diff --git a/NEWS b/NEWS index 68688262a2..21a8cf1e8d 100644 --- a/NEWS +++ b/NEWS @@ -39,8 +39,6 @@ ver 0.24 (not yet released) * switch to C++20 - GCC 10 or clang 11 (or newer) recommended * static partition configuration -* Android - - require Android 7 or newer * Windows - build with libsamplerate - remove JACK DLL support @@ -56,6 +54,8 @@ ver 0.23.14 (not yet released) - wasapi: fix problem setting volume * more libfmt 10 fixes * fix auto-detected systemd unit directory +* Android + - require Android 7 or newer ver 0.23.13 (2023/05/22) * input diff --git a/android/build.py b/android/build.py index ce85453fc6..a26aba1990 100755 --- a/android/build.py +++ b/android/build.py @@ -20,131 +20,13 @@ print("NDK not found in", ndk_path, file=sys.stderr) sys.exit(1) -android_abis = { - 'armeabi-v7a': { - 'arch': 'arm-linux-androideabi', - 'ndk_arch': 'arm', - 'llvm_triple': 'armv7-linux-androideabi', - 'cflags': '-fpic -mfpu=neon -mfloat-abi=softfp', - }, - - 'arm64-v8a': { - 'arch': 'aarch64-linux-android', - 'ndk_arch': 'arm64', - 'llvm_triple': 'aarch64-linux-android', - 'cflags': '-fpic', - }, - - 'x86': { - 'arch': 'i686-linux-android', - 'ndk_arch': 'x86', - 'llvm_triple': 'i686-linux-android', - 'cflags': '-fPIC -march=i686 -mtune=intel -mssse3 -mfpmath=sse -m32', - }, - - 'x86_64': { - 'arch': 'x86_64-linux-android', - 'ndk_arch': 'x86_64', - 'llvm_triple': 'x86_64-linux-android', - 'cflags': '-fPIC -m64', - }, -} - -# select the NDK target -abi_info = android_abis[android_abi] -arch = abi_info['arch'] - # the path to the MPD sources mpd_path = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]) or '.', '..')) sys.path[0] = os.path.join(mpd_path, 'python') # output directories from build.dirs import lib_path, tarball_path, src_path -from build.meson import configure as run_meson - -arch_path = os.path.join(lib_path, arch) -build_path = os.path.join(arch_path, 'build') - -# build host configuration -build_arch = 'linux-x86_64' - -# set up the NDK toolchain - -class AndroidNdkToolchain: - def __init__(self, tarball_path, src_path, build_path, - use_cxx): - self.tarball_path = tarball_path - self.src_path = src_path - self.build_path = build_path - - ndk_arch = abi_info['ndk_arch'] - android_api_level = '24' - - install_prefix = os.path.join(arch_path, 'root') - - self.arch = arch - self.actual_arch = arch - self.install_prefix = install_prefix - - llvm_path = os.path.join(ndk_path, 'toolchains', 'llvm', 'prebuilt', build_arch) - llvm_triple = abi_info['llvm_triple'] + android_api_level - - common_flags = '-Os -g' - common_flags += ' ' + abi_info['cflags'] - - llvm_bin = os.path.join(llvm_path, 'bin') - self.cc = os.path.join(llvm_bin, 'clang') - self.cxx = os.path.join(llvm_bin, 'clang++') - common_flags += ' -target ' + llvm_triple - - common_flags += ' -fvisibility=hidden -fdata-sections -ffunction-sections' - - self.ar = os.path.join(llvm_bin, 'llvm-ar') - self.arflags = 'rcs' - self.ranlib = os.path.join(llvm_bin, 'llvm-ranlib') - self.nm = os.path.join(llvm_bin, 'llvm-nm') - self.strip = os.path.join(llvm_bin, 'llvm-strip') - - self.cflags = common_flags - self.cxxflags = common_flags - self.cppflags = ' -isystem ' + os.path.join(install_prefix, 'include') - self.ldflags = ' -L' + os.path.join(install_prefix, 'lib') + \ - ' -Wl,--exclude-libs=ALL' + \ - ' ' + common_flags - self.ldflags = common_flags - self.libs = '' - - self.is_arm = ndk_arch == 'arm' - self.is_armv7 = self.is_arm and 'armv7' in self.cflags - self.is_aarch64 = ndk_arch == 'arm64' - self.is_windows = False - - libstdcxx_flags = '' - libstdcxx_cxxflags = '' - libstdcxx_ldflags = '' - libstdcxx_libs = '-static-libstdc++' - - if self.is_armv7: - # On 32 bit ARM, clang generates no ".eh_frame" section; - # instead, the LLVM unwinder library is used for unwinding - # the stack after a C++ exception was thrown - libstdcxx_libs += ' -lunwind' - - if use_cxx: - self.cxxflags += ' ' + libstdcxx_cxxflags - self.ldflags += ' ' + libstdcxx_ldflags - self.libs += ' ' + libstdcxx_libs - - self.env = dict(os.environ) - - # redirect pkg-config to use our root directory instead of the - # default one on the build host - import shutil - bin_dir = os.path.join(install_prefix, 'bin') - os.makedirs(bin_dir, exist_ok=True) - self.pkg_config = shutil.copy(os.path.join(mpd_path, 'build', 'pkg-config.sh'), - os.path.join(bin_dir, 'pkg-config')) - self.env['PKG_CONFIG'] = self.pkg_config +from build.toolchain import AndroidNdkToolchain # a list of third-party libraries to be used by MPD on Android from build.libs import * @@ -165,13 +47,17 @@ def __init__(self, tarball_path, src_path, build_path, # build the third-party libraries for x in thirdparty_libs: - toolchain = AndroidNdkToolchain(tarball_path, src_path, build_path, + toolchain = AndroidNdkToolchain(mpd_path, lib_path, + tarball_path, src_path, + ndk_path, android_abi, use_cxx=x.use_cxx) if not x.is_installed(toolchain): x.build(toolchain) # configure and build MPD -toolchain = AndroidNdkToolchain(tarball_path, src_path, build_path, +toolchain = AndroidNdkToolchain(mpd_path, lib_path, + tarball_path, src_path, + ndk_path, android_abi, use_cxx=True) configure_args += [ diff --git a/python/build/autotools.py b/python/build/autotools.py index 46fbd273ba..23c382c785 100644 --- a/python/build/autotools.py +++ b/python/build/autotools.py @@ -1,26 +1,32 @@ import os.path, subprocess, sys +from typing import Collection, Iterable, Optional, Sequence, Union +from collections.abc import Mapping from build.makeproject import MakeProject +from .toolchain import AnyToolchain class AutotoolsProject(MakeProject): - def __init__(self, url, md5, installed, configure_args=[], - autogen=False, - autoreconf=False, - cppflags='', - ldflags='', - libs='', - subdirs=None, + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, + configure_args: Iterable[str]=[], + autogen: bool=False, + autoreconf: bool=False, + per_arch_cflags: Optional[Mapping[str, str]]=None, + cppflags: str='', + ldflags: str='', + libs: str='', + subdirs: Optional[Collection[str]]=None, **kwargs): MakeProject.__init__(self, url, md5, installed, **kwargs) self.configure_args = configure_args self.autogen = autogen self.autoreconf = autoreconf + self.per_arch_cflags = per_arch_cflags self.cppflags = cppflags self.ldflags = ldflags self.libs = libs self.subdirs = subdirs - def configure(self, toolchain): + def configure(self, toolchain: AnyToolchain) -> str: src = self.unpack(toolchain) if self.autogen: if sys.platform == 'darwin': @@ -35,12 +41,16 @@ def configure(self, toolchain): build = self.make_build_path(toolchain) + arch_cflags = '' + if self.per_arch_cflags is not None and toolchain.host_triplet is not None: + arch_cflags = self.per_arch_cflags.get(toolchain.host_triplet, '') + configure = [ os.path.join(src, 'configure'), 'CC=' + toolchain.cc, 'CXX=' + toolchain.cxx, - 'CFLAGS=' + toolchain.cflags, - 'CXXFLAGS=' + toolchain.cxxflags, + 'CFLAGS=' + toolchain.cflags + ' ' + arch_cflags, + 'CXXFLAGS=' + toolchain.cxxflags + ' ' + arch_cflags, 'CPPFLAGS=' + toolchain.cppflags + ' ' + self.cppflags, 'LDFLAGS=' + toolchain.ldflags + ' ' + self.ldflags, 'LIBS=' + toolchain.libs + ' ' + self.libs, @@ -48,10 +58,14 @@ def configure(self, toolchain): 'ARFLAGS=' + toolchain.arflags, 'RANLIB=' + toolchain.ranlib, 'STRIP=' + toolchain.strip, - '--host=' + toolchain.arch, '--prefix=' + toolchain.install_prefix, '--disable-silent-rules', - ] + self.configure_args + ] + + if toolchain.host_triplet is not None: + configure.append('--host=' + toolchain.host_triplet) + + configure.extend(self.configure_args) try: print(configure) @@ -68,7 +82,7 @@ def configure(self, toolchain): return build - def _build(self, toolchain): + def _build(self, toolchain: AnyToolchain) -> None: build = self.configure(toolchain) if self.subdirs is not None: for subdir in self.subdirs: diff --git a/python/build/cmake.py b/python/build/cmake.py index b41d146983..74d5f286a2 100644 --- a/python/build/cmake.py +++ b/python/build/cmake.py @@ -1,18 +1,21 @@ import os import re import subprocess +from typing import cast, Optional, Sequence, TextIO, Union +from collections.abc import Mapping from build.project import Project +from .toolchain import AnyToolchain -def __write_cmake_compiler(f, language, compiler): +def __write_cmake_compiler(f: TextIO, language: str, compiler: str) -> None: s = compiler.split(' ', 1) if len(s) == 2: print(f'set(CMAKE_{language}_COMPILER_LAUNCHER {s[0]})', file=f) compiler = s[1] print(f'set(CMAKE_{language}_COMPILER {compiler})', file=f) -def __write_cmake_toolchain_file(f, toolchain): - if '-darwin' in toolchain.actual_arch: +def __write_cmake_toolchain_file(f: TextIO, toolchain: AnyToolchain) -> None: + if toolchain.is_darwin: cmake_system_name = 'Darwin' elif toolchain.is_windows: cmake_system_name = 'Windows' @@ -21,10 +24,10 @@ def __write_cmake_toolchain_file(f, toolchain): f.write(f""" set(CMAKE_SYSTEM_NAME {cmake_system_name}) -set(CMAKE_SYSTEM_PROCESSOR {toolchain.actual_arch.split('-', 1)[0]}) +set(CMAKE_SYSTEM_PROCESSOR {toolchain.host_triplet.split('-', 1)[0]}) -set(CMAKE_C_COMPILER_TARGET {toolchain.actual_arch}) -set(CMAKE_CXX_COMPILER_TARGET {toolchain.actual_arch}) +set(CMAKE_C_COMPILER_TARGET {toolchain.host_triplet}) +set(CMAKE_CXX_COMPILER_TARGET {toolchain.host_triplet}) set(CMAKE_C_FLAGS_INIT "{toolchain.cflags} {toolchain.cppflags}") set(CMAKE_CXX_FLAGS_INIT "{toolchain.cxxflags} {toolchain.cppflags}") @@ -52,36 +55,39 @@ def __write_cmake_toolchain_file(f, toolchain): set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) """) -def configure(toolchain, src, build, args=(), env=None): - cross_args = [] +def configure(toolchain: AnyToolchain, src: str, build: str, args: list[str]=[], env: Optional[Mapping[str, str]]=None) -> None: + cross_args: list[str] = [] if toolchain.is_windows: - cross_args.append('-DCMAKE_RC_COMPILER=' + toolchain.windres) - - # Several targets need a sysroot to prevent pkg-config from - # looking for libraries on the build host (TODO: fix this - # properly); but we must not do that on Android because the NDK - # has a sysroot already - if '-android' not in toolchain.actual_arch and '-darwin' not in toolchain.actual_arch: - cross_args.append('-DCMAKE_SYSROOT=' + toolchain.install_prefix) - - os.makedirs(build, exist_ok=True) - cmake_toolchain_file = os.path.join(build, 'cmake_toolchain_file') - with open(cmake_toolchain_file, 'w') as f: - __write_cmake_toolchain_file(f, toolchain) + cross_args.append('-DCMAKE_RC_COMPILER=' + cast(str, toolchain.windres)) configure = [ 'cmake', src, - '-DCMAKE_TOOLCHAIN_FILE=' + cmake_toolchain_file, - '-DCMAKE_INSTALL_PREFIX=' + toolchain.install_prefix, '-DCMAKE_BUILD_TYPE=release', '-GNinja', ] + cross_args + args + if toolchain.host_triplet is not None: + # cross-compiling: write a toolchain file + os.makedirs(build, exist_ok=True) + + # Several targets need a sysroot to prevent pkg-config from + # looking for libraries on the build host (TODO: fix this + # properly); but we must not do that on Android because the NDK + # has a sysroot already + if not toolchain.is_android and not toolchain.is_darwin: + cross_args.append('-DCMAKE_SYSROOT=' + toolchain.install_prefix) + + cmake_toolchain_file = os.path.join(build, 'cmake_toolchain_file') + with open(cmake_toolchain_file, 'w') as f: + __write_cmake_toolchain_file(f, toolchain) + + configure.append('-DCMAKE_TOOLCHAIN_FILE=' + cmake_toolchain_file) + if env is None: env = toolchain.env else: @@ -91,16 +97,17 @@ def configure(toolchain, src, build, args=(), env=None): subprocess.check_call(configure, env=env, cwd=build) class CmakeProject(Project): - def __init__(self, url, md5, installed, configure_args=[], - windows_configure_args=[], - env=None, + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, + configure_args: list[str]=[], + windows_configure_args: list[str]=[], + env: Optional[Mapping[str, str]]=None, **kwargs): Project.__init__(self, url, md5, installed, **kwargs) self.configure_args = configure_args self.windows_configure_args = windows_configure_args self.env = env - def configure(self, toolchain): + def configure(self, toolchain: AnyToolchain) -> str: src = self.unpack(toolchain) build = self.make_build_path(toolchain) configure_args = self.configure_args @@ -109,7 +116,7 @@ def configure(self, toolchain): configure(toolchain, src, build, configure_args, self.env) return build - def _build(self, toolchain): + def _build(self, toolchain: AnyToolchain) -> None: build = self.configure(toolchain) subprocess.check_call(['ninja', '-v', 'install'], cwd=build, env=toolchain.env) diff --git a/python/build/download.py b/python/build/download.py index 9cc710c8f3..c170c841ab 100644 --- a/python/build/download.py +++ b/python/build/download.py @@ -1,12 +1,50 @@ -from build.verify import verify_file_digest +from typing import Sequence, Union import os +import sys import urllib.request -def download_and_verify(url, md5, parent_path): +from .verify import verify_file_digest + +def __to_string_sequence(x: Union[str, Sequence[str]]) -> Sequence[str]: + if isinstance(x, str): + return (x,) + else: + return x + +def __get_any(x: Union[str, Sequence[str]]) -> str: + if isinstance(x, str): + return x + else: + return x[0] + +def __download_one(url: str, path: str) -> None: + print("download", url) + urllib.request.urlretrieve(url, path) + +def __download(urls: Sequence[str], path: str) -> None: + for url in urls[:-1]: + try: + __download_one(url, path) + return + except: + print("download error:", sys.exc_info()[0]) + __download_one(urls[-1], path) + +def __download_and_verify_to(urls: Sequence[str], md5: str, path: str) -> None: + __download(urls, path) + if not verify_file_digest(path, md5): + raise RuntimeError("Digest mismatch") + +def download_basename(urls: Union[str, Sequence[str]]) -> str: + return os.path.basename(__get_any(urls)) + +def download_and_verify(urls: Union[str, Sequence[str]], md5: str, parent_path: str) -> str: """Download a file, verify its MD5 checksum and return the local path.""" + base = download_basename(urls) + os.makedirs(parent_path, exist_ok=True) - path = os.path.join(parent_path, os.path.basename(url)) + path = os.path.join(parent_path, base) try: if verify_file_digest(path, md5): return path @@ -16,11 +54,6 @@ def download_and_verify(url, md5, parent_path): tmp_path = path + '.tmp' - print("download", url) - urllib.request.urlretrieve(url, tmp_path) - if not verify_file_digest(tmp_path, md5): - os.unlink(tmp_path) - raise RuntimeError("Digest mismatch") - + __download_and_verify_to(__to_string_sequence(urls), md5, tmp_path) os.rename(tmp_path, path) return path diff --git a/python/build/libs.py b/python/build/libs.py index ee7b6e2dc6..588bc28a4f 100644 --- a/python/build/libs.py +++ b/python/build/libs.py @@ -66,7 +66,8 @@ ) zlib = ZlibProject( - 'http://zlib.net/zlib-1.3.tar.xz', + ('http://zlib.net/zlib-1.3.tar.xz', + 'https://github.com/madler/zlib/releases/download/v1.3/zlib-1.3.tar.xz'), '8a9ba2898e1d0d774eca6ba5b4627a11e5588ba85c8851336eb38de4683050a7', 'lib/libz.a', ) @@ -608,13 +609,15 @@ ) openssl = OpenSSLProject( - 'https://www.openssl.org/source/openssl-3.1.2.tar.gz', - 'a0ce69b8b97ea6a35b96875235aa453b966ba3cba8af2de23657d8b6767d6539', + ('https://www.openssl.org/source/openssl-3.1.3.tar.gz', + 'https://artfiles.org/openssl.org/source/openssl-3.1.3.tar.gz'), + 'f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6', 'include/openssl/ossl_typ.h', ) curl = CmakeProject( - 'https://curl.se/download/curl-8.2.1.tar.xz', + ('https://curl.se/download/curl-8.2.1.tar.xz', + 'https://github.com/curl/curl/releases/download/curl-8_2_1/curl-8.2.1.tar.xz'), 'dd322f6bd0a20e6cebdfd388f69e98c3d183bed792cf4713c8a7ef498cba4894', 'lib/libcurl.a', [ diff --git a/python/build/makeproject.py b/python/build/makeproject.py index 9d498cfbdd..b0acf78076 100644 --- a/python/build/makeproject.py +++ b/python/build/makeproject.py @@ -1,15 +1,17 @@ import subprocess, multiprocessing +from typing import Optional, Sequence, Union from build.project import Project +from .toolchain import AnyToolchain class MakeProject(Project): - def __init__(self, url, md5, installed, - install_target='install', + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, + install_target: str='install', **kwargs): Project.__init__(self, url, md5, installed, **kwargs) self.install_target = install_target - def get_simultaneous_jobs(self): + def get_simultaneous_jobs(self) -> int: try: # use twice as many simultaneous jobs as we have CPU cores return multiprocessing.cpu_count() * 2 @@ -17,17 +19,17 @@ def get_simultaneous_jobs(self): # default to 12, if multiprocessing.cpu_count() is not implemented return 12 - def get_make_args(self, toolchain): + def get_make_args(self, toolchain: AnyToolchain) -> list[str]: return ['--quiet', '-j' + str(self.get_simultaneous_jobs())] - def get_make_install_args(self, toolchain): + def get_make_install_args(self, toolchain: AnyToolchain) -> list[str]: return ['--quiet', self.install_target] - def make(self, toolchain, wd, args): + def make(self, toolchain: AnyToolchain, wd: str, args: list[str]) -> None: subprocess.check_call(['make'] + args, cwd=wd, env=toolchain.env) - def build_make(self, toolchain, wd, install=True): + def build_make(self, toolchain: AnyToolchain, wd: str, install: bool=True) -> None: self.make(toolchain, wd, self.get_make_args(toolchain)) if install: self.make(toolchain, wd, self.get_make_install_args(toolchain)) diff --git a/python/build/meson.py b/python/build/meson.py index 0e19b6f9f6..70ea21a2de 100644 --- a/python/build/meson.py +++ b/python/build/meson.py @@ -1,10 +1,17 @@ import os import subprocess import platform +from typing import Optional, Sequence, Union from build.project import Project +from .toolchain import AnyToolchain -def make_cross_file(toolchain): +def __no_ccache(cmd: str) -> str: + if cmd.startswith('ccache '): + cmd = cmd[7:] + return cmd + +def make_cross_file(toolchain: AnyToolchain) -> str: if toolchain.is_windows: system = 'windows' windres = "windres = '%s'" % toolchain.windres @@ -23,7 +30,7 @@ def make_cross_file(toolchain): cpu = 'arm64-v8a' else: cpu_family = 'x86' - if 'x86_64' in toolchain.arch: + if 'x86_64' in toolchain.host_triplet: cpu = 'x86_64' else: cpu = 'i686' @@ -38,8 +45,8 @@ def make_cross_file(toolchain): with open(path, 'w') as f: f.write(f""" [binaries] -c = '{toolchain.cc}' -cpp = '{toolchain.cxx}' +c = '{__no_ccache(toolchain.cc)}' +cpp = '{__no_ccache(toolchain.cxx)}' ar = '{toolchain.ar}' strip = '{toolchain.strip}' pkgconfig = '{toolchain.pkg_config}' @@ -56,7 +63,7 @@ def make_cross_file(toolchain): root = '{toolchain.install_prefix}' """) - if 'android' in toolchain.arch: + if toolchain.is_android: f.write(""" # Keep Meson from executing Android-x86 test binariees needs_exe_wrapper = true @@ -80,8 +87,7 @@ def make_cross_file(toolchain): """) return path -def configure(toolchain, src, build, args=()): - cross_file = make_cross_file(toolchain) +def configure(toolchain: AnyToolchain, src: str, build: str, args: list[str]=[]) -> None: configure = [ 'meson', 'setup', build, src, @@ -91,27 +97,31 @@ def configure(toolchain, src, build, args=()): '--buildtype', 'plain', '--default-library=static', - - '--cross-file', cross_file, ] + args + if toolchain.host_triplet is not None: + # cross-compiling: write a cross-file + cross_file = make_cross_file(toolchain) + configure.append(f'--cross-file={cross_file}') + env = toolchain.env.copy() subprocess.check_call(configure, env=env) class MesonProject(Project): - def __init__(self, url, md5, installed, configure_args=[], + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, + configure_args: list[str]=[], **kwargs): Project.__init__(self, url, md5, installed, **kwargs) self.configure_args = configure_args - def configure(self, toolchain): + def configure(self, toolchain: AnyToolchain) -> str: src = self.unpack(toolchain) build = self.make_build_path(toolchain) configure(toolchain, src, build, self.configure_args) return build - def _build(self, toolchain): + def _build(self, toolchain: AnyToolchain) -> None: build = self.configure(toolchain) subprocess.check_call(['ninja', '-v', 'install'], cwd=build, env=toolchain.env) diff --git a/python/build/openssl.py b/python/build/openssl.py index dd02941b50..8b0224598e 100644 --- a/python/build/openssl.py +++ b/python/build/openssl.py @@ -1,13 +1,15 @@ import subprocess +from typing import Optional, Sequence, Union from build.makeproject import MakeProject +from .toolchain import AnyToolchain class OpenSSLProject(MakeProject): - def __init__(self, url, md5, installed, + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, **kwargs): MakeProject.__init__(self, url, md5, installed, install_target='install_dev', **kwargs) - def get_make_args(self, toolchain): + def get_make_args(self, toolchain: AnyToolchain) -> list[str]: return MakeProject.get_make_args(self, toolchain) + [ 'CC=' + toolchain.cc, 'CFLAGS=' + toolchain.cflags, @@ -17,45 +19,60 @@ def get_make_args(self, toolchain): 'build_libs', ] - def get_make_install_args(self, toolchain): + def get_make_install_args(self, toolchain: AnyToolchain) -> list[str]: # OpenSSL's Makefile runs "ranlib" during installation return MakeProject.get_make_install_args(self, toolchain) + [ 'RANLIB=' + toolchain.ranlib, ] - def _build(self, toolchain): + def _build(self, toolchain: AnyToolchain) -> None: src = self.unpack(toolchain, out_of_tree=False) # OpenSSL has a weird target architecture scheme with lots of # hard-coded architectures; this table translates between our - # "toolchain_arch" (HOST_TRIPLET) and the OpenSSL target + # host triplet and the OpenSSL target openssl_archs = { # not using "android-*" because those OpenSSL targets want # to know where the SDK is, but our own build scripts # prepared everything already to look like a regular Linux # build - 'arm-linux-androideabi': 'linux-generic32', + 'armv7a-linux-androideabi': 'linux-generic32', 'aarch64-linux-android': 'linux-aarch64', 'i686-linux-android': 'linux-x86-clang', 'x86_64-linux-android': 'linux-x86_64-clang', - # Kobo + # generic Linux 'arm-linux-gnueabihf': 'linux-generic32', # Windows 'i686-w64-mingw32': 'mingw', 'x86_64-w64-mingw32': 'mingw64', + + # Apple + 'x86_64-apple-darwin': 'darwin64-x86_64-cc', + 'aarch64-apple-darwin': 'darwin64-arm64-cc', } - openssl_arch = openssl_archs[toolchain.arch] - - subprocess.check_call(['./Configure', - 'no-shared', - 'no-module', 'no-engine', 'no-static-engine', - 'no-async', - 'no-tests', - 'no-asm', # "asm" causes build failures on Windows - openssl_arch, - '--prefix=' + toolchain.install_prefix], - cwd=src, env=toolchain.env) + configure = [ + './Configure', + 'no-shared', + 'no-module', + 'no-engine', + 'no-static-engine', + 'no-async', + 'no-tests', + 'no-makedepend', + '--libdir=lib', # no "lib64" on amd64, please + '--prefix=' + toolchain.install_prefix, + ] + + if toolchain.is_windows: + # workaround for build failures + configure.append('no-asm') + + if toolchain.host_triplet is not None: + configure.append(openssl_archs[toolchain.host_triplet]) + configure.append(f'--cross-compile-prefix={toolchain.host_triplet}-') + + subprocess.check_call(configure, cwd=src, env=toolchain.env) self.build_make(toolchain, src) diff --git a/python/build/project.py b/python/build/project.py index bf787cf434..f636b51186 100644 --- a/python/build/project.py +++ b/python/build/project.py @@ -1,18 +1,21 @@ import os, shutil import re +from typing import cast, BinaryIO, Optional, Sequence, Union -from build.download import download_and_verify +from build.download import download_basename, download_and_verify from build.tar import untar from build.quilt import push_all +from .toolchain import AnyToolchain class Project: - def __init__(self, url, md5, installed, name=None, version=None, - base=None, - patches=None, + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, + name: Optional[str]=None, version: Optional[str]=None, + base: Optional[str]=None, + patches: Optional[str]=None, edits=None, - use_cxx=False): + use_cxx: bool=False): if base is None: - basename = os.path.basename(url) + basename = download_basename(url) m = re.match(r'^(.+)\.(tar(\.(gz|bz2|xz|lzma))?|zip)$', basename) if not m: raise RuntimeError('Could not identify tarball name: ' + basename) self.base = m.group(1) @@ -39,10 +42,10 @@ def __init__(self, url, md5, installed, name=None, version=None, self.edits = edits self.use_cxx = use_cxx - def download(self, toolchain): + def download(self, toolchain: AnyToolchain) -> str: return download_and_verify(self.url, self.md5, toolchain.tarball_path) - def is_installed(self, toolchain): + def is_installed(self, toolchain: AnyToolchain) -> bool: tarball = self.download(toolchain) installed = os.path.join(toolchain.install_prefix, self.installed) tarball_mtime = os.path.getmtime(tarball) @@ -51,7 +54,7 @@ def is_installed(self, toolchain): except FileNotFoundError: return False - def unpack(self, toolchain, out_of_tree=True): + def unpack(self, toolchain: AnyToolchain, out_of_tree: bool=True) -> str: if out_of_tree: parent_path = toolchain.src_path else: @@ -72,7 +75,7 @@ def unpack(self, toolchain, out_of_tree=True): return path - def make_build_path(self, toolchain, lazy=False): + def make_build_path(self, toolchain: AnyToolchain, lazy: bool=False) -> str: path = os.path.join(toolchain.build_path, self.base) if lazy and os.path.isdir(path): return path @@ -83,5 +86,5 @@ def make_build_path(self, toolchain, lazy=False): os.makedirs(path, exist_ok=True) return path - def build(self, toolchain): + def build(self, toolchain: AnyToolchain) -> None: self._build(toolchain) diff --git a/python/build/quilt.py b/python/build/quilt.py index 876453d2bb..03adf0e427 100644 --- a/python/build/quilt.py +++ b/python/build/quilt.py @@ -1,9 +1,12 @@ import subprocess +from typing import Union -def run_quilt(toolchain, cwd, patches_path, *args): +from .toolchain import AnyToolchain + +def run_quilt(toolchain: AnyToolchain, cwd: str, patches_path: str, *args: str) -> None: env = dict(toolchain.env) env['QUILT_PATCHES'] = patches_path subprocess.check_call(['quilt'] + list(args), cwd=cwd, env=env) -def push_all(toolchain, src_path, patches_path): +def push_all(toolchain: AnyToolchain, src_path: str, patches_path: str) -> None: run_quilt(toolchain, src_path, patches_path, 'push', '-a') diff --git a/python/build/tar.py b/python/build/tar.py index 8247d2c22f..208d0a6061 100644 --- a/python/build/tar.py +++ b/python/build/tar.py @@ -1,6 +1,7 @@ import os, shutil, subprocess -def untar(tarball_path, parent_path, base, lazy=False): +def untar(tarball_path: str, parent_path: str, base: str, + lazy: bool=False) -> str: path = os.path.join(parent_path, base) if lazy and os.path.isdir(path): return path diff --git a/python/build/toolchain.py b/python/build/toolchain.py new file mode 100644 index 0000000000..70ea5289b3 --- /dev/null +++ b/python/build/toolchain.py @@ -0,0 +1,175 @@ +import os.path +import shutil +from typing import Union + +android_abis = { + 'armeabi-v7a': { + 'arch': 'armv7a-linux-androideabi', + 'ndk_arch': 'arm', + 'cflags': '-fpic -mfpu=neon -mfloat-abi=softfp', + }, + + 'arm64-v8a': { + 'arch': 'aarch64-linux-android', + 'ndk_arch': 'arm64', + 'cflags': '-fpic', + }, + + 'x86': { + 'arch': 'i686-linux-android', + 'ndk_arch': 'x86', + 'cflags': '-fPIC -march=i686 -mtune=intel -mssse3 -mfpmath=sse -m32', + }, + + 'x86_64': { + 'arch': 'x86_64-linux-android', + 'ndk_arch': 'x86_64', + 'cflags': '-fPIC -m64', + }, +} + +class AndroidNdkToolchain: + def __init__(self, top_path: str, lib_path: str, + tarball_path: str, src_path: str, + ndk_path: str, android_abi: str, + use_cxx): + # build host configuration + build_arch = 'linux-x86_64' + + # select the NDK target + abi_info = android_abis[android_abi] + host_triplet = abi_info['arch'] + + arch_path = os.path.join(lib_path, host_triplet) + + self.tarball_path = tarball_path + self.src_path = src_path + self.build_path = os.path.join(arch_path, 'build') + + ndk_arch = abi_info['ndk_arch'] + android_api_level = '24' + + install_prefix = os.path.join(arch_path, 'root') + + self.host_triplet = host_triplet + self.install_prefix = install_prefix + + llvm_path = os.path.join(ndk_path, 'toolchains', 'llvm', 'prebuilt', build_arch) + llvm_triple = host_triplet + android_api_level + + common_flags = '-Os -g' + common_flags += ' ' + abi_info['cflags'] + + llvm_bin = os.path.join(llvm_path, 'bin') + self.cc = os.path.join(llvm_bin, 'clang') + self.cxx = os.path.join(llvm_bin, 'clang++') + common_flags += ' -target ' + llvm_triple + + common_flags += ' -fvisibility=hidden -fdata-sections -ffunction-sections' + + self.ar = os.path.join(llvm_bin, 'llvm-ar') + self.arflags = 'rcs' + self.ranlib = os.path.join(llvm_bin, 'llvm-ranlib') + self.nm = os.path.join(llvm_bin, 'llvm-nm') + self.strip = os.path.join(llvm_bin, 'llvm-strip') + + self.cflags = common_flags + self.cxxflags = common_flags + self.cppflags = ' -isystem ' + os.path.join(install_prefix, 'include') + self.ldflags = ' -L' + os.path.join(install_prefix, 'lib') + \ + ' -Wl,--exclude-libs=ALL' + \ + ' ' + common_flags + self.ldflags = common_flags + self.libs = '' + + self.is_arm = ndk_arch == 'arm' + self.is_armv7 = self.is_arm and 'armv7' in self.cflags + self.is_aarch64 = ndk_arch == 'arm64' + self.is_windows = False + self.is_android = True + self.is_darwin = False + + libstdcxx_flags = '' + libstdcxx_cxxflags = '' + libstdcxx_ldflags = '' + libstdcxx_libs = '-static-libstdc++' + + if self.is_armv7: + # On 32 bit ARM, clang generates no ".eh_frame" section; + # instead, the LLVM unwinder library is used for unwinding + # the stack after a C++ exception was thrown + libstdcxx_libs += ' -lunwind' + + if use_cxx: + self.cxxflags += ' ' + libstdcxx_cxxflags + self.ldflags += ' ' + libstdcxx_ldflags + self.libs += ' ' + libstdcxx_libs + + self.env = dict(os.environ) + + # redirect pkg-config to use our root directory instead of the + # default one on the build host + bin_dir = os.path.join(install_prefix, 'bin') + os.makedirs(bin_dir, exist_ok=True) + self.pkg_config = shutil.copy(os.path.join(top_path, 'build', 'pkg-config.sh'), + os.path.join(bin_dir, 'pkg-config')) + self.env['PKG_CONFIG'] = self.pkg_config + +class MingwToolchain: + def __init__(self, top_path: str, + toolchain_path, host_triplet, x64: bool, + tarball_path, src_path, build_path, install_prefix): + self.host_triplet = host_triplet + self.tarball_path = tarball_path + self.src_path = src_path + self.build_path = build_path + self.install_prefix = install_prefix + + toolchain_bin = os.path.join(toolchain_path, 'bin') + self.cc = os.path.join(toolchain_bin, host_triplet + '-gcc') + self.cxx = os.path.join(toolchain_bin, host_triplet + '-g++') + self.ar = os.path.join(toolchain_bin, host_triplet + '-ar') + self.arflags = 'rcs' + self.ranlib = os.path.join(toolchain_bin, host_triplet + '-ranlib') + self.nm = os.path.join(toolchain_bin, host_triplet + '-nm') + self.strip = os.path.join(toolchain_bin, host_triplet + '-strip') + self.windres = os.path.join(toolchain_bin, host_triplet + '-windres') + + common_flags = '-O2 -g' + + if not x64: + # enable SSE support which is required for LAME + common_flags += ' -march=pentium3' + + self.cflags = common_flags + self.cxxflags = common_flags + self.cppflags = '-isystem ' + os.path.join(install_prefix, 'include') + \ + ' -DWINVER=0x0600 -D_WIN32_WINNT=0x0600' + self.ldflags = '-L' + os.path.join(install_prefix, 'lib') + \ + ' -static-libstdc++ -static-libgcc' + self.libs = '' + + # Explicitly disable _FORTIFY_SOURCE because it is broken with + # mingw. This prevents some libraries such as libFLAC to + # enable it. + self.cppflags += ' -D_FORTIFY_SOURCE=0' + + self.is_arm = host_triplet.startswith('arm') + self.is_armv7 = self.is_arm and 'armv7' in self.cflags + self.is_aarch64 = host_triplet == 'aarch64' + self.is_windows = 'mingw32' in host_triplet + self.is_android = False + self.is_darwin = False + + self.env = dict(os.environ) + + # redirect pkg-config to use our root directory instead of the + # default one on the build host + import shutil + bin_dir = os.path.join(install_prefix, 'bin') + os.makedirs(bin_dir, exist_ok=True) + self.pkg_config = shutil.copy(os.path.join(top_path, 'build', 'pkg-config.sh'), + os.path.join(bin_dir, 'pkg-config')) + self.env['PKG_CONFIG'] = self.pkg_config + +AnyToolchain = Union[AndroidNdkToolchain, MingwToolchain] diff --git a/python/build/verify.py b/python/build/verify.py index e22ff274d1..763f224ce7 100644 --- a/python/build/verify.py +++ b/python/build/verify.py @@ -1,6 +1,7 @@ import hashlib +from typing import cast, Any, BinaryIO -def feed_file(h, f): +def feed_file(h: Any, f: BinaryIO) -> None: """Feed data read from an open file into the hashlib instance.""" while True: @@ -10,20 +11,20 @@ def feed_file(h, f): break h.update(data) -def feed_file_path(h, path): +def feed_file_path(h: Any, path: str) -> None: """Feed data read from a file (to be opened by this function) into the hashlib instance.""" with open(path, 'rb') as f: feed_file(h, f) -def file_digest(algorithm, path): +def file_digest(algorithm: Any, path: str) -> str: """Calculate the digest of a file and return it in hexadecimal notation.""" h = algorithm() feed_file_path(h, path) - return h.hexdigest() + return cast(str, h.hexdigest()) -def guess_digest_algorithm(digest): +def guess_digest_algorithm(digest: str) -> Any: l = len(digest) if l == 32: return hashlib.md5 @@ -36,7 +37,7 @@ def guess_digest_algorithm(digest): else: return None -def verify_file_digest(path, expected_digest): +def verify_file_digest(path: str, expected_digest: str) -> bool: """Verify the digest of a file, and return True if the digest matches with the given expected digest.""" algorithm = guess_digest_algorithm(expected_digest) diff --git a/python/build/zlib.py b/python/build/zlib.py index c5aeb8c307..41138c8256 100644 --- a/python/build/zlib.py +++ b/python/build/zlib.py @@ -1,13 +1,15 @@ import subprocess +from typing import Optional, Sequence, Union from build.makeproject import MakeProject +from .toolchain import AnyToolchain class ZlibProject(MakeProject): - def __init__(self, url, md5, installed, + def __init__(self, url: Union[str, Sequence[str]], md5: str, installed: str, **kwargs): MakeProject.__init__(self, url, md5, installed, **kwargs) - def get_make_args(self, toolchain): + def get_make_args(self, toolchain: AnyToolchain) -> list[str]: return MakeProject.get_make_args(self, toolchain) + [ 'CC=' + toolchain.cc + ' ' + toolchain.cppflags + ' ' + toolchain.cflags, 'CPP=' + toolchain.cc + ' -E ' + toolchain.cppflags, @@ -18,13 +20,13 @@ def get_make_args(self, toolchain): 'libz.a' ] - def get_make_install_args(self, toolchain): + def get_make_install_args(self, toolchain: AnyToolchain) -> list[str]: return [ 'RANLIB=' + toolchain.ranlib, self.install_target ] - def _build(self, toolchain): + def _build(self, toolchain: AnyToolchain) -> None: src = self.unpack(toolchain, out_of_tree=False) subprocess.check_call(['./configure', '--prefix=' + toolchain.install_prefix, '--static'], diff --git a/win32/build.py b/win32/build.py index b31d33dc8b..cf077bdb0b 100755 --- a/win32/build.py +++ b/win32/build.py @@ -29,66 +29,12 @@ # output directories from build.dirs import lib_path, tarball_path, src_path +from build.toolchain import MingwToolchain arch_path = os.path.join(lib_path, host_arch) build_path = os.path.join(arch_path, 'build') root_path = os.path.join(arch_path, 'root') -class CrossGccToolchain: - def __init__(self, toolchain_path, arch, - tarball_path, src_path, build_path, install_prefix): - self.arch = arch - self.actual_arch = arch - self.tarball_path = tarball_path - self.src_path = src_path - self.build_path = build_path - self.install_prefix = install_prefix - - toolchain_bin = os.path.join(toolchain_path, 'bin') - self.cc = os.path.join(toolchain_bin, arch + '-gcc') - self.cxx = os.path.join(toolchain_bin, arch + '-g++') - self.ar = os.path.join(toolchain_bin, arch + '-ar') - self.arflags = 'rcs' - self.ranlib = os.path.join(toolchain_bin, arch + '-ranlib') - self.nm = os.path.join(toolchain_bin, arch + '-nm') - self.strip = os.path.join(toolchain_bin, arch + '-strip') - self.windres = os.path.join(toolchain_bin, arch + '-windres') - - common_flags = '-O2 -g' - - if not x64: - # enable SSE support which is required for LAME - common_flags += ' -march=pentium3' - - self.cflags = common_flags - self.cxxflags = common_flags - self.cppflags = '-isystem ' + os.path.join(install_prefix, 'include') + \ - ' -DWINVER=0x0600 -D_WIN32_WINNT=0x0600' - self.ldflags = '-L' + os.path.join(install_prefix, 'lib') + \ - ' -static-libstdc++ -static-libgcc' - self.libs = '' - - # Explicitly disable _FORTIFY_SOURCE because it is broken with - # mingw. This prevents some libraries such as libFLAC to - # enable it. - self.cppflags += ' -D_FORTIFY_SOURCE=0' - - self.is_arm = arch.startswith('arm') - self.is_armv7 = self.is_arm and 'armv7' in self.cflags - self.is_aarch64 = arch == 'aarch64' - self.is_windows = 'mingw32' in arch - - self.env = dict(os.environ) - - # redirect pkg-config to use our root directory instead of the - # default one on the build host - import shutil - bin_dir = os.path.join(install_prefix, 'bin') - os.makedirs(bin_dir, exist_ok=True) - self.pkg_config = shutil.copy(os.path.join(mpd_path, 'build', 'pkg-config.sh'), - os.path.join(bin_dir, 'pkg-config')) - self.env['PKG_CONFIG'] = self.pkg_config - # a list of third-party libraries to be used by MPD on Android from build.libs import * thirdparty_libs = [ @@ -110,8 +56,9 @@ def __init__(self, toolchain_path, arch, ] # build the third-party libraries -toolchain = CrossGccToolchain('/usr', host_arch, - tarball_path, src_path, build_path, root_path) +toolchain = MingwToolchain(mpd_path, + '/usr', host_arch, x64, + tarball_path, src_path, build_path, root_path) for x in thirdparty_libs: if not x.is_installed(toolchain):