diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index fab6fc8..0000000 --- a/.appveyor.yml +++ /dev/null @@ -1,263 +0,0 @@ -# ***** BEGIN LICENSE BLOCK ***** -# This file is part of openfx-io , -# (C) 2018-2021 The Natron Developers -# (C) 2013-2018 INRIA -# -# openfx-io is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# openfx-io is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with openfx-io. If not, see -# ***** END LICENSE BLOCK ***** - -#---------------------------------# -# general configuration # -#---------------------------------# - -# version format -version: 1.0.{build}-{branch} - -# branches to build -branches: - # whitelist - # only: - # - master - - # blacklist - except: - - gh-pages - -# Do not build on tags (GitHub only) -skip_tags: true - -skip_commits: - files: - - docs/* - - LICENSE - - README.md - -pull_requests: - do_not_increment_build_number: true - -#---------------------------------# -# environment configuration # -#---------------------------------# - -# Operating system (build VM template) -image: -- Visual Studio 2019 -#- Ubuntu - -# scripts that are called at very beginning, before repo cloning -init: - - cmd: git config --global core.autocrlf input - - set arch= - - if "%PLATFORM%" == "x64" (set arch= Win64) - - if "%PLATFORM%" == "x86" (set arch=) - - if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2019" (set GENERATOR="Visual Studio 16 2019") - - if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017" (set GENERATOR="Visual Studio 15 2017%arch%") - - if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2015" (set GENERATOR="Visual Studio 14 2015%arch%") - - if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2013" (set GENERATOR="Visual Studio 12 2013%arch%") - - if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Ubuntu" (set GENERATOR="Unix Makefiles") - -# fetch repository as zip archive -#shallow_clone: true # default is "false" - -# set clone depth -clone_depth: 5 # clone entire repository history if not defined - -# clone directory -clone_folder: c:\dev\openfx-io - -# environment variables -environment: - APPVEYOR_SAVE_CACHE_ON_ERROR: true - -# build cache to preserve files/folders between builds -cache: - - c:\tools\vcpkg\installed - - c:\Users\appveyor\AppData\Local\vcpkg\ - -# scripts that run after cloning repository -install: - # Dump appveyor build vars for diagnostics - - cmd: 'echo APPVEYOR_FORCED_BUILD: %APPVEYOR_FORCED_BUILD%' - - cmd: 'echo APPVEYOR_RE_BUILD: %APPVEYOR_RE_BUILD%' - - git submodule update --init --recursive - -#---------------------------------# -# build configuration # -#---------------------------------# - -# build platform, i.e. x86, x64, Any CPU. This setting is optional. -platform: x64 - -# build Configuration, i.e. Debug, Release, etc. -configuration: - - RelWithDebInfo - -# scripts to run before build -# In the Visual Studio build, we set _WIN32_WINNT and WINVER to 0x0600 -before_build: - - cmd: if "%platform%"=="Win32" set VCPKG_ARCH=x86-windows - - cmd: if "%platform%"=="x64" set VCPKG_ARCH=x64-windows - # remove outdated versions - - cmd: vcpkg remove --outdated --recurse - # install dependencies - - cmd: vcpkg install --recurse --triplet %VCPKG_ARCH% - opengl - libpng - openexr - x264 - x265 - libvpx - opus - libtheora - ffmpeg[avcodec,avdevice,avfilter,avformat,avresample,core,gpl,postproc,swresample,swscale,opus,theora,x264,x265] - jasper - lcms - libde265 - libheif - libjpeg-turbo - libraw - tiff - openssl - opencolorio - openimageio[libraw] - - cmd: vcpkg integrate install - - mkdir build - - cd build - - cmd: cmake .. -G %GENERATOR% -DCMAKE_INSTALL_PREFIX="c:/dev/openfx-io/install" -DCMAKE_TOOLCHAIN_FILE=C:/Tools/vcpkg/scripts/buildsystems/vcpkg.cmake - - sh: cmake .. -DCMAKE_INSTALL_PREFIX="c:/dev/openfx-io/install" -DCMAKE_BUILD_TYPE=%configuration% - -# scripts to run after build -#after_build: - -build: - project: c:\dev\openfx-io\build\INSTALL.vcxproj - verbosity: minimal - parallel: true - -# to run your custom scripts instead of automatic MSBuild -# We also compile the tests here instead of later on. -#build_script: - -# to disable automatic builds -#build: off - - -#---------------------------------# -# tests configuration # -#---------------------------------# - -# scripts to run before tests -#before_test: - -# scripts to run after tests -#after_test: - -# to run your custom scripts instead of automatic tests -#test_script: - -# to disable automatic tests -test: off - - -#---------------------------------# -# artifacts configuration # -#---------------------------------# - -artifacts: - # pushing a single file - #- path: test.zip - - # pushing a single file with environment variable in path and "Deployment name" specified - #- path: MyProject\bin\$(configuration) - # name: myapp - - # pushing entire folder as a zip archive - #- path: logs - - # pushing all *.nupkg files in directory - #- path: out\*.nupkg - - - path: install - - -#---------------------------------# -# deployment configuration # -#---------------------------------# - -# providers: Local, FTP, WebDeploy, AzureCS, AzureBlob, S3, NuGet, Environment -# provider names are case-sensitive! -deploy: - - provider: Environment - name: openfx-io - release: openfx-io-$(appveyor_repo_branch)-v$(appveyor_build_version) - artifact: openfx-io-$(appveyor_repo_branch).zip - draft: false - prerelease: true - on: - branch: master # release from master branch only - configuration: release # deploy release builds only - appveyor_repo_tag: true # deploy on tag push only - is_not_pr: true # don't deploy pull requests - -# scripts to run before deployment -#before_deploy: - -# scripts to run after deployment -#after_deploy: - -# to run your custom scripts instead of provider deployments -#deploy_script: - -# to disable deployment -# deploy: off - - -#---------------------------------# -# global handlers # -#---------------------------------# - -# on successful build -on_success: - -# on build failure -on_failure: - -# after build failure or success -on_finish: - - -#---------------------------------# -# notifications # -#---------------------------------# - -# notifications: -# # Email -# - provider: Email -# to: -# - user1@email.com -# - user2@email.com -# subject: 'Build {{status}}' # optional -# message: "{{message}}, {{commitId}}, ..." # optional -# on_build_status_changed: true - -# # Webhook -# - provider: Webhook -# url: http://www.myhook2.com -# headers: -# User-Agent: myapp 1.0 -# Authorization: -# secure: GhD+5xhLz/tkYY6AO3fcfQ== -# on_build_success: false -# on_build_failure: true -# on_build_status_changed: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9846118..7e66ef6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ on: - README.md jobs: - build: + ubuntu_build: name: Build Ubuntu 22.04 runs-on: ubuntu-22.04 steps: @@ -30,7 +30,7 @@ jobs: sudo apt update sudo apt-get install cmake libopencolorio-dev libopenimageio-dev libglu1-mesa-dev libgl-dev libegl-dev liblcms2-dev \ libraw-dev libwebp-dev libtiff-dev libopenjp2-7-dev libpng-dev libavcodec-dev libavformat-dev libavutil-dev \ - libswscale-dev + libswscale-dev patchelf - name: Local SeExpr run: | cd SeExprSrc @@ -52,3 +52,107 @@ jobs: with: name: openfx-io-build-ubuntu_22-release path: Bundle + + - name: Generate testing bundle + run: | + # Generate a self contained bundle that can be used for Natron automated testing. + cp -r Bundle TestingBundle + MANIFEST_NAME=IO.manifest + BINARY_NAME=IO.ofx + BINARY_PATH=TestingBundle/IO.ofx.bundle/Contents/Linux-x86-64 + SEARCH_PATHS="SeExprBuild/lib /usr/lib /usr/lib/x86_64-linux-gnu" + python3 .github/workflows/find_and_copy_deps.py --manifest=${MANIFEST_NAME} ${BINARY_PATH}/${BINARY_NAME} ${BINARY_PATH} ${SEARCH_PATHS} + + patchelf --force-rpath --set-rpath '$ORIGIN' ${BINARY_PATH}/${BINARY_NAME} + + # Generate simple plugin tester to make sure we can load the test binary. + # Replace RTLD_LAZY with RTLD_NOW so that missing symbols cause plugin loading to fail. + sed -e 's/RTLD_LAZY/RTLD_NOW/g' openfx/HostSupport/src/ofxhBinary.cpp > ofxhBinaryStrict.cpp + g++ -o verify_plugin_loads .github/workflows/verify_plugin_loads.cpp ofxhBinaryStrict.cpp -I openfx/HostSupport/include/ -I openfx/include/ + + ./verify_plugin_loads ${BINARY_PATH}/${BINARY_NAME} + + - name: Upload testing artifacts + uses: actions/upload-artifact@v3 + with: + name: openfx-io-build-ubuntu_22-testing + path: TestingBundle + + windows_build: + name: Build Windows Latest + runs-on: windows-latest + defaults: + run: + shell: msys2 {0} + + env: + EXTRA_PKG_CONFIG_PATHS: '/mingw64/ffmpeg-gpl2/lib/pkgconfig/:/mingw64/libraw-gpl2/lib/pkgconfig/' + + steps: + - name: Checkout branch + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Install Windows system packages + uses: msys2/setup-msys2@v2 + with: + msystem: mingw64 + update: true + install: git base-devel mingw-w64-x86_64-wget unzip mingw-w64-x86_64-cc mingw-w64-x86_64-python3 + + - name: Install Natron pacman repository + run: | + mkdir ${GITHUB_WORKSPACE}/natron_pacman_repo + cd ${GITHUB_WORKSPACE}/natron_pacman_repo + wget https://github.com/acolwell/Natron/releases/download/windows-mingw-package-repo/natron_package_repo.zip + unzip natron_package_repo.zip + NATRON_REPO_PATH=`cygpath -u $GITHUB_WORKSPACE` + echo -e "#NATRON_REPO_START\n[natron]\nSigLevel = Optional TrustAll\nServer = file://${NATRON_REPO_PATH}/natron_pacman_repo/\n#NATRON_REPO_END" >> /etc/pacman.conf + pacman -Syl natron + # Install extra Natron packages. + pacman -S --needed --noconfirm mingw-w64-x86_64-natron_openimageio mingw-w64-x86_64-natron_seexpr-git \ + mingw-w64-x86_64-natron_poppler mingw-w64-x86_64-natron_imagemagick mingw-w64-x86_64-natron_ffmpeg-gpl2 + + + - name: Build (release) + run: | + PKG_CONFIG_PATH=${EXTRA_PKG_CONFIG_PATHS}:${PKG_CONFIG_PATH} + make -j2 CONFIG=release + mkdir -p Bundle + mv `find IO -name "MINGW64_NT*-release"`/IO.ofx.bundle Bundle + + - name: Build (debug) + run: | + PKG_CONFIG_PATH=${EXTRA_PKG_CONFIG_PATHS}:${PKG_CONFIG_PATH} + make -j2 CONFIG=debug + + - name: Upload release artifacts + uses: actions/upload-artifact@v3 + with: + name: openfx-io-build-windows_latest-release + path: Bundle + + - name: Generate testing bundle + run: | + # Generate a self contained bundle that can be used for Natron automated testing. + set -x + cp -r Bundle TestingBundle + MANIFEST_NAME=IO.manifest + BINARY_NAME=IO.ofx + BINARY_PATH=TestingBundle/IO.ofx.bundle/Contents/Win64 + SEARCH_PATHS="/mingw64/ffmpeg-gpl2/bin /mingw64/libraw-gpl2/bin /mingw64/bin" + python3 .github/workflows/find_and_copy_deps.py --manifest=${MANIFEST_NAME} ${BINARY_PATH}/${BINARY_NAME} ${BINARY_PATH} ${SEARCH_PATHS} + + "/c/Program Files (x86)/Windows Kits/10/bin/10.0.22621.0/x64/mt.exe" -manifest ${MANIFEST_NAME} -outputresource:"${BINARY_PATH}/${BINARY_NAME};2" + + # Generate simple plugin tester to make sure we can load the test binary. + g++ -o verify_plugin_loads .github/workflows/verify_plugin_loads.cpp openfx/HostSupport/src/ofxhBinary.cpp openfx/HostSupport/src/ofxhUtilities.cpp -I openfx/HostSupport/include/ -I openfx/include/ + + ./verify_plugin_loads ${BINARY_PATH}/${BINARY_NAME} + + - name: Upload testing artifacts + uses: actions/upload-artifact@v3 + with: + name: openfx-io-build-windows_latest-testing + path: TestingBundle diff --git a/.github/workflows/find_and_copy_deps.py b/.github/workflows/find_and_copy_deps.py new file mode 100644 index 0000000..cb238a7 --- /dev/null +++ b/.github/workflows/find_and_copy_deps.py @@ -0,0 +1,231 @@ + +import argparse +import os.path +import re +import shutil +import subprocess +import sys +import tempfile + +import json + +def fix_location(location_path): + if sys.platform == "win32" and location_path[0] == '/': + new_location_path = subprocess.run( + ["cygpath", "-m", location_path], + capture_output=True).stdout.decode("utf-8").strip() + return new_location_path + + return location_path + +def get_deps_for_binary(binary_path, search_directories): + my_name = os.path.basename(binary_path) + proc = subprocess.Popen(['ldd', binary_path], stdout=subprocess.PIPE) + #print("proc ", proc) + + system_libs_to_skip = [] + if sys.platform == "linux": + # System libraries to skip + system_libs_to_skip = [ + "libc.so", + "libdl.so", + "libdrm.so", + "libm.so", + "libpthread.so", + "libresolv.so", + "libselinux.so", + "libudev.so", + "libGL.so", + "libGLX.so", + "libGLdispatch.so", + "libX11.so", + "libXau.so", + "libXdmcp.so", + "libXext.so", + "libXfixes.so", + "libXrender.so", + ] + ret = set() + while True: + line = proc.stdout.readline() + if not line: + break + + line = line.decode("utf-8").strip() + + p = re.compile("^([^ ]+) => ([^(]+)( (.*))?$") + m = p.match(line) + if m: + dep_name = m.group(1) + ldd_dep_location = m.group(2) + + # Skip self, Windows DLLs, and platform system libraries. + if (dep_name == my_name or + "/c/windows/system" in ldd_dep_location.lower() or + "/c/windows/winsxs/" in ldd_dep_location.lower() or + (len(system_libs_to_skip) > 0 and any(lib_name in ldd_dep_location for lib_name in system_libs_to_skip))): + continue + + #print(f"{dep_name} {dep_location}") + + search_dep_location = None + for x in search_directories: + alt_dep_location = os.path.join(x, dep_name) + if os.path.exists(alt_dep_location): + if os.path.isfile(alt_dep_location): + search_dep_location = alt_dep_location + break + + if search_dep_location: + #print("\t", dep_name, " -> ", search_dep_location) + ret.add((dep_name, fix_location(search_dep_location))) + elif ldd_dep_location == "not found": + print(f"{binary_path} depends on {dep_name} but ldd could not find its location.") + sys.exit(1) + else: + print(f"Could not find {dep_name} in search path, but ldd found it at {ldd_dep_location}. Please update search paths and run again.\n") + sys.exit(1) + else: + if ("linux-vdso.so.1" in line or + "ld-linux-x86-64.so" in line or + "statically linked" in line): + continue + print(f"Unexpected line in ldd output: {line}") + sys.exit(1) + return ret + +def get_all_deps_for_binary(binary_key_name, json_dict): + if binary_key_name not in json_dict["binary_deps"]: + return None + + ret = set() + deps_list = json_dict["binary_deps"][binary_key_name].copy() + while len(deps_list) > 0: + dep = deps_list.pop() + if dep not in ret: + ret.add(dep) + deps_list += json_dict["binary_deps"][dep] + return ret + +def main(): + if len(sys.argv) < 3: + print(f"Usage: {sys.argv[0]} [search_directories]") + sys.exit(1) + + parser = argparse.ArgumentParser( + prog='find_and_copy_deps', + description='Finds all the library dependencies of a executable \ + or shared library and copies them to a specified directory.') + parser.add_argument('--json') + parser.add_argument('--manifest') + parser.add_argument('target_binary') + parser.add_argument('output_directory') + parser.add_argument('search_directories', nargs="*", default=[]) + + cmd_line_args = parser.parse_args() + + target_binary = cmd_line_args.target_binary + output_directory = cmd_line_args.output_directory + search_directories = cmd_line_args.search_directories.copy() + + + if not os.path.exists(target_binary): + sys.stderr.write(f"{target_binary} does not exist.\n") + sys.exit(1) + + if not os.path.isfile(target_binary): + sys.stderr.write(f"{target_binary} is not a file.\n") + sys.exit(1) + + if not os.path.isdir(output_directory): + sys.exit(1) + + with tempfile.TemporaryDirectory() as tmpdirname: + print('created temporary directory', tmpdirname) + + binary_name_to_location_map = {} + binary_deps = {} + + ldd_filename_path = target_binary + binary_key_name = os.path.basename(target_binary) + if sys.platform == "win32" and target_binary.endswith(".ofx"): + # Need to make a copy of the plugin and rename it to have a dll extension so ldd + # works properly. + new_filename = os.path.splitext(os.path.basename(target_binary))[0] + ".dll" + ldd_filename_path = os.path.join(tmpdirname,new_filename) + shutil.copyfile(target_binary, ldd_filename_path) + + binary_deps = {} + deps_remaining = set() + + binary_name_to_location_map[binary_key_name] = ldd_filename_path + deps_remaining.add(binary_key_name) + + deps_visited = set() + while len(deps_remaining) > 0: + x = deps_remaining.pop() + + if x in deps_visited: + continue + deps_visited.add(x) + + print(f"Finding dependencies for {x}") + search_directories.append(os.path.dirname(binary_name_to_location_map[x])) + dep_info_list = get_deps_for_binary(binary_name_to_location_map[x], search_directories) + search_directories.pop() + deps_for_this_library = set() + for (dep_name, dep_location) in dep_info_list: + deps_for_this_library.add(dep_name) + + if dep_name not in binary_name_to_location_map: + binary_name_to_location_map[dep_name] = dep_location + elif not os.path.samefile(binary_name_to_location_map[dep_name], dep_location): + print(f"{dep_name} appears to have more than one location. \ + {binary_name_to_location_map[dep_name]} and {dep_location}") + sys.exit(1) + + if dep_name not in deps_visited: + deps_remaining.add(dep_name) + + sorted_deps_list = sorted(deps_for_this_library) + #for y in deps_list: + # print("\t{}".format(y)) + + binary_deps[x] = sorted_deps_list + + json_dict = { + "binary_deps": binary_deps, + "binary_locations": binary_name_to_location_map, + } + + + full_deps_list = sorted(get_all_deps_for_binary(binary_key_name, json_dict)) + + for x in full_deps_list: + src = binary_name_to_location_map[x] + print(f"{x} : {src}") + shutil.copy(src, output_directory) + + if cmd_line_args.json: + with open(cmd_line_args.json, "w", encoding='UTF-8') as write_file: + json.dump(json_dict, write_file, sort_keys=True, indent=2) + + + if cmd_line_args.manifest: + # Creates a manifest that can be used with a command like the one below to + # specify the required DLLs. + # mt -nologo -manifest IO.ofx.manifest -outputresource:"IO.ofx;2" + manifest = '\n' + manifest += '\n' + manifest += f' \n' + + for x in full_deps_list: + manifest += f' \n' + manifest += '' + + with open(cmd_line_args.manifest, "w", encoding='UTF-8') as write_file: + write_file.write(manifest) + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/verify_plugin_loads.cpp b/.github/workflows/verify_plugin_loads.cpp new file mode 100644 index 0000000..322ce97 --- /dev/null +++ b/.github/workflows/verify_plugin_loads.cpp @@ -0,0 +1,51 @@ + + +#include "ofxhBinary.h" +#include "ofxhPluginCache.h" + +int +main(int argc, const char* argv[]) +{ + if (argc < 2) { + std::cerr << "Usage: " << argv[0] << " \n"; + return 1; + } + + const char* const binaryPath = argv[1]; + + // TODO: Change code to _dlHandle = dlopen(_binaryPath.c_str(), RTLD_NOW|RTLD_LOCAL); + OFX::Binary bin(binaryPath); + + if (bin.isInvalid()) { + std::cerr << "error: '" << binaryPath << "' is invalid.\n"; + return 1; + } + + bin.load(); + + if (!bin.isLoaded()) { + std::cerr << "error: '" << binaryPath << "' failed to load.\n"; + return 1; + } + + auto* getNumberOfPlugins = (OFX::Host::OfxGetNumberOfPluginsFunc)bin.findSymbol("OfxGetNumberOfPlugins"); + auto* getPluginFunc = (OFX::Host::OfxGetPluginFunc)bin.findSymbol("OfxGetPlugin"); + + if (getNumberOfPlugins == nullptr || getPluginFunc == nullptr) { + std::cerr << "error: '" << binaryPath << "' missing required symbols.\n"; + return 1; + } + + std::cout << "Num_plugins: " << getNumberOfPlugins() << "\n"; + for (int i = 0; i < getNumberOfPlugins(); ++i) { + auto* plugin = getPluginFunc(i); + if (plugin == nullptr) { + std::cerr << "Failed to get plugin " << i << "\n"; + return 1; + } + + std::cout << "plugin[" << i << "] : " << plugin->pluginIdentifier << " v" << plugin->pluginVersionMajor << "." << plugin->pluginVersionMinor << "\n"; + } + + return 0; +}