From ce4435fc72a17049b0db2e1e5a4a9b9417b13526 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sat, 3 Jun 2023 15:40:24 +0100 Subject: [PATCH 01/45] trying out github actions --- .github/workflows/build-test-publish.yml | 79 ++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 .github/workflows/build-test-publish.yml diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml new file mode 100644 index 00000000..3d7449a7 --- /dev/null +++ b/.github/workflows/build-test-publish.yml @@ -0,0 +1,79 @@ +name: C/C++ CI + +on: + push: + branches: [ "master", "develop" ] + # Run on all pull requests + pull_request: + +jobs: + + + # build-wasm: + # runs-on: ubuntu-latest + + # steps: + # # Checkout the repository + # - name: Checkout + # uses: actions/checkout@v2 + + # # Install Conan + # - name: Install Conan + # run: | + # sudo apt-get update + # sudo apt-get install python3-pip + # pip3 install conan + + # # Configure conan for release build + # - name: Configure Build + # run: | + # ./configure.sh + + + build-linux: + runs-on: ubuntu-latest + + steps: + # Checkout the repository + - name: Checkout + uses: actions/checkout@v2 + + # Install Conan + - name: Install Conan + run: | + sudo apt-get update + sudo apt-get install python3-pip + pip3 install conan + + # Configure conan for release build + - name: Configure Build + run: | + ./configure.sh --build-type=Release + cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake + + # Build the project + - name: Build + run: | + cmake --build build --config Release + + # Run the tests + - name: tests + run: | + GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* GTEST_OUTPUT=xml:test-report.xml ctest . + + + # - name: Python package + # # if the commit is on the master branch or contains the string "release-test" + # if: github.ref == 'refs/heads/master' || contains(github.ref, 'release-test') + # run: | + + + + # build-macos: + # runs-on: macos-latest + + # build-windows: + # runs-on: windows-latest + + + From 0fb1609d22c65003b0b54d0c8091d247231ee15c Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sat, 3 Jun 2023 15:42:05 +0100 Subject: [PATCH 02/45] disable pipelines --- azure-pipelines.yml | 800 ++++++++++++++++++++++---------------------- 1 file changed, 400 insertions(+), 400 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1dcd9391..ecfe0020 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,409 +1,409 @@ -trigger: - batch: true - branches: - include: - - master - - develop - - hotfix/* - - release/* - - "*test_deploy*" -pr: - - master - - develop +# trigger: +# batch: true +# branches: +# include: +# - master +# - develop +# - hotfix/* +# - release/* +# - "*test_deploy*" +# pr: +# - master +# - develop -jobs: - - job: WASM - timeoutInMinutes: 60 - pool: - vmImage: "ubuntu-20.04" - steps: - - task: InstallSSHKey@0 - inputs: - knownHostsEntry: $(griddly_known_hosts_entry) - sshPublicKey: $(griddly_azure_public_key) - sshPassphrase: $(griddly_azure_passphrase) - sshKeySecureFile: griddly - - checkout: self - submodules: true - persistCredentials: true - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.8" - addToPath: true - displayName: Set Python 3.8 - name: py - - bash: | - set -ex - pip install conan==1.59.0 - continueOnError: false - displayName: Install Conan - - bash: | - set -ex - conan install deps/wasm/conanfile_wasm.txt --profile:host deps/wasm/emscripten.profile --profile:build default -s build_type=Release --build missing -if build_wasm - cmake . -B build_wasm -GNinja -DWASM=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake - cmake --build build_wasm --config Release - displayName: Build +# jobs: +# - job: WASM +# timeoutInMinutes: 60 +# pool: +# vmImage: "ubuntu-20.04" +# steps: +# - task: InstallSSHKey@0 +# inputs: +# knownHostsEntry: $(griddly_known_hosts_entry) +# sshPublicKey: $(griddly_azure_public_key) +# sshPassphrase: $(griddly_azure_passphrase) +# sshKeySecureFile: griddly +# - checkout: self +# submodules: true +# persistCredentials: true +# - task: UsePythonVersion@0 +# inputs: +# versionSpec: "3.8" +# addToPath: true +# displayName: Set Python 3.8 +# name: py +# - bash: | +# set -ex +# pip install conan==1.59.0 +# continueOnError: false +# displayName: Install Conan +# - bash: | +# set -ex +# conan install deps/wasm/conanfile_wasm.txt --profile:host deps/wasm/emscripten.profile --profile:build default -s build_type=Release --build missing -if build_wasm +# cmake . -B build_wasm -GNinja -DWASM=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake +# cmake --build build_wasm --config Release +# displayName: Build - - job: Linux - timeoutInMinutes: 60 - pool: - vmImage: "ubuntu-20.04" - steps: - - task: InstallSSHKey@0 - inputs: - knownHostsEntry: $(griddly_known_hosts_entry) - sshPublicKey: $(griddly_azure_public_key) - sshPassphrase: $(griddly_azure_passphrase) - sshKeySecureFile: griddly - - checkout: self - submodules: true - persistCredentials: true - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.8" - addToPath: true - displayName: Set Python 3.8 - name: py - - bash: | - set -ex - pip install conan==1.59.0 - continueOnError: false - displayName: Install Conan - - bash: | - set -ex - conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build missing -if build - cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" - cmake --build build --config Release - displayName: Build - - bash: | - GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* GTEST_OUTPUT=xml:test-report.xml ctest . - continueOnError: true - displayName: Test - - task: PublishTestResults@2 - inputs: - failTaskOnFailedTests: true - testResultsFormat: JUnit # Options: JUnit, NUnit, VSTest, xUnit, cTest - testResultsFiles: test-report.xml - testRunTitle: Ubuntu 20.04 Tests - displayName: Publish C++ Test Results - - script: | - set -ex - cd python - pip install poetry - poetry install - poetry run pip install pytest-azurepipelines - poetry run python -m pytest - displayName: Run and Publish Python Tests +# - job: Linux +# timeoutInMinutes: 60 +# pool: +# vmImage: "ubuntu-20.04" +# steps: +# - task: InstallSSHKey@0 +# inputs: +# knownHostsEntry: $(griddly_known_hosts_entry) +# sshPublicKey: $(griddly_azure_public_key) +# sshPassphrase: $(griddly_azure_passphrase) +# sshKeySecureFile: griddly +# - checkout: self +# submodules: true +# persistCredentials: true +# - task: UsePythonVersion@0 +# inputs: +# versionSpec: "3.8" +# addToPath: true +# displayName: Set Python 3.8 +# name: py +# - bash: | +# set -ex +# pip install conan==1.59.0 +# continueOnError: false +# displayName: Install Conan +# - bash: | +# set -ex +# conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build missing -if build +# cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" +# cmake --build build --config Release +# displayName: Build +# - bash: | +# GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* GTEST_OUTPUT=xml:test-report.xml ctest . +# continueOnError: true +# displayName: Test +# - task: PublishTestResults@2 +# inputs: +# failTaskOnFailedTests: true +# testResultsFormat: JUnit # Options: JUnit, NUnit, VSTest, xUnit, cTest +# testResultsFiles: test-report.xml +# testRunTitle: Ubuntu 20.04 Tests +# displayName: Publish C++ Test Results +# - script: | +# set -ex +# cd python +# pip install poetry +# poetry install +# poetry run pip install pytest-azurepipelines +# poetry run python -m pytest +# displayName: Run and Publish Python Tests - - job: PyPiDeployLinux - dependsOn: Linux - condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), contains(variables['Build.SourceBranch'], 'test_deploy'))) - timeoutInMinutes: 60 - pool: - vmImage: "ubuntu-20.04" - strategy: - matrix: - ManyLinux_2_28_p38: - PythonVersion: 3.8 - PyBin: cp38-cp38 - PLAT: manylinux_2_28_x86_64 - ManyLinux_2_28_p39: - PythonVersion: 3.9 - PyBin: cp39-cp39 - PLAT: manylinux_2_28_x86_64 - ManyLinux_2_28_p310: - PythonVersion: 3.10 - PyBin: cp310-cp310 - PLAT: manylinux_2_28_x86_64 - ManyLinux_2_28_p311: - PythonVersion: 3.11 - PyBin: cp311-cp311 - PLAT: manylinux_2_28_x86_64 - ManyLinux_2014_p38: - PythonVersion: 3.8 - PyBin: cp38-cp38 - PLAT: manylinux2014_x86_64 - ManyLinux_2014_p39: - PythonVersion: 3.9 - PyBin: cp39-cp39 - PLAT: manylinux2014_x86_64 - ManyLinux_2014_p310: - PythonVersion: 3.10 - PyBin: cp310-cp310 - PLAT: manylinux2014_x86_64 - steps: - - task: InstallSSHKey@0 - inputs: - knownHostsEntry: $(griddly_known_hosts_entry) - sshPublicKey: $(griddly_azure_public_key) - sshPassphrase: $(griddly_azure_passphrase) - sshKeySecureFile: griddly - - checkout: self - submodules: true - persistCredentials: true - - task: UsePythonVersion@0 - inputs: - versionSpec: $(PythonVersion) - addToPath: true - displayName: Set Python $(PythonVersion) - - bash: | - set -ex - pip install conan==1.59.0 - continueOnError: false - displayName: Install Conan - - script: | - set -ex - export GRIDDLY_ROOT=$(pwd) - export PYBIN=$(PyBin) - export PYVERSION=$(PythonVersion) +# - job: PyPiDeployLinux +# dependsOn: Linux +# condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), contains(variables['Build.SourceBranch'], 'test_deploy'))) +# timeoutInMinutes: 60 +# pool: +# vmImage: "ubuntu-20.04" +# strategy: +# matrix: +# ManyLinux_2_28_p38: +# PythonVersion: 3.8 +# PyBin: cp38-cp38 +# PLAT: manylinux_2_28_x86_64 +# ManyLinux_2_28_p39: +# PythonVersion: 3.9 +# PyBin: cp39-cp39 +# PLAT: manylinux_2_28_x86_64 +# ManyLinux_2_28_p310: +# PythonVersion: 3.10 +# PyBin: cp310-cp310 +# PLAT: manylinux_2_28_x86_64 +# ManyLinux_2_28_p311: +# PythonVersion: 3.11 +# PyBin: cp311-cp311 +# PLAT: manylinux_2_28_x86_64 +# ManyLinux_2014_p38: +# PythonVersion: 3.8 +# PyBin: cp38-cp38 +# PLAT: manylinux2014_x86_64 +# ManyLinux_2014_p39: +# PythonVersion: 3.9 +# PyBin: cp39-cp39 +# PLAT: manylinux2014_x86_64 +# ManyLinux_2014_p310: +# PythonVersion: 3.10 +# PyBin: cp310-cp310 +# PLAT: manylinux2014_x86_64 +# steps: +# - task: InstallSSHKey@0 +# inputs: +# knownHostsEntry: $(griddly_known_hosts_entry) +# sshPublicKey: $(griddly_azure_public_key) +# sshPassphrase: $(griddly_azure_passphrase) +# sshKeySecureFile: griddly +# - checkout: self +# submodules: true +# persistCredentials: true +# - task: UsePythonVersion@0 +# inputs: +# versionSpec: $(PythonVersion) +# addToPath: true +# displayName: Set Python $(PythonVersion) +# - bash: | +# set -ex +# pip install conan==1.59.0 +# continueOnError: false +# displayName: Install Conan +# - script: | +# set -ex +# export GRIDDLY_ROOT=$(pwd) +# export PYBIN=$(PyBin) +# export PYVERSION=$(PythonVersion) - ./python/manylinux/manylinux-build.sh - displayName: Build Wheels - - task: TwineAuthenticate@1 - displayName: "Twine Authenticate (test)" - inputs: - pythonUploadServiceConnection: pypitest - condition: and(succeeded(), contains(variables['Build.SourceBranch'], 'test_deploy')) - - task: TwineAuthenticate@1 - displayName: "Twine Authenticate (live)" - inputs: - pythonUploadServiceConnection: pypilive - condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - - script: | - pip install twine - cd python - python -m twine upload -r "griddly" --config-file $(PYPIRC_PATH) dist/*.whl - displayName: Deploy To PyPi +# ./python/manylinux/manylinux-build.sh +# displayName: Build Wheels +# - task: TwineAuthenticate@1 +# displayName: "Twine Authenticate (test)" +# inputs: +# pythonUploadServiceConnection: pypitest +# condition: and(succeeded(), contains(variables['Build.SourceBranch'], 'test_deploy')) +# - task: TwineAuthenticate@1 +# displayName: "Twine Authenticate (live)" +# inputs: +# pythonUploadServiceConnection: pypilive +# condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) +# - script: | +# pip install twine +# cd python +# python -m twine upload -r "griddly" --config-file $(PYPIRC_PATH) dist/*.whl +# displayName: Deploy To PyPi - - job: Windows - timeoutInMinutes: 60 - pool: - vmImage: "windows-latest" - steps: - - task: InstallSSHKey@0 - inputs: - knownHostsEntry: $(griddly_known_hosts_entry) - sshPublicKey: $(griddly_azure_public_key) - sshPassphrase: $(griddly_azure_passphrase) - sshKeySecureFile: griddly - - checkout: self - submodules: true - persistCredentials: true - - task: UsePythonVersion@0 - inputs: - versionSpec: 3.8 - addToPath: true - displayName: Set Python 3.8 - name: py - - bash: | - set -ex - pip install conan==1.59.0 - continueOnError: false - displayName: Install Conan - - bash: | - set -ex - conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build missing -if build - cmake . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" - cmake --build build --config Release - displayName: Build - - bash: | - export GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* - export GTEST_OUTPUT=xml:test-report.xml - ctest . -C Release - continueOnError: true - displayName: Test - - task: PublishTestResults@2 - inputs: - failTaskOnFailedTests: true - testResultsFormat: JUnit # Options: JUnit, NUnit, VSTest, xUnit, cTest - testResultsFiles: test-report.xml - testRunTitle: Windows Tests - displayName: Publish C++ Test Results - - bash: | - set -ex - cd python - pip install poetry - poetry install - poetry run pip install pytest-azurepipelines - poetry run python -m pytest - displayName: Run and Publish Python Tests +# - job: Windows +# timeoutInMinutes: 60 +# pool: +# vmImage: "windows-latest" +# steps: +# - task: InstallSSHKey@0 +# inputs: +# knownHostsEntry: $(griddly_known_hosts_entry) +# sshPublicKey: $(griddly_azure_public_key) +# sshPassphrase: $(griddly_azure_passphrase) +# sshKeySecureFile: griddly +# - checkout: self +# submodules: true +# persistCredentials: true +# - task: UsePythonVersion@0 +# inputs: +# versionSpec: 3.8 +# addToPath: true +# displayName: Set Python 3.8 +# name: py +# - bash: | +# set -ex +# pip install conan==1.59.0 +# continueOnError: false +# displayName: Install Conan +# - bash: | +# set -ex +# conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build missing -if build +# cmake . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" +# cmake --build build --config Release +# displayName: Build +# - bash: | +# export GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* +# export GTEST_OUTPUT=xml:test-report.xml +# ctest . -C Release +# continueOnError: true +# displayName: Test +# - task: PublishTestResults@2 +# inputs: +# failTaskOnFailedTests: true +# testResultsFormat: JUnit # Options: JUnit, NUnit, VSTest, xUnit, cTest +# testResultsFiles: test-report.xml +# testRunTitle: Windows Tests +# displayName: Publish C++ Test Results +# - bash: | +# set -ex +# cd python +# pip install poetry +# poetry install +# poetry run pip install pytest-azurepipelines +# poetry run python -m pytest +# displayName: Run and Publish Python Tests - - job: PyPiDeployWindows - condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), contains(variables['Build.SourceBranch'], 'test_deploy'))) - dependsOn: Windows - timeoutInMinutes: 60 - pool: - vmImage: "windows-latest" - strategy: - matrix: - Windows_p38: - PythonVersion: 3.8 - Windows_p39: - PythonVersion: 3.9 - Windows_p310: - PythonVersion: 3.10 - Windows_p311: - PythonVersion: 3.11 - steps: - - task: InstallSSHKey@0 - inputs: - knownHostsEntry: $(griddly_known_hosts_entry) - sshPublicKey: $(griddly_azure_public_key) - sshPassphrase: $(griddly_azure_passphrase) - sshKeySecureFile: griddly - - checkout: self - submodules: true - persistCredentials: true - - task: UsePythonVersion@0 - inputs: - versionSpec: $(PythonVersion) - addToPath: true - displayName: Set Python $(PythonVersion) - name: py - - bash: | - set -ex - pip install conan==1.59.0 - continueOnError: false - displayName: Install Conan - - bash: | - set -ex - conan install deps/conanfile.txt --profile default --profile deps/build.profile -s build_type=Release --build missing -if build - cmake . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" - cmake --build build --config Release - displayName: Build - - script: | - cd python - pip install wheel setuptools - python setup.py bdist_wheel - displayName: Build Wheels - - task: TwineAuthenticate@1 - displayName: "Twine Authenticate (test)" - inputs: - pythonUploadServiceConnection: pypitest - condition: and(succeeded(), contains(variables['Build.SourceBranch'], 'test_deploy')) - - task: TwineAuthenticate@1 - displayName: "Twine Authenticate (live)" - inputs: - pythonUploadServiceConnection: pypilive - condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - - script: | - pip install twine - cd python - python -m twine upload -r "griddly" --config-file $(PYPIRC_PATH) dist/*.whl - displayName: Deploy To PyPi +# - job: PyPiDeployWindows +# condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), contains(variables['Build.SourceBranch'], 'test_deploy'))) +# dependsOn: Windows +# timeoutInMinutes: 60 +# pool: +# vmImage: "windows-latest" +# strategy: +# matrix: +# Windows_p38: +# PythonVersion: 3.8 +# Windows_p39: +# PythonVersion: 3.9 +# Windows_p310: +# PythonVersion: 3.10 +# Windows_p311: +# PythonVersion: 3.11 +# steps: +# - task: InstallSSHKey@0 +# inputs: +# knownHostsEntry: $(griddly_known_hosts_entry) +# sshPublicKey: $(griddly_azure_public_key) +# sshPassphrase: $(griddly_azure_passphrase) +# sshKeySecureFile: griddly +# - checkout: self +# submodules: true +# persistCredentials: true +# - task: UsePythonVersion@0 +# inputs: +# versionSpec: $(PythonVersion) +# addToPath: true +# displayName: Set Python $(PythonVersion) +# name: py +# - bash: | +# set -ex +# pip install conan==1.59.0 +# continueOnError: false +# displayName: Install Conan +# - bash: | +# set -ex +# conan install deps/conanfile.txt --profile default --profile deps/build.profile -s build_type=Release --build missing -if build +# cmake . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" +# cmake --build build --config Release +# displayName: Build +# - script: | +# cd python +# pip install wheel setuptools +# python setup.py bdist_wheel +# displayName: Build Wheels +# - task: TwineAuthenticate@1 +# displayName: "Twine Authenticate (test)" +# inputs: +# pythonUploadServiceConnection: pypitest +# condition: and(succeeded(), contains(variables['Build.SourceBranch'], 'test_deploy')) +# - task: TwineAuthenticate@1 +# displayName: "Twine Authenticate (live)" +# inputs: +# pythonUploadServiceConnection: pypilive +# condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) +# - script: | +# pip install twine +# cd python +# python -m twine upload -r "griddly" --config-file $(PYPIRC_PATH) dist/*.whl +# displayName: Deploy To PyPi - - job: MacOS - timeoutInMinutes: 60 - pool: - vmImage: "macOS-latest" - steps: - - task: InstallSSHKey@0 - inputs: - knownHostsEntry: $(griddly_known_hosts_entry) - sshPublicKey: $(griddly_azure_public_key) - sshPassphrase: $(griddly_azure_passphrase) - sshKeySecureFile: griddly - - checkout: self - submodules: true - persistCredentials: true - - task: UsePythonVersion@0 - inputs: - versionSpec: 3.8 - addToPath: true - displayName: Set Python 3.8 - name: py - - bash: | - set -ex - pip install conan==1.59.0 - continueOnError: false - displayName: Install Conan - - bash: | - set -ex - conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build missing -if build - cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" - cmake --build build --config Release - displayName: Build - - bash: | - GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* GTEST_OUTPUT=xml:test-report.xml ctest . - continueOnError: true - displayName: Test - - task: PublishTestResults@2 - inputs: - failTaskOnFailedTests: true - testResultsFormat: JUnit # Options: JUnit, NUnit, VSTest, xUnit, cTest - testResultsFiles: test-report.xml - testRunTitle: MacOS Tests - displayName: Publish C++ Test Results - - script: | - set -ex - cd python - pip install poetry - poetry install - poetry run pip install pytest-azurepipelines - poetry run python -m pytest - displayName: Run and Publish Python Tests +# - job: MacOS +# timeoutInMinutes: 60 +# pool: +# vmImage: "macOS-latest" +# steps: +# - task: InstallSSHKey@0 +# inputs: +# knownHostsEntry: $(griddly_known_hosts_entry) +# sshPublicKey: $(griddly_azure_public_key) +# sshPassphrase: $(griddly_azure_passphrase) +# sshKeySecureFile: griddly +# - checkout: self +# submodules: true +# persistCredentials: true +# - task: UsePythonVersion@0 +# inputs: +# versionSpec: 3.8 +# addToPath: true +# displayName: Set Python 3.8 +# name: py +# - bash: | +# set -ex +# pip install conan==1.59.0 +# continueOnError: false +# displayName: Install Conan +# - bash: | +# set -ex +# conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build missing -if build +# cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" +# cmake --build build --config Release +# displayName: Build +# - bash: | +# GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* GTEST_OUTPUT=xml:test-report.xml ctest . +# continueOnError: true +# displayName: Test +# - task: PublishTestResults@2 +# inputs: +# failTaskOnFailedTests: true +# testResultsFormat: JUnit # Options: JUnit, NUnit, VSTest, xUnit, cTest +# testResultsFiles: test-report.xml +# testRunTitle: MacOS Tests +# displayName: Publish C++ Test Results +# - script: | +# set -ex +# cd python +# pip install poetry +# poetry install +# poetry run pip install pytest-azurepipelines +# poetry run python -m pytest +# displayName: Run and Publish Python Tests - - job: PyPiDeployMacOS - condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), contains(variables['Build.SourceBranch'], 'test_deploy'))) - dependsOn: MacOS - timeoutInMinutes: 60 - pool: - vmImage: "macOS-latest" - strategy: - matrix: - MacOS_p38: - PythonVersion: 3.8 - MacOS_p39: - PythonVersion: 3.9 - MacOS_p310: - PythonVersion: 3.10 - MacOS_p311: - PythonVersion: 3.11 - steps: - - task: InstallSSHKey@0 - inputs: - knownHostsEntry: $(griddly_known_hosts_entry) - sshPublicKey: $(griddly_azure_public_key) - sshPassphrase: $(griddly_azure_passphrase) - sshKeySecureFile: griddly - - checkout: self - submodules: true - persistCredentials: true - - task: UsePythonVersion@0 - inputs: - versionSpec: $(PythonVersion) - addToPath: true - displayName: Set Python $(PythonVersion) - name: py - - bash: | - set -ex - pip install conan==1.59.0 - continueOnError: false - displayName: Install Conan - - bash: | - set -ex - conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build missing -if build - cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" - cmake --build build --config Release - displayName: Build - - script: | - cd python - pip install wheel setuptools - python setup.py bdist_wheel - displayName: Build Wheels - - task: TwineAuthenticate@1 - displayName: "Twine Authenticate (test)" - inputs: - pythonUploadServiceConnection: pypitest - condition: and(succeeded(), contains(variables['Build.SourceBranch'], 'test_deploy')) - - task: TwineAuthenticate@1 - displayName: "Twine Authenticate (live)" - inputs: - pythonUploadServiceConnection: pypilive - condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - - script: | - pip install twine - cd python - python -m twine upload -r "griddly" --config-file $(PYPIRC_PATH) dist/*.whl - displayName: Deploy To PyPi +# - job: PyPiDeployMacOS +# condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), contains(variables['Build.SourceBranch'], 'test_deploy'))) +# dependsOn: MacOS +# timeoutInMinutes: 60 +# pool: +# vmImage: "macOS-latest" +# strategy: +# matrix: +# MacOS_p38: +# PythonVersion: 3.8 +# MacOS_p39: +# PythonVersion: 3.9 +# MacOS_p310: +# PythonVersion: 3.10 +# MacOS_p311: +# PythonVersion: 3.11 +# steps: +# - task: InstallSSHKey@0 +# inputs: +# knownHostsEntry: $(griddly_known_hosts_entry) +# sshPublicKey: $(griddly_azure_public_key) +# sshPassphrase: $(griddly_azure_passphrase) +# sshKeySecureFile: griddly +# - checkout: self +# submodules: true +# persistCredentials: true +# - task: UsePythonVersion@0 +# inputs: +# versionSpec: $(PythonVersion) +# addToPath: true +# displayName: Set Python $(PythonVersion) +# name: py +# - bash: | +# set -ex +# pip install conan==1.59.0 +# continueOnError: false +# displayName: Install Conan +# - bash: | +# set -ex +# conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build missing -if build +# cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" +# cmake --build build --config Release +# displayName: Build +# - script: | +# cd python +# pip install wheel setuptools +# python setup.py bdist_wheel +# displayName: Build Wheels +# - task: TwineAuthenticate@1 +# displayName: "Twine Authenticate (test)" +# inputs: +# pythonUploadServiceConnection: pypitest +# condition: and(succeeded(), contains(variables['Build.SourceBranch'], 'test_deploy')) +# - task: TwineAuthenticate@1 +# displayName: "Twine Authenticate (live)" +# inputs: +# pythonUploadServiceConnection: pypilive +# condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) +# - script: | +# pip install twine +# cd python +# python -m twine upload -r "griddly" --config-file $(PYPIRC_PATH) dist/*.whl +# displayName: Deploy To PyPi From 8997432cee79aa6a07a383bd936c27a0cf231bd9 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sat, 3 Jun 2023 15:43:57 +0100 Subject: [PATCH 03/45] fix conan version --- .github/workflows/build-test-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 3d7449a7..49ee4e1c 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -43,7 +43,7 @@ jobs: run: | sudo apt-get update sudo apt-get install python3-pip - pip3 install conan + pip3 install conan==1.58.0 # Configure conan for release build - name: Configure Build From 332f5273f811d6d8e9ab80e257cb1b30b180ed99 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sat, 3 Jun 2023 15:48:14 +0100 Subject: [PATCH 04/45] fixing configire.sh --- .github/workflows/build-test-publish.yml | 4 ++-- configure.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 49ee4e1c..f6318863 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -1,4 +1,4 @@ -name: C/C++ CI +name: Griddly Build and Deploy on: push: @@ -43,7 +43,7 @@ jobs: run: | sudo apt-get update sudo apt-get install python3-pip - pip3 install conan==1.58.0 + pip3 install conan==1.59.0 # Configure conan for release build - name: Configure Build diff --git a/configure.sh b/configure.sh index 5f8c5017..aa15bbcb 100755 --- a/configure.sh +++ b/configure.sh @@ -29,8 +29,8 @@ echo $PLATFORM if [[ $PLATFORM == "WASM" ]] then - conan install deps/wasm/conanfile_wasm.txt --profile:host deps/wasm/emscripten.profile --profile:build default -s build_type=$BUILD --build missing -if build_wasm + conan install deps/wasm/conanfile_wasm.txt --pr:h deps/wasm/emscripten.profile --pr:b default -s build_type=$BUILD --build missing -if build_wasm else - conan install deps/conanfile.txt --profile default --profile deps/build.profile -s build_type=$BUILD --build $CONAN_BUILD -if build + conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=$BUILD --build $CONAN_BUILD -if build fi From d8985ed5a7ec44aa65245bc069c570c12c90f870 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sat, 3 Jun 2023 15:49:28 +0100 Subject: [PATCH 05/45] rename piplines file so it does not trigger --- azure-pipelines.yml | 409 ---------------------------------------- old_azure-pipelines.yml | 409 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 409 insertions(+), 409 deletions(-) delete mode 100644 azure-pipelines.yml create mode 100644 old_azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index ecfe0020..00000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,409 +0,0 @@ -# trigger: -# batch: true -# branches: -# include: -# - master -# - develop -# - hotfix/* -# - release/* -# - "*test_deploy*" -# pr: -# - master -# - develop - - -# jobs: -# - job: WASM -# timeoutInMinutes: 60 -# pool: -# vmImage: "ubuntu-20.04" -# steps: -# - task: InstallSSHKey@0 -# inputs: -# knownHostsEntry: $(griddly_known_hosts_entry) -# sshPublicKey: $(griddly_azure_public_key) -# sshPassphrase: $(griddly_azure_passphrase) -# sshKeySecureFile: griddly -# - checkout: self -# submodules: true -# persistCredentials: true -# - task: UsePythonVersion@0 -# inputs: -# versionSpec: "3.8" -# addToPath: true -# displayName: Set Python 3.8 -# name: py -# - bash: | -# set -ex -# pip install conan==1.59.0 -# continueOnError: false -# displayName: Install Conan -# - bash: | -# set -ex -# conan install deps/wasm/conanfile_wasm.txt --profile:host deps/wasm/emscripten.profile --profile:build default -s build_type=Release --build missing -if build_wasm -# cmake . -B build_wasm -GNinja -DWASM=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -# cmake --build build_wasm --config Release -# displayName: Build - -# - job: Linux -# timeoutInMinutes: 60 -# pool: -# vmImage: "ubuntu-20.04" -# steps: -# - task: InstallSSHKey@0 -# inputs: -# knownHostsEntry: $(griddly_known_hosts_entry) -# sshPublicKey: $(griddly_azure_public_key) -# sshPassphrase: $(griddly_azure_passphrase) -# sshKeySecureFile: griddly -# - checkout: self -# submodules: true -# persistCredentials: true -# - task: UsePythonVersion@0 -# inputs: -# versionSpec: "3.8" -# addToPath: true -# displayName: Set Python 3.8 -# name: py -# - bash: | -# set -ex -# pip install conan==1.59.0 -# continueOnError: false -# displayName: Install Conan -# - bash: | -# set -ex -# conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build missing -if build -# cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" -# cmake --build build --config Release -# displayName: Build -# - bash: | -# GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* GTEST_OUTPUT=xml:test-report.xml ctest . -# continueOnError: true -# displayName: Test -# - task: PublishTestResults@2 -# inputs: -# failTaskOnFailedTests: true -# testResultsFormat: JUnit # Options: JUnit, NUnit, VSTest, xUnit, cTest -# testResultsFiles: test-report.xml -# testRunTitle: Ubuntu 20.04 Tests -# displayName: Publish C++ Test Results -# - script: | -# set -ex -# cd python -# pip install poetry -# poetry install -# poetry run pip install pytest-azurepipelines -# poetry run python -m pytest -# displayName: Run and Publish Python Tests - -# - job: PyPiDeployLinux -# dependsOn: Linux -# condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), contains(variables['Build.SourceBranch'], 'test_deploy'))) -# timeoutInMinutes: 60 -# pool: -# vmImage: "ubuntu-20.04" -# strategy: -# matrix: -# ManyLinux_2_28_p38: -# PythonVersion: 3.8 -# PyBin: cp38-cp38 -# PLAT: manylinux_2_28_x86_64 -# ManyLinux_2_28_p39: -# PythonVersion: 3.9 -# PyBin: cp39-cp39 -# PLAT: manylinux_2_28_x86_64 -# ManyLinux_2_28_p310: -# PythonVersion: 3.10 -# PyBin: cp310-cp310 -# PLAT: manylinux_2_28_x86_64 -# ManyLinux_2_28_p311: -# PythonVersion: 3.11 -# PyBin: cp311-cp311 -# PLAT: manylinux_2_28_x86_64 -# ManyLinux_2014_p38: -# PythonVersion: 3.8 -# PyBin: cp38-cp38 -# PLAT: manylinux2014_x86_64 -# ManyLinux_2014_p39: -# PythonVersion: 3.9 -# PyBin: cp39-cp39 -# PLAT: manylinux2014_x86_64 -# ManyLinux_2014_p310: -# PythonVersion: 3.10 -# PyBin: cp310-cp310 -# PLAT: manylinux2014_x86_64 -# steps: -# - task: InstallSSHKey@0 -# inputs: -# knownHostsEntry: $(griddly_known_hosts_entry) -# sshPublicKey: $(griddly_azure_public_key) -# sshPassphrase: $(griddly_azure_passphrase) -# sshKeySecureFile: griddly -# - checkout: self -# submodules: true -# persistCredentials: true -# - task: UsePythonVersion@0 -# inputs: -# versionSpec: $(PythonVersion) -# addToPath: true -# displayName: Set Python $(PythonVersion) -# - bash: | -# set -ex -# pip install conan==1.59.0 -# continueOnError: false -# displayName: Install Conan -# - script: | -# set -ex -# export GRIDDLY_ROOT=$(pwd) -# export PYBIN=$(PyBin) -# export PYVERSION=$(PythonVersion) - -# ./python/manylinux/manylinux-build.sh -# displayName: Build Wheels -# - task: TwineAuthenticate@1 -# displayName: "Twine Authenticate (test)" -# inputs: -# pythonUploadServiceConnection: pypitest -# condition: and(succeeded(), contains(variables['Build.SourceBranch'], 'test_deploy')) -# - task: TwineAuthenticate@1 -# displayName: "Twine Authenticate (live)" -# inputs: -# pythonUploadServiceConnection: pypilive -# condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) -# - script: | -# pip install twine -# cd python -# python -m twine upload -r "griddly" --config-file $(PYPIRC_PATH) dist/*.whl -# displayName: Deploy To PyPi - -# - job: Windows -# timeoutInMinutes: 60 -# pool: -# vmImage: "windows-latest" -# steps: -# - task: InstallSSHKey@0 -# inputs: -# knownHostsEntry: $(griddly_known_hosts_entry) -# sshPublicKey: $(griddly_azure_public_key) -# sshPassphrase: $(griddly_azure_passphrase) -# sshKeySecureFile: griddly -# - checkout: self -# submodules: true -# persistCredentials: true -# - task: UsePythonVersion@0 -# inputs: -# versionSpec: 3.8 -# addToPath: true -# displayName: Set Python 3.8 -# name: py -# - bash: | -# set -ex -# pip install conan==1.59.0 -# continueOnError: false -# displayName: Install Conan -# - bash: | -# set -ex -# conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build missing -if build -# cmake . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" -# cmake --build build --config Release -# displayName: Build -# - bash: | -# export GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* -# export GTEST_OUTPUT=xml:test-report.xml -# ctest . -C Release -# continueOnError: true -# displayName: Test -# - task: PublishTestResults@2 -# inputs: -# failTaskOnFailedTests: true -# testResultsFormat: JUnit # Options: JUnit, NUnit, VSTest, xUnit, cTest -# testResultsFiles: test-report.xml -# testRunTitle: Windows Tests -# displayName: Publish C++ Test Results -# - bash: | -# set -ex -# cd python -# pip install poetry -# poetry install -# poetry run pip install pytest-azurepipelines -# poetry run python -m pytest -# displayName: Run and Publish Python Tests - -# - job: PyPiDeployWindows -# condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), contains(variables['Build.SourceBranch'], 'test_deploy'))) -# dependsOn: Windows -# timeoutInMinutes: 60 -# pool: -# vmImage: "windows-latest" -# strategy: -# matrix: -# Windows_p38: -# PythonVersion: 3.8 -# Windows_p39: -# PythonVersion: 3.9 -# Windows_p310: -# PythonVersion: 3.10 -# Windows_p311: -# PythonVersion: 3.11 -# steps: -# - task: InstallSSHKey@0 -# inputs: -# knownHostsEntry: $(griddly_known_hosts_entry) -# sshPublicKey: $(griddly_azure_public_key) -# sshPassphrase: $(griddly_azure_passphrase) -# sshKeySecureFile: griddly -# - checkout: self -# submodules: true -# persistCredentials: true -# - task: UsePythonVersion@0 -# inputs: -# versionSpec: $(PythonVersion) -# addToPath: true -# displayName: Set Python $(PythonVersion) -# name: py -# - bash: | -# set -ex -# pip install conan==1.59.0 -# continueOnError: false -# displayName: Install Conan -# - bash: | -# set -ex -# conan install deps/conanfile.txt --profile default --profile deps/build.profile -s build_type=Release --build missing -if build -# cmake . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" -# cmake --build build --config Release -# displayName: Build -# - script: | -# cd python -# pip install wheel setuptools -# python setup.py bdist_wheel -# displayName: Build Wheels -# - task: TwineAuthenticate@1 -# displayName: "Twine Authenticate (test)" -# inputs: -# pythonUploadServiceConnection: pypitest -# condition: and(succeeded(), contains(variables['Build.SourceBranch'], 'test_deploy')) -# - task: TwineAuthenticate@1 -# displayName: "Twine Authenticate (live)" -# inputs: -# pythonUploadServiceConnection: pypilive -# condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) -# - script: | -# pip install twine -# cd python -# python -m twine upload -r "griddly" --config-file $(PYPIRC_PATH) dist/*.whl -# displayName: Deploy To PyPi - -# - job: MacOS -# timeoutInMinutes: 60 -# pool: -# vmImage: "macOS-latest" -# steps: -# - task: InstallSSHKey@0 -# inputs: -# knownHostsEntry: $(griddly_known_hosts_entry) -# sshPublicKey: $(griddly_azure_public_key) -# sshPassphrase: $(griddly_azure_passphrase) -# sshKeySecureFile: griddly -# - checkout: self -# submodules: true -# persistCredentials: true -# - task: UsePythonVersion@0 -# inputs: -# versionSpec: 3.8 -# addToPath: true -# displayName: Set Python 3.8 -# name: py -# - bash: | -# set -ex -# pip install conan==1.59.0 -# continueOnError: false -# displayName: Install Conan -# - bash: | -# set -ex -# conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build missing -if build -# cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" -# cmake --build build --config Release -# displayName: Build -# - bash: | -# GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* GTEST_OUTPUT=xml:test-report.xml ctest . -# continueOnError: true -# displayName: Test -# - task: PublishTestResults@2 -# inputs: -# failTaskOnFailedTests: true -# testResultsFormat: JUnit # Options: JUnit, NUnit, VSTest, xUnit, cTest -# testResultsFiles: test-report.xml -# testRunTitle: MacOS Tests -# displayName: Publish C++ Test Results -# - script: | -# set -ex -# cd python -# pip install poetry -# poetry install -# poetry run pip install pytest-azurepipelines -# poetry run python -m pytest -# displayName: Run and Publish Python Tests - -# - job: PyPiDeployMacOS -# condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), contains(variables['Build.SourceBranch'], 'test_deploy'))) -# dependsOn: MacOS -# timeoutInMinutes: 60 -# pool: -# vmImage: "macOS-latest" -# strategy: -# matrix: -# MacOS_p38: -# PythonVersion: 3.8 -# MacOS_p39: -# PythonVersion: 3.9 -# MacOS_p310: -# PythonVersion: 3.10 -# MacOS_p311: -# PythonVersion: 3.11 -# steps: -# - task: InstallSSHKey@0 -# inputs: -# knownHostsEntry: $(griddly_known_hosts_entry) -# sshPublicKey: $(griddly_azure_public_key) -# sshPassphrase: $(griddly_azure_passphrase) -# sshKeySecureFile: griddly -# - checkout: self -# submodules: true -# persistCredentials: true -# - task: UsePythonVersion@0 -# inputs: -# versionSpec: $(PythonVersion) -# addToPath: true -# displayName: Set Python $(PythonVersion) -# name: py -# - bash: | -# set -ex -# pip install conan==1.59.0 -# continueOnError: false -# displayName: Install Conan -# - bash: | -# set -ex -# conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build missing -if build -# cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" -# cmake --build build --config Release -# displayName: Build -# - script: | -# cd python -# pip install wheel setuptools -# python setup.py bdist_wheel -# displayName: Build Wheels -# - task: TwineAuthenticate@1 -# displayName: "Twine Authenticate (test)" -# inputs: -# pythonUploadServiceConnection: pypitest -# condition: and(succeeded(), contains(variables['Build.SourceBranch'], 'test_deploy')) -# - task: TwineAuthenticate@1 -# displayName: "Twine Authenticate (live)" -# inputs: -# pythonUploadServiceConnection: pypilive -# condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) -# - script: | -# pip install twine -# cd python -# python -m twine upload -r "griddly" --config-file $(PYPIRC_PATH) dist/*.whl -# displayName: Deploy To PyPi diff --git a/old_azure-pipelines.yml b/old_azure-pipelines.yml new file mode 100644 index 00000000..1dcd9391 --- /dev/null +++ b/old_azure-pipelines.yml @@ -0,0 +1,409 @@ +trigger: + batch: true + branches: + include: + - master + - develop + - hotfix/* + - release/* + - "*test_deploy*" +pr: + - master + - develop + + +jobs: + - job: WASM + timeoutInMinutes: 60 + pool: + vmImage: "ubuntu-20.04" + steps: + - task: InstallSSHKey@0 + inputs: + knownHostsEntry: $(griddly_known_hosts_entry) + sshPublicKey: $(griddly_azure_public_key) + sshPassphrase: $(griddly_azure_passphrase) + sshKeySecureFile: griddly + - checkout: self + submodules: true + persistCredentials: true + - task: UsePythonVersion@0 + inputs: + versionSpec: "3.8" + addToPath: true + displayName: Set Python 3.8 + name: py + - bash: | + set -ex + pip install conan==1.59.0 + continueOnError: false + displayName: Install Conan + - bash: | + set -ex + conan install deps/wasm/conanfile_wasm.txt --profile:host deps/wasm/emscripten.profile --profile:build default -s build_type=Release --build missing -if build_wasm + cmake . -B build_wasm -GNinja -DWASM=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake + cmake --build build_wasm --config Release + displayName: Build + + - job: Linux + timeoutInMinutes: 60 + pool: + vmImage: "ubuntu-20.04" + steps: + - task: InstallSSHKey@0 + inputs: + knownHostsEntry: $(griddly_known_hosts_entry) + sshPublicKey: $(griddly_azure_public_key) + sshPassphrase: $(griddly_azure_passphrase) + sshKeySecureFile: griddly + - checkout: self + submodules: true + persistCredentials: true + - task: UsePythonVersion@0 + inputs: + versionSpec: "3.8" + addToPath: true + displayName: Set Python 3.8 + name: py + - bash: | + set -ex + pip install conan==1.59.0 + continueOnError: false + displayName: Install Conan + - bash: | + set -ex + conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build missing -if build + cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" + cmake --build build --config Release + displayName: Build + - bash: | + GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* GTEST_OUTPUT=xml:test-report.xml ctest . + continueOnError: true + displayName: Test + - task: PublishTestResults@2 + inputs: + failTaskOnFailedTests: true + testResultsFormat: JUnit # Options: JUnit, NUnit, VSTest, xUnit, cTest + testResultsFiles: test-report.xml + testRunTitle: Ubuntu 20.04 Tests + displayName: Publish C++ Test Results + - script: | + set -ex + cd python + pip install poetry + poetry install + poetry run pip install pytest-azurepipelines + poetry run python -m pytest + displayName: Run and Publish Python Tests + + - job: PyPiDeployLinux + dependsOn: Linux + condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), contains(variables['Build.SourceBranch'], 'test_deploy'))) + timeoutInMinutes: 60 + pool: + vmImage: "ubuntu-20.04" + strategy: + matrix: + ManyLinux_2_28_p38: + PythonVersion: 3.8 + PyBin: cp38-cp38 + PLAT: manylinux_2_28_x86_64 + ManyLinux_2_28_p39: + PythonVersion: 3.9 + PyBin: cp39-cp39 + PLAT: manylinux_2_28_x86_64 + ManyLinux_2_28_p310: + PythonVersion: 3.10 + PyBin: cp310-cp310 + PLAT: manylinux_2_28_x86_64 + ManyLinux_2_28_p311: + PythonVersion: 3.11 + PyBin: cp311-cp311 + PLAT: manylinux_2_28_x86_64 + ManyLinux_2014_p38: + PythonVersion: 3.8 + PyBin: cp38-cp38 + PLAT: manylinux2014_x86_64 + ManyLinux_2014_p39: + PythonVersion: 3.9 + PyBin: cp39-cp39 + PLAT: manylinux2014_x86_64 + ManyLinux_2014_p310: + PythonVersion: 3.10 + PyBin: cp310-cp310 + PLAT: manylinux2014_x86_64 + steps: + - task: InstallSSHKey@0 + inputs: + knownHostsEntry: $(griddly_known_hosts_entry) + sshPublicKey: $(griddly_azure_public_key) + sshPassphrase: $(griddly_azure_passphrase) + sshKeySecureFile: griddly + - checkout: self + submodules: true + persistCredentials: true + - task: UsePythonVersion@0 + inputs: + versionSpec: $(PythonVersion) + addToPath: true + displayName: Set Python $(PythonVersion) + - bash: | + set -ex + pip install conan==1.59.0 + continueOnError: false + displayName: Install Conan + - script: | + set -ex + export GRIDDLY_ROOT=$(pwd) + export PYBIN=$(PyBin) + export PYVERSION=$(PythonVersion) + + ./python/manylinux/manylinux-build.sh + displayName: Build Wheels + - task: TwineAuthenticate@1 + displayName: "Twine Authenticate (test)" + inputs: + pythonUploadServiceConnection: pypitest + condition: and(succeeded(), contains(variables['Build.SourceBranch'], 'test_deploy')) + - task: TwineAuthenticate@1 + displayName: "Twine Authenticate (live)" + inputs: + pythonUploadServiceConnection: pypilive + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + - script: | + pip install twine + cd python + python -m twine upload -r "griddly" --config-file $(PYPIRC_PATH) dist/*.whl + displayName: Deploy To PyPi + + - job: Windows + timeoutInMinutes: 60 + pool: + vmImage: "windows-latest" + steps: + - task: InstallSSHKey@0 + inputs: + knownHostsEntry: $(griddly_known_hosts_entry) + sshPublicKey: $(griddly_azure_public_key) + sshPassphrase: $(griddly_azure_passphrase) + sshKeySecureFile: griddly + - checkout: self + submodules: true + persistCredentials: true + - task: UsePythonVersion@0 + inputs: + versionSpec: 3.8 + addToPath: true + displayName: Set Python 3.8 + name: py + - bash: | + set -ex + pip install conan==1.59.0 + continueOnError: false + displayName: Install Conan + - bash: | + set -ex + conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build missing -if build + cmake . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" + cmake --build build --config Release + displayName: Build + - bash: | + export GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* + export GTEST_OUTPUT=xml:test-report.xml + ctest . -C Release + continueOnError: true + displayName: Test + - task: PublishTestResults@2 + inputs: + failTaskOnFailedTests: true + testResultsFormat: JUnit # Options: JUnit, NUnit, VSTest, xUnit, cTest + testResultsFiles: test-report.xml + testRunTitle: Windows Tests + displayName: Publish C++ Test Results + - bash: | + set -ex + cd python + pip install poetry + poetry install + poetry run pip install pytest-azurepipelines + poetry run python -m pytest + displayName: Run and Publish Python Tests + + - job: PyPiDeployWindows + condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), contains(variables['Build.SourceBranch'], 'test_deploy'))) + dependsOn: Windows + timeoutInMinutes: 60 + pool: + vmImage: "windows-latest" + strategy: + matrix: + Windows_p38: + PythonVersion: 3.8 + Windows_p39: + PythonVersion: 3.9 + Windows_p310: + PythonVersion: 3.10 + Windows_p311: + PythonVersion: 3.11 + steps: + - task: InstallSSHKey@0 + inputs: + knownHostsEntry: $(griddly_known_hosts_entry) + sshPublicKey: $(griddly_azure_public_key) + sshPassphrase: $(griddly_azure_passphrase) + sshKeySecureFile: griddly + - checkout: self + submodules: true + persistCredentials: true + - task: UsePythonVersion@0 + inputs: + versionSpec: $(PythonVersion) + addToPath: true + displayName: Set Python $(PythonVersion) + name: py + - bash: | + set -ex + pip install conan==1.59.0 + continueOnError: false + displayName: Install Conan + - bash: | + set -ex + conan install deps/conanfile.txt --profile default --profile deps/build.profile -s build_type=Release --build missing -if build + cmake . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" + cmake --build build --config Release + displayName: Build + - script: | + cd python + pip install wheel setuptools + python setup.py bdist_wheel + displayName: Build Wheels + - task: TwineAuthenticate@1 + displayName: "Twine Authenticate (test)" + inputs: + pythonUploadServiceConnection: pypitest + condition: and(succeeded(), contains(variables['Build.SourceBranch'], 'test_deploy')) + - task: TwineAuthenticate@1 + displayName: "Twine Authenticate (live)" + inputs: + pythonUploadServiceConnection: pypilive + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + - script: | + pip install twine + cd python + python -m twine upload -r "griddly" --config-file $(PYPIRC_PATH) dist/*.whl + displayName: Deploy To PyPi + + - job: MacOS + timeoutInMinutes: 60 + pool: + vmImage: "macOS-latest" + steps: + - task: InstallSSHKey@0 + inputs: + knownHostsEntry: $(griddly_known_hosts_entry) + sshPublicKey: $(griddly_azure_public_key) + sshPassphrase: $(griddly_azure_passphrase) + sshKeySecureFile: griddly + - checkout: self + submodules: true + persistCredentials: true + - task: UsePythonVersion@0 + inputs: + versionSpec: 3.8 + addToPath: true + displayName: Set Python 3.8 + name: py + - bash: | + set -ex + pip install conan==1.59.0 + continueOnError: false + displayName: Install Conan + - bash: | + set -ex + conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build missing -if build + cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" + cmake --build build --config Release + displayName: Build + - bash: | + GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* GTEST_OUTPUT=xml:test-report.xml ctest . + continueOnError: true + displayName: Test + - task: PublishTestResults@2 + inputs: + failTaskOnFailedTests: true + testResultsFormat: JUnit # Options: JUnit, NUnit, VSTest, xUnit, cTest + testResultsFiles: test-report.xml + testRunTitle: MacOS Tests + displayName: Publish C++ Test Results + - script: | + set -ex + cd python + pip install poetry + poetry install + poetry run pip install pytest-azurepipelines + poetry run python -m pytest + displayName: Run and Publish Python Tests + + - job: PyPiDeployMacOS + condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), contains(variables['Build.SourceBranch'], 'test_deploy'))) + dependsOn: MacOS + timeoutInMinutes: 60 + pool: + vmImage: "macOS-latest" + strategy: + matrix: + MacOS_p38: + PythonVersion: 3.8 + MacOS_p39: + PythonVersion: 3.9 + MacOS_p310: + PythonVersion: 3.10 + MacOS_p311: + PythonVersion: 3.11 + steps: + - task: InstallSSHKey@0 + inputs: + knownHostsEntry: $(griddly_known_hosts_entry) + sshPublicKey: $(griddly_azure_public_key) + sshPassphrase: $(griddly_azure_passphrase) + sshKeySecureFile: griddly + - checkout: self + submodules: true + persistCredentials: true + - task: UsePythonVersion@0 + inputs: + versionSpec: $(PythonVersion) + addToPath: true + displayName: Set Python $(PythonVersion) + name: py + - bash: | + set -ex + pip install conan==1.59.0 + continueOnError: false + displayName: Install Conan + - bash: | + set -ex + conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build missing -if build + cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING="$(py.pythonLocation)" + cmake --build build --config Release + displayName: Build + - script: | + cd python + pip install wheel setuptools + python setup.py bdist_wheel + displayName: Build Wheels + - task: TwineAuthenticate@1 + displayName: "Twine Authenticate (test)" + inputs: + pythonUploadServiceConnection: pypitest + condition: and(succeeded(), contains(variables['Build.SourceBranch'], 'test_deploy')) + - task: TwineAuthenticate@1 + displayName: "Twine Authenticate (live)" + inputs: + pythonUploadServiceConnection: pypilive + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + - script: | + pip install twine + cd python + python -m twine upload -r "griddly" --config-file $(PYPIRC_PATH) dist/*.whl + displayName: Deploy To PyPi From f684ebd4769d882981686ce5ec963d5134f251da Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sat, 3 Jun 2023 16:45:30 +0100 Subject: [PATCH 06/45] trying out some more build steps --- .github/workflows/build-test-publish.yml | 31 ++++++++++++++++++++---- .gitmodules | 24 ------------------ 2 files changed, 26 insertions(+), 29 deletions(-) delete mode 100644 .gitmodules diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index f6318863..61614735 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -30,13 +30,20 @@ jobs: # ./configure.sh - build-linux: + build-binaries-linux: runs-on: ubuntu-latest steps: # Checkout the repository - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + + + # Set python version + - name: Set up Python 3.8 + uses: actions/setup-python@v3 + with: + python-version: 3.8 # Install Conan - name: Install Conan @@ -44,6 +51,7 @@ jobs: sudo apt-get update sudo apt-get install python3-pip pip3 install conan==1.59.0 + pip3 install poetry # Configure conan for release build - name: Configure Build @@ -57,10 +65,23 @@ jobs: cmake --build build --config Release # Run the tests - - name: tests + - name: Test run: | - GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* GTEST_OUTPUT=xml:test-report.xml ctest . - + GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* GTEST_OUTPUT=xml:test-report.xml ctest --test-dir build + + # Setup python environment + - name: Poetry install + run: | + cd python + poetry install + + # Run python tests + - name: Python tests + run: | + cd python + poetry run pytest . + + # - name: Python package # # if the commit is on the master branch or contains the string "release-test" diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 2d56e6e3..00000000 --- a/.gitmodules +++ /dev/null @@ -1,24 +0,0 @@ -[submodule "libs/pybind11"] - path = libs/pybind11 - url = https://github.com/pybind/pybind11.git - ignore = dirty -[submodule "libs/yaml-cpp"] - path = libs/yaml-cpp - url = https://github.com/jbeder/yaml-cpp.git - ignore = dirty -[submodule "libs/glm"] - path = libs/glm - url = https://github.com/g-truc/glm.git - ignore = dirty -[submodule "python/examples/experiments/conditional-action-trees"] - path=python/examples/experiments/conditional-action-trees - url=https://github.com/Bam4d/conditional-action-trees - ignore = dirty -[submodule "python/examples/experiments/rts-self-play"] - path = python/examples/experiments/rts-self-play - url = https://github.com/Bam4d/rts-self-play - ignore = dirty -[submodule "python/examples/experiments/autoregressive-cats"] - path = python/examples/experiments/autoregressive-cats - url = https://github.com/Bam4d/autoregressive-cats - ignore = dirty From 591e3d607a997fb22dbdb302eb52cd324c906e91 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sat, 3 Jun 2023 17:00:41 +0100 Subject: [PATCH 07/45] building multiple python versions and seeing vulkan works --- .github/workflows/build-test-publish.yml | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 61614735..9a5b42bb 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -32,6 +32,9 @@ jobs: build-binaries-linux: runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.7, 3.8, 3.9, 3.10, 3.11] steps: # Checkout the repository @@ -40,13 +43,13 @@ jobs: # Set python version - - name: Set up Python 3.8 + - name: Set up Python uses: actions/setup-python@v3 with: - python-version: 3.8 + python-version: ${{ matrix.python-version }} # Install Conan - - name: Install Conan + - name: Install Build Dependencies run: | sudo apt-get update sudo apt-get install python3-pip @@ -64,6 +67,17 @@ jobs: run: | cmake --build build --config Release + - name: Prepare Vulkan SDK + uses: humbletim/setup-vulkan-sdk@v1.2.0 + with: + vulkan-query-version: 1.3.224.1 + vulkan-components: Vulkan-Loader + vulkan-use-cache: true + + - name: Vulkan + run: | + vulkaninfo + # Run the tests - name: Test run: | From 8838c473e9f52a8e4eec376108b534bf756b178f Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sat, 3 Jun 2023 17:00:51 +0100 Subject: [PATCH 08/45] maybe the tests will work too --- .github/workflows/build-test-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 9a5b42bb..4650ca13 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -81,7 +81,7 @@ jobs: # Run the tests - name: Test run: | - GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* GTEST_OUTPUT=xml:test-report.xml ctest --test-dir build + ctest --test-dir build # Setup python environment - name: Poetry install From 99100533cb78cf76ae67e8bb7f71906c97c63243 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sat, 3 Jun 2023 17:02:56 +0100 Subject: [PATCH 09/45] stupid python versions --- .github/workflows/build-test-publish.yml | 129 +++++++++++------------ 1 file changed, 59 insertions(+), 70 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 4650ca13..05413f1a 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -2,13 +2,11 @@ name: Griddly Build and Deploy on: push: - branches: [ "master", "develop" ] + branches: ["master", "develop"] # Run on all pull requests pull_request: jobs: - - # build-wasm: # runs-on: ubuntu-latest @@ -27,88 +25,79 @@ jobs: # # Configure conan for release build # - name: Configure Build # run: | - # ./configure.sh - + # ./configure.sh build-binaries-linux: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9, 3.10, 3.11] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - # Checkout the repository - - name: Checkout - uses: actions/checkout@v3 - - - # Set python version - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - # Install Conan - - name: Install Build Dependencies - run: | - sudo apt-get update - sudo apt-get install python3-pip - pip3 install conan==1.59.0 - pip3 install poetry - - # Configure conan for release build - - name: Configure Build - run: | - ./configure.sh --build-type=Release - cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake - - # Build the project - - name: Build - run: | - cmake --build build --config Release - - - name: Prepare Vulkan SDK - uses: humbletim/setup-vulkan-sdk@v1.2.0 - with: - vulkan-query-version: 1.3.224.1 - vulkan-components: Vulkan-Loader - vulkan-use-cache: true - - - name: Vulkan - run: | - vulkaninfo - - # Run the tests - - name: Test - run: | - ctest --test-dir build - - # Setup python environment - - name: Poetry install - run: | - cd python - poetry install - - # Run python tests - - name: Python tests - run: | - cd python - poetry run pytest . - - + # Checkout the repository + - name: Checkout + uses: actions/checkout@v3 + + # Set python version + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + # Install Conan + - name: Install Build Dependencies + run: | + sudo apt-get update + sudo apt-get install python3-pip + pip3 install conan==1.59.0 + pip3 install poetry + + # Configure conan for release build + - name: Configure Build + run: | + ./configure.sh --build-type=Release + cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake + + # Build the project + - name: Build + run: | + cmake --build build --config Release + + - name: Prepare Vulkan SDK + uses: humbletim/setup-vulkan-sdk@v1.2.0 + with: + vulkan-query-version: 1.3.224.1 + vulkan-components: Vulkan-Loader + vulkan-use-cache: true + + - name: Vulkan + run: | + vulkaninfo + + # Run the tests + - name: Test + run: | + ctest --test-dir build + + # Setup python environment + - name: Poetry install + run: | + cd python + poetry install + + # Run python tests + - name: Python tests + run: | + cd python + poetry run pytest . # - name: Python package # # if the commit is on the master branch or contains the string "release-test" # if: github.ref == 'refs/heads/master' || contains(github.ref, 'release-test') # run: | - - # build-macos: # runs-on: macos-latest # build-windows: # runs-on: windows-latest - - - From 43222ad22d662878592ba1be2071c10395011c74 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sat, 3 Jun 2023 17:15:06 +0100 Subject: [PATCH 10/45] just ignore the vulkaninfo step and try to run tests with vulkan --- .github/workflows/build-test-publish.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 05413f1a..f6d0b8ce 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -70,10 +70,6 @@ jobs: vulkan-components: Vulkan-Loader vulkan-use-cache: true - - name: Vulkan - run: | - vulkaninfo - # Run the tests - name: Test run: | From 5d6b34711dd0517d6f4d30fab73e237d5084af27 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sat, 3 Jun 2023 17:26:45 +0100 Subject: [PATCH 11/45] remove silly vulkan --- .github/workflows/build-test-publish.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index f6d0b8ce..74098c48 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -63,17 +63,10 @@ jobs: run: | cmake --build build --config Release - - name: Prepare Vulkan SDK - uses: humbletim/setup-vulkan-sdk@v1.2.0 - with: - vulkan-query-version: 1.3.224.1 - vulkan-components: Vulkan-Loader - vulkan-use-cache: true - # Run the tests - name: Test run: | - ctest --test-dir build + GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* GTEST_OUTPUT=xml:test-report.xml ctest --test-dir build # Setup python environment - name: Poetry install From cc82e0784d156a92e6a708a25ed7ef1a2fe426e3 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sat, 3 Jun 2023 17:40:21 +0100 Subject: [PATCH 12/45] dont use python 3.7 --- .github/workflows/build-test-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 74098c48..4aef0c04 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: # Checkout the repository From 38e91622ae8d5219415e0a067751f2558aa66be9 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sat, 3 Jun 2023 21:41:10 +0100 Subject: [PATCH 13/45] 3.11 apparently doesnt work with tensorflow --- .github/workflows/build-test-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 4aef0c04..82635ae3 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10"] steps: # Checkout the repository From 596a94afc946c5dd1b421d8c0ff102bb6dec5513 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sun, 4 Jun 2023 15:57:54 +0100 Subject: [PATCH 14/45] fixing some wierd test dependency that didnt like manylinux --- .github/workflows/build-test-publish.yml | 90 ++++++++++++++++--- .../manylinux/{build-wheels.sh => build.sh} | 9 +- src/Griddly/Core/Observers/SpriteObserver.hpp | 4 +- .../Core/Observers/VectorObserverTest.cpp | 2 +- .../Core/Observers/VulkanObserverTest.cpp | 29 ++++++ .../Core/Observers/VulkanObserverTest.hpp | 22 +---- 6 files changed, 114 insertions(+), 42 deletions(-) rename python/manylinux/{build-wheels.sh => build.sh} (63%) create mode 100644 tests/src/Griddly/Core/Observers/VulkanObserverTest.cpp diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 82635ae3..60744860 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -1,4 +1,4 @@ -name: Griddly Build and Deploy +name: Griddly Build on: push: @@ -6,6 +6,7 @@ on: # Run on all pull requests pull_request: + jobs: # build-wasm: # runs-on: ubuntu-latest @@ -27,11 +28,33 @@ jobs: # run: | # ./configure.sh - build-binaries-linux: + build-linux: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + manylinux_config: + - python-version: "3.8" + py-bin: cp38-cp38 + platform: manylinux_2_28_x86_64 + - python-version: "3.9" + py-bin: cp39-cp39 + platform: manylinux_2_28_x86_64 + - python-version: "3.10" + py-bin: cp310-cp310 + platform: manylinux_2_28_x86_64 + - python-version: "3.11" + py-bin: cp311-cp311 + platform: manylinux2014_x86_64 + - python-version: "3.8" + py-bin: cp38-cp38 + platform: manylinux2014_x86_64 + - python-version: "3.9" + py-bin: cp39-cp39 + platform: manylinux2014_x86_64 + - python-version: "3.10" + py-bin: cp310-cp310 + platform: manylinux2014_x86_64 + steps: # Checkout the repository @@ -53,15 +76,12 @@ jobs: pip3 install poetry # Configure conan for release build - - name: Configure Build - run: | - ./configure.sh --build-type=Release - cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake - - # Build the project - name: Build run: | - cmake --build build --config Release + # ./configure.sh --build-type=Release + # cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake + # cmake --build build --config Release + docker run # Run the tests - name: Test @@ -80,10 +100,52 @@ jobs: cd python poetry run pytest . - # - name: Python package - # # if the commit is on the master branch or contains the string "release-test" - # if: github.ref == 'refs/heads/master' || contains(github.ref, 'release-test') - # run: | + - name: Python Package + if: github.ref == 'refs/heads/master' || contains(github.ref, 'release-test') + run: | + poetry build --format=wheel + + # Upload the built wheels + - name: Upload wheels + if: github.ref == 'refs/heads/master' || contains(github.ref, 'release-test') + uses: actions/upload-artifact@v2 + with: + name: wheels + path: python/dist/*.whl + + publish-wheels: + needs: build-linux + runs-on: ubuntu-latest + + steps: + # Checkout the repository + - name: Checkout + uses: actions/checkout@v3 + + - name: Download wheels + uses: actions/download-artifact@v2 + with: + name: wheels + path: python/dist + + - name: Install Build Dependencies + run: | + sudo apt-get update + sudo apt-get install python3-pip + pip3 install poetry + + - name: Pypi upload (test) + if: contains(github.ref, 'release-test') + run: | + poetry config repositories.test-pypi https://test.pypi.org/legacy/ + poetry config pypi-token.test-pypi ${{ secrets.PYPI_TEST_TOKEN }} + poetry publish -r test-pypi + + # - name: Pypi upload (prod) + # if: github.ref == 'refs/heads/master' + # run: | + # poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} + # poetry publish # build-macos: # runs-on: macos-latest diff --git a/python/manylinux/build-wheels.sh b/python/manylinux/build.sh similarity index 63% rename from python/manylinux/build-wheels.sh rename to python/manylinux/build.sh index 6c61e414..411ceedb 100755 --- a/python/manylinux/build-wheels.sh +++ b/python/manylinux/build.sh @@ -10,7 +10,7 @@ yum install -y ninja-build export CONAN_SYSREQUIRES_SUDO=0 # Get Dependencies -/opt/python/$PYBIN/bin/pip install cmake setuptools wheel twine conan==1.59.0 +/opt/python/$PYBIN/bin/pip install cmake conan==1.59.0 export PATH=$PATH:/opt/python/$PYBIN/bin @@ -19,11 +19,8 @@ echo "Conan Build Finished" # # Cmake Build Griddly cd /opt/Griddly -conan install deps/conanfile.txt --profile default --profile deps/build.profile -s build_type=Release --build=* -if build_manylinux +conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build=* -if build_manylinux /opt/python/$PYBIN/bin/cmake . -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING=/opt/python/$PYBIN -DPYTHON_EXECUTABLE:FILEPATH=/opt/python/$PYBIN/bin/python -S /opt/Griddly -B /opt/Griddly/build_manylinux -/opt/python/$PYBIN/bin/cmake --build /opt/Griddly/build_manylinux --target python_griddly +/opt/python/$PYBIN/bin/cmake --build /opt/Griddly/build_manylinux --config Release -# # # Create Wheel -cd python -/opt/python/$PYBIN/bin/python setup.py bdist_wheel --plat $PLAT diff --git a/src/Griddly/Core/Observers/SpriteObserver.hpp b/src/Griddly/Core/Observers/SpriteObserver.hpp index ec9532a0..e6c8ec5e 100644 --- a/src/Griddly/Core/Observers/SpriteObserver.hpp +++ b/src/Griddly/Core/Observers/SpriteObserver.hpp @@ -19,14 +19,14 @@ enum class TilingMode { }; struct SpriteDefinition { - std::vector images; + std::vector images{}; TilingMode tilingMode = TilingMode::NONE; glm::vec2 offset = {0, 0}; float scale = 1.0; }; struct SpriteObserverConfig : public VulkanGridObserverConfig { - std::map spriteDefinitions; + std::map spriteDefinitions{}; }; class SpriteObserver : public VulkanGridObserver { diff --git a/tests/src/Griddly/Core/Observers/VectorObserverTest.cpp b/tests/src/Griddly/Core/Observers/VectorObserverTest.cpp index 91a4de81..43a02f5b 100644 --- a/tests/src/Griddly/Core/Observers/VectorObserverTest.cpp +++ b/tests/src/Griddly/Core/Observers/VectorObserverTest.cpp @@ -480,7 +480,7 @@ TEST(VectorObserverTest, defaultObserver_global_variables) { 0, 0, false, false}; - + config.globalVariableMapping = {"lightingR", "lightingB"}; uint8_t expectedData[5][5][6] = { diff --git a/tests/src/Griddly/Core/Observers/VulkanObserverTest.cpp b/tests/src/Griddly/Core/Observers/VulkanObserverTest.cpp new file mode 100644 index 00000000..0bb58b5f --- /dev/null +++ b/tests/src/Griddly/Core/Observers/VulkanObserverTest.cpp @@ -0,0 +1,29 @@ +#define STB_IMAGE_STATIC +#define STB_IMAGE_WRITE_STATIC +#define STB_IMAGE_IMPLEMENTATION +#define STB_IMAGE_WRITE_IMPLEMENTATION +#include +#include +#include + +namespace griddly { + +std::unique_ptr loadExpectedImage(std::string filename) { + int width, height, channels; + + stbi_uc* pixels = stbi_load(filename.c_str(), &width, &height, &channels, STBI_rgb_alpha); + + if (!pixels) { + throw std::runtime_error("Failed to load texture image."); + } + + std::unique_ptr spriteData(pixels); + + return std::move(spriteData); +} + +int write_image(std::string filename, uint8_t* imageData, int stride, int width, int height) { + return stbi_write_png(filename.c_str(), width, height, STBI_rgb_alpha, imageData, stride); +} + +} // namespace griddly \ No newline at end of file diff --git a/tests/src/Griddly/Core/Observers/VulkanObserverTest.hpp b/tests/src/Griddly/Core/Observers/VulkanObserverTest.hpp index d664b6b4..0edd0608 100644 --- a/tests/src/Griddly/Core/Observers/VulkanObserverTest.hpp +++ b/tests/src/Griddly/Core/Observers/VulkanObserverTest.hpp @@ -1,3 +1,4 @@ +#pragma once #define STB_IMAGE_STATIC #define STB_IMAGE_WRITE_STATIC #define STB_IMAGE_IMPLEMENTATION @@ -6,31 +7,14 @@ #include #include -#include - using ::testing::Return; namespace griddly { -inline std::unique_ptr loadExpectedImage(std::string filename) { - int width, height, channels; - - stbi_uc* pixels = stbi_load(filename.c_str(), &width, &height, &channels, STBI_rgb_alpha); - - if (!pixels) { - throw std::runtime_error("Failed to load texture image."); - } - - auto spriteSize = width * height * channels; - - std::unique_ptr spriteData(pixels); +std::unique_ptr loadExpectedImage(std::string filename); - return std::move(spriteData); -} +int write_image(std::string filename, uint8_t* imageData, int stride, int width, int height); -inline int write_image(std::string filename, uint8_t* imageData, int stride, int width, int height) { - return stbi_write_png(filename.c_str(), width, height, STBI_rgb_alpha, imageData, stride); -} MATCHER_P3(ObservationResultMatcher, shape, strides, imageData, "") { for (int x = 0; x < shape[1]; x++) { From 62dcb204b930a6d708083e5894b5a51b8d7fbef4 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sun, 4 Jun 2023 16:03:30 +0100 Subject: [PATCH 15/45] another attempt --- .github/workflows/build-test-publish.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 60744860..6737cdf7 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -55,6 +55,11 @@ jobs: py-bin: cp310-cp310 platform: manylinux2014_x86_64 + env: + GRIDDLY_ROOT: ${{ github.workspace }} + PYBIN: ${{ matrix.py-bin }} + PYVERSION: ${{ matrix.python-version }} + PLATFORM: ${{ matrix.platform }} steps: # Checkout the repository @@ -81,7 +86,8 @@ jobs: # ./configure.sh --build-type=Release # cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake # cmake --build build --config Release - docker run + + ./python/manylinux/manylinux-build.sh # Run the tests - name: Test From cf0d7c31af76138cfeeee45cec3b0407710d6529 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sun, 4 Jun 2023 16:05:40 +0100 Subject: [PATCH 16/45] trying to fix --- .github/workflows/build-test-publish.yml | 2 +- python/manylinux/manylinux-build.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 6737cdf7..019b97ab 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -92,7 +92,7 @@ jobs: # Run the tests - name: Test run: | - GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* GTEST_OUTPUT=xml:test-report.xml ctest --test-dir build + GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* GTEST_OUTPUT=xml:test-report.xml ctest --test-dir build_manylinux # Setup python environment - name: Poetry install diff --git a/python/manylinux/manylinux-build.sh b/python/manylinux/manylinux-build.sh index 97462702..d87a5917 100755 --- a/python/manylinux/manylinux-build.sh +++ b/python/manylinux/manylinux-build.sh @@ -5,6 +5,6 @@ set -ex echo $PYVERSION echo $PYBIN echo $GRIDDLY_ROOT -echo $PLAT +echo $PLATFORM -docker run -e PYVERSION=$PYVERSION -e PYBIN=$PYBIN -e PLAT=$PLAT -v "$GRIDDLY_ROOT:/opt/Griddly" quay.io/pypa/$PLAT /opt/Griddly/python/manylinux/build-wheels.sh \ No newline at end of file +docker run -e PYVERSION=$PYVERSION -e PYBIN=$PYBIN -e PLAT=$PLATFORM -v "$GRIDDLY_ROOT:/opt/Griddly" quay.io/pypa/$PLATFORM /opt/Griddly/python/manylinux/build-wheels.sh \ No newline at end of file From 69220fa8a9c88128f908b481d8994204697dd759 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sun, 4 Jun 2023 16:11:48 +0100 Subject: [PATCH 17/45] more fiddling --- .github/workflows/build-test-publish.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 019b97ab..20c2cec7 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -57,9 +57,9 @@ jobs: env: GRIDDLY_ROOT: ${{ github.workspace }} - PYBIN: ${{ matrix.py-bin }} - PYVERSION: ${{ matrix.python-version }} - PLATFORM: ${{ matrix.platform }} + PYBIN: ${{ matrix.manylinux_config.py-bin }} + PYVERSION: ${{ matrix.manylinux_config.python-version }} + PLATFORM: ${{ matrix.manylinux_config.platform }} steps: # Checkout the repository @@ -70,7 +70,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.manylinux_config.python-version }} # Install Conan - name: Install Build Dependencies From d3e3d8daeb56dd33e0df1e90e81766fae7867c1c Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sun, 4 Jun 2023 16:28:11 +0100 Subject: [PATCH 18/45] build not build-wheels --- python/manylinux/manylinux-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/manylinux/manylinux-build.sh b/python/manylinux/manylinux-build.sh index d87a5917..79e2492a 100755 --- a/python/manylinux/manylinux-build.sh +++ b/python/manylinux/manylinux-build.sh @@ -7,4 +7,4 @@ echo $PYBIN echo $GRIDDLY_ROOT echo $PLATFORM -docker run -e PYVERSION=$PYVERSION -e PYBIN=$PYBIN -e PLAT=$PLATFORM -v "$GRIDDLY_ROOT:/opt/Griddly" quay.io/pypa/$PLATFORM /opt/Griddly/python/manylinux/build-wheels.sh \ No newline at end of file +docker run -e PYVERSION=$PYVERSION -e PYBIN=$PYBIN -e PLAT=$PLATFORM -v "$GRIDDLY_ROOT:/opt/Griddly" quay.io/pypa/$PLATFORM /opt/Griddly/python/manylinux/build.sh \ No newline at end of file From 4b33bc342db01c82bfee9d88cafbbac564709c39 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sun, 4 Jun 2023 17:08:23 +0100 Subject: [PATCH 19/45] try to run the tests in the docker container too --- .github/workflows/build-test-publish.yml | 2 +- python/manylinux/manylinux-build.sh | 8 -------- python/manylinux/manylinux-test.sh | 6 ++++++ python/manylinux/test.sh | 3 +++ 4 files changed, 10 insertions(+), 9 deletions(-) create mode 100644 python/manylinux/manylinux-test.sh create mode 100644 python/manylinux/test.sh diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 20c2cec7..b4da8389 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -92,7 +92,7 @@ jobs: # Run the tests - name: Test run: | - GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* GTEST_OUTPUT=xml:test-report.xml ctest --test-dir build_manylinux + ./python/manylinux/manylinux-test.sh # Setup python environment - name: Poetry install diff --git a/python/manylinux/manylinux-build.sh b/python/manylinux/manylinux-build.sh index 79e2492a..87958f99 100755 --- a/python/manylinux/manylinux-build.sh +++ b/python/manylinux/manylinux-build.sh @@ -1,10 +1,2 @@ #!/bin/bash - -set -ex - -echo $PYVERSION -echo $PYBIN -echo $GRIDDLY_ROOT -echo $PLATFORM - docker run -e PYVERSION=$PYVERSION -e PYBIN=$PYBIN -e PLAT=$PLATFORM -v "$GRIDDLY_ROOT:/opt/Griddly" quay.io/pypa/$PLATFORM /opt/Griddly/python/manylinux/build.sh \ No newline at end of file diff --git a/python/manylinux/manylinux-test.sh b/python/manylinux/manylinux-test.sh new file mode 100644 index 00000000..dcc5e037 --- /dev/null +++ b/python/manylinux/manylinux-test.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -ex +set -o pipefail + +docker run -e PYVERSION=$PYVERSION -e PYBIN=$PYBIN -e PLAT=$PLATFORM -e GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* -v "$GRIDDLY_ROOT:/opt/Griddly" quay.io/pypa/$PLATFORM /opt/Griddly/python/manylinux/test.sh \ No newline at end of file diff --git a/python/manylinux/test.sh b/python/manylinux/test.sh new file mode 100644 index 00000000..c972e02f --- /dev/null +++ b/python/manylinux/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +cd /opt/Griddly +ctest --test-dir /opt/Griddly/build_manylinux \ No newline at end of file From b79c95568c219fe1edbde84d68652747ee8d4d8a Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sun, 4 Jun 2023 17:35:05 +0100 Subject: [PATCH 20/45] these should be executable --- python/manylinux/manylinux-test.sh | 0 python/manylinux/test.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 python/manylinux/manylinux-test.sh mode change 100644 => 100755 python/manylinux/test.sh diff --git a/python/manylinux/manylinux-test.sh b/python/manylinux/manylinux-test.sh old mode 100644 new mode 100755 diff --git a/python/manylinux/test.sh b/python/manylinux/test.sh old mode 100644 new mode 100755 From e48036839d23ceb9b73059bdb9a531ed0487a7cf Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sun, 4 Jun 2023 20:04:33 +0100 Subject: [PATCH 21/45] remove 3.11 for now because tensorflow doesn't support it --- .github/workflows/build-test-publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index b4da8389..4199226f 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -42,9 +42,9 @@ jobs: - python-version: "3.10" py-bin: cp310-cp310 platform: manylinux_2_28_x86_64 - - python-version: "3.11" - py-bin: cp311-cp311 - platform: manylinux2014_x86_64 + # - python-version: "3.11" + # py-bin: cp311-cp311 + # platform: manylinux2014_x86_64 - python-version: "3.8" py-bin: cp38-cp38 platform: manylinux2014_x86_64 From c518a13ad9dabaee947cca68c9a35c261174cafe Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sun, 4 Jun 2023 21:12:13 +0100 Subject: [PATCH 22/45] only do the publish if we are on master or release-test --- .github/workflows/build-test-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 4199226f..36b1710e 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -120,6 +120,7 @@ jobs: path: python/dist/*.whl publish-wheels: + if: github.ref == 'refs/heads/master' || contains(github.ref, 'release-test') needs: build-linux runs-on: ubuntu-latest From b1c6d26129055e93813b1147a90971f543f816bd Mon Sep 17 00:00:00 2001 From: Bam4d Date: Sun, 4 Jun 2023 22:04:28 +0100 Subject: [PATCH 23/45] try and use .head_ref for PR --- .github/workflows/build-test-publish.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 36b1710e..63d60a27 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -107,20 +107,20 @@ jobs: poetry run pytest . - name: Python Package - if: github.ref == 'refs/heads/master' || contains(github.ref, 'release-test') + if: github.ref == 'refs/heads/master' || contains(github.head_ref, 'release-test') run: | poetry build --format=wheel # Upload the built wheels - name: Upload wheels - if: github.ref == 'refs/heads/master' || contains(github.ref, 'release-test') + if: github.ref == 'refs/heads/master' || contains(github.head_ref, 'release-test') uses: actions/upload-artifact@v2 with: name: wheels path: python/dist/*.whl publish-wheels: - if: github.ref == 'refs/heads/master' || contains(github.ref, 'release-test') + if: github.ref == 'refs/heads/master' || contains(github.head_ref, 'release-test') needs: build-linux runs-on: ubuntu-latest @@ -142,7 +142,7 @@ jobs: pip3 install poetry - name: Pypi upload (test) - if: contains(github.ref, 'release-test') + if: contains(github.head_ref, 'release-test') run: | poetry config repositories.test-pypi https://test.pypi.org/legacy/ poetry config pypi-token.test-pypi ${{ secrets.PYPI_TEST_TOKEN }} From 409310e7790ba55d4d407f826c922ea697773cc1 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Mon, 5 Jun 2023 12:57:41 +0100 Subject: [PATCH 24/45] fixing python steps --- .github/workflows/build-test-publish.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 63d60a27..a5e3db12 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -109,10 +109,11 @@ jobs: - name: Python Package if: github.ref == 'refs/heads/master' || contains(github.head_ref, 'release-test') run: | + cd python poetry build --format=wheel # Upload the built wheels - - name: Upload wheels + - name: Upload wheel artifacts if: github.ref == 'refs/heads/master' || contains(github.head_ref, 'release-test') uses: actions/upload-artifact@v2 with: @@ -129,7 +130,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Download wheels + - name: Download wheel artifacts uses: actions/download-artifact@v2 with: name: wheels From 73d47e46fc6add4ae3af167080dddf81a85b1404 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Mon, 12 Jun 2023 20:12:31 +0100 Subject: [PATCH 25/45] another attempt at pushing --- python/tools/package_resources.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/tools/package_resources.py b/python/tools/package_resources.py index f277f00a..26fdd0d1 100644 --- a/python/tools/package_resources.py +++ b/python/tools/package_resources.py @@ -47,6 +47,7 @@ def build(setup_kwargs): griddly_lib_dir.mkdir(parents=True, exist_ok=True) for lib in libs_to_copy: + print(f" {lib}") shutil.copy(lib, griddly_lib_dir) # Destination for resources From 238126a8d05461e7cb67f193d43f4c11ec66c104 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Mon, 12 Jun 2023 20:44:07 +0100 Subject: [PATCH 26/45] again try to build each wheelyboi --- .github/workflows/build-test-publish.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index a5e3db12..def382b1 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -115,9 +115,8 @@ jobs: # Upload the built wheels - name: Upload wheel artifacts if: github.ref == 'refs/heads/master' || contains(github.head_ref, 'release-test') - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: - name: wheels path: python/dist/*.whl publish-wheels: @@ -136,6 +135,10 @@ jobs: name: wheels path: python/dist + - name: List wheels + run: | + ls -lah python/dist + - name: Install Build Dependencies run: | sudo apt-get update @@ -145,6 +148,7 @@ jobs: - name: Pypi upload (test) if: contains(github.head_ref, 'release-test') run: | + cd python poetry config repositories.test-pypi https://test.pypi.org/legacy/ poetry config pypi-token.test-pypi ${{ secrets.PYPI_TEST_TOKEN }} poetry publish -r test-pypi From 4f7dba868ac7b58ccd633fa786da63ed4c8423bf Mon Sep 17 00:00:00 2001 From: Bam4d Date: Mon, 12 Jun 2023 21:31:54 +0100 Subject: [PATCH 27/45] download all artifacts not just wheels which no longer exists --- .github/workflows/build-test-publish.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index def382b1..464a3717 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -130,9 +130,8 @@ jobs: uses: actions/checkout@v3 - name: Download wheel artifacts - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: - name: wheels path: python/dist - name: List wheels From 3467950733463b75c922459f10fc25ff0f68542c Mon Sep 17 00:00:00 2001 From: Bam4d Date: Wed, 14 Jun 2023 20:30:26 +0100 Subject: [PATCH 28/45] testing wheely bits --- .github/workflows/build-test-publish.yml | 36 ++++++++++++------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 464a3717..8919e4de 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -81,18 +81,18 @@ jobs: pip3 install poetry # Configure conan for release build - - name: Build - run: | - # ./configure.sh --build-type=Release - # cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake - # cmake --build build --config Release + # - name: Build + # run: | + # # ./configure.sh --build-type=Release + # # cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake + # # cmake --build build --config Release - ./python/manylinux/manylinux-build.sh + # ./python/manylinux/manylinux-build.sh - # Run the tests - - name: Test - run: | - ./python/manylinux/manylinux-test.sh + # # Run the tests + # - name: Test + # run: | + # ./python/manylinux/manylinux-test.sh # Setup python environment - name: Poetry install @@ -132,7 +132,7 @@ jobs: - name: Download wheel artifacts uses: actions/download-artifact@v3 with: - path: python/dist + path: python/dist/**/*.whl - name: List wheels run: | @@ -144,13 +144,13 @@ jobs: sudo apt-get install python3-pip pip3 install poetry - - name: Pypi upload (test) - if: contains(github.head_ref, 'release-test') - run: | - cd python - poetry config repositories.test-pypi https://test.pypi.org/legacy/ - poetry config pypi-token.test-pypi ${{ secrets.PYPI_TEST_TOKEN }} - poetry publish -r test-pypi + # - name: Pypi upload (test) + # if: contains(github.head_ref, 'release-test') + # run: | + # cd python + # poetry config repositories.test-pypi https://test.pypi.org/legacy/ + # poetry config pypi-token.test-pypi ${{ secrets.PYPI_TEST_TOKEN }} + # poetry publish -r test-pypi # - name: Pypi upload (prod) # if: github.ref == 'refs/heads/master' From 31035a2e1abf2c260593a8f924594998404a99a9 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Wed, 14 Jun 2023 20:38:16 +0100 Subject: [PATCH 29/45] more fiddle --- .github/workflows/build-test-publish.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 8919e4de..a2751e76 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -81,18 +81,14 @@ jobs: pip3 install poetry # Configure conan for release build - # - name: Build - # run: | - # # ./configure.sh --build-type=Release - # # cmake . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake - # # cmake --build build --config Release - - # ./python/manylinux/manylinux-build.sh + - name: Build + run: | + ./python/manylinux/manylinux-build.sh - # # Run the tests - # - name: Test - # run: | - # ./python/manylinux/manylinux-test.sh + # Run the tests + - name: Test + run: | + ./python/manylinux/manylinux-test.sh # Setup python environment - name: Poetry install @@ -138,6 +134,10 @@ jobs: run: | ls -lah python/dist + - name: List wheels 2 + run: | + ls -lah python/dist/artifacts + - name: Install Build Dependencies run: | sudo apt-get update From 02d6207c19e953b2303b90630e4186a569024512 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Wed, 14 Jun 2023 20:51:46 +0100 Subject: [PATCH 30/45] dont need to install ninja because conan does that --- python/manylinux/build.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/python/manylinux/build.sh b/python/manylinux/build.sh index 411ceedb..24d474b9 100755 --- a/python/manylinux/build.sh +++ b/python/manylinux/build.sh @@ -3,10 +3,6 @@ set -ex set -o pipefail - -# Download vulkan libs -yum install -y ninja-build - export CONAN_SYSREQUIRES_SUDO=0 # Get Dependencies @@ -14,7 +10,6 @@ export CONAN_SYSREQUIRES_SUDO=0 export PATH=$PATH:/opt/python/$PYBIN/bin - echo "Conan Build Finished" # # Cmake Build Griddly From 43c1175fcb198dec0bdd336f110f175e522a4263 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Wed, 14 Jun 2023 21:33:18 +0100 Subject: [PATCH 31/45] fixing download --- .github/workflows/build-test-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index a2751e76..69b704a2 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -128,7 +128,7 @@ jobs: - name: Download wheel artifacts uses: actions/download-artifact@v3 with: - path: python/dist/**/*.whl + path: python/dist - name: List wheels run: | From 36d272cd176d37ef1244274d056015b4be91106c Mon Sep 17 00:00:00 2001 From: Bam4d Date: Wed, 14 Jun 2023 22:11:18 +0100 Subject: [PATCH 32/45] seriously though --- .github/workflows/build-test-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 69b704a2..731c506e 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -136,7 +136,7 @@ jobs: - name: List wheels 2 run: | - ls -lah python/dist/artifacts + ls -lah python/dist/artifact - name: Install Build Dependencies run: | From ca96f8edaa6a618b8bc6e5924332aacc310a5a15 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Thu, 15 Jun 2023 18:23:19 +0100 Subject: [PATCH 33/45] trying to understand why the wheels are wrong --- .github/workflows/build-test-publish.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 731c506e..35d9633c 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -33,15 +33,15 @@ jobs: strategy: matrix: manylinux_config: - - python-version: "3.8" - py-bin: cp38-cp38 - platform: manylinux_2_28_x86_64 - - python-version: "3.9" - py-bin: cp39-cp39 - platform: manylinux_2_28_x86_64 - - python-version: "3.10" - py-bin: cp310-cp310 - platform: manylinux_2_28_x86_64 + # - python-version: "3.8" + # py-bin: cp38-cp38 + # platform: manylinux_2_28_x86_64 + # - python-version: "3.9" + # py-bin: cp39-cp39 + # platform: manylinux_2_28_x86_64 + # - python-version: "3.10" + # py-bin: cp310-cp310 + # platform: manylinux_2_28_x86_64 # - python-version: "3.11" # py-bin: cp311-cp311 # platform: manylinux2014_x86_64 From 75f92d9dad457e46f086574eeb0a693c47e379e8 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Tue, 20 Jun 2023 20:53:40 +0100 Subject: [PATCH 34/45] try to use docker container for all steps --- .github/workflows/build-test-publish.yml | 53 +++++++++++++----------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 35d9633c..01108573 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -33,18 +33,18 @@ jobs: strategy: matrix: manylinux_config: - # - python-version: "3.8" - # py-bin: cp38-cp38 - # platform: manylinux_2_28_x86_64 - # - python-version: "3.9" - # py-bin: cp39-cp39 - # platform: manylinux_2_28_x86_64 - # - python-version: "3.10" - # py-bin: cp310-cp310 - # platform: manylinux_2_28_x86_64 + - python-version: "3.8" + py-bin: cp38-cp38 + platform: manylinux_2_28_x86_64 + - python-version: "3.9" + py-bin: cp39-cp39 + platform: manylinux_2_28_x86_64 + - python-version: "3.10" + py-bin: cp310-cp310 + platform: manylinux_2_28_x86_64 # - python-version: "3.11" # py-bin: cp311-cp311 - # platform: manylinux2014_x86_64 + # platform: manylinux_2_28_x86_64 - python-version: "3.8" py-bin: cp38-cp38 platform: manylinux2014_x86_64 @@ -54,12 +54,13 @@ jobs: - python-version: "3.10" py-bin: cp310-cp310 platform: manylinux2014_x86_64 - - env: - GRIDDLY_ROOT: ${{ github.workspace }} - PYBIN: ${{ matrix.manylinux_config.py-bin }} - PYVERSION: ${{ matrix.manylinux_config.python-version }} - PLATFORM: ${{ matrix.manylinux_config.platform }} + container: + image: quay.io/pypa/${{ matrix.manylinux_config.platform }} + env: + GRIDDLY_ROOT: ${{ github.workspace }} + PYBIN: ${{ matrix.manylinux_config.py-bin }} + PYVERSION: ${{ matrix.manylinux_config.python-version }} + PLATFORM: ${{ matrix.manylinux_config.platform }} steps: # Checkout the repository @@ -75,37 +76,39 @@ jobs: # Install Conan - name: Install Build Dependencies run: | - sudo apt-get update - sudo apt-get install python3-pip - pip3 install conan==1.59.0 - pip3 install poetry + /opt/python/$PYBIN/bin/pip install poetry cmake conan==1.59.0 + export PATH=$PATH:/opt/python/$PYBIN/bin # Configure conan for release build - name: Build run: | - ./python/manylinux/manylinux-build.sh + cd /opt/Griddly + conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build=* -if build_manylinux + /opt/python/$PYBIN/bin/cmake . -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING=/opt/python/$PYBIN -DPYTHON_EXECUTABLE:FILEPATH=/opt/python/$PYBIN/bin/python -S /opt/Griddly -B /opt/Griddly/build_manylinux + /opt/python/$PYBIN/bin/cmake --build /opt/Griddly/build_manylinux --config Release # Run the tests - name: Test run: | - ./python/manylinux/manylinux-test.sh + cd /opt/Griddly + ctest --test-dir /opt/Griddly/build_manylinux # Setup python environment - name: Poetry install run: | - cd python + cd /opt/Griddly/python poetry install # Run python tests - name: Python tests run: | - cd python + cd /opt/Griddly/python poetry run pytest . - name: Python Package if: github.ref == 'refs/heads/master' || contains(github.head_ref, 'release-test') run: | - cd python + cd /opt/Griddly/python poetry build --format=wheel # Upload the built wheels From 74725d2d7c90c288bebe4101767b784a563517e4 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Tue, 20 Jun 2023 20:57:11 +0100 Subject: [PATCH 35/45] fixing paths --- .github/workflows/build-test-publish.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 01108573..424f7531 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -82,33 +82,31 @@ jobs: # Configure conan for release build - name: Build run: | - cd /opt/Griddly conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build=* -if build_manylinux - /opt/python/$PYBIN/bin/cmake . -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING=/opt/python/$PYBIN -DPYTHON_EXECUTABLE:FILEPATH=/opt/python/$PYBIN/bin/python -S /opt/Griddly -B /opt/Griddly/build_manylinux - /opt/python/$PYBIN/bin/cmake --build /opt/Griddly/build_manylinux --config Release + /opt/python/$PYBIN/bin/cmake . -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING=/opt/python/$PYBIN -DPYTHON_EXECUTABLE:FILEPATH=/opt/python/$PYBIN/bin/python -S . -B build_manylinux + /opt/python/$PYBIN/bin/cmake --build build_manylinux --config Release # Run the tests - name: Test run: | - cd /opt/Griddly - ctest --test-dir /opt/Griddly/build_manylinux + ctest --test-dir build_manylinux # Setup python environment - name: Poetry install run: | - cd /opt/Griddly/python + cd python poetry install # Run python tests - name: Python tests run: | - cd /opt/Griddly/python + cd python poetry run pytest . - name: Python Package if: github.ref == 'refs/heads/master' || contains(github.head_ref, 'release-test') run: | - cd /opt/Griddly/python + cd python poetry build --format=wheel # Upload the built wheels From 4b0810dbad2d092f6521abd54022536e9666b5b9 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Tue, 20 Jun 2023 21:01:41 +0100 Subject: [PATCH 36/45] fixing executable paths --- .github/workflows/build-test-publish.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 424f7531..039d4eda 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -82,32 +82,32 @@ jobs: # Configure conan for release build - name: Build run: | - conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build=* -if build_manylinux + /opt/python/$PYBIN/bin/conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build=* -if build_manylinux /opt/python/$PYBIN/bin/cmake . -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING=/opt/python/$PYBIN -DPYTHON_EXECUTABLE:FILEPATH=/opt/python/$PYBIN/bin/python -S . -B build_manylinux /opt/python/$PYBIN/bin/cmake --build build_manylinux --config Release # Run the tests - name: Test run: | - ctest --test-dir build_manylinux + /opt/python/$PYBIN/bin/ctest --test-dir build_manylinux # Setup python environment - name: Poetry install run: | cd python - poetry install + /opt/python/$PYBIN/bin/poetry install # Run python tests - name: Python tests run: | cd python - poetry run pytest . + /opt/python/$PYBIN/bin/poetry run pytest . - name: Python Package if: github.ref == 'refs/heads/master' || contains(github.head_ref, 'release-test') run: | cd python - poetry build --format=wheel + /opt/python/$PYBIN/bin/poetry build --format=wheel # Upload the built wheels - name: Upload wheel artifacts From f10417f43822c7667c466f598ce57beff74d6616 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Tue, 20 Jun 2023 21:10:37 +0100 Subject: [PATCH 37/45] another attempt at paths --- .github/workflows/build-test-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 039d4eda..ac51dbd1 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -57,7 +57,6 @@ jobs: container: image: quay.io/pypa/${{ matrix.manylinux_config.platform }} env: - GRIDDLY_ROOT: ${{ github.workspace }} PYBIN: ${{ matrix.manylinux_config.py-bin }} PYVERSION: ${{ matrix.manylinux_config.python-version }} PLATFORM: ${{ matrix.manylinux_config.platform }} @@ -77,11 +76,12 @@ jobs: - name: Install Build Dependencies run: | /opt/python/$PYBIN/bin/pip install poetry cmake conan==1.59.0 - export PATH=$PATH:/opt/python/$PYBIN/bin + # Configure conan for release build - name: Build run: | + export PATH=$PATH:/opt/python/$PYBIN/bin /opt/python/$PYBIN/bin/conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build=* -if build_manylinux /opt/python/$PYBIN/bin/cmake . -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING=/opt/python/$PYBIN -DPYTHON_EXECUTABLE:FILEPATH=/opt/python/$PYBIN/bin/python -S . -B build_manylinux /opt/python/$PYBIN/bin/cmake --build build_manylinux --config Release From 74236989df06daf9c488fb99f4fb83e7b75fe2dd Mon Sep 17 00:00:00 2001 From: Bam4d Date: Tue, 20 Jun 2023 22:00:10 +0100 Subject: [PATCH 38/45] remove sudo --- .github/workflows/build-test-publish.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index ac51dbd1..446f0a05 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -81,10 +81,11 @@ jobs: # Configure conan for release build - name: Build run: | + export CONAN_SYSREQUIRES_SUDO=0 export PATH=$PATH:/opt/python/$PYBIN/bin - /opt/python/$PYBIN/bin/conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build=* -if build_manylinux - /opt/python/$PYBIN/bin/cmake . -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING=/opt/python/$PYBIN -DPYTHON_EXECUTABLE:FILEPATH=/opt/python/$PYBIN/bin/python -S . -B build_manylinux - /opt/python/$PYBIN/bin/cmake --build build_manylinux --config Release + conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build=* -if build_manylinux + cmake . -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING=/opt/python/$PYBIN -DPYTHON_EXECUTABLE:FILEPATH=/opt/python/$PYBIN/bin/python -S . -B build_manylinux + cmake --build build_manylinux --config Release # Run the tests - name: Test From 4bea8968b2227e9c71323f0ac72365ac424bc7f6 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Wed, 21 Jun 2023 20:08:41 +0100 Subject: [PATCH 39/45] manylinux already has all the pythons so we should not install another one --- .github/workflows/build-test-publish.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 446f0a05..a7074503 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -66,11 +66,11 @@ jobs: - name: Checkout uses: actions/checkout@v3 - # Set python version - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.manylinux_config.python-version }} + # # Set python version + # - name: Set up Python + # uses: actions/setup-python@v3 + # with: + # python-version: ${{ matrix.manylinux_config.python-version }} # Install Conan - name: Install Build Dependencies @@ -82,10 +82,9 @@ jobs: - name: Build run: | export CONAN_SYSREQUIRES_SUDO=0 - export PATH=$PATH:/opt/python/$PYBIN/bin - conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build=* -if build_manylinux - cmake . -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING=/opt/python/$PYBIN -DPYTHON_EXECUTABLE:FILEPATH=/opt/python/$PYBIN/bin/python -S . -B build_manylinux - cmake --build build_manylinux --config Release + /opt/python/$PYBIN/bin/conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build=* -if build_manylinux + /opt/python/$PYBIN/bin/cmake . -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING=/opt/python/$PYBIN -DPYTHON_EXECUTABLE:FILEPATH=/opt/python/$PYBIN/bin/python -S . -B build_manylinux + /opt/python/$PYBIN/bin/cmake --build build_manylinux --config Release # Run the tests - name: Test From 6d036dc96caaa6c22359955836460e373ffc1c9b Mon Sep 17 00:00:00 2001 From: Bam4d Date: Wed, 21 Jun 2023 20:15:35 +0100 Subject: [PATCH 40/45] add the right python to the path --- .github/workflows/build-test-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index a7074503..07ee394c 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -82,6 +82,7 @@ jobs: - name: Build run: | export CONAN_SYSREQUIRES_SUDO=0 + export PATH=$PATH:/opt/python/$PYBIN/bin /opt/python/$PYBIN/bin/conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build=* -if build_manylinux /opt/python/$PYBIN/bin/cmake . -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING=/opt/python/$PYBIN -DPYTHON_EXECUTABLE:FILEPATH=/opt/python/$PYBIN/bin/python -S . -B build_manylinux /opt/python/$PYBIN/bin/cmake --build build_manylinux --config Release From ae1ea471a314682237febe99a6efdd2480c43fe7 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Wed, 21 Jun 2023 20:43:56 +0100 Subject: [PATCH 41/45] dont need these manylinux build files anymore, also run the tests without the Observer things that crash --- .github/workflows/build-test-publish.yml | 1 + python/manylinux/build.sh | 21 --------------------- python/manylinux/manylinux-build.sh | 2 -- python/manylinux/manylinux-test.sh | 6 ------ python/manylinux/test.sh | 3 --- 5 files changed, 1 insertion(+), 32 deletions(-) delete mode 100755 python/manylinux/build.sh delete mode 100755 python/manylinux/manylinux-build.sh delete mode 100755 python/manylinux/manylinux-test.sh delete mode 100755 python/manylinux/test.sh diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 07ee394c..5d961462 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -90,6 +90,7 @@ jobs: # Run the tests - name: Test run: | + export GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* /opt/python/$PYBIN/bin/ctest --test-dir build_manylinux # Setup python environment diff --git a/python/manylinux/build.sh b/python/manylinux/build.sh deleted file mode 100755 index 24d474b9..00000000 --- a/python/manylinux/build.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -ex -set -o pipefail - -export CONAN_SYSREQUIRES_SUDO=0 - -# Get Dependencies -/opt/python/$PYBIN/bin/pip install cmake conan==1.59.0 - -export PATH=$PATH:/opt/python/$PYBIN/bin - -echo "Conan Build Finished" - -# # Cmake Build Griddly -cd /opt/Griddly -conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=Release --build=* -if build_manylinux -/opt/python/$PYBIN/bin/cmake . -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DPython_ROOT_DIR:STRING=/opt/python/$PYBIN -DPYTHON_EXECUTABLE:FILEPATH=/opt/python/$PYBIN/bin/python -S /opt/Griddly -B /opt/Griddly/build_manylinux -/opt/python/$PYBIN/bin/cmake --build /opt/Griddly/build_manylinux --config Release - - diff --git a/python/manylinux/manylinux-build.sh b/python/manylinux/manylinux-build.sh deleted file mode 100755 index 87958f99..00000000 --- a/python/manylinux/manylinux-build.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -docker run -e PYVERSION=$PYVERSION -e PYBIN=$PYBIN -e PLAT=$PLATFORM -v "$GRIDDLY_ROOT:/opt/Griddly" quay.io/pypa/$PLATFORM /opt/Griddly/python/manylinux/build.sh \ No newline at end of file diff --git a/python/manylinux/manylinux-test.sh b/python/manylinux/manylinux-test.sh deleted file mode 100755 index dcc5e037..00000000 --- a/python/manylinux/manylinux-test.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -ex -set -o pipefail - -docker run -e PYVERSION=$PYVERSION -e PYBIN=$PYBIN -e PLAT=$PLATFORM -e GTEST_FILTER=-*BlockObserverTest*:*SpriteObserverTest* -v "$GRIDDLY_ROOT:/opt/Griddly" quay.io/pypa/$PLATFORM /opt/Griddly/python/manylinux/test.sh \ No newline at end of file diff --git a/python/manylinux/test.sh b/python/manylinux/test.sh deleted file mode 100755 index c972e02f..00000000 --- a/python/manylinux/test.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -cd /opt/Griddly -ctest --test-dir /opt/Griddly/build_manylinux \ No newline at end of file From 0a925db789ed6602b219c5cb92976c314161d65e Mon Sep 17 00:00:00 2001 From: Bam4d Date: Wed, 21 Jun 2023 21:17:11 +0100 Subject: [PATCH 42/45] manylinux builds working and now trying to upload to test pypi --- .github/workflows/build-test-publish.yml | 26 ++++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 5d961462..b03d06e5 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -147,19 +147,19 @@ jobs: sudo apt-get install python3-pip pip3 install poetry - # - name: Pypi upload (test) - # if: contains(github.head_ref, 'release-test') - # run: | - # cd python - # poetry config repositories.test-pypi https://test.pypi.org/legacy/ - # poetry config pypi-token.test-pypi ${{ secrets.PYPI_TEST_TOKEN }} - # poetry publish -r test-pypi - - # - name: Pypi upload (prod) - # if: github.ref == 'refs/heads/master' - # run: | - # poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} - # poetry publish + - name: Pypi upload (test) + if: contains(github.head_ref, 'release-test') + run: | + cd python + poetry config repositories.test-pypi https://test.pypi.org/legacy/ + poetry config pypi-token.test-pypi ${{ secrets.PYPI_TEST_TOKEN }} + poetry publish -r test-pypi + + - name: Pypi upload (prod) + if: github.ref == 'refs/heads/master' + run: | + poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} + poetry publish # build-macos: # runs-on: macos-latest From 3354e210206ccb22b1fe1a09f7d4cf1e977bbc0f Mon Sep 17 00:00:00 2001 From: Bam4d Date: Thu, 22 Jun 2023 17:47:12 +0100 Subject: [PATCH 43/45] another attempt at publishing --- .github/workflows/build-test-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index b03d06e5..8279461e 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -140,6 +140,7 @@ jobs: - name: List wheels 2 run: | ls -lah python/dist/artifact + mv python/dist/artifact/*.whl python/dist/ - name: Install Build Dependencies run: | From 9c5789b80d0358db09c48f87715f65d27232874b Mon Sep 17 00:00:00 2001 From: Bam4d Date: Thu, 22 Jun 2023 19:32:51 +0100 Subject: [PATCH 44/45] release test working --- .github/workflows/build-test-publish.yml | 62 ++++++++++++++---------- configure.sh | 2 +- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 8279461e..f58ff329 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -8,25 +8,41 @@ on: jobs: - # build-wasm: - # runs-on: ubuntu-latest - - # steps: - # # Checkout the repository - # - name: Checkout - # uses: actions/checkout@v2 - - # # Install Conan - # - name: Install Conan - # run: | - # sudo apt-get update - # sudo apt-get install python3-pip - # pip3 install conan - - # # Configure conan for release build - # - name: Configure Build - # run: | - # ./configure.sh + build-wasm: + runs-on: ubuntu-latest + + steps: + # Checkout the repository + - name: Checkout + uses: actions/checkout@v2 + + # Install Conan + - name: Install Build Dependencies + run: | + sudo apt-get update + sudo apt-get install python3-pip + pip3 install conan + + # Build + - name: Build + run: | + conan install deps/wasm/conanfile_wasm.txt -pr:h deps/wasm/emscripten.profile -pr:b default -s build_type=Release --build missing -if build_wasm + cmake . -B build_wasm -GNinja -DWASM=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake + cmake --build build_wasm --config Release + + - name: Upload WASM + if: github.ref == 'refs/heads/master' || contains(github.head_ref, 'release-test') + uses: actions/upload-artifact@v3 + with: + name: griddlyjs.wasm + path: Release/bin/griddlyjs.wasm + + - name: Upload JS + if: github.ref == 'refs/heads/master' || contains(github.head_ref, 'release-test') + uses: actions/upload-artifact@v3 + with: + name: griddlyjs.js + path: Release/bin/griddlyjs.js build-linux: runs-on: ubuntu-latest @@ -72,7 +88,7 @@ jobs: # with: # python-version: ${{ matrix.manylinux_config.python-version }} - # Install Conan + # Install Build Dependencies - name: Install Build Dependencies run: | /opt/python/$PYBIN/bin/pip install poetry cmake conan==1.59.0 @@ -133,11 +149,7 @@ jobs: with: path: python/dist - - name: List wheels - run: | - ls -lah python/dist - - - name: List wheels 2 + - name: Unpack wheels run: | ls -lah python/dist/artifact mv python/dist/artifact/*.whl python/dist/ diff --git a/configure.sh b/configure.sh index aa15bbcb..ca8dbd0e 100755 --- a/configure.sh +++ b/configure.sh @@ -29,7 +29,7 @@ echo $PLATFORM if [[ $PLATFORM == "WASM" ]] then - conan install deps/wasm/conanfile_wasm.txt --pr:h deps/wasm/emscripten.profile --pr:b default -s build_type=$BUILD --build missing -if build_wasm + conan install deps/wasm/conanfile_wasm.txt -pr:h=deps/wasm/emscripten.profile -pr:b=default -s build_type=$BUILD --build missing -if build_wasm else conan install deps/conanfile.txt -pr:b=default -pr:h=default -pr:h=deps/build.profile -s build_type=$BUILD --build $CONAN_BUILD -if build fi From 3f12e515b567a45560fa5f013794f311359c58b2 Mon Sep 17 00:00:00 2001 From: Bam4d Date: Thu, 22 Jun 2023 20:14:48 +0100 Subject: [PATCH 45/45] pinning conan version --- .github/workflows/build-test-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index f58ff329..de082796 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -21,12 +21,12 @@ jobs: run: | sudo apt-get update sudo apt-get install python3-pip - pip3 install conan + pip3 install conan==1.59.0 # Build - name: Build run: | - conan install deps/wasm/conanfile_wasm.txt -pr:h deps/wasm/emscripten.profile -pr:b default -s build_type=Release --build missing -if build_wasm + conan install deps/wasm/conanfile_wasm.txt -pr:h=deps/wasm/emscripten.profile -pr:b=default -s build_type=Release --build missing -if build_wasm cmake . -B build_wasm -GNinja -DWASM=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake cmake --build build_wasm --config Release