Merge pull request #1203 from sys-bio/develop

New release!
sys-bio · Mar 26, 2024 · bcffdf3 · bcffdf3
2 parents e8ba92a + 486d509
commit bcffdf3
Show file tree

Hide file tree

Showing 91 changed files with 2,952 additions and 1,201 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -8,7 +8,7 @@ cmake_minimum_required(VERSION 3.16)
 # Version information and include modules
 
 set(ROADRUNNER_VERSION_MAJOR 2)
-set(ROADRUNNER_VERSION_MINOR 5)
+set(ROADRUNNER_VERSION_MINOR 6)
 set(ROADRUNNER_VERSION_PATCH 0)
 
 set(ROADRUNNER_VERSION "${ROADRUNNER_VERSION_MAJOR}.${ROADRUNNER_VERSION_MINOR}.${ROADRUNNER_VERSION_PATCH}")

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -32,16 +32,30 @@ stages:
         timeoutInMinutes: "0"
         displayName: MacBuildRoadrunnerCpp
         continueOnError: "false"
-        pool:
-          vmImage: 'macOS-11'
         strategy:
           matrix:
-            64-bit Mac Release:
+            64-bit Mac Release x86:
               BuildType: Release
               LLVM_DOWNLOAD_LINK: 'https://github.com/sys-bio/llvm-13.x/releases/download/llvmorg-13.0.0/llvm-13.x-macosx_11_7_x86_64.zip'
-            64-bit Mac Debug:
+              vmImage: macOS-11
+              arch: x86_64
+            64-bit Mac Debug x86:
               BuildType: Debug
               LLVM_DOWNLOAD_LINK: 'https://github.com/sys-bio/llvm-13.x/releases/download/llvmorg-13.0.0/llvm-13.x-macosx_11_7_x86_64.zip'
+              vmImage: macOS-11
+              arch: x86_64
+            # 64-bit Mac Release M1:
+            #   BuildType: Release
+            #   LLVM_DOWNLOAD_LINK: 'https://github.com/sys-bio/llvm-13.x/releases/download/llvmorg-13.0.0/llvm-13.x-macosx_13_2_arm64.zip'
+            #   vmImage: macOS-14
+            #   arch: arm64
+            # 64-bit Mac Debug M1:
+            #   BuildType: Debug
+            #   LLVM_DOWNLOAD_LINK: 'https://github.com/sys-bio/llvm-13.x/releases/download/llvmorg-13.0.0/llvm-13.x-macosx_13_2_arm64.zip'
+            #   vmImage: macOS-14
+            #   arch: arm64
+        pool:
+          vmImage: $(vmImage)
         variables:
           LLVM_CACHE: 'false'
           PythonName: 'py39'
@@ -138,7 +152,7 @@ stages:
               # - https://dev.azure.com/TheRoadrunnerProject/libroadrunner-deps/_build?definitionId=9
               pipeline: 9
               runVersion: 'latest'
-              artifact: libroadrunner-deps-$(Agent.OS)-$(BuildType)
+              artifact: libroadrunner-deps-$(Agent.OS)-$(BuildType)-$(arch)
               path: $(DEPS_INSTALL_PREFIX)
             displayName: Download libroadrunner-deps install artifacts
           - script: |
@@ -191,22 +205,58 @@ stages:
         timeoutInMinutes: "0"
         displayName: MacBuildRoadrunnerPython
         continueOnError: "false"
-        pool:
-          vmImage: 'macOS-11'
         strategy:
           matrix:
-            py38:
-              PythonVersion: 3.8
-              PythonName: py38
-            py39:
+            py39, x86:
+              LLVM_DOWNLOAD_LINK: 'https://github.com/sys-bio/llvm-13.x/releases/download/llvmorg-13.0.0/llvm-13.x-macosx_11_7_x86_64.zip'
+              vmImage: macOS-11
+              arch: x86_64
               PythonVersion: 3.9
               PythonName: py39
-            py310:
+            py310, x86:
+              LLVM_DOWNLOAD_LINK: 'https://github.com/sys-bio/llvm-13.x/releases/download/llvmorg-13.0.0/llvm-13.x-macosx_11_7_x86_64.zip'
+              vmImage: macOS-11
+              arch: x86_64
               PythonVersion: 3.10
               PythonName: py310
-            py311:
+            py311, x86:
+              LLVM_DOWNLOAD_LINK: 'https://github.com/sys-bio/llvm-13.x/releases/download/llvmorg-13.0.0/llvm-13.x-macosx_11_7_x86_64.zip'
+              vmImage: macOS-11
+              arch: x86_64
               PythonVersion: 3.11
               PythonName: py311
+            py312, x86:
+              LLVM_DOWNLOAD_LINK: 'https://github.com/sys-bio/llvm-13.x/releases/download/llvmorg-13.0.0/llvm-13.x-macosx_11_7_x86_64.zip'
+              vmImage: macOS-11
+              arch: x86_64
+              PythonVersion: 3.12
+              PythonName: py312
+            # py39, arm64:
+            #   LLVM_DOWNLOAD_LINK: 'https://github.com/sys-bio/llvm-13.x/releases/download/llvmorg-13.0.0/llvm-13.x-macosx_13_2_arm64.zip'
+            #   vmImage: macOS-11
+            #   arch: arm64_64
+            #   PythonVersion: 3.9
+            #   PythonName: py39
+            # py310, arm64:
+            #   LLVM_DOWNLOAD_LINK: 'https://github.com/sys-bio/llvm-13.x/releases/download/llvmorg-13.0.0/llvm-13.x-macosx_13_2_arm64.zip'
+            #   vmImage: macOS-11
+            #   arch: arm64_64
+            #   PythonVersion: 3.10
+            #   PythonName: py310
+            # py311, arm64:
+            #   LLVM_DOWNLOAD_LINK: 'https://github.com/sys-bio/llvm-13.x/releases/download/llvmorg-13.0.0/llvm-13.x-macosx_13_2_arm64.zip'
+            #   vmImage: macOS-11
+            #   arch: arm64_64
+            #   PythonVersion: 3.11
+            #   PythonName: py311
+            # py312, arm64:
+            #   LLVM_DOWNLOAD_LINK: 'https://github.com/sys-bio/llvm-13.x/releases/download/llvmorg-13.0.0/llvm-13.x-macosx_13_2_arm64.zip'
+            #   vmImage: macOS-11
+            #   arch: arm64_64
+            #   PythonVersion: 3.12
+            #   PythonName: py312
+        pool:
+          vmImage: $(vmImage)
         variables:
           MinicondaRoot : '/usr/local/miniconda'
           PythonRoot: '$(MinicondaRoot)/envs/$(PythonName)'
@@ -217,7 +267,6 @@ stages:
           SWIG_CACHE: 'false'
           MINICONDA_CACHE: 'false'
           LLVM_CACHE: 'false'
-          LLVM_DOWNLOAD_LINK: 'https://github.com/sys-bio/llvm-13.x/releases/download/llvmorg-13.0.0/llvm-13.x-macosx_11_7_x86_64.zip'
         steps:
           - checkout: self
             submodules: recursive
@@ -272,7 +321,7 @@ stages:
               project: 'libroadrunner-deps'
               pipeline: 9
               runVersion: 'latest'
-              artifact: libroadrunner-deps-$(Agent.OS)-Release
+              artifact: libroadrunner-deps-$(Agent.OS)-Release-$(arch)
               path: $(DEPS_INSTALL_PREFIX)
             displayName: Download libroadrunner-deps install artifacts
           - script: |
@@ -370,6 +419,7 @@ stages:
               ls
               echo "$(PythonExecutable) setup.py bdist_wheel"
               $(PythonExecutable) setup.py bdist_wheel
+              rm -r build/
               echo "$(PythonExecutable) setup_rrplugins.py bdist_wheel"
               $(PythonExecutable) setup_rrplugins.py bdist_wheel
             displayName: Generate pip wheel
@@ -413,8 +463,8 @@ stages:
         variables:
           LLVM_CACHE: 'false'
           MinicondaRoot : 'C:\Miniconda'
-          PythonName: 'py39'
-          PythonVersion: '3.9'
+          PythonName: 'py311'
+          PythonVersion: '3.11'
           PythonRoot: '$(MinicondaRoot)\envs\$(PythonName)'
           PythonLibDir: '$(PythonRoot)\Lib'
           PythonScriptsDir: '$(PythonRoot)\Scripts'
@@ -557,9 +607,6 @@ stages:
           vmImage: 'windows-2019'
         strategy:
           matrix:
-            py38:
-              PythonVersion: 3.8
-              PythonName: py38
             py39:
               PythonVersion: 3.9
               PythonName: py39
@@ -569,6 +616,9 @@ stages:
             py311:
               PythonVersion: 3.11
               PythonName: py311
+            py312:
+              PythonVersion: 3.12
+              PythonName: py312
         variables:
           MinicondaRoot : 'C:\Miniconda'
           PythonRoot: '$(MinicondaRoot)\envs\$(PythonName)'
@@ -733,6 +783,7 @@ stages:
               echo "ls in install dir"
               ls
               $(PythonExecutable) setup.py bdist_wheel
+              rm -r build/
               $(PythonExecutable) setup_rrplugins.py bdist_wheel
             displayName: Generate pip wheel
 
@@ -761,7 +812,7 @@ stages:
         displayName: UbuntuBuildRoadrunnerCpp
         continueOnError: "false"
         pool:
-          vmImage: 'Ubuntu-20.04'
+          vmImage: 'Ubuntu-latest'
         strategy:
           matrix:
             64-bit Linux Release:
@@ -963,9 +1014,6 @@ stages:
       - job:
         strategy:
           matrix:
-            py38:
-              PythonVersion: 3.8
-              PythonName: py38
             py39:
               PythonVersion: 3.9
               PythonName: py39
@@ -975,8 +1023,11 @@ stages:
             py311:
               PythonVersion: 3.11
               PythonName: py311
+            py312:
+              PythonVersion: 3.12
+              PythonName: py312
         pool:
-          vmImage: 'ubuntu-20.04'
+          vmImage: 'ubuntu-latest'
         container: sysbiouw/roadrunner-manylinux2014:latest
         variables:
           CCACHE_DIR: '$(Pipeline.Workspace)/ccache'
@@ -1074,6 +1125,7 @@ stages:
 
               echo "cmake command: cmake -DLLVM_INSTALL_PREFIX=/install-llvm-13.x -DRR_DEPENDENCIES_INSTALL_PREFIX=/install-libroadrunner-deps -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIRECTORY) -DBUILD_PYTHON=ON -DBUILD_RR_PLUGINS=ON -DBUILD_TESTS=ON -DPython_ROOT_DIR=$(PythonRoot) -DSWIG_EXECUTABLE=$(SwigExecutable) -DCMAKE_BUILD_TYPE=Release .."
               cmake -DLLVM_INSTALL_PREFIX=/install-llvm-13.x -DRR_DEPENDENCIES_INSTALL_PREFIX=/install-libroadrunner-deps -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIRECTORY) -DBUILD_PYTHON=ON -DBUILD_RR_PLUGINS=ON -DBUILD_TESTS=ON -DPython_ROOT_DIR=$(PythonRoot) -DSWIG_EXECUTABLE=$(SwigExecutable) -DCMAKE_BUILD_TYPE=Release ..
+              rm -r -f $(SOURCE_DIR)/docs $(SOURCE_DIR)/examples
               cmake --build . --target install --config Release -j 12
 
             displayName: Build With Python
@@ -1086,39 +1138,46 @@ stages:
               cd $(BUILD_DIR)
               ctest --extra-verbose --output-on-failure --exclude-regex python_tests_RunStochasticTestSuite .
             displayName: Run Tests
-          - script: |
-              echo ". ~/.bashrc"
-              . ~/.bashrc
-              $(CondaExecutable) activate $(PythonName)
+            # Skip stochastic test suite for now because it keeps pushing us over an hour.
+          # - script: |
+          #     echo ". ~/.bashrc"
+          #     . ~/.bashrc
+          #     $(CondaExecutable) activate $(PythonName)
 
-              echo "cd'ing to build dir $(BUILD_DIR)"
-              cd $(BUILD_DIR)
-              ctest --extra-verbose --output-on-failure --tests-regex python_tests_RunStochasticTestSuite .
-            displayName: Run Stochastic test suite
+          #     echo "cd'ing to build dir $(BUILD_DIR)"
+          #     cd $(BUILD_DIR)
+          #     ctest --extra-verbose --output-on-failure --tests-regex python_tests_RunStochasticTestSuite .
+          #   displayName: Run Stochastic test suite
 
           - script: |
               echo "cd to $(INSTALL_DIRECTORY)"
               cd $(INSTALL_DIRECTORY)
 
+              #Space was filling up, so since we've already installed, we can delete the build directory.
+              echo "rm $(BUILD_DIRECTORY)"
+              rm -r $(BUILD_DIRECTORY)
+
               echo "ls"
               ls
 
               $(PipExecutable) install numpy pytest
               echo "$(PythonExecutable) ./setup.py bdist_wheel"
               $(PythonExecutable) ./setup.py bdist_wheel
+              echo "rm -r build/"
+              rm -r build/
+              echo "$(PythonExecutable) ./setup_rrplugins.py bdist_wheel"
               $(PythonExecutable) ./setup_rrplugins.py bdist_wheel
 
               # cd dist
               # wheel=$(realpath $(ls))
               # echo "wheel is: $wheel"
               # $(PipExecutable) install $wheel
             displayName: Build Pip Wheel
-          - task: CopyFiles@2
-            inputs:
-              sourceFolder: '$(INSTALL_DIRECTORY)'
-              contents: '**'
-              targetFolder: '$(Build.ArtifactStagingDirectory)/roadrunner-manylinux2014-$(PythonName)'
-            displayName: Copy install to artifact staging area
+          - script: |
+              echo "mv $(INSTALL_DIRECTORY)/* $(Build.ArtifactStagingDirectory)/roadrunner-manylinux2014-$(PythonName)"
+              mkdir $(Build.ArtifactStagingDirectory)/roadrunner-manylinux2014-$(PythonName)
+              mv $(INSTALL_DIRECTORY)/* $(Build.ArtifactStagingDirectory)/roadrunner-manylinux2014-$(PythonName)
+            displayName: Move install to staging directory
           - task: PublishBuildArtifacts@1
             inputs:
               pathToPublish: '$(Build.ArtifactStagingDirectory)/roadrunner-manylinux2014-$(PythonName)'

diff --git a/docker/INSTRUCTIONS.txt b/docker/INSTRUCTIONS.txt
@@ -0,0 +1,22 @@
+When updating (i.e. when Python comes out with a new version):
+
+* Change roadrunner-manylinux2014/Dockerfile to reflect new python versions.
+* Make sure docker is running (launching it on the lab mac works)
+* Don't actually use the GUI; just use the command line, once it's running.
+* Run the following commands:
+
+cd roadrunner/docker/
+docker build roadrunner-manylinux2014
+docker images
+
+* Note the IMAGE ID of the thing you just built (We'll say it's 5555):
+
+docker tag 5555 sysbiouw/roadrunner-manylinux2014
+docker login -u "sysbiouw" docker.io
+docker push sysbiouw/roadrunner-manylinux2014
+
+
+
+If you need to update one of the other images, follow the same instructions, but build up from -base to -add-deps, depending on what needs to be updated.  (i.e. if the deps need to be updated, do that first, then the Python with the one after that)
+
+Make sure you build and push roadrunner-manylinux2014-base first, then roadrunner-manylinux2014-add-deps 
diff --git a/docker/roadrunner-manylinux2014-base/Dockerfile b/docker/roadrunner-manylinux2014-base/Dockerfile
@@ -17,9 +17,9 @@ RUN cd gcc-11.1.0 && ./contrib/download_prerequisites
 RUN cd gcc-11.1.0 && mkdir build && cd build && ../configure --prefix=/usr/local/gcc-11.1.0 --disable-multilib
 RUN cd gcc-11.1.0/build && make -j 12 CPPFLAGS=-O3 CXXFLAGS=-O3
 RUN cd gcc-11.1.0/build && make install
-RUN cd .. && rm -rf gcc-11.1.0
+RUN cd .. && rm -rf gcc-11.1.0 gcc-11.1.0.tar.gz
 
-RUN yum install -y nano
+#RUN yum install -y nano
 
 RUN yum install -y pcre-devel.x86_64
 
@@ -29,10 +29,10 @@ RUN  mkdir -p $swig_source_dir && wget -q --no-check-certificate https://netcolo
 RUN  tar -xf swig-4.0.2.tar.gz
 RUN  cd $swig_source_dir && ./configure --prefix=$swig_install_dir
 RUN  cd $swig_source_dir && make -j 12 && make install
+RUN  cd $swig_source_dir && mv $swig_install_dir .. && rm -r -f * && mv ../install-swig* .
+RUN  rm -r -f swig-4.0.2.tar.gz
 
-
-RUN yum install -y ncurses-devel
-RUN git config --global core.editor nano && git config --global pager.branch false
+#RUN git config --global core.editor nano && git config --global pager.branch false
 
 
 #RUN echo -e "export PATH=\"/usr/local/gcc-11.1.0/bin:$PATH\"" >> ~/.bashrc \

diff --git a/docker/roadrunner-manylinux2014/Dockerfile b/docker/roadrunner-manylinux2014/Dockerfile
@@ -11,9 +11,9 @@ FROM sysbiouw/roadrunner-manylinux2014-add-deps
 RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
 RUN bash Miniconda3-latest-Linux-x86_64.sh -b -p /Miniconda3
 
-RUN /Miniconda3/bin/conda create -y --name py38 python=3.8 pytest
 RUN /Miniconda3/bin/conda create -y --name py39 python=3.9 pytest
 RUN /Miniconda3/bin/conda create -y --name py310 python=3.10 pytest
 RUN /Miniconda3/bin/conda create -y --name py311 python=3.11 pytest -c conda-forge
+RUN /Miniconda3/bin/conda create -y --name py312 python=3.12 pytest -c conda-forge
 
 RUN /Miniconda3/bin/conda init && bash ~/.bashrc && . ~/.bashrc