Merge pull request #225 from ESCOMP/test_musica

Add dockerfile to build musica

.github/scripts/branch_pr_issue_closer.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """
 Script name:  branch_PR_issue_closer.py

.github/scripts/pr_mod_file_tests.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """
 Script name:  pr_mod_file_tests.py
@@ -59,28 +59,45 @@ def _file_is_python(filename):
             is_python = True
         else:
             #If no ".py" extension exists, then
-            #open the file and look for a shabang
-            #that contains the word "python".
-            with open(filename, "r", encoding='utf-8') as mod_file:
-                #Loop over lines in file:
-                for line in mod_file:
-
-                    #Ignore blank lines:
-                    if line.strip():
-
-                        #Check that first non-blank
-                        #line is a shabang:
-                        if line[0:2] == '#!':
-                            #If so, then check that the word
-                            #"python" is also present:
-                            if line.find("python") != -1:
-                                #If the word exists, then assume
-                                #it is a python file:
-                                is_python = True
-
-                        #Exit loop, as only the first non-blank
-                        #line should be examined:
-                        break
+            #try to open the file and look for
+            #a shabang that contains the word "python".
+            try:
+                with open(filename, "r", encoding='utf-8') as mod_file:
+                    #Loop over lines in file:
+                    for line in mod_file:
+
+                        #Ignore blank lines:
+                        if line.strip():
+
+                            #Check that first non-blank
+                            #line is a shabang:
+                            if line.startswith("#!"):
+                                #If so, then check that the word
+                                #"python" is also present:
+                                if "python" in line:
+                                    #If the word exists, then assume
+                                    #it is a python file:
+                                    is_python = True
+                                #End if
+                            #End if
+
+                            #Exit loop, as only the first non-blank
+                            #line should be examined:
+                            break
+                        #End if
+                    #End for
+                #End with
+            except UnicodeDecodeError:
+                #Binary files, which we do not care about here,
+                #can result in a decode error, so if that error
+                #is raised just skip the file with a message stating
+                #that it is being skipped (just in case):
+                wmsg = f"WARNING: The file '{filename}' cannot currently be opened,\n"
+                wmsg += "so skipping any attempt at analysing."
+                print(wmsg)
+            #End except
+        #End if
+    #End if
 
     #Return file type result:
     return is_python

.github/scripts/pylint_threshold_test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """
 Modle name: pylint_threshold_test.py

Externals.cfg

@@ -21,7 +21,7 @@ externals = Externals.cfg
 required = True
 
 [cmeps]
-tag = cmeps0.14.24
+tag = cmeps0.14.34
 protocol = git
 repo_url = https://github.com/ESCOMP/CMEPS.git
 local_path = components/cmeps

Externals_CAM.cfg

@@ -23,7 +23,7 @@ required = False
 local_path = src/physics/ncar_ccpp
 protocol = git
 repo_url = https://github.com/NCAR/atmospheric_physics
-tag = atmos_phys0_00_018
+tag = atmos_phys0_00_020
 required = True
 
 [silhs]

README.md

@@ -22,33 +22,33 @@ git checkout development
 Good luck, and have a great day!
 
 ## Docker
-
 If you don't want to worry about installing our dependencies, 
-you can create a build of CAM-SIMA using docker. To do so, you will need to 
-download and install [docker desktop](https://docs.docker.com/desktop/).
+you can create a build of CAM-SIMA using docker.
+
+### Download docker
+Download and install [docker desktop](https://docs.docker.com/desktop/).
 
 #### Windows
 Follow the installation instructions for Windows [here](https://docs.docker.com/desktop/install/windows-install/)
 
-
 #### Linux
 Follow the installation instructions for Linux [here](https://docs.docker.com/desktop/install/linux-install/)
 
 #### MacOS
 Follow the installation instructions for Mac [here](https://docs.docker.com/desktop/install/mac-install/)
 
-### Download docker
 
-### Build the docker file
-1. First, build the esmf docker image. You must tag the build with `esmf`
+### Build the base docker file
+1. First, build the esmf docker image. You must tag the build with `esmf`. This will take quite some time.
 ```
-cd docker
-docker build -f Dockerfile.esmf -t esmf .
+docker build -f docker/Dockerfile.esmf -t esmf .
 ```
-2. Build the CAM-SIMA image
+
+### CAM-SIMA
+
+1. Build the CAM-SIMA image
 ```
-cd ../
-docker build -t cam-sima .
+docker build -f docker/Dockerfile -t cam-sima .
 ```
 3. Run the image
 ```
@@ -57,4 +57,20 @@ docker run --rm -it cam-sima
 4. Submit the case
 ```
 ./case.submit
+```
+
+### MUSICA
+You can include musica chemistry by using the correct docker file
+
+1. Build the MUSICA image
+```
+docker build -f docker/Dockerfile.musica -t musica .
+```
+3. Run the image
+```
+docker run --rm -it musica
+```
+4. Submit the case
+```
+./case.submit
 ```

cime_config/config_component.xml

@@ -301,6 +301,23 @@
     <desc>User mods to apply to specific compset matches. </desc>
   </entry>
 
+  <entry id="CAM_LINKED_LIBS">
+    <type>char</type>
+    <valid_values></valid_values>
+    <default_value>-lmusica -ljsonfortran</default_value>
+    <values match="last">
+      <!--Turn off for physics testbed -->
+      <value compset="_CAM%PHYSTEST"> </value>
+    </values>
+    <group>build_component_cam</group>
+    <file>env_build.xml</file>
+    <desc>
+      CAM-SIMA linked libraries. The libraries are built by
+      CAM-SIMA's buildlib script and should be included
+      during linking of the model executable.
+    </desc>
+  </entry>
+
   <help>
     =========================================
     CAM naming conventions

Dockerfile → docker/Dockerfile

@@ -82,4 +82,4 @@ RUN /home/cam_sima_user/CAM-SIMA/docker/ftp_download.sh
 # # add the snapshot file
 RUN echo "ncdata='/home/cam_sima_user/run_heldsuarez_cam6_nt2_bigg_try005.cam.h5.0001-01-01-00000.nc'" >> user_nl_cam
 
-RUN ./case.build
+RUN ./case.build

docker/Dockerfile.musica

@@ -0,0 +1,119 @@
+# parts of CAM require x86 architecture (gptl, which relies on the rdtsc x86 assembly instruction)
+# esmf is am image you are expected to have built. Read the README file for instructions
+FROM esmf:latest
+
+###################################################
+## Install necessary packages
+###################################################
+RUN dnf -y update \
+    && dnf -y install \
+      ftp \
+      git \
+      hostname \
+      m4 \
+      python \
+      sudo \
+      svn \
+      tree \
+      vim \
+    && dnf clean all
+
+###################################################
+## Make sure the mpi compilers can be found
+###################################################
+ENV PATH="${PATH}:/usr/lib64/mpich/bin/"
+ENV OMP_NUM_THREADS=5
+
+###################################################
+## Build and install Parallel-netcdf
+###################################################
+RUN wget -q https://parallel-netcdf.github.io/Release/pnetcdf-1.12.3.tar.gz
+RUN tar zxf pnetcdf-1.12.3.tar.gz
+RUN cd pnetcdf-1.12.3 && \
+     ./configure --prefix=/usr/local && \
+     make -j 8 install && \
+     ldconfig
+
+ENV FC=gfortran
+
+###################################################
+## Build and install json-fortran
+###################################################
+RUN curl -LO https://github.com/jacobwilliams/json-fortran/archive/8.2.0.tar.gz \
+    && tar -zxvf 8.2.0.tar.gz \
+    && cd json-fortran-8.2.0 \
+    && mkdir build \
+    && cd build \
+    && cmake -D SKIP_DOC_GEN:BOOL=TRUE .. \
+    && make install -j 8
+
+# add a symlink
+RUN ln -s /usr/local/jsonfortran-gnu-8.2.0/lib/libjsonfortran.a /usr/local/lib/libjsonfortran.a
+
+###################################################
+## Build and install MUSICA
+###################################################
+
+RUN git clone https://github.com/NCAR/musica.git
+RUN mkdir /musica/build \
+    && cd /musica/build \
+    && export JSON_FORTRAN_HOME="/usr/local/jsonfortran-gnu-8.2.0" \
+    && cmake \
+             -D ENABLE_TESTS=OFF \
+             -D ENABLE_TUVX=OFF \
+          .. \
+    && make install -j 8
+
+# add a symlink
+RUN ln -s /usr/local/musica-0.3.0/lib64/libmusica.a /usr/local/lib/libmusica.a
+
+###################################################
+## Build CAM-SIMA
+###################################################
+
+# create a user to run the case
+RUN adduser cam_sima_user \
+    && echo "cam_sima_user ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/cam_sima_user \
+    && chmod 0440 /etc/sudoers.d/cam_sima_user
+
+# copy in the CAM-SIMA code and give the proper user permissions
+COPY --chown=cam_sima_user . /home/cam_sima_user/CAM-SIMA
+
+USER cam_sima_user
+WORKDIR /home/cam_sima_user/CAM-SIMA
+
+# pull the dependencies
+RUN ./manage_externals/checkout_externals
+
+# Copy in the machine information for the container
+RUN cp /home/cam_sima_user/CAM-SIMA/docker/config_machines.xml /home/cam_sima_user/CAM-SIMA/ccs_config/machines/
+
+# Set environment variables needed to create and build the case
+ENV USER=$(whoami)
+ENV CASE_NAME=/home/cam_sima_user/case_name
+ENV CESMDATAROOT=/home/cam_sima_user/cesm_data
+ENV CIME_MACHINE=container
+ENV CIME_MODEL=cesm
+ENV ESMFMKFILE=/usr/local/lib/esmf.mk
+
+# Create a case
+RUN ./cime/scripts/create_newcase --case $CASE_NAME  --compset FPHYStest --res ne5_ne5_mg37 --run-unsupported
+
+WORKDIR $CASE_NAME 
+
+RUN ./case.setup
+
+RUN ./xmlchange CAM_CONFIG_OPTS="--dyn none --physics-suites musica"
+RUN ./xmlchange CAM_LINKED_LIBS="-lmusica -ljsonfortran"
+RUN ./xmlchange ROF_NCPL=48
+RUN ./xmlchange STOP_OPTION=nsteps
+RUN ./xmlchange STOP_N=5
+
+# Copy in the grid files and a snapshot file
+RUN chmod +x /home/cam_sima_user/CAM-SIMA/docker/ftp_download.sh
+RUN /home/cam_sima_user/CAM-SIMA/docker/ftp_download.sh
+
+# # add the snapshot file
+RUN echo "ncdata='/home/cam_sima_user/run_heldsuarez_cam6_nt2_bigg_try005.cam.h5.0001-01-01-00000.nc'" >> user_nl_cam
+
+RUN ./case.build

test/.pylintrc

@@ -457,4 +457,4 @@ known-third-party=enchant
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "Exception"
-overgeneral-exceptions=Exception
+overgeneral-exceptions=builtins.Exception