From 680cdadb27ad7e45d4391547563fa1528123c438 Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Wed, 30 Nov 2022 14:06:20 -0700 Subject: [PATCH 1/5] Added matplotlib and importlib-metadata to list of packages installed by pip --- dev/modeltesting/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/modeltesting/Dockerfile b/dev/modeltesting/Dockerfile index 1799982..de95485 100644 --- a/dev/modeltesting/Dockerfile +++ b/dev/modeltesting/Dockerfile @@ -18,7 +18,7 @@ MAINTAINER jamesmcc@ucar.edu ################################### #Install modules -RUN pip install numpy netCDF4 pytest pytest-datadir-ng wrfhydropy==0.0.17 +RUN pip install numpy netCDF4 pytest pytest-datadir-ng wrfhydropy==0.0.17 matplotlib importlib-metadata==4.13.0 #################################### ######### entrypoint ########### From 7583739876a82c71b531cd1e8ad89c4ab94ae028 Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Tue, 29 Aug 2023 12:56:44 -0600 Subject: [PATCH 2/5] Added build code for the WrfHydroForcing docker testing container --- forcing_engine/testing/Dockerfile | 98 +++++++++++++++++++ forcing_engine/testing/Readme.md | 10 ++ forcing_engine/testing/build.sh | 5 + .../testing/install_python_packages.sh | 25 +++++ forcing_engine/testing/run_tests.sh | 21 ++++ 5 files changed, 159 insertions(+) create mode 100644 forcing_engine/testing/Dockerfile create mode 100644 forcing_engine/testing/Readme.md create mode 100755 forcing_engine/testing/build.sh create mode 100755 forcing_engine/testing/install_python_packages.sh create mode 100755 forcing_engine/testing/run_tests.sh diff --git a/forcing_engine/testing/Dockerfile b/forcing_engine/testing/Dockerfile new file mode 100644 index 0000000..de34cad --- /dev/null +++ b/forcing_engine/testing/Dockerfile @@ -0,0 +1,98 @@ +################################### +# WRF-Hydro Forcing Engine testing container +# Purpose: +# Provide pre-built wgrib2, mpi, and ESMF libraries for +# testing and running the Forcing Engine +################################### + +FROM ubuntu:latest +SHELL ["/bin/bash", "-c"] +USER root + +# install build environment +RUN apt-get update \ + && apt-get -y install wget make cmake gcc gfortran pip vim less rsh-client + +# install mpich +RUN MPICH_VERSION="4.1" \ + && MPICH_CONFIGURE_OPTIONS="" \ + && MPICH_MAKE_OPTIONS='-j 2' \ + && mkdir /tmp/mpich-src \ + && cd /tmp/mpich-src \ + && wget http://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz \ + && tar xfz mpich-${MPICH_VERSION}.tar.gz \ + && cd mpich-${MPICH_VERSION} \ + && ./configure ${MPICH_CONFIGURE_OPTIONS} \ + && make ${MPICH_MAKE_OPTIONS} && make install \ + && rm -rf /tmp/mpich-src + + +# add docker user +RUN mkdir /home/docker \ + && useradd -s /bin/bash -d /home/docker -m docker \ + && chown docker /home/docker + + +USER docker + +# Place MPI libraries into the LD path. +RUN export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH +RUN export LIBRARY_PATH=/usr/local/lib +RUN export INCLUDE_PATH=/usr/local/include + +# install conda +RUN cd $HOME \ + && wget https://repo.anaconda.com/miniconda/Miniconda3-py311_23.5.2-0-Linux-x86_64.sh \ + && bash Miniconda3-py311_23.5.2-0-Linux-x86_64.sh -b + +RUN export PATH=$HOME/miniconda3/bin:$PATH \ + && conda init bash + +# install ESMF +RUN mkdir $HOME/.esmf-src \ + && cd $HOME/.esmf-src \ + && wget https://github.com/esmf-org/esmf/archive/refs/tags/v8.5.0.tar.gz \ + && tar xfz v8.5.0.tar.gz \ + && ln -s esmf-8.5.0 esmf \ + && cd esmf \ + && export ESMF_DIR=$PWD \ + && export ESMF_COMM=mpich3 \ + && make + +USER root + +RUN cd /home/docker/.esmf-src/esmf \ + && export ESMF_DIR=$PWD \ + && export ESMF_COMM=mpich3 \ + && make install + +USER docker + +RUN pip install mpi4py h5py netcdf4 numpy + +# install ESMpy +RUN cd $HOME/.esmf-src/esmf/src/addon/esmpy \ + && make install + +# install wgrib2 +RUN cd $HOME \ + && wget https://www.ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz.v3.1.2 \ + && tar xfz wgrib2.tgz.v3.1.2 \ + && cd grib2 \ + && export CC=gcc \ + && export FC=gfortran \ + && make \ + && make lib + +USER docker + +RUN export PATH=$HOME/grib2/wgrib2:$PATH + +#################################### +######### entrypoint ########### +#################################### +COPY install_python_packages.sh /home/docker +COPY run_tests.sh /home/docker + +CMD ["/home/docker/install_python_packages.sh"] +CMD ["/home/docker/run_tests.sh"] diff --git a/forcing_engine/testing/Readme.md b/forcing_engine/testing/Readme.md new file mode 100644 index 0000000..bb62d01 --- /dev/null +++ b/forcing_engine/testing/Readme.md @@ -0,0 +1,10 @@ +![](https://ral.ucar.edu/sites/default/files/public/wrf_hydro_symbol_logo_2017_09_150pxby63px.png) WRF-HYDRO + +# Overview +This container is used for WRF-Hydro Forcing Engine testing on a small domain,single-node machine + +# Usage +1. Make sure you have the NCAR/WrfHydroForcing cloned on your local machine +2. When running this container, make sure the local repo is mapped to /home/docker/WrfHydroForcing +3. To prepare the environment and run all tests, run as such + diff --git a/forcing_engine/testing/build.sh b/forcing_engine/testing/build.sh new file mode 100755 index 0000000..42979a1 --- /dev/null +++ b/forcing_engine/testing/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +docker build "$@" -t wrfhydro/dev:modeltesting . + +exit $? diff --git a/forcing_engine/testing/install_python_packages.sh b/forcing_engine/testing/install_python_packages.sh new file mode 100755 index 0000000..02bfaa0 --- /dev/null +++ b/forcing_engine/testing/install_python_packages.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Install python packages using the repo's requirements.txt and setup.py +# so make sure it exists + +source /home/docker/.bashrc +export PATH=$HOME/grib2/wgrib2:$PATH + +if [ ! -e "/home/docker/WrfHydroForcing/requirements.txt" ]; then + echo "riequirements.txt not found. Please make sure the WrfHydroForcing branch to test " + echo "has been checked out and is mapped to /home/docker/WrfHydroForcing" + exit -1 +fi + +cd /home/docker/WrfHydroForcing +echo "Upgrading pip" +python -m pip install -q --upgrade pip +echo "Installing from requirements.txt" +pip install -q -r requirements.txt +echo "Installing esmpy and mpi4py" +conda install -y -q -c conda-forge esmpy mpi4py > /dev/null 2>&1 +echo "Running setup.py" +python setup.py install > /dev/null 2>&1 + +exit 0 diff --git a/forcing_engine/testing/run_tests.sh b/forcing_engine/testing/run_tests.sh new file mode 100755 index 0000000..9c87ba2 --- /dev/null +++ b/forcing_engine/testing/run_tests.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Run the python script to run tests. All this does is run the run_tests.py script, +# so make sure it exists + +source /home/docker/.bashrc + +if [ ! -e "/home/docker/WrfHydroForcing/core/tests/run_tests.py" ]; then + echo "run_tests.py not found. Please make sure the WrfHydroForcing branch to test " + echo "has been checked out and is mapped to /home/docker/WrfHydroForcing" + exit -1 +fi + +export PATH=/home/docker/grib2/wgrib2:$PATH +cd /home/docker/WrfHydroForcing/core/tests + +args=$@ +python run_tests.py $args + +test_status=$? +exit $test_status From fbf8d183db3adc0a744e9a83b52985abe0891eef Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Tue, 29 Aug 2023 13:52:57 -0600 Subject: [PATCH 3/5] Added final name of dockerhub image --- forcing_engine/testing/Readme.md | 19 +++++++++++++++++-- forcing_engine/testing/build.sh | 2 +- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/forcing_engine/testing/Readme.md b/forcing_engine/testing/Readme.md index bb62d01..cd10ef2 100644 --- a/forcing_engine/testing/Readme.md +++ b/forcing_engine/testing/Readme.md @@ -5,6 +5,21 @@ This container is used for WRF-Hydro Forcing Engine testing on a small domain,si # Usage 1. Make sure you have the NCAR/WrfHydroForcing cloned on your local machine -2. When running this container, make sure the local repo is mapped to /home/docker/WrfHydroForcing -3. To prepare the environment and run all tests, run as such +2. When running this container, make sure the local repo is mapped to `/home/docker/WrfHydroForcing` +3. To prepare the environment and run all tests, run with the default commands: + +``` +docker run -v $REPO_LOCATION:/home/docker/WrfHydroForcing wrfhydro/wrf_hydro_forcing:latest +``` +4. To run the container interactively, override the default command with a shell, e.g.: + +``` +docker run -it -v $REPO_LOCATION:/home/docker/WrfHydroForcing wrfhydro/wrf_hydro_forcing:latest /bin/bash +``` + +Note when running interactively you will need to install the needed python packages manually before running the Forcing Engine or FE tests. This can be done by running this script from within the container: + +``` +/home/docker/install_python_packages.sh +``` diff --git a/forcing_engine/testing/build.sh b/forcing_engine/testing/build.sh index 42979a1..8c9b6dc 100755 --- a/forcing_engine/testing/build.sh +++ b/forcing_engine/testing/build.sh @@ -1,5 +1,5 @@ #!/bin/bash -docker build "$@" -t wrfhydro/dev:modeltesting . +docker build "$@" -t wrfhydro/wrf_hydro_forcing:latest . exit $? From f020c560748173811223a7c70c6d2f239135095d Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Tue, 29 Aug 2023 14:14:13 -0600 Subject: [PATCH 4/5] Fixed incorrect dockerfile specification (can't have two CMD's). Fixed documentation typo --- forcing_engine/testing/Dockerfile | 3 +-- forcing_engine/testing/install_python_packages.sh | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/forcing_engine/testing/Dockerfile b/forcing_engine/testing/Dockerfile index de34cad..609db43 100644 --- a/forcing_engine/testing/Dockerfile +++ b/forcing_engine/testing/Dockerfile @@ -94,5 +94,4 @@ RUN export PATH=$HOME/grib2/wgrib2:$PATH COPY install_python_packages.sh /home/docker COPY run_tests.sh /home/docker -CMD ["/home/docker/install_python_packages.sh"] -CMD ["/home/docker/run_tests.sh"] +CMD ["/bin/bash","-c", "/home/docker/install_python_packages.sh; /home/docker/run_tests.sh"] diff --git a/forcing_engine/testing/install_python_packages.sh b/forcing_engine/testing/install_python_packages.sh index 02bfaa0..0da606b 100755 --- a/forcing_engine/testing/install_python_packages.sh +++ b/forcing_engine/testing/install_python_packages.sh @@ -7,7 +7,7 @@ source /home/docker/.bashrc export PATH=$HOME/grib2/wgrib2:$PATH if [ ! -e "/home/docker/WrfHydroForcing/requirements.txt" ]; then - echo "riequirements.txt not found. Please make sure the WrfHydroForcing branch to test " + echo "requirements.txt not found. Please make sure the WrfHydroForcing branch to test " echo "has been checked out and is mapped to /home/docker/WrfHydroForcing" exit -1 fi From dee822c1caabeb37f05ad1eb8853c2e323e62c0a Mon Sep 17 00:00:00 2001 From: Andrew Gaydos Date: Fri, 8 Sep 2023 11:15:54 -0600 Subject: [PATCH 5/5] Sorted out versioning issues. Now installing mpi4py and esmpy outside of conda, and using python 3.10 --- forcing_engine/testing/Dockerfile | 19 ++++++------ forcing_engine/testing/bash.rc | 30 +++++++++++++++++++ .../testing/install_python_packages.sh | 2 -- 3 files changed, 39 insertions(+), 12 deletions(-) create mode 100644 forcing_engine/testing/bash.rc diff --git a/forcing_engine/testing/Dockerfile b/forcing_engine/testing/Dockerfile index 609db43..2dead2f 100644 --- a/forcing_engine/testing/Dockerfile +++ b/forcing_engine/testing/Dockerfile @@ -11,7 +11,7 @@ USER root # install build environment RUN apt-get update \ - && apt-get -y install wget make cmake gcc gfortran pip vim less rsh-client + && apt-get -y install wget make cmake gcc gfortran pip vim less rsh-client libhdf5-serial-dev libnetcdf-dev # install mpich RUN MPICH_VERSION="4.1" \ @@ -32,7 +32,6 @@ RUN mkdir /home/docker \ && useradd -s /bin/bash -d /home/docker -m docker \ && chown docker /home/docker - USER docker # Place MPI libraries into the LD path. @@ -42,8 +41,8 @@ RUN export INCLUDE_PATH=/usr/local/include # install conda RUN cd $HOME \ - && wget https://repo.anaconda.com/miniconda/Miniconda3-py311_23.5.2-0-Linux-x86_64.sh \ - && bash Miniconda3-py311_23.5.2-0-Linux-x86_64.sh -b + && wget https://repo.anaconda.com/miniconda/Miniconda3-py310_23.5.2-0-Linux-x86_64.sh \ + && bash Miniconda3-py310_23.5.2-0-Linux-x86_64.sh -b RUN export PATH=$HOME/miniconda3/bin:$PATH \ && conda init bash @@ -66,12 +65,16 @@ RUN cd /home/docker/.esmf-src/esmf \ && export ESMF_COMM=mpich3 \ && make install +RUN pip install mpi4py h5py netcdf4 numpy + USER docker -RUN pip install mpi4py h5py netcdf4 numpy +COPY bash.rc /home/docker/.bashrc # install ESMpy -RUN cd $HOME/.esmf-src/esmf/src/addon/esmpy \ +RUN source ~/.bashrc \ + && export ESMFMKFILE=/home/docker/.esmf-src/esmf/DEFAULTINSTALLDIR/lib/libO/Linux.gfortran.64.mpich.default/esmf.mk \ + && cd /home/docker/.esmf-src/esmf/src/addon/esmpy \ && make install # install wgrib2 @@ -84,10 +87,6 @@ RUN cd $HOME \ && make \ && make lib -USER docker - -RUN export PATH=$HOME/grib2/wgrib2:$PATH - #################################### ######### entrypoint ########### #################################### diff --git a/forcing_engine/testing/bash.rc b/forcing_engine/testing/bash.rc new file mode 100644 index 0000000..25f2efa --- /dev/null +++ b/forcing_engine/testing/bash.rc @@ -0,0 +1,30 @@ + +# >>> conda initialize >>> +# !! Contents within this block are managed by 'conda init' !! +__conda_setup="$('/home/docker/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" +if [ $? -eq 0 ]; then + eval "$__conda_setup" +else + if [ -f "/home/docker/miniconda3/etc/profile.d/conda.sh" ]; then + . "/home/docker/miniconda3/etc/profile.d/conda.sh" + else + export PATH="/home/docker/miniconda3/bin:$PATH" + fi +fi +unset __conda_setup +# <<< conda initialize <<< + +export WGRIB2=/home/docker/grib2/wgrib2/wgrib2 + +# make sure we're using mpich and not conda's version +export PATH=/usr/local/bin:$PATH + +export ESMF_DIR=/home/docker/.esmf-src/esmf +export ESMF_COMM=mpich3 + +export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH +export LIBRARY_PATH=/usr/local/lib +export INCLUDE_PATH=/usr/local/include +export ESMFMKFILE=/home/docker/.esmf-src/esmf/DEFAULTINSTALLDIR/lib/libO/Linux.gfortran.64.mpich.default/esmf.mk + +export PYTHONPATH=/usr/local/lib/python3.10/dist-packages:/home/docker/WrfHydroForcing:$PYTHONPATH \ No newline at end of file diff --git a/forcing_engine/testing/install_python_packages.sh b/forcing_engine/testing/install_python_packages.sh index 0da606b..8472135 100755 --- a/forcing_engine/testing/install_python_packages.sh +++ b/forcing_engine/testing/install_python_packages.sh @@ -17,8 +17,6 @@ echo "Upgrading pip" python -m pip install -q --upgrade pip echo "Installing from requirements.txt" pip install -q -r requirements.txt -echo "Installing esmpy and mpi4py" -conda install -y -q -c conda-forge esmpy mpi4py > /dev/null 2>&1 echo "Running setup.py" python setup.py install > /dev/null 2>&1