diff --git a/configs/containergpucuda/compilers.yaml b/configs/containergpucuda/compilers.yaml new file mode 100644 index 00000000..7bf2d53f --- /dev/null +++ b/configs/containergpucuda/compilers.yaml @@ -0,0 +1,15 @@ +compilers: +- compiler: + spec: gcc@11.4.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: ubuntu22.04 + target: any + modules: [] + extra_rpaths: [] + +# See example: https://spack.readthedocs.io/en/latest/gpu_configuration.html diff --git a/configs/containergpucuda/config.yaml b/configs/containergpucuda/config.yaml new file mode 100644 index 00000000..b7797366 --- /dev/null +++ b/configs/containergpucuda/config.yaml @@ -0,0 +1,2 @@ +config: + build_jobs: 20 diff --git a/configs/containergpucuda/packages.yaml b/configs/containergpucuda/packages.yaml new file mode 100644 index 00000000..d878b7c3 --- /dev/null +++ b/configs/containergpucuda/packages.yaml @@ -0,0 +1,79 @@ +packages: +# Global settings + all: + compiler: + - gcc@11.4.0 + providers: + mpi: [mpich] + blas: [netlib-lapack] + lapack: [netlib-lapack] + variants: build_type=Release cuda_arch=80 + mpi: + require: mpich +# GPU-aware MPICH; See - https://spack.readthedocs.io/en/latest/build_settings.html#package-settings-packages-yaml +# mpich: +# require: "+cuda" +# Use Ubuntu libncurses-dev, etc., b/c Spack version fails +# Spack-pinned version of mpich builds fail + mpich: + externals: + - spec: mpich@4.0 + require: "+cuda" + prefix: /usr + ncurses: + externals: + - spec: ncurses@6.3 + prefix: /usr + gdbm: + externals: + - spec: gdbm@1.23 + prefix: /usr + gdbm6: + externals: + - spec: gdbm6@1.23 + prefix: /usr +# Package preferences to be built by Spack for correct Exawind +# Nota bene: use libtool from Spack for correct linking + ascent: + variants: ~fortran~openmp + amr-wind: + variants: +tiny_profile + conduit: + variants: ~fortran~hdf5_compat + boost: + version: [1.78.0] + variants: cxxstd=17 + cmake: + version: [3.26.3] + variants: build_type=Release + trilinos: + require: + - any_of: ["@13.4.0", "@develop"] + hdf5: + version: [1.10.7] + variants: +cxx+hl + libtool: + version: [2.4.7] + masa: + variants: ~fortran~python + netcdf-c: + require: '@4.7.4' + variants: +parallel-netcdf maxdims=65536 maxvars=524288 + openfast: + version: [master] + variants: +cxx + parallel-netcdf: + version: [1.12.2] + variants: ~fortran + perl: + require: '@5.34.1' + tioga: + version: [develop] + hypre: + require: '@develop' + variants: ~fortran + hypre2: + require: '@develop' + variants: ~fortran + yaml-cpp: + version: [0.6.3] diff --git a/env-templates/exawind_containergpucuda.yaml b/env-templates/exawind_containergpucuda.yaml new file mode 100644 index 00000000..d05720c3 --- /dev/null +++ b/env-templates/exawind_containergpucuda.yaml @@ -0,0 +1,9 @@ +spack: + include: + - include.yaml + concretizer: + unify: false + reuse: false + view: false + specs: + - 'exawind+hypre+amr_wind_gpu+nalu_wind_gpu+cuda' diff --git a/hpc_containers/exawind_container_gpucuda/Dockerfile-containergpucuda b/hpc_containers/exawind_container_gpucuda/Dockerfile-containergpucuda new file mode 100644 index 00000000..4d47d497 --- /dev/null +++ b/hpc_containers/exawind_container_gpucuda/Dockerfile-containergpucuda @@ -0,0 +1,132 @@ +LABEL maintainer="Philip Sakievich, Sandia National Laboratories " + +# NVIDIA base images: https://catalog.ngc.nvidia.com/orgs/nvidia/containers/cuda/tags + +ARG REGISTRY=nvcr.io/nvidia +ARG IMAGE=cuda +ARG TAG=11.8.0-devel-ubuntu22.04 +#ARG TAG=12.2.0-devel-ubuntu22.04 + +FROM ${REGISTRY}/${IMAGE}:${TAG} + +# Make bash the default $SHELL +SHELL ["/bin/bash", "-c"] + +# Install Spack Prereqs: https://spack.readthedocs.io/en/latest/getting_started.html#system-prerequisites + +RUN apt-get update -yqq && \ + apt-get upgrade -yqq + +RUN apt-get install -yqq \ + autoconf \ + automake \ + bzip2 \ + ca-certificates \ + clangd \ + coreutils \ + curl \ + emacs-nox \ + file \ + flex \ + gcc \ + gcc-multilib \ + gcc-doc \ + g++ \ + gfortran \ + gfortran-multilib \ + gfortran-doc \ + git \ + git-doc \ + git-man \ + gnupg2 \ + hwloc-nox \ + libbz2-dev \ + libffi-dev \ + libfmt-dev \ + libgdbm-dev \ + libgdbm6 \ + libgmp-dev \ + libhwloc-common \ + libhwloc-dev \ + libhwloc15 \ + libjpeg-dev \ + libmpc-dev \ + libncurses-dev \ + libtool \ + libtool-bin \ + libtool-doc \ + libx11-dev \ + lsb-release \ + m4 \ + make \ + mpich \ + mpich-doc \ + nano \ + python3 \ + python3-distutils \ + python3-venv \ + unzip \ + vim \ + wget \ + wget2 \ + xz-utils \ + zip \ + zlib1g-dev + +RUN apt clean -y + +# Exawind GPU snapshot +WORKDIR /exawind-entry +# +#RUN git clone --recursive https://github.com/sandialabs/spack-manager +# Pre-merge fork +RUN git clone --recursive https://github.com/ajpowelsnl/spack-manager +# Needed by "create-exawind-snapshot.sh" +ENV SPACK_MANAGER_MACHINE=containergpucuda +ENV CONTAINER_BUILD=gpucuda +ENV SPACK_MANAGER=/exawind-entry/spack-manager + +WORKDIR /exawind-entry/spack-manager + +# Nota bene: commented code is needed, but does not work in container env +# Pre-merge branch from ajpowelsnl/spack-manager fork +# RUN git checkout gpucontainer + +# Temp. code: Use branch of Spack w/ patch +# DOESN'T BUILD CORRECTLY +#RUN cd spack +#RUN git remote add amy_fork https://github.com/ajpowelsnl/spack.git +#RUN git fetch amy_fork +#RUN git checkout amy_fork/spack/patch_yaksa + + +# Snapshot will be generated upon running container +RUN echo "pwd" >> /etc/bash.bashrc && \ + echo "cd spack" >> /etc/bash.bashrc && \ + echo "git remote add amy_fork https://github.com/ajpowelsnl/spack.git" >> /etc/bash.bashrc && \ + echo "git fetch amy_fork" >> /etc/bash.bashrc && \ + echo "git checkout amy_fork/spack/patch_yaksa" >> /etc/bash.bashrc && \ + echo "cd .." >> /etc/bash.bashrc && \ + echo "pwd" >> /etc/bash.bashrc && \ + echo "git checkout gpucontainer" >> /etc/bash.bashrc && \ + echo "export SPACK_MANAGER=$SPACK_MANAGER" >> /etc/bash.bashrc && \ + echo "source $SPACK_MANAGER/start.sh && spack-start" >> /etc/bash.bashrc && \ + echo "spack external find --all" >> /etc/bash.bashrc && \ + echo "$SPACK_MANAGER/scripts/create-exawind-snapshot.sh" >> /etc/bash.bashrc && \ + echo "spack clean --all" >> /etc/bash.bashrc && \ + echo "spack env activate -d snapshots/exawind/containergpucuda/$(date +%Y-%m-%d)" >> /etc/bash.bashrc && \ + echo "spack load exawind" >> /etc/bash.bashrc + +# Verify .bashrc +# RUN ["/bin/bash", "-c", "tail -n 6 /etc/bash.bashrc"] + +# Verify executable: +# 66 spack env activate -d snapshots/exawind/containergpucuda/2023-11-01/ +# 67 spack load exawind +# 68 which exawind +# 69 exawind --help + + + +#WORKDIR /exawind-entry +CMD [ "/bin/bash" ] diff --git a/scripts/create-exawind-snapshot.sh b/scripts/create-exawind-snapshot.sh index 2dcef826..444f9fc6 100755 --- a/scripts/create-exawind-snapshot.sh +++ b/scripts/create-exawind-snapshot.sh @@ -1,4 +1,4 @@ -#!/bin/bash -l +#!/bin/bash # # Copyright (c) 2022, National Technology & Engineering Solutions of Sandia, # LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. @@ -52,6 +52,9 @@ elif [[ "${SPACK_MANAGER_MACHINE}" == "summit" ]]; then elif [[ "${SPACK_MANAGER_MACHINE}" == "perlmutter" ]]; then NUM_CORES=8 cmd "nice -n19 spack manager snapshot -m -s exawind%gcc+hypre+cuda+amr_wind_gpu+nalu_wind_gpu" +elif [[ "${SPACK_MANAGER_MACHINE}" == "containergpucuda" ]]; then + cmd "nice -n19 spack -d manager snapshot -m -s exawind%gcc+hypre+cuda+amr_wind_gpu+nalu_wind_gpu" + NUM_CORES=8 elif [[ "${SPACK_MANAGER_MACHINE}" == "snl-hpc" ]]; then # TODO we should probably launch the install through slurm and exit on this one cmd "nice -n19 spack manager snapshot -s exawind+hypre+openfast amr-wind+hypre+openfast" diff --git a/spack b/spack index ee68baf2..de72189f 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit ee68baf254ce8f401704ef1a62b77057487d4a12 +Subproject commit de72189fec9fa67325a93cb64158f41ec51c697b diff --git a/spack-scripting/scripting/cmd/manager_cmds/find_machine.py b/spack-scripting/scripting/cmd/manager_cmds/find_machine.py index 34cc495e..087e93eb 100644 --- a/spack-scripting/scripting/cmd/manager_cmds/find_machine.py +++ b/spack-scripting/scripting/cmd/manager_cmds/find_machine.py @@ -97,6 +97,9 @@ def is_e4s(): "perlmutter": MachineData( lambda: os.environ["NERSC_HOST"] == "perlmutter", "perlmutter-p1.nersc.gov" ), + "containergpucuda": MachineData( + lambda: os.environ["CONTAINER_BUILD"] == "gpucuda", "containgpucuda.nodomain.gov" + ), # General "darwin": MachineData(lambda: sys.platform == "darwin", "darwin.nodomain.gov"), }