Add TChem interface for aerosols

CMakeLists.txt

@@ -144,13 +144,27 @@ if(ENABLE_TCHEM)
     PATHS $ENV{TINES_PATH}/lib64 $ENV{TINES_PATH}/lib)
   include_directories(${TINES_INCLUDE_DIR})
   set(CPP_LIB -lstdc++)
-#  find_path(LAPACK_INCLUDE_DIR lapack.h
-#    DOC "LAPACK include directory"
-#    PATHS $ENV{LAPACK_PATH}/include)
   find_library(LAPACK_LIB openblas
-    PATHS $ENV{LAPACK_PATH}/lib)
-#  include_directories(${LAPACK_INCLUDE_DIR})
-  option(ENABLE_TCHEM_GPU "Enable TChem GPU chemistry support" OFF)
+    PATHS $ENV{LAPACK_PATH}/lib  $ENV{LAPACK_PATH}/lib64)
+  find_path(SUNDIALS_INCLUDE_DIR cvode/cvode.h
+    DOC "SUNDIALS include directory (must have cvode/, sundials/, nvector/ subdirs)"
+    PATHS $ENV{SUNDIALS_HOME}/include /opt/local/include /usr/local/include)
+  find_library(SUNDIALS_NVECSERIAL_LIB sundials_nvecserial
+    DOC "SUNDIALS serial vector library"
+    PATHS $ENV{SUNDIALS_HOME}/lib $ENV{SUNDIALS_HOME}/lib64 /opt/local/lib /usr/local/lib)
+  find_library(SUNDIALS_CVODE_LIB sundials_cvode
+    DOC "SUNDIALS CVODE library"
+    PATHS $ENV{SUNDIALS_HOME}/lib $ENV{SUNDIALS_HOME}/lib64 /opt/local/lib /usr/local/lib)
+  find_library(SUNDIALS_CORE_LIB sundials_core
+    DOC "SUNDIALS core library"
+    PATHS $ENV{SUNDIALS_HOME}/lib $ENV{SUNDIALS_HOME}/lib64 /opt/local/lib /usr/local/lib)
+  set(SUNDIALS_LIBS ${SUNDIALS_NVECSERIAL_LIB} ${SUNDIALS_CVODE_LIB} ${SUNDIALS_CORE_LIB})
+  include_directories(${SUNDIALS_INCLUDE_DIR})
+
+  if (NOT DEFINED ENABLE_TCHEM_GPU)
+    option(ENABLE_TCHEM_GPU "Enable TChem GPU chemistry support" OFF)
+  endif()
+
 endif()
 
 ######################################################################
@@ -239,9 +253,9 @@ add_library(partmclib src/aero_state.F90 src/integer_varray.F90
   ${SUNDIALS_SRC} ${GSL_SRC} ${TCHEM_SRC}
   ${C_SORT_SRC})
 
-target_link_libraries(partmclib ${NETCDF_LIBS} ${SUNDIALS_LIBS}
+target_link_libraries(partmclib ${NETCDF_LIBS}
   ${MOSAIC_LIB} ${GSL_LIBS} ${CAMP_LIB} ${TCHEM_LIB} ${YAML_LIB}
-  ${KOKKOS_LIB} ${KOKKOSKERNEL_LIB} ${TINES_LIB} ${CPP_LIB} ${LAPACK_LIB})
+  ${KOKKOS_LIB} ${KOKKOSKERNEL_LIB} ${TINES_LIB} ${CPP_LIB} ${LAPACK_LIB} ${SUNDIALS_LIBS})
 
 if (ENABLE_TCHEM)
    find_package(OpenMP REQUIRED)

Dockerfile.tchem

@@ -44,6 +44,23 @@ WORKDIR /build/kokkos_build/
 RUN make -j \
     && make install
 
+RUN cmake -S /tchem_dir/external/kokkos-kernels -B /build/kokkoskernels_build \
+          -DCMAKE_INSTALL_PREFIX="/install/kokkoskernels_install" \
+          -DCMAKE_CXX_COMPILER=g++ \
+          -DCMAKE_CXX_FLAGS="-g" \
+          -DCMAKE_EXE_LINKER_FLAGS="-lgfortran" \
+          -DKokkosKernels_ENABLE_EXAMPLES=OFF \
+          -DKokkosKernels_ENABLE_EXPERIMENTAL=OFF \
+          -DKokkosKernels_ENABLE_TESTS=OFF \
+          -DKokkosKernels_ENABLE_COMPONENT_BLAS=OFF \
+          -DKokkosKernels_ENABLE_COMPONENT_GRAPH=OFF \
+          -DKokkosKernels_ENABLE_COMPONENT_LAPACK=OFF \
+          -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+          -DKokkos_ROOT=/install/kokkos_install
+WORKDIR /build/kokkoskernels_build
+RUN make -j8 \
+    && make install
+
 RUN cmake -S /tchem_dir/external/Tines/ext/gtest -B /build/gtest_build \
           -DCMAKE_INSTALL_PREFIX="/install/gtest_install" \
           -DCMAKE_CXX_COMPILER=g++
@@ -70,6 +87,10 @@ RUN cmake -S /tchem_dir/external/Sundials -B /build/sundials_build \
           -DCMAKE_C_FLAGS="-g" \
           -DCMAKE_EXE_LINKER_FLAGS="" \
           -DENABLE_CALIPER:BOOL=OFF \
+          -DENABLE_KOKKOS=ON \
+          -DKokkos_DIR=/install/kokkos_install \
+          -DENABLE_KOKKOS_KERNELS=ON \
+          -DKokkosKernels_DIR=/install/kokkoskernels_install \
           -DCMAKE_BUILD_TYPE=RELEASE
 WORKDIR /build/sundials_build
 RUN make -j \
@@ -105,25 +126,6 @@ WORKDIR /build/tines_build
 RUN make -j \
     && make install
 
-RUN cmake -S /tchem_dir/external/kokkos-kernels -B /build/kokkoskernels_build \
-          -DCMAKE_INSTALL_PREFIX="/install/kokkoskernels_install" \
-          -DCMAKE_CXX_COMPILER=g++ \
-          -DCMAKE_CXX_FLAGS="-g" \
-          -DCMAKE_EXE_LINKER_FLAGS="-lgfortran" \
-          -DKokkosKernels_ENABLE_EXAMPLES=OFF \
-          -DKokkosKernels_ENABLE_EXPERIMENTAL=OFF \
-          -DKokkosKernels_ENABLE_TESTS=OFF \
-          -DKokkosKernels_ENABLE_COMPONENT_BLAS=OFF \
-          -DKokkosKernels_ENABLE_COMPONENT_GRAPH=OFF \
-          -DKokkosKernels_ENABLE_COMPONENT_LAPACK=OFF \
-          -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-          -DKokkos_ROOT=/install/kokkos_install
-WORKDIR /build/kokkoskernels_build
-RUN make -j4 \
-    && make install
-
-
-
 RUN cmake -S /tchem_dir/src -B /tchem_build \
           -DCMAKE_INSTALL_PREFIX=/tchem_install \
           -DCMAKE_CXX_COMPILER=g++ \
@@ -137,12 +139,12 @@ RUN cmake -S /tchem_dir/src -B /tchem_build \
           -DTCHEM_ATM_ENABLE_SACADO_JACOBIAN_ATMOSPHERIC_CHEMISTRY=${SACADO} \
           -DKOKKOS_INSTALL_PATH=/install/kokkos_install \
           -DTINES_INSTALL_PATH=/install/tines_install \
+          -DSUNDIALS_INSTALL_PATH=/install/sundials_install \
           -DTCHEM_ATM_ENABLE_SKYWALKER=ON \
           -DTCHEM_ATM_ENABLE_REAL_TYPE="double" \
           -DSKYWALKER_INSTALL_PATH=/install/skywalker_install \
           -DTCHEM_ATM_ENABLE_KOKKOSKERNELS=ON \
           -DKOKKOSKERNELS_INSTALL_PATH=/install/kokkoskernels_install \
-          -DTCHEM_ATM_ENABLE_SKYWALKER=ON \
           -DGTEST_INSTALL_PATH=/install/gtest_install
 WORKDIR /tchem_build
 RUN make -j \
@@ -169,7 +171,6 @@ RUN cd /build \
              -DCMAKE_Fortran_FLAGS_RELEASE="-O2 -g -Werror -fimplicit-none -Wall -Wextra -Wconversion -Wunderflow -Wimplicit-interface -Wno-compare-reals -Wno-unused -Wno-unused-parameter -Wno-unused-dummy-argument -fbounds-check" \
              -D ENABLE_GSL:BOOL=TRUE \
              -D ENABLE_TCHEM:BOOL=TRUE  \
-             -D ENABLE_SUNDIALS:BOOL=TRUE \
              -DCMAKE_C_FLAGS="-lpthread -ldl -fopenmp" \
              -DCMAKE_Fortran_FLAGS="-lpthread -ldl -fopenmp" \
              /partmc \

src/run_part.F90

@@ -593,18 +593,12 @@ subroutine spec_file_read_run_part(file, run_part_opt, aero_data, &
        call spec_file_read_gas_state(sub_file, gas_data, gas_state_init)
        call spec_file_close(sub_file)
 
-       if (.not. run_part_opt%do_camp_chem) then
-          call spec_file_read_string(file, 'aerosol_data', sub_filename)
-          call spec_file_open(sub_filename, sub_file)
-          call spec_file_read_aero_data(sub_file, aero_data)
-          call spec_file_close(sub_file)
-       ! FIXME: Temporary to run PartMC. Replace with initialization from TChem
-       else if (run_part_opt%do_tchem) then
+       if (.not. (run_part_opt%do_camp_chem .or. run_part_opt%do_tchem)) then
           call spec_file_read_string(file, 'aerosol_data', sub_filename)
           call spec_file_open(sub_filename, sub_file)
           call spec_file_read_aero_data(sub_file, aero_data)
           call spec_file_close(sub_file)
-       else
+       else if (run_part_opt%do_camp_chem) then
 #ifdef PMC_USE_CAMP
           call aero_data_initialize(aero_data, camp_core)
           call aero_state_initialize(aero_state, aero_data, camp_core)