Spack Configuration

This page will show all of our spack configuration files (spack.yaml) used for our production deployments. The spack configuration located in spack-configs directory organized by each subdirectory.

At NERSC, we are building the Extreme-scale Scientific Software Stack (E4S) which is a collection of open-source products software packages part of spack ecosystem for running scientific applications on high-performance computing (HPC) platforms. We acquire the spack configuration from https://github.com/E4S-Project/e4s upon release with list of specs and reference spack branch in order to build the E4S stack. Please see our E4S documentation at https://docs.nersc.gov/applications/e4s/

Perlmutter Spack Develop

This spack configuration will build all packages using spack develop branch on weekly basis. All specs are specified without any version in order to let spack build the latest package which will evolve over time. This stack can be accessible via module load e4s/spack-develop. For more information see https://docs.nersc.gov/applications/e4s/e4s-develop/

spack:
  view: false
  config:
    install_tree:
      concretizer: clingo
    build_stage: $CFS/m3503/perlmutter-spack-develop/spack-stage
    misc_cache: $CFS/m3503/perlmutter-spack-develop/misc_cache
  cdash:
    build-group: DOE nightly E4S builds
    url: https://cdash.spack.io
    project: Spack
    site: NERSC - Perlmutter spack@develop
  compilers:
  - compiler:
      spec: gcc@11.2.0
      paths:
        cc: /opt/cray/pe/craype/default/bin/cc
        cxx: /opt/cray/pe/craype/default/bin/CC
        f77: /opt/cray/pe/craype/default/bin/ftn
        fc: /opt/cray/pe/craype/default/bin/ftn
      flags: {}
      operating_system: sles15
      target: any
      modules:
      - PrgEnv-gnu
      - gcc/11.2.0
      - craype-x86-milan
      - libfabric
  - compiler:
      spec: gcc@10.3.0
      paths:
        cc: /opt/cray/pe/craype/default/bin/cc
        cxx: /opt/cray/pe/craype/default/bin/CC
        f77: /opt/cray/pe/craype/default/bin/ftn
        fc: /opt/cray/pe/craype/default/bin/ftn
      flags: {}
      operating_system: sles15
      target: any
      modules:
      - PrgEnv-gnu
      - gcc/10.3.0
      - craype-x86-milan
      - libfabric
  - compiler:
      spec: cce@13.0.1
      paths:
        cc: /opt/cray/pe/craype/default/bin/cc
        cxx: /opt/cray/pe/craype/default/bin/CC
        f77: /opt/cray/pe/craype/default/bin/ftn
        fc: /opt/cray/pe/craype/default/bin/ftn
      flags: {}
      operating_system: sles15
      target: any
      modules:
      - PrgEnv-cray
      - cce/13.0.1
      - craype-x86-milan
      - libfabric
      environment: {}
      extra_rpaths: []
  - compiler:
      spec: nvhpc@21.11
      paths:
        cc: /opt/cray/pe/craype/default/bin/cc
        cxx: /opt/cray/pe/craype/default/bin/CC
        f77: /opt/cray/pe/craype/default/bin/ftn
        fc: /opt/cray/pe/craype/default/bin/ftn
      flags: {}
      operating_system: sles15
      target: any
      modules:
      - PrgEnv-nvidia
      - nvidia/21.11
      - craype-x86-milan
      - libfabric
  definitions:
  - gcc_compilers: ['%gcc@11.2.0']
  - nvhpc_compilers: ['%nvhpc@21.11']
  - cray_compilers: ['%cce@13.0.1']
  - cray_specs:
    - adios2
    - blaspp
    - fftw
    - hdf5
    - kokkos +openmp +wrapper +cuda cuda_arch=80
    - kokkos-kernels +openmp
    - lapackpp
    - openpmd-api
    - papi
    - petsc +openmp +strumpack
    - slepc
    - strumpack ~slate
    - superlu
    - superlu-dist +openmp
  - gcc_specs:
    - adios2
    - amrex +fortran +hypre +openmp +petsc +shared
    - blaspp
    - gasnet
    - hdf5
    - hipace
    - hypre +openmp +superlu-dist
    - kokkos +openmp +wrapper
    - kokkos-kernels +openmp ^kokkos +openmp +wrapper
    - lapackpp
    - llvm +clang +compiler-rt +libcxx +lld +lldb +llvm_dylib +flang ~cuda
    - openpmd-api
    - papi
    - petsc +openmp +strumpack
    - py-openpmd-viewer +numba
    - py-numba
    - py-warpx ^warpx dims=1
    - py-warpx ^warpx dims=2
    - py-warpx ^warpx dims=3
    - py-warpx ^warpx dims=rz
    - raja
    - slepc
    - strumpack ~slate
    - sundials +openmp +hypre
    - superlu
    - superlu-dist +openmp
    - upcxx +gasnet +mpi
    - warpx dims=1
    - warpx dims=2
    - warpx dims=3
    - warpx dims=rz
  - cuda_specs:
    - amrex +cuda cuda_arch=80
    - blaspp +cuda cuda_arch=80
    - hipace compute=cuda
    - hpctoolkit +cuda +cray +mpi
    - hypre +cuda cuda_arch=80
    - kokkos-kernels +openmp +cuda cuda_arch=80 ^kokkos +openmp +wrapper +cuda cuda_arch=80
    - kokkos +openmp +wrapper +cuda cuda_arch=80
    - lapackpp ^blaspp +cuda cuda_arch=80
    - magma@2.6.1+cuda cuda_arch=80
    - mfem@4.3.0+cuda cuda_arch=80
    - petsc +cuda cuda_arch=80
    - py-warpx ^warpx dims=1 compute=cuda
    - py-warpx ^warpx dims=2 compute=cuda
    - py-warpx ^warpx dims=3 compute=cuda
    - py-warpx ^warpx dims=rz compute=cuda
    - qmcpack +cuda cuda_arch=80
    - raja +cuda cuda_arch=80
    - slepc +cuda cuda_arch=80
    - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack
      +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox
      +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
      +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
    - strumpack ~slate +cuda cuda_arch=80
    - slate +cuda cuda_arch=80
    - superlu-dist +openmp +cuda cuda_arch=80
    - sundials +openmp +cuda cuda_arch=80
    - upcxx +gasnet +mpi
    - umpire ~shared +cuda cuda_arch=80
    - upcxx +cuda
    - warpx dims=1 compute=cuda
    - warpx dims=2 compute=cuda
    - warpx dims=3 compute=cuda
    - warpx dims=rz compute=cuda
    - zfp +cuda cuda_arch=80
  - nvhpc_specs:
     #- adios2 failed due to libffi see https://github.com/libffi/libffi/issues/691
    - amrex +cuda cuda_arch=80
    - blaspp +cuda cuda_arch=80
    - hypre +cuda cuda_arch=80
    - kokkos +openmp +wrapper +cuda cuda_arch=80
    - kokkos-kernels +openmp +cuda cuda_arch=80 ^kokkos +openmp +wrapper +cuda cuda_arch=80
    - lapackpp ^blaspp +cuda cuda_arch=80
    - openpmd-api
    - petsc +cuda cuda_arch=80
    - py-numba
    - raja +cuda cuda_arch=80
    - umpire ~shared +cuda cuda_arch=80
    - upcxx +cuda
    - zfp +cuda cuda_arch=80

  - nersc_specs:
    #- amber+openmp requires tarball and license
    # skipping arm-forge for now this requires a license and gets stuck in CI job.
    #- arm-forge
    - abinit +wannier90
    - amdblis
    - amdfftw
    # requested by user INC0176750. See https://github.com/NVIDIA/AMGX/issues/165 
    - amgx +cuda cuda_arch=80 
    - amdscalapack
    - atompaw
    - berkeleygw
    - boost cxxstd=11
    - boost cxxstd=14
    - boost cxxstd=98
    - cmake
    - dpcpp +openmp
    - eigen
    - elpa
    - fpm
    - lammps
    - llvm-openmp
    - metis
    - mt-metis
    - mumps
    - nccmp
    - nco
    - octave
    - parmetis
    - parallel
    - plumed
    - qmcpack
    - quantum-espresso
    - scotch
    - sparskit
    - superlu-mt
    - wannier90
    - valgrind
    # cuda_arch=80 not supported in spack package yet. See https://github.com/spack/spack/issues/28554
    - cp2k +cuda cuda_arch=70 +elpa +cosma

  specs:
  - matrix:
    - [$cray_specs]
    - [$cray_compilers]
  - matrix:
    - [$gcc_specs]
    - [$gcc_compilers]
  - matrix:
    - [$cuda_specs]
    - [$gcc_compilers]
  - matrix:
    - [$nvhpc_specs]
    - [$nvhpc_compilers]
  - matrix:      
    - [$nersc_specs]
    - [$gcc_compilers]

  gitlab-ci:
    enable-artifacts-buildcache: false
    rebuild-index: true
    before_script:
    - git clone ${SPACK_REPO}
    - pushd spack && git checkout ${SPACK_CHECKOUT_VERSION} && popd
    - . "./spack/share/spack/setup-env.sh"
    script:
    - pushd ${SPACK_CONCRETE_ENV_DIR} && spack env activate --without-view -d $CI_PROJECT_DIR/spack-configs/perlmutter-spack-develop
      && popd
    - spack env st
    - spack -d ci rebuild
    after_script:
    - rm -rf $SPACK_ROOT
    service-job-attributes:
      tags: [perlmutter-login21]
      script:
      - echo "End Pipeline"
    mappings:
    - match: [os=sles15]
      runner-attributes:
        tags: [perlmutter-login21]
  packages:
    all:
      compiler: [gcc@11.2.0, gcc@10.3.0, nvhpc@21.11, cce@13.0.1]
      providers:
        blas: [cray-libsci]
        mpi: [cray-mpich]
        fftw-api: [cray-fftw]
    amrex:
      variants: +fortran +hypre +openmp +petsc +shared
    bzip2:
      version: [1.0.6]
      externals:
      - spec: bzip2@1.0.6
        prefix: /usr
    cray-fftw:
      buildable: false
      # cray-fftw module provided in several MODULEPATH, craype-x86-milan modulefile sets MODULEPATH to cray-fftw optimized build for milan processor
      externals:
      - spec: cray-fftw@3.3.8.12
        modules:
        - craype-x86-milan
        - cray-fftw/3.3.8.12
    cray-libsci:
      buildable: false
      externals:
      - spec: cray-libsci@21.08.1.2
        modules:
        - cray-libsci/21.08.1.2
    cray-mpich:
      buildable: false
      externals:
      - spec: cray-mpich@8.1.15 %gcc@11.2.0
        prefix: /opt/cray/pe/mpich/8.1.15/ofi/gnu/9.1
        modules:
        - cray-mpich/8.1.15
        - cudatoolkit/11.5
      - spec: cray-mpich@8.1.15 %nvhpc@21.11
        prefix: /opt/cray/pe/mpich/8.1.15/ofi/nvidia/20.7
        modules:
        - cray-mpich/8.1.15
        - cudatoolkit/11.5
      - spec: cray-mpich@8.1.15 %cce@13.0.1
        prefix: /opt/cray/pe/mpich/8.1.15/ofi/cray/10.0
        modules:
        - cray-mpich/8.1.15
        - cudatoolkit/11.5
    cuda:
      buildable: false
      version: [11.5.0]
      externals:
      - spec: cuda@11.5.0
        prefix: /opt/nvidia/hpc_sdk/Linux_x86_64/21.11/cuda/11.5
        modules:
        - cudatoolkit/11.5
    diffutils:
      version: [3.6]
      externals:
      - spec: diffutils@3.6
        prefix: /usr
    findutils:
      version: [4.6.0]
      externals:
      - spec: findutils@4.6.0
        prefix: /usr
    hdf5:
      variants: +fortran +hl +shared api=v18
      version: [1.12.1]
    hypre:
      variants: +openmp +superlu-dist
    git:
      version: [2.26.2]
      buildable: false
      externals:
      - spec: git@2.26.2
        prefix: /usr
    gettext:
      version: [0.19.8.1]
      buildable: false
      externals:
      - spec: gettext@0.19.8.1
        prefix: /usr
    libfabric:
      buildable: false
      variants: fabrics=sockets,tcp,udp,rxm
      externals:
      - spec: libfabric@1.11.0.4.414
        prefix: /opt/cray/libfabric/1.11.0.4.114
        modules:
        - libfabric/1.11.0.4.414
    libunwind:
      variants: +pic +xz
    mercury:
      variants: ~bmi
    mesa:
      variants: ~llvm
    mesa18:
      variants: ~llvm
    m4:
      buildable: false
      externals:
      - spec: m4@1.4.18
        prefix: /usr
    mpich:
      variants: ~wrapperrpath
    ncurses:
      variants: +termlib
      externals:
      - spec: ncurses@6.1
        prefix: /usr
    openssl:
      version: [1.1.0i]
      buildable: false
      externals:
      - spec: openssl@1.1.0i
        prefix: /usr
    openssh:
      version: [7.9p1]
      buildable: false
      externals:
      - spec: openssh@7.9p1
        prefix: /usr
    petsc:
      variants: +openmp +strumpack
    pdsh:
      buildable: false
      externals:
      - spec: pdsh@2.34
        prefix: /usr
    readline:
      version: [7.0]
      buildable: false
      externals:
      - spec: readline@7.0
        prefix: /usr
    slurm:
      buildable: false
      version: [20-11-8-1]
      externals:
      - spec: slurm@20-11-8-1
        prefix: /usr
    superlu-dist:
      variants: +openmp
    strumpack:
      variants: ~slate
    tar:
      version: [1.3]
      buildable: false
      externals:
      - spec: tar@1.30
        prefix: /usr
    tcsh:
      version: [6.20.0]
      buildable: false
      externals:
      - spec: tcsh@6.20.0
        prefix: /usr
    trilinos:
      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext
        +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
        +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
        +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
    unzip:
      version: [6.0]
      buildable: false
      externals:
      - spec: unzip@6.0
        prefix: /usr
    util-linux-uuid:
      version: [2.33.1]
      buildable: false
      externals:
      - spec: util-linux-uuid@2.33.1
        prefix: /usr
    xz:
      version: [5.2.3]
      buildable: false
      externals:
      - spec: xz@5.2.3
        prefix: /usr
    zsh:
      version: [5.6]
      buildable: false
      externals:
      - spec: zsh@5.6
        prefix: /usr

Cori Spack Develop

This is the spack configuration for spack develop branch for Cori. Similar to Perlmutter, this stack will build all specs unconstrained with version.

spack:
  view: false
  config:
    build_stage: $CFS/m3503/spack-stage
    misc_cache: $CFS/m3503/misc_cache
    install_tree:
      concretizer: clingo
  compilers:
  - compiler:
      spec: gcc@11.2.0
      paths:
        cc: /opt/cray/pe/craype/default/bin/cc
        cxx: /opt/cray/pe/craype/default/bin/CC
        f77: /opt/cray/pe/craype/default/bin/ftn
        fc: /opt/cray/pe/craype/default/bin/ftn
      flags: {}
      operating_system: cnl7
      target: any
      modules:
      - PrgEnv-gnu
      - gcc/11.2.0
      - craype-haswell
  - compiler:
      spec: intel@19.1.2.254
      paths:
        cc: /opt/cray/pe/craype/default/bin/cc
        cxx: /opt/cray/pe/craype/default/bin/CC
        f77: /opt/cray/pe/craype/default/bin/ftn
        fc: /opt/cray/pe/craype/default/bin/ftn
      flags: {}
      operating_system: cnl7
      target: any
      modules:
      - PrgEnv-intel
      - intel/19.1.2.254
      - craype-haswell
  definitions:
  - gcc_compilers: ['%gcc@11.2.0']
  - intel_compilers: ['%intel@19.1.2.254']
  - gcc_specs:
    - adios2
    - amrex +fortran +hypre +openmp +petsc +shared
    - blaspp
    - fftw
    - gasnet
    - papi
    - hdf5
    - hipace
    - hypre +openmp +superlu-dist
    - kokkos +openmp +wrapper
    - kokkos-kernels +openmp ^kokkos +openmp +wrapper
    - lapackpp
    - petsc +openmp +strumpack
    - raja
    - slepc
    - strumpack ~slate
    - sundials +openmp +hypre
    - superlu
    - superlu-dist +openmp
    - tau +mpi +python
  - intel_specs:
    - adios2
    - amrex +fortran +hypre +openmp +petsc +shared
    - blaspp
    - fftw
    - hdf5
    - hipace
    - lapackpp
    - openpmd-api
    - py-openpmd-viewer +numba
    - py-numba
    - py-warpx ^warpx dims=1
    - py-warpx ^warpx dims=2
    - py-warpx ^warpx dims=3
    - py-warpx ^warpx dims=rz
    - tau +mpi +python
    - upcxx
    - warpx dims=1
    - warpx dims=2
    - warpx dims=3
    - warpx dims=rz    
  - nersc_specs:
    - abinit 
    - cdo 
    - chapel
    - ffmpeg
    - elpa
    - grads
    - gsl
    - gnuplot
    - gromacs
    - libxc
    - libxsmm +shared
    - libint
    - maven
    - metis
    - mpfr
    - octave
    - openjdk
    - parallel
    - parmetis
    - plumed
    - texlive
    - xerces-c
    - valgrind
    - wannier90

  specs:
  - matrix:
    - [$gcc_specs]
    - [$gcc_compilers]
  - matrix:
    - [$intel_specs]
    - [$intel_compilers]
  - $nersc_specs
  packages:
    all:
      compiler: [intel@19.1.2.254, gcc@11.2.0]
      providers:
        blas: [intel-mkl, cray-libsci]
        mpi: [cray-mpich]
        fftw-api: [cray-fftw]
        scalapack: [intel-mkl, cray-libsci]
    amrex:
      variants: +fortran +hypre +openmp +petsc +shared

    bash:
      buildable: false
      externals:
      - spec: bash@4.4.23
        prefix: /
    bzip2:
      version: [1.0.6]
      externals:
      - spec: bzip2@1.0.6
        prefix: /usr
    coreutils:
      buildable: false
      version: [8.29]
      externals:
      - spec: coreutils@8.29
        prefix: /usr
    cpio:
      buildable: false
      externals:
      - spec: cpio@2.12
        prefix: /
    cray-libsci:
      buildable: false
      version: [20.09.1]
      externals:
      - spec: cray-libsci@20.09.1 %gcc
        prefix: /opt/cray/pe/libsci/20.09.1/gnu/8.1
        modules:
        - cray-libsci/20.09.1
      - spec: cray-libsci@20.09.1 %intel
        prefix: /opt/cray/pe/libsci/20.09.1/intel/16.0
        modules:
        - cray-libsci/20.09.1
    cray-fftw:
      buildable: false
      externals:
      - spec: cray-fftw@3.3.8.4
        modules:
        - cray-fftw/3.3.8.4
    cray-mpich:
      buildable: false
      externals:
      - spec: cray-mpich@7.7.19 %intel
        prefix: /opt/cray/pe/mpt/7.7.19/gni/mpich-intel/16.0
        modules:
        - cray-mpich/7.7.19
      - spec: cray-mpich@7.7.19 %gcc
        prefix: /opt/cray/pe/mpt/7.7.19/gni/mpich-gnu/8.2
        modules:
        - cray-mpich/7.7.19
    curl:
      externals:
      - spec: curl@7.66.0+gssapi+ldap+nghttp2
        prefix: /usr
    diffutils:
      version: [3.6]
      externals:
      - spec: diffutils@3.6
        prefix: /usr
    findutils:
      version: [4.6.0]
      externals:
      - spec: findutils@4.6.0
        prefix: /usr
    hdf5:
      variants: +fortran +hl +shared api=v18
    hypre:
      variants: +openmp +superlu-dist
    git:
      version: [2.26.2]
      buildable: false
      externals:
      - spec: git@2.26.2
        prefix: /usr
    gettext:
      version: [0.19.8.1]
      buildable: false
      externals:
      - spec: gettext@0.19.8.1
        prefix: /usr
    intel-mkl:
      buildable: false
      externals:
      - spec: intel-mkl@19.1.2.254
        modules:
        - intel/19.1.2.254
    libunwind:
      variants: +pic +xz
    mercury:
      variants: ~bmi
    mesa:
      variants: ~llvm
    mesa18:
      variants: ~llvm
    m4:
      buildable: false
      externals:
      - spec: m4@1.4.18
        prefix: /usr
    mpich:
      variants: ~wrapperrpath
    ncurses:
      variants: +termlib
      externals:
      - spec: ncurses@6.1
        prefix: /usr
    openssl:
      version: [1.1.0i]
      buildable: false
      externals:
      - spec: openssl@1.1.0i
        prefix: /usr
    openssh:
      version: [7.6p1]
      buildable: false
      externals:
      - spec: openssh@7.6p1
        prefix: /usr
    petsc:
      variants: +openmp +strumpack
    pdsh:
      buildable: false
      externals:
      - spec: pdsh@2.33
        prefix: /usr
    readline:
      version: [7.0]
      buildable: false
      externals:
      - spec: readline@7.0
        prefix: /usr
    sed:
      buildable: false
      externals:
      - spec: sed@4.4
        prefix: /usr
    slurm:
      buildable: false
      version: [20-11-8-1]
      externals:
      - spec: slurm@20-11-8-1
        prefix: /usr
    superlu-dist:
      variants: +openmp
    strumpack:
      variants: ~slate
    tar:
      version: [1.3]
      buildable: false
      externals:
      - spec: tar@1.30
        prefix: /usr
    tcsh:
      version: [6.20.0]
      buildable: false
      externals:
      - spec: tcsh@6.20.0
        prefix: /usr
    trilinos:
      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext
        +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
        +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
        +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
    unzip:
      version: [6.0]
      buildable: false
      externals:
      - spec: unzip@6.0
        prefix: /usr
    util-linux-uuid:
      version: [2.31.1]
      buildable: false
      externals:
      - spec: util-linux-uuid@2.31.1
        prefix: /usr
    xz:
      version: [5.2.3]
      buildable: false
      externals:
      - spec: xz@5.2.3
        prefix: /usr
    zsh:
      version: [5.6]
      buildable: false
      externals:
      - spec: zsh@5.6
        prefix: /usr

Cori E4S 22.02

# This is a Spack Environment file.
#
# It describes a set of packages to be installed, along with
# configuration settings.
spack:
  view: false
  config:
    build_stage: $spack/var/spack/stage
    misc_cache: $spack/var/spack/misc_cache
    install_tree:
      concretizer: clingo
      root: /global/common/software/spackecp/cori/e4s-22.02/software
    module_roots:
      tcl: /global/common/software/spackecp/cori/e4s-22.02/modules
  mirrors:
    source_mirror: /global/common/software/spackecp/mirrors/source_mirror
  modules::
    prefix_inspections:
      bin:
        - PATH
      lib:
        - LIBRARY_PATH
        - LD_LIBRARY_PATH
      lib64:
        - LIBRARY_PATH
        - LD_LIBRARY_PATH
      include:
        - C_INCLUDE_PATH
        - CPLUS_INCLUDE_PATH
        - CPATH
      man:
        - MANPATH
      share/man:
        - MANPATH
      share/aclocal:
        - ACLOCAL_PATH
      lib/pkgconfig:
        - PKG_CONFIG_PATH
      lib64/pkgconfig:
        - PKG_CONFIG_PATH
      share/pkgconfig:
        - PKG_CONFIG_PATH
      '':
        - CMAKE_PREFIX_PATH
    enable:
    - tcl
    tcl:
      blacklist_implicits: true
      hash_length: 0
      naming_scheme: '{name}/{version}-{compiler.name}-{compiler.version}'
      all:
        autoload: direct
        conflict:
        - '{name}'
        environment:
          set:
            '{name}_ROOT': '{prefix}'
      darshan-runtime:
        conflict:
        - darshan
      darshan-util:
        conflict:
        - darshan
      projections:
        all: '{name}/{version}-{compiler.name}-{compiler.version}'
        warpx dims=1: '{name}/{version}-{compiler.name}-{compiler.version}-dims1'
        warpx dims=2: '{name}/{version}-{compiler.name}-{compiler.version}-dims2'
        warpx dims=3: '{name}/{version}-{compiler.name}-{compiler.version}-dims3'
  compilers:
  - compiler:
      spec: gcc@11.2.0
      paths:
        cc: /opt/cray/pe/craype/default/bin/cc
        cxx: /opt/cray/pe/craype/default/bin/CC
        f77: /opt/cray/pe/craype/default/bin/ftn
        fc: /opt/cray/pe/craype/default/bin/ftn
      flags: {}
      operating_system: cnl7
      target: any
      modules:
      - PrgEnv-gnu
      - gcc/11.2.0
      - craype-haswell
  - compiler:
      spec: intel@19.1.2.254
      paths:
        cc: /opt/cray/pe/craype/default/bin/cc
        cxx: /opt/cray/pe/craype/default/bin/CC
        f77: /opt/cray/pe/craype/default/bin/ftn
        fc: /opt/cray/pe/craype/default/bin/ftn
      flags: {}
      operating_system: cnl7
      target: any
      modules:
      - PrgEnv-intel
      - intel/19.1.2.254
      - craype-haswell
  definitions:
  - gcc_compilers: ['%gcc@11.2.0']
  - intel_compilers: ['%intel@19.1.2.254']
  - gcc_specs:
    - adios2@2.7.1
    - amrex@22.02
    - aml@0.1.0
    - arborx@1.1
    - argobots@1.1
    - axom@0.6.1
    - bolt@2.0
    - caliper@2.7.0
    - chai@2.4.0 ~benchmarks ~tests
    - conduit@0.8.2
    - darshan-runtime@3.3.1
    - darshan-util@3.3.1
    - dyninst@12.0.1
    - faodel@1.2108.1
    - flecsi@1.4.2
    - flit@2.1.0
    - gasnet@2021.9.0
    - ginkgo@1.4.0
    - globalarrays@5.8
    - gotcha@1.0.3
    - hdf5@1.10.7 +fortran +hl +shared
    - hdf5@1.12.1 +fortran +hl +shared
    - hdf5@1.13.0 +fortran +hl +shared
    - heffte@2.2.0 +fftw
    - hpx@1.7.1 networking=mpi
    - hypre@2.24.0
    - kokkos@3.5.00 +openmp +wrapper
    - kokkos-kernels@3.5.00 +openmp ^kokkos@3.5.00 +openmp +wrapper
    - legion@21.03.0
    - libquo@1.3.1
    - libunwind@1.5.0
    - mercury@2.1.0
    - metall@0.17
    - mfem@4.3.0
    - mpark-variant@1.4.0
    - nccmp@1.9.0.1 ^netcdf-c@4.8.1
    - nco@5.0.1
    - netlib-scalapack@2.1.0
    - ninja@1.10.2
    - nvhpc@22.1
    - openpmd-api@0.14.4
    - papi@6.0.0.1
    - papyrus@1.0.1
    - parallel-netcdf@1.12.2
    - parsec@3.0.2012 ~cuda
    - pdt@3.25.1
    - petsc@3.16.4 +openmp
    - pumi@2.2.6
    - qthreads@1.16 scheduler=distrib
    - raja@0.14.0
    - stc@0.9.0
    - strumpack@6.3.0~butterflypack~slate
    - sundials@6.1.1
    - superlu@5.3.0
    - superlu-dist@7.2.0
    - swig@4.0.2
    - sz@2.1.12
    - tasmanian@7.7
    - tau@2.31 +mpi +python
    - turbine@1.3.0
    - umap@2.1.0
    - umpire@6.0.0
    - upcxx@2021.9.0
    - veloc@1.5
    - vtk-m@1.7.1
    - zfp@0.5.5
   #- hpctoolkit@2022.01.15
   #- phist@1.9.5 #Unable to locate cray-libsci headers in /opt/cray/pe/libsci/20.09.1/gnu/8.1/include
   #- mpifileutils@0.11.1 ~xattr # failed to install libcircle Unable to find suitable MPI Compiler. Try setting MPICC.
   #- plumed@2.6.3  # Build Failed
   #-precice@2.3.0 # Build failed due to petsc
   #- py-warpx@22.02 ^warpx dims=rz
   #- rempi@1.1.0 failed to find MPICC
   #- slate@2021.05.02 ~cuda # Build failure on blaspp. BLAS++ requires a BLAS library and none was found.
   #- scr@3.0rc2  Build failure
   #- trilinos@13.0.1 +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
    #- unifyfs@0.9.1 # Build failure on dependency mercury
    #- wannier90@3.1.0 #  Error: A PrgEnv-* modulefile must be loaded.
  - intel_specs:
    - adios2@2.7.1
    - arborx@1.1
    - argobots@1.1
    - caliper@2.7.0
    - conduit@0.8.2
    - chai@2.4.0 ~benchmarks ~tests
    - darshan-runtime@3.3.1
    - darshan-util@3.3.1
    - faodel@1.2108.1
    - flecsi@1.4.2
    - flit@2.1.0
    - gasnet@2021.9.0
    - ginkgo@1.4.0
    - globalarrays@5.8
    - gotcha@1.0.3
    - hdf5@1.10.7 +fortran +hl +shared
    - hdf5@1.12.1 +fortran +hl +shared
    - hdf5@1.13.0 +fortran +hl +shared
    - heffte@2.2.0 +fftw
    - hypre@2.24.0
    - kokkos@3.5.00 +openmp +wrapper
    - legion@21.03.0
    - libquo@1.3.1
    - libunwind@1.5.0
    - loki@0.1.7
    - mercury@2.1.0
    - metall@0.17
    - mfem@4.3.0
    - mpark-variant@1.4.0
    - nccmp@1.9.0.1 ^netcdf-c@4.8.1
    - netlib-scalapack@2.1.0
    - ninja@1.10.2
    - openpmd-api@0.14.4
    - papi@6.0.0.1
    - parallel-netcdf@1.12.2
    - parsec@3.0.2012 ~cuda
    - papyrus@1.0.1
    - petsc@3.16.4 +openmp
    - pdt@3.25.1
    - precice@2.3.0
    - pumi@2.2.6
    - qthreads@1.16 scheduler=distrib
    - raja@0.14.0
    - slepc@3.16.2
    - strumpack@6.3.0~butterflypack ~slate
    - sundials@6.1.1
    - superlu@5.3.0
    - superlu-dist@7.2.0
    - swig@4.0.2
    - sz@2.1.12
    - tasmanian@7.7
    - turbine@1.3.0
    - umap@2.1.0
    - umpire@6.0.0
    - upcxx@2021.9.0
    - variorum@0.4.1
    - veloc@1.5
    - vtk-m@1.7.1
    - wannier90@3.1.0
    - warpx dims=1
    - warpx dims=2
    - warpx dims=3
    - zfp@0.5.5
    #- axom@0.6.1 # Build failure in cmake for axom
    #- butterflypack@2.1.0 # Build failure  sed: can't read *.inc: No such file or directory
    #- dyninst@12.0.1 # %intel conflict
    # - hpx@1.7.1 networking=mpi # Failed on asio
    #- lammps@20220107 # Build Failure during cmake
    #- kokkos-kernels@3.5.00 +openmp ^kokkos@3.5.00 +openmp +wrapper  # Build failure
    #- plasma@21.8.29 # %intel conflict
    #- phist@1.9.5  # Error: NoHeadersError: Unable to locate cray-libsci headers in /opt/cray/pe/libsci/20.09.1/intel/16.0/include
    #- plumed@2.6.3
    #- rempi@1.1.0  # FAILED to find MPICC
    #- slate@2021.05.02 ~cuda # %intel conflict
    #- scr@3.0rc2   Build failure
    #- trilinos@13.0.1 +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
    #- warpx dims=rz failed on blaspp

  - nersc_specs:
    - cdo@2.0.3 +curl
    - chapel@1.24.1
    - ffmpeg@4.4.1
    - elpa@2021.11.001
    - grads@2.2.1
    - gsl@2.7
    - gromacs@2021.5
    - libxc@5.1.7
    - libxsmm@1.17 +shared
    - libint@2.6.0
    - nano@4.9
    - maven@3.8.4
    - metis@5.1.0
    - octave@6.4.0
    - openjdk@11.0.12_7
    - parallel@20210922
    - parmetis@4.0.3
    - texlive
    - xerces-c@3.2.3

    #- abinit+openmp failed to install netcdf-fortran
    #- gnuplot build failure on gnuplot
    #- valgrind@3.18.1 Build failure
  specs:
  - matrix:
    - [$gcc_specs]
    - [$gcc_compilers]
  - matrix:
    - [$intel_specs]
    - [$intel_compilers]
  - matrix:
    - [$nersc_specs]
    - [$gcc_compilers]
  packages:
    all:
      compiler: [gcc@11.2.0, intel@19.1.2.254]
      providers:
        blas: [cray-libsci, intel-mkl]
        fftw-api: [cray-fftw]
        mpi: [cray-mpich]
        scalapack: [cray-libsci, intel-mkl]
    amrex:
      variants: +fortran +hypre +openmp +shared
    bash:
      buildable: false
      externals:
      - spec: bash@4.4.23
        prefix: /
    bzip2:
      version: [1.0.6]
      externals:
      - spec: bzip2@1.0.6
        prefix: /usr
    coreutils:
      buildable: false
      version: [8.29]
      externals:
      - spec: coreutils@8.29
        prefix: /usr
    cpio:
      buildable: false
      externals:
      - spec: cpio@2.12
        prefix: /
    cray-libsci:
      buildable: false
      version: [20.09.1]
      externals:
      - spec: cray-libsci@20.09.1 %gcc
        prefix: /opt/cray/pe/libsci/20.09.1/gnu/8.1
        modules:
        - cray-libsci/20.09.1
      - spec: cray-libsci@20.09.1 %intel
        prefix: /opt/cray/pe/libsci/20.09.1/intel/16.0
        modules:
        - cray-libsci/20.09.1
    cray-fftw:
      buildable: false
      externals:
      - spec: cray-fftw@3.3.8.10
        modules:
        - cray-fftw/3.3.8.10
    cray-mpich:
      buildable: false
      externals:
      - spec: cray-mpich@7.7.19 %intel
        prefix: /opt/cray/pe/mpt/7.7.19/gni/mpich-intel/16.0
        modules:
        - cray-mpich/7.7.19
      - spec: cray-mpich@7.7.19 %gcc
        prefix: /opt/cray/pe/mpt/7.7.19/gni/mpich-gnu/8.2
        modules:
        - cray-mpich/7.7.19
    curl:
      externals:
      - spec: curl@7.66.0+gssapi+ldap+nghttp2
        prefix: /usr
    diffutils:
      version: [3.6]
      externals:
      - spec: diffutils@3.6
        prefix: /usr
    elfutils:
      version: [0.168]
      externals:
      - spec: elfutils@0.168
        prefix: /usr
    findutils:
      version: [4.6.0]
      externals:
      - spec: findutils@4.6.0
        prefix: /usr
    hdf5:
      variants: +fortran +hl +shared api=v18
      version: [1.12.1]
      externals:
      - spec: hdf5@1.12.1.1%intel+shared+fortran+hl~mpi
        modules:
        - cray-hdf5/1.12.1.1
      - spec: hdf5@1.12.1.1%intel+shared+fortran+hl+mpi
        modules:
        - cray-hdf5-parallel/1.12.1.1
      - spec: hdf5@1.12.1.1%gcc+shared+fortran+hl~mpi
        modules:
        - cray-hdf5/1.12.1.1
      - spec: hdf5@1.12.1.1%gcc+shared+fortran+hl+mpi
        modules:
        - cray-hdf5-parallel/1.12.1.1
    hypre:
      variants: +openmp +superlu-dist
    gawk:
      buildable: false
      externals:
      - spec: gawk@4.2.1
        prefix: /usr
    git:
      version: [2.26.2]
      buildable: false
      externals:
      - spec: git@2.26.2
        prefix: /usr
    gmake:
      buildable: false
      externals:
      - spec: gmake@4.2.1
        prefix: /usr
    intel-mkl:
      buildable: false
      externals:
      - spec: intel-mkl@19.1.2.254
        modules:
        - intel/19.1.2.254
    krb5:
      buildable: false
      externals:
      - spec: krb5@1.16.3
        prefix: /usr/lib/mit
    libunwind:
      variants: +pic +xz
    mercury:
      variants: ~bmi
    mesa:
      variants: ~llvm
    mesa18:
      variants: ~llvm
    m4:
      buildable: false
      externals:
      - spec: m4@1.4.18
        prefix: /usr
    mpich:
      variants: ~wrapperrpath
    ncurses:
      variants: +termlib
      externals:
      - spec: ncurses@6.1
        prefix: /usr
    netcdf-c:
      version: [4.8.1.1]
      externals:
      - spec: netcdf-c@4.8.1.1~mpi
        modules:
        - cray-netcdf/4.8.1.1
      - spec: netcdf-c@4.8.1.1+mpi
        modules:
        - cray-netcdf-hdf5parallel/4.8.1.1
    openssl:
      buildable: false
      version: [1.1.1d]
      externals:
      - spec: openssl@1.1.1d
        prefix: /usr
    openssh:
      buildable: false
      externals:
      - spec: openssh@8.1p1
        prefix: /usr
    pdsh:
      buildable: false
      externals:
      - spec: pdsh@2.33
        prefix: /usr
    petsc:
      variants: +openmp
    perl:
      version: [5.32.0]
    pkg-config:
      buildable: false
      version: [0.29.2]
      externals:
      - spec: pkg-config@0.29.2
        prefix: /usr
    readline:
      version: [7.0]
      buildable: false
      externals:
      - spec: readline@7.0
        prefix: /usr
    sed:
      buildable: false
      externals:
      - spec: sed@4.4
        prefix: /usr
    slurm:
      buildable: false
      version: [20-11-8-1]
      externals:
      - spec: slurm@20-11-8-1
        prefix: /usr
    superlu-dist:
      variants: +openmp
    strumpack:
      variants: ~slate
    tar:
      version: [1.3]
      buildable: false
      externals:
      - spec: tar@1.30
        prefix: /usr
    tcsh:
      version: [6.20.0]
      buildable: false
      externals:
      - spec: tcsh@6.20.0
        prefix: /usr
    trilinos:
      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext
        +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
        +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
        +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
    unzip:
      version: [6.0]
      buildable: false
      externals:
      - spec: unzip@6.0
        prefix: /usr
    util-linux-uuid:
      version: [2.31.1]
      buildable: false
      externals:
      - spec: util-linux-uuid@2.31.1
        prefix: /usr
    zsh:
      version: [5.6]
      buildable: false
      externals:
      - spec: zsh@5.6
        prefix: /usr

Perlmutter E4S 21.11

Shown below the production spack configuration for Perlmutter E4S 21.11. You can access this stack via module load e4s/21.11-tcl or module load e4s/21.11-lmod on Perlmutter. Please see our user documentation for this stack at https://docs.nersc.gov/applications/e4s/perlmutter/21.11/.

# This is a Spack Environment file.
#
# It describes a set of packages to be installed, along with
# configuration settings.
spack:
  view: false
  config:
    build_stage: $CFS/m3503/spack-stage
    misc_cache: $CFS/m3503/misc_cache
    install_tree:
      concretizer: clingo
      root: /global/common/software/spackecp/perlmutter/e4s-21.11/software
    module_roots:
      tcl: /global/common/software/spackecp/perlmutter/e4s-21.11/modules/tcl
      lmod: /global/common/software/spackecp/perlmutter/e4s-21.11/modules/lmod
  modules:
    prefix_inspections:
      include: [C_INCLUDE_PATH, CPLUS_INCLUDE_PATH, CPATH]
      lib: [LIBRARY_PATH, LD_LIBRARY_PATH]
      lib64: [LIBRARY_PATH, LD_LIBRARY_PATH]
    default:
      enable:
      - lmod
      lmod:
        blacklist_implicits: true
        hash_length: 0
        core_compilers:
        - gcc@7.5.0
        hierarchy:
        - mpi
        defaults:
        - gcc
        - nvhpc
        - cray-mpich@8.1.13 %gcc
        - cray-mpich@8.1.13 %nvhpc
        all:
          autoload: direct
          environment:
            set:
              '{name}_ROOT': '{prefix}'
          suffixes:
            ^cuda: cuda
        warpx:
          suffixes:
            warpx dims=2: dims2
            warpx dims=3: dims3
        projections:
          all: '{name}/{version}'
    tcl:
      hash_length: 0
      naming_scheme: '{name}/{version}-{compiler.name}-{compiler.version}'
      blacklist_implicits: true
      all:
        autoload: direct
        conflict:
        - '{name}'
        environment:
          set:
            '{name}_ROOT': '{prefix}'
        suffixes:
          ^cray-mpich: mpi
          ^cuda: cuda
      warpx:
        suffixes:
          warpx dims=2: dims2
          warpx dims=3: dims3
      blacklist:
      - gcc
      - nvhpc
      - cray-mpich
  compilers:
  - compiler:
      spec: gcc@7.5.0
      paths:
        cc: /usr/bin/gcc
        cxx: /usr/bin/g++
        f77: /usr/bin/gfortran
        fc: /usr/bin/gfortran
      flags: {}
      operating_system: sles15
      target: any
      modules: []
  - compiler:
      spec: nvhpc@21.11
      paths:
        cc: /opt/cray/pe/craype/default/bin/cc
        cxx: /opt/cray/pe/craype/default/bin/CC
        f77: /opt/cray/pe/craype/default/bin/ftn
        fc: /opt/cray/pe/craype/default/bin/ftn
      flags: {}
      operating_system: sles15
      target: any
      modules:
      - PrgEnv-nvidia
      - nvidia/21.11
      - craype-x86-milan
      - libfabric
  - compiler:
      spec: gcc@11.2.0
      paths:
        cc: cc
        cxx: CC
        f77: ftn
        fc: ftn
      flags: {}
      operating_system: sles15
      target: any
      modules:
      - PrgEnv-gnu
      - gcc/11.2.0
      - craype-x86-milan
      - libfabric
      extra_rpaths: []
  definitions:
  - gcc_compilers: ['%gcc@11.2.0']
  - nvhpc_compilers: ['%nvhpc@21.11']
  - compilers: [gcc@11.2.0 %gcc@7.5.0, nvhpc@21.11 %gcc@7.5.0]
  - mpis: [cray-mpich@8.1.13 %gcc@11.2.0, cray-mpich@8.1.13 %nvhpc@21.11]
  - gcc_specs:
    - adios2@2.7.1
    - amrex@21.11 +fortran +hypre +openmp +petsc +shared
    - conduit@0.7.2
    - dyninst@11.0.1
    - gasnet@2021.9.0
    - globalarrays@5.8
    - hdf5@1.12.1
    - hypre@2.23.0 +openmp +superlu-dist
    - kokkos-kernels@3.4.01 +openmp
    - kokkos@3.4.01 +openmp
    - mercury@2.0.1
    - mfem@4.3.0
    - mpark-variant@1.4.0
    - openpmd-api@0.14.3
    - papi@6.0.0.1
    - papyrus@1.0.2
    - parsec@3.0.2012 ~cuda
    - pdt@3.25.1
    - petsc@3.16.1 +openmp +strumpack
    - qthreads@1.16 scheduler=distrib
    - raja@0.14.0
    - slepc@3.16.0
    - strumpack@6.1.0 ~slate
    - sundials@5.8.0 +openmp +hypre
    - superlu-dist@7.1.1 +openmp
    - superlu@5.3.0
    - swig@4.0.2
    - sz@2.1.12
    - tau +mpi +python
    - trilinos@13.0.1 +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
    - umap@2.1.0
    - upcxx@2021.9.0 +gasnet +mpi
    #- warpx dims=2
    #- warpx dims=3
  - cuda_specs:
    - hypre@2.23.0+cuda cuda_arch=80
    - kokkos-kernels@3.4.01 +openmp +cuda cuda_arch=80 ^kokkos +openmp +wrapper +cuda cuda_arch=80
    - kokkos@3.4.01 +openmp +wrapper +cuda cuda_arch=80
    - mfem@4.3.0+cuda cuda_arch=80
    - petsc@3.16.1 +cuda cuda_arch=80
    - slepc@3.16.0 +cuda cuda_arch=80 ^petsc@3.16.1 +cuda cuda_arch=80
    - umpire@6.0.0 ~shared +cuda cuda_arch=80
    - upcxx@2021.9.0 +cuda
    - zfp@0.5.5 +cuda cuda_arch=80
    - raja@0.14.0+cuda cuda_arch=80 # CUB in your include path is not compatible with this release of Thrust
    #- parsec@3.0.2012+cuda cuda_arch=80 # parsec/mca/device/cuda/transfer.c:168: multiple definition of `parsec_CUDA_d2h_max_flows'
    #- amrex@21.11 +cuda cuda_arch=80
    #- magma@2.6.1+cuda cuda_arch=80
    #- strumpack@6.1.0 ~slate +cuda cuda_arch=80
    #- slate@2021.05.02 +cuda cuda_arch=80
    #- superlu-dist@7.1.1 +openmp +cuda cuda_arch=80
    #- sundials@5.8.0 +openmp +cuda cuda_arch=80
  - nvhpc_specs:
    - metis@5.1.0
    - umpire@6.0.0 ~shared +cuda cuda_arch=80
    - upcxx@2021.9.0 +cuda
    - zfp@0.5.5 +cuda cuda_arch=80
    #- hypre@2.23.0 +cuda cuda_arch=80
    # unable to install for nvhpc compiler
    #- amrex@21.11 +cuda cuda_arch=80
    #- petsc@3.16.1 +cuda cuda_arch=80
    #- kokkos-kernels@3.4.01 +openmp +wrapper +cuda cuda_arch=80 ^kokkos +openmp +wrapper +cuda cuda_arch=80
    #- kokkos@3.4.01 +openmp +wrapper +cuda cuda_arch=80
    #- raja@0.14.0+cuda cuda_arch=80
  - nersc_specs:
    - chapel@1.24.1
    - gsl@2.7
    - fftw@3.3.10
    - nccmp@1.9.0.1
    - netcdf-c@4.8.1
    - netcdf-fortran@4.5.3
    - nco@5.0.1
    - metis@5.1.0
    - parallel-netcdf@1.12.2
    - parmetis@4.0.3
    - gromacs@2021.3
    #- plumed@2.6.3
    #- wannier90@3.1.0
  specs:
  - matrix:
    - [$gcc_specs]
    - [$gcc_compilers]
  - matrix:
    - [$cuda_specs]
    - [$gcc_compilers]
  - matrix:
    - [$nvhpc_specs]
    - [$nvhpc_compilers]
  - $compilers
  - $mpis
  - $nersc_specs
  packages:
    all:
      compiler: [gcc@11.2.0, nvhpc@21.11]
      providers:
        blas: [cray-libsci]
        mpi: [cray-mpich]
    amrex:
      variants: +fortran +hypre +openmp +petsc +shared
    bzip2:
      version: [1.0.6]
      externals:
      - spec: bzip2@1.0.6
        prefix: /usr
    cray-libsci:
      buildable: false
      externals:
      - spec: cray-libsci@21.08.1.2
        modules:
        - cray-libsci/21.08.1.2
    cray-mpich:
      buildable: false
      externals:
      - spec: cray-mpich@8.1.13 %gcc@11.2.0
        prefix: /opt/cray/pe/mpich/8.1.13/ofi/gnu/9.1
        modules:
        - cray-mpich/8.1.13
        - cudatoolkit/11.5
      - spec: cray-mpich@8.1.13 %nvhpc@21.11
        prefix: /opt/cray/pe/mpich/8.1.13/ofi/nvidia/20.7
        modules:
        - cray-mpich/8.1.13
        - cudatoolkit/11.5
    cuda:
      buildable: false
      version: [11.5.0]
      externals:
      - spec: cuda@11.5.0
        prefix: /opt/nvidia/hpc_sdk/Linux_x86_64/21.11/cuda/11.5
        modules:
        - cudatoolkit/11.5
    cub:
      buildable: false
      externals:
      - spec: cub@1.13.1
        prefix: /opt/nvidia/hpc_sdk/Linux_x86_64/21.11/math_libs/11.5
        modules:
        - cudatoolkit/11.5
    curl:
      buildable: false
      externals:
      - spec: curl@7.66.0
        prefix: /usr
    diffutils:
      version: [3.6]
      externals:
      - spec: diffutils@3.6
        prefix: /usr
    findutils:
      version: [4.6.0]
      externals:
      - spec: findutils@4.6.0
        prefix: /usr
    hdf5:
      variants: +fortran +hl +shared api=v18
      version: [1.12.1]
    hypre:
      variants: +openmp +superlu-dist
    gcc:
      buildable: false
      externals:
      - spec: gcc@11.2.0 %gcc@7.5.0
        modules:
        - PrgEnv-gnu
        - gcc/11.2.0
        - craype-x86-milan
        - libfabric
    git:
      version: [2.26.2]
      buildable: false
      externals:
      - spec: git@2.26.2
        prefix: /usr
    libfabric:
      buildable: false
      variants: fabrics=sockets,tcp,udp,rxm
      externals:
      - spec: libfabric@1.11.0.4.75
        prefix: /opt/cray/libfabric/1.11.0.4.75
        modules:
        - libfabric/1.11.0.4.75
    libunwind:
      variants: +pic +xz
    mercury:
      variants: ~bmi
    mesa:
      variants: ~llvm
    mesa18:
      variants: ~llvm
    m4:
      buildable: false
      externals:
      - spec: m4@1.4.18
        prefix: /usr
    mpich:
      variants: ~wrapperrpath
    ncurses:
      variants: +termlib
      externals:
      - spec: ncurses@6.1
        prefix: /usr
    nvhpc:
      buildable: false
      externals:
      - spec: nvhpc@21.11 %gcc@7.5.0
        modules:
        - PrgEnv-nvidia
        - nvidia/21.11
        - craype-x86-milan
        - libfabric
    openssl:
      version: [1.1.0i]
      buildable: false
      externals:
      - spec: openssl@1.1.0i
        prefix: /usr
    openssh:
      version: [7.9p1]
      buildable: false
      externals:
      - spec: openssh@7.9p1
        prefix: /usr
    petsc:
      variants: +openmp +strumpack
    pdsh:
      buildable: false
      externals:
      - spec: pdsh@2.34
        prefix: /usr
    readline:
      version: [7.0]
      buildable: false
      externals:
      - spec: readline@7.0
        prefix: /usr
    slurm:
      buildable: false
      version: [20-11-8-1]
      externals:
      - spec: slurm@20-11-8-1
        prefix: /usr
    superlu-dist:
      variants: +openmp
    strumpack:
      variants: ~slate
    tar:
      version: [1.3]
      buildable: false
      externals:
      - spec: tar@1.30
        prefix: /usr
    trilinos:
      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext
        +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
        +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
        +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
    unzip:
      version: [6.0]
      buildable: false
      externals:
      - spec: unzip@6.0
        prefix: /usr
    util-linux-uuid:
      version: [2.33.1]
      buildable: false
      externals:
      - spec: util-linux-uuid@2.33.1
        prefix: /usr
    xz:
      version: [5.2.3]
      buildable: false
      externals:
      - spec: xz@5.2.3
        prefix: /usr
    zsh:
      version: [5.6]
      buildable: false
      externals:
      - spec: zsh@5.6
        prefix: /usr

Cori E4S 21.05

spack:
  view: false
  concretization: separately
  config:
    install_tree:
      root: /global/common/software/spackecp/e4s-21.05/software
    module_roots:
      tcl: /global/common/software/spackecp/e4s-21.05/modules/
    build_stage: $tempdir/user/spack-stage
  modules:
    enable:
    - tcl
    tcl:
      blacklist_implicits: true
      hash_length: 0
      naming_scheme: '{name}/{version}-{compiler.name}-{compiler.version}'
      all:
        conflict:
        - '{name}'
        environment:
          set:
            '{name}_ROOT': '{prefix}'
      darshan-runtime:
        conflict:
        - darshan
      darshan-util:
        conflict:
        - darshan
      projections:
        all: '{name}/{version}-{compiler.name}-{compiler.version}'
        py-warpx ^warpx dims=2: '{name}/{version}-dims2'
        py-warpx ^warpx dims=3: '{name}/{version}-dims3'
        py-warpx ^warpx dims=rz: '{name}/{version}-dimsRZ'
        warpx dims=2: '{name}/{version}-dims2'
        warpx dims=3: '{name}/{version}-dims3'
        warpx dims=rz: '{name}/{version}-dimsRZ'
        boost cxxstd=98: '{name}/{version}-cxxstd=98'
        boost cxxstd=17: '{name}/{version}-cxxstd=17'
        kokkos +openmp: '{name}/{version}-openmp'
        kokkos ~openmp: '{name}/{version}'
  mirrors:
    cori-e4s-21.05: https://cache.e4s.io/21.05
  compilers:
  - compiler:
      spec: intel@19.1.3.304
      paths:
        cc: cc
        cxx: CC
        f77: ftn
        fc: ftn
      flags: {}
      operating_system: cnl7
      target: any
      modules:
      - PrgEnv-intel
      - intel/19.1.3.304
      environment: {}
      extra_rpaths: []
  packages:
    all:
      compiler:
      - intel@19.1.3.304
      providers:
        blas:
        - openblas
        mpi:
        - mpich
      target:
      - haswell
      variants: +mpi
    slurm:
      buildable: false
      version: [20-02-4-1]
      externals:
      - spec: slurm@20-02-4-1
        prefix: /usr
    mpich:
      buildable: false
      externals:
      - spec: mpich@3.1
        modules:
        - cray-mpich/7.7.10
    autoconf:
      version:
      - '2.69'
    automake:
      version:
      - 1.16.3
    berkeley-db:
      version:
      - 18.1.40
    binutils:
      variants: +ld +gold +headers +libiberty ~nls +plugins
      version:
      - 2.33.1
    boost:
      version:
      - 1.75.0
    bzip2:
      version:
      - 1.0.8
    c-blosc:
      version:
      - 1.21.0
    cmake:
      version:
      - 3.20.2
    curl:
      version:
      - 7.76.0
    diffutils:
      version:
      - 3.7
    elfutils:
      version:
      - 0.182
      variants: +bzip2 ~nls +xz
    expat:
      version:
      - 2.2.10
    findutils:
      version:
      - 4.8.0
    gdbm:
      version:
      - 1.18.1
    gettext:
      version:
      - 0.21
    git:
      version:
      - 2.31.0
    glib:
      version:
      - 2.66.8
    hdf5:
      variants: +fortran +hl +shared
      version:
      - 1.10.7
    help2man:
      version:
      - 1.47.16
    hwloc:
      version:
      - 2.4.1
    json-c:
      version:
      - 0.13.1
    libbsd:
      version:
      - 0.10.0
    libfabric:
      version:
      - 1.12.1
      variants: fabrics=sockets,tcp,udp,rxm
    libiconv:
      version:
      - 1.16
    libsigsegv:
      version:
      - 2.12
    libpciaccess:
      version:
      - 0.16
    libtool:
      version:
      - 2.4.6
    libunwind:
      version:
      - 1.5.0
      variants: +pic +xz
    libxml2:
      version:
      - 2.9.10
    lz4:
      version:
      - 1.9.3
    m4:
      version:
      - 1.4.18
    mesa:
      variants: ~llvm
    mesa18:
      variants: ~llvm
    ncurses:
      version:
      - 6.2
      variants: +termlib
    numactl:
      version:
      - 2.0.14
    openblas:
      version:
      - 0.3.10
      variants: threads=openmp
    perl:
      version:
      - 5.32.0
    pkgconf:
      version:
      - 1.7.3
    python:
      version:
      - 3.8.10
    readline:
      version:
      - 8
    sqlite:
      version:
      - 3.34.0
    tar:
      version:
      - 1.32
    texinfo:
      version:
      - 6.5
    xz:
      version:
      - 5.2.5
      variants: +pic
    zlib:
      version:
      - 1.2.11
    zstd:
      version:
      - 1.4.9

  definitions:
  - cuda_specs:
    - amrex@21.05 +cuda cuda_arch=70
    - axom@0.5.0 +cuda cuda_arch=70 ^umpire~shared
    - caliper@2.5.0 +cuda cuda_arch=70
    - chai@2.3.0 +cuda ~benchmarks ~tests cuda_arch=70 ^umpire~shared
    - ginkgo@1.3.0 +cuda cuda_arch=70
    - hpx@1.6.0 +cuda cuda_arch=70
    - kokkos@3.4.00 +cuda +wrapper cuda_arch=70
    - kokkos-kernels@3.2.00 +cuda cuda_arch=70 ^kokkos +cuda +wrapper cuda_arch=70
    - magma@2.5.4 cuda_arch=70
    - raja@0.13.0 +cuda cuda_arch=70
    - slate@2021.05.02 +cuda cuda_arch=70
    - strumpack@5.1.1 +cuda ~slate cuda_arch=70
    - sundials@5.7.0 +cuda cuda_arch=70
    - superlu-dist@6.4.0 +cuda cuda_arch=70
    - tasmanian@7.5 +cuda cuda_arch=70
    - umpire@4.1.2 +cuda ~shared cuda_arch=70
    - zfp +cuda cuda_arch=70
    #- ascent@0.7.1 +cuda ~shared cuda_arch=70
    #- hypre@2.20.0 +cuda cuda_arch=70
    #- mfem@4.2.0 +cuda cuda_arch=70

  - default_specs:
    - adios2@2.7.1
    - adios@1.13.1
    - aml@0.1.0
    - amrex@21.05
    - arborx@1.0
    - argobots@1.1
    - ascent@0.7.1 ~fortran
    - bolt@2.0
    - cabana@0.3.0
    - caliper@2.5.0
    - chai@2.3.0 ~benchmarks ~tests
    - conduit@0.7.2
    - darshan-runtime@3.3.0
    - darshan-util@3.3.0
    - faodel@1.1906.1
    - flecsi@1.4 +cinch
    - flit@2.1.0
    - gasnet@2021.3.0
    - ginkgo@1.3.0
    - globalarrays@5.8
    - gmp@6.2.1
    - gotcha@1.0.3
    - hdf5@1.10.7
    - hypre@2.20.0
    - kokkos-kernels@3.2.00 +openmp
    - kokkos@3.4.00 +openmp
    - legion@21.03.0
    - libnrm@0.1.0
    - libquo@1.3.1
    - libunwind@1.5.0
    - loki@0.1.7
    - mercury@2.0.1
    - metall@0.13
    - mfem@4.2.0
    - mpark-variant@1.4.0
    - ninja@1.10.2
    - openpmd-api@0.13.4
    - papi@6.0.0.1
    - papyrus@1.0.1
    - parallel-netcdf@1.12.2
    - pdt@3.25.1
    - petsc@3.15.0
    - precice@2.2.1
    - pumi@2.2.5
    - py-libensemble@0.7.2
    - py-petsc4py@3.15.0
    - py-warpx@21.05 ^warpx dims=2
    - py-warpx@21.05 ^warpx dims=3
    - py-warpx@21.05 ^warpx dims=rz
    - qthreads@1.16 scheduler=distrib
    - raja@0.13.0
    - scr@3.0rc1
    - slepc@3.15.0
    - stc@0.9.0
    - strumpack@5.1.1 ~slate
    - sundials@5.7.0
    - superlu-dist@6.4.0
    - superlu@5.2.1
    - swig@4.0.2
    - swig@4.0.2-fortran
    - sz@2.1.11.1
    - tasmanian@7.5
    - tau@2.30.1
    - turbine@1.3.0
    - umap@2.1.0
    - umpire@4.1.2
    - upcxx@2021.3.0
    - zfp@0.5.5

    # Explicit conflicts with Cray -or- Intel compiler (prohibited via package.py)
   #- dyninst@11.0.0
   #- hpctoolkit@2021.03.01
   #- plasma@20.9.20
   #- qt@5.15.2
   #- qwt@6.1.6
   #- slate@2021.05.02 ~cuda

    # Cannot build suite-sparse due to OOM killer
   #- fortrilinos@2.0.0 ^trilinos +nox +superlu-dist +stratimikos
   #- omega-h@9.32.5
   #- trilinos@13.0.1
   #- trilinos@13.0.1 +nox +superlu-dist

    # Failed builds
   #- archer@2.0.0 # llvm@8 fails
   #- axom@0.5.0 # thirdparty/sol/sol.hpp(11408): rvalue ref cannot be bound to an lvalue
   #- heffte@2.0.0 # test/test_units_nompi.cpp(499): error: more than one instance of constructor "heffte::box3d::box3d"
   #- hpx@1.6.0 # include/boost/asio/impl/read.hpp(377): no instance of overloaded function "hpx::util::detail::bound
   #- mpifileutils@0.11 ~xattr # libcap: _caps_output.gperf:96:53: unknown type name 'size_t', libcircle: configure: check if MPI setup correctly
   #- nrm@0.1.0 # py-gevent: configure: compiler doesn't halt on function prototype mismatch
   #- py-jupyterhub@1.0.0
   #- rempi@1.1.0 # configure: couldn't find MPI
   #- unifyfs@0.9.1 # configure: couldn't find MPI

  specs:
  - $default_specs

Cori E4S 21.02

spack:
  view: false
  concretization: separately
  config:
    install_tree:
      root: /global/common/software/spackecp/e4s-21.02/software
    module_roots:
      tcl: /global/common/software/spackecp/e4s-21.02/modules/
    build_stage: $tempdir/user/spack-stage
  modules:
    enable:
    - tcl
    tcl:
      blacklist_implicits: true
      hash_length: 0
      naming_scheme: '{name}/{version}-{compiler.name}-{compiler.version}'
      all:
        conflict:
        - '{name}'
        environment:
          set:
            '{name}_ROOT': '{prefix}'
      darshan-runtime:
        conflict:
        - 'darshan'
      darshan-util:
        conflict:
        - 'darshan'
      projections:
        all: '{name}/{version}-{compiler.name}-{compiler.version}'
  mirrors:
    cori-e4s-21.02: /global/common/software/spackecp/mirrors/cori-e4s-21.02
  compilers:
  - compiler:
      spec: intel@19.1.2.254
      paths:
        cc: cc
        cxx: CC
        f77: ftn
        fc: ftn
      flags: {}
      operating_system: cnl7
      target: any
      modules:
      - PrgEnv-intel
      - intel/19.1.2.254
      environment: {unset: []}
      extra_rpaths: []
  - compiler:
      spec: gcc@10.1.0
      paths:
        cc: cc
        cxx: CC
        f77: ftn
        fc: ftn
      operating_system: cnl7
      modules:
      - PrgEnv-gnu
      - gcc/10.1.0
  
  definitions:
  - intel_compiler: ['%intel@19.1.2.254']
  - gcc_compiler: ['%gcc@10.1.0']
  - e4s_intel:    
    - adios2@2.7.1 +hdf5 
    - aml@0.1.0
    - arborx@0.9-beta +openmp     
    - bolt@2.0
    - caliper@2.5.0 +fortran
    - faodel@1.1906.1
    - flecsi@1.4 +cinch +caliper +graphviz +tutorial
    - flit@2.1.0  
    - gasnet@2020.3.0 +udp
    - ginkgo@1.3.0
    - gotcha@1.0.3 +test
    - hdf5@1.10.7    
    - hypre@2.20.0 +mixedint +superlu-dist +openmp
    - libnrm@0.1.0
    - libquo@1.3.1
    - mercury@2.0.0 +udreg
    - mfem@4.2.0 +examples +gnutls +gslib +lapack +libunwind +openmp +threadsafe +pumi +umpire    
    - ninja@1.10.2
    - omega-h@9.32.5 ~trilinos
    - openpmd-api@0.13.2 
    - papi@6.0.0.1 +example +static_tools +powercap +infiniband
    - papyrus@1.0.1 
    - pdt@3.25.1 +pic 
    - precice@2.2.0 +python
    - pumi@2.2.5 +fortran
    - qthreads@1.16 ~hwloc
    - raja@0.13.0 +tests    
    - slepc@3.14.2
    - strumpack@5.1.1 +shared
    - sundials@5.7.0 +examples-cxx +hypre +klu +lapack 
    - superlu@5.2.1
    - superlu-dist@6.4.0 +openmp
    - swig@4.0.2-fortran
    - tasmanian@7.3 +blas +fortran +mpi +python +xsdkflags
    - tau@2.30.1 +mpi ~pdt
    - turbine@1.2.3 +hdf5 +python
    - umap@2.1.0 +tests
    - umpire@4.1.2 +fortran +numa +openmp
    - upcxx@2020.10.0
    - zfp@0.5.5 +aligned +c +fortran +openmp +profile
        
  - e4s_gcc:
    - darshan-runtime@3.2.1 +slurm 
    - darshan-util@3.2.1 +bzip2    
    - dyninst@10.2.1 
    - legion@20.03.0 
    - plasma@20.9.20 
    - slate@2020.10.00 ~cuda  
    

    # skipping package
    #  - adios@1.13.1 +bzip2 +fortran +hdf5 +netcdf
    # - kokkos-kernels@3.2.00 +mkl +openmp
    # - kokkos@3.2.00  +compiler_warnings +deprecated_code +examples +hwloc +memkind +numactl +openmp +pic +tests    
    # - openmpi@4.0.5 +cxx +thread_multiple schedulers=slurm
    # - parallel-netcdf@1.12.1 +burstbuffer
    # - petsc@3.14.4 +X +fftw +jpeg +libpng +libyaml +memkind 
    # - py-jupyterhub@1.0.0
    # - py-libensemble@0.7.1 +mpi +nlopt +petsc4py  +scipy
    # - py-petsc4py@3.14.1
    # - trilinos@13.0.1 

    # _______________________________ ISSUES TO SOLVE _______________________________
    # issue installing vtkh using intel compiler
    # using gcc compiler ascent has dependency for conduit@develop which not pinned to version. There was a build error related to missing HDF5 library
    # - ascent@0.6.0
    
    # Issue detecting fortran compiler https://cdash.spack.io/viewConfigure.php?buildid=105216. Also issue installing conduit since its tied to 'develop'. Tried using conduit@0.7.1 and still failed see https://cdash.spack.io/viewBuildError.php?buildid=105206. Talk to @cyrush at spack slack.   
    # - axom@0.4.0 +mfem +python

    # /usr/lib64/gcc/x86_64-suse-linux/7/../../../../x86_64-suse-linux/bin/ld: /usr/lib/libm.so: error adding symbols: file in wrong format see https://cdash.spack.io/buildSummary.php?buildid=104952
    # - adios2@2.7.1 +hdf5 +dataman +dataspaces   

    # skipping this version for now 5.7 is the latest version 5.8 doesn't exist in spack repo
    # - globalarrays@5.8 +blas +lapack +scalapack
    
    #  Warning: Linking the shared library libhpcrun.la against the static library see  https://cdash.spack.io/viewBuildError.php?buildid=104938
    # - hpctoolkit@2020.08.03 %gcc

    # error: identifier "HPX_SMT_PAUSE" is undefined see https://cdash.spack.io/viewBuildError.php?buildid=105747
    # - hpx@1.6.0 +async_mpi +examples

    # Error building legion with intel compiler https://cdash.spack.io/viewBuildError.php?buildid=105190
    # - legion@20.03.0

    # skip magma because it's a GPU package
    #- magma@2.5.4 cuda_arch=70 ^cuda@10.2.89
   
    # error with intel compiler:  building dtcmp 
    # error with gnu compiler: Could NOT find LibCircle (missing: LibCircle_LIBRARIES
    # - mpifileutils@0.10.1

    # /global/cfs/cdirs/m3503/spack-NSewtxLx/spack-stage/siddiq90/spack-stage-phist-1.9.3-rznbmfuo2mt2erku4rit4peyqxu7iji4/spack-src/fortran_bindings/test/kernels.F90(63): catastrophic error: **Internal compiler error: internal abort** Please report this error along with the circumstances in which it occurred in a Software Problem Report.  Note: File and line given may not be explicit cause of this error. see https://cdash.spack.io/buildSummary.php?buildid=104915
    # - phist@1.9.3

    
    # configure: error: Failed to find C MPI Wrapper. see https://cdash.spack.io/buildSummary.php?buildid=104940
    # - rempi@1.1.0  

    # "%intel@19:" conflicts with "slate" [Does not currently build with icpc >= 2019]
    # - slate@2020.10.00 ^cuda@10.2.89

    # build Error with Intel: error building dtcmp: configure: error: C compiler cannot create executables
    # build Error with GCC: make[2]: *** No rule to make target '/global/cfs/cdirs/m3503/spack-qhLmtUlQ/spack_path_placeholder/spack_path_placeholder/spack_path_placeholder/spack_path_placehol/cray-cnl7-haswell/gcc-10.1.0/libyogrt-1.24-6wngjuplxnjjsivzvilwjsp4gwu4ziuj/lib/libyogrt.a', needed by 'examples/test_ckpt_F'.  Stop. see https://cdash.spack.io/buildSummary.php?buildid=104889
    # - scr@2.0.0
    # error installing ant Error: JAVA_HOME is not defined correctly. https://software.nersc.gov/NERSC/e4s-2102/-/jobs/87103
    # - stc@0.8.3 
    
    # Issue building sundials with raja support see https://cdash.spack.io/viewBuildError.php?buildid=105455. Not sure if raja support with sundials is neccessary.
    # sundials@5.7.0 +examples-cxx +examples-f2003 +f2003 +hypre +klu +lapack +openmp +raja +superlu-dist

    # https://cdash.spack.io/viewBuildError.php?buildid=105513
    # - sz@2.1.11.1 +fortran +python +time_compression +random_access +pastri

    # issue building tau with intel see https://cdash.spack.io/viewBuildError.php?buildid=105235 one of the error points to missing `-lpdb` library. Tau has `+pdt` enabled by default
    # - tau@2.30.1 +adios2 +gasnet +likwid  +ompt +openmp +mpi  +python +scorep +shmem +sqlite

    #  Could NOT find AXL (missing: AXL_LIBRARIES AXL_INCLUDE_DIRS) see https://cdash.spack.io/buildSummary.php?buildid=105476
    # - veloc@1.4

    # issue configure: error: "Couldn't find MPI" see https://cdash.spack.io/viewConfigure.php?buildid=105491
    # issue installing mercury (dependency) for unifyfs with gcc https://cdash.spack.io/buildSummary.php?buildid=105497
    #- unifyfs@0.9.1

  specs:
  - matrix:
    - [$e4s_intel]
    - [$intel_compiler]
  - matrix:    
    - [$e4s_gcc]
    - [$gcc_compiler]

  packages:
    all:
      compiler: [intel@19.1.2.254, gcc@10.1.0]
      target: [haswell]
      providers:
        mpi: [mpich]
        mkl: [cray-libsci, intel-mkl]
        blas: [cray-libsci, intel-mkl]
        scalapack: [cray-libsci, intel-mkl]
        pkgconfig: [pkg-config]
    
    cray-libsci:
      buildable: false
      externals:
      - spec: cray-libsci@19.06.1%intel
        modules:
        - cray-libsci/19.06.1

    fftw:
      buildable: false
      externals:
      - spec: fftw@3.3.8.4%intel
        modules:
        - cray-fftw/3.3.8.4

    hdf5:
      variants: +cxx +debug +fortran +szip +threadsafe +hl
      
    hwloc:
      buildable: false
      externals:
      - spec: hwloc
        prefix: /usr
    intel-mkl:
      buildable: false
      externals:
      - spec: intel-mkl@19.1.2.254
        modules:
        - intel/19.1.2.254

    mpich:
      buildable: false
      externals:
      - spec: mpich@3.1
        modules:
        - cray-mpich/7.7.10
    
    netcdf-c:
      buildable: false
      externals:
        - spec: netcdf-c@4.7.4
          modules:
          - cray-netcdf/4.7.4.0

    openssl:
      buildable: false
      externals:
      - spec: openssl@1.1.1g
        prefix: /usr
    # issue installing version 5.32.1 and confirmed 5.32.0 works
    perl:
      version: [5.32.0]

    # disable slate since we can't build with icc >= 19
    strumpack:
      variants: ~slate

Cori E4S 20.10

spack:
  concretization: separately
  view: false
  config:
    install_tree: /global/common/software/spackecp/e4s-20.10/software
    build_stage: $tempdir/$user/spack-stage
    module_roots:
      tcl: /global/common/software/spackecp/e4s-20.10/modules/
  mirrors::
    e4s-2020-10: /global/common/software/spackecp/mirrors/e4s-2020-10
  modules:
    enable:
    - tcl
    tcl:
      hash_length: 8
      projections:
        all: '{name}/{version}-{compiler.name}-{compiler.version}'
      all:
        conflict:
        - '{name}'
        filter:
          environment_blacklist: []
        load: []
        environment:
          unset: []
      verbose: false
      whitelist: []
      blacklist: []
      blacklist_implicits: false
  
  definitions:
  - e4s:   
    - adios2@2.6.0 
    - aml@0.1.0
    - arborx@0.9-beta +openmp
    - bolt@1.0
    - caliper@2.4.0
    - darshan-runtime@3.2.1 +slurm
    - darshan-util@3.2.1 +bzip2 
    - flit@2.1.0
    - gasnet@2020.3.0 +udp
    - ginkgo@1.2.0
    - globalarrays@5.7 +blas +lapack +scalapack
    - gotcha@1.0.3 +test
    - hdf5@1.10.6 +cxx +debug +fortran +szip +threadsafe +hl
    - hypre@2.20.0 +mixedint
    - kokkos-kernels@3.2.00 +mkl +openmp
    - kokkos@3.2.00 +debug +debug_dualview_modify_check +compiler_warnings +examples +hwloc +memkind +numactl +openmp +pic +tests
    - libnrm@0.1.0
    - libquo@1.3.1
    - mercury@1.0.1 +udreg
    - mfem@4.1.0 +examples +gnutls +gslib +lapack +libunwind +openmp +threadsafe +pumi +umpire  
    - ninja@1.10.1
    - openpmd-api@0.12.0
    - papi@6.0.0.1 +example +static_tools
    - parallel-netcdf@1.12.1
    - pdt@3.25.1 +pic
    - petsc@3.14.0    
    - pumi@2.2.2 +fortran
    - py-libensemble@0.7.0 +mpi +nlopt +scipy
    - py-petsc4py@3.13.0
    - qthreads@1.14 ~hwloc
    - raja@0.12.1
    - slepc@3.14.0 
    - stc@0.8.3 
    - sundials@5.4.0 +examples-cxx +examples-f2003 ~examples-f77 +f2003 +klu +openmp +hypre +lapack 
    - superlu@5.2.1
    - superlu-dist@6.3.1
    - swig@4.0.2
    - sz@2.1.10 +fortran +hdf5 +python +time_compression +random_access +netcdf +pastri
    - tasmanian@7.3 +blas +fortran +mpi +python +xsdkflags 
    - turbine@1.2.3 +hdf5 +python
    - umap@2.1.0 +tests
    - umpire@4.0.1 +fortran +numa +openmp
    - upcxx@2020.3.0
    - veloc@1.4
    - zfp@0.5.5

    
  # - adios@1.13.1 +netcdf +szip +fortran +bzip2  module already installed
  # - ascent@develop skipping package because its on develop
  # - axom@0.3.3 skip build for now, spack ci rebuild issue
  # issue with concretization of dyninst:  "%intel" conflicts with "dyninst" see https://software.nersc.gov/ecp/nersc-e4s/-/jobs/46526
  # - dyninst@10.2.1 +static
  # - faodel@1.1906.1 network=libfabric issue with build https://software.nersc.gov/NERSC/nersc-e4s/-/jobs/60284
  # flesci concretization issue: https://github.com/spack/spack/issues/19292
  # - flecsi@1 +cinch +coverage +doc +doxygen +graphviz +hdf5 +tutorial
  # - hpctoolkit@2020.08.03 +all-static +cray +mpi  # depends on dyninst
  # - hpx@1.5.1 issue with installing boost
  # - legion@20.03.0 failed to build se https://software.nersc.gov/ecp/e4s/facilitypipelines/nersc-e4s/-/jobs/59787
  # - magma@2.5.3 this package requires GPU, this is not applicable for Cori
  # - mpifileutils@0.10.1 +gpfs +lustre # fails on libcircle requires MPICC wrapper see https://software.nersc.gov/ecp/e4s/facilitypipelines/nersc-e4s/-/jobs/57907 we could install OpenMPI and build this with the wrapper.
  #- openmpi            # skip openmpi 
  # - omega-h@9.29.0
  # - papi@6.0.0.1 +example +static_tools   module already installed  
  # - phist@1.9.1 see https://software.nersc.gov/NERSC/nersc-e4s/-/jobs/63409
  # concretization issue "%intel" conflicts with "plasma" see https://software.nersc.gov/ecp/nersc-e4s/-/jobs/46533
  # - plasma@20.9.20
  # - precice@2.1.0 issue finding PETSC
  # - py-jupyterhub@1.0.0
  # error building rempi https://software.nersc.gov/ecp/nersc-e4s/-/jobs/32884
  # - rempi@1.1.0
  # - scr@2.0.0  # async_api=CRAY_DW issue finding DATAWARP libraries -- Could NOT find DATAWARP (missing: DATAWARP_LIBRARIES DATAWARP_INCLUDE_DIRS) see https://software.nersc.gov/ecp/e4s/facilitypipelines/nersc-e4s/-/jobs/59782
  # - slate@develop package tied to develop skipping this build
  # - strumpack@4.0.0 +shared ~butterflypack ~cuda +count_flops +build_dev_tests +build_tests 
  # error building otf2, we can disable tau with otf2 support 
  # - tau@2.29 +craycnl +openmp +mpi
  # - trilinos@13.0.0 +debug +float +openmp +pnetcdf +zlib
  # - unifyfs@0.9.0 +hdf5. can't find MPICC see https://software.nersc.gov/NERSC/nersc-e4s/-/jobs/63408
  
  - arch:
    - '%intel@19.1.2.254 arch=cray-cnl7-haswell'
  specs:
  - matrix:
    - - $e4s
    - - $arch

  compilers:
  - compiler:
      spec: intel@19.1.2.254
      paths:
        cc: cc
        cxx: CC
        f77: ftn
        fc: ftn
      flags: {}
      operating_system: cnl7
      target: any
      modules:
      - PrgEnv-intel
      - intel/19.1.2.254
      environment: {unset: []}
      extra_rpaths: []


  packages:
    all:
      compiler: [intel@19.1.2.254]
      target: [haswell]
      providers:
        mpi: [mpich]
        mkl: [intel-mkl, cray-libsci]
        blas: [intel-mkl, cray-libsci]
        scalapack: [intel-mkl, cray-libsci]
        pkgconfig: [pkg-config]
            
    berkeley-db:
      version: [18.1.4]

    boost:
      version: [1.74.0]   

    bzip2:
      buildable: false
      externals:
      - spec: bzip2
        prefix: /usr

    cmake:
      version: [3.16.5]  # issue with cmake 3.17.3 using 3.16.5 for now see https://github.com/spack/spack/issues/17605  

    cray-libsci:
      buildable: false
      externals:
      - spec: cray-libsci@19.06.1%intel
        modules:
        - cray-libsci/19.06.1
    
    diffutils:
      version: [3.7]
    
    elfutils:
      version: [0.180]
    
    expat:
      version: [2.2.9]

    fftw:
      buildable: false
      externals:
      - spec: fftw@3.3.8.4%intel
        modules: 
          - cray-fftw/3.3.8.4
    
    gdbm:
      version: [1.18.1]
    
    gettext:
      buildable: false      
      externals:
      - spec: gettext
        prefix: /usr
    
    help2man:
      version: [1.47.11]

    hwloc:
      buildable: false
      externals:
      - spec: hwloc
        prefix: /usr 
    
    hypre:
      version: [2.20.0]

    intel-mkl:
      buildable: false
      externals:
      - spec: intel-mkl@19.1.2.254
        modules:
          - intel/19.1.2.254    
    
    libbsd:
      version: [0.10.0]

    libiconv:
      version: [1.16]
    
    libsigsegv:
      version: [2.12]

    libxml2:
      version: [2.9.10]  

    lz4:
      buildable: false
      externals:
      - spec: lz4
        prefix: /usr

    m4:
      buildable: false
      externals:
      - spec: m4
        prefix: /usr  

    mpi:
      buildable: false

    mpich:
      buildable: false
      externals:
      - spec: mpich@3.1
        modules:
        - cray-mpich/7.7.10 

    openssl:
      buildable: false
      externals:
      - spec: openssl@1.1.1g
        prefix: /usr 
     
    ncurses:
      version: [6.2]  

    netcdf:
      buildable: false
      externals:
      - spec: netcdf@4.6.3.2%intel
        modules: 
          - cray-netcdf/4.6.3.2
       
    perl:
      buildable: false
      externals:
      - spec: perl
        prefix: /usr 

    petsc:
      version: [3.14.0]

    pdsh: # required for scr 
      buildable: false
      externals:
      - spec: pdsh
        prefix: /usr    
        
    pkgconf:
      version: [1.7.3]
    
    sqlite:
      version: [3.31.1]
        
    tar:
      buildable: false
      externals:
      - spec: tar
        prefix: /usr

    xz:
      buildable: false
      externals:
      - spec: xz
        prefix: /usr

    zlib:
      version: [1.2.11]