Spack Configuration
This page will show all of our Spack configuration files (spack.yaml) used for our production deployments. The Spack configuration located in spack-configs directory organized by each subdirectory.
At NERSC, we are building the Extreme-scale Scientific Software Stack (E4S) which is a collection of open-source products software packages part of Spack ecosystem for running scientific applications on high-performance computing (HPC) platforms. We acquire the Spack configuration from https://github.com/E4S-Project/e4s upon release with list of specs and reference Spack branch in order to build the E4S stack. Please see our E4S documentation at https://docs.nersc.gov/applications/e4s/
Perlmutter Spack Develop
This Spack configuration will build all packages using Spack develop branch on weekly basis. All specs are specified without any version in order to let Spack build the latest package which will evolve over time.
You may add the mirror into your spack environment by running:
spack mirror add perlmutter-spack-buildcache /global/common/software/spackecp/mirrors/perlmutter-spack-develop
Or you can explicitly add the following lines into your spack.yaml
mirrors: perlmutter-spack-buildcache: file:///global/common/software/spackecp/mirrors/perlmutter-spack-develop
Spack Configuration for spack@develop
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree:
padded_length: 128
mirrors:
perlmutter-spack-buildcache: file:///global/common/software/spackecp/mirrors/perlmutter-spack-develop
source_mirror: file:///global/common/software/spackecp/mirrors/source_mirror
cdash:
build-group: DOE nightly E4S builds
url: https://cdash.spack.io
project: Spack
site: NERSC - Perlmutter spack@develop
definitions:
- gcc_compilers: ['%gcc@11.2.0']
- nvhpc_compilers: ['%nvhpc@21.11']
- cray_compilers: ['%cce@13.0.2']
- cray_specs:
- adios2
- fftw
- hdf5
- papi
- petsc +openmp +strumpack
- superlu
- superlu-dist +openmp
- gcc_specs:
#- adios2
- hdf5
#- hypre +openmp +superlu-dist
- papi
#- petsc +openmp +strumpack
- raja
#- strumpack ~slate
#- sundials +openmp +hypre
#- superlu
#- superlu-dist +openmp
- cuda_specs:
- amrex +cuda cuda_arch=80
- blaspp +cuda cuda_arch=80
- hipace compute=cuda
- hpctoolkit +cuda +cray +mpi
- hypre +cuda cuda_arch=80
- kokkos-kernels +openmp +cuda cuda_arch=80 ^kokkos +openmp +wrapper +cuda cuda_arch=80
- kokkos +openmp +wrapper +cuda cuda_arch=80
- lapackpp ^blaspp +cuda cuda_arch=80
- magma@2.6.1+cuda cuda_arch=80
- mfem@4.3.0+cuda cuda_arch=80
- petsc +cuda cuda_arch=80
- py-warpx ^warpx dims=1 compute=cuda
- py-warpx ^warpx dims=2 compute=cuda
- py-warpx ^warpx dims=3 compute=cuda
- py-warpx ^warpx dims=rz compute=cuda
- qmcpack +cuda cuda_arch=80
- raja +cuda cuda_arch=80
- slepc +cuda cuda_arch=80
- trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack
+ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox
+piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
+teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
- strumpack ~slate +cuda cuda_arch=80
- slate +cuda cuda_arch=80
- superlu-dist +openmp +cuda cuda_arch=80
- sundials +openmp +cuda cuda_arch=80
- upcxx +gasnet +mpi
- umpire ~shared +cuda cuda_arch=80
- upcxx +cuda
- warpx dims=1 compute=cuda
- warpx dims=2 compute=cuda
- warpx dims=3 compute=cuda
- warpx dims=rz compute=cuda
- zfp +cuda cuda_arch=80
- nvhpc_specs:
#- adios2 failed due to libffi see https://github.com/libffi/libffi/issues/691
- amrex +cuda cuda_arch=80
- blaspp +cuda cuda_arch=80
- hypre +cuda cuda_arch=80
- kokkos +openmp +wrapper +cuda cuda_arch=80
- kokkos-kernels +openmp +cuda cuda_arch=80 ^kokkos +openmp +wrapper +cuda cuda_arch=80
- lapackpp ^blaspp +cuda cuda_arch=80
- openpmd-api
- petsc +cuda cuda_arch=80
- py-numba
- raja +cuda cuda_arch=80
- umpire ~shared +cuda cuda_arch=80
- upcxx +cuda
- zfp +cuda cuda_arch=80
- nersc_specs:
#- amber+openmp requires tarball and license
# skipping arm-forge for now this requires a license and gets stuck in CI job.
#- arm-forge
- abinit +wannier90
- amdblis
- amdfftw
# requested by user INC0176750. See https://github.com/NVIDIA/AMGX/issues/165
- amgx +cuda cuda_arch=80
- amdscalapack
- atompaw
- berkeleygw
- boost cxxstd=11
- boost cxxstd=14
- boost cxxstd=98
- cmake
- dpcpp +openmp
- eigen
- elpa
- fpm
- lammps
- llvm-openmp
- metis
- mt-metis
- mumps
- nccmp
- nco
- octave
- parmetis
- parallel
- plumed
- qmcpack
- quantum-espresso
- scotch
- sparskit
- superlu-mt
- wannier90
- valgrind
# cuda_arch=80 not supported in spack package yet. See https://github.com/spack/spack/issues/28554
- cp2k +cuda cuda_arch=70 +elpa +cosma
specs:
#- matrix:
# - [$cray_specs]
# - [$cray_compilers]
- matrix:
- [$gcc_specs]
- [$gcc_compilers]
#- matrix:
# - [$cuda_specs]
# - [$gcc_compilers]
#- matrix:
# - [$nvhpc_specs]
# - [$nvhpc_compilers]
#- matrix:
# - [$nersc_specs]
# - [$gcc_compilers]
gitlab-ci:
enable-artifacts-buildcache: false
rebuild-index: true
before_script:
- module reset
- module use /global/common/software/nersc/$(cat /etc/nersc_modules_rev)/extra_modulefiles
- module load cpu
- module list
- source setup-env.sh
- git clone ${SPACK_REPO}
- pushd spack && git checkout ${SPACK_CHECKOUT_VERSION} && popd
- . spack/share/spack/setup-env.sh
- spack --version
- spack-python --path
script:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view -d .
- spack env st
- spack -d ci rebuild
after_script:
- rm -rf $SPACK_ROOT
service-job-attributes:
tags: [perlmutter-e4s]
script:
- echo "End Pipeline"
mappings:
- match: [os=sles15]
runner-attributes:
tags: [perlmutter-e4s]
Perlmutter E4S 23.05
Production Spack Configuration
GCC spack environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree: $spack/opt/spack
concretizer:
reuse: false
unify: false
mirrors:
perlmutter-e4s-23.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-23.05
specs:
- matrix:
- [$gcc_specs]
- [$gcc_compilers]
- matrix:
- [$nersc_specs]
- [$gcc_compilers]
CCE Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree: $spack/opt/spack
concretizer:
reuse: false
unify: false
mirrors:
perlmutter-e4s-23.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-23.05
specs:
- matrix:
- [$cce_specs]
- [$cce_compilers]
NVHPC Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree: $spack/opt/spack
concretizer:
reuse: false
unify: false
mirrors:
perlmutter-e4s-23.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-23.05
packages:
cmake::
require: '%gcc'
# build failures with xz %nvhpc so reverting to gcc
xz::
require: '%gcc'
specs:
- matrix:
- [$nvhpc_specs]
- [$nvhpc_compilers]
CUDA Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree: $spack/opt/spack
concretizer:
reuse: false
unify: false
mirrors:
perlmutter-e4s-23.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-23.05
specs:
- matrix:
- [$cuda_specs]
- [$gcc_compilers]
DATA Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree: $spack/opt/spack
concretizer:
reuse: false
unify: false
mirrors:
perlmutter-e4s-23.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-23.05
specs:
- matrix:
- [$data_specs]
- [$gcc_compilers]
MATH-LIBS Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree: $spack/opt/spack
concretizer:
reuse: false
unify: false
mirrors:
perlmutter-e4s-23.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-23.05
specs:
- matrix:
- [$math-libs]
- [$gcc_compilers]
TOOLS Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree: $spack/opt/spack
concretizer:
reuse: false
unify: false
mirrors:
perlmutter-e4s-23.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-23.05
specs:
- matrix:
- [$tools]
- [$gcc_compilers]
Shown below is the list of definitions that is used for all of our spack environments.
Definitions for Spack Environments
definitions:
- cce_compilers: ['%cce@=15.0.0']
- gcc_compilers: ['%gcc@=11.2.0']
- nvhpc_compilers: ['%nvhpc@=22.7']
- cce_specs:
- fftw
- hdf5 +fortran +hl +shared
- hypre
- kokkos +openmp
- kokkos-kernels +openmp
- netcdf-c
- netcdf-fortran
- papi
- petsc
- slepc
- sundials
- superlu
- superlu-dist
- math-libs:
- arborx
- fftw@=3.3.8
- fftw@=3.3.9
- fftw@=3.3.10
- ginkgo
- heffte
- hypre
- intel-mkl
- openblas
- petsc
- phist
- pumi
- slate
- slepc
- sundials
- superlu
- superlu-dist
- data_specs:
- datatransferkit
- hdf5
- hdf5-vol-async
- hdf5-vol-cache
- netcdf-c
- netcdf-fortran
- parallel-netcdf
#- hdf5-vol-log Build error
- nvhpc_specs:
- amrex
- boost
- hdf5 ~mpi
- sundials
- superlu
#- superlu-dist
#- hypre
#- kokkos
#- kokkos-kernels
#- papi Build failud due to nvc-Error-Unknown switch: -Wno-error
- cuda_specs:
- adios2@=2.8.3 +cuda cuda_arch=80 # ecp-data-vis-sdk
- amrex +cuda cuda_arch=80
- arborx +cuda cuda_arch=80 ^kokkos +wrapper
- cabana +cuda ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=80
- caliper +cuda cuda_arch=80
- chai ~benchmarks ~tests +cuda cuda_arch=80 ^umpire ~shared
- cusz +cuda cuda_arch=80
- flecsi +cuda cuda_arch=80
- ginkgo +cuda cuda_arch=80
- heffte +cuda cuda_arch=80
- hpx max_cpu_count=512 +cuda cuda_arch=80
- hypre +cuda cuda_arch=80
- kokkos +wrapper +cuda cuda_arch=80
- kokkos-kernels +cuda cuda_arch=80 ^kokkos +wrapper +cuda cuda_arch=80
- lammps +cuda cuda_arch=80
- legion +cuda cuda_arch=80
- magma +cuda cuda_arch=80
- mfem +cuda cuda_arch=80
- mgard +serial +openmp +timing +unstructured +cuda cuda_arch=80
- omega-h +cuda cuda_arch=80
- parsec +cuda cuda_arch=80
- petsc +cuda cuda_arch=80
- slate +cuda cuda_arch=80
- slepc +cuda cuda_arch=80
- strumpack ~slate +cuda cuda_arch=80
- sundials +cuda cuda_arch=80
- superlu-dist +cuda cuda_arch=80
- tasmanian +cuda cuda_arch=80
- umpire ~shared +cuda cuda_arch=80
- zfp +cuda cuda_arch=80
# - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +cusz +mgard +cuda cuda_arch=80 ^cusz +cuda cuda_arch=80 # concretization issue with cuda
# CUDA NOARCH
- hpctoolkit +cuda
- papi +cuda
- tau +mpi +cuda
#- bricks +cuda
# - ecp-data-vis-sdk ~rocm +adios2 ~ascent +hdf5 ~paraview ~pnetcdf ~sz +vtkm +zfp +cuda cuda_arch=80
#- raja +cuda cuda_arch=80
- gcc_specs:
- alquimia
- aml
- amrex
- arborx
- argobots
- bolt
- boost
- butterflypack
- cabana
- caliper
- chai ~benchmarks ~tests
- datatransferkit
- dyninst ^intel-tbb
- flecsi
- flit
- fortrilinos
- ginkgo
- gotcha
- h5bench
- hdf5
- hdf5-vol-async
- hdf5-vol-cache
- heffte +fftw
- hpctoolkit
- hpx max_cpu_count=512 networking=mpi
- hypre
- kokkos +openmp
- kokkos-kernels +openmp
- lammps
- legion
- libnrm
- libpressio +bitgrooming +bzip2 ~cuda ~cusz +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf
- libquo
- libunwind
- likwid
- loki
- mercury
- metall
- mfem
- mgard +serial +openmp +timing +unstructured ~cuda
- mpark-variant
- nccmp
- nco
- netlib-scalapack
- omega-h
- openpmd-api
- papi
- papyrus
- parsec ~cuda
- pdt
- petsc
- phist
- plasma
- plumed
- precice
- pumi
- py-h5py
- py-jupyterhub
- py-libensemble
- py-petsc4py
- qthreads scheduler=distrib
- raja
- slate ~cuda
- slepc
- stc
- strumpack ~slate
- sundials
- superlu
- superlu-dist
- swig@4.0.2-fortran
- sz3
- tasmanian
- tau +mpi +python
- trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
- turbine
- umap
- umpire
- unifyfs
- variorum
# build failures
# - axom ~mpi
#- globalarrays
#- wannier90
# - rempi
# - openfoam
# - mpifileutils ~xattr
# - lbann
# - hdf5-vol-log
# - flux-core
# - ecp-data-vis-sdk ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 ~paraview +pnetcdf +sz +unifyfs +veloc ~visit +vtkm +zfp # visit: font-util: [Makefile:756: install-data-hook] Error 1 share/fonts/X11/cyrillic: failed to write cache paraview: ispc: lex.cpp:398:9: error: 'yywrap' macro redefined [-Werror,-Wmacro-redefined]
# - exaworks
# - conduit ~mpi
- nersc_specs:
- chapel
- gsl
- netcdf-c ~mpi
- netcdf-fortran
- metis
- parmetis
- gromacs
- cdo
# trouble building esmf +mpi
#- ncl ^esmf~mpi
- ncl
- nco
- ncview
- libxc
- libcint
- libint tune=molgw-lmax-7
- intel-mkl
- tools:
- autoconf
- automake
- ccache
- cmake
- git
- gmake
- gawk
- nano
- subversion
Each spack environment is built in a separate directory using spack ci in-order to push specs to buildcache. We have the following spack configuration for each spack environment.
Spack Environments for Spack CI
GCC Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree:
padded_length: 128
concretizer:
reuse: false
unify: false
mirrors:
perlmutter-e4s-23.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-23.05
ci:
enable-artifacts-buildcache: false
rebuild-index: true
target: gitlab
pipeline-gen:
- submapping:
- match: [os=sles15]
build-job:
tags: [perlmutter-e4s]
match_behavior: first
- any-job:
before_script:
- git clone -c feature.manyFiles=true -b e4s-23.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- build-job:
tags: [perlmutter-e4s]
before_script:
- module reset
- module load cpu cray-pmi
- module list
- source setup-env.sh
- git clone -c feature.manyFiles=true -b e4s-23.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- spack-python --path
script:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- spack env st
- export SPACK_GNUPGHOME=$HOME/.gnupg
- spack gpg list
- spack -d ci rebuild
- reindex-job:
tags: [perlmutter-e4s]
script:
- echo "End Pipeline"
- noop-job:
tags: [perlmutter-e4s]
script:
- echo "End Pipeline"
specs:
- matrix:
- [$gcc_specs]
- [$gcc_compilers]
- matrix:
- [$nersc_specs]
- [$gcc_compilers]
CCE Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree:
padded_length: 128
concretizer:
reuse: false
unify: false
mirrors:
perlmutter-e4s-23.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-23.05
ci:
enable-artifacts-buildcache: false
rebuild-index: true
target: gitlab
pipeline-gen:
- submapping:
- match: [os=sles15]
build-job:
tags: [perlmutter-e4s]
match_behavior: first
- any-job:
before_script:
- git clone -c feature.manyFiles=true -b e4s-23.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- build-job:
tags: [perlmutter-e4s]
before_script:
- module reset
- module load cpu cray-pmi
- module list
- source setup-env.sh
- git clone -c feature.manyFiles=true -b e4s-23.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- spack-python --path
script:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- spack env st
- export SPACK_GNUPGHOME=$HOME/.gnupg
- spack gpg list
- spack -d ci rebuild
- reindex-job:
tags: [perlmutter-e4s]
script:
- echo "End Pipeline"
- noop-job:
tags: [perlmutter-e4s]
script:
- echo "End Pipeline"
specs:
- matrix:
- [$cce_specs]
- [$cce_compilers]
NVHPC Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree:
padded_length: 128
concretizer:
reuse: false
unify: false
packages:
cmake::
require: '%gcc'
# build failures with xz %nvhpc so reverting to gcc
xz::
require: '%gcc'
mirrors:
perlmutter-e4s-23.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-23.05
ci:
enable-artifacts-buildcache: false
rebuild-index: true
target: gitlab
pipeline-gen:
- submapping:
- match: [os=sles15]
build-job:
tags: [perlmutter-e4s]
match_behavior: first
- any-job:
before_script:
- git clone -c feature.manyFiles=true -b e4s-23.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- build-job:
tags: [perlmutter-e4s]
before_script:
- module reset
- module load cpu cray-pmi
- module list
- source setup-env.sh
- git clone -c feature.manyFiles=true -b e4s-23.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- spack-python --path
script:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- spack env st
- export SPACK_GNUPGHOME=$HOME/.gnupg
- spack gpg list
- spack -d ci rebuild
- reindex-job:
tags: [perlmutter-e4s]
script:
- echo "End Pipeline"
- noop-job:
tags: [perlmutter-e4s]
script:
- echo "End Pipeline"
specs:
- matrix:
- [$nvhpc_specs]
- [$nvhpc_compilers]
CUDA Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree:
padded_length: 128
concretizer:
reuse: false
unify: false
mirrors:
perlmutter-e4s-23.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-23.05
ci:
enable-artifacts-buildcache: false
rebuild-index: true
target: gitlab
pipeline-gen:
- submapping:
- match: [os=sles15]
build-job:
tags: [perlmutter-e4s]
match_behavior: first
- any-job:
before_script:
- git clone -c feature.manyFiles=true -b e4s-23.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- build-job:
tags: [perlmutter-e4s]
before_script:
- module reset
- module load cpu cray-pmi
- module list
- source setup-env.sh
- git clone -c feature.manyFiles=true -b e4s-23.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- spack-python --path
script:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- spack env st
- export SPACK_GNUPGHOME=$HOME/.gnupg
- spack gpg list
- reindex-job:
tags: [perlmutter-e4s]
script:
- echo "End Pipeline"
- noop-job:
tags: [perlmutter-e4s]
script:
- echo "End Pipeline"
specs:
- matrix:
- [$cuda_specs]
- [$gcc_compilers]
DATA Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree:
padded_length: 128
concretizer:
reuse: false
unify: false
mirrors:
perlmutter-e4s-23.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-23.05
ci:
enable-artifacts-buildcache: false
rebuild-index: true
target: gitlab
pipeline-gen:
- submapping:
- match: [os=sles15]
build-job:
tags: [perlmutter-e4s]
match_behavior: first
- any-job:
before_script:
- git clone -c feature.manyFiles=true -b e4s-23.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- build-job:
tags: [perlmutter-e4s]
before_script:
- module reset
- module load cpu cray-pmi
- module list
- source setup-env.sh
- git clone -c feature.manyFiles=true -b e4s-23.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- spack-python --path
script:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- spack env st
- export SPACK_GNUPGHOME=$HOME/.gnupg
- spack gpg list
- spack -d ci rebuild
- reindex-job:
tags: [perlmutter-e4s]
script:
- echo "End Pipeline"
- noop-job:
tags: [perlmutter-e4s]
script:
- echo "End Pipeline"
specs:
- matrix:
- [$data_specs]
- [$gcc_compilers]
MATH-LIBS Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree:
padded_length: 128
concretizer:
reuse: false
unify: false
mirrors:
perlmutter-e4s-23.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-23.05
ci:
enable-artifacts-buildcache: false
rebuild-index: true
target: gitlab
pipeline-gen:
- submapping:
- match: [os=sles15]
build-job:
tags: [perlmutter-e4s]
match_behavior: first
- any-job:
before_script:
- git clone -c feature.manyFiles=true -b e4s-23.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- build-job:
tags: [perlmutter-e4s]
before_script:
- module reset
- module load cpu cray-pmi
- module list
- source setup-env.sh
- git clone -c feature.manyFiles=true -b e4s-23.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- spack-python --path
script:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- spack env st
- export SPACK_GNUPGHOME=$HOME/.gnupg
- spack gpg list
- spack -d ci rebuild
- reindex-job:
tags: [perlmutter-e4s]
script:
- echo "End Pipeline"
- noop-job:
tags: [perlmutter-e4s]
script:
- echo "End Pipeline"
specs:
- matrix:
- [$math-libs]
- [$gcc_compilers]
TOOLS Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree:
padded_length: 128
concretizer:
reuse: false
unify: false
mirrors:
perlmutter-e4s-23.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-23.05
ci:
enable-artifacts-buildcache: false
rebuild-index: true
target: gitlab
pipeline-gen:
- submapping:
- match: [os=sles15]
build-job:
tags: [perlmutter-e4s]
match_behavior: first
- any-job:
before_script:
- git clone -c feature.manyFiles=true -b e4s-23.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- build-job:
tags: [perlmutter-e4s]
before_script:
- module reset
- module load cpu cray-pmi
- module list
- source setup-env.sh
- git clone -c feature.manyFiles=true -b e4s-23.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- spack-python --path
script:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- spack env st
- export SPACK_GNUPGHOME=$HOME/.gnupg
- spack gpg list
- spack -d ci rebuild
- reindex-job:
tags: [perlmutter-e4s]
script:
- echo "End Pipeline"
- noop-job:
tags: [perlmutter-e4s]
script:
- echo "End Pipeline"
specs:
- matrix:
- [$tools]
- [$gcc_compilers]
Perlmutter E4S 22.11
Production Spack Configuration
GCC spack environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree: $spack/opt/spack
concretizer:
reuse: false
mirrors:
perlmutter-e4s-22.11: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-22.11
specs:
- matrix:
- [$gcc_specs]
- [$gcc_compilers]
- matrix:
- [$utilities]
- [$gcc_compilers]
- matrix:
- [$nersc_specs]
- [$gcc_compilers]
CCE Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree: $spack/opt/spack
concretizer:
reuse: false
mirrors:
perlmutter-e4s-22.11: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-22.11
specs:
- matrix:
- [$cce_specs]
- [$cce_compilers]
NVHPC Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree: $spack/opt/spack
concretizer:
reuse: false
mirrors:
perlmutter-e4s-22.11: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-22.11
packages:
# build failures with xz %nvhpc so reverting to gcc
xz::
require: '%gcc'
specs:
- matrix:
- [$nvhpc_specs]
- [$nvhpc_compilers]
CUDA Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree: $spack/opt/spack
concretizer:
reuse: false
mirrors:
perlmutter-e4s-22.11: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-22.11
specs:
- matrix:
- [$cuda_specs]
- [$gcc_compilers]
Shown below is the list of definitions that is used for all of our spack environments.
Definitions for Spack Environments
definitions:
- cce_compilers: ['%cce@15.0.0']
- gcc_compilers: ['%gcc@11.2.0']
- nvhpc_compilers: ['%nvhpc@22.7']
- gcc_specs:
- adios2@2.8.3
- alquimia@1.0.10
- aml@0.2.0
- amrex@22.11
- arborx@1.3
- argobots@1.1
- axom@0.7.0
- bolt@2.0
- butterflypack@2.2.2
- cabana@0.5.0
- caliper@2.8.0
- chai@2022.03.0 ~benchmarks ~tests
- conduit@0.8.4 ~blt_find_mpi
- datatransferkit@3.1-rc3
- dyninst@12.2.0 ^intel-tbb
#- ecp-data-vis-sdk@1.0 ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 +paraview +pnetcdf +sz +unifyfs +veloc +visit +vtkm +zfp
- flecsi@1.4.2
- flit@2.1.0
- flux-core@0.44.0
- fortrilinos@2.1.0
- gasnet@2022.9.0
- ginkgo@1.4.0
- globalarrays@5.8
- gotcha@1.0.4
- gptune@3.0.0
- h5bench@1.3
- hdf5@1.12.2 +fortran +hl +shared
- hdf5-vol-async@1.3
- heffte@2.3.0 +fftw
- hpctoolkit@2022.10.01
- hpx@1.8.1 networking=mpi
- hypre@2.26.0
- kokkos@3.7.00 +openmp
- kokkos-kernels@3.7.00 +openmp
- lammps@20220623
- legion@21.03.0
- libquo@1.3.1
- libunwind@1.6.2
- mercury@2.1.0
- mfem@4.5.0
- mpark-variant@1.4.0
#- mpifileutils@0.11.1 ~xattr
- nccmp@1.9.0.1
- nco@5.0.1
- netlib-scalapack@2.2.0
- openblas@0.3.20 threads=openmp
- omega-h@9.34.13
- openpmd-api@0.14.5 ~adios2
- papi@6.0.0.1
- papyrus@1.0.2
- parallel-netcdf@1.12.3
- parsec@3.0.2209 ~cuda
- pdt@3.25.1
- petsc@3.18.1
- precice@2.5.0
- pumi@2.2.7
- py-libensemble@0.9.3
- py-h5py +mpi
- py-h5py ~mpi
- py-petsc4py@3.18.1
- py-warpx@22.10 ^warpx dims=2
- py-warpx@22.10 ^warpx dims=3
- py-warpx@22.10 ^warpx dims=rz
- qt@5.14.2
- qthreads@1.16 scheduler=distrib
- quantum-espresso@7.1
- raja@2022.03.0
- slate@2022.07.0 ~cuda
- slepc@3.18.1
- strumpack@7.0.1 ~slate
- sundials@6.4.1
- superlu@5.3.0
- superlu-dist@8.1.1
- tasmanian@7.9
- tau@2.32 +mpi +python
- trilinos@13.0.1 +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext
+ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
+nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
+teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
- umap@2.1.0
- umpire@2022.03.1
- upcxx@2022.9.0
- vtk-m@1.9.0
- zfp@0.5.5
# - bricks@r0.1 build failure
# - plasma@22.9.29 Could NOT find MKL (missing: MKL_INCLUDE_DIRS MKL_LIBRARIES)
# - phist@1.11.2 build error
# - stc@0.9.0 build failure on swig
# - turbine@1.3.0 failed to build swig
# - upcxx+gasnet^gasnet@2022.9.2 # see https://software.nersc.gov/NERSC/spack-infrastructure/-/issues/45 Issue with getting checksum for gasnet
# - visit@3.2.2 ~hdf5
# - wannier90@3.1.0 Error: Type mismatch between actual argument at (1) and actual argument at (2) (COMPLEX(8)/INTEGER(4)).../comms.F90:1214:22: 1214 | call MPI_scatterv(rootglobalarray, counts, displs, MPI_double_precision, &
- cuda_specs:
- adios2@2.8.3 +cuda cuda_arch=80 # ecp-data-vis-sdk
- amrex@22.11 +cuda cuda_arch=80
- arborx@1.3 +cuda cuda_arch=80 ^kokkos@3.7.00 +wrapper
- cabana@0.5.0 +cuda ^kokkos@3.7.00 +wrapper +cuda_lambda +cuda cuda_arch=80
- caliper@2.8.0 +cuda cuda_arch=80
- chai@2022.03.0 ~benchmarks ~tests +cuda cuda_arch=80 ^umpire@2022.03.1 ~shared
- cusz@0.3 +cuda cuda_arch=80
- flecsi@2.1.0 +cuda cuda_arch=80
- ginkgo@1.4.0 +cuda cuda_arch=80
- heffte@2.3.0 +cuda cuda_arch=80
- hpx@1.8.1 +cuda cuda_arch=80
- hypre@2.26.0 +cuda cuda_arch=80
- kokkos-kernels@3.7.00 +cuda cuda_arch=80 ^kokkos@3.7.00 +wrapper +cuda cuda_arch=80
- kokkos@3.7.00 +wrapper +cuda cuda_arch=80
- mfem@4.5.0 +cuda cuda_arch=80
- omega-h@9.34.13 +cuda cuda_arch=80
- petsc@3.18.1 +cuda cuda_arch=80
- slate@2022.07.0 +cuda cuda_arch=80
- slepc@3.18.1 +cuda cuda_arch=80
- strumpack@7.0.1 ~slate +cuda cuda_arch=80
- sundials@6.4.1 +cuda cuda_arch=80
- superlu-dist@8.1.1 +cuda cuda_arch=80
- tasmanian@7.9 +cuda cuda_arch=80
- umpire@2022.03.1 ~shared +cuda cuda_arch=80
- vtk-m@1.9.0 +cuda cuda_arch=80 # ecp-data-vis-sdk
- zfp@0.5.5 +cuda cuda_arch=80 # ecp-data-vis-sdk
#- ascent@0.8.0 +cuda cuda_arch=80 # unable to build vtk-h -- Could NOT find MPI_C (missing: MPI_C_LIB_NAMES) (found version "3.1")
# - dealii@9.4.0 +cuda cuda_arch=80 # CUDA_LIBRARIES: *** Required variable "CUDA_cusparse_LIBRARY" set to NOTFOUND ***
#- magma@2.6.2 +cuda cuda_arch=80 # CMake Error: The following variables are used in this project, but they are set to NOTFOUND.
#- raja@2022.03.0 +cuda cuda_arch=80 # error: "RAJA::expt::Register<int32_t, RAJA::expt::avx2_register> &(const int32_t &)" contains a vector, which is not supported in device code
#- trilinos@13.4.0 +cuda cuda_arch=80
- nvhpc_specs:
- hdf5@1.12.2 +fortran +hl +shared
- kokkos@3.7.00 +openmp
- kokkos-kernels@3.7.00 +openmp
- sundials@6.4.1
- superlu@5.3.0
#- superlu-dist@8.1.1 # error: this OpenMP construct is not supported in NVIDIA subset: The 'taskloop' construct is not supported.
- cce_specs:
- fftw@3.3.10
- hdf5@1.12.2 +fortran +hl +shared
- hypre@2.26.0
- kokkos@3.7.00 +openmp
- kokkos-kernels@3.7.00 +openmp
- netcdf-c@4.9.0
- netcdf-fortran@4.6.0
- sundials@6.4.1
- superlu@5.3.0
- superlu-dist@8.1.1
- nersc_specs:
- chapel@1.24.1
- gsl@2.7.1
- fftw@3.3.10
- fftw@3.3.9
- fftw@3.3.8
- netcdf-c@4.9.0 ~mpi
- netcdf-fortran@4.6.0
- metis@5.1.0
- parmetis@4.0.3
- gromacs@2022.3
- cdo
# trouble building esmf +mpi
- ncl ^esmf~mpi
- nco
- ncview
- libxc
- libcint
- libint tune=molgw-lmax-7
- intel-mkl
- utilities:
- autoconf
- automake
- cmake
- git
- gmake
- gawk
- nano
- subversion
Each spack environment is built in a separate directory using spack ci in-order to push specs to buildcache. We have the following spack configuration for each spack environment.
Spack Environments for Spack CI
GCC Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree:
padded_length: 128
concretizer:
reuse: false
mirrors:
perlmutter-e4s-22.11: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-22.11
# source_mirror: file:///global/cfs/cdirs/m3503/mirrors/source_mirror
gitlab-ci:
enable-artifacts-buildcache: false
rebuild-index: true
before_script:
- module reset
- module load cpu cray-pmi
- module list
- source setup-env.sh
- git clone -c feature.manyFiles=true -b e4s-22.11 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- spack-python --path
script:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- spack env st
- export SPACK_GNUPGHOME=$HOME/.gnupg
- spack gpg list
- spack -d ci rebuild
service-job-attributes:
tags: [perlmutter-e4s]
before_script:
- git clone -b e4s-22.11 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- spack --version
- ls -l /global/common/software/spackecp/mirrors/perlmutter-e4s-22.11/build_cache/_pgp
script:
- echo "End Pipeline"
mappings:
- match: [os=sles15]
runner-attributes:
tags: [perlmutter-e4s]
specs:
- matrix:
- [$gcc_specs]
- [$gcc_compilers]
- matrix:
- [$utilities]
- [$gcc_compilers]
- matrix:
- [$nersc_specs]
- [$gcc_compilers]
CCE Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree:
padded_length: 128
concretizer:
reuse: false
mirrors:
perlmutter-e4s-22.11: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-22.11
gitlab-ci:
enable-artifacts-buildcache: false
rebuild-index: true
before_script:
- module reset
- module load cpu cray-pmi
- module list
- source setup-env.sh
- git clone -c feature.manyFiles=true -b e4s-22.11 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- spack-python --path
script:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- spack env st
- export SPACK_GNUPGHOME=$HOME/.gnupg
- spack gpg list
- spack -d ci rebuild
service-job-attributes:
tags: [perlmutter-e4s]
before_script:
- git clone -b e4s-22.11 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- spack --version
- ls -l /global/common/software/spackecp/mirrors/perlmutter-e4s-22.11/build_cache/_pgp
script:
- echo "End Pipeline"
mappings:
- match: [os=sles15]
runner-attributes:
tags: [perlmutter-e4s]
specs:
- matrix:
- [$cce_specs]
- [$cce_compilers]
NVHPC Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree:
padded_length: 128
concretizer:
reuse: false
packages:
# build failures with xz %nvhpc so reverting to gcc
xz::
require: '%gcc'
mirrors:
perlmutter-e4s-22.11: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-22.11
gitlab-ci:
enable-artifacts-buildcache: false
rebuild-index: true
before_script:
- module reset
- module load cpu cray-pmi
- module list
- source setup-env.sh
- git clone -c feature.manyFiles=true -b e4s-22.11 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- spack-python --path
script:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- spack env st
- export SPACK_GNUPGHOME=$HOME/.gnupg
- spack gpg list
- spack -d ci rebuild
service-job-attributes:
tags: [perlmutter-e4s]
before_script:
- git clone -b e4s-22.11 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- spack --version
- ls -l /global/common/software/spackecp/mirrors/perlmutter-e4s-22.11/build_cache/_pgp
script:
- echo "End Pipeline"
mappings:
- match: [os=sles15]
runner-attributes:
tags: [perlmutter-e4s]
specs:
- matrix:
- [$nvhpc_specs]
- [$nvhpc_compilers]
CUDA Spack Environment
spack:
view: false
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree:
padded_length: 128
concretizer:
reuse: false
mirrors:
perlmutter-e4s-22.11: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-22.11
# source_mirror: file:///global/cfs/cdirs/m3503/mirrors/source_mirror
gitlab-ci:
enable-artifacts-buildcache: false
rebuild-index: true
before_script:
- module reset
- module load cpu cray-pmi
- module list
- source setup-env.sh
- git clone -c feature.manyFiles=true -b e4s-22.11 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- spack-python --path
script:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- spack env st
- export SPACK_GNUPGHOME=$HOME/.gnupg
- spack gpg list
- spack -d ci rebuild
service-job-attributes:
tags: [perlmutter-e4s]
before_script:
- git clone -b e4s-22.11 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- spack --version
- ls -l /global/common/software/spackecp/mirrors/perlmutter-e4s-22.11/build_cache/_pgp
script:
- echo "End Pipeline"
mappings:
- match: [os=sles15]
runner-attributes:
tags: [perlmutter-e4s]
specs:
- matrix:
- [$cuda_specs]
- [$gcc_compilers]
Perlmutter E4S 22.05
Shown below is the production Spack configuration for Perlmutter E4S 22.05. You can access this stack
via module load e4s/22.05 on Perlmutter. Please see
our user documentation for this stack at https://docs.nersc.gov/applications/e4s/perlmutter/22.05/.
Production Spack Configuration
GCC spack environment
spack:
view:
default:
root: $spack/var/spack/environments/gcc/views/default
select: ['%gcc']
exclude: ['py-warpx']
link_type: symlink
link: roots
projections:
all: '{name}/{version}-{compiler.name}-{compiler.version}'
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree: $spack/opt/spack
mirrors:
perlmutter-e4s-22.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-22.05
source_mirror: file:///global/cfs/cdirs/m3503/mirrors/source_mirror
specs:
- $nersc_specs
- $utilities
- matrix:
- [$gcc_specs]
- [$gcc_compilers]
- matrix:
- - py-libensemble@0.9.1 ^py-numpy ~blas ~lapack
- ['%gcc@10.3.0']
CCE Spack Environment
spack:
view:
default:
root: $spack/var/spack/environments/cce/views/default
select: ['%cce']
link_type: symlink
link: roots
projections:
all: '{name}/{version}-{compiler.name}-{compiler.version}'
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree: $spack/opt/spack
mirrors:
perlmutter-e4s-22.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-22.05
source_mirror: file:///global/cfs/cdirs/m3503/mirrors/source_mirror
specs:
- matrix:
- [$cce_specs]
- [$cce_compilers]
NVHPC Spack Environment
spack:
view:
default:
root: $spack/var/spack/environments/nvhpc/views/default
select: ['%nvhpc']
link_type: symlink
link: roots
projections:
all: '{name}/{version}-{compiler.name}-{compiler.version}'
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree: $spack/opt/spack
mirrors:
perlmutter-e4s-22.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-22.05
source_mirror: file:///global/cfs/cdirs/m3503/mirrors/source_mirror
specs:
- matrix:
- [$nvhpc_specs]
- [$nvhpc_compilers]
CUDA Spack Environment
spack:
view:
default:
root: $spack/var/spack/environments/cuda/views/default
select: ['%gcc +cuda']
link_type: symlink
link: roots
projections:
all: '{name}/{version}-{compiler.name}-{compiler.version}'
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree: $spack/opt/spack
mirrors:
perlmutter-e4s-22.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-22.05
source_mirror: file:///global/cfs/cdirs/m3503/mirrors/source_mirror
specs:
- matrix:
- [$cuda_specs]
- [$gcc_compilers]
Shown below is the list of definitions that is used for all of our spack environments.
Definitions for Spack Environments
definitions:
- cce_compilers: ['%cce@15.0.0']
- gcc_compilers: ['%gcc@11.2.0']
- nvhpc_compilers: ['%nvhpc@22.7']
- gcc_specs:
- adios2@2.8.0
- amrex@22.05
- butterflypack@2.1.1
- conduit@0.8.3
- dyninst@12.1.0
- fortrilinos@2.0.0
- gasnet@2022.3.0
- hdf5@1.10.7 +fortran +hl +shared
- heffte@2.2.0 +fftw
- hpctoolkit@2022.04.15
- hpx@1.7.1 networking=mpi
- hypre@2.24.0
- kokkos@3.6.00 +openmp
- kokkos-kernels@3.6.00 +openmp
- lammps@20220107
- libquo@1.3.1
- nccmp@1.9.0.1
- nco@5.0.1
- mfem@4.4.0
- openblas@0.3.20 threads=openmp
- openpmd-api@0.14.4
- papi@6.0.0.1
- parallel-netcdf@1.12.2
- pdt@3.25.1
- petsc@3.17.1
- plasma@21.8.29 ^openblas
- py-warpx@22.05 ^warpx dims=2 ^openblas
- py-warpx@22.05 ^warpx dims=3 ^openblas
- py-warpx@22.05 ^warpx dims=rz ^openblas
- qthreads@1.16 scheduler=distrib
- raja@0.14.0
- slate@2021.05.02 ~cuda
- slepc@3.17.1
- strumpack@6.3.1 ~slate
- sundials@6.2.0
- superlu@5.3.0
- superlu-dist@7.2.0
- tasmanian@7.7
- trilinos@13.0.1 +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext
+ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
+nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
+teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
- vtk-m@1.7.1
- upcxx@2022.3.0
- zfp@0.5.5
#- ascent@0.8.0 # ascent (DIFFERENT ERROR in ParaTools deployment %gcc@11.2.0)
#- plumed@2.6.3 # plumed (SAME ERROR in ParaTools deployment %gcc@11.2.0)
#- variorum@0.4.1 # variorum (SAME ERROR in ParaTools deployment %gcc@11.2.0)
#- wannier90@3.1.0 # wannier90 (SAME ERROR in ParaTools deployment %gcc@11.2.0)
# ascent: Could NOT find MPI_C (missing: MPI_C_LIB_NAMES) (found version "3.1")
# plumed: tools/../../tools/lepton/../../lepton/Operation.h:902:39: error: 'numeric_limits' is not a member of 'std'
# variorum (1/2): make[2]: *** [variorum/CMakeFiles/variorum.dir/build.make:196: variorum/libvariorum.so] Error 1
# variorum (2/2): /usr/bin/ld: Intel/CMakeFiles/variorum_intel.dir/msr_core.c.o:(.bss+0x0): multiple definition of `g_platform'; CMakeFiles/variorum.dir/config_architecture.c.o:(.bss+0x0): first defined here
# wannier90: Error: Type mismatch between actual argument at (1) and actual argument at (2) (COMPLEX(8)/INTEGER(4)).
- cuda_specs:
- adios2@2.8.0 +cuda cuda_arch=80
- hypre@2.24.0 +cuda cuda_arch=80
- kokkos@3.6.00 +wrapper +cuda cuda_arch=80
- kokkos-kernels@3.6.00 +cuda cuda_arch=80 ^kokkos@3.6.00 +wrapper +cuda cuda_arch=80
- lammps@20220107 +cuda cuda_arch=80 +cuda_mps
- hpctoolkit@2022.04.15 +cuda +mpi ~papi
- likwid +cuda
- papi@6.0.0.1 +cuda
- petsc@3.17.1+cuda cuda_arch=80
- raja@0.14.0 +cuda cuda_arch=80
- slate@2021.05.02 +cuda cuda_arch=80
- slepc@3.17.1 +cuda cuda_arch=80
- strumpack@6.3.1 ~slate +cuda cuda_arch=80
- tau@2.31.1 +mpi +cuda
- zfp@0.5.5 +cuda cuda_arch=80
#- heffte@2.2.0 +cuda cuda_arch=80 # heffte (WORKED in ParaTools deployment %gcc@11.2.0)
#- hpx@1.7.1 +cuda cuda_arch=80 # hpx (WORKED in ParaTools deployment %gcc@11.2.0)
#- magma@2.6.2 +cuda cuda_arch=80 # magma (WORKED in ParaTools deployment %gcc@11.2.0)
#- parsec@3.0.2012 +cuda cuda_arch=80 # parsec (SAME ERROR in ParaTools deployment %gcc@11.2.0)
#- sundials@6.2.0 +cuda cuda_arch=80 # sundials (WORKED in ParaTools deployment %gcc@11.2.0)
#- superlu-dist@7.2.0 +cuda cuda_arch=80 # superlu-dist (WORKED in ParaTools deployment %gcc@11.2.0)
#- tasmanian@7.7 +cuda cuda_arch=80 # tasmanian (WORKED in ParaTools deployment %gcc@11.2.0)
#- trilinos@13.2.0 +cuda cuda_arch=80 # trilinos (SAME ERROR in ParaTools deployment %gcc@11.2.0)
#- vtk-m@1.7.1 +cuda cuda_arch=80 # vtk-m (DIFF ERROR in ParaTools deployment %gcc@11.2.0)
# heffte: CMake Error: The following variables are used in this project, but they are set to NOTFOUND: CUDA_cufft_LIBRARY
# hpx: CMake Error: The following variables are used in this project, but they are set to NOTFOUND: CUDA_cublas_LIBRARY
# magma: CMake Error: The following variables are used in this project, but they are set to NOTFOUND: CUDA_cublas_LIBRARY
# parsec: transfer.c:168: multiple definition of `parsec_CUDA_d2h_max_flows';
# sundials: spack-src/examples/sunmatrix/cusparse/test_sunmatrix_cusparse.cu:167: undefined reference to `cusparseCreate'
# superlu-dist: make[2]: *** No rule to make target '/opt/nvidia/hpc_sdk/Linux_x86_ 64/21.11/cuda/11.5/lib64/libcublas.so', needed by 'SRC/CMakeFiles/superlu_dist.dir/cmake_device_link.o'.
# tasmanian: CMake Error at /global/cfs/cdirs/m3503/ci-builds/perlmutter/yUW7FC66... /Modules/FindPackageHandleStandardArgs.cmake:230 (message): Could NOT find TasmanianCudaMathLibs
# trilinos +cuda: CMake Error at /global/cfs....CMakeTestCXXCompiler.cmake:62 (message): The C++ compiler "/opt/cray/pe/mpich/8.1.13/ofi/gnu/9.1/bin/mpicxx" is not able to compile a simple test program.
# vtk-m: spack-src/vtkm/internal/brigand.hpp:1061:131: error: expected class-name before '{' token struct find<true, false, L1, L2, Ls...> : find<true , F<Ts..., L2>::value, L2, Ls...>
- cce_specs:
- adios2@2.8.0
- hdf5@1.10.7 +fortran +hl +shared
- kokkos-kernels@3.6.00 +openmp
- kokkos@3.6.00 +openmp
- petsc@3.17.1 ~strumpack
- sundials@6.2.0
- superlu-dist@7.2.0
- superlu@5.3.0
# - hypre@2.24.0 # error in compilation: clang-15: error: unknown argument: '-qsmp=omp'
#- openblas@0.3.20 threads=openmp # openblas (SAME ERROR in ParaTools deployment %cce@14.0.0)
#- strumpack@6.3.1 ~slate # butterflypack
# openblas: ftn-2307 ftn: ERROR in command line The "-m" option must be followed by 0, 1, 2, 3 or 4. ftn-2103 ftn: WARNING in command line.The -W all option is not supported or invalid and will be ignored.
- nvhpc_specs:
- hdf5@1.10.7 +fortran +hl +shared
- kokkos@3.6.00 +openmp
- kokkos-kernels@3.6.00 +openmp
- sundials@6.2.0
- superlu@5.3.0
- zfp@0.5.5
# - raja@0.14.0 +cuda cuda_arch=80 build failure
# - slate@2021.05.02 +cuda cuda_arch=80
# - strumpack@6.3.1 ~slate build failure
# - superlu-dist@7.2.0 NVIDIA subset: The 'taskloop' construct is not supported.
# ==> Warning: Skipping build of superlu-dist-7.2.0-kmvqtie752awoenzob5pgemabeuzyaiv since parmetis-4.0.3-ew3itm7cvpheb7h4e6lmyteghbq7toeo failed
# ==> Warning: Skipping build of hypre-2.24.0-s2nqxywgft5dfjeps4jgiim6dxvedz3y since superlu-dist-7.2.0-kmvqtie752awoenzob5pgemabeuzyaiv failed
#- hypre@2.24.0 +cuda cuda_arch=80
# failed to build HDF5 dependency with nvhpc
#- raja@0.14.0 +cuda cuda_arch=80
- nersc_specs:
- ccache@4.5.1
- cdo
- gnuplot@5.4.3 +X
- grads
- gromacs@2021.5
- gsl@2.7
- metis@5.1.0
- intel-mkl
- ncl
- ncview@2.1.8
- parmetis@4.0.3
- parallel
- quantum-espresso@7.0
- nwchem@7.0.2 ^cray-libsci
- utilities:
- autoconf
- automake
- cmake
- git
- gmake
- gawk
- nano
- subversion
- xterm
Shown below is the list of spack environments that is used for building the stack into buildcache using spack ci.
Spack Environments for Spack CI
GCC Spack Environment
spack:
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree:
padded_length: 128
concretizer:
reuse: false
cdash:
build-group: DOE nightly E4S builds
url: https://cdash.spack.io
project: Spack
site: NERSC - Perlmutter E4S-22.05
mirrors:
perlmutter-e4s-22.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-22.05
gitlab-ci:
enable-artifacts-buildcache: false
rebuild-index: true
before_script:
- module reset
- module use /global/common/software/nersc/$(cat /etc/nersc_modules_rev)/extra_modulefiles
- module load cpu
- module load cray-pmi
- module list
- source setup-env.sh
- git clone --depth=10 -b e4s-22.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- spack-python --path
script:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- spack env st
- export SPACK_GNUPGHOME=$HOME/.gnupg
- spack gpg list
- spack -d ci rebuild
service-job-attributes:
tags: [perlmutter-e4s]
before_script:
- git clone --depth=10 -b e4s-22.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- spack --version
- ls -l /global/common/software/spackecp/mirrors/perlmutter-e4s-22.05/build_cache/_pgp
script:
- echo "End Pipeline"
mappings:
- match: [os=sles15]
runner-attributes:
tags: [perlmutter-e4s]
specs:
- $nersc_specs
- $utilities
- matrix:
- [$gcc_specs]
- [$gcc_compilers]
- matrix:
- - py-libensemble@0.9.1 ^py-numpy ~blas ~lapack
- ['%gcc@10.3.0']
CCE Spack Environment
spack:
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree:
padded_length: 128
concretizer:
reuse: false
cdash:
build-group: DOE nightly E4S builds
url: https://cdash.spack.io
project: Spack
site: NERSC - Perlmutter E4S-22.05
mirrors:
perlmutter-e4s-22.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-22.05
gitlab-ci:
enable-artifacts-buildcache: false
rebuild-index: true
before_script:
- module reset
- module use /global/common/software/nersc/$(cat /etc/nersc_modules_rev)/extra_modulefiles
- module load cpu
- module load cray-pmi
- module list
- source setup-env.sh
- git clone --depth=10 -b e4s-22.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- spack-python --path
script:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- spack env st
- export SPACK_GNUPGHOME=$HOME/.gnupg
- spack gpg list
- spack -d ci rebuild
service-job-attributes:
tags: [perlmutter-e4s]
before_script:
- git clone --depth=10 -b e4s-22.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- spack --version
- ls -l /global/common/software/spackecp/mirrors/perlmutter-e4s-22.05/build_cache/_pgp
script:
- echo "End Pipeline"
mappings:
- match: [os=sles15]
runner-attributes:
tags: [perlmutter-e4s]
specs:
- matrix:
- [$cce_specs]
- [$cce_compilers]
NVHPC Spack Environment
spack:
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree:
padded_length: 128
concretizer:
reuse: false
cdash:
build-group: DOE nightly E4S builds
url: https://cdash.spack.io
project: Spack
site: NERSC - Perlmutter E4S-22.05
mirrors:
perlmutter-e4s-22.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-22.05
gitlab-ci:
enable-artifacts-buildcache: false
rebuild-index: true
before_script:
- module reset
- module use /global/common/software/nersc/$(cat /etc/nersc_modules_rev)/extra_modulefiles
- module load cpu
- module load cray-pmi
- module list
- source setup-env.sh
- git clone --depth=10 -b e4s-22.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- spack-python --path
script:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- spack env st
- export SPACK_GNUPGHOME=$HOME/.gnupg
- spack gpg list
- spack -d ci rebuild
service-job-attributes:
tags: [perlmutter-e4s]
before_script:
- git clone --depth=10 -b e4s-22.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- spack --version
- ls -l /global/common/software/spackecp/mirrors/perlmutter-e4s-22.05/build_cache/_pgp
script:
- echo "End Pipeline"
mappings:
- match: [os=sles15]
runner-attributes:
tags: [perlmutter-e4s]
specs:
- matrix:
- [$nvhpc_specs]
- [$nvhpc_compilers]
CUDA Spack Environment
spack:
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
config:
concretization: separately
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
concretizer: clingo
install_tree:
padded_length: 128
concretizer:
reuse: false
cdash:
build-group: DOE nightly E4S builds
url: https://cdash.spack.io
project: Spack
site: NERSC - Perlmutter E4S-22.05
mirrors:
perlmutter-e4s-22.05: file:///global/common/software/spackecp/mirrors/perlmutter-e4s-22.05
gitlab-ci:
enable-artifacts-buildcache: false
rebuild-index: true
before_script:
- module reset
- module use /global/common/software/nersc/$(cat /etc/nersc_modules_rev)/extra_modulefiles
- module load cpu
- module load cray-pmi
- module list
- source setup-env.sh
- git clone --depth=10 -b e4s-22.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- which spack
- spack --version
- spack-python --path
script:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- spack env st
- export SPACK_GNUPGHOME=$HOME/.gnupg
- spack gpg list
- spack -d ci rebuild
service-job-attributes:
tags: [perlmutter-e4s]
before_script:
- git clone --depth=10 -b e4s-22.05 $SPACK_REPO
- . spack/share/spack/setup-env.sh
- spack --version
- ls -l /global/common/software/spackecp/mirrors/perlmutter-e4s-22.05/build_cache/_pgp
script:
- echo "End Pipeline"
mappings:
- match: [os=sles15]
runner-attributes:
tags: [perlmutter-e4s]
specs:
- matrix:
- [$cuda_specs]
- [$gcc_compilers]
Perlmutter E4S 21.11
Shown below is the production Spack configuration for Perlmutter E4S 21.11. You can access this stack
via module load e4s/21.11 or module load e4s/21.11 on Perlmutter. Please see
our user documentation for this stack at https://docs.nersc.gov/applications/e4s/perlmutter/21.11/.
Production Spack Environment
# This is a Spack Environment file.
#
# It describes a set of packages to be installed, along with
# configuration settings.
spack:
view: false
config:
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
install_tree:
concretizer: clingo
root: $spack/opt/spack
include:
- /global/common/software/spackecp/perlmutter/spack_settings/compilers.yaml
- /global/common/software/spackecp/perlmutter/spack_settings/packages.yaml
specs:
- matrix:
- [$gcc_specs]
- [$gcc_compilers]
- matrix:
- [$cuda_specs]
- [$gcc_compilers]
- $nersc_specs
definitions:
- gcc_compilers: ['%gcc@11.2.0']
- gcc_specs:
- adios2@2.7.1
- amrex@21.11 +fortran +openmp +shared
- conduit@0.7.2
- dyninst@11.0.1
- gasnet@2021.9.0
- globalarrays@5.8
- hdf5@1.12.1
- kokkos-kernels@3.4.01 +openmp
- kokkos@3.4.01 +openmp
- mercury@2.0.1
- mpark-variant@1.4.0
- openpmd-api@0.14.3
- papi@6.0.0.1
- papyrus@1.0.2
- pdt@3.25.1
- qthreads@1.16 scheduler=distrib
- raja@0.14.0
- strumpack@6.1.0 ~slate
- sundials@5.8.0 +openmp
- superlu-dist@7.1.1 +openmp
- superlu@5.3.0
- swig@4.0.2
- sz@2.1.12
- tau +mpi +python
- trilinos@13.0.1 +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
- umap@2.1.0
- upcxx@2021.9.0 +gasnet +mpi
#- slepc@3.16.0 --dpends on petsc
#- mfem@4.3.0 --depends on hypre
#- petsc@3.16.1 +openmp +strumpack --failed to install
#- hypre@2.23.0 +openmp +superlu-dist --failed to install
#- parsec@3.0.2012 ~cuda --failed to install
#- warpx dims=2
#- warpx dims=3
- cuda_specs:
- kokkos-kernels@3.4.01 +openmp +cuda cuda_arch=80 ^kokkos +openmp +wrapper +cuda cuda_arch=80
- kokkos@3.4.01 +openmp +wrapper +cuda cuda_arch=80
- umpire@6.0.0 ~shared +cuda cuda_arch=80
- upcxx@2021.9.0 +cuda
- zfp@0.5.5 +cuda cuda_arch=80
- raja@0.14.0+cuda cuda_arch=80 # CUB in your include path is not compatible with this release of Thrust
#- mfem@4.3.0+cuda cuda_arch=80 --depends on hypre
#- slepc@3.16.0 +cuda cuda_arch=80 ^petsc@3.16.1 +cuda cuda_arch=80 --depends on petsc
#- petsc@3.16.1 +cuda cuda_arch=80 --failed to install
#- hypre@2.23.0+cuda cuda_arch=80 --failed to install
#- parsec@3.0.2012+cuda cuda_arch=80 # parsec/mca/device/cuda/transfer.c:168: multiple definition of `parsec_CUDA_d2h_max_flows'
#- amrex@21.11 +cuda cuda_arch=80
#- magma@2.6.1+cuda cuda_arch=80
#- strumpack@6.1.0 ~slate +cuda cuda_arch=80
#- slate@2021.05.02 +cuda cuda_arch=80
#- superlu-dist@7.1.1 +openmp +cuda cuda_arch=80
#- sundials@5.8.0 +openmp +cuda cuda_arch=80
- nersc_specs:
- chapel@1.24.1
- gsl@2.7
- fftw@3.3.10
- nccmp@1.9.0.1
- netcdf-c@4.8.1
- netcdf-fortran@4.5.3
- nco@5.0.1
- metis@5.1.0
- parallel-netcdf@1.12.2
- parmetis@4.0.3
- gromacs@2021.3
#- plumed@2.6.3
#- wannier90@3.1.0
Cori E4S 22.02
Production Spack Environment
# This is a Spack Environment file.
#
# It describes a set of packages to be installed, along with
# configuration settings.
spack:
view: false
config:
build_stage: $spack/var/spack/stage
misc_cache: $spack/var/spack/misc_cache
install_tree:
concretizer: clingo
root: /global/common/software/spackecp/cori/e4s-22.02/software
module_roots:
tcl: /global/common/software/spackecp/cori/e4s-22.02/modules
mirrors:
source_mirror: file:///global/cfs/cdirs/m3503/mirrors/source_mirror
modules::
prefix_inspections:
bin:
- PATH
lib:
- LIBRARY_PATH
- LD_LIBRARY_PATH
lib64:
- LIBRARY_PATH
- LD_LIBRARY_PATH
include:
- C_INCLUDE_PATH
- CPLUS_INCLUDE_PATH
- CPATH
man:
- MANPATH
share/man:
- MANPATH
share/aclocal:
- ACLOCAL_PATH
lib/pkgconfig:
- PKG_CONFIG_PATH
lib64/pkgconfig:
- PKG_CONFIG_PATH
share/pkgconfig:
- PKG_CONFIG_PATH
'':
- CMAKE_PREFIX_PATH
enable:
- tcl
tcl:
blacklist_implicits: true
hash_length: 0
naming_scheme: '{name}/{version}-{compiler.name}-{compiler.version}'
all:
autoload: direct
conflict:
- '{name}'
environment:
set:
'{name}_ROOT': '{prefix}'
darshan-runtime:
conflict:
- darshan
darshan-util:
conflict:
- darshan
projections:
all: '{name}/{version}-{compiler.name}-{compiler.version}'
warpx dims=1: '{name}/{version}-{compiler.name}-{compiler.version}-dims1'
warpx dims=2: '{name}/{version}-{compiler.name}-{compiler.version}-dims2'
warpx dims=3: '{name}/{version}-{compiler.name}-{compiler.version}-dims3'
compilers:
- compiler:
spec: gcc@11.2.0
paths:
cc: /opt/cray/pe/craype/default/bin/cc
cxx: /opt/cray/pe/craype/default/bin/CC
f77: /opt/cray/pe/craype/default/bin/ftn
fc: /opt/cray/pe/craype/default/bin/ftn
flags: {}
operating_system: cnl7
target: any
modules:
- PrgEnv-gnu
- gcc/11.2.0
- craype-haswell
- compiler:
spec: intel@19.1.2.254
paths:
cc: /opt/cray/pe/craype/default/bin/cc
cxx: /opt/cray/pe/craype/default/bin/CC
f77: /opt/cray/pe/craype/default/bin/ftn
fc: /opt/cray/pe/craype/default/bin/ftn
flags: {}
operating_system: cnl7
target: any
modules:
- PrgEnv-intel
- intel/19.1.2.254
- craype-haswell
definitions:
- gcc_compilers: ['%gcc@11.2.0']
- intel_compilers: ['%intel@19.1.2.254']
- gcc_specs:
- adios2@2.7.1
- amrex@22.02
- aml@0.1.0
- arborx@1.1
- argobots@1.1
- axom@0.6.1
- bolt@2.0
- caliper@2.7.0
- chai@2.4.0 ~benchmarks ~tests
- conduit@0.8.2
- darshan-runtime@3.3.1
- darshan-util@3.3.1
- dyninst@12.0.1
- faodel@1.2108.1
- flecsi@1.4.2
- flit@2.1.0
- gasnet@2021.9.0
- ginkgo@1.4.0
- globalarrays@5.8
- gotcha@1.0.3
- hdf5@1.10.7 +fortran +hl +shared
- hdf5@1.12.1 +fortran +hl +shared
- hdf5@1.13.0 +fortran +hl +shared
- heffte@2.2.0 +fftw
- hpx@1.7.1 networking=mpi
- hypre@2.24.0
- kokkos@3.5.00 +openmp +wrapper
- kokkos-kernels@3.5.00 +openmp ^kokkos@3.5.00 +openmp +wrapper
- legion@21.03.0
- libquo@1.3.1
- libunwind@1.5.0
- mercury@2.1.0
- metall@0.17
- mfem@4.3.0
- mpark-variant@1.4.0
- nccmp@1.9.0.1 ^netcdf-c@4.8.1
- nco@5.0.1
- netlib-scalapack@2.1.0
- ninja@1.10.2
- nvhpc@22.1
- openpmd-api@0.14.4
- papi@6.0.0.1
- papyrus@1.0.1
- parallel-netcdf@1.12.2
- parsec@3.0.2012 ~cuda
- pdt@3.25.1
- petsc@3.16.4 +openmp
- pumi@2.2.6
- qthreads@1.16 scheduler=distrib
- raja@0.14.0
- stc@0.9.0
- strumpack@6.3.0~butterflypack~slate
- sundials@6.1.1
- superlu@5.3.0
- superlu-dist@7.2.0
- swig@4.0.2
- sz@2.1.12
- tasmanian@7.7
- tau@2.31 +mpi +python
- turbine@1.3.0
- umap@2.1.0
- umpire@6.0.0
- upcxx@2021.9.0
- veloc@1.5
- vtk-m@1.7.1
- zfp@0.5.5
#- hpctoolkit@2022.01.15
#- phist@1.9.5 #Unable to locate cray-libsci headers in /opt/cray/pe/libsci/20.09.1/gnu/8.1/include
#- mpifileutils@0.11.1 ~xattr # failed to install libcircle Unable to find suitable MPI Compiler. Try setting MPICC.
#- plumed@2.6.3 # Build Failed
#-precice@2.3.0 # Build failed due to petsc
#- py-warpx@22.02 ^warpx dims=rz
#- rempi@1.1.0 failed to find MPICC
#- slate@2021.05.02 ~cuda # Build failure on blaspp. BLAS++ requires a BLAS library and none was found.
#- scr@3.0rc2 Build failure
#- trilinos@13.0.1 +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
#- unifyfs@0.9.1 # Build failure on dependency mercury
#- wannier90@3.1.0 # Error: A PrgEnv-* modulefile must be loaded.
- intel_specs:
- adios2@2.7.1
- arborx@1.1
- argobots@1.1
- caliper@2.7.0
- conduit@0.8.2
- chai@2.4.0 ~benchmarks ~tests
- darshan-runtime@3.3.1
- darshan-util@3.3.1
- faodel@1.2108.1
- flecsi@1.4.2
- flit@2.1.0
- gasnet@2021.9.0
- ginkgo@1.4.0
- globalarrays@5.8
- gotcha@1.0.3
- hdf5@1.10.7 +fortran +hl +shared
- hdf5@1.12.1 +fortran +hl +shared
- hdf5@1.13.0 +fortran +hl +shared
- heffte@2.2.0 +fftw
- hypre@2.24.0
- kokkos@3.5.00 +openmp +wrapper
- legion@21.03.0
- libquo@1.3.1
- libunwind@1.5.0
- loki@0.1.7
- mercury@2.1.0
- metall@0.17
- mfem@4.3.0
- mpark-variant@1.4.0
- nccmp@1.9.0.1 ^netcdf-c@4.8.1
- netlib-scalapack@2.1.0
- ninja@1.10.2
- openpmd-api@0.14.4
- papi@6.0.0.1
- parallel-netcdf@1.12.2
- parsec@3.0.2012 ~cuda
- papyrus@1.0.1
- petsc@3.16.4 +openmp
- pdt@3.25.1
- precice@2.3.0
- pumi@2.2.6
- qthreads@1.16 scheduler=distrib
- raja@0.14.0
- slepc@3.16.2
- strumpack@6.3.0~butterflypack ~slate
- sundials@6.1.1
- superlu@5.3.0
- superlu-dist@7.2.0
- swig@4.0.2
- sz@2.1.12
- tasmanian@7.7
- turbine@1.3.0
- umap@2.1.0
- umpire@6.0.0
- upcxx@2021.9.0
- variorum@0.4.1
- veloc@1.5
- vtk-m@1.7.1
- wannier90@3.1.0
- warpx dims=1
- warpx dims=2
- warpx dims=3
- zfp@0.5.5
#- axom@0.6.1 # Build failure in cmake for axom
#- butterflypack@2.1.0 # Build failure sed: can't read *.inc: No such file or directory
#- dyninst@12.0.1 # %intel conflict
# - hpx@1.7.1 networking=mpi # Failed on asio
#- lammps@20220107 # Build Failure during cmake
#- kokkos-kernels@3.5.00 +openmp ^kokkos@3.5.00 +openmp +wrapper # Build failure
#- plasma@21.8.29 # %intel conflict
#- phist@1.9.5 # Error: NoHeadersError: Unable to locate cray-libsci headers in /opt/cray/pe/libsci/20.09.1/intel/16.0/include
#- plumed@2.6.3
#- rempi@1.1.0 # FAILED to find MPICC
#- slate@2021.05.02 ~cuda # %intel conflict
#- scr@3.0rc2 Build failure
#- trilinos@13.0.1 +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
#- warpx dims=rz failed on blaspp
- nersc_specs:
- cdo@2.0.3 +curl
- chapel@1.24.1
- ffmpeg@4.4.1
- elpa@2021.11.001
- grads@2.2.1
- gsl@2.7
- gromacs@2021.5
- libxc@5.1.7
- libxsmm@1.17 +shared
- libint@2.6.0
- nano@4.9
- maven@3.8.4
- metis@5.1.0
- octave@6.4.0
- openjdk@11.0.12_7
- parallel@20210922
- parmetis@4.0.3
- texlive
- xerces-c@3.2.3
#- abinit+openmp failed to install netcdf-fortran
#- gnuplot build failure on gnuplot
#- valgrind@3.18.1 Build failure
specs:
- matrix:
- [$gcc_specs]
- [$gcc_compilers]
- matrix:
- [$intel_specs]
- [$intel_compilers]
- matrix:
- [$nersc_specs]
- [$gcc_compilers]
packages:
all:
compiler: [gcc@11.2.0, intel@19.1.2.254]
providers:
blas: [cray-libsci, intel-mkl]
fftw-api: [cray-fftw]
mpi: [cray-mpich]
scalapack: [cray-libsci, intel-mkl]
amrex:
variants: +fortran +hypre +openmp +shared
bash:
buildable: false
externals:
- spec: bash@4.4.23
prefix: /
bzip2:
version: [1.0.6]
externals:
- spec: bzip2@1.0.6
prefix: /usr
coreutils:
buildable: false
version: [8.29]
externals:
- spec: coreutils@8.29
prefix: /usr
cpio:
buildable: false
externals:
- spec: cpio@2.12
prefix: /
cray-libsci:
buildable: false
version: [20.09.1]
externals:
- spec: cray-libsci@20.09.1 %gcc
prefix: /opt/cray/pe/libsci/20.09.1/gnu/8.1
modules:
- cray-libsci/20.09.1
- spec: cray-libsci@20.09.1 %intel
prefix: /opt/cray/pe/libsci/20.09.1/intel/16.0
modules:
- cray-libsci/20.09.1
cray-fftw:
buildable: false
externals:
- spec: cray-fftw@3.3.8.10
modules:
- cray-fftw/3.3.8.10
cray-mpich:
buildable: false
externals:
- spec: cray-mpich@7.7.19 %intel
prefix: /opt/cray/pe/mpt/7.7.19/gni/mpich-intel/16.0
modules:
- cray-mpich/7.7.19
- spec: cray-mpich@7.7.19 %gcc
prefix: /opt/cray/pe/mpt/7.7.19/gni/mpich-gnu/8.2
modules:
- cray-mpich/7.7.19
curl:
externals:
- spec: curl@7.66.0+gssapi+ldap+nghttp2
prefix: /usr
diffutils:
version: [3.6]
externals:
- spec: diffutils@3.6
prefix: /usr
elfutils:
version: [0.168]
externals:
- spec: elfutils@0.168
prefix: /usr
findutils:
version: [4.6.0]
externals:
- spec: findutils@4.6.0
prefix: /usr
hdf5:
variants: +fortran +hl +shared api=v18
version: [1.12.1]
externals:
- spec: hdf5@1.12.1.1%intel+shared+fortran+hl~mpi
modules:
- cray-hdf5/1.12.1.1
- spec: hdf5@1.12.1.1%intel+shared+fortran+hl+mpi
modules:
- cray-hdf5-parallel/1.12.1.1
- spec: hdf5@1.12.1.1%gcc+shared+fortran+hl~mpi
modules:
- cray-hdf5/1.12.1.1
- spec: hdf5@1.12.1.1%gcc+shared+fortran+hl+mpi
modules:
- cray-hdf5-parallel/1.12.1.1
hypre:
variants: +openmp +superlu-dist
gawk:
buildable: false
externals:
- spec: gawk@4.2.1
prefix: /usr
git:
version: [2.26.2]
buildable: false
externals:
- spec: git@2.26.2
prefix: /usr
gmake:
buildable: false
externals:
- spec: gmake@4.2.1
prefix: /usr
intel-mkl:
buildable: false
externals:
- spec: intel-mkl@19.1.2.254
modules:
- intel/19.1.2.254
krb5:
buildable: false
externals:
- spec: krb5@1.16.3
prefix: /usr/lib/mit
libunwind:
variants: +pic +xz
mercury:
variants: ~bmi
mesa:
variants: ~llvm
mesa18:
variants: ~llvm
m4:
buildable: false
externals:
- spec: m4@1.4.18
prefix: /usr
mpich:
variants: ~wrapperrpath
ncurses:
variants: +termlib
externals:
- spec: ncurses@6.1
prefix: /usr
netcdf-c:
version: [4.8.1.1]
externals:
- spec: netcdf-c@4.8.1.1~mpi
modules:
- cray-netcdf/4.8.1.1
- spec: netcdf-c@4.8.1.1+mpi
modules:
- cray-netcdf-hdf5parallel/4.8.1.1
openssl:
buildable: false
version: [1.1.1d]
externals:
- spec: openssl@1.1.1d
prefix: /usr
openssh:
buildable: false
externals:
- spec: openssh@8.1p1
prefix: /usr
pdsh:
buildable: false
externals:
- spec: pdsh@2.33
prefix: /usr
petsc:
variants: +openmp
perl:
version: [5.32.0]
pkg-config:
buildable: false
version: [0.29.2]
externals:
- spec: pkg-config@0.29.2
prefix: /usr
readline:
version: [7.0]
buildable: false
externals:
- spec: readline@7.0
prefix: /usr
sed:
buildable: false
externals:
- spec: sed@4.4
prefix: /usr
slurm:
buildable: false
version: [20-11-8-1]
externals:
- spec: slurm@20-11-8-1
prefix: /usr
superlu-dist:
variants: +openmp
strumpack:
variants: ~slate
tar:
version: [1.3]
buildable: false
externals:
- spec: tar@1.30
prefix: /usr
tcsh:
version: [6.20.0]
buildable: false
externals:
- spec: tcsh@6.20.0
prefix: /usr
trilinos:
variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext
+ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
+nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
+teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
unzip:
version: [6.0]
buildable: false
externals:
- spec: unzip@6.0
prefix: /usr
util-linux-uuid:
version: [2.31.1]
buildable: false
externals:
- spec: util-linux-uuid@2.31.1
prefix: /usr
zsh:
version: [5.6]
buildable: false
externals:
- spec: zsh@5.6
prefix: /usr
Cori E4S 21.05
Production Spack Environment
spack:
view: false
concretization: separately
config:
install_tree:
root: /global/common/software/spackecp/e4s-21.05/software
module_roots:
tcl: /global/common/software/spackecp/e4s-21.05/modules/
build_stage: $tempdir/user/spack-stage
modules:
enable:
- tcl
tcl:
blacklist_implicits: true
hash_length: 0
naming_scheme: '{name}/{version}-{compiler.name}-{compiler.version}'
all:
conflict:
- '{name}'
environment:
set:
'{name}_ROOT': '{prefix}'
darshan-runtime:
conflict:
- darshan
darshan-util:
conflict:
- darshan
projections:
all: '{name}/{version}-{compiler.name}-{compiler.version}'
py-warpx ^warpx dims=2: '{name}/{version}-dims2'
py-warpx ^warpx dims=3: '{name}/{version}-dims3'
py-warpx ^warpx dims=rz: '{name}/{version}-dimsRZ'
warpx dims=2: '{name}/{version}-dims2'
warpx dims=3: '{name}/{version}-dims3'
warpx dims=rz: '{name}/{version}-dimsRZ'
boost cxxstd=98: '{name}/{version}-cxxstd=98'
boost cxxstd=17: '{name}/{version}-cxxstd=17'
kokkos +openmp: '{name}/{version}-openmp'
kokkos ~openmp: '{name}/{version}'
mirrors:
cori-e4s-21.05: https://cache.e4s.io/21.05
compilers:
- compiler:
spec: intel@19.1.3.304
paths:
cc: cc
cxx: CC
f77: ftn
fc: ftn
flags: {}
operating_system: cnl7
target: any
modules:
- PrgEnv-intel
- intel/19.1.3.304
environment: {}
extra_rpaths: []
packages:
all:
compiler:
- intel@19.1.3.304
providers:
blas:
- openblas
mpi:
- mpich
target:
- haswell
variants: +mpi
slurm:
buildable: false
version: [20-02-4-1]
externals:
- spec: slurm@20-02-4-1
prefix: /usr
mpich:
buildable: false
externals:
- spec: mpich@3.1
modules:
- cray-mpich/7.7.10
autoconf:
version:
- '2.69'
automake:
version:
- 1.16.3
berkeley-db:
version:
- 18.1.40
binutils:
variants: +ld +gold +headers +libiberty ~nls +plugins
version:
- 2.33.1
boost:
version:
- 1.75.0
bzip2:
version:
- 1.0.8
c-blosc:
version:
- 1.21.0
cmake:
version:
- 3.20.2
curl:
version:
- 7.76.0
diffutils:
version:
- 3.7
elfutils:
version:
- 0.182
variants: +bzip2 ~nls +xz
expat:
version:
- 2.2.10
findutils:
version:
- 4.8.0
gdbm:
version:
- 1.18.1
gettext:
version:
- 0.21
git:
version:
- 2.31.0
glib:
version:
- 2.66.8
hdf5:
variants: +fortran +hl +shared
version:
- 1.10.7
help2man:
version:
- 1.47.16
hwloc:
version:
- 2.4.1
json-c:
version:
- 0.13.1
libbsd:
version:
- 0.10.0
libfabric:
version:
- 1.12.1
variants: fabrics=sockets,tcp,udp,rxm
libiconv:
version:
- 1.16
libsigsegv:
version:
- 2.12
libpciaccess:
version:
- 0.16
libtool:
version:
- 2.4.6
libunwind:
version:
- 1.5.0
variants: +pic +xz
libxml2:
version:
- 2.9.10
lz4:
version:
- 1.9.3
m4:
version:
- 1.4.18
mesa:
variants: ~llvm
mesa18:
variants: ~llvm
ncurses:
version:
- 6.2
variants: +termlib
numactl:
version:
- 2.0.14
openblas:
version:
- 0.3.10
variants: threads=openmp
perl:
version:
- 5.32.0
pkgconf:
version:
- 1.7.3
python:
version:
- 3.8.10
readline:
version:
- 8
sqlite:
version:
- 3.34.0
tar:
version:
- 1.32
texinfo:
version:
- 6.5
xz:
version:
- 5.2.5
variants: +pic
zlib:
version:
- 1.2.11
zstd:
version:
- 1.4.9
definitions:
- cuda_specs:
- amrex@21.05 +cuda cuda_arch=70
- axom@0.5.0 +cuda cuda_arch=70 ^umpire~shared
- caliper@2.5.0 +cuda cuda_arch=70
- chai@2.3.0 +cuda ~benchmarks ~tests cuda_arch=70 ^umpire~shared
- ginkgo@1.3.0 +cuda cuda_arch=70
- hpx@1.6.0 +cuda cuda_arch=70
- kokkos@3.4.00 +cuda +wrapper cuda_arch=70
- kokkos-kernels@3.2.00 +cuda cuda_arch=70 ^kokkos +cuda +wrapper cuda_arch=70
- magma@2.5.4 cuda_arch=70
- raja@0.13.0 +cuda cuda_arch=70
- slate@2021.05.02 +cuda cuda_arch=70
- strumpack@5.1.1 +cuda ~slate cuda_arch=70
- sundials@5.7.0 +cuda cuda_arch=70
- superlu-dist@6.4.0 +cuda cuda_arch=70
- tasmanian@7.5 +cuda cuda_arch=70
- umpire@4.1.2 +cuda ~shared cuda_arch=70
- zfp +cuda cuda_arch=70
#- ascent@0.7.1 +cuda ~shared cuda_arch=70
#- hypre@2.20.0 +cuda cuda_arch=70
#- mfem@4.2.0 +cuda cuda_arch=70
- default_specs:
- adios2@2.7.1
- adios@1.13.1
- aml@0.1.0
- amrex@21.05
- arborx@1.0
- argobots@1.1
- ascent@0.7.1 ~fortran
- bolt@2.0
- cabana@0.3.0
- caliper@2.5.0
- chai@2.3.0 ~benchmarks ~tests
- conduit@0.7.2
- darshan-runtime@3.3.0
- darshan-util@3.3.0
- faodel@1.1906.1
- flecsi@1.4 +cinch
- flit@2.1.0
- gasnet@2021.3.0
- ginkgo@1.3.0
- globalarrays@5.8
- gmp@6.2.1
- gotcha@1.0.3
- hdf5@1.10.7
- hypre@2.20.0
- kokkos-kernels@3.2.00 +openmp
- kokkos@3.4.00 +openmp
- legion@21.03.0
- libnrm@0.1.0
- libquo@1.3.1
- libunwind@1.5.0
- loki@0.1.7
- mercury@2.0.1
- metall@0.13
- mfem@4.2.0
- mpark-variant@1.4.0
- ninja@1.10.2
- openpmd-api@0.13.4
- papi@6.0.0.1
- papyrus@1.0.1
- parallel-netcdf@1.12.2
- pdt@3.25.1
- petsc@3.15.0
- precice@2.2.1
- pumi@2.2.5
- py-libensemble@0.7.2
- py-petsc4py@3.15.0
- py-warpx@21.05 ^warpx dims=2
- py-warpx@21.05 ^warpx dims=3
- py-warpx@21.05 ^warpx dims=rz
- qthreads@1.16 scheduler=distrib
- raja@0.13.0
- scr@3.0rc1
- slepc@3.15.0
- stc@0.9.0
- strumpack@5.1.1 ~slate
- sundials@5.7.0
- superlu-dist@6.4.0
- superlu@5.2.1
- swig@4.0.2
- swig@4.0.2-fortran
- sz@2.1.11.1
- tasmanian@7.5
- tau@2.30.1
- turbine@1.3.0
- umap@2.1.0
- umpire@4.1.2
- upcxx@2021.3.0
- zfp@0.5.5
# Explicit conflicts with Cray -or- Intel compiler (prohibited via package.py)
#- dyninst@11.0.0
#- hpctoolkit@2021.03.01
#- plasma@20.9.20
#- qt@5.15.2
#- qwt@6.1.6
#- slate@2021.05.02 ~cuda
# Cannot build suite-sparse due to OOM killer
#- fortrilinos@2.0.0 ^trilinos +nox +superlu-dist +stratimikos
#- omega-h@9.32.5
#- trilinos@13.0.1
#- trilinos@13.0.1 +nox +superlu-dist
# Failed builds
#- archer@2.0.0 # llvm@8 fails
#- axom@0.5.0 # thirdparty/sol/sol.hpp(11408): rvalue ref cannot be bound to an lvalue
#- heffte@2.0.0 # test/test_units_nompi.cpp(499): error: more than one instance of constructor "heffte::box3d::box3d"
#- hpx@1.6.0 # include/boost/asio/impl/read.hpp(377): no instance of overloaded function "hpx::util::detail::bound
#- mpifileutils@0.11 ~xattr # libcap: _caps_output.gperf:96:53: unknown type name 'size_t', libcircle: configure: check if MPI setup correctly
#- nrm@0.1.0 # py-gevent: configure: compiler doesn't halt on function prototype mismatch
#- py-jupyterhub@1.0.0
#- rempi@1.1.0 # configure: couldn't find MPI
#- unifyfs@0.9.1 # configure: couldn't find MPI
specs:
- $default_specs
Cori E4S 21.02
Production Spack Environment
spack:
view: false
concretization: separately
config:
install_tree:
root: /global/common/software/spackecp/e4s-21.02/software
module_roots:
tcl: /global/common/software/spackecp/e4s-21.02/modules/
build_stage: $tempdir/user/spack-stage
modules:
enable:
- tcl
tcl:
blacklist_implicits: true
hash_length: 0
naming_scheme: '{name}/{version}-{compiler.name}-{compiler.version}'
all:
conflict:
- '{name}'
environment:
set:
'{name}_ROOT': '{prefix}'
darshan-runtime:
conflict:
- 'darshan'
darshan-util:
conflict:
- 'darshan'
projections:
all: '{name}/{version}-{compiler.name}-{compiler.version}'
mirrors:
cori-e4s-21.02: /global/common/software/spackecp/mirrors/cori-e4s-21.02
compilers:
- compiler:
spec: intel@19.1.2.254
paths:
cc: cc
cxx: CC
f77: ftn
fc: ftn
flags: {}
operating_system: cnl7
target: any
modules:
- PrgEnv-intel
- intel/19.1.2.254
environment: {unset: []}
extra_rpaths: []
- compiler:
spec: gcc@10.1.0
paths:
cc: cc
cxx: CC
f77: ftn
fc: ftn
operating_system: cnl7
modules:
- PrgEnv-gnu
- gcc/10.1.0
definitions:
- intel_compiler: ['%intel@19.1.2.254']
- gcc_compiler: ['%gcc@10.1.0']
- e4s_intel:
- adios2@2.7.1 +hdf5
- aml@0.1.0
- arborx@0.9-beta +openmp
- bolt@2.0
- caliper@2.5.0 +fortran
- faodel@1.1906.1
- flecsi@1.4 +cinch +caliper +graphviz +tutorial
- flit@2.1.0
- gasnet@2020.3.0 +udp
- ginkgo@1.3.0
- gotcha@1.0.3 +test
- hdf5@1.10.7
- hypre@2.20.0 +mixedint +superlu-dist +openmp
- libnrm@0.1.0
- libquo@1.3.1
- mercury@2.0.0 +udreg
- mfem@4.2.0 +examples +gnutls +gslib +lapack +libunwind +openmp +threadsafe +pumi +umpire
- ninja@1.10.2
- omega-h@9.32.5 ~trilinos
- openpmd-api@0.13.2
- papi@6.0.0.1 +example +static_tools +powercap +infiniband
- papyrus@1.0.1
- pdt@3.25.1 +pic
- precice@2.2.0 +python
- pumi@2.2.5 +fortran
- qthreads@1.16 ~hwloc
- raja@0.13.0 +tests
- slepc@3.14.2
- strumpack@5.1.1 +shared
- sundials@5.7.0 +examples-cxx +hypre +klu +lapack
- superlu@5.2.1
- superlu-dist@6.4.0 +openmp
- swig@4.0.2-fortran
- tasmanian@7.3 +blas +fortran +mpi +python +xsdkflags
- tau@2.30.1 +mpi ~pdt
- turbine@1.2.3 +hdf5 +python
- umap@2.1.0 +tests
- umpire@4.1.2 +fortran +numa +openmp
- upcxx@2020.10.0
- zfp@0.5.5 +aligned +c +fortran +openmp +profile
- e4s_gcc:
- darshan-runtime@3.2.1 +slurm
- darshan-util@3.2.1 +bzip2
- dyninst@10.2.1
- legion@20.03.0
- plasma@20.9.20
- slate@2020.10.00 ~cuda
# skipping package
# - adios@1.13.1 +bzip2 +fortran +hdf5 +netcdf
# - kokkos-kernels@3.2.00 +mkl +openmp
# - kokkos@3.2.00 +compiler_warnings +deprecated_code +examples +hwloc +memkind +numactl +openmp +pic +tests
# - openmpi@4.0.5 +cxx +thread_multiple schedulers=slurm
# - parallel-netcdf@1.12.1 +burstbuffer
# - petsc@3.14.4 +X +fftw +jpeg +libpng +libyaml +memkind
# - py-jupyterhub@1.0.0
# - py-libensemble@0.7.1 +mpi +nlopt +petsc4py +scipy
# - py-petsc4py@3.14.1
# - trilinos@13.0.1
# _______________________________ ISSUES TO SOLVE _______________________________
# issue installing vtkh using intel compiler
# using gcc compiler ascent has dependency for conduit@develop which not pinned to version. There was a build error related to missing HDF5 library
# - ascent@0.6.0
# Issue detecting fortran compiler https://cdash.spack.io/viewConfigure.php?buildid=105216. Also issue installing conduit since its tied to 'develop'. Tried using conduit@0.7.1 and still failed see https://cdash.spack.io/viewBuildError.php?buildid=105206. Talk to @cyrush at spack slack.
# - axom@0.4.0 +mfem +python
# /usr/lib64/gcc/x86_64-suse-linux/7/../../../../x86_64-suse-linux/bin/ld: /usr/lib/libm.so: error adding symbols: file in wrong format see https://cdash.spack.io/buildSummary.php?buildid=104952
# - adios2@2.7.1 +hdf5 +dataman +dataspaces
# skipping this version for now 5.7 is the latest version 5.8 doesn't exist in spack repo
# - globalarrays@5.8 +blas +lapack +scalapack
# Warning: Linking the shared library libhpcrun.la against the static library see https://cdash.spack.io/viewBuildError.php?buildid=104938
# - hpctoolkit@2020.08.03 %gcc
# error: identifier "HPX_SMT_PAUSE" is undefined see https://cdash.spack.io/viewBuildError.php?buildid=105747
# - hpx@1.6.0 +async_mpi +examples
# Error building legion with intel compiler https://cdash.spack.io/viewBuildError.php?buildid=105190
# - legion@20.03.0
# skip magma because it's a GPU package
#- magma@2.5.4 cuda_arch=70 ^cuda@10.2.89
# error with intel compiler: building dtcmp
# error with gnu compiler: Could NOT find LibCircle (missing: LibCircle_LIBRARIES
# - mpifileutils@0.10.1
# /global/cfs/cdirs/m3503/spack-NSewtxLx/spack-stage/siddiq90/spack-stage-phist-1.9.3-rznbmfuo2mt2erku4rit4peyqxu7iji4/spack-src/fortran_bindings/test/kernels.F90(63): catastrophic error: **Internal compiler error: internal abort** Please report this error along with the circumstances in which it occurred in a Software Problem Report. Note: File and line given may not be explicit cause of this error. see https://cdash.spack.io/buildSummary.php?buildid=104915
# - phist@1.9.3
# configure: error: Failed to find C MPI Wrapper. see https://cdash.spack.io/buildSummary.php?buildid=104940
# - rempi@1.1.0
# "%intel@19:" conflicts with "slate" [Does not currently build with icpc >= 2019]
# - slate@2020.10.00 ^cuda@10.2.89
# build Error with Intel: error building dtcmp: configure: error: C compiler cannot create executables
# build Error with GCC: make[2]: *** No rule to make target '/global/cfs/cdirs/m3503/spack-qhLmtUlQ/spack_path_placeholder/spack_path_placeholder/spack_path_placeholder/spack_path_placehol/cray-cnl7-haswell/gcc-10.1.0/libyogrt-1.24-6wngjuplxnjjsivzvilwjsp4gwu4ziuj/lib/libyogrt.a', needed by 'examples/test_ckpt_F'. Stop. see https://cdash.spack.io/buildSummary.php?buildid=104889
# - scr@2.0.0
# error installing ant Error: JAVA_HOME is not defined correctly. https://software.nersc.gov/NERSC/e4s-2102/-/jobs/87103
# - stc@0.8.3
# Issue building sundials with raja support see https://cdash.spack.io/viewBuildError.php?buildid=105455. Not sure if raja support with sundials is neccessary.
# sundials@5.7.0 +examples-cxx +examples-f2003 +f2003 +hypre +klu +lapack +openmp +raja +superlu-dist
# https://cdash.spack.io/viewBuildError.php?buildid=105513
# - sz@2.1.11.1 +fortran +python +time_compression +random_access +pastri
# issue building tau with intel see https://cdash.spack.io/viewBuildError.php?buildid=105235 one of the error points to missing `-lpdb` library. Tau has `+pdt` enabled by default
# - tau@2.30.1 +adios2 +gasnet +likwid +ompt +openmp +mpi +python +scorep +shmem +sqlite
# Could NOT find AXL (missing: AXL_LIBRARIES AXL_INCLUDE_DIRS) see https://cdash.spack.io/buildSummary.php?buildid=105476
# - veloc@1.4
# issue configure: error: "Couldn't find MPI" see https://cdash.spack.io/viewConfigure.php?buildid=105491
# issue installing mercury (dependency) for unifyfs with gcc https://cdash.spack.io/buildSummary.php?buildid=105497
#- unifyfs@0.9.1
specs:
- matrix:
- [$e4s_intel]
- [$intel_compiler]
- matrix:
- [$e4s_gcc]
- [$gcc_compiler]
packages:
all:
compiler: [intel@19.1.2.254, gcc@10.1.0]
target: [haswell]
providers:
mpi: [mpich]
mkl: [cray-libsci, intel-mkl]
blas: [cray-libsci, intel-mkl]
scalapack: [cray-libsci, intel-mkl]
pkgconfig: [pkg-config]
cray-libsci:
buildable: false
externals:
- spec: cray-libsci@19.06.1%intel
modules:
- cray-libsci/19.06.1
fftw:
buildable: false
externals:
- spec: fftw@3.3.8.4%intel
modules:
- cray-fftw/3.3.8.4
hdf5:
variants: +cxx +debug +fortran +szip +threadsafe +hl
hwloc:
buildable: false
externals:
- spec: hwloc
prefix: /usr
intel-mkl:
buildable: false
externals:
- spec: intel-mkl@19.1.2.254
modules:
- intel/19.1.2.254
mpich:
buildable: false
externals:
- spec: mpich@3.1
modules:
- cray-mpich/7.7.10
netcdf-c:
buildable: false
externals:
- spec: netcdf-c@4.7.4
modules:
- cray-netcdf/4.7.4.0
openssl:
buildable: false
externals:
- spec: openssl@1.1.1g
prefix: /usr
# issue installing version 5.32.1 and confirmed 5.32.0 works
perl:
version: [5.32.0]
# disable slate since we can't build with icc >= 19
strumpack:
variants: ~slate
Cori E4S 20.10
Production Spack Environment
spack:
concretization: separately
view: false
config:
install_tree: /global/common/software/spackecp/e4s-20.10/software
build_stage: $tempdir/$user/spack-stage
module_roots:
tcl: /global/common/software/spackecp/e4s-20.10/modules/
mirrors::
e4s-2020-10: /global/common/software/spackecp/mirrors/e4s-2020-10
modules:
enable:
- tcl
tcl:
hash_length: 8
projections:
all: '{name}/{version}-{compiler.name}-{compiler.version}'
all:
conflict:
- '{name}'
filter:
environment_blacklist: []
load: []
environment:
unset: []
verbose: false
whitelist: []
blacklist: []
blacklist_implicits: false
definitions:
- e4s:
- adios2@2.6.0
- aml@0.1.0
- arborx@0.9-beta +openmp
- bolt@1.0
- caliper@2.4.0
- darshan-runtime@3.2.1 +slurm
- darshan-util@3.2.1 +bzip2
- flit@2.1.0
- gasnet@2020.3.0 +udp
- ginkgo@1.2.0
- globalarrays@5.7 +blas +lapack +scalapack
- gotcha@1.0.3 +test
- hdf5@1.10.6 +cxx +debug +fortran +szip +threadsafe +hl
- hypre@2.20.0 +mixedint
- kokkos-kernels@3.2.00 +mkl +openmp
- kokkos@3.2.00 +debug +debug_dualview_modify_check +compiler_warnings +examples +hwloc +memkind +numactl +openmp +pic +tests
- libnrm@0.1.0
- libquo@1.3.1
- mercury@1.0.1 +udreg
- mfem@4.1.0 +examples +gnutls +gslib +lapack +libunwind +openmp +threadsafe +pumi +umpire
- ninja@1.10.1
- openpmd-api@0.12.0
- papi@6.0.0.1 +example +static_tools
- parallel-netcdf@1.12.1
- pdt@3.25.1 +pic
- petsc@3.14.0
- pumi@2.2.2 +fortran
- py-libensemble@0.7.0 +mpi +nlopt +scipy
- py-petsc4py@3.13.0
- qthreads@1.14 ~hwloc
- raja@0.12.1
- slepc@3.14.0
- stc@0.8.3
- sundials@5.4.0 +examples-cxx +examples-f2003 ~examples-f77 +f2003 +klu +openmp +hypre +lapack
- superlu@5.2.1
- superlu-dist@6.3.1
- swig@4.0.2
- sz@2.1.10 +fortran +hdf5 +python +time_compression +random_access +netcdf +pastri
- tasmanian@7.3 +blas +fortran +mpi +python +xsdkflags
- turbine@1.2.3 +hdf5 +python
- umap@2.1.0 +tests
- umpire@4.0.1 +fortran +numa +openmp
- upcxx@2020.3.0
- veloc@1.4
- zfp@0.5.5
# - adios@1.13.1 +netcdf +szip +fortran +bzip2 module already installed
# - ascent@develop skipping package because its on develop
# - axom@0.3.3 skip build for now, spack ci rebuild issue
# issue with concretization of dyninst: "%intel" conflicts with "dyninst" see https://software.nersc.gov/ecp/nersc-e4s/-/jobs/46526
# - dyninst@10.2.1 +static
# - faodel@1.1906.1 network=libfabric issue with build https://software.nersc.gov/NERSC/nersc-e4s/-/jobs/60284
# flesci concretization issue: https://github.com/spack/spack/issues/19292
# - flecsi@1 +cinch +coverage +doc +doxygen +graphviz +hdf5 +tutorial
# - hpctoolkit@2020.08.03 +all-static +cray +mpi # depends on dyninst
# - hpx@1.5.1 issue with installing boost
# - legion@20.03.0 failed to build se https://software.nersc.gov/ecp/e4s/facilitypipelines/nersc-e4s/-/jobs/59787
# - magma@2.5.3 this package requires GPU, this is not applicable for Cori
# - mpifileutils@0.10.1 +gpfs +lustre # fails on libcircle requires MPICC wrapper see https://software.nersc.gov/ecp/e4s/facilitypipelines/nersc-e4s/-/jobs/57907 we could install OpenMPI and build this with the wrapper.
#- openmpi # skip openmpi
# - omega-h@9.29.0
# - papi@6.0.0.1 +example +static_tools module already installed
# - phist@1.9.1 see https://software.nersc.gov/NERSC/nersc-e4s/-/jobs/63409
# concretization issue "%intel" conflicts with "plasma" see https://software.nersc.gov/ecp/nersc-e4s/-/jobs/46533
# - plasma@20.9.20
# - precice@2.1.0 issue finding PETSC
# - py-jupyterhub@1.0.0
# error building rempi https://software.nersc.gov/ecp/nersc-e4s/-/jobs/32884
# - rempi@1.1.0
# - scr@2.0.0 # async_api=CRAY_DW issue finding DATAWARP libraries -- Could NOT find DATAWARP (missing: DATAWARP_LIBRARIES DATAWARP_INCLUDE_DIRS) see https://software.nersc.gov/ecp/e4s/facilitypipelines/nersc-e4s/-/jobs/59782
# - slate@develop package tied to develop skipping this build
# - strumpack@4.0.0 +shared ~butterflypack ~cuda +count_flops +build_dev_tests +build_tests
# error building otf2, we can disable tau with otf2 support
# - tau@2.29 +craycnl +openmp +mpi
# - trilinos@13.0.0 +debug +float +openmp +pnetcdf +zlib
# - unifyfs@0.9.0 +hdf5. can't find MPICC see https://software.nersc.gov/NERSC/nersc-e4s/-/jobs/63408
- arch:
- '%intel@19.1.2.254 arch=cray-cnl7-haswell'
specs:
- matrix:
- - $e4s
- - $arch
compilers:
- compiler:
spec: intel@19.1.2.254
paths:
cc: cc
cxx: CC
f77: ftn
fc: ftn
flags: {}
operating_system: cnl7
target: any
modules:
- PrgEnv-intel
- intel/19.1.2.254
environment: {unset: []}
extra_rpaths: []
packages:
all:
compiler: [intel@19.1.2.254]
target: [haswell]
providers:
mpi: [mpich]
mkl: [intel-mkl, cray-libsci]
blas: [intel-mkl, cray-libsci]
scalapack: [intel-mkl, cray-libsci]
pkgconfig: [pkg-config]
berkeley-db:
version: [18.1.4]
boost:
version: [1.74.0]
bzip2:
buildable: false
externals:
- spec: bzip2
prefix: /usr
cmake:
version: [3.16.5] # issue with cmake 3.17.3 using 3.16.5 for now see https://github.com/spack/spack/issues/17605
cray-libsci:
buildable: false
externals:
- spec: cray-libsci@19.06.1%intel
modules:
- cray-libsci/19.06.1
diffutils:
version: [3.7]
elfutils:
version: [0.180]
expat:
version: [2.2.9]
fftw:
buildable: false
externals:
- spec: fftw@3.3.8.4%intel
modules:
- cray-fftw/3.3.8.4
gdbm:
version: [1.18.1]
gettext:
buildable: false
externals:
- spec: gettext
prefix: /usr
help2man:
version: [1.47.11]
hwloc:
buildable: false
externals:
- spec: hwloc
prefix: /usr
hypre:
version: [2.20.0]
intel-mkl:
buildable: false
externals:
- spec: intel-mkl@19.1.2.254
modules:
- intel/19.1.2.254
libbsd:
version: [0.10.0]
libiconv:
version: [1.16]
libsigsegv:
version: [2.12]
libxml2:
version: [2.9.10]
lz4:
buildable: false
externals:
- spec: lz4
prefix: /usr
m4:
buildable: false
externals:
- spec: m4
prefix: /usr
mpi:
buildable: false
mpich:
buildable: false
externals:
- spec: mpich@3.1
modules:
- cray-mpich/7.7.10
openssl:
buildable: false
externals:
- spec: openssl@1.1.1g
prefix: /usr
ncurses:
version: [6.2]
netcdf:
buildable: false
externals:
- spec: netcdf@4.6.3.2%intel
modules:
- cray-netcdf/4.6.3.2
perl:
buildable: false
externals:
- spec: perl
prefix: /usr
petsc:
version: [3.14.0]
pdsh: # required for scr
buildable: false
externals:
- spec: pdsh
prefix: /usr
pkgconf:
version: [1.7.3]
sqlite:
version: [3.31.1]
tar:
buildable: false
externals:
- spec: tar
prefix: /usr
zlib:
version: [1.2.11]