Merge branch 'develop' into add_dense_hamiltonian

CMake/CheckSincos.cmake

@@ -0,0 +1,29 @@
+INCLUDE(${CMAKE_ROOT}/Modules/CheckCXXSourceCompiles.cmake)
+
+SET(SINCOS_INCLUDE cmath)
+
+SET(ENABLE_MASS FALSE CACHE BOOL "ENABLE MASS math libraries for Power architecture")
+# This needs to go before HAVE_SINCOS
+IF(ENABLE_MASS)
+  INCLUDE(CMake/FindIBMMASS.cmake)
+  IF(HAVE_MASS)
+    SET(CMAKE_REQUIRED_INCLUDES ${MASS_INCLUDE_DIRECTORIES} ${CMAKE_REQUIRED_INCLUDES})
+    SET(CMAKE_REQUIRED_LINK_OPTIONS ${MASS_LINKER_FLAGS})
+    SET(CMAKE_REQUIRED_LIBRARIES ${MASS_LIBRARIES})
+  ENDIF(HAVE_MASS)
+ENDIF(ENABLE_MASS)
+
+MESSAGE("CMAKE_REQUIRED_INCLUDES: ${CMAKE_REQUIRED_INCLUDES}")
+MESSAGE("CMAKE_REQUIRED_LINK_OPTIONS: ${CMAKE_REQUIRED_LINK_OPTIONS}")
+MESSAGE("CMAKE_REQUIRED_LIBRARIES: ${CMAKE_REQUIRED_LIBRARIES}")
+
+SET( SINCOS_TEST_SRC
+  "#include \"${SINCOS_INCLUDE}\"
+int main(void) {
+double input = 1.1;
+double outsin, outcos;;
+sincos(input,&outsin, &outcos);
+}
+")
+
+CHECK_CXX_SOURCE_COMPILES("${SINCOS_TEST_SRC}" HAVE_SINCOS)

CMake/ClangCompilers.cmake

@@ -25,13 +25,6 @@ ADD_DEFINITIONS( -Drestrict=__restrict__ )
 SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -fomit-frame-pointer -fstrict-aliasing")
 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fomit-frame-pointer -fstrict-aliasing -D__forceinline=inline")
 
-# Suppress compile warnings
-SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -Wno-deprecated -Wno-unused-value")
-SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated -Wno-unused-value")
-IF ( CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "3.8.0" )
-  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-undefined-var-template")
-ENDIF()
-
 # Set extra optimization specific flags
 SET( CMAKE_C_FLAGS_RELEASE     "${CMAKE_C_FLAGS_RELEASE} -ffast-math" )
 SET( CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -ffast-math" )

CMake/FindIBMMASS.cmake

@@ -0,0 +1,74 @@
+# Simple file to find IBM MASS (if available)
+INCLUDE( CheckCXXSourceCompiles )
+
+MESSAGE(STATUS "Looking for IBM MASS libraries")
+# Finding and setting the MASS_INCLUDE_DIRECTORIES
+set(SUFFIXES include)
+find_path(MASS_INCLUDE_DIRECTORIES name "mass.h" HINTS ${MASS_ROOT}
+  PATH_SUFFIXES ${SUFFIXES})
+if (NOT MASS_INCLUDE_DIRECTORIES)
+  message(FATAL_ERROR "MASS_INCLUDE_DIRECTORIES not set. \"mass.h\" not found in MASS_ROOT/(${SUFFIXES})")
+endif (NOT MASS_INCLUDE_DIRECTORIES)
+message("MASS_INCLUDE_DIRECTORIES: ${MASS_INCLUDE_DIRECTORIES}")
+
+  # Finding and setting the MASS_LINK_DIRECTORIES
+  # the directory organization varies with platform and targets
+  # these suffixes are not exhaustive
+set(MASS_FIND_LIB "libmass${CMAKE_STATIC_LIBRARY_SUFFIX}")
+set(SUFFIXES lib lib64)
+find_path(MASS_LINK_DIRECTORIES name "${MASS_FIND_LIB}" HINTS ${MASS_ROOT}
+  PATH_SUFFIXES ${SUFFIXES})
+if (NOT MASS_LINK_DIRECTORIES)
+  message(FATAL_ERROR "MASS_LINK_DIRECTORIES not set. ${MASS_FIND_LIB} "
+    "not found in MASS_ROOT/(${SUFFIXES})")
+endif (NOT MASS_LINK_DIRECTORIES)
+message("MASS_LINK_DIRECTORIES: ${MASS_LINK_DIRECTORIES}")
+
+set(MASS_LINKER_FLAGS -L${MASS_LINK_DIRECTORIES} -Wl,-rpath,${MASS_LINK_DIRECTORIES})
+set(MASS_LIBRARIES "-lmass -lmassv")
+
+FILE( WRITE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mass.cxx"
+  "#include <cmath>
+#include <mass.h>
+#include <massv.h>
+#include <iostream>
+
+int main(void) {
+double input = 1.1;
+double outsin, outcos;;
+sincos(input,&outsin, &outcos);
+int in_size = 10;
+double inputv[in_size];
+double resultv[in_size];
+for( int i = 0; i < in_size; ++i)
+  inputv[i] = i;
+vlog10(resultv, inputv, &in_size);
+}
+")
+try_compile(HAVE_MASS ${CMAKE_BINARY_DIR}
+  ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mass.cxx
+  CMAKE_FLAGS
+  "-DINCLUDE_DIRECTORIES=${MASS_INCLUDE_DIRECTORIES} "
+  "-DLINK_DIRECTORIES=${MASS_LINK_DIRECTORIES}"
+  LINK_LIBRARIES "${MASS_LIBRARIES}"
+  COMPILE_DEFINITIONS "${MASS_COMPILE_DEFINITIONS}"
+  OUTPUT_VARIABLE MASS_OUT)
+if ( NOT HAVE_MASS )
+  MESSAGE( "${MASS_OUT}" )
+endif ( NOT HAVE_MASS )
+
+IF ( HAVE_MASS )
+  SET( MASS_FOUND 1 )
+  SET( MASS_FLAGS ${MASS_COMPILE_DEFINITIONS} )
+  include_directories( ${MASS_INCLUDE_DIRECTORIES} )
+  set( HAVE_VECTOR_MATH 1 )
+  set( HAVE_MASSV 1 )
+  set( SINCOS_INCLUDE mass.h )
+  MESSAGE(STATUS "MASS found: HAVE_MASS=${HAVE_MASS}, HAVE_MASSV=${HAVE_MASSV}")
+ELSE( HAVE_MASS )
+  SET( MASS_FOUND 0 )
+  SET( MASS_FLAGS )
+  SET( MASS_LIBRARIES )
+  SET( MASS_LINKER_FLAGS )
+  MESSAGE("MASS not found")
+ENDIF( HAVE_MASS )

CMake/FindVectorMath.cmake

@@ -4,6 +4,7 @@
 # Initial version only for MKL VML. Works for gcc+MKL case. libm and massv detection required.
 #
 
+IF(NOT HAVE_MASS)
 SET( HAVE_VECTOR_MATH 0 )
 
 IF ( HAVE_MKL_VML )
@@ -33,7 +34,7 @@ ELSE()
     ENDIF()
   ENDIF()
 ENDIF()
-
+ENDIF()
 IF ( NOT HAVE_VECTOR_MATH )
   MESSAGE(STATUS "No usable vector math library detected.")
 ENDIF()

CMake/ctest_script.cmake

@@ -177,6 +177,14 @@ IF ( DEFINED QMC_MPI )
   SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DQMC_MPI=${QMC_MPI}" )
 ENDIF()
 
+IF ( DEFINED MPIEXEC )
+  SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPIEXEC=${MPIEXEC}" )
+ENDIF()
+
+IF ( DEFINED MPIEXEC_PREFLAGS )
+  SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS}" )
+ENDIF()
+
 IF ( DEFINED QMC_MIXED_PRECISION )
   SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DQMC_MIXED_PRECISION=${QMC_MIXED_PRECISION}" )
 ENDIF()
@@ -193,6 +201,10 @@ IF ( DEFINED BUILD_AFQMC )
   SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DBUILD_AFQMC=${BUILD_AFQMC}" )
 ENDIF()
 
+IF ( DEFINED ENABLE_MKL )
+  SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DENABLE_MKL=${ENABLE_MKL}" )
+ENDIF()
+
 IF ( DEFINED BLA_VENDOR )
   SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DBLA_VENDOR='${BLA_VENDOR}'" )
 ENDIF()

CMake/macros.cmake

@@ -104,7 +104,7 @@ FUNCTION( RUN_QMC_APP_NO_COPY TESTNAME WORKDIR PROCS THREADS TEST_ADDED TEST_LAB
         IF ( ${TOT_PROCS} GREATER ${TEST_MAX_PROCS} )
             MESSAGE_VERBOSE("Disabling test ${TESTNAME} (exceeds maximum number of processors ${TEST_MAX_PROCS})")
         ELSEIF ( USE_MPI )
-            ADD_TEST( ${TESTNAME} ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} ${QMC_APP} ${ARGN} )
+            ADD_TEST( ${TESTNAME} ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} ${MPIEXEC_PREFLAGS} ${QMC_APP} ${ARGN} )
             SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}"
                 PROCESSORS ${TOT_PROCS} PROCESSOR_AFFINITY TRUE WORKING_DIRECTORY ${WORKDIR}
                 ENVIRONMENT OMP_NUM_THREADS=${THREADS} )

CMake/run_qe.cmake

@@ -4,7 +4,7 @@ INCLUDE("${qmcpack_SOURCE_DIR}/CMake/test_labels.cmake")
 # Runs unit tests
 FUNCTION( ADD_QE_TEST TESTNAME PROCS TEST_BINARY NPOOL WORKDIR TEST_INPUT)
     IF ( USE_MPI )
-        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} ${TEST_BINARY} -npool ${NPOOL} -inp ${TEST_INPUT} )
+        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} ${MPIEXEC_PREFLAGS} ${TEST_BINARY} -npool ${NPOOL} -inp ${TEST_INPUT} )
     ELSE()
         ADD_TEST( NAME ${TESTNAME} COMMAND ${TEST_BINARY} -npool 1 ${TEST_INPUT} )
     ENDIF()

CMake/unit_test.cmake

@@ -5,7 +5,7 @@ INCLUDE("${PROJECT_SOURCE_DIR}/CMake/test_labels.cmake")
 FUNCTION( ADD_UNIT_TEST TESTNAME TEST_BINARY )
     MESSAGE_VERBOSE("Adding test ${TESTNAME}")
     IF ( USE_MPI )
-        ADD_TEST(NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 1 ${TEST_BINARY} ${ARGN})
+        ADD_TEST(NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 1 ${MPIEXEC_PREFLAGS} ${TEST_BINARY} ${ARGN})
         #SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES ENVIRONMENT OMP_NUM_THREADS=1 )
     ELSE()
         ADD_TEST(NAME ${TESTNAME} COMMAND ${TEST_BINARY} ${ARGN})

CMakeLists.txt

@@ -13,11 +13,14 @@ endif(NOT $ENV{CRAYPE_VERSION} MATCHES ".")
 IF(CMAKE_GENERATOR MATCHES "Ninja")
   cmake_policy(SET CMP0058 NEW)
 ENDIF(CMAKE_GENERATOR MATCHES "Ninja")
-# CMake find_package will use <PackageName>_ROOT CMake variable
+# CMP0074: CMake find_package will use <PackageName>_ROOT CMake variable
 # and environment variable in search path.
 # various Find<Package>.cmake modules may not follow this policy
+# CMP0075: check_include_file_cxx and other follow CMAKE_REQUIRED_LIBRARIES
+# when doing link check
 IF(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.12.0")
   cmake_policy(SET CMP0074 NEW)
+  cmake_policy(SET CMP0075 NEW)
 ENDIF(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.12.0")
 
 ######################################################################
@@ -30,6 +33,14 @@ SET(QMCPACK_VERSION_MINOR 8)
 SET(QMCPACK_VERSION_PATCH 9)
 SET(QMCPACK_VERSION "${QMCPACK_VERSION_MAJOR}.${QMCPACK_VERSION_MINOR}.${QMCPACK_VERSION_PATCH}")
 
+######################################################
+# Directory where customize cmake files reside
+######################################################
+SET (PROJECT_CMAKE ${qmcpack_SOURCE_DIR}/CMake)
+
+SET(QMCPACK_UNIT_TEST_DIR ${qmcpack_BINARY_DIR}/tests/bin)
+
+
 IF (COMPILING_ON_BGQ)
   # YL: Put this after PROJECT(qmcpack) because COMPILING_ON_BGQ is defined
   # in BGQ toolchain file which is loaded by PROJECT().
@@ -195,11 +206,12 @@ SET(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH})
 # add macros to compile einspline if necessary
 ######################################################################
 INCLUDE(${CMAKE_ROOT}/Modules/CheckFunctionExists.cmake)
+INCLUDE(${CMAKE_ROOT}/Modules/CheckCXXSymbolExists.cmake)
 INCLUDE(${CMAKE_ROOT}/Modules/CheckIncludeFile.cmake)
 INCLUDE(${CMAKE_ROOT}/Modules/CheckTypeSize.cmake)
-INCLUDE(${CMAKE_ROOT}/Modules/CheckCXXSourceCompiles.cmake)
 
-CHECK_FUNCTION_EXISTS(sincos HAVE_SINCOS)
+INCLUDE(${PROJECT_CMAKE}/CheckSincos.cmake)
+
 CHECK_FUNCTION_EXISTS(posix_memalign HAVE_POSIX_MEMALIGN)
 
 CHECK_INCLUDE_FILE(dlfcn.h HAVE_DLFCN_H)
@@ -260,13 +272,6 @@ SET(HAVE_LIBBLITZ 0)
 SET(HAVE_LIBGSL 0)
 SET(MAC_VECLIB 0)
 
-######################################################
-# Directory where customize cmake files reside
-######################################################
-SET (PROJECT_CMAKE ${qmcpack_SOURCE_DIR}/CMake)
-
-SET(QMCPACK_UNIT_TEST_DIR ${qmcpack_BINARY_DIR}/tests/bin)
-
 ######################################################
 # Git information (see src/CMakeLists.txt for the
 # command to get repository information)
@@ -486,12 +491,8 @@ ELSE(CMAKE_TOOLCHAIN_FILE)
   ELSE()
     # Search for Blas/Lapack
     find_package(LAPACK)
-    # When cmake finds and sets up the MKL libraries with a non-Intel compiler, set the flag
-    # This should be tidied up when FindMKL.cmake supports non-Intel compilers.
     IF("${LAPACK_LIBRARIES}" MATCHES "mkl")
-      MESSAGE(STATUS "MKL found via LAPACK/BLAS")
-      SET( MKL_FOUND 1 )
-      SET( HAVE_MKL 1 )
+      MESSAGE(FATAL_ERROR "MKL found via LAPACK/BLAS. Please use -DENABLE_MKL=1 when invoking cmake for full Intel MKL utilization. -DMKL_ROOT may also be needed for the include files to be found.")
     ENDIF()
   ENDIF()
   IF(LAPACK_FOUND)
@@ -532,6 +533,10 @@ IF ( HAVE_MPI )
     SET( MPIEXEC_NUMPROC_FLAG "-np" )
   ENDIF()
 
+  IF ( NOT DEFINED MPIEXEC_PREFLAGS )
+    SET( MPIEXEC_PREFLAGS "" )
+  ENDIF()
+
   IF ( NOT TEST_MAX_PROCS )
     SET( TEST_MAX_PROCS 100 )
   ENDIF()
@@ -551,6 +556,14 @@ INCLUDE(${CMAKE_ROOT}/Modules/FindThreads.cmake)
 
 SET(QMC_UTIL_LIBS ${LAPACK_LIBRARY} ${BLAS_LIBRARY})
 
+# This needs a better place to live. as does all the IF( HAVE_MKL ) stuff
+IF(HAVE_MASS)
+  SET(CMAKE_CXX_FLAGS ${MASS_FLAGS} ${CMAKE_CXX_FLAGS})
+  SET(CMAKE_C_FLAGS ${MASS_FLAGS} ${CMAKE_C_FLAGS})
+  SET(MASS_LIBRARY "${MASS_LINKER_FLAGS} ${MASS_LIBRARIES}")
+  SET(QMC_UTIL_LIBS ${MASS_LIBRARY} ${QMC_UTIL_LIBS})
+ENDIF(HAVE_MASS)
+
 SET( FFTW_FOUND 0 )
 IF ( HAVE_MKL )
   IF ( HAVE_MKL_FFTW3 )

config/build_olcf_summit.sh

@@ -1,11 +1,17 @@
 #!/bin/bash
+
+BUILD_MODULES=config/load_olcf_summit_modules.sh
+
 module purge
 echo "Purging current module set"
+echo "Sourcing file: $BUILD_MODULES to build QMCPACK"
+
+. $BUILD_MODULES
 
-. config/load_summit_modules.sh
+echo "Either source $BUILD_MODULES or load these same modules to run QMCPACK"
 
-declare -A builds=( ["cpu"]=" " \
-                    ["complex_cpu"]="-DQMC_COMPLEX=1 " \
+declare -A builds=( ["cpu"]="-DENABLE_MASS=1 -DMASS_ROOT=/sw/summit/xl/16.1.1-5/xlmass/9.1.1" \
+                    ["complex_cpu"]="-DQMC_COMPLEX=1 -DENABLE_MASS=1 -DMASS_ROOT=/sw/summit/xl/16.1.1-5/xlmass/9.1.1" \
                     ["legacy_gpu"]="-DQMC_CUDA=1 -DCUDA_ARCH=sm_70 " \
 		    ["complex_legacy_gpu"]="-DQMC_CUDA=1 -DQMC_COMPLEX=1 -DCUDA_ARCH=sm_70 " )
 

config/load_summit_modules.sh → config/load_olcf_summit_modules.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 echo "Loading QMCPACK dependency modules for summit"
-module load gcc
+module load gcc/7.4.0
 module load spectrum-mpi
 module load essl
 module load netlib-lapack

manual/installation.tex

@@ -323,6 +323,7 @@ \subsection{Configuration options}
 MPIEXEC                Specify the mpi wrapper, e.g. srun, aprun, mpirun, etc.
 MPIEXEC_NUMPROC_FLAG   Specify the number of mpi processes flag,
                        e.g. "-n", "-np", etc.
+MPIEXEC_PREFLAGS     Flags to pass to MPIEXEC directly before the executable to run.   
 \end{shade}
 
 \item LLVM/Clang Developer Options\\

manual/running.tex

@@ -112,9 +112,23 @@ \subsection{Performance considerations}
 \label{sec:cpu:performance}
 As walkers are the basic units of workload in QMC algorithms, they are loosely coupled and distributed across all the threads. For this reason, the best strategy to run QMCPACK efficiently is to feed enough walkers to the available threads.
 
-In a VMC calculation, the code automatically raises the actual number of walkers per MPI rank to the number of available threads if the user-specified number of walkers is smaller, see ``walkers/mpi=XXX'' in the VMC output.  In a DMC calculation, the target number of walkers should be chosen to be slightly smaller than a multiple of the total number of available threads across all the MPI ranks belongs to this calculation. Since the number of walkers varies from generation to generation, its dynamical value should be slightly smaller or equal to that multiple most of the time.
-
-To achieve better performance, a mixed-precision version (experimental) has been introduced to the CPU code. The mixed-precision CPU code is more aggressive than the GPU version in using single precision (SP) operations. Current implementation uses SP on most calculations, except for matrix inversions and reductions where double precision is required to retain high accuracy. All the constant spline data in wavefunction, pseudopotentials, and Coulomb potentials are initialized in double precision and later stored in single precision. The mixed-precision code is as accurate as the double-precision code up to a certain system size. Cross checking and verification of accuracy is always required but is particularly important above approximately 1,500 electrons. The mixed precision code is currently tested on solids with real and complex wavefunctions with VMC, VMC using drift, and DMC runs with wavefunction including single Slater determinant and one- and two-body Jastrow factors. Wavefunction optimization is currently not supported.
+In a VMC calculation, the code automatically raises the actual number of walkers per MPI rank to the number of available threads
+if the user-specified number of walkers is smaller, see ``walkers/mpi=XXX'' in the VMC output.
+
+In DMC, for typical small to mid-sized calculations choose the total number of walkers to be a significant multiple of the total number of
+threads (MPI tasks * threads per task). This will ensure a good load balance. e.g. For a calculation on a few nodes with a total
+512 threads, using 5120 walkers may keep the load imbalance around 10\%. For the very largest calculations, the target number of
+walkers should be chosen to be slightly smaller than a multiple of the total number of available threads across all the MPI ranks.
+This will reduce occurrences worse-case load imbalance e.g. where one thread has two walkers while all the others have one.
+
+To achieve better performance, a mixed-precision version (experimental) has been developed in the CPU code. The mixed-precision
+CPU code uses a mixed of single precision (SP) and double precision (DP) operations, while the default code use DP exclusively.
+This mixed precision version is more aggressive than the GPU CUDA version in using single precision (SP) operations. The Current implementation uses SP on most
+calculations, except for matrix inversions and reductions where double precision is required to retain high accuracy. All the
+constant spline data in wavefunction, pseudopotentials, and Coulomb potentials are initialized in double precision and later
+stored in single precision. The mixed-precision code is as accurate as the double-precision code up to a certain system size, and
+may have double the throughput.
+Cross checking and verification of accuracy is always required but is particularly important above approximately 1,500 electrons. 
 
 \subsection{Memory considerations}
 When using threads, some memory objects are shared by all the threads. Usually these memory objects are read only when the walkers are evolving, for instance the ionic distance table and wavefunction coefficients.

nexus/lib/pwscf_input.py

@@ -48,7 +48,7 @@
 import inspect
 from copy import deepcopy
 from superstring import string2val
-from numpy import fromstring,empty,array,float64,ones,pi,dot,ceil
+from numpy import fromstring,empty,array,float64,ones,pi,dot,ceil,ndarray
 from numpy.linalg import inv
 from unit_converter import convert
 from generic import obj
@@ -1973,7 +1973,8 @@ def generate_any_pwscf_input(**kwargs):
     nspin             = kwargs.get_optional('nspin',None)
     nbnd              = kwargs.get_optional('nbnd',None)
     hubbard_u         = kwargs.get_optional('hubbard_u',None)
-
+    occ               = kwargs.get_optional('occupations',None)
+
     #make an empty input file
     pw = PwscfInput()
 
@@ -2184,6 +2185,14 @@ def generate_any_pwscf_input(**kwargs):
     #        )
     #end if
 
+    # occupations card
+    if isinstance(occ,(list,ndarray)):
+        pw.system.occupations = 'from_input'
+        occ_card = occupations()
+        occ_card.occupations = array(occ,dtype=float)
+        pw.occupations = occ_card
+    #end if
+
     # adjust card options, if requested
     options = obj(
         atomic_positions = positions_option,