Further improve LLVM building

.github/workflows/CI.yml

@@ -8,7 +8,7 @@ on:
 env:
   NIGHTLY_TEST_SETTINGS: true
   # store the dependencies in one place
-  CHPL_APT_DEPS: gcc g++ m4 perl python3 python3-pip python3-venv python3-dev bash make mawk git pkg-config cmake llvm-11-dev llvm-11 llvm-11-tools clang-11 libclang-11-dev libedit-dev
+  CHPL_APT_DEPS: gcc g++ m4 perl python3 python3-pip python3-venv python3-dev bash make mawk git pkg-config cmake llvm-11-dev llvm-11 llvm-11-tools clang-11 libclang-11-dev libclang-cpp11-dev libedit-dev
 
 jobs:
   make_check:

compiler/llvm/clangUtil.cpp

@@ -4167,42 +4167,36 @@ void makeBinaryLLVM(void) {
   runtime_libs += CHPL_RUNTIME_SUBDIR;
   runtime_libs += "/list-libraries";
 
+  // TODO: move this logic to printchplenv
+  std::string runtime_ld_override(CHPL_RUNTIME_LIB);
+  runtime_ld_override += "/";
+  runtime_ld_override += CHPL_RUNTIME_SUBDIR;
+  runtime_ld_override += "/override-ld";
+
+  std::vector<std::string> ldOverride;
+  readArgsFromFile(runtime_ld_override, ldOverride);
+  // Substitute $CHPL_HOME $CHPL_RUNTIME_LIB etc
+  expandInstallationPaths(ldOverride);
+
+  std::string clangCC = clangInfo->clangCC;
+  std::string clangCXX = clangInfo->clangCXX;
+  std::string useLinkCXX = clangCXX;
+
+  if (ldOverride.size() > 0)
+    useLinkCXX = ldOverride[0];
+
   std::vector<std::string> runtimeArgs;
   readArgsFromFile(runtime_libs, runtimeArgs);
 
   std::vector<std::string> linkArgs;
   splitStringWhitespace(CHPL_LLVM_CLANG_LINK_ARGS, linkArgs);
 
-  // This might still be needed on CLE 6 systems.
-  // However, it is not currently working on CLE 7 systems
-  // (leading to an error about not being able to find libnuma)
-  // Perhaps we can use CRAYPE_LINK_TYPE=dynamic to opt in to dynamic linking
-  // on CLE 6. Or, make this logic check for CLE 6
-  // with /etc/opt/cray/release/cle-release.
-  /*if (compilingWithPrgEnv()) {
-    if (fLinkStyle == LS_DEFAULT) {
-      // check for indication that the PrgEnv defaults to dynamic linking
-      bool defaultDynamic = false;
-      for (auto & arg : linkArgs) {
-        if (arg == "-Wl,-Bdynamic"   // when PE links with gcc
-            || arg == "-dynamic") {  // when PE links with clang
-          defaultDynamic = true;
-        }
-      }
-
-      // Older Cray PrgEnv defaults to static linking.  If we are asking for
-      // the default link type, and we don't find an explicit dynamic
-      // flag in the gathered PrgEnv arguments, then force static linking
-      // because LLVM's default (dynamic) is different from the PrgEnv
-      // default (static).
-      if (defaultDynamic == false) {
-        fLinkStyle = LS_STATIC;
-      }
-    }
-  }*/
-
-  // start with arguments from CHPL_LLVM_CLANG_C
+  // start with arguments from CHPL_LLVM_CLANG_C unless
+  // using a non-clang compiler to link
   std::vector<std::string> clangLDArgs = clangInfo->clangLDArgs;
+  if (useLinkCXX != clangCXX)
+    clangLDArgs.clear();
+
   for (auto & arg : runtimeArgs) {
     clangLDArgs.push_back(arg);
   }
@@ -4224,24 +4218,6 @@ void makeBinaryLLVM(void) {
   expandInstallationPaths(clangLDArgs);
 
 
-  // TODO: move this logic to printchplenv
-  std::string runtime_ld_override(CHPL_RUNTIME_LIB);
-  runtime_ld_override += "/";
-  runtime_ld_override += CHPL_RUNTIME_SUBDIR;
-  runtime_ld_override += "/override-ld";
-
-  std::vector<std::string> ldOverride;
-  readArgsFromFile(runtime_ld_override, ldOverride);
-  // Substitute $CHPL_HOME $CHPL_RUNTIME_LIB etc
-  expandInstallationPaths(ldOverride);
-
-  std::string clangCC = clangInfo->clangCC;
-  std::string clangCXX = clangInfo->clangCXX;
-  std::string useLinkCXX = clangCXX;
-
-  if (ldOverride.size() > 0)
-    useLinkCXX = ldOverride[0];
-
 
   std::vector<std::string> dotOFiles;
 

compiler/main/driver.cpp

@@ -1694,61 +1694,6 @@ static void validateSettings() {
   checkUnsupportedConfigs();
 }
 
-// This code is off by default - opt in with CHPL_DO_PE_ENV_CHECK
-// It is just here to make it easier to debug problems with the compiler
-// and can be removed in the future.
-static void maybeRelaunchInPrgEnv(int argc, char* argv[]) {
-  // Relaunch in PrgEnv-gnu if CHPL_DO_PE_ENV_CHECK is set and:
-  //  * platform is cray-x* or cray-ex
-  //  * CHPL_TARGET_COMPILER is llvm
-  //  * PE_ENV does not indicate PrgEnv-gnu is loaded
-
-  bool prgEnvPlatform = startsWith(CHPL_TARGET_PLATFORM, "cray-x") ||
-                        0 == strcmp(CHPL_TARGET_PLATFORM, "cray-ex");
-  bool targetLlvm = 0 == strcmp(CHPL_TARGET_COMPILER, "llvm");
-  if (prgEnvPlatform && targetLlvm) {
-    // Default to not doing this.
-    // Keeping this code in for now for testing purposes.
-    if (getenv("CHPL_DO_PE_ENV_CHECK") == nullptr)
-      return;
-
-    // Exit early if we have already tried relaunching
-    if (getenv("CHPL_SKIP_PE_ENV_CHECK") != nullptr)
-      return;
-
-    // check if PrgEnv-gnu is loaded
-    const char* PE_ENV = getenv("PE_ENV");
-    bool prgEnvGnu = PE_ENV != nullptr && 0 == strcmp(PE_ENV, "GNU");
-
-    if (!prgEnvGnu) {
-      // Need to run ourselves again with PrgEnv-gnu
-      std::string exe = std::string(CHPL_HOME) +
-                        "/util/config/run-in-prgenv-gnu.bash";
-      std::vector<const char*> args;
-      args.push_back(exe.c_str());
-      for (int i = 0; i < argc; i++) {
-        args.push_back(argv[i]);
-      }
-      args.push_back(nullptr);
-
-      setenv("CHPL_SKIP_PE_ENV_CHECK", "1", 1);
-
-      if (printSystemCommands) {
-        printf("# relaunching under PrgEnv-gnu\n");
-        for (auto arg : args) {
-          if (arg != nullptr) {
-            printf("\"%s\" ", arg);
-          }
-        }
-        printf("\n");
-      }
-      execv(exe.c_str(), (char* const*) &args[0]);
-      // If that failed for some reason, continue not in PrgEnv-gnu
-      USR_WARN("Relaunching in PrgEnv-gnu failed");
-    }
-  }
-}
-
 int main(int argc, char* argv[]) {
   PhaseTracker tracker;
 
@@ -1790,8 +1735,6 @@ int main(int argc, char* argv[]) {
 
     setupChplGlobals(argv[0]);
 
-    maybeRelaunchInPrgEnv(argc, argv);
-
     postprocess_args();
 
     initCompilerGlobals(); // must follow argument parsing

doc/rst/usingchapel/prereqs.rst

@@ -65,8 +65,8 @@ We have used the following commands to install the above prerequisites:
 
   * Debian, Ubuntu::
 
-      sudo apt-get install gcc g++ m4 perl python3 python3-pip python3-venv python3-dev bash make mawk git pkg-config cmake llvm-11-dev llvm-11 llvm-11-tools clang-11 libclang-11-dev libedit-dev
+      sudo apt-get install gcc g++ m4 perl python3 python3-pip python3-venv python3-dev bash make mawk git pkg-config cmake llvm-11-dev llvm-11 llvm-11-tools clang-11 libclang-11-dev libclang-cpp11-dev libedit-dev
 
   * FreeBSD::
 
-     sudo pkg install gcc m4 perl5 python3 py37-pip bash gmake gawk git pkgconf cmake llvm11
+     sudo pkg install gcc m4 perl5 python3 bash gmake gawk git pkgconf cmake llvm11

make/compiler/Makefile.cray-prgenv

@@ -81,4 +81,4 @@ SPECIALIZE_CFLAGS =
 
 export PE_PKGCONFIG_PRODUCTS := PE_CHAPEL:$(PE_PKGCONFIG_PRODUCTS)
 export PE_CHAPEL_MODULE_NAME := chapel
-export PE_CHAPEL_PKGCONFIG_LIBS := $(shell $(CHPL_MAKE_HOME)/util/config/gather-pe-chapel-pkgconfig-libs.bash "$(CHPL_MAKE_TARGET_PLATFORM)" "$(CHPL_MAKE_COMM)" "$(CHPL_MAKE_COMM_SUBSTRATE)" "$(CHPL_MAKE_AUXFS)")
+export PE_CHAPEL_PKGCONFIG_LIBS := $(CHPL_MAKE_PE_CHPL_PKGCONFIG_LIBS)

runtime/src/comm/ugni/Makefile.share

@@ -46,10 +46,9 @@ ifeq ($(CHPL_MAKE_COMPILER), llvm)
     # if we're compiling with llvm for a cray-prgenv-* build,
     # gather some extra include paths
 
-    # only run gather-cray-prgenv-argumenst.bash once, as an optimization
-    ifndef CHPL_UGNI_PRGENV_CFLAGS
-      export CHPL_UGNI_PRGENV_CFLAGS := $(shell $(CHPL_MAKE_HOME)/util/config/gather-cray-prgenv-arguments.bash compile "$(CHPL_MAKE_TARGET_PLATFORM)" "$(CHPL_MAKE_COMM)" "$(CHPL_MAKE_COMM_SUBSTRATE)" "$(CHPL_MAKE_AUXFS)")
-    endif
+    # these should already be gathered in CHPL_LLVM_CLANG_COMPILE_ARGS
+    # by printchplenv.
+    CHPL_UGNI_PRGENV_CFLAGS := $(CHPL_MAKE_LLVM_CLANG_COMPILE_ARGS)
   endif
 endif
 

third-party/llvm/Makefile

@@ -38,16 +38,9 @@ ifneq ($(LLVM_SUBDIR),llvm-src)
   endif
 endif
 
-# Decide if we want to dynamically link or not
-# This sets the default - it could be overridden during the make
-CHPL_LLVM_DYNAMIC := 1
-ifneq (, $(findstring cray,$(CHPL_MAKE_HOST_PLATFORM)))
-  CHPL_LLVM_DYNAMIC := 0
-endif
-
 CHPL_LLVM_DYNAMIC_OPTS:=""
 ifeq ($(CHPL_LLVM_DYNAMIC),1)
-  CHPL_LLVM_DYNAMIC_OPTS:="-DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_LINK_LLVM_DYLIB=ON"
+  CHPL_LLVM_DYNAMIC_OPTS:=-DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_LINK_LLVM_DYLIB=ON
 endif
 
 CMAKE ?= cmake

third-party/llvm/Makefile.include-bundled

@@ -1,5 +1,12 @@
 include $(THIRD_PARTY_DIR)/llvm/Makefile.share-bundled
 
+# Link statically on Cray systems to avoid problems with
+# finding libstdc++ with different PrgEnv compilers.
+CHPL_LLVM_DYNAMIC := 1
+ifneq (, $(findstring cray,$(CHPL_MAKE_HOST_PLATFORM)))
+  CHPL_LLVM_DYNAMIC := 0
+endif
+
 # Enable RV and Polly if they are in the source directory
 LLVM_RV_FILE = $(LLVM_DIR)/$(LLVM_SUBDIR)/tools/rv/
 LLVM_RV_FILE2 = $(LLVM_DIR)/$(LLVM_SUBDIR)/../rv/
@@ -20,28 +27,20 @@ endif
 ifdef HAVE_LLVM_CONFIG
 ifndef LLVM_LLVM_LIBS
 
-ifeq ($(LLVM_MAJOR_VERSION),10)
-  # frontendopenmp seems to be required to link clang in llvm 10
-  # but not in earlier or later versions
-  export LLVM_LLVM_LIBS=$(shell $(LLVM_CONFIG) --ldflags --system-libs --libs bitreader bitwriter frontendopenmp ipo instrumentation option objcarcopts profiledata all-targets coverage coroutines lto)
-else
-  export LLVM_LLVM_LIBS_TMP=$(shell $(LLVM_CONFIG) --ldflags --system-libs --libs bitreader bitwriter ipo instrumentation option objcarcopts profiledata all-targets coverage coroutines lto)
-  # remove -llibxml2.tbd which seems to appear on some Mac OS X versions
-  # with LLVM 11.
-  export LLVM_LLVM_LIBS=$(subst -llibxml2.tbd,,$(LLVM_LLVM_LIBS_TMP))
-endif
+export LLVM_LLVM_LIBS_TMP=$(shell $(LLVM_CONFIG) --ldflags --system-libs --libs bitreader bitwriter ipo instrumentation option objcarcopts profiledata all-targets coverage coroutines lto)
+# remove -llibxml2.tbd which seems to appear on some Mac OS X versions
+# with LLVM 11.
+export LLVM_LLVM_LIBS=$(subst -llibxml2.tbd,,$(LLVM_LLVM_LIBS_TMP))
 
 endif
 endif
 
-ifeq ($(LLVM_MAJOR_VERSION_9PLUS),9)
-  # -lclangASTMatchers needed for LLVM 9 but not earlier
-  LLVM_CLANG_LIBS=-lclangFrontend -lclangSerialization -lclangDriver -lclangCodeGen -lclangParse -lclangSema -lclangAnalysis -lclangEdit -lclangASTMatchers -lclangAST -lclangLex -lclangBasic
+ifeq ($(CHPL_LLVM_DYNAMIC),1)
+  LLVM_CLANG_LIBS=-lclang-cpp
 else
-  LLVM_CLANG_LIBS=-lclangFrontend -lclangSerialization -lclangDriver -lclangCodeGen -lclangParse -lclangSema -lclangAnalysis -lclangEdit -lclangAST -lclangLex -lclangBasic
+  LLVM_CLANG_LIBS=-lclangFrontend -lclangSerialization -lclangDriver -lclangCodeGen -lclangParse -lclangSema -lclangAnalysis -lclangEdit -lclangASTMatchers -lclangAST -lclangLex -lclangBasic
 endif
 
-
 ifdef CHPL_RV
   LLVM_RV_CFLAGS=-DHAVE_LLVM_RV=1
   LLVM_RV_LIBS=-lRV -lgensleef

third-party/llvm/Makefile.include-system

@@ -1,17 +1,20 @@
 include $(THIRD_PARTY_DIR)/llvm/Makefile.share-system
 
+# Always link dynamically when using a system LLVM.
+# Future work: consider using 'llvm-config --shared-mode'
+# to make this choice.
+CHPL_LLVM_DYNAMIC := 1
+
 ifndef LLVM_LLVM_LIBS
   export LLVM_LLVM_LIBS=$(shell $(LLVM_CONFIG) --ldflags --system-libs --libs bitreader bitwriter ipo instrumentation option objcarcopts profiledata all-targets coverage coroutines lto)
 endif
 
-ifeq ($(LLVM_MAJOR_VERSION_9PLUS),9)
-  # -lclangASTMatchers needed for LLVM 9 but not earlier
-  LLVM_CLANG_LIBS=-lclangFrontend -lclangSerialization -lclangDriver -lclangCodeGen -lclangParse -lclangSema -lclangAnalysis -lclangEdit -lclangASTMatchers -lclangAST -lclangLex -lclangBasic
+ifeq ($(CHPL_LLVM_DYNAMIC),1)
+  LLVM_CLANG_LIBS=-lclang-cpp
 else
-  LLVM_CLANG_LIBS=-lclangFrontend -lclangSerialization -lclangDriver -lclangCodeGen -lclangParse -lclangSema -lclangAnalysis -lclangEdit -lclangAST -lclangLex -lclangBasic
+  LLVM_CLANG_LIBS=-lclangFrontend -lclangSerialization -lclangDriver -lclangCodeGen -lclangParse -lclangSema -lclangAnalysis -lclangEdit -lclangASTMatchers -lclangAST -lclangLex -lclangBasic
 endif
 
-
 # Ubuntu 16.04 needed -fno-rtti for LLVM 3.7
 # tested on that system after installing
 #   llvm-3.7-dev llvm-3.7 clang-3.7 libclang-3.7-dev libedit-dev
@@ -22,3 +25,4 @@ LLVM_CFLAGS=$(LLVM_CONFIG_CFLAGS) -DHAVE_LLVM
 
 LLVM_INCLUDES=-I$(LLVM_CONFIG_INCLUDE_DIR)
 LLVM_LIBS=-L$(LLVM_CONFIG_LIB_DIR) $(LLVM_CLANG_LIBS) $(LLVM_LLVM_LIBS)
+

third-party/llvm/Makefile.version

@@ -9,7 +9,3 @@ ifneq ($(LLVM_VERSION),)
 else
   export LLVM_MAJOR_VERSION:=0
 endif
-
-# LLVM_MAJOR_VERSION_9PLUS will store 9 if the version is >= 9 or
-# be the empty string otherwise.
-export LLVM_MAJOR_VERSION_9PLUS := $(shell test $(LLVM_MAJOR_VERSION) -ge 9 && echo 9)

util/chplenv/chpl_llvm.py

@@ -401,6 +401,10 @@ def get_clang_basic_args():
 
 @memoize
 def gather_pe_chpl_pkgconfig_libs():
+    # Don't do anything if we aren't using a PrgEnv compiler
+    if chpl_compiler.get_prgenv_compiler() == 'none':
+        return ""
+
     import chpl_comm, chpl_comm_substrate, chpl_aux_filesys, chpl_libfabric
     platform = chpl_platform.get('target')
     comm = chpl_comm.get()

util/chplenv/printchplenv.py

@@ -125,6 +125,7 @@
     ChapelEnv('  CHPL_RE2_UNIQ_CFG_PATH', INTERNAL),
     ChapelEnv('  CHPL_THIRD_PARTY_COMPILE_ARGS', INTERNAL),
     ChapelEnv('  CHPL_THIRD_PARTY_LINK_ARGS', INTERNAL),
+    ChapelEnv('  CHPL_PE_CHPL_PKGCONFIG_LIBS', INTERNAL)
 ]
 
 # Global map of environment variable names to values
@@ -269,6 +270,7 @@ def compute_internal_values():
                           if not (arg in seen or seen.add(arg))]
     ENV_VALS['  CHPL_THIRD_PARTY_LINK_ARGS'] = ' '.join(reversed(link_args_3p_dedup))
 
+    ENV_VALS['  CHPL_PE_CHPL_PKGCONFIG_LIBS'] = chpl_llvm.gather_pe_chpl_pkgconfig_libs()
 
 """Return non-empty string if var is set via environment or chplconfig"""
 def user_set(env):