From 0a5d109790c765cb23d3888352ab70d8db9f6d1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20H=C3=BCck?= Date: Fri, 18 Jun 2021 17:24:23 +0200 Subject: [PATCH] Release 1.6 (#78) * Set CMake version to 1.6 * Fix initial demo related issues (Makefile, deprecated MPI symbols) * Rework README.md --- CMakeLists.txt | 2 +- README.md | 424 +++++++++++++++++++++-------- demo/.gitignore | 4 + demo/01_struct_example.c | 9 +- demo/02_broken_struct_example.c | 9 +- demo/Makefile | 63 ++--- demo/applyPass.sh | 43 --- lib/mpi_interceptor/CMakeLists.txt | 3 +- 8 files changed, 353 insertions(+), 204 deletions(-) delete mode 100755 demo/applyPass.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 12e24aa9..452b3947 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.14) project(typeart - VERSION 1.5 + VERSION 1.6 ) set(TYPEART_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/README.md b/README.md index 85151ac0..09c43247 100644 --- a/README.md +++ b/README.md @@ -1,149 +1,347 @@ # TypeART · [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) ![](https://github.com/tudasc/TypeART/workflows/TypeART-CI/badge.svg?branch=master) ![](https://github.com/tudasc/TypeART/workflows/TypeART-CI-ext/badge.svg?branch=master) [![Coverage Status](https://coveralls.io/repos/github/tudasc/TypeART/badge.svg?branch=master)](https://coveralls.io/github/tudasc/TypeART) -TypeART \[[TA18](#ref-typeart-2018); [TA20](#ref-typeart-2020)\] is a type and memory allocation tracking sanitizer. It -consists of an LLVM compiler pass and a corresponding runtime to track relevant memory allocation information during the -execution of a target program. It instruments heap, stack and global variable allocations with a callback to our -runtime. The callback consists of the runtime memory pointer value, what type (built-ins, user-defined structs etc.) and -extent of the value. This allows users of our runtime to query detailed type information behind arbitrary memory -locations, as long as they are mapped. +## What is TypeART? + +TypeART \[[TA18](#ref-typeart-2018); [TA20](#ref-typeart-2020)\] is a type and memory allocation tracking sanitizer +based on the [LLVM](https://llvm.org) compiler toolchain for C/C++ (OpenMP) codes. It consists of an LLVM compiler pass +plugin for instrumentation, and a corresponding runtime to track memory allocations during the execution of a target +program. + +TypeART instruments heap, stack and global variable allocations with a callback to our runtime. The callback consists of +(1) the memory address, (2) the type-layout information of the allocation (built-ins, user-defined structs etc.) and (3) +number of elements. + +## Why use it? + +Employ TypeART whenever you need type-related information of allocations in your program to verify some property, and +generate diagnostics if it doesn't hold. + +For instance, low-level C-language APIs use `void`-pointers as generic types. Often, the user must specify its type and +length manually. This can be error prone. Examples for type unsafe APIs include the Message-Passing Interface (MPI), +checkpointing libraries and numeric solver libraries. With TypeART, it is straightforward to verify that a `void` +-pointer argument to an API is, e.g., a type `T` array with length `n`. ### Use Case: MUST - A dynamic MPI correctness checker -TypeART is used in conjunction with MUST \[[MU13](#ref-must-2013)\] to track memory (de-)allocation relevant to MPI -communication. Thus, MUST can check for type compatibility between the type-less communication buffer and the declared -MPI datatype at all phases of the MPI communication, namely message assembly, message transfer and message disassembly -into the receiving buffer. A brief summary is given in a subsequent section and more information can be found in our -publication: +MUST \[[MU13](#ref-must-2013)\] is a dynamic MPI correctness checker to, e.g., detect deadlocks or a mismatch of MPI +datatypes of the sending and receiving process, see its [project page](https://www.hpc.rwth-aachen.de/must/). -#### References +MUST relies on intercepting MPI calls for its analysis. As a consequence, though, MUST is unaware of the *effective* +type of the allocated `void*` buffers used for the low-level MPI API. To that end, TypeART was developed to track +memory (de-)allocation relevant to MPI communication. With TypeART, MUST can check for type compatibility between the +type-less MPI communication buffer and the declared MPI datatype. - - - - - - - - - - - - - -
[TA18]Hück, Alexander and Lehr, Jan-Patrick and Kreutzer, Sebastian and Protze, Joachim and Terboven, Christian and Bischof, Christian and Müller, Matthias S. - - Compiler-aided type tracking for correctness checking of MPI applications. - In 2nd International Workshop on Software Correctness for HPC Applications (Correctness), - pages 51–58. IEEE, 2018.
[TA20]Hück, Alexander and Protze, Joachim and Lehr, Jan-Patrick and Terboven, Christian and Bischof, Christian and Müller, Matthias S. - - Towards compiler-aided correctness checking of adjoint MPI applications. - In 4th International Workshop on Software Correctness for HPC Applications (Correctness), - pages 40–48. IEEE/ACM, 2020.
[MU13]Hilbrich, Tobias and Protze, Joachim and Schulz, Martin and de Supinski, Bronis R. and Müller, Matthias S. - - MPI Runtime Error Detection with MUST: Advances in Deadlock Detection. - In Scientific Programming, vol. 21, no. 3-4, - pages 109–121, 2013.
+#### Type checking for MPI calls -## Software dependencies +Consider the MPI function `MPI_Send(const void* buffer, int count, MPI_Datatype datatype, ...)`. Without TypeART, MUST +cannot check (1) if the `buffer` argument is compatible with the declared `MPI_Dataype` and (2) if the `count` argument +exceeds the `buffer` allocation size: -TypeART requires [LLVM](https://llvm.org) version 10 and CMake version >= 3.14. +```c +// TypeART tracks this allocation (memory address, type and size): +double* array = (double*) malloc(length*sizeof(double)); +// MUST intercepts this MPI call, and asks TypeARTs runtime library for type information: +// 1. Is the first argument of type double (due to MPI_DOUBLE)? +// 2. Is the allocation at least of size *length*? +MPI_Send((void*) array, length, MPI_DOUBLE, ...) +``` -#### Building TypeART +MUST and TypeART also handle MPI [derived datatypes](https://www.mpi-forum.org/docs/mpi-3.1/mpi31-report/node77.htm) +with complex underlying datastructures, see our [MPI Demo](#13-example-mpi-demo). For more details, see +our [publications](#references), or download the current release (1.8 or higher) of MUST on +its [project page](https://itc.rwth-aachen.de/must/). + +## Table of Contents + +* [1. Using TypeART](#1-using-typeart) + * [1.1 Compiling a target code](#11-compiling-a-target-code) + * [1.1.1 Building with TypeART](#111-building-with-typeart) + * [1.1.2 Options for TypeART passes](#112-options-for-typeart-passes) + * [Example invocations](#example-invocations) + * [1.1.3 Serialized type information](#113-serialized-type-information) + * [1.1.4 Filtering allocations](#114-filtering-allocations) + * [1.2 Executing an instrumented target code](#12-executing-an-instrumented-target-code) + * [1.3 Example: MPI demo](#13-example-mpi-demo) +* [2. Building TypeART](#2-building-typeart) + * [2.1 Optional software requirements](#21-optional-software-requirements) + * [2.2 Building](#22-building) + * [2.2.1 CMake configuration: Options for users](#221-cmake-configuration-options-for-users) +* [References](#references) + +## 1. Using TypeART -TypeART uses CMake to build, cf. [GitHub CI build file](.github/workflows/basic-ci.yml) for a complete recipe to build. -Example build recipe (debug build, installs to default prefix -`${typeart_SOURCE_DIR}/install/typeart`) +Making use of TypeART consists of two phases: -```{.sh} -$> git clone https://github.com/tudasc/TypeART -$> cd TypeART -$> cmake -B build -$> cmake --build build --target install --parallel +1. Compile your code with Clang/LLVM (version 10) using the TypeART LLVM pass plugins to (1) serialize static type + information to a file and (2) instrument all relevant allocations. See [Section 1.1](#11-compiling-a-target-code). +2. Execute the target program with a runtime library (a *client* based on the TypeART runtime) to accept the callbacks + to do some useful analysis with our interface based on the static type information. + See [Section 1.2](#12-executing-an-instrumented-target-code). + +### 1.1 Compiling a target code + +Our LLVM compiler pass plugins instrument allocations and also serialize the static type layouts of these allocations to +a yaml file (default name `types.yaml`). + +#### 1.1.1 Building with TypeART + +A typical compile invocation may first compile code to object files and then link with any libraries: + +```shell +# Compile: +$> clang++ -O2 $(COMPILE_FLAGS) -c code.cpp -o code.o +# Link: +$> clang++ $(LINK_FLAGS) code.o -o binary ``` -#### CMake Configuration: Options for users +With TypeART, the recipe needs to be changed, as we rely on the LLVM `opt` (optimizer) tool to load and apply our +TypeART passes to a target code based on the LLVM intermediate representation (IR): -##### Runtime +1. Compile the code down to LLVM IR, and pipe the output to the LLVM `opt` tool. (Keeping your original compile flags) +2. Apply heap instrumentation with TypeART through `opt`. +3. Optimize the code with -Ox using `opt`. +4. Apply stack and global instrumentation with TypeART through `opt`. +5. Pipe the final output to LLVM `llc` to generate the final object file. +6. Finally, link the TypeART runtime library. + +*Note*: We instrument heap allocations before any optimization, as the compiler may throw out type information of these +allocations (for optimization reasons). -- `SOFTCOUNTERS` (default: **off**) : Enable runtime tracking of #tracked addrs. / #distinct checks / etc. -- `USE_ABSL` (default: **on**) : Enable usage of btree-backed map of the abseil project instead of std::map for the - runtime. -- `USE_BTREE` (default: **off**) : Enable usage of a btree-backed map (alternative to abseil) instead of std::map for - the runtime. +```shell +# Compile: 1.Code-To-LLVM | 2.TypeART_HEAP | 3.Optimize | 4.TypeART_Stack | 5.Object-file +$> clang++ $(COMPILE_FLAGS) $(EMIT_LLVM_IR_FLAGS) code.cpp | opt $(TYPEART_PLUGIN) $(HEAP_ONLY_FLAGS) | opt -O2 -S | opt $(TYPEART_PLUGIN) $(STACK_ONLY_FLAGS) | llc $(TO_OBJECT_FILE) +# Link: +$> clang++ $(LINK_FLAGS) -L$(TYPEART_LIBPATH) -ltypeart-rt code.o -o binary +``` -##### Logging and Passes +#### 1.1.2 Options for TypeART passes + +The main options are shown below. + +| Flag | Default | Description | +| --- | :---: | --- | +| `typeart` | - | Invoke TypeART pass through LLVM `opt` | +| `typeart-outfile` | `types.yaml` | Serialized type layout information of user-defined types | +| `typeart-no-heap` | `false` | Do **not** instrument heap allocations | +| `typeart-alloca` | `false` | Instrument stack and global allocations | +| `typeart-stats` | `false` | Show instrumentation statistic counters | +| `call-filter` | `false` | Filter stack and global allocations. See also [Section 1.1.4](#114-filtering-allocations) | +| `call-filter-str` | `*MPI_*` | Filter string target (glob string) | + +##### Example invocations + +###### Pre-requisites + +1. Loading TypeART plugins with `opt`: + ```shell + TYPEART_PLUGIN=-load $(PLUGIN_PATH)/meminstfinderpass.so \ + -load $(PLUGIN_PATH)/typeartpass.so + ``` +2. Input of `opt` is LLVM IR, e.g.: + ```shell + # Pipe LLVM IR to console + clang++ -g -Xclang -disable-llvm-passes -S -emit-llvm -o - example.cpp + ``` + +###### Examples + +- Heap-only instrumentation (with stats): + ```shell + opt $(TYPEART_PLUGIN) -typeart -typeart-stats + ``` +- Stack- and global-only instrumentation (*no* stats): + ```shell + opt $(TYPEART_PLUGIN) -typeart -typeart-no-heap=true -typeart-alloca + ``` +- Stack- and global-only instrumentation (with default filtering for MPI): + ```shell + opt $(TYPEART_PLUGIN) -typeart -typeart-no-heap=true -typeart-alloca -call-filter + ``` +- Filtering w.r.t. non-standard target API: + ```shell + opt $(TYPEART_PLUGIN) -typeart -typeart-no-heap=true -typeart-alloca -call-filter -call-filter-str=MY_API* + ``` +- Combined instrumentation (with filtering): + ```shell + opt $(TYPEART_PLUGIN) -typeart -typeart-alloca -call-filter + ``` + +Also consult the [demo Makefile](demo/Makefile) for an example recipe, and flags for TypeART. + +#### 1.1.3 Serialized type information + +After instrumentation, the file `types.yaml` contains the static type information. Each user-defined type layout is +extracted and an integer `type-id` is attached to it. Built-in types (e.g., float) have pre-defined ids and byte +layouts. + +The TypeART instrumentation callbacks use the `type-id`. The runtime library correlates the allocation with the +respective type (and layout) during execution. Consider the following struct: + +```c +struct s1_t { + char a[3]; + struct s1_t* b; +} +``` -- `SHOW_STATS` (default: **on**) : Passes show the statistics w.r.t. allocations etc. -- `MPI_LOGGER` (default: **on**) : Enable better logging support in MPI execution context -- `MPI_INTERCEPT_LIB` (default: **on**) : Library can be used by preloading to intercept MPI calls and check whether - TypeART tracks the buffer pointer -- `LOG_LEVEL_` and `LOG_LEVEL_RT` (default **0**) : Granularity of logger. 3 ist most verbose, 0 is least. +The TypeART pass may write a `types.yaml` file with the following content: + +```yaml +- id: 256 // struct type-id + name: struct.s1_t + extent: 16 // byte size + member_count: 2 + offsets: [ 0, 8 ] // byte offsets from struct start + types: [ 0, 10 ] // member type-ids (0->char, 10->pointer) + sizes: [ 3, 1 ] // member (array) length +``` + + +#### 1.1.4 Filtering allocations + +To improve performance, a translation unit-local (TU) data-flow filter for global and stack variables exist. It follows +the LLVM IR `use-def` chain. If the allocation provably never reaches the target API, it can be filtered. Otherwise, it +is instrumented. + +Consider the following example. + +```c +extern foo_bar(float*); // No definition in the TU +void bar(float* x, float* y) { + *x = 2.f; // x is not used after + MPI_Send(y, ...); +} +void foo() { + float a = 1.f, b = 2.f, c = 3.f; + bar(&a, &b); + foo_bar(&c); +} +``` -##### Testing +1. The filter can remove `a`, as the aliasing pointer `x` is never part of an MPI call. +2. `b` is instrumented as the aliasing pointer `y` is part of an MPI call. +3. `c` is instrumented as we cannot reason about the body of `foo_bar`. -- `TEST_CONFIG` (default: **off**) : Set (force) logging levels to appropriate levels for test runner to succeed -- `ENABLE_CODE_COVERAGE` (default: **off**) : Enable code coverage statistics using LCOV 1.14 and genhtml (gcovr - optional) -- `ENABLE_LLVM_CODE_COVERAGE` (default: **off**) : Enable llvm-cov code coverage statistics (llvm-cov and llvm-profdata - required) +### 1.2 Executing an instrumented target code -## Using TypeART +To execute the instrumented code, the TypeART runtime library (or a derivative) has to be loaded to accept the +callbacks. The library also requires access to the `types.yaml` file to correlate the `type-id` with the actual type +layouts. To specify its path, you can use the environment variable `TA_TYPE_FILE`, e.g.: -Making use of TypeART consists of two phases: +```shell +export TA_TYPE_FILE=/shared/types.yaml +env LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(TYPEART_LIBPATH) ./binary +``` -1. Compile and instrument the target code with our LLVM passes, and, -2. execute the target program with a runtime library (based on the TypeART runtime) to accept the callbacks from the - instrumented code and actually do some useful analysis. +An example for pre-loading a TypeART-based library in the context of MPI is found in the demo, +see [Section 1.3](#13-example-mpi-demo). -To that end, the interface [RuntimeInterface.h](runtime/RuntimeInterface.h) can be used to query type information during -the target code execution. +### 1.3 Example: MPI demo -#### Example: MPI Demo +The folder [demo](demo) contains an example of MPI-related type errors that can be detected using TypeART. The code is +compiled with our instrumentation, and executed by preloading the MPI-related check library implemented +in [tool.c](demo/tool.c). The check library uses the TypeART [runtime query interface](lib/runtime/RuntimeInterface.h). +It overloads the required MPI calls and checks that the passed `void*` buffer is correct w.r.t. the MPI derived +datatype. -The folder [demo](demo) contains an example of MPI related type errors that can be detected using TypeART. The code is -compiled with our instrumentation, and executed by preloading the MPI related check library implemented -in [tool.c](demo/tool.c), which is linked against the TypeART runtime and uses the aforementioned query interface. It -overloads the required MPI calls and checks that the passed `void* buffer` is correct. +## 2. Building TypeART -## LLVM pass +TypeART requires LLVM version 10 and CMake version >= 3.14. -The necessary allocation sites and type information are extracted in LLVM passes. TypeART analyzes: +### 2.1 Optional software requirements -- Calls to ```malloc``` and ```free``` to keep track of active pointers referring to objects allocated on the heap, -- relevant stack space allocations, i.e., allocations that cannot be proven to never lead to ```MPI``` functions, -- built-in as well as user-defined types to retrieve type size and the size of the allocation, e.g., for arrays. +- MPI library: Needed for some tests, the [demo](demo), our [MPI interceptor library](lib/mpi_interceptor), and for + logging with our TypeART runtime library within an MPI target application. +- OpenMP-enabled Clang compiler: Needed for some tests. -The type information is necessary to correlate the type of the buffer passed to an MPI call with the MPI datatype the -user declared. In this prototype we restrict ourselves to: +Other smaller, external dependencies are defined within the [externals folder](externals) (depending on configuration +options), see [Section 2.2.1 (Runtime)](#221-cmake-configuration-options-for-users). They are automatically downloaded +during configuration time (internet connection required). -+ primitive types (int, float, long, double, char, unsigned int, unsigned long) -+ arrays of primitive types -+ structs which contain only primitive types +### 2.2 Building -### Example of Instrumentation: Handling malloc +TypeART uses CMake to build, cf. [GitHub CI build file](.github/workflows/basic-ci.yml) for a complete recipe to build. +Example build recipe (debug build, installs to default prefix +`${typeart_SOURCE_DIR}/install/typeart`) -To instrument relevant allocations and extract the necessary type information, the LLVM pass searches for specific -patterns, e.g., how calls to ```malloc``` look like in LLVM IR. Calls to the ```malloc``` function are typically call -instructions followed by a ```bitcast``` instruction to cast the returned pointer to the desired type. +```sh +$> git clone https://github.com/tudasc/TypeART +$> cd TypeART +$> cmake -B build +$> cmake --build build --target install --parallel +``` -~~~{.ll} -; %0 == n * sizeof(float) -%1 = tail call i8* @malloc(i64 %0) -%2 = bitcast i8* %1 to float * -~~~ +#### 2.2.1 CMake configuration: Options for users -The patterns has all the information we require for our instrumentation. Our transformation first detects the type that -the returned pointer is casted to, then it computes the extent of the allocation. The information is passed to our -instrumentation function. +##### Runtime + + +| Option | Default | Description | +| --- | :---: | --- | +| `USE_ABSL` | `ON` | Enable usage of btree-backed map of the [Abseil project](https://abseil.io/) instead of `std::map` | +| `USE_BTREE` | `OFF` | *Deprecated*. Enable usage of a [btree-backed map](https://github.com/ahueck/cpp-btree) (alternative to Abseil) instead of `std::map` | +| `SOFTCOUNTERS` | `OFF` | Enable runtime tracking of #tracked addrs. / #distinct checks / etc. | +| `LOG_LEVEL_RT` | `0` | Granularity of runtime logger. 3 ist most verbose, 0 is least | + + +###### Runtime thread-safety options + +Default mode is to protect the global data structure with a (shared) mutex. Two main options exist: + + +| Option | Default | Description | +| --- | :---: | --- | +| `DISABLE_THREAD_SAFETY` | `OFF` | Disable thread safety of runtime | +| `ENABLE_SAFEPTR` | `OFF` | Instead of a mutex, use a special data structure wrapper for concurrency, see [object_threadsafe](https://github.com/AlexeyAB/object_threadsafe) | + + +##### LLVM passes + + +| Option | Default | Description | +| --- | :---: | --- | +| `SHOW_STATS` | `ON` | Passes show compile-time summary w.r.t. allocations counts | +| `MPI_INTERCEPT_LIB` | `ON` | Library to intercept MPI calls by preloading and check whether TypeART tracks the buffer pointer | +| `MPI_LOGGER` | `ON` | Enable better logging support in MPI execution context | +| `LOG_LEVEL` | `0` | Granularity of pass logger. 3 ist most verbose, 0 is least | + -~~~{.ll} -; %0 == n * sizeof(float) -; %1 holds the returned pointer value -%1 = tail call i8* @malloc(i64 %0) -; compute the number of elements -%2 = udiv i64 %0, 4 -; call TypeART runtime (5 is the type-id for float) -call void @__typeart_alloc(i8 *%1, i32 5, i64 %2) -; original bitcast -%3 = bitcast i8* %1 to float * -~~~ +##### Testing + + +| Option | Default | Description | +| --- | :---: | --- | +| `TEST_CONFIG` | `OFF` | Set (force) logging levels to appropriate levels for test runner to succeed | +| `ENABLE_CODE_COVERAGE` | `OFF` | Enable code coverage statistics using LCOV 1.14 and genhtml (gcovr optional) | +| `ENABLE_LLVM_CODE_COVERAGE` | `OFF` | Enable llvm-cov code coverage statistics (llvm-cov and llvm-profdata required) | +| `ENABLE_ASAN, TSAN, UBSAN` | `OFF` | Enable Clang sanitizers (tsan is mutually exlusive w.r.t. ubsan and asan as they don't play well together) | + + +## References + + + + + + + + + + + + + + +
[TA18]Hück, Alexander and Lehr, Jan-Patrick and Kreutzer, Sebastian and Protze, Joachim and Terboven, Christian and Bischof, Christian and Müller, Matthias S. + + Compiler-aided type tracking for correctness checking of MPI applications. + In 2nd International Workshop on Software Correctness for HPC Applications (Correctness), + pages 51–58. IEEE, 2018.
[TA20]Hück, Alexander and Protze, Joachim and Lehr, Jan-Patrick and Terboven, Christian and Bischof, Christian and Müller, Matthias S. + + Towards compiler-aided correctness checking of adjoint MPI applications. + In 4th International Workshop on Software Correctness for HPC Applications (Correctness), + pages 40–48. IEEE/ACM, 2020.
[MU13]Hilbrich, Tobias and Protze, Joachim and Schulz, Martin and de Supinski, Bronis R. and Müller, Matthias S. + + MPI Runtime Error Detection with MUST: Advances in Deadlock Detection. + In Scientific Programming, vol. 21, no. 3-4, + pages 109–121, 2013.
\ No newline at end of file diff --git a/demo/.gitignore b/demo/.gitignore index 7889e808..dc525784 100644 --- a/demo/.gitignore +++ b/demo/.gitignore @@ -1,3 +1,7 @@ 01_ex 02_ex +toy +toy-stack libtool.so +types.yaml + diff --git a/demo/01_struct_example.c b/demo/01_struct_example.c index cb2ef20d..d82fb48f 100644 --- a/demo/01_struct_example.c +++ b/demo/01_struct_example.c @@ -56,7 +56,7 @@ int main(int argc, char** argv) { MPI_Aint first_var_address; MPI_Aint second_var_address; - MPI_Datatype array_of_types[COUNT] = {MPI_LB, MPI_DOUBLE, MPI_DOUBLE, MPI_INT, MPI_UB}; + MPI_Datatype array_of_types[COUNT - 2] = {MPI_DOUBLE, MPI_DOUBLE, MPI_INT}; MPI_Datatype parttype, fulltype, veltype, postype; MPI_Status status; @@ -92,18 +92,17 @@ int main(int argc, char** argv) { // MPI_Type_create_struct(COUNT, array_of_blocklengths, // array_of_displacements, array_of_types, &fulltype); - MPI_Type_create_struct(COUNT - 2, array_of_blocklengths + 1, array_of_displacements + 1, array_of_types + 1, - &parttype); + MPI_Type_create_struct(COUNT - 2, array_of_blocklengths + 1, array_of_displacements + 1, array_of_types, &parttype); MPI_Type_create_resized(parttype, array_of_displacements[0], array_of_displacements[COUNT - 1], &fulltype); MPI_Type_commit(&fulltype); MPI_Type_free(&parttype); - MPI_Type_create_struct(1, array_of_blocklengths + 1, array_of_displacements + 1, array_of_types + 1, &parttype); + MPI_Type_create_struct(1, array_of_blocklengths + 1, array_of_displacements + 1, array_of_types, &parttype); MPI_Type_create_resized(parttype, array_of_displacements[0], array_of_displacements[COUNT - 1], &veltype); MPI_Type_commit(&veltype); MPI_Type_free(&parttype); - MPI_Type_create_struct(1, array_of_blocklengths + 2, array_of_displacements + 2, array_of_types + 2, &parttype); + MPI_Type_create_struct(1, array_of_blocklengths + 2, array_of_displacements + 2, array_of_types + 1, &parttype); MPI_Type_create_resized(parttype, array_of_displacements[0], array_of_displacements[COUNT - 1], &postype); MPI_Type_commit(&postype); MPI_Type_free(&parttype); diff --git a/demo/02_broken_struct_example.c b/demo/02_broken_struct_example.c index 4f64a157..fc943e4c 100644 --- a/demo/02_broken_struct_example.c +++ b/demo/02_broken_struct_example.c @@ -56,7 +56,7 @@ int main(int argc, char** argv) { MPI_Aint first_var_address; MPI_Aint second_var_address; - MPI_Datatype array_of_types[COUNT] = {MPI_LB, MPI_INT, MPI_DOUBLE, MPI_DOUBLE, MPI_UB}; + MPI_Datatype array_of_types[COUNT - 2] = {MPI_INT, MPI_DOUBLE, MPI_DOUBLE}; MPI_Datatype parttype, fulltype, veltype, postype; MPI_Status status; @@ -86,18 +86,17 @@ int main(int argc, char** argv) { array_of_displacements[3] = array_of_displacements[2] + sizeof(double) * 3; array_of_displacements[4] = array_of_displacements[3] + sizeof(double) * 3 + sizeof(void*); - MPI_Type_create_struct(COUNT - 2, array_of_blocklengths + 1, array_of_displacements + 1, array_of_types + 1, - &parttype); + MPI_Type_create_struct(COUNT - 2, array_of_blocklengths + 1, array_of_displacements + 1, array_of_types, &parttype); MPI_Type_create_resized(parttype, array_of_displacements[0], array_of_displacements[COUNT - 1], &fulltype); MPI_Type_commit(&fulltype); MPI_Type_free(&parttype); - MPI_Type_create_struct(1, array_of_blocklengths + 2, array_of_displacements + 2, array_of_types + 2, &parttype); + MPI_Type_create_struct(1, array_of_blocklengths + 2, array_of_displacements + 2, array_of_types + 1, &parttype); MPI_Type_create_resized(parttype, array_of_displacements[0], array_of_displacements[COUNT - 1], &veltype); MPI_Type_commit(&veltype); MPI_Type_free(&parttype); - MPI_Type_create_struct(1, array_of_blocklengths + 3, array_of_displacements + 3, array_of_types + 3, &parttype); + MPI_Type_create_struct(1, array_of_blocklengths + 3, array_of_displacements + 3, array_of_types + 2, &parttype); MPI_Type_create_resized(parttype, array_of_displacements[0], array_of_displacements[COUNT - 1], &postype); MPI_Type_commit(&postype); MPI_Type_free(&parttype); diff --git a/demo/Makefile b/demo/Makefile index de14decb..2e9b6844 100644 --- a/demo/Makefile +++ b/demo/Makefile @@ -1,55 +1,48 @@ -BUILDDIR ?= ../build +# NOTE: env TYPEART_PATH must be set to install base path of TypeART + MPIRUN ?= mpirun MPICC ?= OMPI_CC=clang mpicc CC ?= cc -TYPEART_PASS_DIR = $(BUILDDIR)/lib -TYPEART_RUNTIME_DIR = $(BUILDDIR)/runtime -CLFLAGS = -L$(TYPEART_RUNTIME_DIR) -ltypeart-rt -CFLAGS = -Xclang -load -Xclang $(TYPEART_PASS_DIR)/analysis/meminstfinderpass.so -Xclang -load -Xclang $(TYPEART_PASS_DIR)/typeartpass.so -mllvm -alloca-array-only=false -mllvm -typeart-alloca +TYPEART_LIBPATH = ${TYPEART_PATH}/lib +TYPEART_INCPATH = ${TYPEART_PATH}/include + +typeart_plugin = -load $(TYPEART_LIBPATH)/meminstfinderpass.so -load $(TYPEART_LIBPATH)/typeartpass.so -typeart +typeart_stack_mode_args = -typeart-no-heap -typeart-alloca -typeart-stats -S +typeart_heap_mode_args = -typeart-no-heap=false -typeart-stats -S + +CLFLAGS = -L$(TYPEART_LIBPATH) -ltypeart-rt -all: libtool.so 01_ex 02_ex toy toy-stack toy-nostack +all: libtool.so 01_ex 02_ex toy toy-stack libtool.so: tool.c - $(MPICC) -I../runtime/ -I../typelib/ $(CLFLAGS) -shared -fPIC $< -o $@ + $(MPICC) -I$(TYPEART_INCPATH) $(CLFLAGS) -shared -fPIC $< -o $@ 01_ex: 01_struct_example.c - $(MPICC) $(CFLAGS) $(CLFLAGS) $< -o $@ -O1 + $(MPICC) -g -O1 -Xclang -disable-llvm-passes -S -emit-llvm $< -o - | opt $(typeart_plugin) $(typeart_heap_mode_args) | opt -O1 -S | opt $(typeart_plugin) $(typeart_stack_mode_args) | llc -x=ir -filetype=obj -o $@.o + $(MPICC) $(CLFLAGS) $@.o -o $@ 02_ex: 02_broken_struct_example.c - $(MPICC) $(CFLAGS) $(CLFLAGS) $< -o $@ -O1 -03_ex: 03_array_example.c - $(MPICC) $(CFLAGS) $(CLFLAGS) $< -o $@ -O1 - -toy: toy.c - $(MPICC) -I../runtime/ -I../typelib/ $(CFLAGS) $(CLFLAGS) $< -o $@ -g + $(MPICC) -g -O1 -Xclang -disable-llvm-passes -S -emit-llvm $< -o - | opt $(typeart_plugin) $(typeart_heap_mode_args) | opt -O1 -S | opt $(typeart_plugin) $(typeart_stack_mode_args) | llc -x=ir -filetype=obj -o $@.o + $(MPICC) $(CLFLAGS) $@.o -o $@ toy-stack: toy.c - $(MPICC) -I../runtime/ -I../typelib/ -S -emit-llvm $< -o $@.ll -g - opt -load $(TYPEART_PASS_DIR)/analysis/meminstfinderpass.so -load $(TYPEART_PASS_DIR)/typeartpass.so -typeart -typeart-alloca -alloca-array-only=false < $@.ll -o $@.ll > /dev/null - llc $@.ll -o $@.s - $(MPICC) $(CLFLAGS) $@.s -o $@ - -toy-nostack: toy.c - $(MPICC) -I../runtime/ -I../typelib/ -DNOSTACK -S -emit-llvm $< -o $@.ll -g - opt -load $(TYPEART_PASS_DIR)/analysis/meminstfinderpass.so -load $(TYPEART_PASS_DIR)/typeartpass.so -typeart < $@.ll -o $@.ll > /dev/null - llc $@.ll -o $@.s - $(MPICC) $(CLFLAGS) $@.s -o $@ + $(MPICC) -I$(TYPEART_INCPATH) -g -O1 -Xclang -disable-llvm-passes -S -emit-llvm $< -o - | opt $(typeart_plugin) $(typeart_heap_mode_args) | opt -O1 -S | opt $(typeart_plugin) $(typeart_stack_mode_args) | llc -x=ir -filetype=obj -o $@.o + $(MPICC) $(CLFLAGS) $@.o -o $@ +toy: toy.c + $(MPICC) -I$(TYPEART_INCPATH) -DNOSTACK -g -O1 -Xclang -disable-llvm-passes -S -emit-llvm $< -o - | opt $(typeart_plugin) $(typeart_heap_mode_args) | opt -O1 -S | opt $(typeart_plugin) $(typeart_stack_mode_args) | llc -x=ir -filetype=obj -o $@.o + $(MPICC) $(CLFLAGS) $@.o -o $@ -runtoy: toy toy-stack toy-nostack - env LD_LIBRARY_PATH=$$LD_LIBRARY_PATH:$(TYPEART_RUNTIME_DIR) ./toy - env LD_LIBRARY_PATH=$$LD_LIBRARY_PATH:$(TYPEART_RUNTIME_DIR) ./toy-stack - env LD_LIBRARY_PATH=$$LD_LIBRARY_PATH:$(TYPEART_RUNTIME_DIR) ./toy-nostack +runtoy: toy toy-stack + env LD_LIBRARY_PATH=$$LD_LIBRARY_PATH:$(TYPEART_LIBPATH) ./toy + env LD_LIBRARY_PATH=$$LD_LIBRARY_PATH:$(TYPEART_LIBPATH) ./toy-stack -run: run01 run02 run03 +run: run01 run02 run01: 01_ex libtool.so - env LD_PRELOAD=./libtool.so LD_LIBRARY_PATH=$$LD_LIBRARY_PATH:$(TYPEART_RUNTIME_DIR) $(MPIRUN) -np 1 01_ex + env LD_PRELOAD=./libtool.so LD_LIBRARY_PATH=$$LD_LIBRARY_PATH:$(TYPEART_LIBPATH) $(MPIRUN) -np 1 01_ex run02: 02_ex libtool.so - env LD_PRELOAD=./libtool.so LD_LIBRARY_PATH=$$LD_LIBRARY_PATH:$(TYPEART_RUNTIME_DIR) $(MPIRUN) -np 1 02_ex -run03: 03_ex libtool.so - env LD_PRELOAD=./libtool.so LD_LIBRARY_PATH=$$LD_LIBRARY_PATH:$(TYPEART_RUNTIME_DIR) $(MPIRUN) -np 4 03_ex - + env LD_PRELOAD=./libtool.so LD_LIBRARY_PATH=$$LD_LIBRARY_PATH:$(TYPEART_LIBPATH) $(MPIRUN) -np 1 02_ex clean: - rm *~ libtool.so 01_ex 02_ex types.yaml + rm *.o libtool.so 01_ex 02_ex toy toy-stack types.yaml diff --git a/demo/applyPass.sh b/demo/applyPass.sh deleted file mode 100755 index 59804bed..00000000 --- a/demo/applyPass.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -target=$1 -pathToPlugin=${2:-../build/lib} -pathToRT=${3:-../build/runtime} -mpi=${4:-1} -outfile=${5} - -tmpDir=/tmp -tmpfile="$tmpDir"/"${target##*/}" -extension="${target##*.}" - -rtDir="$( cd "$pathToRT" && pwd )" - -echo -e Running on "$target" using plugin: "$plugin" - -c_compiler=clang -cxx_compiler=clang++ - -c_compiler_wrapper=$c_compiler -cxx_compiler_wrapper=$cxx_compiler - -if [ $mpi == 1 ]; then - c_compiler_wrapper=mpicc - cxx_compiler_wrapper=mpic++ -fi - -if [ $extension == "c" ]; then - compiler=$c_compiler - compiler_wrapper=$c_compiler_wrapper -else - compiler=$cxx_compiler - compiler_wrapper=$cxx_compiler_wrapper -fi - -if [ -e "types.yaml" ]; then - rm "types.yaml" -fi - -OMPI_CC=$c_compiler OMPI_CXX=$cxx_compiler $compiler_wrapper -S -emit-llvm "$target" -o "$tmpfile".ll -opt -load "${pathToPlugin}/analysis/meminstfinderpass.so" -load "${pathToPlugin}/typeartpass.so" -typeart -typeart-alloca < "$tmpfile".ll -S -o "$tmpfile".ll -llc "$tmpfile".ll -o "$tmpfile".s -OMPI_CC=$c_compiler OMPI_CXX=$cxx_compiler $compiler_wrapper "$tmpfile".s -L"$pathToRT" -ltypeart -o "$outfile" diff --git a/lib/mpi_interceptor/CMakeLists.txt b/lib/mpi_interceptor/CMakeLists.txt index 768517ae..89e3ae50 100644 --- a/lib/mpi_interceptor/CMakeLists.txt +++ b/lib/mpi_interceptor/CMakeLists.txt @@ -39,8 +39,7 @@ if(${MPI_C_FOUND} AND ${PYTHONINTERP_FOUND}) ) if(ENABLE_TSAN) - target_compile_options(interceptor-rt PRIVATE -fsanitize=thread) - target_link_options(interceptor-rt PRIVATE -fsanitize=thread) + target_tsan_options(typeart-rt) endif() target_link_libraries(interceptor-rt