diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..1377554e --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.swp diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..ba4535ce --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,24 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/sphinx/conf.py + +# Build documentation with MkDocs +#mkdocs: +# configuration: mkdocs.yml + +# Optionally build your docs in additional formats such as PDF and ePub +formats: all + +# Optionally set the version of Python and requirements required to build your docs +python: + version: 3.7 + install: + - requirements: docs/sphinx/requirements.txt + diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..a08a9d9b --- /dev/null +++ b/.travis.yml @@ -0,0 +1,63 @@ +services: docker +dist: trusty +language: cpp +env: + global: + - DO_BUILD=yes + - DO_TEST=no + - GTEST_COLOR=1 +matrix: + include: + - compiler: gcc5 + env: + - COMPILER=g++ + - IMG=gcc-5 + - CMAKE_EXTRA_FLAGS="-DENABLE_DEBUG_LOGGING=Off" + - DO_MEMCHECK=no + - compiler: gcc6 + env: + - COMPILER=g++ + - IMG=gcc-6 + - CMAKE_EXTRA_FLAGS="-DENABLE_DEBUG_LOGGING=Off" + - compiler: gcc7 + env: + - COMPILER=g++ + - IMG=gcc-7 + - CMAKE_EXTRA_FLAGS="-DENABLE_DEBUG_LOGGING=Off" + - compiler: gcc8 + env: + - COMPILER=g++ + - IMG=gcc-8 + - CMAKE_EXTRA_FLAGS="-DENABLE_DEBUG_LOGGING=Off" + - compiler: gcc8 + env: + - COMPILER=g++ + - IMG=gcc-8 + - CMAKE_EXTRA_FLAGS="-DENABLE_DEBUG_LOGGING=On" + - compiler: clang4 + env: + - COMPILER=clang++ + - IMG=clang-4 + - CMAKE_EXTRA_FLAGS="-DENABLE_DEBUG_LOGGING=Off" + - compiler: clang5 + env: + - COMPILER=clang++ + - IMG=clang-5 + - CMAKE_EXTRA_FLAGS="-DENABLE_DEBUG_LOGGING=Off" + - compiler: clang6 + env: + - COMPILER=clang++ + - IMG=clang-6 + - CMAKE_EXTRA_FLAGS="-DENABLE_DEBUG_LOGGING=Off" + - compiler: clang6 + env: + - COMPILER=clang++ + - IMG=clang-6 + - CMAKE_EXTRA_FLAGS="-DENABLE_DEBUG_LOGGING=On" + +script: +- docker run --rm --user='root' -v ${TRAVIS_BUILD_DIR}:/home/axom axom/compilers:$IMG chown -R axom /home/axom +- docker run --rm -v ${TRAVIS_BUILD_DIR}:/home/axom -e COMPILER -e DO_BUILD -e DO_TEST -e DO_MEMCHECK -e CMAKE_EXTRA_FLAGS -e GTEST_COLOR axom/compilers:$IMG ./scripts/travis/build_and_test.sh + +after_success: +- if [[ "${CMAKE_EXTRA_FLAGS}" == *"ENABLE_COVERAGE"* ]] ; then bash <(curl -s https://codecov.io/bash) -a "-f" >& /dev/null; fi diff --git a/CMakeLists.txt b/CMakeLists.txt index 5739c883..06fe172b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ ############################################################################# cmake_minimum_required (VERSION 3.5.1) project(umap - VERSION 1.0.0 + VERSION 2.0.0 LANGUAGES CXX C ) @@ -20,14 +20,9 @@ include(cmake/SetupUmapThirdParty.cmake) set(UMAP_DEBUG_LOGGING ${ENABLE_LOGGING}) configure_file( ${PROJECT_SOURCE_DIR}/config/config.h.in - ${PROJECT_BINARY_DIR}/src/include/config.h) + ${PROJECT_BINARY_DIR}/src/umap/config.h) -set (UMAPINCLUDEDIRS - "${CMAKE_BINARY_DIR}/src/include" - "${CMAKE_SOURCE_DIR}/src/include" - "${CMAKE_SOURCE_DIR}/src/logging" - "${CMAKE_SOURCE_DIR}/src/store" - ) +include_directories ( "${CMAKE_SOURCE_DIR}/src" "${CMAKE_BINARY_DIR}/src") add_subdirectory(src) add_subdirectory(examples) diff --git a/README.md b/README.md index 41519b92..b8380bac 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ -# UMAP v1.0.0 +# UMAP v2.0.0 +[![Travis Build Status](https://travis-ci.com/LLNL/umap.svg?branch=develop)](https://travis-ci.com/LLNL/umap) [![Documentation Status](https://readthedocs.org/projects/llnl-umap/badge/?version=develop)](https://llnl-umap.readthedocs.io/en/develop/?badge=develop) Umap is a library that provides an mmap()-like interface to a simple, user- diff --git a/docs/sphinx/index.rst b/docs/sphinx/index.rst index e1bd9b0f..e4f253d8 100644 --- a/docs/sphinx/index.rst +++ b/docs/sphinx/index.rst @@ -1,6 +1,6 @@ -****** -UMAP -****** +*********** +UMAP v2.0.0 +*********** Umap is a library that provides an mmap()-like interface to a simple, user- space page fault handler based on the userfaultfd Linux feature (starting with diff --git a/examples/psort.cpp b/examples/psort.cpp index fab79347..554ca5c2 100644 --- a/examples/psort.cpp +++ b/examples/psort.cpp @@ -16,25 +16,25 @@ #include #include #include +#include #include "errno.h" #include "umap/umap.h" -using namespace std; - -void initialize_and_sort_file( const char* fname, uint64_t arraysize, uint64_t totalbytes ) +void +initialize_and_sort_file( const char* fname, uint64_t arraysize, uint64_t totalbytes, uint64_t psize ) { if ( unlink(fname) ) { int eno = errno; if ( eno != ENOENT ) { - cerr << "Failed to unlink " << fname << ": " - << strerror(eno) << " Errno=" << eno << endl; + std::cerr << "Failed to unlink " << fname << ": " + << strerror(eno) << " Errno=" << eno << std::endl; } } int fd = open(fname, O_RDWR | O_LARGEFILE | O_DIRECT | O_CREAT, S_IRUSR | S_IWUSR); if ( fd == -1 ) { int eno = errno; - cerr << "Failed to create " << fname << ": " << strerror(eno) << endl; + std::cerr << "Failed to create " << fname << ": " << strerror(eno) << std::endl; return; } @@ -43,7 +43,7 @@ void initialize_and_sort_file( const char* fname, uint64_t arraysize, uint64_t t int x; if ( ( x = posix_fallocate(fd, 0, totalbytes) != 0 ) ) { int eno = errno; - cerr << "Failed to pre-allocate " << fname << ": " << strerror(eno) << endl; + std::cerr << "Failed to pre-allocate " << fname << ": " << strerror(eno) << std::endl; return; } } catch(const std::exception& e) { @@ -51,74 +51,90 @@ void initialize_and_sort_file( const char* fname, uint64_t arraysize, uint64_t t return; } catch(...) { int eno = errno; - cerr << "Failed to pre-allocate " << fname << ": " << strerror(eno) << endl; + std::cerr << "Failed to pre-allocate " << fname << ": " << strerror(eno) << std::endl; return; } void* base_addr = umap(NULL, totalbytes, PROT_READ|PROT_WRITE, UMAP_PRIVATE, fd, 0); if ( base_addr == UMAP_FAILED ) { int eno = errno; - cerr << "Failed to umap " << fname << ": " << strerror(eno) << endl; + std::cerr << "Failed to umap " << fname << ": " << strerror(eno) << std::endl; return; } + std::vector pfi; + char* base = (char*)base_addr; + uint64_t PagesInTest = totalbytes / psize; + + std::cout << "Prefetching Pages\n"; + for ( int i{0}; i < PagesInTest; ++i) { + umap_prefetch_item x = { .page_base_addr = &base[i * psize] }; + pfi.push_back(x); + }; + umap_prefetch(PagesInTest, &pfi[0]); + uint64_t *arr = (uint64_t *) base_addr; - cout << "Initializing Array\n"; + + std::cout << "Initializing Array\n"; #pragma omp parallel for for(uint64_t i=0; i < arraysize; ++i) arr[i] = (uint64_t) (arraysize - i); - cout << "Sorting Data\n"; + std::cout << "Sorting Data\n"; __gnu_parallel::sort(arr, &arr[arraysize], std::less(), __gnu_parallel::quicksort_tag()); + if (uunmap(base_addr, totalbytes) < 0) { int eno = errno; - cerr << "Failed to uumap " << fname << ": " << strerror(eno) << endl; + std::cerr << "Failed to uumap " << fname << ": " << strerror(eno) << std::endl; return; } close(fd); } -void verify_sortfile( const char* fname, uint64_t arraysize, uint64_t totalbytes ) +void +verify_sortfile( const char* fname, uint64_t arraysize, uint64_t totalbytes ) { int fd = open(fname, O_RDWR | O_LARGEFILE | O_DIRECT, S_IRUSR | S_IWUSR); if ( fd == -1 ) { - cerr << "Failed to create " << fname << endl; + std::cerr << "Failed to create " << fname << std::endl; return; } void* base_addr = umap(NULL, totalbytes, PROT_READ|PROT_WRITE, UMAP_PRIVATE, fd, 0); if ( base_addr == UMAP_FAILED ) { - cerr << "umap failed\n"; + std::cerr << "umap failed\n"; return; } uint64_t *arr = (uint64_t *) base_addr; - cout << "Verifying Data with\n"; + std::cout << "Verifying Data with\n"; #pragma omp parallel for for(uint64_t i = 0; i < arraysize; ++i) if (arr[i] != (i+1)) { - cerr << "Data miscompare\n"; + std::cerr << "Data miscompare\n"; i = arraysize; } if (uunmap(base_addr, totalbytes) < 0) { - cerr << "uunamp failed\n"; + std::cerr << "uunamp failed\n"; return; } close(fd); } -int main(int argc, char **argv) +int +main(int argc, char **argv) { const char* filename = argv[1]; // Optional: Make umap's pages size double the default system page size // - uint64_t psize = umap_cfg_get_pagesize() * 2; - umap_cfg_set_pagesize( psize ); + // Use UMAP_PAGE_SIZE environment variable to set page size for umap + // + uint64_t psize = umapcfg_get_umap_page_size(); const uint64_t pagesInTest = 64; const uint64_t elemPerPage = psize / sizeof(uint64_t); @@ -129,9 +145,9 @@ int main(int argc, char **argv) // Optional: Set umap's buffer to half the number of pages we need so that // we may simulate an out-of-core experience // - umap_cfg_set_bufsize( pagesInTest / 2 ); - - initialize_and_sort_file(filename, arraySize, totalBytes); + // Use UMAP_BUFSIZE environment variable to set number of pages in buffer + // + initialize_and_sort_file(filename, arraySize, totalBytes, psize); verify_sortfile(filename, arraySize, totalBytes); return 0; } diff --git a/scripts/travis/build_and_test.sh b/scripts/travis/build_and_test.sh new file mode 100755 index 00000000..86ee1ffa --- /dev/null +++ b/scripts/travis/build_and_test.sh @@ -0,0 +1,36 @@ +#!/bin/bash +############################################################################# +# Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +# UMAP Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: LGPL-2.1-only +############################################################################# + +env +function or_die () { + "$@" + local status=$? + if [[ $status != 0 ]] ; then + echo ERROR $status command: $@ + exit $status + fi +} + +or_die mkdir travis-build +cd travis-build +if [[ "$DO_BUILD" == "yes" ]] ; then + or_die cmake -DCMAKE_CXX_COMPILER="${COMPILER}" ${CMAKE_EXTRA_FLAGS} ../ + if [[ ${CMAKE_EXTRA_FLAGS} == *COVERAGE* ]] ; then + or_die make -j 3 + else + or_die make -j 3 VERBOSE=1 + fi + if [[ "${DO_TEST}" == "yes" ]] ; then + or_die ctest -T test --output-on-failure -V + fi + if [[ "${DO_MEMCHECK}" == "yes" ]] ; then + or_die ctest -T memcheck + fi +fi + +exit 0 diff --git a/src/include/umap/umap.h b/src/include/umap/umap.h deleted file mode 100644 index 8cd5e5ae..00000000 --- a/src/include/umap/umap.h +++ /dev/null @@ -1,105 +0,0 @@ -////////////////////////////////////////////////////////////////////////////// -// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other -// UMAP Project Developers. See the top-level LICENSE file for details. -// -// SPDX-License-Identifier: LGPL-2.1-only -////////////////////////////////////////////////////////////////////////////// -#ifndef _UMAP_H_ -#define _UMAP_H_ - -#ifdef __cplusplus - #include - #include "umap/Store.h" - #include - #include -#else // __cplusplus - #include - #include - #include -#endif // __cplusplus - - -#ifdef __cplusplus -/** Allow application to create region of memory to a peristant store - * \param addr Same as input argument for mmap(2) - * \param length Same as input argument of mmap(2) - * \param prot Same as input argument of mmap(2) - * \param flags Same as input argument of mmap(2) - * \param r_pstore pointer to callback function to be used for providing data from - * persistent storage. - * \param w_pstore pointer to callback function to be used for saving data to - * persistent storage. - */ -void* umap_ex( - void* addr, - std::size_t length, - int prot, - int flags, - int fd, - off_t offset, - Store* -); -#endif // __cplusplus - -#ifdef __cplusplus -extern "C" { -#endif -/** Allow application to create region of memory to a peristant store - * \param addr Same as input argument for mmap(2) - * \param length Same as input argument of mmap(2) - * \param prot Same as input argument of mmap(2) - * \param flags Same as input argument of mmap(2) - * \param r_pstore pointer to callback function to be used for providing data from - * persistent storage. - * \param w_pstore pointer to callback function to be used for saving data to - * persistent storage. - */ -void* umap( - void* addr, - size_t length, - int prot, /* See mmap(2) - Subset supported, rest ignored */ - int flags, /* See mmap(2) - Subset supported, rest ignored */ - int fd, /* See mmap(2) */ - off_t offset /* See mmap(2) - umap ignores this */ -); - -int uunmap( void* addr, /* See mmap(2) */ - size_t length /* See mmap(2) */ - ); - -uint64_t* umap_cfg_readenv(const char* env, uint64_t* val); -void umap_cfg_getenv( void ); -uint64_t umap_cfg_get_bufsize( void ); -void umap_cfg_set_bufsize( uint64_t page_bufsize ); -uint64_t umap_cfg_get_uffdthreads( void ); -void umap_cfg_set_uffdthreads( uint64_t numthreads ); -void umap_cfg_flush_buffer( void* region ); -int umap_cfg_get_pagesize( void ); -int umap_cfg_set_pagesize( long psize ); - -struct umap_cfg_stats { - uint64_t dirty_evicts; - uint64_t evict_victims; - uint64_t wp_messages; - uint64_t read_faults; - uint64_t write_faults; -}; - -void umap_cfg_get_stats(void* region, struct umap_cfg_stats* stats); -void umap_cfg_reset_stats(void* region); - -#ifdef __cplusplus -} -#endif - -/* - * flags - */ -#define UMAP_PRIVATE MAP_PRIVATE // Note - UMAP_SHARED not currently supported -#define UMAP_FIXED MAP_FIXED // See mmap(2) - This flag is currently then only flag supported. - -/* - * Return codes - */ -#define UMAP_FAILED (void *)-1 -#endif // _UMAP_H diff --git a/src/logging/spindle_debug.h b/src/logging/spindle_debug.h deleted file mode 100644 index 0de07bbf..00000000 --- a/src/logging/spindle_debug.h +++ /dev/null @@ -1,44 +0,0 @@ -////////////////////////////////////////////////////////////////////////////// -// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other -// UMAP Project Developers. See the top-level LICENSE file for details. -// -// SPDX-License-Identifier: LGPL-2.1-only -////////////////////////////////////////////////////////////////////////////// - -#if !defined(UMAP_SPINDLE_DEBUG_H_) -#define UMAP_SPINDLE_DEBUG_H_ - -#include -#include -#include "config.h" - -#if defined(UMAP_DEBUG_LOGGING) - -#if defined(__cplusplus) -extern "C" { -#endif -#include "spindle_logc.h" -#if defined(__cplusplus) -} -#endif - -#define LOGGING_INIT init_spindle_debugging(0) -#define LOGGING_INIT_PREEXEC init_spindle_debugging(1) -#define LOGGING_FINI fini_spindle_debugging() - -#else -#define LOGGING_INIT -#define LOGGING_INIT_PREEXEC -#define LOGGING_FINI -#define debug_printf(format, ...) -#define debug_printf2(S, ...) debug_printf(S, ## __VA_ARGS__) -#define debug_printf3(S, ...) debug_printf(S, ## __VA_ARGS__) - -#define bare_printf(S, ...) debug_printf(S, ## __VA_ARGS__) -#define bare_printf2(S, ...) debug_printf(S, ## __VA_ARGS__) -#define bare_printf3(S, ...) debug_printf(S, ## __VA_ARGS__) - -#define err_printf(S, ...) debug_printf(S, ## __VA_ARGS__) -#endif - -#endif diff --git a/src/logging/spindle_logc.c b/src/logging/spindle_logc.c deleted file mode 100644 index 2a5b4ea3..00000000 --- a/src/logging/spindle_logc.c +++ /dev/null @@ -1,395 +0,0 @@ -////////////////////////////////////////////////////////////////////////////// -// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other -// UMAP Project Developers. See the top-level LICENSE file for details. -// -// SPDX-License-Identifier: LGPL-2.1-only -////////////////////////////////////////////////////////////////////////////// - -#define _GNU_SOURCE - -#include "config.h" - -#if defined(UMAP_DEBUG_LOGGING) -#include "spindle_debug.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -static char spindle_log_daemon_name[] = "../libexec/umap_logd"; -static char spindle_log_daemon_name2[] = "../../src/umap/umap_logd"; - -static int debug_fd = -1; -static char *tempdir; -static int run_local_only = 1; // Don't use sockets -static char *debug_location; - -FILE *spindle_debug_output_f; -char *spindle_debug_name = "umap"; -int spindle_debug_prints; - -//Timeout in tenths of a second -#define SPAWN_TIMEOUT 300 -#define CONNECT_TIMEOUT 100 - -extern int spindle_mkdir(char *orig_path); - -int fileExists(char *name) -{ - struct stat buf; - return (stat(name, &buf) != -1); -} - -#include -#define MAX_EXE_PATH_STR_SIZE 4096 + 1 -static void getProgramAndPath( char** fpath, char** ppath, char** pname) -{ - static char* fullPath = NULL; - static char* pathPrefix = NULL; - static char* programName = NULL; - char* p, tmp; - ssize_t r; - - if ( fullPath == NULL ) { - fullPath = malloc(MAX_EXE_PATH_STR_SIZE); - if ( fullPath == NULL ) { - fprintf(stderr, "Insufficient memory: %s\n", strerror(errno)); - exit(0); - } - - r = readlink("/proc/self/exe", fullPath, MAX_EXE_PATH_STR_SIZE); - - if ( r == -1 ) { - fprintf(stderr, "readlink failed: %s\n", strerror(errno)); - exit(0); - } - - fullPath[r] = '\0'; - p = strrchr(fullPath, '/'); tmp = *p; *p = '\0'; - pathPrefix = strdup(fullPath); - programName = p+1; - *p = tmp; - } - if (fpath != NULL) *fpath = fullPath; - if (ppath != NULL) *ppath = pathPrefix; - if (pname != NULL) *pname = programName; -} - -/* - * There are two possible places for where the logging daemon will exist. - * Normally, the logging daemon will be in the ../libexec directory of the - * place where umap is installed/deployed. For developers, the other place - * is in the build directory relative to where running umap program is being - * run. - * - * This function will first attempt to find the executable in the installation - * location. If it does not find the file there, it will then check the - * directory relative to where the program being run was built. - * - * If neither are found, this function will print an error and will cause the - * forked daemon to just exit and no logging will be performed. - */ -void spawnLogDaemon(char *tempdir) -{ - int result = fork(); - - if (result == 0) { - result = fork(); - if (result == 0) { - char *params[7]; - int cur = 0; - char* path_prefix; - char* pname_pri; - char* pname_alt; - char* pname; - struct stat sbuf; - - getProgramAndPath( NULL, &path_prefix, NULL); - - pname_pri = malloc(strlen(path_prefix) + strlen(spindle_log_daemon_name) + 1); - if ( pname_pri == NULL ) { - fprintf(stderr, "Insufficient memory: %s\n", strerror(errno)); - exit(0); - } - pname_pri[0] = '\0'; - sprintf(pname_pri, "%s/%s", path_prefix, spindle_log_daemon_name); - - pname_alt = malloc(strlen(path_prefix) + strlen(spindle_log_daemon_name2) + 1); - if ( pname_alt == NULL ) { - fprintf(stderr, "Insufficient memory: %s\n", strerror(errno)); - exit(0); - } - pname_alt[0] = '\0'; - sprintf(pname_alt, "%s/%s", path_prefix, spindle_log_daemon_name2); - - pname = pname_pri; - if ( stat(pname_pri, &sbuf) < 0 ) { - if ( stat(pname_alt, &sbuf) == 0 ) { - pname = pname_alt; - } - } - - params[cur++] = pname; - params[cur++] = tempdir; - if (spindle_debug_prints) { - params[cur++] = "-debug"; - params[cur++] = "umap_output"; - } - params[cur++] = NULL; - - execv(pname, params); - fprintf(stderr, "Error executing %s: %s\n", pname, strerror(errno)); - exit(0); - } - else { - exit(0); - } - } - else - { - int status; - do { - waitpid(result, &status, 0); - } while (!WIFEXITED(status)); - } -} - -int clearDaemon(char *tmpdir) -{ - int fd; - char reset_buffer[512]; - char lock_buffer[512]; - char log_buffer[512]; - int pid; - - /* Only one process can reset the daemon */ - snprintf(reset_buffer, 512, "%s/umap_log_reset", tmpdir); - fd = open(reset_buffer, O_WRONLY | O_CREAT | O_EXCL, 0600); - if (fd == -1) - return 0; - close(fd); - - snprintf(lock_buffer, 512, "%s/umap_log_lock", tmpdir); - snprintf(log_buffer, 512, "%s/umap_log", tmpdir); - - fd = open(lock_buffer, O_RDONLY); - if (fd != -1) { - char pids[32], *cur = pids; - while (read(fd, cur++, 1) == 1 && (cur - pids) < 32); - *cur = '\0'; - pid = atoi(pids); - if (pid && kill(pid, 0) != -1) { - /* The process exists, someone else likely re-created it */ - return 0; - } - } - - unlink(log_buffer); - unlink(lock_buffer); - unlink(reset_buffer); - - return 1; -} - -int connectToLogDaemon(char *path) -{ - int result, pathsize, sockfd; - struct sockaddr_un saddr; - - sockfd = socket(AF_UNIX, SOCK_STREAM, 0); - if (sockfd == -1) - return -1; - - bzero(&saddr, sizeof(saddr)); - pathsize = sizeof(saddr.sun_path); - saddr.sun_family = AF_UNIX; - strncpy(saddr.sun_path, path, pathsize-1); - - int timeout = 0; - for (;;) { - result = connect(sockfd, (struct sockaddr *) &saddr, sizeof(struct sockaddr_un)); - if (result == -1 && (errno == ECONNREFUSED || errno == ENOENT)) { - timeout++; - if (timeout == CONNECT_TIMEOUT) - return -1; - usleep(100000); /* .1 seconds */ - } - else if (result == -1) { - fprintf(stderr, "Error connecting: %s\n", strerror(errno)); - return -1; - } - else { - break; - } - } - - return sockfd; -} - -static void setConnectionSurvival(int fd, int survive_exec) -{ - if (fd == -1) - return; - - if (!survive_exec) { - int fdflags = fcntl(fd, F_GETFD, 0); - if (fdflags < 0) - fdflags = 0; - fcntl(fd, F_SETFD, fdflags | O_CLOEXEC); - unsetenv("UMAP_LOGGING_SOCKET"); - } - else { - int fdflags = fcntl(fd, F_GETFD, 0); - if (fdflags < 0) - fdflags = 0; - fcntl(fd, F_SETFD, fdflags & ~O_CLOEXEC); - char fd_str[32]; - snprintf(fd_str, 32, "%d", debug_fd); - setenv("UMAP_LOGGING_SOCKET", fd_str, 1); - } -} - -static int setup_connection(char *connection_name) -{ - char *socket_file; - int socket_file_len; - int fd, result; - - socket_file_len = strlen(tempdir) + strlen(connection_name) + 2; - socket_file = (char *) malloc(socket_file_len); - snprintf(socket_file, socket_file_len, "%s/%s", tempdir, connection_name); - - int tries = 5; - for (;;) { - /* If the daemon doesn't exist, create it and wait for its existance */ - if (!fileExists(socket_file)) { - spawnLogDaemon(tempdir); - - int timeout = 0; - while (!fileExists(socket_file) && timeout < SPAWN_TIMEOUT) { - usleep(100000); /* .1 seconds */ - timeout++; - } - - if (timeout == SPAWN_TIMEOUT) { - free(socket_file); - return -1; - } - } - - /* Establish connection to daemon */ - fd = connectToLogDaemon(socket_file); - if (fd != -1) - break; - - /* Handle failed connection. */ - if (--tries == 0) - break; - - result = clearDaemon(tempdir); - if (!result) { - /* Give the process clearing the daemon a chance to finish, then - try again */ - sleep(1); - } - } - free(socket_file); - return fd; -} - -void reset_spindle_debugging() -{ - spindle_debug_prints = 0; - init_spindle_debugging(0); -} - -void init_spindle_debugging(int survive_exec) -{ - char *already_setup, *log_level_str; - int log_level = 0; - - log_level_str = getenv("UMAP_LOGGING"); - if (log_level_str) - log_level = atoi(log_level_str); - spindle_debug_prints = log_level; - if (!log_level) - return; - - if (run_local_only) { - spindle_debug_output_f = stdout; - return; - } - - getProgramAndPath( NULL, NULL, &spindle_debug_name); - - if (spindle_debug_prints) - return; - - /* Setup locations for temp and output files */ - tempdir = getenv("TMPDIR"); - if (!tempdir) - tempdir = getenv("TEMPDIR"); - if (!tempdir || !*tempdir) - tempdir = "/tmp"; - if (!fileExists(tempdir)) { - spindle_mkdir(tempdir); - } - - debug_location = log_level ? "./umap_output" : NULL; - - already_setup = getenv("UMAP_LOGGING_SOCKET"); - if (already_setup) { - sscanf(already_setup, "%d", &debug_fd); - } - else { - if (log_level) - debug_fd = setup_connection("umap_log"); - } - - setConnectionSurvival(debug_fd, survive_exec); - - /* Setup the variables */ - if (debug_fd != -1) - spindle_debug_output_f = fdopen(debug_fd, "w"); -} - -void spindle_dump_on_error() -{ - void *stacktrace[256]; - char **syms; - int size, i; - - size = backtrace(stacktrace, 256); - if (size <= 0) - return; - syms = backtrace_symbols(stacktrace, size); - - for (i = 0; i"); - } - - if (syms) - free(syms); -} - -void fini_spindle_debugging() -{ - static unsigned char exitcode[8] = { 0x01, 0xff, 0x03, 0xdf, 0x05, 0xbf, 0x07, '\n' }; - if (debug_fd != -1) - write(debug_fd, &exitcode, sizeof(exitcode)); -} - -int is_debug_fd(int fd) -{ - return (fd == debug_fd); -} -#endif diff --git a/src/logging/spindle_logc.h b/src/logging/spindle_logc.h deleted file mode 100644 index 84420d35..00000000 --- a/src/logging/spindle_logc.h +++ /dev/null @@ -1,92 +0,0 @@ -////////////////////////////////////////////////////////////////////////////// -// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other -// UMAP Project Developers. See the top-level LICENSE file for details. -// -// SPDX-License-Identifier: LGPL-2.1-only -////////////////////////////////////////////////////////////////////////////// - -#if !defined(UMAP_SPINDLE_LOGC_H_) -#define UMAP_SPINDLE_LOGC_H_ - -#include -#include - -extern int spindle_debug_prints; -extern char *spindle_debug_name; -extern FILE *spindle_debug_output_f; - -extern void spindle_dump_on_error(); - -#define BASE_FILE (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/')+1 : __FILE__) - -#define debug_printf(format, ...) \ - do { \ - if (spindle_debug_prints && spindle_debug_output_f) { \ - fprintf(spindle_debug_output_f, "[%s.%d@%s:%u] %s - " format, \ - spindle_debug_name, getpid(), \ - BASE_FILE, __LINE__, __func__, ## __VA_ARGS__); \ - fflush(spindle_debug_output_f); \ - } \ - } while (0) - -#define debug_printf2(format, ...) \ - do { \ - if (spindle_debug_prints > 1 && spindle_debug_output_f) { \ - fprintf(spindle_debug_output_f, "[%s.%d@%s:%u] %s - " format, \ - spindle_debug_name, getpid(), \ - BASE_FILE, __LINE__, __func__, ## __VA_ARGS__); \ - fflush(spindle_debug_output_f); \ - } \ - } while (0) - -#define debug_printf3(format, ...) \ - do { \ - if (spindle_debug_prints > 2 && spindle_debug_output_f) { \ - fprintf(spindle_debug_output_f, "[%s.%d@%s:%u] %s - " format, \ - spindle_debug_name, getpid(), \ - BASE_FILE, __LINE__, __func__, ## __VA_ARGS__); \ - fflush(spindle_debug_output_f); \ - } \ - } while (0) - -#define bare_printf(format, ...) \ - do { \ - if (spindle_debug_prints && spindle_debug_output_f) { \ - fprintf(spindle_debug_output_f, format, ## __VA_ARGS__); \ - fflush(spindle_debug_output_f); \ - } \ - } while (0) - -#define bare_printf2(format, ...) \ - do { \ - if (spindle_debug_prints > 1 && spindle_debug_output_f) { \ - fprintf(spindle_debug_output_f, format, ## __VA_ARGS__); \ - fflush(spindle_debug_output_f); \ - } \ - } while (0) - -#define bare_printf3(format, ...) \ - do { \ - if (spindle_debug_prints > 2 && spindle_debug_output_f) { \ - fprintf(spindle_debug_output_f, format, ## __VA_ARGS__); \ - fflush(spindle_debug_output_f); \ - } \ - } while (0) - -#define err_printf(format, ...) \ - do { \ - if (spindle_debug_prints && spindle_debug_output_f) { \ - fprintf(spindle_debug_output_f, "[%s.%d@%s:%u] - ERROR: " \ - format, spindle_debug_name, getpid(), \ - BASE_FILE, __LINE__, ## __VA_ARGS__); \ - spindle_dump_on_error(); \ - fflush(spindle_debug_output_f); \ - } \ - } while (0) - -void init_spindle_debugging(int survive_exec); -void fini_spindle_debugging(); -void reset_spindle_debugging(); -int is_debug_fd(int fd); - -#endif diff --git a/src/logging/spindle_logd.cpp b/src/logging/spindle_logd.cpp deleted file mode 100644 index e0c8a49b..00000000 --- a/src/logging/spindle_logd.cpp +++ /dev/null @@ -1,508 +0,0 @@ -////////////////////////////////////////////////////////////////////////////// -// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other -// UMAP Project Developers. See the top-level LICENSE file for details. -// -// SPDX-License-Identifier: LGPL-2.1-only -////////////////////////////////////////////////////////////////////////////// - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -//Seconds to live without a child -#define TIMEOUT 10 - -std::string tmpdir; -std::string debug_fname; - -void clean(); -void cleanFiles(); - -class UniqueProcess; -class OutputLog; -class MsgReader; - -UniqueProcess *lockProcess; -OutputLog *debug_log; -MsgReader *debug_reader; - -bool runDebug = false; - -static unsigned char exitcode[8] = { 0x01, 0xff, 0x03, 0xdf, 0x05, 0xbf, 0x07, '\n' }; - -class UniqueProcess -{ -private: - int fd; - std::string logFileLock; - bool unique; -public: - UniqueProcess() - { - unique = false; - logFileLock = tmpdir + std::string("/umap_log_lock"); - fd = open(logFileLock.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600); - if (fd != -1) { - char pid_str[32]; - snprintf(pid_str, 32, "%d", getpid()); - write(fd, pid_str, strlen(pid_str)); - unique = true; - return; - } - if (errno == EEXIST) - return; - fprintf(stderr, "Error creating lock file %s: %s\n", logFileLock.c_str(), strerror(errno)); - } - - ~UniqueProcess() - { - if (fd < 0) - return; - close(fd); - unlink(logFileLock.c_str()); - } - - void cleanFile() { - if (fd < 0) - return; - close(fd); - unlink(logFileLock.c_str()); - fd = -1; - } - - bool isUnique() const - { - return unique; - } -}; - -class OutputInterface -{ -public: - OutputInterface() - { - } - - virtual ~OutputInterface() - { - } - - bool isExitCode(const char *msg1, int msg1_size, const char *msg2, int msg2_size) - { - if (msg1[0] != (char) exitcode[0]) - return false; - if (msg1_size + msg2_size != 8) - return false; - - char code[8]; - memset(code, 0, sizeof(code)); - int i=0; - for (i=0; i conns; - char recv_buffer[MAX_MESSAGE]; - size_t recv_buffer_size, named_buffer_size; - bool error; - std::string socket_path; - pthread_t thrd; - OutputInterface *log; - - bool addNewConnection() { - Connection *con = new Connection(); - socklen_t remote_addr_size = sizeof(struct sockaddr_un); - con->fd = accept(sockfd, (struct sockaddr *) &con->remote_addr, &remote_addr_size); - con->shutdown = false; - if (con->fd == -1) { - fprintf(stderr, "[%s:%u] - Error adding connection: %s\n", __FILE__, __LINE__, strerror(errno)); - delete con; - return false; - } - - int flags = fcntl(con->fd, F_GETFL, 0); - if (flags == -1) flags = 0; - fcntl(con->fd, F_SETFL, flags | O_NONBLOCK); - - con->unfinished_msg[0] = '\0'; - conns.insert(std::make_pair(con->fd, con)); - return true; - } - - bool waitAndHandleMessage() { - fd_set rset; - - for (;;) { - FD_ZERO(&rset); - int max_fd = 0; - if (sockfd != -1) { - FD_SET(sockfd, &rset); - max_fd = sockfd; - } - - for (std::map::iterator i = conns.begin(); i != conns.end(); i++) { - int fd = i->first; - FD_SET(fd, &rset); - if (fd > max_fd) - max_fd = fd; - } - - struct timeval timeout; - timeout.tv_sec = TIMEOUT; - timeout.tv_usec = 0; - - if (!max_fd) { - return false; - } - - int result = select(max_fd+1, &rset, NULL, NULL, conns.empty() ? &timeout : NULL); - if (result == 0) { - return false; - } - if (result == -1) { - fprintf(stderr, "[%s:%u] - Error calling select: %s\n", __FILE__, __LINE__, strerror(errno)); - return false; - } - - if (sockfd != -1 && FD_ISSET(sockfd, &rset)) { - addNewConnection(); - } - - for (std::map::iterator i = conns.begin(); i != conns.end(); i++) { - int fd = i->first; - if (FD_ISSET(fd, &rset)) { - readMessage(i->second); - } - } - - bool foundShutdownProc; - do { - foundShutdownProc = false; - for (std::map::iterator i = conns.begin(); i != conns.end(); i++) { - if (i->second->shutdown) { - conns.erase(i); - foundShutdownProc = true; - break; - } - } - } while (foundShutdownProc); - } - } - - bool readMessage(Connection *con) - { - int result = recv(con->fd, recv_buffer, MAX_MESSAGE, 0); - if (result == -1) { - fprintf(stderr, "[%s:%u] - Error calling recv: %s\n", __FILE__, __LINE__, strerror(errno)); - close(con->fd); - return false; - } - - if (result == 0) { - //A client shutdown - std::map::iterator i = conns.find(con->fd); - assert(i != conns.end()); - i->second->shutdown = true; - if (con->unfinished_msg[0] != '\0') - processMessage(con, "\n", 1); - close(con->fd); - return true; - } - - return processMessage(con, recv_buffer, result); - } - - bool processMessage(Connection *con, const char *msg, int msg_size) { - int msg_begin = 0; - for (int i = 0; i < msg_size; i++) { - if (msg[i] != '\n') - continue; - - if (con->unfinished_msg[0] != '\0') { - log->writeMessage(con->fd, con->unfinished_msg, strlen(con->unfinished_msg), - msg + msg_begin, i+1 - msg_begin); - } - else { - log->writeMessage(con->fd, msg + msg_begin, i+1 - msg_begin, - NULL, 0); - } - con->unfinished_msg[0] = '\0'; - msg_begin = i+1; - } - - if (msg_begin != msg_size) { - int remaining_bytes = msg_size - msg_begin; - strncat(con->unfinished_msg, msg + msg_begin, remaining_bytes); - } - - return true; - } - - static void *main_wrapper(void *mreader) - { - return static_cast(mreader)->main_loop(); - } - - void *main_loop() - { - while (waitAndHandleMessage()); - return NULL; - } - -public: - - MsgReader(std::string socket_suffix, OutputInterface *log_) : - log(log_) - { - error = true; - - sockfd = socket(AF_UNIX, SOCK_STREAM, 0); - if (sockfd == -1) { - fprintf(stderr, "[%s:%u] - Error calling socket: %s\n", __FILE__, __LINE__, strerror(errno)); - return; - } - - struct sockaddr_un saddr; - bzero(&saddr, sizeof(saddr)); - int pathsize = sizeof(saddr.sun_path); - socket_path = tmpdir + std::string("/umap_") + socket_suffix; - saddr.sun_family = AF_UNIX; - if (socket_path.length() > (unsigned) pathsize-1) { - fprintf(stderr, "[%s:%u] - Socket path overflows AF_UNIX size (%d): %s\n", - __FILE__, __LINE__, pathsize, socket_path.c_str()); - return; - } - strncpy(saddr.sun_path, socket_path.c_str(), pathsize-1); - - int result = bind(sockfd, (struct sockaddr *) &saddr, sizeof(saddr)); - if (result == -1) { - fprintf(stderr, "[%s:%u] - Error binding socket: %s\n", - __FILE__, __LINE__, strerror(errno)); - return; - } - - result = listen(sockfd, LISTEN_BACKLOG); - if (result == -1) { - fprintf(stderr, "[%s:%u] - Error listening socket: %s\n", - __FILE__, __LINE__, strerror(errno)); - return; - } - - error = false; - } - - ~MsgReader() - { - for (std::map::iterator i = conns.begin(); i != conns.end(); i++) { - int fd = i->first; - close(fd); - } - conns.clear(); - if (sockfd != -1) { - close(sockfd); - unlink(socket_path.c_str()); - } - } - - void cleanFile() { - close(sockfd); - unlink(socket_path.c_str()); - sockfd = -1; - } - - bool hadError() const { - return error; - } - - void *run() - { - int result = pthread_create(&thrd, NULL, main_wrapper, (void *) this); - if (result < 0) { - fprintf(stderr, "Failed to spawn thread: %s\n", strerror(errno)); - return NULL; - } - return NULL; - } - - void join() - { - void *result; - pthread_join(thrd, &result); - } -}; - -void parseArgs(int argc, char *argv[]) -{ - if (argc < 3) { - fprintf(stderr, "umap_logd cannot be directly invoked\n"); - exit(-1); - } - - tmpdir = argv[1]; - for (int i=0; icleanFile(); - if (debug_reader) - debug_reader->cleanFile(); -} - -void on_sig(int) -{ - clean(); - exit(0); -} - -void registerCrashHandlers() -{ - signal(SIGINT, on_sig); - signal(SIGTERM, on_sig); -} - -int main(int argc, char *argv[]) -{ - registerCrashHandlers(); - parseArgs(argc, argv); - - lockProcess = new UniqueProcess(); - if (!lockProcess->isUnique()) { - return 0; - } - - //When running a spindle session we need all stdout closed - // or a backtick'd `spindle --start-session` may not return. - // since the output daemon could have forked from the spindle - // session we may have its pipe from the backticks open. - close(0); - open("/dev/null", O_RDONLY); - close(1); - open("/dev/null", O_WRONLY); - - if (runDebug) { - debug_log = new OutputLog(debug_fname); - debug_reader = new MsgReader("log", debug_log); - if (debug_reader->hadError()) { - fprintf(stderr, "Debug reader error termination\n"); - return -1; - } - } - - if (runDebug) - debug_reader->run(); - if (runDebug) - debug_reader->join(); - - clean(); - return 0; -} diff --git a/src/logging/spindle_mkdir.c b/src/logging/spindle_mkdir.c deleted file mode 100644 index a32ac8ac..00000000 --- a/src/logging/spindle_mkdir.c +++ /dev/null @@ -1,136 +0,0 @@ -////////////////////////////////////////////////////////////////////////////// -// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other -// UMAP Project Developers. See the top-level LICENSE file for details. -// -// SPDX-License-Identifier: LGPL-2.1-only -////////////////////////////////////////////////////////////////////////////// - -#include -#include -#include -#include -#include - -//#include "ldcs_api.h" -#include "spindle_debug.h" -#include "config.h" - -#define MAX_PATH_LEN 1024 -#if defined(USE_CLEANUP_PROC) -extern void add_cleanup_dir(const char *dir); -#endif - -static int checkdir(char *path) -{ - struct stat buf; - int result = stat(path, &buf); - if (result == -1) { - err_printf("spindle_mkdir failed because stat on existing directory %s failed: %s\n", - path, strerror(errno)); - return -1; - } - if (!S_ISDIR(buf.st_mode) || S_ISLNK(buf.st_mode)) { - err_printf("spindle_mkdir failed because non-directory %s appeared in path during mkdir\n", - path); - return -1; - } - if (buf.st_uid != geteuid()) { - err_printf("spindle_mkdir failed because component %s was owned by %d rather than expected %d\n", - path, buf.st_uid, geteuid()); - return -1; - } - if (buf.st_gid != getegid()) { - err_printf("spindle_mkdir failed because component %s had group %d rather than expected %d\n", - path, buf.st_gid, getegid()); - return -1; - } - if ((buf.st_mode & 0777) != 0700) { - err_printf("spindle_mkdir failed because component %s had unexpected permissions %o\n", - path, buf.st_mode & 0777); - return -1; - } - - return 0; -} - -int spindle_mkdir(char *orig_path) -{ - char path[MAX_PATH_LEN+1]; - int i, path_len, result, do_mkdir = 0, error; - struct stat buf; - char orig_char; - - debug_printf("spindle_mkdir on %s\n", orig_path); - - - strncpy(path, orig_path, sizeof(path)); - path[MAX_PATH_LEN] = '\0'; - path_len = strlen(path); - - i = 0; - while (path[i] == '/') - i++; - - for (; i < path_len+1; i++) { - if (path[i] != '/' && path[i] != '\0') - continue; - orig_char = path[i]; - path[i] = '\0'; - - if (!do_mkdir) { - //Run a stat on an existing path component. As long as a directory - //component already exists, we won't be too picky about its ownership. - result = stat(path, &buf); - if (result == -1) { - error = errno; - if (error == ENOENT) { -#if defined(USE_CLEANUP_PROC) - add_cleanup_dir(path); -#endif - do_mkdir = 1; - } - else { - err_printf("spindle_mkdir failed to stat path component %s: %s\n", - path, strerror(error)); - return -1; - } - } - if (!S_ISDIR(buf.st_mode) && !S_ISLNK(buf.st_mode)) { - err_printf("spindle_mkdir failed because path component %s is not a directory or symlink\n", - path); - return -1; - } - } - - if (do_mkdir) { - result = mkdir(path, 0700); - if (result == -1) { - error = errno; - if (error != EEXIST) { - err_printf("spindle_mkdir failed to make path component %s: %s\n", path, strerror(error)); - return -1; - } - //Someone created this path component while we were doing the mkdir. - //May be a race with other Spindle libraries. Check that it's owned - //by us with appropriate permissions. - if (checkdir(path) == -1) - return -1; - } - } - path[i] = orig_char; - - if (path[i] == '/') - while (path[i+1] == '/') - i++; - } - - if (!do_mkdir) { - //We never did any mkdirs. Ensure that the final directory in the existing path - // is exclusively ours. - if (checkdir(path) == -1) - return -1; - } - - return 0; -} - diff --git a/src/store/CMakeLists.txt b/src/store/CMakeLists.txt deleted file mode 100644 index 93419225..00000000 --- a/src/store/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -############################################################################# -# Copyright 2017-2019 Lawrence Livermore National Security, LLC and other -# UMAP Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: LGPL-2.1-only -############################################################################# -project(storelibs) - diff --git a/src/store/StoreFile.cpp b/src/store/StoreFile.cpp deleted file mode 100644 index 0f45009e..00000000 --- a/src/store/StoreFile.cpp +++ /dev/null @@ -1,69 +0,0 @@ -////////////////////////////////////////////////////////////////////////////// -// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other -// UMAP Project Developers. See the top-level LICENSE file for details. -// -// SPDX-License-Identifier: LGPL-2.1-only -////////////////////////////////////////////////////////////////////////////// -#include -#include -#include "umap/Store.h" -#include "StoreFile.h" -#include "spindle_debug.h" -#include -#include -#include - -#ifdef UMAP_DEBUG_LOGGING -#include -#endif - -StoreFile::StoreFile(void* _region_, size_t _rsize_, size_t _alignsize_, int _fd_) - : region{_region_}, rsize{_rsize_}, alignsize{_alignsize_}, fd{_fd_} -{ -} - -ssize_t StoreFile::read_from_store(char* buf, size_t nb, off_t off) -{ - size_t rval = 0; -#ifdef UMAP_DEBUG_LOGGING - std::stringstream ss; - ss << "pread(fd=" << fd - << ", buf=" << (void*)buf - << ", nb=" << nb - << ", off=" << off - << ")"; - debug_printf("%s\n", ss.str().c_str()); -#endif - rval = pread(fd, buf, nb, off); -#ifdef UMAP_DEBUG_LOGGING - if (rval == -1) { - int eno = errno; - std::cerr << ss.str() << ": " << strerror(eno) << std::endl; - _exit(1); - } -#endif - return rval; -} - -ssize_t StoreFile::write_to_store(char* buf, size_t nb, off_t off) -{ - size_t rval = 0; -#ifdef UMAP_DEBUG_LOGGING - std::stringstream ss; - ss << "pwrite(fd=" << fd - << ", buf=" << (void*)buf - << ", nb=" << nb - << ", off=" << off - << ")"; - debug_printf("%s\n", ss.str().c_str()); -#endif - rval = pwrite(fd, buf, nb, off); -#ifdef UMAP_DEBUG_LOGGING - if (rval == -1) { - int eno = errno; - std::cerr << ss.str() << ": " << strerror(eno) << std::endl; - _exit(1); - } -#endif - return rval; -} diff --git a/src/umap/Buffer.cpp b/src/umap/Buffer.cpp new file mode 100644 index 00000000..19de07ab --- /dev/null +++ b/src/umap/Buffer.cpp @@ -0,0 +1,385 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// + +#include + +#include "umap/Buffer.hpp" +#include "umap/FillWorkers.hpp" +#include "umap/PageDescriptor.hpp" +#include "umap/RegionManager.hpp" +#include "umap/WorkerPool.hpp" +#include "umap/util/Macros.hpp" + +namespace Umap { +// +// Called after data has been placed into the page +// +void Buffer::mark_page_as_present(PageDescriptor* pd) +{ + lock(); + + pd->set_state_present(); + + if ( m_waits_for_state_change ) + pthread_cond_broadcast( &m_state_change_cond ); + + unlock(); +} + +// +// Called after page has been flushed to store and page is no longer present +// +void Buffer::mark_page_as_free( PageDescriptor* pd ) +{ + lock(); + + UMAP_LOG(Debug, "Removing page: " << pd); + pd->region->erase_page_descriptor(pd); + + m_present_pages.erase(pd->page); + + pd->set_state_free(); + pd->spurious_count = 0; + + // + // We only put the page descriptor back onto the free list if it isn't + // deferred. Note: It will be marked as deferred when the page is part of a + // Region that has been unmapped. It will become undeferred later when the + // eviction manager takes it off the end of the end of the buffer. + // + if ( ! pd->deferred ) + release_page_descriptor(pd); + + if ( m_waits_for_state_change ) + pthread_cond_broadcast( &m_state_change_cond ); + + pd->page = nullptr; + + unlock(); +} + +void Buffer::release_page_descriptor( PageDescriptor* pd ) +{ + m_free_pages.push_back(pd); + + if ( m_waits_for_avail_pd ) + pthread_cond_broadcast(&m_avail_pd_cond); +} + +// +// Called from Evict Manager to begin eviction process on oldest present +// page +// +PageDescriptor* Buffer::evict_oldest_page() +{ + PageDescriptor* pd = nullptr; + + lock(); + + while ( m_busy_pages.size() != 0 ) { + pd = m_busy_pages.back(); + + // Deferred means that this page was previously evicted as part of an + // uunmap of a Region. This means that this page descriptor points to a + // page that has already been given back to the system so all we need to + // do is take it off of the busy list and release the descriptor. + // + if ( pd->deferred ) { + UMAP_LOG(Debug, "Deferred Page: " << pd); + + // + // Make sure that the page has truly been flushed. + // + wait_for_page_state(pd, PageDescriptor::State::FREE); + + m_busy_pages.pop_back(); + m_stats.pages_deleted++; + + // + // Jump to the next page descriptor + // + release_page_descriptor(pd); + pd = nullptr; + } + else { + UMAP_LOG(Debug, "Normal Page: " << pd); + wait_for_page_state(pd, PageDescriptor::State::PRESENT); + m_busy_pages.pop_back(); + m_stats.pages_deleted++; + pd->set_state_leaving(); + break; + } + } + + unlock(); + return pd; +} + +// +// Called from uunmap by the unmapping thread of the application +// +// The idea is to go through the entire buffer and remove (evict) all pages +// of the given region descriptor. +// +void Buffer::evict_region(RegionDescriptor* rd) +{ + if (m_rm->get_num_active_regions() > 1) { + lock(); + while ( rd->count() ) { + auto pd = rd->get_next_page_descriptor(); + pd->deferred = true; + wait_for_page_state(pd, PageDescriptor::State::PRESENT); + pd->set_state_leaving(); + m_rm->get_evict_manager()->schedule_eviction(pd); + wait_for_page_state(pd, PageDescriptor::State::FREE); + } + unlock(); + } + else { + m_rm->get_evict_manager()->EvictAll(); + } +} + +bool Buffer::low_threshold_reached( void ) +{ + return m_busy_pages.size() <= m_evict_low_water; +} + +void Buffer::process_page_event(char* paddr, bool iswrite, RegionDescriptor* rd) +{ + WorkItem work; + work.type = Umap::WorkItem::WorkType::NONE; + + lock(); + auto pd = page_already_present(paddr); + + if ( pd != nullptr ) { // Page is already present + if (iswrite && pd->dirty == false) { + work.page_desc = pd; + pd->dirty = true; + pd->set_state_updating(); + UMAP_LOG(Debug, "PRE: " << pd << " From: " << this); + } + else { + static int hiwat = 0; + + pd->spurious_count++; + if (pd->spurious_count > hiwat) { + hiwat = pd->spurious_count; + UMAP_LOG(Info, "New Spurious cound high water mark: " << hiwat); + } + + UMAP_LOG(Debug, "SPU: " << pd << " From: " << this); + unlock(); + return; + } + } + else { // This page has not been brought in yet + pd = get_page_descriptor(paddr, rd); + pd->data_present = false; + work.page_desc = pd; + + rd->insert_page_descriptor(pd); + m_present_pages[pd->page] = pd; + + if (iswrite) + pd->dirty = true; + + UMAP_LOG(Debug, "NEW: " << pd << " From: " << this); + } + + m_rm->get_fill_workers_h()->send_work(work); + + // + // Kick the eviction daemon if the high water mark has been reached + // + if ( m_busy_pages.size() == m_evict_high_water ) { + WorkItem w; + + w.type = Umap::WorkItem::WorkType::THRESHOLD; + w.page_desc = nullptr; + m_rm->get_evict_manager()->send_work(w); + } + + unlock(); +} + +// Return nullptr if page not present, PageDescriptor * otherwise +PageDescriptor* Buffer::page_already_present( char* page_addr ) +{ + while (1) { + auto pp = m_present_pages.find(page_addr); + + // + // Most likely case + // + if ( pp == m_present_pages.end() ) + return nullptr; + + // + // Next most likely is that it is just present in the buffer + // + if ( pp->second->state == PageDescriptor::State::PRESENT ) + return pp->second; + + // There is a chance that the state of this page is not/no-longer + // PRESENT. If this is the case, we need to wait for it to finish + // with whatever is happening to it and then check again + // + UMAP_LOG(Debug, "Waiting for state: (ANY)" << ", " << pp->second); + + ++m_stats.waits; + ++m_waits_for_state_change; + pthread_cond_wait(&m_state_change_cond, &m_mutex); + --m_waits_for_state_change; + } +} + +PageDescriptor* Buffer::get_page_descriptor(char* vaddr, RegionDescriptor* rd) +{ + while ( m_free_pages.size() == 0 ) { + ++m_waits_for_avail_pd; + m_stats.not_avail++; + + ++m_stats.waits; + ++m_waits_for_state_change; + pthread_cond_wait(&m_avail_pd_cond, &m_mutex); + + --m_waits_for_avail_pd; + } + + PageDescriptor* rval; + + rval = m_free_pages.back(); + m_free_pages.pop_back(); + + rval->page = vaddr; + rval->region = rd; + rval->dirty = false; + rval->deferred = false; + rval->set_state_filling(); + rval->spurious_count = 0; + + m_stats.pages_inserted++; + m_busy_pages.push_front(rval); + + return rval; +} + +uint64_t Buffer::apply_int_percentage( int percentage, uint64_t item ) +{ + uint64_t rval; + + if ( percentage < 0 || percentage > 100) + UMAP_ERROR("Invalid percentage (" << percentage << ") given"); + + if ( percentage == 0 || percentage == 100 ) { + rval = item; + } + else { + float f = (float)((float)percentage / (float)100.0); + rval = f * item; + } + return rval; +} + +void Buffer::lock() +{ + int err; + if ( (err = pthread_mutex_trylock(&m_mutex)) != 0 ) { + if (err != EBUSY) + UMAP_ERROR("pthread_mutex_trylock failed: " << strerror(err)); + + if ( (err = pthread_mutex_lock(&m_mutex)) != 0 ) + UMAP_ERROR("pthread_mutex_lock failed: " << strerror(err)); + m_stats.lock_collision++; + } + m_stats.lock++; +} + +void Buffer::unlock() +{ + pthread_mutex_unlock(&m_mutex); +} + +void Buffer::wait_for_page_state( PageDescriptor* pd, PageDescriptor::State st) +{ + UMAP_LOG(Debug, "Waiting for state: " << st << ", " << pd); + + while ( pd->state != st ) { + ++m_stats.waits; + ++m_waits_for_state_change; + + ++m_stats.waits; + ++m_waits_for_state_change; + pthread_cond_wait(&m_state_change_cond, &m_mutex); + + --m_waits_for_state_change; + } +} + +Buffer::Buffer( void ) + : m_rm(RegionManager::getInstance()) + , m_size(m_rm->get_max_pages_in_buffer()) + , m_waits_for_avail_pd(0) + , m_waits_for_state_change(0) +{ + m_array = (PageDescriptor *)calloc(m_size, sizeof(PageDescriptor)); + if ( m_array == nullptr ) + UMAP_ERROR("Failed to allocate " << m_size*sizeof(PageDescriptor) + << " bytes for buffer page descriptors"); + + for ( int i = 0; i < m_size; ++i ) + m_free_pages.push_back(&m_array[i]); + + pthread_mutex_init(&m_mutex, NULL); + pthread_cond_init(&m_avail_pd_cond, NULL); + pthread_cond_init(&m_state_change_cond, NULL); + + m_evict_low_water = apply_int_percentage(m_rm->get_evict_low_water_threshold(), m_size); + m_evict_high_water = apply_int_percentage(m_rm->get_evict_high_water_threshold(), m_size); +} + +Buffer::~Buffer( void ) { + UMAP_LOG(Debug, m_stats); + + assert("Pages are still present" && m_present_pages.size() == 0); + pthread_cond_destroy(&m_avail_pd_cond); + pthread_cond_destroy(&m_state_change_cond); + pthread_mutex_destroy(&m_mutex); + free(m_array); +} + +std::ostream& operator<<(std::ostream& os, const Umap::Buffer* b) +{ + if ( b != nullptr ) { + os << "{ m_size: " << b->m_size + << ", m_waits_for_avail_pd: " << b->m_waits_for_avail_pd + << ", m_present_pages.size(): " << std::setw(2) << b->m_present_pages.size() + << ", m_free_pages.size(): " << std::setw(2) << b->m_free_pages.size() + << ", m_busy_pages.size(): " << std::setw(2) << b->m_busy_pages.size() + << " }" + ; + } + else { + os << "{ nullptr }"; + } + return os; +} + +std::ostream& operator<<(std::ostream& os, const Umap::BufferStats& stats) +{ + os << "Buffer Statisics:\n" + << " Pages Inserted: " << std::setw(12) << stats.pages_inserted<< "\n" + << " Pages Deleted: " << std::setw(12) << stats.pages_deleted<< "\n" + << " Unavailable wait: " << std::setw(12) << stats.not_avail<< "\n" + << " Locks: " << std::setw(12) << stats.lock << "\n" + << " Lock collisions: " << std::setw(12) << stats.lock_collision << "\n" + << " waits: " << std::setw(12) << stats.waits; + return os; +} +} // end of namespace Umap diff --git a/src/umap/Buffer.hpp b/src/umap/Buffer.hpp new file mode 100644 index 00000000..d8e31853 --- /dev/null +++ b/src/umap/Buffer.hpp @@ -0,0 +1,89 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#ifndef _UMAP_Buffer_HPP +#define _UMAP_Buffer_HPP + +#include +#include +#include +#include + +#include "umap/RegionDescriptor.hpp" +#include "umap/PageDescriptor.hpp" + +namespace Umap { + class RegionManager; + + struct BufferStats { + BufferStats() : lock_collision(0), lock(0), pages_inserted(0) + , pages_deleted(0), not_avail(0), waits(0) + {}; + + uint64_t lock_collision; + uint64_t lock; + uint64_t pages_inserted; + uint64_t pages_deleted; + uint64_t not_avail; + uint64_t waits; + }; + + class Buffer { + friend std::ostream& operator<<(std::ostream& os, const Umap::Buffer* b); + friend std::ostream& operator<<(std::ostream& os, const Umap::BufferStats& stats); + public: + void mark_page_as_present(PageDescriptor* pd); + void mark_page_as_free( PageDescriptor* pd ); + + bool low_threshold_reached( void ); + + PageDescriptor* evict_oldest_page( void ); + void process_page_event(char* paddr, bool iswrite, RegionDescriptor* rd); + void evict_region(RegionDescriptor* rd); + + explicit Buffer( void ); + ~Buffer( void ); + + private: + RegionManager* m_rm; + uint64_t m_size; // Maximum pages this buffer may have + PageDescriptor* m_array; + + std::unordered_map m_present_pages; + + std::vector m_free_pages; + std::deque m_busy_pages; + + uint64_t m_evict_low_water; // % to evict too + uint64_t m_evict_high_water; // % to start evicting + + pthread_mutex_t m_mutex; + + int m_waits_for_avail_pd; + pthread_cond_t m_avail_pd_cond; + + int m_waits_for_state_change; + pthread_cond_t m_state_change_cond; + + BufferStats m_stats; + + void release_page_descriptor( PageDescriptor* pd ); + + PageDescriptor* page_already_present( char* page_addr ); + PageDescriptor* get_page_descriptor( char* page_addr, RegionDescriptor* rd ); + uint64_t apply_int_percentage( int percentage, uint64_t item ); + + void lock(); + void unlock(); + void wait_for_page_state( PageDescriptor* pd, PageDescriptor::State st); + }; + + std::ostream& operator<<(std::ostream& os, const Umap::BufferStats& stats); + std::ostream& operator<<(std::ostream& os, const Umap::Buffer* b); + +} // end of namespace Umap + +#endif // _UMAP_Buffer_HPP diff --git a/src/umap/CMakeLists.txt b/src/umap/CMakeLists.txt index 65de1128..31a3bbaf 100644 --- a/src/umap/CMakeLists.txt +++ b/src/umap/CMakeLists.txt @@ -5,18 +5,40 @@ # SPDX-License-Identifier: LGPL-2.1-only ############################################################################# project(umap_libraries) + +set(umapheaders + config.h + Buffer.hpp + EvictManager.hpp + EvictWorkers.hpp + FillWorkers.hpp + PageDescriptor.hpp + RegionManager.hpp + RegionDescriptor.hpp + Uffd.hpp + umap.h + WorkQueue.hpp + WorkerPool.hpp + store/StoreFile.h + store/Store.hpp + util/Exception.hpp + util/Logger.hpp + util/Macros.hpp) + set(umapsrc + Buffer.cpp + EvictManager.cpp + EvictWorkers.cpp + FillWorkers.cpp + PageDescriptor.cpp + RegionManager.cpp + Uffd.cpp umap.cpp - ../store/Store.cpp - ../store/StoreFile.cpp -) - -if ( ENABLE_LOGGING ) - set(umapsrc ${umapsrc} - ../logging/spindle_logc.c - ../logging/spindle_mkdir.c - ) -endif() + store/Store.cpp + store/StoreFile.cpp + util/Exception.cpp + util/Logger.cpp + ${umapheaders}) find_package(Threads REQUIRED) add_library(umap SHARED ${umapsrc} ) @@ -26,27 +48,11 @@ target_link_libraries (umap ${CMAKE_THREAD_LIBS_INIT}) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") -include_directories( AFTER - ${CMAKE_CURRENT_SOURCE_DIR} - ${UMAPINCLUDEDIRS} -) - install(TARGETS umap umap-static LIBRARY DESTINATION lib ARCHIVE DESTINATION lib RUNTIME DESTINATION bin ) -install(FILES - ${CMAKE_SOURCE_DIR}/src/include/umap/umap.h - ${CMAKE_SOURCE_DIR}/src/include/umap/Store.h - DESTINATION include/umap) - -if ( ENABLE_LOGGING ) - add_executable(umap_logd ../logging/spindle_logd.cpp) - target_link_libraries (umap_logd ${CMAKE_THREAD_LIBS_INIT}) +install(FILES umap.h DESTINATION include/umap) - install(FILES ${PROJECT_BINARY_DIR}/umap_logd - DESTINATION libexec - PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_EXECUTE GROUP_READ - ) -endif() +install(FILES store/Store.hpp DESTINATION include/umap/store ) diff --git a/src/umap/EvictManager.cpp b/src/umap/EvictManager.cpp new file mode 100644 index 00000000..4a683296 --- /dev/null +++ b/src/umap/EvictManager.cpp @@ -0,0 +1,91 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// + +#include "umap/Buffer.hpp" +#include "umap/EvictManager.hpp" +#include "umap/EvictWorkers.hpp" +#include "umap/RegionManager.hpp" +#include "umap/Uffd.hpp" +#include "umap/WorkerPool.hpp" +#include "umap/util/Macros.hpp" +#include "umap/store/Store.hpp" + +namespace Umap { + +void EvictManager::EvictMgr( void ) { + while ( 1 ) { + auto w = get_work(); + + if ( w.type == Umap::WorkItem::WorkType::EXIT ) + break; // Time to leave + + while ( ! m_buffer->low_threshold_reached() ) { + WorkItem work; + work.type = Umap::WorkItem::WorkType::EVICT; + work.page_desc = m_buffer->evict_oldest_page(); // Could block + + if ( work.page_desc == nullptr ) + break; + + UMAP_LOG(Debug, m_buffer << ", " << work.page_desc); + + m_evict_workers->send_work(work); + } + } +} + +void EvictManager::EvictAll( void ) +{ + UMAP_LOG(Debug, "Entered"); + + for (auto pd = m_buffer->evict_oldest_page(); pd != nullptr; pd = m_buffer->evict_oldest_page()) { + UMAP_LOG(Debug, "evicting: " << pd); + if (pd->dirty) { + WorkItem work = { .page_desc = pd, .type = Umap::WorkItem::WorkType::FAST_EVICT }; + m_evict_workers->send_work(work); + } + else { + m_buffer->mark_page_as_free(pd); + } + } + + m_evict_workers->wait_for_idle(); + + UMAP_LOG(Debug, "Done"); +} + +void EvictManager::schedule_eviction(PageDescriptor* pd) +{ + WorkItem work = { .page_desc = pd, .type = Umap::WorkItem::WorkType::EVICT }; + + m_evict_workers->send_work(work); +} + +EvictManager::EvictManager( void ) : + WorkerPool("Evict Manager", 1) + , m_buffer(RegionManager::getInstance()->get_buffer_h()) +{ + m_evict_workers = new EvictWorkers( RegionManager::getInstance()->get_num_evictors() + , m_buffer, RegionManager::getInstance()->get_uffd_h()); + start_thread_pool(); +} + +EvictManager::~EvictManager( void ) { + UMAP_LOG(Debug, "Calling EvictAll"); + EvictAll(); + UMAP_LOG(Debug, "Calling stop_thread_pool"); + stop_thread_pool(); + UMAP_LOG(Debug, "Deleting eviction workers"); + delete m_evict_workers; + UMAP_LOG(Debug, "Done"); +} + +void EvictManager::ThreadEntry() { + EvictMgr(); +} + +} // end of namespace Umap diff --git a/src/umap/EvictManager.hpp b/src/umap/EvictManager.hpp new file mode 100644 index 00000000..6495eafe --- /dev/null +++ b/src/umap/EvictManager.hpp @@ -0,0 +1,35 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#ifndef _UMAP_EvictManager_HPP +#define _UMAP_EvictManager_HPP + +#include "umap/EvictWorkers.hpp" + +#include "umap/Buffer.hpp" +#include "umap/PageDescriptor.hpp" +#include "umap/RegionDescriptor.hpp" +#include "umap/WorkerPool.hpp" + +namespace Umap { + class EvictWorkers; + + class EvictManager : public WorkerPool { + public: + EvictManager( void ); + ~EvictManager( void ); + void schedule_eviction(PageDescriptor* pd); + void EvictAll( void ); + + private: + Buffer* m_buffer; + EvictWorkers* m_evict_workers; + + void EvictMgr(void); + void ThreadEntry( void ); + }; +} // end of namespace Umap +#endif // _UMAP_EvictManager_HPP diff --git a/src/umap/EvictWorkers.cpp b/src/umap/EvictWorkers.cpp new file mode 100644 index 00000000..097a58bf --- /dev/null +++ b/src/umap/EvictWorkers.cpp @@ -0,0 +1,70 @@ +/////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#include +#include +#include + +#include "umap/Buffer.hpp" +#include "umap/EvictWorkers.hpp" +#include "umap/RegionManager.hpp" +#include "umap/Uffd.hpp" +#include "umap/WorkerPool.hpp" +#include "umap/util/Macros.hpp" + +namespace Umap { +void EvictWorkers::EvictWorker( void ) +{ + uint64_t page_size = RegionManager::getInstance()->get_umap_page_size(); + + while ( 1 ) { + auto w = get_work(); + + UMAP_LOG(Debug, " " << w << " " << m_buffer); + + if ( w.type == Umap::WorkItem::WorkType::EXIT ) + break; // Time to leave + + auto pd = w.page_desc; + + if ( pd->dirty ) { + auto store = pd->region->store(); + auto offset = pd->region->store_offset(pd->page); + + m_uffd->enable_write_protect(pd->page); + + if (store->write_to_store(pd->page, page_size, offset) == -1) + UMAP_ERROR("write_to_store failed: " + << errno << " (" << strerror(errno) << ")"); + } + + if (w.type != Umap::WorkItem::WorkType::FAST_EVICT) { + if (madvise(pd->page, page_size, MADV_DONTNEED) == -1) + UMAP_ERROR("madvise failed: " << errno << " (" << strerror(errno) << ")"); + } + + UMAP_LOG(Debug, "Removing page: " << w.page_desc); + m_buffer->mark_page_as_free(w.page_desc); + } +} + +EvictWorkers::EvictWorkers(uint64_t num_evictors, Buffer* buffer, Uffd* uffd) + : WorkerPool("Evict Workers", num_evictors), m_buffer(buffer) + , m_uffd(uffd) +{ + start_thread_pool(); +} + +EvictWorkers::~EvictWorkers( void ) +{ + stop_thread_pool(); +} + +void EvictWorkers::ThreadEntry( void ) +{ + EvictWorkers::EvictWorker(); +} +} // end of namespace Umap diff --git a/src/umap/EvictWorkers.hpp b/src/umap/EvictWorkers.hpp new file mode 100644 index 00000000..f5b1bb41 --- /dev/null +++ b/src/umap/EvictWorkers.hpp @@ -0,0 +1,32 @@ +/////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#ifndef _UMAP_EvictWorkers_HPP +#define _UMAP_EvictWorkers_HPP + +#include "umap/config.h" + +#include "umap/Buffer.hpp" +#include "umap/PageDescriptor.hpp" +#include "umap/Uffd.hpp" +#include "umap/WorkerPool.hpp" + +namespace Umap { + class Uffd; + class EvictWorkers : public WorkerPool { + public: + EvictWorkers(uint64_t num_evictors, Buffer* buffer, Uffd* uffd); + ~EvictWorkers( void ); + + private: + Buffer* m_buffer; + Uffd* m_uffd; + + void EvictWorker( void ); + void ThreadEntry( void ); + }; +} // end of namespace Umap +#endif // _UMAP_EvictWorkers_HPP diff --git a/src/umap/FillWorkers.cpp b/src/umap/FillWorkers.cpp new file mode 100644 index 00000000..7c15e5e4 --- /dev/null +++ b/src/umap/FillWorkers.cpp @@ -0,0 +1,87 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#include "umap/config.h" + +#include // calloc +#include +#include // strerror() +#include + +#include "umap/Buffer.hpp" +#include "umap/FillWorkers.hpp" +#include "umap/RegionManager.hpp" +#include "umap/Uffd.hpp" +#include "umap/WorkerPool.hpp" +#include "umap/store/Store.hpp" +#include "umap/util/Macros.hpp" + +namespace Umap { + void FillWorkers::FillWorker( void ) { + char* copyin_buf; + uint64_t page_size = RegionManager::getInstance()->get_umap_page_size(); + uint64_t read_ahead = RegionManager::getInstance()->get_read_ahead(); + std::size_t sz = 2 * page_size; + + if (posix_memalign((void**)©in_buf, page_size, sz)) { + UMAP_ERROR("posix_memalign failed to allocated " + << sz << " bytes of memory"); + } + + if (copyin_buf == nullptr) { + UMAP_ERROR("posix_memalign failed to allocated " + << sz << " bytes of memory"); + } + + while ( 1 ) { + auto w = get_work(); + + UMAP_LOG(Debug, ": " << w << " " << m_buffer); + + if (w.type == Umap::WorkItem::WorkType::EXIT) + break; // Time to leave + + if ( w.page_desc->dirty && w.page_desc->data_present ) { + m_uffd->disable_write_protect(w.page_desc->page); + } + else { + uint64_t offset = w.page_desc->region->store_offset(w.page_desc->page); + + if (w.page_desc->region->store()->read_from_store(copyin_buf, page_size, offset) == -1) + UMAP_ERROR("read_from_store failed"); + + if ( ! w.page_desc->dirty ) { + m_uffd->copy_in_page_and_write_protect(copyin_buf, w.page_desc->page); + } + else { + m_uffd->copy_in_page(copyin_buf, w.page_desc->page); + } + w.page_desc->data_present = true; + } + + m_buffer->mark_page_as_present(w.page_desc); + } + + free(copyin_buf); + } + + void FillWorkers::ThreadEntry( void ) { + FillWorker(); + } + + FillWorkers::FillWorkers( void ) + : WorkerPool("Fill Workers", RegionManager::getInstance()->get_num_fillers()) + , m_uffd(RegionManager::getInstance()->get_uffd_h()) + , m_buffer(RegionManager::getInstance()->get_buffer_h()) + , m_read_ahead(RegionManager::getInstance()->get_read_ahead()) + { + start_thread_pool(); + } + + FillWorkers::~FillWorkers( void ) { + stop_thread_pool(); + } +} // end of namespace Umap diff --git a/src/umap/FillWorkers.hpp b/src/umap/FillWorkers.hpp new file mode 100644 index 00000000..b29b90bd --- /dev/null +++ b/src/umap/FillWorkers.hpp @@ -0,0 +1,32 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#ifndef _UMAP_FillWorkers_HPP +#define _UMAP_FillWorkers_HPP + +#include "umap/Buffer.hpp" +#include "umap/Uffd.hpp" +#include "umap/WorkerPool.hpp" + +namespace Umap { + class Buffer; + class Uffd; + + class FillWorkers : public WorkerPool { + public: + FillWorkers( void ); + ~FillWorkers( void ); + + private: + Uffd* m_uffd; + Buffer* m_buffer; + uint64_t m_read_ahead; + + void FillWorker( void ); + void ThreadEntry( void ); + }; +} // end of namespace Umap +#endif // _UMAP_FillWorker_HPP diff --git a/src/umap/PageDescriptor.cpp b/src/umap/PageDescriptor.cpp new file mode 100644 index 00000000..860c7599 --- /dev/null +++ b/src/umap/PageDescriptor.cpp @@ -0,0 +1,89 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#include + +#include "umap/PageDescriptor.hpp" +#include "umap/RegionDescriptor.hpp" +#include "umap/util/Macros.hpp" + +namespace Umap { + std::string PageDescriptor::print_state( void ) const { + switch (state) { + default: return "???"; + case Umap::PageDescriptor::State::FREE: return "FREE"; + case Umap::PageDescriptor::State::FILLING: return "FILLING"; + case Umap::PageDescriptor::State::PRESENT: return "PRESENT"; + case Umap::PageDescriptor::State::UPDATING: return "UPDATING"; + case Umap::PageDescriptor::State::LEAVING: return "LEAVING"; + } + } + + void PageDescriptor::set_state_free( void ) { + if ( state != LEAVING ) + UMAP_ERROR("Invalid state transition from: " << print_state()); + state = FREE; + } + + void PageDescriptor::set_state_filling( void ) { + if ( state != FREE ) + UMAP_ERROR("Invalid state transition from: " << print_state()); + state = FILLING; + } + + void PageDescriptor::set_state_present( void ) { + if ( state != FILLING && state != UPDATING ) + UMAP_ERROR("Invalid state transition from: " << print_state()); + state = PRESENT; + } + + void PageDescriptor::set_state_updating( void ) { + if ( state != PRESENT ) + UMAP_ERROR("Invalid state transition from: " << print_state()); + state = UPDATING; + } + + void PageDescriptor::set_state_leaving( void ) { + if ( state != PRESENT ) + UMAP_ERROR("Invalid state transition from: " << print_state()); + state = LEAVING; + } + + std::ostream& operator<<(std::ostream& os, const Umap::PageDescriptor* pd) + { + if (pd != nullptr) { + os << "{ " + << (void*)(pd->page) + << ", " << pd->print_state(); + + if ( pd->dirty ) + os << ", DIRTY"; + if ( pd->deferred ) + os << ", DEFERRED"; + if ( pd->spurious_count ) + os << ", spurious: " << pd->spurious_count; + + os << " }"; + } + else { + os << "{ nullptr }"; + } + return os; + } + + std::ostream& operator<<(std::ostream& os, const Umap::PageDescriptor::State st) + { + switch (st) { + default: os << "???"; break; + case Umap::PageDescriptor::State::FREE: os << "FREE"; break; + case Umap::PageDescriptor::State::FILLING: os << "FILLING"; break; + case Umap::PageDescriptor::State::PRESENT: os << "PRESENT"; break; + case Umap::PageDescriptor::State::UPDATING: os << "UPDATING"; break; + case Umap::PageDescriptor::State::LEAVING: os << "LEAVING"; break; + } + return os; + } +} // end of namespace Umap diff --git a/src/umap/PageDescriptor.hpp b/src/umap/PageDescriptor.hpp new file mode 100644 index 00000000..931335a6 --- /dev/null +++ b/src/umap/PageDescriptor.hpp @@ -0,0 +1,38 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#ifndef _UMAP_PageDescriptor_HPP +#define _UMAP_PageDescriptor_HPP + +#include +#include + +namespace Umap { + class RegionDescriptor; + + struct PageDescriptor { + enum State { FREE = 0, FILLING, PRESENT, UPDATING, LEAVING }; + char* page; + RegionDescriptor* region; + State state; + bool dirty; + bool deferred; + bool data_present; + int spurious_count; + + std::string print_state( void ) const; + void set_state_free( void ); + void set_state_filling( void ); + void set_state_updating( void ); + void set_state_present( void ); + void set_state_leaving( void ); + }; + + std::ostream& operator<<(std::ostream& os, const Umap::PageDescriptor::State st); + std::ostream& operator<<(std::ostream& os, const Umap::PageDescriptor* pd); +} // end of namespace Umap + +#endif // _UMAP_PageDescriptor_HPP diff --git a/src/umap/RegionDescriptor.hpp b/src/umap/RegionDescriptor.hpp new file mode 100644 index 00000000..e166f503 --- /dev/null +++ b/src/umap/RegionDescriptor.hpp @@ -0,0 +1,74 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#ifndef _UMAP_RegionDescriptor_HPP +#define _UMAP_RegionDescriptor_HPP + +#include +#include +#include +#include +#include + +#include "umap/PageDescriptor.hpp" +#include "umap/store/Store.hpp" +#include "umap/util/Macros.hpp" + +namespace Umap { + class RegionDescriptor { + public: + RegionDescriptor( char* umap_region, uint64_t umap_size + , char* mmap_region, uint64_t mmap_size + , Store* store ) + : m_umap_region(umap_region), m_umap_region_size(umap_size) + , m_mmap_region(mmap_region), m_mmap_region_size(mmap_size) + , m_store(store) {} + + ~RegionDescriptor( void ) {} + + inline uint64_t store_offset( char* addr ) { + assert("Invalid address for calculating offset" && addr >= start() && addr < end()); + return (uint64_t)(addr - start()); + } + + inline uint64_t size( void ) { return m_umap_region_size; } + inline Store* store( void ) { return m_store; } + inline char* start( void ) { return m_umap_region; } + inline char* end( void ) { return start() + size(); } + inline uint64_t count( void ) { return m_active_pages.size(); } + + inline void insert_page_descriptor(PageDescriptor* pd) { + m_active_pages.insert(pd); + } + + inline void erase_page_descriptor(PageDescriptor* pd) { + UMAP_LOG(Debug, "Erasing PD: " << pd); + m_active_pages.erase(pd); + } + + inline PageDescriptor* get_next_page_descriptor( void ) { + if ( m_active_pages.size() == 0 ) + return nullptr; + + auto it = m_active_pages.begin(); + auto rval = *it; + rval->deferred = false; + erase_page_descriptor(rval); + + return rval; + } + + private: + char* m_umap_region; + uint64_t m_umap_region_size; + char* m_mmap_region; + uint64_t m_mmap_region_size; + Store* m_store; + + std::unordered_set m_active_pages; + }; +} // end of namespace Umap +#endif // _UMAP_RegionDescripto_HPP diff --git a/src/umap/RegionManager.cpp b/src/umap/RegionManager.cpp new file mode 100644 index 00000000..34db2bbc --- /dev/null +++ b/src/umap/RegionManager.cpp @@ -0,0 +1,297 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#include "umap/config.h" + +#include // uint64_t +#include // for reading meminfo +#include // getenv() +#include // string to integer operations +#include // string to integer operations +#include // for max_concurrency +#include +#include // sysconf() + +#include "umap/Buffer.hpp" +#include "umap/EvictManager.hpp" +#include "umap/FillWorkers.hpp" +#include "umap/RegionManager.hpp" +#include "umap/RegionDescriptor.hpp" +#include "umap/store/Store.hpp" +#include "umap/util/Macros.hpp" + +namespace Umap { + +RegionManager* RegionManager::s_fault_monitor_manager_instance = nullptr; + +RegionManager* +RegionManager::getInstance( void ) +{ + if (!s_fault_monitor_manager_instance) { + s_fault_monitor_manager_instance = new RegionManager(); + } + + return s_fault_monitor_manager_instance; +} + +void +RegionManager::addRegion(Store* store, char* region, uint64_t region_size, char* mmap_region, uint64_t mmap_region_size) +{ + UMAP_LOG(Debug, + "store: " << store + << ", region: " << (void*)region + << ", region_size: " << region_size + << ", mmap_region: " << (void*)mmap_region + << ", mmap_region_size: " << mmap_region_size); + + if ( m_active_regions.empty() ) { + UMAP_LOG(Debug, "No active regions, initializing engine"); + UMAP_LOG(Debug, "Creating Buffer"); + m_buffer = new Buffer(); + UMAP_LOG(Debug, "Creating Uffd"); + m_uffd = new Uffd(); + UMAP_LOG(Debug, "Creating FillWorkers"); + m_fill_workers = new FillWorkers(); + UMAP_LOG(Debug, "Creating EvictManager"); + m_evict_manager = new EvictManager(); + } + else { + UMAP_LOG(Debug, "Active regions present, adding new region"); + } + + auto rd = new RegionDescriptor(region, region_size, mmap_region, mmap_region_size, store); + m_active_regions[(void*)region] = rd; + m_uffd->register_region(rd); +} + +void +RegionManager::removeRegion( char* region ) +{ + UMAP_LOG(Debug, "region: " << (void*)region); + + auto it = m_active_regions.find(region); + + if (it == m_active_regions.end()) + UMAP_ERROR("umap fault monitor not found for: " << (void*)region); + + UMAP_LOG(Debug, "Calling unregister_region"); + m_uffd->unregister_region(it->second); + + UMAP_LOG(Debug, "Deleting region"); + delete it->second; + UMAP_LOG(Debug, "Erasing from list region"); + m_active_regions.erase(it); + + if ( m_active_regions.empty() ) { + UMAP_LOG(Debug, "Deleting eviction manager"); + delete m_evict_manager; m_evict_manager = nullptr; + UMAP_LOG(Debug, "Deleting fill workers"); + delete m_fill_workers; m_fill_workers = nullptr; + UMAP_LOG(Debug, "Deleting m_uffd"); + delete m_uffd; m_uffd = nullptr; + UMAP_LOG(Debug, "Deleting m_buffer"); + delete m_buffer; m_buffer = nullptr; + } + UMAP_LOG(Debug, "Done"); +} + +void +RegionManager::prefetch(int npages, umap_prefetch_item* page_array) +{ + for (int i{0}; i < npages; ++i) + m_uffd->process_page(false, (char*)(page_array[i].page_base_addr)); +} + +RegionManager::RegionManager() +{ + m_version.major = UMAP_VERSION_MAJOR; + m_version.minor = UMAP_VERSION_MINOR; + m_version.patch = UMAP_VERSION_PATCH; + + m_system_page_size = sysconf(_SC_PAGESIZE); + + const uint64_t MAX_FAULT_EVENTS = 256; + uint64_t env_value = 0; + if ( (read_env_var("UMAP_MAX_FAULT_EVENTS", &env_value)) != nullptr ) + set_max_fault_events(env_value); + else + set_max_fault_events(MAX_FAULT_EVENTS); + + unsigned int nthreads = std::thread::hardware_concurrency(); + nthreads = (nthreads == 0) ? 16 : nthreads; + + if ( (read_env_var("UMAP_PAGE_FILLERS", &env_value)) != nullptr ) + set_num_fillers(env_value); + else + set_num_fillers(nthreads); + + if ( (read_env_var("UMAP_PAGE_EVICTORS", &env_value)) != nullptr ) + set_num_evictors(env_value); + else + set_num_evictors(nthreads); + + if ( (read_env_var("UMAP_EVICT_HIGH_WATER_THRESHOLD", &env_value)) != nullptr ) + set_evict_high_water_threshold(env_value); + else + set_evict_high_water_threshold(90); + + if ( (read_env_var("UMAP_EVICT_LOW_WATER_THRESHOLD", &env_value)) != nullptr ) + set_evict_low_water_threshold(env_value); + else + set_evict_low_water_threshold(70); + + if ( (read_env_var("UMAP_PAGESIZE", &env_value)) != nullptr ) + set_umap_page_size(env_value); + else + set_umap_page_size(m_system_page_size); + + if ( (read_env_var("UMAP_BUFSIZE", &env_value)) != nullptr ) + set_max_pages_in_buffer(env_value); + else + set_max_pages_in_buffer( get_max_pages_in_memory() ); + + if ( (read_env_var("UMAP_READ_AHEAD", &env_value)) != nullptr ) + set_read_ahead(env_value); + else + set_read_ahead(0); +} + +uint64_t +RegionManager::get_max_pages_in_memory( void ) +{ + static uint64_t total_mem_kb = 0; + const uint64_t oneK = 1024; + const uint64_t percent = 90; // 90% of available memory + + // Lazily set total_mem_kb global + if ( ! total_mem_kb ) { + std::string token; + std::ifstream file("/proc/meminfo"); + while (file >> token) { + if (token == "MemFree:") { + unsigned long mem; + if (file >> mem) { + total_mem_kb = mem; + } else { + UMAP_ERROR("UMAP unable to determine system memory size\n"); + } + } + // ignore rest of the line + file.ignore(std::numeric_limits::max(), '\n'); + } + } + return ( ((total_mem_kb / (get_umap_page_size() / oneK)) * percent) / 100 ); +} + +void +RegionManager::set_max_pages_in_buffer( uint64_t max_pages ) +{ + uint64_t max_pages_in_mem = get_max_pages_in_memory(); + uint64_t old_max_pages_in_buffer = get_max_pages_in_buffer(); + + if ( max_pages > max_pages_in_mem ) { + UMAP_ERROR("Cannot set maximum pages to " + << max_pages + << " because it must be less than the maximum pages in memory " + << max_pages_in_mem); + } + + m_max_pages_in_buffer = max_pages; + + UMAP_LOG(Debug, + "Maximum pages in page buffer changed from " + << old_max_pages_in_buffer + << " to " << get_max_pages_in_buffer() << " pages"); +} + +void +RegionManager::set_read_ahead(uint64_t num_pages) +{ + m_read_ahead = num_pages; +} + +void +RegionManager::set_umap_page_size( uint64_t page_size ) +{ + // + // Must be multiple of system page size + // + if ( page_size % get_system_page_size() ) { + UMAP_ERROR("Specified page size (" << page_size + << ") must be a multiple of the system page size (" + << get_system_page_size() << ")"); + } + + UMAP_LOG(Debug, + "Adjusting page size from " + << get_umap_page_size() << " to " << page_size); + + m_umap_page_size = page_size; +} + +uint64_t* +RegionManager::read_env_var( const char* env, uint64_t* val ) +{ + // return a pointer to val on success, null on failure + char* val_ptr = 0; + if ( (val_ptr = getenv(env)) ) { + uint64_t env_val; + + std::string s(val_ptr); + std::stringstream ss(s); + ss >> env_val; + + if (env_val != 0) { + *val = env_val; + return val; + } + } + return nullptr; +} + +RegionDescriptor* +RegionManager::containing_region( char* vaddr ) +{ + // TODO: change this to judy array once implementation works properly + for ( auto it : m_active_regions ) { + char* b = it.second->start(); + char* e = it.second->end(); + if ( vaddr >= b && vaddr < e ) + return it.second; + } + + UMAP_ERROR("Unable to find addr: " << (void*)vaddr << " in region map"); + + return nullptr; +} + +void +RegionManager::set_num_fillers( uint64_t num_fillers ) +{ + m_num_fillers = num_fillers; +} + +void +RegionManager::set_num_evictors( uint64_t num_evictors ) +{ + m_num_evictors = num_evictors; +} +void +RegionManager::set_evict_high_water_threshold( int percent ) +{ + m_evict_high_water_threshold = percent; +} +void +RegionManager::set_evict_low_water_threshold( int percent ) +{ + m_evict_low_water_threshold = percent; +} +void +RegionManager::set_max_fault_events( uint64_t max_events ) +{ + m_max_fault_events = max_events; +} +} // end of namespace Umap diff --git a/src/umap/RegionManager.hpp b/src/umap/RegionManager.hpp new file mode 100644 index 00000000..4a84af73 --- /dev/null +++ b/src/umap/RegionManager.hpp @@ -0,0 +1,103 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#ifndef _UMAP_RegionManager_HPP +#define _UMAP_RegionManager_HPP + +#include +#include + +#include "umap/Buffer.hpp" +#include "umap/EvictManager.hpp" +#include "umap/FillWorkers.hpp" +#include "umap/Uffd.hpp" +#include "umap/umap.h" +#include "umap/store/Store.hpp" +#include "umap/RegionDescriptor.hpp" + +namespace Umap { +class FillWorkers; +class EvictManager; + +struct Version { + int major; + int minor; + int patch; +}; + +// +// Implemented as a singleton for now. Things can get too weird attempting to +// manage changes in configuration parameters when we have active monitors +// working. So, we only allow changing configuration when there are no active +// monitors +// +class RegionManager { + public: + static RegionManager* getInstance( void ); + + void addRegion( + Store* store + , char* region + , uint64_t region_size + , char* mmap_region + , uint64_t mmap_region_size + ); + + void prefetch(int npages, umap_prefetch_item* page_array); + void removeRegion( char* mmap_region ); + Version get_umap_version( void ) { return m_version; } + long get_system_page_size( void ) { return m_system_page_size; } + uint64_t get_max_pages_in_buffer( void ) { return m_max_pages_in_buffer; } + uint64_t get_read_ahead( void ) { return m_read_ahead; } + uint64_t get_umap_page_size( void ) { return m_umap_page_size; } + uint64_t get_num_fillers( void ) { return m_num_fillers; } + uint64_t get_num_evictors( void ) { return m_num_evictors; } + int get_evict_low_water_threshold( void ) { return m_evict_low_water_threshold; } + int get_evict_high_water_threshold( void ) { return m_evict_high_water_threshold; } + uint64_t get_max_fault_events( void ) { return m_max_fault_events; } + Buffer* get_buffer_h() { return m_buffer; } + Uffd* get_uffd_h() { return m_uffd; } + FillWorkers* get_fill_workers_h() { return m_fill_workers; } + EvictManager* get_evict_manager() { return m_evict_manager; } + RegionDescriptor* containing_region( char* vaddr ); + uint64_t get_num_active_regions( void ) { return (uint64_t)m_active_regions.size(); } + + private: + Version m_version; + uint64_t m_max_pages_in_buffer; + uint64_t m_read_ahead; + long m_umap_page_size; + uint64_t m_system_page_size; + uint64_t m_num_fillers; + uint64_t m_num_evictors; + int m_evict_low_water_threshold; + int m_evict_high_water_threshold; + uint64_t m_max_fault_events; + Buffer* m_buffer; + Uffd* m_uffd; + FillWorkers* m_fill_workers; + EvictManager* m_evict_manager; + + std::unordered_map m_active_regions; + + static RegionManager* s_fault_monitor_manager_instance; + + RegionManager( void ); + + uint64_t* read_env_var( const char* env, uint64_t* val); + uint64_t get_max_pages_in_memory( void ); + void set_max_fault_events( uint64_t max_events ); + void set_max_pages_in_buffer( uint64_t max_pages ); + void set_read_ahead(uint64_t num_pages); + void set_umap_page_size( uint64_t page_size ); + void set_num_fillers( uint64_t num_fillers ); + void set_num_evictors( uint64_t num_evictors ); + void set_evict_low_water_threshold( int percent ); + void set_evict_high_water_threshold( int percent ); +}; + +} // end of namespace Umap +#endif // _UMAP_RegionManager_HPP diff --git a/src/umap/Uffd.cpp b/src/umap/Uffd.cpp new file mode 100644 index 00000000..f1cabc93 --- /dev/null +++ b/src/umap/Uffd.cpp @@ -0,0 +1,315 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#include // sort() +#include // assert() +#include // uint64_t +#include +#include +#include // We all have lists to manage + +#include // strerror() +#include // O_CLOEXEC +#include // ioctl(UFFDIO_*) +#include // poll() +#include // strerror() +#include // ioctl() +#include // syscall() +#include // syscall() + +#include "umap/config.h" +#include "umap/Uffd.hpp" +#include "umap/RegionDescriptor.hpp" +#include "umap/RegionManager.hpp" +#include "umap/util/Macros.hpp" + +namespace Umap { + +struct less_than_key { + inline bool operator() ( const uffd_msg& lhs, const uffd_msg& rhs ) { + if (lhs.arg.pagefault.address == rhs.arg.pagefault.address) + return (lhs.arg.pagefault.flags > rhs.arg.pagefault.flags); + else + return (lhs.arg.pagefault.address < rhs.arg.pagefault.address); + } +}; + +void +Uffd::uffd_handler( void ) +{ + struct pollfd pollfd[3] = { + { .fd = m_uffd_fd, .events = POLLIN } + , { .fd = m_pipe[0], .events = POLLIN } + , { .fd = m_pipe[1], .events = POLLIN } + }; + + // + // For the Uffd worker thread, we use our work queue as a sentinel for + // when it is time to leave (since this particular thread gets its work + // from the m_uffd_fd kernel module. + // + while ( wq_is_empty() ) { + int pollres = poll(&pollfd[0], 3, -1); + + switch (pollres) { + case 1: + break; + case -1: + UMAP_ERROR("poll failed: " << strerror(errno)); + default: + UMAP_ERROR("poll: unexpected result: " << pollres); + } + + if (pollfd[1].revents & POLLIN || pollfd[2].revents & POLLIN) + break; + + if (pollfd[0].revents & POLLERR) + UMAP_ERROR("POLLERR: "); + + if ( !(pollfd[0].revents & POLLIN) ) + continue; + + int readres = read(m_uffd_fd, &m_events[0], m_max_fault_events * sizeof(struct uffd_msg)); + + if (readres == -1) { + if (errno == EAGAIN) + continue; + + UMAP_ERROR("read failed: " << strerror(errno)); + } + + assert("Invalid read result returned" && (readres % sizeof(struct uffd_msg) == 0)); + + int msgs = readres / sizeof(struct uffd_msg); + + assert("invalid message size" && msgs >= 1 && msgs < m_max_fault_events); + + // + // Since uffd page events arrive on the system page boundary which could + // be different from umap's page size, the page address for the incoming + // events are adjusted to the beginning of the umap page address. The + // events are then sorted in page base address / operation type order and + // are processed only once while duplicates are skipped. + // + for (int i = 0; i < msgs; ++i) + m_events[i].arg.pagefault.address &= ~(m_page_size-1); + + std::sort(&m_events[0], &m_events[msgs], less_than_key()); + + char* last_addr = nullptr; + for (int i = 0; i < msgs; ++i) { + if ((char*)(m_events[i].arg.pagefault.address) == last_addr) + continue; + + last_addr = (char*)(m_events[i].arg.pagefault.address); + +#ifndef UMAP_RO_MODE + bool iswrite = (m_events[i].arg.pagefault.flags & (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_WRITE) != 0); +#else + bool iswrite = false; +#endif + + // + // TODO: Since the addresses are sorted, we could optimize the + // search to continue from where it last found something. + // + process_page(iswrite, last_addr); + } + } + UMAP_LOG(Debug, "Good bye"); +} + +void +Uffd::process_page( bool iswrite, char* addr ) +{ + auto rd = m_rm->containing_region(addr); + if ( rd == nullptr ) + UMAP_ERROR("Invalid page: " << addr); + + m_buffer->process_page_event(addr, iswrite, rd); +} + +void +Uffd::ThreadEntry() +{ + uffd_handler(); +} + +Uffd::Uffd( void ) + : WorkerPool("Uffd Manager", 1) + , m_rm(RegionManager::getInstance()) + , m_max_fault_events(m_rm->get_max_fault_events()) + , m_page_size(m_rm->get_umap_page_size()) + , m_buffer(m_rm->get_buffer_h()) +{ + UMAP_LOG(Debug, "\n maximum fault events: " << m_max_fault_events + << "\n page size: " << m_page_size); + + if ((m_uffd_fd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK)) < 0) + UMAP_ERROR("userfaultfd syscall not available in this kernel: " + << strerror(errno)); + + if (pipe2(m_pipe, 0) < 0) + UMAP_ERROR("userfaultfd pipe failed: " << strerror(errno)); + + check_uffd_compatibility(); + m_events.resize(m_max_fault_events); + + start_thread_pool(); +} + +Uffd::~Uffd() +{ + char bye[5] = "bye"; + + write(m_pipe[1], bye, 3); + + stop_thread_pool(); +} + +void +Uffd::enable_write_protect( + void* +#ifndef UMAP_RO_MODE + page_address +#endif + ) +{ +#ifndef UMAP_RO_MODE + struct uffdio_writeprotect wp = { + .range = { .start = (uint64_t)page_address, .len = m_page_size } + , .mode = UFFDIO_WRITEPROTECT_MODE_WP + }; + + if (ioctl(m_uffd_fd, UFFDIO_WRITEPROTECT, &wp) == -1) + UMAP_ERROR("ioctl(UFFDIO_WRITEPROTECT): " << strerror(errno)); +#endif // UMAP_RO_MODE +} + +void +Uffd::disable_write_protect( + void* +#ifndef UMAP_RO_MODE + page_address +#endif +) +{ +#ifndef UMAP_RO_MODE + struct uffdio_writeprotect wp = { + .range = { .start = (uint64_t)page_address, .len = m_page_size } + , .mode = 0 + }; + + if (ioctl(m_uffd_fd, UFFDIO_WRITEPROTECT, &wp) == -1) + UMAP_ERROR("ioctl(UFFDIO_WRITEPROTECT): " << strerror(errno)); +#endif // UMAP_RO_MODE +} + +void +Uffd::copy_in_page(char* data, void* page_address) +{ + struct uffdio_copy copy = { + .dst = (uint64_t)page_address + , .src = (uint64_t)data + , .len = m_page_size + , .mode = 0 + }; + + if (ioctl(m_uffd_fd, UFFDIO_COPY, ©) == -1) + UMAP_ERROR("UFFDIO_COPY failed: " << strerror(errno)); +} + +void +Uffd::copy_in_page_and_write_protect(char* data, void* page_address) +{ + UMAP_LOG(Debug, "(page_address = " << page_address << ")"); + struct uffdio_copy copy = { + .dst = (uint64_t)page_address + , .src = (uint64_t)data + , .len = m_page_size +#ifndef UMAP_RO_MODE + , .mode = UFFDIO_COPY_MODE_WP +#else + , .mode = 0 +#endif + }; + + if (ioctl(m_uffd_fd, UFFDIO_COPY, ©) == -1) + UMAP_ERROR("UFFDIO_COPY failed: " << strerror(errno)); +} + +void +Uffd::register_region( RegionDescriptor* rd ) +{ + struct uffdio_register uffdio_register = { + .range = { .start = (__u64)(rd->start()), .len = rd->size() } +#ifndef UMAP_RO_MODE + , .mode = UFFDIO_REGISTER_MODE_MISSING | UFFDIO_REGISTER_MODE_WP +#else + , .mode = UFFDIO_REGISTER_MODE_MISSING +#endif + }; + + UMAP_LOG(Debug, + "Registering " << (uffdio_register.range.len / m_page_size) + << " pages from: " << (void*)(uffdio_register.range.start) + << " - " << (void*)(uffdio_register.range.start + + (uffdio_register.range.len-1))); + + if (ioctl(m_uffd_fd, UFFDIO_REGISTER, &uffdio_register) == -1) + UMAP_ERROR("ioctl(UFFDIO_REGISTER) failed: " << strerror(errno)); + + if ((uffdio_register.ioctls & UFFD_API_RANGE_IOCTLS) != UFFD_API_RANGE_IOCTLS) + UMAP_ERROR("unexpected userfaultfd ioctl set: " << uffdio_register.ioctls); +} + +void +Uffd::unregister_region( RegionDescriptor* rd ) +{ + // + // Make sure and evict any/all active pages from this region that are still + // in the Buffer + // + m_buffer->evict_region(rd); + + struct uffdio_register uffdio_register = { + .range = { .start = (__u64)(rd->start()), .len = rd->size() } + , .mode = 0 + }; + + UMAP_LOG(Debug, + "Unregistering " << (uffdio_register.range.len / m_page_size) + << " pages from: " << (void*)(uffdio_register.range.start) + << " - " << (void*)(uffdio_register.range.start + + (uffdio_register.range.len-1))); + + if (ioctl(m_uffd_fd, UFFDIO_UNREGISTER, &uffdio_register.range)) + UMAP_ERROR("ioctl(UFFDIO_UNREGISTER) failed: " << strerror(errno)); +} + +void +Uffd::check_uffd_compatibility( void ) +{ + struct uffdio_api uffdio_api = { + .api = UFFD_API +#ifdef UMAP_RO_MODE + , .features = 0 +#else + , .features = UFFD_FEATURE_PAGEFAULT_FLAG_WP +#endif + + , .ioctls = 0 + }; + +if (ioctl(m_uffd_fd, UFFDIO_API, &uffdio_api) == -1) + UMAP_ERROR("ioctl(UFFDIO_API) Failed: " << strerror(errno)); + +#ifndef UMAP_RO_MODE +if ( !(uffdio_api.features & UFFD_FEATURE_PAGEFAULT_FLAG_WP) ) + UMAP_ERROR("UFFD Compatibilty Check - unsupported userfaultfd WP"); +#endif +} +} // end of namespace Umap diff --git a/src/umap/Uffd.hpp b/src/umap/Uffd.hpp new file mode 100644 index 00000000..0b52f0e3 --- /dev/null +++ b/src/umap/Uffd.hpp @@ -0,0 +1,74 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#ifndef _UMAP_Uffd_HPP +#define _UMAP_Uffd_HPP + +#include // sort() +#include // assert() +#include // uint64_t +#include +#include +#include // We all have lists to manage + +#include // strerror() +#include // O_CLOEXEC +#include // ioctl(UFFDIO_*) +#include // poll() +#include // strerror() +#include // ioctl() +#include // syscall() +#include // syscall() + +#include "umap/config.h" +// +// The UFFDIO_COPY_MODE_WP is only defined in later versions of Linux (>5.0) +// +#ifndef UFFDIO_COPY_MODE_WP +#define UMAP_RO_MODE +#endif + +#include "umap/RegionDescriptor.hpp" +#include "umap/RegionManager.hpp" +#include "umap/WorkerPool.hpp" + +namespace Umap { + class RegionManager; + + class PageEvent { + public: + PageEvent(void* paddr, bool iswrite); + }; + + class Uffd : public WorkerPool { + public: + Uffd( void ); + ~Uffd( void); + + void process_page(bool iswrite, char* addr ); + void register_region( RegionDescriptor* region ); + void unregister_region( RegionDescriptor* region ); + + void enable_write_protect( void* ); + void disable_write_protect( void* ); + void copy_in_page(char* data, void* page_address); + void copy_in_page_and_write_protect(char* data, void* page_address); + + private: + RegionManager* m_rm; + uint64_t m_max_fault_events; + uint64_t m_page_size; + Buffer* m_buffer; + int m_uffd_fd; + int m_pipe[2]; + std::vector m_events; + + void uffd_handler( void ); + void ThreadEntry( void ); + void check_uffd_compatibility( void ); + }; +} // end of namespace Umap +#endif // _UMAP_Uffd_HPP diff --git a/src/umap/WorkQueue.hpp b/src/umap/WorkQueue.hpp new file mode 100644 index 00000000..b890acff --- /dev/null +++ b/src/umap/WorkQueue.hpp @@ -0,0 +1,98 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#ifndef _UMAP_WorkQueue_HPP +#define _UMAP_WorkQueue_HPP + +#include + +#include +#include +#include + +#include "umap/Uffd.hpp" +#include "umap/store/Store.hpp" +#include "umap/util/Macros.hpp" + +namespace Umap { +template +class WorkQueue { + public: + WorkQueue(int max_workers) + : m_max_waiting(max_workers) + , m_waiting_workers(0) + , m_idle_waiters(0) + { + pthread_mutex_init(&m_mutex, NULL); + pthread_cond_init(&m_cond, NULL); + pthread_cond_init(&m_idle_cond, NULL); + } + + ~WorkQueue() { + pthread_mutex_destroy(&m_mutex); + pthread_cond_destroy(&m_cond); + pthread_cond_destroy(&m_idle_cond); + } + + void enqueue(T item) { + pthread_mutex_lock(&m_mutex); + m_queue.push_back(item); + pthread_cond_signal(&m_cond); + pthread_mutex_unlock(&m_mutex); + } + + T dequeue() { + pthread_mutex_lock(&m_mutex); + + ++m_waiting_workers; + + while ( m_queue.size() == 0 ) { + if (m_waiting_workers == m_max_waiting && m_idle_waiters) + pthread_cond_signal(&m_idle_cond); + + pthread_cond_wait(&m_cond, &m_mutex); + } + + --m_waiting_workers; + + auto item = m_queue.front(); + m_queue.pop_front(); + + pthread_mutex_unlock(&m_mutex); + return item; + } + + void wait_for_idle( void ) { + pthread_mutex_lock(&m_mutex); + ++m_idle_waiters; + + while ( m_waiting_workers != m_max_waiting ) + pthread_cond_wait(&m_idle_cond, &m_mutex); + + --m_idle_waiters; + pthread_mutex_unlock(&m_mutex); + } + + bool is_empty() { + pthread_mutex_lock(&m_mutex); + bool empty = (m_queue.size() == 0); + pthread_mutex_unlock(&m_mutex); + return empty; + } + + private: + pthread_mutex_t m_mutex; + pthread_cond_t m_cond; + pthread_cond_t m_idle_cond; + std::list m_queue; + uint64_t m_max_waiting; + uint64_t m_waiting_workers; + int m_idle_waiters; +}; + +} // end of namespace Umap + +#endif // _UMAP_WorkQueue_HPP diff --git a/src/umap/WorkerPool.hpp b/src/umap/WorkerPool.hpp new file mode 100644 index 00000000..c17519b7 --- /dev/null +++ b/src/umap/WorkerPool.hpp @@ -0,0 +1,130 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#ifndef _UMAP_Pthread_HPP +#define _UMAP_Pthread_HPP + +#include +#include +#include +#include + +#include "umap/PageDescriptor.hpp" +#include "umap/WorkQueue.hpp" +#include "umap/util/Macros.hpp" + +namespace Umap { + struct WorkItem { + enum WorkType { NONE, EXIT, THRESHOLD, EVICT, FAST_EVICT }; + PageDescriptor* page_desc; + WorkType type; + }; + + static std::ostream& operator<<(std::ostream& os, const Umap::WorkItem& b) + { + os << "{ page_desc: " << b.page_desc; + + switch (b.type) { + default: os << ", type: Unknown(" << b.type << ")"; break; + case Umap::WorkItem::WorkType::NONE: os << ", type: " << "NONE"; break; + case Umap::WorkItem::WorkType::EXIT: os << ", type: " << "EXIT"; break; + case Umap::WorkItem::WorkType::THRESHOLD: os << ", type: " << "THRESHOLD"; break; + case Umap::WorkItem::WorkType::EVICT: os << ", type: " << "EVICT"; break; + case Umap::WorkItem::WorkType::FAST_EVICT: os << ", type: " << "FAST_EVICT"; break; + } + + os << " }"; + return os; + } + + class WorkerPool { + public: + WorkerPool(const std::string& pool_name, uint64_t num_threads) + : m_pool_name(pool_name) + , m_num_threads(num_threads) + , m_wq(new WorkQueue(num_threads)) + { + if (m_pool_name.length() > 15) + m_pool_name.resize(15); + } + + virtual ~WorkerPool() { + stop_thread_pool(); + delete m_wq; + } + + void send_work(const WorkItem& work) { + m_wq->enqueue(work); + } + + WorkItem get_work() { + return m_wq->dequeue(); + } + + bool wq_is_empty( void ) { + return m_wq->is_empty(); + } + + void start_thread_pool() { + UMAP_LOG(Debug, "Starting " << m_pool_name << " Pool of " + << m_num_threads << " threads"); + + for ( uint64_t i = 0; i < m_num_threads; ++i) { + pthread_t t; + + if (pthread_create(&t, NULL, ThreadEntryFunc, this) != 0) + UMAP_ERROR("Failed to launch thread"); + + if (pthread_setname_np(t, m_pool_name.c_str()) != 0) + UMAP_ERROR("Failed to set thread name"); + + m_threads.push_back(t); + } + } + + void stop_thread_pool() { + UMAP_LOG(Debug, "Stopping " << m_pool_name << " Pool of " + << m_num_threads << " threads"); + + WorkItem w = {.page_desc = nullptr, .type = Umap::WorkItem::WorkType::EXIT }; + + // + // This will inform all of the threads it is time to go away + // + for ( uint64_t i = 0; i < m_num_threads; ++i) + send_work(w); + + // + // Wait for all of the threads to exit + // + for ( auto pt : m_threads ) + (void) pthread_join(pt, NULL); + + m_threads.clear(); + + UMAP_LOG(Debug, m_pool_name << " stopped"); + } + + void wait_for_idle( void ) { + m_wq->wait_for_idle(); + } + + protected: + virtual void ThreadEntry() = 0; + + private: + static void* ThreadEntryFunc(void * This) { + ((WorkerPool *)This)->ThreadEntry(); + return NULL; + } + + std::string m_pool_name; + uint64_t m_num_threads; + WorkQueue* m_wq; + std::vector m_threads; + }; +} // end of namespace Umap +#endif // _UMAP_WorkerPool_HPP diff --git a/src/store/Store.cpp b/src/umap/store/Store.cpp similarity index 60% rename from src/store/Store.cpp rename to src/umap/store/Store.cpp index 45441a0a..f707d22c 100644 --- a/src/store/Store.cpp +++ b/src/umap/store/Store.cpp @@ -5,10 +5,12 @@ // SPDX-License-Identifier: LGPL-2.1-only ////////////////////////////////////////////////////////////////////////////// #include "umap/umap.h" -#include "umap/Store.h" -#include "StoreFile.h" +#include "umap/store/Store.hpp" +#include "umap/store/StoreFile.h" -Store* Store::make_store(void* _region_, size_t _rsize_, size_t _alignsize_, int _fd_) -{ - return new StoreFile{_region_, _rsize_, _alignsize_, _fd_}; +namespace Umap { + Store* Store::make_store(void* _region_, size_t _rsize_, size_t _alignsize_, int _fd_) + { + return new StoreFile{_region_, _rsize_, _alignsize_, _fd_}; + } } diff --git a/src/include/umap/Store.h b/src/umap/store/Store.hpp similarity index 94% rename from src/include/umap/Store.h rename to src/umap/store/Store.hpp index a8fa3e5b..53fc5992 100644 --- a/src/include/umap/Store.h +++ b/src/umap/store/Store.hpp @@ -6,9 +6,10 @@ ////////////////////////////////////////////////////////////////////////////// #ifndef _UMAP_STORE_H_ #define _UMAP_STORE_H_ -#include #include +#include +namespace Umap { class Store { public: static Store* make_store(void* _region_, std::size_t _rsize_, std::size_t _alignsize_, int _fd_); @@ -16,4 +17,5 @@ class Store { virtual ssize_t read_from_store(char* buf, std::size_t nb, off_t off) = 0; virtual ssize_t write_to_store(char* buf, std::size_t nb, off_t off) = 0; }; +} // end of namespace Umap #endif diff --git a/src/umap/store/StoreFile.cpp b/src/umap/store/StoreFile.cpp new file mode 100644 index 00000000..01b96e98 --- /dev/null +++ b/src/umap/store/StoreFile.cpp @@ -0,0 +1,60 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#include +#include +#include "StoreFile.h" +#include +#include +#include + +#include "umap/store/Store.hpp" +#include "umap/util/Macros.hpp" + +namespace Umap { + StoreFile::StoreFile(void* _region_, size_t _rsize_, size_t _alignsize_, int _fd_) + : region{_region_}, rsize{_rsize_}, alignsize{_alignsize_}, fd{_fd_} + { + UMAP_LOG(Debug, + "region: " << region << " rsize: " << rsize + << " alignsize: " << alignsize << " fd: " << fd); + } + + ssize_t StoreFile::read_from_store(char* buf, size_t nb, off_t off) + { + size_t rval = 0; + + UMAP_LOG(Debug, "pread(fd=" << fd << ", buf=" << (void*)buf + << ", nb=" << nb << ", off=" << off << ")";); + + rval = pread(fd, buf, nb, off); + + if (rval == -1) { + int eno = errno; + UMAP_ERROR("pread(fd=" << fd << ", buf=" << (void*)buf + << ", nb=" << nb << ", off=" << off + << "): Failed - " << strerror(eno)); + } + return rval; + } + + ssize_t StoreFile::write_to_store(char* buf, size_t nb, off_t off) + { + size_t rval = 0; + + UMAP_LOG(Debug, "pwrite(fd=" << fd << ", buf=" << (void*)buf + << ", nb=" << nb << ", off=" << off << ")";); + + rval = pwrite(fd, buf, nb, off); + if (rval == -1) { + int eno = errno; + UMAP_ERROR("pwrite(fd=" << fd << ", buf=" << (void*)buf + << ", nb=" << nb << ", off=" << off + << "): Failed - " << strerror(eno)); + } + return rval; + } +} diff --git a/src/store/StoreFile.h b/src/umap/store/StoreFile.h similarity index 51% rename from src/store/StoreFile.h rename to src/umap/store/StoreFile.h index 0530b50f..32edfc15 100644 --- a/src/store/StoreFile.h +++ b/src/umap/store/StoreFile.h @@ -7,20 +7,22 @@ #ifndef _UMAP_STORE_FILE_H_ #define _UMAP_STORE_FILE_H_ #include -#include "umap/Store.h" +#include "umap/store/Store.hpp" #include "umap/umap.h" -class StoreFile : public Store { - public: - StoreFile(void* _region_, size_t _rsize_, size_t _alignsize_, int _fd_); +namespace Umap { + class StoreFile : public Store { + public: + StoreFile(void* _region_, size_t _rsize_, size_t _alignsize_, int _fd_); - ssize_t read_from_store(char* buf, size_t nb, off_t off); - ssize_t write_to_store(char* buf, size_t nb, off_t off); - private: - void* region; - void* alignment_buffer; - size_t rsize; - size_t alignsize; - int fd; -}; + ssize_t read_from_store(char* buf, size_t nb, off_t off); + ssize_t write_to_store(char* buf, size_t nb, off_t off); + private: + void* region; + void* alignment_buffer; + size_t rsize; + size_t alignsize; + int fd; + }; +} #endif diff --git a/src/umap/umap.cpp b/src/umap/umap.cpp index d9ef6043..268709ca 100644 --- a/src/umap/umap.cpp +++ b/src/umap/umap.cpp @@ -4,979 +4,179 @@ // // SPDX-License-Identifier: LGPL-2.1-only ////////////////////////////////////////////////////////////////////////////// -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif // _GNU_SOURCE -#include #include -#include -#include -#include -#include -#include -#include -#include -#include // open/close -#include // syscall() -#include // poll() -#include -#include -#include -#include "umap/umap.h" // API to library -#include "umap/Store.h" -#include "config.h" -#include "spindle_debug.h" - -#ifndef UFFDIO_COPY_MODE_WP -#define UMAP_RO_MODE -#endif - -/* - * Note: this implementation is multi-threaded, but the data structures are - * not shared between threads. - */ - -const int umap_Version_Major = UMAP_VERSION_MAJOR; -const int umap_Version_Minor = UMAP_VERSION_MINOR; -const int umap_Version_Patch = UMAP_VERSION_PATCH; - -static const int UMAP_UFFD_MAX_MESSAGES = 256; -static uint64_t uffd_threads; -static uint64_t umap_buffer_size; - -static long umapPageSize; - -class umap_page; -struct umap_PageBlock; -class umap_page_buffer; -class umap_stats; -class UserFaultHandler; - -// -// |------------------------- umap() provided Region ----------------------------| -// |------------------------- umap() provided backing file(s) -------------------| -// |- Page Block 1 -|- Page Block 2 -|- ... -|- Page Block N-1 -|- Page Block N -| -// -// _umap organizes a region of memory into a set of blocks of pages. The blocks -// of pages are then distributed evenly to a set of UserFaultHandler objects. -// -class _umap { - friend UserFaultHandler; - public: - _umap(void* _mmap_region, uint64_t _mmap_rsize, - void* _umap_region, uint64_t _umap_rsize, - int fd, Store* _store_); - ~_umap(); - - static inline void* UMAP_PAGE_BEGIN(const void* a) { - return (void*)( (uint64_t)a & ~(umapPageSize-1) ); - } - - void flushbuffers( void ); - - std::vector ufault_handlers; - - private: - void* mmapRegion; - uint64_t mmapRegionSize; - void* umapRegion; - uint64_t umapRegionSize; - bool uffd_time_to_stop_working; - Store* store; -}; - -class UserFaultHandler { - friend _umap; - friend umap_page_buffer; - public: - UserFaultHandler(_umap* _um, const std::vector& _pblks, uint64_t _pbuf_size); - ~UserFaultHandler(void); - void stop_uffd_worker( void ) noexcept { - _u->uffd_time_to_stop_working = true; - uffd_worker->join(); - }; - bool page_is_in_umap(const void* page_begin); - umap_page_buffer* get_pagebuffer() { return pagebuffer; } - void flushbuffers( void ); - void resetstats( void ); - - umap_stats* stat; - private: - _umap* _u; - std::vector PageBlocks; - uint64_t pbuf_size; - umap_page_buffer* pagebuffer; - std::vector umessages; - - int userfault_fd; - char* copyin_buf; - std::thread* uffd_worker; - - void evict_page(umap_page* page); - void uffd_handler(void); - void pagefault_event(const struct uffd_msg& msg); -#ifndef UMAP_RO_MODE - void enable_wp_on_pages_and_wake(uint64_t, int64_t); - void disable_wp_on_pages(uint64_t, int64_t, bool); -#endif -}; - -class umap_stats { - public: - umap_stats(): - dirty_evicts{0}, - evict_victims{0}, - wp_messages{0}, - read_faults{0}, - write_faults{0} - {}; - - uint64_t dirty_evicts; - uint64_t evict_victims; - uint64_t wp_messages; - uint64_t read_faults; - uint64_t write_faults; -}; - -struct umap_PageBlock { - void* base; - uint64_t length; -}; - -class umap_page_buffer { - public: - umap_page_buffer(UserFaultHandler* _ufh_, uint64_t pbuffersize); - ~umap_page_buffer(); - umap_page* alloc_page_desc(void* page); - void dealloc_page_desc( void ); - bool pages_still_present( void ); - - umap_page* find_inmem_page_desc(void* page_addr); - - private: - uint64_t page_buffer_size; - uint64_t page_buffer_alloc_idx; - uint64_t page_buffer_free_idx; - uint64_t page_buffer_alloc_cnt; - std::unordered_map inmem_page_map; - umap_page* page_descriptor_array; - UserFaultHandler* ufh; -}; - -struct umap_page { - bool page_is_dirty() { return dirty; } - void mark_page_dirty() { dirty = true; } - void mark_page_clean() { dirty = false; } - void* get_page(void) { return page; } - void set_page(void* _p) { page = _p; } - void* page; - bool dirty; -}; - -static std::unordered_map active_umaps; - -static inline bool required_uffd_features_present(int fd) -{ - struct uffdio_api uffdio_api = { - .api = UFFD_API, -#ifdef UMAP_RO_MODE - .features = 0, -#else - .features = UFFD_FEATURE_PAGEFAULT_FLAG_WP, -#endif - .ioctls = 0 - }; - - if (ioctl(fd, UFFDIO_API, &uffdio_api) == -1) { - perror("ERROR: UFFDIO_API Failed: "); - return false; - } - -#ifndef UMAP_RO_MODE - if ( !(uffdio_api.features & UFFD_FEATURE_PAGEFAULT_FLAG_WP) ) { - std::cerr << "UFFD Compatibilty Check - unsupported userfaultfd WP\n"; - return false; - } -#endif - - return true; -} - -// -// Library Interface Entry -// -static int check_uffd_compatibility( void ) -{ - int fd; - - if ((fd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK)) < 0) { - perror("UFFD Compatibilty Check - Unable to open userfaultfd: "); - exit(1); - } - - if ( ! required_uffd_features_present(fd) ) - exit(1); - - close(fd); - +#include // strerror() +#include // strerror() +#include + +#include "umap/config.h" + +#include "umap/RegionManager.hpp" +#include "umap/umap.h" +#include "umap/store/Store.hpp" +#include "umap/util/Macros.hpp" + +void* +umap( + void* region_addr + , uint64_t region_size + , int prot + , int flags + , int fd + , off_t offset +) +{ + UMAP_LOG(Debug, + "region_addr: " << region_addr + << ", region_size: " << region_size + << ", prot: " << prot + << ", flags: " << flags + << ", offset: " << offset + ); + return Umap::umap_ex(region_addr, region_size, prot, flags, fd, 0, nullptr); +} + +int +uunmap(void* addr, uint64_t length) +{ + UMAP_LOG(Debug, "addr: " << addr << ", length: " << length); + auto rm = Umap::RegionManager::getInstance(); + rm->removeRegion((char*)addr); + UMAP_LOG(Debug, "Done"); return 0; } -static inline long get_max_buf_size( void ) -{ static unsigned long total_mem_kb = 0; - const unsigned long oneK = 1024; - const unsigned long percentageToAllocate = 80; // 80% of memory is max - - // Lazily set total_mem_kb global - if ( ! total_mem_kb ) { - std::string token; - std::ifstream file("/proc/meminfo"); - while (file >> token) { - if (token == "MemTotal:") { - unsigned long mem; - if (file >> mem) { - total_mem_kb = mem; - } else { - std::cerr << "UMAP unable to determine system memory size\n"; - total_mem_kb = oneK * oneK; - } - } - // ignore rest of the line - file.ignore(std::numeric_limits::max(), '\n'); - } - } - return ((total_mem_kb / (umapPageSize / oneK)) * percentageToAllocate) / 100; -} - -void* umap(void* base_addr, uint64_t region_size, int prot, int flags, - int fd, off_t offset) +void umap_prefetch( int npages, umap_prefetch_item* page_array ) { - return umap_ex(base_addr, region_size, prot, flags, fd, 0, nullptr); + Umap::RegionManager::getInstance()->prefetch(npages, page_array); } -void* umap_ex(void* base_addr, uint64_t region_size, int prot, int flags, - int fd, off_t offset, Store* _store_) +long +umapcfg_get_system_page_size( void ) { - if (check_uffd_compatibility() < 0) - return NULL; - - if ( (region_size % umapPageSize) ) { - std::cerr << "UMAP: Region size " << region_size << " is not a multple of umapPageSize (" << umapPageSize << ")\n"; - return NULL; - } - - if ( ((uint64_t)base_addr & (umapPageSize - 1)) ) { - std::cerr << "umap: base_addr must be page aligned: " << base_addr - << ", page size is: " << umapPageSize << std::endl; - return NULL; - } - - if (!(flags & UMAP_PRIVATE) || flags & ~(UMAP_PRIVATE|UMAP_FIXED)) { - std::cerr << "umap: Invalid flags: " << std::hex << flags << std::endl; - return UMAP_FAILED; - } - - // - // When dealing with umap-page-sizes that could be multiples of the actual - // system-page-size, it is possible for mmap() to provide a region that is on - // a system-page-boundary, but not necessarily on a umap-page-size boundary. - // - // We always allocate an additional umap-page-size set of bytes so that we can - // make certain that the umap-region begins on a umap-page-size boundary. - // - uint64_t mmap_size = region_size + umapPageSize; - - void* mmap_region = mmap(base_addr, mmap_size, - prot, flags | (MAP_ANONYMOUS | MAP_NORESERVE), -1, 0); - - if (mmap_region == MAP_FAILED) { - perror("ERROR: mmap failed: "); - return UMAP_FAILED; - } - void* umap_region = _umap::UMAP_PAGE_BEGIN((void*)((uint64_t)mmap_region + (umapPageSize-1))); - uint64_t umap_size = region_size; - - try { - active_umaps[umap_region] = new _umap{mmap_region, mmap_size, - umap_region, umap_size, fd, _store_}; - } catch(const std::exception& e) { - std::cerr << __FUNCTION__ << " Failed to launch _umap: " << e.what() << std::endl; - return UMAP_FAILED; - } catch(...) { - std::cerr << "umap failed to instantiate _umap object\n"; - return UMAP_FAILED; - } - return umap_region; + return Umap::RegionManager::getInstance()->get_system_page_size(); } -int uunmap(void* addr, uint64_t length) +uint64_t +umapcfg_get_max_pages_in_buffer( void ) { - auto it = active_umaps.find(addr); - - if (it != active_umaps.end()) { - struct umap_cfg_stats st; - umap_cfg_get_stats(addr, &st); - - debug_printf( "\n\t" - "Dirty Evictions: %" PRIu64 "\n\t" - " Evict Victims: %" PRIu64 "\n\t" - " WP Messages: %" PRIu64 "\n\t" - " Read Faults: %" PRIu64 "\n\t" - " Write Faults: %" PRIu64 "\n", - st.dirty_evicts, - st.evict_victims, - st.wp_messages, - st.read_faults, - st.write_faults); - - delete it->second; - active_umaps.erase(it); - } - return 0; + return Umap::RegionManager::getInstance()->get_max_pages_in_buffer(); } -uint64_t* umap_cfg_readenv(const char* env, uint64_t* val) { - // return a pointer to val on success, null on failure - char* val_ptr = 0; - if ( (val_ptr = getenv(env)) ) { - uint64_t env_val = 0; - if (sscanf(val_ptr, "%" PRIu64, &env_val)) { - *val = env_val; - return val; - } - } - return 0; -} - -void umap_cfg_getenv( void ) { - uint64_t env_value = 0; - if ( (umap_cfg_readenv("UMAP_UFFD_THREADS", &env_value)) ) { - umap_cfg_set_uffdthreads(env_value); - } - - if ( (umap_cfg_readenv("UMAP_BUFSIZE", &env_value)) ) { - umap_cfg_set_bufsize(env_value); - } - - if ( (umap_cfg_readenv("UMAP_PAGESIZE", &env_value)) ) { - umap_cfg_set_pagesize(env_value); - } -} - -uint64_t umap_cfg_get_bufsize( void ) +uint64_t +umapcfg_get_read_ahead( void ) { - return umap_buffer_size; + return Umap::RegionManager::getInstance()->get_read_ahead(); } -void umap_cfg_set_bufsize( uint64_t page_bufsize ) +uint64_t +umapcfg_get_umap_page_size( void ) { - uint64_t max_size = get_max_buf_size(); - uint64_t old_size = umap_buffer_size; - - if ( page_bufsize > max_size ) { - debug_printf("Bufsize of %" PRIu64 " larger than maximum of %ld. Setting to %ld\n", - page_bufsize, max_size, max_size); - umap_buffer_size = max_size; - } - else { - umap_buffer_size = page_bufsize; - } - debug_printf("Bufsize changed from %ld to %lu pages\n", old_size, umap_buffer_size); -} - -uint64_t umap_cfg_get_uffdthreads( void ) -{ - return uffd_threads; + return Umap::RegionManager::getInstance()->get_umap_page_size(); } -void umap_cfg_set_uffdthreads( uint64_t numthreads ) +uint64_t +umapcfg_get_num_fillers( void ) { - uffd_threads = numthreads; + return Umap::RegionManager::getInstance()->get_num_fillers(); } -void umap_cfg_flush_buffer( void* region ) +uint64_t +umapcfg_get_num_evictors( void ) { - auto it = active_umaps.find(region); - - if (it != active_umaps.end()) - it->second->flushbuffers(); + return Umap::RegionManager::getInstance()->get_num_evictors(); } -int umap_cfg_get_pagesize() +int +umapcfg_get_evict_low_water_threshold( void ) { - return umapPageSize; + return Umap::RegionManager::getInstance()->get_evict_low_water_threshold(); } -int umap_cfg_set_pagesize( long psize ) +int +umapcfg_get_evict_high_water_threshold( void ) { - long sys_psize = sysconf(_SC_PAGESIZE); - - /* - * Must be multiple of system page size - */ - if ( psize % sys_psize ) { - std::cerr << "Specified page size (" << psize << ") must be a multiple of system page size (" << sys_psize << ")\n"; - return -1; - } - - debug_printf("Adjusting page size from %ld to %ld\n", umapPageSize, psize); - - umapPageSize = psize; - return 0; -} - -void umap_cfg_get_stats(void* region, struct umap_cfg_stats* stats) -{ - auto it = active_umaps.find(region); - - if (it != active_umaps.end()) { - stats->dirty_evicts = 0; - stats->evict_victims = 0; - stats->wp_messages = 0; - stats->read_faults = 0; - stats->write_faults = 0; - - for ( auto handler : it->second->ufault_handlers ) { - stats->dirty_evicts += handler->stat->dirty_evicts; - stats->evict_victims += handler->stat->evict_victims; - stats->wp_messages += handler->stat->wp_messages; - stats->read_faults += handler->stat->read_faults; - stats->write_faults += handler->stat->write_faults; - } - } -} - -void umap_cfg_reset_stats(void* region) -{ - auto it = active_umaps.find(region); - - if (it != active_umaps.end()) { - for ( auto handler : it->second->ufault_handlers ) - handler->resetstats(); - } -} - -void __attribute ((constructor)) init_umap_lib( void ) -{ - LOGGING_INIT; - - if ((umapPageSize = sysconf(_SC_PAGESIZE)) == -1) { - perror("ERROR: sysconf(_SC_PAGESIZE)"); - throw -1; - } - - umap_buffer_size = get_max_buf_size(); - - unsigned int n = std::thread::hardware_concurrency(); - uffd_threads = (n == 0) ? 16 : n; - - umap_cfg_getenv(); - LOGGING_FINI; -} - -void __attribute ((destructor)) fine_umap_lib( void ) -{ - for (auto it : active_umaps) - delete it.second; -} - -// -// _umap class implementation -// -_umap::_umap( void* _mmap_region, - uint64_t _mmap_rsize, - void* _umap_region, - uint64_t _umap_rsize, - int fd, - Store* _store_) : - mmapRegion{_mmap_region}, mmapRegionSize{_mmap_rsize}, - umapRegion{_umap_region}, umapRegionSize{_umap_rsize}, - uffd_time_to_stop_working{false}, store{_store_} -{ - if ( store == nullptr ) - store = Store::make_store(umapRegion, umapRegionSize, umapPageSize, fd); - - uint64_t region_pages = umapRegionSize / umapPageSize; - - // Shrink buffer size to fit requested region if needed - uint64_t buffer_adjusted_pages = std::min(umap_buffer_size, region_pages); - - // Shrink # of workers if there are too few pages to make it worth it. - uint64_t num_workers = std::min(buffer_adjusted_pages, uffd_threads); - - uint64_t buffer_pages_per_worker = buffer_adjusted_pages / num_workers; - uint64_t buffer_residual_pages = buffer_adjusted_pages % num_workers; - - uint64_t region_pages_per_worker = region_pages / num_workers; - uint64_t region_residual_pages = region_pages % num_workers; - -#ifdef UMAP_DEBUG_LOGGING - std::stringstream ss; - ss << "umap(" - << umapRegion << " - " << (void*)((char*)umapRegion+umapRegionSize) << ")\n\t" - << umapPageSize << " Page Size\n\t" - << umap_buffer_size << " UMAP Buffer Size in Pages\n\t" - << region_pages << " Requested Region Pages\n\t" - << buffer_adjusted_pages << " Adjusted UMAP Buffer Size in Pages\n\t" - << uffd_threads << " Configured Maximum UMAP Threads\n\t" - << num_workers << " UMAP Threads Allocated\n\t" - << buffer_pages_per_worker << " Buffer Pages per worker\n\t" - << buffer_residual_pages << " Residual Buffer pages\n\t" - << region_pages_per_worker << " Region Pages per worker\n\t" - << region_residual_pages << " Risidual Buffer pages" - << std::endl; - debug_printf("%s\n", ss.str().c_str()); -#endif - - try { - uint64_t region_offset = 0; - for (uint64_t worker = 0; worker < num_workers; ++worker) { - umap_PageBlock pb; - uint64_t worker_region_pages = region_pages_per_worker; - uint64_t worker_buffer_pages = buffer_pages_per_worker; - - // - // Distribute residual buffer pages across workers - // - if (buffer_residual_pages) { - buffer_residual_pages--; - worker_buffer_pages++; - } - - // - // Distribute residual buffer pages across workers - // - if (region_residual_pages) { - region_residual_pages--; - worker_region_pages++; - } - - pb.base = (void*)((uint64_t)umapRegion + (region_offset * umapPageSize)); - pb.length = worker_region_pages * umapPageSize; - - std::vector segs{ pb }; - - ufault_handlers.push_back( new UserFaultHandler{this, segs, worker_buffer_pages} ); - region_offset += worker_region_pages; - } - } catch(const std::exception& e) { - std::cerr << __FUNCTION__ << " Failed to launch _umap: " << e.what() << std::endl; - throw -1; - } catch(...) { - std::cerr << "umap failed to instantiate _umap object\n"; - throw -1; - } + return Umap::RegionManager::getInstance()->get_evict_high_water_threshold(); } -void _umap::flushbuffers( void ) +uint64_t +umapcfg_get_max_fault_events( void ) { - for ( auto handler : ufault_handlers ) - handler->flushbuffers(); + return Umap::RegionManager::getInstance()->get_max_fault_events(); } -_umap::~_umap(void) -{ - for ( auto handler : ufault_handlers ) - handler->stop_uffd_worker(); - - for ( auto handler : ufault_handlers ) - delete handler; +namespace Umap { - if (munmap(mmapRegion, mmapRegionSize)) { - perror("munmap failed: "); - } -} - -UserFaultHandler::UserFaultHandler(_umap* _um, const std::vector& _pblks, uint64_t _pbuf_size) - : - stat{ new umap_stats }, - _u{_um}, - PageBlocks{_pblks}, - pbuf_size{_pbuf_size}, - pagebuffer{ new umap_page_buffer{this, _pbuf_size} } +void* +umap_ex( + void* region_addr + , uint64_t region_size + , int prot + , int flags + , int fd + , off_t offset + , Store* store +) { - umessages.resize(UMAP_UFFD_MAX_MESSAGES); - - if (posix_memalign((void**)©in_buf, (uint64_t)umapPageSize, (umapPageSize * 2))) { - std::cerr << "ERROR: posix_memalign: failed\n"; - exit(1); - } - - if (copyin_buf == nullptr) { - std::cerr << "Unable to allocate " << (umapPageSize * 2) << " bytes for temporary buffer\n"; - exit(1); - } - - if ((userfault_fd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK)) < 0) { - perror("ERROR: userfaultfd syscall not available in this kernel"); - throw -1; - } - - if ( ! required_uffd_features_present(userfault_fd) ) - exit(1); + auto rm = RegionManager::getInstance(); + auto umap_psize = rm->get_umap_page_size(); - for ( auto seg : PageBlocks ) { - struct uffdio_register uffdio_register = { - .range = {.start = (uint64_t)seg.base, .len = seg.length}, -#ifndef UMAP_RO_MODE - .mode = UFFDIO_REGISTER_MODE_MISSING | UFFDIO_REGISTER_MODE_WP -#else - .mode = UFFDIO_REGISTER_MODE_MISSING -#endif - }; + UMAP_LOG(Debug, + "region_addr: " << region_addr + << ", region_size: " << region_size + << ", prot: " << prot + << ", flags: " << flags + << ", offset: " << offset + << ", store: " << store + << ", umap_psize: " << umap_psize + ); - debug_printf2("Register %lu Pages from: %p - %p\n", - (seg.length / umapPageSize), seg.base, - (void*)((uint64_t)seg.base + (uint64_t)(seg.length-1))); - - if (ioctl(userfault_fd, UFFDIO_REGISTER, &uffdio_register) == -1) { - perror("ERROR: ioctl/uffdio_register"); - close(userfault_fd); - throw -1; - } - - if ((uffdio_register.ioctls & UFFD_API_RANGE_IOCTLS) != UFFD_API_RANGE_IOCTLS) { - std::cerr << "unexpected userfaultfd ioctl set\n"; - close(userfault_fd); - throw -1; - } - } - - uffd_worker = new std::thread{&UserFaultHandler::uffd_handler, this}; -} - -UserFaultHandler::~UserFaultHandler(void) -{ // - // Now that all of our worker threads have stopped, we can flush everything + // TODO: Allow for non-page-multiple size and zero-fill like mmap does // - for ( auto seg : PageBlocks ) { - struct uffdio_register uffdio_register; - uffdio_register.range.start = (uint64_t)seg.base; - uffdio_register.range.len = seg.length; - - if (ioctl(userfault_fd, UFFDIO_UNREGISTER, &uffdio_register.range)) { - perror("ERROR: UFFDIO_UNREGISTER"); - exit(1); - } + if ( ( region_size % umap_psize ) ) { + UMAP_ERROR("Region size " << region_size + << " is not a multple of umapPageSize (" + << rm->get_umap_page_size() << ")"); } - free(copyin_buf); - delete pagebuffer; - delete stat; - delete uffd_worker; -} - -struct less_than_key -{ - inline bool operator() (const struct uffd_msg& lhs, const struct uffd_msg& rhs) - { - if (lhs.arg.pagefault.address == rhs.arg.pagefault.address) - return (lhs.arg.pagefault.flags >= rhs.arg.pagefault.address); - else - return (lhs.arg.pagefault.address < rhs.arg.pagefault.address); + if ( ( (uint64_t)region_addr & (umap_psize - 1) ) ) { + UMAP_ERROR("region_addr must be page aligned: " << region_addr + << ", page size is: " << rm->get_umap_page_size()); } -}; - -void UserFaultHandler::uffd_handler(void) -{ - prctl(PR_SET_NAME, "UMAP UFFD Hdlr", 0, 0, 0); - for (;;) { - struct pollfd pollfd[1]; - pollfd[0].fd = userfault_fd; - pollfd[0].events = POLLIN; - - if (_u->uffd_time_to_stop_working) { - flushbuffers(); - return; - } - - // wait for a userfaultfd event to occur - int pollres = poll(pollfd, 1, 2000); - - switch (pollres) { - case -1: - perror("ERROR: poll/userfaultfd"); - continue; - case 0: - continue; - case 1: - break; - default: - std::cerr << __FUNCTION__ << " unexpected uffdio poll result\n"; - exit(1); - } - - if (pollfd[0].revents & POLLERR) { - std::cerr << __FUNCTION__ << " POLLERR\n"; - exit(1); - } - if ( !(pollfd[0].revents & POLLIN) ) - continue; - - int readres = read(userfault_fd, &umessages[0], UMAP_UFFD_MAX_MESSAGES * sizeof(struct uffd_msg)); - - if (readres == -1) { - if (errno == EAGAIN) - continue; - perror("ERROR: read/userfaultfd"); - exit(1); - } - - assert(readres % sizeof(struct uffd_msg) == 0); - - int msgs = readres / sizeof(struct uffd_msg); - - if (msgs < 1) { - std::cerr << __FUNCTION__ << "invalid msg size " << readres << " " << msgs; - exit(1); - } - - for (int i = 0; i < msgs; ++i) { - assert("uffd_hander: Unexpected event" && (umessages[i].event == UFFD_EVENT_PAGEFAULT)); - pagefault_event(umessages[i]); // At this point, we know we have had a page fault. Let's handle it. - } - } -} - -void UserFaultHandler::pagefault_event(const struct uffd_msg& msg) -{ - void* page_begin = _umap::UMAP_PAGE_BEGIN( (void*)msg.arg.pagefault.address ); - umap_page* pm = pagebuffer->find_inmem_page_desc(page_begin); - - if (pm != nullptr) { -#ifndef UMAP_RO_MODE - if (msg.arg.pagefault.flags & (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_WRITE)) { - if (pm->page_is_dirty()) - return; - pm->mark_page_dirty(); - disable_wp_on_pages((uint64_t)page_begin, 1, false); - stat->wp_messages++; - debug_printf2("Present page written, marking %p dirty\n", page_begin); - } -#else - if ( msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE ) { - assert("Write operation not allowed without WP support" && 0); - } -#endif - else { - debug_printf2("Spurious fault for page %p which is already present\n", - page_begin); - } - return; + if (!(flags & UMAP_PRIVATE) || flags & ~(UMAP_PRIVATE|UMAP_FIXED)) { + UMAP_ERROR("Invalid flags: " << std::hex << flags); } // - // Page not present, read it in and (potentially) evict someone + // When dealing with umap-page-sizes that could be multiples of the actual + // system-page-size, it is possible for mmap() to provide a region that is on + // a system-page-boundary, but not necessarily on a umap-page-size boundary. // - off_t offset=(uint64_t)page_begin - (uint64_t)_u->umapRegion; - - if (_u->store->read_from_store(copyin_buf, umapPageSize, offset) == -1) { - perror("ERROR: read_from_store failed"); - exit(1); - } - - /* - * Keep trying to obtain a free page descriptor until we get one.. - */ - for ( pm = pagebuffer->alloc_page_desc(page_begin); - pm == nullptr; - pm = pagebuffer->alloc_page_desc(page_begin)) - { - pagebuffer->dealloc_page_desc(); - } - - struct uffdio_copy copy; - copy.src = (uint64_t)copyin_buf; - copy.dst = (uint64_t)page_begin; - copy.len = umapPageSize; - copy.mode = 0; - -#ifndef UMAP_RO_MODE - if (msg.arg.pagefault.flags & (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_WRITE)) { - debug_printf3("Write Fault: Copying in dirty page %p\n", page_begin); - stat->write_faults++; - pm->mark_page_dirty(); - - if (ioctl(userfault_fd, UFFDIO_COPY, ©) == -1) { - perror("ERROR: ioctl(UFFDIO_COPY nowake)"); - exit(1); - } - } -#else - if (msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE) { - assert("Write operation not allowed without WP support" && 0); - } -#endif - else { - debug_printf3("Read Fault: Copying in page %p\n", page_begin); - stat->read_faults++; - pm->mark_page_clean(); - -#ifndef UMAP_RO_MODE - copy.mode = UFFDIO_COPY_MODE_WP; -#else - copy.mode = 0; -#endif - if (ioctl(userfault_fd, UFFDIO_COPY, ©) == -1) { - perror("ERROR: ioctl(UFFDIO_COPY nowake)"); - exit(1); - } - } -} - -bool UserFaultHandler::page_is_in_umap(const void* page_begin) -{ - for ( auto it : PageBlocks ) - if (page_begin >= it.base && page_begin < (void*)((uint64_t)it.base + it.length)) - return true; - return false; -} - -void UserFaultHandler::flushbuffers( void ) -{ - while (pagebuffer->pages_still_present() == true) - pagebuffer->dealloc_page_desc(); -} - -void UserFaultHandler::resetstats( void ) -{ - stat->dirty_evicts = 0; - stat->evict_victims = 0; - stat->wp_messages = 0; - stat->read_faults = 0; - stat->write_faults = 0; -} - -void UserFaultHandler::evict_page(umap_page* pb) -{ - uint64_t* page = (uint64_t*)pb->get_page(); - - stat->evict_victims++; - if (pb->page_is_dirty()) { -#ifdef UMAP_RO_MODE - assert("Dirty page found when running in RO mode" && 0); -#else - stat->dirty_evicts++; - - // Prevent further writes. No need to do this if not dirty because WP is already on. - - enable_wp_on_pages_and_wake((uint64_t)page, 1); - - if (_u->store->write_to_store((char*)page, umapPageSize, (off_t)((uint64_t)page - (uint64_t)_u->umapRegion)) == -1) { - perror("ERROR: write_to_store failed"); - assert(0); - } -#endif - } - - if (madvise((void*)page, umapPageSize, MADV_DONTNEED) == -1) { - perror("ERROR: madvise"); - assert(0); - } - - pb->set_page(nullptr); -} - -#ifndef UMAP_RO_MODE -// -// Enabling WP always wakes up blocked faulting threads that may have been faulted in the specified range. -// -// For reasons which are unknown, the kernel module interface for UFFDIO_WRITEPROTECT does not allow for the caller to submit -// UFFDIO_WRITEPROTECT_MODE_DONTWAKE when enabling WP with UFFDIO_WRITEPROTECT_MODE_WP. UFFDIO_WRITEPROTECT_MODE_DONTWAKE is only -// allowed when disabling WP. -// -void UserFaultHandler::enable_wp_on_pages_and_wake(uint64_t start, int64_t num_pages) -{ - struct uffdio_writeprotect wp; - wp.range.start = start; - wp.range.len = num_pages * umapPageSize; - wp.mode = UFFDIO_WRITEPROTECT_MODE_WP; - - if (ioctl(userfault_fd, UFFDIO_WRITEPROTECT, &wp) == -1) { - perror("ERROR: ioctl(UFFDIO_WRITEPROTECT Enable)"); - exit(1); - } -} - -// -// We intentionally do not wake up faulting thread when disabling WP. This is to handle the write-fault case when the page needs to be copied in. -// -void UserFaultHandler::disable_wp_on_pages(uint64_t start, int64_t num_pages, bool do_not_awaken) -{ - struct uffdio_writeprotect wp; - wp.range.start = start; - wp.range.len = umapPageSize * num_pages; - wp.mode = do_not_awaken ? UFFDIO_WRITEPROTECT_MODE_DONTWAKE : 0; - - if (ioctl(userfault_fd, UFFDIO_WRITEPROTECT, &wp) == -1) { - perror("ERROR: ioctl(UFFDIO_WRITEPROTECT Disable)"); - exit(1); - } -} -#endif - -// -// umap_page_buffer class implementation -// -umap_page_buffer::umap_page_buffer(UserFaultHandler* _ufh_, uint64_t pbuffersize) - : ufh{_ufh_}, page_buffer_size{pbuffersize}, page_buffer_alloc_idx{0}, - page_buffer_free_idx{0}, page_buffer_alloc_cnt{0} -{ - page_descriptor_array = (umap_page *)calloc(page_buffer_size, sizeof(umap_page)); -} - -umap_page_buffer::~umap_page_buffer() -{ - assert(inmem_page_map.size() == 0); - assert(page_buffer_alloc_cnt == 0); + // We always allocate an additional umap-page-size set of bytes so that we can + // make certain that the umap-region begins on a umap-page-size boundary. + // + uint64_t mmap_size = region_size + umap_psize; - free(page_descriptor_array); -} + void* mmap_region = mmap(region_addr, mmap_size, + prot, flags | (MAP_ANONYMOUS | MAP_NORESERVE), -1, 0); -umap_page* umap_page_buffer::alloc_page_desc(void* page) -{ - if ( page_buffer_alloc_cnt < page_buffer_size ) { - umap_page* p = page_descriptor_array + page_buffer_alloc_idx; - page_buffer_alloc_idx = (page_buffer_alloc_idx + 1) % page_buffer_size; - page_buffer_alloc_cnt++; - p->set_page(page); - inmem_page_map[page] = p; - debug_printf3("%p allocated for %p, free idx=%" PRIu64 " alloc idx=%" PRIu64 " cnt=%" PRIu64 "\n", - p, page, page_buffer_free_idx, page_buffer_alloc_idx, page_buffer_alloc_cnt); - return p; + if (mmap_region == MAP_FAILED) { + UMAP_ERROR("mmap failed: " << strerror(errno)); + return UMAP_FAILED; } - return nullptr; -} - -bool umap_page_buffer::pages_still_present( void ) -{ - return page_buffer_alloc_cnt != 0; -} - -void umap_page_buffer::dealloc_page_desc( void ) -{ - umap_page* p = page_buffer_alloc_cnt ? - page_descriptor_array + page_buffer_free_idx : nullptr; + uint64_t umap_size = region_size; + void* umap_region; + umap_region = (void*)((uint64_t)mmap_region + umap_psize - 1); + umap_region = (void*)((uint64_t)umap_region & ~(umap_psize - 1)); - if ( p != nullptr ) { - debug_printf3("%p freed for %p, free idx=%" PRIu64 " alloc idx=%" PRIu64 " cnt=%" PRIu64 "\n", - p, p->get_page(), page_buffer_alloc_idx, - page_buffer_free_idx, page_buffer_alloc_cnt); - page_buffer_free_idx = (page_buffer_free_idx + 1) % page_buffer_size; - page_buffer_alloc_cnt--; - inmem_page_map.erase(p->get_page()); + if ( store == nullptr ) + store = Store::make_store(umap_region, umap_size, umap_psize, fd); - ufh->evict_page(p); - p->mark_page_clean(); - p->set_page(nullptr); - } -} + rm->addRegion(store, (char*)umap_region, umap_size, (char*)mmap_region, mmap_size); -umap_page* umap_page_buffer::find_inmem_page_desc(void* page_addr) -{ - auto it = inmem_page_map.find(page_addr); - return((it == inmem_page_map.end()) ? nullptr : it->second); + return umap_region; } +} // namespace Umap diff --git a/src/umap/umap.h b/src/umap/umap.h new file mode 100644 index 00000000..2ef826df --- /dev/null +++ b/src/umap/umap.h @@ -0,0 +1,91 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#ifndef _UMAP_H_ +#define _UMAP_H_ + +#ifdef __cplusplus + #include + #include "umap/store/Store.hpp" +#else // __cplusplus + #include +#endif // __cplusplus + +#include +#include + +#ifdef __cplusplus +namespace Umap { +/** Allow application to create region of memory to a persistent store + * \param addr Same as input argument for mmap(2) + * \param length Same as input argument of mmap(2) + * \param prot Same as input argument of mmap(2) + * \param flags Same as input argument of mmap(2) + */ +void* umap_ex( + void* addr + , std::size_t length + , int prot + , int flags + , int fd + , off_t offset + , Umap::Store* store +); +} // namespace Umap +#endif // __cplusplus + +#ifdef __cplusplus +extern "C" { +#endif +/** Allow application to create region of memory to a persistent store + * \param addr Same as input argument for mmap(2) + * \param length Same as input argument of mmap(2) + * \param prot Same as input argument of mmap(2) + * \param flags Same as input argument of mmap(2) + */ +void* umap( + void* addr + , size_t length + , int prot + , int flags + , int fd + , off_t offset +); + +int uunmap( + void* addr + , size_t length +); + +struct umap_prefetch_item { + void* page_base_addr; +}; + +void umap_prefetch( int npages, umap_prefetch_item* page_array ); +uint64_t umapcfg_get_umap_page_size( void ); +uint64_t umapcfg_get_max_fault_events( void ); +uint64_t umapcfg_get_num_fillers( void ); +uint64_t umapcfg_get_num_evictors( void ); +uint64_t umapcfg_get_max_pages_in_buffer( void ); +uint64_t umapcfg_get_read_ahead( void ); +int umapcfg_get_evict_low_water_threshold( void ); +int umapcfg_get_evict_high_water_threshold( void ); + +#ifdef __cplusplus +} +#endif + +/* + * flags + */ +#define UMAP_PRIVATE MAP_PRIVATE // Note - UMAP_SHARED not currently supported +#define UMAP_FIXED MAP_FIXED // See mmap(2) - This flag is currently then only flag supported. + +/* + * Return codes + */ +#define UMAP_FAILED (void *)-1 +#endif // _UMAP_H diff --git a/src/umap/util/Exception.cpp b/src/umap/util/Exception.cpp new file mode 100644 index 00000000..d80c254a --- /dev/null +++ b/src/umap/util/Exception.cpp @@ -0,0 +1,39 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#include "umap/util/Exception.hpp" + +#include + +namespace Umap { + +Exception::Exception( + const std::string& message, + const std::string &file, + int line) : + m_message(message), + m_file(file), + m_line(line) +{ + m_what = this->message(); +} + +std::string +Exception::message() const +{ + std::stringstream oss; + oss << "! UMAP Exception [" << m_file << ":" << m_line << "]: "; + oss << m_message; + return oss.str(); +} + +const char* +Exception::what() const throw() +{ + return m_what.c_str(); +} + +} // end of namespace Umap diff --git a/src/umap/util/Exception.hpp b/src/umap/util/Exception.hpp new file mode 100644 index 00000000..983a71cf --- /dev/null +++ b/src/umap/util/Exception.hpp @@ -0,0 +1,36 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#ifndef UMAP_Exception_HPP +#define UMAP_Exception_HPP + +#include +#include + +namespace Umap { + +class Exception : public std::exception { + public: + Exception(const std::string& msg, + const std::string &file, + int line); + + virtual ~Exception() = default; + + std::string message() const; + virtual const char* what() const throw(); + + private: + std::string m_message; + std::string m_file; + int m_line; + + std::string m_what; +}; + +} // end of namespace Umap + +#endif // UMAP_Exception_HPP diff --git a/src/umap/util/Logger.cpp b/src/umap/util/Logger.cpp new file mode 100644 index 00000000..13cb07a5 --- /dev/null +++ b/src/umap/util/Logger.cpp @@ -0,0 +1,127 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#include "umap/util/Logger.hpp" + +#include // for std::cout, std::cerr +#include +#include // for getenv() +#include // for strcasecmp() +#include +#include +#include + +namespace Umap { + +static const char* env_name = "UMAP_LOG_LEVEL"; +static const char* env_name_no_timestamp = "UMAP_LOG_NO_TIMESTAMP_LEVEL"; +static message::Level defaultLevel = message::Info; +std::mutex g_logging_mutex; +Logger* Logger::s_Logger = nullptr; + +static const std::string MessageLevelName[ message::Num_Levels ] = { + "ERROR", + "WARNING", + "INFO", + "DEBUG" +}; + +Logger::Logger(bool log_with_timestamp) noexcept + : m_log_timestamp(log_with_timestamp) +{ + // by default, all message streams are disabled + for ( int i=0 ; i < message::Num_Levels ; ++i ) + m_isEnabled[ i ] = false; +} + +Logger::~Logger() noexcept +{ +} + +void Logger::setLoggingMsgLevel( message::Level level ) noexcept +{ + for ( int i=0 ; i < message::Num_Levels ; ++i ) + m_isEnabled[ i ] = (i<= level) ? true : false; +} + +void Logger::logMessage( message::Level level, + const std::string& message, + const std::string& fileName, + int line ) noexcept +{ + if ( !logLevelEnabled( level ) ) + return; /* short-circuit */ + + std::lock_guard guard(g_logging_mutex); + if (m_log_timestamp) { + std::cout + << getpid() << ":" + << syscall(__NR_gettid) << " " + << "[" << MessageLevelName[ level ] << "]" + << "[" << fileName << ":" << line << "]:" + << message + << std::endl; + } + else { + std::cout + << message + << std::endl; + } +} + +void Logger::initialize() +{ + if ( s_Logger != nullptr ) + return; + + message::Level level = defaultLevel; + char* enval = getenv(env_name); + char* enval_no_timestamp = getenv(env_name_no_timestamp); + bool log_with_timestamp = true; + + if ( enval != NULL || enval_no_timestamp != NULL ) { + + if (enval_no_timestamp != NULL) { + enval = enval_no_timestamp; + log_with_timestamp = false; + } + + bool level_found = false; + for ( int i = 0; i < message::Num_Levels; ++i ) { + if ( strcasecmp( enval, MessageLevelName[ i ].c_str() ) == 0 ) { + level_found = true; + level = (message::Level)i; + break; + } + } + if (! level_found ) { + std::cerr << "No matching logging levels for: " << enval << "\n"; + std::cerr << "Available levels are:\n"; + for ( int i = 0; i < message::Num_Levels; ++i ) { + std::cerr << "\t" << MessageLevelName[ i ] << "\n"; + } + } + } + + s_Logger = new Logger(log_with_timestamp); + s_Logger->setLoggingMsgLevel(level); +} + +void Logger::finalize() +{ + delete s_Logger; + s_Logger = nullptr; +} + +Logger* Logger::getActiveLogger() +{ + if ( s_Logger == nullptr ) + Logger::initialize(); + + return s_Logger; +} + +} /* namespace umap */ diff --git a/src/umap/util/Logger.hpp b/src/umap/util/Logger.hpp new file mode 100644 index 00000000..944405a1 --- /dev/null +++ b/src/umap/util/Logger.hpp @@ -0,0 +1,69 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#ifndef UMAP_Logger_HPP +#define UMAP_Logger_HPP + +#include + +namespace Umap { + +namespace message { +enum Level { + Error, + Warning, + Info, + Debug, + + Num_Levels +}; + +static const std::string MessageLevelName[ Level::Num_Levels ] = { + "ERROR", + "WARNING", + "INFO", + "DEBUG" +}; +} /* namespace messge */ + +class Logger { + public: + + void setLoggingMsgLevel( message::Level level ) noexcept; + + void logMessage( message::Level level, + const std::string& message, + const std::string& fileName, + int line ) noexcept; + + static void initialize(); + + static void finalize(); + + static Logger* getActiveLogger(); + + static Logger* getRootLogger(); + + inline bool logLevelEnabled( message::Level level ) + { + if ( level < 0 || level >= message::Num_Levels || m_isEnabled[ level ] == false ) + return false; + else + return true; + }; + +private: + Logger( bool log_with_timestamp ) noexcept; + ~Logger() noexcept; + + bool m_log_timestamp; + bool m_isEnabled[ message::Num_Levels ]; + static Logger* s_Logger; +}; + +} /* namespace Umap */ + +#endif /* UMAP_Logger_HPP */ diff --git a/src/umap/util/Macros.hpp b/src/umap/util/Macros.hpp new file mode 100644 index 00000000..bdb708d6 --- /dev/null +++ b/src/umap/util/Macros.hpp @@ -0,0 +1,57 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other +// UMAP Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: LGPL-2.1-only +////////////////////////////////////////////////////////////////////////////// +#ifndef UMAP_Macros_HPP +#define UMAP_Macros_HPP + +#include "umap/util/Exception.hpp" +#include "umap/config.h" + +#include +#include + +#ifdef UMAP_ENABLE_ASSERTS +#include +#define UMAP_ASSERT(condition) assert(condition) +#else +#define UMAP_ASSERT(condition) ((void)0) +#endif // UMAP_ENABLE_ASSERTS + +#ifdef UMAP_DEBUG_LOGGING + +#include "umap/util/Logger.hpp" +#define UMAP_LOG( lvl, msg ) \ +{ \ + if (Umap::Logger::getActiveLogger()->logLevelEnabled(Umap::message::lvl)) { \ + std::ostringstream local_msg; \ + local_msg << " " << __func__ << " " << msg; \ + Umap::Logger::getActiveLogger()->logMessage( \ + Umap::message::lvl, local_msg.str(), \ + std::string(__FILE__), __LINE__); \ + } \ +} + +#else + +#define UMAP_LOG( lvl, msg ) ((void)0) + +#endif // UMAP_DEBUG_LOGGING + +#define UMAP_UNUSED_ARG(x) + +#define UMAP_USE_VAR(x) static_cast(x) + +#define UMAP_ERROR( msg ) \ +{ \ + UMAP_LOG(Error, msg); \ + std::ostringstream umap_oss_error; \ + umap_oss_error << " " << __func__ << " " << msg; \ + throw Umap::Exception( umap_oss_error.str(), \ + std::string(__FILE__), \ + __LINE__); \ +} + +#endif // UMAP_Macros_HPP diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d264c78c..86196a4b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -6,4 +6,3 @@ ############################################################################# add_subdirectory(churn) add_subdirectory(pfbenchmark) -add_subdirectory(umapsort) diff --git a/tests/churn/options.cpp b/tests/churn/options.cpp index 2b901b92..5ef7e636 100644 --- a/tests/churn/options.cpp +++ b/tests/churn/options.cpp @@ -4,9 +4,6 @@ // // SPDX-License-Identifier: LGPL-2.1-only ////////////////////////////////////////////////////////////////////////////// -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif // _GNU_SOURCE #include // cout/cerr #include // getopt() @@ -33,14 +30,21 @@ static void usage(char* pname) << " --noinit - No Initialization\n" << " --directio - Use O_DIRECT for file IO\n" << " --usemmap - Use mmap instead of umap\n" - << " -b # of pages in page buffer - default: " << umap_cfg_get_bufsize() << " Pages\n" + << " -b # of pages in page buffer - default: " << umapcfg_get_max_pages_in_buffer() << " Pages\n" << " -c # of churn pages - default: " << NUMCHURNPAGES << " Pages\n" << " -l # of load pages - default: " << NUMLOADPAGES << " Pages\n" << " -t # of churn threads - default: " << NUMCHURNTHREADS << endl << " -r # of load reader threads - default: " << NUMLOADREADERS << endl << " -w # of load writer threads - default: " << NUMLOADWRITERS << endl << " -f [backing file name] - default: " << FILENAME << endl - << " -d # seconds to run test - default: " << TESTDURATION << " seconds\n"; + << " -d # seconds to run test - default: " << TESTDURATION << " seconds\n" + << " \n" + << " Environment Variable Configuration:\n" + << " UMAP_PAGE_FILLERS(env) - currently: " << umapcfg_get_num_fillers() << " fillers\n" + << " UMAP_PAGE_EVICTORS(env)- currently: " << umapcfg_get_num_evictors() << " evictors\n" + << " UMAP_BUFSIZE(env) - currently: " << umapcfg_get_max_pages_in_buffer() << " pages\n" + << " UMAP_PAGESIZE(env) - currently: " << umapcfg_get_umap_page_size() << " bytes\n" + ; exit(1); } @@ -60,7 +64,7 @@ void getoptions(options_t& testops, int& argc, char **argv) testops.num_load_writer_threads=NUMLOADWRITERS; testops.fn=FILENAME; testops.testduration=TESTDURATION; - testops.page_buffer_size = umap_cfg_get_bufsize(); + testops.page_buffer_size = umapcfg_get_max_pages_in_buffer(); while (1) { int option_index = 0; @@ -128,7 +132,5 @@ void getoptions(options_t& testops, int& argc, char **argv) cerr << endl; usage(pname); } - - umap_cfg_set_bufsize(testops.page_buffer_size); } diff --git a/tests/pfbenchmark/nvmebenchmark.cpp b/tests/pfbenchmark/nvmebenchmark.cpp index 03168853..8b007cd7 100644 --- a/tests/pfbenchmark/nvmebenchmark.cpp +++ b/tests/pfbenchmark/nvmebenchmark.cpp @@ -17,10 +17,6 @@ * or randomly (if "--shuffle" command line option is specified). */ -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif // _GNU_SOURCE - #include #include #include diff --git a/tests/pfbenchmark/pfbenchmark.cpp b/tests/pfbenchmark/pfbenchmark.cpp index 2eb7e605..b6ba781d 100644 --- a/tests/pfbenchmark/pfbenchmark.cpp +++ b/tests/pfbenchmark/pfbenchmark.cpp @@ -17,10 +17,6 @@ * or randomly (if "--shuffle" command line option is specified). */ -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif // _GNU_SOURCE - #include #include #include @@ -97,6 +93,7 @@ uint64_t do_read_modify_write_pages(uint64_t page_step, uint64_t pages) void print_stats( void ) { if (!usemmap) { +#if 0 struct umap_cfg_stats s; umap_cfg_get_stats(glb_array, &s); @@ -105,6 +102,7 @@ void print_stats( void ) cout << s.wp_messages << " WP Faults\n"; cout << s.read_faults << " Read Faults\n"; cout << s.write_faults << " Write Faults\n"; +#endif } } diff --git a/tests/umapsort/CMakeLists.txt b/tests/umapsort/CMakeLists.txt deleted file mode 100644 index e75b888f..00000000 --- a/tests/umapsort/CMakeLists.txt +++ /dev/null @@ -1,39 +0,0 @@ -############################################################################# -# Copyright 2017-2019 Lawrence Livermore National Security, LLC and other -# UMAP Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: LGPL-2.1-only -############################################################################# -project(umapsort) - -FIND_PACKAGE( OpenMP REQUIRED ) -if(OPENMP_FOUND) - configure_file( - "${CMAKE_CURRENT_SOURCE_DIR}/perftest_mmap.sh" - "${CMAKE_CURRENT_BINARY_DIR}/perftest_mmap.sh" - COPYONLY - ) - configure_file( - "${CMAKE_CURRENT_SOURCE_DIR}/perftest_umap.sh" - "${CMAKE_CURRENT_BINARY_DIR}/perftest_umap.sh" - COPYONLY - ) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") - set(CMAKE_EXE_LINKER_FLAGS - "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") - add_executable(umapsort umapsort.cpp) - - add_dependencies(umapsort umap) - target_link_libraries(umapsort umap) - - include_directories( ${CMAKE_CURRENT_SOURCE_DIR} ${UMAPINCLUDEDIRS} ) - - install(TARGETS umapsort - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib/static - RUNTIME DESTINATION bin ) -else() - message("Skipping umapsort, OpenMP required") -endif() - diff --git a/tests/umapsort/commandline.hpp b/tests/umapsort/commandline.hpp deleted file mode 100644 index ed7a2525..00000000 --- a/tests/umapsort/commandline.hpp +++ /dev/null @@ -1,155 +0,0 @@ -////////////////////////////////////////////////////////////////////////////// -// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other -// UMAP Project Developers. See the top-level LICENSE file for details. -// -// SPDX-License-Identifier: LGPL-2.1-only -////////////////////////////////////////////////////////////////////////////// -#ifndef _COMMANDLING_HPP -#define _COMMANDLING_HPP - -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif // _GNU_SOURCE - -#include -#include // cout/cerr -#include // getopt() -#include // duh... -#include "umap/umap.h" - -namespace utility { -typedef struct { - int initonly; // Just perform initialization, then quit - int noinit; // Init already done, so skip it - int usemmap; - - long pagesize; - uint64_t numpages; - uint64_t numthreads; - uint64_t bufsize; - uint64_t uffdthreads; - char const* filename; // file name or basename -} umt_optstruct_t; - -static char const* FILENAME = "testfile"; -const uint64_t NUMPAGES = 10000000; -const uint64_t NUMTHREADS = 2; - -using namespace std; - -static void usage(char* pname) -{ - cerr - << "Usage: " << pname << " [--initonly] [--noinit] [--directio]" - << " [--usemmap] [-p #] [-t #] [-b #] [-f name]\n\n" - << " --help - This message\n" - << " --initonly - Initialize file, then stop\n" - << " --noinit - Use previously initialized file\n" - << " --usemmap - Use mmap instead of umap\n" - << " -p # of pages - default: " << NUMPAGES << endl - << " -t # of threads - default: " << NUMTHREADS << endl - << " -u # of uffd threads - default: " << umap_cfg_get_uffdthreads() << " worker threads\n" - << " -b # page buffer size - default: " << umap_cfg_get_bufsize() << " Pages\n" - << " -a # pages to access - default: 0 - access all pages\n" - << " -f [file name] - backing file name.\n" - << " -P # page size - default: " << umap_cfg_get_pagesize() << endl; - exit(1); -} - -void umt_getoptions(utility::umt_optstruct_t* testops, int argc, char *argv[]) -{ - int c; - char *pname = argv[0]; - - testops->initonly = 0; - testops->noinit = 0; - testops->usemmap = 0; - testops->numpages = NUMPAGES; - testops->numthreads = NUMTHREADS; - testops->bufsize = umap_cfg_get_bufsize(); - testops->uffdthreads = umap_cfg_get_uffdthreads(); - testops->filename = FILENAME; - testops->pagesize = umap_cfg_get_pagesize(); - - while (1) { - int option_index = 0; - static struct option long_options[] = { - {"initonly", no_argument, &testops->initonly, 1 }, - {"noinit", no_argument, &testops->noinit, 1 }, - {"usemmap", no_argument, &testops->usemmap, 1 }, - {"help", no_argument, NULL, 0 }, - {0, 0, 0, 0 } - }; - - c = getopt_long(argc, argv, "p:t:f:b:u:P:", long_options, &option_index); - if (c == -1) - break; - - switch(c) { - case 0: - if (long_options[option_index].flag != 0) - break; - - usage(pname); - break; - - case 'P': - if ((testops->pagesize = strtol(optarg, nullptr, 0)) > 0) { - if (umap_cfg_set_pagesize(testops->pagesize) < 0) { - goto R0; - } - break; - } - goto R0; - case 'p': - if ((testops->numpages = strtoull(optarg, nullptr, 0)) > 0) - break; - goto R0; - case 't': - if ((testops->numthreads = strtoull(optarg, nullptr, 0)) > 0) - break; - else goto R0; - case 'b': - if ((testops->bufsize = strtoull(optarg, nullptr, 0)) > 0) - break; - else goto R0; - case 'u': - if ((testops->uffdthreads = strtoull(optarg, nullptr, 0)) > 0) - break; - else goto R0; - case 'f': - testops->filename = optarg; - break; - default: - R0: - usage(pname); - } - } - - if (optind < argc) { - cerr << "Unknown Arguments: "; - while (optind < argc) - cerr << "\"" << argv[optind++] << "\" "; - cerr << endl; - usage(pname); - } - - /* - * Note: Care must be taken when configuring the number of threads - * and the buffer size of umap. When the buffer size is set, it - * apportions the buffer evenly to the umap threads. So setting the - * buffer size requires that the number of threads be set properly - * first. - */ - if (testops->uffdthreads != umap_cfg_get_uffdthreads()) - umap_cfg_set_uffdthreads(testops->uffdthreads); - - umap_cfg_set_bufsize(testops->bufsize); -} - -long umt_getpagesize(void) -{ - return umap_cfg_get_pagesize(); -} -} -#endif // _COMMANDLING_HPP diff --git a/tests/umapsort/perftest_mmap.sh b/tests/umapsort/perftest_mmap.sh deleted file mode 100755 index 847a438b..00000000 --- a/tests/umapsort/perftest_mmap.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash -############################################################################# -# Copyright 2017-2019 Lawrence Livermore National Security, LLC and other -# UMAP Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: LGPL-2.1-only -############################################################################# -function free_mem { - m=`grep MemFree /proc/meminfo | awk -v N=2 '{print $N}'` - fm=$(((${m}/1024)/1024)) - echo $fm GB Free -} - -function drop_page_cache { - echo "Dropping page cache" - echo 3 > /proc/sys/vm/drop_caches -} - -function disable_swap { - echo "Disabling swap" - swapoff -av -} - -function set_readahead { - fs=`mount | grep intel | cut -d " " -f 1` - blockdev --setra $readahead $fs - ra=`blockdev --getra $fs` - echo "Read ahead set to $ra for $fs" -} - -function amounttowaste { - m=`grep MemFree /proc/meminfo | awk -v N=2 '{print $N}'` - echo $m - fm=$(((${m}/1024)/1024)) - waste=$((${fm}-${memtoleave})) - echo $fm GB Available, Wasting $waste GB -} - -function setuptmpfs { - if [ ! -d /mnt/tmpfs ]; then - mkdir -p /mnt/tmpfs - fi - - # Unmount / Reset of already mounted - fs=`stat -f -c '%T' /mnt/tmpfs` - - if [ "$fs" = "tmpfs" ]; then - echo "Resetting tmpfs" - umount /mnt/tmpfs - fi - - fs=`stat -f -c '%T' /mnt/tmpfs` - if [ "$fs" != "tmpfs" ]; then - if [ ! -d /mnt/tmpfs ]; then - mkdir -p /mnt/tmpfs - fi - chmod go+rwx /mnt/tmpfs - mount -t tmpfs -o size=600g tmpfs /mnt/tmpfs - fs=`stat -f -c '%T' /mnt/tmpfs` - echo "/mnt/tmpfs mounted as: $fs" - else - echo "Unable to reset /mnt/tmpfs, exiting" - exit 1 - fi -} - -function waste_memory { - echo "Wasting $waste GB of memory" - echo dd if=/dev/zero of=/mnt/tmpfs/${waste}GB bs=4096 count=$((${waste}*256*1024)) - dd if=/dev/zero of=/mnt/tmpfs/${waste}GB bs=4096 count=$((${waste}*256*1024)) -} - -memtoleave=$((64+6)) -readahead=256 - -set_readahead -disable_swap -setuptmpfs -drop_page_cache -amounttowaste -waste_memory - -for t in 128 64 32 16 -do - rm -f /mnt/intel/sort_perf_data - drop_page_cache - free_mem - cmd="./umapsort --usemmap --directio -f /mnt/intel/sort_perf_data -p $(((96*1024*1024*1024)/4096)) -n 1 -b $(((64*1024*1024*1024)/4096)) -t $t" - date - echo $cmd - time sh -c "$cmd" -done diff --git a/tests/umapsort/perftest_umap.sh b/tests/umapsort/perftest_umap.sh deleted file mode 100755 index dc1676f3..00000000 --- a/tests/umapsort/perftest_umap.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash -############################################################################# -# Copyright 2017-2019 Lawrence Livermore National Security, LLC and other -# UMAP Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: LGPL-2.1-only -############################################################################# -function free_mem { - m=`grep MemFree /proc/meminfo | awk -v N=2 '{print $N}'` - fm=$(((${m}/1024)/1024)) - echo $fm GB Free -} - -function drop_page_cache { - echo "Dropping page cache" - echo 3 > /proc/sys/vm/drop_caches -} - -function disable_swap { - echo "Disabling swap" - swapoff -av -} - -function set_readahead { - fs=`mount | grep intel | cut -d " " -f 1` - blockdev --setra $readahead $fs - ra=`blockdev --getra $fs` - echo "Read ahead set to $ra for $fs" -} - -function amounttowaste { - m=`grep MemFree /proc/meminfo | awk -v N=2 '{print $N}'` - echo $m - fm=$(((${m}/1024)/1024)) - waste=$((${fm}-${memtoleave})) - echo $fm GB Available, Wasting $waste GB -} - -function setuptmpfs { - if [ ! -d /mnt/tmpfs ]; then - mkdir -p /mnt/tmpfs - fi - - # Unmount / Reset of already mounted - fs=`stat -f -c '%T' /mnt/tmpfs` - - if [ "$fs" = "tmpfs" ]; then - echo "Resetting tmpfs" - umount /mnt/tmpfs - fi - - fs=`stat -f -c '%T' /mnt/tmpfs` - if [ "$fs" != "tmpfs" ]; then - if [ ! -d /mnt/tmpfs ]; then - mkdir -p /mnt/tmpfs - fi - chmod go+rwx /mnt/tmpfs - mount -t tmpfs -o size=600g tmpfs /mnt/tmpfs - fs=`stat -f -c '%T' /mnt/tmpfs` - echo "/mnt/tmpfs mounted as: $fs" - else - echo "Unable to reset /mnt/tmpfs, exiting" - exit 1 - fi -} - -function waste_memory { - echo "Wasting $waste GB of memory" - echo dd if=/dev/zero of=/mnt/tmpfs/${waste}GB bs=4096 count=$((${waste}*256*1024)) - dd if=/dev/zero of=/mnt/tmpfs/${waste}GB bs=4096 count=$((${waste}*256*1024)) -} - -memtoleave=$((64+6)) -readahead=0 - -set_readahead -disable_swap -setuptmpfs -drop_page_cache -amounttowaste -waste_memory - -for t in 128 64 32 16 -do - rm -f /mnt/intel/sort_perf_data - drop_page_cache - free_mem - cmd="./umapsort --directio -f /mnt/intel/sort_perf_data -p $(((96*1024*1024*1024)/4096)) -n 1 -b $(((64*1024*1024*1024)/4096)) -t $t" - date - echo $cmd - time sh -c "$cmd" -done diff --git a/tests/umapsort/time.hpp b/tests/umapsort/time.hpp deleted file mode 100644 index 04906f0c..00000000 --- a/tests/umapsort/time.hpp +++ /dev/null @@ -1,23 +0,0 @@ -////////////////////////////////////////////////////////////////////////////// -// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other -// UMAP Project Developers. See the top-level LICENSE file for details. -// -// SPDX-License-Identifier: LGPL-2.1-only -////////////////////////////////////////////////////////////////////////////// - -#ifndef UMAP_TEST_LIB_UTILITY_TIME_HPP -#define UMAP_TEST_LIB_UTILITY_TIME_HPP - -#include - -namespace utility { -inline std::chrono::high_resolution_clock::time_point elapsed_time_sec() { - return std::chrono::high_resolution_clock::now(); -} - -inline double elapsed_time_sec(const std::chrono::high_resolution_clock::time_point &tic) { - auto duration_time = std::chrono::high_resolution_clock::now() - tic; - return static_cast(std::chrono::duration_cast(duration_time).count() / 1e6); -} -} -#endif //UMAP_TEST_LIB_UTILITY_TIME_HPP diff --git a/tests/umapsort/umap_file.hpp b/tests/umapsort/umap_file.hpp deleted file mode 100644 index feada84d..00000000 --- a/tests/umapsort/umap_file.hpp +++ /dev/null @@ -1,136 +0,0 @@ -////////////////////////////////////////////////////////////////////////////// -// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other -// UMAP Project Developers. See the top-level LICENSE file for details. -// -// SPDX-License-Identifier: LGPL-2.1-only -////////////////////////////////////////////////////////////////////////////// -#ifndef _UMAP_FILE_HPP_ -#define _UMAP_FILE_HPP_ - -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include "umap/umap.h" - -namespace utility { - -void* map_in_file( - std::string filename, - bool initonly, - bool noinit, - bool usemmap, - uint64_t numbytes) -{ - int o_opts = O_RDWR | O_LARGEFILE | O_DIRECT; - void* region = NULL; - int fd; - - if ( initonly || !noinit ) { - o_opts |= O_CREAT; - std::cout << "Deleting " << filename << "\n"; - if ( unlink(filename.c_str()) ) { - int eno = errno; - if ( eno != ENOENT ) { - cerr << "Failed to unlink " << filename << ": " - << strerror(eno) << " Errno=" << eno << endl; - } - } - } - - if ( ( fd = open(filename.c_str(), o_opts, S_IRUSR | S_IWUSR) ) == -1 ) { - std::string estr = "Failed to open/create " + filename + ": "; - perror(estr.c_str()); - return NULL; - } - - if ( o_opts & O_CREAT ) { - // If we are initializing, attempt to pre-allocate disk space for the file. - try { - int x; - if ( ( x = posix_fallocate(fd, 0, numbytes) != 0 ) ) { - std::ostringstream ss; - ss << "Failed to pre-allocate " << - numbytes << " bytes in " << filename << ": "; - perror(ss.str().c_str()); - return NULL; - } - } catch(const std::exception& e) { - std::cerr << "posix_fallocate: " << e.what() << std::endl; - return NULL; - } catch(...) { - std::cerr << "posix_fallocate failed to allocate backing store\n"; - return NULL; - } - } - - struct stat sbuf; - if (fstat(fd, &sbuf) == -1) { - std::string estr = "Failed to get status (fstat) for " + filename + ": "; - perror(estr.c_str()); - return NULL; - } - - if ( (off_t)sbuf.st_size != (numbytes) ) { - std::cerr << filename << " size " << sbuf.st_size - << " does not match specified data size of " << (numbytes) << std::endl; - return NULL; - } - - const int prot = PROT_READ|PROT_WRITE; - - if ( usemmap ) { - region = mmap(NULL, numbytes, prot, MAP_SHARED | MAP_NORESERVE, fd, 0); - if (region == MAP_FAILED) { - std::ostringstream ss; - ss << "mmap of " << numbytes << " bytes failed for " << filename << ": "; - perror(ss.str().c_str()); - return NULL; - } - } - else { - int flags = UMAP_PRIVATE; - - region = umap(NULL, numbytes, prot, flags, fd, 0); - if ( region == UMAP_FAILED ) { - std::ostringstream ss; - ss << "umap_mf of " << numbytes - << " bytes failed for " << filename << ": "; - perror(ss.str().c_str()); - return NULL; - } - } - - return region; -} - -void unmap_file(bool usemmap, uint64_t numbytes, void* region) -{ - if ( usemmap ) { - if ( munmap(region, numbytes) < 0 ) { - std::ostringstream ss; - ss << "munmap failure: "; - perror(ss.str().c_str()); - exit(-1); - } - } - else { - if (uunmap(region, numbytes) < 0) { - std::ostringstream ss; - ss << "uunmap of failure: "; - perror(ss.str().c_str()); - exit(-1); - } - } -} - -} -#endif diff --git a/tests/umapsort/umapsort.cpp b/tests/umapsort/umapsort.cpp deleted file mode 100644 index a37cdae7..00000000 --- a/tests/umapsort/umapsort.cpp +++ /dev/null @@ -1,184 +0,0 @@ -////////////////////////////////////////////////////////////////////////////// -// Copyright 2017-2019 Lawrence Livermore National Security, LLC and other -// UMAP Project Developers. See the top-level LICENSE file for details. -// -// SPDX-License-Identifier: LGPL-2.1-only -////////////////////////////////////////////////////////////////////////////// -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif // _GNU_SOURCE - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -#include "umap/umap.h" -#include "commandline.hpp" -#include "umap_file.hpp" -#include "time.hpp" - -using namespace std; - -bool sort_ascending = true; - -void initdata(uint64_t *region, uint64_t rlen) { - fprintf(stderr, "initdata: %p, from %lu to %lu\n", region, (rlen), (rlen - rlen)); -#pragma omp parallel for - for(uint64_t i=0; i < rlen; ++i) - region[i] = (uint64_t) (rlen - i); -} - -uint64_t dump_page( uint64_t* region, uint64_t index ) -{ - uint64_t pageSize = (uint64_t)utility::umt_getpagesize(); - uint64_t elemsPerPage = pageSize / sizeof(uint64_t); - uint64_t pageNumber = index / elemsPerPage; - uint64_t pageStartIndex = pageNumber*elemsPerPage; - uint64_t* page = ®ion[pageStartIndex]; - - fprintf(stderr, "Data miscompare in page %lu\n", pageNumber); - - for ( uint64_t i = pageStartIndex; i < (pageStartIndex + elemsPerPage); ++i ) { - if ( i == index ) - fprintf(stderr, "**%8lu %8lu\n", i, region[i]); - else - fprintf(stderr, " %8lu %8lu\n", i, region[i]); - } - - return pageStartIndex + elemsPerPage; // got to next page -} - -void validatedata(uint64_t *region, uint64_t rlen) { - if (sort_ascending == true) { -// #pragma omp parallel for - for(uint64_t i = 0; i < rlen; ++i) { - if (region[i] != (i+1)) { - fprintf(stderr, "Worker %d found an error at index %lu, %lu != lt %lu!\n", - omp_get_thread_num(), i, region[i], i+1); - - if (i < 3) { - fprintf(stderr, "Context "); - for (int j=0; j < 7; j++) { - fprintf(stderr, "%lu ", region[j]); - } - fprintf(stderr, "\n"); - } - else if (i > (rlen-4)) { - fprintf(stderr, "Context "); - for (uint64_t j=rlen-8; j < rlen; j++) { - fprintf(stderr, "%lu ", region[j]); - } - fprintf(stderr, "\n"); - } - else { - fprintf(stderr, - "Context i-3 i-2 i-1 i i+1 i+2 i+3:%lu %lu %lu %lu %lu %lu %lu\n", - region[i-3], region[i-2], region[i-1], region[i], region[i+1], region[i+2], region[i+3]); - } - i = dump_page( region, i ) - 1; - } - } - } - else { -// #pragma omp parallel for - for(uint64_t i = 0; i < rlen; ++i) { - if(region[i] != (rlen - i)) { - fprintf(stderr, "Worker %d found an error at index %lu, %lu != %lu!\n", - omp_get_thread_num(), i, region[i], (rlen - i)); - - if (i < 3) { - fprintf(stderr, "Context "); - for (int j=0; j < 7; j++) { - fprintf(stderr, "%lu ", region[j]); - } - fprintf(stderr, "\n"); - } - else if (i > (rlen-4)) { - fprintf(stderr, "Context "); - for (uint64_t j=rlen-8; j < rlen; j++) { - fprintf(stderr, "%lu ", region[j]); - } - fprintf(stderr, "\n"); - } - else { - fprintf(stderr, - "Context i-3 i-2 i-1 i i+1 i+2 i+3:%lu %lu %lu %lu %lu %lu %lu\n", - region[i-3], region[i-2], region[i-1], region[i], region[i+1], region[i+2], region[i+3]); - } - exit(1); - } - } - } -} - -int main(int argc, char **argv) -{ - utility::umt_optstruct_t options; - uint64_t pagesize; - uint64_t totalbytes; - uint64_t arraysize; - void* base_addr; - - auto start = utility::elapsed_time_sec(); - pagesize = (uint64_t)utility::umt_getpagesize(); - - umt_getoptions(&options, argc, argv); - - omp_set_num_threads(options.numthreads); - - totalbytes = options.numpages*pagesize; - base_addr = utility::map_in_file(options.filename, options.initonly, options.noinit, options.usemmap, totalbytes); - if (base_addr == nullptr) - return -1; - - fprintf(stderr, "umap INIT took %f seconds\n", utility::elapsed_time_sec(start)); - fprintf(stderr, "%lu pages, %lu bytes, %lu threads\n", options.numpages, totalbytes, options.numthreads); - - uint64_t *arr = (uint64_t *) base_addr; - arraysize = totalbytes/sizeof(uint64_t); - - start = utility::elapsed_time_sec(); - if ( !options.noinit ) { - // init data - initdata(arr, arraysize); - fprintf(stderr, "INIT took %f seconds\n", utility::elapsed_time_sec(start)); - } - - if ( !options.initonly ) - { - start = utility::elapsed_time_sec(); - sort_ascending = (arr[0] != 1); - - if (sort_ascending == true) { - printf("Sorting in Ascending Order\n"); - __gnu_parallel::sort(arr, &arr[arraysize], std::less(), __gnu_parallel::quicksort_tag()); - } - else { - printf("Sorting in Descending Order\n"); - __gnu_parallel::sort(arr, &arr[arraysize], std::greater(), __gnu_parallel::quicksort_tag()); - } - - fprintf(stderr, "Sort took %f seconds\n", utility::elapsed_time_sec(start)); - - start = utility::elapsed_time_sec(); - validatedata(arr, arraysize); - fprintf(stderr, "Validate took %f seconds\n", utility::elapsed_time_sec(start)); - } - - start = utility::elapsed_time_sec(); - utility::unmap_file(options.usemmap, totalbytes, base_addr); - fprintf(stderr, "umap TERM took %f seconds\n", utility::elapsed_time_sec(start)); - - return 0; -} diff --git a/tests/utility/commandline.hpp b/tests/utility/commandline.hpp index 4518521b..fb1c9c30 100644 --- a/tests/utility/commandline.hpp +++ b/tests/utility/commandline.hpp @@ -7,10 +7,6 @@ #ifndef _COMMANDLING_HPP #define _COMMANDLING_HPP -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif // _GNU_SOURCE - #include #include // cout/cerr #include // getopt() @@ -52,14 +48,18 @@ static void usage(char* pname) << " --noinit - Use previously initialized file\n" << " --usemmap - Use mmap instead of umap\n" << " --shuffle - Shuffle memory accesses (instead of sequential access)\n" - << " -p # of pages - default: " << NUMPAGES << endl - << " -t # of threads - default: " << NUMTHREADS << endl - << " -u # of uffd threads - default: " << umap_cfg_get_uffdthreads() << " worker threads\n" - << " -b # page buffer size - default: " << umap_cfg_get_bufsize() << " Pages\n" + << " -p # of pages - default: " << NUMPAGES << " test pages\n" + << " -t # of threads - default: " << NUMTHREADS << " application threads\n" << " -a # pages to access - default: 0 - access all pages\n" << " -f [file name] - backing file name. Or file basename if multiple files\n" << " -d [directory name] - backing directory name. Or dir basename if multiple dirs\n" - << " -P # page size - default: " << umap_cfg_get_pagesize() << endl; + << " \n" + << " Environment Variable Configuration:\n" + << " UMAP_PAGE_FILLERS(env) - currently: " << umapcfg_get_num_fillers() << " fillers\n" + << " UMAP_PAGE_EVICTORS(env)- currently: " << umapcfg_get_num_evictors() << " evictors\n" + << " UMAP_BUFSIZE(env) - currently: " << umapcfg_get_max_pages_in_buffer() << " pages\n" + << " UMAP_PAGESIZE(env) - currently: " << umapcfg_get_umap_page_size() << " bytes\n" + ; exit(1); } @@ -75,11 +75,11 @@ void umt_getoptions(utility::umt_optstruct_t* testops, int argc, char *argv[]) testops->pages_to_access = 0; testops->numpages = NUMPAGES; testops->numthreads = NUMTHREADS; - testops->bufsize = umap_cfg_get_bufsize(); - testops->uffdthreads = umap_cfg_get_uffdthreads(); + testops->bufsize = umapcfg_get_max_pages_in_buffer(); + testops->uffdthreads = umapcfg_get_num_fillers(); testops->filename = FILENAME; testops->dirname = DIRNAME; - testops->pagesize = umap_cfg_get_pagesize(); + testops->pagesize = umapcfg_get_umap_page_size(); while (1) { int option_index = 0; @@ -92,7 +92,7 @@ void umt_getoptions(utility::umt_optstruct_t* testops, int argc, char *argv[]) {0, 0, 0, 0 } }; - c = getopt_long(argc, argv, "p:t:f:b:d:u:a:P:", long_options, &option_index); + c = getopt_long(argc, argv, "p:t:f:d:a:", long_options, &option_index); if (c == -1) break; @@ -103,15 +103,6 @@ void umt_getoptions(utility::umt_optstruct_t* testops, int argc, char *argv[]) usage(pname); break; - - case 'P': - if ((testops->pagesize = strtol(optarg, nullptr, 0)) > 0) { - if (umap_cfg_set_pagesize(testops->pagesize) < 0) { - goto R0; - } - break; - } - goto R0; case 'p': if ((testops->numpages = strtoull(optarg, nullptr, 0)) > 0) break; @@ -120,14 +111,6 @@ void umt_getoptions(utility::umt_optstruct_t* testops, int argc, char *argv[]) if ((testops->numthreads = strtoull(optarg, nullptr, 0)) > 0) break; else goto R0; - case 'b': - if ((testops->bufsize = strtoull(optarg, nullptr, 0)) > 0) - break; - else goto R0; - case 'u': - if ((testops->uffdthreads = strtoull(optarg, nullptr, 0)) > 0) - break; - else goto R0; case 'a': testops->pages_to_access = strtoull(optarg, nullptr, 0); break; @@ -155,23 +138,11 @@ void umt_getoptions(utility::umt_optstruct_t* testops, int argc, char *argv[]) cerr << endl; usage(pname); } - - /* - * Note: Care must be taken when configuring the number of threads - * and the buffer size of umap. When the buffer size is set, it - * apportions the buffer evenly to the umap threads. So setting the - * buffer size requires that the number of threads be set properly - * first. - */ - if (testops->uffdthreads != umap_cfg_get_uffdthreads()) - umap_cfg_set_uffdthreads(testops->uffdthreads); - - umap_cfg_set_bufsize(testops->bufsize); } long umt_getpagesize(void) { - return umap_cfg_get_pagesize(); + return umapcfg_get_umap_page_size(); } } #endif // _COMMANDLING_HPP diff --git a/tests/utility/umap_file.hpp b/tests/utility/umap_file.hpp index 0ff717b1..34f820ac 100644 --- a/tests/utility/umap_file.hpp +++ b/tests/utility/umap_file.hpp @@ -7,10 +7,6 @@ #ifndef _UMAP_FILE_HPP_ #define _UMAP_FILE_HPP_ -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif - #include #include #include