Skip to content

Commit

Permalink
Add option to use pool implemenation which relies on memalign
Browse files Browse the repository at this point in the history
This commit add an option to build pony with the pool implementation
based on posix_memalign instead of the default pool implementation.
This, combined with address/undefined behavior/etc sanitizers, allows
for easier debugging of memory issues within the compiler and runtime.
This commit also includes two small bugfixes in the compiler that
were discovered as part of this:

* lexer.c fix in `normalise_string` to prevent issues when
  `lexer->buflen` == 0
* names.c fix in `names_typeparam` to prevent access to an AST node
  that was just replaced

and a fix to the `Makefile` for running tests with `use=` flags
and a fix to make sure the test runner gets rebuilt whenever ponyc
gets rebuilt
  • Loading branch information
dipinhora committed Jul 4, 2023
1 parent 64dcc74 commit 35cfb4a
Show file tree
Hide file tree
Showing 12 changed files with 191 additions and 31 deletions.
5 changes: 5 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,11 @@ if(PONY_USE_RUNTIMESTATS_MESSAGES)
add_compile_options(-DUSE_RUNTIMESTATS -DUSE_RUNTIMESTATS_MESSAGES)
endif()

if(PONY_USE_POOL_MEMALIGN)
set(PONY_OUTPUT_SUFFIX "-pool_memalign")
add_compile_options(-DUSE_POOL_MEMALIGN)
endif()

# LibPonyC tests assume that our outputs are two directories above the root directory.
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG "${CMAKE_BINARY_DIR}/../debug${PONY_OUTPUT_SUFFIX}")
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE "${CMAKE_BINARY_DIR}/../release${PONY_OUTPUT_SUFFIX}")
Expand Down
9 changes: 8 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,12 @@ endif

srcDir := $(shell dirname '$(subst /Volumes/Macintosh HD/,/,$(realpath $(lastword $(MAKEFILE_LIST))))')
buildDir := $(srcDir)/build/build_$(config)
outDir := $(srcDir)/build/$(config)

# get outDir from CMake install file or fall back to default if the file doesn't exist
outDir := $(subst /libponyrt.tests,,$(shell grep -o -s '$(srcDir)\/build\/$(config).*\/libponyrt.tests' $(buildDir)/cmake_install.cmake))
ifeq ($(outDir),)
outDir := $(srcDir)/build/$(config)
endif

libsSrcDir := $(srcDir)/lib
libsBuildDir := $(srcDir)/build/build_libs
Expand Down Expand Up @@ -141,6 +146,8 @@ define USE_CHECK
PONY_USES += -DPONY_USE_RUNTIMESTATS=true
else ifeq ($1,runtimestats_messages)
PONY_USES += -DPONY_USE_RUNTIMESTATS_MESSAGES=true
else ifeq ($1,pool_memalign)
PONY_USES += -DPONY_USE_POOL_MEMALIGN=true
else
$$(error ERROR: Unknown use option specified: $1)
endif
Expand Down
24 changes: 12 additions & 12 deletions src/libponyc/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -111,10 +111,10 @@ endif (MSVC)

if (NOT MSVC)
add_custom_command(TARGET libponyc POST_BUILD
COMMAND $<$<CONFIG:Debug>:${CMAKE_COMMAND}> ARGS -E copy ${libponyc_BINARY_DIR}/libponyc.a ${CMAKE_BINARY_DIR}/../debug/
COMMAND $<$<CONFIG:Release>:${CMAKE_COMMAND}> ARGS -E copy ${libponyc_BINARY_DIR}/libponyc.a ${CMAKE_BINARY_DIR}/../release/
COMMAND $<$<CONFIG:RelWithDebInfo>:${CMAKE_COMMAND}> ARGS -E copy ${libponyc_BINARY_DIR}/libponyc.a ${CMAKE_BINARY_DIR}/../relwithdebinfo/
COMMAND $<$<CONFIG:MinSizeRel>:${CMAKE_COMMAND}> ARGS -E copy ${libponyc_BINARY_DIR}/libponyc.a ${CMAKE_BINARY_DIR}/../minsizerel/
COMMAND $<$<CONFIG:Debug>:${CMAKE_COMMAND}> ARGS -E copy ${libponyc_BINARY_DIR}/libponyc.a ${CMAKE_BINARY_DIR}/../debug${PONY_OUTPUT_SUFFIX}/
COMMAND $<$<CONFIG:Release>:${CMAKE_COMMAND}> ARGS -E copy ${libponyc_BINARY_DIR}/libponyc.a ${CMAKE_BINARY_DIR}/../release${PONY_OUTPUT_SUFFIX}/
COMMAND $<$<CONFIG:RelWithDebInfo>:${CMAKE_COMMAND}> ARGS -E copy ${libponyc_BINARY_DIR}/libponyc.a ${CMAKE_BINARY_DIR}/../relwithdebinfo${PONY_OUTPUT_SUFFIX}/
COMMAND $<$<CONFIG:MinSizeRel>:${CMAKE_COMMAND}> ARGS -E copy ${libponyc_BINARY_DIR}/libponyc.a ${CMAKE_BINARY_DIR}/../minsizerel${PONY_OUTPUT_SUFFIX}/
)
endif (NOT MSVC)

Expand Down Expand Up @@ -146,10 +146,10 @@ elseif(${CMAKE_HOST_SYSTEM_NAME} MATCHES "Darwin")
)
# copy the generated file after it is built
add_custom_command(TARGET libponyc-standalone POST_BUILD
COMMAND $<$<CONFIG:Debug>:${CMAKE_COMMAND}> ARGS -E copy libponyc-standalone.a ${CMAKE_BINARY_DIR}/../debug/
COMMAND $<$<CONFIG:Release>:${CMAKE_COMMAND}> ARGS -E copy libponyc-standalone.a ${CMAKE_BINARY_DIR}/../release/
COMMAND $<$<CONFIG:RelWithDebInfo>:${CMAKE_COMMAND}> ARGS -E copy libponyc-standalone.a ${CMAKE_BINARY_DIR}/../relwithdebinfo/
COMMAND $<$<CONFIG:MinSizeRel>:${CMAKE_COMMAND}> ARGS -E copy libponyc-standalone.a ${CMAKE_BINARY_DIR}/../minsizerel/
COMMAND $<$<CONFIG:Debug>:${CMAKE_COMMAND}> ARGS -E copy libponyc-standalone.a ${CMAKE_BINARY_DIR}/../debug${PONY_OUTPUT_SUFFIX}/
COMMAND $<$<CONFIG:Release>:${CMAKE_COMMAND}> ARGS -E copy libponyc-standalone.a ${CMAKE_BINARY_DIR}/../release${PONY_OUTPUT_SUFFIX}/
COMMAND $<$<CONFIG:RelWithDebInfo>:${CMAKE_COMMAND}> ARGS -E copy libponyc-standalone.a ${CMAKE_BINARY_DIR}/../relwithdebinfo${PONY_OUTPUT_SUFFIX}/
COMMAND $<$<CONFIG:MinSizeRel>:${CMAKE_COMMAND}> ARGS -E copy libponyc-standalone.a ${CMAKE_BINARY_DIR}/../minsizerel${PONY_OUTPUT_SUFFIX}/
)
elseif(${CMAKE_HOST_SYSTEM_NAME} MATCHES "BSD")
# TODO
Expand All @@ -176,9 +176,9 @@ else()
)
# copy the generated file after it is built
add_custom_command(TARGET libponyc-standalone POST_BUILD
COMMAND $<$<CONFIG:Debug>:${CMAKE_COMMAND}> ARGS -E copy libponyc-standalone.a ${CMAKE_BINARY_DIR}/../debug/
COMMAND $<$<CONFIG:Release>:${CMAKE_COMMAND}> ARGS -E copy libponyc-standalone.a ${CMAKE_BINARY_DIR}/../release/
COMMAND $<$<CONFIG:RelWithDebInfo>:${CMAKE_COMMAND}> ARGS -E copy libponyc-standalone.a ${CMAKE_BINARY_DIR}/../relwithdebinfo/
COMMAND $<$<CONFIG:MinSizeRel>:${CMAKE_COMMAND}> ARGS -E copy libponyc-standalone.a ${CMAKE_BINARY_DIR}/../minsizerel/
COMMAND $<$<CONFIG:Debug>:${CMAKE_COMMAND}> ARGS -E copy libponyc-standalone.a ${CMAKE_BINARY_DIR}/../debug${PONY_OUTPUT_SUFFIX}/
COMMAND $<$<CONFIG:Release>:${CMAKE_COMMAND}> ARGS -E copy libponyc-standalone.a ${CMAKE_BINARY_DIR}/../release${PONY_OUTPUT_SUFFIX}/
COMMAND $<$<CONFIG:RelWithDebInfo>:${CMAKE_COMMAND}> ARGS -E copy libponyc-standalone.a ${CMAKE_BINARY_DIR}/../relwithdebinfo${PONY_OUTPUT_SUFFIX}/
COMMAND $<$<CONFIG:MinSizeRel>:${CMAKE_COMMAND}> ARGS -E copy libponyc-standalone.a ${CMAKE_BINARY_DIR}/../minsizerel${PONY_OUTPUT_SUFFIX}/
)
endif (MSVC)
2 changes: 1 addition & 1 deletion src/libponyc/ast/lexer.c
Original file line number Diff line number Diff line change
Expand Up @@ -607,7 +607,7 @@ static void normalise_string(lexer_t* lexer)

// Trim trailing empty line
size_t trim = 0;
for (size_t i = (lexer->buflen - 1); i>0; i--)
for (ssize_t i = (lexer->buflen - 1); i>0; i--)
{
char c = lexer->buffer[i];
if (c == '\n')
Expand Down
2 changes: 2 additions & 0 deletions src/libponyc/pass/names.c
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,8 @@ static bool names_typeparam(pass_opt_t* opt, ast_t** astp, ast_t* def)
TREE(cap)
TREE(ephemeral)));

ast = *astp;

if(opt->check.frame->iftype_body != NULL)
def = ast_get(ast, name, NULL);

Expand Down
9 changes: 5 additions & 4 deletions src/libponyrt/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ set(_c_src
mem/heap.c
mem/pagemap.c
mem/pool.c
mem/pool_memalign.c
options/options.c
platform/ponyassert.c
platform/threads.c
Expand Down Expand Up @@ -156,9 +157,9 @@ if(PONY_RUNTIME_BITCODE)
)

add_custom_command(TARGET libponyrt_bc POST_BUILD
COMMAND $<$<CONFIG:Debug>:${CMAKE_COMMAND}> ARGS -E copy ${libponyrt_BINARY_DIR}/libponyrt.bc ${CMAKE_BINARY_DIR}/../debug/
COMMAND $<$<CONFIG:Release>:${CMAKE_COMMAND}> ARGS -E copy ${libponyrt_BINARY_DIR}/libponyrt.bc ${CMAKE_BINARY_DIR}/../release/
COMMAND $<$<CONFIG:RelWithDebInfo>:${CMAKE_COMMAND}> ARGS -E copy ${libponyrt_BINARY_DIR}/libponyrt.bc ${CMAKE_BINARY_DIR}/../relwithdebinfo/
COMMAND $<$<CONFIG:MinSizeRel>:${CMAKE_COMMAND}> ARGS -E copy ${libponyrt_BINARY_DIR}/libponyrt.bc ${CMAKE_BINARY_DIR}/../minsizerel/
COMMAND $<$<CONFIG:Debug>:${CMAKE_COMMAND}> ARGS -E copy ${libponyrt_BINARY_DIR}/libponyrt.bc ${CMAKE_BINARY_DIR}/../debug${PONY_OUTPUT_SUFFIX}/
COMMAND $<$<CONFIG:Release>:${CMAKE_COMMAND}> ARGS -E copy ${libponyrt_BINARY_DIR}/libponyrt.bc ${CMAKE_BINARY_DIR}/../release${PONY_OUTPUT_SUFFIX}/
COMMAND $<$<CONFIG:RelWithDebInfo>:${CMAKE_COMMAND}> ARGS -E copy ${libponyrt_BINARY_DIR}/libponyrt.bc ${CMAKE_BINARY_DIR}/../relwithdebinfo${PONY_OUTPUT_SUFFIX}/
COMMAND $<$<CONFIG:MinSizeRel>:${CMAKE_COMMAND}> ARGS -E copy ${libponyrt_BINARY_DIR}/libponyrt.bc ${CMAKE_BINARY_DIR}/../minsizerel${PONY_OUTPUT_SUFFIX}/
)
endif()
14 changes: 9 additions & 5 deletions src/libponyrt/mem/pool.c
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,18 @@
#include <platform.h>
#include <pony/detail/atomics.h>

#ifdef USE_VALGRIND
#include <valgrind/valgrind.h>
#include <valgrind/helgrind.h>
#endif

/// Allocations this size and above are aligned on this size. This is needed
/// so that the pagemap for the heap is aligned.
#define POOL_ALIGN_INDEX (POOL_ALIGN_BITS - POOL_MIN_BITS)
#define POOL_ALIGN_MASK (POOL_ALIGN - 1)

#ifdef POOL_USE_DEFAULT

#ifdef USE_VALGRIND
#include <valgrind/valgrind.h>
#include <valgrind/helgrind.h>
#endif

/// When we mmap, pull at least this many bytes.
#ifdef PLATFORM_IS_ILP32
# define POOL_MMAP (16 * 1024 * 1024) // 16 MB
Expand Down Expand Up @@ -998,6 +1000,8 @@ void ponyint_pool_thread_cleanup()
pool_block_header.largest_size = 0;
}

#endif

size_t ponyint_pool_index(size_t size)
{
#ifdef PLATFORM_IS_ILP32
Expand Down
6 changes: 6 additions & 0 deletions src/libponyrt/mem/pool.h
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@

#include <platform.h>

#if defined(USE_POOL_MEMALIGN)
# define POOL_USE_MEMALIGN
#else
# define POOL_USE_DEFAULT
#endif

PONY_EXTERN_C_BEGIN

/* Because of the way free memory is reused as its own linked list container,
Expand Down
130 changes: 130 additions & 0 deletions src/libponyrt/mem/pool_memalign.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
#define PONY_WANT_ATOMIC_DEFS

#include "pool.h"
#include "alloc.h"
#include "../ds/fun.h"
#include "../sched/cpu.h"
#include "ponyassert.h"
#include <stdint.h>
#include <stdbool.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include <platform.h>
#include <pony/detail/atomics.h>

#ifdef POOL_USE_MEMALIGN

void* ponyint_pool_alloc(size_t index)
{
void* p;
size_t size = POOL_SIZE(index);
if(size < POOL_ALIGN)
{
p = malloc(size);
}
else
{
int code = posix_memalign(&p, POOL_ALIGN, size);
pony_assert(code == 0);
}

return p;
}

void ponyint_pool_free(size_t index, void* p)
{
pony_assert(index < POOL_COUNT);

free(p);
}

static void* pool_alloc_size(size_t size)
{
void* p;
int code = posix_memalign(&p, POOL_ALIGN, size);
pony_assert(code == 0);

return p;
}

void* ponyint_pool_alloc_size(size_t size)
{
size_t index = ponyint_pool_index(size);

if(index < POOL_COUNT)
return ponyint_pool_alloc(index);

size = ponyint_pool_adjust_size(size);
void* p = pool_alloc_size(size);

return p;
}

static void pool_free_size(size_t size, void* p)
{
free(p);
(void)size;
}

void ponyint_pool_free_size(size_t size, void* p)
{
size_t index = ponyint_pool_index(size);

if(index < POOL_COUNT)
return ponyint_pool_free(index, p);

size = ponyint_pool_adjust_size(size);
pool_free_size(size, p);
}

void* ponyint_pool_realloc_size(size_t old_size, size_t new_size, void* p)
{
// Can only reuse the old pointer if the old index/adjusted size is equal to
// the new one, not greater.

if(p == NULL)
return ponyint_pool_alloc_size(new_size);

size_t old_index = ponyint_pool_index(old_size);
size_t new_index = ponyint_pool_index(new_size);
size_t old_adj_size = 0;

void* new_p;

if(new_index < POOL_COUNT)
{
if(old_index == new_index)
return p;

new_p = ponyint_pool_alloc(new_index);
} else {
size_t new_adj_size = ponyint_pool_adjust_size(new_size);

if(old_index >= POOL_COUNT)
{
old_adj_size = ponyint_pool_adjust_size(old_size);

if(old_adj_size == new_adj_size)
return p;
}

new_p = pool_alloc_size(new_adj_size);
}

memcpy(new_p, p, old_size < new_size ? old_size : new_size);

if(old_index < POOL_COUNT)
ponyint_pool_free(old_index, p);
else
pool_free_size(old_adj_size, p);

return new_p;
}

void ponyint_pool_thread_cleanup()
{
}

#endif
16 changes: 8 additions & 8 deletions test/libponyc-run/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,14 @@ foreach(test_c_source ${test_c_sources})
endif()

add_custom_command(TARGET ${test_lib_name} POST_BUILD
COMMAND $<$<CONFIG:Debug>:${CMAKE_COMMAND}> ARGS -E make_directory ${CMAKE_BINARY_DIR}/../debug/test_lib
COMMAND $<$<CONFIG:Release>:${CMAKE_COMMAND}> ARGS -E make_directory ${CMAKE_BINARY_DIR}/../release/test_lib
COMMAND $<$<CONFIG:RelWithDebInfo>:${CMAKE_COMMAND}> ARGS -E make_directory ${CMAKE_BINARY_DIR}/../relwithdebinfo/test_lib
COMMAND $<$<CONFIG:MinSizeRel>:${CMAKE_COMMAND}> ARGS -E make_directory ${CMAKE_BINARY_DIR}/../minsizerel/test_lib
COMMAND $<$<CONFIG:Debug>:${CMAKE_COMMAND}> ARGS -E make_directory ${CMAKE_BINARY_DIR}/../debug${PONY_OUTPUT_SUFFIX}/test_lib
COMMAND $<$<CONFIG:Release>:${CMAKE_COMMAND}> ARGS -E make_directory ${CMAKE_BINARY_DIR}/../release${PONY_OUTPUT_SUFFIX}/test_lib
COMMAND $<$<CONFIG:RelWithDebInfo>:${CMAKE_COMMAND}> ARGS -E make_directory ${CMAKE_BINARY_DIR}/../relwithdebinfo${PONY_OUTPUT_SUFFIX}/test_lib
COMMAND $<$<CONFIG:MinSizeRel>:${CMAKE_COMMAND}> ARGS -E make_directory ${CMAKE_BINARY_DIR}/../minsizerel${PONY_OUTPUT_SUFFIX}/test_lib

COMMAND $<$<CONFIG:Debug>:${CMAKE_COMMAND}> ARGS -E copy $<TARGET_FILE:${test_lib_name}> ${CMAKE_BINARY_DIR}/../debug/test_lib
COMMAND $<$<CONFIG:Release>:${CMAKE_COMMAND}> ARGS -E copy $<TARGET_FILE:${test_lib_name}> ${CMAKE_BINARY_DIR}/../release/test_lib
COMMAND $<$<CONFIG:RelWithDebInfo>:${CMAKE_COMMAND}> ARGS -E copy $<TARGET_FILE:${test_lib_name}> ${CMAKE_BINARY_DIR}/../relwithdebinfo/test_lib
COMMAND $<$<CONFIG:MinSizeRel>:${CMAKE_COMMAND}> ARGS -E copy $<TARGET_FILE:${test_lib_name}> ${CMAKE_BINARY_DIR}/../minsizerel/test_lib
COMMAND $<$<CONFIG:Debug>:${CMAKE_COMMAND}> ARGS -E copy $<TARGET_FILE:${test_lib_name}> ${CMAKE_BINARY_DIR}/../debug${PONY_OUTPUT_SUFFIX}/test_lib
COMMAND $<$<CONFIG:Release>:${CMAKE_COMMAND}> ARGS -E copy $<TARGET_FILE:${test_lib_name}> ${CMAKE_BINARY_DIR}/../release${PONY_OUTPUT_SUFFIX}/test_lib
COMMAND $<$<CONFIG:RelWithDebInfo>:${CMAKE_COMMAND}> ARGS -E copy $<TARGET_FILE:${test_lib_name}> ${CMAKE_BINARY_DIR}/../relwithdebinfo${PONY_OUTPUT_SUFFIX}/test_lib
COMMAND $<$<CONFIG:MinSizeRel>:${CMAKE_COMMAND}> ARGS -E copy $<TARGET_FILE:${test_lib_name}> ${CMAKE_BINARY_DIR}/../minsizerel${PONY_OUTPUT_SUFFIX}/test_lib
)
endforeach()
1 change: 1 addition & 0 deletions test/libponyc-run/runner/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,5 @@ add_custom_command(OUTPUT ${RUNNER_EXECUTABLE}
_test_process_notify.pony
_tester_timer_notify.pony
_tester.pony
$<TARGET_FILE:ponyc>
)
4 changes: 4 additions & 0 deletions test/libponyrt/mem/pool.cc
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

#include <gtest/gtest.h>

#ifdef POOL_USE_DEFAULT

typedef char block_t[32];

TEST(Pool, Fifo)
Expand Down Expand Up @@ -71,6 +73,8 @@ TEST(Pool, LargeAlloc)
ponyint_pool_free_size(1 << 20, p);
}

#endif

TEST(Pool, Index)
{
size_t index = ponyint_pool_index(1);
Expand Down

0 comments on commit 35cfb4a

Please sign in to comment.