Skip to content

Commit

Permalink
fix excess array object alignment
Browse files Browse the repository at this point in the history
  • Loading branch information
@JeffBezanson
JeffBezanson committed Jun 22, 2021
1 parent ae1b469 commit 01a7d27
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 26 deletions.
21 changes: 13 additions & 8 deletions src/array.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,20 +114,25 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
}

int ndimwords = jl_array_ndimwords(ndims);
int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT);
int tsz = sizeof(jl_array_t) + ndimwords*sizeof(size_t);
if (tot <= ARRAY_INLINE_NBYTES) {
// align data area
if (isunboxed && elsz >= 4)
tsz = JL_ARRAY_ALIGN(tsz, JL_SMALL_BYTE_ALIGNMENT); // align data area
tsz = JL_ARRAY_ALIGN(tsz, JL_SMALL_BYTE_ALIGNMENT);
if (tsz+tot > GC_MAX_SZCLASS) {
// object won't be pool allocated, so take advantage of larger alignment
tsz = JL_ARRAY_ALIGN(tsz, JL_CACHE_BYTE_ALIGNMENT);
}
size_t doffs = tsz;
tsz += tot;
tsz = JL_ARRAY_ALIGN(tsz, JL_SMALL_BYTE_ALIGNMENT); // align whole object
// jl_array_t is large enough that objects will always be aligned 16
a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, atype);
assert(((size_t)a & 15) == 0);
// No allocation or safepoint allowed after this
a->flags.how = 0;
data = (char*)a + doffs;
}
else {
tsz = JL_ARRAY_ALIGN(tsz, JL_CACHE_BYTE_ALIGNMENT); // align whole object
data = jl_gc_managed_malloc(tot);
// Allocate the Array **after** allocating the data
// to make sure the array is still young
Expand Down Expand Up @@ -223,7 +228,7 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data,
assert(jl_types_equal(jl_tparam0(jl_typeof(data)), jl_tparam0(atype)));

int ndimwords = jl_array_ndimwords(ndims);
int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords * sizeof(size_t) + sizeof(void*), JL_SMALL_BYTE_ALIGNMENT);
int tsz = sizeof(jl_array_t) + ndimwords * sizeof(size_t) + sizeof(void*);
a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, atype);
// No allocation or safepoint allowed after this
a->flags.pooled = tsz <= GC_MAX_SZCLASS;
Expand Down Expand Up @@ -304,7 +309,7 @@ JL_DLLEXPORT jl_array_t *jl_string_to_array(jl_value_t *str)
jl_array_t *a;

int ndimwords = jl_array_ndimwords(1);
int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t) + sizeof(void*), JL_SMALL_BYTE_ALIGNMENT);
int tsz = sizeof(jl_array_t) + ndimwords*sizeof(size_t) + sizeof(void*);
a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, jl_array_uint8_type);
a->flags.pooled = tsz <= GC_MAX_SZCLASS;
a->flags.ndims = 1;
Expand Down Expand Up @@ -351,7 +356,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data,
"unsafe_wrap: pointer %p is not properly aligned to %u bytes", data, align);

int ndimwords = jl_array_ndimwords(1);
int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT);
int tsz = sizeof(jl_array_t) + ndimwords*sizeof(size_t);
a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, atype);
// No allocation or safepoint allowed after this
a->flags.pooled = tsz <= GC_MAX_SZCLASS;
Expand Down Expand Up @@ -418,7 +423,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
"unsafe_wrap: pointer %p is not properly aligned to %u bytes", data, align);

int ndimwords = jl_array_ndimwords(ndims);
int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT);
int tsz = sizeof(jl_array_t) + ndimwords*sizeof(size_t);
a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, atype);
// No allocation or safepoint allowed after this
a->flags.pooled = tsz <= GC_MAX_SZCLASS;
Expand Down
47 changes: 31 additions & 16 deletions src/staticdata.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -782,14 +782,28 @@ static void jl_write_values(jl_serializer_state *s)
#define JL_ARRAY_ALIGN(jl_value, nbytes) LLT_ALIGN(jl_value, nbytes)
jl_array_t *ar = (jl_array_t*)v;
jl_value_t *et = jl_tparam0(jl_typeof(v));
size_t alen = jl_array_len(ar);
size_t datasize = alen * ar->elsize;
size_t tot = datasize;
int isbitsunion = jl_array_isbitsunion(ar);
if (isbitsunion)
tot += alen;
else if (ar->elsize == 1)
tot += 1;
int ndimwords = jl_array_ndimwords(ar->flags.ndims);
size_t tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords * sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT);
size_t headersize = sizeof(jl_array_t) + ndimwords*sizeof(size_t);
// copy header
ios_write(s->s, (char*)v, tsz);
ios_write(s->s, (char*)v, headersize);
size_t alignment_amt = 0, align16_padding = 0;
if (!ar->flags.ptrarray && ar->elsize >= 4) {
alignment_amt = JL_SMALL_BYTE_ALIGNMENT;
align16_padding = JL_ARRAY_ALIGN(headersize, JL_SMALL_BYTE_ALIGNMENT) - headersize;
}
// use the same cutoff as array.c for stricter alignment
if (headersize + tot + align16_padding > GC_MAX_SZCLASS)
alignment_amt = JL_CACHE_BYTE_ALIGNMENT;
// make some header modifications in-place
jl_array_t *newa = (jl_array_t*)&s->s->buf[reloc_offset];
size_t alen = jl_array_len(ar);
size_t tot = alen * ar->elsize;
if (newa->flags.ndims == 1)
newa->maxsize = alen;
newa->offset = 0;
Expand All @@ -799,8 +813,9 @@ static void jl_write_values(jl_serializer_state *s)

// write data
if (!ar->flags.ptrarray && !ar->flags.hasptr) {
uintptr_t data = LLT_ALIGN(ios_pos(s->const_data), 16);
// realign stream to max(data-align(array), sizeof(void*))
if (alignment_amt < 16)
alignment_amt = 16;
uintptr_t data = LLT_ALIGN(ios_pos(s->const_data), alignment_amt);
write_padding(s->const_data, data - ios_pos(s->const_data));
// write data and relocations
newa->data = NULL; // relocation offset
Expand All @@ -815,22 +830,22 @@ static void jl_write_values(jl_serializer_state *s)
write_pointer(s->const_data);
}
else {
int isbitsunion = jl_array_isbitsunion(ar);
if (ar->elsize == 1 && !isbitsunion)
tot += 1;
ios_write(s->const_data, (char*)jl_array_data(ar), tot);
if (isbitsunion)
ios_write(s->const_data, jl_array_typetagdata(ar), alen);
}
}
else {
newa->data = (void*)tsz; // relocation offset
if (alignment_amt) {
size_t aligned_hdr_sz = JL_ARRAY_ALIGN(headersize, alignment_amt);
write_padding(s->s, aligned_hdr_sz - headersize);
headersize = aligned_hdr_sz;
}
newa->data = (void*)headersize; // relocation offset
arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_array_t, data))); // relocation location
arraylist_push(&s->relocs_list, (void*)(((uintptr_t)DataRef << RELOC_TAG_OFFSET) + item)); // relocation target
if (ar->flags.hasptr) {
// copy all of the data first
const char *data = (const char*)jl_array_data(ar);
ios_write(s->s, data, tot);
ios_write(s->s, data, datasize);
// the rewrite all of the embedded pointers to null+relocation
uint16_t elsz = ar->elsize;
size_t j, np = ((jl_datatype_t*)et)->layout->npointers;
Expand All @@ -840,12 +855,12 @@ static void jl_write_values(jl_serializer_state *s)
size_t offset = i * elsz + jl_ptr_offset(((jl_datatype_t*)et), j) * sizeof(jl_value_t*);
jl_value_t *fld = *(jl_value_t**)&data[offset];
if (fld != NULL) {
arraylist_push(&s->relocs_list, (void*)(uintptr_t)(reloc_offset + tsz + offset)); // relocation location
arraylist_push(&s->relocs_list, (void*)(uintptr_t)(reloc_offset + headersize + offset)); // relocation location
arraylist_push(&s->relocs_list, (void*)backref_id(s, fld)); // relocation target
memset(&s->s->buf[reloc_offset + tsz + offset], 0, sizeof(fld)); // relocation offset (none)
memset(&s->s->buf[reloc_offset + headersize + offset], 0, sizeof(fld)); // relocation offset (none)
}
else {
assert(*(jl_value_t**)&s->s->buf[reloc_offset + tsz + offset] == NULL);
assert(*(jl_value_t**)&s->s->buf[reloc_offset + headersize + offset] == NULL);
}
}
}
Expand Down
12 changes: 10 additions & 2 deletions test/cmdlineargs.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,7 +324,11 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no`
rm(memfile)
@test popfirst!(got) == " 0 g(x) = x + 123456"
@test popfirst!(got) == " - function f(x)"
@test popfirst!(got) == " 80 []"
if Sys.WORD_SIZE == 64
@test popfirst!(got) == " 48 []"
else
@test popfirst!(got) == " 32 []"
end
if Sys.WORD_SIZE == 64
# P64 pools with 64 bit tags
@test popfirst!(got) == " 16 Base.invokelatest(g, 0)"
Expand All @@ -337,7 +341,11 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no`
@test popfirst!(got) == " 8 Base.invokelatest(g, 0)"
@test popfirst!(got) == " 32 Base.invokelatest(g, x)"
end
@test popfirst!(got) == " 80 []"
if Sys.WORD_SIZE == 64
@test popfirst!(got) == " 48 []"
else
@test popfirst!(got) == " 32 []"
end
@test popfirst!(got) == " - end"
@test popfirst!(got) == " - f(1.23)"
@test isempty(got) || got
Expand Down

0 comments on commit 01a7d27

Please sign in to comment.