diff --git a/src/alloc.c b/src/alloc.c index b51d4651b3984..d2487070da54c 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -161,10 +161,7 @@ static jl_value_t *jl_new_bits_internal(jl_value_t *dt, void *data, size_t *len) if (bt == jl_int32_type) return jl_box_int32(*(int32_t*)data); if (bt == jl_float64_type) return jl_box_float64(*(double*)data); - jl_value_t *v = - (jl_value_t*)allocobj((NWORDS(LLT_ALIGN(nb,sizeof(void*)))+1)* - sizeof(void*)); - v->type = (jl_value_t*)bt; + jl_value_t *v = (jl_value_t*)newobj((jl_value_t*)bt, NWORDS(nb)); switch (nb) { case 1: *(int8_t*) jl_data_ptr(v) = *(int8_t*)data; break; case 2: *(int16_t*) jl_data_ptr(v) = *(int16_t*)data; break; @@ -252,7 +249,7 @@ jl_value_t *jl_get_nth_field(jl_value_t *v, size_t i) { jl_datatype_t *st = (jl_datatype_t*)jl_typeof(v); assert(i < jl_tuple_len(st->names)); - size_t offs = jl_field_offset(st,i) + sizeof(void*); + size_t offs = jl_field_offset(st,i); if (st->fields[i].isptr) { return *(jl_value_t**)((char*)v + offs); } @@ -264,7 +261,7 @@ jl_value_t *jl_get_nth_field_checked(jl_value_t *v, size_t i) jl_datatype_t *st = (jl_datatype_t*)jl_typeof(v); if (i >= jl_tuple_len(st->names)) jl_bounds_error_int(v, i+1); - size_t offs = jl_field_offset(st,i) + sizeof(void*); + size_t offs = jl_field_offset(st,i); if (st->fields[i].isptr) { jl_value_t *fval = *(jl_value_t**)((char*)v + offs); if (fval == NULL) @@ -277,7 +274,7 @@ jl_value_t *jl_get_nth_field_checked(jl_value_t *v, size_t i) void jl_set_nth_field(jl_value_t *v, size_t i, jl_value_t *rhs) { jl_datatype_t *st = (jl_datatype_t*)jl_typeof(v); - size_t offs = jl_field_offset(st,i) + sizeof(void*); + size_t offs = jl_field_offset(st,i); if (st->fields[i].isptr) { *(jl_value_t**)((char*)v + offs) = rhs; if(rhs != NULL) gc_wb(v, rhs); @@ -290,7 +287,7 @@ void jl_set_nth_field(jl_value_t *v, size_t i, jl_value_t *rhs) int jl_field_isdefined(jl_value_t *v, size_t i) { jl_datatype_t *st = (jl_datatype_t*)jl_typeof(v); - size_t offs = jl_field_offset(st,i) + sizeof(void*); + size_t offs = jl_field_offset(st,i); if (st->fields[i].isptr) { return *(jl_value_t**)((char*)v + offs) != NULL; } @@ -321,7 +318,7 @@ DLLEXPORT jl_value_t *jl_new_structv(jl_datatype_t *type, jl_value_t **args, uin } for(size_t i=na; i < nf; i++) { if (type->fields[i].isptr) - *(jl_value_t**)((char*)jv+jl_field_offset(type,i)+sizeof(void*)) = NULL; + *(jl_value_t**)((char*)jl_data_ptr(jv)+jl_field_offset(type,i)) = NULL; } return jv; } @@ -331,7 +328,7 @@ DLLEXPORT jl_value_t *jl_new_struct_uninit(jl_datatype_t *type) if (type->instance != NULL) return type->instance; jl_value_t *jv = newstruct(type); if (type->size > 0) - memset(&((void**)jv)[1], 0, type->size); + memset(jl_data_ptr(jv), 0, type->size); return jv; } @@ -360,11 +357,11 @@ DLLEXPORT jl_tuple_t *jl_tuplev(size_t n, jl_value_t **v) jl_tuple_t *jl_tuple1(void *a) { #ifdef OVERLAP_TUPLE_LEN - jl_tuple_t *t = (jl_tuple_t*)alloc_2w(); + jl_tuple_t *t = (jl_tuple_t*)alloc_1w(); #else - jl_tuple_t *t = (jl_tuple_t*)alloc_3w(); + jl_tuple_t *t = (jl_tuple_t*)alloc_2w(); #endif - t->type = (jl_value_t*)jl_tuple_type; + jl_set_typeof(t, jl_tuple_type); jl_tuple_set_len_unsafe(t, 1); jl_tupleset(t, 0, a); return t; @@ -373,11 +370,11 @@ jl_tuple_t *jl_tuple1(void *a) jl_tuple_t *jl_tuple2(void *a, void *b) { #ifdef OVERLAP_TUPLE_LEN - jl_tuple_t *t = (jl_tuple_t*)alloc_3w(); + jl_tuple_t *t = (jl_tuple_t*)alloc_2w(); #else - jl_tuple_t *t = (jl_tuple_t*)alloc_4w(); + jl_tuple_t *t = (jl_tuple_t*)alloc_3w(); #endif - t->type = (jl_value_t*)jl_tuple_type; + jl_set_typeof(t, jl_tuple_type); jl_tuple_set_len_unsafe(t, 2); jl_tupleset(t, 0, a); jl_tupleset(t, 1, b); @@ -434,8 +431,8 @@ jl_tuple_t *jl_tuple_fill(size_t n, jl_value_t *v) DLLEXPORT jl_function_t *jl_new_closure(jl_fptr_t fptr, jl_value_t *env, jl_lambda_info_t *linfo) { - jl_function_t *f = (jl_function_t*)alloc_4w(); - f->type = (jl_value_t*)jl_function_type; + jl_function_t *f = (jl_function_t*)alloc_3w(); assert(NWORDS(sizeof(jl_function_t))==3); + jl_set_typeof(f, jl_function_type); f->fptr = (fptr!=NULL ? fptr : linfo->fptr); f->env = env; f->linfo = linfo; @@ -447,7 +444,7 @@ jl_lambda_info_t *jl_new_lambda_info(jl_value_t *ast, jl_tuple_t *sparams) { jl_lambda_info_t *li = (jl_lambda_info_t*)newobj((jl_value_t*)jl_lambda_info_type, - LAMBDA_INFO_NW); + NWORDS(sizeof(jl_lambda_info_t))); li->ast = ast; li->file = null_sym; li->line = 0; @@ -499,23 +496,23 @@ static jl_sym_t *mk_symbol(const char *str) #endif jl_sym_t *sym; size_t len = strlen(str); - size_t nb = (sizeof(jl_sym_t)+len+1+7)&-8; + size_t nb = (sizeof(jl_typetag_t*)+sizeof(jl_sym_t)+len+1+7)&-8; if (nb >= SYM_POOL_SIZE) { jl_exceptionf(jl_argumenterror_type, "Symbol length exceeds maximum length"); } #ifdef MEMDEBUG - sym = (jl_sym_t*)malloc(nb); + sym = (jl_sym_t*)((jl_typetag_t*)malloc(nb))->value; #else if (sym_pool == NULL || pool_ptr+nb > sym_pool+SYM_POOL_SIZE) { sym_pool = (char*)malloc(SYM_POOL_SIZE); pool_ptr = sym_pool; } - sym = (jl_sym_t*)pool_ptr; + sym = (jl_sym_t*)((jl_typetag_t*)malloc(nb))->value; pool_ptr += nb; #endif - sym->type = (jl_value_t*)jl_sym_type; + jl_set_typeof(sym, jl_sym_type); sym->left = sym->right = NULL; sym->hash = hash_symbol(str, len); strcpy(&sym->name[0], str); @@ -525,7 +522,7 @@ static jl_sym_t *mk_symbol(const char *str) static void unmark_symbols_(jl_sym_t *root) { while (root != NULL) { - root->type = (jl_value_t*)(((uptrint_t)root->type)&~3UL); + jl_set_typeof(root, jl_sym_type); unmark_symbols_(root->left); root = root->right; } @@ -619,7 +616,7 @@ DLLEXPORT jl_sym_t *jl_tagged_gensym(const char *str, int32_t len) jl_typename_t *jl_new_typename(jl_sym_t *name) { - jl_typename_t *tn=(jl_typename_t*)newobj((jl_value_t*)jl_typename_type, 6); + jl_typename_t *tn=(jl_typename_t*)newobj((jl_value_t*)jl_typename_type, NWORDS(sizeof(jl_typename_t))); tn->name = name; tn->module = jl_current_module; tn->primary = NULL; @@ -641,8 +638,7 @@ jl_datatype_t *jl_new_uninitialized_datatype(size_t nfields) { return (jl_datatype_t*) newobj((jl_value_t*)jl_datatype_type, - NWORDS(sizeof(jl_datatype_t) - sizeof(void*) + - nfields*sizeof(jl_fielddesc_t))); + NWORDS(sizeof(jl_datatype_t) + nfields*sizeof(jl_fielddesc_t))); } void jl_compute_field_offsets(jl_datatype_t *st) @@ -766,7 +762,7 @@ jl_datatype_t *jl_new_bitstype(jl_value_t *name, jl_datatype_t *super, jl_uniontype_t *jl_new_uniontype(jl_tuple_t *types) { - jl_uniontype_t *t = (jl_uniontype_t*)newobj((jl_value_t*)jl_uniontype_type,1); + jl_uniontype_t *t = (jl_uniontype_t*)newobj((jl_value_t*)jl_uniontype_type,NWORDS(sizeof(jl_uniontype_t))); // don't make unions of 1 type; Union(T)==T assert(jl_tuple_len(types) != 1); t->types = types; @@ -777,7 +773,7 @@ jl_uniontype_t *jl_new_uniontype(jl_tuple_t *types) jl_typector_t *jl_new_type_ctor(jl_tuple_t *params, jl_value_t *body) { - jl_typector_t *tc = (jl_typector_t*)newobj((jl_value_t*)jl_typector_type,2); + jl_typector_t *tc = (jl_typector_t*)newobj((jl_value_t*)jl_typector_type,NWORDS(sizeof(jl_typector_t))); tc->parameters = params; tc->body = body; return (jl_typector_t*)tc; @@ -791,7 +787,7 @@ jl_value_t *jl_box##nb(jl_datatype_t *t, int##nb##_t x) \ assert(jl_isbits(t)); \ assert(jl_datatype_size(t) == sizeof(x)); \ jl_value_t *v = (jl_value_t*)alloc_##nw##w(); \ - v->type = (jl_value_t*)t; \ + jl_set_typeof(v, t); \ *(int##nb##_t*)jl_data_ptr(v) = x; \ return v; \ } @@ -829,7 +825,7 @@ UNBOX_FUNC(gensym, ssize_t) jl_value_t *pfx##_##typ(c_type x) \ { \ jl_value_t *v = (jl_value_t*)alloc_##nw##w(); \ - v->type = (jl_value_t*)jl_##typ##_type; \ + jl_set_typeof(v, jl_##typ##_type); \ *(c_type*)jl_data_ptr(v) = x; \ return v; \ } @@ -851,7 +847,7 @@ jl_value_t *jl_box_##typ(c_type x) \ if ((u##c_type)idx < (u##c_type)NBOX_C) \ return boxed_##typ##_cache[idx]; \ jl_value_t *v = (jl_value_t*)alloc_##nw##w(); \ - v->type = (jl_value_t*)jl_##typ##_type; \ + jl_set_typeof(v, jl_##typ##_type); \ *(c_type*)jl_data_ptr(v) = x; \ return v; \ } @@ -862,7 +858,7 @@ jl_value_t *jl_box_##typ(c_type x) \ if (x < NBOX_C) \ return boxed_##typ##_cache[x]; \ jl_value_t *v = (jl_value_t*)alloc_##nw##w(); \ - v->type = (jl_value_t*)jl_##typ##_type; \ + jl_set_typeof(v, jl_##typ##_type); \ *(c_type*)jl_data_ptr(v) = x; \ return v; \ } @@ -955,8 +951,8 @@ jl_expr_t *jl_exprn(jl_sym_t *head, size_t n) { jl_array_t *ar = n==0 ? (jl_array_t*)jl_an_empty_cell : jl_alloc_cell_1d(n); JL_GC_PUSH1(&ar); - jl_expr_t *ex = (jl_expr_t*)alloc_4w(); - ex->type = (jl_value_t*)jl_expr_type; + jl_expr_t *ex = (jl_expr_t*)alloc_3w(); assert(NWORDS(sizeof(jl_expr_t))==3); + jl_set_typeof(ex, jl_expr_type); ex->head = head; ex->args = ar; ex->etype = (jl_value_t*)jl_any_type; @@ -972,8 +968,8 @@ JL_CALLABLE(jl_f_new_expr) JL_GC_PUSH1(&ar); for(size_t i=0; i < nargs-1; i++) jl_cellset(ar, i, args[i+1]); - jl_expr_t *ex = (jl_expr_t*)alloc_4w(); - ex->type = (jl_value_t*)jl_expr_type; + jl_expr_t *ex = (jl_expr_t*)alloc_3w(); assert(NWORDS(sizeof(jl_expr_t))==3); + jl_set_typeof(ex, jl_expr_type); ex->head = (jl_sym_t*)args[0]; ex->args = ar; ex->etype = (jl_value_t*)jl_any_type; diff --git a/src/array.c b/src/array.c index d6f80a720f3f6..bb517c41b125d 100644 --- a/src/array.c +++ b/src/array.c @@ -78,7 +78,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims, tsz += tot; tsz = (tsz+15)&-16; // align whole object 16 a = (jl_array_t*)allocobj(tsz); - a->type = atype; + jl_set_typeof(a, atype); a->how = 0; data = (char*)a + doffs; if (tot > 0 && !isunboxed) { @@ -89,7 +89,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims, tsz = (tsz+15)&-16; // align whole object size 16 a = (jl_array_t*)allocobj(tsz); JL_GC_PUSH1(&a); - a->type = atype; + jl_set_typeof(a, atype); // temporarily initialize to make gc-safe a->data = NULL; a->how = 2; @@ -150,7 +150,7 @@ jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, jl_tuple_t *di int ndimwords = jl_array_ndimwords(ndims); int tsz = (sizeof(jl_array_t) + sizeof(void*) + ndimwords*sizeof(size_t) + 15)&-16; a = (jl_array_t*)allocobj(tsz); - a->type = atype; + jl_set_typeof(a, atype); a->pooled = tsz <= 2048; a->ndims = ndims; a->offset = 0; @@ -216,8 +216,8 @@ jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel, elsz = sizeof(void*); int tsz = (sizeof(jl_array_t)+jl_array_ndimwords(1)*sizeof(size_t)+15)&-16; a = (jl_array_t*)allocobj(tsz); + jl_set_typeof(a, atype); a->pooled = tsz <= 2048; - a->type = atype; a->data = data; #ifdef STORE_ARRAY_LEN a->length = nel; @@ -267,8 +267,8 @@ jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_tuple_t *dims, int ndimwords = jl_array_ndimwords(ndims); int tsz = (sizeof(jl_array_t) + ndimwords*sizeof(size_t)+15)&-16; a = (jl_array_t*)allocobj(tsz); + jl_set_typeof(a, atype); a->pooled = tsz <= 2048; - a->type = atype; a->data = data; #ifdef STORE_ARRAY_LEN a->length = nel; @@ -340,8 +340,8 @@ jl_value_t *jl_array_to_string(jl_array_t *a) // TODO: check type of array? jl_datatype_t *string_type = u8_isvalid((char*)a->data, jl_array_len(a)) == 1 ? // ASCII jl_ascii_string_type : jl_utf8_string_type; - jl_value_t *s = (jl_value_t*)alloc_2w(); - s->type = (jl_value_t*)string_type; + jl_value_t *s = (jl_value_t*)alloc_1w(); + jl_set_typeof(s, string_type); jl_set_nth_field(s, 0, (jl_value_t*)a); return s; } diff --git a/src/builtins.c b/src/builtins.c index 4429f25c48d84..d41233a760a37 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -1222,8 +1222,8 @@ size_t jl_static_show_x(JL_STREAM *out, jl_value_t *v, int depth) else if (jl_typeof(v) == NULL) { n += JL_PRINTF(out, ""); } - else if ((uptrint_t)v->type < 4096U) { - n += JL_PRINTF(out, "", (int)(uptrint_t)v->type); + else if ((uptrint_t)jl_typetagof(v)->type < 4096U) { + n += JL_PRINTF(out, "", (int)(uptrint_t)jl_typetagof(v)->type); } else if (jl_is_lambda_info(v)) { jl_lambda_info_t *li = (jl_lambda_info_t*)v; diff --git a/src/ccall.cpp b/src/ccall.cpp index 03f73af8aa406..4107c792c34f0 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -326,7 +326,7 @@ static Value *julia_to_native(Type *ty, jl_value_t *jt, Value *jv, } if (jl_is_mutable_datatype(ety)) { // no copy, just reference the data field - return builder.CreateBitCast(emit_nthptr_addr(jv, (size_t)1), ty); // skip type tag field + return builder.CreateBitCast(jv, ty); } else if (jl_is_immutable_datatype(ety) && jt != (jl_value_t*)jl_voidpointer_type) { // yes copy @@ -336,12 +336,12 @@ static Value *julia_to_native(Type *ty, jl_value_t *jt, Value *jv, else nbytes = tbaa_decorate(tbaa_datatype, builder.CreateLoad( builder.CreateGEP(builder.CreatePointerCast(emit_typeof(jv), T_pint32), - ConstantInt::get(T_size, offsetof(jl_datatype_t,size)/4)), + ConstantInt::get(T_size, offsetof(jl_datatype_t,size)/sizeof(int32_t))), false)); *needStackRestore = true; AllocaInst *ai = builder.CreateAlloca(T_int8, nbytes); ai->setAlignment(16); - builder.CreateMemCpy(ai, builder.CreateBitCast(emit_nthptr_addr(jv, (size_t)1), T_pint8), nbytes, 1); + builder.CreateMemCpy(ai, builder.CreateBitCast(jv, T_pint8), nbytes, 1); return builder.CreateBitCast(ai, ty); } // emit maybe copy @@ -358,16 +358,16 @@ static Value *julia_to_native(Type *ty, jl_value_t *jt, Value *jv, T_int1); builder.CreateCondBr(ismutable, mutableBB, immutableBB); builder.SetInsertPoint(mutableBB); - Value *p1 = builder.CreatePointerCast(emit_nthptr_addr(jv, (size_t)1), ty); // skip type tag field + Value *p1 = builder.CreatePointerCast(jv, ty); builder.CreateBr(afterBB); builder.SetInsertPoint(immutableBB); Value *nbytes = tbaa_decorate(tbaa_datatype, builder.CreateLoad( builder.CreateGEP(builder.CreatePointerCast(jvt, T_pint32), - ConstantInt::get(T_size, offsetof(jl_datatype_t,size)/4)), + ConstantInt::get(T_size, offsetof(jl_datatype_t,size)/sizeof(int32_t))), false)); AllocaInst *ai = builder.CreateAlloca(T_int8, nbytes); ai->setAlignment(16); - builder.CreateMemCpy(ai, builder.CreatePointerCast(emit_nthptr_addr(jv, (size_t)1), T_pint8), nbytes, 1); + builder.CreateMemCpy(ai, builder.CreatePointerCast(jv, T_pint8), nbytes, 1); Value *p2 = builder.CreatePointerCast(ai, ty); builder.CreateBr(afterBB); builder.SetInsertPoint(afterBB); @@ -1149,7 +1149,7 @@ static Value *emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) ary = emit_unbox(largty, emit_unboxed(argi, ctx), jl_tupleref(tt, 0)); } JL_GC_POP(); - return mark_or_box_ccall_result(builder.CreateBitCast(emit_nthptr_addr(ary, addressOf?1:0), lrt), + return mark_or_box_ccall_result(builder.CreateBitCast(ary, lrt), args[2], rt, static_rt, ctx); } if (fptr == (void *) &jl_is_leaf_type || @@ -1226,7 +1226,7 @@ static Value *emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) result = mem; argvals[0] = result; } else { - argvals[0] = builder.CreateBitCast(emit_nthptr_addr(result, (size_t)1), fargt_sig[0]); + argvals[0] = builder.CreateBitCast(result, fargt_sig[0]); } } @@ -1424,7 +1424,7 @@ static Value *emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) Value *newsym = emit_newsym(rt,1,NULL,ctx); assert(newsym != NULL && "Type was not concrete"); if (newsym->getType()->isPointerTy()) { - builder.CreateStore(result,builder.CreateBitCast(emit_nthptr_addr(newsym, (size_t)1), prt->getPointerTo())); + builder.CreateStore(result,builder.CreateBitCast(newsym, prt->getPointerTo())); result = newsym; } else if (lrt != prt) { result = llvm_type_rewrite(result,lrt,rt,true); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index f3544acf3cef9..028d032349e1b 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -769,32 +769,72 @@ static Value *tbaa_decorate(MDNode* md, Instruction* load_or_store) { return load_or_store; } -// --- generating various error checks --- +// --- generating various field accessors --- -static jl_value_t *llvm_type_to_julia(Type *t, bool err=true); +static Value *emit_nthptr_addr(Value *v, ssize_t n) +{ + return builder.CreateGEP(builder.CreateBitCast(v, jl_ppvalue_llvmt), + ConstantInt::get(T_size, (ssize_t)n)); +} + +static Value *emit_nthptr_addr(Value *v, Value *idx) +{ + return builder.CreateGEP(builder.CreateBitCast(v, jl_ppvalue_llvmt), idx); +} + +static Value *emit_nthptr(Value *v, ssize_t n, MDNode *tbaa) +{ + // p = (jl_value_t**)v; p[n] + Value *vptr = emit_nthptr_addr(v, n); + return tbaa_decorate(tbaa,builder.CreateLoad(vptr, false)); +} + +static Value *emit_nthptr(Value *v, Value *idx, MDNode *tbaa) +{ + // p = (jl_value_t**)v; p[n] + Value *vptr = emit_nthptr_addr(v, idx); + return tbaa_decorate(tbaa,builder.CreateLoad(vptr, false)); +} + +static Value *emit_nthptr_recast(Value *v, ssize_t n, MDNode *tbaa, Type* ptype) { + // p = (jl_value_t**)v; *(ptype)&p[n] + Value *vptr = emit_nthptr_addr(v, n); + return tbaa_decorate(tbaa,builder.CreateLoad(builder.CreateBitCast(vptr,ptype), false)); +} + +static Value *emit_typeptr_addr(Value *p) +{ + ssize_t offset = offsetof(jl_typetag_t,type) - offsetof(jl_typetag_t,value); + offset /= (signed)sizeof(jl_value_t*); // important: division must be signed + return emit_nthptr_addr(p, offset); +} static Value *emit_typeof(Value *p) { // given p, a jl_value_t*, compute its type tag if (p->getType() == jl_pvalue_llvmt) { Value *tt = builder.CreateBitCast(p, jl_ppvalue_llvmt); - tt = builder. - CreateLoad(builder.CreateGEP(tt,ConstantInt::get(T_size,0)), - false); - tt = builder. - CreateIntToPtr(builder. CreateAnd(builder.CreatePtrToInt(tt, T_int64), ConstantInt::get(T_int64,~(uptrint_t)3)), jl_pvalue_llvmt); + tt = builder.CreateLoad(emit_typeptr_addr(tt), false); #ifdef OVERLAP_TUPLE_LEN - tt = builder. - CreateIntToPtr(builder. - CreateAnd(builder.CreatePtrToInt(tt, T_int64), - ConstantInt::get(T_int64,0x000ffffffffffffe)), - jl_pvalue_llvmt); + tt = builder.CreateIntToPtr(builder.CreateAnd( + builder.CreatePtrToInt(tt, T_int64), + ConstantInt::get(T_int64,0x000ffffffffffffe)), + jl_pvalue_llvmt); +#else + tt = builder.CreateIntToPtr(builder.CreateAnd( + builder.CreatePtrToInt(tt, T_int64), + ConstantInt::get(T_size,~(uptrint_t)3)), + jl_pvalue_llvmt); #endif return tt; } return literal_pointer_val(julia_type_of(p)); } +// --- generating various error checks --- + +static jl_value_t *llvm_type_to_julia(Type *t, bool err=true); + static void just_emit_error(const std::string &txt, jl_codectx_t *ctx) { Value *zeros[2] = { ConstantInt::get(T_int32, 0), @@ -944,37 +984,6 @@ static Value *emit_bounds_check(Value *a, jl_value_t *ty, Value *i, Value *len, // --- loading and storing --- -static Value *emit_nthptr_addr(Value *v, size_t n) -{ - return builder.CreateGEP(builder.CreateBitCast(v, jl_ppvalue_llvmt), - ConstantInt::get(T_size, n)); -} - -static Value *emit_nthptr_addr(Value *v, Value *idx) -{ - return builder.CreateGEP(builder.CreateBitCast(v, jl_ppvalue_llvmt), idx); -} - -static Value *emit_nthptr(Value *v, size_t n, MDNode *tbaa) -{ - // p = (jl_value_t**)v; p[n] - Value *vptr = emit_nthptr_addr(v, n); - return tbaa_decorate(tbaa,builder.CreateLoad(vptr, false)); -} - -static Value *emit_nthptr(Value *v, Value *idx, MDNode *tbaa) -{ - // p = (jl_value_t**)v; p[n] - Value *vptr = emit_nthptr_addr(v, idx); - return tbaa_decorate(tbaa,builder.CreateLoad(vptr, false)); -} - -static Value *emit_nthptr_recast(Value *v, size_t n, MDNode *tbaa, Type* ptype) { - // p = (jl_value_t**)v; *(ptype)&p[n] - Value *vptr = emit_nthptr_addr(v, n); - return tbaa_decorate(tbaa,builder.CreateLoad(builder.CreateBitCast(vptr,ptype), false)); -} - static Value *ghostValue(jl_value_t *ty); static Value *typed_load(Value *ptr, Value *idx_0based, jl_value_t *jltype, @@ -1122,11 +1131,11 @@ static Value *emit_tuplelen(Value *t,jl_value_t *jt) Type *ty = t->getType(); if (ty == jl_pvalue_llvmt) { //boxed #ifdef OVERLAP_TUPLE_LEN - Value *lenbits = emit_nthptr(t, (size_t)0); + Value *lenbits = builder.CreateLoad(emit_typeptr_addr(t)); return builder.CreateLShr(builder.CreatePtrToInt(lenbits, T_int64), ConstantInt::get(T_int32, 52)); #else - return emit_nthptr_recast(t, 1, tbaa_tuplelen, T_psize); + return emit_nthptr_recast(t, offsetof(jl_tuple_t,length)/sizeof(jl_value_t*), tbaa_tuplelen, T_psize); #endif } else { //unboxed @@ -1134,7 +1143,7 @@ static Value *emit_tuplelen(Value *t,jl_value_t *jt) } } -static Value *emit_tupleset(Value *tuple, Value *ival, Value *x, jl_value_t *jt, jl_codectx_t *ctx) +static Value *emit_tupleset(Value *tuple, Value *ival0, Value *x, jl_value_t *jt, jl_codectx_t *ctx) { if (tuple == NULL) { // A typecheck must have caught this one @@ -1145,17 +1154,17 @@ static Value *emit_tupleset(Value *tuple, Value *ival, Value *x, jl_value_t *jt, if (ty == jl_pvalue_llvmt) { //boxed #ifdef OVERLAP_TUPLE_LEN Value *slot = builder.CreateGEP(builder.CreateBitCast(tuple, jl_ppvalue_llvmt), - ival); + ival0); #else Value *slot = builder.CreateGEP(builder.CreateBitCast(tuple, jl_ppvalue_llvmt), - builder.CreateAdd(ConstantInt::get(T_size,1),ival)); + builder.CreateAdd(ConstantInt::get(T_size,1),ival0)); #endif builder.CreateStore(x,slot); return tuple; } - ConstantInt *idx = dyn_cast(ival); + ConstantInt *idx = dyn_cast(ival0); assert(idx != NULL && "tuplesets must use constant indices"); - unsigned ci = (unsigned)idx->getZExtValue()-1; + unsigned ci = (unsigned)idx->getZExtValue(); if (ty->isVectorTy()) { return mark_julia_type(builder.CreateInsertElement(tuple,x,ConstantInt::get(T_int32,ci)), jt); } @@ -1182,7 +1191,7 @@ static Value *allocate_box_dynamic(Value *jlty, Value *nb, Value *v); static void jl_add_linfo_root(jl_lambda_info_t *li, jl_value_t *val); // Julia semantics -static Value *emit_tupleref(Value *tuple, Value *ival, jl_value_t *jt, jl_codectx_t *ctx) +static Value *emit_tupleref(Value *tuple, Value *ival0, jl_value_t *jt, jl_codectx_t *ctx) { if (tuple == NULL) { // A typecheck must have caught this one @@ -1192,38 +1201,38 @@ static Value *emit_tupleref(Value *tuple, Value *ival, jl_value_t *jt, jl_codect Type *ty = tuple->getType(); if (ty == jl_pvalue_llvmt) { //boxed #ifdef OVERLAP_TUPLE_LEN - Value *slot = builder.CreateGEP(builder.CreateBitCast(tuple, jl_ppvalue_llvmt),ival); + Value *slot = builder.CreateGEP(builder.CreateBitCast(tuple, jl_ppvalue_llvmt),ival0); #else Value *slot = builder.CreateGEP(builder.CreateBitCast(tuple, jl_ppvalue_llvmt), - builder.CreateAdd(ConstantInt::get(T_size,1),ival)); + builder.CreateAdd(ConstantInt::get(T_size,1),ival0)); #endif return builder.CreateLoad(slot); } - ConstantInt *idx = dyn_cast(ival); - unsigned ci = idx ? (unsigned)idx->getZExtValue()-1 : (unsigned)-1; + ConstantInt *idx = dyn_cast(ival0); + unsigned ci = idx ? (unsigned)idx->getZExtValue() : (unsigned)-1; if (ty->isVectorTy()) { - Type *ity = ival->getType(); + Type *ity = ival0->getType(); assert(ity->isIntegerTy()); IntegerType *iity = dyn_cast(ity); // ExtractElement needs i32 *sigh* if (iity->getBitWidth() > 32) - ival = builder.CreateTrunc(ival,T_int32); + ival0 = builder.CreateTrunc(ival0,T_int32); else if (iity->getBitWidth() < 32) - ival = builder.CreateZExt(ival,T_int32); - Value *v = builder.CreateExtractElement(tuple,builder.CreateSub(ival,ConstantInt::get(T_int32,1))); + ival0 = builder.CreateZExt(ival0,T_int32); + Value *v = builder.CreateExtractElement(tuple,ival0); if (idx) { v = mark_julia_type(v,jl_tupleref(jt,ci)); } else { if (sizeof(void*) != 4) - ival = builder.CreateZExt(ival,T_size); + ival0 = builder.CreateZExt(ival0,T_size); if (is_tupletype_homogeneous((jl_tuple_t*)jt)) { v = mark_julia_type(v, jl_t0(jt)); } else { jl_add_linfo_root(ctx->linfo, jt); v = allocate_box_dynamic(emit_tupleref(literal_pointer_val(jt), - ival, jl_typeof(jt), ctx), + ival0, jl_typeof(jt), ctx), ConstantInt::get(T_size,ty->getScalarSizeInBits()/8), v); } } @@ -1260,14 +1269,14 @@ static Value *emit_tupleref(Value *tuple, Value *ival, jl_value_t *jt, jl_codect tbaa_decorate(tbaa_user, builder.CreateStore(tuple,tempSpace)); Value *idxs[2]; idxs[0] = ConstantInt::get(T_size,0); - idxs[1] = builder.CreateSub(ival,ConstantInt::get(T_size,1)); + idxs[1] = ival0; Value *v = builder.CreateGEP(tempSpace,ArrayRef(&idxs[0],2)); if (idx) { v = mark_julia_type(tbaa_decorate(tbaa_user, builder.CreateLoad(v)), jl_tupleref(jt,ci)); } else { jl_add_linfo_root(ctx->linfo, jt); - Value *lty = emit_tupleref(literal_pointer_val(jt), ival, jl_typeof(jt), ctx); + Value *lty = emit_tupleref(literal_pointer_val(jt), ival0, jl_typeof(jt), ctx); size_t i, l = jl_tuple_len(jt); if (is_tupletype_homogeneous((jl_tuple_t*)jt) && jl_isbits(jl_t0(jt))) { v = mark_julia_type(tbaa_decorate(tbaa_user, builder.CreateLoad(v)), jl_t0(jt)); @@ -1299,20 +1308,20 @@ static Value *emit_tupleref(Value *tuple, Value *ival, jl_value_t *jt, jl_codect BasicBlock *after = BasicBlock::Create(getGlobalContext(),"after_switch",ctx->f); BasicBlock *deflt = BasicBlock::Create(getGlobalContext(),"default_case",ctx->f); // Create the switch - SwitchInst *sw = builder.CreateSwitch(ival,deflt,n); + SwitchInst *sw = builder.CreateSwitch(ival0,deflt,n); // Anything else is a bounds error builder.SetInsertPoint(deflt); Value *tmp = builder.CreateAlloca(ty); builder.CreateStore(tuple, tmp); jl_add_linfo_root(ctx->linfo, jt); - builder.CreateCall3(prepare_call(jluboundserror_func), builder.CreatePointerCast(tmp, T_pint8), literal_pointer_val(jt), ival); + builder.CreateCall3(prepare_call(jluboundserror_func), builder.CreatePointerCast(tmp, T_pint8), literal_pointer_val(jt), ival0); builder.CreateUnreachable(); size_t ntuple = jl_tuple_len(jt); PHINode *ret = PHINode::Create(jl_pvalue_llvmt, ntuple); // Now for the cases for (size_t i = 0, j = 0; i < ntuple; ++i) { BasicBlock *blk = BasicBlock::Create(getGlobalContext(),"case",ctx->f); - sw->addCase(ConstantInt::get((IntegerType*)T_size,i+1),blk); + sw->addCase(ConstantInt::get((IntegerType*)T_size,i),blk); builder.SetInsertPoint(blk); jl_value_t *jltype = jl_tupleref(jt,i); Type *ty = julia_struct_to_llvm(jltype); @@ -1351,19 +1360,7 @@ static Value *emit_n_varargs(jl_codectx_t *ctx) static Value *emit_arraysize(Value *t, Value *dim) { -#ifdef STORE_ARRAY_LEN -#ifdef _P64 - int o = 3; -#else - int o = 4; -#endif -#else -#ifdef _P64 - int o = 2; -#else - int o = 3; -#endif -#endif + int o = offsetof(jl_array_t, nrows)/sizeof(void*) - 1; Value *dbits = emit_nthptr(t, builder.CreateAdd(dim, ConstantInt::get(dim->getType(), o)), tbaa_arraysize); @@ -1394,7 +1391,7 @@ static Value *emit_arraylen_prim(Value *t, jl_value_t *ty) { #ifdef STORE_ARRAY_LEN (void)ty; - Value* addr = builder.CreateStructGEP(builder.CreateBitCast(t,jl_parray_llvmt), 2); + Value* addr = builder.CreateStructGEP(builder.CreateBitCast(t,jl_parray_llvmt), 1); //index (not offset) of length field in jl_parray_llvmt return tbaa_decorate(tbaa_arraylen, builder.CreateLoad(addr, false)); #else jl_value_t *p1 = jl_tparam1(ty); @@ -1426,7 +1423,7 @@ static Value *emit_arraylen(Value *t, jl_value_t *ex, jl_codectx_t *ctx) static Value *emit_arrayptr(Value *t) { - Value* addr = builder.CreateStructGEP(builder.CreateBitCast(t,jl_parray_llvmt), 1); + Value* addr = builder.CreateStructGEP(builder.CreateBitCast(t,jl_parray_llvmt), 0); //index (not offset) of data field in jl_parray_llvmt return tbaa_decorate(tbaa_arrayptr, builder.CreateLoad(addr, false)); } @@ -1458,8 +1455,7 @@ static void assign_arrayvar(jl_arrayvar_t &av, Value *ar) static Value *data_pointer(Value *x) { - return builder.CreateGEP(builder.CreateBitCast(x, jl_ppvalue_llvmt), - ConstantInt::get(T_size, 1)); + return builder.CreateBitCast(x, jl_ppvalue_llvmt); } static Value *emit_array_nd_index(Value *a, jl_value_t *ex, size_t nd, jl_value_t **args, @@ -1535,7 +1531,7 @@ static Value *tpropagate(Value *a, Value *b) static Value *init_bits_value(Value *newv, Value *jt, Type *t, Value *v) { - builder.CreateStore(jt, builder.CreateBitCast(newv, jl_ppvalue_llvmt)); + builder.CreateStore(jt, builder.CreateBitCast(emit_typeptr_addr(newv), jl_ppvalue_llvmt)); builder.CreateStore(v , builder.CreateBitCast(data_pointer(newv), PointerType::get(t,0))); return newv; @@ -1548,9 +1544,7 @@ static Value *allocate_box_dynamic(Value *jlty, Value *nb, Value *v) if (v->getType()->isPointerTy()) { v = builder.CreatePtrToInt(v, T_size); } - Value *newv = builder.CreateCall(prepare_call(jlallocobj_func), - builder.CreateAdd(nb, - ConstantInt::get(T_size, sizeof(void*)))); + Value *newv = builder.CreateCall(prepare_call(jlallocobj_func), nb); // TODO: make sure this is rooted. I think it is. return init_bits_value(newv, jlty, v->getType(), v); } @@ -1698,9 +1692,9 @@ static Value *boxed(Value *v, jl_codectx_t *ctx, jl_value_t *jt) make_gcroot(tpl,ctx); for (size_t i = 0; i < n; ++i) { jl_value_t *jti = jl_tupleref(jt,i); - Value *vi = emit_tupleref(v, ConstantInt::get(T_size,i+1), jt, ctx); + Value *vi = emit_tupleref(v, ConstantInt::get(T_size,i), jt, ctx); Value *boxedvi = boxed(vi, ctx, jti); - emit_tupleset(tpl, ConstantInt::get(T_size,i+1), boxedvi, jt, ctx); + emit_tupleset(tpl, ConstantInt::get(T_size,i), boxedvi, jt, ctx); } ctx->argDepth = last_depth; return tpl; @@ -1719,9 +1713,9 @@ static Value *boxed(Value *v, jl_codectx_t *ctx, jl_value_t *jt) if (jb == jl_float64_type) { // manually inline alloc & init of Float64 box. cheap, I know. #ifdef _P64 - Value *newv = builder.CreateCall(prepare_call(jlalloc2w_func)); + Value *newv = builder.CreateCall(prepare_call(jlalloc1w_func)); #else - Value *newv = builder.CreateCall(prepare_call(jlalloc3w_func)); + Value *newv = builder.CreateCall(prepare_call(jlalloc2w_func)); #endif return init_bits_value(newv, literal_pointer_val(jt), t, v); } @@ -1757,7 +1751,7 @@ static void emit_cpointercheck(Value *x, const std::string &msg, emit_typecheck(t, (jl_value_t*)jl_datatype_type, msg, ctx); Value *istype = - builder.CreateICmpEQ(emit_nthptr(t, offsetof(jl_datatype_t,name)/sizeof(char*), tbaa_datatype), + builder.CreateICmpEQ(emit_nthptr(t, (ssize_t)(offsetof(jl_datatype_t,name)/sizeof(char*)), tbaa_datatype), literal_pointer_val((jl_value_t*)jl_pointer_type->name)); BasicBlock *failBB = BasicBlock::Create(getGlobalContext(),"fail",ctx->f); BasicBlock *passBB = BasicBlock::Create(getGlobalContext(),"pass"); @@ -1774,12 +1768,12 @@ static void emit_cpointercheck(Value *x, const std::string &msg, // allocation for known size object static Value* emit_allocobj(size_t static_size) { - if (static_size == sizeof(void*)*2) + if (static_size == sizeof(void*)*1) + return builder.CreateCall(prepare_call(jlalloc1w_func)); + else if (static_size == sizeof(void*)*2) return builder.CreateCall(prepare_call(jlalloc2w_func)); else if (static_size == sizeof(void*)*3) return builder.CreateCall(prepare_call(jlalloc3w_func)); - else if (static_size == sizeof(void*)*4) - return builder.CreateCall(prepare_call(jlalloc4w_func)); else return builder.CreateCall(prepare_call(jlallocobj_func), ConstantInt::get(T_size, static_size)); @@ -1788,10 +1782,8 @@ static Value* emit_allocobj(size_t static_size) // if ptr is NULL this emits a write barrier _back_ static void emit_write_barrier(jl_codectx_t* ctx, Value *parent, Value *ptr) { - /* builder.CreateCall2(wbfunc, builder.CreateBitCast(parent, jl_pvalue_llvmt), builder.CreateBitCast(ptr, jl_pvalue_llvmt)); - return;*/ - parent = builder.CreateBitCast(parent, T_psize); - Value* parent_type = builder.CreateLoad(parent); + Value* parenttag = builder.CreateBitCast(emit_typeptr_addr(parent), T_psize); + Value* parent_type = builder.CreateLoad(parenttag); Value* parent_mark_bits = builder.CreateAnd(parent_type, 1); // the branch hint does not seem to make it to the generated code @@ -1804,7 +1796,7 @@ static void emit_write_barrier(jl_codectx_t* ctx, Value *parent, Value *ptr) builder.CreateCondBr(parent_marked, barrier_may_trigger, cont); builder.SetInsertPoint(barrier_may_trigger); - Value* ptr_mark_bit = builder.CreateAnd(builder.CreateLoad(builder.CreateBitCast(ptr, T_psize)), 1); + Value* ptr_mark_bit = builder.CreateAnd(builder.CreateLoad(builder.CreateBitCast(emit_typeptr_addr(ptr), T_psize)), 1); Value* ptr_not_marked = builder.CreateICmpEQ(ptr_mark_bit, ConstantInt::get(T_size, 0)); builder.CreateCondBr(ptr_not_marked, barrier_trigger, cont); builder.SetInsertPoint(barrier_trigger); @@ -1828,15 +1820,15 @@ static void emit_checked_write_barrier(jl_codectx_t *ctx, Value *parent, Value * builder.SetInsertPoint(cont); } -static void emit_setfield(jl_datatype_t *sty, Value *strct, size_t idx, +static void emit_setfield(jl_datatype_t *sty, Value *strct, size_t idx0, Value *rhs, jl_codectx_t *ctx, bool checked, bool wb) { if (sty->mutabl || !checked) { Value *addr = builder.CreateGEP(builder.CreateBitCast(strct, T_pint8), - ConstantInt::get(T_size, sty->fields[idx].offset + sizeof(void*))); - jl_value_t *jfty = jl_tupleref(sty->types, idx); - if (sty->fields[idx].isptr) { + ConstantInt::get(T_size, sty->fields[idx0].offset)); + jl_value_t *jfty = jl_tupleref(sty->types, idx0); + if (sty->fields[idx0].isptr) { rhs = boxed(rhs, ctx); builder.CreateStore(rhs, builder.CreateBitCast(addr, jl_ppvalue_llvmt)); @@ -1899,9 +1891,9 @@ static Value *emit_newsym(jl_value_t *ty, size_t nargs, jl_value_t **args, jl_co if (might_need_root(args[1]) || fval->getType() != jl_pvalue_llvmt) make_gcroot(f1, ctx); } - Value *strct = emit_allocobj(sizeof(void*)+sty->size); + Value *strct = emit_allocobj(sty->size); builder.CreateStore(literal_pointer_val((jl_value_t*)ty), - emit_nthptr_addr(strct, (size_t)0)); + emit_typeptr_addr(strct)); if (f1) { if (!jl_subtype(expr_type(args[1],ctx), jl_t0(sty->types), 0)) emit_typecheck(f1, jl_t0(sty->types), "new", ctx); diff --git a/src/codegen.cpp b/src/codegen.cpp index 0732452834d3f..2177998c60ad7 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -298,9 +298,9 @@ static Function *jlenter_func; static Function *jlleave_func; static Function *jlegal_func; static Function *jlallocobj_func; +static Function *jlalloc1w_func; static Function *jlalloc2w_func; static Function *jlalloc3w_func; -static Function *jlalloc4w_func; static Function *jl_alloc_tuple_func; static Function *jlsubtype_func; static Function *setjmp_func; @@ -338,7 +338,7 @@ static Function *show_execution_point_func; static std::vector two_pvalue_llvmt; static std::vector three_pvalue_llvmt; -extern "C" DLLEXPORT void gc_wb_slow(void* parent, void* ptr) +extern "C" DLLEXPORT void gc_wb_slow(jl_value_t* parent, jl_value_t* ptr) { gc_wb(parent, ptr); } @@ -1620,9 +1620,9 @@ static Value *emit_lambda_closure(jl_value_t *expr, jl_codectx_t *ctx) if (vari.closureidx != -1) { int idx = vari.closureidx; #ifdef OVERLAP_TUPLE_LEN - val = emit_nthptr((Value*)ctx->envArg, idx+1, tbaa_tuplelen); + val = emit_nthptr((Value*)ctx->envArg, idx, tbaa_tuplelen); #else - val = emit_nthptr((Value*)ctx->envArg, idx+2, tbaa_tuplelen); + val = emit_nthptr((Value*)ctx->envArg, idx+1, tbaa_tuplelen); #endif } else { @@ -1705,8 +1705,7 @@ static Value *emit_getfield(jl_value_t *expr, jl_sym_t *name, jl_codectx_t *ctx) if (strct->getType() == jl_pvalue_llvmt) { Value *addr = builder.CreateGEP(builder.CreateBitCast(strct, T_pint8), - ConstantInt::get(T_size, - sty->fields[idx].offset + sizeof(void*))); + ConstantInt::get(T_size, sty->fields[idx].offset)); JL_GC_POP(); MDNode *tbaa = sty->mutabl ? tbaa_user : tbaa_immut; if (sty->fields[idx].isptr) { @@ -2017,7 +2016,11 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, else if (f->fptr == &jl_f_apply && nargs==3 && ctx->vaStack && symbol_eq(args[3], ctx->vaName) && expr_type(args[2],ctx) == (jl_value_t*)jl_function_type) { Value *theF = emit_expr(args[2],ctx); - Value *theFptr = emit_nthptr_recast(theF,1, tbaa_func, jl_pfptr_llvmt); + Value *theFptr = emit_nthptr_recast( + theF, + offsetof(jl_function_t,fptr)/sizeof(void*), + tbaa_func, + jl_pfptr_llvmt); Value *nva = emit_n_varargs(ctx); #ifdef _P64 nva = builder.CreateTrunc(nva, T_int32); @@ -2054,7 +2057,7 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, if (idx > 0 && (idx < tlen || (idx == tlen && !isseqt))) { // known to be in bounds JL_GC_POP(); - return emit_tupleref(arg1,ConstantInt::get(T_size,idx),tty,ctx); + return emit_tupleref(arg1,ConstantInt::get(T_size,idx-1),tty,ctx); } if (idx==0 || (!isseqt && idx > tlen)) { // known to be out of bounds @@ -2084,9 +2087,9 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, unbox = true; jl_add_linfo_root(ctx->linfo, tty); } - emit_bounds_check(arg1, unbox ? tty : NULL, idx, tlen, ctx); + Value *idx0 = emit_bounds_check(arg1, unbox ? tty : NULL, idx, tlen, ctx); JL_GC_POP(); - return emit_tupleref(arg1,idx,tty,ctx); + return emit_tupleref(arg1,idx0,tty,ctx); } } else if (f->fptr == &jl_f_tuple) { @@ -2124,7 +2127,7 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, } assert(tpl != NULL); Value *elt = emit_unbox(ety,emit_unboxed(args[i+1],ctx),jl_tupleref(rt1,i)); - tpl = emit_tupleset(tpl,ConstantInt::get(T_size,i+1),elt,rt1,ctx); + tpl = emit_tupleset(tpl,ConstantInt::get(T_size,i),elt,rt1,ctx); } JL_GC_POP(); if (ty->isEmptyTy()) @@ -2142,15 +2145,15 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, make_gcroot(arg1, ctx); bool rooted = false; #ifdef OVERLAP_TUPLE_LEN - size_t nwords = nargs+1; + size_t nwords = nargs; #else - size_t nwords = nargs+2; + size_t nwords = nargs+1; #endif Value *tup = emit_allocobj(sizeof(void*)*nwords); #ifdef OVERLAP_TUPLE_LEN - builder.CreateStore(arg1, emit_nthptr_addr(tup, 1)); + builder.CreateStore(arg1, emit_nthptr_addr(tup, 0)); #else - builder.CreateStore(arg1, emit_nthptr_addr(tup, 2)); + builder.CreateStore(arg1, emit_nthptr_addr(tup, 1)); emit_write_barrier(ctx, tup, arg1); #endif ctx->argDepth = last_depth; @@ -2159,22 +2162,14 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, CreateStore(builder. CreateOr(builder.CreatePtrToInt(literal_pointer_val((jl_value_t*)jl_tuple_type), T_int64), ConstantInt::get(T_int64, nargs<<52)), - builder.CreateBitCast(emit_nthptr_addr(tup, (size_t)0), - T_pint64)); + builder.CreateBitCast(emit_typeptr_addr(tup), T_pint64)); #else - builder.CreateStore(literal_pointer_val((jl_value_t*)jl_tuple_type), - emit_nthptr_addr(tup, (size_t)0)); + builder.CreateStore(literal_pointer_val((jl_value_t*)jl_tuple_type), emit_typeptr_addr(tup)); builder.CreateStore(ConstantInt::get(T_size, nargs), - builder.CreateBitCast(emit_nthptr_addr(tup, (size_t)1), T_psize)); -#endif -#ifdef OVERLAP_TUPLE_LEN - size_t offs = 1; -#else - size_t offs = 2; + builder.CreateBitCast(tup, T_psize)); #endif for(i=1; i < nargs; i++) { - builder.CreateStore(V_null, - emit_nthptr_addr(tup, i+offs)); + builder.CreateStore(V_null, emit_nthptr_addr(tup, i+TUPLE_DATA_OFFSET)); } for(i=1; i < nargs; i++) { if (might_need_root(args[i+1]) && !rooted) { @@ -2187,7 +2182,7 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, rooted = true; } Value *argi = boxed(argval,ctx); - builder.CreateStore(argi, emit_nthptr_addr(tup, i+offs)); + builder.CreateStore(argi, emit_nthptr_addr(tup, i+TUPLE_DATA_OFFSET)); emit_write_barrier(ctx, tup, argi); } ctx->argDepth = last_depth; @@ -2356,12 +2351,10 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, if (llvm_st == jl_pvalue_llvmt) { if (is_structtype_all_pointers(stt)) { idx = emit_bounds_check(strct, NULL, idx, ConstantInt::get(T_size, nfields), ctx); - Value *fld = - tbaa_decorate(tbaa_user, builder. - CreateLoad(builder. - CreateGEP(builder. - CreateBitCast(strct, jl_ppvalue_llvmt), - builder.CreateAdd(idx,ConstantInt::get(T_size,1))))); + Value *fld = tbaa_decorate(tbaa_user, builder.CreateLoad( + builder.CreateGEP( + builder.CreateBitCast(strct, jl_ppvalue_llvmt), + idx))); if ((unsigned)stt->ninitialized != jl_tuple_len(stt->types)) { null_pointer_check(fld, ctx); } @@ -2389,7 +2382,7 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, // frobbing the stack Value *fld; if (nfields == 0) { - emit_bounds_check(tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jlnull_var))), + idx = emit_bounds_check(tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jlnull_var))), NULL, idx, ConstantInt::get(T_size, nfields), ctx); fld = UndefValue::get(jl_pvalue_llvmt); } @@ -2405,8 +2398,7 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, jl_add_linfo_root(ctx->linfo, (jl_value_t*)stt); } idx = emit_bounds_check(tempSpace, (jl_value_t*)stt, idx, ConstantInt::get(T_size, nfields), ctx); - Value *ptr = builder.CreateGEP(tempSpace, ConstantInt::get(T_size, 0)); - fld = typed_load(ptr, idx, jt, ctx, stt->mutabl ? tbaa_user : tbaa_immut); + fld = typed_load(tempSpace, idx, jt, ctx, stt->mutabl ? tbaa_user : tbaa_immut); builder.CreateCall(Intrinsic::getDeclaration(jl_Module,Intrinsic::stackrestore), stacksave); } @@ -2632,7 +2624,7 @@ static Value *emit_call(jl_value_t **args, size_t arglen, jl_codectx_t *ctx, jl_ #endif // extract pieces of the function object // TODO: try extractvalue instead - theFptr = emit_nthptr_recast(theFunc, 1, tbaa_func, jl_pfptr_llvmt); + theFptr = emit_nthptr_recast(theFunc, offsetof(jl_function_t,fptr)/sizeof(void*), tbaa_func, jl_pfptr_llvmt); theF = theFunc; } else { @@ -2666,7 +2658,7 @@ static Value *emit_call(jl_value_t **args, size_t arglen, jl_codectx_t *ctx, jl_ myargs = builder.CreateGEP(ctx->argTemp, ConstantInt::get(T_size, argStart+1+ctx->argSpaceOffs)); } - theFptr = emit_nthptr_recast(theFunc, 1, tbaa_func, jl_pfptr_llvmt); + theFptr = emit_nthptr_recast(theFunc, offsetof(jl_function_t,fptr)/sizeof(void*), tbaa_func, jl_pfptr_llvmt); Value *r1 = builder.CreateCall3(prepare_call(theFptr), theFunc, myargs, ConstantInt::get(T_int32,nargs)); builder.CreateBr(mergeBB1); @@ -2756,21 +2748,21 @@ static Value *var_binding_pointer(jl_sym_t *s, jl_binding_t **pbnd, assert(((Value*)ctx->envArg)->getType() == jl_pvalue_llvmt); if (isBoxed(s, ctx)) { #ifdef OVERLAP_TUPLE_LEN - return emit_nthptr_addr(emit_nthptr((Value*)ctx->envArg, idx+1, tbaa_tuplelen), 1); + return builder.CreatePointerCast(emit_nthptr((Value*)ctx->envArg, idx, tbaa_tuplelen), jl_ppvalue_llvmt); #else - return emit_nthptr_addr(emit_nthptr((Value*)ctx->envArg, idx+2, tbaa_tuplelen), 1); + return builder.CreatePointerCast(emit_nthptr((Value*)ctx->envArg, idx+1, tbaa_tuplelen), jl_ppvalue_llvmt); #endif } #ifdef OVERLAP_TUPLE_LEN - return emit_nthptr_addr((Value*)ctx->envArg, idx+1); + return emit_nthptr_addr((Value*)ctx->envArg, idx); #else - return emit_nthptr_addr((Value*)ctx->envArg, idx+2); + return emit_nthptr_addr((Value*)ctx->envArg, idx+1); #endif } Value *l = vi.memvalue; if (l == NULL) return NULL; if (isBoxed(s, ctx)) { - return emit_nthptr_addr(builder.CreateLoad(l,false), 1); + return builder.CreatePointerCast(builder.CreateLoad(l,false), jl_ppvalue_llvmt); } return l; } @@ -2885,9 +2877,8 @@ static Value* emit_assignment(Value *bp, jl_value_t *r, jl_value_t *declType, bo } else { rval = boxed(emit_expr(r, ctx, true),ctx,rt); - if (!is_stack(bp)) { - Value* box = builder.CreateGEP(bp, ConstantInt::get(T_size, -1)); - emit_write_barrier(ctx, box, rval); + if (!is_stack(bp)) { // bp is a jl_box_t* + emit_write_barrier(ctx, bp, rval); } } if (builder.GetInsertBlock()->getTerminator() == NULL) { @@ -3654,8 +3645,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t if (at == jl_pvalue_llvmt) { Value *mem = emit_newsym(jargty, 1, NULL, &ctx); if (mem->getType() == jl_pvalue_llvmt) { - builder.CreateStore(val, builder.CreateBitCast( - emit_nthptr_addr(mem, (size_t)1), val->getType()->getPointerTo())); + builder.CreateStore(val, builder.CreateBitCast(mem, val->getType()->getPointerTo())); val = mem; } else { @@ -4318,7 +4308,7 @@ static Function *emit_function(jl_lambda_info_t *lam) // fetch env out of function object if we need it if (hasCapt) { - ctx.envArg = emit_nthptr(fArg, 2, tbaa_func); + ctx.envArg = emit_nthptr(fArg, offsetof(jl_function_t,env)/sizeof(jl_value_t*), tbaa_func); } // step 8. set up GC frame @@ -4751,14 +4741,10 @@ extern "C" void jl_fptr_to_llvm(void *fptr, jl_lambda_info_t *lam, int specsig) extern "C" DLLEXPORT jl_value_t *jl_new_box(jl_value_t *v) { - jl_value_t *box = (jl_value_t*)alloc_2w(); -#ifdef OVERLAP_TUPLE_LEN - box->type = (size_t)jl_box_any_type; -#else - box->type = jl_box_any_type; -#endif - if(v) gc_wb(box, v); - ((jl_value_t**)box)[1] = v; + jl_value_t *box = (jl_value_t*)alloc_1w(); + jl_set_typeof(box, jl_box_any_type); + // if (v) gc_wb(box, v); // write block not needed: box was just allocated + box->fieldptr[0] = v; return box; } @@ -4882,9 +4868,9 @@ static void init_julia_llvm_env(Module *m) assert(jl_func_sig != NULL); jl_pfptr_llvmt = PointerType::get(PointerType::get(jl_func_sig, 0), 0); - Type* vaelts[] = {valueStructElts[0], T_pint8 + Type* vaelts[] = {T_pint8 #ifdef STORE_ARRAY_LEN - , T_size + , T_size #endif }; Type* jl_array_llvmt = @@ -5090,13 +5076,14 @@ static void init_julia_llvm_env(Module *m) jlcall_func_to_llvm("jl_apply_generic", (void*)&jl_apply_generic, m); jlgetfield_func = jlcall_func_to_llvm("jl_f_get_field", (void*)&jl_f_get_field, m); - std::vector wbargs(0); - wbargs.push_back(jl_pvalue_llvmt); - wbargs.push_back(jl_pvalue_llvmt); queuerootfun = Function::Create(FunctionType::get(T_void, args_1ptr, false), Function::ExternalLinkage, "gc_queue_root", m); add_named_global(queuerootfun, (void*)&gc_queue_root); + + std::vector wbargs(0); + wbargs.push_back(jl_pvalue_llvmt); + wbargs.push_back(jl_pvalue_llvmt); wbfunc = Function::Create(FunctionType::get(T_void, wbargs, false), Function::ExternalLinkage, "gc_wb_slow", m); @@ -5232,6 +5219,12 @@ static void init_julia_llvm_env(Module *m) add_named_global(jlallocobj_func, (void*)&allocobj); std::vector empty_args(0); + jlalloc1w_func = + Function::Create(FunctionType::get(jl_pvalue_llvmt, empty_args, false), + Function::ExternalLinkage, + "alloc_1w", m); + add_named_global(jlalloc1w_func, (void*)&alloc_1w); + jlalloc2w_func = Function::Create(FunctionType::get(jl_pvalue_llvmt, empty_args, false), Function::ExternalLinkage, @@ -5244,12 +5237,6 @@ static void init_julia_llvm_env(Module *m) "alloc_3w", m); add_named_global(jlalloc3w_func, (void*)&alloc_3w); - jlalloc4w_func = - Function::Create(FunctionType::get(jl_pvalue_llvmt, empty_args, false), - Function::ExternalLinkage, - "alloc_4w", m); - add_named_global(jlalloc4w_func, (void*)&alloc_4w); - std::vector atargs(0); atargs.push_back(T_size); jl_alloc_tuple_func = diff --git a/src/dump.c b/src/dump.c index 0bdb3b451ef5a..e8dc961b39c89 100644 --- a/src/dump.c +++ b/src/dump.c @@ -869,7 +869,7 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) int has_instance = (flags>>3)&1; if (has_instance) { dt->instance = jl_deserialize_value(s, &dt->instance); - dt->instance->type = (jl_value_t*)dt; + jl_set_typeof(dt->instance, dt); } assert(tree_literal_values==NULL && mode != MODE_AST); backref_list.items[pos] = dt; @@ -879,7 +879,7 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) arraylist_push(&flagref_list, (void*)(uptrint_t)pos); if (has_instance) { arraylist_push(&flagref_list, dt); - arraylist_push(&flagref_list, &dt->instance->type); + arraylist_push(&flagref_list, &jl_typetagof(dt->instance)->type); arraylist_push(&flagref_list, (void*)(uptrint_t)-1); } } @@ -1039,7 +1039,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t } } if (mode == MODE_MODULE) { - aty = jl_deserialize_value(s, &a->type); + aty = jl_deserialize_value(s, &jl_typetagof(a)->type); assert(aty == jl_typeof(a)); } return (jl_value_t*)a; @@ -1066,7 +1066,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t return (jl_value_t*)e; } else if (vtag == (jl_value_t*)jl_tvar_type) { - jl_tvar_t *tv = (jl_tvar_t*)newobj((jl_value_t*)jl_tvar_type, 4); + jl_tvar_t *tv = (jl_tvar_t*)newobj((jl_value_t*)jl_tvar_type, NWORDS(sizeof(jl_tvar_t))); if (usetable) arraylist_push(&backref_list, tv); tv->name = (jl_sym_t*)jl_deserialize_value(s, NULL); @@ -1080,7 +1080,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t } else if (vtag == (jl_value_t*)jl_function_type) { jl_function_t *f = - (jl_function_t*)newobj((jl_value_t*)jl_function_type, 3); + (jl_function_t*)newobj((jl_value_t*)jl_function_type, NWORDS(sizeof(jl_function_t))); if (usetable) arraylist_push(&backref_list, f); f->linfo = (jl_lambda_info_t*)jl_deserialize_value(s, (jl_value_t**)&f->linfo); @@ -1093,7 +1093,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t else if (vtag == (jl_value_t*)jl_lambda_info_type) { jl_lambda_info_t *li = (jl_lambda_info_t*)newobj((jl_value_t*)jl_lambda_info_type, - LAMBDA_INFO_NW); + NWORDS(sizeof(jl_lambda_info_t))); if (usetable) arraylist_push(&backref_list, li); li->ast = jl_deserialize_value(s, &li->ast); @@ -1241,8 +1241,8 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t case 4: v = jl_box32(dt, *(int32_t*)data); break; case 8: v = jl_box64(dt, *(int64_t*)data); break; default: - v = (jl_value_t*)allocobj(sizeof(void*)+nby); - v->type = (jl_value_t*)dt; + v = (jl_value_t*)allocobj(nby); + jl_set_typeof(v, dt); memcpy(jl_data_ptr(v), data, nby); } } @@ -1276,14 +1276,14 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t } // TODO: put WeakRefs on the weak_refs list if (mode == MODE_MODULE) { - dt = (jl_datatype_t*)jl_deserialize_value(s, (jl_value_t**)&v->type); + dt = (jl_datatype_t*)jl_deserialize_value(s, &jl_typetagof(v)->type); assert((jl_value_t*)dt == jl_typeof(v)); } return v; } else if (vtag == (jl_value_t*)Singleton_tag) { assert(mode != MODE_MODULE_LAMBDAS); - jl_value_t *v = (jl_value_t*)allocobj(sizeof(void*)); + jl_value_t *v = (jl_value_t*)allocobj(0); if (usetable) { uptrint_t pos = backref_list.len; arraylist_push(&backref_list, (void*)v); @@ -1670,14 +1670,14 @@ jl_module_t *jl_restore_new_module(const char *fname) jl_value_t **loc = (jl_value_t**)flagref_list.items[i++]; int offs = (int)(intptr_t)flagref_list.items[i++]; if (t != dt) { - dt->type = (jl_value_t*)(ptrint_t)2; // invalidate the old value to help catch errors + jl_set_typeof(dt, (ptrint_t)2); // invalidate the old value to help catch errors if ((jl_value_t*)dt == o) { if (loc) *loc = (jl_value_t*)t; if (offs > 0) backref_list.items[offs] = t; } } if (t->instance != v) { - v->type = (jl_value_t*)(ptrint_t)1; // invalidate the old value to help catch errors + jl_set_typeof(v, (ptrint_t)1); // invalidate the old value to help catch errors if (v == o) { if (loc) *loc = v; if (offs > 0) backref_list.items[offs] = v; diff --git a/src/gc.c b/src/gc.c index 6b267f20125c7..ea813e55fcac8 100644 --- a/src/gc.c +++ b/src/gc.c @@ -333,11 +333,12 @@ static inline void objprofile_count(void* ty, int old, int sz) #endif } -static inline void gc_setmark_other(void *o, int mark_mode) -{ - _gc_setmark(o, mark_mode); - verify_val(o); -} +//static inline void gc_setmark_other(jl_value_t *v, int mark_mode) // unused function +//{ +// jl_typetag_t *o = jl_typetagof(v); +// _gc_setmark(o, mark_mode); +// verify_val(o); +//} #define inc_sat(v,s) v = (v) >= s ? s : (v)+1 @@ -406,8 +407,10 @@ static inline int gc_setmark_pool(void *o, int mark_mode) } -static inline int gc_setmark(void *o, int sz, int mark_mode) +static inline int gc_setmark(jl_value_t *v, int sz, int mark_mode) { + jl_typetag_t *o = jl_typetagof(v); + sz += sizeof(jl_typetag_t); #ifdef MEMDEBUG return gc_setmark_big(o, mark_mode); #endif @@ -417,7 +420,7 @@ static inline int gc_setmark(void *o, int sz, int mark_mode) return gc_setmark_big(o, mark_mode); } -#define gc_typeof(v) ((jl_value_t*)(((uptrint_t)jl_typeof(v))&(~(uintptr_t)3))) +#define gc_typeof(v) jl_typeof(v) #define gc_val_buf(o) ((buff_t*)(((void**)(o))-1)) inline void gc_setmark_buf(void *o, int mark_mode) @@ -598,7 +601,7 @@ void *jl_gc_managed_realloc(void *d, size_t sz, size_t oldsz, int isaligned, jl_ { maybe_collect(); - if (gc_bits(owner) == GC_MARKED) { + if (gc_bits(jl_typetagof(owner)) == GC_MARKED) { perm_scanned_bytes += sz - oldsz; live_bytes += sz - oldsz; } @@ -656,8 +659,8 @@ static arraylist_t weak_refs; DLLEXPORT jl_weakref_t *jl_gc_new_weakref(jl_value_t *value) { - jl_weakref_t *wr = (jl_weakref_t*)alloc_2w(); - wr->type = (jl_value_t*)jl_weakref_type; + jl_weakref_t *wr = (jl_weakref_t*)alloc_1w(); + jl_set_typeof(wr, jl_weakref_type); wr->value = value; arraylist_push(&weak_refs, wr); return wr; @@ -674,9 +677,10 @@ static void sweep_weak_refs(void) return; do { wr = (jl_weakref_t*)lst[n]; - if (gc_marked(wr)) { + jl_typetag_t *o = jl_typetagof(wr); + if (gc_marked(o)) { // weakref itself is alive - if (!gc_marked(wr->value)) + if (!gc_marked(o->value)) wr->value = (jl_value_t*)jl_nothing; n++; } @@ -742,8 +746,7 @@ static void run_finalizers(void) while (to_finalize.len > 0) { f = arraylist_pop(&to_finalize); o = arraylist_pop(&to_finalize); - int ok = 1;run_finalizer((jl_value_t*)o, (jl_value_t*)f); - assert(ok); (void)ok; + run_finalizer((jl_value_t*)o, (jl_value_t*)f); } JL_GC_POP(); } @@ -927,7 +930,7 @@ static void sweep_malloced_arrays(void) mallocarray_t **pma = &mallocarrays; while (ma != NULL) { mallocarray_t *nxt = ma->next; - if (gc_marked(ma->a)) { + if (gc_marked(jl_typetagof(ma->a))) { pma = &ma->next; } else { @@ -959,7 +962,7 @@ static inline gcval_t *reset_page(pool_t *p, gcpage_t *pg, gcval_t *fl) return beg; } -static __attribute__((noinline)) void add_page(pool_t *p) +static __attribute__((noinline)) void add_page(pool_t *p) { char *data = malloc_page(); if (data == NULL) @@ -972,7 +975,7 @@ static __attribute__((noinline)) void add_page(pool_t *p) p->newpages = fl; } -static inline void *__pool_alloc(pool_t* p, int osize, int end_offset) +static inline void *__pool_alloc(pool_t* p, int osize, int end_offset) { gcval_t *v, *end; if (__unlikely((allocd_bytes += osize) >= 0)) { @@ -1248,7 +1251,7 @@ static gcval_t** sweep_page(pool_t* p, gcpage_t* pg, gcval_t **pfl, int sweep_ma return pfl; } -extern void jl_unmark_symbols(void); +//extern void jl_unmark_symbols(void); static void gc_sweep_once(int sweep_mask) { @@ -1270,10 +1273,10 @@ static void gc_sweep_once(int sweep_mask) JL_PRINTF(JL_STDOUT, "GC sweep big %.2f (freed %d/%d with %d rst)\n", (clock_now() - t0)*1000, big_freed, big_total, big_reset); t0 = clock_now(); #endif - if (sweep_mask == GC_MARKED) - jl_unmark_symbols(); + //if (sweep_mask == GC_MARKED) + // jl_unmark_symbols(); #ifdef GC_TIME - JL_PRINTF(JL_STDOUT, "GC sweep symbols %.2f\n", (clock_now() - t0)*1000); + //JL_PRINTF(JL_STDOUT, "GC sweep symbols %.2f\n", (clock_now() - t0)*1000); #endif } @@ -1329,7 +1332,7 @@ int max_msp = 0; static arraylist_t tasks; static arraylist_t rem_bindings; -static arraylist_t _remset[2]; +static arraylist_t _remset[2]; // contains jl_value_t* static arraylist_t *remset = &_remset[0]; static arraylist_t *last_remset = &_remset[1]; void reset_remset(void) @@ -1340,27 +1343,31 @@ void reset_remset(void) remset->len = 0; } -DLLEXPORT void gc_queue_root(void *ptr) +DLLEXPORT void gc_queue_root(jl_value_t *ptr) { - assert(gc_bits(ptr) != GC_QUEUED); - gc_bits(ptr) = GC_QUEUED; + jl_typetag_t *o = jl_typetagof(ptr); + assert(gc_bits(o) != GC_QUEUED); + gc_bits(o) = GC_QUEUED; arraylist_push(remset, ptr); } -void gc_queue_binding(void *bnd) + +void gc_queue_binding(jl_binding_t *bnd) { - assert(gc_bits(bnd) != GC_QUEUED); - gc_bits(bnd) = GC_QUEUED; - arraylist_push(&rem_bindings, (void*)((void**)bnd + 1)); + buff_t *buf = gc_val_buf(bnd); + assert(gc_bits(buf) != GC_QUEUED); + gc_bits(buf) = GC_QUEUED; + arraylist_push(&rem_bindings, bnd); } static int push_root(jl_value_t *v, int d, int); -static inline int gc_push_root(void *v, int d) +static inline int gc_push_root(void *v, int d) // v isa jl_value_t* { - assert((v) != NULL); - verify_val(v); - int bits = gc_bits(v); - if (!gc_marked(v)) { - return push_root((jl_value_t*)(v),d, bits); + assert(v != NULL); + jl_typetag_t* o = jl_typetagof(v); + verify_val(o); + int bits = gc_bits(o); + if (!gc_marked(o)) { + return push_root((jl_value_t*)v, d, bits); } return bits; } @@ -1368,12 +1375,13 @@ static inline int gc_push_root(void *v, int d) void jl_gc_setmark(jl_value_t *v) // TODO rename this as it is misleading now { // int64_t s = perm_scanned_bytes; - if (!gc_marked(v)) { + jl_typetag_t *o = jl_typetagof(v); + if (!gc_marked(o)) { // objprofile_count(jl_typeof(v), 1, 16); #ifdef MEMDEBUG - gc_setmark_big(v, GC_MARKED_NOESC); + gc_setmark_big(o, GC_MARKED_NOESC); #else - gc_setmark_pool(v, GC_MARKED_NOESC); + gc_setmark_pool(o, GC_MARKED_NOESC); #endif } // perm_scanned_bytes = s; @@ -1412,7 +1420,7 @@ __attribute__((noinline)) static int gc_mark_module(jl_module_t *m, int d) for(i=1; i < m->bindings.size; i+=2) { if (table[i] != HT_NOTFOUND) { jl_binding_t *b = (jl_binding_t*)table[i]; - gc_setmark_buf(b, gc_bits(m)); + gc_setmark_buf(b, gc_bits(jl_typetagof(m))); #ifdef GC_VERIFY void* vb = gc_val_buf(b); verify_parent("module", m, &vb, "binding_buff"); @@ -1444,7 +1452,7 @@ static void gc_mark_task_stack(jl_task_t *ta, int d) { if (ta->stkbuf != NULL || ta == jl_current_task) { if (ta->stkbuf != NULL) { - gc_setmark_buf(ta->stkbuf, gc_bits(ta)); + gc_setmark_buf(ta->stkbuf, gc_bits(jl_typetagof(ta))); } #ifdef COPY_STACKS ptrint_t offset; @@ -1501,7 +1509,7 @@ static int push_root(jl_value_t *v, int d, int bits) int refyoung = 0; if (vt == (jl_value_t*)jl_weakref_type) { - bits = gc_setmark(v, jl_datatype_size(jl_weakref_type), GC_MARKED_NOESC); + bits = gc_setmark(v, sizeof(jl_weakref_t), GC_MARKED_NOESC); goto ret; } if ((jl_is_datatype(vt) && ((jl_datatype_t*)vt)->pointerfree)) { @@ -1534,27 +1542,28 @@ static int push_root(jl_value_t *v, int d, int bits) } else if (((jl_datatype_t*)(vt))->name == jl_array_typename) { jl_array_t *a = (jl_array_t*)v; + jl_typetag_t *o = jl_typetagof(v); int todo = !(bits & GC_MARKED); if (a->pooled) MARK(a, #ifdef MEMDEBUG - bits = gc_setmark_big(a, GC_MARKED_NOESC); + bits = gc_setmark_big(o, GC_MARKED_NOESC); #else - bits = gc_setmark_pool(a, GC_MARKED_NOESC); + bits = gc_setmark_pool(o, GC_MARKED_NOESC); #endif if (a->how == 2 && todo) { - objprofile_count(MATY, gc_bits(a) == GC_MARKED, array_nbytes(a)); - if (gc_bits(a) == GC_MARKED) + objprofile_count(MATY, gc_bits(o) == GC_MARKED, array_nbytes(a)); + if (gc_bits(o) == GC_MARKED) perm_scanned_bytes += array_nbytes(a); else scanned_bytes += array_nbytes(a); }); else MARK(a, - bits = gc_setmark_big(a, GC_MARKED_NOESC); + bits = gc_setmark_big(o, GC_MARKED_NOESC); if (a->how == 2 && todo) { - objprofile_count(MATY, gc_bits(a) == GC_MARKED, array_nbytes(a)); - if (gc_bits(a) == GC_MARKED) + objprofile_count(MATY, gc_bits(o) == GC_MARKED, array_nbytes(a)); + if (gc_bits(o) == GC_MARKED) perm_scanned_bytes += array_nbytes(a); else scanned_bytes += array_nbytes(a); @@ -1569,7 +1578,7 @@ static int push_root(jl_value_t *v, int d, int bits) void* val_buf = gc_val_buf((char*)a->data - a->offset*a->elsize); verify_parent("array", v, &val_buf, "buffer ('loc' addr is meaningless)"); #endif - gc_setmark_buf((char*)a->data - a->offset*a->elsize, gc_bits(v)); + gc_setmark_buf((char*)a->data - a->offset*a->elsize, gc_bits(o)); } if (a->ptrarray && a->data!=NULL) { size_t l = jl_array_len(a); @@ -1604,7 +1613,7 @@ static int push_root(jl_value_t *v, int d, int bits) refyoung = GC_MARKED_NOESC; } else if(vt == (jl_value_t*)jl_symbol_type) { - gc_setmark_other(v, GC_MARKED); // symbols have their own allocator + //gc_setmark_other(v, GC_MARKED); // symbols have their own allocator and are never freed } else if( #ifdef GC_VERIFY @@ -1624,7 +1633,7 @@ static int push_root(jl_value_t *v, int d, int bits) jl_fielddesc_t* fields = dt->fields; for(int i=0; i < nf; i++) { if (fields[i].isptr) { - jl_value_t **slot = (jl_value_t**)((char*)v + fields[i].offset + sizeof(void*)); + jl_value_t **slot = (jl_value_t**)((char*)v + fields[i].offset); jl_value_t *fld = *slot; if (fld) { verify_parent("object", v, slot, "field(%d)", i); @@ -1667,8 +1676,10 @@ static void visit_mark_stack_inc(int mark_mode) { while(mark_sp > 0 && !should_timeout()) { jl_value_t* v = mark_stack[--mark_sp]; - assert(gc_bits(v) == GC_QUEUED || gc_bits(v) == GC_MARKED || gc_bits(v) == GC_MARKED_NOESC); - push_root(v, 0, gc_bits(v)); + assert(gc_bits(jl_typetagof(v)) == GC_QUEUED || + gc_bits(jl_typetagof(v)) == GC_MARKED || + gc_bits(jl_typetagof(v)) == GC_MARKED_NOESC); + push_root(v, 0, gc_bits(jl_typetagof(v))); } } @@ -1749,9 +1760,9 @@ static void post_mark(arraylist_t *list, int dryrun) for(size_t i=0; i < list->len; i+=2) { jl_value_t *v = (jl_value_t*)list->items[i]; jl_value_t *fin = (jl_value_t*)list->items[i+1]; - int isfreed = !gc_marked(v); + int isfreed = !gc_marked(jl_typetagof(v)); gc_push_root(fin, 0); - int isold = list == &finalizer_list && gc_bits(v) == GC_MARKED && gc_bits(fin) == GC_MARKED; + int isold = list == &finalizer_list && gc_bits(jl_typetagof(v)) == GC_MARKED && gc_bits(jl_typetagof(fin)) == GC_MARKED; if (!dryrun && (isfreed || isold)) { // remove from this list if (i < list->len - 2) { @@ -2046,10 +2057,9 @@ void jl_gc_collect(int full) reset_remset(); // avoid counting remembered objects & bindings twice in perm_scanned_bytes for(int i = 0; i < last_remset->len; i++) { - uintptr_t item = (uintptr_t)last_remset->items[i]; - void* ptr = (void*)(item & ~(uintptr_t)1); - objprofile_count(jl_typeof(ptr), 2, 0); - gc_bits(ptr) = GC_MARKED; + jl_value_t *item = (jl_value_t*)last_remset->items[i]; + objprofile_count(jl_typeof(item), 2, 0); + gc_bits(jl_typetagof(item)) = GC_MARKED; } for (int i = 0; i < rem_bindings.len; i++) { void *ptr = rem_bindings.items[i]; @@ -2057,9 +2067,8 @@ void jl_gc_collect(int full) } for (int i = 0; i < last_remset->len; i++) { - uintptr_t item = (uintptr_t)last_remset->items[i]; - void* ptr = (void*)(item & ~(uintptr_t)1); - push_root(ptr, 0, gc_bits(ptr)); + jl_value_t * item = (jl_value_t*)last_remset->items[i]; + push_root(item, 0, GC_MARKED); } // 2. mark every object in a remembered binding @@ -2167,7 +2176,7 @@ void jl_gc_collect(int full) // so that we don't trigger the barrier again on them. if (sweep_mask == GC_MARKED_NOESC) { for (int i = 0; i < remset->len; i++) { - gc_bits(((uintptr_t)remset->items[i] & ~(uintptr_t)1)) = GC_QUEUED; + gc_bits(jl_typetagof(remset->items[i])) = GC_QUEUED; } for (int i = 0; i < rem_bindings.len; i++) { void *ptr = rem_bindings.items[i]; @@ -2270,51 +2279,54 @@ void *reallocb(void *b, size_t sz) } } -DLLEXPORT void *allocobj(size_t sz) +#define jl_valueof(v) (((jl_typetag_t*)(v))->value) + +DLLEXPORT jl_value_t *allocobj(size_t sz) { + sz += sizeof(void*); #ifdef MEMDEBUG - return alloc_big(sz); + return jl_valueof(alloc_big(sz)); #endif if (sz <= 2048) - return pool_alloc(&pools[szclass(sz)]); + return jl_valueof(pool_alloc(&pools[szclass(sz)])); else - return alloc_big(sz); + return jl_valueof(alloc_big(sz)); } -DLLEXPORT void *alloc_2w(void) +DLLEXPORT jl_value_t *alloc_1w(void) { #ifdef MEMDEBUG - return alloc_big(2*sizeof(void*)); + return jl_valueof(alloc_big(2*sizeof(void*))); #endif #ifdef _P64 - return _pool_alloc(&pools[2], 2*sizeof(void*)); + return jl_valueof(_pool_alloc(&pools[2], 2*sizeof(void*))); #else - return _pool_alloc(&pools[0], 2*sizeof(void*)); + return jl_valueof(_pool_alloc(&pools[0], 2*sizeof(void*))); #endif } -DLLEXPORT void *alloc_3w(void) +DLLEXPORT jl_value_t *alloc_2w(void) { #ifdef MEMDEBUG - return alloc_big(3*sizeof(void*)); + return jl_valueof(alloc_big(3*sizeof(void*))); #endif #ifdef _P64 - return _pool_alloc(&pools[4], 3*sizeof(void*)); + return jl_valueof(_pool_alloc(&pools[4], 3*sizeof(void*))); #else - return _pool_alloc(&pools[1], 3*sizeof(void*)); + return jl_valueof(_pool_alloc(&pools[1], 3*sizeof(void*))); #endif } -DLLEXPORT void *alloc_4w(void) +DLLEXPORT jl_value_t *alloc_3w(void) { #ifdef MEMDEBUG - return alloc_big(4*sizeof(void*)); + return jl_valueof(alloc_big(4*sizeof(void*))); #endif #ifdef _P64 - return _pool_alloc(&pools[6], 4*sizeof(void*)); + return jl_valueof(_pool_alloc(&pools[6], 4*sizeof(void*))); #else - return pool_alloc(&pools[2]); + return jl_valueof(pool_alloc(&pools[2])); #endif } diff --git a/src/gf.c b/src/gf.c index baeeadd757b07..d3e620883085d 100644 --- a/src/gf.c +++ b/src/gf.c @@ -22,7 +22,7 @@ extern "C" { static jl_methtable_t *new_method_table(jl_sym_t *name) { jl_methtable_t *mt = (jl_methtable_t*)allocobj(sizeof(jl_methtable_t)); - mt->type = (jl_value_t*)jl_methtable_type; + jl_set_typeof(mt, jl_methtable_type); mt->name = name; mt->defs = (jl_methlist_t*)JL_NULL; mt->cache = (jl_methlist_t*)JL_NULL; @@ -1270,7 +1270,7 @@ jl_methlist_t *jl_method_list_insert(jl_methlist_t **pml, jl_tuple_t *type, l = l->next; } jl_methlist_t *newrec = (jl_methlist_t*)allocobj(sizeof(jl_methlist_t)); - newrec->type = (jl_value_t*)jl_method_type; + jl_set_typeof(newrec, jl_method_type); newrec->sig = type; newrec->tvars = tvars; newrec->va = (jl_tuple_len(type) > 0 && diff --git a/src/interpreter.c b/src/interpreter.c index 8f436a51e2296..9eeb96d1c284e 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -344,7 +344,7 @@ static jl_value_t *eval(jl_value_t *e, jl_value_t **locals, size_t nl, size_t ng temp = b->value; check_can_assign_type(b); b->value = (jl_value_t*)dt; - gc_wb_binding(((void**)b)-1, dt); + gc_wb_binding(b, dt); super = eval(args[2], locals, nl, ngensym); jl_set_datatype_super(dt, super); b->value = temp; diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 3f7fca096a7e1..0052c99b9cc2f 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -264,9 +264,9 @@ static Value *emit_unbox(Type *to, Value *x, jl_value_t *jt) Type *ety = jl_llvmtuple_eltype(to,jt,i); if (ety == T_void) continue; - Value *ref = emit_tupleref(x,ConstantInt::get(T_size,i+1),jt,NULL); + Value *ref = emit_tupleref(x,ConstantInt::get(T_size,i),jt,NULL); Value *elt = emit_unbox(ety,ref,jl_tupleref(jt,i)); - tpl = emit_tupleset(tpl,ConstantInt::get(T_size,i+1),elt,jt,NULL); + tpl = emit_tupleset(tpl,ConstantInt::get(T_size,i),elt,jt,NULL); } return tpl; } @@ -630,11 +630,11 @@ static Value *emit_pointerref(jl_value_t *e, jl_value_t *i, jl_codectx_t *ctx) ConstantInt::get(T_size, sizeof(void*)+size)); builder.CreateStore(literal_pointer_val((jl_value_t*)ety), - emit_nthptr_addr(strct, (size_t)0)); + emit_typeptr_addr(strct)); im1 = builder.CreateMul(im1, ConstantInt::get(T_size, LLT_ALIGN(size, ((jl_datatype_t*)ety)->alignment))); thePtr = builder.CreateGEP(builder.CreateBitCast(thePtr, T_pint8), im1); - builder.CreateMemCpy(builder.CreateBitCast(emit_nthptr_addr(strct, (size_t)1), T_pint8), + builder.CreateMemCpy(builder.CreateBitCast(strct, T_pint8), thePtr, size, 1); return mark_julia_type(strct, ety); } @@ -690,7 +690,7 @@ static Value *emit_pointerset(jl_value_t *e, jl_value_t *x, jl_value_t *i, jl_co im1 = builder.CreateMul(im1, ConstantInt::get(T_size, LLT_ALIGN(size, ((jl_datatype_t*)ety)->alignment))); builder.CreateMemCpy(builder.CreateGEP(builder.CreateBitCast(thePtr, T_pint8), im1), - builder.CreateBitCast(emit_nthptr_addr(val, (size_t)1), T_pint8), size, 1); + builder.CreateBitCast(val, T_pint8), size, 1); } else { if (val == NULL) { diff --git a/src/jlapi.c b/src/jlapi.c index 2d2c3a02a1b29..1d460ff8147d5 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -283,6 +283,17 @@ DLLEXPORT const char* jl_ver_string(void) return JULIA_VERSION_STRING; } +// Create function versions of some useful macros +#undef jl_typetagof +DLLEXPORT jl_typetag_t *jl_typetagof(jl_value_t *v) { + return jl_typetagof__MACRO(v); +} + +#undef jl_typeof +DLLEXPORT jl_value_t *jl_typeof(jl_value_t *v) { + return jl_typeof__MACRO(v); +} + #ifdef __cplusplus } #endif diff --git a/src/jltypes.c b/src/jltypes.c index 7acf6c1a52fec..05ea1fccf6bcd 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2937,13 +2937,13 @@ void jl_init_types(void) { // create base objects jl_datatype_type = jl_new_uninitialized_datatype(10); - jl_datatype_type->type = (jl_value_t*)jl_datatype_type; + jl_set_typeof(jl_datatype_type, jl_datatype_type); jl_typename_type = jl_new_uninitialized_datatype(4); jl_sym_type = jl_new_uninitialized_datatype(0); jl_symbol_type = jl_sym_type; jl_tuple_type = jl_alloc_tuple(1); - jl_tuple_type->type = (jl_value_t*)jl_tuple_type; + jl_set_typeof(jl_tuple_type, jl_tuple_type); #ifdef OVERLAP_TUPLE_LEN jl_tuple_set_len_unsafe(jl_tuple_type, 1); #endif diff --git a/src/julia.h b/src/julia.h index 0fb4e7529a8e0..458a79698b70c 100644 --- a/src/julia.h +++ b/src/julia.h @@ -39,6 +39,9 @@ extern "C" { #define NORETURN #endif +#define container_of(ptr, type, member) \ + ((type *) ((char *)(ptr) - offsetof(type, member))) + #ifdef _MSC_VER #if _WIN64 #define JL_ATTRIBUTE_ALIGN_PTRSIZE(x) __declspec(align(8)) x @@ -53,19 +56,40 @@ extern "C" { // core data types ------------------------------------------------------------ -#ifdef OVERLAP_TUPLE_LEN -#define JL_DATA_TYPE \ - size_t type : 52; \ - size_t _resvd : 12; -#else #define JL_DATA_TYPE \ - struct _jl_value_t *type; -#endif + struct _jl_value_t *fieldptr[0]; typedef struct _jl_value_t { JL_DATA_TYPE + struct _jl_value_t *_silence_Wextern_c_compat; } jl_value_t; +typedef struct { + union { + jl_value_t *type; + uintptr_t type_bits; + struct { + uintptr_t gc_bits:2; +#ifdef OVERLAP_TUPLE_LEN +#ifdef _P64 + uintptr_t unmarked:50; +#else +#error OVERLAP_TUPLE_LEN requires 64-bit pointers +#endif + uintptr_t length:12; +#endif + }; + }; + jl_value_t value[0]; +} jl_typetag_t; + +#define jl_typetagof__MACRO(v) container_of((v),jl_typetag_t,value) +#define jl_typeof__MACRO(v) ((jl_value_t*)(jl_typetagof__MACRO(v)->type_bits&~(size_t)3)) +#define jl_typetagof jl_typetagof__MACRO +#define jl_typeof jl_typeof__MACRO +#define jl_set_typeof(v,t) (jl_typetagof(v)->type = (jl_value_t*)(t)) +#define jl_typeis(v,t) (jl_typeof(v)==(jl_value_t*)(t)) + typedef struct _jl_sym_t { JL_DATA_TYPE struct _jl_sym_t *left; @@ -80,11 +104,8 @@ typedef struct _jl_gensym_t { } jl_gensym_t; typedef struct { -#ifdef OVERLAP_TUPLE_LEN - size_t type : 52; - size_t length : 12; -#else JL_DATA_TYPE +#ifndef OVERLAP_TUPLE_LEN size_t length; #endif jl_value_t *data[]; @@ -172,8 +193,6 @@ typedef struct _jl_lambda_info_t { int32_t specFunctionID; // index that this specFunction will have in the codegen table } jl_lambda_info_t; -#define LAMBDA_INFO_NW (NWORDS(sizeof(jl_lambda_info_t))-1) - typedef struct _jl_function_t { JL_DATA_TYPE jl_fptr_t fptr; @@ -433,37 +452,36 @@ extern jl_sym_t *inert_sym; // GC write barrier -DLLEXPORT void gc_queue_root(void *root); -void gc_queue_binding(void *bnd); +DLLEXPORT void gc_queue_root(jl_value_t *root); // root isa jl_value_t* +void gc_queue_binding(jl_binding_t *bnd); void gc_setmark_buf(void *buf, int); -DLLEXPORT void gc_wb_slow(void* parent, void* ptr); -static inline void gc_wb_binding(void *bnd, void *val) +static inline void gc_wb_binding(jl_binding_t *bnd, void *val) // val isa jl_value_t* { - if (__unlikely((*(uintptr_t*)bnd & 1) == 1 && (*(uintptr_t*)val & 1) == 0)) + if (__unlikely((*((uintptr_t*)bnd-1) & 1) == 1 && (*(uintptr_t*)jl_typetagof(val) & 1) == 0)) gc_queue_binding(bnd); } -static inline void gc_wb(void *parent, void *ptr) +static inline void gc_wb(void *parent, void *ptr) // parent and ptr isa jl_value_t* { - if (__unlikely((*((uintptr_t*)parent) & 1) == 1 && - (*((uintptr_t*)ptr) & 1) == 0)) - gc_queue_root(parent); + if (__unlikely((*((uintptr_t*)jl_typetagof(parent)) & 1) == 1 && + (*((uintptr_t*)jl_typetagof(ptr)) & 1) == 0)) + gc_queue_root((jl_value_t*)parent); } -static inline void gc_wb_buf(void *parent, void *bufptr) +static inline void gc_wb_buf(void *parent, void *bufptr) // parent isa jl_value_t* { // if parent is marked and buf is not - if (__unlikely((*((uintptr_t*)parent) & 1) == 1)) + if (__unlikely((*((uintptr_t*)jl_typetagof(parent)) & 1) == 1)) // (*((uintptr_t*)bufptr) & 3) != 1)) - gc_setmark_buf(bufptr, *(uintptr_t*)parent & 3); + gc_setmark_buf(bufptr, *(uintptr_t*)jl_typetagof(parent) & 3); } -static inline void gc_wb_back(void *ptr) +static inline void gc_wb_back(void *ptr) // ptr isa jl_value_t* { // if ptr is marked - if(__unlikely((*((uintptr_t*)ptr) & 1) == 1)) { - gc_queue_root(ptr); + if(__unlikely((*((uintptr_t*)jl_typetagof(ptr)) & 1) == 1)) { + gc_queue_root((jl_value_t*)ptr); } } @@ -471,16 +489,13 @@ static inline void gc_wb_back(void *ptr) // object accessors ----------------------------------------------------------- #ifdef OVERLAP_TUPLE_LEN -#define jl_typeof(v) ((jl_value_t*)((uptrint_t)((jl_value_t*)(v))->type & 0x000ffffffffffffeULL)) +#define jl_tuple_len(t) (jl_typetagof(t)->length) +#define jl_tuple_set_len_unsafe(t,n) (jl_typetagof(t)->length=(n)) #else -#define jl_typeof(v) ((jl_value_t*)((uptrint_t)((jl_value_t*)(v))->type & ((uintptr_t)~3))) -#endif - -#define jl_typeis(v,t) (jl_typeof(v)==(jl_value_t*)(t)) - #define jl_tuple_len(t) (((jl_tuple_t*)(t))->length) #define jl_tuple_set_len_unsafe(t,n) (((jl_tuple_t*)(t))->length=(n)) -#define jl_tuple_data(t) (((jl_tuple_t*)(t))->data) +#endif +#define jl_tuple_data(t) (((jl_tuple_t*)(t))->data) #define TUPLE_DATA_OFFSET offsetof(jl_tuple_t,data)/sizeof(void*) #ifdef STORE_ARRAY_LEN @@ -521,8 +536,8 @@ STATIC_INLINE jl_value_t *jl_cellset(void *a, size_t i, void *x) return (jl_value_t*)x; } -# define jl_t0(t) jl_tupleref(t,0) -# define jl_t1(t) jl_tupleref(t,1) +#define jl_t0(t) jl_tupleref(t,0) +#define jl_t1(t) jl_tupleref(t,1) #define jl_exprarg(e,n) (((jl_value_t**)jl_array_data(((jl_expr_t*)(e))->args))[n]) #define jl_exprargset(e, n, v) jl_cellset(((jl_expr_t*)(e))->args, n, v) @@ -531,9 +546,9 @@ STATIC_INLINE jl_value_t *jl_cellset(void *a, size_t i, void *x) #define jl_symbolnode_sym(s) ((jl_sym_t*)jl_fieldref(s,0)) #define jl_symbolnode_type(s) (jl_fieldref(s,1)) -#define jl_linenode_line(x) (((ptrint_t*)x)[1]) -#define jl_labelnode_label(x) (((ptrint_t*)x)[1]) -#define jl_gotonode_label(x) (((ptrint_t*)x)[1]) +#define jl_linenode_line(x) (((ptrint_t*)x)[0]) +#define jl_labelnode_label(x) (((ptrint_t*)x)[0]) +#define jl_gotonode_label(x) (((ptrint_t*)x)[0]) #define jl_getfieldnode_val(s) (jl_fieldref(s,0)) #define jl_getfieldnode_name(s) ((jl_sym_t*)jl_fieldref(s,1)) #define jl_getfieldnode_type(s) (jl_fieldref(s,2)) @@ -543,14 +558,14 @@ STATIC_INLINE jl_value_t *jl_cellset(void *a, size_t i, void *x) #define jl_cell_data(a) ((jl_value_t**)((jl_array_t*)a)->data) -#define jl_string_data(s) ((char*)((jl_array_t*)((jl_value_t**)(s))[1])->data) -#define jl_iostr_data(s) ((char*)((jl_array_t*)((jl_value_t**)(s))[1])->data) +#define jl_string_data(s) ((char*)((jl_array_t*)(s)->fieldptr[0])->data) +#define jl_iostr_data(s) ((char*)((jl_array_t*)(s)->fieldptr[0])->data) #define jl_gf_mtable(f) ((jl_methtable_t*)((jl_function_t*)(f))->env) #define jl_gf_name(f) (jl_gf_mtable(f)->name) // get a pointer to the data in a datatype -#define jl_data_ptr(v) (&((void**)(v))[1]) +#define jl_data_ptr(v) ((v)->fieldptr) // struct type info #define jl_field_offset(st,i) (((jl_datatype_t*)st)->fields[i].offset) @@ -806,13 +821,13 @@ DLLEXPORT ssize_t jl_unbox_gensym(jl_value_t *v); #ifdef _P64 #define jl_box_long(x) jl_box_int64(x) -#define jl_box_ulong(x) jl_box_uint64(x) +#define jl_box_ulong(x) jl_box_uint64(x) #define jl_unbox_long(x) jl_unbox_int64(x) #define jl_is_long(x) jl_is_int64(x) #define jl_long_type jl_int64_type #else #define jl_box_long(x) jl_box_int32(x) -#define jl_box_ulong(x) jl_box_uint32(x) +#define jl_box_ulong(x) jl_box_uint32(x) #define jl_unbox_long(x) jl_unbox_int32(x) #define jl_is_long(x) jl_is_int32(x) #define jl_long_type jl_int32_type @@ -1174,12 +1189,12 @@ void jl_gc_free_array(jl_array_t *a); void jl_gc_track_malloced_array(jl_array_t *a); void jl_gc_count_allocd(size_t sz); void jl_gc_run_all_finalizers(void); -DLLEXPORT void *alloc_2w(void); -DLLEXPORT void *alloc_3w(void); -DLLEXPORT void *alloc_4w(void); +DLLEXPORT jl_value_t *alloc_1w(void); +DLLEXPORT jl_value_t *alloc_2w(void); +DLLEXPORT jl_value_t *alloc_3w(void); void *allocb(size_t sz); void *reallocb(void*, size_t); -DLLEXPORT void *allocobj(size_t sz); +DLLEXPORT jl_value_t *allocobj(size_t sz); DLLEXPORT void jl_clear_malloc_data(void); DLLEXPORT int64_t jl_gc_num_pause(void); @@ -1195,11 +1210,11 @@ DLLEXPORT int64_t jl_gc_num_full_sweep(void); #define jl_gc_unpreserve() #define jl_gc_n_preserved_values() (0) -STATIC_INLINE void *alloc_2w() { return allocobj(2*sizeof(void*)); } -STATIC_INLINE void *alloc_3w() { return allocobj(3*sizeof(void*)); } -STATIC_INLINE void *alloc_4w() { return allocobj(4*sizeof(void*)); } +STATIC_INLINE jl_value_t *alloc_1w() { return allocobj(1*sizeof(void*)); } +STATIC_INLINE jl_value_t *alloc_2w() { return allocobj(2*sizeof(void*)); } +STATIC_INLINE jl_value_t *alloc_3w() { return allocobj(3*sizeof(void*)); } #define allocb(nb) malloc(nb) -#define allocobj(nb) malloc(nb) +#define allocobj(nb) (((jl_typetag_t*)malloc((nb)+sizeof(jl_typetag_t)))->value) #endif // async signal handling ------------------------------------------------------ diff --git a/src/julia_internal.h b/src/julia_internal.h index 0c3e2618dc7fb..d7a5c7e25d16e 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -13,22 +13,22 @@ STATIC_INLINE jl_value_t *newobj(jl_value_t *type, size_t nfields) jl_value_t *jv = NULL; switch (nfields) { case 1: - jv = (jl_value_t*)alloc_2w(); break; + jv = (jl_value_t*)alloc_1w(); break; case 2: - jv = (jl_value_t*)alloc_3w(); break; + jv = (jl_value_t*)alloc_2w(); break; case 3: - jv = (jl_value_t*)alloc_4w(); break; + jv = (jl_value_t*)alloc_3w(); break; default: - jv = (jl_value_t*)allocobj((1+nfields) * sizeof(void*)); + jv = (jl_value_t*)allocobj(nfields * sizeof(void*)); } - jv->type = type; + jl_set_typeof(jv, type); return jv; } STATIC_INLINE jl_value_t *newstruct(jl_datatype_t *type) { - jl_value_t *jv = (jl_value_t*)allocobj(sizeof(void*) + type->size); - jv->type = (jl_value_t*)type; + jl_value_t *jv = (jl_value_t*)allocobj(type->size); + jl_set_typeof(jv, type); return jv; } diff --git a/src/module.c b/src/module.c index 8d7070f074ba7..68b5b2ad2350a 100644 --- a/src/module.c +++ b/src/module.c @@ -17,7 +17,7 @@ jl_module_t *jl_current_module=NULL; jl_module_t *jl_new_module(jl_sym_t *name) { jl_module_t *m = (jl_module_t*)allocobj(sizeof(jl_module_t)); - m->type = (jl_value_t*)jl_module_type; + jl_set_typeof(m, jl_module_type); JL_GC_PUSH1(&m); assert(jl_is_symbol(name)); m->name = name; @@ -368,7 +368,7 @@ DLLEXPORT void jl_checked_assignment(jl_binding_t *b, jl_value_t *rhs) } } b->value = rhs; - gc_wb_binding(((void**)b)-1, rhs); + gc_wb_binding(b, rhs); } DLLEXPORT void jl_declare_constant(jl_binding_t *b) diff --git a/src/task.c b/src/task.c index 962d8a14fae52..b60069ee81d91 100644 --- a/src/task.c +++ b/src/task.c @@ -809,7 +809,7 @@ DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize) { size_t pagesz = jl_page_size; jl_task_t *t = (jl_task_t*)allocobj(sizeof(jl_task_t)); - t->type = (jl_value_t*)jl_task_type; + jl_set_typeof(t, jl_task_type); ssize = LLT_ALIGN(ssize, pagesz); t->ssize = ssize; t->current_module = NULL; @@ -915,7 +915,7 @@ void jl_init_tasks(void *stack, size_t ssize) runnable_sym = jl_symbol("runnable"); jl_current_task = (jl_task_t*)allocobj(sizeof(jl_task_t)); - jl_current_task->type = (jl_value_t*)jl_task_type; + jl_set_typeof(jl_current_task, jl_task_type); #ifdef COPY_STACKS jl_current_task->ssize = 0; // size of saved piece jl_current_task->bufsz = 0; diff --git a/ui/repl.c b/ui/repl.c index d93e7c1cd00f3..6d9b2821f42d5 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -321,7 +321,7 @@ static int true_main(int argc, char *argv[]) int i; for (i=0; i < argc; i++) { jl_value_t *s = (jl_value_t*)jl_cstr_to_string(argv[i]); - s->type = (jl_value_t*)jl_utf8_string_type; + jl_set_typeof(s,jl_utf8_string_type); jl_arrayset(args, s, i); } }