Skip to content

Record hidden ptr in layout #92

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 5, 2025
Merged

Record hidden ptr in layout #92

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
075a6a4
Add fields about hidden pointers in jl_datatype_layout_t
qinsoon Jul 18, 2025
baf66d0
Record hidden pointer in layout
qinsoon Jul 28, 2025
3d7144d
Move the new field to the end of the struct
qinsoon Aug 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions base/runtime_internals.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -521,7 +521,9 @@ struct DataTypeLayout
size::UInt32
nfields::UInt32
npointers::UInt32
nhidden_pointers::UInt32
firstptr::Int32
firsthiddenptr::Int32
alignment::UInt16
flags::UInt16
# haspadding : 1;
Expand Down
137 changes: 124 additions & 13 deletions src/datatype.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *modu
tn->partial = NULL;
tn->atomicfields = NULL;
tn->constfields = NULL;
tn->hiddenptrfields = NULL;
tn->max_methods = 0;
tn->constprop_heustic = 0;
return tn;
Expand Down Expand Up @@ -126,6 +127,7 @@ static uint32_t _hash_djb2(uint32_t hash, const char *mem, size_t s) JL_NOTSAFEP
return hash;
}


static uint32_t _hash_layout_djb2(uintptr_t _layout, void *unused) JL_NOTSAFEPOINT
{
(void)unused;
Expand All @@ -138,11 +140,20 @@ static uint32_t _hash_layout_djb2(uintptr_t _layout, void *unused) JL_NOTSAFEPOI
const char *pointers = jl_dt_layout_ptrs(layout);
assert(pointers);
size_t pointers_size = layout->first_ptr < 0 ? 0 : (layout->npointers << layout->flags.fielddesc_type);
const char *hidden_pointers = NULL;
size_t hidden_ptrs_size = 0;
if (layout->first_hidden_ptr >= 0) {
hidden_pointers = jl_dt_layout_hidden_ptrs(layout);
hidden_ptrs_size = layout->nhidden_pointers << layout->flags.fielddesc_type;
}

uint_t hash = 5381;
hash = _hash_djb2(hash, (char *)layout, own_size);
hash = _hash_djb2(hash, fields, fields_size);
hash = _hash_djb2(hash, pointers, pointers_size);
if (hidden_ptrs_size > 0) {
hash = _hash_djb2(hash, hidden_pointers, hidden_ptrs_size);
}
return hash;
}

Expand All @@ -163,6 +174,13 @@ static int layout_eq(void *_l1, void *_l2, void *unused) JL_NOTSAFEPOINT
size_t pointers_size = l1->first_ptr < 0 ? 0 : (l1->npointers << l1->flags.fielddesc_type);
if (memcmp(p1, p2, pointers_size))
return 0;
if (l1->first_hidden_ptr >= 0 && l2->first_hidden_ptr >= 0) {
const char *h1 = jl_dt_layout_hidden_ptrs(l1);
const char *h2 = jl_dt_layout_hidden_ptrs(l2);
size_t hidden_ptrs_size = l1->nhidden_pointers << l1->flags.fielddesc_type;
if (memcmp(h1, h2, hidden_ptrs_size))
return 0;
}
return 1;
}

Expand All @@ -174,15 +192,18 @@ HTIMPL_R(layoutcache, _hash_layout_djb2, layout_eq)
static htable_t layoutcache;
static int layoutcache_initialized = 0;


static jl_datatype_layout_t *jl_get_layout(uint32_t sz,
uint32_t nfields,
uint32_t npointers,
uint32_t nhidden_pointers,
uint32_t alignment,
int haspadding,
int isbitsegal,
int arrayelem,
jl_fielddesc32_t desc[],
uint32_t pointers[]) JL_NOTSAFEPOINT
uint32_t pointers[],
uint32_t hidden_pointers[]) JL_NOTSAFEPOINT
{
assert(alignment); // should have been verified by caller

Expand Down Expand Up @@ -212,11 +233,13 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz,
}
}
int32_t first_ptr = (npointers > 0 ? (int32_t)pointers[0] : -1);
int32_t first_hidden_ptr = (nhidden_pointers > 0 ? (int32_t)hidden_pointers[0] : -1);

// allocate a new descriptor, on the stack if possible.
size_t fields_size = nfields * jl_fielddesc_size(fielddesc_type);
size_t pointers_size = first_ptr < 0 ? 0 : (npointers << fielddesc_type);
size_t flddesc_sz = sizeof(jl_datatype_layout_t) + fields_size + pointers_size;
size_t hidden_ptrs_size = first_hidden_ptr < 0 ? 0 : (nhidden_pointers << fielddesc_type);
size_t flddesc_sz = sizeof(jl_datatype_layout_t) + fields_size + pointers_size + hidden_ptrs_size;
int should_malloc = flddesc_sz >= jl_page_size;
jl_datatype_layout_t *mallocmem = (jl_datatype_layout_t *)(should_malloc ? malloc(flddesc_sz) : NULL);
jl_datatype_layout_t *allocamem = (jl_datatype_layout_t *)(should_malloc ? NULL : alloca(flddesc_sz));
Expand All @@ -233,6 +256,8 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz,
flddesc->flags.padding = 0;
flddesc->npointers = npointers;
flddesc->first_ptr = first_ptr;
flddesc->nhidden_pointers = nhidden_pointers;
flddesc->first_hidden_ptr = first_hidden_ptr;

// fill out the fields of the new descriptor
jl_fielddesc8_t *desc8 = (jl_fielddesc8_t *)jl_dt_layout_fields(flddesc);
Expand Down Expand Up @@ -272,6 +297,25 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz,
}
}

// fill out hidden pointer descriptors
if (first_hidden_ptr >= 0 && hidden_pointers) {
uint8_t *hptrs8 = (uint8_t *)jl_dt_layout_hidden_ptrs(flddesc);
uint16_t *hptrs16 = (uint16_t *)jl_dt_layout_hidden_ptrs(flddesc);
uint32_t *hptrs32 = (uint32_t *)jl_dt_layout_hidden_ptrs(flddesc);
uint32_t *src_descs = (uint32_t *)hidden_pointers;
for (size_t i = 0; i < nhidden_pointers; i++) {
if (fielddesc_type == 0) {
hptrs8[i] = src_descs[i];
}
else if (fielddesc_type == 1) {
hptrs16[i] = src_descs[i];
}
else {
hptrs32[i] = src_descs[i];
}
}
}

if (__unlikely(!layoutcache_initialized)) {
htable_new(&layoutcache, 4096);
layoutcache_initialized = 1;
Expand Down Expand Up @@ -299,6 +343,7 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz,
return ret;
}


// Determine if homogeneous tuple with fields of type t will have
// a special alignment and vector-ABI beyond normal rules for aggregates.
// Return special alignment if one exists, 0 if normal alignment rules hold.
Expand Down Expand Up @@ -502,7 +547,7 @@ void jl_get_genericmemory_layout(jl_datatype_t *st)
jl_value_t *addrspace = jl_tparam2(st);
if (!jl_is_typevar(eltype) && !jl_is_type(eltype)) {
// this is expected to have a layout, but since it is not constructable, we don't care too much what it is
static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, -1, sizeof(void*), {0}};
static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, 0, -1, -1, sizeof(void*), {0}};
st->layout = &opaque_ptr_layout;
st->has_concrete_subtype = 0;
return;
Expand Down Expand Up @@ -571,7 +616,7 @@ void jl_get_genericmemory_layout(jl_datatype_t *st)
else
arrayelem = 0;
assert(!st->layout);
st->layout = jl_get_layout(elsz, nfields, npointers, al, haspadding, isbitsegal, arrayelem, NULL, pointers);
st->layout = jl_get_layout(elsz, nfields, npointers, 0, al, haspadding, isbitsegal, arrayelem, NULL, pointers, NULL);
st->zeroinit = zi;
//st->has_concrete_subtype = 1;
//st->isbitstype = 0;
Expand Down Expand Up @@ -630,17 +675,17 @@ void jl_compute_field_offsets(jl_datatype_t *st)
// if we have no fields, we can trivially skip the rest
if (st == jl_symbol_type || st == jl_string_type) {
// opaque layout - heap-allocated blob
static const jl_datatype_layout_t opaque_byte_layout = {0, 0, 1, -1, 1, { .haspadding = 0, .fielddesc_type=0, .isbitsegal=1, .arrayelem_isboxed=0, .arrayelem_isunion=0 }};
static const jl_datatype_layout_t opaque_byte_layout = {0, 0, 1, 0, -1, -1, 1, { .haspadding = 0, .fielddesc_type=0, .isbitsegal=1, .arrayelem_isboxed=0, .arrayelem_isunion=0 }};
st->layout = &opaque_byte_layout;
return;
}
else if (st == jl_simplevector_type || st == jl_module_type) {
static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, -1, sizeof(void*), { .haspadding = 0, .fielddesc_type=0, .isbitsegal=1, .arrayelem_isboxed=0, .arrayelem_isunion=0 }};
static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, 0, -1, -1, sizeof(void*), { .haspadding = 0, .fielddesc_type=0, .isbitsegal=1, .arrayelem_isboxed=0, .arrayelem_isunion=0 }};
st->layout = &opaque_ptr_layout;
return;
}
else {
static const jl_datatype_layout_t singleton_layout = {0, 0, 0, -1, 1, { .haspadding = 0, .fielddesc_type=0, .isbitsegal=1, .arrayelem_isboxed=0, .arrayelem_isunion=0 }};
static const jl_datatype_layout_t singleton_layout = {0, 0, 0, 0, -1, -1, 1, { .haspadding = 0, .fielddesc_type=0, .isbitsegal=1, .arrayelem_isboxed=0, .arrayelem_isunion=0 }};
st->layout = &singleton_layout;
}
}
Expand Down Expand Up @@ -672,6 +717,7 @@ void jl_compute_field_offsets(jl_datatype_t *st)
size_t descsz = nfields * sizeof(jl_fielddesc32_t);
jl_fielddesc32_t *desc;
uint32_t *pointers;
uint32_t *hidden_pointers;
int should_malloc = descsz >= jl_page_size;
if (should_malloc)
desc = (jl_fielddesc32_t*)malloc_s(descsz);
Expand All @@ -685,12 +731,22 @@ void jl_compute_field_offsets(jl_datatype_t *st)
int homogeneous = 1;
int needlock = 0;
uint32_t npointers = 0;
uint32_t nhidden_pointers = 0;
jl_value_t *firstty = jl_field_type(st, 0);
for (i = 0; i < nfields; i++) {
jl_value_t *fld = jl_field_type(st, i);
int isatomic = jl_field_isatomic(st, i);
int ishiddenptr = jl_field_ishiddenptr(st, i);
size_t fsz = 0, al = 1;
if (jl_islayout_inline(fld, &fsz, &al) && (!isatomic || jl_is_datatype(fld))) { // aka jl_datatype_isinlinealloc
if (ishiddenptr) {
fsz = sizeof(void*);
al = fsz;
if (al > MAX_ALIGN)
al = MAX_ALIGN;
desc[i].isptr = 0; // Hidden pointers are stored as non-pointer fields
nhidden_pointers++;
}
else if (jl_islayout_inline(fld, &fsz, &al) && (!isatomic || jl_is_datatype(fld))) { // aka jl_datatype_isinlinealloc
if (__unlikely(fsz > max_size))
// Should never happen
throw_ovf(should_malloc, desc, st, fsz);
Expand Down Expand Up @@ -724,6 +780,7 @@ void jl_compute_field_offsets(jl_datatype_t *st)
if (!zeroinit)
zeroinit = ((jl_datatype_t*)fld)->zeroinit;
npointers += fld_npointers;
nhidden_pointers += ((jl_datatype_t*)fld)->layout->nhidden_pointers;
}
}
else {
Expand Down Expand Up @@ -782,25 +839,61 @@ void jl_compute_field_offsets(jl_datatype_t *st)
pointers = (uint32_t*)malloc_s(npointers * sizeof(uint32_t));
else
pointers = (uint32_t*)alloca(npointers * sizeof(uint32_t));
if (should_malloc && nhidden_pointers) {
hidden_pointers = (uint32_t*)malloc_s(nhidden_pointers * sizeof(uint32_t));
} else {
hidden_pointers = (uint32_t*)alloca(nhidden_pointers * sizeof(uint32_t));
}
size_t ptr_i = 0;
size_t hptr_i = 0;
for (i = 0; i < nfields; i++) {
jl_value_t *fld = jl_field_type(st, i);
uint32_t offset = desc[i].offset / sizeof(jl_value_t**);
if (desc[i].isptr)
int ishiddenptr = jl_field_ishiddenptr(st, i);
if (ishiddenptr) {
// Direct hidden pointer field
hidden_pointers[hptr_i] = offset;
hptr_i++;
}
else if (desc[i].isptr)
pointers[ptr_i++] = offset;
else if (jl_is_datatype(fld)) {
// Handle nested datatype with regular/hidden pointers
int j, npointers = ((jl_datatype_t*)fld)->layout->npointers;
for (j = 0; j < npointers; j++) {
pointers[ptr_i++] = offset + jl_ptr_offset((jl_datatype_t*)fld, j);
}
// Copy hidden pointers from nested field
int nhidden = ((jl_datatype_t*)fld)->layout->nhidden_pointers;
for (j = 0; j < nhidden; j++) {
hidden_pointers[hptr_i] = offset + jl_hidden_ptr_offset((jl_datatype_t*)fld, j);
hptr_i++;
}
}
}
assert(ptr_i == npointers);
st->layout = jl_get_layout(sz, nfields, npointers, alignm, haspadding, isbitsegal, 0, desc, pointers);
assert(hptr_i == nhidden_pointers);
st->layout = jl_get_layout(sz, nfields, npointers, nhidden_pointers, alignm, haspadding, isbitsegal, 0, desc, pointers, hidden_pointers);

// Validation: Ensure no overlap between pointer and hidden pointer offsets
if (npointers > 0 && nhidden_pointers > 0) {
for (size_t p = 0; p < npointers; p++) {
for (size_t hp = 0; hp < nhidden_pointers; hp++) {
if (pointers[p] == hidden_pointers[hp]) {
jl_errorf("Field offset conflict: field at offset %u appears in both regular pointer offsets and hidden pointer offsets",
pointers[p]);
}
}
}
}

// All hidden pointers are assumed to be PtrOrOffset type
if (should_malloc) {
free(desc);
if (npointers)
free(pointers);
if (nhidden_pointers)
free(hidden_pointers);
}
st->zeroinit = zeroinit;
}
Expand All @@ -813,7 +906,7 @@ void jl_compute_field_offsets(jl_datatype_t *st)
return;
}

JL_DLLEXPORT jl_datatype_t *jl_new_datatype(
JL_DLLEXPORT jl_datatype_t *jl_new_datatype_with_hiddenptrs(
jl_sym_t *name,
jl_module_t *module,
jl_datatype_t *super,
Expand All @@ -822,7 +915,8 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype(
jl_svec_t *ftypes,
jl_svec_t *fattrs,
int abstract, int mutabl,
int ninitialized)
int ninitialized,
uint32_t* hiddenptrfields)
{
jl_datatype_t *t = NULL;
jl_typename_t *tn = NULL;
Expand Down Expand Up @@ -912,6 +1006,7 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype(
}
tn->atomicfields = atomicfields;
tn->constfields = constfields;
tn->hiddenptrfields = hiddenptrfields;

if (t->name->wrapper == NULL) {
t->name->wrapper = (jl_value_t*)t;
Expand All @@ -933,6 +1028,22 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype(
return t;
}

JL_DLLEXPORT jl_datatype_t *jl_new_datatype(
jl_sym_t *name,
jl_module_t *module,
jl_datatype_t *super,
jl_svec_t *parameters,
jl_svec_t *fnames,
jl_svec_t *ftypes,
jl_svec_t *fattrs,
int abstract, int mutabl,
int ninitialized)
{
return jl_new_datatype_with_hiddenptrs(
name, module, super, parameters, fnames, ftypes, fattrs,
abstract, mutabl, ninitialized, NULL);
}

JL_DLLEXPORT jl_datatype_t *jl_new_primitivetype(jl_value_t *name, jl_module_t *module,
jl_datatype_t *super,
jl_svec_t *parameters, size_t nbits)
Expand Down Expand Up @@ -962,7 +1073,7 @@ JL_DLLEXPORT jl_datatype_t *jl_new_primitivetype(jl_value_t *name, jl_module_t *
bt->ismutationfree = 1;
bt->isidentityfree = 1;
bt->isbitstype = (parameters == jl_emptysvec);
bt->layout = jl_get_layout(nbytes, 0, 0, alignm, 0, 1, 0, NULL, NULL);
bt->layout = jl_get_layout(nbytes, 0, 0, 0, alignm, 0, 1, 0, NULL, NULL, NULL);
bt->instance = NULL;
return bt;
}
Expand Down
18 changes: 11 additions & 7 deletions src/jltypes.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3007,27 +3007,29 @@ void jl_init_types(void) JL_GC_DISABLED
jl_typename_type->name->mt = jl_nonfunction_mt;
jl_typename_type->super = jl_any_type;
jl_typename_type->parameters = jl_emptysvec;
jl_typename_type->name->n_uninitialized = 16 - 2;
jl_typename_type->name->names = jl_perm_symsvec(16, "name", "module",
jl_typename_type->name->n_uninitialized = 17 - 2;
jl_typename_type->name->names = jl_perm_symsvec(17, "name", "module",
"names", "atomicfields", "constfields",
"wrapper", "Typeofwrapper", "cache", "linearcache",
"mt", "partial",
"hash", "n_uninitialized",
"flags", // "abstract", "mutable", "mayinlinealloc",
"max_methods", "constprop_heuristic");
"max_methods", "constprop_heuristic",
"hiddenptrfields");
const static uint32_t typename_constfields[1] = { 0x00003a27 }; // (1<<0)|(1<<1)|(1<<2)|(1<<5)|(1<<9)|(1<<11)|(1<<12)|(1<<13) ; TODO: put back (1<<3)|(1<<4) in this list
const static uint32_t typename_atomicfields[1] = { 0x00000180 }; // (1<<7)|(1<<8)
jl_typename_type->name->constfields = typename_constfields;
jl_typename_type->name->atomicfields = typename_atomicfields;
jl_precompute_memoized_dt(jl_typename_type, 1);
jl_typename_type->types = jl_svec(16, jl_symbol_type, jl_any_type /*jl_module_type*/,
jl_typename_type->types = jl_svec(17, jl_symbol_type, jl_any_type /*jl_module_type*/,
jl_simplevector_type, jl_any_type/*jl_voidpointer_type*/, jl_any_type/*jl_voidpointer_type*/,
jl_type_type, jl_type_type, jl_simplevector_type, jl_simplevector_type,
jl_methtable_type, jl_any_type,
jl_any_type /*jl_long_type*/, jl_any_type /*jl_int32_type*/,
jl_any_type /*jl_uint8_type*/,
jl_any_type /*jl_uint8_type*/,
jl_any_type /*jl_uint8_type*/);
jl_any_type /*jl_uint8_type*/,
jl_any_type/*jl_voidpointer_type*/);

jl_methtable_type->name = jl_new_typename_in(jl_symbol("MethodTable"), core, 0, 1);
jl_methtable_type->name->wrapper = (jl_value_t*)jl_methtable_type;
Expand Down Expand Up @@ -3338,11 +3340,12 @@ void jl_init_types(void) JL_GC_DISABLED
memory_datatype->ismutationfree = 0;

jl_datatype_t *jl_memoryref_supertype = (jl_datatype_t*)jl_apply_type1((jl_value_t*)jl_ref_type, jl_svecref(tv, 1));
const static uint32_t memoryref_hiddenptrfields[1] = { 0x00000001 }; // (1<<0) - field 0 is a hidden pointer
jl_datatype_t *memoryref_datatype =
jl_new_datatype(jl_symbol("GenericMemoryRef"), core, jl_memoryref_supertype, tv,
jl_new_datatype_with_hiddenptrs(jl_symbol("GenericMemoryRef"), core, jl_memoryref_supertype, tv,
jl_perm_symsvec(2, "ptr_or_offset", "mem"),
jl_svec(2, pointer_void, memory_datatype),
jl_emptysvec, 0, 0, 2);
jl_emptysvec, 0, 0, 2, memoryref_hiddenptrfields);
jl_genericmemoryref_typename = memoryref_datatype->name;
jl_genericmemoryref_type = (jl_unionall_t*)jl_genericmemoryref_typename->wrapper;
memoryref_datatype->ismutationfree = 0;
Expand Down Expand Up @@ -3856,6 +3859,7 @@ void jl_init_types(void) JL_GC_DISABLED
jl_svecset(jl_typename_type->types, 13, jl_uint8_type);
jl_svecset(jl_typename_type->types, 14, jl_uint8_type);
jl_svecset(jl_typename_type->types, 15, jl_uint8_type);
jl_svecset(jl_typename_type->types, 16, jl_voidpointer_type); // hiddenptrfields
jl_svecset(jl_methtable_type->types, 4, jl_long_type);
jl_svecset(jl_methtable_type->types, 5, jl_module_type);
jl_svecset(jl_methtable_type->types, 6, jl_array_any_type);
Expand Down
Loading