diff --git a/src/data/reconstructing_datatypes.jl b/src/data/reconstructing_datatypes.jl index 92837142..508eb7b4 100644 --- a/src/data/reconstructing_datatypes.jl +++ b/src/data/reconstructing_datatypes.jl @@ -305,18 +305,18 @@ function jlconvert(rr::ReadRepresentation{T,DataTypeODR()}, ptr::Ptr, header_offset::RelOffset) where T mypath = String(jlconvert(ReadRepresentation{UInt8,Vlen{UInt8}}(), f, ptr, NULL_REFERENCE)) - + ptr += odr_sizeof(Vlen{UInt8}) track_weakref!(f, header_offset, SelfReferentialPlaceholder()) - params = types_from_refs(f, ptr+odr_sizeof(Vlen{UInt8})) + params = types_from_refs(f, ptr) + ptr += odr_sizeof(Vlen{RelOffset}) # For cross-platform compatibility convert integer type parameters to system precision params = [p isa Union{Int64,Int32} ? Int(p) : p for p in params] - hasparams = !isempty(params) m = _resolve_type(mypath, params) reconstruct = m isa UnknownType - if !reconstruct && hasparams + if !reconstruct && !isempty(params) try m = m{params...} catch e @@ -329,14 +329,24 @@ function jlconvert(rr::ReadRepresentation{T,DataTypeODR()}, end if reconstruct - dataptr= ptr+odr_sizeof(Vlen{UInt8})+odr_sizeof(Vlen{RelOffset}) - if jlconvert_isinitialized(ReadRepresentation{String,Vlen{String}}(), dataptr) - fieldnames = jlconvert(ReadRepresentation{String, Vlen{Vlen{String}}}(), f, dataptr, NULL_REFERENCE) + if jlconvert_isinitialized(ReadRepresentation{DataType, RelOffset}(), ptr) + # There exists a parametric type definition + pt = jlconvert(ReadRepresentation{DataType, RelOffset}(), f, ptr, NULL_REFERENCE) + # Instantiating with the (reconstructed) parametric type works + m = pt.name.wrapper{params...} else - fieldnames = String[] + ptr += odr_sizeof(RelOffset) + # Load fieldnames if they exist + if jlconvert_isinitialized(ReadRepresentation{String,Vlen{Vlen{String}}}(), ptr) + fieldnames = jlconvert(ReadRepresentation{String, Vlen{Vlen{String}}}(), f, ptr, NULL_REFERENCE) + else + fieldnames = String[] + end + ptr += odr_sizeof(Vlen{String}) + fieldtypes = types_from_refs(f, ptr) + # Construct a new (nonparametric) type using fields + m = create_type(m.name, params, fieldnames, fieldtypes) end - fieldtypes = types_from_refs(f, ptr+odr_sizeof(Vlen{UInt8})+odr_sizeof(Vlen{RelOffset})+odr_sizeof(Vlen{RelOffset})) - m = create_type(m.name, params, fieldnames, fieldtypes) end track_weakref!(f, header_offset, m) return m diff --git a/src/data/unknown_types.jl b/src/data/unknown_types.jl index 7a4edd1a..b5f7afd4 100644 --- a/src/data/unknown_types.jl +++ b/src/data/unknown_types.jl @@ -2,18 +2,51 @@ module ReconstructedTypes end + + +symbolify_typevars(type::TypeVar) = type.name +symbolify_typevars(T) = T + +function symbolify_typevars(T::Type) + if !isempty(T.parameters) + #ex = Expr(:curly, Symbol(T.name.wrapper), symbolify_typevars.(T.parameters)...) + ex = :($(Symbol(T.name.wrapper)){$(symbolify_typevars.(T.parameters)...)}) + println(ex) + return ex + else + return T + end +end + + + # Construct a datatype from type parameters, field names, and field types function create_type(T, typeparams, fieldnames, fieldtypes) reconname = gensym(T) @warn "Unable to match type $T. Reconstructing to $reconname" - typeparamnames = Symbol.(('A':'Z')[1:length(typeparams)]) - fieldtypes = [(ft isa SelfReferentialPlaceholder ? reconname : ft) for ft in fieldtypes] - Core.eval(ReconstructedTypes, - Expr(:struct, false, :($reconname{$(typeparamnames...)}), - Expr(:block, Any[ Expr(Symbol("::"), Symbol(fieldnames[i]), fieldtypes[i]) for i = 1:length(fieldtypes) ]..., - # suppress default constructors, plus a bogus `new()` call to make sure - # ninitialized is zero. - Expr(:if, false, Expr(:call, :new))))) + typeparamnames = map(1:length(typeparams)) do n + tp = typeparams[n] + if tp isa TypeVar + # We can even return the sub typing relationships + # However, some types are already unknown. + # If field types need to be reconstructed, + # things would break again. + #return :($(tp.name) <: $(tp.ub)) + return tp.name + else + return gensym(:T) + end + end + fieldtypes = Any[ + (ft isa SelfReferentialPlaceholder ? reconname : symbolify_typevars(ft)) + for ft in fieldtypes] + ex = Expr(:struct, false, :($reconname{$(typeparamnames...)}), + Expr(:block, Any[ + Expr(Symbol("::"), Symbol(fieldnames[i]), fieldtypes[i]) + for i = 1:length(fieldtypes) ]...,)) + println(ex) + Core.eval(ReconstructedTypes, ex + ) T = getfield(ReconstructedTypes, reconname) if !isempty(typeparams) return T{typeparams...} diff --git a/src/data/writing_datatypes.jl b/src/data/writing_datatypes.jl index 5b32bc08..3726357b 100644 --- a/src/data/writing_datatypes.jl +++ b/src/data/writing_datatypes.jl @@ -11,29 +11,40 @@ const DataTypeODR = OnDiskRepresentation{ (0, odr_sizeof(Vlen{String}), odr_sizeof(Vlen{String})+odr_sizeof(Vlen{RelOffset}), - odr_sizeof(Vlen{String})+2*odr_sizeof(Vlen{RelOffset})), + odr_sizeof(Vlen{String})+odr_sizeof(RelOffset)+odr_sizeof(Vlen{RelOffset}), + odr_sizeof(Vlen{String})+odr_sizeof(RelOffset)+2*odr_sizeof(Vlen{RelOffset}), + odr_sizeof(Vlen{String})+odr_sizeof(RelOffset)+3*odr_sizeof(Vlen{RelOffset})), Tuple{String, Vector{Any}, + Any, Vector{String}, - Vector{Any}}, + Vector{Any}, + String}, Tuple{Vlen{String}, + Vlen{RelOffset}, + RelOffset, Vlen{RelOffset}, Vlen{RelOffset}, - Vlen{RelOffset}}} + Vlen{String}}} const OldDataTypeODR = OnDiskRepresentation{(0, odr_sizeof(Vlen{String})),Tuple{String,Vector{Any}},Tuple{Vlen{String},Vlen{RelOffset}}} const H5TYPE_DATATYPE = CompoundDatatype( - odr_sizeof(Vlen{String})+odr_sizeof(Vlen{RelOffset})+odr_sizeof(Vlen{RelOffset})+odr_sizeof(Vlen{RelOffset}), - [:name, :parameters, :fieldnames, :fieldtypes], + 2*odr_sizeof(Vlen{String})+odr_sizeof(RelOffset)+3*odr_sizeof(Vlen{RelOffset}), + [:name, :parameters, :typedef, :fieldnames, :fieldtypes, :debuginfo], [0, odr_sizeof(Vlen{String}), odr_sizeof(Vlen{String})+odr_sizeof(Vlen{RelOffset}), - odr_sizeof(Vlen{String})+2*odr_sizeof(Vlen{RelOffset})], + odr_sizeof(Vlen{String})+odr_sizeof(RelOffset)+ odr_sizeof(Vlen{RelOffset}), + odr_sizeof(Vlen{String})+odr_sizeof(RelOffset)+2*odr_sizeof(Vlen{RelOffset}), + odr_sizeof(Vlen{String})+odr_sizeof(RelOffset)+3*odr_sizeof(Vlen{RelOffset})], [H5TYPE_VLEN_UTF8, VariableLengthDatatype(ReferenceDatatype()), + ReferenceDatatype(), VariableLengthDatatype(H5TYPE_VLEN_UTF8), - VariableLengthDatatype(ReferenceDatatype())] + VariableLengthDatatype(ReferenceDatatype()), + H5TYPE_VLEN_UTF8 + ] ) const H5TYPE_OLD_DATATYPE = CompoundDatatype( @@ -45,31 +56,57 @@ const H5TYPE_OLD_DATATYPE = CompoundDatatype( function h5convert!(out::Pointers, ::DataTypeODR, f::JLDFile, T::DataType, wsession::JLDWriteSession) + # Figure out basic stuff + hasparams = !isempty(T.parameters) + # Is this a regular parametric datatype ? + # Has type parameters and none of them are TypeVars + regularparametric = hasparams && !any(isa.(T.parameters, TypeVar)) + # Should fieldtypes and field names be recorded? + # If datatype is defined in KNOWN_MODULES (e.g. Core) + # then we don't want to write out fieldnames and fieldtypes + # because (a) they will already be known and (b) because + # they can introduce a long tail of further intrinsic types + fieldnames = T.name.names + writefields = !isempty(fieldnames) && T.name.module ∉ KNOWN_MODULES + # If this is a regular parametric type then the types should + # be extracted from the parametric type def + writefields = writefields && !regularparametric + t = typename(T) store_vlen!(out, UInt8, f, unsafe_wrap(Vector{UInt8}, t), f.datatype_wsession) out += odr_sizeof(Vlen{UInt8}) - if isempty(T.parameters) - h5convert_uninitialized!(out, Vlen{UInt8}) - else + if hasparams refs = refs_from_types(f, T.parameters, wsession) store_vlen!(out, RelOffset, f, refs, f.datatype_wsession) + else + h5convert_uninitialized!(out, Vlen{RelOffset}) end out += odr_sizeof(Vlen{RelOffset}) - fieldnames = T.name.names - # If datatype is defined in KNOWN_MODULES (e.g. Core) - # then we don't want to write out fieldnames and fieldtypes - # because (a) they will already be known and (b) because - # they can introduce a long tail of further intrinsic types - if isempty(fieldnames) || T.name.module in KNOWN_MODULES - h5convert_uninitialized!(out, Vlen{String}) - out += odr_sizeof(Vlen{String}) - h5convert_uninitialized!(out, Vlen{RelOffset}) + + if regularparametric + t = T.name.wrapper + while t isa UnionAll + t = t.body + end + h5convert!(out, RelOffset, f, t, f.datatype_wsession) else + h5convert_uninitialized!(out, RelOffset) + end + out += odr_sizeof(RelOffset) + + + if writefields store_vlen!(out, Vlen{String}, f, string.(fieldnames), wsession) out += odr_sizeof(Vlen{String}) refs = refs_from_types(f, T.types, wsession) store_vlen!(out, RelOffset, f, refs, f.datatype_wsession) + else + h5convert_uninitialized!(out, Vlen{String}) + out += odr_sizeof(Vlen{String}) + h5convert_uninitialized!(out, Vlen{RelOffset}) end + out += odr_sizeof(Vlen{RelOffset}) + store_vlen!(out, UInt8, f, unsafe_wrap(Vector{UInt8}, string(T)), f.datatype_wsession) nothing end diff --git a/src/datasets.jl b/src/datasets.jl index 9f62d405..de1a4b6f 100644 --- a/src/datasets.jl +++ b/src/datasets.jl @@ -438,7 +438,7 @@ function write_dataset(f::JLDFile, dataspace::WriteDataspace, datatype::H5Dataty push!(wsession.objects, data) end - if data isa Type + if data isa DataType id = length(f.juliatypes_group)+1 f.juliatypes_group[@sprintf("%08d", id)] = h5offset(f, header_offset) f.juliatype_locations[data] = h5offset(f, header_offset)