store parametric type def

JuliaIO · Aug 25, 2021 · 14be6ca · 14be6ca
1 parent 496493a
commit 14be6ca
Show file tree

Hide file tree

Showing 4 changed files with 118 additions and 38 deletions.
diff --git a/src/data/reconstructing_datatypes.jl b/src/data/reconstructing_datatypes.jl
@@ -305,18 +305,18 @@ function jlconvert(rr::ReadRepresentation{T,DataTypeODR()},
                    ptr::Ptr,
                    header_offset::RelOffset) where T
     mypath = String(jlconvert(ReadRepresentation{UInt8,Vlen{UInt8}}(), f, ptr, NULL_REFERENCE))
-
+    ptr += odr_sizeof(Vlen{UInt8})
     track_weakref!(f, header_offset, SelfReferentialPlaceholder())
 
-    params = types_from_refs(f, ptr+odr_sizeof(Vlen{UInt8}))
+    params = types_from_refs(f, ptr)
+    ptr += odr_sizeof(Vlen{RelOffset})
     # For cross-platform compatibility convert integer type parameters to system precision
     params = [p isa Union{Int64,Int32} ? Int(p) : p for p in params]
-    hasparams = !isempty(params)
 
     m = _resolve_type(mypath, params)
 
     reconstruct = m isa UnknownType
-    if !reconstruct && hasparams
+    if !reconstruct && !isempty(params)
         try
             m = m{params...}
         catch e
@@ -329,14 +329,24 @@ function jlconvert(rr::ReadRepresentation{T,DataTypeODR()},
     end
 
     if reconstruct
-        dataptr= ptr+odr_sizeof(Vlen{UInt8})+odr_sizeof(Vlen{RelOffset})
-        if jlconvert_isinitialized(ReadRepresentation{String,Vlen{String}}(), dataptr)
-            fieldnames = jlconvert(ReadRepresentation{String, Vlen{Vlen{String}}}(), f, dataptr, NULL_REFERENCE)
+        if jlconvert_isinitialized(ReadRepresentation{DataType, RelOffset}(), ptr)
+            # There exists a parametric type definition
+            pt = jlconvert(ReadRepresentation{DataType, RelOffset}(), f, ptr, NULL_REFERENCE)
+            # Instantiating with the (reconstructed) parametric type works
+            m = pt.name.wrapper{params...}
         else
-            fieldnames = String[]
+            ptr += odr_sizeof(RelOffset)
+            # Load fieldnames if they exist
+            if jlconvert_isinitialized(ReadRepresentation{String,Vlen{Vlen{String}}}(), ptr)
+                fieldnames = jlconvert(ReadRepresentation{String, Vlen{Vlen{String}}}(), f, ptr, NULL_REFERENCE)
+            else
+                fieldnames = String[]
+            end
+            ptr += odr_sizeof(Vlen{String})
+            fieldtypes = types_from_refs(f, ptr)
+            # Construct a new (nonparametric) type using fields
+            m = create_type(m.name, params, fieldnames, fieldtypes)
         end
-        fieldtypes = types_from_refs(f, ptr+odr_sizeof(Vlen{UInt8})+odr_sizeof(Vlen{RelOffset})+odr_sizeof(Vlen{RelOffset}))
-        m = create_type(m.name, params, fieldnames, fieldtypes)
     end
     track_weakref!(f, header_offset, m)
     return m

diff --git a/src/data/unknown_types.jl b/src/data/unknown_types.jl
@@ -2,18 +2,51 @@
 
 module ReconstructedTypes end
 
+
+
+symbolify_typevars(type::TypeVar) = type.name
+symbolify_typevars(T) = T
+
+function symbolify_typevars(T::Type)
+    if !isempty(T.parameters)
+        #ex = Expr(:curly, Symbol(T.name.wrapper), symbolify_typevars.(T.parameters)...)
+        ex = :($(Symbol(T.name.wrapper)){$(symbolify_typevars.(T.parameters)...)})
+        println(ex)
+        return ex
+    else
+        return T
+    end
+end
+
+
+
 # Construct a datatype from type parameters, field names, and field types
 function create_type(T, typeparams, fieldnames, fieldtypes)
     reconname = gensym(T)
     @warn "Unable to match type $T. Reconstructing to $reconname"
-    typeparamnames = Symbol.(('A':'Z')[1:length(typeparams)])
-    fieldtypes = [(ft isa SelfReferentialPlaceholder ? reconname : ft) for ft in fieldtypes]
-    Core.eval(ReconstructedTypes,
-              Expr(:struct, false, :($reconname{$(typeparamnames...)}),
-                   Expr(:block, Any[ Expr(Symbol("::"), Symbol(fieldnames[i]), fieldtypes[i]) for i = 1:length(fieldtypes) ]...,
-                        # suppress default constructors, plus a bogus `new()` call to make sure
-                        # ninitialized is zero.
-                        Expr(:if, false, Expr(:call, :new)))))
+    typeparamnames = map(1:length(typeparams)) do n
+        tp = typeparams[n]
+        if tp isa TypeVar
+            # We can even return the sub typing relationships
+            # However, some types are already unknown.
+            # If field types need to be reconstructed, 
+            # things would break again.
+            #return :($(tp.name) <: $(tp.ub))
+            return tp.name
+        else
+            return gensym(:T)
+        end    
+    end
+    fieldtypes = Any[
+            (ft isa SelfReferentialPlaceholder ? reconname : symbolify_typevars(ft))
+            for ft in fieldtypes]
+    ex = Expr(:struct, false, :($reconname{$(typeparamnames...)}),
+            Expr(:block, Any[ 
+                Expr(Symbol("::"), Symbol(fieldnames[i]), fieldtypes[i])
+                 for i = 1:length(fieldtypes) ]...,))
+    println(ex)
+    Core.eval(ReconstructedTypes, ex
+              )
     T = getfield(ReconstructedTypes, reconname)
     if !isempty(typeparams)
         return T{typeparams...}

diff --git a/src/data/writing_datatypes.jl b/src/data/writing_datatypes.jl
@@ -11,29 +11,40 @@ const DataTypeODR = OnDiskRepresentation{
     (0,
     odr_sizeof(Vlen{String}),
     odr_sizeof(Vlen{String})+odr_sizeof(Vlen{RelOffset}),
-    odr_sizeof(Vlen{String})+2*odr_sizeof(Vlen{RelOffset})),
+    odr_sizeof(Vlen{String})+odr_sizeof(RelOffset)+odr_sizeof(Vlen{RelOffset}),
+    odr_sizeof(Vlen{String})+odr_sizeof(RelOffset)+2*odr_sizeof(Vlen{RelOffset}),
+    odr_sizeof(Vlen{String})+odr_sizeof(RelOffset)+3*odr_sizeof(Vlen{RelOffset})),
     Tuple{String,
         Vector{Any},
+        Any,
         Vector{String},
-        Vector{Any}},
+        Vector{Any},
+        String},
     Tuple{Vlen{String},
+        Vlen{RelOffset},
+        RelOffset,
         Vlen{RelOffset},
         Vlen{RelOffset},
-        Vlen{RelOffset}}}
+        Vlen{String}}}
 
 const OldDataTypeODR = OnDiskRepresentation{(0, odr_sizeof(Vlen{String})),Tuple{String,Vector{Any}},Tuple{Vlen{String},Vlen{RelOffset}}}
 
 const H5TYPE_DATATYPE = CompoundDatatype(
-    odr_sizeof(Vlen{String})+odr_sizeof(Vlen{RelOffset})+odr_sizeof(Vlen{RelOffset})+odr_sizeof(Vlen{RelOffset}),
-    [:name, :parameters, :fieldnames, :fieldtypes],
+    2*odr_sizeof(Vlen{String})+odr_sizeof(RelOffset)+3*odr_sizeof(Vlen{RelOffset}),
+    [:name, :parameters, :typedef, :fieldnames, :fieldtypes, :debuginfo],
     [0,
      odr_sizeof(Vlen{String}),
      odr_sizeof(Vlen{String})+odr_sizeof(Vlen{RelOffset}),
-     odr_sizeof(Vlen{String})+2*odr_sizeof(Vlen{RelOffset})],
+     odr_sizeof(Vlen{String})+odr_sizeof(RelOffset)+ odr_sizeof(Vlen{RelOffset}),
+     odr_sizeof(Vlen{String})+odr_sizeof(RelOffset)+2*odr_sizeof(Vlen{RelOffset}),
+     odr_sizeof(Vlen{String})+odr_sizeof(RelOffset)+3*odr_sizeof(Vlen{RelOffset})],
     [H5TYPE_VLEN_UTF8,
     VariableLengthDatatype(ReferenceDatatype()),
+    ReferenceDatatype(),
     VariableLengthDatatype(H5TYPE_VLEN_UTF8),
-    VariableLengthDatatype(ReferenceDatatype())]
+    VariableLengthDatatype(ReferenceDatatype()),
+    H5TYPE_VLEN_UTF8
+    ]
 )
 
 const H5TYPE_OLD_DATATYPE = CompoundDatatype(
@@ -45,31 +56,57 @@ const H5TYPE_OLD_DATATYPE = CompoundDatatype(
 
 
 function h5convert!(out::Pointers, ::DataTypeODR, f::JLDFile, T::DataType, wsession::JLDWriteSession)
+    # Figure out basic stuff
+    hasparams = !isempty(T.parameters)
+    # Is this a regular parametric datatype ?
+    # Has type parameters and none of them are TypeVars
+    regularparametric = hasparams && !any(isa.(T.parameters, TypeVar))
+    # Should fieldtypes and field names be recorded?
+    # If datatype is defined in KNOWN_MODULES (e.g. Core)
+    # then we don't want to write out fieldnames and fieldtypes
+    # because (a) they will already be known and (b) because
+    # they can introduce a long tail of further intrinsic types
+    fieldnames = T.name.names
+    writefields = !isempty(fieldnames) && T.name.module ∉ KNOWN_MODULES
+    # If this is a regular parametric type then the types should
+    # be extracted from the parametric type def
+    writefields = writefields && !regularparametric
+
     t = typename(T)
     store_vlen!(out, UInt8, f, unsafe_wrap(Vector{UInt8}, t), f.datatype_wsession)
     out += odr_sizeof(Vlen{UInt8})
-    if isempty(T.parameters)
-        h5convert_uninitialized!(out, Vlen{UInt8})
-    else
+    if hasparams
         refs = refs_from_types(f, T.parameters, wsession)
         store_vlen!(out, RelOffset, f, refs, f.datatype_wsession)
+    else
+        h5convert_uninitialized!(out, Vlen{RelOffset})
     end
     out += odr_sizeof(Vlen{RelOffset})
-    fieldnames = T.name.names
-    # If datatype is defined in KNOWN_MODULES (e.g. Core)
-    # then we don't want to write out fieldnames and fieldtypes
-    # because (a) they will already be known and (b) because
-    # they can introduce a long tail of further intrinsic types
-    if isempty(fieldnames) || T.name.module in KNOWN_MODULES
-        h5convert_uninitialized!(out, Vlen{String})
-        out += odr_sizeof(Vlen{String})
-        h5convert_uninitialized!(out, Vlen{RelOffset})
+
+    if regularparametric
+        t = T.name.wrapper
+        while t isa UnionAll
+            t = t.body
+        end
+        h5convert!(out, RelOffset, f, t, f.datatype_wsession)
     else
+        h5convert_uninitialized!(out, RelOffset)
+    end
+    out += odr_sizeof(RelOffset)
+
+
+    if writefields
         store_vlen!(out, Vlen{String}, f, string.(fieldnames), wsession)
         out += odr_sizeof(Vlen{String})
         refs = refs_from_types(f, T.types, wsession)
         store_vlen!(out, RelOffset, f, refs, f.datatype_wsession)
+    else
+        h5convert_uninitialized!(out, Vlen{String})
+        out += odr_sizeof(Vlen{String})
+        h5convert_uninitialized!(out, Vlen{RelOffset})
     end
+    out += odr_sizeof(Vlen{RelOffset})
+    store_vlen!(out, UInt8, f, unsafe_wrap(Vector{UInt8}, string(T)), f.datatype_wsession)
 
     nothing
 end

diff --git a/src/datasets.jl b/src/datasets.jl
@@ -438,7 +438,7 @@ function write_dataset(f::JLDFile, dataspace::WriteDataspace, datatype::H5Dataty
         push!(wsession.objects, data)
     end
 
-    if data isa Type
+    if data isa DataType
         id = length(f.juliatypes_group)+1
         f.juliatypes_group[@sprintf("%08d", id)] = h5offset(f, header_offset)
         f.juliatype_locations[data] = h5offset(f, header_offset)