Merge pull request #1476 from JuliaRobotics/22Q1/enh/jsononlysrz

packed factors are nested JSON only,

src/Deprecated.jl

@@ -42,6 +42,16 @@ export FunctorInferenceType, PackedInferenceType
 ## Deprecate code below before v0.28
 ##==============================================================================
 
+function Base.convert(::Type{String}, 
+                      obj::FluxModelsDistribution)
+  #
+  @error "Obsolete, FluxModelsSerialization should not return String for general cases of PackedSamplableBelief"
+  # convert to packed type first
+  packed = convert(PackedFluxModelsDistribution, obj)
+  # FIXME, should not return String for general cases of PackedSamplableBelief 
+  return JSON2.write(packed)
+end
+
 
 # import IncrementalInference: decodefg, loadjld
 

src/ExportAPI.jl

@@ -305,6 +305,8 @@ export PackedZeroMeanDiagNormal, PackedZeroMeanFullNormal, PackedDiagNormal, Pac
 export PackedManifoldKernelDensity
 export PackedAliasingScalarSampler, PackedHeatmapGridDensity, PackedLevelSetGridNormal
 
+export Mixture, PackedMixture
+
 export sampleTangent
 export samplePoint
 

src/Factors/Circular.jl

@@ -72,16 +72,14 @@ $(TYPEDEF)
 
 Serialized object for storing PriorCircular.
 """
-mutable struct PackedPriorCircular  <: AbstractPackedFactor
-  datastr::String
-  # PackedPriorCircular() = new()
-  # PackedPriorCircular(x::String) = new(x)
+Base.@kwdef struct PackedPriorCircular  <: AbstractPackedFactor
+  Z::PackedSamplableBelief
 end
 function convert(::Type{PackedPriorCircular}, d::PriorCircular)
-  return PackedPriorCircular(convert(String, d.Z)) # TODO, PackedSamplableBelief
+  return PackedPriorCircular(convert(PackedSamplableBelief, d.Z))
 end
 function convert(::Type{PriorCircular}, d::PackedPriorCircular)
-  distr = convert(SamplableBelief, d.datastr)
+  distr = convert(SamplableBelief, d.Z)
   return PriorCircular{typeof(distr)}(distr)
 end
 
@@ -97,16 +95,14 @@ $(TYPEDEF)
 
 Serialized object for storing CircularCircular.
 """
-mutable struct PackedCircularCircular  <: AbstractPackedFactor
-  datastr::String
-  # PackedCircularCircular() = new()
-  # PackedCircularCircular(x::String) = new(x)
+Base.@kwdef struct PackedCircularCircular  <: AbstractPackedFactor
+  Z::PackedSamplableBelief
 end
 function convert(::Type{CircularCircular}, d::PackedCircularCircular)
-  return CircularCircular(convert(SamplableBelief, d.datastr))
+  return CircularCircular(convert(SamplableBelief, d.Z))
 end
 function convert(::Type{PackedCircularCircular}, d::CircularCircular)
-  return PackedCircularCircular(convert(String, d.Z)) # TODO, PackedSamplableBelief
+  return PackedCircularCircular(convert(PackedSamplableBelief, d.Z))
 end
 
 

src/Factors/DefaultPrior.jl

@@ -26,11 +26,11 @@ $(TYPEDEF)
 
 Serialization type for Prior.
 """
-mutable struct PackedPrior <: AbstractPackedFactor
-  Z::String
+Base.@kwdef mutable struct PackedPrior <: AbstractPackedFactor
+  Z::PackedSamplableBelief
 end
 function convert(::Type{PackedPrior}, d::Prior)
-  PackedPrior(convert(String, d.Z)) # TODO, PackedSamplableBelief
+  PackedPrior(convert(PackedSamplableBelief, d.Z))
 end
 function convert(::Type{Prior}, d::PackedPrior)
   Prior(convert(SamplableBelief, d.Z))

src/Factors/EuclidDistance.jl

@@ -30,14 +30,14 @@ Base.convert(::Type{<:MB.AbstractManifold}, ::InstanceType{EuclidDistance}) = Ma
 $(TYPEDEF)
 Serialization type for `EuclidDistance` binary factor.
 """
-mutable struct PackedEuclidDistance <: AbstractPackedFactor
+Base.@kwdef mutable struct PackedEuclidDistance <: AbstractPackedFactor
   _type::String
-  Z::String
+  Z::PackedSamplableBelief
 end
 
 function convert(::Type{PackedEuclidDistance}, d::EuclidDistance)
   PackedEuclidDistance( "/application/JuliaLang/PackedSamplableBelief",
-                        convert(String, d.Z) ) # TODO, PackedSamplableBelief
+                        convert(PackedSamplableBelief, d.Z) )
 end
 
 function convert(::Type{<:EuclidDistance}, d::PackedEuclidDistance)

src/Factors/GenericFunctions.jl

@@ -155,10 +155,10 @@ function mahalanobus_distance2(M, X, inv_Σ)
 end
 
 
-mutable struct PackedManifoldPrior <: AbstractPackedFactor
+Base.@kwdef mutable struct PackedManifoldPrior <: AbstractPackedFactor
   varType::String 
   p::Vector{Float64}  #NOTE This is a fixed point from where the measurement `Z` likely stored as a coordinate
-  Z::String
+  Z::PackedSamplableBelief
 end
 
 
@@ -171,8 +171,7 @@ function convert(::Union{Type{<:AbstractPackedFactor}, Type{<:PackedManifoldPrio
   c = AMP.makeCoordsFromPoint(obj.M, obj.p)
 
   # TODO convert all distributions to JSON
-  Zst = convert(String, obj.Z)
-  # Zst = string(obj.Z)
+  Zst = convert(PackedSamplableBelief, obj.Z) # String
 
   PackedManifoldPrior(varT, c, Zst)
 end
@@ -190,7 +189,6 @@ function convert(::Union{Type{<:AbstractFactor}, Type{<:ManifoldPrior}},
   # u0 = getPointIdentity(obj.varType)
   p = AMP.makePointFromCoords(M, obj.p, e0) #, u0)
 
-  @show obj.Z
   Z = convert(SamplableBelief, obj.Z)
 
   ManifoldPrior(M, p, Z)

src/Factors/GenericMarginal.jl

@@ -14,7 +14,7 @@ end
 
 getSample(::CalcFactor{<:GenericMarginal}) = [0]
 
-mutable struct PackedGenericMarginal <: AbstractPackedFactor
+Base.@kwdef mutable struct PackedGenericMarginal <: AbstractPackedFactor
     Zij::Array{Float64,1}
     Cov::Array{Float64,1}
     W::Array{Float64,1}

src/Factors/LinearRelative.jl

@@ -52,11 +52,11 @@ Base.convert(::Type{<:MB.AbstractManifold}, ::InstanceType{LinearRelative{N}}) w
 $(TYPEDEF)
 Serialization type for `LinearRelative` binary factor.
 """
-mutable struct PackedLinearRelative <: AbstractPackedFactor
-  Z::String
+Base.@kwdef mutable struct PackedLinearRelative <: AbstractPackedFactor
+  Z::PackedSamplableBelief
 end
 function convert(::Type{PackedLinearRelative}, d::LinearRelative)
-  PackedLinearRelative(convert(String, d.Z)) # TODO, PackedSamplableBelief
+  PackedLinearRelative(convert(PackedSamplableBelief, d.Z))
 end
 function convert(::Type{LinearRelative}, d::PackedLinearRelative)
   LinearRelative(convert(SamplableBelief, d.Z))

src/Factors/Mixture.jl

@@ -1,5 +1,4 @@
 
-export Mixture, PackedMixture
 
 
 _defaultNamesMixtures(N::Int) = ((Symbol[Symbol("c$i") for i in 1:N])...,)
@@ -107,28 +106,27 @@ $(TYPEDEF)
 
 Serialization type for `Mixture`.
 """
-mutable struct PackedMixture <: AbstractPackedFactor
+Base.@kwdef mutable struct PackedMixture <: AbstractPackedFactor
   N::Int
   # store the packed type for later unpacking
   F_::String 
   S::Vector{String}
-  components::Vector{String}
-  diversity::String
+  components::Vector{PackedSamplableBelief}
+  diversity::PackedSamplableBelief
 end
 
 
 function convert(::Type{<:PackedMixture}, obj::Mixture{N,F,S,T}) where {N,F,S,T}
-  allcomp = String[]
+  allcomp = PackedSamplableBelief[]
   for val in obj.components
-    dtr_ = convert(String, val) # PackedSamplableBelief
+    dtr_ = convert(PackedSamplableBelief, val)
     # FIXME ON FIRE, likely to be difficult for non-standard "Samplable" types -- e.g. Flux models in RoME
-    # dtr_ = JSON2.write(packDistribution(val))
     push!(allcomp, dtr_)
   end
   pm = DFG.convertPackedType(obj.mechanics)
   pm_ = convert(pm, obj.mechanics)
   sT = string(typeof(pm_))
-  dvst = convert(String, obj.diversity) # PackedSamplableBelief
+  dvst = convert(PackedSamplableBelief, obj.diversity)
   PackedMixture( N, sT, string.(collect(S)), allcomp, dvst )
 end
 

src/Factors/MsgPrior.jl

@@ -35,13 +35,13 @@ getManifold(mp::MsgPrior{<:ManifoldKernelDensity}) = mp.Z.manifold
 
 
 
-struct PackedMsgPrior <: AbstractPackedFactor
-  Z::String
+Base.@kwdef struct PackedMsgPrior <: AbstractPackedFactor
+  Z::PackedSamplableBelief
   infoPerCoord::Vector{Float64}
 end
 
 function convert(::Type{PackedMsgPrior}, d::MsgPrior)
-  PackedMsgPrior(convert(String, d.Z), d.infoPerCoord) # TODO PackedSamplableBelief
+  PackedMsgPrior(convert(PackedSamplableBelief, d.Z), d.infoPerCoord)
 end
 function convert(::Type{<:MsgPrior}, d::PackedMsgPrior)
   MsgPrior(convert(SamplableBelief, d.Z), d.infoPerCoord)

src/Factors/PartialPrior.jl

@@ -29,13 +29,13 @@ $(TYPEDEF)
 
 Serialization type for `PartialPrior`.
 """
-mutable struct PackedPartialPrior <: AbstractPackedFactor
-  Z::String
+Base.@kwdef struct PackedPartialPrior <: AbstractPackedFactor
+  Z::PackedSamplableBelief
   partials::Vector{Int}
 end
 
 function convert(::Type{PackedPartialPrior}, d::PartialPrior)
-  PackedPartialPrior(convert(String, d.Z), [d.partial...;]) # TODO, PackedSamplableBelief
+  PackedPartialPrior(convert(PackedSamplableBelief, d.Z), [d.partial...;])
 end
 function convert(::Type{PartialPrior}, d::PackedPartialPrior)
   PartialPrior(convert(SamplableBelief, d.Z),(d.partials...,))

src/Factors/PartialPriorPassThrough.jl

@@ -24,8 +24,8 @@ getSample(cf::CalcFactor{<:PartialPriorPassThrough}) = sampleTangent(getManifold
 
 Required internal density to store its type
 """
-mutable struct PackedPartialPriorPassThrough <: AbstractPackedFactor
-  Z::String # PackedHeatmapGridDensity
+Base.@kwdef mutable struct PackedPartialPriorPassThrough <: AbstractPackedFactor
+  Z::PackedSamplableBelief # PackedHeatmapGridDensity
   partial::Vector{Int}
 end
 
@@ -34,25 +34,16 @@ function convert( ::Union{Type{<:AbstractPackedFactor}, Type{<:PackedPartialPrio
                   obj::PartialPriorPassThrough )
   #
 
-  # TODO, PackedSamplableBelief
-  str = convert(String, obj.Z)
-  # po = convert(PackedSamplableBelief, obj.Z)
-  # str = JSON2.write(po)
-  PackedPartialPriorPassThrough(str, Int[obj.partial...])
+  po = convert(PackedSamplableBelief, obj.Z)
+  PackedPartialPriorPassThrough(po, Int[obj.partial...])
 end
 
 
 function convert( ::Union{Type{<:AbstractFactor}, Type{<:PartialPriorPassThrough}},
                   obj::PackedPartialPriorPassThrough )
   #
 
-  # get as bland obj to extract type
   dens = convert(SamplableBelief, obj.Z)
-  # _up = JSON2.read(obj.Z)
-  # _typ = DFG.getTypeFromSerializationModule(_up._type)
-  # # now redo with type
-  # pdens = JSON2.read(obj.Z, _typ)
-  # dens = convert(SamplableBelief, pdens)
   PartialPriorPassThrough(dens, tuple(obj.partial...))
 end
 

src/Serialization/entities/AdditionalDensities.jl

@@ -1,7 +1,7 @@
 
 
 
-struct PackedManifoldKernelDensity <: PackedSamplableBelief
+Base.@kwdef struct PackedManifoldKernelDensity <: PackedSamplableBelief
   _type::String
   varType::String
   pts::Vector{Vector{Float64}}
@@ -18,7 +18,7 @@ Base.@kwdef struct PackedAliasingScalarSampler <: PackedSamplableBelief
 end
 
 
-mutable struct PackedHeatmapGridDensity <: PackedSamplableBelief
+Base.@kwdef mutable struct PackedHeatmapGridDensity <: PackedSamplableBelief
   _type::String
   data::Vector{Vector{Float64}}
   domain::Tuple{Vector{Float64}, Vector{Float64}}
@@ -29,7 +29,7 @@ mutable struct PackedHeatmapGridDensity <: PackedSamplableBelief
 end
 
 
-mutable struct PackedLevelSetGridNormal <: PackedSamplableBelief
+Base.@kwdef mutable struct PackedLevelSetGridNormal <: PackedSamplableBelief
   _type::String
   level::Float64
   sigma::Float64

src/Serialization/entities/FluxModelsSerialization.jl

@@ -2,7 +2,7 @@
 export PackedFluxModelsDistribution
 
 
-mutable struct PackedFluxModelsDistribution <: IIF.PackedSamplableBelief
+Base.@kwdef mutable struct PackedFluxModelsDistribution <: PackedSamplableBelief
   # standardized _type field
   _type::String
   # shape of the input data
@@ -21,6 +21,6 @@ mutable struct PackedFluxModelsDistribution <: IIF.PackedSamplableBelief
   serializeHollow::Bool
     # TODO remove requirement and standardize sampler API
     # specialSampler::Symbol
-  # field name usage to direct the IIF serialization towards JSON method
+  # TODO, only use ._type.  Legacy, field name usage to direct the IIF serialization towards JSON method
   PackedSamplableTypeJSON::String
 end

src/Serialization/services/FluxModelsSerialization.jl

@@ -79,16 +79,6 @@ function packDistribution(obj::FluxModelsDistribution)
 end
 
 
-function Base.convert(::Type{String}, # ::Union{Type{<:PackedSamplableBelief},Type{<:PackedFluxModelsDistribution}}, 
-                      obj::FluxModelsDistribution)
-  #
-  # convert to packed type first
-  packed = packDistribution(obj)
-  # FIXME, should not return String for general cases of PackedSamplableBelief 
-  return JSON2.write(packed)
-end
-
-
 
 function unpackDistribution(obj::PackedFluxModelsDistribution)
   #
@@ -111,10 +101,20 @@ end
 
 
 
+function Base.convert(::Union{Type{<:PackedSamplableBelief},Type{<:PackedFluxModelsDistribution}}, 
+                      obj::FluxModelsDistribution)
+  #
+  # convert to packed type first
+  return packDistribution(obj)
+end
+
+
+
 function convert( ::Union{Type{<:SamplableBelief},Type{<:FluxModelsDistribution}}, 
                   obj::PackedFluxModelsDistribution )
   #
-  return unpackDistirbution(obj)
+  return unpackDistribution(obj)
 end
 
+
 #

src/Serialization/services/SerializingDistributions.jl

@@ -12,6 +12,24 @@ packDistribution(dtr::FullNormal) = PackedFullNormal(; mu=dtr.μ, cov=dtr.Σ.mat
 
 packDistribution(dtr::AliasingScalarSampler) = PackedAliasingScalarSampler(; domain=dtr.domain, weights=dtr.weights.values )
 
+# A more specialized constructor to help serialization processes reading Any[Any[1,2,3,..]] rather than floats. 
+function PackedHeatmapGridDensity(
+    _type::String, 
+    data::AbstractVector, # {Any} 
+    domain::AbstractVector, # {Any}
+    hint_callback::String, 
+    bw_factor::Float64, 
+    N::Int64 )
+  #
+  data_ = Vector{Vector{Float64}}(undef, length(data))
+  for (i,dat) in enumerate(data)
+    data_[i] = float.(dat)
+  end
+  domain_ = tuple(float.(domain[1]), float.(domain[2]))
+
+  PackedHeatmapGridDensity(_type, data_, domain_, hint_callback, bw_factor, N)
+end
+
 function packDistribution( obj::HeatmapGridDensity )
   #
   data_ = obj.data
@@ -101,6 +119,10 @@ unpackDistribution(dtr::PackedLevelSetGridNormal) = LevelSetGridNormal( dtr.leve
 convert(::Type{<:PackedSamplableBelief}, obj::StringThemSamplableBeliefs) = packDistribution(obj)
 convert(::Type{<:SamplableBelief}, obj::PackedSamplableBelief) = unpackDistribution(obj)
 
+function convert(::Type{<:PackedSamplableBelief}, nt::NamedTuple)
+  distrType = DFG.getTypeFromSerializationModule(nt._type)
+  distrType(;nt...)
+end
 
 ##===================================================================================
 

test/testCircular.jl

@@ -32,6 +32,7 @@ gt = rem2pi.(collect(0:4), RoundNearest)
 d = "/tmp/caesar/random/testfg"
 saveDFG(d,fg)
 lfg = loadDFG(d)
+##
 Base.rm(d*".tar.gz")
 # check loaded fg for all variable and factors
 @test issetequal(ls(fg), ls(lfg))