diff --git a/src/DataBlobs/services/BlobEntry.jl b/src/DataBlobs/services/BlobEntry.jl index 8406b18a..c9c5fc18 100644 --- a/src/DataBlobs/services/BlobEntry.jl +++ b/src/DataBlobs/services/BlobEntry.jl @@ -114,6 +114,14 @@ function getBlobEntryFirst(var::AbstractDFGVariable, key::Regex) ) end +function getBlobEntryFirst(var::Variable, key::Regex) + firstIdx = findfirst(x->contains(string(x.label), key), var.blobEntries) + if isnothing(firstIdx) + throw(KeyError("$key")) + end + return var.blobEntries[firstIdx] +end + getBlobEntry(var::AbstractDFGVariable, key::AbstractString) = getBlobEntry(var,Regex(key)) #TODO split betweeen getfirstBlobEntry and getBlobEntry diff --git a/src/DistributedFactorGraphs.jl b/src/DistributedFactorGraphs.jl index 521d21b1..eedf9b71 100644 --- a/src/DistributedFactorGraphs.jl +++ b/src/DistributedFactorGraphs.jl @@ -347,6 +347,12 @@ include("entities/AbstractDFGSummary.jl") include("services/AbstractDFG.jl") +#Blobs +include("DataBlobs/services/BlobEntry.jl") +include("DataBlobs/services/BlobStores.jl") +include("DataBlobs/services/BlobPacking.jl") +include("DataBlobs/services/HelpersDataWrapEntryBlob.jl") + # In Memory Types include("GraphsDFG/GraphsDFG.jl") @reexport using .GraphsDFGs @@ -362,11 +368,6 @@ include("services/DFGVariable.jl") include("services/DFGFactor.jl") include("Deprecated.jl") include("services/CompareUtils.jl") -#Blobs -include("DataBlobs/services/BlobEntry.jl") -include("DataBlobs/services/BlobStores.jl") -include("DataBlobs/services/BlobPacking.jl") -include("DataBlobs/services/HelpersDataWrapEntryBlob.jl") # Include the FilesDFG API. include("FileDFG/FileDFG.jl") diff --git a/src/FileDFG/services/FileDFG.jl b/src/FileDFG/services/FileDFG.jl index 383a1a9d..faa60f62 100644 --- a/src/FileDFG/services/FileDFG.jl +++ b/src/FileDFG/services/FileDFG.jl @@ -149,7 +149,6 @@ function loadDFG!(dfgLoadInto::AbstractDFG, dst::AbstractString; overwriteDFGMet end # extract the factor graph from fileDFG folder - variables = DFGVariable[] factors = DFGFactor[] varFolder = "$folder/variables" factorFolder = "$folder/factors" @@ -160,43 +159,62 @@ function loadDFG!(dfgLoadInto::AbstractDFG, dst::AbstractString; overwriteDFGMet varFiles = sort(readdir(varFolder; sort=false); lt=natural_lt) factorFiles = sort(readdir(factorFolder; sort=false); lt=natural_lt) - @showprogress 1 "loading variables" for varFile in varFiles - jstr = read("$varFolder/$varFile", String) - try - packedData = JSON3.read(jstr, PackedVariable) - push!(variables, unpackVariable(packedData)) - catch ex - @error("JSON3 is having trouble reading $varFolder/$varFile into a PackedVariable") - @show jstr - throw(ex) + + if isa(dfgLoadInto, GraphsDFG) && GraphsDFGs._variablestype(dfgLoadInto) == Variable + variables = @showprogress 1 "loading variables" map(varFiles) do varFile + jstr = read("$varFolder/$varFile", String) + return JSON3.read(jstr, PackedVariable) + end + else + variables = DFGVariable[] + @showprogress 1 "loading variables" for varFile in varFiles + jstr = read("$varFolder/$varFile", String) + try + packedData = JSON3.read(jstr, PackedVariable) + push!(variables, unpackVariable(packedData)) + catch ex + @error("JSON3 is having trouble reading $varFolder/$varFile into a PackedVariable") + @show jstr + throw(ex) + end end end - @info "Loaded $(length(variables)) variables - $(map(v->v.label, variables))" + + @info "Loaded $(length(variables)) variables"#- $(map(v->v.label, variables))" @info "Inserting variables into graph..." # Adding variables map(v->addVariable!(dfgLoadInto, v), variables) - @showprogress 1 "loading factors" for factorFile in factorFiles - jstr = read("$factorFolder/$factorFile", String) - try - packedData = JSON3.read(jstr, PackedFactor) - push!(factors, unpackFactor(dfgLoadInto, packedData)) - catch ex - @error("JSON3 is having trouble reading $factorFolder/$factorFile into a PackedFactor") - @show jstr - throw(ex) + if isa(dfgLoadInto, GraphsDFG) && GraphsDFGs._factorstype(dfgLoadInto) == PackedFactor + factors = @showprogress 1 "loading factors" map(factorFiles) do factorFile + jstr = read("$factorFolder/$factorFile", String) + return JSON3.read(jstr, PackedFactor) + end + else + @showprogress 1 "loading factors" for factorFile in factorFiles + jstr = read("$factorFolder/$factorFile", String) + try + packedData = JSON3.read(jstr, PackedFactor) + push!(factors, unpackFactor(dfgLoadInto, packedData)) + catch ex + @error("JSON3 is having trouble reading $factorFolder/$factorFile into a PackedFactor") + @show jstr + throw(ex) + end end end - @info "Loaded $(length(variables)) factors - $(map(f->f.label, factors))" + @info "Loaded $(length(factors)) factors"# - $(map(f->f.label, factors))" @info "Inserting factors into graph..." # # Adding factors map(f->addFactor!(dfgLoadInto, f), factors) - # Finally, rebuild the CCW's for the factors to completely reinflate them - # NOTE CREATES A NEW DFGFactor IF CCW TYPE CHANGES - @info "Rebuilding CCW's for the factors..." - @showprogress 1 "build factor operational memory" for factor in factors - rebuildFactorMetadata!(dfgLoadInto, factor) + if isa(dfgLoadInto, GraphsDFG) && GraphsDFGs._factorstype(dfgLoadInto) != PackedFactor + # Finally, rebuild the CCW's for the factors to completely reinflate them + # NOTE CREATES A NEW DFGFactor IF CCW TYPE CHANGES + @info "Rebuilding CCW's for the factors..." + @showprogress 1 "build factor operational memory" for factor in factors + rebuildFactorMetadata!(dfgLoadInto, factor) + end end # remove the temporary unzipped file diff --git a/src/GraphsDFG/entities/GraphsDFG.jl b/src/GraphsDFG/entities/GraphsDFG.jl index 70d51e32..2e2b1f82 100644 --- a/src/GraphsDFG/entities/GraphsDFG.jl +++ b/src/GraphsDFG/entities/GraphsDFG.jl @@ -21,7 +21,7 @@ mutable struct GraphsDFG{T <: AbstractParams, V <: AbstractDFGVariable, F <:Abst sessionBlobEntries::OrderedDict{Symbol, BlobEntry} addHistory::Vector{Symbol} #TODO: Discuss more - is this an audit trail? solverParams::T # Solver parameters - blobStores::Dict{Symbol, <:AbstractBlobStore} + blobStores::Dict{Symbol, AbstractBlobStore} end """ @@ -46,7 +46,7 @@ function GraphsDFG{T,V,F}( sessionBlobEntries::OrderedDict{Symbol, BlobEntry} = OrderedDict{Symbol, BlobEntry}(), addHistory::Vector{Symbol} = Symbol[], solverParams::T=T(), - blobstores=Dict{Symbol, AbstractBlobStore}(), + blobStores=Dict{Symbol, AbstractBlobStore}(), ) where {T <: AbstractParams, V <:AbstractDFGVariable, F<:AbstractDFGFactor} # Validate the userLabel, robotLabel, and sessionLabel !isValidLabel(userLabel) && error("'$userLabel' is not a valid User label") @@ -67,7 +67,7 @@ function GraphsDFG{T,V,F}( sessionBlobEntries, addHistory, solverParams, - blobstores + blobStores ) end @@ -92,7 +92,7 @@ GraphsDFG( robotData::Dict{Symbol, SmallDataTypes}, sessionData::Dict{Symbol, SmallDataTypes}, solverParams::AbstractParams, - blobstores=Dict{Symbol, AbstractBlobStore}() + blobStores=Dict{Symbol, AbstractBlobStore}() ) = GraphsDFG{typeof(solverParams),DFGVariable,DFGFactor}( FactorGraph{Int,DFGVariable,DFGFactor}(); description, @@ -103,7 +103,7 @@ GraphsDFG( robotData, sessionData, solverParams, - blobstores + blobStores ) @@ -116,7 +116,7 @@ function GraphsDFG{T,V,F}( robotData::Dict{Symbol, SmallDataTypes}, sessionData::Dict{Symbol, SmallDataTypes}, solverParams::T, - blobstores=Dict{Symbol, AbstractBlobStore}() + blobStores=Dict{Symbol, AbstractBlobStore}() ) where {T <: AbstractParams, V <:AbstractDFGVariable, F<:AbstractDFGFactor} return GraphsDFG{T,V,F}( FactorGraph{Int,V,F}(); @@ -128,6 +128,6 @@ function GraphsDFG{T,V,F}( robotData, sessionData, solverParams, - blobstores + blobStores ) end \ No newline at end of file diff --git a/src/GraphsDFG/services/GraphsDFG.jl b/src/GraphsDFG/services/GraphsDFG.jl index f49bb60f..d7c031e1 100644 --- a/src/GraphsDFG/services/GraphsDFG.jl +++ b/src/GraphsDFG/services/GraphsDFG.jl @@ -1,4 +1,10 @@ +function getDFGMetadata(fg::GraphsDFG) + metafields = Set(fieldnames(GraphsDFG)) + setdiff!(metafields, [:g, :solverParams]) + metaprops = NamedTuple(k => getproperty(fg, k) for k in metafields) + return metaprops +end function exists(dfg::GraphsDFG{P,V,F}, node::V) where {P <: AbstractParams, V <: AbstractDFGVariable, F <: AbstractDFGFactor} return haskey(dfg.g.variables, node.label) diff --git a/src/GraphsDFG/services/GraphsDFGSerialization.jl b/src/GraphsDFG/services/GraphsDFGSerialization.jl index 51bc83a4..c62d0f94 100644 --- a/src/GraphsDFG/services/GraphsDFGSerialization.jl +++ b/src/GraphsDFG/services/GraphsDFGSerialization.jl @@ -14,8 +14,10 @@ using InteractiveUtils addHistory::Vector{Symbol} solverParams::T solverParams_type::String = string(typeof(solverParams)) - #TODO - # blobStores::Dict{Symbol, AbstractBlobStore} + # TODO remove Union.Nothing in DFG v0.24 + typePackedVariable::Union{Nothing,Bool} = false # Are variables packed or full + typePackedFactor::Union{Nothing,Bool} = false # Are factors packed or full + blobStores::Union{Nothing, Dict{Symbol, FolderStore{Vector{UInt8}}}} end StructTypes.StructType(::Type{PackedGraphsDFG}) = StructTypes.AbstractType() @@ -27,6 +29,9 @@ function StructTypes.subtypes(::Type{PackedGraphsDFG}) NamedTuple(map(s->Symbol(s)=>PackedGraphsDFG{s}, subs)) end +_variablestype(fg::GraphsDFG{<:AbstractParams,T,<:AbstractDFGFactor}) where T = T +_factorstype(fg::GraphsDFG{<:AbstractParams,<:AbstractDFGVariable, T}) where T = T + ## """ $(SIGNATURES) @@ -34,18 +39,50 @@ Packing function to serialize DFG metadata from. """ function packDFGMetadata(fg::GraphsDFG) commonfields = intersect(fieldnames(PackedGraphsDFG), fieldnames(GraphsDFG)) + + setdiff!(commonfields, [:blobStores]) + blobStores = Dict{Symbol, FolderStore{Vector{UInt8}}}() + foreach(values(fg.blobStores)) do store + if store isa FolderStore{Vector{UInt8}} + blobStores[store.key] = store + else + @warn "BlobStore $(store.key) of type $(typeof(store)) is not supported yet and will not be saved" + end + end + props = (k => getproperty(fg, k) for k in commonfields) - return PackedGraphsDFG(;props...) + return PackedGraphsDFG(; + typePackedVariable = _variablestype(fg) == Variable, + typePackedFactor = _factorstype(fg) == PackedFactor, + blobStores, + props... + ) end function unpackDFGMetadata(packed::PackedGraphsDFG) commonfields = intersect(fieldnames(GraphsDFG), fieldnames(PackedGraphsDFG)) + + #FIXME Deprecate remove in DFG v0.24 + setdiff!(commonfields, [:blobStores]) + blobStores = Dict{Symbol, AbstractBlobStore}() + !isnothing(packed.blobStores) && merge!(blobStores, packed.blobStores) + props = (k => getproperty(packed, k) for k in commonfields) - GraphsDFG(;props...) + + VT = isnothing(packed.typePackedVariable) || !packed.typePackedVariable ? DFGVariable : Variable + FT = isnothing(packed.typePackedFactor) || !packed.typePackedFactor ? DFGFactor : PackedFactor + # VT = isnothing(packed.typePackedVariable) || packed.typePackedVariable ? Variable : DFGVariable + # FT = isnothing(packed.typePackedFactor) || packed.typePackedFactor ? PackedFactor : DFGFactor + GraphsDFG{typeof(packed.solverParams), VT, FT}(;blobStores, props...) end function unpackDFGMetadata!(dfg::GraphsDFG, packed::PackedGraphsDFG) commonfields = intersect(fieldnames(GraphsDFG), fieldnames(PackedGraphsDFG)) + + #FIXME Deprecate remove Nothing union in DFG v0.24 + setdiff!(commonfields, [:blobStores]) + !isnothing(packed.blobStores) && merge!(dfg.blobStores, packed.blobStores) + props = (k => getproperty(packed, k) for k in commonfields) foreach(props) do (k,v) setproperty!(dfg, k, v) diff --git a/src/services/AbstractDFG.jl b/src/services/AbstractDFG.jl index a7ce3be1..d686230a 100644 --- a/src/services/AbstractDFG.jl +++ b/src/services/AbstractDFG.jl @@ -1266,7 +1266,7 @@ function mergeVariableData!(dfg::AbstractDFG, sourceVariable::AbstractDFGVariabl :solverDataDict in fieldnames(typeof(var)) && mergeVariableSolverData!(var, sourceVariable) #update if its not a InMemoryDFGTypes, otherwise it was a reference - # if satelite nodes are used it can be updated seprarately + # if satelite nodes are used it can be updated separately # !(isa(dfg, InMemoryDFGTypes)) && updateVariable!(dfg, var) return var diff --git a/src/services/DFGFactor.jl b/src/services/DFGFactor.jl index be1471ae..9bbe667d 100644 --- a/src/services/DFGFactor.jl +++ b/src/services/DFGFactor.jl @@ -118,6 +118,7 @@ Get the variable ordering for this factor. Should be equivalent to listNeighbors unless something was deleted in the graph. """ getVariableOrder(fct::DFGFactor) = fct._variableOrderSymbols::Vector{Symbol} +getVariableOrder(fct::PackedFactor) = fct._variableOrderSymbols::Vector{Symbol} getVariableOrder(dfg::AbstractDFG, fct::Symbol) = getVariableOrder(getFactor(dfg, fct)) ##------------------------------------------------------------------------------ diff --git a/src/services/DFGVariable.jl b/src/services/DFGVariable.jl index cc6ef095..f38faef4 100644 --- a/src/services/DFGVariable.jl +++ b/src/services/DFGVariable.jl @@ -450,6 +450,8 @@ Note: Rather use SmallData CRUD """ getSmallData(v::DFGVariable) = v.smallData +getSmallData(v::Variable) = JSON3.read(base64decode(v.metadata)) + """ $(SIGNATURES) Set the small data for a variable. diff --git a/src/services/Serialization.jl b/src/services/Serialization.jl index d7b21063..d3c2abd1 100644 --- a/src/services/Serialization.jl +++ b/src/services/Serialization.jl @@ -162,7 +162,7 @@ end ## Variable Packing and unpacking ##============================================================================== -function packVariable(v::AbstractDFGVariable; includePPEs::Bool=true, includeSolveData::Bool=true, includeDataEntries::Bool=true) +function packVariable(v::DFGVariable; includePPEs::Bool=true, includeSolveData::Bool=true, includeDataEntries::Bool=true) return PackedVariable(; id=v.id, label = v.label, @@ -177,7 +177,11 @@ function packVariable(v::AbstractDFGVariable; includePPEs::Bool=true, includeSol blobEntries = collect(values(v.dataDict)), _version = string(DFG._getDFGVersion())) end - + +function packVariable(v::Variable; includePPEs::Bool=true, includeSolveData::Bool=true, includeDataEntries::Bool=true) + return v +end + function unpackVariable(variable::PackedVariable; skipVersionCheck::Bool=false) !skipVersionCheck && _versionCheck(variable) @@ -205,8 +209,9 @@ function unpackVariable(variable::PackedVariable; skipVersionCheck::Bool=false) smallData=metadata, dataDict=dataDict, solvable=variable.solvable ) - end +end +DFGVariable(v::Variable) = unpackVariable(v) ##============================================================================== ## Factor Packing and unpacking @@ -250,6 +255,8 @@ function packFactor(f::DFGFactor) return props end +packFactor(f::PackedFactor) = f + function reconstFactorData end function decodePackedType(dfg::AbstractDFG, varOrder::AbstractVector{Symbol}, ::Type{T}, packeddata::GenericFunctionNodeData{PT}) where {T<:FactorOperationalMemory, PT}