diff --git a/Project.toml b/Project.toml index 208bcc4a..7eaa7e09 100644 --- a/Project.toml +++ b/Project.toml @@ -36,14 +36,14 @@ ComputationalResources = "0.3" Distributions = "0.25.3" InvertedIndices = "1" LossFunctions = "0.5, 0.6, 0.7, 0.8" -MLJModelInterface = "1.5" +MLJModelInterface = "1.6" Missings = "0.4, 1" OrderedCollections = "1.1" Parameters = "0.12" PrettyTables = "1" ProgressMeter = "1.7.1" ScientificTypes = "3" -StatisticalTraits = "3" +StatisticalTraits = "3.2" StatsBase = "0.32, 0.33" Tables = "0.2, 1.0" julia = "1.6" \ No newline at end of file diff --git a/src/composition/learning_networks/machines.jl b/src/composition/learning_networks/machines.jl index 266af003..2593fa8e 100644 --- a/src/composition/learning_networks/machines.jl +++ b/src/composition/learning_networks/machines.jl @@ -225,6 +225,22 @@ $DOC_SIGNATURES """ glb(mach::Machine{<:Union{Composite,Surrogate}}) = glb(mach.fitresult) +""" + report(fitresult::CompositeFitresult) + +Return a tuple combining the report from `fitresult.glb` (a `Node` report) with the +additions coming from nodes declared as report nodes in `fitresult.signature`, but without +merging the two. + +$DOC_SIGNATURES + +**Private method** +""" +function report(fitresult::CompositeFitresult) + basic = report(glb(fitresult)) + additions = _call(_report_part(signature(fitresult))) + return (; basic, additions) +end """ fit!(mach::Machine{<:Surrogate}; @@ -245,11 +261,10 @@ See also [`machine`](@ref) """ function fit!(mach::Machine{<:Surrogate}; kwargs...) - glb_node = glb(mach) - fit!(glb_node; kwargs...) + glb = MLJBase.glb(mach) + fit!(glb; kwargs...) mach.state += 1 - report_additions_ = _call(_report_part(signature(mach.fitresult))) - mach.report = merge(report(glb_node), report_additions_) + mach.report = MLJBase.report(mach.fitresult) return mach end @@ -347,7 +362,7 @@ the following: - Calls `fit!(mach, verbosity=verbosity, acceleration=acceleration)`. -- Records a copy of `model` in a variable called `cache`. +- Records (among other things) a copy of `model` in a variable called `cache` - Returns `cache` and outcomes of training in an appropriate form (specifically, `(mach.fitresult, cache, mach.report)`; see [Adding @@ -396,6 +411,7 @@ function return!(mach::Machine{<:Surrogate}, # record the current hyper-parameter values: old_model = deepcopy(model) + glb = MLJBase.glb(mach) cache = (; old_model) setfield!(mach.fitresult, @@ -647,9 +663,8 @@ function restore!(mach::Machine{<:Composite}) return mach end - -function setreport!(mach::Machine{<:Composite}, report) - basereport = MLJBase.report(glb(mach)) - report_additions = Base.structdiff(report, basereport) - mach.report = merge(basereport, report_additions) +function setreport!(copymach::Machine{<:Composite}, mach) + basic = report(glb(copymach.fitresult)) + additions = mach.report.additions + copymach.report = (; basic, additions) end diff --git a/src/composition/models/inspection.jl b/src/composition/models/inspection.jl index b4950629..b6aa7617 100644 --- a/src/composition/models/inspection.jl +++ b/src/composition/models/inspection.jl @@ -5,8 +5,9 @@ try_scalarize(v) = length(v) == 1 ? v[1] : v function machines_given_model_name(mach::Machine{M}) where M<:Composite network_model_names = getfield(mach.fitresult, :network_model_names) names = unique(filter(name->!(name === nothing), network_model_names)) - network_models = MLJBase.models(glb(mach)) - network_machines = MLJBase.machines(glb(mach)) + glb = MLJBase.glb(mach) + network_models = MLJBase.models(glb) + network_machines = MLJBase.machines(glb) ret = LittleDict{Symbol,Any}() for name in names mask = map(==(name), network_model_names) @@ -17,22 +18,27 @@ function machines_given_model_name(mach::Machine{M}) where M<:Composite return ret end -function tuple_keyed_on_model_names(item_given_machine, mach) +function tuple_keyed_on_model_names(machines, mach, f) dict = MLJBase.machines_given_model_name(mach) names = tuple(keys(dict)...) named_tuple_values = map(names) do name - [item_given_machine[m] for m in dict[name]] |> try_scalarize + [f(m) for m in dict[name]] |> try_scalarize end return NamedTuple{names}(named_tuple_values) end -function report(mach::Machine{<:Composite}) - dict = mach.report.report_given_machine - return merge(tuple_keyed_on_model_names(dict, mach), mach.report) +function report(mach::Machine{<:Union{Composite,Surrogate}}) + report_additions = mach.report.additions + report_basic = mach.report.basic + report_components = mach isa Machine{<:Surrogate} ? NamedTuple() : + MLJBase.tuple_keyed_on_model_names(report_basic.machines, mach, MLJBase.report) + return merge(report_components, report_basic, report_additions) end function fitted_params(mach::Machine{<:Composite}) - fp = fitted_params(mach.model, mach.fitresult) - dict = fp.fitted_params_given_machine - return merge(MLJBase.tuple_keyed_on_model_names(dict, mach), fp) + fp_basic = fitted_params(mach.model, mach.fitresult) + machines = fp_basic.machines + fp_components = + MLJBase.tuple_keyed_on_model_names(machines, mach, MLJBase.fitted_params) + return merge(fp_components, fp_basic) end diff --git a/src/composition/models/methods.jl b/src/composition/models/methods.jl index 880b377c..532f0a1a 100644 --- a/src/composition/models/methods.jl +++ b/src/composition/models/methods.jl @@ -31,24 +31,25 @@ function update(model::M, # underlying learning network machine. network_model_names = getfield(fitresult, :network_model_names) - old_model = cache.old_model - glb_node = glb(fitresult) # greatest lower bound + old_model = cache.old_model + glb = MLJBase.glb(fitresult) # greatest lower bound of network, a node - if fallback(model, old_model, network_model_names, glb_node) + if fallback(model, old_model, network_model_names, glb) return fit(model, verbosity, args...) end - fit!(glb_node; verbosity=verbosity) + fit!(glb; verbosity=verbosity) + # Retrieve additional report values - report_additions_ = _call(_report_part(signature(fitresult))) + report = MLJBase.report(fitresult) # record current model state: cache = (; old_model = deepcopy(model)) return (fitresult, cache, - merge(report(glb_node), report_additions_)) + report) end diff --git a/src/composition/models/pipelines.jl b/src/composition/models/pipelines.jl index 455712f2..d0e9088c 100644 --- a/src/composition/models/pipelines.jl +++ b/src/composition/models/pipelines.jl @@ -608,7 +608,7 @@ MMI.target_scitype(p::SupervisedPipeline) = target_scitype(supervised_component( # ## Training losses function MMI.training_losses(pipe::SupervisedPipeline, pipe_report) - mach = supervised(pipe_report.machines) + mach = supervised(pipe_report.basic.machines) _report = report(mach) return training_losses(mach.model, _report) end diff --git a/src/composition/models/transformed_target_model.jl b/src/composition/models/transformed_target_model.jl index 06599625..dd225e1e 100644 --- a/src/composition/models/transformed_target_model.jl +++ b/src/composition/models/transformed_target_model.jl @@ -224,7 +224,7 @@ end # # TRAINING LOSSES function training_losses(model::SomeTT, tt_report) - mach = first(tt_report.machines) + mach = first(tt_report.basic.machines) return training_losses(mach) end diff --git a/src/machines.jl b/src/machines.jl index f9827f33..f6781985 100644 --- a/src/machines.jl +++ b/src/machines.jl @@ -469,10 +469,11 @@ end # Not one, but *two*, fit methods are defined for machines here, # `fit!` and `fit_only!`. -# - `fit_only!`: trains a machine without touching the learned -# parameters (`fitresult`) of any other machine. It may error if -# another machine on which it depends (through its node training -# arguments `N1, N2, ...`) has not been trained. +# - `fit_only!`: trains a machine without touching the learned parameters (`fitresult`) of +# any other machine. It may error if another machine on which it depends (through its node +# training arguments `N1, N2, ...`) has not been trained. It's possible that a dependent +# machine `mach` may have it's report mutated if `reporting_operations(mach.model)` is +# non-empty. # - `fit!`: trains a machine after first progressively training all # machines on which the machine depends. Implicitly this involves @@ -909,13 +910,14 @@ function serializable(mach::Machine{<:Any, C}) where C setfield!(copymach, fieldname, ()) # Make fitresult ready for serialization elseif fieldname == :fitresult + # this `save` does the actual emptying of fields copymach.fitresult = save(mach.model, getfield(mach, fieldname)) else setfield!(copymach, fieldname, getfield(mach, fieldname)) end end - setreport!(copymach, mach.report) + setreport!(copymach, mach) return copymach end @@ -997,6 +999,8 @@ function save(file::Union{String,IO}, serialize(file, smach) end +setreport!(copymach, mach) = + setfield!(copymach, :report, mach.report) -setreport!(mach::Machine, report) = - setfield!(mach, :report, report) +# NOTE. there is also a specialization for `setreport!` for `Composite` models, defined in +# /src/composition/learning_networks/machines/ diff --git a/src/operations.jl b/src/operations.jl index d67cce06..188488d1 100644 --- a/src/operations.jl +++ b/src/operations.jl @@ -37,37 +37,59 @@ warn_serializable_mach(operation) = "The operation $operation has been called on "deserialised machine mach whose learned parameters "* "may be unusable. To be sure, first run restore!(mach)." +# Given return value `ret` of an operation with symbol `operation` (eg, `:predict`) return +# `ret` in the ordinary case that the operation does not include an "report" component ; +# otherwise update `mach.report` with that component and return the non-report part of +# `ret`: +function get!(ret, operation, mach) + if operation in reporting_operations(mach.model) + report = last(ret) + if isnothing(mach.report) || isempty(mach.report) + mach.report = report + else + mach.report = merge(mach.report, report) + end + return first(ret) + end + return ret +end + # 0. operations on machine, given rows=...: for operation in OPERATIONS - if operation != :inverse_transform + quoted_operation = QuoteNode(operation) # eg, :(:predict) - ex = quote - function $(operation)(mach::Machine{<:Model,false}; rows=:) - # catch deserialized machine with no data: - isempty(mach.args) && _err_serialized($operation) - return ($operation)(mach, mach.args[1](rows=rows)) - end - function $(operation)(mach::Machine{<:Model,true}; rows=:) - # catch deserialized machine with no data: - isempty(mach.args) && _err_serialized($operation) - model = mach.model - return ($operation)(model, - mach.fitresult, - selectrows(model, rows, mach.data[1])...) - end - end - eval(ex) + operation == :inverse_transform && continue + ex = quote + function $(operation)(mach::Machine{<:Model,false}; rows=:) + # catch deserialized machine with no data: + isempty(mach.args) && _err_serialized($operation) + ret = ($operation)(mach, mach.args[1](rows=rows)) + return get!(ret, $quoted_operation, mach) + end + function $(operation)(mach::Machine{<:Model,true}; rows=:) + # catch deserialized machine with no data: + isempty(mach.args) && _err_serialized($operation) + model = mach.model + ret = ($operation)( + model, + mach.fitresult, + selectrows(model, rows, mach.data[1])..., + ) + return get!(ret, $quoted_operation, mach) + end end + eval(ex) + end # special case of Static models (no training arguments): transform(mach::Machine{<:Static}; rows=:) = _err_rows_not_allowed() inverse_transform(mach::Machine; rows=:) = - throw(ArgumentError("`inverse_transform()(mach)` and "* + throw(ArgumentError("`inverse_transform(mach)` and "* "`inverse_transform(mach, rows=...)` are "* "not supported. Data or nodes "* "must be explictly specified, "* @@ -77,22 +99,32 @@ _symbol(f) = Base.Core.Typeof(f).name.mt.name for operation in OPERATIONS + quoted_operation = QuoteNode(operation) # eg, :(:predict) + ex = quote # 1. operations on machines, given *concrete* data: function $operation(mach::Machine, Xraw) if mach.state != 0 mach.state == -1 && @warn warn_serializable_mach($operation) - return $(operation)(mach.model, - mach.fitresult, - reformat(mach.model, Xraw)...) + ret = $(operation)( + mach.model, + mach.fitresult, + reformat(mach.model, Xraw)..., + ) + get!(ret, $quoted_operation, mach) else error("$mach has not been trained.") end end function $operation(mach::Machine{<:Static}, Xraw, Xraw_more...) - return $(operation)(mach.model, mach.fitresult, - Xraw, Xraw_more...) + ret = $(operation)( + mach.model, + mach.fitresult, + Xraw, + Xraw_more..., + ) + get!(ret, $quoted_operation, mach) end # 2. operations on machines, given *dynamic* data (nodes): diff --git a/test/composition/learning_networks/machines.jl b/test/composition/learning_networks/machines.jl index 85c962d6..128c4a89 100644 --- a/test/composition/learning_networks/machines.jl +++ b/test/composition/learning_networks/machines.jl @@ -88,7 +88,6 @@ end @test Θ.transform == Wout Θ.report.some_stuff == rnode @test report(mach).some_stuff == :stuff - @test report(mach).machines == fitted_params(mach).machines # supervised @@ -281,7 +280,7 @@ end end # Testing extra report field : it is a deepcopy - @test smach.report.cv_report === mach.report.cv_report + @test report(smach).cv_report === report(mach).cv_report @test smach.fitresult isa MLJBase.CompositeFitresult @@ -356,7 +355,8 @@ end metalearner = FooBarRegressor(lambda=1.), resampling = dcv, model_1 = DeterministicConstantRegressor(), - model_2=ConstantRegressor()) + model_2=ConstantRegressor() + ) filesizes = [] for n in [100, 500, 1000] diff --git a/test/composition/models/pipelines.jl b/test/composition/models/pipelines.jl index b4e768e3..5bade4f4 100644 --- a/test/composition/models/pipelines.jl +++ b/test/composition/models/pipelines.jl @@ -53,12 +53,20 @@ end mutable struct StaticKefir <: Static alpha::Float64 # non-zero to be invertible end +MLJBase.reporting_operations(::Type{<:StaticKefir}) = (:transform, :inverse_transform) -# for `alpha != 0` the map `x -> kefir(x, alpha) is invertible: +# piece-wise linear function that is linear only for `alpha=1`: kefir(x, alpha) = x > 0 ? x * alpha : x / alpha -MLJBase.transform(model::StaticKefir, _, X) = broadcast(kefir, model.alpha, X) -MLJBase.inverse_transform(model::StaticKefir, _, W) = broadcast(kefir, 1/(model.alpha), W) +MLJBase.transform(model::StaticKefir, _, X) = ( + broadcast(kefir, X, model.alpha), + (; first = first(X)), +) + +MLJBase.inverse_transform(model::StaticKefir, _, W) = ( + broadcast(kefir, W, 1/(model.alpha)), + (; last = last(W)), +) d = MyDeterministic(:d) p = MyProbabilistic(:p) @@ -597,7 +605,26 @@ end @test transform(mach, inverse_transform(mach, X)) ≈ X end +@testset "accessing reports generated by Static models" begin + X = Float64[4, 5, 6] + pipe = UnivariateStandardizer() |> StaticKefir(3) + mach = machine(pipe, X) + fit!(mach, verbosity=0) + r = report(mach).static_kefir + @test isnothing(r) || isempty(r) # tranform has not been called yet + transform(mach, X) # adds to report of mach, ie mutates mach + r = report(mach).static_kefir + @test report(mach).static_kefir.first == -1 + transform(mach, [5, 6]) # mutates `mach` + r = report(mach).static_kefir + @test keys(r) == (:first, ) + @test r.first == 0 + inverse_transform(mach, [1, 2, 3]) + r = report(mach).static_kefir + @test r.first == 0.0 + @test r.last == 3 end -true +end # module +true diff --git a/test/composition/models/static_transformers.jl b/test/composition/models/static_transformers.jl index 5796bdcf..0236f3a6 100644 --- a/test/composition/models/static_transformers.jl +++ b/test/composition/models/static_transformers.jl @@ -9,18 +9,35 @@ import Random.seed! seed!(1234) +struct PlainTransformer <: Static + ftr::Symbol +end + +MLJBase.transform(transf::PlainTransformer, verbosity, X) = + selectcols(X, transf.ftr) + +@testset "nodal machine constructor for static transformers" begin + X = (x1=rand(3), x2=[1, 2, 3]); + mach = machine(PlainTransformer(:x2)) + @test transform(mach, X) == [1, 2, 3] +end + struct YourTransformer <: Static ftr::Symbol end +MLJBase.reporting_operations(::Type{<:YourTransformer}) = (:transform,) +# returns `(output, report)`: MLJBase.transform(transf::YourTransformer, verbosity, X) = - selectcols(X, transf.ftr) + (selectcols(X, transf.ftr), (; nrows=nrows(X))) @testset "nodal machine constructor for static transformers" begin X = (x1=rand(3), x2=[1, 2, 3]); mach = machine(YourTransformer(:x2)) - fit!(mach, verbosity=0) @test transform(mach, X) == [1, 2, 3] + @test report(mach).nrows == 3 + transform(mach, (x2=["a", "b"],)) + @test report(mach).nrows == 2 end x1 = rand(30) diff --git a/test/machines.jl b/test/machines.jl index 5438c3c9..62abc607 100644 --- a/test/machines.jl +++ b/test/machines.jl @@ -431,17 +431,6 @@ end rm(filename) end -@testset "Test Misc functions used in `serializable`" begin - X, y = make_regression(100, 1) - mach = machine(DeterministicConstantRegressor(), X, y) - fit!(mach, verbosity=0) - # setreport! default - @test mach.report isa NamedTuple - MLJBase.setreport!(mach, "toto") - @test mach.report == "toto" -end - - end # module true