From a2714a4bc831c914a7ff2831e2e48e1f809523ba Mon Sep 17 00:00:00 2001 From: jeremie Date: Sat, 6 Apr 2024 11:03:47 -0400 Subject: [PATCH 1/9] tablesAPI --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 54d549e..6ae48cd 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,5 @@ docs/site/ # environment. Manifest.toml -/data +data/ +.vscode/ From cbb5f8906db1c80fcc6832fe4c176ab3ce920776 Mon Sep 17 00:00:00 2001 From: jeremie Date: Sun, 7 Apr 2024 23:01:45 -0400 Subject: [PATCH 2/9] up --- Project.toml | 3 +- experiments/aicrowd-test.jl | 98 ++++++++++------------ src/EvoLinear.jl | 2 +- src/callback.jl | 19 ++--- src/linear/Linear.jl | 5 +- src/linear/fit.jl | 152 ++++++++++++++++++++-------------- src/linear/structs.jl | 57 ++++++++----- src/{metric.jl => metrics.jl} | 0 src/splines/Splines.jl | 2 +- src/splines/fit.jl | 51 ++++++------ 10 files changed, 210 insertions(+), 179 deletions(-) rename src/{metric.jl => metrics.jl} (100%) diff --git a/Project.toml b/Project.toml index 09dfdcd..d61aa2f 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "EvoLinear" uuid = "ab853011-1780-437f-b4b5-5de6f4777246" authors = ["jeremie and contributors"] -version = "0.4.3" +version = "0.5.0" [deps] Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" @@ -14,6 +14,7 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" [compat] Distributions = "0.25" diff --git a/experiments/aicrowd-test.jl b/experiments/aicrowd-test.jl index 3c91ff0..19b7714 100644 --- a/experiments/aicrowd-test.jl +++ b/experiments/aicrowd-test.jl @@ -1,4 +1,3 @@ -using Revise using CSV using DataFrames using EvoLinear @@ -9,9 +8,9 @@ using Random: seed! using AWS: AWSCredentials, AWSConfig, @service @service S3 aws_creds = AWSCredentials(ENV["AWS_ACCESS_KEY_ID_JDB"], ENV["AWS_SECRET_ACCESS_KEY_JDB"]) -aws_config = AWSConfig(; creds = aws_creds, region = "ca-central-1") +aws_config = AWSConfig(; creds=aws_creds, region="ca-central-1") -path = "share/data/insurance-aicrowd.csv" +path = "share/data/aicrowd/insurance-aicrowd.csv" raw = S3.get_object( "jeremiedb", path, @@ -19,10 +18,9 @@ raw = S3.get_object( aws_config, ) df = DataFrame(CSV.File(raw)) -transform!(df, "claim_amount" => ByRow(x -> x > 0 ? 1.0f0 : 0.0f0) => "event") -target = "event" -feats = [ +target_name = "event" +feature_names = [ "vh_age", "vh_value", "vh_speed", @@ -34,57 +32,49 @@ feats = [ "pol_sit_duration", ] +transform!(df, "claim_amount" => ByRow(x -> x > 0 ? 1.0f0 : 0.0f0) => "event") + pol_cov_dict = Dict{String,Float64}("Min" => 1, "Med1" => 2, "Med2" => 3, "Max" => 4) pol_cov_map(x) = get(pol_cov_dict, x, 4) transform!(df, "pol_coverage" => ByRow(pol_cov_map) => "pol_coverage") -setdiff(feats, names(df)) - seed!(123) nobs = nrow(df) -id_train = sample(1:nobs, Int(round(0.8 * nobs)), replace = false) +id_train = sample(1:nobs, Int(round(0.8 * nobs)), replace=false) -df_train = dropmissing(df[id_train, [feats..., target]]) -df_eval = dropmissing(df[Not(id_train), [feats..., target]]) - -x_train = Matrix{Float32}(df_train[:, feats]) -x_eval = Matrix{Float32}(df_eval[:, feats]) -y_train = Vector{Float32}(df_train[:, target]) -y_eval = Vector{Float32}(df_eval[:, target]) +dtrain = dropmissing(df[id_train, [feature_names..., target_name]]) +deval = dropmissing(df[Not(id_train), [feature_names..., target_name]]) config = EvoLinearRegressor( - T = Float32, - loss = :logistic, - L1 = 0.0, - L2 = 0.0, - nrounds = 1000, - eta = 0.2, + loss=:logistic, + L1=0.0, + L2=0.0, + nrounds=1000, + eta=0.2, ) # @time m = fit_evotree(config; x_train, y_train, print_every_n=25); -@time m, logger = EvoLinear.fit( - config; - x_train, - y_train, - x_eval, - y_eval, - early_stopping_rounds = 100, - print_every_n = 10, - metric = :logloss, - return_logger = true, +@time m = EvoLinear.fit(config, dtrain; + feature_names, + target_name, + deval, + metric=:logloss, + early_stopping_rounds=100, + print_every_n=10, ); -p_linear = m(x_eval); -EvoLinear.Metrics.logloss(p_linear, y_eval) + +p_linear = m(deval); +EvoLinear.Metrics.logloss(p_linear, deval[:, target_name]) config = EvoSplineRegressor( - T = Float32, - loss = :logistic, - nrounds = 600, - eta = 1e-3, - knots = Dict(1 => 4, 2 => 4, 3 => 4, 4 => 4, 5 => 4, 6 => 4, 7 => 4, 8 => 4, 9 => 4), - act = :elu, - batchsize = 4096, - device = :cpu, + T=Float32, + loss=:logistic, + nrounds=600, + eta=1e-3, + knots=Dict(1 => 4, 2 => 4, 3 => 4, 4 => 4, 5 => 4, 6 => 4, 7 => 4, 8 => 4, 9 => 4), + act=:elu, + batchsize=4096, + device=:cpu, ) @time m, logger = EvoLinear.fit( config; @@ -92,10 +82,10 @@ config = EvoSplineRegressor( y_train, x_eval, y_eval, - early_stopping_rounds = 50, - print_every_n = 10, - metric = :logloss, - return_logger = true, + early_stopping_rounds=50, + print_every_n=10, + metric=:logloss, + return_logger=true, ); # @time m = EvoLinear.fit(config; x_train, y_train); p_spline = m(x_eval') @@ -129,9 +119,9 @@ watchlist = Dict("eval" => DMatrix(x_eval, y_eval)) dtrain; watchlist, num_round, - nthread = nthread, - verbosity = 0, - eval_metric = metric_xgb, + nthread=nthread, + verbosity=0, + eval_metric=metric_xgb, params_xgb..., ); p_xgb_tree = XGBoost.predict(m_xgb, x_eval) @@ -155,11 +145,11 @@ metrics = ["logloss"] @time m_xgb = xgboost( x_train, nrounds, - label = y_train, - param = params_xgb, - metrics = metrics, - nthread = nthread, - silent = 1, + label=y_train, + param=params_xgb, + metrics=metrics, + nthread=nthread, + silent=1, ); p_xgb_linear = XGBoost.predict(m_xgb, x_eval) diff --git a/src/EvoLinear.jl b/src/EvoLinear.jl index 25269a4..e520a77 100644 --- a/src/EvoLinear.jl +++ b/src/EvoLinear.jl @@ -9,7 +9,7 @@ import MLJModelInterface: fit, update, predict, schema export EvoLinearRegressor, EvoSplineRegressor include("utils.jl") -include("metric.jl") +include("metrics.jl") include("callback.jl") include("losses.jl") diff --git a/src/callback.jl b/src/callback.jl index 4c456d2..4260abf 100644 --- a/src/callback.jl +++ b/src/callback.jl @@ -1,32 +1,25 @@ - module CallBacks using ..EvoLinear.Metrics export init_logger, update_logger! -struct CallBackLinear{F,M,V,Y} +struct CallBack{F,M,V,Y} feval::F x::M p::V y::Y w::V end -function (cb::CallBackLinear)(logger, iter, m) - m(cb.p, cb.x; proj = true) - metric = cb.feval(cb.p, cb.y, cb.w) - update_logger!(logger, iter, metric) - return nothing -end -function init_logger(; T, metric, maximise, early_stopping_rounds) +function init_logger(; metric, maximise, early_stopping_rounds) logger = Dict( :name => String(metric), :maximise => maximise, :early_stopping_rounds => early_stopping_rounds, :nrounds => 0, :iter => Int[], - :metrics => T[], + :metrics => Float32[], :iter_since_best => 0, :best_iter => 0, :best_metric => 0.0, @@ -34,7 +27,11 @@ function init_logger(; T, metric, maximise, early_stopping_rounds) return logger end -function update_logger!(logger, iter, metric) +function update_logger!(logger, m, cb, iter) + + m(cb.p, cb.x; proj=true) + metric = cb.feval(cb.p, cb.y, cb.w) + logger[:nrounds] = iter push!(logger[:iter], iter) push!(logger[:metrics], metric) diff --git a/src/linear/Linear.jl b/src/linear/Linear.jl index 79b10f6..2099935 100644 --- a/src/linear/Linear.jl +++ b/src/linear/Linear.jl @@ -6,12 +6,13 @@ using ..EvoLinear: sigmoid, logit, mk_rng using ..EvoLinear.Metrics using ..EvoLinear.Losses using ..EvoLinear.CallBacks -import ..EvoLinear.CallBacks: CallBackLinear +import ..EvoLinear.CallBacks: CallBack using Base.Threads: @threads # using StatsBase using Statistics: mean, std using LoopVectorization +import Tables import MLJModelInterface as MMI import MLJModelInterface: fit, update, predict, schema @@ -20,7 +21,7 @@ export EvoLinearRegressor, EvoLinearModel, init, fit!, get_loss_type include("structs.jl") include("loss.jl") -include("predict.jl") +# include("predict.jl") include("fit.jl") end \ No newline at end of file diff --git a/src/linear/fit.jl b/src/linear/fit.jl index 3d469c0..850b1a8 100644 --- a/src/linear/fit.jl +++ b/src/linear/fit.jl @@ -1,25 +1,42 @@ -function init(config::EvoLinearRegressor{L,T}, x, y; w = nothing) where {L,T} - cache = init_cache(config, x, y; w) - m = EvoLinearModel(L; coef = zeros(T, size(x, 2)), bias = zero(T)) - return m, cache -end +function init(config::EvoLinearRegressor{L}, dtrain; feature_names, target_name, weight_name=nothing) where {L} + + T = Float32 + nobs = Tables.DataAPI.nrow(dtrain) + nfeats = length(feature_names) + + x = zeros(T, nobs, nfeats) + @threads for j in axes(x, 2) + @views x[:, j] .= Tables.getcolumn(dtrain, feature_names[j]) + end + + y = Tables.getcolumn(dtrain, target_name) + y = convert(Vector{T}, y) + + w = isnothing(weight_name) ? ones(T, nobs) : convert(Vector{T}, Tables.getcolumn(dtrain, weight_name)) + ∑w = sum(w) -function init_cache(::EvoLinearRegressor{L,T}, x, y; w = nothing) where {L,T} ∇¹, ∇² = zeros(T, size(x, 2)), zeros(T, size(x, 2)) ∇b = zeros(T, 2) - w = isnothing(w) ? ones(T, size(y)) : convert(Vector{T}, w) - ∑w = sum(w) + + info = Dict( + :nrounds => 0, + :feature_names => feature_names, + :target_name => target_name, + :weight_name => weight_name) + cache = ( - ∇¹ = ∇¹, - ∇² = ∇², - ∇b = ∇b, - x = convert(Matrix{T}, x), - y = convert(Vector{T}, y), - w = w, - ∑w = ∑w, - info = Dict(:nrounds => 0), + x=x, + y=y, + w=w, + ∑w=∑w, + ∇¹=∇¹, + ∇²=∇², + ∇b=∇b, ) - return cache + + m = EvoLinearModel(L; coef=zeros(T, size(x, 2)), bias=zero(T), info=info) + + return m, cache end @@ -48,54 +65,52 @@ Provided a `config`, `EvoLinear.fit` takes `x` and `y` as features and target in - `:tweedie_deviance` """ function fit( - config::EvoLinearRegressor{L,T}; - x_train, - y_train, - w_train = nothing, - x_eval = nothing, - y_eval = nothing, - w_eval = nothing, - metric = nothing, - print_every_n = 9999, - early_stopping_rounds = 9999, - verbosity = 1, - fnames = nothing, - return_logger = false, -) where {L,T} - - m, cache = init(config, x_train, y_train; w = w_train) - + config::EvoLinearRegressor, + dtrain; + feature_names, + target_name, + weight_name=nothing, + deval=nothing, + metric=nothing, + print_every_n=9999, + early_stopping_rounds=9999, + verbosity=1 +) + + feature_names = Symbol.(feature_names) + target_name = Symbol(target_name) + weight_name = isnothing(weight_name) ? nothing : Symbol(weight_name) logger = nothing - if !isnothing(metric) && !isnothing(x_eval) && !isnothing(y_eval) - cb = CallBackLinear(config; metric, x_eval, y_eval, w_eval) + + m, cache = init(config, dtrain; feature_names, target_name, weight_name) + + if !isnothing(metric) && !isnothing(deval) + cb = CallBack(config, deval; metric, feature_names, target_name, weight_name) logger = init_logger(; - T, metric, - maximise = is_maximise(cb.feval), + maximise=is_maximise(cb.feval), early_stopping_rounds, ) - cb(logger, 0, m) + update_logger!(logger, m, cb, 0) (verbosity > 0) && @info "initialization" metric = logger[:metrics][end] end for iter = 1:config.nrounds fit!(m, cache, config) if !isnothing(logger) - cb(logger, iter, m) + update_logger!(logger, m, cb, iter) if iter % print_every_n == 0 && verbosity > 0 @info "iter $iter" metric = logger[:metrics][end] end (logger[:iter_since_best] >= logger[:early_stopping_rounds]) && break end end - if return_logger - return (m, logger) - else - return m - end + + m.info[:logger] = logger + return m end -function fit!(m::EvoLinearModel{L}, cache, config::EvoLinearRegressor{L,T}) where {L,T} +function fit!(m::EvoLinearModel, cache, config::EvoLinearRegressor) ∇¹, ∇², ∇b = cache.∇¹ .* 0, cache.∇² .* 0, cache.∇b .* 0 x, y, w = cache.x, cache.y, cache.w @@ -103,17 +118,17 @@ function fit!(m::EvoLinearModel{L}, cache, config::EvoLinearRegressor{L,T}) wher if config.updater == :all # update all coefs then bias - p = m(x; proj = true) - update_∇_bias!(L, ∇b, x, y, p, w) + p = m(x; proj=true) + update_∇_bias!(m.loss, ∇b, x, y, p, w) update_bias!(m, ∇b) - p = m(x; proj = true) - update_∇!(L, ∇¹, ∇², x, y, p, w) + p = m(x; proj=true) + update_∇!(m.loss, ∇¹, ∇², x, y, p, w) update_coef!(m, ∇¹, ∇², ∑w, config) else @error "invalid updater" end - cache[:info][:nrounds] += 1 + m.info[:nrounds] += 1 return nothing end @@ -128,17 +143,28 @@ function update_bias!(m, ∇b) return nothing end -function CallBackLinear( - ::EvoLinearRegressor{L,T}; +function CallBack( + config::EvoLinearRegressor, + deval; metric, - x_eval, - y_eval, - w_eval = nothing, -) where {L,T} + feature_names, + target_name, + weight_name=nothing +) + T = Float32 + nobs = Tables.DataAPI.nrow(deval) + nfeats = length(feature_names) feval = metric_dict[metric] - x = convert(Matrix{T}, x_eval) - p = zeros(T, length(y_eval)) - y = convert(Vector{T}, y_eval) - w = isnothing(w_eval) ? ones(T, size(y)) : convert(Vector{T}, w_eval) - return CallBackLinear(feval, x, p, y, w) -end \ No newline at end of file + + x = zeros(T, nobs, nfeats) + @threads for j in axes(x, 2) + @views x[:, j] .= Tables.getcolumn(deval, feature_names[j]) + end + y = Tables.getcolumn(deval, target_name) + y = convert(Vector{T}, y) + p = zero(y) + + w = isnothing(weight_name) ? ones(T, nobs) : convert(Vector{T}, Tables.getcolumn(deval, weight_name)) + + return CallBack(feval, x, p, y, w) +end diff --git a/src/linear/structs.jl b/src/linear/structs.jl index 43fa1d1..a34e60b 100644 --- a/src/linear/structs.jl +++ b/src/linear/structs.jl @@ -1,11 +1,10 @@ -mutable struct EvoLinearRegressor{L,T} <: MMI.Deterministic +mutable struct EvoLinearRegressor{L} <: MMI.Deterministic updater::Symbol nrounds::Int - eta::T - L1::T - L2::T + eta::Float32 + L1::Float32 + L2::Float32 rng - device::Symbol end @@ -115,9 +114,7 @@ function EvoLinearRegressor(; kwargs...) :eta => 1, :L1 => 0, :L2 => 0, - :rng => 123, - :device => :cpu, - :T => Float32 + :rng => 123 ) args_ignored = setdiff(keys(kwargs), keys(args)) @@ -134,29 +131,29 @@ function EvoLinearRegressor(; kwargs...) end args[:rng] = mk_rng(args[:rng]) - T = args[:T] L = loss_types[args[:loss]] - model = EvoLinearRegressor{L,T}( + model = EvoLinearRegressor{L}( args[:updater], args[:nrounds], - T(args[:eta]), - T(args[:L1]), - T(args[:L2]), - args[:rng], - args[:device]) + args[:eta], + args[:L1], + args[:L2], + args[:rng]) return model end -mutable struct EvoLinearModel{L<:Loss,C,B} - coef::C +mutable struct EvoLinearModel{L<:Loss,A,B} + loss::Type{L} + coef::A bias::B + info::Dict{Symbol,Any} end -EvoLinearModel(loss::Type; coef, bias) = EvoLinearModel{loss,typeof(coef),eltype(coef)}(coef, bias) -EvoLinearModel(loss::Symbol; coef, bias) = EvoLinearModel(loss_types[loss]; coef, bias) -get_loss_type(::EvoLinearModel{L,C,B}) where {L,C,B} = L -get_loss_type(::EvoLinearRegressor{L,T}) where {L,T} = L +EvoLinearModel(loss::Type{<:Loss}; coef, bias, info) = EvoLinearModel(loss, coef, bias, info) +EvoLinearModel(loss::Symbol; coef, bias, info) = EvoLinearModel(loss_types[loss]; coef, bias, info) +get_loss_type(m::EvoLinearModel) = m.loss +get_loss_type(::EvoLinearRegressor{L}) where {L} = L function (m::EvoLinearModel{L})(x::AbstractMatrix; proj::Bool=true) where {L} p = x * m.coef .+ m.bias @@ -168,6 +165,24 @@ function (m::EvoLinearModel{L})(p::AbstractVector, x::AbstractMatrix; proj::Bool proj ? proj!(L, p) : nothing return nothing end +function (m::EvoLinearModel{L})(data; proj::Bool=true) where {L} + + Tables.istable(data) || error("data must be Table compatible") + + T = Float32 + feature_names = m.info[:feature_names] + nobs = Tables.DataAPI.nrow(data) + nfeats = length(feature_names) + + x = zeros(T, nobs, nfeats) + @threads for j in axes(x, 2) + @views x[:, j] .= Tables.getcolumn(data, feature_names[j]) + end + + p = x * m.coef .+ m.bias + proj ? proj!(L, p) : nothing + return p +end function proj!(::L, p) where {L<:Type{MSE}} return nothing diff --git a/src/metric.jl b/src/metrics.jl similarity index 100% rename from src/metric.jl rename to src/metrics.jl diff --git a/src/splines/Splines.jl b/src/splines/Splines.jl index 5bc6b67..6c61ef4 100644 --- a/src/splines/Splines.jl +++ b/src/splines/Splines.jl @@ -6,7 +6,7 @@ using ..EvoLinear: sigmoid, logit, mk_rng using ..EvoLinear.Metrics using ..EvoLinear.Losses using ..EvoLinear.CallBacks -import ..EvoLinear.CallBacks: CallBackLinear +import ..EvoLinear.CallBacks: CallBack import ..EvoLinear: init, fit!, get_loss_type import MLJModelInterface as MMI diff --git a/src/splines/fit.jl b/src/splines/fit.jl index f4856af..a906235 100644 --- a/src/splines/fit.jl +++ b/src/splines/fit.jl @@ -1,20 +1,20 @@ -function init(config::EvoSplineRegressor{L,T}, x, y; w = nothing) where {L,T} +function init(config::EvoSplineRegressor{L,T}, x, y; w=nothing) where {L,T} @info "starting spline" device = config.device == :cpu ? Flux.cpu : Flux.gpu nfeats = size(x, 2) dtrain = DataLoader( - (x = Matrix{T}(x') |> device, y = T.(y) |> device), - batchsize = config.batchsize, + (x=Matrix{T}(x') |> device, y=T.(y) |> device), + batchsize=config.batchsize, ) loss = loss_fn[L] - m = EvoSplineModel(config; nfeats, mean = mean(y)) |> device + m = EvoSplineModel(config; nfeats, mean=mean(y)) |> device opt = Optimisers.NAdam(config.eta) opts = Optimisers.setup(opt, m) - cache = (dtrain = dtrain, loss = loss, opts = opts, info = Dict(:nrounds => 0)) + cache = (dtrain=dtrain, loss=loss, opts=opts, info=Dict(:nrounds => 0)) return m, cache end @@ -28,27 +28,27 @@ function fit( config::EvoSplineRegressor{L,T}; x_train, y_train, - w_train = nothing, - x_eval = nothing, - y_eval = nothing, - w_eval = nothing, - metric = nothing, - print_every_n = 9999, - early_stopping_rounds = 9999, - verbosity = 1, - fnames = nothing, - return_logger = false, + w_train=nothing, + x_eval=nothing, + y_eval=nothing, + w_eval=nothing, + metric=nothing, + print_every_n=9999, + early_stopping_rounds=9999, + verbosity=1, + fnames=nothing, + return_logger=false, ) where {L,T} - m, cache = init(config, x_train, y_train; w = w_train) - + m, cache = init(config, x_train, y_train; w=w_train) logger = nothing + if !isnothing(metric) && !isnothing(x_eval) && !isnothing(y_eval) - cb = CallBackLinear(config; metric, x_eval, y_eval, w_eval) + cb = CallBack(config; metric, x_eval, y_eval, w_eval) logger = init_logger(; T, metric, - maximise = is_maximise(cb.feval), + maximise=is_maximise(cb.feval), early_stopping_rounds, ) cb(logger, 0, m) @@ -74,25 +74,26 @@ end function fit!(m, cache) for d in cache[:dtrain] - grads = gradient(model -> cache[:loss](model(d[:x]; proj = false), d[:y]), m)[1] + grads = gradient(model -> cache[:loss](model(d[:x]; proj=false), d[:y]), m)[1] Optimisers.update!(cache[:opts], m, grads) end cache[:info][:nrounds] += 1 return nothing end -function CallBackLinear( - config::EvoSplineRegressor{L,T}; +function CallBack( + config::EvoSplineRegressor; metric, x_eval, y_eval, - w_eval = nothing, -) where {L,T} + w_eval=nothing, +) + T = Flota32 device = config.device == :cpu ? Flux.cpu : Flux.gpu feval = metric_dict[metric] x = convert(Matrix{T}, x_eval') p = zeros(T, length(y_eval)) y = convert(Vector{T}, y_eval) w = isnothing(w_eval) ? ones(T, size(y)) : convert(Vector{T}, w_eval) - return CallBackLinear(feval, x |> device, p |> device, y |> device, w |> device) + return CallBack(feval, x |> device, p |> device, y |> device, w |> device) end From fbc76592a89e5e249b88cbc4b25a67fe08ec4aed Mon Sep 17 00:00:00 2001 From: jeremie Date: Tue, 30 Apr 2024 16:58:56 -0400 Subject: [PATCH 3/9] up --- experiments/aicrowd-test.jl | 41 +++++++++++++++++++++---------------- src/linear/loss.jl | 12 +++++------ src/losses.jl | 14 ++++++------- 3 files changed, 36 insertions(+), 31 deletions(-) diff --git a/experiments/aicrowd-test.jl b/experiments/aicrowd-test.jl index 19b7714..c778a57 100644 --- a/experiments/aicrowd-test.jl +++ b/experiments/aicrowd-test.jl @@ -50,7 +50,7 @@ config = EvoLinearRegressor( L1=0.0, L2=0.0, nrounds=1000, - eta=0.2, + eta=0.5, ) # @time m = fit_evotree(config; x_train, y_train, print_every_n=25); @@ -92,12 +92,22 @@ p_spline = m(x_eval') # p_spline = m(x_eval' |> EvoLinear.Splines.gpu) |> EvoLinear.Splines.cpu EvoLinear.Metrics.logloss(p_spline, y_eval) + +###################################################### +# XGBoost +###################################################### +x_train = Matrix(dtrain[:, feature_names]) +y_train = dtrain[:, target_name] + +x_eval = Matrix(deval[:, feature_names]) +y_eval = deval[:, target_name] + params_xgb = Dict( :objective => "reg:logistic", :booster => "gbtree", :eta => 0.05, :max_depth => 4, - :lambda => 10.0, + :lambda => 1.0, :gamma => 0.0, :subsample => 0.5, :colsample_bytree => 0.8, @@ -113,15 +123,15 @@ num_round = 250 metric_xgb = "logloss" @info "xgboost train:" -dtrain = DMatrix(x_train, y_train) watchlist = Dict("eval" => DMatrix(x_eval, y_eval)) @time m_xgb = xgboost( - dtrain; + (x_train, y_train); watchlist, num_round, - nthread=nthread, + nthread, verbosity=0, eval_metric=metric_xgb, + early_stopping_rounds=10, params_xgb..., ); p_xgb_tree = XGBoost.predict(m_xgb, x_eval) @@ -135,21 +145,16 @@ params_xgb = Dict( :print_every_n => 5, ) -nthread = Threads.nthreads() -nthread = 8 - -nrounds = 250 -metrics = ["logloss"] - @info "xgboost train:" @time m_xgb = xgboost( - x_train, - nrounds, - label=y_train, - param=params_xgb, - metrics=metrics, - nthread=nthread, - silent=1, + (x_train, y_train); + watchlist, + num_round, + eval_metric=metric_xgb, + early_stopping_rounds=10, + nthread, + verbosity=0, + params_xgb..., ); p_xgb_linear = XGBoost.predict(m_xgb, x_eval) diff --git a/src/linear/loss.jl b/src/linear/loss.jl index 4a2ca19..a0f4534 100644 --- a/src/linear/loss.jl +++ b/src/linear/loss.jl @@ -74,7 +74,7 @@ end # The prediction p is assumed to be on the projected basis (exp(pred_linear)) # Derivative is w.r.t to β on the linear basis ################################### -function update_∇!(::Type{Poisson}, ∇¹, ∇², x, y, p, w, feat) +function update_∇!(::Type{PoissonDev}, ∇¹, ∇², x, y, p, w, feat) ∇1, ∇2 = zero(eltype(p)), zero(eltype(p)) @turbo for i in axes(x, 1) ∇1 += 2 * (p[i] - y[i]) * x[i, feat] * w[i] @@ -84,7 +84,7 @@ function update_∇!(::Type{Poisson}, ∇¹, ∇², x, y, p, w, feat) ∇²[feat] = ∇2 return nothing end -function update_∇_bias!(::Type{Poisson}, ∇_bias, x, y, p, w) +function update_∇_bias!(::Type{PoissonDev}, ∇_bias, x, y, p, w) ∇1, ∇2 = zero(eltype(p)), zero(eltype(p)) @turbo for i in axes(x, 1) ∇1 += 2 * (p[i] - y[i]) * w[i] @@ -102,7 +102,7 @@ end # The prediction p is assumed to be on the projected basis (exp(pred_linear)) # Derivative is w.r.t to β on the linear basis ################################### -function update_∇!(::Type{Gamma}, ∇¹, ∇², x, y, p, w, feat) +function update_∇!(::Type{GammaDev}, ∇¹, ∇², x, y, p, w, feat) ∇1, ∇2 = zero(eltype(p)), zero(eltype(p)) @turbo for i in axes(x, 1) ∇1 += 2 * (1 - y[i] / p[i]) * x[i, feat] * w[i] @@ -112,7 +112,7 @@ function update_∇!(::Type{Gamma}, ∇¹, ∇², x, y, p, w, feat) ∇²[feat] = ∇2 return nothing end -function update_∇_bias!(::Type{Gamma}, ∇_bias, x, y, p, w) +function update_∇_bias!(::Type{GammaDev}, ∇_bias, x, y, p, w) ∇1, ∇2 = zero(eltype(p)), zero(eltype(p)) @turbo for i in axes(x, 1) ∇1 += 2 * (1 - y[i] / p[i]) * w[i] @@ -130,7 +130,7 @@ end # The prediction p is assumed to be on the projected basis (exp(pred_linear)) # Derivative is w.r.t to β on the linear basis ################################### -function update_∇!(::Type{Tweedie}, ∇¹, ∇², x, y, p, w, feat) +function update_∇!(::Type{TweedieDev}, ∇¹, ∇², x, y, p, w, feat) rho = eltype(p)(1.5) ∇1, ∇2 = zero(eltype(p)), zero(eltype(p)) @turbo for i in axes(x, 1) @@ -141,7 +141,7 @@ function update_∇!(::Type{Tweedie}, ∇¹, ∇², x, y, p, w, feat) ∇²[feat] = ∇2 return nothing end -function update_∇_bias!(::Type{Tweedie}, ∇_bias, x, y, p, w) +function update_∇_bias!(::Type{TweedieDev}, ∇_bias, x, y, p, w) rho = eltype(p)(1.5) ∇1, ∇2 = zero(eltype(p)), zero(eltype(p)) @turbo for i in axes(x, 1) diff --git a/src/losses.jl b/src/losses.jl index 7fad44d..ecb8c03 100644 --- a/src/losses.jl +++ b/src/losses.jl @@ -1,20 +1,20 @@ module Losses -export Loss, MSE, Logistic, Poisson, Gamma, Tweedie, loss_types +export Loss, MSE, Logistic, PoissonDev, GammaDev, TweedieDev, loss_types abstract type Loss end struct MSE <: Loss end struct Logistic <: Loss end -struct Poisson <: Loss end -struct Gamma <: Loss end -struct Tweedie <: Loss end +struct PoissonDev <: Loss end +struct GammaDev <: Loss end +struct TweedieDev <: Loss end const loss_types = Dict( :mse => MSE, :logistic => Logistic, - :poisson => Poisson, - :gamma => Gamma, - :tweedie => Tweedie, + :poisson_deviance => PoissonDev, + :gamma_deviance => GammaDev, + :tweedie_deviance => TweedieDev, ) end From 81877c7655576c82aee000532dc5d5fc0fafb224 Mon Sep 17 00:00:00 2001 From: "jeremie.desgagne.bouchard" Date: Sat, 8 Jun 2024 12:44:56 -0400 Subject: [PATCH 4/9] cleanup for Linear only --- Project.toml | 2 - experiments/random-mse.jl | 58 ++++++--- src/EvoLinear.jl | 23 ++-- src/MLJ.jl | 38 +----- src/{callback.jl => callbacks.jl} | 35 +++++- src/fit.jl | 144 +++++++++++++++++++++ src/linear/structs.jl | 2 + src/losses.jl | 173 ++++++++++++++++++++++++-- src/predict.jl | 90 ++++++++++++++ src/splines/Splines.jl | 32 ----- src/splines/fit.jl | 99 --------------- src/splines/loss.jl | 7 -- src/splines/structs.jl | 113 ----------------- src/{splines/models.jl => structs.jl} | 87 +++++++------ test/runtests.jl | 1 - test/spline.jl | 70 ----------- 16 files changed, 534 insertions(+), 440 deletions(-) rename src/{callback.jl => callbacks.jl} (58%) create mode 100644 src/fit.jl create mode 100644 src/predict.jl delete mode 100644 src/splines/Splines.jl delete mode 100644 src/splines/fit.jl delete mode 100644 src/splines/loss.jl delete mode 100644 src/splines/structs.jl rename src/{splines/models.jl => structs.jl} (66%) delete mode 100644 test/spline.jl diff --git a/Project.toml b/Project.toml index d61aa2f..f36be79 100644 --- a/Project.toml +++ b/Project.toml @@ -5,7 +5,6 @@ version = "0.5.0" [deps] Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" -Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" LoopVectorization = "bdcacae8-1622-11e9-2a5c-532679323890" MLJModelInterface = "e80e1ace-859a-464e-9ed9-23947d8ae3ea" @@ -18,7 +17,6 @@ Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" [compat] Distributions = "0.25" -Flux = "0.13, 0.14" LoopVectorization = "0.12" MLJModelInterface = "1.0" Optimisers = "0.2, 0.3" diff --git a/experiments/random-mse.jl b/experiments/random-mse.jl index 65d1d96..88f9830 100644 --- a/experiments/random-mse.jl +++ b/experiments/random-mse.jl @@ -1,27 +1,41 @@ using Revise +using DataFrames using EvoLinear using BenchmarkTools nobs = 1_000_000 nfeats = 100 T = Float32 +train_pct = 0.8 -x_train = randn(T, nobs, nfeats) +x = randn(T, nobs, nfeats) coef = randn(T, nfeats) -y_train = x_train * coef .+ rand(T, nobs) * T(0.1) +y = x * coef .+ rand(T, nobs) * T(0.1) -config = EvoLinearRegressor(nrounds=10, loss=:mse, L1=0e-1, L2=1) -@time m, logger = EvoLinear.fit(config; x_train, y_train, metric=:mae, x_eval = x_train, y_eval = y_train, print_every_n = 5, return_logger = true); +dtot = DataFrame(x, :auto) +dtot.y = y +target_name = :y +feature_names = setdiff(Symbol.(names(dtot)), [target_name]) + +train_idx = 1:round(Int, train_pct * nobs) +eval_idx = setdiff(1:nobs, train_idx) +dtrain = dtot[train_idx, :] +deval = dtot[eval_idx, :] + +config = EvoLinearRegressor(nrounds=100, loss=:mse, eta=0.1, L1=0e-1, L2=1) +# @time m = EvoLinear.fit(config, dtrain; target_name, feature_names); +@time m = EvoLinear.fit(config, dtrain; target_name, feature_names, deval=dtrain, metric=:mae, print_every_n=5); +# @time m, logger = EvoLinear.fit(config; x_train, y_train, metric=:mae, x_eval = x_train, y_eval = y_train, print_every_n = 5, return_logger = true); # @btime m, logger = EvoLinear.fit(config; x_train, y_train, metric=:mae, x_eval = x_train, y_eval = y_train, print_every_n = 5, return_logger = true); sum(m.coef .== 0) config = EvoLinear.EvoLinearRegressor(nrounds=10, loss=:mse, L1=1e-1, L2=1e-2) # @btime m = EvoLinear.fit(config; x_train, y_train); -@time m0, cache = EvoLinear.Linear.init(config, x_train, y_train) -@time EvoLinear.Linear.fit!(m0, cache, config); +@time m0, cache = EvoLinear.init(config, dtrain; feature_names, target_name) +@time EvoLinear.fit!(m0, cache, config); # @btime EvoLinear.fit!($m, $cache, $config; x=$x, y=$y) -@code_warntype EvoLinear.Linear.fit!(m0, cache, config) +@code_warntype EvoLinear.fit!(m0, cache, config) logger[:nrounds] p = EvoLinear.Linear.predict_proj(m, x_train) @@ -33,22 +47,32 @@ p = EvoLinear.Linear.predict_proj(m, x_train) metric = EvoLinear.Metrics.mse(p, y_train) metric = EvoLinear.Metrics.mae(p, y_train) + +##################################### +# XGBoost +##################################### +@info "xgboost train" using XGBoost + +x_train = dtrain[:, feature_names] +y_train = dtrain[:, target_name] + # xgboost aprams params_xgb = [ - "booster" => "gblinear", - "updater" => "shotgun", # shotgun / coord_descent - "eta" => 1.0, - "lambda" => 0.0, - "objective" => "reg:squarederror", - "print_every_n" => 5] + :booster => "gblinear", + :updater => "shotgun", # shotgun / coord_descent + :eta => 0.1, + :lambda => 0.0, + :num_round => nrounds, + :objective => "reg:squarederror", + :print_every_n => 5] nthread = Threads.nthreads() -nrounds = 10 # metrics = ["rmse"] -metrics = ["mae"] +metric_xgb = ["mae"] # metrics = ["logloss"] -@info "xgboost train:" -@time m_xgb = xgboost(x_train, nrounds, label=y_train, param=params_xgb, metrics=metrics, nthread=nthread, silent=0); +dtrain_xgb = DMatrix(x_train, y_train) +watchlist = Dict("train" => DMatrix(x_train, y_train)) +@time m_xgb = xgboost(dtrain_xgb; watchlist, nthread, verbosity=0, silent=0, eval_metric=metric_xgb, params_xgb...) diff --git a/src/EvoLinear.jl b/src/EvoLinear.jl index e520a77..edb04f3 100644 --- a/src/EvoLinear.jl +++ b/src/EvoLinear.jl @@ -1,25 +1,30 @@ module EvoLinear -using Statistics: mean, std +import Base.Threads: @threads +import Statistics: mean, std +import Tables import Random import MLJModelInterface as MMI import MLJModelInterface: fit, update, predict, schema -export EvoLinearRegressor, EvoSplineRegressor +export EvoLinearRegressor, EvoLinearModel include("utils.jl") -include("metrics.jl") -include("callback.jl") + include("losses.jl") +using .Losses + +include("metrics.jl") +using .Metrics -include("linear/Linear.jl") -using .Linear +include("structs.jl") -include("splines/Splines.jl") -using .Splines +include("callbacks.jl") +using .CallBacks -const EvoLinearTypes = Union{EvoLinearRegressor,EvoSplineRegressor} +include("fit.jl") +include("predict.jl") include("MLJ.jl") diff --git a/src/MLJ.jl b/src/MLJ.jl index f3aab3d..a1c8d4c 100644 --- a/src/MLJ.jl +++ b/src/MLJ.jl @@ -6,15 +6,6 @@ function MMI.fit(model::EvoLinearRegressor, verbosity::Int, A, y) report = (coef = fitresult.coef, bias = fitresult.bias, names = A.names) return fitresult, cache, report end -function MMI.fit(model::EvoSplineRegressor, verbosity::Int, A, y) - fitresult, cache = EvoLinear.Splines.init(model, A.matrix, y) - while cache[:info][:nrounds] < model.nrounds - fit!(fitresult, cache) - end - report = nothing - # report = (coef = fitresult.coef, bias = fitresult.bias, names = A.names) - return fitresult, cache, report -end function okay_to_continue(model, fitresult, cache) return model.nrounds - cache[:info][:nrounds] >= 0 && @@ -32,27 +23,11 @@ function MMI.update(model::EvoLinearRegressor, verbosity::Integer, fitresult, ca end return fitresult, cache, report end -function MMI.update(model::EvoSplineRegressor, verbosity::Integer, fitresult, cache, A, y) - if okay_to_continue(model, fitresult, cache) - while cache[:info][:nrounds] < model.nrounds - fit!(fitresult, cache) - end - # report = (coef = fitresult.coef, bias = fitresult.bias, names = A.names) - report = nothing - else - fitresult, cache, report = fit(model, verbosity, A, y) - end - return fitresult, cache, report -end function predict(::EvoLinearRegressor, fitresult, A) pred = fitresult(A.matrix) return pred end -function predict(::EvoSplineRegressor, fitresult, A) - pred = fitresult(A.matrix') - return pred -end # Generate names to be used by feature_importances in the report MMI.reformat(::EvoLinearTypes, X, y) = @@ -73,7 +48,7 @@ MMI.iteration_parameter(::Type{<:EvoLinearTypes}) = :nrounds # Metadata MMI.metadata_pkg.( - (EvoLinearRegressor, EvoSplineRegressor), + (EvoLinearRegressor), name = "EvoLinear", uuid = "ab853011-1780-437f-b4b5-5de6f4777246", url = "https://github.com/jeremiedb/EvoLinear.jl", @@ -92,14 +67,3 @@ MMI.metadata_model( weights = false, path = "EvoLinear.EvoLinearRegressor", ) - -MMI.metadata_model( - EvoSplineRegressor, - input_scitype = Union{ - MMI.Table(MMI.Continuous, MMI.Count, MMI.OrderedFactor), - AbstractMatrix{MMI.Continuous}, - }, - target_scitype = AbstractVector{<:MMI.Continuous}, - weights = false, - path = "EvoLinear.EvoSplineRegressor", -) diff --git a/src/callback.jl b/src/callbacks.jl similarity index 58% rename from src/callback.jl rename to src/callbacks.jl index 4260abf..15de5e5 100644 --- a/src/callback.jl +++ b/src/callbacks.jl @@ -1,8 +1,12 @@ module CallBacks -using ..EvoLinear.Metrics +import Base.Threads: @threads +import Tables -export init_logger, update_logger! +import ..EvoLinear: EvoLinearTypes +import ..EvoLinear.Metrics: metric_dict + +export CallBack, init_logger, update_logger! struct CallBack{F,M,V,Y} feval::F @@ -12,6 +16,33 @@ struct CallBack{F,M,V,Y} w::V end +function CallBack( + config::EvoLinearTypes, + deval; + metric, + feature_names, + target_name, + weight_name=nothing +) + T = Float32 + nobs = Tables.DataAPI.nrow(deval) + nfeats = length(feature_names) + feval = metric_dict[metric] + + x = zeros(T, nobs, nfeats) + @threads for j in axes(x, 2) + @views x[:, j] .= Tables.getcolumn(deval, feature_names[j]) + end + y = Tables.getcolumn(deval, target_name) + y = convert(Vector{T}, y) + p = zero(y) + + w = isnothing(weight_name) ? ones(T, nobs) : convert(Vector{T}, Tables.getcolumn(deval, weight_name)) + + return CallBack(feval, x, p, y, w) +end + + function init_logger(; metric, maximise, early_stopping_rounds) logger = Dict( :name => String(metric), diff --git a/src/fit.jl b/src/fit.jl new file mode 100644 index 0000000..813e73c --- /dev/null +++ b/src/fit.jl @@ -0,0 +1,144 @@ +function init(config::EvoLinearRegressor{L}, dtrain; feature_names, target_name, weight_name=nothing) where {L} + + T = Float32 + nobs = Tables.DataAPI.nrow(dtrain) + nfeats = length(feature_names) + + x = zeros(T, nobs, nfeats) + @threads for j in axes(x, 2) + @views x[:, j] .= Tables.getcolumn(dtrain, feature_names[j]) + end + + y = Tables.getcolumn(dtrain, target_name) + y = convert(Vector{T}, y) + + w = isnothing(weight_name) ? ones(T, nobs) : convert(Vector{T}, Tables.getcolumn(dtrain, weight_name)) + ∑w = sum(w) + + ∇¹, ∇² = zeros(T, size(x, 2)), zeros(T, size(x, 2)) + ∇b = zeros(T, 2) + + info = Dict( + :nrounds => 0, + :feature_names => feature_names, + :target_name => target_name, + :weight_name => weight_name) + + cache = ( + x=x, + y=y, + w=w, + ∑w=∑w, + ∇¹=∇¹, + ∇²=∇², + ∇b=∇b, + ) + + m = EvoLinearModel(L; coef=zeros(T, size(x, 2)), bias=zero(T), info=info) + + return m, cache +end + + +""" + fit(config::EvoLinearRegressor; + x, y, w=nothing, + x_eval=nothing, y_eval=nothing, w_eval=nothing, + metric=:none, + print_every_n=1) + +Provided a `config`, `EvoLinear.fit` takes `x` and `y` as features and target inputs, plus optionally `w` as weights and train a Linear boosted model. + +# Arguments +- `config::EvoLinearRegressor`: + +# Keyword arguments +- `x::AbstractMatrix`: Features matrix. Dimensions are `[nobs, num_features]`. +- `y::AbstractVector`: Vector of observed targets. +- `w=nothing`: Vector of weights. Can be be either a `Vector` or `nothing`. If `nothing`, assumes a vector of 1s. +- `metric=nothing`: Evaluation metric to be tracked through each iteration. Default to `nothing`. Can be one of: + + - `:mse` + - `:logistic` + - `:poisson_deviance` + - `:gamma_deviance` + - `:tweedie_deviance` +""" +function fit( + config::EvoLinearRegressor, + dtrain; + feature_names, + target_name, + weight_name=nothing, + deval=nothing, + metric=nothing, + print_every_n=9999, + early_stopping_rounds=9999, + verbosity=1 +) + + feature_names = Symbol.(feature_names) + target_name = Symbol(target_name) + weight_name = isnothing(weight_name) ? nothing : Symbol(weight_name) + logger = nothing + + m, cache = init(config, dtrain; feature_names, target_name, weight_name) + + if !isnothing(metric) && !isnothing(deval) + cb = CallBack(config, deval; metric, feature_names, target_name, weight_name) + logger = init_logger(; + metric, + maximise=is_maximise(cb.feval), + early_stopping_rounds, + ) + update_logger!(logger, m, cb, 0) + (verbosity > 0) && @info "initialization" metric = logger[:metrics][end] + end + + for iter = 1:config.nrounds + fit!(m, cache, config) + if !isnothing(logger) + update_logger!(logger, m, cb, iter) + if iter % print_every_n == 0 && verbosity > 0 + @info "iter $iter" metric = logger[:metrics][end] + end + (logger[:iter_since_best] >= logger[:early_stopping_rounds]) && break + end + end + + m.info[:logger] = logger + return m +end + +function fit!(m::EvoLinearModel, cache, config::EvoLinearRegressor) + + ∇¹, ∇², ∇b = cache.∇¹ .* 0, cache.∇² .* 0, cache.∇b .* 0 + x, y, w = cache.x, cache.y, cache.w + ∑w = cache.∑w + + if config.updater == :all + # update all coefs then bias + p = m(x; proj=true) + update_∇_bias!(m.loss, ∇b, x, y, p, w) + update_bias!(m, ∇b) + + p = m(x; proj=true) + update_∇!(m.loss, ∇¹, ∇², x, y, p, w) + update_coef!(m, ∇¹, ∇², ∑w, config) + else + @error "invalid updater" + end + m.info[:nrounds] += 1 + return nothing +end + +function update_coef!(m, ∇¹, ∇², ∑w, config) + update = -∇¹ ./ (∇² .+ config.L2 * ∑w) + update[abs.(update). MSE, :logistic => Logistic, - :poisson_deviance => PoissonDev, - :gamma_deviance => GammaDev, - :tweedie_deviance => TweedieDev, + :poisson_deviance => Poisson, + :gamma_deviance => Gamma, + :tweedie_deviance => Tweedie, ) +# function init_∇¹(x) +# ∇¹ = zeros(size(x, 2)) +# return ∇¹ +# end +# function init_∇²(x) +# ∇² = zeros(size(x, 2)) +# return ∇² +# end + +""" + update_∇!(L, ∇¹, ∇², x, y, p, w) + +Update gradients w.r.t each feature. Each feature gradient update is dispatch according to the loss type (`mse`, `logistic`...). +""" +function update_∇!(L, ∇¹, ∇², x, y::A, p::A, w::A) where {A} + @threads for feat in axes(x, 2) + update_∇!(L, ∇¹, ∇², x, y, p, w, feat) + end + return nothing +end + +################################### +# linear +################################### +function update_∇!(::Type{MSE}, ∇¹, ∇², x, y, p, w, feat) + ∇1, ∇2 = zero(eltype(p)), zero(eltype(p)) + @turbo for i in axes(x, 1) + ∇1 += 2 * (p[i] - y[i]) * x[i, feat] * w[i] + ∇2 += 2 * x[i, feat]^2 * w[i] + end + ∇¹[feat] = ∇1 + ∇²[feat] = ∇2 + return nothing +end +function update_∇_bias!(::Type{MSE}, ∇_bias, x, y, p, w) + ∇1, ∇2 = zero(eltype(p)), zero(eltype(p)) + @turbo for i in axes(x, 1) + ∇1 += 2 * (p[i] - y[i]) * w[i] + ∇2 += 2 * w[i] + end + ∇_bias[1] = ∇1 + ∇_bias[2] = ∇2 + return nothing +end + +################################### +# logistic +################################### +function update_∇!(::Type{Logistic}, ∇¹, ∇², x, y, p, w, feat) + ∇1, ∇2 = zero(eltype(p)), zero(eltype(p)) + @turbo for i in axes(x, 1) + ∇1 += (p[i] - y[i]) * x[i, feat] * w[i] + ∇2 += p[i] * (1 - p[i]) * x[i, feat]^2 * w[i] + end + ∇¹[feat] = ∇1 + ∇²[feat] = ∇2 + return nothing +end +function update_∇_bias!(::Type{Logistic}, ∇_bias, x, y, p, w) + ∇1, ∇2 = zero(eltype(p)), zero(eltype(p)) + @turbo for i in axes(x, 1) + ∇1 += (p[i] - y[i]) * w[i] + ∇2 += p[i] * (1 - p[i]) * w[i] + end + ∇_bias[1] = ∇1 + ∇_bias[2] = ∇2 + return nothing +end + +################################### +# Poisson +# Deviance = 2 * (y * log(y/μ) + μ - y) +# https://www.casact.org/sites/default/files/old/rpm_2013_handouts_paper_1497_handout_795_0.pdf +# The prediction p is assumed to be on the projected basis (exp(pred_linear)) +# Derivative is w.r.t to β on the linear basis +################################### +function update_∇!(::Type{Poisson}, ∇¹, ∇², x, y, p, w, feat) + ∇1, ∇2 = zero(eltype(p)), zero(eltype(p)) + @turbo for i in axes(x, 1) + ∇1 += 2 * (p[i] - y[i]) * x[i, feat] * w[i] + ∇2 += 2 * p[i] * x[i, feat]^2 * w[i] + end + ∇¹[feat] = ∇1 + ∇²[feat] = ∇2 + return nothing +end +function update_∇_bias!(::Type{Poisson}, ∇_bias, x, y, p, w) + ∇1, ∇2 = zero(eltype(p)), zero(eltype(p)) + @turbo for i in axes(x, 1) + ∇1 += 2 * (p[i] - y[i]) * w[i] + ∇2 += 2 * p[i] * w[i] + end + ∇_bias[1] = ∇1 + ∇_bias[2] = ∇2 + return nothing +end + +################################### +# Gamma +# Deviance = 2 * (log(μ/y) + y/μ - 1) +# https://www.casact.org/sites/default/files/old/rpm_2013_handouts_paper_1497_handout_795_0.pdf +# The prediction p is assumed to be on the projected basis (exp(pred_linear)) +# Derivative is w.r.t to β on the linear basis +################################### +function update_∇!(::Type{Gamma}, ∇¹, ∇², x, y, p, w, feat) + ∇1, ∇2 = zero(eltype(p)), zero(eltype(p)) + @turbo for i in axes(x, 1) + ∇1 += 2 * (1 - y[i] / p[i]) * x[i, feat] * w[i] + ∇2 += 2 * y[i] / p[i] * x[i, feat]^2 * w[i] + end + ∇¹[feat] = ∇1 + ∇²[feat] = ∇2 + return nothing +end +function update_∇_bias!(::Type{Gamma}, ∇_bias, x, y, p, w) + ∇1, ∇2 = zero(eltype(p)), zero(eltype(p)) + @turbo for i in axes(x, 1) + ∇1 += 2 * (1 - y[i] / p[i]) * w[i] + ∇2 += 2 * y[i] / p[i] * w[i] + end + ∇_bias[1] = ∇1 + ∇_bias[2] = ∇2 + return nothing +end + +################################### +# Tweedie +# Deviance = 2 * (y²⁻ᵖ/(1-p)(2-p) - yμ¹⁻ᵖ/(1-p) + μ²⁻ᵖ/(2-p)) +# https://www.casact.org/sites/default/files/old/rpm_2013_handouts_paper_1497_handout_795_0.pdf +# The prediction p is assumed to be on the projected basis (exp(pred_linear)) +# Derivative is w.r.t to β on the linear basis +################################### +function update_∇!(::Type{Tweedie}, ∇¹, ∇², x, y, p, w, feat) + rho = eltype(p)(1.5) + ∇1, ∇2 = zero(eltype(p)), zero(eltype(p)) + @turbo for i in axes(x, 1) + ∇1 += 2 * (p[i]^(2 - rho) - y[i] * p[i]^(1 - rho)) * x[i, feat] * w[i] + ∇2 += 2 * ((2 - rho) * p[i]^(2 - rho) - (1 - rho) * y[i] * p[i]^(1 - rho)) * x[i, feat]^2 * w[i] + end + ∇¹[feat] = ∇1 + ∇²[feat] = ∇2 + return nothing +end +function update_∇_bias!(::Type{Tweedie}, ∇_bias, x, y, p, w) + rho = eltype(p)(1.5) + ∇1, ∇2 = zero(eltype(p)), zero(eltype(p)) + @turbo for i in axes(x, 1) + ∇1 += 2 * (p[i]^(2 - rho) - y[i] * p[i]^(1 - rho)) * w[i] + ∇2 += 2 * ((2 - rho) * p[i]^(2 - rho) - (1 - rho) * y[i] * p[i]^(1 - rho)) * w[i] + end + ∇_bias[1] = ∇1 + ∇_bias[2] = ∇2 + return nothing +end + end diff --git a/src/predict.jl b/src/predict.jl new file mode 100644 index 0000000..57662d1 --- /dev/null +++ b/src/predict.jl @@ -0,0 +1,90 @@ + +function (m::EvoLinearModel{L})(x::AbstractMatrix; proj::Bool=true) where {L} + p = x * m.coef .+ m.bias + proj ? proj!(L, p) : nothing + return p +end +function (m::EvoLinearModel{L})(p::AbstractVector, x::AbstractMatrix; proj::Bool=true) where {L} + p .= x * m.coef .+ m.bias + proj ? proj!(L, p) : nothing + return nothing +end +function (m::EvoLinearModel{L})(data; proj::Bool=true) where {L} + + Tables.istable(data) || error("data must be Table compatible") + + T = Float32 + feature_names = m.info[:feature_names] + nobs = Tables.DataAPI.nrow(data) + nfeats = length(feature_names) + + x = zeros(T, nobs, nfeats) + @threads for j in axes(x, 2) + @views x[:, j] .= Tables.getcolumn(data, feature_names[j]) + end + + p = x * m.coef .+ m.bias + proj ? proj!(L, p) : nothing + return p +end + +function proj!(::L, p) where {L<:Type{MSE}} + return nothing +end +function proj!(::L, p) where {L<:Type{Logistic}} + p .= sigmoid.(p) + return nothing +end +function proj!(::L, p) where {L<:Union{Type{Poisson},Type{Gamma},Type{Tweedie}}} + p .= exp.(p) + return nothing +end + +# """ +# predict_linear(m, x) + +# Returns the predictions on the linear basis from model `m` using the features matrix `x`. + +# # Arguments + +# - `m::EvoLinearModel`: model generating the predictions. +# - `x`: features matrix `[nobs, num_features]` for which predictions are generated. +# """ +# function predict_linear(m::EvoLinearModel, x) +# p = x * m.coef .+ m.bias +# return p +# end + +# function predict_linear!(p, m::EvoLinearModel, x) +# p .= x * m.coef .+ m.bias +# return nothing +# end + +# """ +# predict_proj(m, x) + +# Returns the predictions on the projected basis from model `m` using the features matrix `x`. + +# - `MSE`: `pred_proj = pred_linear` +# - `Logistic`: `pred_proj = sigmoid(pred_linear)` +# - `Poisson`: `pred_proj = exp(pred_linear)` +# - `Gamma`: `pred_proj = exp(pred_linear)` +# - `Tweedie`: `pred_proj = exp(pred_linear)` + +# # Arguments + +# - `m::EvoLinearModel`: model generating the predictions. +# - `x`: features matrix `[nobs, num_features]` for which predictions are generated. +# """ +# function predict_proj(m::EvoLinearModel{MSE}, x) +# p = predict_linear(m, x) +# return p +# end +# function predict_proj(m::EvoLinearModel{Logistic}, x) +# p = sigmoid(predict_linear(m, x)) +# return p +# end +# function predict_proj(m::EvoLinearModel{L}, x) where {L<:Union{Poisson,Gamma,Tweedie}} +# p = exp.(predict_linear(m, x)) +# return p +# end \ No newline at end of file diff --git a/src/splines/Splines.jl b/src/splines/Splines.jl deleted file mode 100644 index 6c61ef4..0000000 --- a/src/splines/Splines.jl +++ /dev/null @@ -1,32 +0,0 @@ -module Splines - -using ..EvoLinear -using ..EvoLinear: sigmoid, logit, mk_rng - -using ..EvoLinear.Metrics -using ..EvoLinear.Losses -using ..EvoLinear.CallBacks -import ..EvoLinear.CallBacks: CallBack -import ..EvoLinear: init, fit!, get_loss_type - -import MLJModelInterface as MMI -import MLJModelInterface: fit, update, predict, schema - -using Statistics: mean, std -using StatsBase: quantile -using Distributions: Normal -using Flux -using Flux: DataLoader -using Flux: update!, @functor -using Optimisers -using SparseArrays -using LinearAlgebra - -export EvoSplineRegressor, EvoSplineModel - -include("models.jl") -include("loss.jl") -include("structs.jl") -include("fit.jl") - -end \ No newline at end of file diff --git a/src/splines/fit.jl b/src/splines/fit.jl deleted file mode 100644 index a906235..0000000 --- a/src/splines/fit.jl +++ /dev/null @@ -1,99 +0,0 @@ -function init(config::EvoSplineRegressor{L,T}, x, y; w=nothing) where {L,T} - - @info "starting spline" - device = config.device == :cpu ? Flux.cpu : Flux.gpu - nfeats = size(x, 2) - dtrain = DataLoader( - (x=Matrix{T}(x') |> device, y=T.(y) |> device), - batchsize=config.batchsize, - ) - loss = loss_fn[L] - - m = EvoSplineModel(config; nfeats, mean=mean(y)) |> device - - opt = Optimisers.NAdam(config.eta) - opts = Optimisers.setup(opt, m) - - cache = (dtrain=dtrain, loss=loss, opts=opts, info=Dict(:nrounds => 0)) - return m, cache -end - - -""" - fit(config::EvoSplineRegressor; x_train, y_train, x_eval = nothing, y_eval = nothing) - -Train a splined linear model. -""" -function fit( - config::EvoSplineRegressor{L,T}; - x_train, - y_train, - w_train=nothing, - x_eval=nothing, - y_eval=nothing, - w_eval=nothing, - metric=nothing, - print_every_n=9999, - early_stopping_rounds=9999, - verbosity=1, - fnames=nothing, - return_logger=false, -) where {L,T} - - m, cache = init(config, x_train, y_train; w=w_train) - logger = nothing - - if !isnothing(metric) && !isnothing(x_eval) && !isnothing(y_eval) - cb = CallBack(config; metric, x_eval, y_eval, w_eval) - logger = init_logger(; - T, - metric, - maximise=is_maximise(cb.feval), - early_stopping_rounds, - ) - cb(logger, 0, m) - (verbosity > 0) && @info "initialization" metric = logger[:metrics][end] - end - - for iter = 1:config.nrounds - fit!(m, cache) - if !isnothing(logger) - cb(logger, iter, m) - if iter % print_every_n == 0 && verbosity > 0 - @info "iter $iter" metric = logger[:metrics][end] - end - (logger[:iter_since_best] >= logger[:early_stopping_rounds]) && break - end - end - if return_logger - return (m, logger) - else - return m - end -end - -function fit!(m, cache) - for d in cache[:dtrain] - grads = gradient(model -> cache[:loss](model(d[:x]; proj=false), d[:y]), m)[1] - Optimisers.update!(cache[:opts], m, grads) - end - cache[:info][:nrounds] += 1 - return nothing -end - -function CallBack( - config::EvoSplineRegressor; - metric, - x_eval, - y_eval, - w_eval=nothing, -) - T = Flota32 - device = config.device == :cpu ? Flux.cpu : Flux.gpu - feval = metric_dict[metric] - x = convert(Matrix{T}, x_eval') - p = zeros(T, length(y_eval)) - y = convert(Vector{T}, y_eval) - w = isnothing(w_eval) ? ones(T, size(y)) : convert(Vector{T}, w_eval) - return CallBack(feval, x |> device, p |> device, y |> device, w |> device) -end diff --git a/src/splines/loss.jl b/src/splines/loss.jl deleted file mode 100644 index 739c055..0000000 --- a/src/splines/loss.jl +++ /dev/null @@ -1,7 +0,0 @@ -const loss_fn = Dict( - MSE => Flux.Losses.mse, - Logistic => Flux.Losses.logitbinarycrossentropy, - # :poisson => Poisson, - # :gamma => Gamma, - # :tweedie => Tweedie, -) \ No newline at end of file diff --git a/src/splines/structs.jl b/src/splines/structs.jl deleted file mode 100644 index 01e04e5..0000000 --- a/src/splines/structs.jl +++ /dev/null @@ -1,113 +0,0 @@ -struct Spline{S,B,W,F} - mat::S - b::B - w::W - act::F -end - -@functor Spline -Flux.trainable(m::Spline) = (b = m.b, w = m.w) - -function (m::Spline)(x) - m.w * m.act.(m.mat * x .- m.b) -end - -function Spline(; nfeats, act, knots = Dict{Int,Int}()) - T = Float32 - nknots = sum(values(knots)) - sp = spzeros(T, nknots, nfeats) - b = randn(T, nknots) - cum = 0 - for (k, v) in knots - sp[cum+1:cum+v, k] .= 1 - b[cum+1:cum+v] .= quantile.(Normal(0, 1), collect(1:v) ./ (v + 1)) - cum += v - end - w = randn(T, 1, nknots) ./ T(100) ./ T(sqrt(nfeats)) - m = Spline(sp, b, w, act) - return m -end - -struct Linear{B,W} - b::B - w::W -end - -@functor Linear -Flux.trainable(m::Linear) = (b = m.b, w = m.w) - -function (m::Linear)(x) - m.w * x .+ m.b -end - -function Linear(; nfeats, mean = 0) - T = Float32 - b = ones(T, 1) .* T(mean) - w = randn(T, 1, nfeats) ./ T(100) ./ T(sqrt(nfeats)) - m = Linear(b, w) - return m -end - -struct EvoSplineModel{L,A,B,C} - loss::Type{L} - bn::A - linear::B - spline::C -end - -@functor EvoSplineModel -Flux.trainable(m::EvoSplineModel) = (bn = m.bn, linear = m.linear, spline = m.spline) - -const act_dict = Dict( - :sigmoid => Flux.sigmoid_fast, - :tanh => tanh, - :relu => relu, - :elu => elu, - :gelu => gelu, - :softplus => softplus, -) - -function EvoSplineModel(config::EvoSplineRegressor{L,T}; nfeats, mean = 0) where {L,T} - bn = BatchNorm(nfeats; affine = true) - linear = Linear(; nfeats, mean) - act = act_dict[config.act] - spline = isnothing(config.knots) ? nothing : Spline(; nfeats, config.knots, act) - m = EvoSplineModel{L,typeof(bn),typeof(linear),typeof(spline)}(L, bn, linear, spline) - return m -end - -get_loss_type(::EvoSplineModel{L,A,B,C}) where {L,A,B,C} = L -get_loss_type(::EvoSplineRegressor{L,T}) where {L,T} = L - -function (m::EvoSplineModel{L,A,B,C})(x; proj = true) where {L,A,B,C} - _x = x |> m.bn - if isnothing(m.spline) - p = m.linear(_x) |> vec - else - p = (m.linear(_x) .+ m.spline(_x)) |> vec - end - proj ? proj!(L, p) : nothing - return p -end -function (m::EvoSplineModel{L,A,B,C})(p, x; proj = true) where {L,A,B,C} - _x = x |> m.bn - if isnothing(m.spline) - p .= m.linear(_x) |> vec - else - p .= (m.linear(_x) .+ m.spline(_x)) |> vec - end - proj ? proj!(L, p) : nothing - return nothing -end - -function proj!(::L, p) where {L<:Type{MSE}} - return nothing -end -function proj!(::L, p) where {L<:Type{Logistic}} - p .= sigmoid.(p) - return nothing -end -function proj!(::L, p) where {L<:Union{Type{Poisson},Type{Gamma},Type{Tweedie}}} - p .= exp.(p) - return nothing -end diff --git a/src/splines/models.jl b/src/structs.jl similarity index 66% rename from src/splines/models.jl rename to src/structs.jl index 291c1ed..f0c21ee 100644 --- a/src/splines/models.jl +++ b/src/structs.jl @@ -1,26 +1,23 @@ -mutable struct EvoSplineRegressor{L,T} <: MMI.Deterministic +mutable struct EvoLinearRegressor{L} <: MMI.Deterministic + updater::Symbol nrounds::Int - opt::Symbol - batchsize::Int - act::Symbol - eta::T - L2::T - knots::Union{Dict,Nothing} - rng::Any - device::Symbol + eta::Float32 + L1::Float32 + L2::Float32 + rng end - """ - EvoSplineRegressor(; kwargs...) + EvoLinearRegressor(; kwargs...) -A model type for constructing a EvoSplineRegressor, based on [EvoLinear.jl](https://github.com/jeremiedb/EvoLinear.jl), and implementing both an internal API and the MLJ model interface. +A model type for constructing a EvoLinearRegressor, based on [EvoLinear.jl](https://github.com/jeremiedb/EvoLinear.jl), and implementing both an internal API and the MLJ model interface. # Keyword arguments - `loss=:mse`: loss function to be minimised. Can be one of: + - `:mse` - `:logistic` - `:poisson` @@ -37,11 +34,11 @@ A model type for constructing a EvoSplineRegressor, based on [EvoLinear.jl](http # Internal API -Do `config = EvoSplineRegressor()` to construct an hyper-parameter struct with default hyper-parameters. +Do `config = EvoLinearRegressor()` to construct an hyper-parameter struct with default hyper-parameters. Provide keyword arguments as listed above to override defaults, for example: ```julia -EvoSplineRegressor(loss=:logistic, L1=1e-3, L2=1e-2, nrounds=100) +EvoLinearRegressor(loss=:logistic, L1=1e-3, L2=1e-2, nrounds=100) ``` ## Training model @@ -49,7 +46,7 @@ EvoSplineRegressor(loss=:logistic, L1=1e-3, L2=1e-2, nrounds=100) A model is built using [`fit`](@ref): ```julia -config = EvoSplineRegressor() +config = EvoLinearRegressor() m = fit(config; x, y, w) ``` @@ -66,11 +63,11 @@ preds = m(x) From MLJ, the type can be imported using: ```julia -EvoSplineRegressor = @load EvoSplineRegressor pkg=EvoLinear +EvoLinearRegressor = @load EvoLinearRegressor pkg=EvoLinear ``` Do `model = EvoLinearRegressor()` to construct an instance with default hyper-parameters. -Provide keyword arguments to override hyper-parameter defaults, as in `EvoSplineRegressor(loss=...)`. +Provide keyword arguments to override hyper-parameter defaults, as in `EvoLinearRegressor(loss=...)`. ## Training model @@ -95,7 +92,7 @@ Train the machine using `fit!(mach, rows=...)`. The fields of `fitted_params(mach)` are: -- `:fitresult`: the `SplineModel` object returned by EvoSplineRegressor fitting algorithm. +- `:fitresult`: the `EvoLinearModel` object returned by EvoLnear.jl fitting algorithm. ## Report @@ -106,32 +103,26 @@ The fields of `report(mach)` are: - `:names`: Names of each of the features. """ -function EvoSplineRegressor(; kwargs...) +function EvoLinearRegressor(; kwargs...) # defaults arguments args = Dict{Symbol,Any}( :loss => :mse, + :updater => :all, :nrounds => 10, - :opt => :Adam, - :batchsize => 1024, - :act => :relu, - :eta => 1e-3, - :L2 => 0.0, - :knots => nothing, - :rng => 123, - :device => :cpu, - :T => Float32, + :eta => 1, + :L1 => 0, + :L2 => 0, + :rng => 123 ) args_ignored = setdiff(keys(kwargs), keys(args)) args_ignored_str = join(args_ignored, ", ") - length(args_ignored) > 0 && - @info "Following $(length(args_ignored)) provided arguments will be ignored: $(args_ignored_str)." + length(args_ignored) > 0 && @info "Following $(length(args_ignored)) provided arguments will be ignored: $(args_ignored_str)." args_default = setdiff(keys(args), keys(kwargs)) args_default_str = join(args_default, ", ") - length(args_default) > 0 && - @info "Following $(length(args_default)) arguments were not provided and will be set to default: $(args_default_str)." + length(args_default) > 0 && @info "Following $(length(args_default)) arguments were not provided and will be set to default: $(args_default_str)." args_override = intersect(keys(args), keys(kwargs)) for arg in args_override @@ -139,20 +130,28 @@ function EvoSplineRegressor(; kwargs...) end args[:rng] = mk_rng(args[:rng]) - T = args[:T] - L = loss_types[Symbol(args[:loss])] + L = loss_types[args[:loss]] - model = EvoSplineRegressor{L,T}( + model = EvoLinearRegressor{L}( + args[:updater], args[:nrounds], - Symbol(args[:opt]), - args[:batchsize], - Symbol(args[:act]), - args[:T](args[:eta]), - args[:T](args[:L2]), - args[:knots], - args[:rng], - Symbol(args[:device]), - ) + args[:eta], + args[:L1], + args[:L2], + args[:rng]) return model end + +mutable struct EvoLinearModel{L<:Loss,A,B} + loss::Type{L} + coef::A + bias::B + info::Dict{Symbol,Any} +end +EvoLinearModel(loss::Type{<:Loss}; coef, bias, info) = EvoLinearModel(loss, coef, bias, info) +EvoLinearModel(loss::Symbol; coef, bias, info) = EvoLinearModel(loss_types[loss]; coef, bias, info) +get_loss_type(m::EvoLinearModel) = m.loss +get_loss_type(::EvoLinearRegressor{L}) where {L} = L + +const EvoLinearTypes = Union{EvoLinearRegressor} diff --git a/test/runtests.jl b/test/runtests.jl index adf6a09..0b55a21 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,7 +6,6 @@ using Random: seed! @testset "EvoLinear.jl" begin @testset "Core API" begin include("linear.jl") - include("spline.jl") end @testset "MLJ API" begin include("MLJ.jl") diff --git a/test/spline.jl b/test/spline.jl deleted file mode 100644 index e65b021..0000000 --- a/test/spline.jl +++ /dev/null @@ -1,70 +0,0 @@ -@testset "Constructors" begin - - seed!(121) - - nobs = 1_000 - nfeats = 10 - T = Float32 - - x_train = randn(T, nobs, nfeats) - coef = randn(T, nfeats) - y_train = x_train * coef .+ rand(T, nobs) * T(0.1) - - config = EvoSplineRegressor(nrounds = 10, loss = :mse) - m = EvoLinear.EvoSplineModel(config; nfeats, mean = mean(y_train)) - -end - -@testset "MSE" begin - - seed!(121) - - nobs = 10_000 - nfeats = 10 - T = Float32 - - x_train = randn(T, nobs, nfeats) - coef = randn(T, nfeats) - y_train = x_train * coef .+ rand(T, nobs) * T(0.1) - - config = EvoSplineRegressor(nrounds = 100, loss = :mse, knots = Dict(1 => 4, 10 => 4)) - m0 = EvoLinear.fit(config; x_train, y_train, metric = :mse) - m1, cache = EvoLinear.Linear.init(config, x_train, y_train) - for i = 1:config.nrounds - EvoLinear.fit!(m1, cache) - end - - coef_diff = m0.linear.w .- m1.linear.w - @info "max coef diff" maximum(coef_diff) - @info "min coef diff" minimum(coef_diff) - @test maximum(abs.(coef_diff)) < 0.01 - - p = m0(x_train') - - metric_mse = EvoLinear.Metrics.mse(p, y_train) - metric_mae = EvoLinear.Metrics.mae(p, y_train) - @test metric_mse < 0.05 - @test metric_mae < 0.2 - -end - -@testset "Logistic" begin - - seed!(121) - nobs = 10_000 - nfeats = 10 - T = Float32 - - x_train = randn(T, nobs, nfeats) - coef = randn(T, nfeats) - y_train = EvoLinear.sigmoid(x_train * coef .+ rand(T, nobs) * T(0.1)) - w = ones(T, nobs) - - config = EvoSplineRegressor(nrounds = 100, loss = :logistic, knots = Dict(1 => 4, 10 => 4)) - m = EvoLinear.fit(config; x_train, y_train, metric = :logloss) - p = m(x_train') - - metric = EvoLinear.Metrics.logloss(p, y_train) - @test metric < 0.45 - -end From 68d210812e8547548c364ef8840c73bb4f9b2de1 Mon Sep 17 00:00:00 2001 From: "jeremie.db" Date: Fri, 14 Jun 2024 17:22:58 -0400 Subject: [PATCH 5/9] vitepress docs --- docs/Project.toml | 1 + docs/make.jl | 28 +++-- docs/package.json | 15 +++ docs/src/.vitepress/config.mts | 48 ++++++++ docs/src/.vitepress/theme/index.ts | 19 +++ docs/src/.vitepress/theme/style.css | 179 ++++++++++++++++++++++++++++ docs/src/assets/favicon.ico | Bin 0 -> 1529 bytes docs/src/assets/logo.png | Bin 0 -> 7077 bytes docs/src/assets/style.css | 5 +- docs/src/index.md | 57 +++++---- docs/src/intro.md | 30 +++++ 11 files changed, 337 insertions(+), 45 deletions(-) create mode 100644 docs/package.json create mode 100644 docs/src/.vitepress/config.mts create mode 100644 docs/src/.vitepress/theme/index.ts create mode 100644 docs/src/.vitepress/theme/style.css create mode 100644 docs/src/assets/favicon.ico create mode 100644 docs/src/assets/logo.png create mode 100644 docs/src/intro.md diff --git a/docs/Project.toml b/docs/Project.toml index dfa65cd..9441786 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,2 +1,3 @@ [deps] Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +DocumenterVitepress = "4710194d-e776-4893-9690-8d956a29c365" diff --git a/docs/make.jl b/docs/make.jl index 804d5ad..131c95d 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,26 +1,28 @@ using Documenter +using DocumenterVitePress using EvoLinear -makedocs( +makedocs(; sitename="EvoLinear.jl", authors="Jeremie Desgagne-Bouchard", modules=[EvoLinear], + format=DocumenterVitepress.MarkdownVitepress( + repo="github.com/jeremiedb/EvoLinear.jl", + devbranch="main", + devurl="dev" + ), + warnonly=true, + checkdocs=:all, pages=[ "Home" => "index.md", "API" => "api.md", - ], - format=Documenter.HTML( - sidebar_sitename=true, - edit_link="main", - assets=["assets/style.css"] - ) + ] ) -# Documenter can also automatically deploy documentation to gh-pages. -# See "Hosting Documentation" and deploydocs() in the Documenter manual -# for more information. deploydocs( - repo="github.com/jeremiedb/EvoLinear.jl.git", - target="build", - devbranch="main" + repo="github.com/jeremiedb/EvoLinear.jl", + target="build", # this is where Vitepress stores its output + branch="gh-pages", + devbranch="main", + push_preview=true ) diff --git a/docs/package.json b/docs/package.json new file mode 100644 index 0000000..5633b49 --- /dev/null +++ b/docs/package.json @@ -0,0 +1,15 @@ +{ + "scripts": { + "docs:dev": "vitepress dev build/.documenter", + "docs:build": "vitepress build build/.documenter", + "docs:preview": "vitepress preview build/.documenter" + }, + "dependencies": { + "@shikijs/transformers": "^1.1.7", + "markdown-it": "^14.1.0", + "markdown-it-footnote": "^4.0.0", + "markdown-it-mathjax3": "^4.3.2", + "vitepress": "^1.1.4", + "vitepress-plugin-tabs": "^0.5.0" + } +} diff --git a/docs/src/.vitepress/config.mts b/docs/src/.vitepress/config.mts new file mode 100644 index 0000000..ea5b763 --- /dev/null +++ b/docs/src/.vitepress/config.mts @@ -0,0 +1,48 @@ +import { defineConfig } from 'vitepress' +import { tabsMarkdownPlugin } from 'vitepress-plugin-tabs' +import mathjax3 from "markdown-it-mathjax3"; +import footnote from "markdown-it-footnote"; + +// https://vitepress.dev/reference/site-config +export default defineConfig({ + base: 'REPLACE_ME_DOCUMENTER_VITEPRESS',// TODO: replace this in makedocs! + title: 'REPLACE_ME_DOCUMENTER_VITEPRESS', + description: 'REPLACE_ME_DOCUMENTER_VITEPRESS', + lastUpdated: true, + cleanUrls: true, + outDir: 'REPLACE_ME_DOCUMENTER_VITEPRESS', // This is required for MarkdownVitepress to work correctly... + head: [['link', { rel: 'icon', href: 'REPLACE_ME_DOCUMENTER_VITEPRESS_FAVICON' }]], + ignoreDeadLinks: true, + + markdown: { + math: true, + config(md) { + md.use(tabsMarkdownPlugin), + md.use(mathjax3), + md.use(footnote) + }, + theme: { + light: "github-light", + dark: "github-dark"} + }, + themeConfig: { + outline: 'deep', + logo: 'REPLACE_ME_DOCUMENTER_VITEPRESS', + search: { + provider: 'local', + options: { + detailedView: true + } + }, + nav: 'REPLACE_ME_DOCUMENTER_VITEPRESS', + sidebar: 'REPLACE_ME_DOCUMENTER_VITEPRESS', + editLink: 'REPLACE_ME_DOCUMENTER_VITEPRESS', + socialLinks: [ + { icon: 'github', link: 'REPLACE_ME_DOCUMENTER_VITEPRESS' } + ], + footer: { + message: 'Made with DocumenterVitepress.jl
', + copyright: `© Copyright ${new Date().getUTCFullYear()}.` + } + } +}) diff --git a/docs/src/.vitepress/theme/index.ts b/docs/src/.vitepress/theme/index.ts new file mode 100644 index 0000000..463b5d8 --- /dev/null +++ b/docs/src/.vitepress/theme/index.ts @@ -0,0 +1,19 @@ +// .vitepress/theme/index.ts +import { h } from 'vue' +import type { Theme } from 'vitepress' +import DefaultTheme from 'vitepress/theme' + +import { enhanceAppWithTabs } from 'vitepress-plugin-tabs/client' +import './style.css' + +export default { + extends: DefaultTheme, + Layout() { + return h(DefaultTheme.Layout, null, { + // https://vitepress.dev/guide/extending-default-theme#layout-slots + }) + }, + enhanceApp({ app, router, siteData }) { + enhanceAppWithTabs(app) + } +} satisfies Theme \ No newline at end of file diff --git a/docs/src/.vitepress/theme/style.css b/docs/src/.vitepress/theme/style.css new file mode 100644 index 0000000..d2ca479 --- /dev/null +++ b/docs/src/.vitepress/theme/style.css @@ -0,0 +1,179 @@ +@import url(https://fonts.googleapis.com/css?family=Space+Mono:regular,italic,700,700italic); +@import url(https://fonts.googleapis.com/css?family=Space+Grotesk:regular,italic,700,700italic); + +/* Customize default theme styling by overriding CSS variables: +https://github.com/vuejs/vitepress/blob/main/src/client/theme-default/styles/vars.css + */ + + /* Layouts */ + +/* + :root { + --vp-layout-max-width: 1440px; +} */ + +.VPHero .clip { + white-space: pre; + max-width: 500px; +} + +/* Fonts */ + + :root { + /* Typography */ + --vp-font-family-base: "Barlow", "Inter var experimental", "Inter var", + -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen, Ubuntu, + Cantarell, "Fira Sans", "Droid Sans", "Helvetica Neue", sans-serif; + + /* Code Snippet font */ + --vp-font-family-mono: "Space Mono", Menlo, Monaco, Consolas, "Courier New", + monospace; +} + +.mono { + /* + Disable contextual alternates (kind of like ligatures but different) in monospace, + which turns `/>` to an up arrow and `|>` (the Julia pipe symbol) to an up arrow as well. + This is pretty bad for Julia folks reading even though copy+paste retains the same text. + */ + font-feature-settings: 'calt' 0; +} + +/* Colors */ + +:root { + --julia-blue: #4063D8; + --julia-purple: #9558B2; + --julia-red: #CB3C33; + --julia-green: #389826; + + --vp-c-brand: #389826; + --vp-c-brand-light: #3dd027; + --vp-c-brand-lighter: #9499ff; + --vp-c-brand-lightest: #bcc0ff; + --vp-c-brand-dark: #535bf2; + --vp-c-brand-darker: #454ce1; + --vp-c-brand-dimm: #212425; +} + + /* Component: Button */ + +:root { + --vp-button-brand-border: var(--vp-c-brand-light); + --vp-button-brand-text: var(--vp-c-white); + --vp-button-brand-bg: var(--vp-c-brand); + --vp-button-brand-hover-border: var(--vp-c-brand-light); + --vp-button-brand-hover-text: var(--vp-c-white); + --vp-button-brand-hover-bg: var(--vp-c-brand-light); + --vp-button-brand-active-border: var(--vp-c-brand-light); + --vp-button-brand-active-text: var(--vp-c-white); + --vp-button-brand-active-bg: var(--vp-button-brand-bg); +} + +/* Component: Home */ + +:root { + --vp-home-hero-name-color: transparent; + --vp-home-hero-name-background: -webkit-linear-gradient( + 120deg, + #9558B2 30%, + #CB3C33 + ); + + --vp-home-hero-image-background-image: linear-gradient( + -45deg, + #9558B2 30%, + #389826 30%, + #CB3C33 + ); + --vp-home-hero-image-filter: blur(40px); +} + +@media (min-width: 640px) { + :root { + --vp-home-hero-image-filter: blur(56px); + } +} + +@media (min-width: 960px) { + :root { + --vp-home-hero-image-filter: blur(72px); + } +} + +/* Component: Custom Block */ + +:root.dark { + --vp-custom-block-tip-border: var(--vp-c-brand); + --vp-custom-block-tip-text: var(--vp-c-brand-lightest); + --vp-custom-block-tip-bg: var(--vp-c-brand-dimm); + + /* // Tweak the color palette for blacks and dark grays */ + --vp-c-black: hsl(220 20% 9%); + --vp-c-black-pure: hsl(220, 24%, 4%); + --vp-c-black-soft: hsl(220 16% 13%); + --vp-c-black-mute: hsl(220 14% 17%); + --vp-c-gray: hsl(220 8% 56%); + --vp-c-gray-dark-1: hsl(220 10% 39%); + --vp-c-gray-dark-2: hsl(220 12% 28%); + --vp-c-gray-dark-3: hsl(220 12% 23%); + --vp-c-gray-dark-4: hsl(220 14% 17%); + --vp-c-gray-dark-5: hsl(220 16% 13%); + + /* // Backgrounds */ + /* --vp-c-bg: hsl(240, 2%, 11%); */ + --vp-custom-block-info-bg: hsl(220 14% 17%); + /* --vp-c-gutter: hsl(220 20% 9%); + + --vp-c-bg-alt: hsl(220 20% 9%); + --vp-c-bg-soft: hsl(220 14% 17%); + --vp-c-bg-mute: hsl(220 12% 23%); + */ +} + + /* Component: Algolia */ + +.DocSearch { + --docsearch-primary-color: var(--vp-c-brand) !important; +} + +/* Component: MathJax */ + +mjx-container > svg { + display: block; + margin: auto; +} + +mjx-container { + padding: 0.5rem 0; +} + +mjx-container { + display: inline-block; + margin: auto 2px -2px; +} + +mjx-container > svg { + margin: auto; + display: inline-block; +} + +/** + * Colors links + * -------------------------------------------------------------------------- */ + + :root { + --vp-c-brand-1: #CB3C33; + --vp-c-brand-2: #CB3C33; + --vp-c-brand-3: #CB3C33; + --vp-c-sponsor: #ca2971; + --vitest-c-sponsor-hover: #c13071; +} + +.dark { + --vp-c-brand-1: #91dd33; + --vp-c-brand-2: #91dd33; + --vp-c-brand-3: #91dd33; + --vp-c-sponsor: #91dd33; + --vitest-c-sponsor-hover: #e51370; +} \ No newline at end of file diff --git a/docs/src/assets/favicon.ico b/docs/src/assets/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..9d67c8aedce9665286a15c35ae649fb539140e5a GIT binary patch literal 1529 zcmV004R>004l5008;`004mK004C`008P>0026e000+ooVrmw00009 za7bBm000ie000ie0hKEb8vpy{D4^001R)MObuXVRU6WV{&C- zbY%cCFflkSFf=VNHB>M#Iy5jkGchYLGCD9YVj~`90000bbVXQnWMOn=I&E)cX=Zr< zGB7bXEig1KFf~*#Fgi3aIx{gVFfuwYFcW^J*#H0p`$j%B<0Vt@vTN6VHpX66+-~~j4Vnt#Rr__E4)1XaovHsz28AdGNbuSA9R}C8|oW6MSqml zc0+u0BRYQkg)GEoyD22(F5o5L<+aIJxgj1H8bLjz50Fy04_Awo90WxuodJ$bUML74v4n!{x#rbmsh)H-GIqx0e9R7lW zy6NI@BVe;}qb@QSy?u6suQ0>Br;#&7&YL?O?z=!C)#}!?nEwI|?LEkOqZSwo2J{m+ zQ1T*7nNMl6fb~o-GKy+3H5x$C@FS;FkjH2j-2{%6=VHT?k8vK0oXB=(u2{Wz2=nPO zGM6=?vdT(Txd`D88PHr^h$mJo(`JEPpIA}6`x_uI7=lqxO+pTaZCDl`gYWCM;Nhsn zoadr1OOR(=W)+|cX43-3rX;MlRN&B8{Bm6Gy8%#bh_qOCQW9J?9QrsLudPi}iRyd#mI61_knqQ2oj)W0gRe0%M4Kyi zbRM-JC5bvnLtFO+`SuzJ8C`N|0W(!giV)-1y_L^3d>}>&p!eB58m?F`zjUXW* z2JO@h_b&?7n71*r0L2BDpr?{kQQvYJ1@C+YjMD~wc2vBd3G+*0x zKo@9$qzeQ_>=6AVv>n=n^kl6z=n_+?hSaiFF>3_6FZbiQSF2&WM)?!Rk(Ki}8a^!0 zW`Qez4&s^3Rp`G+Ct@gx)3Mg0ni#!~;;k7dF|Fe~kJ`NhGYW7{jw8)fgPs$VM~Xyz zawIycEm*?(Tf@86$MMmwRvbqN_s4;2Vcj11{OaB62BoO==dFRCp@}9 zy~WJ?jv{Ee2|cH-P#*`t+Ej?Xrk$E|^1_Oe@YAJ1I&ebhOrq0b#EAgWiL=&i=sdhl zbFM;1{&iZ#&j=QQY1B1$Du^sl5NlFO>RL6cqD(0geKcqftGCuGcw{qu^OfD7wy8$N z<(~HyUIH3Vb}K2l`{;=_B5=2KugWspl^z(hpH;b~%G0Jwb-BXoZXy^tT7$XQoLsg`!2Lc2^IA9k9fg{}n z1;QaT6%|n-n2?}U0fmI-T|3_UeBX~eklAa!^UlmWv)A59JLKUgBdID0gTZ8+ozRD2 zFd+{39Ni!eEUFK*Yrvn47o2<}U@*h&&_^idzI7BZsze_2i9CW2jf@Vuh=bXl#0TTF zoWp}o;SS@1P7(=SI1~(qFm*=T9*y}lHEerPtuvQCM-EJ0_Xl?1!YuXuR`Fyv?S4I&60jzRS7daOzX1ul439AbBvfv zrnH{?0Kb3$UDyC`qHwl(c(7w;VX7~F9k)b`SXDrk z-HcO?tC_nk-|78Jys@f3TEBe4I=(VGz1*&VL^V?8HuIzoY*s-Bv8IX2qPg z((KWMNhra-)7Bpj#r}@q_J7W{->uIvD66^}|M}+H6aJk|;uo3^$5oGx|9-Zz+T!Yy z-|QJ;k=V5`{u^w?l#yJm)E4_me)pC}U~VPWhy@8P22*@Jj=#?JihnBEx@UfoC$S*$ z=mhyLe|TnL&@Y((nmH+Y0Tusk+;ik@+<6OFnc6|C-aBjD=vC7|Q=-JAroq+(J;#N~ z&GKDh6lvEDT8yt_yQfdkrjmmsbow>(QM*2_U`*I; zysmsCxfGUKGbW5WKy+P+kB%5;`U?g3H}#bOAt!lZA13V{37SGAZimNCC^c?>-GqY3z~!wQX;Or{rR>|&6qe7^G#Tb zQGToH&>4W#X|L3}J$3$}%kL>#-IStu>)5LgzA}M3Fw_cOKmLR)u#2S~b1@^w37i{??pIvKhjjAh&yeA}->gnPRIOX69HoUZ(TF z`TdA9ALQIh_JbU3sUX)HM_=P>)%Px6G8g9ds>}JojvVCQF#c^hN0BSV=^D!cO9Z-6 zi5uPQ>Az|H*2If8)b(>(t&zZQ&nC`(R?AQDB7& zurA(mjj%+-H!L=Mt{KQPzkmWxAEeK#6R4kJ=hJ_9rZGvWC;Huzgl;%G9vGFPHc_d? z>-IcK*qHX{*K4uVc{c(VDVI~fw6OV!+-hhbd|7yO%u>aZ z_%lA6iJ~*#X5pi7dfr(t-B~;H+gIX7NXCj}v&ka``xzP!l$s7rZT>a%0_~}sW+JF-U*?OE5I}@J}*S%A>TXXf&OleSj zScTs-bv;0ReSloavSqCa6j#oJm0q?p-Rl-bxjXGsUX=PDS_E^j z;aa&C!6Z+r2%qJ!`B{<;SpW=pmUo9> zLdRE14F3dcvvp`hRM(H*;9vtOGg_l;*HY?DX+JI9E8?3Kd@{?GI-S zz}Wy)Zl2IAgtpWZP$r+e`0F?QkiiX;xQGJ{!(cYv+QaW{JAFV|Da4n;@4XaQzYz*g z!1lvcf=L|+U-j8%z=>?}xgD?Z>OgOUayL-6p=PtY7Q;5J))jf;27G*7vb8jiqRd7P zULDqv=Oo-CP9g)Gimr_5A4TXV+*0n|Z)OCd7JCY555zrCby~YG7UVUpdi({5yW6o% zL!R?@`_LlGfN)%>VsnVmWkP5n8O)kg(G<#-NMG1ya^)C)`#uh z9}=9&#CqC-nO2sW&^`q9(mdAv?pe^O>eNGU-M4}9uyorKB!pCi6PMOTT*RFGx`F78dys72M z8|7fl5npWP^R)28B4Ju(nXT=QP1-@D1K}7}yPvhuJ@ChN4P6^bg5=dJUY0pJ*7%o& zoQ~?WL(IZTrBv;;f2!gZDVwt4{YD6-)9 zrVx&q=}>vSmqww2{Xwe~{_;LEtx9LE@La2h0>%MCYq{Bcn_%)`(PXjbv|?81f&U>& zoL~i4OosIIxiy~tVE+emg*4IGzAKPO8eZuQ}kof!TAKDm)3#>7urbBmMgYqPt))=W)tj*ZqQMAf!D~Dw% z2D37l6_mYlsivXY{h-<6(?C~(G{!~9bW{k$d2H`0H+IkA)QoELxx&r0XcY<>3iFNA zR;VI)L-jQnkg$R-ji~w1gAL%YoutR>Byo&G%OK_@Qa`%$9RRlSgDuU_D0whVVFEb~$t6T8prPQMl-rozj83%Gya zV=~MQq2%`D?>hbkjzfjck0EF80%4N&cJ*cQp_60zZG)aw?#j1|iaff}I7m^$Dq_6=(Z&hXfi2P~5QHTF|})RxAzPutzb2-QDNLFB-O2LaTW1 z9sh4K_nYmA@P>rz^WMqaDJpc=s6o@WO@eO?nh+!IJ^WtiDmga{ni&wIi0pb*$Qk4A z$1T@9;}}^~qP{Y?UaKT9de1yF5J+Fe-9* zy#W{Bt|j&x)4_nQxV3}Pdar#*CvG8Mz;rb<*#ILBWD3TcKp>!JyF)QXurg_BNcBs^ z>`+OuPGwpnHq;uFJaJAiEuqbbkV2r;+%BjFqYeKblUrlC943wtf3QOUd-3Azn_?ma z%z3m}2ee)fh%z0!`#ypxm*4$uNV-hA#}m3M#dqB`pRKybVw

+|zf5p2Ymx$*&T`aBW^AzrpPu4%X)WEUd?@Gvh=?dDDGda6E zt*=}<`p3`!O@m{PSSz5qD~OomzD8TrO?S@g_93`m>hM zDlyArL-${;g_f<}o6Gh5MKy(wZ)E`Vhe{RGGv#J*pN+dc`!|$H7m3r5qOkQgd6;L4 zY+#J2s=Wic9X#1`q#7Lq~~dy55L21Wwc)Ix>)!R#qGsuEpzJacFCPtOM~&h z-?*swZ1oK_Sv=k^!%u#s-zL8$fW*EyG!`$9(=znsI^VzZ6;GNF6EzVaKVBsVEN^4j zxYe|L6Z5TUG5R+=PNf?h9jeUdvRrj$b=Hozz;S`THD7e7bNORSq<|@SuY`TI9`J}X zd~snqYGG`6__q@|k=SLLVZw~D?rQnY%%rdKScv?7xi#$2XXkB<)_+bGh^63L`ac_J z*Oe=E`sDNII~J$`>H#uq!e!EG`38J?n=|*7_b(RjtOB#)q}3vTwWr0rgbE`p4LvIkaENlkdrzz+qfXPf{Cl zM0bUb=(X9>=`+|#8O}H5-&bOHTUxu36PejNm^|azD<)JhtytyL?Cu%Y0@B2IS6N*R zS_NE;Nxd1HJ*1j`5Znc*gzwjet+W`A4)~sbik0WM?xMU%A1tb>iNS^FaZFWMdJm29;oeSEc?(wo}?!6AUARWaPWTf%=MPf@jtp>=J z*$}BQML$SRWM^|eFgSDi>n^MeJl5*wC%qKCyJIH{jv12Ju{=6dGb*7A)s^U2mI=?{ z&>6UAtXUpCgI(@U3Q5)n%6H2%T4sD;;ApIXFYjzfb6J}!(JB@?zE-mx9~roklFPPS zME^}AYW1sJJrPHpZlt%N1d*Imwz(Z7Xx{R1eQTO&-(KJI6_#>AH7Gz~j&%sQ$#5=GuOZv2_Ry!O z?@W&Mu`^GFB{AX^*wD8B;$O%MKbdA)f$bBa65KL5N%R@lp`uF& z-!f_a`~j|ZRd`WQ=1}a;l5LeDaGXzL(fOpIn8O4D39^-isYA;Y+>E<&(p{k{swQ=}3QV8pwDXyBS>2u6_0cL)3b-5|{mI%d zDQbAl%He&tIr2CICo+jWceVttcH{@nq;BP~x$FTr&cH3tkA=ADwNiW(>;HoTM5^zf zR?n0?DAmd+cd=!oIp; zv^#heoF1(MB0*zO3iSANc0rWk1lDYcJssrw2GO+{qT0e zmy)ADzQ#=HKuhxcz>;e#Inat>Z2_)(-KNf6u_=cYNfsK-&d??S8?WW!)FWWu$Wl4m z)!vqDvK#Zs;plg3hnM+Ms&YB+Fs&0{beNd=Ms=B&xx}v}^|xtX`a4J#q*}nuo;0P9 z;Sx`LnSZQPL+A0`g!jfNdLa&i!f)u6WYN3B<$bkFyzk5W z4hp^^0aVSYW(w$JgXb(3$05`$gU`c* z=5s$~OE<>3fio~3J5WUUkR%{}*@XLD-QiRzB?-Ts4m~_c>q4aWNp&caEa^P@uO5~D zTSx<-I=s8dgpRC~GAc;~-XW1%lYMyf?%+&3R;h^~Fp}5=XpnD3?{4ZfEd*{lh`I|PQDtTiJP9? z6)i(;$INGGR?Z`3?}0~tD#7m?T?`lj&6GJ57EDx#rC0mqFflYSxC|OBxwN>V@TWJ* zZIOlm(K-7e@>U$uW7Y8UoqtZXzr z_6+sn>n<>J`jfW`Y)c&<{a%BGKDsmWl|8gfboIfC^VnyHiz$kr6D`!644-b*gRc$S zl226};?dhI)R!$#NYReX)lfz}6~_c)qFaW7#23}WFvWU6G9X2*kzWQR{`)PgQD}ji zwXj8tq0mmjqxw4iGnp!qtt~%WB{!>+q5OB2kWcGQXuL&`T_= zs-!%J4O|*LhP@GF6e;Y5yhWf~|oDbS;m8vje zh!PdurD4eiKOX6GWNb&+5eirR?1q#j!Btec-cm}Fze@TQ3l+&8`y-nQ`@pHs@MBJha+_D6MZ8({bIEak=}|vrm=J@h%&lLW;JGNna>j3% zfp^lfUt^8v`RmJx&LBbrjRBx#?WM)bWh zff|((;8mXVh?23~pombJnmALLa`QYEV70mKX+_O@U+UxvrPf>%MyUfzXkR;_V&&lm z@G|T6p)UoZ+|nF%@CttvA+m5`yO0(msKIPwPg@@x*mxf^=Jv;XwVp(&3CVMwO^nzo z25Lm*!OAL{Bs~76yhnjw7oJ=hO?#6irN!9#?YWeJe(2tQ@E$POYK8vfN??@9q`bBb z+5UAa%;!+gKGBWxoDCyO!K5?a61!qF)xe$~S&3lOL2(vIig8O3O2F&0_u@UBv9W(2 z#`L}C%?ihRo(OLx6*a?lDREGIniTIvq=GQ`u|tY?c`2=7oD$1+uoDWM%i)WRM%^zu zXsM}PP@DX;Q2uO?41RD`=t2N_!8W^!%D2jxpWJA@RPCvge_cLTekjKeX$X_xPb@Dl za|5XSxaO71{MM@IgCf5?^6=FDa?XyFZO`+>j2VOVziV9Keob?A*BgGl*kF1V*Tpao z*hDJ+90ETHldB}pR*2soKDOoC>YbcUFAoQjku)h-n4(UM0!Kgb RD)@;F=6uitU1=AX^dA img { max-height: 10rem; diff --git a/docs/src/index.md b/docs/src/index.md index 1b6e35f..a700e2d 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,30 +1,29 @@ -# EvoLinear.jl - -ML library implementing linear boosting with L1 and L2 regularization. - -For tree based boosting, consider [EvoTrees.jl](https://github.com/Evovest/EvoTrees.jl). - -Supported loss functions: - -- mse (squared-error) -- logistic (logloss) regression -- poisson -- gamma -- tweedie - -## Installation - -``` -pkg> add https://github.com/jeremiedb/EvoLinear.jl +```@raw html +--- +layout: home + +hero: + text: "EvoLinear.jl" + tagline: Regularized linear models for tabular data + image: + src: /logo.png + alt: Evovest + + actions: + - theme: brand + text: Quick start + link: /quick-start + - theme: alt + text: Design + link: /design + - theme: alt + text: Models + link: /models + - theme: alt + text: Tutorials + link: /tutorials-logistic-titanic.md + - theme: alt + text: Source code + link: https://github.com/jeremiedb/EvoLinear.jl +--- ``` - -## Getting started - -Build a configuration struct with `EvoLinearRegressor`. Then `EvoLinear.fit` takes `x::Matrix` and `y::Vector` as inputs, plus optionally `w::Vector` as weights and fits a linear boosted model. - -```julia -using EvoLinear -config = EvoLinearRegressor(loss=:mse, L1=1e-1, L2=1e-2, nrounds=10) -m = EvoLinear.fit(config; x, y, metric=:mse) -p = EvoLinear.predict_proj(m, x) -``` \ No newline at end of file diff --git a/docs/src/intro.md b/docs/src/intro.md new file mode 100644 index 0000000..1b6e35f --- /dev/null +++ b/docs/src/intro.md @@ -0,0 +1,30 @@ +# EvoLinear.jl + +ML library implementing linear boosting with L1 and L2 regularization. + +For tree based boosting, consider [EvoTrees.jl](https://github.com/Evovest/EvoTrees.jl). + +Supported loss functions: + +- mse (squared-error) +- logistic (logloss) regression +- poisson +- gamma +- tweedie + +## Installation + +``` +pkg> add https://github.com/jeremiedb/EvoLinear.jl +``` + +## Getting started + +Build a configuration struct with `EvoLinearRegressor`. Then `EvoLinear.fit` takes `x::Matrix` and `y::Vector` as inputs, plus optionally `w::Vector` as weights and fits a linear boosted model. + +```julia +using EvoLinear +config = EvoLinearRegressor(loss=:mse, L1=1e-1, L2=1e-2, nrounds=10) +m = EvoLinear.fit(config; x, y, metric=:mse) +p = EvoLinear.predict_proj(m, x) +``` \ No newline at end of file From 10a81e4cff1a56e7e51817d24a8bf6bf50d0f116 Mon Sep 17 00:00:00 2001 From: "jeremie.db" Date: Fri, 14 Jun 2024 17:23:18 -0400 Subject: [PATCH 6/9] vitepress docs --- experiments/MLJ-spline.jl | 78 --------------------------------------- experiments/spline.jl | 69 ---------------------------------- 2 files changed, 147 deletions(-) delete mode 100644 experiments/MLJ-spline.jl delete mode 100644 experiments/spline.jl diff --git a/experiments/MLJ-spline.jl b/experiments/MLJ-spline.jl deleted file mode 100644 index b8458e9..0000000 --- a/experiments/MLJ-spline.jl +++ /dev/null @@ -1,78 +0,0 @@ -using Revise -using EvoLinear -using EvoLinear: logit, sigmoid -using StatsBase: sample -using MLJBase - -################################################## -### Regression - small data -################################################## -features = rand(10_000) .* 5 .- 2 -X = reshape(features, (size(features)[1], 1)) -Y = sin.(features) .* 0.5 .+ 0.5 -Y = logit(Y) + randn(size(Y)) -Y = sigmoid(Y) -y = Y -X = MLJBase.table(X) - -# linear regression -model = EvoSplineRegressor(loss=:mse, nrounds=10, knots = Dict(1 => 4)) -mach = machine(model, X, y) -train, test = partition(eachindex(y), 0.7, shuffle=true); # 70:30 split -fit!(mach, rows=train, verbosity=1) - -mach.model.nrounds += 2 -fit!(mach, rows=train, verbosity=1) -mach.cache[:info][:nrounds] - -# predict on train data -pred_train = predict(mach, selectrows(X, train)) -mean(abs.(pred_train - selectrows(Y, train))) - -# predict on test data -pred_test = predict(mach, selectrows(X, test)) -mean(abs.(pred_test - selectrows(Y, test))) - -@test MLJBase.iteration_parameter(EvoLinearRegressor) == :nrounds - - -################################################## -### Regression - matrix data -################################################## -X = MLJBase.matrix(X) -model = EvoLinearRegressor(loss=:logistic, nrounds=4) - -mach = machine(model, X, y) -train, test = partition(eachindex(y), 0.7, shuffle=true); # 70:30 split -fit!(mach, rows=train, verbosity=1) - -mach.model.nrounds += 2 -fit!(mach, rows=train, verbosity=1) - -pred_train = predict(mach, selectrows(X, train)) -mean(abs.(pred_train - selectrows(Y, train))) - - -#################################################################################### -# tests that `update` handles data correctly in the case of a cold restart: -#################################################################################### -X = MLJBase.table(rand(5, 2)) -y = rand(5) -model = EvoLinearRegressor(loss=:mse) -data = MLJBase.reformat(model, X, y); -f, c, r = MLJBase.fit(model, 2, data...); -c[:info] -model.L2 = 0.1 -model.nrounds += 2 -MLJBase.update(model, 2, f, c, data...) -c[:info][:nrounds] - -X = rand(5, 2) -y = rand(5) -model = EvoLinearRegressor(loss=:mse) -data = MLJBase.reformat(model, X, y); -f, c, r = MLJBase.fit(model, 2, data...); -model.L2 = 0.1 -model.nrounds += 2 -MLJBase.update(model, 2, f, c, data...) -MLJBase.update(model, 2, f, c, data...) \ No newline at end of file diff --git a/experiments/spline.jl b/experiments/spline.jl deleted file mode 100644 index cff33b4..0000000 --- a/experiments/spline.jl +++ /dev/null @@ -1,69 +0,0 @@ -using Revise -using EvoLinear -using AlgebraOfGraphics, GLMakie -using DataFrames -using BenchmarkTools -using Random: seed! - -nobs = 1_000 -nfeats = 1 -T = Float32 - -seed!(123) -x_train = rand(T, nobs, nfeats) .* 2 -coef = randn(T, nfeats) - -y_train = sin.(x_train[:, 1]) .+ randn(T, nobs) .* 0.1f0 -df = DataFrame(hcat(x_train, y_train), ["x", "y"]); -draw(data(df) * mapping(:x, :y) * visual(Scatter, markersize = 5, color = "gray")) - -config = EvoLinearRegressor(nrounds = 100, loss = :mse, L1 = 0e-1, L2 = 1) -@time ml = EvoLinear.fit( - config; - x_train, - y_train, - x_eval = x_train, - y_eval = y_train, - metric = :mae, - print_every_n = 10, - early_stopping_rounds = 5 -); - -x_pred = - reshape(range(start = minimum(x_train), stop = maximum(x_train), length = 100), :, 1) -pl = ml(x_pred) - -dfp = DataFrame(hcat(x_pred, pl), ["x", "p"]); -plt = - data(df) * mapping(:x, :y) * visual(Scatter, markersize = 5, color = "gray") + - data(dfp) * mapping(:x, :p) * visual(Lines, markersize = 5, color = "navy") -draw(plt) - -config = EvoSplineRegressor( - loss = :mse, - nrounds = 200, - knots = Dict(1 => 4), - act = :elu, - eta = 1e-2, - batchsize = 200, -) -ms = EvoLinear.fit(config; x_train, y_train) -@time ms = EvoLinear.fit( - config; - x_train, - y_train, - x_eval = x_train, - y_eval = y_train, - metric = :mae, - print_every_n = 10, - early_stopping_rounds = 5 -); -# fit!(loss, m, dtrain, opts) -pl = ml(x_pred) -ps = ms(x_pred') -dfp = DataFrame(hcat(x_pred, pl, ps), ["x", "p_linear", "p_spline"]); -plt = - data(df) * mapping(:x, :y) * visual(Scatter, markersize = 5, color = "gray") + - data(dfp) * mapping(:x, :p_linear) * visual(Lines, markersize = 5, color = "navy") + - data(dfp) * mapping(:x, :p_spline) * visual(Lines, markersize = 5, color = "darkgreen") -draw(plt) From 639873a3d03c9aaa28e8d73afc25f01e3935825f Mon Sep 17 00:00:00 2001 From: "jeremie.db" Date: Fri, 14 Jun 2024 17:27:49 -0400 Subject: [PATCH 7/9] up --- docs/src/assets/favicon.ico | Bin 0 -> 1529 bytes docs/src/assets/logo.png | Bin 0 -> 7077 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/src/assets/favicon.ico create mode 100644 docs/src/assets/logo.png diff --git a/docs/src/assets/favicon.ico b/docs/src/assets/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..9d67c8aedce9665286a15c35ae649fb539140e5a GIT binary patch literal 1529 zcmV004R>004l5008;`004mK004C`008P>0026e000+ooVrmw00009 za7bBm000ie000ie0hKEb8vpy{D4^001R)MObuXVRU6WV{&C- zbY%cCFflkSFf=VNHB>M#Iy5jkGchYLGCD9YVj~`90000bbVXQnWMOn=I&E)cX=Zr< zGB7bXEig1KFf~*#Fgi3aIx{gVFfuwYFcW^J*#H0p`$j%B<0Vt@vTN6VHpX66+-~~j4Vnt#Rr__E4)1XaovHsz28AdGNbuSA9R}C8|oW6MSqml zc0+u0BRYQkg)GEoyD22(F5o5L<+aIJxgj1H8bLjz50Fy04_Awo90WxuodJ$bUML74v4n!{x#rbmsh)H-GIqx0e9R7lW zy6NI@BVe;}qb@QSy?u6suQ0>Br;#&7&YL?O?z=!C)#}!?nEwI|?LEkOqZSwo2J{m+ zQ1T*7nNMl6fb~o-GKy+3H5x$C@FS;FkjH2j-2{%6=VHT?k8vK0oXB=(u2{Wz2=nPO zGM6=?vdT(Txd`D88PHr^h$mJo(`JEPpIA}6`x_uI7=lqxO+pTaZCDl`gYWCM;Nhsn zoadr1OOR(=W)+|cX43-3rX;MlRN&B8{Bm6Gy8%#bh_qOCQW9J?9QrsLudPi}iRyd#mI61_knqQ2oj)W0gRe0%M4Kyi zbRM-JC5bvnLtFO+`SuzJ8C`N|0W(!giV)-1y_L^3d>}>&p!eB58m?F`zjUXW* z2JO@h_b&?7n71*r0L2BDpr?{kQQvYJ1@C+YjMD~wc2vBd3G+*0x zKo@9$qzeQ_>=6AVv>n=n^kl6z=n_+?hSaiFF>3_6FZbiQSF2&WM)?!Rk(Ki}8a^!0 zW`Qez4&s^3Rp`G+Ct@gx)3Mg0ni#!~;;k7dF|Fe~kJ`NhGYW7{jw8)fgPs$VM~Xyz zawIycEm*?(Tf@86$MMmwRvbqN_s4;2Vcj11{OaB62BoO==dFRCp@}9 zy~WJ?jv{Ee2|cH-P#*`t+Ej?Xrk$E|^1_Oe@YAJ1I&ebhOrq0b#EAgWiL=&i=sdhl zbFM;1{&iZ#&j=QQY1B1$Du^sl5NlFO>RL6cqD(0geKcqftGCuGcw{qu^OfD7wy8$N z<(~HyUIH3Vb}K2l`{;=_B5=2KugWspl^z(hpH;b~%G0Jwb-BXoZXy^tT7$XQoLsg`!2Lc2^IA9k9fg{}n z1;QaT6%|n-n2?}U0fmI-T|3_UeBX~eklAa!^UlmWv)A59JLKUgBdID0gTZ8+ozRD2 zFd+{39Ni!eEUFK*Yrvn47o2<}U@*h&&_^idzI7BZsze_2i9CW2jf@Vuh=bXl#0TTF zoWp}o;SS@1P7(=SI1~(qFm*=T9*y}lHEerPtuvQCM-EJ0_Xl?1!YuXuR`Fyv?S4I&60jzRS7daOzX1ul439AbBvfv zrnH{?0Kb3$UDyC`qHwl(c(7w;VX7~F9k)b`SXDrk z-HcO?tC_nk-|78Jys@f3TEBe4I=(VGz1*&VL^V?8HuIzoY*s-Bv8IX2qPg z((KWMNhra-)7Bpj#r}@q_J7W{->uIvD66^}|M}+H6aJk|;uo3^$5oGx|9-Zz+T!Yy z-|QJ;k=V5`{u^w?l#yJm)E4_me)pC}U~VPWhy@8P22*@Jj=#?JihnBEx@UfoC$S*$ z=mhyLe|TnL&@Y((nmH+Y0Tusk+;ik@+<6OFnc6|C-aBjD=vC7|Q=-JAroq+(J;#N~ z&GKDh6lvEDT8yt_yQfdkrjmmsbow>(QM*2_U`*I; zysmsCxfGUKGbW5WKy+P+kB%5;`U?g3H}#bOAt!lZA13V{37SGAZimNCC^c?>-GqY3z~!wQX;Or{rR>|&6qe7^G#Tb zQGToH&>4W#X|L3}J$3$}%kL>#-IStu>)5LgzA}M3Fw_cOKmLR)u#2S~b1@^w37i{??pIvKhjjAh&yeA}->gnPRIOX69HoUZ(TF z`TdA9ALQIh_JbU3sUX)HM_=P>)%Px6G8g9ds>}JojvVCQF#c^hN0BSV=^D!cO9Z-6 zi5uPQ>Az|H*2If8)b(>(t&zZQ&nC`(R?AQDB7& zurA(mjj%+-H!L=Mt{KQPzkmWxAEeK#6R4kJ=hJ_9rZGvWC;Huzgl;%G9vGFPHc_d? z>-IcK*qHX{*K4uVc{c(VDVI~fw6OV!+-hhbd|7yO%u>aZ z_%lA6iJ~*#X5pi7dfr(t-B~;H+gIX7NXCj}v&ka``xzP!l$s7rZT>a%0_~}sW+JF-U*?OE5I}@J}*S%A>TXXf&OleSj zScTs-bv;0ReSloavSqCa6j#oJm0q?p-Rl-bxjXGsUX=PDS_E^j z;aa&C!6Z+r2%qJ!`B{<;SpW=pmUo9> zLdRE14F3dcvvp`hRM(H*;9vtOGg_l;*HY?DX+JI9E8?3Kd@{?GI-S zz}Wy)Zl2IAgtpWZP$r+e`0F?QkiiX;xQGJ{!(cYv+QaW{JAFV|Da4n;@4XaQzYz*g z!1lvcf=L|+U-j8%z=>?}xgD?Z>OgOUayL-6p=PtY7Q;5J))jf;27G*7vb8jiqRd7P zULDqv=Oo-CP9g)Gimr_5A4TXV+*0n|Z)OCd7JCY555zrCby~YG7UVUpdi({5yW6o% zL!R?@`_LlGfN)%>VsnVmWkP5n8O)kg(G<#-NMG1ya^)C)`#uh z9}=9&#CqC-nO2sW&^`q9(mdAv?pe^O>eNGU-M4}9uyorKB!pCi6PMOTT*RFGx`F78dys72M z8|7fl5npWP^R)28B4Ju(nXT=QP1-@D1K}7}yPvhuJ@ChN4P6^bg5=dJUY0pJ*7%o& zoQ~?WL(IZTrBv;;f2!gZDVwt4{YD6-)9 zrVx&q=}>vSmqww2{Xwe~{_;LEtx9LE@La2h0>%MCYq{Bcn_%)`(PXjbv|?81f&U>& zoL~i4OosIIxiy~tVE+emg*4IGzAKPO8eZuQ}kof!TAKDm)3#>7urbBmMgYqPt))=W)tj*ZqQMAf!D~Dw% z2D37l6_mYlsivXY{h-<6(?C~(G{!~9bW{k$d2H`0H+IkA)QoELxx&r0XcY<>3iFNA zR;VI)L-jQnkg$R-ji~w1gAL%YoutR>Byo&G%OK_@Qa`%$9RRlSgDuU_D0whVVFEb~$t6T8prPQMl-rozj83%Gya zV=~MQq2%`D?>hbkjzfjck0EF80%4N&cJ*cQp_60zZG)aw?#j1|iaff}I7m^$Dq_6=(Z&hXfi2P~5QHTF|})RxAzPutzb2-QDNLFB-O2LaTW1 z9sh4K_nYmA@P>rz^WMqaDJpc=s6o@WO@eO?nh+!IJ^WtiDmga{ni&wIi0pb*$Qk4A z$1T@9;}}^~qP{Y?UaKT9de1yF5J+Fe-9* zy#W{Bt|j&x)4_nQxV3}Pdar#*CvG8Mz;rb<*#ILBWD3TcKp>!JyF)QXurg_BNcBs^ z>`+OuPGwpnHq;uFJaJAiEuqbbkV2r;+%BjFqYeKblUrlC943wtf3QOUd-3Azn_?ma z%z3m}2ee)fh%z0!`#ypxm*4$uNV-hA#}m3M#dqB`pRKybVw

+|zf5p2Ymx$*&T`aBW^AzrpPu4%X)WEUd?@Gvh=?dDDGda6E zt*=}<`p3`!O@m{PSSz5qD~OomzD8TrO?S@g_93`m>hM zDlyArL-${;g_f<}o6Gh5MKy(wZ)E`Vhe{RGGv#J*pN+dc`!|$H7m3r5qOkQgd6;L4 zY+#J2s=Wic9X#1`q#7Lq~~dy55L21Wwc)Ix>)!R#qGsuEpzJacFCPtOM~&h z-?*swZ1oK_Sv=k^!%u#s-zL8$fW*EyG!`$9(=znsI^VzZ6;GNF6EzVaKVBsVEN^4j zxYe|L6Z5TUG5R+=PNf?h9jeUdvRrj$b=Hozz;S`THD7e7bNORSq<|@SuY`TI9`J}X zd~snqYGG`6__q@|k=SLLVZw~D?rQnY%%rdKScv?7xi#$2XXkB<)_+bGh^63L`ac_J z*Oe=E`sDNII~J$`>H#uq!e!EG`38J?n=|*7_b(RjtOB#)q}3vTwWr0rgbE`p4LvIkaENlkdrzz+qfXPf{Cl zM0bUb=(X9>=`+|#8O}H5-&bOHTUxu36PejNm^|azD<)JhtytyL?Cu%Y0@B2IS6N*R zS_NE;Nxd1HJ*1j`5Znc*gzwjet+W`A4)~sbik0WM?xMU%A1tb>iNS^FaZFWMdJm29;oeSEc?(wo}?!6AUARWaPWTf%=MPf@jtp>=J z*$}BQML$SRWM^|eFgSDi>n^MeJl5*wC%qKCyJIH{jv12Ju{=6dGb*7A)s^U2mI=?{ z&>6UAtXUpCgI(@U3Q5)n%6H2%T4sD;;ApIXFYjzfb6J}!(JB@?zE-mx9~roklFPPS zME^}AYW1sJJrPHpZlt%N1d*Imwz(Z7Xx{R1eQTO&-(KJI6_#>AH7Gz~j&%sQ$#5=GuOZv2_Ry!O z?@W&Mu`^GFB{AX^*wD8B;$O%MKbdA)f$bBa65KL5N%R@lp`uF& z-!f_a`~j|ZRd`WQ=1}a;l5LeDaGXzL(fOpIn8O4D39^-isYA;Y+>E<&(p{k{swQ=}3QV8pwDXyBS>2u6_0cL)3b-5|{mI%d zDQbAl%He&tIr2CICo+jWceVttcH{@nq;BP~x$FTr&cH3tkA=ADwNiW(>;HoTM5^zf zR?n0?DAmd+cd=!oIp; zv^#heoF1(MB0*zO3iSANc0rWk1lDYcJssrw2GO+{qT0e zmy)ADzQ#=HKuhxcz>;e#Inat>Z2_)(-KNf6u_=cYNfsK-&d??S8?WW!)FWWu$Wl4m z)!vqDvK#Zs;plg3hnM+Ms&YB+Fs&0{beNd=Ms=B&xx}v}^|xtX`a4J#q*}nuo;0P9 z;Sx`LnSZQPL+A0`g!jfNdLa&i!f)u6WYN3B<$bkFyzk5W z4hp^^0aVSYW(w$JgXb(3$05`$gU`c* z=5s$~OE<>3fio~3J5WUUkR%{}*@XLD-QiRzB?-Ts4m~_c>q4aWNp&caEa^P@uO5~D zTSx<-I=s8dgpRC~GAc;~-XW1%lYMyf?%+&3R;h^~Fp}5=XpnD3?{4ZfEd*{lh`I|PQDtTiJP9? z6)i(;$INGGR?Z`3?}0~tD#7m?T?`lj&6GJ57EDx#rC0mqFflYSxC|OBxwN>V@TWJ* zZIOlm(K-7e@>U$uW7Y8UoqtZXzr z_6+sn>n<>J`jfW`Y)c&<{a%BGKDsmWl|8gfboIfC^VnyHiz$kr6D`!644-b*gRc$S zl226};?dhI)R!$#NYReX)lfz}6~_c)qFaW7#23}WFvWU6G9X2*kzWQR{`)PgQD}ji zwXj8tq0mmjqxw4iGnp!qtt~%WB{!>+q5OB2kWcGQXuL&`T_= zs-!%J4O|*LhP@GF6e;Y5yhWf~|oDbS;m8vje zh!PdurD4eiKOX6GWNb&+5eirR?1q#j!Btec-cm}Fze@TQ3l+&8`y-nQ`@pHs@MBJha+_D6MZ8({bIEak=}|vrm=J@h%&lLW;JGNna>j3% zfp^lfUt^8v`RmJx&LBbrjRBx#?WM)bWh zff|((;8mXVh?23~pombJnmALLa`QYEV70mKX+_O@U+UxvrPf>%MyUfzXkR;_V&&lm z@G|T6p)UoZ+|nF%@CttvA+m5`yO0(msKIPwPg@@x*mxf^=Jv;XwVp(&3CVMwO^nzo z25Lm*!OAL{Bs~76yhnjw7oJ=hO?#6irN!9#?YWeJe(2tQ@E$POYK8vfN??@9q`bBb z+5UAa%;!+gKGBWxoDCyO!K5?a61!qF)xe$~S&3lOL2(vIig8O3O2F&0_u@UBv9W(2 z#`L}C%?ihRo(OLx6*a?lDREGIniTIvq=GQ`u|tY?c`2=7oD$1+uoDWM%i)WRM%^zu zXsM}PP@DX;Q2uO?41RD`=t2N_!8W^!%D2jxpWJA@RPCvge_cLTekjKeX$X_xPb@Dl za|5XSxaO71{MM@IgCf5?^6=FDa?XyFZO`+>j2VOVziV9Keob?A*BgGl*kF1V*Tpao z*hDJ+90ETHldB}pR*2soKDOoC>YbcUFAoQjku)h-n4(UM0!Kgb RD)@;F=6uitU1=AX^dA Date: Fri, 14 Jun 2024 18:14:43 -0400 Subject: [PATCH 8/9] up --- Project.toml | 3 - docs/src/tutorials-logistic-titanic.md | 101 ++++++++++++++++++++++++ docs/src/tutorials-regression-boston.md | 79 ++++++++++++++++++ test/linear.jl | 56 +++++-------- 4 files changed, 200 insertions(+), 39 deletions(-) create mode 100644 docs/src/tutorials-logistic-titanic.md create mode 100644 docs/src/tutorials-regression-boston.md diff --git a/Project.toml b/Project.toml index cb7bdc8..c27978f 100644 --- a/Project.toml +++ b/Project.toml @@ -8,9 +8,7 @@ Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" LoopVectorization = "bdcacae8-1622-11e9-2a5c-532679323890" MLJModelInterface = "e80e1ace-859a-464e-9ed9-23947d8ae3ea" -Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" -SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" @@ -19,7 +17,6 @@ Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Distributions = "0.25" LoopVectorization = "0.12" MLJModelInterface = "1.0" -Optimisers = "0.2, 0.3" Statistics = "1" StatsBase = "0.33, 0.34" julia = "1.6" diff --git a/docs/src/tutorials-logistic-titanic.md b/docs/src/tutorials-logistic-titanic.md new file mode 100644 index 0000000..f5bf3f8 --- /dev/null +++ b/docs/src/tutorials-logistic-titanic.md @@ -0,0 +1,101 @@ +# Logistic Regression on Titanic Dataset + +We will use the Titanic dataset, which is included in the MLDatasets package. It describes the survival status of individual passengers on the Titanic. The model will be approached as a logistic regression problem, although a Classifier model could also have been used (see the `Classification - Iris` tutorial). + +## Getting started + +To begin, we will load the required packages and the dataset: + +```julia +using EvoLinear +using MLDatasets +using DataFrames +using Statistics: mean +using StatsBase: median +using CategoricalArrays +using Random +Random.seed!(123) +``` + +## Preprocessing + +A first step in data processing is to prepare the input features in a model compatible format. + +EvoLinear's Tables API supports input that are either `Real` (incl. `Bool`) or `Categorical`. `Bool` variables are treated as unordered, 2-levels categorical variables. +A recommended approach for `String` features such as `Sex` is to convert them into an unordered `Categorical`. + +For dealing with features with missing values such as `Age`, a common approach is to first create an `Bool` indicator variable capturing the info on whether a value is missing. +Then, the missing values can be imputed (replaced by some default values such as `mean` or `median`, or more sophisticated approach such as predictions from another model). + +```julia +df = MLDatasets.Titanic().dataframe + +# convert string feature to Categorical +transform!(df, :Sex => categorical => :Sex) +transform!(df, :Sex => ByRow(levelcode) => :Sex) + +# treat string feature and missing values +transform!(df, :Age => ByRow(ismissing) => :Age_ismissing) +transform!(df, :Age => (x -> coalesce.(x, median(skipmissing(x)))) => :Age); + +# remove unneeded variables +df = df[:, Not([:PassengerId, :Name, :Embarked, :Cabin, :Ticket])] + +``` + +The full data can now be split according to train and eval indices. +Target and feature names are also set. + +```julia +train_ratio = 0.8 +train_indices = randperm(nrow(df))[1:Int(round(train_ratio * nrow(df)))] + +dtrain = df[train_indices, :] +deval = df[setdiff(1:nrow(df), train_indices), :] + +target_name = "Survived" +feature_names = setdiff(names(df), [target_name]) +``` + +## Training + +Now we are ready to train our model. We first define a model configuration using the [`NeuroTreeRegressor`](@ref) model constructor. +Then, we use [`NeuroTreeModels.fit`](@ref) to train a boosted tree model. We pass the optional `deval` argument to enable the usage of early stopping. + +```julia +config = EvoLinearRegressor( + loss=:logistic, + nrounds=2000, + L2=0.1, + eta=1e-1, +) + +m = EvoLinear.fit( + config, + dtrain; + deval, + target_name, + feature_names, + metric=:logloss, + print_every_n=100, + early_stopping_rounds=10, +) +``` + + +## Diagnosis + +We can get predictions by passing training and testing data to our model. We can then evaluate the accuracy of our model, which should be around 85%. + +```julia +p_train = m(dtrain) +p_eval = m(deval) +``` + +```julia-repl +julia> mean((p_train .> 0.5) .== dtrain[!, target_name]) +0.8036465638148668 + +julia> mean((p_eval .> 0.5) .== deval[!, target_name]) +0.797752808988764 +``` diff --git a/docs/src/tutorials-regression-boston.md b/docs/src/tutorials-regression-boston.md new file mode 100644 index 0000000..60a5bbf --- /dev/null +++ b/docs/src/tutorials-regression-boston.md @@ -0,0 +1,79 @@ +# Regression on Boston Housing Dataset + +We will use the Boston Housing dataset, which is included in the MLDatasets package. It's derived from information collected by the U.S. Census Service concerning housing in the area of Boston. Target variable represents the median housing value. + +## Getting started + +To begin, we will load the required packages and the dataset: + +```julia +using EvoLinear +using MLDatasets +using DataFrames +using Statistics: mean, std +using CategoricalArrays +using Random +Random.seed!(123) +``` + +## Preprocessing + +Before we can train our model, we need to preprocess the dataset. We will split our data according to train and eval indices, and separate features from the target variable. + +```julia +df = MLDatasets.BostonHousing().dataframe +feature_names = setdiff(names(df), ["MEDV"]) + +train_ratio = 0.8 +train_indices = randperm(nrow(df))[1:Int(round(train_ratio * nrow(df)))] + +dtrain = df[train_indices, :] +deval = df[setdiff(1:nrow(df), train_indices), :] + +_mean, _std = mean(dtrain.MEDV), std(dtrain.MEDV) +transform!(dtrain, :MEDV => (x -> (x .- _mean) ./ _std) => "target") +transform!(deval, :MEDV => (x -> (x .- _mean) ./ _std) => "target") + +target_name = "target" +``` + +## Training + +Now we are ready to train our model. We first define a model configuration using the [`NeuroTreeRegressor`](@ref) model constructor. +Then, we use [`NeuroTreeModels.fit`](@ref) to train a boosted tree model. We pass the optional `deval` argument to enable the usage of early stopping. + +```julia +config = EvoLinearRegressor( + loss=:mse, + nrounds=5000, + eta=5e-1, +) + +m = EvoLinear.fit( + config, + dtrain; + deval, + target_name, + feature_names, + metric=:mse, + print_every_n=100, + early_stopping_rounds=10, +) +``` + +## Diagnosis + +Finally, we can get predictions by passing training and testing data to our model. We can then apply various evaluation metric, such as the MAE (mean absolute error): + +```julia +p_train = m(dtrain) .* _std .+ _mean +p_eval = m(deval) .* _std .+ _mean +``` + +```julia-repl +julia> mean(abs.(p_train .- dtrain[!, "MEDV"])) +0.8985784079860025 + +julia> mean(abs.(p_eval .- deval[!, "MEDV"])) +2.3287859731914597 +``` \ No newline at end of file diff --git a/test/linear.jl b/test/linear.jl index a04bb0b..6bd346e 100644 --- a/test/linear.jl +++ b/test/linear.jl @@ -10,10 +10,10 @@ coef = randn(T, nfeats) y_train = x_train * coef .+ rand(T, nobs) * T(0.1) - config = EvoLinearRegressor(nrounds = 10, loss = :mse, L1 = 0, L2 = 1) - m = EvoLinear.Linear.EvoLinearModel(:mse; coef = rand(3), bias = rand()) - m = EvoLinear.Linear.EvoLinearModel(:mse; coef = rand(Float32, 3), bias = rand(Float32)) - m = EvoLinear.Linear.EvoLinearModel(EvoLinear.Linear.loss_types[:mse]; coef = rand(3), bias = rand()) + config = EvoLinearRegressor(nrounds=10, loss=:mse, L1=0, L2=1) + m = EvoLinear.Linear.EvoLinearModel(:mse; coef=rand(3), bias=rand()) + m = EvoLinear.Linear.EvoLinearModel(:mse; coef=rand(Float32, 3), bias=rand(Float32)) + m = EvoLinear.Linear.EvoLinearModel(EvoLinear.Linear.loss_types[:mse]; coef=rand(3), bias=rand()) end @@ -29,8 +29,8 @@ end coef = randn(T, nfeats) y_train = x_train * coef .+ rand(T, nobs) * T(0.1) - config = EvoLinearRegressor(nrounds = 10, loss = :mse, L1 = 0, L2 = 1) - m0 = EvoLinear.fit(config; x_train, y_train, metric = :mse) + config = EvoLinearRegressor(nrounds=10, loss=:mse, L1=0, L2=1) + m0 = EvoLinear.fit(config; x_train, y_train, metric=:mse) m1, cache = EvoLinear.Linear.init(config, x_train, y_train) for i = 1:config.nrounds EvoLinear.fit!(m1, cache, config) @@ -40,10 +40,6 @@ end @info "max coef diff" maximum(coef_diff) @info "min coef diff" minimum(coef_diff) @test all(m0.coef .≈ m1.coef) - - p = EvoLinear.Linear.predict_linear(m0, x_train) - p = EvoLinear.Linear.predict_linear!(p, m0, x_train) - p = EvoLinear.Linear.predict_proj(m0, x_train) p = m0(x_train) metric_mse = EvoLinear.Metrics.mse(p, y_train) @@ -65,15 +61,15 @@ end coef = randn(T, nfeats) y_train = x_train * coef .+ rand(T, nobs) * T(0.1) - config = EvoLinearRegressor(nrounds = 10, loss = :mse, L1 = 0, L2 = 0) - m0 = EvoLinear.fit(config; x_train, y_train, metric = :mse) + config = EvoLinearRegressor(nrounds=10, loss=:mse, L1=0, L2=0) + m0 = EvoLinear.fit(config; x_train, y_train, metric=:mse) - config = EvoLinearRegressor(nrounds = 10, loss = :mse, L1 = 1e-1, L2 = 0) - m1 = EvoLinear.fit(config; x_train, y_train, metric = :mse) + config = EvoLinearRegressor(nrounds=10, loss=:mse, L1=1e-1, L2=0) + m1 = EvoLinear.fit(config; x_train, y_train, metric=:mse) @test sum(m1.coef .== 0) >= 5 - config = EvoLinearRegressor(nrounds = 10, loss = :mse, L1 = 0, L2 = 1) - m2 = EvoLinear.fit(config; x_train, y_train, metric = :mse) + config = EvoLinearRegressor(nrounds=10, loss=:mse, L1=0, L2=1) + m2 = EvoLinear.fit(config; x_train, y_train, metric=:mse) @test sum(abs.(m2.coef)) < sum(abs.(m0.coef)) end @@ -91,8 +87,8 @@ end y_train = EvoLinear.sigmoid(x_train * coef .+ rand(T, nobs) * T(0.1)) w = ones(T, nobs) - config = EvoLinearRegressor(nrounds = 10, loss = :logistic, L1 = 1e-2, L2 = 1e-3) - m = EvoLinear.fit(config; x_train, y_train, metric = :logloss) + config = EvoLinearRegressor(nrounds=10, loss=:logistic, L1=1e-2, L2=1e-3) + m = EvoLinear.fit(config; x_train, y_train, metric=:logloss) p = m(x_train) p1 = EvoLinear.Linear.predict_proj(m, x_train) @@ -119,13 +115,9 @@ end y_train = exp.(x_train * coef .+ rand(T, nobs) * T(0.1)) w = ones(T, nobs) - config = EvoLinearRegressor(nrounds = 10, loss = :poisson, L1 = 1e-2, L2 = 1e-3) - m = EvoLinear.fit(config; x_train, y_train, metric = :poisson_deviance) - + config = EvoLinearRegressor(nrounds=10, loss=:poisson, L1=1e-2, L2=1e-3) + m = EvoLinear.fit(config; x_train, y_train, metric=:poisson_deviance) p = m(x_train) - p1 = EvoLinear.Linear.predict_proj(m, x_train) - - @test all(p .== p1) metric = EvoLinear.Metrics.poisson_deviance(p, y_train) metric_w = EvoLinear.Metrics.poisson_deviance(p, y_train, w) @@ -147,13 +139,9 @@ end y_train = exp.(x_train * coef .+ rand(T, nobs) * T(0.1)) w = ones(T, nobs) - config = EvoLinearRegressor(nrounds = 10, loss = :gamma, L1 = 1e-2, L2 = 1e-3) - m = EvoLinear.fit(config; x_train, y_train, metric = :gamma_deviance) - + config = EvoLinearRegressor(nrounds=10, loss=:gamma, L1=1e-2, L2=1e-3) + m = EvoLinear.fit(config; x_train, y_train, metric=:gamma_deviance) p = m(x_train) - p1 = EvoLinear.Linear.predict_proj(m, x_train) - - @test all(p .== p1) metric = EvoLinear.Metrics.gamma_deviance(p, y_train) metric_w = EvoLinear.Metrics.gamma_deviance(p, y_train, w) @@ -175,13 +163,9 @@ end y_train = exp.(x_train * coef .+ rand(T, nobs) * T(0.1)) w = ones(T, nobs) - config = EvoLinearRegressor(nrounds = 10, loss = :tweedie, L1 = 1e-2, L2 = 1e-3) - m = EvoLinear.fit(config; x_train, y_train, metric = :tweedie_deviance) - + config = EvoLinearRegressor(nrounds=10, loss=:tweedie, L1=1e-2, L2=1e-3) + m = EvoLinear.fit(config; x_train, y_train, metric=:tweedie_deviance) p = m(x_train) - p1 = EvoLinear.Linear.predict_proj(m, x_train) - - @test all(p .== p1) metric = EvoLinear.Metrics.tweedie_deviance(p, y_train) metric_w = EvoLinear.Metrics.tweedie_deviance(p, y_train, w) From 8cf85f35028f086d7770e0b4b8586a3663eeb385 Mon Sep 17 00:00:00 2001 From: "jeremie.db" Date: Fri, 14 Jun 2024 18:21:19 -0400 Subject: [PATCH 9/9] up --- test/linear.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/linear.jl b/test/linear.jl index 6bd346e..3c33d9a 100644 --- a/test/linear.jl +++ b/test/linear.jl @@ -115,7 +115,7 @@ end y_train = exp.(x_train * coef .+ rand(T, nobs) * T(0.1)) w = ones(T, nobs) - config = EvoLinearRegressor(nrounds=10, loss=:poisson, L1=1e-2, L2=1e-3) + config = EvoLinearRegressor(nrounds=10, loss=:poisson_deviance, L1=1e-2, L2=1e-3) m = EvoLinear.fit(config; x_train, y_train, metric=:poisson_deviance) p = m(x_train) @@ -139,7 +139,7 @@ end y_train = exp.(x_train * coef .+ rand(T, nobs) * T(0.1)) w = ones(T, nobs) - config = EvoLinearRegressor(nrounds=10, loss=:gamma, L1=1e-2, L2=1e-3) + config = EvoLinearRegressor(nrounds=10, loss=:gamma_deviance, L1=1e-2, L2=1e-3) m = EvoLinear.fit(config; x_train, y_train, metric=:gamma_deviance) p = m(x_train) @@ -163,7 +163,7 @@ end y_train = exp.(x_train * coef .+ rand(T, nobs) * T(0.1)) w = ones(T, nobs) - config = EvoLinearRegressor(nrounds=10, loss=:tweedie, L1=1e-2, L2=1e-3) + config = EvoLinearRegressor(nrounds=10, loss=:tweedie_deviance, L1=1e-2, L2=1e-3) m = EvoLinear.fit(config; x_train, y_train, metric=:tweedie_deviance) p = m(x_train)