diff --git a/kan/.ipynb_checkpoints/KANLayer-checkpoint.py b/kan/.ipynb_checkpoints/KANLayer-checkpoint.py new file mode 100644 index 00000000..60c7b0eb --- /dev/null +++ b/kan/.ipynb_checkpoints/KANLayer-checkpoint.py @@ -0,0 +1,323 @@ +import torch +import torch.nn as nn +import numpy as np +from .spline import * +from .utils import sparse_mask + + +class KANLayer(nn.Module): + """ + KANLayer class + + + Attributes: + ----------- + in_dim: int + input dimension + out_dim: int + output dimension + size: int + the number of splines = input dimension * output dimension + k: int + the piecewise polynomial order of splines + grid: 2D torch.float + grid points + noises: 2D torch.float + injected noises to splines at initialization (to break degeneracy) + coef: 2D torch.tensor + coefficients of B-spline bases + scale_base: 1D torch.float + magnitude of the residual function b(x) + scale_sp: 1D torch.float + mangitude of the spline function spline(x) + base_fun: fun + residual function b(x) + mask: 1D torch.float + mask of spline functions. setting some element of the mask to zero means setting the corresponding activation to zero function. + grid_eps: float in [0,1] + a hyperparameter used in update_grid_from_samples. When grid_eps = 0, the grid is uniform; when grid_eps = 1, the grid is partitioned using percentiles of samples. 0 < grid_eps < 1 interpolates between the two extremes. + weight_sharing: 1D tensor int + allow spline activations to share parameters + lock_counter: int + counter how many activation functions are locked (weight sharing) + lock_id: 1D torch.int + the id of activation functions that are locked + device: str + device + + Methods: + -------- + __init__(): + initialize a KANLayer + forward(): + forward + update_grid_from_samples(): + update grids based on samples' incoming activations + initialize_grid_from_parent(): + initialize grids from another model + get_subset(): + get subset of the KANLayer (used for pruning) + lock(): + lock several activation functions to share parameters + unlock(): + unlock already locked activation functions + """ + + def __init__(self, in_dim=3, out_dim=2, num=5, k=3, noise_scale=0.1, scale_base=1.0, scale_sp=1.0, base_fun=torch.nn.SiLU(), grid_eps=0.02, grid_range=[-1, 1], sp_trainable=True, sb_trainable=True, save_plot_data = True, device='cpu', sparse_init=False): + '''' + initialize a KANLayer + + Args: + ----- + in_dim : int + input dimension. Default: 2. + out_dim : int + output dimension. Default: 3. + num : int + the number of grid intervals = G. Default: 5. + k : int + the order of piecewise polynomial. Default: 3. + noise_scale : float + the scale of noise injected at initialization. Default: 0.1. + scale_base : float + the scale of the residual function b(x). Default: 1.0. + scale_sp : float + the scale of the base function spline(x). Default: 1.0. + base_fun : function + residual function b(x). Default: torch.nn.SiLU() + grid_eps : float + When grid_eps = 0, the grid is uniform; when grid_eps = 1, the grid is partitioned using percentiles of samples. 0 < grid_eps < 1 interpolates between the two extremes. Default: 0.02. + grid_range : list/np.array of shape (2,) + setting the range of grids. Default: [-1,1]. + sp_trainable : bool + If true, scale_sp is trainable. Default: True. + sb_trainable : bool + If true, scale_base is trainable. Default: True. + device : str + device + + Returns: + -------- + self + + Example + ------- + >>> model = KANLayer(in_dim=3, out_dim=5) + >>> (model.in_dim, model.out_dim) + (3, 5) + ''' + super(KANLayer, self).__init__() + # size + self.out_dim = out_dim + self.in_dim = in_dim + self.num = num + self.k = k + + # shape: (size, num) + ### grid size: (batch, in_dim, out_dim, G + 1) => (batch, in_dim, G + 2*k + 1) + + grid = torch.linspace(grid_range[0], grid_range[1], steps=num + 1)[None,:].expand(self.in_dim, num+1) + grid = extend_grid(grid, k_extend=k) + self.grid = torch.nn.Parameter(grid).requires_grad_(False) + noises = (torch.rand(self.num+1, self.in_dim, self.out_dim) - 1 / 2) * noise_scale / num + # shape: (size, coef) + self.coef = torch.nn.Parameter(curve2coef(self.grid[:,k:-k].permute(1,0), noises, self.grid, k)) + #if isinstance(scale_base, float): + if sparse_init: + mask = sparse_mask(in_dim, out_dim) + else: + mask = 1. + + self.scale_base = torch.nn.Parameter(torch.ones(in_dim, out_dim) * scale_base * mask).requires_grad_(sb_trainable) # make scale trainable + #else: + #self.scale_base = torch.nn.Parameter(scale_base.to(device)).requires_grad_(sb_trainable) + self.scale_sp = torch.nn.Parameter(torch.ones(in_dim, out_dim) * scale_sp * mask).requires_grad_(sp_trainable) # make scale trainable + self.base_fun = base_fun + + self.mask = torch.nn.Parameter(torch.ones(in_dim, out_dim)).requires_grad_(False) + self.grid_eps = grid_eps + + ### remove weight_sharing & lock parts + #self.weight_sharing = torch.arange(out_dim*in_dim).reshape(out_dim, in_dim) + #self.lock_counter = 0 + #self.lock_id = torch.zeros(out_dim*in_dim).reshape(out_dim, in_dim) + + + def forward(self, x): + ''' + KANLayer forward given input x + + Args: + ----- + x : 2D torch.float + inputs, shape (number of samples, input dimension) + + Returns: + -------- + y : 2D torch.float + outputs, shape (number of samples, output dimension) + preacts : 3D torch.float + fan out x into activations, shape (number of sampels, output dimension, input dimension) + postacts : 3D torch.float + the outputs of activation functions with preacts as inputs + postspline : 3D torch.float + the outputs of spline functions with preacts as inputs + + Example + ------- + >>> model = KANLayer(in_dim=3, out_dim=5) + >>> x = torch.normal(0,1,size=(100,3)) + >>> y, preacts, postacts, postspline = model(x) + >>> y.shape, preacts.shape, postacts.shape, postspline.shape + (torch.Size([100, 5]), + torch.Size([100, 5, 3]), + torch.Size([100, 5, 3]), + torch.Size([100, 5, 3])) + ''' + batch = x.shape[0] + # x: shape (batch, in_dim) => shape (size, batch) (size = out_dim * in_dim) + #x = torch.einsum('ij,k->ikj', x, torch.ones(self.out_dim, device=self.device)).reshape(batch, self.size).permute(1, 0) + preacts = x[:,None,:].clone().expand(batch, self.out_dim, self.in_dim) + + base = self.base_fun(x) # (batch, in_dim) + y = coef2curve(x_eval=x, grid=self.grid, coef=self.coef, k=self.k) # y shape: (batch, in_dim, out_dim) + + postspline = y.clone().permute(0,2,1) # postspline shape: (batch, out_dim, in_dim) + + y = self.scale_base[None,:,:] * base[:,:,None] + self.scale_sp[None,:,:] * y + y = self.mask[None,:,:] * y + + postacts = y.clone().permute(0,2,1) + + y = torch.sum(y, dim=1) # shape (batch, out_dim) + return y, preacts, postacts, postspline + + def update_grid_from_samples(self, x): + ''' + update grid from samples + + Args: + ----- + x : 2D torch.float + inputs, shape (number of samples, input dimension) + + Returns: + -------- + None + + Example + ------- + >>> model = KANLayer(in_dim=1, out_dim=1, num=5, k=3) + >>> print(model.grid.data) + >>> x = torch.linspace(-3,3,steps=100)[:,None] + >>> model.update_grid_from_samples(x) + >>> print(model.grid.data) + tensor([[-1.0000, -0.6000, -0.2000, 0.2000, 0.6000, 1.0000]]) + tensor([[-3.0002, -1.7882, -0.5763, 0.6357, 1.8476, 3.0002]]) + ''' + batch = x.shape[0] + #x = torch.einsum('ij,k->ikj', x, torch.ones(self.out_dim, ).to(self.device)).reshape(batch, self.size).permute(1, 0) + x_pos = torch.sort(x, dim=0)[0] + y_eval = coef2curve(x_pos, self.grid, self.coef, self.k) + num_interval = self.grid.shape[1] - 1 - 2*self.k + ids = [int(batch / num_interval * i) for i in range(num_interval)] + [-1] + grid_adaptive = x_pos[ids, :].permute(1,0) + margin = 0.01 + h = (grid_adaptive[:,[-1]] - grid_adaptive[:,[0]])/num_interval + grid_uniform = grid_adaptive[:,[0]] + h * torch.arange(num_interval+1,)[None, :].to(x.device) + grid = self.grid_eps * grid_uniform + (1 - self.grid_eps) * grid_adaptive + self.grid.data = extend_grid(grid, k_extend=self.k) + self.coef.data = curve2coef(x_pos, y_eval, self.grid, self.k) + + def initialize_grid_from_parent(self, parent, x): + ''' + update grid from a parent KANLayer & samples + + Args: + ----- + parent : KANLayer + a parent KANLayer (whose grid is usually coarser than the current model) + x : 2D torch.float + inputs, shape (number of samples, input dimension) + + Returns: + -------- + None + + Example + ------- + >>> batch = 100 + >>> parent_model = KANLayer(in_dim=1, out_dim=1, num=5, k=3) + >>> print(parent_model.grid.data) + >>> model = KANLayer(in_dim=1, out_dim=1, num=10, k=3) + >>> x = torch.normal(0,1,size=(batch, 1)) + >>> model.initialize_grid_from_parent(parent_model, x) + >>> print(model.grid.data) + tensor([[-1.0000, -0.6000, -0.2000, 0.2000, 0.6000, 1.0000]]) + tensor([[-1.0000, -0.8000, -0.6000, -0.4000, -0.2000, 0.0000, 0.2000, 0.4000, + 0.6000, 0.8000, 1.0000]]) + ''' + batch = x.shape[0] + # preacts: shape (batch, in_dim) => shape (size, batch) (size = out_dim * in_dim) + #x_eval = torch.einsum('ij,k->ikj', x, torch.ones(self.out_dim, ).to(self.device)).reshape(batch, self.size).permute(1, 0) + x_eval = x + pgrid = parent.grid # (in_dim, G+2*k+1) + pk = parent.k + y_eval = coef2curve(x_eval, pgrid, parent.coef, pk) + + h = (pgrid[:,[-pk]] - pgrid[:,[pk]])/self.num + grid = pgrid[:,[pk]] + torch.arange(self.num+1,) * h + grid = extend_grid(grid, k_extend=self.k) + self.grid.data = grid + self.coef.data = curve2coef(x_eval, y_eval, self.grid, self.k) + + def get_subset(self, in_id, out_id): + ''' + get a smaller KANLayer from a larger KANLayer (used for pruning) + + Args: + ----- + in_id : list + id of selected input neurons + out_id : list + id of selected output neurons + + Returns: + -------- + spb : KANLayer + + Example + ------- + >>> kanlayer_large = KANLayer(in_dim=10, out_dim=10, num=5, k=3) + >>> kanlayer_small = kanlayer_large.get_subset([0,9],[1,2,3]) + >>> kanlayer_small.in_dim, kanlayer_small.out_dim + (2, 3) + ''' + spb = KANLayer(len(in_id), len(out_id), self.num, self.k, base_fun=self.base_fun) + spb.grid.data = self.grid[in_id] + spb.coef.data = self.coef[in_id][:,out_id] + spb.scale_base.data = self.scale_base[in_id][:,out_id] + spb.scale_sp.data = self.scale_sp[in_id][:,out_id] + spb.mask.data = self.mask[in_id][:,out_id] + + spb.in_dim = len(in_id) + spb.out_dim = len(out_id) + return spb + + + def swap(self, i1, i2, mode='in'): + + with torch.no_grad(): + def swap_(data, i1, i2, mode='in'): + if mode == 'in': + data[i1], data[i2] = data[i2].clone(), data[i1].clone() + elif mode == 'out': + data[:,i1], data[:,i2] = data[:,i2].clone(), data[:,i1].clone() + + if mode == 'in': + swap_(self.grid.data, i1, i2, mode='in') + swap_(self.coef.data, i1, i2, mode=mode) + swap_(self.scale_base.data, i1, i2, mode=mode) + swap_(self.scale_sp.data, i1, i2, mode=mode) + swap_(self.mask.data, i1, i2, mode=mode) + diff --git a/kan/.ipynb_checkpoints/LBFGS-checkpoint.py b/kan/.ipynb_checkpoints/LBFGS-checkpoint.py new file mode 100644 index 00000000..212477f2 --- /dev/null +++ b/kan/.ipynb_checkpoints/LBFGS-checkpoint.py @@ -0,0 +1,493 @@ +import torch +from functools import reduce +from torch.optim import Optimizer + +__all__ = ['LBFGS'] + +def _cubic_interpolate(x1, f1, g1, x2, f2, g2, bounds=None): + # ported from https://github.com/torch/optim/blob/master/polyinterp.lua + # Compute bounds of interpolation area + if bounds is not None: + xmin_bound, xmax_bound = bounds + else: + xmin_bound, xmax_bound = (x1, x2) if x1 <= x2 else (x2, x1) + + # Code for most common case: cubic interpolation of 2 points + # w/ function and derivative values for both + # Solution in this case (where x2 is the farthest point): + # d1 = g1 + g2 - 3*(f1-f2)/(x1-x2); + # d2 = sqrt(d1^2 - g1*g2); + # min_pos = x2 - (x2 - x1)*((g2 + d2 - d1)/(g2 - g1 + 2*d2)); + # t_new = min(max(min_pos,xmin_bound),xmax_bound); + d1 = g1 + g2 - 3 * (f1 - f2) / (x1 - x2) + d2_square = d1**2 - g1 * g2 + if d2_square >= 0: + d2 = d2_square.sqrt() + if x1 <= x2: + min_pos = x2 - (x2 - x1) * ((g2 + d2 - d1) / (g2 - g1 + 2 * d2)) + else: + min_pos = x1 - (x1 - x2) * ((g1 + d2 - d1) / (g1 - g2 + 2 * d2)) + return min(max(min_pos, xmin_bound), xmax_bound) + else: + return (xmin_bound + xmax_bound) / 2. + + +def _strong_wolfe(obj_func, + x, + t, + d, + f, + g, + gtd, + c1=1e-4, + c2=0.9, + tolerance_change=1e-9, + max_ls=25): + # ported from https://github.com/torch/optim/blob/master/lswolfe.lua + d_norm = d.abs().max() + g = g.clone(memory_format=torch.contiguous_format) + # evaluate objective and gradient using initial step + f_new, g_new = obj_func(x, t, d) + ls_func_evals = 1 + gtd_new = g_new.dot(d) + + # bracket an interval containing a point satisfying the Wolfe criteria + t_prev, f_prev, g_prev, gtd_prev = 0, f, g, gtd + done = False + ls_iter = 0 + while ls_iter < max_ls: + # check conditions + #print(f_prev, f_new, g_new) + if f_new > (f + c1 * t * gtd) or (ls_iter > 1 and f_new >= f_prev): + bracket = [t_prev, t] + bracket_f = [f_prev, f_new] + bracket_g = [g_prev, g_new.clone(memory_format=torch.contiguous_format)] + bracket_gtd = [gtd_prev, gtd_new] + break + + if abs(gtd_new) <= -c2 * gtd: + bracket = [t] + bracket_f = [f_new] + bracket_g = [g_new] + done = True + break + + if gtd_new >= 0: + bracket = [t_prev, t] + bracket_f = [f_prev, f_new] + bracket_g = [g_prev, g_new.clone(memory_format=torch.contiguous_format)] + bracket_gtd = [gtd_prev, gtd_new] + break + + # interpolate + min_step = t + 0.01 * (t - t_prev) + max_step = t * 10 + tmp = t + t = _cubic_interpolate( + t_prev, + f_prev, + gtd_prev, + t, + f_new, + gtd_new, + bounds=(min_step, max_step)) + + # next step + t_prev = tmp + f_prev = f_new + g_prev = g_new.clone(memory_format=torch.contiguous_format) + gtd_prev = gtd_new + f_new, g_new = obj_func(x, t, d) + ls_func_evals += 1 + gtd_new = g_new.dot(d) + ls_iter += 1 + + + # reached max number of iterations? + if ls_iter == max_ls: + bracket = [0, t] + bracket_f = [f, f_new] + bracket_g = [g, g_new] + + # zoom phase: we now have a point satisfying the criteria, or + # a bracket around it. We refine the bracket until we find the + # exact point satisfying the criteria + insuf_progress = False + # find high and low points in bracket + low_pos, high_pos = (0, 1) if bracket_f[0] <= bracket_f[-1] else (1, 0) + while not done and ls_iter < max_ls: + # line-search bracket is so small + if abs(bracket[1] - bracket[0]) * d_norm < tolerance_change: + break + + # compute new trial value + t = _cubic_interpolate(bracket[0], bracket_f[0], bracket_gtd[0], + bracket[1], bracket_f[1], bracket_gtd[1]) + + # test that we are making sufficient progress: + # in case `t` is so close to boundary, we mark that we are making + # insufficient progress, and if + # + we have made insufficient progress in the last step, or + # + `t` is at one of the boundary, + # we will move `t` to a position which is `0.1 * len(bracket)` + # away from the nearest boundary point. + eps = 0.1 * (max(bracket) - min(bracket)) + if min(max(bracket) - t, t - min(bracket)) < eps: + # interpolation close to boundary + if insuf_progress or t >= max(bracket) or t <= min(bracket): + # evaluate at 0.1 away from boundary + if abs(t - max(bracket)) < abs(t - min(bracket)): + t = max(bracket) - eps + else: + t = min(bracket) + eps + insuf_progress = False + else: + insuf_progress = True + else: + insuf_progress = False + + # Evaluate new point + f_new, g_new = obj_func(x, t, d) + ls_func_evals += 1 + gtd_new = g_new.dot(d) + ls_iter += 1 + + if f_new > (f + c1 * t * gtd) or f_new >= bracket_f[low_pos]: + # Armijo condition not satisfied or not lower than lowest point + bracket[high_pos] = t + bracket_f[high_pos] = f_new + bracket_g[high_pos] = g_new.clone(memory_format=torch.contiguous_format) + bracket_gtd[high_pos] = gtd_new + low_pos, high_pos = (0, 1) if bracket_f[0] <= bracket_f[1] else (1, 0) + else: + if abs(gtd_new) <= -c2 * gtd: + # Wolfe conditions satisfied + done = True + elif gtd_new * (bracket[high_pos] - bracket[low_pos]) >= 0: + # old low becomes new high + bracket[high_pos] = bracket[low_pos] + bracket_f[high_pos] = bracket_f[low_pos] + bracket_g[high_pos] = bracket_g[low_pos] + bracket_gtd[high_pos] = bracket_gtd[low_pos] + + # new point becomes new low + bracket[low_pos] = t + bracket_f[low_pos] = f_new + bracket_g[low_pos] = g_new.clone(memory_format=torch.contiguous_format) + bracket_gtd[low_pos] = gtd_new + + #print(bracket) + if len(bracket) == 1: + t = bracket[0] + f_new = bracket_f[0] + g_new = bracket_g[0] + else: + t = bracket[low_pos] + f_new = bracket_f[low_pos] + g_new = bracket_g[low_pos] + return f_new, g_new, t, ls_func_evals + + + +class LBFGS(Optimizer): + """Implements L-BFGS algorithm. + + Heavily inspired by `minFunc + `_. + + .. warning:: + This optimizer doesn't support per-parameter options and parameter + groups (there can be only one). + + .. warning:: + Right now all parameters have to be on a single device. This will be + improved in the future. + + .. note:: + This is a very memory intensive optimizer (it requires additional + ``param_bytes * (history_size + 1)`` bytes). If it doesn't fit in memory + try reducing the history size, or use a different algorithm. + + Args: + lr (float): learning rate (default: 1) + max_iter (int): maximal number of iterations per optimization step + (default: 20) + max_eval (int): maximal number of function evaluations per optimization + step (default: max_iter * 1.25). + tolerance_grad (float): termination tolerance on first order optimality + (default: 1e-7). + tolerance_change (float): termination tolerance on function + value/parameter changes (default: 1e-9). + history_size (int): update history size (default: 100). + line_search_fn (str): either 'strong_wolfe' or None (default: None). + """ + + def __init__(self, + params, + lr=1, + max_iter=20, + max_eval=None, + tolerance_grad=1e-7, + tolerance_change=1e-9, + tolerance_ys=1e-32, + history_size=100, + line_search_fn=None): + if max_eval is None: + max_eval = max_iter * 5 // 4 + defaults = dict( + lr=lr, + max_iter=max_iter, + max_eval=max_eval, + tolerance_grad=tolerance_grad, + tolerance_change=tolerance_change, + tolerance_ys=tolerance_ys, + history_size=history_size, + line_search_fn=line_search_fn) + super().__init__(params, defaults) + + if len(self.param_groups) != 1: + raise ValueError("LBFGS doesn't support per-parameter options " + "(parameter groups)") + + self._params = self.param_groups[0]['params'] + self._numel_cache = None + + def _numel(self): + if self._numel_cache is None: + self._numel_cache = reduce(lambda total, p: total + p.numel(), self._params, 0) + return self._numel_cache + + def _gather_flat_grad(self): + views = [] + for p in self._params: + if p.grad is None: + view = p.new(p.numel()).zero_() + elif p.grad.is_sparse: + view = p.grad.to_dense().view(-1) + else: + view = p.grad.view(-1) + views.append(view) + device = views[0].device + return torch.cat(views, dim=0) + + def _add_grad(self, step_size, update): + offset = 0 + for p in self._params: + numel = p.numel() + # view as to avoid deprecated pointwise semantics + p.add_(update[offset:offset + numel].view_as(p), alpha=step_size) + offset += numel + assert offset == self._numel() + + def _clone_param(self): + return [p.clone(memory_format=torch.contiguous_format) for p in self._params] + + def _set_param(self, params_data): + for p, pdata in zip(self._params, params_data): + p.copy_(pdata) + + def _directional_evaluate(self, closure, x, t, d): + self._add_grad(t, d) + loss = float(closure()) + flat_grad = self._gather_flat_grad() + self._set_param(x) + return loss, flat_grad + + + @torch.no_grad() + def step(self, closure): + """Perform a single optimization step. + + Args: + closure (Callable): A closure that reevaluates the model + and returns the loss. + """ + + torch.manual_seed(0) + + assert len(self.param_groups) == 1 + + # Make sure the closure is always called with grad enabled + closure = torch.enable_grad()(closure) + + group = self.param_groups[0] + lr = group['lr'] + max_iter = group['max_iter'] + max_eval = group['max_eval'] + tolerance_grad = group['tolerance_grad'] + tolerance_change = group['tolerance_change'] + tolerance_ys = group['tolerance_ys'] + line_search_fn = group['line_search_fn'] + history_size = group['history_size'] + + # NOTE: LBFGS has only global state, but we register it as state for + # the first param, because this helps with casting in load_state_dict + state = self.state[self._params[0]] + state.setdefault('func_evals', 0) + state.setdefault('n_iter', 0) + + # evaluate initial f(x) and df/dx + orig_loss = closure() + loss = float(orig_loss) + current_evals = 1 + state['func_evals'] += 1 + + flat_grad = self._gather_flat_grad() + opt_cond = flat_grad.abs().max() <= tolerance_grad + + # optimal condition + if opt_cond: + return orig_loss + + # tensors cached in state (for tracing) + d = state.get('d') + t = state.get('t') + old_dirs = state.get('old_dirs') + old_stps = state.get('old_stps') + ro = state.get('ro') + H_diag = state.get('H_diag') + prev_flat_grad = state.get('prev_flat_grad') + prev_loss = state.get('prev_loss') + + n_iter = 0 + # optimize for a max of max_iter iterations + while n_iter < max_iter: + # keep track of nb of iterations + n_iter += 1 + state['n_iter'] += 1 + + ############################################################ + # compute gradient descent direction + ############################################################ + if state['n_iter'] == 1: + d = flat_grad.neg() + old_dirs = [] + old_stps = [] + ro = [] + H_diag = 1 + else: + # do lbfgs update (update memory) + y = flat_grad.sub(prev_flat_grad) + s = d.mul(t) + ys = y.dot(s) # y*s + if ys > tolerance_ys: + # updating memory + if len(old_dirs) == history_size: + # shift history by one (limited-memory) + old_dirs.pop(0) + old_stps.pop(0) + ro.pop(0) + + # store new direction/step + old_dirs.append(y) + old_stps.append(s) + ro.append(1. / ys) + + # update scale of initial Hessian approximation + H_diag = ys / y.dot(y) # (y*y) + + # compute the approximate (L-BFGS) inverse Hessian + # multiplied by the gradient + num_old = len(old_dirs) + + if 'al' not in state: + state['al'] = [None] * history_size + al = state['al'] + + # iteration in L-BFGS loop collapsed to use just one buffer + q = flat_grad.neg() + for i in range(num_old - 1, -1, -1): + al[i] = old_stps[i].dot(q) * ro[i] + q.add_(old_dirs[i], alpha=-al[i]) + + # multiply by initial Hessian + # r/d is the final direction + d = r = torch.mul(q, H_diag) + for i in range(num_old): + be_i = old_dirs[i].dot(r) * ro[i] + r.add_(old_stps[i], alpha=al[i] - be_i) + + if prev_flat_grad is None: + prev_flat_grad = flat_grad.clone(memory_format=torch.contiguous_format) + else: + prev_flat_grad.copy_(flat_grad) + prev_loss = loss + + ############################################################ + # compute step length + ############################################################ + # reset initial guess for step size + if state['n_iter'] == 1: + t = min(1., 1. / flat_grad.abs().sum()) * lr + else: + t = lr + + # directional derivative + gtd = flat_grad.dot(d) # g * d + + # directional derivative is below tolerance + if gtd > -tolerance_change: + break + + # optional line search: user function + ls_func_evals = 0 + if line_search_fn is not None: + # perform line search, using user function + if line_search_fn != "strong_wolfe": + raise RuntimeError("only 'strong_wolfe' is supported") + else: + x_init = self._clone_param() + + def obj_func(x, t, d): + return self._directional_evaluate(closure, x, t, d) + loss, flat_grad, t, ls_func_evals = _strong_wolfe( + obj_func, x_init, t, d, loss, flat_grad, gtd) + self._add_grad(t, d) + opt_cond = flat_grad.abs().max() <= tolerance_grad + else: + # no line search, simply move with fixed-step + self._add_grad(t, d) + if n_iter != max_iter: + # re-evaluate function only if not in last iteration + # the reason we do this: in a stochastic setting, + # no use to re-evaluate that function here + with torch.enable_grad(): + loss = float(closure()) + flat_grad = self._gather_flat_grad() + opt_cond = flat_grad.abs().max() <= tolerance_grad + ls_func_evals = 1 + + # update func eval + current_evals += ls_func_evals + state['func_evals'] += ls_func_evals + + ############################################################ + # check conditions + ############################################################ + if n_iter == max_iter: + break + + if current_evals >= max_eval: + break + + # optimal condition + if opt_cond: + break + + # lack of progress + if d.mul(t).abs().max() <= tolerance_change: + break + + if abs(loss - prev_loss) < tolerance_change: + break + + state['d'] = d + state['t'] = t + state['old_dirs'] = old_dirs + state['old_stps'] = old_stps + state['ro'] = ro + state['H_diag'] = H_diag + state['prev_flat_grad'] = prev_flat_grad + state['prev_loss'] = prev_loss + + return orig_loss diff --git a/kan/.ipynb_checkpoints/MultKAN-checkpoint.py b/kan/.ipynb_checkpoints/MultKAN-checkpoint.py new file mode 100644 index 00000000..39dd62b5 --- /dev/null +++ b/kan/.ipynb_checkpoints/MultKAN-checkpoint.py @@ -0,0 +1,1815 @@ +import torch +import torch.nn as nn +import numpy as np +from .KANLayer import KANLayer +#from .Symbolic_MultKANLayer import * +from .Symbolic_KANLayer import Symbolic_KANLayer +from .LBFGS import * +import os +import glob +import matplotlib.pyplot as plt +from tqdm import tqdm +import random +import copy +#from .MultKANLayer import MultKANLayer +import pandas as pd +from sympy.printing import latex +from sympy import * +import sympy +import yaml +from .spline import curve2coef +from .utils import SYMBOLIC_LIB +from .hypothesis import plot_tree + +class MultKAN(nn.Module): + + # include mult_ops = [] + def __init__(self, width=None, grid=3, k=3, mult_arity = 2, noise_scale=1.0, scale_base_mu=0.0, scale_base_sigma=1.0, base_fun='silu', symbolic_enabled=True, affine_trainable=False, grid_eps=1.0, grid_range=[-1, 1], sp_trainable=True, sb_trainable=True, seed=1, save_act=True, sparse_init=False, auto_save=True, first_init=True, ckpt_path='./model', state_id=0, round=0): + + super(MultKAN, self).__init__() + + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + + ### initializeing the numerical front ### + + self.act_fun = [] + self.depth = len(width) - 1 + + for i in range(len(width)): + if type(width[i]) == int: + width[i] = [width[i],0] + + self.width = width + + # if mult_arity is just a scalar, we extend it to a list of lists + # e.g, mult_arity = [[2,3],[4]] means that in the first hidden layer, 2 mult ops have arity 2 and 3, respectively; + # in the second hidden layer, 1 mult op has arity 4. + if isinstance(mult_arity, int): + self.mult_homo = True # when homo is True, parallelization is possible + else: + self.mult_homo = False # when home if False, for loop is required. + self.mult_arity = mult_arity + + width_in = self.width_in + width_out = self.width_out + + self.base_fun_name = base_fun + if base_fun == 'silu': + base_fun = torch.nn.SiLU() + elif base_fun == 'identity': + base_fun = torch.nn.Identity() + + self.grid_eps = grid_eps + self.grid_range = grid_range + + + for l in range(self.depth): + # splines + scale_base = scale_base_mu * 1 / np.sqrt(width_in[l]) + \ + scale_base_sigma * (torch.randn(width_in[l], width_out[l + 1]) * 2 - 1) * 1/np.sqrt(width_in[l]) + sp_batch = KANLayer(in_dim=width_in[l], out_dim=width_out[l+1], num=grid, k=k, noise_scale=noise_scale, scale_base=scale_base, scale_sp=1., base_fun=base_fun, grid_eps=grid_eps, grid_range=grid_range, sp_trainable=sp_trainable, sb_trainable=sb_trainable, sparse_init=sparse_init) + self.act_fun.append(sp_batch) + + self.node_bias = [] + self.node_scale = [] + self.subnode_bias = [] + self.subnode_scale = [] + + globals()['self.node_bias_0'] = torch.nn.Parameter(torch.zeros(3,1)).requires_grad_(False) + #self.node_bias_0 = torch.nn.Parameter(torch.zeros(3,1)).requires_grad_(False) + exec('self.node_bias_0' + " = torch.nn.Parameter(torch.zeros(3,1)).requires_grad_(False)") + + for l in range(self.depth): + exec(f'self.node_bias_{l} = torch.nn.Parameter(torch.zeros(width_in[l+1],)).requires_grad_(affine_trainable)') + exec(f'self.node_scale_{l} = torch.nn.Parameter(torch.ones(width_in[l+1],)).requires_grad_(affine_trainable)') + exec(f'self.subnode_bias_{l} = torch.nn.Parameter(torch.zeros(width_out[l+1],)).requires_grad_(affine_trainable)') + exec(f'self.subnode_scale_{l} = torch.nn.Parameter(torch.ones(width_out[l+1],)).requires_grad_(affine_trainable)') + exec(f'self.node_bias.append(self.node_bias_{l})') + exec(f'self.node_scale.append(self.node_scale_{l})') + exec(f'self.subnode_bias.append(self.subnode_bias_{l})') + exec(f'self.subnode_scale.append(self.subnode_scale_{l})') + + + self.act_fun = nn.ModuleList(self.act_fun) + + self.grid = grid + self.k = k + self.base_fun = base_fun + + ### initializing the symbolic front ### + self.symbolic_fun = [] + for l in range(self.depth): + sb_batch = Symbolic_KANLayer(in_dim=width_in[l], out_dim=width_out[l+1]) + self.symbolic_fun.append(sb_batch) + + self.symbolic_fun = nn.ModuleList(self.symbolic_fun) + self.symbolic_enabled = symbolic_enabled + self.affine_trainable = affine_trainable + self.sp_trainable = sp_trainable + self.sb_trainable = sb_trainable + + self.save_act = save_act + + self.node_scores = None + self.edge_scores = None + self.subnode_scores = None + + self.cache_data = None + self.acts = None + + self.auto_save = auto_save + self.state_id = 0 + self.ckpt_path = ckpt_path + self.round = round + + if auto_save: + if first_init: + if not os.path.exists(ckpt_path): + # Create the directory + os.makedirs(ckpt_path) + print(f"checkpoint directory created: {ckpt_path}") + print('saving model version 0.0') + + history_path = self.ckpt_path+'/history.txt' + with open(history_path, 'w') as file: + file.write(f'### Round {self.round} ###' + '\n') + file.write('init => 0.0' + '\n') + self.saveckpt(path=self.ckpt_path+'/'+'0.0') + else: + self.state_id = state_id + + self.input_id = torch.arange(self.width_in[0],) + + def initialize_from_another_model(self, another_model, x): + another_model(x) # get activations + batch = x.shape[0] + + self.initialize_grid_from_another_model(another_model, x) + + for l in range(self.depth): + spb = self.act_fun[l] + #spb_parent = another_model.act_fun[l] + + # spb = spb_parent + preacts = another_model.spline_preacts[l] + postsplines = another_model.spline_postsplines[l] + self.act_fun[l].coef.data = curve2coef(preacts[:,0,:], postsplines.permute(0,2,1), spb.grid, k=spb.k) + self.act_fun[l].scale_base.data = another_model.act_fun[l].scale_base.data + self.act_fun[l].scale_sp.data = another_model.act_fun[l].scale_sp.data + self.act_fun[l].mask.data = another_model.act_fun[l].mask.data + + for l in range(self.depth): + self.node_bias[l].data = another_model.node_bias[l].data + self.node_scale[l].data = another_model.node_scale[l].data + + self.subnode_bias[l].data = another_model.subnode_bias[l].data + self.subnode_scale[l].data = another_model.subnode_scale[l].data + + for l in range(self.depth): + self.symbolic_fun[l] = another_model.symbolic_fun[l] + + return self.to(device) + + def log_history(self, method_name): + + if self.auto_save: + + # save to log file + #print(func.__name__) + with open(self.ckpt_path+'/history.txt', 'a') as file: + file.write(str(self.round)+'.'+str(self.state_id)+' => '+ method_name + ' => ' + str(self.round)+'.'+str(self.state_id+1) + '\n') + + # update state_id + self.state_id += 1 + + # save to ckpt + self.saveckpt(path=self.ckpt_path+'/'+str(self.round)+'.'+str(self.state_id)) + print('saving model version '+str(self.round)+'.'+str(self.state_id)) + + + def refine(self, new_grid): + + model_new = MultKAN(width=self.width, + grid=new_grid, + k=self.k, + mult_arity=self.mult_arity, + base_fun=self.base_fun_name, + symbolic_enabled=self.symbolic_enabled, + affine_trainable=self.affine_trainable, + grid_eps=self.grid_eps, + grid_range=self.grid_range, + sp_trainable=self.sp_trainable, + sb_trainable=self.sb_trainable, + ckpt_path=self.ckpt_path, + auto_save=True, + first_init=False, + state_id=self.state_id, + round=self.round) + + model_new.initialize_from_another_model(self, self.cache_data) + model_new.cache_data = self.cache_data + model_new.grid = new_grid + + self.log_history('refine') + model_new.state_id += 1 + + return model_new + + + def saveckpt(self, path='model'): + + model = self + + dic = dict( + width = model.width, + grid = model.grid, + k = model.k, + mult_arity = model.mult_arity, + base_fun_name = model.base_fun_name, + symbolic_enabled = model.symbolic_enabled, + affine_trainable = model.affine_trainable, + grid_eps = model.grid_eps, + grid_range = model.grid_range, + sp_trainable = model.sp_trainable, + sb_trainable = model.sb_trainable, + state_id = model.state_id, + auto_save = model.auto_save, + ckpt_path = model.ckpt_path, + round = model.round + ) + + for i in range (model.depth): + dic[f'symbolic.funs_name.{i}'] = model.symbolic_fun[i].funs_name + + with open(f'{path}_config.yml', 'w') as outfile: + yaml.dump(dic, outfile, default_flow_style=False) + + torch.save(model.state_dict(), f'{path}_state') + torch.save(model.cache_data, f'{path}_cache_data') + + @staticmethod + def loadckpt(path='model'): + with open(f'{path}_config.yml', 'r') as stream: + config = yaml.safe_load(stream) + + state = torch.load(f'{path}_state') + + model_load = MultKAN(width=config['width'], + grid=config['grid'], + k=config['k'], + mult_arity = config['mult_arity'], + base_fun=config['base_fun_name'], + symbolic_enabled=config['symbolic_enabled'], + affine_trainable=config['affine_trainable'], + grid_eps=config['grid_eps'], + grid_range=config['grid_range'], + sp_trainable=config['sp_trainable'], + sb_trainable=config['sb_trainable'], + state_id=config['state_id'], + auto_save=config['auto_save'], + first_init=False, + ckpt_path=config['ckpt_path'], + round = config['round']+1) + + model_load.load_state_dict(state) + model_load.cache_data = torch.load(f'{path}_cache_data') + + depth = len(model_load.width) - 1 + for l in range(depth): + out_dim = model_load.symbolic_fun[l].out_dim + in_dim = model_load.symbolic_fun[l].in_dim + funs_name = config[f'symbolic.funs_name.{l}'] + for j in range(out_dim): + for i in range(in_dim): + fun_name = funs_name[j][i] + model_load.symbolic_fun[l].funs_name[j][i] = fun_name + model_load.symbolic_fun[l].funs[j][i] = SYMBOLIC_LIB[fun_name][0] + model_load.symbolic_fun[l].funs_sympy[j][i] = SYMBOLIC_LIB[fun_name][1] + model_load.symbolic_fun[l].funs_avoid_singularity[j][i] = SYMBOLIC_LIB[fun_name][3] + return model_load + + def rewind(self, model_id): + + self.round += 1 + self.state_id = model_id.split('.')[-1] + + history_path = self.ckpt_path+'/history.txt' + with open(history_path, 'a') as file: + file.write(f'### Round {self.round} ###' + '\n') + + self.saveckpt(path=self.ckpt_path+'/'+f'{self.round}.{self.state_id}') + + print('rewind to model version '+f'{self.round-1}.{self.state_id}'+', renamed as '+f'{self.round}.{self.state_id}') + + return MultKAN.loadckpt(path=self.ckpt_path+'/'+str(model_id)) + + + def checkout(self, model_id): + return MultKAN.loadckpt(path=self.ckpt_path+'/'+str(model_id)) + + @property + def width_in(self): + width = self.width + width_in = [width[l][0]+width[l][1] for l in range(len(width))] + return width_in + + @property + def width_out(self): + width = self.width + if self.mult_homo == True: + width_out = [width[l][0]+self.mult_arity*width[l][1] for l in range(len(width))] + else: + width_out = [width[l][0]+int(np.sum(self.mult_arity[l])) for l in range(len(width))] + return width_out + + @property + def n_sum(self): + width = self.width + n_sum = [width[l][0] for l in range(1,len(width)-1)] + return n_sum + + @property + def n_mult(self): + width = self.width + n_mult = [width[l][1] for l in range(1,len(width)-1)] + return n_mult + + @property + def feature_score(self): + self.attribute() + if self.node_scores == None: + return None + else: + return self.node_scores[0] + + def update_grid_from_samples(self, x): + for l in range(self.depth): + self.get_act(x) + self.act_fun[l].update_grid_from_samples(self.acts[l]) + + def update_grid(self, x): + self.update_grid_from_samples(x) + + def initialize_grid_from_another_model(self, model, x): + model(x) + for l in range(self.depth): + self.act_fun[l].initialize_grid_from_parent(model.act_fun[l], model.acts[l]) + + def forward(self, x, singularity_avoiding=False, y_th=10.): + + assert x.shape[1] == self.width_in[0] + + x = x[:,self.input_id.long()] + + # cache data + self.cache_data = x + + self.acts = [] # shape ([batch, n0], [batch, n1], ..., [batch, n_L]) + self.acts_premult = [] + self.spline_preacts = [] + self.spline_postsplines = [] + self.spline_postacts = [] + self.acts_scale = [] + self.acts_scale_spline = [] + self.subnode_actscale = [] + self.edge_actscale = [] + # self.neurons_scale = [] + + self.acts.append(x) # acts shape: (batch, width[l]) + + for l in range(self.depth): + + x_numerical, preacts, postacts_numerical, postspline = self.act_fun[l](x) + #print(preacts, postacts_numerical, postspline) + + if self.symbolic_enabled == True: + x_symbolic, postacts_symbolic = self.symbolic_fun[l](x, singularity_avoiding=singularity_avoiding, y_th=y_th) + else: + x_symbolic = 0. + postacts_symbolic = 0. + + x = x_numerical + x_symbolic + + if self.save_act: + # save subnode_scale + self.subnode_actscale.append(torch.std(x, dim=0).detach()) + + # subnode affine transform + x = self.subnode_scale[l][None,:] * x + self.subnode_bias[l][None,:] + + if self.save_act: + postacts = postacts_numerical + postacts_symbolic + + # self.neurons_scale.append(torch.mean(torch.abs(x), dim=0)) + #grid_reshape = self.act_fun[l].grid.reshape(self.width_out[l + 1], self.width_in[l], -1) + input_range = torch.std(preacts, dim=0) + 0.1 + output_range_spline = torch.std(postacts_numerical, dim=0) # for training, only penalize the spline part + output_range = torch.std(postacts, dim=0) # for visualization, include the contribution from both spline + symbolic + # save edge_scale + self.edge_actscale.append(output_range) + + self.acts_scale.append((output_range / input_range).detach()) + self.acts_scale_spline.append(output_range_spline / input_range) + self.spline_preacts.append(preacts.detach()) + self.spline_postacts.append(postacts.detach()) + self.spline_postsplines.append(postspline.detach()) + + self.acts_premult.append(x.detach()) + + # multiplication + dim_sum = self.width[l+1][0] + dim_mult = self.width[l+1][1] + + if self.mult_homo == True: + for i in range(self.mult_arity-1): + if i == 0: + x_mult = x[:,dim_sum::self.mult_arity] * x[:,dim_sum+1::self.mult_arity] + else: + x_mult = x_mult * x[:,dim_sum+i+1::self.mult_arity] + + else: + for j in range(dim_mult): + acml_id = dim_sum + np.sum(self.mult_arity[l+1][:j]) + for i in range(self.mult_arity[l+1][j]-1): + if i == 0: + x_mult_j = x[:,[acml_id]] * x[:,[acml_id+1]] + else: + x_mult_j = x_mult_j * x[:,[acml_id+i+1]] + + if j == 0: + x_mult = x_mult_j + else: + x_mult = torch.cat([x_mult, x_mult_j], dim=1) + + if self.width[l+1][1] > 0: + x = torch.cat([x[:,:dim_sum], x_mult], dim=1) + + # x = x + self.biases[l].weight + # node affine transform + x = self.node_scale[l][None,:] * x + self.node_bias[l][None,:] + + self.acts.append(x.detach()) + + + return x + + def set_mode(self, l, i, j, mode, mask_n=None): + if mode == "s": + mask_n = 0.; + mask_s = 1. + elif mode == "n": + mask_n = 1.; + mask_s = 0. + elif mode == "sn" or mode == "ns": + if mask_n == None: + mask_n = 1. + else: + mask_n = mask_n + mask_s = 1. + else: + mask_n = 0.; + mask_s = 0. + + self.act_fun[l].mask.data[i][j] = mask_n + self.symbolic_fun[l].mask.data[j,i] = mask_s + + def fix_symbolic(self, l, i, j, fun_name, fit_params_bool=True, a_range=(-10, 10), b_range=(-10, 10), verbose=True, random=False, log_history=True): + if not fit_params_bool: + self.symbolic_fun[l].fix_symbolic(i, j, fun_name, verbose=verbose, random=random) + r2 = None + else: + x = self.acts[l][:, i] + mask = self.act_fun[l].mask + y = self.spline_postacts[l][:, j, i] + #y = self.postacts[l][:, j, i] + r2 = self.symbolic_fun[l].fix_symbolic(i, j, fun_name, x, y, a_range=a_range, b_range=b_range, verbose=verbose) + if mask[i,j] == 0: + r2 = - 1e8 + self.set_mode(l, i, j, mode="s") + + if log_history: + self.log_history('fix_symbolic') + return r2 + + def unfix_symbolic(self, l, i, j, log_history=True): + self.set_mode(l, i, j, mode="n") + self.symbolic_fun[l].funs_name[j][i] = "0" + if log_history: + self.log_history('unfix_symbolic') + + def unfix_symbolic_all(self): + for l in range(len(self.width) - 1): + for i in range(self.width[l]): + for j in range(self.width[l + 1]): + self.unfix_symbolic(l, i, j) + + def get_range(self, l, i, j, verbose=True): + x = self.spline_preacts[l][:, j, i] + y = self.spline_postacts[l][:, j, i] + x_min = torch.min(x) + x_max = torch.max(x) + y_min = torch.min(y) + y_max = torch.max(y) + if verbose: + print('x range: [' + '%.2f' % x_min, ',', '%.2f' % x_max, ']') + print('y range: [' + '%.2f' % y_min, ',', '%.2f' % y_max, ']') + return x_min, x_max, y_min, y_max + + def plot(self, folder="./figures", beta=3, mask=False, metric='backward', scale=0.5, tick=False, sample=False, in_vars=None, out_vars=None, title=None, varscale=1.0): + + global Symbol + + if not self.save_act: + print('cannot plot since data are not saved. Set save_act=True first.') + + # forward to obtain activations + if self.acts == None: + if self.cache_data == None: + raise Exception('model hasn\'t seen any data yet.') + self.forward(self.cache_data) + + if metric == 'backward': + self.attribute() + + + if not os.path.exists(folder): + os.makedirs(folder) + # matplotlib.use('Agg') + depth = len(self.width) - 1 + for l in range(depth): + w_large = 2.0 + for i in range(self.width_in[l]): + for j in range(self.width_out[l+1]): + rank = torch.argsort(self.acts[l][:, i]) + fig, ax = plt.subplots(figsize=(w_large, w_large)) + + num = rank.shape[0] + + #print(self.width_in[l]) + #print(self.width_out[l+1]) + symbolic_mask = self.symbolic_fun[l].mask[j][i] + numeric_mask = self.act_fun[l].mask[i][j] + if symbolic_mask > 0. and numeric_mask > 0.: + color = 'purple' + alpha_mask = 1 + if symbolic_mask > 0. and numeric_mask == 0.: + color = "red" + alpha_mask = 1 + if symbolic_mask == 0. and numeric_mask > 0.: + color = "black" + alpha_mask = 1 + if symbolic_mask == 0. and numeric_mask == 0.: + color = "white" + alpha_mask = 0 + + + if tick == True: + ax.tick_params(axis="y", direction="in", pad=-22, labelsize=50) + ax.tick_params(axis="x", direction="in", pad=-15, labelsize=50) + x_min, x_max, y_min, y_max = self.get_range(l, i, j, verbose=False) + plt.xticks([x_min, x_max], ['%2.f' % x_min, '%2.f' % x_max]) + plt.yticks([y_min, y_max], ['%2.f' % y_min, '%2.f' % y_max]) + else: + plt.xticks([]) + plt.yticks([]) + if alpha_mask == 1: + plt.gca().patch.set_edgecolor('black') + else: + plt.gca().patch.set_edgecolor('white') + plt.gca().patch.set_linewidth(1.5) + # plt.axis('off') + + plt.plot(self.acts[l][:, i][rank].cpu().detach().numpy(), self.spline_postacts[l][:, j, i][rank].cpu().detach().numpy(), color=color, lw=5) + if sample == True: + plt.scatter(self.acts[l][:, i][rank].cpu().detach().numpy(), self.spline_postacts[l][:, j, i][rank].cpu().detach().numpy(), color=color, s=400 * scale ** 2) + plt.gca().spines[:].set_color(color) + + '''lock_id = self.act_fun[l].lock_id[j * self.width[l] + i].long().item() + if lock_id > 0: + im = plt.imread(f'{folder}/lock.png') + newax = fig.add_axes([0.15, 0.7, 0.15, 0.15]) + plt.text(500, 400, lock_id, fontsize=15) + newax.imshow(im) + newax.axis('off')''' + + plt.savefig(f'{folder}/sp_{l}_{i}_{j}.png', bbox_inches="tight", dpi=400) + plt.close() + + def score2alpha(score): + return np.tanh(beta * score) + + + if metric == 'forward_n': + scores = self.acts_scale + elif metric == 'forward_u': + scores = self.edge_actscale + elif metric == 'backward': + scores = self.edge_scores + else: + raise Exception(f'metric = \'{metric}\' not recognized') + + alpha = [score2alpha(score.cpu().detach().numpy()) for score in scores] + + # draw skeleton + width = np.array(self.width) + width_in = np.array(self.width_in) + width_out = np.array(self.width_out) + A = 1 + y0 = 0.3 # height: from input to pre-mult + z0 = 0.1 # height: from pre-mult to post-mult (input of next layer) + + neuron_depth = len(width) + min_spacing = A / np.maximum(np.max(width_out), 5) + + max_neuron = np.max(width_out) + max_num_weights = np.max(width_in[:-1] * width_out[1:]) + y1 = 0.4 / np.maximum(max_num_weights, 5) # size (height/width) of 1D function diagrams + y2 = 0.15 / np.maximum(max_neuron, 5) # size (height/width) of operations (sum and mult) + + fig, ax = plt.subplots(figsize=(10 * scale, 10 * scale * (neuron_depth - 1) * (y0+z0))) + # fig, ax = plt.subplots(figsize=(5,5*(neuron_depth-1)*y0)) + + # -- Transformation functions + DC_to_FC = ax.transData.transform + FC_to_NFC = fig.transFigure.inverted().transform + # -- Take data coordinates and transform them to normalized figure coordinates + DC_to_NFC = lambda x: FC_to_NFC(DC_to_FC(x)) + + # plot scatters and lines + for l in range(neuron_depth): + + n = width_in[l] + + # scatters + for i in range(n): + plt.scatter(1 / (2 * n) + i / n, l * (y0+z0), s=min_spacing ** 2 * 10000 * scale ** 2, color='black') + + # plot connections (input to pre-mult) + for i in range(n): + if l < neuron_depth - 1: + n_next = width_out[l+1] + N = n * n_next + for j in range(n_next): + id_ = i * n_next + j + + symbol_mask = self.symbolic_fun[l].mask[j][i] + numerical_mask = self.act_fun[l].mask[i][j] + if symbol_mask == 1. and numerical_mask > 0.: + color = 'purple' + alpha_mask = 1. + if symbol_mask == 1. and numerical_mask == 0.: + color = "red" + alpha_mask = 1. + if symbol_mask == 0. and numerical_mask == 1.: + color = "black" + alpha_mask = 1. + if symbol_mask == 0. and numerical_mask == 0.: + color = "white" + alpha_mask = 0. + + + if mask == True: + plt.plot([1 / (2 * n) + i / n, 1 / (2 * N) + id_ / N], [l * (y0+z0), l * (y0+z0) + y0/2 - y1], color=color, lw=2 * scale, alpha=alpha[l][j][i] * self.mask[l][i].item() * self.mask[l + 1][j].item()) + plt.plot([1 / (2 * N) + id_ / N, 1 / (2 * n_next) + j / n_next], [l * (y0+z0) + y0/2 + y1, l * (y0+z0)+y0], color=color, lw=2 * scale, alpha=alpha[l][j][i] * self.mask[l][i].item() * self.mask[l + 1][j].item()) + else: + plt.plot([1 / (2 * n) + i / n, 1 / (2 * N) + id_ / N], [l * (y0+z0), l * (y0+z0) + y0/2 - y1], color=color, lw=2 * scale, alpha=alpha[l][j][i] * alpha_mask) + plt.plot([1 / (2 * N) + id_ / N, 1 / (2 * n_next) + j / n_next], [l * (y0+z0) + y0/2 + y1, l * (y0+z0)+y0], color=color, lw=2 * scale, alpha=alpha[l][j][i] * alpha_mask) + + + # plot connections (pre-mult to post-mult, post-mult = next-layer input) + if l < neuron_depth - 1: + n_in = width_out[l+1] + n_out = width_in[l+1] + mult_id = 0 + for i in range(n_in): + if i < width[l+1][0]: + j = i + else: + if i == width[l+1][0]: + if isinstance(self.mult_arity,int): + ma = self.mult_arity + else: + ma = self.mult_arity[l+1][mult_id] + current_mult_arity = ma + if current_mult_arity == 0: + mult_id += 1 + if isinstance(self.mult_arity,int): + ma = self.mult_arity + else: + ma = self.mult_arity[l+1][mult_id] + current_mult_arity = ma + j = width[l+1][0] + mult_id + current_mult_arity -= 1 + #j = (i-width[l+1][0])//self.mult_arity + width[l+1][0] + plt.plot([1 / (2 * n_in) + i / n_in, 1 / (2 * n_out) + j / n_out], [l * (y0+z0) + y0, (l+1) * (y0+z0)], color='black', lw=2 * scale) + + + + plt.xlim(0, 1) + plt.ylim(-0.1 * (y0+z0), (neuron_depth - 1 + 0.1) * (y0+z0)) + + + plt.axis('off') + + for l in range(neuron_depth - 1): + # plot splines + n = width_in[l] + for i in range(n): + n_next = width_out[l + 1] + N = n * n_next + for j in range(n_next): + id_ = i * n_next + j + im = plt.imread(f'{folder}/sp_{l}_{i}_{j}.png') + left = DC_to_NFC([1 / (2 * N) + id_ / N - y1, 0])[0] + right = DC_to_NFC([1 / (2 * N) + id_ / N + y1, 0])[0] + bottom = DC_to_NFC([0, l * (y0+z0) + y0/2 - y1])[1] + up = DC_to_NFC([0, l * (y0+z0) + y0/2 + y1])[1] + newax = fig.add_axes([left, bottom, right - left, up - bottom]) + # newax = fig.add_axes([1/(2*N)+id_/N-y1, (l+1/2)*y0-y1, y1, y1], anchor='NE') + if mask == False: + newax.imshow(im, alpha=alpha[l][j][i]) + else: + ### make sure to run model.prune_node() first to compute mask ### + newax.imshow(im, alpha=alpha[l][j][i] * self.mask[l][i].item() * self.mask[l + 1][j].item()) + newax.axis('off') + + + # plot sum symbols + N = n = width_out[l+1] + for j in range(n): + id_ = j + path = os.path.dirname(os.path.abspath(__file__)) + "/assets/img/sum_symbol.png" + im = plt.imread(path) + left = DC_to_NFC([1 / (2 * N) + id_ / N - y2, 0])[0] + right = DC_to_NFC([1 / (2 * N) + id_ / N + y2, 0])[0] + bottom = DC_to_NFC([0, l * (y0+z0) + y0 - y2])[1] + up = DC_to_NFC([0, l * (y0+z0) + y0 + y2])[1] + newax = fig.add_axes([left, bottom, right - left, up - bottom]) + newax.imshow(im) + newax.axis('off') + + # plot mult symbols + N = n = width_in[l+1] + n_sum = width[l+1][0] + n_mult = width[l+1][1] + for j in range(n_mult): + id_ = j + n_sum + path = os.path.dirname(os.path.abspath(__file__)) + "/assets/img/mult_symbol.png" + im = plt.imread(path) + left = DC_to_NFC([1 / (2 * N) + id_ / N - y2, 0])[0] + right = DC_to_NFC([1 / (2 * N) + id_ / N + y2, 0])[0] + bottom = DC_to_NFC([0, (l+1) * (y0+z0) - y2])[1] + up = DC_to_NFC([0, (l+1) * (y0+z0) + y2])[1] + newax = fig.add_axes([left, bottom, right - left, up - bottom]) + newax.imshow(im) + newax.axis('off') + + if in_vars != None: + n = self.width_in[0] + for i in range(n): + if isinstance(in_vars[i], sympy.Expr): + plt.gcf().get_axes()[0].text(1 / (2 * (n)) + i / (n), -0.1, f'${latex(in_vars[i])}$', fontsize=40 * scale * varscale, horizontalalignment='center', verticalalignment='center') + else: + plt.gcf().get_axes()[0].text(1 / (2 * (n)) + i / (n), -0.1, in_vars[i], fontsize=40 * scale * varscale, horizontalalignment='center', verticalalignment='center') + + + + if out_vars != None: + n = self.width_in[-1] + for i in range(n): + if isinstance(out_vars[i], sympy.Expr): + plt.gcf().get_axes()[0].text(1 / (2 * (n)) + i / (n), (y0+z0) * (len(self.width) - 1) + 0.15, f'${latex(out_vars[i])}$', fontsize=40 * scale * varscale, horizontalalignment='center', verticalalignment='center') + else: + plt.gcf().get_axes()[0].text(1 / (2 * (n)) + i / (n), (y0+z0) * (len(self.width) - 1) + 0.15, out_vars[i], fontsize=40 * scale * varscale, horizontalalignment='center', verticalalignment='center') + + if title != None: + plt.gcf().get_axes()[0].text(0.5, (y0+z0) * (len(self.width) - 1) + 0.3, title, fontsize=40 * scale, horizontalalignment='center', verticalalignment='center') + + + def reg(self, reg_metric, lamb_l1, lamb_entropy, lamb_coef, lamb_coefdiff): + + if reg_metric == 'edge_forward_n': + acts_scale = self.acts_scale_spline + + if reg_metric == 'edge_forward_u': + acts_scale = self.edge_actscale + + if reg_metric == 'edge_backward': + acts_scale = self.edge_scores + + if reg_metric == 'node_backward': + acts_scale = self.node_attribute_scores + + reg_ = 0. + for i in range(len(acts_scale)): + vec = acts_scale[i] + + l1 = torch.sum(vec) + p_row = vec / (torch.sum(vec, dim=1, keepdim=True) + 1) + p_col = vec / (torch.sum(vec, dim=0, keepdim=True) + 1) + entropy_row = - torch.mean(torch.sum(p_row * torch.log2(p_row + 1e-4), dim=1)) + entropy_col = - torch.mean(torch.sum(p_col * torch.log2(p_col + 1e-4), dim=0)) + #entropy_row = torch.max(-torch.sum(p_row * torch.log2(p_row + 1e-4), dim=1)) + #entropy_col = torch.max(-torch.sum(p_col * torch.log2(p_col + 1e-4), dim=0)) + reg_ += lamb_l1 * l1 + lamb_entropy * (entropy_row + entropy_col) # both l1 and entropy + '''vec = vec.reshape(-1,) + p = vec / (torch.sum(vec) + 1e-4) + entropy = - torch.sum(p * torch.log2(p + 1e-4)) + reg_ += lamb_l1 * l1 + lamb_entropy * entropy # both l1 and entropy''' + + # regularize coefficient to encourage spline to be zero + for i in range(len(self.act_fun)): + coeff_l1 = torch.sum(torch.mean(torch.abs(self.act_fun[i].coef), dim=1)) + coeff_diff_l1 = torch.sum(torch.mean(torch.abs(torch.diff(self.act_fun[i].coef)), dim=1)) + reg_ += lamb_coef * coeff_l1 + lamb_coefdiff * coeff_diff_l1 + + return reg_ + + def get_reg(self, reg_metric, lamb_l1, lamb_entropy, lamb_coef, lamb_coefdiff): + return self.reg(reg_metric, lamb_l1, lamb_entropy, lamb_coef, lamb_coefdiff) + + def disable_symbolic_in_fit(self, lamb): + + old_save_act = self.save_act + if lamb == 0.: + self.save_act = False + + # skip symbolic if no symbolic is turned on + depth = len(self.symbolic_fun) + no_symbolic = True + for l in range(depth): + no_symbolic *= torch.sum(torch.abs(self.symbolic_fun[l].mask)) == 0 + + old_symbolic_enabled = self.symbolic_enabled + + if no_symbolic: + self.symbolic_enabled = False + + return old_save_act, old_symbolic_enabled + + def get_params(self): + return self.parameters() + + + def fit(self, dataset, opt="LBFGS", steps=100, log=1, lamb=0., lamb_l1=1., lamb_entropy=2., lamb_coef=0., lamb_coefdiff=0., update_grid=True, grid_update_num=10, loss_fn=None, lr=1.,start_grid_update_step=-1, stop_grid_update_step=50, batch=-1, + metrics=None, save_fig=False, in_vars=None, out_vars=None, beta=3, save_fig_freq=1, img_folder='./video', singularity_avoiding=False, y_th=1000., reg_metric='edge_backward', display_metrics=None): + + if lamb > 0. and not self.save_act: + print('setting lamb=0. If you want to set lamb > 0, set self.save_act=True') + + old_save_act, old_symbolic_enabled = self.disable_symbolic_in_fit(lamb) + + pbar = tqdm(range(steps), desc='description', ncols=100) + + if loss_fn == None: + loss_fn = loss_fn_eval = lambda x, y: torch.mean((x - y) ** 2) + else: + loss_fn = loss_fn_eval = loss_fn + + grid_update_freq = int(stop_grid_update_step / grid_update_num) + + if opt == "Adam": + optimizer = torch.optim.Adam(self.get_params(), lr=lr) + elif opt == "LBFGS": + optimizer = LBFGS(self.get_params(), lr=lr, history_size=10, line_search_fn="strong_wolfe", tolerance_grad=1e-32, tolerance_change=1e-32, tolerance_ys=1e-32) + + results = {} + results['train_loss'] = [] + results['test_loss'] = [] + results['reg'] = [] + if metrics != None: + for i in range(len(metrics)): + results[metrics[i].__name__] = [] + + if batch == -1 or batch > dataset['train_input'].shape[0]: + batch_size = dataset['train_input'].shape[0] + batch_size_test = dataset['test_input'].shape[0] + else: + batch_size = batch + batch_size_test = batch + + global train_loss, reg_ + + def closure(): + global train_loss, reg_ + optimizer.zero_grad() + pred = self.forward(dataset['train_input'][train_id], singularity_avoiding=singularity_avoiding, y_th=y_th) + train_loss = loss_fn(pred, dataset['train_label'][train_id]) + if self.save_act: + if reg_metric == 'edge_backward': + self.attribute() + if reg_metric == 'node_backward': + self.node_attribute() + reg_ = self.get_reg(reg_metric, lamb_l1, lamb_entropy, lamb_coef, lamb_coefdiff) + else: + reg_ = torch.tensor(0.) + objective = train_loss + lamb * reg_ + objective.backward() + return objective + + if save_fig: + if not os.path.exists(img_folder): + os.makedirs(img_folder) + + for _ in pbar: + + if _ == steps-1 and old_save_act: + self.save_act = True + + train_id = np.random.choice(dataset['train_input'].shape[0], batch_size, replace=False) + test_id = np.random.choice(dataset['test_input'].shape[0], batch_size_test, replace=False) + + if _ % grid_update_freq == 0 and _ < stop_grid_update_step and update_grid and _ >= start_grid_update_step: + self.update_grid(dataset['train_input'][train_id]) + + if opt == "LBFGS": + optimizer.step(closure) + + if opt == "Adam": + pred = self.forward(dataset['train_input'][train_id], singularity_avoiding=singularity_avoiding, y_th=y_th) + train_loss = loss_fn(pred, dataset['train_label'][train_id]) + if self.save_act: + if reg_metric == 'edge_backward': + self.attribute() + if reg_metric == 'node_backward': + self.node_attribute() + reg_ = self.get_reg(reg_metric, lamb_l1, lamb_entropy, lamb_coef, lamb_coefdiff) + else: + reg_ = torch.tensor(0.) + loss = train_loss + lamb * reg_ + optimizer.zero_grad() + loss.backward() + optimizer.step() + + test_loss = loss_fn_eval(self.forward(dataset['test_input'][test_id]), dataset['test_label'][test_id]) + + + if metrics != None: + for i in range(len(metrics)): + results[metrics[i].__name__].append(metrics[i]().item()) + + results['train_loss'].append(torch.sqrt(train_loss).cpu().detach().numpy()) + results['test_loss'].append(torch.sqrt(test_loss).cpu().detach().numpy()) + results['reg'].append(reg_.cpu().detach().numpy()) + + if _ % log == 0: + if display_metrics == None: + pbar.set_description("| train_loss: %.2e | test_loss: %.2e | reg: %.2e | " % (torch.sqrt(train_loss).cpu().detach().numpy(), torch.sqrt(test_loss).cpu().detach().numpy(), reg_.cpu().detach().numpy())) + else: + string = '' + data = () + for metric in display_metrics: + string += f' {metric}: %.2e |' + try: + results[metric] + except: + raise Exception(f'{metric} not recognized') + data += (results[metric][-1],) + pbar.set_description(string % data) + + + if save_fig and _ % save_fig_freq == 0: + self.plot(folder=img_folder, in_vars=in_vars, out_vars=out_vars, title="Step {}".format(_), beta=beta) + plt.savefig(img_folder + '/' + str(_) + '.jpg', bbox_inches='tight', dpi=200) + plt.close() + + self.log_history('fit') + # revert back to original state + self.symbolic_enabled = old_symbolic_enabled + return results + + def prune_node(self, threshold=1e-2, mode="auto", active_neurons_id=None, log_history=True): + + if self.acts == None: + self.get_act() + + mask_up = [torch.ones(self.width_in[0], )] + mask_down = [] + active_neurons_up = [list(range(self.width_in[0]))] + active_neurons_down = [] + num_sums = [] + num_mults = [] + mult_arities = [[]] + + if active_neurons_id != None: + mode = "manual" + + for i in range(len(self.acts_scale) - 1): + + mult_arity = [] + + if mode == "auto": + self.attribute() + overall_important_up = self.node_scores[i+1] > threshold + + elif mode == "manual": + overall_important_up = torch.zeros(self.width_in[i + 1], dtype=torch.bool) + overall_important_up[active_neurons_id[i]] = True + + + num_sum = torch.sum(overall_important_up[:self.width[i+1][0]]) + num_mult = torch.sum(overall_important_up[self.width[i+1][0]:]) + if self.mult_homo == True: + overall_important_down = torch.cat([overall_important_up[:self.width[i+1][0]], (overall_important_up[self.width[i+1][0]:][None,:].expand(self.mult_arity,-1)).T.reshape(-1,)], dim=0) + else: + overall_important_down = overall_important_up[:self.width[i+1][0]] + for j in range(overall_important_up[self.width[i+1][0]:].shape[0]): + active_bool = overall_important_up[self.width[i+1][0]+j] + arity = self.mult_arity[i+1][j] + overall_important_down = torch.cat([overall_important_down, torch.tensor([active_bool]*arity)]) + if active_bool: + mult_arity.append(arity) + + num_sums.append(num_sum.item()) + num_mults.append(num_mult.item()) + + mask_up.append(overall_important_up.float()) + mask_down.append(overall_important_down.float()) + + active_neurons_up.append(torch.where(overall_important_up == True)[0]) + active_neurons_down.append(torch.where(overall_important_down == True)[0]) + + mult_arities.append(mult_arity) + + active_neurons_down.append(list(range(self.width_out[-1]))) + mask_down.append(torch.ones(self.width_out[-1], )) + + if self.mult_homo == False: + mult_arities.append(self.mult_arity[-1]) + + self.mask_up = mask_up + self.mask_down = mask_down + + # update act_fun[l].mask up + for l in range(len(self.acts_scale) - 1): + for i in range(self.width_in[l + 1]): + if i not in active_neurons_up[l + 1]: + self.remove_node(l + 1, i, mode='up',log_history=False) + + for i in range(self.width_out[l + 1]): + if i not in active_neurons_down[l]: + self.remove_node(l + 1, i, mode='down',log_history=False) + + model2 = MultKAN(copy.deepcopy(self.width), grid=self.grid, k=self.k, base_fun=self.base_fun_name, mult_arity=self.mult_arity, ckpt_path=self.ckpt_path, auto_save=True, first_init=False, state_id=self.state_id, round=self.round) + model2.load_state_dict(self.state_dict()) + + width_new = [self.width[0]] + + for i in range(len(self.acts_scale)): + + if i < len(self.acts_scale) - 1: + num_sum = num_sums[i] + num_mult = num_mults[i] + model2.node_bias[i].data = model2.node_bias[i].data[active_neurons_up[i+1]] + model2.node_scale[i].data = model2.node_scale[i].data[active_neurons_up[i+1]] + model2.subnode_bias[i].data = model2.subnode_bias[i].data[active_neurons_down[i]] + model2.subnode_scale[i].data = model2.subnode_scale[i].data[active_neurons_down[i]] + model2.width[i+1] = [num_sum, num_mult] + + model2.act_fun[i].out_dim_sum = num_sum + model2.act_fun[i].out_dim_mult = num_mult + + model2.symbolic_fun[i].out_dim_sum = num_sum + model2.symbolic_fun[i].out_dim_mult = num_mult + + width_new.append([num_sum, num_mult]) + + model2.act_fun[i] = model2.act_fun[i].get_subset(active_neurons_up[i], active_neurons_down[i]) + model2.symbolic_fun[i] = self.symbolic_fun[i].get_subset(active_neurons_up[i], active_neurons_down[i]) + + model2.cache_data = self.cache_data + model2.acts = None + + width_new.append(self.width[-1]) + model2.width = width_new + + if self.mult_homo == False: + model2.mult_arity = mult_arities + + if log_history: + self.log_history('prune_node') + model2.state_id += 1 + + return model2 + + def prune_edge(self, threshold=3e-2, log_history=True): + + if self.acts == None: + self.get_act() + + for i in range(len(self.width)-1): + #self.act_fun[i].mask.data = ((self.acts_scale[i] > threshold).permute(1,0)).float() + old_mask = self.act_fun[i].mask.data + self.act_fun[i].mask.data = ((self.edge_scores[i] > threshold).permute(1,0)*old_mask).float() + + if log_history: + self.log_history('fix_symbolic') + + def prune(self, node_th=1e-2, edge_th=3e-2): + + if self.acts == None: + self.get_act() + + self = self.prune_node(node_th, log_history=False) + #self.prune_node(node_th, log_history=False) + self.forward(self.cache_data) + self.attribute() + self.prune_edge(edge_th, log_history=False) + self.log_history('prune') + return self + + def prune_input(self, threshold=1e-2, active_inputs=None, log_history=True): + + if active_inputs == None: + self.attribute() + input_score = self.node_scores[0] + input_mask = input_score > threshold + print('keep:', input_mask.tolist()) + input_id = torch.where(input_mask==True)[0] + + else: + input_id = torch.tensor(active_inputs, dtype=torch.long) + + model2 = MultKAN(copy.deepcopy(self.width), grid=self.grid, k=self.k, base_fun=self.base_fun, mult_arity=self.mult_arity, ckpt_path=self.ckpt_path, auto_save=True, first_init=False, state_id=self.state_id, round=self.round) + model2.load_state_dict(self.state_dict()) + + model2.act_fun[0] = model2.act_fun[0].get_subset(input_id, torch.arange(self.width_out[1])) + model2.symbolic_fun[0] = self.symbolic_fun[0].get_subset(input_id, torch.arange(self.width_out[1])) + + model2.cache_data = self.cache_data + model2.acts = None + + model2.width[0] = [len(input_id), 0] + model2.input_id = input_id + + if log_history: + self.log_history('prune_input') + model2.state_id += 1 + + return model2 + + def remove_edge(self, l, i, j, log_history=True): + self.act_fun[l].mask[i][j] = 0. + if log_history: + self.log_history('remove_edge') + + def remove_node(self, l ,i, mode='all', log_history=True): + if mode == 'down': + self.act_fun[l - 1].mask[:, i] = 0. + self.symbolic_fun[l - 1].mask[i, :] *= 0. + + elif mode == 'up': + self.act_fun[l].mask[i, :] = 0. + self.symbolic_fun[l].mask[:, i] *= 0. + + else: + self.remove_node(l, i, mode='up') + self.remove_node(l, i, mode='down') + + if log_history: + self.log_history('remove_node') + + + def attribute(self, l=None, i=None, out_score=None, plot=True): + + # output (out_dim, in_dim) + + if l != None: + self.attribute() + out_score = self.node_scores[l] + + if self.acts == None: + self.get_act() + + def score_node2subnode(node_score, width, mult_arity, out_dim): + + assert np.sum(width) == node_score.shape[1] + if isinstance(mult_arity, int): + n_subnode = width[0] + mult_arity * width[1] + else: + n_subnode = width[0] + int(np.sum(mult_arity)) + + #subnode_score_leaf = torch.zeros(out_dim, n_subnode).requires_grad_(True) + #subnode_score = subnode_score_leaf.clone() + #subnode_score[:,:width[0]] = node_score[:,:width[0]] + subnode_score = node_score[:,:width[0]] + if isinstance(mult_arity, int): + #subnode_score[:,width[0]:] = node_score[:,width[0]:][:,:,None].expand(out_dim, node_score[width[0]:].shape[0], mult_arity).reshape(out_dim,-1) + subnode_score = torch.cat([subnode_score, node_score[:,width[0]:][:,:,None].expand(out_dim, node_score[width[0]:].shape[0], mult_arity).reshape(out_dim,-1)], dim=1) + else: + acml = width[0] + for i in range(len(mult_arity)): + #subnode_score[:, acml:acml+mult_arity[i]] = node_score[:, width[0]+i] + subnode_score = torch.cat([subnode_score, node_score[:, width[0]+i]].expand(out_dim, mult_arity[i]), dim=1) + acml += mult_arity[i] + return subnode_score + + + node_scores = [] + subnode_scores = [] + edge_scores = [] + + l_query = l + if l == None: + l_end = self.depth + else: + l_end = l + + # back propagate from the queried layer + out_dim = self.width_in[l_end] + if out_score == None: + node_score = torch.eye(out_dim).requires_grad_(True) + else: + node_score = torch.diag(out_score).requires_grad_(True) + node_scores.append(node_score) + + device = self.act_fun[0].grid.device + + for l in range(l_end,0,-1): + + # node to subnode + if isinstance(self.mult_arity, int): + subnode_score = score_node2subnode(node_score, self.width[l], self.mult_arity, out_dim=out_dim) + else: + mult_arity = self.mult_arity[l] + subnode_score = score_node2subnode(node_score, self.width[l], mult_arity) + + subnode_scores.append(subnode_score) + # subnode to edge + #print(self.edge_actscale[l-1].device, subnode_score.device, self.subnode_actscale[l-1].device) + edge_score = torch.einsum('ij,ki,i->kij', self.edge_actscale[l-1], subnode_score.to(device), 1/(self.subnode_actscale[l-1]+1e-4)) + edge_scores.append(edge_score) + + # edge to node + node_score = torch.sum(edge_score, dim=1) + node_scores.append(node_score) + + self.node_scores_all = list(reversed(node_scores)) + self.edge_scores_all = list(reversed(edge_scores)) + self.subnode_scores_all = list(reversed(subnode_scores)) + + self.node_scores = [torch.mean(l, dim=0) for l in self.node_scores_all] + self.edge_scores = [torch.mean(l, dim=0) for l in self.edge_scores_all] + self.subnode_scores = [torch.mean(l, dim=0) for l in self.subnode_scores_all] + + # return + if l_query != None: + if i == None: + return self.node_scores_all[0] + else: + + # plot + if plot: + in_dim = self.width_in[0] + plt.figure(figsize=(1*in_dim, 3)) + plt.bar(range(in_dim),self.node_scores_all[0][i].detach().numpy()) + plt.xticks(range(in_dim)); + + return self.node_scores_all[0][i] + + def node_attribute(self): + self.node_attribute_scores = [] + for l in range(1, self.depth+1): + node_attr = self.attribute(l) + self.node_attribute_scores.append(node_attr) + + def feature_interaction(self, l, neuron_th = 1e-2, feature_th = 1e-2): + + dic = {} + width = self.width_in[l] + + for i in range(width): + score = self.attribute(l,i,plot=False) + + if torch.max(score) > neuron_th: + features = tuple(torch.where(score > torch.max(score) * feature_th)[0].detach().numpy()) + if features in dic.keys(): + dic[features] += 1 + else: + dic[features] = 1 + + return dic + + def suggest_symbolic(self, l, i, j, a_range=(-10, 10), b_range=(-10, 10), lib=None, topk=5, verbose=True, r2_loss_fun=lambda x: np.log2(1+1e-5-x), c_loss_fun=lambda x: x, weight_simple = 0.8): + + r2s = [] + cs = [] + + if lib == None: + symbolic_lib = SYMBOLIC_LIB + else: + symbolic_lib = {} + for item in lib: + symbolic_lib[item] = SYMBOLIC_LIB[item] + + # getting r2 and complexities + for (name, content) in symbolic_lib.items(): + r2 = self.fix_symbolic(l, i, j, name, a_range=a_range, b_range=b_range, verbose=False, log_history=False) + if r2 == -1e8: # zero function + r2s.append(-1e8) + else: + r2s.append(r2.item()) + self.unfix_symbolic(l, i, j, log_history=False) + c = content[2] + cs.append(c) + + r2s = np.array(r2s) + cs = np.array(cs) + r2_loss = r2_loss_fun(r2s).astype('float') + cs_loss = c_loss_fun(cs) + + loss = weight_simple * cs_loss + (1-weight_simple) * r2_loss + + sorted_ids = np.argsort(loss)[:topk] + r2s = r2s[sorted_ids][:topk] + cs = cs[sorted_ids][:topk] + r2_loss = r2_loss[sorted_ids][:topk] + cs_loss = cs_loss[sorted_ids][:topk] + loss = loss[sorted_ids][:topk] + + topk = np.minimum(topk, len(symbolic_lib)) + + if verbose == True: + # print results in a dataframe + results = {} + results['function'] = [list(symbolic_lib.items())[sorted_ids[i]][0] for i in range(topk)] + results['fitting r2'] = r2s[:topk] + results['r2 loss'] = r2_loss[:topk] + results['complexity'] = cs[:topk] + results['complexity loss'] = cs_loss[:topk] + results['total loss'] = loss[:topk] + + df = pd.DataFrame(results) + print(df) + + '''if verbose == True: + print('function', ',', 'r2', ',', 'c', ',', 'r2 loss', ',', 'c loss', ',', 'total loss') + for i in range(topk): + print(list(symbolic_lib.items())[sorted_ids[i]][0], ',', r2s[i], ',', cs[i], ',', r2_loss[i], ',', cs_loss[i], ',', loss[i])''' + + best_name = list(symbolic_lib.items())[sorted_ids[0]][0] + best_fun = list(symbolic_lib.items())[sorted_ids[0]][1] + best_r2 = r2s[0] + best_c = cs[0] + + '''if best_r2 < 1e-3: + # zero function + zero_id = list(SYMBOLIC_LIB).index('0') + best_r2 = 0.0 + best_name = '0' + best_fun = list(symbolic_lib.items())[zero_id][1] + best_c = 0.0 + print('behave like a zero function')''' + + return best_name, best_fun, best_r2, best_c; + + def auto_symbolic(self, a_range=(-10, 10), b_range=(-10, 10), lib=None, verbose=1): + for l in range(len(self.width_in) - 1): + for i in range(self.width_in[l]): + for j in range(self.width_out[l + 1]): + #if self.symbolic_fun[l].mask[j, i] > 0. and self.act_fun[l].mask[i][j] == 0.: + if self.symbolic_fun[l].mask[j, i] > 0. and self.act_fun[l].mask[i][j] == 0.: + print(f'skipping ({l},{i},{j}) since already symbolic') + elif self.symbolic_fun[l].mask[j, i] == 0. and self.act_fun[l].mask[i][j] == 0.: + self.fix_symbolic(l, i, j, '0', verbose=verbose > 1, log_history=False) + print(f'fixing ({l},{i},{j}) with 0') + else: + name, fun, r2, c = self.suggest_symbolic(l, i, j, a_range=a_range, b_range=b_range, lib=lib, verbose=False) + self.fix_symbolic(l, i, j, name, verbose=verbose > 1, log_history=False) + if verbose >= 1: + print(f'fixing ({l},{i},{j}) with {name}, r2={r2}, c={c}') + + self.log_history('auto_symbolic') + + def symbolic_formula(self, compute_digit=5, display_digit=3, var=None, normalizer=None, simplify=False, output_normalizer = None): + + symbolic_acts = [] + symbolic_acts_premult = [] + x = [] + + def ex_round(ex1, n_digit): + ex2 = ex1 + for a in sympy.preorder_traversal(ex1): + if isinstance(a, sympy.Float): + ex2 = ex2.subs(a, round(a, n_digit)) + return ex2 + + # define variables + if var == None: + for ii in range(1, self.width[0][0] + 1): + exec(f"x{ii} = sympy.Symbol('x_{ii}')") + exec(f"x.append(x{ii})") + elif type(var[0]) == Symbol: + x = var + else: + x = [sympy.symbols(var_) for var_ in var] + + x0 = x + + if normalizer != None: + mean = normalizer[0] + std = normalizer[1] + x = [(x[i] - mean[i]) / std[i] for i in range(len(x))] + + symbolic_acts.append(x) + + for l in range(len(self.width_in) - 1): + num_sum = self.width[l + 1][0] + num_mult = self.width[l + 1][1] + y = [] + for j in range(self.width_out[l + 1]): + yj = 0. + for i in range(self.width_in[l]): + a, b, c, d = self.symbolic_fun[l].affine[j, i] + sympy_fun = self.symbolic_fun[l].funs_sympy[j][i] + try: + yj += c * sympy_fun(a * x[i] + b) + d + except: + print('make sure all activations need to be converted to symbolic formulas first!') + return + yj = self.subnode_scale[l][j] * yj + self.subnode_bias[l][j] + if simplify == True: + y.append(sympy.simplify(yj)) + else: + y.append(yj) + + symbolic_acts_premult.append(y) + + mult = [] + for k in range(num_mult): + if isinstance(self.mult_arity, int): + mult_arity = self.mult_arity + else: + mult_arity = self.mult_arity[l+1][k] + for i in range(mult_arity-1): + if i == 0: + mult_k = y[num_sum+2*k] * y[num_sum+2*k+1] + else: + mult_k = mult_k * y[num_sum+2*k+i+1] + mult.append(mult_k) + + y = y[:num_sum] + mult + + for j in range(self.width_in[l+1]): + y[j] = self.node_scale[l][j] * y[j] + self.node_bias[l][j] + + x = y + symbolic_acts.append(x) + + if output_normalizer != None: + output_layer = symbolic_acts[-1] + means = output_normalizer[0] + stds = output_normalizer[1] + + assert len(output_layer) == len(means), 'output_normalizer does not match the output layer' + assert len(output_layer) == len(stds), 'output_normalizer does not match the output layer' + + output_layer = [(output_layer[i] * stds[i] + means[i]) for i in range(len(output_layer))] + symbolic_acts[-1] = output_layer + + + self.symbolic_acts = [[symbolic_acts[l][i] for i in range(len(symbolic_acts[l]))] for l in range(len(symbolic_acts))] + self.symbolic_acts_premult = [[symbolic_acts_premult[l][i] for i in range(len(symbolic_acts_premult[l]))] for l in range(len(symbolic_acts_premult))] + + out_dim = len(symbolic_acts[-1]) + #return [symbolic_acts[-1][i] for i in range(len(symbolic_acts[-1]))], x0 + + if simplify: + return [symbolic_acts[-1][i] for i in range(len(symbolic_acts[-1]))], x0 + else: + return [symbolic_acts[-1][i] for i in range(len(symbolic_acts[-1]))], x0 + + + def expand_depth(self): + + self.depth += 1 + + # add kanlayer, set mask to zero + dim_out = self.width_in[-1] + layer = KANLayer(dim_out, dim_out, num=self.grid, k=self.k) + layer.mask *= 0. + self.act_fun.append(layer) + + self.width.append([dim_out, 0]) + self.mult_arity.append([]) + + # add symbolic_kanlayer set mask to one. fun = identity on diagonal and zero for off-diagonal + layer = Symbolic_KANLayer(dim_out, dim_out) + layer.mask += 1. + + for j in range(dim_out): + for i in range(dim_out): + if i == j: + layer.fix_symbolic(i,j,'x') + else: + layer.fix_symbolic(i,j,'0') + + self.symbolic_fun.append(layer) + + self.node_bias.append(torch.nn.Parameter(torch.zeros(dim_out,)).requires_grad_(self.affine_trainable)) + self.node_scale.append(torch.nn.Parameter(torch.ones(dim_out,)).requires_grad_(self.affine_trainable)) + self.subnode_bias.append(torch.nn.Parameter(torch.zeros(dim_out,)).requires_grad_(self.affine_trainable)) + self.subnode_scale.append(torch.nn.Parameter(torch.ones(dim_out,)).requires_grad_(self.affine_trainable)) + + def expand_width(self, layer_id, n_added_nodes, sum_bool=True, mult_arity=2): + + def _expand(layer_id, n_added_nodes, sum_bool=True, mult_arity=2, added_dim='out'): + l = layer_id + in_dim = self.symbolic_fun[l].in_dim + out_dim = self.symbolic_fun[l].out_dim + if sum_bool: + + if added_dim == 'out': + new = Symbolic_KANLayer(in_dim, out_dim + n_added_nodes) + old = self.symbolic_fun[l] + in_id = np.arange(in_dim) + out_id = np.arange(out_dim + n_added_nodes) + + for j in out_id: + for i in in_id: + new.fix_symbolic(i,j,'0') + new.mask += 1. + + for j in out_id: + for i in in_id: + if j > n_added_nodes-1: + new.funs[j][i] = old.funs[j-n_added_nodes][i] + new.funs_avoid_singularity[j][i] = old.funs_avoid_singularity[j-n_added_nodes][i] + new.funs_sympy[j][i] = old.funs_sympy[j-n_added_nodes][i] + new.funs_name[j][i] = old.funs_name[j-n_added_nodes][i] + new.affine.data[j][i] = old.affine.data[j-n_added_nodes][i] + + self.symbolic_fun[l] = new + self.act_fun[l] = KANLayer(in_dim, out_dim + n_added_nodes, num=self.grid, k=self.k) + self.act_fun[l].mask *= 0. + + self.node_scale[l].data = torch.cat([torch.ones(n_added_nodes), self.node_scale[l].data]) + self.node_bias[l].data = torch.cat([torch.zeros(n_added_nodes), self.node_bias[l].data]) + self.subnode_scale[l].data = torch.cat([torch.ones(n_added_nodes), self.subnode_scale[l].data]) + self.subnode_bias[l].data = torch.cat([torch.zeros(n_added_nodes), self.subnode_bias[l].data]) + + + + if added_dim == 'in': + new = Symbolic_KANLayer(in_dim + n_added_nodes, out_dim) + old = self.symbolic_fun[l] + in_id = np.arange(in_dim + n_added_nodes) + out_id = np.arange(out_dim) + + for j in out_id: + for i in in_id: + new.fix_symbolic(i,j,'0') + new.mask += 1. + + for j in out_id: + for i in in_id: + if i > n_added_nodes-1: + new.funs[j][i] = old.funs[j][i-n_added_nodes] + new.funs_avoid_singularity[j][i] = old.funs_avoid_singularity[j][i-n_added_nodes] + new.funs_sympy[j][i] = old.funs_sympy[j][i-n_added_nodes] + new.funs_name[j][i] = old.funs_name[j][i-n_added_nodes] + new.affine.data[j][i] = old.affine.data[j][i-n_added_nodes] + + self.symbolic_fun[l] = new + self.act_fun[l] = KANLayer(in_dim + n_added_nodes, out_dim, num=self.grid, k=self.k) + self.act_fun[l].mask *= 0. + + + else: + + if isinstance(mult_arity, int): + mult_arity = [mult_arity] * n_added_nodes + + if added_dim == 'out': + n_added_subnodes = np.sum(mult_arity) + new = Symbolic_KANLayer(in_dim, out_dim + n_added_subnodes) + old = self.symbolic_fun[l] + in_id = np.arange(in_dim) + out_id = np.arange(out_dim + n_added_nodes) + + for j in out_id: + for i in in_id: + new.fix_symbolic(i,j,'0') + new.mask += 1. + + for j in out_id: + for i in in_id: + if j < out_dim: + new.funs[j][i] = old.funs[j][i] + new.funs_avoid_singularity[j][i] = old.funs_avoid_singularity[j][i] + new.funs_sympy[j][i] = old.funs_sympy[j][i] + new.funs_name[j][i] = old.funs_name[j][i] + new.affine.data[j][i] = old.affine.data[j][i] + + self.symbolic_fun[l] = new + self.act_fun[l] = KANLayer(in_dim, out_dim + n_added_subnodes, num=self.grid, k=self.k) + self.act_fun[l].mask *= 0. + + self.node_scale[l].data = torch.cat([self.node_scale[l].data, torch.ones(n_added_nodes)]) + self.node_bias[l].data = torch.cat([self.node_bias[l].data, torch.zeros(n_added_nodes)]) + self.subnode_scale[l].data = torch.cat([self.subnode_scale[l].data, torch.ones(n_added_subnodes)]) + self.subnode_bias[l].data = torch.cat([self.subnode_bias[l].data, torch.zeros(n_added_subnodes)]) + + if added_dim == 'in': + new = Symbolic_KANLayer(in_dim + n_added_nodes, out_dim) + old = self.symbolic_fun[l] + in_id = np.arange(in_dim + n_added_nodes) + out_id = np.arange(out_dim) + + for j in out_id: + for i in in_id: + new.fix_symbolic(i,j,'0') + new.mask += 1. + + for j in out_id: + for i in in_id: + if i < in_dim: + new.funs[j][i] = old.funs[j][i] + new.funs_avoid_singularity[j][i] = old.funs_avoid_singularity[j][i] + new.funs_sympy[j][i] = old.funs_sympy[j][i] + new.funs_name[j][i] = old.funs_name[j][i] + new.affine.data[j][i] = old.affine.data[j][i] + + self.symbolic_fun[l] = new + self.act_fun[l] = KANLayer(in_dim + n_added_nodes, out_dim, num=self.grid, k=self.k) + self.act_fun[l].mask *= 0. + + _expand(layer_id-1, n_added_nodes, sum_bool, mult_arity, added_dim='out') + _expand(layer_id, n_added_nodes, sum_bool, mult_arity, added_dim='in') + if sum_bool: + self.width[layer_id][0] += n_added_nodes + else: + if isinstance(mult_arity, int): + mult_arity = [mult_arity] * n_added_nodes + + self.width[layer_id][1] += n_added_nodes + self.mult_arity[layer_id] += mult_arity + + def perturb(self, mag=0.02, mode='all'): + if mode == 'all': + for i in range(self.depth): + self.act_fun[i].mask += self.act_fun[i].mask*0. + mag + + if mode == 'minimal': + for l in range(self.depth): + funs_name = self.symbolic_fun[l].funs_name + for j in range(self.width_out[l+1]): + for i in range(self.width_in[l]): + if funs_name[j][i] != '0': + self.act_fun[l].mask.data[i][j] = mag + + self.log_history('perturb') + + + def module(self, start_layer, chain): + #chain = '[-1]->[-1,-2]->[-1]->[-1]' + groups = chain.split('->') + n_total_layers = len(groups)//2 + #start_layer = 0 + + for l in range(n_total_layers): + current_layer = cl = start_layer + l + id_in = [int(i) for i in groups[2*l][1:-1].split(',')] + id_out = [int(i) for i in groups[2*l+1][1:-1].split(',')] + + in_dim = self.width_in[cl] + out_dim = self.width_out[cl+1] + id_in_other = list(set(range(in_dim)) - set(id_in)) + id_out_other = list(set(range(out_dim)) - set(id_out)) + self.act_fun[cl].mask.data[np.ix_(id_in_other,id_out)] = 0. + self.act_fun[cl].mask.data[np.ix_(id_in,id_out_other)] = 0. + self.symbolic_fun[cl].mask.data[np.ix_(id_out,id_in_other)] = 0. + self.symbolic_fun[cl].mask.data[np.ix_(id_out_other,id_in)] = 0. + + self.log_history('module') + + def tree(self, x=None, in_var=None, style='tree', sym_th=1e-3, sep_th=1e-1, skip_sep_test=False, verbose=False): + if x == None: + x = self.cache_data + plot_tree(self, x, in_var=in_var, style=style, sym_th=sym_th, sep_th=sep_th, skip_sep_test=skip_sep_test, verbose=verbose) + + + def speed(self, compile=False): + self.symbolic_enabled=False + self.save_act=False + self.auto_save=False + if compile == True: + return torch.compile(self) + else: + return self + + def get_act(self, x=None): + if isinstance(x, dict): + x = x['train_input'] + if x == None: + if self.cache_data != None: + x = self.cache_data + else: + raise Exception("missing input data x") + save_act = self.save_act + self.save_act = True + self.forward(x) + self.save_act = save_act + + def get_fun(self, l, i, j): + inputs = self.spline_preacts[l][:,j,i] + outputs = self.spline_postacts[l][:,j,i] + # they are not ordered yet + rank = np.argsort(inputs) + inputs = inputs[rank] + outputs = outputs[rank] + plt.figure(figsize=(3,3)) + plt.plot(inputs, outputs, marker="o") + return inputs, outputs + + + def history(self, k='all'): + + with open(self.ckpt_path+'/history.txt', 'r') as f: + data = f.readlines() + n_line = len(data) + if k == 'all': + k = n_line + + data = data[-k:] + for line in data: + print(line[:-1]) + @property + def n_edge(self): + depth = len(self.act_fun) + complexity = 0 + for l in range(depth): + complexity += torch.sum(self.act_fun[l].mask > 0.) + return complexity.item() + + def evaluate(self, dataset): + evaluation = {} + evaluation['test_loss'] = torch.sqrt(torch.mean((self.forward(dataset['test_input']) - dataset['test_label'])**2)).item() + evaluation['n_edge'] = self.n_edge + evaluation['n_grid'] = self.grid + # add other metrics (maybe accuracy) + return evaluation + + def swap(self, l, i1, i2, log_history=True): + + self.act_fun[l-1].swap(i1,i2,mode='out') + self.symbolic_fun[l-1].swap(i1,i2,mode='out') + self.act_fun[l].swap(i1,i2,mode='in') + self.symbolic_fun[l].swap(i1,i2,mode='in') + + def swap_(data, i1, i2): + data[i1], data[i2] = data[i2], data[i1] + + swap_(self.node_scale[l-1].data, i1, i2) + swap_(self.node_bias[l-1].data, i1, i2) + swap_(self.subnode_scale[l-1].data, i1, i2) + swap_(self.subnode_bias[l-1].data, i1, i2) + + if log_history: + self.log_history('swap') + + @property + def connection_cost(self): + + cc = 0. + for t in self.edge_scores: + + def get_coordinate(n): + return torch.linspace(0,1,steps=n+1)[:n] + 1/(2*n) + + in_dim = t.shape[0] + x_in = get_coordinate(in_dim) + + out_dim = t.shape[1] + x_out = get_coordinate(out_dim) + + dist = torch.abs(x_in[:,None] - x_out[None,:]) + cc += torch.sum(dist * t) + + return cc + + def auto_swap_l(self, l): + + num = self.width_in[1] + for i in range(num): + ccs = [] + for j in range(num): + self.swap(l,i,j,log_history=False) + self.get_act() + self.attribute() + cc = self.connection_cost.detach().clone() + ccs.append(cc) + self.swap(l,i,j,log_history=False) + j = torch.argmin(torch.tensor(ccs)) + self.swap(l,i,j,log_history=False) + + def auto_swap(self): + depth = self.depth + for l in range(1, depth): + self.auto_swap_l(l) + + self.log_history('auto_swap') + +KAN = MultKAN diff --git a/kan/.ipynb_checkpoints/__init__-checkpoint.py b/kan/.ipynb_checkpoints/__init__-checkpoint.py new file mode 100644 index 00000000..254a757b --- /dev/null +++ b/kan/.ipynb_checkpoints/__init__-checkpoint.py @@ -0,0 +1,3 @@ +from .MultKAN import * +from .utils import * +torch.use_deterministic_algorithms(True) \ No newline at end of file diff --git a/kan/.ipynb_checkpoints/spline-checkpoint.py b/kan/.ipynb_checkpoints/spline-checkpoint.py new file mode 100644 index 00000000..6a14510c --- /dev/null +++ b/kan/.ipynb_checkpoints/spline-checkpoint.py @@ -0,0 +1,182 @@ +import torch + + +def B_batch(x, grid, k=0, extend=True, device='cpu'): + ''' + evaludate x on B-spline bases + + Args: + ----- + x : 2D torch.tensor + inputs, shape (number of splines, number of samples) + grid : 2D torch.tensor + grids, shape (number of splines, number of grid points) + k : int + the piecewise polynomial order of splines. + extend : bool + If True, k points are extended on both ends. If False, no extension (zero boundary condition). Default: True + device : str + devicde + + Returns: + -------- + spline values : 3D torch.tensor + shape (number of splines, number of B-spline bases (coeffcients), number of samples). The numbef of B-spline bases = number of grid points + k - 1. + + Example + ------- + >>> num_spline = 5 + >>> num_sample = 100 + >>> num_grid_interval = 10 + >>> k = 3 + >>> x = torch.normal(0,1,size=(num_spline, num_sample)) + >>> grids = torch.einsum('i,j->ij', torch.ones(num_spline,), torch.linspace(-1,1,steps=num_grid_interval+1)) + >>> B_batch(x, grids, k=k).shape + torch.Size([5, 13, 100]) + ''' + + '''# x shape: (size, x); grid shape: (size, grid) + def extend_grid(grid, k_extend=0): + # pad k to left and right + # grid shape: (batch, grid) + h = (grid[:, [-1]] - grid[:, [0]]) / (grid.shape[1] - 1) + + for i in range(k_extend): + grid = torch.cat([grid[:, [0]] - h, grid], dim=1) + grid = torch.cat([grid, grid[:, [-1]] + h], dim=1) + grid = grid.to(device) + return grid + + if extend == True: + grid = extend_grid(grid, k_extend=k) + + grid = grid.unsqueeze(dim=2).to(device) + x = x.unsqueeze(dim=1).to(device) + + if k == 0: + value = (x >= grid[:, :-1]) * (x < grid[:, 1:]) + else: + B_km1 = B_batch(x[:, 0], grid=grid[:, :, 0], k=k - 1, extend=False, device=device) + value = (x - grid[:, :-(k + 1)]) / (grid[:, k:-1] - grid[:, :-(k + 1)]) * B_km1[:, :-1] + ( + grid[:, k + 1:] - x) / (grid[:, k + 1:] - grid[:, 1:(-k)]) * B_km1[:, 1:]''' + + x = x.unsqueeze(dim=2) + grid = grid.unsqueeze(dim=0) + + if k == 0: + value = (x >= grid[:, :, :-1]) * (x < grid[:, :, 1:]) + else: + B_km1 = B_batch(x[:,:,0], grid=grid[0], k=k - 1) + + value = (x - grid[:, :, :-(k + 1)]) / (grid[:, :, k:-1] - grid[:, :, :-(k + 1)]) * B_km1[:, :, :-1] + ( + grid[:, :, k + 1:] - x) / (grid[:, :, k + 1:] - grid[:, :, 1:(-k)]) * B_km1[:, :, 1:] + + # in case grid is degenerate + value = torch.nan_to_num(value) + return value + + + +def coef2curve(x_eval, grid, coef, k, device="cpu"): + ''' + converting B-spline coefficients to B-spline curves. Evaluate x on B-spline curves (summing up B_batch results over B-spline basis). + + Args: + ----- + x_eval : 2D torch.tensor) + shape (number of splines, number of samples) + grid : 2D torch.tensor) + shape (number of splines, number of grid points) + coef : 2D torch.tensor) + shape (number of splines, number of coef params). number of coef params = number of grid intervals + k + k : int + the piecewise polynomial order of splines. + device : str + devicde + + Returns: + -------- + y_eval : 2D torch.tensor + shape (number of splines, number of samples) + + Example + ------- + >>> num_spline = 5 + >>> num_sample = 100 + >>> num_grid_interval = 10 + >>> k = 3 + >>> x_eval = torch.normal(0,1,size=(num_spline, num_sample)) + >>> grids = torch.einsum('i,j->ij', torch.ones(num_spline,), torch.linspace(-1,1,steps=num_grid_interval+1)) + >>> coef = torch.normal(0,1,size=(num_spline, num_grid_interval+k)) + >>> coef2curve(x_eval, grids, coef, k=k).shape + torch.Size([5, 100]) + ''' + # x_eval: (size, batch), grid: (size, grid), coef: (size, coef) + # coef: (size, coef), B_batch: (size, coef, batch), summer over coef + + b_splines = B_batch(x_eval, grid, k=k) # (batch, in_dim, n_coef) + y_eval = torch.einsum('ijk,jlk->ijl', b_splines, coef.to(b_splines.device)) + + return y_eval + + +def curve2coef(x_eval, y_eval, grid, k): + ''' + converting B-spline curves to B-spline coefficients using least squares. + + Args: + ----- + x_eval : 2D torch.tensor + shape (number of splines, number of samples) + y_eval : 2D torch.tensor + shape (number of splines, number of samples) + grid : 2D torch.tensor + shape (number of splines, number of grid points) + k : int + the piecewise polynomial order of splines. + device : str + devicde + + Example + ------- + >>> num_spline = 5 + >>> num_sample = 100 + >>> num_grid_interval = 10 + >>> k = 3 + >>> x_eval = torch.normal(0,1,size=(num_spline, num_sample)) + >>> y_eval = torch.normal(0,1,size=(num_spline, num_sample)) + >>> grids = torch.einsum('i,j->ij', torch.ones(num_spline,), torch.linspace(-1,1,steps=num_grid_interval+1)) + torch.Size([5, 13]) + ''' + ''' + # x_eval: (size, batch); y_eval: (size, batch); grid: (size, grid); k: scalar + mat = B_batch(x_eval, grid, k, device=device).permute(0, 2, 1) + # coef = torch.linalg.lstsq(mat, y_eval.unsqueeze(dim=2)).solution[:, :, 0] + coef = torch.linalg.lstsq(mat.to(device), y_eval.unsqueeze(dim=2).to(device), + driver='gelsy' if device == 'cpu' else 'gels').solution[:, :, 0]''' + batch = x_eval.shape[0] + in_dim = x_eval.shape[1] + out_dim = y_eval.shape[2] + n_coef = grid.shape[1] - k - 1 + #mat = B_batch(x_eval, grid, k, device=device).permute(0, 2, 1) + mat = B_batch(x_eval, grid, k) # (batch, in_dim, G+k) + mat = mat.permute(1,0,2)[:,None,:,:].expand(in_dim, out_dim, batch, n_coef) # (in_dim, out_dim, batch, n_coef) + # coef shape: (in_dim, outdim, G+k) + y_eval = y_eval.permute(1,2,0).unsqueeze(dim=3) # y_eval: (in_dim, out_dim, batch, 1) + #print(mat) + device = mat.device + coef = torch.linalg.lstsq(mat, y_eval, + driver='gelsy' if device == 'cpu' else 'gels').solution[:,:,:,0] + return coef + + +def extend_grid(grid, k_extend=0): + # pad k to left and right + # grid shape: (batch, grid) + h = (grid[:, [-1]] - grid[:, [0]]) / (grid.shape[1] - 1) + + for i in range(k_extend): + grid = torch.cat([grid[:, [0]] - h, grid], dim=1) + grid = torch.cat([grid, grid[:, [-1]] + h], dim=1) + + return grid \ No newline at end of file diff --git a/kan/.ipynb_checkpoints/utils-checkpoint.py b/kan/.ipynb_checkpoints/utils-checkpoint.py new file mode 100644 index 00000000..8d8bd9c4 --- /dev/null +++ b/kan/.ipynb_checkpoints/utils-checkpoint.py @@ -0,0 +1,350 @@ +import numpy as np +import torch +from sklearn.linear_model import LinearRegression +import sympy +import yaml +from sympy.utilities.lambdify import lambdify + +# sigmoid = sympy.Function('sigmoid') +# name: (torch implementation, sympy implementation) + +# singularity protection functions +f_inv = lambda x, y_th: ((x_th := 1/y_th), y_th/x_th*x * (torch.abs(x) < x_th) + torch.nan_to_num(1/x) * (torch.abs(x) >= x_th)) +f_inv2 = lambda x, y_th: ((x_th := 1/y_th**(1/2)), y_th * (torch.abs(x) < x_th) + torch.nan_to_num(1/x**2) * (torch.abs(x) >= x_th)) +f_inv3 = lambda x, y_th: ((x_th := 1/y_th**(1/3)), y_th/x_th*x * (torch.abs(x) < x_th) + torch.nan_to_num(1/x**3) * (torch.abs(x) >= x_th)) +f_inv4 = lambda x, y_th: ((x_th := 1/y_th**(1/4)), y_th * (torch.abs(x) < x_th) + torch.nan_to_num(1/x**4) * (torch.abs(x) >= x_th)) +f_inv5 = lambda x, y_th: ((x_th := 1/y_th**(1/5)), y_th/x_th*x * (torch.abs(x) < x_th) + torch.nan_to_num(1/x**5) * (torch.abs(x) >= x_th)) +f_sqrt = lambda x, y_th: ((x_th := 1/y_th**2), x_th/y_th*x * (torch.abs(x) < x_th) + torch.nan_to_num(torch.sqrt(torch.abs(x))*torch.sign(x)) * (torch.abs(x) >= x_th)) +f_power1d5 = lambda x, y_th: torch.abs(x)**1.5 +f_invsqrt = lambda x, y_th: ((x_th := 1/y_th**2), y_th * (torch.abs(x) < x_th) + torch.nan_to_num(1/torch.sqrt(torch.abs(x))) * (torch.abs(x) >= x_th)) +f_log = lambda x, y_th: ((x_th := torch.e**(-y_th)), - y_th * (torch.abs(x) < x_th) + torch.nan_to_num(torch.log(torch.abs(x))) * (torch.abs(x) >= x_th)) +f_tan = lambda x, y_th: ((clip := x % torch.pi), (delta := torch.pi/2-torch.arctan(y_th)), - y_th/delta * (clip - torch.pi/2) * (torch.abs(clip - torch.pi/2) < delta) + torch.nan_to_num(torch.tan(clip)) * (torch.abs(clip - torch.pi/2) >= delta)) +f_arctanh = lambda x, y_th: ((delta := 1-torch.tanh(y_th) + 1e-4), y_th * torch.sign(x) * (torch.abs(x) > 1 - delta) + torch.nan_to_num(torch.arctanh(x)) * (torch.abs(x) <= 1 - delta)) +f_arcsin = lambda x, y_th: ((), torch.pi/2 * torch.sign(x) * (torch.abs(x) > 1) + torch.nan_to_num(torch.arcsin(x)) * (torch.abs(x) <= 1)) +f_arccos = lambda x, y_th: ((), torch.pi/2 * (1-torch.sign(x)) * (torch.abs(x) > 1) + torch.nan_to_num(torch.arccos(x)) * (torch.abs(x) <= 1)) +f_exp = lambda x, y_th: ((x_th := torch.log(y_th)), y_th * (x > x_th) + torch.exp(x) * (x <= x_th)) + +SYMBOLIC_LIB = {'x': (lambda x: x, lambda x: x, 1, lambda x, y_th: ((), x)), + 'x^2': (lambda x: x**2, lambda x: x**2, 2, lambda x, y_th: ((), x**2)), + 'x^3': (lambda x: x**3, lambda x: x**3, 3, lambda x, y_th: ((), x**3)), + 'x^4': (lambda x: x**4, lambda x: x**4, 3, lambda x, y_th: ((), x**4)), + 'x^5': (lambda x: x**5, lambda x: x**5, 3, lambda x, y_th: ((), x**5)), + '1/x': (lambda x: 1/x, lambda x: 1/x, 2, f_inv), + '1/x^2': (lambda x: 1/x**2, lambda x: 1/x**2, 2, f_inv2), + '1/x^3': (lambda x: 1/x**3, lambda x: 1/x**3, 3, f_inv3), + '1/x^4': (lambda x: 1/x**4, lambda x: 1/x**4, 4, f_inv4), + '1/x^5': (lambda x: 1/x**5, lambda x: 1/x**5, 5, f_inv5), + 'sqrt': (lambda x: torch.sqrt(x), lambda x: sympy.sqrt(x), 2, f_sqrt), + 'x^0.5': (lambda x: torch.sqrt(x), lambda x: sympy.sqrt(x), 2, f_sqrt), + 'x^1.5': (lambda x: torch.sqrt(x)**3, lambda x: sympy.sqrt(x)**3, 4, f_power1d5), + '1/sqrt(x)': (lambda x: 1/torch.sqrt(x), lambda x: 1/sympy.sqrt(x), 2, f_invsqrt), + '1/x^0.5': (lambda x: 1/torch.sqrt(x), lambda x: 1/sympy.sqrt(x), 2, f_invsqrt), + 'exp': (lambda x: torch.exp(x), lambda x: sympy.exp(x), 2, f_exp), + 'log': (lambda x: torch.log(x), lambda x: sympy.log(x), 2, f_log), + 'abs': (lambda x: torch.abs(x), lambda x: sympy.Abs(x), 3, lambda x, y_th: ((), torch.abs(x))), + 'sin': (lambda x: torch.sin(x), lambda x: sympy.sin(x), 2, lambda x, y_th: ((), torch.sin(x))), + 'cos': (lambda x: torch.cos(x), lambda x: sympy.cos(x), 2, lambda x, y_th: ((), torch.cos(x))), + 'tan': (lambda x: torch.tan(x), lambda x: sympy.tan(x), 3, f_tan), + 'tanh': (lambda x: torch.tanh(x), lambda x: sympy.tanh(x), 3, lambda x, y_th: ((), torch.tanh(x))), + 'sgn': (lambda x: torch.sign(x), lambda x: sympy.sign(x), 3, lambda x, y_th: ((), torch.sign(x))), + 'arcsin': (lambda x: torch.arcsin(x), lambda x: sympy.asin(x), 4, f_arcsin), + 'arccos': (lambda x: torch.arccos(x), lambda x: sympy.acos(x), 4, f_arccos), + 'arctan': (lambda x: torch.arctan(x), lambda x: sympy.atan(x), 4, lambda x, y_th: ((), torch.arctan(x))), + 'arctanh': (lambda x: torch.arctanh(x), lambda x: sympy.atanh(x), 4, f_arctanh), + '0': (lambda x: x*0, lambda x: x*0, 0, lambda x, y_th: ((), x*0)), + 'gaussian': (lambda x: torch.exp(-x**2), lambda x: sympy.exp(-x**2), 3, lambda x, y_th: ((), torch.exp(-x**2))), + #'cosh': (lambda x: torch.cosh(x), lambda x: sympy.cosh(x), 5), + #'sigmoid': (lambda x: torch.sigmoid(x), sympy.Function('sigmoid'), 4), + #'relu': (lambda x: torch.relu(x), relu), +} + +def create_dataset(f, + n_var=2, + f_mode = 'col', + ranges = [-1,1], + train_num=1000, + test_num=1000, + normalize_input=False, + normalize_label=False, + device='cpu', + seed=0): + ''' + create dataset + + Args: + ----- + f : function + the symbolic formula used to create the synthetic dataset + ranges : list or np.array; shape (2,) or (n_var, 2) + the range of input variables. Default: [-1,1]. + train_num : int + the number of training samples. Default: 1000. + test_num : int + the number of test samples. Default: 1000. + normalize_input : bool + If True, apply normalization to inputs. Default: False. + normalize_label : bool + If True, apply normalization to labels. Default: False. + device : str + device. Default: 'cpu'. + seed : int + random seed. Default: 0. + + Returns: + -------- + dataset : dic + Train/test inputs/labels are dataset['train_input'], dataset['train_label'], + dataset['test_input'], dataset['test_label'] + + Example + ------- + >>> f = lambda x: torch.exp(torch.sin(torch.pi*x[:,[0]]) + x[:,[1]]**2) + >>> dataset = create_dataset(f, n_var=2, train_num=100) + >>> dataset['train_input'].shape + torch.Size([100, 2]) + ''' + + np.random.seed(seed) + torch.manual_seed(seed) + + if len(np.array(ranges).shape) == 1: + ranges = np.array(ranges * n_var).reshape(n_var,2) + else: + ranges = np.array(ranges) + + + train_input = torch.zeros(train_num, n_var) + test_input = torch.zeros(test_num, n_var) + for i in range(n_var): + train_input[:,i] = torch.rand(train_num,)*(ranges[i,1]-ranges[i,0])+ranges[i,0] + test_input[:,i] = torch.rand(test_num,)*(ranges[i,1]-ranges[i,0])+ranges[i,0] + + if f_mode == 'col': + train_label = f(train_input) + test_label = f(test_input) + elif f_mode == 'row': + train_label = f(train_input.T) + test_label = f(test_input.T) + else: + print(f'f_mode {f_mode} not recognized') + + # if has only 1 dimension + if len(train_label.shape) == 1: + train_label = train_label.unsqueeze(dim=1) + test_label = test_label.unsqueeze(dim=1) + + def normalize(data, mean, std): + return (data-mean)/std + + if normalize_input == True: + mean_input = torch.mean(train_input, dim=0, keepdim=True) + std_input = torch.std(train_input, dim=0, keepdim=True) + train_input = normalize(train_input, mean_input, std_input) + test_input = normalize(test_input, mean_input, std_input) + + if normalize_label == True: + mean_label = torch.mean(train_label, dim=0, keepdim=True) + std_label = torch.std(train_label, dim=0, keepdim=True) + train_label = normalize(train_label, mean_label, std_label) + test_label = normalize(test_label, mean_label, std_label) + + dataset = {} + dataset['train_input'] = train_input.to(device) + dataset['test_input'] = test_input.to(device) + + dataset['train_label'] = train_label.to(device) + dataset['test_label'] = test_label.to(device) + + return dataset + + + +def fit_params(x, y, fun, a_range=(-10,10), b_range=(-10,10), grid_number=101, iteration=3, verbose=True, device='cpu'): + ''' + fit a, b, c, d such that + + .. math:: + |y-(cf(ax+b)+d)|^2 + + is minimized. Both x and y are 1D array. Sweep a and b, find the best fitted model. + + Args: + ----- + x : 1D array + x values + y : 1D array + y values + fun : function + symbolic function + a_range : tuple + sweeping range of a + b_range : tuple + sweeping range of b + grid_num : int + number of steps along a and b + iteration : int + number of zooming in + verbose : bool + print extra information if True + device : str + device + + Returns: + -------- + a_best : float + best fitted a + b_best : float + best fitted b + c_best : float + best fitted c + d_best : float + best fitted d + r2_best : float + best r2 (coefficient of determination) + + Example + ------- + >>> num = 100 + >>> x = torch.linspace(-1,1,steps=num) + >>> noises = torch.normal(0,1,(num,)) * 0.02 + >>> y = 5.0*torch.sin(3.0*x + 2.0) + 0.7 + noises + >>> fit_params(x, y, torch.sin) + r2 is 0.9999727010726929 + (tensor([2.9982, 1.9996, 5.0053, 0.7011]), tensor(1.0000)) + ''' + # fit a, b, c, d such that y=c*fun(a*x+b)+d; both x and y are 1D array. + # sweep a and b, choose the best fitted model + for _ in range(iteration): + a_ = torch.linspace(a_range[0], a_range[1], steps=grid_number, device=device) + b_ = torch.linspace(b_range[0], b_range[1], steps=grid_number, device=device) + a_grid, b_grid = torch.meshgrid(a_, b_, indexing='ij') + post_fun = fun(a_grid[None,:,:] * x[:,None,None] + b_grid[None,:,:]) + x_mean = torch.mean(post_fun, dim=[0], keepdim=True) + y_mean = torch.mean(y, dim=[0], keepdim=True) + numerator = torch.sum((post_fun - x_mean)*(y-y_mean)[:,None,None], dim=0)**2 + denominator = torch.sum((post_fun - x_mean)**2, dim=0)*torch.sum((y - y_mean)[:,None,None]**2, dim=0) + r2 = numerator/(denominator+1e-4) + r2 = torch.nan_to_num(r2) + + + best_id = torch.argmax(r2) + a_id, b_id = torch.div(best_id, grid_number, rounding_mode='floor'), best_id % grid_number + + + if a_id == 0 or a_id == grid_number - 1 or b_id == 0 or b_id == grid_number - 1: + if _ == 0 and verbose==True: + print('Best value at boundary.') + if a_id == 0: + a_range = [a_[0], a_[1]] + if a_id == grid_number - 1: + a_range = [a_[-2], a_[-1]] + if b_id == 0: + b_range = [b_[0], b_[1]] + if b_id == grid_number - 1: + b_range = [b_[-2], b_[-1]] + + else: + a_range = [a_[a_id-1], a_[a_id+1]] + b_range = [b_[b_id-1], b_[b_id+1]] + + a_best = a_[a_id] + b_best = b_[b_id] + post_fun = fun(a_best * x + b_best) + r2_best = r2[a_id, b_id] + + if verbose == True: + print(f"r2 is {r2_best}") + if r2_best < 0.9: + print(f'r2 is not very high, please double check if you are choosing the correct symbolic function.') + + post_fun = torch.nan_to_num(post_fun) + reg = LinearRegression().fit(post_fun[:,None].detach().cpu().numpy(), y.detach().cpu().numpy()) + c_best = torch.from_numpy(reg.coef_)[0].to(device) + d_best = torch.from_numpy(np.array(reg.intercept_)).to(device) + return torch.stack([a_best, b_best, c_best, d_best]), r2_best + + +def sparse_mask(in_dim, out_dim): + + in_coord = torch.arange(in_dim) * 1/in_dim + 1/(2*in_dim) + out_coord = torch.arange(out_dim) * 1/out_dim + 1/(2*out_dim) + + dist_mat = torch.abs(out_coord[:,None] - in_coord[None,:]) + in_nearest = torch.argmin(dist_mat, dim=0) + in_connection = torch.stack([torch.arange(in_dim), in_nearest]).permute(1,0) + out_nearest = torch.argmin(dist_mat, dim=1) + out_connection = torch.stack([out_nearest, torch.arange(out_dim)]).permute(1,0) + all_connection = torch.cat([in_connection, out_connection], dim=0) + mask = torch.zeros(in_dim, out_dim) + mask[all_connection[:,0], all_connection[:,1]] = 1. + + return mask + + +def add_symbolic(name, fun, c=1, fun_singularity=None): + ''' + add a symbolic function to library + + Args: + ----- + name : str + name of the function + fun : fun + torch function or lambda function + + Returns: + -------- + None + + Example + ------- + >>> print(SYMBOLIC_LIB['Bessel']) + KeyError: 'Bessel' + >>> add_symbolic('Bessel', torch.special.bessel_j0) + >>> print(SYMBOLIC_LIB['Bessel']) + (, Bessel) + ''' + exec(f"globals()['{name}'] = sympy.Function('{name}')") + if fun_singularity==None: + fun_singularity = fun + SYMBOLIC_LIB[name] = (fun, globals()[name], c, fun_singularity) + + +def ex_round(ex1, n_digit): + ex2 = ex1 + for a in sympy.preorder_traversal(ex1): + if isinstance(a, sympy.Float): + ex2 = ex2.subs(a, round(a, n_digit)) + return ex2 + + +def augment_input(orig_vars, aux_vars, x): + + # if x is a tensor + if isinstance(x, torch.Tensor): + + for aux_var in aux_vars: + func = lambdify(orig_vars, aux_var,'numpy') # returns a numpy-ready function + aux_value = torch.from_numpy(func(*[x[:,[i]].numpy() for i in range(len(orig_vars))])) + x = torch.cat([x, aux_value], dim=1) + + # if x is a dataset + elif isinstance(x, dict): + x['train_input'] = augment_input(orig_vars, aux_vars, x['train_input']) + x['test_input'] = augment_input(orig_vars, aux_vars, x['test_input']) + + return x + + +def batch_jacobian(func, x, create_graph=False): + # x in shape (Batch, Length) + def _func_sum(x): + return func(x).sum(dim=0) + return torch.autograd.functional.jacobian(_func_sum, x, create_graph=create_graph)[0] + +def batch_hessian(model, x, create_graph=False): + # x in shape (Batch, Length) + jac = lambda x: batch_jacobian(model, x, create_graph=True) + def _jac_sum(x): + return jac(x).sum(dim=0) + return torch.autograd.functional.jacobian(_jac_sum, x, create_graph=create_graph).permute(1,0,2) \ No newline at end of file diff --git a/kan/KANLayer.py b/kan/KANLayer.py index a07378de..60c7b0eb 100644 --- a/kan/KANLayer.py +++ b/kan/KANLayer.py @@ -120,29 +120,28 @@ def __init__(self, in_dim=3, out_dim=2, num=5, k=3, noise_scale=0.1, scale_base= grid = extend_grid(grid, k_extend=k) self.grid = torch.nn.Parameter(grid).requires_grad_(False) noises = (torch.rand(self.num+1, self.in_dim, self.out_dim) - 1 / 2) * noise_scale / num - noises = noises.to(device) # shape: (size, coef) - self.coef = torch.nn.Parameter(curve2coef(self.grid[:,k:-k].permute(1,0), noises, self.grid, k, device)) + self.coef = torch.nn.Parameter(curve2coef(self.grid[:,k:-k].permute(1,0), noises, self.grid, k)) #if isinstance(scale_base, float): if sparse_init: mask = sparse_mask(in_dim, out_dim) else: mask = 1. - self.scale_base = torch.nn.Parameter(torch.ones(in_dim, out_dim, device=device) * scale_base * mask).requires_grad_(sb_trainable) # make scale trainable + self.scale_base = torch.nn.Parameter(torch.ones(in_dim, out_dim) * scale_base * mask).requires_grad_(sb_trainable) # make scale trainable #else: #self.scale_base = torch.nn.Parameter(scale_base.to(device)).requires_grad_(sb_trainable) - self.scale_sp = torch.nn.Parameter(torch.ones(in_dim, out_dim, device=device) * scale_sp * mask).requires_grad_(sp_trainable) # make scale trainable + self.scale_sp = torch.nn.Parameter(torch.ones(in_dim, out_dim) * scale_sp * mask).requires_grad_(sp_trainable) # make scale trainable self.base_fun = base_fun - self.mask = torch.nn.Parameter(torch.ones(in_dim, out_dim, device=device)).requires_grad_(False) + self.mask = torch.nn.Parameter(torch.ones(in_dim, out_dim)).requires_grad_(False) self.grid_eps = grid_eps ### remove weight_sharing & lock parts #self.weight_sharing = torch.arange(out_dim*in_dim).reshape(out_dim, in_dim) #self.lock_counter = 0 #self.lock_id = torch.zeros(out_dim*in_dim).reshape(out_dim, in_dim) - self.device = device + def forward(self, x): ''' @@ -181,7 +180,7 @@ def forward(self, x): preacts = x[:,None,:].clone().expand(batch, self.out_dim, self.in_dim) base = self.base_fun(x) # (batch, in_dim) - y = coef2curve(x_eval=x, grid=self.grid, coef=self.coef, k=self.k, device=self.device) # y shape: (batch, in_dim, out_dim) + y = coef2curve(x_eval=x, grid=self.grid, coef=self.coef, k=self.k) # y shape: (batch, in_dim, out_dim) postspline = y.clone().permute(0,2,1) # postspline shape: (batch, out_dim, in_dim) @@ -219,16 +218,16 @@ def update_grid_from_samples(self, x): batch = x.shape[0] #x = torch.einsum('ij,k->ikj', x, torch.ones(self.out_dim, ).to(self.device)).reshape(batch, self.size).permute(1, 0) x_pos = torch.sort(x, dim=0)[0] - y_eval = coef2curve(x_pos, self.grid, self.coef, self.k, device=self.device) + y_eval = coef2curve(x_pos, self.grid, self.coef, self.k) num_interval = self.grid.shape[1] - 1 - 2*self.k ids = [int(batch / num_interval * i) for i in range(num_interval)] + [-1] grid_adaptive = x_pos[ids, :].permute(1,0) margin = 0.01 h = (grid_adaptive[:,[-1]] - grid_adaptive[:,[0]])/num_interval - grid_uniform = grid_adaptive[:,[0]] + h * torch.arange(num_interval+1,).to(self.device)[None, :] + grid_uniform = grid_adaptive[:,[0]] + h * torch.arange(num_interval+1,)[None, :].to(x.device) grid = self.grid_eps * grid_uniform + (1 - self.grid_eps) * grid_adaptive self.grid.data = extend_grid(grid, k_extend=self.k) - self.coef.data = curve2coef(x_pos, y_eval, self.grid, self.k, device=self.device) + self.coef.data = curve2coef(x_pos, y_eval, self.grid, self.k) def initialize_grid_from_parent(self, parent, x): ''' @@ -264,20 +263,13 @@ def initialize_grid_from_parent(self, parent, x): x_eval = x pgrid = parent.grid # (in_dim, G+2*k+1) pk = parent.k - y_eval = coef2curve(x_eval, pgrid, parent.coef, pk, device=self.device) - '''print(x_pos.shape) - sp2 = KANLayer(in_dim=1, out_dim=self.in_dim, k=1, num=x_pos.shape[1] - 2*self.k - 1, scale_base=0., device=self.device) + y_eval = coef2curve(x_eval, pgrid, parent.coef, pk) - print(sp2.grid[:,sp2.k:-sp2.k].shape, x_pos[:,self.k:-self.k].shape, sp2.grid.shape) - sp2.coef.data = curve2coef(sp2.grid[:,sp2.k:-sp2.k], x_pos[:,self.k:-self.k], sp2.grid, k=1, device=self.device) - y_eval = coef2curve(x_eval, parent.grid, parent.coef, parent.k, device=self.device) - percentile = torch.linspace(-1, 1, self.num + 1).to(self.device) - self.grid.data = sp2(percentile.unsqueeze(dim=1))[0].permute(1, 0)''' h = (pgrid[:,[-pk]] - pgrid[:,[pk]])/self.num grid = pgrid[:,[pk]] + torch.arange(self.num+1,) * h grid = extend_grid(grid, k_extend=self.k) self.grid.data = grid - self.coef.data = curve2coef(x_eval, y_eval, self.grid, self.k, self.device) + self.coef.data = curve2coef(x_eval, y_eval, self.grid, self.k) def get_subset(self, in_id, out_id): ''' @@ -301,7 +293,7 @@ def get_subset(self, in_id, out_id): >>> kanlayer_small.in_dim, kanlayer_small.out_dim (2, 3) ''' - spb = KANLayer(len(in_id), len(out_id), self.num, self.k, base_fun=self.base_fun, device=self.device) + spb = KANLayer(len(in_id), len(out_id), self.num, self.k, base_fun=self.base_fun) spb.grid.data = self.grid[in_id] spb.coef.data = self.coef[in_id][:,out_id] spb.scale_base.data = self.scale_base[in_id][:,out_id] diff --git a/kan/LBFGS.py b/kan/LBFGS.py index a699b8c2..212477f2 100644 --- a/kan/LBFGS.py +++ b/kan/LBFGS.py @@ -267,7 +267,8 @@ def _gather_flat_grad(self): else: view = p.grad.view(-1) views.append(view) - return torch.cat(views, 0) + device = views[0].device + return torch.cat(views, dim=0) def _add_grad(self, step_size, update): offset = 0 diff --git a/kan/MultKAN.py b/kan/MultKAN.py index 044c264f..39dd62b5 100644 --- a/kan/MultKAN.py +++ b/kan/MultKAN.py @@ -24,7 +24,7 @@ class MultKAN(nn.Module): # include mult_ops = [] - def __init__(self, width=None, grid=3, k=3, mult_arity = 2, noise_scale=1.0, scale_base_mu=0.0, scale_base_sigma=1.0, base_fun='silu', symbolic_enabled=True, affine_trainable=False, grid_eps=1.0, grid_range=[-1, 1], sp_trainable=True, sb_trainable=True, device='cpu', seed=1, save_act=True, sparse_init=False, auto_save=True, first_init=True, ckpt_path='./model', state_id=0, round=0): + def __init__(self, width=None, grid=3, k=3, mult_arity = 2, noise_scale=1.0, scale_base_mu=0.0, scale_base_sigma=1.0, base_fun='silu', symbolic_enabled=True, affine_trainable=False, grid_eps=1.0, grid_range=[-1, 1], sp_trainable=True, sb_trainable=True, seed=1, save_act=True, sparse_init=False, auto_save=True, first_init=True, ckpt_path='./model', state_id=0, round=0): super(MultKAN, self).__init__() @@ -69,7 +69,7 @@ def __init__(self, width=None, grid=3, k=3, mult_arity = 2, noise_scale=1.0, sca # splines scale_base = scale_base_mu * 1 / np.sqrt(width_in[l]) + \ scale_base_sigma * (torch.randn(width_in[l], width_out[l + 1]) * 2 - 1) * 1/np.sqrt(width_in[l]) - sp_batch = KANLayer(in_dim=width_in[l], out_dim=width_out[l+1], num=grid, k=k, noise_scale=noise_scale, scale_base=scale_base, scale_sp=1., base_fun=base_fun, grid_eps=grid_eps, grid_range=grid_range, sp_trainable=sp_trainable, sb_trainable=sb_trainable, device=device, sparse_init=sparse_init) + sp_batch = KANLayer(in_dim=width_in[l], out_dim=width_out[l+1], num=grid, k=k, noise_scale=noise_scale, scale_base=scale_base, scale_sp=1., base_fun=base_fun, grid_eps=grid_eps, grid_range=grid_range, sp_trainable=sp_trainable, sb_trainable=sb_trainable, sparse_init=sparse_init) self.act_fun.append(sp_batch) self.node_bias = [] @@ -82,10 +82,10 @@ def __init__(self, width=None, grid=3, k=3, mult_arity = 2, noise_scale=1.0, sca exec('self.node_bias_0' + " = torch.nn.Parameter(torch.zeros(3,1)).requires_grad_(False)") for l in range(self.depth): - exec(f'self.node_bias_{l} = torch.nn.Parameter(torch.zeros(width_in[l+1],)).requires_grad_(affine_trainable).to(device)') - exec(f'self.node_scale_{l} = torch.nn.Parameter(torch.ones(width_in[l+1],)).requires_grad_(affine_trainable).to(device)') - exec(f'self.subnode_bias_{l} = torch.nn.Parameter(torch.zeros(width_out[l+1],)).requires_grad_(affine_trainable).to(device)') - exec(f'self.subnode_scale_{l} = torch.nn.Parameter(torch.ones(width_out[l+1],)).requires_grad_(affine_trainable).to(device)') + exec(f'self.node_bias_{l} = torch.nn.Parameter(torch.zeros(width_in[l+1],)).requires_grad_(affine_trainable)') + exec(f'self.node_scale_{l} = torch.nn.Parameter(torch.ones(width_in[l+1],)).requires_grad_(affine_trainable)') + exec(f'self.subnode_bias_{l} = torch.nn.Parameter(torch.zeros(width_out[l+1],)).requires_grad_(affine_trainable)') + exec(f'self.subnode_scale_{l} = torch.nn.Parameter(torch.ones(width_out[l+1],)).requires_grad_(affine_trainable)') exec(f'self.node_bias.append(self.node_bias_{l})') exec(f'self.node_scale.append(self.node_scale_{l})') exec(f'self.subnode_bias.append(self.subnode_bias_{l})') @@ -101,7 +101,7 @@ def __init__(self, width=None, grid=3, k=3, mult_arity = 2, noise_scale=1.0, sca ### initializing the symbolic front ### self.symbolic_fun = [] for l in range(self.depth): - sb_batch = Symbolic_KANLayer(in_dim=width_in[l], out_dim=width_out[l+1], device=device) + sb_batch = Symbolic_KANLayer(in_dim=width_in[l], out_dim=width_out[l+1]) self.symbolic_fun.append(sb_batch) self.symbolic_fun = nn.ModuleList(self.symbolic_fun) @@ -110,7 +110,6 @@ def __init__(self, width=None, grid=3, k=3, mult_arity = 2, noise_scale=1.0, sca self.sp_trainable = sp_trainable self.sb_trainable = sb_trainable - self.device = device self.save_act = save_act self.node_scores = None @@ -125,7 +124,6 @@ def __init__(self, width=None, grid=3, k=3, mult_arity = 2, noise_scale=1.0, sca self.ckpt_path = ckpt_path self.round = round - if auto_save: if first_init: if not os.path.exists(ckpt_path): @@ -145,10 +143,10 @@ def __init__(self, width=None, grid=3, k=3, mult_arity = 2, noise_scale=1.0, sca self.input_id = torch.arange(self.width_in[0],) def initialize_from_another_model(self, another_model, x): - another_model(x.to(another_model.device)) # get activations + another_model(x) # get activations batch = x.shape[0] - self.initialize_grid_from_another_model(another_model, x.to(another_model.device)) + self.initialize_grid_from_another_model(another_model, x) for l in range(self.depth): spb = self.act_fun[l] @@ -157,7 +155,7 @@ def initialize_from_another_model(self, another_model, x): # spb = spb_parent preacts = another_model.spline_preacts[l] postsplines = another_model.spline_postsplines[l] - self.act_fun[l].coef.data = curve2coef(preacts[:,0,:], postsplines.permute(0,2,1), spb.grid, k=spb.k, device=self.device) + self.act_fun[l].coef.data = curve2coef(preacts[:,0,:], postsplines.permute(0,2,1), spb.grid, k=spb.k) self.act_fun[l].scale_base.data = another_model.act_fun[l].scale_base.data self.act_fun[l].scale_sp.data = another_model.act_fun[l].scale_sp.data self.act_fun[l].mask.data = another_model.act_fun[l].mask.data @@ -172,7 +170,7 @@ def initialize_from_another_model(self, another_model, x): for l in range(self.depth): self.symbolic_fun[l] = another_model.symbolic_fun[l] - return self + return self.to(device) def log_history(self, method_name): @@ -204,7 +202,6 @@ def refine(self, new_grid): grid_range=self.grid_range, sp_trainable=self.sp_trainable, sb_trainable=self.sb_trainable, - device=self.device, ckpt_path=self.ckpt_path, auto_save=True, first_init=False, @@ -237,7 +234,6 @@ def saveckpt(self, path='model'): grid_range = model.grid_range, sp_trainable = model.sp_trainable, sb_trainable = model.sb_trainable, - device = model.device, state_id = model.state_id, auto_save = model.auto_save, ckpt_path = model.ckpt_path, @@ -271,7 +267,6 @@ def loadckpt(path='model'): grid_range=config['grid_range'], sp_trainable=config['sp_trainable'], sb_trainable=config['sb_trainable'], - device=config['device'], state_id=config['state_id'], auto_save=config['auto_save'], first_init=False, @@ -859,7 +854,7 @@ def get_params(self): def fit(self, dataset, opt="LBFGS", steps=100, log=1, lamb=0., lamb_l1=1., lamb_entropy=2., lamb_coef=0., lamb_coefdiff=0., update_grid=True, grid_update_num=10, loss_fn=None, lr=1.,start_grid_update_step=-1, stop_grid_update_step=50, batch=-1, - metrics=None, save_fig=False, in_vars=None, out_vars=None, beta=3, save_fig_freq=1, img_folder='./video', device='cpu', singularity_avoiding=False, y_th=1000., reg_metric='edge_backward', display_metrics=None): + metrics=None, save_fig=False, in_vars=None, out_vars=None, beta=3, save_fig_freq=1, img_folder='./video', singularity_avoiding=False, y_th=1000., reg_metric='edge_backward', display_metrics=None): if lamb > 0. and not self.save_act: print('setting lamb=0. If you want to set lamb > 0, set self.save_act=True') @@ -900,8 +895,8 @@ def fit(self, dataset, opt="LBFGS", steps=100, log=1, lamb=0., lamb_l1=1., lamb_ def closure(): global train_loss, reg_ optimizer.zero_grad() - pred = self.forward(dataset['train_input'][train_id].to(self.device), singularity_avoiding=singularity_avoiding, y_th=y_th) - train_loss = loss_fn(pred, dataset['train_label'][train_id].to(self.device)) + pred = self.forward(dataset['train_input'][train_id], singularity_avoiding=singularity_avoiding, y_th=y_th) + train_loss = loss_fn(pred, dataset['train_label'][train_id]) if self.save_act: if reg_metric == 'edge_backward': self.attribute() @@ -927,17 +922,19 @@ def closure(): test_id = np.random.choice(dataset['test_input'].shape[0], batch_size_test, replace=False) if _ % grid_update_freq == 0 and _ < stop_grid_update_step and update_grid and _ >= start_grid_update_step: - self.update_grid(dataset['train_input'][train_id].to(device)) + self.update_grid(dataset['train_input'][train_id]) if opt == "LBFGS": optimizer.step(closure) if opt == "Adam": - pred = self.forward(dataset['train_input'][train_id].to(self.device), singularity_avoiding=singularity_avoiding, y_th=y_th) - train_loss = loss_fn(pred, dataset['train_label'][train_id].to(self.device)) + pred = self.forward(dataset['train_input'][train_id], singularity_avoiding=singularity_avoiding, y_th=y_th) + train_loss = loss_fn(pred, dataset['train_label'][train_id]) if self.save_act: - if reg_metric == 'fa': + if reg_metric == 'edge_backward': self.attribute() + if reg_metric == 'node_backward': + self.node_attribute() reg_ = self.get_reg(reg_metric, lamb_l1, lamb_entropy, lamb_coef, lamb_coefdiff) else: reg_ = torch.tensor(0.) @@ -946,7 +943,7 @@ def closure(): loss.backward() optimizer.step() - test_loss = loss_fn_eval(self.forward(dataset['test_input'][test_id].to(self.device)), dataset['test_label'][test_id].to(self.device)) + test_loss = loss_fn_eval(self.forward(dataset['test_input'][test_id]), dataset['test_label'][test_id]) if metrics != None: @@ -1055,7 +1052,7 @@ def prune_node(self, threshold=1e-2, mode="auto", active_neurons_id=None, log_hi if i not in active_neurons_down[l]: self.remove_node(l + 1, i, mode='down',log_history=False) - model2 = MultKAN(copy.deepcopy(self.width), grid=self.grid, k=self.k, base_fun=self.base_fun_name, device=self.device, mult_arity=self.mult_arity, ckpt_path=self.ckpt_path, auto_save=True, first_init=False, state_id=self.state_id, round=self.round) + model2 = MultKAN(copy.deepcopy(self.width), grid=self.grid, k=self.k, base_fun=self.base_fun_name, mult_arity=self.mult_arity, ckpt_path=self.ckpt_path, auto_save=True, first_init=False, state_id=self.state_id, round=self.round) model2.load_state_dict(self.state_dict()) width_new = [self.width[0]] @@ -1135,7 +1132,7 @@ def prune_input(self, threshold=1e-2, active_inputs=None, log_history=True): else: input_id = torch.tensor(active_inputs, dtype=torch.long) - model2 = MultKAN(copy.deepcopy(self.width), grid=self.grid, k=self.k, base_fun=self.base_fun, device=self.device, mult_arity=self.mult_arity, ckpt_path=self.ckpt_path, auto_save=True, first_init=False, state_id=self.state_id, round=self.round) + model2 = MultKAN(copy.deepcopy(self.width), grid=self.grid, k=self.k, base_fun=self.base_fun, mult_arity=self.mult_arity, ckpt_path=self.ckpt_path, auto_save=True, first_init=False, state_id=self.state_id, round=self.round) model2.load_state_dict(self.state_dict()) model2.act_fun[0] = model2.act_fun[0].get_subset(input_id, torch.arange(self.width_out[1])) @@ -1227,6 +1224,8 @@ def score_node2subnode(node_score, width, mult_arity, out_dim): else: node_score = torch.diag(out_score).requires_grad_(True) node_scores.append(node_score) + + device = self.act_fun[0].grid.device for l in range(l_end,0,-1): @@ -1239,7 +1238,8 @@ def score_node2subnode(node_score, width, mult_arity, out_dim): subnode_scores.append(subnode_score) # subnode to edge - edge_score = torch.einsum('ij,ki,i->kij', self.edge_actscale[l-1], subnode_score, 1/(self.subnode_actscale[l-1]+1e-4)) + #print(self.edge_actscale[l-1].device, subnode_score.device, self.subnode_actscale[l-1].device) + edge_score = torch.einsum('ij,ki,i->kij', self.edge_actscale[l-1], subnode_score.to(device), 1/(self.subnode_actscale[l-1]+1e-4)) edge_scores.append(edge_score) # edge to node @@ -1508,10 +1508,10 @@ def expand_depth(self): self.symbolic_fun.append(layer) - self.node_bias.append(torch.nn.Parameter(torch.zeros(dim_out,)).requires_grad_(self.affine_trainable).to(self.device)) - self.node_scale.append(torch.nn.Parameter(torch.ones(dim_out,)).requires_grad_(self.affine_trainable).to(self.device)) - self.subnode_bias.append(torch.nn.Parameter(torch.zeros(dim_out,)).requires_grad_(self.affine_trainable).to(self.device)) - self.subnode_scale.append(torch.nn.Parameter(torch.ones(dim_out,)).requires_grad_(self.affine_trainable).to(self.device)) + self.node_bias.append(torch.nn.Parameter(torch.zeros(dim_out,)).requires_grad_(self.affine_trainable)) + self.node_scale.append(torch.nn.Parameter(torch.ones(dim_out,)).requires_grad_(self.affine_trainable)) + self.subnode_bias.append(torch.nn.Parameter(torch.zeros(dim_out,)).requires_grad_(self.affine_trainable)) + self.subnode_scale.append(torch.nn.Parameter(torch.ones(dim_out,)).requires_grad_(self.affine_trainable)) def expand_width(self, layer_id, n_added_nodes, sum_bool=True, mult_arity=2): diff --git a/kan/spline.py b/kan/spline.py index 4cfbc01f..6a14510c 100644 --- a/kan/spline.py +++ b/kan/spline.py @@ -60,13 +60,13 @@ def extend_grid(grid, k_extend=0): value = (x - grid[:, :-(k + 1)]) / (grid[:, k:-1] - grid[:, :-(k + 1)]) * B_km1[:, :-1] + ( grid[:, k + 1:] - x) / (grid[:, k + 1:] - grid[:, 1:(-k)]) * B_km1[:, 1:]''' - x = x.unsqueeze(dim=2).to(device) - grid = grid.unsqueeze(dim=0).to(device) + x = x.unsqueeze(dim=2) + grid = grid.unsqueeze(dim=0) if k == 0: value = (x >= grid[:, :, :-1]) * (x < grid[:, :, 1:]) else: - B_km1 = B_batch(x[:,:,0], grid=grid[0], k=k - 1, device=device) + B_km1 = B_batch(x[:,:,0], grid=grid[0], k=k - 1) value = (x - grid[:, :, :-(k + 1)]) / (grid[:, :, k:-1] - grid[:, :, :-(k + 1)]) * B_km1[:, :, :-1] + ( grid[:, :, k + 1:] - x) / (grid[:, :, k + 1:] - grid[:, :, 1:(-k)]) * B_km1[:, :, 1:] @@ -113,18 +113,14 @@ def coef2curve(x_eval, grid, coef, k, device="cpu"): ''' # x_eval: (size, batch), grid: (size, grid), coef: (size, coef) # coef: (size, coef), B_batch: (size, coef, batch), summer over coef - '''if coef.dtype != x_eval.dtype: - coef = coef.to(x_eval.dtype) - y_eval = torch.einsum('ij,ijk->ik', coef, B_batch(x_eval, grid, k, device=device))''' - b_splines = B_batch(x_eval, grid, k=k).to(device) # (batch, in_dim, n_coef) - # coef (in_dim, out_dim, n_coef) - #print(b_splines.shape, coef.shape) - y_eval = torch.einsum('ijk,jlk->ijl', b_splines, coef) + b_splines = B_batch(x_eval, grid, k=k) # (batch, in_dim, n_coef) + y_eval = torch.einsum('ijk,jlk->ijl', b_splines, coef.to(b_splines.device)) + return y_eval -def curve2coef(x_eval, y_eval, grid, k, device="cpu"): +def curve2coef(x_eval, y_eval, grid, k): ''' converting B-spline curves to B-spline coefficients using least squares. @@ -163,14 +159,15 @@ def curve2coef(x_eval, y_eval, grid, k, device="cpu"): out_dim = y_eval.shape[2] n_coef = grid.shape[1] - k - 1 #mat = B_batch(x_eval, grid, k, device=device).permute(0, 2, 1) - mat = B_batch(x_eval, grid, k, device=device) # (batch, in_dim, G+k) + mat = B_batch(x_eval, grid, k) # (batch, in_dim, G+k) mat = mat.permute(1,0,2)[:,None,:,:].expand(in_dim, out_dim, batch, n_coef) # (in_dim, out_dim, batch, n_coef) # coef shape: (in_dim, outdim, G+k) y_eval = y_eval.permute(1,2,0).unsqueeze(dim=3) # y_eval: (in_dim, out_dim, batch, 1) #print(mat) - coef = torch.linalg.lstsq(mat.to(device), y_eval.to(device), + device = mat.device + coef = torch.linalg.lstsq(mat, y_eval, driver='gelsy' if device == 'cpu' else 'gels').solution[:,:,:,0] - return coef.to(device) + return coef def extend_grid(grid, k_extend=0): diff --git a/pykan.egg-info/PKG-INFO b/pykan.egg-info/PKG-INFO index da283df5..e7307237 100644 --- a/pykan.egg-info/PKG-INFO +++ b/pykan.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: pykan -Version: 0.0.1 +Version: 0.2.1 Summary: Kolmogorov Arnold Networks Author: Ziming Liu Author-email: zmliu@mit.edu @@ -13,48 +13,175 @@ License-File: LICENSE kan_plot -# Kolmogorov-Arnold Newtworks (KANs) +# !! Major Updates on July 13, 2024 -This the github repo for the paper "KAN: Kolmogorov-Arnold Networks" [link]. The documentation can be found here [link]. +* `model.train()` has been changed to `model.fit()` +* Some other small features are changed (e.g., create_dataset has been moved to kan.utils). I have updated and checked the notebooks in `./tutorials` are runnable on CPUs, so please refer to those tutorials for updated/new functionalities. Documentation hasn't been updated yet but will be updated soon. -Kolmogorov-Arnold Networks (KANs) are promising alternatives of Multi-Layer Perceptrons (MLPs). KANs have strong mathematical foundations just like MLPs: MLPs are based on the [universal approximation theorem](https://en.wikipedia.org/wiki/Universal_approximation_theorem), while KANs are based on [Kolmogorov-Arnold representation theorem](https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Arnold_representation_theorem). KANs and MLPs are dual: KANs have activation functions on edges, while MLPs have activation functions on nodes. This simple change makes KANs better (sometimes much better!) than MLPs in terms of both model accuracy and interpretability. +For pypi users, this is the most recent version 0.2.1. + +New functionalities include (documentation later): +* including multiplications in KANs. [Tutorial](https://github.com/KindXiaoming/pykan/blob/master/tutorials/Interp_1_Hello%2C%20MultKAN.ipynb) +* the speed mode. Speed up your KAN using `model = model.speed()` if you never use the symbolic functionalities. [Tutorial](https://github.com/KindXiaoming/pykan/blob/master/tutorials/Example_2_speed_up.ipynb) +* Compiling symbolic formulas into KANs. [Tutorial](https://github.com/KindXiaoming/pykan/blob/master/tutorials/Interp_3_KAN_Compiler.ipynb) +* Feature attribution and pruning inputs. [Tutorial](https://github.com/KindXiaoming/pykan/blob/master/tutorials/Interp_4_feature_attribution.ipynb) + +# Kolmogorov-Arnold Networks (KANs) + +This is the github repo for the paper ["KAN: Kolmogorov-Arnold Networks"](https://arxiv.org/abs/2404.19756). Find the documentation [here](https://kindxiaoming.github.io/pykan/). Here's [author's note](https://github.com/KindXiaoming/pykan?tab=readme-ov-file#authors-note) responding to current hype of KANs. + +Kolmogorov-Arnold Networks (KANs) are promising alternatives of Multi-Layer Perceptrons (MLPs). KANs have strong mathematical foundations just like MLPs: MLPs are based on the universal approximation theorem, while KANs are based on Kolmogorov-Arnold representation theorem. KANs and MLPs are dual: KANs have activation functions on edges, while MLPs have activation functions on nodes. This simple change makes KANs better (sometimes much better!) than MLPs in terms of both model **accuracy** and **interpretability**. A quick intro of KANs [here](https://kindxiaoming.github.io/pykan/intro.html). mlp_kan_compare +## Accuracy +**KANs have faster scaling than MLPs. KANs have better accuracy than MLPs with fewer parameters.** + +Please set `torch.set_default_dtype(torch.float64)` if you want high precision. + +**Example 1: fitting symbolic formulas** +Screenshot 2024-04-30 at 10 55 30 + +**Example 2: fitting special functions** +Screenshot 2024-04-30 at 11 07 20 + +**Example 3: PDE solving** +Screenshot 2024-04-30 at 10 57 25 + +**Example 4: avoid catastrophic forgetting** +Screenshot 2024-04-30 at 11 04 36 + +## Interpretability +**KANs can be intuitively visualized. KANs offer interpretability and interactivity that MLPs cannot provide. We can use KANs to potentially discover new scientific laws.** + +**Example 1: Symbolic formulas** +Screenshot 2024-04-30 at 11 04 56 + +**Example 2: Discovering mathematical laws of knots** +Screenshot 2024-04-30 at 11 05 25 + +**Example 3: Discovering physical laws of Anderson localization** +Screenshot 2024-04-30 at 11 05 53 + +**Example 4: Training of a three-layer KAN** + +![kan_training_low_res](https://github.com/KindXiaoming/pykan/assets/23551623/e9f215c7-a393-46b9-8528-c906878f015e) + + + ## Installation -There are two ways to install pykan, through pypi or github. +Pykan can be installed via PyPI or directly from GitHub. -**Installation via github** +**Pre-requisites:** -```python -git clone https://github.com/KindXiaoming/pykan.git +``` +Python 3.9.7 or higher +pip +``` + +**For developers** + +``` +pip clone https://github.com/KindXiaoming/pykan.git cd pykan pip install -e . ``` -**Installation via pypi (soon)** +**Installation via github** -```python +``` +pip install git+https://github.com/KindXiaoming/pykan.git +``` + +**Installation via PyPI:** +``` pip install pykan ``` +Requirements -To install requirements: +```python +# python==3.9.7 +matplotlib==3.6.2 +numpy==1.24.4 +scikit_learn==1.1.3 +setuptools==65.5.0 +sympy==1.11.1 +torch==2.2.2 +tqdm==4.66.2 +``` + +After activating the virtual environment, you can install specific package requirements as follows: ```python pip install -r requirements.txt ``` +**Optional: Conda Environment Setup** +For those who prefer using Conda: +``` +conda create --name pykan-env python=3.9.7 +conda activate pykan-env +pip install git+https://github.com/KindXiaoming/pykan.git # For GitHub installation +# or +pip install pykan # For PyPI installation +``` + +## Computation requirements + +Examples in [tutorials](tutorials) are runnable on a single CPU typically less than 10 minutes. All examples in the paper are runnable on a single CPU in less than one day. Training KANs for PDE is the most expensive and may take hours to days on a single CPU. We use CPUs to train our models because we carried out parameter sweeps (both for MLPs and KANs) to obtain Pareto Frontiers. There are thousands of small models which is why we use CPUs rather than GPUs. Admittedly, our problem scales are smaller than typical machine learning tasks, but are typical for science-related tasks. In case the scale of your task is large, it is advisable to use GPUs. + ## Documentation -The documenation can be found here []. +The documentation can be found [here](https://kindxiaoming.github.io/pykan/). ## Tutorials **Quickstart** -Get started with [hellokan.ipynb](./hellokan.ipynb) notebook +Get started with [hellokan.ipynb](./hellokan.ipynb) notebook. **More demos** -Jupyter Notebooks in [docs/Examples](./docs/Examples) and [docs/API_demo](./docs/API\_demo) are ready to play. You may also find these examples in documentation. +More Notebook tutorials can be found in [tutorials](tutorials). + +## Advice on hyperparameter tuning +Many intuition about MLPs and other networks may not directy transfer to KANs. So how can I tune the hyperparameters effectively? Here is my general advice based on my experience playing with the problems reported in the paper. Since these problems are relatively small-scale and science-oriented, it is likely that my advice is not suitable to your case. But I want to at least share my experience such that users can have better clues where to start and what to expect from tuning hyperparameters. + +* Start from a simple setup (small KAN shape, small grid size, small data, no reguralization `lamb=0`). This is very different from MLP literature, where people by default use widths of order `O(10^2)` or higher. For example, if you have a task with 5 inputs and 1 outputs, I would try something as simple as `KAN(width=[5,1,1], grid=3, k=3)`. If it doesn't work, I would gradually first increase width. If that still doesn't work, I would consider increasing depth. You don't need to be this extreme, if you have better understanding about the complexity of your task. + +* Once an acceptable performance is achieved, you could then try refining your KAN (more accurate or more interpretable). + +* If you care about accuracy, try grid extention technique. An example is [here](https://kindxiaoming.github.io/pykan/Examples/Example_1_function_fitting.html). But watch out for overfitting, see below. + +* If you care about interpretability, try sparsifying the network with, e.g., `model.train(lamb=0.01)`. It would also be advisable to try increasing lamb gradually. After training with sparsification, plot it, if you see some neurons that are obvious useless, you may call `pruned_model = model.prune()` to get the pruned model. You can then further train (either to encourage accuracy or encouarge sparsity), or do symbolic regression. + +* I also want to emphasize that accuracy and interpretability (and also parameter efficiency) are not necessarily contradictory, e.g., Figure 2.3 in [our paper](https://arxiv.org/pdf/2404.19756). They can be positively correlated in some cases but in other cases may dispaly some tradeoff. So it would be good not to be greedy and aim for one goal at a time. However, if you have a strong reason why you believe pruning (interpretability) can also help accuracy, you may want to plan ahead, such that even if your end goal is accuracy, you want to push interpretability first. + +* Once you get a quite good result, try increasing data size and have a final run, which should give you even better results! + +Disclaimer: Try the simplest thing first is the mindset of physicists, which could be personal/biased but I find this mindset quite effective and make things well-controlled for me. Also, The reason why I tend to choose a small dataset at first is to get faster feedback in the debugging stage (my initial implementation is slow, after all!). The hidden assumption is that a small dataset behaves qualitatively similar to a large dataset, which is not necessarily true in general, but usually true in small-scale problems that I have tried. To know if your data is sufficient, see the next paragraph. + +Another thing that would be good to keep in mind is that please constantly checking if your model is in underfitting or overfitting regime. If there is a large gap between train/test losses, you probably want to increase data or reduce model (`grid` is more important than `width`, so first try decreasing `grid`, then `width`). This is also the reason why I'd love to start from simple models to make sure that the model is first in underfitting regime and then gradually expands to the "Goldilocks zone". + +## Citation +```python +@article{liu2024kan, + title={KAN: Kolmogorov-Arnold Networks}, + author={Liu, Ziming and Wang, Yixuan and Vaidya, Sachin and Ruehle, Fabian and Halverson, James and Solja{\v{c}}i{\'c}, Marin and Hou, Thomas Y and Tegmark, Max}, + journal={arXiv preprint arXiv:2404.19756}, + year={2024} +} +``` + +## Contact +If you have any questions, please contact zmliu@mit.edu + +## Author's note +I would like to thank everyone who's interested in KANs. When I designed KANs and wrote codes, I have math & physics examples (which are quite small scale!) in mind, so did not consider much optimization in efficiency or reusability. It's so honored to receive this unwarranted attention, which is way beyond my expectation. So I accept any criticism from people complaning about the efficiency and resuability of the codes, my apology. My only hope is that you find `model.plot()` fun to play with :). + +For users who are interested in scientific discoveries and scientific computing (the orginal users intended for), I'm happy to hear your applications and collaborate. This repo will continue remaining mostly for this purpose, probably without signifiant updates for efficiency. In fact, there are already implmentations like [efficientkan](https://github.com/Blealtan/efficient-kan) or [fouierkan](https://github.com/GistNoesis/FourierKAN/) that look promising for improving efficiency. + +For users who are machine learning focus, I have to be honest that KANs are likely not a simple plug-in that can be used out-of-the box (yet). Hyperparameters need tuning, and more tricks special to your applications should be introduced. For example, [GraphKAN](https://github.com/WillHua127/GraphKAN-Graph-Kolmogorov-Arnold-Networks) suggests that KANs should better be used in latent space (need embedding and unembedding linear layers after inputs and before outputs). [KANRL](https://github.com/riiswa/kanrl) suggests that some trainable parameters should better be fixed in reinforcement learning to increase training stability. +The most common question I've been asked lately is whether KANs will be next-gen LLMs. I don't have good intuition about this. KANs are designed for applications where one cares about high accuracy and/or interpretability. We do care about LLM interpretability for sure, but interpretability can mean wildly different things for LLM and for science. Do we care about high accuracy for LLMs? I don't know, scaling laws seem to imply so, but probably not too high precision. Also, accuracy can also mean different things for LLM and for science. This subtlety makes it hard to directly transfer conclusions in our paper to LLMs, or machine learning tasks in general. However, I would be very happy if you have enjoyed the high-level idea (learnable activation functions on edges, or interacting with AI for scientific discoveries), which is not necessariy *the future*, but can hopefully inspire and impact *many possible futures*. As a physicist, the message I want to convey is less of "KANs are great", but more of "try thinking of current architectures critically and seeking fundamentally different alternatives that can do fun and/or useful stuff". +I would like to welcome people to be critical of KANs, but also to be critical of critiques as well. Practice is the only criterion for testing understanding (实践是检验真理的唯一标准). We don't know many things beforehand until they are really tried and shown to be succeeding or failing. As much as I'm willing to see success mode of KANs, I'm equally curious about failure modes of KANs, to better understand the boundaries. KANs and MLPs cannot replace each other (as far as I can tell); they each have advantages in some settings and limitations in others. I would be intrigued by a theoretical framework that encompasses both and could even suggest new alternatives (physicists love unified theories, sorry :). diff --git a/pykan.egg-info/SOURCES.txt b/pykan.egg-info/SOURCES.txt index f4f9203b..ccb3b065 100644 --- a/pykan.egg-info/SOURCES.txt +++ b/pykan.egg-info/SOURCES.txt @@ -4,8 +4,15 @@ setup.py kan/KAN.py kan/KANLayer.py kan/LBFGS.py +kan/MLP.py +kan/MultKAN.py kan/Symbolic_KANLayer.py kan/__init__.py +kan/ckpt.py +kan/compiler.py +kan/experiment.py +kan/feynman.py +kan/hypothesis.py kan/spline.py kan/utils.py kan/assets/img/mult_symbol.png diff --git a/tutorials/.ipynb_checkpoints/API_10_device-checkpoint.ipynb b/tutorials/.ipynb_checkpoints/API_10_device-checkpoint.ipynb new file mode 100644 index 00000000..99b43ede --- /dev/null +++ b/tutorials/.ipynb_checkpoints/API_10_device-checkpoint.ipynb @@ -0,0 +1,173 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "134e7f9d", + "metadata": {}, + "source": [ + "# Demo 10: Device\n", + "\n", + "All other demos have by default used device = 'cpu'. In case we want to use cuda, we should pass the device argument to model and dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "7a4ac1e1-84ba-4bc3-91b6-a776a5e7711c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cpu\n" + ] + } + ], + "source": [ + "from kan import KAN, create_dataset\n", + "import torch\n", + "\n", + "torch.use_deterministic_algorithms(False)\n", + "\n", + "#device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", + "device = 'cpu'\n", + "print(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "2075ef56", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "checkpoint directory created: ./model\n", + "saving model version 0.0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "| train_loss: 6.83e-01 | test_loss: 7.21e-01 | reg: 1.04e+03 | : 100%|█| 50/50 [00:19<00:00, 2.62it\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "saving model version 0.1\n" + ] + } + ], + "source": [ + "model = KAN(width=[4,100,100,100,1], grid=3, k=3, seed=0).to(device)\n", + "f = lambda x: torch.exp((torch.sin(torch.pi*(x[:,[0]]**2+x[:,[1]]**2))+torch.sin(torch.pi*(x[:,[2]]**2+x[:,[3]]**2)))/2)\n", + "dataset = create_dataset(f, n_var=4, train_num=1000, device=device)\n", + "\n", + "# train the model\n", + "#model.train(dataset, opt=\"LBFGS\", steps=20, lamb=1e-3, lamb_entropy=2.);\n", + "model.fit(dataset, opt=\"Adam\", lr=1e-3, steps=50, lamb=1e-3, lamb_entropy=5., update_grid=False);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2f182cc1-51bf-4151-a253-a52fe854919e", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "f6f8125e-d26d-4c97-9e5f-988099bb4737", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cuda\n" + ] + } + ], + "source": [ + "device = 'cuda'\n", + "print(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "95017dfa-3a2a-43e0-8b68-fb220ca5abc9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "checkpoint directory created: ./model\n", + "saving model version 0.0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "| train_loss: 6.83e-01 | test_loss: 7.21e-01 | reg: 1.04e+03 | : 100%|█| 50/50 [00:01<00:00, 26.90it\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "saving model version 0.1\n" + ] + } + ], + "source": [ + "model = KAN(width=[4,100,100,100,1], grid=3, k=3, seed=0).to(device)\n", + "f = lambda x: torch.exp((torch.sin(torch.pi*(x[:,[0]]**2+x[:,[1]]**2))+torch.sin(torch.pi*(x[:,[2]]**2+x[:,[3]]**2)))/2)\n", + "dataset = create_dataset(f, n_var=4, train_num=1000, device=device)\n", + "\n", + "# train the model\n", + "#model.train(dataset, opt=\"LBFGS\", steps=20, lamb=1e-3, lamb_entropy=2.);\n", + "model.fit(dataset, opt=\"Adam\", lr=1e-3, steps=50, lamb=1e-3, lamb_entropy=5., update_grid=False);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8230d562-2635-4adc-b566-06ac679b166a", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tutorials/.ipynb_checkpoints/Unchecked_API_10_device-checkpoint.ipynb b/tutorials/.ipynb_checkpoints/Unchecked_API_10_device-checkpoint.ipynb deleted file mode 100644 index cbac3cf4..00000000 --- a/tutorials/.ipynb_checkpoints/Unchecked_API_10_device-checkpoint.ipynb +++ /dev/null @@ -1,110 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "134e7f9d", - "metadata": {}, - "source": [ - "# Demo 10: Device\n", - "\n", - "All other demos have by default used device = 'cpu'. In case we want to use cuda, we should pass the device argument to model and dataset." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "7a4ac1e1-84ba-4bc3-91b6-a776a5e7711c", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "cuda\n" - ] - } - ], - "source": [ - "from kan import KAN, create_dataset\n", - "import torch\n", - "\n", - "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", - "print(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "2075ef56", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "train loss: 5.78e-03 | test loss: 5.89e-03 | reg: 7.32e+00 : 100%|██| 50/50 [00:26<00:00, 1.85it/s]\n" - ] - } - ], - "source": [ - "model = KAN(width=[4,2,1,1], grid=3, k=3, seed=0, device=device)\n", - "f = lambda x: torch.exp((torch.sin(torch.pi*(x[:,[0]]**2+x[:,[1]]**2))+torch.sin(torch.pi*(x[:,[2]]**2+x[:,[3]]**2)))/2)\n", - "dataset = create_dataset(f, n_var=4, train_num=3000, device=device)\n", - "\n", - "# train the model\n", - "#model.train(dataset, opt=\"LBFGS\", steps=20, lamb=1e-3, lamb_entropy=2.);\n", - "model.train(dataset, opt=\"LBFGS\", steps=50, lamb=5e-5, lamb_entropy=2.);" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "3acdcdee-71ca-42a1-98aa-7f7df4a29077", - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAHiCAYAAAAkiYF/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGtElEQVR4nO3deXyU9Z0H8M/zTDLJ5E4mCQRC7oRw5/ACL7QIiiBVtEVxPVrXVldp66utblvRaNd17bqK22O3ddeKIGolIpcJCmKpBwgBDCJJICQkkGMmySRzZc7f/sEOReTI8cw8c3zer5cvXy/JzHzlN08+z+/3/A5JCCFARESkIFntAoiIKPwwXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUlyU2gUQhQIhBHp6emCxWJCQkAC9Xg9JktQuiyhosedCdB4mkwkrVqxAcXExMjIykJ+fj4yMDBQXF2PFihUwmUxql0gUlCSeREl0drW1tVi8eDFsNhuAk70XH1+vJS4uDmvXrsW8efNUqZEoWDFciM6itrYWN954I4QQ8Hq95/w5WZYhSRI2bdrEgCE6DcOF6AwmkwnZ2dmw2+3nDRYfWZah0+nQ3t6OlJQU/xdIFAL4zIXoDK+++ipsNtuQggUAvF4vbDYbVq5c6efKiEIHey5EpxFCoLi4GM3NzRjOpSFJEgoKCtDU1MRZZERguBB9jdFoREZGxqher9frFayIKDRxWIzoNBaLZVSvN5vNClVCFNoYLkSnSUhIGNXrExMTFaqEKLQxXIhOo9frUVhYOOznJpIkobCwEGlpaX6qjCi0MFyITiNJEh5++OERvXbZsmV8mE/0//hAn+gMXOdCNHrsuRCdISUlBWvXroUkSZDl818ivhX61dXVDBai0zBciM5i3rx52LRpE3Q6HSRJ+sZwl++/6XQ6bN68GXPnzlWpUqLgxHAhOod58+ahvb0dL774IgoKCr72ZwUFBXjxxRdx/PhxBgvRWfCZC9EQCCHw0Ucf4frrr0dNTQ2uvvpqPrwnOg/2XIiGQJIkpKSkfO3fRHRuDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSnCSEEGoXQRQIQgi0tbWN+PVerxcOhwMxMTGQ5ZHfl02YMAGSJI349UShIErtAogCxeVyYe3atSgoKBjxewghRhQMXq8XjY2NiIqKwsMPPwytVjviGohCAcOFIsrMmTNx2WWXBfxza2pq8Jvf/Ab/+7//G/DPJlIDn7kQ+ZkQAj/+8Y+RlpaG9PR0tcshCgj2XIj8rK6uDq2trVi7di2ftVDEYM+FyI+EEHjggQeg1+tx/fXXq10OUcCw50LkR/X19aivr8crr7wyqhlmRKGG33YiPxFC4Pvf/z70ej1uu+02tcshCij2XIj85LPPPsOBAwewatUqaDQatcshCij2XIj8wOv14p577kF2djZuvvlmtcshCjj2XIj8YNWqVTh27Bg++OADPmuhiMRvPZHCLBYLHnnkEcyaNQuzZs1SuxwiVTBciBQkhMCyZcvgcDiwcuVKrmuhiMVwIVLQrl278MYbb+DRRx/FuHHj1C6HSDUMFyKF2O12LFmyBLm5uXjsscfYa6GIxgf6RAoQQuChhx6CwWDA559/jqgoXloU2XgFEI2SEALV1dV4/fXX8dRTT6G0tFTtkohUx2ExolFqbm7GfffdhyuuuAKPPPIIh8OIwHAhGpX+/n7ccMMNSE5Oxl/+8heuxCf6fxwWIxohh8OBm2++GQaDATt27EBKSoraJREFDYYL0Qi43W7ce++92LVrF9asWYMpU6aoXRJRUGG4EA2T2+3GD3/4Q7z77rtYsWIFFixYwOcsRGfgMxeiYXA6nbj//vuxZs0aPPPMM/j+97/PYCE6C/ZciIZoYGAA99xzD7Zs2YJnn30WDz30EIOF6BwYLkQXIITA0aNHcfvtt6OhoQF//OMfcfvttzNYiM6D4UJ0Hl6vFxs2bMA//dM/QaPRYP369bjyyisZLEQXwGcuRGchhEBPTw+WLVuGpUuXYuLEidixYweDhWiIGC5EZ3C5XKiursbll1+O119/HY899hg2bdqEnJwcBgvREHFYjOj/eTwe7Nq1C08//TT++te/4uKLL8bq1atRUVHBUCEaJoYLRTQhBFwuF3bu3IkVK1bg/fffx7hx4/C73/0OS5YsQUxMjNolEoUkhgtFJCEEurq6UFNTg1deeQV1dXXIzs7GU089hbvuugspKSnsrRCNAsOFIorL5cLatWuxbt06fPTRR+jv70dZWRl++9vfYtGiRUhOTmaoECmA4UIRxeFw4N5770Vubi6++93v4tZbb0V5eTmio6MZKkQKkoQQQu0iiALB6XTiv//7v9Hb24upU6ciPj4+4IHS0NCAH/7wh9BqtQH9XKJAY7hQxBBCoLGxUdUeihACJSUl7CVR2GO4EA2REAJCCEiSxHAgugAuoiQaov379yM+Ph779+9XuxSioMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiGQAiBvr6+r/2biM6N4UJ0HiaTCStWrEBxcTGuvfZaOBwOXHvttSguLsaKFStgMpnULpEoKEmCt2BEZ1VbW4vFixfDZrMBwNd6K5IkAQDi4uKwdu1azJs3T5UaiYIVw4XoLGpra3HjjTdCCAGv13vOn5NlGZIkYdOmTQwYotMwXIjOYDKZkJ2dDbvdft5g8ZFlGTqdDu3t7UhJSfF/gUQhgM9ciM7w6quvwmazDSlYAMDr9cJms2HlypV+rowodLDnQnQaIQSKi4vR3Nw8rBlhkiShoKAATU1Np57HEEUyhgvRaYxGIzIyMkb1er1er2BFRKGJw2JEp7FYLKN6vdlsVqgSotDGcCE6TUJCwqhen5iYqFAlRKGN4UJ0Gp1Oh7S0tBG9trCwcMSvJQo3DBciAB6PB2vWrMHMmTPh8XhG9B4zZ85EX1+fwpURhSaGC0W8HTt2YM6cOfjRj36Eyy67DDt27EB8fDxkeWiXhyRJiI2NRU5ODp577jls2rQJdrvdz1UTBTfOFqOI1djYiKqqKrz//vu46KKL8PTTT6OyshLA8Ffob968GbNnz8ZHH32EDz/8ENHR0Zg7dy4uu+wyaDSaQP0vEQUNhgtFHKPRiOeeew6vvfYaxo8fj+XLl2PhwoXfWJ8y1L3FqqurMXfu3FN/NjAwgJqaGuzevRvp6em48cYbMXnyZK5/oYjCcKGI4XA48Mc//hEvvPACZFnGI488gvvuuw9arfacrzGZTFi5ciVeeuklHDly5NR/LywsxLJly3D33XcjOTn5rK89ceIENm7ciKamJhQWFmLhwoUYP3684v9fRMGI4UJhTwiBd955B7/+9a/R2dmJe+65Bz/96U+HNbNLCIHe3l6YzWYkJiYiLS1tSD0RIQQOHTqEjRs3wmAwoLKyEtdff/05A4koXDBcKKzt2rULy5cvR11dHa6//no88cQTKCwsDHgdXq8XO3fuRG1tLZxOJ66++mrMnj0bMTExAa+FKBAYLhSWWlpa8NRTT2Hjxo2YPn06qqqqcPnll6tdFgYHB7Ft2zbs2LEDOp0O8+bNw8UXXzzkmWlEoYLhQmHFZDLhP/7jP/A///M/yMjIwC9/+UssXrw46H559/X14b333sPevXuRlZWFBQsWoKSkRO2yiBTDcKGw4HQ68corr+D555+H0+nEj370I/zwhz+ETqdTu7Tzamtrw/r169HS0oKJEydi4cKFGDNmjNplEY0aw4VCmhAC7733HqqqqtDa2oqlS5fi0UcfRWZmptqlDZkQAgcOHMCmTZvQ29uLSy+9FHPnzuU+ZRTSGC4Usvbt24fly5fjs88+wzXXXIOqqiqUlpaqXdaIeTwefPzxx/jggw/g9XpxzTXX4KqrrkJ0dLTapRENG8OFQk57ezueeeYZvP322ygtLUVVVRWuueYatctSjM1mwwcffIBPPvkEiYmJuOGGG1BeXs5FmBRSGC4UMsxmM1566SX813/9F5KSkvDYY4/h9ttvR1RUlNql+YXRaMTmzZtRX1+P7OxsLFy4EAUFBWqXRTQkDBcKem63G6tXr8a//du/wWKx4MEHH8RDDz006rNXQsXRo0exYcMGtLW1YerUqbjxxhuRnp6udllE58VwoaAlhMDWrVtRVVWFhoYGfOc738EvfvELjBs3Tu3SAk4IgX379mHz5s0YGBjArFmzcN111yEuLk7t0ojOiuFCQengwYNYvnw5/vrXv+Lyyy9HVVUVpk+frnZZqnO5XPjb3/6GrVu3QpZlzJkzB7NmzQrboUEKXQwXCipdXV149tln8frrr6OgoABPPPEE5s2bx4fZZ7BYLNiyZQt27tyJ1NRUzJ8/H9OmTePfEwUNhgsFBZvNht///vf47W9/i5iYGPz85z/HXXfdxWm4F9DV1YVNmzbhq6++Ql5eHhYuXIicnBy1yyJiuJC6vF4v3nrrLTzzzDPo7e3Ffffdh5/85CfcNXiYmpqasGHDBnR0dKCsrAzz589Hamqq2mVRBGO4kGp27NiBJ554AgcOHMCiRYvw+OOP8657FLxeL/bs2YOamhrYbDZcccUV+Na3voXY2Fi1S6MIxHChgGtqakJVVRW2bNmCyspKPP3007jooovULitsOBwOfPTRR9i+fTu0Wi3mzp2LSy+9lMctU0AxXChgenp68Nxzz2HlypUYP348Hn/8cdx00018CO0n/f39qK2txe7du5GRkYEFCxagtLSUf98UEAwX8jvf8cIvvvgiJEka0vHCpJzTj1suKirCggULeNwy+R3DhfxGieOFSRk8bpkCjeFCfnHm8cLLly9HUVGR2mVFPI/Hg507d2LLli08bpn8iuFCimppacHTTz+NDRs2BNXxwvR1Zx63fP311+Oiiy4KuhM7KXQxXEgRJpMJL7zwAl5++WWkp6fjl7/8JW699Vb+sgpyPG6Z/IXhQqPidDrx5z//Gc8//zwcDgeWLVuGBx54IOiPF6avO3bsGDZs2ICWlhaUlpZiwYIFPG6ZRoXhQiMSDscL09fxuGVSEsOFhm3fvn144okn8Omnn+Kaa67Bk08+iUmTJqldFinE7Xbjk08+4XHLNCoMFxqy048XnjhxIqqqqnDttdeqXRb5ie+45Y8//hhJSUk8bpmGheFCF2SxWPDSSy/hD3/4AxITE/HYY4/hjjvu4BkiEcJoNGLTpk04cOAAsrOzcdNNNyE/P1/tsijIMVzonE4/XthsNuPBBx/Eww8/HDHHC9PXNTc3Y8OGDWhvb+dxy3RBDBf6BiEEtm3bhieffBINDQ247bbb8Itf/IJbhhCEENi7dy/ee+89DAwM4PLLL8ecOXN43DJ9A8OFvubgwYN44okn8NFHH2HWrFmoqqrCjBkz1C6LgozL5cKOHTuwbds2HrdMZ8VwIQB/P154zZo1yMvLw5NPPsnjhemCzGYz3n//fXz22WdIS0vjcct0CsMlwp15vPBPf/pT3H333dyxmIalq6sLGzduxKFDh3jcMgFguEQsr9eLv/zlL/iXf/mXU8cL//jHP0ZKSorapVEIa2xsxMaNG9HR0YHy8nLccMMNPG45QjFcItDf/vY3PPHEE6ivr8eiRYvwq1/9Crm5uWqXRWHC6/Vi9+7dqKmpgd1ux5VXXolrr72Wxy1HGIZLBGlqasJTTz2F2tpaVFZW4qmnnsLFF1+sdlkUps523PJll13GzUwjBMMlAvT09OA3v/kNXn31VYwbNw6PP/44Fi1axIeuFBD9/f2oqanBnj17eNxyBGG4hDGHw4E//elPeOGFFyBJEn7yk5/gvvvu48FQpIoTJ05gw4YNOHz4MIqKirBw4UKMGzdO7bLITxguYUgIgXXr1uHXv/41Tpw4gXvuuQc/+9nPeLwwqY7HLUcOhkuY+fzzz7F8+XLs2bOHxwtT0DrzuOXZs2dj9uzZnAIfRhguYaK1tRVPP/001q9fj2nTpqGqqgpXXHGF2mURndfpxy3HxcXh+uuvR2VlJR/6hwGGS4gzmUx48cUX8fLLL0Ov1/N4YQpJfX192Lx5M/bt24esrCwsXLgQxcXFapdFo8BwCVFOpxOvvvoq/v3f/x0OhwMPP/wwHnzwQR4vTCGNxy2HD4ZLiBFCoKamBlVVVWhpacEdd9yBRx99lBcghQ0hBOrr67F582b09fWdOm6ZRz2EFoZLCDn9eOHZs2ejqqqKxwtT2DrzuOVrr70WV155JY9bDhEMlxDg9XqxbNkyvPXWWzxemCLOmcct33777SgoKFC7LLoAhkuACCHQ3t4+4tdbrVZoNJpR78+UnZ3NldGkCiEETCbTiF9vt9vR3NyMgoKCUT1bTElJ4TUQADzZJ0BcLhfWr1+PvLy8c/6M0+lER0cHUlJS/LKo7OjRo7j//vu5loBU4fF4sH//fuj1+nP+jNPphNlsRmJi4lm/p8nJyejp6RlxDUajEVdeeSUPNQsA/g0H0CWXXHLWjSJ9q5YfeeQRHDp0CKmpqXjsscdw2223KXqHtWvXLsXei2gk8vLyznqDJYTA0aNHUVtbi76+PqSmpuK2225DTk6OotfA0aNHFXsvOj8uhggC7e3tWLRoEbq6urB8+XIUFxfjRz/6Ed5//31w1JIiQUdHB15++WVIkoQFCxZACIE//vGPoxpGI3UxXFTm8XiwZMkSSJKEzZs345577sFrr72G6dOn4wc/+AGsVqvaJRL5ldvtxssvv4z4+Hg8+OCDmDlzJh588EHIsoxXXnmFN1ghiuGispUrV6KpqQmvvPIKxo4dC0mSoNVqsXLlSgwODuLxxx/nxUVhSwiBbdu2wWw24/vf//6pB/Xx8fFYsmQJOjo60NjYqHKVNBIMFxU5nU5UVVVh1qxZmDlz5tf+LDMzE3fffTfWrFmDgYEBlSok8i+3241t27Zh6tSp31gIPHnyZKSlpeHtt9/mDVYIYrio6LXXXoPNZsN//ud/fuOhpSRJ+NWvfgVJkvDss8+qVCGRf3300UfweDxYvHjxWa+BxYsXw2Qy4fjx4ypVSCPFcFGJ1+vFs88+i/LycmRnZ5/1Z+Lj43HDDTfgtddeg8vlCnCFRP7l9Xrx4YcforCwEPHx8Wf9maKiIsTExGDdunXsvYQYhotKdu/ejf7+fjz77LPnnGopSRKqqqrgdDrx3nvvBbhCIv9qbm6G0+nEt7/97XNeA7IsY/bs2WhtbYXD4QhwhTQaDBcVCCHw5JNPIjk5GWVlZef92ezsbGRnZ+Ppp5/mnRuFDSEENm3ahLi4uAtuuuo7l+izzz4LRGmkEIaLCux2O/bs2YP777//ggvEJEnCo48+itbW1lGtTCYKJi6XC+3t7bjqqqsueA3ExMQgKysL27dv5w1WCGG4qOCdd96BEAL/+I//OKSfX7RoEWRZxu9//3s/V0YUGPv27QMAzJo164I/K0kSbrjhBlitVvT39/u5MlIKwyXAhBB48cUXMWHChCHvHxYTE4OLL74Yr776Ku/cKOQJIbB161akpKQMeSPWkpISSJKE7du3+7c4UgzDJcBsNhtaW1vx0EMPDXnPJEmS8Itf/AJms5l7I1HIc7lc6O3txezZs4d8DciyjIKCAnz++ee8wQoRDJcAW7duHQDgtttuG9brLrnkEkRHR+P555/3Q1VEgVNfXw8AqKioGPJrJEnCvHnz4HQ6+ewxRDBcAux3v/sdsrKyzjmv/1w0Gg3mzJmDDRs2wOv1+qk6Iv/bvn07EhIShn02UW5uLmRZxrZt2/xUGSmJ4RJALpcLR44cwfe+970RbSP+85//HIODgzh48KAfqiPyP6/Xi87OTlx66aXDvgYkSUJxcTH27dvHobEQwHAJIIfDgdtvvx133HHHiF4/adIk6HQ6PPfccwpXRhQYTqcTpaWluOyyy4b9WkmSMHfuXLhcLhgMBj9UR0piuARQQkICXnjhBaSnp4/o9bIs46abbsLWrVvh8XgUro7I/2JjY3HvvfeO+KTV7OxsaDQabN26VeHKSGkMlwCTJGlUJ+s98sgjcLlc2Llzp4JVEQWOLMsjvgZkWUZpaSm++OKLYT17FELwhizAGC4hJi8vD0lJSXjmmWc47kwRae7cuXC73ejo6Bjya3p6evCHP/wBg4ODfqyMTsdwCTGSJOGee+7B7t27h7yRnxCCuypT2Bg7diyio6NRW1s75Busv/71rzh27BhiYmL8XB35MFxC0AMPPACv14vq6uoh/XxraysWLVrE88gpLMiyjIqKCjQ0NAxpaEwIgbq6OuTn549qSJqGh+ESgtLS0pCfn4/nnntuSHduv/nNb7B//34kJiYGoDoi/5szZw68Xu+QpuWbTCY4HA7MmTMnAJWRD8MlBEmShOXLl+PEiRNobW0978+63W6sX78e8+bNg0ajCVCFRP6VnJyMpKQkbNq06YI3WB988MGp7WMocBguIWrevHmIjY3F8uXLz3tx1dbWwuFw4PHHHw9gdUT+JUkS5s+fj56envMO93o8HtTV1WHSpEmQZf66CyT+bYeoqKgofO9738OWLVswMDBw1p8RQmD58uXIyclBXl5eYAsk8rOysjJERUXh3XffPecN1hdffAG3240FCxbweUuAMVxC2E9/+lPIsnzO3sunn36KtrY2/Ou//isvLAo7Go0GV155JQ4ePAiLxfKNP/d6vVi/fj0yMzOh1+tVqDCyMVxCWHx8PO699168+eab33j24nQ68YMf/AC5ubn41re+pVKFRP41Z84caDQavPHGG1+7wRJC4LPPPoPFYsGSJUt4c6UChksIkyQJv/rVr5CcnIylS5fCarUCODnO/M///M8wGAz485//zLFmClvR0dFYuHAhGhsbsWfPHgghIIRAd3c3NmzYgNLSUmRnZ6tdZkSKUrsAGh2dTofXXnsNt9xyC2655Rbcf//92Lp1K9auXYuf/exnmDx5stolEvmNJEm47LLL8NVXX+Evf/kLenp6kJSUhNraWiQkJGDp0qXstaiE4RIGLr74YqxcuRK//OUv8ZOf/ASJiYl4/PHH8eCDD/LCorAnyzLuuusuvPPOO9ixYwe8Xi8mTJiAJUuWDPvMGFKOJLhBVUA4nU688sorKCoq8ttnWK1WtLW1IT09HXq9/hvB0tTUhO9973vQarV+q4HoXNxuNz799FNkZGT45f2FEDCZTHC73UhLSzvruq7u7m7MmjULUVG8r/Y3hkuACCFw+PBhVXsSQggUFRWxN0OqEEIExTksGRkZvAYCgOESInxbhms0Gl4YFLE8Hg9sNhvi4uK440SQ4zSiEFFfX49x48ahvr5e7VKIVNPZ2YmnnnoKnZ2dapdCF8BwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXEKAEAJ9fX3wer3o6+uDEELtkogCTgiB3t5e9Pf3o7e3l9dBkGO4BDGTyYQVK1aguLgYs2fPRk9PD2bPno3i4mKsWLECJpNJ7RKJ/O7062D69On405/+hOnTp/M6CHKSYPwHpdraWixevBg2mw0AvnaXJkkSACAuLg5r167FvHnzVKmRyN94HYQuhksQqq2txY033gghBLxe7zl/TpZlSJKETZs28cKisMPrILQxXIKMyWRCdnY27Hb7eS8oH1mWodPp0N7ejpSUFP8XSBQAvA5CH5+5BJlXX30VNpttSBcUAHi9XthsNqxcudLPlREFDq+D0MeeSxARQqC4uBjNzc3DmgkjSRIKCgrQ1NR0ahyaKFTxOggPDJcgYjQakZGRMarX6/V6BSsiCjxeB+GBw2JBxGKxjOr1ZrNZoUqI1MPrIDwwXIJIQkLCqF6fmJioUCVE6uF1EB4YLkFEr9ejsLBw2OPFkiShsLAQaWlpfqqMKHB4HYQHhksQkSQJDzzwwIi2tVi2bBkfYlJYkCQJDz/88Ihey+sgeDBcgkhnZye0Wi20Wu2QLxBZlhEXF4e77rrLz9URBc7dd9+NuLg4yPLQfkVJkoTY2Fj8wz/8g58ro6FiuASJPXv24M0330R6ejrWrFkDWZYveGH5ViZXV1dz4RiFlZSUFKxduxaSJF3wRst3HTz//PPo7u6G0+kMUJV0PpyKrLLBwUHU1taiubkZlZWVuPzyy6HRaM67p5JPfHw8qqurMXfu3ECXTRQQ7733HhYvXgy73Q5Jks65t1h1dTWuuOIKtLS0wO12IycnhzdcKmO4qOjEiRPYvHkzXC4X5s2bh4KCgq/9uclkwsqVK/HSSy/hyJEjp/57VlYWrrrqKvzhD39AampqoMsmCpienh589dVX2LVrF37/+99/7TooLCzEsmXLcPfddyM5ORkA4PF4cOzYMZhMJmRkZGDcuHFDHlojZTFcVCCEwO7du/Hxxx8jKysL8+fPP+/0Sd85FmazGYmJifB6vVi1ahVuuOEGlJaWBrByosBqaGhAbGwscnNzv3EdpKWlnXPIzGg0or29HbGxscjPz0dMTEyAKyeGS4DZ7XbU1NSgpaUFF198MWbNmjWiO6vq6mrY7XYsXbrUD1USqW9gYAAtLS0oKipCXFzcsF9vt9tx9OhRuFwu5OTksJcfYOwvBtDx48exatUqdHV14eabb8YVV1wx4i57ZWUluru70d7ernCVRMHBYDAgLi5uRMECADqdDqWlpUhOTkZLSwva2tqGvBEmjV6U2gVEAiEEdu3ahU8//RTjxo3D/PnzR70KOTc3F3q9Hnv27EF2drZClRIFB7vdDqvVitzc3FG9jyzLyMvLQ2JiItrb22G1WpGXl4fY2FiFKqVzYc/Fz2w2G6qrq/HJJ5/gkksuwa233jrqYPGprKxEc3Mz+vr6FHk/omBhMBig1WqRlJSkyPvp9XqUlJRACIGGhgb09vYq8r50bgwXP2pra8OqVatgNBqxePHiET9fOZfS0lLExcWhrq5OsfckUpvL5UJ/fz/S09MVXW2v0+kwceJEpKSkoLW1FceOHeMwmR8xXPxACIFPP/0Ub7/9NtLS0nDnnXciJydH8c/RaDQoKyvDwYMHYbfbFX9/IjUYjUbIsuyXPcJkWUZubi5yc3PR19eHhoYGDA4OKv45xHBRnMViwdtvv42dO3di1qxZWLx4MeLj4/32edOnTwcAfPHFF377DKJA8Xq96O3tRVpaml/Xp6SlpWHixIkATk537unp8dtnRSqGi4JaW1uxatUq9PX14dZbb8Wll17q9030dDodJk+ejH379sHtdvv1s4j8rbe3F16vF+np6X7/rNjYWEycOBGpqak4duwYWltbOUymIIaLArxeLz7++GNUV1cjMzMTd955Z0BncFVUVMBms6GhoSFgn0mkNCEEjEYjkpOTER0dHZDPlGUZOTk5yMvLg8lkwqFDhzjErBCGyyj5hsE+//xzXHHFFbj55ptHPC9/pFJTU1FQUIA9e/YE9HOJlDQwMACn0zmqI45HKjU1FaWlpZBlGQ0NDTAajQGvIdwwXEbh6NGjeO2119Df34/bbrsNF198sWpnSVRWVqKnpwetra2qfD7RaBkMBiQkJECn06ny+TExMSgpKYFer0dbWxtaWlrg8XhUqSUccBHlCPiGwXbv3o38/HzMmzdPtQvCJzs7G5mZmdizZ8+oF54RBZrNZoPNZkNeXp6qdciyjAkTJiAhIQHHjh2DzWZDfn6+6td3KGLPZZgGBgbw1ltvoa6uDldddRUWLVoUNF+8yspKtLa2cuYLhRyDwYCYmBjFFk2Olm+YTKPRoKGhAQaDQe2SQg7DZRiOHDmC1atXw2q14jvf+Q4qKyuD6kjVkpISJCQk8NkLhRSn03lq0WQw8Q2Tpaeno729HUePHuUw2TAwXIbA4/Fg+/btWL9+PcaPH4+lS5ciKytL7bK+QZZllJeX49ChQ7BarWqXQzQkRqMRGo0mKHctliQJ2dnZyM/Ph9lsxqFDh04d4Efnx3C5gP7+frz55pvYv38/Zs+ejZtuuimoN72bNm0aZFnG/v371S6F6II8Hg96e3uRnp4e1Id6paSkoLS0FFFRUWhsbER3d7faJQW94G3NINDU1IRVq1ZhcHAQS5YsQXl5udolXVBMTAymTp2K/fv3c1ElBb3e3l4IIaDX69Uu5YK0Wi1KSkqQkZGB48ePo7m5mcNk58FwOQu3241t27Zh48aNyM3NxdKlSzFmzBi1yxqy8vJyOBwOHDx4UO1SiM7Jt2gyNTUVUVGhMXFVkiSMHz8eBQUFsFgsHII+D4bLGUwmE958800cOHAA1157LRYsWBByR6QmJyejqKgIdXV14EGjFKz6+/vhcrmC7kH+UCQnJ6O0tBTR0dFoamriMNlZMFxO09DQgNWrV8PpdOL222/HjBkz1C5pxCorK9HX14ejR4+qXQrRWRkMBiQmJgb1M8zz0Wq1KC4uRmZmJo4fP44jR45wKPo0DBecHAb74IMPsHnzZuTn52Pp0qWqbEGhpKysLGRlZXFaMgUli8UCu90ekr2W00mShHHjxqGwsBA2mw2HDh2CxWJRu6ygEPHh0tvbizVr1uCrr77CnDlzMH/+fGi1WrXLUkRlZSXa29vZZaegYzQaERsbi8TERLVLUURSUhJKS0sRExODw4cPo6urS+2SVBfR4fLVV1/h9ddfh9frxe23345p06apXZKiCgsLkZSUxN4LBRWHw4GBgYGQHx04U3R0NIqKijBmzBicOHEChw8fjuhhsogMF5fLhS1btqCmpgbFxcW44447Qr57fjayLKOiogKNjY3sqlPQMBqNiIqKQkpKitqlKE6SJGRlZaGoqAh2uz2ih8kiLlx6enqwZs0aNDQ0YO7cuZg3b17Azo5Qw5QpUxAdHY29e/eqXQoR3G73qUWTwbR1ktISExNPDZM1NTWhs7Mz4mZuRlS4fPnll3j99dcBAHfccQemTJmickX+p9VqMW3aNNTX18PpdKpdDkW4np4eSJKEtLQ0tUvxO98wWVZWFjo6OnDkyBG4XC61ywqYiAgXl8uFmpoabNmyBaWlpbjjjjtCYkWwUsrKyuByufDll1+qXQpFMCEEenp6QmrR5GhJkoSxY8eiuLgYg4ODOHToEMxms9plBUTYh4vRaMTq1atx+PBh3HDDDbjuuusi5ovtk5iYiJKSEuzduzfiuuYUPPr6+uB2u8Py+eaFJCQkoLS0FDqdDocPH0ZHR0fYX4thHS5ffPEFXn/9dURFRWHp0qUoLS1VuyTVVFRUoL+/H4cPH1a7FIpQRqMRSUlJIbfjhVKioqJQVFSEcePGobOzE4cPHw7rYbKwDBen04nNmzdj69atmDJlCpYsWRKU23kH0pgxY5Cdnc1pyaQKs9mMwcHBsJt+PBJjxoxBcXExHA4HDh06hIGBAbVL8ouwC5fu7m6sXr0aR48exY033ohvfetbETcMdi4VFRXo6OhAR0eH2qVQhDEYDNDpdIiPj1e7lKDgGyaLi4vDkSNHcOLEibAbJgurcNm3bx/eeOMNaLVaLF26FCUlJWqXFFQKCgqQkpLC3gsF1ODgICwWC3stZ4iKikJhYSHGjx+P7u5uNDU1hdWMzrAIF4fDgY0bN+LDDz/EtGnTsGTJkrBcoDVakiShoqIChw8fRn9/v9rlUIQwGAyIjo5GcnKy2qUEpczMTBQXF8PlcuHQoUNhc22GfLh0dXVh9erVOHbsGBYuXIhrrrkGGo1G7bKC1pQpUxATE8NFlRQQLpcLJpMp7BdNjlZ8fDxKS0uRkJCA5uZmHD9+POSHyUI6XOrq6vDGG28gNjYWS5cuRVFRkdolBb2oqCjMmDEDBw4cgMPhULscCnORtGhytDQaDQoKCjB+/HgYDAY0NjaG9DBZSIbL4OAg1q9fj48++gjl5eX47ne/yy73MMyYMQNerxf19fVql0JhzOv1oqenB2lpaRxNGIbMzEyUlJTA7Xbj0KFDMJlMapc0IiEXLh0dHVi1ahWOHz+ORYsW4aqrruIXd5h8XfC9e/fC6/WqXQ6Fqb6+Png8nohcNDlacXFxKC0tRWJiIo4ePYr29vaQGyYLmXARQmD37t146623kJCQgDvvvBMFBQVqlxWyKioqYLFY0NjYqHYpFKaMRiOSk5PD5nykQNNoNMjPz0d2djaMRiMaGxtDaig7JMLFbrfj3XffxY4dO1BRUYHvfOc7YXPIkFrS09ORm5vLacnkFwMDA3A4HJx+rICMjAyUlJTA4/Hg0KFD6OvrU7ukIQn6cDl+/DhWrVqFzs5O3Hzzzbjyyishy0FfdkiorKxEd3c32tvb1S6FwozBYEBcXBzi4uLULiUsxMXFYeLEiUhOTkZLSwva2tqCfkg7aJeuCyHw+eef45NPPsG4ceMwf/58JCQkqF1WWMnNzYVer8eePXuQnZ2tdjkUJux2O6xWK3Jzc9UuJaxoNBrk5eUhISEB7e3tsFqtyM/PD9q92oKyC2Cz2fDOO+/g448/xiWXXIJbb72VweInlZWVaG5uDpmuNgU/g8EArVaLpKQktUsJS+np6Zg4cSK8Xm9QD5MFXbi0tbVh1apVMBgMuOWWWzBr1iwOg/nRxIkTERcXh7q6OrVLoTDgcrnQ39/PRZN+ptPpUFpaipSUFLS0tODYsWNBN0wWNL+1hRD47LPPsHbtWqSlpeHOO+9ktzoAoqKiUFZWhoMHD8Jut6tdDoU4o9EIWZa5aDIAZFlGbm4ucnJy0NfXh4aGBgwODqpd1ilBES5WqxVr167FZ599hssuuwyLFy/m7qkBNH36dAAnz78hGimv14ve3l6kpaVxtCGA9Ho9Jk6cCABoaGhAb2+vyhWdJAkFVuYIIUZ1dGdraytOnDiBSZMmjWrDycTExIjtio+2DRobG2GxWFBRUTGqOiK5DUKdEGJUh1dZLBZ0dHQgPz9/VMdcREdHR+x3aDRt4PV60dXVBYvFgoKCglEtLleiDRQJF4/Hg7q6OkiShKSkpGHftQgh4Ha7ER0dPezPFkLA4XDAbrejoqIiYlfr+9oAAJKSkob99+B2u6HRaEb0hfJdEFarNaLbINR5vV4YjcZRzT7yeDwjan8hxKl9tNLT0yO25+P1emEwGCDL8oh+HwInr+WRhPvpbZCRkTHqNlBsKnJGRgY2bdqEm2++GePGjVPqbS+ou7sb7777LubMmROwzwxWGRkZ2LhxI2655ZaAtoHRaMSGDRswe/bsgH0m+YdOp4PRaMT48eMDurLearWivb0dOTk5AfvMYKXT6dDa2orCwsKArhOyWCxoa2tDXl6eIu+n2O2BVquFx+PBgQMHlHrLIfnyyy9hs9m4xQT+3gaBfnZy4MAB9Pf3sw3CgCzLMJvNAd8ssaenB16vl71enFzP4vF4YDQaA/q5vjZQqteoWLhIkoSMjAw0NzcHbIM1IQSampqg1+sjdoz2TJmZmQFvg8bGRs4OCiPR0dHo7e0N6HfIbDZzNf9pdDod+vv7A9oGAwMDiraBogOb06dPx+DgIDwej5Jve05erxc2mw1Tp04NyOeFghkzZsDhcMDtdgfk87xeL6xWK9sgjKSkpAT0HBEhBDweD29QTpOeng6PxxPQcPF4PNDr9Yq9p6Lh4tul+Pjx40q+7Tl1dnYCAIqLiwPyeaHA1wZtbW0B+byOjg4AODUVkkKf75d8oNZMWK1WAOCK/tP4zqeyWCwB+TxfGyh5Lpai4RIbG4uoqCjs27dPybc9p3379kGj0bA7fZqYmBhER0cHrA327t3LNggzvmdnPT09Afk8o9EISZL4vOU0vpmbBoMhIJ9nMBgUbwNFw0WSJEyYMAFtbW1+784JIdDS0oLx48fzectpJElCXl5eQA4XEkKgtbUV2dnZbIMwE6gxfyEELBYL10edwbesw2KxBKQNzGYzEhISFG0DxSeTl5eXw+12+30rkcHBQbhcLpSXl/v1c0JReXk5PB6P37vUdrsdLpdr1AsvKbhIknRqzN/f+1W53W4IIXha5VlkZGScWgPoT75nO0qfvaN4uPjWV3z11VdKv/XX+E5QnDBhgl8/JxSNHTsWAPw+LdzXxtyuP/z4nn8MDAz49XN8O/pyWPWbfFtg+Xt40rddjNI7zyseLhqNBklJSdi/f7/funNCCOzduxcJCQkcpz0L38aB9fX1fm2D/fv3j2g3AAp+sixDo9HAYDD49TvU09MDrVbLIbFziImJQU9Pj1/bwGg0+mXLHcXDRZIklJWVYWBgwG9Tkr1eL0wmE6ZPn84v5VlIkoSKigpYrVa/dak9Hg/6+/tRVlbGNghDkiQhLS3NrzPGfNsGcXv+s/OtHXS5XH4bnvRt+eKPNvDLBj6TJk0CALS0tPjj7XHs2DEAwJQpU/zy/uGgpKQEwN+HD5V29OhRAMDkyZP98v6kPt+aB980VaX5htxGs1ltuEtNTQUA9Pf3++X9fZvd+mONkV/CJTY2FrGxsdi1a5fi3Tnf8cdarZbjtOeh1WoRHx+P3bt3+6UNdu3adaqdKTxFR0dDlmV0dXX55TvU3d0NjUbDYdXzkGUZUVFRfmuDrq4uaDSaUe1ifS5+CRdJkjB16lQYDAbFu3NCCHR0dGDy5MnsSp+Hb3iyt7dX8eFJ386tHJYMb5IkISUlxS89FyEEBgcHuXXTBfiGxhwOh1+eu9hsNqSlpfmlDfy2r3V5efmpdRBK8q2hqaysVPR9w5HvEDClh8aOHj0KIQTKysoUfV8KPpmZmQCUHxrzDcdwCvKF+YYnld5M1NcGSk9B9vFbuMTFxSE2NhaffvqpYokrhMAnn3yCmJgYxafNhaOYmBgkJiZi586dirbBp59+Cp1Ox2HJCOAbGuvs7FT0O9TZ2YmoqCgOiQ2BRqNBdHS0okNjvjbwvbc/+C1cfDOWDAaDYjOWPB4Purq6OENpiCRJwiWXXAKTyQSHw6HIe7pcLhiNRlx00UVsgwjgW1Bps9kU+8Xm9XrhcDiQkZHB79AQSJKEMWPGwOFwKDbELYSAzWbz60w9vx73NmPGDADKLeb78ssvAYCr8ofBN3PPd0rlaO3fvx/A34fcKPz5hq58Cx5Hy7cokLsgD51v1phSZ7z4Fk76a0gM8HO4aLVapKenKzJrTAiBnTt3Ii0tbVTHsEaaqKgojB8/HnV1dYq0we7du5GZmem3rjQFH41Gg5iYGEWGZYQQMBgM0Ol0EXuU8UjIsoy4uDh0d3cr0gadnZ2IiYnx67CkX1tXkiRcffXVsNvto07c3t5e2Gw2XHXVVexKD4OvDVwuF06cODGq9+ru7sbg4CBmz57NNoggkiQhKysLbrd71MOrdrsdHo8H48aN43doGCRJwvjx40+dYTUavvOe/L3pr99vHbKzsxEdHY0PP/xwxIkrhMCHH36I6OhonrE9ApmZmdDpdNi2bduo2mDbtm3QarWn9o+jyJGYmAhZlnHixIlRfYeOHz9+6i6chicuLg4ajWZUO54LIdDe3g5ZlpGYmKhwhV/n93CRJAmXXnopTpw4MeLEtdvtaG9vx0UXXcSu9AhIkoQrrrgCRqNxxCt9rVYrOjs7MXPmTN5xRiDfeguLxTLih8oulwt2ux1jx47ld2gEfD1I327kI+F2u2GxWJCZmen3NgjIb+qysjLIsjyi3osQAtu3b4csy9zafRQmTZqEqKgofPDBByNqg61bt0Kj0fBBfgTzPfwdSe/F12vx7VlGI+Nb8DiSM7NObwPf+iV/Cki4aDQaVFZW4vDhw8NejGWz2dDU1ISysjK/bFEQKTQaDWbNmoW2trZhL8ayWCxobm7GRRddxHUJEcw3LdlkMg17eYHL5YLZbEZmZiZHH0ZBlmWMGTMGZrMZTqdzWK91u90wmUwBmwIekFb2DY1FRUWhpqZmyIkrhEBtbS00Gg2HYxRQVlYGrVaLzZs3D6sN3nvvPURHR+OSSy5hG0QwSZJODWkNZ9zft1OHLMt+nfoaKXwB3draOuw2OL0N/S1gtxBRUVG4+uqr0d7ePuQtYdra2nDs2DFceeWV7LUoQKPR4LrrrkN3dzeampqG9MVsbW3F8ePHMXv2bLYBnRr3N5vNQxqFEEJgYGAAdrsd48ePZ69FAbIsIzs7GzabbciHuZnNZlgsloAeSR7Qlp46dSrS09Px3nvvXfAY5MHBQWzatAl6vR7Tpk3jHbNCiouLkZWVhS1btlxwgsXg4CA2b96M9PR0Hm9AAE6Gi16vR0xMDFpbWy/4cN/tdqOtrQ06nY5b6ysoNTUVOp0Ora2tFxyidLvdaGlpQWxsrN82qTybgIaLJElYtGgRPB4P1q1bd86/FLfbjXfffRcejweLFi3i3Y6CJEnCTTfdBEmSUF1dfc5xW7fbjXfeeQcejwff/va3Ge50iiRJyMvLg9frRUtLyzl3Pvd4PKfO/cnLy+N3SEGSJCE/Px9CCDQ3N5+zDbxeL44cOQIAKCgoCGgbBPy3dkJCAhYsWACDwYB169bBarWeGp4RQsBqteLdd99FZ2cn5s+f7/e52JFIp9Nh0aJF6O3tRXV1NSwWyzfaYN26dejq6sKCBQu4SSh9g1arRU5ODqxW66m759O/Q06nE83NzXA4HMjLy+OQqh9ER0cjPz8fNpsNzc3NcLlcX2sDl8uFI0eOwG63Iy8vL+C7akhCgd3oPB4Puru7kZWVNaSfF0LgyJEjqK2thVarxdSpU6HX69Hb24sDBw7A4XDguuuuQ3Fx8ZCT9sSJExgzZkzEzmbybeo51AWOQgi0tLRg8+bNiIqKwrRp06DX69HX14f6+no4HA7MnTt3WG1w/PhxjB07NmLbINR5vV7Y7XbEx8cP6eeFEDCZTGhvb0dUVNSprZkGBwdPnfuek5ODxMTEIX+HrFZrRG8NM5I26O/vx7Fjx6DRaE4NWfrawOv1IicnB8nJyUNuA4vFgri4uFG3gSrhApz8S+nt7cUnn3yC48ePw+VyndoHa9asWcM+RIjhMrxwAU62QV9fHz7++GO0t7d/ow2Gu2MqwyW0DfcXG/D3Q786OztPjULIsoyEhASMHTsWWq12WN8hhsvI26CjowNWqxVer/dUG2RlZSEmJmZYbRB04bJ///5TO3cOl9VqxeDgIGJjY4f1l3q6vr4+zJgxI2J/sY2mDXzbbw8ODiImJgbx8fEjGpuN9DYIdV6vFz09PSPeGNblcsHj8SAqKmrEw2AOhwN6vT6iw8VoNI7o+HAhBNxuN9xu96k2GMl1PDg4iPT09OAIF98dsNpSU1Mj9qEh24BGSwih2Lk/ozHcO+1wEk5toEi4jJbL5YLFYhlxz4dGz+FwwG63c7oojZhvx15OAFGP2+2G0+kMio1Bg6LvuXv3bqxevRqDg4NqlxKxtm/fjrVr155zSiPRhZhMJjQ3Nw97WxJSjm+BtFInVo5GUITL9OnT4fV6FTuxkoavvLwcAwMDOHz4sNqlUIgyGAxISkqCVqtVu5SI5Htek56eHhTPPYMiXOLj41FaWoq9e/cGReJGoszMTEyYMAF79uxRuxQKQWazGQ6Hg3uHqai3txcejydo2iAowgUAKisrYbFY0NTUpHYpEauyshKdnZ2jPrGSIo/BYEBcXNyIZ3vS6HV3dyMlJSVoeo5BEy56vR65ubm8c1ZRXl4eUlNT2QY0LHa7HRaLBenp6WqXErH6+/vhcDgCck7LUAVNuAAn75y7u7vR3t6udikRSZKkU+fuDPfMF4pcRqMR0dHRSE5OVruUiNXd3Y34+Pig6jkGVbjk5uZCr9fzzllFkyZNgk6nw969e9UuhUKAy+WCyWQa9m4OpBybzXbq6OJgElThApzsvTQ3NwfFgsBIFBUVhRkzZuDAgQOcGk4X1NPTw6OLVdbd3Y2YmJig6zkGXbiUlpYiLi4OdXV1apcSsWbMmAEhBOrr69UuhYKYb7sYvV4fFFNfI5HT6Qzo0cXDEXThotFoUFZWhoMHD17wQDHyj7i4OEyaNAn79u3j1HA6p76+Pni9Xuj1erVLiVgGgwGyLAdlGwRduAAnF1UCwBdffKFyJZGroqICFosFjY2NapdCQUgIAYPBgOTk5KCZ+hppPB4Penp6FNlk0h+CryKcPMxq8uTJ2Ldv3wWP8CT/0Ov1yMvL4+QKOiuz2Qyn08npxyryndcSLIsmzxSU4QKcvHO22WxoaGhQu5SIVVlZCYPBgLa2NrVLoSBjMBgQHx8fFBskRiJfzzE1NTXgJ0wOVdCGS2pqKgoKCnjnrKKcnBykp6ezDehrbDYbrFYrey0q6u/vh9PpDLrpx6cL2nABTt459/T0oLW1Ve1SIlZlZSWOHj2K3t5etUuhIGE0GqHVapGUlKR2KRGru7sbCQkJ0Ol0apdyTkEdLtnZ2cjMzOSds4omTpyI+Ph4Tg0nACenvvb39wfl1NdIYbVaYbVag7rXAgR5uAAn75xbW1thNBrVLiUi+aaGf/XVV5waTujp6YEsyzzYT0XBumjyTEEfLiUlJUhISOCds4p8U8P379+vciWkJt/U10g+415tDocDJpMp6HstQAiEiyzLKC8vx6FDh2C1WtUuJyLFxsZiypQp2L9/P6eGR7De3l4IIYJywV6kMBgMiIqKContdoI+XABg2rRpkGWZd84q8k0NP3TokNqlkAqEEDAajUhJSQnaqa/hLtgXTZ4p+CsEEBMTg6lTp/LOWUUpKSkoLCzk5IoI1d/fD5fLxenHKjIajRBCBO2iyTOFRLgAJ894dzgc+PLLL9UuJWJVVlait7cXLS0tapdCAWY0GoN+6ms48y2aTEtLQ1RUlNrlDEnIhEtycjKKiopQV1cHIYTa5USk8ePHY8yYMey9RBir1QqbzRYyd8zhqK+vDy6XKyQe5PuETLgAJ++cTSYTmpub1S4lYlVWVuLYsWMwGAxql0IBYjAYEBMTg8TERLVLiVjd3d1ISkpCbGys2qUMWUiFS1ZWFrKysjgtWUXFxcVITExkG0QIp9OJgYEB9lpUZLFYYLfbQ6rXAoRYuAAn75zb29vR1dWldikR6fSp4RaLRe1yyM98U19TUlLULiVidXV1QafThVzPMeTCpaioCMnJybxzVtHUqVOh0Wg4NTzMeTwe9PX1cdGkigYHBzEwMBByvRYgBMNFkiSUl5ejoaEBZrNZ7XIiUkxMDKZNm4YvvvgCLpdL7XLIT3p6erhoUmUGgwHR0dEhud1OyIULAEyZMgVarRb79u1Tu5SI5ZsafvDgQbVLIT8QQqCnpwepqakhM/U13LjdbvT29iI9PT0kNwkNyXDRarWYNm0a6uvr4XQ61S4nIiUlJaG4uJhTw8OUyWTiokmV+TbrDdU2CMlwAU7eObtcLi6qVBGnhocvg8GAxMTEkJr6Gk68Xm/ILZo8U8iGS0JCAkpKSlBXVwev16t2ORFp7NixGDduHBdVhhmLxYLBwUFOP1ZRX18f3G53SD7I9wnZcAFObqY4MDCAI0eOqF1KxKqsrMTx48c5NTyMGAwGxMbGIiEhQe1SIlZ3dzeSk5MRExOjdikjFtLhMmbMGGRnZ/POWUWFhYVITk5mG4QJh8MBs9nMXouKBgYGMDg4GNK9FiDEwwU4eefc0dGBjo4OtUuJSJIkoaKiAo2NjZwaHgZ8U1+5aFI93d3diIuLC/meY8iHS35+PlJTU3nnrCLf1PC9e/eqXQqNgtvtPrVoMhSnvoYDu90Os9kc8r0WIAzCxXfnfPjwYfT396tdTkSKjo7G9OnTOTU8xPX09ECSJC6aVFF3d3fY9BxDPlwAYPLkyYiJieGds4rKysrgdrtx4MABtUuhEfB6vacWTWo0GrXLiUgulwt9fX3IzMwMi55jWIRLVFQUZsyYgQMHDsDhcKhdTkRKSEjAxIkTsXfvXk4ND0EmkwlutztkF+yFA6PRGFY9x7AIFwCYMWMGvF4v6uvr1S4lYvmmhh8+fFjtUmiYDAYDkpKSQnrqayjzLZrU6/Vh03MMm3CJj49HaWkp75xVlJmZiQkTJnByRYgxm81wOBycfqyi3t5eeDyesHiQ7xM24QKcvHO2WCxobGxUu5SIVVFRgc7OTpw4cULtUmiIDAYDdDod4uPj1S4lYnV3dyMlJQVarVbtUhQTVuGSnp6O3Nxc3jmriFPDQ8vg4CAsFgt7LSrq7++Hw+EIq14LEGbhApxcVNnd3Y329na1S4lIp08NN5lMapdDF+BbNJmcnKx2KRGru7sb8fHxYddzDLtwyc3NhV6v552ziiZPnozY2FhODQ9yLpcLJpMpZM8LCQc2mw0WiyXsei1AGIYLcLL30tzcjL6+PrVLiUi+qeFffvklBgcH1S6HzsG3aDItLU3tUiJWd3c3tFptWPYcwzJcSktLERcXh7q6OrVLiVhlZWWcGh7EfIsm09LSwmbqa6hxOp0wmUxhs2jyTKF5Cs0FaDQalJWVYdeuXZg5c+aprmdCQgL3TQqQuLg4lJaWYt++fSgvL4fJZGIbqMh3bLGvDQDA4/Fw0WQAndkGg4ODkGU5bBZNniksey4AkJOTg+3bt6OkpAQZGRnIz89HRkYGiouLsWLFCj5sDoCioiJs3rwZBQUFbAOVmEwmrFixAsXFxV9rg4qKClRXV8Nms6ldYtg7VxvMnDkT69atw8DAgNol+ocIQzU1NSI+Pl4A+MY/kiQJSZJEfHy8qKmpUbvUsMU2UJ+vDXx/32drC7aBf0VyG4RduNTU1AiNRiNkWT5rQ/r+kWVZaDSasGxUtbEN1Mc2UF+kt4EkhBDK94fUYTKZkJ2dDbvdPqQtYGRZhk6nQ3t7e1hscR0M2AbqYxuoj20QZs9cXn31VdhstiHvLeb1emGz2bBy5Uo/VxY52AbqYxuoj20AhE3PRQiB4uJiNDc3Yzj/S5IkoaCgAE1NTZzBNEpsA/WxDdTHNjgpbMLFaDSOan8ko9EYtlMCA4VtoD62gfrYBieFzbCYxWIZ1evNZrNClUQutoH62AbqYxucFDbh4lsYNlKJiYkKVRK52AbqYxuoj21wUtiEi16vR2Fh4bDHKiVJQmFhIfdXUgDbQH1sA/WxDU4Km3CRJAkPP/zwiF67bNmysHiApja2gfrYBupjG5wUNg/0Ac4tDwZsA/WxDdTHNgijngsApKSkYO3atZAkCbJ8/v81WZYhSRKqq6vDpjGDAdtAfWwD9bENEN57i51tP5/T97Wqra1Vu9SwxTZQH9tAfZHcBmEZLkII0dfXJ1asWCEKCwu/1qCFhYVixYoVwmQyqV1i2GMbqI9toL5IbYOweuZyNkII9Pb2wmw2IzExEWlpaWHzwCxUsA3UxzZQX6S1QdiHCxERBV5YPdAnIqLgwHAhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLF/R96o8qByf2ilAAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "model.plot()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8230d562-2635-4adc-b566-06ac679b166a", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.7" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/tutorials/API_10_device.ipynb b/tutorials/API_10_device.ipynb new file mode 100644 index 00000000..99b43ede --- /dev/null +++ b/tutorials/API_10_device.ipynb @@ -0,0 +1,173 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "134e7f9d", + "metadata": {}, + "source": [ + "# Demo 10: Device\n", + "\n", + "All other demos have by default used device = 'cpu'. In case we want to use cuda, we should pass the device argument to model and dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "7a4ac1e1-84ba-4bc3-91b6-a776a5e7711c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cpu\n" + ] + } + ], + "source": [ + "from kan import KAN, create_dataset\n", + "import torch\n", + "\n", + "torch.use_deterministic_algorithms(False)\n", + "\n", + "#device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", + "device = 'cpu'\n", + "print(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "2075ef56", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "checkpoint directory created: ./model\n", + "saving model version 0.0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "| train_loss: 6.83e-01 | test_loss: 7.21e-01 | reg: 1.04e+03 | : 100%|█| 50/50 [00:19<00:00, 2.62it\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "saving model version 0.1\n" + ] + } + ], + "source": [ + "model = KAN(width=[4,100,100,100,1], grid=3, k=3, seed=0).to(device)\n", + "f = lambda x: torch.exp((torch.sin(torch.pi*(x[:,[0]]**2+x[:,[1]]**2))+torch.sin(torch.pi*(x[:,[2]]**2+x[:,[3]]**2)))/2)\n", + "dataset = create_dataset(f, n_var=4, train_num=1000, device=device)\n", + "\n", + "# train the model\n", + "#model.train(dataset, opt=\"LBFGS\", steps=20, lamb=1e-3, lamb_entropy=2.);\n", + "model.fit(dataset, opt=\"Adam\", lr=1e-3, steps=50, lamb=1e-3, lamb_entropy=5., update_grid=False);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2f182cc1-51bf-4151-a253-a52fe854919e", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "f6f8125e-d26d-4c97-9e5f-988099bb4737", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cuda\n" + ] + } + ], + "source": [ + "device = 'cuda'\n", + "print(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "95017dfa-3a2a-43e0-8b68-fb220ca5abc9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "checkpoint directory created: ./model\n", + "saving model version 0.0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "| train_loss: 6.83e-01 | test_loss: 7.21e-01 | reg: 1.04e+03 | : 100%|█| 50/50 [00:01<00:00, 26.90it\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "saving model version 0.1\n" + ] + } + ], + "source": [ + "model = KAN(width=[4,100,100,100,1], grid=3, k=3, seed=0).to(device)\n", + "f = lambda x: torch.exp((torch.sin(torch.pi*(x[:,[0]]**2+x[:,[1]]**2))+torch.sin(torch.pi*(x[:,[2]]**2+x[:,[3]]**2)))/2)\n", + "dataset = create_dataset(f, n_var=4, train_num=1000, device=device)\n", + "\n", + "# train the model\n", + "#model.train(dataset, opt=\"LBFGS\", steps=20, lamb=1e-3, lamb_entropy=2.);\n", + "model.fit(dataset, opt=\"Adam\", lr=1e-3, steps=50, lamb=1e-3, lamb_entropy=5., update_grid=False);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8230d562-2635-4adc-b566-06ac679b166a", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tutorials/Unchecked_API_10_device.ipynb b/tutorials/Unchecked_API_10_device.ipynb deleted file mode 100644 index cbac3cf4..00000000 --- a/tutorials/Unchecked_API_10_device.ipynb +++ /dev/null @@ -1,110 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "134e7f9d", - "metadata": {}, - "source": [ - "# Demo 10: Device\n", - "\n", - "All other demos have by default used device = 'cpu'. In case we want to use cuda, we should pass the device argument to model and dataset." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "7a4ac1e1-84ba-4bc3-91b6-a776a5e7711c", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "cuda\n" - ] - } - ], - "source": [ - "from kan import KAN, create_dataset\n", - "import torch\n", - "\n", - "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", - "print(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "2075ef56", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "train loss: 5.78e-03 | test loss: 5.89e-03 | reg: 7.32e+00 : 100%|██| 50/50 [00:26<00:00, 1.85it/s]\n" - ] - } - ], - "source": [ - "model = KAN(width=[4,2,1,1], grid=3, k=3, seed=0, device=device)\n", - "f = lambda x: torch.exp((torch.sin(torch.pi*(x[:,[0]]**2+x[:,[1]]**2))+torch.sin(torch.pi*(x[:,[2]]**2+x[:,[3]]**2)))/2)\n", - "dataset = create_dataset(f, n_var=4, train_num=3000, device=device)\n", - "\n", - "# train the model\n", - "#model.train(dataset, opt=\"LBFGS\", steps=20, lamb=1e-3, lamb_entropy=2.);\n", - "model.train(dataset, opt=\"LBFGS\", steps=50, lamb=5e-5, lamb_entropy=2.);" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "3acdcdee-71ca-42a1-98aa-7f7df4a29077", - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAHiCAYAAAAkiYF/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGtElEQVR4nO3deXyU9Z0H8M/zTDLJ5E4mCQRC7oRw5/ACL7QIiiBVtEVxPVrXVldp66utblvRaNd17bqK22O3ddeKIGolIpcJCmKpBwgBDCJJICQkkGMmySRzZc7f/sEOReTI8cw8c3zer5cvXy/JzHzlN08+z+/3/A5JCCFARESkIFntAoiIKPwwXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUlyU2gUQhQIhBHp6emCxWJCQkAC9Xg9JktQuiyhosedCdB4mkwkrVqxAcXExMjIykJ+fj4yMDBQXF2PFihUwmUxql0gUlCSeREl0drW1tVi8eDFsNhuAk70XH1+vJS4uDmvXrsW8efNUqZEoWDFciM6itrYWN954I4QQ8Hq95/w5WZYhSRI2bdrEgCE6DcOF6AwmkwnZ2dmw2+3nDRYfWZah0+nQ3t6OlJQU/xdIFAL4zIXoDK+++ipsNtuQggUAvF4vbDYbVq5c6efKiEIHey5EpxFCoLi4GM3NzRjOpSFJEgoKCtDU1MRZZERguBB9jdFoREZGxqher9frFayIKDRxWIzoNBaLZVSvN5vNClVCFNoYLkSnSUhIGNXrExMTFaqEKLQxXIhOo9frUVhYOOznJpIkobCwEGlpaX6qjCi0MFyITiNJEh5++OERvXbZsmV8mE/0//hAn+gMXOdCNHrsuRCdISUlBWvXroUkSZDl818ivhX61dXVDBai0zBciM5i3rx52LRpE3Q6HSRJ+sZwl++/6XQ6bN68GXPnzlWpUqLgxHAhOod58+ahvb0dL774IgoKCr72ZwUFBXjxxRdx/PhxBgvRWfCZC9EQCCHw0Ucf4frrr0dNTQ2uvvpqPrwnOg/2XIiGQJIkpKSkfO3fRHRuDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSnCSEEGoXQRQIQgi0tbWN+PVerxcOhwMxMTGQ5ZHfl02YMAGSJI349UShIErtAogCxeVyYe3atSgoKBjxewghRhQMXq8XjY2NiIqKwsMPPwytVjviGohCAcOFIsrMmTNx2WWXBfxza2pq8Jvf/Ab/+7//G/DPJlIDn7kQ+ZkQAj/+8Y+RlpaG9PR0tcshCgj2XIj8rK6uDq2trVi7di2ftVDEYM+FyI+EEHjggQeg1+tx/fXXq10OUcCw50LkR/X19aivr8crr7wyqhlmRKGG33YiPxFC4Pvf/z70ej1uu+02tcshCij2XIj85LPPPsOBAwewatUqaDQatcshCij2XIj8wOv14p577kF2djZuvvlmtcshCjj2XIj8YNWqVTh27Bg++OADPmuhiMRvPZHCLBYLHnnkEcyaNQuzZs1SuxwiVTBciBQkhMCyZcvgcDiwcuVKrmuhiMVwIVLQrl278MYbb+DRRx/FuHHj1C6HSDUMFyKF2O12LFmyBLm5uXjsscfYa6GIxgf6RAoQQuChhx6CwWDA559/jqgoXloU2XgFEI2SEALV1dV4/fXX8dRTT6G0tFTtkohUx2ExolFqbm7GfffdhyuuuAKPPPIIh8OIwHAhGpX+/n7ccMMNSE5Oxl/+8heuxCf6fxwWIxohh8OBm2++GQaDATt27EBKSoraJREFDYYL0Qi43W7ce++92LVrF9asWYMpU6aoXRJRUGG4EA2T2+3GD3/4Q7z77rtYsWIFFixYwOcsRGfgMxeiYXA6nbj//vuxZs0aPPPMM/j+97/PYCE6C/ZciIZoYGAA99xzD7Zs2YJnn30WDz30EIOF6BwYLkQXIITA0aNHcfvtt6OhoQF//OMfcfvttzNYiM6D4UJ0Hl6vFxs2bMA//dM/QaPRYP369bjyyisZLEQXwGcuRGchhEBPTw+WLVuGpUuXYuLEidixYweDhWiIGC5EZ3C5XKiursbll1+O119/HY899hg2bdqEnJwcBgvREHFYjOj/eTwe7Nq1C08//TT++te/4uKLL8bq1atRUVHBUCEaJoYLRTQhBFwuF3bu3IkVK1bg/fffx7hx4/C73/0OS5YsQUxMjNolEoUkhgtFJCEEurq6UFNTg1deeQV1dXXIzs7GU089hbvuugspKSnsrRCNAsOFIorL5cLatWuxbt06fPTRR+jv70dZWRl++9vfYtGiRUhOTmaoECmA4UIRxeFw4N5770Vubi6++93v4tZbb0V5eTmio6MZKkQKkoQQQu0iiALB6XTiv//7v9Hb24upU6ciPj4+4IHS0NCAH/7wh9BqtQH9XKJAY7hQxBBCoLGxUdUeihACJSUl7CVR2GO4EA2REAJCCEiSxHAgugAuoiQaov379yM+Ph779+9XuxSioMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiGQAiBvr6+r/2biM6N4UJ0HiaTCStWrEBxcTGuvfZaOBwOXHvttSguLsaKFStgMpnULpEoKEmCt2BEZ1VbW4vFixfDZrMBwNd6K5IkAQDi4uKwdu1azJs3T5UaiYIVw4XoLGpra3HjjTdCCAGv13vOn5NlGZIkYdOmTQwYotMwXIjOYDKZkJ2dDbvdft5g8ZFlGTqdDu3t7UhJSfF/gUQhgM9ciM7w6quvwmazDSlYAMDr9cJms2HlypV+rowodLDnQnQaIQSKi4vR3Nw8rBlhkiShoKAATU1Np57HEEUyhgvRaYxGIzIyMkb1er1er2BFRKGJw2JEp7FYLKN6vdlsVqgSotDGcCE6TUJCwqhen5iYqFAlRKGN4UJ0Gp1Oh7S0tBG9trCwcMSvJQo3DBciAB6PB2vWrMHMmTPh8XhG9B4zZ85EX1+fwpURhSaGC0W8HTt2YM6cOfjRj36Eyy67DDt27EB8fDxkeWiXhyRJiI2NRU5ODp577jls2rQJdrvdz1UTBTfOFqOI1djYiKqqKrz//vu46KKL8PTTT6OyshLA8Ffob968GbNnz8ZHH32EDz/8ENHR0Zg7dy4uu+wyaDSaQP0vEQUNhgtFHKPRiOeeew6vvfYaxo8fj+XLl2PhwoXfWJ8y1L3FqqurMXfu3FN/NjAwgJqaGuzevRvp6em48cYbMXnyZK5/oYjCcKGI4XA48Mc//hEvvPACZFnGI488gvvuuw9arfacrzGZTFi5ciVeeuklHDly5NR/LywsxLJly3D33XcjOTn5rK89ceIENm7ciKamJhQWFmLhwoUYP3684v9fRMGI4UJhTwiBd955B7/+9a/R2dmJe+65Bz/96U+HNbNLCIHe3l6YzWYkJiYiLS1tSD0RIQQOHTqEjRs3wmAwoLKyEtdff/05A4koXDBcKKzt2rULy5cvR11dHa6//no88cQTKCwsDHgdXq8XO3fuRG1tLZxOJ66++mrMnj0bMTExAa+FKBAYLhSWWlpa8NRTT2Hjxo2YPn06qqqqcPnll6tdFgYHB7Ft2zbs2LEDOp0O8+bNw8UXXzzkmWlEoYLhQmHFZDLhP/7jP/A///M/yMjIwC9/+UssXrw46H559/X14b333sPevXuRlZWFBQsWoKSkRO2yiBTDcKGw4HQ68corr+D555+H0+nEj370I/zwhz+ETqdTu7Tzamtrw/r169HS0oKJEydi4cKFGDNmjNplEY0aw4VCmhAC7733HqqqqtDa2oqlS5fi0UcfRWZmptqlDZkQAgcOHMCmTZvQ29uLSy+9FHPnzuU+ZRTSGC4Usvbt24fly5fjs88+wzXXXIOqqiqUlpaqXdaIeTwefPzxx/jggw/g9XpxzTXX4KqrrkJ0dLTapRENG8OFQk57ezueeeYZvP322ygtLUVVVRWuueYatctSjM1mwwcffIBPPvkEiYmJuOGGG1BeXs5FmBRSGC4UMsxmM1566SX813/9F5KSkvDYY4/h9ttvR1RUlNql+YXRaMTmzZtRX1+P7OxsLFy4EAUFBWqXRTQkDBcKem63G6tXr8a//du/wWKx4MEHH8RDDz006rNXQsXRo0exYcMGtLW1YerUqbjxxhuRnp6udllE58VwoaAlhMDWrVtRVVWFhoYGfOc738EvfvELjBs3Tu3SAk4IgX379mHz5s0YGBjArFmzcN111yEuLk7t0ojOiuFCQengwYNYvnw5/vrXv+Lyyy9HVVUVpk+frnZZqnO5XPjb3/6GrVu3QpZlzJkzB7NmzQrboUEKXQwXCipdXV149tln8frrr6OgoABPPPEE5s2bx4fZZ7BYLNiyZQt27tyJ1NRUzJ8/H9OmTePfEwUNhgsFBZvNht///vf47W9/i5iYGPz85z/HXXfdxWm4F9DV1YVNmzbhq6++Ql5eHhYuXIicnBy1yyJiuJC6vF4v3nrrLTzzzDPo7e3Ffffdh5/85CfcNXiYmpqasGHDBnR0dKCsrAzz589Hamqq2mVRBGO4kGp27NiBJ554AgcOHMCiRYvw+OOP8657FLxeL/bs2YOamhrYbDZcccUV+Na3voXY2Fi1S6MIxHChgGtqakJVVRW2bNmCyspKPP3007jooovULitsOBwOfPTRR9i+fTu0Wi3mzp2LSy+9lMctU0AxXChgenp68Nxzz2HlypUYP348Hn/8cdx00018CO0n/f39qK2txe7du5GRkYEFCxagtLSUf98UEAwX8jvf8cIvvvgiJEka0vHCpJzTj1suKirCggULeNwy+R3DhfxGieOFSRk8bpkCjeFCfnHm8cLLly9HUVGR2mVFPI/Hg507d2LLli08bpn8iuFCimppacHTTz+NDRs2BNXxwvR1Zx63fP311+Oiiy4KuhM7KXQxXEgRJpMJL7zwAl5++WWkp6fjl7/8JW699Vb+sgpyPG6Z/IXhQqPidDrx5z//Gc8//zwcDgeWLVuGBx54IOiPF6avO3bsGDZs2ICWlhaUlpZiwYIFPG6ZRoXhQiMSDscL09fxuGVSEsOFhm3fvn144okn8Omnn+Kaa67Bk08+iUmTJqldFinE7Xbjk08+4XHLNCoMFxqy048XnjhxIqqqqnDttdeqXRb5ie+45Y8//hhJSUk8bpmGheFCF2SxWPDSSy/hD3/4AxITE/HYY4/hjjvu4BkiEcJoNGLTpk04cOAAsrOzcdNNNyE/P1/tsijIMVzonE4/XthsNuPBBx/Eww8/HDHHC9PXNTc3Y8OGDWhvb+dxy3RBDBf6BiEEtm3bhieffBINDQ247bbb8Itf/IJbhhCEENi7dy/ee+89DAwM4PLLL8ecOXN43DJ9A8OFvubgwYN44okn8NFHH2HWrFmoqqrCjBkz1C6LgozL5cKOHTuwbds2HrdMZ8VwIQB/P154zZo1yMvLw5NPPsnjhemCzGYz3n//fXz22WdIS0vjcct0CsMlwp15vPBPf/pT3H333dyxmIalq6sLGzduxKFDh3jcMgFguEQsr9eLv/zlL/iXf/mXU8cL//jHP0ZKSorapVEIa2xsxMaNG9HR0YHy8nLccMMNPG45QjFcItDf/vY3PPHEE6ivr8eiRYvwq1/9Crm5uWqXRWHC6/Vi9+7dqKmpgd1ux5VXXolrr72Wxy1HGIZLBGlqasJTTz2F2tpaVFZW4qmnnsLFF1+sdlkUps523PJll13GzUwjBMMlAvT09OA3v/kNXn31VYwbNw6PP/44Fi1axIeuFBD9/f2oqanBnj17eNxyBGG4hDGHw4E//elPeOGFFyBJEn7yk5/gvvvu48FQpIoTJ05gw4YNOHz4MIqKirBw4UKMGzdO7bLITxguYUgIgXXr1uHXv/41Tpw4gXvuuQc/+9nPeLwwqY7HLUcOhkuY+fzzz7F8+XLs2bOHxwtT0DrzuOXZs2dj9uzZnAIfRhguYaK1tRVPP/001q9fj2nTpqGqqgpXXHGF2mURndfpxy3HxcXh+uuvR2VlJR/6hwGGS4gzmUx48cUX8fLLL0Ov1/N4YQpJfX192Lx5M/bt24esrCwsXLgQxcXFapdFo8BwCVFOpxOvvvoq/v3f/x0OhwMPP/wwHnzwQR4vTCGNxy2HD4ZLiBFCoKamBlVVVWhpacEdd9yBRx99lBcghQ0hBOrr67F582b09fWdOm6ZRz2EFoZLCDn9eOHZs2ejqqqKxwtT2DrzuOVrr70WV155JY9bDhEMlxDg9XqxbNkyvPXWWzxemCLOmcct33777SgoKFC7LLoAhkuACCHQ3t4+4tdbrVZoNJpR78+UnZ3NldGkCiEETCbTiF9vt9vR3NyMgoKCUT1bTElJ4TUQADzZJ0BcLhfWr1+PvLy8c/6M0+lER0cHUlJS/LKo7OjRo7j//vu5loBU4fF4sH//fuj1+nP+jNPphNlsRmJi4lm/p8nJyejp6RlxDUajEVdeeSUPNQsA/g0H0CWXXHLWjSJ9q5YfeeQRHDp0CKmpqXjsscdw2223KXqHtWvXLsXei2gk8vLyznqDJYTA0aNHUVtbi76+PqSmpuK2225DTk6OotfA0aNHFXsvOj8uhggC7e3tWLRoEbq6urB8+XIUFxfjRz/6Ed5//31w1JIiQUdHB15++WVIkoQFCxZACIE//vGPoxpGI3UxXFTm8XiwZMkSSJKEzZs345577sFrr72G6dOn4wc/+AGsVqvaJRL5ldvtxssvv4z4+Hg8+OCDmDlzJh588EHIsoxXXnmFN1ghiuGispUrV6KpqQmvvPIKxo4dC0mSoNVqsXLlSgwODuLxxx/nxUVhSwiBbdu2wWw24/vf//6pB/Xx8fFYsmQJOjo60NjYqHKVNBIMFxU5nU5UVVVh1qxZmDlz5tf+LDMzE3fffTfWrFmDgYEBlSok8i+3241t27Zh6tSp31gIPHnyZKSlpeHtt9/mDVYIYrio6LXXXoPNZsN//ud/fuOhpSRJ+NWvfgVJkvDss8+qVCGRf3300UfweDxYvHjxWa+BxYsXw2Qy4fjx4ypVSCPFcFGJ1+vFs88+i/LycmRnZ5/1Z+Lj43HDDTfgtddeg8vlCnCFRP7l9Xrx4YcforCwEPHx8Wf9maKiIsTExGDdunXsvYQYhotKdu/ejf7+fjz77LPnnGopSRKqqqrgdDrx3nvvBbhCIv9qbm6G0+nEt7/97XNeA7IsY/bs2WhtbYXD4QhwhTQaDBcVCCHw5JNPIjk5GWVlZef92ezsbGRnZ+Ppp5/mnRuFDSEENm3ahLi4uAtuuuo7l+izzz4LRGmkEIaLCux2O/bs2YP777//ggvEJEnCo48+itbW1lGtTCYKJi6XC+3t7bjqqqsueA3ExMQgKysL27dv5w1WCGG4qOCdd96BEAL/+I//OKSfX7RoEWRZxu9//3s/V0YUGPv27QMAzJo164I/K0kSbrjhBlitVvT39/u5MlIKwyXAhBB48cUXMWHChCHvHxYTE4OLL74Yr776Ku/cKOQJIbB161akpKQMeSPWkpISSJKE7du3+7c4UgzDJcBsNhtaW1vx0EMPDXnPJEmS8Itf/AJms5l7I1HIc7lc6O3txezZs4d8DciyjIKCAnz++ee8wQoRDJcAW7duHQDgtttuG9brLrnkEkRHR+P555/3Q1VEgVNfXw8AqKioGPJrJEnCvHnz4HQ6+ewxRDBcAux3v/sdsrKyzjmv/1w0Gg3mzJmDDRs2wOv1+qk6Iv/bvn07EhIShn02UW5uLmRZxrZt2/xUGSmJ4RJALpcLR44cwfe+970RbSP+85//HIODgzh48KAfqiPyP6/Xi87OTlx66aXDvgYkSUJxcTH27dvHobEQwHAJIIfDgdtvvx133HHHiF4/adIk6HQ6PPfccwpXRhQYTqcTpaWluOyyy4b9WkmSMHfuXLhcLhgMBj9UR0piuARQQkICXnjhBaSnp4/o9bIs46abbsLWrVvh8XgUro7I/2JjY3HvvfeO+KTV7OxsaDQabN26VeHKSGkMlwCTJGlUJ+s98sgjcLlc2Llzp4JVEQWOLMsjvgZkWUZpaSm++OKLYT17FELwhizAGC4hJi8vD0lJSXjmmWc47kwRae7cuXC73ejo6Bjya3p6evCHP/wBg4ODfqyMTsdwCTGSJOGee+7B7t27h7yRnxCCuypT2Bg7diyio6NRW1s75Busv/71rzh27BhiYmL8XB35MFxC0AMPPACv14vq6uoh/XxraysWLVrE88gpLMiyjIqKCjQ0NAxpaEwIgbq6OuTn549qSJqGh+ESgtLS0pCfn4/nnntuSHduv/nNb7B//34kJiYGoDoi/5szZw68Xu+QpuWbTCY4HA7MmTMnAJWRD8MlBEmShOXLl+PEiRNobW0978+63W6sX78e8+bNg0ajCVCFRP6VnJyMpKQkbNq06YI3WB988MGp7WMocBguIWrevHmIjY3F8uXLz3tx1dbWwuFw4PHHHw9gdUT+JUkS5s+fj56envMO93o8HtTV1WHSpEmQZf66CyT+bYeoqKgofO9738OWLVswMDBw1p8RQmD58uXIyclBXl5eYAsk8rOysjJERUXh3XffPecN1hdffAG3240FCxbweUuAMVxC2E9/+lPIsnzO3sunn36KtrY2/Ou//isvLAo7Go0GV155JQ4ePAiLxfKNP/d6vVi/fj0yMzOh1+tVqDCyMVxCWHx8PO699168+eab33j24nQ68YMf/AC5ubn41re+pVKFRP41Z84caDQavPHGG1+7wRJC4LPPPoPFYsGSJUt4c6UChksIkyQJv/rVr5CcnIylS5fCarUCODnO/M///M8wGAz485//zLFmClvR0dFYuHAhGhsbsWfPHgghIIRAd3c3NmzYgNLSUmRnZ6tdZkSKUrsAGh2dTofXXnsNt9xyC2655Rbcf//92Lp1K9auXYuf/exnmDx5stolEvmNJEm47LLL8NVXX+Evf/kLenp6kJSUhNraWiQkJGDp0qXstaiE4RIGLr74YqxcuRK//OUv8ZOf/ASJiYl4/PHH8eCDD/LCorAnyzLuuusuvPPOO9ixYwe8Xi8mTJiAJUuWDPvMGFKOJLhBVUA4nU688sorKCoq8ttnWK1WtLW1IT09HXq9/hvB0tTUhO9973vQarV+q4HoXNxuNz799FNkZGT45f2FEDCZTHC73UhLSzvruq7u7m7MmjULUVG8r/Y3hkuACCFw+PBhVXsSQggUFRWxN0OqEEIExTksGRkZvAYCgOESInxbhms0Gl4YFLE8Hg9sNhvi4uK440SQ4zSiEFFfX49x48ahvr5e7VKIVNPZ2YmnnnoKnZ2dapdCF8BwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXEKAEAJ9fX3wer3o6+uDEELtkogCTgiB3t5e9Pf3o7e3l9dBkGO4BDGTyYQVK1aguLgYs2fPRk9PD2bPno3i4mKsWLECJpNJ7RKJ/O7062D69On405/+hOnTp/M6CHKSYPwHpdraWixevBg2mw0AvnaXJkkSACAuLg5r167FvHnzVKmRyN94HYQuhksQqq2txY033gghBLxe7zl/TpZlSJKETZs28cKisMPrILQxXIKMyWRCdnY27Hb7eS8oH1mWodPp0N7ejpSUFP8XSBQAvA5CH5+5BJlXX30VNpttSBcUAHi9XthsNqxcudLPlREFDq+D0MeeSxARQqC4uBjNzc3DmgkjSRIKCgrQ1NR0ahyaKFTxOggPDJcgYjQakZGRMarX6/V6BSsiCjxeB+GBw2JBxGKxjOr1ZrNZoUqI1MPrIDwwXIJIQkLCqF6fmJioUCVE6uF1EB4YLkFEr9ejsLBw2OPFkiShsLAQaWlpfqqMKHB4HYQHhksQkSQJDzzwwIi2tVi2bBkfYlJYkCQJDz/88Ihey+sgeDBcgkhnZye0Wi20Wu2QLxBZlhEXF4e77rrLz9URBc7dd9+NuLg4yPLQfkVJkoTY2Fj8wz/8g58ro6FiuASJPXv24M0330R6ejrWrFkDWZYveGH5ViZXV1dz4RiFlZSUFKxduxaSJF3wRst3HTz//PPo7u6G0+kMUJV0PpyKrLLBwUHU1taiubkZlZWVuPzyy6HRaM67p5JPfHw8qqurMXfu3ECXTRQQ7733HhYvXgy73Q5Jks65t1h1dTWuuOIKtLS0wO12IycnhzdcKmO4qOjEiRPYvHkzXC4X5s2bh4KCgq/9uclkwsqVK/HSSy/hyJEjp/57VlYWrrrqKvzhD39AampqoMsmCpienh589dVX2LVrF37/+99/7TooLCzEsmXLcPfddyM5ORkA4PF4cOzYMZhMJmRkZGDcuHFDHlojZTFcVCCEwO7du/Hxxx8jKysL8+fPP+/0Sd85FmazGYmJifB6vVi1ahVuuOEGlJaWBrByosBqaGhAbGwscnNzv3EdpKWlnXPIzGg0or29HbGxscjPz0dMTEyAKyeGS4DZ7XbU1NSgpaUFF198MWbNmjWiO6vq6mrY7XYsXbrUD1USqW9gYAAtLS0oKipCXFzcsF9vt9tx9OhRuFwu5OTksJcfYOwvBtDx48exatUqdHV14eabb8YVV1wx4i57ZWUluru70d7ernCVRMHBYDAgLi5uRMECADqdDqWlpUhOTkZLSwva2tqGvBEmjV6U2gVEAiEEdu3ahU8//RTjxo3D/PnzR70KOTc3F3q9Hnv27EF2drZClRIFB7vdDqvVitzc3FG9jyzLyMvLQ2JiItrb22G1WpGXl4fY2FiFKqVzYc/Fz2w2G6qrq/HJJ5/gkksuwa233jrqYPGprKxEc3Mz+vr6FHk/omBhMBig1WqRlJSkyPvp9XqUlJRACIGGhgb09vYq8r50bgwXP2pra8OqVatgNBqxePHiET9fOZfS0lLExcWhrq5OsfckUpvL5UJ/fz/S09MVXW2v0+kwceJEpKSkoLW1FceOHeMwmR8xXPxACIFPP/0Ub7/9NtLS0nDnnXciJydH8c/RaDQoKyvDwYMHYbfbFX9/IjUYjUbIsuyXPcJkWUZubi5yc3PR19eHhoYGDA4OKv45xHBRnMViwdtvv42dO3di1qxZWLx4MeLj4/32edOnTwcAfPHFF377DKJA8Xq96O3tRVpaml/Xp6SlpWHixIkATk537unp8dtnRSqGi4JaW1uxatUq9PX14dZbb8Wll17q9030dDodJk+ejH379sHtdvv1s4j8rbe3F16vF+np6X7/rNjYWEycOBGpqak4duwYWltbOUymIIaLArxeLz7++GNUV1cjMzMTd955Z0BncFVUVMBms6GhoSFgn0mkNCEEjEYjkpOTER0dHZDPlGUZOTk5yMvLg8lkwqFDhzjErBCGyyj5hsE+//xzXHHFFbj55ptHPC9/pFJTU1FQUIA9e/YE9HOJlDQwMACn0zmqI45HKjU1FaWlpZBlGQ0NDTAajQGvIdwwXEbh6NGjeO2119Df34/bbrsNF198sWpnSVRWVqKnpwetra2qfD7RaBkMBiQkJECn06ny+TExMSgpKYFer0dbWxtaWlrg8XhUqSUccBHlCPiGwXbv3o38/HzMmzdPtQvCJzs7G5mZmdizZ8+oF54RBZrNZoPNZkNeXp6qdciyjAkTJiAhIQHHjh2DzWZDfn6+6td3KGLPZZgGBgbw1ltvoa6uDldddRUWLVoUNF+8yspKtLa2cuYLhRyDwYCYmBjFFk2Olm+YTKPRoKGhAQaDQe2SQg7DZRiOHDmC1atXw2q14jvf+Q4qKyuD6kjVkpISJCQk8NkLhRSn03lq0WQw8Q2Tpaeno729HUePHuUw2TAwXIbA4/Fg+/btWL9+PcaPH4+lS5ciKytL7bK+QZZllJeX49ChQ7BarWqXQzQkRqMRGo0mKHctliQJ2dnZyM/Ph9lsxqFDh04d4Efnx3C5gP7+frz55pvYv38/Zs+ejZtuuimoN72bNm0aZFnG/v371S6F6II8Hg96e3uRnp4e1Id6paSkoLS0FFFRUWhsbER3d7faJQW94G3NINDU1IRVq1ZhcHAQS5YsQXl5udolXVBMTAymTp2K/fv3c1ElBb3e3l4IIaDX69Uu5YK0Wi1KSkqQkZGB48ePo7m5mcNk58FwOQu3241t27Zh48aNyM3NxdKlSzFmzBi1yxqy8vJyOBwOHDx4UO1SiM7Jt2gyNTUVUVGhMXFVkiSMHz8eBQUFsFgsHII+D4bLGUwmE958800cOHAA1157LRYsWBByR6QmJyejqKgIdXV14EGjFKz6+/vhcrmC7kH+UCQnJ6O0tBTR0dFoamriMNlZMFxO09DQgNWrV8PpdOL222/HjBkz1C5pxCorK9HX14ejR4+qXQrRWRkMBiQmJgb1M8zz0Wq1KC4uRmZmJo4fP44jR45wKPo0DBecHAb74IMPsHnzZuTn52Pp0qWqbEGhpKysLGRlZXFaMgUli8UCu90ekr2W00mShHHjxqGwsBA2mw2HDh2CxWJRu6ygEPHh0tvbizVr1uCrr77CnDlzMH/+fGi1WrXLUkRlZSXa29vZZaegYzQaERsbi8TERLVLUURSUhJKS0sRExODw4cPo6urS+2SVBfR4fLVV1/h9ddfh9frxe23345p06apXZKiCgsLkZSUxN4LBRWHw4GBgYGQHx04U3R0NIqKijBmzBicOHEChw8fjuhhsogMF5fLhS1btqCmpgbFxcW44447Qr57fjayLKOiogKNjY3sqlPQMBqNiIqKQkpKitqlKE6SJGRlZaGoqAh2uz2ih8kiLlx6enqwZs0aNDQ0YO7cuZg3b17Azo5Qw5QpUxAdHY29e/eqXQoR3G73qUWTwbR1ktISExNPDZM1NTWhs7Mz4mZuRlS4fPnll3j99dcBAHfccQemTJmickX+p9VqMW3aNNTX18PpdKpdDkW4np4eSJKEtLQ0tUvxO98wWVZWFjo6OnDkyBG4XC61ywqYiAgXl8uFmpoabNmyBaWlpbjjjjtCYkWwUsrKyuByufDll1+qXQpFMCEEenp6QmrR5GhJkoSxY8eiuLgYg4ODOHToEMxms9plBUTYh4vRaMTq1atx+PBh3HDDDbjuuusi5ovtk5iYiJKSEuzduzfiuuYUPPr6+uB2u8Py+eaFJCQkoLS0FDqdDocPH0ZHR0fYX4thHS5ffPEFXn/9dURFRWHp0qUoLS1VuyTVVFRUoL+/H4cPH1a7FIpQRqMRSUlJIbfjhVKioqJQVFSEcePGobOzE4cPHw7rYbKwDBen04nNmzdj69atmDJlCpYsWRKU23kH0pgxY5Cdnc1pyaQKs9mMwcHBsJt+PBJjxoxBcXExHA4HDh06hIGBAbVL8ouwC5fu7m6sXr0aR48exY033ohvfetbETcMdi4VFRXo6OhAR0eH2qVQhDEYDNDpdIiPj1e7lKDgGyaLi4vDkSNHcOLEibAbJgurcNm3bx/eeOMNaLVaLF26FCUlJWqXFFQKCgqQkpLC3gsF1ODgICwWC3stZ4iKikJhYSHGjx+P7u5uNDU1hdWMzrAIF4fDgY0bN+LDDz/EtGnTsGTJkrBcoDVakiShoqIChw8fRn9/v9rlUIQwGAyIjo5GcnKy2qUEpczMTBQXF8PlcuHQoUNhc22GfLh0dXVh9erVOHbsGBYuXIhrrrkGGo1G7bKC1pQpUxATE8NFlRQQLpcLJpMp7BdNjlZ8fDxKS0uRkJCA5uZmHD9+POSHyUI6XOrq6vDGG28gNjYWS5cuRVFRkdolBb2oqCjMmDEDBw4cgMPhULscCnORtGhytDQaDQoKCjB+/HgYDAY0NjaG9DBZSIbL4OAg1q9fj48++gjl5eX47ne/yy73MMyYMQNerxf19fVql0JhzOv1oqenB2lpaRxNGIbMzEyUlJTA7Xbj0KFDMJlMapc0IiEXLh0dHVi1ahWOHz+ORYsW4aqrruIXd5h8XfC9e/fC6/WqXQ6Fqb6+Png8nohcNDlacXFxKC0tRWJiIo4ePYr29vaQGyYLmXARQmD37t146623kJCQgDvvvBMFBQVqlxWyKioqYLFY0NjYqHYpFKaMRiOSk5PD5nykQNNoNMjPz0d2djaMRiMaGxtDaig7JMLFbrfj3XffxY4dO1BRUYHvfOc7YXPIkFrS09ORm5vLacnkFwMDA3A4HJx+rICMjAyUlJTA4/Hg0KFD6OvrU7ukIQn6cDl+/DhWrVqFzs5O3Hzzzbjyyishy0FfdkiorKxEd3c32tvb1S6FwozBYEBcXBzi4uLULiUsxMXFYeLEiUhOTkZLSwva2tqCfkg7aJeuCyHw+eef45NPPsG4ceMwf/58JCQkqF1WWMnNzYVer8eePXuQnZ2tdjkUJux2O6xWK3Jzc9UuJaxoNBrk5eUhISEB7e3tsFqtyM/PD9q92oKyC2Cz2fDOO+/g448/xiWXXIJbb72VweInlZWVaG5uDpmuNgU/g8EArVaLpKQktUsJS+np6Zg4cSK8Xm9QD5MFXbi0tbVh1apVMBgMuOWWWzBr1iwOg/nRxIkTERcXh7q6OrVLoTDgcrnQ39/PRZN+ptPpUFpaipSUFLS0tODYsWNBN0wWNL+1hRD47LPPsHbtWqSlpeHOO+9ktzoAoqKiUFZWhoMHD8Jut6tdDoU4o9EIWZa5aDIAZFlGbm4ucnJy0NfXh4aGBgwODqpd1ilBES5WqxVr167FZ599hssuuwyLFy/m7qkBNH36dAAnz78hGimv14ve3l6kpaVxtCGA9Ho9Jk6cCABoaGhAb2+vyhWdJAkFVuYIIUZ1dGdraytOnDiBSZMmjWrDycTExIjtio+2DRobG2GxWFBRUTGqOiK5DUKdEGJUh1dZLBZ0dHQgPz9/VMdcREdHR+x3aDRt4PV60dXVBYvFgoKCglEtLleiDRQJF4/Hg7q6OkiShKSkpGHftQgh4Ha7ER0dPezPFkLA4XDAbrejoqIiYlfr+9oAAJKSkob99+B2u6HRaEb0hfJdEFarNaLbINR5vV4YjcZRzT7yeDwjan8hxKl9tNLT0yO25+P1emEwGCDL8oh+HwInr+WRhPvpbZCRkTHqNlBsKnJGRgY2bdqEm2++GePGjVPqbS+ou7sb7777LubMmROwzwxWGRkZ2LhxI2655ZaAtoHRaMSGDRswe/bsgH0m+YdOp4PRaMT48eMDurLearWivb0dOTk5AfvMYKXT6dDa2orCwsKArhOyWCxoa2tDXl6eIu+n2O2BVquFx+PBgQMHlHrLIfnyyy9hs9m4xQT+3gaBfnZy4MAB9Pf3sw3CgCzLMJvNAd8ssaenB16vl71enFzP4vF4YDQaA/q5vjZQqteoWLhIkoSMjAw0NzcHbIM1IQSampqg1+sjdoz2TJmZmQFvg8bGRs4OCiPR0dHo7e0N6HfIbDZzNf9pdDod+vv7A9oGAwMDiraBogOb06dPx+DgIDwej5Jve05erxc2mw1Tp04NyOeFghkzZsDhcMDtdgfk87xeL6xWK9sgjKSkpAT0HBEhBDweD29QTpOeng6PxxPQcPF4PNDr9Yq9p6Lh4tul+Pjx40q+7Tl1dnYCAIqLiwPyeaHA1wZtbW0B+byOjg4AODUVkkKf75d8oNZMWK1WAOCK/tP4zqeyWCwB+TxfGyh5Lpai4RIbG4uoqCjs27dPybc9p3379kGj0bA7fZqYmBhER0cHrA327t3LNggzvmdnPT09Afk8o9EISZL4vOU0vpmbBoMhIJ9nMBgUbwNFw0WSJEyYMAFtbW1+784JIdDS0oLx48fzectpJElCXl5eQA4XEkKgtbUV2dnZbIMwE6gxfyEELBYL10edwbesw2KxBKQNzGYzEhISFG0DxSeTl5eXw+12+30rkcHBQbhcLpSXl/v1c0JReXk5PB6P37vUdrsdLpdr1AsvKbhIknRqzN/f+1W53W4IIXha5VlkZGScWgPoT75nO0qfvaN4uPjWV3z11VdKv/XX+E5QnDBhgl8/JxSNHTsWAPw+LdzXxtyuP/z4nn8MDAz49XN8O/pyWPWbfFtg+Xt40rddjNI7zyseLhqNBklJSdi/f7/funNCCOzduxcJCQkcpz0L38aB9fX1fm2D/fv3j2g3AAp+sixDo9HAYDD49TvU09MDrVbLIbFziImJQU9Pj1/bwGg0+mXLHcXDRZIklJWVYWBgwG9Tkr1eL0wmE6ZPn84v5VlIkoSKigpYrVa/dak9Hg/6+/tRVlbGNghDkiQhLS3NrzPGfNsGcXv+s/OtHXS5XH4bnvRt+eKPNvDLBj6TJk0CALS0tPjj7XHs2DEAwJQpU/zy/uGgpKQEwN+HD5V29OhRAMDkyZP98v6kPt+aB980VaX5htxGs1ltuEtNTQUA9Pf3++X9fZvd+mONkV/CJTY2FrGxsdi1a5fi3Tnf8cdarZbjtOeh1WoRHx+P3bt3+6UNdu3adaqdKTxFR0dDlmV0dXX55TvU3d0NjUbDYdXzkGUZUVFRfmuDrq4uaDSaUe1ifS5+CRdJkjB16lQYDAbFu3NCCHR0dGDy5MnsSp+Hb3iyt7dX8eFJ386tHJYMb5IkISUlxS89FyEEBgcHuXXTBfiGxhwOh1+eu9hsNqSlpfmlDfy2r3V5efmpdRBK8q2hqaysVPR9w5HvEDClh8aOHj0KIQTKysoUfV8KPpmZmQCUHxrzDcdwCvKF+YYnld5M1NcGSk9B9vFbuMTFxSE2NhaffvqpYokrhMAnn3yCmJgYxafNhaOYmBgkJiZi586dirbBp59+Cp1Ox2HJCOAbGuvs7FT0O9TZ2YmoqCgOiQ2BRqNBdHS0okNjvjbwvbc/+C1cfDOWDAaDYjOWPB4Purq6OENpiCRJwiWXXAKTyQSHw6HIe7pcLhiNRlx00UVsgwjgW1Bps9kU+8Xm9XrhcDiQkZHB79AQSJKEMWPGwOFwKDbELYSAzWbz60w9vx73NmPGDADKLeb78ssvAYCr8ofBN3PPd0rlaO3fvx/A34fcKPz5hq58Cx5Hy7cokLsgD51v1phSZ7z4Fk76a0gM8HO4aLVapKenKzJrTAiBnTt3Ii0tbVTHsEaaqKgojB8/HnV1dYq0we7du5GZmem3rjQFH41Gg5iYGEWGZYQQMBgM0Ol0EXuU8UjIsoy4uDh0d3cr0gadnZ2IiYnx67CkX1tXkiRcffXVsNvto07c3t5e2Gw2XHXVVexKD4OvDVwuF06cODGq9+ru7sbg4CBmz57NNoggkiQhKysLbrd71MOrdrsdHo8H48aN43doGCRJwvjx40+dYTUavvOe/L3pr99vHbKzsxEdHY0PP/xwxIkrhMCHH36I6OhonrE9ApmZmdDpdNi2bduo2mDbtm3QarWn9o+jyJGYmAhZlnHixIlRfYeOHz9+6i6chicuLg4ajWZUO54LIdDe3g5ZlpGYmKhwhV/n93CRJAmXXnopTpw4MeLEtdvtaG9vx0UXXcSu9AhIkoQrrrgCRqNxxCt9rVYrOjs7MXPmTN5xRiDfeguLxTLih8oulwt2ux1jx47ld2gEfD1I327kI+F2u2GxWJCZmen3NgjIb+qysjLIsjyi3osQAtu3b4csy9zafRQmTZqEqKgofPDBByNqg61bt0Kj0fBBfgTzPfwdSe/F12vx7VlGI+Nb8DiSM7NObwPf+iV/Cki4aDQaVFZW4vDhw8NejGWz2dDU1ISysjK/bFEQKTQaDWbNmoW2trZhL8ayWCxobm7GRRddxHUJEcw3LdlkMg17eYHL5YLZbEZmZiZHH0ZBlmWMGTMGZrMZTqdzWK91u90wmUwBmwIekFb2DY1FRUWhpqZmyIkrhEBtbS00Gg2HYxRQVlYGrVaLzZs3D6sN3nvvPURHR+OSSy5hG0QwSZJODWkNZ9zft1OHLMt+nfoaKXwB3draOuw2OL0N/S1gtxBRUVG4+uqr0d7ePuQtYdra2nDs2DFceeWV7LUoQKPR4LrrrkN3dzeampqG9MVsbW3F8ePHMXv2bLYBnRr3N5vNQxqFEEJgYGAAdrsd48ePZ69FAbIsIzs7GzabbciHuZnNZlgsloAeSR7Qlp46dSrS09Px3nvvXfAY5MHBQWzatAl6vR7Tpk3jHbNCiouLkZWVhS1btlxwgsXg4CA2b96M9PR0Hm9AAE6Gi16vR0xMDFpbWy/4cN/tdqOtrQ06nY5b6ysoNTUVOp0Ora2tFxyidLvdaGlpQWxsrN82qTybgIaLJElYtGgRPB4P1q1bd86/FLfbjXfffRcejweLFi3i3Y6CJEnCTTfdBEmSUF1dfc5xW7fbjXfeeQcejwff/va3Ge50iiRJyMvLg9frRUtLyzl3Pvd4PKfO/cnLy+N3SEGSJCE/Px9CCDQ3N5+zDbxeL44cOQIAKCgoCGgbBPy3dkJCAhYsWACDwYB169bBarWeGp4RQsBqteLdd99FZ2cn5s+f7/e52JFIp9Nh0aJF6O3tRXV1NSwWyzfaYN26dejq6sKCBQu4SSh9g1arRU5ODqxW66m759O/Q06nE83NzXA4HMjLy+OQqh9ER0cjPz8fNpsNzc3NcLlcX2sDl8uFI0eOwG63Iy8vL+C7akhCgd3oPB4Puru7kZWVNaSfF0LgyJEjqK2thVarxdSpU6HX69Hb24sDBw7A4XDguuuuQ3Fx8ZCT9sSJExgzZkzEzmbybeo51AWOQgi0tLRg8+bNiIqKwrRp06DX69HX14f6+no4HA7MnTt3WG1w/PhxjB07NmLbINR5vV7Y7XbEx8cP6eeFEDCZTGhvb0dUVNSprZkGBwdPnfuek5ODxMTEIX+HrFZrRG8NM5I26O/vx7Fjx6DRaE4NWfrawOv1IicnB8nJyUNuA4vFgri4uFG3gSrhApz8S+nt7cUnn3yC48ePw+VyndoHa9asWcM+RIjhMrxwAU62QV9fHz7++GO0t7d/ow2Gu2MqwyW0DfcXG/D3Q786OztPjULIsoyEhASMHTsWWq12WN8hhsvI26CjowNWqxVer/dUG2RlZSEmJmZYbRB04bJ///5TO3cOl9VqxeDgIGJjY4f1l3q6vr4+zJgxI2J/sY2mDXzbbw8ODiImJgbx8fEjGpuN9DYIdV6vFz09PSPeGNblcsHj8SAqKmrEw2AOhwN6vT6iw8VoNI7o+HAhBNxuN9xu96k2GMl1PDg4iPT09OAIF98dsNpSU1Mj9qEh24BGSwih2Lk/ozHcO+1wEk5toEi4jJbL5YLFYhlxz4dGz+FwwG63c7oojZhvx15OAFGP2+2G0+kMio1Bg6LvuXv3bqxevRqDg4NqlxKxtm/fjrVr155zSiPRhZhMJjQ3Nw97WxJSjm+BtFInVo5GUITL9OnT4fV6FTuxkoavvLwcAwMDOHz4sNqlUIgyGAxISkqCVqtVu5SI5Htek56eHhTPPYMiXOLj41FaWoq9e/cGReJGoszMTEyYMAF79uxRuxQKQWazGQ6Hg3uHqai3txcejydo2iAowgUAKisrYbFY0NTUpHYpEauyshKdnZ2jPrGSIo/BYEBcXNyIZ3vS6HV3dyMlJSVoeo5BEy56vR65ubm8c1ZRXl4eUlNT2QY0LHa7HRaLBenp6WqXErH6+/vhcDgCck7LUAVNuAAn75y7u7vR3t6udikRSZKkU+fuDPfMF4pcRqMR0dHRSE5OVruUiNXd3Y34+Pig6jkGVbjk5uZCr9fzzllFkyZNgk6nw969e9UuhUKAy+WCyWQa9m4OpBybzXbq6OJgElThApzsvTQ3NwfFgsBIFBUVhRkzZuDAgQOcGk4X1NPTw6OLVdbd3Y2YmJig6zkGXbiUlpYiLi4OdXV1apcSsWbMmAEhBOrr69UuhYKYb7sYvV4fFFNfI5HT6Qzo0cXDEXThotFoUFZWhoMHD17wQDHyj7i4OEyaNAn79u3j1HA6p76+Pni9Xuj1erVLiVgGgwGyLAdlGwRduAAnF1UCwBdffKFyJZGroqICFosFjY2NapdCQUgIAYPBgOTk5KCZ+hppPB4Penp6FNlk0h+CryKcPMxq8uTJ2Ldv3wWP8CT/0Ov1yMvL4+QKOiuz2Qyn08npxyryndcSLIsmzxSU4QKcvHO22WxoaGhQu5SIVVlZCYPBgLa2NrVLoSBjMBgQHx8fFBskRiJfzzE1NTXgJ0wOVdCGS2pqKgoKCnjnrKKcnBykp6ezDehrbDYbrFYrey0q6u/vh9PpDLrpx6cL2nABTt459/T0oLW1Ve1SIlZlZSWOHj2K3t5etUuhIGE0GqHVapGUlKR2KRGru7sbCQkJ0Ol0apdyTkEdLtnZ2cjMzOSds4omTpyI+Ph4Tg0nACenvvb39wfl1NdIYbVaYbVag7rXAgR5uAAn75xbW1thNBrVLiUi+aaGf/XVV5waTujp6YEsyzzYT0XBumjyTEEfLiUlJUhISOCds4p8U8P379+vciWkJt/U10g+415tDocDJpMp6HstQAiEiyzLKC8vx6FDh2C1WtUuJyLFxsZiypQp2L9/P6eGR7De3l4IIYJywV6kMBgMiIqKContdoI+XABg2rRpkGWZd84q8k0NP3TokNqlkAqEEDAajUhJSQnaqa/hLtgXTZ4p+CsEEBMTg6lTp/LOWUUpKSkoLCzk5IoI1d/fD5fLxenHKjIajRBCBO2iyTOFRLgAJ894dzgc+PLLL9UuJWJVVlait7cXLS0tapdCAWY0GoN+6ms48y2aTEtLQ1RUlNrlDEnIhEtycjKKiopQV1cHIYTa5USk8ePHY8yYMey9RBir1QqbzRYyd8zhqK+vDy6XKyQe5PuETLgAJ++cTSYTmpub1S4lYlVWVuLYsWMwGAxql0IBYjAYEBMTg8TERLVLiVjd3d1ISkpCbGys2qUMWUiFS1ZWFrKysjgtWUXFxcVITExkG0QIp9OJgYEB9lpUZLFYYLfbQ6rXAoRYuAAn75zb29vR1dWldikR6fSp4RaLRe1yyM98U19TUlLULiVidXV1QafThVzPMeTCpaioCMnJybxzVtHUqVOh0Wg4NTzMeTwe9PX1cdGkigYHBzEwMBByvRYgBMNFkiSUl5ejoaEBZrNZ7XIiUkxMDKZNm4YvvvgCLpdL7XLIT3p6erhoUmUGgwHR0dEhud1OyIULAEyZMgVarRb79u1Tu5SI5ZsafvDgQbVLIT8QQqCnpwepqakhM/U13LjdbvT29iI9PT0kNwkNyXDRarWYNm0a6uvr4XQ61S4nIiUlJaG4uJhTw8OUyWTiokmV+TbrDdU2CMlwAU7eObtcLi6qVBGnhocvg8GAxMTEkJr6Gk68Xm/ILZo8U8iGS0JCAkpKSlBXVwev16t2ORFp7NixGDduHBdVhhmLxYLBwUFOP1ZRX18f3G53SD7I9wnZcAFObqY4MDCAI0eOqF1KxKqsrMTx48c5NTyMGAwGxMbGIiEhQe1SIlZ3dzeSk5MRExOjdikjFtLhMmbMGGRnZ/POWUWFhYVITk5mG4QJh8MBs9nMXouKBgYGMDg4GNK9FiDEwwU4eefc0dGBjo4OtUuJSJIkoaKiAo2NjZwaHgZ8U1+5aFI93d3diIuLC/meY8iHS35+PlJTU3nnrCLf1PC9e/eqXQqNgtvtPrVoMhSnvoYDu90Os9kc8r0WIAzCxXfnfPjwYfT396tdTkSKjo7G9OnTOTU8xPX09ECSJC6aVFF3d3fY9BxDPlwAYPLkyYiJieGds4rKysrgdrtx4MABtUuhEfB6vacWTWo0GrXLiUgulwt9fX3IzMwMi55jWIRLVFQUZsyYgQMHDsDhcKhdTkRKSEjAxIkTsXfvXk4ND0EmkwlutztkF+yFA6PRGFY9x7AIFwCYMWMGvF4v6uvr1S4lYvmmhh8+fFjtUmiYDAYDkpKSQnrqayjzLZrU6/Vh03MMm3CJj49HaWkp75xVlJmZiQkTJnByRYgxm81wOBycfqyi3t5eeDyesHiQ7xM24QKcvHO2WCxobGxUu5SIVVFRgc7OTpw4cULtUmiIDAYDdDod4uPj1S4lYnV3dyMlJQVarVbtUhQTVuGSnp6O3Nxc3jmriFPDQ8vg4CAsFgt7LSrq7++Hw+EIq14LEGbhApxcVNnd3Y329na1S4lIp08NN5lMapdDF+BbNJmcnKx2KRGru7sb8fHxYddzDLtwyc3NhV6v552ziiZPnozY2FhODQ9yLpcLJpMpZM8LCQc2mw0WiyXsei1AGIYLcLL30tzcjL6+PrVLiUi+qeFffvklBgcH1S6HzsG3aDItLU3tUiJWd3c3tFptWPYcwzJcSktLERcXh7q6OrVLiVhlZWWcGh7EfIsm09LSwmbqa6hxOp0wmUxhs2jyTKF5Cs0FaDQalJWVYdeuXZg5c+aprmdCQgL3TQqQuLg4lJaWYt++fSgvL4fJZGIbqMh3bLGvDQDA4/Fw0WQAndkGg4ODkGU5bBZNniksey4AkJOTg+3bt6OkpAQZGRnIz89HRkYGiouLsWLFCj5sDoCioiJs3rwZBQUFbAOVmEwmrFixAsXFxV9rg4qKClRXV8Nms6ldYtg7VxvMnDkT69atw8DAgNol+ocIQzU1NSI+Pl4A+MY/kiQJSZJEfHy8qKmpUbvUsMU2UJ+vDXx/32drC7aBf0VyG4RduNTU1AiNRiNkWT5rQ/r+kWVZaDSasGxUtbEN1Mc2UF+kt4EkhBDK94fUYTKZkJ2dDbvdPqQtYGRZhk6nQ3t7e1hscR0M2AbqYxuoj20QZs9cXn31VdhstiHvLeb1emGz2bBy5Uo/VxY52AbqYxuoj20AhE3PRQiB4uJiNDc3Yzj/S5IkoaCgAE1NTZzBNEpsA/WxDdTHNjgpbMLFaDSOan8ko9EYtlMCA4VtoD62gfrYBieFzbCYxWIZ1evNZrNClUQutoH62AbqYxucFDbh4lsYNlKJiYkKVRK52AbqYxuoj21wUtiEi16vR2Fh4bDHKiVJQmFhIfdXUgDbQH1sA/WxDU4Km3CRJAkPP/zwiF67bNmysHiApja2gfrYBupjG5wUNg/0Ac4tDwZsA/WxDdTHNgijngsApKSkYO3atZAkCbJ8/v81WZYhSRKqq6vDpjGDAdtAfWwD9bENEN57i51tP5/T97Wqra1Vu9SwxTZQH9tAfZHcBmEZLkII0dfXJ1asWCEKCwu/1qCFhYVixYoVwmQyqV1i2GMbqI9toL5IbYOweuZyNkII9Pb2wmw2IzExEWlpaWHzwCxUsA3UxzZQX6S1QdiHCxERBV5YPdAnIqLgwHAhIiLFMVyIiEhxDBciIlIcw4WIiBTHcCEiIsUxXIiISHEMFyIiUhzDhYiIFMdwISIixTFciIhIcQwXIiJSHMOFiIgUx3AhIiLF/R96o8qByf2ilAAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "model.plot()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8230d562-2635-4adc-b566-06ac679b166a", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.7" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/tutorials/model/0.0_cache_data b/tutorials/model/0.0_cache_data deleted file mode 100644 index d8d78888..00000000 Binary files a/tutorials/model/0.0_cache_data and /dev/null differ diff --git a/tutorials/model/0.0_config.yml b/tutorials/model/0.0_config.yml deleted file mode 100644 index 14d2b95d..00000000 --- a/tutorials/model/0.0_config.yml +++ /dev/null @@ -1,51 +0,0 @@ -affine_trainable: false -auto_save: true -base_fun_name: silu -ckpt_path: ./model -device: cpu -grid: 3 -grid_eps: 1.0 -grid_range: -- -1 -- 1 -k: 3 -mult_arity: 2 -round: 0 -sb_trainable: true -sp_trainable: true -state_id: 0 -symbolic.funs_name.0: -- - '0' - - '0' - - '0' - - '0' -- - '0' - - '0' - - '0' - - '0' -- - '0' - - '0' - - '0' - - '0' -- - '0' - - '0' - - '0' - - '0' -- - '0' - - '0' - - '0' - - '0' -symbolic.funs_name.1: -- - '0' - - '0' - - '0' - - '0' - - '0' -symbolic_enabled: true -width: -- - 4 - - 0 -- - 5 - - 0 -- - 1 - - 0 diff --git a/tutorials/model/0.0_state b/tutorials/model/0.0_state deleted file mode 100644 index dd684190..00000000 Binary files a/tutorials/model/0.0_state and /dev/null differ diff --git a/tutorials/model/0.1_cache_data b/tutorials/model/0.1_cache_data deleted file mode 100644 index 53ca4ae1..00000000 Binary files a/tutorials/model/0.1_cache_data and /dev/null differ diff --git a/tutorials/model/0.1_config.yml b/tutorials/model/0.1_config.yml deleted file mode 100644 index 1cab8a4e..00000000 --- a/tutorials/model/0.1_config.yml +++ /dev/null @@ -1,51 +0,0 @@ -affine_trainable: false -auto_save: true -base_fun_name: silu -ckpt_path: ./model -device: cpu -grid: 3 -grid_eps: 1.0 -grid_range: -- -1 -- 1 -k: 3 -mult_arity: 2 -round: 0 -sb_trainable: true -sp_trainable: true -state_id: 1 -symbolic.funs_name.0: -- - '0' - - '0' - - '0' - - '0' -- - '0' - - '0' - - '0' - - '0' -- - '0' - - '0' - - '0' - - '0' -- - '0' - - '0' - - '0' - - '0' -- - '0' - - '0' - - '0' - - '0' -symbolic.funs_name.1: -- - '0' - - '0' - - '0' - - '0' - - '0' -symbolic_enabled: false -width: -- - 4 - - 0 -- - 5 - - 0 -- - 1 - - 0 diff --git a/tutorials/model/0.1_state b/tutorials/model/0.1_state deleted file mode 100644 index 3bc98435..00000000 Binary files a/tutorials/model/0.1_state and /dev/null differ diff --git a/tutorials/model/0.2_cache_data b/tutorials/model/0.2_cache_data deleted file mode 100644 index 53ca4ae1..00000000 Binary files a/tutorials/model/0.2_cache_data and /dev/null differ diff --git a/tutorials/model/0.2_config.yml b/tutorials/model/0.2_config.yml deleted file mode 100644 index 20146554..00000000 --- a/tutorials/model/0.2_config.yml +++ /dev/null @@ -1,51 +0,0 @@ -affine_trainable: false -auto_save: true -base_fun_name: silu -ckpt_path: ./model -device: cpu -grid: 3 -grid_eps: 1.0 -grid_range: -- -1 -- 1 -k: 3 -mult_arity: 2 -round: 0 -sb_trainable: true -sp_trainable: true -state_id: 2 -symbolic.funs_name.0: -- - '0' - - '0' - - '0' - - '0' -- - '0' - - '0' - - '0' - - '0' -- - '0' - - '0' - - '0' - - '0' -- - '0' - - '0' - - '0' - - '0' -- - '0' - - '0' - - '0' - - '0' -symbolic.funs_name.1: -- - '0' - - '0' - - '0' - - '0' - - '0' -symbolic_enabled: true -width: -- - 4 - - 0 -- - 5 - - 0 -- - 1 - - 0 diff --git a/tutorials/model/0.2_state b/tutorials/model/0.2_state deleted file mode 100644 index 3bc98435..00000000 Binary files a/tutorials/model/0.2_state and /dev/null differ diff --git a/tutorials/model/history.txt b/tutorials/model/history.txt deleted file mode 100644 index 987e3a99..00000000 --- a/tutorials/model/history.txt +++ /dev/null @@ -1,4 +0,0 @@ -### Round 0 ### -init => 0.0 -0.0 => fit => 0.1 -0.1 => prune_input => 0.2