From 7246593fa4f7cf60bbe4bc027c4cfc3856e9b644 Mon Sep 17 00:00:00 2001 From: chicm-ms <38930155+chicm-ms@users.noreply.github.com> Date: Mon, 16 Sep 2019 16:16:06 +0800 Subject: [PATCH 01/22] Fix multiphase issue with gridsearch tuner and batch tuner (#1539) * Fix multiphase issue with gridsearch tuner and batch tuner --- docs/en_US/AdvancedFeature/MultiPhase.md | 9 +++++++-- src/nni_manager/core/nnimanager.ts | 7 +++++-- src/sdk/pynni/nni/msg_dispatcher.py | 6 +++++- src/sdk/pynni/nni/trial.py | 3 ++- test/config_test/multi_phase/multi_phase.py | 3 +++ test/pipelines-it-local-windows.yml | 2 +- test/pipelines-it-local.yml | 2 +- test/pipelines-it-pai-windows.yml | 2 +- test/pipelines-it-pai.yml | 2 +- test/pipelines-it-remote-windows.yml | 2 +- test/pipelines-it-remote.yml | 2 +- 11 files changed, 28 insertions(+), 12 deletions(-) diff --git a/docs/en_US/AdvancedFeature/MultiPhase.md b/docs/en_US/AdvancedFeature/MultiPhase.md index 831a6c7544..9dbc22edc1 100644 --- a/docs/en_US/AdvancedFeature/MultiPhase.md +++ b/docs/en_US/AdvancedFeature/MultiPhase.md @@ -8,8 +8,6 @@ Typically each trial job gets a single configuration (e.g., hyperparameters) fro The above cases can be supported by the same feature, i.e., multi-phase execution. To support those cases, basically a trial job should be able to request multiple configurations from tuner. Tuner is aware of whether two configuration requests are from the same trial job or different ones. Also in multi-phase a trial job can report multiple final results. -Note that, `nni.get_next_parameter()` and `nni.report_final_result()` should be called sequentially: __call the former one, then call the later one; and repeat this pattern__. If `nni.get_next_parameter()` is called multiple times consecutively, and then `nni.report_final_result()` is called once, the result is associated to the last configuration, which is retrieved from the last get_next_parameter call. So there is no result associated to previous get_next_parameter calls, and it may cause some multi-phase algorithm broken. - ## Create multi-phase experiment ### Write trial code which leverages multi-phase: @@ -23,6 +21,9 @@ It is pretty simple to use multi-phase in trial code, an example is shown below: for i in range(5): # get parameter from tuner tuner_param = nni.get_next_parameter() + # nni.get_next_parameter returns None if there is no more hyper parameters can be generated by tuner. + if tuner_param is None: + break # consume the params # ... @@ -32,6 +33,10 @@ It is pretty simple to use multi-phase in trial code, an example is shown below: # ... ``` +In multi-phase experiments, at each time the API ```nni.get_next_parameter()``` is called, it returns a new hyper parameter generated by tuner, then the trail code consume this new hyper parameter and report final result of this hyper parameter. `nni.get_next_parameter()` and `nni.report_final_result()` should be called sequentially: __call the former one, then call the later one; and repeat this pattern__. If `nni.get_next_parameter()` is called multiple times consecutively, and then `nni.report_final_result()` is called once, the result is associated to the last configuration, which is retrieved from the last get_next_parameter call. So there is no result associated to previous get_next_parameter calls, and it may cause some multi-phase algorithm broken. + +Note that, ```nni.get_next_parameter``` returns None if there is no more hyper parameters can be generated by tuner. + __2. Experiment configuration__ To enable multi-phase, you should also add `multiPhase: true` in your experiment YAML configure file. If this line is not added, `nni.get_next_parameter()` would always return the same configuration. diff --git a/src/nni_manager/core/nnimanager.ts b/src/nni_manager/core/nnimanager.ts index c516117ca5..888901e44c 100644 --- a/src/nni_manager/core/nnimanager.ts +++ b/src/nni_manager/core/nnimanager.ts @@ -691,8 +691,11 @@ class NNIManager implements Manager { }; this.log.info(`updateTrialJob: job id: ${tunerCommand.trial_job_id}, form: ${JSON.stringify(trialJobForm)}`); await this.trainingService.updateTrialJob(tunerCommand.trial_job_id, trialJobForm); - await this.dataStore.storeTrialJobEvent( - 'ADD_HYPERPARAMETER', tunerCommand.trial_job_id, content, undefined); + if (tunerCommand['parameters'] !== null) { + // parameters field is set as empty string if no more hyper parameter can be generated by tuner. + await this.dataStore.storeTrialJobEvent( + 'ADD_HYPERPARAMETER', tunerCommand.trial_job_id, content, undefined); + } break; case NO_MORE_TRIAL_JOBS: if (!['ERROR', 'STOPPING', 'STOPPED'].includes(this.status.status)) { diff --git a/src/sdk/pynni/nni/msg_dispatcher.py b/src/sdk/pynni/nni/msg_dispatcher.py index 3abf8066c6..fc9de474a2 100644 --- a/src/sdk/pynni/nni/msg_dispatcher.py +++ b/src/sdk/pynni/nni/msg_dispatcher.py @@ -22,6 +22,7 @@ from collections import defaultdict import json_tricks +from nni import NoMoreTrialError from .protocol import CommandType, send from .msg_dispatcher_base import MsgDispatcherBase from .assessor import AssessResult @@ -144,7 +145,10 @@ def handle_report_metric_data(self, data): assert data['trial_job_id'] is not None assert data['parameter_index'] is not None param_id = _create_parameter_id() - param = self.tuner.generate_parameters(param_id, trial_job_id=data['trial_job_id']) + try: + param = self.tuner.generate_parameters(param_id, trial_job_id=data['trial_job_id']) + except NoMoreTrialError: + param = None send(CommandType.SendTrialJobParameter, _pack_parameter(param_id, param, trial_job_id=data['trial_job_id'], parameter_index=data['parameter_index'])) else: raise ValueError('Data type not supported: {}'.format(data['type'])) diff --git a/src/sdk/pynni/nni/trial.py b/src/sdk/pynni/nni/trial.py index 132fd96834..89ceeb4a49 100644 --- a/src/sdk/pynni/nni/trial.py +++ b/src/sdk/pynni/nni/trial.py @@ -43,7 +43,8 @@ def get_next_parameter(): - """Returns a set of (hyper-)paremeters generated by Tuner.""" + """Returns a set of (hyper-)paremeters generated by Tuner. + Returns None if no more (hyper-)parameters can be generated by Tuner.""" global _params _params = platform.get_next_parameter() if _params is None: diff --git a/test/config_test/multi_phase/multi_phase.py b/test/config_test/multi_phase/multi_phase.py index 39e77d8083..7aec26f779 100644 --- a/test/config_test/multi_phase/multi_phase.py +++ b/test/config_test/multi_phase/multi_phase.py @@ -4,5 +4,8 @@ if __name__ == '__main__': for i in range(5): hyper_params = nni.get_next_parameter() + print('hyper_params:[{}]'.format(hyper_params)) + if hyper_params is None: + break nni.report_final_result(0.1*i) time.sleep(3) diff --git a/test/pipelines-it-local-windows.yml b/test/pipelines-it-local-windows.yml index a0cf163fbc..56a6e99bdc 100644 --- a/test/pipelines-it-local-windows.yml +++ b/test/pipelines-it-local-windows.yml @@ -18,7 +18,7 @@ jobs: displayName: 'generate config files' - script: | cd test - python config_test.py --ts local --local_gpu --exclude smac,bohb,multi_phase_batch,multi_phase_grid + python config_test.py --ts local --local_gpu --exclude smac,bohb displayName: 'Examples and advanced features tests on local machine' - script: | cd test diff --git a/test/pipelines-it-local.yml b/test/pipelines-it-local.yml index ec19565d13..4f054546fc 100644 --- a/test/pipelines-it-local.yml +++ b/test/pipelines-it-local.yml @@ -31,7 +31,7 @@ jobs: displayName: 'Built-in tuners / assessors tests' - script: | cd test - PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts local --local_gpu --exclude multi_phase_batch,multi_phase_grid + PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts local --local_gpu displayName: 'Examples and advanced features tests on local machine' - script: | cd test diff --git a/test/pipelines-it-pai-windows.yml b/test/pipelines-it-pai-windows.yml index 2452f320f1..0291ecfdca 100644 --- a/test/pipelines-it-pai-windows.yml +++ b/test/pipelines-it-pai-windows.yml @@ -65,5 +65,5 @@ jobs: python --version python generate_ts_config.py --ts pai --pai_host $(pai_host) --pai_user $(pai_user) --pai_pwd $(pai_pwd) --vc $(pai_virtual_cluster) --nni_docker_image $(docker_image) --data_dir $(data_dir) --output_dir $(output_dir) --nni_manager_ip $(nni_manager_ip) - python config_test.py --ts pai --exclude multi_phase,smac,bohb,multi_phase_batch,multi_phase_grid + python config_test.py --ts pai --exclude multi_phase,smac,bohb displayName: 'Examples and advanced features tests on pai' \ No newline at end of file diff --git a/test/pipelines-it-pai.yml b/test/pipelines-it-pai.yml index c038c01cce..5e44c7a6be 100644 --- a/test/pipelines-it-pai.yml +++ b/test/pipelines-it-pai.yml @@ -76,6 +76,6 @@ jobs: python3 generate_ts_config.py --ts pai --pai_host $(pai_host) --pai_user $(pai_user) --pai_pwd $(pai_pwd) --vc $(pai_virtual_cluster) \ --nni_docker_image $TEST_IMG --data_dir $(data_dir) --output_dir $(output_dir) --nni_manager_ip $(nni_manager_ip) - PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts pai --exclude multi_phase_batch,multi_phase_grid + PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts pai PATH=$HOME/.local/bin:$PATH python3 metrics_test.py displayName: 'integration test' diff --git a/test/pipelines-it-remote-windows.yml b/test/pipelines-it-remote-windows.yml index a4c2addacf..fb7a088823 100644 --- a/test/pipelines-it-remote-windows.yml +++ b/test/pipelines-it-remote-windows.yml @@ -39,7 +39,7 @@ jobs: cd test python generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) --remote_port $(Get-Content port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip) Get-Content training_service.yml - python config_test.py --ts remote --exclude cifar10,smac,bohb,multi_phase_batch,multi_phase_grid + python config_test.py --ts remote --exclude cifar10,smac,bohb displayName: 'integration test' - task: SSH@0 inputs: diff --git a/test/pipelines-it-remote.yml b/test/pipelines-it-remote.yml index 923e4215f3..8fe96552fb 100644 --- a/test/pipelines-it-remote.yml +++ b/test/pipelines-it-remote.yml @@ -53,7 +53,7 @@ jobs: python3 generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) \ --remote_port $(cat port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip) cat training_service.yml - PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts remote --exclude cifar10,multi_phase_batch,multi_phase_grid + PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts remote --exclude cifar10 PATH=$HOME/.local/bin:$PATH python3 metrics_test.py displayName: 'integration test' - task: SSH@0 From 55f48d27eedae9bb7f88c928cb979433f4592196 Mon Sep 17 00:00:00 2001 From: QuanluZhang Date: Mon, 16 Sep 2019 16:19:30 +0800 Subject: [PATCH 02/22] Merge dev-nas-tuner back to master (#1531) * PPO tuner for NAS, supports NNI's NAS interface (#1380) --- azure-pipelines.yml | 2 +- docs/en_US/Tuner/BuiltinTuner.md | 57 +- docs/en_US/Tuner/PPOTuner.md | 20 + docs/img/enas_search_space.png | Bin 0 -> 80580 bytes docs/img/ppo_cifar10.png | Bin 0 -> 253123 bytes docs/img/ppo_mnist.png | Bin 0 -> 101368 bytes .../mnist-nas/classic_mode/config_hpo.yml | 16 + examples/trials/mnist-nas/config_ppo.yml | 19 + examples/trials/nas_cifar10/README.md | 9 +- .../trials/nas_cifar10/config_pai_ppo.yml | 31 + examples/trials/nas_cifar10/config_ppo.yml | 21 + examples/trials/nas_cifar10/data/download.sh | 1 + examples/trials/nas_cifar10/macro_cifar10.sh | 31 + .../trials/nas_cifar10/macro_cifar10_pai.sh | 31 + examples/trials/nas_cifar10/src/__init__.py | 0 .../nas_cifar10/src/cifar10/__init__.py | 0 .../nas_cifar10/src/cifar10/data_utils.py | 74 +++ .../nas_cifar10/src/cifar10/general_child.py | 423 +++++++++++++ .../trials/nas_cifar10/src/cifar10/models.py | 196 ++++++ .../src/cifar10/nni_child_cifar10.py | 162 +++++ .../trials/nas_cifar10/src/cifar10_flags.py | 45 ++ examples/trials/nas_cifar10/src/common_ops.py | 255 ++++++++ examples/trials/nas_cifar10/src/utils.py | 262 ++++++++ .../random_nas_tuner/random_nas_tuner.py | 9 +- .../rest_server/restValidationSchemas.ts | 2 +- src/sdk/pynni/nni/constants.py | 4 +- .../nni/hyperopt_tuner/hyperopt_tuner.py | 2 + src/sdk/pynni/nni/msg_dispatcher.py | 7 +- src/sdk/pynni/nni/nas_utils.py | 171 ++++- src/sdk/pynni/nni/ppo_tuner/__init__.py | 0 src/sdk/pynni/nni/ppo_tuner/distri.py | 198 ++++++ src/sdk/pynni/nni/ppo_tuner/model.py | 166 +++++ src/sdk/pynni/nni/ppo_tuner/policy.py | 219 +++++++ src/sdk/pynni/nni/ppo_tuner/ppo_tuner.py | 599 ++++++++++++++++++ src/sdk/pynni/nni/ppo_tuner/requirements.txt | 3 + src/sdk/pynni/nni/ppo_tuner/util.py | 266 ++++++++ src/sdk/pynni/nni/tuner.py | 3 +- tools/nni_annotation/.gitignore | 1 + tools/nni_annotation/test_annotation.py | 15 +- tools/nni_cmd/config_schema.py | 18 + tools/nni_cmd/constants.py | 3 +- 41 files changed, 3292 insertions(+), 49 deletions(-) create mode 100644 docs/en_US/Tuner/PPOTuner.md create mode 100644 docs/img/enas_search_space.png create mode 100644 docs/img/ppo_cifar10.png create mode 100644 docs/img/ppo_mnist.png create mode 100644 examples/trials/mnist-nas/classic_mode/config_hpo.yml create mode 100644 examples/trials/mnist-nas/config_ppo.yml create mode 100644 examples/trials/nas_cifar10/config_pai_ppo.yml create mode 100644 examples/trials/nas_cifar10/config_ppo.yml create mode 100755 examples/trials/nas_cifar10/data/download.sh create mode 100644 examples/trials/nas_cifar10/macro_cifar10.sh create mode 100644 examples/trials/nas_cifar10/macro_cifar10_pai.sh create mode 100644 examples/trials/nas_cifar10/src/__init__.py create mode 100644 examples/trials/nas_cifar10/src/cifar10/__init__.py create mode 100644 examples/trials/nas_cifar10/src/cifar10/data_utils.py create mode 100644 examples/trials/nas_cifar10/src/cifar10/general_child.py create mode 100644 examples/trials/nas_cifar10/src/cifar10/models.py create mode 100644 examples/trials/nas_cifar10/src/cifar10/nni_child_cifar10.py create mode 100644 examples/trials/nas_cifar10/src/cifar10_flags.py create mode 100644 examples/trials/nas_cifar10/src/common_ops.py create mode 100644 examples/trials/nas_cifar10/src/utils.py create mode 100644 src/sdk/pynni/nni/ppo_tuner/__init__.py create mode 100644 src/sdk/pynni/nni/ppo_tuner/distri.py create mode 100644 src/sdk/pynni/nni/ppo_tuner/model.py create mode 100644 src/sdk/pynni/nni/ppo_tuner/policy.py create mode 100644 src/sdk/pynni/nni/ppo_tuner/ppo_tuner.py create mode 100644 src/sdk/pynni/nni/ppo_tuner/requirements.txt create mode 100644 src/sdk/pynni/nni/ppo_tuner/util.py create mode 100644 tools/nni_annotation/.gitignore diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f142de7bd3..1563e4a0ee 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -15,7 +15,7 @@ jobs: displayName: 'Install nni toolkit via source code' - script: | python3 -m pip install flake8 --user - IGNORE=./tools/nni_annotation/testcase/*:F821,./examples/trials/mnist-nas/*/mnist*.py:F821 + IGNORE=./tools/nni_annotation/testcase/*:F821,./examples/trials/mnist-nas/*/mnist*.py:F821,./examples/trials/nas_cifar10/src/cifar10/general_child.py:F821 python3 -m flake8 . --count --per-file-ignores=$IGNORE --select=E9,F63,F72,F82 --show-source --statistics displayName: 'Run flake8 tests to find Python syntax errors and undefined names' - script: | diff --git a/docs/en_US/Tuner/BuiltinTuner.md b/docs/en_US/Tuner/BuiltinTuner.md index d4f4fdfb09..1825e8dcd1 100644 --- a/docs/en_US/Tuner/BuiltinTuner.md +++ b/docs/en_US/Tuner/BuiltinTuner.md @@ -20,6 +20,7 @@ Currently we support the following algorithms: |[__Metis Tuner__](#MetisTuner)|Metis offers the following benefits when it comes to tuning parameters: While most tools only predict the optimal configuration, Metis gives you two outputs: (a) current prediction of optimal configuration, and (b) suggestion for the next trial. No more guesswork. While most tools assume training datasets do not have noisy data, Metis actually tells you if you need to re-sample a particular hyper-parameter. [Reference Paper](https://www.microsoft.com/en-us/research/publication/metis-robustly-tuning-tail-latencies-cloud-systems/)| |[__BOHB__](#BOHB)|BOHB is a follow-up work of Hyperband. It targets the weakness of Hyperband that new configurations are generated randomly without leveraging finished trials. For the name BOHB, HB means Hyperband, BO means Bayesian Optimization. BOHB leverages finished trials by building multiple TPE models, a proportion of new configurations are generated through these models. [Reference Paper](https://arxiv.org/abs/1807.01774)| |[__GP Tuner__](#GPTuner)|Gaussian Process Tuner is a sequential model-based optimization (SMBO) approach with Gaussian Process as the surrogate. [Reference Paper](https://papers.nips.cc/paper/4443-algorithms-for-hyper-parameter-optimization.pdf), [Github Repo](https://github.com/fmfn/BayesianOptimization)| +|[__PPO Tuner__](#PPOTuner)|PPO Tuner is an Reinforcement Learning tuner based on PPO algorithm. [Reference Paper](https://arxiv.org/abs/1707.06347)| ## Usage of Built-in Tuners @@ -38,7 +39,7 @@ Note: Please follow the format when you write your `config.yml` file. Some built TPE, as a black-box optimization, can be used in various scenarios and shows good performance in general. Especially when you have limited computation resource and can only try a small number of trials. From a large amount of experiments, we could found that TPE is far better than Random Search. [Detailed Description](./HyperoptTuner.md) -**Requirement of classArg** +**Requirement of classArgs** * **optimize_mode** (*maximize or minimize, optional, default = maximize*) - If 'maximize', the tuner will target to maximize metrics. If 'minimize', the tuner will target to minimize metrics. @@ -66,7 +67,7 @@ tuner: Random search is suggested when each trial does not take too long (e.g., each trial can be completed very soon, or early stopped by assessor quickly), and you have enough computation resource. Or you want to uniformly explore the search space. Random Search could be considered as baseline of search algorithm. [Detailed Description](./HyperoptTuner.md) -**Requirement of classArg:** +**Requirement of classArgs** * **optimize_mode** (*maximize or minimize, optional, default = maximize*) - If 'maximize', the tuner will target to maximize metrics. If 'minimize', the tuner will target to minimize metrics. @@ -91,7 +92,7 @@ tuner: Anneal is suggested when each trial does not take too long, and you have enough computation resource(almost same with Random Search). Or the variables in search space could be sample from some prior distribution. [Detailed Description](./HyperoptTuner.md) -**Requirement of classArg** +**Requirement of classArgs** * **optimize_mode** (*maximize or minimize, optional, default = maximize*) - If 'maximize', the tuner will target to maximize metrics. If 'minimize', the tuner will target to minimize metrics. @@ -117,7 +118,7 @@ tuner: Its requirement of computation resource is relatively high. Specifically, it requires large initial population to avoid falling into local optimum. If your trial is short or leverages assessor, this tuner is a good choice. And, it is more suggested when your trial code supports weight transfer, that is, the trial could inherit the converged weights from its parent(s). This can greatly speed up the training progress. [Detailed Description](./EvolutionTuner.md) -**Requirement of classArg** +**Requirement of classArgs** * **optimize_mode** (*maximize or minimize, optional, default = maximize*) - If 'maximize', the tuner will target to maximize metrics. If 'minimize', the tuner will target to minimize metrics. @@ -156,7 +157,7 @@ nnictl package install --name=SMAC Similar to TPE, SMAC is also a black-box tuner which can be tried in various scenarios, and is suggested when computation resource is limited. It is optimized for discrete hyperparameters, thus, suggested when most of your hyperparameters are discrete. [Detailed Description](./SmacTuner.md) -**Requirement of classArg** +**Requirement of classArgs** * **optimize_mode** (*maximize or minimize, optional, default = maximize*) - If 'maximize', the tuner will target to maximize metrics. If 'minimize', the tuner will target to minimize metrics. @@ -243,7 +244,7 @@ tuner: It is suggested when you have limited computation resource but have relatively large search space. It performs well in the scenario that intermediate result (e.g., accuracy) can reflect good or bad of final result (e.g., accuracy) to some extent. [Detailed Description](./HyperbandAdvisor.md) -**Requirement of classArg** +**Requirement of classArgs** * **optimize_mode** (*maximize or minimize, optional, default = maximize*) - If 'maximize', the tuner will target to maximize metrics. If 'minimize', the tuner will target to minimize metrics. * **R** (*int, optional, default = 60*) - the maximum budget given to a trial (could be the number of mini-batches or epochs) can be allocated to a trial. Each trial should use TRIAL_BUDGET to control how long it runs. @@ -277,7 +278,7 @@ NetworkMorphism requires [PyTorch](https://pytorch.org/get-started/locally) and It is suggested that you want to apply deep learning methods to your task (your own dataset) but you have no idea of how to choose or design a network. You modify the [example](https://github.com/Microsoft/nni/tree/master/examples/trials/network_morphism/cifar10/cifar10_keras.py) to fit your own dataset and your own data augmentation method. Also you can change the batch size, learning rate or optimizer. It is feasible for different tasks to find a good network architecture. Now this tuner only supports the computer vision domain. [Detailed Description](./NetworkmorphismTuner.md) -**Requirement of classArg** +**Requirement of classArgs** * **optimize_mode** (*maximize or minimize, optional, default = maximize*) - If 'maximize', the tuner will target to maximize metrics. If 'minimize', the tuner will target to minimize metrics. * **task** (*('cv'), optional, default = 'cv'*) - The domain of experiment, for now, this tuner only supports the computer vision(cv) domain. @@ -313,7 +314,7 @@ Note that the only acceptable types of search space are `choice`, `quniform`, `u Similar to TPE and SMAC, Metis is a black-box tuner. If your system takes a long time to finish each trial, Metis is more favorable than other approaches such as random search. Furthermore, Metis provides guidance on the subsequent trial. Here is an [example](https://github.com/Microsoft/nni/tree/master/examples/trials/auto-gbdt/search_space_metis.json) about the use of Metis. User only need to send the final result like `accuracy` to tuner, by calling the NNI SDK. [Detailed Description](./MetisTuner.md) -**Requirement of classArg** +**Requirement of classArgs** * **optimize_mode** (*'maximize' or 'minimize', optional, default = 'maximize'*) - If 'maximize', the tuner will target to maximize metrics. If 'minimize', the tuner will target to minimize metrics. @@ -347,7 +348,7 @@ nnictl package install --name=BOHB Similar to Hyperband, it is suggested when you have limited computation resource but have relatively large search space. It performs well in the scenario that intermediate result (e.g., accuracy) can reflect good or bad of final result (e.g., accuracy) to some extent. In this case, it may converges to a better configuration due to Bayesian optimization usage. [Detailed Description](./BohbAdvisor.md) -**Requirement of classArg** +**Requirement of classArgs** * **optimize_mode** (*maximize or minimize, optional, default = maximize*) - If 'maximize', tuners will target to maximize metrics. If 'minimize', tuner will target to minimize metrics. * **min_budget** (*int, optional, default = 1*) - The smallest budget assign to a trial job, (budget could be the number of mini-batches or epochs). Needs to be positive. @@ -386,7 +387,7 @@ Note that the only acceptable types of search space are `choice`, `randint`, `un As a strategy in Sequential Model-based Global Optimization(SMBO) algorithm, GP Tuner uses a proxy optimization problem (finding the maximum of the acquisition function) that, albeit still a hard problem, is cheaper (in the computational sense) and common tools can be employed. Therefore GP Tuner is most adequate for situations where the function to be optimized is a very expensive endeavor. GP can be used when the computation resource is limited. While GP Tuner has a computational cost that grows at *O(N^3)* due to the requirement of inverting the Gram matrix, so it's not suitable when lots of trials are needed. [Detailed Description](./GPTuner.md) -**Requirement of classArg** +**Requirement of classArgs** * **optimize_mode** (*'maximize' or 'minimize', optional, default = 'maximize'*) - If 'maximize', the tuner will target to maximize metrics. If 'minimize', the tuner will target to minimize metrics. * **utility** (*'ei', 'ucb' or 'poi', optional, default = 'ei'*) - The kind of utility function(acquisition function). 'ei', 'ucb' and 'poi' corresponds to 'Expected Improvement', 'Upper Confidence Bound' and 'Probability of Improvement' respectively. @@ -415,3 +416,39 @@ tuner: selection_num_warm_up: 100000 selection_num_starting_points: 250 ``` + + + +![](https://placehold.it/15/1589F0/000000?text=+) `PPO Tuner` + +> Built-in Tuner Name: **PPOTuner** + +Note that the only acceptable type of search space is `mutable_layer`. `optional_input_size` can only be 0, 1, or [0, 1]. + +**Suggested scenario** + +PPOTuner is a Reinforcement Learning tuner based on PPO algorithm. When you are using NNI NAS interface in your trial code to do neural architecture search, PPOTuner is recommended. It has relatively high data efficiency but is suggested when you have large amount of computation resource. You could try it on very simple task, such as the [mnist-nas](https://github.com/microsoft/nni/tree/master/examples/trials/mnist-nas) example. [Detailed Description](./PPOTuner.md) + +**Requirement of classArgs** + +* **optimize_mode** (*'maximize' or 'minimize'*) - If 'maximize', the tuner will target to maximize metrics. If 'minimize', the tuner will target to minimize metrics. +* **trials_per_update** (*int, optional, default = 20*) - The number of trials to be used for one update. This number is recommended to be larger than `trialConcurrency` and `trialConcurrency` be a aliquot devisor of `trials_per_update`. Note that trials_per_update should be divisible by minibatch_size. +* **epochs_per_update** (*int, optional, default = 4*) - The number of epochs for one update. +* **minibatch_size** (*int, optional, default = 4*) - Mini-batch size (i.e., number of trials for a mini-batch) for the update. Note that, trials_per_update should be divisible by minibatch_size. +* **ent_coef** (*float, optional, default = 0.0*) - Policy entropy coefficient in the optimization objective. +* **lr** (*float, optional, default = 3e-4*) - Learning rate of the model (lstm network), constant. +* **vf_coef** (*float, optional, default = 0.5*) - Value function loss coefficient in the optimization objective. +* **max_grad_norm** (*float, optional, default = 0.5*) - Gradient norm clipping coefficient. +* **gamma** (*float, optional, default = 0.99*) - Discounting factor. +* **lam** (*float, optional, default = 0.95*) - Advantage estimation discounting factor (lambda in the paper). +* **cliprange** (*float, optional, default = 0.2*) - Cliprange in the PPO algorithm, constant. + +**Usage example** + +```yaml +# config.yml +tuner: + builtinTunerName: PPOTuner + classArgs: + optimize_mode: maximize +``` \ No newline at end of file diff --git a/docs/en_US/Tuner/PPOTuner.md b/docs/en_US/Tuner/PPOTuner.md new file mode 100644 index 0000000000..6afdf89503 --- /dev/null +++ b/docs/en_US/Tuner/PPOTuner.md @@ -0,0 +1,20 @@ +PPO Tuner on NNI +=== + +## PPOTuner + +This is a tuner generally for NNI's NAS interface, it uses [ppo algorithm](https://arxiv.org/abs/1707.06347). The implementation inherits the main logic of the implementation [here](https://github.com/openai/baselines/tree/master/baselines/ppo2) (i.e., ppo2 from OpenAI), and is adapted for NAS scenario. + +It could successfully tune the [mnist-nas example](https://github.com/microsoft/nni/tree/master/examples/trials/mnist-nas), and has the following result: + +![](../../img/ppo_mnist.png) + +We also tune [the macro search space for image classification in the enas paper](https://github.com/microsoft/nni/tree/master/examples/trials/nas_cifar10) (with limited epoch number for each trial, i.e., 8 epochs), which is implemented using the NAS interface and tuned with PPOTuner. Use Figure 7 in the [enas paper](https://arxiv.org/pdf/1802.03268.pdf) to show how the search space looks like + +![](../../img/enas_search_space.png) + +The figure above is a chosen architecture, we use it to show how the search space looks like. Each square is a layer whose operation can be chosen from 6 operations. Each dash line is a skip connection, each square layer could choose 0 or 1 skip connection getting the output of a previous layer. __Note that__ in original macro search space each square layer could choose any number of skip connections, while in our implementation it is only allowed to choose 0 or 1. + +The result is shown in figure below (with the experiment config [here](https://github.com/microsoft/nni/blob/master/examples/trials/nas_cifar10/config_ppo.yml)): + +![](../../img/ppo_cifar10.png) diff --git a/docs/img/enas_search_space.png b/docs/img/enas_search_space.png new file mode 100644 index 0000000000000000000000000000000000000000..9280cc37bbbcfa853a1e222b2c876409db9f5c5d GIT binary patch literal 80580 zcmb5Wby(DE+XYH9ba#UiBHi67NGj5xbO}gz3@xC7N~e^9D4=vBrN9sdh%^X@2na}X z?%Ch@zVr8)YwzoQw=pxn=ec9uYpt6k!<$+pgp7n37#Jiv+Umv_7}zKb3`{-*7W~To zsI?aS500~nz6u7$+jOEcdtCT$0&i{e2N)Qnz36{22kuuoU|?L<>8Pui2H9-ojO0i}UL5oE@E8>2iQL08CXe{H{bPS)QcYF0BkVY4y~Y0* zw?V-}D)Nr-(?cWyl{a?}YO2aaE1fm`=O_!3j9FUvZeJn||84fOueOmSEV6!sk2sF_ z)TKPWMX||0SR?KsARsvE{?Qu9MVxOyoqu?7e(F3{u*kVVj($p$&Hs5qK7#Y3-Q^D{ zbbY0}ySuN%qc!s_XyVcQr zxj*0DVlUo!E_kp$&c)7-VUM6gRT!{w26df8VPew?S|ti!xT7Sqaw9l6Ik7t5!`mP+ zx8J#c^_pqm>oI)i_BokWpe6D$0y&V%lyJjy;srsEH+I9jcL)3X1&ZOxm9@3^vqc@( zM)Om4rt6)5{P^L+fKYotNkd1Mu#>@K;4Kn#*DvwEn?>LEPcm7-+N5YwTDu~Q&LjS zK`8wFX^<=m(e23dUla~|o%}3}j7p07aeuypgE0Q^-@o~H?sPrl?|UqR1T{?reU3)!yE&@(B@namH}H2`;7c@`A~TT;`nG#5e9|z`^F{djn~Ym)|;m ze73N#P^y*A`H&8$7B8zHXz7xWPLR0PX}%*IF_t4iWHSP>N^!BX*qcnlzmv93ht9FO zlK*ng@z#&F;0I#`3h8W0n)KeYO+L$**m&$`o1ghTJw0#S_=rT}sCT1c@kmN`DaQ(x z1`1I}B#8$VPS?Uu_LC|Z?+2_7KgF4cc=Yn}djJ0Y%30KS3Jw2zx6jv*yQ~9gEUiu> zj7W|)uQfXKN!TJ>gf!PMiYV|xH8eF@l_K&; zMV^elj*0Z&oZ>T-?x`>++#W9-c*LPbegOyR`*Sp(ODhd7h*Q@6OLZEH9Ouvf@{2{r zccnWPAAzncj*0*Clx?hvVNG`3SoX?q^7HePk_bo;@Bv|v*xd$*&WQ7n-!t!u-1;Z= z_EcX7?JnW^`^LVh(95n1KMg<%S+_ViIyyQyWCne?3_0AIhRlP1IM3EbavSf@8_Pn9 zW~}w0ix6Kv4l4xw1w;WsQal-(O^e?~Vj==b#v;42xEP(;N7el9x1eo@-p54|k+PcG zwKfVN`|DX*^hh54oGTea@(KzR7r$ydv9Pc>xwy>x@7=wt`{2JL(+wr~Xq6}PzzJ2P zvfQ7lb~KVJ6}-2?XHxzNGKE|r>`1sPhqiXGAtqyIW0I$Z&gv2p-BpEhy(f<7=UCx@ z5t2c^E`N2=`TyYfh<@Q{AZv23GJIF?y{DomT)*yOFyM(dGX($`A(nZ{{0qfYil_Bo3Fmic7%tXpBzl_jY8ec#QLAF zTZ54wdbm}5pioq84tg-5h_|f0rKQTdRU#a`_nrOC1cML8<$nyQCzP=X`=O=!Z zF5@LmPEI)X(b3UI85CTa>|x9OB)`t-{^EE?S zl}_iOo7lDg2@i&lziT4{>1-6=8eB6H5(eJb_mbm%jp56cx_|RW`-1!nm&}^z7s|mF zN=(Fr6t$w6X^j?UnsNgvxL2?{v zK)wZ^pGfo^v8iMev)v-?t}(V+KB^HsSZLnp_b{~CKH;R+oS z(+#n)B1Q?fmBGi@E6Lt|e&XM@&yN54@#^DDLGUqS1!u&vysdQfLKK=?&1g zAt)$_;irE3?@#)_@2|{+khQj*HdV%DnyHMBmytiydGt>8MRkw^s@vr9QlX)t)Kt2E zpbu$l4?>Xe4fpl!%ccC+c!aP*qy3>z+L6~iRzEL(>pwMdHi&xTE zcH_CMj0_2@JpE$mzwPK3zqht3E5g21o2#g(G~6bz+x?U+M#iS7o~82u%Z4cQd<$^J zbe&@+^qD^pFc7UIiwt5;P?`NUrwA4y%hKVPpxTI2p2K~J6UP&b+4mp`Y;0`y&x;KT z6-P%$#p~9W2h#7)wdu&GBqePGQlLRn`K5lLVmovNNO3j#h96Bn_biJ4KndVuW$^ty zBVj1FD2ia3^16TDAR)>tEiFylV;OGX*Vo#Lt6W)i$gU;80UdrDls$G@MGg=Yt{8f+ zZrAv(42nPsAmPvpQ;VJyg0Xb=LXF#@mUWBjfT4Vi=xY;F{R7dI>(RwsZ10X`x{h`W~Qbx z``?@#AIr$eO`0m@p-*2#OmzxT|Q z_TiKX>a3XSj5O#Cg7$m6;$}W1Opl6&AJqB3KgXOy_k~3qmCyxhc=QBFi|?x+O)U8Z}yx_8yOkFaXQy; zXHjngnrP^g*wcZ}>gnkr2RK;w_U+r)SnQEMv#o*m{rvoL9$F12TQqswLyZJbmx_iu z*x1-{1a(js%q`HVQI}hw6e+0pp!&|+lI&~-0h)i#CAR#3!8&B z{c}{`-TbV3>AAM|MS z;|GUoL7TR301&o+v~oSHn1U*sE#}-2d0CJK=)_Urzw_8TIsW?--fRQ=975_{L&G(* z*LqC1Cru@3PL6gukreaB%;FtX{Ut;Adq$ixF}LIfDZE znVFe}buClVDfqA<>@$}A#omM@DjrGCnan6Tc6K`ezTT$myZ~y~=pb6lucIS~4eHL9 z8cPJ!{mqRHBLn&uEo9df6(iu<)IYntaTGO&2X>`+@|52UQ2Aq8>3esFJt))N_x}BZ zUT}R@^*@O1f+~WhI9WWeV)@=<<7+J&NuupLw_AXEhXm~J-W_g0PX?Ke>Lz5a^Vv=# zIp%lnz=2~eLbO+IN8ss{2A50Jqd~^{y7YrJJ!EI(<;}m}0V%zH^!EJh%!QwA#o5x* zQb36(GdYv zkk{J2_A|u2t4;!x19MBu(2$V-PzywW*Z23Sx6kk0JyF1wq7me;nZVu>G=2fVnZL)j zGlGPGASo?vbPgAt{~aA2;yWJWB^phuP@z#AP-6UO(RZg;3}7JgXa_U;F_+fn=H|o~ z4wyWO1l`^O07hzP=jG==So@^oo&>mnS=u{_gu?!=F^9!-axb@BNzd!oueI6Jp+*|B`$8&79`T+g7{DYU#$$CuU`S6K^PcQ z@3i0jKxlD!B{wO)S*=X>DP*pS22%J{Kvveoe{-{VC1BClaCg1&B#8O8;K|9!40;O` zfDPA(D>IT%aZF51PM)3bt=>2u{|=|iPzDc8%BIM?qyK&6x^0I%yiR{NCcFLInywco zYlV&uH&V6U z<7uK(lX5gx!+vcMbiv6xj{$DFy1I@d#-UX3T)A?E)fuw+Ia1;Ou z4JKz~LIRtNuR6Vd>iYn=UkUdwFP?e@Zhce1eg||$UArNVRYw>@G@N!$PL8a!G-Ly+ z*n?|!8M2i;=yALwuM(*+k8gyo@Y(Q7+>x6E~cjt{AAdk3hH|4|E)~($7 zIY?o~8LuBL0@f{=hjmcbP81HHeFM2*C?Mj!pv=ZrxW4Wpo2jr5bsXZJgh3Rk_$w|h zjzu9v{4}-F{;qC2`aG&$0o3qa9nQ|pHF=W36=63T!&U&?VBMmy{t03Jsu)lEn|L0l#w; zLM2C{dkF@A1Ljqw&ruAQE!sghh(AzDcNV&#rfD4kTLOu^HP?O$zeqtsM)vRU(5BIg zrH4+)#^XzM_M2~KUD4Q(oZ75>%JndELoHIF{|ZbD;!L{6wcY>M?XymC+$DHC-rE59 zwCHC#!sQ~eX?KE;b{3(~0$NNH-X?YY3o&j!)u{kI0m=}7yVM=Xs`|E! za&qOlkI=Ut%V?!BCkZ31WbgG85C4{?rlIkN_JMqS>$O#L40Vg@*`K}86HyV@o9J>y zjZlt?XTvN~&fwPXn$9HE&y#*@Sgd+PXW#J8MSwRyiH`&*l)&RX$6@QDZ-aw_5YC*$ z{Vb9mcVN_YX_$big-Ox_ain~-&>egA#caLyyOOe_Tjg_$pnK+HAa zNk<@-9e8+nj#3e7-b9OWioiB4Laatz>~>CaaFwk~LX%jKXB@Eq;qFY6HSXnsWO7UoLKvBsfnho`wgJXS?_Z z&x<_E4IWkMRLS6PM|E!S^H5O{XhY-T+I$-d=??T@)c~p6T$>Ck7^W12L6$J)m4xd^ zVQK7|cde~*7GCP(7T9Yz6ER$Yr-z=%>(xwEUMub(U7-k*Qoc{u{t4uQ-&MeTSu25SHL7gZ+Z-XlQs ztDBo43tgDHTH%}P>y@RYYCkOWI%}4emrF}ZVDyX?ep3oW9h5lEuD!jzi7#JRrM#GX zyeELx;>3^|#rLGTRbN~(eQT$G(JNYlAgw#!&XIA*8+ZrEXR7CFFu^N7;GBXIBknqJ z1xR9hjCJ53qY5(B2HWY7h5o+2!?{vIB$fu2dITnV$g&<2N|~!CuB|6Aoj-Mk^s}EsRdH#K_&QMz0^k zIZx+JpjJlMLVkP&{>bx6kIQ`&mgl!O#ZXkH4$%`+>ey)i$?2a?=x$oQ1gOnY8TD4} z1V%}Z%8CjtWtar70Vx2PgX~rlbNX}#dP{gbp3V(KQj$(K@VhsbMW8xUGBD^Yi(H)j zHIR~Ognnw2_R_4jptzW0^D2=)eFTgSdc@DI7L{V~wbfodPJmpu zc<+(415Db26o9cj*##d}Su#hJgIpd1R>q+WYutfTG#(S8Ey&8xw+0lxfr7TSyR&hM z!Bn308|HIg~q)S%#sSBjiub zR8_G^zBWTQ0TMd$l|4N)Z)oE}Bcz1H#MiBwg!PyDzQagN=8rGz$ON1%#2$VNq2S%>w68ybV#T#vM>>5s%VO;!#`(dYtzkxYw z5B}RfJw6V!h%ri%)6#wc^3EcMQF&`|5xN8!vUvC@zoogk5f4LKntbpcYUX>ClU??+XS6iY zP-1EMjFC=49>7HL-fr}=MixU>F47_V7AC&U%uF0vSdry~CE}efVQu~X?Mg%Xh9a3> zpcm_@faQB2lN-{9;4AD71LH!D*RjmVp2HaZF+NxQ0e;~2EKDIAo14kPE8E*?zlvKE zlasMLQIR+{wovtT1%G@1oaJpBbrX3N`&wP$BYc-~mb#FOvtuG^US$T>v`H9PA%ciT zHK|$4d$e|cf0t@9M$>Mcj7kx0v9Z07=M;1A+@HUDCsE@?%3-^xG`ypT2=ak~^?SBi ztjEK<{!Z662puV&3qW-+Ta2d-7g>zP+LcDJv++QE06#eH#x$tC z@U-4}%;{)S+c@ZJQ-nn~wim&6+B($+$hSq#ytNQ6T;HqB>*Ld3aU3Z{ZNViQQb%MK z6tMaiX=-Xhk9z=Dg20~Mcw5nF7yxs-!7l;Pq&QV-M#dQ^HOVc_&HjJpSPBF*vPE;D zcRCt0x3;1M*Urd7+O*WPG!QannXsShAp1|@i#O;pPQ9^Ld$R^{NRH=`$}A%)Ry$6V zcXcl@B?V}yY!v2vxjqxXz`b2O5=I=qa+Xmb5)7-1ud=iI3u2KcKv6JFRb$t%7uJNz zSyw6g?5=TfCrEz=WTbc+ASFj_so!&Y^gN!Fg|HFFzO4>4v#jjw@k|wrWav6IF)(?G z*c&h-g6L>yG`0zEmMi+}|AN_V@N%j+g&6KlF9M61QbJ1;bKr3);<3@`INcNKL{ap_ zv$!`tpln(^IPvV+v!nr;PKxxzwLmuus9M%_4g{XtDc&b-Db>6|6`-~>sH|o3nVcN{ z{-{~ta1HAAu;R?ydkLkbTqAu6DAx5~zlhx1MI5V9LD=H1^u54i5EBzeRrJwCrybxh(e7+Xc8L9<5CJdwojp;Ejdt!Y08Rmby2W8c&EPt!am8dnM z&T!7O3*>`=8M;AzBmA;z9B_!Azkl5J*c8!R(Hq`H=&Fm9s02p&M&R-$*zr-qc|%k3 zmpjbwY#qR*h z!=P&5{0>>w+uQ52JEcTYFj`6WOON5IsZf`##SQ99AithVibO9wq!F;leCDCWT?LaL z2)!F9w#c)$UO!|vv!4ZW^Wk>6d3apk`f0FU7>{){!I`x~2vS68akMxnwb1CU7PLlQ zb~HB!1IBc%zJzqg9B^%YM{-d+IH--I7WEY4paS2hv@?L*AusRK8EK@YKn#yBAZLn%6K&Q3o`pRd{pc%MI?c zuj~;PnQMPKFY$~6H>Z-qqDWa}2OUH$HYB-y?rU3A@kY0m6 zcn+AX>FMm-qq~QPUp{}nN(b%&SIAwoj{&WM!sM)zVAUA-$A+K(6;LF5J|rn$Dx>?; z*=(Wr_|^l1Hw|OZNqm%Q&dFnr;^Ja}qo8whu{j&7RjwNTDIR~3Bmyc-`4FYv20QxM zZcXW?f))bftV_(tu?TX-Vqvon3h(q=K6{annKVn7AAEDoJB>X3fcky9b&^BvT=-OUp(;9B z1qe&t1T75>nCu1gx%sfx^YijRNgo=RyvbXEbr5lOMDY|)qciN7lk7ndV^3Ia!0(yW zvBF5S+RG*v-~qbV5Q{f6J$<6Hh6JTa-9E6GIo?=GsgpIN?hB7T$kh=x)4Im0r33sonB;pe;?r~SUfY;dH-*=gA15>*_L1UOCWwU|Y{vbJ zZ?%Nb00E1InHi6inVycWqU8o_I+2trKZ{72US+6J>qK7#bQ z9EhAL{Y@aKCOP{EM@!}&o9boT0u|B-NJUbze9me_rYtHbQ+E$tiHNX9VRjSnuX7lB z0!dWlaJ68N02Q`6Op`N#;Q{6h2No~}1QP0qLY6X6!V zW_Xe78k&f?{b#NnyS%p)WB2}S(_5I&fqiu$MG2a|x}8dUy-?QwCCC_UaWLL%XlQ^I z91h+VhCR)?j5IbSn-|&?@(zllna1djJ!Mhvl!}rs(giocwcjoynT+ShKMP?Bq5Cod zlQIXp<$8`TmC+cjjX8+PuCM8Bm~$eF15?-TgKntt10YaG8!&LOXA3&UA+m@>gE*oN zchF{B5(Yu4TN}zq;3iaCTwKJ^Uyw+fSiesF7kUBM87AcuV1QdTsQhk8e$5i)cH4Yu z1IkzsZo}J;K(>PV=)DWfj!gK8xYuv^1yJbr4-X|JB+Be%z^QTj_U(OfASr-x$yrMg zcDjcaYNm*&z&6H1F!1moxJ(%ef@p0?Nr_=X8gNszH#Ei61V~H>k^K^)qP{SzApmr} zfjow8;KzG_r7#(TmaZzz>&rVj6g``o0CW_!y>SFlP0~Zk#W50=Y{u(KCIgK&N4^c(snbZh0%CupsHB)7g^j79YxEfrO7z%9Gsn<3CqV2pH) z?FFs6EP_d!25Sqroafw?MQ0Y4mXm&Jh^r}10u{weOG`byy()Q#(r-(h?r=a*S^WL| z(^=#+e>%uOXs5|1$UIoPbLWm9cxkS*Lq%ti3qYwN$vL>f9s+5&5V1JMKGWjwe&fcC z9FAl*d$uK0L|P#j5|Y18xlP40!sofzC zq19B35dkXyGg*<2itYyI%sTWPZz~F5&3?dKWYT%XF)U0TL57*h&BDTBaj^%(FdQ3I zyp)3z902LTU&tuTTc8wX8Uq;*&Mk$Y9gJ~cQXy&{D&jg2ECwvdM7@9!RHwEagoLOK zAdR>lsylM&?BKYt>w+Heau$3IoXk{I^}r0B-CE-7d{>!HoZg_5!M$U)F)d1fz7Rff z-xY0Wc-GcGpwRu*(Hr*PUHTA>jX!}35dT@ULha{DG#Lx7JbQQd8_N=n9@CZ;j)m*p(<~`1jeKyV$@-?T5wz9-+7pZt8Fa@H*3T9#Gp#aaSF(7VtW#`g zYjO(>f~QVgNGh53!|sxER|_O-=3-K^}Ks(tA1P&miC%w6DX!&{b4 zXS=)t{y&ai>j)|@#Ixtu*DUXxVS=t40EI@S;wf}Dl~_Cwg>C{NwL#Pcx+=h9&4={S z!gh#AV;ExGj-Wy<{`z&pJp*_LC3*S02ol*RycT^601iCvAA4|ncY!<*9fjbkD6@Od zX>}U{+&2pI7eu%jrsuQ))KCnaS~Ly9PxqJ>CxLB0>2BN(NGxJw9b(>(rf|9eB7+|b z^%Odp56B`qJyLE0o3oyi<*D!9`EP3RJ6dz_Pz%{Gj|pU?OHbrMZ7zESPOCo~8%ZGv z0cUrgAa03e!6!I1%m|{y%tt6Mu3f)g4az;v7eF{_ty$5(l8fGAu0#GZ*mwN`f>0Ke z*``?#OTkJo{)Gw1k;g+0;PRWDzp_n$f4AK8I*PXEYH9nm#jN5wG|Te&E7UoR#7Qh0 zA@DQ7odCH*5l=q%-jivB?E?Xml-HGple1$nQC4|!H>|Hv!a#@Koau}Fxw?v5q(zUF z$@ScLjCQnpb^N&>yIdQ4Lq3uM_=kl1%Bg}8R(3??daof-OMcHcKj2|AGZ-}DHOY*Q zjfnx6=gI|o!r-B4-;^yQbDAJcJBM04Mu%&6x84$pRW9JuhYuR(_5(Zb40DS~RgPdJ zs_ZlUm$V>hR5ScnpSLlMhm!GdO~1Bh`N#AE0yeNL@=~)f0dkB!&*nzpB?palnzT=?^-_k8j`T^$z9K z-nT8BKjSsx#C2!rjw}~_2ik9VczBa?W`R*mFWUMR^dn4OMs-@N`~Hs>iJr=qxOD$v zKk&dsMPW>_B1)a^fKVVi^(|YMj8*>4yLUg}8rd!t&|^wIHE1gQ1s7-oo{0$61{@6( zsm3>N4B2GS3OnQx8oMk4OB{0y^R{M!_!U$?v~OsYh<$Zx@$COs-GC=`RF*WRG&p06 zS_6@z)ACd$#)PBc9XLATD^&hQ?G}NpaU9JUMK5?KqwO5M^ezqjCbBL-i-WHiKB-m%c;_Tq0U=DJg>t zYqI+21^=RG0x~kKtbUow$+GjUX|a=dIj>pi25)f`r>8&ozhPoU)kQ(hylrUk^Y&Jz zKj3=P0h<~PM)J~>!~FCO8Y_wwb)1$yOd2_+VStAaLsLWLto%$YPIn{;sxIo<`doYq{HQlw%}C6%ED zBKD49Fxt2#CiD*PW?5W|Ni|H8r50RYsj-5WpG^jL5p1?xyY&*S&3?EP+K%CVwD19} zdPOsOz7FiSNC)XFeIJ*7zU;{+0Zpk4DCha&U(uLS8BTG-M3*rf0055$*dh)018Hw9 zVO%%u8=C2R4J8@uCuqf9TA!86e`Xxz*2dyC1=O!)y_5dQM8cYwtgNhKzJi_`VLHoi z5)$d?S0*qNf#AjQkUB^js=C4IHGgxBoq04rKHu?ZRaw5gyj(oL1;`zEHSi5VIzdxj z{P}a>eB|;Xq53@slMV? zJEAgVar8Y2c>szhAiVLRq0d0MeW|rI_<)Zjscixew0Kep+DbrlLJ97WV^}|!e92Az z8Z%Jk7iEkP#Z_BIa?L=MT=u2civ{DGr$L50lZK`g@m>5o3N9I)#8wEbH1cg9@I!{l z`-9vBj}bmHGO{t$#ZuNhy1{;=0qpn{HX(_gRxlh)0mol?A1hn;!-OFD=sYAectHQO?-N6cw=p-Gv;b zJaf@^K!UuCz!!Z4(9Rp+3SBi|Ty>|4|e z13h_vN=kCF>f-0V1a^5%yUI6|O0S#2&(b7>#KywvS^|sB>;)vy;{vbGf6!!ajBgH+ zRAo)ge|pyB?*O#2F>}JMk8{aDBUCf_#NnUJH3#rb0M44!`miO?X1I3^x))+Tm$JSP zny9NTuaZkUiu#@FWW2$NKwSA_Jc6QBG6I5PH;rt=KY5DbAyA2?N}}+f{^yEP_g}W6 zO;zXW0!7(CU{lUUX`n1FZ`%sgx1iQxo_(6bxxy(RPy^f&7jc#xv=%2D7F?Sph?SvR z7~vq&)`Q5kq5&v%6^N>-)ckl@NzjKxfTf*hS03EBH7awAJtz#8dvspd+igx&J%0Rn z`yBWLo-)f>blB#``L1118}8JWkx{70E(ym+A6 zOaUz9XYg*+a!E1dp*3qbMVG{OIGR)AP zy{Rxf9scfoVCqP%7a{OpnpUY|t9IE}UFsFS=<1G$h|syv0wSY_o-7XQ7s907b#nzE z40!=8JuvDSc!L;#gTNAqa`n z;P5r7=57F#-V_&hVE zJ)kU~*NO?n8wNSOCF7Hz6TG*n=w^M>N3S|Cat0A0HoV zJYpcgl<862#{tNcPFM-pj%n$K)SiHqLVKS;72a`-6G&TjG`WY|h;|lO3_m_S?G09@ ze@(%9!y>hvdy!${E4IEnOnT5wVM_eDoI~N2&@thn3dQ(3Na}WW>^(Y%2Hgpm!Ravh z{?Kf2PV~-o9L|hZ%A3U#5kvP!MtATCe@LcLJJ^*wS|XGH*B|6l0us67qY(njS~rWv zOMny6Sgwe8w+Av75FMcAKJ#`1rVq^V4kk?5+JiU6r|hApKq)AB+Xkb`*S<~KqfVC? z%(@a7I@8n1^xH?sL0-getPUxnp7{fJYTQ`D-o(Gcv=9JxSr8pcLuQ|&=h>K)@91rT zOW?|Y$KYi;%+CgmupL1BJCErnB|*9kN;h(#?qwZ5o5B935Wz zu7UYCp{FoEHuU`g62c{wC%D+Y%LoTV-_LL(2hYztVH$z6cI~o_d-u*Y$753iyC0}AjD3KaCF&#LlV$=_l6_6XnnLoh{{VwPgIMMrKRP=V z`E2y$e%Ea~raz5|o*q?Ma3xS_C=aX4e;ox)$GxC$V4$Kdg~C4{^i&%vwy!T1Kp=*@XFh>o3>x%jXv+7NMLUSieQ=() zp~E{Bwwv65*;rmyv%ME71#|_Le&(C+C91`YKnemy%EM-$3+M%H)Jhd;hw%coZN)`0 zg>k`HjLmp&Q{)_=3=jijc`|B8zQ4W*SY46$TzOSROjRRzf8EHAE%@%v$Rqo@K(5Er z$i@H;pT;wQW0rUB9G#q$+ZA+yHyIWmX3@(+yFWK19W=6`-{WVQw2v%;aRh8ns((etcgmSMz@Ltw(c>?-j{lnB$IQ?YdCMzKAfa(sx0S9;VC3;H*QJT*x?(*o3 zh0G(CqrKef8#OW<_)9vlaBO^k8!Q!zf-j@)%kO|(#`%vk3{1R0LpT-GeuLx##(0T~ z6X;~NM*>@gSeVu57QM|j%q4n<@GRAVz>!TOslx?)exyxB7uruhajnqWv5hFl{YBfu zs3Y5F8(gVG$=G!0iX%#E*}#x|xVNX9A+lOo3nn=5h>Z+LTwmebqk(NoL(^MLO}rH@ zG0@H@!hEtm6+yFmF_wvuiau|*t9n-zgZCV+DJeP$L@REP!=C}&b2is zU$j+LRYIzfyv%+c|^4H5&lB-q+ph}*%Qm} z;9|c5@z+Y#=xyL!?1UgCeP36nOC^-DS?0&87ROiz3Q*5Co;a*b2pJhBv#3E|prD?8 z>K;@@Gi%p?+#qSvytfuXRXmJdO~(SVf^IUY2ml}Uo6i}LczH|DDO+NVBkAqToz zTKK%#C!c3A`T&xargsJjLD@%Zw4qDw2S{k+?3=QEN@M+xN4c7=zHaw>HuWm=n?v(*V6=-}mWDL?jsP0r~)# zdrD6B`T+>QJ)e{itJJTt!efW!saVk&eD7Wcsk^qjDKtJ_^STwNe#cEAF>?9Qg<6%b z)yk~F1$XqbYV6zoF{gq??>)ne*5+m)S~DnD<8Q7nr(0~&ggaRj!2rh{qt4u1|+S|c;lPqk*pj-o!MFSbdMflN< zJ?(*%_!aSc-bOp?U`Ini@9FANW=t#8K|WC}{OJ0RD(PmPG*Y0kuh?xNd9veXCL-;YM1 z9*8-J&cojW+rK7Pxnb-OnejjkkbJoM*lc4qC4}<)=)`)7jLBChC*$E`Bubn~JxRKX z9vDGh2RQ#}VAT)cL)TldR`e9v&mhh9!J8Ze)Y+8kpP6_3z(qjKC|3g|9!k{hK{h?g z#UJ2832&Hg1QH zuW!jy@eg2>qgM!NSdi}483aMbH`GQ<7xFIQkNyo~2kYXk_03K6K#|UK)vg8JVC~tv z+qQt&g@uLJf#Du5X9B^o0h)c-l~^QgcTbP1M&VZ&Hx`t7RhE{gHPCir8!K6|EuAuS zOH+4Q0i#^jsFU!NW|Wm2@{>F&u5iLa1wKkT8lT#w%ap zL-hjwHOxi-OaddEEm$Vi#r;>gF6XTK4AgQJ5`nfVs2Gj#BNisY!@~O0SaS2fMBXZf z`h=>yEA2nrk_HAkFBcbWqBVz1w6qI|MaP*{myHk&SiM(Q0(#tAcK>*NGgqyoa`}jb zQ|I+eO@NMyDi$Mb=IAI$*OcSPU%c!}Z(N;{!yx(6durm^G4Dqj_uc}xK4VQ!$91_u zBXtM1fNX7T6$`AblC3ZTn#J+@f^(g{4I;4e`W zq&#Rm*#7Yy{{KiCo469jFAzQqr#^nncwxQ!wbNTXqeedA8AY8QdLz2&SmwvM9|4TUn&{eqj$#H3hOE_K{pK_*JP<^*~aTf&v4J$ zTLDwUmiuHtpkw*eFIbfY>u}L$;>QebQXgfC7Opzq)+630QHeSHyMV1sG9ZW>w%?J@EH08`1>O z367A>`+M^B(XhKikdRzn#(8h5+FT5hjm_}=HjBy;g-2#fvX=_rNzX= z4mz`;Q(QCA7m7#KEi4kfGI|WSa&2|>>9nB4(-7eNLF7O$kqwBUH^{E4!CM0@-?*7t zTh{^#v8(v|rUZ;RG5ubE3RjBb@6653K``CYWR3a*RfW(JD|!%^h$q!L=X%+OK{YTf zAk8$fnZR7fNI~)VYX1UwZslrdKSMja2WAv}62`uPNz=XaQs5qG5vQ>KVjk|OKm#*0 ziG1q5t-qiruB{*J6PPzNvfi`?h;W=3f_3c3`s_pRzkmPC-XDty`oj=z?k*osh?czq z`?N&==bJ;pd*JP@-TmoFbqUNy^h`{p1xw3GvLXAGsj2!v)qrRO^Ab9QPD;YT20WzQ z2R>~qL}e@vjV(z0Q2pws6XdzycEzA8bo;->r&Tg?l0R^2(X)6%cJRz5oQ}8-X<&Z5 z^;5Wi5m8Y}s-HkB!%qvDg|~thva8b>4aiLN}2Nwn6@80cwm9^Xd{qn<_GAdV3EeN zZ+L!SX!=t?TnFeB%<{(V(h(MwkWW!nHO@1 zN8%viL6kXejB2;ZQ6~c~<70A>gGL89-SKE5gAfLD0id7!B?DtEe=RPmw5~W^gT_aO zn;+vGt6|~0IWFK8rz7ee|5) zUVXOiLpZENp(zD%%X3zLJHr3^$Zy z=X)QoJbyJzshX+>hX5SdAtz^L z6=Vp1Art)`7N5+(BDO#O5j8#C4Ds@0sWc5ZQuyg8R|_7Xe+~#fK0Xsu?!1G!9}T0( zU3HiXgshsn2I|Wfm+ycNKMk#VGeLzkdTk}M zdlJ>%-3_J|bH4$TxRA>Wpsmnrx>HZi^t!h=P7?V{(-23erzry%7*7mCCtfj74I3$hDr*=3h3aXe+Dv{DgiE6+8o20`!nY<4$5u4>aNN)p{} zN1_{;K!YDkF-(+O1SxudTj<=~z0TjZ1V066kK!?ypWIzsvWBGCzJas%$#-LLythEj z3cHd)@Hy>`VI!)@P6htPleg`#Aj%YS1({OUh1e&=rF`0yYc{i+9xNv3)if%1YQ(j%z?lk?9 z&nJ}g=p1-kaEpb8o`R|k){i@&>W^%mE-McnN0}+^)q){c4t#HlPm&H`w+}D{|F6vm zn0(i4ReyrR735^9f3&a{G4x07F%9ll>fb@!d0WMT-ldO&vNZ29nK*O!Z+$a@=`O21 zGat!V4;dvsNAz0Qo;-C-GgJu(#TNIaV&Kzp@$o0^o}k&kp0PMrV^!CF9urxFJ{F_A zJvZ%d?C0DwP%6wd zy}E!!fg7ngc{yPihIu0&Xa2gRByDXF@IBNcCCMo%&`L6@`LdUaF9{FMPXpV+yJbL2 zBbI(BDihTNTqAl9L2Zx>)Ks%b7R@%I2E-1*?)#J7bXW$b3xDFri>9+CL`SJpU~yJg7*3M@fK&pHaQj!3BWDb2v-fw z3wmS%VgRgc^*!JP+W?bDftshX(7$_3P(1WEKe2(O@P!Bm+S&>$AO35Y-DoKT%5-1-#n1J12-MyybV|iK z7TzXL!}*z@Ji$+-NV_=rYTNLgNLo|>ADJf=DnoK$x+A<5wwa)!uA|tJN`mtM;0{=$ zFkH^+8XFr^Qi#HhYt3p|x3wQ+9D1MfT2=8C%p~bo9K9xyZ1w@WCbSx@Bj8S8{s*n~W#95!<^5qwyHNN-=l|j8x&xv7|F}CNJJ};- zuTUZ@BeF;K9vNkGLS~#DQnp00_b7=Aab`vtk*{&-NXyDdX8hjw`?nnTJkRI(eBSTZ zdW&c2#93B5pS78aw5xufEtZ|YUHrB@(@Y9IUsv|&`}|E)51+$Bg2q4<>{eWQ>FkBxMaT_XeWcLrx>sG=qtDlW7&8F1?(KJQ=i9RO+vE^UcfGBsRgR_j6w1 z8jYppe*`IUvGh4au&45ROf)}M+#O!msrzZ=qz>mADXbvcz*T$=;+WonL?rFzgYz@c z15Tw$Y=442Vh}M8D2UzlPgG>sZm-#<@(Au9u>HM%|Guej{(51fd2P<6Ur56cK}Ts# z15#2_(8Jp2Q6caf^SGTTt+C3Fp5|5WfqOM)hxcO6PdOvC8CE{*mBd9*d!$(8+%%|j z{3hYwfBdL*wUX9 zl*W|*7desB*n*)Cwx-^9uj1ntBuPNk7JIcF3Pv>1%9y4L8KdgI8SvEAVQ7hnH@r)s zO`0oZyi}~bUi{K*CigEu&rnT$AIUvyg?2xTsEt0dDY-#$`)CHyS4~V`J~lDoKk$$Z zZG4vr^?Iq$@VeX_pz|P?VMk>s2m3D%o&#+cP^>#!S-JsOnrL#+wn-LwGBZgDk>_j5 zREZQcf<&D7(WH4m*_zR| zY5Npz_DR2;0rl$nm==?^nB)i!dUzMrlYWT|6A#V{2WN=y<%-+h@%4oOMtkMLDxjl! z_+X!KfJh#2>8UkdlIoKB%}eD0HMaZg!2|7Kbhz$VvPkQoPp{_7#s9UfuF{+{*$r2&XI!M{b z4oJe7lFcLZj7IIfQz*GDZdJs9z1^cm9S~h_5+m``8q0I zmMY?9ON#~OH3{*?`#P|Lh}q4uLR#J(?)6*&fh{155M*)>QYNSD zb|kJ)jf!*A>#(5|vqWTrH&Hw*#-E2?dXrK{EwH3!goG}NE($+{_L%ZD6gVKRsaWU# z_mF9>7@n>%Yc~1Z5!7KnfBh=nV4mh5B71h|Nh<4NZ)?y82Xa z#5MoJ#!Ev{xgVwlIRGujS0f0AgC%d}3mStyu0^v$?b2A7wd$ODqiSMw9?`>SJKs}q3uO^nKB*hf>LvnUQ6|OnER<87vj`B2MzwTvcv}69M zLdVYc(ffD*^78!uv8xpaMr$2tPqVQ92ABi{(kR&f)czxv<5<5BgvvR+D*EPtdQb5lO4s$3gom;ioP!zqyH!fk3d3s)v1T%TP+@pBcWkR1*`(zBfU$2TGUy2kZn2?Npsj^!9q%xX3ohZ{${?qyv0A00bQZ_NL$@ zSc(!>*GSP#3Ja45C0YSBPO0Ks!G$p{-IC2LF8OC4P>5-#m`9Pf(hMci3$_@X32?W#K1q4qI`sRqd8zvW%s;~^-Sx^qt1@I!L$NZT2+JB zR8@3l@cNv<+6UUj>3!~Dqv9OA#{#=X-W*EW?GCqj-$tGaiOsLmkMS{8rPs5P7*$O;;dR6w7Z&7->T(P z;)Sj6k!?^g2o|jj<%zkGu}c)`)`}jJmTSZ7j(zOzQvbh)q)3Bs}r_9(y__s3JtkH7>Th%dgML;K1=Vobp z8g(v<9)>D;SdR?_v6iWnf+Xy~-#%$?Z|~`UoJoq~59PkFrCcU2{*0zA;0Jt%;rJta zaG{-;-($%L&KgaiN^{&+OOsP6(wzcT@s|@#OG_4&lI-FhQZUEGbzR6_0d(y_{No3% zu1;BO3#4|)lb9(w%4bG7=0w3w7xRf*)x#6n{O-6G$#8sm-|(6fE|b2gu~LXDQgiP3 zplKlW-2Tm~uNg9d^w`fY7A3p#PFn8O*WL3WDFYWeJ699{fC_6bB;s=PM(^?alui!rpM56qO0v0r<{&mA7lN5Um|hvdV$$U76S9 zZZroi&?~8~@!jX;T_1CwW_RMZam%2}xE?S|Wn!Uj@kD!8&DV%^kc`x#356_{y=d8Xri&M1;Uq6b$J~?TqDL&{tZK0+x(_r z3lfM)aLW}jy#w!d#5FHuFl3JU7Idmq5p9G<@=>xes}}QF=UbmAeB7q7F85Sxh_ z&RB!tX6${)2IaAQH!uLI@Z_GfYg2dB@B=czfHz7>W!NZ!P)Dm-u;^09r^!hMw1$%~ z6+)IjkDgDP4b}SvuUC`%WQi1mvG1WJO*f2-d!uaLU@o_Y^p2 zTlY3Mp+oKk3tN1JeLP&gyzUEl&gjo0P29WNhnQ35aUn+_Q;bw1uEKdhC;>*Asg32%A&S(3Y$EUw4)Tm zYv|?TT^5QMiDY%`5O?n22kLe1-C)tXmZ1o@%L5L26f2a^q9^z?rf3cUX~+VqJuAXv zOkqxj``|UrHPP@&?%R*_fS~mn(%@BbL&pEF+ezf!yT?sItF9*}O=zOo7N4d@$8w@>y zLXZr{PlMQd%-z(Rc$-gaIDs}>+=f4VM-x7_932h~4;-AtNl@a^8^}FBY=n#wSex&) z=(8!JIkF!D@&^dZekpie2;$p7`kA@+@AaD!0&|Fc_tik4@${+>JNM13Lv`IOk=t0B z-~AsynxMv{5jd5oVF||^gC`d_q;4>CNu)f^Exd5~Hg=?Jq#Fq5XzMBfY~9 zch)W*&Pj-HC-y_kq6ZKY-m}R0ZTI_Y@wgf(mkDXg@i6OH5>k?=Q0i;FhVpS6qIIvo{|qQU{gJB z1KbU5Ahbv%hY(R>5IKAeM5Q_m*nfY;TXSg5O5*AwLajG&=WqW0BB^9>>9z}Ik*QQB zJFrMiJ9K{)8Ic!CEnA3!thi;CA|m67r}S1AA2gGWZ+F1&GG0ul=mRe zR^w@<=2S4VwdHWT^M-2!QX9dk6{FitNbW%rvh_Yh6X+lym_k@UlCU`h_xUjj`1!Qa z=RB2J2-#!Tjt9Vt@D&7wV^Sc85Mml`V?X!5P0ICB+PX0`c2qLLD5mU&LZ|UwCG0}* z+Nea`HFv1BkZLx{qURn`JA0;5#IE}3|8KGqs;Ew% zZ8nu7;?KYESu{e5fuZ~h)WiTx)Tj0GBbzA}%XFNunPJRo<~z~|&j z;Rn2Aq^09ja5d&)p3FzPtF4D$o}v46FQa;CphDq`pFEGr70+Z6zMvSA=5?P=<4&Ps z?&{W?@V(s0zKSsFRw)qAcZKhbLZ>Pd0So{7;CpztiLux`{Qx{I4Grm;#~iX7fFV6t za%3!lS0!5dCrK!j=II{(EYgz~R$;Sq&Z2@EICQZF(q|j8*S{a6TFKvy2ys;5*m2fK zHCR|z;FZt3_zxZaslPqoY+-t7}OrF@l;<5|qTAx*DM5}Lw5%0S$jnQ00NhU=-I_$*}%%6c0; z58HVno$BYgYEvN+SZyr;diTY4fqD;?GEEBv73OCB{&M@yohJMCNF7FfYTPCVOhVYa z2hjobVV_2!J}u$VYJY^PKyCmiTC_d}zEC({j81FsaT$ex?sEzlb^y=Z?D60^@isGK zjL$~w!Si(gCU=caGz@A(Bdg$YR!l#9W#4httNta@6TRxp=Pw#C>LYO>{ez!G?=+8y zr66VJbUS>f2L5#Oe1I~;6W*Nt!7fkG#DLK9;6Z^28II0I-_?VpNdlD#OLy&0fxw3X z&mSM2aeQll8rYDAUCE}^{_dk&zs}%d)$T0DEPq>GFuN<5rx`u}8||1YV9%elV)gTR zvC~*Df0C$vDfa%a4DU~(onxx^74PX=)MQDYOQ7D_ES64xh&}>Pj*E+HN?XWM6wN`4 z-To#e&5#lJ>D~c!fjl0(?`N#;?bb-A>l68!A!IzKgiXq{p)<|=T`EF zc!zI_h)zLz6s!;4A|&Sb=kSzZ|8<)fe$I+Rd%RK8LOc;}FdUTtE>(i;;ugZbM#|us z!l(ZJ_#RPo_)2N@!j-r0?74+>ZhkZ>s8o3-0e=}l4}8`$&M%s90`9+sK;$A693Wf2 zL4vRxd>U;+OsGl9ls>`}Z}qclu$U09e_vyf?ej&#;Jk9dQ6V3!C?pQ02lB-_@BcJk zf}-Nak&+1U#-`MiSLqDhy!9G-r#BAxK zUPas}12GJA|3ta)>;jz%W4y$=MO%xX56_&N-t&lr0}?TBQ!6?Cs~vc|(7Vwgbeh!x>pyDTO5BY5 zoqwhmFTv(vjQ9O)GyQrsYT}BUtET2@-kXD{#6Gf`>524WXBHvlP~MSjy}!REC6i)2 z0;y$v5wg5Dy1y*7qeWc(r5iE$;NOut5l1qH& zJ?AlV=;Wt1ZNZbTGo55ZKHD)UxviYkTjZk1wC#8MW+E<6@r|WQ*WRk|s!MsweL+Imz*~gF zqzOdN%OogIs4k&^VMOi$&EU@hM9bn=Uc_stDjwo`-bI-Mt_EAd0qk?I$pJ1%*yo6G zfEaKCJqA!_pdbB)^Z3GbHA>27`}R}ZCXRCaEu$%lUl&b_3sGKbN$diFse$Wn*VGH+ z9^7QGO%BIELNIXZW$c*#t7CqB3u$>7H=3xqdOz3I`l&x3&A1_zifrwyj@&c<953u( z@~A+*piS-*n^mRBD_vp!9-fzox`5U16W3e)jE=q{zm``Df$F3PewLeniQOTDe!j$PY)8 zr6pH`r^5EpT13A4dXz%*)pRz2xEM-2MESuahWfali@muCG*t8@gMt_C@`zp)tNj@G zbL*?*zyK-mtn$9NKrqF)ylY6h$p8lqY;c57^AX+q-ELPeT_U2xvoPkcKThXr3G(R| zq2)Y(vDT)mYbE%cbiFc^Ud5;mX}DM7HLZIl0_eV zWyHT>XJKy%bt#W2@BHu(+H~}2 z{YWJDTcx8rl#?ZpaWoxpxO_%NMpuB3fg7U3z(6XL_Rj&$dRpg4$&dvKi2zXhYd9r= zo%9!EyKbCnVw`#tM9CV;$S}Y}gSi`^r88^nMfIiPf34=myv|-L4R-SMTKV`+1D3x6 z@rv1Awc+B*dQE=TrLDYgv1*h{^ze6+qd9KV>}IIpmFds1$@3dIk1~lKO!>UHyS%R= z`#Zhhc{f(+)|@`o>c{`T1+wUhFD;QvjEtkv?PqxL?cJ6D(Av#2)Tuier3Z<7k;J&f z8n;vq7?cMRwm|WVm}LC|-iN!_{MAHIl~|+HtDYpfgT;3o(sL>C5|$e*tq6b?T3vaN zmW@jjg5k|cQha#L6l4eXt?>D0bvB~t`Hu@91KKL(%AeR@ZX}l^c~HiL-20hQIuU7_ zt258$V>}VpMVVMkZ}~UdAOKt6DNTt3>T7q0L*Ryx!C~)cLmS=e7s3s6+Zzk_V-u;T zG=Zr600KGbvhLwwn0e&W8=D4<>%eaq4(^LK!E@#AKdLdE3qJNp6ZwF6G zpw9}qfgkWau+RP&?NnB_kN=q=ke|#f!&368R#9ZNd3tgr>d#LmlD`{Dv3tluO^ijo z0uN6>d_bQjHnk}!CW-lJ1znX0PKGFkRgVYtAEzBlL!%t2k32r9hx9!aR12{sWU+fR znzyD7jQ9PIQ&9baOxZFk)rbv{%Jp>Vo0pb(N6lD#NP8ap)|re9A@!Q$j&S zRGXQ}VE^PW8PfhU9WSF|Gd_rJuFJ8BdYb*m*wWxrPmQVfv8<;tsmVmUP3jAa7rj1iX zdq4yOT~JGniM9B}nuwtDZh8Yhl`CL}f@+~W8GdVthzar@} z4za+QlFyv}r0~%r@Mp~Ec)ctpvsq+g{WBN`hxUxqohJ0wvmbvYAL4C7}4?Ttj=`Q}u0a6Xu8)`J*|4 z{dWX|Szg$-mT3uSVmVn^!I@~NuHFmExlNYJ7TAg72bl0~R1;}&X}E*CreleecsNUt z1=lFHoC6_ui96Z#J00 z8+&JF=HnBbjnt~m^rL5-B6Ua%l{HgIB%AaP_o-hjHAiN9tV5rUw{zI>nqHsEbQlmA zf!C8zQ-9Md>1oQ-#diXQhC;oN!aKA+vh}{vaqCfQYh7r7asfZ8>;A|tUAXFbxFRc$A-W2b+&v0JIv6 zFxAKdKnsyNBgly>l)6o6qki1(HpZoFo&+@nUF&qR`Sp10dK8X3mordCTic&|QNk#P z5(jVx1V!B_YD)>bhT z$wmEMBXf%67H!FIjd>>R=H^_O>(l>VBsVBx8Xk2TK9*53x_(wgz9VFJsp_5wQ72(( zCdbSP)}`+0`?BeE5l3qGJ@@)HZE`hPuN14~L2oJMGlQiW)NH2`)Z95;=Lpu8L4A4C zTTd+vg?f|QSXm`Sf~Q%43lyJA76u4g2x4oGdVUl}i!Pd2u8CHZFod65`gAHbwJ-YD zlx_e8mBa>=S%FSN8SuxAwU{0zuvIDjotqgJ7xnV+GH4rn;{?rIf+pvimp|qT=M?7t z{CnDVQ`3=MT-o5#2Sl)ukrD7oaXDN!hek&97A@^lk$dm+^a2C7At8)yk&1sbx+Cs2?ulnRD?6rTaEvPj4+<#zC^KF}YKXh7wNJ_uA^D+k@qE zka%^Lb}>gh@!S9f07t#;td}kj{dUI5HIy#yZ?SG2#JqN&U=DffF^SL{$@tv9SK0^u zLJ6xE7IIopN=ZLrLzj#WC&WN95+scr1A%s|*1JgaIkDaJzoc6(5&@(yEXHiQUv`^v z8F*Z4yIeA*L}7}+8QK>NPl-)NxkZ`Unsz^vurB3aiCR4WvYZSPJqWj108h4ElD2(@ zKAzqZ;3SyGmmE_=fS+`;OPtH;FzCLT2eygy@_=t2Nb$akrQ%I-`!dBbAL{el zoimfG@{H||-ft6Vt&t(|Usdp+y&RhDjF-=FSaP@$>)sfslEoAfP68sFFzKM$AO(oA zV^5RZNVmzTWPEX~vuJWwqyXf7?E(0h$30FkDR#&~rZxu0!@~oz8cr!!@~D~u)y;j| zuyb^U<^VqKyCi2p*-otDPmeUB=mJ4vSLQd5lwQwVt8@Ba7Ae&^@6 z>MJj3In5C`sD`AaW#d$+JEcVL+!#g07Sn7BL6ewchiCRA1m-M-(xZOU zl0@XUoS&?2-#0o0LT373eZ(2c>9##Amy6&B#Sn|TgmvU=*8V;F{lQv}+zgen7$@Ze zTGdY%3q)ShrgK8K6szL@>%))OPTQl5%JOkIdWuZk+}el@6%}AB%Da4;FU5) zjJ+L3^kmK2SCf71J=>!$d;9WA3Nn0yHNz9GcaEmp4Bah=hd##uFh)ZlQQqvrshEkc8cad!`~gfXfa?tfdbGN@_D2!K{g0-4G11G z-se;WXy)`pF^!E&yV~TcJJZ^_7tzr;Kfki&?B*f^{|~B`s8ji4!O`yLxb{6}5f({h z@A;NjO*%Pnhn_s=wwG!wDmtYNS{H?6>aMC8^!E#EoQkg8%BZ(!;lX;_fJf5FNz3|D zNM9$?+cK{~a}5#-kn~g5D=ug>++d`ykf;2+zs2I_CVO7gxQE~g^K<<%%5bm~Fw3;^ zvAlITqI``gHpp{kNGRmu1r;qmYzfNd>jmV@IlzvC6kM82UZ~XE~ z0TQoy5VLc$kn9h|C;t+DMPfV|xC#&@Ae3gwo&>|c-qlOI06HRG1KOu9T&9o#)Em5(0^*f8){%MV=7hR>YK* z7iMQ?M@KgxWDOcv*-5cPHJ$Q(pjpg$+EgIGXLkaeHN%%vkg;>XfVDbXCVgn>RC@qjTEy&Nph ztSxiVv3o)^8H0F>KOMiVF6^)NBg+z#+qAXiE(SfYzw$}T7<1?Ej;$vHF?`~Fz#cTm z!gq+-Lu&muGlRLW^cWd-uvL^{m+ZRu!uY6SObxGnV>$1QUVoxg2z4tZ)Xu#`(x^8! z`ksh{3O{vE8z4rRt~dz*u9Fa5`1t%{!nC-3%?7*c<;!+IT3tCe+JoP$+{_SW71f>U zf}FN*jUopo2;3!He-c9z+ZBvVWuGbD9GXwuAj43`JOzcFOHo}W2iC`9h{*#6Y@IY^ zh%jMk)W59;V;jFM=Qx~mVDL3aJ8$k(rI@vd>F}V%@H9_XIT8ZC*sn=Rnc#_%;JpS! z#o;kGH<`uXJ#M{KVz9q>Ep(VCo^_;uF6c*eqo5HL&k#B(jq3INobjPxD2QaCOH!%q z{lnDqBi+cOfp7CZ^MLg}M7KjsZwAf0IaM=^OUF*dc2SE2(E(tkRsIiJyY^0~G(+ShVr4oQR_@$7VLs`fX{-c1;^r3}M#e>e_0 ztpWcs-3et(uRIh{peJp0pFoOVG=V5v;Kb&fmTbT|xi z^P^j5-?2WzeZ_^ePUhQ0*1drFMf*RW=PduZQROqM81&O$rq}CEqrWC>NAQ_}g{!*o zJdl2@MTjq6@ATK^aj(Kl@5E%cya*JvJQqD1d)zP78|jEQOf7cijC2mc&tmznBvwQ# zk}`9Cc>N`#k6T5~EY-BBU9rYe+O3&3Y8YT0(_5B}cS|V*J}H%ukrin17!q#4O=4XO zt*2H{70EFFz2@CleeM$bSbYzy703YC`TJM1_Oa%D=zq3tJu{_gaMhPD?Y(rdjQ+m& zyDRg2hLr*lG+ugo-FNpJ8gMqd&>j=Xj`PBn3|h(Wl6p!o@*rJ|S}J7DPCY~Nxdhj< z<#Xj9Rp*5aadLq_c?WIpYC9uQpl-?TQkMAx5%P0S4pqM4wGPXRqX#q{&vT-9RcD6_ zV_>t29wcHvF1}Yn2VIOzls1zZ~z zJjpH_v86;Gfld$T|Km?1u5X%~9sE>;$=t{9arvwZJ{tA0TcxzeMH*zJt~IGCPQa;IOd>|tuNbf zU19x+`5VO4={HRH#&D)oNdf5sR0}YoubP^W zDB^ao5)ou%z#WMe2 zP2gY#DHO4|v@?YLt?}mK1-egf*e%{F?7gkF2P zQx{kYJ0%j@W;@>q~kXTEML{Jl2k%#B{wLaQ| zya(JSPhN5>EeY~#(>v*W2eHuYs?+{;s>pZm_1l!QXVPk6{6YDsYd$<8KE2KKK7EOZ z4Rl<0Nz5XRXhhP=a^m%HGHbJgvp?$#>dm4q8FOQ{lEDF%MC7mvzT4HQ>fTfC)8rXT zBcRg8Qy)KQ_WDmkx$MTKM}2*LKZ5`SwL|Xx5Wpzg{_@plaaAkeEg#mRw|Dm6yZ0{s z;}(zb6X`F;&MO^M*A7e!%lDt>3;}Yuw5~mRF8?W(<_TleoJOf6E(RE03RMs!hFQ(i zX54V1ZtZ|oTv<17+-MLkRZE!-lgT_R+aKBLNdlU2K$A-S&>-SnBDqkCxHa}I1gl+p z1XlK>m$sy++$!Uq;#hUslWc#nU>{7t$^r;D>&8-WvXp$&J zC#39VgU8bTfP4Tng=Bb(sE_<1v|R};b`L*`qT79JeGA(|q(>6Z;%YoaSDc$fOfe$^ zz1{W?xyNqGWS*i#Skd`{1~q(H3dc4^Mro|==0vS7LR#|*EztO=QAS*;lrS9rYeZM4 zW5lrU#}&=xu;>7r!ut1?E7Di`8g&dAE?Q$bfjbz2@d*3_AZ}EGQ~ZeD?VDR$=e}kE8gn%@UxbIN72s* z-?!{=R=6ShErp&dQ!?RJ>mpuL-NnNLW^HI7+94@V zTlq=jZ^_w*YghzZM?Rlcq|S}Eb7e!4BHPRyBSG;qb9Yzo-7I`>V0K9=BU`dDFesE2 zZTsOeZtyB(8NY67qJI$%{wfTqaI2y3kYWtYf{N9DR{sUZHp;y`s@a%+{U#{VeCDTK zsj+45p70m1`;w*3nuBWD`Ie)W7oIoC4Br}BS!^&aO@OIQPO>;r|1T>K&hd>O(w$`B zpTRIuh}9^3EF{X$Uk0Rq#EqmD@7WkfXtpnSUQumnZI#uCm+APHUB|TnSNZh}=^DT^ zS3av5))!kJ}DBRE>>n%}~;9$zNVosn7Y*jS-@ zG3VY!Sl@B!{mi)H4n<@_`^<~z+SLg%EFHR+S6<881@oHH6$U{Brt2iDmVv_{gmh}E zn#eaRU`O|RmBW$mw}_T=m0FA>k|OV>+W9)^A$WihYc(W<#)P5 zJ>V24D=RLV&=9PyNZODg4+5|fyTO7PzcC?gn)JEj0M}beYDi0wM`iISDb%L&QFv!W zn_iB0MuEs>+17;S)gQfy+;@c<hx#2+_S* zBL+V;W|Z1>|9p1Y*Pi(`@&VtxFISTqg>Ux`G38rKjn;nl;IZ5=v&zmfFKF8)d8~3a zoQq6yt}KpXB?90cxLG=?M6KU=R@hKyUl8hpB-!ZqJ25I>9jhI$!VyYmqf^ibgR0B* zSZ>Llbf}T~Zns`Kb@0Z2MG(s+qSmZNqmTb2d9fQA)cmYFM?LD+6=@e6Z)g8T%97)B zH}}u^m)&F3l$1-2QX=R0qm$8(vA);X(!@;G=7l$_H@Q}6Yf>^-sjFg9AlrBy{z=oU z!{mipEgUAPwKVs~$*vV;Xw3}IQXc3=Je`4x0~pj^8Fg`+W}w?sGmXn*+lj^>z#ztS74x&ed zdB0$?jk-l7bcy~&O<4Piu(CUy5>g4<(o@1XTxa0U(fp#BipPPnUf$}{s~X!V*~Rlo z;1>XYK>4a9M74lpDJ|ExG{fM0m{76X%miCH3rk%VKiBg&_<<%UvI*+O=hu5z!&)-= z$wqB`&5N7()2XqaL7riT@TBbI+zr597g~eyb_L0oac1k^hcibOMz|zdH4OWMBc%5G zpr|os#Trm)7^M6VwcfYQlDO9%~OfM%K{R?lnVr83e z`+=B=yGG?Uv8ag)4LeF*??BItpa5jthr`+|1FPJ1Bc zG=I1VI?a@>Q!6vnhy3v9j%NjSS@pg=GqGoWZyi}SDr>T0#g9Y!lkcUl_?5lrBgYUf z0sJTB$7pskS>#&6d=WLq>B94IqXx5*<0WZ)SVkd=a&1n=?%baB!>-{nlA6+{r1lGI zP()pos$X6tCmRL)ocW~8skZ`7brHwTCaE);-U$h$1Og0AgnL0o^Hje;F1CjrvqM%Eu`>w{1}VeG%i(twId?zDGn2O)pYn-v{mryHhXP|q;i*c-M-ajoYf$}1 zl%VdRVYaA%b2T=`9Ygln8>NwC7R%RnZhKN!CH*(-c-sUFx#wt9!BIb8Pt^WPU2NU9 zY&=5ta3$}jwZ&`cM)wB4nlDjWQ9~5lcfLAL-qW) z?OCPI=F7y0Ly}}v+}yjshI9-qHiwDM;4v|SfI1kI1m>UtZf(=<@aQX8c#s4&dky2% z0fy?9>_Kl&X{Ib-{F?m_Dd!nmYCOtI8z+44{r%a{6pGir|TOtuMp z3FAS`h-AHSyZ&Sv)F{D%DSn~a?y&mb=70cGQel-=j)jfdrK}`xb$_Zn7O3GlXh*?p zjn9XCMAq}(y81Ewj71M8P>Lwk1v)R{z4 z>bm?|lYL=-mk*J!x^7N&Ul<%6HQcRi04O14)_=Ft-{0l7@{^eXJ2t*GX%V7eZDgyy zH-1)iQTpXXmCVWEROYe-$etcjK9y_`p-0G`uCD0ZrO6gGC*VUw&9z?l3p`w1Gx&Xa zH@-wAX8H+8Oo$VK^y^27#s?2Jk5G=wG4p&mHl#BB_HEkk%YSyuXZ4}C@>lP;Kulb| zN>uT2wD8;AAkU4SfdR58iBLJ`*we4ShU)DQvR`Lj)@GDae!ybWEKFF?9C(LnlTY#i z7UVx3tdH|Vk_km6d6LUK#AFojDw&ymU2N>}x>uVNfF2#5Mr2tbc91>Een^z#V~s}9n&7YwPs-MZ z+iI2?NX^=rK+c#keK#;@Z|iu7ySRLHeM?nvF_b0r&yFx|U-N}wio!%*22Fmp_3M^T z?2i}!IUq2wC9j=)zGk>Ug!5d8$7pAWnpNR{nOLfgy{LYB_+NvS#zKi;g6=54o(i7! z^YDH5%@vhEP+dOo#cYM_7YH0`4lEE8_f=Bl)bR+6L)V^eT0Qlqtqpbui~clvtHz+W zps`7@lw9z?uJtx1jM@BRmX8cCE339*d^67d|G2sdXF_w~8Y83D`nx4L_z*GRBFckUphz2~N0-JqzA zi|gvIq_}!}Sl{j=s73eT5CJbryoI|f%p@8;$^ zUb7Ny>-z6VZ|6#06H`j2Oae#pDn&i9u%y z;wzex$8^lBjXh6~Ap8$xrfoV-$}GdgC`gclfW$byQfkbXw_opk zMh~6t$DU%?xkh;~%hE|2H?()I%oZwryh3UkMJ)>eTB}+-=1qN-)223<_``p5?R{{? zOY|S@E*Y5nSt?apj8?ehD+C1%Ll0JJoSI>GYxHGi>5GTb==tDMKj6jEFEi4*^7cw9 zDkyA2O}M!MG`nGNG}%;N-!d2{0R06282AdkVAQC4G2k2!fI$w44-Wk?XnvIZxS9jV zC6)n%ziTOSJs_#QsPQIv>OicfW;RQt9*geqC@>!&satg?=5baE7h8? zVm|||yuo1M;KO9q_&tr@9YPB*ckgmQ>__5XD!)ElSz277Q@)Ts$84vVHCR9X*|uDk zk-As!Eo2>O3xZ7FC%-Wsb8qB-%Lj8sdPdY&)K5FyzaXCqJ@~(a9k;{lYCgJV2pkum z5{Bg(Mj{pQ6vPV))`iERngtf$=n}gqwJw*MZy|8|zwzqx;j^_?^CZfb4NU{?pPgb) zlI2m~IOAzr0>q&>84;^on&a8UPh#LV#0>}6kG z2-y8;^P=gYAL@y>b4$3y`n|DhpAx3*IT~g{3mU<4{}843wuEA1LFofJe`V&VT%v(PQ|Cj~cfQ=EY#m}lb96oSwytnsV-n-R{Qasq zfO{LBDofG!x<$gKegk zCQ2W7cE)a5-T7c6s%)?j5zz)S!RQf0vY$(1V|Ru!W&3|T@HzVXRn848=!E`CGoIe_ z6ea|ATB}n|kUVB%{rY`ClIiqcyZ0aWhUU@yy%CS?tD)ObZ`%AEM`s2tm#Xj zdJzLM44$Pu(TO*Mi%~^`g-Vq$zban74rPxd7AoN^NxfzcM=Rf^0v!=*^0#G`Q9N0m zaPu-U_kDk;B#!Y=Elb_NlP;tuy(jF2b4+s;-0AR9q3A+_E!Hp z`-7^=AZZwVG{M)P-Zzkt@1ZU-k{jS!=iTqyZP9QnY57%!q`EW^My~JNW3&NK5A34B zh^d5kNLexwFh#2;r!>YCW8F8X<{6oT9|T%|6ljceV0u>S`iv#GrOs1l=d~(w|2|*_ z_PIe!&-pH|N6dlBgm6bK@MO!;Ng%P&udkHC!yx8xpo)!1czJn+Z$p+dm@m+0l0_K| zi7Ti)q1}r{l0q`o|6Lfa2w60JaC8CnRDuijSmNEFhK@JP z#w2d#_33EYnY-x*ZL%U46fO+)du7OY+yz?!3Ke}L;+io?D9n>xuV*}0c_U)c9(Th zDrwXF6UY~lH740Ls%BoAM3n%}6G}tBOmg!xz&J+YWT&fVv^a`}eKmS-`@GKFzj$v$k!ddW73fXwu0ysCfa414EZ}CCTWrZ+SeJTPWaH`=i9 z+7OF|yhy?6vx|6`_-G1Y0^_5)$#0|pm`CT2idmD;$)oUCN0X_SiD%#BZEWj}QdKCU z_^tq{1Crd)xVdqNhy2Ol)rmZK@VK9Bl#Gaco-HHw&9#aVQx(Z(==%P;S4AEM(kzE!mB6n48h$*fwf#qC?;4cH?DUuCHj)G)gdQWu_ zg0qMYJ&25?M_af*W0$7f+z{5`*Ym?$e1Tg?Dfd#C*WEgi_{`SIn{U#KMDIN{w-B7_DLOhEdR53Ddz z`=km6b*_kD7@BoW8@LsqOZ7|!v+9L&orzIjBf$$*J=!B9r`17Y1bVGsm(?hL7*sel zB+SkT+pk`+#y(3+Q>~(jzO#MOPA==}Zl$^%aqHJ#3AjLDT?M8DL&F2OIY^Uozw_(0 zHsxK!NB&*QL(kC}p&Oo)6jlV%v1S|WhZFgWiHY3`0|FK61Q}i>0eygA#3nO_O+gd? z#>R=v(7ItDoa2h-Q{g9vAc415^&!7m1S$Iwjh-8J+j_&;U6cKl$!jWDA{?IX7J*2K z_vS153oep4pYISZ0!b!DXLrt}gL3G7Vg{e<6+lLmd4<5_KAU93p=PeWnn^BdQ}`Sy zWCgWlJYT5vmc$f^um9xZj<}@($G7@->ijMs+(VE2e?ByQlN{+2!T-zk7Yz)+#EZ0G@JL+)|XEh1kw%gx(EpM7;f z@Cc*H=gF0BXFuy3_g1GAv$p#GXJt@dC9+b9b#jzKdp`H4a(oPITBa4@DIP3t>mGyr z6Qf2Q=ot9@*b;OjmOdw~_+6O?yj4dc68!x37{<8i&>JV1s$E(yietugml9&`dKmkZqG^*98Dd zHIQ!7MDJb^#c=l7gsTm%gwT<(MP*b8wZC|Anw50DwcY(d=Q3WnTU`$)7JKIm*W&k@ z_5&i?OF?E=vpca_U8KkIIz*MjIR+kv`x%s~(O3UuNS@Pi7ks+7I1QqH3t5dRgPVwe z07y$Hue&{t<>v~EJrD1U#*bcJZ<3NGr;G~uZ?`4pJd%Q0#l;QFZ(sM5Fi)s~SjXj> zovgV{%kN%p1{LSQN$XH-gpIW|S_gh7ba86R=r*Sr06);D!gjeRy~+^GR|4v) zUPsp^%00)6){JcPKdp+oZc>S}$)74ndoyrrtf^?R% zT1d!7Aqa?yu>FS(ZBfXoscy=xtCD{v!*53(SMn3p2L-zaEaV7V$DKLU9p3+S|F8>y z&GN0!*Po}eY2RL;3>3igTfILm7|Y$-!oYo`=LFdL;sp@S%NrxZ!eCUi{Y)^cEi1&wE!=){H|LqwHrF49$~*Pie%q?P)J_|6AmgE>;Gv3UCu#dMSGLfrn&#Ta!#?0v$3A zUTSr3L9GBHhVjwpdB_c!t|Mdw3+McNy1zNIezV-@$i<7A9zm%V?kFY1!=TM^9(#&D zAd#;H&jbuyFETru{6n#vP~OcEH9a;+a4?8O-WcRB7yP7QB#LmT{% zot=i72Xh9s?C+R2J1;MFci|*Gnxe2d!~7dlAH6X@&(h^H^tOmFOPvU<(J`IXxV?f; z%luA+JaKqXI)UWPDGk=7llPqdjp6Z>HFyC{00e_7B2@#=Vx*Sfp z$QmqM+ug6|{vE~^MG~wDn`yauj=wLZCQT-hqO|sCr4bnXAy@A#TLPNO_i07^yc(-y zs0sZX)IMi`?L+67%S*d3x&-2Phh^qI`Po>g%kzUibo%B>3pnyUs!;o`FcBP(8+_d% z7-7eeqGk74Cx6j)yhdyuH;dz8T>1T^b3SIxq902`-5~b?br&{*PSEa=s8OWvLL2Tp zJ{DNIVXNl@8q%!@miTgddkgK`5Rn6%g+&8>H|h*14|cErcmf-c9P$riRXkf;ZTtS4 zskDl(bY@bDz#`AcTT4`oI2oAo&#AoYOISvn_zahTm+q(re4YZf}F72E67@c&xDd86`7Vl!O%xUQlEFg@GT$ z1u0!ur~&N50@wuiFJ-yY(96z@RF{&cC5IB>)QWwW`z&MpGVGSd@E~ko3I8pmvqcvu zgu3m{Zsw$!fC*Hkg7uv{3GwkUhC64v*bhj_=?ZA6Y;^Me)qm;WOyQg06^7}5zws5Q zZ$7zt(ji{H_$^6v|Mkru)3^A)XMh-PbYx`U2bd&}gMjsZY=QU&xo9fJ+=;~H-#*ya zgY!GVQ0!qDc=mvub4W@`wF%0;GsW9sSO5mtJ2`>(mwRmWIg|u%A1s4BE;3T|*>yWX z5S@c;a$^s?8Q#mYL3tkAuR1 zlsc~tAc;`T2{7S>DGJ;}8=Zyqh+>#Nya4Yl<}v7B*G5-6*ECi7y?-9Zwxp`_%c|)$ z);Apz?bxBJk-toOus@(>H9aYxxfy!=U$kysXPVFx?(SFg{v-@rY4a0cp2o_|F4aqa zi)%CR9TS~&aJuv5*Y1{L);-wD?B^-Mo?)Bgf7)9=YE36?Z1JA297&1nF`OVxN@8Y? z3l=`*#R@L45i%Y>YZhmwpDwlkQ;L(}i;#IgADM?STL#G5J9Fky9n2^J6w^F@UI z**rj0)`{ z3{G(1-UznyScAgC!eBmLmk|r61G;sZgEdm)M(cUiN3-jAeXR9Q zrFy1tYTXke5KF;>hpN%dE~DBoiglZtgMP>o+!c``cH6 zVxfFflwmVk>;S)pJ{TEkF9R6)_t);RnmQMcckaFY|MXMSGDPfNElf_TJ^H#}@l5Xi zOL7{UWJ=4Uch5kOs;$+W$DLIjqC-gdfh>)bD3sE)(Y@&tN7INVo~%Xp$aPkqlGyvq za)$=k4fkC8eeK74w&ivSP;Dd=f{TM(BZ^B^qcKYwz*E`-ClgqHqR6)wbpIlvC%CcO`c_HR^Uzy0NrI{Kyb?|<35SPnV=GIoX@#v3)Uak7t z&)62P(X3ibWI{lT-J&%E!pieb^`?uTra)`=uLr7l+1x~0{r_N51F(i|UsOr6L8meI zh=eUV6NNae7QD9bxF>eI>?ay0(e@!UoW!O);O7zipluskkJ>Y{L7rKIgGCS zd@Jz71+_Ub>MeHWGa|TC`nTTq#_ts06ngylR=h{Go<;5t=K2YgShI zZ?v+&DM95Wuqh(A1G*mc1aFY1Kw#}l@H3;fG*df3_CO_`hQ0nO&)Mne=7)SHzx0zv ztc4gm?2eAk;dQ>%i?ZFS7euI+f7|l^23gCEYNaqqp` z-YP=Zbvf@qOLF!UIq{*HB`Iwgp=ckdQx09)&zI0vW!5IdZGT!3>GJbqbcN5)w_H-! zRQW?`sl@UiF}7Vuy2-8jvgb9qHzmGGy+C=SKYHDWH-VqPk0uYdB_?Z~ zuLTd77GL3EJ~@R?0Zd3pcy%cB`SC&9lpA&+)ROOK+qlydb#zV^2g==mrIAExrvZ7Q zy-06e8EFQ`)B4Ja)ykFh+8jyXTpc2{S5tffXlPHswla6|a9%^}#q=nvj> z)+a$gd=I%5IPh>?#HpcJJCbZx?Qi2h_|ocB_jab)%p4a{*(=lH@#IL~#Qexnswdc$ zDY2FkS>BjzVZRq#I7g27c!9h6aO2-Oc(dF=h_P9)Ax##Io_+XnuV(GS97{$|a=A?n zgoUX#mJ1Sd+15V-ZY(8!R12;rLM3jydARLFFFEM!cJ(>{a4)CPz&J#LdK3P)`OTXs zFc8Zw-hFZ{#&lep5~c>&Ma93yU^Tu6odRs4_nSKQidg#Jni&Yc+?vAo{jO#e5Js0s*7JW+F(ST zlk+$ZlPcHB%1YZReX>)NX}rJ%PIAl+2GTb4*HO*`Ap>)N;+k8VLyYr*zW}yf=zNA8DL!BkePAgl=dijl-!k<+EIwYe<6~`i zE=jxu&DY-WC7J!qz?{?M&?T`aWUJjp9o&x6BX>U>67P|Jlsqs(u=4V)JyU9fs<@xV zi>Sr3RsKmBN;PSS?YQhfyC^7ahJm0Hv6@?6Rsz60O~|fXPVG;nzYBa=$O{4h+|d}) zBx7P?pkF(}kIL1GJ-;|TXa8@u`rYqc37l!mvKSLblm<&lgtWA4XSab1lgj1xV*C>rIhcN8K+BoCZcTJv9q}^37!Y9I-pdKPNj1 zD(|tlqC}zpr1<^oarU9s+I0uw_LY$%*Voq2rX?VFqFx5RRHvh>h=DIBydkzd46{m!LlEwn$c%LVO znfYhy?#^Gge#O7ZI%3_lF*-DtAa}Mq%(F(knDKU(iM17xz#g*26$# zYebi43ms1##G}F_txn+wPM&*mAs_eyz@sZBoZ2XcCAyR|_9L%BA?#Cq9bYv!`+=@H zemNi%oP5x`Q&?-R)IH5?Q)FjK_j^3*VLChfb{_ALgvyq&w%Krk_8kz(EL#P-{1lh5 zmT`UiA9o3`l8Uy%ZS)HkkdQ<0$9G?#-2JHo6Pp>=-GLOwqPfn-mk%a2#y#VJ12QFl zk91PRKl`&^{{KX#iMXYN&*$4|m&u84!MMS#Z9%W+28&J>dHIUzmqm+)3yzfHPS<2S zVb^jf5BVl&mW9f32FH>$Pm^_kHzWUDgVxBa4lqmdXvW(6 zPAh^W{Lkxw=XIO67H8yw|N2#$uNw#cPf(7ShPbx07${q5eK+0=Oyhm(Y=Y4lfAp57 zh;VCiFmKq^YT5*D!9>ujZRGchKq4X%gbG~U6IzjD#5vcpw7}=~nS8LJk&*vc1_z8~ z4lf*4p;Ltfrn-z>DUkE9$XAq2?9UQXo)gcmJDAR`$q)@1Iid zwC6q8|8KEeJw0H(28>mNZmRkW()-OqzipzBdrD6^5Dl9E=<5>y{igezhVTC6Tgl>+ zy>;zfLL%R*D29pf{iUTYMhAz5@oZh{J?K)wE9mlMos8%sz2Eo{#eZX&?|_aB2`sV1 z9Zo|^RglXH(Y_BJ;-{OxT;^Qp4UZPN{o-l|)*IJ{Du(ZXh$n{$zfpf*j&5iOm znMXNr4w0zyvUCJsozZ6X%L_EYGG!Q9HrlJKAcT!UP_ z{U(?$$*0&_$POw%A{Y7cSLU_9x!{h$gi8cn*LWuknj3|n_KCcOu@5sX=i&@`uS@U! zc^DWg?`M0>p&mT5#P@Y7#M`H)U&5;kboFCX8jp&afJ+L!gsRe^@WQgd4nCvD{M+IA z5iEOW@L;(8c%Qc(GWMqBoZ%<&mAP$mB`dBO8XE5Q>FK^Iz*vI2=7yBi{LG9ecuztj zvP)#K17)0l4CD|EvGT5#t(;v?36C1unu55&C3qk9Yd}JXO$>Pl|27ho;v`AOU?Vp+ z+hJQNjxuC}Cpm-*9VDIa8W-78Ockub4l^~?FrC_8A>>5xs(Xg_XR-?U`LcnO0 z0K@r^_zoSi93)DRk4n|YPPPQR6fCFXObrN&KtSUI7FghV>EUs-wP}g{-?H6Dg6M79l~TF4Ob>OMia%JgHI)K;uD|M|7QDKi;}#%I5rmt zK6d5#U%)a4G_JEWNc>wxp4+Bf_h`ZgQf}u>O9;oPXx4ts%5G;ssCOKSB$$Ypvg=mg zL}c4_4;qN!i+XsDGfJ&l;+o7<<`KpgUDH@QnEFjnkx=LG%!b!;B233EFve=<@7)U-@GmJ;acS zIhpG%62t-gI}l72kRX3XCG?bEDh+s&E|KrO0;m4+m!q#$cra=p)0;5*R;H;!BHCH* zL2Fj=jT6t($a(-eAwle7B~nZpWsplO4t6{ zTE!&Klk{{tB>@itfXJoZBHK9ErsHR`mgl_HLy?cjmu&}VGm6~r6FU5W96=IS|v?(C6 zZr2Jk7fXAxYA8d6EhE6eu?j&aJok0XEPfWFktR|{|?3tyh2!A$Or+Zp&zE>euW1ZIQBh%1<0DmC44cq7;j7wYie&V)U zt3$>50=jf)BamX1@JUxgR84{`n6ahj8w!LJff5p!JLPp6Ah4&j#=oEgv%Lsg!#!*n zJy$^160b3+&0u2Vu@gCPzQdIHhDK;vm7^a2h2gD4$o#8hlwiW*yf1PrS`sVv$Ln9- zGe9c^d`gku#WFOhFl%^)1i5O!$jG?12@SswTsIL7devlz#+T2rmKsC=2f=j}M6oJG z$XZ*D&CN*ydJF4>vecY>oDvIc8N@1(HdS#e?^QL_I1I};rp-^!38 zgi`>Z0KW8&Le*91KNX(8b>_~F2aXa%*iJor`wNbuJ^0T2JNtdAL(4YwZ}%f#NB|UI zQxY_rUlHVgdd6J~A4u04CQJyy0oO+r_8=@7uXI>=^@bj6g=mAlt*WXjb)FRn zYO`P$DsyOkTb{TH?Jwr0;{o}-49)$=ww$cjUX1~Qm4mYNWew-9Y9ZfXX@`-5#HCQx{CZhWP zZN8%eKSc%IGU;lSx`|;M;&Kk{y8@;2Ih`>~R_~H1KQ})>Aj%MyIDW(8VC2rif19`k)^PNfB!mFfI`hmlh1EnQp`uz5dLuO65fy0lt42aw{ zQAz=8IQDSM`@@lK5Bu#^4p{0_g;J?UPoZ&~0sfm-HfGs?8N`J8BvbNGld3uP zi|~KMlKBO8yVi9&Qs1RVTFN(EX?0S%EX>me)W(Cb(IfEw@^yA~> zhje3X#@rn_on8T4vJ%r;>02-H*RS(yVj-;RIsYx(dznFAkezKEOhH98loUl1vshJ? zyg^gj-bv_v*Xx{(7f5dMkk;6Z&I1P|n&vY#aEmY*fG+~3@wK(JRfPVZn@hIxaAD^q z8+{pt_Z75>AR{;>)}wA%hXsL+jZIQQLP0@6Z^%z@mwmw`0L=$9i`AF~Gsc7nbE zCD@p84zFN?DZDi#1umiBMyHSxa9UIRXoI$|2PMcH=6-y9JleENdmXy2vKj4OIFVP& z(MKPbJqF&eD2UutbzWdVf&%i9S`n9e0&yJ}kM)&BcS1G@~?ltU*s&J$8Qn^xKB;dUuxbl+FL1L6AF;Ul5 zSLZ@o-C)sJSXe0SzYM0=*KEx>b_y?(TC%5w=xx*x*Tj&n+ZN zON;jcmxo9szPg%+^$A!WeI?ub_Ds@~vd$0Q?>~S3P*PT;A$b;rOGvPO9n*31q3D{9 z`SV&@t_O;>^ExSGW>!U|#i=oN8foBBV<#p;EbUYu@K6I-X zILxU-b{h-iVlJEB^;8Xn%3 zH9LGJN$l5(n=6ciFVl*GQ3l+qQ&LjK$e~KqWM88Q0s;F;=i=#0YwBnSROQJ2CJ9-@ zz)yV?I0v#TsN3CXuYWXteDXmpL$7qnGlc4%~%W_;Gd?g*f`4 z$BR$O%x;jpSYGh{ln{Lq1}dc`B-FRA5fD68iVOqs%F#6q>J(I-ZN@5b-Pte^W7@@^ zA543`a}ruwT7ahEUOyB}1cDiuxg-?Dt0;sVVw{G;e3O@#*JGA7qhYwGJi|I}Y{m7Y^yq%pZ;Esa!5-y`@LE7$2%YpY~ zruEQ7L2q&B0=hqlk0T)p%PVUmoOg9{%H$fvJY}|!JW!d9Db!f5(Ft>R-$v)`Z2NJjzaSEb7+ohdntGg zi#=RjHA7J_5T9HfZyPmB7)}njav5zsDo`b3YR02fyOZ?T5pkGgVpXNtKFNVR>5p#cGIQH=Hkoj1ia8E1O7;5vV@;6sl=0ZV2vFNaqzwZdM z194^dSuMtZ2)pYY#P&^QzC@J3cVO1-Jb;ulKtjqfcX%-BQ;2#9RwXNN^7B{ryg=v% zrqL?d=Ka=5&SMVI|K>rH3?2I6lXh=?zh}D_I5L-URr#%F(4qZjAVU~(0c+;<342diHM{H3&)nd z(V}XsufJ2S22p|u;I9Rw>m&PjsqjPIRAGepPiUp!{)M+4qIfVZNN%_|IvyJRNg=C)UkdqS@{U#_1_V8nwBvT9=&MN66rft@3K4ks1 zE&Kz6g&fy`Z{0!ex(wAD2jt^3&(iXt;Xq907xkP)ZiZB!%P9CQKSC9rsrsgr;|Y;0~4 zzb1I~NEOfir4K^~%fVAJ*=%@Mw^=Tdii&JhR51F8gzfk(&tVce)s$qp8Hj^R)#;4h zy8gE?Bro)5!ycbp#U)7H3zuL&+IcN_@4gADNp#B^nTDk7JUF zf@W|z7Kv5c3KyAe8J03K#KOYDkHvcs_><82F!SPbtrQj%czSu!PAlKvoS=ad4oz}7 zXQ5K0+`;oN*ezNhzzW*B31ZlR+~EtHze$HXncu=J8g9?rUFO{R@7r;{c{9Iqg?{?q zRrkQ2?8~Pp(MSN1r<(z}EVg+-eIMUe$V0&2AL1NK;4!XlKpt`6B(57Zt)Tg#7laVs zQORkqB8TK4BGTWuESyQqvIs}$pup$=oQR>()uz$C{&NT?m^AA0g0MEBNCVRtvQYVj zgqWb%dU~2nQC;!oFmn%Zyn&l1@)GuV0^;yjT{N|FFRVA1?|!`4_(H8~`UO%zJMM$E z;^E_v+P1EXswNyB3IaM03hUjKkr5v4Vj-%f+4nF`94~HQZrm9vGd9~w3MW%ER$;hK zco+opYU0n|->o(l=qrANQ)2B}h{uc&MNNISHK^RSs zaaJ7+XcJ5v9N0vdfA~)+M*#*mx3roY!fb2^*G@-bvIp%q_3t*Yxl{;N9#@5=pJM#v zqB@Lk$v)B$Qch^iP~(_!11TlR?kXuI6(oFZ5+``(@c39LW?dc6Qx4!%X~%n7Vc_`yr9Ca#Lg$D9inuhLel)YH%58(+kEkZ_|~n43pLL_k>hKp&nO_=rO7x$Eql zp~-Je{a$1^7=m-^zPWDT|MrscoYhbpQar|4!07bd-$?-OP}xp^HIDy7;hO!a!$Sn5 z6&b!(F;^W{IYNt2a@Gg8QXSmh53jNJziQ z1`7d3@q-CIT%>)*Lvy~et%K=uY;$p+jYiIEEH zW9{VwAzsiQ0t(epE~yNDqk(~eYb+P%Esc$h&!781ovzDXTv=h;>?R=TrdB6nnx~yT zh5_&9**JG5T(@PG?@fp|bX{CFAeGxUQli3rGMQYgvVSr666frZh}C#PD%7h(`?2Lf9SaL7S=pz-#bk)m^3MYN zxVQ{biy0j1j*v&^3#mbX8&&kA)V_qaZBN!nM+fb67DX*hFYQCWY}p}31Mxn#$_FZH zDxB(G_&7L+Pfk_z^z?i@{L0jj7m19Zt?-G;5Hd!lI4JOlM(9|Vr*wJrpEuZa?Stl( zh;=qL|30c8)A|*)w73co?{(_^A>rZjre<(|`ZpkjrKe9FOkJG&!x9;V`F3=4$o}<% zCP;Y?ur%MW90i6J?XhFl+?@F=c0!mW%#Hu-1mO+eVta!_mio6Em$MG#MvdRB1*QPC zO-;U}Mif6t6eQ?D*jkGh0&g`yJ&?>;fywfSP=Gvt1XDmF` z!>_Rl1?ALzH$ZU-3N5shTsA5hA_BLWn|7g}W?^QoD~dq3y;^A%H-w4_iHU_SO;@1D zoS2xnX#9{YQTG~c$fIX8*>%A$e8~3e*RKPwRtl$fq~%A|oC#Zow=t@?Y&k+woR+ zTflkhTkI5KCCX}Q^6su_klI5^BAk!45Iwb6LO+V15IB}I3)iix7HG>#f~A={&N8wt zR&l^A2433`KHVTbzFOEg1iiD&p)g(qs5|prtRTkwm-mFX?|3y;O?A0CcZmTY>g^5ceI1QY|!;dH~K0Ll1`)OfpOo#uSQ#t8Mr*}T`lm__kK1;|OQj&F)_#})~_4NhY zg`b=UU)}$XRJy!~iHKY5m*-b}ilU{cdjus_KJ1srNhf#Z>3KJN69!33g_UC~a~F$W zzruhNkH8o42;@m9I&zV052gIIr&e)6g zke|*vUOV?a-lJu;?CdecRBNzW8WYTssmjgCp}vUz^B;qC+y(FG$?2)8`qhmy0t*Bb z4P$Pwr&ESWQM7%gLuYNi(Tche`@RKY(p49mtO-8F7)Ha++C^yel@so?{H`i450|v42 zNmU#VNr#hn{?jz^6wSsN!0D<}Jy!&AB0L}vsBf{Z_cx+XH7UpCnzp;~=i~1tIz$mP zT|%sLUT~Av(^+GxYN|&K0&+DsHwSGiArr2>*GtSnuGF+NxaSrBOJkB*o1q@PZr$#d zR#H*|oXwFG8V*T#v%3FLD@C3S|1i3QMh4#BesW8;$0rDQ`C!ik6|xYjxN_7*%saud ztoGJc0C)vtPB1`gV+o2EX5VY9uC<0Gg_53}h1JO8cm>}E{NVy96LUTgYjybh_h!ii zaXB#^1p2 zs@RmZ*u|=P`WK)~B%F$U>jF%qGP>?d&H8e9oG`plezNuleJ8vcv!6f9h>7VK8g7-m z=LO4IUEN*SO7*P!uzx6%uV#P2!X(Q>G<+uMhvW zISt_f2Ul)9_CWig#%4}Y&vizGZcH$HbD3Zes6ydc!}}dtd&CDh+0j?meL4sVdeBAS&NNs<}SsWlV%-dOCr+L z0_dUL$K$>xCgHVfu$L4u^v@@EihF$zMK=Kl2GYwu{D=k2V+f{m}yBN|E80ApkF?YuG)mFgi80eV)}sEsj-uw<2}+dw|7O1h`5sR`AF z@Mb}9u`&>-&`y0#Nfx&aT^@2u!^Ri%o5kh4e_(BC8y8f@0Utyoc5`q@k2=SUdRh6c zTaKvCD;QiVExxbx^|5oALu(=Ufs>4L3!XIG!jQ#iU1|zsyW_PU%Z_18na9VmBeBq3 zM@B~aHN5h8(b4f7V!{I~e+f~~`A$9i&!Zb1jls+;%sUlVv`9-yX?lD2@sW^{Vme?g zyi~lmWJ3~hm^nD*LonmGa>Kt-`^FO z%5!patNYhr`&#()sqvK1{*A5&nWTvJrDaDht;aD({3h;rQ-e!O%%>bTZcy|Q{~W!V z4jS`w2EWQJNmm4bg@4CC^zn)bcq9rF+>9sX( z?j9OLxB(G|`YcgKB_7rxgp`vcQtJl?Mkf3S<|F{_5GWPX*$j`dofEQi*lNKoz9bzF zy%ub;I_wA-IsD{>FlZjd#d?>id*aIIsF4diqz&E}%jBV39G!w!`<8(Lz+v{?fvzW3 zvO~)60WfQVsgB#?G}c3X8MU&kc}?{L$H9}l6e?~X>`;4ERm`i<(Gf@Jb(U_qatqiN z!s2qMt3d9)S1B@Cd9076hND0?W=pC z%8+zDNgD%u-?nsKz_t@A)iEjk2*xf9U+#VyU302M`cLj9zxo&_7PNIMH1yQj5IIF= z?&tC8H2)QZ=-9(XySbvm`KJsCzc_5?ejOf`i4H=&qB#|~#-?P;Ov|_G82377wb)8> z8Cnb$X`kEhJqkSABphx{OneI@gOJhgv55&ZpDcwxw*nSxue-Yg8ZX3Bu6?yjSyUBj ztf;^$QQb~mYPH76uz{B{G9m)xG9JC87{9Su)cq)zQTzLb+s8Oc^RRpy;VVsQJ1f2! z#BLMx+~^q_%%5o~DRG!3qils15r@8r;zJdKTen~!AC@or7WoEJjV=EjLP;0C~XDOd0S*`eC2QSAm;EId(Ky(F1JiIn(6Rd)hOA&+myZ%<`#scv|9Yz4fQ zd%BB@nG+&V4`DQ@W@(_0>nF*%e)|Om)}xne+L3%~oAi9; zj+EOI&m2hp>f-n`&HxsiCgpC=2w<~33Jg<|srT?%aeTZ#Kfm)(?jDr?p1bAZQ2o4I ze@?R z4}Jk)%$x0S->A$US`sk{iFKB3+&8B%mCRLo-o0zq`-@u;D1Wq2&lE!)AGk-Hd)K3uMnLocx!4MQqZ(5nBYgQPP$j!BG^JZ^S_QtsE6ouEn7R6**a1S(QjA8*331@dUOMt%6FY zu}P_VlwG=Z-lsQdbYfwUAbhinSBa7F+Vc5r)pxada&TD>{e}*%2;P`?@7@7!M)AO@ zg@MK&AeQ4|@Aw`?h`-daSPsyQ`y|cFnSBPkVea(x>(^hDU+pgUU4CNHJ+G;O$Z?VrY;g1DV$dj~-NMrX>n<&Z z`j{J?@Zy=u5QRdBacF&4DF* z{T}Se;L=uItx%7lC9SZ{DE$hPLWXy%;{X;QsmgopKwR?j@~Yq_5}nqM11KY#F8;!} z$y(~GwLXk4Gc&CoA8z)X!LpF%!m$7Zpu>0W2=) z^rf?F05p?&34aW|sKNwJR|uw6Crq_al51)TV@4s1Vm59>as!>ua)}fnz@NkLlKaX+ zqUSMPrV71L2h+%40x=CW^$riUCj@GG@Csz0s_tfh^`&2i!CbZ?$d_{_f%-ZC&29Tr z;q)u*L^8HFuv`XYI0&cx(oAi@-&^|lu^nYkzQLvtOf(;Ih>A^3)z7pd zAI-G|?|8YypKKzS*5ayXjwn{(GsZ51EN&E#t(;Fd@^$upqP%IOs||JUaDtQElJ^Z& zP=_kdG9|G8B7s+~0$S_(g zCg-IkX%+YtRwgGWq1mVY&!AOGH(no{y*uTrNqf9QiU8umDiv$6g=+kTzX9Huhki9* z%rCQE2du*&KB5pVYW%Lgp#cvU7q%${e`csS>o80XOpMUBaIj?n2_(tujWF8({`+s( zYbb&gN-F8xIaJ%VcxsiPc__J0pVBV}Tuyt)_I!cra&~@BToH&-bpi58;Et#XMnmBS zAVBguNka!<=I|n$OS4aKyx-NE)*B$#d!9?aDz}RIwXUT@UrsSUI}2Vx9KAT1V6`S* zs3TZTA$74ZG&FP<7#FZf(f%-SSFN*Xy?fmRx+sEuagIJ!pMZ8X07{_ww?eQFz>8WY z;Mqwen*<+Ci#9GzNPwMcEso>2hxaWH7D+rr6pCo=#l57b0T$4cZ8(JE)D!vV_5CIO z>+0eXKYZQ1J@zFZEWrlv9%`7Io1?6Dc6Q229)5VZXjb8^M~O^Dp&U~vw7js3{)1ft`-jlmMcdlxk^>_#z7GaMUxV`H!&Ixk5&=>*W*`Edr3Ua|n+xAuAWe zQ|KP}#|>~2NPd78YW||s6Q{;YxT`j;K1j+Sjw8+eIx$fT{yUw@vYWvV9@H!p@c+qh zHvfXb=$IqZtWQSwXH<^vK-a)v^TB+g{m-K#&hm8sQZc7NP{B)TY(V~8JQ^z2v>f7J~~t)TJnL1hsVBM9IPJ+KKD897GuxZp6Ebe z%(!xEXQ%JV8^s@ANn&@_*3|Ci0PI%xiQ|Q_>ktDfx+Y}eP9 z2qXM<{FB=^{cmc}h;yVY!_@@KerClm>STw0{vT|rr9d3vdPAH1kh!s@CUNd<=#FDu zXxjkX*(Q6ssp~&`VXy3Mco=g+qikAQd_&R);*bt=b`)lr$ufE0RRr!#mqX4BlcU+{ zMAW+jF$q}=$Zp{lAXn8iWCPB`1@?(6&B!u9c5OiBgR3~C?C zl(LxqhZUY;ri9`*DGS1KO!{XuDyqx!%SLK%6&wLv3ojR(iE=FM_c6yIl66kVdSThB zh9VUb^&2U&$-HRj#vJj|@7q@(b9$w=*u75lN?Usu%irr~p18(9peHsU@}{irwzYL2 zOB#m3E>t&wK$&GzL4hL|WNr$Lr;MCF42?6#N8%@s~G`EwYcCY#e&J zWSQ!MTx#R27WrdfX{Vy88q8T@GCtUqnVk(i5rh3;Pz&S~mhzsW2ZVzUNwv2U~NvYwhDn%|%PO8V=xH!#@rQb8AWw@BQ%Ikh(!S$9v z%Xb}@FY_fL*=b#%A8+nAl=&xWIIE;gA85WR_-XwH*a#a7>*V1Kp@OUn)jP^sWAY81tSj(IXgk?GOh=;0Yh)qyM@8)L`uE}AZ-5U| zo{-R3-_nx)iZS=}F@pr+5;PL#jmH0$z}WaphgjjWXKc$gwY7iGkFPF?^F2WnL!H4w zN4E_VPxx{!Cs1RMEX%Pc02CqB%NH}uibr)>B>1KIMY?UKm&yoh*>BPVtKfD|NWcs) z6)Mk#S(QW)`?*V#q<+%78KEHZPxe4 z&ety|o}0&PV|%_kRo3Z%+a!(oX4WkEeTUMLg*lTjpcJUm%4obMn9(&GhEj?@n%u*l z5iZJGcw>Sogz#pzjiyB)8=JK&On8@u1}=^#^kWwncBl{>Fj5AFLrsi6rkuPykOt-d zlE9#?wqE`kW_X1M~YUFiMkih%kPoAAq}}tX7G6xCaJb zu&x(IQ7Y;*!th66YEK_$X6*$z_MfNEjML_>=?dla)R7Q9RgGN}0}}^Es+xwPhaWdR zG>nHbNLAI-Wr)9g$|u6YTgihiRv%AtDFU`B%K;CD0Hni{Vh>T_&@*>&;a1XQ*#MrB zB*$|PmDV3i-PsiiFgNpXaY;B0;(XyRh>eW}gv?p+jthHzIbj9^G^qd{3#HJqmx(|p z@Ie_r|4UD3dqo8&om*02J~e&YN6?Whc6w_EhJvaLja9rG+Ubsj*5@!N(B1zi0_8)` z#I(H-PNI4%z*1+=^f9(QAa+?6lZi|q5<;P?Kb3WzE7&ip1Y= zQg6K}D?2^uaY1x7A`y=@YYB_pFf27aGcySVg<|E*%&og`AD1Ipq9`dT%iei<{Rij^ zP|oCC51Jl*B6!6Oe+A<|xWmiOKlb&jT++rX#Ht$1(cKHUI2yI>?LmM7>6iwQU~nVb z+QdaJxxygXF_Xljnp)wE8B<3M=cb;*qJp_GtObEi3yEGL&x_C#iS1O?{4<=Qd8g4Bu+BW(|>`_Z3FT@piW+; ziKSrtG5{nxU&IElNP~44vuYk(Z~&JET6PHhb7+J6LXYtT{5?E$2>qO#Fk3?|P6ECl zQoscHzs*hDqD;O+pVj&4Do)az5@0gV_N*#EW(JFOCm^%yssQyp3Ms=-CJHXRScWwx zXlUZ{GAZQ7V#m~+k#pEYSQ}E@xO^MUx6i zq9L>pp@~Y;l1ivlLh^fG_w#%Hd0wykzN>St>pMQ5_xdEEJ=kGsS*RvD?=?9&Ij?B> zb!~h|OPoa5QL7fWuPpw6IU{v9xt-VYDg6tSN^Ap; zqW?^Hg|Y)|+c7Y*nQ>YUj*f6#*Pi|rIN;p`4&7cgD4DG&j)qrQ*xH$^Q6lvtIH&`a z4rQs0jk>q4`mfHry9&~G6l)~J7BNfovM5;0)|&sG7%B4gb{+v-g@enYhyr&R7E9Q; z<<`=43>hIEK-mDlQSM<4k@{U7BJ{V#Vop@K)G7bFd6L1v(qaGp55o_xj`l!8^&6Wz zag_)uXVceH9;Gv@KvoY9LzN=GBUH!PGm;mLe?J;!vMn;q#z8ZFx1Y9m@afZQ#tAI` z+G!Q%%_z~Hbox2?nUqMCED+Mijvmc@yS}e@?EB+?*W7H?hojio5jb%S^_FvX3hzc? zLBX8hyUN0abYZyzZFF^Xan8OVpq!nZ8$L(bae9KRMRB;cww5~Ve(B1UZQ+ZP1j=Ex z4QKH;9S=G7$SqP*F~$cabpy7d_liZ$dt5_9Lq|ua@c}@6!{FW*oOb$ZM_!IPO#pod zQD**ee)i2KZ^>1Kr7W`1E6WS1iK4dj%S^=BS-h9muU`+I`1SA@TRvGhWO-#J(a-Q6 zlSnUt+(UL*D=-D+45~((zOIFWrF;GhQmBy{=?k8{+8zZ!9<}eh=nTa$@e2|-mM9!fL9{>*^D6Ii}jLdCuoj}uFyUx4xoRg4M)R+D}ijyYOAnht= zNlD3A!T6Sz^)I&{qi8qdrHL{30|SY|vnxnWw8!K|N%MPC>75rhQ`1*=9HM#s7{|4a z5?fL9bv2wlnXep8|L)A>0ZvzDFYogSBrCcfnyRJL>xETS;=;nIj9aj4hT|r)nA#12 z{e>|e;^ZCX;oLvtd5Q4q-4q@r>gU)n)r=b5r6MN$9Ov7l<_Y#kgN&dwYv>jXR@ zwXJ-o0L>X#N!11Dz7jMVDr9nwC)Cmquh!hVmjL4=rx|M|i%7b#MkUH3b=8D_fXEZ- zuU|lwl62}iy+u)7VTw;DLn+lHhf*q=qLR|`@^V~UoI6f|$5=(K`z2LW4ua{Ky$(@1 znrUjY{cZBC-<@4tvb{&J&rmQ_TsKftSF(OwfvWSqNA>%kC}{3 z%QO!)@32EFDtbx&BJS{K4bje(aKm1@G$>(Fz{2n;Usr)&>k3-WQM&PUrgQjy_313>NSHFXbeZ;!e~Yel`w4(j*@zcd~>dHIyz)^^|G&-1qP zI;p@O4~mS()Qh(1__yJPau2r>zP|qcoNEW7o=3bzwZ+%}m^G6z=He++zRwNxc9-RNPlunK1+)_In|)Fh|zs6_4}Nt zPBA`l#}PT?PF{=Vua{GDxetSH*$+<^Zzku>+>bC^S_7SQ&SzOC8u{JVZN;?9qC{O| ztqVQ30yxKtKEC#hv*lAzU|_M^+StrFaiX4e#TXZ(Saen>)ktnNRHWj5TPkCKWk*a9hZfMcqHZSKy-#l^CrLL-}Nx$MB zbQm+=PH>sPF9dG2b$K`rMBPzuZ^LE`Gp6~*^?fdi$WzjM|8MbAn&g@J@;MGsZNp=e zV=~^3f7jMt#HP`oGqbfln_GntYneU%6ee8yok4kvIE!ArdR5`RI-x%WLH2|Cdczg$ zyIFikyKGcIwRi9GS~DwgkUO<%e(ozx;VP!H`aC^7fIq*q6kJN{JP3-`@w{~VtqJ63 zjS2V6mZgu=e&EhsSdcSna-1!%lHy}o|`@74dxvFZ7k7mDjoYhFFHk9i| z8D#Lbe4GJ_M&=hAcNuzr|KXyXWG!PYITYf(VNCgF%agB=BHYCF1d(Mm4wWcW=fX$5 za>#2FBIU=iK8LE^+fev-hwCrt?{VAP80>?u$9gY9KsXxOX+zG;(UYN}oE#jv!UC+7 z!h!o-TrQuyn3|FCDkx}o&OL}LNcy3laX|v}NxX6`S&>8b9n2u!%fooXt-Oh9CMups zj*Rc}TQ7@=p;k6u%3MOJ4>!Va?z%M9qvD4T|CU;1yUE&;&2YiSD! z3hI2Qd+%ZpM5&gaK?SxY4#kZmTv&bPnyS_99J5ws%Yl)!pnB{?^9u@~1hW00hG{{N zF~W_i{BWazzZ`laaOdg|1EDR(A}SX9W0yk~HLEYoxGrZgrvG{GDX03;xAPz0zlWP| z#NX1*WK%Qq0(?~k0OK}teH;H?xBl~pJGG7jN*yfuku4^IhHU4DZ#w400tIR_Ov(Pn zn3U&%gB=~Hl5-j{GAs2rTx9I(hTqO_m&-v3xuD35r#8cQD6LTqg9Y(Qt74PZ!Sdr0 zx_a<>y`7{P%?F|j1m&C83_|8HA`EC+-G$7khVDQOzE%N};HaBhbWYN4!gc=L_rU1Z z`My-%TWPN95a^^DKf*wf+I4J*5trUTE5Kj)0* z&1ly-=^eMk8(TeNY|AE0x|Q4`cXEu+%L-dNk;qdCm2&g)>pHLchpcxrNenCX3rY{X zyb%vhkd3+pm*ug%Tdq`;*8Zjzv-`ERFU1yJc;;{{%=TB1UPdS0$Fo#_%p1ZkGONU) zV=M`gDpZ^VO#6ZX{@kgCX*$xzrQJY}=f-!QvbM1a3Jtxg=EWff?Eh`$uC$ZU5B$ey za`*+#xp=VWZ6stQbyoYPan_^>sBB=Q^1A|UA__41UNQOTPV%}SDn}N5Att{`FKxk9 zW?}c_bLT8{gVIW9!(>1kd3@e0_mJuIuf+sEy02f*pdg4%ep2|XDHveO25D}jwq;$s zxcZ0oS9I&xV_a>hcI3g1gjl>`=Y2!lFK_C2@gqMq~*egk}P zlR+b)_B{Z})A*au3!|KfuRI?r83R6hs<280(p2!N*fXPME0VRmLCL42c_=(*Nvehh(zj-Xh2AEDsU(9*j8LAAUoj*^)Pfxb!rGJusLOvdjif_&}pz^ab zGd)jC7LC96HY(Foa&nvyI0CwGxaa)asJuYY`cfuZ1w{vu%1(wrbB8Wl=7EP&VMRF&(tc$q2m;Em3`kGQ~lp? zAJr#aDqhxuWsf~8=XWQn&;(WG;fj4_$xKtDsZ<;4JIUTtN{WhDdyFoWsjpx|Q38}z z$vd%ocg!>~MeCM+)7=cQIg@a|MRk!Zbk27$S|TFlR)Pc7d>)~0-Vjif)Kpc+P4yppKmFc% z-K^%Heb5AZ?M}vRG&}1r_DIsi1}AdTUS<479SJdo*i2$?d>dRoQWX^+#Rh3#Q!OhL z^`8^bI_r}50XzsKc>Q-db~0~saZie{ufk#z5))(p1AiUCx!G6j7|kt<+(WX@%Fo|^ zYBO`@z`y{OF`KY@a;V}P$u78m?RM>2K4KU4su8@3v2o515V*NJBoFuprGLI!%}q(Cu{os>e4t38}8oImQ~P3pgwI& z+-&x(a{mL(GPeGWB=aks=O+_Ig!w(n#iR$|U4@j|9?^XVt`Nb;2XZB`*UH!BI2!OEfhYAA8D)qvzk4xz z1ppn1k1tax{29G5`^J8T;Ud5oy|o+1h?ocG7OI`u1~pBSlK<%Tv3@f}&0^N{h|Z7q z==;7tN4l-ko~bCaWKS_RZUdr`l4`8+?m;Mg zHexJhP=48ispw&Aiv+wiBD36CYHDUyl)F*PYLpokL~iA5AfBOrx(S5z_hExCFg0LB z)km`|&~a`B+4lbTd*tAT|Cp+k{{Us*XgY{5&&LZt_nR;|*z)UIy7bo59Ni|n4uuCI z|I9D!2pBV_W=YY!J1KTy^ZYn!CtPFYZCNxw%@`_W@+4lC-siHiA!1n%W|~N9kFJ$0 z7JbU1bS0?XGR}`ldWnYJst9 zJKp;jQU!fkXXyY91TLFu7o&*kPw+O{e1`(SGL=xp|OmmYf0_=w92hJ-Yyo%)zgMGhHgWxn*O-t6@e9XY__seY!c9$0&@b>w zNObA(N8^>>sH{8*EMEumyGn}WzVwWYOs4A0B%6+kQA16^Tz20ze%C>VeHVQpD}(g+ ziO@~2$>%VAfS)<2eg=1=7kKf-SQMluzmF)*XjDcQk|1K?E!#`JbGDr2$jggHk*jbu zdLG3Hv)xA{Vwtsx*w=9}OL{!A#ZNeny^HLPy4_nc85)dsR9?Ssv8e?-8v*1Vdh=ud zW{M{+_hPH~`bB6x?4BnHVvfBlw5!iKV@wWtdeT!fig?*x-S$eU^B_+ql`&mvP)jWR zuvr!jjyTZTVE{Fn9zK-noDN5Un);Sm{vs+_W!HKNP4;|!g^?16>j39u9<11MVE`WE zC$MDP(Qq8JwdFV$C%g{a=<>yj-GQ6J{Bd4>y06E#X;cbD^!Da?p0lWF!y_Vk3##_s zIRqvnD%f+ue_^h}KK>Dm;fm`f^3h;@(2sPFS{WP1!sX|aAYyc~Tk}ANl&5CcO8ZXR5_}vcXWFH0gJnT_wJ_Ua@!)A zGokHtS`(ZUl3vt0K$66`INAb|sPJ>MWh4lsf$ZtFB0E(bb{IDl%gf3#{${av??oW% z=uGDz^}%&p?Ds12-D5zpEG#T{?%b)CGGSyBM2*WZ;9ZM6o4bdfaR|FV1|H$yki|vI zKu7nR?^`z{BJk5dI}iBCd@`BoemBSz3ybj%VD+u<@9VXeP#dC zryCwWabKjZQXC#lp8I+jVJZ3V&&dVQ3fuv72bX?nwj5y;3)X){xA)%B^Dr=M&2ylu zqOTM_cI+4$2}|59MJNMs_)LnivQ@s2cf3}3sE8|PyxrcCYw~#UtO!)am3Bqx@@IZ6 z(x0QE&-kO+KTv!rPRiQu`On!mHZM=zOhuE*7Gqe(Y-FXDV9z6YKDAm>`Xgy)?~)`0-K0ty2Ih6=^n~d?LI;StKL{tB<0(E z=dcC*OR1UB@lW)87_+g4T7K+n2$ZvW11|y8mw8W^%uZo|TxvY^X2}%qc7EGd1Uh(o z%AMwFq>H4f7ytqeOMEY_+52kLcB%4C66taHNFfZaU!e5+%P%~HL>5HZF}-;I;JerIiT@W&y1gNS&qqnCK!J)OT$Mth+$<<1;)I=gRvfT1;ZHt&HlZ zb_%^y*}&`d{8&PziBHyxw~fI@^sy0%gxntzU{Qle{g8S|6NmE-{Dwt-!}ZW1Cfb>5 z20#Qh`|ZgEf;(N;^vJv}Rymb*k+zI6n=!Art`SA?!rozEeBgBGrx|A|t-{u!C;T)1nCBps=y>_k zCD0z1qZDX#dhJ#c#6)?6rXQ7RU%SfeXU<*3<=~_aRJr>A>5z*$2p%&}Hon~S^q1fz z-Mu_Li&BdUcbPwr(d8I&-myc;g3<-778-%^iHVBmxEo_UPQ!UR^l=^*UtaInSaau2%w4)?im)@uCC5f}^R)5mLC4y& z=H3xn@1ouqn~$r6aq(F@J5Q`XEl#}+zv~q%J_Ea)uN#y4arOtzP~HZc=8tj8wRLsZ ze|Sp-lU z3-R%ENH@wl27{d4bF*U;%+n0cN$_C4M*(Z^s=Nsg4UqmqniZz%W)Zb|v}Ct+CgSKrEGH z*|*EbB26W!hAdTJwkPc^wq>gbT4&1RiHhFxs-dDoX*tcc;Db@#DvnRqA$B zu3SF*lDENbE9C)qwpD>d`8~w4LB2fQWLGFoa@q<8xl}5oKf*h-EaYrL^7EH3MUlq= zvM21>KkBbGdQYN3|2f@q4nfArntK7)3ltMPZygSD#d0O%=nH02;b^4%9xW45*8#qD z1(YTfn1xxH9bhwCdG`Tf-r~v5!l7e>>&mnuCo^+lWyQS2@90s5sXzP+w3!*sW+ogU(soNSSeHV4K_pbDMK7HvJrLF_zlA znJXfpl))n&=Jy>QP^d-MaCa5hwR3s5pdo}lN!{WrXN;al2L+u+kCbWkwQEPtjwK** zV|sQr*KYMYK22;~956_FJ*4#e2m;mn`=v9ZR0j>`XOOGz1tPTeh@Er-Jj%;0N{781vL6u717WsFwjch? zGo?(Yx6uarAW_f-^e7)8i^=V|WNKl7kdkyb3`1Wso?ZkmMqo~%ie@Oo&3?Cb)5Elx zSRP}23}{kfB7+!<=hgT6zhGfkr+dXm*y92E^aS?VP}Tt#Dm7fqrJ?xYQA~-gsmONV zjB<2v7}eY7?A#m9VJpHet}l3Y_rlLTot)|!>X{NaBR;%+o7-rNYaalC#g9wfdw~VP zw&Hm88#*2-|B44%5Kn4ooOemtX=s)7J$!XuYI*0xzcJrwIr2Jg0;9^-1CLJYR8c2Y zOS!s9%9Gs|uqe#FSSFr6h{!sm(D>%acuYW!knN_br3IDy79}ziBm2}p@~l(Um7hfM z4RM9{Te=)ZYp!tCVrt{~90}`yvu=mXx5P*!q;i|1a29>c!lyuaMxXQ*T~F+}VoTm~ z7VOXZ<7a@5Dz3y;(^}|93MwMPrdj|+It*5;gG+g4Gw$e$-JRnsEk&g zb8U~4lP`p;b&}>Ot#6Lmy$ADEMy6+y0Y*KvX%da7cPh&R{+eLg=jXj7*WqfOpVQt# z;^y80Q=sTyesmicfd+Im=?p?NvYLvii?id`ar%JQW+|}#-~?EnINjd-^gjmN1VX@- z@0@Tvqrj2O9J6dDL`C4OIx=P4O_U*N+6Ag8P=4|O+V0~tA!DrxxQ)&UZlTeP?*52m zidPb5HddFKF2c1U3yII#Q;=k1M==%qTBaX7p;a8Py^%Rw-Q>-hSzZe}I!gW7~h z=9ulf0Qr!%k8CLkYc`0lEE%jr{9nfp^ukF*nw@^1s>mi9uw33}zM#Rn=Q|iU(_1Dz zuQmDxTF1${CT0b>AtfVcqaQ2>oq0^C(?EEqJ1{8eRtjdd;w?)cplQY}lbV`3!)!|p z#<)>-V+>2_mwf;sJmi)cjH=GRdL&UU5fTzIbYUB#rE;JxCJ_CedL=@Z?ez#hIWRCZ zWGoqR>eP?3PiS2gv<|Sj|Ao*X;zf5JA1en(Vn%1&&sMV_T%Ni4sX$$__Ba-Ie}boOEDKYFwr{j|+X(sHx?uoI2mWoL z+Wf2Pf4@$oZY=SZT!x3vz;9SvI4NdN;%CCOJ~00ggnH-d1|xOzBvpEqA#kU*x$6G! zD$d8$cirA2a$G|HeXPK?3wCyCXgp&TQdL>eq-+_-HGsEH(m_~<2J#|AZDdjyst)%8 z0|XD$uU3nS0;r#3d^45_YTDRM@tJ?km?61GpSMo3+F0_Qu9E8L6(1P793)>uc$DanWhV!=IkBYHx3k zOGW(^w|H7ZgU+l&p0Oe2Mf_urBS&^0@w!`>t`0{uv=r5uX21Hbo5s{bBHX1SDZ-Hj zP1j)Tg^1@g->yPPbR}q^lLOE~dk&7EV&~Fj^0$$P4{#DYaDQAYJbBGK20IZ%*~<5i zR~4Q?Y=YpptNdPPaHz0V5}F9dIekL8`kKG8?yG2F&x@W0!kgfpX6iOb)j+IDuXrG))Nc1~;1zA`y+S8Wd`Fvw2|Dh6xF1p3{K&tq^4d@ZF7&*&-`h%( zepDlgFQNiwFnAV{)6(J|qT%w`9~Bif5H{Sw+`WD9yMp|DdSL@PXZNRGDq+A-^ug`-R%DFhyaKZPSD*P9`>H+Vk>5;^n$5wuS|Aum7|Zx1ieL zxpx-8T)wI&OU54)2KZ(wpsaN28X6c#7vGw9?Yp5YN{H<*4}a*6C(Pc1GOR6H7&qCd z(g?2RTpv$Z*vbC)nY2wA{Lc()`fs^}BEdhLK|7VLHnyK@d|zP&k)}CEM1Tu!DaI9C zdU0(1yvo5V5nhnZYr}!k;W{9>Wy=CCg0fSr@mH@Ngh&+?=w1( z(NmP_#Ho`ZAqASpg+IxaDzp>uoI8Zw!bvy+6W9SB#>U2yYniW}SM)m1kYGo)tF`B! zRt99-0~`k-D+4%N%uhP(*>gRXwo6pYwzEevB^A<`4=5<&%SJ00to9EBaN=XR%4TreB$Z ztA^ylSyzPCUOMeZ;=1qMr`<+s~+v*~-(4kkz$5 zRr?A@)Y`5}UZix;yfHh|7FM&Gn4HWsfTs8CkI0Ik0&%%*Oa>H@?xYw}iFl~Kr$>nL zxKJJD+pzy8v3W?bh@14pWiyB7H7UA^TQk$~RWLR97Tn+H0G?5b7q zfz?908s2Yd{1l@dME2}BoqURc|D@fX?jKL~m-vysJ$)J*mu|G50S9+{_vfcS1LGnt z(>P1MlDlz_M|Aid5w&6&|KH`4!l*>nDCRQven0p1t9ywi50r_`U5{wA7x z0xD8}BepTgGyCsTVm8l<=k2&HG<JpidzefBBO9BDGx9s~$*lvFPJeEkuACy} zH*U23DI32F%+*6Gwfo86*O$~781AO-O`I|BYAVzwBfq)j6-A-9 zuMdNnm}s+Z;KBy@6D)qbw4c-0NG5B0JWZndbqz!RcSvi1IMg8oA0Ht?)Ptu?5b09{ z=n#3hG3=6q`1edjRG?j~kcZN*Kl=&{2Nr48p|JT~-4gwc?a!V~OoZLThoSU9L=Q{m zwo@yL2M7yCEyr`cf(>h;&Hy{vQMy3Qk{ySX4IIQ5>85R!O48+68k9LHCy&uk% zf%>p4Q|~+e-bat7*7fMVvYTJUh}2C3RCzpEX^YM1IRno5%TCLsOjG#T?UcMpKtaem z=oW4-(q5Gd)lyVsTi=d`c)Dp-tFAsYP{*H@>qXX=L;D{_oFAqQ>vU*%$?7WL+{Z;| zkg+Ay741?%U=>G}h6Qp22$V&Bl4BHRI_lBz>aL&;XfCOHh1_Lk$<958<*Vw5`g^U!k;-%R|?m^0F_uhCfd3O2&A6bV$Pw5e<1)uP41WhzfV1! zenzP;Va|js(R$NcGmk>s-S+l_btfdgU(6IharJ`b#7I*S^+9!8D1Bwu0W63M3GFkc z{pd*nK6@J?w$Z%_D0LltwZ3aVkZZf{eOh9~{))}zPl@A>_?wu}l7wIH$SHU9wE7!k z&RmKgJou1XD)6Y2#bE0xsy@M?Px8x1148+{v$N0CA#8_#t!&+iy~Z(@bx+RiO+6p` z*@UCAlvYAKFn0rYYvv8d>m2lqG^ysBl*oA!W>k-eSQ4`L?72cZ`{7PBk_}J1N-t6< zT!9opdSGECim^l4Pg}6rU@;+Rr_UgB!N*>TCaVHFRNK6Hbz-R=ORaBe7B}FFJA5>B z>cd6oiv$#C;!k-%>4Ri8JlE=;erR%P{N5RJL-Z~Otgiy(*q_x^Q&Uqzhl!N(ii)EC zQFgD_`Qcd)X*QU&8`jycYNvLAr;$C=w@o_?MFtQGtb(rqE&Z*!raL9oc;VQxx zO3TYFGkIbe<7Mu*0nVIuS;?K(0r17NNKpMxvE+1<{r z8~3?4wTij>ul2tV&YyoHLCas1(xYnsQS%wrQUxvFK~>Ker1HzdC8SrXVm9jW>Q{m5F--lbcsL^NmpOQV3E$LATET-o` zDUmRNq849L9{Hr@^PwOxP{Is!|E;Fyycc`S`p!wB>xh;A6E%s(7&xg41|J!`RJpGBItjlz z)acHhbRL3EBM5yDm&fx=(U*A|Hy{&Mm)PIffQ*72QC=i|6w~PjynK8Knup;QsfL;Z z{|6pQRM_3)MG2MV6JxUPP;E-E3-Zk&jh(}t|9#bfoB%EQE+AcKM{ z@9IkIW!e&yn%5I1r!9DSc(7Y7?wZN5x*7-!v2s^KPs7rQT^I>Bp6~JF?rkvvAt4D_ z{F^x^Ske14OhAo^EdI@!n#y_yQprX}lIOeu+H~aes8<^bG0P$CYXL1m8GWq2zCKEs zjftD~H1}N#J^-Q~$~H*;pOV#mb0GHz~DM1L}5ZoRt{)DcqBzU9V6mtSAjrIz8 z$mu{iNr0Mk17n2j-qi=w9*q66yc4>mIQzZCvq+^k)pWMVGoQjh92J$1=p|zpOTm#lRM>_VMn)|# zkv(Qu#M0=lhqS%ANWI}T-O0$lnosvy3Ci~PKUlj+%bnTZH zTjZ!>LqkIbAXAR9V{)>x$?rP^LmH6^hR(2a&W;u@2Wxf9EiV{IHc!b&bXF5AmXwsFI+RxYSAXl)9>0K1 zNBm3jJ?U!0C&SPxI5qbSKq_nLAjl;LsaWUTH>Hu74!s#C*(@#MkdTgpZnKmz>Bs}r z*t{EU74O;<7+$^t^~1`B)7a?XbqKA-wwP+%#A#`1CH9%Xv)_Oh+VB*yl*-PvWH|Kx z>>p=!$SN(WtE($28pD%zZ_`M)>%7LrEh)&*I{>Rt%T7E2#5z>JBU-E&wV`)2t z5z}R8g;RnQje0t;Yj46ehxMWSxL~DxX8@%A>U&QBF9f0*%ARPCwrt5)4I@p+CQ}01EBIU zUa}4M%6t@M3WUJx$UXAkr^u3a1EiU=*0cvgv1!S>0#t^#^7CSjT}BUJAe721tIB!+ z^mp&u_X}F(ro)ML5b3pWIieFlID!wGE;w@wi|Z(2Yu~nm@IoZF;T!4T>-A<){M2Z6 z5%d5Ub^a_pRUX4teA?p8)1Zp@MXUUAB0vF^Jv!1Daw8*y{Z)fUj&8^7q#v@48$fDw zf1GqAva1_irN=#WZr{ek;u>_)95YX?@ya=*fC6jG!`P(_Q=N{(04r5Q_EVCSY(_EM zct?(=bOcRs`GcXJ9&Tp6<3s8L^V0s8TMPE5>YhYsDAxk1NO@b&HfdrwK!?KS(Qm15 zwwXE*3XR0}XokevWmJ`f;7Dc>(U{O0PTf~Ax}0k5xOAhR?`mzWIul7yK-rZEnV2^P zw%|X6w$&m3lY~eoysk5saQ_o zW((^MF28>LU8xmsU^*c&;;~nrUz$(zFzLI{68^PA$zM&@Q_eai5k;Lz%`ad-sGz;K zYHtQn0QVd*AaeyvzK7?}uU{>R$qgsJ=9$($LIeS>)B;!pzVCJGc6QA6b$18T!AY%G zot>SH@0+}vQ8Rc1G6m@PI%5T%urtX(MwxeX4WO;|9!UL4Uk`zP!$Cpvl5wCm+E+q} z!_9yMiO%+RI?Y;@c5CKWywL(I|FU((K%Nrn?>_2)39b^HKE+&kXeA-&Q|pTt*7o+95t$&$7%xfRq%hQncbr%B?QW8MtVXL z^z`XdlJjvtKk#eelj;Lp!}tJ5^Z)$4E7@Xldr@>^T=vfON+{caIDloK{R4Ucui0Xu zNL^q-bVvVT7)Br^I38fR&}(>zATAgtwEDS>vxuq9C~ioX zw*U=;J$tB2UqPV~v={)Mzkk2Y{cS^MLl*vE6-gr;h&g4o1;I6T6#o_tU-IkH`<>b3_ql5x&8HKTdg>nyN3Nl2$N<_)wE^zG{CZ zdG9$Qr0p*mcvxs>d~X8we4G>)*9D3qa!Z4gbMK+?CFP96c!i-!v%CL)j3)RQ9}Y07 zdDTb!x51i%-;AukYH(DR%0}+tIGb^5xEp>k6f%soC;7UE_WNTD;$KAT~l3MT}vxzl*j zM~8^TLd^Gm^-4)s7YEV1?pHhV)TJzz@G~&llZ9m$pzJ`EL!Pqfl;R9h;~zcMytVL6 zBt&=!WLJ0Yqv-z0d@gSXwj_bD@#QBJT)D#WNom`I(6P+S(5=WjJAVAQU)Q@D=9(p> z&iH7i$-XYVph^_ff=@o7;w7Y@;a>9FujB$M3*dN+muN&X4x%AwSar^i_b*!Y zWg_vI(9oQR8{!@g$Az=3oK6CwaS)JFyb<-RqEKq|oDZ90ZGB<>PKc zy{8Y1%xWG#`{e1m(T__0<8okA73N^@?FOicTfH|gv0i|*t5n3-^(Aakqxu4s zlvMG4{V+1B`MAGhZH&767>rg5FA7bzVIEA*TS;0OMxp>s&-w)Bq13lfsW8yHS#10W z{7WF`phj*ff(j60^2o-&5!H)bKR$jR|CpK2Mm1Jq`qYq%vW||B`Idw^Y#HZTZ=-Fr zbk8ubGF4(*+7V6x&M~tXEkv{-<+=9lDwfdaSC^kD1wJFI+GYQ|8^gYg6!s24h&^{j zur=ne`&$|iWy9%kMc2DX2F<;XKjHQU=oGeb4iRc%Vg&$X9J^P5Izd@3Z)5n-=yk92 zlQn-D(j|V~b0g6wP6h-Jz?JWs`eHbSX-t>FDls!V>rl5PP$S=bP1@HE`s(^?TwkgS z^7GyLYFS**L;g{_?fC|zBsG~*f24H6Lk-fPnz}YHj)C2q>e;hrj9SJvehf2!w(K8?iU+3n$AG#-Nc2r=e_MP5f zsjPmrx>x0q13y%W|AE{FAWSgrgibUc`TC2B16 zJ?Rv!!0V9_afz5e{m{Um8}p1Bd<+C1225Hd`D8`LJ7?KUR4^IEcTprjrG9guYNnT@ zG_}tQXw&m8!hijZQDq)h&^OAXX7&>2{D4A_f$N20hb*A}vYuALPy5m>vA%^)E3Sni z*!{>@Jk!aT_shh@T}bZ9Cu+6qo+G)BQVI7WVm6QqW{Wy%#*s3C>XeO@HI7{aXhj_s zSZm@+f+S((Em}b}2V=1Cx1`=PP`Z2Uz-X0idZ$Okr-XDt7^o_~E z1N8wiR|<1;`CV_Lw}YrUXVV^zNP%q_-=ikWM5pzQKq%1Fq9qc4EyLMtWzuM`NG-~@ zX19<3S}$GTAGGy8MLQ>ja3`cX9Ogp&9g#XGG1~F%WyKE0Eil-MY2E#GD<7#Lx*d z%t+zai#r|v2%*U8A?Dvu=Z>FyB+5g5i2Jv=W;29Nmb0zTqFe=c?D+HfYSjFfRdE*E zXv26B!!wt(X+NA(BQZz6XJ%oEy;geU4Q2>rpg<*UnSonbattKGZ#eZBi|8aePt-71`s?`O0Uz_5^4`f+O200GgcHYai% zXb3C=O`|X2T8*3)6h%c>0_uS)0uq*XA8T`c7;z6*2)gxJFZRhmEOi`}XQo~XX{+II zH`Dz&-9sBF^oWiFw?k@D@}4nPwC0%A%e5y#B_{I>^FN6vEfhZ#i(_{7^x0 zq+_%u(rLlKRRkK|d-@__we7z(Xba1FNolEvxA)$73R~>~u1Ebp*a7kcLuMnoEoyxj z1z(_O7O9e(hqstlxS+HW)`56!v&V+~giql2M>o*llo=A4=9kOEh>%&0q$|R_kh2+zW~w*rl#KP zDA3)7A0A)FFRQ2hf{Sc zj_{;;_0b!fZ#7F&T*^g65W-e_?A_Vrp{ZyV4r^Cb752Z9t<*^-T>st84xLzXW~>&7 zH6V~gN~?GIblEtsgv9r`x$RP9)QkkJSuxg%20h`6Y!9=Da1M>b*hUYJ?7vkXzX&4f z$}yd8=?sLCA?sst^$doQ^doT_E=3G497jnp`a2QPwx`i(v$(d{gmFHKVU0BF%=lP5 zz?ZxVEB~0a+l0Uv8^wSMJvQpdohoChgot8ci6_pUjvUuwI2CU)fPML5a_MTz0uDsn zn!Dhk`oiR9iTo!uLADvjUVkhs)H6lX+}?I$E0rj(IyyVQ;Hg&QEUvtDYb#2a2&-hC zt1T3o(S8)cNpD{)t*qo;xPU>TOa<4o**LVHLWhKAq1xUvAwItQ=CMTCR$^6=T?#*7 z>}LhmEpx{1f9NZbI%uxOEOM(PE6V{qMY&AAMs9kI6)rX`iygKE4ZuQwE9FvduK8Vh z!L`D(#X91kk5SJ!KaPCpug4@e(G--P zkRJE`?+> z6Blh@KtNGOJ3>KRPz9)YQg^*_*@9*grhH&wvSfDtN#p=&!t2!RQ* zuDR49o?hZ%{p69u0!*w#?kd_yRAtR*>w3ab;A7(B5ZhHLru>cQQvRX)>`in{uc0E| zKwKsxrtjgyD&AURWX7~y{~@*f1AIb2wD24?zPf@>Hs6%i7aMJ5Wrd!hBT^tvYCEM1 z?+B06+&PA$vJtH^*V}5W@wu_F02Ie_TBR!>0z}X}n38{L3h1$NxRBD8eL( zx*x!1zagT3{L@_0c+>~b-?K6^7wLEbHUI$op~jPS4)8UiPw=+3{mEcUEQC8Y;|Qk- zz42a`efwGxN6|EkpyVP5)m5n8?hi%u7`~yl@C23QbOd!S@F*jdEiZ)s+au;u(fmTl zj|Q;iv#SMXdhgIk8BITrl7^s|7|vV&p+0dh++tfW?POY36fE)vS=sL08tJMWf#0D$ z!RJfMk28V9Xca-nRSE1;OcS_P2TypA3b2YW*GE=$4v+1uu*Qpb?=Iw;dgOTi?;RS) zX`YXG031+q9}rP@%o^S&nzzfl0&DW?z@0dkpVipVsIX#*0b5J~%A5b9DkK%OQdE(= z#Wb`1E)9@9AnfQaaByD08H(^6?_xqfICwy(m<+%b3=?#3JML!ex@PngGZ!A7 z{3<**Iq6aF01Q^npc>s1N;bmxO6WE9hjDkVt+SbwHz83n@!~bx0*I=r{XXapD5@d_ z?7zpI*o`?%jbRJY=_qIbvh2v$wtx9^6*B4KON9{2Ve!MyP#v(qH4e32SsH-aq$|i& zj4ptPQ%r1s;2*jzfs{*^!XeRuda-W|9Toa&d~=plSYwa@AZ3-{rciJ50Un+DwS#4K zX@*xwOgET<=y{9mD_3mob=k}#|F`N|8E>h?%P-6Q`_W~9t2-gTtng?Gm2v)VCI)=F z{TSZ9T>9;O-EDw?Kmd}6d>eGtnB_DH^NRM;@-pILb(Sxf{GxWB1k%&fO?SP3$~8b!r4WaCn!%VQl|z<0YJ?4_2$R_jH>`B5R)OQ$I{UNldPyk zsycD{6;)di(_{s{S^1BM|BLQKR13x?nYnEF&vynyzyazLRitO(iME|h zPDa9?V#UrG+N>P5>F;nmJ0Ip?VHgiYhe^Ch1~$R&G8=)}8hA@@I^U~ARO}pf73x+j z{DDGzUhrL|A)HvNnbGR@d@Lc5cCH#bC(;*7yx#8QM4BV4kp~|Qzk0>4iD^R!c17~V zwmBG>Tff529>d=Ac#8ECuCR<0Z1qt?QYsK?=ccYFPs)&62hb}4Kl>#*n#%9U<~aM_|Nqg~pa3Aj^RC&CL9X}`)I@0v1MgzKpAgxyu3y*;8>z}&wLkjurf7m$8Ui#lJ*pi9>|giaiwqaiM)R=+>-hJ_lpo6UWvtk z$Pv&J8w*5n2|;z@9pLS(dAWtcMA>g;2E7(-$&!0!E1#vh0LL3`UE&T?QS+L=$b#+=wXNZx>*apJ21R#bDi$DGP z{%tQy>Hj{B`QSK!SDfsqzhX1~!BIiv^+cmMe}_kgDOb|CJk6hjkk?wJUJy6Xy8D&QYiS5aTg%tg@vz(imLG6={h z3CHpg57ksQbt1ai=TS^{wj&gh8+Kk{)02~XGBPsq{=MFx{cMTB{}qsKn-@oMSpiE= zhvXj*dorQi-O6O^wfK@{oB>$V4> zRX;}L%;1e6ozTIMNtDQ@OGrrQS=44@klFcPrNCxxwSeD;PD&7%?|VeALit>vrK(DV zgn#|_d2-STkQEO9=GQM@<^#n51%#$9y$#h2(h~{e;bVg~E*{G+*jl?|Bd9ofu|A=A za6?WO$lGzQ6UOv8TO2QsgF@;A9S(m(NVM}p5he{cqY9l#8&iQH|w@p zmdH%b*AM5Zxy&KOi&-w_sviai8}K8fF@Kc0&=l>0k9jII@ABm@Gcyp;w&Ho{a}W`x zBG9N1v0Ac*w9`Gd5Q0o1zj&IwChk?+3qIpaUwUc;vXCC+*NyCo)u_ebX+&;bpM z+4g^1_O2%8C$Pm1T)oHvnh^M)0o)n$V(yL*jWn+3z*`G|_nR2MlQjb^38z!*=iFhM#4&5Oo5X_&4&A_mrrv!|m*8nT0h+!9_7yH22BzNyQ+F;s0=)VH zbSD}xS%OAVwp_5-n*yxma&mlLXFjL_mTkZRu!P4vD%K0)z5#o+guk6{ltuwMYYD8wjGP*CV{AEZ^Gpx}$3prHMc5Fnqte(JD?f}((u zla^5Tf<9_POf%PVKW!1*6`uCCY^LA+K51R>X$(hF62@c$9(G)t4>SPof6OA&cWtWg zI9-W>Hj2&RH13omq!bULN!}d!kff$^lI1$KeEN8NVBr4iLESy=;G>A`_?(&s{ZY;M zX;#hn%4^$=?x#)2Pre*712pY-xa1(5k1+D$|LK=bVeH79;>|sie^~Qh|4j(Pq<9$j zk5Buv@={(reBZmT98vB6c;$a@2CwLv?El)=zZM01(|+&1YTcL(W&F!Yf>5S?F#meo zR-BI}K8DVRU4K1qxJ?`4-xe7gBLCzNsc5`5{?E7juTwy_-d(!_`?p0L6>#{{X=H?i z-`E51hpRnL4F!Cz_R3W5mR4Wtg)c|AYgPh*9tDcYN=g5xLI3NOigZ{)-jlx-unaqH z>MHK-?|N}n27m@FPpR;apENZsHQR0$Z4=Yd2<^BHfdAwmf1aoCt+R}^NbEdHWuPk* zag-e9a#}@9#GZLQBZGLM#!5x|{eE;b%6*d9J?{SAUfT39Pxhwp-CF2H)40Eb@^D@{ z^FP~;Dy6e5+-4Wq?D}ap@MU*ue%|XnP$0|u!1JG-HwZ^N1FoA&F%-d>n0c>ZHDKwR z%R>SPPlSr_EfzTf*uHLf4|M6z&i7pKD}^!}h33q@$cQ?bBCP#lc|)azoT8b|DJBj3hlj3pIiP&v z!12a&D zPuiQMx_?OyYoBScctu4;1L|^Ga~l4ZNJhkhb@UX+13z6y{_~YW(?SmQlcVhoFh3{s zpPxz*0$E5ZL7))x>z`APHVT}2njwnTr*Vn@RJ{Hty<$l~7RG;%DSWT-&sB#79#X~F z!%Khe{-uybQa~2UPV`OXGyQY&N`-;OD*|+D{C53IEtAHAETrBuwfU6tKO6r~yHE&* z)G~#2rUQY$lrySG$UDd7GyBz%jDpJZUXl&FT z8Xl&fH1W2vDbX3Tl9BnM)rnE(peob+A6uILeYsR2omoiJ$H&Lj9>=;LX0P}B^D-!9?VK+m}1f$lzsE9q+3g2%ff%>l?3EwdQ z7mD+DbIHZaiD!q<3|+I~KRf?-IQ&TQBYo0_s;W;&A(14QqW5i%tD+q}*}~qc9;Ykz z`ks|Ab7W7Tjf@WCLyzM{iwnwCJ;$~gb`@XYmbg{M-#yi_8feJf3_Z0)SuMVTUunH`Oh~Z&G9FH+-=xQ z{w04Te1LS|A6h@_TK@BS{{JohQ^rpCK*5KDTK<1o#ShL0QFe z&D(sOX>k0*g?+&)1UYD9!SEjYPk|TM&#NY@+bPsGO!M&z-uir`t(^Z;DCdEZsj~mc zH#*UfpRTe|1aub;mP^ZkQeQA)2MJD=j|_^8AdgK8gb@=`r>w}aIU{>VO>6kF79XdY zmh%7owR7eV_$s{~I`sZ`|HMiZ_m}Nx^=mdHZh4^PWRvkI8%ZRY+YJZHYB2?eQ#sYQ zn@#1h^Lh++p8aeJq%Z#sZ$dr6%|W#cE0vh@hMf_asU{5l2xs7Doz;X&@=mg#Z^Twz zP25ESi9Dc_=zx(Sn3?)J?@r43blboCr*=ME`{R&~%|6*x&hkPtMpR5x>ftQQE!#A^ zJVh2Y10H(&gU~Zly8Y z7g-touOjcbcAE7Gn1Y!pgIdsw8SF0?1FLKnyRWEy<-;B?vlT{VlIaF?EnTt(f3>$6 z#cR|J0yPBqY@g9!gPSotOXnWqMe&MyIz8)k^gkb@I*NKOI^I&_m0uJh*59*cekFge zGg!^XZotOpaZl>7`^&Clb^U3QpNC52E~b(nPh zO#k;zp)whXW!wob@D$Ow!XUf}LfbD90Z_MF< z8JDYHmzzErimK7aWy2<7k+AATb6YG$H{llLq9BF&M z77}Z&!>!;c;fi4&j4?5zVj%4B)W`l28`#|QkOM9~`F$Ps?+*JB<^(RakZ#~vC>q`q zg|B=}3D8piW-<4n1g=|#f|_xKw_+q2Zf-jdf2V*(5vOoLbe>akyYNsFR~wMaNHvqh zrwhmxwMl0~Vu$v7d4&YC%>gk75r9P{>Fco`t2dF{lzngCVE-YhTP`xU_wX5;$xXNa zQcO>#O(b}X|E{biLKaj|veaYk<)*Gn=VwS@PN!h!mJF^c6enA0W0H2z>i;#YByzD8 ztROjIruxaF+VAdRruwta!3?x9^tfvHK73e3geFXa$`l3d1B?DsZW4~~Dd-PZ~Jon1r|KPz zD3VR?tH}=FALEzz{W`wb?B<G#*~wB2>~~< zX&KOs^z^+f{N z>76WV@?aL`ges49tMRdGU@=SJU(huYC=#4OgvB(QfI?HZ3p^Yw{lF$ ze@>O4{3h+A+IqzFiQL`i>;HJQLZ^mG>Qd--KDfrh~ zZY83@^9w7?lW%4z_EX#(EIN$m)h%udiZzqR#d1*fL!KFx-L*vizYz#tR{)tJsojsW z+k@*D_#Iv8*`+S#?m*dhPcMhNZ@cUf;rYoUW6&^I!)w{td$PoKx%l!vxdz;{xC*Qa zJ$wNk1kWW@adKB;dmXdG-@$tVU%PAxwtn_lj5x;63|KknRlo)*f0%v(vX6~r_!gJz8hZP@Rxle)A7UarhV~ad_Q{D;vXZ?PIL`Go>aGQ)ZOD78o5$X> zD>mGq4Q1|<6xR8ERNXLB3iR{uGH>9-JT+v5Q}3(bQS@@Oq@^c>Ud*YQ7V~j=nqfr; zNz0OmZJTY1D!9JuL%9348kjfOx(cbo+u6fPiTGy@LS7}_5TNsthb^a@KevC-{&<92 zk_!PxaJ1q*W2|uq>WdCbdtCp^@%0&lL1O{l7vfDE+R99$N=dR6%n5i4j9M9WC*+ne zVges20hz*cHg@Lv%IXd2@z19#)odfOQ%A``V!_qT822a4yXNM<=~>#Wtj=2>uCLh> zUQpf}TbnB?Yc|M@uSKrmW`FnNmP2woyyV{UlPX?y$b1_r7+KY#sg$!WBz&~efj)bB zEz}9~LP$yiaO`jbsi+4YO8)SB$r;7jo6gd~?Fj5gkg&GsIBoNh82eqXATL>l79Wcu zS&Ju7!D>LaFQ?#pc6N*!=TcO!h5$($49tVQ(VcNoO0lkXjALB41qPtqS<_Yj`=cmg z@0&FG9k^R|%^km;f(I}N?$Z8b#ICS-~)-j1gmT;g=gqHlE$dY(@MeZBy@a;#Cf=s~dpxLItBr>_ZZp zl0Ob9Ded9q^mcGIGYr38b%(d%k`H2B;P%m*+s;f^_lskHsf*EFWrZbaEYbLy5(b5r zwJ~&V!7W)wed_VIKpe2=+5-Z4$JZ0qbgr)o;Ezml=bSEiRciS(^0V88!-&AkWt9jve&5UQV-(luaUbgMLPjm=B5rlM zNzp|+?GH!^6O15R)30N>d7Mf07R+T*REbD*_yGqrg;5OdSqz$#uni0f1Qu=t9x}Dp z`W*ZV(?c*UJb}rjsv00RHm(Il8QWPkrr;ZZxMjO=+I?zbhF;E@v%1tgy$Xb4b&fww zBg8Tm;PM@O`uk+ru$HEGK#liUbp>oTW^bW9l%+&_8`>}sQ0sQ9!h}7Wf7}*YEzIgX z)lA;z{YhE9&KtnJ7Cwwig(ZWtwRqs+Eh##&aJv^|rXI z-e7FjeZ;yFN#Z%Fp)A{(vkL!ZZ3X9kJAsKPt1@Th<2S}zI|FbR=J>3qyep)2iv6mf zsGdxt(kr${at6I1Xwdd|7$T@ zY}6CsQ)V*)4gazhzp=#gImw*j;w;-932~i~(h}^U>zIX|9T+XP@&a_dj(1)Y1v z7Et1Fs$sH-Ed7V*th_=7H?(P|u}!TA(cG2l9jN)IzTnMptN&L<#1eq3yyF@xgpGdN z+h^UW`M^yWyguoCUpzO4jIYh7(V!{k;unkc&aNnm&vfvjRt~zLQV=zzazjXUg-QgL zIx#!L_=zfF=^%Eq!x?Cz%4x`2`qj(iAzaPpG~FvH2K=*z2_{>y<%Ke zm0hh|+7Qt3xcuVRL5=ay>^MvUerk8xs(h}d-vm9KKp7els${4!;B_hyk3_k1sEKK z0y4J7$-vk*6o72De1h3T?N*a}{F43A%OrTuvYrY$lLsO8Y;HD;#?%5bqSUP4(oHu0 zz{5I=`fxqdO3RYsFJq!}YGwAW4I^&L^Hpbl{s6ZdOnFGPckW-;#=ElB#R>#%app!z zsPNl9+_Ut!eRYR4_`d-oFrAWsULg)jHeH8>Z(_@K_7h zxzCBUYVT-`kw7|xoqV)8TM)&Ws{G~S7CWuWr59UlJZB%U{f7~bshRmwv|>|0)FX6; zY%8`qga@ujh~6RVa*BjP^kn5MzQuOutgZZ4?nv%z#WEh^)%*t6lrPlI$8WF_nB6ij zzz&b%j0N*aq-k}RZ9}^d3IAhmi?3Q_8@}btK$@d&bq!pXw9qbJS9^taDwR>AV+?Zr zd*2zeLz>8o)GSf>Xnb3gXQcC;g}Pn{5v z6C5+?FJ32{MHPA*ACSN0g{6iIY)SkL>6Ai$n9;pkPTArHJo0&5lkj8NL^;d+?D6C} zF(Z6~g722313f&2LwA43Y8Vvs3qAY!Vlf@T=7fj?dfx*$lxQoTI!aVMPXX)WTt$X@ znf~@}i9LC+$;aqSqPLnI+~PeX(PL0rz1=rm$8{1Z5m~Xh4<~+fK%Tlj$+_vVrB4bQ z^aJD--8EF06|Yh6yQ~$rASeClQ?;u_yz@Rbpi-28Rr@Y@f2?ec4go%Gho9stdYDivgQhwboLHW&;XaS6&7^6!Re2l zw!3Kl((|Xzl&-M&X1CgW_D~T8^+wAbJMd|T)_ZQv@jNU(8zTI(#na zU+tY!3hoD$Xn@LUGaD7d+w!yfvO0K~Mw=7R`0+-DgTc3Lvb~t^L0TChIyKZx&r3^y zwuR}Yeiy+23SBIG`|8w+8_##5>GiL_h1#LCx4abyS(dv$)iV!LGt(Mo2>OShdKc37 z8zl4zgyOCSUbhlaP>6v|TRrEZ-`k`w@UM{=ciBg{FDInGeem|(%SgE(M;vMk#S^eW zc6p88U~YQDz2fO+r{8{Qnod+uC1gf4j22_hTz#Bh82}tr0BnU3?vq?OZ$govO16NJ zS$eC6_{2VCT;q){Lm)*GxK^~5nF0ao<*JL+rL|hg2QdO3fW)tuXV3BBV!XnL2R6Ae z)cTM#F}C1c1w+g_8O(;NlGKm(Lkol_IyyAgEjvtv;I!**sChTmaqc8=D6Hi>KgABe zBISSpq}SCSTs#LvZ4?~)iccNf#^DyOhBAp;Ry2dJk7vr85rJOj(k|SI^iAjj@A7KS z;;W-PaoG$Fg*A40(ZX*Zp3rPg$T(n)J(ifC?GCbzsB?v}2TxaY&0m05gfZTMc*9r2 zrT>}*C?dF=!A{7iHx&D{h);lc!lbB2JpJl-D4E3Ld95I+A8W@r5=|I!&a;@E&(k3* z)4yGQsdO>HTMGQuDdtZ4EEyCr^82nb?xXT6#uV%Sg0EOv;vbDtd$jAm;17lD3i8`l zw)$)>YFw0>&2R~C>_3@d%ZPV2$=4mfoWPjoi{Q6~Qk!jLrNzz@pY*yN=8wE-0#8{K zbyo~1C$RSEtc#Ac?s^`AVD865MVb9koF0opn3?)7=|8vT^AN4aTo)rUo6r4v+W{TM zEchYKqex{fwW!*hChhSiWkGT^qy;Nx`CfVVQ0Q6O#`j5$m(-O`aT@dG=s0sLh1*?5 zUuvMEh)31~Fg7CJQr;iP(VI$|=OS1_K`n=3`EgFzw{|u zL}pkq*0&yShzlSNm}VR|U8Un^{WOKUSukwkJW|&ZfFcnH9E%a$nCt)Cy5Pt#@NKol zKlCtYUI>t}4WW{{(TEUYRg*Rfthz6jwaKjLhF4~W+q~CZiGv$!R(E5b24Ml=>Ac#g z5%PB*U%Ng85S70vAed2iH!wtfo>kBsDk*p(Uu`jy}2e8f8UM*ygQbq>_hwm zAy^i0PesbTpS{_-l2c0w&h1i4BT^dEYssvU!%3>gbOkPiHB9^+_HMHpTVb+EvD8(~ z!u{ldSy0*qD{z9*l_piA{>E$A*8gdWoQ#MlsS)`cL+nC9BW@HCDNi zOD7>|LwDR;Vsw(AEF2PC5hj)sF~x^lBG5G$Lz%u?Mk~!*tjMqgg!6qMX=MC4;ZRnQ z(2$C!mU{g3pm+oSo+<9V8huMEOXt=ApozT6L(?X&;irt}Nb0kzJb7wGUsUX~aKVM< zn>^vjiGn-A@lG2Tzbr=jQzDk8?0r=;z|)O%Tx;NX%BX3-funJdVc4D0doluEK&H=D z?&>>sQruuEXUChoLj_Ax%x-UtnDrKv>6mqr9kc9{MGA)tX$JO+UG<(tZ{Y~@ z0uCiZYp>DLzwo9KxV7>y20P*0LD(7ZYsEJHVQ{Rp_tBVA%x466su<)C+^B(ZE4UD* zRveiS=5;rmJ3#W3+3H~X>&Xhsq1C%%zji0n5+4W<+S6WgJB_D0b#ra8YZqnbaYIgx zV~lc^=ra`DxZJihpQ+~Qs<@n}?BAmj6qL8#HHH4-Ox*1*C0jwU!S(DT7^QjMkR!%g zD`n?%@7CsbF0xc$p5D@e)3WU6|FZERrPb}%nD0BVjIc+)%;v+IzCQrl;4I2-yq=0W z$csN~2aqzX+AWC7lSXa|oHJDQk;1gEb>X>88^Rgk^-)}Kf z{Ra29o1k}Y=^>V>Od%cU{(24qXa*|c)lu1)>`W=*U&KxjEfbymxe3@vnRI;$n@YzU z@fSXR+qcRaq%hG{twYwg8jhZC^tbTx@A19#0Q<&Qn2r?E^T{jeH(%h_;N?5UB;sn= zcl#&irqDFlquGntErdVTa7Hd6H$Y_&kUT%0CKuWof7mcJeX=xqy7*9cq?gNn7r0Am zQl4VJV54>UmS|HLhA=WshqMj0Cov^)&mI?o(O^y&pYM7rF*!w&Td+@djmXRSU?ZL_ zza&MXPc?bQS@B7jMYOcUEqdfG8SFY2KW3+)cu6yiDDzp2Dk&sxWSKYFNVVVkpDn4? zK|I%Q{cL3NGhR_Ks!l@-7tEcm@A#PMeVkc_bbw_dxXJzEjZl%$TBy}88wD-R`~mRN zeYakKNPKWf%TFiYj| zin68gcZ4*RS09dTIzanN1jO4hw~jN1H5>qQIkFU)Lbc=_n)A~8&pK_wKH;e->rgU1 z7X%Tdy?pch1QtW|4u&K*W6tk}et9zlf1q&6%Ue=OuD1|{C?v;Wix!vngX)IVbn)?n zzbr~X?5v_Da!6rD-sS*(yCW|9`lncPYMxQ6`=hFt^AuN?U(L%e$sr>&_uc+_O8u)X zPB|SEl7SaRbr;n7_Jf(~D~frN*-d zO%Zf0-e{!b#c@ulpP(F7F%W=+w6!;PZ)cU@FV=u~9@lvOqSp@Lxp+RRlCe)+q?xU; z*xlqEjxldrf7`)vOSjr807iBE8%j9T*!aDrW#gB{gg?W%>fKVYjHyo`VufbwQ-^&F zX;|rKo1Y==!|Otu9uv~EL=Ei10eRisTUI()+<-+e&pDG|Ele$hg%9p~yvFFte!$`m zZTMWGHV$tCobs zUT@NIK^wkycKS}w2?<#=jSrOV@F!wGqeEwI92Ri~D&I{>7^qBVbL6j+I^ZZzizaLv>tO!nJU#6sOsEM1f zxV2|bZL@VPEVP}_9zbY!N>@GWi4up(qQblGnFFE-7*I-n8fl*I3NbQQ!hT~mi}=nrYQKs2IkAR+PCib~^s{(jq$VGnaRb@|l7^k58P@q|>IO<~gZPM4(; zz2M>N+b5iNYIExl2lDDe9ZN%4pT!ru@PBxJkmpCvDD<(-toF++?31PlfH;? zzrB876T^IIU?`Gw&={z zq5GGIA`&qlDI^};Ng*)8q6-}-6y#~oh^94ZnSu!e6Ic)vw+vbo;GkwPD$acP0$wIt z>v7{OFg^ud6p@Lcst*l8Sy{bSA|KR$v*@sp%DKXfADA4%Cn6-nMbJK3(cPpf3gK>v4r5NN4b$#Hig7!L*BUyY zjY|8_rz24=>m&kltZOmh_7{3rAFiuDd#`Y=bsp&>!~}?qgi2j! zKZCmKI9yMj(x`)8wt_k;?jt-ZvfTYuNbvJW4zFg0r!C@o!tqy;D?eKyI2$zPp1*$$ zoO2%#S&9iW4lker7&j9Nw;=bOSCL6%b{32TRRLMCK0B{kh0bhLWCT-MnNq|-BdoC2 z;l30e`#g!`_D-ZjG^or=UkI_O|M`R~7%e?ndEp(tu_seCrOH&a||8=nEuvrOvk0e|JO#ON9J=M-i<=C#d7!KhYyZ!2eK4nl4_c4=m zF4y&CPfEACOy=jshk_=nkD{J)GA#_IqWlDwik~S8X+T3D4v!})1khW8H^a1LAH8L~ zpjGH6v78m$KL6kLb`&HAaM>@7%&Bkx}f2L z;QoX*?KC4jmPRA477T2-r5*@Uq93nID^sPn zmg>0_%I;*=Zpb_KFIk7%*JClKXN4GEANh-)(5SIO+OInK1iY%`R@&$7Lds1nyr&=Z z7iuhTo<+-CmvK^usw9By8$dl%8K^<;KoCe8^+8qPHfX;^r-$TeAz*!l6)4(Ms`m^V zwP>c%a2qa>=0W~D&kr>rjq?@)G|$GqM@2<(!O}Ve^k%jo7XI*@(9XeG<57nwvx9g1 zWW?upf^=zbLum1vl*t$On33{&KO1BV`SXYW;cbmp+ZkAwb9|v;yD<%nK9@O5rP%Tv z8j2mBSH*9$wr2_yzs?6Ppk-m*v1^ko9%~0amg2e;w9ee=f_O1fdqyxCZTxc<-eOzgrriSZt?S625I*t3}RcI(xCW`4lYG*#F@D7_G=H z6eY?CR4hZSFutV>7^s>B^+27bK=86ocBTWu=uy>uEiv-ABSbvI4k?ep>a|JjP+MUk zZ*jhejT{HpccDSexL?Vu>Sx*MrQ5KyX5U|uIwEuk7~1;{aA(53IMq1(!YWRu?7H5V zpWVVrAbwp84oyu<+wu zf7}|i-Cmq}yKWJFeO#1Z_I`3NbD#I_F3uK0OiB!;552*owRKU0FZF2%n|W{CJ-(oS zPl+m=`aBwwh^@jRPFUUwp+>QY{OO_&uS@ji1}CHWxR>KP@p&9Vi;O4=4K0ky$rEqjXEIQ{vABT~dpa0Nw|iwk1rMSCwZ z7ZlO6b3X}IMb@|GE6SiHU_sjP15@p)a=#0O>#J896#9%3) z-{23u;)?mksxVD74lfsc;mIcC*3_qxY)N_?x~u&AuBprXTMY4(+cxj`V3E1JLRtI{`$#mEs35v4wdyoJq z{K!+BiB8D5=Yj+W4U5GUwqm#5-T!7iET;WPt*HAc!)yZ4Y8KsrsymzTr$np;e%D#A zZ*)2uhk#R+4ul^%o))A5Dq$F2iL^YM{ooQ7w0^kIOrxV-8Sn5N7W^alpX# z`8Q5ok?%jQTI!FK@mvc-dsJQcU+RZu0gtnsoD*0)?ayxkae0MmBMy6eNr~H;BNRDH z{sX>;>C_qlHG-8s0k3tXu~jJ|-eC9E9jtR_RZ-ECrV|g6UK=p$x$>Z#!WHT_`Bg%G z|2!O6*uYybqnZ`R-3Q5^^;K`_y2FVia)Sq*PldfyfRWR0=h7?1ciYJJJU~rKG=Z3g zNir8Y1{M~c%J5D(g47imPskjkpBW1}tsA<=q8JSvkY815Jb?;RrtItA9?Osy^Xcg6kvH@og@b~0jPi|1CvOYvmN9K!g_A-Tyc9sl!h~(;N znR~-p2?0*Ev0F*6hp9C};3)%`Yt%Nk$5`JNmG6QB=~g)rB>XxTIcY^TJ<5k}bX`s$ z^F*U4o={oiduapiwyDDq627UZ7uRm6W%7*?MDp-CNAOXMV0h49iVF zkgeC9I8!th@HK@Yr1YGdgjVL;hAy?DSYY+6BOgt`Lg3ww_x<9Jxs3fQzFgznGZzKz zlIB7MrR*B~qSF;s1~KcFHN2+*G4;S$QSJQ;H;LXOH{qVM1%dT3xP$}t)dsbpgT**h zxPwe1(()X2H-ru_&IukEr$1Y+n=2_N&y^#l^GEG5^2DBEi^3D)Sk~drg>RgKFl`Qz zWhtJ8H`pwh|Mo6wDtQ;eRWqBErM+~==#l=6R)+dTw!a|H zR{V?1QY$P!A{+0Zk#OzwI4e5r$WvFO=eu#S*-^+Ch3dl8A-%k7nS#>tj(@B|UUq49 z+)OnqY%XZ@va#aDgeVjyi~kfabFHfr0U6a2!G#dZg|6`$J$~rcP{a{`wUSLs)TQTY z3BQp*a$e?ZO*fDCMYfr);(wBdy^q4;yde=2Zn#01|9*>a_TfV$Quae?U?DV|200P4agG<(b-a1dX3K zDTwJH!<8xj8Mj|j=Q@gqXJ)6VFyVX+_S&GI_oSJqg+Br(c);f(krq%}flph}h57^6uQDa@t}hM3qbQCHwuf_eRiMpY0X&A~G~Lr&13QsKM0q-^ z*P3b4xd^d5$Hln7RmEwmwk3k~u_luD?I{CJ(b?2`-!J6%h+5Emlvs-NO;J{Esm!{~ z7On~7-(b$0gv5hlqVepAYtZzw-3k}tO{DE7-_)keiPSwy;L)~VMX7C$zzkxPux5B(7yMn$?_ zO5~hfR_m!MzE17$5CM;ooGcJouzKSaAv)Uw#-XG+4hz=lqM+DI2SkKt5EI#R#dgjA zrG#};ba>(jQ70kudi;%0^uWywWPU_68JfO6d0C?dBDc!7*g;sR_inzLKR0s(G%ets z;$J9raxxe&k960GKeAItW)2&7je|of7hIIC2gSz&1FwintPk)<0-~$c^j8J6LtmmrfCpB!~)Y8}T+Z1Df+;5RJjPbv36f z<=3p+G##e9VXKxCu5_jF)R0BVMc1LcqPSLon8z*xw^L983iLW3p5@()Qr-oIenf@y zq6~PqS2zP$DxRsrA=d2iE%7*{$_vFZ6`GLs;D0T#`e;+IgIqYX*AMacLD+on*K z*=9@HLcarJ8f`!&p~m6m^Rm6q7=R)~wA9z{liAwNo5=Aw&-AX6A9$>mdc$E7p!I~o z5r)#^B&x-ixm&vwGmVjgn>)bi5ebf;3L;&YwUtnUC;f)VspA1lMEogUlt@64|F#vy zbrHHZ(A0VD{BD#}Za>NyMD?~RrsPBE}{_2DdmsL-9lkk}k^}buG&3BG(>!}c$Pd&{8_QxpQ1GID@>8 zimT5P*QFNa)O!k*-Be5*7V6VaSe*uVY9dSYPwqIzyB*+K4^67Yr(~}iS?^1`AGr)l z%G&iS?Sp@=xG_G@zMrC6u4=+%@JE2V$FJ3 zIvuYE-e$`OM*}Ji{oizJ zYJ7^U$pzor_+};X+RLPk$t!((34+ql&K%cFuD@Rz*BU-(%jX%kcI6JmjCo|{b&>G% z+xbw3B?V<-CS?MOMNrJ9B6(c$(6pR-YC3y&khX4_P6?u1Q;fh2eVgt3Wovnu6-0ho zeMQscua;wMe;-4wv9ln}(cKn=mxUFSZ?4~d{X|3^UB$*Z`wD5wf?^Q-%5){r8`zB= z*Bq!VJN%lJkz(bHJKf>;Lxj+GJJItuu5AIC0aYUGk2b8f$EW*#8@6c@KYQ=2P3rg5 zCE#-fbhq&e2gxOXer0+uJDxyyk+-2X1;2o` z=0u_<_g=t>1z1}U2e@qgW8qC1d8o)|4TXyI(e5L(OqA-xxNXr;sY=N*o_u@5FYALI zzx%cMFZ)|&S0DqXaih%h^xW!pP+~3U(Bk%@9z@NZHT}N|o7=P?qZ1=#2_NUiPWrtm z<&)A|d+?l@{F%YVr3(`o?@4HA82p4QSY+s83n9Kv)4{-O=|ipPPkIT|wRzZ~`x?%sv; z?kY&g8mV9A49=Xtz6`nQXKmHl753(G_+twC)d#SL`MB&e$E z*mF5yrr)8rcI+3!o@ypL1;Hp~k=QImA870u29n2Ds#cJCz`1Z#1x_o`dO9JPMKsCnimc9$~IvhEx zUFEu}%v(7>duUtXV&tM6*@kqWX>Ebf)p-^PDFF*B%@fuL$k=5zLJQIY0=rv{gf^5x z^fe)gn_FITVTb>k1V8ha<^%a@=EssFU;Qd#^9!M!9FMGd*n{5P2W>5WPXfAC{WV2*?Oa z*14wtJuk*`&!6wozY36T=BA$Ne#dY2wectHF1O8l}m1ZgIMRKo8a3-(+?#Yoom z)z6aXlebDapO=geUAg_JN5^3~SAtc(F1p!xcBZ!+Tbz z>O(;vPH6$}jYYF4naKdrj&YZ@HC0mgo30tA0d>BVrzZOi=>CbVdnhC22T zj+mTmpR{VpvWcis67ze`-=A1~uFAPP!g=%k=zf3q3HLj0C(lD?SfLpD?gGXU{7!L4 zMHUnlYCA1#($Y05m<}g!y2(`mNiara!BgK@K!1aR9l;V|oBW=8ob=Bb=Tm@+OlwjUl=T2fT2Su{igrir~b^^?%h;yyTgR3GV(E>>JAE<$&WZ~$w`WZ$`?vc=+5O9fgcNpO|wP`vlVCJ;r?ajD3I(b>NWB`n$wz!pqPjb zk~(3Ui>e=$@>+9A0OA-&B-e}HkJz)&HA_V1BQ)3(cj?by1`06N6DK{dVSLlb6pQ1* z`9rCDDl8@8Bo?aTl&uANwbiu?L^5>NowEN_56MQUT zWZoky{K^3J6&*X<@mkfCxO(gci(F;0hoy@xe3`%E}W?BSjN!lD)u#Pd&;dv;@Hj!fG z54b#!oE^ajTIGgBeaA8v+eMi;1_0?sfbHH90nCbte+!$K6hsFxe+ePYIDR<;DuMW& z6~=?G)vP(_M+Fyg=OQzX z(~pBhlT#9nhxF3Ysoo0=UNA?T+1(b4Qc&Iq;kwh@KKgo=C+Cs_lRudqGh22dVXOvJ zjf|+SR&n(^@=?M(s$goS-4fm6-;xJ9E!T7PYcr}LdaWSmQ2RWY$PNzNH4AH%qEe(isYL0?{Jj=o%|qSEt7*&$rL_<`I10&*>m zGqTTIt)c(5t#G=)Hz+H(I9L&{neN5g`UBvi`&7PAsQ8L9cKlycd$iq1>*Oqx`#oLx zWz$H`3S&^X4u=FjIviqJu{aJlWPDpOW_Dq z9RD$8coYRf#;0p!%A4*m(@33r)q6Lr{KjnQ8=VocN^55&D9wgyV2d3?3#BW`sko}` z-O?SII|5g_a!%_L`7$@RC;Octs-mJnv#V`h)-Q*o9iRlaKZZmXBNN;S?g2lkXjxfS zR6HdX3CZQVR>whRkWcJ_A3ke`xG{96>l--by5wuFgK$hBV8yTo=)24883 z)F2JYkdW!_Bk=q#%?b;DjlP`1kZqWXWzumWnY{wQJ5TjNSvLfE%a(vb z`ZD5Od?5kpa%ULS`Ru)nSyePc&-dfDrRFUNI*}3e*d)FL7G3V&wC@-OL*pA;R(icp zMdo?y73=f_x@GOAYUkm+I)f8qc~jIh&SP@^rS7x7T`ZbXqxpA#SaP9(kO9y7_UTBt zNEh%bWH;29C%QIphn;3)wWO7NVg@TsM^%Zn-k*e$S#q-{94?)7FBUEQbYJ@#Nu0*6 zYL}b>lud{+TuDRMB(WO!A{`s8jhzPe5nzqRv^~f8;BOYZWNc)=x}L|~Z4NFh5Olhn ztB`{odg*&BAv5zXctxT2aX`CcPPMgc3lLJIg#5~qa(0@>7DUNSHf}E&++4ur1?}>O zFd;jmmM%4>!BMY}u{rX=;KIsjCWCVM#yMUhO*(xC1S;!Y<*z@Ep%hT%wPS%mU&b@g+yy@Ke#kVrc(D$Yksm2_jKGc=HfUxc#m2+q0tz zY095(e)*EZ#@`Tjh-$Kq=l-k{Q#zYk?J!Hqb^LrGrR|oQu#oJ5e-U-S$DOt|YQ7Ou z!)`n0OwJW__ge_BNc6Vx6wAziL=fSV)1l0FPs^u|D$SgA_u&t{b#SnWzkR_*us?Ei z;9?QIfE}_0+s#jFahDe}nEJT2=*@g$N`wU#Ye~{ek?mR2N{F|`YH5~O;W|j*brrHk zyC>zAl!iKA@r>q&Htx4rzrBlOGMnR^hAFUM9crb#5Dn9uTI5Ck9{|NbI={=T5bj4r z;JdaJ39$qI_L&o=pX+R;IR#;!1?o}G!EhNKljg1gd0=Ppw1VLx^8e~cmEgmlddJO= zIgFz`kG9v4D{wY1&&i!z3glR2yBSR5`Mr6g$I=;2xGN%1F!w*@q%XsL&HIg+54d8n zi8`&Rr(cd#w8{@(zhFdpo(WuuWF-0GM2wG#9Ib3Kd9p8Mg1sGd`d_~}X$IUrd2{H2 zHGAAh{9ODBQ6y5G<1$l`IYns>M+J!$?(WjimIiZNv^mM1KnsD^n2hwV%<-SQ!rc;$Hv|M;u2UIbVEsq~lZL+2 z_oB_0MwMH#R%&kR=3^PY-I%t8Wd>8#c9-NV$$B`PEZ@4&Bu_g0rJk%NYI?VP;a-nV z!*CvJHhD3q^V+!v(_tX^3f2t-FZq`@FPWfOxV+IcnE)MMIxTd@ot=)!TZU%|9nlSi znUkh3Zz=Ncc!%d`;mHL*koX_9A_IcvCvINrDrN>Akr^VNzh{dHI!p%;%Pj_sBO&2N z%q<05&eV7YZLuzqy*xz(6h2>Z#`uIa|92<$wI*m=Pxk_-hlL zPq)OBY6NxORn|&NWYpXT_=TlIPe?=#)kCdAFru`$y@wadc!Xe>ar&bZN-uf)CLmUyor+8}sc zZ0Iy%Nq%PX6}vd${fN~wKACO8f)lI9?9>FoO#}h{oI)afI{s`z5PJ4_jqxLGml2?D z=c;UDVQiP#PedNKRN36)5#JaPiH$ZQ5YHNdTrgo8QLui(0LPgt7Mxg5lemu&*lyn7 zsefs^B1+qYP|9C)Oq3+XM$3xqR6n;H45T5pVKvHak9&$SBgT0Kz^w%*0|u1GT3C`l z16AD1LnGrRh7i{m5e=6Z1cm;IBbX&(D);|)UOH=wW?VWxe%}^veS;ed2F*Ojj52i; zmeX&ZZ!&^7f-(Z9({&!tA=aM0`iyu7&GHxT+hLvy1UMsv$H&NyRXOHb6EyDmvoM9rwsCCrFg7| zdCy{D&wv8rZ{Ra;`z>6z)|$F)y0Zq%ITU8`J;c{FL!M#l=r*Di6USr zmc%?$G11ZHS=v^d(1 zzxVH+J!J%6a{v)JQAMB9-w1#{rYBdAWi~=M?;rfu!+`h~*2Ro72(itRPiCEv?6Ydu zS))nka_&Fe_*jl?OTExDI4ZpZ!=@~V^XNmoAE@gvjAPKsiqhoA^+4;}w=UOjfYTm( zDSWCh9f4P|nW-r*DVYC3mrWyT(fvD$O(jThM=H}5S*dd0jv^_@Og2^zPSKiZ$Jb6V z4RE1Peq%wVu?WH9g3&TN`p9&pNJ!HO02T@75v+_v2(|*Ra~ph8g;kg)kc3Nr!5J0W zdHhh{>D0NEIoR~+PG-$u`QkfCdZ5d8gx0}WxQ*t%A6F64cHfaUcWpwgsyiyl39T2_*8k%Zq z_oU3OaKU{$i{-`BbwXrp8j>^^X>79WbX~iA<>6g3wkSJO+is?-xAqK3dSbj>w`N($ zU`AJX3iG-<9g!crenHMp%;KVR#Pdr>G*qI)(>FM3I+C^%6KZ?6bsyiiTAn#t?X`@< zHzZa$i!mWk{`-N;#uCq#kB-!{Cu-%PfN8#Veca<8g+fD7_923E3ufx%5XlCQ&4msuPDr^0AD!Mr*2(utmSJvdJRvWbL%#Ngg=4Y$-W@B1 z_N+7UzGN05;vV7~)s__}`Q-T4h=P20xg6@yo+i#!btQ-@GIk6p`pFz=7*$MQfI0^hP7Ty!|eLl8tjaOi|+Xa4FVbln*L!Nt-Yi8P< zh|=Sp!3t);j*og=f4q98!T6;o=%`I@K_xdAd0c-mAcp|Wbwv=y{mk<81UX*WX5LA( z)vQ3z>&cnM`nqY`Le>^|jZ`9RXGXgB~Y;f&bdk2lH8}{SgVK4w;pZczU zXjn$a#rU|le(rG{!-b(OGLE%-Y>auo7iJ|1B0IB|ZL)*XR=~E)aW8^TZkDgfO_zsu zuawu$HcI~lOSa=4WnhN7Zrh4%sqgGHffeuGR%DJ%dxXIz&KuF4_bh`>LA>JFLz_?g z%;JZ;3k!17SMUUOWi}ixNJ!jI&hz1U=HwBjDIdqPc@4)z3(pq!2Ci;wy#gmY%ZO`9 zz;6=Vwy(^RyS6Uk>Gr0i)BjI}>FKjtG+R=!yQN^(8gAQ0+A=IS@Uc`@-(eaSOp~L7 z^~L+Qn3oSzH+X0nHO8`cF(<5JK7H#t`G*(Jno5xADyi|Y#vc}qCHygKS~voM!C-j# zB7WjWnFi7AoAbls)`L#;+Prk{$gw@+_4(89Aka8MNbl?$G~%dj%-XzYJj_ShT$m*} zsr7QXxyLkGigQxTJwBad(=Z^SyEDCbs!p~P`H%K*D$bPXcRbb{T#vSi=~J(ssW;Q~ zK6%T!pn*1>d57~m77uhX0u#R4B1xDgc+kkmsN`lQi;W^3cuxGL<6g;B!SQh;rVvPI zHp%*!4BVFnF=E6wpFi!l3JaFU?wr41c1@jq6CD`rn0(il1|glaa}AxYf|_5U9X}W} ze&Qux=deNg%{`077MNO8yp^yW$8)NvTCFr{pma9(%L&@+8T&=h1n_ck?Tum zBx(jur>1ONE3Vm0(nt5LmItp{;k8~1)>fnQIy5|Hgc#aKW&m<%MAAp%OlLVaZT3N& zr&C@;_*c#}NK<#ev6$Oknj@=%%+SN~bE?EFKDWUD&W&d!R6_Ki{&3craGPqsPg6fV8~h>UaZc7Ekag^fyAaw4Tq}0smo?@R{3!yQH&L`Rqp75N`;M>NK#xQkP)%Om#7#tov!(;c5cw#lcZe&^zi- zgebc|Vl3_5t{R_ENnUJYK3!%}&{wBrVG~pT5vUNS5DK|>5U=QaAP#={sUxy1HBqi# zmurN2_GEjS_iV?YIlpl&fH2LB1bb~60C7ugJwCS`tj7R=fV58W&m%f|p2F6E?Va@z z`*V+DZCcabYXm-)Q5O(^l4CK`1b1x=kn(J!OY$y67+66wV}Tjw+%wF8aV7-8{viT&dafa2 zajeA1x{0s3d-?=^Nti(7r!B!fBMGCwgYA z@NP*?@SGzB<$5z%0UIocnf1WEdAYI2)NSw{85^FE`q{(d-iCXf`_v}(;kjXOgaIAO z&wYx^4+dR0K7%$`QsbJ3dyaotCf~ia&{*m}f1<|R7d#JkX2YuGDW+a#fcS+|b%sMy zT(rr9dzXQzmTm?BJwYG2e{Ayf-3$DDVY80BG3;vT9x(jP_M_wGetq&lq6vy=^9cOJ zvY&d3d(_;+#x+f}+`qF};$mWizA1Tg-FQBENAM|Qo`95`GxxT+z7v^~56=>9gXLz{ z$&eAKV`F2)yg%vt_6!>9bi{Pp*TlbJduIUJz_LctZ+C?=lFr~NEbG=4*(Q3d8F898 zfBfn(m2n{wFLHz!DpjVl@NiHug)ZK}3W0*<&^G`!0bH2|#hIs>N7l=HMf2eV*5nBUHJjvyRTI zTbfBkd$;|=wTz%VSjG?lgy1sx3najcO-wEO?knf~8s-Q`bXo%wzSR6Weq~&v0b_v? zxb0NG@!M*rc{{hIA&hUyz=X9iE>q^qg@zSQCfrNZ-Jng-M=3ZG}kZ#*B`pq>Ycy=)H9YOyeUp{NB zUJ>+~dj<_3?hiyO1RHx@oCBYfi+2cZ8S6FygP%XRbE7eG;Q64vbE>x8ShL5+#h8F? zh(5Fz5x;_^UmdG4#s=&UBhH8E^&i={&a^!P#Zbh&$L`o5KR;9@&z-0-=Azuc+~?do zdFjc9J0y7%8m(vzvP>b0&_iZaOGjsps z#|MoI5yW3=dfEt_7k|d!b^;#10|WGN-nH%ha-zD!h`zKLxc5m>QCRAZNLo^ynZ458 zKjLRa&UFpEwpiqv_&!k{3uh5bxIOP9@}#b|Zm1Y+Xy~FY_gLGr9q%meD>EoRF6Q{u zLDaQ_!viK}18pCIw6^sO$X`CZQ%aVlm>G(=vf=sLw?0qyuE~|JKYQBLuZXi&?8OS6 zx|P{yxC25I?jQ2MznH1>-B-@a&XqZ`u_(hU?8BCPtow}#tmflnO`Cj3N}Y@RzOTNAJIBLljDKDW`6$nQ4@d5Cw(060F(NyAV^z(d z4|!v?&a=WF*NS=u?MQ`bzssXP=92)UD1UPAR(YqqS-Sd$O(jjxfKX9w!&N>SmZnl; z#4ReySS#}~bEb3kUGhhFZ*nfW;_vTX_KcD{Goli5P9JfuTkPL?nYg9l(U}PzZh7dk zD zml!W92CCja!ytxk$C+vwRwdTgmUG)c;mNtq`8`+%iH3Y&qEnoe8gHij1+J@YpKQ5? zDE+P9`di-~t7wtGc;C*z-DmY1%RRTm^xuU})Q>hO4`pHc=D+X1Y^=C!AS*X(k!Oxn z8^4L%L(XKatypFo$oPQ7x}qRpfX%s|+jj&l+|0B)tZ=Y0pfg8f)h%@o(9oP8-_BUN z(SfC7;!Jk>E`Ca9lI(Q6zNnPA1nRM=d3Znw7)R$8kz_s+)*Q}6CzUkO zUN_$lTxjwN(+GWTsjX#6i*n?K|fL5Dqs4y4%1n@ZBzcF zfuUy(R~y0EGQ-*of1a&ij}SG{>|-cd#PEswtRQjHqa* z_S)m~*)?&Eoy+WZ?sukWhng;Lx8;*IsHC`(&f{# z7PtFP^@)gubr)g@i9uqg(TCli&Oah1B7`%opQc+Cf*iu^Xhn2z8BAMNe z>CnM~Gczzw-2+o+LOi5n?d+p)PhlO4<=>w_w8L0}GLX{MHzMiDvBsJ)n2`qUfT;v> z=kWu-^GrT>qShRrdfpkZ&3Szh2S}6^@j6hjVcid3zvvf~xi~x7^M0JTPY_rKM-fU! zy`nTC5%mH7c9Y^g7Ok|`KI<~D&M^>rUp`YWoxMZmKD}{$o;P4ovQb` zV(1$1OlN06BPM#>2#Vb2SV8vm`(`asw{U-rj*iKX-@G9AY%BK8#@SSqDg4>=&#zxF z!N%YcJMnyD#I-!#y?w^DuXntg7%(7NF`N70*GH>u_U3cn`dl-xK%4;Fj2tQ_7~rsE zroBeg_jV0$@U{ z|MNv#8G*bob8@?c`vPlW1|)5w@9wm?WjO|3Yw|MWObxE3YE0V0@A#OR_EqX|_8D^sRqf1Uph{Qmhe(G2Yskq9dZl#ReCorcSGK59*L>wF5DK;@90`c6s1mu}vAM$1JjQfvw zV|&k_{QOX*iLvy%H?K7YK*l|Tv3T{=6L7VO;I_D#pnXi;;S{CWrmn3|4qt9IgDD)7 z^Kza`X>OX~h!whD8W&mZ2;9oM9kvL#yc2NSVXfII5BDtf4MM0#oc6e!h>P}E{&Q>+ z$KPC(Y0B$&JQ1^uYrnm+x1Wq`GZKzfT(}Yr;N2206FXw=w*E?% z;hT(6d@4Nrl+!~}fc>fqZp7nmtq_BaG!!7_QF6QlKRsl2(% z8?>~c!(AYxA#MGGlLyz*Hr#0lSqnyyA^DqM9G4I7EeR^HxRrqcng904HO7q%*2j5{ zfJ-#;+){ed>;{4mNuN7eD>bcs($w8&Ivq?~r4x7ewn9I##2%N9%cpKxCoi6^Hx^rF z%45983hLHP`L0pE?Y?fG`5BWtjBMxDk#yTX)H3{Ux+SKxW~U_>@q$hubpxW(T%^r~ z8QuVdR?MN3;1om|(6$M2z%B7kTuO;&oTOUFX`iUisdMT5n+K?9_O9c=sw} z$wwXxcrXo|&Nl;|ykD$U6zemC_-OsffpW7S+i;!l-cls@Zl62@!%3YUN~K6T(0#DjhH}wSZ3`k$(fOr(&z5lD9jM*=p8gZbg_O6mQK`o{P1(1&`G7! zYjjdvb4HCDmj?J$#b0P^x5w(w=5^=RLcidUcKJ*{2~vyYrZX`^k~j)(@iyQd*rsJ) zgaj)tIP)U7lTG+*4vZ^d*V#|{6`b?P$e3~E=M*9BHdB2fO#I8+m%M`yloxBWzCnEA zV$tl$0oL|AOPnH|JqGSLj3uF6AY!JMBZxT%dU$Sd%fPcv_PrB=jFSlhA8Z5bXl6+~ z{cffA2)u`1I9Y49Ww6;MW*af1%m#mF5SrOiBt*KK*5wH^^yVrdW?~_FqN?5070jMP zI6_FKEN1ph3<7$l5j1HxGAoY6J7dIQl*4aC7v5V0qa(@+c zx_!N?dp-QfVrA%epY2-dcU^71d{SQ+f4=@>Pu};~z1xb+^m<&M5u}a4k@(5CiQoK@ zesQEyo;QB5J!SR07o*IepvSs1%81ne^P|@pD^czd=Kx8tc6DN0v=Ln4!~28xEUYm$ z;CurU)~0`a&t@aWc?7&TRfb@g7#}UTQeno!x`K4$hQU7D z4BQcbm6~bQ;~0WIRl+Pc?vV#}tuXI!o=xa^ zt|fvvdwP=h_>^mG+vmv-n>r!{)I9#%a41CXOZKs$O-;_rfkmp)QZw} zd2=`Jw?<`P!kV7t93JcPv9V|q?}Pe{vf&2Cb1t+AsSC#?DK1I|hCBcz~TuzJXC+Lkl_nS5u;7D6-)RO%;djXaY)=3-lK zLoguo30nrUG%4O)1a?|(up#vo)8**cI4zp3g%;La|37oh12?{8c;to~LK1TC<0T|Fgj^B=353Z6V`Gdlw%c~w zrtNmC=Xp?ft2MV;vs5Liq^kG*)~Qvs>!?anPk?!E`D<6{)H(a?v(MgZuk~M3CeJ_L z?Gt|@q`Bam{eYrJK_S;`al?WNS#X(x7rBlB5C97h+#$Sv09?*@56U0iKRF!&=itF+ z>9N}TLh9*?$Ddm~$ND_oCk10k1>o+Xcy_pHRi1wn)T^r<-Hb)^l7Z4hcC%hw{InXT69%!|n5}(aILn&*6^_PxJIkevl@ZxWjCf4T! z-a4vndrBF~86LKO{nPnqXE21aPrr<2<>yLHc2L*N>{=ph80Z_a`Q7X0r5v;8xs=cD41fx$WzrGea*Pk4Ed17p z`Q?HFc04~rngRRW{RFiA;!7vpLvrh?8ktGDF!v(B9(QlyH`Ymcah@Ir$}&KUj4~7b zVVPf9C1+{6^JO-BZt*l)bUg_VW!z~+Steqi(=FQ}i?1Jif#eD~z@h=w4Q zA#;*BfuRFA>672tE4Qz%Rj>t68X%FVQa~uY1=&~O|F*CBZgtsz{^zc9LTVp$8GteV z0WR_00p+lrVsv8;a2|72y-$OYxKZK#Cp|MN7LXp2W6-~Y; z=K-?e4ey5Vdth5k21W?O_jEpT8*f`(Bk5{&An&~4VStHlBERr1NrRvO-b74O!2HUs z%ao^J7=Q=UcN`=DY9Y>d3r}*)Ee;5PM?PRcVdgG;b9~TJNCY&Bq-QuHvuEWc_#bV+ z|2<<;ks8nTo$J-~0?6e6#iY-m$n%G`tw^L8WFPqN@7}0|WS`mZp4_L#0{9QcRE|T)o9vg~U)G9f^v&Yg z#VQjjHm9G$=Nt! zGk$hYyFd3r=3uOur!KGL80S&^5AXPzS^2Vcsh6siH4Z;uo>-ea7vE{4Fh3~o-r^O# zMjpj*0mi8MJoj_(6G|^|%x96qrihtGj zW#h)XW{qGp`+1?stL5kA$^QT6xT+&S5u0PY zpM={{kOV^Dq4>f`UNRgI0n9?J?3%0b2?+0k(zHn!`>MI6si3-?*$=Lswu(2E=7L|7sp5%|gSOi}J>qF3qcDp-p&HY+E%M@P)}Y#RX}UYQmF# zQUv4u;>*X?l5h6}RD!XynO{r7qKX26um`ajvjHIRUM>2@&c;L@J=z2`17P^I+olGP zZeLZS8@SsKz%Re?#{K1`XL1a%@^S#nnY4Ym=Pb-ntP26wXyd|i70~Q&J_&70 z7qWck+PU(bU1w9g87v1O3Kj|@@8U`c#f4NW2*RY&!4u3Mb~@900CWLLLkQDGC>8*e z&D|kY$fh$UAaGwPp;VB&8|4aqYvI*b+U~;2^adcjSNxzTJRb!$od@Za_Jj3jYw(4` zt(wQ!ZPU$v2QWfvgNCzsA4OC-ynR(|qKdhDUAph&Ydms0B$H= z=DkD6ItnPB$3&%;6l80-EugQ32qTM;Vgf~)ka@h%$p4N9k`ZeKK$JpX7=|%ik>inE z%%6IAa{@eVcEaGf7KMvY=)d{FAm;nlz7oz=^ z=@Y}5fzp6{JQw3};C!!o#lCd3O@8OD)e8EXr?lJFnXW-y8^2J$Nb|)KDM1K0ilXZQ z%QbDi!qH!2jBI==u@2v0I>>kuPka2 zMV>Xpyu$0&K=Mp7hD9#|nChHiE(QYf(AGLt)OkNHt(!md!mmF*>B(U3^{*Z86hIk3 zPe3UEoarJ-Gp$Kd1ys(;lc)DJ3+cN7R#+PVF@3`kdHl^LEr>~uT9cQ}o6qeN-`T@h zA1>Xvbao<@(T@(cCgf@NoV1{+EfD8K(pVUAAAm%NB4-pBD~#s`X0PG>8IgCM$k(e4 zOz%JMMQYfbXYrM?Ra zp25$Ui+I1<(71cMJ*)faMt`}--S3Y#G|7&m?aFiTVq}i5np-AIYl`K;%?owEdF=IM zO&7e?$s_!!M>bF54a`LVHkaV(fTWCt5#Ep6kF=)+rdQ4>Q==aS2mn@e55~o=xA6kN z>wEsF)(K%8SbNY9q*-+cfVXr6h;3wP004L)IKR7(;Q7f>qMiURiEyVST-2MXee%0At~MfBNk`3dUI*jHB*f zF+B34$_tr$%yo=D$e|c!*;8!1xRz8$>7tTX)s`moOE1r~=bel{~&E6D@(q= z@4U(ttZ&{shJ=ZzCvS5P(wdMOgE@Tf#`)@v{p^7jl~d3iNb>|`2ZL(3vvfJEquiHH zN=;e5@?hF$`4}JJ8hgG+?^%~H#Blt+jq~LnpF5_JDhiOtJ*F9V-sMOnTfV!yNmC@j z)A0!BA94Y0rjwBq*iLRv;$FXT#|n*L!H9rdgf0OOW-YRYj4+DOcl0Z2FhR8E@-FTR z?;-6E^B0-T92X6Y=Nx-}-e*y6Q2xv9%O~+5yYE}^^Z!%8G(v`jOkRzbb+y~O8hyG} zZDeAIQuvjgxWktE(=I(an3cnw0P6ZrjF5NNzZKz=U$%i+qN4Ht4 zm&U!p{O4ZlzsXJZXAf=Gyrt;NekI<2e~QH5E#VTzuZQXrKrX+Bk33GgR(Ru^ZN^jT7}dj6f!#6)~7{_C`$DU^$Sy;!QU409!3u4)a=rN85t+p z|53)=c}Tm<;(R(2AHq=Q(mU~<>J4e1QB+VMQR*2#41E9b+G+I&L}3SH%{QLxA-3xEUp>+#{OiDwu90r@8n zHTP=2tc{nCcF5d#K7LcYXa~6n#f)`xpt)PZW0Ax?^hO!}hX*S_IM?B|Z{Bj~=&X5i zzkYpZqpX=*CZy*eceT47%>$hRR~W;-_T$siaHd-ZN5|!2Y*axXGrqZihrc__6Z%nF zkSEJ#n1kBdKw16Kffgyw&rz9+^~G~gz>T%;2Y2ZQ4kASC03hkP!7SRKMjU=%1`D71WH-EARE|HXe zVvXhnJP^r1Y0fb{t+O|*d1w8_@PJ8(q~Y1zdC#?^+aThFae@DkH#S{FlfN(gZDYy$ zVP3)W0fPX|85@jR7O4e1W_vN`0<_@q2Ontf533P`xkI{zN1W6nL{crRDwhBCl*hl^ ze$+T=Be^eY0J&zSgo6a2G(>+4+}*UOJdvvBWT$7~hW|3=@Ds)x2P%xNj48Y@6dBiC z;D!&t`|w)jnHh_ihYlvXmd;E5yA4f=^l+w|k$Ud2*Uu_I&oSmGAUz-(`R|#d+W)wR zG~dyo4eU+aFrJ!?>x^tKfXDOv=IP2Qk$4_qo;ATw;}_!X=|+K(H<<`f(hJdM9LKfW zO!~w)Pk!XrPj{+=0MCv0F?s;0rNC?0vlur3V|by>9j`IZdzS5rexrME4&W}I%p*Wx zj?@2pH!aZp3;l|m$mA2pxWs-yK-xNgL?L}aZe~yB zT>#kHIs2Qs<+=F1;4gTum*-{cJ#elv0-*Om^{j-^7(*X?1w$5c({u$=NsCA!Y>Wst zPK;fUbXZxeye!@G0D^HoVQ)5InmNjR!fTl^+k38C*3K`}cfi;isBZn`^F&);g7>m6 zbezV<6@+K(*pC=b8w;*M=b#|xt0&M)CN$cXdBwZJ`47Dw0w`C*a*T*2QdoEeCMj$< z7K1M*L0_&#r=<8PME^;{$+J<{qa-Uw`vyM(9A+eNs=*+G$Nk^^@UZ50C#_352M1&< zQiGtQve)uX7@OaI`)VD>_usr!@5}$JKs>w9d`=n= zSa-}jfj0yp=waJLy}g*O%DW=MT5aOUF7?h%U$r z%Iz!X$o-q9cF~{e7EkRxuhq`OqZ3k6n4@9bZ;w}-pX$1|@^4FLhy7OeG z3My{cBFY8hUs;?xWwr85eYlk42({^mKL4Z$1vW1`v$5rJ`@y66K?~aA7{58r6=`pQ zmqBImwe=cLd*ciTXPQRDyxQDmD14L-n5IqJu-)@#V$eYdQ19CoLJTKVFvtW3xo!Wv ze8)?de+a~`aOgfCUP&`eDKWpINYf2HuelSGt&juoY7u?qpDBk|(k|ZCfOrVj2#J>F z*fqRe4UjfdA1S{1#GZ2+Qph=Ysv*Pz98O2%`m-n%&c(p+>vyb36nf$qzynGTDI-V$ zgyLeJ$zSRB#R11}Nb#X7B^0PHzI;NBb=DT%(tsI806_fv{PV|B^3>wt1#crS=;hFH zibeSduOket{zw)S<#a>}E%s(3qB6eq`hTMO!D}$<^ZVx z(*gJ}u;cwq^Z^8*OJoTkIr0v3{`S>#(kh^M=J3zImhn6?k2e%F<})DIBU|gFyeL1V zNSuMH+?&1_0F6=);nju0igN8I-WedfpWJ;I!<7LQ>|vIh6M%v5oJ;==n5HfIe9Ma2 z^8BH8HK4Ih$zKg%0{}!w{0mua3S52k?zNKuHMc+C+jCxxqbR!kGx-KdA*#6>qw3{I zdI9=_w>|)qMSiSyoE`pi*I6}0@XWE1m{(9LCK|!PJ<|b(9RJ7XkEIwx@O=E;cX$f3 z+t*CrFT{&oF)r+JBYuH- zEdUSSd*fpN=`;R2ow1v-!KjK7Z-g#l-iLO~{bVlcnglHB3wdjDah0IMp_>@-y!>O{ zqZ2Tnn2$tLHeU?Mn`e?ybO5n)i*jZC{0ar!NpFMq;_a)v;$8l|7hvOe?pmb=yTS0d z7RqA}^@GLq1zG&PH!M~?0^?3`L5}8h-Fx<;79z9$@@YVqtyAU@{WdahS9iSl4V~8V zQS&!I?bmjmmQOu=on1QRCw!NJW2Bz}^zz1zRhN7(<~_*0S>r}LGpXAE1*szeXhS*& zw+0B{9#IbfR79pR4rWA}S5c_>i@){yS$*&5HjFn0q5C&2lno1}H-2CYg6KD_ka2?y zj00pI4B?a$RsBeRjQ{V6Jxx+It3dwkm6K`=^YZL^>0#in08`3ec@J@KM|ciX7CuBPi7B}4W#QM^#JdV`L%grl~AP4h(1{=rs6R`zfrB101xx$O}A^< z3qJSUF0Lxy>}u>%!}N#Wwo==MNBql6C*)9ER^k0K4%vaMw5c@gc`o#a<&_g1 z(ls!uhJ#geOVki*a{v$=u$Fm*69LW#%vGKh{X^mn<^TX707*naRD@4A*E*?@%;1Hg zYdT;W2P*E5hcX+Rnu9cHx~Oc#yk>8J?=fFkL(B!<9dmUoj=7M-pHyG)T=s`d`T;31 zEPAHAC{JW~Oy3jl0&jWr(<5!Y8qq?!5^85q5D0FUEltNuSN$#QW!YbUp^Nlz(t8$NqWy2|3>0pE4)7C*JS*+1Z+EgLj5ZMw{$c z$dGtCvuA>X7_k?A@x>FmE;+{d!DDteUX+3GK_Qi+Yy1J&hgg>sai)e2e2{S>(uKLj zd$;*%_hAmJu}cDyjdyt#-qT)M8I2=RKs}!7D9)d_rnn$i`+xU(Yw4~3dF2{PhB4^t z>ywXt>|=7|$Pt+{XO8^+-~YYT)lDwEaPHhWdGygorM=y=R(|0NUy#k4Hz!)%w{M?( zUtbQ!?ey zLm;Gh7tDxUc^ei^wu)Q-epWxR_`mPQ#i|^`^iiP5b!7SaE@doT4-9)=ln}<$72seb zCKzxY3|+w`cA7X*K^=u1kBqBDm%WKjfw{y*6ZGYRRTX@5c{WL!ZN5&=@2IvnA zmkr7V;F;gN7v`=@##GOA4qbC#+VzHT`!GL4ssi_%YxSMyC$$V?dvahB`b&BO?!$KoLrB{2>6zYOn)4ppbw)j&?KILv5&B9L z##D6f;_xJ%!1?=|dQyPte|_nsoM=y`x&ics0H?|A=r7EL?BO@)*uIr6eM_c3Vr)gR zatV;@&NXvW#uGm2dNbPjCpwZIj&^LOpC}UP$e%Zsb5OX`5e2dQbgs+ueoQwXcwP&? zwCnKNCbzr`vf6L%=f;HkTXLJ@IhsG3mbR%K>=h)xx z+_*sYpFfZY$>(?=kfrhn01|SEcPyYlD}7=A!0Ch(hRjm~3OSF|fq(eU^=jC}yA^Lm zm&#}5$*z*JfFPC%5t);|AgPtSNxOwLajy%ZX61 zyI_Fk*}R;z2B3R-Fu)woYU>hE5FMNS9346n-M?{xeEs!C0TV$FAqP5j2FAw6^%>lE zhKD{TNs1_%p^0`s^K{aC-S3@m_dR=YC?hZpK{u@s~3tH(REeddu7?UEiUGBM%J)e%x{ zdGaXzG2NQ;&6vY?09-(4ba9M@L|yoaeY~nTUxr7AG+G5f)Z%5haEe1_;CWm6LR#Dwj*)z)-n(V4)8C%Zn4ZhKp&#ZD#TvLcJgPEr z5+%|uDL{;Dz;keo@}IaYX+|#2gx>&6H(eZ-9mm_XKs%0jw9PZJzUbSiL{$f}bhA7! z?*@Y}>m1MOio#r#Wx%b7(1Lo7cY5;_gJ+QTy#CQBybD9XpT2989BA&5um8AFsz(Rz|97maom7c+s_U=CZzw96Kmsh}xf^)st_zkex6+}@~u%^RQTq0cbY7!m_2#XZD!0Ur>%$t^1r+vTfeaWGe zj$r;jON4b_q}SV!J$X^P$FSo0z@}#e6DKNw4ay~~tdMhg;%5|^edjK!(xHme(%CAw zCr-G;F9=u^Bzk)P`9#iCQb(ZB|1{{8;~nz7Jxv<2#{HMqmdb~=Ez{gum(!njUAIWS z`TFTZt})gJ0^;yUuQmr@a=4{O{`}$V(t6a)gY$2AC!^^^YD{ubw1$v5|hsO?{;E=Jmz|-(ijY@=Z&p z+`H`_T^Wjgp_tFhO?sP{l5W@8CraVv5^@$i(IsRDiqN$=o0C z)=%^bP-E{oykR05c8k1*Vv>#s4V?~{PIui@#|RtUd6G3hqBS%C4wEn<#3p78tdo9=S zk5mW$`211n3r|R2n6w$5Lg+UZ;6?SrgDuMIQOZ#8NgIa86uGcnvV5HGG5c6-WPhG@zzpue6h!7>oD|5&&rTV$zM(Pgx3v$DT~%5j|Mi~LlVniN`To9UdErQ_ z&Nt*ELbhoaVy(`Umd`0s4?mQ3`e(A%+6A6N4#oviySN0ff#R&|Dh6=n$bj9uAB02s~C9e*<;tI^#rPJ=ZzF@8Rk8VebAS2B zb!o@d&H>m(38xr30KnX`0)1xJ3*64ll>6C!&-94{4rvA7INhZ=VBu$gU5ppw%iARN zdMG@m00h8-5&8-I=E;Wh39mYSV_pIT0dBs0ydwcZ@f)d9nETU3RDO5&xx{#3>tHAW3=A_}(`iA#1Mh$8W5c~|0I*XPc#hto@vPduzU^sUPFUxd< z7i({P|Myhjn5LmXPD=LN1M?Vv_}^3xTNp*wEvS&@?jZ%l7E~3H~oyo4W+q3 z1$;-GU?n`*A@l&|p{pMu(<0}Aow!lztp6sbU@H08apD~tk0~z#FX5Tl_W(@cy&qn` zQvUrF&-jA@24EV@mVFrSYJha^jgc1K%X(WguT(A$jOy~Z9?rY($q$S+@bk-An9eO5CJ>0FC9$h{d0#_DiOpUT^R`^;fA zQ0cxI=lk>>V>9w7G!k*-&IwQc;G9?>rqA&42RAC;BigX8s#w19dZW%Og<`Sa(?*|TR;yrKD#DH9++9&p$68`p}0i8M_8h1`hzsK;CTr#ZNq@h6VLf-r6|Ojr&}kv-?s?bhI|}-jcAQ9rJ93|gw=PPgO1M_{wRRbk&;Iam z3V{FI;a2&Bcdk>f2s;lCEWB(0&k$S?{6F*bA+4&NE;S0*p^OnZLFlYi4!8DxTA$~g z)Cz7NNRec@E)gWZwBw|wD8x^~$c*A~^YYA5GYHjwUgh57T71tWVgxb(hHfAH#dYjA zz;q*1mhEqVaNa*&EnnDvLcuG9TUTkZbNK{FbAdj4|+-r++J^t?=}~aHBYJmq@_iLs?mVN44@ww(8{Ue?*>AmV zm25xKCIG-L0n!3G{LM3mQWQ*bHdYj-Dq41(h1uHQSNe$&jHm%4WIy;fN_nOfL|f{r zCdoxx7iUiI1h7f5oau<%spOr7KSLC>{=hBE<$X6TRz(?)Lf4r9#{>-00R9-qp4r!; z3O@=TN+60E#*wO$d{qKzlcKfvZ(bl@eI=Q8hB=cP^zMlwZ%RGqPi07|hUp+tbtW0r+IBwyJb3RY%ZwX&ct3FK(u~Flu0_2Dyg+d_!mrB(O5D8V zyi?{nWMBf{8!rqhc*J*r8I)n?zCkoGUVbW)x^kn|)c9pJvEF8TB? z)+a7u=Arulih2N!2RXTO5*cn`MX``#$-G2u&A_9X>qZG6$1%p*Z+sqT?p2)-5YhA93QsWu?A`qbe*5kNGYGICCmnJHXF(VH!ATl1JknvsKHTv2s$2})I zej~bL*XblCLewezk}>B~0Xj{~;k`#p8?+N1CEZP~dO_0;bBr}c$_mmW0N|pN;!O*X z>Nkj3G#HUj

%Z9#WW-Ela9Xu>o@61E^APHsO&P6D>^0d+a_qsz7aRS)ovW#lK&0 z51wl{IURI<=^TxF81oa)%szU{@*2%c4?p8sSQoSMvUObYa;a01Y>4M4QlGJ8e2{_I zo6#k2TUD#I9N;;;AH1-O@`JLpwnRZ$`tI(j7%o@OE!BOE^CAAv`0`4}CeXX05{|{B zw4^|Cf?gVugXeqWM)`n(v+zU=pwwpY*E;goMc^LDugn|PJo%UTdwe`~4`Zyne_q-S zJfr877RbA|EYy2q5Yl!=z33hyhX^=geBf^wRRE589@6LGD8iZryk`v||3+i%h5Fb! z)`wbpbuR~$hIogt5ynF20U)xr=oQ-}^(juI6eNcK!7uQ9W*z808wapy%XowFM0=wc zI03JJ=N+q4sD zHPX`3k^)SVl4I-Et@7%tugZfDKB&Mo*OBsr6d%yHzx{1_{PD;2`zJo}2|e#pUH}&2 zbhISH@pR{6)z(D$eIYI8!^CEBCkabOQejLUmqgAb_ZZ(1)-wUqN&d6*malkNufi=1H}TyL^xf# zkiko7^Jm`GLQa)&QcSo+ z-aBKMb{tPdQs7*vhb6)pO7%~Z{9eA+mzE5^l2q&cfL6BhF zGZ7vMR7R$nGJ?vwdFArUH!smm6(s@!+_sGoCEEnTlkJ1o9GGS!MFHdf2+Vk~5;pu# zFC0@(R^BazutK8~2`>bMigaX~y89afBw!QZ8dah3GW2_ux#!R1yH~Iz)vNo7-A!5? z)D`Sz($?ji1Nh;eZ>%lCQ{4iEymO4Bu3*eu!T1Mg1vu#vprwI&9EU%VyY^CvHc-YZ zit=Y{@SEx$rr7w;Nho3Q+?VifKOJuym=c)wpZ}Bio4tgBPNX1kX+=%uXd*k0s1+)- zKX<4l0S+enn+&=R5m8yII zR9P6O|2h;W2vE>~LF6db1KwY(7gDT2?0*l}*Xcgxf_eN#Bto697$L{W9h^yq(?&1h ze<)9Qc>+SGBaHKzfNApf5>g9L3dh9wK26iCp? zPF395cX@|^uIzJ+0pK+I5z6CC2tfP?`-la7cUbscp$2T+Z0ao!y={mzqK>KMSz`X!YqMo6Mu z&jRAcC_*?Zqh6Bk-oJF?5@8+eJ=-nOSU?T>@O6L>ywY{z>xEu1E##`Sr1)X#T(AVLaYy0R!Y1yEwXxrF?P3`9XPm+*9kFC!na7qMpptiZ1^(!mcO z*t|ef1t8xu2Piy;$MwIza$1V>gYx3>RL3LEIo3L$^$pNTF;d~_OWTminf9Hz#@YiI zCQXk^=pPVsz$NAzMBSovL?i_B39!h2KRrKQ#E)}=lzod2KVyWvPJdW$HXqsZIES`K zU&FY##!Ky9ybY5<;iVfF&6NuSA&r{W!Haj6bKrY;YV&OD-B#lYL*wUPI4+Hy{o0S! zbIUbVrme+~`1DBfWFsph-!e+f8TLUhbq`*`N#4brgm)I_3+<;up0D}O5ygPIn(p8L3q^?7X5<;y# zVU=x|ugH;>He*T6EG=B9C&mBASg^(zZ~9= z#WKsqyxoJgRzi3jkF%DVcNk- z8LxES>F?LB(YjxjKOE4Ov88w~23XeUNRBgC;%gPYb>F7>y7%x7&Fh$FHUlQ}*&^lC z2onRej;ahqKUjyP0wbLp@6px>&x!MsOUw_9MV!a}LOx|uR`E={OLrdVIAw|QFvgE} ziL(VMt+vqY-JpSROxHeh({yd?A9I#edJxy~s%T?q z)YCgK6lU>X9H?e|$(Zon-A&pbYlku6A99BOdF&jX2fhs^bN{A!>cqi4Iqs4xX97a= z52G3~iMzhsYu?J=Q)kOndJ+KWOQFB|tG^QdL4eN?;5L9XIj8^lkN;SKYSMnJU%x(W zu4=s~&Hqq1P>d*sf%OhR6i|dP*O?H^fc#eonUDz)ELbKS;WcaO+`dfzVHr&a%mGxrc#MkgBSQEPdE_XP0Rfy{ zD?)%rz%lUQTHS|sZNu~5!=BTB{8Ii#DIraP{~i>6V7%QFEBixMtOyQuVB{ z>-eP54QtFLlne8|<~YI!dFg6O@-NF6!h2}%56PkC9u>4L9GkWgC%cA5Q;LJSeYu+7 z5xUZm@ooE!wTnRRC-x^i`CMQc%f1Ura}9=96fis;G7-YtvDRK0iA2@w6hke>HzPdE z2!Xy5!cz#(IG6BpusQ}DcMHR~=b*lF~ zL%#2)_MBIv6MbNhv;CZY;yrK1Jbj04ed=9E*dyoqWeWJ%bf7f%p}qv`mas?0er`pf z77t4|4vY~VVuzZ0B|JVZHM80-1RHbn~0SZEv5( z(VYTk$iWyWx!2p*%+Yz2uFrs>0MUMG?ei2ic+*#2ZA>U!tSb~Z<_~KNg&Uq^AUH}P zWW3b%*5o*2Nhq`tO6}5`VwKeaEK%0+cBlV%F~S4s1LFeVf`=A4vT=HV7Xg;j7mOhA zN75bGTqSos#l`3w`>M_Nd1XZ!9*jHz--J()uit|ZJP-lE3L~hW*b`W1JRkFWh*0Qg zOm1FYD|fD~opBwC&i;@Vdb0M_Lo47FX(H7|I=zMYI1q}c_ui8cE80{#VKh=7@v)cDz#9bo#X$c*1+qhI;Ag!v&W$3zBc)ge4>CT z=6FJX@WiFhtUK1J=~AaV)90Nh|FxgspF{(}>i{W8JL4xnQe=6{Y0B@&9sr}vlbI0X z&A+d`)+oCgFC@SgKtcLmRg|ZCAn9os%N00q9B!4|s25&tfCRu3fRsNC7kyYcH)*K+ z`p(k|wpn^Y0C02^^v|~buzK>A#p`UO^9m=86W|^5M)$e=Y~^j+kG89koAw~a{8(GR zAUFPB4{XqLkncZm+foG;KlA;An%+PS__zQ7AOJ~3K~#lvc~8hxq@bW({V%>Kh#thl zpS>Tjm1rQ>dza^DEyHh-_wWp)PKZm$sqpe3R@OKXeaapK-JyjMds9+I5IqSfi{Xg= zaSt17YlBZfC87iuRut;}(fcs?lkb(W@SHP(dS0AG>E-MlOJ^6WC#SXNw(mKjc{*iT zVUE0cdMI&!7m(w5|J!|Qb^U`SSaiZ?e|RL3j*91?Pw);)@#*T3^pW+7S1fI_SCN;Q z{T#8!dmpJlAoG)RKmEM})_v84D2I>1uzAiMC7xdTsf9X;sp z|9xA%K2K^9qRY?|e&yyR^2L`=>e%q?yg&Y#;h6Vlec`=8q`P4alkVe(`&;DM1Fbry zq>aNd14m1uA!v`jO~gcvLCjm`33DkLCCXu3@5B8VUt4e7yRtA(=9CrcT7s7tfi)a# z?oFJ_aqh#p7%X`{G6OEwquOoAF>TtBiL3>c0g`iHd}u2`pNs6HblANr`JzkTDN41{WV+rx z&e{uy@r1PH?EP9C$P)J*xYx||dn6i@#-50TA~7k<3CQva3gL(_JLbo8M|GX+lCIRu z3flbHzHu2G4oUw=G{qAC+<~+5+L3^?55aW33binp-k}i*Mw39$OS=!s`q@baIMY7= ztekhcC!+J48Y{DM1F~**p3I#!_50;rzR@}&r@JE(iHcMgX33hWTv=K%J#f=8G$BtO zAC^!gs_%<&7z&R`MIpAufOIlNWK zOXoS!IgMafk{1X4$6|7(ZL;^{?&D3eB)eNe{b31(@D9ML?e&`tav&6zp|Hq>&WJQL zBte0zJ#H`T8x+>EOWXIKk-D;M?U#F;@2~GWEPd5Ey3UsrkIVUf`YtlNC@7mtC-apP z-QKwQ+j({Wafx-Ok7V(mYk%)-9+88c9w5^<64lV*(tJY8ll<-Yk;BrFpA-n~+N=H9 z*_-~kdOKUBp&^-@uPQhy&EpYO8m$j`!GP{NXB$t8^duQ%L+gly;$8!GE}spDPRK-8 zdVbd(nvkbXgk%(nQbdL$G3BWxxydouakxn;CN5sq9%B6f-d$ZfE{(n8D&$ra2Iacy zfb89q+{@Xo{rf4`PYg{+L&M}XU~Ql69hV&nYPh%t-T%D#8!;sW74Pe-S^ zBeMN$st5G-^YY}nIjPFF+s=*UA~%*B@9mYxw_lL$XHwgE}!k-#b1eu5EHzMULFHB-H~ealuvkQY53&6^b*_ zc$@H$JSPSz3g?NwMi;Qb7i1wOb)h<$apL!&mBCks@1H#faC|WWMyTJylwH! zp9y~d?CIn@Kum1y>Q_xBClJu}W{UHH6X)d3qi1was4NJ|rrKO7o56gw=R4CIk>?sy z@Aa+M24u9mCGDDJ|4+BhvLXSUA5DGZGCZ7=Ye&btaOa`Wh-6K8>nIS5N_b>cV&h4G zyC9N&pQNnx%9078pX?Qtto@o?nLXYq+na|}PUl$;H(iv0v8WVeV~j(!71?oA1YW`Z z|ITH}Jmb9igB>;kB8j;wA~)8=WJlAOjKqheBrhQ6T1TbfRI6TqQrSN`A+z$}5lP-3 z9KIm!T}he4S`S2{GCVw-0J-4?1$hBkmDeL}r+Xy*%CM`GpDhF9GCDe@;El4|xOO`{ z5tA=IzhBeaFvA_ofThBDJ{N7ViF`D$a zU0yscjh%#TCuIVLxb@nUc+lM6N$Xn{ICnCEV_NuH04=c?NB;M1& z$b>w7DkL4l6EZXk@FY@{nhQ}~^1*9x@T$YuI&fA9e z5g8nf3An<#>O7e>+$s$Xt%AI9zH=x=PGkL}r+_~J_%deQLm}B*nm3CRw1Nkc=w430+S#dBe4AN1w{eaT^7#=MjUhQxV%NYvQMx(+#e zaC9OnBcl^C5lQmY@?4SX;(#>u#$;Fx9|1>a*W|m}-=j_K@~=;K>sm&BKo+|gk46K! z2t-A%m*}`eMx)X8{F_rhd;zlpBo6+=7U_(r{Q_q8MmFmMkpGk^LQ`5(;_IB|(i0@ZMQCJfinT z#vdCEOW(Msb2Bf0?@K#mc}0%raFIh^8XISDsBxK_hfd*1hoSMP@+bJ>zQ%UheF`ir zORYKiL6K1|wac0Pv8dK-&=t-?(TISk^oOD{81M`f1EEpfGl9mw*D$D;7svGk#(H3w zqQ*o=s6~ZEqW54vvzB=#WDn#kF6Nnva{>~KMdV^nube)5Q28Wlj6G`mnGw%8!08nI zRWD|dzQJMXIe$X;2WD$e_mG6b$vMwEWiC!2n_$@D&BVtTxu?BjRD!Vy8B?!kk6-e> z+k41o9~DrEDB!t9UcqeWG~*hRD2`t24RM_p!vnJjc}0oSwIHVmhG?+T?V)zvN4*og z|Nm(3SqXQy%bMyWZ+3t8e|NJ~_kjo+W>9v+O^9)q~CA2;RdO%Pv*Xwbb^Td{j*2p%!Q;E=gFa< z{I~noO$tRVX&;d7&B@}0h`q(*Q4Mv%qC3B$C}SQc?n^J~HxH}ji?9kjVz&Cyap1xB zRd;#btOP@KfPga{so;`Gse4 zpZ4H+dErp1==}>fFO~YGne!Y@*DjVn@&$GFdz&EVpC8N7@D1~BnyQWJi^KAC<6emc z##C_h1U!+x5s`~(@$`0HSQSi>Fvz_8B&fM~ah=q!PR)5c)%|{!exvH}&ZEp(4=kZJ zacW7sqO|Ldpr%kHx%ksbrR5Q@pgF_A4BRtg@hQxRcZg@tXB7mdl-fS1RK z^GBmzDA&p1eCdki%PlKrXUvm)=E9Kt$8(1z7G<08fSkeHQu+05lk;I>9slZM4U&~x zpaMSk>KYU2%&AG_iJYpxSMuAOr6GC#P^%C=iw6XG+$>b=O4_aI8J1Vt4rvptO*|=> zx8!Q4$OaXe2_d#HD?G|RxNW&wexEp zz0lOA;T#z(oTpx!SJMA<*Q!U#(KmG?cYnuAVQF2hGz}*eR{MS3>g9yzj1cDPF#tu8Yb)Z$BaA!i4}_@kqoo?ZIuEHRRK}(O=Le$NDE5 zH&-sNllqnE`5j(S!MqMJ$R|uL>#=dk$<0@91Z#WFqGdvXsw<(HyzgjPo905s^9I7B z?Zaw0R;}T2UjdysotMtWL$C!b2S{7lL;HX`?jqrbbtGevyz<~m25}GW%RCJ zRH5FeJP~OY$lW)S{WZ~<8GQL}Lz86XMiYFROWH<6X4kDw@#ddFQ#%zN3o$b_`0+>vw)z*~Cm!fI&=MP&Dd7L^l5!eg37g>j3-vZSK0K$^oM z3spW9PqG=3|tStRwV(y#j?!za}{+1lXqjlP2`!LbnqLI7zP z9Y7>xxZY8bp{@z3n_s40v=l_c%Lz|uQh5-%?Gn887bl`Zm^=ANhr?s?H_sfAh{>aw4OODvt5)=7;<7+ZJ z;MIblE%VMBj^;?oycKDSlkVu)BRPE|a$zv+0afuR3%nPh)yo!1{dEiFGfy9su|P~^ z|AIpCo(KFg${D0JN$uIEg;V|A=qn@!Mkew@q@ko@s!W^lu z-;%hDu`hh}v__I}4bcZg#t^v@D6W+H`c!#ucmE0XNW>G4s8`-QxWzY*MbwKP&tSab z@K~HzUL>D?V!!l-m=zwt6$=C;G%iwB?8*16p)FhLHS+RDN0A%iC|s||-5Y8$79za! z7LmVva=*0p3~QhDd9woYwzYF)QDspgFE!8ip2X88zmOaBvDeQ^Ag4e|;{Fzk7R{6T zOkWR6AkLY3drSAO!twxRqInC0*Oo;4>s$1Wp5~oOlSyyL68Vh!uY*Bw&+8v z=ap)N$=1bH>IM7fkMC6vVj@{6e8(#vjIdU+B`Yh5eQ^KW+#FfDc)q-^u1eFvco8bu zLSbo}U;p&Nak)Uj%6zY=>u6|9J(+oLS&>nhh^Iy}o`Y_L{!&<&G}hKu<)~Mwn-d-$ zO^S$-=zzomUOFQ4Fl8Jm(vAmv_eekv_j-C^^H{EIT{2G!FEqu+Ph@ytL=8CEIXP13 zc#~GwtxD|qdtW>uxjEy~7mg^PsqczC70|yoBF+7t>=&BwQlS*(cO)>ok8jhBH#dF!!lj;duABcqXlW_p=qecd6lvGMY@Zhx$|6^QW zt)$x-%PW)ec{Ngq6OoUCCDl@2zf@D^kg|rTF`_PsLi~BAYXVH;<%}ovv17+Hr!=1s z$MKRTCC8ur*`F!+%y}FGoW^5%&z?OBVAs~x%Fxh|7C$cR6cY&BRVDcfQnLX5@23yz0=H{-HFjxkP;#iTaS`I|g$~&nm;cG)J>%qEww4Gr z<~Ng9?D$MRDHyY=wnSQc!^)%xt+eNz$@S^ZAym%|73Ums6?2>rhv{fhRgqRWg>k}6 zc{Ze!(U`Zf5YhwV^EN;a0Cb<-Efiz-B1EU9dssI=3Oa3?BJ|y^xmKU##U|{X5E(!e z@>s#}prdW6;qV*-hBIDHXyz#CYRLTtpCC`GOQaKc?qI9jxO}!M172!^fQD5OGP-287a~eJ zBXZ3W3QRkiJ2c`|_T_y+FCTAD2%zpgGJPlPh)uP~I_uAn3S+V*k7ESgm1;qq^Ed;LlNBM-=qHuyj2w3GJH`Z_H(f{@MIkM3Fg(pt zJP>wgbfm3U!)fhrKKGvMk>9;*m3t^9_pALtY6Rp5OQB)MGkv05q$A^@_8TcXSYPmN zm)Kt+!X}&JmCh-=LRcY62)Pj{RQK-d7OMQhxr~9ISowhO0Q=XH{PV}|)fXO>g&NOh zN3QNCa?_v1i-OVeOk^lRQvdjY4Z?m%3W0N7gF<-xp0nM$?x^UF2dydJwCl=-%ro}m zD6kdJ=r&Xw+I8NYi<{^Xxg?u-BkR6sRB zRXv=%wK0xr8VkrXfc<$niJX>zeT?7l-#giu$lhdWc>dwda|-G*B8&;LAjZhG^U8G` ztq**XTHq_MrYe#5Z(N|pY0@b0-Y#bjV3!mIBa1F?VjA zB7a(cwym12z%=~cI3awoH#DlUB}zZW`u}|Mtkl<4E5KscG7f+tfYY24%nb^%2gi|N z{6s$H2jW1n|2OtlLdOAk-~+S)_!fyoRrfi)^Ngnaq4=0Pp4JX2dmz%$(T=xYa_<39Vhs$2$93QM(ScUkd8$*_G5bg?E_3j1Nsqw$WiFDh7GhihThOn9 zS>y8NnTzt!mPPV|1I?jqn?Pn#?b}a9jpP zJdb{^r%zLn{my!#uLMm)wnhNWcwfl%7-!!e&qvHT7%@o=q}L@1aC@uAptmxvq($KOKJ;IQhf=_9Y{JWZZETeHs_7r*S7`c5Qdarti!YrN za>rY01CO@^;wi*<4)#Qkd-66ok9`rNAHW`KoqY2jzGIc9PdeP(qwkot``U>M5+0pU z@Gdkqq3f4Et8)dpeMchF7aCVS!80M>&?YrBTrz&?Uwg*cD0+y_%wB5G!S%coyA~OW z+6WN*%A#d#oOz)(-QacJI8eueb(gE}_#3aENfcwnK>2}NmZg=kkVmR#iI8H;Pt@D_ z(|2#wlnz$(kUp{>cJzg`2rUL5_yBy7dBGe(rp16K7rpvK_P5Pn3_*aUq~Bm%kQ3Om z(6>GEA~Z;m!}xn^Ues8^xHD(j>mhR9(-wOJV+hdB*l;gb4qz?A^EiiptR6m%Z0sp2 z3vTeI_ia*s1|H`U>tl4-i@ri8=GkL0^5xTDKzW?COZuu$eRse7!F_9W>>s#pp@Q$E zMxj6OCUlK{$gd|c-)s!r=jA)cF$mFbhK%ukd{>jEq1$|gPC)*9{Ve^R zs^MOxp5e*-p$~mX9)0vtX>V_rwQJYPXFvN{^`3Tb0T}(!kA76%_rCWf4wF-Q^XAR^ z_s2f=G2x%(pXN97u1;Lwd;#>uyDO`>QU!FZQB)Nt-wfc4QU6FXw9t<6Y04`C&a1~g zA%f#fr2K-Q=G^P-XG&glI&FPRY_ zYv*VBWTQk_@#mI=nUH9og-79wTUXYmSZ*`5XUEB}csV&1?Rx*GA0)EhvC_?yZeCHN zN(p1HegZL1Xoiz%Zmxjmb>j(`UT#g8JH`H%%udc1?gyiCa|=>w_@Vl-O9J!gJAz+l z--vF`Yvz?+7PyT^^glkI^x~&~2hR5jAQohskpAKPkFbEGm!Lw0E07Y7bwA`Q;!b73RL=`VS+^960xWp+-gT4qFH>y&_e?kE z{pVci-+&tFs3MWy;86@{{em)C?-bQzZdI2Qs4{?qf&mDg zLIdJQh-s2fx_KT8)tEK2o|=Kdf3^qK1tcr(?TRwWzDPcPwm0@x2xXGoqZER}NI@+E z%b(7?|H1y|1Td&%d&U3&AOJ~3K~%j24LA+C`P=J?7wdH<{D3GAibVs!FyAq-0Aw;};CofYNx1@lu1I1` z;~i&@XMcavDeyQvGNk+d{K{ha+K*2u_>Do1xye4lJ0L$Te1?1Ag$Ds3-?^q%jqU&P z(h2Dt0KABYP6q-C;JADohYvpiApG84tK`#99gG7g0reb2j;F6i?B(r!!^t^<(KX96 z?D6b2N8Yj4D}?6PC1?v^jJ**9Fmjhm?DKdQ0xBEz$0G9No+br1>=-#ufB&A;=u_n6 zzTr^?j9udb>-7J;a9rMh(-K)Tzg&LlhQ;y^&mGhCuCgqSG*Pd{fNCo`JuxQ4#Qab@ z`@&LEo1=g`ePE8b^917x1}vf$cn1Ji>=%HMRs#tcAKipy#rp(^<2wKrd=Fh^Anbuz z!Qhxa3-_`yr7avK0KqAg$35T?e!@%oMB9KGhlw_JBMo0T(v~Pv%5}`+&Vf_%myc{o zd8g+u4khmE^}z$o-M!raD*ycz#fndOo{zDuD9M%T()^5tf_V?{RU`Y(+=8#F7pUg( z4eD5No`KSg3kDQqCn8B906bE|@eT~|WE|-a;4(SN%^1zRVNG){d#8ZXtULD^SqAVv zYP4)xRH1Sx`@xucWb35SH40B5Jp#ZX-?=9M<45maoi-AK{=RZoh0OO?_8;wX~$Dq7^x>MeL-6GXRbTH7ZU-zON87uxltP7mW z0H8hH)N_=Ar@@zzH;Epku%y)`!J{@B3#NG9Bj1?u7AH04IrG^Y5$i$nlR3ng08leN zOgwu>oW*f|=RBqnx#Qt^R-h9@uImkstoY1vhPCDa6|YLvZJ+``zwIsv>}P>m#|=Ys(8% z{$?%V81$_-&S*c8OKi_#o$w~`nKt6xa2);QrDGjB_Q(^zbJr^MOn>YRQliD=!eEDD zd*o05&9|i*V>T@+PvAX_4gVMm#x5KQYgU8z-dHCa7M9753bo7T<5C;F8c$P)+SUvSg{cVf;Wr|=$mcmLz@eTq5ou83;*h5AKT z>myUxINbTXCo46>$morpejuRUr zYmG@wK@O^HW7&kEGpU(qENqub3Ujnjhzm;dI|~$`7H?`lQ*p1K>N|p>-vgNHz!k-d zz18eo3`3`r9s~4#pc?v2JcXTRGJZu+!76RUCIi!^`nA!%rXKkO7CUD{w%-WfqmT?t zo?>KxOtzueIeb2Qutm;xAt=T)q|+5b-1E|XH_ssdZaYe}>iiiJ7Zv2>q!z)~@n%DuG3W?308 zqVs`UW7C33{xG8IsRGwR91Ijn<+>+2HU9aP+QtE02AK6>BU#Dg;2BH` z6WSrP%ZeN|oas&xHr;k^Tvn~%E}>)G0w~SQ+9V`_hBgdEv|%XAxf)D6za+(5WDB`9fb$r0VRWom`j>Ji>oGuQ4-B@DQ)=s zb0vTO)Ar*6xM9TJ{=wUm>4P{<1Q{|U3V}<2L+FE%Z0v>7Gq_8u+G`T0W&auurIKe zGEY$8-*w#ry%!)Y_rd^wIgxF9&s~%QO}#oktLK!-&D&Ogjm#}Gs4#g4qR`smRttr(gzT~gTZ+p)r z*Cy{Viqk|iCIQk808UFlWeg%-rXu8K;~AWX=l{RGa!M}&05M<%&(eMnD{g3@wCQGm1PVA=2QY+mQh0qxc`Lxp8z;MT9eUpMNdSd) z%yZx|2RH}lgO23&w;=AI;VCZMhk41E^4?q-8b>3D^^A^zE?`7xya98x4WGf0Kmnxq zNu(#!P<+*Oy)b!r2KPiaLKeAW^;|7j26zqcK)&F;qs!pwXrQSUKn-T41*HkkCuM;V zvKxgnKO6^zaO5z8hY=_Q4@R$Xy{37mZ~=4^cN~;6B+`0##wwemjGYpWSwV{OCH4PB z)y4A2*33o5R@Ba#r2k=zMhD(~wo46y0EI<)L6zOnNzfnJuW{Oe-^TB5kEgYG0Es|$ zzdU^_^%v4{87R%kMY-7uJQ8Ka7_;tlPG@InS{B}s5r!7_1tP-PQ)rpTwo`a!Z#)9R zI^jJU73b$F01W6&N(CGixDQ0s2}Eia)`;R0dc>2Zc#h&=jv%9fui&`CIsr^oih22_ znKKm3eP&;a{Q4cs<@%*w%d2&cU}GX`ku?YLZ0^|EJmJ$DW7d?%<2aU>KXBssD`Oq_V&@p zxTX^$r5L;fxgXxm^D-Y7lox60Cx+pc^^)_pc6p*1Gg^KNC}GbvevI(GF$4ZyvQ{8_5@J~Hi0Fkvgt^FSR-}O zLI3vs*SpQWrN3V@VEU&qDk`kNbc)JTDocO!^+sLnZl6#%Y%}FJ9v@rksssQ#9TCFi zR}9mgm+6=>cmMBLo)AR5e1hJuEHD_0F+h&OK-i?y0l@w?r~CVtZd$C*avG1pI6&(M zB?%{^0zE>eOMc;k@G!!OvADVfh~+B6w8aMfaQ)1xVIy2`Khmn6G}bQQ({*)|6$Sg< z|B0vDZ``p`KJ$ZvGBSp{jpwOhY!b!>v%#9icpq-*(@^)XzIs|yPS73}sd*K}s+coY zc&#DOvYEoL(-DFu^NWoi5$fx&H>x1VW(mOzv*VUY3;@%4Ir{Hm9$Xw}6NT8w5NaTV zQG_PEsf-Z#r^>BKSjghp#k%1mTtbWu`O2s?>=Fv2Db5^65PN)glU7XTxrf3L4M}7z z{o4*tDRQqTL@yn|pntGGHLM9ii!tvW9MRM_b{%WugSV}a=MJ?hpohmYjGq)cE-^lz z{@#9R>KfMPK~U5%va#_?j(Tw0#Y+Z-%LrhThy%i}PzGI4nsXX23~2T8+g8>jKv}!i z{|T>Pjsh+i0eIAu=1($ik!QNeNzD_NnN$JgT2h{%x457_^W&NQsf8%%mrGdwQ54t+ z7gZN&s2!lL33H4qmE-MQ+MIFQaKEDrmKUY+68HB@{Z1u#1EskZ{`}3?&&ppro{w&u zD4u|o@HNH+rHZ+Qmnj6Vdf$ypG-vmUlO=i%JdrBQ(|NnQf;#Kni^K2)MA}AZ!-?4tKJh(Z%H?3V~pMVO?pTn)a%0~!g#?a6DMTiH4 z1aP}jV+{9WZzX>uimj`7TVDV}T~OMNA3Wci$ZO9r_#AV@=D3}Q%z)gud2ywb7v$+Y zA;OIDOc$DN*Jb)7YL0&{!CUc~BZnk1AK(yPZpa^iH2@IHmQ)Fzk{F+iSOetC9iQ+# zr${A)Twrw9x_QFCOCbPnD%zjDFqlwG%|nfLkpGUg^=itLOKDfH?HJIg68Iv3hACZq zH>2U9mLB=V8|q}+>RS2tS5C>EbKMD0Dkm!-{pvvAd3W*yk$^dlH#_&iP-9@EzmG&^ z96H~d0B=0V%L8Kr0s{F5<@S9yrh;3fCD?vAHLo|kV}nnorhPn9jd=FZ zSWL2m08>GI|4VCU$)op7PDf(bWcmbvhL}5{(Frd%avW#@2mpj~$7bc6GM$e^Rhf4% zbHE728_)?*fi?~FV$RYIW8kKRBK6+Czj|8rG0x>010>jkD~bvfTr%Eo_wj$W^bREq ziu}epgh#VBSgUw%v*t;6#W9SLtQ~kcktn}<%My)zai8^;l{IPrr5^@Lvrj+()}MPW zH7XInhR6lm?~OMYsLO950DQbmPj%_qcN&6jcNu2H~WIzDzXI1Dh8}Nj%bnF*Rb0 z(NAfYcaGlpwl$>C8kL==y41M|U4&<2{LqP*W0{E5SVlyX0qVVWqFoIM$Y+G^V+di6 z>9Yf>ipc-jeoQ{~p6hfE=lT_MOJ(<&ivnwdci{C&{$Gr&M7S_t(a~rFos;*)*zw8S zW9W+Va})dIuidtMO2b(rQo%51EMWAbcYAql(T|Wb z;mO~5vq|Mmdv-qQk2eQW#~Wj@n~o6O2hU4*`Njp6+8*g<$TzE#A$hs-&-VVX8uSMcuo4j%yq-8J_ut{O*$RFkAl&CEmHJ zUkr~bK(6{Y#v6(~MkqSkr_aOl?`piD`PCOx70KpBRr=n@>COJ<4b`aT%4dJaXaW%p zYNX>C2&f!@r}l^MSo#05cOUS29oJ#t2fdK!g#ZW=B-ncsNlBJOHHs=$vt?Pfp1K6+D_y+wq@CpEXitCr$osTCHCG6Kms6o@BO{~{m*>od-E>72Y`~j zyHxhz_i*34W$xUWGpC<3F`m|*G16z*9du%L4Sxf(V69je8Ci{1!lUGSe#d{|oot&s zS8VBzg*Nc1KluEkh4Iw4k=?I8dtT_KTH&1h;B6~o-FY9HTn z=Yuk|Zhc~7-wu?#NWdFiLV%Mj&Y=mV3^zEP-_zRozW&6zFY5vQxa~ zlEp<}QUcthJJ9pZOB(m#eH$Rz+GFU!DaYjc!(WuoWSr6%X{1VUTD96TN$QB;vuZ#CXD_p zUL0lr+O$;$-kUBzJr*m&B*92@?4SrwlWK8&E6}YAnr*2o_Lggwl~3HWs&8b3_}%o= ze~3QhJAJ#&)DpU7D)KymqYENsx}Y?JG=o$u5hR}EE{L=_S;I7f@+)t;yxjflmhc1; zG^Is6-+2)2h#66)5SL^)`exqEIbr~OFN2pl$O#k|Ms?qWAs|j_J~0xuX{PRX{ZGF9 zWJF`;^PC;Q4Emm~dL;?$pfS#8$2t;532+n#L=X4|=_3iE5HMf@HpMI>RP%cGNC^GK zH(VABQ|(~<>gr|l`$Dh#F-MA_`Np7TOq(3cFWXXoyRCb~u=>P#yCOv}@1bChWpJ~> z;&+_CDEeTYCXFB8U$avtjb78*6LI(M(?*y0$oDuB0{+t{A1I6t2^04pF8amT@_ftj zWE+m6V!FVfEbWZJwuOww`S2aj#HNW1Y@!RgAZqrnZ+ki-Q(QA|-}mC>{qg#-8_q8` zKeR4Vb0Y

Fi=tR0f_d-I_LeQu$XOx+hYyv_-i6!IzyI5pwGGEt_|0Srr>K4ae{8 zoTyt!_)TE=Mpx0#jLvZlLnNF~NYokoe-Z|jHnAo+2-?yM*(aqt7^MNh4Df<`CL?F1 zBTkwyY0Sv)cxb?Xr_UX!x60Uguooea?Gl_*IP5-7sA-MTlrlS36Rg`g={_f7gmXk> zCxP(1(`J^pU%M>QkXwhK_C9^znnmS}9&17U^R+wnM&zdcrZ9Kqne)p1t4A_+N)d6O6w#$YS2yiE5Qc6$bIW*wclb*e z;J{yhV?{rrM}HaNDdt@eb&581jevO{K~b79f!#L&Y=JR7`I>W_eQM0U?V6Kwux9&% zj%iRBbnRr$Nt*`vlg)6>n0sn@L+{+6=Ly7Pm5=D&ZwwhnMVzeOBASO1GE&ySX2;5L!FAs!?p_^rfTiz#v%r|ffssI_>(Q>LB(oQzY(lq&wK}Qj(j9?5dqKdL&9;5jATw@U$X<++f=O&LPsVwC?T#YD4{(4<+xGrfT(GFf{*W>8#fR4g56iW} z$Q;2CO>n@z!!?2ut^jTLE<6g)6sVF$YXKJO>-sxoe24L(mOeZMGZpK-}?A{Wzo#(!Q05hC!(PYci(+&34ZsDCpVY>e#>KJ_OvO5 zQB+1;3wSC>gFb!L&#Y6Xgsj0p#R$YmXIwsY|LXFh<@3TI*0$q!MteXO@El;E{>WX= z#y4pr)?i$z26*2ps`_7l=$`TmufH_(k?+0!>~OA%RHRnToj$pI=hbHfUbt6aNX}z8 zUnY)^u@?MgTwB8uIwmsoM9_60{_S0(_j}3bmDVF?DQl<1(i7`<#GD;GFgoIrQw{#u zFp%pIEvNek8tFQ-(8kwXyd<&^1UE5|kK5BDRJ|AHnlgTL5d4CEYfRg<=bg@x3llgx z;VAkBtS|Fp!({SM)in?H+ku15d8hjQa74E2dv19=o>yu@KeJEy9>?sp^+hMgGrsSo z%cExUlk0ax2A+Ow*txI#`%m28ryXhmGj{tviM^g-bR7hukIB3(kcr73zw(`zMWE7? z8+HZWlF_|OkP82s??T(M@80M41mkD(Oxc2LG{((%;tv2m;)n2#g4qO1*E4H~Izh?Oo2cIUqGZ$5HwY#1yP*E8@SjND;MsJQI?(~JRDBY6?}h;LE|Jf798(?zGY9DdDG-knS8nFz3*?!bqYoR<{b@#&*r>!>=|2o z8=#B*?15f{v>?0$<7TZTpdpX0-&tOD;i9ttaOEl5KM@F0`Tq^mCmN;y7q0WNVa)5c z)cTEx2yZ50V`BZNor5MraD;^?tp#cH7;iViS+vzyKtbMpfCp#Fm_or7XO1qPfuO*( z|Kqa{hmiTUYtD>_ZRs)(tl3&9el1+Ld_fR~_T2ON-C{&o20;p=-D_RLtLyuZj3{>P z{b^`*E|^xOXYDOPR7af{;}60dfsD{Vc>GpDO2}jDcJ>)e2n1o!twbZ+Hx5$pS|CKh z8ww91fhanvCChL`u}Oo^mmXa|$|$WJsc)h9bjoDpR19)JBvWkr(f&^8AsD4m+BFQ; z#u;rCoVMH2l7we?t=1gRa`qwHCU9t*w;yp7!Jr-V zCj!D1rfcIq85MTJ&8YKKPP%OviZ-i3|1%mV*Pke&3l`5jrZkVpw5_{G3~-qF$M0QL z?p?JlyrIVQoQ1O@3!q)I-*eqrF*gJcAi{U;_Pu?hG{%YXe&8D`Vq-7&z}>M7pAR;X z_W6AxPdL}$3@>>a@0<{MaKlBTBd$G{>p0u^hdg$1H`~^CH6pF?a~7W3kFsqA0|G>5 zsH7sKmS{#xWm_kEG#BctDi@=KigiXnNhSJtMtj0zf>d980c_v9Db~x6-f&)C9(7c+ zyHz5sYq*xi@KHAtptkP|IweYX&p~O|Q~Ifd?Y&(mr6>mJk^m2+#2Yg>r)#;i0jgVCpPR1XN>VnFdEjO)Y2}9>TmfpXPG$|1n73IzwUK& zuxsC;^6E>Mln>puGRB>uKr|{PNwiLDlr^to!SVR)SkJWQ3A&fZZ&{{FMCJ`sCrcm@wJ*VbIF5f0ivyxE=^wfqd)*o_z+>wS#gZPcy%(B(T; zZi=+kv7S5Lc`{gQly3MoCAEh&XAg3{0{P z$ZDc{eE%=M`SLOOn9>ao#iqoEBZ~TE7o0Zcy@S3d@yFUwI?gw3-&2io#-H*?LDhe_ zS<|3vOK2YcVDPY>;aJKZ9NxwR?E&N=rI6H} zRDb!*Ij2PACb>Gn+{>0|4GVa^cJWp9R}<1Hgr>M2vPO^iD=8(a==9ms!nxpd^5yUU zj&sW4BS*`!c{2vmxols3&chK!=1i?u$)LDs=Cp_yB>VlhFG|C%j%bQHnwJJ&-z`H9 z&nGAN5!jG4GvAX(_YMMb7uhNKl=iz0Y|hHo;G%rkKYP^$kp?`dP3IV5Ik-hs!C_+v zx6gF2Rbg=j**dEsTaE99`iYE;)F1m2~?71-U0_GIsLW0ECq6~3>q63;{)g% z`CwXcdr*1+03ZNKL_t(O*ua-;PFEhh#jzqXF>l*{yrZZ+_QR$MiAZ-5zbU`Mz6M zgrSTRI>+_=#WTWi!@xzBfY(dsof5KY(hu|a2cLf=WL2LmN9@$elVbBQ&akJ}?;N-p zw)R9I%ANvfo$rgIvMi$Cm9VM97{^@CWMTaNr*6D3oU0ltT7G{ zao$F2()C0pAc$8in^P_w!!Z3vH$N89BH0Z}K-SF4bB-1t1X7RAqG#yC;Ir^OV;+yv z&%geXFmBJ9Iw=f%daj=C^Qq6=zV}0SJX`L0W^)h+#pK^``RV0~Ge^=D+-uD4TfKEu z3X%R2+PWaZU|lDD*gZB-NWgsUaLRMof@hBh>S%xcsg0r3So~^nKHv0M(t^)2w$A-)u4_<`W^=B;Wr!)ovM zJE87#4~!UXd@rTO1|%(X4O7+uWs0zuz*|heGA`;<@dN{c9|f`xPnC#Xv~lOj5>2Tz zd#^uI-uLyVL&@-M)-6MPrXlv*Kfy0jbgD2`g~+@evsgT9^mpgA_V|Zicx;rT;+99& z#|8=vzvnNSUAF8V5uiMG#=Pla41jl{SqSMA+XPV&3F%?aza3X^9jOhN=O1})XG9`F zJGi)J+xkNBX9!K@3eE9Ds0u-JXsG!RNd3*gBc= z9Hqj1iw=^mHEq)P@WvC8?PT0WLl-_Z}x{eS+{gH`D336CON&RTeC z(AjQ_xh^OI`k#PanCHu%nbQ~C@|r$A_}u8oCfAfo;J{2sC6M7o8oagb*@SLHDIu#{ zOP>)*jWp$a@W5AGv?!E2!nT&eK-MWSrJZz=cy|H=b19R89@>btJI)jk$^(N)w=|5% ziY^RVFS~mYuibmmfkE|@IpH#z&iLcagj{`7BOGMZNVeI|83PH$gmNPtPsFXr6};rt z7mqH{kk5~G6g8oZge7yQm+yP=Ic4h}PKTjV4Td5NCzeZ>&Mr~|6J%Tb-TkY}s?9y# zar&IXICs2tp3ioVjJ%ZM1T^L7P|Yq#0~Y0L<;LBWCzhk(%tRxqf|*YaKJMB-S}w@D zx>+B2{$!4#o(4geC(D4MNa1Jr0HX?_-rC3CtwWKc>N87o|Gn?OKIQ~&aI9W?&ipa! zNDtbS*c$%jH(fTUYopGgTWYHZ{*Xm3gg=WT0{(qp8`(IPLOa9ehtXF~p-G_(9vP~} zk!sbc^#>WpeAlNRSW|9$a#P4NWQSK>ykwNIzU|2W{K`|Y$&z!%$vmfgjK5Uzr_xOK zkZ0h8%_60p{_dw9ismStuE8DII^@Bhc=d&aAuXeu$Th~aI&SMckR>v^P%Ta+U-Tp8 zrnawS`RX(foi%cWLi_%&kBnKD&WXHBUM!1`VTlo9Xk4v5OL_glm)DHKFF(4jZxn{-Cip#_kBrsmo3UWz`-9I(cRm;f2T@lUZH2aV0gpz97}4Czp-)CXZ_&txm5h0< zQCb_agE%g}=ep4)MB4Tl#~580HIRP&AdDPjM0}kCAo&J^ z!PH5j^^TUhTTgu(J_;-Ftpr9D#?vq`Od20!mf6;HA!7o!IC2v38a0K@xiofb04SAW zzV&zfI0nJGIhpt2K?%Om_|r7HM)JI;OA|y!8;hj#jLC=g9U3Lmc-9v()G|0L8RK1O z-hvrZ%DvB`mm|h@*L*8J*#(i`KYqh`Wy8(`~B%=t44!@qaY7cMBLu=2wj<=4w z?~r-)p*s$aa<=$3`~wJ%Kquhh#0ga&W}qVrKlU$4m-#cMMX8k5!LLt=ws1IQG3Tr|b7QWS%{{dj$zL~x^BkNtuIBORIBT_*+Hoyt z13F8;mfbR7Gg`*D7Tkv){Lsre8i#`xDOsVk7{}a=moDk!i02p-*#N8)W19Grx`AM& zgpAas-+uC*)zL3bH_yO}T)Dd2~Z;(2M68 zlZ>nIUD+q}yanI6Z)ijx$$N9HCW7db^-8Yd2<%_)$<6*Yp6QH|=~Go=Z?;)rbt-Wn zW|poiTXyY@Z)CgCZvoNpFWbYTwNtrH#SwTiJUq30?@P`;M%RLu@J?VtY(gtCerP!8 zH<=`X^YcG^;(<{CFYHi1edEQEEk!R7m3-UNo5CpVIWmCp(HFFLXt+A>vrR{G)UST< zuJWJXb!~e(en$^m*t)M*ZlrfIaY&?o%tJrgyJ!2ERQG#g%lvkMZ#%;@Wo`J-{!s!F zM#cNFbN`4ppK-+k7?jd)F`rDTE+t+HV+*%EwXtm4wXe*cKDC@acUlpxEAkhJJiYA9 zxxonUT(KzxEqy@fAs-{)WlQIT5&DLU78gI?3Q%^kZl-n!WPf?oe@IA z2XB{_&Ur!+I~u9HgbqdcTcGTTdLBQe3T=6>8%rOUP=8H{ty7G)z;(*0{2CZcQVUl1D%ym{5K&bM3w3y0gl|&uyzESz;I25razG z&-|UD=3A!B7{MtUwh)lpe&qdQ9j$TTNKsB8)W7815d&VHZ;yoNZuGqC>f|v@+j*QI zZ(}>_ml4A{0E`9dxvI}4_5bm8L)Ek1;D>SR0=yz1V88?^OjvZc8eAfj8O;Dg^T+RA z72fY3dBsRXEuj+(_dXTPOQUpL=xyqyX4mmH?X;1{WBW&YanD|GYPoRn?4stU?^TOZ zZNhHaD5JyQ^mf_yP7zGXk60h+8PC3aD@TT8_@|5syw!f!P_6yjpW0LyrbP|hBI>Ru z=$<}jLYX;za(UN_&yI*CHMc)^p|8H)^GQw5Z(#&bx+ewVJP)5Z0J}igRqp4@ zk2MaRUP`)qIV-zBAnVq>gDWJKsm!B4<&Ai`X#ILNwl@kjGt ztF(xS#vcLU-w&Y^O-N`XNLnmUWruK_f_pyWxOkSx$Kf!DjO@EG918V03y*b@Qo_15 zEg49@>#DP&-aE%4dQGrZr*=>oi_bQO0ewZ!37Us&7<0H9pjyF^(ForC=Ev^q^FRjg zmR6vxAHREbSvG&hNoRU+>a|8`wkN1&@bug4EK19@%`c>>&j2P_QHOB>krx#t6^^yD+fFN`F;6Oj%*u?op~kK^O(c9hbif$jP3?=z+y z>tt{bW5_oPXy+i9G-(6_Dfo=M_!8rOt-m`ij67M3S5`_XmG5vKDz|c?k}H%lk=Yzj zvF38o*?(dG;lgR4Ox0LGhUAb=pd6v^3@P?Z&|m$gl(fn=dP27JT~BP>Ssq-qt-o0w zdVufW_VngabWyN?l2XVefZjv5N(jZwuqJ+ghMs0&_|!jT9v&%zaHsW$%m zFQ0ihjF8r_5*+%CM~VPq98ucGCcEmxGFDRZ&>}yE?9Er55#2T*}BO=|F`Zv5aWcd>AxR1h_2=sN-_P=%g-%eer$ah z!SKLMyDAT7z)E1GCKWhU=~c}*v4}o>|C)%7&wGv03?}OZ2$ge9~o8NE&_uXH+y9ZvJJ*C z{UjRKzEq=i$PMGhMR3(`fBe2tYlR`-n4K)3E%adh-!22B7kFf|W%EexI~lOw5gicC z&{sT}PA4;gY%O4k@j1c#EjV>b1lQcT^6os{Gqw$d?4_o8W}lh2DHBFY8G1nWCmr>Y zUci;05bZKdayaQv&Ko>qxJC2ON%V~#L{1Yd^8GJ4ryohKM4O0Ca!o>_+W<8sZZRiG zbDglZ(Rp;-d1Df$I!0Yl(wTuP}d^jA>WiLjK7cQ9-0e1Aj zggc(z6p`g*Lu-9)uWTGg$ksiF!uSmzZA$5VU3&2T+v+B#Xr&BV^N%NS44_TqM@IRS z)YsqE zD@EiwoCdzjS8;AcG^YVR?= zjTNCiW@iRmXgzs(*@CgT!XQPdO}EVZyGO|X@7{h@Fks9_@%=Oax|he2lD1{{{<3_* z%&0Gfky75Xj#{@(f7?}b5Qus>8N;;qwnk}fG4lS;zkf1>4|srKQLq^fL;y0FB60~I zynSU5Y6Ol!(A+QpPncL0te`qWC76FkT6%Whfl8Rv6->jfao(j6{?VaGheg<4ap5KX z$Ys&AHaB8S+pv9q`R?n_ivA+*VIZn>JB-s()w*fYS1g;`_poRGyUI+SmlmQw*=NGF z-)*?i0){XMtPuto#rYGAr=NfO_AA3v%X@DQsz5?$J{(c%g!upcQxA?ZdVOjB`tpl! zyflV0+iQRT=^53Jf2f{zH?m!5xG2$yg-jB357rDfU3Qti#>3)a>kAV7`< zKm3YwBUSUOPi%}e!Eus~zI1@K&r*V{M@l0|VUBX}SKnAs&R;yMY}&Oy6qzcpP3#Mc z=u2s81#f8#--F(zec2rVE~2Rk?x*w+B14ewNqFWN%^5mcqoT9?TSj}Jxr8&DkS0(t zoy(1X2+Is>`wtI=uy6BF#@j5L5rN=|T4#l)rXI48D74J}Ym7{VjV2B(9WFN*^k*t((n@lG#F1+cg zGeTj-ju-39vkcdM+(X{dj^Kc{4te&~QRFQ73}_FT6@A4S zpkK-WXv?)U&a}@zyn1`osLpffplE1E>D>4^LxBD=N@)L+_pXZejSGb^+hu@J!e;ZX zV&=@xu}6TAwm*XziUVxAYCuuGoN=6PujqJq#PN!hY{F56AeBA+_b3 z=>Z}z-*R>Kje7`l*HC$c1bXJk=#~^4uFJE|S`Mdkplm}OD zjrVhkFi?rO9aQeBME(xQ$3|U4{8|N`1z$M>t^TX>xmvRk_W@N;>fH60I z*=E1bQD?PADvLc8(asoT?XTIsFN}|ziD(9P=nsD{6pqgiFx z6T%VK7_L0)8+Y6PYNV1u%eB&Y-sJN6NGQh8sWd(&h2fZg}mQ z@;waj=&JFe69Ahta!PtOA?h_x8Xp^W?%01QvS9ZQskL0nmY%^f@9jV3J9-WTb4DMv z86+J)@d!ugkjaI;+ls|oowfF3n{Sh!k-pjR%55Q5hg)J%dK(aO5^YfYe z)|AiP|6Huog|nxHF_WxjomO1V(Heb3L##{sw6O!)eaH37!p?v{Dz!&O@trwGz87$w z!41#rzkB=~DEeQ6nPgKP38(19QFPOL1ZQ}!av7YEz7MUr>dblNmp^z{$cLP3zF{<& z)6p{i=<88ZI!dEyGo3a1ZCpggR|ilv-sZFVWYcIsEMq&%@irg6jm-u>MlX$r3<UB_!`B-0_eCM37S9oG1r^H6aZ%KnPuHAvdle&>(yaf+CgtZcdFYBO(vFD7Y>l zD59zX&v$U`<|cyg>n=aNU)NAyL@D3%OpVC$T?{)v@Y0d!=Y0IVUwyjVv2t^~8^eHU z$?%MG3k)lmiSKLyL$sdM1`Q)jln)sf8xJt*@-Py-DX~@6th+w@Ugty%X`9jqR&Uu8 z^{~73-SeH@b(QA{`p*Zf_g{L`W#Oe%f9kXMuPMV*CxsBAoo6@iE}y!0P5IGI9bby0 z8k@!|`<(9;Ep@UWYUtJ{HU-^*TQjCjjL2#ma<*x@+1~fR^qdfU8L7+-gCKmxNLxTZ zFmB*l3NGe{(0|(OY2}Arc5ZlIZ-2VhTc+fs1F0+Qwr%^JA@A4Uc2)Vqn;$Qatl3tE z#tns`mLPfJaw@f7-8TQR|277qL-cazzKAV(8k&IPrhUMs0}ft_YHz9!{eF=@j=GD zi+fRU=!W|Vd4N)9Jun{oZU*%$&zcv}1+XBaKt+4#ADT69hT6Rg!>BAONa>AJr%mkF zNfkZAF`^b^M#OoqvH8^x+!d+)uf1eZIe+o&NWCT8iuS`x&t5pAy!py%7%>)p#)41W zyE>d!jB9}LoU?+!C+#`c7^Th+;k-Vr;e2KsZ$asW zlZAJ`=7Qh>>N%qKYE+A?X@N6~ATFY$jmU5B$=~spcfb0gFl4F8D7wieT&)~4+@u+V zqtbpOVaQe2SOj=AYGs=zBZI*`DfEEi8A?gd5Ruxa&4P{xV@;nZBhiyeoQTj>-T_XC z0AqOb&76$ZnePZ?E|lZ2;6;7l$2U;Iluek>SMKy>kX&URwA11nMaz3^pmg2x`Q@uU zN04=6&8evmI7VV)ydGVkQN>s=PQ+$gLt!L?=hl`eT(T69fh2=zEC-T%IOb*N+`eK< z@N;uz+!zV~V>!M?$t1sY@r6;S3&x94V#}Tp?dP`{USIQQl^k8bIH;gVchLoHxq!ssug6%m;C}GR3r-6MK{_~PHNLfH%vRGMMAVr2%4(S(9V$~NO^I)T zoBACza2)^YgmUnx&BcadzTu<2eBN~B8GR@CPrUk~aLjSkn%is6o_CDlv2}Jv2YUa> z9HTe*Nf2z{r-g;HmPHvJ0WWX`-V2CiEERwv>%mVKICKOqaV#g0nPtPE!PW^KDAyyA z&$kj(FY^ya^7n06mI?PLwpx2@P1>#{0Y70388(MPa)+vpA4<7Cgd+wxHn z#_F&84)Y{}Q{V?#humUr-~yo28nRO+Pl!?iVZ1xaP(4~!MK92sNfTA)n;`w$)=}S(ZQ_fj5qrCs?E5oUf?@?2o5%KVzY5>M3>D95G z8_SzOh3|jK^0H(9!6>!DX}Duw%@`%O7z;RQ{G3M{03Mk1KIx8696w=PM9+H0I2=4u zaS;#NEt{^lM+!XDS3ky=4TO`_@3T&s9ENXm{M^=kF@Ee6Xy@PD@l1a#Ju46cZdYh5 zianmFoYJc=SsdkDVpG`hrI4+az))J}MQ6RIKg;+x}X&6bK5J;9{C1$`Hm4|H99l; zdTq%%hM(SvC)wNK$&EXM{$*4&8OvCsX=-YdE576Wg<)5)URoL2c;*O_w|#@!>si|M z!*@Otb_Wi*oO9y^A7y!PE(@%~?Z}Vl7$+`r@3UZ&G4z$?U-;KA+sUBXlc{ zI~hB0h$Dnynw{aup}NWC)JbqCYK-)U*tY;Yu+_Kn2It@#*`V%UcCg3i>Imd`TYJWN z*7V6i@89>er(=xpiUegx7_49DF!Dt&^e;I<{|TEEZ;C_$MOn?KZx(nb)Fw)ZB}}<# z(wbdE;H1kXAacC(ye@sD>nTN;kj0%~v8f~H2Vuti>~ebTISazOcsz#8!{4T&X_MDb zv|R#&u#y=G-oZ2coA0^mi8!7s_8Znl%L`Q6>!R;153kSfo;B$_!=X=+Ne<9od#vV3ZUUXbgNG0Up3P zv{HIkn;PcY>n=OJZ%~SxODV6qxYohO(9F$K8+QdUmS)Krk%j%G4vBK8^H&`&Q>RpK zDItsD!8Oc73OylQOuYNXy743QBDiv#ECY@$^N!S3`Ctu!Xn4e=vfx2 zKVmfCSo-g`JRTbbY9o}qUTradi$XBJCj;wt?2lB|ychFQW6t=tDav`>(YKyC|I~gB zPUm7Qd!F$cy*$2eEXB|mPSKYC`IRR};f|EVcfaQ}?`8 zT-$d2-i^Y`>&K&0dC#|cRBx_T9<*n-@Zb8_eNl@MJyD7XFK8iYFhQKr)rz<7jO+6? zOcVYIP+xfHxw3nI74+bE*2-+(+y2h^r8EP&VNDT6uRm9c{IP~~&zS?pMffVxs-<(T zi>jr(CNDwD7?Ye!fIoPUjTA-2U$A6m`RUhQ9CiK~!w8GlE}u8bV07*B`JPJiFJtsPnVFrPyp}+Xrll_h0+~a+N zXfM8+eAb#c9&I{f{;8D@_7qWjUU2L&(p-7g+=z_i$QJb_8vo^;jhDkn(Och zWyVB&pUh>0E$gyIL@Ud{@KYm@+I7SL$Y}kJ>&`CMoVB2=-nP5^<=0m9O72wWHe;6e zqX!HDA#bSHAIjLoIG1CgkLZyf^3zYg_QLG;agU;oMZ~S_)%0WtOkf1kzBY3$>JLYq z(gC?v@_t8i$5&zO^cfhp)8|c(@>Bkfw!(-z97fP_kuqQFkXjeJOO4J;!^nYgM8w3k1nr+XEO82z|54?BQ^eI928SoQj;HICyyKi;*#l@F)Umf@V?x!D$O`akO zx95oQEuuK#62~})yY)l{Zgs6}@QSnMm)Bf+dgwCFF$^S}He+h6!8XXm9O^SZQH@bm zH-cy6Wf8petG8c!R(a9c^N!({bfD!6N6MWsV!ZdOPnE|vRJwTX!kOi5*Dj0wY?M+Y zr{asRxp?uw(ofmf_DIJ3S8u+ey#MwU<^E^41@2FuQa51y)2}>He(Srh9>sU_S>M?W z0&xwF;0J5A?kV$5om@8V8VWhkHR}q#UVZkw@-sJHTz>Yww*`(_`KfzZ$CXy8OAQln>uA;1kZQ}NZCoGVmHXk9oD4r>4G_} z`APMPb?aX5_}jZyh3sV`GxLnU((kNy#@jB870A-P&)*yYb9;5u+i$Q=eNtCH$Trmf zPcUbU(#8}p5IT3xX_I3Q4*l`X_zS~)Zh(olGA5A+v;2_ftUGOGd-{^=kt~>H|Iirw zVOU#(bO^F%M#giXr(C0tL5HmI@ip30`6-Snk=Ae|U{5LA_HGY;PakDS1j#SS2)+$20XoeQo8ZCq-nsarXP@|$H(efc z{l!Ptj*>OWxJ)DT&73>W8ACt%cjRc@3{~sorz|01LrA!zAI22k>U+GtSH{M;)0< z7bklEah5&2W=9x}okB0IMWyHPVA1C?g!Ue)4ncB(d#o>DtjJK-<|U^Oc0|)Xt@C|* zj)YT_vrZcoPsf!DmduJwBfP0M4dJ9C+q8yhWkXe>U99=Ko-CM1$p>i#xQMm z)aq3;je<@&Yz;JoJII!F60wC~7RXq@p!P3be`!QkNI96kX?Ga2S|b4?=BMs^E;cao zjzl5?83hQlviJ}qeFCS=9*G>X^XKg=HuaU$!$U>BX0BqbD#~Ws;!}QtTvp#Wa-=@`XVDS^4^P2pB=(T&a3mH5R*yfedbcMTwBYJ z!jB=(J7sDJgOpy10D+s3)WxA}%Vs)l`dXg%jhD}yJ4)ep4fCLEe(&|m%bG0@l})=x zgl|G9rkl6{e;EwZC>?E&Qz|0H>!kuXm3O&(GM2d#FZlb`5%Ct>YB>v?n@E z8wu8^ek%-;sBE-8Qi*!sTm;T{G&@WqGM z_YFk(tj%*?dj7&l-4iuQQI06Wv*(X8G`gR#q;?jXCt5(1REs;=rsMjZ`vw|+2q8AF zz@$TY;2Fm>r%nvVjfnn#Y!K1P`N&~{p{u27)0l)+2ls>l|3AF*no(H~m_BD*YXF2t z(spfhakB8Ycdv>lP|7b_bkpTahz51xOnW(+wUDlxuNDoKq z@Pzno&ns0Rismy9tOC>ClwGX|hz1bt<)lwcvaU3uobNWso`bdPC#b3XX+f$=eh_`OYsZhdk? zj9L2}#~6CZh!|0my+}>>=r7v9h>PFL_JNDJF%$az`=5KL?ASXJAw|*om*07L(93+! zPu_TO`Re0!Et5qU&eX>xbocpbkCCBgBFhuL?2Zk zU1-jE)u1v*Zh?P(I2h3vH8ua!-#u7fcK*UhZ*Q7;-oJj}_7$}@=J*;Jr~zxSHB2i{ zf?nICk<9ZQ=baXG`gkuhrs|P0qDjxkUtV^>!t%W@uH}iG7dZyMqzRKjazo??)@+NK z;}lAQe`YdB%}?OOi+MX@Pl1}j0jC%!TKOaI&m$ugSx0Sn_~U!doI5(D*}bAKDU?de zB$#UeA1ISbhtZ8wdCY5MI?*^F*4J>|R0%D}nv7_W`)Y(#37G@k@Ld-ystQ%s+=gpt zPXZ+muOvq?_G^oCj4|2!&R;Y$czkU?pHenvu;Xpyj?uqSD1Dn=w!Ay-0cb5J#v{+| zh!P|2#rNNW3Ij;SviaplZa6Pe(d}uF-+``q##l>LucYDM z`XJ94M_Llc7#{LH*B^;>^VquWqnu?*_1J%c{LDZGr|=tcHX!40%(h^^8)G~BlqqHL z-jSeK3C3|Gre$(Ngq#u1BthJ&Qa{lWoOJTKX=o`<#P|MEuXw+Rb;wYJMbAl zd}#Id$j0q&h}Bz)c(Qv;i23n167|5y#G)bN!uWa94~ws{Tu}FlyQ^630<%@x9zPPT}GYj%%F)qU5>_P znIF4*b#fp5wO(kDLq;F>7s z2!9!`vu$r0`I=7R~|mirghp z(0J#Vzx|qJ{VW7Bo6VkEo`#l8KUDQevd`n|cLYs>is%CaAV{Z{F*7b?sTOkWkx%iI z7WC8D2Yrf{*VPj&t9N70g)33ks<&aBC{wrvC(sk^?-I}z^`PtGv97X+WH(z1Z>7`6 z+G;!1``xo`c1o^8M{_4*mQIEj=%do5XsU10u5-Razx5MuLywiH;>h@}>z4K9X!!DT zue-SX)O&9$L)9`Jy~uB$D3kHjDM8;EiPLctPPwl=9kjr{KP@PC``>=#rR5Jk|40Na zct2yi@wQ$`4)6yA-JLWs49VGsaq)h)_+~f;4-?Egy}7r{O_NNF)~s)pFzWV_sb4ZQp_0XC%d{bWN^v->;ym`;ic@l-3OG0 zsdIbJs;yD-MBoNK&i3Hjlz?=kiug`4;@mPiw>I!Ug>vKe5tR^5xViVl#MeG9&Id~Z(L^JKc@sqOg}((nIE8KzB4 zDimq~|A};$&AX$9E5wyvz@UY2*&Q7atkoGC3#v!HMn+-&A{W!RY2jgrQdBS~UJ;8t zi_$w@yPTnOWTP?Zun0RLPhov^hk_V~_oSu-*<*teV4>L{5O#3bRH3%%g=L z+n;^)$?`95x-{yOx9w)%;FZnx7zpty!Xlz(b~o+Z-@_r~d^m8CP-5KGo8!Su=A}JR zuY)3AyV3XVWz_sJwE4-^`%sHj-oc`%oMjK$>eD#^JV{<-4Azi+3W_bGZoMUTvT)%y^ zQsFvflMvaBvTfVQ-#K|0Zms#ixxCOE;ek{mjtvoZmoJ+aOf+xdr5DYa>&JP<5b3V3 zyL3sf&e~8}Jm-`?^I5lJZ$3Wi_=Sho72jk{65ts$+GsmB>oez93txPAZQujt0-nz7 zrR=h#*(z-`t=6)MKXCOd7^LHD_D^>2d3=N?=ejykjho$!rm?D(r&R}6ebcXO%l ze$iRwmWS8JdLVo$86#rW#xdrJ5PKqERQmfz*Y|JZAhV$kBk81R$BvmB8AwH}WUXDE zc?v-CsD5Lm1g}!^Cjm|}s#?F9a*4i+w6k_>2tz4^w}WUu0r1~`_}*-;f4pbq*8av{ zPU4dfuGt=u1`NldK5XvOZk}m}^e37UMeuFg(81t8)+r@)5NP}T->7wCU4LM;4gYqB zLT>zhSFVHMUbaUl2IIyRwXxAUbxr^JwiU6VnKnd3Xm`%3lVj7%LGKkciZ5l`VXT@u zDI!Ni3cIdWx6Ka$V>jm%fM-AavUAGkA6i$|Zr>YgmjgF9O@fowe+xF1YmL&bq0z=Q zan(6d=DGP;KChg>ij8~9QyX_gi4n>Re6;asyQcu&@r!S~BxC}8Ot1lSD_`V!y!u-O zQCXr>@F?R+mavJaNO2C~=L`7a|M~exf{(#jfZiw*L)aH}ZjA8sHZy{(CQ2WjOoxry zvd?bb8+>1;%}tlpjhE3QKl+-=kN1$#^gY1_gcL~*7{-JIiv9a2j6Uis5jOWP5ja48#@D)0(r}8yaI2t#DHDP= z=G+>S1kq_~DNC(QJammL;63PuAJ3R?G-uX~S_UoJ7_O0Pl+B#4q&2RS0TLJhI1o8N z80L&^*XYFfocKlIjflUPTYVZ|j_jyPI(}fzQr=~tKrcn1p-De;(?xOoCoemv+;H)t z^4lM~KMXbS#&zQfm*_1VXZAip`z}9YPB=5S?K>FwVyMs|-$Z$5^ls%x_0I9IUw`{m zVT8sH7R;PhK6aNnoi$P!{ZmHZBX>Sq{&`Qvvv#c~^1FUBjBvCZ>9tP|wqY^DgT5x< znY-USbTmeQ<1@>n zFu45fovWh!43sdQzRG&!tYrWcfJAQyqg4-WtZ8eBu8Gb@H1Ei8gz?Lt#)zShQI6a_ zbNa-X6OMZTpA)pn$d%8~4-(+HvU2#My^+j~zNYPy!)$2saCv6qp7Ow|?a?=e5p)3E zo;zbo1gS(hl_`_LU}#<7yCNxN36kBEl+f=@JsM|<8H|9M$BZ_NyNrg@WAm@6qhyur zZyxFQxv3fligDGxHEletJ;w9cv(kDVPi{XLKQazDFhuFo&lrbOukTsLV*Tg5dS@H` z`(v*>zpUQ6Ck$=I4=ugKZ*Z)zqBS%A?Wc*Z)%Z_pIDKXH7`)XOO2xUu)2%{mh{H2ad|IY$9m z__fH`-+24gK|_?K8QxY6(r6W%hjN47t8bi}oI&PQcAemi1RLtMAaZ`utf^)HAu`su zFsf@O!I%ZEP8{E-u@#@|rjL1_Z-;xNG^NCVv1c^K7tp+}p5y!L67&BE{Wd>c2uEhn z-ui^Km(M8ok)XU3oJ^yAp3kE`k>Qulo8C7{=QZ^1+|y?E>1nPL-=AHQT_2f0zoQTM8yXGhI=PWH&MWF0 z=quTEEFdS@U!VmgaLA9b(cu6c%KW9{sC_QPc1J# zcYYWKDa4Rk>LxJByxF{*j5i`h03~{^*@TAA}zkc@NFt}6$bS2{bwKlRgq8^(zy9i3DX@#{xMrmC~ z^vcuCNA!N}Zh&;`9mk$gcRB^E{#Cop9vzO&Trz^Jf2|i~!T8k*LI^o8SGz;EbkDIo zvsk5Gj0TS@Bi^vo}afLbL3?B>DOI!%=zbgg5YfY5K;s? z1v!j)5bUJ6uuU>d*fEiDGjCHRa$` zOMDSr5re6W5f-=Y*%{+Xplfl5A)!8BBI?2hkw1Oo@xL2A_DmOg_Ji{7*IpRPQwVgc zw#LSJ8Nnm?_w^#Vx^26kag=eDkjofHkxY+&-amEnXhV?m1TaeP z2zx3~ch{y}HkGm-ZNSymV=|tYYHos*?VGm~0R>B`3(AOo8!IJ!F!2`der9vI`JoX{ z@|^qEl%IdYrTuxc{(tq&mzM|EY>Q|u%#5-PjKLs-&!F1dBE8hq7PSm)glhMBQJO70 z_}qxm#d*xzyoAEtOOYc$PUti{&A|3t+bIb zM>l;Vv!x8 zKJtmXR|Sox09un2X;E$NYwDKHJn<&F36Nesw{VALlzB z&snR)liGf>r!y+f3ON$)05#LQN^ZBk_HUcv$ySl-NSSfBB~aB zOXEvpzbo@+PK^joiezhiu)cDQX&3(^FTiW~E`z`Zjwqut5WDvuis(bMlJdKA|4@19 zxeLobebxC9LF@gDF0Ii9eef;zFJY_`r9FQpW6%D9PVoG`JTk)B7&zsKJnwkLvN`1) z*Pb0I?&$>Exa(LaGd{`Kd4%GucJ!g0L0?kV&o)I%lBKro-4Qg+d2`4h$v9HejYs%I zH@YuntyVruInX$Bz`$we(%6OX`|;kyE$?MeOQW@lsz2pt;>K$D1 zZ6S{g4F?^|n$+Hty&SxFdaG2)=Q;=Oj7~%I-!-sDNB5$oJ@AK`x;|;xxy}027{|yT2@CYM< ztg%1;>QkdyAVYe`-`@RfKUhidkj#rMyY`16P}JRwX*H^~eK-w>9BTu0v1H+tz0zwF%D1oJD>@OzP* z&~4)Q;n-|+cweQ=Dfj31dOkMJ>ltY9Jn|~w;P)-oMb_gthHt)VX^a~>${>_Zrfjp~ zv6-phZ{s1D?04SzqP%V3(OP(P-HyOl-<v%D#0-&TW+}UwAjeK9ZzykI%u3Xu5(VES`KdC7y2>U!@O+FA{eTN zDWceuZ?Z{nF1J8#6F4GM0v**?W9Y|P<7l)d;B11jYv2DJ=alz+>4~7fmBPw@$?T+h zCtHTGLVpE+w7Q0IHbw~|x=F765}UmX72`Pysw~;f{>|sB?dU+@FP@-8g*88MVtohx z%E?E6$#>UC`nvBx@ZcF!!wzPTo^r;gflV$&bBW2#qDN&Bno(}fQ zG7cFS`87x*uL zC)zhxX^%l;oJ-o6^H+OZg7%IjcXKNF?*H-SCql2&ze7ilhJBLV3NL@l)n`OjWVT0c zZ8JHXzu}_Al|JQ7FVN$+wPAYw&i$iOS=VjfABF&)Ibd->Sm5Y>2+vTM!YTrMixl;U z)|nDQRvO&m!2$nC#eDy5EBf9z#E7Q3CRP`FwjckLNC$Ri5tE z=F4jomIRSwBK$Et2EuK7tHCG;v>JAa%c0u&!n!ew6d38N1lR=qRZHw=-+OysiDoPn zwb3$n_pjO_O>uHC;EdkGoDl8m(G@8*ExeyU@`{mqgZVB;o1ux$myQo8c)X>?==`3r z)!yI!ZoU|$343jX6^Bf&Df7%|VKdMcMC958CnNe?Bd9N)T_gP3d&l}+L=)pBT99;s zB8^eEpmc-iW{OsVoz7cG1rVNb&=Eulj~rM}uiq6$C-#ELb`9Y z#%x&i!!JMY7@FFB&x!tK!0Uo2z2EuNgCQJ?E}_)7IFQS0UxphM^Y+)ibwefDI3mw`YSWI#lpD9Wwl)OuaE;MB-eu}1My-}vy_9kHh2 z0lo*{*CpJDWPQ6r36WC#jp;1gm+kwx7;;Qd_?{ifVX>u3piaQ%+* z8*i_hZFQeH(chxoIO<3&_^+RLTz^*3*A|yf)J|R(nS%#e4e*b`Ct8Lv6kX?-y?#Jl zZU&BY+CQ~nclq$0DK5{mGc~j!m7<9&NJ z&WTbv9AdLhnH2T5+pGj~yX8rSh4=Ic->JWhmBx`FMH;(|7bQ7tf+h1KGd1Z6=Bowk zAi?+hNgxxU33#El@qcnZLw;X3O5lW{}k| z7AP5$<>-Re_4Is(G@FA)0E(KllhwbkvR!^h+h)qcHm?-HY3}vuv9&w<*_Xc9=JdH| z!(A&jhb}|M@@{Lyx;Z4kW|B>4M+~0MTPOH8S^?mE(B}B7KA+$38eV4eVPgf@6-1lf zwQ@_8qx3%f)p|IKBiuI2=*qBlN=-0I-_1&BFi++GM<+sMop~LMNg&%qThhKhfxp>K}81Flu z*<3d5s97A=r*DHdB1E;%`K(Xgv-6PO?G-UM^laH(`a$NCm8LA-fBE#opnnO?Bde{5(cgG`hD;}-Up9y}2w&klXe6%LU zk|8F=;L!pW*xzJLq75w_VOP6q+1xU5LM^+?A;oEv!64=q@2UYu-m-5iqtC(9yxklPeRvZu$Ij-r|{|{0~wn)T+$Q3BAL&5EOl1 z%Y5PbiQw%oIx{v+dbUTf3G|u5?pXpqMFjX>rEL)S`Lj7b_Ky-q#*M224dc&?&s!ML zPNGsW%~0em!$GTDWAqrQbgq^;dEb_1Qwm=ghle&&@ z+vXAQ{_3;mhjN&Ua};B_2AnH&IDYWxm~$ze);@^&qX8<-T(;PA~4iu{_$6iI5`M|HnZx2vIez& zwcW+1hLD{1gg}#j^y;>wa}0U14p#r%bgxg77ACC84W4 z7y%J}H$S*G=po!Q5AZ}L$M|toi6UJy+@crh1hM|nLF@Mx8~N@&7>cOzq`aqL-Mcvi zPK}+{YxF*T;>Q%g$GrE7Gv`EPrc9%5)U@58Bii##|{BvW>Al@3dK=6dT{we2lKi18U@ED%n8OV75k6hv$|Y)=3wRT!o{=0D4NeXib7Pb zAkvh3bisF^Axe|A^?~6G+A&B*YujW1lYNj7V=!FGo}pUP_2~HVVN5!afsgEnAHmCn zYnIP1S1y}d?p?L5oId-MvT4`WFc^du-|JuAONIjsQsiJ`Ichxp-HRB?XB^XL+@8zd z?|NoStapDKcVPZSOT(9s-1$s6899pv(fpJNvTvUEJ4Ks>y>`d`!nN?lhu4J?Yc9x3 z-eug;F?ag1#I5K=Zv($KUcNMzZyzZyWKPi>c**HNo=O8$FQ}~CaM8lp_k&S` z(rE2@hEX41(8d{aPmSN-|MiulW=f;9bM%c*-Tz$FV9tJ}24h}OBV&+! z$vq;%Iqo?L7*ZLs(vb= zhSY}yxWhnc9J-s&yXX8xHNrCQ7bUr3)1LC_`|76h-bI!&k5IE`Ukn+YWAOQc47m>; z9x5v~Rb#yJKRE`B_>7}?QUc#M*@^)bJ|%832*Ih4#V6L_8f#N2f}ectMG=kp`=5I_ z_V`HyX{@Cv=~v}`I*#Z;S!LC@SKpd*n*7~7dA3Jtg6@-3;kdc9_TU~M`v`1se_bE> zB_%R+jiGfSbzg%vx6=-XVwQREKHp0wBToe-sJ53dfcWkNIN8!RctWQM3^h1Co@p(# z1}Ep?XL4yJ#C*54qdmtK0uR)8jm;m&l^y#IM7f>zp8VanJ<$Y!z}3{m5dnXZb?`3+ z+a3FSgQ)hQpl!OHPLD;iMuHgd_~i2fN#F??(mb~M2>n$c*5*Bx7G%Hd1>!q?|MQQQ zAHU)JD2EoC%k}6GeG$(*Y|P<&F9@kel7n`UeaML9zXWTXyk*4DQyVUtdpH1m+bNSL z#fETb*XErEV*Tcr)bTMU3Qbur8;2`jB_;0*qzo2Wtd>%YngSfO$uKdJ3tHNn( z)8H1&qv+oQJ$~68NUe=tHTJ+D>Ws+OJL%r-TNbG3H>B5%8)eoaNs59E{w8;N?jUb zdY#P;+qL-ddsm132!15La9ueh`m*=IKl+v|6s07x%uxHy`TMO;Y$%_)cTN5o^a1U; z{+xwn|6!$hCYO6xZH;*$gPL{Sxwz?yrJ*O~I|dyY`~UFfM@I3VEVq=-PV_Z(Ds5QL zeR*V1qc6~NBND_5^aBQUd+PER#b!(xBn1kBV2TuIz`(d=*TL{u^(CF2;A+97@V{km zt--YWK#k`090XzXMWDul=eF;Sg@=Gp!~w#fA16~&Z^5h)Pq2k&p;=52Pz}1ZJN5>l zr!+zM&)m1B?=|x-#I0S-&Jj4(BBM0jJ-~qCeoTkA^qU2X#;&8%9WZTuCSUL%A++kL zGEltvsx$kqu(_d#GxdrPP;FpF?V@%wL)FQG_wt0d=doE-$}u(I{iJwl|PWdtj~k0cXye5q->S zV;$Y^TU)5iuNtxld^XxZHwFQN=~zHWq%5u6B*JuG@L7?6y#5SgAHMV1Q2YpGghS&a ziuJ|kF6gKAwQZ|G+=5hgim@MK%2<=>q)9K;>rH%YpUp3ZSCnr;fpL&>f72DGk6D*n z#GEw5FFmq0mX|e8xc|VqmGK_JHSap797D1ioGqAt{qfU|J7>=RZAZ&5{M8*nN2DYY zl0|wj+UL&9+1AD#Blx8)2J4~W$sr{D(i<-g1rCn2@9-=kbj`Lsk!CGJfRHC@h4WA8 z4gv-mEZX5cw>&<|FlQ4;X}~}fD7u)y2}6KmliwzsC_hB_&M$e{`L#Y=)!?$=xp4y0!{?J1{B zn;61;j$ilQvHlLHX|th>F{7w-on4Ga@GhO{+I#I~OZu4vW3_*x^XLPG9*+{e3(^TRBeW9@Y}L6#+vHNz=QEuUr_Lc`v8<!|(h0Q^%AsaxL$fI*}tLZV%a#0m$P@R(_4neEC`;{;ZBm}*0>VP^AH*7}KIvnzT$4i&aEx+|W*OWiH`H>h4 zyf}>|#^#sbbb0jCSmDoIU{u^Tpp=ZgPAH-1GA9|jq>TjfCP<|k-+(J>;*uA>`dAH; z&>zl}1cr{WV8MVgRqn+LHty;frqh`*RL2IbWwf%%pJ?WsuY5^or>0i z;DbGR58os+ZQVN*`xEUwa5%mne`Bz*UK2cGuHcyaGWaGlWAi?3!U@?DaIemJm9e0Y zbuD|sSkOmXzNF6_1n>ZMxF5Z^^z=C)3!%B6?DYx#7PQ64?we{;ynsJ>Psd5-J|E#U~zU*AkG+;!kk=&GSt^=h??B7NU48H*J=FT;}bUFr2pim_O8KcKe1`)#OjO?h?>v0dWJRD2G$+! z%^%syT;?;70PkSSV1>wNeBYULW2EoH<-*0YVgqI8v_Z5? zzeMGr`M%u;4@ZVYJka}3`2+oKX$QHJ!29QVp4n*RhcY&TQXwF?)`iqbOViR8B&RX_5IDxc@`gO2n&^7x$TQ_A*z zb%So>Ngk1Xj^@(q^wnGlXrnh+o4!$yB0QH>1OKcKfhfwqkY%j{yx*F5WX<--$mMVX z0wR=|K+DKH90Cuj?SA-3$Vtj&$S_iJ#rV(f8LYFOHiI~QU+^X!&)!>P6gVlv5iRe6 zz0B+nL9W4P$PZ|fAA4w=zqn?ir1L-RR`A69!F5vtU(ggKwL;hEm7%qfc`M_2zkNTt zb}V!Jii;MN*Ic|f`2BBx{Jy|r_>7K_3Cz6jGVbu;=uz+O8IT#nhiZg7-ioFR+^e*{ zt_kyK-efnKPkOuKTd!FbGMv)DHdVDgQU^xol>5z+zc*dJq=?doCyQ)++hi$s$2R}I z<+1f))Xu*h8OYIyc3rgBzg>U-=D7R+heyK#pK|uK(L=-mKsuht6i5{|-+O3nAf-hX zNGlzK$8hn?Fz_%`BDfY2q<~=;r$}&(o0SO=B*Sp`SY*8q}Z(H zL?DGwlw2zF78yZ=KsjN2WdPbB;ECekHB(2(r#sd(H2j|5o8$ zFIgT8pHhmE8B@l=pmJ`r0nAf9E$?C|LP%Q1D1rq+&4?JvC!-L!?WxVdV9FU^Y9_#- zR6N2WNX0JSV+|Vs<)l+V$ZLc{Mod5if^!6AbA?&KgRH|$sOm-!GI|Yyq#uK>bS}n-PdD!ExWb<$bEk*lIG^{O=1zYZ5Jc>nJ1M6&0%N#jWD$`asbAyA#pYn5 zUbL^Ah3}+lieL^JH@<{X_}Q&{M!k#SMHE2`#*U|2mVqb{eI(2qW6XBSA6`4kDD7c%;-%-G7Ab|_eeGF6>qL(GZVKmE(epoh(n@9;igFyPKlu^P_06}j&17nR!fFJ0Uxk}!O zPyON>FX=Z&5l%klqzIS5vHjA%Q}L~vb4{2qzDWB9+9mW7##@EfxJyBAK?<@Ta}JjX zmFPFcy9F5?AG>E&tSLN3CQ3%0iRLC$+x+&jrE|jQciQYaF1d!2hb2x-kQb|#>nW3i z_M#(cY-r!>cewC(_pXWWvJv9@Ze3AkO`B3?Oq~$QfM~>P&YoAUI_p@OBHL0Oujxm<8f&F&I0*1_j?|o^ zE6$kLuhW_ethG()FfV{%CY{F4F=hafu?+Zl)?P--o^cg_N@$%N&QU7A@5RgeP73GV zc*Pk}79kDMj3AUJn{NTiHhMv^H-_8y9gfJV-~8^YW3wIjnRD*`AAH%lf9>&`BBJiR#-Nc0=P-{GSrjG#KGZ`luY zj?768MWfIRu??_?q5Ep?I8c66j++ zbm!BEF(5I1O`JNZuTNdIYHnn0wcm`MG}F~zmmG~1x9VJ4PXNuh?(7BSrpuO=kKXxg zd2-{bz1qeuRbxV^a=VO>HYoK*%Ho^?1wQ*fWOwMm0?ptYa7i1 zL2t`&&~JvXu^`tK{m5EC?+*3K>KQL>`W^;4Jdkma&Sok3<{B|OuADjVlw%_L$9iXb zUNBtWV%1#e^9xLSxJ0P{a;^2nIgo%xvH{Qu{FoyB6(MYV$ft1QkoWY!^x*FgKDV{l zHY;V7oZCL$4cX)8Sh-g-c)`8TZiy(=Rr~ffrATY`HX3|nNWE?m!6VB&bi3XLtLE~V zO}op^J!Gq?W#8ewF|Iz?GvvfJ3)Xw_k0o=b^?99h)`EQnI2jT@9Frf$8Z6_=8FNnU z8%RBew_CGa;0Wr%J6~M0upZx_6vA-mTyMQ*MAqou-~OF_f;XbHhlUw91tQ7Vq(>bs zi>6PC;kRDlA$!qxUArvgD0{hBzwi$&HztYa?tX-7YJc4v)B=2*c zgU&k8m(+vF|L(D_^iw+=g|hSnru^E6?&;f*w2RNll(jcltJesIP?m{}j+4#!a>}1~ zoXkx$k^GvpOrTVZStD2KH(p3L!OL&BaB6p-K#Nakz!E);`Nu71v93U58Spg5RY*LKv>M0!A7c(2*a-}+Vp>O zcOPJTT~~pqk9x7XtZLbo+-14Oj?+vW$LY-p34}5*149Nv8yL9Er89&(cV-wEhG7T; z17Xq#A%&3QIL$4IYusc@vaD|P-n;bQ=Uw}(@8AE?CrbwI^I(>qU+4SIcg{Zh?7jBt zYwhM|JDcUHFKnZ%Lbwn}5h}-;Mf1)*E2DJw{Vmtdi&S5N+L#f;n z2;P$QyJK^-nwZ#>qaWC998jeDfsJkcp@6`=@7Bx0(4X|r&} z$6vx9e0fQH#CWEREOSmV``z!qDQXkxYZ}d6h(QvtDbEz?+9>UJsXnj0a!!vSIro`I z)`rsLGiFefUui{!;Cm;`(W_wq_^LmQO+?7)Aa)yHwmF!prHjx@yE!pTq-LoB4LAIm z57Bs-OoH+Be$E#tsf7KE(9+AbVLjEb@VudEruA<=^EjLTwcGc_`C_kG8(`2yAC!pCX@~PFjQkg z0XTfTY&0YC=|{fn(ukV&9Ljbw(&;TW zqtkBIo%Sqa_UP&zQJ?F!MRW7|Ldq_KdLDPL^Bim72islR^PHo|W5&CFQNVR=^7@{CjL#H1v{&6gmGVWk zlp5>J+TBqm@&_Cr8$QIT_t_x09HC(N^8q_MMC@{WRI{+T+IW(-B8sw5 zD8kDxnHf8xD>sII!-b@w#;-P*ML&l%%{|Mu3^7c@i!0|%iHsYMVPLS1@f0TVo|mRpN8P#QG_&$Y?ula?s&Z(N+h ztCX38BhJ}mE@*P3AeTKU^pV`6*5=z^azXQf&pzHv7;QtrLB&>0|8!DiKNzYrNN(I?Hg%dY?II zd^kA!l}UR0b>}x9zkgMPi{OyHzH2AyBI z{o3ZTIa30^wP9S*YS%5Cb85EdwtYt$`%RGBME1YxGMlm=4&$BuCFnyNc9+sk@CNm^ z=J9;bS2ium9vwEf!@~J8|J(^-z>JqT%m2oVD)i0eB(K?3S*wU z%4dNKN|R;kI6bW&EI4=KkPKeIk_F3uh-oZLTOAF5%T5`wyGm@3xn&#);v?z2d*PBN)O%@OO zM$SaLWyj8(Jg%8CZcLOZfq!HHG!G7N*3-?<-~aj5r7@1yBm7K&r=FdLMCaX$-k6Vx z<3@*poJ?%K6RZO|flL^@vgl&PfajWm&x6-9tQ&(#qep~%jt;{z?Jyi`%ebfGnSqky zs|IB?0a}O7fp|`9w8JYI^krt&(63~s-=|c%VP20`X7*y9JKG~1Wa}VnS^hy=+Hlg2el%A{|iF1w^1_|OIAp8z}|djj)IK5#BK{j}c{ zytjrD$OeK-&}u-Z&>v$D;P|_49GW!~Kg8y}gPFv|hF6^O@K|&=o)3(J@rSee4^KcI zQ)V~rWR27-;-a?fDCeqv?`X3Dyr!1y4b9m4lnu|@6Pw@ z>L8ve)9+IctqB`|@9_9vyy}t|0Kh;$zYFg`uNSRNPJ;TzSjM#=kf%26imb9MHI(o0 z&+Rx=d-`Yab_AOmAe9Ar-qdl;Prm-j9#L@3uik!LD3qde8Q~ZJHtjwThH2~5<|Gz0 z<_!6rW8AfvfQ{fFSwzkAhkQf^gsREn)P}O~FL)cHzI@xg^paokxU-fIo5D6`_=NudTmlR7aje zP|au7fpQ+v@cwfqkB>AB0-9QrNvvJtcNUigvnEB|$_uAYj3|7+r9hX@aAe%k&xi)% zS(Ls!LRo>q`gu0nnLc53^Vpgl!O&+<8Q;9?h6~SVtXFHS3u;lB%epU|Ie&i7@?N?x zf=l9xxi;LxxQtPGwxi#z*^|dhbc=b(NQ8Vw9*zHxzxImQEMoh< zL*d<~P~^JK>$*pdB1UO|7p8Fg*6Zf=1kGB&V}!;~GTkTJQ(Kf#ny0*^S2xXbI|rJ{ z<448ywXhJM+f9cu`lA*;wFz(k;LA@3Gs|bFqgdD1W(57%x65Wr94Tl8TA8&;F>{e* z$zZi33X4I4Aph}it%x*IV@jzYOmX&gQ*-nALyrSPj2V7QA)+f6dEbpYZ7NynM!Ob0 zS5hVvnU>x>v3AE0Wr#4Wt$insMFc0jl!D)dAA8N^vBBZST?`!~8yiXH{80WWu=gz8 z5)q~%xoixiY=G#Meuilgi)ttX&JjYGwLB=c7Xw1KAHH)$y>`)@hC-C^U02K+QW6D? zeR{*La8_}k8uS0Zz&cyHX?N60q*yRs`6HBynnt_uTj`8w7MirEQ{u{+t-<;Y!&EyW z)(sO7NUSw*?cyX*P~Gf8>yF4S%XX~wK_$^CorPf)RI!im;fkN#?QE> zfbIO#8~2>nFwMY9@jA5oP)~6*ew6ikbKs0b(HA{b>)I5*6nE4ZJ#@& z{<(W8|293!4Pl+5RO$mf1R_XoY>J`ivyZHa8n!8upb6d(?R7*e%%nLG?P{(=iIu@~ z_;_r<^SfVM9FavLoNIVy4=Yj%dy7Y?G?3ETId%IDye(o>8(F-)K67-*l=2>7bRJgf^~yd8 zsM^i($mckUkjol|a9@Azg2XnYPlX=J^g9*?I``lmbyQNTfAyN(wf}G!jwz-I+6hc+ z`!kf6l&b_jaA>PzdC#)VVI0#oR1JAXdALdAdQKS*Wi+31i!aw;Jr%4TJWD40iQ5)7 zHt8XAS_9H_pV_oG%DknMUSLQay*$`L(E{(d+#!Ve~*8}~!VgB^y9XDJU&Qs^e2Z9om(qJeOxol-Keu%IZ zC7ilMX7Q;V-Hqmt2lV-mqf5pjNBqGfr8G<$GV=0C=ZtE;>xwx&qqKhi-W`jhR8U^S z@c-ktEo{E=)W&EJ?&7~>!g^*zPs=c^gL$+;p1kd<^P0P!Ru-m2))(2T81(ZwDTDcL z%7?dE0LF%Kao4b;ArDIFH|~r{8+I2*tSDeB(j4jEkG|&8o*ZjFn<^GUnZp{;T!Bvv z*n{K+J{KcCddH!lR1%|j7wEENcUimm89;y0A2?OhLca-M!W$1CWmFo7h`_Uv@08<$ zpBV+rDMdDz-+7N?TM7O858wTg9v#fSc8{{3 zHjOzLjsoq$!K8`SIQ|JAbFT#Fx^kSXEdf^68G|AxQ-TQAhrHY2qs_DlW1E{Vn;A}N zr2`&XwIgg6-U(iwY@=0;n>FfvQr6a<&&C!#Pmo#tnuW8Y?3Q!-2d^0-$$IDkFoyKR zlrK`YF?ZGyeDbdNDjN!2NaQj-1ODry5;^_!b4~*wqW4Jq+4HOsS#?*PKP8-tb*V)#vrvd-WBwnr>!z$V#!eIS>Y#;gOkD^h@JqgIP2ekOv>$Gjv}YYXxY+ zJfH>cyL`@+=I7pYW&GwpcB?=7%F^PnCR?hPGTRN^c=t{7d-O}Y z@4NN#;B{r+pi#{=7fcPkMqoxy$GT~L1P-9##6cLY5pJvTo=N z?WWMh%+*TMtq24yABKs zdrG}wI6DES=QY(Phvalw{OH-jIa8WTo+nB>I`ty>Z&12_5p?!!x+o&)5d#58nKd0fG|Wr zgjASZGAii+|JU2D4x@Os)9;A6_XCS;$%fs*GzicH59faBbyr5}sQ1H+E}uI&h=4XQ z=wH44+NhmFSb_M@XN3FOQyZGUtBT(1%lmG^`Soe%@w`V7y+XYV}yeHqel%7uQmmMu_6JJ0HT*fOZ5Y7i^SoP;>lOX z7i~B1qo_iNBX0CZdo`d6xN$B zpB;k1&wluUV6GIuRQ`-ht|9%f!I&{3K>gkwPemHquz|9%f@h8$A!WWWi~4=?vl0j| zv<$-99mP14Z4jJ*2ro)B0f%r7ccg1FaP{g!f-knM2f0a%n+0Bl7a=}r?E+aEZfU9;`@c?P7In%k#v75Zb*Q3?tEfH z2(1L68qh)p_tSysDt}#Ifb4=bM!3mwA3thDv-NN(!;|M2Dy}^5)b_G1wZq?X-T5&N z1i>&^wLdZ*{Bb9un#ca9`=m!F4X_porOB`IIj1{%honPN$CM^AzZ?V1e{hcBhf#uZ z!I5FC-N&eP@$5;VjAZ+0nFuCjr?kUx0`$i{l+BvAF}5c#dJ#-1r<}X>MmKrQvQ2x# zFfGNs3uwXTA6*xQA?rzvJvfAJr)RV zJxe$D4AU~J{`$ey#W>Q^2W_a;o&2eun=C(3KPNv+1+`JE=u2zd{cyy)aDpUUKYv;{ zQd}dV$2h~IZ!a&qo4Y6HIPckWh_X4<5Tf)%EbP2UdS$!GU((_H34YwYY;$XTE)Maa z*KMR-ZW^_!<Nm%E-DmAyHmkB? zMM`XO8ZeM?KzkRwo*{l!>qyA{xCULdFM-eRe(8nbY?DFw_?jINEu=J*XL`T7-)f`- zvZnnEYG7op(7gAPNUB#&8q97Gx%rX%R`kjjpu^LuJ0HmA(Wy&|fj>_RcLi)#MJw^lS)UobV4&Q9~auUXTxcj9?Is546b)Q8*@*cyBJ z#WRQSv~0p}AukOy&EwDQi2FjOX&Dz<^}AnK9MP0(wijLlW6ddj#Jy-d2)V9J{Z``y zja{~JZy2W0&!fjz4GDCRHFC$3>%(X*8wp>*Go2^TeesF)&3D%9i2Lo&kR|BQjl=s( z8h^Mu2AYQB6us+0I_$|Ef@(P5W=|g9yy5D(L-=WyI6|W`+bQwfyBbfBMzwqngHB!A z$a&FfAN< zwf^*d*a)LjZezrQhpsQimC~5t9c!XCFyyj?H*qmwOoA=#vX^Eii z{Z>9eshtO9`fh7}Y1`gr-qdrN8|I%IctUr=M^@W_y)}Zl%Q@NzhhBmbNoQl=W0bMk zTx15E9O$L0D=}rC1Tr2ulFs?~x*f6KQ$FK@Y3IaRO*-z#XxyE@2`vrE`?^Qb^ZvR} z>+AY}c34l20-mM~j1D=)T1Y&kBOg6+s5x}}MDr`3d?YsgJ7=tn)!9QbvhrCRLN;GS zAGNI=^X(bdDt<*KQsT1O@Y%v& zeslQ{eB|u%&N806uxQ@Y2*gl22MIhIY&v=c^XKRQcsADA}IC#pi z5`ms_T74}%jx5(y@*c$JT6b#+@7y zG>Lq{*-ZX}hjc)=xaUxjy&)aE3!d2S>jv+P$7SsvI`9+!V=Dys2>G+S4jAjLRqHnQ z)NuWTF{4A?XA|SBNr?NyF;JrXa}MY)8EEKpW^HoVsHOmREZ-9I%$fW6+EUV!a|BQ8 zLas&UWk+QRDBt1NkG%TQ=Pe1?ZSSAr_kUJ~>HqPSrHxHghL4*Pf|~^`tz1ezPg*|^ zY19RqZ~W$WUmr#68K)r_;zU@?ona|@(rXB#2^PDkyTiw9q%g<;h$4uCwnTejbclg- z3>J)>=S_Y7+D3M2wfpGuYG-+y0)6V?HA6}Oh&1xY+rDncIq^&cnkVoZPi+bhJz)sp zKU2!R_p*pPjZg`PncDf@* zMA@_%z~XhgqCTYgG;X|F6pbxU9SWh4vp__L8ml$r9CYiz<~0Q2|NQyIAz&hA1Wr5k z{@XV$>KTT4fsda!5s^rYsP%>u7=2NRvc+_~Ek$XiYU(ktqIB&}M=AqTwLjZ~gv^7L2DY=PYhy=K{*m-qq*Ynz(jJsy~psDQ5!dr z6E6f9GqOyFdFcw}3fM?T(d5u-zPBCF{+_UqTp)pzN;N%G z+Owz|>sU0dOcF+&?|u12Avi13U^i;h%{Zu@GNVmqoOJuMbi=Mt*7S*>%ix!k*LW4$ zT?Ru9udZRb+uoVJ-?^9*=RmaEEd#;;B$7CRal|}LKBtt#P!k#*Cn!B1hzRXRCm(6+ zU)J~H2l`X5%bEtbmf`pPuUpuoY0lXPL6l!8iEWKv3K8X8hqjDqOBlfe<4Vbe?|3gGrFFXZkhawU9Nf$(vy19ByPri? z0V#miDWLbMWo#Xodvye@N8^_3#C66ijV{I{VfK{%Yc79hyyF13hHGA6-6yF1>|MXU zWK+=Aaic~>E#C5b*(4?-*(pBw>EArKCi+ZaR#V=%5cttJ>#&Y0_gU#20Ds6_@czir zLaWFOIUYvny|1{ar@*5Z=nz0tMZR&iB)y9t+Wj-qF#XV~5*6fq$t~uUfm-xP@a>Vq zo4@22iNxR zIo|y2hwl&kp>QjEkR=!LT1SyY+F!P5cgPL`7RCAXees_eQHf|+oQ>aF_&4^Eye%Bb}f_%Nc>XoV{? zW>0ROQDWoJ(IEzQDY|gp-X#oV45S=Q@Ks83;$HpC=lh)N&wC9U%>hw136(k6H7sR2 z=%{s~M5i^A^Wt~o4#-Hm4jc=-iMfc{W z**9Jpc;&k?8~_Gc2YA@Of7>;2?sWfs{pn57ugcdNxHxUXIX(LXBPJXWh?S~T7(oXz zlWDMZ<=d3^s`m>}{saD#3Ghj6pta~5=TFirauqxwh5z!~u8pyzQ;=~O(fWaYb>U@~ z%!oY{oX4X^wwqIbXU(=DmkONt>f(*fkKS@=^TgU6VTZ6r(f$NZr#VxGW?&lm=i6Ab4iCK=&_~YTV14h7Im!HWdKquimSqPpk-@G@1OS;$E z#GM>dM~2%JwANAF|8(n6;q0~s{xJZZ&y1>Uk28&v2;eiG2RDrs9Tv@{_NTl^4?{<; zYH5=-Rr5sc;KL%&w}wNq%puy!=2wF(ClSg00+Tp^tOL5Z`A?{ISGECw*QZh1cY9I@ zij}c_>y>k3zSzIWwOz=7EcCZFO6PUt<>%Y0Ca?Rad>mZ8|CD=q7JXAvfr3fZ@-Kz&Moh%f`$<~Bv!Z;=1+F$h1nS&>DK8B>M|Q-_@cJ*CjX zCM+c|20^aft>-4spYA9!E`;$TM`Pn8!Zcy82DP=M>@(8v;>9K@?FAOd*Phz+g1qz; z$$lVJBOxOl@_QZ$%#@|BflDfqC}kcY3-y`{r-ee3%8~ok_7cevweMQ5^U)Jh&ZJrm z52lR)As!-NW{euqtZE}mD3AmQ;|m^Lx1)LOuDLzoAp7^l$4kmies`oKit5V6&xkZ? z#K?#)$jFiUdV($^2wXyp5JQpGwMRRzCpaKHQZ7<>MofR`=8KyjYA)_Ed)NQJfA>V2 zN?HtnQUawO*O3F}UW8_r<)G+N65trcmbVWsB7$&}a6n1&9(O;zsadscZzwegs?8b* z;)H7gtn0t;F|htU(%~v3ntKXu8IzrKHU{kY~; zm(A>(8Ni^GDadf&yHcXyKS9ZYVu+$-f?5F@@_@7a@}Ya4ZMvy<6lF%H@nc3cM~)s3 z#iMQxesTo~r#XnV|vnGxi)oeO)C?eb_m_;cW=#`|9wpSab3Cl`i*jOhGmiV6a z4rMugT_dX?;rF10N9t%&54`USPH#+?%-&hv!0F~esBNl^g(tQq- zy3RM+k-XLO^raT!v!AsCIF!5g2ZjXo%sERapZ*v{0L7l8jsu9}Jfo|Tf|MH^%@2B4 z=tPO0w(%1jnJ`Y&Yk3%YVSex`YcC8oCkG;0hjB{%+K=78GNPvNoEr4=$J;u#Hk$l@ zKKEpAJ-Tkymi>YAu1iRqE%=+tdfB*J1nbXq_A|-*>f=wp;mYRTXErz6_8e};jTzA_ zI&W(5$JZ{J8)aFnRqtozlWou^@5LY@TS;bu_YPxuQ_{TgB6E7~#Bq%%KG`_RbcslW zvv3SvtHX--S?4P@?+d!X`M@xJI#Ax`Pw!e528_H$0LL%A^_u71#_6W^=63SOc*kTA4%29{Sm>m#=+v3)D)v@7CnUzr89P zwQ)fkjKUCRJe7tKg&gqnDmb31j?1j`tsh1x7I}R zx|9iOIY1vK9xe{-(QWoz9fjq0YXOk|=g&B|VRTfcDrGCbDLZ8yF>NMVl$5CI`4i=o@|Q)ZkrZfx|&GprTu`a?g-+jJWI>gn~nVs1p+^52|P5<+R7 zQ6s_#j>c#g%^5usPcCICri>dM>Eu_;nbcQT=#HE3SIkNNSon7GZ{N(HzNbMB-B!+m ztoWh(o();ryC}lK!C<~P1D0>v9VPO#k50n91apDzc^9&CSC9MP*Om-17%LA+uFDKF z`Y9aHW+}Nd)R}6I;AP^f=W>L8&r25!S^LJ2lY~<*%U`C>lYKqgBRrPLNH47a%uDKn=WjA z_QMZ`aXI^dhSEPa(&yXE5;TW%5&w{J!TDlsWeJDEJG)HU<^v_)`2^@Dy+EOh(;qoee3?$-+5j8jN{FWN#i0LAlqZ3QCf>Em-P4f_ASZ#&gR%pm&OzDQ}k@}o&!Pi zWMBzCWn;L7&zU)CT(A5W=U(VAZGCa}W`8FK zibFE_C)uZ71A=0#$69_*ea8A)xn+N|bHDo2N1C@?e?hZs!=52~iEt8J*lxtkCO|F_ z6H|Cod8^Dy?e~r^mUI7X4AV8EsRut&x#|<`W;T>=k%1Hu6I=sDk--B|z??;)FqDay zL~z(Ud1M%RrC+8naKnNb5gDc}sRjt$$l)WJ$*p%@GzBOEz@QKojA`MV)7Ids`&Or9 zoX%)WAX8hbYcQs$^5Q1g;7&w_^A;UXS(k`qx=r4bN2M4N2rm5~H(RWKBgom|)x4w- z`1rb=VL17>Z&(yUpZB7i5LzjQ2^mR|M?xYX_yNSgqNf0)QQEm9M^7|=`|#T4r#p(> z(@}RXZteZYBh2g8g_@bjM+rg-sq7LX(hryZCQ$6 zDi)L?O0V}U8%`9Vop4~M_BwI_{w1i(g;1IQ1jcN5$oPs)3)=oWfBSeC>#x0_MAf-w?8w41;CL!I@Erc?my~aj`jq#Y?z@+=uZq4U9LlaK?LEOm0Z66+Wt5V6 z<;uy;=H1Fu3=2bgzkMGlc3luTNcl0o=)Qg*9~fvxwoX-?R9g2he(a&x49Og$EgZjq zF-o}^pr1KU<{w`o6pJ2Bx#4w-PECQ0o7?~JgMLse64NJ+3xjiB=Q(z@C#S)m8ql5m_$!O(9EZyY%s6E>fduH3B$ zn{AL~9$L9040x%u`Hk_JBA>FObJpbOk%KiVorCMgh7CkLYx2hHubx{{n~%$6!;khw zL>0%3dC{giid|T-W#15ofZrKlClE>0Up;!}LWOcQHN|`a{N%f^Lp~^fi}- zg71DM<&?i6>%ezo>KZ{2Xtwls_+X7CelV`rhG`D+kKezt7n#QS@x3=K=#}^3EZ1-R z6g@yj$y*Ps*cOh9?|u1#h{Q&};2FcM=t>|%?l10rHk4{Sub_0VTDdkJEvq`SceRSqAF8P_Dn6QSInWs311tvj6oU;bI%?Ti4U%v z67Nq2au!T1^XpLye8tyT054Q7x8uJLGVfYHcr zJ{xEAAkgVwy}E4rN7h$nNRSP>$vEd(nYOJxjtTmJ;G?fE*%L=g8YsO{r zMDl$p_2%o5)jNhba^Nweh#K7~?>VQAzD>IDm%7wb(>fbv0~%9v1os)35;95?eG-A$ z<&P0G)F-->Tn67h_vpG_U3G18?j1WhFvOvg(`?Mm1H(Q%De_46q^zUoi&!1J^30#$ z^)I~Ts@P*mmXG;TT4BS^10fr0!CEeOQ20j3N*zAqI=|Af>2By_mhk!D*OxTE`i|>@ zrqn%xOZ_1Ac(MZiX?=vTDxNXW*mV3iKl5nFTuGnk@pNgWEN$q^F(JEvG+)9=F2&sj^eLAwXym1L#v}Uwfki=E!|M;1--QytGO7X z4M6Igv1X2*Y&PsZ8T81Qbb&)#J@KU*O6gR;zva4lK_`^`)^H>zW<=g10CESUdUaHoA5j<#e*n zaeZBW#vjm!+9>Te>t1GlZJ5q`y3cSP&M5iE{sGW+!j~2kjF8T4dkCugMf- zfA!_bu2at$8`)U>X!{S|d~tK&*swbn--$e$jrC8Fo4H9Gewe5c;UhRINkH zM#|DX?q!dJ>zp7jb{LexU*vC+QE}bZNCnAx%E14=dvKFC=K|>u4H|2m6S>>1<%#fP zYN@z)pX0j-6t%TP1S_*3HTvX};AqF_9MjTkpPph_a+V2x-=a%ImMK7={hQFv%LuKlhwd zB9#5UWaI8w(3nC3A;50bgm|8CDVA#MQ08lJ9dI5)Nnx762ngpt`TB+NZsr{mCj?Rm zpWQlij$JQ(y9>sLp`2hick1}?ye-|dH~P#QTnldm8{RPe$P_`$k5L-Fhz!x?V<%5E ze}32UIL9chKMZDHe|lpWDNB%XWkh!h)B-c8?YA|#4NN>B}9 z42gQKgXa;{ZF~2J^F;r`&_j7>l>iJBfje8cWagyiiNmGlSKh;rpTd5gb2Pr@#5^s%@0RKZzMF`~vEuLXK7*;^` zn}EmYl+G@{QP|K-m!@*2zwl~NTLWB8^i<;AlLmSQl7#QN{PfYAdCl1zMW}q_TPx#I z{fC46qcD$A#F#Kt1LS{g<-hdBI?><#5lvrKi8XUVIRyLfwRmpBS_-arP}ln6nG>QmX>Eke z4T|E$+Rek12%*&Al^4yLd`4$G+J%NHeZt5K7&b(}Bz)IpGb8;uzdKsP3^!fCH|(s= zppxSUUm~JKCP`~x7ul_Tp^{_PCR$7Q$M;{dV0tKklpza>!YgI(B=JP}` z>;DhDa*)r-lCYDzXOjoXg=rM?JO($%Tz~q!t);8kuA{zMclw{e`Nlxp-B4Ef9nNIq zylC#!=iRr#-X+QjB|k&U$R^|`k<|S_WH$pkdaQ&F+CpaV9yX%1_N>3j|FlZ>3OW8abH&+dLE`f5I9Gk6d4S3_+~A{urDRuk+b8os;Qh<8bZ5 zB{UZux8a}&*B^Lgkwu?ew=2fd`&%>KC5`9Wt;aN<<49%|U;#gq9~%p@2S7uDBOf># zI?^B9@pSaX`h;^qH_M;6uEogUnxFCm&qZ zi!{Wu!YSE03>lePR+Y^RkyBHS5U2sK7&`FPpL+emFoc+s1kMUhDdTUAA0Idwn{5x< zFH>|vGiy?Ds=xe_8G*A(Hg=(35DaIHb&(*T;)id(B)H~6`C-{EFpU`({{93z(OA+ka`O{}92-M2jSXilCka2P$l zc4uR=uZ*4+m1z7K)C8HSKQEK5mOtxu+?Sb2&MTQ{!)bD>+%j zQD0t3sN+G8)HgPNWHdaiuC;AGZA1xjF7ZFXHGlKanqGMUcx6uD!@$WBw1jqeZ|xh4 zod48KwTYH09fnrA-`Y>``Te(D5#x+L&|NuxfBCN;$x9;R>Y91e8v7W?_PKcGVCE&> zjvk@^H3)tH==e(%dJYCiX!wIP4G z0c}=W{cJ!sGnsYgzEWBVul0`jGM$7x4e#(8_7|Cn-su&W_QzEQ-Yl1z&ll+P`R}ZY zY-hBHgYZ2sn}5nW%`kiBP-{IlG zLuEKN&EWiX@2|fD-ykZ#lR| z_MY!EWAVMqwuCN{*Kwr0?4lV#r_d97i=b_pk(t*yF5Ou04{nhe;1GO5qGF$Nnte1x-!nG+0spIRsQ4sw_P5QeXhA++JxrM?isArm2DxUhCvKa#B@liXBQV@46GjKa zP2=_T7fu}#0qq`ylvjmNg#p)T-hMy-+~S!NA=Cwn!49F_nu$O5n#-E6KD{wgIE{f# z`07+`gq2ZKxd{ffxl_hPWL39)-)+i6kQa3XFkR#0+3Gaiw|sMKD3r|E7$u|^pGVu` z?s@6jotl*}AvPkgd4Jwp3$%F-(GhuLZh?Cc7=|f|9L7y~tIKDI)?xtBt$l~L4N=y# zv0~fah!(KfS~IK|g?J`0Y@q!TLD#-}QzAhS!mlje*!;jNFA70d-->Z3Zi{rrwuxXC z2%r)NwTF6^`{#E*69$9Yz@R^v#jBnt zHP|&bUNj^A2mqoiJ@18pH#g6iP69!Y(VqY;GKSDWcw4%0Z){MK*L9DwA#AiDQh?Ei zVZz#Z?<*Gc%5CI5Ha>X8C9|48xbvx=VvxUO&B%d{;7nNsiNjLyfp<2J>yN7`XYPx^ z%x@_Wr|0}zf1>QH1a(EG4wLdiRg#Z3@FwyWq{z95dztSRH%EFZEE%$IMyuR zvM2N>PC>O;YcFtKf2Lz99fSgaGN} z(4kMQiL2Ln(D6}SQQZ%t_>RBMxW~kKdqxar)P)KLBV+aRhOUAJo%yvWG)ec&o|DW ze{5YS)%Yt}MuhKwe(y_zwlZu>cdUWJ`RnhxE*zQqGiS;_2iiqOBP% z$~%OOBrl0)Swm67u@^O6WZJCB+q+nxN2dhcAAMzM$hUdT*A{P#>;&`I4@3gih6~pi z8-M-q2MKFhYc%6 z=-8MOIc%Ae@pGN`#b>PpVHgZ{Nz?I>x-V!X-G)QP^)|3l*34#czxK}Sq6FOUesOWg zBz0fAnI+1x8AEdous~%(^0{-SjtfIxy|HPwmn8%W9u@i8eg4j!Fs@hc-t|)K`@!tV zL(eO9pwt8!phbO?IVfsbc_#Xo%%Bxys1`)(!rOHC!e=G zj%#DDo-aQBTsZX58pg-WaMMp?oO@YV_vD+Txk(qS9et5;VStLdSc0SI^5m#om-vkN zr{|N4Qupu-Yi-AYqe1tzNv}lXvo1G-5@$lKcWA53RFNlQa69A6wqV7G^7M{)%-Aub z!nj7S6#3kZc()PeN554z6h?1l041&SjHE+BxT>{k&2seX7YIBdf46kl^F=)yS8Zl( zZP&_VCS$u(iBH#l`HA(>j{gL>Jh$^iuXZ>70Y~s-xFa)Hwt~z9&Jld<%6U_U7~+2F zbyqZtpA%SCOXJ9KA(*EIx;jX^O#a5%2EXeP zZ24QiqcQpLH&^t2qE9`yeRX8<+iyrV4lv()4<2nk`Ouo?>hlXN?A`PNKm7cg7ljOE z-mD*imT;OwPy{%90OZ+!dGjT`;1}ocFJR0vFoCV6rh|T0R@I&`^}2E%-jFBAt4i~R zQ!`2oj_l3BnX-D}86caIb=lMKSI;|BP*>dePyD~38o)pC&OTrFq-6g5n-+!Fe(}9u zYHt1RTZ6#VUe^sf4)kq8iuZzvJMMdBVS&VPI?+W@; zOhpL4@v8GqS)|%X5Y49O`X?U15Q;z?%r4kkxe78f^b&-m{r&zOgBx?Wf@ev@3}Yi9 z3^V1y!R#3`r9uN~b__m7-@1Y^}V|R5ASON zMdce$Zwv;GfKD7YIs~<-r#5oXlg-n~07R*A9bixpk;{0Sr3A8F*E0ZM9x3P|R)jip7+JhezdCEb_?D}hU;OyPaUa=% z2M)ubtRTPyTM7iFh%ksSBWmUVPEp9g+$m>HUveD*=0o=^k6Oz~m?@Qvue|)8VGMyN zhJXEm|!d!UX0TexKJsX)0FWDDIDZ}ZI@|^(5xFCw63+Rw>x4t`MATEv4AQOrL=Y+MT z96{N%vncj}n*=q=Zs-+b!9HSsfAWlp;Wl&3eA z%n=b0)^&Qb-L&)I(b(XENB&HJei{2x24ygvk+5siTH}*_(U$KNq3vsIYBB~rta4sf-yT^rbk)${U-5J%^75En-MPGhNRBki5&4oP#aN4Z#~O!zt|pYcb~FSc#%` zPV^93BAPJ)uD^QW?4Uh)?jPfr{L`36G$0->8d)?^gq;=R5m{f|IsMjW@4&H|AcLq2 z=(J5(YCv;x4D!5aafUa>S&GkV7S4{;!~8vuwWC=?rZOSGx-!Pe@@3YOUDdS`Qy0o0}UY$~g+IiE@X`O%t$A0r& z*GDusK_6~^d-=BJ-+lDKXqtfz??_{_TE{eHajMO>}C?3kMBqT5KefNP< zYQSe`_vCXsqhHoR4P-b@VmsQ~NGjf`%_M=7=nUoGc%cV)MwZ+0E(cn;9cfZw3b?$tGDcp zzMMDhoG_|0I*4|Ko4|YIeBed)HYw0XdymX0%d&3TAM=tp8VB?`+kDrJ7e>^ac5OPW z92Kx;7^RJ0&WH0#=Fo`%Igx`HPbZ&nDl6wkw-PD%m8Uiak3ql9kNG1fle5e-`Stc2 zE;z;55RJG0=`VSv#=H1wSlL_>4!|YR>1ZNDgvi_kV|~lQvT1jo`|^_;Lhi}$*80wa z1C5%o*S9Ab{>YASoPlDXHC_(GS6cq)Qjw2$KD9mqD$o=Wx#$WVLxeQ?B@@RzGpCGe z#vC9MlyVY<7Yy<&?TM(B)dwHpDOrn*VvU(|@>T|9oj3KIX4#>kyp0p@1MhxG7~OM@ zrK5NGg?0I*$DV7;z?7G!7o(SCDD$6yPH}K)qi0xa9e9lV${nRVaP-*8=Kd91qij~{ zv&JH2Qho9c^p=)|H=^?42imUwzBxy?;D|A_4$4@Sq4yhH0~s{oGvl~%lS!22R+JI_ z$=eq89L;2_A9?kq%^!VrX~=TPPjG6_8&$~-=1yO<2?WwiXo>+ePE`LH2jk448Bx_! z#sz=V2GCD5P~V4upW9mRc|K{Sb%!=tH^$Eqn4^R|=esQEtikNNOrERGACzm!Q%O_x z!*l3OANl5rs5Q-a2V`fcH+}m{=AYI8m)WJR&$``lbI3cK?k8i-A?X#mPVHNPy(U=6edUo}>;CIFc_>u;x%!g+f_pv8Ew{aE?|-D~N_ z-O*2YXESifM@D*~hH34~ynFA<7c^gZ?780K&?_K|fBJpmnlJ`fDq57boWaI68}B2(SDczyGxYv%PF1NAj$`?*^K-hE$s;<=Cu z$RqTlfqetbpWgjU^IPwFNiWl>-v;G)RE|gee8EDgGdXXAO#v00Z)hetU$6=qLq5}v zNbEC}E~?jAS(Bv%5q<2R1DuM1FXFc%jQIQApSiPn*Nwk@%9~&${lNK8KeVP-u7WKZ zk;KHaUYG^@zFPlp-BM5gb-5hnG2#y!eoYY9yQ*TZht_6;W*3hkuT>thD`TWOU zThgeNL%_li%VKI7LCakAg`g4aviDPo-ePznd=MO6E~@-=ef_>yT-1E#;Wdqo6DWl0 zPS-ow|F5qvi%2`~Ln$Ksso!KX#uQTgmV$N&0l~V;2pG4LzBn!BF~@~ro4}TH>O2Kd z-%kuHifdk5zdr^qnvtLlHy8q)3niyz^!g%2new5|=;;7sQ!)YJ=80s0x_e^4ft{6IveFz6*N1cM_*+%zB zfTIMtSJVn75$U*zH4a+v{?g{pJ{!>rz3HlOGQvG+ z<*T>si+bS{A%3Pk%4o@pTUze)LOAl2zs)95|VyW=n~NPrR( zzdwL3B87q#7y?omw7&oR?q_>^_lI6}Nz{Ln9>>TbGQSHQ$w=h@NsFbj6q6ViAd^#O^Um~=wa>F$^_Yu5=i+%ZwELUB12?&^Q~vM z1no&&XF%{?8BLCV0kqdtGyeG& zG19i^rY?;^cR4a;)QOfj^14!!L2z6yu)TplULB8QCiq~oJnL&xw4Fv)A`qk5`%I zF2g5!#?FIBn!SfbOc$e48tmP}5EF`YWRH|*#RKy{<=w`Vv9x}Im&~6Yo4C7#EuA6J znAnI-AJ7TwvW6}WDmUgYd4+4qO@H{6C1LbJBg_dn)<(VK8Om_XpK(q&k@frX*^_&Y zrhJ~LW|6+KvAnanjCX_=qeg@=^cO$=P+TrL7!Oi2lwn*1>zGj^d;BijBs&Rq(XXU0 z@C1)cCsAI@04y``1D|~?XlSTVZ8@NN>_zv&Q*GlD_ys_>@$)nW?g=`4GH6+@BWvB- zq%SJDAk!=XErGY_f)TV9l{NYKk31NW-2$Vw>^T^HbC0okd#kskQQCR3OBY1)&qWL1EqT;Bg+oAp(kNZuQ@;ng=sD?VW=QnMQ4Vb4nnqD$hdvlLdZjrj z@ytW@S@ICsCiNZH(eOBgWPWSW@43dTA2JOdRqN*u ztQ@+TwdXz4265!GqZ6%>ST>XS96o$Rv-{A|=!f6KX}IHH^N9ymHox(%bk*dYryld2 z^B1(!daA`P>(v;rkDLj<@c8-&;>qh|C4Bt;RlN=AoQET5BpxJCir$2mgDtxcwa=$R z4${K%c|@$Su3;K|QLc~EnNE*qh3wEWAM(oIJ+?0PD#(7F?UCO2k_G36&B>acG=5a` z-@g1*;2u0PUI}Ee8I>4iGfF{%Kwv0YPP8<9C7%HSXk?M+lSz{X#|DB|Q=_ zD-0J=POwXC4G%uMH3U|OWkFLWD17~lz4d)Q|KO`GYCiYK$VhD`Y+3vmLP8RQT!hYJ zoHi*UsMK|i^BxMum!8h1ry9DmX`Qp=9yMQVXc{7x4(!%Zcx(wjn_C0sTdTYpMZey zsoA}B!=N*O@NLnX-x@M{Dd*7!Xap2dC&x~dR9%c(+la0@qE1r*7V~5(Ukmgjq;82Ihc5 zbMs}hLm;kG8S8fQGfy^L$As$f%HJJ#9yr`wHlly6;K$dMF~$f8AcRoNH^!<}6trPn zj02&8z(b*8Bur3~ysrD`JX|AqNugx`PY|6#8NlS`O+6=SnAQxLQ)$DadfsjI0|^M`j3Xo?jo?^6kiz(fcRoGjC$%8I>+)F^|5tO6!m*z?dmC8O>t8IeKii2+RwBo90D&pL3LW$_#o~gXoYPyZSs~S1PIYDHjwZ zMzD6-2i;S$!kBi28v-C9P6;2;_M+O*?rSfc9@z)d1Su*M3ej`5kg#apl)4-7StP6e z>!VIxk~l4_A{E(c%v6bjty>> zgNC8!=(x05wfj>z7afHbv{M&j!*E+~G?(KpVtEkqY%yL~i@-Q!F5 zPq7&z#e#D*l}xmFAdDKrW3x<=Ith0#+uX>^0TeHPXA2CG_=ooKG)MFuJ<=a@oOBru zGCsj)drpY<0@kOsE-I8^+M4Q?XmcH-k$Tmq1JOTS{rY#{txyS0OUpB%7D{h?TQm(Ctktm5V{q>gCb^V0b-pU<{!a^^9F>8A2Ivikfn zsu}mBVL5iB}2zW)FNU*Wg>EMmm&I|lxl=W`o+xn+9y{+#R zbRttsW={?1prJ%-lD5I?RQ!x}$}~2cTQGA{)D2}MU?@(&yR9QQ=-wKPgDEF_!VeMn z_LxYp2Jz(8+x9l|rj`1_Xc~T!V2XDF;a-Cca~HL= zL^*;?FSiEBjr23)P9_XlT{L40YUeYuxL=C)>C&{#iIR=kuhdV1{#Uhjvt3So^8xU} z#I?Y$Rz^cx$YUH{-p!D6c3Crr_Ydl~*814dBLkPrgLh#}_7-S=iSl%Jc`q`%_0EW6 z&HL;Z<0zWc+OV&SI@{hSZoZ@pv=r z2)VS(BRX5ldv#j_Da5uX$&X)oazpPD^gH!<{2JbC%NQAN^WwSQE0%I1dE0^vv$tM1 z=nQvXKaj!P4+I|hqfej@_b=btJhW-P8S^(dcE;*K7mtM-+Sx2{o&xT6V24B z58j9D0#B_;vN0NIXa&Y(aLjvdTG0IMch_AH-l-=aV4{C zo;qq|$OyMBnth6+tJ{|EL*?Rf1)T~Ht%3d{agjrc^I^^Q1I@?3y&??HY)fS0@+xIK ztRHwYed3^go9&@vbXt5MWTtiutYOne$D!-mMECCl#?T+0^B+IAB<3650c$bo>f|bG z%WvlCeXqGZoK{=+9t^qG{FBvdAkSagoZjd!yP^gTPO{+iF(aGdCk8^k6C4I_fiW=m z^u}NM#3RiQyyBwXo+N*E*Rml|*6aa-Ef&t55_&M4Jsq$Q0GsO8;JI{dK@}N1Bg>bp z<$c+vR#tn?wiFnShOl>}&H$_9?Vl88|Wr{a@Lq z7c&e>kT2%9ILp3^c*95-eHd*he|Bc?@Re9tgjBSKB5>jKiJ{cxV-S`l8~60Q&CZLC zvw&@QNH9&oE$^w1ga~!67)NZzSc7PR*mOta>N#Tqi7iTXZ4!~y`C>3rYb@cz_pNMJ zZXNV!WAxOGcU)hhYi)p((MNfY_9IeZ*g!-R+Nl?l=n|H^K97t*-(0>mYENR=(!QV1 zOJl4pM#9FRXj6m;Ld`|9CPa-y?}+hW9InKa>@qH(CHurE{LUNBOn^lU^VU%$2}idr zniHv#8NG_I)kdPHM(m2HyV}r7SwL87{J{L5*|;Y*sbQ~Pc;4ig%db7PF?J$1M){5y zFetv};#|N?QgOqG{AymEPr~aSjNCYk9W}by6KQ0{IBvWMmYCi1jrpJ5^~{indnE}( z5+`nnDp%`}v5JS&_^KiIi3e8qgb29!!#7_#q-02zmpOR&cNB zq5S>dZ@(rVA4l~H`$4X|z(Ed!)|@cjG8USXgab!bhp=u=mF_4C#vmS1pfR5u zF9HX~^v7RY+T68tW2`3|-VnZ&a1qVl6~NGtx?UnhS~M=Yp!PT{6x9RQJdd%{ABOsi zi%Pkl>@Ov*28_QSa4zLK6tQ6~F|LFWvc+F(#d`N#w9_0A(ik#6{LPg^HYV{5#@+w= z#l<~ljJDRssqDvw-3NOHw>+0ev>I)%LH%fv*2YD;`*&9F3_+PNfBpPvJ%?T1V-C!} z=O>7oV+^bftNELw4J&Q5r0y&5F1O!UXwwY~rpNf#I8FJ6`?3X;S+c2Ij%T*Z;MNbE z*H#*(T|>ySfh8f{7%=Fh(cJInPn!^iQ18XKz-eJTk`EBpx|^5vo?T?X_^Rz5RPr}N zQ5Rlw1qauxa6~eUf3(AwTwiAg<^45{ts|=PcjF`rLtRf1)z$^PPT-h58;~lUMrqf> zHEE^{l+Q=Ax^2r^z&|N;a7X`~xUWQTHO;3VUK2I$;0#U+ ziR}OYAOJ~3K~(yqgSRf49W{F0Dq0q=MQ8EGPdu}#(3dkH`)+fgN#jR1-+X3s&@F)qoVZ<>J!xDN%N+>EXtfAr)5=n4jgELW`X^G8 zflX@YvW`T18sfp%PwlJYd8Kz$GuM)vW*WVm@;fUy$gO5S_J`9>RAiCI;aRng+aM7*oBx8&TVeFVs>PIbT`Sh z5px1vR)jXEfwCRS6Dc(XcY&zqlLM`6Ve>+U>Xt?!J=quMjG?SNMp~QjC6I;fb)_6a z4QDbQX$Sn}y@O+c76Wws_kQqY%{y*5zuA98L}|$iCClTp=EGc|`;5dnKXG#y9PPtY zzQGlEId$TguwjrR)H@cvO^)T@E(W+UAtTW(6TI8&uAbXGxMJH7hYK0yj>p$WfEc+K z9ayz>fA9u;!TLw1@dU(1#5lYx{G@n3jsUFe<07LPzbtrPz=S$O5~7$c7Rny%ZM2K`6Z z;J3isTnEF3l@dud-28_E8VAUFXbCVM9Cp?c8r=o_G6NxU6VdK9{p9`n8u!B?SsBXU z$%7*UrOQS+@seF_t(vz9@lkCf508(V92{bM0m(x$4|z7a&NwFUN5lA zvX%vS#}k94P23~c2OK^&=w$Z2cA!qXct-4z!v=?^oi}A%v#=^38x6D$#MH0xt)w61 zb|v3%Xq!}@4Jg-E<4NALD>j|XZ!gZ{*^R)9^WFOP+5+>}1v~kw^9E@M8dBUDctnqM zfs)guPZlyo>*KAbolf5V=a0el0ZW1LIwMw@% z*IhU@42=Z++9;jD3}oya^Of9I1V*s*LRA9=z!CKXn~J^doY?n z>cdmdJ=5H?Y)cpl8JCq9up|5TzI=Wgbvdpl&|)kEI$#Vf%r{;=FBpc69~cZVcmln8 z$F6tXFTCaIM(S;CcsX4kMbdupvB9XCy$AO+Yqsxge*4`wL=94cZEcj+9g$q3$p}Rj z`ztP;)y$b(YL;R+2qT9BZu5g$K3&}`d(q$R2hA0WEv2>vVQdu9*R7AeC@90 zri;#Pe&!8V#&~x7)0HEG(0VsA%b<#c=_&^Uf6u{fs^6DQL|KgaGOB^-T*l6 zJ7vB$P_|9SHD~*uxsE9FDdWSC<39M2!a&^)nhr+@Up8f@(U^<%eQ7Ii+oH=~<~73FfekQ*#HAC4F$blO_c$+zIrj7ej{=$-fb4ez~Y+2)8~ zc;#iYPI2PowT?>XbOC*|xg$VR%v~xJ-D~qZArcsWcoRl~fhMw)-g)DNL*_K^;RyKk zcU~XSW$-y^uW?n&I~Dr1+eOH4#~ecmHf{uKDf4j0Dm8yKq(Oo)oFaM06YFCgi8O}q zaOKkvt_j66BT@@(7}l)cUCL`|ub3c;&BkZWDEIF=P|AETVsTiaSuYfnP6#|cVIw{U zFZ5y{%3d&XB_zFvXL&l$!F$fs!2jXHB3&G=EtoNJ$S&*Jl7bJ^;;uJYx>uU_u7eTn z&(QzL2Udjv)wLqb0jE#BKFEw@hQGY8Y|@i!T0ejQKU~@?QMM^9{urn^2^hPqUy6|p zU#0gxxBX!3e6Q>nKtGMIzk1W^u2V8DZki~QMadR)kp3(*J7uIB7ff${_~wg4W*|Gk z0Z^|sky^V=}xK zj-p3C4>bG3P@_x?-oq*G`LZTb(d|AfzuOb!@bQc+E0Jep5c4&9L{acKe({w3hfnl2 zg~MM(@i9RC&BJS&|M;F8PSFL(#0=ADUJbe4>e}5(YoY&YblSK(anyZy&t-EaN7=nF zKZFCK;J7-`0}!TVw>BNc4>K~&{b-%dZj}c#Z%DuC8{d}kuwKF`+&bU0uXrzqHHR0x z5~1ZWN*2lTNRZzA#M&J}W8s}=Q{p|t2Fu9bTVl6COD{B-Iu z$54idbwCdTc*Of|87v`L-|;}S8;7g=tt;~YPh690!5j`86Me3g_^+kQrPY)J>u2=)%&3b_)}Sm{Zt$AH3gnk! z$SVpJ9_rcV4$Uky*evXi``lX2^&7gT#16-q zn&tLwQYt{nxK^_ZN0zan7tjN{M!bK0%ViO$V2=m*C*rZ69ir|_ekS82q}=Dba&jfY zx%~R*n(d*R?>28IZ5}!zHVnM*+=)Z7M_sL-uU#}ZvZVBzywlItfrfSipnk(f#9MPs zo)eL%HY)4_ojM`rbKS1uB!?|w)h}79*0nL$A7jg5B4F$ zai7eJy#7qbq;baCe4uS;tn4o`?U~@3^QRA~q3`->6Glfs;F<0f?JYg8Nwjh2fnKD4{>G5++SFsoI(2~BQwMj2{+cB^+$YO}<1w9hqNq9L z!HkKgFWc#NdUTzA8aH+Rv%zteNF)td*~Z%(k~rqinAlu+?$8KwP3XHU6zfLzGun^0 z_Jdk(?f3hO;aq<$_{Grkf5bi#Mz`)h6w0KUkdOtTI9<&li${9A6cvmDLqum8QxkYJ z>!>2aza7Obb_@<;B$1@)Y@vuWjgZVIY_gk#_yza5CA(p=}x`{}T z+H0f4UDf7B-q+Zg8%72N`xl;fH2NhH+OE$GyqKl*&+eEpgo~0RjEMe#7pM#auDX3S z1ilQ|ux4a<>WMZCqNUDZ5$;^C9VyLgwwEX`=MXR(01*r+bEkuS2a8dbVJDbNdntTO zoFp_#1EKi9Gm4}gZvotLA0kND*?Z`s-aBKcp;(&qwojb+#gR~qwIUmXS;rNp{wKwS zQ7-B&wu%g5N--EXDaE;LgaN+_!7eWV2Q$suI9zwZ^ybbd*N0KSQ!p{YtKGR1hs)D# z9>h^l-c4)L*X~-fIfQo1oiZ$n&KSEl!SmmG&Aevqj(rV-5nwnIxy4h<;E@79<_YH` zb?wAJb7Ie-FcOHQ#h~pH?ivOm>r$kKowO4tQou0o%Qx?BK6KBs&5ajLk7&s{14AnP zXC5lX4E-@*j3$Is1|K?aL?VqS3TM8HBf8cF!k*f^Us=2%Xv#Tb#{^9i;Zy_Rl0n@V z+l`ztfTIA7f)feG5KJ65^nP?fIT2~(){v_RXZlRYGcM8__cqFD=;t6kJsmC1XP)Uu za8+jy4F;5V2BIv3LU{uWlePJ!c3{N?)qjMSI^#v{|ED+X?v*+Cnb%+0+`Vj5Y;2;$ zLcCwaBW%i~W@a5ZV^ha3KE6H#*L;+9apif_BASX3dgD&@q)W6ZVfM!P=Qig}AK$CX z?LKQI=ONn>iJ?wizv!#HCXX_SL_#sxn{SyR(h@0XsT}0B89A8W;rdU#{>q5L(Z6oF z579H~ms2?5tO(88VP*~e^c${hzO{U7q?5xD%AUH)BATpa*T7{w4Xsh`#GlBkUwy~5 zflt|=`q-sQR{GYxha%!9d9A+vyEk7oBvYhr!)NJ-%FX~Z$seQRdv2QFz4Vmt-3WX< zFOAaP3GYs$5bn=W!rAy=zOXpn70oce#ulA69(d0yFQ3``pKq-Q+=1_PloXZT(r2)l zQQ@G@S#u)wN(vz-Q%fU)3>I35Ch41{ZLMUw?x00$O|I;(FaB?9Y8Mt9Qge&KOKC45>6QmW0zXWbEh;MVd&MD9)B*z3Et+K`1Z3~ z0xvl<(g;*{9q+w#)}$D3xG&`#;8|aPYE$4H`H9?MZ2X0MHc)m-*AE5_U_T6h=Bi#h zN~`QXc(|m&S3C!PEF3o&aiH~H;i14h`4a^UFVh$r z{P)B_i~&8~J;v#pU^`zxor_Q~Qu^s0-SFE8E@<-oLw_E>9- zxttE@mOd$U!Pu$96=z8;7wQ8efzo<53AApkM^Sq*uC0inkJ0y&13?pIVI}<3>lQ{` zblo{<;xxgy;~=)*ztVDWkx$7^OJED_^h$Xx> zTn4Pu>*r66R8aWGyF2^a`*9e8L+o{XRbjH1Uo>}^7q4j4AHM0*;`p)kK(q6}mS+0E zK(u>&Z%O@NEs%+yOF-G1;}!=x|E=0Ol+v?4jvl<~66p`O#=V*u z6JF*8k7<`e9PaB?1IYPa$LnYG`pt+ha*wP}%=cWz9H~)i1r%KV7k91h$|Blg02A#b zhcHE_eUiXK^=nT3Fm*z#fozM?tQRCd<++@9teLXI+97@47dEA(WQ=Od4BAnX5_-!8GO8ArU34XJN>uAKpS@spMU!_ulSK2&Eu6ft_ zjrQOnMe9(Uqvy*-0aQ_i?S8fY4ji}ZP_s+@l}_G#*D%&gE5#6dCSXkc$y=6(&VU0Q zIY7OXHqSj$8}yj#*y z1}p0`4FS&o=o^+bfB40xOY~LT(L{7~rkFE#4EW@3#t~_AM+=;le&FbZAvap%_CwC4 zsNeUz`ohzi;$Wmecn#nEilyNtE;9h!o&bkmz2uuajfZy`t~toJ`~TTD%O0XtTcBBu z6@h`!2tC&9Al#&E6YQS$j_k;%== zJD?6TA)sO`+IiI}o{}-+Q9~%^F-bv`-b|ASEyqrVU}l3-x}hsz8?HPagmc3F>?sqQ6LR&A8PmMxl6j#7;z=i`xGUWU zC?@328VQb_IMK+jDVH8_8$rgnKeE0Q&QvfA5lf(eCe(2h3QzjZ2}X}Uy#1->=fAzA zy@2?JzfBR2cx1`{I2vZhzM-`HkDq)j_6I>GnEc~Sc#=_MQPlI$`tZGL8wve$%~)H6 zDdC1jtTR6y=8}>HW6wL6lF8Wm{n|_C zHw(Yc-1oe!QYo!3!nr?k|8o&a?(bTukbU*7*jq2KCZ+M%w(n>#%M^7K&kTU;wjBsh z6~a-b%8)KXZr?Uw%4@LZgtuczeT)}e^XngfG{%&Yt|;vbQ*-#}iDvPfNs(6MhD+v0 z)XMn&{^qOecQkiDJ(OcTpOI-i^6xW!zx(>7jnA1t8WRQ$!hxk+7|2uM!7LwWFW^a1 zdz19K27bJ9(ah%7CpHaB)xhXR7@szIpm})hj=&o!glb200^|>jnFRk1hYYeR%>_dOYhJZn6XCTV`D=a*{Rx*$7rl{;*YJp@r z3_>pa>^*t`ocC?lEFLDXxK18sd>w;Gj!g;{&ijq8`=i^RZfZ{)=TaD|pZDOp!e17n zKslojrHC1Thfw&4AOw=P%YOHdDXG^ehxQyQo{{jYaT1l3%9+JmjErQx_OyzwJFOSaO4(#4pW|D+$% zSt*#z!ybrYd+RleyPkWD!W4bpsqB|jke%pyRN5nOYtaAB<8r?IN%6{%$aqgFSi^M} z&TV`E9`MNV6V2?Y6QZb67PP~|dj{ng-i;vwPfT%=dqhF?5MHNF2%6^{0~TCauNl`* z80Z_OIPz?sVyVnB26t8X99sjIS^o+6j9acMMXS?WvEY!R5Ms1gx8p!VQ3tFO$_v1; z1pmvRg%o zo%r5jxGD|_qSx@Ic&S_~nv@$~e(g& zf-lM_8MGrKPO^=SyN0|!;ah1GrIf>q;#1Hd`&-*Dzi@7>g@@K|k7xysNYW*d4Mqa% zA5EcvNQF1t$Eo2=pdTXlcyHqq6deSfiGR>9%3JRXd|P|^iSGT%cU{}uw|ZN6q8l-} zmf_u3T{J)ThkasgyT@L#CK;`zv-y)do{6>ZT%R=)PmrUPA$w-?hRaWnHtt+9sxOMJ zV<#uYoQf!*XbmPKXi*c!kedI_Mwj#$Gkj$YWby>G*xI*b;1x76y~gvJW#>*0Ck9HW z^yGFpt5W?Kfp0#q@!WQPLE8-@g97*cTke$Wj0nGErSH5ZqW9oS{hiywdG&oS_BVY9V-23VhQuv&A8MUgUy1kN;{0t*!l90lTN9*I zc+(Y&nhm=Sgi_k^i_e+f{NQUZ9!5*?d+Ai=8ts`+xMO<%5dSW{)o4Ip=t-te5;axU z&FYRri~G?h5c!G{AJ2xadwX-^9K$Hw4gY=nOBOYM z^u=eoxX6AnrgT+F|8wlJk8pSmjLy=0@<1okqy6|&fX1t-kkOouadv$F8L|x2ZTn63 z@ji-q%fF3>fOAKZi9~6%fkPgL&Papw+>VA^L{=7gf;awVhySaXHiJn~tJ$QiM9@=U z;CW*cu=>ki1=(7hm=T-h#L3~I(OdT(4rI#%nFR`gpFeFvD2g#xkW>)#6DNbQ0@~L%ERWSYUV2{S@;k7nFse3j3TrHBgkD}%@^l4L z(y`XHU!CVKWWYkp!?+EQ1wx*LB+qLHvW73U587?B2EFatCBf(zH;i-wi()9W$Bt=) z^{^76NM)rUh{X)njO*4JZ~INo;(#{)ZUS@pjJw|PJe{K z?$2b-9Yq9o^P#mnn_vFu!|_~36CQ>2(g=lO|Jdk%KJ!f97;m$S;2Gy&P@)QSf22g2 zJ8fdu+YA$|U=0P5J&76Moqgw1FN6jAr*2suITe5Z^D6_{d%rP#C!*jD0c-pi#H(qt z^1ObD+IVs4!)dU>jQpBVKDE1^HqyN zp+W&&R(cTt6*x@-Y7bbO*6?jBHZ?Cfe`c%`YqSQ>D)*$XQkhSCj1dE)4dl@kxo|eP zcg?oufi*iCRqo{m6?%q2=8bCKQ$$ahAot}!cUa z+v~QBbx9aBR|>hc0R!6$YHz@xImU9*xDaln>mgX?TDnIu&%Wz=OHcp+AOJ~3K~$z$ zIBRmZ*rewLZcrI}`owPGxOF>*CN75#h!ZGHH(|WM^41LX`lD}k$^`Wrk64^8Ssre3zJ{plXO3i@lF*zvvuF0 zNLRwk{a4<3ZA6@G+kYq&PP04AYu=OJm|3Z%9(rzjc)Qyt_Kp+`PjA`NFyf?Qkn)58 z0*WqS2tS&8w0}&~7hX5^5=Q*SWu?dpr4k7KV2pO+&n`ZJBj781i2hq!3Sz~cy?SBu zXJ1+!inT&(5~mb=i5TmRLWM3&pN?I{i)GZrQ&%*+e@tLt|LJxQ922l%1a zUff)=V2GaOJ=O%TzfXL1LnzDOeYg}1VY=ab#)?h5LqR~O$Rbb-MsSREL*TUMW=#C&PEqhK3P?(;kUUBW{jG&Cxu{byo&8uk zXKM4<`c4tN&Oz%3jq$G52!Mlx^a6NKF*iU7XiUS2pz|J#<`fJEj~?vsnpxB^pV@rF}*LO+lbQ6JtV#_M5}N z*)1iKYvF}nfLEgsWULjQOprBiYJ^xTB5%<6REXfOc!4p#>^N8o^Wk}{g*EC)v`8zX z{9oN&($i6Bj6K@d>_1haHj-X*YzTg8puhRi*AE#h@dXrUBK%I_=bFK0{Mi2H+*y+v zDaKwbP>>sI>!Cz{4L19w_BzfohpWPC4QI6Xr7lb4G3@8oiI zpY^PL^Yh+I=MU?qGz)htJ&6osl0^8yrQ_Bk79bBpvc z{bK_UG48-8o@-8TU9l;=_0H4|99qj#u4OC$aAq2_;BfQlV_$hTddLAG!LBxz|K@F1 zh4af7R}NQ-7|*256ykEW!+AdFm32RJ()g~yKimCUzeA3lG|&w08ls2AOj2+k{B6EN znctCNMoK%t^H3h$y@!rA(72W-t+S=ygkR%B>v-zJdR# z)$o}|Hb!JA`bUAEhI%};K7?}hRKd3)^Vx?{0DDYxqzBJTkH8}|-%)qzVc;qTFT6Wh znH<*(hN~M3jp=n=eeAFAes);2gEcPNsZBl-FSbv#>B${Wm9%xigLDjj_IJ`&vUn}e zN$&#x&>u;e8toN!AovPts2FWe4i-9NZ@XXnFYHtJxENj2d4-36yDmiizV&Z<1xDZB z{>+MaEo~qWNN{d4FQ`f|JWF!>+Om27cHdYR|Sy&OPSil2? z5lY1X%N&Ivq4MY7dR16Xg>*n#pR7Dt;v{IK_?AqBBLrfF z6@@?p1R%$&6y}S=!~SEHLf{lIsoR9819^44U)*%@ywkmUA(|qD4Qn8|osgmQ`SVH!ZUr1pRIi03Y48b`)V_5Wl@WzXqyPnz_47_WMYZ)hnEKiJ>V^rTU z%|KW$p-d>n&wcR0@Y2m^9GO<~GGZbyITToo4U`CqQGEKbjonai9%sTwv)VC1v}>?F0y+nMaOGv!bsh7nS=dA@rpe*ar*`tiD8K z1k~CU2F^2*5Mq7)yLY}M3b^sy^(=~YNR~I**?`dT;Rn_=TXr2P;h)W5gPp?6P%~jr`27xNu%D&OFaBgC7e4hyrv-L2=*OZOxK7OPies%br1K%*nClR48zG$nl`y zUEqC;hZQ9>KF2HjY~PO-JSTw#9qc(-7+n2B)Js0RCOk~ua`mFTJib{$jVWa&r7Q*` z=h_9)&rmoQgAZ-QW9xT^Vw&PUafE9z4|@)Eiq~|Wf}vO7h!yr#7talxRTKEmWv6}d zvXj?7K_87h55{<>4u@upG2xC0S!a|{aPVv(pJXlI^Z25N;hkWsV_vP(|5K2Ao|XCE zy!Gk^E`if|8^EWD`|JLsqKL45N=5FDA&z%BC6gR~ZD0`a-WbjV(I;sH$pV__1ZK+G z!CV1Mthw_SrJx!ln5;7aqvri`$Dr5-{iH~E zF(8*-Z7kco>rfaad$l386R@{?fwg)jsFeoNKK<7(tUT@V+S}j0{F5&~6UvFakVhhX z#j7(Fpw5}+c7%`OE1qLsg{dAo#=G@c&?fkfSEe~;;5pe!{iMyr1-TZjSqcDuzxv{H zx`JmGX>kuaoy>U(fq9&NQL_lFMz*FfP)f} zwq?FV$Km56hYFvD&(hB~bW$cT5U$)LM|0V;-d~?Gl8mC_&%V4S=4hb5FO)TlJH~rT zJkRT?z|yLB!B;fJay5G1iN|ecLsQBBfWJqX3yTWgrA%JcT(T4Dwh` z1xFTt(+`SK{FOC>*Ab>Hjho1YhzB;J z%H|DAk5m_-#b3#vedOqgVGb?ESYatS!8yy|f41&967#1%O^&{C^TGG&YeaN)92+K0 z8i-uliYw+!QEW2o;AbWbpxt95h1ZUKM?!w#obh5HbG+LJ(W%bOoNG>;ES_>Dnyo~c z8dGb>Iz16}eojW+5d7qSfA1@!AT%8I>4%?>v@&pF!D*%8d)I7h{@0gRN3L*2TQVek zfPb1iemEbpt167_?(cdTXQ1_6nl3-66gi`X!~uCd0}MKl`^<5krg#QM=NALy1Yj=l zr<5Bp$DOrrsq$?8=!5HeHp{#8+NHB2<(^_>Xo!B1f8_a=`tJ3YpWA%of%RdCws*;= zVDCqlGW_b3lV&3@|9=VN|c!@w)m**6kW z21j#;WR<~Y+up+s86fF`XU>~8DV%!B_?YPe@M7)6l*hl(-`1G4oxOl#l`)ns0lyP_ zaInxT=`nfVOIazcWDTMG?WBE55)b!&amA*PoAO%wt_I^`ZPa{$^B13Uy79Zdk5R0h zg6GRmz0h2-a7O3@G77~#=pG|v>M7tLyn8tAotV^(jwL50z)1;Y3yuk7rx~Zl1r4%S zq-X$mVK_D-;XCq#F))r~`{3{JYR685Q-dfuhEnIvc__3eH_h$?Lq?Z;_DJ81bmU}s z-2HWg?}hVE*X4}V&ew9>xYbbBcF{r9$Kk`2AJI<54$-fyOX2;?{2RS7&73Jx#heTS zx-s8*;BZ$ag704pzBVTxdEmLOBaY`-_hiRRIa2pEe>0{$y=8APq|BTYiuF7=VFJsjMA~qE5MS!CVTLZ4pkn>%fsvwqQVcZ$VlZ>hfKP zMj)Mu0}<+l2}C3bn{~L?x^4Tz^8GV!xx9H|!=CWqhREc)>g7!)1n#a?TSB1;2~HW0 zktmGL^vUCfd5scsws$tE4JH(VC2G&1qs@~W75eKPLWbGrNvPPBT)=w{@}Lx&mGQ4e?)4Qfyfce zx@OH#-dcy0a5EPQUM4qX%h3W80LQUDP#4z9;QBIw2w z6Il~RMDBH$F9>32&Qnm!wpb1oiO{CUZroLbG3S$}QaVt)y#J<4o93oVL&0LMMvEjr z{_uw8(BY0UFAD9HOcusy=fPtkjC&t>zMtB(Iv52x!C5&S55+@=*oQ*4LwFSeK!oip zOnmOUAG-14$O-*#KJ;LRQE^4d(?$rdAw#I@Vi*aIFmO` z2@Q(_)qikKd)^dBY=h?#PWpv)b-ejp14qiU2Sgc+23~#fyk^VpgUx-~{L3GCIQA2dPt0tM)3rMy+Qs>oj%?>1qed3^n@=I1{6Kqx%n0{Qf$lOc%M!-dK0i)Zkd zqX?~)Q~BN*f}DSMpZE5w!!y|Pion|+kq=T5CA{;x^F|CI%WKZ`O@W7jMFS!tu4K+R zYoU<3P3y2$O6NHWh!nW8?iiuW;e~%H46otC8*|zh_n{e-0tudPy~3}Ib(F8?&+W}S zP*zHHz_z=&<5a5xn#aH1QzULL^H!~&=G=KJ`RiTLU zZ2gV<#x%`e-c>w>d3jT;^vW%%z@hbq|DXs3sodnmVBaLJ&OK^RD6R~=LGgMTY$lH@ zQB2m7z8YsZjrTMD7rhy#Xzt{ej)pq_iR@|T?a%XQ6$P~RYbfJbiedleXG+Mo^Z2G% zN2gA8!Ch~0{9?)cJgrAzs@o6 zpoQ|aQcPHzQrOrB7K*;}=(nyIOy2gA#nIlA&+l$F?J4ze94PBe-qKTzNI5)#2TBP7 zA6vur9GOhh_FUqWXfX14&uD(}J7&bZ*^h~bd+}9yPku92F-+Cq&C!b*hd5u17xk6p zw;bo|w;zmg!2{Gh2mbViW%DEbi@jk!tuJ#LYc-5-{gEo*^p>R%8f;{FOi%y%nv67Z z%v#;J^I#Zj_8cn4XwQ&>>~}x6GWaRRs^tr(hw`>=XSBa5f*I4vTOjuvV-I{+=Vi6e z{?jKPA11poKD_$k;a>6Dfd}%V&-!T%TTg(&m7&5~F}`&XKlPH(W4pTC^26 zj^aP5^6U?5D$`@OZ^O4U`tbHlFn0`L<}wws=q8@UI-WgQVORxU<(Z$>yysOH#Xey0 z`K`}95ptFN&Unu_D@Dn9bEo#4=X~bZ`%P(2ru1Ag5xhqq@qMqjpjp0nW*C9cN-3*4 zJ|6K%V^fs#MSo)dmsE5GM`XVgg%ep_M{I3s8mdgz;&Fb zYUiaiMBphzB>UT2M~#(^!-_SlZR0r@g*f50PEs1<-BNHU+@DH$4v}69uI8&C!G?f zn;IA-eyEE0s?YWMEj@=6OEqxXOIleU|5o|kdnbMqz8&OIdlBX?w;-SzBco*Hfx)b`-lw6g2D4a|FY%EW=F z^ja&&cxMY6O)oUoBspx{lfWDJ4Og67kfbo)zEW`K=yA;3VDo*ix}cH39pXjsM+@>} z5eRle^c{v2!^ra2%L@~;AVh^{v?6#pu22@l0e2Dh2z{7_+QX;r)09qeHi%1~Gn|^x z{|g^}C=~DrIOOAPFfP`}O_!e=`P-}sQ0FU4#Kl@KEe7HN94BNxzF|*zO*_^W_#BhI z|t+W6SFzxIj8x{(9*{kcx-V_(njcfWjT?4`$^-x*$tl)MBR?@Iuo z6t@1`@Hh898_FfQ-pnmyjlJO28Ncjro^pg2OgEu&)1D!Nj`k?D0*rYkvLXkA?TMT-Elv_4m}~y3m%Ltz8ip-O597qk2Re6*!IS4=D)x*i9)eOu z-*b&j_V<`l%#VD=yDdR9Ko(La>`Sk;`PuC;Ue>!0dP69M1JMRbT%U{-V;tl)1Z=|` zkD?_jPr@t71JBrP-&zCz^W}ogk0loG895A8J8xTcPe{h_!w1NPE+ zlLP-t6OB@;3L~$+^x$xYRfhFK+1Dq=!|WCQ`ECl=qp^45QeSs1-B$0o%9Dj zvQLxc3f_E7DG;C4C?|3@+IelTkv7T#T-`_756dYy1F}}vnn&{|Z;+Xc~rlu*W z!=y3EHId%k-f7eBQ(@4|9M>X~IEr*AYjn$cDLAO)D~IRIr|*4-rFuCQ^NA3E1wnq^%(UFuTc+g%z0c|ee7`A@+o@|&#s=P;vjyifLk7pk4{2SgS zU!)@5arGh4gzftd2flHSvGR_dWo+$%qsLD-ATVMn2-yoL{1a}u;@n7UG*p0#f_Ho< zyuAw^hsW%_tG71)?1qcF{J8r?3f;GcGgk@Om(K9w5%SZ*QH;mwEF^lx`edM}!9LGx z@;HOzb32NHGLKoalxg^qtv93eUM<0By;VqlsJ?yWf6Pz5R^Qd98Mt-?; zDMS-Lpi@8frsaX(>h|7r<+;uO_~J9c=ZO)s|LhAmmSU9gfim`w?|3HO*}U4TH(q*9 z^VVyZ3`_6wJD+m9fB#S^)NYL&9vqB9+R}QZ z{H%S>gQD;7;=uZZQ;(f25xkDMo}Qt;FD`DHP8RpZ zq-t%YQbI@V(=V>r(nza!!?JT4DND4;Anu!z@sICV70UJWp4N|BpWG6P>9<_7xcR_W z)(mqZf{V_dGdVuep!I?}LLzCO>mvH06H#v-AU`_rF@n9p7F*$V(d@gr17D9*9PC7$9!M4cxDt!+*NsalPwsf zI6qp~QfqwZ{4J(F#d3*^pm~d!A?CWbO)~S9>(M5T??D7dkzO*Fn8+2?t1OjhG;b@i_l#3T)viMf9H8iVB~AXG=&MI$ny!?-Vh5q-v>VoF(Rok20hmJJUCr*wK z<`fF!#!>!9*#5vh&jyoVoOo;jLbShAfXn81K>gQSS9HC<3F>lqQwq6Od%$%R0fe?X z?45GdJ3_paOrFN-&dO}ld#3p&I6S>+Z&;#T^PMkS5@VVDNFwC#81K%BR47D0dMv%& zos<*VPW=t>P#Ef;e<0^WM@=gQT7u`MH8qdre=fyeQjHuN`4s+M*mhfA*AScPEz>p{D8N6Mp0J z1;h4?lo;|kdOr-c_L8~qTIQFbm%1JSbRFiKm*et9GaKzCVD{U2s}&gK89|t3d}>>z z-AhSDKuO@yM{y@-m%W0(8)M9Gg1uOjX5|PXdF7&W!s~RT=j6TK?~@Nd->mHvJ-l%K zl&D(oJm&L?g)>7*l-K(fdYEgaJ}%A3y=&l{W8q=0U<3uwXyB3cJHor2@=TwCS?%B& z_pjM{pn2mJ!&@S_7$3rZC~P{X!(h}D&vu9442F*MbgKK#K=k`xSUJpS%3uK3Qy_y} z(-`5DoE_6geI;;Do;VN+NDPU*b&5FxjPZZ`#A6Y~zz8eHeS(5nKXJ?D-5hFLt6Bw8 zI9XqSr_B@3?~3)v5XU<{7}FS+eIck{e(ua}VWn)}H>UNok8g|ut$Cc6hH>_eqY33D z98=ijcw{@i_Z~bpOxc04DuziRc_vuDbGT5N*V^ytT}{v%3CjYP&t8Aet1fI7&zjWm+Gy8)F#fM! zer~Ko<6V#W&X+D}KJn0oX6^QUfj{L3eb@EHb4b*jxszJ}03ZNKL_t&&Px1_nr*Pb| z`$)58@4==S92e^Y9?BXS@NL1qB{h`a@D?w|ng%i!6SiwZp=(4T$(#A;bGvexhmd*8 z6$=~DZMhG;x3REytPSH#No4F4!J06xKRh)l3;*RCmqj0V+@umJ$L+^%zAO~psjN%d z>^&Gh|I;TQi|3o4o3A)G6iLe$&FCs8J>ML`%M`#AZRShv{R9H452G(Uik2vrqaa(Y zn5KLo(AeX=z`PF{NGWc9ef*|es=@p70+KVq5qU7xu5Z|aP)f> zi5jPW{oNEIb=&zHz9R6aM=89#cH6!PTRnfylu*D}BXAKA87NhdIpU3IjqAT?524mI z%U#cDL2iY=(|m*X~st={}11E=`aQV54`50$n(B_=l)LgL-F$UKKjq-^3*1dAw@BnW8)jH#4Ewu z48O(~KM&98w>;GLlHb+>&v9!9Z-6E+Ds9_`rz!YWk)jl1FPJ-Rn1aOfqzIs(L4(zx zkjJ`K@|x4OQrdkr-O(5G1Md`Zy~rc6Pc7&QCWyp2S<)6dtl!ohK_9J6aPRAuEeJgG zC28F{N^s%$14oV*UVXy2m@mT}M;n|xIEHv0uUI_wNYTFc zJ-aO*)%~WNt`*Zx-~p2lWQ>!-BEh=9a`DXOz>#85%;WOu3_k|~iDVaEq{KDI> z9;Q&uKJgy59{XE`z1nFwdpTBUgPc?YJOxB7ko|x@TL;GEV-IbJ-}X@|LLXSWqpO&{ zePziV95;6UldHQOnOmR75B)_at(W6)1>PQ=G&gMpApZU*Z@#SgozJa^X5jgxv(^UC zk5t6lw;Xb$9R0z{Ba`El)MyFz2Y}Dr3*Ucn<(B5c`O^Y_TeHc_<~<{QqdVH8B+tIp z*WUTe3n6!EFWY2HVtCAP&Fg>U^_MoUxNuJM?DhlA$rFRk?N4nEI+k)hnVJ(sFYj9W zXX;F{aT%9vGv$~zuY=#sb9pTq@0jtihjo%N31i zQ@SU{+?Sp|JLHi1-p@U;@wD_LGp7uX0z{{MzR_U6;s?)}HZh{F&>VR2r{B^N9IFT8 zh0!`Pd8r>&=oclHd`tcuj;+6mUy+9d zrBDhX`6_d#U|#kI;qPU6CgdU*7<>pw-nC%Nq}7;F3I$?@f9lPbXOofBiXe%xCIGYn z!Xh+T#Nip#A#R8q0rGvXyChOI5Ksu@m>Wu}dsc4?!ApA(qx1!d%<~x=cON)SF-<@j zE&!CPwNdc}*+hI}9ug=DdjTaDrm-;m!>^FH%cllRpQP;Gr?&Xj*R+2_QbWx1P@hu4 z+||@{ZX`U^h3t5@1L0Q~3v&ly7!RIX<_?p!Zrg$81NW?nad!Q{z?i6HYJ5`(=pI5t zml;jfBZ0>Yh74>3uu z(_hSn&>dhLLm?NQ3?=6-p_>JrQoAY#NHq6rtdyh0TCZ8IXTfY6Syxh{$t$ohl8!IK9HJS{% z_pxvFomV2}5tDC>c~fRiRze(H+e>lZ>v@E;T3Jfraq!5=sA}(eVevxWcz&1{-rKzS z+rW7Ahqpg7>@dMrRpPcnYS=}cT<{-#Afp3{Q0~144ws^1aP)><`y*N*DjQ!gCm8(1 zon0eIr(uFVM*O8qX9wO{{leZ3lUOderju&IU)EVh@)%=65HCFYCiMf>6Cshd&N@iX zuh8B_So)YSex#M0M_C)>tZ#eQVK#Y*@p=H}6O#nIzkLTN?BuFru<%~MTC+wmUckPf z=&Qqp8Kwy2lrYv!Dp24;kczVS<1GL2M3NhoA{tm@nAOBf_3gX%9~q{YuFv)SJw#T9 zozD3kcCht_NUTFgPBb@PMR~e8=z&XkKAxMlunx?I6raf-`qEp|2-cJ_o8*Z=rm>D58iljUfTV(_y6dQ zr$gbI2cO>D#I-LuZ&@4?63vj7N`_c2@Y~pugPAhZ+_#^r>x2x#oUwq{uQ_i{W1leg zl8e2aW#V(r%n4zv&wlupb9aBIqB^0EatD7_KMNCYzcM=Z0%LDHdEuCw8tlWFeZ?ct z_5o3(MHz4jB&+8g$YzBx9l8Q*0S z%yDfr%(kV@>V?zswLI?t-q!wxpRZgrGd#O;FK0U=egEwP&qbJW?pypIx=J~^ec_BS zn$+WG?ZDT($INL0oW_6$Kh}A-^E`%#q6>cWKB*YMqq)!g!nLMKU(%3ZjTn2jQmKRR zPCn=_g)yUoItq$x!5QsBaGozs*$>}T3MN{Yyjy{_jrRuWA#85yb(Y>|@+J2&M7`(L z7X?k`9q9RZIVqxQ;IS-H0RFX}C?NGejFHuaj2D!i6lQ!<@I@5zaEEmRYCg2OHE_r! z3#Ny$(s8m6#gTn$tO?eP%~!PEfGZz1z>A4;vL}x2D!GuYGk6%kzF_)t$KebPRKNPMN5WBP;=sT#rF6EZcx|u!3-8K@ z&j_0ylazN9a5bO@z2M5lGnxtG#)QGcdGm{3hkH&OE(%&}lRQSLX5S=;5Hu$E0Bgd2 zN~NW1&<`@umOZDUJ`N*GrWVLHeE;1;>l?Y(6W|^WRB(-mgI=&~;f&Z%$wTHf%NLcL zzj%o@z}5JHuD(O3yHutale_nHHWd9wjkC^HGsgHxy^+ReYXG0wju7bOY~a~D52&wm zdTJ>JBz-87?^;#7r{_#9a-sV;PGsTve23R8n?Ef2C6DKKFJ1@EPyQ^ggCEHwaLVef z`@)z}r;{*>_Ws<;%XwVKaWLD+Zw9RdG|Hz2_*LO+cKknp3^JM#UOBd5C-OnEGc^ks zb!|a%8s34yw}#P1Vbu40EBqdLM8C=24pI5;moJTcpxJdy3M+Dk5@42iktE1l^#`}E z42B-_2ZFIVQ51}XV+=BaEX4=mL|72!8B+mxj!0qAhN%|>O-4n-#7#au`gOXu#d3_dy$iD+gLg#N`FE^AnX5nnl?c+->Uc({p!fW)gUz#B_cg!yo|ktORdc5(RsyjL?;X=j88^21zE@w^2%BROufh5g zwxa(9N_c4^d&fTe=*F&6+TRpbfB)6>F*cuneA6)Hj}TW%6_!@a^#8MHW{jI@q8v!T zxEd}RpjkR+S{SjM zlY)WqEP-qF--}jARt^q(P13X$8r6|$mE%`CJ*6@x=+1+^L+P#Gptd9 z;NRTy>@eJv_azRrW*MZcGj$Kzv?8yU^R5l%;`WuB11C{hh$`~jqbE+3`UF$PM|vqV zn8%SZVW45KBYb5FqpKFrYQD01YX}%n5_%@ruQ7#Z2BwV(MV06O^qa4UVm9{KuYLT{ zP7l8X?K%iSexVP(wp@H$z^G`CfQ{7XuWE1fDjm+_=!#o} zTJp%V4l~V+6UI@9q~ccAEu%}FRxF<<99J%1bbyy9dRTj&z4W{}&8jW?BQ;jugIBcY z+H_*`o`Zo)cqw`>bc*6YK2MPN;>Kn3!)sr@a$&=@SCTo?Ho8$f#0&njMy(0PKLEE( zKHlHF<*J3@9mLQ;X^ueET&^8u{xP zBIyPgo`HGdoga~d7r4B|5;ba~Xg-W+l4 z!cU44sNr*uZ45r3myy7ApL=XmS1Ijpxb*KHe6G3vg1Ox_@AydlYkRQQ&-R;a*bB&c zKl+Ac5h-ha)CH>LtX}u@;&mti=@RNplyY`k$3BW@8iO{p*C#%wrt^7yMu6ns_4nW2 z|6K5$d5rua`a=DQHo!57XYw2ab~-A+WuJQF`7WQVCReZYr+M#4-=ZY$S-m}U5qXVB zt+t^d&(+`NxXnvif7Y;J4Ap((PybWBj`}R{O`;(^2#ZGVpoj@BM7^{Xg{K=C8l9HkiP?l?MQ?%0wDj zfwe|977O8lL$PpFc|?=($B?8q?YJ2w^}xMrwnclH(qjL3c)D~thzoS>hzQrdCejI2wT5f*43n7l-IS=WVID0F%7 z77vBNZpi$(vnDm){qob5Z*!-fZo$LsT{vI(?BSU2p`JBe;h2{~4c?DHP=u$UB1DRq zBwzxJBaj!e4Fr+%=Ux*EFm8^Xmx=QK1R+#Lg>vuHgcjgvS z7q)FoFhV>PGvqw)$?w8+%?cqXJz+fWpT2s?0JC9N31vsb2=J6}C$^tzzV{El6au69 zBm@vrQ_1{mAA2m2E+O6AQ;eEN3I@tL#1wN$kP1P+lQY%0@`_XFiqbm)L&1vA6DyyA zeRVIwR*2ZN=!ftj#ub)u4$o^%WmOON>xqame!GvNm zLG~F#dH8sFj^pNw0HUp;keD1EE^$-mKg7fy%eBK>LRS_{pBRF;C9`SwftY8{0fgMSKQJu2koa zoIDf>1>mDpz&XBp%id4Dc3kH8i-pW3=pF1WYu(V^`SjLCp0>7< zscyeQRk1F-w|300j~P08LfG@z5Ttm%$;~To_V>K}g61O+tZTm7@yxn{iFE2;n8cYtsP6`NX%laoEyAIt^5n7SPRI21Ne9M=A^w_B` zZpn3L4()3+%y0d+hs`;GoKgr__sJj`n*?*EE&U~!CKS*2vX>~E4j(-cYaFxB$$}!G zrinaY9$2?MlweW}@d8eTPPXa0>*b>`-FZbBQJ-Hhb5cYj!MA^Y*Xr;dNyP!0hE~;z zN%vALz3tk?&6l2fAsA?apmE@N;YWZ!0@%b<4KMI6!~Ht zgu=r?_Ecm2^hBdj?)YHBd0N$wN=wQ}AN_)hc&29>t?QP~X*TWJ5co0E z%gD>wO8_*_y%g4XfsZ`+T%;{xr*UuY^LlTJ=E4Ipyc&a4w&*iHLN31>EDq33uIzJMv&+8JG;}hO@1p2m}#$2Sb;WAplX(~1EzagFEloXy%Klvp8r zg~6uqj0{M1A=-?#(iahelqh(V#FzG|_Ir6H$+dhY@O*}G{EgyR zy(_)|O@YttO6#41q89Yfa9)9MuzktHE5SPU{>d+1d;Y9OIQQOz$KzcB=X4fa6b`fLGYH4S)TWp(wzEO%wFv<}1%_e!7bA%zIL?%-fEffUaD+ zaC&q8?5V?WbzV5w&+p zCB_7wHc+Tgd>FSHtUGMRfBw{y!5>_D?(~pH4j(y5$J$Iq!x@1iC-`6#*Tc_?Rxv-; zggydFw?k3jy+pM+=G?@iLn&42bb&Qqr;ZG(I_m@t{P@Eg zf|luzaj|9@uM%##a#7&e{M&cid$>MV|CW2T4Gc6Cthq*N9-zMVOnXkj-*>;f>P=TP|K(GUHyqU9 zQ){1Y!}oEHc0HWXw(L3B(|cXJz19$ZgCR?Q6RtUL<}h06xb#r95ihT;zZrJbq-aM0 zl6Ub=qfugJdqrk|hvjuIg>msbwTH4aChvUdP;tawdIs%@ENO$tA+#Ldj)%ba$c;P_ zlmfoP&vKkXgVO0@_}P*FvfkS1dq%$c%x8Y;O_zt>#(tYRp}%RTiX7=~d$1kFnbGu8 zpn5O-I-u8Bw?mzt+$l|DAh`93&CL(Irg!99`?(w|>*%!h`G3g2DgE!BxV2fkb${Rl zMX_tear@aE?kK0xczDo$>%j&WTzG6qNuBK!JyQ9L9)Qe~Mx@Pq4hMb<$4WYm6~)@P|6~6CmQqaTK%Y8#s9RY#kL9;etppnow0N4BT*ALam|n%MQDv6bP(EwPijo z?Bpl=={GHJ?t6B7ghEf6Fg7fzkUQ%RL6pTBi5voU?fECJJMtl?e+(ESk7l39~`A|*_l>#w_Da`TpJ7De%_1BXsD zGpCMgge%s9Dg}siCYUGV=-q$pjhB_M+kHoZuo(*;@d@2@CkKy(LL(VPOmUg(Q>9{k zi@T>-=B>vY=Z~v23Y5n7ku~D}lkxy|3Ljy5Fes^%#(WwYUT?g`DRf;^jt_QoSY5u5 zXZGP>)G>#EaFELBd>37eW2z+%(Y9a?7fhMhJiBdQsfV81UIeV8CnH@82agE@Jaj!1u^d{z zCr=jRlQo$ND{a?nJfC%@Z|_KvSbHqHzD{v+ru*x2@E?Yrm!5B+%an)m0iG#zjRz;C z1cQu_$yh(UrTT_)mfH)v2TSoAZF_fs5tjFG^WMXaA|DJg7%c|1y$6qW8Dt)Dlv(Z2 zHws!fuO2(fAhk%^ugK?K8q$M z=Dq_VoY6r+ETf@y)ENp!2>p!_;RnXNuZ?54}UZSaBkRFNRDZQfd zT0)#6!D#1~pWYIRjqtziY<%+^;5&Dp3^l3;{)IWG@#(Vveqfp zFt|JtMQC`BrE{k=@UpZ`#@M!_bcLfBVQ+h4b9mq}-u#!(Jkj+A=B-AlPRWsi7B5bH zt=F{vTW1vO-d(;;bIk}dZNj+vFuOGX03ZNKL_t($@tn!chwoh%&m+j7*_4C$HlD@k zOs&Anag^dB71L}*xX3h#IR(~)qGvvw{-rH(SPx*Dc$k6COD>1G8wiyHIaLt z-mc4c>GO4EdXi~4HAIWvb8AnkLX zcw|GY&pd7){K)Gs>v~tI)ABvDmxh6o;UXhM;6Yf>*xKWGH@u68z%?D-oM#;c9Q^R` zb2}m}g)}0-KDBQ?+NlBH7+wjjN}TE#ykL*uGx2fOy!{${(XrxLor)>xPkIUBXpz3P zK_qDNlJ?Oiy!=H1P$=8$lv<-zK4fb}c~~iFGUHeoY)K00Dj$}g0 zBiErHA-|M%OTYX0rpuZ~n5aKr=8?riq0-WqhMrtchNYCwnZp=vCkpT@!!W+Hg7 zDzzJ3fXIe_^_Jyz6Fq;E6F6q+n`gpn^aO2CZ12Cl_m#2kj6Zp3{q}>wyUBHK9kn5O zq4sfq8>?PG2CCueC9|7<@rJR@6PtE7M~+H`H6haU)Ypvkn_htDvh=0;BBJAKp_vSn zj3FuGoIF+b-*>+3yl^UUyk7hIV;w%Ses@G%WdF#LBc(7rTMCM7!*|U)kLGxN%|GOC zcoP2ykGvSVz|jy5Q%4BiMgJIefpsr(9*uw(?I$&H+OTQ&>Fck3&1n9HJMAaMkpCf| z9sPqBs{9`9no7c zl{xp+n}-NAA|l@>4-&*=3IVp0qX@&zD_%R4 z+cz&S!mxJaUF2yBIXI6skZuJc#UKHSi&Sty&@b#h7=;1TTP-dc;lJ8~tNjE_V6Gb; zTWtd9I@ipYG(K{)DWH)`AR%^{3cz*pq5r`bRt^(x6$toeue~^OQROq+-xxj$8_c&A zVE*;VdSJOXrh~l6I*L(3DaL~k{fUP+gmv7t7%Gq+!pZzQrj2CGrgw_}>3bR%JX0Ii z7D0oY)@v4pG9*3Vtc`EKx>$5E#}BRB8G$`?X$K>=*Od?-ltzs_^o?pGKh2M3$?z4qep5{-gG)>X%I$!NrwoGM;Wxz1~a ze7yVgiBYKW&Zk#~a7t*SSfo&+sJ6}k0B;i$j?o%M?3Ob?$Ra-xZs!*A#qb{1s+2 zhv-~-MDj{(=dwlDD42PF6P}ZI(KmSZ`CVJ1D5BqtQt(9JrSSt{7~!R)eMUxL?BFc( zZ+_8Ep6rxwaIiJapsh$O;nKdr2zwv@D0c2yy)_im(&_N{f@1*O=-K8dol2$R@t$)g zj}Px5=1+vR!%{&yt6m;yk?t-FM5C=Y~Km%?QwUwHNH$Ho%?MioQ@dS_4*%YqgIyF{1qH zyRHpofV0xX)dEGk&0c?V45$tLVPv>xO>_C88G#FX-6Qw>s?8;TzjZ_T+u1I$kN)=q zYa1zBc&k&^TdPvnF^H#d`Rng~X@rbQb%1_9^TOUPjU@o``o&LD)VTk5KmTO&lea7n z;}8RhJv(#K_~y0C<~PYFiD0;Q&9;~?yUu!@HDyffBZ@#?@#=U`02_O2iyrbL(*Gg5h3}ZQPV?dtMu~|KFr7 zR9(3P`uY>MEDw5Q3>a?ERQrV?r5B_E-?>76i*Hjeg`#WllJXYUX?mt8?pN{1t`l`I zlA*m36(zNtegZ`_A-@ z_wjjX?T(_LKT+nvm>Xu1R?@#P0R6}tmi3IF(tfY?T#ldV>1RK;c;@6-pY@d@Gjd)s1MLA^gzGD1c|O zPgic*8)+UnYcNnShWU&Jtbeb<;pm1=rcH7eMR*PPSbK6Lu!b_fbY3GGv!y}9e_lO<`>T^&8hiK9MHZ`K6U&E{Rg>)LHip4Qc<*{V0mKW zZ~vizA(b&jQ9eB6Yc8E11~o>o4VCBpZL>Fr5jpW;yI*+3Pd@s5MDsa^FMUssl(u;v zG#hJg1fy6l2LrTb?_pl#W6!E+=AiU}d!FsSgYgvER6F%$UwVBj^!oVvT@l$LWtbFQ zwda4{Q}bVW?EbaeM^sGHO>sE7d)2mZOv4*9eu_jCg)dtG{x*#7k8eE90Dr@!1=o9) z2!@2hVHTqn+>>j#jfe93Z?^AmNsnohp!$3-Ku{8d%Rabndn^>fg++tenlOHB6nP`C z@Ywjus&WTmDYHJ?WB{XHd#7gG?QhIc+tbE1m_rKX)m!%lgRc(>a;j0#_VSj!9Wo#8B!3=o_$_0b7J$}8%jYN?J79YtLTJtELsGr@TBc9|I;T7 z1T(4N5yDej*@S>ipE$Hu+V4>RO=uUYa_-ED5ek^+jC~!hXsx{GH5WGj^Y*8^bAsVn zuE52t$<1Hhy{0*SyeMz84Z;*_EGs5YJ%V2i!qF%_F@PfhhLSQ^ZmU6pa~~n8FyFxt z@jmW=dG#zp3Py;+fRf(+5W0(?TgwQR>x9Bfy@An|e~I$yrezBnAug$u*0kqYH+Xcv zLkT;kFV@qWuP))t_K`F*6vz;5Fo>tiSkY600j7eWOR@K>-}TZEHfq5u--$v>zxAK@ zW~|e`%rQ#gUOZ<#6A{FmW@wxABi0T^ZIuFjowP1SjK>Qh2b0!jG=vV<>&ZI!nYSzt zB^#w9<=NIwssL?Z&J)t3_)ovID!hHt(#k}Jh#_Y7oVEPf$2K-nweT>TK5_gorL^{V zsU!M?+?r^yQJJ6g-lYJ;Ks>)mkyNB_km8hwU=fauGkC)l=XSl#vY(^*#<>6b_slAu zJ$-#$;lp_G{!75*uH3w*`FHPl$*|n~_7$ZxpftC3^nJYCqZ11wXNgd1^8Ce+I8JyV ztP(B(Wry_?1YH}Q6ee+!*L-K4SsxT-@J$0oFTFBvRBDLAm5>_u9hm-CL?nOG$xQZ<@hSPM0y~F;rXP2_OtX@~@}TnE_0Q zaT^j}w~wa*PuOxh6j11@K7;k!i!zm*qZrMyEFh@KXeT9Uc4;O28vNYi9CjK?db8yUCr~3Vp?b{fg3KM zG>tcgkH?N_wr0^cZBp{O4&J9Y^f5p9Cg=5^!vAc*(=4M>M!Hu1 zN^6C}il-C>@c+DHRq#XBA3T{}t;W=~`fH5Py1~x=HBV1&+8t|24*IVZ(pX>LFY;y6 zu0tVQ*2n#(P?f48K|WRMxmF%XvBpql9?k>=SRbq6r~dN3?xkpH@~pYv7}Wph_NSv= z>+F*G)0?+myLg!5%eo@;=YF9;WOxt~{qK4>W4!$(OTrUSbO0L7GkYYEueui`?yS#I ze!Ta!7d3zSrPV^|B1YUG4{%B0p zTqx=6qVfGBrh3S>;4Hj}F*aWKF5~FEMC<{InqFzWvJJ9PFF0G#>t6TN z$2dXYYir)aI^y^wH#x@x@~HQ$A@QAK-izlo2Ohj1!27?s+NXK16fN>YKUvkA$wGd~ zx8$|SPv+mZuiVmn_};Zaa~*@TIddpl2wx^0%IjauH(nQ?py;1(bB#87Hvl)tk4?@> z+RAeoUgNl@7Lo`f1ruKkpxX-BDKuz9@F-{4vCbHF7)tRm92FvW{V2R(_4ShvKi{m| zzAtz%#?38zjs$ODUx&OK4nW1ZOY3-fuzB8Q{g+#8*Jm?$(I4b-@gW6X3-~=)k zhZ@D^Yo)aF&T&kr9SHM2-w}noZm50`V=aU5NGWIXUU-o{tIf+?6=|dLswU>YjwrN7 zf3T_#H*{tB^~r}f40{Lb)f&3&?Pu5FFkFz?61+Qu+%JB|HF>D}-Ak4s|KJ01zjQC| z{NQ{sQe;Pc?OU!|6vh&{`t2>wB(=VhflSmdP?Q)?fzH*ug~*w21sINTzZKr7^A7W; zO=#YH^}-0p=N=yxljuL~wczRQZi&vU*hu)ZF}G>g%YM>sK{2;aeI4GxB)%X3(8FiJ@f zi-1+~iz?OSdQ2qiVG5m;1T0Ow1OMe4E{icDxKoY@6I?WVN;jp0XLveA83r*6!>AxD zO$dOjjR%ihzxVkS-D@#1)(0UV$0<)LCh*d!6Pg%W@{gwCBI|n#Iv4>zJW|;>Ls|QVXI@AvLCW_!)oe2BG&n zOfv?U$D8-^HzUsUiDQ~MgA6oH^DFPXHbUM!4`Z92nj!$ozaq33WyoM4a_cS&Y#6(r z{kAKc-~arR;oYkp&(b$*qQGF}qNGT{;;6kBpk-cJNet1<~3-&7VkE> zwFZU}7{mZ%u1Si^-~99wu_m=+oDd`oCq?Cn6DOO+bEfoUOtqt+#-|>AKAug$9WPp= z<7s!!j0w$Kt}4ZYFfEt{$`1@^QP>QI0p&+tzbq8`|Kp2~cZ*sgP|qCY?Rcs?2RRnz zDm^&!SQe-1FV5>>xQKa@3vl8<2-X-)9Am` z#-d7iq9uffv;32Xj>S53+`C%8))!{qdj5@1KAyKk-d0{~sg!oYUQ-1*A&7ct4Xm+J zbkI7pj&8dA^!#ulSmgVTebZ?qa8X}hGk$PP(`Mxp3CxV94~J@j>Ws zFJRnJ{OCjXt*!QFQ97q^*uKx)S<}YQnn@_wz6plWbA^Lp&#l>=yj&S%kI8r0U-19Q z;)#CH=BR*C)?Tea0T%chUecbi6E;bagGrq=Wn8>B??(Xt!dvoEGz=}Sa{G$rJ_h}@L_$D572PBfo>WMklK*AwQ>qzL3i z)(gBJAUN`ThO_j?7=i;GrZxV^L5Uon#Skg4^zG_B>@UU%m%u@UVoD8?RgHb zTxmS~wjS(xhJ>U=`s6(*T+msC)TGuyi`zx>^1XU}ON~G=|DpTVMXH`$dxRuKF4Mqg z4Hzp6aO(K}ZViE0kDa}fQ{Y;{4&}SG=^gL~6tSEmM#DeqYp(fT{qA*4F5zJ?Sd3Ap z2FDF+E8o$zcwISd6?K$4pbg$n8IBCCPw&F|I_t=MrLlGUzU|GP!^fH*zxlH6Ww$=L zsk?6-%oHtW3cwVpAto0*3Q(K24r*He1zbT`h+O)S(b0+pw$D>ZVr2E%w4^Ld{ z63_uMxrItRd?o!CZJ0{MqJ7c}v!cY4=E@Bi&x| zTJr4oD#%pRt$Y@lM9LyUJlV!O@;XLGc!JCO$#D9v>q?Qib=wX^bRyacZzSAu)xt9> zkwqd>I@tsEGsQWll^XEj@@JAS09oXFU$L}jY6s`NFUJ7mt8I#Ip8gd0cyjMz4dd60 zrxE6vZtv+kd~WkH=J8xVa*Wq%(>%I!VX-5zK| z;qs}*|&09W9%dop&hGU*gL8s zb=h98@8UsyI^S!|>)H=eg}nc!W$|2!G@q{(q;rz`CP8W6CG^oX$Qbx$69 z-;I}qgM_#~GF2Pk5(b`@Fh* zljrn7dL6P{;#g;`MGE3JonjD2$7)~{8c8W#UpJcHWWPExwZ7-trL%)3JBAOVe?&w4 z2I$MzlIo^&`#-9$-V)G1sxFScGniY*ZrV*h+`Ru!?+MRbh#P~z3+di9+ndWPgs){q z*)n8tq$f%5@LdX?96b@j_ej@`blk=z__K@&c}*aI`TM8;>i%ZOz9T`XF|om*L_sV$ zABV2b`(Dg9=Jp-eofo+pF#^Glb(k%Rwaksg0xJ~be?RbCFsI2Z6Fic^!b~`haO)%Q zrN4jZxd{7XeeM;8|9!8#Adryp$NaJMB*W)DRI9eJd6+;V$vip-*&R4?D)Mw+xnx!_ z>+N@ce&>O>=MQgtDv(K@7v9GTiP2uay`%-velMUb`C2m))r3{skOan*g^wjzQ82R0-i@K5b;o#&(-_u7iLW{4W0rFHs1&!?5j8o$C*#L;>@#A zus>7abKsHnJ7S(PPu??|_jZ^|YrZI@a{dXFyf}`xDvD#I%?-ni@df@Sh*O5bd!b|u zuiKKMh7krWus1PVKnNw`c;~|T zQ(}MDBkH|G?-2Cqs~N&lY4zq#@efSr3;Pa*l7WzKeC!2tgQlSm*Ih6t?!##Fbc4I+ z&zKlOFWy18E@dU}e(6SzMvmiS&8d?Gj}>HO4ULUlr<2A}&KD1FZ{y8iKDXo z`oxA%Fk{r=Nb?L8F(^33TPkr=J{!Lf9(kQ8gTOT%?{I4f18-kQMYaEMNl)Xrxk{X+ zt?Mo<>(<)K^^(uXedW8o63mrj<_;b7p71U2d2+hHec&+fVGaMz=T;0$9Ux7IqIG8j z`7A&2)eW&G3H{bSnvHfCm*;mJ48ykmr?FO_8wuu$AYKFdXx@yMcSr?9;#$}0TU{`$zW>GihWiM2 znM*tGWn>)*5rgWU9FFu0g`VIeuUs-O(hXU2=m{EzGqFEjdC~l4 z#im_dB^umxYOp`>sQqNDe)OhGo4Z$S52f|VMhc>#o@v86PCg{hP|D%S>f=`|o*7<~ z6#pk8g5qSecKg2Y+D!h;{d1=d&&Q4D^d9>Q;J5) z&%e5RO%#GeyFrX8+Sa)XzHrPsR)AUr6nerF{)6|Fy!QD3Jo%Wu;wXc*=DzAn0!opw z=SV1m;Uar7jcwTm-2THim8b-fQ;Z46PL}A%uy17)X9pL<8@aB?Z4(=5-1?%HjyB;F z4p(i~4qgQNj+|L;EIjj$;2Y3H5qRpUP&|$VA_VFP4A)r`XxKrQ7001BWNkl)#pSVoa=4^VJ6U^|mL66v+(YoH*LfND9rt z@8nR9TV!AFrH%#MYW^szIlOR4sH1op%X{%~pMPRw&}8*IL|Raq_ln5jIq7|$*swe1 zPz@IQf-+R(i#e9gjUqmQQw7Mio5$1HE!Sfn|3-b=8?{neyLc)*-Dv3P+4>=hs11*A z+%?RhBhOoBXa@OVBtYY8rL^m;68oG3kbUkq9Sf(#Iuaq<{K$jrntNAoi?N1}p5J+> zd2Id8a3m0=_+mf-=q{fo72>Q(;lPJxfZF-&#kAY&S`ny^ZP*=7CiA9GjJ1pI)R60? zKIXSI*9-83y#PLfXM%oo9meozrBti-sRT_wYd}LujkLuS9mjf%?HNOU{PBQis;-`QON6 z5!pX!T(f%1zF41><~J@|5NUXx*|I01w&lQo`?cqNP0`KY$i9BPZS`JxUvI1bAKfo3 zgvGpXOcAmusxe^cF+(8a%|QWiCSaX;Wc{vSqzNVj8;n>Ylr!C1pA+W&#g&_*uprC& zlG#(DIAIbPLOB8}^bj+TX|?GwKZH6PWkx1xtQ&rjzcFjZg?B^<=3`Vj9|$Gh%*jL6 zItskIpV<}+HHLx`6vK^KSIB2;hiMIdtgkNwS;Fiu{q0x7qqJAphhsk8t8F3Gu3}vu z_{!Q4{vZd>#oQ(M?Kt6}7Z_Je1p&DRN9BBJTZn`u)Y~{SOW_ka#a}p8^ zF)$RYp?K`%V#O+q9V?Z=q)@bt9~cvY1cX`(CM@{!OD6bDSj3=s`-|pHYuHod;G{Ehh9@hmK=DD5ld>I^M67BbIWT7la~CglUE)>tEXx652>*o~iJD zdkz&QPF}WN0+=u=sU@;_UouEk49Ab3Y$i__z6Rw#?*(!*{=e+q37j2QbtnGY-P-qk zvo>#%<$VPV*nnA#0RsjSLP#c)WJ30YA(P1rlZ1q`6ppQHj*LikOa(X3}!L5 zvGKl0wj@i|zFE6kYAvn)f4;ZgdGEgJ*HTL|B+TS|KHc?Ry{cPPx9&anoZs1uY1Vei z39tbX^a&Wpj(f(l5N`X&fS5!23n#PO7rx-zfm zs>qu&q5t)jCv&P7_l+kKyA<>Tq?ROwi6{tbT34fh7B6Ve{MKvF&hz8zE4F5@IJ|?x z3rvoFaPPoa<0*q`4FeM8Nf>m{6=%1K3CUB(v&`Jc`OzBd<3sCqHpip><%wsE{`J?N z%3iYOoxU{&0mFDS!;*Pbjsw>Wu*_$?&HdfCdPg=w0xKEmJcH+a<5de2VO$&XpaZ6& zNC^)zeBXSHr#apw-$WG(&4U*&R&=lLSbtvU&4KdzfJokF!ut;QR=}e7UqM*LLRXqQ zc})7@7oD5Og79iefMWn}m8WmJf-dLk2k%^!*JXX>o#5B_k@fnQUwu)Im~b4!Cc|Ic zKTj~;_M$3`Mc{mvA9U70SB_o@#a~}ZucDs-f8I$opDMoLx{-3myD`?hGi1D*ZyER> zGcrG$>zRMCaBGLB5DW&od|FwK{_i^1t?124vSN zBR!0_670@TP(4QWz}DZ^5M$bX8&`NA`2at!0~B$6TYOSDC=gzg_!izw1~SLJ>&B(> z`VlfDX9ETxMZw4wj)`#}a*TQ9UdVg#Ox3%rTBM-M>;7x6yR^KeL7&&}RP&-rnI?*@ zf8cmtM>B65baYDdr%lLpD~v^6^2l`=R`FSpTbvdU*|-Fy?Gx%Rg2np*Qhec|4OM>d zel-ayV++7dq<|djC5`VOQ-w_984!>8aU*-O;m^AR7KE2$0O;EB4o{VS`i3RxYb&fV(bEapb?eh6j%wB-VNg&QTL0o}ys624KluE` z*???*;7QswU$rHluW`&sP4*;0=JD2$CGY{DfAmw_GkLZIrS1Rp1M8Yzhu*22-qu$h ze#YgGyy%=9$;H5K4*Ja;H`mB>z-;652yndNx#?BfXTFJg;Jp~2v}<|>^Tc=`4k#V> z-)U>E^1zO?66L$zeW3CfF6DLiC2x930DH3Ld9$nZ6+9BFbG*rFT$*>tdk}#Ze8gP9 z8_R0|r+JE#&yjvVbo5w`APhCc+JkonjclZ)@SPXwKPWKU0A(4^U(-le=l3^XvnZz; zc<`BB`R~bQe(uC4lI?OI7fDKX6qpbO^|{zIs2MKxyOm3q^5OPAh6g zi43UM+i+i;lA&wp)*W-g9_{OR{ z2FMY^A(4aSm~zhMT?f-QpW2$Emrv9Nc@?1Q`9u*66b|NxmK`Z@Kv^ZRukh+C8utc*x+=dhp&oT{;ogGgO=Au(hou#x@Zi z;S75C=s=eBMeCKhqBGAt+Mf*{CEt`*%sNMYe0<}s#FJZ}SzFBsGUhMec+nu|welXG z>(AFOos;|paY`!Fh)R+2(R)^ROhXdqd_QbYUE=#!{8X#ARZPV#T0jv2*Ix(`7MNC$ z%;1ON6;c)g;OW8h;x}%&JWuAqXSDY4METMqo3h6grlx>FRMjM;_)d9cH71e#>FvvN zY6}d=VsBDo{&H_ouB}gN(@8NdKx=<+B#bgLylqmx5 z<~^0QiV{A0B+rT}Ul@g7rSJdwg~zk!c~vF51$M_BVrJoa9s7W?1oy5+AiR6CFuQJi zCNR1t>lDxr!!C)(2rF(Ux?_3EdhU0Mkmm^@l)x!WXE;Q_;VpFds8FLS7b3yRqScAQ zYt?giB4CKI7-Y0cui!jj0!I7drL!_96UPv!{U($a=@+k)H>ccD0HYw!_KUB&aL5!N z1iasR^A)+6qQy81oQ(zNrpxB%6f$L{T-VCiCfH9LopT>WUck6!xjlc*#0(hy$!*Ik zL5t^GV@~Ex))cBja1@^sl-3Sl@TC{dX?mHPN0G86uEV?DqEV7Lq`1VpU*8fuK4)i*GZ*6r*~ zzyIk+(*Jn-HPsw1ZtPR{t<4_)`Y}UVls_PALt?7usV6*H+zA*Zi7r^NxxX5B4Q0#k z?&bJEmHGy-zHr)v?Ck-#weSLo%ry}Uj0Vz%5Oi;QbW^i=`Pk|nYdlF=uqi?poHZrY z6^%HVFolm1N-UP%4G)K`#0#vw&&>-ok0)b3eJBYdHol$bwp9p7J0R?eM3! zEzjNuEsv|NJon>yjb(G}Ki_<51{WV(`)n3u6GVhz2m%E5B$iDBQjdtlM>g7O9%J0S1|A;hYvma3IV5N1n5<}?2$&{U7>_YV zCpnNT$kt9QI?D}l13ze7Uz_tpY;d8 z&zk3V@5fxK*6*r#pEL@*jjh+B0Hpj;B#~zk(hgaeS1a(_xK=pz?3okuJhL8-9aYT@ zdHK;7AKjR4dt_6SS5usSx5@3&W{#Dfj55y4R>{3D};@p2lF zrZH%Z&g*nTcD`i(v@D-%gF(wFa~kWceV@2@P5SjWUY=#R@;VVQ7yCpfg+Vd%XR`H( z?({!@_<30o{`CDVuVCl!;H*1!U zWoJ!o<~&_EeSCHp`QV+avn(}s)W~dTlpanxj(b*bPb0g#)1kg&SsubC%{gn0?T=r# zq*>s-yiemnga6ZfvoA?zGq z$c?;B$#f{z^EkfT@(`9Lhc_}ZFY(e6l(t`=oC%e=x65{b@Ij&-O9PAjoNJ`^0WEZi5UMFr;)WN^hUha7&sYl*{RLp?6dBZ zE6C3!P1}27$G-B|`e%SIxty#DxUcK;qEtm+#TbqA>70S3#6Dng85Lu$8poB6m9pW5 z{KuI}bROd`df7#Dv#b;Clw0xx-Wi~D{DyaMSPWoVzmOZH;bb^iI(JfAYMhYEcq<*4-7*u9s$EJegd=bex*?r!MbSXg!I6gXLCds*$)3w z55yQA(bJQb&Yjq1TqBFaeIh#|UKt+RWQ{YS^X5*@vP#xPiiY&@P$a7Xqt}z0_GJ0w zoS75T^Uj%`{?}LA0ek0jcq)m^MK&w%8!kkymbO#sIqQyo^H&oFvL+8(#lE=P9+z2`vMHYGa$>}!$>^q**VsvVc$4p(B zE06PgCn7ut@d6W~T#|4+sS<9uXkISN62Aef^c^q9U-`&=ISmWN(OCDMdDQ*VjTdE$ zDNmCC+MUniNqPuM+@E~$@$@@yxhk5?fHW@y(GmL4IL}H`HbV7eg;BZ^px-mk9>{&A z@PW`z=_Fi>5GoB3pu8I6RjyCKOw3OLmG{JBPmu(DT!NJ%F^r*~EehiZHzd^a@~!tZ zMXD`T&Hd}r&%NS;wt@`$>jPh3neW>J1Eh^%0AVKJQUf@b)+W3&706;ZS-0c940yTc z7}n+(zA$jK;yb{v31ccS1~@CSfYK)Sr6UM)K^c7Kku$0Cn&)AT^zTo|C(n|wg=&vo#rP@eo00I1I;xKQn{q5}dTqo9j`czFHp96rym63Rt!nf#Fw=FhwN zf4=ZoHke8WWt|43^D>(;sby@jo_ZGV&OE`tw9UYS2-kHY)O&ZO2M+JUAAaGv>F!nA zbA14OTU*t7)swwpto=ONzwzXjrneo>NBz%;5rqG&ZWIpoTlDR*{{GB=<;z+<=JPc>0 z6!U6+VC~Kf5c8Z0eh6fgx7#sN5x9;Bf|z#!aJUCD7qV7|>`SIHcTin%%R2{BlKuqP z!wdL^3+Lpufno2%cRih?VX__=Yo3pnCb^Db(R^eaBfA)1fIxLH?y^=_<- zrA(L)sQr7@CG*n8oxRQXMb6L%j0ZPdI4`FgThmBG$EYd_u1+!5-9sISbu6calJG}{ zkOtexKUMFza)#NI!<1WVit#7~U!9B*#dJ9pFj$>Cb7C$6*U9))KR3QBb{^hr($eq_ zHYV{Fya2!(#?d&(IOX-nKxVF!g?aal9W_`c&2>l${mMx8tmL2|G7rA(y4E@Tkbx85 zHs{KD7w7QIeQYBq72fdUI)-w)&k`{F>#sf4G$d+=L4geForO$ff7BBp|C2qhJ%46L zP};G)u<@BEHt)&n&7?6SvH|d9&P)MJ;SVK+CQANr{h)uke_iI`9Fn@bst6$ZOXOOX zBlbk}W7nUTi=Q%9PZ~Eekt<6;1<&a_dNil&qc;KKPL-mdZv8y^Y)}|-<9lrDAY|-6 z({05$&$N~5{GWX3()5+bHf4hnAs@ee)y4C&Bhzzpg~6?eGNIShX{WWP0D$Yiclyro zW?c)9WXC-J+*x_fN2(May>Ga3QJ&9x4jyji@+GU)=b`na1f}De6UL6reAv4-zN@0} zRoWi>8SSg3dW>^z8BF7TVn2NiZvg5BDd<_I4Pi94mWkd&b;C=X7hcNoy}ngjrB2qF z8!w&Tl*8)3!>Kyap`$IslPFd?mi1}!an4nh?0CVvYVP7e(v1ea?<-GcnLfUgO*|i` z2}Beu;Rd>{Jy&n5TyUOY+rHCozI z`~L6CZ+98i}mXn{?J zbiDfi{moUm;6?>RS6)cL1TnJ`m^Oi``|HYiy*6Y;rp5fnG?=GKvPb)OW{+7_i_e5C z3OpqVaTNFxa8BSzhSyaFsjz#%N&vwB9LFu@bCH+Fj9H`NsAGSCIy(FhXAZRvw4vM6g!bN0SD$vSjGXN zAG&LG&WXjdCX_Yf=Z#k{>gf4aw#}>YOOI~K)?_&vLm~H^!AQN|38Q#eK&U~Vx|(_V|=*Is z8DeX^#Tblzw(|6sm4bO(1Ji>Dvb*~zzSW#=GD$hD3m&3S5r9N0hfm6DzrU~Y#LHk( zO2;!`7a%Z0VSi3P6rr1yOJ58v2!dY1}B0Ud@zLkX{) z#yg|c?8##Wr82s7;ne0@SEi8j4Ehhqc*P>cm|DWNd(%g;M(7KC9_SdkC-rdkCIlmLvWu)=)r;z3V&*UyUa@x*o?1YiKu`@i~R22hJBkuSrT1Q6zZ z0{9FsA=mt)&p(#EyyJe1#aCQ>Rvw>coh0Pbm|_suc3wY3CXKG@7ytrF4?BRG9GO$b zj!b{MyXA>!EV%!7ynJc)#$gZ$GT*F)Ln-+spNw;CpV=|kFtF>uk#z2yiRsv8M%K#6 znAeTtRg?n{#k8j?1}d+4#!egQU_4uUpNb?IXbZQ**PhG}k(T4FTHOzzSb~f@;8_!j~eaD!~2pIjz`_?u=PW!Ce#vrgQ*fDv-oME^sjg}P6 zeTSZEI8*Srt@YjGkJXa)z@B<{_wMprcpb4 zGcOHUl8lJITCqe01;x18yr(yI8NEt2`Ia_Piw-|N78c89dEe1}w?E ztl@E9$;{(3p9p7#Xh+{#TdX@F)A|j7t4KJ52H;jSh&db>d9x=Z=>dnL5-7yW$X&*Pv`ma-ol;5|&&HCz zMrHy}{W%AZB)OaCOdgYS`&P!wzC6#WXweER;$h0bqzxna(+{q%imo>RY<=;IH(of@ zE$96JoD#%2;lU|o<6il!alE3s_5Jv?zn{8c=^#geIIrAVOF9`W%46!ErQgCM%Q5O0 zG7XCL%xC=+Kpj5oT(YwHPd5|pU!P04k=j4JCisn1f^ss8bX7+KKl;v^o!lDoS<+y< z>!yp_*4|oknYVrN-nHq;M&5Jtpi^$|GJVfba+PShr?ksTLSZ{3@DLixMTkFMX@1g7ao42+DF zC0#m%b7LGM@~ac!d*en`#-i^$y(8D*A#-yu39PAp-i)HIKnbvtT>`yQo1jkCXpSaz zdVIsKCNS+cPEFF@FfgJCV->ib-!xd&&p(y#`mfY8PvzV=R>}=Be%%mp{+!9#5u=Qh zbPk*;BFKC%bscjt0tZU@a^Z}j>n6l~d^fk}&7M4HjL`>0NPj=miIJc0vnPAwHH-2> z`s9<9f?0(Ob>{__&=W$o#}eisguXI>wFGSBQ9{x%+_M{1J7HW!7JXa zh+vSt1$&Ms7A|!EnjNQ|F9tX#90{O!{P^)4a>hfI7ZJfRD|q=T)jx`iMD8r_tG69U zk8IeP1*r~z-fAslBtBW`OQ^ti+Yu7u& zbb0j2Um7HA^ao#XcJmrM%g1NstI7q4@lpWd-r_vlGS?*`Sg0x>Lq1g=-JJw!mPddd zCCLJfF%gJ;Ccx;2^Jh(NhRu}M8|w1~b0_B80z91=_3B30^s<5}}vt2$B491Qco5MhXZM-L0=l}AnPc^}I z=b9sG@tJ$^EOHhyns~1S9SWZ|2`Q?Jqq}PPvFwwhtX?^t@~SZ4XCGcqgKPq8Avk-c z7(cFGE~2{+9L`Fx{zk?H<@?89GFaf1U(P+&Me*=($_>byRsdJ_;VZxk{T%&G-_cHh zfZm(@ril0GqL~xZ?|u5={N4z`!Dk4i<45*nkOYvyqly9>{R+Uk{qapXXF6~R$>H{? zMs8>VCZ6U#C^2$BD#FGX)`|G`73owFj#P|mU8RK}HTyn3ud_jJ9_?`EoPkpAG1bygRPv>J88h-m9AI@{xTBc3+ zMOdd`D~A8g=)d{aoO9I$8@Ak0XX^{@fZfS{6`9Mv1a=XU&<+f z@{!d)0&p0hjrCaL;t-yB4B+I4yH;+`N|(73*K?e4PyP@Ah&!_Vl!V$APrB5v7Hv=wDu3A}ypFF`i>gJ0Fh-erF83H+-n{nv-tB zU(h|PQufXU)=4n|808sMwU=_SQ8}_psz;WTztunEH;ZRY%*L8;KfNR8mffq|qGaUC z%U2Hbf$aVEmB%+_C0B}v>OR7Uvr6rC^OYP+I*+5cuW}m(*>a!HWQ{rj8r6R=Kh3il zlg6aW7f#FTIJqRq+QrK(fGh^L0}Y#O$>*CzY%kpJ7SuS|-pNp0diOKD?l02uT3 zkhFf?$vSO3B6G*Q@jLHXQ7xjkgf5%!-(Iyn7fu8a8;`~S2L~SXQdhW-Z#`8-G4lTI zL_D2K!mIsBIU82a2p8?uf-cA)d%O?Z;DZpOOx*W3hBk)x&7 zVx%S~8AH5$r9UW%T=weA=BID1+?sOn!||iiOD~+;jI1b+?ev+h@)uuyvh64H`Bz?Z zaofJ<`U~)pXKYE2tlyb_^|h^92iE+$+~N3>@nK_LABU{Ic~|FjKvJ^R4I2!E=HRpm zW7?t);y&qkc!2R*_ZFVp$hYnH8RyMPcP!u1G_1t=moFT;2rjwvmv6i52UmLys+H-Re z-3>c?vp&P2@%2|M$i@)kZi~7PJqnLcY8oKo8D#Z;(5)rB+mI> zZ)H}n&NFJ!ugLSmO;7v1zSmQ^UQ6OSm9zep9!p6Dvi#L=|6*GDW51Pw((+cr8{;qU zc&Zt0>ZRt~h!e-;;|U5DEQ?f-31>n6^5dJDA-aAeB*`&kfiFc}i@o!C+Yrnt;ud@0 zNy&r>h`;}XS7-0bRonKao%^cXfMp1v1(8K_I9|AMzqa)-eBp0hsq)%PsZx3)k0%dO z0%d?sLVJ0>-2V8u^z^pnRbRBaBR`XH2qE=HjtNn13AAGxd8x~LseiWc09S>f|I7_b z)5>l8b7-Dw#wygC_yJ;3hDuO6+PZ&DE8i1u4~vw)lC(#W24wg4^`{vVN3}6)UTdYG z|D83@X2n85NPS`)@NzxSSA{KWn?gotM;KMu)L9Epg$QQAUP<+VJ@Re6c*(5vH(zg0 zJE6^I_x0uQCZ91#ulsJw8=>@3^@c7z8uZY_i8`}+n`?~(p2c>dAn9?OM}>N$jS zc(y`Mabn{kC@26WjF^zjXicH~>f@V}Ji`(8-8j6t5sF(DQQ#Tw9hz72b3MF%XI7S6 zFO*p)^E&6L1U!G?l3BUhY_ri5eE6AHT#!@f5Fi01010ys)B#KuE?ngnmrkVeXa^)Z&?&=qwyo3-QLGqQEC%_s2LTMFuH2wqFrug7b zycR-GjO#4l+)9ICACFhXg?NNYQAD5$PZz&O?({exUo2_j=n;7?+vnb0m-SXI=$BnM zJ1v_xEy+J#lJVrd>kr4{Cx)Yb(^tSz#W2m|IsrCIlIs;8qf8KP0m%#^lx70m`Ew^{ zL8Lq80KAe92AcxMG1raYeICYAPO|KZA*bWP$n00>*~tI2>~oSdynUIA#b>PQQEZUV4E_CM_e_8=OjapwYDTi zjEy@FWpI$}gvSDIFI_l28}fJ};ln(lN|L7NXfL24cxKg;VW@Mgv=DV-NWboa*=gI} zEouJLikA}q7R(r*e&JQi>W4SKGiG}B0IKmXIo(Bn;ImR%2y6Y?ayilava#l^&sU6Y zL?VMCvV?hrI68>7*_0}trtKhoV9a`Ua!sV^iQjFd$S5hsI6mq;yvuX_!1EX7f|%v8 z4AI6Ruh~2oMNZ^A)|GLLY-Ju)5q{)b(TLSO0S2@2C$G6xUUCNiNO@6?XV+{zZU)^$ z*~OpbdB)3(X#kP?*Eg=^xC-ZN7jLaqPJ+CeG z8061qm_KLcq--$esDPgkpCJfar(!x98>b8@`7Bh9OGfU6Fxg5aa&pb7nN79j!rMdtytI z_dB0s%21%R-;E0t;3++#CxeT@MWtujn=K%UiN?BO9M#8R$-j-sIO0@zYQCA1{F^Pp zqXXd$WTS{UGe6BkGBZOuCz>)HNQWj)y371}bluLp4%26%mII#UeFOQVPGYzqI{{lt zBCp`JbWnY14IsN-IIlh8pgbo&)2)pKM$1?`;x}8*YOKXOvd?GdJ+y9D6QE<{l0&wn zPV?mHZF_TMD7mc@88ZycJUpd@^lX4vN7KbPi6|HyV7&hNhi|n&K;F!vg!CQH;*xmx z`zzPU^7j*cE&%}FKp?;NIF2I`<5#Cb>T%4OZuN1UIE!(#h;b(hjt6^5*2+%r(=q11 z)KcUoB)uEiXsCBm?)TpD&C=P0@l*{1q@eXTR&2?9mwdTk+Jv^0FnDo2T^J*FHpJ{Z zoOv@N!f^Uj$_mde`qtWg>x1i?Yg|r~c$gkgIUD7h!@H)noDS_Xw}+C;iBa3x;3gJcyjt$=QXh)ydPCY}AR! zkpCJ%Y4;itCB|jQL-7o}qe~Cz*dJRCC@)zyr&-5Jzg)R!dLEB;Jvz>*Z|izM{r5Bd zju%JmEwWw1u6(A={)?aAF3J9j-_CP%1N~KzmU?R}{k)+3wTRbme+CFFTm%$biY|b{ zU=HP+5LYK2poAU*9VI;kC(6M2b0?*rxPD1i3hL*@_bKB>rm#G#1PmCeQ{5_9|Goa* z;#d#E&qDIZp5}sj_KXRcDFNk$z?I}&R%6WW*69|7v1mDl!Xs~QY27Q&C17`=vS_2pId9syoaeJVcF~N9xj$L4k=%L& z4ewKWg%XsG^X^);BNrKp@3s_B%$E@5y;6O(5uz|FoN%yHA^j-FTuLn^t8Q_P_B>xn zt}WvWfiVEqUwGxRtT6rK{m-Olwjaofg)zw^#2BX70_*C)H062TV;fU=?Yj;yP@ols zFvznK)a2}yUkfnPY0)k0IdN*fM@k|ucV3o4^eHOkxaLtjdBn&(hZK(+E_pAG2jbrM zbmak@&vnCH1kV1ntEuExK!AJPu=7BgGHzNHHXI7P7Qz(oR2wX*q7D3w9@UfHal?}I znfuqbfg3seTW)7Bv;j2a4+@nW?a$%6gkSGW4%WKBaR1sJS)dihMmaBupyl~WVz`LH zQn5G^+zCN{{k4_3e_77Gdtk5k3zWbUzPfyK4y%^(gNFzK+=3nCec%PgA;lE1NLUet{o?-diIOId2U1j^G*6*F{^AqOv{R1z+RUx#T{pFI^HYB6 zU?-q~$B(?B-Cf-|LgVHuPMjV@pHsj+Z%Kr8`S84d`5t+>m9%Qx{`AUAS~-CMBNSPD z1VFz2Xm6e??lbt4{tRW`xT(M;1tz;w?~x-JponV__CJ2_Gfkt1{&FwXcpS*#lzY35 zrZqdN0xr^b1cfKI;iZ?2Re##Z(`8^F$u0lx*Iyo|47s%~0x&-R@P?dH#{KW^J)CwO zsM^-wg!%y4G(>6poi|^Zo_V%6^GG@OBh3mPNik!zMvMwl2FagY5}wQ`XoiH0!8@Ez zU4mJRgXS(C(uuSoG63%~CQGAsr{fjF@%NFtpKkII#{qK~L52eTN=4Lo(bxD2V1vQU zIQy~d&d=BXkFUNcDFW%f%q9Kj*iHm+aPCuE_vCjdJ%-|W#z)YIfv)s(uUOWWI^#!P zbWY{{_T<+5>>+!Q7qllT0N*BG02YJvQ=Rac0OjSa#J3htpO{{A#bBV`xz>~qd{d6e z>YehWyXmq8*+3QJ-*Lb|zmc)p zJWCd+qFN?oBko73gbW*_WC-|TfYPq# zx`22!1bA@?F}ANO=ee$GkC+GUSAkmV!N~}a{`ME1lkQ%er3gf@@EuBcemmrMm zi>A12;o0d$SI^FofPa4b^5&^Ltc`DL9ddu3say}eUk0|imv*u2Hrw+d`oC>wBTsvM zpYMzyLGp0_d?RJs`|o%vEid#O_o450_4eiW^7d=bNq6SF>Xlc1$N}!lJ2bAyMCK1q z{Ib>w4{BqpB{{?$R9jx-X>|20@{E^;T4T;VNEeAk+ssv(Yf20n^eoh$yh;8z<76oG3zz z998P~92q|P?KSC^s}`j?08b`9Z%!*R;_j8(+dyge&xwsAkhdBIt;x^N?mw365h$w6 zz=hX8zG+YTtMIRIRpy-_@_fD7s{Bt1s)nyFV%BIQdzd$&=P~az-bj+lxvF*@eT+ zO_!hf^L4t`_hR2r&v~+IzxkTQ*-0)(dXMfoA&_0RMV9*DolobvWIv$a95~V3DbGKh z&qKV*(`o8o{G9(If$8IY2iijXqn9sRI5jIv@!d8ob|IJ%YA_rMcxF6`$?X^J+>-$t z={CYz?|c9OaM_~Iqckg--QDR2Ua+{Cj2+YG6-2S8s8ZAj2X%^K67~IcpjRkh-FBk7@ZcDVp&Z=r`p|u;KOIjCr%g=f%$S%Jo&fg%)tAhl zoIZ8m;9N;^Mo${kS<&aoS10|Z_*w{!IzS(Lk3wdiD`o@aBuMj0E(y43fiB5Apr8f8 z!DdYyour&0d=hRcw)We`m>{@Ytb>eKz|8M`=8;^X+ILC^!SI|}lbgyn#%vrT;3Llq zV6af0lFph|87;h{k^Z*X;xX<`)xZ5$Kx|<#^&dCC52Ck~nifOYt z6C5d9ic|q8fofqLHYfED0gYiQWL!e<`Lib_IlJl}ca+lIjiQjAi9$!YBuL>YJVprt zV@GwTmI~Q=s_2K8ESsDD@xEtTKd2tPY;Jz<3_}DGUPu8-$AR9pG!Z}Xk|pW-WpmQ! z9^8euJtEj>_4JVq&=lmR&#{gv3Y;wc1oLiSL^s0P>yS|>l_;bZ9yS1-(=%=QZ- z_s&WZwp%CQfi=q*?WE#8d!XG2i?3P_<+pVLK9R?Hp&$a@0pN^I!7JjrofI}ew*Bq_ zPvTt=L^)*~W5l9ZL}5kq(|sc|8*ei&j2*Y)yE)BMnnAH#tE25s+of}-4-nsy#eWwJ-{X1{UU@#1yHRdS9A~38o|x4SNz}Z;%|=(hQY^UrW7_4I14kd#_7bP$zkehXSVS$NFvz zl>JLtLySp)gyE1j7%YQCknq?l32+RoF4J(4m3WH&;PcPUfDc0#5S6j04k*0jg4Vql zpTho2nmBeuUK1D){24buO6%Lq_TY|u+Ls>Nl-ETHvhfJW{`1?GXW6b?TX_q5Ukr@p z_;>z=Ifvrpbm3W38=lC$NmLU8WHD-58_c=aiI+*LhC{a9-*NanFRy6ImBxzoz}7fG`UXF))AG%G zn+3f^!_1x1GJw{P{kw0jY@4rmSm}h~S|a+$bk^2*PUG|bgU{w1#tbAP>VkxIi$Hn9 zRSPm$@BIQ@-8LEHRHIX)-4ku;Qozg{DLS$z%XPo?PY-7~-FX}z7y_d2qAiB9S(94) zEA|g(%QKkHS(10+FXYbhw``oO740;i=(T0koPNgFfK{D{t}B0VNpwWfF=Z_S|~o zJZ=sJaV|3!MOv8GLFP0W*LwclAhN0SEu7NK*|J#ZnO=|KEIiihZOyNFWykOGnbVyV zDqX#8FF`mL6u;%_g?T-79Q{_^l;M!bN4iLMmN?DlvmS`zv4vIWv+qZew!`14&=(%r z)C8rSC(YESzVpm;h83O4y?j3&`~5jmhSAeM@zN#f%a3hL8=mb=imPF)3i`m zz)$&+3cz=w6kvt$FYg+!2*NT3NqNRhg)Zb!%YlJrHF!c8FCD6Wgj!O%S*=SCAeKLQ zoXUmPJ-zV!ndyPGJKBzw3y|Uwq!5lOtw8`o_w}EOd{hKVsU^ZwIF6k&sRd-lKEOH# zP9gAI6l3H$kY`^ucS_Fjc~Ie;qg9yRo1eEh3jsG@u`n&0H#LX*VbHvI37egWV$z8s zk7B3a+>at~ye23c1S*1&+Y2F%C-k~!N2i~C`O-9d%DDXgC?&#TI}rvxlrS)U^x(Wz z-aQbZ685Wp(Z>MCRf;GL4CJC}ovwAN`xp**UjrNU^}=Zrat>+z5IG1by1=@U(Oa

oof++bOmS{E2nBZo zcG|O`lXNwM-9jBJ+BG!T&p5IXM_|7b>q1jYQs{M>ikGUC@Ck){UyP(7sY4wkh^Qcz z);=HbIIHPo*x2-zypW$fO$u0FQx)`pQ3UJqds9>OJD2rj@m`-`O9X(mg~!oy1A&N?H7VT+S&>QIUCB_{kqGr4Q2JzF#EffvUN#=lm4JFY7v6 z{yayKlk=x9B7LSEF*{KT0Re4wO^%8#^?$BNP!*8K5?!FrDpVp9_{hEw0zT}%TUTNo z!yq;Ard0X2AdP{6?8xfa@A}3LK3dI0iZuqDTj9c@$59>n>IP!KJ#IwHoNwhP!8U8_ zMH{CmZrg7!y{dF*c0t>nTcxI`Xn6@)i>$NlFpe_%UwEC?$}6N>N{fODHqb8*6v-D` zn+rT-n;v(~x90F%OEMq56Bzu<`m<_nK{jU>JzKxMaIX0yiapUORKC$i-TIA&Gf-dvq>P-0sEOL z_nM?k7=!cDQhBTUMKU=oiC+E=VyxVdK)df+T@Bs{-^?2M(pU|90LNM+x}O)K2j~;Xo95`ztsAk*gHq$oHaO1{&k9`<9Z@)b+!*#oZowgPo_%BH zWNG80fk=e%!q|d*DX1itl#;5I`QdISi)StuCH_x;`gh*{ED7OIkrG2Tq|S)IXrH~T ziZ(mHx?sSk1(a&oT zb)?ktY6s4gUJNq0C^J`p!T^cw3mep8nr=U#7wvU(J$hbDfWssg!b0(o9cWw44_9D*ew+YCLGWy9NF4eAJF+36Tu z*I%7BhuJ-HHoUakrluKVeLNG?_ye@{0cve8^EH?)K1L09HNyl0R;ZtVf|4=I_s`bW zwXm9v5n=Pl(pYD;W#lQwdghsR|wUQjm7Jd1^Pd)M zG-E#QOuFoLf1@fg7Vms~N>^zr`9Yq)T72|%xu-oo)(sES#!b#jRnQ(3ic;_f)t%)9 ziGsDFWSLGq9Vge3iqjveUgCu{#z`GdpA|?k(%oV7;}e%7X?vwG2FC7Uy!(oeDBfZC zUE?I*xE~)=%gyqTO_1l2QMj@QEJa*ST9VV+^=PD%6ES+Ra`<__m010z>9e+Q`fjc` z%+Cy|cj*%+cu1Twg+|xHZ$|(fby07W1yI7Q{R?z8$8^BQ0mm{+u{S_PHT-=Vu;Bd6 zJLfmn+boZ^&&1nuZ@(%Y*ehhvjO9gJN}A!Nq^G~xA}N{IMc2_4ax{bEw8mpNsLgv9 zD8dn{FprEk-wMxbH0b;y5m}0fd{)6eX&;k#WzaqaUgH$?;i-AEc@kH0*D8k=`@7f+ zZ{kX5w8EK#^}>=!Ucz5BGEWseEn}iT-1l$wKXup|5P0%ff%64HLM#u7zZ1su(Nt zjoRF9KkI1ADcnVOUGN$^>pZltK=?HzT5EcxPZjWb6hF(BwqG*t4lr2{EnHK*HMXi= z`QZbt`R3|DHu_XujY@CHVV3d8o)LXlj^>nJVhsi44#tbfH}4TCGVe9nh^`){1rI`A zpYFlUG6Grw9I?3i4J}rao&NORWLy^y$$u{vi0^fD-8} z#k1Ln?FsLLOxjMQ$qpAC<*k>Ls>4E2!7Q_oNBf@-`p0@qrJiK@2VLhG=1qxb>XD1alVuIGN7f`P@)V1~TQTigE7OaC#@`(qoFOsG=3oAF z8%q6CL!&&s7FR$Il%6am%p-tA&Q{aIjbNC_Gu5MzQlMj zg2;>_lcufxNFmMAYrqfw=26}7R@~0IsuW`AMX>ekETi+@PjH#V1;x@ohX2O=G$iPk z=h+}b@c@6&m9xUYkEs)FQtyyAlfw5Dev3r%VMn494rrKJz$?XL~)9M z5_t);Zw09tRPNbV=tn{@^Zl}QqZj{vKyY5Q-G_cM{ z7@1Jix8DkCp94URgi-rz4tF=i8~%FYQ3FbWaBuSvgspE}H9wFpONxwoXwI4Tgv_UC zFe9$=6N`mlOTfz*XdqE0QB3uS%KMxTFAp{DVYHxJ&$z8_aFEDki&QT-vXB<>yi1?d zA{1~m83)B4!@zTu?4|`M0A|a;t~fRq_KU)sD7$ub7B5uR`&`gD!A#xVVR07pZ?j`J zQ(b?-RqQBt)mU$%u(ZFf7uHQ~z6&=*`zCyFn}dvVsGrnyJ8H$OGwep?zyv!@K}_KE z$GzjsFaD3HgRCevNL=NRs95SzUS(s#fH*Ds&La>}iPff9%QrFi*#g#q$V(gm!B^#n za_EM|HAd@EX70LjmSA^a&+<@O2@DcX5iF;6+t}#S0kLP3@->GCKdJ|akyt9<3Gx2I z5+DueR3(PIEYH{rS#Sw~N=d=t<-L~Yu3R~>Fc(0m)rF^vrtf&Zp4#ku0-rFes4gJyh{@nSMGB z|SINnPsXWugCis+@G4~XY48b=I%#B-VUGo9V2NI2iXrg z+oWLuLwK6HsuUpLo%L%&QE6%W+v8X~nO1ETMf|hum^IwKaRE2RYXanuFho9@*{U2R zyb2G}%gK_63iS6kkySO&*a!-CJxRe#Xh?t5y05s1pMNVVdZqhAlr`WXKCE#&`J_{w zqq2+hnyChb$?@pScx*y|N;G9L%xPF9O8#v&8Pp^&(7qb2jf$CGwQMM&?eKAn9LcKu z?{1$k(qklyWa##NlE^_u0X9xnTG);8qE8;q#0oh%9l^1W*05)ap7)j2MrgcxDB zOu3o_EVN)>j8;)tf+pMd1ArbKJ_?3|?~;Ups*@Ur(<1e@0%ogO%Q5;l?@7$J_m!lw zcY|Z|Buos96R}ld4(#}+C+NQ&&4Bt`i$7CoZ_gG!1=^~wh>}D}{=QLn-k~g9GCBhp zJz)DMuCYhC`L;Dpko29OMwqytJ{7NzH{ZD8&oHo%2WyTAl+Bw|ongMLnoHiT{@G+R zNEXQ&MUT&|llo+oTFdi%6~`T=pr~mHRlMgiy#gMmk94wY?lKl)qY9kV2ND&W2;v@)R(s#6_f1e?b5(wtMc-9cg3%D z_sRN&_j(=FE-?%Fh!aA>;$>2V(-5X$BV)H-ddL@!w2|d7a-G#88SY4!S=X0R)i<&@ zYs)?j)7nue35A1#*lD1>`ftdBv7$IuE-QQEbUt@?)@&^jb5UNc^zQSu>NQer!uM1yBghRXh}? z*=cEy{dGlZKXZ>}PgPdDHgp3(SI{>dU-hHU<@RGV8vtyDL_gVDxn|$qCT7oQb%UOt ze**o*2n;Brj_M>9czI?|*XT%A=yA!%7u9(?yDKdPEKxM5Y_824MI2rv&H)js0eoqz zjqsUd#YQwE%fxteCJr9S5k`hEBf{#MEbZ!oj*VEtzw^W2D+NEuQkcJzn#5<3)3BVa z(>>am-O0SoDcphzQ6Nz#F#XbLnN}`W*7*&T-)Pi#-tG?`!?*U-D+M|DsSvYMFlwp-W8i%JiD$81aBCEtNo!r!9EI< zg6MWkG^UR6ZqGL#2I#vnm)J)~(7D!by(L@x9TzmbBZ1!Pm>O$uZ25G;&F2vZhh)~x z6r)kZH^iwV&%maD5e7e5jY(im9$f(5{V37jrSG2=k-|u1NOm?|&B}x5@y4lX=$qxw z!=>%VZqd`y^2D~JGW?(phSW!(YS|GxKtk?$Yy&Dg>F*W;cmQF;JAl#Xodk)25=lll z^`1Jh4}_UWM$q;z)(~k3{7`Jjkezzab1crD&Cr5R{Q7ln8(J2~YNC#AKS}U|5Z`~c z-~dTQqtC2h&l)kgGI-a?QUn8CXVfH@inNyBS}M|C(UG+VXJ&%thw2_S_ZBrB)*7|dxU zm7>UF0#kAQSuB3P&Vl@Ezcqx7_iL(?xu+TOH7Q?o*;v?^c!(^YlXJu0(;@-3t_Wa$ z$HQO=YtvYr;#3WSC6hK4Qkoog$cIi^(3xYLM*VKb{P`1j01lF}@^`1S6iN8Qc^5#EB(Q~d1WcJH7y8zXe|W$mW)CRM zp~b&(z`n7&b9n)cub*1UYyvWP-APk*dQnUrhbMbfd+0g2bP<~YDnuki$5|e_PKBc! z%y<18R)Hd{)mqe;mf&Vpm*f3 z$cs>WM{c>AYN*vH(NC!+ex9K{+6kN%it4?@>FkyX&y|)oVdYwp?I&Z^-Rc%-25`q4 zDTuDuys+PqWC;p6bOlyY%G)3d-BO)RvWP?o5e%MaLD2dhju(y+j{(Pa?~77lJ@aWF zq78m<=j3qj%ET49UcB_5DyVuY|ZQU^9R?HS1a%4K{sy12`KiP4bST|4<|q5e~j zK?91Ca(IBg7p=Bx1 zZIqkEg8?{xMV0p(&>su;0Re6sRQ>e%!z?IM2Pb`_|4Z+VF+n&L6t|L!S+tCq z`9SA@$`xtMxPR$kZQ;~@5x)pAxp~zB4mB6OY@T-K(c!gu3;>lRgrGU+7FkMh>90li zlswVY*QF`X$=~;D#{9$a8KVw)!jZPcgnM>a_^QD3%z3;X42~c28K!`=Q0y8k;;-Gm zSEz<$siX67Ewl53;vP>H401raFMrnS3ad+fB7X@F(qgWc&?T|1G91V0m69&R8|EQ0 zZ3U$+8Sqfu!A6N6c0?i$-k1_5fnr(Pj~!VqOfz(I3{XPQaCn^Gwx;Q?0;UARRw^Lq^h!~5?< zd2xiYduC-XcjaVJ{<$`SO1Ks7F_7e2gsr<8L0;pj1_bK41A~KMFqeG&2PHdv*nzCr zVu{sL)K~bme5WpQ${X062|`n;m|}FXFZ|?TfBy{gv^Ql~LgM3WI&*iEqNfCJytM4;octe{uP{=fFrZ{LUZ$~ho%mzN2}qe6^YL=h?xq;> z285(=)6=BOZtJ*{;}1S&v70*vUV9OWe48{emVxO@q_4FuHBn~C(h{Tz_wIW|Bp1D6 ziCr#k-Us7r#o8B70F)3oj0n^(6Qn$$;r!k90h$Qdy%8--g8d+QIVp?d5P7U!Z`rP; zRH`Hvw-&D)fVOk?4jE)}@H?gTU1t8SHrF<YX`TfN;o&b5eT82fgxyV2 z0kW1?o_i7sP>-O^DRDS4U6Q8+oX~S*PHe}3$oQsRXc84`?~_xzYTO3Yg9Uyo+_b~_ z9$`N8Vry>!aVlXHEq}=52$$5(N2DYHuGMcY(lv~VyYEi|*5>C8K>MB4FyI_os97SY zO^H+J@?CVX*RX5Mt9raSHtHvYAPrL4`3af`LO^;;wHiVVAkW+=w zUH9M(iUfrnMk@C?YJXL9NB>wwp;179TRuPg^H0CPz@NALS|15VU8cOOOiEurX>+P_ z@UHdNM5&UlF5^mHQaN6KRm-k^m18FG@&^$Sh_FE}eZk=gKXyPvt`jD*e}{&hG=%_% zrj$MVLYY_2*b@jm!WgtVq6JzRG66dLg~kV{j4LVANq5_?kSJ>R6>I{nzDPYO!{TD- z;-c^zOfTb=&8}13;fLVEKvB~m_bvd%cFuoWt)~$|DVtEo-rjwa4L9fpz07ac!9j## z!EY@uk4dx8@SS*Nd`t{23zFp86K4v6PU6yUiFM^@FkY{r90E3zs|G*gZF4x zl>Pl!s|1TAQS=XeMuvVBl%z$dJtV0d69a%UAd&VA2Qux13E-hKD7ywbeaDs&%XuO| zaY)Gi8wLy52NwptKfKkqQUE2)p(`&fE#dUcoPr*gKrLwHN@B68Wb@nhW^5tLn2Xn6 z%(=(1XG$%i)%Bo|+v@uIJFvbVRu9r~hkKmov9~un`z_T>vpz1V+LG)aIUS00bR7Uc zr)Btu$`P))+s~-j;zmlD2SqI?E8~>Ty!D7JqPt?I(8!mtyV)1JJBm@8Kb`Nz1o9}5 zxK5p|2iL3f>RbggW!BT{i}F9PSx6N7nfltc2a_N3HeH>Ra-F-rp#gN8LBk>`=*7WdPqzys8g=|J|Eu&15GYZnyr@T7 z*fQ05G4|$_&-UFs{qd8E3G%)V+Qwi;lXSd2P0z^4sHCk;FE5BlrNJf1qAlp07sVE! zI-Yh(+7HV}@|v*lekOnIr|S9S*Kje)7!DvJsqQaSiUyQS%r&i5I>inC&`dvmU5*wW z7y>@$es?Yh47!?)7bwpkvHpR${Atr(U_6_O5HCPt6a2>x{O5BBd*HmUpbb80B6KeW z)m^@+0q5xg$ZBd1g|xM<{v&{gjq)ExK!0~E{+Y9*FvYNKbL`>N-F;IJ0{R0v<%j`? zEKEEMrI>VyBESRym`Ei;~lTraC9@Lz~~2WdzK^(igZq5>_c zZ(tYQAM*P*O9}`4h&cQV|2W3+_Y{B_!Zla3^t8eb|9Zs#GJxEBAl&L0@C2{*kGSi% zxe(W@Sv4h88~?gXtWSYmx@S(~_eVVN`(Ex{&2lxD9P$3&!~n9xZ?Ce$0}4R6@-N)9 z@D7-@HxQPCcE$NEw2;QHc0>u#fWe7Et^cAULZB#0xWyAW9N*%9OVEbV%6 ziiTI$d+(0aDJj{;%ZnhU9Q?0zp?Gz5l~P&J#w(%$c0B@SCZ#Eoj{SOA^~V3(MI_LX zk^n?hO`Sl(`FYloSw`n=i!PXQ90B#SX-&!BBcSiM`Q8O7b5BoX!HEHgTqK%H zexU3;?GHl#Q#66`Yx?WGQ0LZW;s3JO-&BCu;sX3T^(=<`zmfB=C4t50(slP(LjH|S zBq*kU7!iw0arwV)>R}JlL3;$*KG6GW}J(D*tzH z>+gBj?@GuZJ%NQnw|ceyt`uvC$K{?Hf+;^2pM4MHPWSdQzUzj+b)&H1RVY+A5m)-R zDu=wsxkncFB#Pq+uQfd556dFFaCMJ%6R{ls{KDPWvgYC1AvQ-`t!aO-sIbRXs4O@c zS^DsoAX7j<)CCr@l>j<;6lOF!aZ^)cX=?=^@+Yo~A@Vwo{P8pP;;_FddIy9A9PQZ$ zD&jj3-4|^9oR3>UF-pDT1EDtPQNKr5K!Vh7jm8qs@7dp8@ZPf*fXBhF>g)6)yZ09X z(UGsm?xQ0=GyPYkPIZU-Dqu5fh5mgTUpli{3 z{aF7^U@0D;KY8+`Bk8;|Ny5dY+`?1eV^0jUo|&Pu2t`tsO`PBLZon2gFCY^6R^==ne!JTjuTwy_bOz~`dXl%rd%mX)0iJ4YPkHPylgr1#%3do-pbh{vW$M{Ake zGSJZM@;ObFUPJvizvUW658qXj9~UR5PCtBl{+r7^Ju8sFF1ZuK8aX_&=oRr!4K>eLZbH`1n0k#$(p=xYMQnOxNNr>to#V z(ozh(k>si&1>95V3>5GFJ9kF;oHtn>dpln&$JrcZ!GT7bCO`m2Hb{EyjYUt}Oo@4T zB=-LFRp6IMOG`V#VXFBnMez@D`&$?R=kj&tBKhU6FHSzzW6b7Ht%LGHR#x`!EoA5t z1LJ@$Dx1$l3l`ghwWW+n=I4TA`k^->Z zXEI*se(U*8rU#{`?yEhvm_JDkxaWc@wxS>(I{9A5qcY?JbNI`u=oHDQm+SOU29HUr za?MlSRR^=|2-KCUK2ip^D97E8wzpL86g!1PAD6Vndo-(IRF{u4iRHU?B|ILfX4ZfOlGajG@)f+>^J3_r( z9IWq0=-bIh!Dr>*jmi=Ek8gvbG`p*Gemj8pmx}&5fM`&??rbi2Y2bn_c=9niTGu9f z!}0LF`_blCW@N#n@GscNObIyurng?B+zt93S*kP*$KwBqxc<&wP%fl+Rsow;O_qOS zbgIXIipDLT*0lIHHx5|KE)D9n*Ggyt#6%;r%JKsEIG<@&zKWAa0-#s^{#?E>M5k&H zR^q$1ha_P0@@2@CEGo|cC>+1!Pn@ox$+o4`K7^=etCNl~H zx}ZtL;n7}IgeYghn_JEq3!h8e_%eCtKR0cyZH*=mDerL6Nbj`M37zm2mUe&4dr&M8 zSbK3;gojx>lYYbo%kFaI{X_y3M(RcB*>t3oCux($cd1`y1<7r#;cbt!Ck%XMO9;ve za;wD|%@v{`Daz(#V|D5pmBfn7sLgMf?bTW_y{XI);j#X*Yf_TTU0?A~5tK=}?zKKd zn{TIp?d@U9E3fO7u^(BVlVWKeFAzE5qJM%upcLTI;zs|P4r&`C&+nKdgF#5%7cFpQz zHw0eAaKu~jF0*J$yu8jZdQgwK_VS2#)!@j{nP{k#$-max4LXA=u|Z&pEq=CoTtzqO zHKaK)dCg9>LVy_-Y`^K$-@NHm5S_2jS6EUWTzfkNZD0bXaGBK?+jO{p=um4jv9S#_ zmb+tZXr3=laUo{oK#fo(arctDOfCyf?9~%Wz;IpE;;Gf$a8O&JakD%HJQ^_}DZR6f&zvb{@gt1{M}{gJxCtYNR!P zP0R=8i)yBy@*uL$W#)I4i)eUMJ;36slxFXXCO+1aTbs)- zPg2&R7L?c+N^kU8+wwkmkx=`^>G9mB@=$H)jG#nDIjDfG)zV`O8ieMuJJ*=W?6v2w ztgEZ5EGjCh3`RtQc@77YE**Qbd}{J*k0vsAJXZMBG-;H1-X?pTh>&&MK+M^y*e3{q zXKpsLs+e`@uUtdiLBm=(QI@NJ9uM??k^AzTz}-_pzLvjMr$AXILMZ6{4qkeDaZ)y? z`1F=leiNQcZT>2Pg2W`-)$Lxdq7aJ`|JMAvRw>59f_#34HZ)8**rreUoyhayb`_FA zoIS;i`pXTV(KspS^SR1+);45Fby)0>OBqo`zVE~E(x`f~iXlX@4|nwFgiha`{5h{G zHGaF7iWA2nzCd1>I;#jYoB^_&xf1Nq+CQYOk|=iw(Mdm(W~PLmVZgW#@wqgonjAH4 zu|STAN48G6W0ZBulazVd<3}t@nfppp31f6TDrD((hJDDOwF@;EB*jbUf&38W+PrO$ z5LhnKY@HqwR7-eSZS)c#QSNM9)0{@YF45D~jwLLXNtQvuTbuS9FG%7l~|I-yo2 zyAhK@-HRG z+9}$I?5JYyQ@oz~$^3njiG+pJi4j@T4rHFtwlB#Hkx8{etM=o~n?&&x~^jb;ywjZ${mxF81Lsxwog9;HTq#ia}Z77j+KdH_Rgf z=n@my_w)qzEKv3k?ibCQMGMMz_ugx(OaRN(r9$F5rpR z=708ha~qj@STGB#u_QS2OQ;?!b4f|OoTZ#qJT3UxXsQS5wUXenbd_^yYI@q)(^Jwr z3;yPr6w@<*`lbMI(SdYs{pMI}Na0rKM>c2AM1`Jl zlbl#hmdV~HCKlKO0vSedu)!a!FE|{|~7WeddxbiW@*r#tC zTwHOVK<87ZetyR5*cIjao`WG_)-{r z{qi()aBMy*LlX5Q*|LP**mtm|!so ze!090OKv#&SZI?Td!JQN7jtBcuhOIaI{HqetP+bqhlFv3)stlEmu@6bCUSCJYY6oAgfRAdZRzX$o z*7lzA_%4vkG~C2O#zwuB*6=KDexB@RaeO?95ze}=EUiyqsOcI8k6b+WgR8_&>5{nq z2$JSunJDQ;c;c4$?MG}~KA#-+7FweS_brY!Tj#h`rhq{IE?nn{vMZf5__um;I zZ$M`zy2>qy6C_Ai>`(`grlqBkDHEOU)0VT;dMQNAk%`XU_C+QysXeVoUoppS=7<*@ z;SKC|6SQ6dtlni?kZ_v`7f*Z>yimQh(Vwu5JMG@yXs`-zIptVZA@T|!AZP>RGWBuR zMIUF$xosPi;8m@n8Xt|sSwQP1Z>;3ygI8SIX?t?XA=LLg5Z(G^+YT3Xg9i?Lqm%7q z^6xSUE#a7OY&b4=&Bq0A4r52m;*||YD?_3l8_9x;MoFxb$;tLDm$r!D%bd4%=u>LB zS|)z$quB9#alVDDW9~)u^|LEH4YSZis*rW;b)IYWX%*#Pt$>{jB(Kk#S`_{TVJl^7~rao=O*JlO_9P+7DT( zaWvnsui5XS-#DWyS0Kl4Gi zL!SWk6*eW{c`IhF9si~y2PU}8Pd?-FBbi^LL?@D_D)=}{;U3e}De~!)_1j|j#-OVf zg-rLrQ`@BqEl<5pLIvI6pZNhmMGXW^@XsFrjEClnw`e`KUSoyynt&!WfCO~AoxbQz zXw!@IHQ&63Jy?-5fa2~*?5?W!l0KkMst^TS?v*-#;9ECB>q1{DADbb!?K7-tzIe+& z*I(~#C*eDL4=!6?sM9w%JBA~b*g<}H2DF%F!WMvs^zBqDLo>-Cn2ETy!Hd}JJPP8i z68Zjvl{12LSY$J-BZgDY5}oy=aIj+f%N zQd;z2@m-Atlg%qsNbgz0ibODHlSWX{)V8Ed#mQ<)Fh4og$KA%#>6kVWcap)<%zR<8 z^RiF*!iDc?#AkB!8VL* z#2lWygyn&Bel7_$-umR#+qUG?bRPI_W~Mz;|PGZVleEN|3&ioJ1Ko=nzEK z=eRwjeut2OC9_4aEp}hgtJs`8qi*4dKeMa#>vva>f2cAuAEkMGCU^>vl9+TOii^g} zp3f`Z@N_dwa38iC+4@v&K1I+P&$Y|Tfy{Gn9^ONIa9_CG8XonadtIE)=Q!ZM5MP`v z#b+98+&@|;pC|IA#eJz3r!YcWHv^A2N)t*6?T_Xq!_mrD;uTJB4sah*mKTikTkw|h zbUC;s!U=tBxm)?6$1&NGMwfR8{mdO17o`2rb z&5!5YF_`zPg&~7b&$aQUDA6k)RA{ng4^uPdx|z_KCIA85R=q_}N_KzOKHcZgy)3TAFMulBDcM ztQ)hVmZ%J^zkE&Tn5HvqV%NnSe+k5$b*sf8*Rj}=lNLq$b*AN4k+Z-wn!Ax?y?8!# zX(V`$LKM@~FO$i>9^E@0I{CC>bQ9B9XXF5VX=--NpFh40E5nW_#9r2Js!b1{oLLX* zl&1EO2AtuQ+LREKUWmL!++kCgDMZtnsFemGI)y6Yyhk=W_Yt}NdR86G6jDez#;+Qq zAy(+JqlZ}Ls$V;BH@k_MN}gHc?uvJ`nf0eT3l-secA@x6?->Dw#Y;)xNiWyVz{KnPLf5o<_u$xq>I-Y?v`nn;2yvIDX&RXl)4iXW$}X`-VV} z7nYY?UjU#o$kH9aeQfDZSEC&Z5O9$uT)*OR-!;uPecp`Al@^rC%L$I5%Q<-bB9*_k zJ_g4vlXv?$Ya`3fh(?_?MkEaOK%OJVJ&%O@%#q67@%`85SgI8n0|OjMhrKDffQP!S88>?oQ9pI{Y7 z|HCetZycs6HcK{vPzBWPgNQy)&G~}wPDdzZ8_T)Eyf`(99AmdI@r!HC9NZ#I9T1`E zc@vQ@6Rp^BTD>{#AK;aO$f!=jLd`vX3ZxTg*heQC$)~~fH6N$g_TDrR6gzKu_Ue13 z+je|Hwd1NB+83U7L_1}k!1LHVj>}}2#x|ciZC(-jhIox{;#TKi7ex4?R_HSu3l{IR zMe>aN-21)mB{nuTl4cbTI6IBh)TTjp1%&J?_WY1Il0bs8#>d5?#OO&7g^>+FN>3eU z`a#B?Cr5OTO4cV&yD9eV*Tt=26g^g(yT{s_L*)H^EM)W8SlH7LkDdna3Z?DLSsn2H z88KhK@0%~UCJS76SmfSYy|fG=UdDahv!&W+V}nmc_Hu1JyHBs(e8h43e&O@vk`tx_ zaYn)fb;xX(!l`$y*J-IT@v5E7id>%*OEN0c$snmI>tgd;T*9f~*^@S|0CJ`C>G1VO zcVP1{bU(w36yR=yn6j1iMP}}me>(~OHik>OyuW)312GSur8vnnyUP#eeFtJ=j|Tvd zXHKt>-y(<|R1`aqFNnABdy4%8pHnge-m#mr^UdayW% zPhtPskANUc&k}TQck3oWB8K^$+q~w4o@~k+sjL zPbE^zuBm{zYFDaj-2rcCuF{y(i0@R1Wrr;JGO;?XnO&w}hpb}dJdpGi2jnt`NP@T4 z26Z5Km*h?lUmD7?SO2>JK9>Yf~yE;`)G4Tg$uUxWcl;% zGw!x80p|Rc3(JjxnbvY=(ymnZ@q`)_3>RS6u(&wLmh6g~3`}pKO5>x(3?t{lh?azE zXkPe)CuS5CVhBZKkXTbu^I299?Fw5d!9T+rNxq^gxGnnMBYSF_XFB}?BSp#Bu)+2w zX0(#lu%v*;=S7ZDoy8ctWaDP+Rmxy~Lxj;aae+q!p8WeJ+^%?F&7)Baxyy%`F8mcZ9EC zj%C|M(Z{5P4aTOAPej0|n6TX@^HcQHrs<%UHs{=#Rr8;Z(RzfNMDKR_7I&{jxzHF` zUtIS?XFl|cRnYd#R^()0 z!>52_pk#7JXs5Ct7wtb5t?mttK?>Cc-Zo*&slUjyP!=j`A3SKU92|q}pI=-B1P$o* zarcg=NY>xYDuj35DtELtLK}D&Ms4iuhGCBh1aO~|s#YjQzs=56F1OuoesM=Ylqvk| zTW^Ar421a(O89=3H6{lo#oBLt;IUBS(IEUJFIQ zS02^6i&rv+*e4?A#27RAZS;}33el z{jaM0org@pUSWOb0Xd=NvF#5Xxbz(QhWGGEdz>9r$wzskB1u)9U~XIl_I)02b@ADq zK@I{32)d}4ot2s*{DoGg#GWY+)fJDeJTG4DwS`VOG~Jo0zP*~_$RGzZ@swa6_mM|N z)dwqlN4EIvrXG5&1iI(_Yo$f_>l+KN^QIVfrVo16+IZ()_et?|S2Z zw1p=GK4?zRaENen5!~t8`^Mtr{?zCU5pdh7&bwlhZlp#zFLrd`$b$-N6<%e5o!`Q< zrff#fzr@{UxH$7Hnrla!Wn?=?fHwj6EU@n8^YL=wD7yRu9)n%dpl@Gr2k#NaDV^Vf zwk$7L+JdE%xos1D=JAQB2dP;TI7tXh~wzA1mf2yyru$%sGZ4f#)vy&6awdUQJPPkhNW!M$nM;{kjQL#cz> zY$yx4E*~~`FnEI3C4^Rv+PnmZdo|?Eu=&@>;;^EHOfpWCI7tezl{Vk@>5?H%VB2P` z^>iNd0ZVmz?E= zE;95PSF!7~f^H^#@ONS+U1E8WNY&aGar3|r>(?`+st)CCSV(9Uli$AS`7Is%> zF^`&lBXpO)9JVu|S2s}`bv`UEilj^2bnJ&caF3+Zp1<@v=n8Tkc|@-6)gQyTvpXludaymup|sTh_WwV=)k{6s21oF!ApNgpgptH?Y>iH7oW?c(_mmY} zV2TGR9ppdm;@7=FQqU_{JtL|%bN=^~0}NnF9aiM8@BHVZH7KG?K?S9Boz?#9BVT>7 z0dX}&-&g7na{Ob_IT|Eu8S*qHKKlQ>!cRKRc>q(A@c%ZodY3)H@6~o*mHbYrscROzv6U(-&|453s5jod5s; literal 0 HcmV?d00001 diff --git a/docs/img/ppo_mnist.png b/docs/img/ppo_mnist.png new file mode 100644 index 0000000000000000000000000000000000000000..3c5a00c176d5482e2d478902cf3751081d539f02 GIT binary patch literal 101368 zcmeEtRZwKx5+&~L?yily6k51L<1URhH16*1?%F`pxVu~9(2cvhJ5%?)nTa>=-p~1% zk0FAnI2EV%-Y0kF%C&OmiBMLQMn)h&00RRl_vjVs(;@RodR`I zf&aG|YZHxxxc-lo{p)s3jH)t=zb{%g@c->aR|1vRmX@0RKHjwqJofIE3~S2Ef5yT< zuC1-1qodO*X5$eO)}vILPfSi~XllB;y6SxV&8$N@QbwM~LqkK5>u`vOaNeTp6A;v)u>H4F3}p_Q+|KJZ+n=4D65-(; zo}4hUu*Am3`tHQ=e0^Q_x)5?b{e5~`m9p%xsaezYe2l3iI5R(QMvTDF;Cb91hDzC= z#-d;MZpep(jBNL(4w_FwxLW*2dR`tW2}y2lZqvKvf4fC=WAxx*g~;3OTto!?Nqj_W zYwP^{yfmp#GG}*yjz&_h&I~0ix{lD@5B>H_mP&w{2ZYH~FvpESNAt4EgtIBi|dIPt@*SmT3 zU*{Rde_Px_kkN6)!NCD4sc041}=i5%4$otxbZ8Pib6xZwvXZjK0S1Qbhfr;#G)PMUOP%E z`P?3=2<{bxk@N|XdY!Vp!pO?X($zw&LN)gxdwF^)X7dIJZhi-C+QTrZ9}yAJfNpVF znaIQHHw%~x?QY+#N2OI*zFu^qVeK(XTU%RG zQ%Vobf<$Re$86bA??7-ttieX}ah%mKlHe*SX=&ZEzXi8xBSpHWb$)U2E@;JXCpF!Q znVR`Oe%SYiB0qpU0kj(L&7J^$dw4T!Y8`EQegL^H%+KSJ1EBimQj(sc30M9yf#B zLpr@45*hHzlw}tOOLOiH^88|V-1%oWuW9qVF+R3qMlSzIDk

7#kaB@i<7?Cm=QV z>8^A`*t?wmW?Kb!Kge+_Z&5yNfu#Oh3`J{1k)2dzWoG8OZ^IW*o}Zkoba>qSwfx*% zma1q_P=TB$Y)33jA=A7-nr8E~L@sS;c)0ol^hC?cI;+~N-ZP%ofqA|nhdRZtAkzr( z^UJ~>VV`$CtZ#h0KlG74oSvTM3VFp}m*x3VDDc0N@H@jv-=D3}+U8vuTy%d3voJ9* z6jI(JVbX=g3Eii+c-z~fOx5{?Bqw8;S%A!2*F+rA-ripA_ut_O1KN`jPix6N0!+-H z+ekvR*Y~5cJkZPr#%}Uj;B_Q_VvmlF9J-!hc#09z)RdSjQ@|6FKyfPT+6CC(b?CfX zyuAGMZujNOpP$D4r3zVCK8~`o2tU^9Em7%P&hTlJQOrYx0Z#~&l$0Nj!y?MLEzJL5 zL9Sr5Wi>=|me$93$E`kSB3ynGHzBSNma581IT;y`lj`QLTh}KQJa)_TOG}o-?0~c# z$*K2)^l6qeP+WE8omcsV~mg?zJLY=i-vM6q1p=P-oyjG0?XkWc9 z+Aq57R+>tRiyImmkWf)CR$DQ7^cX#llW3JFCT(guEjoYluo!m9XPSG3!uuXqj9TmR z4*o3#MmL6l;DnHXfZO&rnI2yf3I5TUuLxHBUsDBaZEZELd)lnExnN^sJ41mmbEDQ2 z)}HR90%Vfju`yX$(Vvw*TCz$p^+3~ymx7*L*oiMK~hZ>+5g*FhI9baxAQqU zIUuy}{R0V$0@rWAUNdCqht(+7iD=c>5BdW^C5t`K(|YSeCxY~|zJ90u8o%~gW##AQt^LXArn;({J1Fe%=8+ z3l8ca>Ml5}&vC`ue1T03^y`dTTy!+Z1h7CHe0+L=y&gT=`}Pb%CG7}j&*O@U3J}KOH2Z9Y;1(7YIc{0z`80Xl?1ON3JAYa& zsSKUS1C$-(k$My^UsN#9J&zk2IIsOWK`f%Yyc~oWC&4_ag=H}D8nc^fDD`!9in)Rj z8%_ktvo+oCvaC_r*#d6%W@b>sZ}V!9LPA1j6S0A7wWiW~daE-<(pOlm@qb5V7n2Il zqid=bQUVLK!EF@|8+{}FP}|i)*Txkug2mR;+kGsde{mh)mcjS zTpt>eQe8t3;sSa3&*9b#5L~72GW`1b3PK^4UPIgqqWb!J=Wl=Z!t;8lg)d04ri1iysUt@|u4_#_Ju`Cy zb+>s4d=Wm}YJ&*tV`2)geo^|&f5Vv>|`8fDk1(BzJ z&?C`};f)~JN>6{hol^oK&^johQ&Unpt^Y}-)b9V(yqlk!o4dNYs;;iSxw(Oag#4#L zxgrI~V=4YYE191C)42cFB-`7@Uyls3Ox^q6SS-@e^|t+t8Y(L4#!hrut|vp+w8pME1L#XZcIX82CP!pOF47LXDVM zK`Qc}_UJE#`VT7bUqbSqtVZofdsvYM_YVSg2mxf!f7*h-q^kBmXxP7GaQ)wu!)JT> z$zKb1(etix(el4OhvB)g{pZr9KCuZ2EKLjOYzXiRbJ_gcL;D?!i0Du7#_Ufkj|rMk z*j`1Y#4kp(hbZyywB(kTVTP-V#-q-+B7R54&b2DwM(ISYz*H64;NN__IlN4}FI^Vj zbw{+V&!k_wc9^OvXnp)ncHtBSt%G>V`tiee978Z&XQlafdo>)5#s0L;k=wzv#FrN% zE?Mg#{=Uyp|7(4p?UUV*3|Q=il9qz`ve|cxdBMNsKaTzCKXQRpwPwX>Z1^!fkyVQm zfd5S$AZKRr7SfkG_KW=Fa16ZuDd6g;l@BQ7Mt12sT;ECcAW zT+Qo4$~LLV7-$$V%_hZ9cy7r2p)_O5uqU3<)gSsQ3tZVN^9P)~+nO#yls-iUI`zjt zw&s`5PlvjG6~Lb;Z>DGEb!)F*eY1tuIZ>+J0~w~2YH)FHo7;oP28q$oU|y$>73E(Vt#g&v8fhz@SF+68F)P_JH{pcwWp#at<>`k@ zEgonVMqqp8B&(e{b_t@TY3m3XiI#Dl?(zsr#(aAGBym|f2XtVEPQ0YU zXmHf_q-ZUF;Wl`?bnfQiB6M{bMCzFF+~($F#f>?vdlvX#N0VMi?nP;VDeZCU$l!zj zFQ1Iw6aV;)wa<8shGIfulw-k5)?H#q b?0t5ZX9}hlz;-wkHkc5sEtuQG%t8D$N zcYgT^TNNF*8Ci@5mk+Q`CieN@98Prmd0#5S9nc%cg6$F}FQqSJMM@OLwEWvm+Gma{ zxU05;^+zh5%LOZKo!l}^H5Ne9)pR4HqZDK{o^b)OvOwc1h6YrH*VMJknnSV6vftOg zBT-Hytdv)ToXARGT+-dy%N(uxhBFOSPI{J>I_C}2qKIkbMm_`%u6GC6W(lbmoqe2Z zm9w8g26J2yRaDyun+1IJbDFf;tOPJan(7uY-=&hVt1WQ^)=RS=r>G4uQVY}gI;P~{ znO(VX=Z!t>I;nitu~S8cA+4OA7AcxELAlg*rCequc7}LPihf>i8H+s~TA%6RfzfBO z_1Uu;l9QfYx=?uD&~NUgMWO1p?;1S&&cdid77nbz-1|w-zVoO5i&9w0smymAjY3l^ zlkp6_1gX-G@5dkq960{57NLPR=GUUp#o7u3ZIyE~w2;^|RfD58xr$>L@x(zaVmMtT zm=oT%+bxZ#9(NtvoBGan;s?7)>B~s(fm*WYid3P=l)f^)IMzL&fz@!Ud(aVJ|C%Hx zNkM(}cJuUcduSX%=k2|S9Hx2+pDU2vbYul1d^=}c5l0I2y*9djU^B1(b`w$!c}-0w z{HC=er&6o(M%LYc^!W)%N%yge-BC$>XCidn!Z0Ksy6acF>RjmQ7sCaxPtcMd{whr0 z^Cd~>D%(QOT#ha*ohsX7a*#bHeU{`_Sh2W^z7G7srJSwEEke7jn+hg`zW)7QJV{$W zz6{Iu(y6{mpfMf2JQ#}j?*DIOOhrvUFiH%N!FvQb_mb>NHD+`>B*(WKSM%^k3r4t{=>eiP=gar)5-0Zbf>3!gmc-dI#ni06Gr}ITeNT9 zGUGOIiWs;pZ>}XNhl&yg-?~>jyKz%U)y``m5N9ciPr7p_XmZEw5JIq)M7Y=HMN07` zLs5vuF;jB-qcDHsBA21xgS<7UQaaT3W031kJ1r`Rg9JJfw0Y5?>#zIojdZWHnr=Q` z_0iVvsLf+));@$2u?mw0tackI%6B`}PV(Qcwin^yA$R74ih4JyQVo`hyq(c*Jo4I1 z@jHt`7o!cHQi|})8*mYHfiFd56{T_>ICPw?I#%D{LQ58O=&2;I8|X&EEpM_M@AbOA z+j)~yyln|ZJe-Bo+RZ=v$i$s;aQ=L_i1z!sbmo^miL6wqP_6qI$PBsjm=CX!V@`a$ z{BgS4QB~}F1$^FtO~;Q?pziI!G(%)iQv$cC<@00Vuj%8F@q|zochk=d zLu&J*g^K|vC3azRC(1(GmHf1&mX4%P=mV2wbyv()UT+WGR!k0FH3Ruzh61@B6)h!R zW~zsiGbOm3sC4uZj|OyTI!5j^f_GbM@Q<v z8xn_Kd5GlEVclpox9Yc{^F*>knId)`q2kV)89t?aOx0q8BbORiG;Z4mI{Z$x#qNcj zdYihNJ>P=Iikb_A?b>`HiTaX^1yW?dxW~H5JTY&oGRcVoprnP zry0ZcpY;OspD0W9i%Dde}jCpT0pqLTg$&lW_;RFevF=r3}FwPXj9FF$EJ< zEK79R+d^qKgez4n{nN6MwZy@1IqF&&-B+ueD3dsN#+Ok`7tdwB3mZ$_J?%$O#2!vgk&k&8g{%o-WtKOL!#&PD~>`jJFF_iZR)1v|5JnS%J7L{|HeD1^- zZ&n7<*JB+SXXEjKiPP0rK`A9i-H>##&$usQ9>df7YJ4qbb2HI*UYH>a2T$P*4))0l z&Zs*pTM<^Q4tbM4W`?tKTcNQUTcDI=TZ?o4VMHHFjD9u#?~=XLW{frnb8BJr=$>gnZT#p4Fev?}Fr z%~H4xNO&>OMbSWdO!MEr1tNRlm=VuTtSmZB+F~(;vke}`W{^D3Nt+0%O~1Amj1_N) zg*?R7xBC9FT!tMUq%^}8yi)SQH2h$;aZtUBlAgV?N>HUSAf=`xeL5LE6|ZRAI&vv< zBD~3ZVdHlYPKkMeqi)qjZJ6PXzQw4mDI`Le8`-heJ@<^$LVJB5SMNcGkS%$a zZ#k;u!8iMy%nQW6^HrMs=w2}aygt5hKMx>&KZbQ9G!N(gu^Rm7WX2y`#j|nf7$;0n z42|vG8#XZxs0ZNdI9=Ah$vhgESo$N{r#8=hYm_jQ8a1ng@Rm(L_O5y;yiOKQ4WEix zj$-OWOStb#;5>$%hTMcj2%DcfgpFo@;)*?QnpUxm8)oeGU3CYj&!@!}8#3yD{zIIb z4cPd+EF6Bm#P7Tm^gmM;Bz8_LK_rNHCK#D`Hn|uf74nu?t|ll8clIUm*{W6pmM(); zmB}np(+@j5C(%thlKld}+v_jPDPF#1lYp1GgFuv3^h_54CAH)V5WvXjAqtGjo=Lu!s|Kb}z-mSEu z9+UbKqEh;{tOIMN4+`FOZhaw`$ca#cJ>kvw-Ja-~Qi}r`OEHX@S|2MwkdbBEj{@Xq zg}YeALQRKmedyXJ5Vn!|!X04^IAos@=ZN9%##tQ)x5&O(Pu|tVpB`c`ydcANIx7{s zAUq3)zNWt4Kr@$p%L0LEGanNVg3b~Xz9K)9iPuk4+Wd73&RYWp>rR7ZgT*c*~mmydGaOsoy_JKzLF|?O?9+L81tu@t$?#iX! z!B3dttYjp*9HK~s$(}G?U_pd*T!NqgUZ;8ZQ-=@DsxpZklHA)=j4BqWb}Q$&+uHx7 zuF2lUxj>{~7iPFj7j{_9?tX-oM8@w}d8~817LEHO2knW`L(n&$@hh6x2x_?BPL?k}?Au-P1o!?y4 zEa=hJo#CiRp<1s>xpmvHj$sh+d=++f1xm&rd1s;b2z-8QedFKNZPi|chw6rW_~na= zhZHWgW-1FI;?GZ`DhHi0!AF$o?AjJ*vm6YSVS^5hj>4w9Ml;n~iJGKv2Q*wGK@*q4 z3uvn&wY@?g5hol-KhP#DqPnEWj?$4PZS%Z|qL8koIjbJ@H2c{e%NxwS9NYjp2xtUr zNO?H&=ZQ@$jYCtqrH2pGU7S--q}w5&ar zh8jP*Qg%9;g_VRCyP+>`OPGbqlAJ;F`2DfsHCWu>bfInX^fpBML^JDHz)rL7kV^Fq zN`O2iPjk5vyHFQ_wNgY_9p;%QH0t;}=nNbrvdgZs?3h;uR-wrr-n@1xXng2J9Qtyy z)&TzS*Ufi2HY`L0aGv5UxE-r)OxQ$V0y0IXp64ML=^sn;p!H|rBPIh^5M9pGR5-6! zaJpu(u&E)PO3t<7)b+$gixYQ{AD#ZH9CcWvb|c}F?-%i?@UhLB0&g$LZ_^qs`W!p3 zqm68;#t}vx#=UvJlG32(oc6!N)-`P)F#Toh*H1Dd(eDi`rc9Kd=oasyp&C3m$c(6P z47JDYuH3Fm-330)1&XHsnYMYLrY-7QIU1Hr2o1Hn&PFXhm>&XEW61-D?4iNC0TN3hzyzt;@TB7W@9 zDTGv?p+S`uzj2MSWs6aoBN$2Egq$unc7qq#Z7%jc>`3);TZev+Okwjg6)lvulu;mz zkv|Ob4hBta{c19XJzRWqz4%yFhHD??swDfcn-K{dSHOAjd={Q(Y+0n!7P$`3E@-{* ztbNDamsHK7gah52|D0+82})gVtzPIHm1HZP${Vfjq)iC^pg`+CDB@bEAGs z89@gb&e-5jMD($7$(*sLD@$d;gRBf8!q3$vJHsgMxS2f72Z<(!;X3b?^p~Kau9=z| zFOT&e0ftEC5DtFKc~LelfwEOi)eE86BDX=p1eB0@ek=ONH)%Cfi1qF{EB3k}*f)lX`<@GX zPcCujR)xh-NI_^^iVs##9RzEvwaMS74h);|u&@wp}+}4hxVDdvT(rRH6JyYC-LsI zq3zYUp+MH_>@lzgEaMR42$ntCiAF$kyR9>#hIqV1dp$OHS3=R$(`RaR^|hI2+{*Yg zAO}&0TRHjDSiJ8h*Oi_ma->(}X+T4&5v>Ny>^+5Pxl>DbZw?})P|8-&z8$hpX9lwB zHlg`8JtLo&yTGm?l$M0ZNAD8^5v+=HI{9Xk5-~fGO$SlN`_IYW83fSmrsZ5hC$MDK zqRT6|RB@QA|V0(Dk5PEc_t-VzHAyQ(GF9!QGLjE-BlJ`0%5z}N$~&udMLK__JmQESO^ z3_C+)WeIOnNGtjk1?9e|78af8x2&cyaH0&s01oi9xjNs5ZIkF=T?1_QNlUbjW?C^w zp=A^Zg4Et2`?#NlX)^wVGhl!Q}={8oQ;a4Abo$^d5H-@ZY3!Cogn4w zsDg4EguNQ_0y#lTH-PvH>gJ}l|b-EP|BX$`8}X(FZTg4Fu^wl%yvcg@qUmBSe} zPL<#6%B$S|GR3nW2qyA_U+;={t0LV6fq{C;%8P8tnoCI=tKC>hKNgTE7lW~KAD14- zlqI=X&(fiof*eYy(CfK$VS&4iMp94p}MGWTm7>!<41fAm|DK-gj6w zF=MKgThR+gua)HIXGc%}=0WJiaPyBQSIRLbxy_ZT;av_JB9FE1FR)5jD6f^o)^pa# z%ggK5T)y{_#}7&3Pg=pEn8NPIT!cV{(r*uzqR2kYH8=8prd7%pGybUa&0AtoH~D!x zpOPcaD#=v|J;E;3>0}L#L3bh|}E;}i+P>$Lf#rKe=4X0T= z2{YZkI?^`Yr@4vr`_7{-wld(4Ij!^5H4;dPY+XVom}04{nsx3+QkV9U&T{V; zU>M6ei1>bpdg8?GLnk*6J;Xh6MU9s)MH7%_vra#w0&M#Jq*3DDf5%j~nUMDyeu@6_ zTh1a21eBYGw~^c5U)*$LIXgx`G1|hce;3{=ol5|Rp`2IoYSTdGFsEod1;;O`*sk65 zQ`*3u6;JOHVPwijAev=adCol-I3-3v)iV*0+^xgmx^4fB5TFj=sIEu2qo%t(C3}W? zm3+S%&E-#Kn)P=l0%o#P48?+!i1AN%XG08xBwO{E(dcG!W_0R^m2G`9$=qq#9YT7X z8nW+bcY?*U*nh}UFjbzGe1m`Vy=-fP5M)5YroSK=G{TUE>nzn}2;71V6ncq*NUnhG zJdPs~XV>7pd#l!fE}^~9#6D|U^}*31TvX*{R;bZn7xn;Pf{iRzsD%-nnDxDoeWVe$ zo1XeQXm*HjR0-g+hLtB4MVRiqo2P;x1(X~8sUT9vjb^)O&bm_tP9TOq=(dhOG5jT5OiK0{Fc77jx#5LFe#qJxlAjQ2`y(cK zky^dEf-j0kvK54#nbp&xfN`*l&T&3YcBZb1GTnZ{QA7-6X*E8E%Q?!?y@7`^Kf4w@5)&`Eja}?2iYZ z%bRYC`$b;+*I_RN5R1B=(kIx%fSVV>VTftjy={v zVK>^u)s!V$+t#};(qzf|J9G8B_-akg(~2p2>U`tTVi#oZ7un+vmIqx)Cw7HD?Xoj7 zDi4YJC||D(s*xj5_~{{_KqE5J z0zUR6nm#Jg*+CRt{c)?O)bod|XQx#p+%k3s|yUPB}5YeG}C`Y{Ep zL_aXv?2&sXCQ&NJUq!zNnc&J}t~7E$WkU8#e8HL8hObL%-FOMEWUc!}Q6c3pk9B5v zbY^1Ytuv_na1ogeF^_T@+gzgv*QKo+hT36ZGY?}v!t|MEasv5>hwH*=mL>O6g_CRC zM~kHEIb@9Xr|XhOC*6mCP#kFW!(Vi9c{f|!zS0(UeB1W7pF^Ub7v)O|4B(G+gkXU2 zk=UjTEmQ2LJ(Wv1+Kq9R7SmAo?GFqcgUUa z2q<6;P}G2;N>#N(ynh4|B%053xqLj(ue?X(Qp(t zp;ch$`0bI>sK_XjsatIqJOX0m&>jGig>6(ylA?^A8Gg^kL|1fl(ABygB02VwDfq=N zQ8Uum4p2cq!0OJ3(3 z9U~w8u9k`lOSCYRz>l1s#GV0`k}71)k5EnmlJ@i;fsZZDXnqNazY8X3ewO+>>|<%WUz3JluWYkUE9$M7Qi)ou;;Tv8c+LokSs8 zC-dX8qv@xfDmfJmafpKEAzlVjT3^?ak+vO#Ezb&J6v(p@oxMZSEQo49K`VT`je=8= zyLGNA<$+8%oNN<2^fP=ExP~0Pg7HMJY4FzNAH_6X-&o?X2EN_gjL~N(T3F3UhEgGA z1TRO)%S#>UTxS+GLL|{o@R=_pdiTc|e=k2OqyAEE%;P{s;sQrUbvXoohmm2Z#|+54 zSqFTGbn7H8Ras}#TI?4h=m@v#bv&7I59r8Og_LB;>-}Um}k&)T8QL zhqKH)1(O)kb92PUB`sZATQt~OQiG(w;O}>=85^zIELbPuOUq$hW5Omyt=tZ+fz+c( zDg|)a7#a;ON(Z*9Md#H!*qHyj9)z$9o%tGxY0@>LbTtAJ{W9lCJwrtNwi&u29XeUw zoCd4MZS*GW!OfYhHHGKq4`Um8h*{!TtGju!Nhwh8>5R;RB=)+PUO0QQ61^OKahax0 zuzcIt+FhiOe-@brYBbH5BgMYXZC7!q+GoOOg#pls#9>u%jV@ZuGwKTSBt*DU=$Go( z)eHO7*57eO@7bev2ud{qSxf2`)+!0^ZYJ2mXcBIf1n!*DVr;amH5fAO2dUut`aBt&H1$SZ26>Eb_|862{YP~8CHgcUh4 z6Tn0yZqNrD+ge8;fkAzslKft%-=~EW^N`$3Sye&FRc&9EH=0g7p%F<91;pcBSWQkp zKr)Oq`W{g+iKw8M2j>`+U&ZQxD$cooo8>YDAvL`w%_sIq9r8$sf#x1yFsT#zC>5fO zSqcNj8hG)9RRj@(^dW|}978jZp$wK>J(Mv}ipfNUlsVo2zcb1q=?iSl@y}6$^#7wFn12?iV zIT5`^G!E=YE@8@qpV||96YFXkoF6Co;8KG5tPJ#lUv6I3Bs5K${ zwrzZ{+6gZPlAi0{C-dZIi6TR7;1l5bHELo6)|?rM4ocAE;c~2<%IS(2ispoW z;`khWxk-%m?$NhX2mdWaP5x6WsXZ~#27#S|cOJ5#vP}F6a>f=4OVAE61g0Hta^ZkB z#Xh$-VroNE@I+J?162qsJsioh-RbWVJ~o_mC0+I_05o{MOAS;&Jy|Y?_baX%;ZF8)WpoItf&QM&7XYe`e~skMP?4oQ zSLE!#L<<&h@_+`$JEN7dQ}`y~lPJsh*FQ3{GpTWczxGtV8K`D5apE{=gv-mm`j?GU zpVcdS_=hm3et$lzZ>=F32c4L%eTw91_#n|Q^B^}D|Fa(3y&ZO!pMz+Rd1^`#jstK> ze1>9^qPrEe2k26fFo>XF5yM$EuqtT)kc+K@D6MuH1f}9IHGpV27z1e8DQi31_AUKB z0z0JZc8}pbA-5R=l;yuAe?XnV?jd1Tu)mHLm2osWAa-Vx;cuF{hJh1XI%1D~4OfzTJt~ZYJpwHEY{P?Y;a0WjG99Z6`LX)wHOTd%Hnvg7>J&_y^GDJjISWB|%hKxhlETo)HbM6>Qm)KQEZqVe7?= zIHJGRjUNkp58Pj3o857AVqcXw&c?BiUzOTVMY1T zp={KQ);B`u<~9H-=X>)r?d5#)@Z(E}Ifvtb5>%nbcl7)INy?PRFw!6LSttH7XZ?qu^M=tE! z@i15yIAK%OBuLb?Q7|-_GD1t%{)l(xVOOPIBQ5pWUdW9D4mT(P=Xvjzk_@s$up(K2 zO}~`YF%DM;=qdCmO#2EV6R%O&3!JifZDS#JuQXLs;UmACtAQFlwmQr>)ZV?VnvFY6 zWZ*0wa~vE}pKg`5X?;7)TL^K4;L)daM*K5K$z#Q+{G9`!`baD3OFpYoI5Dzd6(UCB z8qRshG3mQKCq1y8viMYPN^Ov2 zI7nC*Lsm8;VBgu$pV;DPN1m1A7jxYbpRYtkFge(q4Yaw5%_PUI)mLy+JBNm{q01~ZPk8nNYEXusqz#@G7*w9TW`tt0 zqS6-bP;Ys6YK-Swr>l~cltSWWpp+6cO72n6G$5X|yB&JRp&B(+Jica6#Lw(rWjP0y zheG@8=OJRH-jP}HuDY*smMT~+sr<&ZGEp?B$)afHj0Om4 zn}%&uVx#FiWdi<(sBF@Gn}KcC-u6l)bW_QgZSlTQJGVjhj9pyo=nn&;L8Sr4aNTUO zsug3!xvvFlp{72&I2i@(&F2>Dkw6UsERN4;dqzH#5#@r9f$HDBiZ?jw1-|THHF8l< z<9q~eJ6_LS>0uyznk(8vo@Z+??u}AJSFV{(-)Rle z+2yEby+olkg7F4M59zMv&DueH$IPw*;m23|sT1^qM=03bPWc^gB1%=_6;?BL-&>OfLE-0Fbd7lZ^HPHBLMI2y#9GU8^?+b!!vCt73Aype0&u`KoNgN(%Ltj*jD^#gxAhT-d?K&?+zhj z;WRV}UXs_MGfT6fNyDs)X>jcw7xE6GZt zk2Nv4`S9z&RLVYly7UcEXk;Wb1>6)SZpT>}jTvH=Kmrj?r0*DZX;wqRo1R~h#Np-D z6(M=cW?alWMT$$NMg6otH&e{Kbl&wsmeYjHG(DWb>R!YFq118>=l3VfA~UBwKxJ|z zsjsw`>%{K>P4Q7OBGrma)j}6$p!I;-I)a2#+LnY4pUuLbR};~)l|(TFZ69yH#NW$I z(dRNWP$7b-1lPG#c%WB1i}(wM5-k8D2NIyOkMTC42-b7~3^K{2+bG3r$m4M7`cF5*;QBx2YS6urHKh&-$tkf!d9^qoM&NWPAGM+*?U(c zh2K8l3G49_o#usLj=J7gs4TlXJ}JIPR2mR$MeD!w=DB%ew;&s_Bv9Bk3&+mne)Pg7 zvJO~!&dxoJqlo)PYLp=#GN|;;Q}s$eP)DE)vjhjVY~3V6I~_$3 z25!%dvx-GBpzX;^GS90@5CK9r1T)kY**LkmOtn-oH=@S0aQv}V8+af}C!LkC$bf%R z?*REK_!G~R{4!`8Sk<9A!G}MPWg{6fau*A73n`tefl~A_q!xw@^8tyQ^HrUz_52EAtEPCi%yR`5%{jATEHU3$A-s3qoS-iMlKw=)&IpXl!CGTq#E1hr* z5OTXO?12gaUTFF)#D=)4%z#bkZe&+p?c;NqpBFFxKZ0)N{EPYIUj4WHhfk`gJ5fC-!L&ln=q^^IF`5GvP#A#A-a~?wgt0oIK6yJMU z=O>aCg=1|f@5E;93XtkV54_e^BIEVKUE9$fYPV&W4vG1Mo|k(ISYCJwckTy=z7ui! zgK&giyuJ@2{7ZthLRaEPfd&Hh;kt*I%5bbw#p3JzlUGN3oPjk&lLPG?PE42&wHL$% zVTod}R2372^os--ne?`ZYZ*-Y=zEnwJAlLzgEb|MiaVksLb4i$Ux$Y9*5;Xtwnkt3 z95t0v;5LoKGeWswIgB$rm&Uwt2YN(~`YErN7;iZD%e{8YclAR9bMrjg5NUpD;5>7*KiKljVS8DbW($9yjlvDo9>QPY+H~S^DFJpmt0;&w>^N2^2{$ zz>`ra1BvxPcVXuQFBStKd!MR}%Aa`T?N;fhmUUp_H*45DjX5p;?tO(a_cL{rWpujI%yQIn>=C|m8v9sQIcVU8By2?C zR=Bm&@Ln|fk@&P|kQ-Jd$~`!i!^7b_m9yh*O?vg{0ARKoQBtIfivXe&oDik-6~nS6 z3SiESGwMpH%R`6S;5YcVO@Wl7Z2~0Q$OdY^?W&^hQr^NpQpBCty8`oX>5$iYl_N4m zE*I%YJ>8xG(uYjBfmzdOk{R7tUj~lLDgqrDmF(D_kBji<%PR~eQNB#ZJ<7LS0UUX} zP1LiN;w&kI=P>Jr_q_m8l8r@^QU0+Hhump zbWBFp!>2M(hyS=ktHE+CIf<&^&@(FHr=XBsaSPMbG*`$vB=fvvoo8O9lL{=4Y9`fq& z&r(i-YG@SACWDz)ST(H-qBqtBGdRT1;nop6*lGgaxVuDYo0^Yl2Bgwo8DF{9XWILs zvNy%r`sBplzbOf$Jm*@E8Pf%nk@iH`HY0CF z%8f06Z86L%>O2g!(1jIJL^>}0@Br5p)a}3x19d5Q&~FDJLZ>O5%;!Vis9#|*Hwu3a zLQ$nbFlnNS5-L8UUMRNOm#A$I+-Gq`2m^+7V3!8w>C@-t>QA?3k1cRABGD}#f!>eb z&{C)UB$3GRhO z!AOsI4nKBG72h|8&Q9PtI7iM~U z^HK2!mIo_^l!&f}sw-(g*T^Vg4a}FgeAWAc?pk=mYOl59v{Y`~CwV3|&q&)9QiRA^ zwL-VTd}YrOY|zWL9k|OhF1Ud$3zRQN;GerLspE=N$ukkaKNrxidJ^}lt%Eq^3}5kD z4}d{`)&O1=CFbFJH4=yZi`|8B96}6Q)w>29REZs}0EySqdheE0MI13nCEuXSC=d7QirOZ!f9 zC@m5Jv}cid#v~lzOO#!cS@vZlegnU>Sq-m71KpXH6=%hNNHL?#|uv6VG>gj?=C1aVt%sDhXRuTAs|r zuL_>Q^JG+w5S#cH$B$`|^BFNkYDAKm-bxw1TbO&ENx>exiyOin|4H7UK+W|yBrgM@ zYQ;D^CfQ=qzo2)g1vk4w7zea@^zy;dT+HIw<@=Rs75$$QEF)V-Q@Lnz>8D6mh4hPj z*8fD!L4&^ZJw1Em{AW6QpxGU2``!*W4ykh+G(yqR7UAUL`507O^LF9xyQ@D6;V6cS zk(JcuGLDmBDai&lcNP&{jqb8v;2)dyoOdsxi{z)V(r~}T6%|ku)#4u{3KhU69a*W) z<5h+G2>GEl%72lvm5bqZFgKI7vv(z6a)jGJx#Ug~A$U`jJKR!&% z>@-!VMl7pL&Y=$Zf|;SzhzJoBhaEpFGkf+x(gYR*KLcEM<`fSzcsj;gg|S;!ql0X< z-TuT81tBn{Vr3I^c5<9Z>|6dB%V8_8B~AnLVl&-zoU}bIx}{5($6vi9^7s<5PB*JD zsySC6+A`hYsQntnufzYgJoLmJHS`Vy(Z+2fNNM1}_OAO27HC_rSW+e-go6c=1S>RI zau#>KtLq2iBGm7@ZL-0N>N6pl`H-|KM{lh;ksA0XYzMJWFgBSCO7V+C2rX$5N^p=+ zI_+s95ljql^H2GX$et{dMd_0M%^UsiSLH|rv;2QbiH>i^jD;Jq&$Q-!pOl7* z4sVg0s*$`~@;aLXujBDn{#}(PKm0r54g8`SgNx^pw#;xjII74Y_JaIRb6s=LmQii?-zLwvy0&%!JtsG32$N8KJAX^sjz@xsef1xcBsymKl>6T^1c#q)DXpSxfaGjIBa(+vVof9fVnkas?uS<$Pq<# z1_sIhF4aHepQzfi?K@$omfT|2+ga7rQDVy*WT%cX%O+X7g~o;-V$GiVS-yShgse1r zx>;j-`ly)miB2bfeU@-R8u!{!Dx2N1|2sb_#&DJyx_8x7MvOyF$N_f1fd73!s(l(& z$~_Wc3pI}IyKn`wB#VE#PKWdgor6>eFL5lHyX=AQ)$||79@QUex6Eie_f|jk?FS^_ zciV(lN)>m9hOCW0$58CR?2t>?Vnl=3kmla7iqU4ssG17f`vGR5IC6sNM(S5~u9#m* zR0(!yRSztlKT8xyL_WoGJZlyEgW@0kx=Ot zRQ(8LeDlLA%%&^>2ZB}Bk4Eu?45tUKs+B88;?S0#X9o2AfA3vfOAAb6U# zns@h1;lZ7zo5U?GCN4+cc5Q!apTArVi|Xs9>ltSEy$Dw07seUC zOR)frpiLSUTffgU{ZDZf3a6M%LM9o@BKNsSiMLn#7~y$sPYjJ$b`sF_r+LX!&lqX7 z%)qNq*ctWO;kBsm9O&L$(s@kuvIo+Au1K$O-g<3}W)_({zhM881xD0>Xosq)a9LQk zivEUxwe01s=ezMAI8=YHbuV43@E6Ac7KJ=vy0DajOcjv`my3UVJCih z|Ispr*hgsmneiE8ZmCJm-nPPep1A0{1YbXrg!9(DPb*HKP;}E`pl}OUKh)YSd8`UW@Dy0gm zJ+=_uyZ^`E1-|-iUa(+Cv9uM7J*g^)wVf>xxtlx?${YlF#vY}s!eHwJLwjf@(v`~K zLw-XS@Db=Kl){Q1zxgmPMfU_SsBBzA->|?sL@#GSpL>H*4dYbU@zpf&p(4ru(zg&S z@>&0a3DxKNp>gk}o8n2sGo1#|ba!0bXq zZXvTb7|4?lHk9pL#;HCkPP%=zL@{P*X}0)@H%Bx{HAwN>S+?fV4Rr<&oD;f@lG606 z6fP{)%_iblucoA$!Yzq7i#-8|8@xTFn=5XlIUn1};aMieQ-G2dn$XgdrV&->-D@N3o&B2Nr*eHNYxg${lr{bmV+xu0u2>py%e`0c9_iN}hirNNZuv1e>as zZ<%XQ;PhER(rh0dL%y8wBqrk=5@!mYD^$5Y7X%x!a4HQN%2--Ec)yY&VS@EX^F7c5BOo^TS`bk{^NSxvMtJw$@n;0%U=iXxxGR@#5S zV@@Rs4hSAZ&|Pm#aVw^bs`V;1hU3!oEOX0+*MQY=;oJhlm~!0&xEAgpYKx+_i8d9{ zAN6Tnug^$JV}H^(@v50sG7d@*+xz`@G9OL6hAZn zE)Ao{$AlLe@7eHixXjwMs>JDsyANzS4AacuF&lueebh(=EAaSqmjxWr`~3%L81<&j zGGXw0*_>f&@SOB$KS}=p&E4`V{D7C4Z^5Gaqr|!2B*i{lO$_H^$*(mCY)2YQM70H@ zNrHx|irFMNIK9pzC?`J)q@C0*sDDnc<|XY$%S*w&xnMV&vchvlNYqi}>g)b!`=}cM z@=Pu07Gfsar__X-PWytvq@4LjQa1%3OvjXLopgQ+e`8m{7)2J{2Zr&DkX9wCwHU6* zf#up2&xzcAa0sCHVedmJ72&KS#<_m35UA)(V$SCbdymOU`Wkz#<{VS;=Wvaf7;n7N zM#Lp5st60JV!N?1xIO4$Xm!*uiKw`&iqB|cJv16oAN(@Cr+ATQ<|z9QH8vye+p1~R z8fcbW)j5qTLgG3@#uB?o=vL9s-qN6CgmynUEqVv;FJ9QEN_~L(yf7C?PCI<~F73X1 z8v%P~iMWt*8Y$q!Ij*FQHPe1w(UF?#aeT+t|Ki0OfIOS)n?RcAzsb|cdr!>b8_Ppq zq|1U&&k?Qi!`X%AV9J?<=8uHg#K({FGfagUyKqca3TgPnUSF&b56Ox06UILhG>CR+ z-YFPH1pfSMK;bVU6a=0B6t>79GJ%HP{)1G)Wn~A6t^`WrFh>pI`>Ca8<<8i zHxHu(>jq$Ykjs-Kd1Z)y(h}W$4vg%?AogFsQ+s*f>9Bn_?}34<$jOYB5HF=Ce*lu1 zgJg&g5Vd0KqW`vAHK!`d%z_E{Fz8A<9=}CTxc-q5Cgp(tx8zv<7;5VWhA%w?R1X1u zUcH{C-4>(uJxhVb8hjhHQ+b|ePOkVqu1^=@>H~9cKsLwYyNQX(=Qs#d`SX3loAkcm zJced4BUZ^a(xwhC@4j9yJZ17Fv4efjfrjl&ksR(&yL1!D6G#-bH?0*R1h`o$@x!DM zQA&I@b{Pt`7X@@tCz)muTS&Nwi*K zu1aw+)>F2V%AmffQ^8}Z!007%ZSk0;iPdtOBUMJLJu$1VK3ub$a5CxroUyx7|E4c+ z{#;ZM1-Q};cOy%M+_MbAWL>(}%6%S=E2f*#i4*x4YHo|DW@ahUGHWp%ODjv;o@566 zz`^v-EU{-Wb987(Tsu$!)#W^37;=M5b1UsmM8n5Lkyx_-pYLnNv$}m|(yF#bW zWP9w;M!c1Hu7eAspY}*ZM@&coO1_xkV~aq5+-uK!g~W*E%j<<^f^LAZp!LzA&Ea87 z7q9HnN%Q`PPTtv}LLdz2;fx0ziFzQ4MN|xKZkhzd8-RYL5N#-@-pX{cP9rr-v*~dI zADrXVsq_p4pJouoWQ-GdyU$~SF8(N36C3#k;rI1WnsBIbJ`12 zWjTEw+R$Q>_1K|cSkS{PxKn+W@q~|JiYhnpdEWnf?p*9+(@y0mQD-*5ITcB1RBJxt zHGk4$rX3(XNZcYK34hC`f?AoC(S2pNV!DC(jfMSf;y0`nF$Haz8v*)*^AdZKQ9Brw zaDQiD18un6HfEkEhNyfnVvy{+Ys`k;B4E}i0pSfw0vZkZ5{Kci>6&X&N@Zh*!B?f~ z#&VIDSbN=~ziQ2=!QQ_P0aZy-v#bwU3!&w(d#zOI-hb=3Vc7xDq%!zBnq9X-D+0gM zCB>n(q<@vL1wnXk{HNur;c8!?+^4>l^nB4HW9i|b-X!utoUD1ZKC-MrRjJ!@!s6_DhVSZIXi@BA$(&BQ1 zr3F}8Q>O=6tD_%{$6X&Q%Yqz9xVfK+yWR;)G8dAi3rOl0q2 zg=OfAY!rY?S8jH4e1o4%IpS~`=iX4-Uz3^PR*JhXM1%RCGQWC*ylK^|%j1PYZTAwd z+Fmgy=z8cXoDvnbT;f+6efY46GmyIeAWWIrwH(Xs^Fo^uO6a!8OJqw)Bm|YD}o`D7SI*{9N5xHv>wl;qlnKjef*$y_>xi$aLU=wBVLT! z8g}PELDHBuXno z%VS-`BNeh3-8BzXAaUQ5DZ^dmrGKu(9q!o<1j~}D9)I-wb)2C_N}c>R<PCM7%jRf{?sH$5`jh#0;}ZlbA!8op{5jn9;hWm-LDv;5e><5 zD=hx`*_2FRTTAlmF1NQ;;UMSwcC=xMEy<_Qt7y~@H`5euANaIAnu@llgaqi58Yijj zKd7P{2u;j!lB5mwQJwC1UX>B9O2sA&XG5cG51+im}ew~O&n+$3X|KU$f zgCVgBU}yW|@;!)L0zBWB))PgkdwQ3S0vxQURlssc)Uu6jtccUEj}s2`Nw76%*9)~OO)1X8>1$Q51F$$b#~2v#vImjEoQ*Y&?; z$di$xUdl>2kaJguWpfqSuxAo)3e|iaS};d8vfK_1a}hi?juG<_l3ftorP>t@A!4uo zOzd3;YNM{k&;6NcS*)UszD|Y3Vxoj2^@`pVYtM8=LpMHN@ZjMr2tOu?FvvFg2fPF9 zAH=VRqk8pcBL|`>3O}q!SgBr0%CjTC-qqK@Q{#QL>Ha(Pn6P-E2m}EOs-hL(XC`74 zMzMb)V1EE9^dJA1@vAU~(s6HeET)J=7QL#N9sFUN-fnoxcs8w~>!I zwMo>6D|(gMO`x`L^>BXgGZ9i0YZi^pKz^u|Dv`V?mh!O^tk`*h^HB9bf#>WheR}SR z+1g~)xb)g2=)C3|^X3O9NS=QaS-_t0iKu$!($HT2%!$Cg31onij^DR0?s3B5ZO8n} zZV5&Dy0Kx$Q8m=#OoPQ)*38yi(l(ak!g!wmH&NE5eS9g^&!%|@NCo~zuvtm>SkKZ( z7+n9BYLt%dByLBtZJg%Es5i+V1NI?{m532meC8H1`B)4RKi7;U98?N7R{VsOEVmZu zMX#NOM(5KK?&*(v;Avh|b6OOgt!$hEMaZa*H-j<-9><$S4r~CZsy<{yO5TuOa{)Na zsK37YR0q6V$jidJ)eT03#xAPV{9DOQFRbsc4PRkm!`epbMN=+B`t`95YHB%x9efqn zC`Fv2j$#u>KXt2=%g8&Bi=ssocHKx->8AP>h^|#__U>mBM+HjW+x9^186NzFbhr|U zh9p)4gf&Za(0O3|Un%xwSW|kAiCt~kR0~R+up8}xpbw&IgMI!Mcw>3JHz>3I10Pn)Y$y94DIQ2G&3UeT~ z(nY+>LPc6pp+ceP%OI?E&f^91JXW_ersBc!mPw}@qxfB3aqsuR$Kw8Ud_PDwU9gwX zjd$_wf$mfsVm42w`Mj=_Lc(jY_MFQLFDcZsbP;05dD9QMWA>uw)5j=<;TkDJpw^@o zPAmJ|>?5t&$$*Z!t8ebQ_MsQ{%QS2XRpmVpKlzXvlGPz_!N8wHP->`psP{*DtH{H- zqM-}7iYt}S8}E+E5H3@&m)a;4RwftPI#W>#D$^SS-pkNE+`)p^MjO0+e>DVP-Z%Xi zQFYmzMPWDC1S<256kcs&zZ-R9w5vh$8#c`OPlm4C9gvUy?%p`qvoJFc>x^akxSD)C zVi~)k%ss7OJ7myIGTD2*=#N`HCcfY?p!fuV2mu=1@wK6U_eFIQJf3bzCn z_2bxVi{pJn2T9p?6>@oqx8}6@g`a!W@{IMjBuKR2rwEkr^wsc8ujK~;AlXswIDiS# zhdTw{7qQRt&b;N*L;f-7%^J);r<(ltP>nNmK0M-NCXhbeSLmJO&e;eC-VZ0&bV838m+*WI zw$3QV-gGi3E|+q$T)PqLz=QL{;4G8LVQu zhw^!GvQFK333=UuAD$oP9Z_2Od}Ss0>WHM%=lUVZRmY26qcfH4B?f} zG$}mB2~eu~Af^D}2$VRkgvdI}M?iZTe}`r}R=+gDWKX|iYfO6C95wCO9S@6r%1@*) zxI03d&1HJraFNdc*K;C>C`JWOBE;714_D$l*}lH~O-ays7AG~Emb+;)XwEG^=_vL; zQVt56e`McjYJJe5dRNlcd9cU#gG>SJ8^QlCgWXSt$yyI#rqv2u$ZVy^%L0rmJ)#`m1&tgiZJ9U6_8qQ_SlA(1y!3c8CL|39K)hy`nV0wdAHDN{YPN?82ef4E4R&gB zs?UOFgnAd}->2^mpqe|iLGG_anbBQjuifO=UX|3~tCSK-Dtrcgyf14bLPJyR*Qv@! zf7LCA0Ydio;l~ke4-dace6bnPPYJF_wM3q#JCQQ82Zee8On1u44*kl($FV!Vmy>j6 zAUu?}D8=+5QXc32j)9^xltv}}^OI!URQJi}?M8a%H>ajiHSns@+z`z4ds!aAv)$4G zqOu;wx?@Tz=3O|HIk(0(t;fgreP9sd&?&>|G`tq*-E{slE~0TbD;g700c-bhXaDQ) zUd8dq^r_TRI7u}foXd9w1bU~iie=mPdb{IxB3LQm^2wms@h=aA#qi)^E;J2xw59Q zeCc8P*B$a*9q(E_8Ozf*1xRnZdLpnRvP7&#N|nAxjD)Kq zG@?!YyJ&AC=?fsKyQ#f+yI-^UV?Q-|0qoOfq-p`9mo$C3f01&E{yBhUJ!FN5t7R6b zXQAmoA5^@D<{L8O8ml5_#-ZF#BE!e%8Mo{}@fD$dMu~~@KEIYV-(kzpyTf8P;?~|L+7B2oPStSH+Peb2G zfGt*#Eh}_ux{Pf4aw^!Ox>#gIadUroZYe|yJ~Bj`$M!8A zdr^A=$+~a0_8DV`Dw$QUEzgw4?GgcE02h_DRLzYdwDBXcW@=w6ZTVd=9sNN~rb<{N z3@cd1rwJ~MwZt5GjmVq|$IbyW=By$gxc>g&r4zw)E2_W}AL0rnuk;iF9us}TI+5iI z$@oWOInur;QD9P)8An}2OMS(HxK%_{gAHjLY*>^|y#p$rhKhw=1#*c`(8bd_Gt04a zqG}HO7JCONaUh}{D_j3_8g7>0#xy50- zvCeEL1jELbvf?!o*neugS#3~c%y+Ahr4f-=--maCswGm$*A9WHC~<@VTPY9s(^Be6 zNivL_CWCz8bq!_EGDQ>vZH9<=O5IRj5GU--h^S*bm(gP*r7M*+ZUpeXz;P8O_akxCMAViN4(& z&?MvfCcRYWJE!lx6!DbMv!|6m8jt?yfm_ej72y=k)nAHd-7)D}%XtwI)dgcw9Tqh@ z&zzhUW1zykcl?YrejM09*&gOBb^^Z8%wn-kt@P3!+N;M*a26W=sjO1?t048dwN!Dm zS?-%li9~5(lVxsRu3YUUHX-Kpr{NgZG)|GBIy5}Z44MyjbYAI5y>VdP0MI;rw`kSf zRZ#p-q*;AfHEaX~_D!_PhVC!xpLZ1_?q%lGarda0BZ0rtqAqqUGdt_&heEnAWV>8r zP;)im@n_XHaO&Nqi7^X`tXzD9F%G)g*T?ZkNLx!Pb6@vwT*;3193NrvzU+`tSW^2K zL3f7w6M#P129#7>HHk-v!5IMZ(o2NzO`tmtld(2v5o&n zW))@H4CNaFb2h{3!k_MxYx%D~R$gbYY}-77_rL18deo^_h^i6eR(EkGN+HEH81Nb} zKCVO9uaYuzvYg1tRO(K8^)@kaES5c)H2HrurM9XS1q|ZU%1tRG6N?QY7OVurW4KBp zBdqN*k0#&jPm~ zEy}U$rM<=3kV`_2IEt6gejD%mBvxB=gS_Qg8|&-Bbgt;UnSIarl zixFr@liCuk7HM^eaQ&u4{#)4a^L7i3yQYd66OksLgqvnT=nU>7p(5yS(Y&)NjJ)-T zXw;GgOgdupiJDZ0gYQ@9NiZQMy@?W&2+4?e+y|d3YO5!(Ufyk|*{^E^+F9fvydo@> z9NnIRHo3$xnezmVa|SdU*JUSI$>M9OoKdRkxDmSV7gOA4p&z&e4=ijlYG^u)%Ul~7 z@I;!@HId5oCT?mFAA~Go39dMGyi7I*&xz+;FD6-4O$n-n^$T&apmJCP>EZM%wRGSb zwdVFqz2L!Z36s~LR)fu5kohM`dNp)P@s)1PMOT(${Pt+bQLg(F@JfKt^MZ`?MD{|H zj%z&1;xo_1;Pn0Q$cvZEb zh|Hbs6B4m~`UUPjMa_ZO?mrYL-im)jMbuPl)$LT3WY^Wsb}n_PJz0LH3XtB@U*Ey9 z9)(^%q}Z6LOV44I;K<{-i7O&OlF=_TT}3L}`St%dy;S3xewtGzk@ZN60?1uv=`P!O zYz^=Ay5=jjxcbxAXFsN1K6YNxvZaV!K3B;*pF4x=N=Y<6_`kk=Czf#7&#FW0c?A8_ zmw-M}-P2h*I)=|E3!?`xWca7rf0=L&zgF~zQ1D$vw*J}0(XvATqT2_6A6L5l$DLp_ zwO5vyL>uG?$Q|lo1UA+r%ogOHw^esuuLBYVk#-@N#4;Kw>VleUj&W3%D0NRP))_%g zz~PFQp<~Kz``TU70A$L_otGK=FlHFw3{OZ zqPAo8Q|-M&FyWBwB-9ktf{9$Ocij@H@raw8Pvawd|EJg1f7kr_PJqP4%K{x%#h`jC z46iMDFLd!3zif>=y^x-3H`dh2vK3m}(;|(?2;sUb^3i*hVrxu@M2~&Tn6!#M5ET)! zF{ug|^>}8Lff?|IP{J1R1^P5hcP=Bo@M4*0Y^L*l6svEU9zln8nKP!|bAgfY$1&jX zweI<%w8b$GT1M^fZajQVMY1dH9qT{&qw55CtqAb*6=zbfzWD_b8GUL2ABDTwIM&-` zdW~;Tb*{?@pKVoq0B(*pMs~20eX^CVS$-z?gk8?wixV z#v^2@L?Ys$+7y3p^Up;?W~cFXJ?j@Za)KAV(3QlH%BN`>Lzn9-+cD=1^({#sB?O$` z)0pUmC93VIR;)gV|Y*9}J_;oyPR0MExH}#G!48FAP|JbB0V|bm> z3?KU=OLLODC3dCtm9$iq8O!HI4>6tVxJvKLzEdWt0gy(A0T-A?j0D(n@*Cg=r;GRD zR=4PSENm=MZpO&+i(AJiMHK=l*=#gg@pG}>r`Rv?M0cLtig3eldXYK#ADcN3$ix1N z7YF9KQb$R<)$kRU6bZ8RFURWC?LrMCa7Ef%;l{gaYlmz_8^$PDEibb$R#A=$Fc@-) zXWc|G2k<*u?&j8)GA<~TeArtF^TvSYWb>P$(E{JP=qI&JF3@yUOlpw95SO5Hoh@&I z^`gc0mopE{6OMfi?jgdFa#lz3BuQlCf2PgYQdKtB|D5CKMfe231)?U#U|84iU$H?k zclAXvXdBpogE%2(1aM-cGmbYuH2?RMcdZ07erEaGC&41=xv8yBR#k6~=3=tD;jYEB zIqqJ0ad0qyk2zee5Lt#*V}=}A$krUF-H2HQNl9ZrkSw5kx}nnrKq)i;N}i%5R^u z6mbLnzE`^bXvI0Gsa2-=Q3dN19QYaV7rO1oHqv|kv}yH0;m*CAFYF&9i^rCCo+1pn zZbopTDp8odn^|1|G{P3JR9)Yw>$bi{XD_y)wrU>o+m-lpQ)2Qp=Ze+3?bN&F9A6?J zNb(HV0F13Itv*lm|0LL^$Qy(mN3$en*jSnP7S_L}zF>j~7>OL-UA3E@6J#`C5lT!4 zIPU;s>S~XQ;!>I@r`1YY1G4w?PT3e8O^Hj;4*! zv|2K=gc~_fQX*8}@U{fg-D{ggxQi+95izzpli~JZHhvXkVl&yElm4W`Xsil?1YOds z_xRZd^J_8roj&{~<$|dJF-(sR>9N5X-TnC8%$cZp9)o$J4GXaSn0jgg6-ns%R->c_ z;$3RHLu44oNSrwbB3-LP3SC$OVp@{Rk^(Y> zt*pO@D}wnC`D7i;QB`w+nSVwFp;u@MOhn)6IbmpY3CWc6!+Ke70`*9#>9tUzysgxS zCzG*6#aormvJk?6KPAH)Q%5?nfzBhe)BNu4%%%dmAW#)5H!3@}v<~k@(FsYNZzc0(=A`bQjR`a$(!>`2{Kogaj^i zm}kBkc!h0bBBxCiS+!sN`yi(hztZQH^SJ8z^V2Ft2DS%xos8djM@=*iT!S{arJjkM z5rnf_&**NsfpJL9E|gDY9+SphiG>r0fMYZe8OHR#S(ykr&wm&CZJsg8=ZSp^-qGph zO06B((I*(nJ%s7?gUCu1r_c0tuut$aJfls6fCrXCq|gu|`zf|)1wn#IZr!g?e5F$L zcv~BdJUb2!zQ_}eoL_<3*hI*=U9{ar=Jl$LjYuO7cEB>RhIYn{@*njmKnzS}t<%V94AZ7@A_-=e7EMx;8UbeUhhlmAVrY+ys= zDVSaISZ_8sPxAH$;2j45dqPLna-}Q{s$G6>oJ9Qcse}E6r6#q!R@cptoC?OW)`Gz= zBIWSfISTL^H)yu^=QPCCD^WY_^x=MG9oQ!oEzGu7Y}$TxrP@*Aj^q=eSd?J}qN7y5CfvgM%2O*sJs#C@G{6b&AC zKn6-QV1@MkQ?y;;?I-L%*wjqma?qt*;~;%)N_J*RQ`Im;FTdScC6r7FiB7@bfm3!h zokt)oKSOp-u)wcV<9K&>U1gcP7l-Mz#fa}}{}#;+{#iZt;PGi;mmEdwHgp%bze~Bd zWbf=Z-3awPEWML;aY_yTc=X8J=|}Cs7@U>4r&O>yb2Ob>sb9rdt{U>&_%hGIz}5JEeTMocV(4KAOl* z5PQIF2E5ME@sS4N!|$OsxI*qE$)I~FWh;EL}e z-Mr_bQRwIL=4C=eDU;s^2ljy$2T8+Zq4?m}UZ#q35A+9lP2tTVH}hY$B`~zz^Z}y_ zCcYE1nJcE4bdN9ApBLcSt<$?g!+OJ`O9J zS-~)}9kq(zvaxn0b-iaGq%Oo-H$jtH^!j0jSiZmi%m6A@l$EQdiJ@CMY!HUwvLVY9 z9&Y8|(IiMUgEU{OvOgzh-D%{gpPK4D39FY+^mDzINeY4FvaL`u_cm6m?@KbCFKvQZXm&3HY|qBS*;wE? z=UfQX%9KKzGj^JvkyFf~dfFgLP3)K$hfeWlmdbn0&Fu%hP^k9r+hzOmzy>6po%Ul3 zgb4Cr-F=O+1Jo8Y&r2`aT^8a>n?7$gtXiHsTeUbu_&NaiXXayN{HKdJDsx)%FnK5Y3l?rvP53`2a=Q+AjYB zThC}^ZUkD7|7CIdMIP4w{O1r9p?FMDedN6hMt|eD+xu{sR6As?%$uOuU{&6 z)fUCh4V?^13E~$ATtNzE3PIv(tDuLt^o-AnqCOVij0Z^ z8+J{N3Pmob)?Z(rLJ4-h-vW!?adTX2=KBp}P`|vQGYH7R{l(wi_TyRp=0=uVvc`$W zZTo$voaqj^9O+8u5{i>uXEs&IjVu_v@ zRk_G#6nde$=yNy6vO#X`i#BHI@N%8Z&OY3--b)BD=|<$~YyXW^Tb2W6oXMZ42Gji6 zSVS)_j5I4xS7C;jYQQ%@xd|{m-($f*8jxR?fZR@=Nl>eyt0=Mz%yhr?L+8Q$leOU6EyNh6R@j!l__%goCdh(jh@AUwAZT9QPTQ#Iwhx;r@p)X zSK2a}(FkS;ZD_Zg#0j&D4g%Y55*jC|aNdG=gCwxUvhn3m$8B^%9S4H-qtVY`?U!JiCUqdcy6n|XIOOww_bWGO`Bv%f`xaG@R@!mz2F20i)ggBy2 ztnm0gm-^}VHE>K9y*=~ldWCf5J^LD*!(>^0)muM2yh2Mw^-CfE&R~G)Y8%oK?%i6R z=P=+l2VUo!@Faz{LkRR7XP>(wwb&vXk)qofEA>~RsDF-?JhRMc3LgW*@;%A|sZ@(V zUia6t$|satV`A*B(I7d4L@^5kBSWyK9R;K8o>48|TKMF4B!ip*y zAEdZ%Lh^2mkqMP6GGHoB_Ztz9Npo49C zBN@_VE6q@51Nc4vk(i*!xdBhnS-wxfaTmi=Zf&)2qhGag=c*V!X7+X*^eP{AsR4qX zLJIgXb(QKfy~qV6wrjpb%lYD0Nc3Fn)opRsUx?la?z?VyRBsR;+iaMnBq= zHSJI`UPT3#T#m_Q!y}7cHdKPwFwlT*R%cjdq~6C{#4;`~rRPOk&3LYtUlM==FQNz+ z2#!-n%1TzqRRiNJ!$F@N3Z|R#Wj9T>1F9QRIT*Hk@70(pXZl76rukEQxgOXa zH_|Kd!zO3h2M8tQ8!^Th7pC^Li#ERi$KTL+!*9J**0eGH&p@33P0%PGl2M(bC6&7U zNl7^(NI~ej`m9twJiRjyXEOGHH{XyQHP)#y&Mm(^`b~Px<%~QAyX8_^2cPqg&TY-K z&9;1(i$hl8r9#TLe0(0l&u^Ou4hpKh{2t6y!$kvrryuU--z7?6!l(zkzThuIZ9-h4p zU&s^lzqzc9cJ!~zw)`zb#z=A<;yAD^hAV#)nSVNT3?n{ueYd*&9{UPOET7Bx#zVR6 z`;wR^?qs?2FoX7pkuj>tBLieJgNrtp$SNbQ? zYYRUsLo($-5NJB7Hu~qoxEvNLMwE4@pCK)vfr&u*&3I?!T5|=2P%=tzAN|VxkZsJm z!8c@UtACXfYRR>a3%fLigi|QFT3uKA&q>_1k`Aa&86K8o0_RtRW|V%ykH0>MZv$${ zfQ~uCFMojJ*{)qINnK-2WLAv7^i%PpKx@&AP=#r)v)7UO<5=Ee^3u5#I@n(^!vWc+ z)}yj+n8Df_DYWDy!w8pYke6tRb5rk)mrKOIxF%4?d;w_Edjx|8&%#l>&t(5e62OhO z6f94?_OllgmTqAWUA&j&@Rr1)N`D5j2U|IR=eL2=oafY^E6*6fO`rd`qUcmdOfY`H zSP}sabgjR?UGaa;Mhgh?#pRjE!F9G{XnA#-HZp^Vfp)P3rIQCh#r-?7y!NBFDvpUz zMrAtpx|YYwnh+S>j;N}jzdUc1 zDJ?Wu<0}l!p%HIo8MS(54EEKQT{u-n;rBWSaUtAUWmtW=m)>dRC1!S9m=y4t zJ@?W688s8J4Omu_@MX%afl5R)gXbab-ols@fJAc18KwF1W#O|-MK^-0hUM@d96P|* zFKH&NH0u3V17%nXv9)x5<11S83oklqz;mxIlTL}k!1m+aueEe@1tIWNTEL6rX68uM zDpLY{`<>}{F#iW+*AXYmUJ%esWQzFDLwDRgYR*>kCvsG0ys!GS(L)^y(8b5wqd!Yv zFayIRAtVGsW%UgvG|V|~ur2r}@ZIGx8eZj)cQvs48KpLaI@ypH>?e)AOEy&joZ~Ys z?*~x+n>Yu0{exoD=_OXU(2322y}G$TF^F_QW1jx?&6}JiH>Hu=99;si0N2vmRBpWYfA&84JiLNx-Bs81@{(of!88t0 zEU+tFQYQTSj2qI}W3=E)(}fdHU)t(#?L6RtVAPp|v=eN#^T#*4f39rt=O&F8y(WY~ z6LXxE>+Q+dZNst&_$P0Crg#6&6dA7~B;7VlbY~8IKIvxoriNdu)ggZ^O{GQ#1m0G( zlhIS#w(1PmXF*jV0X9M{utq?^X3tZwOYSch_SPrlGC?7A9n$Kv=8Xrp zp?}mnOw2mr(Pi}Qzf<}v70(;f)Xv$WD6$e|>RMhQm^l~v9z`qaDJvRO=leyCuJb&2 zrp4XUL8V+Sj2ijc!=aB5@ds`M0;52y81IrzjRf9Q(Z$S?~^!01IfqJrg79ui^E8~>yf2l*E0rGVsI zC#-Cy7_8hy)yL$mT9JFqLX?-8PLx%%E>E&-v9$e}yRs_BcXD(v#atx%fJ&i8hYU=oCC&gPr6*qk%AF5(}&8SbEpnt<4(#5i+}Ge*p}sconr7xzIT5NOeWN_XbK3!8*j~ zXgemfZh5WORcCUqWF0up;n+W(Ycd9Os7w8IASUuwX?^obCmRa}yJ!;bxaLfYa94(d z;P#x*QtxwbQQ_2anWpv6!ZPK$s+-vf+nnQDzuIn!<*VHAxMx3Ga?p!yGRyEyOC!x9 zBA}wC*z-#Cp6eM4lmNmmr~s6d6}ylY>J_GR-tr?lGwXuPu@N zct&CWsE^q_bHLd%j%gAi(z)!-sFeEopV`C2O*!27?FV{!SZgWXno=O1&C>_LAly~Mk= zn@C?oze#kw{*i!rroXBGZ#kC!X(lf-t7UDu*O$AzMuanN50E(GKp;E4`==a+VKFNk z8Xqn*`fMl0ak83uP>d?v3=e50AI_^e;?QKS7(tY102>w4a02tQmHmC@P3~SU!8Bn? ze(6YBWrcB49-=3q@%j%@M+GCibYYBT4me*n>DU&oRO)7!D`Q~TJOfTv;WK(e!o;O_ z*$u;qYsfQk!EFc;SC8ouxg7a)|M|BN^qsV(g?v{z0HejT?|~lEohsZ=_lY{C-b^7~ z4-RJ$p+m$^Ob=>4oV>$cknpaaf_wuvBlAg1ADJ3_oC!8RQjLFO1QZO>Gk!!6n6pX@ zk-u{;y&P9x{W+L}T&?qxi~uyN&u!jy7_Lr#XbNb66p18~O;}4)B@Nv#!nELz%u!({ z3<2ziqHx4dF~BzQz!ep4H7~4Q(pAb(kvy>OVnX>69?wYW{=+u}ts+hlW0qwrKlZt7 zMO3G>b9L;$&`m2WJ3U7lW10X#(kc7-WB79Gqruj$r36yZ=m?Wgq*5RGAfenno=FRP zBP?q~pr4dXu@YlZL4dlgQYEjBkQ>^(= zyOv5Nl#aMDm4h(-?3~e3+&_2bmm@gF(pj-SSlR{K=FLHL@|rV>sq$-?oE<|+u?-_3 zPWK~`*Z&c9&3}2m@qS2)%eCB=t>s!=ww7(%Et|`>v21I(mTlLQv3RcUIX|3#;CWuJ z`@Zh$gSYZ7(?t3okJLjarVOWS7ioX&l}v=Oe79jQY!ojO8Bxl1{_+f>m|!TtRXbPW zWnUsjs3HGuD?TInCrRWSTkRyW(xTu%9$r!gTAT#Iiaq(fIEr8Nqmb0U#A!|tKgCBH zWt!8MG1x`I{DwMXXX_-~h9dy+0f^b=M3=qVkr2gZTS;vX`GGmw>9iYJw|La-rweFF z*;MUy2jYS~5SO^TO;eT7@Q;X3Ui5h5eturr2;YoIGXj2U<-i^~OUwBR`>bcI$i6tu zzSP(6EpD~U_-HP#iX)=Y*Xu1Oc=ZPns(O^DZ6OeCBZ`}Qee(7V7Hl(D2^WF^UXxz- zM%@AO5_ZCO3X^zN_~|)u7b0NpsTsd-Dc4=}XMl!udkAE`FW_}2 z;X-eH#jm3!OOg8XS%vGv)Y@?pktXH219|gS*I@sQ*baWAk)1~QPsPf|TGdIo0UuKX zk)OfS@6Tfk>S0M`B`!lxV13zH2xf`b z7yjr@&sQHA8a7Ns(^B6lvK~z)^wz9D>4W11u!yI4X+4%4g>3}6k6Xs}AY!&z+RJq4 zfHvT{=@-(*6>5<9&2~dF9r2Gwv)v)xu}3W)+|Xja+;BAYs|UTVF&c}$}D3ueXQ zl1zXleSUOxZToq73ngCC-C#n)=t zFRFto%z4rb#1^Na??wCGDcu+Q8MnSuxA1weR_c)4_|G-m>(sSyb-jP0dRHZkSfEwU zNHrHFi{D${-FY0Jei^;ARj~1*{{-iKl;j@&p62n6>O39UeBWq!`JYE1#ZWJXVRtVt z65kad`BR6H+6O$urCjUCPnXQzqoNTMP+1W1WUij~R@9)3@B8!j#cvD!+KWzpnU8F;QL^O{L#xLzH;6n!8S2 zcN5&Vy%29L_N?J4$nS2u>P)?PE-K6OD%Cz=EwdXyi|zl^pn^xwU`_X_;55;OvKm#m zFsJA%-ATHo9Yz%ds>0v0`(%StLYp2mYxq*&4GnucqcJ zYQ?daUf(>t?}V4ScTZ9HAKhWbGDJQX?&c;}EC8kZibX+acq-_tTP())EiPlRivQL+#N?HZB2aRJWrT71I`kxMG_2y?yEf-<39tcAud4*}=@% zB{E`sGH!0FwB$0r(Yzr;syeOI5b+W}1)VZAs!EsEhwj)*_ZA$u&sLln(~!aOYp^5u_g)}ILjb!mcNBo&O25}v)R6gqgo&hO zsQ;4|zxsU_PC39P6aG=%gnWY{Da?-$x?om0x(!b98#Lr?ya1S;iYof@yJ~Fm-^~v{ ziB}7fzT6l(XL|_8rCTX5j(p#5-gd^^+j{(Ev69r}hm2@0;r(sVKaDXc?zy)0p2A7~ z9kR64av8iZHa{KddqGj5GoYnR8W@H9bI4SAkx1*_B)La3KHP*7E#r1XPCD8C#bsAT|eYtGaVT5_xdR`IW z^GydOT65QiPha}+UFG4ukf}8`*0t3?`h=KVu0k>O*D2f!miur2sj`V5M%UqHsu1`g zAh-ZFutcm(rwCqBhS@D1YIfqcoBch0SVEG=XvSIx33bj}A3vGNa#0xpwoX&P0WN-4 zW@bjC4Xerl%XPDs^79!@|I5N9M|&ZEkVwwY5T-5GGm7B7ofTW$jjg&1*o@IPxKc9P zX!H?qz#CklM|ytj_x+qo3vsvC_6~!cWId_CvB91$_AYCJp2_bRi-05C{%N)(PC7w z9hLGK&z#j?C`;n))#&gVu*AF|2s~rRDh}yA?u!8kb5pQ8zP%p?pz`8*503;>r$yXc__)6S z45-~~V&9673HndAT%F$=4$Ph`#@7e0iL{NKQa``^X0N>mveLCp2Mx zJ;SoM9}3RF0un4YxK10aAwnm5SJnc&&WqGmtYGY}h^7Jd-J)_HMm@cu2p6eSQCNqon)a)sLc22SkIfB#z-c4oiTph` z8$leDQ@JU=Bu{_xe)1KrK74dCP`#&)y0HXhnk58Fh6`hkynoAu?>ChPStV7#;&xnBl_d(#efUO?8n zoFeB`U5x34p&cohPdH*ud?)aWB`2e-)so?Tvyy-*o44FV81=GvEQnHoQmC?_P_09H%7p(bI-#yEW^JvJ6bVJzmoV z{2sbOZ&3q2vN(w-KNq!!0-|QCBmc?Uw`hU*i$qT#IXTJH$-b{`KAIum2QeQy#3-sL zp|8PR>R+AbJR&rfaDck(DdEv{r4(*zvPvBZSg+=;|6-4!4rnvfI*9;dV{2S0j`Hul zgq-gztBCT`&y-oZB=zHQcvCs@89s(OXd*+39E^OC4dwVsI2b607N33~wVNUrw`_nI z(}3XJg1pjcJEDoHZ@F5tHwXMcrCknx{?i4=k@TM>lB}ZBRA$rjBu&)-| zp1%PXN!GRSjT;l`^R}djq{|esjY#7(QCT|Ueuj&%)Oy0_K6cqX$HgA=w~+81U9k$U zjCcI-#_YE_2Fl3PD<#usD2w{6zE$o#x?F)87lY#Kq}jzfnAUu|pKlF9&&G>twfc(Z4~fki$lTd#c7HuuI6`qggYuaT&FLqB&qeUhq8_h zZ)(xuYzXEpOM~v8v`F=mt25g<>saksQm7`zJ%tjjS_LpI99UupW?8lMeMaIPT6tj+)b+nBek+|{Y7D85Fe!Xe6vF1M=U+}SJKF98gaZ{PXp`G!M_#r*!d>dX zNH2ril)f|yhhhBmu!pEn{NZ+KXU-oaZgBF*)H4t0P5BxgZ?aekn$2C#2+tFN2_g8! zgrok?haZE03HxDXQ4kIc?!Be<^gqA7?BxdRhHIMRQ;j@?xk*doBn(3sGAez+I0q|% zBAh-7`<0oR?=1c$HE-cZSKk8!^7e29g;bjs-p?!uF1o~4HMjEK!&R%J1*W%q z)QI&7rUq{pI9h?GY2jT?L$c2xhP&^96^;a7|9675Jiywj$3MySo)M-fwO!`00G8Nd z(ieRMnm#It5~K`Pc7C0Or7XGCz-h^7{i-gnT|}jsEqc*DL#c}T>Kp!sV86VS2VIhl zaMZ(faqXNK8czM%3`*YPLP9xHD)@FvvG?c?YW6?zNT`sYfsw9h0@f7>sL`k4HW*$G zp9=|6&#pA-t1K_?tTZdP*sVsd()%W>K z-{q)Eh(w~Ze(AAFMqG!|e3<`q*WRmJjzNm=_}%ih)wHSUC+suRQHzId<8g_v2YY+f ztfG%oMIRB{@k0EkLS7>8%NzZkRn+ zIP(=gJ}a4T*2Vsa`P!T5bd#2IBfewUbK5fbugU(2pIt%-JJVHl%a7}3)*vPN?C#PV zd9i-Bdx3Gv`1A!d>dM^>wiDJT;q>PJVP`~hIraW5ZU_)zov6_ToI(hGr@~1wj+0Kl z83A04otJ|2T^29aGjKk(Mr0|!8fD}d5=;t^`R44z%)}dh`>1I74_j5TVNmtK;6!oZ zZIDO?R{*kRJEm()pQw?zs1c(Owi^jlf4{o=dm z`)z)L_dN39hL))b^+@yUNqBaY%+K@^O~E#d2xWXaMxl?gZbRGw%&5v-dEOUcmsqN@ zlf*l)G$lSbsmYutboQ1U@s|%5QWU)PYVlgRJzjP(N%$vHjhM8BP+{v&6;sgp`H8LW zfr9h@k=~Kf**+G0(wRNniq!jy6`@3O`4yB_LExow-TYJtrTzKJlZ^RGY*5je{&4zJ z<ouyp!r>egVYJgjy(}f~&|5K|Oc)v17 z=NE9!>SVD5>7(W!8`Ww-C8axNR^encljp-{!fBJ1%RZ?t-Ew`@HS}Y2LrM>r|v32E09q6-MknZehxI?=Ro(BV%LRR1r)_l^@64Y8;@#=az+GHn_1?Y8^Qlc5>N5&LCQ zc~69bAcr;1q#R7ik~ox}yWVbNORCTFAtdn(NFC9ki@8YRyz zAjt2isq~K3IA@W;k#gWn9NtrxcR3^W{)#lL8HG zxi4IFQ<1>gVR^JOT}~?))`bb}M!30CN)N6?q`mC6s4pqX!>;&NINob)Z#(n8yg94g zsJcw0z4BtPtAA(~aD}D4^5~uofH^lvTHEHs@amR9jTXQovNWcz^LEnuTIRKjU))e` zaA4ewY7hME)wxcI(iG@bL*zHY8Yw7q@#}sBNg88Kqmi!TuS7NFmsv*x#x;9#>=o}; z>Vwbi=s;S=*LJY$?gvL@C#6u06eYX-yw(Dxw3>B-6#y&ziKoSaD2f%P(r_;b%c?>Y7xP+)m)q&8qfXvop`;GY;KZ>gPiT`vLhrR z50Ds&c-l?v%@tJfvgxZ`y?Lq2CROy4D@Dhms1^ZM^ zH{pzwH0)3#K1y^rjqw9Ggd=fv8*ONaeE5H}vnZhs+M^U%@DT z3&OYEyvDQ#xiq1U3wyGN@@+q==W3XwY@(%7v1{BRcxmbZ{upHUae~ZP+VIR$HvG40 zHNy5gr3d58cZBO?__9O{jZNu!rm5=5%J)0kNs;l7vnn>g!%4?`raSwDdk|#WM;*B}sSb zPVRoraz9E)y=TIC**HUn%!Cl>sK-#^`o5bU0X$$IZOS)OSNDf&=hlE$LC0HvGl4eA z4Im3%=GqrmXCVdYB!)A$@PKkeKg0;P%vBhYRW>d;YsD5weX zb`OAO^K~?g?<2A|^m&V1wN>t;Yu!{@vIXxm<8CyWhl*KR=b)tf|+t;BmR0u~tG6ez|KTj|ikg29)0BXk4X(eUf_FeHQD!;8H~TicM$s?i-PR7P zjk%lO72^rS=1!#p+Z>xn`fuySt5~+g1uCs6PKNfP776d@m($xyCdM#a9#30mpbV`x z*T?mgy4YonH4gm()@xR0w#xSm%I$w=rdHSJ@;Xgk&x`~uHva<)HK`jwx_@cuy`7NJ zp+_+H5M?}Lu*9_m>SW9anXwsX6d`-!eSgW|Y2;@g?LRIq3k0M)UvD};#)8Qa9%_aH zg`B4P<=Z0ty02u&O=H&EC~_5-eicWKlr4JXB&X#TXn%+dms;`Z9JgLDt+Zgh zR|>qRtE`{B@E%zI>1~$KluuS)7?JAq1gOc5iNai{TbN!&Q zf=2_Uqg##4mie5N&=H3SH$vCG>f+*MGf6*8Ua7tuOjD;>`JJUpUR8twb zZTMe7ONI)PZ-0iB#Ml4GE&&E&4K5PpJ&D7yy?Z8T*{~ln{_>`ReX|I;fX8mqb}k=KlsQu`HYx)sP|T(DNdz_SZbY38 zvTW<$W{z1P4~NnIID;oO0NYej7CdmN10w&)vUU0qO04CTDv7Ao-URbS3I_HoeEmoZ zAcX6iL9G@(t)7hnkRbOL!q(SwbSPS4r#2OJkfK;luk znfZ@F@f}25XU(#>TO61n=?ECy)Md#(7x!{6X`hMYhqi?UPf!Y!8 z>Zvr(tWXPW4Ho?9)(}t+4F#y0ZUlWNLdxM+RwF#Km;aPI`H(ocJO8H|wQ!B|AB94Q z*+~#gOes)AqUArSG=N@B@PY^o8PEt8KR2r^y+kw~zp?G@j#nLR$|iQB!RkvE9Ms`0 zv|IY6wraGh8cbYiMQKfGXdd2TU@9nHc<$%G(TW*7;{B<0P$ST({h@w8b86_cG4kl) zZ>-}(HU@@FMTq)`z0ezun0X5hMfd17 zLqxEPR(|LyL@vQ1M?Jy?hNW7PA)%!{h)xM9yl+<|4j+4IDL3vt z@3{{H5<!l&((@ndH3(h+%C(is`9HVxgAxd_Xlu7+t)n0 zd_wFH2QN~_;LnbaqfmoYimoJzd}Epp2-X8a4;Q7svk*V=dlTb&l509e6vOE{wdcib z%Ar5& zLC3iDil%z8Z;*04cFKHr7V?^^`f}FsUO( z$QH~ve+~xgLy{$z5#0bdY^i8NW~T7p<4ia?AclOtG*f+!Y8kiK&ewV2i^zWRVDz>5ZIM8C9Sp1v$-BPjV}YwLu`s8 zHD381hH=iDaKO{#6hx&mq(x;y8_oAFoqLU3k)*3;GV5&qYAjHC4F^l`PzuVIf1P(# zF5jJoyn-+u)_~JS8cdDHp_E0?JD%66@0@!IglDxxyY3gqrFb(8SHctIB&(x8sAb9wR2*tCAC9b);6PUyi9f z7T71y2KZZ(^zW9G@pBdBqz9~cSfOJlXT-sHcBZ)j>a9WA&Dv0Wh;HtTL4^O!NTtJ2hR+ zuYj-~<8iW+YWouDxR_b^S}D%UpcIJw;|@M?m;ES?6pA+D4jNq8!zYjBum8f zLo@#{mhHK){U|w8yuNB0NFl+mvhywx8M3$3LswLZ3rbWRm2HB#F^hY0>KZUGm+ZM< zg+Kg36~;u2lFeg<7=IU~l$CF|RQ0~hE-if_v$k@a;*+0ev9GT~@><>9n%i~hi(U%R zzB@iH0MxFjlun4&AK?p3?Xa1JhCFMHJ9x5_IVSj1i>xk}N_ythx$;PyAXUVgpmkdt zvU*$Pw)y$YA-8fXynJDAE zwvsgGS0F^>tmPo6G2qX*D_So=P+O^2j$DTlNwY4wIfTz}h2Z#jk&ppuO5;LTO1Cbf z3fcda=BwA|vwdW}@b}7L%~15PMuUTVn_Fr6H^I*3X9UWZ<#KDcaBoF9%NTiM3f}pJ zv)x52V7Pk5CSTryuUf=g%?wFKO4S8r{w9LfVY1=p#`I%@^H~M|L4xHQqS}5g55p`y zZhS@SA6rcbS%b{0>GDs{Qk8MqdxL)yd;4=NT(^Mh{C_yPO~Py9DkUkaV6neag#Tll z1%?H8yt%9T>VNUiltP!)uKnf4AjyuhK)`dicsGwY`)8QUBjsl;1f=AYv=OVLPVKoU znBC}&BBvV5vy02r$Ij;JY(=4eD}kGQ@q7{`uVK@n;Ue7;-2BX((@d;(Nx@v-f@zX| zZ)_Wf@D%A539uqMz70azZHBLhDV5RF>K1XCDue3%#RAm!m=y-AnSCa0TCc4dZBa7g zo}29fQs5;FPV}Sg3I=}na?1Da*?)LAim0Z|`|Y$LCQ8ua==hi%Hs{+12IFJKc*Gm`!NQt_}rR7H-8F0+>6cC50^{5;#xQ^DJon9f)z!`^dN0> z#a1k@C{dZ^)Zdpgsd2D54BlCjt-??;fYyr4ey+RudvWX4d4)?|B{uKWT1bydJO>9M zz7EYnp?AD!!I^a;T`onN!Otj3jKB?YmX6%78*7jIS4k$vb{PyN(<)qeb?>t#c8l5G zx+ekVgz2`%FmD9b=U*(t&XiIUM+Ewe)X9!cpJ85HX*l@i47@6;OlDx)I{sT+R%OLO z*ZKXO;SX}l&I&X3p#23%mjNs(+;4I+t6dPKg|X`a#S$hDLE|aZ1YL3MVgdm{@Ru(cNkrTIfKqq=1A%yAsIZ>7M&Rgkkl?v`iNaDl$B15CyT+#Q<*IrVoAhs1wAkk&qJ#_tC;P`!Xo?T&z+{>(UB3x+ z)v8(*N=K6pm0I!~ypRbuj}fha21{E{uG$>-hFo-?jO$Fy3?ZMj%!22>;(UEZ70NyRGNbC|3L zl~-j%J}vD;=%K;f*W|P;qO_W)xgl;`%rKjZ6VL#F zc)P9`{!bZ~+7OT?hx#2Vk8d?x?lrm1Vop|Xi<-vl1dYV*2<#Z9_QaU@9K)?9qKduBzt2L@1NJ__M#dD zi0C!vO%az5NE?UtUD!v;9T9_Q(!ZvbFB`a3`kXfH;z;imY;iA!7$W6>^N071FuqZS z83z%@K>Fv+3{6SVxY@}g+4vGElK73j{BhX9$5sE&|2m;grG?Jc?ohq>_r{o(k>>sWd2H$7|~`45-X1TVE=f* zr-Q%dC>U76rXCbqARy41x@pdb{@9r~wIPAN7|bgo6LTiiez-3i65?q%7#*!Y5?j>fC8=lc9sg~39 zoAqve&jj*Hmv3k@NNN7+qYO+Vb*+i$81}lx{-T;ZXPU_DBiwwQ=FoLdNA8CT`9CmX zU2`+ux{1mCl(?yHEV<8$HtMivy)qovAC)`}x_aRbz6kA@DgkLmdYF_g6Z6pd5gwKQLO;={ui?IweP^Xwz6#&x9>|Ua<9oaD8|5H-OTWtd>^h0 zOb~?$BS|dHgvB5W2_GdYf#J(19W$*_HG)4@NB$v^gf}W8g9^eF;49ZHCk9Okv-?q> zc_ejW6Ddn3ZHB!w>Nt&l2A#rlgmc{L0ZueuX;;-#e4W`0M3%5Fcfic$)R*oVA<80{ z+_*BS{!FrR&8-#QA!8|XbEYK|i03G3@U}smECp+8EimyRK2mk!g0XL_spGZawq{UM z;ls)~sPptx4@I*WxE(T?WL25!SgEppjh6jh0zI4eH|5B^Ck z!@lYVnR`ArJ<#e;-pnjUTHhAAG_d=KhS;_%)l%#M>jHngvN7ljvJyf=eMtYIwOaYlFBh@d~#VX#_H(yV=U-UV!15m{;dVPy^D-wV-R&jAyaY7Il` z59pPK9&N_3l#7b~vcUz^MIkbJ>vdmMauCiV-P$f`Z9}<|YNDWT8YyABojlP0zLNT(0!Z(3IFAqnj{7(k55S_j&M7OW4J$lCT|KU#o7c;k)o0bvIPzlIO#Sf^tI5 zSYwtlV;g1mLpWLPce)K1cgXUm=Z#( zsYhUCo40u>;zG^Y&A&}U28#Mipdc~hw7T2uH~nOi&Rwjztw2lB9Gq(?uemo z4WpoU0xawX{&Dj=@{-I)!xwy2y#j~+la;5cmU<{3J_s5UDaC~N;f z8nTMveU*cvvDCQ__pyjoe{5u?P>S zz=Hkl-4_)GQn|iQ{ZCJxl%Cftg0Mvy+TeuVnppdrh1%zg5;QJW_mbMJ>g~E#^sxqE zoCl?B-grdb_xb@nlIJQ* z7>Ny9hAev*hw{fYd|Cc?yU0V6L8FOC@*G+;s1{!Acx(2)!(qHCRS0MY_~PVWhYCo6 zU!YSZ9sa?2Umb7xT*tQ_7>J#Aw7{7a89X77Vf)Ou$w%8YJm?_(exwpSqjpc&|NOVl zt|+jpU;}wh5!^31DtRo9OS`p*U3r}~B6;sPi7^dkdmtMg8k&Jw#)sLj0y;ig0*KO5;j#w%k0c?C|NDgpA%Ls(o%_*$ z`)`=+^U75NWni|15h&0!0Q!G4!HXc*R(=FYY}0Y}tIGgVa{X2J4*Mt&2U4}KlG3+yTXh~gQ+jo6;|3xEY$y(&=e zu3-RxNOH;kmzmMIz3vp4OF05K8u^+Y^Ks##R^=0GD{jfb<+dJ-Lpl^)K{I8K+;uTZt9Qb+zTyUyH?ML$kkNjTjX^xTq!`2UVz#H2sy zI`fL?%^c$2M;omc$ne9-jGuIY2}*@j6*%j)dwrE{PHduKdh?k($wt^Qw7=2uzHSeh z(v;*7f{Da-(f>JU8s%In)28a6pfc)*y(J;eNr+gLVx>X|c4Hhfgky9<=-dL8B;my& zk99E`CjofG@+I}H0wcz-Q_EE;L~GAlN*z>kRjVga z3SPF0v0o(f0;a2^zLJPM7K7DNV}JYS4*VLr@}N2N24;C7Kzu>7TJsH0LUIqD>x*7F zM!fpPZhai7Djmy%73Xv9riyDsWN_OSn9z^d>*!eEGa9QTxg zI`P&|+)vE9#AGg!$*^5OLS?)DdqA^qz-Sqf`Dt?e19H*)4?qyYA5m)j<N!g5biEAybzbr4HX;or#3Ru_LmPf`|cYK!PuUE9{zFdBVdsh z4LyzV7af^gd76GxsZfbR70#xcc-SB7f@r=21yFwC&lj@fPMn`+6MyrYS#+yrXX>QHXekxp3-H!FMQYT+s}isha549W5+bSAx8dJTEc z9EU$Lec8HX>iu0`ehf8?Vy66Pjz%r>F$tQm{wK5ay3f+{Is@pUO#jU+{oU^7>mYk} z4SelaBnxf`W)!8_{{S1U8)P6oDW8Nnro^huUwK8a?SrNC1!;r;@F&@(=Ds7Fe?n7z zi`z?RIYz`G5(8_eK_hP}w|Hp!Nhre3RvyVL9~}2whdGJJW9I)?Vp)cfG{Y3T=(scTPrL?%*T%w`W4rc=y2Kbc%lk(HC}qHu|{G z!iml&s{Yf4$UfZ7MziNS!27eW5YpWu+oRhE_SRxv|oJGN_J5J2=O z-FpMC9Qi)WNP0PKY2)AcA$hPWkaAm2_Eg~EQ1@UZDFk&b_S*Jaj_8(Lxe<1v(0g_( z&Vq=$v<#8do#kT4U)y3Qn&=!Jy+-1MrL;8|C=Oy-MEmLtS3F=RsD&r~dP@Q@s{oUAW!W~fXMiU&-Y>z0Eex+E=I zaq@>Rah8#qG`@oB6=2dWOw1#UTg|(42;aFy|c0_>hPh&m^vy z-Tm&1To0`d2okg?Kgz=7=97<(y*8G6G3Ip?<$Kw>e`Wj}t~GvfU32FC>H^wc%HYUg z1;f~vn{Cle&{b$&xY!N8sV5*ke%VO{IxiBcga(bwZ|e!^DNEw5a-FeHSxf)=XVF@e zzr}c!QX{qqDc}9c1$wpFSP@{GikbQt_k1~3_b|o8TG^J`0|sxEzXbwQ$BT<~5%M%E z$6vTeX;!shD(nstr<>VaCf9uE_+pf3o)3p zYa_-vL>bo!3MTxjy_gfk|LSnXyyC>z*emTRIo8vY`9*2Hbt~RaH$4UEd63M|cT^cg zLgVQv#>8LMVuUDLr9pU>PA_W3jXp!=?eLncQgR1Ue_NXs-M!AL%Y%N`wE(f{NawDz zqxr3d7QTFz9n~_-Ac@7ZZ5ckWCzETZJnJpPotKQ1lbIlBcyfggQ%iEmd`6a=W4ca4 z*JgVgAy#E^xtp7@^076oHCKL8@tq(D>-}EH)Y_AL8MEMXD(Q?Uo4Q`JM3HoRZ$ROo zB1(b=&wBf>IKW%*tgn{pH;7Hs41Mnn@d)wVQEm4=E@5ACxnUGKwD*bX`lHY0ly3ayJzn-Vg54`xFG1kla zH$qZw$6W-iv80tXS7~?!KD;SNFE|>pSeB~lBN32IaWO97lTXZnKW5n#FwNDiyArzK ztziS-Fo(yy4FO2ijk|e8xZxJ0k6TW;j$F5Kl^c>JyHoI9@d(^aqGFaKWwZ-v`5z8% z`@sXveR{-++)KhKME|Uc&pxumg!wYK$aDTp!b+qly7_bA_^^j>`1#Rq^C>=4PFN+W zi4a*geOX|5Aiw0sTGpBVr%+X~c%gn#ys%tH$ECJNJ3aFgxLYHsO<(R6P-|dwml;q} zTv>Rzi?pz`Gy|9%kq@ZXIg`Hg{90!Hs6UaPr^o{AXTj)lc*9F#-%+h_j$1&i_I5ak zqA-DbBti@&>g$RR_=H><$A8n4#oV{Mpc%RlITy~~sUJc(uJ8XU5=m6l$4M+~6GYy| zdWm1!Hh5RnUrIcY7%QKB`au&DJ~JMNsil2{mmvXS2o(N0x0WLKcQ`qo>N;B!8sNJF zo(gi-FXe$F-?`qVtSLvF2)5vG5$hdC`R+DkItVWCC#X(vHW2^rSQb=~!qJj2Cj%4=ebldf@PSUV>F0H9jK!|-bjyY}O; zANM=@!oDiP5l<@n?!cCe>cUU+RKF=M93iRIr_#j`U|hSEg$>^oa^2Ez%eS!r0&_~WW5#q0rlvX5tpZNUlG^sxsB2J~@mhT}`TV)e-RYBU=e?I29{Ej>aq`r`Ph2KI1x-w6XOD0RzTe^gmLS z%~y17FoQ)}J@tbRODu$rMJ*!{(e!5NjBxbPc!_fUEAKk+Q%FXxm|s0<(LX@G1pWbD z5BDMJFE{yXGpS;HTi?k|*0?G@ctda5C#g8>zeN(jgGKDSV56zW7&L{%p%rvJccVOC6I%959DDE@5*wcd4Mn(PStH zo}S&{Is3lLPH!-oKO4=I05+(DwNzUWF zu`p&8skm7BBr*{aO|VOUfyjVCzmTBnjJ&8k3%*^iM$W&y`vmu@dLJ`(>QJH#K7#0z zfF)OUOFw$cLsfWmo&>x&%)I7*|5gh591nqZWrT?5XHGxRv0vU`wzT>P;0gGE7J*24 zGJSRuFJb*@wmuFJ0KdIrY0b1e@fFTbREBQZ`Lxle? zSj>6JpcToy? z4R;l;BNtzpNYYv-y0dvv2Sx4gdcQUMDrtpzT5depUP_4844^YhJN?sHE{q$czIn!g4iBwHmb1KgwXHT?IN>Y)m9%yx4UjjRjXP$v?_pBkVo+Iv9( z#{=sF#sVycaJ1sX4k6Y*Blwwir#<~bs0(|HJ>#QeL*$iUiQr@F(gP&UW>c~EO#uo%dE``cQcH+HENQcI>@H z+`{m-J-RF6a@5T|+(YA`c1d7^{J;?=Gw@kB%Uzk@*Q~|fBxSHhg>ObkXuTL8Wk0ED zIIiK1aQIQ>=5fKPJoFgo#ar4FnqzGF{)egW3~K88x(=cuBA}vxR0Rd;0@9mEQ%dNe zgHnZ1B=i!nO9v(NrXYj>A<{dDQbUs#3|*>(CM6*KIsE?f&b*9YoXKtHp0oGbYps25 zqtWfP;Cs1wT>(xr{6mIL5<5swA7z1U0Hash6qm;}pA?oE7F4XIe$D>T?#)oqrCFQU zzf5gHeyCcT{pEb;=fw|E^K@uNm*DwY(fC)Dw7R{PBJfkH3N)9iMvlsBC?c<7NpZ$U(D7{2s z+^PFWX?U%j?bto$GhiP(RG@Ht`W-NGa^)+6>!CKJV9fJeDXpmtY%?`|Wo0_^G{oNK9 z6Ezrj3qhPw)7kBCGBKh#&veJfAlgL6LmI`kOX_)MrHJ*;GH2?0gG!X#j=Inp7-q6Pr!uRB9LgHTtaP z6{EoSTEnga;VO!dkkhYZN+&90jCmJ+?@!rWG>ODo_~(p|W$(;)MA1?iauAcw&a{i+ zB24^fyHfP4UmY};u{3b4ygq3(2zJtL&5krjnK0D&mrRXk{HZ#{Ay1UbGaKD7XjS>l$+s?lDvMLZval6 zpFwumY3msMpl*9Et=v}dF7muRZI2YQAH6M}%&0cKnI$>{ z*cIvW8El>ZEU@6;@s|}eX55iv(`;^@zdui-HCT6z7i_X;KYmhEvsd)a8r25O46hO; z*yQ@@+)7TaS#VseS`vr>TL%ZXeCy$dnIGS(^CER|l4X6K7~e~0h(Ew6ro5%vZy+Pb>r6~L>rz4H5#3`dWO zLlOf7yf5E+Uq*RdLE<=}Pk2#`{&mTP^iHoa{mI6}*PrgK86750l?Dov9+kE9a8h z5%4`TU9?y`FIU|{5`NmdwKShy=+{8qcUT|$>4w*%65kg!Ll?L8_Eg@?0lbnX3B#M`H1#)= z)Y*rh{i6*Ekpev(#>O+kc;E2I_{7zhaNek`9MK3q#*!`w=FZH(0MHbjiPWs3#C*2OFL*qm^tWj>(SLePuP zIEK&#M)`fY=XOU+nhsx`y&f(Ojjc;|C>A9nD6Tq$kWS?Hhd5V8LIVA62FDi%%swaM zR038@uixX`$h;-26if~Aj7pbHm8tl7@7|4j1$3UCnH3vu(0ng~Vhy`Lm$P#HHNDhm z^1)W_sM_yW#$kQzBj=i`b762V39P~`r|VNmH;7yPmRlSrapr|ONdjmm8Fq=6bAD0$ zb`LN=U-H3G&X~CFG2s|3kf2}BO#Sul!3viQx#W9wt-%EdBup4Vy1O{^ z%4fhied{-@4gg@fviy!kRO-L3U zTH2|}-l)H^Y2a*9cwovzL(bQaLKE03^PrBx%dUsc`Y8nf}0)6{~oHv zu7rI}{EaTgEZaDf!^PNtc8_y)>i>A~LTTK(=#oTdk^f{8(v&s@tJ|}>Gd9?2{^)3V zJxkk+hod6{ek{fN1oKe)4=*WX@Y4z<1Y#A*T6>=65=gMR@e-Fx-oTFj z%*w*k2BbRu_%{m|r+t7FVpu zc*K}N`82pBvY>g+MI|I`PTM(Y#X`k4e%pCQ4e8$?n-dUSOnWm7{pd)8s)+QjHpKNp zWTttD)K9RDA#qQMYR_;ZNg3MQrWr=z988!@HLsE?n|}=-&Q@$1dydVEeY|&t!CmUo zpC7-PT3)K{P(UC>%rX^b2p0xKPtcB2VOi+mto3Xh66Sl>ueK#SI0tqvBgfWl`RjyI zL8HNNA=2ZXVWdvnh`fEDJL?sZye&C(?!dg?_3c6BpH%ji+Aqq=f5?&w*oxLemxom~ zmePm+e!uDNaOsavvEZLOkFN;yCZ2Byt@wz9yW0mAyV=XOwA*kk+Ivp(GU?yq`4XW- zOBt#C=DO=uZ1Kj8ll(v%$FBR24O_IOCmI-_tfA7Xf^wJb6~A$HnQQN#g+Q)S$wzCB;tq)!5JN4udc8?``y?Ieo7>*Mvma_7 z%x)e1`|9wgZhU8gJ~j5bcX7}748%5LlPh>E;~l9{b?*WVm_y=oLvuqP9B!mJv!t_F zY96zW{thOqI_i65m?S+Ts|5TsGh|QmxK?{4t%@) z;nsyCr>M6bp^a4E_*230wQcSMGJ1Fy*YmbHsd>rz8C!Y*DeX_mTb~eW2;`f=2g7*7 z_RONnr^r`E8(AKuDh(z$noaYmP)&vg^LIUI#l77^uC4 zF?EsKKNuu5z`f^9lVJycwkzc}e`PaiFmt~T`W+Ze2Jwi~3deZTHlIsFm1h~mQOVr? z&BlX(2oAe-$I7m)PFKY9aGlP>Tpj{>QLPNsfwFHMi`;n@!%Zryqb=>54*px%=j^b# zWw2VdWl|8RR5$o{5v)jI6Ed_mJCwTUq1LpNxp@WeoU?KnO{UciioB$~&%tU;GU=Ha zea|>#^n|mi8X8oha28TQeILZgOOkn5W5mD7jwK12GGGP6*_I?M*ZfIwqj21H+U-_DNsf z6qzWd>cP9;C!~B7CMG?KIhzW#D%_s$<+5V?(^43HOt`BSurG2$ z78a9Vv0c#p(?u6qQ7V$CQ@fBbrG*hY&T|N?Vm*%v(!o3#F$2*oOzVHM;#Yl$xmlZ| zeb&p3m4vA*kpo=L-ihx`3!I5A<1qc-;wPOQnCnYDpM_uX7QmObprkEJD#7BM3lNs0 zz^=By%k;{X%BMv3wAA-2c#-@eY;6efVp;8obpAM8Q-TiK2?Ou_ z?1R(yPTzZw!HFGim^Ku6=Jd3QtU~OgU%a8>qL0hkUqX?tTfcr(f1qIgbHf<6jIq^n zSbt;5GUEG%Dmlyy7vIWAOuAP*vdt%_jLB`>Te%qUsxp5w)021+628jaWR@1{S2;ID zzwk_V`{dXvrt);1m&mn-F8}3{@o`a*sUS)S?X;giaNQPpa0UYT>ezwl#LT`yJj8>^ z%IF<~_jw7zXx5aQY3K?@9HVx$8_sp4p6I7su;$rLH8NDyPK6j9-v|jPTmjz-tCEDB z$-ImjmMmI1n_CnQCGJ16I+?@P9t#X3iEck^gYmiNuPaE-Goosn#$|A`#%8fP^4pSA z`B!|(KjpmadkmqrJD4~>`+H+|QHvaBJ{0*LK{n1BFHV|bfU~WgJF2>1YK)}baN0Cw zn&~@4UtRY&gj`nKSmP`6xr?(aZ{HcJCaNhWraTEJ51Ww#i<3UQ8cqowc68 z@K#AFefQ8$Ci{5#*)f1?ZBwG?EvQZ!k?`{`C;%ylVaySXAWn_vUIAeHSc4OHx}!Nf6&?{7X7-b2pvBc%6Gy9BH!+0>=0WUJ~anYmMba$XO$TLgSu#^hR>P zP4vV<*f9MgihE*O&{f~M!NEZ(_wj#aSx2rv-qG?x)4viKvwW1d*<=uN!zMP@r^xL} z$D$%4y5ljHOp*u}apq)%RWWHIq)K_HgUBMeA%u@+F8m^J zjCT0<*)rqxyzh?}mP!7_Mr8gV6qTAmte!1e8aNi0;7x%w0oN@SDCF9msx9p>)0LQ*_)0jTxeg} zjIl~LElo}HD))(Zw5%;nGbgzE&4|{kq#y3X!^0vC+aLcNwu*#o>@VY#wz|bBenSKb={ z)CA<(P1mh3ACZMZWj_7~vszWikgDIxZzw$>pDUuZm-WeBQ=N~MT0%pD7SM%y%mR73 z3SfISCDOh^L(w|j~}pwg+Ny~ z{9`zkGEu1Vpk4R(v*>sVE8&sjql4Cu>~g+m<&;LNOVw5UGmfDB z87&T+?vlQXQ((NS8kaM3vaIm{Irbp>G{@}d*3Lz6hFbm=7Q3)6S>Tit8b4gH9kzey zt`cnN^0;2DZ=IQMnS&E4`^o0xt6s3d&D~+y`hD~b4*arSNJRcRVsDn>vd*9v&4|l> z%au(fy7@-`yXz+HG$?UQ#psoTzTR0nKc6X8H5@jHo&bT&n0zxFkCJGA556VJbd)~o zQc-NvbW9$x;p7NGy!!mF$#iqvtFNN660fFPh`~IDLMJQWN$p#6_r6RFB??(s7W0b! z%a%-Z`$cOnyltS*iAerrW|ljQu3Ap$+ZCYIM8>oG%X2@ah1Q^y zvyHyRp8=^bdhiq4XuF!p_Lx5-cVafpLs8T+_e0zjUlFBtAk3xlH*`w~VjADZOdrOo`Awh_o&) z1_lG2@w~`)-*gX8rkh{e254$VSZL04xk4|c94wTFMyYX~IGVs4Cjgr=PtNO%xKz0zFww(uQW4&b{@8!8C zlKQK3Cs4?^*0%7 z$$Qs6ws*aqCWLKtKRdxMhnOec-)J! zbWU6R=3Q!EE4lf-GjMJBGJ)N&yBkB}_oVO0L7reib;{AX{_E(I*4v6pnmke|e2q4l59s8iIDq_b}+ZycV7C zJ7hB1TsOG`Gm_b}SjnR|;Z+;^&MzzsmcdL6$c!x?BY)<>JbI(V{6&h&zk^w8 zN2s-Rn!(flK5lcKmI+xS7$a0FAyzzMnvj)0!g7ew<&Giwu~Nk{db9PeR~tLAmy0MK zBsz?H)_>ii=bO)NdAj4`lLc4gSYW81x}QHz(*~f(-Gj@t3X*7MpZy8XymGkxIm6$8 z2Z)d}7m?wZt!AZ&i#K7aGH8M$$$u#~skXKjym3Lsu1Jwa zX4|W=H8nM_spxrlc~54Tn*L3!hk&t?^^`LLkXp{r#<+&ey8$+<(|%!=P?tj8;F;R z;tadE(y@wy-Gjn#=NhJZudWR4169u#UN567>X&p{W}!9g`bmX}!M;1$EfZf3ujRTF zF_cQ2Ve_KF#~QkfU)ktrwo_t)2S}L`CQ=fFyP;_)9|>JncU<^57}B6vJE#CCPZ-5{BeH$`ZXtV zTmNITc%Qu2@nJw!kn@&OgvV{PH7hNpA>J!I|3p#$UzWyJ!dr9T=Z&AbVSKANj)!Eb zj9%FS)!zM+ibGpIS#_sl{~Cim|FE*kZ_n;=)dWPaonk_*?Dzla? z-+!6b8>h0y2E-?9=xfsHF+{`);f*c>}-4h#8v{mwzg*76mqoHr<9eN zDk~+mxV880{d>Q)@e~xwd;3rS3qx}%hI{B@Qkt~kXoXVUs_F-|rc4glcLsU<-tfjN z6X-R58ow`6LLzfkK0kK!OlsjatM@!7XKBGV7=~F&$_k{WE+t^^iz3m@^97L5+RGZ3 zY65G5XF4R;K+SFan&F8_mtPG<@7WXCLZ8Pqu73iqs&*V#_c{J`>k;Y+i)?_xH2qcG%$3B}p~%_VzaUCWwR~U?bEh zMMrkFNgv!!4=AShx8^qkBaMBZm<{mDp&DF@rm}=4J<$uHy^N9jV`O|}D|U0N`&L7q zA@K>;9aYs8GwfASWVOfFHj2c-cCZESibLk9_J4e>mXdX>#=MY zl5Su|wkjQd0Fu8=plS>PzVHa_8P9q{t0)0!ns}J!zF{h7@e@&3kmK~kRQTKr=8ef* zvH5^z)6jZTo&yLZ+|7mGv+r6I0;(VI*anX@*b((x9Suv=*a!t3vw z>+4MGx&pGkxPR6mDV8j7-^sg1z8C|u*7?$5lz~^*u{$$ss7`0oM`QjGYymcjGWJ1( z{ZxTwmw+Jv9#d?6d+T;GIF5s*lOT6SJfQ_OSB_)Qi!qLqQ-k^Z7w04b?@)P#{3B>8jnTn_uuB@ zVbaH(H5S35Wk#2jLTyt@2T}YHbK)cP^psPcf#GN&;(oF}k8vRCs)h-X#m)B8l*nBh z5}mflU6T-QyYPxqdac{O8PYfJ9&A-lK`0^OWB#%xGBG!nLv`ozilD&eM#r##X5i41 z<1b&n3=J89!9$_0l~^7MG)Z0%z4OS}IN@J0B^6xP;Z zoAGmhwDVOG-QscRl%q5Gdvm(N?y^1HR={;lbnWhmCeCcMrtk+hH}V4;^h3PWRPN!L z<^*mDh5dcy2>)1p=F^q__SBnna#^_rvDIbg^8Rf5EN-@0DUv6mZyW--KR48Gq^B1La5D11ca}__0bN|^Ny`Ge zZxIL(rsrj2s~!zx!ajK!3oEqTHVyNZWA4ZG=_2ome6uVJbbE{)93^yj+D4YrnzzpQ ztO9!TZqr)>j^gJ>9Ul(q9Qy9wIe?AD97K<3yCp3 zSNE#+qA(%rtc3h53Z306U3jra^9cZCK1VQhyj|OmteT`a&R$*%j5tK$5f{Vb@JY zRjsgV@rB&#uINpbgi@%qLt*pAcRiC{!AjMg@3E1xR#&G}c25lO7Vi?D2fWznU!s)J z&HK%G!lX4PD>zXSXV=DPNTmx2*RHSEk9pF@0=P;=z=9fYdyzIaHlkUj6c1L)T3cI3 zuV(cnfgqQjUT@b3*e0uQ{{cxY;(h3sXP?Wo zOp_P##9$ok-k3xpmRm2!GCfmj04dg&x1g<=&&}j);Y_h7k!1iYTeumWtym44u=DDm z8-#O|3XI{5Du(NGS{3C^S{vHP?H5s>j|#@r(XV{hF)p}$hUuBTf4y}SF?;6tN1AEYRIGxM9r2JEN;P|78D z_hl;dV6aN}iCU-30LPJHs~b0N1nsU}Ke1^FX=vpt)2bS4a2u=q^5ymhF3~bJI@)c# zT3;|=t4GSvz@W6OtaUeHT@wn0>gbS#<}<#Cq9rpY|JcCO4LL6cAa=p_G(;x3fioE! zyd(ga(=R!ci7X!+_gf+_78;lGF$ViLm=i|uAhY$xj-T*yM^=3> ze{C@Hk4FyH-7szJZiy*^c^zdSntr!DZD@%{!L1wW-xOD#pp~NdPBGCA@f`8W9ROcZ zSTWeQc;Xf32Fdpbjk`XL^%|9D9ti#^k%{8`B%eOjFTo-1#1Da7*3av}AQf#qJvT%` z_qx8cEAIcvX>L}o*>C3rHN}Wcs8HhiXazjPI#WYZLc-_I&(9#uI8SlagE^?Cre?i9 z@cqh0bRZO2Ci(4gOdvbn;=A*x!0v9`dZLt7K^#(aK4xl#Q*I!wD4s%*IkN-M89OY! zgdx(2Rl&0xG5@GP)`_KBA26}Md?gyZcwzc)7Kh<>xBC> zw%r0#&x#IzQGef*Q%6OQ2n^aQ7w8Y>#Lt{8TE2*-3k&?MSG*$-oWwf)LTMp+kZ``J zHiZ0yPAS9y4XemKCnUX)};xm=45y3Zd7;V0+B%Z3+V0EJ73x^&ccp}WK z7@$?h0%wZ`uNGG7a*w5cYSUMVmq+%@TbzBGmyAs?GJ`?&8lahcYOL}aQznc$_yUNn)02PXoTPQVuFJim{Bn(2aK@mgJMIpQQ zTXKMlK*kRT;yju>zN|ro|@t}4hyeo`}%k58sGTb%L8XEG0&u7zxbM(r_Ft`j^XNK_OBiV&AN69@Ln9@6!G6XczB}piBT;G>IDIRh2K?We9D=< zTt3$VsC|+%JlYrHUweD#@XORf5%lI6=8TJoC1``a&G*7eXmHZladU->{*-;yJEh#| zY+l#%^3fyR`)JFNf`c80%CK6?VfZm$C7`cQX7!3c>cK01YloX2e7f_E?|0q9KH3UR z`pZ+XO8iwtM#{Wpx~gwRsybQ}EuoV92FnLF$Hy2MOh}0IdXGk^?ttbFMM+*jN9M3> z`mp4<qdzW8TB-tCy)I2;V}{ z{iIiWj;vuak?L{5FBAsW6fyJDxdLCJyGRw+P_M2z|pulrjsm9@K_Z!~~FKZg(Rv@skl2^Q;UqlHu?n9vA z<1g=i&+M}}g(Bl0kQ6O@GP7go@qEb{yUjbK#rIn+n@>V7TtZHr(w9hqg{`A%0q(yw zW5clHM>Y3noa@Vj4%M+_m{q95I#-Z~x8vGTp)p=@g#TBr#1)wa-M*J)$4gWzZ8Y<9 zfnjPrD!#Md2yjpWZ-w#zZRjSTYgFck9QEvP97vNpTQQ3D))~ZFf_U7`{$QIgnfld7 z?yr4bF;JUmoP2@LzC}3D!^uc(z?j$dCiyHM9FFi_JUMz2lNpQ*F04e}|Feke3Tz{L zY~zMAuGvm58#;#W&sYOQC2SlJ+fz-i29iGfLJezOJ3BrgCcF5GfyR$$7BY$8aimxI zTf5UkmkFK~6o-7Ey`l8|PsQTXNLRplJbE-Y=7r#6 zN4$j1n35S0Zr2v>aQQ%|CT-h=-P&IASLxaYDk5|6jup@*0NaeG+qZ9ZLw+8$+b2hNJYtwBP6tEJ*2XNSsr6V)}=aml7EG_ z3a`q|&v?FPjwi|%U9!u?2Yzo5^ZJ*19$lF?d?OUUkS1YCIuC&)Uj`zo1muac`MXuH z=%Clk4EO^J{0lu40o~C~BWok0|8IZE*6zNY;KECZrlPp;$@VH22hPiKrDN)-4@E|M zE{j8;f9y|{Y&+m~r{@y%3qI>;D&J8R+=-*r&u}8>1 zA*jdnhLB}3WeHlr-=%1HdWc{5{`UM4&%DN;3JI*WBv_rvl&3d$d2CbzZMeK2dUqXK z$a&f0WX=E*4%qL`7Tn2^G}zX$Dufs2#TBfbznW#VCXH=$dKlVL#J;5bdiIEjEp<-t zj+H}ke!iBp2xO_ehd`^LZQ+j{@$2@;o1T4}VLe5%>7{${2K=gc*pzWtJ-7YOgBIRcoV{E`=)1UjX7f}?KvLz z(v)E5bvG2bCy92m4ae#?#fh0pNo$AWb13?zQ?(nB(t~cP3n9hs@h7YLGBqG>sEBT}Uc_WU>HuE->>T0bXu*Ymb z@wQSduwP+lT0h3OCZ^P!+f#sCUZ-31V~&kzn`ylYf!t$7kSe+`LA>cU%z3>-BuA`q zaa%zuqDP=^)RYP_W^ZGgw(GBZ-6AT;5pIrj!08lAR?*{3SgR~CvF<}1ijX{1zx)?@ z*MwAWp;eUA@*BgNa}CFfmP;Oe*A^>cWyd*)w2<(p-`d z_~pzny>GH&*Ad3~R`$nzt)VvCLvq5oE!@x@ihEi6FEQ*(IsoiF`;l^WzPT{T2hrIV z#XVRS+F({V9+M9^vESv&5Ndb)JP<0R=%B=O_Zr~1`DPD@PA>G0KN270m}` z;>Rgbev>CP1^YaTjbuaAA&3=jqkocO<{CU|kEL zFns`RV3Z+ekBll>|KhVK(F7(AJQRjsjL!&n?iUTnE7z(_{ne1C>`+!wJ9YoA93kRg z`2JjM)?{(tGwWsA2e-dar9KD1h>)NYs6w_@JGv1Z1Rb*NQh58mYQUoHC#)M6{CBt8S z8Ak_VXNtrWB;WfN0hf#BkkXpeq_e5MG#p`_wcnpD)t3MgKATX%aUXZU{-&WQ;t+&5 z;KAVn?>;qaA14Lh_M1LVVCwAqIF)bg^zEODW(&K;@c>Y%0w2?o zx5Ah!mfEAZ{=UCsyU4+nho5s*$%KGik!>*Gzk1YnOLVnNjuy}EQe>Pmt=ME|RAZ(O z#*WjkWwHD7yaI*d!sfx2!)d*yPd-T(m`H^s5oHCN{=1*JM_`cdqM`I+$}jV)L{I?@ zANJYybee!MOz8`1S8aMGT0ye?b>e3WpCg%7Zo8O2p`*2-17tU7+6GgT!36Kf9cIz* zz)H5)%FToiiV9N4fSPzk<@szR(&*I$bB;P=8uQ{#gl@1qwPN!=0MPb^!B=!?@g6Qk zV2q5&Ivqe%2R4P7TWIT?baQEad;Q78$iRb#W`l87zD1SId`X+LQ9e(M7w>>i*~c>Z z@+-}lM;jN%6!0b0*?H#$jsMo^c#MG`@a{TP{_(>veH31;x7&T?cQEnIDQdwt>e*D} zPtDi5w4aCR=O-4sY60DDU9fVB!lup-F0*CmuW9BAqjoz8^ZPtMy+s4B`(er367ZW6 z_E4b$2Wf-8cc=FU+2Tf9*QJ|Rk4gXiO?Ej_*gIaSv>qd64HeamNPjvhdNKc8q=RsM z?BJ_ejVBhkixSzx-UUTdKRk+zZ*usV+fjV+tKR;i z$}9aiA?0Kj>`Skzs*t-uzn`o%v3Hz0CLM;Ai*d}{iV~JtgH59vCih!UK$TgeoV(*x z)k*kQDuIRW>5yfw#P-WWRLx;yUlGy@#^V*!3Aha~qP8?ZECZ{-3sY(mFkdhfhooM# z^p2K04qol-oHjr8Mz;Uyc2drZ{mCz^dnt9n$zpcP+#I%B<}1I~q9s4?>P zVE1(m9)SgG*5ci20DDfWmEBSNl|$8Yu!rMvcy{#2sd8LyX>Bw8A!h_5t>7w23*~FV zz{b{w`ns{Bi$Rc^%4ZLMrRO)$r-8f-d7ym@9kZBTTf{pwSMwj6{xmsb_uu=W(|Z~Q z^anppG`Y1wJwU+J0?+>W8t+bVL=lWE&!a!uIdoc%n+cOD3|pII$V2uw-*1kt=B2NF zYnX1(haYYG9@QoBtFeo$lt;Y|=&El)5G8=c{@rSr7<`sTjL zsgpra6!-v)x_g6Aat|>L^cMC4um#`OvR4jIU!2dUxP2*fL`wk^J!a9&qt0sG`{e%H zn1D;rnq%KG7!j%;?i<~c`1GLvRSbQxapdUHCmF!D7n_#EOeeXFC*3%S?cqqvsc(l3 zOMSK?T!(x$0$37-rnfs^Vg^g0=-k6yXZm$aLPvbdLs zD3Q(uxN{Es#EsAIjX`&NX+)rFj8;NJ9q0W>Pz3I_5wALUMD8E$4TQ?Y37yt7DC1N5 zr*ll~{R;pygkpuihbPcTS9CD#vi1aoAcFw_Xt12S_CR}&M{a1_Jv$v(p1ncpsQ?mI zc6zPfcQb;5^`sqZI-*lepKH>N$uD{$zhttY46L8I0S$Gu6NoG%oY_fWnPfu%wUYrH z_*?otnwutNal)dosjE0b*f0WU(={Sd#zc5L($sD0eCojO;drGCZ6VV9I%T?x5We9_ zVj;)8Pt@Hp$D&g(N-rR3pVgo@$xYXl4Y-}BgntsC9P`g16r314Y6Kd7aL_LPF=)A! zIk2`76#Y5{`(uJy^4#So(8z_M(cxYYplvV6kj1=QHhA1dUOUdhn?1Zn)$@{`n|(K) z3pJPo+PFoQ;v&#o*wYcwa*2D?%OY!UYYk?AdeoQTBS2HZu|iLmiq5h^veM*XrKeV@lB(~)6OZb)lFOn-zAbLH*SD1+k@_l zchkT79F=sOc+iwXM3nye?#&6$%O2Uu{cXB2l2*amdp5bs92XF4ws0h^@2X#^!$)L_ zbzD@?HcY>8YAef^B)lIfbg=opoaN02yeUbAfexI1pOx49-T5Y=ugYgQ@U+6dl1`w` zyig)U>D1k2-hoL|q!KF~|RrK8aoZDazN+N|QCPsm}OUdDCOQ4T4Z3C}`Rlh1M) zW#|6X+tJ;xWbX6J)5G4)qtmz&8>l!0M!05PIP zQjFb&?xMaiGTne(xkqJZrnsa)F%6mc{x%3SPv?fTdNG{)RyvO`72TM1C51cXp&*+9 zryh@s(j&Rgemb={LSEfOTO*(o($cy0n=lu!BaNaUTi`n# z{d8lBWWD9D4WGQq%Y%ZSksTMp->g*r_9(Te${Sv<`dWhUx2Ac0$9vzu^Zi$gy0pV7lyqR;*J2KYlEjB3!M+h{s}*u?5P$a-_?t8{~- z_O1jZ1C&?}3*=2-x1oLRLk4T*{#aBMpYz-y&^TH8aKKNxnU%eJ_3>{252N&-5LiGxnYC*nCW z=1EMfrFC?3@;=dI@mwn}8tWPs$mJGts^V<(VAH?XHo=L`bV3h*Aw2 zQ0WazZ7i(@&d$ylIN&U)ggUf(T3c8*dDD&*z`k zF*md+8v)#2C`)x5uBma3xL&94KZXhc?(%TC$6Num_ql#dV!a!E;m@j*E%fQw%pnP z4E#$|Da=?6O}_J%aszcgBezL1f4uQRJvlPn>?ACt#7wW3;<`AeZGp#NYoc z_ym|M6R@{{clXFfvu3japgCaou?HZ}6NJg)>=D?w#H1*G8KCJG2>?0kLy_pG4?1nj z`PW$D*}-W5y@_S(`Cra&j+R?ERB^$!M2q+%OX)#jyWkxZ!{p17wNN>M!7XQ@3c<5| z^`0B*>cg;}&x^w8B(P+p#3Ml|x+X!#|6Z)25m0(*4{P=MbAoe9)+FfwIRUXv9GjI0 z$s92KeW=3(o{Qpxp@tg2kOAtqTm72a?P~iMcnH{9 z5`%stgis{^U?fsO9=MW9WKd30xj=pRuYwaiCqv%!B4M3%d!@GjYgeKV^8`K`Iq}r( z5+U^v<;8rkOjm;gsuYb5mwhR>0BLAzf$PD@*YPD7C__)#7PkMQd5~+2-8m(ziG%ss zbHK=b&=j-KbL@hySeVO*(G> zJpLcntT0sDN9d<(H^@6_4U3EVIo_uwI?cNj$&_p*1htrg&I2M9^AK5=Uqx-`z@!lP zO-d;Xo%{oLygTEyY&eAe{9$oYqI_VVA-ps|UwEl*#}{xX-^YZp5OPBDbZA*@!Y$2hY2jdCp8@y6LIu0Po1$@zAUJ_*LRox2-W_|fbbaY8vO<9{1UA`9Ffj)de+Mp0N2GCPF>o&wOl2VuN9SG z(9i90?&Y{Uv))c}iLb-`u10_F(_SlN0z!jqiI^=SwgGJ^W{Bu2iG*Y!v zeIRB#&nDF7)TtKgdHV#|bwm6rTdHZZzHE3>JyC0kUkCIJ@S*|TEmL6G5eB9|8csOZtA(HL`3j zT(XcJTs984F2OU?fZ+%=I(lq=1f&hK{A++4e5E7miRI}XOx$vqz~hQgE0O$NzqA1` z{WY{KL3pR!%ct^inK=W-@kKL1-QRpKf+L{(Fjl2T)97fK;jQf+z|`!(`GTYgvpO;q zKJ9BXwETPxTr!OLzm=ZaP58Zz81u`MV#mWrl%|f%I*5fLCOa-}7e0HrE~FLl@-l@6 z)-`_eGAfR=p{0!9$F11Z-#M@S$3qI9@V=nLk?ZhA7}j}{K^_<($*~1fbp{|$fzYSr zcUx&F*b)e3VAg~AXAHtOFPiSp)2_HpXqzdL0q$vr{Wt9$bUx1#m}f&WAkb?TD2kln ztYV{>p6*T_NS^+W77m?4PE<7280dv>7u1}K z-3-%<=)}CdJ{lr2bY0`3Y`Z<@i)YJe&-Vd-*;wKN=eZ(7CM^y@U|L_h;*W3T+M-hG zKI9h2)xG z7f$o&w=bXRb6IAa+Ewz5N@yX1Z8iEgTmYbGG*D z-yoBM+G-Mj>!M+x0f+_e=G&asFAqqDqrxHr;OPN`K%6@?ZF4-MKCE^nGwvEFL1u0k z1_z>#uV782;sX>7{9Lp#mOg1~Bc+71Qw-JwEHO4eftB#%?Ji7rz8yOSA(ImyU<$yzKK0?0d&d&oBvO-+a{D@UsVDq6em22&f#<`LE?pk zDWCSJnaaK2QfX*+XHNmXT6Y$Tbn{+w z;r;AqzsK?I@B7|;9Ph6^f8E~9HDk;%u5rbAo|i>s(A!sY^(hZiyN3X0*%yn8(6Nqqgc8-m|HV~L*gBbjV?{%Se-lNpIIO>)!l`l`Y= zEBV4*A?$H?l>pZRiw@(Xi^On2+B?XXWDo-1+BdG~lpkaLBwaaIPB-Nfd!5pyZO`lb zA^VX%LRYj1Mdwp^^>D=tNa&pvB3e+{AW(kfzx}>c3tSwSMTE#KLU}U+S#0gTz$9_$ z^6VzPr!eE9@{BeaRxbiU?g>k!S@L=0Fz`_RTzs(ZjjZi|Ent)cxi&jmOhKONA>>rX zsd-eTGViZ5qDIeY9%&dI_X$8wWv4@I1L_YRD@CGOa7mxA;vS(&0 zSLdmt@B6)hogz|LD)p&?dvpAPt<&I2bM=|=E=Y@YbK+bs%<0^*cAG+BQBjZ4nqcvr zo)&(q?#Ck-yf}_@L1IyA1r3cJ0FHWL1NB4>c2~SMt7v}z{*Ad=EOZVA3PlQt1BFW( zL8(K`kJdUoi0s$M-^OZPEeIO0A$94#W=s|rCQHkm z5cnXDb&{LIfO~{Zin!Rcx%^0n;KES(1}RH6@Ns|AR5V!U!b0f4OYIbTKSyMQM5i8j zM#iqqF$0S&HkQ;Qau)Civ`kXsP_Y%M4W&_#k?AdSE36tOkk$SVN?Fl>O-55vqRnD| zv9kDWPLY2mE)+A zykpnwVya^L5H*l6V&7kU7rV(z}^nw|wca3#N6@%Y|R>|9SSGYd=VyLS?tuI#+L%IqYT#}YYpR8%73<0-A?-(WQ4_plj4<4vWy zXj$UeMGC*KfL<{9MV?l7jcLbw^RiV=AY}WP+w)EijTf-FNu;u<-E2|9WnQq%^@8~g z2O zEz9e&kmTO@fmz`Df(|2}ZP)?cgP-yq9dk($6dHCeRzk6Byi3jE?ay?085C_fkbeLZ z><*Y!J=f`vU%zq|Pw@m+FOL8NXsuJrsYHkm-8bL)^_{w^s?$P$@z|I#)~iW)v6Ppa zn|9|1Lphs`{md+%N4@aVkHB>?X44l7Oq7iML!xl2zvN-9G0E?;n###YOVGj4>?!OO zx`XCYTMIC1#ezsz=Wy|f{UVS(fMOMS#zEkAVpHq?5b@@A^^)#X{8Xf#V2UiLo{DA-9jgg?n; ze-V58N28MH_**qkXQKg2?OvJ>RS`<9DZ8|aYkM<+1jvmgu3((x_h3#_v?NkfQ1lhq zrKl%{h0&3`q)W7)F>HugKB<*Dt--YV;%NfcqUxP=u$+F%kZxr)(oKEwHlp}y9dGaZ zQm3BmFR&T2ir>v#u=7PQW+>F{pnvC=%nCZ<$bsq>7GKoMKg_b#D9ej3Vp$iz1|v}A z^&_9!uM2!Ur)l(2h5d>BAYDnCWxs)J43Q!sfLeVc3~bXY2Gap=8J2Uy?~@J1Q;DsKhpl-o|IiIu9VizXP0 z{CSyYa}vkrioQ}iMSRn%d~y|eo_ub;$Rg%$7!+dQGr~eDupL@(5*O_2ns%@mEZr)e zOtiEw;_z>)IRqWb7>A%PJMKoN)QLJRfKqEo> zIL0r1dE1CA((QMBd6|}H?z=mjIsukjP#X&)Ne}o!VN9bh<{U7)Z=c#NV0dAWD|(iI zY~+Da|L}gbBx1!Xduh=jpZ+VX{;R)O{mfMJ!oC~Uy|C8^VRe%Aw9K>BJ2UlY9v93K zycw|^yny-vaa4Wndaz>#6TNc((KK`Y`VMaH4zX*Zi(xHxe(}rHarP11okKRAKuCc@ z0&`m&O$^qfGec{mk+)^8H=m7x!3b8_$iZw^W}!N}=5sN+{7p55uaS$}u;h}w_P1a{ z0eMBV7@|o&0)sE*6!qv&fc3Ffy!&(%z?c~yN&>*k;>x`rX|9Tuw7qTM61 zd<;7YMam0tyJ?-QX5Vf+fAuUi_oEI_5Aeqd2CPXR7m=X#S(Cp#Ky3{x?8*Ds{zZv(EN2YEphO3oV*UHF<8DhDX! zYo8PWX3hk#F>=WWr_h6W(+sq-ZzO69*+VHj*zx99SruD7mmKlES)_!71VjH^?$u?d5Kq1Ct@}>1{`9i<;o}T;U4mP<})s)0@C2wU6kFZM}X?lvx;g z={5#5zZ$Z4ct-bqEp6It2qvHXpzQRgH7*+lv|e|6ZQ~`97_Ytsu$M9@DlhLC0JP{< zkD|=|77p_S5cdGQ1n>m}2ecOy7^$;vSY#N6`VY9F4QRX1Q3GuWwgp9})X!-h5HN_; zp1C>T4}~OSy%E8S_AWy&~8 zey!~Lx{K$_9D(AZHoGm^#4%#-l<%oM37))1{g35qpFe(aJC^~NGl;+rO5bLzovA+c zRz(w(%`XnRNh%e4iN~zip314NsJB-H=kKR&O&ips>dUWnu1PZ+Kz*8}A+p|@rqzby zd-2Va+gfe(k(V~nnl;<3Y^`PnHZ%$Rp7i^2kGf7XfBq5iNqZ&kSks9sGn!4mKOMVy z6K`|F%?Kfbp(^|9QtwA7AB^tP+wvBfCNQbX@Aal8dkd~XI@JH;&AYbbD(f+-2aV>x z$yr^k>sdekj`PiHKCaeDH51;Ij2I`!hc5;EwJv!{{o-KX0GOS1Jj=8abw$lgHsb=H zB4KZ$FGsz(BBWa2b`y8xt&`(j^_XmVf7lCc3U2}V=3v3f-|i<6u%NWDm2jFK?SUz3 zyAAlJcsKWPM%kr%p82ecczh{RBSj=vuw87c?KEuJpl-0Gq3NFzFo)82tshlOosCxt zrnPwy|6{g~QcMZl9Pe%kRXH9vTcF85OU_g8hbk$h@tkp7A8>-O4hNi%QD6o1xP zvS=#xhjgFz>%gGg$KzVndvIHDAng1q#VPdHhZ|7%`KH>|^@H%39}5y}`@L@m+i7)T z_qf~Ja!9@YYlSPybF{B3u-bCx)$8x;i9-?(q->^3ru21%%@m|nJf*!W&O;$JA`M#y z5@8=q%aB|U6YBKXBAs~gn`)X9gi4d$jr((|9+glR{o>bctPI~s;>v5~I5spfIwqJa&gJ8Z*JzqCs`>P&R(`xA zKCK*cAjfuVmHKWqQrwW}2UB4yf1dUeP$LCX&Q~hI{<`@`Ge7acVuRJZu_{31Q7AlQ z_kq)60znRtEH2iec)Sbw{{)N3w)X7vbhaRXzV^oN1b+toFkY=9UBuyotJ6cJHYI)= zpyzlbqK}XCJ3tu+_DBE1D@&DF^`PgysCM4DDy8^f8*b}jNfGYPOerYseC3P+w?*J1l<%D9LnH^FeUEoZr9`Us9GO*M~c;L zz1LN0B_*P?VDk|O8^(G*WH~LYB{x%H_d~M7$N?iMlq0y7`i7IyTltIEZ$>RbXC$)+ zh?>k?O%qEW4TqKwDywy>mHOa0UKi-#6Jw~rMH48j5ALV=)y?R{5%-y*PyGWj9(oZ) zX6F~{PYa6YJKkJB5(+R~cc(&08ak+|mUq>CG{<=_el#Fu*OaRDYR)xlMDL*S?iAZJ zqtBxKttJJXZb8bDlJdd1BhSEU+<0C zTszhj>X|U9>#bzt@?Dl@cfze?Q~4Dt!boc5*~x_mdGwMK|m^hy;WlOT* zwKi6La>~J}Z>g%&VE=P?7NW`F8=CF(fq8q#4R>X~3uEnsz#xLk^3Idqf)AbkLVAM( zH-;nFUZc{(n|VK83*KAy9MzfVIA8T{Ab-OkkS7c<4{xFNrycwP+Y-by%Duz|sS8NE z!y(a#o3i8*n}Ci7XrbcB-HdwI8WW<6Rmp;DJ~#O7X8;3uOwbz8{A@Ej^s+0jb#NI@ z6bP#w739oQvU-j@ zi3Ng$U7`}>K)h0d1cE$6ajVMRE@QbYT5mcV+4ueB8QhC*;vp$IeNM1HGEwMuC@ll# z>M`Bj=8{p*lsWEf5O~Y!sp)l3xq~x|sMpEnl+%Q&T~K>gcF$_n%y>pd*egC->Zy6+>W4=)DJ zhs@(t70JmMI7=pNQEfrl2uDxEy##FtE}xuj>|iR7$B84$vl)VljMKQ-!}@7msAV7r ztuF2?o{8Iv=!@DjNa2#7&UC)TMtMQ?BOpg`Ct;{?Lsiqr>EaCN&Kz0Aq=83SEpPN| zO2OH!7fp-!Ym)6=;GSRfEVZqSR47l0h+60T+l{&oKNAqw!6rVO0eb9zKGoou^4I-) z(Ot9QdJLTXU(P~y1we7Dc525?H<$gnpGMOiHkb!hJ%B1fH4bo1xO$0qF*5L(2@`ZX z^RJ%4)Byk*e{H<>8TPQ+SxeG*)|{Y@b%kboe8>22UDrRqXa~Hw*~G-S1!}2HczAcp zHQ0Wt+vR<_-7ij@=MX~*Bf6$Z8W-Ca-IHAECpr3g>U_FpX_PGUu{UE~sDOqAN6clojw7acLKKAh)CR9xIOcY;#(8sV zr{TH3EQ3m9&N0;M*N|kS`N7=@#0gwE!CM1)8R`C)%jmDpXqC|5wkw2D6LcllT!J3{ z7DZC`{$?#zrikds!J^Vj-1zCMA~Tc_U8<%s<9Z&g!obcPAQbuCDBU9#0Me+}Ee0(ICSTzjSx-(?)n?Ynh|g$$`D-tV*44uuhi>m zX>7PSmug?Pa&?k?n)w-&^yGWayfA(f-wUPEgCi(85CipIF4a$eV$!I9Bw%Bxa7ws4 zbGxlR=udo`wR<&D1|`~54Fw-w1(1NaoG}f-@9ZBBRy;u`f_;ZR<=qnph??ikVA7U= z6Q>5DGa&l+pl-Z(Pck)ZS_3!XE7QRr4^tMNATBr&tCnq`uf$)S4&cptE)#ZaNrM>e z2Y?WpERm)G%3xip5~wQUwf+F%>M8pZNri&aHM#TgpfA;Jhl@-fC}lvnS0L7k*idl= zx~wXHb%IC_Ot5ed^6;DE8ay9N%;o zXBC%ErPhjhr=t7- zw*~^s;5EZZU-I0vk&{Fr5XF90@w}2bgtMxs*R(Lmc2w2i^2uzPuxGX~@1W9)Gajwg z-}IbzsBB1M zA_7q{5(ghp1B zKzB^%y2o=Z+BI_2IpY3|Qh$gm_o)ET*7asa7Z7lzKTZDQw5nb}EUGseK?#Au2V_&! z{l7mG#r-TX*|<#xK|tdYfj~IBh*(6F0%VnBxHH-z2*!b#VYRX13VutsA+>%{j?EG9 zX1JO)1hkPjOY36T&x1bR2IN5ehQEI4KNsuS%aZ!}DVR@)7X}}9WXa&%0ue?`J54I^ zmVhdb9BICYnf3kxG9mXqOX*||p9whp10kZGd!826RXQoz@C%9sx1LggBT>eW{+k&? zP^WOl;n}Fx+y4juFpi4vop(Z`(YF0XSTm?1f;y#qynsP&&)Bymm4^?hL|w0Qa9DeK z>T;feVf}ln+hLG@M7I^=bU!|TfQ5B;mgFwm{y{xkraC)uPc~`EK>4FKCajKDp-n2&nx2tRc*4ue-1BNski#XBDD6&?j{x_f}p@7Oo^Y3 z8MmJws?cNa$u{zZ3QX_Kr7d@mU%R*m{+{>gY(#@Tf5yw}sxmTqhdwh(Un4n5Su3M+ zZMLgoe|xs7s>;4KR6O7ykg-K{L)+3aP5Y9zvvXn5%c0$g8_7v9^sjq_K*o0((USvo zY{Uk;H8q{r!U@|-fJgzi?(ZH~`5P*ZAY*c0Mn>rA@`skm@IdMWdXJ?)lG!4Rj8dFu ze#|>9c4v=%jMjAp{jAOCc=Ae+29*h6pSk%IIK}LsV;4bjy%S8@->u*KBT^^8x~$Lj zghxbx>JKf{31VYe4l6)s#)hkWK|2vjiaF}+uz5qE$3mB*yYcpU951F9*;q1Wip4jXc`Cs0fNmdEM@9y+JfkDaz@a*U`}dFUSWd6|8tUO0Pn#*_#T4 z*Xr++1s-DUQ2LCZ5vU+9U%oXF2ZE$ZYSG)NPyf9Nro-v_wr#qSb=!GwaF4od0v$_VkjXu!b zSz7%~t!K$zVbosqecyykYhsAny`1KN-28>#c16T8q)B_s3<2}0<)~_x^y$WhHP8!H zI-HW z{A-orLz~cMOG$oxR8pg56oQxrlPD;6|NhVGyjY}InUDbQY-w+AfB$|kh(JMo3sg80 zMcqI|MSHcbv9U2}f0a2~#(h-rxlW0bD99jz+-&{#=clTp;X#)vsACfoiH9yWW_qrr z$y~`&o@UIXYQa+Km$-bD9zJBO7$$5+-#M*k*3N={%L*C7J41i6a!y6m+s;H~ zd~&!_qS&La7%kH~Hgo(*!8?tf%QdX#GMG?Y)fA;K|DPnY3i~)n_BK;sqLw~gdD0@< z)!Pf@?^j!?*;Y`-h#>Qgv@bIHbI~X5Gazz$j>!Pt;dO@TL(2pa)$T0-x+2>X(bk(7 z*yJ8Ah#4iYJe<@%+%DDKV0VY2GooHRm)k;2LSiZFE~W<(sP^{DAymSvORo0zIiP#` zhwm~UO!5MvYGTGx;ETm}yc^&%)@y2@;K!E-UAxpc@C`0c05z3$BSEQz`ug9!!+Q#f z`W#T<&Cn`bh&4;>$+uLybEjso%&qzWPzScz>Uk&sO>GIAXUSl1!L9|_n2UpCvxw#* zZ|u|18y}Nvi&9D0Tt#$~w9AgBwB^>st!0jUWsc^48M_5u3*1xB?B!1EW!${>u0yaK zD_a49fgo6<$0WMmDot}9AHTjse18h-Qog@!ne7g;Pb(`c_S0>3so59xO!8T`5_A#v zi!oCOFoqUxu*R{1+-}?4u5vWBPGZ@GTTL&om1La}KWcgF9Nzc>Z)dP(+Fvv3N3A>t z4I*ld3j3IunG>2(+3yNeK?Y72_GSV|$T%GjSg}Z#M8Ea_3Rt@OPcx3mcRdLV7{6JE z9LTb$*=VGdhA8&DP=8=Wro13&6x`hScN_Pf;!%{R;uvfH1u2fX7MmR)k^+}HvrC;h z{uy(u|2N4U`W5?hna%ENRtJcRL4zU5&kjGT`dx0<%TCM8hhCWe8N@~K6P2QCd42ie zpiy`R8J#Js1 zJf?bN@+UpP?1x+rRlC^r`92d%CGw%^SLu^q4nTsIF(;1;86jaNZ|b9 zGtgw&T-D|kPQ;ymB5)OlH#n?fp)g17XCN&B>|coXxJ|>f+Gfa4qQ@-6pC3YjpF{H$ zys+!Si-@Lh&laMUTptq2GEWU7o>@PL_qQ5`yAEkxYM(Uv`OD_}x1OP$1$4Vp9mvzZ zgi`X3;7=B^mFQ#wq_!vT8p1f>2t-&ShZYRwO$zEg#{}2F31kdx$Kl$hAIZcip=G}r z)6k~?=&xt820&!wJ%xs_0RY$2Oy*m2l#~BKBe;Hl#~1;j@w>?n@GU%0lK;4y8FK>~ zIjmk?yYG}{C1u=iO{ub8tc1+XGlypTiio{M?qoZojwG6{lNG*9f!G9g8)W)IE<4n# z-mV${qHB`GHh$3c*B_-d2vd#|MdRBQi)1Vdwa06A?_^0m;#HN`^0xiXT5nFK1lNZ_ zgX{BrF0C!#%3VVo2Vku%L7C(*A*HWTm)ex5yrJ+d!cQ5rb$qS(5Gj`4)$NrI)}9xB zcpjN2;vQ0>3~xeUZLdzW2DqCRW(wQ`tFzkAkQRlEAg%-DyDmhx?Pts0CzSG)gDs1P z&hNA-(iU#Mqm)hy(2cV?>a89=NV&Q~52@utf6|+g9v$*5-W4>9KtCZ;K%yP$96QXn z?$fjaw+RMm(CdA&nT3d1psF2hun0Ie;- z2}tTtC$BjdeuxxgdLl2~QUMzWge*U*0SI+K+8wH;RwoEap6HY!AzF@FZvfL0g}4Cv&%15w{GkutK~rbGNup;9cn{Ws4tz zTw2c_b_fq!kriwxymEW5h?2Fhc=SLpF;GaYmjjX?7$6IXSOkG+%3-GM?*;G#c=NN7 z3e#nEMo_5*_STeiq(7`Keft>>>sC$5(T5MgbiT`9ffjlCQmu-5^2{m>d!J(@%#|FK$>bUPK1xL;D3e zR@=&;HQ?grTu?s`gdLM0vNX4T#FO-U1#dZCd=G~+MV8Q&E6B^92637%rKj2s0S;*f zUQTh>YJ!kVHKaTV&OdjGUhe-8(Km`y*^#(O57Ao)ZCuqJY^dYbF`p5o{v+vc4xCeO zl9|>(*0~IJ16mmbSFNDnI*@+IqoY;CeT&Z0TFZDo7pQpKb;W3(N~ea{%zJCT1#f9O z3Hn0d0?oVnK%b9PRuCBIvPjn(wC#PPSqSn_&!gyye;kL-QM#E5o=4L8dFCPu_iBXV zhj6dN47akA%0N!3GY$*7&o}#eOPgZ0@Rt}fe zfq|F+K`Lg!wLdDq0#~?4k6gT+HS3gkb!6o*tA_e?2lr=C-h9(U#s;jD^&?oz@3X>S z9nbBb)y@MUC^!(4K6(YMnj`oTRu!Z>2B9y<0^fPvjD-esYG6P9{D?ab#SNX5T%aFG zsV-xTss&VE{)VI|R5%@4-ft!K!dTuv@hgg(%zj4TXKLu>&uz=(v%sqOyVv4Q4hfI{ z=GOn-xXsnWp-dAN1n%(Og5?c7#0ZCA7D(8JepTDJFMudJ;EtFJf{wdm?KdH4hrR|h zTxtQmPUwBtU`MAxhOQ&rr$W1!4-U`umWCH%MW58y1o9}Gp zK$(Qpnh8>phO~P!GEGksXn^sQ;kQoDp^p1IZg*91c>;Rqmaq=ST}C9M1JM9H+)7_p z9^PGhaoM?UZTKlW~b^K}pJ_y%#Mxult0SJB^?{cq_Z=51RU zH}`ms-~fV}hr_!FN2aL!AS7)D!0V1lysa@O4;&*9_jdvqXDC_3C_Wr|k@$dO7F5sw zrkVXna1Se23q}UhR>L}LYIAr=0=2`RG|#{CWQ^nQ{2FBXcK``U{{98oWcV{8x+cT_ zj#M@-UGF_T_q011@)`gVMx?snkeI_|Wb}mq1p$OvmjCTa@_UsZF3M5N0@L^(KkqE= zu^OjTvZegd`P-YS8J2?#akz`2Mj(ENY(luu?2VmT;ATOT{SV)k>+sRkpm5uyIs4O4 zRz^xeD)c@McM9f-3wPDSs}S7(ul@mLH)t`G>bM7P|AAcv_O7S&y?=cM6v_qaVm^Y7 z8IH?H{$NHzzOS_Q5L*HlryUMcmS#mSp7B>NA2V#T~TSaHT8gf|h1Bn0x>`F+LnmFsx9}@*8Vkq0U66adJ2P zVHpf=j{L-8Fcb>9kygFxV5kriR2d2YCuqQAE?EnE-Y&i_KXQG)Cs1FkPlf#fzGK$} z1z!%H9g-~q(T^LtS4lwZPZlwvz-#G7G%&fSfwI>M+Xt==CeaTfE-LaIA8uS=vAu@K zPqc|ozoBB{7G5k?H3AK!4c`cRMIa5iP?SP2u^5-GI=(<$&BuELHn?nH53sh*49H%- zwXSivG@#%CLz@3fKmgH;@bZ0H{H2ShkhUwfvNB>8QFCg-?H^7t&tbwH@Mc57vEHVA z)@IZjz@oGtz)xDfVh9YHghP`%0^&EuPb?Ay!#fd06TpAvu(;MxJ~5L`dQkzDd)PVc z6Mh1>V0HeSIuBv~F;wybq)nfmRJ0g8+=x)fssZUH{`W|6CL=SuyxuCzL-DtWF-TXu z%z2o6MyN?(^a&&}a}s2D;?AIGDQnLNN173Qd=z)Sbc9S2uUjT)6l_FIIKv|FyB#6g8zkLL<@?5x&h*WEsZRGbH-v#cIS-SFDC_jQu zVr&#f=|(R<#U=2r{+v#h6E?y_RjS++e{Y3|P$D0bK(8*9=muJhYWJI$=Ff6-@H{no zNR7%o=-&1bzi#3|TYbphcS=(QdzTf1=#@+omVxLXh#J6wA+evqG^MfuhNVQ$h^iSR z5nGo4lV{=R+Wgz431#QtQW{m08?Y7=Ft0b6o}y?wf6X*C_|6OisT;5Xq4z4}~WeLOXpU z0sO#+iu0XmLcJt=4c0V@{6E!ME|-BA@5~Of!X+(){xk#3l?AMM%^u*$!4(Ha&ZnZs z^l%k#=o^*rOE90Q!e|xNBNQtEEKsfsqCVfo13vds8_$G_>!glR{WZ1(3W%vt`{4Xz z>z&EeoeaJ-N0-O-jL~0T0U7|flh9rfGD%RR!5tFX2;)k(944JiNIdXJyE5LChJ*oo z2i%8Urjj*)O89g!L)ij%99q?eJn_1Z_`s7eKD9I&3OxmZQ|)DX-NUtHKW+{hVG$M| zrUpPOiBN4mPf$st=aGLrrg7$3C3+* zbkC4(EzjO}LBp{eJik{bGBX`rptn;i5GSQ@PeS_zdCh1e&=+NVdc8BMU9-oRuU$|` z8S0mOHc_*3iC1&39;-UE6Y5$wEHi4}7x86;gnHwCPT+_b$p%SzYt{=|z6b94GD8QD z?jLi$EqvYQqvU^7_4K#mq z7|5Y@orzn7;mn2fGUx_oLP>5}x9|oxhf(@Xh!e*p9n4p`hFzT@rbQCSn<^Rys_TkXqs!N4OIa2F8oF|LZE zi!lr8lQQ)w9RoGyi8XQN!+p*LViUa{RA?iz#RYvL0^`^^G&tggGe>qY;~VzP5r_?@trx| z)~NXSUC=!*-(7APIr!mA<5^^0nL#jgnH>QwJoGtdL})=f0~BkPs^99q%RU&W06GGX|9`aZg=YI*RNf=ZXBS5Z6lx9-rv`^(=5E`w_tl?ZQC@unuRMDFMqE)BH}}UmdLvkky6beFd%Sf?PJk}^ ziGMin&qKB@Lv(#5T1F0Jt8!t6fYqm0waF`k!kL7piYE{Pzaf}&%!igpkSa;qtQ=MIcPIUmB7@h{w zvI5bL^wUdhe8X3A4*gj_ALusA8Ex4R#c^bE<1shblxkG;Qmy+W{lR-r;ZDN^4spq z=mpJR&}I?pKWk6rtv>2SCaMVR`@et`RHt_1WV>$W4QA%at;r-Pc!;^Jr}veOO`DK3 zr5?e=f?P3J6A-=&lu4Z;L;9cMzAnn|B(%oizpl^fiOTF1&YFdwZQ!Xk=t0BO~KHP^h{^ zsQKgOt1gIN5`|SubPz53JO=Xf3Ey{T)GEIGP2hP?fwT7i)I-CwcW!QOGRdoau2?Jx zqtH`XdKlt>abkq3&`Td5A1F=C_Z3o%nnK;~>B_{v7J;Usq9T0aa*v%T*N)p>e?cOM zRr~*z?UpI|lw3R7lWV4+q@<*w5nJHISeN=Y$fON`QruZ+o&sjbZ6empX}&iyEG*l3 zQ5Smv7Q@?|AJropVR(hc>nJX6(({2p4y$2Uy)x1K|?SBz;uZA z6-Eh3_r3E2kk7=*cogyS?y_5b^i{!Yikwykgx zPL{Y~gF<)>DhcHIbnj((eKi>28|M`c+ z|BM0Zzdr^jv82omq zAfvqwq-g){D43+y)>dV-Jk@y?hqwK|el0;EwtVTw+4=c-aIEkt1)y;fLb$BJzjbc1 z-N?v@;)czoiY3JD{#R>Nz(atzyfkE2|8-GtLjkqH!t&wQBYS1#Ht>FQfys^#r(e=N zvqyGrxPSvZCV^=eZ?Vex%S<9*O4W1uhkgG6!K`*83{TFx;GF9D{@c&ahAH2;ejRbJ^gPx& zFqjvcH{iVi)G`Cd?Cj1c%|Ve#198c~S4Y>f!B3w)HO1Yq>^gf1Xq3)#-8|^FRKC;x zZ@4ZP(FP!n^%f+<)^}-4dpj@|ztTX~4a=+Iy(@}K! zw=7o~08oI|sEOVT-LwasLz3tAbZd%q1$4+E(Q{XHUj$zTfDWDER|WGA23x_8^4H0- z?&F^y2^c2W`aHZ3xFtE?ASi3j+rkNRfitu)B z!Ham!g102{yf9csVWoG9VH+~IxD5n_DU^C*anR!S1q99haQ90TKM~a}-_^J*0PRxqG9y zUH4+mjO6%S`Fhv6t#;wR1*T4Py zEXnp@e{ZCZh!af=ZY3tu6+{lQNyd_CyeJQa4K`r91P29yAN^7w$#t?hVcC|LZJp)- zOJkTeM$*buaN8<$m;TH8!T=lpogmXKM->RcHLQX*)nI%Kll+2tekRSqSe$rupZi># zJ|^n%ua(-V^DqprQ}1NHG9Wrv#YBtm`kmYLnBnS4qc5##2HnRoS$MCW>q@|Y$O*Wb zyd@)JYe_vOUne3ex;H6)pyBRD52D1>ghIl?`HVpGooY=P*${_NmZwx#mV)1kfP%Kd zk3XA{Db~59xLCug5kMnjU5bu-t#Vu=dhm$;9`yWo=4jx*Vl7tBz+;i}fA`}Uem5D~ zN^vgb2CZ_i?)fSv(pUE-C|7+9R!?H4&ey1kEDZaTImwR;AT0v?{7iA3l)gW_ z1umVeB6@oIzjJ$N^PQit4xebC^eL~f@J_gb*we`b=e)KtA|5oc2(^NyW-lyFE>AN$ zLBXdR3j@l^%Jz=eA74G^cguiVB+R!GMjGaCH!=m`rQj=E>zL}`w(>!~f+QDAXxvwT zxqL3p8%F8zud{xw6mrSIgD{(XnoM{4hW44>@RJ*$%2BRg8K;Ex7Ws?dy}`NhTHXdtJ7JWAwVWniuHx68pJ5TJ>Znp?SD=Q zymu$_PT!R+DHXUI{Nj}8Kh_1#dJEajziS&wApW&<2rl&>h^imbjP&r3xX$P*KW|^m zsa>=*5ofzUTd_aeXItU1wQQ1o`>PhwNG+AX<3F&^EzhEh2Nr^Yg2eZ+p-3FLH6AV1 zhxCFhlquDvwZ$?d|OkhtvSM^ytwe={>)s zSqGt8*QYR=ttkV&GVsSbEfAtfRZdAs$yp=Jrigp{7GrbCIlL&>^I&HvNz@G_I8?vt zo*zEt4OcLx4N?YYvv~~g zA7zW5J$v?s?I63;1oY0$edAlP=`ScKh*&w8Guo@&$+t9_3m#aZUEC3Po?!#Bz3uON z7Td7+Sj%N)zd!53<(JW0U>)a@)sPu|6_ZGa>CMTxW=Ddk`9LO-3o>X1+3I6L$L#b9 z^YW~Ea->#Ik0f5!3wUY2X1JO}U#_|3-*H7=5x|0p97+BmnDlKQaLB50;}^{S-~2= zTjl!wuvb|Nv_&e%zkOU9idjl>_~CVg_!EJVX)1&Axyx8Tq8pd@XIJz$9J82v2u{6T zqKWwqqhvrmS>@h*o~bd6r96Tyal*^i0;BqJ%c;h^HCSWDh(A916P#xD;&F(np}&ji zxjiulMZ-}N4{PFPc;N&oJ#P*RIMc!SB!vq3K@shX$Lm2ht;g=haPj&O`f`m1(qk#F zQ1|ZF=KZSO`beDv+(aXcD81*`{iEt4y0E?6qLH{s*|?XnZKT@ok#<=UQ~07ewYH`P zbBm3I<=(jz=rgEu?h^LkejLC5cn-rG4C!SqT2FxY1<3p}d+`tv>Fn9J2<=!~Q$z|ApdR>Z0YBQm=xJ4R7NNo{sXl3)oxF%gnSK?y@0ypWqwWJ@$$&H(?sw(kuzg-(y+e z!+J?l5=349P@VhEg>!c*B^0x9vb2_j>i1h8pQH0)d}8I8ijm6(o0;vmY?0+dm}x{# zPHsHQFtW8g(~;Jz%3-w2{>^MvI@n3UpM-eiV80?~vtn~Pxwe~WAz#>e;YF_5oYM|? z<)02N?S&ABGL3V@x{$q}cWO<^_qT;Cb`du6wM&sd9kZ(cK(e^W=8y`vflZK0_8NfR z{l*R5F#5k1j$%2$72&mMJQMnJc#(|Zpmy#HWq8#PsU13yFnKR$goMORg5qL5k1=EI z823iSK0}_7UjbRN#-)%(a&)+hJvWMRc`k-_6hW$ zPjw9~lEiAgSScd@(N8K>$(%4QN*wfKD2Dq~mibwa-E(w}!FVeu`f}}`vt!&hnBs4x z*>(9Rd>VS|LK<)?#Y!{aSEdB9M^lPzZa1k%T*c-{;KP2>y{RwfFJm(|D`*EV&(_lK zy}#55(LsLm(uqFa-m>YI1M^Rdly+YJtsg}{{ScXAQCY?L=u zbG*+{XcV_<=wQ|S5Pa*Ok}bd#I1b?P)&(FDjJTOdldh7;`)oSqDZW3~`Z&>HwXc=` zj#WWtW-02db!#x58Ez>ta-OD(7d5&xf-)4L_qWDhzs7KUubj9dT|mry!{@-xv~|=y z>-~b4G9CLjo{@5b4L|iXKJz#_*P9gz=k|_DWY2_ZSa+MJsLc{akxO3jfqH87=Kk(V zOZ6{7ND{l;yeNxdFn~n4;k$S-4!6+pwmFVUwq zGF0VzA+P{KBB4g+hR1$|fws``7l;|IZf@0*lFU(-Uj7$18dg{Hv@N*B#5fo3+i4S= zvYG#+diUIgr)>&E(xO-`3TeO1wQ9GM*-LvHLtP4FL|Sb7ne%CpXT7-l%iPrRbiO&< zvso}27N^-HDp4uzzAP@g8SS-2f{hG{R*HqNh+|EtflM|_28)l`xZ|Ih1rZGRb>P2=5ImPonkBroF7wr24 zOWO52V5v9Xk0dAoTceZ7FPn}yAX=lhkGYJaQIz3Ttz~AJDdh!2=0NG-&DI*}>`niX z$$;T2p7H8e+dDe^8EbCUFD+%ch|)#%mpChwEUxkpO}M?{zH#TeUm3IK*L7tu7<+GA zG9LAl+L>j*UN=!gdcwUipgEETS+VnLNk!;0XV2#EG594x+W+I89zn&vOvumSIWI}K zV78OqPREHUG_B22zduWQv*-L}eDpTc+fbLJ{6#YuA}eD9IO4(EvS z?_8xzH{IMy3b^yQaeH6$HR`hgOE*kgk5D#adGj9gGc_KotJh=p&4nFCQ%h45v-B(U zx1oPg9EOUOU!jz{&GiR~75C{8gYv0{b3ERlhjgVtBS_0St~%|Vg7}_Fe{Uq4T3#Zo zTGH%;>clP@nGwBt|9e(3q*pXUH=$`EHI+&tJ+&oxAvQ4z5qId;CYXJ@SI$aZKtaf$9Cem19$C-<=c9I3AMM*oo*uRe zQk*(=S0FR|7Qu~ji1h6mz+ks8YGl7OR1fgV=X&`#WM!C=@#a}GBa8d>|* zzpB9Q0)kBDTOyzg;xsxVW~xRic_jo{)J23X5+l{0T^{_D3e~grb7pUF5xgvheWMnn z&tqJTl`Tyq82x>q@jxUG6J-Vcj43t7lBGlC=AY6tFx=t{Kw85{AAMnF|wWTKjl6{~(ZGj=R_$@~;c_7nQ;rOw2LY6@1n%1Gc12{R?K zM?TB@_Jii!NY71njo(AzB#bA~MfdIW0*w|RmlAyJ7D@AK{)-e`gI)o8Ga#XOxoKGB zp81a+Qm_46LbUTS1g2MMb>I4(CvYxvIcj5JadGJK5=9l|@BIAydP~CWKq$5_dTL-< zl{wFFHU?vvJ*U9?!cfuSd)T**u5e)*9I)4r>L_f5T+eTwhM|)^Za4J+TH8Bt>dFPu zI77&d^#b!X2o84jeShiLPKGH4aU9i2p7Q6;_VyS_sDFm*Lh&TV@)FjMST7)n(K8u# zr?&j0G>&$D)0RZhUPY-=T@t;34Z+@Mat;H+6j`s}i^xj%+qvmR@s5b%$?%U8^Jq(1 zRZltmCq@pk9*vyS$WkMo--=eP2o6{RwY=Dx@Z z2<>5%BJgLBd}6_4jt|O_Cxh*+Iuid>FGp5Xo-uH6xX#8VD-5om@21^;i%)No#9rw0 zWId59s+LZH-gzWWH6(1k&+hcxwbt`56xtrjglT`wf0E8Fccf-KncG||XYRTEX5OUR zXlw1XLqM0$NF99$SMQsN%>^YzxJkxtni zYbFXxuo%xy`8+%?3J0(n>pAMBzf_jl=Q&+A2AuO(Z}0p*NJ!*g7T;Z};#SyIysE{= z!$YS#KmE*4erLLu&1l{vd-Hw0+~aNKd#0wJwDWHNulBz3E2{oqTcvSsDZwG68Cn>I z5TqMqkS-B{p^=i3R!U+9fgy%&5J{z^#36?6776K=20`F#?%z4*dH#Xt-ScMFVr>?z z+56hp_xfC)!g|HY`xx2J)NXbzlhaq_|EuRN#gu!i+Dl|FiMIvhJFTp*9b2-mV@4PS zaKc{WAG&`yQ=EPS&#CYEREb=-xJZ zXb?u@D;Rnxr0|593!#58o^)P^DgX#KC;z8+)PDx*g=fXD0}>j5PHrx%n~U@Ow--LI zUxp{jY?Lk&U|6Scd#VvNS2&CZFcNVDEPS2_c_vxlRig;@qKUiys0*ALFHf}P zsaA0jGp9kO*Vj84mxB-WYdWuisnlm-7>F_@3l|F{+1#YA~WSZ(ai6)_nd2e zRfB!0HIwJ%A{Xf}l~FxqzUqme(20Iy6iLrBQ~wju!dEtmF!L4jK{Bs ztJ7Q7a*J45=6{rrc|O7xA{gpsgxjr1IA4!k^nnn2NLC7A$TZJ@a7s} zNpDju&89%FPlYbD@V&q4tq%cE(y>=HKVreKRl}C^ar0O^|AriTe(pB)-}^5C?R-Sp zq3}ih(98yEuk@Nb=je^#+%ju?wcniTAO^Y|wfvcXWEqMuardHz@6zTO1nSJ!s-pd_ z)|wtaL2}Csv_-DnrE0PoS%oYonf9BEDoUNd!y}sWKZO5$uIZgVf`LNDK}+HU<^DD= zVS30EZ4cjHlj`;wf^Xd)AGMND7CY@r(~ zPH~Jc@S%iqzSv!whZ}%>OJ5015{^R0iGMZK%IU^5ysdmcA)5D|CVMND``Ue7EhDRl zGBN_1UGN7xsGE$!B2%B>8pKN`2aJNheyvxWES~q>;=gZIY0ABA(mV1B_bI!NYo6E_ z@SnVQf6~5sV)ORSYpr{;7Xx=rOueS9H@vig*zbw|nc%_xGWwZAAJ6*s6pR~{kUztt zA5mIUCSR0j|x7S_a*{&O~neT{kd~~*M|*(<5KMz z)P0&VT7C6*O~%$}deN>fZkj2S+a_1{8pZTjeVT+DclKan6I!tuuuf*Ryd6IbvmpnK zCiCj{)+nV)+G#ir^cePv)Y~GH@QyYZHF8yK7c+semu64CQm2}pGfi;+_z9( zkH4vMdBmclS5e*_;xLf_X;^x{TPT*4DHl9Kigv*Ddb)R?otpXSA?^P$r%>yI5~4^5 zT?YWbgy8nfb!2OEpFKj6sF#>(2RbBX#9D&x(i9h($2x_Cgx%k&uu6dsI~NS?3frQfy92Yn)c03a)7*a0L6`FMENR9cUyoKelV zTnHiS(z?lfxd`&HZubd^3NFna-jYN3f>uiP3Pt3dzv}%dvliCvX zV|UtlP0*aTXf?^OXR2xuhjlHNs48ingXK6jmGb&ky3i}2f!|o5^Gy%#;S;a*BE&K5 zPH}!&SX110yhg8`bHKSV6z_3Qqd(UY^(CyhtKL}?;Nz5yi(%bw56haMHhdZ2c_sQ= zJ{TM8A&30Om*h_aBPN2+-wd-Aq?L*iCOaP0Yl%>)$ztF6DaHW+#%I4C_b)X?pS0Q$HVALe1**Bv5f z2gC(X4hG-7QwxL1|Dm-np}lCid@o2inkLFVtbY4EuIImhlEQ7Kqb z!!z}&-lXpFJW`p?Wt93zEiPx@Ppb=SC28B+`4D)NC*}9=Tg;`i=N+gxgqgS%u4;LL z5r%-ji^CwQijr-*q6YX@WzzM|8d@g0JQPox6cKMzSTCj+`E=k|Ef*N@?!WKv+u)S> zqt^azRw(|(%IbI|ION$7tuo^iFAxG#k>=bGIZ<$NN`B5@?2pyfprg{RniM^i;WL`_68D!r^`rBEB;Z#WKhi0sJ_%@1km5iQWid-UcekVU;hyGRYMH(_r;@Tm7{(BBUb18r zTMdUB9JixV^k9sK^BmEXadV}`>l&iw)T<&&XY`GLR4tcFsnvvymaX6KUT+tuO2|1m+M) z@&^iuJ9Z=s*w&(=amiGZt-M`LHNy3aL7*pq87U;;*7lFsvUp<@Qt(+sB|@iuilu3%4B)<8|*+QrA2H66c#mL^k^(aSQ!t3_t+MP9~of_App?qGlI#k z=JD-rFUs^)KWq&IMp5Kh#IcUkfsRF&fMN&ABz)aJZA@2{W;#@I2pyVefq;#tMVc9( z_|Cpn{n3GKOyqk~AXps=F}D-Q*KXpSBJ%9E5w!q9EjC_!DfGwt8MOZxeTaZ!yYx7- zf*#8@tFU_z@L-KyAEi@>#IPN_&fNuAIwYp-E^*QKUM0moKYUc&KN9MSYrFs(ti+L)}TbI+=q09swQgbTQTl6Q&Pa{v|iC zzU>v(xyLv|CAo5Rm{)C(L|U?3N*K-Yl>C91S&`XTkbug2dZhgrP1p~7SEBmMy*4t$ zN%l&Wc+i!kpC3TwT4rO4`tR`yG=m+(0sqh4BjG`g)m`rM?|5W_+c8ge?Jk>a8T92m zP|hxuJnMy;5epO3x3<$2T(b`xw?`gcCAO?}ziB3#()+mCcbD+E2V};PsSi_4$2!)w z8>#m4H{TH;8sMVzL_icwDyhP?ZNB!Mt%|ioK85~{1)zp#ARUA_9Gt!=4|C9pFbGuETA~cT(@kn1Eu+mh{4!(k=O_!Is51h3G#= zqtAFLM+|4f2Hdnl-SHwwO?ZPt`GGkc-pcg;f56m;*V1PTcwW6=hSro-i~3xMAu=Vd z>aIw=B{GyChEb|V(5aoX?_S7$xUWGTn}fGM8E^c!X!u0mm+6jr6(tv&Ix~q}^w}3`{|o9zkI& zM*wQR2U%rC@2 zg<%(DAZ#h~iFb_d!iYBzj&GyfHHt))e}Yf`?l%5jl*LD_Dk`BveSw7)jTB7bsWH); zb%&eKHdwA6tON8MFgh|N1EydX0u`H2ZNO9l5}0=X&xfB0OI3z2s#Zeb-_1eKQgoYR zJMF)NK5F$$W623x3t5@rmeSoAC|u%YaXnmdlkW*<(NWT}_qW3gY!#01Dq3y!&{%`+ zf1zxvIFJpM6G`)anEEf4jXVVH&G3>Uz@tvHA_D zO@Dz@d%sL&DZOK523`B;w1f<|vA7)K?bW-0IoGy-B0#pNicG&zKF zE6aEO&G@+d?>ut$qHi3+e+XwOA4h$oX#3nVru5CvzJ_DSN<$2Ql^lOB{7%kdhwOqe{4-@*?K9ax4R>ScY5fw~zBgOpI z8ebN(HXRiOIl<<_aPm<%0!kHUP+D-9b8P``CVxl~5Z4;c+w43%SP zWafQxv(}_SQ8vy3S0AN3M7Jt@R5Ihk$53Lr!}&C(ux|}#_y4BAYz1}XS}QrM%R6aS zwzoMVqSKywWSDf&I|b~t$@t$!$f#8-sp|G%7!^(#B|O%(nz1GkLU5HFKJ5}<+eXUE zc~b6!CMLQl1^Rc;f?YPLM(^L{O>T9hZ)D-FzDti(E)9xn6HOXfTY)j8M7C~TmI&JqiA$y0Tuyn`z$E?d&E0UId;B>m~3Q|cJ>$9AAbBDGp@+zO6aXCqGOT*t@SnQ@`lY%e@B_5k{n+@>xYcu zBX$AL9`_U2QIe{NWn7PZM}#gX+@ry%AfsSdrPb&6N_)Way6J!LMWHZul6%c`eby;;`colc zZ2s^iMTbWvDTJc*@4zWJbXlNc0DPX%vkL!crYv`; zTFc0VY(KD9XmZf`V`=zPCRQh?BJy?s5qb*3a%AKx9J3jJKqk{u87uPsU+Btt`V(L)%H)Spzb-NG!De*_SleNPXm$2Hzby+P5j zi$NH_ldJU^_+|ppxMoS9K^KaIv}@wHTWSzQnPBh&s=0a$62XJRH@JN4UClfkj*|j2 zp(Ms4T+I94*;HF}ZKV{yLb&<*OHWH+H)as)FQU6rXqCbkBU`W3-T`cRjlOJfX0a9& z!cua;969#WfX6;wmkv#(w)XF{Z_F1d8-m$HrdqiF`_u`d2jU4e@aa1WyY%`k|1P|i z3sXZpb|f4d1`XK3e9Z~{+VXe*5Znj(lKD|*RFs59fb*LFQ7%SWRN}_4GH;6gtP{(6 zr@z`hdiZGA)8Wl>oJ!B*(Mckkl50@Ht3;vL!1?dDsm1HYh`q%ZDn8k9Sz zNgVe_JCFXz=ix%Uu5RYj+sgg$0JV|)&0p_Z350q^XAnO`uKBg=!!pfEJUh}2J*9>< zi^ZLYE(KrY7)i}HvV2YN%0XUkh4UbRpTpZ7cV%KehpHhW@E3DR^Fls43=DjM4CXVD zsJ-;nu@{(;4zQOxu>WQ&Dn{|d_1F6)LhLoL1BRbNv!cuvO4o&1$RK6K?Fj@BRed3> zj)JHlY4hW&szk&e!}HaACgQ*9)jV)`-F}_DMLJM7S)`mgypsA;w`$scBq0-Vgfz~z zF)_+AbuT3!{v-6(c1=D*ZcqboTX?A0Un3Up#GBPPXECG{MpRplKMOpd@>2Tfokmye z&5UpgFCXbe;$_eUHLmgF&#N!T3Fr@~4$0rlCf+vPR)V`-g~$&?I#XSA@V zkrMiZ7r{KFKT@(<_60dh%g@o{sb@7nu|mmJp5nbNVDh8Bg0_N2&9c+tg}xNn5BZ22 z=O|QlPpsH5M~6Tye0Mk;sB(Pn0?T~tmOE380c0@Q<<5#|RAl;#%-IBWlgN`5?-bKO zfhtN!0jKhy^aWp1_JYy4(`;RQmq52#&S4DDp};|pjs_P*0mdmS-L z$}GGxIg&));W|_?_?(^|!=k2A4JUj) zDCq`2&VA@KCLNVgEm5amHJ^U8N7?{ai7qP&nJ&9sX1j8z%hg;+E;v#d^k6420XU?F zmD5f`tA9#=u&o;wN62T_$J~4ihp2LwYez-myK0W&M*`~fw(%4v5#ZkOw&^J2sazjd z1oa1g6)d-P4D%%tny_)Td%+2GKF0QpJ2ksHShvvS3x{{!3>RzQq<1ZhZ*C$gsdjvA z@krGk_*WPMNSG{ZdNqN!wnbzKE$?GXui;2x^Rfh7Urb-&Ra7|2ql>H^F>>e>WWtpv2}>-?AhghKNde z4mSqRgXL~v$ioN-=Za>xk#%vU`apRjzMGr-M!9YgvSBfb_a2k9#^hCEg@0HPJ{8OO zKR_LP z2et}D1|2PFZ}~zfL8_wCSo>xR$FQT8b_q|xd>`tH=G}&^Z>P<_ht-PA^Hlqc%7*0L z5drxOMR^m(rJ?r2HPVQZIlpaB9vrdfYb?FF1!u^g87Y^1%f9`kPGC3D@sI*p4rgDh z$fRl8JJmX~r#W{9TkH%3mw=Og*>Lb3f=l(y4j^YX2GW=ym(CjTk2MM=HCoiOioivC zVFLcY0Q)7^$kn4mU(!4n$0am<2sEe8T(T_{raAjw9#7KNYN3}hxoLZW15MH|6+@64 zTqG6wJhxl!n#zwMLqYHWh{+(0pH-k}`nI_wyDuZQI{oHmsd^(->^BgNV8yv|cR0Ea zCXfoM{)BNf-!hB8%jyi|C8*$3?7@KW3EgjsnjF|dhu{D^x6Eqvc=Bb%ivr7@f$mdh z#V5!26hA|!4)c*q_$lfd@@v}UVnpwx+lf49fDdjN{|Zm%vbZR-XFkyQ9_s;MU&MlEp`zJr%W6GAZo5Cl9|h@$XRtLfY^bsw3wIVFy1&ay=!yw?1K0deus|nvQk1vSjwXlP{ zYwq42z6tt^a{(@M)~tLQImFmEk2KPQw=Z+M;h7uI8-A?s1`p7|pT@QEfedy4VRb4) zp-Nf2Lu2H*@co^C2N2#ZrI^19r?uCajAE2S1UulA?&_D$G`w5q$UGc$*S9;+_+WNR zpqpUW3ILuGz>Z&U*m2}vcH9EGS~5F9(g4FeVaw|Nr1e|&KD~7si)$-j*dJ82HQj8) z^h!3PQz&`cSnZDe^>{t|nnaI~!DNr^jd%gnO=PPBB$(6Gbm9G^Dq8bzDyyb_-*(qn z0-Oi=IOytcb6O+$I@kB58Ry2b)5wc-=xzJ>Qn_H9qKCAZdnd8-DzLr~)B6y_V-|A< zdE=Pb@tnt{b4+iQ*@u0FQiv>t{XxRaak=;q_=!ziZb&Ebq$y)B;j8!t0At0bY`L3& z`ep_06oeV>gkuKmnJoADdnf2-`{2xh!Prtc;0@d#7#{r`Ohoq|WaV0S{42kGGfZMt z6+v3{jc_JjiUSC6q*^@ zP>~SU9=`l%M&haT6~2N5S#8!8`#Ifr;#$T2&?$prU@kG5`^FPYH3`xJHuow4TQ za}IDC2~uzr`c2z{nv$ksxx9Jse*wRp BG5i1k literal 0 HcmV?d00001 diff --git a/examples/trials/mnist-nas/classic_mode/config_hpo.yml b/examples/trials/mnist-nas/classic_mode/config_hpo.yml new file mode 100644 index 0000000000..3c04a62f9f --- /dev/null +++ b/examples/trials/mnist-nas/classic_mode/config_hpo.yml @@ -0,0 +1,16 @@ +authorName: default +experimentName: example_mnist +trialConcurrency: 1 +maxExecDuration: 1h +maxTrialNum: 10 +#choice: local, remote, pai +trainingServicePlatform: local +#choice: true, false +useAnnotation: true +tuner: + builtinTunerName: TPE +trial: + command: python3 mnist.py --batch_num 200 + codeDir: . + gpuNum: 0 + nasMode: classic_mode diff --git a/examples/trials/mnist-nas/config_ppo.yml b/examples/trials/mnist-nas/config_ppo.yml new file mode 100644 index 0000000000..9be8e78570 --- /dev/null +++ b/examples/trials/mnist-nas/config_ppo.yml @@ -0,0 +1,19 @@ +authorName: NNI-example +experimentName: example_mnist +trialConcurrency: 1 +maxExecDuration: 100h +maxTrialNum: 10000 +#choice: local, remote, pai +trainingServicePlatform: local +#choice: true, false +useAnnotation: true +tuner: + #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner + #SMAC, PPO (SMAC and PPO should be installed through nnictl) + builtinTunerName: PPOTuner + classArgs: + optimize_mode: maximize +trial: + command: python3 mnist.py + codeDir: . + gpuNum: 0 diff --git a/examples/trials/nas_cifar10/README.md b/examples/trials/nas_cifar10/README.md index 2f3b52a869..e6f03e0b58 100644 --- a/examples/trials/nas_cifar10/README.md +++ b/examples/trials/nas_cifar10/README.md @@ -2,7 +2,14 @@ === Now we have an NAS example [NNI-NAS-Example](https://github.com/Crysple/NNI-NAS-Example) run in NNI using NAS interface from our contributors. + +We have included its trial code in this folder, and provided example config files to show how to use PPO tuner to tune the trial code. + +> Download data + +- `cd data && . download.sh` +- `tar xzf cifar-10-python.tar.gz && mv cifar-batches cifar10` Thanks our lovely contributors. -And welcome more and more people to join us! \ No newline at end of file +And welcome more and more people to join us! diff --git a/examples/trials/nas_cifar10/config_pai_ppo.yml b/examples/trials/nas_cifar10/config_pai_ppo.yml new file mode 100644 index 0000000000..38156376bd --- /dev/null +++ b/examples/trials/nas_cifar10/config_pai_ppo.yml @@ -0,0 +1,31 @@ +authorName: Unknown +experimentName: enas_macro +trialConcurrency: 20 +maxExecDuration: 2400h +maxTrialNum: 20000 +#choice: local, remote +trainingServicePlatform: pai +#choice: true, false +useAnnotation: true +multiPhase: false +versionCheck: false +nniManagerIp: 0.0.0.0 +tuner: + builtinTunerName: PPOTuner + classArgs: + optimize_mode: maximize + trials_per_update: 60 + epochs_per_update: 20 + minibatch_size: 6 +trial: + command: sh ./macro_cifar10_pai.sh + codeDir: ./ + gpuNum: 1 + cpuNum: 1 + memoryMB: 8196 + image: msranni/nni:latest + virtualCluster: nni +paiConfig: + userName: your_account + passWord: your_pwd + host: 0.0.0.0 diff --git a/examples/trials/nas_cifar10/config_ppo.yml b/examples/trials/nas_cifar10/config_ppo.yml new file mode 100644 index 0000000000..74c0dbea8e --- /dev/null +++ b/examples/trials/nas_cifar10/config_ppo.yml @@ -0,0 +1,21 @@ +authorName: Unknown +experimentName: enas_macro +trialConcurrency: 4 +maxExecDuration: 2400h +maxTrialNum: 20000 +#choice: local, remote +trainingServicePlatform: local +#choice: true, false +useAnnotation: true +multiPhase: false +tuner: + builtinTunerName: PPOTuner + classArgs: + optimize_mode: maximize + trials_per_update: 60 + epochs_per_update: 12 + minibatch_size: 10 +trial: + command: sh ./macro_cifar10.sh + codeDir: ./ + gpuNum: 1 diff --git a/examples/trials/nas_cifar10/data/download.sh b/examples/trials/nas_cifar10/data/download.sh new file mode 100755 index 0000000000..f00ac25724 --- /dev/null +++ b/examples/trials/nas_cifar10/data/download.sh @@ -0,0 +1 @@ +wget https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz diff --git a/examples/trials/nas_cifar10/macro_cifar10.sh b/examples/trials/nas_cifar10/macro_cifar10.sh new file mode 100644 index 0000000000..863256d802 --- /dev/null +++ b/examples/trials/nas_cifar10/macro_cifar10.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -e +export PYTHONPATH="$(pwd)" + +python3 src/cifar10/nni_child_cifar10.py \ + --data_format="NCHW" \ + --search_for="macro" \ + --reset_output_dir \ + --data_path="data/cifar10" \ + --output_dir="outputs" \ + --train_data_size=45000 \ + --batch_size=100 \ + --num_epochs=8 \ + --log_every=50 \ + --eval_every_epochs=1 \ + --child_use_aux_heads \ + --child_num_layers=12 \ + --child_out_filters=36 \ + --child_l2_reg=0.0002 \ + --child_num_branches=6 \ + --child_num_cell_layers=5 \ + --child_keep_prob=0.50 \ + --child_drop_path_keep_prob=0.60 \ + --child_lr_cosine \ + --child_lr_max=0.05 \ + --child_lr_min=0.001 \ + --child_lr_T_0=10 \ + --child_lr_T_mul=2 \ + --child_mode="subgraph" \ + "$@" + diff --git a/examples/trials/nas_cifar10/macro_cifar10_pai.sh b/examples/trials/nas_cifar10/macro_cifar10_pai.sh new file mode 100644 index 0000000000..226955edc7 --- /dev/null +++ b/examples/trials/nas_cifar10/macro_cifar10_pai.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -e +export PYTHONPATH="$(pwd)" + +python3 src/cifar10/nni_child_cifar10.py \ + --data_format="NCHW" \ + --search_for="macro" \ + --reset_output_dir \ + --data_path="data/cifar10" \ + --output_dir="outputs" \ + --train_data_size=45000 \ + --batch_size=100 \ + --num_epochs=30 \ + --log_every=50 \ + --eval_every_epochs=1 \ + --child_use_aux_heads \ + --child_num_layers=12 \ + --child_out_filters=36 \ + --child_l2_reg=0.0002 \ + --child_num_branches=6 \ + --child_num_cell_layers=5 \ + --child_keep_prob=0.50 \ + --child_drop_path_keep_prob=0.60 \ + --child_lr_cosine \ + --child_lr_max=0.05 \ + --child_lr_min=0.001 \ + --child_lr_T_0=10 \ + --child_lr_T_mul=2 \ + --child_mode="subgraph" \ + "$@" + diff --git a/examples/trials/nas_cifar10/src/__init__.py b/examples/trials/nas_cifar10/src/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/trials/nas_cifar10/src/cifar10/__init__.py b/examples/trials/nas_cifar10/src/cifar10/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/trials/nas_cifar10/src/cifar10/data_utils.py b/examples/trials/nas_cifar10/src/cifar10/data_utils.py new file mode 100644 index 0000000000..b8a8c36339 --- /dev/null +++ b/examples/trials/nas_cifar10/src/cifar10/data_utils.py @@ -0,0 +1,74 @@ +import os +import sys +import pickle +import numpy as np +import tensorflow as tf + + +def _read_data(data_path, train_files): + """Reads CIFAR-10 format data. Always returns NHWC format. + + Returns: + images: np tensor of size [N, H, W, C] + labels: np tensor of size [N] + """ + images, labels = [], [] + for file_name in train_files: + print(file_name) + full_name = os.path.join(data_path, file_name) + with open(full_name, "rb") as finp: + data = pickle.load(finp, encoding='latin1') + batch_images = data["data"].astype(np.float32) / 255.0 + batch_labels = np.array(data["labels"], dtype=np.int32) + images.append(batch_images) + labels.append(batch_labels) + images = np.concatenate(images, axis=0) + labels = np.concatenate(labels, axis=0) + images = np.reshape(images, [-1, 3, 32, 32]) + images = np.transpose(images, [0, 2, 3, 1]) + + return images, labels + + +def read_data(data_path, num_valids=5000): + print("-" * 80) + print("Reading data") + + images, labels = {}, {} + + train_files = [ + "data_batch_1", + "data_batch_2", + "data_batch_3", + "data_batch_4", + "data_batch_5", + ] + test_file = [ + "test_batch", + ] + images["train"], labels["train"] = _read_data(data_path, train_files) + + if num_valids: + images["valid"] = images["train"][-num_valids:] + labels["valid"] = labels["train"][-num_valids:] + + images["train"] = images["train"][:-num_valids] + labels["train"] = labels["train"][:-num_valids] + else: + images["valid"], labels["valid"] = None, None + + images["test"], labels["test"] = _read_data(data_path, test_file) + + print("Prepropcess: [subtract mean], [divide std]") + mean = np.mean(images["train"], axis=(0, 1, 2), keepdims=True) + std = np.std(images["train"], axis=(0, 1, 2), keepdims=True) + + print("mean: {}".format(np.reshape(mean * 255.0, [-1]))) + print("std: {}".format(np.reshape(std * 255.0, [-1]))) + + images["train"] = (images["train"] - mean) / std + if num_valids: + images["valid"] = (images["valid"] - mean) / std + images["test"] = (images["test"] - mean) / std + + return images, labels diff --git a/examples/trials/nas_cifar10/src/cifar10/general_child.py b/examples/trials/nas_cifar10/src/cifar10/general_child.py new file mode 100644 index 0000000000..4e80dc340e --- /dev/null +++ b/examples/trials/nas_cifar10/src/cifar10/general_child.py @@ -0,0 +1,423 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import numpy as np +import tensorflow as tf +from src.common_ops import create_weight, batch_norm, batch_norm_with_mask, global_avg_pool, conv_op, pool_op +from src.utils import count_model_params, get_train_ops, get_C, get_strides +from src.cifar10.models import Model + + +class GeneralChild(Model): + def __init__(self, + images, + labels, + cutout_size=None, + fixed_arc=None, + out_filters_scale=1, + num_layers=2, + num_branches=6, + out_filters=24, + keep_prob=1.0, + batch_size=32, + clip_mode=None, + grad_bound=None, + l2_reg=1e-4, + lr_init=0.1, + lr_dec_start=0, + lr_dec_every=10000, + lr_dec_rate=0.1, + lr_cosine=False, + lr_max=None, + lr_min=None, + lr_T_0=None, + lr_T_mul=None, + optim_algo=None, + sync_replicas=False, + num_aggregate=None, + num_replicas=None, + data_format="NHWC", + name="child", + mode="subgraph", + *args, + **kwargs + ): + + super(self.__class__, self).__init__( + images, + labels, + cutout_size=cutout_size, + batch_size=batch_size, + clip_mode=clip_mode, + grad_bound=grad_bound, + l2_reg=l2_reg, + lr_init=lr_init, + lr_dec_start=lr_dec_start, + lr_dec_every=lr_dec_every, + lr_dec_rate=lr_dec_rate, + keep_prob=keep_prob, + optim_algo=optim_algo, + sync_replicas=sync_replicas, + num_aggregate=num_aggregate, + num_replicas=num_replicas, + data_format=data_format, + name=name) + + self.lr_cosine = lr_cosine + self.lr_max = lr_max + self.lr_min = lr_min + self.lr_T_0 = lr_T_0 + self.lr_T_mul = lr_T_mul + self.out_filters = out_filters * out_filters_scale + self.num_layers = num_layers + self.mode = mode + + self.num_branches = num_branches + self.fixed_arc = fixed_arc + self.out_filters_scale = out_filters_scale + + pool_distance = self.num_layers // 3 + self.pool_layers = [pool_distance - 1, 2 * pool_distance - 1] + + + + def _factorized_reduction(self, x, out_filters, stride, is_training): + """Reduces the shape of x without information loss due to striding.""" + assert out_filters % 2 == 0, ( + "Need even number of filters when using this factorized reduction.") + if stride == 1: + with tf.variable_scope("path_conv"): + inp_c = get_C(x, self.data_format) + w = create_weight("w", [1, 1, inp_c, out_filters]) + x = tf.nn.conv2d(x, w, [1, 1, 1, 1], "SAME", + data_format=self.data_format) + x = batch_norm(x, is_training, data_format=self.data_format) + return x + + stride_spec = get_strides(stride, self.data_format) + # Skip path 1 + path1 = tf.nn.avg_pool( + x, [1, 1, 1, 1], stride_spec, "VALID", data_format=self.data_format) + with tf.variable_scope("path1_conv"): + inp_c = get_C(path1, self.data_format) + w = create_weight("w", [1, 1, inp_c, out_filters // 2]) + path1 = tf.nn.conv2d(path1, w, [1, 1, 1, 1], "SAME", + data_format=self.data_format) + + # Skip path 2 + # First pad with 0"s on the right and bottom, then shift the filter to + # include those 0"s that were added. + if self.data_format == "NHWC": + pad_arr = [[0, 0], [0, 1], [0, 1], [0, 0]] + path2 = tf.pad(x, pad_arr)[:, 1:, 1:, :] + concat_axis = 3 + else: + pad_arr = [[0, 0], [0, 0], [0, 1], [0, 1]] + path2 = tf.pad(x, pad_arr)[:, :, 1:, 1:] + concat_axis = 1 + + path2 = tf.nn.avg_pool( + path2, [1, 1, 1, 1], stride_spec, "VALID", data_format=self.data_format) + with tf.variable_scope("path2_conv"): + inp_c = get_C(path2, self.data_format) + w = create_weight("w", [1, 1, inp_c, out_filters // 2]) + path2 = tf.nn.conv2d(path2, w, [1, 1, 1, 1], "SAME", + data_format=self.data_format) + + # Concat and apply BN + final_path = tf.concat(values=[path1, path2], axis=concat_axis) + final_path = batch_norm(final_path, is_training, + data_format=self.data_format) + + return final_path + + def _model(self, images, is_training, reuse=False): + '''Build model''' + with tf.variable_scope(self.name, reuse=reuse): + layers = [] + + out_filters = self.out_filters + with tf.variable_scope("stem_conv"): + w = create_weight("w", [3, 3, 3, out_filters]) + x = tf.nn.conv2d( + images, w, [1, 1, 1, 1], "SAME", data_format=self.data_format) + x = batch_norm(x, is_training, data_format=self.data_format) + layers.append(x) + + def add_fixed_pooling_layer(layer_id, layers, out_filters, is_training): + '''Add a fixed pooling layer every four layers''' + out_filters *= 2 + with tf.variable_scope("pool_at_{0}".format(layer_id)): + pooled_layers = [] + for i, layer in enumerate(layers): + with tf.variable_scope("from_{0}".format(i)): + x = self._factorized_reduction( + layer, out_filters, 2, is_training) + pooled_layers.append(x) + return pooled_layers, out_filters + + def post_process_out(out, optional_inputs): + '''Form skip connection and perform batch norm''' + with tf.variable_scope("skip"): + inputs = layers[-1] + if self.data_format == "NHWC": + inp_h = inputs.get_shape()[1].value + inp_w = inputs.get_shape()[2].value + inp_c = inputs.get_shape()[3].value + out.set_shape([None, inp_h, inp_w, out_filters]) + elif self.data_format == "NCHW": + inp_c = inputs.get_shape()[1].value + inp_h = inputs.get_shape()[2].value + inp_w = inputs.get_shape()[3].value + out.set_shape([None, out_filters, inp_h, inp_w]) + optional_inputs.append(out) + pout = tf.add_n(optional_inputs) + out = batch_norm(pout, is_training, + data_format=self.data_format) + layers.append(out) + return out + + global layer_id + layer_id = -1 + + def get_layer_id(): + global layer_id + layer_id += 1 + return 'layer_' + str(layer_id) + + def conv3(inputs): + # res_layers is pre_layers that are chosen to form skip connection + # layers[-1] is always the latest input + with tf.variable_scope(get_layer_id()): + with tf.variable_scope('branch_0'): + out = conv_op( + inputs[0][0], 3, is_training, out_filters, out_filters, self.data_format, start_idx=None) + out = post_process_out(out, inputs[1]) + return out + + def conv3_sep(inputs): + with tf.variable_scope(get_layer_id()): + with tf.variable_scope('branch_1'): + out = conv_op( + inputs[0][0], 3, is_training, out_filters, out_filters, self.data_format, start_idx=None, separable=True) + out = post_process_out(out, inputs[1]) + return out + + def conv5(inputs): + with tf.variable_scope(get_layer_id()): + with tf.variable_scope('branch_2'): + out = conv_op( + inputs[0][0], 5, is_training, out_filters, out_filters, self.data_format, start_idx=None) + out = post_process_out(out, inputs[1]) + return out + + def conv5_sep(inputs): + with tf.variable_scope(get_layer_id()): + with tf.variable_scope('branch_3'): + out = conv_op( + inputs[0][0], 5, is_training, out_filters, out_filters, self.data_format, start_idx=None, separable=True) + out = post_process_out(out, inputs[1]) + return out + + def avg_pool(inputs): + with tf.variable_scope(get_layer_id()): + with tf.variable_scope('branch_4'): + out = pool_op( + inputs[0][0], is_training, out_filters, out_filters, "avg", self.data_format, start_idx=None) + out = post_process_out(out, inputs[1]) + return out + + def max_pool(inputs): + with tf.variable_scope(get_layer_id()): + with tf.variable_scope('branch_5'): + out = pool_op( + inputs[0][0], is_training, out_filters, out_filters, "max", self.data_format, start_idx=None) + out = post_process_out(out, inputs[1]) + return out + + """@nni.mutable_layers( + { + layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()], + fixed_inputs:[x], + layer_output: layer_0_out + }, + { + layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()], + fixed_inputs:[layer_0_out], + optional_inputs: [layer_0_out], + optional_input_size: [0, 1], + layer_output: layer_1_out + }, + { + layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()], + fixed_inputs:[layer_1_out], + optional_inputs: [layer_0_out, layer_1_out], + optional_input_size: [0, 1], + layer_output: layer_2_out + }, + { + layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()], + fixed_inputs:[layer_2_out], + optional_inputs: [layer_0_out, layer_1_out, layer_2_out], + optional_input_size: [0, 1], + layer_output: layer_3_out + } + )""" + layers, out_filters = add_fixed_pooling_layer( + 3, layers, out_filters, is_training) + layer_0_out, layer_1_out, layer_2_out, layer_3_out = layers[-4:] + """@nni.mutable_layers( + { + layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()], + fixed_inputs: [layer_3_out], + optional_inputs: [layer_0_out, layer_1_out, layer_2_out, layer_3_out], + optional_input_size: [0, 1], + layer_output: layer_4_out + }, + { + layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()], + fixed_inputs: [layer_4_out], + optional_inputs: [layer_0_out, layer_1_out, layer_2_out, layer_3_out, layer_4_out], + optional_input_size: [0, 1], + layer_output: layer_5_out + }, + { + layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()], + fixed_inputs: [layer_5_out], + optional_inputs: [layer_0_out, layer_1_out, layer_2_out, layer_3_out, layer_4_out, layer_5_out], + optional_input_size: [0, 1], + layer_output: layer_6_out + }, + { + layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()], + fixed_inputs: [layer_6_out], + optional_inputs: [layer_0_out, layer_1_out, layer_2_out, layer_3_out, layer_4_out, layer_5_out, layer_6_out], + optional_input_size: [0, 1], + layer_output: layer_7_out + } + )""" + layers, out_filters = add_fixed_pooling_layer( + 7, layers, out_filters, is_training) + layer_0_out, layer_1_out, layer_2_out, layer_3_out, layer_4_out, layer_5_out, layer_6_out, layer_7_out = layers[ + -8:] + """@nni.mutable_layers( + { + layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()], + fixed_inputs: [layer_7_out], + optional_inputs: [layer_0_out, layer_1_out, layer_2_out, layer_3_out, layer_4_out, layer_5_out, layer_6_out, layer_7_out], + optional_input_size: [0, 1], + layer_output: layer_8_out + }, + { + layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()], + fixed_inputs: [layer_8_out], + optional_inputs: [layer_0_out, layer_1_out, layer_2_out, layer_3_out, layer_4_out, layer_5_out, layer_6_out, layer_7_out, layer_8_out], + optional_input_size: [0, 1], + layer_output: layer_9_out + }, + { + layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()], + fixed_inputs: [layer_9_out], + optional_inputs: [layer_0_out, layer_1_out, layer_2_out, layer_3_out, layer_4_out, layer_5_out, layer_6_out, layer_7_out, layer_8_out, layer_9_out], + optional_input_size: [0, 1], + layer_output: layer_10_out + }, + { + layer_choice: [conv3(), conv3_sep(), conv5(), conv5_sep(), avg_pool(), max_pool()], + fixed_inputs:[layer_10_out], + optional_inputs: [layer_0_out, layer_1_out, layer_2_out, layer_3_out, layer_4_out, layer_5_out, layer_6_out, layer_7_out, layer_8_out, layer_9_out, layer_10_out], + optional_input_size: [0, 1], + layer_output: layer_11_out + } + )""" + + x = global_avg_pool(layer_11_out, data_format=self.data_format) + if is_training: + x = tf.nn.dropout(x, self.keep_prob) + with tf.variable_scope("fc"): + if self.data_format == "NHWC": + inp_c = x.get_shape()[3].value + elif self.data_format == "NCHW": + inp_c = x.get_shape()[1].value + else: + raise ValueError( + "Unknown data_format {0}".format(self.data_format)) + w = create_weight("w", [inp_c, 10]) + x = tf.matmul(x, w) + return x + + + # override + def _build_train(self): + print("-" * 80) + print("Build train graph") + logits = self._model(self.x_train, is_training=True) + log_probs = tf.nn.sparse_softmax_cross_entropy_with_logits( + logits=logits, labels=self.y_train) + self.loss = tf.reduce_mean(log_probs) + + self.train_preds = tf.argmax(logits, axis=1) + self.train_preds = tf.to_int32(self.train_preds) + self.train_acc = tf.equal(self.train_preds, self.y_train) + self.train_acc = tf.to_int32(self.train_acc) + self.train_acc = tf.reduce_sum(self.train_acc) + + tf_variables = [var + for var in tf.trainable_variables() if var.name.startswith(self.name)] + self.num_vars = count_model_params(tf_variables) + print("Model has {} params".format(self.num_vars)) + + self.global_step = tf.Variable( + 0, dtype=tf.int32, trainable=False, name="global_step") + + self.train_op, self.lr, self.grad_norm, self.optimizer = get_train_ops( + self.loss, + tf_variables, + self.global_step, + clip_mode=self.clip_mode, + grad_bound=self.grad_bound, + l2_reg=self.l2_reg, + lr_init=self.lr_init, + lr_dec_start=self.lr_dec_start, + lr_dec_every=self.lr_dec_every, + lr_dec_rate=self.lr_dec_rate, + lr_cosine=self.lr_cosine, + lr_max=self.lr_max, + lr_min=self.lr_min, + lr_T_0=self.lr_T_0, + lr_T_mul=self.lr_T_mul, + num_train_batches=self.num_train_batches, + optim_algo=self.optim_algo, + sync_replicas=False, + num_aggregate=self.num_aggregate, + num_replicas=self.num_replicas) + + # override + def _build_valid(self): + if self.x_valid is not None: + print("-" * 80) + print("Build valid graph") + logits = self._model(self.x_valid, False, reuse=True) + self.valid_preds = tf.argmax(logits, axis=1) + self.valid_preds = tf.to_int32(self.valid_preds) + self.valid_acc = tf.equal(self.valid_preds, self.y_valid) + self.valid_acc = tf.to_int32(self.valid_acc) + self.valid_acc = tf.reduce_sum(self.valid_acc) + + # override + def _build_test(self): + print("-" * 80) + print("Build test graph") + logits = self._model(self.x_test, False, reuse=True) + self.test_preds = tf.argmax(logits, axis=1) + self.test_preds = tf.to_int32(self.test_preds) + self.test_acc = tf.equal(self.test_preds, self.y_test) + self.test_acc = tf.to_int32(self.test_acc) + self.test_acc = tf.reduce_sum(self.test_acc) + + + def build_model(self): + + self._build_train() + self._build_valid() + self._build_test() diff --git a/examples/trials/nas_cifar10/src/cifar10/models.py b/examples/trials/nas_cifar10/src/cifar10/models.py new file mode 100644 index 0000000000..089fe846a6 --- /dev/null +++ b/examples/trials/nas_cifar10/src/cifar10/models.py @@ -0,0 +1,196 @@ +import os +import sys + +import numpy as np +import tensorflow as tf + + +class Model(object): + def __init__(self, + images, + labels, + cutout_size=None, + batch_size=32, + eval_batch_size=100, + clip_mode=None, + grad_bound=None, + l2_reg=1e-4, + lr_init=0.1, + lr_dec_start=0, + lr_dec_every=100, + lr_dec_rate=0.1, + keep_prob=1.0, + optim_algo=None, + sync_replicas=False, + num_aggregate=None, + num_replicas=None, + data_format="NHWC", + name="generic_model", + seed=None, + ): + """ + Args: + lr_dec_every: number of epochs to decay + """ + print("-" * 80) + print("Build model {}".format(name)) + + self.cutout_size = cutout_size + self.batch_size = batch_size + self.eval_batch_size = eval_batch_size + self.clip_mode = clip_mode + self.grad_bound = grad_bound + self.l2_reg = l2_reg + self.lr_init = lr_init + self.lr_dec_start = lr_dec_start + self.lr_dec_rate = lr_dec_rate + self.keep_prob = keep_prob + self.optim_algo = optim_algo + self.sync_replicas = sync_replicas + self.num_aggregate = num_aggregate + self.num_replicas = num_replicas + self.data_format = data_format + self.name = name + self.seed = seed + + self.global_step = None + self.valid_acc = None + self.test_acc = None + print("Build data ops") + with tf.device("/cpu:0"): + # training data + self.num_train_examples = np.shape(images["train"])[0] + + self.num_train_batches = ( + self.num_train_examples + self.batch_size - 1) // self.batch_size + x_train, y_train = tf.train.shuffle_batch( + [images["train"], labels["train"]], + batch_size=self.batch_size, + capacity=50000, + enqueue_many=True, + min_after_dequeue=0, + num_threads=16, + seed=self.seed, + allow_smaller_final_batch=True, + ) + self.lr_dec_every = lr_dec_every * self.num_train_batches + + def _pre_process(x): + x = tf.pad(x, [[4, 4], [4, 4], [0, 0]]) + x = tf.random_crop(x, [32, 32, 3], seed=self.seed) + x = tf.image.random_flip_left_right(x, seed=self.seed) + if self.cutout_size is not None: + mask = tf.ones( + [self.cutout_size, self.cutout_size], dtype=tf.int32) + start = tf.random_uniform( + [2], minval=0, maxval=32, dtype=tf.int32) + mask = tf.pad(mask, [[self.cutout_size + start[0], 32 - start[0]], + [self.cutout_size + start[1], 32 - start[1]]]) + mask = mask[self.cutout_size: self.cutout_size + 32, + self.cutout_size: self.cutout_size + 32] + mask = tf.reshape(mask, [32, 32, 1]) + mask = tf.tile(mask, [1, 1, 3]) + x = tf.where(tf.equal(mask, 0), x=x, y=tf.zeros_like(x)) + if self.data_format == "NCHW": + x = tf.transpose(x, [2, 0, 1]) + + return x + self.x_train = tf.map_fn(_pre_process, x_train, back_prop=False) + self.y_train = y_train + + # valid data + self.x_valid, self.y_valid = None, None + if images["valid"] is not None: + images["valid_original"] = np.copy(images["valid"]) + labels["valid_original"] = np.copy(labels["valid"]) + if self.data_format == "NCHW": + images["valid"] = tf.transpose( + images["valid"], [0, 3, 1, 2]) + self.num_valid_examples = np.shape(images["valid"])[0] + self.num_valid_batches = ( + (self.num_valid_examples + self.eval_batch_size - 1) + // self.eval_batch_size) + self.x_valid, self.y_valid = tf.train.batch( + [images["valid"], labels["valid"]], + batch_size=self.eval_batch_size, + capacity=5000, + enqueue_many=True, + num_threads=1, + allow_smaller_final_batch=True, + ) + + # test data + if self.data_format == "NCHW": + images["test"] = tf.transpose(images["test"], [0, 3, 1, 2]) + self.num_test_examples = np.shape(images["test"])[0] + self.num_test_batches = ( + (self.num_test_examples + self.eval_batch_size - 1) + // self.eval_batch_size) + self.x_test, self.y_test = tf.train.batch( + [images["test"], labels["test"]], + batch_size=self.eval_batch_size, + capacity=10000, + enqueue_many=True, + num_threads=1, + allow_smaller_final_batch=True, + ) + + # cache images and labels + self.images = images + self.labels = labels + + def eval_once(self, sess, eval_set, child_model, verbose=False): + """Expects self.acc and self.global_step to be defined. + + Args: + sess: tf.Session() or one of its wrap arounds. + feed_dict: can be used to give more information to sess.run(). + eval_set: "valid" or "test" + """ + + assert self.global_step is not None + global_step = sess.run(self.global_step) + print("Eval at {}".format(global_step)) + + if eval_set == "valid": + assert self.x_valid is not None + assert self.valid_acc is not None + num_examples = self.num_valid_examples + num_batches = self.num_valid_batches + acc_op = self.valid_acc + elif eval_set == "test": + assert self.test_acc is not None + num_examples = self.num_test_examples + num_batches = self.num_test_batches + acc_op = self.test_acc + else: + raise NotImplementedError("Unknown eval_set '{}'".format(eval_set)) + + total_acc = 0 + total_exp = 0 + + for batch_id in range(num_batches): + acc = sess.run(acc_op) + + total_acc += acc + total_exp += self.eval_batch_size + if verbose: + sys.stdout.write( + "\r{:<5d}/{:>5d}".format(total_acc, total_exp)) + if verbose: + print("") + print("{}_accuracy: {:<6.4f}".format( + eval_set, float(total_acc) / total_exp)) + return float(total_acc) / total_exp + + def _model(self, images, is_training, reuse=None): + raise NotImplementedError("Abstract method") + + def _build_train(self): + raise NotImplementedError("Abstract method") + + def _build_valid(self): + raise NotImplementedError("Abstract method") + + def _build_test(self): + raise NotImplementedError("Abstract method") diff --git a/examples/trials/nas_cifar10/src/cifar10/nni_child_cifar10.py b/examples/trials/nas_cifar10/src/cifar10/nni_child_cifar10.py new file mode 100644 index 0000000000..5481ba7b07 --- /dev/null +++ b/examples/trials/nas_cifar10/src/cifar10/nni_child_cifar10.py @@ -0,0 +1,162 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import os +import shutil +import logging +import tensorflow as tf +from src.cifar10.data_utils import read_data +from src.cifar10.general_child import GeneralChild +import src.cifar10_flags +from src.cifar10_flags import FLAGS + + +def build_logger(log_name): + logger = logging.getLogger(log_name) + logger.setLevel(logging.DEBUG) + fh = logging.FileHandler(log_name+'.log') + fh.setLevel(logging.DEBUG) + logger.addHandler(fh) + return logger + + +logger = build_logger("nni_child_cifar10") + + +def build_trial(images, labels, ChildClass): + '''Build child class''' + child_model = ChildClass( + images, + labels, + use_aux_heads=FLAGS.child_use_aux_heads, + cutout_size=FLAGS.child_cutout_size, + num_layers=FLAGS.child_num_layers, + num_cells=FLAGS.child_num_cells, + num_branches=FLAGS.child_num_branches, + fixed_arc=FLAGS.child_fixed_arc, + out_filters_scale=FLAGS.child_out_filters_scale, + out_filters=FLAGS.child_out_filters, + keep_prob=FLAGS.child_keep_prob, + drop_path_keep_prob=FLAGS.child_drop_path_keep_prob, + num_epochs=FLAGS.num_epochs, + l2_reg=FLAGS.child_l2_reg, + data_format=FLAGS.data_format, + batch_size=FLAGS.batch_size, + clip_mode="norm", + grad_bound=FLAGS.child_grad_bound, + lr_init=FLAGS.child_lr, + lr_dec_every=FLAGS.child_lr_dec_every, + lr_dec_rate=FLAGS.child_lr_dec_rate, + lr_cosine=FLAGS.child_lr_cosine, + lr_max=FLAGS.child_lr_max, + lr_min=FLAGS.child_lr_min, + lr_T_0=FLAGS.child_lr_T_0, + lr_T_mul=FLAGS.child_lr_T_mul, + optim_algo="momentum", + sync_replicas=FLAGS.child_sync_replicas, + num_aggregate=FLAGS.child_num_aggregate, + num_replicas=FLAGS.child_num_replicas + ) + + return child_model + + +def get_child_ops(child_model): + '''Assemble child op to a dict''' + child_ops = { + "global_step": child_model.global_step, + "loss": child_model.loss, + "train_op": child_model.train_op, + "lr": child_model.lr, + "grad_norm": child_model.grad_norm, + "train_acc": child_model.train_acc, + "optimizer": child_model.optimizer, + "num_train_batches": child_model.num_train_batches, + "eval_every": child_model.num_train_batches * FLAGS.eval_every_epochs, + "eval_func": child_model.eval_once, + } + return child_ops + + +class NASTrial(): + + def __init__(self): + images, labels = read_data(FLAGS.data_path, num_valids=0) + + self.output_dir = os.path.join(os.getenv('NNI_OUTPUT_DIR'), '../..') + self.file_path = os.path.join( + self.output_dir, 'trainable_variable.txt') + + self.graph = tf.Graph() + with self.graph.as_default(): + self.child_model = build_trial(images, labels, GeneralChild) + + self.total_data = {} + + self.child_model.build_model() + self.child_ops = get_child_ops(self.child_model) + config = tf.ConfigProto( + intra_op_parallelism_threads=0, + inter_op_parallelism_threads=0, + allow_soft_placement=True) + + self.sess = tf.train.SingularMonitoredSession(config=config) + + logger.debug('initlize NASTrial done.') + + def run_one_step(self): + '''Run this model on a batch of data''' + run_ops = [ + self.child_ops["loss"], + self.child_ops["lr"], + self.child_ops["grad_norm"], + self.child_ops["train_acc"], + self.child_ops["train_op"], + ] + loss, lr, gn, tr_acc, _ = self.sess.run(run_ops) + global_step = self.sess.run(self.child_ops["global_step"]) + log_string = "" + log_string += "ch_step={:<6d}".format(global_step) + log_string += " loss={:<8.6f}".format(loss) + log_string += " lr={:<8.4f}".format(lr) + log_string += " |g|={:<8.4f}".format(gn) + log_string += " tr_acc={:<3d}/{:>3d}".format(tr_acc, FLAGS.batch_size) + if int(global_step) % FLAGS.log_every == 0: + logger.debug(log_string) + return loss, global_step + + def run(self): + '''Run this model according to the `epoch` set in FALGS''' + max_acc = 0 + while True: + _, global_step = self.run_one_step() + if global_step % self.child_ops['num_train_batches'] == 0: + acc = self.child_ops["eval_func"]( + self.sess, "test", self.child_model) + max_acc = max(max_acc, acc) + '''@nni.report_intermediate_result(acc)''' + if global_step / self.child_ops['num_train_batches'] >= FLAGS.num_epochs: + '''@nni.report_final_result(max_acc)''' + break + + +def main(_): + logger.debug("-" * 80) + + if not os.path.isdir(FLAGS.output_dir): + logger.debug( + "Path {} does not exist. Creating.".format(FLAGS.output_dir)) + os.makedirs(FLAGS.output_dir) + elif FLAGS.reset_output_dir: + logger.debug( + "Path {} exists. Remove and remake.".format(FLAGS.output_dir)) + shutil.rmtree(FLAGS.output_dir) + os.makedirs(FLAGS.output_dir) + logger.debug("-" * 80) + trial = NASTrial() + + trial.run() + + +if __name__ == "__main__": + tf.app.run() diff --git a/examples/trials/nas_cifar10/src/cifar10_flags.py b/examples/trials/nas_cifar10/src/cifar10_flags.py new file mode 100644 index 0000000000..2374f76b90 --- /dev/null +++ b/examples/trials/nas_cifar10/src/cifar10_flags.py @@ -0,0 +1,45 @@ +import tensorflow as tf +from src.utils import DEFINE_boolean +from src.utils import DEFINE_float +from src.utils import DEFINE_integer +from src.utils import DEFINE_string +flags = tf.app.flags +FLAGS = flags.FLAGS + +DEFINE_boolean("reset_output_dir", False, "Delete output_dir if exists.") +DEFINE_string("data_path", "", "") +DEFINE_string("output_dir", "", "") +DEFINE_string("data_format", "NHWC", "'NHWC' or 'NCWH'") +DEFINE_string("search_for", None, "Must be [macro|micro]") +DEFINE_integer("train_data_size", 45000, "") +DEFINE_integer("batch_size", 32, "") + +DEFINE_integer("num_epochs", 300, "") +DEFINE_integer("child_lr_dec_every", 100, "") +DEFINE_integer("child_num_layers", 5, "") +DEFINE_integer("child_num_cells", 5, "") +DEFINE_integer("child_filter_size", 5, "") +DEFINE_integer("child_out_filters", 48, "") +DEFINE_integer("child_out_filters_scale", 1, "") +DEFINE_integer("child_num_branches", 4, "") +DEFINE_integer("child_num_aggregate", None, "") +DEFINE_integer("child_num_replicas", 1, "") +DEFINE_integer("child_block_size", 3, "") +DEFINE_integer("child_lr_T_0", None, "for lr schedule") +DEFINE_integer("child_lr_T_mul", None, "for lr schedule") +DEFINE_integer("child_cutout_size", None, "CutOut size") +DEFINE_float("child_grad_bound", 5.0, "Gradient clipping") +DEFINE_float("child_lr", 0.1, "") +DEFINE_float("child_lr_dec_rate", 0.1, "") +DEFINE_float("child_keep_prob", 0.5, "") +DEFINE_float("child_drop_path_keep_prob", 1.0, "minimum drop_path_keep_prob") +DEFINE_float("child_l2_reg", 1e-4, "") +DEFINE_float("child_lr_max", None, "for lr schedule") +DEFINE_float("child_lr_min", None, "for lr schedule") +DEFINE_string("child_skip_pattern", None, "Must be ['dense', None]") +DEFINE_string("child_fixed_arc", None, "") +DEFINE_boolean("child_use_aux_heads", False, "Should we use an aux head") +DEFINE_boolean("child_sync_replicas", False, "To sync or not to sync.") +DEFINE_boolean("child_lr_cosine", False, "Use cosine lr schedule") +DEFINE_integer("log_every", 50, "How many steps to log") +DEFINE_integer("eval_every_epochs", 1, "How many epochs to eval") diff --git a/examples/trials/nas_cifar10/src/common_ops.py b/examples/trials/nas_cifar10/src/common_ops.py new file mode 100644 index 0000000000..e0933f6e53 --- /dev/null +++ b/examples/trials/nas_cifar10/src/common_ops.py @@ -0,0 +1,255 @@ +import numpy as np +import tensorflow as tf +from tensorflow.python.training import moving_averages + + +def lstm(x, prev_c, prev_h, w): + ifog = tf.matmul(tf.concat([x, prev_h], axis=1), w) + i, f, o, g = tf.split(ifog, 4, axis=1) + i = tf.sigmoid(i) + f = tf.sigmoid(f) + o = tf.sigmoid(o) + g = tf.tanh(g) + next_c = i * g + f * prev_c + next_h = o * tf.tanh(next_c) + return next_c, next_h + + +def stack_lstm(x, prev_c, prev_h, w): + next_c, next_h = [], [] + for layer_id, (_c, _h, _w) in enumerate(zip(prev_c, prev_h, w)): + inputs = x if layer_id == 0 else next_h[-1] + curr_c, curr_h = lstm(inputs, _c, _h, _w) + next_c.append(curr_c) + next_h.append(curr_h) + return next_c, next_h + + +def create_weight(name, shape, initializer=None, trainable=True, seed=None): + if initializer is None: + initializer = tf.contrib.keras.initializers.he_normal(seed=seed) + return tf.get_variable(name, shape, initializer=initializer, trainable=trainable) + + +def create_bias(name, shape, initializer=None): + if initializer is None: + initializer = tf.constant_initializer(0.0, dtype=tf.float32) + return tf.get_variable(name, shape, initializer=initializer) + + +def conv_op(inputs, filter_size, is_training, count, out_filters, + data_format, ch_mul=1, start_idx=None, separable=False): + """ + Args: + start_idx: where to start taking the output channels. if None, assuming + fixed_arc mode + count: how many output_channels to take. + """ + + if data_format == "NHWC": + inp_c = inputs.get_shape()[3].value + elif data_format == "NCHW": + inp_c = inputs.get_shape()[1].value + + with tf.variable_scope("inp_conv_1"): + w = create_weight("w", [1, 1, inp_c, out_filters]) + x = tf.nn.conv2d(inputs, w, [1, 1, 1, 1], + "SAME", data_format=data_format) + x = batch_norm(x, is_training, data_format=data_format) + x = tf.nn.relu(x) + + with tf.variable_scope("out_conv_{}".format(filter_size)): + if start_idx is None: + if separable: + w_depth = create_weight( + "w_depth", [filter_size, filter_size, out_filters, ch_mul]) + w_point = create_weight( + "w_point", [1, 1, out_filters * ch_mul, count]) + x = tf.nn.separable_conv2d(x, w_depth, w_point, strides=[1, 1, 1, 1], + padding="SAME", data_format=data_format) + x = batch_norm( + x, is_training, data_format=data_format) + else: + w = create_weight( + "w", [filter_size, filter_size, inp_c, count]) + x = tf.nn.conv2d( + x, w, [1, 1, 1, 1], "SAME", data_format=data_format) + x = batch_norm( + x, is_training, data_format=data_format) + else: + if separable: + w_depth = create_weight( + "w_depth", [filter_size, filter_size, out_filters, ch_mul]) + #test_depth = w_depth + w_point = create_weight( + "w_point", [out_filters, out_filters * ch_mul]) + w_point = w_point[start_idx:start_idx+count, :] + w_point = tf.transpose(w_point, [1, 0]) + w_point = tf.reshape( + w_point, [1, 1, out_filters * ch_mul, count]) + + x = tf.nn.separable_conv2d(x, w_depth, w_point, strides=[1, 1, 1, 1], + padding="SAME", data_format=data_format) + mask = tf.range(0, out_filters, dtype=tf.int32) + mask = tf.logical_and( + start_idx <= mask, mask < start_idx + count) + x = batch_norm_with_mask( + x, is_training, mask, out_filters, data_format=data_format) + else: + w = create_weight( + "w", [filter_size, filter_size, out_filters, out_filters]) + w = tf.transpose(w, [3, 0, 1, 2]) + w = w[start_idx:start_idx+count, :, :, :] + w = tf.transpose(w, [1, 2, 3, 0]) + x = tf.nn.conv2d( + x, w, [1, 1, 1, 1], "SAME", data_format=data_format) + mask = tf.range(0, out_filters, dtype=tf.int32) + mask = tf.logical_and( + start_idx <= mask, mask < start_idx + count) + x = batch_norm_with_mask( + x, is_training, mask, out_filters, data_format=data_format) + x = tf.nn.relu(x) + return x + +def pool_op(inputs, is_training, count, out_filters, avg_or_max, data_format, start_idx=None): + """ + Args: + start_idx: where to start taking the output channels. if None, assuming + fixed_arc mode + count: how many output_channels to take. + """ + + if data_format == "NHWC": + inp_c = inputs.get_shape()[3].value + elif data_format == "NCHW": + inp_c = inputs.get_shape()[1].value + + with tf.variable_scope("conv_1"): + w = create_weight("w", [1, 1, inp_c, out_filters]) + x = tf.nn.conv2d(inputs, w, [1, 1, 1, 1], + "SAME", data_format=data_format) + x = batch_norm(x, is_training, data_format=data_format) + x = tf.nn.relu(x) + + with tf.variable_scope("pool"): + if data_format == "NHWC": + actual_data_format = "channels_last" + elif data_format == "NCHW": + actual_data_format = "channels_first" + + if avg_or_max == "avg": + x = tf.layers.average_pooling2d( + x, [3, 3], [1, 1], "SAME", data_format=actual_data_format) + elif avg_or_max == "max": + x = tf.layers.max_pooling2d( + x, [3, 3], [1, 1], "SAME", data_format=actual_data_format) + else: + raise ValueError("Unknown pool {}".format(avg_or_max)) + + if start_idx is not None: + if data_format == "NHWC": + x = x[:, :, :, start_idx: start_idx+count] + elif data_format == "NCHW": + x = x[:, start_idx: start_idx+count, :, :] + + return x + + +def global_avg_pool(x, data_format="NHWC"): + if data_format == "NHWC": + x = tf.reduce_mean(x, [1, 2]) + elif data_format == "NCHW": + x = tf.reduce_mean(x, [2, 3]) + else: + raise NotImplementedError("Unknown data_format {}".format(data_format)) + return x + + +def batch_norm(x, is_training, name="bn", decay=0.9, epsilon=1e-5, + data_format="NHWC"): + if data_format == "NHWC": + shape = [x.get_shape()[3]] + elif data_format == "NCHW": + shape = [x.get_shape()[1]] + else: + raise NotImplementedError("Unknown data_format {}".format(data_format)) + + with tf.variable_scope(name, reuse=None if is_training else True): + offset = tf.get_variable( + "offset", shape, + initializer=tf.constant_initializer(0.0, dtype=tf.float32)) + scale = tf.get_variable( + "scale", shape, + initializer=tf.constant_initializer(1.0, dtype=tf.float32)) + moving_mean = tf.get_variable( + "moving_mean", shape, trainable=False, + initializer=tf.constant_initializer(0.0, dtype=tf.float32)) + moving_variance = tf.get_variable( + "moving_variance", shape, trainable=False, + initializer=tf.constant_initializer(1.0, dtype=tf.float32)) + + if is_training: + x, mean, variance = tf.nn.fused_batch_norm( + x, scale, offset, epsilon=epsilon, data_format=data_format, + is_training=True) + update_mean = moving_averages.assign_moving_average( + moving_mean, mean, decay) + update_variance = moving_averages.assign_moving_average( + moving_variance, variance, decay) + with tf.control_dependencies([update_mean, update_variance]): + x = tf.identity(x) + else: + x, _, _ = tf.nn.fused_batch_norm(x, scale, offset, mean=moving_mean, + variance=moving_variance, + epsilon=epsilon, data_format=data_format, + is_training=False) + return x + + +def batch_norm_with_mask(x, is_training, mask, num_channels, name="bn", + decay=0.9, epsilon=1e-3, data_format="NHWC"): + + shape = [num_channels] + indices = tf.where(mask) + indices = tf.to_int32(indices) + indices = tf.reshape(indices, [-1]) + + with tf.variable_scope(name, reuse=None if is_training else True): + offset = tf.get_variable( + "offset", shape, + initializer=tf.constant_initializer(0.0, dtype=tf.float32)) + scale = tf.get_variable( + "scale", shape, + initializer=tf.constant_initializer(1.0, dtype=tf.float32)) + offset = tf.boolean_mask(offset, mask) + scale = tf.boolean_mask(scale, mask) + + moving_mean = tf.get_variable( + "moving_mean", shape, trainable=False, + initializer=tf.constant_initializer(0.0, dtype=tf.float32)) + moving_variance = tf.get_variable( + "moving_variance", shape, trainable=False, + initializer=tf.constant_initializer(1.0, dtype=tf.float32)) + + if is_training: + x, mean, variance = tf.nn.fused_batch_norm( + x, scale, offset, epsilon=epsilon, data_format=data_format, + is_training=True) + mean = (1.0 - decay) * (tf.boolean_mask(moving_mean, mask) - mean) + variance = (1.0 - decay) * \ + (tf.boolean_mask(moving_variance, mask) - variance) + update_mean = tf.scatter_sub( + moving_mean, indices, mean, use_locking=True) + update_variance = tf.scatter_sub( + moving_variance, indices, variance, use_locking=True) + with tf.control_dependencies([update_mean, update_variance]): + x = tf.identity(x) + else: + masked_moving_mean = tf.boolean_mask(moving_mean, mask) + masked_moving_variance = tf.boolean_mask(moving_variance, mask) + x, _, _ = tf.nn.fused_batch_norm(x, scale, offset, + mean=masked_moving_mean, + variance=masked_moving_variance, + epsilon=epsilon, data_format=data_format, + is_training=False) + return x diff --git a/examples/trials/nas_cifar10/src/utils.py b/examples/trials/nas_cifar10/src/utils.py new file mode 100644 index 0000000000..65d57af7f1 --- /dev/null +++ b/examples/trials/nas_cifar10/src/utils.py @@ -0,0 +1,262 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import sys +import numpy as np +import tensorflow as tf + + +user_flags = [] + + +def DEFINE_string(name, default_value, doc_string): + tf.app.flags.DEFINE_string(name, default_value, doc_string) + global user_flags + user_flags.append(name) + + +def DEFINE_integer(name, default_value, doc_string): + tf.app.flags.DEFINE_integer(name, default_value, doc_string) + global user_flags + user_flags.append(name) + + +def DEFINE_float(name, default_value, doc_string): + tf.app.flags.DEFINE_float(name, default_value, doc_string) + global user_flags + user_flags.append(name) + + +def DEFINE_boolean(name, default_value, doc_string): + tf.app.flags.DEFINE_boolean(name, default_value, doc_string) + global user_flags + user_flags.append(name) + + +def print_user_flags(line_limit=80): + print("-" * 80) + + global user_flags + FLAGS = tf.app.flags.FLAGS + + for flag_name in sorted(user_flags): + value = "{}".format(getattr(FLAGS, flag_name)) + log_string = flag_name + log_string += "." * (line_limit - len(flag_name) - len(value)) + log_string += value + print(log_string) + + +def get_C(x, data_format): + """ + Args: + x: tensor of shape [N, H, W, C] or [N, C, H, W] + """ + if data_format == "NHWC": + return x.get_shape()[3].value + elif data_format == "NCHW": + return x.get_shape()[1].value + else: + raise ValueError( + "Unknown data_format '{0}'".format(data_format)) + +def get_HW(x, data_format): + """ + Args: + x: tensor of shape [N, H, W, C] or [N, C, H, W] + """ + return x.get_shape()[2].value + +def get_strides(stride, data_format): + """ + Args: + x: tensor of shape [N, H, W, C] or [N, C, H, W] + """ + if data_format == "NHWC": + return [1, stride, stride, 1] + elif data_format == "NCHW": + return [1, 1, stride, stride] + else: + raise ValueError( + "Unknown data_format '{0}'".format(data_format)) + + +class TextColors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + + +class Logger(object): + def __init__(self, output_file): + self.terminal = sys.stdout + self.log = open(output_file, "a") + + def write(self, message): + self.terminal.write(message) + self.terminal.flush() + self.log.write(message) + self.log.flush() + + +def count_model_params(tf_variables): + """ + Args: + tf_variables: list of all model variables + """ + + num_vars = 0 + for var in tf_variables: + num_vars += np.prod([dim.value for dim in var.get_shape()]) + return num_vars + + +def get_train_ops( + loss, + tf_variables, + train_step, + clip_mode=None, + grad_bound=None, + l2_reg=1e-4, + lr_warmup_val=None, + lr_warmup_steps=100, + lr_init=0.1, + lr_dec_start=0, + lr_dec_every=10000, + lr_dec_rate=0.1, + lr_dec_min=None, + lr_cosine=False, + lr_max=None, + lr_min=None, + lr_T_0=None, + lr_T_mul=None, + num_train_batches=None, + optim_algo=None, + sync_replicas=False, + num_aggregate=None, + num_replicas=None, + get_grad_norms=False, + moving_average=None): + """ + Args: + clip_mode: "global", "norm", or None. + moving_average: store the moving average of parameters + """ + + if l2_reg > 0: + l2_losses = [] + for var in tf_variables: + l2_losses.append(tf.reduce_sum(var ** 2)) + l2_loss = tf.add_n(l2_losses) + loss += l2_reg * l2_loss + + grads = tf.gradients(loss, tf_variables) + grad_norm = tf.global_norm(grads) + + grad_norms = {} + for v, g in zip(tf_variables, grads): + if v is None or g is None: + continue + if isinstance(g, tf.IndexedSlices): + grad_norms[v.name] = tf.sqrt(tf.reduce_sum(g.values ** 2)) + else: + grad_norms[v.name] = tf.sqrt(tf.reduce_sum(g ** 2)) + + if clip_mode is not None: + assert grad_bound is not None, "Need grad_bound to clip gradients." + if clip_mode == "global": + grads, _ = tf.clip_by_global_norm(grads, grad_bound) + elif clip_mode == "norm": + clipped = [] + for g in grads: + if isinstance(g, tf.IndexedSlices): + c_g = tf.clip_by_norm(g.values, grad_bound) + c_g = tf.IndexedSlices(g.indices, c_g) + else: + c_g = tf.clip_by_norm(g, grad_bound) + clipped.append(g) + grads = clipped + else: + raise NotImplementedError("Unknown clip_mode {}".format(clip_mode)) + + if lr_cosine: + assert lr_max is not None, "Need lr_max to use lr_cosine" + assert lr_min is not None, "Need lr_min to use lr_cosine" + assert lr_T_0 is not None, "Need lr_T_0 to use lr_cosine" + assert lr_T_mul is not None, "Need lr_T_mul to use lr_cosine" + assert num_train_batches is not None, ("Need num_train_batches to use" + " lr_cosine") + + curr_epoch = train_step // num_train_batches + + last_reset = tf.Variable(0, dtype=tf.int32, trainable=False, + name="last_reset") + T_i = tf.Variable(lr_T_0, dtype=tf.int32, trainable=False, name="T_i") + T_curr = curr_epoch - last_reset + + def _update(): + update_last_reset = tf.assign( + last_reset, curr_epoch, use_locking=True) + update_T_i = tf.assign(T_i, T_i * lr_T_mul, use_locking=True) + with tf.control_dependencies([update_last_reset, update_T_i]): + rate = tf.to_float(T_curr) / tf.to_float(T_i) * 3.1415926 + lr = lr_min + 0.5 * (lr_max - lr_min) * (1.0 + tf.cos(rate)) + return lr + + def _no_update(): + rate = tf.to_float(T_curr) / tf.to_float(T_i) * 3.1415926 + lr = lr_min + 0.5 * (lr_max - lr_min) * (1.0 + tf.cos(rate)) + return lr + + learning_rate = tf.cond( + tf.greater_equal(T_curr, T_i), _update, _no_update) + else: + learning_rate = tf.train.exponential_decay( + lr_init, tf.maximum(train_step - lr_dec_start, 0), lr_dec_every, + lr_dec_rate, staircase=True) + if lr_dec_min is not None: + learning_rate = tf.maximum(learning_rate, lr_dec_min) + + if lr_warmup_val is not None: + learning_rate = tf.cond(tf.less(train_step, lr_warmup_steps), + lambda: lr_warmup_val, lambda: learning_rate) + + if optim_algo == "momentum": + opt = tf.train.MomentumOptimizer( + learning_rate, 0.9, use_locking=True, use_nesterov=True) + elif optim_algo == "sgd": + opt = tf.train.GradientDescentOptimizer( + learning_rate, use_locking=True) + elif optim_algo == "adam": + opt = tf.train.AdamOptimizer(learning_rate, beta1=0.0, epsilon=1e-3, + use_locking=True) + else: + raise ValueError("Unknown optim_algo {}".format(optim_algo)) + + if sync_replicas: + assert num_aggregate is not None, "Need num_aggregate to sync." + assert num_replicas is not None, "Need num_replicas to sync." + + opt = tf.train.SyncReplicasOptimizer( + opt, + replicas_to_aggregate=num_aggregate, + total_num_replicas=num_replicas, + use_locking=True) + + if moving_average is not None: + opt = tf.contrib.opt.MovingAverageOptimizer( + opt, average_decay=moving_average) + + train_op = opt.apply_gradients( + zip(grads, tf_variables), global_step=train_step) + + if get_grad_norms: + return train_op, learning_rate, grad_norm, opt, grad_norms + else: + return train_op, learning_rate, grad_norm, opt diff --git a/examples/tuners/random_nas_tuner/random_nas_tuner.py b/examples/tuners/random_nas_tuner/random_nas_tuner.py index d7f6214aa6..c13bc72c6b 100644 --- a/examples/tuners/random_nas_tuner/random_nas_tuner.py +++ b/examples/tuners/random_nas_tuner/random_nas_tuner.py @@ -2,13 +2,14 @@ from nni.tuner import Tuner + def random_archi_generator(nas_ss, random_state): '''random ''' chosen_archi = {} - print("zql: nas search space: ", nas_ss) for block_name, block_value in nas_ss.items(): - assert block_value['_type'] == "mutable_layer", "Random NAS Tuner only receives NAS search space whose _type is 'mutable_layer'" + assert block_value['_type'] == "mutable_layer", \ + "Random NAS Tuner only receives NAS search space whose _type is 'mutable_layer'" block = block_value['_value'] tmp_block = {} for layer_name, layer in block.items(): @@ -19,13 +20,12 @@ def random_archi_generator(nas_ss, random_state): tmp_layer['chosen_layer'] = value[index] elif key == 'optional_inputs': tmp_layer['chosen_inputs'] = [] - print("zql: optional_inputs", layer['optional_inputs']) if layer['optional_inputs']: if isinstance(layer['optional_input_size'], int): choice_num = layer['optional_input_size'] else: choice_range = layer['optional_input_size'] - choice_num = random_state.randint(choice_range[0], choice_range[1]+1) + choice_num = random_state.randint(choice_range[0], choice_range[1] + 1) for _ in range(choice_num): index = random_state.randint(len(layer['optional_inputs'])) tmp_layer['chosen_inputs'].append(layer['optional_inputs'][index]) @@ -37,6 +37,7 @@ def random_archi_generator(nas_ss, random_state): chosen_archi[block_name] = tmp_block return chosen_archi + class RandomNASTuner(Tuner): '''RandomNASTuner ''' diff --git a/src/nni_manager/rest_server/restValidationSchemas.ts b/src/nni_manager/rest_server/restValidationSchemas.ts index 56b1b2c633..23423218bc 100644 --- a/src/nni_manager/rest_server/restValidationSchemas.ts +++ b/src/nni_manager/rest_server/restValidationSchemas.ts @@ -174,7 +174,7 @@ export namespace ValidationSchemas { checkpointDir: joi.string().allow('') }), tuner: joi.object({ - builtinTunerName: joi.string().valid('TPE', 'Random', 'Anneal', 'Evolution', 'SMAC', 'BatchTuner', 'GridSearch', 'NetworkMorphism', 'MetisTuner', 'GPTuner'), + builtinTunerName: joi.string().valid('TPE', 'Random', 'Anneal', 'Evolution', 'SMAC', 'BatchTuner', 'GridSearch', 'NetworkMorphism', 'MetisTuner', 'GPTuner', 'PPOTuner'), codeDir: joi.string(), classFileName: joi.string(), className: joi.string(), diff --git a/src/sdk/pynni/nni/constants.py b/src/sdk/pynni/nni/constants.py index ab726baa1b..5fc515da7b 100644 --- a/src/sdk/pynni/nni/constants.py +++ b/src/sdk/pynni/nni/constants.py @@ -30,7 +30,8 @@ 'NetworkMorphism': 'nni.networkmorphism_tuner.networkmorphism_tuner', 'Curvefitting': 'nni.curvefitting_assessor.curvefitting_assessor', 'MetisTuner': 'nni.metis_tuner.metis_tuner', - 'GPTuner': 'nni.gp_tuner.gp_tuner' + 'GPTuner': 'nni.gp_tuner.gp_tuner', + 'PPOTuner': 'nni.ppo_tuner.ppo_tuner' } ClassName = { @@ -44,6 +45,7 @@ 'NetworkMorphism':'NetworkMorphismTuner', 'MetisTuner':'MetisTuner', 'GPTuner':'GPTuner', + 'PPOTuner': 'PPOTuner', 'Medianstop': 'MedianstopAssessor', 'Curvefitting': 'CurvefittingAssessor' diff --git a/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py b/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py index 7d1e6f7caa..9c54e03df8 100644 --- a/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py +++ b/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py @@ -27,6 +27,7 @@ import hyperopt as hp import numpy as np from nni.tuner import Tuner +from nni.nas_utils import rewrite_nas_space from nni.utils import NodeType, OptimizeMode, extract_scalar_reward, split_index logger = logging.getLogger('hyperopt_AutoML') @@ -240,6 +241,7 @@ def _choose_tuner(self, algorithm_name): return hp.anneal.suggest raise RuntimeError('Not support tuner algorithm in hyperopt.') + @rewrite_nas_space def update_search_space(self, search_space): """ Update search space definition in tuner by search_space in parameters. diff --git a/src/sdk/pynni/nni/msg_dispatcher.py b/src/sdk/pynni/nni/msg_dispatcher.py index fc9de474a2..1467b27695 100644 --- a/src/sdk/pynni/nni/msg_dispatcher.py +++ b/src/sdk/pynni/nni/msg_dispatcher.py @@ -101,11 +101,16 @@ def handle_initialize(self, data): self.tuner.update_search_space(data) send(CommandType.Initialized, '') + def send_trial_callback(self, id, params): + """For tuner to issue trial config when the config is generated + """ + send(CommandType.NewTrialJob, _pack_parameter(id, params)) + def handle_request_trial_jobs(self, data): # data: number or trial jobs ids = [_create_parameter_id() for _ in range(data)] _logger.debug("requesting for generating params of {}".format(ids)) - params_list = self.tuner.generate_multiple_parameters(ids) + params_list = self.tuner.generate_multiple_parameters(ids, st_callback=self.send_trial_callback) for i, _ in enumerate(params_list): send(CommandType.NewTrialJob, _pack_parameter(ids[i], params_list[i])) diff --git a/src/sdk/pynni/nni/nas_utils.py b/src/sdk/pynni/nni/nas_utils.py index efc25194a4..70e66b318e 100644 --- a/src/sdk/pynni/nni/nas_utils.py +++ b/src/sdk/pynni/nni/nas_utils.py @@ -17,10 +17,16 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT # OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # ================================================================================================== +import functools +import logging from . import trial +_logger = logging.getLogger(__name__) +_MUTABLE_LAYER_SPACE_PREFIX = "_mutable_layer" + + def classic_mode( mutable_id, mutable_layer_id, @@ -34,13 +40,11 @@ def classic_mode( without touching the full model graph.''' if trial.get_current_parameter() is None: trial.get_next_parameter() - mutable_block = trial.get_current_parameter(mutable_id) - chosen_layer = mutable_block[mutable_layer_id]["chosen_layer"] - chosen_inputs = mutable_block[mutable_layer_id]["chosen_inputs"] - real_chosen_inputs = [optional_inputs[input_name] - for input_name in chosen_inputs] - layer_out = funcs[chosen_layer]( - [fixed_inputs, real_chosen_inputs], **funcs_args[chosen_layer]) + + chosen_layer, chosen_inputs = _get_layer_and_inputs_from_tuner(mutable_id, mutable_layer_id, + list(optional_inputs.keys())) + real_chosen_inputs = [optional_inputs[input_name] for input_name in chosen_inputs] + layer_out = funcs[chosen_layer]([fixed_inputs, real_chosen_inputs], **funcs_args[chosen_layer]) return layer_out @@ -173,20 +177,44 @@ def reload_tensorflow_variables(tf, session): tf: tensorflow module ''' subgraph_from_tuner = trial.get_next_parameter() - for mutable_id, mutable_block in subgraph_from_tuner.items(): + mutable_layers = set() + for subgraph_key in subgraph_from_tuner: + if "/" in subgraph_key: + # has to remove the last, could be layer_choice or whatever + mutable_id, mutable_layer_id = _decompose_general_key(subgraph_key[:subgraph_key.rfind("/")]) + if mutable_id is not None: + mutable_layers.add((mutable_id, mutable_layer_id)) + mutable_layers = sorted(list(mutable_layers)) + for mutable_id, mutable_layer_id in mutable_layers: if mutable_id not in name_space: + _logger.warning("{} not found in name space".format(mutable_id)) continue - for mutable_layer_id, mutable_layer in mutable_block.items(): - name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) - # extract layer information from the subgraph sampled by tuner - chosen_layer = name_space[name_prefix]['funcs'].index( - mutable_layer["chosen_layer"]) - chosen_inputs = [1 if inp in mutable_layer["chosen_inputs"] - else 0 for inp in name_space[name_prefix]['optional_inputs']] - # load these information into pre-defined tensorflow variables - tf_variables[name_prefix]['funcs'].load(chosen_layer, session) - tf_variables[name_prefix]['optional_inputs'].load( - chosen_inputs, session) + name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) + # get optional inputs names + optional_inputs = name_space[name_prefix]['optional_inputs'] + # extract layer information from the subgraph sampled by tuner + chosen_layer, chosen_inputs = _get_layer_and_inputs_from_tuner(mutable_id, mutable_layer_id, optional_inputs) + chosen_layer = name_space[name_prefix]['funcs'].index(chosen_layer) + chosen_inputs = [1 if inp in chosen_inputs else 0 for inp in optional_inputs] + # load these information into pre-defined tensorflow variables + tf_variables[name_prefix]['funcs'].load(chosen_layer, session) + tf_variables[name_prefix]['optional_inputs'].load( + chosen_inputs, session) + + +def _construct_general_key(mutable_id, mutable_layer_id): + # Mutable layer key in a general (search space) format + # that is, prefix/mutable_id/mutable_layer_id + return _MUTABLE_LAYER_SPACE_PREFIX + "/" + mutable_id + "/" + mutable_layer_id + + +def _decompose_general_key(key): + # inverse operation of above + if not key.startswith(_MUTABLE_LAYER_SPACE_PREFIX): + return None, None + else: + _, mutable_id, mutable_layer_id = key.split("/", maxsplit=2) + return mutable_id, mutable_layer_id def darts_training(tf, session, loss, feed_dict): @@ -205,4 +233,107 @@ def training_update(nas_mode, tf=None, session=None, loss=None, feed_dict=None): if nas_mode == 'darts_mode': darts_training(tf, session, loss, feed_dict) elif nas_mode == 'enas_mode': - reload_tensorflow_variables(tf, session) \ No newline at end of file + reload_tensorflow_variables(tf, session) + + +def _get_layer_and_inputs_from_tuner(mutable_id, mutable_layer_id, optional_inputs): + # optional_inputs should be name(key)s of the optional inputs + try: + mutable_block = trial.get_current_parameter(mutable_id) + + # There is a NAS tuner + chosen_layer = mutable_block[mutable_layer_id]["chosen_layer"] + chosen_inputs = mutable_block[mutable_layer_id]["chosen_inputs"] + except KeyError: + # Try to find converted NAS parameters + params = trial.get_current_parameter() + expected_prefix = _construct_general_key(mutable_id, mutable_layer_id) + chosen_layer = params[expected_prefix + "/layer_choice"] + + # find how many to choose + optional_input_size = int(params[expected_prefix + "/optional_input_size"]) # convert uniform to randint + + # find who to choose, can duplicate + optional_input_state = params[expected_prefix + "/optional_input_chosen_state"] + chosen_inputs = [] + # make sure dict -> list produce stable result by sorting + optional_inputs_keys = sorted(optional_inputs) + for i in range(optional_input_size): + chosen_inputs.append(optional_inputs_keys[optional_input_state % len(optional_inputs)]) + optional_input_state //= len(optional_inputs) + + _logger.info("%s_%s: layer: %s, optional inputs: %s" % (mutable_id, mutable_layer_id, + chosen_layer, chosen_inputs)) + return chosen_layer, chosen_inputs + + +def convert_nas_search_space(search_space): + """ + Args: + param search_space: raw search space + return: the new search space, mutable_layers will be converted into choice + """ + ret = dict() + for k, v in search_space.items(): + if "_type" not in v: + # this should not happen + _logger.warning("There is no _type in one of your search space values with key '%s'" + ". Please check your search space" % k) + ret[k] = v + elif v["_type"] != "mutable_layer": + ret[k] = v + else: + _logger.info("Converting mutable_layer search space with key '%s'" % k) + # v["_value"] looks like {'mutable_layer_1': {'layer_choice': ...} ...} + values = v["_value"] + for layer_name, layer_data in values.items(): + # there should be at most layer_choice, optional_inputs, optional_input_size in layer_data + + # add "_mutable_layer" as prefix so that they can be recovered later + layer_key = _construct_general_key(k, layer_name) + + if layer_data.get("layer_choice"): # filter out empty choice and no choice + layer_choice = layer_data["layer_choice"] + else: + raise ValueError("No layer choice found in %s" % layer_key) + + if layer_data.get("optional_input_size"): + input_size = layer_data["optional_input_size"] + if isinstance(input_size, int): + input_size = [input_size, input_size] + if input_size[0] > input_size[1] or input_size[0] < 0: + _logger.error("Might not be able to handle optional_input_size < 0, please double check") + input_size[1] += 1 + else: + _logger.info("Optional input choices are set to empty by default in %s" % layer_key) + input_size = [0, 1] + + if layer_data.get("optional_inputs"): + total_state_size = len(layer_data["optional_inputs"]) ** (input_size[1] - 1) + else: + _logger.info("Optional inputs not found in %s" % layer_key) + total_state_size = 1 + + converted = { + layer_key + "/layer_choice": { + "_type": "choice", "_value": layer_choice + }, + layer_key + "/optional_input_size": { + "_type": "randint", "_value": input_size + }, + layer_key + "/optional_input_chosen_state": { + "_type": "randint", "_value": [0, total_state_size] + } + } + _logger.info(converted) + ret.update(converted) + + return ret + + +def rewrite_nas_space(func): + @functools.wraps(func) + def wrap(self, search_space): + search_space = convert_nas_search_space(search_space) + return func(self, search_space) + return wrap diff --git a/src/sdk/pynni/nni/ppo_tuner/__init__.py b/src/sdk/pynni/nni/ppo_tuner/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sdk/pynni/nni/ppo_tuner/distri.py b/src/sdk/pynni/nni/ppo_tuner/distri.py new file mode 100644 index 0000000000..4666acc2da --- /dev/null +++ b/src/sdk/pynni/nni/ppo_tuner/distri.py @@ -0,0 +1,198 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, +# to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +functions for sampling from hidden state +""" + +import tensorflow as tf + +from .util import fc + + +class Pd: + """ + A particular probability distribution + """ + def flatparam(self): + raise NotImplementedError + def mode(self): + raise NotImplementedError + def neglogp(self, x): + # Usually it's easier to define the negative logprob + raise NotImplementedError + def kl(self, other): + raise NotImplementedError + def entropy(self): + raise NotImplementedError + def sample(self): + raise NotImplementedError + def logp(self, x): + return - self.neglogp(x) + def get_shape(self): + return self.flatparam().shape + @property + def shape(self): + return self.get_shape() + def __getitem__(self, idx): + return self.__class__(self.flatparam()[idx]) + +class PdType: + """ + Parametrized family of probability distributions + """ + def pdclass(self): + raise NotImplementedError + def pdfromflat(self, flat, mask, nsteps, size, is_act_model): + return self.pdclass()(flat, mask, nsteps, size, is_act_model) + def pdfromlatent(self, latent_vector, init_scale, init_bias): + raise NotImplementedError + def param_shape(self): + raise NotImplementedError + def sample_shape(self): + raise NotImplementedError + def sample_dtype(self): + raise NotImplementedError + + def param_placeholder(self, prepend_shape, name=None): + return tf.placeholder(dtype=tf.float32, shape=prepend_shape+self.param_shape(), name=name) + def sample_placeholder(self, prepend_shape, name=None): + return tf.placeholder(dtype=self.sample_dtype(), shape=prepend_shape+self.sample_shape(), name=name) + +class CategoricalPd(Pd): + """ + categorical prossibility distribution + """ + def __init__(self, logits, mask_npinf, nsteps, size, is_act_model): + self.logits = logits + self.mask_npinf = mask_npinf + self.nsteps = nsteps + self.size = size + self.is_act_model = is_act_model + def flatparam(self): + return self.logits + def mode(self): + return tf.argmax(self.logits, axis=-1) + + @property + def mean(self): + return tf.nn.softmax(self.logits) + def neglogp(self, x): + """ + return tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=x) + Note: we can't use sparse_softmax_cross_entropy_with_logits because + the implementation does not allow second-order derivatives... + """ + if x.dtype in {tf.uint8, tf.int32, tf.int64}: + # one-hot encoding + x_shape_list = x.shape.as_list() + logits_shape_list = self.logits.get_shape().as_list()[:-1] + for xs, ls in zip(x_shape_list, logits_shape_list): + if xs is not None and ls is not None: + assert xs == ls, 'shape mismatch: {} in x vs {} in logits'.format(xs, ls) + + x = tf.one_hot(x, self.logits.get_shape().as_list()[-1]) + else: + # already encoded + assert x.shape.as_list() == self.logits.shape.as_list() + + return tf.nn.softmax_cross_entropy_with_logits_v2( + logits=self.logits, + labels=x) + + def kl(self, other): + """kl""" + a0 = self.logits - tf.reduce_max(self.logits, axis=-1, keepdims=True) + a1 = other.logits - tf.reduce_max(other.logits, axis=-1, keepdims=True) + ea0 = tf.exp(a0) + ea1 = tf.exp(a1) + z0 = tf.reduce_sum(ea0, axis=-1, keepdims=True) + z1 = tf.reduce_sum(ea1, axis=-1, keepdims=True) + p0 = ea0 / z0 + return tf.reduce_sum(p0 * (a0 - tf.log(z0) - a1 + tf.log(z1)), axis=-1) + + def entropy(self): + """compute entropy""" + a0 = self.logits - tf.reduce_max(self.logits, axis=-1, keepdims=True) + ea0 = tf.exp(a0) + z0 = tf.reduce_sum(ea0, axis=-1, keepdims=True) + p0 = ea0 / z0 + return tf.reduce_sum(p0 * (tf.log(z0) - a0), axis=-1) + + def sample(self): + """sample from logits""" + if not self.is_act_model: + re_res = tf.reshape(self.logits, [-1, self.nsteps, self.size]) + masked_res = tf.math.add(re_res, self.mask_npinf) + re_masked_res = tf.reshape(masked_res, [-1, self.size]) + + u = tf.random_uniform(tf.shape(re_masked_res), dtype=self.logits.dtype) + return tf.argmax(re_masked_res - tf.log(-tf.log(u)), axis=-1) + else: + u = tf.random_uniform(tf.shape(self.logits), dtype=self.logits.dtype) + return tf.argmax(self.logits - tf.log(-tf.log(u)), axis=-1) + + @classmethod + def fromflat(cls, flat): + return cls(flat) + +class CategoricalPdType(PdType): + """ + to create CategoricalPd + """ + def __init__(self, ncat, nsteps, np_mask, is_act_model): + self.ncat = ncat + self.nsteps = nsteps + self.np_mask = np_mask + self.is_act_model = is_act_model + def pdclass(self): + return CategoricalPd + + def pdfromlatent(self, latent_vector, init_scale=1.0, init_bias=0.0): + """add fc and create CategoricalPd""" + pdparam, mask, mask_npinf = _matching_fc(latent_vector, 'pi', self.ncat, self.nsteps, + init_scale=init_scale, init_bias=init_bias, + np_mask=self.np_mask, is_act_model=self.is_act_model) + return self.pdfromflat(pdparam, mask_npinf, self.nsteps, self.ncat, self.is_act_model), pdparam, mask, mask_npinf + + def param_shape(self): + return [self.ncat] + def sample_shape(self): + return [] + def sample_dtype(self): + return tf.int32 + +def _matching_fc(tensor, name, size, nsteps, init_scale, init_bias, np_mask, is_act_model): + """ + add fc op, and add mask op when not in action mode + """ + if tensor.shape[-1] == size: + assert False + return tensor + else: + mask = tf.get_variable("act_mask", dtype=tf.float32, initializer=np_mask[0], trainable=False) + mask_npinf = tf.get_variable("act_mask_npinf", dtype=tf.float32, initializer=np_mask[1], trainable=False) + res = fc(tensor, name, size, init_scale=init_scale, init_bias=init_bias) + if not is_act_model: + re_res = tf.reshape(res, [-1, nsteps, size]) + masked_res = tf.math.multiply(re_res, mask) + re_masked_res = tf.reshape(masked_res, [-1, size]) + return re_masked_res, mask, mask_npinf + else: + return res, mask, mask_npinf diff --git a/src/sdk/pynni/nni/ppo_tuner/model.py b/src/sdk/pynni/nni/ppo_tuner/model.py new file mode 100644 index 0000000000..330f10369d --- /dev/null +++ b/src/sdk/pynni/nni/ppo_tuner/model.py @@ -0,0 +1,166 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, +# to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +the main model of policy/value network +""" + +import tensorflow as tf + +from .util import initialize, get_session + +class Model: + """ + We use this object to : + __init__: + - Creates the step_model + - Creates the train_model + + train(): + - Make the training part (feedforward and retropropagation of gradients) + + save/load(): + - Save load the model + """ + def __init__(self, *, policy, nbatch_act, nbatch_train, + nsteps, ent_coef, vf_coef, max_grad_norm, microbatch_size=None, np_mask=None): + """ + init + """ + self.sess = sess = get_session() + + with tf.variable_scope('ppo2_model', reuse=tf.AUTO_REUSE): + # CREATE OUR TWO MODELS + # act_model that is used for sampling + act_model = policy(nbatch_act, 1, sess, np_mask=np_mask, is_act_model=True) + + # Train model for training + if microbatch_size is None: + train_model = policy(nbatch_train, nsteps, sess, np_mask=np_mask, is_act_model=False) + else: + train_model = policy(microbatch_size, nsteps, sess, np_mask=np_mask, is_act_model=False) + + # CREATE THE PLACEHOLDERS + self.A = A = train_model.pdtype.sample_placeholder([None]) + self.ADV = ADV = tf.placeholder(tf.float32, [None]) + self.R = R = tf.placeholder(tf.float32, [None]) + # Keep track of old actor + self.OLDNEGLOGPAC = OLDNEGLOGPAC = tf.placeholder(tf.float32, [None]) + # Keep track of old critic + self.OLDVPRED = OLDVPRED = tf.placeholder(tf.float32, [None]) + self.LR = LR = tf.placeholder(tf.float32, []) + # Cliprange + self.CLIPRANGE = CLIPRANGE = tf.placeholder(tf.float32, []) + + neglogpac = train_model.pd.neglogp(A) + + # Calculate the entropy + # Entropy is used to improve exploration by limiting the premature convergence to suboptimal policy. + entropy = tf.reduce_mean(train_model.pd.entropy()) + + # CALCULATE THE LOSS + # Total loss = Policy gradient loss - entropy * entropy coefficient + Value coefficient * value loss + + # Clip the value to reduce variability during Critic training + # Get the predicted value + vpred = train_model.vf + vpredclipped = OLDVPRED + tf.clip_by_value(train_model.vf - OLDVPRED, - CLIPRANGE, CLIPRANGE) + # Unclipped value + vf_losses1 = tf.square(vpred - R) + # Clipped value + vf_losses2 = tf.square(vpredclipped - R) + + vf_loss = .5 * tf.reduce_mean(tf.maximum(vf_losses1, vf_losses2)) + + # Calculate ratio (pi current policy / pi old policy) + ratio = tf.exp(OLDNEGLOGPAC - neglogpac) + + # Defining Loss = - J is equivalent to max J + pg_losses = -ADV * ratio + + pg_losses2 = -ADV * tf.clip_by_value(ratio, 1.0 - CLIPRANGE, 1.0 + CLIPRANGE) + + # Final PG loss + pg_loss = tf.reduce_mean(tf.maximum(pg_losses, pg_losses2)) + approxkl = .5 * tf.reduce_mean(tf.square(neglogpac - OLDNEGLOGPAC)) + clipfrac = tf.reduce_mean(tf.to_float(tf.greater(tf.abs(ratio - 1.0), CLIPRANGE))) + + # Total loss + loss = pg_loss - entropy * ent_coef + vf_loss * vf_coef + + # UPDATE THE PARAMETERS USING LOSS + # 1. Get the model parameters + params = tf.trainable_variables('ppo2_model') + # 2. Build our trainer + self.trainer = tf.train.AdamOptimizer(learning_rate=LR, epsilon=1e-5) + # 3. Calculate the gradients + grads_and_var = self.trainer.compute_gradients(loss, params) + grads, var = zip(*grads_and_var) + + if max_grad_norm is not None: + # Clip the gradients (normalize) + grads, _grad_norm = tf.clip_by_global_norm(grads, max_grad_norm) + grads_and_var = list(zip(grads, var)) + # zip aggregate each gradient with parameters associated + # For instance zip(ABCD, xyza) => Ax, By, Cz, Da + + self.grads = grads + self.var = var + self._train_op = self.trainer.apply_gradients(grads_and_var) + self.loss_names = ['policy_loss', 'value_loss', 'policy_entropy', 'approxkl', 'clipfrac'] + self.stats_list = [pg_loss, vf_loss, entropy, approxkl, clipfrac] + + + self.train_model = train_model + self.act_model = act_model + self.step = act_model.step + self.value = act_model.value + self.initial_state = act_model.initial_state + + initialize() + + def train(self, lr, cliprange, obs, returns, masks, actions, values, neglogpacs, states=None): + """ + train the model. + Here we calculate advantage A(s,a) = R + yV(s') - V(s) + Returns = R + yV(s') + """ + advs = returns - values + + # Normalize the advantages + advs = (advs - advs.mean()) / (advs.std() + 1e-8) + + td_map = { + self.train_model.X : obs, + self.A : actions, + self.ADV : advs, + self.R : returns, + self.LR : lr, + self.CLIPRANGE : cliprange, + self.OLDNEGLOGPAC : neglogpacs, + self.OLDVPRED : values + } + if states is not None: + td_map[self.train_model.S] = states + td_map[self.train_model.M] = masks + + return self.sess.run( + self.stats_list + [self._train_op], + td_map + )[:-1] diff --git a/src/sdk/pynni/nni/ppo_tuner/policy.py b/src/sdk/pynni/nni/ppo_tuner/policy.py new file mode 100644 index 0000000000..65e2db414e --- /dev/null +++ b/src/sdk/pynni/nni/ppo_tuner/policy.py @@ -0,0 +1,219 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, +# to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +build policy/value network from model +""" + +import tensorflow as tf + +from .distri import CategoricalPdType +from .util import lstm_model, fc, observation_placeholder, adjust_shape + + +class PolicyWithValue: + """ + Encapsulates fields and methods for RL policy and value function estimation with shared parameters + """ + + def __init__(self, env, observations, latent, estimate_q=False, vf_latent=None, sess=None, np_mask=None, is_act_model=False, **tensors): + """ + Parameters: + ---------- + env: RL environment + observations: tensorflow placeholder in which the observations will be fed + latent: latent state from which policy distribution parameters should be inferred + vf_latent: latent state from which value function should be inferred (if None, then latent is used) + sess: tensorflow session to run calculations in (if None, default session is used) + **tensors: tensorflow tensors for additional attributes such as state or mask + """ + + self.X = observations + self.state = tf.constant([]) + self.initial_state = None + self.__dict__.update(tensors) + + vf_latent = vf_latent if vf_latent is not None else latent + + vf_latent = tf.layers.flatten(vf_latent) + latent = tf.layers.flatten(latent) + + # Based on the action space, will select what probability distribution type + self.np_mask = np_mask + self.pdtype = CategoricalPdType(env.action_space.n, env.nsteps, np_mask, is_act_model) + + self.act_latent = latent + self.nh = env.action_space.n + + self.pd, self.pi, self.mask, self.mask_npinf = self.pdtype.pdfromlatent(latent, init_scale=0.01) + + # Take an action + self.action = self.pd.sample() + + # Calculate the neg log of our probability + self.neglogp = self.pd.neglogp(self.action) + self.sess = sess or tf.get_default_session() + + assert estimate_q is False + self.vf = fc(vf_latent, 'vf', 1) + self.vf = self.vf[:, 0] + + if is_act_model: + self._build_model_for_step() + + def _evaluate(self, variables, observation, **extra_feed): + sess = self.sess + feed_dict = {self.X: adjust_shape(self.X, observation)} + for inpt_name, data in extra_feed.items(): + if inpt_name in self.__dict__.keys(): + inpt = self.__dict__[inpt_name] + if isinstance(inpt, tf.Tensor) and inpt._op.type == 'Placeholder': + feed_dict[inpt] = adjust_shape(inpt, data) + + return sess.run(variables, feed_dict) + + def _build_model_for_step(self): + # multiply with weight and apply mask on self.act_latent to generate + self.act_step = step = tf.placeholder(shape=(), dtype=tf.int64, name='act_step') + with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): + from .util import ortho_init + nin = self.act_latent.get_shape()[1].value + w = tf.get_variable("w", [nin, self.nh], initializer=ortho_init(0.01)) + b = tf.get_variable("b", [self.nh], initializer=tf.constant_initializer(0.0)) + logits = tf.matmul(self.act_latent, w)+b + piece = tf.slice(self.mask, [step, 0], [1, self.nh]) + re_piece = tf.reshape(piece, [-1]) + masked_logits = tf.math.multiply(logits, re_piece) + + npinf_piece = tf.slice(self.mask_npinf, [step, 0], [1, self.nh]) + re_npinf_piece = tf.reshape(npinf_piece, [-1]) + + def sample(logits, mask_npinf): + new_logits = tf.math.add(logits, mask_npinf) + u = tf.random_uniform(tf.shape(new_logits), dtype=logits.dtype) + return tf.argmax(new_logits - tf.log(-tf.log(u)), axis=-1) + + def neglogp(logits, x): + # return tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=x) + # Note: we can't use sparse_softmax_cross_entropy_with_logits because + # the implementation does not allow second-order derivatives... + if x.dtype in {tf.uint8, tf.int32, tf.int64}: + # one-hot encoding + x_shape_list = x.shape.as_list() + logits_shape_list = logits.get_shape().as_list()[:-1] + for xs, ls in zip(x_shape_list, logits_shape_list): + if xs is not None and ls is not None: + assert xs == ls, 'shape mismatch: {} in x vs {} in logits'.format(xs, ls) + + x = tf.one_hot(x, logits.get_shape().as_list()[-1]) + else: + # already encoded + assert x.shape.as_list() == logits.shape.as_list() + + return tf.nn.softmax_cross_entropy_with_logits_v2( + logits=logits, + labels=x) + + self.act_action = sample(masked_logits, re_npinf_piece) + self.act_neglogp = neglogp(masked_logits, self.act_action) + + + def step(self, step, observation, **extra_feed): + """ + Compute next action(s) given the observation(s) + + Parameters: + ---------- + observation: observation data (either single or a batch) + **extra_feed: additional data such as state or mask (names of the arguments should match the ones in constructor, see __init__) + + Returns: + ------- + (action, value estimate, next state, negative log likelihood of the action under current policy parameters) tuple + """ + extra_feed['act_step'] = step + a, v, state, neglogp = self._evaluate([self.act_action, self.vf, self.state, self.act_neglogp], observation, **extra_feed) + if state.size == 0: + state = None + return a, v, state, neglogp + + def value(self, ob, *args, **kwargs): + """ + Compute value estimate(s) given the observation(s) + + Parameters: + ---------- + observation: observation data (either single or a batch) + **extra_feed: additional data such as state or mask (names of the arguments should match the ones in constructor, see __init__) + + Returns: + ------- + value estimate + """ + return self._evaluate(self.vf, ob, *args, **kwargs) + + +def build_lstm_policy(model_config, value_network=None, estimate_q=False, **policy_kwargs): + """ + build lstm policy and value network, they share the same lstm network. + the parameters all use their default values. + """ + policy_network = lstm_model(**policy_kwargs) + + def policy_fn(nbatch=None, nsteps=None, sess=None, observ_placeholder=None, np_mask=None, is_act_model=False): + ob_space = model_config.observation_space + + X = observ_placeholder if observ_placeholder is not None else observation_placeholder(ob_space, batch_size=nbatch) + + extra_tensors = {} + + # encode_observation is not necessary anymore as we use embedding_lookup + encoded_x = X + + with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): + policy_latent = policy_network(encoded_x, 1, model_config.observation_space.n) + if isinstance(policy_latent, tuple): + policy_latent, recurrent_tensors = policy_latent + + if recurrent_tensors is not None: + # recurrent architecture, need a few more steps + nenv = nbatch // nsteps + assert nenv > 0, 'Bad input for recurrent policy: batch size {} smaller than nsteps {}'.format(nbatch, nsteps) + policy_latent, recurrent_tensors = policy_network(encoded_x, nenv, model_config.observation_space.n) + extra_tensors.update(recurrent_tensors) + + _v_net = value_network + + assert _v_net is None or _v_net == 'shared' + vf_latent = policy_latent + + policy = PolicyWithValue( + env=model_config, + observations=X, + latent=policy_latent, + vf_latent=vf_latent, + sess=sess, + estimate_q=estimate_q, + np_mask=np_mask, + is_act_model=is_act_model, + **extra_tensors + ) + return policy + + return policy_fn diff --git a/src/sdk/pynni/nni/ppo_tuner/ppo_tuner.py b/src/sdk/pynni/nni/ppo_tuner/ppo_tuner.py new file mode 100644 index 0000000000..0a041ff567 --- /dev/null +++ b/src/sdk/pynni/nni/ppo_tuner/ppo_tuner.py @@ -0,0 +1,599 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, +# to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +ppo_tuner.py including: + class PPOTuner +""" + +import os +os.environ["CUDA_VISIBLE_DEVICES"] = "" +import copy +import logging +import numpy as np +import json_tricks +from gym import spaces + +import nni +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward + +from .model import Model +from .util import set_global_seeds +from .policy import build_lstm_policy + + +logger = logging.getLogger('ppo_tuner_AutoML') + +def constfn(val): + """wrap as function""" + def f(_): + return val + return f + + +class ModelConfig: + """ + Configurations of the PPO model + """ + def __init__(self): + self.observation_space = None + self.action_space = None + self.num_envs = 0 + self.nsteps = 0 + + self.ent_coef = 0.0 + self.lr = 3e-4 + self.vf_coef = 0.5 + self.max_grad_norm = 0.5 + self.gamma = 0.99 + self.lam = 0.95 + self.cliprange = 0.2 + self.embedding_size = None # the embedding is for each action + + self.noptepochs = 4 # number of training epochs per update + self.total_timesteps = 5000 # number of timesteps (i.e. number of actions taken in the environment) + self.nminibatches = 4 # number of training minibatches per update. For recurrent policies, + # should be smaller or equal than number of environments run in parallel. + +class TrialsInfo: + """ + Informations of each trial from one model inference + """ + def __init__(self, obs, actions, values, neglogpacs, dones, last_value, inf_batch_size): + self.iter = 0 + self.obs = obs + self.actions = actions + self.values = values + self.neglogpacs = neglogpacs + self.dones = dones + self.last_value = last_value + + self.rewards = None + self.returns = None + + self.inf_batch_size = inf_batch_size + #self.states = None + + def get_next(self): + """ + get actions of the next trial + """ + if self.iter >= self.inf_batch_size: + return None, None + actions = [] + for step in self.actions: + actions.append(step[self.iter]) + self.iter += 1 + return self.iter - 1, actions + + def update_rewards(self, rewards, returns): + """ + after the trial is finished, reward and return of this trial is updated + """ + self.rewards = rewards + self.returns = returns + + def convert_shape(self): + """ + convert shape + """ + def sf01(arr): + """ + swap and then flatten axes 0 and 1 + """ + s = arr.shape + return arr.swapaxes(0, 1).reshape(s[0] * s[1], *s[2:]) + self.obs = sf01(self.obs) + self.returns = sf01(self.returns) + self.dones = sf01(self.dones) + self.actions = sf01(self.actions) + self.values = sf01(self.values) + self.neglogpacs = sf01(self.neglogpacs) + + +class PPOModel: + """ + PPO Model + """ + def __init__(self, model_config, mask): + self.model_config = model_config + self.states = None # initial state of lstm in policy/value network + self.nupdates = None # the number of func train is invoked, used to tune lr and cliprange + self.cur_update = 1 # record the current update + self.np_mask = mask # record the mask of each action within one trial + + set_global_seeds(None) + assert isinstance(self.model_config.lr, float) + self.lr = constfn(self.model_config.lr) + assert isinstance(self.model_config.cliprange, float) + self.cliprange = constfn(self.model_config.cliprange) + + # build lstm policy network, value share the same network + policy = build_lstm_policy(model_config) + + # Get the nb of env + nenvs = model_config.num_envs + + # Calculate the batch_size + self.nbatch = nbatch = nenvs * model_config.nsteps # num of record per update + nbatch_train = nbatch // model_config.nminibatches # get batch size + # self.nupdates is used to tune lr and cliprange + self.nupdates = self.model_config.total_timesteps // self.nbatch + + # Instantiate the model object (that creates act_model and train_model) + self.model = Model(policy=policy, nbatch_act=nenvs, nbatch_train=nbatch_train, + nsteps=model_config.nsteps, ent_coef=model_config.ent_coef, vf_coef=model_config.vf_coef, + max_grad_norm=model_config.max_grad_norm, np_mask=self.np_mask) + + self.states = self.model.initial_state + + logger.info('=== finished PPOModel initialization') + + def inference(self, num): + """ + generate actions along with related info from policy network. + observation is the action of the last step. + + Parameters: + ---------- + num: the number of trials to generate + """ + # Here, we init the lists that will contain the mb of experiences + mb_obs, mb_actions, mb_values, mb_dones, mb_neglogpacs = [], [], [], [], [] + # initial observation + # use the (n+1)th embedding to represent the first step action + first_step_ob = self.model_config.action_space.n + obs = [first_step_ob for _ in range(num)] + dones = [True for _ in range(num)] + states = self.states + # For n in range number of steps + for cur_step in range(self.model_config.nsteps): + # Given observations, get action value and neglopacs + # We already have self.obs because Runner superclass run self.obs[:] = env.reset() on init + actions, values, states, neglogpacs = self.model.step(cur_step, obs, S=states, M=dones) + mb_obs.append(obs.copy()) + mb_actions.append(actions) + mb_values.append(values) + mb_neglogpacs.append(neglogpacs) + mb_dones.append(dones) + + # Take actions in env and look the results + # Infos contains a ton of useful informations + obs[:] = actions + if cur_step == self.model_config.nsteps - 1: + dones = [True for _ in range(num)] + else: + dones = [False for _ in range(num)] + + #batch of steps to batch of rollouts + np_obs = np.asarray(obs) + mb_obs = np.asarray(mb_obs, dtype=np_obs.dtype) + mb_actions = np.asarray(mb_actions) + mb_values = np.asarray(mb_values, dtype=np.float32) + mb_neglogpacs = np.asarray(mb_neglogpacs, dtype=np.float32) + mb_dones = np.asarray(mb_dones, dtype=np.bool) + last_values = self.model.value(np_obs, S=states, M=dones) + + return mb_obs, mb_actions, mb_values, mb_neglogpacs, mb_dones, last_values + + def compute_rewards(self, trials_info, trials_result): + """ + compute the rewards of the trials in trials_info based on trials_result, + and update the rewards in trials_info + + Parameters: + ---------- + trials_info: info of the generated trials + trials_result: final results (e.g., acc) of the generated trials + """ + mb_rewards = np.asarray([trials_result for _ in trials_info.actions], dtype=np.float32) + # discount/bootstrap off value fn + mb_returns = np.zeros_like(mb_rewards) + mb_advs = np.zeros_like(mb_rewards) + lastgaelam = 0 + last_dones = np.asarray([True for _ in trials_result], dtype=np.bool) # ugly + for t in reversed(range(self.model_config.nsteps)): + if t == self.model_config.nsteps - 1: + nextnonterminal = 1.0 - last_dones + nextvalues = trials_info.last_value + else: + nextnonterminal = 1.0 - trials_info.dones[t+1] + nextvalues = trials_info.values[t+1] + delta = mb_rewards[t] + self.model_config.gamma * nextvalues * nextnonterminal - trials_info.values[t] + mb_advs[t] = lastgaelam = delta + self.model_config.gamma * self.model_config.lam * nextnonterminal * lastgaelam + mb_returns = mb_advs + trials_info.values + + trials_info.update_rewards(mb_rewards, mb_returns) + trials_info.convert_shape() + + def train(self, trials_info, nenvs): + """ + train the policy/value network using trials_info + + Parameters: + ---------- + trials_info: complete info of the generated trials from the previous inference + nenvs: the batch size of the (previous) inference + """ + # keep frac decay for future optimization + if self.cur_update <= self.nupdates: + frac = 1.0 - (self.cur_update - 1.0) / self.nupdates + else: + logger.warning('current update (self.cur_update) %d has exceeded total updates (self.nupdates) %d', + self.cur_update, self.nupdates) + frac = 1.0 - (self.nupdates - 1.0) / self.nupdates + lrnow = self.lr(frac) + cliprangenow = self.cliprange(frac) + self.cur_update += 1 + + states = self.states + + assert states is not None # recurrent version + assert nenvs % self.model_config.nminibatches == 0 + envsperbatch = nenvs // self.model_config.nminibatches + envinds = np.arange(nenvs) + flatinds = np.arange(nenvs * self.model_config.nsteps).reshape(nenvs, self.model_config.nsteps) + for _ in range(self.model_config.noptepochs): + np.random.shuffle(envinds) + for start in range(0, nenvs, envsperbatch): + end = start + envsperbatch + mbenvinds = envinds[start:end] + mbflatinds = flatinds[mbenvinds].ravel() + slices = (arr[mbflatinds] for arr in (trials_info.obs, trials_info.returns, trials_info.dones, + trials_info.actions, trials_info.values, trials_info.neglogpacs)) + mbstates = states[mbenvinds] + self.model.train(lrnow, cliprangenow, *slices, mbstates) + + +class PPOTuner(Tuner): + """ + PPOTuner + """ + + def __init__(self, optimize_mode, trials_per_update=20, epochs_per_update=4, minibatch_size=4, + ent_coef=0.0, lr=3e-4, vf_coef=0.5, max_grad_norm=0.5, gamma=0.99, lam=0.95, cliprange=0.2): + """ + initialization, PPO model is not initialized here as search space is not received yet. + + Parameters: + ---------- + optimize_mode: maximize or minimize + trials_per_update: number of trials to have for each model update + epochs_per_update: number of epochs to run for each model update + minibatch_size: minibatch size (number of trials) for the update + ent_coef: policy entropy coefficient in the optimization objective + lr: learning rate of the model (lstm network), constant + vf_coef: value function loss coefficient in the optimization objective + max_grad_norm: gradient norm clipping coefficient + gamma: discounting factor + lam: advantage estimation discounting factor (lambda in the paper) + cliprange: cliprange in the PPO algorithm, constant + """ + self.optimize_mode = OptimizeMode(optimize_mode) + self.model_config = ModelConfig() + self.model = None + self.search_space = None + self.running_trials = {} # key: parameter_id, value: actions/states/etc. + self.inf_batch_size = trials_per_update # number of trials to generate in one inference + self.first_inf = True # indicate whether it is the first time to inference new trials + self.trials_result = [None for _ in range(self.inf_batch_size)] # results of finished trials + + self.credit = 0 # record the unsatisfied trial requests + self.param_ids = [] + self.finished_trials = 0 + self.chosen_arch_template = {} + + self.actions_spaces = None + self.actions_to_config = None + self.full_act_space = None + self.trials_info = None + + self.all_trials = {} # used to dedup the same trial, key: config, value: final result + + self.model_config.num_envs = self.inf_batch_size + self.model_config.noptepochs = epochs_per_update + self.model_config.nminibatches = minibatch_size + + self.send_trial_callback = None + logger.info('=== finished PPOTuner initialization') + + def _process_one_nas_space(self, block_name, block_space): + """ + process nas space to determine observation space and action space + + Parameters: + ---------- + block_name: the name of the mutable block + block_space: search space of this mutable block + + Returns: + ---------- + actions_spaces: list of the space of each action + actions_to_config: the mapping from action to generated configuration + """ + actions_spaces = [] + actions_to_config = [] + + block_arch_temp = {} + for l_name, layer in block_space.items(): + chosen_layer_temp = {} + + if len(layer['layer_choice']) > 1: + actions_spaces.append(layer['layer_choice']) + actions_to_config.append((block_name, l_name, 'chosen_layer')) + chosen_layer_temp['chosen_layer'] = None + else: + assert len(layer['layer_choice']) == 1 + chosen_layer_temp['chosen_layer'] = layer['layer_choice'][0] + + if layer['optional_input_size'] not in [0, 1, [0, 1]]: + raise ValueError('Optional_input_size can only be 0, 1, or [0, 1], but the pecified one is %s' + % (layer['optional_input_size'])) + if isinstance(layer['optional_input_size'], list): + actions_spaces.append(["None", *layer['optional_inputs']]) + actions_to_config.append((block_name, l_name, 'chosen_inputs')) + chosen_layer_temp['chosen_inputs'] = None + elif layer['optional_input_size'] == 1: + actions_spaces.append(layer['optional_inputs']) + actions_to_config.append((block_name, l_name, 'chosen_inputs')) + chosen_layer_temp['chosen_inputs'] = None + elif layer['optional_input_size'] == 0: + chosen_layer_temp['chosen_inputs'] = [] + else: + raise ValueError('invalid type and value of optional_input_size') + + block_arch_temp[l_name] = chosen_layer_temp + + self.chosen_arch_template[block_name] = block_arch_temp + + return actions_spaces, actions_to_config + + def _process_nas_space(self, search_space): + """ + process nas search space to get action/observation space + """ + actions_spaces = [] + actions_to_config = [] + for b_name, block in search_space.items(): + if block['_type'] != 'mutable_layer': + raise ValueError('PPOTuner only accept mutable_layer type in search space, but the current one is %s'%(block['_type'])) + block = block['_value'] + act, act_map = self._process_one_nas_space(b_name, block) + actions_spaces.extend(act) + actions_to_config.extend(act_map) + + # calculate observation space + dedup = {} + for step in actions_spaces: + for action in step: + dedup[action] = 1 + full_act_space = [act for act, _ in dedup.items()] + assert len(full_act_space) == len(dedup) + observation_space = len(full_act_space) + + nsteps = len(actions_spaces) + + return actions_spaces, actions_to_config, full_act_space, observation_space, nsteps + + def _generate_action_mask(self): + """ + different step could have different action space. to deal with this case, we merge all the + possible actions into one action space, and use mask to indicate available actions for each step + """ + two_masks = [] + + mask = [] + for acts in self.actions_spaces: + one_mask = [0 for _ in range(len(self.full_act_space))] + for act in acts: + idx = self.full_act_space.index(act) + one_mask[idx] = 1 + mask.append(one_mask) + two_masks.append(mask) + + mask = [] + for acts in self.actions_spaces: + one_mask = [-np.inf for _ in range(len(self.full_act_space))] + for act in acts: + idx = self.full_act_space.index(act) + one_mask[idx] = 0 + mask.append(one_mask) + two_masks.append(mask) + + return np.asarray(two_masks, dtype=np.float32) + + def update_search_space(self, search_space): + """ + get search space, currently the space only includes that for NAS + + Parameters: + ---------- + search_space: search space for NAS + + Returns: + ------- + no return + """ + logger.info('=== update search space %s', search_space) + assert self.search_space is None + self.search_space = search_space + + assert self.model_config.observation_space is None + assert self.model_config.action_space is None + + self.actions_spaces, self.actions_to_config, self.full_act_space, obs_space, nsteps = self._process_nas_space(search_space) + + self.model_config.observation_space = spaces.Discrete(obs_space) + self.model_config.action_space = spaces.Discrete(obs_space) + self.model_config.nsteps = nsteps + + # generate mask in numpy + mask = self._generate_action_mask() + + assert self.model is None + self.model = PPOModel(self.model_config, mask) + + def _actions_to_config(self, actions): + """ + given actions, to generate the corresponding trial configuration + """ + chosen_arch = copy.deepcopy(self.chosen_arch_template) + for cnt, act in enumerate(actions): + act_name = self.full_act_space[act] + (block_name, layer_name, key) = self.actions_to_config[cnt] + if key == 'chosen_inputs': + if act_name == 'None': + chosen_arch[block_name][layer_name][key] = [] + else: + chosen_arch[block_name][layer_name][key] = [act_name] + elif key == 'chosen_layer': + chosen_arch[block_name][layer_name][key] = act_name + else: + raise ValueError('unrecognized key: {0}'.format(key)) + return chosen_arch + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Returns multiple sets of trial (hyper-)parameters, as iterable of serializable objects. + """ + result = [] + self.send_trial_callback = kwargs['st_callback'] + for parameter_id in parameter_id_list: + had_exception = False + try: + logger.debug("generating param for %s", parameter_id) + res = self.generate_parameters(parameter_id, **kwargs) + except nni.NoMoreTrialError: + had_exception = True + if not had_exception: + result.append(res) + return result + + def generate_parameters(self, parameter_id, **kwargs): + """ + generate parameters, if no trial configration for now, self.credit plus 1 to send the config later + """ + if self.first_inf: + self.trials_result = [None for _ in range(self.inf_batch_size)] + mb_obs, mb_actions, mb_values, mb_neglogpacs, mb_dones, last_values = self.model.inference(self.inf_batch_size) + self.trials_info = TrialsInfo(mb_obs, mb_actions, mb_values, mb_neglogpacs, + mb_dones, last_values, self.inf_batch_size) + self.first_inf = False + + trial_info_idx, actions = self.trials_info.get_next() + if trial_info_idx is None: + self.credit += 1 + self.param_ids.append(parameter_id) + raise nni.NoMoreTrialError('no more parameters now.') + + self.running_trials[parameter_id] = trial_info_idx + new_config = self._actions_to_config(actions) + return new_config + + def _next_round_inference(self): + """ + """ + self.finished_trials = 0 + self.model.compute_rewards(self.trials_info, self.trials_result) + self.model.train(self.trials_info, self.inf_batch_size) + self.running_trials = {} + # generate new trials + self.trials_result = [None for _ in range(self.inf_batch_size)] + mb_obs, mb_actions, mb_values, mb_neglogpacs, mb_dones, last_values = self.model.inference(self.inf_batch_size) + self.trials_info = TrialsInfo(mb_obs, mb_actions, mb_values, mb_neglogpacs, + mb_dones, last_values, self.inf_batch_size) + # check credit and submit new trials + for _ in range(self.credit): + trial_info_idx, actions = self.trials_info.get_next() + if trial_info_idx is None: + logger.warning('No enough trial config, trials_per_update is suggested to be larger than trialConcurrency') + break + assert self.param_ids + param_id = self.param_ids.pop() + self.running_trials[param_id] = trial_info_idx + new_config = self._actions_to_config(actions) + self.send_trial_callback(param_id, new_config) + self.credit -= 1 + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + receive trial's result. if the number of finished trials equals self.inf_batch_size, start the next update to + train the model + """ + trial_info_idx = self.running_trials.pop(parameter_id, None) + assert trial_info_idx is not None + + value = extract_scalar_reward(value) + if self.optimize_mode == OptimizeMode.Minimize: + value = -value + + self.trials_result[trial_info_idx] = value + self.finished_trials += 1 + + if self.finished_trials == self.inf_batch_size: + self._next_round_inference() + + def trial_end(self, parameter_id, success, **kwargs): + """ + to deal with trial failure + """ + if not success: + if parameter_id not in self.running_trials: + logger.warning('The trial is failed, but self.running_trial does not have this trial') + return + trial_info_idx = self.running_trials.pop(parameter_id, None) + assert trial_info_idx is not None + # use mean of finished trials as the result of this failed trial + values = [val for val in self.trials_result if val is not None] + logger.warning('zql values: {0}'.format(values)) + self.trials_result[trial_info_idx] = (sum(values) / len(values)) if len(values) > 0 else 0 + self.finished_trials += 1 + if self.finished_trials == self.inf_batch_size: + self._next_round_inference() + + def import_data(self, data): + """ + Import additional data for tuning + + Parameters + ---------- + data: a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + """ + logger.warning('PPOTuner cannot leverage imported data.') diff --git a/src/sdk/pynni/nni/ppo_tuner/requirements.txt b/src/sdk/pynni/nni/ppo_tuner/requirements.txt new file mode 100644 index 0000000000..138951469b --- /dev/null +++ b/src/sdk/pynni/nni/ppo_tuner/requirements.txt @@ -0,0 +1,3 @@ +enum34 +gym +tensorflow \ No newline at end of file diff --git a/src/sdk/pynni/nni/ppo_tuner/util.py b/src/sdk/pynni/nni/ppo_tuner/util.py new file mode 100644 index 0000000000..ac958e54de --- /dev/null +++ b/src/sdk/pynni/nni/ppo_tuner/util.py @@ -0,0 +1,266 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, +# to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +util functions +""" + +import os +import random +import multiprocessing +import numpy as np +import tensorflow as tf +from gym.spaces import Discrete, Box, MultiDiscrete + +def set_global_seeds(i): + """set global seeds""" + rank = 0 + myseed = i + 1000 * rank if i is not None else None + tf.set_random_seed(myseed) + np.random.seed(myseed) + random.seed(myseed) + +def batch_to_seq(h, nbatch, nsteps, flat=False): + """convert from batch to sequence""" + if flat: + h = tf.reshape(h, [nbatch, nsteps]) + else: + h = tf.reshape(h, [nbatch, nsteps, -1]) + return [tf.squeeze(v, [1]) for v in tf.split(axis=1, num_or_size_splits=nsteps, value=h)] + +def seq_to_batch(h, flat=False): + """convert from sequence to batch""" + shape = h[0].get_shape().as_list() + if not flat: + assert len(shape) > 1 + nh = h[0].get_shape()[-1].value + return tf.reshape(tf.concat(axis=1, values=h), [-1, nh]) + else: + return tf.reshape(tf.stack(values=h, axis=1), [-1]) + +def lstm(xs, ms, s, scope, nh, init_scale=1.0): + """lstm cell""" + nbatch, nin = [v.value for v in xs[0].get_shape()] + with tf.variable_scope(scope): + wx = tf.get_variable("wx", [nin, nh*4], initializer=ortho_init(init_scale)) + wh = tf.get_variable("wh", [nh, nh*4], initializer=ortho_init(init_scale)) + b = tf.get_variable("b", [nh*4], initializer=tf.constant_initializer(0.0)) + + c, h = tf.split(axis=1, num_or_size_splits=2, value=s) + for idx, (x, m) in enumerate(zip(xs, ms)): + c = c*(1-m) + h = h*(1-m) + z = tf.matmul(x, wx) + tf.matmul(h, wh) + b + i, f, o, u = tf.split(axis=1, num_or_size_splits=4, value=z) + i = tf.nn.sigmoid(i) + f = tf.nn.sigmoid(f) + o = tf.nn.sigmoid(o) + u = tf.tanh(u) + c = f*c + i*u + h = o*tf.tanh(c) + xs[idx] = h + s = tf.concat(axis=1, values=[c, h]) + return xs, s + +def lstm_model(nlstm=128, layer_norm=False): + """ + Builds LSTM (Long-Short Term Memory) network to be used in a policy. + Note that the resulting function returns not only the output of the LSTM + (i.e. hidden state of lstm for each step in the sequence), but also a dictionary + with auxiliary tensors to be set as policy attributes. + + Specifically, + S is a placeholder to feed current state (LSTM state has to be managed outside policy) + M is a placeholder for the mask (used to mask out observations after the end of the episode, but can be used for other purposes too) + initial_state is a numpy array containing initial lstm state (usually zeros) + state is the output LSTM state (to be fed into S at the next call) + + + An example of usage of lstm-based policy can be found here: common/tests/test_doc_examples.py/test_lstm_example + + Parameters: + ---------- + nlstm: int LSTM hidden state size + layer_norm: bool if True, layer-normalized version of LSTM is used + + Returns: + ------- + function that builds LSTM with a given input tensor / placeholder + """ + + def network_fn(X, nenv=1, obs_size=-1): + with tf.variable_scope("emb", reuse=tf.AUTO_REUSE): + w_emb = tf.get_variable("w_emb", [obs_size+1, 32]) + X = tf.nn.embedding_lookup(w_emb, X) + + nbatch = X.shape[0] + nsteps = nbatch // nenv + + h = tf.layers.flatten(X) + + M = tf.placeholder(tf.float32, [nbatch]) #mask (done t-1) + S = tf.placeholder(tf.float32, [nenv, 2*nlstm]) #states + + xs = batch_to_seq(h, nenv, nsteps) + ms = batch_to_seq(M, nenv, nsteps) + + assert not layer_norm + h5, snew = lstm(xs, ms, S, scope='lstm', nh=nlstm) + + h = seq_to_batch(h5) + initial_state = np.zeros(S.shape.as_list(), dtype=float) + + return h, {'S':S, 'M':M, 'state':snew, 'initial_state':initial_state} + + return network_fn + +def ortho_init(scale=1.0): + """init approach""" + def _ortho_init(shape, dtype, partition_info=None): + #lasagne ortho init for tf + shape = tuple(shape) + if len(shape) == 2: + flat_shape = shape + elif len(shape) == 4: # assumes NHWC + flat_shape = (np.prod(shape[:-1]), shape[-1]) + else: + raise NotImplementedError + a = np.random.normal(0.0, 1.0, flat_shape) + u, _, v = np.linalg.svd(a, full_matrices=False) + q = u if u.shape == flat_shape else v # pick the one with the correct shape + q = q.reshape(shape) + return (scale * q[:shape[0], :shape[1]]).astype(np.float32) + return _ortho_init + +def fc(x, scope, nh, *, init_scale=1.0, init_bias=0.0): + """fully connected op""" + with tf.variable_scope(scope): + nin = x.get_shape()[1].value + w = tf.get_variable("w", [nin, nh], initializer=ortho_init(init_scale)) + b = tf.get_variable("b", [nh], initializer=tf.constant_initializer(init_bias)) + return tf.matmul(x, w)+b + +def _check_shape(placeholder_shape, data_shape): + """ + check if two shapes are compatible (i.e. differ only by dimensions of size 1, or by the batch dimension) + """ + + return True + +# ================================================================ +# Shape adjustment for feeding into tf placeholders +# ================================================================ +def adjust_shape(placeholder, data): + """ + adjust shape of the data to the shape of the placeholder if possible. + If shape is incompatible, AssertionError is thrown + + Parameters: + placeholder: tensorflow input placeholder + data: input data to be (potentially) reshaped to be fed into placeholder + + Returns: + reshaped data + """ + if not isinstance(data, np.ndarray) and not isinstance(data, list): + return data + if isinstance(data, list): + data = np.array(data) + + placeholder_shape = [x or -1 for x in placeholder.shape.as_list()] + + assert _check_shape(placeholder_shape, data.shape), \ + 'Shape of data {} is not compatible with shape of the placeholder {}'.format(data.shape, placeholder_shape) + + return np.reshape(data, placeholder_shape) + +# ================================================================ +# Global session +# ================================================================ + +def get_session(config=None): + """Get default session or create one with a given config""" + sess = tf.get_default_session() + if sess is None: + sess = make_session(config=config, make_default=True) + return sess + +def make_session(config=None, num_cpu=None, make_default=False, graph=None): + """Returns a session that will use CPU's only""" + if num_cpu is None: + num_cpu = int(os.getenv('RCALL_NUM_CPU', multiprocessing.cpu_count())) + if config is None: + config = tf.ConfigProto( + allow_soft_placement=True, + inter_op_parallelism_threads=num_cpu, + intra_op_parallelism_threads=num_cpu) + config.gpu_options.allow_growth = True + + if make_default: + return tf.InteractiveSession(config=config, graph=graph) + else: + return tf.Session(config=config, graph=graph) + +ALREADY_INITIALIZED = set() + +def initialize(): + """Initialize all the uninitialized variables in the global scope.""" + new_variables = set(tf.global_variables()) - ALREADY_INITIALIZED + get_session().run(tf.variables_initializer(new_variables)) + + ALREADY_INITIALIZED.update(new_variables) + +def observation_placeholder(ob_space, batch_size=None, name='Ob'): + """ + Create placeholder to feed observations into of the size appropriate to the observation space + + Parameters: + ---------- + ob_space: gym.Space observation space + batch_size: int size of the batch to be fed into input. Can be left None in most cases. + name: str name of the placeholder + + Returns: + ------- + tensorflow placeholder tensor + """ + + assert isinstance(ob_space, (Discrete, Box, MultiDiscrete)), \ + 'Can only deal with Discrete and Box observation spaces for now' + + dtype = ob_space.dtype + if dtype == np.int8: + dtype = np.uint8 + + return tf.placeholder(shape=(batch_size,) + ob_space.shape, dtype=dtype, name=name) + +def explained_variance(ypred, y): + """ + Computes fraction of variance that ypred explains about y. + Returns 1 - Var[y-ypred] / Var[y] + + interpretation: + ev=0 => might as well have predicted zero + ev=1 => perfect prediction + ev<0 => worse than just predicting zero + + """ + assert y.ndim == 1 and ypred.ndim == 1 + vary = np.var(y) + return np.nan if vary == 0 else 1 - np.var(y-ypred)/vary diff --git a/src/sdk/pynni/nni/tuner.py b/src/sdk/pynni/nni/tuner.py index e8c345b3f3..c9c72d479b 100644 --- a/src/sdk/pynni/nni/tuner.py +++ b/src/sdk/pynni/nni/tuner.py @@ -17,11 +17,10 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT # OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # ================================================================================================== - - import logging import nni + from .recoverable import Recoverable _logger = logging.getLogger(__name__) diff --git a/tools/nni_annotation/.gitignore b/tools/nni_annotation/.gitignore new file mode 100644 index 0000000000..36e264cf44 --- /dev/null +++ b/tools/nni_annotation/.gitignore @@ -0,0 +1 @@ +_generated diff --git a/tools/nni_annotation/test_annotation.py b/tools/nni_annotation/test_annotation.py index 7550ba2466..a9ac3bc8ff 100644 --- a/tools/nni_annotation/test_annotation.py +++ b/tools/nni_annotation/test_annotation.py @@ -39,17 +39,18 @@ def setUpClass(cls): shutil.rmtree('_generated') def test_search_space_generator(self): - search_space = generate_search_space('testcase/annotated') + shutil.copytree('testcase/annotated', '_generated/annotated') + search_space = generate_search_space('_generated/annotated') with open('testcase/searchspace.json') as f: self.assertEqual(search_space, json.load(f)) def test_code_generator(self): - code_dir = expand_annotations('testcase/usercode', '_generated', nas_mode='classic_mode') - self.assertEqual(code_dir, '_generated') - self._assert_source_equal('testcase/annotated/nas.py', '_generated/nas.py') - self._assert_source_equal('testcase/annotated/mnist.py', '_generated/mnist.py') - self._assert_source_equal('testcase/annotated/dir/simple.py', '_generated/dir/simple.py') - with open('testcase/usercode/nonpy.txt') as src, open('_generated/nonpy.txt') as dst: + code_dir = expand_annotations('testcase/usercode', '_generated/usercode', nas_mode='classic_mode') + self.assertEqual(code_dir, '_generated/usercode') + self._assert_source_equal('testcase/annotated/nas.py', '_generated/usercode/nas.py') + self._assert_source_equal('testcase/annotated/mnist.py', '_generated/usercode/mnist.py') + self._assert_source_equal('testcase/annotated/dir/simple.py', '_generated/usercode/dir/simple.py') + with open('testcase/usercode/nonpy.txt') as src, open('_generated/usercode/nonpy.txt') as dst: assert src.read() == dst.read() def test_annotation_detecting(self): diff --git a/tools/nni_cmd/config_schema.py b/tools/nni_cmd/config_schema.py index 1568b9f713..abbce3dd82 100644 --- a/tools/nni_cmd/config_schema.py +++ b/tools/nni_cmd/config_schema.py @@ -142,6 +142,24 @@ def setPathCheck(key): Optional('includeIntermediateResults'): setType('includeIntermediateResults', bool), Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), }, + 'PPOTuner': { + 'builtinTunerName': 'PPOTuner', + 'classArgs': { + 'optimize_mode': setChoice('optimize_mode', 'maximize', 'minimize'), + Optional('trials_per_update'): setNumberRange('trials_per_update', int, 0, 99999), + Optional('epochs_per_update'): setNumberRange('epochs_per_update', int, 0, 99999), + Optional('minibatch_size'): setNumberRange('minibatch_size', int, 0, 99999), + Optional('ent_coef'): setType('ent_coef', float), + Optional('lr'): setType('lr', float), + Optional('vf_coef'): setType('vf_coef', float), + Optional('max_grad_norm'): setType('max_grad_norm', float), + Optional('gamma'): setType('gamma', float), + Optional('lam'): setType('lam', float), + Optional('cliprange'): setType('cliprange', float), + }, + Optional('includeIntermediateResults'): setType('includeIntermediateResults', bool), + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + }, 'customized': { 'codeDir': setPathCheck('codeDir'), 'classFileName': setType('classFileName', str), diff --git a/tools/nni_cmd/constants.py b/tools/nni_cmd/constants.py index 04a3dbbaff..d22a509c46 100644 --- a/tools/nni_cmd/constants.py +++ b/tools/nni_cmd/constants.py @@ -80,7 +80,8 @@ PACKAGE_REQUIREMENTS = { 'SMAC': 'smac_tuner', - 'BOHB': 'bohb_advisor' + 'BOHB': 'bohb_advisor', + 'PPOTuner': 'ppo_tuner' } TUNERS_SUPPORTING_IMPORT_DATA = { From 59ce65c50fe89e1c61b86211683975d2a8cd4e44 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Mon, 16 Sep 2019 18:23:29 +0800 Subject: [PATCH 03/22] fix nnictl log trial bug (#1550) --- tools/nni_cmd/nnictl_utils.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tools/nni_cmd/nnictl_utils.py b/tools/nni_cmd/nnictl_utils.py index 5575c25f2f..b6fada56e8 100644 --- a/tools/nni_cmd/nnictl_utils.py +++ b/tools/nni_cmd/nnictl_utils.py @@ -351,6 +351,7 @@ def log_stderr(args): def log_trial(args): ''''get trial log path''' trial_id_path_dict = {} + trial_id_list = [] nni_config = Config(get_config_filename(args)) rest_port = nni_config.get_config('restServerPort') rest_pid = nni_config.get_config('restServerPid') @@ -363,23 +364,27 @@ def log_trial(args): if response and check_response(response): content = json.loads(response.text) for trial in content: - trial_id_path_dict[trial['id']] = trial['logPath'] + trial_id_list.append(trial.get('id')) + if trial.get('logPath'): + trial_id_path_dict[trial.get('id')] = trial['logPath'] else: print_error('Restful server is not running...') exit(1) - if args.id: - if args.trial_id: - if trial_id_path_dict.get(args.trial_id): - print_normal('id:' + args.trial_id + ' path:' + trial_id_path_dict[args.trial_id]) - else: - print_error('trial id is not valid.') - exit(1) + if args.trial_id: + if args.trial_id not in trial_id_list: + print_error('Trial id {0} not correct, please check your command!'.format(args.trial_id)) + exit(1) + if trial_id_path_dict.get(args.trial_id): + print_normal('id:' + args.trial_id + ' path:' + trial_id_path_dict[args.trial_id]) else: - print_error('please specific the trial id.') + print_error('Log path is not available yet, please wait...') exit(1) else: + print_normal('All of trial log info:') for key in trial_id_path_dict: - print('id:' + key + ' path:' + trial_id_path_dict[key]) + print_normal('id:' + key + ' path:' + trial_id_path_dict[key]) + if not trial_id_path_dict: + print_normal('None') def get_config(args): '''get config info''' From 04d2d7cb1f2c3f279fa314e51351bdf80dc4d374 Mon Sep 17 00:00:00 2001 From: Guoxin Date: Tue, 17 Sep 2019 20:13:21 -0700 Subject: [PATCH 04/22] #1546 add doc for gp-tuner: support only numerical values (#1551) * add doc & err catch for gp-tuner: support only numerical values --- docs/en_US/Tuner/BuiltinTuner.md | 5 ++--- docs/en_US/Tuner/GPTuner.md | 2 ++ docs/en_US/Tuner/MetisTuner.md | 2 +- docs/en_US/Tutorial/SearchSpaceSpec.md | 2 +- src/sdk/pynni/nni/gp_tuner/target_space.py | 8 ++++++++ 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/docs/en_US/Tuner/BuiltinTuner.md b/docs/en_US/Tuner/BuiltinTuner.md index 1825e8dcd1..c7d27fc5d1 100644 --- a/docs/en_US/Tuner/BuiltinTuner.md +++ b/docs/en_US/Tuner/BuiltinTuner.md @@ -308,8 +308,7 @@ tuner: > Built-in Tuner Name: **MetisTuner** -Note that the only acceptable types of search space are `choice`, `quniform`, `uniform` and `randint`. - +Note that the only acceptable types of search space are `quniform`, `uniform` and `randint` and numerical `choice`. Only numerical values are supported since the values will be used to evaluate the 'distance' between different points. **Suggested scenario** Similar to TPE and SMAC, Metis is a black-box tuner. If your system takes a long time to finish each trial, Metis is more favorable than other approaches such as random search. Furthermore, Metis provides guidance on the subsequent trial. Here is an [example](https://github.com/Microsoft/nni/tree/master/examples/trials/auto-gbdt/search_space_metis.json) about the use of Metis. User only need to send the final result like `accuracy` to tuner, by calling the NNI SDK. [Detailed Description](./MetisTuner.md) @@ -381,7 +380,7 @@ advisor: > Built-in Tuner Name: **GPTuner** -Note that the only acceptable types of search space are `choice`, `randint`, `uniform`, `quniform`, `loguniform`, `qloguniform`. +Note that the only acceptable types of search space are `randint`, `uniform`, `quniform`, `loguniform`, `qloguniform`, and numerical `choice`. Only numerical values are supported since the values will be used to evaluate the 'distance' between different points. **Suggested scenario** diff --git a/docs/en_US/Tuner/GPTuner.md b/docs/en_US/Tuner/GPTuner.md index 9ef49db2bb..0159c3d5c9 100644 --- a/docs/en_US/Tuner/GPTuner.md +++ b/docs/en_US/Tuner/GPTuner.md @@ -7,4 +7,6 @@ Bayesian optimization works by constructing a posterior distribution of function GP Tuner is designed to minimize/maximize the number of steps required to find a combination of parameters that are close to the optimal combination. To do so, this method uses a proxy optimization problem (finding the maximum of the acquisition function) that, albeit still a hard problem, is cheaper (in the computational sense) and common tools can be employed. Therefore Bayesian Optimization is most adequate for situations where sampling the function to be optimized is a very expensive endeavor. +Note that the only acceptable types of search space are `randint`, `uniform`, `quniform`, `loguniform`, `qloguniform`, and numerical `choice`. + This optimization approach is described in Section 3 of [Algorithms for Hyper-Parameter Optimization](https://papers.nips.cc/paper/4443-algorithms-for-hyper-parameter-optimization.pdf). diff --git a/docs/en_US/Tuner/MetisTuner.md b/docs/en_US/Tuner/MetisTuner.md index 7c0c8e3e37..d653796e01 100644 --- a/docs/en_US/Tuner/MetisTuner.md +++ b/docs/en_US/Tuner/MetisTuner.md @@ -15,6 +15,6 @@ It finds the global optimal point in the Gaussian Process space. This point repr It identifies the next hyper-parameter candidate. This is achieved by inferring the potential information gain of exploration, exploitation, and re-sampling. -Note that the only acceptable types of search space are `choice`, `quniform`, `uniform` and `randint`. +Note that the only acceptable types of search space are `quniform`, `uniform` and `randint` and numerical `choice`. More details can be found in our paper: https://www.microsoft.com/en-us/research/publication/metis-robustly-tuning-tail-latencies-cloud-systems/ \ No newline at end of file diff --git a/docs/en_US/Tutorial/SearchSpaceSpec.md b/docs/en_US/Tutorial/SearchSpaceSpec.md index ea0d6bdf9e..b892a5e1e5 100644 --- a/docs/en_US/Tutorial/SearchSpaceSpec.md +++ b/docs/en_US/Tutorial/SearchSpaceSpec.md @@ -94,7 +94,7 @@ All types of sampling strategies and their parameter are listed here: Known Limitations: -* Note that Metis Tuner only supports numerical `choice` now +* GP Tuner and Metis Tuner support only **numerical values** in search space(`choice` type values can be no-numeraical with other tuners, e.g. string values). Both GP Tuner and Metis Tuner use Gaussian Process Regressor(GPR). GPR make predictions based on a kernel function and the 'distance' between different points, it's hard to get the true distance between no-numerical values. * Note that for nested search space: diff --git a/src/sdk/pynni/nni/gp_tuner/target_space.py b/src/sdk/pynni/nni/gp_tuner/target_space.py index 831bc335df..56481fa691 100644 --- a/src/sdk/pynni/nni/gp_tuner/target_space.py +++ b/src/sdk/pynni/nni/gp_tuner/target_space.py @@ -55,6 +55,14 @@ def __init__(self, pbounds, random_state=None): [item[1] for item in sorted(pbounds.items(), key=lambda x: x[0])] ) + # check values type + for _bound in self._bounds: + if _bound['_type'] == 'choice': + try: + [float(val) for val in _bound['_value']] + except ValueError: + raise ValueError("GP Tuner supports only numerical values") + # preallocated memory for X and Y points self._params = np.empty(shape=(0, self.dim)) self._target = np.empty(shape=(0)) From 0b7d62601d5c06193f164237570d5238034b9639 Mon Sep 17 00:00:00 2001 From: QuanluZhang Date: Fri, 20 Sep 2019 16:03:44 +0800 Subject: [PATCH 05/22] support specifying gpus for tuner and advisor (#1556) * support specifying gpu for tuner and advisor --- docs/en_US/Tutorial/ExperimentConfig.md | 63 ++++++++++++------- examples/trials/nas_cifar10/config_ppo.yml | 3 + src/nni_manager/common/utils.ts | 11 ++++ .../rest_server/restValidationSchemas.ts | 9 ++- src/sdk/pynni/nni/ppo_tuner/ppo_tuner.py | 1 - test/naive_test/local.yml | 2 - tools/nni_cmd/config_schema.py | 27 ++++---- tools/nni_cmd/launcher.py | 10 +++ 8 files changed, 81 insertions(+), 45 deletions(-) diff --git a/docs/en_US/Tutorial/ExperimentConfig.md b/docs/en_US/Tutorial/ExperimentConfig.md index cb5dfa4bc6..d610d25302 100644 --- a/docs/en_US/Tutorial/ExperimentConfig.md +++ b/docs/en_US/Tutorial/ExperimentConfig.md @@ -35,7 +35,7 @@ tuner: classArgs: #choice: maximize, minimize optimize_mode: - gpuNum: + gpuIndices: trial: command: codeDir: @@ -71,14 +71,13 @@ tuner: classArgs: #choice: maximize, minimize optimize_mode: - gpuNum: + gpuIndices: assessor: #choice: Medianstop builtinAssessorName: classArgs: #choice: maximize, minimize optimize_mode: - gpuNum: trial: command: codeDir: @@ -113,14 +112,13 @@ tuner: classArgs: #choice: maximize, minimize optimize_mode: - gpuNum: + gpuIndices: assessor: #choice: Medianstop builtinAssessorName: classArgs: #choice: maximize, minimize optimize_mode: - gpuNum: trial: command: codeDir: @@ -245,11 +243,11 @@ machineList: * __builtinTunerName__ and __classArgs__ * __builtinTunerName__ - __builtinTunerName__ specifies the name of system tuner, NNI sdk provides four kinds of tuner, including {__TPE__, __Random__, __Anneal__, __Evolution__, __BatchTuner__, __GridSearch__} + __builtinTunerName__ specifies the name of system tuner, NNI sdk provides different tuners introduced [here](../Tuner/BuiltinTuner.md). * __classArgs__ - __classArgs__ specifies the arguments of tuner algorithm. If the __builtinTunerName__ is in {__TPE__, __Random__, __Anneal__, __Evolution__}, user should set __optimize_mode__. + __classArgs__ specifies the arguments of tuner algorithm. Please refer to [this file](../Tuner/BuiltinTuner.md) for the configurable arguments of each built-in tuner. * __codeDir__, __classFileName__, __className__ and __classArgs__ * __codeDir__ @@ -264,16 +262,16 @@ machineList: __classArgs__ specifies the arguments of tuner algorithm. - * __gpuNum__ - - __gpuNum__ specifies the gpu number to run the tuner process. The value of this field should be a positive number. If the field is not set, NNI will not set `CUDA_VISIBLE_DEVICES` in script (that is, will not control the visibility of GPUs on trial command through `CUDA_VISIBLE_DEVICES`), and will not manage gpu resource. + * __gpuIndices__ - Note: users could only specify one way to set tuner, for example, set {tunerName, optimizationMode} or {tunerCommand, tunerCwd}, and could not set them both. + __gpuIndices__ specifies the gpus that can be used by the tuner process. Single or multiple GPU indices can be specified, multiple GPU indices are seperated by comma(,), such as `1` or `0,1,3`. If the field is not set, `CUDA_VISIBLE_DEVICES` will be '' in script, that is, no GPU is visible to tuner. * __includeIntermediateResults__ If __includeIntermediateResults__ is true, the last intermediate result of the trial that is early stopped by assessor is sent to tuner as final result. The default value of __includeIntermediateResults__ is false. + Note: users could only use one way to specify tuner, either specifying `builtinTunerName` and `classArgs`, or specifying `codeDir`, `classFileName`, `className` and `classArgs`. + * __assessor__ * Description @@ -282,7 +280,7 @@ machineList: * __builtinAssessorName__ and __classArgs__ * __builtinAssessorName__ - __builtinAssessorName__ specifies the name of system assessor, NNI sdk provides one kind of assessor {__Medianstop__} + __builtinAssessorName__ specifies the name of built-in assessor, NNI sdk provides different assessors introducted [here](../Assessor/BuiltinAssessor.md). * __classArgs__ __classArgs__ specifies the arguments of assessor algorithm @@ -305,11 +303,39 @@ machineList: __classArgs__ specifies the arguments of assessor algorithm. - * __gpuNum__ + Note: users could only use one way to specify assessor, either specifying `builtinAssessorName` and `classArgs`, or specifying `codeDir`, `classFileName`, `className` and `classArgs`. If users do not want to use assessor, assessor fileld should leave to empty. + +* __advisor__ + * Description - __gpuNum__ specifies the gpu number to run the assessor process. The value of this field should be a positive number. + __advisor__ specifies the advisor algorithm in the experiment, there are two kinds of ways to specify advisor. One way is to use advisor provided by NNI sdk, need to set __builtinAdvisorName__ and __classArgs__. Another way is to use users' own advisor file, and need to set __codeDirectory__, __classFileName__, __className__ and __classArgs__. + * __builtinAdvisorName__ and __classArgs__ + * __builtinAdvisorName__ - Note: users' could only specify one way to set assessor, for example,set {assessorName, optimizationMode} or {assessorCommand, assessorCwd}, and users could not set them both.If users do not want to use assessor, assessor fileld should leave to empty. + __builtinAdvisorName__ specifies the name of a built-in advisor, NNI sdk provides [different advisors](../Tuner/BuiltinTuner.md). + + * __classArgs__ + + __classArgs__ specifies the arguments of the advisor algorithm. Please refer to [this file](../Tuner/BuiltinTuner.md) for the configurable arguments of each built-in advisor. + * __codeDir__, __classFileName__, __className__ and __classArgs__ + * __codeDir__ + + __codeDir__ specifies the directory of advisor code. + * __classFileName__ + + __classFileName__ specifies the name of advisor file. + * __className__ + + __className__ specifies the name of advisor class. + * __classArgs__ + + __classArgs__ specifies the arguments of advisor algorithm. + + * __gpuIndices__ + + __gpuIndices__ specifies the gpus that can be used by the tuner process. Single or multiple GPU indices can be specified, multiple GPU indices are seperated by comma(,), such as `1` or `0,1,3`. If the field is not set, `CUDA_VISIBLE_DEVICES` will be '' in script, that is, no GPU is visible to tuner. + + Note: users could only use one way to specify advisor, either specifying `builtinAdvisorName` and `classArgs`, or specifying `codeDir`, `classFileName`, `className` and `classArgs`. * __trial(local, remote)__ @@ -560,7 +586,6 @@ machineList: classArgs: #choice: maximize, minimize optimize_mode: maximize - gpuNum: 0 trial: command: python3 mnist.py codeDir: /nni/mnist @@ -586,14 +611,12 @@ machineList: classArgs: #choice: maximize, minimize optimize_mode: maximize - gpuNum: 0 assessor: #choice: Medianstop builtinAssessorName: Medianstop classArgs: #choice: maximize, minimize optimize_mode: maximize - gpuNum: 0 trial: command: python3 mnist.py codeDir: /nni/mnist @@ -620,7 +643,6 @@ machineList: classArgs: #choice: maximize, minimize optimize_mode: maximize - gpuNum: 0 assessor: codeDir: /nni/assessor classFileName: myassessor.py @@ -628,7 +650,6 @@ machineList: classArgs: #choice: maximize, minimize optimize_mode: maximize - gpuNum: 0 trial: command: python3 mnist.py codeDir: /nni/mnist @@ -656,7 +677,6 @@ machineList: classArgs: #choice: maximize, minimize optimize_mode: maximize - gpuNum: 0 trial: command: python3 mnist.py codeDir: /nni/mnist @@ -780,7 +800,6 @@ machineList: builtinAssessorName: Medianstop classArgs: optimize_mode: maximize - gpuNum: 0 trial: codeDir: . worker: diff --git a/examples/trials/nas_cifar10/config_ppo.yml b/examples/trials/nas_cifar10/config_ppo.yml index 74c0dbea8e..8de1c5123f 100644 --- a/examples/trials/nas_cifar10/config_ppo.yml +++ b/examples/trials/nas_cifar10/config_ppo.yml @@ -15,6 +15,9 @@ tuner: trials_per_update: 60 epochs_per_update: 12 minibatch_size: 10 + #could use the No. 0 gpu for this tuner + #if want to specify multiple gpus, here is an example of specifying three gpus: 0,1,2 + gpuIndices: 0 trial: command: sh ./macro_cifar10.sh codeDir: ./ diff --git a/src/nni_manager/common/utils.ts b/src/nni_manager/common/utils.ts index 99d293c6a3..5ae7fc80cb 100644 --- a/src/nni_manager/common/utils.ts +++ b/src/nni_manager/common/utils.ts @@ -219,6 +219,11 @@ function getMsgDispatcherCommand(tuner: any, assessor: any, advisor: any, multiP if (advisor.classFileName !== undefined && advisor.classFileName.length > 1) { command += ` --advisor_class_filename ${advisor.classFileName}`; } + if (advisor.gpuIndices !== undefined) { + command = `CUDA_VISIBLE_DEVICES=${advisor.gpuIndices} ` + command; + } else { + command = `CUDA_VISIBLE_DEVICES='' ` + command; + } } else { command += ` --tuner_class_name ${tuner.className}`; if (tuner.classArgs !== undefined) { @@ -243,6 +248,12 @@ function getMsgDispatcherCommand(tuner: any, assessor: any, advisor: any, multiP command += ` --assessor_class_filename ${assessor.classFileName}`; } } + + if (tuner.gpuIndices !== undefined) { + command = `CUDA_VISIBLE_DEVICES=${tuner.gpuIndices} ` + command; + } else { + command = `CUDA_VISIBLE_DEVICES='' ` + command; + } } return command; diff --git a/src/nni_manager/rest_server/restValidationSchemas.ts b/src/nni_manager/rest_server/restValidationSchemas.ts index 23423218bc..2e79c1b2e7 100644 --- a/src/nni_manager/rest_server/restValidationSchemas.ts +++ b/src/nni_manager/rest_server/restValidationSchemas.ts @@ -170,8 +170,8 @@ export namespace ValidationSchemas { classFileName: joi.string(), className: joi.string(), classArgs: joi.any(), - gpuNum: joi.number().min(0), - checkpointDir: joi.string().allow('') + checkpointDir: joi.string().allow(''), + gpuIndices: joi.string() }), tuner: joi.object({ builtinTunerName: joi.string().valid('TPE', 'Random', 'Anneal', 'Evolution', 'SMAC', 'BatchTuner', 'GridSearch', 'NetworkMorphism', 'MetisTuner', 'GPTuner', 'PPOTuner'), @@ -179,9 +179,9 @@ export namespace ValidationSchemas { classFileName: joi.string(), className: joi.string(), classArgs: joi.any(), - gpuNum: joi.number().min(0), checkpointDir: joi.string().allow(''), - includeIntermediateResults: joi.boolean() + includeIntermediateResults: joi.boolean(), + gpuIndices: joi.string() }), assessor: joi.object({ builtinAssessorName: joi.string().valid('Medianstop', 'Curvefitting'), @@ -189,7 +189,6 @@ export namespace ValidationSchemas { classFileName: joi.string(), className: joi.string(), classArgs: joi.any(), - gpuNum: joi.number().min(0), checkpointDir: joi.string().allow('') }), clusterMetaData: joi.array().items(joi.object({ diff --git a/src/sdk/pynni/nni/ppo_tuner/ppo_tuner.py b/src/sdk/pynni/nni/ppo_tuner/ppo_tuner.py index 0a041ff567..1bc86ae750 100644 --- a/src/sdk/pynni/nni/ppo_tuner/ppo_tuner.py +++ b/src/sdk/pynni/nni/ppo_tuner/ppo_tuner.py @@ -23,7 +23,6 @@ class PPOTuner """ import os -os.environ["CUDA_VISIBLE_DEVICES"] = "" import copy import logging import numpy as np diff --git a/test/naive_test/local.yml b/test/naive_test/local.yml index a894c3c5ee..edf2a50322 100644 --- a/test/naive_test/local.yml +++ b/test/naive_test/local.yml @@ -14,14 +14,12 @@ tuner: className: NaiveTuner classArgs: optimize_mode: maximize - gpuNum: 0 assessor: codeDir: . classFileName: naive_assessor.py className: NaiveAssessor classArgs: optimize_mode: maximize - gpuNum: 0 trial: command: python3 naive_trial.py codeDir: . diff --git a/tools/nni_cmd/config_schema.py b/tools/nni_cmd/config_schema.py index abbce3dd82..da943564fb 100644 --- a/tools/nni_cmd/config_schema.py +++ b/tools/nni_cmd/config_schema.py @@ -76,7 +76,7 @@ def setPathCheck(key): 'optimize_mode': setChoice('optimize_mode', 'maximize', 'minimize'), }, Optional('includeIntermediateResults'): setType('includeIntermediateResults', bool), - Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), }, ('Evolution'): { 'builtinTunerName': setChoice('builtinTunerName', 'Evolution'), @@ -85,12 +85,12 @@ def setPathCheck(key): Optional('population_size'): setNumberRange('population_size', int, 0, 99999), }, Optional('includeIntermediateResults'): setType('includeIntermediateResults', bool), - Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), }, ('BatchTuner', 'GridSearch', 'Random'): { 'builtinTunerName': setChoice('builtinTunerName', 'BatchTuner', 'GridSearch', 'Random'), Optional('includeIntermediateResults'): setType('includeIntermediateResults', bool), - Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), }, 'TPE': { 'builtinTunerName': 'TPE', @@ -100,7 +100,7 @@ def setPathCheck(key): Optional('constant_liar_type'): setChoice('constant_liar_type', 'min', 'max', 'mean') }, Optional('includeIntermediateResults'): setType('includeIntermediateResults', bool), - Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), }, 'NetworkMorphism': { 'builtinTunerName': 'NetworkMorphism', @@ -112,7 +112,7 @@ def setPathCheck(key): Optional('n_output_node'): setType('n_output_node', int), }, Optional('includeIntermediateResults'): setType('includeIntermediateResults', bool), - Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), }, 'MetisTuner': { 'builtinTunerName': 'MetisTuner', @@ -124,7 +124,7 @@ def setPathCheck(key): Optional('cold_start_num'): setType('cold_start_num', int), }, Optional('includeIntermediateResults'): setType('includeIntermediateResults', bool), - Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), }, 'GPTuner': { 'builtinTunerName': 'GPTuner', @@ -140,7 +140,7 @@ def setPathCheck(key): Optional('selection_num_starting_points'): setType('selection_num_starting_points', int), }, Optional('includeIntermediateResults'): setType('includeIntermediateResults', bool), - Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), }, 'PPOTuner': { 'builtinTunerName': 'PPOTuner', @@ -158,7 +158,7 @@ def setPathCheck(key): Optional('cliprange'): setType('cliprange', float), }, Optional('includeIntermediateResults'): setType('includeIntermediateResults', bool), - Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), }, 'customized': { 'codeDir': setPathCheck('codeDir'), @@ -166,7 +166,7 @@ def setPathCheck(key): 'className': setType('className', str), Optional('classArgs'): dict, Optional('includeIntermediateResults'): setType('includeIntermediateResults', bool), - Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), } } @@ -178,7 +178,7 @@ def setPathCheck(key): Optional('R'): setType('R', int), Optional('eta'): setType('eta', int) }, - Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), }, 'BOHB':{ 'builtinAdvisorName': Or('BOHB'), @@ -194,14 +194,14 @@ def setPathCheck(key): Optional('bandwidth_factor'): setNumberRange('bandwidth_factor', float, 0, 9999), Optional('min_bandwidth'): setNumberRange('min_bandwidth', float, 0, 9999), }, - Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), }, 'customized':{ 'codeDir': setPathCheck('codeDir'), 'classFileName': setType('classFileName', str), 'className': setType('className', str), Optional('classArgs'): dict, - Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), } } @@ -212,7 +212,6 @@ def setPathCheck(key): Optional('optimize_mode'): setChoice('optimize_mode', 'maximize', 'minimize'), Optional('start_step'): setNumberRange('start_step', int, 0, 9999), }, - Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), }, 'Curvefitting': { 'builtinAssessorName': 'Curvefitting', @@ -223,14 +222,12 @@ def setPathCheck(key): Optional('threshold'): setNumberRange('threshold', float, 0, 9999), Optional('gap'): setNumberRange('gap', int, 1, 9999), }, - Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), }, 'customized': { 'codeDir': setPathCheck('codeDir'), 'classFileName': setType('classFileName', str), 'className': setType('className', str), Optional('classArgs'): dict, - Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999) } } diff --git a/tools/nni_cmd/launcher.py b/tools/nni_cmd/launcher.py index 1a1f3f78aa..ee8df7075c 100644 --- a/tools/nni_cmd/launcher.py +++ b/tools/nni_cmd/launcher.py @@ -296,10 +296,20 @@ def set_experiment(experiment_config, mode, port, config_file_name): request_data['multiThread'] = experiment_config.get('multiThread') if experiment_config.get('advisor'): request_data['advisor'] = experiment_config['advisor'] + if request_data['advisor'].get('gpuNum'): + print_error('gpuNum is deprecated, please use gpuIndices instead.') + if request_data['advisor'].get('gpuIndices') and isinstance(request_data['advisor'].get('gpuIndices'), int): + request_data['advisor']['gpuIndices'] = str(request_data['advisor'].get('gpuIndices')) else: request_data['tuner'] = experiment_config['tuner'] + if request_data['tuner'].get('gpuNum'): + print_error('gpuNum is deprecated, please use gpuIndices instead.') + if request_data['tuner'].get('gpuIndices') and isinstance(request_data['tuner'].get('gpuIndices'), int): + request_data['tuner']['gpuIndices'] = str(request_data['tuner'].get('gpuIndices')) if 'assessor' in experiment_config: request_data['assessor'] = experiment_config['assessor'] + if request_data['assessor'].get('gpuNum'): + print_error('gpuNum is deprecated, please remove it from your config file.') #debug mode should disable version check if experiment_config.get('debug') is not None: request_data['versionCheck'] = not experiment_config.get('debug') From 99f7d79c7e772f7c58415b459db95fa9cd09b273 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Thu, 26 Sep 2019 10:38:54 +0800 Subject: [PATCH 06/22] Support experiment view (#1524) --- docs/en_US/Tutorial/Nnictl.md | 30 +++ .../common/experimentStartupInfo.ts | 24 ++- src/nni_manager/common/log.ts | 26 ++- src/nni_manager/common/manager.ts | 8 +- src/nni_manager/core/nnimanager.ts | 26 ++- src/nni_manager/main.ts | 24 ++- src/nni_manager/rest_server/restHandler.ts | 34 ++-- tools/nni_cmd/launcher.py | 173 ++++++++---------- tools/nni_cmd/nnictl.py | 8 +- 9 files changed, 208 insertions(+), 145 deletions(-) diff --git a/docs/en_US/Tutorial/Nnictl.md b/docs/en_US/Tutorial/Nnictl.md index 5b5899e42d..6dac0bfc7f 100644 --- a/docs/en_US/Tutorial/Nnictl.md +++ b/docs/en_US/Tutorial/Nnictl.md @@ -10,6 +10,7 @@ nnictl support commands: * [nnictl create](#create) * [nnictl resume](#resume) +* [nnictl view](#view) * [nnictl stop](#stop) * [nnictl update](#update) * [nnictl trial](#trial) @@ -104,6 +105,35 @@ Debug mode will disable version check function in Trialkeeper. nnictl resume [experiment_id] --port 8088 ``` + + +![](https://placehold.it/15/1589F0/000000?text=+) `nnictl view` + +* Description + + You can use this command to view a stopped experiment. + +* Usage + + ```bash + nnictl view [OPTIONS] + ``` + +* Options + + |Name, shorthand|Required|Default|Description| + |------|------|------ |------| + |id| True| |The id of the experiment you want to view| + |--port, -p| False| |Rest port of the experiment you want to view| + +* Example + + > view an experiment with specified port 8088 + + ```bash + nnictl view [experiment_id] --port 8088 + ``` + ![](https://placehold.it/15/1589F0/000000?text=+) `nnictl stop` diff --git a/src/nni_manager/common/experimentStartupInfo.ts b/src/nni_manager/common/experimentStartupInfo.ts index c0c0c7a2a7..90c5d21363 100644 --- a/src/nni_manager/common/experimentStartupInfo.ts +++ b/src/nni_manager/common/experimentStartupInfo.ts @@ -33,11 +33,11 @@ class ExperimentStartupInfo { private initTrialSequenceID: number = 0; private logDir: string = ''; private logLevel: string = ''; + private readonly: boolean = false; - public setStartupInfo(newExperiment: boolean, experimentId: string, basePort: number, logDir?: string, logLevel?: string): void { + public setStartupInfo(newExperiment: boolean, experimentId: string, basePort: number, logDir?: string, logLevel?: string, readonly?: boolean): void { assert(!this.initialized); assert(experimentId.trim().length > 0); - this.newExperiment = newExperiment; this.experimentId = experimentId; this.basePort = basePort; @@ -52,6 +52,10 @@ class ExperimentStartupInfo { if (logLevel !== undefined && logLevel.length > 1) { this.logLevel = logLevel; } + + if (readonly !== undefined) { + this.readonly = readonly; + } } public getExperimentId(): string { @@ -84,6 +88,12 @@ class ExperimentStartupInfo { return this.logLevel; } + public isReadonly(): boolean { + assert(this.initialized); + + return this.readonly; + } + public setInitTrialSequenceId(initSequenceId: number): void { assert(this.initialized); this.initTrialSequenceID = initSequenceId; @@ -121,10 +131,14 @@ function getExperimentStartupInfo(): ExperimentStartupInfo { } function setExperimentStartupInfo( - newExperiment: boolean, experimentId: string, basePort: number, logDir?: string, logLevel?: string): void { + newExperiment: boolean, experimentId: string, basePort: number, logDir?: string, logLevel?: string, readonly?: boolean): void { component.get(ExperimentStartupInfo) - .setStartupInfo(newExperiment, experimentId, basePort, logDir, logLevel); + .setStartupInfo(newExperiment, experimentId, basePort, logDir, logLevel, readonly); +} + +function isReadonly(): boolean { + return component.get(ExperimentStartupInfo).isReadonly(); } export { ExperimentStartupInfo, getBasePort, getExperimentId, isNewExperiment, getExperimentStartupInfo, - setExperimentStartupInfo, setInitTrialSequenceId, getInitTrialSequenceId }; + setExperimentStartupInfo, setInitTrialSequenceId, getInitTrialSequenceId, isReadonly }; diff --git a/src/nni_manager/common/log.ts b/src/nni_manager/common/log.ts index d12e235836..e2ca62f9c6 100644 --- a/src/nni_manager/common/log.ts +++ b/src/nni_manager/common/log.ts @@ -26,7 +26,7 @@ import { Writable } from 'stream'; import { WritableStreamBuffer } from 'stream-buffers'; import { format } from 'util'; import * as component from '../common/component'; -import { getExperimentStartupInfo } from './experimentStartupInfo'; +import { getExperimentStartupInfo, isReadonly } from './experimentStartupInfo'; import { getLogDir } from './utils'; const FATAL: number = 1; @@ -76,6 +76,7 @@ class Logger { private level: number = INFO; private bufferSerialEmitter: BufferSerialEmitter; private writable: Writable; + private readonly: boolean = false; constructor(fileName?: string) { let logFile: string | undefined = fileName; @@ -95,6 +96,8 @@ class Logger { if (logLevel !== undefined) { this.level = logLevel; } + + this.readonly = isReadonly(); } public close() { @@ -134,14 +137,21 @@ class Logger { public fatal(...param: any[]): void { this.log('FATAL', param); } - + + /** + * if the experiment is not in readonly mode, write log content to stream + * @param level log level + * @param param the params to be written + */ private log(level: string, param: any[]): void { - const buffer: WritableStreamBuffer = new WritableStreamBuffer(); - buffer.write(`[${(new Date()).toLocaleString()}] ${level} `); - buffer.write(format(param)); - buffer.write('\n'); - buffer.end(); - this.bufferSerialEmitter.feed(buffer.getContents()); + if (!this.readonly) { + const buffer: WritableStreamBuffer = new WritableStreamBuffer(); + buffer.write(`[${(new Date()).toLocaleString()}] ${level} `); + buffer.write(format(param)); + buffer.write('\n'); + buffer.end(); + this.bufferSerialEmitter.feed(buffer.getContents()); + } } } diff --git a/src/nni_manager/common/manager.ts b/src/nni_manager/common/manager.ts index 4933465b92..bbe42d7c11 100644 --- a/src/nni_manager/common/manager.ts +++ b/src/nni_manager/common/manager.ts @@ -24,6 +24,10 @@ import { TrialJobStatus } from './trainingService'; type ProfileUpdateType = 'TRIAL_CONCURRENCY' | 'MAX_EXEC_DURATION' | 'SEARCH_SPACE' | 'MAX_TRIAL_NUM'; type ExperimentStatus = 'INITIALIZED' | 'RUNNING' | 'ERROR' | 'STOPPING' | 'STOPPED' | 'DONE' | 'NO_MORE_TRIAL' | 'TUNER_NO_MORE_TRIAL'; +namespace ExperimentStartUpMode { + export const NEW = 'new'; + export const RESUME = 'resume'; +} interface ExperimentParams { authorName: string; @@ -95,7 +99,7 @@ interface NNIManagerStatus { abstract class Manager { public abstract startExperiment(experimentParams: ExperimentParams): Promise; - public abstract resumeExperiment(): Promise; + public abstract resumeExperiment(readonly: boolean): Promise; public abstract stopExperiment(): Promise; public abstract getExperimentProfile(): Promise; public abstract updateExperimentProfile(experimentProfile: ExperimentProfile, updateType: ProfileUpdateType): Promise; @@ -115,4 +119,4 @@ abstract class Manager { public abstract getStatus(): NNIManagerStatus; } -export { Manager, ExperimentParams, ExperimentProfile, TrialJobStatistics, ProfileUpdateType, NNIManagerStatus, ExperimentStatus }; +export { Manager, ExperimentParams, ExperimentProfile, TrialJobStatistics, ProfileUpdateType, NNIManagerStatus, ExperimentStatus, ExperimentStartUpMode }; diff --git a/src/nni_manager/core/nnimanager.ts b/src/nni_manager/core/nnimanager.ts index 888901e44c..991fab633e 100644 --- a/src/nni_manager/core/nnimanager.ts +++ b/src/nni_manager/core/nnimanager.ts @@ -59,6 +59,7 @@ class NNIManager implements Manager { private waitingTrials: string[]; private trialJobs: Map; private trialDataForTuner: string; + private readonly: boolean; private trialJobMetricListener: (metric: TrialJobMetric) => void; @@ -72,6 +73,7 @@ class NNIManager implements Manager { this.waitingTrials = []; this.trialJobs = new Map(); this.trialDataForTuner = ''; + this.readonly = false; this.log = getLogger(); this.dataStore = component.get(DataStore); @@ -88,6 +90,9 @@ class NNIManager implements Manager { } public updateExperimentProfile(experimentProfile: ExperimentProfile, updateType: ProfileUpdateType): Promise { + if (this.readonly) { + return Promise.reject(new Error('Error: can not update experiment profile in readonly mode!')); + } switch (updateType) { case 'TRIAL_CONCURRENCY': this.updateTrialConcurrency(experimentProfile.params.trialConcurrency); @@ -109,6 +114,9 @@ class NNIManager implements Manager { } public importData(data: string): Promise { + if (this.readonly) { + return Promise.reject(new Error('Error: can not import data in readonly mode!')); + } if (this.dispatcher === undefined) { return Promise.reject( new Error('tuner has not been setup') @@ -124,6 +132,9 @@ class NNIManager implements Manager { } public addCustomizedTrialJob(hyperParams: string): Promise { + if (this.readonly) { + return Promise.reject(new Error('Error: can not add customized trial job in readonly mode!')); + } if (this.currSubmittedTrialNum >= this.experimentProfile.params.maxTrialNum) { return Promise.reject( new Error('reach maxTrialNum') @@ -136,6 +147,9 @@ class NNIManager implements Manager { } public async cancelTrialJobByUser(trialJobId: string): Promise { + if (this.readonly) { + return Promise.reject(new Error('Error: can not cancel trial job in readonly mode!')); + } this.log.info(`User cancelTrialJob: ${trialJobId}`); await this.trainingService.cancelTrialJob(trialJobId); await this.dataStore.storeTrialJobEvent('USER_TO_CANCEL', trialJobId, ''); @@ -180,13 +194,16 @@ class NNIManager implements Manager { return this.experimentProfile.id; } - public async resumeExperiment(): Promise { + public async resumeExperiment(readonly: boolean): Promise { this.log.info(`Resuming experiment: ${this.experimentProfile.id}`); //Fetch back the experiment profile const experimentId: string = getExperimentId(); this.experimentProfile = await this.dataStore.getExperimentProfile(experimentId); + this.readonly = readonly; + if (readonly) { + return Promise.resolve(); + } const expParams: ExperimentParams = this.experimentProfile.params; - setInitTrialSequenceId(this.experimentProfile.maxSequenceId + 1); // Set up multiphase config @@ -196,7 +213,7 @@ class NNIManager implements Manager { // Set up versionCheck config if (expParams.versionCheck !== undefined) { - this.trainingService.setClusterMetadata('versionCheck', expParams.versionCheck.toString()); + this.trainingService.setClusterMetadata('version_check', expParams.versionCheck.toString()); } const dispatcherCommand: string = getMsgDispatcherCommand(expParams.tuner, expParams.assessor, expParams.advisor, @@ -247,6 +264,9 @@ class NNIManager implements Manager { } public async setClusterMetadata(key: string, value: string): Promise { + if (this.readonly) { + return Promise.reject(new Error('Error: can not set cluster metadata in readonly mode!')); + } this.log.info(`NNIManager setClusterMetadata, key: ${key}, value: ${value}`); let timeoutId: NodeJS.Timer; // TO DO: move timeout value to constants file diff --git a/src/nni_manager/main.ts b/src/nni_manager/main.ts index b946894ac3..fec5a8819e 100644 --- a/src/nni_manager/main.ts +++ b/src/nni_manager/main.ts @@ -26,7 +26,7 @@ import * as component from './common/component'; import { Database, DataStore } from './common/datastore'; import { setExperimentStartupInfo } from './common/experimentStartupInfo'; import { getLogger, Logger, logLevelNameMap } from './common/log'; -import { Manager } from './common/manager'; +import { Manager, ExperimentStartUpMode } from './common/manager'; import { TrainingService } from './common/trainingService'; import { getLogDir, mkDirP, parseArg, uniqueString } from './common/utils'; import { NNIDataStore } from './core/nniDataStore'; @@ -43,10 +43,10 @@ import { function initStartupInfo( startExpMode: string, resumeExperimentId: string, basePort: number, - logDirectory: string, experimentLogLevel: string): void { - const createNew: boolean = (startExpMode === 'new'); + logDirectory: string, experimentLogLevel: string, readonly: boolean): void { + const createNew: boolean = (startExpMode === ExperimentStartUpMode.NEW); const expId: string = createNew ? uniqueString(8) : resumeExperimentId; - setExperimentStartupInfo(createNew, expId, basePort, logDirectory, experimentLogLevel); + setExperimentStartupInfo(createNew, expId, basePort, logDirectory, experimentLogLevel, readonly); } async function initContainer(platformMode: string): Promise { @@ -108,15 +108,15 @@ if (!['local', 'remote', 'pai', 'kubeflow', 'frameworkcontroller'].includes(mode } const startMode: string = parseArg(['--start_mode', '-s']); -if (!['new', 'resume'].includes(startMode)) { +if (![ExperimentStartUpMode.NEW, ExperimentStartUpMode.RESUME].includes(startMode)) { console.log(`FATAL: unknown start_mode: ${startMode}`); usage(); process.exit(1); } const experimentId: string = parseArg(['--experiment_id', '-id']); -if (startMode === 'resume' && experimentId.trim().length < 1) { - console.log(`FATAL: cannot resume experiment, invalid experiment_id: ${experimentId}`); +if ((startMode === ExperimentStartUpMode.RESUME) && experimentId.trim().length < 1) { + console.log(`FATAL: cannot resume the experiment, invalid experiment_id: ${experimentId}`); usage(); process.exit(1); } @@ -133,7 +133,15 @@ if (logLevel.length > 0 && !logLevelNameMap.has(logLevel)) { console.log(`FATAL: invalid log_level: ${logLevel}`); } -initStartupInfo(startMode, experimentId, port, logDir, logLevel); +const readonlyArg: string = parseArg(['--readonly', '-r']); +if (!('true' || 'false').includes(readonlyArg.toLowerCase())) { + console.log(`FATAL: readonly property should only be true or false`); + usage(); + process.exit(1); +} +const readonly = readonlyArg.toLowerCase() == 'true' ? true : false; + +initStartupInfo(startMode, experimentId, port, logDir, logLevel, readonly); mkDirP(getLogDir()) .then(async () => { diff --git a/src/nni_manager/rest_server/restHandler.ts b/src/nni_manager/rest_server/restHandler.ts index 7c11b15e72..415411649e 100644 --- a/src/nni_manager/rest_server/restHandler.ts +++ b/src/nni_manager/rest_server/restHandler.ts @@ -25,9 +25,9 @@ import * as path from 'path'; import * as component from '../common/component'; import { DataStore, MetricDataRecord, TrialJobInfo } from '../common/datastore'; import { NNIError, NNIErrorNames } from '../common/errors'; -import { isNewExperiment } from '../common/experimentStartupInfo'; +import { isNewExperiment, isReadonly } from '../common/experimentStartupInfo'; import { getLogger, Logger } from '../common/log'; -import { ExperimentProfile, Manager, TrialJobStatistics} from '../common/manager'; +import { ExperimentProfile, Manager, TrialJobStatistics, ExperimentStartUpMode } from '../common/manager'; import { ValidationSchemas } from './restValidationSchemas'; import { NNIRestServer } from './nniRestServer'; import { getVersion } from '../common/utils'; @@ -86,11 +86,11 @@ class NNIRestHandler { return router; } - private handle_error(err: Error, res: Response, isFatal: boolean = false): void { + private handle_error(err: Error, res: Response, isFatal: boolean = false, errorCode: number = 500): void { if (err instanceof NNIError && err.name === NNIErrorNames.NOT_FOUND) { res.status(404); } else { - res.status(500); + res.status(errorCode); } res.send({ error: err.message @@ -169,13 +169,13 @@ class NNIRestHandler { this.handle_error(err, res); }); } else { - this.nniManager.resumeExperiment().then(() => { + this.nniManager.resumeExperiment(isReadonly()).then(() => { res.send(); }).catch((err: Error) => { // Resume experiment is a step of initialization, so any exception thrown is a fatal this.handle_error(err, res); }); - } + } }); } @@ -193,18 +193,18 @@ class NNIRestHandler { router.put( '/experiment/cluster-metadata', expressJoi(ValidationSchemas.SETCLUSTERMETADATA), async (req: Request, res: Response) => { - // tslint:disable-next-line:no-any - const metadata: any = req.body; - const keys: string[] = Object.keys(metadata); - try { - for (const key of keys) { - await this.nniManager.setClusterMetadata(key, JSON.stringify(metadata[key])); + // tslint:disable-next-line:no-any + const metadata: any = req.body; + const keys: string[] = Object.keys(metadata); + try { + for (const key of keys) { + await this.nniManager.setClusterMetadata(key, JSON.stringify(metadata[key])); + } + res.send(); + } catch (err) { + // setClusterMetata is a step of initialization, so any exception thrown is a fatal + this.handle_error(NNIError.FromError(err), res, true); } - res.send(); - } catch (err) { - // setClusterMetata is a step of initialization, so any exception thrown is a fatal - this.handle_error(NNIError.FromError(err), res, true); - } }); } diff --git a/tools/nni_cmd/launcher.py b/tools/nni_cmd/launcher.py index ee8df7075c..e2fac2cb42 100644 --- a/tools/nni_cmd/launcher.py +++ b/tools/nni_cmd/launcher.py @@ -118,12 +118,17 @@ def start_rest_server(port, platform, mode, config_file_name, experiment_id=None node_command = 'node' if sys.platform == 'win32': node_command = os.path.join(entry_dir[:-3], 'Scripts', 'node.exe') - cmds = [node_command, entry_file, '--port', str(port), '--mode', platform, '--start_mode', mode] + cmds = [node_command, entry_file, '--port', str(port), '--mode', platform] + if mode == 'view': + cmds += ['--start_mode', 'resume'] + cmds += ['--readonly', 'true'] + else: + cmds += ['--start_mode', mode] if log_dir is not None: cmds += ['--log_dir', log_dir] if log_level is not None: cmds += ['--log_level', log_level] - if mode == 'resume': + if mode in ['resume', 'view']: cmds += ['--experiment_id', experiment_id] stdout_full_path, stderr_full_path = get_log_path(config_file_name) with open(stdout_full_path, 'a+') as stdout_file, open(stderr_full_path, 'a+') as stderr_file: @@ -156,7 +161,6 @@ def set_trial_config(experiment_config, port, config_file_name): def set_local_config(experiment_config, port, config_file_name): '''set local configuration''' - #set machine_list request_data = dict() if experiment_config.get('localConfig'): request_data['local_config'] = experiment_config['localConfig'] @@ -177,7 +181,7 @@ def set_local_config(experiment_config, port, config_file_name): fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) return False, err_message - return set_trial_config(experiment_config, port, config_file_name) + return set_trial_config(experiment_config, port, config_file_name), None def set_remote_config(experiment_config, port, config_file_name): '''Call setClusterMetadata to pass trial''' @@ -345,7 +349,6 @@ def set_experiment(experiment_config, mode, port, config_file_name): {'key': 'frameworkcontroller_config', 'value': experiment_config['frameworkcontrollerConfig']}) request_data['clusterMetaData'].append( {'key': 'trial_config', 'value': experiment_config['trial']}) - response = rest_post(experiment_url(port), json.dumps(request_data), REST_TIME_OUT, show_error=True) if check_response(response): return response @@ -357,6 +360,33 @@ def set_experiment(experiment_config, mode, port, config_file_name): print_error('Setting experiment error, error message is {}'.format(response.text)) return None +def set_platform_config(platform, experiment_config, port, config_file_name, rest_process): + '''call set_cluster_metadata for specific platform''' + print_normal('Setting {0} config...'.format(platform)) + config_result, err_msg = None, None + if platform == 'local': + config_result, err_msg = set_local_config(experiment_config, port, config_file_name) + elif platform == 'remote': + config_result, err_msg = set_remote_config(experiment_config, port, config_file_name) + elif platform == 'pai': + config_result, err_msg = set_pai_config(experiment_config, port, config_file_name) + elif platform == 'kubeflow': + config_result, err_msg = set_kubeflow_config(experiment_config, port, config_file_name) + elif platform == 'frameworkcontroller': + config_result, err_msg = set_frameworkcontroller_config(experiment_config, port, config_file_name) + else: + raise Exception(ERROR_INFO % 'Unsupported platform!') + exit(1) + if config_result: + print_normal('Successfully set {0} config!'.format(platform)) + else: + print_error('Failed! Error is: {}'.format(err_msg)) + try: + kill_command(rest_process.pid) + except Exception: + raise Exception(ERROR_INFO % 'Rest server stopped!') + exit(1) + def launch_experiment(args, experiment_config, mode, config_file_name, experiment_id=None): '''follow steps to start rest server and start experiment''' nni_config = Config(config_file_name) @@ -381,8 +411,10 @@ def launch_experiment(args, experiment_config, mode, config_file_name, experimen exit(1) log_dir = experiment_config['logDir'] if experiment_config.get('logDir') else None log_level = experiment_config['logLevel'] if experiment_config.get('logLevel') else None - if log_level not in ['trace', 'debug'] and (args.debug or experiment_config.get('debug') is True): - log_level = 'debug' + #view experiment mode do not need debug function, when view an experiment, there will be no new logs created + if mode != 'view': + if log_level not in ['trace', 'debug'] and (args.debug or experiment_config.get('debug') is True): + log_level = 'debug' # start rest server rest_process, start_time = start_rest_server(args.port, experiment_config['trainingServicePlatform'], mode, config_file_name, experiment_id, log_dir, log_level) nni_config.set_config('restServerPid', rest_process.pid) @@ -416,83 +448,14 @@ def launch_experiment(args, experiment_config, mode, config_file_name, experimen except Exception: raise Exception(ERROR_INFO % 'Rest server stopped!') exit(1) - - # set remote config - if experiment_config['trainingServicePlatform'] == 'remote': - print_normal('Setting remote config...') - config_result, err_msg = set_remote_config(experiment_config, args.port, config_file_name) - if config_result: - print_normal('Successfully set remote config!') - else: - print_error('Failed! Error is: {}'.format(err_msg)) - try: - kill_command(rest_process.pid) - except Exception: - raise Exception(ERROR_INFO % 'Rest server stopped!') - exit(1) - - # set local config - if experiment_config['trainingServicePlatform'] == 'local': - print_normal('Setting local config...') - if set_local_config(experiment_config, args.port, config_file_name): - print_normal('Successfully set local config!') - else: - print_error('Set local config failed!') - try: - kill_command(rest_process.pid) - except Exception: - raise Exception(ERROR_INFO % 'Rest server stopped!') - exit(1) - - #set pai config - if experiment_config['trainingServicePlatform'] == 'pai': - print_normal('Setting pai config...') - config_result, err_msg = set_pai_config(experiment_config, args.port, config_file_name) - if config_result: - print_normal('Successfully set pai config!') - else: - if err_msg: - print_error('Failed! Error is: {}'.format(err_msg)) - try: - kill_command(rest_process.pid) - except Exception: - raise Exception(ERROR_INFO % 'Restful server stopped!') - exit(1) - - #set kubeflow config - if experiment_config['trainingServicePlatform'] == 'kubeflow': - print_normal('Setting kubeflow config...') - config_result, err_msg = set_kubeflow_config(experiment_config, args.port, config_file_name) - if config_result: - print_normal('Successfully set kubeflow config!') - else: - if err_msg: - print_error('Failed! Error is: {}'.format(err_msg)) - try: - kill_command(rest_process.pid) - except Exception: - raise Exception(ERROR_INFO % 'Restful server stopped!') - exit(1) - - #set frameworkcontroller config - if experiment_config['trainingServicePlatform'] == 'frameworkcontroller': - print_normal('Setting frameworkcontroller config...') - config_result, err_msg = set_frameworkcontroller_config(experiment_config, args.port, config_file_name) - if config_result: - print_normal('Successfully set frameworkcontroller config!') - else: - if err_msg: - print_error('Failed! Error is: {}'.format(err_msg)) - try: - kill_command(rest_process.pid) - except Exception: - raise Exception(ERROR_INFO % 'Restful server stopped!') - exit(1) - + if mode != 'view': + # set platform configuration + set_platform_config(experiment_config['trainingServicePlatform'], experiment_config, args.port, config_file_name, rest_process) + # start a new experiment print_normal('Starting experiment...') # set debug configuration - if experiment_config.get('debug') is None: + if mode != 'view' and experiment_config.get('debug') is None: experiment_config['debug'] = args.debug response = set_experiment(experiment_config, mode, args.port, config_file_name) if response: @@ -519,8 +482,23 @@ def launch_experiment(args, experiment_config, mode, config_file_name, experimen print_normal(EXPERIMENT_SUCCESS_INFO % (experiment_id, ' '.join(web_ui_url_list))) -def resume_experiment(args): - '''resume an experiment''' +def create_experiment(args): + '''start a new experiment''' + config_file_name = ''.join(random.sample(string.ascii_letters + string.digits, 8)) + nni_config = Config(config_file_name) + config_path = os.path.abspath(args.config) + if not os.path.exists(config_path): + print_error('Please set correct config path!') + exit(1) + experiment_config = get_yml_content(config_path) + validate_all_content(experiment_config, config_path) + + nni_config.set_config('experimentConfig', experiment_config) + launch_experiment(args, experiment_config, 'new', config_file_name) + nni_config.set_config('restServerPort', args.port) + +def manage_stopped_experiment(args, mode): + '''view a stopped experiment''' update_experiment() experiment_config = Experiments() experiment_dict = experiment_config.get_all_experiments() @@ -528,38 +506,31 @@ def resume_experiment(args): experiment_endTime = None #find the latest stopped experiment if not args.id: - print_error('Please set experiment id! \nYou could use \'nnictl resume {id}\' to resume a stopped experiment!\n' \ - 'You could use \'nnictl experiment list --all\' to show all experiments!') + print_error('Please set experiment id! \nYou could use \'nnictl {0} {id}\' to {0} a stopped experiment!\n' \ + 'You could use \'nnictl experiment list --all\' to show all experiments!'.format(mode)) exit(1) else: if experiment_dict.get(args.id) is None: print_error('Id %s not exist!' % args.id) exit(1) if experiment_dict[args.id]['status'] != 'STOPPED': - print_error('Only stopped experiments can be resumed!') + print_error('Only stopped experiments can be {0}ed!'.format(mode)) exit(1) experiment_id = args.id - print_normal('Resuming experiment %s...' % experiment_id) + print_normal('{0} experiment {1}...'.format(mode, experiment_id)) nni_config = Config(experiment_dict[experiment_id]['fileName']) experiment_config = nni_config.get_config('experimentConfig') experiment_id = nni_config.get_config('experimentId') new_config_file_name = ''.join(random.sample(string.ascii_letters + string.digits, 8)) new_nni_config = Config(new_config_file_name) new_nni_config.set_config('experimentConfig', experiment_config) - launch_experiment(args, experiment_config, 'resume', new_config_file_name, experiment_id) + launch_experiment(args, experiment_config, mode, new_config_file_name, experiment_id) new_nni_config.set_config('restServerPort', args.port) -def create_experiment(args): - '''start a new experiment''' - config_file_name = ''.join(random.sample(string.ascii_letters + string.digits, 8)) - nni_config = Config(config_file_name) - config_path = os.path.abspath(args.config) - if not os.path.exists(config_path): - print_error('Please set correct config path!') - exit(1) - experiment_config = get_yml_content(config_path) - validate_all_content(experiment_config, config_path) +def view_experiment(args): + '''view a stopped experiment''' + manage_stopped_experiment(args, 'view') - nni_config.set_config('experimentConfig', experiment_config) - launch_experiment(args, experiment_config, 'new', config_file_name) - nni_config.set_config('restServerPort', args.port) +def resume_experiment(args): + '''resume an experiment''' + manage_stopped_experiment(args, 'resume') \ No newline at end of file diff --git a/tools/nni_cmd/nnictl.py b/tools/nni_cmd/nnictl.py index 3af676f2fa..8da30fdfb7 100644 --- a/tools/nni_cmd/nnictl.py +++ b/tools/nni_cmd/nnictl.py @@ -21,7 +21,7 @@ import argparse import pkg_resources -from .launcher import create_experiment, resume_experiment +from .launcher import create_experiment, resume_experiment, view_experiment from .updater import update_searchspace, update_concurrency, update_duration, update_trialnum, import_data from .nnictl_utils import * from .package_management import * @@ -66,6 +66,12 @@ def parse_args(): parser_resume.add_argument('--debug', '-d', action='store_true', help=' set debug mode') parser_resume.set_defaults(func=resume_experiment) + # parse view command + parser_resume = subparsers.add_parser('view', help='view a stopped experiment') + parser_resume.add_argument('id', nargs='?', help='The id of the experiment you want to view') + parser_resume.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of restful server') + parser_resume.set_defaults(func=view_experiment) + # parse update command parser_updater = subparsers.add_parser('update', help='update the experiment') #add subparsers for parser_updater From 3e62e60b0376e54a33b7f8a2aed90560a61f7fe7 Mon Sep 17 00:00:00 2001 From: liuzhe-lz <40699903+liuzhe-lz@users.noreply.github.com> Date: Thu, 26 Sep 2019 16:34:35 +0800 Subject: [PATCH 07/22] Refactor web UI to support incremental metric loading (#1557) * Refactor web UI to support incremental metric loading * refactor * Remove host job * Move sequence ID to NNI manager * implement incremental loading --- .../common/experimentStartupInfo.ts | 22 +- src/nni_manager/common/manager.ts | 5 +- src/nni_manager/common/trainingService.ts | 28 +- src/nni_manager/core/nnimanager.ts | 65 ++- src/nni_manager/core/sqlDatabase.ts | 8 +- src/nni_manager/core/test/dataStore.test.ts | 2 +- .../core/test/mockedTrainingService.ts | 8 +- src/nni_manager/core/test/nnimanager.test.ts | 2 +- src/nni_manager/core/test/sqlDatabase.test.ts | 8 +- src/nni_manager/rest_server/restHandler.ts | 24 + .../rest_server/restValidationSchemas.ts | 2 +- .../rest_server/test/mockedNNIManager.ts | 12 +- .../frameworkcontrollerTrainingService.ts | 21 +- .../kubeflow/kubeflowTrainingService.ts | 26 +- .../kubernetes/kubernetesData.ts | 10 +- .../kubernetes/kubernetesTrainingService.ts | 16 +- .../local/localTrainingService.ts | 126 +--- .../training_service/pai/paiData.ts | 8 +- .../pai/paiTrainingService.ts | 39 +- .../remote_machine/remoteMachineData.ts | 8 +- .../remoteMachineTrainingService.ts | 109 +--- .../test/localTrainingService.test.ts | 6 +- .../test/paiTrainingService.test.ts | 9 +- .../test/remoteMachineTrainingService.test.ts | 12 +- src/webui/src/App.tsx | 167 +++--- src/webui/src/components/Modal/Compare.tsx | 10 +- .../src/components/Modal/ExperimentDrawer.tsx | 2 +- src/webui/src/components/Modal/LogDrawer.tsx | 4 +- src/webui/src/components/Overview.tsx | 552 +++--------------- src/webui/src/components/SlideBar.tsx | 48 +- src/webui/src/components/TrialsDetail.tsx | 390 ++----------- .../src/components/overview/BasicInfo.tsx | 47 +- .../src/components/overview/NumInput.tsx | 85 +++ .../src/components/overview/Progress.tsx | 247 +++----- .../src/components/overview/SuccessTable.tsx | 156 ++--- .../src/components/overview/TrialProfile.tsx | 43 +- .../components/public-child/DefaultMetrc.tsx | 35 +- .../public-child/IntermediateVal.tsx | 34 +- .../src/components/public-child/OpenRow.tsx | 70 +-- .../trial-detail/DefaultMetricPoint.tsx | 365 ++++-------- .../src/components/trial-detail/Duration.tsx | 17 +- .../components/trial-detail/Intermediate.tsx | 110 ++-- .../src/components/trial-detail/Para.tsx | 99 ++-- .../src/components/trial-detail/TableList.tsx | 451 ++++++-------- src/webui/src/static/const.ts | 9 +- src/webui/src/static/datamodel.ts | 7 + src/webui/src/static/function.ts | 56 +- src/webui/src/static/interface.ts | 146 +++-- src/webui/src/static/model/experiment.ts | 87 +++ src/webui/src/static/model/trial.ts | 195 +++++++ src/webui/src/static/model/trialmanager.ts | 156 +++++ src/webui/tslint.json | 19 +- 52 files changed, 1689 insertions(+), 2494 deletions(-) create mode 100644 src/webui/src/components/overview/NumInput.tsx create mode 100644 src/webui/src/static/datamodel.ts create mode 100644 src/webui/src/static/model/experiment.ts create mode 100644 src/webui/src/static/model/trial.ts create mode 100644 src/webui/src/static/model/trialmanager.ts diff --git a/src/nni_manager/common/experimentStartupInfo.ts b/src/nni_manager/common/experimentStartupInfo.ts index 90c5d21363..5675facdde 100644 --- a/src/nni_manager/common/experimentStartupInfo.ts +++ b/src/nni_manager/common/experimentStartupInfo.ts @@ -30,7 +30,6 @@ class ExperimentStartupInfo { private newExperiment: boolean = true; private basePort: number = -1; private initialized: boolean = false; - private initTrialSequenceID: number = 0; private logDir: string = ''; private logLevel: string = ''; private readonly: boolean = false; @@ -93,17 +92,6 @@ class ExperimentStartupInfo { return this.readonly; } - - public setInitTrialSequenceId(initSequenceId: number): void { - assert(this.initialized); - this.initTrialSequenceID = initSequenceId; - } - - public getInitTrialSequenceId(): number { - assert(this.initialized); - - return this.initTrialSequenceID; - } } function getExperimentId(): string { @@ -118,14 +106,6 @@ function isNewExperiment(): boolean { return component.get(ExperimentStartupInfo).isNewExperiment(); } -function setInitTrialSequenceId(initSequenceId: number): void { - component.get(ExperimentStartupInfo).setInitTrialSequenceId(initSequenceId); -} - -function getInitTrialSequenceId(): number { - return component.get(ExperimentStartupInfo).getInitTrialSequenceId(); -} - function getExperimentStartupInfo(): ExperimentStartupInfo { return component.get(ExperimentStartupInfo); } @@ -141,4 +121,4 @@ function isReadonly(): boolean { } export { ExperimentStartupInfo, getBasePort, getExperimentId, isNewExperiment, getExperimentStartupInfo, - setExperimentStartupInfo, setInitTrialSequenceId, getInitTrialSequenceId, isReadonly }; + setExperimentStartupInfo, isReadonly }; diff --git a/src/nni_manager/common/manager.ts b/src/nni_manager/common/manager.ts index bbe42d7c11..65ab4b77ed 100644 --- a/src/nni_manager/common/manager.ts +++ b/src/nni_manager/common/manager.ts @@ -83,7 +83,7 @@ interface ExperimentProfile { logDir?: string; startTime?: number; endTime?: number; - maxSequenceId: number; + nextSequenceId: number; revision: number; } @@ -115,6 +115,9 @@ abstract class Manager { public abstract getClusterMetadata(key: string): Promise; public abstract getMetricData(trialJobId?: string, metricType?: MetricType): Promise; + public abstract getMetricDataByRange(minSeqId: number, maxSeqId: number): Promise; + public abstract getLatestMetricData(): Promise; + public abstract getTrialJobStatistics(): Promise; public abstract getStatus(): NNIManagerStatus; } diff --git a/src/nni_manager/common/trainingService.ts b/src/nni_manager/common/trainingService.ts index e4a9f1547e..2dfa0a9589 100644 --- a/src/nni_manager/common/trainingService.ts +++ b/src/nni_manager/common/trainingService.ts @@ -23,20 +23,12 @@ * define TrialJobStatus */ type TrialJobStatus = 'UNKNOWN' | 'WAITING' | 'RUNNING' | 'SUCCEEDED' | 'FAILED' | 'USER_CANCELED' | 'SYS_CANCELED' | 'EARLY_STOPPED'; -type JobType = 'TRIAL' | 'HOST'; interface TrainingServiceMetadata { readonly key: string; readonly value: string; } -/** - * define JobApplicationForm - */ -interface JobApplicationForm { - readonly jobType: JobType; -} - interface HyperParameters { readonly value: string; readonly index: number; @@ -45,18 +37,11 @@ interface HyperParameters { /** * define TrialJobApplicationForm */ -interface TrialJobApplicationForm extends JobApplicationForm { +interface TrialJobApplicationForm { + readonly sequenceId: number; readonly hyperParameters: HyperParameters; } -/** - * define HostJobApplicationForm - */ -interface HostJobApplicationForm extends JobApplicationForm { - readonly host: string; - readonly cmd: string; -} - /** * define TrialJobDetail */ @@ -69,8 +54,7 @@ interface TrialJobDetail { readonly tags?: string[]; readonly url?: string; readonly workingDirectory: string; - readonly form: JobApplicationForm; - readonly sequenceId: number; + readonly form: TrialJobApplicationForm; isEarlyStopped?: boolean; } @@ -112,8 +96,8 @@ abstract class TrainingService { public abstract getTrialJob(trialJobId: string): Promise; public abstract addTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void; public abstract removeTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void; - public abstract submitTrialJob(form: JobApplicationForm): Promise; - public abstract updateTrialJob(trialJobId: string, form: JobApplicationForm): Promise; + public abstract submitTrialJob(form: TrialJobApplicationForm): Promise; + public abstract updateTrialJob(trialJobId: string, form: TrialJobApplicationForm): Promise; public abstract get isMultiPhaseJobSupported(): boolean; public abstract cancelTrialJob(trialJobId: string, isEarlyStopped?: boolean): Promise; public abstract setClusterMetadata(key: string, value: string): Promise; @@ -135,5 +119,5 @@ class NNIManagerIpConfig { export { TrainingService, TrainingServiceError, TrialJobStatus, TrialJobApplicationForm, TrainingServiceMetadata, TrialJobDetail, TrialJobMetric, HyperParameters, - HostJobApplicationForm, JobApplicationForm, JobType, NNIManagerIpConfig + NNIManagerIpConfig }; diff --git a/src/nni_manager/core/nnimanager.ts b/src/nni_manager/core/nnimanager.ts index 991fab633e..adbdfcda34 100644 --- a/src/nni_manager/core/nnimanager.ts +++ b/src/nni_manager/core/nnimanager.ts @@ -26,7 +26,7 @@ import { Deferred } from 'ts-deferred'; import * as component from '../common/component'; import { DataStore, MetricDataRecord, MetricType, TrialJobInfo } from '../common/datastore'; import { NNIError } from '../common/errors'; -import { getExperimentId, setInitTrialSequenceId } from '../common/experimentStartupInfo'; +import { getExperimentId } from '../common/experimentStartupInfo'; import { getLogger, Logger } from '../common/log'; import { ExperimentParams, ExperimentProfile, Manager, ExperimentStatus, @@ -62,7 +62,7 @@ class NNIManager implements Manager { private readonly: boolean; private trialJobMetricListener: (metric: TrialJobMetric) => void; - + constructor() { this.currSubmittedTrialNum = 0; this.trialConcurrencyChange = 0; @@ -204,7 +204,6 @@ class NNIManager implements Manager { return Promise.resolve(); } const expParams: ExperimentParams = this.experimentProfile.params; - setInitTrialSequenceId(this.experimentProfile.maxSequenceId + 1); // Set up multiphase config if (expParams.multiPhase && this.trainingService.isMultiPhaseJobSupported) { @@ -301,6 +300,37 @@ class NNIManager implements Manager { return this.dataStore.getMetricData(trialJobId, metricType); } + public async getMetricDataByRange(minSeqId: number, maxSeqId: number): Promise { + const trialJobs = await this.dataStore.listTrialJobs(); + const targetTrials = trialJobs.filter(trial => ( + // FIXME: can this be undefined? + trial.sequenceId !== undefined && minSeqId <= trial.sequenceId && trial.sequenceId <= maxSeqId + )); + const targetTrialIds = new Set(targetTrials.map(trial => trial.id)); + + const allMetrics = await this.dataStore.getMetricData(); + return allMetrics.filter(metric => targetTrialIds.has(metric.trialJobId)); + } + + public async getLatestMetricData(): Promise { + // FIXME: this can take a long time + const allMetrics: MetricDataRecord[] = await this.dataStore.getMetricData(); + const finals: MetricDataRecord[] = []; + const latestIntermediates: Map = new Map(); + for (const metric of allMetrics) { + if (metric.type !== 'PERIODICAL') { + finals.push(metric); + } else { + const old: MetricDataRecord | undefined = latestIntermediates.get(metric.trialJobId); + if (old === undefined || old.sequence <= metric.sequence) { + latestIntermediates.set(metric.trialJobId, metric); + } + } + } + return finals.concat(Array.from(latestIntermediates.values())); + // FIXME: unit test + } + public getExperimentProfile(): Promise { // TO DO: using Promise.resolve() const deferred: Deferred = new Deferred(); @@ -383,7 +413,7 @@ class NNIManager implements Manager { if (this.dispatcher === undefined) { throw new Error('Error: tuner has not been setup'); } - this.trainingService.removeTrialJobMetricListener(this.trialJobMetricListener); + this.trainingService.removeTrialJobMetricListener(this.trialJobMetricListener); this.dispatcher.sendCommand(TERMINATE); let tunerAlive: boolean = true; // gracefully terminate tuner and assessor here, wait at most 30 seconds. @@ -456,11 +486,7 @@ class NNIManager implements Manager { case 'EARLY_STOPPED': this.trialJobs.delete(trialJobId); finishedTrialJobNum++; - if (trialJobDetail.form.jobType === 'TRIAL') { - hyperParams = (trialJobDetail.form).hyperParameters.value; - } else { - throw new Error('Error: jobType error, not TRIAL'); - } + hyperParams = trialJobDetail.form.hyperParameters.value; this.dispatcher.sendCommand(TRIAL_END, JSON.stringify({ trial_job_id: trialJobDetail.id, event: trialJobDetail.status, @@ -473,11 +499,7 @@ class NNIManager implements Manager { // TO DO: push this job to queue for retry this.trialJobs.delete(trialJobId); finishedTrialJobNum++; - if (trialJobDetail.form.jobType === 'TRIAL') { - hyperParams = (trialJobDetail.form).hyperParameters.value; - } else { - throw new Error('Error: jobType error, not TRIAL'); - } + hyperParams = trialJobDetail.form.hyperParameters.value; this.dispatcher.sendCommand(TRIAL_END, JSON.stringify({ trial_job_id: trialJobDetail.id, event: trialJobDetail.status, @@ -576,7 +598,7 @@ class NNIManager implements Manager { } this.currSubmittedTrialNum++; const trialJobAppForm: TrialJobApplicationForm = { - jobType: 'TRIAL', + sequenceId: this.experimentProfile.nextSequenceId++, hyperParameters: { value: hyperParams, index: 0 @@ -584,7 +606,7 @@ class NNIManager implements Manager { }; this.log.info(`submitTrialJob: form: ${JSON.stringify(trialJobAppForm)}`); const trialJobDetail: TrialJobDetail = await this.trainingService.submitTrialJob(trialJobAppForm); - await this.storeMaxSequenceId(trialJobDetail.sequenceId); + await this.storeExperimentProfile(); this.trialJobs.set(trialJobDetail.id, Object.assign({}, trialJobDetail)); const trialJobDetailSnapshot: TrialJobDetail | undefined = this.trialJobs.get(trialJobDetail.id); if (trialJobDetailSnapshot != undefined) { @@ -703,7 +725,7 @@ class NNIManager implements Manager { assert(tunerCommand.trial_job_id !== undefined); const trialJobForm: TrialJobApplicationForm = { - jobType: 'TRIAL', + sequenceId: -1, // FIXME: multi-phase tuner should use sequence ID instead of trial job ID hyperParameters: { value: content, index: tunerCommand.parameter_index @@ -757,7 +779,7 @@ class NNIManager implements Manager { revision: 0, execDuration: 0, logDir: getExperimentRootDir(), - maxSequenceId: 0, + nextSequenceId: 0, params: { authorName: '', experimentName: '', @@ -788,13 +810,6 @@ class NNIManager implements Manager { return Promise.resolve(chkpDir); } - - private async storeMaxSequenceId(sequenceId: number): Promise { - if (sequenceId > this.experimentProfile.maxSequenceId) { - this.experimentProfile.maxSequenceId = sequenceId; - await this.storeExperimentProfile(); - } - } } export { NNIManager }; diff --git a/src/nni_manager/core/sqlDatabase.ts b/src/nni_manager/core/sqlDatabase.ts index 182011dcba..8ae29413bc 100644 --- a/src/nni_manager/core/sqlDatabase.ts +++ b/src/nni_manager/core/sqlDatabase.ts @@ -54,7 +54,7 @@ create table ExperimentProfile ( startTime integer, endTime integer, logDir text, - maxSequenceId integer, + nextSequenceId integer, revision integer); create index ExperimentProfile_id on ExperimentProfile(id); `; @@ -67,7 +67,7 @@ function loadExperimentProfile(row: any): ExperimentProfile { startTime: row.startTime === null ? undefined : row.startTime, endTime: row.endTime === null ? undefined : row.endTime, logDir: row.logDir === null ? undefined : row.logDir, - maxSequenceId: row.maxSequenceId, + nextSequenceId: row.nextSequenceId, revision: row.revision }; } @@ -144,7 +144,7 @@ class SqlDB implements Database { exp.startTime === undefined ? null : exp.startTime, exp.endTime === undefined ? null : exp.endTime, exp.logDir === undefined ? null : exp.logDir, - exp.maxSequenceId, + exp.nextSequenceId, exp.revision ]; this.log.trace(`storeExperimentProfile: SQL: ${sql}, args: ${JSON.stringify(args)}`); @@ -183,7 +183,7 @@ class SqlDB implements Database { event: TrialJobEvent, trialJobId: string, timestamp: number, hyperParameter?: string, jobDetail?: TrialJobDetail): Promise { const sql: string = 'insert into TrialJobEvent values (?,?,?,?,?,?)'; const logPath: string | undefined = jobDetail === undefined ? undefined : jobDetail.url; - const sequenceId: number | undefined = jobDetail === undefined ? undefined : jobDetail.sequenceId; + const sequenceId: number | undefined = jobDetail === undefined ? undefined : jobDetail.form.sequenceId; const args: any[] = [timestamp, trialJobId, event, hyperParameter, logPath, sequenceId]; this.log.trace(`storeTrialJobEvent: SQL: ${sql}, args: ${JSON.stringify(args)}`); diff --git a/src/nni_manager/core/test/dataStore.test.ts b/src/nni_manager/core/test/dataStore.test.ts index d0303990bb..6794706672 100644 --- a/src/nni_manager/core/test/dataStore.test.ts +++ b/src/nni_manager/core/test/dataStore.test.ts @@ -80,7 +80,7 @@ describe('Unit test for dataStore', () => { execDuration: 0, startTime: Date.now(), endTime: Date.now(), - maxSequenceId: 0, + nextSequenceId: 0, revision: 0 } const id: string = profile.id; diff --git a/src/nni_manager/core/test/mockedTrainingService.ts b/src/nni_manager/core/test/mockedTrainingService.ts index 027234de9e..f50fb62113 100644 --- a/src/nni_manager/core/test/mockedTrainingService.ts +++ b/src/nni_manager/core/test/mockedTrainingService.ts @@ -41,9 +41,9 @@ class MockedTrainingService extends TrainingService { url: 'http://test', workingDirectory: '/tmp/mocked', form: { - jobType: 'TRIAL' + sequenceId: 0, + hyperParameters: { value: '', index: 0 } }, - sequenceId: 0 }; public jobDetail2: TrialJobDetail = { id: '3456', @@ -55,9 +55,9 @@ class MockedTrainingService extends TrainingService { url: 'http://test', workingDirectory: '/tmp/mocked', form: { - jobType: 'TRIAL' + sequenceId: 1, + hyperParameters: { value: '', index: 1 } }, - sequenceId: 0 }; public listTrialJobs(): Promise { diff --git a/src/nni_manager/core/test/nnimanager.test.ts b/src/nni_manager/core/test/nnimanager.test.ts index 1b22ba7315..2eac8b1c8c 100644 --- a/src/nni_manager/core/test/nnimanager.test.ts +++ b/src/nni_manager/core/test/nnimanager.test.ts @@ -101,7 +101,7 @@ describe('Unit test for nnimanager', function () { params: updateExperimentParams, id: 'test', execDuration: 0, - maxSequenceId: 0, + nextSequenceId: 0, revision: 0 } diff --git a/src/nni_manager/core/test/sqlDatabase.test.ts b/src/nni_manager/core/test/sqlDatabase.test.ts index f48e0d978e..d292776a3c 100644 --- a/src/nni_manager/core/test/sqlDatabase.test.ts +++ b/src/nni_manager/core/test/sqlDatabase.test.ts @@ -64,10 +64,10 @@ const expParams2: ExperimentParams = { }; const profiles: ExperimentProfile[] = [ - { params: expParams1, id: '#1', execDuration: 0, logDir: '/log', startTime: Date.now(), endTime: undefined, maxSequenceId: 0, revision: 1,}, - { params: expParams1, id: '#1', execDuration: 0, logDir: '/log', startTime: Date.now(), endTime: Date.now(), maxSequenceId: 0, revision: 2 }, - { params: expParams2, id: '#2', execDuration: 0, logDir: '/log', startTime: Date.now(), endTime: Date.now(), maxSequenceId: 0, revision: 2 }, - { params: expParams2, id: '#2', execDuration: 0, logDir: '/log', startTime: Date.now(), endTime: Date.now(), maxSequenceId: 0, revision: 3 } + { params: expParams1, id: '#1', execDuration: 0, logDir: '/log', startTime: Date.now(), endTime: undefined, nextSequenceId: 0, revision: 1,}, + { params: expParams1, id: '#1', execDuration: 0, logDir: '/log', startTime: Date.now(), endTime: Date.now(), nextSequenceId: 1, revision: 2 }, + { params: expParams2, id: '#2', execDuration: 0, logDir: '/log', startTime: Date.now(), endTime: Date.now(), nextSequenceId: 0, revision: 2 }, + { params: expParams2, id: '#2', execDuration: 0, logDir: '/log', startTime: Date.now(), endTime: Date.now(), nextSequenceId: 2, revision: 3 } ]; const events: TrialJobEventRecord[] = [ diff --git a/src/nni_manager/rest_server/restHandler.ts b/src/nni_manager/rest_server/restHandler.ts index 415411649e..83c95a2987 100644 --- a/src/nni_manager/rest_server/restHandler.ts +++ b/src/nni_manager/rest_server/restHandler.ts @@ -72,6 +72,8 @@ class NNIRestHandler { this.addTrialJob(router); this.cancelTrialJob(router); this.getMetricData(router); + this.getMetricDataByRange(router); + this.getLatestMetricData(router); this.exportData(router); // Express-joi-validator configuration @@ -262,6 +264,28 @@ class NNIRestHandler { }); } + private getMetricDataByRange(router: Router): void { + router.get('/metric-data-range/:min_seq_id/:max_seq_id', async (req: Request, res: Response) => { + const minSeqId = Number(req.params.min_seq_id); + const maxSeqId = Number(req.params.max_seq_id); + this.nniManager.getMetricDataByRange(minSeqId, maxSeqId).then((metricsData: MetricDataRecord[]) => { + res.send(metricsData); + }).catch((err: Error) => { + this.handle_error(err, res); + }); + }); + } + + private getLatestMetricData(router: Router): void { + router.get('/metric-data-latest/', async (req: Request, res: Response) => { + this.nniManager.getLatestMetricData().then((metricsData: MetricDataRecord[]) => { + res.send(metricsData); + }).catch((err: Error) => { + this.handle_error(err, res); + }); + }); + } + private exportData(router: Router): void { router.get('/export-data', (req: Request, res: Response) => { this.nniManager.exportData().then((exportedData: string) => { diff --git a/src/nni_manager/rest_server/restValidationSchemas.ts b/src/nni_manager/rest_server/restValidationSchemas.ts index 2e79c1b2e7..99bbe4bb96 100644 --- a/src/nni_manager/rest_server/restValidationSchemas.ts +++ b/src/nni_manager/rest_server/restValidationSchemas.ts @@ -209,7 +209,7 @@ export namespace ValidationSchemas { startTime: joi.number(), endTime: joi.number(), logDir: joi.string(), - maxSequenceId: joi.number() + nextSequenceId: joi.number() } }; } diff --git a/src/nni_manager/rest_server/test/mockedNNIManager.ts b/src/nni_manager/rest_server/test/mockedNNIManager.ts index 299c473aa6..3c4a502ec8 100644 --- a/src/nni_manager/rest_server/test/mockedNNIManager.ts +++ b/src/nni_manager/rest_server/test/mockedNNIManager.ts @@ -85,9 +85,9 @@ export class MockedNNIManager extends Manager { // tslint:disable-next-line:no-http-string url: 'http://test', workingDirectory: '/tmp/mocked', - sequenceId: 0, form: { - jobType: 'TRIAL' + sequenceId: 0, + hyperParameters: { value: '', index: 0 } } }; deferred.resolve(jobDetail); @@ -129,6 +129,12 @@ export class MockedNNIManager extends Manager { public getMetricData(trialJobId: string, metricType: MetricType): Promise { throw new MethodNotImplementedError(); } + public getMetricDataByRange(minSeqId: number, maxSeqId: number): Promise { + throw new MethodNotImplementedError(); + } + public getLatestMetricData(): Promise { + throw new MethodNotImplementedError(); + } public getExperimentProfile(): Promise { const profile: ExperimentProfile = { params: { @@ -148,7 +154,7 @@ export class MockedNNIManager extends Manager { execDuration: 0, startTime: Date.now(), endTime: Date.now(), - maxSequenceId: 0, + nextSequenceId: 0, revision: 0 }; diff --git a/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts b/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts index 51d56d5b7c..f54bd11e9e 100644 --- a/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts +++ b/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts @@ -25,7 +25,7 @@ import * as path from 'path'; import * as component from '../../../common/component'; import { getExperimentId } from '../../../common/experimentStartupInfo'; import { - JobApplicationForm, NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus + NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../../common/trainingService'; import { delay, generateParamFileName, getExperimentRootDir, uniqueString } from '../../../common/utils'; import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../../common/containerJobData'; @@ -55,7 +55,6 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple super(); this.fcJobInfoCollector = new FrameworkControllerJobInfoCollector(this.trialJobsMap); this.experimentId = getExperimentId(); - this.nextTrialSequenceId = -1; } public async run(): Promise { @@ -77,7 +76,7 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple } } - public async submitTrialJob(form: JobApplicationForm): Promise { + public async submitTrialJob(form: TrialJobApplicationForm): Promise { if (this.fcClusterConfig === undefined) { throw new Error('frameworkcontrollerClusterConfig is not initialized'); } @@ -91,14 +90,13 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple } const trialJobId: string = uniqueString(5); - const curTrialSequenceId: number = this.generateSequenceId(); // Set trial's NFS working folder const trialWorkingFolder: string = path.join(this.CONTAINER_MOUNT_PATH, 'nni', getExperimentId(), trialJobId); const trialLocalTempFolder: string = path.join(getExperimentRootDir(), 'trials-local', trialJobId); const frameworkcontrollerJobName: string = `nniexp${this.experimentId}trial${trialJobId}`.toLowerCase(); //Generate the port used for taskRole this.generateContainerPort(); - await this.prepareRunScript(trialLocalTempFolder, curTrialSequenceId, trialJobId, trialWorkingFolder, form); + await this.prepareRunScript(trialLocalTempFolder, trialJobId, trialWorkingFolder, form); //upload code files const trialJobOutputUrl: string = await this.uploadCodeFiles(trialJobId, trialLocalTempFolder); @@ -113,7 +111,6 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple trialWorkingFolder, form, frameworkcontrollerJobName, - curTrialSequenceId, trialJobOutputUrl ); @@ -248,8 +245,8 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple return `${portScript} . /mnt/frameworkbarrier/injector.sh && ${command}`; } - private async prepareRunScript(trialLocalTempFolder: string, curTrialSequenceId: number, trialJobId: string, - trialWorkingFolder: string, form: JobApplicationForm): Promise { + private async prepareRunScript(trialLocalTempFolder: string, trialJobId: string, + trialWorkingFolder: string, form: TrialJobApplicationForm): Promise { if (this.fcTrialConfig === undefined) { throw new Error('frameworkcontroller trial config is not initialized'); } @@ -264,16 +261,16 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple for (const taskRole of this.fcTrialConfig.taskRoles) { const runScriptContent: string = await this.generateRunScript('frameworkcontroller', trialJobId, trialWorkingFolder, - this.generateCommandScript(taskRole.command), curTrialSequenceId.toString(), + this.generateCommandScript(taskRole.command), form.sequenceId.toString(), taskRole.name, taskRole.gpuNum); await fs.promises.writeFile(path.join(trialLocalTempFolder, `run_${taskRole.name}.sh`), runScriptContent, { encoding: 'utf8' }); } // Write file content ( parameter.cfg ) to local tmp folders const trialForm : TrialJobApplicationForm = (form); - if (trialForm !== undefined && trialForm.hyperParameters !== undefined) { - await fs.promises.writeFile(path.join(trialLocalTempFolder, generateParamFileName(trialForm.hyperParameters)), - trialForm.hyperParameters.value, { encoding: 'utf8' }); + if (form !== undefined) { + await fs.promises.writeFile(path.join(trialLocalTempFolder, generateParamFileName(form.hyperParameters)), + form.hyperParameters.value, { encoding: 'utf8' }); } } diff --git a/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts b/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts index e70246176a..de61deb3ef 100644 --- a/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts +++ b/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts @@ -27,7 +27,7 @@ import * as component from '../../../common/component'; import { getExperimentId } from '../../../common/experimentStartupInfo'; import { - JobApplicationForm, NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus + NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../../common/trainingService'; import { delay, generateParamFileName, getExperimentRootDir, uniqueString } from '../../../common/utils'; import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../../common/containerJobData'; @@ -59,7 +59,6 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber super(); this.kubeflowJobInfoCollector = new KubeflowJobInfoCollector(this.trialJobsMap); this.experimentId = getExperimentId(); - this.nextTrialSequenceId = -1; this.log.info('Construct Kubeflow training service.'); } @@ -84,7 +83,7 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber this.log.info('Kubeflow training service exit.'); } - public async submitTrialJob(form: JobApplicationForm): Promise { + public async submitTrialJob(form: TrialJobApplicationForm): Promise { if (this.kubernetesCRDClient === undefined) { throw new Error('Kubeflow job operator client is undefined'); } @@ -96,10 +95,9 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber const trialJobId: string = uniqueString(5); const trialWorkingFolder: string = path.join(this.CONTAINER_MOUNT_PATH, 'nni', getExperimentId(), trialJobId); const kubeflowJobName: string = `nni-exp-${this.experimentId}-trial-${trialJobId}`.toLowerCase(); - const curTrialSequenceId: number = this.generateSequenceId(); const trialLocalTempFolder: string = path.join(getExperimentRootDir(), 'trials-local', trialJobId); //prepare the runscript - await this.prepareRunScript(trialLocalTempFolder, trialJobId, trialWorkingFolder, curTrialSequenceId, form); + await this.prepareRunScript(trialLocalTempFolder, trialJobId, trialWorkingFolder, form); //upload files to sotrage const trialJobOutputUrl: string = await this.uploadCodeFiles(trialJobId, trialLocalTempFolder); let initStatus: TrialJobStatus = 'WAITING'; @@ -113,7 +111,6 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber trialWorkingFolder, form, kubeflowJobName, - curTrialSequenceId, trialJobOutputUrl ); @@ -236,8 +233,8 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber return Promise.resolve(trialJobOutputUrl); } - private async prepareRunScript(trialLocalTempFolder: string, trialJobId: string, trialWorkingFolder: string, curTrialSequenceId: number, - form: JobApplicationForm): Promise { + private async prepareRunScript(trialLocalTempFolder: string, trialJobId: string, trialWorkingFolder: string, + form: TrialJobApplicationForm): Promise { if (this.kubeflowClusterConfig === undefined) { throw new Error('Kubeflow Cluster config is not initialized'); } @@ -262,7 +259,7 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber if (kubeflowTrialConfig.worker !== undefined) { const workerRunScriptContent: string = await this.generateRunScript('kubeflow', trialJobId, trialWorkingFolder, kubeflowTrialConfig.worker.command, - curTrialSequenceId.toString(), 'worker', + form.sequenceId.toString(), 'worker', kubeflowTrialConfig.worker.gpuNum); await fs.promises.writeFile(path.join(trialLocalTempFolder, 'run_worker.sh'), workerRunScriptContent, { encoding: 'utf8' }); } @@ -272,7 +269,7 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber if (tensorflowTrialConfig.ps !== undefined) { const psRunScriptContent: string = await this.generateRunScript('kubeflow', trialJobId, trialWorkingFolder, tensorflowTrialConfig.ps.command, - curTrialSequenceId.toString(), + form.sequenceId.toString(), 'ps', tensorflowTrialConfig.ps.gpuNum); await fs.promises.writeFile(path.join(trialLocalTempFolder, 'run_ps.sh'), psRunScriptContent, { encoding: 'utf8' }); } @@ -281,16 +278,15 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber if (pytorchTrialConfig.master !== undefined) { const masterRunScriptContent: string = await this.generateRunScript('kubeflow', trialJobId, trialWorkingFolder, pytorchTrialConfig.master.command, - curTrialSequenceId.toString(), 'master', + form.sequenceId.toString(), 'master', pytorchTrialConfig.master.gpuNum); await fs.promises.writeFile(path.join(trialLocalTempFolder, 'run_master.sh'), masterRunScriptContent, { encoding: 'utf8' }); } } // Write file content ( parameter.cfg ) to local tmp folders - const trialForm : TrialJobApplicationForm = (form); - if (trialForm !== undefined && trialForm.hyperParameters !== undefined) { - await fs.promises.writeFile(path.join(trialLocalTempFolder, generateParamFileName(trialForm.hyperParameters)), - trialForm.hyperParameters.value, { encoding: 'utf8' }); + if (form !== undefined) { + await fs.promises.writeFile(path.join(trialLocalTempFolder, generateParamFileName(form.hyperParameters)), + form.hyperParameters.value, { encoding: 'utf8' }); } } diff --git a/src/nni_manager/training_service/kubernetes/kubernetesData.ts b/src/nni_manager/training_service/kubernetes/kubernetesData.ts index b52e9a3049..f49f67eee0 100644 --- a/src/nni_manager/training_service/kubernetes/kubernetesData.ts +++ b/src/nni_manager/training_service/kubernetes/kubernetesData.ts @@ -19,7 +19,7 @@ 'use strict'; -import { JobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../common/trainingService'; +import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../common/trainingService'; /** * KubeflowTrialJobDetail @@ -33,21 +33,19 @@ export class KubernetesTrialJobDetail implements TrialJobDetail { public tags?: string[]; public url?: string; public workingDirectory: string; - public form: JobApplicationForm; + public form: TrialJobApplicationForm; public kubernetesJobName: string; - public sequenceId: number; public queryJobFailedCount: number; constructor(id: string, status: TrialJobStatus, submitTime: number, - workingDirectory: string, form: JobApplicationForm, - kubernetesJobName: string, sequenceId: number, url: string) { + workingDirectory: string, form: TrialJobApplicationForm, + kubernetesJobName: string, url: string) { this.id = id; this.status = status; this.submitTime = submitTime; this.workingDirectory = workingDirectory; this.form = form; this.kubernetesJobName = kubernetesJobName; - this.sequenceId = sequenceId; this.tags = []; this.queryJobFailedCount = 0; this.url = url; diff --git a/src/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts b/src/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts index 6a1df6e0f2..62e4916599 100644 --- a/src/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts +++ b/src/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts @@ -26,7 +26,7 @@ import * as azureStorage from 'azure-storage'; import { EventEmitter } from 'events'; import { Base64 } from 'js-base64'; import { String } from 'typescript-string-operations'; -import { getExperimentId, getInitTrialSequenceId } from '../../common/experimentStartupInfo'; +import { getExperimentId } from '../../common/experimentStartupInfo'; import { getLogger, Logger } from '../../common/log'; import { NNIManagerIpConfig, TrialJobDetail, TrialJobMetric @@ -53,7 +53,6 @@ abstract class KubernetesTrainingService { protected readonly trialLocalNFSTempFolder: string; protected stopping: boolean = false; protected experimentId! : string; - protected nextTrialSequenceId: number; protected kubernetesRestServerPort?: number; protected readonly CONTAINER_MOUNT_PATH: string; protected azureStorageClient?: azureStorage.FileService; @@ -74,7 +73,6 @@ abstract class KubernetesTrainingService { this.trialJobsMap = new Map(); this.trialLocalNFSTempFolder = path.join(getExperimentRootDir(), 'trials-nfs-tmp'); this.experimentId = getExperimentId(); - this.nextTrialSequenceId = -1; this.CONTAINER_MOUNT_PATH = '/tmp/mount'; this.genericK8sClient = new GeneralK8sClient(); this.logCollection = 'none'; @@ -93,9 +91,7 @@ abstract class KubernetesTrainingService { const jobs: TrialJobDetail[] = []; for (const [key, value] of this.trialJobsMap) { - if (value.form.jobType === 'TRIAL') { - jobs.push(await this.getTrialJob(key)); - } + jobs.push(await this.getTrialJob(key)); } return Promise.resolve(jobs); @@ -222,14 +218,6 @@ abstract class KubernetesTrainingService { return Promise.resolve(); } - protected generateSequenceId(): number { - if (this.nextTrialSequenceId === -1) { - this.nextTrialSequenceId = getInitTrialSequenceId(); - } - - return this.nextTrialSequenceId++; - } - // tslint:disable: no-unsafe-any no-any protected async createAzureStorage(vaultName: string, valutKeyName: string, accountName: string, azureShare: string): Promise { try { diff --git a/src/nni_manager/training_service/local/localTrainingService.ts b/src/nni_manager/training_service/local/localTrainingService.ts index 88e006a3f9..1a7c70d3a1 100644 --- a/src/nni_manager/training_service/local/localTrainingService.ts +++ b/src/nni_manager/training_service/local/localTrainingService.ts @@ -26,10 +26,10 @@ import * as path from 'path'; import * as ts from 'tail-stream'; import * as tkill from 'tree-kill'; import { NNIError, NNIErrorNames } from '../../common/errors'; -import { getExperimentId, getInitTrialSequenceId } from '../../common/experimentStartupInfo'; +import { getExperimentId } from '../../common/experimentStartupInfo'; import { getLogger, Logger } from '../../common/log'; import { - HostJobApplicationForm, HyperParameters, JobApplicationForm, TrainingService, TrialJobApplicationForm, + HyperParameters, TrainingService, TrialJobApplicationForm, TrialJobDetail, TrialJobMetric, TrialJobStatus } from '../../common/trainingService'; import { @@ -76,21 +76,19 @@ class LocalTrialJobDetail implements TrialJobDetail { public tags?: string[]; public url?: string; public workingDirectory: string; - public form: JobApplicationForm; - public sequenceId: number; + public form: TrialJobApplicationForm; public pid?: number; public gpuIndices?: number[]; constructor( id: string, status: TrialJobStatus, submitTime: number, - workingDirectory: string, form: JobApplicationForm, sequenceId: number) { + workingDirectory: string, form: TrialJobApplicationForm) { this.id = id; this.status = status; this.submitTime = submitTime; this.workingDirectory = workingDirectory; this.form = form; this.url = `file://localhost:${workingDirectory}`; - this.sequenceId = sequenceId; this.gpuIndices = []; } } @@ -125,7 +123,6 @@ class LocalTrainingService implements TrainingService { private initialized: boolean; private stopping: boolean; private rootDir!: string; - private trialSequenceId: number; private readonly experimentId! : string; private gpuScheduler!: GPUScheduler; private readonly occupiedGpuIndexNumMap: Map; @@ -145,7 +142,6 @@ class LocalTrainingService implements TrainingService { this.initialized = false; this.stopping = false; this.log = getLogger(); - this.trialSequenceId = -1; this.experimentId = getExperimentId(); this.jobStreamMap = new Map(); this.log.info('Construct local machine training service.'); @@ -169,9 +165,7 @@ class LocalTrainingService implements TrainingService { const jobs: TrialJobDetail[] = []; for (const key of this.jobMap.keys()) { const trialJob: TrialJobDetail = await this.getTrialJob(key); - if (trialJob.form.jobType === 'TRIAL') { - jobs.push(trialJob); - } + jobs.push(trialJob); } return jobs; @@ -182,9 +176,6 @@ class LocalTrainingService implements TrainingService { if (trialJob === undefined) { throw new NNIError(NNIErrorNames.NOT_FOUND, 'Trial job not found'); } - if (trialJob.form.jobType === 'HOST') { - return this.getHostJob(trialJobId); - } if (trialJob.status === 'RUNNING') { const alive: boolean = await isAlive(trialJob.pid); if (!alive) { @@ -219,28 +210,21 @@ class LocalTrainingService implements TrainingService { this.eventEmitter.off('metric', listener); } - public submitTrialJob(form: JobApplicationForm): Promise { - if (form.jobType === 'HOST') { - return this.runHostJob(form); - } else if (form.jobType === 'TRIAL') { - const trialJobId: string = uniqueString(5); - const trialJobDetail: LocalTrialJobDetail = new LocalTrialJobDetail( - trialJobId, - 'WAITING', - Date.now(), - path.join(this.rootDir, 'trials', trialJobId), - form, - this.generateSequenceId() - ); - this.jobQueue.push(trialJobId); - this.jobMap.set(trialJobId, trialJobDetail); - - this.log.debug(`submitTrialJob: return: ${JSON.stringify(trialJobDetail)} `); - - return Promise.resolve(trialJobDetail); - } else { - return Promise.reject(new Error(`Job form not supported: ${JSON.stringify(form)}`)); - } + public submitTrialJob(form: TrialJobApplicationForm): Promise { + const trialJobId: string = uniqueString(5); + const trialJobDetail: LocalTrialJobDetail = new LocalTrialJobDetail( + trialJobId, + 'WAITING', + Date.now(), + path.join(this.rootDir, 'trials', trialJobId), + form + ); + this.jobQueue.push(trialJobId); + this.jobMap.set(trialJobId, trialJobDetail); + + this.log.debug(`submitTrialJob: return: ${JSON.stringify(trialJobDetail)} `); + + return Promise.resolve(trialJobDetail); } /** @@ -248,16 +232,12 @@ class LocalTrainingService implements TrainingService { * @param trialJobId trial job id * @param form job application form */ - public async updateTrialJob(trialJobId: string, form: JobApplicationForm): Promise { + public async updateTrialJob(trialJobId: string, form: TrialJobApplicationForm): Promise { const trialJobDetail: undefined | TrialJobDetail = this.jobMap.get(trialJobId); if (trialJobDetail === undefined) { throw new Error(`updateTrialJob failed: ${trialJobId} not found`); } - if (form.jobType === 'TRIAL') { - await this.writeParameterFile(trialJobDetail.workingDirectory, (form).hyperParameters); - } else { - throw new Error(`updateTrialJob failed: jobType ${form.jobType} not supported.`); - } + await this.writeParameterFile(trialJobDetail.workingDirectory, form.hyperParameters); return trialJobDetail; } @@ -279,13 +259,7 @@ class LocalTrainingService implements TrainingService { return Promise.resolve(); } - if (trialJob.form.jobType === 'TRIAL') { - tkill(trialJob.pid, 'SIGKILL'); - } else if (trialJob.form.jobType === 'HOST') { - await cpp.exec(`pkill -9 -P ${trialJob.pid}`); - } else { - throw new Error(`Job type not supported: ${trialJob.form.jobType}`); - } + tkill(trialJob.pid, 'SIGKILL'); this.setTrialJobStatus(trialJob, getJobCancelStatus(isEarlyStopped)); return Promise.resolve(); @@ -409,7 +383,7 @@ class LocalTrainingService implements TrainingService { { key: 'NNI_SYS_DIR', value: trialJobDetail.workingDirectory }, { key: 'NNI_TRIAL_JOB_ID', value: trialJobDetail.id }, { key: 'NNI_OUTPUT_DIR', value: trialJobDetail.workingDirectory }, - { key: 'NNI_TRIAL_SEQ_ID', value: trialJobDetail.sequenceId.toString() }, + { key: 'NNI_TRIAL_SEQ_ID', value: trialJobDetail.form.sequenceId.toString() }, { key: 'MULTI_PHASE', value: this.isMultiPhase.toString() } ]; if (gpuNum !== undefined) { @@ -562,7 +536,7 @@ class LocalTrainingService implements TrainingService { const scriptName: string = getScriptName('run'); await fs.promises.writeFile(path.join(trialJobDetail.workingDirectory, scriptName), runScriptContent.join(getNewLine()), { encoding: 'utf8', mode: 0o777 }); - await this.writeParameterFile(trialJobDetail.workingDirectory, (trialJobDetail.form).hyperParameters); + await this.writeParameterFile(trialJobDetail.workingDirectory, trialJobDetail.form.hyperParameters); const trialJobProcess: cp.ChildProcess = runScript(path.join(trialJobDetail.workingDirectory, scriptName)); this.setTrialJobStatus(trialJobDetail, 'RUNNING'); trialJobDetail.startTime = Date.now(); @@ -589,60 +563,10 @@ class LocalTrainingService implements TrainingService { this.jobStreamMap.set(trialJobDetail.id, stream); } - private async runHostJob(form: HostJobApplicationForm): Promise { - const jobId: string = uniqueString(5); - const workDir: string = path.join(this.rootDir, 'hostjobs', jobId); - await cpp.exec(`mkdir -p ${workDir}`); - const wrappedCmd: string = `cd ${workDir} && ${form.cmd}>stdout 2>stderr`; - this.log.debug(`runHostJob: command: ${wrappedCmd}`); - const process: cp.ChildProcess = cp.exec(wrappedCmd); - const jobDetail: LocalTrialJobDetail = { - id: jobId, - status: 'RUNNING', - submitTime: Date.now(), - workingDirectory: workDir, - form: form, - sequenceId: this.generateSequenceId(), - pid: process.pid - }; - this.jobMap.set(jobId, jobDetail); - this.log.debug(`runHostJob: return: ${JSON.stringify(jobDetail)} `); - - return jobDetail; - } - - private async getHostJob(jobId: string): Promise { - const jobDetail: LocalTrialJobDetail | undefined = this.jobMap.get(jobId); - if (jobDetail === undefined) { - throw new NNIError(NNIErrorNames.NOT_FOUND, `Host Job not found: ${jobId}`); - } - try { - await cpp.exec(`kill -0 ${jobDetail.pid}`); - - return jobDetail; - } catch (error) { - if (error instanceof Error) { - this.log.debug(`getHostJob: error: ${error.message}`); - this.jobMap.delete(jobId); - throw new NNIError(NNIErrorNames.NOT_FOUND, `Host Job not found: ${error.message}`); - } else { - throw error; - } - } - } - private async writeParameterFile(directory: string, hyperParameters: HyperParameters): Promise { const filepath: string = path.join(directory, generateParamFileName(hyperParameters)); await fs.promises.writeFile(filepath, hyperParameters.value, { encoding: 'utf8' }); } - - private generateSequenceId(): number { - if (this.trialSequenceId === -1) { - this.trialSequenceId = getInitTrialSequenceId(); - } - - return this.trialSequenceId++; - } } export { LocalTrainingService }; diff --git a/src/nni_manager/training_service/pai/paiData.ts b/src/nni_manager/training_service/pai/paiData.ts index 8ac4b77ed1..d10be902e1 100644 --- a/src/nni_manager/training_service/pai/paiData.ts +++ b/src/nni_manager/training_service/pai/paiData.ts @@ -19,7 +19,7 @@ 'use strict'; -import { JobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../common/trainingService'; +import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../common/trainingService'; /** * PAI trial job detail @@ -34,20 +34,18 @@ export class PAITrialJobDetail implements TrialJobDetail { public tags?: string[]; public url?: string; public workingDirectory: string; - public form: JobApplicationForm; - public sequenceId: number; + public form: TrialJobApplicationForm; public hdfsLogPath: string; public isEarlyStopped?: boolean; constructor(id: string, status: TrialJobStatus, paiJobName : string, - submitTime: number, workingDirectory: string, form: JobApplicationForm, sequenceId: number, hdfsLogPath: string) { + submitTime: number, workingDirectory: string, form: TrialJobApplicationForm, hdfsLogPath: string) { this.id = id; this.status = status; this.paiJobName = paiJobName; this.submitTime = submitTime; this.workingDirectory = workingDirectory; this.form = form; - this.sequenceId = sequenceId; this.tags = []; this.hdfsLogPath = hdfsLogPath; } diff --git a/src/nni_manager/training_service/pai/paiTrainingService.ts b/src/nni_manager/training_service/pai/paiTrainingService.ts index ff742f0fc0..4e949e7708 100644 --- a/src/nni_manager/training_service/pai/paiTrainingService.ts +++ b/src/nni_manager/training_service/pai/paiTrainingService.ts @@ -30,10 +30,10 @@ import { EventEmitter } from 'events'; import { Deferred } from 'ts-deferred'; import { String } from 'typescript-string-operations'; import { MethodNotImplementedError } from '../../common/errors'; -import { getExperimentId, getInitTrialSequenceId } from '../../common/experimentStartupInfo'; +import { getExperimentId } from '../../common/experimentStartupInfo'; import { getLogger, Logger } from '../../common/log'; import { - HyperParameters, JobApplicationForm, NNIManagerIpConfig, TrainingService, + HyperParameters, NNIManagerIpConfig, TrainingService, TrialJobApplicationForm, TrialJobDetail, TrialJobMetric } from '../../common/trainingService'; import { delay, generateParamFileName, @@ -70,7 +70,6 @@ class PAITrainingService implements TrainingService { private readonly paiTokenUpdateInterval: number; private readonly experimentId! : string; private readonly paiJobCollector : PAIJobInfoCollector; - private nextTrialSequenceId: number; private paiRestServerPort?: number; private nniManagerIpConfig?: NNIManagerIpConfig; private copyExpCodeDirPromise?: Promise; @@ -90,7 +89,6 @@ class PAITrainingService implements TrainingService { this.expRootDir = path.join('/nni', 'experiments', getExperimentId()); this.experimentId = getExperimentId(); this.paiJobCollector = new PAIJobInfoCollector(this.trialJobsMap); - this.nextTrialSequenceId = -1; this.paiTokenUpdateInterval = 7200000; //2hours this.logCollection = 'none'; this.log.info('Construct OpenPAI training service.'); @@ -112,9 +110,7 @@ class PAITrainingService implements TrainingService { const jobs: TrialJobDetail[] = []; for (const [key, value] of this.trialJobsMap) { - if (value.form.jobType === 'TRIAL') { - jobs.push(await this.getTrialJob(key)); - } + jobs.push(await this.getTrialJob(key)); } return Promise.resolve(jobs); @@ -142,7 +138,7 @@ class PAITrainingService implements TrainingService { this.metricsEmitter.off('metric', listener); } - public async submitTrialJob(form: JobApplicationForm): Promise { + public async submitTrialJob(form: TrialJobApplicationForm): Promise { if (this.paiClusterConfig === undefined) { throw new Error(`paiClusterConfig not initialized!`); } @@ -151,7 +147,6 @@ class PAITrainingService implements TrainingService { this.log.info(`submitTrialJob: form: ${JSON.stringify(form)}`); const trialJobId: string = uniqueString(5); - const trialSequenceId: number = this.generateSequenceId(); //TODO: use HDFS working folder instead const trialWorkingFolder: string = path.join(this.expRootDir, 'trials', trialJobId); const paiJobName: string = `nni_exp_${this.experimentId}_trial_${trialJobId}`; @@ -171,7 +166,6 @@ class PAITrainingService implements TrainingService { Date.now(), trialWorkingFolder, form, - trialSequenceId, hdfsLogPath); this.trialJobsMap.set(trialJobId, trialJobDetail); @@ -181,16 +175,12 @@ class PAITrainingService implements TrainingService { return deferred.promise; } - public async updateTrialJob(trialJobId: string, form: JobApplicationForm): Promise { + public async updateTrialJob(trialJobId: string, form: TrialJobApplicationForm): Promise { const trialJobDetail: undefined | TrialJobDetail = this.trialJobsMap.get(trialJobId); if (trialJobDetail === undefined) { throw new Error(`updateTrialJob failed: ${trialJobId} not found`); } - if (form.jobType === 'TRIAL') { - await this.writeParameterFile(trialJobId, (form).hyperParameters); - } else { - throw new Error(`updateTrialJob failed: jobType ${form.jobType} not supported.`); - } + await this.writeParameterFile(trialJobId, form.hyperParameters); return trialJobDetail; } @@ -397,11 +387,10 @@ class PAITrainingService implements TrainingService { await fs.promises.writeFile(path.join(trialLocalTempFolder, 'install_nni.sh'), runScriptContent, { encoding: 'utf8' }); // Write file content ( parameter.cfg ) to local tmp folders - const trialForm : TrialJobApplicationForm = (trialJobDetail.form); - if (trialForm !== undefined) { + if (trialJobDetail.form !== undefined) { await fs.promises.writeFile( - path.join(trialLocalTempFolder, generateParamFileName(trialForm.hyperParameters)), - trialForm.hyperParameters.value, { encoding: 'utf8' } + path.join(trialLocalTempFolder, generateParamFileName(trialJobDetail.form.hyperParameters)), + trialJobDetail.form.hyperParameters.value, { encoding: 'utf8' } ); } const hdfsCodeDir: string = HDFSClientUtility.getHdfsTrialWorkDir(this.paiClusterConfig.userName, trialJobId); @@ -416,7 +405,7 @@ class PAITrainingService implements TrainingService { `$PWD/${trialJobId}/nnioutput`, trialJobId, this.experimentId, - trialJobDetail.sequenceId, + trialJobDetail.form.sequenceId, this.isMultiPhase, this.paiTrialConfig.command, nniManagerIp, @@ -507,14 +496,6 @@ class PAITrainingService implements TrainingService { return deferred.promise; } - private generateSequenceId(): number { - if (this.nextTrialSequenceId === -1) { - this.nextTrialSequenceId = getInitTrialSequenceId(); - } - - return this.nextTrialSequenceId++; - } - private async statusCheckingLoop(): Promise { while (!this.stopping) { try { diff --git a/src/nni_manager/training_service/remote_machine/remoteMachineData.ts b/src/nni_manager/training_service/remote_machine/remoteMachineData.ts index 02ad95d627..7c2ba2f0e5 100644 --- a/src/nni_manager/training_service/remote_machine/remoteMachineData.ts +++ b/src/nni_manager/training_service/remote_machine/remoteMachineData.ts @@ -22,7 +22,7 @@ import * as fs from 'fs'; import { Client, ConnectConfig } from 'ssh2'; import { Deferred } from 'ts-deferred'; -import { JobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../common/trainingService'; +import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../common/trainingService'; import { GPUInfo, GPUSummary } from '../common/gpuData'; /** @@ -82,20 +82,18 @@ export class RemoteMachineTrialJobDetail implements TrialJobDetail { public tags?: string[]; public url?: string; public workingDirectory: string; - public form: JobApplicationForm; - public sequenceId: number; + public form: TrialJobApplicationForm; public rmMeta?: RemoteMachineMeta; public isEarlyStopped?: boolean; public gpuIndices: GPUInfo[]; constructor(id: string, status: TrialJobStatus, submitTime: number, - workingDirectory: string, form: JobApplicationForm, sequenceId: number) { + workingDirectory: string, form: TrialJobApplicationForm) { this.id = id; this.status = status; this.submitTime = submitTime; this.workingDirectory = workingDirectory; this.form = form; - this.sequenceId = sequenceId; this.tags = []; this.gpuIndices = []; } diff --git a/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts b/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts index 35631f1ce9..4733df6809 100644 --- a/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts +++ b/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts @@ -30,11 +30,11 @@ import { Deferred } from 'ts-deferred'; import { String } from 'typescript-string-operations'; import * as component from '../../common/component'; import { NNIError, NNIErrorNames } from '../../common/errors'; -import { getExperimentId, getInitTrialSequenceId } from '../../common/experimentStartupInfo'; +import { getExperimentId } from '../../common/experimentStartupInfo'; import { getLogger, Logger } from '../../common/log'; import { ObservableTimer } from '../../common/observableTimer'; import { - HostJobApplicationForm, HyperParameters, JobApplicationForm, NNIManagerIpConfig, TrainingService, TrialJobApplicationForm, + HyperParameters, NNIManagerIpConfig, TrainingService, TrialJobApplicationForm, TrialJobDetail, TrialJobMetric } from '../../common/trainingService'; import { @@ -172,9 +172,7 @@ class RemoteMachineTrainingService implements TrainingService { const deferred: Deferred = new Deferred(); for (const [key, value] of this.trialJobsMap) { - if (value.form.jobType === 'TRIAL') { - jobs.push(await this.getTrialJob(key)); - } + jobs.push(await this.getTrialJob(key)); } deferred.resolve(jobs); @@ -228,33 +226,26 @@ class RemoteMachineTrainingService implements TrainingService { * @param form trial job description form */ // tslint:disable-next-line:informative-docs - public async submitTrialJob(form: JobApplicationForm): Promise { + public async submitTrialJob(form: TrialJobApplicationForm): Promise { if (this.trialConfig === undefined) { throw new Error('trial config is not initialized'); } - if (form.jobType === 'HOST') { - return this.runHostJob(form); - } else if (form.jobType === 'TRIAL') { - // Generate trial job id(random) - const trialJobId: string = uniqueString(5); - const trialWorkingFolder: string = unixPathJoin(this.remoteExpRootDir, 'trials', trialJobId); + // Generate trial job id(random) + const trialJobId: string = uniqueString(5); + const trialWorkingFolder: string = unixPathJoin(this.remoteExpRootDir, 'trials', trialJobId); - const trialJobDetail: RemoteMachineTrialJobDetail = new RemoteMachineTrialJobDetail( - trialJobId, - 'WAITING', - Date.now(), - trialWorkingFolder, - form, - this.generateSequenceId() - ); - this.jobQueue.push(trialJobId); - this.trialJobsMap.set(trialJobId, trialJobDetail); + const trialJobDetail: RemoteMachineTrialJobDetail = new RemoteMachineTrialJobDetail( + trialJobId, + 'WAITING', + Date.now(), + trialWorkingFolder, + form + ); + this.jobQueue.push(trialJobId); + this.trialJobsMap.set(trialJobId, trialJobDetail); - return Promise.resolve(trialJobDetail); - } else { - return Promise.reject(new Error(`Job form not supported: ${JSON.stringify(form)}, jobType should be HOST or TRIAL.`)); - } + return Promise.resolve(trialJobDetail); } /** @@ -262,20 +253,16 @@ class RemoteMachineTrainingService implements TrainingService { * @param trialJobId trial job id * @param form job application form */ - public async updateTrialJob(trialJobId: string, form: JobApplicationForm): Promise { + public async updateTrialJob(trialJobId: string, form: TrialJobApplicationForm): Promise { const trialJobDetail: undefined | TrialJobDetail = this.trialJobsMap.get(trialJobId); if (trialJobDetail === undefined) { throw new Error(`updateTrialJob failed: ${trialJobId} not found`); } - if (form.jobType === 'TRIAL') { - const rmMeta: RemoteMachineMeta | undefined = (trialJobDetail).rmMeta; - if (rmMeta !== undefined) { - await this.writeParameterFile(trialJobId, (form).hyperParameters, rmMeta); - } else { - throw new Error(`updateTrialJob failed: ${trialJobId} rmMeta not found`); - } + const rmMeta: RemoteMachineMeta | undefined = (trialJobDetail).rmMeta; + if (rmMeta !== undefined) { + await this.writeParameterFile(trialJobId, form.hyperParameters, rmMeta); } else { - throw new Error(`updateTrialJob failed: jobType ${form.jobType} not supported.`); + throw new Error(`updateTrialJob failed: ${trialJobId} rmMeta not found`); } return trialJobDetail; @@ -558,7 +545,7 @@ class RemoteMachineTrainingService implements TrainingService { await this.allocateSSHClientForTrial(trialJobDetail); await this.launchTrialOnScheduledMachine( - trialJobId, trialWorkingFolder, trialJobDetail.form, rmScheduleInfo); + trialJobId, trialWorkingFolder, trialJobDetail.form, rmScheduleInfo); trialJobDetail.status = 'RUNNING'; trialJobDetail.url = `file://${rmScheduleInfo.rmMeta.ip}:${trialWorkingFolder}`; @@ -628,7 +615,7 @@ class RemoteMachineTrainingService implements TrainingService { trialWorkingFolder, trialJobId, getExperimentId(), - trialJobDetail.sequenceId.toString(), + trialJobDetail.form.sequenceId.toString(), this.isMultiPhase, unixPathJoin(trialWorkingFolder, '.nni', 'jobpid'), command, @@ -657,38 +644,6 @@ class RemoteMachineTrainingService implements TrainingService { SSHClientUtility.remoteExeCommand(`bash ${unixPathJoin(trialWorkingFolder, 'run.sh')}`, sshClient); } - private async runHostJob(form: HostJobApplicationForm): Promise { - const rmMeta: RemoteMachineMeta = this.getRmMetaByHost(form.host); - const sshClientManager: SSHClientManager | undefined = this.machineSSHClientMap.get(rmMeta); - if (sshClientManager === undefined) { - throw new Error('sshClient not found.'); - } - const sshClient: Client = sshClientManager.getFirstSSHClient(); - const jobId: string = uniqueString(5); - const localDir: string = path.join(this.expRootDir, 'hostjobs-local', jobId); - const remoteDir: string = this.getHostJobRemoteDir(jobId); - await cpp.exec(`mkdir -p ${localDir}`); - await SSHClientUtility.remoteExeCommand(`mkdir -p ${remoteDir}`, sshClient); - const runScriptContent: string = String.Format( - HOST_JOB_SHELL_FORMAT, remoteDir, path.join(remoteDir, 'jobpid'), form.cmd, path.join(remoteDir, 'code') - ); - await fs.promises.writeFile(path.join(localDir, 'run.sh'), runScriptContent, { encoding: 'utf8' }); - await SSHClientUtility.copyFileToRemote( - path.join(localDir, 'run.sh'), unixPathJoin(remoteDir, 'run.sh'), sshClient); - // tslint:disable-next-line: no-floating-promises - SSHClientUtility.remoteExeCommand(`bash ${unixPathJoin(remoteDir, 'run.sh')}`, sshClient); - - const jobDetail: RemoteMachineTrialJobDetail = new RemoteMachineTrialJobDetail( - jobId, 'RUNNING', Date.now(), remoteDir, form, this.generateSequenceId() - ); - jobDetail.rmMeta = rmMeta; - jobDetail.startTime = Date.now(); - this.trialJobsMap.set(jobId, jobDetail); - this.log.debug(`runHostJob: return: ${JSON.stringify(jobDetail)} `); - - return jobDetail; - } - private getRmMetaByHost(host: string): RemoteMachineMeta { for (const [rmMeta, client] of this.machineSSHClientMap.entries()) { if (rmMeta.ip === host) { @@ -765,13 +720,7 @@ class RemoteMachineTrainingService implements TrainingService { } let jobpidPath: string; - if (trialJobDetail.form.jobType === 'TRIAL') { - jobpidPath = unixPathJoin(trialJobDetail.workingDirectory, '.nni', 'jobpid'); - } else if (trialJobDetail.form.jobType === 'HOST') { - jobpidPath = unixPathJoin(this.getHostJobRemoteDir(jobId), 'jobpid'); - } else { - throw new Error(`Job type not supported: ${trialJobDetail.form.jobType}`); - } + jobpidPath = unixPathJoin(trialJobDetail.workingDirectory, '.nni', 'jobpid'); return jobpidPath; } @@ -791,14 +740,6 @@ class RemoteMachineTrainingService implements TrainingService { await SSHClientUtility.copyFileToRemote(localFilepath, unixPathJoin(trialWorkingFolder, fileName), sshClient); } - - private generateSequenceId(): number { - if (this.trialSequenceId === -1) { - this.trialSequenceId = getInitTrialSequenceId(); - } - - return this.trialSequenceId++; - } } export { RemoteMachineTrainingService }; diff --git a/src/nni_manager/training_service/test/localTrainingService.test.ts b/src/nni_manager/training_service/test/localTrainingService.test.ts index d0d6daf3ad..2d95bb80cc 100644 --- a/src/nni_manager/training_service/test/localTrainingService.test.ts +++ b/src/nni_manager/training_service/test/localTrainingService.test.ts @@ -76,7 +76,7 @@ describe('Unit Test for LocalTrainingService', () => { // submit job const form: TrialJobApplicationForm = { - jobType: 'TRIAL', + sequenceId: 0, hyperParameters: { value: 'mock hyperparameters', index: 0 @@ -95,7 +95,7 @@ describe('Unit Test for LocalTrainingService', () => { // submit job const form: TrialJobApplicationForm = { - jobType: 'TRIAL', + sequenceId: 0, hyperParameters: { value: 'mock hyperparameters', index: 0 @@ -121,4 +121,4 @@ describe('Unit Test for LocalTrainingService', () => { it('Test multiphaseSupported', () => { chai.expect(localTrainingService.isMultiPhaseJobSupported).to.be.equals(true) }) -}); \ No newline at end of file +}); diff --git a/src/nni_manager/training_service/test/paiTrainingService.test.ts b/src/nni_manager/training_service/test/paiTrainingService.test.ts index e55fe9e483..2a52362d42 100644 --- a/src/nni_manager/training_service/test/paiTrainingService.test.ts +++ b/src/nni_manager/training_service/test/paiTrainingService.test.ts @@ -24,6 +24,7 @@ import * as chaiAsPromised from 'chai-as-promised'; import * as fs from 'fs'; import * as tmp from 'tmp'; import * as component from '../../common/component'; +import { TrialJobApplicationForm } from '../../common/trainingService'; import { cleanupUnitTest, prepareUnitTest } from '../../common/utils'; import { TrialConfigMetadataKey } from '../common/trialConfigMetadataKey'; import { PAITrainingService } from '../pai/paiTrainingService'; @@ -84,12 +85,16 @@ describe('Unit Test for PAITrainingService', () => { console.log(`paiCluster is ${paiCluster}`) await paiTrainingService.setClusterMetadata(TrialConfigMetadataKey.PAI_CLUSTER_CONFIG, paiCluster); await paiTrainingService.setClusterMetadata(TrialConfigMetadataKey.TRIAL_CONFIG, paiTrialConfig); + const form: TrialJobApplicationForm = { + sequenceId: 0, + hyperParameters: { value: '', index: 0 } + }; try { - const trialDetail = await paiTrainingService.submitTrialJob({jobType : 'TRIAL'}); + const trialDetail = await paiTrainingService.submitTrialJob(form); chai.expect(trialDetail.status).to.be.equals('WAITING'); } catch(error) { console.log('Submit job failed:' + error); chai.assert(error) } }); -}); \ No newline at end of file +}); diff --git a/src/nni_manager/training_service/test/remoteMachineTrainingService.test.ts b/src/nni_manager/training_service/test/remoteMachineTrainingService.test.ts index 7509ea2ade..f8f5025e49 100644 --- a/src/nni_manager/training_service/test/remoteMachineTrainingService.test.ts +++ b/src/nni_manager/training_service/test/remoteMachineTrainingService.test.ts @@ -99,11 +99,11 @@ describe('Unit Test for RemoteMachineTrainingService', () => { await remoteMachineTrainingService.setClusterMetadata( TrialConfigMetadataKey.TRIAL_CONFIG, `{"command":"sleep 1h && echo ","codeDir":"${localCodeDir}","gpuNum":1}`); const form: TrialJobApplicationForm = { - jobType: 'TRIAL', - hyperParameters: { - value: 'mock hyperparameters', - index: 0 - } + sequenceId: 0, + hyperParameters: { + value: 'mock hyperparameters', + index: 0 + } }; const trialJob = await remoteMachineTrainingService.submitTrialJob(form); @@ -137,7 +137,7 @@ describe('Unit Test for RemoteMachineTrainingService', () => { // submit job const form: TrialJobApplicationForm = { - jobType: 'TRIAL', + sequenceId: 0, hyperParameters: { value: 'mock hyperparameters', index: 0 diff --git a/src/webui/src/App.tsx b/src/webui/src/App.tsx index c3b31d422a..b55129d6e3 100644 --- a/src/webui/src/App.tsx +++ b/src/webui/src/App.tsx @@ -1,104 +1,111 @@ import * as React from 'react'; import { Row, Col } from 'antd'; -import axios from 'axios'; -import { COLUMN, MANAGER_IP } from './static/const'; +import { COLUMN } from './static/const'; +import { EXPERIMENT, TRIALS } from './static/datamodel'; import './App.css'; import SlideBar from './components/SlideBar'; interface AppState { - interval: number; - whichPageToFresh: string; - columnList: Array; - concurrency: number; + interval: number; + columnList: Array; + experimentUpdateBroadcast: number; + trialsUpdateBroadcast: number; } class App extends React.Component<{}, AppState> { - public _isMounted: boolean; - constructor(props: {}) { - super(props); - this.state = { - interval: 10, // sendons - whichPageToFresh: '', - columnList: COLUMN, - concurrency: 1 - }; - } + private timerId: number | null; - changeInterval = (interval: number) => { - if (this._isMounted === true) { - this.setState(() => ({ interval: interval })); + constructor(props: {}) { + super(props); + this.state = { + interval: 10, // sendons + columnList: COLUMN, + experimentUpdateBroadcast: 0, + trialsUpdateBroadcast: 0, + }; } - } - changeFresh = (fresh: string) => { - // interval * 1000 - if (this._isMounted === true) { - this.setState(() => ({ whichPageToFresh: fresh })); + async componentDidMount() { + await Promise.all([ EXPERIMENT.init(), TRIALS.init() ]); + this.setState(state => ({ experimentUpdateBroadcast: state.experimentUpdateBroadcast + 1 })); + this.setState(state => ({ trialsUpdateBroadcast: state.trialsUpdateBroadcast + 1 })); + this.timerId = window.setTimeout(this.refresh, this.state.interval * 1000); } - } - changeColumn = (columnList: Array) => { - if (this._isMounted === true) { - this.setState(() => ({ columnList: columnList })); + changeInterval = (interval: number) => { + this.setState({ interval: interval }); + if (this.timerId === null && interval !== 0) { + window.setTimeout(this.refresh); + } else if (this.timerId !== null && interval === 0) { + window.clearTimeout(this.timerId); + } } - } - changeConcurrency = (val: number) => { - if (this._isMounted === true) { - this.setState(() => ({ concurrency: val })); + // TODO: use local storage + changeColumn = (columnList: Array) => { + this.setState({ columnList: columnList }); } - } - getConcurrency = () => { - axios(`${MANAGER_IP}/experiment`, { - method: 'GET' - }) - .then(res => { - if (res.status === 200) { - const params = res.data.params; - if (this._isMounted) { - this.setState(() => ({ concurrency: params.trialConcurrency })); - } + render() { + const { interval, columnList, experimentUpdateBroadcast, trialsUpdateBroadcast } = this.state; + if (experimentUpdateBroadcast === 0 || trialsUpdateBroadcast === 0) { + return null; // TODO: render a loading page + } + const reactPropsChildren = React.Children.map(this.props.children, child => + React.cloneElement( + // tslint:disable-next-line:no-any + child as React.ReactElement, { + interval, + columnList, changeColumn: this.changeColumn, + experimentUpdateBroadcast, + trialsUpdateBroadcast, + }) + ); + return ( + + + + + + + + + + + {reactPropsChildren} + + + + ); + } + + private refresh = async () => { + const [ experimentUpdated, trialsUpdated ] = await Promise.all([ EXPERIMENT.update(), TRIALS.update() ]); + if (experimentUpdated) { + this.setState(state => ({ experimentUpdateBroadcast: state.experimentUpdateBroadcast + 1 })); + } + if (trialsUpdated) { + this.setState(state => ({ trialsUpdateBroadcast: state.trialsUpdateBroadcast + 1 })); } - }); - } - componentDidMount() { - this._isMounted = true; - this.getConcurrency(); - } + if ([ 'DONE', 'ERROR', 'STOPPED' ].includes(EXPERIMENT.status)) { + // experiment finished, refresh once more to ensure consistency + if (this.state.interval > 0) { + this.setState({ interval: 0 }); + this.lastRefresh(); + } - componentWillUnmount() { - this._isMounted = false; - } - render() { - const { interval, whichPageToFresh, columnList, concurrency } = this.state; - const reactPropsChildren = React.Children.map(this.props.children, child => - React.cloneElement( - // tslint:disable-next-line:no-any - child as React.ReactElement, { - interval, whichPageToFresh, - columnList, changeColumn: this.changeColumn, - concurrency, changeConcurrency: this.changeConcurrency - }) - ); - return ( - - - - - - - - - - - {reactPropsChildren} - - - - ); - } + } else if (this.state.interval !== 0) { + this.timerId = window.setTimeout(this.refresh, this.state.interval * 1000); + } + } + + private async lastRefresh() { + await EXPERIMENT.update(); + await TRIALS.update(true); + this.setState(state => ({ experimentUpdateBroadcast: state.experimentUpdateBroadcast + 1 })); + this.setState(state => ({ trialsUpdateBroadcast: state.trialsUpdateBroadcast + 1 })); + } } export default App; diff --git a/src/webui/src/components/Modal/Compare.tsx b/src/webui/src/components/Modal/Compare.tsx index 40599c3e6b..39719767c5 100644 --- a/src/webui/src/components/Modal/Compare.tsx +++ b/src/webui/src/components/Modal/Compare.tsx @@ -2,12 +2,13 @@ import * as React from 'react'; import { Row, Modal } from 'antd'; import ReactEcharts from 'echarts-for-react'; import IntermediateVal from '../public-child/IntermediateVal'; +import { TRIALS } from '../../static/datamodel'; import '../../static/style/compare.scss'; -import { TableObj, Intermedia, TooltipForIntermediate } from 'src/static/interface'; +import { TableRecord, Intermedia, TooltipForIntermediate } from 'src/static/interface'; // the modal of trial compare interface CompareProps { - compareRows: Array; + compareRows: Array; visible: boolean; cancelFunc: () => void; } @@ -105,11 +106,12 @@ class Compare extends React.Component { // render table column --- initColumn = () => { - const { compareRows } = this.props; const idList: Array = []; const sequenceIdList: Array = []; const durationList: Array = []; + const compareRows = this.props.compareRows.map(tableRecord => TRIALS.getTrial(tableRecord.id)); + const parameterList: Array = []; let parameterKeys: Array = []; if (compareRows.length !== 0) { @@ -147,7 +149,7 @@ class Compare extends React.Component { const temp = compareRows[index]; return ( - + ); })} diff --git a/src/webui/src/components/Modal/ExperimentDrawer.tsx b/src/webui/src/components/Modal/ExperimentDrawer.tsx index 2433eec439..2541811bf3 100644 --- a/src/webui/src/components/Modal/ExperimentDrawer.tsx +++ b/src/webui/src/components/Modal/ExperimentDrawer.tsx @@ -58,7 +58,7 @@ class ExperimentDrawer extends React.Component { trialMessage: trialMessagesArr }; if (this._isCompareMount === true) { - this.setState(() => ({ experiment: JSON.stringify(result, null, 4) })); + this.setState({ experiment: JSON.stringify(result, null, 4) }); } } })); diff --git a/src/webui/src/components/Modal/LogDrawer.tsx b/src/webui/src/components/Modal/LogDrawer.tsx index 89a9e90798..bd4abcdb18 100644 --- a/src/webui/src/components/Modal/LogDrawer.tsx +++ b/src/webui/src/components/Modal/LogDrawer.tsx @@ -51,13 +51,13 @@ class LogDrawer extends React.Component { setDispatcher = (value: string) => { if (this._isLogDrawer === true) { - this.setState(() => ({ isLoadispatcher: false, dispatcherLogStr: value })); + this.setState({ isLoadispatcher: false, dispatcherLogStr: value }); } } setNNImanager = (val: string) => { if (this._isLogDrawer === true) { - this.setState(() => ({ isLoading: false, nniManagerLogStr: val })); + this.setState({ isLoading: false, nniManagerLogStr: val }); } } diff --git a/src/webui/src/components/Overview.tsx b/src/webui/src/components/Overview.tsx index 7366b45511..22d52e5458 100644 --- a/src/webui/src/components/Overview.tsx +++ b/src/webui/src/components/Overview.tsx @@ -1,16 +1,14 @@ import * as React from 'react'; -import axios from 'axios'; import { Row, Col } from 'antd'; -import { MANAGER_IP } from '../static/const'; -import { Experiment, TableObj, Parameters, TrialNumber } from '../static/interface'; -import { getFinal } from '../static/function'; +import { EXPERIMENT, TRIALS } from '../static/datamodel'; +import { Trial } from '../static/model/trial'; import SuccessTable from './overview/SuccessTable'; import Title1 from './overview/Title1'; import Progressed from './overview/Progress'; import Accuracy from './overview/Accuracy'; import SearchSpace from './overview/SearchSpace'; import BasicInfo from './overview/BasicInfo'; -import TrialPro from './overview/TrialProfile'; +import TrialInfo from './overview/TrialProfile'; require('../static/style/overview.scss'); require('../static/style/logPath.scss'); @@ -18,486 +16,70 @@ require('../static/style/accuracy.css'); require('../static/style/table.scss'); require('../static/style/overviewTitle.scss'); -interface OverviewState { - tableData: Array; - experimentAPI: object; - searchSpace: object; - status: string; - errorStr: string; - trialProfile: Experiment; - option: object; - noData: string; - accuracyData: object; - bestAccuracy: number; - accNodata: string; - trialNumber: TrialNumber; - isTop10: boolean; - titleMaxbgcolor?: string; - titleMinbgcolor?: string; - // trial stdout is content(false) or link(true) - isLogCollection: boolean; - isMultiPhase: boolean; +interface OverviewProps { + experimentUpdateBroadcast: number; + trialsUpdateBroadcast: number; } -interface OverviewProps { - interval: number; // user select - whichPageToFresh: string; - concurrency: number; - changeConcurrency: (val: number) => void; +interface OverviewState { + trialConcurrency: number; + metricGraphMode: 'max' | 'min'; } class Overview extends React.Component { - - public _isMounted = false; - public intervalID = 0; - public intervalProfile = 1; - constructor(props: OverviewProps) { super(props); this.state = { - searchSpace: {}, - experimentAPI: {}, - status: '', - errorStr: '', - trialProfile: { - id: '', - author: '', - experName: '', - runConcurren: 1, - maxDuration: 0, - execDuration: 0, - MaxTrialNum: 0, - startTime: 0, - tuner: {}, - trainingServicePlatform: '' - }, - tableData: [], - option: {}, - noData: '', - // accuracy - accuracyData: {}, - accNodata: '', - bestAccuracy: 0, - trialNumber: { - succTrial: 0, - failTrial: 0, - stopTrial: 0, - waitTrial: 0, - runTrial: 0, - unknowTrial: 0, - totalCurrentTrial: 0 - }, - isTop10: true, - isLogCollection: false, - isMultiPhase: false + trialConcurrency: EXPERIMENT.trialConcurrency, + metricGraphMode: (EXPERIMENT.optimizeMode === 'minimize' ? 'min' : 'max'), }; } - // show session - showSessionPro = () => { - axios(`${MANAGER_IP}/experiment`, { - method: 'GET' - }) - .then(res => { - if (res.status === 200) { - let sessionData = res.data; - let trialPro = []; - const tempara = sessionData.params; - const trainingPlatform = tempara.trainingServicePlatform; - // assessor clusterMeteData - const clusterMetaData = tempara.clusterMetaData; - const endTimenum = sessionData.endTime; - const assessor = tempara.assessor; - const advisor = tempara.advisor; - let optimizeMode = 'other'; - if (tempara.tuner !== undefined) { - if (tempara.tuner.classArgs !== undefined) { - if (tempara.tuner.classArgs.optimize_mode !== undefined) { - optimizeMode = tempara.tuner.classArgs.optimize_mode; - } - } - } - // default logCollection is true - const logCollection = tempara.logCollection; - let expLogCollection: boolean = false; - const isMultiy: boolean = tempara.multiPhase !== undefined - ? tempara.multiPhase : false; - if (optimizeMode !== undefined) { - if (optimizeMode === 'minimize') { - if (this._isMounted) { - this.setState({ - isTop10: false, - titleMinbgcolor: '#999' - }); - } - } else { - if (this._isMounted) { - this.setState({ - isTop10: true, - titleMaxbgcolor: '#999' - }); - } - } - } - if (logCollection !== undefined && logCollection !== 'none') { - expLogCollection = true; - } - trialPro.push({ - id: sessionData.id, - author: tempara.authorName, - revision: sessionData.revision, - experName: tempara.experimentName, - runConcurren: tempara.trialConcurrency, - logDir: sessionData.logDir ? sessionData.logDir : 'undefined', - maxDuration: tempara.maxExecDuration, - execDuration: sessionData.execDuration, - MaxTrialNum: tempara.maxTrialNum, - startTime: sessionData.startTime, - endTime: endTimenum ? endTimenum : undefined, - trainingServicePlatform: trainingPlatform, - tuner: tempara.tuner, - assessor: assessor ? assessor : undefined, - advisor: advisor ? advisor : undefined, - clusterMetaData: clusterMetaData ? clusterMetaData : undefined, - logCollection: logCollection - }); - // search space format loguniform max and min - const temp = tempara.searchSpace; - const searchSpace = temp !== undefined - ? JSON.parse(temp) : {}; - Object.keys(searchSpace).map(item => { - const key = searchSpace[item]._type; - let value = searchSpace[item]._value; - switch (key) { - case 'quniform': - case 'qnormal': - case 'qlognormal': - searchSpace[item]._value = [value[0], value[1]]; - break; - - default: - - } - }); - if (this._isMounted) { - this.setState({ - experimentAPI: res.data, - trialProfile: trialPro[0], - searchSpace: searchSpace, - isLogCollection: expLogCollection, - isMultiPhase: isMultiy - }); - } - } - }); - this.checkStatus(); - - } - - checkStatus = () => { - axios(`${MANAGER_IP}/check-status`, { - method: 'GET' - }) - .then(res => { - if (res.status === 200) { - const errors = res.data.errors; - if (errors.length !== 0) { - if (this._isMounted) { - this.setState({ - status: res.data.status, - errorStr: res.data.errors[0] - }); - } - } else { - if (this._isMounted) { - this.setState({ - status: res.data.status, - }); - } - } - } - }); - } - - showTrials = () => { - this.isOffInterval(); - axios(`${MANAGER_IP}/trial-jobs`, { - method: 'GET' - }) - .then(res => { - if (res.status === 200) { - const tableData = res.data; - const topTableData: Array = []; - const profile: TrialNumber = { - succTrial: 0, - failTrial: 0, - stopTrial: 0, - waitTrial: 0, - runTrial: 0, - unknowTrial: 0, - totalCurrentTrial: 0 - }; - // currently totoal number - profile.totalCurrentTrial = tableData.length; - Object.keys(tableData).map(item => { - switch (tableData[item].status) { - case 'WAITING': - profile.waitTrial += 1; - break; - - case 'UNKNOWN': - profile.unknowTrial += 1; - break; - - case 'FAILED': - profile.failTrial += 1; - break; - - case 'RUNNING': - profile.runTrial += 1; - break; - - case 'USER_CANCELED': - case 'SYS_CANCELED': - case 'EARLY_STOPPED': - profile.stopTrial += 1; - break; - case 'SUCCEEDED': - profile.succTrial += 1; - const desJobDetail: Parameters = { - parameters: {}, - intermediate: [], - multiProgress: 1 - }; - const duration = (tableData[item].endTime - tableData[item].startTime) / 1000; - const acc = getFinal(tableData[item].finalMetricData); - // if hyperparameters is undefine, show error message, else, show parameters value - const tempara = tableData[item].hyperParameters; - if (tempara !== undefined) { - const tempLength = tempara.length; - const parameters = JSON.parse(tempara[tempLength - 1]).parameters; - desJobDetail.multiProgress = tempara.length; - if (typeof parameters === 'string') { - desJobDetail.parameters = JSON.parse(parameters); - } else { - desJobDetail.parameters = parameters; - } - } else { - desJobDetail.parameters = { error: 'This trial\'s parameters are not available.' }; - } - if (tableData[item].logPath !== undefined) { - desJobDetail.logPath = tableData[item].logPath; - } - topTableData.push({ - key: topTableData.length, - sequenceId: tableData[item].sequenceId, - id: tableData[item].id, - duration: duration, - status: tableData[item].status, - acc: acc, - description: desJobDetail - }); - break; - default: - } - }); - // choose top10 or lowest10 - const { isTop10 } = this.state; - if (isTop10 === true) { - topTableData.sort((a: TableObj, b: TableObj) => { - if (a.acc !== undefined && b.acc !== undefined) { - return JSON.parse(b.acc.default) - JSON.parse(a.acc.default); - } else { - return NaN; - } - }); - } else { - topTableData.sort((a: TableObj, b: TableObj) => { - if (a.acc !== undefined && b.acc !== undefined) { - return JSON.parse(a.acc.default) - JSON.parse(b.acc.default); - } else { - return NaN; - } - }); - } - topTableData.length = Math.min(10, topTableData.length); - let bestDefaultMetric = 0; - if (topTableData[0] !== undefined) { - if (topTableData[0].acc !== undefined) { - bestDefaultMetric = JSON.parse(topTableData[0].acc.default); - } - } - if (this._isMounted) { - this.setState({ - tableData: topTableData, - trialNumber: profile, - bestAccuracy: bestDefaultMetric - }); - } - this.checkStatus(); - // draw accuracy - this.drawPointGraph(); - } - }); - } - - // trial accuracy graph Default Metric - drawPointGraph = () => { - - const { tableData } = this.state; - const sourcePoint = JSON.parse(JSON.stringify(tableData)); - sourcePoint.sort((a: TableObj, b: TableObj) => { - if (a.sequenceId !== undefined && b.sequenceId !== undefined) { - return a.sequenceId - b.sequenceId; - } else { - return NaN; - } - }); - const accarr: Array = []; - const indexarr: Array = []; - Object.keys(sourcePoint).map(item => { - const items = sourcePoint[item]; - if (items.acc !== undefined) { - accarr.push(items.acc.default); - indexarr.push(items.sequenceId); - } - }); - const accOption = { - // support max show 0.0000000 - grid: { - left: 67, - right: 40 - }, - tooltip: { - trigger: 'item' - }, - xAxis: { - name: 'Trial', - type: 'category', - data: indexarr - }, - yAxis: { - name: 'Default metric', - type: 'value', - scale: true, - data: accarr - }, - series: [{ - symbolSize: 6, - type: 'scatter', - data: accarr - }] - }; - if (this._isMounted) { - this.setState({ accuracyData: accOption }, () => { - if (accarr.length === 0) { - this.setState({ - accNodata: 'No data' - }); - } else { - this.setState({ - accNodata: '' - }); - } - }); - } - } - clickMaxTop = (event: React.SyntheticEvent) => { event.stopPropagation(); // #999 panel active bgcolor; #b3b3b3 as usual - this.setState(() => ({ isTop10: true, titleMaxbgcolor: '#999', titleMinbgcolor: '#b3b3b3' })); - this.showTrials(); + this.setState({ metricGraphMode: 'max' }); } clickMinTop = (event: React.SyntheticEvent) => { event.stopPropagation(); - this.setState(() => ({ isTop10: false, titleMaxbgcolor: '#b3b3b3', titleMinbgcolor: '#999' })); - this.showTrials(); - } - - isOffInterval = () => { - const { status } = this.state; - const { interval } = this.props; - if (status === 'DONE' || status === 'ERROR' || status === 'STOPPED' || - interval === 0 - ) { - window.clearInterval(this.intervalID); - window.clearInterval(this.intervalProfile); - return; - } + this.setState({ metricGraphMode: 'min' }); } - componentWillReceiveProps(nextProps: OverviewProps) { - const { interval, whichPageToFresh } = nextProps; - window.clearInterval(this.intervalID); - window.clearInterval(this.intervalProfile); - if (whichPageToFresh.includes('/oview')) { - this.showTrials(); - this.showSessionPro(); - } - if (interval !== 0) { - this.intervalID = window.setInterval(this.showTrials, interval * 1000); - this.intervalProfile = window.setInterval(this.showSessionPro, interval * 1000); - } + changeConcurrency = (val: number) => { + this.setState({ trialConcurrency: val }); } - componentDidMount() { - this._isMounted = true; - const { interval } = this.props; - this.showTrials(); - this.showSessionPro(); - if (interval !== 0) { - this.intervalID = window.setInterval(this.showTrials, interval * 1000); - this.intervalProfile = window.setInterval(this.showSessionPro, interval * 1000); - } - } + render() { + const { trialConcurrency, metricGraphMode } = this.state; + const { experimentUpdateBroadcast } = this.props; - componentWillUnmount() { - this._isMounted = false; - window.clearInterval(this.intervalID); - window.clearInterval(this.intervalProfile); - } + const searchSpace = this.convertSearchSpace(); - render() { + const bestTrials = this.findBestTrials(); + const bestAccuracy = bestTrials.length > 0 ? bestTrials[0].accuracy! : NaN; + const accuracyGraphData = this.generateAccuracyGraph(bestTrials); + const noDataMessage = bestTrials.length > 0 ? '' : 'No data'; - const { - trialProfile, searchSpace, tableData, accuracyData, - accNodata, status, errorStr, trialNumber, bestAccuracy, isMultiPhase, - titleMaxbgcolor, titleMinbgcolor, isLogCollection, experimentAPI - } = this.state; - const { concurrency } = this.props; - trialProfile.runConcurren = concurrency; - Object.keys(experimentAPI).map(item => { - if (item === 'params') { - const temp = experimentAPI[item]; - Object.keys(temp).map(index => { - if (index === 'trialConcurrency') { - temp[index] = concurrency; - } - }); - } - }); + const titleMaxbgcolor = (metricGraphMode === 'max' ? '#999' : '#b3b3b3'); + const titleMinbgcolor = (metricGraphMode === 'min' ? '#999' : '#b3b3b3'); return (
{/* status and experiment block */} - + {/* status graph */} {/* experiment parameters search space tuner assessor... */} @@ -512,7 +94,10 @@ class Overview extends React.Component { {/* the scroll bar all the trial profile in the searchSpace div*/}
- +
@@ -541,24 +126,79 @@ class Overview extends React.Component { - + trial.info.id)}/>
); } + + private convertSearchSpace(): object { + const searchSpace = Object.assign({}, EXPERIMENT.searchSpace); + Object.keys(searchSpace).map(item => { + const key = searchSpace[item]._type; + let value = searchSpace[item]._value; + switch (key) { + case 'quniform': + case 'qnormal': + case 'qlognormal': + searchSpace[item]._value = [value[0], value[1]]; + break; + default: + } + }); + return searchSpace; + } + + private findBestTrials(): Trial[] { + let bestTrials = TRIALS.sort(); + if (this.state.metricGraphMode === 'max') { + bestTrials.reverse().splice(10); + } else { + bestTrials.splice(10); + } + return bestTrials; + } + + private generateAccuracyGraph(bestTrials: Trial[]): object { + const xSequence = bestTrials.map(trial => trial.sequenceId); + const ySequence = bestTrials.map(trial => trial.accuracy); + + return { + // support max show 0.0000000 + grid: { + left: 67, + right: 40 + }, + tooltip: { + trigger: 'item' + }, + xAxis: { + name: 'Trial', + type: 'category', + data: xSequence + }, + yAxis: { + name: 'Default metric', + type: 'value', + scale: true, + data: ySequence + }, + series: [{ + symbolSize: 6, + type: 'scatter', + data: ySequence + }] + }; + } } + export default Overview; diff --git a/src/webui/src/components/SlideBar.tsx b/src/webui/src/components/SlideBar.tsx index 36d80ba00d..1200fb8a7d 100644 --- a/src/webui/src/components/SlideBar.tsx +++ b/src/webui/src/components/SlideBar.tsx @@ -26,7 +26,6 @@ interface SliderState { interface SliderProps extends FormComponentProps { changeInterval: (value: number) => void; - changeFresh: (value: string) => void; } interface EventPer { @@ -35,7 +34,6 @@ interface EventPer { class SlideBar extends React.Component { - public _isMounted = false; public divMenu: HTMLDivElement | null; public selectHTML: Select | null; @@ -57,32 +55,26 @@ class SlideBar extends React.Component { method: 'GET' }) .then(res => { - if (res.status === 200 && this._isMounted) { + if (res.status === 200) { this.setState({ version: res.data }); } }); } handleMenuClick = (e: EventPer) => { - if (this._isMounted) { this.setState({ menuVisible: false }); } + this.setState({ menuVisible: false }); switch (e.key) { // to see & download experiment parameters case '1': - if (this._isMounted === true) { - this.setState(() => ({ isvisibleExperimentDrawer: true })); - } + this.setState({ isvisibleExperimentDrawer: true }); break; // to see & download nnimanager log case '2': - if (this._isMounted === true) { - this.setState(() => ({ activeKey: 'nnimanager', isvisibleLogDrawer: true })); - } + this.setState({ activeKey: 'nnimanager', isvisibleLogDrawer: true }); break; // to see & download dispatcher log case '3': - if (this._isMounted === true) { - this.setState(() => ({ isvisibleLogDrawer: true, activeKey: 'dispatcher' })); - } + this.setState({ isvisibleLogDrawer: true, activeKey: 'dispatcher' }); break; case 'close': case '10': @@ -96,13 +88,10 @@ class SlideBar extends React.Component { } handleVisibleChange = (flag: boolean) => { - if (this._isMounted === true) { - this.setState({ menuVisible: flag }); - } + this.setState({ menuVisible: flag }); } getInterval = (value: string) => { - if (value === 'close') { this.props.changeInterval(0); } else { @@ -203,13 +192,9 @@ class SlideBar extends React.Component { fresh = (event: React.SyntheticEvent) => { event.preventDefault(); event.stopPropagation(); - if (this._isMounted) { - this.setState({ isdisabledFresh: true }, () => { - const whichPage = window.location.pathname; - this.props.changeFresh(whichPage); - setTimeout(() => { this.setState(() => ({ isdisabledFresh: false })); }, 1000); - }); - } + this.setState({ isdisabledFresh: true }, () => { + setTimeout(() => { this.setState({ isdisabledFresh: false }); }, 1000); + }); } desktopHTML = () => { @@ -330,27 +315,18 @@ class SlideBar extends React.Component { } // close log drawer (nnimanager.dispatcher) closeLogDrawer = () => { - if (this._isMounted === true) { - this.setState(() => ({ isvisibleLogDrawer: false, activeKey: '' })); - } + this.setState({ isvisibleLogDrawer: false, activeKey: '' }); } // close download experiment parameters drawer closeExpDrawer = () => { - if (this._isMounted === true) { - this.setState(() => ({ isvisibleExperimentDrawer: false })); - } + this.setState({ isvisibleExperimentDrawer: false }); } componentDidMount() { - this._isMounted = true; this.getNNIversion(); } - componentWillUnmount() { - this._isMounted = false; - } - render() { const mobile = ({this.mobileHTML()}); const tablet = ({this.tabeltHTML()}); @@ -376,4 +352,4 @@ class SlideBar extends React.Component { } } -export default Form.create()(SlideBar); \ No newline at end of file +export default Form.create()(SlideBar); diff --git a/src/webui/src/components/TrialsDetail.tsx b/src/webui/src/components/TrialsDetail.tsx index 93bb03cec7..9ea12ca154 100644 --- a/src/webui/src/components/TrialsDetail.tsx +++ b/src/webui/src/components/TrialsDetail.tsx @@ -1,10 +1,8 @@ import * as React from 'react'; -import axios from 'axios'; -import { MANAGER_IP } from '../static/const'; import { Row, Col, Tabs, Select, Button, Icon } from 'antd'; const Option = Select.Option; -import { TableObj, Parameters, ExperimentInfo } from '../static/interface'; -import { getFinal } from '../static/function'; +import { EXPERIMENT, TRIALS } from '../static/datamodel'; +import { Trial } from '../static/model/trial'; import DefaultPoint from './trial-detail/DefaultMetricPoint'; import Duration from './trial-detail/Duration'; import Title1 from './overview/Title1'; @@ -16,37 +14,22 @@ import '../static/style/trialsDetail.scss'; import '../static/style/search.scss'; interface TrialDetailState { - accSource: object; - accNodata: string; - tableListSource: Array; - searchResultSource: Array; - isHasSearch: boolean; - experimentLogCollection: boolean; - entriesTable: number; // table components val - entriesInSelect: string; - searchSpace: string; - isMultiPhase: boolean; + tablePageSize: number; // table components val whichGraph: string; - hyperCounts: number; // user click the hyper-parameter counts - durationCounts: number; - intermediateCounts: number; - experimentInfo: ExperimentInfo; - searchFilter: string; - searchPlaceHolder: string; + searchType: string; + searchFilter: (trial: Trial) => boolean; } interface TrialsDetailProps { - interval: number; - whichPageToFresh: string; columnList: Array; changeColumn: (val: Array) => void; + experimentUpdateBroacast: number; + trialsUpdateBroadcast: number; } class TrialsDetail extends React.Component { - public _isMounted = false; public interAccuracy = 0; - public interTableList = 1; public interAllTableList = 2; public tableList: TableList | null; @@ -73,335 +56,68 @@ class TrialsDetail extends React.Component constructor(props: TrialsDetailProps) { super(props); - this.state = { - accSource: {}, - accNodata: '', - tableListSource: [], - searchResultSource: [], - experimentLogCollection: false, - entriesTable: 20, - entriesInSelect: '20', - searchSpace: '', + tablePageSize: 20, whichGraph: '1', - isHasSearch: false, - isMultiPhase: false, - hyperCounts: 0, - durationCounts: 0, - intermediateCounts: 0, - experimentInfo: { - platform: '', - optimizeMode: 'maximize' - }, - searchFilter: 'id', - searchPlaceHolder: 'Search by id' + searchType: 'id', + searchFilter: trial => true, }; } - getDetailSource = () => { - this.isOffIntervals(); - axios - .all([ - axios.get(`${MANAGER_IP}/trial-jobs`), - axios.get(`${MANAGER_IP}/metric-data`) - ]) - .then(axios.spread((res, res1) => { - if (res.status === 200 && res1.status === 200) { - const trialJobs = res.data; - const metricSource = res1.data; - const trialTable: Array = []; - Object.keys(trialJobs).map(item => { - let desc: Parameters = { - parameters: {}, - intermediate: [], - multiProgress: 1 - }; - let duration = 0; - const id = trialJobs[item].id !== undefined - ? trialJobs[item].id - : ''; - const status = trialJobs[item].status !== undefined - ? trialJobs[item].status - : ''; - const begin = trialJobs[item].startTime; - const end = trialJobs[item].endTime; - if (begin) { - if (end) { - duration = (end - begin) / 1000; - } else { - duration = (new Date().getTime() - begin) / 1000; - } - } - const tempHyper = trialJobs[item].hyperParameters; - if (tempHyper !== undefined) { - const getPara = JSON.parse(tempHyper[tempHyper.length - 1]).parameters; - desc.multiProgress = tempHyper.length; - if (typeof getPara === 'string') { - desc.parameters = JSON.parse(getPara); - } else { - desc.parameters = getPara; - } - } else { - desc.parameters = { error: 'This trial\'s parameters are not available.' }; - } - if (trialJobs[item].logPath !== undefined) { - desc.logPath = trialJobs[item].logPath; - } - - const acc = getFinal(trialJobs[item].finalMetricData); - // deal with intermediate result list - const mediate: Array = []; - Object.keys(metricSource).map(key => { - const items = metricSource[key]; - if (items.trialJobId === id) { - // succeed trial, last intermediate result is final result - // final result format may be object - if (typeof JSON.parse(items.data) === 'object') { - mediate.push(JSON.parse(items.data).default); - } else { - mediate.push(JSON.parse(items.data)); - } - } - }); - desc.intermediate = mediate; - trialTable.push({ - key: trialTable.length, - sequenceId: trialJobs[item].sequenceId, - id: id, - status: status, - duration: duration, - acc: acc, - description: desc, - startTime: begin, - endTime: (end !== undefined) ? end : undefined - }); - }); - // update search data result - const { searchResultSource, entriesInSelect } = this.state; - if (searchResultSource.length !== 0) { - const temp: Array = []; - Object.keys(searchResultSource).map(index => { - temp.push(searchResultSource[index].id); - }); - const searchResultList: Array = []; - for (let i = 0; i < temp.length; i++) { - Object.keys(trialTable).map(key => { - const item = trialTable[key]; - if (item.id === temp[i]) { - searchResultList.push(item); - } - }); - } - - if (this._isMounted) { - this.setState(() => ({ - searchResultSource: searchResultList - })); - } - } - if (this._isMounted) { - this.setState(() => ({ tableListSource: trialTable })); - } - if (entriesInSelect === 'all' && this._isMounted) { - this.setState(() => ({ - entriesTable: trialTable.length - })); - } - } - })); - } - // search a trial by trial No. & trial id searchTrial = (event: React.ChangeEvent) => { const targetValue = event.target.value; - if (targetValue === '' || targetValue === ' ') { - const { tableListSource } = this.state; - if (this._isMounted) { - this.setState(() => ({ - isHasSearch: false, - tableListSource: tableListSource, - })); - } - } else { - const { tableListSource, searchFilter } = this.state; - const searchResultList: Array = []; - Object.keys(tableListSource).map(key => { - const item = tableListSource[key]; - switch (searchFilter) { - case 'id': - if (item.id.toUpperCase().includes(targetValue.toUpperCase())) { - searchResultList.push(item); - } - break; - case 'Trial No.': - if (item.sequenceId.toString() === targetValue) { - searchResultList.push(item); - } - break; - case 'status': - if (item.status.toUpperCase().includes(targetValue.toUpperCase())) { - searchResultList.push(item); - } - break; - case 'parameters': - const strParameters = JSON.stringify(item.description.parameters, null, 4); - if (strParameters.includes(targetValue)) { - searchResultList.push(item); - } - break; - default: - } - }); - if (this._isMounted) { - this.setState(() => ({ - searchResultSource: searchResultList, - isHasSearch: true - })); - } - } - } - - // close timer - isOffIntervals = () => { - const { interval } = this.props; - if (interval === 0) { - window.clearInterval(this.interTableList); + let filter = (trial: Trial) => true; + if (!targetValue.trim()) { + this.setState({ searchFilter: filter }); return; - } else { - axios(`${MANAGER_IP}/check-status`, { - method: 'GET' - }) - .then(res => { - if (res.status === 200 && this._isMounted) { - const expStatus = res.data.status; - if (expStatus === 'DONE' || expStatus === 'ERROR' || expStatus === 'STOPPED') { - window.clearInterval(this.interTableList); - return; - } - } - }); } + switch (this.state.searchType) { + case 'id': + filter = trial => trial.info.id.toUpperCase().includes(targetValue.toUpperCase()); + break; + case 'Trial No.': + filter = trial => trial.info.sequenceId.toString() === targetValue; + break; + case 'status': + filter = trial => trial.info.status.toUpperCase().includes(targetValue.toUpperCase()); + break; + case 'parameters': + // TODO: support filters like `x: 2` (instead of `"x": 2`) + filter = trial => JSON.stringify(trial.info.hyperParameters, null, 4).includes(targetValue); + break; + default: + alert(`Unexpected search filter ${this.state.searchType}`); + } + this.setState({ searchFilter: filter }); } - handleEntriesSelect = (value: string) => { - // user select isn't 'all' - if (value !== 'all') { - if (this._isMounted) { - this.setState(() => ({ entriesTable: parseInt(value, 10) })); - } - } else { - const { tableListSource } = this.state; - if (this._isMounted) { - this.setState(() => ({ - entriesInSelect: 'all', - entriesTable: tableListSource.length - })); - } - } + handleTablePageSizeSelect = (value: string) => { + this.setState({ tablePageSize: value === 'all' ? -1 : parseInt(value, 10) }); } handleWhichTabs = (activeKey: string) => { - // const which = JSON.parse(activeKey); - if (this._isMounted) { - this.setState(() => ({ whichGraph: activeKey })); - } + this.setState({ whichGraph: activeKey }); } test = () => { alert('TableList component was not properly initialized.'); } - getSearchFilter = (value: string) => { + updateSearchFilterType = (value: string) => { // clear input value and re-render table if (this.searchInput !== null) { this.searchInput.value = ''; - if (this._isMounted === true) { - this.setState(() => ({ isHasSearch: false })); - } - } - if (this._isMounted === true) { - this.setState(() => ({ searchFilter: value, searchPlaceHolder: `Search by ${value}` })); - } - } - - // get and set logCollection val - checkExperimentPlatform = () => { - axios(`${MANAGER_IP}/experiment`, { - method: 'GET' - }) - .then(res => { - if (res.status === 200) { - const trainingPlatform: string = res.data.params.trainingServicePlatform !== undefined - ? - res.data.params.trainingServicePlatform - : - ''; - // default logCollection is true - const logCollection = res.data.params.logCollection; - let expLogCollection: boolean = false; - const isMultiy: boolean = res.data.params.multiPhase !== undefined - ? res.data.params.multiPhase : false; - const tuner = res.data.params.tuner; - // I'll set optimize is maximize if user not set optimize - let optimize: string = 'maximize'; - if (tuner !== undefined) { - if (tuner.classArgs !== undefined) { - if (tuner.classArgs.optimize_mode !== undefined) { - if (tuner.classArgs.optimize_mode === 'minimize') { - optimize = 'minimize'; - } - } - } - } - if (logCollection !== undefined && logCollection !== 'none') { - expLogCollection = true; - } - if (this._isMounted) { - this.setState({ - experimentInfo: { platform: trainingPlatform, optimizeMode: optimize }, - searchSpace: res.data.params.searchSpace, - experimentLogCollection: expLogCollection, - isMultiPhase: isMultiy - }); - } - } - }); - } - - componentWillReceiveProps(nextProps: TrialsDetailProps) { - const { interval, whichPageToFresh } = nextProps; - window.clearInterval(this.interTableList); - if (interval !== 0) { - this.interTableList = window.setInterval(this.getDetailSource, interval * 1000); - } - if (whichPageToFresh.includes('/detail')) { - this.getDetailSource(); } - } - - componentDidMount() { - - this._isMounted = true; - const { interval } = this.props; - this.getDetailSource(); - this.interTableList = window.setInterval(this.getDetailSource, interval * 1000); - this.checkExperimentPlatform(); - } - - componentWillUnmount() { - this._isMounted = false; - window.clearInterval(this.interTableList); + this.setState({ searchType: value }); } render() { - - const { - tableListSource, searchResultSource, isHasSearch, isMultiPhase, - entriesTable, experimentInfo, searchSpace, experimentLogCollection, - whichGraph, searchPlaceHolder - } = this.state; - const source = isHasSearch ? searchResultSource : tableListSource; + const { tablePageSize, whichGraph } = this.state; const { columnList, changeColumn } = this.props; + const source = TRIALS.filter(this.state.searchFilter); + const trialIds = TRIALS.filter(this.state.searchFilter).map(trial => trial.id); + return (
@@ -409,10 +125,9 @@ class TrialsDetail extends React.Component @@ -420,7 +135,7 @@ class TrialsDetail extends React.Component @@ -440,7 +155,7 @@ class TrialsDetail extends React.Component Show + (this.searchInput) = text} @@ -481,14 +196,11 @@ class TrialsDetail extends React.Component trial.tableRecord)} columnList={columnList} changeColumn={changeColumn} + trialsUpdateBroadcast={this.props.trialsUpdateBroadcast} ref={(tabList) => this.tableList = tabList} />
@@ -496,4 +208,4 @@ class TrialsDetail extends React.Component } } -export default TrialsDetail; \ No newline at end of file +export default TrialsDetail; diff --git a/src/webui/src/components/overview/BasicInfo.tsx b/src/webui/src/components/overview/BasicInfo.tsx index dfddde7a1e..b47fca53f0 100644 --- a/src/webui/src/components/overview/BasicInfo.tsx +++ b/src/webui/src/components/overview/BasicInfo.tsx @@ -1,68 +1,45 @@ +import { Col, Row, Tooltip } from 'antd'; import * as React from 'react'; -import { - Row, Col, - Tooltip -} from 'antd'; -import { Experiment } from '../../static/interface'; +import { EXPERIMENT } from '../../static/datamodel'; +import { formatTimestamp } from '../../static/function'; interface BasicInfoProps { - trialProfile: Experiment; - status: string; + experimentUpdateBroadcast: number; } class BasicInfo extends React.Component { - constructor(props: BasicInfoProps) { super(props); } render() { - const { trialProfile } = this.props; return (

Name

-
{trialProfile.experName}
+
{EXPERIMENT.profile.params.experimentName}

ID

-
{trialProfile.id}
+
{EXPERIMENT.profile.id}

Start time

-
- {new Date(trialProfile.startTime).toLocaleString('en-US')} -
+
{formatTimestamp(EXPERIMENT.profile.startTime)}

End time

-
- { - trialProfile.endTime - ? - new Date(trialProfile.endTime).toLocaleString('en-US') - : - 'none' - } -
+
{formatTimestamp(EXPERIMENT.profile.endTime)}

Log directory

- - {trialProfile.logDir} + + {EXPERIMENT.profile.logDir || 'unknown'}

Training platform

-
- { - trialProfile.trainingServicePlatform - ? - trialProfile.trainingServicePlatform - : - 'none' - } -
+
{EXPERIMENT.profile.params.trainingServicePlatform}
 ); } } -export default BasicInfo; \ No newline at end of file +export default BasicInfo; diff --git a/src/webui/src/components/overview/NumInput.tsx b/src/webui/src/components/overview/NumInput.tsx new file mode 100644 index 0000000000..0c014a3233 --- /dev/null +++ b/src/webui/src/components/overview/NumInput.tsx @@ -0,0 +1,85 @@ +import * as React from 'react'; +import { Button, Row } from 'antd'; + +interface ConcurrencyInputProps { + value: number; + updateValue: (val: string) => void; +} + +interface ConcurrencyInputStates { + editting: boolean; +} + +class ConcurrencyInput extends React.Component { + private input = React.createRef(); + + constructor(props: ConcurrencyInputProps) { + super(props); + this.state = { editting: false }; + } + + save = () => { + if (this.input.current !== null) { + this.props.updateValue(this.input.current.value); + this.setState({ editting: false }); + } + } + + cancel = () => { + this.setState({ editting: false }); + } + + edit = () => { + this.setState({ editting: true }); + } + + render() { + if (this.state.editting) { + return ( + + + + + + ); + } else { + return ( + + + + + ); + } + } +} + +export default ConcurrencyInput; diff --git a/src/webui/src/components/overview/Progress.tsx b/src/webui/src/components/overview/Progress.tsx index 39d6ee3322..88cc6e645b 100644 --- a/src/webui/src/components/overview/Progress.tsx +++ b/src/webui/src/components/overview/Progress.tsx @@ -1,192 +1,99 @@ import * as React from 'react'; -import { Row, Col, Popover, Button, message } from 'antd'; +import { Row, Col, Popover, message } from 'antd'; import axios from 'axios'; -import { MANAGER_IP, CONTROLTYPE } from '../../static/const'; -import { Experiment, TrialNumber } from '../../static/interface'; +import { MANAGER_IP } from '../../static/const'; +import { EXPERIMENT, TRIALS } from '../../static/datamodel'; import { convertTime } from '../../static/function'; +import ConcurrencyInput from './NumInput'; import ProgressBar from './ProgressItem'; import LogDrawer from '../Modal/LogDrawer'; import '../../static/style/progress.scss'; import '../../static/style/probar.scss'; interface ProgressProps { - trialProfile: Experiment; concurrency: number; - trialNumber: TrialNumber; bestAccuracy: number; - status: string; - errors: string; changeConcurrency: (val: number) => void; + experimentUpdateBroadcast: number; } interface ProgressState { - btnName: string; - isEnable: boolean; - userInputVal: string; // get user input - cancelSty: string; isShowLogDrawer: boolean; } class Progressed extends React.Component { - - public conInput: HTMLInputElement | null; - public _isMounted = false; constructor(props: ProgressProps) { super(props); this.state = { - btnName: 'Edit', - isEnable: true, - userInputVal: this.props.trialProfile.runConcurren.toString(), - cancelSty: 'none', isShowLogDrawer: false }; } - editTrialConcurrency = () => { - const { btnName } = this.state; - if (this._isMounted) { - if (btnName === 'Edit') { - // user click edit - this.setState(() => ({ - isEnable: false, - btnName: 'Save', - cancelSty: 'inline-block' - })); - } else { - // user click save button - axios(`${MANAGER_IP}/experiment`, { - method: 'GET' - }) - .then(rese => { - if (rese.status === 200) { - const { userInputVal } = this.state; - const experimentFile = rese.data; - const trialConcurrency = experimentFile.params.trialConcurrency; - if (userInputVal !== undefined) { - if (userInputVal === trialConcurrency.toString() || userInputVal === '0') { - message.destroy(); - message.info( - `trialConcurrency's value is ${trialConcurrency}, you did not modify it`, 2); - } else { - experimentFile.params.trialConcurrency = parseInt(userInputVal, 10); - // rest api, modify trial concurrency value - axios(`${MANAGER_IP}/experiment`, { - method: 'PUT', - headers: { - 'Content-Type': 'application/json;charset=utf-8' - }, - data: experimentFile, - params: { - update_type: CONTROLTYPE[1] - } - }).then(res => { - if (res.status === 200) { - message.destroy(); - message.success(`Update ${CONTROLTYPE[1].toLocaleLowerCase()} - successfully`); - this.props.changeConcurrency(parseInt(userInputVal, 10)); - } - }) - .catch(error => { - if (error.response.status === 500) { - if (error.response.data.error) { - message.error(error.response.data.error); - } else { - message.error( - `Update ${CONTROLTYPE[1].toLocaleLowerCase()} failed`); - } - } - }); - // btn -> edit - this.setState(() => ({ - btnName: 'Edit', - isEnable: true, - cancelSty: 'none' - })); - } - } - } - }); - } - } - } - - cancelFunction = () => { - const { trialProfile } = this.props; - if (this._isMounted) { - this.setState( - () => ({ - btnName: 'Edit', - isEnable: true, - cancelSty: 'none', - })); + editTrialConcurrency = async (userInput: string) => { + if (!userInput.match(/^[1-9]\d*$/)) { + message.error('Please enter a positive integer!', 2); + return; } - if (this.conInput !== null) { - this.conInput.value = trialProfile.runConcurren.toString(); + const newConcurrency = parseInt(userInput, 10); + if (newConcurrency === this.props.concurrency) { + message.info(`Trial concurrency has not changed`, 2); + return; } - } - getUserTrialConcurrency = (event: React.ChangeEvent) => { - const value = event.target.value; - if (value.match(/^[1-9]\d*$/) || value === '') { - this.setState(() => ({ - userInputVal: value - })); - } else { - message.error('Please enter a positive integer!', 2); - if (this.conInput !== null) { - const { trialProfile } = this.props; - this.conInput.value = trialProfile.runConcurren.toString(); + const newProfile = Object.assign({}, EXPERIMENT.profile); + newProfile.params.trialConcurrency = newConcurrency; + + // rest api, modify trial concurrency value + try { + const res = await axios.put(`${MANAGER_IP}/experiment`, newProfile, { + params: { update_type: 'TRIAL_CONCURRENCY' } + }); + if (res.status === 200) { + message.success(`Successfully updated trial concurrency`); + // NOTE: should we do this earlier in favor of poor networks? + this.props.changeConcurrency(newConcurrency); + } + } catch (error) { + if (error.response && error.response.data.error) { + message.error(`Failed to update trial concurrency\n${error.response.data.error}`); + } else if (error.response) { + message.error(`Failed to update trial concurrency\nServer responsed ${error.response.status}`); + } else if (error.message) { + message.error(`Failed to update trial concurrency\n${error.message}`); + } else { + message.error(`Failed to update trial concurrency\nUnknown error`); } } } isShowDrawer = () => { - if (this._isMounted === true) { - this.setState(() => ({ isShowLogDrawer: true })); - } + this.setState({ isShowLogDrawer: true }); } closeDrawer = () => { - if (this._isMounted === true) { - this.setState(() => ({ isShowLogDrawer: false })); - } + this.setState({ isShowLogDrawer: false }); } - componentWillReceiveProps() { - const { trialProfile } = this.props; - if (this.conInput !== null) { - this.conInput.value = trialProfile.runConcurren.toString(); - } - } + render() { + const { bestAccuracy } = this.props; + const { isShowLogDrawer } = this.state; - componentDidMount() { - this._isMounted = true; - } + const count = TRIALS.countStatus(); + const stoppedCount = count.get('USER_CANCELED')! + count.get('SYS_CANCELED')! + count.get('EARLY_STOPPED')!; + const bar2 = count.get('RUNNING')! + count.get('SUCCEEDED')! + count.get('FAILED')! + stoppedCount; - componentWillUnmount() { - this._isMounted = false; - } + const bar2Percent = (bar2 / EXPERIMENT.profile.params.maxTrialNum) * 100; + const percent = (EXPERIMENT.profile.execDuration / EXPERIMENT.profile.params.maxExecDuration) * 100; + const remaining = convertTime(EXPERIMENT.profile.params.maxExecDuration - EXPERIMENT.profile.execDuration); + const maxDuration = convertTime(EXPERIMENT.profile.params.maxExecDuration); + const maxTrialNum = convertTime(EXPERIMENT.profile.params.maxTrialNum); + const execDuration = convertTime(EXPERIMENT.profile.execDuration); - render() { - const { trialProfile, trialNumber, bestAccuracy, status, errors } = this.props; - const { isEnable, btnName, cancelSty, isShowLogDrawer } = this.state; - const bar2 = trialNumber.totalCurrentTrial - trialNumber.waitTrial - trialNumber.unknowTrial; - const bar2Percent = (bar2 / trialProfile.MaxTrialNum) * 100; - const percent = (trialProfile.execDuration / trialProfile.maxDuration) * 100; - const runDuration = convertTime(trialProfile.execDuration); - const temp = trialProfile.maxDuration - trialProfile.execDuration; - let remaining; let errorContent; - if (temp < 0) { - remaining = '0'; - } else { - remaining = convertTime(temp); - } - if (errors !== '') { + if (EXPERIMENT.error) { errorContent = (
- {errors} + {EXPERIMENT.error}
); @@ -196,9 +103,9 @@ class Progressed extends React.Component {

Status

- {status} + {EXPERIMENT.status} { - status === 'ERROR' + EXPERIMENT.status === 'ERROR' ? {

Best metric

-
{bestAccuracy.toFixed(6)}
+
{isNaN(bestAccuracy) ? 'N/A' : bestAccuracy.toFixed(6)}

Spent

-
{convertTime(trialProfile.execDuration)}
+
{execDuration}
 @@ -247,54 +154,32 @@ class Progressed extends React.Component { {/* modify concurrency */}

Concurrency

- - this.conInput = input} - /> - - - +

Running

-
{trialNumber.runTrial}
+
{count.get('RUNNING')}

Succeeded

-
{trialNumber.succTrial}
+
{count.get('SUCCEEDED')}

Stopped

-
{trialNumber.stopTrial}
+
{stoppedCount}

Failed

-
{trialNumber.failTrial}
+
{count.get('FAILED')}
 @@ -309,4 +194,4 @@ class Progressed extends React.Component { } } -export default Progressed; \ No newline at end of file +export default Progressed; diff --git a/src/webui/src/components/overview/SuccessTable.tsx b/src/webui/src/components/overview/SuccessTable.tsx index 18d7ee55a6..1e99d2416c 100644 --- a/src/webui/src/components/overview/SuccessTable.tsx +++ b/src/webui/src/components/overview/SuccessTable.tsx @@ -2,131 +2,83 @@ import * as React from 'react'; import { Table } from 'antd'; import OpenRow from '../public-child/OpenRow'; import DefaultMetric from '../public-child/DefaultMetrc'; -import { TableObj } from '../../static/interface'; +import { TRIALS } from '../../static/datamodel'; +import { TableRecord } from '../../static/interface'; import { convertDuration } from '../../static/function'; import '../../static/style/tableStatus.css'; import '../../static/style/openRow.scss'; interface SuccessTableProps { - tableSource: Array; - trainingPlatform: string; - logCollection: boolean; - multiphase: boolean; + trialIds: string[]; } -class SuccessTable extends React.Component { - - public _isMounted = false; +function openRow(record: TableRecord) { + return ( + + ); +} +class SuccessTable extends React.Component { constructor(props: SuccessTableProps) { super(props); - - } - - openRow = (record: TableObj) => { - const { trainingPlatform, logCollection, multiphase } = this.props; - return ( - - ); - } - - componentDidMount() { - this._isMounted = true; - } - - componentWillUnmount() { - this._isMounted = false; } render() { - const { tableSource } = this.props; - - let bgColor = ''; - const columns = [{ - title: 'Trial No.', - dataIndex: 'sequenceId', - key: 'sequenceId', - width: 140, - className: 'tableHead' - }, { - title: 'ID', - dataIndex: 'id', - key: 'id', - width: 60, - className: 'tableHead leftTitle', - render: (text: string, record: TableObj) => { - return ( -
{record.id}
- ); - }, - }, { - title: 'Duration', - dataIndex: 'duration', - key: 'duration', - width: 140, - sorter: (a: TableObj, b: TableObj) => (a.duration as number) - (b.duration as number), - render: (text: string, record: TableObj) => { - let duration; - if (record.duration !== undefined) { - // duration is nagative number(-1) & 0-1 - if (record.duration > 0 && record.duration < 1 || record.duration < 0) { - duration = `${record.duration}s`; - } else { - duration = convertDuration(record.duration); - } - } else { - duration = 0; + const columns = [ + { + title: 'Trial No.', + dataIndex: 'sequenceId', + width: 140, + className: 'tableHead' + }, { + title: 'ID', + dataIndex: 'id', + width: 60, + className: 'tableHead leftTitle', + render: (text: string, record: TableRecord) => { + return ( +
{record.id}
+ ); + }, + }, { + title: 'Duration', + dataIndex: 'duration', + width: 140, + render: (text: string, record: TableRecord) => { + return ( +
{convertDuration(record.duration)}
+ ); + }, + }, { + title: 'Status', + dataIndex: 'status', + width: 150, + className: 'tableStatus', + render: (text: string, record: TableRecord) => { + return ( +
{record.status}
+ ); } - return ( -
{duration}
- ); - }, - }, { - title: 'Status', - dataIndex: 'status', - key: 'status', - width: 150, - className: 'tableStatus', - render: (text: string, record: TableObj) => { - bgColor = record.status; - return ( -
- {record.status} -
- ); - } - }, { - title: 'Default metric', - dataIndex: 'acc', - key: 'acc', - sorter: (a: TableObj, b: TableObj) => { - if (a.acc !== undefined && b.acc !== undefined) { - return JSON.parse(a.acc.default) - JSON.parse(b.acc.default); - } else { - return NaN; + }, { + title: 'Default metric', + dataIndex: 'accuracy', + render: (text: string, record: TableRecord) => { + return ( + + ); } - }, - render: (text: string, record: TableObj) => { - return ( - - ); } - }]; + ]; return (
- + ); } } diff --git a/src/webui/src/components/overview/TrialProfile.tsx b/src/webui/src/components/overview/TrialProfile.tsx index 4820fa7ccd..dd55dd0868 100644 --- a/src/webui/src/components/overview/TrialProfile.tsx +++ b/src/webui/src/components/overview/TrialProfile.tsx @@ -1,9 +1,11 @@ import * as React from 'react'; import MonacoEditor from 'react-monaco-editor'; import { MONACO } from '../../static/const'; +import { EXPERIMENT } from '../../static/datamodel'; interface TrialInfoProps { - experiment: object; + experimentUpdateBroadcast: number; + concurrency: number; } class TrialInfo extends React.Component { @@ -12,32 +14,21 @@ class TrialInfo extends React.Component { super(props); } - componentWillReceiveProps(nextProps: TrialInfoProps) { - const experiments = nextProps.experiment; - Object.keys(experiments).map(key => { - switch (key) { - case 'id': - case 'logDir': - case 'startTime': - case 'endTime': - experiments[key] = undefined; - break; - case 'params': - const params = experiments[key]; - Object.keys(params).map(item => { - if (item === 'experimentName' || item === 'searchSpace' - || item === 'trainingServicePlatform') { - params[item] = undefined; - } - }); - break; - default: + render() { + const blacklist = [ + 'id', 'logDir', 'startTime', 'endTime', + 'experimentName', 'searchSpace', 'trainingServicePlatform' + ]; + // tslint:disable-next-line:no-any + const filter = (key: string, val: any) => { + if (key === 'trialConcurrency') { + return this.props.concurrency; } - }); - } + return blacklist.includes(key) ? undefined : val; + }; + const profile = JSON.stringify(EXPERIMENT.profile, filter, 2); - render() { - const { experiment } = this.props; + // FIXME: highlight not working? return (
{ height="361" language="json" theme="vs-light" - value={JSON.stringify(experiment, null, 2)} + value={profile} options={MONACO} />
diff --git a/src/webui/src/components/public-child/DefaultMetrc.tsx b/src/webui/src/components/public-child/DefaultMetrc.tsx index d31b288c63..02a8894419 100644 --- a/src/webui/src/components/public-child/DefaultMetrc.tsx +++ b/src/webui/src/components/public-child/DefaultMetrc.tsx @@ -1,45 +1,22 @@ import * as React from 'react'; -import { TableObj } from '../../static/interface'; +import { TRIALS } from '../../static/datamodel'; +import { formatAccuracy } from '../../static/function'; interface DefaultMetricProps { - record: TableObj; + trialId: string; } class DefaultMetric extends React.Component { - constructor(props: DefaultMetricProps) { super(props); - } render() { - const { record } = this.props; - let accuracy; - if (record.acc !== undefined) { - accuracy = record.acc.default; - } - let wei = 0; - if (accuracy !== undefined) { - if (accuracy.toString().indexOf('.') !== -1) { - wei = accuracy.toString().length - accuracy.toString().indexOf('.') - 1; - } - } + const accuracy = TRIALS.getTrial(this.props.trialId).accuracy; return ( -
- { - record.acc !== undefined && record.acc.default !== undefined - ? - wei > 6 - ? - JSON.parse(record.acc.default).toFixed(6) - : - record.acc.default - : - '--' - } -
+
{accuracy !== undefined ? formatAccuracy(accuracy) : '--'}
); } } -export default DefaultMetric; \ No newline at end of file +export default DefaultMetric; diff --git a/src/webui/src/components/public-child/IntermediateVal.tsx b/src/webui/src/components/public-child/IntermediateVal.tsx index b5bd015843..2382ed4fbc 100644 --- a/src/webui/src/components/public-child/IntermediateVal.tsx +++ b/src/webui/src/components/public-child/IntermediateVal.tsx @@ -1,46 +1,18 @@ import * as React from 'react'; -import { TableObj } from '../../static/interface'; +import { TRIALS } from '../../static/datamodel'; interface IntermediateValProps { - record: TableObj; + trialId: string; } class IntermediateVal extends React.Component { - constructor(props: IntermediateValProps) { super(props); - } render() { - const { record } = this.props; - const interArr = record.description.intermediate; - let lastVal; - let wei = 0; - if (interArr !== undefined) { - lastVal = interArr[interArr.length - 1]; - } - let result: string = JSON.stringify(lastVal); - if (lastVal !== undefined) { - if (lastVal.toString().indexOf('.') !== -1) { - wei = lastVal.toString().length - lastVal.toString().indexOf('.') - 1; - if (wei > 6) { - result = `${lastVal.toFixed(6)}`; - } - } - // some trial haven't final result - if (record.acc !== undefined) { - if (record.acc.default !== undefined) { - result = `${result} (FINAL)`; - } - } else { - result = `${result} (LATEST)`; - } - } else { - result = '--'; - } return ( -
{result}
+
{TRIALS.getTrial(this.props.trialId).formatLatestAccuracy()}
); } } diff --git a/src/webui/src/components/public-child/OpenRow.tsx b/src/webui/src/components/public-child/OpenRow.tsx index dcaf22a6d8..f6d842c9fc 100644 --- a/src/webui/src/components/public-child/OpenRow.tsx +++ b/src/webui/src/components/public-child/OpenRow.tsx @@ -2,7 +2,8 @@ import * as React from 'react'; import * as copy from 'copy-to-clipboard'; import PaiTrialLog from '../public-child/PaiTrialLog'; import TrialLog from '../public-child/TrialLog'; -import { TableObj } from '../../static/interface'; +import { EXPERIMENT, TRIALS } from '../../static/datamodel'; +import { Trial } from '../../static/model/trial'; import { Row, Tabs, Button, message, Modal } from 'antd'; import { MANAGER_IP } from '../../static/const'; import '../../static/style/overview.scss'; @@ -11,10 +12,7 @@ import JSONTree from 'react-json-tree'; const TabPane = Tabs.TabPane; interface OpenRowProps { - trainingPlatform: string; - record: TableObj; - logCollection: boolean; - multiphase: boolean; + trialId: string; } interface OpenRowState { @@ -24,7 +22,6 @@ interface OpenRowState { class OpenRow extends React.Component { - public _isMounted: boolean; constructor(props: OpenRowProps) { super(props); this.state = { @@ -33,20 +30,16 @@ class OpenRow extends React.Component { }; } - showFormatModal = (record: TableObj) => { + showFormatModal = (trial: Trial) => { // get copy parameters - const params = JSON.stringify(record.description.parameters, null, 4); + const params = JSON.stringify(trial.info.hyperParameters, null, 4); // open modal with format string - if (this._isMounted === true) { - this.setState(() => ({ isShowFormatModal: true, formatStr: params })); - } + this.setState({ isShowFormatModal: true, formatStr: params }); } hideFormatModal = () => { // close modal, destroy state format string data - if (this._isMounted === true) { - this.setState(() => ({ isShowFormatModal: false, formatStr: '' })); - } + this.setState({ isShowFormatModal: false, formatStr: '' }); } copyParams = () => { @@ -62,68 +55,54 @@ class OpenRow extends React.Component { this.hideFormatModal(); } - componentDidMount() { - this._isMounted = true; - } - - componentWillUnmount() { - this._isMounted = false; - } render() { - const { trainingPlatform, record, logCollection, multiphase } = this.props; const { isShowFormatModal, formatStr } = this.state; + const trialId = this.props.trialId; + const trial = TRIALS.getTrial(trialId); let isClick = false; - let isHasParameters = true; - if (record.description.parameters.error) { - isHasParameters = false; - } - const openRowDataSource = record.description.parameters; - const trialink: string = `${MANAGER_IP}/trial-jobs/${record.id}`; - const logPathRow = record.description.logPath !== undefined - ? - record.description.logPath - : - 'This trial\'s log path are not available.'; + const trialLink: string = `${MANAGER_IP}/trial-jobs/${trialId}`; + const logPathRow = trial.info.logPath || 'This trial\'s log path is not available.'; + const multiProgress = trial.info.hyperParameters === undefined ? 0 : trial.info.hyperParameters.length; return ( { - multiphase + EXPERIMENT.multiPhase ? Trails for multiphase experiment will return a set of parameters, we are listing the latest parameter in webportal.
For the entire parameter set, please refer to the following " - {trialink}". + {trialLink}".
- Current Phase: {record.description.multiProgress}. + Current Phase: {multiProgress}. :
} { - isHasParameters + trial.info.hyperParameters !== undefined ? { isClick ? - 
{JSON.stringify(openRowDataSource, null, 4)}
+ 
{JSON.stringify(trial.info.hyperParameters, null, 4)}
: true} // default expandNode getItemString={() => ()} // remove the {} items - data={openRowDataSource} + data={trial.info.hyperParameters} /> } @@ -138,15 +117,16 @@ class OpenRow extends React.Component { { - trainingPlatform !== 'local' + // FIXME: this should not be handled in web UI side + EXPERIMENT.trainingServicePlatform !== 'local' ? : - + } @@ -170,4 +150,4 @@ class OpenRow extends React.Component { } } -export default OpenRow; \ No newline at end of file +export default OpenRow; diff --git a/src/webui/src/components/trial-detail/DefaultMetricPoint.tsx b/src/webui/src/components/trial-detail/DefaultMetricPoint.tsx index 966df304ef..3fedaa9413 100644 --- a/src/webui/src/components/trial-detail/DefaultMetricPoint.tsx +++ b/src/webui/src/components/trial-detail/DefaultMetricPoint.tsx @@ -1,278 +1,41 @@ import * as React from 'react'; import { Switch } from 'antd'; import ReactEcharts from 'echarts-for-react'; -import { filterByStatus } from '../../static/function'; -import { TableObj, DetailAccurPoint, TooltipForAccuracy } from '../../static/interface'; +import { EXPERIMENT, TRIALS } from '../../static/datamodel'; +import { Trial } from '../../static/model/trial'; +import { TooltipForAccuracy } from '../../static/interface'; require('echarts/lib/chart/scatter'); require('echarts/lib/component/tooltip'); require('echarts/lib/component/title'); interface DefaultPointProps { - showSource: Array; - height: number; - whichGraph: string; - optimize: string; + trialIds: string[]; + visible: boolean; + trialsUpdateBroadcast: number; } interface DefaultPointState { - defaultSource: object; - accNodata: string; - succeedTrials: number; - isViewBestCurve: boolean; + bestCurveEnabled: boolean; } class DefaultPoint extends React.Component { - public _isDefaultMounted = false; - constructor(props: DefaultPointProps) { super(props); - this.state = { - defaultSource: {}, - accNodata: '', - succeedTrials: 10000000, - isViewBestCurve: false - }; - } - - defaultMetric = (succeedSource: Array, isCurve: boolean) => { - const { optimize } = this.props; - const accSource: Array = []; - const drawSource: Array = succeedSource.filter(filterByStatus); - const lengthOfSource = drawSource.length; - const tooltipDefault = lengthOfSource === 0 ? 'No data' : ''; - if (this._isDefaultMounted === true) { - this.setState(() => ({ - succeedTrials: lengthOfSource, - accNodata: tooltipDefault - })); - } - if (lengthOfSource === 0) { - const nullGraph = { - grid: { - left: '8%' - }, - xAxis: { - name: 'Trial', - type: 'category', - }, - yAxis: { - name: 'Default metric', - type: 'value', - } - }; - if (this._isDefaultMounted === true) { - this.setState(() => ({ - defaultSource: nullGraph - })); - } - } else { - const resultList: Array[] = []; - // lineListDefault: [[sequenceId, default metric], []] - const lineListDefault: Array[] = []; - Object.keys(drawSource).map(item => { - const temp = drawSource[item]; - if (temp.acc !== undefined) { - if (temp.acc.default !== undefined) { - const searchSpace = temp.description.parameters; - lineListDefault.push([temp.sequenceId, temp.acc.default]); - accSource.push({ - acc: temp.acc.default, - index: temp.sequenceId, - searchSpace: searchSpace - }); - } - } - }); - // deal with best metric line - const bestCurve: Array[] = []; // best curve data source - if (lineListDefault[0] !== undefined) { - bestCurve.push([lineListDefault[0][0], lineListDefault[0][1], accSource[0].searchSpace]); - } - if (optimize === 'maximize') { - for (let i = 1; i < lineListDefault.length; i++) { - const val = lineListDefault[i][1]; - const latest = bestCurve[bestCurve.length - 1][1]; - if (val >= latest) { - bestCurve.push([lineListDefault[i][0], val, accSource[i].searchSpace]); - } else { - bestCurve.push([lineListDefault[i][0], latest, accSource[i].searchSpace]); - } - } - } else { - for (let i = 1; i < lineListDefault.length; i++) { - const val = lineListDefault[i][1]; - const latest = bestCurve[bestCurve.length - 1][1]; - if (val <= latest) { - bestCurve.push([lineListDefault[i][0], val, accSource[i].searchSpace]); - } else { - bestCurve.push([lineListDefault[i][0], latest, accSource[i].searchSpace]); - } - } - } - Object.keys(accSource).map(item => { - const items = accSource[item]; - let temp: Array; - temp = [items.index, items.acc, items.searchSpace]; - resultList.push(temp); - }); - // isViewBestCurve: false show default metric graph - // isViewBestCurve: true show best curve - if (isCurve === true) { - if (this._isDefaultMounted === true) { - this.setState(() => ({ - defaultSource: this.drawBestcurve(bestCurve, resultList) - })); - } - } else { - if (this._isDefaultMounted === true) { - this.setState(() => ({ - defaultSource: this.drawDefaultMetric(resultList) - })); - } - } - } - } - - drawBestcurve = (realDefault: Array[], resultList: Array[]) => { - return { - grid: { - left: '8%' - }, - tooltip: { - trigger: 'item', - enterable: true, - position: function (point: Array, data: TooltipForAccuracy) { - if (data.data[0] < realDefault.length / 2) { - return [point[0], 80]; - } else { - return [point[0] - 300, 80]; - } - }, - formatter: function (data: TooltipForAccuracy) { - const result = '
' + - '
Trial No.: ' + data.data[0] + '
' + - '
Optimization curve: ' + data.data[1] + '
' + - '
Parameters: ' + - '
' + JSON.stringify(data.data[2], null, 4) + '
' + - '
' + - '
'; - return result; - } - }, - xAxis: { - name: 'Trial', - type: 'category', - }, - yAxis: { - name: 'Default metric', - type: 'value', - scale: true - }, - series: [ - { - type: 'line', - lineStyle: { color: '#FF6600' }, - data: realDefault - }, - { - symbolSize: 6, - type: 'scatter', - data: resultList - }] - }; - } - - drawDefaultMetric = (resultList: Array[]) => { - return { - grid: { - left: '8%' - }, - tooltip: { - trigger: 'item', - enterable: true, - position: function (point: Array, data: TooltipForAccuracy) { - if (data.data[0] < resultList.length / 2) { - return [point[0], 80]; - } else { - return [point[0] - 300, 80]; - } - }, - formatter: function (data: TooltipForAccuracy) { - const result = '
' + - '
Trial No.: ' + data.data[0] + '
' + - '
Default metric: ' + data.data[1] + '
' + - '
Parameters: ' + - '
' + JSON.stringify(data.data[2], null, 4) + '
' + - '
' + - '
'; - return result; - } - }, - xAxis: { - name: 'Trial', - type: 'category', - }, - yAxis: { - name: 'Default metric', - type: 'value', - scale: true - }, - series: [{ - symbolSize: 6, - type: 'scatter', - data: resultList - }] - }; + this.state = { bestCurveEnabled: false }; } loadDefault = (checked: boolean) => { - // checked: true show best metric curve - const { showSource } = this.props; - if (this._isDefaultMounted === true) { - this.defaultMetric(showSource, checked); - // ** deal with data and then update view layer - this.setState(() => ({ isViewBestCurve: checked })); - } - } - - // update parent component state - componentWillReceiveProps(nextProps: DefaultPointProps) { - - const { whichGraph, showSource } = nextProps; - const { isViewBestCurve } = this.state; - if (whichGraph === '1') { - this.defaultMetric(showSource, isViewBestCurve); - } + this.setState({ bestCurveEnabled: checked }); } shouldComponentUpdate(nextProps: DefaultPointProps, nextState: DefaultPointState) { - const { whichGraph } = nextProps; - if (whichGraph === '1') { - const { succeedTrials, isViewBestCurve } = nextState; - const succTrial = this.state.succeedTrials; - const isViewBestCurveBefore = this.state.isViewBestCurve; - if (isViewBestCurveBefore !== isViewBestCurve) { - return true; - } - if (succeedTrials !== succTrial) { - return true; - } - } - // only whichGraph !== '1', default metric can't update - return false; - } - - componentDidMount() { - this._isDefaultMounted = true; - } - - componentWillUnmount() { - this._isDefaultMounted = false; + return nextProps.visible; } render() { - const { height } = this.props; - const { defaultSource, accNodata } = this.state; + const graph = this.generateGraph(); + const accNodata = (graph === EmptyGraph ? 'No data' : ''); + return (
@@ -282,10 +45,10 @@ class DefaultPoint extends React.Component
); } + + private generateGraph() { + const trials = TRIALS.getTrials(this.props.trialIds).filter(trial => trial.sortable); + if (trials.length === 0) { + return EmptyGraph; + } + const graph = generateGraphConfig(trials[trials.length - 1].sequenceId); + if (this.state.bestCurveEnabled) { + (graph as any).series = [ generateBestCurveSeries(trials), generateScatterSeries(trials) ]; + } else { + (graph as any).series = [ generateScatterSeries(trials) ]; + } + return graph; + } +} + +const EmptyGraph = { + grid: { + left: '8%' + }, + xAxis: { + name: 'Trial', + type: 'category', + }, + yAxis: { + name: 'Default metric', + type: 'value', + } +}; + +function generateGraphConfig(maxSequenceId: number) { + return { + grid: { + left: '8%', + }, + tooltip: { + trigger: 'item', + enterable: true, + position: (point: Array, data: TooltipForAccuracy) => ( + [ (data.data[0] < maxSequenceId ? point[0] : (point[0] - 300)), 80 ] + ), + formatter: (data: TooltipForAccuracy) => ( + '
' + + '
Trial No.: ' + data.data[0] + '
' + + '
Default metric: ' + data.data[1] + '
' + + '
Parameters: 
' + JSON.stringify(data.data[2], null, 4) + '
' + + '
' + ), + }, + xAxis: { + name: 'Trial', + type: 'category', + }, + yAxis: { + name: 'Default metric', + type: 'value', + scale: true, + }, + series: undefined, + }; +} + +function generateScatterSeries(trials: Trial[]) { + const data = trials.map(trial => [ + trial.sequenceId, + trial.accuracy, + trial.info.hyperParameters, + ]); + return { + symbolSize: 6, + type: 'scatter', + data, + }; +} + +function generateBestCurveSeries(trials: Trial[]) { + let best = trials[0]; + const data = [[ best.sequenceId, best.accuracy, best.info.hyperParameters ]]; + + for (let i = 1; i < trials.length; i++) { + const trial = trials[i]; + const delta = trial.accuracy! - best.accuracy!; + const better = (EXPERIMENT.optimizeMode === 'minimize') ? (delta < 0) : (delta > 0); + if (better) { + data.push([ trial.sequenceId, trial.accuracy, trial.info.hyperParameters ]); + best = trial; + } else { + data.push([ trial.sequenceId, best.accuracy, trial.info.hyperParameters ]); + } + } + + return { + type: 'line', + lineStyle: { color: '#FF6600' }, + data, + }; } -export default DefaultPoint; \ No newline at end of file +export default DefaultPoint; diff --git a/src/webui/src/components/trial-detail/Duration.tsx b/src/webui/src/components/trial-detail/Duration.tsx index d47b5107ce..c8add154b4 100644 --- a/src/webui/src/components/trial-detail/Duration.tsx +++ b/src/webui/src/components/trial-detail/Duration.tsx @@ -22,8 +22,6 @@ interface DurationState { class Duration extends React.Component { - public _isMounted = false; - constructor(props: DurationProps) { super(props); @@ -142,15 +140,12 @@ class Duration extends React.Component { trialId: trialId, trialTime: trialTime }); - if (this._isMounted) { - this.setState({ - durationSource: this.getOption(trialRun[0]) - }); - } + this.setState({ + durationSource: this.getOption(trialRun[0]) + }); } componentDidMount() { - this._isMounted = true; const { source } = this.props; this.drawDurationGraph(source); } @@ -187,10 +182,6 @@ class Duration extends React.Component { return false; } - componentWillUnmount() { - this._isMounted = false; - } - render() { const { durationSource } = this.state; return ( @@ -206,4 +197,4 @@ class Duration extends React.Component { } } -export default Duration; \ No newline at end of file +export default Duration; diff --git a/src/webui/src/components/trial-detail/Intermediate.tsx b/src/webui/src/components/trial-detail/Intermediate.tsx index 9a9dfa1e6d..e9117a43f9 100644 --- a/src/webui/src/components/trial-detail/Intermediate.tsx +++ b/src/webui/src/components/trial-detail/Intermediate.tsx @@ -24,7 +24,6 @@ interface IntermediateProps { class Intermediate extends React.Component { static intervalMediate = 1; - public _isMounted = false; public pointInput: HTMLInputElement | null; public minValInput: HTMLInputElement | null; public maxValInput: HTMLInputElement | null; @@ -45,12 +44,10 @@ class Intermediate extends React.Component drawIntermediate = (source: Array) => { if (source.length > 0) { - if (this._isMounted) { - this.setState(() => ({ - length: source.length, - detailSource: source - })); - } + this.setState({ + length: source.length, + detailSource: source + }); const trialIntermediate: Array = []; Object.keys(source).map(item => { const temp = source[item]; @@ -118,11 +115,9 @@ class Intermediate extends React.Component }, series: trialIntermediate }; - if (this._isMounted) { - this.setState(() => ({ - interSource: option - })); - } + this.setState({ + interSource: option + }); } else { const nullData = { grid: { @@ -139,71 +134,60 @@ class Intermediate extends React.Component name: 'Metric' } }; - if (this._isMounted) { - this.setState(() => ({ interSource: nullData })); - } + this.setState({ interSource: nullData }); } } // confirm btn function [filter data] filterLines = () => { - if (this._isMounted) { - const filterSource: Array = []; - this.setState({ isLoadconfirmBtn: true }, () => { - const { source } = this.props; - // get input value - const pointVal = this.pointInput !== null ? this.pointInput.value : ''; - const minVal = this.minValInput !== null ? this.minValInput.value : ''; - const maxVal = this.maxValInput !== null ? this.maxValInput.value : ''; - // user not input message - if (pointVal === '' || minVal === '') { - alert('Please input filter message'); + const filterSource: Array = []; + this.setState({ isLoadconfirmBtn: true }, () => { + const { source } = this.props; + // get input value + const pointVal = this.pointInput !== null ? this.pointInput.value : ''; + const minVal = this.minValInput !== null ? this.minValInput.value : ''; + const maxVal = this.maxValInput !== null ? this.maxValInput.value : ''; + // user not input message + if (pointVal === '' || minVal === '') { + alert('Please input filter message'); + } else { + // user not input max value + const position = JSON.parse(pointVal); + const min = JSON.parse(minVal); + if (maxVal === '') { + Object.keys(source).map(item => { + const temp = source[item]; + const val = temp.description.intermediate[position - 1]; + if (val >= min) { + filterSource.push(temp); + } + }); } else { - // user not input max value - const position = JSON.parse(pointVal); - const min = JSON.parse(minVal); - if (maxVal === '') { - Object.keys(source).map(item => { - const temp = source[item]; - const val = temp.description.intermediate[position - 1]; - if (val >= min) { - filterSource.push(temp); - } - }); - } else { - const max = JSON.parse(maxVal); - Object.keys(source).map(item => { - const temp = source[item]; - const val = temp.description.intermediate[position - 1]; - if (val >= min && val <= max) { - filterSource.push(temp); - } - }); - } - if (this._isMounted) { - this.setState({ filterSource: filterSource }); - } - this.drawIntermediate(filterSource); - } - const counts = this.state.clickCounts + 1; - if (this._isMounted) { - this.setState({ isLoadconfirmBtn: false, clickCounts: counts }); + const max = JSON.parse(maxVal); + Object.keys(source).map(item => { + const temp = source[item]; + const val = temp.description.intermediate[position - 1]; + if (val >= min && val <= max) { + filterSource.push(temp); + } + }); } - }); - } + this.setState({ filterSource: filterSource }); + this.drawIntermediate(filterSource); + } + const counts = this.state.clickCounts + 1; + this.setState({ isLoadconfirmBtn: false, clickCounts: counts }); + }); } switchTurn = (checked: boolean) => { - if (this._isMounted) { - this.setState({ isFilter: checked }); - } + this.setState({ isFilter: checked }); if (checked === false) { this.drawIntermediate(this.props.source); } } componentDidMount() { - this._isMounted = true; const { source } = this.props; this.drawIntermediate(source); } @@ -272,10 +256,6 @@ class Intermediate extends React.Component return false; } - componentWillUnmount() { - this._isMounted = false; - } - render() { const { interSource, isLoadconfirmBtn, isFilter } = this.state; return ( diff --git a/src/webui/src/components/trial-detail/Para.tsx b/src/webui/src/components/trial-detail/Para.tsx index 94d6d70883..156aa78547 100644 --- a/src/webui/src/components/trial-detail/Para.tsx +++ b/src/webui/src/components/trial-detail/Para.tsx @@ -40,8 +40,6 @@ message.config({ class Para extends React.Component { - public _isMounted = false; - private chartMulineStyle = { width: '100%', height: 392, @@ -121,15 +119,12 @@ class Para extends React.Component { this.swapGraph(paraData, swapAxisArr); } this.getOption(paraData, lengthofTrials); - if (this._isMounted === true) { - this.setState(() => ({ paraBack: paraData })); - } + this.setState({ paraBack: paraData }); } hyperParaPic = (source: Array, searchSpace: string) => { // filter succeed trials [{}, {}, {}] - const origin = source.filter(filterByStatus); - const dataSource: Array = JSON.parse(JSON.stringify(origin)); + const dataSource = source.filter(filterByStatus); const lenOfDataSource: number = dataSource.length; const accPara: Array = []; // specific value array @@ -139,15 +134,13 @@ class Para extends React.Component { // nest search space let isNested: boolean = false; Object.keys(searchRange).map(item => { - if (typeof searchRange[item]._value[0] === 'object') { + if (searchRange[item]._value && typeof searchRange[item]._value[0] === 'object') { isNested = true; return; } }); const dimName = Object.keys(searchRange); - if (this._isMounted === true) { - this.setState(() => ({ dimName: dimName })); - } + this.setState({ dimName: dimName }); const parallelAxis: Array = []; // search space range and specific value [only number] @@ -324,23 +317,21 @@ class Para extends React.Component { color: ['#CA0000', '#FFC400', '#90EE90'] } }; - if (this._isMounted === true) { - this.setState({ - paraNodata: 'No data', - option: optionOfNull, - sutrialCount: 0, - succeedRenderCount: 0 - }); - } + this.setState({ + paraNodata: 'No data', + option: optionOfNull, + sutrialCount: 0, + succeedRenderCount: 0 + }); } else { Object.keys(dataSource).map(item => { - const temp = dataSource[item]; - eachTrialParams.push(temp.description.parameters); + const trial = dataSource[item]; + eachTrialParams.push(trial.description.parameters.error || ''); // may be a succeed trial hasn't final result // all detail page may be break down if havn't if - if (temp.acc !== undefined) { - if (temp.acc.default !== undefined) { - accPara.push(temp.acc.default); + if (trial.acc !== undefined) { + if (trial.acc.default !== undefined) { + accPara.push(JSON.parse(trial.acc.default)); } } }); @@ -361,14 +352,12 @@ class Para extends React.Component { }); }); } - if (this._isMounted) { - // if not return final result - const maxVal = accPara.length === 0 ? 1 : Math.max(...accPara); - const minVal = accPara.length === 0 ? 1 : Math.min(...accPara); - this.setState({ max: maxVal, min: minVal }, () => { - this.getParallelAxis(dimName, parallelAxis, accPara, eachTrialParams, lenOfDataSource); - }); - } + // if not return final result + const maxVal = accPara.length === 0 ? 1 : Math.max(...accPara); + const minVal = accPara.length === 0 ? 1 : Math.min(...accPara); + this.setState({ max: maxVal, min: minVal }, () => { + this.getParallelAxis(dimName, parallelAxis, accPara, eachTrialParams, lenOfDataSource); + }); } } @@ -376,11 +365,9 @@ class Para extends React.Component { percentNum = (value: string) => { let vals = parseFloat(value); - if (this._isMounted) { - this.setState({ percent: vals }, () => { - this.reInit(); - }); - } + this.setState({ percent: vals }, () => { + this.reInit(); + }); } // deal with response data into pic data @@ -445,22 +432,17 @@ class Para extends React.Component { } }; // please wait the data - if (this._isMounted) { - this.setState(() => ({ - option: optionown, - paraNodata: '', - succeedRenderCount: lengthofTrials, - sutrialCount: paralleData.length - })); - } + this.setState({ + option: optionown, + paraNodata: '', + succeedRenderCount: lengthofTrials, + sutrialCount: paralleData.length + }); } // get swap parallel axis getSwapArr = (value: Array) => { - - if (this._isMounted) { - this.setState(() => ({ swapAxisArr: value })); - } + this.setState({ swapAxisArr: value }); } reInit = () => { @@ -471,9 +453,7 @@ class Para extends React.Component { swapReInit = () => { const { clickCounts, succeedRenderCount } = this.state; const val = clickCounts + 1; - if (this._isMounted) { - this.setState({ isLoadConfirm: true, clickCounts: val, }); - } + this.setState({ isLoadConfirm: true, clickCounts: val, }); const { paraBack, swapAxisArr } = this.state; const paralDim = paraBack.parallelAxis; const paraData = paraBack.data; @@ -523,11 +503,9 @@ class Para extends React.Component { }); this.getOption(paraBack, succeedRenderCount); // please wait the data - if (this._isMounted) { - this.setState(() => ({ - isLoadConfirm: false - })); - } + this.setState({ + isLoadConfirm: false + }); } sortDimY = (a: Dimobj, b: Dimobj) => { @@ -585,7 +563,6 @@ class Para extends React.Component { } componentDidMount() { - this._isMounted = true; this.reInit(); } @@ -623,10 +600,6 @@ class Para extends React.Component { return false; } - componentWillUnmount() { - this._isMounted = false; - } - render() { const { option, paraNodata, dimName, isLoadConfirm } = this.state; return ( @@ -687,4 +660,4 @@ class Para extends React.Component { } } -export default Para; \ No newline at end of file +export default Para; diff --git a/src/webui/src/components/trial-detail/TableList.tsx b/src/webui/src/components/trial-detail/TableList.tsx index 0c72de7a5e..318f4ecbc5 100644 --- a/src/webui/src/components/trial-detail/TableList.tsx +++ b/src/webui/src/components/trial-detail/TableList.tsx @@ -2,14 +2,14 @@ import * as React from 'react'; import axios from 'axios'; import ReactEcharts from 'echarts-for-react'; import { Row, Table, Button, Popconfirm, Modal, Checkbox, Select, Icon } from 'antd'; +import { ColumnProps } from 'antd/lib/table'; const Option = Select.Option; const CheckboxGroup = Checkbox.Group; import { MANAGER_IP, trialJobStatus, COLUMN_INDEX, COLUMNPro } from '../../static/const'; -import { convertDuration, intermediateGraphOption, killJob, filterByStatus } from '../../static/function'; -import { TableObj, TrialJob } from '../../static/interface'; +import { convertDuration, formatTimestamp, intermediateGraphOption, killJob } from '../../static/function'; +import { TableRecord } from '../../static/interface'; import OpenRow from '../public-child/OpenRow'; import Compare from '../Modal/Compare'; -import IntermediateVal from '../public-child/IntermediateVal'; // table default metric column import '../../static/style/search.scss'; require('../../static/style/tableStatus.css'); require('../../static/style/logPath.scss'); @@ -26,14 +26,11 @@ echarts.registerTheme('my_theme', { }); interface TableListProps { - entries: number; - tableSource: Array; - updateList: Function; - platform: string; - logCollection: boolean; - isMultiPhase: boolean; + pageSize: number; + tableSource: Array; columnList: Array; // user select columnKeys changeColumn: (val: Array) => void; + trialsUpdateBroadcast: number; } interface TableListState { @@ -41,7 +38,7 @@ interface TableListState { modalVisible: boolean; isObjFinal: boolean; isShowColumn: boolean; - selectRows: Array; + selectRows: Array; isShowCompareModal: boolean; selectedRowKeys: string[] | number[]; intermediateData: Array; // a trial's intermediate results (include dict) @@ -56,10 +53,9 @@ interface ColumnIndex { class TableList extends React.Component { - public _isMounted = false; public intervalTrialLog = 10; public _trialId: string; - public tables: Table | null; + public tables: Table | null; constructor(props: TableListProps) { super(props); @@ -78,46 +74,35 @@ class TableList extends React.Component { }; } - showIntermediateModal = (id: string) => { - - axios(`${MANAGER_IP}/metric-data/${id}`, { - method: 'GET' - }) - .then(res => { - if (res.status === 200) { - const intermediateArr: number[] = []; - // support intermediate result is dict because the last intermediate result is - // final result in a succeed trial, it may be a dict. - // get intermediate result dict keys array - let otherkeys: Array = ['default']; - if (res.data.length !== 0) { - otherkeys = Object.keys(JSON.parse(res.data[0].data)); - } - // intermediateArr just store default val - Object.keys(res.data).map(item => { - const temp = JSON.parse(res.data[item].data); - if (typeof temp === 'object') { - intermediateArr.push(temp.default); - } else { - intermediateArr.push(temp); - } - }); - const intermediate = intermediateGraphOption(intermediateArr, id); - if (this._isMounted) { - this.setState(() => ({ - intermediateData: res.data, // store origin intermediate data for a trial - intermediateOption: intermediate, - intermediateOtherKeys: otherkeys, - intermediateId: id - })); - } + showIntermediateModal = async (id: string) => { + const res = await axios.get(`${MANAGER_IP}/metric-data/${id}`); + if (res.status === 200) { + const intermediateArr: number[] = []; + // support intermediate result is dict because the last intermediate result is + // final result in a succeed trial, it may be a dict. + // get intermediate result dict keys array + let otherkeys: Array = ['default']; + if (res.data.length !== 0) { + otherkeys = Object.keys(JSON.parse(res.data[0].data)); + } + // intermediateArr just store default val + Object.keys(res.data).map(item => { + const temp = JSON.parse(res.data[item].data); + if (typeof temp === 'object') { + intermediateArr.push(temp.default); + } else { + intermediateArr.push(temp); } }); - if (this._isMounted) { + const intermediate = intermediateGraphOption(intermediateArr, id); this.setState({ - modalVisible: true + intermediateData: res.data, // store origin intermediate data for a trial + intermediateOption: intermediate, + intermediateOtherKeys: otherkeys, + intermediateId: id }); } + this.setState({ modalVisible: true }); } // intermediate button click -> intermediate graph for each trial @@ -147,37 +132,29 @@ class TableList extends React.Component { } const intermediate = intermediateGraphOption(intermediateArr, intermediateId); // re-render - if (this._isMounted) { - this.setState(() => ({ - intermediateOption: intermediate - })); - } + this.setState({ + intermediateOption: intermediate + }); } hideIntermediateModal = () => { - if (this._isMounted) { - this.setState({ - modalVisible: false - }); - } + this.setState({ + modalVisible: false + }); } hideShowColumnModal = () => { - if (this._isMounted) { - this.setState({ - isShowColumn: false - }); - } + this.setState({ + isShowColumn: false + }); } // click add column btn, just show the modal of addcolumn addColumn = () => { // show user select check button - if (this._isMounted) { - this.setState({ - isShowColumn: true - }); - } + this.setState({ + isShowColumn: true + }); } // checkbox for coloumn @@ -191,8 +168,8 @@ class TableList extends React.Component { switch (checkedValues[m]) { case 'Trial No.': case 'ID': - case 'StartTime': - case 'EndTime': + case 'Start Time': + case 'End Time': case 'Duration': case 'Status': case 'Operation': @@ -229,27 +206,17 @@ class TableList extends React.Component { wantResult.push(want[i].name); }); - if (this._isMounted) { - this.props.changeColumn(wantResult); - } + this.props.changeColumn(wantResult); } - openRow = (record: TableObj) => { - const { platform, logCollection, isMultiPhase } = this.props; + openRow = (record: TableRecord) => { return ( - + ); } - fillSelectedRowsTostate = (selected: number[] | string[], selectedRows: Array) => { - if (this._isMounted === true) { - this.setState(() => ({ selectRows: selectedRows, selectedRowKeys: selected })); - } + fillSelectedRowsTostate = (selected: number[] | string[], selectedRows: Array) => { + this.setState({ selectRows: selectedRows, selectedRowKeys: selected }); } // open Compare-modal compareBtn = () => { @@ -258,47 +225,33 @@ class TableList extends React.Component { if (selectRows.length === 0) { alert('Please select datas you want to compare!'); } else { - if (this._isMounted === true) { - this.setState({ isShowCompareModal: true }); - } + this.setState({ isShowCompareModal: true }); } } // close Compare-modal hideCompareModal = () => { // close modal. clear select rows data, clear selected track - if (this._isMounted) { this.setState({ isShowCompareModal: false, selectedRowKeys: [], selectRows: [] }); - } - } - - componentDidMount() { - this._isMounted = true; - } - - componentWillUnmount() { - this._isMounted = false; } render() { - - const { entries, tableSource, updateList, columnList } = this.props; + const { pageSize, columnList } = this.props; + const tableSource: Array = JSON.parse(JSON.stringify(this.props.tableSource)); + console.log('rerender table', tableSource); const { intermediateOption, modalVisible, isShowColumn, selectRows, isShowCompareModal, selectedRowKeys, intermediateOtherKeys } = this.state; const rowSelection = { selectedRowKeys: selectedRowKeys, - onChange: (selected: string[] | number[], selectedRows: Array) => { + onChange: (selected: string[] | number[], selectedRows: Array) => { this.fillSelectedRowsTostate(selected, selectedRows); } }; let showTitle = COLUMNPro; - let bgColor = ''; - const trialJob: Array = []; const showColumn: Array = []; // only succeed trials have final keys - if (tableSource.filter(filterByStatus).length >= 1) { - const temp = tableSource.filter(filterByStatus)[0].acc; + if (tableSource.filter(record => record.status === 'SUCCEEDED').length >= 1) { + const temp = tableSource.filter(record => record.status === 'SUCCEEDED')[0].accuracy; if (temp !== undefined && typeof temp === 'object') { - if (this._isMounted) { // concat default column and finalkeys const item = Object.keys(temp); // item: ['default', 'other-keys', 'maybe loss'] @@ -311,169 +264,33 @@ class TableList extends React.Component { }); showTitle = COLUMNPro.concat(want); } - } } } - trialJobStatus.map(item => { - trialJob.push({ - text: item, - value: item - }); - }); - Object.keys(columnList).map(key => { - const item = columnList[key]; + for (const item of columnList) { switch (item) { case 'Trial No.': - showColumn.push({ - title: 'Trial No.', - dataIndex: 'sequenceId', - key: 'sequenceId', - width: 120, - className: 'tableHead', - sorter: (a: TableObj, b: TableObj) => (a.sequenceId as number) - (b.sequenceId as number) - }); + showColumn.push(SequenceIdColumnConfig); break; case 'ID': - showColumn.push({ - title: 'ID', - dataIndex: 'id', - key: 'id', - width: 60, - className: 'tableHead leftTitle', - // the sort of string - sorter: (a: TableObj, b: TableObj): number => a.id.localeCompare(b.id), - render: (text: string, record: TableObj) => { - return ( -
{record.id}
- ); - } - }); + showColumn.push(IdColumnConfig); break; - case 'StartTime': - showColumn.push({ - title: 'StartTime', - dataIndex: 'startTime', - key: 'startTime', - width: 160, - render: (text: string, record: TableObj) => { - const start = record.startTime !== undefined ? record.startTime : -1; - return ( - - { - start !== -1 - ? - new Date(start).toLocaleString('en-US') - : - '--' - } - - ); - }, - }); + case 'Start Time': + showColumn.push(StartTimeColumnConfig); break; - case 'EndTime': - showColumn.push({ - title: 'EndTime', - dataIndex: 'endTime', - key: 'endTime', - width: 160, - render: (text: string, record: TableObj) => { - const end = record.endTime !== undefined ? record.endTime : -1; - return ( - - { - end !== -1 - ? - new Date(end).toLocaleString('en-US') - : - '--' - } - - ); - }, - }); + case 'End Time': + showColumn.push(EndTimeColumnConfig); break; case 'Duration': - showColumn.push({ - title: 'Duration', - dataIndex: 'duration', - key: 'duration', - width: 100, - // the sort of number - sorter: (a: TableObj, b: TableObj) => (a.duration as number) - (b.duration as number), - render: (text: string, record: TableObj) => { - let duration; - if (record.duration !== undefined) { - // duration is nagative number(-1) & 0-1 - if (record.duration > 0 && record.duration < 1 || record.duration < 0) { - duration = `${record.duration}s`; - } else { - duration = convertDuration(record.duration); - } - } else { - duration = 0; - } - return ( -
{duration}
- ); - }, - }); + showColumn.push(DurationColumnConfig); break; case 'Status': - showColumn.push({ - title: 'Status', - dataIndex: 'status', - key: 'status', - width: 150, - className: 'tableStatus', - render: (text: string, record: TableObj) => { - bgColor = record.status; - return ( - {record.status} - ); - }, - filters: trialJob, - onFilter: (value: string, record: TableObj) => { - return record.status.indexOf(value) === 0; - }, - // onFilter: (value: string, record: TableObj) => record.status.indexOf(value) === 0, - sorter: (a: TableObj, b: TableObj): number => a.status.localeCompare(b.status) - }); + showColumn.push(StatusColumnConfig); break; case 'Intermediate count': - showColumn.push({ - title: 'Intermediate count', - dataIndex: 'progress', - key: 'progress', - width: 86, - render: (text: string, record: TableObj) => { - return ( - {`#${record.description.intermediate.length}`} - ); - }, - }); + showColumn.push(IntermediateCountColumnConfig); break; case 'Default': - showColumn.push({ - title: 'Default metric', - className: 'leftTitle', - dataIndex: 'acc', - key: 'acc', - width: 120, - sorter: (a: TableObj, b: TableObj) => { - const oneArr = a.description.intermediate; - const otherArr = b.description.intermediate; - const one = (oneArr[oneArr.length - 1] !== undefined) ? oneArr[oneArr.length - 1] : 0; - const other = (otherArr[otherArr.length - 1] !== undefined) - ? otherArr[otherArr.length - 1] : 0; - return one - other; - }, - render: (text: string, record: TableObj) => { - return ( - - ); - } - }); + showColumn.push(AccuracyColumnConfig); break; case 'Operation': showColumn.push({ @@ -481,7 +298,7 @@ class TableList extends React.Component { dataIndex: 'operation', key: 'operation', width: 120, - render: (text: string, record: TableObj) => { + render: (text: string, record: TableRecord) => { let trialStatus = record.status; const flag: boolean = (trialStatus === 'RUNNING') ? false : true; return ( @@ -499,7 +316,7 @@ class TableList extends React.Component {
| null) => this.tables = table} + ref={(table: Table | null) => this.tables = table} columns={showColumn} rowSelection={rowSelection} expandedRowRender={this.openRow} dataSource={tableSource} className="commonTableStyle" - pagination={{ pageSize: entries }} + pagination={pageSize > 0 ? { pageSize } : false} /> {/* Intermediate Result Modal */} { } } +const SequenceIdColumnConfig: ColumnProps = { + title: 'Trial No.', + dataIndex: 'sequenceId', + width: 120, + className: 'tableHead', + sorter: (a, b) => a.sequenceId - b.sequenceId +}; + +const IdColumnConfig: ColumnProps = { + title: 'ID', + dataIndex: 'id', + width: 60, + className: 'tableHead leftTitle', + sorter: (a, b) => a.id.localeCompare(b.id), + render: (text, record) => ( +
{record.id}
+ ) +}; + +const StartTimeColumnConfig: ColumnProps = { + title: 'Start Time', + dataIndex: 'startTime', + width: 160, + render: (text, record) => ( + {formatTimestamp(record.startTime)} + ) +}; + +const EndTimeColumnConfig: ColumnProps = { + title: 'End Time', + dataIndex: 'endTime', + width: 160, + render: (text, record) => ( + {formatTimestamp(record.endTime, '--')} + ) +}; + +const DurationColumnConfig: ColumnProps = { + title: 'Duration', + dataIndex: 'duration', + width: 100, + sorter: (a, b) => a.duration - b.duration, + render: (text, record) => ( +
{convertDuration(record.duration)}
+ ) +}; + +const StatusColumnConfig: ColumnProps = { + title: 'Status', + dataIndex: 'status', + width: 150, + className: 'tableStatus', + render: (text, record) => ( + {record.status} + ), + sorter: (a, b) => a.status.localeCompare(b.status), + filters: trialJobStatus.map(status => ({ text: status, value: status })), + onFilter: (value, record) => (record.status === value) +}; + +const IntermediateCountColumnConfig: ColumnProps = { + title: 'Intermediate count', + dataIndex: 'intermediateCount', + width: 86, + render: (text, record) => ( + {`#${record.intermediateCount}`} + ) +}; + +const AccuracyColumnConfig: ColumnProps = { + title: 'Default metric', + className: 'leftTitle', + dataIndex: 'accuracy', + width: 120, + sorter: (a, b, sortOrder) => { + if (a.accuracy === undefined) { + return sortOrder === 'ascend' ? -1 : 1; + } else if (b.accuracy === undefined) { + return sortOrder === 'ascend' ? 1 : -1; + } else { + return a.accuracy - b.accuracy; + } + }, + render: (text, record) => ( + // TODO: is this needed? +
{record.latestAccuracy}
+ ) +}; + export default TableList; diff --git a/src/webui/src/static/const.ts b/src/webui/src/static/const.ts index fcfd40bc88..fbc300ad2e 100644 --- a/src/webui/src/static/const.ts +++ b/src/webui/src/static/const.ts @@ -1,3 +1,7 @@ +// when there are more trials than this threshold, metrics will be updated in group of this size to avoid freezing +const METRIC_GROUP_UPDATE_THRESHOLD = 100; +const METRIC_GROUP_UPDATE_SIZE = 20; + const MANAGER_IP = `/api/v1/nni`; const DOWNLOAD_IP = `/logs`; const trialJobStatus = [ @@ -65,9 +69,10 @@ const COLUMN_INDEX = [ // defatult selected column const COLUMN = ['Trial No.', 'ID', 'Duration', 'Status', 'Default', 'Operation']; // all choice column !dictory final -const COLUMNPro = ['Trial No.', 'ID', 'StartTime', 'EndTime', 'Duration', 'Status', +const COLUMNPro = ['Trial No.', 'ID', 'Start Time', 'End Time', 'Duration', 'Status', 'Intermediate count', 'Default', 'Operation']; export { MANAGER_IP, DOWNLOAD_IP, trialJobStatus, COLUMNPro, - CONTROLTYPE, MONACO, COLUMN, COLUMN_INDEX, DRAWEROPTION + CONTROLTYPE, MONACO, COLUMN, COLUMN_INDEX, DRAWEROPTION, + METRIC_GROUP_UPDATE_THRESHOLD, METRIC_GROUP_UPDATE_SIZE, }; diff --git a/src/webui/src/static/datamodel.ts b/src/webui/src/static/datamodel.ts new file mode 100644 index 0000000000..b47f7b2e84 --- /dev/null +++ b/src/webui/src/static/datamodel.ts @@ -0,0 +1,7 @@ +import { Experiment } from './model/experiment'; +import { TrialManager } from './model/trialmanager'; + +const EXPERIMENT = new Experiment(); +const TRIALS = new TrialManager(); + +export { EXPERIMENT, TRIALS }; diff --git a/src/webui/src/static/function.ts b/src/webui/src/static/function.ts index 857c2fb0d4..be352ff35c 100644 --- a/src/webui/src/static/function.ts +++ b/src/webui/src/static/function.ts @@ -1,9 +1,12 @@ import axios from 'axios'; import { message } from 'antd'; import { MANAGER_IP } from './const'; -import { FinalResult, FinalType, TableObj } from './interface'; +import { MetricDataRecord, FinalType, TableObj } from './interface'; const convertTime = (num: number) => { + if (num <= 0) { + return '0'; + } if (num % 3600 === 0) { return num / 3600 + 'h'; } else { @@ -15,24 +18,28 @@ const convertTime = (num: number) => { // trial's duration, accurate to seconds for example 10min 30s const convertDuration = (num: number) => { + if (num < 1) { + return '0s'; + } const hour = Math.floor(num / 3600); - const min = Math.floor(num / 60 % 60); + const minute = Math.floor(num / 60 % 60); const second = Math.floor(num % 60); - const result = hour > 0 ? `${hour} h ${min} min ${second}s` : `${min} min ${second}s`; - if (hour <= 0 && min === 0 && second !== 0) { - return `${second}s`; - } else if (hour === 0 && min !== 0 && second === 0) { - return `${min}min`; - } else if (hour === 0 && min !== 0 && second !== 0) { - return `${min}min ${second}s`; - } else { - return result; + let result = [ ]; + if (hour > 0) { + result.push(`${hour}h`); + } + if (minute > 0) { + result.push(`${minute}min`); + } + if (second > 0) { + result.push(`${second}s`); } + return result.join(' '); }; // get final result value // draw Accuracy point graph -const getFinalResult = (final: Array) => { +const getFinalResult = (final?: MetricDataRecord[]) => { let acc; let showDefault = 0; if (final) { @@ -51,7 +58,7 @@ const getFinalResult = (final: Array) => { }; // get final result value // acc obj -const getFinal = (final: Array) => { +const getFinal = (final?: MetricDataRecord[]) => { let showDefault: FinalType; if (final) { showDefault = JSON.parse(final[final.length - 1].data); @@ -101,7 +108,7 @@ const intermediateGraphOption = (intermediateArr: number[], id: string) => { }; // kill job -const killJob = (key: number, id: string, status: string, updateList: Function) => { +const killJob = (key: number, id: string, status: string, updateList?: Function) => { axios(`${MANAGER_IP}/trial-jobs/${id}`, { method: 'DELETE', headers: { @@ -113,7 +120,9 @@ const killJob = (key: number, id: string, status: string, updateList: Function) message.destroy(); message.success('Cancel the job successfully'); // render the table - updateList(); + if (updateList) { + updateList(); // FIXME + } } else { message.error('fail to cancel the job'); } @@ -160,7 +169,22 @@ const downFile = (content: string, fileName: string) => { } }; +function formatTimestamp(timestamp?: number, placeholder?: string = 'N/A'): string { + return timestamp ? new Date(timestamp).toLocaleString('en-US') : placeholder; +} + +function metricAccuracy(metric: MetricDataRecord): number { + const data = JSON.parse(metric.data); + return typeof data === 'number' ? data : NaN; +} + +function formatAccuracy(accuracy: number): string { + // TODO: how to format NaN? + return accuracy.toFixed(6).replace(/0+$/, '').replace(/\.$/, ''); +} + export { convertTime, convertDuration, getFinalResult, getFinal, downFile, - intermediateGraphOption, killJob, filterByStatus, filterDuration + intermediateGraphOption, killJob, filterByStatus, filterDuration, + formatAccuracy, formatTimestamp, metricAccuracy, }; diff --git a/src/webui/src/static/interface.ts b/src/webui/src/static/interface.ts index ee7062feab..44789ab7f0 100644 --- a/src/webui/src/static/interface.ts +++ b/src/webui/src/static/interface.ts @@ -1,3 +1,5 @@ +// tslint:disable:no-any + // draw accuracy graph data interface interface TableObj { key: number; @@ -12,6 +14,19 @@ interface TableObj { endTime?: number; } +interface TableRecord { + key: string; + sequenceId: number; + startTime: number; + endTime?: number; + id: string; + duration: number; + status: string; + intermediateCount: number; + accuracy?: number; + latestAccuracy: string; // formatted string +} + interface SearchSpace { _value: Array; _type: string; @@ -32,26 +47,6 @@ interface Parameters { multiProgress?: number; } -interface Experiment { - id: string; - author: string; - revision?: number; - experName: string; - logDir?: string; - runConcurren: number; - maxDuration: number; - execDuration: number; - MaxTrialNum: number; - startTime: number; - endTime?: number; - trainingServicePlatform: string; - tuner: object; - assessor?: object; - advisor?: object; - clusterMetaData?: object; - logCollection?: string; -} - // trial accuracy interface AccurPoint { acc: number; @@ -74,21 +69,6 @@ interface TooltipForAccuracy { data: Array; } -interface TrialNumber { - succTrial: number; - failTrial: number; - stopTrial: number; - waitTrial: number; - runTrial: number; - unknowTrial: number; - totalCurrentTrial: number; -} - -interface TrialJob { - text: string; - value: string; -} - interface Dimobj { dim: number; name: string; @@ -108,10 +88,6 @@ interface ParaObj { parallelAxis: Array; } -interface FinalResult { - data: string; -} - interface Intermedia { name: string; // id type: string; @@ -119,13 +95,93 @@ interface Intermedia { hyperPara: object; // each trial hyperpara value } -interface ExperimentInfo { - platform: string; - optimizeMode: string; +interface MetricDataRecord { + timestamp: number; + trialJobId: string; + parameterId: string; + type: string; + sequence: number; + data: string; +} + +interface TrialJobInfo { + id: string; + sequenceId: number; + status: string; + startTime?: number; + endTime?: number; + hyperParameters?: string[]; + logPath?: string; + finalMetricData?: MetricDataRecord[]; + stderrPath?: string; +} + +interface ExperimentParams { + authorName: string; + experimentName: string; + description?: string; + trialConcurrency: number; + maxExecDuration: number; // seconds + maxTrialNum: number; + searchSpace: string; + trainingServicePlatform: string; + multiPhase?: boolean; + multiThread?: boolean; + versionCheck?: boolean; + logCollection?: string; + tuner?: { + className: string; + builtinTunerName?: string; + codeDir?: string; + classArgs?: any; + classFileName?: string; + checkpointDir: string; + gpuNum?: number; + includeIntermediateResults?: boolean; + }; + assessor?: { + className: string; + builtinAssessorName?: string; + codeDir?: string; + classArgs?: any; + classFileName?: string; + checkpointDir: string; + gpuNum?: number; + }; + advisor?: { + className: string; + builtinAdvisorName?: string; + codeDir?: string; + classArgs?: any; + classFileName?: string; + checkpointDir: string; + gpuNum?: number; + }; + clusterMetaData?: { + key: string; + value: string; + }[]; +} + +interface ExperimentProfile { + params: ExperimentParams; + id: string; + execDuration: number; + logDir?: string; + startTime?: number; + endTime?: number; + maxSequenceId: number; + revision: number; +} + +interface NNIManagerStatus { + status: string; + errors: string[]; } export { - TableObj, Parameters, Experiment, AccurPoint, TrialNumber, TrialJob, - DetailAccurPoint, TooltipForAccuracy, ParaObj, Dimobj, FinalResult, FinalType, - TooltipForIntermediate, SearchSpace, Intermedia, ExperimentInfo + TableObj, TableRecord, Parameters, ExperimentProfile, AccurPoint, + DetailAccurPoint, TooltipForAccuracy, ParaObj, Dimobj, FinalType, + TooltipForIntermediate, SearchSpace, Intermedia, MetricDataRecord, TrialJobInfo, + NNIManagerStatus, }; diff --git a/src/webui/src/static/model/experiment.ts b/src/webui/src/static/model/experiment.ts new file mode 100644 index 0000000000..e5e751c5c6 --- /dev/null +++ b/src/webui/src/static/model/experiment.ts @@ -0,0 +1,87 @@ +import axios from 'axios'; +import { MANAGER_IP } from '../const'; +import { ExperimentProfile, NNIManagerStatus } from '../interface'; + +function compareProfiles(profile1?: ExperimentProfile, profile2?: ExperimentProfile): boolean { + if (!profile1 || !profile2) { + return false; + } + const copy1 = Object.assign({}, profile1, { execDuration: undefined }); + const copy2 = Object.assign({}, profile2, { execDuration: undefined }); + return JSON.stringify(copy1) === JSON.stringify(copy2); +} + +class Experiment { + private profileField?: ExperimentProfile = undefined; + private statusField?: NNIManagerStatus = undefined; + + public async init(): Promise { + while (!this.profileField || !this.statusField) { + await this.update(); + } + } + + public async update(): Promise { + const profilePromise = axios.get(`${MANAGER_IP}/experiment`); + const statusPromise = axios.get(`${MANAGER_IP}/check-status`); + const [ profileResponse, statusResponse ] = await Promise.all([ profilePromise, statusPromise ]); + let updated = false; + if (statusResponse.status === 200) { + updated = JSON.stringify(this.statusField) === JSON.stringify(statusResponse.data); + this.statusField = statusResponse.data; + } + if (profileResponse.status === 200) { + updated = updated || compareProfiles(this.profileField, profileResponse.data); + this.profileField = profileResponse.data; + } + return updated; + } + + get profile(): ExperimentProfile { + if (!this.profileField) { + throw Error('Experiment profile not initialized'); + } + return this.profileField!; + } + + get trialConcurrency(): number { + return this.profile.params.trialConcurrency; + } + + get optimizeMode(): string { + const tuner = this.profile.params.tuner; + return (tuner && tuner.classArgs && tuner.classArgs.optimize_mode) ? tuner.classArgs.optimize_mode : 'unknown'; + } + + get trainingServicePlatform(): string { + return this.profile.params.trainingServicePlatform; + } + + get searchSpace(): object { + return JSON.parse(this.profile.params.searchSpace); + } + + get logCollectionEnabled(): boolean { + return !!(this.profile.params.logCollection && this.profile.params.logCollection !== 'none'); + } + + get multiPhase(): boolean { + return !!(this.profile.params.multiPhase); + } + + get status(): string { + if (!this.statusField) { + throw Error('Experiment status not initialized'); + } + return this.statusField!.status; + } + + get error(): string { + if (!this.statusField) { + throw Error('Experiment status not initialized'); + } + return this.statusField!.errors[0] || ''; + } +} + +export { Experiment }; diff --git a/src/webui/src/static/model/trial.ts b/src/webui/src/static/model/trial.ts new file mode 100644 index 0000000000..f2a6e13896 --- /dev/null +++ b/src/webui/src/static/model/trial.ts @@ -0,0 +1,195 @@ +import { MetricDataRecord, TrialJobInfo, TableObj, TableRecord, Parameters, FinalType } from '../interface'; +import { getFinal, formatAccuracy, metricAccuracy } from '../function'; + +class Trial implements TableObj { + private metricsInitialized: boolean = false; + private infoField: TrialJobInfo | undefined; + private intermediates: (MetricDataRecord | undefined)[] = [ ]; + private final: MetricDataRecord | undefined; + private finalAcc: number | undefined; + + constructor(info?: TrialJobInfo, metrics?: MetricDataRecord[]) { + this.infoField = info; + if (metrics) { + this.updateMetrics(metrics); + } + } + + public compareAccuracy(otherTrial: Trial): number | undefined { + if (!this.sortable || !otherTrial.sortable) { + return undefined; + } + return this.finalAcc! - otherTrial.finalAcc!; + } + + get info(): TrialJobInfo { + return this.infoField!; + } + + get intermediateMetrics(): MetricDataRecord[] { + const ret: MetricDataRecord[] = [ ]; + for (let i = 0; i < this.intermediates.length; i++) { + if (this.intermediates[i]) { + ret.push(this.intermediates[i]!); + } else { + break; + } + } + return ret; + } + + get accuracy(): number | undefined { + return this.finalAcc; + } + + get sortable(): boolean { + return this.finalAcc !== undefined && !isNaN(this.finalAcc); + } + + /* table obj start */ + + get tableRecord(): TableRecord { + const endTime = this.info.endTime || new Date().getTime(); + const duration = (endTime - this.info.startTime!) / 1000; + + return { + key: this.info.id, + sequenceId: this.info.sequenceId, + id: this.info.id, + startTime: this.info.startTime!, + endTime: this.info.endTime, + duration, + status: this.info.status, + intermediateCount: this.intermediates.length, + accuracy: this.finalAcc, + latestAccuracy: this.formatLatestAccuracy(), + }; + } + + get key(): number { + return this.info.sequenceId; + } + + get sequenceId(): number { + return this.info.sequenceId; + } + + get id(): string { + return this.info.id; + } + + get duration(): number { + const endTime = this.info.endTime || new Date().getTime(); + return (endTime - this.info.startTime!) / 1000; + } + + get status(): string { + return this.info.status; + } + + get acc(): FinalType | undefined { + return getFinal(this.info.finalMetricData); + } + + get description(): Parameters { + let ret: Parameters = { + parameters: { }, + intermediate: [ ], + multiProgress: 1 + }; + const tempHyper = this.info.hyperParameters; + if (tempHyper !== undefined) { + const getPara = JSON.parse(tempHyper[tempHyper.length - 1]).parameters; + ret.multiProgress = tempHyper.length; + if (typeof getPara === 'string') { + ret.parameters = JSON.parse(getPara); + } else { + ret.parameters = getPara; + } + } else { + ret.parameters = { error: 'This trial\'s parameters are not available.' }; + } + if (this.info.logPath !== undefined) { + ret.logPath = this.info.logPath; + } + + const mediate: number[] = [ ]; + for (const items of this.intermediateMetrics) { + if (typeof JSON.parse(items.data) === 'object') { + mediate.push(JSON.parse(items.data).default); + } else { + mediate.push(JSON.parse(items.data)); + } + } + ret.intermediate = mediate; + return ret; + } + + get color(): string | undefined { + return undefined; + } + + /* table obj end */ + + public initialized(): boolean { + return !!(this.infoField && this.metricsInitialized); + } + + public updateMetrics(metrics: MetricDataRecord[]): boolean { + // parameter `metrics` must contain all known metrics of this trial + this.metricsInitialized = true; + const prevMetricCnt = this.intermediates.length + (this.final ? 1 : 0); + if (metrics.length <= prevMetricCnt) { + return false; + } + for (const metric of metrics) { + if (metric.type === 'PERIODICAL') { + this.intermediates[metric.sequence] = metric; + } else { + this.final = metric; + this.finalAcc = metricAccuracy(metric); + } + } + return true; + } + + public updateLatestMetrics(metrics: MetricDataRecord[]): boolean { + // this method is effectively identical to `updateMetrics`, but has worse performance + this.metricsInitialized = true; + let updated = false; + for (const metric of metrics) { + if (metric.type === 'PERIODICAL') { + updated = updated || !this.intermediates[metric.sequence]; + this.intermediates[metric.sequence] = metric; + } else { + updated = updated || !this.final; + this.final = metric; + this.finalAcc = metricAccuracy(metric); + } + } + return updated; + } + + public updateTrialJobInfo(trialJobInfo: TrialJobInfo): boolean { + const same = (this.infoField && this.infoField.status === trialJobInfo.status); + this.infoField = trialJobInfo; + if (trialJobInfo.finalMetricData) { + this.final = trialJobInfo.finalMetricData[trialJobInfo.finalMetricData.length - 1]; + this.finalAcc = metricAccuracy(this.final); + } + return !same; + } + + public formatLatestAccuracy(): string { // TODO: this should be private + if (this.accuracy !== undefined) { + return `${formatAccuracy(this.accuracy)} (FINAL)`; + } else if (this.intermediates.length === 0) { + return '--'; + } else { + const latest = this.intermediates[this.intermediates.length - 1]!; + return `${formatAccuracy(metricAccuracy(latest))} (LATEST)`; + } + } +} + +export { Trial }; diff --git a/src/webui/src/static/model/trialmanager.ts b/src/webui/src/static/model/trialmanager.ts new file mode 100644 index 0000000000..aa5b3793f3 --- /dev/null +++ b/src/webui/src/static/model/trialmanager.ts @@ -0,0 +1,156 @@ +import axios from 'axios'; +import { MANAGER_IP, METRIC_GROUP_UPDATE_THRESHOLD, METRIC_GROUP_UPDATE_SIZE } from '../const'; +import { MetricDataRecord, TableRecord, TrialJobInfo } from '../interface'; +import { Trial } from './trial'; + +class TrialManager { + private trials: Map = new Map(); + private infoInitialized: boolean = false; + private metricInitialized: boolean = false; + private maxSequenceId: number = 0; + private doingBatchUpdate: boolean = false; + private batchUpdatedAfterReading: boolean = false; + + public async init(): Promise { + while (!this.infoInitialized || !this.metricInitialized) { + await this.update(); + } + } + + public async update(lastTime?: boolean): Promise { + const [ infoUpdated, metricUpdated ] = await Promise.all([ this.updateInfo(), this.updateMetrics(lastTime) ]); + return infoUpdated || metricUpdated; + } + + public getTrial(trialId: string): Trial { + return this.trials.get(trialId)!; + } + + public getTrials(trialIds: string[]): Trial[] { + return trialIds.map(trialId => this.trials.get(trialId)!); + } + + public table(trialIds: string[]): TableRecord[] { + return trialIds.map(trialId => this.trials.get(trialId)!.tableRecord); + } + + public toArray(): Trial[] { + const trials = Array.from(this.trials.values()).filter(trial => trial.initialized()); + return trials.sort((trial1, trial2) => trial1.sequenceId - trial2.sequenceId); + } + + public filter(callback: (trial: Trial) => boolean): Trial[] { + const trials = Array.from(this.trials.values()).filter(trial => trial.initialized() && callback(trial)); + return trials.sort((trial1, trial2) => trial1.sequenceId - trial2.sequenceId); + } + + public succeededTrials(): Trial[] { + return this.filter(trial => trial.status === 'SUCCEEDED'); + } + + public sort(): Trial[] { + return this.filter(trial => trial.sortable).sort((trial1, trial2) => trial1.compareAccuracy(trial2)!); + } + + public countStatus(): Map { + const cnt = new Map([ + [ 'UNKNOWN', 0 ], + [ 'WAITING', 0 ], + [ 'RUNNING', 0 ], + [ 'SUCCEEDED', 0 ], + [ 'FAILED', 0 ], + [ 'USER_CANCELED', 0 ], + [ 'SYS_CANCELED', 0 ], + [ 'EARLY_STOPPED', 0 ], + ]); + for (const trial of this.trials.values()) { + if (trial.initialized()) { + cnt.set(trial.info.status, cnt.get(trial.info.status)! + 1); + } + } + return cnt; + } + + private async updateInfo(): Promise { + const response = await axios.get(`${MANAGER_IP}/trial-jobs`); + let updated = false; + if (response.status === 200) { + for (const info of response.data as TrialJobInfo[]) { + if (this.trials.has(info.id)) { + updated = this.trials.get(info.id)!.updateTrialJobInfo(info) || updated; + } else { + this.trials.set(info.id, new Trial(info, undefined)); + updated = true; + } + this.maxSequenceId = Math.max(this.maxSequenceId, info.sequenceId); + } + this.infoInitialized = true; + } + return updated; + } + + private async updateMetrics(lastTime?: boolean): Promise { + if (this.trials.size < METRIC_GROUP_UPDATE_THRESHOLD || lastTime) { + return await this.updateAllMetrics(); + } else { + this.updateManyMetrics(); + const ret = (await this.updateLatestMetrics()) || this.batchUpdatedAfterReading; + this.batchUpdatedAfterReading = false; + return ret; + } + } + + private async updateAllMetrics(): Promise { + const response = await axios.get(`${MANAGER_IP}/metric-data`); + return (response.status === 200) && this.doUpdateMetrics(response.data as MetricDataRecord[], false); + } + + private async updateLatestMetrics(): Promise { + const response = await axios.get(`${MANAGER_IP}/metric-data-latest`); + return (response.status === 200) && this.doUpdateMetrics(response.data as MetricDataRecord[], true); + } + + private async updateManyMetrics(): Promise { + if (this.doingBatchUpdate) { + return; + } + this.doingBatchUpdate = true; + for (let i = 0; i < this.maxSequenceId; i += METRIC_GROUP_UPDATE_SIZE) { + const response = await axios.get(`${MANAGER_IP}/metric-data-range/${i}/${i + METRIC_GROUP_UPDATE_SIZE}`); + if (response.status === 200) { + const updated = this.doUpdateMetrics(response.data as MetricDataRecord[], false); + this.batchUpdatedAfterReading = this.batchUpdatedAfterReading || updated; + } + } + this.doingBatchUpdate = false; + } + + private doUpdateMetrics(allMetrics: MetricDataRecord[], latestOnly: boolean): boolean { + let updated = false; + for (const [ trialId, metrics ] of groupMetricsByTrial(allMetrics).entries()) { + if (this.trials.has(trialId)) { + const trial = this.trials.get(trialId)!; + updated = (latestOnly ? trial.updateLatestMetrics(metrics) : trial.updateMetrics(metrics)) || updated; + } else { + this.trials.set(trialId, new Trial(undefined, metrics)); + updated = true; + } + } + this.metricInitialized = true; + return updated; + } +} + +function groupMetricsByTrial(metrics: MetricDataRecord[]): Map { + const ret = new Map(); + for (const metric of metrics) { + if (ret.has(metric.trialJobId)) { + ret.get(metric.trialJobId)!.push(metric); + } else { + ret.set(metric.trialJobId, [ metric ]); + } + } + return ret; +} + +export { TrialManager }; diff --git a/src/webui/tslint.json b/src/webui/tslint.json index 322c3c28f5..1bfcc7878e 100644 --- a/src/webui/tslint.json +++ b/src/webui/tslint.json @@ -11,10 +11,6 @@ ], "ban": false, "class-name": true, - "comment-format": [ - true, - "check-space" - ], "curly": true, "eofline": false, "forin": true, @@ -40,19 +36,9 @@ "static-before-instance", "variables-before-functions" ], - "no-any": true, "no-arg": true, "no-bitwise": true, - "no-console": [ - true, - "log", - "error", - "debug", - "info", - "time", - "timeEnd", - "trace" - ], + "no-console": false, "no-consecutive-blank-lines": true, "no-construct": true, "no-debugger": true, @@ -64,7 +50,6 @@ "no-switch-case-fall-through": true, "no-trailing-whitespace": false, "no-unused-expression": true, - "no-use-before-declare": true, "one-line": [ true, "check-catch", @@ -123,4 +108,4 @@ "check-typecast" ] } -} \ No newline at end of file +} From 958efabf066b975a5cdcde293c9de33ec1430cb2 Mon Sep 17 00:00:00 2001 From: apatsekin Date: Thu, 26 Sep 2019 20:36:24 -0700 Subject: [PATCH 08/22] Typos in function invocation and exception msg (#1572) --- src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py b/src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py index 46aeadf1ea..67658a6f60 100644 --- a/src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py +++ b/src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py @@ -79,7 +79,7 @@ def trial_end(self, trial_job_id, success): self.completed_avg_history[trial_job_id].append(history_sum / cnt) self.running_history.pop(trial_job_id) else: - logger.warning('trial_end: trial_job_id does not in running_history') + logger.warning('trial_end: trial_job_id does not exist in running_history') def assess_trial(self, trial_job_id, trial_history): """assess_trial @@ -112,7 +112,7 @@ def assess_trial(self, trial_job_id, trial_history): logger.exception(error) except Exception as error: logger.warning('unrecognized exception in medianstop_assessor:') - logger.excpetion(error) + logger.exception(error) self._update_data(trial_job_id, num_trial_history) if self.high_better: From 4b5b6876f70c83c1c1e91d48e9084ef11c91ee84 Mon Sep 17 00:00:00 2001 From: Lijiao <35484733+lvybriage@users.noreply.github.com> Date: Fri, 27 Sep 2019 14:31:56 +0800 Subject: [PATCH 09/22] add search space key as table additional column (#1574) add search space key as table additional column --- src/webui/src/components/Modal/Compare.tsx | 7 +- src/webui/src/components/TrialsDetail.tsx | 1 - .../src/components/overview/Progress.tsx | 2 +- .../src/components/public-child/OpenRow.tsx | 21 ++--- .../trial-detail/DefaultMetricPoint.tsx | 2 +- .../components/trial-detail/Intermediate.tsx | 4 +- .../src/components/trial-detail/Para.tsx | 2 +- .../src/components/trial-detail/TableList.tsx | 79 +++++++++++-------- src/webui/src/static/const.ts | 8 +- 9 files changed, 66 insertions(+), 60 deletions(-) diff --git a/src/webui/src/components/Modal/Compare.tsx b/src/webui/src/components/Modal/Compare.tsx index 39719767c5..8a7278c966 100644 --- a/src/webui/src/components/Modal/Compare.tsx +++ b/src/webui/src/components/Modal/Compare.tsx @@ -26,11 +26,12 @@ class Compare extends React.Component { const idsList: Array = []; Object.keys(compareRows).map(item => { const temp = compareRows[item]; + const trial = TRIALS.getTrial(temp.id); trialIntermediate.push({ name: temp.id, - data: temp.description.intermediate, + data: trial.description.intermediate, type: 'line', - hyperPara: temp.description.parameters + hyperPara: trial.description.parameters }); idsList.push(temp.id); }); @@ -208,7 +209,7 @@ class Compare extends React.Component { > {this.intermediate()} - # Intermediate + # Intermediate result {this.initColumn()} diff --git a/src/webui/src/components/TrialsDetail.tsx b/src/webui/src/components/TrialsDetail.tsx index 9ea12ca154..b6f1897f29 100644 --- a/src/webui/src/components/TrialsDetail.tsx +++ b/src/webui/src/components/TrialsDetail.tsx @@ -117,7 +117,6 @@ class TrialsDetail extends React.Component const { columnList, changeColumn } = this.props; const source = TRIALS.filter(this.state.searchFilter); const trialIds = TRIALS.filter(this.state.searchFilter).map(trial => trial.id); - return (
diff --git a/src/webui/src/components/overview/Progress.tsx b/src/webui/src/components/overview/Progress.tsx index 88cc6e645b..398fbf2ff7 100644 --- a/src/webui/src/components/overview/Progress.tsx +++ b/src/webui/src/components/overview/Progress.tsx @@ -86,7 +86,7 @@ class Progressed extends React.Component { const percent = (EXPERIMENT.profile.execDuration / EXPERIMENT.profile.params.maxExecDuration) * 100; const remaining = convertTime(EXPERIMENT.profile.params.maxExecDuration - EXPERIMENT.profile.execDuration); const maxDuration = convertTime(EXPERIMENT.profile.params.maxExecDuration); - const maxTrialNum = convertTime(EXPERIMENT.profile.params.maxTrialNum); + const maxTrialNum = EXPERIMENT.profile.params.maxTrialNum; const execDuration = convertTime(EXPERIMENT.profile.execDuration); let errorContent; diff --git a/src/webui/src/components/public-child/OpenRow.tsx b/src/webui/src/components/public-child/OpenRow.tsx index f6d842c9fc..c990b03b94 100644 --- a/src/webui/src/components/public-child/OpenRow.tsx +++ b/src/webui/src/components/public-child/OpenRow.tsx @@ -59,7 +59,6 @@ class OpenRow extends React.Component { const { isShowFormatModal, formatStr } = this.state; const trialId = this.props.trialId; const trial = TRIALS.getTrial(trialId); - let isClick = false; const trialLink: string = `${MANAGER_IP}/trial-jobs/${trialId}`; const logPathRow = trial.info.logPath || 'This trial\'s log path is not available.'; const multiProgress = trial.info.hyperParameters === undefined ? 0 : trial.info.hyperParameters.length; @@ -76,7 +75,7 @@ class OpenRow extends React.Component {
For the entire parameter set, please refer to the following " {trialLink}". -
+
Current Phase: {multiProgress}. : @@ -87,18 +86,12 @@ class OpenRow extends React.Component { ? - { - isClick - ? - 
{JSON.stringify(trial.info.hyperParameters, null, 4)}
- : - true} // default expandNode - getItemString={() => ()} // remove the {} items - data={trial.info.hyperParameters} - /> - } + true} // default expandNode + getItemString={() => ()} // remove the {} items + data={trial.description.parameters} + />
); diff --git a/src/webui/src/components/trial-detail/Para.tsx b/src/webui/src/components/trial-detail/Para.tsx index 156aa78547..68b3814021 100644 --- a/src/webui/src/components/trial-detail/Para.tsx +++ b/src/webui/src/components/trial-detail/Para.tsx @@ -326,7 +326,7 @@ class Para extends React.Component { } else { Object.keys(dataSource).map(item => { const trial = dataSource[item]; - eachTrialParams.push(trial.description.parameters.error || ''); + eachTrialParams.push(trial.description.parameters || ''); // may be a succeed trial hasn't final result // all detail page may be break down if havn't if if (trial.acc !== undefined) { diff --git a/src/webui/src/components/trial-detail/TableList.tsx b/src/webui/src/components/trial-detail/TableList.tsx index 318f4ecbc5..2048e91fc0 100644 --- a/src/webui/src/components/trial-detail/TableList.tsx +++ b/src/webui/src/components/trial-detail/TableList.tsx @@ -7,6 +7,7 @@ const Option = Select.Option; const CheckboxGroup = Checkbox.Group; import { MANAGER_IP, trialJobStatus, COLUMN_INDEX, COLUMNPro } from '../../static/const'; import { convertDuration, formatTimestamp, intermediateGraphOption, killJob } from '../../static/function'; +import { TRIALS } from '../../static/datamodel'; import { TableRecord } from '../../static/interface'; import OpenRow from '../public-child/OpenRow'; import Compare from '../Modal/Compare'; @@ -159,8 +160,9 @@ class TableList extends React.Component { // checkbox for coloumn selectedColumn = (checkedValues: Array) => { - // 7: because have seven common column, "Intermediate count" is hidden by default - let count = 7; + // 9: because have nine common column, + // [Intermediate count, Start Time, End Time] is hidden by default + let count = 9; const want: Array = []; const finalKeys: Array = []; const wantResult: Array = []; @@ -174,7 +176,7 @@ class TableList extends React.Component { case 'Status': case 'Operation': case 'Default': - case 'Intermediate count': + case 'Intermediate result': break; default: finalKeys.push(checkedValues[m]); @@ -231,13 +233,12 @@ class TableList extends React.Component { // close Compare-modal hideCompareModal = () => { // close modal. clear select rows data, clear selected track - this.setState({ isShowCompareModal: false, selectedRowKeys: [], selectRows: [] }); + this.setState({ isShowCompareModal: false, selectedRowKeys: [], selectRows: [] }); } render() { const { pageSize, columnList } = this.props; const tableSource: Array = JSON.parse(JSON.stringify(this.props.tableSource)); - console.log('rerender table', tableSource); const { intermediateOption, modalVisible, isShowColumn, selectRows, isShowCompareModal, selectedRowKeys, intermediateOtherKeys } = this.state; const rowSelection = { @@ -248,25 +249,41 @@ class TableList extends React.Component { }; let showTitle = COLUMNPro; const showColumn: Array = []; + + // parameter as table column + const trialMess = TRIALS.getTrial(tableSource[0].id); + const trial = trialMess.description.parameters; + const parameterColumn: Array = Object.keys(trial); + const parameterStr: Array = []; + parameterColumn.forEach(value => { + parameterStr.push(`${value} (search space)`); + }); + showTitle = COLUMNPro.concat(parameterStr); + // only succeed trials have final keys if (tableSource.filter(record => record.status === 'SUCCEEDED').length >= 1) { const temp = tableSource.filter(record => record.status === 'SUCCEEDED')[0].accuracy; if (temp !== undefined && typeof temp === 'object') { - // concat default column and finalkeys - const item = Object.keys(temp); - // item: ['default', 'other-keys', 'maybe loss'] - if (item.length > 1) { - const want: Array = []; - item.forEach(value => { - if (value !== 'default') { - want.push(value); - } - }); - showTitle = COLUMNPro.concat(want); - } + // concat default column and finalkeys + const item = Object.keys(temp); + // item: ['default', 'other-keys', 'maybe loss'] + if (item.length > 1) { + const want: Array = []; + item.forEach(value => { + if (value !== 'default') { + want.push(value); + } + }); + showTitle = COLUMNPro.concat(want); + } } } for (const item of columnList) { + const paraColumn = item.match(/ \(search space\)$/); + let cc; + if (paraColumn !== null) { + cc = paraColumn.input; + } switch (item) { case 'Trial No.': showColumn.push(SequenceIdColumnConfig); @@ -286,7 +303,7 @@ class TableList extends React.Component { case 'Status': showColumn.push(StatusColumnConfig); break; - case 'Intermediate count': + case 'Intermediate result': showColumn.push(IntermediateCountColumnConfig); break; case 'Default': @@ -332,27 +349,22 @@ class TableList extends React.Component { }, }); break; - - case 'Intermediate result': + case (cc): + // remove SEARCH_SPACE title + const realItem = item.replace(' (search space)', ''); showColumn.push({ - title: 'Intermediate result', - dataIndex: 'intermediate', - key: 'intermediate', - width: '16%', + title: realItem, + dataIndex: item, + key: item, + width: '6%', render: (text: string, record: TableRecord) => { + const eachTrial = TRIALS.getTrial(record.id); return ( - + {eachTrial.description.parameters[realItem]} ); }, }); break; - default: // FIXME alert('Unexpected column type'); @@ -369,6 +381,7 @@ class TableList extends React.Component { expandedRowRender={this.openRow} dataSource={tableSource} className="commonTableStyle" + scroll={{x: 'max-content'}} pagination={pageSize > 0 ? { pageSize } : false} /> {/* Intermediate Result Modal */} @@ -495,7 +508,7 @@ const StatusColumnConfig: ColumnProps = { }; const IntermediateCountColumnConfig: ColumnProps = { - title: 'Intermediate count', + title: 'Intermediate result', dataIndex: 'intermediateCount', width: 86, render: (text, record) => ( diff --git a/src/webui/src/static/const.ts b/src/webui/src/static/const.ts index fbc300ad2e..368daa624c 100644 --- a/src/webui/src/static/const.ts +++ b/src/webui/src/static/const.ts @@ -38,11 +38,11 @@ const COLUMN_INDEX = [ index: 2 }, { - name: 'StartTime', + name: 'Start Time', index: 3 }, { - name: 'EndTime', + name: 'End Time', index: 4 }, { @@ -54,7 +54,7 @@ const COLUMN_INDEX = [ index: 6 }, { - name: 'Intermediate count', + name: 'Intermediate result', index: 7 }, { @@ -70,7 +70,7 @@ const COLUMN_INDEX = [ const COLUMN = ['Trial No.', 'ID', 'Duration', 'Status', 'Default', 'Operation']; // all choice column !dictory final const COLUMNPro = ['Trial No.', 'ID', 'Start Time', 'End Time', 'Duration', 'Status', -'Intermediate count', 'Default', 'Operation']; +'Intermediate result', 'Default', 'Operation']; export { MANAGER_IP, DOWNLOAD_IP, trialJobStatus, COLUMNPro, CONTROLTYPE, MONACO, COLUMN, COLUMN_INDEX, DRAWEROPTION, From f1210a9cd9be7143ea807b9a36cfc89109097b62 Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Mon, 30 Sep 2019 16:49:45 +0800 Subject: [PATCH 10/22] Add support for @nni.training_update in codegen (#1564) * add support for training_update in codegen and prettify code in nni annotation --- tools/nni_annotation/code_generator.py | 11 +++++---- .../nni_annotation/search_space_generator.py | 3 +-- .../nni_annotation/specific_code_generator.py | 24 ++++++++++++------- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/tools/nni_annotation/code_generator.py b/tools/nni_annotation/code_generator.py index 9b8797612b..3b7a22159e 100644 --- a/tools/nni_annotation/code_generator.py +++ b/tools/nni_annotation/code_generator.py @@ -22,6 +22,7 @@ import ast import astor + # pylint: disable=unidiomatic-typecheck def parse_annotation_mutable_layers(code, lineno, nas_mode): @@ -79,7 +80,8 @@ def parse_annotation_mutable_layers(code, lineno, nas_mode): fields['optional_inputs'] = True elif k.id == 'optional_input_size': assert not fields['optional_input_size'], 'Duplicated field: optional_input_size' - assert type(value) is ast.Num or type(value) is ast.List, 'Value of optional_input_size should be a number or list' + assert type(value) is ast.Num or type(value) is ast.List, \ + 'Value of optional_input_size should be a number or list' optional_input_size = value fields['optional_input_size'] = True elif k.id == 'layer_output': @@ -118,6 +120,7 @@ def parse_annotation_mutable_layers(code, lineno, nas_mode): nodes.append(node) return nodes + def parse_annotation(code): """Parse an annotation string. Return an AST Expr node. @@ -198,7 +201,7 @@ def convert_args_to_dict(call, with_lambda=False): if type(arg) in [ast.Str, ast.Num]: arg_value = arg else: - # if arg is not a string or a number, we use its source code as the key + # if arg is not a string or a number, we use its source code as the key arg_value = astor.to_source(arg).strip('\n"') arg_value = ast.Str(str(arg_value)) arg = make_lambda(arg) if with_lambda else arg @@ -311,7 +314,6 @@ def visit(self, node): return self._visit_children(node) - def _visit_string(self, node): string = node.value.s if string.startswith('@nni.'): @@ -325,7 +327,7 @@ def _visit_string(self, node): call_node.args.insert(0, ast.Str(s=self.nas_mode)) return expr - if string.startswith('@nni.report_intermediate_result') \ + if string.startswith('@nni.report_intermediate_result') \ or string.startswith('@nni.report_final_result') \ or string.startswith('@nni.get_next_parameter'): return parse_annotation(string[1:]) # expand annotation string to code @@ -341,7 +343,6 @@ def _visit_string(self, node): raise AssertionError('Unexpected annotation function') - def _visit_children(self, node): self.stack.append(None) self.generic_visit(node) diff --git a/tools/nni_annotation/search_space_generator.py b/tools/nni_annotation/search_space_generator.py index 04bca78584..f33fbab8f5 100644 --- a/tools/nni_annotation/search_space_generator.py +++ b/tools/nni_annotation/search_space_generator.py @@ -64,7 +64,6 @@ def generate_mutable_layer_search_space(self, args): 'optional_input_size': args[6].n if isinstance(args[6], ast.Num) else [args[6].elts[0].n, args[6].elts[1].n] } - def visit_Call(self, node): # pylint: disable=invalid-name self.generic_visit(node) @@ -108,7 +107,7 @@ def visit_Call(self, node): # pylint: disable=invalid-name else: # arguments of other functions must be literal number assert all(isinstance(ast.literal_eval(astor.to_source(arg)), numbers.Real) for arg in node.args), \ - 'Smart parameter\'s arguments must be number literals' + 'Smart parameter\'s arguments must be number literals' args = [ast.literal_eval(astor.to_source(arg)) for arg in node.args] key = self.module_name + '/' + name + '/' + func diff --git a/tools/nni_annotation/specific_code_generator.py b/tools/nni_annotation/specific_code_generator.py index 1e68ca7c61..5a43b5201b 100644 --- a/tools/nni_annotation/specific_code_generator.py +++ b/tools/nni_annotation/specific_code_generator.py @@ -28,6 +28,7 @@ para_cfg = None prefix_name = None + def parse_annotation_mutable_layers(code, lineno): """Parse the string of mutable layers in annotation. Return a list of AST Expr nodes @@ -102,6 +103,7 @@ def parse_annotation_mutable_layers(code, lineno): nodes.append(node) return nodes + def parse_annotation(code): """Parse an annotation string. Return an AST Expr node. @@ -182,7 +184,7 @@ def convert_args_to_dict(call, with_lambda=False): if type(arg) in [ast.Str, ast.Num]: arg_value = arg else: - # if arg is not a string or a number, we use its source code as the key + # if arg is not a string or a number, we use its source code as the key arg_value = astor.to_source(arg).strip('\n"') arg_value = ast.Str(str(arg_value)) arg = make_lambda(arg) if with_lambda else arg @@ -217,7 +219,7 @@ def test_variable_equal(node1, node2): if len(node1) != len(node2): return False return all(test_variable_equal(n1, n2) for n1, n2 in zip(node1, node2)) - + return node1 == node2 @@ -294,7 +296,6 @@ def visit(self, node): return self._visit_children(node) - def _visit_string(self, node): string = node.value.s if string.startswith('@nni.'): @@ -303,19 +304,27 @@ def _visit_string(self, node): return node # not an annotation, ignore it if string.startswith('@nni.get_next_parameter'): - deprecated_message = "'@nni.get_next_parameter' is deprecated in annotation due to inconvenience. Please remove this line in the trial code." + deprecated_message = "'@nni.get_next_parameter' is deprecated in annotation due to inconvenience. " \ + "Please remove this line in the trial code." print_warning(deprecated_message) - return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), args=[ast.Str(s='Get next parameter here...')], keywords=[])) + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast.Str(s='Get next parameter here...')], keywords=[])) + + if string.startswith('@nni.training_update'): + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast.Str(s='Training update here...')], keywords=[])) if string.startswith('@nni.report_intermediate_result'): module = ast.parse(string[1:]) arg = module.body[0].value.args[0] - return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), args=[ast.Str(s='nni.report_intermediate_result: '), arg], keywords=[])) + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast.Str(s='nni.report_intermediate_result: '), arg], keywords=[])) if string.startswith('@nni.report_final_result'): module = ast.parse(string[1:]) arg = module.body[0].value.args[0] - return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), args=[ast.Str(s='nni.report_final_result: '), arg], keywords=[])) + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast.Str(s='nni.report_final_result: '), arg], keywords=[])) if string.startswith('@nni.mutable_layers'): return parse_annotation_mutable_layers(string[1:], node.lineno) @@ -327,7 +336,6 @@ def _visit_string(self, node): raise AssertionError('Unexpected annotation function') - def _visit_children(self, node): self.stack.append(None) self.generic_visit(node) From 168d74e448d19aad13ad7cde670787248c827867 Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Tue, 8 Oct 2019 19:36:50 +0800 Subject: [PATCH 11/22] Add example for customized advisor and some refactoring (#1569) Add example for customized advisor and some refactoring --- docs/en_US/Tuner/CustomizeAdvisor.md | 24 ++- docs/en_US/sdk_reference.rst | 3 + .../mnist_keras_customized_advisor/config.yml | 20 +++ .../dummy_advisor.py | 95 ++++++++++++ .../mnist_keras.py | 137 ++++++++++++++++++ .../search_space.json | 5 + .../hyperband_advisor/hyperband_advisor.py | 69 +++++---- src/sdk/pynni/nni/msg_dispatcher.py | 18 ++- src/sdk/pynni/nni/msg_dispatcher_base.py | 87 ++++++++++- src/sdk/pynni/tests/test_assessor.py | 6 +- src/sdk/pynni/tests/test_tuner.py | 14 +- 11 files changed, 408 insertions(+), 70 deletions(-) create mode 100644 examples/tuners/mnist_keras_customized_advisor/config.yml create mode 100644 examples/tuners/mnist_keras_customized_advisor/dummy_advisor.py create mode 100644 examples/tuners/mnist_keras_customized_advisor/mnist_keras.py create mode 100644 examples/tuners/mnist_keras_customized_advisor/search_space.json diff --git a/docs/en_US/Tuner/CustomizeAdvisor.md b/docs/en_US/Tuner/CustomizeAdvisor.md index 8dcb8330d4..aefdd959ad 100644 --- a/docs/en_US/Tuner/CustomizeAdvisor.md +++ b/docs/en_US/Tuner/CustomizeAdvisor.md @@ -1,16 +1,12 @@ # **How To** - Customize Your Own Advisor -*Advisor targets the scenario that the automl algorithm wants the methods of both tuner and assessor. Advisor is similar to tuner on that it receives trial parameters request, final results, and generate trial parameters. Also, it is similar to assessor on that it receives intermediate results, trial's end state, and could send trial kill command. Note that, if you use Advisor, tuner and assessor are not allowed to be used at the same time.* +*Warning: API is subject to change in future releases.* -So, if user want to implement a customized Advisor, she/he only need to: +Advisor targets the scenario that the automl algorithm wants the methods of both tuner and assessor. Advisor is similar to tuner on that it receives trial parameters request, final results, and generate trial parameters. Also, it is similar to assessor on that it receives intermediate results, trial's end state, and could send trial kill command. Note that, if you use Advisor, tuner and assessor are not allowed to be used at the same time. -1. Define an Advisor inheriting from the MsgDispatcherBase class -1. Implement the methods with prefix `handle_` except `handle_request` -1. Configure your customized Advisor in experiment YAML config file +If a user want to implement a customized Advisor, she/he only needs to: -Here is an example: - -**1) Define an Advisor inheriting from the MsgDispatcherBase class** +**1. Define an Advisor inheriting from the MsgDispatcherBase class.** For example: ```python from nni.msg_dispatcher_base import MsgDispatcherBase @@ -20,13 +16,11 @@ class CustomizedAdvisor(MsgDispatcherBase): ... ``` -**2) Implement the methods with prefix `handle_` except `handle_request`** - -Please refer to the implementation of Hyperband ([src/sdk/pynni/nni/hyperband_advisor/hyperband_advisor.py](https://github.com/Microsoft/nni/tree/master/src/sdk/pynni/nni/hyperband_advisor/hyperband_advisor.py)) for how to implement the methods. +**2. Implement the methods with prefix `handle_` except `handle_request`**.. You might find [docs](https://nni.readthedocs.io/en/latest/sdk_reference.html#nni.msg_dispatcher_base.MsgDispatcherBase) for `MsgDispatcherBase` helpful. -**3) Configure your customized Advisor in experiment YAML config file** +**3. Configure your customized Advisor in experiment YAML config file.** -Similar to tuner and assessor. NNI needs to locate your customized Advisor class and instantiate the class, so you need to specify the location of the customized Advisor class and pass literal values as parameters to the \_\_init__ constructor. +Similar to tuner and assessor. NNI needs to locate your customized Advisor class and instantiate the class, so you need to specify the location of the customized Advisor class and pass literal values as parameters to the `__init__` constructor. ```yaml advisor: @@ -38,3 +32,7 @@ advisor: classArgs: arg1: value1 ``` + +## Example + +Here we provide an [example](../../../examples/tuners/mnist_keras_customized_advisor). diff --git a/docs/en_US/sdk_reference.rst b/docs/en_US/sdk_reference.rst index 5c3047ba37..64de1ee45e 100644 --- a/docs/en_US/sdk_reference.rst +++ b/docs/en_US/sdk_reference.rst @@ -50,6 +50,9 @@ Assessor Advisor ------------------------ +.. autoclass:: nni.msg_dispatcher_base.MsgDispatcherBase + :members: + .. autoclass:: nni.hyperband_advisor.hyperband_advisor.Hyperband :members: diff --git a/examples/tuners/mnist_keras_customized_advisor/config.yml b/examples/tuners/mnist_keras_customized_advisor/config.yml new file mode 100644 index 0000000000..0d8d987ac3 --- /dev/null +++ b/examples/tuners/mnist_keras_customized_advisor/config.yml @@ -0,0 +1,20 @@ +authorName: default +experimentName: example_customized_advisor +trialConcurrency: 4 +maxExecDuration: 1h +maxTrialNum: 200 +#choice: local, remote, pai +trainingServicePlatform: local +searchSpacePath: search_space.json +#choice: true, false +useAnnotation: false +advisor: + codeDir: . + classFileName: dummy_advisor.py + className: DummyAdvisor + classArgs: + k: 3 +trial: + command: python3 mnist_keras.py --epochs 100 --num_train 600 --num_test 100 + codeDir: . + gpuNum: 0 diff --git a/examples/tuners/mnist_keras_customized_advisor/dummy_advisor.py b/examples/tuners/mnist_keras_customized_advisor/dummy_advisor.py new file mode 100644 index 0000000000..5123b598fa --- /dev/null +++ b/examples/tuners/mnist_keras_customized_advisor/dummy_advisor.py @@ -0,0 +1,95 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import logging +from collections import defaultdict + +import json_tricks +import numpy as np +from nni import parameter_expressions as param +from nni.msg_dispatcher_base import MsgDispatcherBase +from nni.protocol import CommandType, send +from nni.utils import MetricType + +logger = logging.getLogger('customized_advisor') + + +class DummyAdvisor(MsgDispatcherBase): + """WARNING: Advisor API is subject to change in future releases. + + This advisor creates a new trial when validation accuracy of any one of the trials just dropped. + The trial is killed if the validation accuracy doesn't improve for at least k last-reported metrics. + To demonstrate the high flexibility of writing advisors, we don't use tuners or the standard definition of + search space. This is just a demo to customize an advisor. It's not intended to make any sense. + """ + def __init__(self, k=3): + super(DummyAdvisor, self).__init__() + self.k = k + self.random_state = np.random.RandomState() + + def handle_initialize(self, data): + logger.info("Advisor initialized: {}".format(data)) + self.handle_update_search_space(data) + self.parameters_count = 0 + self.parameter_best_metric = defaultdict(float) + self.parameter_cooldown = defaultdict(int) + send(CommandType.Initialized, '') + + def _send_new_trial(self): + self.parameters_count += 1 + new_trial = { + "parameter_id": self.parameters_count, + "parameters": { + "optimizer": param.choice(self.searchspace_json["optimizer"], self.random_state), + "learning_rate": param.loguniform(self.searchspace_json["learning_rate"][0], + self.searchspace_json["learning_rate"][1], + self.random_state) + }, + "parameter_source": "algorithm" + } + logger.info("New trial sent: {}".format(new_trial)) + send(CommandType.NewTrialJob, json_tricks.dumps(new_trial)) + + def handle_request_trial_jobs(self, data): + logger.info("Request trial jobs: {}".format(data)) + for _ in range(data): + self._send_new_trial() + + def handle_update_search_space(self, data): + logger.info("Search space update: {}".format(data)) + self.searchspace_json = data + + def handle_trial_end(self, data): + logger.info("Trial end: {}".format(data)) # do nothing + + def handle_report_metric_data(self, data): + logger.info("Metric reported: {}".format(data)) + if data['type'] == MetricType.REQUEST_PARAMETER: + raise ValueError("Request parameter not supported") + elif data["type"] == MetricType.PERIODICAL: + parameter_id = data["parameter_id"] + if data["value"] > self.parameter_best_metric[parameter_id]: + self.parameter_best_metric[parameter_id] = data["value"] + self.parameter_cooldown[parameter_id] = 0 + else: + self.parameter_cooldown[parameter_id] += 1 + logger.info("Accuracy dropped, cooldown {}, sending a new trial".format( + self.parameter_cooldown[parameter_id])) + self._send_new_trial() + if self.parameter_cooldown[parameter_id] >= self.k: + logger.info("Send kill signal to {}".format(data)) + send(CommandType.KillTrialJob, json_tricks.dumps(data["trial_job_id"])) diff --git a/examples/tuners/mnist_keras_customized_advisor/mnist_keras.py b/examples/tuners/mnist_keras_customized_advisor/mnist_keras.py new file mode 100644 index 0000000000..ee74a085ca --- /dev/null +++ b/examples/tuners/mnist_keras_customized_advisor/mnist_keras.py @@ -0,0 +1,137 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import argparse +import logging + +import os +import keras +import numpy as np +from keras import backend as K +from keras.callbacks import TensorBoard +from keras.datasets import mnist +from keras.layers import Conv2D, Dense, Flatten, MaxPooling2D +from keras.models import Sequential + +import nni + +LOG = logging.getLogger('mnist_keras') +K.set_image_data_format('channels_last') +TENSORBOARD_DIR = os.environ['NNI_OUTPUT_DIR'] + +H, W = 28, 28 +NUM_CLASSES = 10 + + +def create_mnist_model(hyper_params, input_shape=(H, W, 1), num_classes=NUM_CLASSES): + """ + Create simple convolutional model + """ + layers = [ + Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape), + Conv2D(64, (3, 3), activation='relu'), + MaxPooling2D(pool_size=(2, 2)), + Flatten(), + Dense(100, activation='relu'), + Dense(num_classes, activation='softmax') + ] + + model = Sequential(layers) + + if hyper_params['optimizer'] == 'Adam': + optimizer = keras.optimizers.Adam(lr=hyper_params['learning_rate']) + else: + optimizer = keras.optimizers.SGD(lr=hyper_params['learning_rate'], momentum=0.9) + model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy']) + + return model + + +def load_mnist_data(args): + """ + Load MNIST dataset + """ + (x_train, y_train), (x_test, y_test) = mnist.load_data() + + x_train = (np.expand_dims(x_train, -1).astype(np.float) / 255.)[:args.num_train] + x_test = (np.expand_dims(x_test, -1).astype(np.float) / 255.)[:args.num_test] + y_train = keras.utils.to_categorical(y_train, NUM_CLASSES)[:args.num_train] + y_test = keras.utils.to_categorical(y_test, NUM_CLASSES)[:args.num_test] + + LOG.debug('x_train shape: %s', (x_train.shape,)) + LOG.debug('x_test shape: %s', (x_test.shape,)) + + return x_train, y_train, x_test, y_test + + +class SendMetrics(keras.callbacks.Callback): + """ + Keras callback to send metrics to NNI framework + """ + + def on_epoch_end(self, epoch, logs={}): + """ + Run on end of each epoch + """ + LOG.debug(logs) + # Should this be val_acc or val_accuracy? Seems inconsistent behavior of Keras? + nni.report_intermediate_result(logs["val_accuracy"]) + + +def train(args, params): + """ + Train model + """ + x_train, y_train, x_test, y_test = load_mnist_data(args) + model = create_mnist_model(params) + + model.fit(x_train, y_train, batch_size=args.batch_size, epochs=args.epochs, verbose=1, + validation_data=(x_test, y_test), callbacks=[SendMetrics(), TensorBoard(log_dir=TENSORBOARD_DIR)]) + + _, acc = model.evaluate(x_test, y_test, verbose=0) + LOG.debug('Final result is: %d', acc) + nni.report_final_result(acc) + + +def generate_default_params(): + """ + Generate default hyper parameters + """ + return { + 'optimizer': 'Adam', + 'learning_rate': 0.001 + } + + +if __name__ == '__main__': + PARSER = argparse.ArgumentParser() + PARSER.add_argument("--batch_size", type=int, default=200, help="batch size", required=False) + PARSER.add_argument("--epochs", type=int, default=10, help="Train epochs", required=False) + PARSER.add_argument("--num_train", type=int, default=60000, + help="Number of train samples to be used, maximum 60000", required=False) + PARSER.add_argument("--num_test", type=int, default=10000, help="Number of test samples to be used, maximum 10000", + required=False) + + ARGS, UNKNOWN = PARSER.parse_known_args() + + # get parameters from tuner + RECEIVED_PARAMS = nni.get_next_parameter() + LOG.debug(RECEIVED_PARAMS) + PARAMS = generate_default_params() + PARAMS.update(RECEIVED_PARAMS) + # train + train(ARGS, PARAMS) diff --git a/examples/tuners/mnist_keras_customized_advisor/search_space.json b/examples/tuners/mnist_keras_customized_advisor/search_space.json new file mode 100644 index 0000000000..dadb04bc25 --- /dev/null +++ b/examples/tuners/mnist_keras_customized_advisor/search_space.json @@ -0,0 +1,5 @@ +{ + "README": "To demonstrate the flexibility, this search space does not follow the standard definition.", + "optimizer": ["Adam", "SGD"], + "learning_rate": [0.001, 0.1] +} diff --git a/src/sdk/pynni/nni/hyperband_advisor/hyperband_advisor.py b/src/sdk/pynni/nni/hyperband_advisor/hyperband_advisor.py index f596e5ea3b..7e376c6d9e 100644 --- a/src/sdk/pynni/nni/hyperband_advisor/hyperband_advisor.py +++ b/src/sdk/pynni/nni/hyperband_advisor/hyperband_advisor.py @@ -21,18 +21,17 @@ hyperband_advisor.py """ -import sys -import math import copy import logging -import numpy as np -import json_tricks +import math +import sys -from nni.protocol import CommandType, send +import json_tricks +import numpy as np +from nni.common import multi_phase_enabled from nni.msg_dispatcher_base import MsgDispatcherBase -from nni.common import init_logger, multi_phase_enabled +from nni.protocol import CommandType, send from nni.utils import NodeType, OptimizeMode, MetricType, extract_scalar_reward -import nni.parameter_expressions as parameter_expressions _logger = logging.getLogger(__name__) @@ -53,6 +52,7 @@ def create_parameter_id(): _next_parameter_id += 1 return _next_parameter_id - 1 + def create_bracket_parameter_id(brackets_id, brackets_curr_decay, increased_id=-1): """Create a full id for a specific bracket's hyperparameter configuration @@ -77,6 +77,7 @@ def create_bracket_parameter_id(brackets_id, brackets_curr_decay, increased_id=- increased_id]) return params_id + def json2parameter(ss_spec, random_state): """Randomly generate values for hyperparameters from hyperparameter space i.e., x. @@ -100,7 +101,7 @@ def json2parameter(ss_spec, random_state): _index = random_state.randint(len(_value)) chosen_params = json2parameter(ss_spec[NodeType.VALUE][_index], random_state) else: - chosen_params = eval('parameter_expressions.' + # pylint: disable=eval-used + chosen_params = eval('parameter_expressions.' + # pylint: disable=eval-used _type)(*(_value + [random_state])) else: chosen_params = dict() @@ -114,6 +115,7 @@ def json2parameter(ss_spec, random_state): chosen_params = copy.deepcopy(ss_spec) return chosen_params + class Bracket(): """A bracket in Hyperband, all the information of a bracket is managed by an instance of this class @@ -137,12 +139,12 @@ def __init__(self, s, s_max, eta, R, optimize_mode): self.bracket_id = s self.s_max = s_max self.eta = eta - self.n = math.ceil((s_max + 1) * (eta**s) / (s + 1) - _epsilon) # pylint: disable=invalid-name - self.r = R / eta**s # pylint: disable=invalid-name + self.n = math.ceil((s_max + 1) * (eta ** s) / (s + 1) - _epsilon) # pylint: disable=invalid-name + self.r = R / eta ** s # pylint: disable=invalid-name self.i = 0 - self.hyper_configs = [] # [ {id: params}, {}, ... ] - self.configs_perf = [] # [ {id: [seq, acc]}, {}, ... ] - self.num_configs_to_run = [] # [ n, n, n, ... ] + self.hyper_configs = [] # [ {id: params}, {}, ... ] + self.configs_perf = [] # [ {id: [seq, acc]}, {}, ... ] + self.num_configs_to_run = [] # [ n, n, n, ... ] self.num_finished_configs = [] # [ n, n, n, ... ] self.optimize_mode = OptimizeMode(optimize_mode) self.no_more_trial = False @@ -153,7 +155,7 @@ def is_completed(self): def get_n_r(self): """return the values of n and r for the next round""" - return math.floor(self.n / self.eta**self.i + _epsilon), math.floor(self.r * self.eta**self.i + _epsilon) + return math.floor(self.n / self.eta ** self.i + _epsilon), math.floor(self.r * self.eta ** self.i + _epsilon) def increase_i(self): """i means the ith round. Increase i by 1""" @@ -185,7 +187,6 @@ def set_config_perf(self, i, parameter_id, seq, value): else: self.configs_perf[i][parameter_id] = [seq, value] - def inform_trial_end(self, i): """If the trial is finished and the corresponding round (i.e., i) has all its trials finished, it will choose the top k trials for the next round (i.e., i+1) @@ -195,16 +196,17 @@ def inform_trial_end(self, i): i: int the ith round """ - global _KEY # pylint: disable=global-statement + global _KEY # pylint: disable=global-statement self.num_finished_configs[i] += 1 - _logger.debug('bracket id: %d, round: %d %d, finished: %d, all: %d', self.bracket_id, self.i, i, self.num_finished_configs[i], self.num_configs_to_run[i]) + _logger.debug('bracket id: %d, round: %d %d, finished: %d, all: %d', self.bracket_id, self.i, i, + self.num_finished_configs[i], self.num_configs_to_run[i]) if self.num_finished_configs[i] >= self.num_configs_to_run[i] \ - and self.no_more_trial is False: + and self.no_more_trial is False: # choose candidate configs from finished configs to run in the next round assert self.i == i + 1 this_round_perf = self.configs_perf[i] if self.optimize_mode is OptimizeMode.Maximize: - sorted_perf = sorted(this_round_perf.items(), key=lambda kv: kv[1][1], reverse=True) # reverse + sorted_perf = sorted(this_round_perf.items(), key=lambda kv: kv[1][1], reverse=True) # reverse else: sorted_perf = sorted(this_round_perf.items(), key=lambda kv: kv[1][1]) _logger.debug('bracket %s next round %s, sorted hyper configs: %s', self.bracket_id, self.i, sorted_perf) @@ -214,7 +216,7 @@ def inform_trial_end(self, i): for k in range(next_n): params_id = sorted_perf[k][0] params = self.hyper_configs[i][params_id] - params[_KEY] = next_r # modify r + params[_KEY] = next_r # modify r # generate new id increased_id = params_id.split('_')[-1] new_id = create_bracket_parameter_id(self.bracket_id, self.i, increased_id) @@ -223,7 +225,7 @@ def inform_trial_end(self, i): return [[key, value] for key, value in hyper_configs.items()] return None - def get_hyperparameter_configurations(self, num, r, searchspace_json, random_state): # pylint: disable=invalid-name + def get_hyperparameter_configurations(self, num, r, searchspace_json, random_state): # pylint: disable=invalid-name """Randomly generate num hyperparameter configurations from search space Parameters @@ -236,7 +238,7 @@ def get_hyperparameter_configurations(self, num, r, searchspace_json, random_sta list a list of hyperparameter configurations. Format: [[key1, value1], [key2, value2], ...] """ - global _KEY # pylint: disable=global-statement + global _KEY # pylint: disable=global-statement assert self.i == 0 hyperparameter_configs = dict() for _ in range(num): @@ -263,6 +265,7 @@ def _record_hyper_configs(self, hyper_configs): self.num_configs_to_run.append(len(hyper_configs)) self.increase_i() + class Hyperband(MsgDispatcherBase): """Hyperband inherit from MsgDispatcherBase rather than Tuner, because it integrates both tuner's functions and assessor's functions. This is an implementation that could fully leverage available resources, i.e., high parallelism. @@ -277,14 +280,15 @@ class Hyperband(MsgDispatcherBase): optimize_mode: str optimize mode, 'maximize' or 'minimize' """ + def __init__(self, R=60, eta=3, optimize_mode='maximize'): """B = (s_max + 1)R""" super(Hyperband, self).__init__() - self.R = R # pylint: disable=invalid-name + self.R = R # pylint: disable=invalid-name self.eta = eta - self.brackets = dict() # dict of Bracket - self.generated_hyper_configs = [] # all the configs waiting for run - self.completed_hyper_configs = [] # all the completed configs + self.brackets = dict() # dict of Bracket + self.generated_hyper_configs = [] # all the configs waiting for run + self.completed_hyper_configs = [] # all the completed configs self.s_max = math.floor(math.log(self.R, self.eta) + _epsilon) self.curr_s = self.s_max @@ -302,12 +306,11 @@ def __init__(self, R=60, eta=3, optimize_mode='maximize'): self.job_id_para_id_map = dict() def handle_initialize(self, data): - """data is search space - + """callback for initializing the advisor Parameters ---------- - data: int - number of trial jobs + data: dict + search space """ self.handle_update_search_space(data) send(CommandType.Initialized, '') @@ -348,14 +351,8 @@ def _get_one_trial_job(self): } return ret - def handle_update_search_space(self, data): """data: JSON object, which is search space - - Parameters - ---------- - data: int - number of trial jobs """ self.searchspace_json = data self.random_state = np.random.RandomState() diff --git a/src/sdk/pynni/nni/msg_dispatcher.py b/src/sdk/pynni/nni/msg_dispatcher.py index 1467b27695..64459e3a57 100644 --- a/src/sdk/pynni/nni/msg_dispatcher.py +++ b/src/sdk/pynni/nni/msg_dispatcher.py @@ -42,8 +42,9 @@ TODO: move this logic to NNI manager ''' + def _sort_history(history): - ret = [ ] + ret = [] for i, _ in enumerate(history): if i in history: ret.append(history[i]) @@ -51,17 +52,20 @@ def _sort_history(history): break return ret + # Tuner global variables _next_parameter_id = 0 _trial_params = {} '''key: trial job ID; value: parameters''' _customized_parameter_ids = set() + def _create_parameter_id(): global _next_parameter_id # pylint: disable=global-statement _next_parameter_id += 1 return _next_parameter_id - 1 + def _pack_parameter(parameter_id, params, customized=False, trial_job_id=None, parameter_index=None): _trial_params[parameter_id] = params ret = { @@ -77,6 +81,7 @@ def _pack_parameter(parameter_id, params, customized=False, trial_job_id=None, p ret['parameter_index'] = 0 return json_tricks.dumps(ret) + class MsgDispatcher(MsgDispatcherBase): def __init__(self, tuner, assessor=None): super(MsgDispatcher, self).__init__() @@ -123,7 +128,7 @@ def handle_update_search_space(self, data): def handle_import_data(self, data): """Import additional data for tuning - data: a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + data: a list of dictionaries, each of which has at least two keys, 'parameter' and 'value' """ self.tuner.import_data(data) @@ -154,7 +159,8 @@ def handle_report_metric_data(self, data): param = self.tuner.generate_parameters(param_id, trial_job_id=data['trial_job_id']) except NoMoreTrialError: param = None - send(CommandType.SendTrialJobParameter, _pack_parameter(param_id, param, trial_job_id=data['trial_job_id'], parameter_index=data['parameter_index'])) + send(CommandType.SendTrialJobParameter, _pack_parameter(param_id, param, trial_job_id=data['trial_job_id'], + parameter_index=data['parameter_index'])) else: raise ValueError('Data type not supported: {}'.format(data['type'])) @@ -188,7 +194,8 @@ def _handle_final_metric_data(self, data): customized = True else: customized = False - self.tuner.receive_trial_result(id_, _trial_params[id_], value, customized=customized, trial_job_id=data.get('trial_job_id')) + self.tuner.receive_trial_result(id_, _trial_params[id_], value, customized=customized, + trial_job_id=data.get('trial_job_id')) def _handle_intermediate_metric_data(self, data): """Call assessor to process intermediate results @@ -223,7 +230,8 @@ def _handle_intermediate_metric_data(self, data): _logger.debug('BAD, kill %s', trial_job_id) send(CommandType.KillTrialJob, json_tricks.dumps(trial_job_id)) # notify tuner - _logger.debug('env var: NNI_INCLUDE_INTERMEDIATE_RESULTS: [%s]', dispatcher_env_vars.NNI_INCLUDE_INTERMEDIATE_RESULTS) + _logger.debug('env var: NNI_INCLUDE_INTERMEDIATE_RESULTS: [%s]', + dispatcher_env_vars.NNI_INCLUDE_INTERMEDIATE_RESULTS) if dispatcher_env_vars.NNI_INCLUDE_INTERMEDIATE_RESULTS == 'true': self._earlystop_notify_tuner(data) else: diff --git a/src/sdk/pynni/nni/msg_dispatcher_base.py b/src/sdk/pynni/nni/msg_dispatcher_base.py index c98749e981..9680494c6c 100644 --- a/src/sdk/pynni/nni/msg_dispatcher_base.py +++ b/src/sdk/pynni/nni/msg_dispatcher_base.py @@ -18,7 +18,6 @@ # OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # ================================================================================================== -#import json_tricks import os import threading import logging @@ -39,7 +38,12 @@ QUEUE_LEN_WARNING_MARK = 20 _worker_fast_exit_on_terminate = True + class MsgDispatcherBase(Recoverable): + """This is where tuners and assessors are not defined yet. + Inherits this class to make your own advisor. + """ + def __init__(self): if multi_thread_enabled(): self.pool = ThreadPool() @@ -49,7 +53,8 @@ def __init__(self): self.default_command_queue = Queue() self.assessor_command_queue = Queue() self.default_worker = threading.Thread(target=self.command_queue_worker, args=(self.default_command_queue,)) - self.assessor_worker = threading.Thread(target=self.command_queue_worker, args=(self.assessor_command_queue,)) + self.assessor_worker = threading.Thread(target=self.command_queue_worker, + args=(self.assessor_command_queue,)) self.default_worker.start() self.assessor_worker.start() self.worker_exceptions = [] @@ -72,7 +77,8 @@ def run(self): if multi_thread_enabled(): result = self.pool.map_async(self.process_command_thread, [(command, data)]) self.thread_results.append(result) - if any([thread_result.ready() and not thread_result.successful() for thread_result in self.thread_results]): + if any([thread_result.ready() and not thread_result.successful() for thread_result in + self.thread_results]): _logger.debug('Caught thread exception') break else: @@ -112,7 +118,8 @@ def command_queue_worker(self, command_queue): def enqueue_command(self, command, data): """Enqueue command into command queues """ - if command == CommandType.TrialEnd or (command == CommandType.ReportMetricData and data['type'] == 'PERIODICAL'): + if command == CommandType.TrialEnd or ( + command == CommandType.ReportMetricData and data['type'] == 'PERIODICAL'): self.assessor_command_queue.put((command, data)) else: self.default_command_queue.put((command, data)) @@ -142,14 +149,14 @@ def process_command(self, command, data): _logger.debug('process_command: command: [{}], data: [{}]'.format(command, data)) command_handlers = { - # Tunner commands: + # Tuner commands: CommandType.Initialize: self.handle_initialize, CommandType.RequestTrialJobs: self.handle_request_trial_jobs, CommandType.UpdateSearchSpace: self.handle_update_search_space, CommandType.ImportData: self.handle_import_data, CommandType.AddCustomizedTrialJob: self.handle_add_customized_trial, - # Tunner/Assessor commands: + # Tuner/Assessor commands: CommandType.ReportMetricData: self.handle_report_metric_data, CommandType.TrialEnd: self.handle_trial_end, @@ -163,22 +170,88 @@ def handle_ping(self, data): pass def handle_initialize(self, data): + """Initialize search space and tuner, if any + This method is meant to be called only once for each experiment, after calling this method, + dispatcher should `send(CommandType.Initialized, '')`, to set the status of the experiment to be "INITIALIZED". + Parameters + ---------- + data: dict + search space + """ raise NotImplementedError('handle_initialize not implemented') def handle_request_trial_jobs(self, data): + """The message dispatcher is demanded to generate `data` trial jobs. + These trial jobs should be sent via `send(CommandType.NewTrialJob, json_tricks.dumps(parameter))`, + where `parameter` will be received by NNI Manager and eventually accessible to trial jobs as "next parameter". + Semantically, message dispatcher should do this `send` exactly `data` times. + + The JSON sent by this method should follow the format of + { + "parameter_id": 42 + "parameters": { + // this will be received by trial + }, + "parameter_source": "algorithm" // optional + } + Parameters + ---------- + data: int + number of trial jobs + """ raise NotImplementedError('handle_request_trial_jobs not implemented') def handle_update_search_space(self, data): - raise NotImplementedError('handle_update_search_space not implemented') + """This method will be called when search space is updated. + It's recommended to call this method in `handle_initialize` to initialize search space. + *No need to* notify NNI Manager when this update is done. + Parameters + ---------- + data: dict + search space + """ + raise NotImplementedError('handle_update_search_space not implemented') def handle_import_data(self, data): + """Import previous data when experiment is resumed. + Parameters + ---------- + data: list + a list of dictionaries, each of which has at least two keys, 'parameter' and 'value' + """ raise NotImplementedError('handle_import_data not implemented') def handle_add_customized_trial(self, data): + """Experimental API. Not recommended for usage. + """ raise NotImplementedError('handle_add_customized_trial not implemented') def handle_report_metric_data(self, data): + """Called when metric data is reported or new parameters are requested (for multiphase). + When new parameters are requested, this method should send a new parameter. + Parameters + ---------- + data: dict + a dict which contains 'parameter_id', 'value', 'trial_job_id', 'type', 'sequence'. + type: can be `MetricType.REQUEST_PARAMETER`, `MetricType.FINAL` or `MetricType.PERIODICAL`. + `REQUEST_PARAMETER` is used to request new parameters for multiphase trial job. In this case, + the dict will contain additional keys: `trial_job_id`, `parameter_index`. Refer to `msg_dispatcher.py` + as an example. + Raises + ------ + ValueError + Data type is not supported + """ raise NotImplementedError('handle_report_metric_data not implemented') def handle_trial_end(self, data): + """Called when the state of one of the trials is changed + Parameters + ---------- + data: dict + a dict with keys: trial_job_id, event, hyper_params. + trial_job_id: the id generated by training service. + event: the job’s state. + hyper_params: the string that is sent by message dispatcher during the creation of trials. + """ raise NotImplementedError('handle_trial_end not implemented') diff --git a/src/sdk/pynni/tests/test_assessor.py b/src/sdk/pynni/tests/test_assessor.py index 9f992377cd..f1b2913b7a 100644 --- a/src/sdk/pynni/tests/test_assessor.py +++ b/src/sdk/pynni/tests/test_assessor.py @@ -28,9 +28,9 @@ import json from unittest import TestCase, main +_trials = [] +_end_trials = [] -_trials = [ ] -_end_trials = [ ] class NaiveAssessor(Assessor): def assess_trial(self, trial_job_id, trial_history): @@ -47,12 +47,14 @@ def trial_end(self, trial_job_id, success): _in_buf = BytesIO() _out_buf = BytesIO() + def _reverse_io(): _in_buf.seek(0) _out_buf.seek(0) nni.protocol._out_file = _in_buf nni.protocol._in_file = _out_buf + def _restore_io(): _in_buf.seek(0) _out_buf.seek(0) diff --git a/src/sdk/pynni/tests/test_tuner.py b/src/sdk/pynni/tests/test_tuner.py index c1fd3594ee..f2330bd32c 100644 --- a/src/sdk/pynni/tests/test_tuner.py +++ b/src/sdk/pynni/tests/test_tuner.py @@ -32,7 +32,7 @@ class NaiveTuner(Tuner): def __init__(self): self.param = 0 - self.trial_results = [ ] + self.trial_results = [] self.search_space = None self.accept_customized_trials() @@ -57,12 +57,14 @@ def update_search_space(self, search_space): _in_buf = BytesIO() _out_buf = BytesIO() + def _reverse_io(): _in_buf.seek(0) _out_buf.seek(0) nni.protocol._out_file = _in_buf nni.protocol._in_file = _out_buf + def _restore_io(): _in_buf.seek(0) _out_buf.seek(0) @@ -70,7 +72,6 @@ def _restore_io(): nni.protocol._out_file = _out_buf - class TunerTestCase(TestCase): def test_tuner(self): _reverse_io() # now we are sending to Tuner's incoming stream @@ -94,21 +95,20 @@ def test_tuner(self): self.assertEqual(e.args[0], 'Unsupported command: CommandType.KillTrialJob') _reverse_io() # now we are receiving from Tuner's outgoing stream - self._assert_params(0, 2, [ ], None) - self._assert_params(1, 4, [ ], None) + self._assert_params(0, 2, [], None) + self._assert_params(1, 4, [], None) command, data = receive() # this one is customized data = json.loads(data) self.assertIs(command, CommandType.NewTrialJob) self.assertEqual(data['parameter_id'], 2) self.assertEqual(data['parameter_source'], 'customized') - self.assertEqual(data['parameters'], { 'param': -1 }) + self.assertEqual(data['parameters'], {'param': -1}) - self._assert_params(3, 6, [[1,4,11,False], [2,-1,22,True]], {'name':'SS0'}) + self._assert_params(3, 6, [[1, 4, 11, False], [2, -1, 22, True]], {'name': 'SS0'}) self.assertEqual(len(_out_buf.read()), 0) # no more commands - def _assert_params(self, parameter_id, param, trial_results, search_space): command, data = receive() self.assertIs(command, CommandType.NewTrialJob) From 3274ca3094bf05d4fb9d6afa554a2bd71001b2d8 Mon Sep 17 00:00:00 2001 From: chicm-ms <38930155+chicm-ms@users.noreply.github.com> Date: Wed, 9 Oct 2019 10:08:22 +0800 Subject: [PATCH 12/22] Fix multi phase integration test cases (#1591) * Fix multiphase it cases --- test/config_test/multi_phase/multi_phase_batch.test.yml | 4 ++-- test/config_test/multi_phase/multi_phase_grid.test.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/config_test/multi_phase/multi_phase_batch.test.yml b/test/config_test/multi_phase/multi_phase_batch.test.yml index 089cec9c04..1a488d368a 100644 --- a/test/config_test/multi_phase/multi_phase_batch.test.yml +++ b/test/config_test/multi_phase/multi_phase_batch.test.yml @@ -1,8 +1,8 @@ authorName: nni experimentName: default_test maxExecDuration: 5m -maxTrialNum: 8 -trialConcurrency: 4 +maxTrialNum: 2 +trialConcurrency: 2 searchSpacePath: ./search_space.json tuner: diff --git a/test/config_test/multi_phase/multi_phase_grid.test.yml b/test/config_test/multi_phase/multi_phase_grid.test.yml index 793224e40e..aeb0a0103d 100644 --- a/test/config_test/multi_phase/multi_phase_grid.test.yml +++ b/test/config_test/multi_phase/multi_phase_grid.test.yml @@ -1,8 +1,8 @@ authorName: nni experimentName: default_test maxExecDuration: 5m -maxTrialNum: 8 -trialConcurrency: 4 +maxTrialNum: 2 +trialConcurrency: 2 searchSpacePath: ./search_space.json tuner: From e93d2c25e9301c00bb62c749f815f0258517a218 Mon Sep 17 00:00:00 2001 From: liuzhe-lz <40699903+liuzhe-lz@users.noreply.github.com> Date: Wed, 9 Oct 2019 11:38:26 +0800 Subject: [PATCH 13/22] Merge model compression dev branch to master (#1571) * [Proposal] demo compressor (#1402) model compression * update doc for model compression (#1509) * Update Overview.md * Change Doc (#1510) * refactor compression sdk (#1562) * refactor compression sdk * bugfix * bugfix * update ut * Sync model compression doc and implementation (#1575) * update doc * formatting * bugfix * add import to examples --- azure-pipelines.yml | 10 + docs/en_US/Compressor/AutoCompression.md | 3 + docs/en_US/Compressor/Overview.md | 185 ++++++++++++++++++ docs/en_US/Compressor/Pruner.md | 132 +++++++++++++ docs/en_US/Compressor/Quantizer.md | 78 ++++++++ docs/img/agp_pruner.png | Bin 0 -> 8576 bytes .../model_compress/configure_example.yaml | 9 + examples/model_compress/main_tf_pruner.py | 130 ++++++++++++ examples/model_compress/main_tf_quantizer.py | 117 +++++++++++ examples/model_compress/main_torch_pruner.py | 95 +++++++++ .../model_compress/main_torch_quantizer.py | 87 ++++++++ src/sdk/pynni/nni/compression/__init__.py | 0 .../nni/compression/tensorflow/__init__.py | 3 + .../compression/tensorflow/builtin_pruners.py | 112 +++++++++++ .../tensorflow/builtin_quantizers.py | 74 +++++++ .../nni/compression/tensorflow/compressor.py | 152 ++++++++++++++ .../compression/tensorflow/default_layers.py | 8 + .../pynni/nni/compression/torch/__init__.py | 3 + .../nni/compression/torch/builtin_pruners.py | 131 +++++++++++++ .../compression/torch/builtin_quantizers.py | 76 +++++++ .../pynni/nni/compression/torch/compressor.py | 162 +++++++++++++++ .../nni/compression/torch/default_layers.py | 6 + src/sdk/pynni/tests/test_compressor.py | 116 +++++++++++ 23 files changed, 1689 insertions(+) create mode 100644 docs/en_US/Compressor/AutoCompression.md create mode 100644 docs/en_US/Compressor/Overview.md create mode 100644 docs/en_US/Compressor/Pruner.md create mode 100644 docs/en_US/Compressor/Quantizer.md create mode 100644 docs/img/agp_pruner.png create mode 100644 examples/model_compress/configure_example.yaml create mode 100644 examples/model_compress/main_tf_pruner.py create mode 100644 examples/model_compress/main_tf_quantizer.py create mode 100644 examples/model_compress/main_torch_pruner.py create mode 100644 examples/model_compress/main_torch_quantizer.py create mode 100644 src/sdk/pynni/nni/compression/__init__.py create mode 100644 src/sdk/pynni/nni/compression/tensorflow/__init__.py create mode 100644 src/sdk/pynni/nni/compression/tensorflow/builtin_pruners.py create mode 100644 src/sdk/pynni/nni/compression/tensorflow/builtin_quantizers.py create mode 100644 src/sdk/pynni/nni/compression/tensorflow/compressor.py create mode 100644 src/sdk/pynni/nni/compression/tensorflow/default_layers.py create mode 100644 src/sdk/pynni/nni/compression/torch/__init__.py create mode 100644 src/sdk/pynni/nni/compression/torch/builtin_pruners.py create mode 100644 src/sdk/pynni/nni/compression/torch/builtin_quantizers.py create mode 100644 src/sdk/pynni/nni/compression/torch/compressor.py create mode 100644 src/sdk/pynni/nni/compression/torch/default_layers.py create mode 100644 src/sdk/pynni/tests/test_compressor.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1563e4a0ee..a2932fd217 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,6 +10,11 @@ jobs: steps: - script: python3 -m pip install --upgrade pip setuptools --user displayName: 'Install python tools' + - script: | + python3 -m pip install torch==0.4.1 --user + python3 -m pip install torchvision==0.2.1 --user + python3 -m pip install tensorflow==1.12.0 --user + displayName: 'Install dependencies for integration' - script: | source install.sh displayName: 'Install nni toolkit via source code' @@ -50,6 +55,11 @@ jobs: steps: - script: python3 -m pip install --upgrade pip setuptools displayName: 'Install python tools' + - script: | + python3 -m pip install torch==0.4.1 --user + python3 -m pip install torchvision==0.2.1 --user + python3 -m pip install tensorflow --user + displayName: 'Install dependencies for integration' - script: | source install.sh displayName: 'Install nni toolkit via source code' diff --git a/docs/en_US/Compressor/AutoCompression.md b/docs/en_US/Compressor/AutoCompression.md new file mode 100644 index 0000000000..fc24f17211 --- /dev/null +++ b/docs/en_US/Compressor/AutoCompression.md @@ -0,0 +1,3 @@ +# Automatic Model Compression on NNI + +TBD. \ No newline at end of file diff --git a/docs/en_US/Compressor/Overview.md b/docs/en_US/Compressor/Overview.md new file mode 100644 index 0000000000..96453caad5 --- /dev/null +++ b/docs/en_US/Compressor/Overview.md @@ -0,0 +1,185 @@ +# Compressor +NNI provides an easy-to-use toolkit to help user design and use compression algorithms. It supports Tensorflow and PyTorch with unified interface. For users to compress their models, they only need to add several lines in their code. There are some popular model compression algorithms built-in in NNI. Users could further use NNI's auto tuning power to find the best compressed model, which is detailed in [Auto Model Compression](./AutoCompression.md). On the other hand, users could easily customize their new compression algorithms using NNI's interface, refer to the tutorial [here](#customize-new-compression-algorithms). + +## Supported algorithms +We have provided two naive compression algorithms and four popular ones for users, including three pruning algorithms and three quantization algorithms: + +|Name|Brief Introduction of Algorithm| +|---|---| +| [Level Pruner](./Pruner.md#level-pruner) | Pruning the specified ratio on each weight based on absolute values of weights | +| [AGP Pruner](./Pruner.md#agp-pruner) | To prune, or not to prune: exploring the efficacy of pruning for model compression. [Reference Paper](https://arxiv.org/abs/1710.01878)| +| [Sensitivity Pruner](./Pruner.md#sensitivity-pruner) | Learning both Weights and Connections for Efficient Neural Networks. [Reference Paper](https://arxiv.org/abs/1506.02626)| +| [Naive Quantizer](./Quantizer.md#naive-quantizer) | Quantize weights to default 8 bits | +| [QAT Quantizer](./Quantizer.md#qat-quantizer) | Quantization and Training of Neural Networks for Efficient Integer-Arithmetic-Only Inference. [Reference Paper](http://openaccess.thecvf.com/content_cvpr_2018/papers/Jacob_Quantization_and_Training_CVPR_2018_paper.pdf)| +| [DoReFa Quantizer](./Quantizer.md#dorefa-quantizer) | DoReFa-Net: Training Low Bitwidth Convolutional Neural Networks with Low Bitwidth Gradients. [Reference Paper](https://arxiv.org/abs/1606.06160)| + +## Usage of built-in compression algorithms + +We use a simple example to show how to modify your trial code in order to apply the compression algorithms. Let's say you want to prune all weight to 80% sparsity with Level Pruner, you can add the following three lines into your code before training your model ([here](https://github.com/microsoft/nni/tree/master/examples/model_compress) is complete code). + +Tensorflow code +```python +from nni.compression.tensorflow import LevelPruner +config_list = [{ 'sparsity': 0.8, 'op_types': 'default' }] +pruner = LevelPruner(config_list) +pruner(tf.get_default_graph()) +``` + +PyTorch code +```python +from nni.compression.torch import LevelPruner +config_list = [{ 'sparsity': 0.8, 'op_types': 'default' }] +pruner = LevelPruner(config_list) +pruner(model) +``` + +You can use other compression algorithms in the package of `nni.compression`. The algorithms are implemented in both PyTorch and Tensorflow, under `nni.compression.torch` and `nni.compression.tensorflow` respectively. You can refer to [Pruner](./Pruner.md) and [Quantizer](./Quantizer.md) for detail description of supported algorithms. + +The function call `pruner(model)` receives user defined model (in Tensorflow the model can be obtained with `tf.get_default_graph()`, while in PyTorch the model is the defined model class), and the model is modified with masks inserted. Then when you run the model, the masks take effect. The masks can be adjusted at runtime by the algorithms. + +When instantiate a compression algorithm, there is `config_list` passed in. We describe how to write this config below. + +### User configuration for a compression algorithm + +When compressing a model, users may want to specify the ratio for sparsity, to specify different ratios for different types of operations, to exclude certain types of operations, or to compress only a certain types of operations. For users to express these kinds of requirements, we define a configuration specification. It can be seen as a python `list` object, where each element is a `dict` object. In each `dict`, there are some keys commonly supported by NNI compression: + +* __op_types__: This is to specify what types of operations to be compressed. 'default' means following the algorithm's default setting. +* __op_names__: This is to specify by name what operations to be compressed. If this field is omitted, operations will not be filtered by it. +* __exclude__: Default is False. If this field is True, it means the operations with specified types and names will be excluded from the compression. + +There are also other keys in the `dict`, but they are specific for every compression algorithm. For example, some , some. + +The `dict`s in the `list` are applied one by one, that is, the configurations in latter `dict` will overwrite the configurations in former ones on the operations that are within the scope of both of them. + +A simple example of configuration is shown below: +```python +[ + { + 'sparsity': 0.8, + 'op_types': 'default' + }, + { + 'sparsity': 0.6, + 'op_names': ['op_name1', 'op_name2'] + }, + { + 'exclude': True, + 'op_names': ['op_name3'] + } +] +``` +It means following the algorithm's default setting for compressed operations with sparsity 0.8, but for `op_name1` and `op_name2` use sparsity 0.6, and please do not compress `op_name3`. + +### Other APIs + +Some compression algorithms use epochs to control the progress of compression, and some algorithms need to do something after every minibatch. Therefore, we provide another two APIs for users to invoke. One is `update_epoch`, you can use it as follows: + +Tensorflow code +```python +pruner.update_epoch(epoch, sess) +``` +PyTorch code +```python +pruner.update_epoch(epoch) +``` + +The other is `step`, it can be called with `pruner.step()` after each minibatch. Note that not all algorithms need these two APIs, for those that do not need them, calling them is allowed but has no effect. + +__[TODO]__ The last API is for users to export the compressed model. You will get a compressed model when you finish the training using this API. It also exports another file storing the values of masks. + +## Customize new compression algorithms + +To simplify writing a new compression algorithm, we design programming interfaces which are simple but flexible enough. There are interfaces for pruner and quantizer respectively. + +### Pruning algorithm + +If you want to write a new pruning algorithm, you can write a class that inherits `nni.compression.tensorflow.Pruner` or `nni.compression.torch.Pruner` depending on which framework you use. Then, override the member functions with the logic of your algorithm. + +```python +# This is writing a pruner in tensorflow. +# For writing a pruner in PyTorch, you can simply replace +# nni.compression.tensorflow.Pruner with +# nni.compression.torch.Pruner +class YourPruner(nni.compression.tensorflow.Pruner): + def __init__(self, config_list): + # suggest you to use the NNI defined spec for config + super().__init__(config_list) + + def bind_model(self, model): + # this func can be used to remember the model or its weights + # in member variables, for getting their values during training + pass + + def calc_mask(self, weight, config, **kwargs): + # weight is the target weight tensor + # config is the selected dict object in config_list for this layer + # kwargs contains op, op_type, and op_name + # design your mask and return your mask + return your_mask + + # note for pytorch version, there is no sess in input arguments + def update_epoch(self, epoch_num, sess): + pass + + # note for pytorch version, there is no sess in input arguments + def step(self, sess): + # can do some processing based on the model or weights binded + # in the func bind_model + pass +``` + +For the simpliest algorithm, you only need to override `calc_mask`. It receives each layer's weight and selected configuration, as well as op information. You generate the mask for this weight in this function and return. Then NNI applies the mask for you. + +Some algorithms generate mask based on training progress, i.e., epoch number. We provide `update_epoch` for the pruner to be aware of the training progress. + +Some algorithms may want global information for generating masks, for example, all weights of the model (for statistic information), model optimizer's information. NNI supports this requirement using `bind_model`. `bind_model` receives the complete model, thus, it could record any information (e.g., reference to weights) it cares about. Then `step` can process or update the information according to the algorithm. You can refer to [source code of built-in algorithms](https://github.com/microsoft/nni/tree/master/src/sdk/pynni/nni/compressors) for example implementations. + +### Quantization algorithm + +The interface for customizing quantization algorithm is similar to that of pruning algorithms. The only difference is that `calc_mask` is replaced with `quantize_weight`. `quantize_weight` directly returns the quantized weights rather than mask, because for quantization the quantized weights cannot be obtained by applying mask. + +``` +# This is writing a Quantizer in tensorflow. +# For writing a Quantizer in PyTorch, you can simply replace +# nni.compression.tensorflow.Quantizer with +# nni.compression.torch.Quantizer +class YourPruner(nni.compression.tensorflow.Quantizer): + def __init__(self, config_list): + # suggest you to use the NNI defined spec for config + super().__init__(config_list) + + def bind_model(self, model): + # this func can be used to remember the model or its weights + # in member variables, for getting their values during training + pass + + def quantize_weight(self, weight, config, **kwargs): + # weight is the target weight tensor + # config is the selected dict object in config_list for this layer + # kwargs contains op, op_type, and op_name + # design your quantizer and return new weight + return new_weight + + # note for pytorch version, there is no sess in input arguments + def update_epoch(self, epoch_num, sess): + pass + + # note for pytorch version, there is no sess in input arguments + def step(self, sess): + # can do some processing based on the model or weights binded + # in the func bind_model + pass + + # you can also design your method + def your_method(self, your_input): + #your code + + def bind_model(self, model): + #preprocess model +``` + +__[TODO]__ Will add another member function `quantize_layer_output`, as some quantization algorithms also quantize layers' output. + +### Usage of user customized compression algorithm + +__[TODO]__ ... diff --git a/docs/en_US/Compressor/Pruner.md b/docs/en_US/Compressor/Pruner.md new file mode 100644 index 0000000000..59db5b16c8 --- /dev/null +++ b/docs/en_US/Compressor/Pruner.md @@ -0,0 +1,132 @@ +Pruner on NNI Compressor +=== + +## Level Pruner + +This is one basic pruner: you can set a target sparsity level (expressed as a fraction, 0.6 means we will prune 60%). + +We first sort the weights in the specified layer by their absolute values. And then mask to zero the smallest magnitude weights until the desired sparsity level is reached. + +### Usage + +Tensorflow code +``` +from nni.compression.tensorflow import LevelPruner +config_list = [{ 'sparsity': 0.8, 'op_types': 'default' }] +pruner = LevelPruner(config_list) +pruner(model_graph) +``` + +PyTorch code +``` +from nni.compression.torch import LevelPruner +config_list = [{ 'sparsity': 0.8, 'op_types': 'default' }] +pruner = LevelPruner(config_list) +pruner(model) +``` + +#### User configuration for Level Pruner +* **sparsity:** This is to specify the sparsity operations to be compressed to + +*** + +## AGP Pruner +In [To prune, or not to prune: exploring the efficacy of pruning for model compression](https://arxiv.org/abs/1710.01878), authors Michael Zhu and Suyog Gupta provide an algorithm to prune the weight gradually. + +>We introduce a new automated gradual pruning algorithm in which the sparsity is increased from an initial sparsity value si (usually 0) to a final sparsity value sf over a span of n pruning steps, starting at training step t0 and with pruning frequency ∆t: +![](../../img/agp_pruner.png) +>The binary weight masks are updated every ∆t steps as the network is trained to gradually increase the sparsity of the network while allowing the network training steps to recover from any pruning-induced loss in accuracy. In our experience, varying the pruning frequency ∆t between 100 and 1000 training steps had a negligible impact on the final model quality. Once the model achieves the target sparsity sf , the weight masks are no longer updated. The intuition behind this sparsity function in equation + +### Usage +You can prune all weight from %0 to 80% sparsity in 10 epoch with the code below. + +First, you should import pruner and add mask to model. + +Tensorflow code +```python +from nni.compression.tensorflow import AGP_Pruner +config_list = [{ + 'initial_sparsity': 0, + 'final_sparsity': 0.8, + 'start_epoch': 1, + 'end_epoch': 10, + 'frequency': 1, + 'op_types': 'default' +}] +pruner = AGP_Pruner(config_list) +pruner(tf.get_default_graph()) +``` +PyTorch code +```python +from nni.compression.torch import AGP_Pruner +config_list = [{ + 'initial_sparsity': 0, + 'final_sparsity': 0.8, + 'start_epoch': 1, + 'end_epoch': 10, + 'frequency': 1, + 'op_types': 'default' +}] +pruner = AGP_Pruner(config_list) +pruner(model) +``` + +Second, you should add code below to update epoch number when you finish one epoch in your training code. + +Tensorflow code +```python +pruner.update_epoch(epoch, sess) +``` +PyTorch code +```python +pruner.update_epoch(epoch) +``` +You can view example for more information + +#### User configuration for AGP Pruner +* **initial_sparsity:** This is to specify the sparsity when compressor starts to compress +* **final_sparsity:** This is to specify the sparsity when compressor finishes to compress +* **start_epoch:** This is to specify the epoch number when compressor starts to compress +* **end_epoch:** This is to specify the epoch number when compressor finishes to compress +* **frequency:** This is to specify every *frequency* number epochs compressor compress once + +*** + +## Sensitivity Pruner +In [Learning both Weights and Connections for Efficient Neural Networks](https://arxiv.org/abs/1506.02626), author Song Han and provide an algorithm to find the sensitivity of each layer and set the pruning threshold to each layer. + +>We used the sensitivity results to find each layer’s threshold: for example, the smallest threshold was applied to the most sensitive layer, which is the first convolutional layer... The pruning threshold is chosen as a quality parameter multiplied by the standard deviation of a layer’s weights + +### Usage +You can prune weight step by step and reach one target sparsity by Sensitivity Pruner with the code below. + +Tensorflow code +```python +from nni.compression.tensorflow import SensitivityPruner +config_list = [{ 'sparsity':0.8, 'op_types': 'default' }] +pruner = SensitivityPruner(config_list) +pruner(tf.get_default_graph()) +``` +PyTorch code +```python +from nni.compression.torch import SensitivityPruner +config_list = [{ 'sparsity':0.8, 'op_types': 'default' }] +pruner = SensitivityPruner(config_list) +pruner(model) +``` +Like AGP Pruner, you should update mask information every epoch by adding code below + +Tensorflow code +```python +pruner.update_epoch(epoch, sess) +``` +PyTorch code +```python +pruner.update_epoch(epoch) +``` +You can view example for more information + +#### User configuration for Sensitivity Pruner +* **sparsity:** This is to specify the sparsity operations to be compressed to + +*** diff --git a/docs/en_US/Compressor/Quantizer.md b/docs/en_US/Compressor/Quantizer.md new file mode 100644 index 0000000000..be91dcc339 --- /dev/null +++ b/docs/en_US/Compressor/Quantizer.md @@ -0,0 +1,78 @@ +Quantizer on NNI Compressor +=== + +## Naive Quantizer + +We provide Naive Quantizer to quantizer weight to default 8 bits, you can use it to test quantize algorithm without any configure. + +### Usage +tensorflow +```python +nni.compressors.tensorflow.NaiveQuantizer()(model_graph) +``` +pytorch +```python +nni.compressors.torch.NaiveQuantizer()(model) +``` + +*** + +## QAT Quantizer +In [Quantization and Training of Neural Networks for Efficient Integer-Arithmetic-Only Inference](http://openaccess.thecvf.com/content_cvpr_2018/papers/Jacob_Quantization_and_Training_CVPR_2018_paper.pdf), authors Benoit Jacob and Skirmantas Kligys provide an algorithm to quantize the model with training. + +>We propose an approach that simulates quantization effects in the forward pass of training. Backpropagation still happens as usual, and all weights and biases are stored in floating point so that they can be easily nudged by small amounts. The forward propagation pass however simulates quantized inference as it will happen in the inference engine, by implementing in floating-point arithmetic the rounding behavior of the quantization scheme +>* Weights are quantized before they are convolved with the input. If batch normalization (see [17]) is used for the layer, the batch normalization parameters are “folded into” the weights before quantization. +>* Activations are quantized at points where they would be during inference, e.g. after the activation function is applied to a convolutional or fully connected layer’s output, or after a bypass connection adds or concatenates the outputs of several layers together such as in ResNets. + + +### Usage +You can quantize your model to 8 bits with the code below before your training code. + +Tensorflow code +```python +from nni.compressors.tensorflow import QAT_Quantizer +config_list = [{ 'q_bits': 8, 'op_types': 'default' }] +quantizer = QAT_Quantizer(config_list) +quantizer(tf.get_default_graph()) +``` +PyTorch code +```python +from nni.compressors.torch import QAT_Quantizer +config_list = [{ 'q_bits': 8, 'op_types': 'default' }] +quantizer = QAT_Quantizer(config_list) +quantizer(model) +``` + +You can view example for more information + +#### User configuration for QAT Quantizer +* **q_bits:** This is to specify the q_bits operations to be quantized to + + +*** + +## DoReFa Quantizer +In [DoReFa-Net: Training Low Bitwidth Convolutional Neural Networks with Low Bitwidth Gradients](https://arxiv.org/abs/1606.06160), authors Shuchang Zhou and Yuxin Wu provide an algorithm named DoReFa to quantize the weight, activation and gradients with training. + +### Usage +To implement DoReFa Quantizer, you can add code below before your training code + +Tensorflow code +```python +from nni.compressors.tensorflow import DoReFaQuantizer +config_list = [{ 'q_bits': 8, 'op_types': 'default' }] +quantizer = DoReFaQuantizer(config_list) +quantizer(tf.get_default_graph()) +``` +PyTorch code +```python +from nni.compressors.torch import DoReFaQuantizer +config_list = [{ 'q_bits': 8, 'op_types': 'default' }] +quantizer = DoReFaQuantizer(config_list) +quantizer(model) +``` + +You can view example for more information + +#### User configuration for QAT Quantizer +* **q_bits:** This is to specify the q_bits operations to be quantized to diff --git a/docs/img/agp_pruner.png b/docs/img/agp_pruner.png new file mode 100644 index 0000000000000000000000000000000000000000..889f42e7647e705f773b58141b2ef6c3067d3e95 GIT binary patch literal 8576 zcmd6tRa6{7x2Pe(onV7|kl^kFC%C%}9w0D4@WEvugF}EMI3xsz;LZ#|0s%sR!EFc< z++FVEKWD9bU+(K!=b^iIt*)-#yK2`i`6@wAM~x7V1`iDljqsJavH=>}lW3G|f`f&+ zH$KUgMm^Ad4b&9TYR2gPq7oP`3fc;2X!R-hcR);38W*H)?u&*-)c5Z~AN2a_h=#^Y z@k&|2D9~nq30Kc>YZ(i~#qzA0V#=aau^Pc=TaBRT|GesH&4-L)YJOMEwA0l@(ZpRG zHT(sWbc%bLyO6VWO~{_hOu>?xWbTX`YNg^hi{+@k`RC^;JIMtb8#8X?UAVA|D$&lX zs+^6?b+M*AM~L!v7MO8fUk0_u%; zw>#XqYDs-5;h7OP6%|jQNK0A)w-~Y-#0A!C`dhqu^bXM7q0a`N31V=k`3G`?`Ni8qhiG9H@;O{);B@ckG}_CPATcC%E<8s%r-p`!Tgj%Z+24h-S)IU<$F^&bA**vNAj+Ez$gvVk zR$b_0Hab}A?+e@}XhsK;{9sJU>~{r~EzVjW7OHP{Il!JW#Dk}fcRdw&srdk?{Gn5E z>PO#ait+O(oAMZM@rs~HKzpi4`=qelAr*xiS+>QlXq;@|FUCI+WXHJbT4NOVHpWJc z))%5<^B;TnK0#@04q`}(mpBR1by99GO~4br#F3LLk#hkv3Vm28ZJklxCnef^B+etS7ady$mHMeCv|H5FrHxw$}87g~pO6g_wBdbYX3<^)Z-d+k}(CI7= z=tr56@Xw6Ilw4b$i9V1Zd|0_sgdsYUz7KfxnU08{N)=7Z6VhU!%~hL1jy#a;U}*kF$rXV|VlWVqmMzlE${gd$gcn_UYJYkJ zbqYB981Up2~(V&7yk`gfWt&IwE&SSG&_svGoV0l_aV#J$u{}A?xBYd5T`v&e;i+oh# zJ>vZl?@cEJICXK8M-L*?qzzT%TjEmadS3lHlq6>EFE&Q?HxaB@7*W}iHc6FYd3W?? zjJCGVc(HBo@cp+dFS)D^PnF$87#}}~^GG{JrYUtKkcc|ypKTL= zXLBs9qaTyUi>Al5jQ?~nHcMs8ZBazX4I1AH@#sEN9D6yWgw$nJ8h520>mjY)!O1bY z(xJy)h~SLD-h;zv-P~L_ygR1CA8B7S;(tghO4Rb?rj%pqZr_XbnunO=}MHW zk_h3UIfmHSx8B@y^!UzqI#G?J89_7)_x2N6);8xXn1VeW z4#%w%W`BICKD)_)ISKp$GNp{)`cGENgtQK)e+cM$gZo+bbkn!I|B5D$3Pmh$%MPWM z*(Zf2Nn^LHneBFo8$q}f{8foGjBS}`GiUdO)aNfS3Esf<3d*k~kMgGQCy9E;$$X<{z~M8L!1J95Qw(C3Hko3Se2YboUx((!6Z51?VPS0-$N*uR#JOI`32_ z%q=_1DpT}37~EFQEokdiB;0Qq-kz%gv!5N%Yi)*UZ42V;+E35;rEYADv^ zr4z$Pms8&Ic6~eYjRKxFwd}1F(Bfw+SWkq!R#u8ni&xl;?V3I53zp!gPU^P4bztur zs~Xf?i~*MT7l!>j5TB_F{z^e4_nxEL1exfk`awDKeuR{R=}F`*p675MH#KVJr?r@<{*dy{Tl4L~_KNz|e3VUZaHVXDc!`$ zD)ZW4(YBiaaZl5(fIPv&+>K;Ap-&_YW4n`Hq_1X@T`Q;7!@Q&oY5pBOgw~XA4cQNk z+IYsdqOKZQl})1$Xp}Nyx*jm}CelR3GR`X5U7`g&RiUr%K)rMYE^i1rc;2z()}6SD z359Z$K$7KDNA#r4ygj)R38;#!p{Bf2aJft}%NU%p_YxoTa94iF7M93c#LdG@h841W z4SsMERA^HuW?R4;zC?C|XNy;W!fz${)ER|aCo8)j=;iV$Vzz{G>9uY=6Myf<)e;TQ zWaPjL*?5Le(@7XZs9y7LKqH&30yp2F%XZZt{WRG9-2Oo$w|TR0x6tp(FavI&uOMdK?{2m;x93H&aqpbd@t4)Gg$;HhxF=Ms#Pw!14 zxJwOHZ8||iR&Nib0JO;Yt!+n!yr{}kZnKYm{g#IWkn>9&kVVJSt^BFHj1BdaGdQu% z7|5cD?uB^kdf*-O`If;sR5>Q7-EMq(W~l8TRuq9>cX2t0w*qd({M(uhq6%QK63_}K zq0{-!$}C9TR!y)rM7c6xqKRLH)1Fe?ynM~J9N5aK-UEs02q0M#BDjgB1F^`VC;VtD zUa5o>JF(R^W8(s=3x!P>M#`co&r0V%4&)h%2wLKiw6IRk%T#0;~dEHEA>~B=Pozu^Y%Fo>5Pq=eMpi{D{9R= zvxvlxD+Z=Qdcv+S3hpzR1XquJ;j4}fM9i1@B~?fY1065;K&ngeN;3z0cI|5U!kbbq zy4n_-&xREBgh_=@^CoJNdFiub6AEc~MnZj)QBjFL~@ zX&B)Xi(RnuNfGv?20>CSe2>GU)gV;8%8#;Ir$J0t3H`eFS&Ki0OS35^|@W}c|$N^{pyRcuv`RJyPN;m{~Dp8H)x3e=6J@ztL1))W~e(H(s z7q22mJ!L$YpY=tN09J{1VZj3VYj}@JkvMqTB=?wpdUc89zcvLpTjhf06WlO&Gp%72 zk??tiTgs+rkD^r=3nvNs0Rm9vl1N?iAVCg@`4uyU`0%RXk~za0ARqjY0C^A=1bH&q zD+V~Iy(E7a<9azj3hVyRs*Ci{`>CWm`(K5c$Ns0#&8BFd%%_*cRj6t5&ZnkVe!w~z zJR+s4!(+-0Nt<^>Jbh_{b|m)2R@6sUtvpGe7W>4=>~XyvQ3RqH@b zqa9Dn9DWl~TiMwOFRZp9q^a|D!tv%=k6ocV-WK3`8Pfn@l(*xm2H3rb&^D`d`Lcua zMCaFA1%Qa3;<{1^n;WK9JK_LxBRC7eFKeK@|^9zWUMgdY|Zqy!hVWk~K%!e*;Ajhqrb zm_;5C+NKoL_ut|<=6|#BOdnA{4r=)1W(jcA2QO2cLJC&yTbccqb+FNGC$hC$+w}4qJ+{Ju|+)$1;>nBIxoJQpRWzS=5bhHx0GCd~L84IfoN+Ce6>H+^-hJlr}&eP7B8VBk>l6ShT#Ho-B0 z4Krq@p5rpIm;z{}fTs$ETUpg6tGKJ&bB}h`K7R_PqD1nK;-9#b9@evF+Q@rqf!)W! zP@mDC!vnhrKI|xnx$f<#wmYgr+6)X|`V~p{GEiBZg|D*=%#qVoqKR-VkE#Fm=EE_j zGEM4x7hgPoerJU-S`eZA+rRkRMUmaYOkYiL>ISm8fm$Nt{^Qv}C|Mib{c`;PwP1PD ztMi!q%`Z(+ej5C>-*4O70e0a<8+sN(`G*^qp@N5xs%xJL1OJ$CUc|EyPfrsFnP2&> zVQ<$=Vp_jFvlCflPPbWe{?t!TQT-mME<^GcAzA6F-gYyvpuE#qK! zIGHt99e*Rwz@m2e172|6-M2g3PEmdp+Z&Eoi`!MVnps+*4!a1mjwf#4E!3o<-=-#n zgzBX$)>d;P!E3HwGkd2AwtFTXN`FpGi1a>Zdr^!nT=XC9zp9RdAxngf8L?g0t$J{8_MEkM54WQ;E z%QZp3yittAkJn`IN%p)T0t0^n2rS!E+xC4uCQkkwwy&;;OzBC34A2vaW?oj61-TM3 z@L2YEzAGXo)z~@4n!@;?r%~c}vYUo4+lAx6N{WjTeR7v^V}ybhLQQx*0l!Uoht}FcwOxp5Z_4Jd#QsLP|AO zFc@T>g`d`%#j|@QgOqoWRVYk2^XU^W3F&j(k!lT!VbU)G!YIJnKfB>;4bFv z6JjyFHa*XVtdi(*AV6SZKQgxHcf}hcEh_+dm9$WU^xN-eZ=D(197^*U9=vwtMde?} z6MP#Yu6r{0le)!8#;8p^SHN4TD&2%7X4#rA%sd2IotP* z{f~pC2k*^Zn3bI&#iMrfRuz1%W8)wcAI2}m;~!rGZggg&*+iF`jY+VshvXV)DAy54 z`;zE~Q5Ec*#JQxib^~5{DEmDJOf4&o<9git1&DYt;NZKCIpwAGQF|)iXSM3Yg8JmT zO}k>jJ=O&uGQ-AEJ5j!HVKfK-vX(%fKQ09@H>kJS!kK0-%#px<28@f9Lgbbz1Zx{oQbj^CrHMS;6bBog!{9=`tcz_n-~Lg_=q>**#vq9w%Z`w6`TYO{<@w zyOE;Pl3zTV);^y$7C_J@o)>$&W}-SIDR80nUJtkUJ}QlE|GC*YHbS>it@Rttif=8< zI|ALAo=iB-#R-JsJ5fW`Wxj?6|`^X6!RwIB8q(?3@5AoRGRz8V34+pe8#kEFy|=dR>|YbWu;dW z0%nxWIB{KX+^~vk4e8etiG#IZ=nASpeAd-)p|T4q4?#ZlN>m>yb*K$GomZe94_LPL zpmSsTTpMvx&O~6cw)l-$5WePGGB3x#aPaY+d5_4{Ht5j|%G}qy>g8lB?#Qy>Qf11H zWA>D%&bEe{>Z73u!AaV>IOg*gd`*FP)7i-PvSgQ``>3^@&>h)8zRpY4rJZaI(jHx*ZTHOBQ;=94>~B((dLE-v{KL-?$H5oCgve<22S?lJX?q1VE<#p!=XiQL)hn5zlvoDTjuadh1~I>1Qi<3VyDDgln{Bm}2zR7H zbnNwyKDUihc}Oz{2nYkN!B~kO@hm7dUU+1PJY*2YXRkfcUqDoTI^$~I!Q~q!m0sOz zq#1cm9Jwalb^HKum#FcE_oN`Y8TQEM{Z7D?LD*jkKP|iJS2x`p$2wpHOWT4dj|UC|5;%5Y3VCL>Hq^R<_dM)1c@w)a*FM zfhQkMrPUF}D+c!R(RDJ@>_okSH}ew!$q+|%{o}{|r8bkG)(NUr_IBV|$30MN$?(85 zjF9bpTKeWFr9PXwk=29A5$G4Db^B2LHlP>&qH=RZxV$?@>Nh32{u<7#ABi~4A=rjx zQMCv9;5@lDmt}g`c{?lo8ZEJG&nw5kfMoaJ@IwsxfMH$F|MdmhZh=OHzM}7H$%kxb zAN|b+e2XF#Q3E<4WORiRqp|V1Wa%9OE3s}d<`)TZYo0o#@qp3lNz3zJ6fL%&d8Du? zk1AeAv8-P&mk&@o4j;1!Qxh}DsQ{rZ;VKehWsH=$~Zd^&B6stsaUJIv?8VqaxC z0QWrMuh+Ma{;k6?=a#@S{?KYt2fYUtvLD0@$nsWx*ybA53o;&}7pi|Zx{eIDW`#6( zZ58ZJ8@>;XUOEL!IpIeOSlbfYdgyxkM7h^q8^bAInE$V}v2!&_V-bfjL>AE^9r2jc1=aB%36GQJ*@lu; z9=ybI&L(Y>Y#>ANp$U`Cn`X^}#Gc(V`h_*Dn$AL5{QS7m-5*m{>8N!zJqdEL{A1HX zGw+-b)La5V-F0uY&Vuo0>GV}Q-g7I@UZ|Q%N@UqV`y%0)l65mOY(0`=md=}|1PIu_@o2XQQP1($+|MUO4r zVMkw=yd2x?Tlq$N_WVW>aW@;pB^D&d zJq9dKW2WSCxB!P?T_jE@T~fn1r#heNci_4^%L(>g-Ru4#zW0zkNk@`~h!Y~qs+?2( z-a?t!n)mfB+MCBdRk!6-mCUl>v{dkPA<7*X@rXamu=NijJ?AwE$1&aXC}KB4C9iG* zOBiyv5y?QkKI4^$$yEJWnh?qtyw={PPu+yC+KNEt8!3>yq6*FAlMQDKhxhBXfjl!W z@c14;*J#1_J+ZmMzy0L!wDb8nVgJ3sMcuGvssis$u^ElO*$;7GepQJlRj>Qfwlf~Ee8?3pT*x5NuMuzC!sSbPgn0YEJHpn=bp9yh6E| zU0!Dm;bQ8F)KFm(ODhBr>H;t@5&(%XE7QoGcrw7p3J~jRhw3s0zG|pl`O-=Ig4@1t zA;dqgjS~zZP54;UknotT0^&7!U#3m00j9T?L>S8VNx4x3boC603Ab#W7w>&fz-toV z{}1+IG3%6(AzTXi3D;-W)zZabF5O4bmN95SLf85k&@n_O?lp!e;LvOD`dGr|WCY}< zpu0_M9@%j2@KGSJ(U(oaFlHT?U@~n8J^^P!9K6?IaSNpwWmFe&56T5!8pi)2Rh)ku zX<%^uejR9sjCUOkn&cO6zLJ91P)1e%1rER*dj`5fR+xN(3yF+s=(Luu?oM-j|*<>o(%$4tag6+MT_P@JOLBnC|^$= zD=hD$Y1pE3Q}~-48QY@_&{=3S|I)lfjal_K{EO`scPXM5C9HTT-uSr$#jC`dNqYGA zAOYDiGdkE)KwJ)q7L2kVx$RLrfEIKm39_{%@KBMMRzCmUBAH!(uYz>_4n0^)GQvX+ z3daf0t)e!nf0MCQBJ`v=X#qb=xsP9>{9@b(DSrVgZGrS|q!8WZqY$c;+wAkU{pryX*ii;JQm9G#wUkso{1p=LSXU_v=&c~7!PBw7os8% zj$u1Ki}M0cvOX!2BG=-`el#7%EJy`)3uxW^zoh3x+t38^;{=x3U9x(Sf|WR{gQDmx zc+tw5Ml26y{B~}k&xW^|D*Zhq^S3%Fg+m5uJJofSY}(S5`2I5(sDIM5<__Zv%X5h$ z{q(rg+EkAsee5Xmy03qZ*1;PG-!v*Qtyado8p5p~P+Jc`%CVz({6yeCpbBJQge{`r zH2-@W>k17N^D+zVCgI5E23paekUh5=Fx8cslHFU{PI8Pi?Ru(GVeaaBt9}?PC3EO^ z=S`17yp|}iYN1Yo^LX<88H8zKT$w9}fX?3x>+pLV%nL>RUFGniQ4|h`*&-3e%TZE1 wC92B)9S11xN9p+QV1TRO|5O&<)kpLZM<2qs&_mQ|2{g1= final_sparsity: + _logger.warning('your end epoch <= start epoch or initial_sparsity >= final_sparsity') + return final_sparsity + + now_epoch = tf.minimum(self.now_epoch, tf.constant(end_epoch)) + span = int(((end_epoch - start_epoch-1)//freq)*freq) + assert span > 0 + base = tf.cast(now_epoch - start_epoch, tf.float32) / span + target_sparsity = (final_sparsity + + (initial_sparsity - final_sparsity)* + (tf.pow(1.0 - base, 3))) + return target_sparsity + + def update_epoch(self, epoch, sess): + sess.run(self.assign_handler) + sess.run(tf.assign(self.now_epoch, int(epoch))) + + +class SensitivityPruner(Pruner): + """ + Use algorithm from "Learning both Weights and Connections for Efficient Neural Networks" + https://arxiv.org/pdf/1506.02626v3.pdf + + I.e.: "The pruning threshold is chosen as a quality parameter multiplied + by the standard deviation of a layers weights." + """ + def __init__(self, config_list): + """ + Configure Args: + sparsity: chosen pruning sparsity + """ + super().__init__(config_list) + self.layer_mask = {} + self.assign_handler = [] + + def calc_mask(self, weight, config, op_name, **kwargs): + target_sparsity = config['sparsity'] * tf.math.reduce_std(weight) + mask = tf.get_variable(op_name + '_mask', initializer=tf.ones(weight.shape), trainable=False) + self.layer_mask[op_name] = mask + + weight_assign_handler = tf.assign(weight, mask*weight) + # use control_dependencies so that weight_assign_handler will be executed before mask_update_handler + with tf.control_dependencies([weight_assign_handler]): + threshold = tf.contrib.distributions.percentile(weight, target_sparsity * 100) + # stop gradient in case gradient change the mask + new_mask = tf.stop_gradient(tf.cast(tf.math.greater(weight, threshold), weight.dtype)) + mask_update_handler = tf.assign(mask, new_mask) + self.assign_handler.append(mask_update_handler) + return mask + + def update_epoch(self, epoch, sess): + sess.run(self.assign_handler) diff --git a/src/sdk/pynni/nni/compression/tensorflow/builtin_quantizers.py b/src/sdk/pynni/nni/compression/tensorflow/builtin_quantizers.py new file mode 100644 index 0000000000..3dde1f2f1c --- /dev/null +++ b/src/sdk/pynni/nni/compression/tensorflow/builtin_quantizers.py @@ -0,0 +1,74 @@ +import logging +import tensorflow as tf +from .compressor import Quantizer + +__all__ = [ 'NaiveQuantizer', 'QAT_Quantizer', 'DoReFaQuantizer' ] + +_logger = logging.getLogger(__name__) + + +class NaiveQuantizer(Quantizer): + """ + quantize weight to 8 bits + """ + def __init__(self, config_list): + super().__init__(config_list) + self.layer_scale = { } + + def quantize_weight(self, weight, config, op_name, **kwargs): + new_scale = tf.reduce_max(tf.abs(weight)) / 127 + scale = tf.maximum(self.layer_scale.get(op_name, tf.constant(0.0)), new_scale) + self.layer_scale[op_name] = scale + orig_type = weight.dtype + return tf.cast(tf.cast(weight / scale, tf.int8), orig_type) * scale + + +class QAT_Quantizer(Quantizer): + """ + Quantizer using the DoReFa scheme, as defined in: + Quantization and Training of Neural Networks for Efficient Integer-Arithmetic-Only Inference + http://openaccess.thecvf.com/content_cvpr_2018/papers/Jacob_Quantization_and_Training_CVPR_2018_paper.pdf + """ + def __init__(self, config_list): + """ + Configure Args: + q_bits + """ + super().__init__(config_list) + + def quantize_weight(self, weight, config, **kwargs): + a = tf.stop_gradient(tf.reduce_min(weight)) + b = tf.stop_gradient(tf.reduce_max(weight)) + n = tf.cast(2 ** config['q_bits'], tf.float32) + scale = b-a/(n-1) + + # use gradient_override_map to change round to idetity for gradient + with tf.get_default_graph().gradient_override_map({'Round': 'Identity'}): + qw = tf.round((weight-a)/scale)*scale +a + + return qw + + +class DoReFaQuantizer(Quantizer): + """ + Quantizer using the DoReFa scheme, as defined in: + Zhou et al., DoReFa-Net: Training Low Bitwidth Convolutional Neural Networks with Low Bitwidth Gradients + (https://arxiv.org/abs/1606.06160) + """ + def __init__(self, config_list): + """ + Configure Args: + q_bits + """ + super().__init__(config_list) + + def quantize_weight(self, weight, config, **kwargs): + a = tf.math.tanh(weight) + b = a/(2*tf.reduce_max(tf.abs(weight))) + 0.5 + + scale = pow(2, config['q_bits'] - 1) + # use gradient_override_map to change round to idetity for gradient + with tf.get_default_graph().gradient_override_map({'Round': 'Identity'}): + qw = tf.round(b*scale)/scale + r_qw = 2 * qw - 1 + return r_qw diff --git a/src/sdk/pynni/nni/compression/tensorflow/compressor.py b/src/sdk/pynni/nni/compression/tensorflow/compressor.py new file mode 100644 index 0000000000..3e8b638054 --- /dev/null +++ b/src/sdk/pynni/nni/compression/tensorflow/compressor.py @@ -0,0 +1,152 @@ +import tensorflow as tf +import logging +from . import default_layers + +_logger = logging.getLogger(__name__) + + +class LayerInfo: + def __init__(self, op): + self.op = op + self.name = op.name + self.type = op.type + + +class Compressor: + """ + Abstract base TensorFlow compressor + """ + def __init__(self, config_list): + self._bound_model = None + self._config_list = config_list + + def __call__(self, model): + self.compress(model) + return model + + def compress(self, model): + """ + Compress given graph with algorithm implemented by subclass. + This will edit the graph. + """ + assert self._bound_model is None, "Each NNI compressor instance can only compress one model" + self._bound_model = model + self.bind_model(model) + for op in model.get_operations(): + layer = LayerInfo(op) + config = self._select_config(layer) + if config is not None: + self._instrument_layer(layer, config) + + def compress_default_graph(self): + """ + Compress the default graph with algorithm implemented by subclass. + This will edit the graph. + """ + self.compress(tf.get_default_graph()) + + + def bind_model(self, model): + """ + This method is called when a model is bound to the compressor. + Users can optionally overload this method to do model-specific initialization. + It is guaranteed that only one model will be bound to each compressor instance. + """ + pass + + def update_epoch(self, epoch, sess): + """ + if user want to update mask every epoch, user can override this method + """ + pass + + def step(self, sess): + """ + if user want to update mask every step, user can override this method + """ + pass + + + def _instrument_layer(self, layer, config): + raise NotImplementedError() + + def _select_config(self, layer): + ret = None + for config in self._config_list: + op_types = config.get('op_types') + if op_types == 'default': + op_types = default_layers.op_weight_index.keys() + if op_types and layer.type not in op_types: + continue + if config.get('op_names') and layer.name not in config['op_names']: + continue + ret = config + if ret is None or ret.get('exclude'): + return None + return ret + + +class Pruner(Compressor): + """ + Abstract base TensorFlow pruner + """ + def __init__(self, config_list): + super().__init__(config_list) + + def calc_mask(self, weight, config, op, op_type, op_name): + """ + Pruners should overload this method to provide mask for weight tensors. + The mask must have the same shape and type comparing to the weight. + It will be applied with `multiply()` operation. + This method works as a subgraph which will be inserted into the bound model. + """ + raise NotImplementedError("Pruners must overload calc_mask()") + + def _instrument_layer(self, layer, config): + """ + it seems the graph editor can only swap edges of nodes or remove all edges from a node + it cannot remove one edge from a node, nor can it assign a new edge to a node + we assume there is a proxy operation between the weight and the Conv2D layer + this is true as long as the weight is `tf.Value` + not sure what will happen if the weight is calculated from other operations + """ + weight_index = _detect_weight_index(layer) + if weight_index is None: + _logger.warning('Failed to detect weight for layer {}'.format(layer.name)) + return + weight_op = layer.op.inputs[weight_index].op + weight = weight_op.inputs[0] + mask = self.calc_mask(weight, config, op=layer.op, op_type=layer.type, op_name=layer.name) + new_weight = weight * mask + tf.contrib.graph_editor.swap_outputs(weight_op, new_weight.op) + + +class Quantizer(Compressor): + """ + Abstract base TensorFlow quantizer + """ + def __init__(self, config_list): + super().__init__(config_list) + + def quantize_weight(self, weight, config, op, op_type, op_name): + raise NotImplementedError("Quantizer must overload quantize_weight()") + + def _instrument_layer(self, layer, config): + weight_index = _detect_weight_index(layer) + if weight_index is None: + _logger.warning('Failed to detect weight for layer {}'.format(layer.name)) + return + weight_op = layer.op.inputs[weight_index].op + weight = weight_op.inputs[0] + new_weight = self.quantize_weight(weight, config, op=layer.op, op_type=layer.type, op_name=layer.name) + tf.contrib.graph_editor.swap_outputs(weight_op, new_weight.op) + + +def _detect_weight_index(layer): + index = default_layers.op_weight_index.get(layer.type) + if index is not None: + return index + weight_indices = [ i for i, op in enumerate(layer.op.inputs) if op.name.endswith('Variable/read') ] + if len(weight_indices) == 1: + return weight_indices[0] + return None diff --git a/src/sdk/pynni/nni/compression/tensorflow/default_layers.py b/src/sdk/pynni/nni/compression/tensorflow/default_layers.py new file mode 100644 index 0000000000..0f44ca2987 --- /dev/null +++ b/src/sdk/pynni/nni/compression/tensorflow/default_layers.py @@ -0,0 +1,8 @@ +op_weight_index = { + 'Conv2D': None, + 'Conv3D': None, + 'DepthwiseConv2dNative': None, + + 'Mul': None, + 'MatMul': None, +} diff --git a/src/sdk/pynni/nni/compression/torch/__init__.py b/src/sdk/pynni/nni/compression/torch/__init__.py new file mode 100644 index 0000000000..baf2f84628 --- /dev/null +++ b/src/sdk/pynni/nni/compression/torch/__init__.py @@ -0,0 +1,3 @@ +from .compressor import LayerInfo, Compressor, Pruner, Quantizer +from .builtin_pruners import * +from .builtin_quantizers import * diff --git a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py new file mode 100644 index 0000000000..7309ce1eb3 --- /dev/null +++ b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py @@ -0,0 +1,131 @@ +import logging +import torch +from .compressor import Pruner + +__all__ = [ 'LevelPruner', 'AGP_Pruner', 'SensitivityPruner' ] + +logger = logging.getLogger('torch pruner') + + +class LevelPruner(Pruner): + """Prune to an exact pruning level specification + """ + def __init__(self, config_list): + """ + we suggest user to use json configure list, like [{},{}...], to set configure + format : + [ + { + 'sparsity': 0, + 'support_type': 'default' + }, + { + 'sparsity': 50, + 'support_op': conv1 + } + ] + if you want input multiple configure from file, you'd better use load_configure_file(path) to load + """ + super().__init__(config_list) + + def calc_mask(self, weight, config, **kwargs): + w_abs = weight.abs() + k = int(weight.numel() * config['sparsity']) + if k == 0: + return torch.ones(weight.shape) + threshold = torch.topk(w_abs.view(-1), k, largest = False).values.max() + return torch.gt(w_abs, threshold).type(weight.type()) + + +class AGP_Pruner(Pruner): + """ + An automated gradual pruning algorithm that prunes the smallest magnitude + weights to achieve a preset level of network sparsity. + + Michael Zhu and Suyog Gupta, "To prune, or not to prune: exploring the + efficacy of pruning for model compression", 2017 NIPS Workshop on Machine + Learning of Phones and other Consumer Devices, + https://arxiv.org/pdf/1710.01878.pdf + """ + def __init__(self, config_list): + """ + Configure Args + initial_sparsity + final_sparsity: you should make sure initial_sparsity <= final_sparsity + start_epoch: start epoch numer begin update mask + end_epoch: end epoch number stop update mask, you should make sure start_epoch <= end_epoch + frequency: if you want update every 2 epoch, you can set it 2 + """ + super().__init__(config_list) + self.mask_list = {} + self.now_epoch = 1 + + def calc_mask(self, weight, config, op_name, **kwargs): + mask = self.mask_list.get(op_name, torch.ones(weight.shape)) + target_sparsity = self.compute_target_sparsity(config) + k = int(weight.numel() * target_sparsity) + if k == 0 or target_sparsity >= 1 or target_sparsity <= 0: + return mask + # if we want to generate new mask, we should update weigth first + w_abs = weight.abs()*mask + threshold = torch.topk(w_abs.view(-1), k, largest = False).values.max() + new_mask = torch.gt(w_abs, threshold).type(weight.type()) + self.mask_list[op_name] = new_mask + return new_mask + + def compute_target_sparsity(self, config): + end_epoch = config.get('end_epoch', 1) + start_epoch = config.get('start_epoch', 1) + freq = config.get('frequency', 1) + final_sparsity = config.get('final_sparsity', 0) + initial_sparsity = config.get('initial_sparsity', 0) + if end_epoch <= start_epoch or initial_sparsity >= final_sparsity: + logger.warning('your end epoch <= start epoch or initial_sparsity >= final_sparsity') + return final_sparsity + + if end_epoch <= self.now_epoch: + return final_sparsity + + span = ((end_epoch - start_epoch-1)//freq)*freq + assert span > 0 + target_sparsity = (final_sparsity + + (initial_sparsity - final_sparsity)* + (1.0 - ((self.now_epoch - start_epoch)/span))**3) + return target_sparsity + + def update_epoch(self, epoch): + if epoch > 0: + self.now_epoch = epoch + + +class SensitivityPruner(Pruner): + """ + Use algorithm from "Learning both Weights and Connections for Efficient Neural Networks" + https://arxiv.org/pdf/1506.02626v3.pdf + + I.e.: "The pruning threshold is chosen as a quality parameter multiplied + by the standard deviation of a layers weights." + """ + def __init__(self, config_list): + """ + configure Args: + sparsity: chosen pruning sparsity + """ + super().__init__(config_list) + self.mask_list = {} + + + def calc_mask(self, weight, config, op_name, **kwargs): + mask = self.mask_list.get(op_name, torch.ones(weight.shape)) + # if we want to generate new mask, we should update weigth first + weight = weight*mask + target_sparsity = config['sparsity'] * torch.std(weight).item() + k = int(weight.numel() * target_sparsity) + if k == 0: + return mask + + w_abs = weight.abs() + threshold = torch.topk(w_abs.view(-1), k, largest = False).values.max() + new_mask = torch.gt(w_abs, threshold).type(weight.type()) + self.mask_list[op_name] = new_mask + return new_mask diff --git a/src/sdk/pynni/nni/compression/torch/builtin_quantizers.py b/src/sdk/pynni/nni/compression/torch/builtin_quantizers.py new file mode 100644 index 0000000000..9f2b9ccd95 --- /dev/null +++ b/src/sdk/pynni/nni/compression/torch/builtin_quantizers.py @@ -0,0 +1,76 @@ +import logging +import torch +from .compressor import Quantizer + +__all__ = [ 'NaiveQuantizer', 'QAT_Quantizer', 'DoReFaQuantizer' ] + +logger = logging.getLogger(__name__) + + +class NaiveQuantizer(Quantizer): + """ + quantize weight to 8 bits + """ + def __init__(self, config_list): + super().__init__(config_list) + self.layer_scale = {} + + def quantize_weight(self, weight, config, op_name, **kwargs): + new_scale = weight.abs().max() / 127 + scale = max(self.layer_scale.get(op_name, 0), new_scale) + self.layer_scale[op_name] = scale + orig_type = weight.type() # TODO: user layer + return weight.div(scale).type(torch.int8).type(orig_type).mul(scale) + + +class QAT_Quantizer(Quantizer): + """ + Quantizer using the DoReFa scheme, as defined in: + Quantization and Training of Neural Networks for Efficient Integer-Arithmetic-Only Inference + http://openaccess.thecvf.com/content_cvpr_2018/papers/Jacob_Quantization_and_Training_CVPR_2018_paper.pdf + """ + def __init__(self, config_list): + """ + Configure Args: + q_bits + """ + super().__init__(config_list) + + def quantize_weight(self, weight, config, **kwargs): + if config['q_bits'] <= 1: + return weight + a = torch.min(weight) + b = torch.max(weight) + n = pow(2, config['q_bits']) + scale = (b-a)/(n-1) + zero_point = a + out = torch.round((weight - zero_point)/scale) + out = out*scale + zero_point + orig_type = weight.dtype + return out.type(orig_type) + + +class DoReFaQuantizer(Quantizer): + """ + Quantizer using the DoReFa scheme, as defined in: + Zhou et al., DoReFa-Net: Training Low Bitwidth Convolutional Neural Networks with Low Bitwidth Gradients + (https://arxiv.org/abs/1606.06160) + """ + def __init__(self, config_list): + """ + configure Args: + q_bits + """ + super().__init__(config_list) + + def quantize_weight(self, weight, config, **kwargs): + out = weight.tanh() + out = out /( 2 * out.abs().max()) + 0.5 + out = self.quantize(out, config['q_bits']) + out = 2 * out -1 + return out + + def quantize(self, input_ri, q_bits): + scale = pow(2, q_bits)-1 + output = torch.round(input_ri*scale)/scale + return output diff --git a/src/sdk/pynni/nni/compression/torch/compressor.py b/src/sdk/pynni/nni/compression/torch/compressor.py new file mode 100644 index 0000000000..6282a2138c --- /dev/null +++ b/src/sdk/pynni/nni/compression/torch/compressor.py @@ -0,0 +1,162 @@ +import torch +import logging +from . import default_layers + +_logger = logging.getLogger(__name__) + + +class LayerInfo: + def __init__(self, name, module): + self.module = module + self.name = name + self.type = type(module).__name__ + + self._forward = None + + +class Compressor: + """ + Abstract base PyTorch compressor + """ + def __init__(self, config_list): + self._bound_model = None + self._config_list = config_list + + def __call__(self, model): + self.compress(model) + return model + + def compress(self, model): + """ + Compress the model with algorithm implemented by subclass. + The model will be instrumented and user should never edit it after calling this method. + """ + assert self._bound_model is None, "Each NNI compressor instance can only compress one model" + self._bound_model = model + self.bind_model(model) + for name, module in model.named_modules(): + layer = LayerInfo(name, module) + config = self._select_config(layer) + if config is not None: + self._instrument_layer(layer, config) + + + def bind_model(self, model): + """ + This method is called when a model is bound to the compressor. + Users can optionally overload this method to do model-specific initialization. + It is guaranteed that only one model will be bound to each compressor instance. + """ + pass + + def update_epoch(self, epoch): + """ + if user want to update model every epoch, user can override this method + """ + pass + + def step(self): + """ + if user want to update model every step, user can override this method + """ + pass + + + def _instrument_layer(self, layer, config): + raise NotImplementedError() + + def _select_config(self, layer): + ret = None + for config in self._config_list: + op_types = config.get('op_types') + if op_types == 'default': + op_types = default_layers.weighted_modules + if op_types and layer.type not in op_types: + continue + if config.get('op_names') and layer.name not in config['op_names']: + continue + ret = config + if ret is None or ret.get('exclude'): + return None + return ret + + +class Pruner(Compressor): + """ + Abstract base PyTorch pruner + """ + def __init__(self, config_list): + super().__init__(config_list) + + def calc_mask(self, weight, config, op, op_type, op_name): + """ + Pruners should overload this method to provide mask for weight tensors. + The mask must have the same shape and type comparing to the weight. + It will be applied with `mul()` operation. + This method is effectively hooked to `forward()` method of the model. + """ + raise NotImplementedError("Pruners must overload calc_mask()") + + + def _instrument_layer(self, layer, config): + # TODO: support multiple weight tensors + # create a wrapper forward function to replace the original one + assert layer._forward is None, 'Each model can only be compressed once' + if not _check_weight(layer.module): + _logger.warning('Module {} does not have parameter "weight"'.format(layer.name)) + return + layer._forward = layer.module.forward + + def new_forward(*input): + # apply mask to weight + old_weight = layer.module.weight.data + mask = self.calc_mask(old_weight, config, op=layer.module, op_type=layer.type, op_name=layer.name) + layer.module.weight.data = old_weight.mul(mask) + # calculate forward + ret = layer._forward(*input) + # recover original weight + layer.module.weight.data = old_weight + return ret + + layer.module.forward = new_forward + + +class Quantizer(Compressor): + """ + Base quantizer for pytorch quantizer + """ + def __init__(self, config_list): + super().__init__(config_list) + + def __call__(self, model): + self.compress(model) + return model + + def quantize_weight(self, weight, config, op, op_type, op_name): + """ + user should know where dequantize goes and implement it in quantize method + we now do not provide dequantize method + """ + raise NotImplementedError("Quantizer must overload quantize_weight()") + + def _instrument_layer(self, layer, config): + assert layer._forward is None, 'Each model can only be compressed once' + if not _check_weight(layer.module): + _logger.warning('Module {} does not have parameter "weight"'.format(layer.name)) + return + layer._forward = layer.module.forward + + def new_forward(*input): + weight = layer.module.weight.data + new_weight = self.quantize_weight(weight, config, op=layer.module, op_type=layer.type, op_name=layer.name) + layer.module.weight.data = new_weight + return layer._forward(*input) + + layer.module.forward = new_forward + + +def _check_weight(module): + try: + return isinstance(module.weight, torch.nn.Parameter) and isinstance(module.weight.data, torch.Tensor) + except AttributeError: + return False diff --git a/src/sdk/pynni/nni/compression/torch/default_layers.py b/src/sdk/pynni/nni/compression/torch/default_layers.py new file mode 100644 index 0000000000..185df8bfff --- /dev/null +++ b/src/sdk/pynni/nni/compression/torch/default_layers.py @@ -0,0 +1,6 @@ +weighted_modules = [ + 'Conv1d', 'Conv2d', 'Conv3d', 'ConvTranspose1d', 'ConvTranspose2d', 'ConvTranspose3d', + 'Linear', 'Bilinear', + 'PReLU', + 'Embedding', 'EmbeddingBag', +] diff --git a/src/sdk/pynni/tests/test_compressor.py b/src/sdk/pynni/tests/test_compressor.py new file mode 100644 index 0000000000..1c6021b0cd --- /dev/null +++ b/src/sdk/pynni/tests/test_compressor.py @@ -0,0 +1,116 @@ +from unittest import TestCase, main +import nni.compression.tensorflow as tf_compressor +import nni.compression.torch as torch_compressor +import torch +import torch.nn.functional as F +import tensorflow as tf + +def weight_variable(shape): + return tf.Variable(tf.truncated_normal(shape, stddev = 0.1)) + +def bias_variable(shape): + return tf.Variable(tf.constant(0.1, shape = shape)) + +def conv2d(x_input, w_matrix): + return tf.nn.conv2d(x_input, w_matrix, strides = [ 1, 1, 1, 1 ], padding = 'SAME') + +def max_pool(x_input, pool_size): + size = [ 1, pool_size, pool_size, 1 ] + return tf.nn.max_pool(x_input, ksize = size, strides = size, padding = 'SAME') + + +class TfMnist: + def __init__(self): + images = tf.placeholder(tf.float32, [ None, 784 ], name = 'input_x') + labels = tf.placeholder(tf.float32, [ None, 10 ], name = 'input_y') + keep_prob = tf.placeholder(tf.float32, name='keep_prob') + + self.images = images + self.labels = labels + self.keep_prob = keep_prob + + self.train_step = None + self.accuracy = None + + self.w1 = None + self.b1 = None + self.fcw1 = None + self.cross = None + with tf.name_scope('reshape'): + x_image = tf.reshape(images, [ -1, 28, 28, 1 ]) + with tf.name_scope('conv1'): + w_conv1 = weight_variable([ 5, 5, 1, 32 ]) + self.w1 = w_conv1 + b_conv1 = bias_variable([ 32 ]) + self.b1 = b_conv1 + h_conv1 = tf.nn.relu(conv2d(x_image, w_conv1) + b_conv1) + with tf.name_scope('pool1'): + h_pool1 = max_pool(h_conv1, 2) + with tf.name_scope('conv2'): + w_conv2 = weight_variable([ 5, 5, 32, 64 ]) + b_conv2 = bias_variable([ 64 ]) + h_conv2 = tf.nn.relu(conv2d(h_pool1, w_conv2) + b_conv2) + with tf.name_scope('pool2'): + h_pool2 = max_pool(h_conv2, 2) + with tf.name_scope('fc1'): + w_fc1 = weight_variable([ 7 * 7 * 64, 1024 ]) + self.fcw1 = w_fc1 + b_fc1 = bias_variable([ 1024 ]) + h_pool2_flat = tf.reshape(h_pool2, [ -1, 7 * 7 * 64 ]) + h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, w_fc1) + b_fc1) + with tf.name_scope('dropout'): + h_fc1_drop = tf.nn.dropout(h_fc1, 0.5) + with tf.name_scope('fc2'): + w_fc2 = weight_variable([ 1024, 10 ]) + b_fc2 = bias_variable([ 10 ]) + y_conv = tf.matmul(h_fc1_drop, w_fc2) + b_fc2 + with tf.name_scope('loss'): + cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels = labels, logits = y_conv)) + self.cross = cross_entropy + with tf.name_scope('adam_optimizer'): + self.train_step = tf.train.AdamOptimizer(0.0001).minimize(cross_entropy) + with tf.name_scope('accuracy'): + correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(labels, 1)) + self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) + +class TorchMnist(torch.nn.Module): + def __init__(self): + super().__init__() + self.conv1 = torch.nn.Conv2d(1, 20, 5, 1) + self.conv2 = torch.nn.Conv2d(20, 50, 5, 1) + self.fc1 = torch.nn.Linear(4 * 4 * 50, 500) + self.fc2 = torch.nn.Linear(500, 10) + + def forward(self, x): + x = F.relu(self.conv1(x)) + x = F.max_pool2d(x, 2, 2) + x = F.relu(self.conv2(x)) + x = F.max_pool2d(x, 2, 2) + x = x.view(-1, 4 * 4 * 50) + x = F.relu(self.fc1(x)) + x = self.fc2(x) + return F.log_softmax(x, dim = 1) + +class CompressorTestCase(TestCase): + def test_tf_pruner(self): + model = TfMnist() + configure_list = [{'sparsity':0.8, 'op_types':'default'}] + tf_compressor.LevelPruner(configure_list).compress_default_graph() + + + def test_tf_quantizer(self): + model = TfMnist() + tf_compressor.NaiveQuantizer([{'op_types': 'default'}]).compress_default_graph() + + def test_torch_pruner(self): + model = TorchMnist() + configure_list = [{'sparsity':0.8, 'op_types':'default'}] + torch_compressor.LevelPruner(configure_list).compress(model) + + def test_torch_quantizer(self): + model = TorchMnist() + torch_compressor.NaiveQuantizer([{'op_types': 'default'}]).compress(model) + + +if __name__ == '__main__': + main() From 313b0f67bf9900ad9366df9ad62fd96656b92b48 Mon Sep 17 00:00:00 2001 From: chicm-ms <38930155+chicm-ms@users.noreply.github.com> Date: Wed, 9 Oct 2019 15:57:25 +0800 Subject: [PATCH 14/22] Fix gp tuner (#1592) * fix gp tuner --- azure-pipelines.yml | 2 +- src/sdk/pynni/nni/gp_tuner/gp_tuner.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a2932fd217..336d2375b8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -58,7 +58,7 @@ jobs: - script: | python3 -m pip install torch==0.4.1 --user python3 -m pip install torchvision==0.2.1 --user - python3 -m pip install tensorflow --user + python3 -m pip install tensorflow==1.13.1 --user displayName: 'Install dependencies for integration' - script: | source install.sh diff --git a/src/sdk/pynni/nni/gp_tuner/gp_tuner.py b/src/sdk/pynni/nni/gp_tuner/gp_tuner.py index 3f0b5506bc..5398cd7b13 100644 --- a/src/sdk/pynni/nni/gp_tuner/gp_tuner.py +++ b/src/sdk/pynni/nni/gp_tuner/gp_tuner.py @@ -83,7 +83,7 @@ def update_search_space(self, search_space): """ self._space = TargetSpace(search_space, self._random_state) - def generate_parameters(self, parameter_id): + def generate_parameters(self, parameter_id, **kwargs): """Generate next parameter for trial If the number of trial result is lower than cold start number, gp will first randomly generate some parameters. @@ -123,7 +123,7 @@ def generate_parameters(self, parameter_id): logger.info("Generate paramageters:\n %s", results) return results - def receive_trial_result(self, parameter_id, parameters, value): + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): """Tuner receive result from trial. Parameters From 4cf0e28b11850c3b05aa3a9ea2c5c75587595b74 Mon Sep 17 00:00:00 2001 From: chicm-ms <38930155+chicm-ms@users.noreply.github.com> Date: Wed, 9 Oct 2019 17:07:24 +0800 Subject: [PATCH 15/22] fix hyperband tuner (#1594) --- src/sdk/pynni/nni/hyperband_advisor/hyperband_advisor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sdk/pynni/nni/hyperband_advisor/hyperband_advisor.py b/src/sdk/pynni/nni/hyperband_advisor/hyperband_advisor.py index 7e376c6d9e..699d02c050 100644 --- a/src/sdk/pynni/nni/hyperband_advisor/hyperband_advisor.py +++ b/src/sdk/pynni/nni/hyperband_advisor/hyperband_advisor.py @@ -32,6 +32,7 @@ from nni.msg_dispatcher_base import MsgDispatcherBase from nni.protocol import CommandType, send from nni.utils import NodeType, OptimizeMode, MetricType, extract_scalar_reward +import nni.parameter_expressions as parameter_expressions _logger = logging.getLogger(__name__) From feef81fffbb31dcecfd7983447c09216537f9908 Mon Sep 17 00:00:00 2001 From: Tim Yagan <30977192+TimYagan@users.noreply.github.com> Date: Wed, 9 Oct 2019 11:08:34 +0200 Subject: [PATCH 16/22] Fixed the sequencing of the font attributes (#1579) Fixed the sequencing of the font attributes to ensure that the parent attributes are not overwritten --- src/webui/src/index.css | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webui/src/index.css b/src/webui/src/index.css index 373637d4d4..118fc88769 100644 --- a/src/webui/src/index.css +++ b/src/webui/src/index.css @@ -19,8 +19,8 @@ time, mark, audio, video { margin: 0; padding: 0; border: 0; + font: inherit; font-size: 100%; - font: inherit; } /* HTML5 display-role reset for older browsers */ article, aside, details, figcaption, figure, @@ -48,4 +48,4 @@ q:before, q:after { table { border-collapse: collapse; border-spacing: 0; -} \ No newline at end of file +} From f60bf1d97bd2159c01facdb3ca8b0a61363daf52 Mon Sep 17 00:00:00 2001 From: chicm-ms <38930155+chicm-ms@users.noreply.github.com> Date: Thu, 10 Oct 2019 11:06:06 +0800 Subject: [PATCH 17/22] Remove unneccessary dependencies in pipeline (#1597) * refactor pipeline dependencies --- test/pipelines-it-kubeflow.yml | 5 ----- test/pipelines-it-local.yml | 4 ++-- test/pipelines-it-pai.yml | 5 ----- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/test/pipelines-it-kubeflow.yml b/test/pipelines-it-kubeflow.yml index e9c160636c..3173a7312c 100644 --- a/test/pipelines-it-kubeflow.yml +++ b/test/pipelines-it-kubeflow.yml @@ -39,11 +39,6 @@ jobs: displayName: 'Install nni toolkit via source code' - script: | - python3 -m pip install scikit-learn==0.20.0 --user - python3 -m pip install torch==0.4.1 --user - python3 -m pip install torchvision==0.2.1 --user - python3 -m pip install keras==2.1.6 --user - python3 -m pip install tensorflow==1.12.0 --user sudo apt-get install swig -y PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC PATH=$HOME/.local/bin:$PATH nnictl package install --name=BOHB diff --git a/test/pipelines-it-local.yml b/test/pipelines-it-local.yml index 4f054546fc..31a2e8cdb6 100644 --- a/test/pipelines-it-local.yml +++ b/test/pipelines-it-local.yml @@ -9,8 +9,8 @@ jobs: displayName: 'Install nni toolkit via source code' - script: | python3 -m pip install scikit-learn==0.20.0 --user - python3 -m pip install torch==0.4.1 --user - python3 -m pip install torchvision==0.2.1 --user + python3 -m pip install torch==1.2.0 --user + python3 -m pip install torchvision==0.4.0 --user python3 -m pip install keras==2.1.6 --user python3 -m pip install tensorflow-gpu==1.12.0 --user sudo apt-get install swig -y diff --git a/test/pipelines-it-pai.yml b/test/pipelines-it-pai.yml index 5e44c7a6be..26c1e05cd8 100644 --- a/test/pipelines-it-pai.yml +++ b/test/pipelines-it-pai.yml @@ -39,11 +39,6 @@ jobs: displayName: 'Install nni toolkit via source code' - script: | - python3 -m pip install scikit-learn==0.20.0 --user - python3 -m pip install torch==0.4.1 --user - python3 -m pip install torchvision==0.2.1 --user - python3 -m pip install keras==2.1.6 --user - python3 -m pip install tensorflow-gpu==1.12.0 --user sudo apt-get install swig -y PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC PATH=$HOME/.local/bin:$PATH nnictl package install --name=BOHB From b869dd48dfe36392e7b78c70ea35eb6d4b4779dc Mon Sep 17 00:00:00 2001 From: chicm-ms <38930155+chicm-ms@users.noreply.github.com> Date: Thu, 10 Oct 2019 11:50:37 +0800 Subject: [PATCH 18/22] fix mnist-pytorch example (#1596) --- examples/trials/mnist-pytorch/mnist.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/trials/mnist-pytorch/mnist.py b/examples/trials/mnist-pytorch/mnist.py index 91c68c12a9..af7dcce982 100644 --- a/examples/trials/mnist-pytorch/mnist.py +++ b/examples/trials/mnist-pytorch/mnist.py @@ -5,6 +5,7 @@ https://github.com/pytorch/examples/blob/master/mnist/main.py """ +import os import argparse import logging import nni @@ -84,15 +85,18 @@ def main(args): device = torch.device("cuda" if use_cuda else "cpu") kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} + + data_dir = os.path.join(args['data_dir'], nni.get_trial_id()) + train_loader = torch.utils.data.DataLoader( - datasets.MNIST(args['data_dir'], train=True, download=True, + datasets.MNIST(data_dir, train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=args['batch_size'], shuffle=True, **kwargs) test_loader = torch.utils.data.DataLoader( - datasets.MNIST(args['data_dir'], train=False, transform=transforms.Compose([ + datasets.MNIST(data_dir, train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])), From 1bd24577200a6eb9ba8ecf22457e99278a34abc6 Mon Sep 17 00:00:00 2001 From: Lawrence Wu Date: Thu, 10 Oct 2019 22:39:35 -0700 Subject: [PATCH 19/22] Update AdvancedNas.md (#1601) --- docs/en_US/AdvancedFeature/AdvancedNas.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en_US/AdvancedFeature/AdvancedNas.md b/docs/en_US/AdvancedFeature/AdvancedNas.md index 62f8be1fa8..99d0d9c213 100644 --- a/docs/en_US/AdvancedFeature/AdvancedNas.md +++ b/docs/en_US/AdvancedFeature/AdvancedNas.md @@ -63,7 +63,7 @@ sudo mount -t nfs 10.10.10.10:/tmp/nni/shared /mnt/nfs/nni ``` where `10.10.10.10` should be replaced by the real IP of NFS server machine in practice. -## Asynchornous Dispatcher Mode for trial dependency control +## Asynchronous Dispatcher Mode for trial dependency control The feature of weight sharing enables trials from different machines, in which most of the time **read after write** consistency must be assured. After all, the child model should not load parent model before parent trial finishes training. To deal with this, users can enable **asynchronous dispatcher mode** with `multiThread: true` in `config.yml` in NNI, where the dispatcher assign a tuner thread each time a `NEW_TRIAL` request comes in, and the tuner thread can decide when to submit a new trial by blocking and unblocking the thread itself. For example: ```python def generate_parameters(self, parameter_id): From 5bd994de72e70f93cac7a9655a08eab0fdb1e1b1 Mon Sep 17 00:00:00 2001 From: chicm-ms <38930155+chicm-ms@users.noreply.github.com> Date: Sat, 12 Oct 2019 11:26:44 +0800 Subject: [PATCH 20/22] Fix dispatcher CUDA_VISIBLE_DEVICES envvar for windows (#1604) * Fix dispatcher CUDA_VISIBLE_DEVICES for windows --- src/nni_manager/common/manager.ts | 5 ++--- src/nni_manager/common/utils.ts | 11 ----------- src/nni_manager/core/nnimanager.ts | 19 ++++++++++++++++++- src/nni_manager/core/test/dataStore.test.ts | 3 +-- src/nni_manager/core/test/sqlDatabase.test.ts | 3 +-- 5 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/nni_manager/common/manager.ts b/src/nni_manager/common/manager.ts index 65ab4b77ed..98c5cd8b14 100644 --- a/src/nni_manager/common/manager.ts +++ b/src/nni_manager/common/manager.ts @@ -49,8 +49,8 @@ interface ExperimentParams { classArgs?: any; classFileName?: string; checkpointDir: string; - gpuNum?: number; includeIntermediateResults?: boolean; + gpuIndices?: string; }; assessor?: { className: string; @@ -59,7 +59,6 @@ interface ExperimentParams { classArgs?: any; classFileName?: string; checkpointDir: string; - gpuNum?: number; }; advisor?: { className: string; @@ -68,7 +67,7 @@ interface ExperimentParams { classArgs?: any; classFileName?: string; checkpointDir: string; - gpuNum?: number; + gpuIndices?: string; }; clusterMetaData?: { key: string; diff --git a/src/nni_manager/common/utils.ts b/src/nni_manager/common/utils.ts index 5ae7fc80cb..99d293c6a3 100644 --- a/src/nni_manager/common/utils.ts +++ b/src/nni_manager/common/utils.ts @@ -219,11 +219,6 @@ function getMsgDispatcherCommand(tuner: any, assessor: any, advisor: any, multiP if (advisor.classFileName !== undefined && advisor.classFileName.length > 1) { command += ` --advisor_class_filename ${advisor.classFileName}`; } - if (advisor.gpuIndices !== undefined) { - command = `CUDA_VISIBLE_DEVICES=${advisor.gpuIndices} ` + command; - } else { - command = `CUDA_VISIBLE_DEVICES='' ` + command; - } } else { command += ` --tuner_class_name ${tuner.className}`; if (tuner.classArgs !== undefined) { @@ -248,12 +243,6 @@ function getMsgDispatcherCommand(tuner: any, assessor: any, advisor: any, multiP command += ` --assessor_class_filename ${assessor.classFileName}`; } } - - if (tuner.gpuIndices !== undefined) { - command = `CUDA_VISIBLE_DEVICES=${tuner.gpuIndices} ` + command; - } else { - command = `CUDA_VISIBLE_DEVICES='' ` + command; - } } return command; diff --git a/src/nni_manager/core/nnimanager.ts b/src/nni_manager/core/nnimanager.ts index adbdfcda34..4bae632333 100644 --- a/src/nni_manager/core/nnimanager.ts +++ b/src/nni_manager/core/nnimanager.ts @@ -369,7 +369,8 @@ class NNIManager implements Manager { NNI_CHECKPOINT_DIRECTORY: dataDirectory, NNI_LOG_DIRECTORY: getLogDir(), NNI_LOG_LEVEL: getLogLevel(), - NNI_INCLUDE_INTERMEDIATE_RESULTS: includeIntermediateResultsEnv + NNI_INCLUDE_INTERMEDIATE_RESULTS: includeIntermediateResultsEnv, + CUDA_VISIBLE_DEVICES: this.getGpuEnvvarValue() }; let newEnv = Object.assign({}, process.env, nniEnv); const tunerProc: ChildProcess = getTunerProc(command,stdio,newCwd,newEnv); @@ -379,6 +380,22 @@ class NNIManager implements Manager { return; } + private getGpuEnvvarValue(): string { + let cudaDevices: string | undefined; + + if (this.experimentProfile.params.advisor !== undefined) { + cudaDevices = this.experimentProfile.params.advisor.gpuIndices; + } else if (this.experimentProfile.params.tuner !== undefined) { + cudaDevices = this.experimentProfile.params.tuner.gpuIndices; + } + + if (cudaDevices === undefined) { + return ''; + } else { + return cudaDevices; + } + } + private updateTrialConcurrency(trialConcurrency: number): void { // we assume trialConcurrency >= 0, which is checked by restserver this.trialConcurrencyChange += (trialConcurrency - this.experimentProfile.params.trialConcurrency); diff --git a/src/nni_manager/core/test/dataStore.test.ts b/src/nni_manager/core/test/dataStore.test.ts index 6794706672..db7d702586 100644 --- a/src/nni_manager/core/test/dataStore.test.ts +++ b/src/nni_manager/core/test/dataStore.test.ts @@ -72,8 +72,7 @@ describe('Unit test for dataStore', () => { }`, tuner: { className: 'testTuner', - checkpointDir: '/tmp/cp', - gpuNum: 0 + checkpointDir: '/tmp/cp' } }, id: 'exp123', diff --git a/src/nni_manager/core/test/sqlDatabase.test.ts b/src/nni_manager/core/test/sqlDatabase.test.ts index d292776a3c..0c6717c113 100644 --- a/src/nni_manager/core/test/sqlDatabase.test.ts +++ b/src/nni_manager/core/test/sqlDatabase.test.ts @@ -40,8 +40,7 @@ const expParams1: ExperimentParams = { searchSpace: 'SS', tuner: { className: 'testTuner', - checkpointDir: '/tmp', - gpuNum: 0 + checkpointDir: '/tmp' } }; From ca2253c389a5739b0500c4ceb6015101113fe336 Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Mon, 14 Oct 2019 10:19:29 +0800 Subject: [PATCH 21/22] cmd -> cmd.exe (#1603) --- src/nni_manager/common/utils.ts | 10 +++++----- src/nni_manager/training_service/common/util.ts | 2 +- .../training_service/local/localTrainingService.ts | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/nni_manager/common/utils.ts b/src/nni_manager/common/utils.ts index 99d293c6a3..446f4d0ab1 100644 --- a/src/nni_manager/common/utils.ts +++ b/src/nni_manager/common/utils.ts @@ -445,10 +445,10 @@ function getTunerProc(command: string, stdio: StdioOptions, newCwd: string, newE /** * judge whether the process is alive */ -async function isAlive(pid:any): Promise { +async function isAlive(pid: any): Promise { let deferred : Deferred = new Deferred(); let alive: boolean = false; - if(process.platform ==='win32'){ + if (process.platform === 'win32') { try { const str = cp.execSync(`powershell.exe Get-Process -Id ${pid} -ErrorAction SilentlyContinue`).toString(); if (str) { @@ -458,7 +458,7 @@ async function isAlive(pid:any): Promise { catch (error) { } } - else{ + else { try { await cpp.exec(`kill -0 ${pid}`); alive = true; @@ -473,11 +473,11 @@ async function isAlive(pid:any): Promise { /** * kill process */ -async function killPid(pid:any): Promise { +async function killPid(pid: any): Promise { let deferred : Deferred = new Deferred(); try { if (process.platform === "win32") { - await cpp.exec(`cmd /c taskkill /PID ${pid} /F`); + await cpp.exec(`cmd.exe /c taskkill /PID ${pid} /F`); } else{ await cpp.exec(`kill -9 ${pid}`); diff --git a/src/nni_manager/training_service/common/util.ts b/src/nni_manager/training_service/common/util.ts index cfc6f9b26b..ef05ac57b3 100644 --- a/src/nni_manager/training_service/common/util.ts +++ b/src/nni_manager/training_service/common/util.ts @@ -156,7 +156,7 @@ export async function execRemove(directory: string): Promise { */ export async function execKill(pid: string): Promise { if (process.platform === 'win32') { - await cpp.exec(`cmd /c taskkill /PID ${pid} /T /F`); + await cpp.exec(`cmd.exe /c taskkill /PID ${pid} /T /F`); } else { await cpp.exec(`pkill -P ${pid}`); } diff --git a/src/nni_manager/training_service/local/localTrainingService.ts b/src/nni_manager/training_service/local/localTrainingService.ts index 1a7c70d3a1..2d4d1a1745 100644 --- a/src/nni_manager/training_service/local/localTrainingService.ts +++ b/src/nni_manager/training_service/local/localTrainingService.ts @@ -490,7 +490,7 @@ class LocalTrainingService implements TrainingService { const script: string[] = []; if (process.platform === 'win32') { script.push( - `cmd /c ${localTrialConfig.command} 2>${path.join(workingDirectory, 'stderr')}`, + `cmd.exe /c ${localTrialConfig.command} 2>${path.join(workingDirectory, 'stderr')}`, `$NOW_DATE = [int64](([datetime]::UtcNow)-(get-date "1/1/1970")).TotalSeconds`, `$NOW_DATE = "$NOW_DATE" + (Get-Date -Format fff).ToString()`, `Write $LASTEXITCODE " " $NOW_DATE | Out-File ${path.join(workingDirectory, '.nni', 'state')} -NoNewline -encoding utf8`); From d6b61e2f8d486bc973072c434482d0fa0d40b9a7 Mon Sep 17 00:00:00 2001 From: liuzhe-lz <40699903+liuzhe-lz@users.noreply.github.com> Date: Mon, 14 Oct 2019 11:21:31 +0800 Subject: [PATCH 22/22] Resolve comments in PR 1571 (#1590) * Resolve comments in PR 1571 * try to pass ut * fix typo * format doc-string * use tensorflow.compat.v1 * Revert "use tensorflow.compat.v1" This reverts commit 97a4ed923677c6dfd545fd654c55c424cf490a19. --- docs/en_US/Compressor/Overview.md | 6 +-- docs/en_US/Compressor/Pruner.md | 2 +- examples/model_compress/main_tf_pruner.py | 4 +- examples/model_compress/main_tf_quantizer.py | 4 +- examples/model_compress/main_torch_pruner.py | 6 +-- .../model_compress/main_torch_quantizer.py | 5 +- .../compression/tensorflow/builtin_pruners.py | 26 +++++----- .../tensorflow/builtin_quantizers.py | 17 +++--- .../nni/compression/tensorflow/compressor.py | 52 ++++++++----------- .../nni/compression/torch/builtin_pruners.py | 37 +++++-------- .../compression/torch/builtin_quantizers.py | 17 +++--- .../pynni/nni/compression/torch/compressor.py | 33 +++++------- src/sdk/pynni/tests/test_compressor.py | 6 +-- 13 files changed, 90 insertions(+), 125 deletions(-) diff --git a/docs/en_US/Compressor/Overview.md b/docs/en_US/Compressor/Overview.md index 96453caad5..e3b0fb7c13 100644 --- a/docs/en_US/Compressor/Overview.md +++ b/docs/en_US/Compressor/Overview.md @@ -7,7 +7,7 @@ We have provided two naive compression algorithms and four popular ones for user |Name|Brief Introduction of Algorithm| |---|---| | [Level Pruner](./Pruner.md#level-pruner) | Pruning the specified ratio on each weight based on absolute values of weights | -| [AGP Pruner](./Pruner.md#agp-pruner) | To prune, or not to prune: exploring the efficacy of pruning for model compression. [Reference Paper](https://arxiv.org/abs/1710.01878)| +| [AGP Pruner](./Pruner.md#agp-pruner) | Automated gradual pruning (To prune, or not to prune: exploring the efficacy of pruning for model compression) [Reference Paper](https://arxiv.org/abs/1710.01878)| | [Sensitivity Pruner](./Pruner.md#sensitivity-pruner) | Learning both Weights and Connections for Efficient Neural Networks. [Reference Paper](https://arxiv.org/abs/1506.02626)| | [Naive Quantizer](./Quantizer.md#naive-quantizer) | Quantize weights to default 8 bits | | [QAT Quantizer](./Quantizer.md#qat-quantizer) | Quantization and Training of Neural Networks for Efficient Integer-Arithmetic-Only Inference. [Reference Paper](http://openaccess.thecvf.com/content_cvpr_2018/papers/Jacob_Quantization_and_Training_CVPR_2018_paper.pdf)| @@ -72,7 +72,7 @@ It means following the algorithm's default setting for compressed operations wit ### Other APIs -Some compression algorithms use epochs to control the progress of compression, and some algorithms need to do something after every minibatch. Therefore, we provide another two APIs for users to invoke. One is `update_epoch`, you can use it as follows: +Some compression algorithms use epochs to control the progress of compression (e.g. [AGP](./Pruner.md#agp-pruner)), and some algorithms need to do something after every minibatch. Therefore, we provide another two APIs for users to invoke. One is `update_epoch`, you can use it as follows: Tensorflow code ```python @@ -138,7 +138,7 @@ Some algorithms may want global information for generating masks, for example, a The interface for customizing quantization algorithm is similar to that of pruning algorithms. The only difference is that `calc_mask` is replaced with `quantize_weight`. `quantize_weight` directly returns the quantized weights rather than mask, because for quantization the quantized weights cannot be obtained by applying mask. -``` +```python # This is writing a Quantizer in tensorflow. # For writing a Quantizer in PyTorch, you can simply replace # nni.compression.tensorflow.Quantizer with diff --git a/docs/en_US/Compressor/Pruner.md b/docs/en_US/Compressor/Pruner.md index 59db5b16c8..c6c74efd8a 100644 --- a/docs/en_US/Compressor/Pruner.md +++ b/docs/en_US/Compressor/Pruner.md @@ -38,7 +38,7 @@ In [To prune, or not to prune: exploring the efficacy of pruning for model compr >The binary weight masks are updated every ∆t steps as the network is trained to gradually increase the sparsity of the network while allowing the network training steps to recover from any pruning-induced loss in accuracy. In our experience, varying the pruning frequency ∆t between 100 and 1000 training steps had a negligible impact on the final model quality. Once the model achieves the target sparsity sf , the weight masks are no longer updated. The intuition behind this sparsity function in equation ### Usage -You can prune all weight from %0 to 80% sparsity in 10 epoch with the code below. +You can prune all weight from 0% to 80% sparsity in 10 epoch with the code below. First, you should import pruner and add mask to model. diff --git a/examples/model_compress/main_tf_pruner.py b/examples/model_compress/main_tf_pruner.py index 3c6acc3d5c..b00a01925f 100644 --- a/examples/model_compress/main_tf_pruner.py +++ b/examples/model_compress/main_tf_pruner.py @@ -127,4 +127,6 @@ def main(): }) print('final result is', test_acc) -main() + +if __name__ == '__main__': + main() diff --git a/examples/model_compress/main_tf_quantizer.py b/examples/model_compress/main_tf_quantizer.py index cc6e0b8fc9..c1e6214ebf 100644 --- a/examples/model_compress/main_tf_quantizer.py +++ b/examples/model_compress/main_tf_quantizer.py @@ -114,4 +114,6 @@ def main(): }) print('final result is', test_acc) -main() + +if __name__ == '__main__': + main() diff --git a/examples/model_compress/main_torch_pruner.py b/examples/model_compress/main_torch_pruner.py index 39ceb378a1..b8474a8a00 100644 --- a/examples/model_compress/main_torch_pruner.py +++ b/examples/model_compress/main_torch_pruner.py @@ -89,7 +89,7 @@ def main(): test(model, device, test_loader) pruner.update_epoch(epoch) - - -main() + +if __name__ == '__main__': + main() diff --git a/examples/model_compress/main_torch_quantizer.py b/examples/model_compress/main_torch_quantizer.py index 27e3dcfac0..adbfab0582 100644 --- a/examples/model_compress/main_torch_quantizer.py +++ b/examples/model_compress/main_torch_quantizer.py @@ -81,7 +81,6 @@ def main(): train(model, device, train_loader, optimizer) test(model, device, test_loader) - - -main() +if __name__ == '__main__': + main() diff --git a/src/sdk/pynni/nni/compression/tensorflow/builtin_pruners.py b/src/sdk/pynni/nni/compression/tensorflow/builtin_pruners.py index b014d8bc99..c2b7e4453d 100644 --- a/src/sdk/pynni/nni/compression/tensorflow/builtin_pruners.py +++ b/src/sdk/pynni/nni/compression/tensorflow/builtin_pruners.py @@ -10,8 +10,8 @@ class LevelPruner(Pruner): def __init__(self, config_list): """ - Configure Args: - sparsity + config_list: supported keys: + - sparsity """ super().__init__(config_list) @@ -21,8 +21,7 @@ def calc_mask(self, weight, config, **kwargs): class AGP_Pruner(Pruner): - """ - An automated gradual pruning algorithm that prunes the smallest magnitude + """An automated gradual pruning algorithm that prunes the smallest magnitude weights to achieve a preset level of network sparsity. Michael Zhu and Suyog Gupta, "To prune, or not to prune: exploring the @@ -32,12 +31,12 @@ class AGP_Pruner(Pruner): """ def __init__(self, config_list): """ - Configure Args - initial_sparsity: - final_sparsity: you should make sure initial_sparsity <= final_sparsity - start_epoch: start epoch numer begin update mask - end_epoch: end epoch number stop update mask - frequency: if you want update every 2 epoch, you can set it 2 + config_list: supported keys: + - initial_sparsity + - final_sparsity: you should make sure initial_sparsity <= final_sparsity + - start_epoch: start epoch numer begin update mask + - end_epoch: end epoch number stop update mask + - frequency: if you want update every 2 epoch, you can set it 2 """ super().__init__(config_list) self.now_epoch = tf.Variable(0) @@ -77,8 +76,7 @@ def update_epoch(self, epoch, sess): class SensitivityPruner(Pruner): - """ - Use algorithm from "Learning both Weights and Connections for Efficient Neural Networks" + """Use algorithm from "Learning both Weights and Connections for Efficient Neural Networks" https://arxiv.org/pdf/1506.02626v3.pdf I.e.: "The pruning threshold is chosen as a quality parameter multiplied @@ -86,8 +84,8 @@ class SensitivityPruner(Pruner): """ def __init__(self, config_list): """ - Configure Args: - sparsity: chosen pruning sparsity + config_list: supported keys + - sparsity: chosen pruning sparsity """ super().__init__(config_list) self.layer_mask = {} diff --git a/src/sdk/pynni/nni/compression/tensorflow/builtin_quantizers.py b/src/sdk/pynni/nni/compression/tensorflow/builtin_quantizers.py index 3dde1f2f1c..a7ed2b9338 100644 --- a/src/sdk/pynni/nni/compression/tensorflow/builtin_quantizers.py +++ b/src/sdk/pynni/nni/compression/tensorflow/builtin_quantizers.py @@ -8,8 +8,7 @@ class NaiveQuantizer(Quantizer): - """ - quantize weight to 8 bits + """quantize weight to 8 bits """ def __init__(self, config_list): super().__init__(config_list) @@ -24,15 +23,14 @@ def quantize_weight(self, weight, config, op_name, **kwargs): class QAT_Quantizer(Quantizer): - """ - Quantizer using the DoReFa scheme, as defined in: + """Quantizer using the DoReFa scheme, as defined in: Quantization and Training of Neural Networks for Efficient Integer-Arithmetic-Only Inference http://openaccess.thecvf.com/content_cvpr_2018/papers/Jacob_Quantization_and_Training_CVPR_2018_paper.pdf """ def __init__(self, config_list): """ - Configure Args: - q_bits + config_list: supported keys: + - q_bits """ super().__init__(config_list) @@ -50,15 +48,14 @@ def quantize_weight(self, weight, config, **kwargs): class DoReFaQuantizer(Quantizer): - """ - Quantizer using the DoReFa scheme, as defined in: + """Quantizer using the DoReFa scheme, as defined in: Zhou et al., DoReFa-Net: Training Low Bitwidth Convolutional Neural Networks with Low Bitwidth Gradients (https://arxiv.org/abs/1606.06160) """ def __init__(self, config_list): """ - Configure Args: - q_bits + config_list: supported keys: + - q_bits """ super().__init__(config_list) diff --git a/src/sdk/pynni/nni/compression/tensorflow/compressor.py b/src/sdk/pynni/nni/compression/tensorflow/compressor.py index 3e8b638054..3c0cf04d1c 100644 --- a/src/sdk/pynni/nni/compression/tensorflow/compressor.py +++ b/src/sdk/pynni/nni/compression/tensorflow/compressor.py @@ -13,20 +13,21 @@ def __init__(self, op): class Compressor: - """ - Abstract base TensorFlow compressor - """ + """Abstract base TensorFlow compressor""" + def __init__(self, config_list): self._bound_model = None self._config_list = config_list def __call__(self, model): + """Compress given graph with algorithm implemented by subclass. + The graph will be editted and returned. + """ self.compress(model) return model def compress(self, model): - """ - Compress given graph with algorithm implemented by subclass. + """Compress given graph with algorithm implemented by subclass. This will edit the graph. """ assert self._bound_model is None, "Each NNI compressor instance can only compress one model" @@ -39,30 +40,26 @@ def compress(self, model): self._instrument_layer(layer, config) def compress_default_graph(self): - """ - Compress the default graph with algorithm implemented by subclass. - This will edit the graph. + """Compress the default graph with algorithm implemented by subclass. + This will edit the default graph. """ self.compress(tf.get_default_graph()) def bind_model(self, model): - """ - This method is called when a model is bound to the compressor. - Users can optionally overload this method to do model-specific initialization. + """This method is called when a model is bound to the compressor. + Compressors can optionally overload this method to do model-specific initialization. It is guaranteed that only one model will be bound to each compressor instance. """ pass def update_epoch(self, epoch, sess): - """ - if user want to update mask every epoch, user can override this method + """If user want to update mask every epoch, user can override this method """ pass def step(self, sess): - """ - if user want to update mask every step, user can override this method + """If user want to update mask every step, user can override this method """ pass @@ -87,15 +84,13 @@ def _select_config(self, layer): class Pruner(Compressor): - """ - Abstract base TensorFlow pruner - """ + """Abstract base TensorFlow pruner""" + def __init__(self, config_list): super().__init__(config_list) def calc_mask(self, weight, config, op, op_type, op_name): - """ - Pruners should overload this method to provide mask for weight tensors. + """Pruners should overload this method to provide mask for weight tensors. The mask must have the same shape and type comparing to the weight. It will be applied with `multiply()` operation. This method works as a subgraph which will be inserted into the bound model. @@ -103,13 +98,11 @@ def calc_mask(self, weight, config, op, op_type, op_name): raise NotImplementedError("Pruners must overload calc_mask()") def _instrument_layer(self, layer, config): - """ - it seems the graph editor can only swap edges of nodes or remove all edges from a node - it cannot remove one edge from a node, nor can it assign a new edge to a node - we assume there is a proxy operation between the weight and the Conv2D layer - this is true as long as the weight is `tf.Value` - not sure what will happen if the weight is calculated from other operations - """ + # it seems the graph editor can only swap edges of nodes or remove all edges from a node + # it cannot remove one edge from a node, nor can it assign a new edge to a node + # we assume there is a proxy operation between the weight and the Conv2D layer + # this is true as long as the weight is `tf.Value` + # not sure what will happen if the weight is calculated from other operations weight_index = _detect_weight_index(layer) if weight_index is None: _logger.warning('Failed to detect weight for layer {}'.format(layer.name)) @@ -122,9 +115,8 @@ def _instrument_layer(self, layer, config): class Quantizer(Compressor): - """ - Abstract base TensorFlow quantizer - """ + """Abstract base TensorFlow quantizer""" + def __init__(self, config_list): super().__init__(config_list) diff --git a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py index 7309ce1eb3..858db63a94 100644 --- a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py +++ b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py @@ -12,19 +12,8 @@ class LevelPruner(Pruner): """ def __init__(self, config_list): """ - we suggest user to use json configure list, like [{},{}...], to set configure - format : - [ - { - 'sparsity': 0, - 'support_type': 'default' - }, - { - 'sparsity': 50, - 'support_op': conv1 - } - ] - if you want input multiple configure from file, you'd better use load_configure_file(path) to load + config_list: supported keys: + - sparsity """ super().__init__(config_list) @@ -38,8 +27,7 @@ def calc_mask(self, weight, config, **kwargs): class AGP_Pruner(Pruner): - """ - An automated gradual pruning algorithm that prunes the smallest magnitude + """An automated gradual pruning algorithm that prunes the smallest magnitude weights to achieve a preset level of network sparsity. Michael Zhu and Suyog Gupta, "To prune, or not to prune: exploring the @@ -49,12 +37,12 @@ class AGP_Pruner(Pruner): """ def __init__(self, config_list): """ - Configure Args - initial_sparsity - final_sparsity: you should make sure initial_sparsity <= final_sparsity - start_epoch: start epoch numer begin update mask - end_epoch: end epoch number stop update mask, you should make sure start_epoch <= end_epoch - frequency: if you want update every 2 epoch, you can set it 2 + config_list: supported keys: + - initial_sparsity + - final_sparsity: you should make sure initial_sparsity <= final_sparsity + - start_epoch: start epoch numer begin update mask + - end_epoch: end epoch number stop update mask, you should make sure start_epoch <= end_epoch + - frequency: if you want update every 2 epoch, you can set it 2 """ super().__init__(config_list) self.mask_list = {} @@ -99,8 +87,7 @@ def update_epoch(self, epoch): class SensitivityPruner(Pruner): - """ - Use algorithm from "Learning both Weights and Connections for Efficient Neural Networks" + """Use algorithm from "Learning both Weights and Connections for Efficient Neural Networks" https://arxiv.org/pdf/1506.02626v3.pdf I.e.: "The pruning threshold is chosen as a quality parameter multiplied @@ -108,8 +95,8 @@ class SensitivityPruner(Pruner): """ def __init__(self, config_list): """ - configure Args: - sparsity: chosen pruning sparsity + config_list: supported keys: + - sparsity: chosen pruning sparsity """ super().__init__(config_list) self.mask_list = {} diff --git a/src/sdk/pynni/nni/compression/torch/builtin_quantizers.py b/src/sdk/pynni/nni/compression/torch/builtin_quantizers.py index 9f2b9ccd95..f5b4b5223a 100644 --- a/src/sdk/pynni/nni/compression/torch/builtin_quantizers.py +++ b/src/sdk/pynni/nni/compression/torch/builtin_quantizers.py @@ -8,8 +8,7 @@ class NaiveQuantizer(Quantizer): - """ - quantize weight to 8 bits + """quantize weight to 8 bits """ def __init__(self, config_list): super().__init__(config_list) @@ -24,15 +23,14 @@ def quantize_weight(self, weight, config, op_name, **kwargs): class QAT_Quantizer(Quantizer): - """ - Quantizer using the DoReFa scheme, as defined in: + """Quantizer using the DoReFa scheme, as defined in: Quantization and Training of Neural Networks for Efficient Integer-Arithmetic-Only Inference http://openaccess.thecvf.com/content_cvpr_2018/papers/Jacob_Quantization_and_Training_CVPR_2018_paper.pdf """ def __init__(self, config_list): """ - Configure Args: - q_bits + config_list: supported keys: + - q_bits """ super().__init__(config_list) @@ -51,15 +49,14 @@ def quantize_weight(self, weight, config, **kwargs): class DoReFaQuantizer(Quantizer): - """ - Quantizer using the DoReFa scheme, as defined in: + """Quantizer using the DoReFa scheme, as defined in: Zhou et al., DoReFa-Net: Training Low Bitwidth Convolutional Neural Networks with Low Bitwidth Gradients (https://arxiv.org/abs/1606.06160) """ def __init__(self, config_list): """ - configure Args: - q_bits + config_list: supported keys: + - q_bits """ super().__init__(config_list) diff --git a/src/sdk/pynni/nni/compression/torch/compressor.py b/src/sdk/pynni/nni/compression/torch/compressor.py index 6282a2138c..5d74a464b0 100644 --- a/src/sdk/pynni/nni/compression/torch/compressor.py +++ b/src/sdk/pynni/nni/compression/torch/compressor.py @@ -15,9 +15,8 @@ def __init__(self, name, module): class Compressor: - """ - Abstract base PyTorch compressor - """ + """Abstract base PyTorch compressor""" + def __init__(self, config_list): self._bound_model = None self._config_list = config_list @@ -27,8 +26,7 @@ def __call__(self, model): return model def compress(self, model): - """ - Compress the model with algorithm implemented by subclass. + """Compress the model with algorithm implemented by subclass. The model will be instrumented and user should never edit it after calling this method. """ assert self._bound_model is None, "Each NNI compressor instance can only compress one model" @@ -42,22 +40,19 @@ def compress(self, model): def bind_model(self, model): - """ - This method is called when a model is bound to the compressor. + """This method is called when a model is bound to the compressor. Users can optionally overload this method to do model-specific initialization. It is guaranteed that only one model will be bound to each compressor instance. """ pass def update_epoch(self, epoch): - """ - if user want to update model every epoch, user can override this method + """if user want to update model every epoch, user can override this method """ pass def step(self): - """ - if user want to update model every step, user can override this method + """if user want to update model every step, user can override this method """ pass @@ -82,15 +77,13 @@ def _select_config(self, layer): class Pruner(Compressor): - """ - Abstract base PyTorch pruner - """ + """Abstract base PyTorch pruner""" + def __init__(self, config_list): super().__init__(config_list) def calc_mask(self, weight, config, op, op_type, op_name): - """ - Pruners should overload this method to provide mask for weight tensors. + """Pruners should overload this method to provide mask for weight tensors. The mask must have the same shape and type comparing to the weight. It will be applied with `mul()` operation. This method is effectively hooked to `forward()` method of the model. @@ -122,9 +115,8 @@ def new_forward(*input): class Quantizer(Compressor): - """ - Base quantizer for pytorch quantizer - """ + """Base quantizer for pytorch quantizer""" + def __init__(self, config_list): super().__init__(config_list) @@ -133,8 +125,7 @@ def __call__(self, model): return model def quantize_weight(self, weight, config, op, op_type, op_name): - """ - user should know where dequantize goes and implement it in quantize method + """user should know where dequantize goes and implement it in quantize method we now do not provide dequantize method """ raise NotImplementedError("Quantizer must overload quantize_weight()") diff --git a/src/sdk/pynni/tests/test_compressor.py b/src/sdk/pynni/tests/test_compressor.py index 1c6021b0cd..83735a20a2 100644 --- a/src/sdk/pynni/tests/test_compressor.py +++ b/src/sdk/pynni/tests/test_compressor.py @@ -1,9 +1,9 @@ from unittest import TestCase, main -import nni.compression.tensorflow as tf_compressor -import nni.compression.torch as torch_compressor +import tensorflow as tf import torch import torch.nn.functional as F -import tensorflow as tf +import nni.compression.tensorflow as tf_compressor +import nni.compression.torch as torch_compressor def weight_variable(shape): return tf.Variable(tf.truncated_normal(shape, stddev = 0.1))

qZbHmt2c%y-^+ar%SLK9cj?QG-G_(T}q`jM2EREr9;Zk8RE&R)kl@ zI`!El^QYx`T!ugA^Q47Hys@e@8r~x%rW9X!iTvYz>+;yw&&C-(5P%(^&pazDiN`(h z)LS%eWIB3G0k_^JsO4F7`M|*OJZ6R4DDqc&)M+EqGvG}stgqX7py5e8G09E+SB;Qw z$BOLWVIwL5@s=`1C?9`t>!W!bmOznXD5Bqda%*nOc?vE$hc%wZ426SmYAi{!Qi9UX z*Po8Lq2a|290gzp;T?p-KYZ)e`8nU+D6AO2S8cRB@7+tC?Bhv4qDSuSD)oc8CSPrt zV_VT2g#`giyuO#so0JQ1)Z6p@?Aa6212tZ6|AIzAnt4;krXPKA``C?k2-mkiz9qkZ zg}S_BsUQGc0ZKbos)$YmYb`mdp!*_a%U@O|ot231}Zr);h^q&ER7Gd3&zqvpz1#yz}?-uUwYr z*?;@QgQ-65ou??ZEl*xWI!oe=z=$3Mh>o$x!`uB@w@dyMQ3;(!mZ`pS%bxV{Z@2O} z>uW|L(SHg=2UT8pd-K``ROS_E%nvnW$;c>pQMBXP{Qc667iVMD=O5WHC=A*h zPVO8zqAMGt02-VlK6U@vtfOdCRGFNSorruR9aBlxqzmVF_L2h{GIpYBepYAt9N$Ma z1TS+-$|1Z?VMV;E1f`u>9|KpPGc*0>Eo0K}eEOkAcz7l6NM96DOyti_h1=u$)g2A} zpK-`s#>~Ho0Dshdp8dekIfx>x94%~yK%C=uc}Sq znZlFZ@x~m3vGGx^^SPl@_{R{q;o1Ee91^t_eTgqDm^LCCPb#C3{PI;8{PsOnO%GoE zi>ioA3Y{zlx8w>?ZOKH@JL}ix1v}4HbOJ+~X zg{6B>G$F66B+%r6ol zSLkse+t+{px9_bU=*$UPzn}J}J$v?~cfIRf>4_(vNOR}TO@Hzyf07n29_$tS%rnoV zcfRwTY0H)hY<>Ux-=D6y;)=HGMq9Vsa!b1T=9}Ao8|n)owo|GBh4NSm4LFN#gfop9 zIWjBA+M~`AuqdPi%TVB^QU)6#h?Xo$DyBK=NO$T#QYpa}Jqn0)ApkCU-AYhe`||t< zskpHSmA0Qp=eyrl5$2ougZu9wVC zw?5Dcxr_68W)X@3P4}(YnHO$3sgbJdK-kj_8H#Jzdhefp0DxIphzU;*ta~kHv zE$T{w^9b<9K~uzRU^2!lBBxVWdF6E?X(o)v`eML?E*p4u^)|sU*Y_UEP~5UjMJrg7 zDJB=rpPD^E%HJ7Uyc=zVhl}4``{fr6Ex1@@g~RnPUf#08baD5)9U0s!O zEyStHbS$(qHH;cvsq{<&I4bA`aG;pqx8~VF-WcxBy9vS@q!-|+i9Y~2c~m{QX>SH1 zBJ3J}0E8>>chhC_(c<@q;VN=(--=! zPR8vWk8f#uZy5*1n2~C3I}h34iKo}w^PK<;NW{AUgYrDS>Y{mr!Z$me`=Ps5Hwu>Y zq;aFh<^GT#y`DA$52Tz?pi81?{ppvNXCNsIBKXzeV^v7HH3GT8H%KP0_ssFv_TQqAF-BJ zO4>krw)b!vI53b6z={rw8WGkk&#ZAuAh$Ln#-H{PTN#vrQp1gAab5ReO$mdZZJl_m z$M5yud{=v+6EQmT&gaP&Bsykg;TkvIKqMk-_Vf-{gRe z@hNgU6Vd!X8NK1EMcIStkqx_Y2)L9EcyOd4dur?c^yAl^m!8~GDaJfY zhm+UQb%zq#!+>|Pd-J=o=&ut2)E3X0l=JZF1F4n5KxBO>L22jXAzyiHbDI&%xRs8g zjT+t7my0$Z&vR2(N=1RZhcF7(73lA**_r#kY|k9QGbW7g$$f$+m%JJOFb^~ja4HQp z(GD3?fj2U!f<~TG?sY)C^XmjSv!0drC(Y7O@5K3d^Q$hMmp*gHExNu&c zPxWiW_e&N`O`m&sWYbWKXW&se8lj;O^U}5!;~(;81IkgaBkm{`84; zwIn<#fN7qyjH@N_ApY5xUywa6%l+~B+}YJwV)P|*l;m0gn9mua2EfZNnwvfA@bXy8 zv|)_?>T51(8XN1fS9u-jN90T8oM665L&1~1zShS%FIzSz1NC&A;9~TP^1Y89HKHj) z;CmCs^yG+}GWCY`$ZDOuh{^r`?#1V2xyLyo&j9EmNklIguOIkE%PT+b4JbGq!pHoT z)O)x)%|CoJkA<0v+)f;m-gC?4naBO}1MAYZJqnq(zkg$Yv_aH}fXRAHP7SazZQIwE z+ZbOsY$)=0y1+CZC?#V_Quo9l4%Zg~z z^7!xeXN7wG;US?Xx#r`WTG5no4-6Fl62#!HV0MtV^ZAE24hc+iw&X|?#1Qq9*Pq{v ztcm^KtL>kCS*pdBvw-{iA(T@AGKl`&m$K847of+WP-kB1iMNVV+ zj6QbHn!GSu)Bua2h?d?yNBXOLtpS2!<}8V?yK+JL@3%gh3nzvZ&$5fT$DqVG6pHFZ zbc=mt!AXA!a}L>LIq%grLpbEpJCoJ*OxXVtb0I`h`E7-1l+ zWIQ;Ld{fACl|r!4*$D5<2MpsM3nQSNHBH}y2Z~VUKlsce=}p%xPA^?H7?>o8^H#-# z^=I{dDsZE_0km4g^i4?=v3hC@{UA(9XyW=EJfDtdYqbijKYZ=EIh0S~rP8w=C=wy| zt|QN4NkEk^JiI9z61=-o`6!PY&S$w%1C);A<=X|yQz&C>#c#0%Qh9v9YVHREsnH$! z;YVI{ZVpF{<73N!^63ZGH^DWEC}Av=Q_7D&9?pu`co*_ZyN0>+;k(yl4^8cf1W@%` z@rn$BjvJLL82{YMmkkn3fPB202+LJI_YvtGFOz$)-D@H4(^e{&0Hw?QE4S>=!kAo$ z1nFJ9)!5*jbgUoPR2ediVSh6xj?Q^j%QgtQ0q_YM??iJZkI8dCSEnx)U2Osn84eT= zInt{zR29-5xnYfW;O0eVP02<6%*jPFC$!C}`ZYsIiTxBk!m{^#v{4w#c!;z#KIar9 zaRN(qcS;MVwZQHiB2AFvt$O5W+H~YtjuhePOgIK)0=y}(<{72?s>RcqEe06a+|l?D zLk9DT^_GBMpRk6#MGer`>GmYK`~U9F z)mhuUQbKHbNes8x*|ubc(1WOY1espUs*w$v9dX4 zq@!sF_Po|SL*J2Osk^H(Tv+dTZSg{v5&;m(3zlKh9OTg#M8F5`%QfiTcpe5W9?**9 zmP84$9_H>YK+YH+o?Dr<4BB;7c`v;i3dR+SXJo!BB}|a}`QTlva|8!oF>mU)tW3u5 zws@6sW(>JEZ2`LQs^OCU;eBmf6{+>}>1BAZca?K07tdB+Lu6JxJl{Q&85rqSZP}L& zA3K(2O&XUJpNV3(-bal4ibsuU1EtGr#ODAM;(KhPuYK=;Kjy)qDSouRhgG_Y}t|w)Gp^_qFkAYskcLBh%V#Rq}2?HA7vU6o{6dq)c7K z8whxDDrDY~kr+?vg?Y$u{z@VXF=ClN`qdZ;+Oh|H)FLS=tN|s-ahtvRA`KKn+K8U) zA$uUgkdO6eSuq=^LmN5L8#N0AK%mjrqw}- zxgE#3KAs;kiv64*MDcj8(;@5B*)u0*zTtDwviVcee|g=dIpxCt{?@9@OX9biuUt^A z+p#01oC-*;>i{b8E4;Y$q?FQ(9uQMmt`)H_h&3%7Ejqjn(JJEjWXzEmriLN1w%GZxS(f3Yi zy5wo**ua3yIRr0ND}|@|BkKplK%onr=sKN#>IBbw0<~iSIMWS<9|D zXL?8D-Kk#ZRE{s#6Ei&KXxRpQBkKx7G4S%>=kW8Un#`nW7!%$a*g{+mdy>MfBnFAE zJhA0p1ek`2)a5Dn_Q-~vgY?Dnp3Bc7&dby?ew#mSTrO-|9`|pbJNLj~j@0?MQ`)(;b?|NeB{b=T$dmMvSB=Fgv>)~;PU$Q#=Jv(K(_4BAefJUIcZ1Hk4{y<)|R^yW9e z`IPSALwOCjPl-z z0st6sJKQDx!LZ*pbc{|kpwu3r+ z0puv8_3?Yx=Ea`k2rR?k|N3iBWq?$nkdo;hEWrIrD6K5=6;M+x=>S<1zfMBMuD@VT zR?xH(fkP@vYz2P~WxRiON(O_22*Qo2#;-EdQ(bQjJ+h0b$3aKl#xYv?p8sg!v4OEBSWb>~yt z1_9F)7Jxsn+q+}5I2j!$p>&{)yAL!A>=;jDMs#Pz|5$$o{*CDB&WeAGsiAVYN7KM1 zUIZmkQqP++IbFP?20{a_pygcWU{^nC#+YA-PDj_Mnx_FiHFedDuJ9}FWYX-XEU5ZZ`E0J4L*x$?v zW3u6gA;TC#yh#+*V5s@_>Syy}3nb8&o*e-qqED|mXGR93{^47zQ+H0yP=yWyr+IuU zwn``tB}|{0e*`N+XOPbfUv-ZiX$=1Uo2%2EE4StAxJ~`!*mvD@aU0)=wp>eJf9k$x zGKlQS+`DJw2_gSuPznCVKHGBSJ-%UA2J@ZI^S+FfP&vFz$)S&%ZFY5KUaL=MO&*({ zZS$T(neP&GfAz!nrMc6_r3F*Rr<*QckW)MWwKBhvlBSV2-u>t={aG3cB7(s+@%6Z_ zJ`bLyzug~!ULh(*0(D@*c@U6V2q01{=- zniXS#@+6wcenjc?$1um>qi7#*rLsVo<1Sq|J*TQEujg~Q7S@0B9T+Kxr+xzr0!4Qp z+?a)Db0nkZwRU*^)Pa$*Z6KZd2W0Wy)EBS0yed#>+P6j>efO;Pukym7R?rO_B?dj@HTlOAoiy8r<@_xmmUvS>u?41YLF<;F~ZTKv7 zUYNc06rfqRi_{|;C`HC!U2d=@dtL?C6jhRkHb|K5>#yjXN+OP_Z>2NA56N)GnNaA}sBkwR;1#doZL{!T9oC@yB z#d2%5q7qz}0S~W4W=XSXlU?iOGgvu>w{cmaK?mO7+C}f`A8p_i}-G&DKX|Aer9KmY%@yY#-w>-PUE$XN5+yu!T11! zn59R5PCv~Y!@T=G_wf3hUQWAHCyq{6EuNm&5)n>_y!4)1uE+p2-ZXdW*p5yTc#rxm zp26Kp(yM|Y|iT=*_`3@>1}(PvZ1!fj{fQqehk+feqag_)yn9v7_jt; zQW1tz9V6L;&yahM=JCkr!60f)gaes>Qt=Iyj?_57j}%umw}N>3zw^}>pOL3~v~@aL zoJ=Z6I8aF(PUzR)Ys$Dz!ysQ?yOV6dL9vs=SzV?n+xc#OuFFhut=_)Q*i8IhZl$!7O}D7&Yugo8luk zmAr+nlkc^sN-;s{I2Dp>jaNe4vvYY8&!67%upBDCk`UO_r57)qoo;<-eHQE~g5jZ{ zKjquR_rIe9`m3q)xp36xDkFSD(y3l^}+3>>P#;&lu#rW)J29_>SLf z2@7?P)$;fH=d*?wbBrkwe!92sXjP)+jY~$_>k^=V#Ns3&);-Yy?uNp#qS>UX^&+C@VY68;$FACq;GsowA=e!0fg@5z4 zm2LT|MPGC(N&zekPZaM}>K?p9kwik*JvH#0!o}l&zJ(W1`~x)qQ-{JOUwU*?o?oT#O;|P_c|0l@Stkm-Jpab2 z{`aiiT}LuVCbdclOq=%r1;r{#Qa}lP`hj)1@MrlOaE+k_?c878i_fi@P##?w1YF15 z!%p8a{4Kvk$hq}Ac zfB)Pg`5jb((}E!da^PO1YKW(?uM;U8`M$j?vf%Fi@kY-Tp2yZj?-{R%yov-?pk?o& zV@*J?Q`vDH{?EQcwF*{$75TUg@(Bwx{I47b$(4cJ-bRnD*Q}-sP3v zd+>1f=)Cs)nS;O|)xbabowe!NeU$+c2>tr27UVf|CLzO~gXeMGL)?2M6P0A%Q7Yw; z50Yx46EN-fp%lY7cc?9e88BUvT=UW}*Z$4RhbcUhF*klmmbx-cNw1aoCqQ4?Q zgdSUO@h&0f3{^ZG?_RY%r&n`5d|A2~JkDBGg86_d1`#xE!l=AXQR*m@F<-4$BKNFC zc$aYx^-g=owyj<-*K&CXODFpSRPoL+uk8^~Ya&JjqQSP8#Mh zFxLG0sh2Hj0s=#|aVGntf1(XWn7)QgMXu)6EbYh61BcRismcq>fB=*(<&=SjAxMpq zIw_(RptRotO!Oi|7$IX@dOmzX3Z$WA$Fgl}H-)t%>6ENRCj)QgR4()5uHCNiXIG8{ zEZd9E_=)wO9)zbowPjCQzHx6ZLI#`zutecUQD=z2$k`Cw;c^ zct9jf{zA~qY_{%$#np}#zIM=QDPle7@89dLq4O1d5c#%4}UyUqQ8|P@%n3jBSnt2vrc5$ z6A^i*@5u2cZ=eT%_QCb(XNt5C92M&1d4K*D7c|$-`tRS%@3DT@uZf=ym#hidh%5cm z^l{B`HQe*dh8ox9xx@LaxNO~9)4acT^^P{6S6UIgAW1wZxgw-+be#ROUpdUo}$u4{BBcYSGmqff#+m>F4rA({jBLykAPkvi=6# zg!4gJgTVSFl2bhvUa~#&y4>gUP=$T#$355IBXw`R-RTo63@PI#`w!=5IT8PS z{S^z6zkj1s?}4E?(5bi8JOcoID)hU*`@6{>`t@J`_2#L6`?r6afonOZ&jgs3Q!WaD zXngzLtr=t?1pZ|MXd~nwKVDguD5C^D-nH^t0uK~s*uDQqE)Ie*vFIxF1MCnk0q`Ex z-JKJq|M>OiX8|=HW>h4HwOA=1;AZk18*4v=Ljv7Jy3u zI594Acv3z9Mc;mUd%E_#Svi%0`==z;(|1HWw%hj|&O#dR#1PbNtMm@*^8LsKXwgxi z%;Kg!%6UEab=mH4KI;cyA~1$x7nW;V)s`tv!LSH?au!oKSu`oFQW>yz%4v1!{HawK z?FlZdFb+U_+QczAwLx8Rj#I;fB))gr0t9&Xxo__Z={KJVWC4-_-9K~x`m}t@?wpg; zI~qHpC&>)y*dKVo;z58U=6%7u9XZ4^uFXp?JRM?RY!!kF3IN`F5}H~9ZgEVx9msn^1|E}0gZ5W?Rm3@^w4s@`ie)}uYK%0*#H%oh4*dTvMEsRgx*C? zM5zWuI0O(Z<)gXF;J_HfyWkIQeWckP-V3iGzlSmwKjN!s(ew#Tj~INF66;-fk9i)Y zlwNwloSgn3&W$Y)jE7rE=Ak~4zY*{-XUe#?P*b_!DT|bgn6DX38tBS=oj|F8YtRop z|Li;-0`LVGQRaX3qu)t@3W9s-$^EUB+nT^MMcOYGKjzv&bx6v;0Fa704967eAM`$v{bIoQ)1h`Luuvaec9{Xar)bJ1HdWg zdyMfty%hkzyLVUOq3~aCzAU<-Z7YI84Flo&#;rW@Wzo3uU>=SEg}((YSYDbX0n zJa&JSXdoP*-4@Rept@|&+FIsb=i%b7B;N9We*5wqYVUaoz(C3}H@}^bHpbGJ4kAJ<<0Gqete7E z`O~Q#;QNUHc;^Vc{>`(l$UKuWYJPt3P7w_$=d={^T`!o|=~|ukS%X(>-jh8^<9EEW zEW8}hXbAv0HjMpo-sx@u^Iu!OIlrf)86>TY%Vm22x1nhF!B)hOF-}giMliZvxp-Q0 zZr9J}9dC^#8-3=%b=lbXr?)+kWf2}T6klMVu{~qbn7qDtziL%T!9#MpsF9Ob+sijq z9%A@M5I$9{6C*Mx%J7OO=t}@-5Ky)xAV($Bi5iO90Vz8)h!prApMNy-2Rz?#3_OhK zCsMS^_I*}#)q0C>MeP7QmRtZ>08G}sE9-5X$@e9*C+Au%WgCoTS1g{EK5_3eY4XUP zBrOHw?&@v(GY~JL$y#5M{w6aiejX8Z@p~ElVqYcm)m*Y0UbmioxD=lG_2c#9iGXQ5 ztpugxd}HTiX;&Dk5uO;w{-=-LmvxtD$M%u+J8}4O*C~Jg#)@_?S`FN_YTJ;;p-vZ| z=SY1)ce4f{V9`(>^wCB*_xppzX-C9+<58yJm0Mu^tyDPkD0G*s^Hty_A>HNyToYILS1*{vKCh8CSS2|NC$}ewL~4i`}LPK<<(N3tRMT`{tghIFlt0vxov-v z!jN1sR7w|4V~h&Yzm!zgQz`d89J#b|CnU^nX(+A_Xf7_mZ z4}4dEY3VuEuWuLrI^Ci)9sm5#|C~Pl@sDROX(>DuOmsflwryMb$)EhmPNxogA0?v% zrQ<}*RDLNTrT0E9Utk3o96Q#Zb92f!EUysd5GVljxcjMAct6V`VIDK0WCMsP5Uk_% z!cneCCw>}^Pi#rc=1$4^^g@B>Y5nGF77Y?qEr>h^OT|^l90f_Co*)bPe|YJV zCI}S!W6LX$^1W$ye;Oe7HF6YN5Tsv-LSvK*e}LV(0u*g^+VV7cWZkX|I9PZov6v+Q zg0fF(1GoWD=wgH!z#v6o{m$OBq)~uT+n5^fa|;F_2QWUI1)O)%pWnVBhu+3LN;e>^ zP;S_g0E?Xpkd?>y3@qeT+Iyrw?dd(-1l4&p@fcdOeP8;|ANx+eCLsh^W(?`ST>}GI zDAmqi@!sg}9#k>;=U#C^_P*JB=x_pDoG8C$sl;lN;zV!?rIpf2z^pIgjs>2bHEmp5 z-`GFYf~I+Vk_6 z{n8_wvVkb(gm=Uv!gudVi0uD&+Y?#22Z$j6kZ@DyM;^og?EEfdSHC^Jsbx*K9}slY z<@57702p5Gj5Sf9g<`}LuI>S2J`h6V8Uzu~fpKixbFdBI`qaJ8WB}Urj3Mt4I2`kD z_13*<{!7nK|8DN7DUdvMcpH3Lv8uzz@_mLO!ni>^E|7j9ln1cn9{la;SNQQ*F0T5j zOWHk)oi}1ccLv_QpZt8ewUj-~@Fe3XX!`ikna}ddM&2i{@RwaQ*t^H?ywIg~2#`D4 zQHxB9qcz(e&j2^UUsMcgdOr_%zUpb5&$z|w4QNpU)6yLH!)G?{K9t9ec04=f0(ihc z7K#^d!61AdA0}*xYQgI~6aEGSB0M`+TgE}(-oBi(*ms3GyGICv9mwZ}Qb>Uvd8i8g zr3CWwzv}F1P2fI?=(^UAzWg`r{N?5dpl_af>cg{f2iOQ5EoqjL( zsfK>xyva!pVuoA5zAbNEJlQz#F7*o!$b3=Mvl9i7R{stEtgkE9EM5}j+Uh(?+Si`goaa-$o#B30WChR^o|s1t9nXAi z+?WwLN9b_dDUV}_p@dso=S&%sqdo9KH2@e3ti54qnlyG#4(qoTpA1BlqKF}xtid3K zC`0dg(-paH1YSRT(zskPUe$Qx%~&^Pg~yjhP3KyB&Xg$2AGy0#Yz)64^C?1E8s%f1 z)X#WuxgIi<1h9e^hC=s_8+9F6 zw^R4Bj^9rQ6v69)??gMc_=ev?E+~H+&S!nAjXKpq{`&9r@2>x@n=Z-*O?<~#m73&C zh+~74H6okFj2e-3qoE3dpU!pWO&y!p`v6hRb*V+%KR!b~`0ck$OwVlJpZl2iEcukN z;Byae%Hv&al*<-O&mNGcdJp(fC#v(_PRG>u1<$PSueYT?rA{j8bPb)_ASX+oQxe0= zJ70Zax$BfZzvE>~lU(0YFu8wq0y^pG(Wv=k4P|Wl{KFfXptRq4%YN~ZO{WA*GkU%M zt1DWvXiDjomzxO^aGiu_kZZdvFj~tDTUS zPS&R1$#0#AesU%x1!X6qCx`~3f06ka(52ox84b1WbTS5FY8qNQU27=$c&PoI z+UT@Tq=WY#WKI6aq16f4d@2<3%c-2x=~!#K$U<{~Tz9I`bvIlzFWqoardpZ#nGuz&vNe?E4l_r33ZY3|&)>EVYT zPPg55TXTPWxbemt+uD7}OI{K`fPhVeqC~e+7VOSeaBVM|GBWKt)SvbrZZYQT7mZ05 z&O9*?_r=wR(z-n@FeBfnloeyU)9yp9w(OrXu_wK8X?xm?U5AdRk3XRZOZDmq5HO`Z zeaF+3F)fc)=g*%!BE9bX&I;hY8~f5jTaTn?59V@I`8Eioto($+7cU%>uAM(R3tBGq z^sb}n(6NCuc0@|&Pp`sU9)Ii7lt zR3H<~NX|EMmFJl=p(nlV;t6T2PH)hyPaVwfBEHWU*OP8pHa=e?_SLr!Y&o31v#I3~ z6~EuSbX;1s^H^HHm$&%w4ET*3(Un1>38T8xg2^M&SrdDb&vp9zs@}9=Z?%}}^C&;{HX5K*FTVUAF7Hw@d_oY7(=2H zr;h1KF{e)SEcM^^^&QWk?IagD3GF=8pH}TUnpW>RmbM>YAgJbq(~P^xqkGarTjf-5 zZ)p11?)3Vl9?T%#vu*QlAk7@dQ!S{`Ai) zdQ)G&Q0?O>k0iI$m2!UVDPy~H|CIdCJ>Vhcu;+CThx-T8gpplo;p7qNiaDdxqA7#z zp~lX>!^gA7ta;ao-0w#p-JkXzKAsAUIp2@t%{%X6>X<6b9sig-raJ|n!>f(cn3OmR zuhHpadonNQ;WTSpPrBx;G0pyZblc%{_l6e#*l|!Vbfo}_Ky|-yBfHZ0MyJjk*OT6S zVT(s^J#Z|o*?la}A$=1IEI>n_;9L4|*TE`P1f!-tbD)3(?-#$AF>XYqtrt(o{ny%adY=v)8AyNs z@ZPk$ueu4(HDM%!#XySreBj9Oyms>b41LdU;l~cNt;2E7Rr5w? z-e%63%U8}DmFLV?*Y>4VI|t{UeCtKy)6}tp$KQs1$I|Clw#x~wv0&1O^x7q@^_8Nt zde@P3=vc~Q1Rrj#U~yN~7H|IIs;bT2BgYdl0^l`cd{27yxntAfDea(G``LrPnP*>E zc`!Y-^JpGlc>d^~t~76APkPg`@#(J~*q4nX<$7<;l`7m@S~7i9UXPwPzkQ7`mj3z4 zgLyrRt7;S9tkdml4yLDf9m~-vB2=_{?$nX#ve~23*Vk2sXy>@E8_pS<7EKsMUv=)-yw*SYY$bQOzOiuG?9u6o?T6Ey z>#I3#j>0GLpcgF|o7b&qyVKS)e&C_O^5ux0l>WmN?d$p%S07C4_OwTPy!QNYS=P|@ z9nTy}D;xJ0*CJc&J6r*UvCq7)1{r(i(z(+{rt_z@#>zumj--1xwZiN10esKadhDI= z>XS+*Wm&J38Q*rvghY-x5m}dSU3oC!bL2tyBmSph*UaaWf2=$D{dDNjZAa4G8(Pnx zjT_D$o0d#(J?BvEe&n(JxzFmqzwMHVY4Yf<^iL}eq~$xvycKW4ZzqlJO7FO8QXZG} z^XLck1mt~_TlG-2JP)Laqk7U6b4RC_Eb45~tGDO-#(l@K%u)VE<}@BoL_>|Yi)M{V z*DW0W_2U})Vta49Kw^D*OR=8Cv8S^An}_xdT3@D(?M^>U7ij<1(+S@2`J1BJ1mgb&}X}vU#a3y!wNW9!Q59vSYOWl0{?FvY98JJ8^t$Kew_s zZQR!`C|o#WWV(KF`@AN5Z`ofRH*0)%x?skMb>L{f(>D5$JVy4mruK9Xq#wR`VwyRA z@Y>v|jWgM|yRSceZsnn5O-5usvVX`e(e_VWIWfz*@x9!VQK_z>E8Ma6P+DnS9tKSq z)s=qa(w6haFitt?Aw%IH!yT6{U|sycgOz+)UJ%{^_c&cLzCZM6yBEH;=&LU{>(tx* z^3#XXn%#r@`huAw(~~>e_gn8#-Jk>Cc=_2wU;FFN^ra`CZOM=17=3vr^vNfB6Fs(0 zKYYoAL1V9e%ya#H+W~W+H^1{}`{G%nGA0z?|8Lup-N$t8vf6zM! z5QQW_f&_c-6h%pv)g`M-w%iiOYbS9M$2G}1b(8GpO}=bmJKpssaW+{ywqqymmSn3_ zjp`Cfv3HQ@jp)7a_dD~>d-o0I0S_QZi89Xj0W)vPotZm#?!D)K&iNlbj&1Nt<1xK= z?_T-zr#~(C-+#XzXSV~;Tvk@5T-3G=5c<_uU$vj%<<6ZuW&8H+;or~J?+b_i+s4i1 zg|dD7a^a|Ln|tL*b($;^zNmM|ySv}K!mblCYQ z0#DcPm0pKjo(?vBY)~?Ni95#9Qf2%0t#+y|Nt(9*Y^$`6B+8%w21(d!k8XM_AyLLi zsMwt>IlXfYB(n=(X(s&t#fnDh9rV0(pc%khpmS(M zj`inD{ZOY2CydJA=%7G3sX!fGC}f>aPEt?a6XV51{7EX959c(f}C{qc5E9KNiHfEH*&zv^)HDSfD5dp+fxST=cNe|43mW!-x?|i z?@5xH;nCgJJt#e+Nm6}&P)5cQ6}WVP>cgDcl&*?=alqR5)vB`K&O#JJo+LN8usE@eAo3mBL|Aek1F1f z#L)NMI})K6$;!;qXDG@|mv>#cIqDqy@%N7&kz;iO`VQz9GSi0T9 zdG}RIWqW{iAL@_%p%L{e#52u2tLJ2UVfO9))QRrXk)?dAxuu;- z{oy_X;d?i~E0TN+4+*?$p>iqc8jxpd4!eiz`l8agvVHqXUFRKt`H{m?M`st3%E46m z{r9XN=Pk_cv5IENto6KoGo5)Y5k2p_W@?cTlm};aMMsYejU~v)@Tl~O=k?ezI3};Q zXG?m9mjpARzc&ni&_9fYD9qYNQl-69B)30baxzoYixw{_zOC-D;i$Pu`-n(VVR=+o zJ!Aj3kG`g3wDUNUnVT*jxMj24KQby%PtpJ4IgjzmP1MCRtSQaaC@;TPUZ^^C@%W)y zKe}4YhZ#@2=CXzklkLD359O!&AR~`!@GbVM_Y)sU{tl@FeFu zgr>`6$?5_5=5r^*(R6rS61{}CFBD0Ps~TB>F*Gq*`V-Qne-OZSscd(?$CEErNM>8a z^NR9Qp|To=t^jl17 zLbOA9AQO zNUm1?b@%vtt}K`BpvsdTd(AyCGI~`fMZb+#S$RWcxK!0SBXl{AqXASP& z;B`XxYukEsY+NyKPM9w5)%N3(+}$FrT?0Co(X&Sq#eZV)TAihUe5NrJ?PIqLXOv$s*@ zarxZmJ}33{^|Ef=I{C_1zM|gK?wtZg144h`10M)){`%Ly9+e<5yyF5^2>BynnDNpQ z>H-<}l4N-(7i7x*>f?unaA-Xz zAwk}E#gf2|AMVgMDxxPZOiWRzjmJt0(P^_tMPr95b0zs%IuQmGdjA#W!T@Sv>MQ2u zXh^!9XG{1ET_O7{Gyx_6#!5nx0RQf=B!$2p5ro*LI?v%0;n4XDL{Du z{l^d0X!yS>yoyPTDc34>$3zRBPA1|ORzuAua9oHt5dhdiaE!htz7DrcL!l-ru@#U_Nb6XMSR$NfW7 z(l;d zM}@COH^HdV2>o1joGx%C2qg9tTX@HK~VxclTzLe9w&49LW!h_>n`Lx4Z?zZixacNWN9s+AYUwI@Ab8cp6?IT%T(Lypw1VSyq&z2@_e*%Sw~cZf}xK zeZzwqlo*g>N#KYg!=%ZC5L!Up`ExR(CR-(WE_`H1N0ubhBhixesq2;1=9w@yf!2Y)MZ}3{%Bu-QY z0ra8pxif|m778f&lh-Sit_nJYSe&8mz4E=h(LJDLK$%4S(j;gflv$jUuGfNlkiKQ* zf?WC2dp9ZXyBDgMA!amV?E2^3CzO7~u0h!Cg8WRW_t6uOgTdry`O6qYO!m)s$)A8+{|=&A&W*QpbJjfUyDNY>+S}Bh-n@ z)la$fd#*r3a=xoiHkB19U5KA&72aa_B2Bm#-lwY=df#(G_!-#$i~TjqpYM&vcDCr) zjscs#gmH%P)14p{Tcf-kL3@a!ocJg`Z_ol*b4vvAOJ~3K~(0ACORJ(hF&SMjk#Am z-x`1s!uy<`m!ZkgpfSVPz*tEeBRTfGITMdBb}#-V6iS4)`-O2i9)V-k`ZYVA?-!#S zZ4RFGvbk9TsIq#&99dN|N1w%x4O%j$`DHRgx&L_oZ>YbF{>#5_qTeJ%X08oaJkfXY zyt7#4(GULj`?qSp$-qQ{u$d6}%9j`CD7_NRZfS9@(*Ddu*)v@R3CX|x;%Nn_C>u(_ z_wOu^nM{46>n|?K5|XX@g@nCRmB$X&gh6S}#n}48*Uk#ut9~%kEgOAp*}UwCBl#=6$-epz0eD_{EX6>9t=vGj%10ry9QAv!B#33|_s z>pf~f67n*RFz-j#v-bkWF*#ar5`sFDv3D||-@TB&*Iz!mgg<`(36_KAf&O!6`gcoc zjQ5v|qt{GPh4{;UyZ>e`iaur(i*~;o$}Wsv%F3BtMrvUE3^uawg4O_?fA+JVjVpug z-}uHi!rSqv4)S^q9}rq`=|24XCi}f%VZM5&Ft`9nxcq{g`yVe?L?uSR>WL!8u?Q1z zJbBFv)UJo3@YIod1&K&TXrMIvQI4>jqKKeKxPsDx(tMz*O*>A26F5-;df{}1gOgA! zPr(%Vho_E*Pyf)arCKK5UHk#IGe7`Nl7*H`_pUI|yFx-WSGYGgBLUG#&_;*Nq=)A< zaX~`c2A3mV8hpDB}sh$>e|uoC#&|-gqu67r9g}bxmUn8cHg*efshc25b~fT8SsToCX@%5=a+Nm$A^0Pk8yu9;W6G?wo^}iL&K7i zlBmfuaQNSo;kHEp*#Td^`=Y~Ns_%PLUOw5V+^yI8z&`i-D13i; z-&O^;m?SaLVcXfJZuzmC(`2Y{c+p<))WJg>FX-Dh_bI3DecP8P2QNHhfCvESw{0wu zlDSzEP7DANtv$Q<1K4%Ep*;$icK63xfB%)`nuYvdpE)I6{ewE;3w4VMsx!%l8~^rA ziGyM@zH>rgnjsIv)Swd` zz$nNzdmieXdg96HSB$IBfPCtE@7Wf)ZDXnXn1So}Y%i0VwjNCaNu7CZ6uJ_RpS^at zSKAEj!}>D5$sh#;zH>OSU4ftZcWzuQN9x+8vAs|GZ1`$XbP<&7pYN~I`n2;H8{iPO zwrSgK34nk%8*PSfi$0Wo9KOzdXIpemz_@HepZ|?LC#Bl~?Ep~ldcqr4`)&StUALX4 zeR^_Ik}NLDk;>*S1#AuEGb0Yhd6V7&IGDYmnQgo;z6*-xz4D!4fMtHfZve8Y^F2a@ z1ray2X~s0ZQO-p%43dWS$hX0MlwJDh{=s2QS_cS?*A(?^V!*%sW^P6sN=-@BM0^;j ziqg|Hiph>)$+*II?)FX03Gjrqc>#t~+Dq!3q$s*A63rI*4E%1{mKgMGbB|K+ET5Nq zLCK(8MHDb$T_|yv%_HeI>>J@esCQR3ESx)z3lT%qU7HuGv4=Wj{zjr_9V5p?)-TGJ zd$vZLHQe(2>sfHQg@isLTj}GH<>S|`l;id7^5mg9jUps(1Hkof-MCu613PE3C4B@u zyYz#uxXG7bW? zUz}qA3*NPQzSx|Eh{5U>qEYa2%?$+7+~>2LB<{8gT_=$sf3bw<0g~x5UxlmQB2{A@ zKiPJSHW<191L>QHNO)jpdDOQOR0jEa@R~HiIq0E%)zbo|Nz%uhf?PO^LwQ*#^6u3O zp9wW`ne*e!*ZI5PlVxygXTJ1>(J<~z}#w0Gk#pKexQ8V3&a zI#-C)LeK5&9gLaK*gc+g`tvV8dN^t<*mJT`{^-t)6SfI|Kf~Mz!yX*==({+!SzZtK zngCYlLVl@g>C*g4t^*XukSA%=;<-_dVr-wTcf0?(_WMV#om0J>F;Mm7O{My6*#4<5 z8FQIeUa0V`GOspSRNy6&V~%#A!d#!fAaNayY`-}r*zdPudH1$5)!)hGf#7&>@7D3o zBA4Rc{NI3GGYj6gUt;;h4Xf2b5GNYuDVxgjr^Ge?OI&qRdi2N)b4sUO9tQ))eD{Sj zvcI}TLC})9nR3_WMd74TJOzM&KU*)`Q9w!XgSQicWfSF@i5?#<;3^H%05}7*^x)7x zJc$7*2o3WK9)*6dLOi>fR6#9)T4-KRfX^s-2#FY6uN&LWzK^|og_Ir1(oj(DljMOr zeIe$6ra=Nw0ayzV<_Z=&fLyq9_jt3cCv)Z(KB2ex6w!_7G<>j-dfChRpCJxWwz>eW8SBc3*_d;w_h-lO|+w zhgN{>s@$ImlpRQ9i?_$T5}1^j&>l##h60TT-Sul0D9C90DVM_)9FYXUn~cdS$u3dU zFgoCgjWEEwo{S{eeD+v_dJEz8#6E!Dibl_K->$)u+XpNFaAh(?Tc>Wv#yrXZ>J{%s z|1&L}m#d!MUHx7{L`eNj05y3>ie%VEh08o62?;!y7;-(113U)=+pK6UdN z1)S&yTtFq(_1Mm&F6ayAeWBFEBL-xG1MT&2b*uITc#QiMSSGC`g)t$K3!Y4_075Z|$D0ixvAJtNlda9m%aH3<2PaP? z@gsfy(Yki64~zlSFZW}hG+>>z!`IH77m|^ph~W8Yh9J8S-WlIK=l|RDr);wf4h?ET zwL7;gl%m`Wy~dH64y`ZxO|CgNDNT(M^c60eNPkKDCE*YC5EMEazL0O_mJOxCAMKUz z(_d%4!?nG*+sKgqv%QrHEU}#l7`Nwim)h?Fgia(RmsPKtL=V*DSd7-4Ku^2E6Bf_>h56q6f#gjtD9w34 zf3r$~X+RFZSsP>BPJnH6>&yr6N+Y2h227%$xCUb`1{V{;l-+RykFm~vwv0(@_n>~W z)HTUr0XPB4cow}je8ZvSX&ggz!%ZP?CCaA{rTqi08Ys?o#t+6+zu@iWjvKrKzc6?C z>@Dl1Fo$TXWTg_r!0!s*g1`6Uxm|;|^)6f^IVm!?$&N!A^Ct$S4KrVy4!weV->m=E z4XZSfReLW~AE%-se%SAKv>-_l(M1@_=*yv^A(0PWXbb@WD8%LPezZ=;MCTcn(9(Z> z|L?73h4RAj=)}G_3HTGFK43PyH?DT=Z`f+9YSPIyb|Qi z_GV>E6evwu^yzrk-LN)zyomRDLB}yJ9;@w8gByuHZ(d)j<5SSFv9@7gXRg2)v8W(B z49Wp=k-*r5cS6h20Ff3Xe&%KG41fE8is9K&@#nwPI31q zJ9&59?U&2C-@(i78=Veeo^Zpu1+wROgBn8_kIYdjdkXJ^@4CmKyQFL=59tRcGJA6WiP|6nnf`c5VNO`r4=QV_+wi() zj5kp|`GwQ+*AHxun$y|-i?Jl?r>%ELaTD}mzDbNXE-1})F{GjY2049PjQhIOW$K(@ z2x_@t$|VfsBgpT4mtT9*NeE(Q~v7TZ3(gHbL;{oJ$4Ro}NVZgNVFNY=b3fEcW za=7Qq@g5_y@5}M-C%NZ`_fzM!Gu5e?@;D+i=z$U(tG z(QpSK11~LX++R|x$%X$oe&hAPKxN^5#(9PUXaF>T1AnnPsL8CYWR2bzVe$yy1~efk zS7h%cOdbLFVUSq0AWxn;TptAzB!xVc!UZVx1!3xuFk$v7$ij#l!aG}g2GxrN>$gz@ za2$gbN+MawKXub;S?Tk%fN%Blx2;!RV?2iUSDu%Q)Zr*4jY)_)9=bQBp?OJi^SY9l zmS{Lr4_3G7goXMC7+{dFfnjZ_=k-CD@poQ09WEWuBS1$iupEz)j-a3e3GZ57q~17e zdnKVsUOXw1m4?@3xCpTjWPrDa_eDaKc@{k978iJYw2?EsHNDReIMfSJOaldg2pL#y{$g$qV7juDAd?+bhp^Ye+kwcx`K+ zGrV|R=jyFhIf#=EV2TCv+`6unxzu5VcE#c)Y&P|K*qT}1**%2m# zB->k3kUfrq0_FC*FP%|Em}BvFd*F(4_3Ze_)yw6P12yVV4&c(#J*dy^8PP=Q+In>| zb?xeT>JiL$On97qL+*V85dQDSk0@|y+nM+igipfZVcS zq0$Y&7m4sR+D{6}DH8=>EiXDI(jqv9^8AKEyO+t9vBrnV&ilcu6{<9F9Q_7fzkKrc z4%j|2GAaNq6A7MboR0y3=YZ$;?yY6o-)&h^D0s_4;jwhy997_7JlUkj^SXrmC^i9k=a%xjqq8t}r)p`*ij} zM?r|Cy&u}9ADE`kzHaRTrN+RZU6hldb1j$hfj$dQa<~0pl=ByRLIm$R1PU=c`dzsyfM$Z@?Hu=5@-Eo7z;^vCLSFCgaTBdCpPwV2f6scQ z?85khaS!7lUh_c$;5b#m07ScoRu5Y>Va|R>j;WRjNFO8^fo56&gYZ-|0gPmpG3P3Ob)?w0EKq z#Yq2#-~@2X`VzTmeQ+co^G^%VqF?*gp3`CHNgTZH-cqJ{UM^-m5!H{*NQ6Cl7rNke zYoh4@FXr`U%l_rAfp;u04TyqOo4O?>b%^a8#nW zVU^<=_iQUu9wP?eFTQV^RJHbmJ5x5VT@S?;L7EdU*>uHXc5lW}kkE>|fbHJzT zc_n(`=>XpV5rB&sDam0^wzDlga;m9Ic?t>3G=W|LZ&Cz6$RBRac0giyjsbN|2veO5 zu>bhGS4dOmBB?yzC2yQp84B3lX=2fp^DhGyjUXy(o+&;WYpulWpDz>Ko}Gg6TQhJZO0XTmt?RG zg+$`@0MFob!6v$yhkAjXjJRm0_dwi9&Jv>}d!v5z6 zcPOpFBl~NlsZdJHUtS_wph7c zL1Bd+gSJArDc^(%-wAE}LWK77m-la#uRU>8!F>g#(|Ctz<2)1|*tsOiV;`W^?ZfbP z`|%s+H1v^4ES|#%-kxyw19F5VLFBpP4LbSaiFhA`kN?dFwucQKb}a1>BZe#RwPT#& zdw}Z=Z(;iCG5V;)1j$MB`iJH7=LGWgx@Apeo#*572fwZ zo;$5xDDbM%J{eo+`-q0{d*hnnGZsfa=hJ6Fk6FhxPh!7czd)-_nQanC?M?IyZ6Kou$ns>cu41xKBTb0DfXz0t99pLaD*nMF{ep zn-_*@dmv0?svcE7gIWd*_x#8c84yk2?3@0@Hv#fbdtZC>A2cs29G0U%XkPhM5)f zay5+G%JeT!*0;;oo;a?QGVJ5qp`F8*PUMV@qyF=z@|(FD6hGCiy=oLDtQ^25UcnF+ zB(BGN(0>0ms2BKiC02BX+lJlagVyl`CP7zgPrqcRr^*#e3udgp_tO>G^)c$qw74F! zRyJTFbG>+Vf~Su~!rZM4yo@ITm~H-P$J>%VX}W?A#k`JhZL+)+IMWQ0%yCY7*ccvk zT?`r~l+Pc1>_6>3F4wP_FLSceN3`L_aKbgPalb{Yx-(4=DWvEXPxH?l33gA*qhduXuk|VN(2#+mdqo_WnhA#J}7?y z^9h_gFv8IXVPIoSjz=&s0jA@iGe0+D#z8RZ_1ZP_$3;Y3y(*GW8QQxq+_h21e#SAP zUfoFHc%CEHcO9!AaLXj>gukf@RBGs+L{QUip<==r=B*Of$m-rhaEo-0xbSs6rTx@f z%uil9-K_lnL06dSI22UI+ie?qpn=ld8&soE1+7~&S1ejP);&LRpgIa?!71aH(tH9SKaQE#0Am17HCPAb;+}?73t0^3ush4Z~-!yKmcK{n(9o z9)$_c=l}b3gxi@Hqon*fX$nN4&@rL0kRZaR0eqP>F_1E8+E5ms7oYpm@jj^uDd7y@ zfCT_pHmShk>MCLGx$&%14odZbDi8P|oc45Mm)4PWbR>rH3m$mxgt#<6OD8Wz4}fP= z7PJx}Zteu-TJ``c1FL}x3^Yi9?qU=w6{tQuS_)_&s8=_+5gw*Np%Wxg@&nU=qr5w~ zI|2S^(vyw@PXB7VBDLzP(D&wKJGZu;o8%jMqfW$JyKmyxO- zc34B92f@>hP+i`+3r-SLjh6s`>Jx`+!n})2M)3^BI}3~$&$s?Tk2b~X8<3Jg5QS*V zk^-#{yFYhnNkN$m{^>oNl(*0W_7lR?1%MXR4FW$NNj^-tXX7Zvv;9H^&(sg@ih%n2 z&$VdsJ}VawEPx|`1HgAc62zYSWJ9NN6R%%5S2JSUc~e~i-uxhxC=}2PrH=%cnQ18s z3gGz>3zX(sLEP+YFV4;o65Dn4r$q73@|~l2{n?N9YCC{K-X9_!&ol51VMs#xYwsIW z@5R6Q`N8n1D1r2ecs~Nh0Gj;xwMyB2v|j5H1(fzleG+9tKk9l5^N#&I)(TX?PadLq z;bOd}f82dc{`js<;cK%J3#2u!)JNZ}Q7=+^uk;y&)e@xA|L#9!hRCtB5wWZh7gqx?+P!~{-I=T%jmEMP(tqj z@A;*Z@tP}9LU{C|pkhP;sGBS_daCzp{VE|y{;|In&sAejfyL3*$*oc9dy1@{F5=~nM zOoh(qPw&~Rv`l=1ycd8V#u@}iLRvrf%{e?XfBwYH%(-}fHm|UIv?cX{f#Wxey-11w zc=@cne9EJUz&Jp=BWDH)h@tZ!p)K!@kZR@%jCpvPac#y3!hl;8&`gmlmleqec9rWl z3D0-XoC%}H(?{#&NKL!WLFfzb-nuAkymY;s?4Ez^n&mNpng{n)M~$&~lhbb%rzEKt z-An+}+z0Jox}2EMb-0mPl!wPT{ojQOeI4HF{u~h-2(Z(hGz@_dmj%ys_XvnQ02z>X!EF3uN0f}TT8 z9-bw_cb}jJVvI%fO>P1%94gQsUOjU@iVynfqjmCyJ2vP%Y!L<Emw$06$|(Yejn7X)MZq<25crp8PDb4l{Uf?dEFo&nXfxa~Io9!PCn93af;m!8 zZh0Z%EoUmnF%JZse!Iw$;_N8nEB6IO zT4{(L>|)+0^c_Kh6T!{vOEl8P9GUDm9Q@vC5-XCZwxvfV1E$Y4_l&C!=2`TeF74#F zgJw6(*1%<40}+kpGJXjEt)~=tgz+dNfJrE{{Nde$;$@Rs1}%VTfXqL+dy`UR0Ft6u z`YnB21E3GSXn279LYN+&T_}mBC^Df#xm{MA<3Bv+?;F=H(9jJIz!MLUf)0oTLU{L? z$FLOusNB&zph+2mLO1O=TjCLi*9Ff4o<|5Wo??Ka@7qx>t4k)I$Wa#8!V{NJGn7b@ ztzZoWSO921p&d;0i~@FTU}}_geHVZQ>ap!{G6T-I%DSDy#OtoDi^fgj;9kZnU~k2F z<>BYdQG)U6^vlWxbL7v;Hfx)~BOFCImZC?#XmZbFoNCxPPLJbI@U`#0CB-@Oa*t}@q;|>z&K1={1 z2&cr*0?>jY#pHbDg2;pc&piJA>cKaar__ulltD6p=?M2|p2+_5{Rum1KL8@!o6m7O zd@7u60MkM8gICT9kq{_FWJ|w;WFV#E@0+Lq6TB@4x<|s-1vt88Ly7GUT*j*JyzEr@ zAe~!L$`p(Wi41I$dRY4oWXvr8uLNtFkFk`)W;M%n*dv!^^@?U@D zQP1Pq_OT=soq2-^R6~D${}z3}yhFZyH({+E_wwmRO>$_zbs6Gfre3*cKpEO03j23n zJfq*n`b8udT__}pqFuNaHyONNFAJvtWU%udqhfOcm|$T$=A%pUWU(m$N<_L50{@XB)a zfNbvSlOMf)P61~qNl;#QAFflt=+;oglzVT0)ZcXY-RV1j(pssJ2rfUh|N8&<)Nuuj zp^M@B18}#tK^VB}b{YRNzN3q`(jC)6G+M@FIxchA#;NLGq%HFYsyeHAoV;qOx%lPOD z$A96DjqawXb#y2kuAuI?DF&TE)Ctr4!8e72#(}C9)$N%VV95Pt5~5aHy9QLxqX@{A|^w{Kjnx&XKVc`S&IzF7USd;Lqj9?P-WuU7@#7e;r4`2~6+ zznGh1OrDjg*T6(L!_-TgJ>z24Ktv0?SXX!}%1!dZUs3Q-m{4c|O~yt=Mia&qm|;?C z$rkJuQNm4PL_BK3FFVFv9;$8C>)?UpN<~xWlz?e?^hi)w+1#ZQB>)Xq33JQ-KTjSD zPj=W2Ksp(R9o(KKD11zq5coO&vE!Kh_KmAF>7yMp(UST#?_vOQ1{ek-cR)fpAv~I( zljd$O^dA6<`X$>v4_Ta(p&o@OGfY%b_q8J)@`A{Z*LG;=ivM2lUc$?iY|ixGOq%g> z0vIwc1OGV}@*D3DfOtG9Om5mx5>}d~%7N%|FQO#UMro@`O_7iwzk1EGs8AZ(mJ6;m zgcKhJFF*~vR+wP(Ufi}kZ;rkQf%9`B;TxoGQE~yK zOsFU1L|un^MFKWkWC`FC^^0=n2chv={KO3_<*Sb$RRa_TKgt3iLdkZ;{e=PSsY7-8 zt@6r-Mn)90LCIsnpul#>0O$eSkw_8Br2QYSZJ!jF#_)hq#RS?SKs1-b+V5Jg(Ay6tSiY^ z&(omiiFFNkI|fUr3{2iQD{}69k3gMdLbxD(4GOgho^AkU-+Qq_+XmasNWqfG0`3p6 zfbf5S*x$Nwm44gwi|1Opq^6~JoToZKkYD`s0Lmv3ABN^w@4yU5Zdm}dh1s6>CB~5; z!9YacY+iyKLwiDjbjPf}`^BLspp$S=+Ao~y#;YDwe$Y0?ctB~Mi}3};24j%hemQR< zrO=FQfD?={03Uqot4ea^H{QKc{`2LtY9yd9$V?&Wrq}-QJcMq7ejvIR7p0B)(j$k$ zw+u-B$(z?`UIZes{B?i4zEf@uz&nDH15@akCePD4d;w<#z}&uf>Ywip*?YD{y|bA& z1__?yK>`RuIT$Rh52Zh2?1J(LdX?vnH7HN=M<+4pMSse7?M5!dD%ah9)4EbQd%jBm zPEDYbF}`v*UVrjNm7EQEZ_@`dA8-vtM5$q%cIB!S5hFEy27efADVK=Nzkl?wwnOHF z3VKU|eB|mCfrfXt+@SU5&PX4|+{5cH@LF!xs5#oZ2|##|pc9yLg}a~r9$Kz{dg_?g zGv5z=5~4&*PE(-Y;=4nS7#$teXLWT9`XdC#o!`HGz3MdF8S{#G<(M;Huybfcd;?0$G;L@DE#8@cm1JJ(&JzoD1$K6sYtfP(0&09iztGS9a@rZZ_s{`ThwmABW% z=&rutzN>2>*n`!*SDrUWc>jd~)69L9>A{7`P~pjoNb%w zrRg^_UGC*l9uaOIxn{Y37e@PaDej$q7EWpU02Ag6n?v-Rd>7N@l(+KUE=%T+i3uKE zod2GP1|WxuU&`Z%hFrBESBM64=N;TqWPOvux}jc~HI53@}Ji~*> zcJF5Oy2E-|G$&KJdt)6x*>+Rj&C8s+AZa4X9qrTTG5qDcbgEGfRJAE^0tgS#!C-|_ zYD7(M6*2o#Dn4f9ZT%+s3k@ilr16}co)2fzO%zoeF}Y`%YO<` z(oaj5sLkj4SV2HBU(RhnRxsp2s@<4IKY~RJ2|GFqY!= z3#YARBe#CRe}1grfQ0|=xl@9d7kvmuh}$-n$_;CyJ(GFPAO)O6j>g}J$LObTSsUw8 zQ{E0=zh9up=^iN8e#iEG&Cd55>CvMo|YsMYBq*d&_?+dp?W~Rx#yJp z=5?zyipM6KT!RFEe(`fdv)=^)dGAdfeM(z0G}5l#cPQ&FVB|l?e+_;EW(Cb3xE5`c zHev=f@&l}PB2wsAg1m3>6b;HZ5OjTa+wjn+8sg~Z5WXcFGoV7jP`qJbez?xvb8VgW zzP-Fq+bUy|31c!*WEfE~X847Ki@6!8@*CH$(lMZ^t4|dJ!rL**xI$mH=R~7y_a%$8 zW8A*bwy}(-?{tr6-6BZ*+pSHx&*&aA(VUUF1WM#^#+U*}%&(vbM**kK5#~tLxgS); zF#Vl9r^8fk&sbcLt(^OIjx7!B zK6gIy{n_}!F%6x7##8qcO-c6JEd8Ggj1X-@-})QZMUoZA0=!~v=e|fPjbnfr%b7DV zcf-p&fyqHg&Ug3gI7?aRE%am5hkc`NvSLm#fZNVz$#+Trg08V{Vg5Mdt?Pu~p2u|% z<~Y!r`Nid^_wV!n=ABGe=sxg>hi=3`Y06?QjHfz=OT5q#e|^|BV>xTs{)x`@N7L9j zJ^jJdG(m|6(bdo&U7-!oCr;I=$1P)aeexQh{Y>;NPF}*)k6_Fr(IHVr(0AY|4JeJX z-8)5Szdw4tQgH>IiD-ylxoW98`q(qlwsAIt){MT6#LL&MnHP4>vg3l5M2LO=m9ru2 zWK10+nY%rGtYOmVhoJI)8MdM4&~Gz#BIrEynkOl$!5TrX6sMlAThaosA-fZo{As~{SHiOc5& zT?O8G0Npm}z|*RuH{#iPAsktDFG0 smtN^1fNIwp$r^RWr%x^JPj0P)FQJkDbFZ zNYQuspP@n^eCudkn^5m?lEUGL0QjM31Hj-lgxG}7CYNkuLJBXVc|fp@{eWa7TLat! zz#-uY+x=mVTJQwV!2W-E=0udoGoEC4dm1ofWv~Qv!i$^mN%uX%+c_0sLWY+$69YVo zZD2Mp9KXVCz*f+^qz>^;LjVwYU(|O|cDjO!gbM%8jjJ`87VY}seO2Q);|S$-J*MFC z^k@HO68_4WrYU>p`Rj#2z@JzKFz)%|jp8@**tu0LJyF0keBmT%B$*4syT5kTe3_S* zp#Uug6A~T*DrBXnsE0Ah{`^AHFOo?@Sz$ukLI4qt)U`(e(YQM&NKT+p-CSgE&cOu}BzNMRoT7zlp) z#yR=Ioe_6hynnA+(WwcHtvsSJf`DDZ?lBtiZg>VuNJ{(SH(<4fQ6@lNF(#SmDN*wP z%71u&wP2Lu9JqEdjNZR}amXl(cXxtrhw|gm12x+AlM+3zVho-x#e@5v?A}@0_rW?~ z5wuk-0dw#`Gy|`D4(rdJXi(2x+s?N@d%0oFf=Rvo>>QSOUNL^vclJqYVxoGH;-!et zR`BTFbD~MwLW%2kE-MU=Gk8zAo~n4G`+;f7A&P53ex~-P{%~}=7E0bP_EifB6ESx4 z{(=PBm{{T(Biv)G|D)e!euLr7m8(}o`a5?!>mS~=RK2~Si&5iA-;m~xh_qYpj8-DY z92yw5q9_^F3cvBR{o(6pf2n|JJZmvdLEB=oxF{>oh>)K-ethSg2lrKJ1O;RKlEPf2 zAmIC$iZD(Aq%fAKvp`~kCauQHio}!g3hzBAnh6hQl5G3MK)aVuH|cjkcst3{$PW@r z+n;(WCAW*BG*(e8yX4g9XJAE3}^0&A-*@{sf}kciBO4j!}Hjji}1({qSkTm_v=re zP#O!{mY!NApSyKk)V)ESw67=Hhzeh_+jcO3VN9fU(FqXh{89;A*u4WIN-sv^bmwr4 zyC&#eFP>_Q`fi}P3i4pKb8LCl%3_V^vfnJ>YW5puZ9fLIAj!)bPy2y^AxMJsoS@@o zw_ScUfUX9g^gC64{6++n<{62yz)|rNH$>CDVl*nawU{VR1kR9T!axQl33MKWJQ)|z_3IYs#PVXi zTAL91%Z|0(jsrj^r2L`1)v6dWhvghoYKwByHPpJNe=wAp-c#;gJQWGEwz5##P{65Q z1TPW(`~v8Nvd-U@#f7R+f`|~#4A5%cON1%Xd4KuQ!>Vut!%X~9=1C-YypH6F9yo7C4ipo=kch;DcSgHYK&TtwJ4pdWKmtHV zEQfg5`USBA-irYR3l z?t@n?RU?mkpZD!pqDmzRh7god62iKI@&IKEKu|0$`|agL!XF-IeDeluVl;sB8HJoa zd2E#D5DDROVI_~g$K-Rjt(9w57ArVI!Z^xIPfil}U`-Czw2lL&0TKwEgVWU&JTzko z3a?*y&LA%#k`dzdjFEyQU&a`V0TfQnt>uL=J(jqJ^VGM6Hx+b;Bz4e5Y^&-7prpR1zrLP#^)O(KndjG4Fct+4i6% z6hV)?SsmVm=LmgUQBJ0U)!`i>kKLd;caO344_&oXCzgk5+GTWXR3n%$PSCy(fOz@> zKxKe-*DHtws&{T#sQn{eyEaZ)*(~Yf4IpLQCa4Od3UdXw@1l0Z&D|OuMn4azgHUJN0=SZ`yjaVLKp^^;AmBf!tc=vickJ%x{h{0e zkZ5!WbYCQOE6tbNL;TX;e*Sb+0z1A#`YgD}8GraTF|1jCX$CO&I*jKSquutdR8p%V z5iKNLURz3nu28=>FEa zrSV`!nET|7dXXehvjJo=uO+tz5mL;d4Q%II=o=W8oXiyEccv}ad^za;iPC~w*%cgG za29bXXYSlurftmT4a`L_ESf;;f>GW2Pwtbbm}L{pE9~5vT+$x}ao=aAM|1`HHgYCm zJfII`z8tSdG7)Y>g);BtS#dDJ=nzYIM?|Q)lr``;`y)a)KPyI`Z@Oo!Ys_w+t%0{= z4d4{k&^{g@hQev4gq$8hVo@?d`+ixsVcaJvqzUw(5ALlT=R{)r<5I@L2hXlt${Kh_ z0n>P;4h}`UA(_mO+{M5oJfWEUF%dG0H=rnCUQ@N|F=%6zW^zTDcpPJ0CtRvzZif8V zJ2r+VQ@l_TEy6F#@q}MNadG>mXljahUp&0C&^Z{d8=VwMRZ>$D1&^#CVZzO14*?MU z-J4hIgvFiY6y~NYJ&1X%0CeCHNtktePsAecmd}JYNa1&nao3j?XA6HOa7;0o=HBf7 z05xm^4TxeP&nWl+Dm?dRzVnJIi6rf5>l)Dai6+gJzCaD{U4 zym7rOXvL1GF>&^~t?@8fSDJ{ZLz`s(Q>`ICD#Yf=mM zSX<9YOO?e%(WG@apYSb}$4HjtdaqbHWCVA)5N-Rhy3TRHG@ueRIVe)rrT}@lCZGbz zErSH5l)sYvtf=I59E(-f>dB?Xam$Id-93hN66Nt{-vfZiJ}0pZPCEF-;tiG<3JWD{ zbt1(q=o;|%R)r={iwkp{1h^H<+@y#0B3jv#7ZveSB#B5f_{%S+cP*K z2?-v+&ikOBrcZ_r!xca<{!kXb`q+^$7-`20I^TkAWBIP zh_WaQ5<tusv> z;c=PwzjaBWdKY6rp!~_?_ikUWR4W)qpiTMKo-=AhqTPFjCa(m(Z@V=*!7>UR48vYp4T)4{RdzgZ3aUxz$}0-<+_Hg;(%mUZkhgd z_`F{~+bp%MebV0BFLSff)hiOf-J*c}=fwKW902gxr7U4=!%)rhnUEZmy0CUa-;E*8 zFN_@@zjlR?&@eYSQNJ&MN(>sGxn)iGaJbzuielt~8y0#J04M5)WS1DbT>%hxDa+h_ zldS{L_|?|*-pw3}ZRu|(eVQ|8@Cgc3kmJbcgod=Atbz$u2h zSbc!MZt1hSLdFo=hp`Vw7AS^hLTGCkIQ{LIag#*PHm=bhGdA+g1p(83z!hT?Kt@^$ zhP6@2Oi$LxGIRpGk8MtE&p@Aj#j?V%!P2(7ON`6(*EWXRKIW(2c>YwBQwT}9|LCqw zQC`M&e9)3_%>YgMC}`}scN;g^MpU7J(v-p1fFsT3vVy2+t)TMR=FcBs$C2#W6+q_C z+`3jn?dc#} zc5!a74&^V8dE1}fy;=1fDCYR>(zm(CuV0ua5ARQqq2Y*^ImSR}q-JHd1}?W6Fy{#W zebaBvlwh>MsN|9OUc~KaPm6< z5<~%A=Rg2Wgvn`;09v>L;Ff=&I;IxGOuB z7AQRm-~hA(gyDd%@h;(1-3_ zj*^LIEtM7YNF0~|03ZNKL_t&!a1St#mk)h23Nv6f#)|9K%vX=(bIqOd;J#}4^B?Wg zD^fCEu>MIX+;9K<%n9|Bg?0$eV>e+HCx7JXW%9^@TB&L6R?k(wkJ~pb)V|FHrR~vP zt2iG9rU7UF{Y8)XoO__(BOw%hK-iF`?IS_G`02B0=ayT5=fp6E0gG`EV3&C`<6P2c zf*L$n2i!zR8p4=~=X9)Z5->MNh*l!}_`l!3RY(H-(y3X_%Qb8Q1e=HU zkV6e*<>P(r=fNeK>R4@?_K9{}tmXT!ELTsD^W9!zSvaT-q;$a+-aEz#xbLAUvOa-t z?xDR^+Q&lQWuTq4S0atdi=z#LvF?TK^ygoB^ss_578;5A1qBfX`aiyF)3~_@=P{R{ zo-jc9<&!t9k>^h|NK;3@0%|83I@B=W9{0@A2DxQ@BuNbhlOW;U>^tXith@V9H*_d1 z5EKkhU}4ZVAs5K+-n=em1c)6s-6c^h@UpqV&O)e<-T8*Sb{xA23h^k;W% zT_g|hi$vo<9|7e{Jd%?>zTe?IJg$Qz$jOE=mw6Bd1w6JHXGzFOdki{0sN9*_#$4vp zx2{z~G7cGqIjI_j8kAsirmi_zIm|64qMNbvNu2xn_iT{Y&oqUSh^NUFOA8h7w_|3P zvo&zJ)Bt@GdV-&4er?Ij8d^w}Tvm{!QGv|WiFQU`ot5Ysh^6N-R^7Y3OkOzAsCo@L zGc;{MQ614G&i;C@j>JA6s{DxL5-_c~|KcPsa#o22P@ov8Ks&+pw5 zRz%%%m^ho40T%oGWN+e?nVLF|E`Z4>A@rznc(vg{SJQG%!ySJ2<~7Pu+0u=-GJLSP z%HhlNko`K$ot7YSsH!zE?3WPKcs)_vXP9jLnpN{P^cYV`yxQ=VVFHM9jPO3yCYj&? zGbBv`L`O+R@rLW}XRlYv`bBxFtfCwP;*$sv?>|C5Ov;OL#GgcsdJPhQs;NkBfHKbc zK}tGYd~RrAtn1(bSJ6mT`cC!S1b_h0YwGG(kcqMlgu=KH1PT+Th_M2?0tPvNJb-b& z4dt}TNKs%sUZoggL6F#X=rCS9nJaZ|@O67$Fy>VOXja$W=YeWt9zgcQ!8-NopDg^4 z#G@~tgtxd~-#dV!DPs!@#)Fn9`V8{$TNmV}DUb!l!__N`m1ZXB8Qg8$^B3Q@O@WTl z(Ge+`n-vA-;+e{X!37sFFxaFW{$3(Fn4nuCR=0frI(?2Sd~1L_v~dHk`3BcsM2{0c zh+_AH5i4HdT|o2V3IGgkGoCiav3GARQ!i)(U*VP{F~~&X)$!!s3$%N}q7498Q<^7u zm7DNAk>CU{*QBR^NZ$p^gown`)HAt>5nG>b_1G2h6tDSG(i~Y7G04G2w z5#UC*!amx%0ab?Yp-Zs$ z?VUOCe?UXNJ?MB$xFo+R=WtDH6d=X@eB-&3VZKtpzB{*g-p_VE?czPoH~gnR+-tkS zOTIHpB+Ev=H=q<3IQpdD~QX^x}K>^V^%Hq-~5d!gF^GhINFL{P5q z&0~zlJDvoHI5g1jGcWPaBi-{Z_Bv5{+!OOg43+>$Hjd5owK1Bwpfu&<{X|p(lnX}n z#&+fejNw5-8))x|fZH5{anSuUVBh z9d-I2fB%iB-~8z88`ftSw}a@Lg6wgI91KoDW2*JV{&HgV-R^O;U*o=|Hm5lU47Dhsy0z-)qqI$knqj&S@%;dE$*RsfAVmZw08PKzs&yxt=pA6oP znO(m9YTz9OOjErdyLP2K`Q&aH9b2r5BQzIy51`N!D(ccia6DmAmZZ;pJrLN#TWJC) z@zcGP@&~uC4};R|W0H~>QvBDI&J7IhGNsYVvL(v(;pzyGVcSud09OG>{4!?llD-}A zZ~6sr9R(Xu1K<`xVTL~$uT+3)6P|a|;sW*b3xm?3ozP=Y{O4|4r&*N&L-D%2Sc(dl zCH#;-yj{NZ$RW*$ZTF>h?r8of<6s6OBcqa>=y5JTdALrKgaA$vI?ww*-q0a`ao=VQ z4Q}olPzA&kc&i7NfV=+5EZgyfpNatZ!hz?8CMP8&D2+=z<)Dx==#eOgXQq<{yv7?8 zV30(Pc+VMFhnF5e#kz%&b^|E0n=nd{?_xSHQO-;DB2{z1m4hV7v>2{ zPmoaNMCAsZ<8A{4TQNVjry;aAx5I1OJEZ*Qc)bHYkigLR3zL%*)X`}nGmbHx0OYK{)F=rz6PKOP%s&1chG$VZKLn_{;OxB`a%G_uRJ=S{SVh! zK7URWC`~yS2{K2=6eOjOB>@{CnkBGuuLY$~JnzZn&$jdN)H>PFDS!3go4N@v4f<`2 zXY_e^Z4pTV)ePRxd;@M23!K-~H^v+Gk@W4F6-C1L;|dfq&?unzT7SZRu7~iQ#}a56 zisz)O=c{|nx`h#NAFvd`NJu0JRL6h`C+nl{!#zLNItJIE`m8?REq-7cLoOVpCX6SK z9OxCI7|!<$guSJyOZpLj=*^4g%D%HL>W$8MV~p_$3F_5rhO7(1ySqon{W zSecnChn;)?90%}iMgkHgK6kuP&Uf`mc6yo+K{64&hh3K?;0>J8yi+s2yTA>M54(@*oX_@Q+`;=aD4CxfGu@@1 zmcYs(63Ydp*-l6^3MTK|FL!KRq%d?Ame;$$ms)cJ7_NMzy5{% zrr%|r02PNB7%2}Rc*l}p5S1~Me#eZJ^gHi$c#+-v!AS#eW>@I5U0OYNpMRb2(KWDG zTlF7@hxb$jnh^db6QTW4YMmryc)91r(I*QdWK zE6kQtBuY##=ewiI1rYRy%5z^Rz(Lc)7FnSbgE~deTx7-wC((y)F~x9vhrqf3GX?7CQlryRZyC573zh= zVJLDc6=0gI9g-{!Zdv*+z{pq%p1pEfXW~t^5FO0vp&4*+b=}8o5LGJ#z6OXtaYC6mW`##WpBse@)tjHphixIyu&zV!=gO-yI&j%gQlE=m;E2#vq{Ij*UmI47d^=f36aJV z9qJ~+%JCisph1k|SAPWj`^5FDH3|cR8Q{vsvizu|BT%70zXXU*dtjWUZ?10b6?lJx zI8*~Jqnl#&4ME#6@Va6g{k=mNvqrUV_r{6@O{560LgJ_=57!FbtpMqaxp>1EwGQ`s z*Oszyv3Mx{*E=@oH_C*V{*MTOn%1K__rQ}LqXF}SqU>~STmQWKMA*bkNcP#LPUUtF z@=AyI7SB=oOMuPm*UZ;&{r%_8%lBV7r|p6M50E!38p$QvhX3Q`3N@r`D=&x|^TWr+ zu~L}rfkG}|%xN}Xvi4|2!AFe%A^|#^eC>%N@+Wt1);0je*_VECNWVYY1{5mhp$fec zK-8AS1!3cem0@5O&x@hg`r3D`o+ocsMT40*9WeIu9vT1mc3dGL?xhgg1;zw~xhLZ^ zUlzu8j7qMYJKrr|dGv@TW@Nm_V-q^MPu{p@oCiI1R;`IP6Lg*#lrY&XJD&9u_3crh zw4HOTzWt&A({A!JzA^e>B#TkODV*RTB{JPfOsD$SJ{j9q4eSsAm-*&3LwflF5dBv0mVRxa-vc*g3rzO8r#&Y`IgIZNQMv?y4OUjHCHHsI0aZM2 zaYCls6qx3nc;0u3Q9w7_VkuEpH~wpqZGJjJQXV{cxgO7sB1c`{x@n=@u_@SN*}V?$ z$bc_*J65F0IP(BIsB?f`5FU&Yl-Jnaui@o0nYEa23_}j@U{OK#loehgFHjxOqh7OMe!`t&_Wvk53&yWTu<1>5U?sbK5+BYiByPI@< zaSX%cxC2FmK4X1pzBF|7s%Iy_qYDP|eJf9QcB)c3Q9oC%C{j?G^5CkLO(I0V0BG{3 zV?<(twp+#uyvOFxNmm0cAdbq&sPwfB%kc21dY-wV&+MNp*$Q<}&VnEzLIX@l1*O>!k1|Q10sHt?0D1sdF*MW9F}5-$;3*0X(yzs!ZJ;dw_l^Mp3N#Y6 z^$!h8QBFp%ggA5k_W2f?){ud8=`U}@7z`VDoQr1iyn-ofVO)rL#mHv!&OnaV{>DK^M zWiBX9N#udLa``-&Y81fq?_&1Mi&X>my_i06v92>)?raUr*1(0=z&jF{9&WDCFnV_a z!@!M46^i=O;+3-3$^35BNT@yMESj4o>q_%gfwX-rnRqcl!TW%)UA!GptN_LEwnq60 z5_po~oh7`VNjBVY*RJxi9+O-PUQ&QB1SJg*8N4G+;6S}{S&=NC?4YB7A|@~>V%$G-8$WLv=Mm;jxnH<*gFJemR_fY&6o^8} zjzx{Yd4x(6>S=<9_-AijtI8Kj1|ENaawr6@_n#df?~;2*QRNSh7J%Eq;W1T?8$0`? zqPf%a8ct1Cus$kDRLBz-?|3G1CWMCm**%+s0a822w}cPTKxuYay|Xkv!&go&dv?Vi7C~ufzI*%GM~O%jk%ghz_uea%M{2sA!gNzu+aCvoeWDz>7Iv0i1C` zY0l%jAcQ?0U^?FGT&&{&`R{f-MBVFDG<8dDTel=9CTjmQ9WOP!ic zEPYI3f~+gam-k)~ZMeF6#XRM$#S59ffPRK?1Tb#>!q{L6eHkDkeFyEHd*_d5{qbv+ z+RoJo!}pPxpfpM(6r|qh$Gtik2=}dk&5U!1)h*uyWdLd;$Lcz?Kh_`4KaK}*$K$!W zrB}xbc;VsKr?1>lHaBqWaEchW`q~)}tj7bJb`LfUGR;`P7&_JOb^FpHY3dsA_-6-( zq%b>OqgjI5o|_m{6A})3*B^!CVvLBIQ5hLc&~cm_pGl+YZ{M&=b0jbhxiko|o|ks) z7q|<91Zo_(RiSnw3gKW)t0p(bo0x=~UHyX!V7uqS<^@g7L}R&oynh{|H9#I^V~nM} z=47M>CXt~Hz3SsrX8gu;ydh-J?dtQ!5BIt0|M8ZlT}+2Jl`)72CBFcITOUmGV|?ZQ z`Ia$O%%maHe{JTJM&rajr7F=mUV`RCiSl%Z`eek*SK^jKUs&OIr?L;a6juP}9zRqo z0Mi)u$UA}YfGDk4W50X8trI2V7v`x)rW)ct2~T_<}^G^*!oc5;ejr6%e3$z0Z= z9JnWZTBd4_dt@8_LzK29P*(%jiM3rq)9%w@FgdoNXJ z{v-^&7?j9ObE*2(%cz=WZ*{f?-ZnMxjs&Jh2YThdT(d&{`k@1=fG}_qCI>JJK#VsU z;g5ite)HVeh?puA9cw_BEXo$(&%{$uR-hp%3y?U!$f?{xKoJb7S}L_CF+;^!;>_3KtnYEUCdbt30$XlGK!I;KA#L@HTez={IQ} zqX0ti-a?5(KZ@Y3iC2Z6Uy#hsLH&*YSxA0@7ylCnYNTguM7^!>xPmGNE_YY(-2F2A z`Hm+x;n2`cA$X%ME6$bYj@Rpb+qnRHv{3+>e|!Fv7>7N5!R?zCNojs0^xJi$asg4k z7uq|Yj}s-YzoDLZ_iC?y{>P_|E68QnCdvd5=}cp%jzs{07;Mep2=I&-Duz`9it&0x z044dx5exu4KhZ9X4O107!__-{k`wVjwmxrAqdPqM6YdkC9vG8(XXV8?I_^(WsV?K?7>2X_M17|` zb+}gB6b3`QxOw}uQ^dT}-S)?Jix7wfKuz_YIsVBbwIPFa1pFsyO0{{rSEs{qslgx#v!IoxTEt$+Ji6 zqly3|$9o={7ChjYD^FK`@akCw^f;e!g~&to*4My05}2l1=jCNYfzEiyF!4koW&(s&`02@hv2<~cNbZu0cs`-x%8JC z&u_x7&oy-`t7R<(9ZYEqJre8>1p;j#PjBZtGcgyITy3G^8x_yP2o$RH9v(OKd6b)|U%zbABE0&i|HieTpapI~4d9u1Bn&M!h0%-%*9w$1^-M6=*Me7^z3s15T zHmx4Cco0rkzO(0altO>5xl1`o{R$<<3>4dm!ouidP)so};K7H&3lPG+0-&>fqVuQw z443LWJZt@+G}qp9ym89FGzlu&dIyx6h%)lBQsmyP-g|^gx~`*F=?MU^08Qz`)tlOR z*t3mYQBfxBdiHpO0@J4(yX3nsp4L8>?RZFkXxCDC<@9+qE~0!E=6DJyeJ7xgEAL)2 ze?pFBJTakL;2oA2;F#K#Iv zrw%5PSmDR+*yTSTT97`XezH=ds1y9h#ryjsS1;2jrr!Ptg%RgsIQ;tV;|j(ye=y@B z`)DJyaoVAuDkt9S1|7#dhqh654*#y{8FoMRIb6Qag2wqe+Q=~ScaC@+$4Ja@{9)OjRM694VK&0!;w?R@{0 z<;r)@_YcQza#Dl`n|=$btyr`WR>oA9(E0!{y7Oh8bHk!SdEkojsTLP?JW(XH0TVOQ z(Pz36rZ#dnPI3K*ubqoZ+DN|-ji9Sf;JENlyN^YUHAGHgj3jFDQj6DP&)ij+AK`^c zyCwQ(CIxP142jS&VqjdDpCz|$ni}m=y!R8!L1%U8rGH0u&y1%*Abe|g|12uzyh@G?KpBGLViQQO)RMa|>iKl^*O z24-vEt*e1|EHDj#hh_c$v-ciga$MJ)?-@+aIbkp}800}D62Sy8i8+ZBMM(~lZF!$% zOO_?s-nG5^Y_IIM`@ZL0uXi2Sw!99aY)KR)Do~=B6G#FC0V2oAIp+-EeZSLldu~@( z&vf_n00T^&@0+Ra3b$_EuDW&a|NPJY7{(*|#|%adV~a^R6B8!s2AGCjH=Cv!KT}ih zh#!>h7#LSpuCE*t>r5*6K_Z{ue9)H^%Q?|yz{W%@&cT>G-O{hXG@;+<%-M$i9s?vm z55RTXm}tTxWB|_~jS0E|(wK0sF3Aj;um(wz`C58X@GS&W*)R?4i4zm9_+>r-sI_-w zPGB0~fiQm9LzK6rJa^6s3eXD^J-~6=n)U$*#9$Md1cIzXKbwgFq2Q4iT7-ZH=sjau zDQJ66DEr-%1A8GHgYjsGk?|RAZ`goj{k~~+k&i)-rnF6e45#oA`)psCFWWtSO3A0OQ&M;m)(WWpyC5Du$!mQ#ndY0|JAr5TuC=9MgE{Nn6NO$m8vMWW@K)_RX>-H0{n~41g~X9KVu(#Z#)4!|} zT_(&5dR!)CKThu-fN89MB+!Y6_OqJ-+~U~y`&Z8>n2p9P#(T;{18|Ra4+b+P*>Bmh zWg3>wy8@7p=al;xfM9KLmWKVp*ykO|xp;rYT^3CuM>zYp57x=o_n#NoLcCLK<|^l< z-iBq!`DinkeU9!t&fS z&8aZ6m+un}^1pfcOh8V9ebpr)b1XQIx4(byP`w~c!+ipC9|;`pL&iizhmkb(+4t-H zeIbeMkcZeQ+jdtK$o_LPY%sR1F3bw}&CUKjgA?LR1ZuXi z-t@U<9ZQ~1A`09h8k$^me(htU{rvIWtK^x3wf_DbdA!GV&Oe!vbLeEAmuX*}zq<5G z^D)Ml!wyFb<<)olqOeTaXtg^XplmzcIuJH6jdQrBeL(x~(%kfUNe-+l$qtHI!m$|5 zm=njvzrI_eTGd<=ed=fSu633CG3^oQ61gJ&A~jg*5CrIB!jbxdtn>MMs@3VtyOKyM zw7`izxLOuUmdB8nu2#LS#(6cObb9WwV%nh|O+9$e;w)E(;N+B)WT%D985K?Wu{g#l znfd&YCP6NYMDw1xw*c=i&%74!THxAhf%%QX7WuXQGD*9Rw@72xu!cJmHf(@tCd6^e zd>{xR-vt?|ApuLAcUq90-BeZEPK}#>x7DMN~bXxS(l$KpSx>=JaeEm zroI2=t@HlFX?NJ5g;^=u?*0waAE-+#5(R1Uk$Eu6JDH&sJlO|$?*4}Mph zQRb|t^E=^OAJlhiMt7c-dtl*Qye#8Wmz0rU_6M{BB(xn?wmgK{&n!){9P^zeG8Hg3 z0oYEr45;oeGA*!PW6dVA&%2PH(F=C7jFg$^6SwUoXzfA!G&w9G{*gCdTQPYtfuR8_9v!(rcP!_SocfETGWYVoN8=iVoVvl zyO278Y3pXR>F4g=AUsp;eWU6`!KsHss-G z-vBg?ag8|6oJJeC8Ic`mr~cpj9dn`38lPI1o>k<>ptq3=i7 zs?KMfcNzD%{S0s&Z=dV%1Jite>c%Sh*shfUP7c+5Nj1Utr93YSvoFV)?L-R4- zRc_{#?wYvh-gnRTTiE5I_5Sy-p9y%DKDw(y!X-r5W$~GO3EV^G-eKJNu_#&UNPIDt zGKNF;g`-WfW)8h-v;RUYam@bJOQ!;eJtTDf)Xj4<@saW3Z?PBz44e2sy_jP<-!F@$ zz6((gNJoG9;^}~dh4Ymdx_M2J@V6L^e#(2yhHUA|2ojRp z$9kW=7VuhN;alLk2BsMd!q~yciVkasi8Vk0VZ1;HNG{9$c-z@K(YnQ`y-EywYr@PHyw0~p}VbNn)4T4EkomyG<8{y)b~hoLW2D4t${k5_Hh7W)By$r;3Gg6L}~+O0(8H8+?Ozn z<6x%!%3T{mCbVQ9CzEn{PH@s7yAICJ*4RB@laT>nC1H!mp#ZD{h?M4}$^BcFy8{-( z%AOh;qL{&$db+U|v z3kfdM65~};Csvd3P<4#OdM5TnTmZh=wr6#L{Wak8zr1xpWh%@{!1>SLv(Z&Hz-YnK zH!4edx|;<^AR^?GH&yxB!<2X1x)POYFlDp6d981f0dslMQWJoo2`GV#025XuhO$x< z6tp6_mJ>j>%jEMn->LB@s|8h-D+vi2}0UC;S#}@0QF|_o0N@7XTbaG~h2`?M`!QdYkRM>j1Z%M)Fzi1;dD;eW)wC zyMQZ1ZhU6n8twD#WAEHpD*U0Ji!4^}$fRCB?*k*P1(1jniGBsmbYn;X;LyUsiIF(- z$=$1h!hMm8_@6JFP^XtU_U$tr@_UbM3vgKC9JN7Ov+v;O46`04K5{MS5!3&mwQJbY zPwzjkjsuPxpS-NO1Gsl>Iipaa=;mB3LnZR+7#N8tRM?G?J1{aS^_@e0(8RQ{eSSV` zl!0?)48K2uyjn)GCYIygVf;lO^Yynb2niDG6x$?QY1LZZMsS4wle|xu9-z^+p zS&*R-Ur4!xW8B|P;M6|u^;50t*yh>7c&8u>hGU%M*S76inWuf8z0Q0NrFqWwRmJY( zG#vK}X>;|#(VY8dZ&{;`9N5eBdG}T<*GQ(`A)oVzJ|ogGoUO z#x(0-YA}$%0O7iaT4ZF%hm-(hFxiJahqfrdbSML)fO$jsuK}k4Jnr2Z7}93IfKcVc zdOd?$bdlda+u>)JEt5C}QyAdF4uW|@8x>`z$=w@EW@VdEKXZM~y22=7Vrh(lFkr{? zA2uqZ@S*MHveISZcgZ5C*poc~2`2qEC{5Wg-(ij6aa|}Y&Pq|{D7u~iBthH0wpHW> zYVtj?t5W#85~g^X%ZUJR7s3GBo$L?JbEu{(Brpww^FKViRTDV@ECS};vNkkW?5sE8 ze;D^t$KbeM;}nMlAfqwvSq26?cMNdCkts3G7gL5=4p?B5S;z8U->Fe;V6!eW6}{fg z0sV*Bw%C^RnIwp8?i=@I(GN8O` zQ)vJwO__jk06z^HwGbz(3Nuyuhi%)+^ZjA;h7r&0oIrF33HUV1ajbj1;Xg>Dr`wf0jnsVqjzVm*aeEEJ~a?|;g&9Swef$|WhynDBl zO7BR%j%|?vK_1S4ZzK-3b?J#rayraRIV^F85P1lzzI(`*ERa45=h?MpI0Rs#FSLF7 zwlx}g!n=s^08(D`L$)zSrqAPiRM|Dg`UOG!5a>hIx&TWx?hsFNQ_lWZJ%}nIep^pj$Ofxq7@sUO$nhBXQ#wO9o!Pw`!QlCwtVCVT3%K70u&$axWzSs1> z5YUM1dhu9`Mn!^Kn7PGbxJKsLfBNI&0ps(RPPE8Z9@xC7>bV%&%B#cm+X71ln1&?< z2uyM)UTT2BboLCSVEqB^y6Nn>{q%QMmnum0CqFu>3@u)GOoo_f!l?e(?v-wbX{a0^ z8~_B9Aw{$=`@&h7ATd$;&(9uKS%^EUOI#-m`!h6{V!}a6~v=WvEtS$hh zhOS}NEhQuYq!G{;2M&PpUsPy&aMd|+Q15ib(gZ=<#Det#Cs!Q!6LVo_m+t(?x5}jDu zcT;(;vPjTXH^8PjI5}yVJifCsfF=Efd#jb5+c`8QX^9E4xja{|b0w?P|9*s<%dTaWpLE@9Q&B>GfbG; zjzRh6yEW=C1vEug0`@ncB@PezUmG-~Y}z@LbLm2-Pj1W9fhG3{j!`?x0Y{3PjXJ<; z_Q8>WRFZ8Z+r&;Z2PR`;9SrN!Exl5jJsRWMZ>Y>uT~*^;z(HYbl(sk7M@?($q)pad7kaZlP22;r6$$$mIi_N||hDY1_w z(eR@?D`jYOQYB`>0XB$~TBv9#yKeSpz$2VdfUM!v(LVNSe@465jA4n)jZ}7fL zOCi@ucw-F8v-=yn{7_CNd!L2;%)5{0nK^+mp05>o8R|&Ec|7g^F`KW*GL&piECKWH z9;)-tO}HqMc>dsn2H}0_S!`Rt-2aT<8RM~J-U*2$l>OuD=QKLTY_I9?>9ZTzM6-S| zE_q**5ZfkqAY*Bw0vBVQ7h9d4e)~c|geQ(eq}D#O&lhRF80yRK>m-fvo%NAK_AiEO z@#^5UK(s9|GZK%sglp(Hz{-?)rGwuWDtYj-;)(scm%kL)N{fQe@SJ+8g$O0nJkV(*Lj;tC@ddyDph*4DxKB??3}7M| z2_(A=l30Fb?;3km$j{to;ofWHfWV22oDjzIYbKKF+XAW`;y01hXwvKkJNo&(CoG{oTd}B$ZuNm?aH8TpNkz;`Dv@ z^!^&D?HJHHI?oGW$2*ob0o1assxUJElx7=%{7;WJ3+#6c`Iz5+Xp7&e&A8UtFT&d*GdeH)5{7{~T`+&(|FZH099kLj3*?YSAr^6(AiL46$8j1z_X7Jz{$HkeE} zp?P+0Sy!ZyDsJZpwT@gOPN&NE4>d^N@VLMdHsd+pC}E})qoHg&DA~$8V7|tfaLPqX z*f8?vTR!h3oBeFctH=+GxH9X_k~W^Lk23qYEpeoN_h7x8YxhN0tS`xyCw8yYzQ=9< zNY|mG|F^H4R)>!{_P(uUL5UyDwrhQ92po}K2jSf~#cTr*4y8wK@3+vOc?bOCo9FdC zhXZ7I;-V%*JyPE-=_!fw>04L3l5Q5VUR%m@1o;Qsa8S+`Enx3E*Js`>PC$ucY8{d~ zbCu(553dD?W_jUgv(B4%k7GEM$E=7)yCkBEkZf`qQA}nm;oP>^zW@Aft7XT^JSDI( z<|GoCh!)SnYk?1^liS;SlkomC_KK_g*)0l=Dur5qm82Y;s1GaNdEZAovs5uUW9ltXR~kYuaU#`UCRDoqEC{d$*Iedm6D(u zr+V!4Y>BX5V|amikuC&Ng``9<7-12RywExEHFcP4Nd|FaTmoz?CjGD-0hX}c)0(sA z+Xtm*$Om`;Fgrm6Q;xkf;00v@+8S^8C-(R>y9wK6NJ7FHwi{y|tavBW2BsbftzZV) z1Nef>6a~GV$t8BJnj7RX=L}~F`;vr~x#=lE36|z!%K*x}b-F#^G-xBgeyVLwfG*&m zFuvTgxlH(jQH<_!sDwUY6cyGkFrUA3V1}v6y+d-lg|Y{>tvx0tKwm-S2N?tk9qT#kM5}nKz>bnPhebWq4TZ;t{gmLR;Ml!-|BN-j}006>p+-xq(Q+X#q zhS>~2-3F!EmuQ_(%xbjE-So`q3+&`l=A36+2g3rU&Hnb%)_KoP^IZphG>HI>kznc9 z@2w7CPn&JgE@4gYKi8oEYi2U?8U+CWxG6W(xC2kU2s6~?t?7_BcJywAUNP4_NkjzDPtT!gg$dyd7j)}T`F&!o&jJ0WC2ZVZE?yu zRM#aBZJUWapw8hqHT;!x`M_`SOSi9;iK*3k&e?Q=xwp73qOAV=^Tz^8gBb98k8TTV zj9?CQ_A9KX?FiX5;|9iaBsfe>ge~to5@4KM7ih(Y4U?h$G08`kKN)g@VivpAG}^PSp&J`7--=P4Y9aHulv z^{$EF$6eWQ~~!08;g zq(mRazdt$J95X;T+?w(%{>okJH6IOq0DaGG)6U;;%b0DyQ1h8k>t`#=+ctL#FrF;U zO;e`_z%=d5_<_D+&+6bPKAsUbvk53}+ZWq9=FYJZf?8}>w(#|F%Byik#?jmx%mEjJ z5hd#g%z;9741_ov`mzp7d4zk5}_dhhLB=h9re$_oi^1x2K@=tG^ z*GZ{49v0WH-@8$auBBNY&eHj^XW{^>aZ5#>GIB_e1JLkKZ=MgxjLSK{aPP*jCmPNP zE|d5#Kd?!jeZNkgIZz*z1cr&9vn|c_u-*ox&3Xo{gg-a;j_5U@KNgG0DdiK|{bw(n z2uO&-J;yo2H9&YbGg;={w#|2)TRY9SOt<4n+=9MhOYg8g4*+1!1Hxhj001BWNklza^L!5;m;lj{rBgO2WS>k?tB5F zaO#F0qA`G1Krrg{=RZ9u_iSG7cewHF0>WbSRXFzEJKw2^+?ei3ZS1=Xcn|fPuzn zClUanl?-re3}ElAGwm^Rgdu13KYwyU*{tRq{4W3Dk*xx#9EttH-nBvmKy&Y~0+BaV zjvxJ$M>w%kYO0)$bzjY%94+U(Ek72-5WG? zePCoLz80_Y^_`U$R=NhZz$ zb{XM*e(I(wZELj3sTYzpuh4^w~|ug==s+);Pz>mSD*ew{sH}#`rK)KhH6(eeQiE z?+g!^4s2Dx`o$Q^e`xw$!ZT^!RVu|bO;~Bg6XY<+B;A8!+INQ{4>=Br_^X0?D{><~ zE}R2+XEKJR-?D9U*}|O9KF3o1OafbHD|h7cvA21judfy$ z(vdN&ZAungB%n0)+P<%9d6NA1`9)ru&<{N)EW>j0ZU zeNlFb+&Y~&jYN2E=I(2C+cR*TNwUV|l5@d!VuJN+AF1{OC4d}+m=PZ7wB=Fg`og;U zi=UpsW28V>_@tcN9fHB*N=8^`O?bzA2!8UbfX@{N7AA{xgexLR! zAW$gJZK#RlyOPGYu&Hv2dJ zjr%vLGzDSpgpU&;5ejULWxed44yT6!zG^#$G_fns5}+%Z?Z)`+$rO$J2&^a)XGLPj z=gg)HPBOaJPPM57iYb?YBaS?@S)CT@nP5Nq+L1AIrDv$ofH%u?Q&sa3eb4%yL6vn{ zRg|R+DvmqR)F%gPX8K6_P9~gg77i#4I8ImxziB9NEm9B+rjVPVV;+sU0sm<40=_y8 z&@fiUJ>;AyQZ|5iq`))`s^2Kt7*sXxQ5e}wR1>G2FUIMG)C4j;wmD@v>}~u`pSGk* zG?C`OsD{=u@pYQd!8NJc;KX&Z8(2uBV@q%P59h8IV#^h)F5Sf87k>vKxy{p znYd$Psk}C=G0iq)Dki2xCNIuRc1SRKwspWYvcs-ZEI#uLAYWnRaO^Ti#(dXIo(}q8 z>dG~c%$NucV?@x{Nm9*vmLviEu%TNu>CJkcb$)ZhmNx78eQ_q_Wu$1rP{u3a<`yw! zguBGCi>C8J7~`J5e))7jlmLk_X*(dpy1Wykw>(sUtJgeV~=`{$2GJ81eUGf$tCAC%6Hf zNmXAz-6lUg+~7}e%lr11?pPZT#cVmxkW7-19ihX_mdB6M_k*SL*$~MDj;+RvdeF2j!k}1}A z4$B91UCMGsYmo`*|9RuAvil-Q41|U<0FrI6wn^?7P6D!~?h)0BrbBd_Y@2dnGBLq8 z+t#m1K3EU%^wFIa0RT*>bM{O#mH^Aqhx8@PidWrHq!#8fj(g*DyDZO1)rpHqIE9SD zXZFT^G6;CwKh1uN6gcc24AcQN*H+UxBtJOZsCwC~OG;cOPwrmnn&f~ma@v>VSGJM3 z>Pi~YGzTtp`cFw1IZQJfu<>tP1Ru_|4a(`(UVR49WpyTySXq!R{9P@B$q8HcN*fL_ zWC;K#3@A+<661X_?f|stV#CHX{Rv<-JxkBXgnVQF1$p&ki#qyZ1r)$ug)IwcM*l)m zNc7j=IMo^iOe3Ly1WzaUsQ4;Cc z022V7?LaZd#nKTLN)M9XDOUE$n^$SrIPVvX`fP&AcEr?!i*SMIP-C-DIXAd(NY3okYQ-rO${-DV`#`N93}QdVj6XioBrwXI zwz)hr5P88_C$ zI)33;i=bH_sU2(Vk=IYPYocTNIC3Fesqq!(bITVI=`}3m{{QmU1&16}gGO$K+h;6f z-6AcU$U@_QHOJy;wjH%*Jxj*^Y-cV0~&C0IR)?#8$QavNT!Q_?1E0IiM zntDf6x`HOUESG0Ix^beR-yl2Rdbd`+VNdO;^6OKQdT zw^)=6;v$jZKu_4sPBi!WL20(5fd$y%1W(yU7~1FWsg^fRwQ15Q2En1UY2D5hi**1a zFw>momQ=#I&(-)2@XA2kwll2*VJB#_kGqne0cGn+u6J3aEH7Qd%VT|hCTsLBu$3?d z^I54$sx!)Wz@bY+6ABQ|Xe|32UY9P3_G_?i+WTAdSl9@$Fqi>**9WrJ@7lCn;^UXe zspfu}ym(Py1{-G`G8l%X#5HVOo-0q?v{GeZ2>qt6Xi5UEljzI|R=d?1K%74liQ%@- zT2E`j?$bo1*}*XsnNHQn740aQc@F_sWeoiWtjcZ*QLDY zkId*vBY8l-j%12$TUG@3@%HveK8JEF6lNx?q{eLQa~*QLp+}FwdBHo55K)o};#iEP8Ku*|fNUs1y z;DCvAJZH`w$&x6sKmGCXfI|+|cFR{Dh@CpvI zb^NODCnb4gY9iWD`?vLv27uCLzlNS+^D$tFKJ4YoadNb=SH>o$q<6S0$N|Y1?Sl?(R z7Y?^wc-BdRYYe9b#0lQ7;U?NO$6k+1qQ{&$61dKH&v%CH%vwzCf&-TRo?JV;3z2wy zXpV6x?}2C|%beH#&iRf2P@3(KrM6Y%yS}HK$9vzeP76S0^UP}juLTxw3xpnlF5WsX zXw4Y#k|d7-eyD-_Y!$fBKEq66LfAVzsh|jKG$vWg;(hv^0G-PuPJZsT)&Al-`iFH8 zKRP)j=x+jyz&iN;2Mu8Z)1g8>5YI2p&QvcU;IC~rR2FE$B$zC=y>qTZ!$|DOAd_QQ zzP6Eh`%q1nD=-Zk5YMFZoJ_c3J=>tPDHAr(T)?#L6-UR8K`EAe>5}S4{^)R{hUo5C znQw}nbxF7=za10)(wuZDpJr1cw_$_Q6v8^%CR}|#2E}%wIm&=NMiMHg5vmQ_(~YfO zo|k@A7#ZI>P%H0W=u)tfi6#IhOc-NahplsJ)?e)<3CNsAG{a*%E9PXv;^62S8dWFB z#5NJWD3W>uq`NVnzVY@2e}X&8W8y`9Z>Y!>$fW?$XzCh(3#k&M zivR&b8Q>#;J&Z*3-751kB{^}KvTcz9`N-xnl`{!-JnI{}hjcT}91>|_^yh@yQx|~g zCw4^-Vp1M}@?1$%^~7QPLt+I@efnuwwyTS?6&NC6BLGTzQiA3SVDc9$BV?Pt(Pl^@ zQ}RJwk9_OBI&CL3ls|FPD#@8n$T;6~BMCrMkk2m4d*?eOAudinar4|PC!Wn89&YqW zPT)wqcuBg4$CUvO^W7~0UnKP<&%=NE=ncwxMJJQ*yn7GTc1nkpV}Sdc_aDG5jKjIC zC@W3DXpxydn`4~X>2sA6$#xuDIIf)5&>biz#L0Xc~>J#2FsYS7vqhSErH`G%EobVrln8TFVBhg zQ~}2oYfBHbs;q!4A1{HRNSHRZ%`r9+ka+34@;Dbg@40yeOrzA%HgU7cE<9OtG z3Nup`G<0j~gk^z!lG$gr`@`(xT-LM)%YCsAvUAr<+d8i~lJCsZ+ziM0wQA%;;qPi# zsQf<6k?cA1)u@|yPOk;L7Vui&T5SOjn0EQ)nDjXZ*e<)dt|HNvF(y(Bur6J`q#%g_ zQ_T`d58H@l1^7j8))--_O+6#>(utO^!aHtUT@WzQryKwW6ShQxC*MSBTwqGWfCD4} zSa!3np)px2GohMz-QBX=w)D9(U?}s8_imIob@Jvr=hpIkSyhzbOTgrEQYNW$8KL~f z2+uZVCQ5ui-!Y&QMYAo`5}^5SUpph0XNr;gx2=!|x2;f+*zGtQC~(_uuR|~Ow;$f3 zGBL=sz}y1V{<~LB+q?NcM>8n*9~|Qkgsi`H+Lu6#q5%@RhsGo&H%&jYt~@_IWcXZ+ zs}oUiFpu2~cKvKDRcEqJ*gpU6&h^sVGpy`e*tAUO=VLfRPBr)G1Rsr47>*bTSeuG_jz${(-Sc zdG3RT0C^&_Z6TM02-iB2nQ%TP#82$5luOzd2eu~t!-cj11*Yj|kqBTS!F6vb&(-J{ zz(-@Py`u^g@*8pNu|`pc8vW>OMtM^gFUmzrqJSUP0OXXr*o=P4)^br)ee{@{ZlM?klGWzuZuk_8&9;}mJx_$ogOn{v;p!7`b&b9{wi8EVgKH*5k zoZ8~Zvn0eP1W2^Z92MCo?ho!m+jdkH%Hg^$G08J=v@nL?FFSQ^wmt7!BCbp#NXo`x zOxW>4nHl3yKJrrl#k|&Wkhc*^YZ<`wrOOw!|1t0Hgm@fEGi+&gU*nLBAoAeHN1LR+ zYf#WWM|0F9*5lDnO!9%E6cZwmux!v=ahT!yvY2__DIS`OUc>_$|=8t;kQ8jf*G?IZ_TzrasrfGk`8B;4myV zKxPAIxFzd^757KqJ)+kN$bvKk_u5TsisiO-MXF(I+9#Z4L)WkZ8a$uIh#>gI+t>QR zC$pXOIZ&sYQWBRb=)7-3i99)-u!mPVdFT6$W=*GE&b9?Q zaHDN2fNnSAJsXahy#UPkGt5o1eq-6(U|TrnaqfeIw0m$=<*K+J0c?$94Q6RLhMwbV`qW}^H+(mQEHkg=qRF}%P57eoaD)VL7Suhn5 z>__W39BpG*2$6Ed{Fski@p=fx{rSrB>b#s<`+xj9uaACH=GW*+CP6s#y(1s;E86dlNWClbr z#{fnHLIGkMkcE5m;Dt^t{|}zHL60^2&$WE#e7lCY+WR3Ba8GZUI-Z;oFEBG^1Epc< zk!*|PLNFo2O{VwSDPJE&J&=XK*Mh8q0j7Dj+WMO9Y6^*qPFbPqgvlG$Ci?a^_!@BF z0(_zWz`5mQAFnR*0fbDrUN{<9f2u*^M>L<&?{`|WBl5%6^|klKSvZ@a%}@Q#w+|}d z3Y*J-i5$1JB9Ik3TUkqen+@*Ir!8oIfJ@#>wyi33XSmw?@EMk>G4f8pz(z)jh@+d= z6uC;xnEmIo1h_}y0~r94+o7{g0!`k-^u^&Cz(lO1U&1-(&+lEMQAjcj_1q7hfb&CC<8r^PPKDa~5}jM0A8RubHc^I)2`^Lf`qu2}$z9Xh80{&ULMB|M}Zi z%a-ySjS^!lPg3;78bdCCK#KNQ^e6((E)Do|p^>f2Piak#gUYfGFK1 zB4O&%C1oi(6XyXoj!jO+Yh5}!-0cjNJ>Jyg3>UTu16zs1z<`6#@zIw9gP0y{(rkeu^M z7;;=jlIu#GFPwAO0Hkw&#w=BZnE?q9*@tK9XKr4ly7fe#Fp-Bv8;K!H;AUgf>vvZb z1W1>#?~Su)C%ct3*Sb(iu~!ykXmY+2jlG)K5!nPI)ihU`*V=Y|OC*b&W(A7qCP)>w zgbPV?0j_b*hht8*d^pEMv(Go)sgdIr0i|oOLmeoe-n%B6;)0GTFG$z-6YpvIN}SHO zt}9mZO3=CQSmUN9APpJNP57kR; z$DjfgFj+`;XhQkjE~71k9%y_2h-xLf9RtGxHVX`qcg}Sx`xdqYNv2>lhGJmAsQ&bA zYvup@;ZcE+!vG(^$J7t~<+&jdw;Ij84sA=LZ~2!mo|ZF~1~xjwu$;OF$NdSHP}~I2 zv$Je-7IgG%P?~jU=mF-#jB+!#N#4LnFmT4PZtF>w$RA9q)WigBH@EY+t=qM#Kunf- zvwc@pfwnu}VRZ1f6z$IHQt2KXlY_NgGCpxhigQxrzRhLYHg1V4VZOsAUMx}t$|$oE z-I3JJ%}lZ_!thq)r3Emm(TeAKoDvo!9s^MO>9HntT#(G||GH~^K=A;e&mF3lx~?Gw zw>PcG(F9)sJ57JV@dJwtpaw@4VZSR2Gc=0AZGSYh-LkB|cfM2R0;U0JN$l5W>7#Ov z^pwN^P>=`?{%k`|hfL_UZ7t7>NP>pC)24=1XO|Vp=ce9ao#2DCrzR&TSZMbp#_>hJ z04QwqfeW)zTu%c~N#qKe`2E9^dM^x)O$9jA(4Pme57(%VVWEe6%(b{54A!r`epX2I zYwTk;uAGxe4BP*IzkFJHhkZ^FBzCwj3Nn*5VudndW$Q|__VFpu3w&?5%olc_X~ii zn>Ow893Kl5hg4B^iZI^8aIo%MALu-H)-}@aIP-B<9JtWwKNL_JCq4Jo?m3);{)4XJ z57nhPLbQ)_m*A@gg?8!PTK05@6|~2bO<-oXXH6xG|$F}ND84dJlCWa@a4~cdeSdbK@+06 z_*)NdnKg{=hRQs-ZGB0QTn1oDB!Eb?KN-wE81LtUvu`x4O*9q>F-AKUw1ck{001BWNkl1akCRr<)kG_dw<~lfc(p10j1sA*G}{TV+jp&By*S~qalx8wn-35LM@~z z2rd8Sd$k&ZF4GCD0CtTwtpQg7=8iV>$kvJ(_TRS-)JkiwPcnk`BMA{ns+_idd5-wB zc17>Ky@B|)?#}VBZ#f_L>rjZ~jzb|dPMzn3sYL{YZD?lCWk_TI3Xxi{1a5%R822_nKFRC8^57O(kvo&K!JH=+%lYEzyyikrWVp zqz<)eQ9vA>e#T=&u*K4T?$*_sa8M=7k`rZhh{T7nJm+F63sMJBGiTcRwZB|joUQCA zyN(w_B>S)vh1>gItDk=}5HRV(W{YGOx4_?;yZwNC}>KEihLLEGb|b zi3$Ts+YNKBeZc-2^m%(`WXr(M~c{!oWy7C77A7 zo@$l$z8OHfI44yTAjbOIjb3%E%Ux{cp$CoQ0hyNdB{{D5VW>LWpfvlyM*Yq-Fldgs zV?!Xj)~sJFOTri{3R;C{lSvj99~Y-d694YiGxB?nZc{B%WIb%FD$0~2b=~p`_hbK< z%3S~qlgQMT)-f8?91Ek`fYNqZocFgM-XbTP`ebx+N}m}(;@Rx2v+eTz4;u94&VEP% zA)&)L_+4~Q`fF|PaDj-+HTzL)2wDuc|F z1%sBp20c*&`kQU`vZ2cs-I&=z^n(+ap4mF{iR-^*ZIMvU)B7(7`l2S8&8w%{u2_^2|HBPnOu(O8bZcl*ZZ{e`&7Qc>BO)#pZiBA{mFv= z=(~sImNiB4nSEPXH;gYOW1ukJab1xu`_0YD+lGy#| z!zMo{%{g&i(Qlk+F!nOTFzJr1z}`<>4DA4kcQ0{ zY0?+MOB;JfBtA}*#S@8!Bg`WJY4n_7y-lKB(I4xJ$S(q2I`7I1R7#JzL834|XSj z$D3!`0vO~RV>2;BDcJMOkf?{ha142rZ@pU^u?U{J!8HwoHIotc0mQ?yj{1L^NyJ`tGwmbQ5FOh?#l00VG%I?)~F868t2 zpF(-Z8hV4+^@hzpF?G=oO0(}WiPL-nYf7?Iht>Qx-tsE90E3e-jReA&lOf$R=q}SW zTd_4A!32V_iA&n&5w+no01}(o&Gw^>Jp!{ol%$xm4Db8=TL8x?{rGp^uh)DIlt?ZB zB32fgkssz;@74IFN6PZjt{5R?A=@I{u|}vm5D{TOY4#Z$oswq{)qB8ns0Let>&ZPr z*uJsn4mD`>A(BU)c`e|z!2DW(#Me%1?OR-lY^mRxqO732ROR{Up5DLn$Me32Y=I>O zOv7&E0cq?W4rq{dvoN1|(f-q0=lyIC=46D&ttko3qWJcKT6yzyyL1hW3G8F!H}W!) zRnwXi7iOjgoF6?|8c)7}uSh zmMG~L6Qpx!OhIV?2a*{YU;%I%gS|G08|ylmDM zmeVZ*0$9T21bK!Qfc|B1vZ^Rkulak2>g8ZfmyAzcl#=W;xo^vIiNpbLC9erE)lHB1 z-SeGM1Jl(1TGZd>UWQeil{yym6U;;i1a0!KfQ5vPs8dHBfZKMy;JAVf+~UxvU+W^} z;V<0fLu;Q;3dC%JO^jwT&jtx4(vss$^2lk(Z$fJy#dIpO0f<%75p*Iyq1DO`Hf(HfSRwxpFNSzpY!ueeZDaGZN~0c2yP1 zk3VcuW-M(wc@gO!(PW0S1K=cmrPH=o=BcC>`y1olHejY(l#|)dTVDAVz)9t1Zp_k9 z(qnd#W4E&0)+H=+4+9O=ue9_Aj`K{J&C}9K=KGagA}2zUPa@9B@-xCt4tgc)xDe+e zX9s_bLwK$;BGBkxi28&ob4_uk5H&^rg##Hfgl8i4-DH#rwEp1am6{*T$iWhY^s(KM<5iqnNjhghv-p_pg+G}U!g=5Xi zIA$HpVkAUhjR0r?f;$KWAWkkz zaw4o!k^l14c9k{x&cQk#i#jphuX|0Ht}(Pp(I31P%^5QD%E?KYm^v#3)4Fy_+f&=4^pWGS1n)UB+l|O{i9Ud5UzxN`kF5&mDo@`N;QEF0xY$?x?9V-hW zDdLLn3&YNtz>i9VN{Z*KAR&rhzJIeEs_)VSU_9G5ttkr9F#XicYqY;1~zh)fEMSL7Q2 zjCpn@r)B_11!}qm!%|FCg$(FiY zO!f5YvUpoy$pO>l`Dp=(`sVs+TKj@oH$6iWnrI6U&NhJLLv`JLP?~KF+!)lA=BCQ8 z-CgY}6W|0!Oqtnff#oxw=@=Z9r1*FRx!LbjbFUE25g)%yL2tlefEAMX!T11RW|9v8 z6p6ulrrwDTk`g+$Or{!ohNZS6aFV#mMRS(AoNVsXBt-zbXgM!-LxRYe)XpRrd@w#s)7)aiY)A}>|$+qyy%8h-J<`DG(UkG>!)MVY6Z z%P<6;n(eviu^XU-VvgH7^X9spOA_9_U)!atnV7tIyhZh~@7`1@zqoImynLcn+WJQX z#uXqtpfZ0*2a! z_RD!pOkNb60e1V62oq@l27wpa2i5sUB1)dComF`OptLFXbjwUwt69hLcw?`i$K26B zCL{~Z&q&c|lB+quq}|bscRDq1UQ=jSea_EBx8Rs%aGDgqOeL5&2w9J!>{PjD%W~H; z_zhsd|Nb){1Z{_Xz7*_-M;i6s;#y(nKYYXDlR7ZJ14(!Iv!6S&&AI&Iy&L82bM5+k zlVEgPdA>iNO7I^Yl2b*m$W@gWg3M)Hm0Y*3oxNgVCeT(|4$efC;lF|+^`Qg?0y!r@W`96BI*(8x=Bx+Qwe`6Ocf4u!j8-gR z^@O!k7r>#cv?QJQH}(umT4KB(tSZhF z%ucBa$~)T78*^Z~wsT0nvH!d#ALJa!<9uRIl~kAIM0^R^sfqI24{Z^`x_PDwGLiOh zA6$o1I1FK1&lEk>=H5~Hjd_C7=G?!yZ>{|FXtT8R4NFR5f|TT>Naui0zt!w>%i735 zM01?mlCXSOzqY|noU4s^UJNJ=aEs&!23Y))51W0A@!gf$=ln;`{A~wkl(rWRh>5 zZWF@hh;p%wet_+N_43I8*7wnd9tB=;=Gk^J__ZNlfBS-<{||#0_Bn=rIq#WYy?aAc z_p*J?`FwurrYa4)ZR{OZr`y`%?5Lb+^|r9;a|REy(UoZ7#NYXB%Db30z-c;dtMVQ_GrWyN&1L3L-y*qL-pD( zFn;9knfL1See{KUs27hoRDESooL$x}PVfW`?(Q`17TgK$ z?(XgoJV4_X+}+)RySuvwcetII`R1Kl)jzt5sxGQ~Kj-Yd_L3JtBC8CaBS@IRLNb8K zFmk`Fvd8GRH3)ON1Tz>C#5WHUp+a8wzKCzyMpc$-wa_3$j`nzn0Knqgio+76cHP?H zfvZ@eVN2@}4$*YS%CyGEXqw_R-qD9cBd30{Rciu46rE{)l3WfH6;O_LA(LAwn`R81pn|MNSBF3nhE0 za?x@`n@*awTQ-*+*Rob*WcrSLl=r&1dMKJ%P^Q~4N)?cif$4y6mTF+v8?#flh`4pv45%_S3 zOvA=7;`j;r8OA8Kw2?>gKh2oo9?u}<8xez{3YTL$S0(Vjd=WB~UUi&F_#Uyd6*2*0 zllgR4fhXxZtTayajFAnK{9@he054e1p1gf}nRFRD)oq-_KR-ODy*bRBvUE{Q+T`lF zOa*?XBE8ZLKW^X(CjGai#6wU@$)`%1v7axEtwH3$83N56+F5)GMXklb zjF8pVWn&2--#NZtkm&Y+Zq7WZPfGxo?Sku8krOaDoKTx(hhSxgF^J~S-oUq602 zgV`+wpBb%c9h#zMq~0;Xz@>Niox->Hm1*EU#k=-{E!aTj<6vi_`nS;x+&TI`BE^% z$AQ10Vo;63PTNfEy`?(ct75UO7Z^jyFSqL|d``a7jLBuPK`>-Ilkl0Y9z6oNpX0QR zUYYQtOovmnF>mv*aa%Puoxgt?{>T!TL9ZT-E<*@%zJqlA)A=t}O0Fe%uGSE?;TTy@ zr>fIQ%IS}OmN`6jOcO7Y|L*9uKj^E;K&2f517s%}J}RKbCdigtTbu)+UoVDAnk_jK zyQOW|Zzi!UYi3cJD+5Z%mJL*);PTf-lCpbYwMICu{Z1*kH!D)ph=OPwT#V{Gd zpC^y!!LMLiyfWpnHYtKJil-or%DkxKT1?ayL!-IhYQ(sXeq$SzdX*wOn<`Kqp-k-$N5x+VzK)UP3Wi!aG2RzZKH%>&swZdibBabF$;%4IIx7amI*H-3*c_fKEMlv>q7Hplil|+{wnLV`J-N*@)LDy*h&uL9l70DCWZ24N&mstR)}`m<;B!%}K7mv?|j4204%v{&|^|-Gd7ZScqJe zX`Qgj0fZq}L~8Oimuh;LjKY?!M>oPktwSBL`vEqDQj=wLgsX~W_LN7#N%+uRnFyG? zleK~m5%`ZIs1>*X^N`}*=Ya1q9qm>BX3_nNGvrRsq=i5Sr@xX^rQZvU_EoH!@2NG1 z#jeq{^<>re>fYsgJqYRgA_O@TU6GhVJh&H%-P zIDpcM3$M%hgdm-a(d3W!B&&$EJF-*>?g}?|5{Z>PcO!YQYQdLI@AhvCm}a0bGwmlr z0e)i2G%uCHkZXNkEVBZt#}sc24dV%`Hk?g^_)x=yfl~hL))pzH_$j_iwZWc~UOEr! z{U$@z>N9tgaPy1*UG-w7H3Tp|XuuCwcq4~vZ_#z`&jZNeC*q9f7-p=W)cQHeA zbL8`L4KMgF>Pu~#0^h5+PuE1ue@%pKn!5`V^qgDC15!3 z?C-X{VlR%~%)Y@1_Emk`^CBq$$)*<^xXUB!ln>829eLC^Pk)BgtR^@5+!?3u$&c1> zjS5PA`pkvnwH1mn;%Rqu{cW1~X@Jls1kFN2yHv`3-{g_?O4>MzMmI~9`%rB`x4JR zF+k^>#WVMC*{R1^m37NZI?gYDQ4;TS3Q5kbPbnuOr`%YSAOxWu-<{7ZAgk|g=pSLb zu*OsBIn$6C1fRb>1y&!*7n(#fOu>=7-->@y5ov(X-IxiYx$7MM?EXy7OhvbR(c=&> zZHibD{b1D!yn+XdP8c`}N%N6ex_<=n=o{T5G?2CSunR_jmia=q#vl`C5S*7-1=0qg zfu+5Dr}Vmx_dYpLF`aj*(^knSd2R?vz2aYgJBA7XYa{G(*cR4%TCbapH#VnxbUVYx zS%fmRb;R9f=k-aEl*7GXHtnLNe%hUT;BfilZ!wEnqY*j-dr5%o8WGA(`UQH3M_UpB z`i#H2LHnowc4k-{pt~g1m;6+c@Wj-0F1=%2!e;rj_brNd-j1(XBX58~w>@7{-eZ%n z^jHZNcY><^`-|AXJaG&`mrYaoT{PvH%ZAzC`=*&+bIYhrb;NC>>LzHmEP`b}4xd?O z&C!Q7!42boju6o`2x*8En6bB=j`Y1uC;S-zFN1rBopvJQsgx6mZFQN4ph#0}4x}d77Xt+^A-P{=clZu+_rSlH{iOUhulC!3mW_G8(YK!hNP^F;#CmaO4Pf|xg$K6Y6x+GhTUYWG1%fQ2n8p< z(@UWNls5s!1fHqOvrH3vS&9f&AgeK$=DDX_%hYXvQ2p_LTmWaSk78c0YTb#Zhf}t` zckzi{TP)>Rb;iqb1{IILWEBXWl&7OlLm-)llj3_kysnzGlc1E}F`=^A+ym(tqcNk=rydcpj zDqlaB#f_YxxgOdRLZ@Hx^x;;dUg3m$o{wH!GV!3e=lQw|9eU*0-MxDFL+{0W04kou z&F@}`jmU(tb}vK4LbcGpM+Kb>!k>*7c@vJKuL&*SZ_OZ#>c4$3=-&CjR7dS*9sdpz zmL@_Y3{Itg$TPdaks$yUIJl1&smS9dLhivmJU=|1-Ugda8JH5Eyg{Omlt*;a0vL1Y zbjP(M<7~G;uCv7r4w6l;t{%TnQFE90mOnw>Rh8S1VBHSp+Kk=ZvXo<3K?#(|txRNu z%es~Ryx}Ead6L$!vujJMhbus&#!{-$a(MMhhLL!J$ftqJpXpnWcvCGp1m(E2NJ zbg1;U@fG0hU}FsVY!5Mype!w;ls6_38-@P0$hpJDyZ-BuVNCGv%ws(DI+6$KocJ13 zGaMu!htSeUZN;Bllv~%)ZWFW#p=PyybAvNs$I(uRUYEi}R)A)57w2{6h{R3rB2^O+ z?=amY(z|u|KoGZ>O%aIsM4a93kw}zyi^Z5V6ExYhtKU|xk2o?W%C~1iHhOMw#4ql7 zueE>)Yfk6Hdzuun-c}$3-eFLtM|Mq)^K!^BIr`F|^DZLk_zc2i-nbB__W3sEXzem= z6Ji3G-+bcoAD1ple;7NwQZ|S3|B-9SVbRN`Z(C_o+kfdI*$gb~V9wX~y*8-Z;`I-} zEy<*uHvIU}c(XEN?MH|v$pzNl^>D-vT8tLeig8X`OQzZjCWD;vo=(-|p6+V61h_9b zDj9%Ha!s^1DG%o{i5?)|HkRS~x(dkawHz3~4;dDvjNhW#+qb`}D7t#WDF(Qz)0Ous zqqn(6NG*y&maxLQQ9wI;a^)eDtDB~ZtFAW8oebU7zXkL;9kY5cv4(n@w*P0Nh%1~d`js;OW?QBrZ9sE7U{zl>^8@V_qLpk7!sNV^`eqSZZ?AHYc)0B z(>&J-7=D7+xEM}0APqHblwHX;gHih4FU-F~1%gA1KY>GcMx((Qm7N>a`L()U8z5s2%I1`MEi{?zH-ms{x}>}^!GtC)wni9;k#K& z=d+CLCQH)wW3s3+O2OEWkL)lC$iK`0=*x|@`Ub4H5#0=G*D}O?u>W}DxDOt z$3)$~Hd>Ggu*HF~Q;pZG_1R^fPm%e+GCGfRwfhLAf9^Q7aqkXUD1-Z5oyvKw^QNn?F6$?( zl&RHb(oN9rSPO0_veKtd@s>~}R|q7j$Ouu5|sZ0ulbnqA{%t{s&hm^uV5$NS(L z^m-!B-i%Lh*V@LOG=d-wd0Owrhc`BrlpROXXT&dwUZ-ZM1On3oWRC}mqaq^{s@A$R zkmsDGE{B4RDJVx8N7{?kEf3na8Dc?R)gqHWPtdDCP!SU5(idB7HA{hf-3%B&6j@OH zjj)D=Tv|R^es>x!ha3WFlDOg#o0Eg2_ave(ejYK&|@>9mQ zso&>CBKbt5-!4|ZlYHFUR!~gl;rXAKYn11l2mztwfC|eykX9I_z@-Yx?Yy)gE|fBY z$ZuWyJVJl`QhvPBt4K#tW(Rt)N;KQdSbd9H8vM)_)Zw}U8^0p=bbMUnXbZ%JqQsR&zmBOtCUk+mO0cfTK)9@JT6N9u^?%Zc|J*x^ zItWm?Y0T~Iot^woBWj%_lyVtAXJ(Y1>I7Z@8Z9jD!kTJolzPuI$@vZQ$-gh`-Vf}I zUtF{C`nUVS_MNg{G8QZwSq#-}7IRaq{ayq!Gi=Y%QW(l2Yxiq(Fe*LAE1`ee?~|_5 z?!}#xZ1O9V;Z(RzgUsO4iU)PE&JWr8Q+u;*Fgz~8Q)zB|PP^`GE80QaJCQ*yUE~GCnIyo0?b?vg}kn|u{kKY_w!y`(n>VXD$CD$fE7r8-EM4GWX4Y zs|fxB#cX+6fj=xC@K!lmTg$xP%71((@OyQ53iEq}1=oK+QR^x_b#ycp)&j56_<;5M z0}C%!x0(I%lzr;^-QeT>?fUt&N?J-vYKSF6C9~aKW2f-WFLO2b4VlWPdnWV^WWpjw zS-6MrpenAl*B7~NW{&GP!&^z~P(*RXk?!C28#Fm3so z?YxR0s-XSf^fus~t>@g3%AgSUeL;~OpxfOuu|voW^IUyTWCj&2AH=Y^uGq^HbEI5* z4sP3g5HSA}G=taYvQ#c3*fZE?T;Y2NV!`+{uKQWC2FJBLftF;Ciy=1#m(PahWnk{t zvkB1lNN};Y=%~|}Bbm5C8b?K1&#VC^u9l9XlkS{%EvgU1S6g~r-rR!lFsTKUt}0?K zl7E$7Lc-lkryv?28iz`F!c$ufop@%}k;H1F4bJZg8A$~gs;=X?X8ECk>T8FwrSSD; zSkN_juq-P7#wiCZkv=(dcNYKVZ(6JKK>L;&>@5M_)7kG4ns<@p(*H$zG+*QC%b68X zjvcEQYmCj12}rP?;OG99(YTF2U&kZlRymfOFz(~}B`_gnbJ;?)M$|Bho}fK|S-fl{ z#5kX92qd^fSRwq0+cz$efk&$qmrk*45k4b;s$Q6Y__^%HW1sFFljtPJjcf^9t~0qr zI=P?I0qY~to$dDD+~@y}+az`Y(BOruD=WtK_6qkG>t$@Em6b^tw5k#|HWk@OHl-ua zr8IgamH4w`8l5ji{(%TE47>wUPMwvv=Y{p`TGxT_;v++gOUpoY6T8H=Rwn6WBa z2?Xm}lxwsk6VH!9u3QBf%QfJYA0tB)E{o;%HBGuuwPBjgHiER+$~#7_ml`@`oT5{Y zgIwYHa*n7Eei?h58A=qCfn0j!Nx|KOwfVsyk(l{jH%KCL>q~xR!72%3q#`iV6E{kT zGay+SKH2d)m}HopXF9jTw{r3pg+dKdj*?9MtNKr}#IB@`FHayJx)3_khZA8J zmP|_n8D~W&Nl@D>+IeT2+3wZZ=ZB|%*U4o3^A%RcZ|h-qlIy_!0;}wDc{bcKbN^f3 z_yt|x`}^_g!bofkt7Wi;l5qRhUU)-(f)D@Y&5kYjzyw!`Bu-A zOrRQtSM+mnwo*x=*-J51Ct84?N~PhDYV*3pUvosO;LxdklFDBHdk}&?u!j$@;z+$DM}WLB&H#zI?ZjIr`9b-?e#@ zVl0@JJ*QWFHkxjX;&O)YoB!EvLi=7xD$kqB1%=&t=m6&-8iD6od{5meS-$~>Uy!Rm zjDE4m@0hPs2<~BMebK!kujC@<@onq8PPfdKsqV37{WQOD&YEEO`#?J7>fU@`GF<}F z;}Lj&cfA#p}Z)S3bT zjqcGsT-JY46*H7KvbnNV{XRQWmB^@(HG`iPcf@2YR}C={@!6qo$FL{+;@8-_s7--> zlAo(*8Xa@etubQm;)BL06)efCjGZRjM8$S~mgp zV|P)Txek8O=#UdC5%_;zDf9tciZUYL_)%Uv;gnE*_OZOzKjf8`a^9a8yKGZm6-I2m z$P$otV>-VCWU8MX@Z&a>)9J;qf^1O(;eNE#r{YGmcRf}w=K;k$Q&zjn$*IgT&{gI3 zmy!=iHXeTi*@Bdo=jLaHw}Dwnba_Sz3kYz=x?FGqbRflW^9!U`2st8TV zcfRfKqjX#r;VwR=@lDMxPZ~I>t4Wc*uI=kuHRg9E+oag~cNo%9hR}{r0w>Sa89>4S z2#^5jWr?zB6rFZ4Jqh^f9q6?ynjUM&O=_blS-dlL$jrm1Vp$g1&OalRg(TgcDCUa& z@%HB%V*zV#PCm|8f`=Lm1oClfHKV1`#VVJOpBH}nrT9kc@&bdf!FHYAbDAcNGey3!!hpM?zEO!Vhi5+MpQW~25SH;6%>-rP$z zp!c>1Ihu7d!M4AEsIG(g@?~HBw z)l#T6#AJMThQ+T3O6O}`V&yq%Bh9J{YmN#Lln>r2lY|-MR0>@a(e0LZGIDo2!D8(?WddNb-`toRKLRX<#-`BuJ=RRrpp?y;{ItpFCU)t@#RYGP%X7jqs_`1glWJ=o6XAW4;lfSn3R2Mx7~z< z1*Nq#=pGwPC;&M{7}ocMK9lKla52uwvTs8F+t-DsEaxn6!l)^jheILEG{ys(&7M|bYECGeS)7mXRhNH_&d*gmQ}MWTQ-Dg%rW7uLR{>_{ z@}tu#uMFby&M0}$aPe0IJovp>LVO_siq*eCP`F$7e!SZu04mzOU$5AdN zyf33bCxtg|J5AeV7R13HV-Jl^=i3G^DStT}z{okk=ZZPNHWX~-HLHw$B>S~r(V{xq zE+zHSS$<*P9x{nN=4H3uQEOKK}S0?d#n zOhn+@UAT01wTzdckoWvbOUcq`9hmeLnf}D#ad;7wELfoJ3?x1d}eSA#<~L7M60HF?l1oG})vfh}|Zy zuJ;p_>$;6Yj2;_9r5_&sIq7W}bb4)kn#IlK<9CeO^LuEaLd%6QYZ5$Q;29x4bA2bl zNq1jq`Gl!&iqg@aAaN9SKhMu&5=2sn*Ka(&+-O%^V8~c*zBLO=+Z#u#NLqJHVd@dH zb*eE~Oml&B{FS5Z2_r_Y)t{nxY)m6E?B#iuQ6D>D-B|eAWK(=oV;8x>vxVJNXr$Zb z9kAx}=9oBk=>BThRFB6)e0)3{&&yt+-iObY8gH9=L#3%eq1BYe5zfQmD*V&e+5USx zW#RY^`hltEMb~vmr1A2#Qr$z> z6t1m{O3v&11V(&998I!Un2C-{9HO?$yWudWE{)yl=k`=^>%IFu#+Xmm&Sr&$>Z7}5 z5w-gWX-dyiAroa%$JpiS5K;TLi&fV;@W@Y{cO#07FYTvT7{i1cdsZJV6yhM`qL9FP zUV%}??TW#}7DjL?%ud%q7_O-)hPS?5rJsomt7?x;so`Gm>_!6Cw~bYuIcbFG-qbuV z-H3hLg|^t(xXGotc*{C!CU<6s=(EIfxQN_OFQb$6-Uf<|uj7Y}qp=%pMEGw(@!5&Q zZnot~^C|$`d7&D3t+^2cr&%@rR;p3Vy^wWeQx}Jr-GRtb@0Y6;69lNn6qhDjgPd3; z%@6_iQ5=g49AH3>BK6p?JzGHl>-pO(p!~d$c%X8HnPJQ!>*`fJTiwV;)on|=2(NvL zTEl`mi3G!xOaJ-k=^5er1|^MdwKIF8Rg`_c`1oS>fJ$Eu+?2;zS&FGl33#quSZc;g zc%Ck0NZMVsQ7aeWI9uRnto^<_ZdLB{MV}oSv57%Pu%Lc{_z$l`^>zeeOq8UCZ_-^3 zsMZdy^O&<3&urbDduw5BH_;s>Ehey9MdbNu zEZh)hK4@myI1uuElJ_PGl%gnErZGt%|3#Y1=6Z4W_l?I523 zK5L87nQzxv*(*m$WAIwq@kUX~=UaABc)?~S21(v`3v4meI8`a*P2C5MAa_nEp3&Gf zpYeGW?A@eXn^hJ!ueJGa%OsBo-6jQcexH9+m_POnig2FU`HvGdRZ#&B0&8%0S=m*C z4UsbD)t3OAHbvaKC!CR|A2+^1$%Z&(M%4l5uvpyC5oaYO7Ikvj>Ut)0jEcNhxOW@N z*y%IH4@-~vuYIS3&=GR`f3(~;2XpP@Kap>G<6k*EEpLvctn@B9*EWd1XM3@G@cT}N zSQmJt&a9mCA>a_YoJm;#K`VYsi*xMwH?tM!7n7|#1xu;gS3dzBI=z<)q}+@2mY)e8 z@&9dl{`E#Qv^#^kkd)N-H7VX0un8Xmld(=HW$d#n-k$Pt^aj&Y>rB^=(qA&(a%OQi zeqB4;k-SsdoYG0m_v*xd^Lgjg>22_Hb(9dA!fz;P?OSGjBK+y=8I}M- z$H9oeBt?b3S4)CM0jMUw^Ik(~YP0NpD@NL%Rz4ZiyOcc8M{y+1$WDYNr!PJhD^Q%4iYx-tkBG2z7aED`cjjLuYVs z+jBr@tlc5J#?NdE8=2$`AXyp6-s#~vgocwic0E)U(STxP0{P+o`9!#+B8yoP&93_o zk9}^zpwgI;%D~{_^Jeqq@BU*XSr^Qs)uL5TlQ+EDRu7$m*HNAgTyoAklk)o(+kOZx zx0b~fsxML&oZi4q?i}{g$jP&iL`6p9O8$AMiE7|d8h(V2lhIhll9wxpdL(toofeS4omg9t@BKMy z#*o+9<>Y5J>1a0a@THotZPVP3E2GXl#l%3+{jHB9+%g9wTdp2@z0OWfEX?)5cLI#P zkB2U&>IYyc`8Aq$!2g(yRHpB^&Qy!_m`A4I=yWy9OeSPpgWGC+Zc~eq(wZC-bFU(_ zi3__dPDq^1825Vm|LBNI%}2+3OvL<+;4u1H+?u;RhY=Z5mGS8^zyq*UKJu{I=1LCT z3o8~`s0|1wRiLP{UBcMc>MY_e68{{wq;|$KHg9$#6FSn@Q7X?CDlnb0>tf6@>D_es z<FHD_TaORlgXd&dj>8X9q5BZR%)L|XV~ocpDzE3O}_Yi6aH zqht9f4)>~dc6O&Uf|S~FyS=lEou9u%GztP|6y}bkED~Xp4#WB?s3z-7Cq>|90jv}u z%f|~*-%_R|&|v|1Ow{O6PA8;O2)!`(1++oB%k(rzAE8dV(7%VF@A>@Uu$TDNkh?q| z&)bYsBT`z-Vp-irDJ>5-ZsBAe@YcXty4UE3QfH8WJPs8h|n$^|yJ6Z*xY)$$Q= z!&;a4s%S>yApwzsydFSUW_Ax$Xpg{ou;&P{roE-=nZ1lxS%Wafa4C}VVQ+{5`|7oB;0l_e41LD)d+^x#sq5VeKToC_0R3rVB z;8weIR1pS7SN%bk-ez44-bY8Swe{*%R`g@tr#aUbjFxW) zM!8LcV0Twu#EjjowWXW@eYR44lE_P3>@7WtbzKpWsX-~SpZd)?$9RgpgSZR}b>CW$ z11!E;OEg+YhBpM5F?j^jGMJL2Yj@eOZ}b}0k+jrK`Ka_>+^HnhK6<~@?~qVy>|KmY zd#iG#7=Kzo0kp_4WTcaCG#3j0!IDL8%+lkXqiT*#NQke9)tP2(`%1VK!{#-aR!KEh zi=LFdQm;P6&-4_STB=U%bjVf%0N{*F#_G$NJ8MVOWu9A4s`x3UOX*%<$B*59ryi0@ zDAMI4PFjbpm`b4MTR4cTUgQHslaXT41Gs7Y%4^Lp@PJhtt*Rd5!w%imsYLKNIJn6q z4jXuA5zl}MAII03D8r8hB!q&I$rN_VMEhZM!UjD7PstT8ao!sJR`~bn9bgGt8?T&E zEgwOZQJ8<Rpd39F_5aQGU~r;X;53DKWqjHNJ&5H3&BN`y4-(2vQBkp@xjWh| zHhpI{vBF$FR_kXsiWoTf+4ZLSRdkb0rtzul+)54(!HzHyvqo56#@2?y)oTw4xMg=# z23&&&8UpSwt4Z!pB?weqf4!$!1^SfeKtX*)dq8$l496h9!|or>_c7|} z`Sc4i!o}XEUUb8wOs0xJoBOcy?aND^^JQoFbje;BsI#x)GeS@I0=nF1agZimu6yyX z%lN0zb$>|DGu#d8B-YQ>H5d+8p7%SPX45MSOeuhzk6Swj@U^r1^0rkCGE;)YT||r` zES0vohe-@TrZIg6+9BX*ng;4#UHjPjoG6vaH{~?H4L7kE^N39Zd>yzEaMrBP4Lls5Ohb)SKJl9xx?sNM z^>4d@71p2G8eBzjQF(o>GD_&~-LaOGl#GX4rJ?bggs9Qbl9K&rh>j!vycl6IW|&%~ zkrQXpd90+r$dG2Ux*c-v*WogtbRuX-^-aL<4Su;e6^8lhc`wwmXH&biNnY=M_&s`; zK8a7w$5lssv-)*XFY{KZ7{vOPuCu{@mnLL|+c8v@+Q&>j7)iF%#aHb=MJl7+3qnxa zi)bt%(_P2}ZY{~6re9FQ-Y`xQHh*^LXEohEbrd9++ZGflI!g;nZC(;Zep67R>E0-@ zVE;I@Q)kHTbPd-}Vly>`F;b;aa-F`a*&e(@N-{*Yt{L9Q8m|FBs@+Wr#wwtnf{t6{ zI`_T?i^DQ;hLJ8mXGIg6yTyepp{LQds;kGA%)V_iqB5*bzU|$q^KYi;xbMi)1BxEo!-MdGcPio2g6y9w}A`0eniflWo`sB((M^iHd|tbw^p~*b&79a=Z?$A*pkA*Qdp>l z$I&^ox(dh1U_%EhmK{C$`pC1D`BS(o9dBCV=9Bq5r9e`Mk;Hd{;bDn{J7ipUVOwis z45W08oN)sP2;BS= zeTknGsS|`c22=Nyc4wGty^iUFHQV;IR%x_6k^ypzAIh*uM;`6WL&YFw(oR&I;e#Gmynf=7X7V#(QUO>sfUn# zPfl7ku3qEp81=GTUErTx$){ldCRNg(=!yh~ly}kbs*Sx$QX+E$6TTossP6iRFFjZL z`snkeRux`@DR`MIW{o!{{JYJlFyP5wS4@t z8Zgq7nrMESnrc(NBjij4p%ibHgwC^^W_?PaI^63jihGQ!)M}EEW!hMAGMOKtm6a2` zZjXZ)Mcf{clfI`j02btbtpR$fr_xZFmvi*-q!Q`!El|N_O(6B_q*om|G zy_w>-6ju0TCnaA{`l~T4fQ=&A43|gN;b7)+U>+mT;cc$~^c`iO(sjiCwv5s`Db25` zljnXmwI<%pYksEB9OV`{M%H=@KhvLlDnLoAK!Ux}uBm0wv(+KwcQOvBmX(!DnnS#? zvmrXhrfmGQB!D=WbPG)Sr(Scz1lOnwC$;Z0*F&-T?>o2s-2!5>4#G3YWYu8y2>pDN zko6i8;Lxc0m$Bnhvq-8lu}ckcp1Pf3O(jo}U0n~gsF6Hu)paN=jkw7-2NzW|)1^zT z@MHnZ5Xx(B(Qq<8)Hhg{8c9$GOM(q9d9~nE810dtxp}NZ~v~nT=dCoZ9IAtB6 zjStu$Kh&dOI1P7t4r|^zcnu({5Af zd^nJ-(r_9oYi!lmEo(EIH5{e5zKTrw#z`zKb$wtAkn~IZebyt;mM3>I=^`;XRczvh z>qD$)`Upz9W_;Cdy9{fZ=={>c!bEdr*tSFn>D887b}P z&Cjwtn_kIGi_Z?+W_Yk=#_O24|+{95l(=;wS9it zbQ2Q~M=c%W@oJTdE z=BF`R);O$}NjIvC6@F!8+hKv`LC6wPcBFP&6AdCo>_M?^3xJyMPqY%6u|}Y>-&&uIr^yX9) z{=rM77R_SSErE#exWLS@KF5pL>LZo~wz}@^=c6OSIJZgdBw8QRKqN2@dg?^-0xl