diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index adc9429602..718e752b64 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -5,9 +5,13 @@ about: Report an issue or question while using nni instance (deployment). --- +**Describe the issue**: + + + **Environment**: - NNI version: -- NNI mode (local|remote|pai): +- Training service (local|remote|pai|aml|etc): - Client OS: - Server OS (for remote mode only): - Python version: @@ -15,15 +19,22 @@ about: Report an issue or question while using nni instance (deployment). - Is conda/virtualenv/venv used?: - Is running in Docker?: + +**Configuration**: + - Experiment config (remember to remove secrets!): + - Search space: + + **Log message**: - - nnimanager.log: + - nnimanager.log: - dispatcher.log: - nnictl stdout and stderr: - - -**What issue meet, what's expected?**: + -**How to reproduce it?**: -**Additional information**: +**How to reproduce it?**: \ No newline at end of file diff --git a/README.md b/README.md index 0b99dacb98..d2cbd8d712 100644 --- a/README.md +++ b/README.md @@ -328,6 +328,27 @@ Join IM discussion groups: |![image](https://user-images.githubusercontent.com/39592018/80665738-e0574a80-8acc-11ea-91bc-0836dc4cbf89.png)| OR |![image](https://github.com/scarlett2018/nniutil/raw/master/wechat.png)| +## Test status + +### Essentials + +| Type | Status | +| :---: | :---: | +| Fast test | [![Build Status](https://msrasrg.visualstudio.com/NNIOpenSource/_apis/build/status/fast%20test?branchName=master)](https://msrasrg.visualstudio.com/NNIOpenSource/_build/latest?definitionId=54&branchName=master) | +| Full linux | [![Build Status](https://msrasrg.visualstudio.com/NNIOpenSource/_apis/build/status/full%20test%20-%20linux?repoName=microsoft%2Fnni&branchName=master)](https://msrasrg.visualstudio.com/NNIOpenSource/_build/latest?definitionId=62&repoName=microsoft%2Fnni&branchName=master) | +| Full windows | [![Build Status](https://msrasrg.visualstudio.com/NNIOpenSource/_apis/build/status/full%20test%20-%20windows?branchName=master)](https://msrasrg.visualstudio.com/NNIOpenSource/_build/latest?definitionId=63&branchName=master) | + +### Training services + +| Type | Status | +| :---: | :---: | +| Remote - linux to linux | [![Build Status](https://msrasrg.visualstudio.com/NNIOpenSource/_apis/build/status/integration%20test%20-%20remote%20-%20linux%20to%20linux?branchName=master)](https://msrasrg.visualstudio.com/NNIOpenSource/_build/latest?definitionId=64&branchName=master) | +| Remote - linux to windows | [![Build Status](https://msrasrg.visualstudio.com/NNIOpenSource/_apis/build/status/integration%20test%20-%20remote%20-%20linux%20to%20windows?branchName=master)](https://msrasrg.visualstudio.com/NNIOpenSource/_build/latest?definitionId=67&branchName=master) | +| Remote - windows to linux | [![Build Status](https://msrasrg.visualstudio.com/NNIOpenSource/_apis/build/status/integration%20test%20-%20remote%20-%20windows%20to%20linux?branchName=master)](https://msrasrg.visualstudio.com/NNIOpenSource/_build/latest?definitionId=68&branchName=master) | +| OpenPAI | [![Build Status](https://msrasrg.visualstudio.com/NNIOpenSource/_apis/build/status/integration%20test%20-%20openpai%20-%20linux?branchName=master)](https://msrasrg.visualstudio.com/NNIOpenSource/_build/latest?definitionId=65&branchName=master) | +| Frameworkcontroller | [![Build Status](https://msrasrg.visualstudio.com/NNIOpenSource/_apis/build/status/integration%20test%20-%20frameworkcontroller?branchName=master)](https://msrasrg.visualstudio.com/NNIOpenSource/_build/latest?definitionId=70&branchName=master) | +| Kubeflow | [![Build Status](https://msrasrg.visualstudio.com/NNIOpenSource/_apis/build/status/integration%20test%20-%20kubeflow?branchName=master)](https://msrasrg.visualstudio.com/NNIOpenSource/_build/latest?definitionId=69&branchName=master) | + ## Related Projects Targeting at openness and advancing state-of-art technology, [Microsoft Research (MSR)](https://www.microsoft.com/en-us/research/group/systems-and-networking-research-group-asia/) had also released few other open source projects. diff --git a/examples/notebooks/retrieve_nni_info_with_python.ipynb b/examples/notebooks/retrieve_nni_info_with_python.ipynb deleted file mode 100644 index 97f9ae7e08..0000000000 --- a/examples/notebooks/retrieve_nni_info_with_python.ipynb +++ /dev/null @@ -1,232 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Python wrapper for nni restful APIs\n", - "\n", - "nni provides nnicli module as a python wrapper for its restful APIs, which can be used to retrieve nni experiment and trial job information in your python code. This notebook shows how to use nnicli module.\n", - "\n", - "For a full nnicli API reference, please refer to [this documentation](https://nni.readthedocs.io/en/latest/nnicli_ref.html)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Start nni experiment using specified configuration file\n", - "Let's use a configruation file in nni examples directory to start an experiment. Make sure you have installed nni, seaborn and pytorch in your environment." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "tags": [] - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": "authorName: default\nexperimentName: example_mnist_pytorch\ntrialConcurrency: 1\nmaxExecDuration: 1h\nmaxTrialNum: 10\n#choice: local, remote, pai\ntrainingServicePlatform: local\nsearchSpacePath: search_space.json\n#choice: true, false\nuseAnnotation: false\ntuner:\n #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner, GPTuner\n #SMAC (SMAC should be installed through nnictl)\n builtinTunerName: TPE\n classArgs:\n #choice: maximize, minimize\n optimize_mode: maximize\ntrial:\n command: python3 mnist.py\n codeDir: .\n gpuNum: 0\n" - } - ], - "source": [ - "! cat ../trials/mnist-pytorch/config.yml" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "tags": [] - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": "INFO: expand searchSpacePath: search_space.json to /home/xxx/nni/examples/trials/mnist-pytorch/search_space.json\nINFO: expand codeDir: . to /home/xxx/nni/examples/trials/mnist-pytorch/.\nINFO: Starting restful server...\nINFO: Successfully started Restful server!\nINFO: Setting local config...\nINFO: Successfully set local config!\nINFO: Starting experiment...\nINFO: Successfully started experiment!\n------------------------------------------------------------------------------------\nThe experiment id is OhHNEkLQ\nThe Web UI urls are: http://127.0.0.1:8080 http://xxx.xxx.xxx.xxx:8080 http://172.17.0.1:8080\n------------------------------------------------------------------------------------\n\nYou can use these commands to get more information about the experiment\n------------------------------------------------------------------------------------\ncommands description\n1. nnictl experiment show show the information of experiments\n2. nnictl trial ls list all of trial jobs\n3. nnictl top monitor the status of running experiments\n4. nnictl log stderr show stderr log content\n5. nnictl log stdout show stdout log content\n6. nnictl stop stop an experiment\n7. nnictl trial kill kill a trial job by id\n8. nnictl --help get help information about nnictl\n------------------------------------------------------------------------------------\nCommand reference document https://nni.readthedocs.io/en/latest/Tutorial/Nnictl.html\n------------------------------------------------------------------------------------\n\n" - } - ], - "source": [ - "from nnicli import Experiment\n", - "exp = Experiment()\n", - "exp.start_experiment(config_file='../trials/mnist-pytorch/config.yml')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Retrieve nni experiment and trial job information" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": "{'status': 'DONE', 'errors': []}" - }, - "metadata": {}, - "execution_count": 4 - } - ], - "source": [ - "exp.get_experiment_status()" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": "[{'trialJobStatus': 'SUCCEEDED', 'trialJobNumber': 10}]" - }, - "metadata": {}, - "execution_count": 5 - } - ], - "source": [ - "exp.get_job_statistics()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": "{'id': 'OhHNEkLQ',\n 'revision': 181,\n 'execDuration': 1680,\n 'logDir': '/home/xxx/nni-experiments/OhHNEkLQ',\n 'nextSequenceId': 11,\n 'params': {'authorName': 'default',\n 'experimentName': 'example_mnist_pytorch',\n 'trialConcurrency': 1,\n 'maxExecDuration': 3600,\n 'maxTrialNum': 10,\n 'searchSpace': '{\"batch_size\": {\"_type\": \"choice\", \"_value\": [16, 32, 64, 128]}, \"hidden_size\": {\"_type\": \"choice\", \"_value\": [128, 256, 512, 1024]}, \"lr\": {\"_type\": \"choice\", \"_value\": [0.0001, 0.001, 0.01, 0.1]}, \"momentum\": {\"_type\": \"uniform\", \"_value\": [0, 1]}}',\n 'trainingServicePlatform': 'local',\n 'tuner': {'builtinTunerName': 'TPE',\n 'classArgs': {'optimize_mode': 'maximize'},\n 'checkpointDir': '/home/xxx/nni-experiments/OhHNEkLQ/checkpoint'},\n 'versionCheck': True,\n 'clusterMetaData': [{'key': 'codeDir',\n 'value': '/home/xxx/nni/examples/trials/mnist-pytorch/.'},\n {'key': 'command', 'value': 'python3 mnist.py'}]},\n 'startTime': 1597942817897,\n 'endTime': 1597944680966}" - }, - "metadata": {}, - "execution_count": 6 - } - ], - "source": [ - "exp.get_experiment_profile()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": "[TrialJob(trialJobId: PTWOZ status: SUCCEEDED hyperParameters: [TrialHyperParameters(parameter_id: 0 parameter_source: algorithm parameters: {'batch_size': 32, 'hidden_size': 1024, 'lr': 0.1, 'momentum': 0.1922378994556755} parameter_index: 0)] logPath: file://localhost:/home/xxx/nni-experiments/OhHNEkLQ/trials/PTWOZ startTime: 1597942828086 endTime: 1597943039314 finalMetricData: [TrialMetricData(timestamp: 1597943031202 trialJobId: PTWOZ parameterId: 0 type: FINAL sequence: 0 data: 99.36)] stderrPath: None),\n TrialJob(trialJobId: InH3J status: SUCCEEDED hyperParameters: [TrialHyperParameters(parameter_id: 1 parameter_source: algorithm parameters: {'batch_size': 16, 'hidden_size': 256, 'lr': 0.1, 'momentum': 0.8122758606731078} parameter_index: 0)] logPath: file://localhost:/home/xxx/nni-experiments/OhHNEkLQ/trials/InH3J startTime: 1597943053122 endTime: 1597943249425 finalMetricData: [TrialMetricData(timestamp: 1597943249044 trialJobId: InH3J parameterId: 1 type: FINAL sequence: 0 data: 10.1)] stderrPath: None),\n TrialJob(trialJobId: aj2DE status: SUCCEEDED hyperParameters: [TrialHyperParameters(parameter_id: 2 parameter_source: algorithm parameters: {'batch_size': 64, 'hidden_size': 128, 'lr': 0.0001, 'momentum': 0.4401062752065499} parameter_index: 0)] logPath: file://localhost:/home/xxx/nni-experiments/OhHNEkLQ/trials/aj2DE startTime: 1597943258156 endTime: 1597943416538 finalMetricData: [TrialMetricData(timestamp: 1597943409420 trialJobId: aj2DE parameterId: 2 type: FINAL sequence: 0 data: 85.45)] stderrPath: None),\n TrialJob(trialJobId: w3wpE status: SUCCEEDED hyperParameters: [TrialHyperParameters(parameter_id: 3 parameter_source: algorithm parameters: {'batch_size': 64, 'hidden_size': 1024, 'lr': 0.1, 'momentum': 0.26330740737640446} parameter_index: 0)] logPath: file://localhost:/home/xxx/nni-experiments/OhHNEkLQ/trials/w3wpE startTime: 1597943428190 endTime: 1597943586757 finalMetricData: [TrialMetricData(timestamp: 1597943580752 trialJobId: w3wpE parameterId: 3 type: FINAL sequence: 0 data: 99.33)] stderrPath: None),\n TrialJob(trialJobId: ekUrl status: SUCCEEDED hyperParameters: [TrialHyperParameters(parameter_id: 4 parameter_source: algorithm parameters: {'batch_size': 64, 'hidden_size': 1024, 'lr': 0.001, 'momentum': 0.6196562297063133} parameter_index: 0)] logPath: file://localhost:/home/xxx/nni-experiments/OhHNEkLQ/trials/ekUrl startTime: 1597943598222 endTime: 1597943757003 finalMetricData: [TrialMetricData(timestamp: 1597943747959 trialJobId: ekUrl parameterId: 4 type: FINAL sequence: 0 data: 97.91)] stderrPath: None),\n TrialJob(trialJobId: CBvzn status: SUCCEEDED hyperParameters: [TrialHyperParameters(parameter_id: 5 parameter_source: algorithm parameters: {'batch_size': 16, 'hidden_size': 256, 'lr': 0.0001, 'momentum': 0.6226217880666888} parameter_index: 0)] logPath: file://localhost:/home/xxx/nni-experiments/OhHNEkLQ/trials/CBvzn startTime: 1597943768253 endTime: 1597943971248 finalMetricData: [TrialMetricData(timestamp: 1597943970892 trialJobId: CBvzn parameterId: 5 type: FINAL sequence: 0 data: 96.15)] stderrPath: None),\n TrialJob(trialJobId: Thriw status: SUCCEEDED hyperParameters: [TrialHyperParameters(parameter_id: 6 parameter_source: algorithm parameters: {'batch_size': 128, 'hidden_size': 512, 'lr': 0.1, 'momentum': 0.05546862979056} parameter_index: 0)] logPath: file://localhost:/home/xxx/nni-experiments/OhHNEkLQ/trials/Thriw startTime: 1597943978282 endTime: 1597944124673 finalMetricData: [TrialMetricData(timestamp: 1597944120237 trialJobId: Thriw parameterId: 6 type: FINAL sequence: 0 data: 99.25)] stderrPath: None),\n TrialJob(trialJobId: dE0HP status: SUCCEEDED hyperParameters: [TrialHyperParameters(parameter_id: 7 parameter_source: algorithm parameters: {'batch_size': 128, 'hidden_size': 1024, 'lr': 0.01, 'momentum': 0.3669870499772513} parameter_index: 0)] logPath: file://localhost:/home/xxx/nni-experiments/OhHNEkLQ/trials/dE0HP startTime: 1597944138317 endTime: 1597944291430 finalMetricData: [TrialMetricData(timestamp: 1597944291080 trialJobId: dE0HP parameterId: 7 type: FINAL sequence: 0 data: 98.8)] stderrPath: None),\n TrialJob(trialJobId: swAW3 status: SUCCEEDED hyperParameters: [TrialHyperParameters(parameter_id: 8 parameter_source: algorithm parameters: {'batch_size': 16, 'hidden_size': 1024, 'lr': 0.001, 'momentum': 0.32479400764440947} parameter_index: 0)] logPath: file://localhost:/home/xxx/nni-experiments/OhHNEkLQ/trials/swAW3 startTime: 1597944303349 endTime: 1597944510877 finalMetricData: [TrialMetricData(timestamp: 1597944503684 trialJobId: swAW3 parameterId: 8 type: FINAL sequence: 0 data: 98.64)] stderrPath: None),\n TrialJob(trialJobId: LcnOg status: SUCCEEDED hyperParameters: [TrialHyperParameters(parameter_id: 9 parameter_source: algorithm parameters: {'batch_size': 64, 'hidden_size': 128, 'lr': 0.001, 'momentum': 0.07252783892989623} parameter_index: 0)] logPath: file://localhost:/home/xxx/nni-experiments/OhHNEkLQ/trials/LcnOg startTime: 1597944523381 endTime: 1597944680557 finalMetricData: [TrialMetricData(timestamp: 1597944672287 trialJobId: LcnOg parameterId: 9 type: FINAL sequence: 0 data: 95.97)] stderrPath: None)]" - }, - "metadata": {}, - "execution_count": 7 - } - ], - "source": [ - "exp.list_trial_jobs()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Visualizing nni experiment result\n", - "\n", - "With the retrieved trial job information, we can do some analysis by visualizing the metric data, below is a simple example." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": "
", - "image/svg+xml": "\n\n\n\n \n \n \n \n 2020-08-21T01:35:59.389563\n image/svg+xml\n \n \n Matplotlib v3.3.1, https://matplotlib.org/\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3EAAAF4CAYAAAD65HCtAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAABGyklEQVR4nO3deXwM9+PH8XeykaTOCBKJKOWLusWVqLriVgSljv5US6lSZ5XUFbTVhqIoLaotrbrqCGnrqqsl7lt6uDUSQtxFrt3fH6mtVBJHNpJJXs/Hw+ORnc/OZz77sTM775nPzNhZLBaLAAAAAACGYJ/RDQAAAAAAPDxCHAAAAAAYCCEOAAAAAAyEEAcAAAAABkKIAwAAAAADIcQBAAAAgIEQ4gAANrVnzx41bdr0od67fPlyde7cOdmyiIgIeXt7KyEh4YH1TJ8+XUOGDHnoNu7du1dNmjSRt7e3NmzYoNdff10rVqx46PlTsnPnTtWtWzfN9aSVn5+ftm/fntHNAACkE4eMbgAAIPPz9va2/n379m05OjrKZDJJksaOHavWrVtby6tXr661a9emeZmenp7av39/mutJzrRp0/Tyyy+rW7dukqRGjRqly3Iyg+nTp+vMmTP6+OOPM7opAAAbIcQBAB7o3jDl5+en999/X88999x974uPj5eDQ+b/aYmIiFCpUqUytA1G6SsAQObDcEoAwGO7O3xw9uzZql27tt599937hhTOnj1bjRo1kre3t1q0aKH169c/VN3h4eEqU6aM4uPjJUkXLlxQ7969VbNmTTVu3FhLlixJ8v7Y2FgNHDhQ3t7eatu2rX7//fdk623UqJH++usv9e7dW97e3oqNjVXXrl21dOlSSf8O8QwKClKNGjXk5+enLVu2WOdftmyZmjdvLm9vbzVs2FCLFi166P4qU6aMFixYoCZNmqhJkyaSpE2bNsnf31/Vq1dXp06dkrR79uzZqlOnjry9vdW0aVOFhoZKkgICAjRlyhTr+1Iaxrl161bNmjVLP/30k7y9va1nTJcvX66GDRvK29tbfn5+WrVq1UN/BgBAxuMQIAAgTS5duqRr165p06ZNMpvNOnjwYJLyokWLasGCBSpUqJDWrFmjd955R+vWrZObm9sjLWfw4MEqVaqUfvnlF508eVKvvfaaihYtqlq1akmSfv75Z02aNEkTJ07U/Pnz1adPH61du1Y5cuRIUs+GDRtSPZsoSYcOHVLbtm21Y8cOLV68WCNGjNAvv/wiOzs7FShQQLNmzVLRokW1e/du9ezZUxUrVlT58uUf6nNs2LBBS5YskbOzs8LCwjR8+HB9/vnnqlChglatWqU+ffpozZo1Cg8P14IFC/T999/L3d1d4eHhMpvNj9RndevW1RtvvJFkOOWtW7f0/vvv6/vvv1eJEiUUFRWla9euPVK9AICMxZk4AECa2Nvbq3///nJ0dJSzs/N95c2bN5e7u7vs7e3VokULFStWTIcOHXqkZURGRmrfvn0aMmSInJycVLZsWXXo0EHBwcHW95QvX17NmjVTjhw59Nprryk2Nva+QPmwPD099dJLL8lkMqlt27a6ePGiLl26JEmqX7++nn76adnZ2almzZqqXbu29uzZ89B19+rVSy4uLnJ2dtbixYvVsWNHVa5c2bqsHDly6MCBAzKZTIqNjdWJEycUFxcnLy8vPf3004/1ef7L3t5ex44d0507d+Tm5pbhQ0sBAI+GM3EAgDTJnz+/nJycUixfuXKlvvrqK507d05S4pmgK1euPNIyoqKilC9fPuXOnds6zdPTU0eOHLG+Lly4sPVve3t7ubu7Kyoq6pGWc1fBggWtfz/11FPWdkvSli1bNGPGDJ0+fVpms1l37txR6dKlH7puDw8P698RERFauXKlvv32W+u0uLg4RUVFqWbNmho+fLimT5+u48eP6/nnn1dAQIDc3d0f6zPdlTNnTk2ZMkVffvmlRowYoapVq2rYsGEqWbJkmuoFADw5nIkDAKSJnZ1dimXnzp3TyJEjNWrUKO3cuVN79ux5rLM+bm5uunbtmm7evGmdFhkZmSTQnD9/3vq32WzWhQsXHnnI5oPExsaqf//+6t69u7Zt26Y9e/aobt26slgsD13Hvf3l4eGh3r17a8+ePdZ/Bw8eVMuWLSVJrVq10sKFC7Vp0ybZ2dlZh0Q+9dRTunPnjrWeu2cJH7S8u+rUqaOvvvpKv/76q0qUKKFRo0Y9dPsBABmPEAcASDe3b9+WnZ2dXF1dJSXeFOTYsWOPXI+Hh4e8vb01efJkxcTE6Pfff9f333+f5NEGR48e1bp16xQfH6958+bJ0dFRlStXttlnkRJDXGxsrFxdXeXg4KAtW7Zo27Ztj11fhw4dtGjRIh08eFAWi0W3bt3S5s2bdfPmTZ08eVKhoaGKjY2Vo6OjnJycZG+f+LNdtmxZbdmyRVevXtXFixc1b968FJdRoEABnTt3zno93aVLl7RhwwbdunVLjo6Oypkzp7VeAIAxsNUGAKSb//3vf+revbs6deqk5557Tn/++aeqVq36WHVNnjxZ586dU506dfTWW2+pX79+SW5M0rBhQ/3444+qUaOGgoODNX369PtuapJWuXPn1siRIzVw4EDVqFFDISEh8vPze+z6KlasqPfee0/jxo1TjRo11KRJEy1fvlxSYmCcNGmSfHx89Pzzz+vy5csaPHiwJMnf31/PPvus/Pz81L17d7Vo0SLFZTRr1kyS5OPjo7Zt28psNuvrr79WnTp1VLNmTe3evVtjxox57M8AAHjy7CyPMgYEAIAn5K+//lLTpk119OjRVIdsAgCQ3XAmDgCQKf3555/y9PQkwAEA8B/cnRIAkOl89dVX+uKLLzRy5MiMbgoAAJkOwykBAAAAwEAYTgkAAAAABkKIAwAAAAADIcQBAAAAgIFk6hubXLnyt8xmLtkDAAAAkL3Y29spf/5cyZZl6hBnNlsIcQAAAABwD4ZTAgAAAICBEOIAAAAAwEAIcQAAAABgINkqxK1fv0YNG9ZRvXq+8vdvrjNnTqc6/b8WLvxW9erVUoMGtVWvnq/mzPksSXlw8HLVq+erunV9VK+er6KioiRJCQkJGjZssGrUqKSaNSvr22/nWedJrQzA40nPdT21MtZ14MlK67o+aVKQ6tSpqXr1aqlRo7rauHGDtaxfv96qXPlZNWhQWw0a1NaUKROtZVFRUerQwV++vt6qX/857d27+6HKAMBW7CwWS6a9c0h09E2b3djk6tUr8vX11g8/rFfJkqW0dOkiff/9Ys2a9WWy0xcvXnFfHTduXFfu3HlkZ2enmzdvqG5dX33zzWKVL19BBw7s01tvvaFly0Lk7u6u69evydHRSc7Ozlq8+DstW7ZEixYt1+XLl9Ww4fNavXqtnn66WKplAB5deq/rqZWxrgNPji3W9Y0bN8jX9znlzJlTR44cVps2LXT48J966qmn1K9fb1Wp4q0ePd64b74BA/qoWLHiGjx4qHbsCNWAAW9qx479srOzS7Uss1u/fo0++ugDxcfHycUlv6ZN+0zFihXXunU/6aOPPpDFYpHFYtGQIQFq2bL1ffNfvHhRAwa8qXPnzik+Pk61a9fR+PET5eDgoISEBA0f/o42btwgOzs79e8/WP/3f90k6bHLgKzO3t5OBQrkTr7sCbclw5w6dVKFCrmpZMlSkqRGjZpo06afU5weHR19Xx158uS1boRv3bqtuLg46+vPP5+hPn36y93dXZKUN28+OTs7S0o8Q9e166uyt7dXwYIF1bz5C1q1auUDywA8uvRe11MrY10HnhxbrOt+fo2UM2dOSVL58hVksVh05crlBy47OHiFunXrIUny9a0lJycnHTiw74FlmdnVq1fUr19vzZ79pbZs2aH/+79uGjp0kCwWi/r2fUMzZszWpk3bNGPGbPXr11tms/m+OqZO/VilSpXRli2h2rw5VIcOHdAPP6ySJH3//WKdOnVSO3ce0I8//qyJEz/U2bNn0lSWmaV0Nnjdup/k5/e8GjSorfr1n1NIyKoU60hphNddx48fU7Fi7goMHGGdduvWLfXs+apq1qys556rpnXrfnqoMhhPtglxJUv+T1FRF7R//15J0rJlSyRJt2/fTnb6uXN/JVvPmjU/qk6dmqpWrbz69u2vcuXKS5L+/PMPnTlzSq1bN1PDhnU0efIE3T3Jee5cuLy8ilrrKFKkqCIiwh9YBuDRpfe6nloZ6zrw5NhqXb9r8eLvVLz4M/L0LGKd9tlnM1Svnq9eeaWz/vzzD0nS5cvRkiwqUKCA9X1Finjp3LlzqZZldqmFX3t7O12/fl2SdP36Nbm7u8ve/v5dyMQRCjdlNpsVExOj2Ng4FS7sKenxD3IZ8QCYLQLxgQP7NHHih1qyJFhbt+7U6tVrlTdvXmt5QkKChgwZoObNX0gy38yZ05QnTx7t2nVQ3367WIMG9dPNmzcfWAbjeWCICwoKkp+fn8qUKaM///zTOv3UqVPq2LGjmjZtqo4dO+r06dMPVZZR8ubNp9mzv9aoUe+qceN6unTpovLlc1G+fC7JTjeZkn+EXrNmLfTLL7sUGrpPS5cu1vHjxyQlrkxhYUe1dGmwgoN/1M8/r9eSJQuf5EcEoPRf1x9UltUkdzT57Nkz1uuEGjSorWrVKqh06aeTnf/ChQt65ZVOqlevlmrXrq6lSxdZyzZt+lmNG9eTl1fBJEeSJa4vxIPZal2XpO3bf1VQ0AeaNetL67Thw0dr164D2rJlh154oZU6dWqnhISEJ/HRMkRKoTgiIlxz5sxTt26dVLVqeXXr1lmffjor2ToGDx6qkyePq0KFUqpQoZQaNGgoHx9fSY9/kMuIB8BsEYhTG+ElSdOmTVbjxs1UosT/kswXHLxcr7zymiSpRIn/qUoVb23cuP6BZTCeB4a4hg0basGCBSpSpEiS6YGBgerSpYvWrl2rLl26aPTo0Q9VlpHq1WugkJB1Wr9+i7p3f0N37txW8eLPpDg9NV5eReXtXU3r1q3557WXWrb0l5OTk3LnzqNmzV6wbgiLFPFSePi/RwDPnftLnp5eDywD8HjSc11PrSyrrespHU1++uli2rRpm/Vf8+YvqF27DsnWERj4ripX9taWLaEKDl6j8ePH6dy5xB2wYsWKa/Lk6erbt/9982XF4VWwPVus67t371SfPj01b953+t//Slmne3h4WneuO3bsor//vqmIiHNydU08y3bv8Mxz58JVpEiRVMsyu5RCscnkoKlTJ2vevEXat++ovvlmiXr2fDXZMzirVq1UuXLldeTIMR069LtCQ7dp9eqVT/7DZDBbBOLURngdOXJYmzb9rN69+943X3h4uLy8/j2odu+Z4NTKYDwPDHHVq1eXh4dHkmnR0dEKCwtTy5YtJUktW7ZUWFiYLl++nGpZRrtw4YIkyWw2a/z4serWrbty5cqV4vT/ujuUQkrsg23btqps2XKSpHbtOmjz5o2yWCyKi4vTL79sVvnyFSVJrVu31TfffC2z2axLly7pp59+UKtW/g8sA/B40nNdT60sq63rD3NtUWxsrJYtW6IuXbomW8fRo0fk59dIklSwYEFVqFBRwcGJN5goUaKkKlaslOwZkqw2vArpI63r+v79e9Wr12uaO3e+KlWqkqQsMjLC+vfGjRtkMpnk4ZE4NLBVqzaaN2+uJGnHjlDdvn1blSt7P7Ass0su/F67dlUXLkRaz6j5+PgqZ86cOnbsj/vmnzt3ll588SXZ29srb958atbsBf3661ZJj3+Qy4gHwGwRiFMa4RUXF6chQ/pr4sRPZDKZMuDTIbNIeWxBKiIjI+Xu7m798phMJrm5uSkyMlIWiyXFMldXV9u1/DF89NF72rVrh2JjY1W/fkONHDk21emS1Lnzixo2bISqVKmqb775Sps3b5SDQw5ZLBb16NFLDRo0lCS1bdteBw7s1/PP15C9vb3q12+ol19+RZLUoUMn7d27Rz4+VSRJb789TMWKFX9gGYDHk57remplWW1dv/dosrd3tSTXFt295mft2h9VuLDnfTvAd1WqVEUrVixTlSpVdfbsGe3evVNFiyY/9PJeWW14FdJHWtf1YcMG686d2xoyZKC1fMaM2SpXrrzeequ3Ll2Kkp2dvfLkyaP58xfJwSFxt2nkyLHq06enFi/+Ts7OT2nGjDnWs3aplWV2Fy5ckLu7e5LwW7JkKUVEROj48WP63/9K6c8//9DFi1HJntl8+uli2rhxg6pWra7Y2Fht3bpZL7zQStK/B7JeeKG1Ll++rJ9++kGrVq1JU1lmVq9eA9Wr10BS4mMnZsyYlmog9vaulmT+e0d4OTk5WUd41a5dR6dPn1KXLu0lSdeuXZPFYtHNmzc0adI0eXl5KTz8rAoWLCgpcXv5/PN1rXWmVAYDsjykBg0aWP744w+LxWKxHD582NKiRYsk5c2bN7ccOXIk1TIAgLGsX7/eUrt2bUu1atUsI0eOtLi4uFgOHjxoLW/evLll6tSpKc4fFRVlefnlly2VKlWytGzZ0vLSSy9ZBg0alOQ9gYGBlrfffjvJtAoVKlh27dplfR0UFGTp16/fA8sAPL4ePXpYnn32WUuJEiUsvXv3tty+fdtisVgs3377raVChQqWSpUqWSpVqmRZsWKFdZ7mzZtbdu/ebbFYLJbjx49bGjVqZKlQoYKlbNmylj59+lji4uIsFovFEh8fb+ndu7elRIkSlhIlSlhmzZplreNxyzKzyMhIi8VisSQkJFi6d+9uGTBggCUyMtKSJ08ey++//26xWCyWsLAwS/78+S3R0dH3zb9gwQLLyy+/bDGbzZbY2FhLkyZNLLNnz77vff/dfgYGBlpef/11i8Visfz5558WNzc3y/Xr1x9YBuN5rDNxHh4eunDhghISEmQymZSQkKCoqCh5eHjIYrGkWPaobPmcOADAo6tc2UcrViTehjoqKkoTJ05U3rxuunjxhiIjI7RlyxZNmTJTFy/eSKEGZ02Z8u8D0Tt3flG+vnWSvP/vv2N061ZskmmFC3vq8OHfVbz4s5KkP/44Li+vp3Xx4o1UywA8vg8/nJLk9Y0bcbpxI05NmrRWkyZJnwt3d32bN2+x9XXevG767rvlSd535cpt69/jxk3QuHET7qsjLWWZ1ZAhw5KcDX777REymZwVFDRZbdu2k51d4tnZKVNmKCEhhy5evJHkLHHDhi9o69btKlPmWesIr9atX7rvs/93+/nqq73Vv/+beuaZEjKZTJo4caru3JHu3LmRahkyp9SeE/dYIa5AgQIqW7asQkJC5O/vr5CQEJUtW9Y6XDK1MlvJk9dZzk45bFqnUd2JidON63cyuhlAunDJ46gczk4Z3YxMIe5OjK7eiH2iy0xueNXda4sWL/5OjRo1td7MITmXL0crb958cnBw0C+/bNFvv4Vp7txvHrjcrDi8Skr5Ycp37tzRqFHvauvWTXJ2dlb16jU1adK0++bftOlnjR8/Tr/9dlQ9eryhsWM/sJZNmDBeX3/9hdzdEw+a1qzpo6CgyZISnw81YEAfHTy4Xw4ODhoz5n01adL8gWVPUv58T8nB8bF2S7Kc+Nh4Xbl2+8FvRKY1ZcqnyU5v376j2rfvmGzZwoXLrH/b29vrvfc+1HvvfZjqcoYOHZ7kda5cuTR37vxk35taWWaX0razWrUK/ww5Tbxz56hRY63XYd/r+PFjGjp0kPWa7rFjP1D9+n6SpBdfbP3P40GkhIR4/f77b9q0abvKl6+QqbedD9xavv/++1q3bp0uXbqk1157TS4uLvrhhx80ZswYBQQEaObMmcqbN6+CgoKs86RWZivOTjnUZegCm9drRN9NeFk3RIhD1pTD2Uk//nNL5OyuxfyvpCcc4lK7tmjRogUaP37CffPcezR5//69Gj58qEwmk1xdC+ibbxZbH668Y0eo3njjNd24cUMWi0UrVy7TlCmfys+vUZa7vlD6926fP/ywXiVLltLSpYs0dOggLV68QuPGjZKzs5N27NgvOzu7+x7qe9fdO3qGhKzUnTsx95V36NA5SbC7697nQ508eVytWjXTzp0HlDt37lTLniQHRwcdnLn5iS4zs6rcp36a68iXz1mOjhzslqTY2Dhdu8Z+klGltu2UpLlzv7HeYCwl/fu/qVdf7aGXXuqskyePq23blgoN3aecOXNq2bJ/H7j+448h+uij91S+fAVJmXvb+cAQN3LkSI0cOfK+6SVLltTSpUuTnSe1MgCAcaR0NFmSduzYn+z0e48mN2zYRDt3Nkn2fb6+tXTw4O/JliUO9ZnyyGWZWXJ3++zbt5ciIs5pyZJFOnDgN9nZ2UmS3Nzckq2jRImSkqSffgqRdH+IS0lw8HJNn/75P3X8+3yo1q3bploG43J0zKFJkyZldDMyhbfffltK48Hu/C5OcsjhaJsGGVx8XKyuXH347U9apbTtvPdOyQ8SFnZEfn6NJSVu51xc8uvnn9ffd5fohQu/UefO/95tOTNvOxm3AADAE5DS3T5Pnz6l/Pnz6+OPP9K2bb8oV65cCggYJV/fWo+8jJUrl2nz5o1yc3PT0KHDVaOGjySeHQWklUMOR20NGZPRzcgU6rYco0c5iJRWqd0pWZLefPN1WSwW+fj4asSIQOXL53JfHZUqVdHy5UvUq1cfHTiwTydOHEvy6Aop8fKBrVs3a8qUGdZpmXnbSYgDgGwkX96n5OjEpl+SYmPide36k7vu6N5nR8XExKhhw0bKl89FdnZ2OnPmtCpWrKQxY97X3r271bVrR+3ceUB58uR96Pq7deuhQYPeUY4cObR580Z169ZZv/66O9VrFgEgs0tp22kyOWjVqjUqUsRLMTExGjkyQAEBQ/TZZ1/cV8e0aZ9p9Oh3tXDhApUpU0Y+PrXk4JD0OXtLlixUgwaNrI9gyOz4JQeAbMTRyUHjR3yf0c3IFIZ/0P6JLzO5Z0d5eRWVg4OD2rXrIEmqVq2GXF0L6MSJ46pSpepD1+3u7m79u359P3l6eun333/Tc889z7OjABhactvO4sWfsd5oy8nJSa+99rpeeaVTsvMXL/6M5s9fZH39/PM1VLr0s0nes2jRtwoMfC/JtMy87TTG0ycBAMgCLly4IElJ7vZZtOjTql27jjZv3ihJOnHimC5duqhnninxSHVHRkZY/z58+JD++uuM9RqSVq3aaP78ryRJJ08e1/79+6x3cEutDAAyg+S2nZJ0/fo1SbLeHKt8+YrJzn/x4kVZLImPLVu0aIEcHZ1Ut259a/muXTt1/fp1NWyY9BruzLzt5EwcAABPSEp3+5w48RMNHNhXY8aMkINDDs2YMdt6Xce9d/tM7Y6eH3wwVocOHZC9vUmOjol13D0717fvAPXv/6Zq1qwsk8mkSZOmKXfuPA8sA4DMILlt5/nzkerevasSEhJkNieodOlnNWHCZOs8DRrU1sKF36twYQ+tXfujpk+fIjs7OxUv/oy+/nqB9UZSUuJZuJde6iyTKekQy8y87bSz3I2lmVBqD/suVCgPjxj4x3cTXjbEgy+Bx1GoUB4eMfCPFvO/SvO6XqhQHoZT/mP4B+3ZdmYihQrl4RED/6jcp75N1nXuTpno7bfftkl/cmOTRHVbjmHb+YSk9rBvhlMCAAAAgIEwnBIAgMeUL6+jHJ2cMroZmUJsTIyuXX+yD6MHYDx5XZ6SUw4iiCTFxMXr+tXHu0syPQgAwGNydHLS5HffyOhmZAqDP5wliRAHIHVOORw0eMWWjG5GpjC5bb3HnpfhlAAAAABgIIQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMygXXrfpKf3/Nq0KC26td/TiEhqyRJ1apV0HPPVVODBrXVoEFtbdy4IdV6tm37RYULu2ju3FnWaW3atFD16pWsdSxc+K217MSJY2revKF8fb3VvHlDnTx5/KHKAAAAkHEcMroBQHZnsVjUt+8bWrVqjcqWLaejR4+oZcsmatGipSRp7txvVLZsuQfWc/PmDb333mg1bNj4vrLx44PUpEnz+6a/884gde/eUx06dNLSpYs0ZMhALV8e8sAyAAAAZBzOxAGZgL29na5fvy5Jun79mtzd3WVv/2ir5+jRw9W37wC5uhZ4qPdfvHhRhw4dVLt2HSRJ7dp10KFDB3Xp0qVUywAAAJCxCHFABrOzs9OcOfPUrVsnVa1aXt26ddann/47HPLNN19XvXq1NHToIF27djXZOn7+eZ2uX7+uVq3aJFs+duwo1avnqzfffF2RkRGSpIiIcHl4eMhkMkmSTCaTChcurIiI8FTLAAAAkLEIcUAGi4+P19SpkzVv3iLt23dU33yzRD17vqqbN29q1ao12rx5u9at2yyLRQoIGHLf/NeuXdV7743Rhx9+nGz9M2bM1rZte7Rx4zaVKlVaPXu+ms6fCAAAAOmJa+KADHbkyCFduBApHx9fSZKPj69y5sypY8f+kLd3NUmSk5OTXnvtdb3ySqf75v/tt98UFXVezZo1kCRFR0dr3bqfdOXKFQ0ZEqAiRbwkJZ5N69XrTU2c+KHMZrM8Pb0UGRmphIQEmUwmJSQk6Pz58/L09JLFYkmxDAAAABmLM3FABvPwKKKIiAgdP35MkvTnn3/o4sUoFS7soevXr0lKvPnJypXLVL58xfvm9/WtpbCwk9q794j27j2iVq38NXTocA0ZEqD4+HhFRUVZ37t8+fcqW7a87O3tVahQIVWoUFHLly/9p2ypKlaspIIFC6ZaBgAAgIzFmTggg7m7u2vChMnq0aOr7OwSj6t88slMxcTE6OWXX1JCQoLM5gSVLv2sJkyYbJ0v8XEB36twYY8U606so4NiY2MlWVS4sIdmz/7KWj5x4ifq1+8NTZoUJBcXF3366eyHKgMAAEDGIcQBmUD79h3Vvn3H+6Zv3PhrivNs2rQt2enTp39u/TtXrlxav35LinWUKlVaa9ZseuQyAAAAZByGUwIAAACAgXAmDrCxvPmc5OTomNHNyBRiYmN1/VpMRjcDAAAgSyHEATbm5OioV78akNHNyBS+fm2qJEIcAACALTGcEgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMAAAAAA0lziNu0aZPatGkjf39/tW7dWuvWrZMknTp1Sh07dlTTpk3VsWNHnT59Oq2LAgAAAIBszyEtM1ssFg0dOlQLFixQ6dKl9fvvv6tz585q1KiRAgMD1aVLF/n7+ys4OFijR4/W/PnzbdVuAAAAAMiW0nwmzt7eXjdu3JAk3bhxQ25ubrpy5YrCwsLUsmVLSVLLli0VFhamy5cvp3VxAAAAAJCtpelMnJ2dnT755BP16dNHOXPm1N9//63Zs2crMjJS7u7uMplMkiSTySQ3NzdFRkbK1dXVJg0HAAAAgOwoTSEuPj5es2bN0syZM1WtWjXt3btXAwcO1IQJE2zSuAIFctuknuygUKE8Gd0EIFl8N22L/rQt+tO26E/boS9ti/60LfrTdh63L9MU4n777TdFRUWpWrVqkqRq1arpqaeekpOTky5cuKCEhASZTCYlJCQoKipKHh4ej1R/dPRNmc2WZMv48iR18eKNjG4C/sF3M6m0fjfpz6ToT9uiP20rLf1JXybFd9O26E/bYl23ndT60t7eLsWTWmm6Jq5w4cI6f/68Tp48KUk6ceKEoqOjVaxYMZUtW1YhISGSpJCQEJUtW5ahlAAAAACQRmk6E1eoUCGNGTNGAwYMkJ2dnSRp/PjxcnFx0ZgxYxQQEKCZM2cqb968CgoKskmDAQAAACA7S1OIk6TWrVurdevW900vWbKkli5dmtbqAQAAAAD3SPMjBgAAAAAATw4hDgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAc0lpBTEyMxo8fr9DQUDk5OalKlSp67733dOrUKQUEBOjq1atycXFRUFCQihcvboMmAwAAAED2leYQN3HiRDk5OWnt2rWys7PTpUuXJEmBgYHq0qWL/P39FRwcrNGjR2v+/PlpbjAAAAAAZGdpGk75999/a+XKlRowYIDs7OwkSQULFlR0dLTCwsLUsmVLSVLLli0VFhamy5cvp73FAAAAAJCNpelM3F9//SUXFxd9+umn2rlzp3LlyqUBAwbI2dlZ7u7uMplMkiSTySQ3NzdFRkbK1dXVJg0HAAAAgOwoTSEuISFBf/31l8qVK6dhw4bp4MGD6t27t6ZOnWqTxhUokNsm9WQHhQrlyegmAMniu2lb9Kdt0Z+2RX/aDn1pW/SnbdGftvO4fZmmEOfh4SEHBwfrsMnKlSsrf/78cnZ21oULF5SQkCCTyaSEhARFRUXJw8PjkeqPjr4ps9mSbBlfnqQuXryR0U3AP/huJpXW7yb9mRT9aVv0p22lpT/py6T4btoW/WlbrOu2k1pf2tvbpXhSK03XxLm6usrHx0fbtm2TJJ06dUrR0dEqXry4ypYtq5CQEElSSEiIypYty1BKAAAAAEijNN+dcuzYsRo+fLiCgoLk4OCgCRMmKG/evBozZowCAgI0c+ZM5c2bV0FBQbZoLwAAAABka2kOcUWLFtU333xz3/SSJUtq6dKlaa0eAAAAAHCPNA2nBAAAAAA8WYQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZisxD36aefqkyZMvrzzz8lSQcOHFDr1q3VtGlTde/eXdHR0bZaFAAAAABkWzYJcUePHtWBAwdUpEgRSZLZbNY777yj0aNHa+3atapevbo+/vhjWywKAAAAALK1NIe42NhYjRs3TmPGjLFOO3LkiJycnFS9enVJUqdOnbRmzZq0LgoAAAAAsr00h7ipU6eqdevW8vLysk6LjIyUp6en9bWrq6vMZrOuXr2a1sUBAAAAQLbmkJaZ9+/fryNHjmjIkCG2ak8SBQrkTpd6s6JChfJkdBOAZPHdtC3607boT9uiP22HvrQt+tO26E/bedy+TFOI2717t06cOKGGDRtKks6fP68ePXqoa9euioiIsL7v8uXLsre3l4uLyyPVHx19U2azJdkyvjxJXbx4I6ObgH/w3Uwqrd9N+jMp+tO26E/bSkt/0pdJ8d20LfrTtljXbSe1vrS3t0vxpFaahlP26tVLv/76qzZu3KiNGzeqcOHCmjt3rl5//XXduXNHe/bskSQtWrRIzZo1S8uiAAAAAABK45m4lNjb22vChAkKDAxUTEyMihQpookTJ6bHogAAAAAgW7FpiNu4caP176pVq2r16tW2rB4AAAAAsj2bPewbAAAAAJD+CHEAAAAAYCCEOAAAAAAwEEIcAAAAABgIIQ4AAAAADIQQBwAAAAAGQogDAAAAAAMhxAEAAACAgRDiAAAAAMBACHEAAAAAYCCEOAAAAAAwEEIcAAAAABgIIQ4AAAAADIQQBwAAAAAGQogDAAAAAAMhxAEAAACAgRDiAAAAAMBACHEAAAAAYCCEOAAAAAAwEEIcAAAAABgIIQ4AAAAADIQQBwAAAAAGQogDAAAAAAMhxAEAAACAgRDiAAAAAMBACHEAAAAAYCCEOAAAAAAwEEIcAAAAABgIIQ4AAAAADIQQBwAAAAAGQogDAAAAAAMhxAEAAACAgRDiAAAAAMBACHEAAAAAYCCEOAAAAAAwEEIcAAAAABgIIQ4AAAAADIQQBwAAAAAGQogDAAAAAAMhxAEAAACAgRDiAAAAAMBACHEAAAAAYCCEOAAAAAAwEEIcAAAAABgIIQ4AAAAADIQQBwAAAAAGQogDAAAAAAMhxAEAAACAgRDiAAAAAMBACHEAAAAAYCCEOAAAAAAwEEIcAAAAABgIIQ4AAAAADIQQBwAAAAAG4pCWma9cuaKhQ4fq7NmzcnR0VLFixTRu3Di5urrqwIEDGj16tGJiYlSkSBFNnDhRBQoUsFW7AQAAACBbStOZODs7O73++utau3atVq9eraJFi+rjjz+W2WzWO++8o9GjR2vt2rWqXr26Pv74Y1u1GQAAAACyrTSFOBcXF/n4+FhfV6lSRRERETpy5IicnJxUvXp1SVKnTp20Zs2atLUUAAAAAGC7a+LMZrMWLlwoPz8/RUZGytPT01rm6uoqs9msq1ev2mpxAAAAAJAtpemauHu99957ypkzp/7v//5P69evt0mdBQrktkk92UGhQnkyuglAsvhu2hb9aVv0p23Rn7ZDX9oW/Wlb9KftPG5f2iTEBQUF6cyZM/r8889lb28vDw8PRUREWMsvX74se3t7ubi4PFK90dE3ZTZbki3jy5PUxYs3MroJ+AffzaTS+t2kP5OiP22L/rSttPQnfZkU303boj9ti3XddlLrS3t7uxRPaqV5OOXkyZN15MgRzZgxQ46OjpKkChUq6M6dO9qzZ48kadGiRWrWrFlaFwUAAAAA2V6azsQdO3ZMs2bNUvHixdWpUydJkpeXl2bMmKEJEyYoMDAwySMGAAAAAABpk6YQV6pUKf3xxx/JllWtWlWrV69OS/UAAAAAgP+w2d0pAQAAAADpjxAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADAQQhwAAAAAGAghDo8lMHCEqlevKDe3vPrttzDr9BMnjql584by9fVW8+YNdfLk8WTn37TpZzVuXE9eXgUVGDgiSVlCQoKGDRusGjUqqWbNyvr223lpLgMAAACyCkIcHkvz5i0VHPyTihZ9Osn0d94ZpO7de2rHjv3q3r2nhgwZmOz8xYoV1+TJ09W3b//7yr7/frFOnTqpnTsP6Mcff9bEiR/q7NkzaSoDAAAAsgpCHB6Lr28tFSnilWTaxYsXdejQQbVr10GS1K5dBx06dFCXLl26b/4SJUqqYsVKMpkc7isLDl6url1flb29vQoWLKjmzV/QqlUr01QGAAAAZBWEONhMRES4PDw8ZDKZJEkmk0mFCxdWRET4I9Vz7ly4vLyKWl8XKVLUWsfjlgEAAABZBSEOAAAAAAyEEAeb8fT0UmRkpBISEiQl3mjk/Pnz8vT0esCcSRUp4qXw8L+sr8+d+8tax+OWAQAAAFkFIQ42U6hQIVWoUFHLly+VJC1fvlQVK1ZSwYIFH6me1q3b6ptvvpbZbNalS5f0008/qFUr/zSVAQAAAFkFIQ6PZfjwd1S58rOKiDin9u1bq06dmpKkiRM/0dy5s+Tr6625c2dp4sRPrPN07vyiDhzYJ0nasSNUlSs/q88/n6H5879S5crPauPGDZKkDh06qVixZ+TjU0XNm/vp7beHqVix4mkqAwAAALKK+28NCDyE8eMnavz4ifdNL1WqtNas2ZTsPAsXLrP+7etbSwcP/p7s+0wmkyZOnGLTMgAAACCr4EwcAAAAABgIZ+IgScqfz1EOjk4Z3YxMIT42RleuxWZ0MwAAAIBkEeIgSXJwdNLeCa9ndDMyhWpDv5BEiAMAAEDmxHBKAAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADAQQhwAAAAAGAghDgAAAAAMhBAHAAAAAAZCiAMAAAAAAyHEAQAAAICBEOIAAAAAwEAIcQAAAABgIIQ4AAAAADCQdA1xp06dUseOHdW0aVN17NhRp0+fTs/FAQAAAECWl64hLjAwUF26dNHatWvVpUsXjR49Oj0XBwAAAABZXrqFuOjoaIWFhally5aSpJYtWyosLEyXL19Or0UCAAAAQJbnkF4VR0ZGyt3dXSaTSZJkMpnk5uamyMhIubq6PlQd9vZ2qZYXzJ8rze3MKh7UVw/DMW8BG7Qka0hrfxbM/XDf8ezAFt/Npwry3bzLFv2ZzyWnDVqSNdiiP/O68P28K639mSOPs41aYnw2+W7mzWuDlmQNtuhPp6dc0t6QLCKt/Zk/p5ONWmJ8qfVlamV2FovFkh4NOnLkiIYNG6YffvjBOq1FixaaOHGiypcvnx6LBAAAAIAsL92GU3p4eOjChQtKSEiQJCUkJCgqKkoeHh7ptUgAAAAAyPLSLcQVKFBAZcuWVUhIiCQpJCREZcuWfeihlAAAAACA+6XbcEpJOnHihAICAnT9+nXlzZtXQUFBKlGiRHotDgAAAACyvHQNcQAAAAAA20rX58QBAAAAAGyLEAcAAAAABkKIAwAAAAADIcQBAAAAgIEQ4gAAAADAQBwyugFPmp+fnxwdHeXo6Ciz2awePXro66+/liRdunRJZrNZbm5ukqRWrVppxowZ2r17txwcHGSxWPTcc8+pTZs2GjZsmCRpzZo1mj9/vr777jtJ0pIlSzRv3jyZzWaZzWa1bdtWvXr1kr29vRYuXKhFixYlac/58+dVrFgxLVmy5Ml1whPg5+enzz//XKVLl07xPQEBAapQoYL+7//+zzotKChIOXPmVL9+/fTbb79p+PDhMpvNio+PV9WqVTVq1Cg5Ojpq586dCgoK0vLly5/Ex8lUDh8+rK+//lqTJk3SqVOnNHr0aF28eFEODg6qWLGiAgMD5ezsrPDwcDVp0kSlSpWS2WxWXFycqlevrrfeekuFCxeWlPh/sH37duXPn99a/5tvvqlmzZpl1Md74vr06aPw8HDZ29srZ86cGjVqlMqWLWuTuu9ub5ycnKzTZsyYIS8vL5vUn1mFh4frxRdf1M6dO+8r69q1q7p3764GDRpYp/Xv31/169dXu3btkq2vTJky2rdvn3LlypVubc5IcXFxmjlzpn788Uc5OjrKZDLJ19dXderUUZ8+fVS8eHFJktls1ptvvqkWLVpkbIMNoEOHDoqNjVVcXJxOnz6tUqVKSZJu3LghFxeXh/7tmDp1qkqVKkWf/+PuutiqVasUt22nTp1SQECArl69KhcXFwUFBVm/w8ntG7Rr107Dhg2Tj4+Ppk+fru+++05ubm6KiYlR1apVNWbMGDk6Oj7pj5ppbNmyRb169dKnn36qxo0bS5Lq1aunadOmqXLlypISt6Hnzp3TsmXLJEmXL19WvXr1tHv3bu3atUsff/yx7OzsFB8fr0aNGmngwIGys7PLsM9kKw+zr/mwUtt/z6yyXYiTpGnTpql06dIKCwtTp06dtHnzZrm6umr69Om6deuWNaBJ0jfffKOjR4+qcuXKOn78uIoUKaI9e/ZYy3ft2qWaNWtKklauXKl58+Zpzpw58vT01LVr19S3b1+ZzWb16dNHnTt3VufOna3znjp1Sp07d06yPPzrmWee0eLFi62Be8CAAVq0aJFeeeWVjG5ahqpYsaImTZokScqRI4feffddlStXTmazWYMHD9bcuXPVt29fSVKePHkUHBwsSYqNjdVnn32mTp06afXq1cqTJ48kqVevXkmCdHYTFBRk7YsNGzZo+PDhWrFihc3qv7u9waOLj4+Xg0PW/5l69913FRMTo2XLlil37tyKj4/XsmXLFBsbq5IlS1oDx/Hjx9WhQwc1a9YsU+9YZAZLly6V9O8BhbvbwbsHAB9GQkKCBgwYkG5tNLqUtm2BgYHq0qWL/P39FRwcrNGjR2v+/PkPXe/dA+WxsbHq2rVrtv/dX7ZsmXx9fbVs2TJriKtZs6Z27txpDXFhYWHKmTOnbt68qdy5c2vXrl2qWLGinJ2dVa1aNa1YsUImk0lxcXHq3LmzKlWqpIYNG2bkx8pUHrT/nlll61+BcuXKKVeuXAoPD0/xPXdXFCkxsDVp0kRxcXG6efOmdZqPj48kafr06Ro2bJg8PT0lSfny5dPYsWM1a9YsxcbGJqn31q1b6tevn9566y1Vq1YtPT5eptC1a1cFBQWpc+fOatiwoT7++OOHntfZ2dl69C0+Pl537tzJdjsub7/9ttq1a6dWrVqpb9++unbtmnbu3Gk9Y+Hl5aVy5cpJkuzt7VWpUiVFREQkW5ejo6MGDBggd3d3rVq16ol9hoy0aNEijR07VpJ06NAhlSlTRocOHZIkjRkzRosXL7YGOEm6efOm9ejkw8wrJR6ZnjZtmvz9/dW0aVOtXbv2iX2+zODgwYPq2rWr2rVrp3bt2mnz5s1JymNjYzVw4EB99NFHepjHkgYEBGjEiBHq0qWLXnzxxXRqdeZx+vRpbdiwQe+//75y584tSXJwcFDHjh2VM2fOJO+9ceOGcufOLXt7ewUHB1sP1kiJ28jnn39ef/31l0aPHi1/f3/5+/urcePGqlGjhqTEvh09erReeeUVNWnSREOHDn2o/5OsJiEhQaNHj1arVq3UunVrnThxQlJiwGvVqpXeffdd+fv7a+vWrQoICNC3334rSapTp46io6MlST179lSvXr0kSdHR0apbt27GfJh0tm7dOjVr1kz+/v6aMWPGA98fHR2tsLAwtWzZUpLUsmVLhYWF6fLly4+8bEdHR1WrVk2nTp165Hkzk9u3b6t///5q0aKFWrdurQEDBmjw4MH66aefJElz5sxRtWrVlJCQIElq0aKF9TNfuXJFO3bs0KRJk7R//35dvHhRUuK+6a5duyQlHtwpVqyYatSoYT3JcO++aa5cuWQymSRJMTExiouLy9L7UidOnFD37t3VqlUrtWrVynpQNrX90YfZf//222/VpEkTvfjii5o2bZq1fzNS1j/EmYodO3YoJibGepo/OT4+PlqzZo169eqlXbt26dVXX1VkZKT27NmjSpUq6ezZs/L29tbNmzcVHh6uKlWqJJm/ZMmScnBw0OnTp5McsRo+fLjKlSuXLc6AREZGasGCBfr777/VqFEjtW/f3trns2fPth4xlaSoqCh16dLF+vrChQvq1auXzp49q3r16umll1560s3PUCNGjJCrq6skacqUKZozZ47q1KmT7Hvv3LmjZcuWafDgwanWWbFiRR07dsz6+r//Bx999JHNhhNmtFq1almHS4eGhsrb21s7duxQpUqVFBoaqu7du0tK7Odt27bJYrHoiy++eKR5JVl3qk+ePKnOnTurevXqKlCggKTEYS53hxyZTKYsNQT4+vXrCgwM1OzZs+Xm5qaoqCi1b99es2bNkiRdvXpV/fr1U+PGjR/pSPpvv/2mb7/99r4QkxWFhYWpWLFiypcvX7LlJ06ckL+/v2JjY3Xu3DlNmDBBktSkSRONHz9ely9flqurq7Zu3aoSJUqoaNGiGjdunKTEYZo9evRI8jtz7Ngxff3117Kzs1Pbtm21fft21a5dO/0/aCZy/Phxffjhhxo3bpw+++wzzZw50zq64fjx4xo3bpy8vb0lKclBGR8fH+3YsUNNmjRReHi47OzsFBcXp9DQ0EyxQ2drly5d0qhRo7Rw4UKVKFFCc+bMSVKe3LYtMjJS7u7u1tBgMpnk5uamyMhI62/ZvfNJiQcyknPjxg1t27bN8PtJv/76q/7++2/9+OOPkqRr165p3bp1Cg0NVfPmzbVjxw6VKlVKhw8flqenp27duqVnnnlGkrRq1SrVr19fBQsWVOPGjbVixQr16tVLvr6++vDDDxUfH6+dO3eqZs2aevrpp7Vz507Vr19fu3bt0ogRI6xtOHz4sIYPH64zZ86oc+fOql+/fkZ0RbqLj49Xnz59NHDgQDVv3lxSYhC+K7n90YIFCz5w/91sNmvWrFkKDg6Wq6ur3n///Sf5sVKULUPc3Q1I7ty5NX36dOXNmzfF9/r4+Gj8+PGKj4/X0aNHVbFiRUVGRmrnzp26c+eOKlWqJGdnZ+uZueT8d9zxl19+qdOnT993fVxWdXfoT548eVSyZEmdPXvWGuL+O5Tvv8Nc3N3dFRwcrFu3bumdd97R+vXr9cILLzzJ5meo4OBgrV69WnFxcbp165aKFy+ebIiLj4/XoEGD5Ovr+8hDJLLycMpixYopJiZG58+fV2hoqAYNGqTPP/9crVq1UlxcnJ5++mlJ0gcffCApcUjFhAkTNGfOnIeeV0q8/kaSSpQooXLlyunAgQPW/4esPJxy//79Cg8PV8+ePa3T7l53ERsbqy5duqhfv37WH9PU3LudbNasWbYIcA/j3uGUJ06cUNeuXeXt7S13d3c1atRIISEheuWVV7RixYr7rikcMWKESpcurW7dulmnNWrUyLoDXa5cOZ09ezbbhbhnnnnGOoKhSpUq2rRpk7WsWLFi1gD3X7Vq1dL27dvl7u6uKlWqyGKx6ODBg9q+fbt8fX2fSNufpIMHD6pcuXIqUaKEJKljx45Jzl487rbtv/P993u7cuVKbd++Xfb29qleK2sUzz77rE6cOKGxY8eqZs2aql+/vnx9fTV79mzFxsbq/Pnz6tGjh7Zv3y5PT88kBwSWL1+ugIAASVLbtm01YsQI9erVS0WLFlW+fPl09OhR7d69W926dVPRokU1d+5cXb58WWfPnlXVqlWt9VSsWFGrV6/W5cuX1b9/f+3Zs8d6hj4rOXXqlOLj45P85tx7zX9y+6MFCxZMsb67v0u7du1SvXr1rAci2rdvr9WrV6fTp3h4Wfd8aiqmTZum4OBgLViw4IE/XndXlNWrV+vpp5+Wg4ODatSoYb1Y9O71cLlz55aXl5cOHDiQZP4TJ04oLi5OxYoVk5Q4XGPOnDmaPn26nJ2d0+XzZTb3HnEzmUzWIQOPImfOnGrRokWmWGmelD179mjhwoX64osvtHr1ag0cOPC+YblS4tCgIUOGKF++fBo5cuQD6z18+LD1Iv/swNfXV5s2bVJ0dLR8fHx08eJFbd68Odkj523atNHOnTutR+4eZd7syGKxqEyZMgoODrb+27Jli/Lnz68cOXKocuXK2rhxY5J13tXVVVevXk1Sz5UrV6w/jpKyVYArV66czpw5o2vXrj3wvSVLllSRIkW0b98+SYk7dStXrtSVK1e0a9cuNW3a1PreqVOn6ubNmxo+fHiSOmyxPTa6e2+SYW9vr/j4eOvr1L57vr6+Cg0NVWhoqHx9feXr66sdO3Zox44dqlWrVrq22Sg8PDx04cIF6/cqISFBUVFR8vDweOg62rRpo+DgYK1YsUKDBg2yntUzqqJFiyokJES1a9dWaGio/P395ebmJrPZrB9++EFVqlRRrVq1FBoamuS7dOTIER0/flwjRoyQn5+f3nnnHYWHh2vv3r2S/r3c5+jRo6pQoYIKFiyoO3fuaOPGjapUqVKSdf0uV1dX1a1bV2vWrHmifZBZJLf9e9j998woW4a4R1WzZk19/vnn1sBWqFAh3bp1S1u2bEmyM/fWW29pwoQJioyMlJR4ynzMmDHq2bOnnJycdP78eb399tsKCgpS0aJFM+SzGMlff/1lDS2xsbH6+eefs+wZjeRcv35duXPnlouLi2JjY613nbqX2WxWQECATCaTPvjgg1TvNhUbG6tPP/1U58+fV+vWrdOz6ZmKr6+v5syZYz26XrVqVc2ZM0e1atXS33//bV1fJWnjxo3Kly+fXFxcHjjvve7+35w+fVphYWH3DcvIqry9vXXmzBnt2LHDOu3QoUOyWCyys7PT+PHjlTt3bg0aNEhxcXGSpNq1a2vlypWKiYmRJP3+++86ceKEKlWqlCGfIaMVL15cfn5+Gj16tHVER0JCgpYuXapbt24lee+FCxd0+vRp60iG6tWr6+bNm5o8ebIaNWqkp556SlLi0ftff/1VkyZNytLXvjxpRYoUkclk0ooVK1SrVi3VqlVLy5cvl4ODg/VamqykSpUqCgsLsw53vHfYfUoKFCigsmXLKiQkRJIUEhKismXLJjlIk92cP39eJpNJjRo10rvvvqvLly/r6tWr8vX11fTp0/Xcc8/Jw8NDV69e1a+//mr9fVm2bJlef/11bdy40fqvX79+1t+bmjVratmyZSpatKhy5MghKXGbPGfOnCT7pqdOnZLZbJaUeD+GrVu3Ztl9qWeeeUYODg7W6w2lpMMpU/Kg/feaNWtq69at1ms7bXnzs7TIlsMpH5WPj49WrlyZ5NRz1apVFRwcnGRnrW3btrpz54569Oghi8WihIQE+fv7680335QkzZw5Uzdu3NCkSZOs4++lxItO7z6iAP/at2+fvvjiC9nZ2clsNqtGjRrWuwQlJCQke5QpK6lTp45WrVqlpk2bKn/+/KpevboOHz4s6d9T/Fu3btWqVatUunRp65CTqlWrKjAwUFLiNQX+/v5KSEiwPmJg0aJFSW7m8d9r4jp16pTkLqpG5+vrq6FDh1p/GH19fbV48WL5+vrq9u3bGjBggG7fvi17e3vly5dPn3/+ubV/U5v3XgkJCWrTpo1u376tcePGWa+Hk+6//uP9999XxYoV0/tjPxH58uXTzJkzNXHiRI0fP15xcXEqWrSoRo0aJSnxexoYGKigoCD17dtX06dPV/v27RUREaH27dvL3t5ezs7OmjJlSpIhL9nNRx99pBkzZujFF19Ujhw5ZDabVa9ePXl6elqvibNYLIqPj9eAAQOSXLPapk0bTZ06VQsWLLBO+/TTTyUlrssSvzG2VKtWLe3du9f6KCJnZ2dVr149g1uVPgoUKKD33ntPvXv3lrOzs5o0aZKkPKVt25gxYxQQEKCZM2cqb968D3030Kzqjz/+sO7zmc1m9erVS+7u7qpVq5b1zpOSVK1aNYWGhsrd3V0xMTH64YcftHDhwiR1tWzZUq1bt9bIkSPl4+Oj4cOHq02bNtbyGjVqaMmSJdaTDpL0888/W+9OaTab1ahRI+slAFnBa6+9luRs7dSpU/XJJ59o5syZsrOzU/fu3ZP0UXIetP/+7LPP6vXXX1enTp2UO3du+fr6JtmPyih2lux4ayoY3hdffKETJ07oww8/zOimPHE//vijli1bprlz52Z0U6Cs/wwzAACyu7uPb5AS72Z55syZR7rjenrgTBwMp1u3brp165b1Lm3ZyaJFi/Tll19q9OjRGd0UAACAbGHSpEnat2+fdcTJ3bsAZyTOxAEAAACAgXDFMwAAAAAYCCEOAAAAAAyEEAcAAAAABkKIAwAAAAADIcQBAAAAgIEQ4gAAAADAQP4fP5kyMgy0kxMAAAAASUVORK5CYII=\n" - }, - "metadata": {} - } - ], - "source": [ - "import seaborn as sns\n", - "import matplotlib.pyplot as plt\n", - "sns.set(style=\"whitegrid\")\n", - "\n", - "jobs = exp.list_trial_jobs()\n", - "job_ids = [x.trialJobId for x in jobs]\n", - "final_metrics = [float(x.finalMetricData[0].data) for x in jobs]\n", - "\n", - "data = {'job id': job_ids, 'final metrics': final_metrics}\n", - "sns.set(rc={'figure.figsize':(15, 6)})\n", - "\n", - "plt.title('Trial job final results')\n", - "ax = sns.barplot(x='job id', y='final metrics', data=data) \n", - "\n", - "for i,p in enumerate(ax.patches):\n", - " ax.annotate('{:.4f}'.format(p.get_height()), (p.get_x() + p.get_width() / 2., p.get_height()),\n", - " ha='center', va='center', fontsize=11, color='black', rotation=0, xytext=(0, 5),\n", - " textcoords='offset points') " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Stop nni experiment" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "tags": [] - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": "INFO: Stopping experiment OhHNEkLQ\nINFO: Stop experiment success.\n" - } - ], - "source": [ - "exp.stop_experiment()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.5-final" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} \ No newline at end of file diff --git a/examples/notebooks/tabular_data_classification_in_AML.ipynb b/examples/notebooks/tabular_data_classification_in_AML.ipynb new file mode 100644 index 0000000000..c37d8aa755 --- /dev/null +++ b/examples/notebooks/tabular_data_classification_in_AML.ipynb @@ -0,0 +1,396 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tabular Data Classification with NNI in AML" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This simple example is to use NNI NAS 2.0(Retiarii) framework to search for the best neural architecture for tabular data classification task in Azure Machine Learning training platform.\n", + "\n", + "The video demo is https://www.youtube.com/watch?v=PDVqBmm7Cro and https://www.bilibili.com/video/BV1oy4y1W7GF." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Prepare the dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The first step is to prepare the dataset. Here we use the Titanic dataset as an example." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import torch\n", + "import pandas as pd\n", + "\n", + "from sklearn.preprocessing import LabelEncoder\n", + "from torchvision.datasets.utils import download_url\n", + "\n", + "class TitanicDataset(torch.utils.data.Dataset):\n", + " def __init__(self, root: str, train: bool = True):\n", + " filename = 'train.csv' if train else 'eval.csv'\n", + " if not os.path.exists(os.path.join(root, filename)):\n", + " download_url(os.path.join(\n", + " 'https://storage.googleapis.com/tf-datasets/titanic/', filename), root, filename)\n", + "\n", + " df = pd.read_csv(os.path.join(root, filename))\n", + " object_colunmns = df.select_dtypes(include='object').columns.values\n", + " for idx in df.columns:\n", + " if idx in object_colunmns:\n", + " df[idx] = LabelEncoder().fit_transform(df[idx])\n", + " \n", + " self.x = torch.tensor(df.iloc[:, 1:].values)\n", + " self.y = torch.tensor(df.iloc[:, 0].values)\n", + "\n", + " def __len__(self):\n", + " return len(self.y)\n", + "\n", + " def __getitem__(self, idx):\n", + " return self.x[idx], self.y[idx]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "train_dataset = TitanicDataset('./data', train=True)\n", + "test_dataset = TitanicDataset('./data', train=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Define the Model Space" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Model space is defined by users to express a set of models that they want to explore, which contains potentially good-performing models. In Retiarii(NNI NAS 2.0) framework, a model space is defined with two parts: a base model and possible mutations on the base model." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 2.1: Define the Base Model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Defining a base model is almost the same as defining a PyTorch (or TensorFlow) model. Usually, you only need to replace the code ``import torch.nn as nn`` with ``import nni.retiarii.nn.pytorch as nn`` to use NNI wrapped PyTorch modules. Below is a very simple example of defining a base model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import nni.retiarii.nn.pytorch as nn\n", + "import torch.nn.functional as F\n", + "\n", + "class Net(nn.Module):\n", + "\n", + " def __init__(self, input_size):\n", + " super().__init__()\n", + "\n", + " self.fc1 = nn.Linear(input_size, 16)\n", + " self.bn1 = nn.BatchNorm1d(16)\n", + " self.dropout1 = nn.Dropout(0.0)\n", + "\n", + " self.fc2 = nn.Linear(16, 16)\n", + " self.bn2 = nn.BatchNorm1d(16)\n", + " self.dropout2 = nn.Dropout(0.0)\n", + "\n", + " self.fc3 = nn.Linear(16, 2)\n", + "\n", + " def forward(self, x):\n", + "\n", + " x = self.dropout1(F.relu(self.bn1(self.fc1(x))))\n", + " x = self.dropout2(F.relu(self.bn2(self.fc2(x))))\n", + " x = F.sigmoid(self.fc3(x))\n", + " return x\n", + " \n", + "model_space = Net(len(train_dataset.__getitem__(0)[0]))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 2.2: Define the Model Mutations" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A base model is only one concrete model, not a model space. NNI provides APIs and primitives for users to express how the base model can be mutated, i.e., a model space that includes many models. The following will use inline Mutation APIs as a simple example. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import nni.retiarii.nn.pytorch as nn\n", + "import torch.nn.functional as F\n", + "\n", + "class Net(nn.Module):\n", + "\n", + " def __init__(self, input_size):\n", + " super().__init__()\n", + "\n", + " self.hidden_dim1 = nn.ValueChoice(\n", + " [16, 32, 64, 128, 256, 512, 1024], label='hidden_dim1')\n", + " self.hidden_dim2 = nn.ValueChoice(\n", + " [16, 32, 64, 128, 256, 512, 1024], label='hidden_dim2')\n", + "\n", + " self.fc1 = nn.Linear(input_size, self.hidden_dim1)\n", + " self.bn1 = nn.BatchNorm1d(self.hidden_dim1)\n", + " self.dropout1 = nn.Dropout(nn.ValueChoice([0.0, 0.25, 0.5]))\n", + "\n", + " self.fc2 = nn.Linear(self.hidden_dim1, self.hidden_dim2)\n", + " self.bn2 = nn.BatchNorm1d(self.hidden_dim2)\n", + " self.dropout2 = nn.Dropout(nn.ValueChoice([0.0, 0.25, 0.5]))\n", + "\n", + " self.fc3 = nn.Linear(self.hidden_dim2, 2)\n", + "\n", + " def forward(self, x):\n", + "\n", + " x = self.dropout1(F.relu(self.bn1(self.fc1(x))))\n", + " x = self.dropout2(F.relu(self.bn2(self.fc2(x))))\n", + " x = F.sigmoid(self.fc3(x))\n", + " return x\n", + "\n", + "model_space = Net(len(train_dataset.__getitem__(0)[0]))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Besides inline mutations, Retiarii also provides ``mutator``, a more general approach to express complex model space." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Explore the Defined Model Space" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the NAS process, the search strategy repeatedly generates new models, and the model evaluator is for training and validating each generated model. The obtained performance of a generated model is collected and sent to the search strategy for generating better models.\n", + "\n", + "Users can choose a proper search strategy to explore the model space, and use a chosen or user-defined model evaluator to evaluate the performance of each sampled model." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 3.1: Choose a Search Strategy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import nni.retiarii.strategy as strategy\n", + "\n", + "simple_strategy = strategy.TPEStrategy()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 3.2: Choose or Write a Model Evaluator" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the context of PyTorch, Retiarii has provided two built-in model evaluators, designed for simple use cases: classification and regression. These two evaluators are built upon the awesome library PyTorch-Lightning." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import nni.retiarii.evaluator.pytorch.lightning as pl\n", + "\n", + "trainer = pl.Classification(train_dataloader=pl.DataLoader(train_dataset, batch_size=16),\n", + " val_dataloaders=pl.DataLoader(\n", + " test_dataset, batch_size=16),\n", + " max_epochs=20)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Configure the Experiment" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After all the above are prepared, it is time to configure an experiment to do the model search. The basic experiment configuration is as follows: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from nni.retiarii.experiment.pytorch import RetiariiExeConfig, RetiariiExperiment\n", + "\n", + "exp = RetiariiExperiment(model_space, trainer, [], simple_strategy)\n", + "\n", + "exp_config = RetiariiExeConfig('aml')\n", + "exp_config.experiment_name = 'titanic_example'\n", + "exp_config.trial_concurrency = 2\n", + "exp_config.max_trial_number = 20\n", + "exp_config.max_experiment_duration = '2h'\n", + "exp_config.trial_gpu_number = 1\n", + "exp_config.nni_manager_ip = '' # your nni_manager_ip" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Running NNI experiments on the AML(Azure Machine Learning) training service is also simple, you only need to configure the following additional fields:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "exp_config.training_service.use_active_gpu = True\n", + "exp_config.training_service.subscription_id = '' # your subscription id\n", + "exp_config.training_service.resource_group = '' # your resource group\n", + "exp_config.training_service.workspace_name = '' # your workspace name\n", + "exp_config.training_service.compute_target = '' # your compute target\n", + "exp_config.training_service.docker_image = '' # your docker image" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Run and View the Experiment" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can launch the experiment now! \n", + "\n", + "Besides, NNI provides WebUI to help users view the experiment results and make more advanced analysis." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "exp.run(exp_config, 8081 + random.randint(0, 100))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Export the top Model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Exporting the top model script is also very convenient." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print('Final model:')\n", + "for model_code in exp.export_top_models():\n", + " print(model_code)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.8" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts b/ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts index 4e88c588a5..828fa086b6 100644 --- a/ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts +++ b/ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts @@ -66,7 +66,6 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple await this.fcJobInfoCollector.retrieveTrialStatus(this.kubernetesCRDClient); if (this.kubernetesJobRestServer.getErrorMessage !== undefined) { throw new Error(this.kubernetesJobRestServer.getErrorMessage); - this.stopping = true; } } } diff --git a/ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts b/ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts index 67f8fd4a06..e54c6f3f04 100644 --- a/ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts +++ b/ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts @@ -60,7 +60,6 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber await this.kubeflowJobInfoCollector.retrieveTrialStatus(this.kubernetesCRDClient); if (this.kubernetesJobRestServer.getErrorMessage !== undefined) { throw new Error(this.kubernetesJobRestServer.getErrorMessage); - this.stopping = true; } } this.log.info('Kubeflow training service exit.');