From 1bb37595ff0ed772231ca669b55b6581489f807f Mon Sep 17 00:00:00 2001 From: Mahdi Date: Sun, 14 Apr 2024 12:45:18 -0400 Subject: [PATCH 01/29] add FedPFT baseline --- baselines/fedpft/.gitignore | 3 + baselines/fedpft/EXTENDED_README.md | 123 ++++++++++ baselines/fedpft/LICENSE | 202 ++++++++++++++++ baselines/fedpft/README.md | 109 +++++++++ baselines/fedpft/fedpft/__init__.py | 1 + baselines/fedpft/fedpft/client.py | 170 ++++++++++++++ baselines/fedpft/fedpft/conf/base.yaml | 15 ++ .../fedpft/fedpft/conf/client/fedavg.yaml | 2 + .../fedpft/fedpft/conf/client/fedpft.yaml | 2 + .../fedpft/fedpft/conf/dataset/CIFAR100.yaml | 10 + .../fedpft/conf/dataset/Caltech101.yaml | 10 + baselines/fedpft/fedpft/conf/model/clip.yaml | 9 + .../fedpft/fedpft/conf/model/resnet50.yaml | 8 + .../fedpft/fedpft/conf/strategy/fedavg.yaml | 12 + .../fedpft/fedpft/conf/strategy/fedpft.yaml | 22 ++ baselines/fedpft/fedpft/dataset.py | 108 +++++++++ .../fedpft/fedpft/dataset_preparation.py | 1 + baselines/fedpft/fedpft/main.py | 88 +++++++ baselines/fedpft/fedpft/models.py | 221 ++++++++++++++++++ baselines/fedpft/fedpft/server.py | 94 ++++++++ baselines/fedpft/fedpft/strategy.py | 145 ++++++++++++ baselines/fedpft/fedpft/utils.py | 102 ++++++++ baselines/fedpft/pyproject.toml | 143 ++++++++++++ 23 files changed, 1600 insertions(+) create mode 100644 baselines/fedpft/.gitignore create mode 100644 baselines/fedpft/EXTENDED_README.md create mode 100644 baselines/fedpft/LICENSE create mode 100644 baselines/fedpft/README.md create mode 100644 baselines/fedpft/fedpft/__init__.py create mode 100644 baselines/fedpft/fedpft/client.py create mode 100644 baselines/fedpft/fedpft/conf/base.yaml create mode 100644 baselines/fedpft/fedpft/conf/client/fedavg.yaml create mode 100644 baselines/fedpft/fedpft/conf/client/fedpft.yaml create mode 100644 baselines/fedpft/fedpft/conf/dataset/CIFAR100.yaml create mode 100644 baselines/fedpft/fedpft/conf/dataset/Caltech101.yaml create mode 100644 baselines/fedpft/fedpft/conf/model/clip.yaml create mode 100644 baselines/fedpft/fedpft/conf/model/resnet50.yaml create mode 100644 baselines/fedpft/fedpft/conf/strategy/fedavg.yaml create mode 100644 baselines/fedpft/fedpft/conf/strategy/fedpft.yaml create mode 100644 baselines/fedpft/fedpft/dataset.py create mode 100644 baselines/fedpft/fedpft/dataset_preparation.py create mode 100644 baselines/fedpft/fedpft/main.py create mode 100644 baselines/fedpft/fedpft/models.py create mode 100644 baselines/fedpft/fedpft/server.py create mode 100644 baselines/fedpft/fedpft/strategy.py create mode 100644 baselines/fedpft/fedpft/utils.py create mode 100644 baselines/fedpft/pyproject.toml diff --git a/baselines/fedpft/.gitignore b/baselines/fedpft/.gitignore new file mode 100644 index 00000000000..4ab8207aedb --- /dev/null +++ b/baselines/fedpft/.gitignore @@ -0,0 +1,3 @@ +outputs/ +multirun/ +.ruff_cache/ \ No newline at end of file diff --git a/baselines/fedpft/EXTENDED_README.md b/baselines/fedpft/EXTENDED_README.md new file mode 100644 index 00000000000..9c8f5bc72fa --- /dev/null +++ b/baselines/fedpft/EXTENDED_README.md @@ -0,0 +1,123 @@ + +# Extended Readme + +> The baselines are expected to run in a machine running Ubuntu 22.04 + +While `README.md` should include information about the baseline you implement and how to run it, this _extended_ readme provides info on what's the expected directory structure for a new baseline and more generally the instructions to follow before your baseline can be merged into the Flower repository. Please follow closely these instructions. It is likely that you have already completed steps 1-2. + +1. Fork the Flower repository and clone it. +2. Navigate to the `baselines/` directory and from there run: + ```bash + # This will create a new directory with the same structure as this `baseline_template` directory. + ./dev/create-baseline.sh + ``` +3. All your code and configs should go into a sub-directory with the same name as the name of your baseline. + * The sub-directory contains a series of Python scripts that you can edit. Please stick to these files and consult with us if you need additional ones. + * There is also a basic config structure in `/conf` ready be parsed by [Hydra](https://hydra.cc/) when executing your `main.py`. +4. Therefore, the directory structure in your baseline should look like: + ```bash + baselines/ + ├── README.md # describes your baseline and everything needed to use it + ├── EXTENDED_README.md # to remove before creating your PR + ├── pyproject.toml # details your Python environment + └── + ├── *.py # several .py files including main.py and __init__.py + └── conf + └── *.yaml # one or more Hydra config files + + ``` +> :warning: Make sure the variable `name` in `pyproject.toml` is set to the name of the sub-directory containing all your code. + +5. Add your dependencies to the `pyproject.toml` (see below a few examples on how to do it). Read more about Poetry below in this `EXTENDED_README.md`. +6. Regularly check that your coding style and the documentation you add follow good coding practices. To test whether your code meets the requirements, please run the following: + ```bash + # After activating your environment and from your baseline's directory + cd .. # to go to the top-level directory of all baselines + ./dev/test-baseline.sh + ./dev/test-baseline-structure.sh + ``` + Both `test-baseline.sh` and `test-baseline-structure.sh` will also be automatically run when you create a PR, and both tests need to pass for the baseline to be merged. + To automatically solve some formatting issues and apply easy fixes, please run the formatting script: + ```bash + # After activating your environment and from your baseline's directory + cd .. # to go to the top-level directory of all baselines + ./dev/format-baseline.sh + ``` +7. Ensure that the Python environment for your baseline can be created without errors by simply running `poetry install` and that this is properly described later when you complete the `Environment Setup` section in `README.md`. This is specially important if your environment requires additional steps after doing `poetry install`. +8. Ensure that your baseline runs with default arguments by running `poetry run python -m .main`. Then, describe this and other forms of running your code in the `Running the Experiments` section in `README.md`. +9. Once your code is ready and you have checked: + * that following the instructions in your `README.md` the Python environment can be created correctly + + * that running the code following your instructions can reproduce the experiments in the paper + + , then you just need to create a Pull Request (PR) to kickstart the process of merging your baseline into the Flower repository. + +> Once you are happy to merge your baseline contribution, please delete this `EXTENDED_README.md` file. + + +## About Poetry + +We use Poetry to manage the Python environment for each individual baseline. You can follow the instructions [here](https://python-poetry.org/docs/) to install Poetry in your machine. + + +### Specifying a Python Version (optional) +By default, Poetry will use the Python version in your system. In some settings, you might want to specify a particular version of Python to use inside your Poetry environment. You can do so with [`pyenv`](https://github.com/pyenv/pyenv). Check the documentation for the different ways of installing `pyenv`, but one easy way is using the [automatic installer](https://github.com/pyenv/pyenv-installer): +```bash +curl https://pyenv.run | bash # then, don't forget links to your .bashrc/.zshrc +``` + +You can then install any Python version with `pyenv install ` (e.g. `pyenv install 3.9.17`). Then, in order to use that version for your baseline, you'd do the following: + +```bash +# cd to your baseline directory (i.e. where the `pyproject.toml` is) +pyenv local + +# set that version for poetry +poetry env use + +# then you can install your Poetry environment (see the next setp) +``` + +### Installing Your Environment +With the Poetry tool already installed, you can create an environment for this baseline with commands: +```bash +# run this from the same directory as the `pyproject.toml` file is +poetry install +``` + +This will create a basic Python environment with just Flower and additional packages, including those needed for simulation. Next, you should add the dependencies for your code. It is **critical** that you fix the version of the packages you use using a `=` not a `=^`. You can do so via [`poetry add`](https://python-poetry.org/docs/cli/#add). Below are some examples: + +```bash +# For instance, if you want to install tqdm +poetry add tqdm==4.65.0 + +# If you already have a requirements.txt, you can add all those packages (but ensure you have fixed the version) in one go as follows: +poetry add $( cat requirements.txt ) +``` +With each `poetry add` command, the `pyproject.toml` gets automatically updated so you don't need to keep that `requirements.txt` as part of this baseline. + + +More critically however, is adding your ML framework of choice to the list of dependencies. For some frameworks you might be able to do so with the `poetry add` command. Check [the Poetry documentation](https://python-poetry.org/docs/cli/#add) for how to add packages in various ways. For instance, let's say you want to use PyTorch: + +```bash +# with plain `pip` you'd run a command such as: +pip install torch==1.13.1+cu117 torchvision==0.14.1+cu117 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu117 + +# to add the same 3 dependencies to your Poetry environment you'd need to add the URL to the wheel that the above pip command auto-resolves for you. +# You can find those wheels in `https://download.pytorch.org/whl/cu117`. Copy the link and paste it after the `poetry add` command. +# For instance to add `torch==1.13.1+cu117` and a x86 Linux system with Python3.8 you'd: +poetry add https://download.pytorch.org/whl/cu117/torch-1.13.1%2Bcu117-cp38-cp38-linux_x86_64.whl +# you'll need to repeat this for both `torchvision` and `torchaudio` +``` +The above is just an example of how you can add these dependencies. Please refer to the Poetry documentation to extra reference. + +If all attempts fail, you can still install packages via standard `pip`. You'd first need to source/activate your Poetry environment. +```bash +# first ensure you have created your environment +# and installed the base packages provided in the template +poetry install + +# then activate it +poetry shell +``` +Now you are inside your environment (pretty much as when you use `virtualenv` or `conda`) so you can install further packages with `pip`. Please note that, unlike with `poetry add`, these extra requirements won't be captured by `pyproject.toml`. Therefore, please ensure that you provide all instructions needed to: (1) create the base environment with Poetry and (2) install any additional dependencies via `pip` when you complete your `README.md`. \ No newline at end of file diff --git a/baselines/fedpft/LICENSE b/baselines/fedpft/LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/baselines/fedpft/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/baselines/fedpft/README.md b/baselines/fedpft/README.md new file mode 100644 index 00000000000..3f045810835 --- /dev/null +++ b/baselines/fedpft/README.md @@ -0,0 +1,109 @@ +--- +title: Parametric Feature Transfer, One-shot Federated Learning with Foundation Models +url: https://arxiv.org/abs/2402.01862 +labels: [foundation-models, pre-trained, one-shot, one-round] # please add between 4 and 10 single-word (maybe two-words) labels (e.g. system heterogeneity, image classification, asynchronous, weight sharing, cross-silo). Do not use "" +dataset: [CIFAR100, Caltech101] # list of datasets you include in your baseline. Do not use "" +--- + +# FedPFT: One-shot Federated Learning with Foundation Models + +> Note: If you use this baseline in your work, please remember to cite the original authors of the paper as well as the Flower paper. + +**Paper:** [arxiv.org/abs/2402.01862](https://arxiv.org/abs/2402.01862) + +**Authors:** Mahdi Beitollahi, Alex Bie, Sobhan Hemati, Leo Maxime Brunswic, Xu Li, Xi Chen, Guojun Zhang. + +**Abstract:** In one-shot federated learning (FL), clients collaboratively train a global model in a single round of communication. Existing approaches for one-shot FL enhance communication efficiency at the expense of diminished accuracy. This paper introduces FedPFT (Federated Learning with Parametric Feature Transfer), a methodology that harnesses the transferability of foundation models to enhance both accuracy and communication efficiency in one-shot FL. The approach involves transferring per-client parametric models (specifically, Gaussian mixtures) of features extracted from foundation models. Subsequently, each parametric model is employed to generate synthetic features for training a classifier head. Experimental results on eight datasets demonstrate that FedPFT enhances the communication-accuracy frontier in both centralized and decentralized FL scenarios, as well as across diverse data-heterogeneity settings such as covariate shift and task shift, with improvements of up to 20.6%. Additionally, FedPFT adheres to the data minimization principle of FL, as clients do not send real features. We demonstrate that sending real features is vulnerable to potent reconstruction attacks. Moreover, we show that FedPFT is amenable to formal privacy guarantees via differential privacy, demonstrating favourable privacy-accuracy tradeoffs. + + +## About this baseline + +**What’s implemented:** The code in this directory replicates the centralized experiments in *Parametric Feature Transfer, One-shot Federated Learning with Foundation Models* (Beitollahi et al., 2024) for CIFAR100 and Caltech101 datasets, which proposed the FedPFT algorithm. Concretely, it replicates the results in Section 5.2. + +**Datasets:** CIFAR100 and Caltech101 from HuggingFace + +**Hardware Setup:** These experiments were run on a desktop machine with 8 CPU threads and Nvidia 4070 with 8Gigs of ram. + +**Contributors:** Mahdi Beitollahi + + +## Experimental Setup + +**Task:** Image classification + +**Model:** This directory utilize two pre-trained, frozen models as shown in Table 1 of the paper: +* ResNet50 pre-trained on ImageNet is used for CIFAR100 dataset(see `models/resnet50`). +* CLIP, ViT-B/32 pre-trained on web dataset is used for Caltech101 dataset (see `models/clip_vit`) + +**Dataset:** This baseline includes the CIFAR100 and Caltech101 datasets. By default, it will be partitioned into 50 clients following a Dirichlet distribution with $\alpha$=0.1. + +| Dataset | #classes | #partitions | partitioning method | partition settings | +| :------ | :---: | :---: | :---: | :---: | +| CIFAR100 | 100 | 50 | Dirichlet distribution | $\alpha$=0.1 | +| Caltech101 | 101 | 50 | Dirichlet distribution | $\alpha$=0.1 | + +**Training Hyperparameters:** The following table shows the main hyperparameters for this baseline with their default value (i.e. the value used if you run `python main.py` directly) + +| Description | Default Value | +| ----------- | ----- | +| total clients | 50 | +| clients per round | 50 | +| number of rounds | 1 | +| client resources | {'num_cpus': 2.0, 'num_gpus': 0.0 }| +| data partition | distribution with $\alpha$=0.1 | +| Number of mixtures | 2 | +| Covariance type | spherical | +| tolerance | 1e-12 | +| maximum GMM iterations | 1e3 | + + +## Environment Setup + +To construct the Python environment, simply run: + +```bash +# Set directory to use python 3.10 (install with `pyenv install ` if you don't have it) +pyenv local 3.10.12 + +# Tell poetry to use python3.10 +poetry env use 3.10.12 + +# Install +poetry install +``` + + +## Running the Experiments + +To run this FedProx with CIFAR100 baseline, first ensure you have activated your Poetry environment (execute `poetry shell` from this directory), then: + +```bash +python -m fedpft.main # this will run using the default settings in the `conf/config.yaml` + +# you can override settings directly from the command line +python -m fedprox.main dataset=Caltech101 model=clip # will set dataset to Caltech101 and the pre-trained model to Clip-ViT/B32 +``` + +To run using FedAvg: +```bash +# this will use a frozen, pre-trained model and train the classifier head +python -m fedpft.main strategy=FedAvg client=FedAvg + +``` + + +## Expected Results + + +With the following command, we run both FedPFT and FedAvg configurations. + +```bash +python -m fedprox.main --multirun dataset=CIFAR100, Caltech101 + +# FedAvg +python -m fedprox.main --multirun strategy=fedavg client=fedavg dataset=CIFAR100, Caltech101 +``` + +The above commands would generate results that you can plot and would look like the plot shown below. This plot was generated using the jupyter notebook in the `docs/` directory of this baseline after running the `--multirun` commands above. + +![](_static/FedProx_mnist.png) \ No newline at end of file diff --git a/baselines/fedpft/fedpft/__init__.py b/baselines/fedpft/fedpft/__init__.py new file mode 100644 index 00000000000..a5e567b5913 --- /dev/null +++ b/baselines/fedpft/fedpft/__init__.py @@ -0,0 +1 @@ +"""Template baseline package.""" diff --git a/baselines/fedpft/fedpft/client.py b/baselines/fedpft/fedpft/client.py new file mode 100644 index 00000000000..434055808f8 --- /dev/null +++ b/baselines/fedpft/fedpft/client.py @@ -0,0 +1,170 @@ +"""Define your client class and a function to construct such clients. + +Please overwrite `flwr.client.NumPyClient` or `flwr.client.Client` and create a function +to instantiate your client. +""" + +from collections import OrderedDict +from typing import Callable, Dict, List, Tuple + +import flwr as fl +import torch +from flwr.common.typing import NDArrays, Scalar +from hydra.utils import instantiate +from omegaconf import DictConfig +from torch import nn +from torch.utils.data import DataLoader + +from fedpft.models import extract_features, test, train +from fedpft.utils import gmmparam_to_ndarrays, learn_gmm + + +class FedPFTClient(fl.client.NumPyClient): + """Flower FedPFTClient.""" + + def __init__( + self, + trainloader: DataLoader, + testloader: DataLoader, + feature_extractor: torch.nn.Module, + num_classes: int, + device: torch.device, + ) -> None: + """FedPFT client strategy. + + Implementation based on https://arxiv.org/abs/2402.01862 + + Parameters + ---------- + trainloader : DataLoader + Dataset used for learning GMMs + testloader : DataLoader + Dataset used for evaluating `classifier_head` sent from the server + feature_extractor : torch.nn.Module + Model used to extract features of each client + num_classes : int + Number of total classes in the dataset + device : torch.device + Device used to extract features and evaluate `classifier_head` + """ + self.trainloader = trainloader + self.testloader = testloader + self.feature_extractor = feature_extractor + self.classifier_head = nn.Linear( + feature_extractor.hidden_dimension, num_classes + ) + self.device = device + + def get_parameters(self, config) -> NDArrays: + """Return the parameters of the `classifier_head`.""" + return [ + val.cpu().numpy() for _, val in self.classifier_head.state_dict().items() + ] + + def set_parameters(self, parameters: NDArrays) -> None: + """Set the parameters of the `classifier_head`.""" + params_dict = zip(self.classifier_head.state_dict().keys(), parameters) + state_dict = OrderedDict({k: torch.tensor(v) for k, v in params_dict}) + self.classifier_head.load_state_dict(state_dict, strict=True) + + def fit( + self, parameters: NDArrays, config: Dict[str, Scalar] + ) -> Tuple[NDArrays, int, Dict]: + """Fit a GMM on features and return GMM parameters.""" + # Extracting features + features, labels = extract_features( + dataloader=self.trainloader, + feature_extractor=self.feature_extractor, + device=self.device, + ) + + # Learning GMM + gmm_list = learn_gmm( + features=features, + labels=labels, + n_mixtures=int(config["n_mixtures"]), + cov_type=config["cov_type"], + seed=int(config["seed"]), + tol=float(config["tol"]), + max_iter=int(config["max_iter"]), + ) + + # Reshaping GMM parameters into an NDArray + return [array for gmm in gmm_list for array in gmmparam_to_ndarrays(gmm)], 0, {} + + def evaluate( + self, parameters: NDArrays, config: Dict[str, Scalar] + ) -> Tuple[float, int, Dict]: + """Evaluate `classifier_head` on the test data.""" + self.set_parameters(parameters) + loss, acc = test( + classifier_head=self.classifier_head, + dataloader=self.testloader, + feature_extractor=self.feature_extractor, + device=self.device, + ) + return loss, len(self.testloader.dataset), {"accuracy": acc} + + +class FedAvgClient(FedPFTClient): + """Flower FedAvgClient.""" + + def fit( + self, parameters: NDArrays, config: Dict[str, Scalar] + ) -> Tuple[NDArrays, int, Dict]: + """Train the classifier head.""" + self.set_parameters(parameters) + + # train classifier head + opt = torch.optim.AdamW( + params=self.classifier_head.parameters(), lr=float(config["lr"]) + ) + train( + classifier_head=self.classifier_head, + dataloader=self.trainloader, + feature_extractor=self.feature_extractor, + device=self.device, + num_epochs=int(config["num_epochs"]), + opt=opt, + ) + return self.get_parameters(config={}), len(self.trainloader.dataset), {} + + +def generate_client_fn( + client_cfg: DictConfig, + trainloaders: List[DataLoader], + testloaders: List[DataLoader], + feature_extractor: torch.nn.Module, + num_classes: int, + device: torch.device, +) -> Callable[[str], fl.client.NumPyClient]: + """Generate the client function that creates the Flower Clients. + + Parameters + ---------- + client_cfg : DictConfig + Type of client + trainloaders : List[DataLoader] + List of train dataloaders for clients + testloaders : List[DataLoader] + List of test dataloaders for clients + feature_extractor : torch.nn.Module + Pre-trained model as the backbone + num_classes : int + Number of classes in the dataset + device : torch.device + Device to load the `feature_extractor` + """ + + def client_fn(cid: str) -> fl.client.NumPyClient: + """Create a FedPFT client.""" + return instantiate( + client_cfg, + trainloader=trainloaders[int(cid)], + testloader=testloaders[int(cid)], + feature_extractor=feature_extractor, + num_classes=num_classes, + device=device, + ) + + return client_fn diff --git a/baselines/fedpft/fedpft/conf/base.yaml b/baselines/fedpft/fedpft/conf/base.yaml new file mode 100644 index 00000000000..ab1477bd696 --- /dev/null +++ b/baselines/fedpft/fedpft/conf/base.yaml @@ -0,0 +1,15 @@ +--- + +num_clients: 2 +dirichlet_alpha: 0.1 +num_rounds: 1 +num_cpus: 1 +num_gpus: 0.1 +batch_size: 64 +device: cuda + +defaults: + - strategy: fedpft + - client: fedpft + - model: resnet50 + - dataset: CIFAR100 diff --git a/baselines/fedpft/fedpft/conf/client/fedavg.yaml b/baselines/fedpft/fedpft/conf/client/fedavg.yaml new file mode 100644 index 00000000000..10fc2b0f922 --- /dev/null +++ b/baselines/fedpft/fedpft/conf/client/fedavg.yaml @@ -0,0 +1,2 @@ +--- +_target_: fedpft.client.FedAvgClient \ No newline at end of file diff --git a/baselines/fedpft/fedpft/conf/client/fedpft.yaml b/baselines/fedpft/fedpft/conf/client/fedpft.yaml new file mode 100644 index 00000000000..6ef0f175976 --- /dev/null +++ b/baselines/fedpft/fedpft/conf/client/fedpft.yaml @@ -0,0 +1,2 @@ +--- +_target_: fedpft.client.FedPFTClient \ No newline at end of file diff --git a/baselines/fedpft/fedpft/conf/dataset/CIFAR100.yaml b/baselines/fedpft/fedpft/conf/dataset/CIFAR100.yaml new file mode 100644 index 00000000000..322c2d80c18 --- /dev/null +++ b/baselines/fedpft/fedpft/conf/dataset/CIFAR100.yaml @@ -0,0 +1,10 @@ +--- +_target_: fedpft.dataset.Dataset +name: cifar100 +dataset: CIFAR100 +num_classes: 100 +image_column_name: img +partition_by: fine_label +num_clients: ${num_clients} +dirichlet_alpha: ${dirichlet_alpha} +batch_size: ${batch_size} \ No newline at end of file diff --git a/baselines/fedpft/fedpft/conf/dataset/Caltech101.yaml b/baselines/fedpft/fedpft/conf/dataset/Caltech101.yaml new file mode 100644 index 00000000000..96dcc50fa8d --- /dev/null +++ b/baselines/fedpft/fedpft/conf/dataset/Caltech101.yaml @@ -0,0 +1,10 @@ +--- +_target_: fedpft.dataset.Dataset +name: caltech101 +dataset: clip-benchmark/wds_vtab-caltech101 +num_classes: 102 +image_column_name: webp +partition_by: cls +num_clients: ${num_clients} +dirichlet_alpha: ${dirichlet_alpha} +batch_size: ${batch_size} \ No newline at end of file diff --git a/baselines/fedpft/fedpft/conf/model/clip.yaml b/baselines/fedpft/fedpft/conf/model/clip.yaml new file mode 100644 index 00000000000..23d350a2347 --- /dev/null +++ b/baselines/fedpft/fedpft/conf/model/clip.yaml @@ -0,0 +1,9 @@ +feature_extractor: + _target_: fedpft.models.clip_vit + name: openai/clip-vit-base-patch32 +transform: + _target_: fedpft.models.transform + mean: [0.48145466, 0.4578275, 0.40821073] + std: [0.26862954, 0.26130258, 0.27577711] +image_input_size: 224 +hidden_dimension: 768 diff --git a/baselines/fedpft/fedpft/conf/model/resnet50.yaml b/baselines/fedpft/fedpft/conf/model/resnet50.yaml new file mode 100644 index 00000000000..260d9151e68 --- /dev/null +++ b/baselines/fedpft/fedpft/conf/model/resnet50.yaml @@ -0,0 +1,8 @@ +feature_extractor: + _target_: fedpft.models.resnet50 +transform: + _target_: fedpft.models.transform + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] +image_input_size: 224 +hidden_dimension: 2048 diff --git a/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml b/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml new file mode 100644 index 00000000000..3a4e290ced2 --- /dev/null +++ b/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml @@ -0,0 +1,12 @@ +--- +_target_: fedpft.strategy.FedAvg +fraction_fit: 1 +fraction_evaluate: 1 +accept_failures: False +on_fit_config_fn: + _target_: fedpft.server.fedavg_get_on_fit_config_fn + lr: 0.001 + num_epochs: 10 +evaluate_metrics_aggregation_fn: + _target_: fedpft.server.weighted_average + _partial_: true \ No newline at end of file diff --git a/baselines/fedpft/fedpft/conf/strategy/fedpft.yaml b/baselines/fedpft/fedpft/conf/strategy/fedpft.yaml new file mode 100644 index 00000000000..c4982074633 --- /dev/null +++ b/baselines/fedpft/fedpft/conf/strategy/fedpft.yaml @@ -0,0 +1,22 @@ +--- +_target_: fedpft.strategy.FedPFT +fraction_fit: 1 +fraction_evaluate: 1 +accept_failures: False +num_classes: ${dataset.num_classes} +feature_dimension: ${model.hidden_dimension} +device: ${device} +server_batch_size: 32 +num_epochs: 1 +server_opt: + lr: 1e-4 +on_fit_config_fn: + _target_: fedpft.server.fedpft_get_on_fit_config_fn + n_mixtures: 2 + cov_type: spherical + seed: 0 + tol: 1e-12 + max_iter: 10000 +evaluate_metrics_aggregation_fn: + _target_: fedpft.server.weighted_average + _partial_: true diff --git a/baselines/fedpft/fedpft/dataset.py b/baselines/fedpft/fedpft/dataset.py new file mode 100644 index 00000000000..733234074ef --- /dev/null +++ b/baselines/fedpft/fedpft/dataset.py @@ -0,0 +1,108 @@ +"""Dataset creation.""" + +from typing import Callable, Dict + +from flwr_datasets.federated_dataset import FederatedDataset +from flwr_datasets.partitioner import DirichletPartitioner +from torch.utils.data import DataLoader +from torchvision import transforms + + +class Dataset: + """Dataset class.""" + + def __init__( + self, + dataset: str, + num_clients: int, + batch_size: int, + dirichlet_alpha: float, + partition_by: str, + image_column_name: str, + transform: transforms, + image_input_size: int, + seed: int = 0, + split_size: float = 0.8, + **kwargs, + ) -> None: + """Load the dataset and partition it using dirichlet distribution. + + Parameters + ---------- + dataset : str + Name of dataset to be downloaded from HuggingFace. + num_clients: int + Number of clients. + batch_size: int + Batch size of training and testing dataloaders of clients. + dirichlet_alpha: float + Alpha parameter of Dirichlet distribution. + partition_by: str + Label named used for partitioning the dataset. + image_column_name: str + Column name of image in the dataset. + transform: transforms + Transformation of each batch. + image_input_size: int + Input size of pre-trained model. + seed: int, optional + Seed for partitioning the dataset. Default is 0. + split_size: float, optional + The portion of dataset to be used as training and rest as test. + """ + self.dataset = dataset + self.num_clients = num_clients + self.image_input_size = image_input_size + self.transform = transform + self.batch_size = batch_size + self.dirichlet_alpha = dirichlet_alpha + self.partition_by = partition_by + self.seed = seed + self.split_size = split_size + self.image_column_name = image_column_name + + def get_loaders(self): + """Partition the datasets and return a list of dataloaders.""" + partitioner = DirichletPartitioner( + num_partitions=self.num_clients, + partition_by=self.partition_by, + alpha=self.dirichlet_alpha, + min_partition_size=10, + self_balancing=True, + ) + + fds = FederatedDataset( + dataset=self.dataset, partitioners={"train": partitioner} + ) + # Create train/val for each partition and wrap it into DataLoader + trainloaders, testloaders = [], [] + for partition_id in range(self.num_clients): + partition = fds.load_partition(partition_id) + partition = partition.with_transform(self.apply_batch_transforms()) + partition = partition.train_test_split( + train_size=self.split_size, seed=self.seed + ) + trainloaders.append( + DataLoader(partition["train"], batch_size=self.batch_size) + ) + testloaders.append( + DataLoader(partition["test"], batch_size=self.batch_size) + ) + + return trainloaders, testloaders + + def apply_batch_transforms(self) -> Callable[[Dict], Dict]: + """Apply batch transforms for each batch.""" + + def batch_transform(batch): + batch_img = [ + self.transform( + img.resize((self.image_input_size, self.image_input_size)) + ) + for img in batch[self.image_column_name] + ] + batch_label = list(batch[self.partition_by]) + + return {"img": batch_img, "label": batch_label} + + return batch_transform diff --git a/baselines/fedpft/fedpft/dataset_preparation.py b/baselines/fedpft/fedpft/dataset_preparation.py new file mode 100644 index 00000000000..83a9c5dd9e2 --- /dev/null +++ b/baselines/fedpft/fedpft/dataset_preparation.py @@ -0,0 +1 @@ +"""Handle the dataset partitioning and (optionally) complex downloads.""" diff --git a/baselines/fedpft/fedpft/main.py b/baselines/fedpft/fedpft/main.py new file mode 100644 index 00000000000..9860b1232bf --- /dev/null +++ b/baselines/fedpft/fedpft/main.py @@ -0,0 +1,88 @@ +"""Run FL with frozen, pre-trained models.""" + +import pickle +from pathlib import Path + +import flwr as fl +import hydra +import torch +from hydra.core.hydra_config import HydraConfig +from hydra.utils import instantiate +from omegaconf import DictConfig, OmegaConf + +from fedpft.client import generate_client_fn + + +@hydra.main(config_path="conf", config_name="base", version_base=None) +def main(cfg: DictConfig) -> None: + """Run federated learning with frozen, pre-trained models. + + Parameters + ---------- + cfg : DictConfig + An omegaconf object that stores the hydra config. + """ + # Print Config + print(OmegaConf.to_yaml(cfg)) + + # Set device + device = torch.device(cfg.device) + + # Prepare dataset + trainloaders, testloaders = instantiate( + cfg.dataset, + transform=cfg.model.transform, + image_input_size=cfg.model.image_input_size, + ).get_loaders() + + # Define clients + client_fn = generate_client_fn( + client_cfg=cfg.client, + trainloaders=trainloaders, + testloaders=testloaders, + feature_extractor=instantiate(cfg.model.feature_extractor), + num_classes=cfg.dataset.num_classes, + device=device, + ) + + # Setup strategy + strategy = instantiate(cfg.strategy) + + # Start simulation + history = fl.simulation.start_simulation( + client_fn=client_fn, + num_clients=cfg.num_clients, + config=fl.server.ServerConfig(num_rounds=cfg.num_rounds), + strategy=strategy, + client_resources={"num_cpus": cfg.num_cpus, "num_gpus": cfg.num_gpus}, + ) + + # Save results + accuracy_per_round = history.metrics_distributed["accuracy"] + print(accuracy_per_round) + save_path = HydraConfig.get().runtime.output_dir + + strategy_name = strategy.__class__.__name__ + + def format_variable(x): + return f"{x!r}" if isinstance(x, bytes) else x + + file_suffix: str = ( + f"_{format_variable(strategy_name)}" + f"_{format_variable(cfg.dataset.name)}" + f"_clients={format_variable(cfg.num_clients)}" + f"_rounds={format_variable(cfg.num_rounds)}" + f"_finalacc={format_variable(accuracy_per_round[-1][1]):.2f}" + ) + filename = "results" + file_suffix + ".pkl" + + print(f">>> Saving {filename}") + results_path = Path(save_path) / filename + results = {"history": history} + + with open(str(results_path), "wb") as hist_file: + pickle.dump(results, hist_file, protocol=pickle.HIGHEST_PROTOCOL) + + +if __name__ == "__main__": + main() diff --git a/baselines/fedpft/fedpft/models.py b/baselines/fedpft/fedpft/models.py new file mode 100644 index 00000000000..0514b2f3283 --- /dev/null +++ b/baselines/fedpft/fedpft/models.py @@ -0,0 +1,221 @@ +"""Models, training and eval functions.""" + +import logging +from typing import List, Optional, Tuple + +import numpy as np +import torch +import torch.utils +import torchvision.transforms as transforms +from flwr.common.logger import log +from torch import nn +from torch.utils.data import DataLoader +from torchvision import models +from transformers import CLIPModel + + +def resnet50() -> torch.nn.modules: + """Return ResNet-50 model as feature extractor.""" + resnet50 = models.resnet50(weights=models.ResNet50_Weights.DEFAULT) + + # Remove last layer and flatten outputs + resnet50 = torch.nn.Sequential( + *(list(resnet50.children())[:-1]), torch.nn.Flatten() + ) + + # Set the hidden_dimension + resnet50.hidden_dimension = 2048 + + return resnet50 + + +def clip_vit(name: str) -> torch.nn.modules: + """Return CLIP-ViT as feature extractor. + + Parameters + ---------- + name : str + Name of the CLIP model on transformer library, + e.g. `openai/clip-vit-base-patch32`. + """ + + class ClipVit(nn.Module): + """Wrap outputs to return only pooled outputs.""" + + def __init__(self, vision_model): + super().__init__() + self.vision_model = vision_model + self.hidden_dimension = vision_model.config.hidden_size + + def forward(self, input): + output = self.vision_model(input) + return output[1] # return pooled_output (CLS token) + + vision_model = CLIPModel.from_pretrained(name).vision_model + + return ClipVit(vision_model) + + +def transform(mean: List, std: List) -> transforms.Compose: + """Return `transforms.Compose` function for normalizing images. + + Parameters + ---------- + mean : List + Sequence of means for each channel + std : List + Sequence of standard deviations for each channel. + + Returns + ------- + transforms.Compose + Transform function for normalizing images + """ + tr = transforms.Compose( + [ + transforms.ToTensor(), + transforms.Normalize(mean, std), + ] + ) + return tr + + +def extract_features( + dataloader: DataLoader, feature_extractor: torch.nn.Module, device: torch.device +) -> Tuple[np.array, np.array]: + """Extract features and labels from images using feature extractor. + + Parameters + ---------- + dataloader : DataLoader + Dataloader containing {'img': img, 'label': label} + dicts to be extracted. + feature_extractor : torch.nn.Module + Model for extracting features. + device : torch.device + Device for loading `feature_extractor`. + + Returns + ------- + features : np.array + 2D array containing features extracted from `feature_extractor`. + labels : np.array + 2D array containing labels of `features`. + """ + features, labels = [], [] + for dict in dataloader: + batch_samples = dict["img"].to(device) + batch_label = dict["label"].to(device) + with torch.no_grad(): + feature = feature_extractor(batch_samples) + features.append(feature.cpu().detach().numpy()) + labels.append(batch_label) + + # reshape feauturs and labels into a single numpy array + features = np.concatenate(features, axis=0, dtype=np.float64) + labels = np.concatenate(labels, dtype=int) + + return features, labels + + +def test( + classifier_head: torch.nn.Linear, + dataloader: DataLoader, + feature_extractor: torch.nn.Module, + device: torch.device, +) -> Tuple[float, float]: + """"Evaluates the `classifier_head` on the dataset. + + Parameters + ---------- + classifier_head : torch.nn.Linear + Classifier head model. + dataloader : DataLoader + Dataset used for evaluating `classifier_head` containing + {'img': img, 'label': label} dicts. + feature_extractor : torch.nn.Module + Model used for extracting features from the `dataloader`. + device : torch.device + Device for loading `feature_extractor`. + + Returns + ------- + loss : float + CrossEntropy Loss of `classifier_head` on the dataset. + accuracy : float + Accuracy of `classifier_head` on the dataset. + """ + classifier_head.eval() + feature_extractor.eval() + classifier_head.to(device) + feature_extractor.to(device) + + correct, total, loss = 0, 0, 0 + for dict in dataloader: + samples = dict["img"].to(device) + labels = dict["label"].to(device) + with torch.no_grad(): + feature = feature_extractor(samples) + output = classifier_head(feature) + pred = torch.max(output, 1)[1].data.squeeze() + correct += (pred == labels).sum().item() + total += samples.shape[0] + running_loss = nn.CrossEntropyLoss()(output, labels) + loss += running_loss + + return loss.cpu().item(), correct / total + + +def train( + classifier_head: torch.nn.Linear, + dataloader: DataLoader, + opt: torch.optim.Optimizer, + num_epochs: int, + device: torch.device, + feature_extractor: Optional[torch.nn.Module] = None, + verbose: Optional[bool] = False, +) -> None: + """Trains the `classifier_head`. + + Parameters + ---------- + classifier_head : torch.nn.Linear + Classifier head model. + dataloader : DataLoader + Dataset used for evaluating `classifier_head` + containing {'img': img, 'label': label} dicts. + opt : torch.optim.Optimizer + Optimizer for the `classifier_head`. + num_epochs: int + Number of epochs to train the `classifier_head`. + device : torch.device + Device for loading `feature_extractor`. + feature_extractor : torch.nn.Module, Optional + Model used for extracting features from the `dataloader`, optional. + `verbose` : bool, Optional + Whether or not log the accuracy during the training. Defaults to False. + """ + classifier_head.to(device) + if feature_extractor: + feature_extractor.eval() + feature_extractor.to(device) + + for epoch in range(num_epochs): + correct, total, loss = 0, 0, 0 + for _, dict in enumerate(dataloader): + classifier_head.zero_grad() + samples = dict["img"].to(device) + labels = dict["label"].to(device) + if feature_extractor: + with torch.no_grad(): + samples = feature_extractor(samples) + output = classifier_head(samples) + pred = torch.max(output, 1)[1].data.squeeze() + correct += (pred == labels).sum().item() + total += samples.shape[0] + running_loss = nn.CrossEntropyLoss()(output, labels) + loss += running_loss + running_loss.backward() + opt.step() + if verbose: + log(logging.INFO, f"Epoch:{epoch+1} --- Accuracy: {correct/total}") diff --git a/baselines/fedpft/fedpft/server.py b/baselines/fedpft/fedpft/server.py new file mode 100644 index 00000000000..00d88360e9f --- /dev/null +++ b/baselines/fedpft/fedpft/server.py @@ -0,0 +1,94 @@ +"""Create global evaluation function.""" + +from typing import Callable, Dict, List, Tuple + +from flwr.common import Metrics + + +def fedpft_get_on_fit_config_fn( + n_mixtures: int, cov_type: str, seed: int, tol: float, max_iter: int +) -> Callable[[int], Dict[str, str]]: + """Return a function which returns FedPFT training configurations. + + Parameters + ---------- + n_mixtures : int + Number of mixtures for GMMs + cov_type : str + Type of covariance + seed : int + Seed for learning and sampling from the GMMs + tol : float + Error tolerance for learning GMMs + max_iter : int + Maximum number of iteration for EM algorithm + + Returns + ------- + Callable[[int], Dict[str, str]] + Function to return a config with the `lr` and `num_epochs` + """ + + def fit_config(server_round: int) -> Dict[str, str]: + """Return a configuration for training Gaussian Mixtures.""" + config = { + "n_mixtures": str(n_mixtures), + "cov_type": cov_type, + "seed": str(seed), + "tol": str(tol), + "max_iter": str(max_iter), + } + return config + + return fit_config + + +def fedavg_get_on_fit_config_fn( + lr: float, + num_epochs: int, +) -> Callable[[int], Dict[str, str]]: + """Return a function which returns FedAvg training configurations. + + Parameters + ---------- + lr : float + Client's learning rate + num_epochs : int + Number of epochs for local learning of clients + + Returns + ------- + Callable[[int], Dict[str, str]] + Function to return a config with the `lr` and `num_epochs` + """ + + def fit_config(server_round: int) -> Dict[str, str]: + """Return a configuration number of epochs and learning rate.""" + config = { + "lr": str(lr), + "num_epochs": str(num_epochs), + } + return config + + return fit_config + + +def weighted_average(metrics: List[Tuple[int, Metrics]]) -> Metrics: + """Aggregate with weighted average during evaluation. + + Parameters + ---------- + metrics : List[Tuple[int, Metrics]] + The list of metrics to aggregate. + + Returns + ------- + Metrics + The weighted average metric. + """ + # Multiply accuracy of each client by number of examples used + accuracies = [num_examples * float(m["accuracy"]) for num_examples, m in metrics] + examples = [num_examples for num_examples, _ in metrics] + + # Aggregate and return custom metric (weighted average) + return {"accuracy": int(sum(accuracies)) / int(sum(examples))} diff --git a/baselines/fedpft/fedpft/strategy.py b/baselines/fedpft/fedpft/strategy.py new file mode 100644 index 00000000000..f9546140124 --- /dev/null +++ b/baselines/fedpft/fedpft/strategy.py @@ -0,0 +1,145 @@ +"""FedPFT strategy.""" + +from typing import Dict, List, Optional, Tuple, Union + +import torch +from flwr.common import ( + FitRes, + Parameters, + Scalar, + ndarrays_to_parameters, + parameters_to_ndarrays, +) +from flwr.server.client_proxy import ClientProxy +from flwr.server.strategy import FedAvg +from omegaconf import DictConfig +from sklearn.mixture import GaussianMixture as GMM +from torch.utils.data import DataLoader + +from fedpft.models import train +from fedpft.utils import chunks, ndarrays_to_gmmparam + + +class FedPFT(FedAvg): + """Implementation of FedPFT. + + https://arxiv.org/abs/2402.01862 + Authors: + Mahdi Beitollahi, Alex Bie, Sobhan Hemati, Leo Maxime Brunswic, + Xu Li, Xi Chen, Guojun Zhang. + """ + + def __init__( + self, + *args, + num_classes: int, + feature_dimension: int, + server_opt: DictConfig, + server_batch_size: int, + num_epochs: int, + device: torch.device, + **kwargs, + ) -> None: + """Create FedPFT strategy. + + Parameters + ---------- + num_classes : int + Number of classes in the dataset. + feature_dimension : int + Size of feature embeddings + server_opt : DictConfig + Configuration of server optimizer for training classifier head. + server_batch_size : int + Batch size of synthetic features. + num_epochs : int + Number of epochs to train the classifier head. + + Attributes + ---------- + device : torch.device() + Device to train the classifier head at the server. + """ + super().__init__(*args, **kwargs) + self.num_classes = num_classes + self.feature_dimension = feature_dimension + self.server_opt = server_opt + self.server_batch_size = server_batch_size + self.num_epochs = num_epochs + self.device = device + + def aggregate_fit( + self, + server_round: int, + results: List[Tuple[ClientProxy, FitRes]], + failures: List[Union[Tuple[ClientProxy, FitRes], BaseException]], + ) -> Tuple[Optional[Parameters], Dict[str, Scalar]]: + """Learn a classifier head by generating samples from the GMMs.""" + # Do not aggregate if there are failures. + if not self.accept_failures and failures: + raise Exception("there are failures and failures are not accepted") + + config = self.on_fit_config_fn(server_round) + + # Sample from the GMMs to create synthetic feature dataset + synthetic_features_dataset = [] + for _, fit_res in results: + # Convert byte parameters into ndarrays and GMMParameters + ndarray = parameters_to_ndarrays(fit_res.parameters) + all_gmm_parameters = [ + ndarrays_to_gmmparam(array) for array in chunks(ndarray, 5) + ] + + # Sample from GMM_label pairs to create synthetic features + for gmm_parameter in all_gmm_parameters: + gmm = GMM( + n_components=int(config["n_mixtures"]), + covariance_type=config["cov_type"], + random_state=int(config["seed"]), + tol=float(config["tol"]), + max_iter=int(config["max_iter"]), + ) + # Set values of the GMMs + gmm.means_ = gmm_parameter.means.astype("float32") + gmm.weights_ = gmm_parameter.weights.astype("float32") + gmm.covariances_ = gmm_parameter.covariances.astype("float32") + + # Sample features + syn_features, _ = gmm.sample(gmm_parameter.num_samples) + syn_features = torch.tensor(syn_features, dtype=torch.float32) + gmm_labels = torch.tensor( + [int(gmm_parameter.label)] * int(gmm_parameter.num_samples) + ) + + # Add to train data + synthetic_features_dataset += list(zip(syn_features, gmm_labels)) + + # Train a classifier head + synthetic_features_dataset = [ + {"img": img, "label": label} for img, label in synthetic_features_dataset + ] + synthetic_loader = DataLoader( + synthetic_features_dataset, + batch_size=self.server_batch_size, + shuffle=True, + ) + classifier_head = torch.nn.Linear(self.feature_dimension, self.num_classes) + opt = torch.optim.AdamW( + params=classifier_head.parameters(), lr=self.server_opt.lr + ) + + train( + classifier_head=classifier_head, + dataloader=synthetic_loader, + device=self.device, + num_epochs=self.num_epochs, + opt=opt, + verbose=True, + ) + + # Send the classifier head to clients + classifier_ndarray = [ + val.cpu().numpy() for _, val in classifier_head.state_dict().items() + ] + + return ndarrays_to_parameters(classifier_ndarray), {} diff --git a/baselines/fedpft/fedpft/utils.py b/baselines/fedpft/fedpft/utils.py new file mode 100644 index 00000000000..c1a27c14647 --- /dev/null +++ b/baselines/fedpft/fedpft/utils.py @@ -0,0 +1,102 @@ +"""Utility functions.""" + +from dataclasses import dataclass +from typing import List + +import numpy as np +from flwr.common import NDArrays +from sklearn.mixture import GaussianMixture + + +@dataclass +class GMMParameters: + """GMM parameters.""" + + label: int + means: NDArrays + weights: NDArrays + covariances: NDArrays + num_samples: int + + +def gmmparam_to_ndarrays(gmm: GMMParameters) -> NDArrays: + """Convert gmm object to NumPy ndarrays.""" + return [gmm.label, gmm.means, gmm.weights, gmm.covariances, gmm.num_samples] + + +def ndarrays_to_gmmparam(ndarrays: NDArrays) -> GMMParameters: + """Convert NumPy ndarray to GMM object.""" + return GMMParameters( + label=ndarrays[0], + means=ndarrays[1], + weights=ndarrays[2], + covariances=ndarrays[3], + num_samples=ndarrays[4], + ) + + +def learn_gmm( + features: np.array, + labels: np.array, + n_mixtures: int, + cov_type: str, + seed: int, + tol: float = 1e-12, + max_iter: int = 1000, +) -> List[GMMParameters]: + """Learn a list of 16-bits GMMs for each label. + + Parameters + ---------- + features : np.array + A 2-d array with size (n_samples, feature_dimension) containing + extracted features for all the samples. + labels : np.array + An array with size (n_samples) containing labels associated for + each sample in `features`. + n_mixtures : int + Number of mixtures in each Gaussian Mixture. + cov_type : str + Covariance type of Gaussian Mixtures, e.g. spherical. + seed: int + Seed for learning and sampling from Gaussian Mixtures. + tol: float + Tolerance of Gaussian Mixtures. + max_iter: int + Number of maximum iterations to learn the Gaussian Mixtures. + + Returns + ------- + List[GMMParameters] + Returns a list containing the GMMParameters for each class. + """ + gmm_list = [] + for label in np.unique(labels): + cond_features = features[label == labels] + if ( + len(cond_features) > n_mixtures + ): # number of samples should be larger than `n_mixtures`. + gmm = GaussianMixture( + n_components=n_mixtures, + covariance_type=cov_type, + random_state=seed, + tol=tol, + max_iter=max_iter, + ) + gmm.fit(cond_features) + gmm_list.append( + GMMParameters( + label=label, + means=gmm.means_.astype("float16"), + weights=gmm.weights_.astype("float16"), + covariances=gmm.covariances_.astype("float16"), + num_samples=len(cond_features), + ) + ) + return gmm_list + + +def chunks(lst, n): + """Yield successive n-sized chunks from lst.""" + for i in range(0, len(lst), n): + yield lst[i : i + n] diff --git a/baselines/fedpft/pyproject.toml b/baselines/fedpft/pyproject.toml new file mode 100644 index 00000000000..30e47defbda --- /dev/null +++ b/baselines/fedpft/pyproject.toml @@ -0,0 +1,143 @@ +[build-system] +requires = ["poetry-core>=1.4.0"] +build-backend = "poetry.masonry.api" + +[tool.poetry] +name = "fedpft" # <----- Ensure it matches the name of your baseline directory containing all the source code +version = "1.0.0" +description = "Flower Baselines" +license = "Apache-2.0" +authors = ["The Flower Authors "] +readme = "README.md" +homepage = "https://flower.ai" +repository = "https://github.com/adap/flower" +documentation = "https://flower.ai" +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Scientific/Engineering :: Mathematics", + "Topic :: Software Development", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", + "Typing :: Typed", +] + +[tool.poetry.dependencies] +python = ">=3.8.15, <3.12.0" # don't change this +flwr = { extras = ["simulation"], version = "1.5.0" } +hydra-core = "1.3.2" # don't change this +torch = {url = "https://download.pytorch.org/whl/cu117/torch-1.13.0%2Bcu117-cp310-cp310-linux_x86_64.whl"} +scikit-learn = "1.2.2" +flwr-datasets = "0.1.0" +torchvision = {url = "https://download.pytorch.org/whl/cu117/torchvision-0.14.0%2Bcu117-cp310-cp310-linux_x86_64.whl"} +transformers = "4.39.3" +datasets = "2.18.0" + +[tool.poetry.dev-dependencies] +isort = "==5.13.2" +black = "==24.2.0" +docformatter = "==1.7.5" +mypy = "==1.4.1" +pylint = "==2.8.2" +flake8 = "==3.9.2" +pytest = "==6.2.4" +pytest-watch = "==4.2.0" +ruff = "==0.0.272" +types-requests = "==2.27.7" + +[tool.isort] +line_length = 88 +indent = " " +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true + +[tool.black] +line-length = 88 +target-version = ["py38", "py39", "py310", "py311"] + +[tool.pytest.ini_options] +minversion = "6.2" +addopts = "-qq" +testpaths = [ + "flwr_baselines", +] + +[tool.mypy] +ignore_missing_imports = true +strict = false +plugins = "numpy.typing.mypy_plugin" + +[tool.pylint."MESSAGES CONTROL"] +disable = "bad-continuation,duplicate-code,too-few-public-methods,useless-import-alias" +good-names = "i,j,k,_,x,y,X,Y" +signature-mutators = "hydra.main.main" + +[tool.pylint.typecheck] +generated-members = "numpy.*, torch.*, tensorflow.*" + +[[tool.mypy.overrides]] +module = [ + "importlib.metadata.*", + "importlib_metadata.*", +] +follow_imports = "skip" +follow_imports_for_stubs = true +disallow_untyped_calls = false + +[[tool.mypy.overrides]] +module = "torch.*" +follow_imports = "skip" +follow_imports_for_stubs = true + +[tool.docformatter] +wrap-summaries = 88 +wrap-descriptions = 88 + +[tool.ruff] +target-version = "py38" +line-length = 88 +select = ["D", "E", "F", "W", "B", "ISC", "C4"] +fixable = ["D", "E", "F", "W", "B", "ISC", "C4"] +ignore = ["B024", "B027"] +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".hg", + ".mypy_cache", + ".nox", + ".pants.d", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "venv", + "proto", +] + +[tool.ruff.pydocstyle] +convention = "numpy" From 302543c9502e92f6887787e11d782d5b3d1ca1df Mon Sep 17 00:00:00 2001 From: Mahdi Date: Sun, 14 Apr 2024 22:41:38 -0400 Subject: [PATCH 02/29] fixd model to gpu bug --- baselines/fedpft/fedpft/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/baselines/fedpft/fedpft/models.py b/baselines/fedpft/fedpft/models.py index 0514b2f3283..7ebc9beed4a 100644 --- a/baselines/fedpft/fedpft/models.py +++ b/baselines/fedpft/fedpft/models.py @@ -102,6 +102,8 @@ def extract_features( labels : np.array 2D array containing labels of `features`. """ + feature_extractor.to(device) + features, labels = [], [] for dict in dataloader: batch_samples = dict["img"].to(device) @@ -109,7 +111,7 @@ def extract_features( with torch.no_grad(): feature = feature_extractor(batch_samples) features.append(feature.cpu().detach().numpy()) - labels.append(batch_label) + labels.append(batch_label.cpu().detach().numpy()) # reshape feauturs and labels into a single numpy array features = np.concatenate(features, axis=0, dtype=np.float64) From 02724f9a600694feb1a62d5913145bc1cfa5a0f4 Mon Sep 17 00:00:00 2001 From: Mahdi Date: Sun, 14 Apr 2024 22:42:26 -0400 Subject: [PATCH 03/29] fixed config --- baselines/fedpft/fedpft/conf/base.yaml | 4 ++-- baselines/fedpft/fedpft/conf/strategy/fedavg.yaml | 2 +- baselines/fedpft/fedpft/conf/strategy/fedpft.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/baselines/fedpft/fedpft/conf/base.yaml b/baselines/fedpft/fedpft/conf/base.yaml index ab1477bd696..01b1495c241 100644 --- a/baselines/fedpft/fedpft/conf/base.yaml +++ b/baselines/fedpft/fedpft/conf/base.yaml @@ -1,10 +1,10 @@ --- -num_clients: 2 +num_clients: 50 dirichlet_alpha: 0.1 num_rounds: 1 num_cpus: 1 -num_gpus: 0.1 +num_gpus: 1 batch_size: 64 device: cuda diff --git a/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml b/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml index 3a4e290ced2..5f9e1d9e777 100644 --- a/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml +++ b/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml @@ -6,7 +6,7 @@ accept_failures: False on_fit_config_fn: _target_: fedpft.server.fedavg_get_on_fit_config_fn lr: 0.001 - num_epochs: 10 + num_epochs: 1 evaluate_metrics_aggregation_fn: _target_: fedpft.server.weighted_average _partial_: true \ No newline at end of file diff --git a/baselines/fedpft/fedpft/conf/strategy/fedpft.yaml b/baselines/fedpft/fedpft/conf/strategy/fedpft.yaml index c4982074633..5612193071d 100644 --- a/baselines/fedpft/fedpft/conf/strategy/fedpft.yaml +++ b/baselines/fedpft/fedpft/conf/strategy/fedpft.yaml @@ -7,12 +7,12 @@ num_classes: ${dataset.num_classes} feature_dimension: ${model.hidden_dimension} device: ${device} server_batch_size: 32 -num_epochs: 1 +num_epochs: 50 server_opt: lr: 1e-4 on_fit_config_fn: _target_: fedpft.server.fedpft_get_on_fit_config_fn - n_mixtures: 2 + n_mixtures: 1 cov_type: spherical seed: 0 tol: 1e-12 From a7730cc3a35aab6efc581a1a018ddbcc006a2276 Mon Sep 17 00:00:00 2001 From: Mahdi Date: Sun, 14 Apr 2024 22:43:04 -0400 Subject: [PATCH 04/29] added notebook for visualization --- .../fedpft/docs/viz_and_plot_results.ipynb | 204 ++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 baselines/fedpft/docs/viz_and_plot_results.ipynb diff --git a/baselines/fedpft/docs/viz_and_plot_results.ipynb b/baselines/fedpft/docs/viz_and_plot_results.ipynb new file mode 100644 index 00000000000..866ffb5d8c7 --- /dev/null +++ b/baselines/fedpft/docs/viz_and_plot_results.ipynb @@ -0,0 +1,204 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "5e0cf2a9-b782-48de-ac45-128726a26e64", + "metadata": {}, + "outputs": [ + { + "ename": "ModuleNotFoundError", + "evalue": "No module named 'matplotlib'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[2], line 7\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mos\u001b[39;00m\n\u001b[0;32m 6\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[1;32m----> 7\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mmatplotlib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpyplot\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mplt\u001b[39;00m\n", + "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'matplotlib'" + ] + } + ], + "source": [ + "import pickle\n", + "import yaml\n", + "from pathlib import Path\n", + "import os\n", + "\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "7ea3e149-ce6f-4ba0-aa41-e0501a04efe3", + "metadata": {}, + "outputs": [], + "source": [ + "def saveFig(name, fig):\n", + " fig.savefig(\n", + " name,\n", + " dpi=None,\n", + " facecolor=fig.get_facecolor(),\n", + " edgecolor=\"none\",\n", + " orientation=\"portrait\",\n", + " format=\"png\",\n", + " transparent=False,\n", + " bbox_inches=\"tight\",\n", + " pad_inches=0.2,\n", + " metadata=None,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "4b010856-0d99-4d81-8fb0-7a927f10eeaf", + "metadata": {}, + "outputs": [], + "source": [ + "# Update the path belows to the directories containing the results for FedPFT and FedAvg\n", + "path_fedpft_resutls_cifar100 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','16-36-16')\n", + "path_fedpft_resutls_caltech101 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','16-44-20')\n", + "\n", + "path_fedavg_resutls_cifar100 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','18-16-41')\n", + "path_fedavg_resutls_caltech101 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','16-36-16')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2e3e165c-1ce6-4efa-a4e1-1372586e436e", + "metadata": {}, + "outputs": [], + "source": [ + "# load results\n", + "def read_accuracies(path_to_pickle):\n", + " for result in list(Path(path_to_pickle).glob(\"*.pkl\")):\n", + " with open(result, \"rb\") as handle:\n", + " data = pickle.load(handle)\n", + "\n", + " accuracies = data['history'].metrics_distributed['accuracy']\n", + " return accuracies\n" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "77b70c73", + "metadata": {}, + "outputs": [], + "source": [ + "fedpft_cifar = read_accuracies(path_fedpft_resutls_cifar100)\n", + "fedpft_caltech = read_accuracies(path_fedpft_resutls_caltech101)\n", + "fedavg_cifar = read_accuracies(path_fedavg_resutls_cifar100)\n", + "fedavg_caltech = read_accuracies(path_fedavg_resutls_caltech101)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "6f4c87ad", + "metadata": {}, + "outputs": [], + "source": [ + "fedavg_cifar = [(1, 0.06924765515865097),\n", + " (2, 0.1315106765116743),\n", + " (3, 0.16773099181800039),\n", + " (4, 0.1946717222111355),\n", + " (5, 0.2171223308720814),\n", + " (6, 0.2375773298742766),\n", + " (7, 0.2597285970864099),\n", + " (8, 0.276092596288166),\n", + " (9, 0.290560766314109),\n", + " (10, 0.3036320095789264),\n", + " (11, 0.3128118140091798),\n", + " (12, 0.3261823987228098),\n", + " (13, 0.33745759329475156),\n", + " (14, 0.3477349830373179),\n", + " (15, 0.35831171422869684),\n", + " (16, 0.36679305527838757),\n", + " (17, 0.37407703053282776),\n", + " (18, 0.3817601277190182),\n", + " (19, 0.38824585910995807),\n", + " (20, 0.3942326880862103)]" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "id": "e1a678de", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1wAAAD0CAYAAACPQifaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABdAUlEQVR4nO3deVhU1f8H8PcMy7CDOuygyGK4YqHgviSCVi6ZipaifE2zpDLKFFNcE0sj/ZVluVbuZlmmqYjikqilWe4KiijIqjAsAgNzf38QkxODyjDDDPh+Pc88Oueec+9nzoye+cw991yRIAgCiIiIiIiISOvE+g6AiIiIiIiosWLCRUREREREpCNMuIiIiIiIiHSECRcREREREZGOMOEiIiIiIiLSESZcREREREREOsKEi4iIiIiISEeYcBEREREREekIEy4iIiIiIiIdYcJFRERERESkI0y4SC+Sk5Px2muvwdPTE2ZmZrCxsUH37t2xfPly3L9/X1nPw8MDL7zwgkpbkUik9uHk5KRSLy8vD2ZmZhCJRLh06ZLaOMaPH6+yD4lEglatWiE6OholJSXV6m/duhVjxoyBj48PRCIR+vTpU+NrLC0txfTp0+Hi4gJzc3MEBgYiLi5Obd3jx4+jR48esLCwgJOTE9566y0UFhbWuG9N/bfPbGxs0Lt3b+zevVvrx6qSkpKiPN6OHTuqbZ87dy5EIhFycnJqve/jx49j7ty5yMvLq7atT58+aj8nAwYMqFa3Nu8VERERUW0Y6zsAevLs3r0bI0aMgEQiQVhYGNq1a4eysjIcO3YM06ZNw4ULF/D1118/dB/9+/dHWFiYSpm5ubnK8+3btysTsY0bN2LhwoVq9yWRSLB69WoAQH5+Pn766ScsWLAAycnJ2Lhxo0rdL7/8EqdPn0bnzp2Rm5v70BjHjx+P77//HlOnToWPjw/Wr1+P5557DocOHUKPHj2U9c6ePYt+/fqhdevWiI2Nxe3bt7F06VJcu3YNv/7660OPoYmqvhMEATdv3sSXX36JQYMG4ddff0VISIjWj/eg+fPnY9iwYRCJRFrZ3/HjxzFv3jyMHz8ednZ21ba7ubkhJiZGpczFxaVavcd9r4iIiIhqTSCqR9evXxesrKwEX19fIT09vdr2a9euCcuWLVM+b9GihfD888+r1AEgTJky5ZHH6tWrlzBs2DDhnXfeEVq2bKm2zrhx4wRLS0uVMoVCIXTp0kUQiURCRkaGyrbU1FShoqJCEARBaNu2rdC7d2+1+z158qQAQFiyZImy7P79+4KXl5fQtWtXlboDBw4UnJ2dhfz8fGXZqlWrBADCvn37Hvk6a0Nd3128eFEAIAwcOFCrx6py48YNAYDQsWNHAYCwY8cOle1z5swRAAjZ2dm13veSJUsEAMKNGzeqbevdu7fQtm3bR+6jNu8VERERUW1xSiHVq48//hiFhYVYs2YNnJ2dq2339vbG22+/XefjpKam4ujRoxg1ahRGjRqFGzdu4Pjx44/VViQSoUePHhAEAdevX1fZ5u7uDrH40f9svv/+exgZGWHSpEnKMjMzM0yYMAGJiYm4desWAEAmkyEuLg5jxoyBjY2Nsm5YWBisrKywbdu2x4q5Llq3bg2pVIrk5GSV8tLSUsyZMwfe3t6QSCRwd3fH+++/j9LSUpV6cXFx6NGjB+zs7GBlZYWnnnoKM2fOrHacUaNGoVWrVpg/fz4EQXhkXCdPnsSAAQNga2sLCwsL9O7dG7/99pty+9y5czFt2jQAQMuWLZVTBlNSUlT2U15e/tDpmY/7XhERERFpglMKqV7t2rULnp6e6NatW532U1JSUu2aH2tra0gkEgDA5s2bYWlpiRdeeAHm5ubw8vLCxo0bH/u4VV/amzRpolF8f/75J1q1aqWSRAFAQEAAgMpphO7u7jh37hzKy8vRqVMnlXqmpqbo2LEj/vzzT42OXxv5+fm4d+8evLy8lGUKhQKDBw/GsWPHMGnSJLRu3Rrnzp3Dp59+iqtXr2Lnzp0AgAsXLuCFF15Ahw4dMH/+fEgkEiQlJakkRlWMjIwwa9YshIWF4ccff8SwYcNqjOngwYMYOHAg/P39MWfOHIjFYqxbtw7PPvssjh49ioCAAAwbNgxXr17F5s2b8emnn0IqlQIA7O3tlfu5evUqLC0tUVZWBkdHR0ycOBHR0dEwMTFR1nnc94qIiIhIE0y4qN7IZDKkpaVhyJAhdd7XmjVrsGbNGpWydevWYfz48QCAjRs3YsiQIcrrukJDQ/H1119j+fLlMDau/rGvSt7y8/Oxc+dO7NixA+3atcNTTz2lUXx37txRewavqiw9PV1Z78Hy/9Y9evSoRsd/mKpkVRAEpKamYtasWaioqMDw4cOVdTZt2oQDBw7g8OHDKtcwtWvXDpMnT8bx48fRrVs3xMXFoaysDL/++qsy4XmYl19+GQsWLMD8+fPx4osvqr2WSxAETJ48GX379sWvv/6qrPPaa6+hbdu2mDVrFvbv348OHTrgmWeewebNmzF06FB4eHio7MfLywt9+/ZF+/btUVRUhO+//x4LFy7E1atXsXXrVmW9x32viIiIiDTBhIvqjUwmA1B5JqquhgwZgoiICJWytm3bAgD+/vtvnDt3TmWxhNGjR2PRokXYt28fnn/+eZV2RUVFKmdFAKBHjx745ptvNF7c4f79+8qzbQ8yMzNTbn/wz5rqPrhio7b8N1k1MTHB+++/j8jISGXZ9u3b0bp1a/j6+qqcSXz22WcBAIcOHUK3bt2UC1X89NNPCA8Pf+R0y6qzXOPGjcPOnTvx4osvVqtz9uxZXLt2DbNmzaq2MEm/fv3w3XffQaFQPPJY/03Ix44di0mTJmHVqlV455130KVLFwCP/14RERERaYIJF9WbqilbBQUFdd6Xm5sbgoKC1G7bsGEDLC0t4enpiaSkJACVX549PDywcePGagmXmZkZdu3aBQC4ffs2Pv74Y2RlZVVb9bA2zM3Nq13rBEC51HzVvqv+rKnuo2LIyMhQeW5ra/vINlXJallZGX7//XcsWrQIxcXFKgnMtWvXcOnSpWqJaJWsrCwAlWcOV69ejVdffRUzZsxAv379MGzYMAwfPrzGhOiVV15RnuUaOnRote3Xrl0DAIwbN67G15Cfn6/RdM93330Xq1atwoEDB5QJ1+O+V0RERESaYMJF9cbGxgYuLi44f/68zo4hCAI2b96MoqIitGnTptr2rKwsFBYWwsrKSllmZGSkkryFhITA19cXr732Gn7++WeN4nB2dkZaWlq18qophFVLk1dNW6sq/29ddUuY//c4D3pwWmVNHkxWn3vuOUilUkRERKBv377K66oUCgXat2+P2NhYtfuouqbJ3NwcR44cwaFDh7B7927s3bsXW7duxbPPPov9+/fDyMioWtuqs1zjx4/HTz/9VG27QqEAACxZsgQdO3ZUe/wH37/aqIr77t27yrLHfa+IiIiINMGEi+rVCy+8gK+//hqJiYno2rWr1vd/+PBh3L59G/Pnz0fr1q1Vtt27dw+TJk3Czp07MWbMmBr34ezsjHfeeQfz5s3DiRMnlGdCaqNjx444dOgQZDKZymIMJ0+eVG4HKq+JMjY2xh9//IGRI0cq65WVleHs2bMqZer89+a8VdMqa+O1117Dp59+ilmzZimvq/Ly8sJff/2Ffv36PXJapVgsRr9+/dCvXz/ExsZi0aJF+OCDD3Do0KEaz0KOGTMGCxcuxLx58zB48GCVbVWLd9jY2NTYvkptp3xWrTr54Jm7x32viIiIiDTBZeGpXr3//vuwtLTEq6++iszMzGrbk5OTsXz5co33XzWdcNq0aRg+fLjKY+LEifDx8al2M2N13nzzTVhYWGDx4sUaxTF8+HBUVFSo3MC5tLQU69atQ2BgoPJMi62tLYKCgrBhwwaVqZbfffcdCgsLMWLEiIceJygoSOWhbvGHRzE2Nsa7776LS5cuKc84jRw5EmlpaVi1alW1+vfv30dRUREA1TNFVaoSFHXT9KpUneU6e/ZstbOI/v7+8PLywtKlS9Uu556dna38u6WlJQAgLy9PpY5MJqt2fEEQlDe/fvAGz4/7XhERERFpgme4qF55eXlh06ZNCA0NRevWrREWFoZ27dqhrKwMx48fx/bt2x85Ja4mpaWl2LFjB/r3769c8OC/Bg8ejOXLlyMrKwsODg417qtZs2YIDw/HF198gUuXLinPlh05cgRHjhwBUPnFv6ioSPklvlevXujVqxcAIDAwECNGjEBUVBSysrLg7e2Nb775BikpKdUWc/jwww/RrVs39O7dG5MmTcLt27fxySefIDg4GAMGDNCoL2pr/PjxiI6OxkcffYShQ4di7Nix2LZtGyZPnoxDhw6he/fuqKiowOXLl7Ft2zbs27cPnTp1wvz583HkyBE8//zzaNGiBbKysvDFF1/Azc1NZXVDdaqu5Tp79qxKuVgsxurVqzFw4EC0bdsW4eHhcHV1RVpaGg4dOgQbGxvlNXf+/v4AgA8++ACjRo2CiYkJBg0ahDNnzmD06NEYPXo0vL29cf/+ffz444/47bffMGnSJDzzzDPK49XmvSIiIiKqNX3edZmeXFevXhUmTpwoeHh4CKampoK1tbXQvXt34bPPPhNKSkqU9Vq0aCE8//zzKm0BCFOmTKm2zx07dggAhDVr1tR43ISEBAGAsHz5ckEQBGHcuHGCpaWl2rrJycmCkZGRMG7cOGXZnDlzBABqH3PmzFFpf//+feG9994TnJycBIlEInTu3FnYu3ev2mMdPXpU6Natm2BmZibY29sLU6ZMEWQyWY2vQ1M19Z0gCMLcuXMFAMKhQ4cEQRCEsrIy4aOPPhLatm0rSCQSoUmTJoK/v78wb948IT8/XxAEQYiPjxeGDBkiuLi4CKampoKLi4swevRo4erVq8r93rhxQwAgLFmypNox161bp+y/7OxslW1//vmnMGzYMKFZs2aCRCIRWrRoIYwcOVKIj49XqbdgwQLB1dVVEIvFAgDhxo0bwvXr14URI0YIHh4egpmZmWBhYSH4+/sLK1euFBQKRbU4avNeEREREdWGSBAEQQ95HhERERERUaPHa7iIiIiIiIh0hAkXERERERGRjjDhIiIiIiIi0hEmXERERI9w5MgRDBo0CC4uLhCJRNi5c+cj2yQkJOCZZ56BRCKBt7c31q9fr/M4iYjI8DDhIiIieoSioiL4+flhxYoVj1X/xo0beP7559G3b1+cPXsWU6dOxauvvop9+/bpOFIiIjI0XKWQiIioFkQiEX788UcMHTq0xjrTp0/H7t27cf78eWXZqFGjkJeXh71799ZDlEREZCh442MACoUC6enpsLa2hkgk0nc4RERPFEEQUFBQABcXF4jFjWPiRWJiIoKCglTKQkJCMHXq1BrblJaWorS0VPlcoVDg7t27aNasGccmIqJ6pO1xyWATrhUrVmDJkiXIyMiAn58fPvvsMwQEBDyy3ZYtWzB69GgMGTLksebYA0B6ejrc3d3rGDEREdXFrVu34Obmpu8wtCIjIwOOjo4qZY6OjpDJZLh//z7Mzc2rtYmJicG8efPqK0QiInoEbY1LBplwbd26FZGRkVi5ciUCAwOxbNkyhISE4MqVK3BwcKixXUpKCt577z307NmzVseztrYGUNmpNjY2tY5XLpdj//79CA4OhomJSa3bNzbsD+1if2oX+1P76tqnMpkM7u7uyv+Ln1RRUVGIjIxUPs/Pz0fz5s1x48aNOvWNXC7HoUOH0LdvX37mH8B+0S32r26xf3Xr7t27aNWqldbGJYNMuGJjYzFx4kSEh4cDAFauXIndu3dj7dq1mDFjhto2FRUVeOWVVzBv3jwcPXoUeXl5j328qqkaNjY2GidcFhYWsLGx4Yce7A9tY39qF/tT+7TVp41p2pyTkxMyMzNVyjIzM2FjY6P27BYASCQSSCSSauVNmzbVaGyqUvX+NGvWjJ/5B7BfdIv9q1vs3/qhrXHJ4BKusrIynD59GlFRUcoysViMoKAgJCYm1thu/vz5cHBwwIQJE3D06NGHHuO/8+RlMhmAyg+vXC6vdcxVbTRp2xixP7SL/ald7E/tq2ufNsb3omvXrtizZ49KWVxcHLp27aqniIiISF8MLuHKyclBRUWF2rnvly9fVtvm2LFjWLNmDc6ePftYx6hpnvz+/fthYWFR65irxMXFady2MWJ/aBf7U7vYn9qnaZ8WFxdrORLtKywsRFJSkvL5jRs3cPbsWTRt2hTNmzdHVFQU0tLS8O233wIAJk+ejM8//xzvv/8+/ve//+HgwYPYtm0bdu/era+XQEREemJwCVdtFRQUYOzYsVi1ahWkUuljtfnvPPmq6weCg4M1nlIYFxeH/v3787Qu2B/axv7ULvZn3eUVy3EjpwjXc4pwI6cYydkFkOVmYf3r/TS+hsvQ/fHHH+jbt6/yedUYMm7cOKxfvx537txBamqqcnvLli2xe/duvPPOO1i+fDnc3NywevVqhISE1HvsRESkXwaXcEmlUhgZGamd++7k5FStfnJyMlJSUjBo0CBlmUKhAAAYGxvjypUr8PLyUmlT0zx5ExOTOn0Bq2v7xob9oV3sT+1ifz6cvEKBW3eLkZxdhOvZhbieXYTrOZV/5haVVatvbSLSuE8bwvvQp08fPOy2levXr1fb5s8//9RhVERE1BAYXMJlamoKf39/xMfHK28qqVAoEB8fj4iIiGr1fX19ce7cOZWyWbNmoaCgAMuXL+dy70RED3G3qAzXswuR/E9SlfxPYpWaW4xyRc0Jxn8VlwOFpeVo0gCSJyIiovpkcAkXUDlVY9y4cejUqRMCAgKwbNkyFBUVKVctDAsLg6urK2JiYmBmZoZ27dqptLezswOAauVERE+qnMJSXMssxLWsApU/1Z2tehgHawk87S3haW8FT6klvBys0NxOgr8TE2AlMcghhYiISK8McnQMDQ1FdnY2oqOjkZGRgY4dO2Lv3r3KhTRSU1O1ctdnIqLGRBAE5BSW4VpmAa5lFeLqP38mZRXibi0SK4mxGC2llvCyt4Kn/b9/tpRawtqs+hksuVyO841nRXciIiKtMsiECwAiIiLUTiEEgISEhIe2VTeXnoioMckvluNCej6u/JNUVSVZecWPv8S6vbUEPg5W8LK3glfVWSt7S7jYmkMsZgZFRESkDQabcBERUaXcwlKcT5fhfFp+5SM9H7fu3n/s9g7WErRytIa3gxVaOVrDx9EKPg5WsLMw1WHUREREBDDhIiIyKJmykn8SKxnOpeXjQno+7uSXPFZbJxuzf5Kpf5MqHwdr2FpwIQsiIiJ9YcJFRKQHgiAgPb8E525XJlWVZ65kyC4ofWRbcxMjtHGxQXtXW/g6WcPnn7NXtuZMrIiIiAwNEy4iIh2TVyiQlFWIi+kyXLojw8V/Ho9zvZWVxBhtXWzQztUW7Vwrk6yWUisY8RorIiKiBoEJFxGRFuUXy3HxzgOJVboMSVmFKKtQPLKtrbkJ2rn+k1y52KKdqy1aNLXgAhZEREQNGBMuIiINCIKAW3fvK89WVZ29Sst7vMUspFaSf6YF2iiTK7cm5hCJmFwRERE1Jky4iIgeQ5asBGdS83D2Vh7+TL2Hi+kyFJSWP7KdWAR42luhjbMNWjvboI2LDVo7W8PB2qweoiYiIiJ9Y8JFRPQfJfIKnE/Lx58PJFjpj7FSoKWpEVo/kFi1cbZBK0drmJsa1UPUREREZIiYcBHRE00QBKTkFuPP1Hv/JFd5uHRHhnKF8NB2zrZmaPNAYtXa2QbNeb0VERER/QcTLiJ6ohSUlONSngjJB5Pxd7oMZ2/lPXK1QAtTI3Rws0VH9ybo6G6Hp5vbwdGGUwKJiIjo0ZhwEVGjVl6hwF+383Dkag6OJeXg7K08VCiMgEvJauuLRIC3vRWebm6Hju5N8HRzO/g4WMHYSFzPkRMREVFjwISLiBoVQRBwI6cIx5JycPRaDk4k5z50cYtmlqbKs1Yd3Zugg7stbMx4A2EiIiLSDiZcRNTg3Ssqw2/JOTh2rTLJetjS7J5SS7gaFeDFXn7o5CGFe1MuxU5ERES6w4SLiBqc0vIKnE65h6NJlUnW+fR8CDWscdHU0hTdvaXo6SNFD28p7C2NsWfPHjzXwRkmJjyTRURERLrFhIuIDJ5CIeBShgzHk3JxLCkHp27cxX15hdq6psZidPZogp4+9ujhLUUbZxuVlQPl8ocvkEFERESkTUy4iMjgCIKA1LvF+C0pF78l5yAxORd3i8pqrN/a2UZ5BiugZVOYmfC+V0RERGQYmHARkUHILijF8eQc5Vmsh12H5WgjQQ9ve/T0kaK7txT21pJ6jJSeVCtWrMCSJUuQkZEBPz8/fPbZZwgICKix/rJly/Dll18iNTUVUqkUw4cPR0xMDMzMeEsBIqInCRMuItKLghI5Tt24W3kWKykHVzILaqxrLTFGF69m6O7VDN29pfB2sOJCF1Svtm7disjISKxcuRKBgYFYtmwZQkJCcOXKFTg4OFSrv2nTJsyYMQNr165Ft27dcPXqVYwfPx4ikQixsbF6eAVERKQvTLiIqF6UlStwJvUejidV3g/rr9v5qFCoX+nC1FiMTi2aoLu3FN28mqG9qy3vg0V6FRsbi4kTJyI8PBwAsHLlSuzevRtr167FjBkzqtU/fvw4unfvjpdffhkA4OHhgdGjR+PkyZP1GjcREekfEy4i0pmM/BIkXMnCoStZ+C0pF4U13A9LJAI6uNqim7cU3b2k6OTRhNdhkcEoKyvD6dOnERUVpSwTi8UICgpCYmKi2jbdunXDhg0bcOrUKQQEBOD69evYs2cPxo4dW+NxSktLUVpaqnwuk8kAVC70UpfFXqracsEYVewX3WL/6hb7V7e03a9MuIhIa8orFDiTmvdPkpWNS3dkNdb1srf85wyWFF09m8HWgku0k2HKyclBRUUFHB0dVcodHR1x+fJltW1efvll5OTkoEePHhAEAeXl5Zg8eTJmzpxZ43FiYmIwb968auX79++HhYVF3V4EgLi4uDrvozFiv+gW+1e32L+6UVxcrNX9MeEiojrJLijF4avZOHQlC0evZkNWov4sVhMLE/RuZY+ePvbo7i2Fky0XDqDGKyEhAYsWLcIXX3yBwMBAJCUl4e2338aCBQswe/ZstW2ioqIQGRmpfC6TyeDu7o7g4GDY2NhoHItcLkdcXBz69+/Pe889gP2iW+xf3WL/6lZubq5W98eEi4hqpUIh4K/beUi4XHkW61xafo11O7jZos9TDuj7lD06uNnBSMyFLqjhkUqlMDIyQmZmpkp5ZmYmnJyc1LaZPXs2xo4di1dffRUA0L59exQVFWHSpEn44IMPIBZXvyZRIpFAIqm+4qaJiYlWvlBpaz+NDftFt9i/usX+1Q1t9ykTLiJ6JFmJHAcvVV6LdeRqNu4Vq5/bbGNmjF6t7NH3KQf0amXP5dqpUTA1NYW/vz/i4+MxdOhQAIBCoUB8fDwiIiLUtikuLq6WVBkZVV6XKAjqF4shIqLGSaOE6+TJkwgMDNR2LERkQIrLynHgUhZ++SsdCVezUVauUFuvjbMN+vrao89TDnja3Y6rCZLe6HJsioyMxLhx49CpUycEBARg2bJlKCoqUq5aGBYWBldXV8TExAAABg0ahNjYWDz99NPKKYWzZ8/GoEGDlIkXERE9GTRKuLp27Yr27dtj4sSJGDNmDOzs7LQcFhHpQ4m8AglXsrHr73QcvJSF+/KKanWsJMbo6SNFn6fs0buVA6/FIoOhy7EpNDQU2dnZiI6ORkZGBjp27Ii9e/cqF9JITU1VOaM1a9YsiEQizJo1C2lpabC3t8egQYPw4Ycfai0mIiJqGDRKuMaMGYMdO3bgrbfewvvvv4/hw4dj4sSJ6Nmzp7bjIyIdKytX4FhSNn756w72X8xUu3S71EqCFzo4I7itIzq1aApTY57FIsOj67EpIiKiximECQkJKs+NjY0xZ84czJkzRyvHJiKihkujb03ffvst0tPT8dlnn8HX1xcbNmxAnz594Ovri08++QQ5OTnajpOItKi8QoFj13Iw/fu/0fnDA/jf+j/ww59pKslWEwsTjA5ojk0TA3FyZj/MHdwW3bykTLbIYHFsIiIiQ6TxNydbW1tMmTIFZ86cwR9//IFJkyYhMzMT06ZNg5ubG0JDQ3HgwAFtxkpEdaBQCDh5PRezd55Hl5h4jFlzElv/uIX8+/8ugGFtZozh/m5YH94Zpz4IQsyw9ujmJeXqgtRgcGwiIiJDo5Wfqp955hl8+eWXSE9Px/r16yGVSvH9998jJCQEnp6e+Pjjj1FQUKCNQxFRLeQUlmLv+QzM/fkCui0+iNCvT+C7EzeRU1imrGNhaoTBfi5YFdYJf8wKwtIRfujzlANMuPgFNXAcm4iIyBBobVn4e/fu4dtvv8Xq1auRnp4OkUiE7t2749KlS5gxYwaWLVuGn376CZ07d9bWIYnoAYIgICW3GL+n3MUfKXfxR8o9XM8pUltXYizGs74OeKGDC571dYC5KVdNo8aJYxMREelbnROuQ4cOYdWqVdi5cydKSkpgb2+PadOm4bXXXoOnpydKS0uxdu1avP/++3jzzTdx4sQJbcRN9MSTVyhwMV32T4J1D3/cvKty5uq/TIxE6OVjj0F+Lghq4wgrCW/DR40XxyYiIjIUGn3jyszMxLp167BmzRpcv34dgiCgd+/emDx5MoYNG6Zyd2aJRILXX38dSUlJWLFixWMfY8WKFViyZAkyMjLg5+eHzz77DAEBAWrr/vDDD1i0aBGSkpIgl8vh4+ODd999F2PHjtXk5REZpMLScpxPycPvKffwR8pd/Jmap3bZ9iomRiJ0cLNDJ48m6NyiKTq3bApbc96Nnhqv+hibiIiIakujhMvNzQ0KhQJNmjTB1KlTMWnSJDz11FMPbWNvb4+yspp/fX/Q1q1bERkZiZUrVyIwMBDLli1DSEgIrly5AgcHh2r1mzZtig8++AC+vr4wNTXFL7/8gvDwcDg4OCAkJESTl0ikd4Ig4M9bedj9Vxr2/W2Ed04chEKoub61mTH8WzRBZ4+m6OzRFB3cbGFmwqmC9OTQ9dhERESkCY0SrsDAQEyePBkjRoyARCJ5rDYzZszAjBkzHqtubGwsJk6ciPDwcADAypUrsXv3bqxdu1btPvr06aPy/O2338Y333yDY8eOMeGiBqUqydrz9x3sOXcH6fkl/2ypvkqgs63ZP8lVE3TyaIpWjtZcTZCeaLoem4iIiDShUcJ17NgxbcehVFZWhtOnTyMqKkpZJhaLERQUhMTExEe2FwQBBw8exJUrV/DRRx+prVNaWorS0lLlc5lMBgCQy+WQy+Vq2zxMVRtN2jZG7I/aEQQBf93Ox94Lmfj1fOYDSda/RBDg42CFTh5N4N+8CTq1sIOLnblKHUVFORQ1zzCkf/DzqX117VNtvRe6HJuIiIg0pVHCdfv2bZw5cwa9evWCnZ1dte337t3D0aNH4e/vD1dX11rtOycnBxUVFXB0dFQpd3R0xOXLl2tsl5+fD1dXV5SWlsLIyAhffPEF+vfvr7ZuTEwM5s2bV618//79sLCwqFW8D4qLi9O4bWPE/qiZIACphcCfuWL8dVeEu6XVz0yJRQKeshXQsZmA9k0EWJrkA8gH0lJwNg04W+9RNy78fGqfpn1aXFyslePrcmwiIiLSlEYJ18KFC7F9+3akp6er3W5hYYH//e9/GDVqFD7//PM6Bfi4rK2tcfbsWRQWFiI+Ph6RkZHw9PSsNt0QAKKiohAZGal8LpPJ4O7ujuDgYNjY2NT62HK5HHFxcejfv7/KRdlPKvaHeoIg4O80GX49n4G9FzKRllf9TJaxWIRuXk0xsJ0TgnwdYGdhwv7UMvan9tW1T6tmGdSVIY5NREREGiVcBw8eRHBwcI1z5CUSCYKDg3HgwIFa71sqlcLIyAiZmZkq5ZmZmXBycqqxnVgshre3NwCgY8eOuHTpEmJiYtQmXBKJRG3sJiYmdfoCVtf2jQ37458k63Y+9py7g93n7uD2vfvV6hiLRejuLcXz7Z0R3NYRdhamavfF/tQu9qf2adqn2nofdDk2ERERaUqjhCstLQ0vvfTSQ+u0aNECu3btqvW+TU1N4e/vj/j4eAwdOhQAoFAoEB8fj4iIiMfej0KhULlOi6g+peYW44c/b+PHP9NwM7f6dCljsQjdvKV44RFJFhE9Pl2OTURERJrSKOEyNTV95BQQmUwGkUizFdMiIyMxbtw4dOrUCQEBAVi2bBmKioqUqxaGhYXB1dUVMTExACqvyerUqRO8vLxQWlqKPXv24LvvvsOXX36p0fGJNCErkePXc3ew43QaTqXcrbbdSHkmywnBbZzQxJJJFpE26XpsIiIi0oRGCVf79u2xa9cuxMbGqp26UVJSgp9//hnt27fXKKjQ0FBkZ2cjOjoaGRkZ6NixI/bu3atcSCM1NRVisVhZv6ioCG+88QZu374Nc3Nz+Pr6YsOGDQgNDdXo+ESPq0Ih4FhSDnacvo19FzJQWq5Q2S4SAd29pBjk58wki0jHdD02ERERaUKjhCs8PBwTJkzA4MGD8eWXX8LT01O5LTk5GW+88QbS09Mxf/58jQOLiIiocQphQkKCyvOFCxdi4cKFGh+LqLauZhZgx+nKKYNZBdWnrno7WOGlZ9zw4tOucLI100OERE+e+hibiIiIakvjhGvPnj3YsWMHfH190bJlS7i6uiItLQ03btxAeXk5QkNDlVMAiRqD3MJS/PxXOnacuY3zadWnLTWxMMFgPxe85O+G9q62nLZEVM84NhERkSHSKOECgG3btmHFihX44osvcPnyZVy7dg0A0KZNG0yZMgWvv/661oIk0pfS8gocupyF70+nIeFKFsoVgsp2Y7EIz/o6YNgzbnjW1wGmxuIa9kRE9YFjExERGRqNEy6RSKSc9ldUVIT8/HzY2trC0tJSm/ER6UVSViE2nLiJnWfTkFcsr7a9g5sthj3tisEdXdGU12URGQyOTUREZGi08nO8paUlXFxcOKBRg1ZeocC+Cxl4ZfUJBMUexvrjKSrJlqONBK/19sT+d3rh54geGN+9JZMtIgOm7bFpxYoV8PDwgJmZGQIDA3Hq1KmH1s/Ly8OUKVPg7OwMiUSCVq1aYc+ePVqJhYiIGg6Nz3ARNRY5haXY+vstbDxxE+n5JSrbJMZiDGjnhJeecUN3bymMxLwui+hJtHXrVkRGRmLlypUIDAzEsmXLEBISgitXrsDBwaFa/bKyMvTv3x8ODg74/vvv4erqips3b8LOzq7+gyciIr3SOOG6desWFi5ciAMHDiA9PR1lZWXV6ohEIpSXl9cpQCJdEAQBZ1Lz8G1iCvacuwN5heq1WS2aWWBslxYY4e8OWwsTPUVJRLWlq7EpNjYWEydOVC64sXLlSuzevRtr167FjBkzqtVfu3Yt7t69i+PHj8PEpPL/EA8Pj9q/ICIiavA0SriuX7+OwMBA3Lt3D23btkVpaSlatGgBMzMzXL9+HXK5HH5+fvwljwzO/bIK/PxXGr5NvIkL6aorDYpEwLNPOWBs1xbo5WMPMc9mETUouhqbysrKcPr0aURFRSnLxGIxgoKCkJiYqLbNzz//jK5du2LKlCn46aefYG9vj5dffhnTp0+HkZGR2jalpaUoLf33NhNVN3GWy+WQy6tfS/q4qtrWZR+NEftFt9i/usX+1S1t96tGCde8efOQn5+P+Ph49O7dG2KxGOHh4YiOjsadO3fw+uuv4+LFizhw4IBWgyXSVEpOETacuIntp28j/77qPyI7CxOEdnLHmC4t4N7UQk8RElFd6WpsysnJQUVFBRwdHVXKHR0dcfnyZbVtrl+/joMHD+KVV17Bnj17kJSUhDfeeANyuRxz5sxR2yYmJgbz5s2rVr5//35YWNT9/6a4uLg676MxYr/oFvtXt9i/ulFcXKzV/WmUcB04cADPPfccevfurSwThMopWc7Ozti6dSvat2+PmTNn4quvvtJOpES1VKEQkHAlC98m3sThq9nVtndws8XYLi0wyM8FZibqf3EmoobDkMYmhUIBBwcHfP311zAyMoK/vz/S0tKwZMmSGhOuqKgoREZGKp/LZDK4u7sjODgYNjY2Gscil8sRFxeH/v37K6c3EvtF19i/usX+1a3c3Fyt7k+jhCsnJwe+vr7/7sTYWCUTlEgk6N+/P3bu3FnnAIlqq7xCgc2/38LXR5Jx6+59lW2mxmK80MEZYV090NHdTj8BEpFO6GpskkqlMDIyQmZmpkp5ZmYmnJyc1LZxdnaGiYmJyvTB1q1bIyMjA2VlZTA1rb7CqUQigUQiqVZuYmKilS9U2tpPY8N+0S32r26xf3VD232qUcIllUpRVFSk8jwlJUV1x8bGyMvLq0tsRLWWcCULH+6+hGtZhSrlrnbmGNOlBUI7u3Mpd6JGSldjk6mpKfz9/REfH4+hQ4cCqDyDFR8fj4iICLVtunfvjk2bNkGhUEAsrrwDy9WrV+Hs7Kw22SIiosZLo4TLx8cHycnJyucBAQHYt28frl+/Dk9PT2RnZ+P777+Hl5eX1gIlephrmQVYuPtStamDvVrZI6xLC/T1deCS7kSNnC7HpsjISIwbNw6dOnVCQEAAli1bhqKiIuWqhWFhYXB1dUVMTAwA4PXXX8fnn3+Ot99+G2+++SauXbuGRYsW4a233tLOiyUiogZDo4Rr4MCBmDt3LvLy8mBnZ4epU6di165d6NChA1q3bo2kpCTIZDLMnTtXy+ESqbpbVIZP465i06lUVCj+Xdr96eZ2mPV8G/i3aKLH6IioPulybAoNDUV2djaio6ORkZGBjh07Yu/evcqFNFJTU5VnsgDA3d0d+/btwzvvvIMOHTrA1dUVb7/9NqZPn66tl0tERA2ERgnX66+/jj59+ijnpvfp0wdbtmzB3Llzcf78ebRo0QILFy7ExIkTtRosUZXS8gp8e/wm/u/gNRSU/Hs/HRdbM0wf6IvBfi4QiXhGi+hJouuxKSIiosYphAkJCdXKunbtihMnTmh0LCIiajw0SrhsbGwQGBioUjZixAiMGDFCK0ER1UQQBOy7kImYXy/hZu6/F8NbmBrhjT5eeLWnJ1ccJHpCcWwiIiJDpFHC9eyzz6J79+5YsGCBtuMhqtH5tHws+OUiTt64qywTiYCR/u54N7gVHGzM9BgdEekbxyYiIjJEGiVcJ0+eRJcuXbQdC5FaWbISLNl3Bd+fuQ3h38u00NWzGWa90BptXWz1FxwRGQyOTUREZIg0Srh8fX1x8+ZNbcdCpOJ+WQVWHb2OlYeTUVxWoSz3aGaBmc+1Rv82jrxOi4iUODYREZEh0ijhevPNNxEREYGLFy+iTZs22o6JnnAKhYCf/0rHR3sv405+ibLcxswYb/XzQVhXD5gaix+yByJ6EnFsIiIiQ6RRwuXp6Yk+ffqgS5cueO2119C5c2c4Oqo/29CrV686B0lPBkEQcPhqNpbuv4LzaTJluZFYhDGBzfF2UCvetJiIasSxiYiIDJFGCVefPn0gEokgCAI++eSTh07rqqioqHEbUZVTN+5i6b4rOJVyV6W871P2+OD51vB2sNZTZETUUHBsIiIiQ6RRwhUdHc1rZ0grzt3Ox9L9V3D4arZKeVsXG0wf4Iterez1FBkRNTQcm4iIyBBplHDNnTtXy2HQkyYpqwCxcVex51yGSrmXvSXeDX4KA9o6QSzmFycienwcm4iIyBBplHARaerW3WIsj7+GH87chuKBJd5d7cwxNcgHLz7tCmMjLohBRERERI0DEy6qF1myEnx+KAmbT6VCXvFvpiW1kuCtft4I7ewOibGRHiMkIiIiItI+jRIusVj8WPPkRSIRysvLNTkENRJ5xWVYefg61h+/gRK5Qllua26Cyb29MK5bC1iYMu8norrj2ERERIZIo2+6vXr1Ujuo5efn49q1aygqKoKfnx/s7OzqGh81UCUVwOeHkrH2t5soKP33i42FqREm9GiJV3t6wtbcRI8RElFjw7GJiIgMkUYJV0JCQo3biouLMWPGDOzduxdxcXGaxkUNVGl5Bb45fhPLzxihqDxZWW5qJMaYLi3wRl8vSK0keoyQiBorjk1ERGSItL46gYWFBf7v//4Ptra2mDZtmrZ3TwZKEATsPZ+B/rFHsOjXKygqr/yV2UgswugAdyRM64PoQW2YbBGRXnBsIiIifdHZxTM9e/bEhg0bdLV7MiAX02VY8MtFJF7PVSl/ob0T3g3xRUuppZ4iIyJSxbGJiIjqm84SruzsbBQWFupq92QAcgpL8cn+q9j6e6rKEu9dWjZBT+tsTBrRASYmvE6LiAwHxyYiIqpvWk+4FAoFNm7ciK1bt6JTp07a3j0ZgLJyBb45noL/i7+msiBG86YWmPlcazzbqil+/fVXPUZIRKSKYxMREemLRgmXp6en2vLy8nJkZWVBLpfDxMQEMTExdQqODIsgCIi/lIUP91zCjZwiZbmVxBgRz3ojvLsHJMZGkMvleoySiJ5UHJuIiMgQabRohkKhgCAI1R4mJiZo164dJk2ahNOnT6N3794aB7ZixQp4eHjAzMwMgYGBOHXqVI11V61ahZ49e6JJkyZo0qQJgoKCHlqfau9qZgHC1p7Cq9/+oUy2RCIgtJM7Dr7XG5N7e/HGxUSkV/UxNhEREdWWRme4UlJStByGqq1btyIyMhIrV65EYGAgli1bhpCQEFy5cgUODg7V6ickJGD06NHo1q0bzMzM8NFHHyE4OBgXLlyAq6urTmNt7O4VleHTA1ex8WQqKh64UCvAoymiB7VBO1dbPUZHRPQvXY9NK1aswJIlS5CRkQE/Pz989tlnCAgIeGS7LVu2YPTo0RgyZAh27typ0xiJiMjw6GzRjLqIjY3FxIkTER4eDgBYuXIldu/ejbVr12LGjBnV6m/cuFHl+erVq7Fjxw7Ex8cjLCysWv3S0lKUlpYqn8tkMgCAXC7XaDpcVZvGNJVOXqHAxlO38NnBZMhK/r1Oy9XODNNDWmFAW0eIRCK1r7kx9oc+sT+1i/2pfXXt04bwXtT2h8AqKSkpeO+999CzZ896jJaIiAyJRgnX7du3cebMGfTq1Qt2dnbVtt+7dw9Hjx6Fv79/rc8wlZWV4fTp04iKilKWicViBAUFITEx8bH2UVxcDLlcjqZNm6rdHhMTg3nz5lUr379/PywsLGoV74May800L94TYedNMTLvi5RlpmIB/V0V6ONcCCH1DH5NffR+Gkt/GAr2p3axP7VP0z4tLi7WyvF1OTbV9odAAKioqMArr7yCefPm4ejRo8jLy6vtSyIiokZAo4Rr4cKF2L59O9LT09Vut7CwwP/+9z+MGjUKn3/+ea32nZOTg4qKCjg6OqqUOzo64vLly4+1j+nTp8PFxQVBQUFqt0dFRSEyMlL5XCaTwd3dHcHBwbCxsalVvEDlr7NxcXHo379/g14G/fa9+5i76xIOX8tRKX+xozPe7e8DRxuzx9pPY+kPQ8H+1C72p/bVtU+rZhnUla7GJk1/CJw/fz4cHBwwYcIEHD169JHH0fbsiyo8q6se+0W32L+6xf7VLW33q0YJ18GDBxEcHAyJRKJ2u0QiQXBwMA4cOFCn4DSxePFibNmyBQkJCTAzU58gSCQStbGbmJjU6QtYXdvriyAI2H76NubvuojCB5Z592/RBNEvtIGfu51G+22o/WGo2J/axf7UPk37VFvvg67GJk1+CDx27BjWrFmDs2fPPvZxdDX7ogrP6qrHftEt9q9usX91Q1szL6polHClpaXhpZdeemidFi1aYNeuXbXet1QqhZGRETIzM1XKMzMz4eTk9NC2S5cuxeLFi3HgwAF06NCh1sd+EuUUliLqh3OIu/hvfzvbmmHGQF8M9nOBSCR6SGsiIsOhy7GpNgoKCjB27FisWrUKUqn0sdtpe/ZFFZ7VVY/9olvsX91i/+pWbm6uVvenUcJlamr6yCkgMplMoy/rpqam8Pf3R3x8PIYOHQqgcqnf+Ph4RERE1Nju448/xocffoh9+/bxppaPad+FDMz84Rxyi8qUZcP93RA9qA1szPiPl4gaFl2NTbX9ITA5ORkpKSkYNGiQskyhUAAAjI2NceXKFXh5eVVrp6vZF9reT2PDftEt9q9usX91Q9t9qtF9uNq3b49du3apzDV/UElJCX7++We0b99eo6AiIyOxatUqfPPNN7h06RJef/11FBUVKS9WDgsLU5lL/9FHH2H27NlYu3YtPDw8kJGRgYyMDBQWFmp0/MZOViLHu9v+wmvfnVYmW80sTfHVWH8sHeHHZIuIGiRdjU0P/hBYpeqHwK5du1ar7+vri3PnzuHs2bPKx+DBg9G3b1+cPXsW7u7utXthRETUoGmUcIWHh+P27dsYPHgwrl+/rrItOTkZQ4YMQXp6Ol599VWNggoNDcXSpUsRHR2Njh074uzZs9i7d69y/nxqairu3LmjrP/ll1+irKwMw4cPh7Ozs/KxdOlSjY7fmB1PzsHAZUex48xtZVn/No7Y904vhLR9+JRNIiJDpsuxqTY/BJqZmaFdu3YqDzs7O1hbW6Ndu3YwNTWt+4slIqIGQ6MpheHh4dizZw927NgBX19ftGzZEq6urkhLS8ONGzdQXl6O0NBQ5UCkiYiIiBqnECYkJKg81/XNLhuDEnkFPt57BWt/u6Ess5IYY86gNhju78ZrtYiowdPl2BQaGors7GxER0cjIyMDHTt2rPZDoFis0W+YRETUyGl84+Nt27ZhxYoV+OKLL3D58mVcu3YNANCmTRtMmTIFr7/+utaCpLo5dzsf72w7i6Ssf6dYdvFsiqUj/ODWpO4rXxERGQpdjk21+SHwv9avX6/xcYmIqGHTOOESiUTKwaeoqAj5+fmwtbWFpaWlNuOjOpBXKPDFoWR8dvAayhUCAMDUWIzpA3wR3s0DYjHPahFR48KxiYiIDI3GCdeDLC0tOZgZmOTsQkRuPYu/bucry9q52uDTkR3h42itx8iIiOoHxyYiIjIEGk04/+233xAZGYmMjAy12+/cuYPIyEicOHGiTsFR7SkUAtb/dgPPLT+qTLaMxCK81c8HP77RnckWETVaHJuIiMgQaZRwxcbGYteuXTXeiNjZ2Rm//PILPv300zoFR7WTnncfY9eexNxdF1FaXnnPF0+pJXa83g2R/VvBxIgXdBNR48WxiYiIDJFGUwp///139OvX76F1evXqhbi4OI2Coto7cT0Xk779A7KScmXZ+G4emD7AF+amRnqMjIiofnBsIiIiQ6RRwpWVlQVXV9eH1nFyckJWVpZGQVHtHLiYiSmbzijPajnbmmHJcD/08JHqOTIiovrDsYmIiAyRRgmXnZ0dUlNTH1rn5s2bsLKy0igoenw7Tt/G+zv+RsU/qxD2ecoey0c9DVtzEz1HRkRUvzg2ERGRIdLoop4uXbrgxx9/xK1bt9RuT01Nxc6dO9GtW7c6BUcPt/bYDby7/S9lsjWkowtWhXViskVETySOTUREZIg0SrgiIyNRXFyM7t2749tvv8WdO3cAVK4A9c0336B79+64f/8+3n33Xa0GS5UEQUDs/iuY/8tFZdm4ri3w6ciOXBiDiJ5YHJuIiMgQaTSlsFevXoiNjcW7776L8PBwAJU3mxSEyjMtYrEYy5cvR69evbQXKQGoXPZ9zs8X8N2Jm8qyt/v5YGqQD0Qi3siY6o9cLkdFRYW+w6g1uVwOY2NjlJSUNMj4DdF/+9TIyAgmJvV/pp1jExERGSKNb3z89ttvo2/fvli5ciV+//135Ofnw87ODgEBAZg8eTLatWuH0tJSSCQSbcb7RCsrV+C97X/h57/SlWVzBrVBePeWeoyKnjQymQw5OTkoLS3VdygaEQQBTk5OuHXrFn+k0BJ1fSqRSCCVSmFjY1OvsXBsIiIiQ6NxwgUAHTp0wBdffFGt/MyZM5gyZQq2bNmC3NzcuhyC/nG/rAKvbzyNhCvZACpvZrx0RAe8+LSbniOjJ4lMJkNaWhqsrKwglUphYmLS4JIWhUKBwsJCWFlZQSzmFFxteLBPRSIR5HI58vPzkZaWBgD1nnRxbCIiIkNSp4TrQXl5ediwYQPWrFmDv//+G4IgwNzcXFu7f6Ll35djwvrf8cfNewAAibEYX7zyDPq1dtRzZPSkycnJgZWVFdzc3BpcolVFoVCgrKwMZmZmTLi05L99am5uDmtra9y+fRs5OTn1nnA9iGMTERHpW50TrgMHDmDNmjX46aefUFpaCkEQ0LVrV4SHhyM0NFQbMT7RsmQlCFt7CpczCgAA1hJjrB7XCYGezfQcGT1p5HI5SktLIZVKG2yyRfVHJBLB1tYWaWlpkMvl9X5NF8cmIiIyFBolXLdu3cK6deuwbt06pKamQhAEuLq6Ii0tDePHj8fatWu1HecTKTW3GGPWnETq3WIAgNTKFOvDA9DO1VbPkdGTqGqBCX0shkANU9VnpaKiol4+NxybiIjIED12wiWXy7Fz506sWbMG8fHxqKiogKWlJV555RWEhYXh2WefhbGxMYyNtTZL8Yl2OUOGsDWnkFVQuTCBq505NrwaiJZSSz1HRk86nt2ix1UfnxWOTUREZOgeewRycXHB3bt3IRKJ0LdvX4SFhWHYsGGwtGQCoG2nb95D+LpTkJWUAwB8HKzw7YQAONvyugMiogdxbCIiIkP32AlXbm4uxGIx3nnnHbz//vuwt7fXZVxPrMNXszH5u9O4L6+cvuXnbof14zujiaWpniMjIjI8HJuIiMjQPfYSXePHj4e5uTliY2Ph5uaGwYMHY/v27SgrK9NlfE+UX/5Ox6vf/K5Mtrp7N8PGVwOZbBER1YBjExERGbrHTrjWrl2LO3fu4KuvvsIzzzyDX375BaNGjYKjoyNee+01HDt2TJdxNnqbTqbizc1/Ql4hAAAGtnPC2vGdYSXhdQdETzKRSIQ+ffroOwyDxbGJiIgMXa1uQmNlZYVXX30ViYmJuHDhAqZOnQpTU1OsWrUKvXv3hkgkwpUrV3Dz5k1dxdsoJSbnYuaP5yBU5loI7eSOz19+BhJjI/0GRkQqUlJSIBKJHvrIy8ur15hu3rwJIyMjiEQiLFmypF6PbSjqa2xasWIFPDw8YGZmhsDAQJw6darGuqtWrULPnj3RpEkTNGnSBEFBQQ+tT0REjZfGd/1s3bo1PvnkE6SlpWHbtm0IDg6GSCTC0aNH4eXlhX79+uG7777TZqyNUom8Ah/8eE75fGLPllj8UnsYibkSHJGh8vLywpw5c9Q+zMzM6jWWtWvXQqFQQCQScdlz6G5s2rp1KyIjIzFnzhycOXMGfn5+CAkJQVZWltr6CQkJGD16NA4dOoTExES4u7sjODgYaWlpdX2JRETUwNR5vpqxsTGGDx+O4cOH4/bt21i3bh3Wr1+PQ4cOISEhAWPHjtVGnI3WFwnJuJ5TBADwb9EEUQNbc9ltIgPn7e2NuXPn6jsMKBQKrF+/HlKpFC+88ALWr1+P48ePo1u3bvoOTe+0PTbFxsZi4sSJCA8PBwCsXLkSu3fvxtq1azFjxoxq9Tdu3KjyfPXq1dixYwfi4+MRFham+QsjIqIGR6sXCLm5uWH27NmYPXs24uPj+WvrIyRlFeDLhCQAgLFYhEUvtoeYZ7aIGoW///4bixYtwuHDh5GbmwtnZ2cMHjwYc+fORbNmzarVX716NZYtW4akpCTY29tj9OjRmD9//kOPERcXh9TUVERERCA0NBTr16/HmjVrVBKuBQsWIDo6Gt98843aL/o//PADXnrpJcycORMffvihSvmiRYtw4cIF2NjYYPDgwfj444/x9NNPA6icXtlQ1HVsKisrw+nTpxEVFaUsE4vFCAoKQmJi4mPto7i4GHK5HE2bNq2xTmlpKUpLS5XPZTIZgMp7jcnl8lrF/KCqtnXZR2PEftEt9q9usX91S9v9qrMVGfr164d+/frpavcNnkIhYOYP55WLZLzW2xNPOVnrOSoi0oaff/4ZI0eOhFgsxpAhQ+Du7o6LFy/i888/x759+3Dy5Ek0adJEWb8qKXJ0dMTEiRNhYmKCrVu34tKlSw89zpo1awAAYWFh6Ny5Mzw9PbFt2zYsX74cVlZWAIAxY8Zgzpw52LBhg9qEq2p63YNnfNauXYsJEybAxsYGYWFhsLW1xZ49e9C/f3/I5XKYmJjUuY/0RZOxKScnBxUVFXB0dFQpd3R0xOXLlx9rH9OnT4eLiwuCgoJqrBMTE4N58+ZVK9+/fz8sLCxqFbM6cXFxdd5HY8R+0S32r26xf3WjuLhYq/vjEnh6su2PWziVchcA0KKZBd581kfPERHVzaDPjiG7oPTRFfXI3lqCn6bUfbpdUlKS2imFAwYMgI+PD8aOHQupVIrffvsNLVq0UG7fsmULRo8ejejoaHz22WfKfc2fPx+urq44c+YMHBwcAABz585FQEBAjTHk5ubip59+gq+vLzp37gygMrmaP38+tm7digkTJgAAWrZsie7du+PgwYO4c+cOnJ2dlfu4e/cu9uzZg06dOsHX1xcAkJeXh7fffhuWlpb4448/4ONT+X/TokWLEBISgtOnT6u8Jnq0xYsXY8uWLUhISHjoNX5RUVGIjIxUPpfJZMprv2xsbDQ+vlwuR1xcHPr379+gk2VtY7/oFvtXt9i/upWbm6vV/THh0oPsglIs2vPvL9cfDm0PMxOuSEgNW3ZBKTJkJfoOo14kJyerPRNhZ2eHxMREyGQyfP7559USk1GjRmHJkiXYsmWLMuHatGkTysvLERkZqUy2AMDGxgazZs2q8Vqj7777DmVlZSrbw8LCMH/+fKxZs0aZcAGVZ6+OHTuGzZs3q3yh37p1K8rKyjBmzBhl2U8//YTCwkK89dZbymQLqLwmauHChU/k9WFSqRRGRkbIzMxUKc/MzISTk9ND2y5duhSLFy/GgQMH0KFDh4fWlUgkkEgk1cpNTEy08oVKW/tpbNgvusX+1S32r25ou0+ZcOnBgl8uQlZSDgB48WlX9PCR6jkiorqzt67+RdHQaCvGkJAQ7N27V+220NBQAMDJkyeRnJxcbXtJSQlycnKQk5MDqVSKv/76CwDQs2fPanXVlVVZs2YNRCKRSrLk5eWFbt264fjx47h06RJat24NABg5ciTeeustfPfddyoJ14YNG2BsbIzRo0cry6ri6dGjR7VjBgYGwtj4yRs2TE1N4e/vj/j4eAwdOhRA5YIl8fHxiIiIqLHdxx9/jA8//BD79u1Dp06d6ilaIiIyNE/eyKlnh69m4+e/0gEAdhYmmPV8az1HRKQdu96s/gXdECkUCp3u/+7dyqnCK1aseGi9oqIiSKVS5OfnA4DK2a0q/71mqMrJkydx/vx59O3bF82bN1fZFhYWhuPHj2Pt2rXK+3LZ2dnhhRdewI4dO3Dx4kW0adMGycnJOH78OJ577jmVY1ct1KAuHrFYDKn0yfyBKDIyEuPGjUOnTp0QEBCAZcuWoaioSLlqYVhYGFxdXRETEwMA+OijjxAdHY1NmzbBw8MDGRkZACrvGVZ1fR0REdWdXC5HRUXFI+sZGRnp7WwgE656dL+sArN2/nvPrZnPtUYzK8M/K0BEj6/qWptz586hXbt2j6xva2sLAMjKyqo2BfG/U9iqVC2WcejQoRpvI/Htt99i0aJFysFl7Nix2LFjB7777jvExMRgw4YNynJ18au7v5RCoUBOTg5cXV0f+boam9DQUGRnZyM6OhoZGRno2LEj9u7dq0yKU1NTIRb/e2vLL7/8EmVlZRg+fLjKfubMmWMQtxQgImroZDIZcnJyVFZ3fRSJRAKpVFqn62I1wYSrHi2Pv4Zbd+8DALp4NsUIfzc9R0RE2hYYGIgffvgBiYmJj5Vw+fn54YcffsDRo0eVi19UOXr0aLX6RUVF2LJlCywsLFSmAj7o999/x99//41ffvkFL774IgDgueeeQ7NmzbBp0yZ8+OGH2LhxI6ytrTFkyJBq8QDAb7/9hhEjRqhsO3XqFMrLyx/5mhqriIiIGqcQJiQkqDxvSMvmExE1NDKZDGlpabCysoJUKoWJiclD72MrCALkcjny8/OVN6Cvz6RL/Ogq9W/FihXw8PCAmZkZAgMDcerUqRrrXrhwAS+99BI8PDwgEomwbNmy+gu0Fi7dkWHV0esAAFMjMT58sT1vcEzUCIWHh8Pa2hoffPABLly4UG17cXExTpw4oXz+8ssvw8jICLGxsSpnlWQyGRYuXFit/fbt21FQUIDhw4dj9erVah9VUwmrzoQBlRcAh4aGIjU1FR9//DGuXbuGl156Cebm5ir7HzJkCKysrLBmzRqVa9DKy8sxe/ZszTuGiIhIS3JycmBlZQU3NzfY2NjA3NwcZmZmNT7Mzc1hY2MDNzc3WFlZIScnp17jNbiEa+vWrYiMjMScOXNw5swZ+Pn5ISQkRO30FqDyy4unpycWL178yNWi9KVCISDqh3OoUFTec+uNvl7wsuccfqLGyN7eHps3b0ZhYSH8/Pzwwgsv4L333sObb76JQYMGwcnJSWVKmbe3N6Kjo5GWloYOHTrgrbfeQmRkJNq3b6+ySmCVqiSq6tohdYKCguDm5oa9e/ciPT1dWV41fTA6Olrl+YPs7OwQGxuLwsJC+Pv7Y/LkyZg+fTqefvpp3Lt3Dy4uLipT54iIiOqTXC5HaWkpbG1ta33yQiQSwdbWFqWlpfV602iDm1IYGxuLiRMnKr9MrFy5Ert378batWsxY8aMavU7d+6snIajbrs6paWlKvM9qy4Sl8vlGnX+o+72veFkKs7eygMAeEot8Wr3Fo36zuC8+7l2GUp/yuVyCIIAhUKh84UndEkQBOWftX0dVfUf1XbgwIE4ffo0li5divj4eMTFxcHS0hJubm4YP348XnnlFZX2s2bNgpOTE5YvX46vvvoKDg4OCA0Nxbx585QLLCgUCly5cgXHjh1Dy5Yt0bNnz4fGEBYWhkWLFmHdunWIiooCAAQEBMDHxwfXrl2Dm5sbevXqpXYfEyZMgK2tLRYvXoz169fD1tYWgwYNwuLFi9GyZUt4eXmptKupTxUKhXIah5FRzbe+0Pdnm4iIGo6qBTI0XQCjql1FRUW9LaJhUAlXWVkZTp8+rfxyAFSuihUUFITExEStHScmJkbtPXT2798PCwsLjfer7m7feaXA4r+MAFRm4M875iN+v/rlpBsb3v1cu/Tdn8bGxnByckJhYSHKysr0Gos2FBQU1LpN06ZNce/ePQD//lBTE2dnZ3zyySc1bv9v+5EjR2LkyJEqZXK5XOV4zs7OyuePin/atGmYNm1atWM9OEW7sLCwxvbBwcEIDg5WKbt+/ToKCwvh6emp9vX/N6aysjLcv38fR44ceei1X8XFxQ99LURERP+l6aU5+rikx6ASrpycHFRUVFRbCtnR0RGXL1/W2nGioqJU7kUjk8ng7u6O4OBgjS6ge9jdviM2n0VpReV0yBH+rnhraNu6Bd8A8O7n2mUo/VlSUoJbt27BysoKZmZmeoujrgRBQEFBAaytrXkdZQ3u3bsHCwsLlZvw3r9/XzkV8aWXXlL5v7KmPi0pKYG5uTl69er10M/Mo5JXIiKihsygEq76IpFIVL5IVKnr3br/2/7AxUzsu1iZbEmtTPHB822eqASEdz/XLn33Z0VFBUQiEcRicYO+hqdqylvVa6Hqjh49igkTJiA4OBjNmzdHTk4ODh48iJSUFDz77LMYPXq0St/V1KdisRgikeiRn13+P0FERI2ZQSVcUqkURkZG1e49k5mZabALYtSkqLQc0T+dVz6f/UIb2FmY6jEiIqLH07ZtW/Tv3x+//fYbdu7cCaBycY8FCxbgvffeY6JKRERUCwaVcJmamsLf3x/x8fEYOnQogMpfTuPj42u894mh+mT/VaTnlwAAevpIMdjPRc8RERE9Hh8fH2zZskXfYRARETUKBpVwAUBkZCTGjRuHTp06ISAgAMuWLUNRUZFy1cKwsDC4uroiJiYGQOVF2RcvXlT+PS0tDWfPnoWVlRW8vb318hrO3c7H+uM3AAASYzEWDm3Ha0WIiIiIiJ5ABpdwhYaGIjs7G9HR0cjIyEDHjh2xd+9e5UIaqampKtNZ0tPT8fTTTyufL126FEuXLkXv3r2RkJBQ3+GjvEKBGT/8jX9uuYW3g3zQopllvcdBRERERET6Z3AJFwBERETUOIXwv0mUh4eH8h4whmD98RRcSK9cccvXyRoTe3rqOSIi7TKkf29k2PhZISIiXdF0jNHH2MQrn7UoLe8+Ptl/FQAgEgGLhrWHiRG7mBqHqhvX8ia19LiqPisPu+kxERFRbdT1+4g+xiZmA1oiCMC8Xy7hvrzy7tdjAlvgmeZN9BwVkfaYmJhAIpEgPz+fZy7okQRBQH5+PiQSCZd9JyIiranL9xF9jU0GOaWwIfrrrgiHruYAABysJZg24Ck9R0SkfVKpFGlpabh9+zZsbW1hYmLS4BaEUSgUKCsrQ0lJCZc315IH+1QkEkEulyM/Px+FhYVwdXXVd3hERNTI1Pb7iCAIeh2bmHBpQUGJHDtu/PvFbd7gtrAx4y+61PjY2NgAAHJycpCWlqbnaDQjCALu378Pc3PzBpcsGip1fSqRSODq6qr8zBAREWmLpt9H9DU2MeHSgqVx1yCTV37J6OfrgAHtGtZNmolqw8bGBjY2NpDL5aioqNB3OLUml8tx5MgR9OrVi1PdtOS/fWpkZMS+JSIinart9xF9jk1MuOro9M172Pz7bQCAhakR5vOeW/SEMDExaZBfqo2MjFBeXg4zM7MGGb8hYp8SEZG+NITvI7yAoY7EIqBlMwsAwNR+3nC1M9dzREREREREZCiYcNXR082b4Ocp3TC8ZQXGBrrrOxwiIiIiIjIgTLi0QGIsRk8nAca85xYRERERET2AGQIREdFjWLFiBTw8PGBmZobAwECcOnXqofW3b98OX19fmJmZoX379tizZ089RUpERIaECRcREdEjbN26FZGRkZgzZw7OnDkDPz8/hISEICsrS23948ePY/To0ZgwYQL+/PNPDB06FEOHDsX58+frOXIiItI3JlxERESPEBsbi4kTJyI8PBxt2rTBypUrYWFhgbVr16qtv3z5cgwYMADTpk1D69atsWDBAjzzzDP4/PPP6zlyIiLSNy4Lj8qbdgKATCbTqL1cLkdxcTFkMpnBL0tZH9gf2sX+1C72p/bVtU+r/u+t+r/Y0JSVleH06dOIiopSlonFYgQFBSExMVFtm8TERERGRqqUhYSEYOfOnTUep7S0FKWlpcrn+fn5AIC7d+9CLpdrHH/V+5Obm8vP/APYL7rF/tUt9q9u3b17F4D2xiUmXAAKCgoAAO7uXGWQiEhfCgoKYGtrq+8wqsnJyUFFRQUcHR1Vyh0dHXH58mW1bTIyMtTWz8jIqPE4MTExmDdvXrXyli1bahA1ERHVVW5urlbGJSZcAFxcXHDr1i1YW1trdNNimUwGd3d33Lp1CzY2NjqIsGFhf2gX+1O72J/aV9c+FQQBBQUFcHFx0UF0DUdUVJTKWTGFQoG7d++iWbNmGo1NVfiZV4/9olvsX91i/+pWfn4+mjdvjqZNm2plf0y4UDk1xM3Nrc77sbGx4Yf+AewP7WJ/ahf7U/vq0qeGeGarilQqhZGRETIzM1XKMzMz4eTkpLaNk5NTreoDgEQigUQiUSmzs7PTLGg1+JlXj/2iW+xf3WL/6pZYrJ3lLrhoBhER0UOYmprC398f8fHxyjKFQoH4+Hh07dpVbZuuXbuq1AeAuLi4GusTEVHjxTNcREREjxAZGYlx48ahU6dOCAgIwLJly1BUVITw8HAAQFhYGFxdXRETEwMAePvtt9G7d2988skneP7557Flyxb88ccf+Prrr/X5MoiISA+YcGmBRCLBnDlzqk0FeVKxP7SL/ald7E/texL6NDQ0FNnZ2YiOjkZGRgY6duyIvXv3KhfGSE1NVZl60q1bN2zatAmzZs3CzJkz4ePjg507d6Jdu3b1HvuT8P5ogv2iW+xf3WL/6pa2+1ckGOo6vERERERERA0cr+EiIiIiIiLSESZcREREREREOsKEi4iIiIiISEeYcBEREREREekIE646OHLkCAYNGgQXFxeIRCLs3LlT3yHpzdy5cyESiVQevr6++g6rQXnU50kQBERHR8PZ2Rnm5uYICgrCtWvX9BNsA/Co/hw/fny1z+yAAQP0E2wDEBMTg86dO8Pa2hoODg4YOnQorly5olKnpKQEU6ZMQbNmzWBlZYWXXnqp2s1/qf5wjFKP45V2cezSLY5lulVfYxsTrjooKiqCn58fVqxYoe9QDELbtm1x584d5ePYsWP6DqlBedTn6eOPP8b//d//YeXKlTh58iQsLS0REhKCkpKSeo60YXicf58DBgxQ+cxu3ry5HiNsWA4fPowpU6bgxIkTiIuLg1wuR3BwMIqKipR13nnnHezatQvbt2/H4cOHkZ6ejmHDhukx6icbx6iacbzSHo5dusWxTLfqbWwTSCsACD/++KO+w9CbOXPmCH5+fvoOo9H47+dJoVAITk5OwpIlS5RleXl5gkQiETZv3qyHCBsWdf8+x40bJwwZMkQv8TQGWVlZAgDh8OHDgiBUfh5NTEyE7du3K+tcunRJACAkJibqK0z6x5M+Rj2I45XucOzSLY5luqersY1nuEhrrl27BhcXF3h6euKVV15BamqqvkNqNG7cuIGMjAwEBQUpy2xtbREYGIjExEQ9RtawJSQkwMHBAU899RRef/115Obm6jukBiM/Px8A0LRpUwDA6dOnIZfLVT6jvr6+aN68OT+jZHA4XtUPjl31g2OZ9uhqbGPCRVoRGBiI9evXY+/evfjyyy9x48YN9OzZEwUFBfoOrVHIyMgAADg6OqqUOzo6KrdR7QwYMADffvst4uPj8dFHH+Hw4cMYOHAgKioq9B2awVMoFJg6dSq6d++Odu3aAaj8jJqamsLOzk6lLj+jZGg4XtUfjl26x7FMe3Q5thlrM1B6cg0cOFD59w4dOiAwMBAtWrTAtm3bMGHCBD1GRqTeqFGjlH9v3749OnToAC8vLyQkJKBfv356jMzwTZkyBefPn+d1L9QgcbyixoRjmfbocmzjGS7SCTs7O7Rq1QpJSUn6DqVRcHJyAoBqq+JkZmYqt1HdeHp6QiqV8jP7CBEREfjll19w6NAhuLm5KcudnJxQVlaGvLw8lfr8jJKh43ilOxy76h/HMs3oemxjwkU6UVhYiOTkZDg7O+s7lEahZcuWcHJyQnx8vLJMJpPh5MmT6Nq1qx4jazxu376N3NxcfmZrIAgCIiIi8OOPP+LgwYNo2bKlynZ/f3+YmJiofEavXLmC1NRUfkbJoHG80h2OXfWPY1nt1NfYximFdVBYWKjyC8KNGzdw9uxZNG3aFM2bN9djZPXvvffew6BBg9CiRQukp6djzpw5MDIywujRo/UdWoPxqM/T1KlTsXDhQvj4+KBly5aYPXs2XFxcMHToUP0FbcAe1p9NmzbFvHnz8NJLL8HJyQnJycl4//334e3tjZCQED1GbbimTJmCTZs24aeffoK1tbVy7rqtrS3Mzc1ha2uLCRMmIDIyEk2bNoWNjQ3efPNNdO3aFV26dNFz9E8mjlHqcbzSLo5dusWxTLfqbWzT8mqKT5RDhw4JAKo9xo0bp+/Q6l1oaKjg7OwsmJqaCq6urkJoaKiQlJSk77AalEd9nhQKhTB79mzB0dFRkEgkQr9+/YQrV67oN2gD9rD+LC4uFoKDgwV7e3vBxMREaNGihTBx4kQhIyND32EbLHV9CUBYt26dss79+/eFN954Q2jSpIlgYWEhvPjii8KdO3f0F/QTjmOUehyvtItjl25xLNOt+hrbRP8cjIiIiIiIiLSM13ARERERERHpCBMuIiIiIiIiHWHCRUREREREpCNMuIiIiIiIiHSECRcREREREZGOMOEiIiIiIiLSESZcREREREREOsKEi4iIiIiISEeYcBFRrYlEIvTp00ffYRAREQHguESGjQkXkQ6kpKRAJBKpPExMTODq6oqRI0fijz/+0HeIRET0BOG4RKQ/xvoOgKgx8/LywpgxYwAARUVFOH36NLZv346dO3fiwIED6NWrl54jJCKiJwnHJaL6x4SLSIe8vb0xd+5clbLFixcjKioKs2fPxuHDh/UTGBERPZE4LhHVP04pJKpnEyZMAACcPn1apTwnJwdTp05Fy5YtIZFI4ODggJEjR+L8+fPV9tGnTx+IRCK1+x8/fjxEIhFSUlKUZevXr4dIJML69euxf/9+dOvWDRYWFmjWrBnGjRuH3NxctftavXo12rVrBzMzM7i7u+P9999HSUmJhq+ciIgMEcclIt3iGS4iPTE2/vefX3Z2Nrp27Yrk5GT06dMHo0aNwo0bN/D9999j9+7d2LdvH3r06FHnY/7888/YvXs3Bg0ahG7duuHIkSP49ttvkZycjGPHjqnUXbBgAaKjo+Ho6IiJEyfCxMQEW7duxaVLl+ocBxERGR6OS0S6wYSLqJ6tXr0aAFQGqunTpyM5ORlRUVFYtGiRsnzPnj14/vnnER4ejitXrkAsrttJ6V27diEhIQHdu3cHAFRUVCAoKAgJCQk4ceIEunTpAgBISkrC/Pnz4erqijNnzsDBwQEAMHfuXAQEBNQpBiIiMiwcl4h0i1MKiXQoKSkJc+fOxdy5czFt2jQ8++yzmDlzJhwdHbFkyRIAQFlZGTZv3oxmzZph1qxZKu2fe+459O/fH0lJSfjtt9/qHM/LL7+sHNQAwMjICOPGjQMA/P7778ryTZs2oby8HJGRkcpBDQBsbGyqxUhERA0HxyWi+sczXEQ6lJycjHnz5qmUOTk54ejRo/D29gYAXL58GSUlJejbty8sLCyq7aNv376Ii4vD2bNn0bNnzzrF4+/vX63Mzc0NAJCXl6cs++uvvwBA7fHqGgMREekPxyWi+sczXEQ6FBISAkEQIAgCsrKysGTJEmRlZWHw4MEoLCwEAMhkMgCAo6Oj2n04Ozur1KsLGxubamVVc/YrKiqUZfn5+QCg8itilZriJCIiw8dxiaj+MeEiqif29vZ47733MHPmTFy6dEk5BaJqsMnMzFTbLiMjQ6UeAOWc+fLy8mr1qwalurC1tQUAZGVlVdtWU5xERNSwcFwiqh9MuIjq2cyZM+Hi4oIvvvgCKSkp8PX1hZmZGX7//XcUFxdXq5+QkAAA6Nixo7KsSZMmAIC0tDSVugqFQjntoi78/PwAAEePHq22TV0ZERE1XByXiHSLCRdRPTM3N8f06dMhl8uxYMECmJqaYvTo0cjJyUFMTIxK3b1792Lfvn3w9vZWuai4c+fOACrvY/Kg2NhY3Lhxo84xvvzyyzAyMkJsbKzKr4kymQwLFy6s8/6JiMhwcFwi0i0mXER6MGnSJLi4uCjvNfLRRx/B09MTCxcuRL9+/TBz5ky8/PLLGDRoECwsLLBu3TqVpXfDw8PRpEkTzJ07Fy+++CLee+899OnTB4sXL0bv3r3rHJ+3tzeio6ORlpaGDh064K233kJkZCTat28PHx+fOu+fiIgMC8clIt1hwkWkB2ZmZoiKikJ5eTnmzZsHe3t7nDx5Em+99RaSk5OxdOlSxMXFYejQoTh58mS1m0s6Ojri0KFD6NevH/bv349Vq1bBzs4OJ06cgIeHh1ZijI6OxqpVq9CsWTN89dVX2L59O0aOHIlt27ZpZf9ERGQ4OC4R6Y5IEARB30EQERERERE1RjzDRUREREREpCNMuIiIiIiIiHSECRcREREREZGOMOEiIiIiIiLSESZcREREREREOsKEi4iIiIiISEeYcBEREREREekIEy4iIiIiIiIdYcJFRERERESkI0y4iIiIiIiIdIQJFxERERERkY4w4SIiIiIiItKR/wcClh2xYZMmhQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def viz():\n", + " fig, axs = plt.subplots(figsize=(10, 2), nrows=1, ncols=2)\n", + " \n", + " # cifar100 - fedavg\n", + " axs[0].plot([r for r, _ in fedavg_cifar], [a for _, a in fedavg_cifar], label='FedAvg', linewidth=2.0)\n", + " \n", + " axs[0].set_title('CIFAR100 - ResNet50')\n", + " \n", + " for ax in axs:\n", + " ax.set_xticks([1, 5, 10 , 15, 20])\n", + " ax.grid()\n", + " ax.legend(fontsize=14, loc='lower right')\n", + " ax.set_xlabel(\"Round\", fontsize=14)\n", + " ax.set_ylabel(\"Accuracy\", fontsize=14)\n", + "\n", + " return fig\n", + "\n", + "f = viz()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "92460065", + "metadata": {}, + "outputs": [], + "source": [ + "saveFig(\"FedProx_mnist.png\", f)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 056919e1b2bbcc1c07849d256342669e13a524f2 Mon Sep 17 00:00:00 2001 From: Mahdi Date: Sun, 14 Apr 2024 22:43:34 -0400 Subject: [PATCH 05/29] completed readme file --- baselines/fedpft/README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/baselines/fedpft/README.md b/baselines/fedpft/README.md index 3f045810835..3370fd22d8f 100644 --- a/baselines/fedpft/README.md +++ b/baselines/fedpft/README.md @@ -51,10 +51,10 @@ dataset: [CIFAR100, Caltech101] # list of datasets you include in your baseline. | number of rounds | 1 | | client resources | {'num_cpus': 2.0, 'num_gpus': 0.0 }| | data partition | distribution with $\alpha$=0.1 | -| Number of mixtures | 2 | +| Number of mixtures | 1 | | Covariance type | spherical | | tolerance | 1e-12 | -| maximum GMM iterations | 1e3 | +| maximum EM iterations | 1e3 | ## Environment Setup @@ -98,10 +98,13 @@ python -m fedpft.main strategy=FedAvg client=FedAvg With the following command, we run both FedPFT and FedAvg configurations. ```bash -python -m fedprox.main --multirun dataset=CIFAR100, Caltech101 +# FedPFT +python -m fedprox.main dataset=CIFAR100 model=resnet50 +python -m fedprox.main dataset=Caltech101 model=clip -# FedAvg -python -m fedprox.main --multirun strategy=fedavg client=fedavg dataset=CIFAR100, Caltech101 +# FedAvg with pre-trained, frozen models +python -m fedpft.main strategy=fedavg client=fedavg dataset=CIFAR100 model=resnet50 num_rounds=20 +python -m fedpft.main strategy=fedavg client=fedavg dataset=Caltech101 model=clip num_rounds=20 fedavg.num_epochs=10 fedavg.lr=0.01 num_gpus=0.2 ``` The above commands would generate results that you can plot and would look like the plot shown below. This plot was generated using the jupyter notebook in the `docs/` directory of this baseline after running the `--multirun` commands above. From 6d9218e5a75803dc02a480ddf435415709c6808e Mon Sep 17 00:00:00 2001 From: Mahdi Date: Sun, 14 Apr 2024 23:47:09 -0400 Subject: [PATCH 06/29] fixed cofig --- baselines/fedpft/fedpft/conf/strategy/fedavg.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml b/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml index 5f9e1d9e777..166bcd10aef 100644 --- a/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml +++ b/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml @@ -5,8 +5,8 @@ fraction_evaluate: 1 accept_failures: False on_fit_config_fn: _target_: fedpft.server.fedavg_get_on_fit_config_fn - lr: 0.001 - num_epochs: 1 + lr: 0.01 + num_epochs: 10 evaluate_metrics_aggregation_fn: _target_: fedpft.server.weighted_average _partial_: true \ No newline at end of file From a58786b2e2f6e6737f75ef5c66182b94be3c5b44 Mon Sep 17 00:00:00 2001 From: Mahdi Date: Sun, 14 Apr 2024 23:47:39 -0400 Subject: [PATCH 07/29] fixed readme --- baselines/fedpft/README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/baselines/fedpft/README.md b/baselines/fedpft/README.md index 3370fd22d8f..c25f6de8f71 100644 --- a/baselines/fedpft/README.md +++ b/baselines/fedpft/README.md @@ -75,19 +75,19 @@ poetry install ## Running the Experiments -To run this FedProx with CIFAR100 baseline, first ensure you have activated your Poetry environment (execute `poetry shell` from this directory), then: +To run this FedPFT with CIFAR100 baseline, first ensure you have activated your Poetry environment (execute `poetry shell` from this directory), then: ```bash python -m fedpft.main # this will run using the default settings in the `conf/config.yaml` # you can override settings directly from the command line -python -m fedprox.main dataset=Caltech101 model=clip # will set dataset to Caltech101 and the pre-trained model to Clip-ViT/B32 +python -m fedpft.main dataset=Caltech101 model=clip # will set dataset to Caltech101 and the pre-trained model to Clip-ViT/B32 ``` To run using FedAvg: ```bash # this will use a frozen, pre-trained model and train the classifier head -python -m fedpft.main strategy=FedAvg client=FedAvg +python -m fedpft.main strategy=FedAvg client=FedAvg num_rounds=20 dataset=Caltech101 model=clip num_gpus=0.2 ``` @@ -99,14 +99,14 @@ With the following command, we run both FedPFT and FedAvg configurations. ```bash # FedPFT -python -m fedprox.main dataset=CIFAR100 model=resnet50 -python -m fedprox.main dataset=Caltech101 model=clip +python -m fedpft.main dataset=CIFAR100 model=resnet50 +python -m fedpft.main dataset=Caltech101 model=clip # FedAvg with pre-trained, frozen models -python -m fedpft.main strategy=fedavg client=fedavg dataset=CIFAR100 model=resnet50 num_rounds=20 -python -m fedpft.main strategy=fedavg client=fedavg dataset=Caltech101 model=clip num_rounds=20 fedavg.num_epochs=10 fedavg.lr=0.01 num_gpus=0.2 +python -m fedpft.main strategy=fedavg client=fedavg dataset=CIFAR100 model=resnet50 num_rounds=20 strategy.on_fit_config_fn.num_epochs=1=1 num_gpus=0.5 +python -m fedpft.main strategy=fedavg client=fedavg dataset=Caltech101 model=clip num_rounds=20 num_gpus=0.2 ``` -The above commands would generate results that you can plot and would look like the plot shown below. This plot was generated using the jupyter notebook in the `docs/` directory of this baseline after running the `--multirun` commands above. +The above commands would generate results that you can plot and would look like the plot shown below. This plot was generated using the jupyter notebook in the `docs/` directory of this baseline after running the commands above. -![](_static/FedProx_mnist.png) \ No newline at end of file +![](_static/FedPft.png) \ No newline at end of file From 1127ce6aaa84c5f6085910535880e81e821b893c Mon Sep 17 00:00:00 2001 From: Mahdi Date: Mon, 15 Apr 2024 00:12:08 -0400 Subject: [PATCH 08/29] add plots --- baselines/fedpft/_static/FedPft.png | Bin 0 -> 28010 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 baselines/fedpft/_static/FedPft.png diff --git a/baselines/fedpft/_static/FedPft.png b/baselines/fedpft/_static/FedPft.png new file mode 100644 index 0000000000000000000000000000000000000000..76028f4f24b02b424b9bbbb2ffff0a37d260f45c GIT binary patch literal 28010 zcmd43WmHye*EV{Ql2QsH2ndLDmr6<_-QAMXNH-!SAR-DS@|J?VuAAN>5Ue{XdJkL3gdBloRQ;{RYyN-uKp$HY^r8Q6}3>y>* z4F?As{$#@QSv&l8$3sTXL(|#X!^_<5AxhcY!{w2)$0IunI?so0?sm>je4Ij@{OokL z9v&|4ce%J6|K|rdo!xA>$isyD;X`m;r+CD^WF*$06U!iX!=Pnwd_>1_)jqwsPIt9{Jhwob z&A_Ipgsd|JJru8(V4)!>O(^6!3XSwd%YnObE#o(RAtD~HjkMn)a`ZT%DEfc@46%^~ zT#}If_YZr2%U^O%su<}N8PA$VTONy#*6x&A4C2B;m71Z-}Tyf)}Maw z^^vQP^AtgpbZ86(*Q=unLVYDA0!C6?LFcJrTYP-{FlqWhL#1Qi*yrZvuCmx1ckf;& zV^Rvv&%a$NMAWd`e@}5jk&*NRk&TT_?q8E8e_S>; zHhD(U$xp6Qj@0ku;&sL&a&vD@dCwC*dGdsgfkEownQ^_Rc}FBkfoZcCBdJ1`^T(8w zlw?sqK_eq0Ge<{4Q&UrT)&<4~Yd@=Or<+6{J$f{=w)U)0Gk@;y>L}Jmea_B7N=XUF z-N%2Pv_}v*oawpkNV6o!mpf03B`xmcq$DTntp#L?`jMNPn|mMai(~`*UPj+vD6#w)j-p0YlZ~sxG`+&Ci)#=|+(_|4JUVndoGbbk#tE@`%c5H7y zzxLi1OYE7MnF%gyI>xiMw!X$=Asw>Xi5b62(x$Wd9p32Hf zm!+?iU(@c8U7VkA1)gqyzJD^mzM&zT=ya7$e75bxM@K4hE|X$U(^s!v z=`{z6qUi42p}cgNx+PmChWN&fSjUNvEG6~UebowCezklcdonQRmGJBuk8n=YQv{?o$6h1k7 z%ktnma~qrd$_AYh1HDSCD2>#Zn3!JQH|PJBi{awRA}p6)xfz;>h{ymuqjgLZMY%ZY z*5kjcgVlDL-oCzb`|A@}r6=t~;%0LlQQjA42PD+g&%WoYTK*X>lz;G`9X8a}RT&wX z+4{{UsrILYv+L_6yB7TX{8#WPIHk{y|MHl(VScKwXJBE$MLzmuvpF6fj`>)*xqVH# zc#zoA*A&h7^B1sD5?}zG(5Sjno+>s?EX8h+@ce&)7{Ii?h}3uIN4|6AR;8Z zio*GpAzQzpbm4ScC5_q5_-ON$kST8H zVQnt#`p}q|2MH&1XS3z9yz-_2f3KP}2gVXo3BI-Nd&Bx=^+!=eyx4KU@MoXHt(z*D zVzD=+o|<((zxgTXLaaYSv}o`nX%y!AL`?)8&L^+Uj3HhjA)(~tD`(RVn^TS40s@_f z+w)f)b#!#D-ScG6&d%Oi9%MjCC@SJXo8h+Tz!@7KXC3)_a3HtXn^fU<;1p}YNeG*3 zY+{1p?p=dIQ9eGhtgNg8-7*?>4vyKSrKkE;);Ib1H0>HABXO#or)fMrJqwI#d9U-^ z2K($T6;j*4#d*)i3YEk0#0gciGWz{3Q?AUUQSF1h_So3iTUfF~C~H^0w70c!yDurB zB+~Cbz9J>ELsZn$-_Pk0S&$dRM$hZi*(aVV=y-p$)OdDz8Rx-+2mFu5h#f|XS@y2M z9+v*{iUSR0+4W4lforkxys+Rvuy$-r@7axeQ{pUxK<$PBxw~&yo{sxpFH(Y6xKc#mPc}a;rp{1t> z*V>v|*rxk%^k$gH^1w|fM%hp}a>rGn7W4`zYQNV~Zy?zitH3}$>E31ohJ%9x94`u} z&EF3{?GPRQc(2W-?yp;F)Jo;tkfr5nYwM>{5tosrd){M(LqMQa zA6ru+M9ygx`TRLCatk-^dZ199JOVz>7#&#&B?cebzP#q*2|V@o_VKwXC`b;)QO3aF zIy*c2mCtFs5Aoh>7d;&qQ0wUEFzbxIiY#rZaeeeJxt+8`@!Lm-hb@?L7MI?Tzw%R#E8+00i028R2umd|_!weSbav_)inr($e zFNhrc_t#R8JwLngtG;d2G>8N6RTgxWP$ecd?vHIx1q6`_KK`Ld@ud0UoDD@hQfkbG zbAQ9-DV*2B;(N9Obge&s*0#*Q#zdK~ja9T&*?a>qhB-Ue8FL>_x7JgIM?WiNxOsR; zph%TepN)TTz(TcU2A#7&p=CF1YUsWhn|P=vL}G^$cB0A`NXb`x z5wy^b+TGdRB@+=9O&aIFF`}1tAQ%)B)Y{P@z5ZP0=;Mxg?5}z+-SO$YmG4-ZnwqnV zUtZe>mBC$OLkGyq%d>xdM_3p~>M4PGz0&LU>qOMTbSPrj^reHxf1V2T?kbt=e7j7+ zWujFr>b)gVURg;x<+mxC`&j7F7@Z!Vq(&ydR_rL^z{A-~xsMHNoH#z!)vgs@U%cx1xvL%nW!lS^S5O1JLZ+|-q@<;v3`vQlbc$b`hB=IunnOR2 z{u^=mMl@VLwWzPwL({^-LKhd8;IJ?Z*isRceAaJamkGPfNJQN(l~+~OP3!#frMFkH zLS(ZUtE4D4rG00j+#CbW(#-O5Bsqrx9WO6wm37~J7ngF2bWy+kmWc@}6B84_A6g=H zp6hrL)W_%&Jh-1jFfIYoj28KLy7qHA=%PgUN})>2PGsVP@bDpv9MKHM!xSk%Sd+>F&^FCz&=&W0dJx9iy#eSQ7efh;Lh@aK5i`J%F>`}gmM z%FgfNTR1wFxH*!eI`>X@0=`^u*#644h>VHR^fmcl|AW3}WweYoG&B@21e8wB8EAxZ z1_p05=xdQmGWjWsUz)oNjuol}?z|%K%=a0U&}H|fuabb0x#1}jPz#s0bnyEhy09Bm z;~00+oC~m+o=s^(+3k#%#cg_yy<0}AUot~d;+Wx8&K=u9Su$-Unsy=!8 zl%ACp4-lu~7gz~(xHOB-w<~^zIy&S~Z(ctAaWxO-G{tX=o3UIb4a_?J!H0pbFWv~= zQFeQg2Y(|8PZVao+^i5EJ7huNMC;o5LDL6d_0;BD29yd)jlwG=BqXeS7jIHh%wZw* z8hr&UEG=jM{CUqjtavYnhm8&YTc)_K{g>4E)zPxDJxX~o{GI5lY+JJtwAl)YOu2W( zRmTxAX2vnD^H4Xs#FW@GkfT7xZun7YU2I|03`U~);X+Me*EvjNn%dfO zSsVX=52vxS!t%2je1ByLoE`W$*O?YH7$mDK5FGAu{ZB(s^W5{#=|U@Z$2Iiy^V*ZA zWH2>3{iDikmPYTY4vy@Or(Ru1`KS*!enTIyoy$G7A1RJX zOQU9FWNaB6R3FTMJ(QEfSkg!vxHl+wIDK)R?>zb-U%cHWmVf#0=oREsQ3G&RH(|Yx*85Lo$Wy=VvG3PTxxmg^&>v zkc@?^^;^BP1WDiD>0Vp&@#4tVfg?D0e135_Pb3m}dJQ(BR-GvhU0{qbG}g80=H>zX zH-Ocmpw(NphGGc=A2LVqz?I*5iW(p=z|ZI*X)*wYy6l&NyJD%@zkVfv^4f8U*8i;@ z7izN3!)WHCCSYPZ<=bLN3@KL=;WJPkxQQk zdTr0iXQs@90A zs;Zp5Ju9l!w!GUl{#2l)rA5%HY0%k+amGZi=w-zuRx}i_J%!aO zupfH2u_V_k7^{&B4u!RaMJR01d8odh zytlJ^eQ(`i$SVZslClqR9nL?cUl?&F$;LZVU2R%?Dev?XHWxSD{|sr%$-z z&9F9Cpi}ZjG4QqNC1c{FMasT60&IV`6@$ylWXcZ$^CVRt>F41;D1D@qkz<$??llNgIXt89>A z#D8XUtb*aL=NgX0sh}gw8Ois&I7vxK)yhmnPd9@uq=17Hd^GmfqZoP{0dy!C`1{;& z6p_O@?Y~RtyT2A?(&(F1z!mKPLpKgCZtH8)z$kc(mJ8hpV~u`7Oe$%SWVcliWC1&) zwJ%i&nFgV=Ug0pPZfOh0cl|TW0{H0?Ou6#%@-2Ww)QfaDU;=C!u7zyC56JsB90)Sz=8!r4+uzk9v8CgA+&w=mEt^jyHk%|ls&d7h|0 z?asU8X9}7`G$Ji9wWB1|w|d-P*Vo^Zjibhupnl(EJ4g>4fdxG9t=-*r_*)f^S9SdN zf7&RHhTpn*f07s-d04RU%Pm;@@TqAhq(*;oxp|pdI2gKcp=f)a zBl3{oM$sZi=VWAMEdVY7C&vNHTDUu}iM#T+R6d)2vMIn22M323xCu*+ zKiO=)FzN2@Rx8#W&^m_kmi1(^#u?ZasQXeOpGAFka!?XTp~8~vPFFNdhf>lyJgjw1 z!2V^nve=EXIyqb0f}wp_m$qk=e3(Ge0)wbw`?E3^?xl2JFJCp2eEg#$8;riVmoHzw zZ)W!P`HL4CGn?ufSbx*$aP9*yIpomn4Nno@(!836X1Y z@$#Y&JDkC|$;+!@*9apnZfR+0C=8nxtiKWND-4^E6nQ8>E4Oxa$!=|LlfZaW=1K|=AeKv1SOhd<9@wE&QA9P^b0wN@( zwX$Gj#Q_*SqNfU38+NH-M;TonY6m#Dv|>TtpldS%5SC^#OZ- zG;&B$9pb?yC5{y}uN8BlMqPtbjSIEWF~Y`UjCQ>9e3)}Te?RG?s-}j<%+gYxH&{qX$?)ekXeg~qw--vCf$K_xWk zL%vi}%&22zL2-wcFP%6ReW>v^GW0Ii@AeC z(S9){O3%Om$R3Sc7?b|w_9_d&yz#$$jp~sg+}5JyfEWo$0*U?iJVJT~hNN-WaszqF zRH8DV$sBheT9^C-gyk&KEt`WiMs*bK0EG|@>?=BP1miJi4KGeI$;L7 zIBaMM*kfCLLepk2bVE7+{@_$vI>)qO>EGobd%ewxw$5kQ`ojcNY?%mkxP^+Dl>hG% zU3N6RsBz6I8Hsw~KNasi&PWNy8$m1@(V<6Ya`7F1M@zmt8o_t|Q~L@U9;jXdJ?E1j z#qwnBGWz-;gKJ~GSLeDnc5GyU(!Wn&-7GE%ng5KYsyZ{(*oM#VWA3?`@#%bfer_K= z#{Hi{$O>CEBS6TRP2@6~U zz6azJeh1d(veG#=vc_q^iQxehh0+&VR>pM>3e5e{prf5zAPA@fVGqkb7b`T?0^&k1 zvjMCnMna85Occq4g@s-+3CT*zAh`{+8DJN| zzP9$Oo&x6BHYk693Jkgu$Vf;iQ9?pyU&t3fav1*mu!NK*4CX?pW(e&E^e!5z*tnh? zc19Oe0Un!v>ZHFyPLq)^I?MyC0xtFv%G=xf<-=~+4NvC>8O#>s7e6L2_`s^l$3LqS z2418s_rYs!uMH!aR6r*%*vAzVFn^e!7V##ALoNw40Q6bZrAwC(iUU}!r@*$1JM!}J zC8VSjdS|?NfsO(a4gT$nGi*{;_{b6e|2ULKH3QiVRlL4_F^)7eEwNM<_wf1Y=^MFw z=h%S@0|Cy(%}vAJ%gwC~Ak_~i_suC$YVkm7s8*lhfw~-Q7%OFnAj2}S?DjSKl~%aq z?0Rp90@ucPfGvM1d=5L8)Y*`xLcAI|%4@3YY<8J>xw&?CUf{7_cpS;ap@J?r&L>I? zH`wFY2O=YT)6~grZrudM5Xc{3BuRkZf{rKo=n?xXPGbWzS3!6dn3pcGW#s4NwDk26 z0wMGSDLPWh_>##|J#_p5!rny1Lu7{&(O$ zvw3Y^cBV8oiYh59lhD$Z4b#xlA}a2OiQ!^>vZbXZMX76zjg3$cBY@1+)6*+>WNBxI z3$%BVr@Q-ZwzytSKE}w1Jw@TKrssdZ+>1R<cpMy^{K$0T#npU zTH4nfzAii*3*=Q|N=gL2Vi13O27rj3nOS~a5a^(GpxEKw%`E6uG<;6>9pO4O9nE!E zS4YrH1F)#YOZD&j?cpqsUriRk$H5tJVuyXRvSN`jxFBYIgkN2191?V1SZ+#3ny}wv zHFt17H^9o~HD}z~Bi!@zdh+)=NpIb4N>1%_tzULhqTnf74i~7~4rXH+e6SC{%BDjn zAfV0T_YN3vBV*$Vs~%EROO8TfIdp1&n9-AgUf!6j4f`g3K@IKlhm#q6wYld(sr9t* z(izikQCTnvLozBVQog>pq>~FJa5krA^0ud0*vf>S*I;F(Wqo4KUtGu;y~YVGxABQC zaYtK0!5jPnenH$WQ;FDhaaJ6>k^njvafAEIv<6Q!DX^X5@RGPQU=ro6Ot z9SN~o0~hMoq<7b|?np+a=+4&-IEfcjWDMk*R@Bcv3S$Jneft(Yf(KfnMs5jwA~NyE zy}RFPxxZ^dq&qMc1a-rh#VM<^?OFHRg$301wjd7C>zV!=i}dU~>dh4jjKQ@&gGBiF zeIwr&%&nPdYH|?%51?zdY4cN*QnXPs`?bfe)&s*=Z4(maD#ykqSEBWy{E(1n3x}d| z@}gyuPPf-mjx!g5R1ykP8j+f}gl6yU6IH{vx6>N{>vVkB_w^-yF1i!dqk$X^$~fxU z_zcm;zc-@6h+*7o`u6VzA-g{P`Ik1tj8ULuWp%~>Ow*30^OueNd$+-)1og>j=`%Ze z3W`N~rSr0w2P^&T>OU=A^BjEetCO!)a4+Y#2?3~o+rbL@W8t61nh?B8*qB>QZvDg(t?xP#-^1 z+skA9yK|d|d*>H1grXuHg^&4biLOIMPtq>ZF<%+V1=dpvX}Lhsyn{c&Ds2YE8Q>!} z5v|x0s2L!DMc#Vws-?I0J;6IB%a>^cM$Md4+cTL#9ieOE0(7@zrW00V;ap{!_fDI9 zV2z>U6}!vP-=8T?1Z;&dO$KAX#e&`vsLI^N5dadl5NEZ)Q$b@-xF||iiOqJW3cK9vWz_MZdm{E zKB|H&$?I-FN{!ujY|vsPfq;>Hb`1??Lt~o$@`C#Kl&UJdwA#A&%|zFse)7`uQf{5z zdH$SEY;GR^;s};@xcDbcd&H`clkrmC|F#XnAdGlv5Mu#Yn}~>j;=TfHanwM`N(2)v zGDefL<^1T<;ikB&j)d}XDIvcj9={#>a9O6IOV0(z>dnV<0vf;1FRrt+e~Q17aPzyJ zt02h1YZEm@P?|bnow$KQ!oUx!7&imxCLJA1A$d-?XM-m`4$W#LF_In)LNj-@~CiFM_XxD|v2MQ8`vzr{c`#A&FI$K9yBOR=$`m8Xf zIZP#{Dl_mH7jz9#a34g2!fpmvGX};MpvdBbgHg|}3*hbV@7EDzt>xxp7}wi-(pH^f z8H|?B>00fu_(Vo3Di8-#aPi-(N_ze_)j|8c?j`CQXmWCtl~rWV6X$W<-PdX5G~o%r z;0Y8;NMd55Mtx!;83=@ho~giA-vmJLUrK#W8eqvb##o)wPF||zs*V+iUb(ohN^kow6>rRi+R8*BJyH( z;~eYlu>DE)p`i2-)*8kNxP4-f_eL_Y)6n(k@aFR8Y z4pn9nC8KSWcF4SD;}}$7V}6tCBkK0;#h~4bwHcJmO{UAvbvT-uXJ02@z0O+5OT4W* z2Xe<}0Q|G7t1v<2pg_rI*6GvZvntRteTpb!FVI5Gtp8ctYzrb|+U z#}Ps!>f2(Jsa=J}>}U!d$fnip+Y>a9DkTlxSlhIxrMrK7DqzoezPcWJkxAW!7p$pX z&n^6)yPwEpxqB#ErNmX@5A$d__>_RsC-B=}Yi;n^E%(|qL9i(>TgfnXFe$%z0`l|A zhkD}_#HeT2vV5`@m%8|@r7=^Ks96seqIRTAY!^8isztFrrW?=7=uG`ybj0R&Tx9a| z^Gkm7hF(m}q)=2$jHbW8-@fK(@4G4wkco(pgF>_fkS-8j>u@1a)yyK>H#QEeZR{M{ z5fN`vQ(ymSH?XVWQjPu-^Hs@pXz_IPsc69apXVhdA+oVSVZ=D5HFbP{kih>i$?+9g!C z6|eMvKo@K)R;=X0LT^ibN#g3=FX~Z?jW&8y^lKv=eWf)onPbU%ii1F18` zdKKsZ>7##Aot>S5`Vpc9$IXg;$J zT^)__`d0quhkFc{&&`nn61Cb&CrhU69hv-2eSd!u;LV>v8G+f%j~EjT{3dP4nFf_E zS2F}`a>@L5q#IL>G0^)~c-iT*Equ-l0%mVr+~F+sIC(AKQKiet`GEwd|FSProEBIcpdsV{$%D2#0N0C9k>eMg3LGf} zkTEw9LW)xHTqZg;K*riT4=YILM7_K+a~|NDY6^R_XKK+gTO`u_6I`HI=}$%hxZCSFL*HL7_VbpHC4@c>2#r^^Cv&%+canh zSAx#|QXTIHq<+8pn(wRf)z6K7E#n_rx?O(9CeY}iz}@}(CgM=K%N*>gOD7OpBUmrx|}a#n&K^2F4fgz}0cDnnG0qi#W5+?R2R zs^AklM<;#JTl8WBfxvglb)P4-0vVamzvrQWs$X+*6lK$Myf@o77S;U;o`Azp4lZ2W z6DSmgH^g2fB!qw_h)C0Q1YwbQTgL(}yG$_Kk*)S^kt>q0~lo=43NOXllUpk+gI!Y@tVAlX(zp9wG-J;5Z{A zBl_Xs;2`26hB6)sHi1991lf38EDE)##Ea8q_3Z=g#yUM)AKxW%^PM1vQEYbhYFhCG zK^`H>n%k-!xT5=;-yEx|FkW#{*qAQhs?zgX-{tC$qrMXgE*eB`e*9-R1O`4>1J{dl zm~U+?q4Zqivwj9^`yMKpykg~js~pT4XEq9RpJsi|=p=4Nc92w6tf4d`X&b%)mMvL-7ZXd;3j! z8ueqtf(wtig_nlGiyR$k=|>??Svi-O=IIinY&w_vwVN(`cHo$G;=Pq6Y=l0OLk5p( z2UucKGFo-9sHm1CwwAxEy)t_DLzhmT%cH!!(0&x%erqY`$8T@-p1I?__IKko=|wd6 zMDx1G57N*tzKOd}*#AHeJVaamWf#%^m1{`z%tKZ-1btNF@iguJwt$x4H; z-`+9_II*7rj=iCpAtI@x^Q!kQG=`r)f7V?NFZhZRs^_uD)iF26Khb2Zj7+Kp+C?*~ ztvMR&ovK<>uy%RP7uaH_YW~eHiwr6BnrW6N^_|Uu+NXq-1F#+j9H2>qHvo?7;PWH{ zPz1za8%&RY!v>4>`Nmw2g?KG^Q@n5Jo!?(O+Rd&h(ZKTFRh}uS3AXO*aA>>Ek9x23 zBVbbc>Q(o|*#yOqaQqOiCgN{tXmb?m-}7VE1qF{|<0A`-;e;827aPPM5+HMcod&}e zCUAa|^71$tVgb>xQ)Z#$v;zrH3iYA-J`>rur)MW+JM&$HbQ~qi>qXia{{9{J;T8>F z^D&Sn2`n+S)8owjtSm8fpSUi7F6e}5@L|XlIM?I1-{ZsKv+nsGY{bGhv5aF%qEHA) z1T9Y{S;uF+5B66<`!Rd?5DNxO#I|?cvBv$k3RvY3nW5uz-VRIvmxzKH?#=PFBw%Z&K6&TpJ(o| z1^d^?UK9NVrlCn9@da|Q@=Mqj2p-yF7>i z{wS`&bFcOBaG?AVI<%>&Nw2~J3v>#X)e#QiyLZVy8G{0B{WUoR%GJx|Pf*ktBl&Li zehW2*k-8e@%D#^X;xLW9&XP&u?mFl=DXryJ*RPyY8$%HiFV-8-WEtT-%}bvht(gCh z(9_tROzxPPqT%7;=|xLUK^$|p#xE)#ejz3%4(74!LZLuGivehYSYiQS#*I4yU&DP7ba zwn!0-g{P1JLmFr}i1T>60dSMB?;bO#0GGg5AmX=AfLJvFIL0(?0-J~g4ZvllT@6_c z#NiI|4XaB%Vn`hvBmsH96SM++QCYx~yt`lLQ~$%`zB02HITF0}^P|Yn!yqdwggk8x z$0s#vU>0+l9$Aa~LPLXoUGU1xI&i852`H|zP%e3)m*TfgO%WxgX=uOj<2iZVSLGba z6oRb-iiqbJ5~+f7&z<4XspNP51gBXrny4C% zc*P0YJ9kJ${PnQBHpgfC)9pX`jb0|A8Kb|QzlC2{TwJMMv+-cKfFCj6*AGMiMRgd{)b4mQ^3MUM=FMaY*2%Qd0 zoaw;DEj3D@6K&k8vRGO|ec=w2b;re$;!T)2 zqmCzA02?k8y+{5Ve_`;z244I@6-;$^@!qu~c2 zJ}o_6dn7(CP8JRj3PVb2C7B))TPRbNz_;Fyv%9WjjQxpwKe- zZjK-tj2ux>SY-|t=s@Rj7%qvu5lm_s(x@_z?TRboTDJ(YH&r;%G7P$8yu`#hnRvTp zAZxB%7KMY4;k!5B0K_UbGD?8OeX!K{9yt2sS5H2cmj}nk6M=4a_519X|CzHM27sr{ zsNPe#&=K)K{rz*b>1^NdQ*EuCs4VN96s0u9qnc^p%}#w>qx~L>h$i7z)J&nX>T*ta zM|FEI^9rd$2iO**$Hkp<7Z>05WRRAA`2hCo(eCfTy2lRb>*D|CsR@`myjUtBo$)Df zS)jDW6~s*%P;ocN@GGYSzbMZfX&2H5ejl$F8YrTe`D5_Wl{CTi`(r2ize{}>{b?jY zp6+Cuj$MADq$DgrHZ3hKhF{t#O!hGQ-^@McZC-(4w{Th-{XQPf;geC{!%h=Ty*GTT zSvI!AqS(v%t)rodu!{7p{MAY6_kl-PhsO@vHtz1O=){Cv7fkf;)jSN+lv8-IBqb#e zp^G3AemXS+q7`t7aN_X$$15R#VkBwdDCN%H47m@O!&^7_IAxFfZ#wiw;j0qXCLG=lh z&c4ed1qQpvlz1!Z8ZU$&?u1~>S<%!U1ZoG~0Ae~^Z*xi_;&SjCD1niR%A>V*;NaJ* z^I(~qn**Jl&mTfDw(v(GtIbeO5Z6rL`H_uC&^a|UWuyT>Gc#VZq{O(W0jhjf<&*yqXk!f&?+9KS-kXWSK@PDk>pgx(}u0Vmim*LIkk?JY_S`$F&`1 zra3>q?EU)&O(DXz~*Ym|@lRgP`enpP$}A0x+MngCim$R4mvW zQU%ed1S7>K#!x4Kn(hJ<-wNv=9v;5pc93I3Nhx8K0|Ak{m>r#kre%Qv71vfriEO`P zA6}d^-Dl;f*Nci=@JLEUGI1+Ix1e_E4kA`F=sygstcvS`;3ky@9~v=WSYT3GT3gWo z(f-SW3H9`I0aOC(M)dEURQCtD8uAaex>ynVl6+yaIix4CkB+vf$zEG<%3F&2@F2i= zPe$gm*bmV4J(E6)Ki>8luyd7NR8@%+{&m(BJLhpwI~u?ELcSxWaS0fMz~aPx%0rT1|I-8)__v$d2tKUZXt>%h&$EN zQgiMQ(xr+GjwXeL*2z5Y$Y9g+g*5v9lKWtPG1E;_veu+zTg1goaiWD(SM8y5h(>*P zV`T4_mvVKrklde-53ZvuC@Jpni~qZN`?BHJgP%_#BZt9U0hD@2y!bireVPbS1i1>t z_*yQU3P!ciu&}G&{>!}Z6me#Q^k$J5LSV1Y2KqFdVoUEi%=; zYv;X#g9sQoU$N_>qYz#U40BNsYs1mZQ)*4W=e2e8I}-_5-a7uZck3qFH4(iAPD<2L z|J8_3WR%yJn0kxeo)62xg-`epAtqJ`{(%4N7D8Er*0{p<5^w#;9RPSow$iV^e}2`$ zZA=AFa947?KY>4E>$g=~Sy%q5_# zAXeL^_5w9fY@*ReO0o9WNb)n=1#VlUn>4LJ$3JV{|LfC4S{C{)8<+@yp-i$T(Bl{qP9D$!?@t2 zhp3jCd&!*y*&GcPutu0eLa@3gzBf*qUJ@D<2^AGS>|kz+PkcglvaDFIIQs&c-1GI` zk|2x6dO2W;sqv}}Bt0Oi)(%G9fovK4fN=Gjpaa)?VZ=~iT%6@M03%~I%xre*-7;Y) zzO8;l-~4t&P0T8BHnCX;MvJ}kI&&x1j7GnZ2%^qRzXS5-1qNQ}ugY3R$GysM@F4s1 z<0DuP7;fH_RFCTAdd9Tg+mG$JP8w5rI-2;w9t#s2Jq#OdWaKz1zE>!aPvH&g>L{Lj zq9;!1j($}N-(L#+guLWbdCg@tvv`^;T_#BU+TN)`|W7}EF`75MMZ0-a#x0OE8MN+gq=zn(T>@e_3@S4g7m%G z;NgFEjpzP4FB@ACZ|p

d_7*2>qXO)q5X%DR0Q)4ebQi-R}&>+5*hiF25l@RuX$zmJKxU0O+++dK}!@eb4paJy!^nUCZXxtxnYR1b#yXM;$X3yg?J zh_@UNJ+Pc$zz>DE#)s81(`Fig(NB|-u7V4L8=U-BU^Vas(*X{+Gl(ID)(S!(8j4Er zk(z3K`(K|C(=`fWTUynQ0PO&w zi$STvfao65M8KF2L7GcB8g@~~Iqa}1lkwuwSl3a9Cm1R!$f8uo*D8F&yzyrsD-3** zOz;#7i~)&r4!_ra0Pg4#&h~*fP|?wXpQoi6Wv^FNRz3ms3LgY&nNqey+_t8G(3wB1 zICCXpopSGFLtTzNqAokTM1QB%;+I6RV#WcoC$y-jh!|W_ zwyG*B5GXzpu#zFpp^f-_pNh6NaO^uK#Viob(kuDU4G&<}g zVuNbiEL*?rxo|Zw?`A=oxeX-ctLqwtT1a{mB>`S&-EmdPfcGm7dDWY(ZeNvq`2#k) zziRa>2syu6cERpbo09uNCbgou#I(fpxVEtTe0V|NT}i{s*D7UqsQ9c2Adnm5H61_! z`JIsb{Co?24-bAO)r@D5x;C3_3$G(+U+(CLl#&TN6R7DMH%MEyJoad5yYHMul4KA= zE@70`^o}{Lif?PwGMUJN1> z5`n3#s)83T5P+Y*vkvkC$>KpxLp4u*9Di0~!MZ7a0S&JLg5{6K$|ZSTViGs|h`28i zAYmpz9x^9LeiqXC9%x4(b|SejrS!X3k#JfyXjA4;r?3P1Q0|&NYvWkma=wrR@;)KY z-U+RZs_jt&LiwKC0uIxA_5ScrC50#^^WZwtkFXR##|OTxgw z$!~M2r8fyO?8h;&@`iJBZsIdZUSNI%`>}5QBbaSb!JyeO+`TZKM}r-ToD}c}l-JaJ z1_qZ91!2#y-nu{I3lQHxfiwAK1H`an5I+jo8$Y?uV>dK5qM>O0x6txb(xagfG2d!| zv(pBFDL@a})jjieuuFP*0s;cety|oS_=K)z(lJkaydit9?$bWWXyG&@o-X(4|H#L` zbJ=I-Hb-|k^L0v%!QWc1nd=kvS;0u6=cOjus;YK~tcQUDfgCC+L<@lY z8-S~9AjK3vzcQ`L<)k4aBZJSSIQCnia@X93X&>xJ=aY2;&ZGAiiQE$-YZWd$Owtq> zNingoAZURDvaY;}iYy=y^(gmEbX11;rKPWEBPCxP`FPMmLxW{vP=ZnQU;^*4=0A^v11T%u~eM4zpIMRf?pw` zw)PIn^{^WPC*=>nkOL)o&-pIr$m)g@YwkQM=t*UtvhoU9_D>W-+(pp(kxsH>lc^^lcuOPCuM2$X#dcLKbXAosTqs5Rz4D<2~9CGfIxyUt02^^(fD>4*uW7c>eAisa;% zD>DSKJVJIGtr)JaWKm!sl#Ly77Lm>|{HlDM|9zR$=+j%zk1rcw;*10ye1(@F49@V@ zh@th^hcD3N<_-;rh-S9T%?_MeS|XZB6D}j^E#&PGNIdoI_%T8+A@(PTvLKAU86XYC z)rcog5N}2S1k#XnG!!AoWNx8>_r9ch3JMSMx*()Bg4=<0>yU|w31UK5peL4tabCs` zm|t}W^TTWpJs2MbJDvIf@DlB>{lowYAHi7wyJX>~T*2N9y@0vvoC+p{z7jpOg@s7* zOW5dwg1&?ik-aGQ#K0Ag`_JJO1N2;V?vR1>8#M8;w8rkXVHDsa<95#jCj{RhNs=5G z*NHJD-#w;+_m4oOcWBszw;CWRKh|}h_)&NxM{ImE68$u3(91o8;(px`!aNd?qgE?i z2Wkgdf$P_nW6P6M`FH1JHUtKBbS%Iq-xULZ&pj~>HWIAW>}-2PcgJaWb5|Pco#g_XKumj?(MGZ_#*u*%eFNR( z@?`|N$dh%wNnHRS`@!kg_2$zZd~i4AcpibCQW11bfpMBuNtb#K`%(0yCHk@O)v7 z%Lb#hg(71g2_s|3f&w>1UGW0w=`GEQYZX-TN7Ej!jgcTJUr3=ACnw3!Cop%VzDZ7= zL0&5ceCvbi3t(U&Kp;`E`N3gy4&omWl-MPg5IS#y7Ym^8xXY8e!JFHS6;)nr=+*}l%&%hzHJr^5k$4gU3w&E;R99bRn4Pco4VU`_0y!$F5} zTnWE6Gg8oW0*r@?ts?i=-f8V(|aH1mWn#tq0|holV*YO|3QWN|g`Lb}zjrb|TNRjE^@| zA!#Oj?;e8fyL)?w9N5PZ6(}G=ZDV2cJE(Ly`T0m-{{O1(+~cWE*Eaqm>A*Ne=pZSl zrI2cr4yuJhC0e8=XDS(rYLszGltU#&Oi9jR5=uoyP76s*R-)08L(~u{rCo@9-D>u` z_h-+%f4%>_{;RYOzxDi{=f1D|y1omBN6o#r@y4?&lOp&Wg?TpTh8F&I4pZlB9rcI( zK1iE44seUf-arIXgQg-20;bc8!Wkb&k+l=2t z{hK?G&%aZV7c1K%-m4QmQd?bJg3uVMj{JDaIOV}owV5| ztH}fm&mQ3?3cP)|%xBV((2O$Hwe}tqvsJj|C`|0UW5*f^A>hx1G1peFyR81-(05zakIW17y!aX1H80jKz5YY)#a7+IyySi zX$Hbj(xST?GdqhKdR{~>4U>^H-}<=g*&AFk0yohXk0SQzbvId*7y_#$w0%D88iq!_B`^*@fTiWL*CY{e$*!S7K+qrfvqC7<*YYaFJu3u_@;jp+! zd*op@O2$J~)d{FE%5Y=1_4P%f>Zf{m$Bs1p{Rkn!xDW#9Ng`TT8j`>pB6Dre=b@qU zCr@Nd&(0HId_Q-_QUV&M0bbt+cQ zzNo6YC(f1{R-p?Q3qlrHs=YWSDfvO8v)UAm7}cY^OYxV(>(wvoFaG7SiENHrw(L$& zsAxg~JNd2sU4m$733r4S&HI?7eekV4jkBqGfaZ);q$Gm0adSIx%9yZsHeB9%1a}S{ zQLg$h(*ZB|*vHqvz*(+FNjJthIcwq0hxRC-3y`a?tg7QWTdJ$Se41~3N9++#XL>?H z)!vSE-1(b6tjM!Cl-jql)7g?Eci>CrJ!i|NgpA3LR(qa}jut|xO(zu8X{hr@Zipw% zMLmsXb$9FqL^!$*p@rS(G+RljoYPQfJO$~zvg)Wg1QbDm?sn&brpm9ACmsx}UH@)Q zX@RGds5~#_>5#g(sEcD-f5oQ3=rfj4dLaXsRL)p-IVQ7kIk2uOq5bFn_`43Nw-l$w z0}{Lisi0j$d$Fk)vamDo!?Kc7QY5ELIf8nh9Q5?Wg4t1O#4>DNcLG-Kx{GYQ;s0(*)E zruu&w*h1nIreSetcqjti%a)R~JP4Hw1xtgsoba#nyf%BPX7%^?m*M!>JJ4;(ebMaQ zVu($j{(zxtjGoGdADdaLy{>-Bep_-F-Jgq>wNe&C(wnNXH#*z}9{eRZ{n~I#6a5hF z{ow;M^@%Nw(yAUs-xwnq76MJi-}btNcWb`WhCM#X2em+I&CKB=;9KB=0;;h zPfyRLG!q$|NCoSS&R095XD!@$@8DzCWJgC2*7q;d%esbF?!@5Z)64UF-Xe0(+EEi8 z8d`!*z#PKd{_gro^r@|T_`9jW_^mg^K6Pufe)3#(=Gr7=M{I5j!WfRZe`F4WH^v2F zqh3?vzMBHo8>miJO9(ATA{KqfLM$*(%&2DQ?O*5G3=LKK;LIEp(Z~u~Hf@0|84}I) z`=T*d#x6SVCi__6GVcnYH}vjd?B7;lt|J;)71n?Dod?xPj7E3V` zp-!3f=H=xDrKF@Vs{n)v=B*?zZI9FK+qXr|rJe~IAcs9#S} zF(;ZoH;S35HxWjMS#fdoJntn2l_7N0uA|U>y7lb*lIkeo*kR_~N-M;Jqf@_sN$f|y@+u{zYqz@6nuo!XgpOk^fDJXbz$PexRa2ySmZW4S<``Es zp_5Sx`NfNuD*hVkX8wVn_<^VG%mMfmV_=oDOnRM`o?eET8Y*BW@xz1Ec;9m$gRto( z;o>PDj}8r8RK`yYF3fa3$eyC4voca4tlY4F+S047zg!hP1!E4=in+doP*TY;Uu)xk zDKSIgk&#-rOUO7dw)T|E5g0#iIwEozn#Q%vJL6LcPp1px$#?0Qq53VsT-AzdB ziK6|V=j6eXr;D__{(Jd(>uiHm_iqD1y!~~*qI`A$c>?E)Kik+)N@%QA%g+xkEG#T8pPlR# zUKke6zE~Lc54@`EYd3FFu!6>@uo0K>`7@|baFB^uuR^W>1Xb)E%lh*2XYaWPRyF#! zxtQ_e6Wx2@z!UF-2Aqcrbj|%g4{eqZDte`Y1(fd8Fjk6~S zE!Yf?OUh%;fM`Ec0~fr7ev8J7gAX^*JHCW>d_w0iWssIFn-1oyk^J4+ju)k(lI%T{ zO0b64jHWkbZ(F<{$l2=UIJY`u&1L@=Ccp@=MuhtzS}O#p)nM4wp{3e> z*T3la995;{{)~b{U0{dE>FrnB7{BB<>g^NsyC`mI3b|8FLa34*`I3YJ^Qfv*n0( z2idt&R6b9xi~H{#`)d%ovE2+)RChNk*prBP}M zt^nBX?l18|!y;s~EQZhpz>oxEr(iX$2Vkt|XK!q8nawHypuyK@;!Z8n-GIAI1L%Uk z;sWC1MH#C5F}2EbI%|>!WHj5t;&kcy#!NHxaB#5@W5eXxC6v^Hiq99-OT^68>Wn>m zRu~^KqH(eXV*Ad#$<@k_^vfcTrcT8j=X~x~B(os0wRUu**1zfK2!@K5)vThaxv_T; zed_K;yFal3w%|TV8Ertc04Nv@PWCp-7KaQqW80cN(BQCjP#RNY`>{e`QF3=J)RK4i zw3cRL->FTq5BQK=r(T#KJI(>S|>pRD!CQ$kjZ5{F3C< zsa7frwglgeNPjBZ^v#7P?-*br#J+GQZftwV5M0pY_U&5=YNlXE8v79r#I{R6Q3Cda z62Xr%V;vuJn0BWBApZ=^JpCMH=xPXaE5IE8F!QXrdiTKGiGmnz5&|4!>l8S)LC-zu zAGBD|aNrg7JGR#Vmf@M@8rkE*lD&suJs1R%;|bUhi*2gV?IB*#5@l_ogi^JhNV93f#ycC;(VZ;NqzRyfPWp#v^ucrD5|Z;fIJA{MdAP z=W4)4!R!K4N@%r&T_g(VGBnrHuOWN08Aa?-T^hBMBQe%A{vj@w#^ghyOLgDX%EuI9 z*jmEnj)&MHx6*eB8JYS{)to@WV2-Ay7^>+l`Igz+WU%?uEXv8bvKO~BynO~hXc0{t z9Yxo&xhV+Sh%y0!;?=5Bjwqf}`BOezGv~`~!9Eb7c6K}Q7Aat}$My;ka=KI!0X)n6Es1qsP z5g{Qh9ZaJq!d?QjBi#%89F$UmP{%a9a*GzFvPxSk24txRa%>OG9()EcxHee3HU;Z< zP%LeG#^y_6SQp61WbQIGHZCR0Hg>cXMppjIf%gNI*Z09x_!1X%HjgU?8|1wCgapji}n=yDu!~kPZ zq$yCQ;UXHSs-W>AZ;|vX2N9N1ZHwKe;*^`9udlzt$h|;t+_*fyp`Ym64+w95{7DX? z>5yarq_w2t0EaBCG9ap`oy86=jEbXS-v=3yeVa954|1^Ot$eA{tCPj&r0-)}ZNf#C zU^13N0Zi+ARKthkDFmSD$@oGI@?MbcM*QnRvHKa)13stkIhTR!&O#SD2`A7R>@mTR zNxO7+jkB^20Agnn97fVe+6uu2R@&TbLu@@*vp>wdQEJf`%LGHJRc|mhGn0ioVjkyL zs|_eXBnb@NhFX45C3Va{A+T!IgUtHs)7ZCzjIPZhu9w%xa%03&H zStuW1Z8E<-3QHw?>x%Hq)#1xbc0lH#xgo|nY@C4TbB;hlvIos&or#IX?Hp9#bOaK+ z3d{i9F(Jo~AD0?=iL)s*E;e+3eQo{>nDSTN#z2z*WCSG7R)Ntv^3nNi-=0X9F$C6TE}H z?5th_NEPZ*iVV64M_gtm5O80$CV=`+_VoNn+$j7Tj80hjiC+)rUm!Kix(+8(!7`wY=|=M z0i$1ZnT!>-u1;9OU+7wjqi57%P-r8V8l*6**f+po_J-fWoWrX>+#=P+uMWij^;qZ zMJGXNNuaSMcF)$+kntx@z%4X3oYA3K*|K;(4*4!8g|Vlz zk{_RH(}xFtfk}%+IviIpKBYa^$5V*?`4lCUk+Z~MGvF3&FCOkU1>xKgzXX=@bZ`fb zVFge@NBtD`_~>*E1=zyS6(6N!8@|heG!JqFY88JHBqP89UvwL6q&5*Lrti@q*A=$z zU5HPkOttI&GDR}Vfj-Rnbm42!UBa+n2%<5pVMJbw*AGxY5orjsSnwll2D{ s>{y-qx1z(?@BKgcR<8flO-3hJ`-@C{F-=Vg-+RUw8*VhXvc^8>pAVFp_W%F@ literal 0 HcmV?d00001 From 9084b4b7573bfd1ccfed210ae45d962332f0be27 Mon Sep 17 00:00:00 2001 From: Mahdi Date: Mon, 15 Apr 2024 00:12:31 -0400 Subject: [PATCH 09/29] add plot notebooks --- .../fedpft/docs/viz_and_plot_results.ipynb | 88 ++++++------------- 1 file changed, 25 insertions(+), 63 deletions(-) diff --git a/baselines/fedpft/docs/viz_and_plot_results.ipynb b/baselines/fedpft/docs/viz_and_plot_results.ipynb index 866ffb5d8c7..68077f7b59c 100644 --- a/baselines/fedpft/docs/viz_and_plot_results.ipynb +++ b/baselines/fedpft/docs/viz_and_plot_results.ipynb @@ -2,22 +2,10 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 16, "id": "5e0cf2a9-b782-48de-ac45-128726a26e64", "metadata": {}, - "outputs": [ - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'matplotlib'", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[1;32mIn[2], line 7\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mos\u001b[39;00m\n\u001b[0;32m 6\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[1;32m----> 7\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mmatplotlib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpyplot\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mplt\u001b[39;00m\n", - "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'matplotlib'" - ] - } - ], + "outputs": [], "source": [ "import pickle\n", "import yaml\n", @@ -30,7 +18,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 17, "id": "7ea3e149-ce6f-4ba0-aa41-e0501a04efe3", "metadata": {}, "outputs": [], @@ -52,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 18, "id": "4b010856-0d99-4d81-8fb0-7a927f10eeaf", "metadata": {}, "outputs": [], @@ -61,13 +49,13 @@ "path_fedpft_resutls_cifar100 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','16-36-16')\n", "path_fedpft_resutls_caltech101 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','16-44-20')\n", "\n", - "path_fedavg_resutls_cifar100 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','18-16-41')\n", - "path_fedavg_resutls_caltech101 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','16-36-16')\n" + "path_fedavg_resutls_cifar100 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','23-24-25')\n", + "path_fedavg_resutls_caltech101 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','22-32-11')\n" ] }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 19, "id": "2e3e165c-1ce6-4efa-a4e1-1372586e436e", "metadata": {}, "outputs": [], @@ -84,7 +72,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 20, "id": "77b70c73", "metadata": {}, "outputs": [], @@ -97,49 +85,13 @@ }, { "cell_type": "code", - "execution_count": 2, - "id": "6f4c87ad", - "metadata": {}, - "outputs": [], - "source": [ - "fedavg_cifar = [(1, 0.06924765515865097),\n", - " (2, 0.1315106765116743),\n", - " (3, 0.16773099181800039),\n", - " (4, 0.1946717222111355),\n", - " (5, 0.2171223308720814),\n", - " (6, 0.2375773298742766),\n", - " (7, 0.2597285970864099),\n", - " (8, 0.276092596288166),\n", - " (9, 0.290560766314109),\n", - " (10, 0.3036320095789264),\n", - " (11, 0.3128118140091798),\n", - " (12, 0.3261823987228098),\n", - " (13, 0.33745759329475156),\n", - " (14, 0.3477349830373179),\n", - " (15, 0.35831171422869684),\n", - " (16, 0.36679305527838757),\n", - " (17, 0.37407703053282776),\n", - " (18, 0.3817601277190182),\n", - " (19, 0.38824585910995807),\n", - " (20, 0.3942326880862103)]" - ] - }, - { - "cell_type": "code", - "execution_count": 59, + "execution_count": 21, "id": "e1a678de", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n" - ] - }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1wAAAD0CAYAAACPQifaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABdAUlEQVR4nO3deVhU1f8H8PcMy7CDOuygyGK4YqHgviSCVi6ZipaifE2zpDLKFFNcE0sj/ZVluVbuZlmmqYjikqilWe4KiijIqjAsAgNzf38QkxODyjDDDPh+Pc88Oueec+9nzoye+cw991yRIAgCiIiIiIiISOvE+g6AiIiIiIiosWLCRUREREREpCNMuIiIiIiIiHSECRcREREREZGOMOEiIiIiIiLSESZcREREREREOsKEi4iIiIiISEeYcBEREREREekIEy4iIiIiIiIdYcJFRERERESkI0y4SC+Sk5Px2muvwdPTE2ZmZrCxsUH37t2xfPly3L9/X1nPw8MDL7zwgkpbkUik9uHk5KRSLy8vD2ZmZhCJRLh06ZLaOMaPH6+yD4lEglatWiE6OholJSXV6m/duhVjxoyBj48PRCIR+vTpU+NrLC0txfTp0+Hi4gJzc3MEBgYiLi5Obd3jx4+jR48esLCwgJOTE9566y0UFhbWuG9N/bfPbGxs0Lt3b+zevVvrx6qSkpKiPN6OHTuqbZ87dy5EIhFycnJqve/jx49j7ty5yMvLq7atT58+aj8nAwYMqFa3Nu8VERERUW0Y6zsAevLs3r0bI0aMgEQiQVhYGNq1a4eysjIcO3YM06ZNw4ULF/D1118/dB/9+/dHWFiYSpm5ubnK8+3btysTsY0bN2LhwoVq9yWRSLB69WoAQH5+Pn766ScsWLAAycnJ2Lhxo0rdL7/8EqdPn0bnzp2Rm5v70BjHjx+P77//HlOnToWPjw/Wr1+P5557DocOHUKPHj2U9c6ePYt+/fqhdevWiI2Nxe3bt7F06VJcu3YNv/7660OPoYmqvhMEATdv3sSXX36JQYMG4ddff0VISIjWj/eg+fPnY9iwYRCJRFrZ3/HjxzFv3jyMHz8ednZ21ba7ubkhJiZGpczFxaVavcd9r4iIiIhqTSCqR9evXxesrKwEX19fIT09vdr2a9euCcuWLVM+b9GihfD888+r1AEgTJky5ZHH6tWrlzBs2DDhnXfeEVq2bKm2zrhx4wRLS0uVMoVCIXTp0kUQiURCRkaGyrbU1FShoqJCEARBaNu2rdC7d2+1+z158qQAQFiyZImy7P79+4KXl5fQtWtXlboDBw4UnJ2dhfz8fGXZqlWrBADCvn37Hvk6a0Nd3128eFEAIAwcOFCrx6py48YNAYDQsWNHAYCwY8cOle1z5swRAAjZ2dm13veSJUsEAMKNGzeqbevdu7fQtm3bR+6jNu8VERERUW1xSiHVq48//hiFhYVYs2YNnJ2dq2339vbG22+/XefjpKam4ujRoxg1ahRGjRqFGzdu4Pjx44/VViQSoUePHhAEAdevX1fZ5u7uDrH40f9svv/+exgZGWHSpEnKMjMzM0yYMAGJiYm4desWAEAmkyEuLg5jxoyBjY2Nsm5YWBisrKywbdu2x4q5Llq3bg2pVIrk5GSV8tLSUsyZMwfe3t6QSCRwd3fH+++/j9LSUpV6cXFx6NGjB+zs7GBlZYWnnnoKM2fOrHacUaNGoVWrVpg/fz4EQXhkXCdPnsSAAQNga2sLCwsL9O7dG7/99pty+9y5czFt2jQAQMuWLZVTBlNSUlT2U15e/tDpmY/7XhERERFpglMKqV7t2rULnp6e6NatW532U1JSUu2aH2tra0gkEgDA5s2bYWlpiRdeeAHm5ubw8vLCxo0bH/u4VV/amzRpolF8f/75J1q1aqWSRAFAQEAAgMpphO7u7jh37hzKy8vRqVMnlXqmpqbo2LEj/vzzT42OXxv5+fm4d+8evLy8lGUKhQKDBw/GsWPHMGnSJLRu3Rrnzp3Dp59+iqtXr2Lnzp0AgAsXLuCFF15Ahw4dMH/+fEgkEiQlJakkRlWMjIwwa9YshIWF4ccff8SwYcNqjOngwYMYOHAg/P39MWfOHIjFYqxbtw7PPvssjh49ioCAAAwbNgxXr17F5s2b8emnn0IqlQIA7O3tlfu5evUqLC0tUVZWBkdHR0ycOBHR0dEwMTFR1nnc94qIiIhIE0y4qN7IZDKkpaVhyJAhdd7XmjVrsGbNGpWydevWYfz48QCAjRs3YsiQIcrrukJDQ/H1119j+fLlMDau/rGvSt7y8/Oxc+dO7NixA+3atcNTTz2lUXx37txRewavqiw9PV1Z78Hy/9Y9evSoRsd/mKpkVRAEpKamYtasWaioqMDw4cOVdTZt2oQDBw7g8OHDKtcwtWvXDpMnT8bx48fRrVs3xMXFoaysDL/++qsy4XmYl19+GQsWLMD8+fPx4osvqr2WSxAETJ48GX379sWvv/6qrPPaa6+hbdu2mDVrFvbv348OHTrgmWeewebNmzF06FB4eHio7MfLywt9+/ZF+/btUVRUhO+//x4LFy7E1atXsXXrVmW9x32viIiIiDTBhIvqjUwmA1B5JqquhgwZgoiICJWytm3bAgD+/vtvnDt3TmWxhNGjR2PRokXYt28fnn/+eZV2RUVFKmdFAKBHjx745ptvNF7c4f79+8qzbQ8yMzNTbn/wz5rqPrhio7b8N1k1MTHB+++/j8jISGXZ9u3b0bp1a/j6+qqcSXz22WcBAIcOHUK3bt2UC1X89NNPCA8Pf+R0y6qzXOPGjcPOnTvx4osvVqtz9uxZXLt2DbNmzaq2MEm/fv3w3XffQaFQPPJY/03Ix44di0mTJmHVqlV455130KVLFwCP/14RERERaYIJF9WbqilbBQUFdd6Xm5sbgoKC1G7bsGEDLC0t4enpiaSkJACVX549PDywcePGagmXmZkZdu3aBQC4ffs2Pv74Y2RlZVVb9bA2zM3Nq13rBEC51HzVvqv+rKnuo2LIyMhQeW5ra/vINlXJallZGX7//XcsWrQIxcXFKgnMtWvXcOnSpWqJaJWsrCwAlWcOV69ejVdffRUzZsxAv379MGzYMAwfPrzGhOiVV15RnuUaOnRote3Xrl0DAIwbN67G15Cfn6/RdM93330Xq1atwoEDB5QJ1+O+V0RERESaYMJF9cbGxgYuLi44f/68zo4hCAI2b96MoqIitGnTptr2rKwsFBYWwsrKSllmZGSkkryFhITA19cXr732Gn7++WeN4nB2dkZaWlq18qophFVLk1dNW6sq/29ddUuY//c4D3pwWmVNHkxWn3vuOUilUkRERKBv377K66oUCgXat2+P2NhYtfuouqbJ3NwcR44cwaFDh7B7927s3bsXW7duxbPPPov9+/fDyMioWtuqs1zjx4/HTz/9VG27QqEAACxZsgQdO3ZUe/wH37/aqIr77t27yrLHfa+IiIiINMGEi+rVCy+8gK+//hqJiYno2rWr1vd/+PBh3L59G/Pnz0fr1q1Vtt27dw+TJk3Czp07MWbMmBr34ezsjHfeeQfz5s3DiRMnlGdCaqNjx444dOgQZDKZymIMJ0+eVG4HKq+JMjY2xh9//IGRI0cq65WVleHs2bMqZer89+a8VdMqa+O1117Dp59+ilmzZimvq/Ly8sJff/2Ffv36PXJapVgsRr9+/dCvXz/ExsZi0aJF+OCDD3Do0KEaz0KOGTMGCxcuxLx58zB48GCVbVWLd9jY2NTYvkptp3xWrTr54Jm7x32viIiIiDTBZeGpXr3//vuwtLTEq6++iszMzGrbk5OTsXz5co33XzWdcNq0aRg+fLjKY+LEifDx8al2M2N13nzzTVhYWGDx4sUaxTF8+HBUVFSo3MC5tLQU69atQ2BgoPJMi62tLYKCgrBhwwaVqZbfffcdCgsLMWLEiIceJygoSOWhbvGHRzE2Nsa7776LS5cuKc84jRw5EmlpaVi1alW1+vfv30dRUREA1TNFVaoSFHXT9KpUneU6e/ZstbOI/v7+8PLywtKlS9Uu556dna38u6WlJQAgLy9PpY5MJqt2fEEQlDe/fvAGz4/7XhERERFpgme4qF55eXlh06ZNCA0NRevWrREWFoZ27dqhrKwMx48fx/bt2x85Ja4mpaWl2LFjB/r3769c8OC/Bg8ejOXLlyMrKwsODg417qtZs2YIDw/HF198gUuXLinPlh05cgRHjhwBUPnFv6ioSPklvlevXujVqxcAIDAwECNGjEBUVBSysrLg7e2Nb775BikpKdUWc/jwww/RrVs39O7dG5MmTcLt27fxySefIDg4GAMGDNCoL2pr/PjxiI6OxkcffYShQ4di7Nix2LZtGyZPnoxDhw6he/fuqKiowOXLl7Ft2zbs27cPnTp1wvz583HkyBE8//zzaNGiBbKysvDFF1/Azc1NZXVDdaqu5Tp79qxKuVgsxurVqzFw4EC0bdsW4eHhcHV1RVpaGg4dOgQbGxvlNXf+/v4AgA8++ACjRo2CiYkJBg0ahDNnzmD06NEYPXo0vL29cf/+ffz444/47bffMGnSJDzzzDPK49XmvSIiIiKqNX3edZmeXFevXhUmTpwoeHh4CKampoK1tbXQvXt34bPPPhNKSkqU9Vq0aCE8//zzKm0BCFOmTKm2zx07dggAhDVr1tR43ISEBAGAsHz5ckEQBGHcuHGCpaWl2rrJycmCkZGRMG7cOGXZnDlzBABqH3PmzFFpf//+feG9994TnJycBIlEInTu3FnYu3ev2mMdPXpU6Natm2BmZibY29sLU6ZMEWQyWY2vQ1M19Z0gCMLcuXMFAMKhQ4cEQRCEsrIy4aOPPhLatm0rSCQSoUmTJoK/v78wb948IT8/XxAEQYiPjxeGDBkiuLi4CKampoKLi4swevRo4erVq8r93rhxQwAgLFmypNox161bp+y/7OxslW1//vmnMGzYMKFZs2aCRCIRWrRoIYwcOVKIj49XqbdgwQLB1dVVEIvFAgDhxo0bwvXr14URI0YIHh4egpmZmWBhYSH4+/sLK1euFBQKRbU4avNeEREREdWGSBAEQQ95HhERERERUaPHa7iIiIiIiIh0hAkXERERERGRjjDhIiIiIiIi0hEmXERERI9w5MgRDBo0CC4uLhCJRNi5c+cj2yQkJOCZZ56BRCKBt7c31q9fr/M4iYjI8DDhIiIieoSioiL4+flhxYoVj1X/xo0beP7559G3b1+cPXsWU6dOxauvvop9+/bpOFIiIjI0XKWQiIioFkQiEX788UcMHTq0xjrTp0/H7t27cf78eWXZqFGjkJeXh71799ZDlEREZCh442MACoUC6enpsLa2hkgk0nc4RERPFEEQUFBQABcXF4jFjWPiRWJiIoKCglTKQkJCMHXq1BrblJaWorS0VPlcoVDg7t27aNasGccmIqJ6pO1xyWATrhUrVmDJkiXIyMiAn58fPvvsMwQEBDyy3ZYtWzB69GgMGTLksebYA0B6ejrc3d3rGDEREdXFrVu34Obmpu8wtCIjIwOOjo4qZY6OjpDJZLh//z7Mzc2rtYmJicG8efPqK0QiInoEbY1LBplwbd26FZGRkVi5ciUCAwOxbNkyhISE4MqVK3BwcKixXUpKCt577z307NmzVseztrYGUNmpNjY2tY5XLpdj//79CA4OhomJSa3bNzbsD+1if2oX+1P76tqnMpkM7u7uyv+Ln1RRUVGIjIxUPs/Pz0fz5s1x48aNOvWNXC7HoUOH0LdvX37mH8B+0S32r26xf3Xr7t27aNWqldbGJYNMuGJjYzFx4kSEh4cDAFauXIndu3dj7dq1mDFjhto2FRUVeOWVVzBv3jwcPXoUeXl5j328qqkaNjY2GidcFhYWsLGx4Yce7A9tY39qF/tT+7TVp41p2pyTkxMyMzNVyjIzM2FjY6P27BYASCQSSCSSauVNmzbVaGyqUvX+NGvWjJ/5B7BfdIv9q1vs3/qhrXHJ4BKusrIynD59GlFRUcoysViMoKAgJCYm1thu/vz5cHBwwIQJE3D06NGHHuO/8+RlMhmAyg+vXC6vdcxVbTRp2xixP7SL/ald7E/tq2ufNsb3omvXrtizZ49KWVxcHLp27aqniIiISF8MLuHKyclBRUWF2rnvly9fVtvm2LFjWLNmDc6ePftYx6hpnvz+/fthYWFR65irxMXFady2MWJ/aBf7U7vYn9qnaZ8WFxdrORLtKywsRFJSkvL5jRs3cPbsWTRt2hTNmzdHVFQU0tLS8O233wIAJk+ejM8//xzvv/8+/ve//+HgwYPYtm0bdu/era+XQEREemJwCVdtFRQUYOzYsVi1ahWkUuljtfnvPPmq6weCg4M1nlIYFxeH/v3787Qu2B/axv7ULvZn3eUVy3EjpwjXc4pwI6cYydkFkOVmYf3r/TS+hsvQ/fHHH+jbt6/yedUYMm7cOKxfvx537txBamqqcnvLli2xe/duvPPOO1i+fDnc3NywevVqhISE1HvsRESkXwaXcEmlUhgZGamd++7k5FStfnJyMlJSUjBo0CBlmUKhAAAYGxvjypUr8PLyUmlT0zx5ExOTOn0Bq2v7xob9oV3sT+1ifz6cvEKBW3eLkZxdhOvZhbieXYTrOZV/5haVVatvbSLSuE8bwvvQp08fPOy2levXr1fb5s8//9RhVERE1BAYXMJlamoKf39/xMfHK28qqVAoEB8fj4iIiGr1fX19ce7cOZWyWbNmoaCgAMuXL+dy70RED3G3qAzXswuR/E9SlfxPYpWaW4xyRc0Jxn8VlwOFpeVo0gCSJyIiovpkcAkXUDlVY9y4cejUqRMCAgKwbNkyFBUVKVctDAsLg6urK2JiYmBmZoZ27dqptLezswOAauVERE+qnMJSXMssxLWsApU/1Z2tehgHawk87S3haW8FT6klvBys0NxOgr8TE2AlMcghhYiISK8McnQMDQ1FdnY2oqOjkZGRgY4dO2Lv3r3KhTRSU1O1ctdnIqLGRBAE5BSW4VpmAa5lFeLqP38mZRXibi0SK4mxGC2llvCyt4Kn/b9/tpRawtqs+hksuVyO841nRXciIiKtMsiECwAiIiLUTiEEgISEhIe2VTeXnoioMckvluNCej6u/JNUVSVZecWPv8S6vbUEPg5W8LK3glfVWSt7S7jYmkMsZgZFRESkDQabcBERUaXcwlKcT5fhfFp+5SM9H7fu3n/s9g7WErRytIa3gxVaOVrDx9EKPg5WsLMw1WHUREREBDDhIiIyKJmykn8SKxnOpeXjQno+7uSXPFZbJxuzf5Kpf5MqHwdr2FpwIQsiIiJ9YcJFRKQHgiAgPb8E525XJlWVZ65kyC4ofWRbcxMjtHGxQXtXW/g6WcPnn7NXtuZMrIiIiAwNEy4iIh2TVyiQlFWIi+kyXLojw8V/Ho9zvZWVxBhtXWzQztUW7Vwrk6yWUisY8RorIiKiBoEJFxGRFuUXy3HxzgOJVboMSVmFKKtQPLKtrbkJ2rn+k1y52KKdqy1aNLXgAhZEREQNGBMuIiINCIKAW3fvK89WVZ29Sst7vMUspFaSf6YF2iiTK7cm5hCJmFwRERE1Jky4iIgeQ5asBGdS83D2Vh7+TL2Hi+kyFJSWP7KdWAR42luhjbMNWjvboI2LDVo7W8PB2qweoiYiIiJ9Y8JFRPQfJfIKnE/Lx58PJFjpj7FSoKWpEVo/kFi1cbZBK0drmJsa1UPUREREZIiYcBHRE00QBKTkFuPP1Hv/JFd5uHRHhnKF8NB2zrZmaPNAYtXa2QbNeb0VERER/QcTLiJ6ohSUlONSngjJB5Pxd7oMZ2/lPXK1QAtTI3Rws0VH9ybo6G6Hp5vbwdGGUwKJiIjo0ZhwEVGjVl6hwF+383Dkag6OJeXg7K08VCiMgEvJauuLRIC3vRWebm6Hju5N8HRzO/g4WMHYSFzPkRMREVFjwISLiBoVQRBwI6cIx5JycPRaDk4k5z50cYtmlqbKs1Yd3Zugg7stbMx4A2EiIiLSDiZcRNTg3Ssqw2/JOTh2rTLJetjS7J5SS7gaFeDFXn7o5CGFe1MuxU5ERES6w4SLiBqc0vIKnE65h6NJlUnW+fR8CDWscdHU0hTdvaXo6SNFD28p7C2NsWfPHjzXwRkmJjyTRURERLrFhIuIDJ5CIeBShgzHk3JxLCkHp27cxX15hdq6psZidPZogp4+9ujhLUUbZxuVlQPl8ocvkEFERESkTUy4iMjgCIKA1LvF+C0pF78l5yAxORd3i8pqrN/a2UZ5BiugZVOYmfC+V0RERGQYmHARkUHILijF8eQc5Vmsh12H5WgjQQ9ve/T0kaK7txT21pJ6jJSeVCtWrMCSJUuQkZEBPz8/fPbZZwgICKix/rJly/Dll18iNTUVUqkUw4cPR0xMDMzMeEsBIqInCRMuItKLghI5Tt24W3kWKykHVzILaqxrLTFGF69m6O7VDN29pfB2sOJCF1Svtm7disjISKxcuRKBgYFYtmwZQkJCcOXKFTg4OFSrv2nTJsyYMQNr165Ft27dcPXqVYwfPx4ikQixsbF6eAVERKQvTLiIqF6UlStwJvUejidV3g/rr9v5qFCoX+nC1FiMTi2aoLu3FN28mqG9qy3vg0V6FRsbi4kTJyI8PBwAsHLlSuzevRtr167FjBkzqtU/fvw4unfvjpdffhkA4OHhgdGjR+PkyZP1GjcREekfEy4i0pmM/BIkXMnCoStZ+C0pF4U13A9LJAI6uNqim7cU3b2k6OTRhNdhkcEoKyvD6dOnERUVpSwTi8UICgpCYmKi2jbdunXDhg0bcOrUKQQEBOD69evYs2cPxo4dW+NxSktLUVpaqnwuk8kAVC70UpfFXqracsEYVewX3WL/6hb7V7e03a9MuIhIa8orFDiTmvdPkpWNS3dkNdb1srf85wyWFF09m8HWgku0k2HKyclBRUUFHB0dVcodHR1x+fJltW1efvll5OTkoEePHhAEAeXl5Zg8eTJmzpxZ43FiYmIwb968auX79++HhYVF3V4EgLi4uDrvozFiv+gW+1e32L+6UVxcrNX9MeEiojrJLijF4avZOHQlC0evZkNWov4sVhMLE/RuZY+ePvbo7i2Fky0XDqDGKyEhAYsWLcIXX3yBwMBAJCUl4e2338aCBQswe/ZstW2ioqIQGRmpfC6TyeDu7o7g4GDY2NhoHItcLkdcXBz69+/Pe889gP2iW+xf3WL/6lZubq5W98eEi4hqpUIh4K/beUi4XHkW61xafo11O7jZos9TDuj7lD06uNnBSMyFLqjhkUqlMDIyQmZmpkp5ZmYmnJyc1LaZPXs2xo4di1dffRUA0L59exQVFWHSpEn44IMPIBZXvyZRIpFAIqm+4qaJiYlWvlBpaz+NDftFt9i/usX+1Q1t9ykTLiJ6JFmJHAcvVV6LdeRqNu4Vq5/bbGNmjF6t7NH3KQf0amXP5dqpUTA1NYW/vz/i4+MxdOhQAIBCoUB8fDwiIiLUtikuLq6WVBkZVV6XKAjqF4shIqLGSaOE6+TJkwgMDNR2LERkQIrLynHgUhZ++SsdCVezUVauUFuvjbMN+vrao89TDnja3Y6rCZLe6HJsioyMxLhx49CpUycEBARg2bJlKCoqUq5aGBYWBldXV8TExAAABg0ahNjYWDz99NPKKYWzZ8/GoEGDlIkXERE9GTRKuLp27Yr27dtj4sSJGDNmDOzs7LQcFhHpQ4m8AglXsrHr73QcvJSF+/KKanWsJMbo6SNFn6fs0buVA6/FIoOhy7EpNDQU2dnZiI6ORkZGBjp27Ii9e/cqF9JITU1VOaM1a9YsiEQizJo1C2lpabC3t8egQYPw4Ycfai0mIiJqGDRKuMaMGYMdO3bgrbfewvvvv4/hw4dj4sSJ6Nmzp7bjIyIdKytX4FhSNn756w72X8xUu3S71EqCFzo4I7itIzq1aApTY57FIsOj67EpIiKiximECQkJKs+NjY0xZ84czJkzRyvHJiKihkujb03ffvst0tPT8dlnn8HX1xcbNmxAnz594Ovri08++QQ5OTnajpOItKi8QoFj13Iw/fu/0fnDA/jf+j/ww59pKslWEwsTjA5ojk0TA3FyZj/MHdwW3bykTLbIYHFsIiIiQ6TxNydbW1tMmTIFZ86cwR9//IFJkyYhMzMT06ZNg5ubG0JDQ3HgwAFtxkpEdaBQCDh5PRezd55Hl5h4jFlzElv/uIX8+/8ugGFtZozh/m5YH94Zpz4IQsyw9ujmJeXqgtRgcGwiIiJDo5Wfqp955hl8+eWXSE9Px/r16yGVSvH9998jJCQEnp6e+Pjjj1FQUKCNQxFRLeQUlmLv+QzM/fkCui0+iNCvT+C7EzeRU1imrGNhaoTBfi5YFdYJf8wKwtIRfujzlANMuPgFNXAcm4iIyBBobVn4e/fu4dtvv8Xq1auRnp4OkUiE7t2749KlS5gxYwaWLVuGn376CZ07d9bWIYnoAYIgICW3GL+n3MUfKXfxR8o9XM8pUltXYizGs74OeKGDC571dYC5KVdNo8aJYxMREelbnROuQ4cOYdWqVdi5cydKSkpgb2+PadOm4bXXXoOnpydKS0uxdu1avP/++3jzzTdx4sQJbcRN9MSTVyhwMV32T4J1D3/cvKty5uq/TIxE6OVjj0F+Lghq4wgrCW/DR40XxyYiIjIUGn3jyszMxLp167BmzRpcv34dgiCgd+/emDx5MoYNG6Zyd2aJRILXX38dSUlJWLFixWMfY8WKFViyZAkyMjLg5+eHzz77DAEBAWrr/vDDD1i0aBGSkpIgl8vh4+ODd999F2PHjtXk5REZpMLScpxPycPvKffwR8pd/Jmap3bZ9iomRiJ0cLNDJ48m6NyiKTq3bApbc96Nnhqv+hibiIiIakujhMvNzQ0KhQJNmjTB1KlTMWnSJDz11FMPbWNvb4+yspp/fX/Q1q1bERkZiZUrVyIwMBDLli1DSEgIrly5AgcHh2r1mzZtig8++AC+vr4wNTXFL7/8gvDwcDg4OCAkJESTl0ikd4Ig4M9bedj9Vxr2/W2Ed04chEKoub61mTH8WzRBZ4+m6OzRFB3cbGFmwqmC9OTQ9dhERESkCY0SrsDAQEyePBkjRoyARCJ5rDYzZszAjBkzHqtubGwsJk6ciPDwcADAypUrsXv3bqxdu1btPvr06aPy/O2338Y333yDY8eOMeGiBqUqydrz9x3sOXcH6fkl/2ypvkqgs63ZP8lVE3TyaIpWjtZcTZCeaLoem4iIiDShUcJ17NgxbcehVFZWhtOnTyMqKkpZJhaLERQUhMTExEe2FwQBBw8exJUrV/DRRx+prVNaWorS0lLlc5lMBgCQy+WQy+Vq2zxMVRtN2jZG7I/aEQQBf93Ox94Lmfj1fOYDSda/RBDg42CFTh5N4N+8CTq1sIOLnblKHUVFORQ1zzCkf/DzqX117VNtvRe6HJuIiIg0pVHCdfv2bZw5cwa9evWCnZ1dte337t3D0aNH4e/vD1dX11rtOycnBxUVFXB0dFQpd3R0xOXLl2tsl5+fD1dXV5SWlsLIyAhffPEF+vfvr7ZuTEwM5s2bV618//79sLCwqFW8D4qLi9O4bWPE/qiZIACphcCfuWL8dVeEu6XVz0yJRQKeshXQsZmA9k0EWJrkA8gH0lJwNg04W+9RNy78fGqfpn1aXFyslePrcmwiIiLSlEYJ18KFC7F9+3akp6er3W5hYYH//e9/GDVqFD7//PM6Bfi4rK2tcfbsWRQWFiI+Ph6RkZHw9PSsNt0QAKKiohAZGal8LpPJ4O7ujuDgYNjY2NT62HK5HHFxcejfv7/KRdlPKvaHeoIg4O80GX49n4G9FzKRllf9TJaxWIRuXk0xsJ0TgnwdYGdhwv7UMvan9tW1T6tmGdSVIY5NREREGiVcBw8eRHBwcI1z5CUSCYKDg3HgwIFa71sqlcLIyAiZmZkq5ZmZmXBycqqxnVgshre3NwCgY8eOuHTpEmJiYtQmXBKJRG3sJiYmdfoCVtf2jQ37458k63Y+9py7g93n7uD2vfvV6hiLRejuLcXz7Z0R3NYRdhamavfF/tQu9qf2adqn2nofdDk2ERERaUqjhCstLQ0vvfTSQ+u0aNECu3btqvW+TU1N4e/vj/j4eAwdOhQAoFAoEB8fj4iIiMfej0KhULlOi6g+peYW44c/b+PHP9NwM7f6dCljsQjdvKV44RFJFhE9Pl2OTURERJrSKOEyNTV95BQQmUwGkUizFdMiIyMxbtw4dOrUCQEBAVi2bBmKioqUqxaGhYXB1dUVMTExACqvyerUqRO8vLxQWlqKPXv24LvvvsOXX36p0fGJNCErkePXc3ew43QaTqXcrbbdSHkmywnBbZzQxJJJFpE26XpsIiIi0oRGCVf79u2xa9cuxMbGqp26UVJSgp9//hnt27fXKKjQ0FBkZ2cjOjoaGRkZ6NixI/bu3atcSCM1NRVisVhZv6ioCG+88QZu374Nc3Nz+Pr6YsOGDQgNDdXo+ESPq0Ih4FhSDnacvo19FzJQWq5Q2S4SAd29pBjk58wki0jHdD02ERERaUKjhCs8PBwTJkzA4MGD8eWXX8LT01O5LTk5GW+88QbS09Mxf/58jQOLiIiocQphQkKCyvOFCxdi4cKFGh+LqLauZhZgx+nKKYNZBdWnrno7WOGlZ9zw4tOucLI100OERE+e+hibiIiIakvjhGvPnj3YsWMHfH190bJlS7i6uiItLQ03btxAeXk5QkNDlVMAiRqD3MJS/PxXOnacuY3zadWnLTWxMMFgPxe85O+G9q62nLZEVM84NhERkSHSKOECgG3btmHFihX44osvcPnyZVy7dg0A0KZNG0yZMgWvv/661oIk0pfS8gocupyF70+nIeFKFsoVgsp2Y7EIz/o6YNgzbnjW1wGmxuIa9kRE9YFjExERGRqNEy6RSKSc9ldUVIT8/HzY2trC0tJSm/ER6UVSViE2nLiJnWfTkFcsr7a9g5sthj3tisEdXdGU12URGQyOTUREZGi08nO8paUlXFxcOKBRg1ZeocC+Cxl4ZfUJBMUexvrjKSrJlqONBK/19sT+d3rh54geGN+9JZMtIgOm7bFpxYoV8PDwgJmZGQIDA3Hq1KmH1s/Ly8OUKVPg7OwMiUSCVq1aYc+ePVqJhYiIGg6Nz3ARNRY5haXY+vstbDxxE+n5JSrbJMZiDGjnhJeecUN3bymMxLwui+hJtHXrVkRGRmLlypUIDAzEsmXLEBISgitXrsDBwaFa/bKyMvTv3x8ODg74/vvv4erqips3b8LOzq7+gyciIr3SOOG6desWFi5ciAMHDiA9PR1lZWXV6ohEIpSXl9cpQCJdEAQBZ1Lz8G1iCvacuwN5heq1WS2aWWBslxYY4e8OWwsTPUVJRLWlq7EpNjYWEydOVC64sXLlSuzevRtr167FjBkzqtVfu3Yt7t69i+PHj8PEpPL/EA8Pj9q/ICIiavA0SriuX7+OwMBA3Lt3D23btkVpaSlatGgBMzMzXL9+HXK5HH5+fvwljwzO/bIK/PxXGr5NvIkL6aorDYpEwLNPOWBs1xbo5WMPMc9mETUouhqbysrKcPr0aURFRSnLxGIxgoKCkJiYqLbNzz//jK5du2LKlCn46aefYG9vj5dffhnTp0+HkZGR2jalpaUoLf33NhNVN3GWy+WQy6tfS/q4qtrWZR+NEftFt9i/usX+1S1t96tGCde8efOQn5+P+Ph49O7dG2KxGOHh4YiOjsadO3fw+uuv4+LFizhw4IBWgyXSVEpOETacuIntp28j/77qPyI7CxOEdnLHmC4t4N7UQk8RElFd6WpsysnJQUVFBRwdHVXKHR0dcfnyZbVtrl+/joMHD+KVV17Bnj17kJSUhDfeeANyuRxz5sxR2yYmJgbz5s2rVr5//35YWNT9/6a4uLg676MxYr/oFvtXt9i/ulFcXKzV/WmUcB04cADPPfccevfurSwThMopWc7Ozti6dSvat2+PmTNn4quvvtJOpES1VKEQkHAlC98m3sThq9nVtndws8XYLi0wyM8FZibqf3EmoobDkMYmhUIBBwcHfP311zAyMoK/vz/S0tKwZMmSGhOuqKgoREZGKp/LZDK4u7sjODgYNjY2Gscil8sRFxeH/v37K6c3EvtF19i/usX+1a3c3Fyt7k+jhCsnJwe+vr7/7sTYWCUTlEgk6N+/P3bu3FnnAIlqq7xCgc2/38LXR5Jx6+59lW2mxmK80MEZYV090NHdTj8BEpFO6GpskkqlMDIyQmZmpkp5ZmYmnJyc1LZxdnaGiYmJyvTB1q1bIyMjA2VlZTA1rb7CqUQigUQiqVZuYmKilS9U2tpPY8N+0S32r26xf3VD232qUcIllUpRVFSk8jwlJUV1x8bGyMvLq0tsRLWWcCULH+6+hGtZhSrlrnbmGNOlBUI7u3Mpd6JGSldjk6mpKfz9/REfH4+hQ4cCqDyDFR8fj4iICLVtunfvjk2bNkGhUEAsrrwDy9WrV+Hs7Kw22SIiosZLo4TLx8cHycnJyucBAQHYt28frl+/Dk9PT2RnZ+P777+Hl5eX1gIlephrmQVYuPtStamDvVrZI6xLC/T1deCS7kSNnC7HpsjISIwbNw6dOnVCQEAAli1bhqKiIuWqhWFhYXB1dUVMTAwA4PXXX8fnn3+Ot99+G2+++SauXbuGRYsW4a233tLOiyUiogZDo4Rr4MCBmDt3LvLy8mBnZ4epU6di165d6NChA1q3bo2kpCTIZDLMnTtXy+ESqbpbVIZP465i06lUVCj+Xdr96eZ2mPV8G/i3aKLH6IioPulybAoNDUV2djaio6ORkZGBjh07Yu/evcqFNFJTU5VnsgDA3d0d+/btwzvvvIMOHTrA1dUVb7/9NqZPn66tl0tERA2ERgnX66+/jj59+ijnpvfp0wdbtmzB3Llzcf78ebRo0QILFy7ExIkTtRosUZXS8gp8e/wm/u/gNRSU/Hs/HRdbM0wf6IvBfi4QiXhGi+hJouuxKSIiosYphAkJCdXKunbtihMnTmh0LCIiajw0SrhsbGwQGBioUjZixAiMGDFCK0ER1UQQBOy7kImYXy/hZu6/F8NbmBrhjT5eeLWnJ1ccJHpCcWwiIiJDpFHC9eyzz6J79+5YsGCBtuMhqtH5tHws+OUiTt64qywTiYCR/u54N7gVHGzM9BgdEekbxyYiIjJEGiVcJ0+eRJcuXbQdC5FaWbISLNl3Bd+fuQ3h38u00NWzGWa90BptXWz1FxwRGQyOTUREZIg0Srh8fX1x8+ZNbcdCpOJ+WQVWHb2OlYeTUVxWoSz3aGaBmc+1Rv82jrxOi4iUODYREZEh0ijhevPNNxEREYGLFy+iTZs22o6JnnAKhYCf/0rHR3sv405+ibLcxswYb/XzQVhXD5gaix+yByJ6EnFsIiIiQ6RRwuXp6Yk+ffqgS5cueO2119C5c2c4Oqo/29CrV686B0lPBkEQcPhqNpbuv4LzaTJluZFYhDGBzfF2UCvetJiIasSxiYiIDJFGCVefPn0gEokgCAI++eSTh07rqqioqHEbUZVTN+5i6b4rOJVyV6W871P2+OD51vB2sNZTZETUUHBsIiIiQ6RRwhUdHc1rZ0grzt3Ox9L9V3D4arZKeVsXG0wf4Iterez1FBkRNTQcm4iIyBBplHDNnTtXy2HQkyYpqwCxcVex51yGSrmXvSXeDX4KA9o6QSzmFycienwcm4iIyBBplHARaerW3WIsj7+GH87chuKBJd5d7cwxNcgHLz7tCmMjLohBRERERI0DEy6qF1myEnx+KAmbT6VCXvFvpiW1kuCtft4I7ewOibGRHiMkIiIiItI+jRIusVj8WPPkRSIRysvLNTkENRJ5xWVYefg61h+/gRK5Qllua26Cyb29MK5bC1iYMu8norrj2ERERIZIo2+6vXr1Ujuo5efn49q1aygqKoKfnx/s7OzqGh81UCUVwOeHkrH2t5soKP33i42FqREm9GiJV3t6wtbcRI8RElFjw7GJiIgMkUYJV0JCQo3biouLMWPGDOzduxdxcXGaxkUNVGl5Bb45fhPLzxihqDxZWW5qJMaYLi3wRl8vSK0keoyQiBorjk1ERGSItL46gYWFBf7v//4Ptra2mDZtmrZ3TwZKEATsPZ+B/rFHsOjXKygqr/yV2UgswugAdyRM64PoQW2YbBGRXnBsIiIifdHZxTM9e/bEhg0bdLV7MiAX02VY8MtFJF7PVSl/ob0T3g3xRUuppZ4iIyJSxbGJiIjqm84SruzsbBQWFupq92QAcgpL8cn+q9j6e6rKEu9dWjZBT+tsTBrRASYmvE6LiAwHxyYiIqpvWk+4FAoFNm7ciK1bt6JTp07a3j0ZgLJyBb45noL/i7+msiBG86YWmPlcazzbqil+/fVXPUZIRKSKYxMREemLRgmXp6en2vLy8nJkZWVBLpfDxMQEMTExdQqODIsgCIi/lIUP91zCjZwiZbmVxBgRz3ojvLsHJMZGkMvleoySiJ5UHJuIiMgQabRohkKhgCAI1R4mJiZo164dJk2ahNOnT6N3794aB7ZixQp4eHjAzMwMgYGBOHXqVI11V61ahZ49e6JJkyZo0qQJgoKCHlqfau9qZgHC1p7Cq9/+oUy2RCIgtJM7Dr7XG5N7e/HGxUSkV/UxNhEREdWWRme4UlJStByGqq1btyIyMhIrV65EYGAgli1bhpCQEFy5cgUODg7V6ickJGD06NHo1q0bzMzM8NFHHyE4OBgXLlyAq6urTmNt7O4VleHTA1ex8WQqKh64UCvAoymiB7VBO1dbPUZHRPQvXY9NK1aswJIlS5CRkQE/Pz989tlnCAgIeGS7LVu2YPTo0RgyZAh27typ0xiJiMjw6GzRjLqIjY3FxIkTER4eDgBYuXIldu/ejbVr12LGjBnV6m/cuFHl+erVq7Fjxw7Ex8cjLCysWv3S0lKUlpYqn8tkMgCAXC7XaDpcVZvGNJVOXqHAxlO38NnBZMhK/r1Oy9XODNNDWmFAW0eIRCK1r7kx9oc+sT+1i/2pfXXt04bwXtT2h8AqKSkpeO+999CzZ896jJaIiAyJRgnX7du3cebMGfTq1Qt2dnbVtt+7dw9Hjx6Fv79/rc8wlZWV4fTp04iKilKWicViBAUFITEx8bH2UVxcDLlcjqZNm6rdHhMTg3nz5lUr379/PywsLGoV74May800L94TYedNMTLvi5RlpmIB/V0V6ONcCCH1DH5NffR+Gkt/GAr2p3axP7VP0z4tLi7WyvF1OTbV9odAAKioqMArr7yCefPm4ejRo8jLy6vtSyIiokZAo4Rr4cKF2L59O9LT09Vut7CwwP/+9z+MGjUKn3/+ea32nZOTg4qKCjg6OqqUOzo64vLly4+1j+nTp8PFxQVBQUFqt0dFRSEyMlL5XCaTwd3dHcHBwbCxsalVvEDlr7NxcXHo379/g14G/fa9+5i76xIOX8tRKX+xozPe7e8DRxuzx9pPY+kPQ8H+1C72p/bVtU+rZhnUla7GJk1/CJw/fz4cHBwwYcIEHD169JHH0fbsiyo8q6se+0W32L+6xf7VLW33q0YJ18GDBxEcHAyJRKJ2u0QiQXBwMA4cOFCn4DSxePFibNmyBQkJCTAzU58gSCQStbGbmJjU6QtYXdvriyAI2H76NubvuojCB5Z592/RBNEvtIGfu51G+22o/WGo2J/axf7UPk37VFvvg67GJk1+CDx27BjWrFmDs2fPPvZxdDX7ogrP6qrHftEt9q9usX91Q1szL6polHClpaXhpZdeemidFi1aYNeuXbXet1QqhZGRETIzM1XKMzMz4eTk9NC2S5cuxeLFi3HgwAF06NCh1sd+EuUUliLqh3OIu/hvfzvbmmHGQF8M9nOBSCR6SGsiIsOhy7GpNgoKCjB27FisWrUKUqn0sdtpe/ZFFZ7VVY/9olvsX91i/+pWbm6uVvenUcJlamr6yCkgMplMoy/rpqam8Pf3R3x8PIYOHQqgcqnf+Ph4RERE1Nju448/xocffoh9+/bxppaPad+FDMz84Rxyi8qUZcP93RA9qA1szPiPl4gaFl2NTbX9ITA5ORkpKSkYNGiQskyhUAAAjI2NceXKFXh5eVVrp6vZF9reT2PDftEt9q9usX91Q9t9qtF9uNq3b49du3apzDV/UElJCX7++We0b99eo6AiIyOxatUqfPPNN7h06RJef/11FBUVKS9WDgsLU5lL/9FHH2H27NlYu3YtPDw8kJGRgYyMDBQWFmp0/MZOViLHu9v+wmvfnVYmW80sTfHVWH8sHeHHZIuIGiRdjU0P/hBYpeqHwK5du1ar7+vri3PnzuHs2bPKx+DBg9G3b1+cPXsW7u7utXthRETUoGmUcIWHh+P27dsYPHgwrl+/rrItOTkZQ4YMQXp6Ol599VWNggoNDcXSpUsRHR2Njh074uzZs9i7d69y/nxqairu3LmjrP/ll1+irKwMw4cPh7Ozs/KxdOlSjY7fmB1PzsHAZUex48xtZVn/No7Y904vhLR9+JRNIiJDpsuxqTY/BJqZmaFdu3YqDzs7O1hbW6Ndu3YwNTWt+4slIqIGQ6MpheHh4dizZw927NgBX19ftGzZEq6urkhLS8ONGzdQXl6O0NBQ5UCkiYiIiBqnECYkJKg81/XNLhuDEnkFPt57BWt/u6Ess5IYY86gNhju78ZrtYiowdPl2BQaGors7GxER0cjIyMDHTt2rPZDoFis0W+YRETUyGl84+Nt27ZhxYoV+OKLL3D58mVcu3YNANCmTRtMmTIFr7/+utaCpLo5dzsf72w7i6Ssf6dYdvFsiqUj/ODWpO4rXxERGQpdjk21+SHwv9avX6/xcYmIqGHTOOESiUTKwaeoqAj5+fmwtbWFpaWlNuOjOpBXKPDFoWR8dvAayhUCAMDUWIzpA3wR3s0DYjHPahFR48KxiYiIDI3GCdeDLC0tOZgZmOTsQkRuPYu/bucry9q52uDTkR3h42itx8iIiOoHxyYiIjIEGk04/+233xAZGYmMjAy12+/cuYPIyEicOHGiTsFR7SkUAtb/dgPPLT+qTLaMxCK81c8HP77RnckWETVaHJuIiMgQaZRwxcbGYteuXTXeiNjZ2Rm//PILPv300zoFR7WTnncfY9eexNxdF1FaXnnPF0+pJXa83g2R/VvBxIgXdBNR48WxiYiIDJFGUwp///139OvX76F1evXqhbi4OI2Coto7cT0Xk779A7KScmXZ+G4emD7AF+amRnqMjIiofnBsIiIiQ6RRwpWVlQVXV9eH1nFyckJWVpZGQVHtHLiYiSmbzijPajnbmmHJcD/08JHqOTIiovrDsYmIiAyRRgmXnZ0dUlNTH1rn5s2bsLKy0igoenw7Tt/G+zv+RsU/qxD2ecoey0c9DVtzEz1HRkRUvzg2ERGRIdLoop4uXbrgxx9/xK1bt9RuT01Nxc6dO9GtW7c6BUcPt/bYDby7/S9lsjWkowtWhXViskVETySOTUREZIg0SrgiIyNRXFyM7t2749tvv8WdO3cAVK4A9c0336B79+64f/8+3n33Xa0GS5UEQUDs/iuY/8tFZdm4ri3w6ciOXBiDiJ5YHJuIiMgQaTSlsFevXoiNjcW7776L8PBwAJU3mxSEyjMtYrEYy5cvR69evbQXKQGoXPZ9zs8X8N2Jm8qyt/v5YGqQD0Qi3siY6o9cLkdFRYW+w6g1uVwOY2NjlJSUNMj4DdF/+9TIyAgmJvV/pp1jExERGSKNb3z89ttvo2/fvli5ciV+//135Ofnw87ODgEBAZg8eTLatWuH0tJSSCQSbcb7RCsrV+C97X/h57/SlWVzBrVBePeWeoyKnjQymQw5OTkoLS3VdygaEQQBTk5OuHXrFn+k0BJ1fSqRSCCVSmFjY1OvsXBsIiIiQ6NxwgUAHTp0wBdffFGt/MyZM5gyZQq2bNmC3NzcuhyC/nG/rAKvbzyNhCvZACpvZrx0RAe8+LSbniOjJ4lMJkNaWhqsrKwglUphYmLS4JIWhUKBwsJCWFlZQSzmFFxteLBPRSIR5HI58vPzkZaWBgD1nnRxbCIiIkNSp4TrQXl5ediwYQPWrFmDv//+G4IgwNzcXFu7f6Ll35djwvrf8cfNewAAibEYX7zyDPq1dtRzZPSkycnJgZWVFdzc3BpcolVFoVCgrKwMZmZmTLi05L99am5uDmtra9y+fRs5OTn1nnA9iGMTERHpW50TrgMHDmDNmjX46aefUFpaCkEQ0LVrV4SHhyM0NFQbMT7RsmQlCFt7CpczCgAA1hJjrB7XCYGezfQcGT1p5HI5SktLIZVKG2yyRfVHJBLB1tYWaWlpkMvl9X5NF8cmIiIyFBolXLdu3cK6deuwbt06pKamQhAEuLq6Ii0tDePHj8fatWu1HecTKTW3GGPWnETq3WIAgNTKFOvDA9DO1VbPkdGTqGqBCX0shkANU9VnpaKiol4+NxybiIjIED12wiWXy7Fz506sWbMG8fHxqKiogKWlJV555RWEhYXh2WefhbGxMYyNtTZL8Yl2OUOGsDWnkFVQuTCBq505NrwaiJZSSz1HRk86nt2ix1UfnxWOTUREZOgeewRycXHB3bt3IRKJ0LdvX4SFhWHYsGGwtGQCoG2nb95D+LpTkJWUAwB8HKzw7YQAONvyugMiogdxbCIiIkP32AlXbm4uxGIx3nnnHbz//vuwt7fXZVxPrMNXszH5u9O4L6+cvuXnbof14zujiaWpniMjIjI8HJuIiMjQPfYSXePHj4e5uTliY2Ph5uaGwYMHY/v27SgrK9NlfE+UX/5Ox6vf/K5Mtrp7N8PGVwOZbBER1YBjExERGbrHTrjWrl2LO3fu4KuvvsIzzzyDX375BaNGjYKjoyNee+01HDt2TJdxNnqbTqbizc1/Ql4hAAAGtnPC2vGdYSXhdQdETzKRSIQ+ffroOwyDxbGJiIgMXa1uQmNlZYVXX30ViYmJuHDhAqZOnQpTU1OsWrUKvXv3hkgkwpUrV3Dz5k1dxdsoJSbnYuaP5yBU5loI7eSOz19+BhJjI/0GRkQqUlJSIBKJHvrIy8ur15hu3rwJIyMjiEQiLFmypF6PbSjqa2xasWIFPDw8YGZmhsDAQJw6darGuqtWrULPnj3RpEkTNGnSBEFBQQ+tT0REjZfGd/1s3bo1PvnkE6SlpWHbtm0IDg6GSCTC0aNH4eXlhX79+uG7777TZqyNUom8Ah/8eE75fGLPllj8UnsYibkSHJGh8vLywpw5c9Q+zMzM6jWWtWvXQqFQQCQScdlz6G5s2rp1KyIjIzFnzhycOXMGfn5+CAkJQVZWltr6CQkJGD16NA4dOoTExES4u7sjODgYaWlpdX2JRETUwNR5vpqxsTGGDx+O4cOH4/bt21i3bh3Wr1+PQ4cOISEhAWPHjtVGnI3WFwnJuJ5TBADwb9EEUQNbc9ltIgPn7e2NuXPn6jsMKBQKrF+/HlKpFC+88ALWr1+P48ePo1u3bvoOTe+0PTbFxsZi4sSJCA8PBwCsXLkSu3fvxtq1azFjxoxq9Tdu3KjyfPXq1dixYwfi4+MRFham+QsjIqIGR6sXCLm5uWH27NmYPXs24uPj+WvrIyRlFeDLhCQAgLFYhEUvtoeYZ7aIGoW///4bixYtwuHDh5GbmwtnZ2cMHjwYc+fORbNmzarVX716NZYtW4akpCTY29tj9OjRmD9//kOPERcXh9TUVERERCA0NBTr16/HmjVrVBKuBQsWIDo6Gt98843aL/o//PADXnrpJcycORMffvihSvmiRYtw4cIF2NjYYPDgwfj444/x9NNPA6icXtlQ1HVsKisrw+nTpxEVFaUsE4vFCAoKQmJi4mPto7i4GHK5HE2bNq2xTmlpKUpLS5XPZTIZgMp7jcnl8lrF/KCqtnXZR2PEftEt9q9usX91S9v9qrMVGfr164d+/frpavcNnkIhYOYP55WLZLzW2xNPOVnrOSoi0oaff/4ZI0eOhFgsxpAhQ+Du7o6LFy/i888/x759+3Dy5Ek0adJEWb8qKXJ0dMTEiRNhYmKCrVu34tKlSw89zpo1awAAYWFh6Ny5Mzw9PbFt2zYsX74cVlZWAIAxY8Zgzpw52LBhg9qEq2p63YNnfNauXYsJEybAxsYGYWFhsLW1xZ49e9C/f3/I5XKYmJjUuY/0RZOxKScnBxUVFXB0dFQpd3R0xOXLlx9rH9OnT4eLiwuCgoJqrBMTE4N58+ZVK9+/fz8sLCxqFbM6cXFxdd5HY8R+0S32r26xf3WjuLhYq/vjEnh6su2PWziVchcA0KKZBd581kfPERHVzaDPjiG7oPTRFfXI3lqCn6bUfbpdUlKS2imFAwYMgI+PD8aOHQupVIrffvsNLVq0UG7fsmULRo8ejejoaHz22WfKfc2fPx+urq44c+YMHBwcAABz585FQEBAjTHk5ubip59+gq+vLzp37gygMrmaP38+tm7digkTJgAAWrZsie7du+PgwYO4c+cOnJ2dlfu4e/cu9uzZg06dOsHX1xcAkJeXh7fffhuWlpb4448/4ONT+X/TokWLEBISgtOnT6u8Jnq0xYsXY8uWLUhISHjoNX5RUVGIjIxUPpfJZMprv2xsbDQ+vlwuR1xcHPr379+gk2VtY7/oFvtXt9i/upWbm6vV/THh0oPsglIs2vPvL9cfDm0PMxOuSEgNW3ZBKTJkJfoOo14kJyerPRNhZ2eHxMREyGQyfP7559USk1GjRmHJkiXYsmWLMuHatGkTysvLERkZqUy2AMDGxgazZs2q8Vqj7777DmVlZSrbw8LCMH/+fKxZs0aZcAGVZ6+OHTuGzZs3q3yh37p1K8rKyjBmzBhl2U8//YTCwkK89dZbymQLqLwmauHChU/k9WFSqRRGRkbIzMxUKc/MzISTk9ND2y5duhSLFy/GgQMH0KFDh4fWlUgkkEgk1cpNTEy08oVKW/tpbNgvusX+1S32r25ou0+ZcOnBgl8uQlZSDgB48WlX9PCR6jkiorqzt67+RdHQaCvGkJAQ7N27V+220NBQAMDJkyeRnJxcbXtJSQlycnKQk5MDqVSKv/76CwDQs2fPanXVlVVZs2YNRCKRSrLk5eWFbt264fjx47h06RJat24NABg5ciTeeustfPfddyoJ14YNG2BsbIzRo0cry6ri6dGjR7VjBgYGwtj4yRs2TE1N4e/vj/j4eAwdOhRA5YIl8fHxiIiIqLHdxx9/jA8//BD79u1Dp06d6ilaIiIyNE/eyKlnh69m4+e/0gEAdhYmmPV8az1HRKQdu96s/gXdECkUCp3u/+7dyqnCK1aseGi9oqIiSKVS5OfnA4DK2a0q/71mqMrJkydx/vx59O3bF82bN1fZFhYWhuPHj2Pt2rXK+3LZ2dnhhRdewI4dO3Dx4kW0adMGycnJOH78OJ577jmVY1ct1KAuHrFYDKn0yfyBKDIyEuPGjUOnTp0QEBCAZcuWoaioSLlqYVhYGFxdXRETEwMA+OijjxAdHY1NmzbBw8MDGRkZACrvGVZ1fR0REdWdXC5HRUXFI+sZGRnp7WwgE656dL+sArN2/nvPrZnPtUYzK8M/K0BEj6/qWptz586hXbt2j6xva2sLAMjKyqo2BfG/U9iqVC2WcejQoRpvI/Htt99i0aJFysFl7Nix2LFjB7777jvExMRgw4YNynJ18au7v5RCoUBOTg5cXV0f+boam9DQUGRnZyM6OhoZGRno2LEj9u7dq0yKU1NTIRb/e2vLL7/8EmVlZRg+fLjKfubMmWMQtxQgImroZDIZcnJyVFZ3fRSJRAKpVFqn62I1wYSrHi2Pv4Zbd+8DALp4NsUIfzc9R0RE2hYYGIgffvgBiYmJj5Vw+fn54YcffsDRo0eVi19UOXr0aLX6RUVF2LJlCywsLFSmAj7o999/x99//41ffvkFL774IgDgueeeQ7NmzbBp0yZ8+OGH2LhxI6ytrTFkyJBq8QDAb7/9hhEjRqhsO3XqFMrLyx/5mhqriIiIGqcQJiQkqDxvSMvmExE1NDKZDGlpabCysoJUKoWJiclD72MrCALkcjny8/OVN6Cvz6RL/Ogq9W/FihXw8PCAmZkZAgMDcerUqRrrXrhwAS+99BI8PDwgEomwbNmy+gu0Fi7dkWHV0esAAFMjMT58sT1vcEzUCIWHh8Pa2hoffPABLly4UG17cXExTpw4oXz+8ssvw8jICLGxsSpnlWQyGRYuXFit/fbt21FQUIDhw4dj9erVah9VUwmrzoQBlRcAh4aGIjU1FR9//DGuXbuGl156Cebm5ir7HzJkCKysrLBmzRqVa9DKy8sxe/ZszTuGiIhIS3JycmBlZQU3NzfY2NjA3NwcZmZmNT7Mzc1hY2MDNzc3WFlZIScnp17jNbiEa+vWrYiMjMScOXNw5swZ+Pn5ISQkRO30FqDyy4unpycWL178yNWi9KVCISDqh3OoUFTec+uNvl7wsuccfqLGyN7eHps3b0ZhYSH8/Pzwwgsv4L333sObb76JQYMGwcnJSWVKmbe3N6Kjo5GWloYOHTrgrbfeQmRkJNq3b6+ySmCVqiSq6tohdYKCguDm5oa9e/ciPT1dWV41fTA6Olrl+YPs7OwQGxuLwsJC+Pv7Y/LkyZg+fTqefvpp3Lt3Dy4uLipT54iIiOqTXC5HaWkpbG1ta33yQiQSwdbWFqWlpfV602iDm1IYGxuLiRMnKr9MrFy5Ert378batWsxY8aMavU7d+6snIajbrs6paWlKvM9qy4Sl8vlGnX+o+72veFkKs7eygMAeEot8Wr3Fo36zuC8+7l2GUp/yuVyCIIAhUKh84UndEkQBOWftX0dVfUf1XbgwIE4ffo0li5divj4eMTFxcHS0hJubm4YP348XnnlFZX2s2bNgpOTE5YvX46vvvoKDg4OCA0Nxbx585QLLCgUCly5cgXHjh1Dy5Yt0bNnz4fGEBYWhkWLFmHdunWIiooCAAQEBMDHxwfXrl2Dm5sbevXqpXYfEyZMgK2tLRYvXoz169fD1tYWgwYNwuLFi9GyZUt4eXmptKupTxUKhXIah5FRzbe+0Pdnm4iIGo6qBTI0XQCjql1FRUW9LaJhUAlXWVkZTp8+rfxyAFSuihUUFITExEStHScmJkbtPXT2798PCwsLjfer7m7feaXA4r+MAFRm4M875iN+v/rlpBsb3v1cu/Tdn8bGxnByckJhYSHKysr0Gos2FBQU1LpN06ZNce/ePQD//lBTE2dnZ3zyySc1bv9v+5EjR2LkyJEqZXK5XOV4zs7OyuePin/atGmYNm1atWM9OEW7sLCwxvbBwcEIDg5WKbt+/ToKCwvh6emp9vX/N6aysjLcv38fR44ceei1X8XFxQ99LURERP+l6aU5+rikx6ASrpycHFRUVFRbCtnR0RGXL1/W2nGioqJU7kUjk8ng7u6O4OBgjS6ge9jdviM2n0VpReV0yBH+rnhraNu6Bd8A8O7n2mUo/VlSUoJbt27BysoKZmZmeoujrgRBQEFBAaytrXkdZQ3u3bsHCwsLlZvw3r9/XzkV8aWXXlL5v7KmPi0pKYG5uTl69er10M/Mo5JXIiKihsygEq76IpFIVL5IVKnr3br/2/7AxUzsu1iZbEmtTPHB822eqASEdz/XLn33Z0VFBUQiEcRicYO+hqdqylvVa6Hqjh49igkTJiA4OBjNmzdHTk4ODh48iJSUFDz77LMYPXq0St/V1KdisRgikeiRn13+P0FERI2ZQSVcUqkURkZG1e49k5mZabALYtSkqLQc0T+dVz6f/UIb2FmY6jEiIqLH07ZtW/Tv3x+//fYbdu7cCaBycY8FCxbgvffeY6JKRERUCwaVcJmamsLf3x/x8fEYOnQogMpfTuPj42u894mh+mT/VaTnlwAAevpIMdjPRc8RERE9Hh8fH2zZskXfYRARETUKBpVwAUBkZCTGjRuHTp06ISAgAMuWLUNRUZFy1cKwsDC4uroiJiYGQOVF2RcvXlT+PS0tDWfPnoWVlRW8vb318hrO3c7H+uM3AAASYzEWDm3Ha0WIiIiIiJ5ABpdwhYaGIjs7G9HR0cjIyEDHjh2xd+9e5UIaqampKtNZ0tPT8fTTTyufL126FEuXLkXv3r2RkJBQ3+GjvEKBGT/8jX9uuYW3g3zQopllvcdBRERERET6Z3AJFwBERETUOIXwv0mUh4eH8h4whmD98RRcSK9cccvXyRoTe3rqOSIi7TKkf29k2PhZISIiXdF0jNHH2MQrn7UoLe8+Ptl/FQAgEgGLhrWHiRG7mBqHqhvX8ia19LiqPisPu+kxERFRbdT1+4g+xiZmA1oiCMC8Xy7hvrzy7tdjAlvgmeZN9BwVkfaYmJhAIpEgPz+fZy7okQRBQH5+PiQSCZd9JyIiranL9xF9jU0GOaWwIfrrrgiHruYAABysJZg24Ck9R0SkfVKpFGlpabh9+zZsbW1hYmLS4BaEUSgUKCsrQ0lJCZc315IH+1QkEkEulyM/Px+FhYVwdXXVd3hERNTI1Pb7iCAIeh2bmHBpQUGJHDtu/PvFbd7gtrAx4y+61PjY2NgAAHJycpCWlqbnaDQjCALu378Pc3PzBpcsGip1fSqRSODq6qr8zBAREWmLpt9H9DU2MeHSgqVx1yCTV37J6OfrgAHtGtZNmolqw8bGBjY2NpDL5aioqNB3OLUml8tx5MgR9OrVi1PdtOS/fWpkZMS+JSIinart9xF9jk1MuOro9M172Pz7bQCAhakR5vOeW/SEMDExaZBfqo2MjFBeXg4zM7MGGb8hYp8SEZG+NITvI7yAoY7EIqBlMwsAwNR+3nC1M9dzREREREREZCiYcNXR082b4Ocp3TC8ZQXGBrrrOxwiIiIiIjIgTLi0QGIsRk8nAca85xYRERERET2AGQIREdFjWLFiBTw8PGBmZobAwECcOnXqofW3b98OX19fmJmZoX379tizZ089RUpERIaECRcREdEjbN26FZGRkZgzZw7OnDkDPz8/hISEICsrS23948ePY/To0ZgwYQL+/PNPDB06FEOHDsX58+frOXIiItI3JlxERESPEBsbi4kTJyI8PBxt2rTBypUrYWFhgbVr16qtv3z5cgwYMADTpk1D69atsWDBAjzzzDP4/PPP6zlyIiLSNy4Lj8qbdgKATCbTqL1cLkdxcTFkMpnBL0tZH9gf2sX+1C72p/bVtU+r/u+t+r/Y0JSVleH06dOIiopSlonFYgQFBSExMVFtm8TERERGRqqUhYSEYOfOnTUep7S0FKWlpcrn+fn5AIC7d+9CLpdrHH/V+5Obm8vP/APYL7rF/tUt9q9u3b17F4D2xiUmXAAKCgoAAO7uXGWQiEhfCgoKYGtrq+8wqsnJyUFFRQUcHR1Vyh0dHXH58mW1bTIyMtTWz8jIqPE4MTExmDdvXrXyli1bahA1ERHVVW5urlbGJSZcAFxcXHDr1i1YW1trdNNimUwGd3d33Lp1CzY2NjqIsGFhf2gX+1O72J/aV9c+FQQBBQUFcHFx0UF0DUdUVJTKWTGFQoG7d++iWbNmGo1NVfiZV4/9olvsX91i/+pWfn4+mjdvjqZNm2plf0y4UDk1xM3Nrc77sbGx4Yf+AewP7WJ/ahf7U/vq0qeGeGarilQqhZGRETIzM1XKMzMz4eTkpLaNk5NTreoDgEQigUQiUSmzs7PTLGg1+JlXj/2iW+xf3WL/6pZYrJ3lLrhoBhER0UOYmprC398f8fHxyjKFQoH4+Hh07dpVbZuuXbuq1AeAuLi4GusTEVHjxTNcREREjxAZGYlx48ahU6dOCAgIwLJly1BUVITw8HAAQFhYGFxdXRETEwMAePvtt9G7d2988skneP7557Flyxb88ccf+Prrr/X5MoiISA+YcGmBRCLBnDlzqk0FeVKxP7SL/ald7E/texL6NDQ0FNnZ2YiOjkZGRgY6duyIvXv3KhfGSE1NVZl60q1bN2zatAmzZs3CzJkz4ePjg507d6Jdu3b1HvuT8P5ogv2iW+xf3WL/6pa2+1ckGOo6vERERERERA0cr+EiIiIiIiLSESZcREREREREOsKEi4iIiIiISEeYcBEREREREekIE646OHLkCAYNGgQXFxeIRCLs3LlT3yHpzdy5cyESiVQevr6++g6rQXnU50kQBERHR8PZ2Rnm5uYICgrCtWvX9BNsA/Co/hw/fny1z+yAAQP0E2wDEBMTg86dO8Pa2hoODg4YOnQorly5olKnpKQEU6ZMQbNmzWBlZYWXXnqp2s1/qf5wjFKP45V2cezSLY5lulVfYxsTrjooKiqCn58fVqxYoe9QDELbtm1x584d5ePYsWP6DqlBedTn6eOPP8b//d//YeXKlTh58iQsLS0REhKCkpKSeo60YXicf58DBgxQ+cxu3ry5HiNsWA4fPowpU6bgxIkTiIuLg1wuR3BwMIqKipR13nnnHezatQvbt2/H4cOHkZ6ejmHDhukx6icbx6iacbzSHo5dusWxTLfqbWwTSCsACD/++KO+w9CbOXPmCH5+fvoOo9H47+dJoVAITk5OwpIlS5RleXl5gkQiETZv3qyHCBsWdf8+x40bJwwZMkQv8TQGWVlZAgDh8OHDgiBUfh5NTEyE7du3K+tcunRJACAkJibqK0z6x5M+Rj2I45XucOzSLY5luqersY1nuEhrrl27BhcXF3h6euKVV15BamqqvkNqNG7cuIGMjAwEBQUpy2xtbREYGIjExEQ9RtawJSQkwMHBAU899RRef/115Obm6jukBiM/Px8A0LRpUwDA6dOnIZfLVT6jvr6+aN68OT+jZHA4XtUPjl31g2OZ9uhqbGPCRVoRGBiI9evXY+/evfjyyy9x48YN9OzZEwUFBfoOrVHIyMgAADg6OqqUOzo6KrdR7QwYMADffvst4uPj8dFHH+Hw4cMYOHAgKioq9B2awVMoFJg6dSq6d++Odu3aAaj8jJqamsLOzk6lLj+jZGg4XtUfjl26x7FMe3Q5thlrM1B6cg0cOFD59w4dOiAwMBAtWrTAtm3bMGHCBD1GRqTeqFGjlH9v3749OnToAC8vLyQkJKBfv356jMzwTZkyBefPn+d1L9QgcbyixoRjmfbocmzjGS7SCTs7O7Rq1QpJSUn6DqVRcHJyAoBqq+JkZmYqt1HdeHp6QiqV8jP7CBEREfjll19w6NAhuLm5KcudnJxQVlaGvLw8lfr8jJKh43ilOxy76h/HMs3oemxjwkU6UVhYiOTkZDg7O+s7lEahZcuWcHJyQnx8vLJMJpPh5MmT6Nq1qx4jazxu376N3NxcfmZrIAgCIiIi8OOPP+LgwYNo2bKlynZ/f3+YmJiofEavXLmC1NRUfkbJoHG80h2OXfWPY1nt1NfYximFdVBYWKjyC8KNGzdw9uxZNG3aFM2bN9djZPXvvffew6BBg9CiRQukp6djzpw5MDIywujRo/UdWoPxqM/T1KlTsXDhQvj4+KBly5aYPXs2XFxcMHToUP0FbcAe1p9NmzbFvHnz8NJLL8HJyQnJycl4//334e3tjZCQED1GbbimTJmCTZs24aeffoK1tbVy7rqtrS3Mzc1ha2uLCRMmIDIyEk2bNoWNjQ3efPNNdO3aFV26dNFz9E8mjlHqcbzSLo5dusWxTLfqbWzT8mqKT5RDhw4JAKo9xo0bp+/Q6l1oaKjg7OwsmJqaCq6urkJoaKiQlJSk77AalEd9nhQKhTB79mzB0dFRkEgkQr9+/YQrV67oN2gD9rD+LC4uFoKDgwV7e3vBxMREaNGihTBx4kQhIyND32EbLHV9CUBYt26dss79+/eFN954Q2jSpIlgYWEhvPjii8KdO3f0F/QTjmOUehyvtItjl25xLNOt+hrbRP8cjIiIiIiIiLSM13ARERERERHpCBMuIiIiIiIiHWHCRUREREREpCNMuIiIiIiIiHSECRcREREREZGOMOEiIiIiIiLSESZcREREREREOsKEi4iIiIiISEeYcBFRrYlEIvTp00ffYRAREQHguESGjQkXkQ6kpKRAJBKpPExMTODq6oqRI0fijz/+0HeIRET0BOG4RKQ/xvoOgKgx8/LywpgxYwAARUVFOH36NLZv346dO3fiwIED6NWrl54jJCKiJwnHJaL6x4SLSIe8vb0xd+5clbLFixcjKioKs2fPxuHDh/UTGBERPZE4LhHVP04pJKpnEyZMAACcPn1apTwnJwdTp05Fy5YtIZFI4ODggJEjR+L8+fPV9tGnTx+IRCK1+x8/fjxEIhFSUlKUZevXr4dIJML69euxf/9+dOvWDRYWFmjWrBnGjRuH3NxctftavXo12rVrBzMzM7i7u+P9999HSUmJhq+ciIgMEcclIt3iGS4iPTE2/vefX3Z2Nrp27Yrk5GT06dMHo0aNwo0bN/D9999j9+7d2LdvH3r06FHnY/7888/YvXs3Bg0ahG7duuHIkSP49ttvkZycjGPHjqnUXbBgAaKjo+Ho6IiJEyfCxMQEW7duxaVLl+ocBxERGR6OS0S6wYSLqJ6tXr0aAFQGqunTpyM5ORlRUVFYtGiRsnzPnj14/vnnER4ejitXrkAsrttJ6V27diEhIQHdu3cHAFRUVCAoKAgJCQk4ceIEunTpAgBISkrC/Pnz4erqijNnzsDBwQEAMHfuXAQEBNQpBiIiMiwcl4h0i1MKiXQoKSkJc+fOxdy5czFt2jQ8++yzmDlzJhwdHbFkyRIAQFlZGTZv3oxmzZph1qxZKu2fe+459O/fH0lJSfjtt9/qHM/LL7+sHNQAwMjICOPGjQMA/P7778ryTZs2oby8HJGRkcpBDQBsbGyqxUhERA0HxyWi+sczXEQ6lJycjHnz5qmUOTk54ejRo/D29gYAXL58GSUlJejbty8sLCyq7aNv376Ii4vD2bNn0bNnzzrF4+/vX63Mzc0NAJCXl6cs++uvvwBA7fHqGgMREekPxyWi+sczXEQ6FBISAkEQIAgCsrKysGTJEmRlZWHw4MEoLCwEAMhkMgCAo6Oj2n04Ozur1KsLGxubamVVc/YrKiqUZfn5+QCg8itilZriJCIiw8dxiaj+MeEiqif29vZ47733MHPmTFy6dEk5BaJqsMnMzFTbLiMjQ6UeAOWc+fLy8mr1qwalurC1tQUAZGVlVdtWU5xERNSwcFwiqh9MuIjq2cyZM+Hi4oIvvvgCKSkp8PX1hZmZGX7//XcUFxdXq5+QkAAA6Nixo7KsSZMmAIC0tDSVugqFQjntoi78/PwAAEePHq22TV0ZERE1XByXiHSLCRdRPTM3N8f06dMhl8uxYMECmJqaYvTo0cjJyUFMTIxK3b1792Lfvn3w9vZWuai4c+fOACrvY/Kg2NhY3Lhxo84xvvzyyzAyMkJsbKzKr4kymQwLFy6s8/6JiMhwcFwi0i0mXER6MGnSJLi4uCjvNfLRRx/B09MTCxcuRL9+/TBz5ky8/PLLGDRoECwsLLBu3TqVpXfDw8PRpEkTzJ07Fy+++CLee+899OnTB4sXL0bv3r3rHJ+3tzeio6ORlpaGDh064K233kJkZCTat28PHx+fOu+fiIgMC8clIt1hwkWkB2ZmZoiKikJ5eTnmzZsHe3t7nDx5Em+99RaSk5OxdOlSxMXFYejQoTh58mS1m0s6Ojri0KFD6NevH/bv349Vq1bBzs4OJ06cgIeHh1ZijI6OxqpVq9CsWTN89dVX2L59O0aOHIlt27ZpZf9ERGQ4OC4R6Y5IEARB30EQERERERE1RjzDRUREREREpCNMuIiIiIiIiHSECRcREREREZGOMOEiIiIiIiLSESZcREREREREOsKEi4iIiIiISEeYcBEREREREekIEy4iIiIiIiIdYcJFRERERESkI0y4iIiIiIiIdIQJFxERERERkY4w4SIiIiIiItKR/wcClh2xYZMmhQAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1MAAAD0CAYAAAB+SXxXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8WgzjOAAAACXBIWXMAAA9hAAAPYQGoP6dpAABrxElEQVR4nO3deVxU1fsH8M+wg4obmyCBqIm7hfueG6aZZpo7aoqpoCmZWwVuZWWplVu5fnPfyiVNJdTURNPU0nJFcUEBQQVlHZjz++P+ZmCYAYZhhhng83695iVz7vZwGOeZZ+6558qEEAJERERERERUJBamDoCIiIiIiKg0YjFFRERERESkBxZTREREREREemAxRUREREREpAcWU0RERERERHpgMUVERERERKQHFlNERERERER6YDFFRERERESkBxZTREREREREemAxRURERKSnzp07o3PnziV+3A0bNkAmk+H8+fMlfmxzFh0dDZlMhg0bNqja5syZA5lMZrKYTPUaoZLBYopMIioqCu+99x58fHxgZ2cHR0dHtGvXDt988w3S0tJU63l7e+ONN95Q21Ymk2l9uLm5qa337Nkz2NnZQSaT4erVq1rjGDVqlNo+bG1t8fLLLyM0NBTp6eka62/fvh3Dhw9H3bp1IZPJCnxzzMjIwIwZM+Du7g57e3u0atUK4eHhWtc9ffo02rdvDwcHB7i5uWHy5Ml48eJFvvvWV94+c3R0RKdOnXDgwAGDH0tJmdhkMhl2796tsVyZ5BISEoq879OnT2POnDl49uyZxrLOnTtrfZ307NlTY92i/K2IqHTTNf/o6+HDh5gzZw4uXbpU/GAN5M8//8TEiRPh5+cHa2vrQguLtWvXon79+rCzs0PdunXx3Xffaaxz/fp1TJ06FW3btlXl2ujoaCP9BpLjx4+jf//+cHNzg42NDVxcXNCnTx/89NNPRj0uACgUCjg7O+PLL7+ElZUVhg8fnu+6z58/h729Pfr376+xLL/clPcxZ84cjW1z59PcebxZs2ZYtmwZsrOzDfkrk46sTB0AlT8HDhzAwIEDYWtri4CAADRq1AiZmZk4deoUPvzwQ/z777/44YcfCtxH9+7dERAQoNZmb2+v9nznzp2qImvz5s1YsGCB1n3Z2tpizZo1AICkpCTs3bsX8+fPR1RUFDZv3qy27sqVK/HXX3+hRYsWSExMLDDGUaNGYdeuXZgyZQrq1q2LDRs2oFevXjh27Bjat2+vWu/SpUvo2rUr6tevj8WLF+PBgwf46quvcPPmTfz6668FHkMfyr4TQuDu3btYuXIl+vTpg19//RX+/v4GP15u8+bNQ//+/Q32DeHp06cxd+5cjBo1ClWqVNFYXrNmTSxcuFCtzd3dXWM9Xf9WRFS6GSL/FObhw4eYO3cuvL290axZM8MEXkwHDx7EmjVr0KRJE/j4+ODGjRv5rvv9999j/PjxePvttxESEoKTJ09i8uTJSE1NxYwZM1TrRUZG4ttvv0WDBg1Qv359oxePYWFhmDdvHurWrYv33nsPXl5eSExMxMGDB/H2229j8+bNGDp0qNZtP/74Y8ycObNYx//zzz+RkJCA3r1749ixY9i7dy9SU1Ph4OCgse5PP/2E9PR0VcF15MgR1bKPPvoIY8eOVT0/d+4cvv32W8yePRv169dXtTdp0iTfWIYMGYJevXoBkD63HDx4EJMmTcLdu3exaNGiYv2epAdBVIJu374tKlasKHx9fcXDhw81lt+8eVMsXbpU9dzLy0v07t1bbR0AIigoqNBjdezYUfTv319MnTpV1KpVS+s6I0eOFBUqVFBrUygUonXr1kImk4nY2Fi1Zffu3RPZ2dlCCCEaNmwoOnXqpHW/Z8+eFQDEokWLVG1paWmidu3aok2bNmrrvv7666JGjRoiKSlJ1bZ69WoBQBw+fLjQ37MotPXdf//9JwCI119/3aDHUrpz544AIJo1ayYAiN27d6stDwsLEwDE48ePi7zvRYsWCQDizp07Gss6deokGjZsWOg+ivK3IqLSq6j5R1edOnVSywXnzp0TAMT69euLEW3h1q9fLwCIc+fOFbpubGysSE1NFUIIERQUJPL7+JeamiqqV6+ukXeHDRsmKlSoIJ48eaJqS0xMFMnJyUKIgt+LDWHnzp0CgBgwYIDIzMzUWH7o0CGxf/9+IUROzjF0/3/yySfCy8tLCCHExo0bBQCxdetWrev26NFDVK5cWaSnpxe6X+XvduzYsULXVf5uufOVENLnlhYtWgh3d/dC90GGx2F+VKK+/PJLvHjxAmvXrkWNGjU0ltepUwfvv/9+sY9z7949nDx5EoMHD8bgwYNx584dnD59WqdtZTIZ2rdvDyEEbt++rbbM09MTFhaF/7fZtWsXLC0tMW7cOFWbnZ0dxowZg8jISNy/fx8AkJycjPDwcAwfPhyOjo6qdQMCAlCxYkXs2LFDp5iLo379+nByckJUVJRae0ZGBsLCwlCnTh3Y2trC09MT06dPR0ZGhtp64eHhaN++PapUqYKKFSuiXr16mD17tsZxBg8ejJdffhnz5s2DEKLQuM6ePYuePXuicuXKcHBwQKdOnfDHH3+ols+ZMwcffvghAKBWrVqqIQ95h5lkZWUVOGRS178VEZVuRc0/69evR5cuXeDi4gJbW1s0aNAAK1euLPAYx48fR4sWLQAAo0ePVr0v5b5+p7D3NqWYmBiMGTMG7u7usLW1Ra1atTBhwgRkZmaqrZeRkYGQkBA4OzujQoUKeOutt/D48WO1dVxdXTVGb2hz7NgxJCYmYuLEiWrtQUFBSElJURsSXq1aNVSqVKnQfRrCJ598gmrVqmHdunWwtrbWWO7v769xSUBu2q6ZkslkCA4OxubNm1GvXj3Y2dnBz88PJ06c0LqPAwcOoHfv3gCAt956CxUqVMCWLVs01ouPj0dERAQGDBgAW1tbAMa/Zkomk8HV1RVWVhxwZgrsdSpR+/fvh4+PD9q2bVus/aSnp2tcY1OpUiXVG9fWrVtRoUIFvPHGG7C3t0ft2rWxefNmnY+r/EBetWpVveK7ePEiXn75ZbUCCQBatmwJQBra5+npicuXLyMrKwvNmzdXW8/GxgbNmjXDxYsX9Tp+USQlJeHp06eoXbu2qk2hUODNN9/EqVOnMG7cONSvXx+XL1/GkiVLcOPGDezZswcA8O+//+KNN95AkyZNMG/ePNja2uLWrVtaPxhYWlri448/RkBAAH7++WetY8mVjh49itdffx1+fn4ICwuDhYWF6oPNyZMn0bJlS/Tv3x83btzA1q1bsWTJEjg5OQEAnJ2dVfu5ceMGKlSogMzMTLi6uiIwMBChoaFqyVjXvxURlW5FzT8rV65Ew4YN8eabb8LKygr79+/HxIkToVAoEBQUpHWb+vXrY968eQgNDcW4cePQoUMHAFAdU5f3NkAaKtiyZUs8e/YM48aNg6+vL2JiYrBr1y6kpqbCxsZGdcxJkyahatWqCAsLQ3R0NJYuXYrg4GBs3769yH2kzDl5c5Kfnx8sLCxw8eLFAq8VMoabN2/i2rVrePfddw1evP3+++/Yvn07Jk+eDFtbW6xYsQI9e/bEn3/+iUaNGqnWi42NxcWLFzFv3jwAQIUKFdC3b1/s2rULT548QbVq1VTrbt++HdnZ2Rg2bJhBY80tNTVV9RkoOTkZv/76Kw4dOoRZs2YZ7ZhUAFOfGqPyIykpSQAQffv21Xmb/Ib5aXvkPqXfuHFjMWzYMNXz2bNnCycnJyGXy9X2pRzm9/jxY/H48WNx69Yt8dVXXwmZTCYaNWokFApFvrEVNMyvYcOGokuXLhrt//77rwAgVq1aJYTIOb1/4sQJjXUHDhwo3Nzc8j2+PgCIMWPGiMePH4v4+Hhx/vx50bNnT41hAxs3bhQWFhbi5MmTatuvWrVKABB//PGHEEKIJUuWFDpEL/ewhKysLFG3bl3RtGlTVd/mHeanUChE3bp1hb+/v1r/p6amilq1aonu3bur2goaWvLuu++KOXPmiN27d4sff/xRvPnmmwKAeOedd9TW0/VvRUSllz75RzksLjd/f3/h4+Oj1qbrML+ivLcFBAQICwsLrUP4lNsqh/l169ZNbX9Tp04VlpaW4tmzZ1p/r4KG+QUFBQlLS0uty5ydncXgwYO1LjPmML+9e/cKAGLJkiU6ra9tmJ8yz+Sm/Oxw/vx5Vdvdu3eFnZ2deOutt9TWXbt2rbC3t1d7TRw4cEAAEN9//73auq1btxYeHh6qSwKE0HyN5KbPMD9tjwkTJhT4mYWMh8P8qMQkJycDgEG+Werbty/Cw8PVHsrJE/755x9cvnwZQ4YMUa0/ZMgQJCQk4PDhwxr7SklJgbOzM5ydnVGnTh1MmzYN7dq1w969e/WeKCEtLU11liw3Ozs71fLc/+a3riFmlspr7dq1cHZ2houLC5o3b46IiAhMnz4dISEhqnV27tyJ+vXrw9fXFwkJCapHly5dAEhDQQCoJn3Yu3cvFApFocdWnp36+++/VWe38rp06RJu3ryJoUOHIjExUXXslJQUdO3aFSdOnNDpWGvXrkVYWBj69++PESNGYO/evQgMDMSOHTtw5swZ1Xq6/q2IqPTSJ//kHhaXlJSEhIQEdOrUCbdv30ZSUlKRY9D1vU2hUGDPnj3o06ePxhkiABp5ady4cWptHTp0QHZ2Nu7evVvkGNPS0tTOeuVmrJxUGEN+dsirTZs28PPzUz1/6aWX0LdvXxw+fFhtZryDBw/itddeU3tN9OjRA87OzmpD/e7cuYMzZ85gyJAhOl0SoK9x48apPvvs3r0bQUFB+P7779XyOJUcDvOjEqMcRvX8+fNi76tmzZro1q2b1mWbNm1ChQoV4OPjg1u3bgGQkoC3tzc2b96sGvOsZGdnh/379wMAHjx4gC+//BLx8fE6jS/Pj729vca1RQBU060r9638N791C4shNjZW7XnlypUL3aZv374IDg5GZmYmzp07h88++wypqalqb/w3b97E1atX1YbM5RYfHw8AGDRoENasWYOxY8di5syZ6Nq1K/r3748BAwbkm0iGDRuG+fPnY968eejXr5/G8ps3bwIARo4cme/vkJSUpNcQzA8++ACrV6/Gb7/9htatWwPQ/W9FRKWXPvnnjz/+QFhYGCIjI5Gamqq2LCkpCZUrVy5SDLq+t2VmZiI5OVltmFlBXnrpJbXnyvfGp0+fFik+QHq/y3tNlpIuOUlXSUlJaoWZjY2N2lC53Az52SGvunXrarS9/PLLSE1NxePHj+Hm5ga5XI7w8HCNmWGtrKwwaNAgrFixAjExMfDw8FAVVsUd4peZmYknT56oteXOx3Xr1lX7DKScJXfp0qV499130bhx42Idn4qGxRSVGEdHR7i7u+PKlStGO4YQAlu3bkVKSgoaNGigsTw+Ph4vXrxAxYoVVW2WlpZqb0r+/v7w9fXFe++9h3379ukVR40aNRATE6PR/ujRIwA503MrL4JWtuddV9s03nmPk9v69esxatSoArfJXYj26tULTk5OCA4Oxmuvvaa6jkmhUKBx48ZYvHix1n0oryGyt7fHiRMncOzYMRw4cACHDh3C9u3b0aVLFxw5cgSWlpYa2yrPTo0aNQp79+7VWK4867Ro0aJ8pxXO/fcrCmXcuZOUrn8rIiq9ipp/oqKi0LVrV/j6+mLx4sXw9PSEjY0NDh48iCVLluh0djwvXd/b8n6ILoy291kAOk30k1eNGjWQnZ2N+Ph4uLi4qNozMzORmJhosPfD999/H//73/9Uzzt16oTjx49rXdfX1xcAcPnyZYMcu6hOnTqF5ORk1VTkuQ0fPhzLli3D1q1bMW3aNGzduhUNGjQo9pT4p0+fxmuvvabWdufOnQK36dq1K5YtW4YTJ06wmCphLKaoRL3xxhv44YcfEBkZiTZt2hh8/7///jsePHiAefPmqd2vAZC+pRs3bhz27NlT4AW0NWrUwNSpUzF37lycOXNGdQajKJo1a4Zjx44hOTlZbWKDs2fPqpYDQKNGjWBlZYXz58/jnXfeUa2XmZmJS5cuqbVpk/fGsg0bNixyrO+99x6WLFmCjz/+GG+99RZkMhlq166Nv//+G127di10qKOFhQW6du2Krl27YvHixfjss8/w0Ucf4dixY/mePRw+fDgWLFiAuXPn4s0331RbppwIw9HRMd/tlYo6DFM5O2Pub/h0/VsRUelWlPyzf/9+ZGRkYN++fWpnfpRDnAuS3/uSru9tzs7OcHR0NOoXj/lRvt+dP39erXg4f/48FAqFwd4Pp0+frpaHCxpp8PLLL6NevXrYu3cvvvnmG72/TNNGebYwtxs3bsDBwUGVJw4cOIAGDRrA29tbY91WrVqhdu3a2LJlC7p3745///0Xn376abHjatq0qUZ+d3Nz0xiNkltWVhYAFDh7LRkHr5miEjV9+nRUqFABY8eORVxcnMbyqKgofPPNN3rvXznE78MPP8SAAQPUHoGBgahbt67GjXi1mTRpEhwcHPD555/rFceAAQOQnZ2tdvPHjIwMrF+/Hq1atVKdIalcuTK6deuGTZs2qQ1h2LhxI168eIGBAwcWeJxu3bqpPbRN91sYKysrfPDBB7h69arqTNE777yDmJgYrF69WmP9tLQ0pKSkAIDWb1CVyVbb0Dkl5dmpS5cuaZz98/PzQ+3atfHVV19pTQq5p/ytUKECAODZs2dq6yQnJ2scXwihunFz7psT6/q3IqLSrSj5R3m2J/fZnaSkJKxfv77Q4+T3vqTre5uFhQX69euH/fv34/z58xrr6XPGSVddunRBtWrVNKaAX7lyJRwcHDSGyeurQYMGarkr93VL2sydOxeJiYkYO3asqmjI7ciRI/jll1+KHEdkZCQuXLigen7//n3s3bsXPXr0UL0GDh48WODvPWzYMFy8eBFhYWGQyWT53ji4KKpWraqR35XX8eZHeblC06ZNi318KhqemaISpfwGZ9CgQahfv77aHehPnz6NnTt3FjpMLT8ZGRnYvXs3unfvnu+bzptvvolvvvlGYwhDXtWrV8fo0aOxYsUKXL16VXWW68SJE6p7UDx+/BgpKSmqD+gdO3ZEx44dAUjfVg0cOBCzZs1CfHw86tSpg//973+Ijo7G2rVr1Y716aefom3btujUqRPGjRuHBw8e4Ouvv0aPHj3Qs2dPvfqiqEaNGoXQ0FB88cUX6NevH0aMGIEdO3Zg/PjxOHbsGNq1a4fs7Gxcu3YNO3bswOHDh9G8eXPMmzcPJ06cQO/eveHl5YX4+HisWLECNWvWRPv27Qs8pvLaqUuXLqm1W1hYYM2aNXj99dfRsGFDjB49Gh4eHoiJicGxY8fg6OioShrKBPzRRx9h8ODBsLa2Rp8+fXDhwgUMGTIEQ4YMQZ06dZCWloaff/4Zf/zxB8aNG4dXX31Vdbyi/K2IqPQqSv7p0aMHbGxs0KdPH7z33nt48eIFVq9eDRcXF63DsvMep0qVKli1ahUqVaqEChUqoFWrVqhVq5bO722fffYZjhw5osoL9evXx6NHj7Bz506cOnVKNfmPru7evYuNGzcCgKpAU+YuLy8vjBgxAoA0dHv+/PkICgrCwIED4e/vj5MnT2LTpk349NNP1a5rSkpKwnfffQcAqtthLFu2DFWqVEGVKlUQHBxcpBgLMmjQIFy+fBmffvopLl68iCFDhsDLywuJiYk4dOgQIiIitN7zqTCNGjWCv7+/2tTogFS8AdLQuqtXrxZ4f7Hhw4dj3rx52Lt3L9q1a6f1DJahXbhwAZs2bQIgXUsWERGB3bt3o23btujRo4fRj095mHQuQSq3bty4IQIDA4W3t7ewsbERlSpVEu3atRPfffed2h3D85saPSgoSGOfu3fvFgDE2rVr8z3u8ePHBQDxzTffCCFypkbXJioqSlhaWoqRI0eq2pTTq2p7hIWFqW2flpYmpk2bJtzc3IStra1o0aKFOHTokNZjnTx5UrRt21bY2dkJZ2dnERQUpLqzvCHl13dCCDFnzhy16VkzMzPFF198IRo2bChsbW1F1apVhZ+fn5g7d65ISkoSQggREREh+vbtK9zd3YWNjY1wd3cXQ4YMETdu3FDtN787tguRM7UvtEyvfvHiRdG/f39RvXp1YWtrK7y8vMQ777wjIiIi1NabP3++8PDwEBYWFqqpeW/fvi0GDhwovL29hZ2dnXBwcBB+fn5i1apVWqeOLcrfiohKN13zz759+0STJk2EnZ2d8Pb2Fl988YVYt26dxhTg2qa93rt3r2jQoIGwsrLSmKZb1/e2u3fvioCAAOHs7CxsbW2Fj4+PCAoKEhkZGUKInPfPvNOnHzt2TGOqbWWbtoe2Kbt/+OEHUa9ePWFjYyNq164tlixZovHeWdA03V5eXoX/IfSgzDkuLi7CyspKODs7iz59+oi9e/dqxKXL1OhBQUFi06ZNom7dusLW1la88sorav22bNkyUblyZY3bquTVokULAUCsWLFC63JjTo1uZWUlfHx8xIcffiieP39e6D7I8GRCGPF8MRERERGRmZHJZAgKCsKyZcvyXadXr16oWLEiduzYUYKRUWnDYX5ERERERHl07twZHTp0MHUYZOZYTBERERER5TF9+nRTh0ClAGfzIyIiIiIi0gOLKSIiokKcOHECffr0gbu7O2QyGfbs2VPoNsePH8err74KW1tb1KlTBxs2bDB6nESkGyFEgddLEemKxRQREVEhUlJS0LRpUyxfvlyn9e/cuYPevXvjtddew6VLlzBlyhSMHTsWhw8fNnKkRERUkjibHxERURHIZDL8/PPP6NevX77rzJgxAwcOHMCVK1dUbYMHD8azZ89w6NChEoiSiIhKAiegAKBQKPDw4UNUqlQJMpnM1OEQEZUrQgg8f/4c7u7usLAoGwMmIiMj0a1bN7U2f39/TJkyJd9tMjIykJGRoXquUCjw5MkTVK9enbmJiKgEFSUvsZgC8PDhQ3h6epo6DCKicu3+/fuoWbOmqcMwiNjYWLi6uqq1ubq6Ijk5GWlpabC3t9fYZuHChZg7d25JhUhERIXQJS+xmAJQqVIlAFKHOTo6Fnl7uVyOI0eOoEePHrC2tjZ0eKUO+8Ow2J+Gxf40vOL2aXJyMjw9PVXvxeXVrFmzEBISonqelJSEl156CXfu3NGrb+RyOY4dO4bXXnut3L/W2ReGxz41LPanYRW3P58/f45atWrp9N7LYgpQDZ9wdHTUu5hycHCAo6Mj/wOA/WFo7E/DYn8anqH6tCwNZXNzc0NcXJxaW1xcHBwdHbWelQIAW1tb2NraarRXq1atWLmpevXq5f61zr4wPPapYbE/Dau4/ancRpe8VDYGpxMREZmRNm3aICIiQq0tPDwcbdq0MVFERERkDCymiIiICvHixQtcunQJly5dAiBNfX7p0iXcu3cPgDRELyAgQLX++PHjcfv2bUyfPh3Xrl3DihUrsGPHDkydOtUU4RMRkZGwmCIiIirE+fPn8corr+CVV14BAISEhOCVV15BaGgoAODRo0eqwgoAatWqhQMHDiA8PBxNmzbF119/jTVr1sDf398k8RMRkXHwmikiIqJCdO7cGQXdlnHDhg1at7l48aIRoyIiIlPjmSkiIiIiIiI9sJgiIiIiIiLSA4spIiIiIiIiPbCYIiIiIiIi0gOLKSIiotIuLc246xMRkVYspoiIiEqz1auBJk2A+/d1W//+fWn91auNGxcRUTnAYoqIiKi0SksDvvwSuHUL6Ny58ILq/n1pvVu3pO14hoqIqFhYTBEREZVW9vbA0aOAjw9w+3bBBZWykLp9W1r/6FFpeyIi0huLKSIiotLM0xM4frzggipvIXX8uLQdEREVC4spIiKi0k5bQRUTIy2LiWEhRURkJCymiIiIyoK8BVWvXlJ7r14spIiIjMQsi6nly5fD29sbdnZ2aNWqFf78888C13/27BmCgoJQo0YN2Nra4uWXX8bBgwdLKFoiIiIzkbugio6W2qKjWUgRERmJ2RVT27dvR0hICMLCwnDhwgU0bdoU/v7+iI+P17p+ZmYmunfvjujoaOzatQvXr1/H6tWr4eHhUcKRExERmQFPT2DjRvW2jRtZSBERGYGVqQPIa/HixQgMDMTo0aMBAKtWrcKBAwewbt06zJw5U2P9devW4cmTJzh9+jSsra0BAN7e3iUZMhERkfm4fx8YMUK9bcQInpkiIjICsyqmMjMz8ddff2HWrFmqNgsLC3Tr1g2RkZFat9m3bx/atGmDoKAg7N27F87Ozhg6dChmzJgBS0tLrdtkZGQgIyND9Tw5ORkAIJfLIZfLixy3cht9ti2L2B+Gxf40LPan4RW3T/m3MKDcs/bVry+1eXsDV69K7SyoiIgMyqyKqYSEBGRnZ8PV1VWt3dXVFdeuXdO6ze3bt3H06FEMGzYMBw8exK1btzBx4kTI5XKEhYVp3WbhwoWYO3euRvuRI0fg4OCgd/zh4eF6b1sWsT8Mi/1pWOxPw9O3T1NTUw0cSTmVd/rzgweBv/+W/u3aNWeWPxZUREQGY1bFlD4UCgVcXFzwww8/wNLSEn5+foiJicGiRYvyLaZmzZqFkJAQ1fPk5GR4enqiR48ecHR0LHIMcrkc4eHh6N69u2qoYXnG/jAs9qdhsT8Nr7h9qhwdQMWg7T5Sbm5SMeXhIT1XLmdBRURkMGZVTDk5OcHS0hJxcXFq7XFxcXBzc9O6TY0aNWBtba02pK9+/fqIjY1FZmYmbGxsNLaxtbWFra2tRru1tXXhHwTS0vK9Y7zW7QtYv6zTqT9JZ+xPw2J/Gp6+fcq/QzHld0Pe3MMnlbP8saAiIjIosyqmbGxs4Ofnh4iICPTr1w+AdOYpIiICwcHBWrdp164dtmzZAoVCAQsLaXLCGzduoEaNGloLqWJZvRr48kvg6FHdEtD9+0CXLsD06UBgoGFjISIiSkuT8owu95HKW1B16QL880+5/cKPiAxPoZC+xynuIyureHFkZcnw9981UbmyDJ07G+RXy5dZFVMAEBISgpEjR6J58+Zo2bIlli5dipSUFNXsfgEBAfDw8MDChQsBABMmTMCyZcvw/vvvY9KkSbh58yY+++wzTJ482bCBpaVJhdStW7p9o5f7m8IvvwSGD2fCIiIiw7K3l76w0/WLPmVBpfyij3mJqNRKSwNu3JDml7l2Dbh+HSjuJahCSIVMYcVOZqb2doXCML9b8VkB8ENMjKL8FVODBg3C48ePERoaitjYWDRr1gyHDh1STUpx79491RkoAPD09MThw4cxdepUNGnSBB4eHnj//fcxY8YMwwZmby8lKl2GSOQdcnH0KBMWEREZR2Bg0b6w8/TkGSmiUiQhQSqWlEWT8t/oaKn4IdMyu2IKAIKDg/Md1nf8+HGNtjZt2uDMmTNGjgr5jznPfT1XfmPXiYiIjKWohRELKSKzkp0NxMU54NAhGW7eVC+aEhJMG5tMBlhbF/6wsdFtPV0eVlbScfWVnZ2Nf//9F337NgBgUej6xWGWxZRZ01ZQRURIy2JicqafZSFFREREVO4IAaSkSEWQ8vH4sfaflc+fPLGCQtFd52NUqgT4+kq3k/P1zXlUr178+K2s1AubfG7batbkcgUOHryDHj3qG/1YLKb0kbeg6tULWLhQ+peFFBEREVGJSE/P//qd4jyKus+UFPUiKT29qL+J9tMw7u7qRZPyX3f34p25IcNhMaWv3AVVdLTUFh3NQoqIiIhID3I5kJiY/xkcbW1paaaOungqVgScnIDq1RWwtIxDp04uaNjQEvXrA/XqAZUrmzpCKgyLqeLw9AQ2bgS6dctp27iRhRQRERGZTEFDzJQPIQAHB6BCBenf/B75LbeyAl68sEZMjFQEpaZqPlJStLcrl6Wk5BRPjx8DSUmm7rnisbaWCiPlw9m54OfVq+dcviiXZ+PgwT/Rq1cvWFuXwnF15RiLqeK4fx8YMUK9bcQInpkiIiIigxICuHcPuHxZ+vhRUKFUMmdrrAH0KokDabC0zClIqlUDbG2NMylCUbZ1cAAcHTn0rjxiMaWv3LP21f//i9u8vaWpV3hneSKiMmf58uVYtGgRYmNj0bRpU3z33Xdo2bJlvusvXboUK1euxL179+Dk5IQBAwZg4cKFsLOzK8GoqTRKSgKuXJFmsP/nH6mAunwZSE42dWTGUbVqwWdw8rZXrsyihcyHXsXU2bNn0apVK0PHUnrknf784EHg77+lf5Wz+bGgIqJyIjUVePrU1tRhGDU3bd++HSEhIVi1ahVatWqFpUuXwt/fH9evX4eLi4vG+lu2bMHMmTOxbt06tG3bFjdu3MCoUaMgk8mwePFio8RIpU9WlnTTVWXBpPz37t2i78vKKv9CRFubhYXuw/K0taekKPDs2WN4eTmjYkWLIg0RzN1etaoUO1FppdfLt02bNmjcuDECAwMxfPhwVKlSxcBhmTFt95Fyc5OKKQ8P7fehYkFFRKWIQgE8eQLEx0uPuLiCf37xwho1arTHsGGmjduYuWnx4sUIDAzE6NGjAQCrVq3CgQMHsG7dOsycOVNj/dOnT6Ndu3YYOnQoAMDb2xtDhgzB2bNnDRYTlR5CALGx6gXTP/8A//0nzRqni5deApo0ARo3BurWzSmQlP+W9BAz6RqfM/9/jY9x7+NDZM70KqaGDx+O3bt3Y/LkyZg+fToGDBiAwMBAdOjQwdDxmZf8bsgrl+esk9+NfVlQEZEJpafnFEGFFUiPH0s3kCyKZ89Mf2bKWLkpMzMTf/31F2bNmqVqs7CwQLdu3RAZGal1m7Zt22LTpk34888/0bJlS9y+fRsHDx7EiLzX2eaSkZGBjIwM1fPk/x/TJZfLIc+dZ3Sk3EafbcuakuyL1FTgv/9kuHwZuHJFhsuXZbhyRYaEBN0qnUqVBBo1EmjcWKBxY6BRI4GGDQUK+24gK6v4sRcFX1+Gxf40rOL2Z1G206uY+vHHH/Hdd99h06ZNWLt2LTZt2oTNmzejbt26CAwMxMiRI+Hk5KTPrs1XWhrQpYtu95HKW1B16SJ9BcU7zhORgQgBPH2q25mj+HjjXGtRrRrg4gK4uCiQlRWP7GwXWFsb/ji6MlZuSkhIQHZ2NlxdXdXaXV1dce3aNa3bDB06FAkJCWjfvj2EEMjKysL48eMxe/bsfI+zcOFCzJ07V6P9yJEjcHBwKHLcSuHh4XpvW9YYsi8UCiAurgKiox1x964joqMdce+eIx49qgAhCi+cLCwUcHdPgbd3Ery8nsPLKwleXslwcUlTO8OUlAScPm2wsA2Ory/DYn8alr79mZqaqvO6eo9SrVy5MoKCghAUFIQLFy5g9erV2LZtGz788EN89NFH6Nu3LwIDA9Et97ThpZm9PTB9OvDll8DRo4WfaVIWVF26SNuxkCKiQggBPH8uFUGxsdKjoJ8N/QWmjY2yOAJcXdX/zfuzszNUhZM03Oc8LC1NM7NXbuaSm44fP47PPvsMK1asQKtWrXDr1i28//77mD9/Pj755BOt28yaNQshISGq58nJyfD09ESPHj3g6OhY5BjkcjnCw8PRvXt3WJuyyjUDxe2LhATpLFPOmSbg339lSE3V7WyTm1vO2aaGDaV/69fH/09GYgfAtbBdmB2+vgyL/WlYxe3P5CJ8A2mQS/5effVVrFy5EosXL8bOnTsxe/Zs7Nq1C7t27YKXlxfGjx+PCRMmoFKlSoY4nOkEBgLDh+teGHl68owUEUGhkIbOPXyo/lAWRrkLJUNPaVylSsFFUe7nZW2GLEPlJicnJ1haWiIuLk6tPS4uDm5ublq3+eSTTzBixAiMHTsWANC4cWOkpKRg3Lhx+Oijj2BhoXmNia2tLWxtNYdLWltbF+vDVXG3L0sK64uMDGlS3rzXNj16pNv+7e2Bhg1zrm1S/uvsLANQhv5z5cLXl2GxPw1L3/4syjYGmz/l6dOn+PHHH7FmzRo8fPgQMpkM7dq1w9WrVzFz5kwsXboUe/fuRYsWLQx1SNMoamHEQoqozFIOtYuJ0SyU8hZNhrqewcJCOivk5iYVQW5u2gsj5cPGxjDHLa0MkZtsbGzg5+eHiIgI9OvXDwCgUCgQERGB4OBgrdukpqZqFEyWltKNOIUQhvnlSG+579mUu2i6fl236wVlMmnEf96iqXZt6R5IRFR+FLuYOnbsGFavXo09e/YgPT0dzs7O+PDDD/Hee+/Bx8cHGRkZWLduHaZPn45JkybhzJkzhoibiMjo0tOlD1x370qP6Oicnx88kAqlXPMFFEv16jnFkfKR+7nyZycnfljThaFzU0hICEaOHInmzZujZcuWWLp0KVJSUlSz+wUEBMDDwwMLFy4EAPTp0weLFy/GK6+8ohrm98knn6BPnz6qoopKzv37wKFD3vj1VwtcuVK0ezZVq5ZTLCkLp4YNgYoVjRszEZUOehVTcXFxWL9+PdauXYvbt29DCIFOnTph/Pjx6N+/v9qpMVtbW0yYMAG3bt3C8uXLDRY4EVFxvXihvVBS/hwbW7z9y2TS2SF3d+0PZaHEM0iGYczcNGjQIDx+/BihoaGIjY1Fs2bNcOjQIdWkFPfu3VM7E/Xxxx9DJpPh448/RkxMDJydndGnTx98+umnhv/FSSuFAoiIAFasAPbts4JC0bTA9a2tgfr11c82NWkC1KhRtoa/EpFh6VVM1axZEwqFAlWrVsWUKVMwbtw41KtXr8BtnJ2dkanrzRSIiPSUnQ0kJmqf1U66LskS1651wpgxVkhM1P84VatKt5bLXRzlfe7qCpPOblfeGDs3BQcH5zus7/jx42rPraysEBYWhrCwMJ32TYbz9CmwYQOwciVw86ayVb0a8vTUHKJXrx7/vxJR0elVTLVq1Qrjx4/HwIEDtV4sq83MmTO13tiQiEgXycnAnTvSsLu8BVLuoikhQboeIn8WAKoUerwaNQAvL+nh7a3+80svcYiPOWJuKt8uXACWLwe2btWcyMXdXaBDh+t47706aNbMClWrmiZGIip79CqmTp06Zeg4iKicS0+XhtbduaP98eSJ4Y5lYaFAzZoyeHvLtBZMnp6AnZ3hjkclg7mp/ElPB3bskIbynT2rubxLF2DiROD117MQHn4d7dvX5tknIjIovYqpBw8e4MKFC+jYsSOqaLkl99OnT3Hy5En4+fnBw8OjuDESURmQnS1N2pBfsfTwYfH2b2enffrvvG1Vq8rx55+/ok+f1zn9bBnD3FR+3L4NrFoFrFsHjeG6jo7AyJHAhAnSNVCA4e/JRkSkpFcxtWDBAuzcuRMP8/n04+DggHfffReDBw/GsmXLihUgEZUOQkhD7fIrlu7d0296cAsLoGZNoFYt6eHtnTO7Xe5iqWJF3S4Sl8sBS0tOTV0WMTeVbdnZwKFD0lmoX3/VHM7bpAkQFAQMHcphuERUcvQqpo4ePYoePXrkOybd1tYWPXr0wG+//Vas4IjIvDx7llMcRUerF0vR0UBqqn77dXHJKZbyPl56iReFk26Ym8qm5GTpLNSqVdJ7TW7W1sDAgdJQvrZtOeseEZU8vYqpmJgYvP322wWu4+Xlhf379+sVFBGZRmamdAbp9m3pQ8vt2+o/P32q334dHfMvlry9gQoVDPprUDnF3FT2HDgAvPeedGPs3F56CRg/HhgzRvoyhojIVPQqpmxsbJBcyN3ukpOTIeNXRERmRQjg8WPNIkn58/370r1ZisrWViqK8iuYqlblN8ZkfMxNZUdCAjBlCrB5s3q7v790Fqp3b968mojMg17FVOPGjbF//34sXrxY63CK9PR07Nu3D40bNy52gERUdC9eANeuAVevAv/9J/0bFSUVTCkpRd+f8rolHx/txZKbm7QOkSkxN5V+QgA7dwLBwdIXP0o9egDffQe8/LLpYiMi0kavYmr06NEYM2YM3nzzTaxcuRI+Pj6qZVFRUZg4cSIePnyIefPmGSxQItKUmKheMCl/vn+/6PuqVk0qjHx8coom5b8vvQTY2Bg+fiJDYm4q3R4+lCaQ2LMnp61KFWDJEml2Pp5QJCJzpHcxdfDgQezevRu+vr6oVasWPDw8EBMTgzt37iArKwuDBg3C6NGjDR0vUbkjBJCYaIeICBlu3FAvmnJ/c1sYGxtpKJ62YqlWLelDC1FpxtxUOgkBrF8PhIQASUk57W+9Jd2Et0YN08VGRFQYvYopANixYweWL1+OFStW4Nq1a7h58yYAoEGDBggKCsKECRMMFiRReZCZCdy6JQ3PU39Y4flzf533U6WKdG+VBg2kf5U/v/QSh+JR2cfcVLpERwPjxgHh4TltLi7AsmXAgAE8G0VE5k/vYkomkyE4OBjBwcFISUlBUlISKleujAqclouoQE+faiuYpGuasrO1baH904Sra07BlLtwcnPjBxAqv5ibSgeFQjrrNGuW+nWcw4cDS5cC1aubLDQioiLRu5jKrUKFCkxURHkkJAAXLwJXrqgXTfHxuu9DJgO8vASqVYtHx45OaNjQUlU4Va1qvNiJygLmJvN0/bo0pfkff+S01awJfP890KuX6eIiItKHQYopovLu0SPgwgX1x717um9vbw/4+uY86tWT/q1bF7C2zsLBg2fQq1cvWFtzLmAiKp2ysoCvvgLmzAEyMnLax48HvvhCuh8dEVFpo3cxdf/+fSxYsAC//fYbHj58iMzMTI11ZDIZsrKyihUgkTkRQiqS8hZOsbG6bV+jhnrRpHzUrJn/9UxyueHiJyrrmJvM099/A+++K71fKtWuDaxZA3TubLKwiIiKTa9i6vbt22jVqhWePn2Khg0bIiMjA15eXrCzs8Pt27chl8vRtGlTVOH0YFSKKRTSdUx5C6cnTwrftlIl4JVXgFdfBZo2la5pqlcPqFzZ+HETlVfMTeZHLgfmzQM+/1w6MwVIXxxNnSq1OziYNj4iouLSq5iaO3cukpKSEBERgU6dOsHCwgKjR49GaGgoHj16hAkTJuC///7Db7/9Zuh4iQxOoZDONv37r/rj6lUgNbXw7atWBfz8pMJJ+ahdmzPnEZU05ibzIoR0NmrTppy2hg2BdeuAli1NFxcRkSHp9XHvt99+Q69evdCpUydVmxACAFCjRg1s374dADB79my9A1u+fDm8vb1hZ2eHVq1a4c8//9Rpu23btkEmk6Ffv356H5vKJiGAu3eBgwelcfujR0sJ3dFRus/SG28AM2YAP/4I/PWX9kLK1VW6QPrjj4GffpKm9U1MlKb1/eILYNAg6TonFlJEJa8kchPp7vvvcwopKysgLEw6u89CiojKEr3OTCUkJMDX1zdnJ1ZWSM31ydPW1hbdu3fHnty3MS+C7du3IyQkBKtWrUKrVq2wdOlS+Pv74/r163Bxccl3u+joaEybNg0dOnTQ67hUdsTHSzPpKc8yXbki3eT2xQvdtpfJpLNLDRuqn3Fydzdu3ESkP2PnJtLd+fPA++/nPN+6VbpvFBFRWaNXMeXk5ISUXDeGcHJyQnR0tPqOrazw7NkzvYJavHgxAgMDVXepX7VqFQ4cOIB169Zh5syZWrfJzs7GsGHDMHfuXJw8eVLvY1Ppk5QknUk6dy7noetMejKZdFaqYUP1h6+vNMMeEZUexs5NpJsnT6TCSTn3x5QpLKSIqOzSq5iqW7cuoqKiVM9btmyJw4cP4/bt2/Dx8cHjx4+xa9cu1K5du8j7zszMxF9//YVZs2ap2iwsLNCtWzdERkbmu928efPg4uKCMWPG4OTJkwUeIyMjAxm55mVNTk4GAMjlcsj1mDpNuY0+25ZFxuyP9HTg779lOHdOhvPnpceNG7rdodbbW6BBA4H69aV/GzYU8PXN/wJoc/lz8vVlWOxPwytunxrqb2HM3ES6USiAgABpSDUAtGkjDYEmIiqr9CqmXn/9dcyZMwfPnj1DlSpVMGXKFOzfvx9NmjRB/fr1cevWLSQnJ2POnDlF3ndCQgKys7Ph6uqq1u7q6opr165p3ebUqVNYu3YtLl26pNMxFi5ciLlz52q0HzlyBA7FmFooPDxc723LouL2R3a2DPfuVcKtW1Vw82ZV3LpVBXfvOiI7u+ALkuzsslC79jPUrv0MXl7JeOml56hZ8zns7bPV1ouN1X1Kc3PA15dhsT8NT98+TdVlphcdGDM3kW4+/xw4cED62ckJ2LEDsLExbUxERMakVzE1YcIEdO7cGZaW0g1EO3fujG3btmHOnDm4cuUKvLy8sGDBAgQGBho0WG2eP3+OESNGYPXq1XByctJpm1mzZiEkJET1PDk5GZ6enujRowcc9bhroFwuR3h4OLp37w5ra+sib1/W6Nsf2dnAX3/J8OuvMhw7JsPFizKkpRV81snaWqBpU4HmzaWHn590tsnSsjKAsjEPOV9fhsX+NLzi9qlydEBxmVNuKo+OHgU++UT6WSYDtmyR7qFHRFSW6VVMOTo6olWrVmptAwcOxMCBA4sdkJOTEywtLREXF6fWHhcXBzc3N431o6KiEB0djT59+qjaFAoFAGls/PXr1zWGdNja2sLW1lZjX9bW1sX6cFXc7csaXfojMRE4fFiaYe/wYSAhIf91LSyA+vWBFi1yHk2ayGBrq9swv9KOry/DYn8anr59aqi/gzFzExXs4UNgyBBpmB8AzJkDdO9u0pCIiEqEXsVUly5d0K5dO8yfP9/Q8cDGxgZ+fn6IiIhQTW+uUCgQERGB4OBgjfV9fX1x+fJltbaPP/4Yz58/xzfffANPT0+Dx0j6UyikqXF//VUqoM6elaYs18bHR71wevVVoGLFko2XiEoPY+Ymyp9cLt0WIj5eeu7vL90+goioPNCrmDp79ixat25t6FhUQkJCMHLkSDRv3hwtW7bE0qVLkZKSoprdLyAgAB4eHli4cCHs7OzQqFEjte2Vd7fP206m8fQpcOSIVED9+mtOws2rUiXpm8xevYCePQEPj5KNk4hKN2PnJtJu9mzg1CnpZ09P6d5SvNceEZUXehVTvr6+uKucqscIBg0ahMePHyM0NBSxsbFo1qwZDh06pJqU4t69e7DgO7XZEgK4fdsRn39ugSNHgNOnc4Z+5NWwoVQ89eoFtG3LC5WJSH/Gzk2k6eefpZugA4C1tTThhI6XLxMRlQl6FVOTJk1CcHAw/vvvPzRo0MDQMQEAgoODtQ7rA4Djx48XuO2GDRsMHxAVKD1duvh4717gl1+s8PDha1rXc3AAunWTiqfXXwdeeqmEAyWiMqskchPluHULGDUq5/nXXwM8MUhE5Y1exZSPjw86d+6M1q1b47333kOLFi3g6uoKmUxzIoCOHTsWO0gyT4mJ0hS4e/dKk0fk3CtT/XXg6ysVTr16AR06AFrm/iAiKjZj56bly5dj0aJFiI2NRdOmTfHdd9+hZcuW+a7/7NkzfPTRR/jpp5/w5MkTeHl5YenSpejVq1eRj21u0tKkG/EqJ2J85x0gn+8/iYjKNL2Kqc6dO0Mmk0EIga+//lprolLKzs7OdxmVPlFRUvG0d680Rl7b8D07O4GGDeMQEOCMN96whI9PycdJROWPMXPT9u3bERISglWrVqFVq1ZYunQp/P39cf36dbi4uGisn5mZie7du8PFxQW7du2Ch4cH7t69q7qmt7SbNAn4+2/p53r1gDVrpOnQiYjKG72KqdDQ0AKTFJUdCgVw7lxOAfXff9rXc3IC3ngD6NsX6Nw5C7//fha9evWCtbVlyQZMROWWMXPT4sWLERgYqJoIadWqVThw4ADWrVuHmTNnaqy/bt06PHnyBKdPn1ZN/e7t7W2U2Era+vXA2rXSzw4OwO7d0gRCRETlkV7FFO8eX7alpwMREVLxtH8/EBurfb26daXiqW9foE0b4P/vkwm5vORiJSJSMlZuyszMxF9//YVZs2ap2iwsLNCtWzdERkZq3Wbfvn1o06YNgoKCsHfvXjg7O2Po0KGYMWOG6qbCeWVkZCAjI0P1XHkzY7lcDrkeb6zKbfTZNj9//w1MnGgF5XDu5cuz8PLLwuzf943RF+Ud+9Sw2J+GVdz+LMp2ehVTVPZkZ0vTl69bJ01fnnP9Uw6ZTLq4WFlA+fqWfJxERCUtISEB2dnZqhlllVxdXXHt2jWt29y+fRtHjx7FsGHDcPDgQdy6dQsTJ06EXC5HWFiY1m0WLlyIuXPnarQfOXIEDg4OescfHh6u97a5paRYYdq0TkhPl8609ex5B1Wr/oODBw2y+xJhqL6gHOxTw2J/Gpa+/ZmamqrzuiymyrnYWKmA+uEHQNuMwnZ20r2f+vaVhvHl+SxBRERaKBQKuLi44IcffoClpSX8/PwQExODRYsW5VtMzZo1CyEhIarnycnJ8PT0RI8ePeDo6FjkGORyOcLDw9G9e3fVUEN9CQEMGmSJR4+k25K8+qoCO3bUhJ1dzWLtt6QYsi9Iwj41LPanYRW3P5UjA3ShVzFlYWGh07h0mUyGrKwsfQ5BRqRQSNOYf/89sGcPkPdP5OQE9OkjFVDdu0tj4omIzJ2xcpOTkxMsLS0RFxen1h4XFwc3Nzet29SoUQPW1tZqQ/rq16+P2NhYZGZmwkbLTfVsbW1hq2W6U2tr62J9uCru9gCweLGULwCgShVg1y4LVKpU+u73aIi+IHXsU8NifxqWvv1ZlG30KqY6duyoNWElJSXh5s2bSElJQdOmTcvMrEVlxePHwIYN0lmoW7fUl8lkgL8/MH480Ls3YMVzllQIuVxeKmfrlMvlsLKyQnp6eqmM3xzl7VNLS0uTfBgwVm6ysbGBn58fIiIi0K9fPwDSmaeIiIh874fYrl07bNmyBQqFQnWT+Rs3bqBGjRpaCylzduoUMH16zvONG4FatUwXD1FBmJsI0N6fxspNen1kLuimuampqZg5cyYOHTrEcZ9mQAjgxAnpLNTu3UBmpvpyV1dgzBhg7FgmR9JNcnIyEhIS1C6UL02EEHBzc8P9+/c5K6mBaOtTW1tbODk56TU8TV/GzE0hISEYOXIkmjdvjpYtW2Lp0qVISUlRze4XEBAADw8PLFy4EAAwYcIELFu2DO+//z4mTZqEmzdv4rPPPsPkyZP1+t1MJT4eGDRIuq4WAGbNkoZ8E5kb5ibKLb/+NEZuMvj5BwcHB3z77bdo0aIFPvzwQ6xfv97QhyAdPHkC/PijVERpuz66a1fpLNSbbwKl7EtSMqHk5GTExMSgYsWKcHJygrW1dal701coFHjx4gUqVqyoOmNAxZO7T2UyGeRyOZKSkhATEwMAJVpQ5ae4uWnQoEF4/PgxQkNDERsbi2bNmuHQoUOqSSnu3bun9nry9PTE4cOHMXXqVDRp0gQeHh54//33MWPGDIP+XsaUnQ0MHQo8fCg979wZmDfPpCERacXcRHnl7U8hhNFyk9EGc3Xo0AGbNm0y1u5JCyGAyEhg1Spg505pivPcnJyA0aOBwEBpWnOiokpISEDFihVRs2bNUpeolBQKBTIzM2FnZ8eEZSB5+9Te3h6VKlXCgwcPkJCQYBbFlFJxclNwcHC+w/q0nRVr06YNzpw5o9exzMHKldJtMgDAzQ3YupVDwMk8MTdRXtr601i5yWhvi48fP8aLFy+MtXvKRaEAdu0CPvss5470uXXqBLz3HtC/P6Dl2mYincjlcmRkZMDJyanUJisqOTKZDJUrV0ZMTAzkcrnZXFDN3KQbuRxYtCjn+bZtUkFFZG6Ym6gojJGbDF5MKRQKbN68Gdu3b0fz5s0NvXvKJTsb2LEDWLAA+O8/9WVVqwIjRwLjxgH165smPipblBdwmsuHYjJ/ytdKdna2yV83zE1Fs2sXcO+e9HOvXtKXckTmiLmJisrQuUmvYsrHx0dre1ZWFuLj41WVnvJCXDKsrCxg+3apiMp7PVTLlkBQEDBwIGBvb5r4qGzjN3+kq5J+rTA3GYYQ6melPvzQdLEQ6Yq5iXRl6NeKXsWUQqHQGoi1tTUaNWqEFi1aIDg4GA0bNix2gJQjKwvYskUqom7eVF/Wrh0QFgZ06yZNc05EVN4wNxnG0aPAxYvSz82b86wUEVFB9CqmoqOjDRwGFUQuBzZtAj79FIiKUl/WoYNURHXpwiKKiMo35ibDyH1Wato05hYiooJwuhAzJpcDa9cC9eoB776rXkh17gwcOybdQ6prVyY7orJKJpOhc+fOpg6Dyol//gEOH5Z+9vYG3n7bpOEQkZlibsqhVzH14MED7Nu3D8+ePdO6/OnTp9i3b59qHncqmsxM4IcfpOnLx44F7tzJWda1K/D771IhxdcwUcmKjo6GTCYr8JHf+6Kx3L17F5aWlpDJZFiU+5RCOcTcVHxff53zc0gIp0InKg2Ym0xLr7fJBQsWYOfOnXiovJNfHg4ODnj33XcxePBgLFu2rFgBlicZGcC6dcDChcD9++rLuneXhvO1a2ea2IgoR+3atTF8+HCty+zs7Eo0lnXr1qmuFVq3bh0+LMezBTA3Fc+DB9J1uYA0I+y775o2HiIqGuYm09CrmDp69Ch69OgB23xuWmRra4sePXrgt99+K1Zw5UV2NrBmjTSxxIMH6st69gRCQ4E2bUwTGxFpqlOnDubMmWPqMKBQKLBhwwY4OTnhjTfewIYNG3D69Gm0bdvW1KGZBHNT8Xz7rTTREQBMnAhUqGDaeIioaJibTEOvYX4xMTHw9vYucB0vLy8OpdDBhQtA69bA+PHqhVTv3sDZs8Cvv7KQIiqN/vnnHwwePBg1atSAjY0NvLy8MGnSJCQmJmpdf82aNWjUqBHs7Ozg6emJ6dOnIz09vcBjhIeH4969exg8eDDGjBkDAFi7dq3aOvPnz4dMJsOPP/6odR8//fQTZDIZPvroI4325s2bw97eHq6urggMDMTTp0/h7e1d6Pu/qTA36S85Gfj+e+lnW1tg0iTTxkNExsHcZHh6FVM2NjZITk4ucJ3k5GTO+V+A58+BKVOAFi2A8+dz2vv0Ac6dA375RbpnFBGVPvv27UPLli2xb98+dO7cGVOmTEHjxo2xbNkytGnTBk+fPlVbf/78+QgMDERCQgICAwMxcOBAbN++HQMHDizwOMrkFBAQgPbt28PHxwc7duzAixcvVOsMHz4cMpkMmzZt0rqPjRs3AgBGjBihalu3bh3efvtt3Lx5EwEBARg5ciQiIyPRvXt3yOVyvfqkJDA36e+HH6SCCgACAgBXV9PGQ0SGx9xkJEIPHTp0EJ6eniI9PV3r8rS0NFGzZk3Rtm1bfXZf4pKSkgQAkZSUpNf2mZmZYs+ePSIzM7PQdRUKIXbvFsLDQwjp1ojSo2FDIU6e1OvwZqco/UGFM5f+TEtLE//9959IS0szaRzFlZ2dLZ4+fSqys7OLvO2dO3cEAFG7dm0RFham8YiMjBQJCQnC0dFReHh4iOjoaLXtt27dKgCI4OBgVdvNmzeFlZWV8PDwEHFxcar2pKQkUa9ePQFAdOrUSSOWhIQEYWNjI3x9fVVtoaGhAoBYs2aN2rrt27cXlpaW4uHDh2rtiYmJwsbGRjRv3lzV9vTpU1GxYkVRoUIFcePGDVW7XC4XXbp0EQCEl5eX2n7y61NdXzPFfQ9WYm5Sp+t7R0aGek66dk2vw5k1c3kfLUvMpU+Zm5ibtOWmgvpTl9dMUd5/9bpmavTo0RgzZgzefPNNrFy5Uu2u81FRUZg4cSIePnyIefPmFaPMK3uio6WhE7/8ktNmby9NLDF1KmBjY7LQiIqteXMgNtbUURTMzQ3488/i7ycqKgpz587VaK9SpQoiIyORnJyMZcuWwcvLS2354MGDsWjRImzbtg3fffcdAGDLli3IyspCSEgIXFxcVOs6Ojri448/VvtWLreNGzciMzNTbXlAQADmzZuHtWvXqoZWANI3e6dOncLWrVsREhKiat++fTsyMzPVLljeu3cvXrx4gcmTJ6Nu3bqqdisrKyxYsMCsx7wzN+ln+3ZAOfLxzTel23EQlRXMTcxNxqZ3MXXw4EHs3r0bvr6+qFWrFjw8PBATE4M7d+4gKysLgwYNwujRow0db6kklwNLlgBz5wKpqTntvXoBy5YBtWqZLjYiQ4mNzflAVtb5+/vj0KFDWpcNGjQIAHD27FlE5b3LNoD09HQkJCQgISEBTk5O+PvvvwEAHTp00FhXW5vS2rVrIZPJ1JJN7dq10bZtW5w+fRpXr15F/fr1AQDvvPMOJk+ejI0bN6olrE2bNsHKygpDhgxRtSnjad++vcYxW7VqBSszniubuanohFC/SW8ZnnCLyinmJglzk/HofeQdO3Zg+fLlWLFiBa5du4abN28CABo0aICgoCBMmDDBYEGWZqdPA++9B1y5ktPm7g588410M0QO3aeyws3N1BEUriRifPLkCQBg+fLlBa6XkpICJycnJCUlAYDaN39KrvlcuHL27FlcuXIFr732Gl566SW1ZQEBATh9+jTWrVunurdHlSpV8MYbb2D37t3477//0KBBA0RFReH06dPo1auX2rGV1xxpi8fCwgJOTk4F/l6mxtxUNEeOAJcvSz+3bs3bb1DZw9wkYW4yHr2LKZlMhuDgYAQHByMlJQVJSUmoXLkyKnAuVQDAkyfAzJnA6tU5bRYWQHAwMH8+4OhoutiIjCH3RCrmTKEw7v4d//8/9+XLl9GoUaNC169cuTIAID4+XmPoRVxcnNZtlBf3Hjt2LN/JFH788Ud89tlnsLa2BiANp9i9ezc2btyIhQsXqi76zTtUQxl/fHy8xj4VCgUSEhLg4eFR6O9lKsxNRZP7rNS0afyCj8oe5iYJc5Px6DWbX14VKlSAu7s7kxWkIRMbNwK+vuqFlJ+fNB72m29YSBGVZa1atQIAREZG6rR+06ZNAQAnT57UWKatLSUlBdu2bYODgwPGjBmj9dGkSRPEx8fjl1wXaPbq1QvVq1fHli1boFAosHnzZlSqVAl9+/bVGs8ff/yhcew///wTWcobEZUCzE0Fu3gRiIiQfq5TB+jXz6ThEJERMTcZUaFTVGhx6tQpMXXqVPHo0SOtyx8+fCimTp0qIiMj9dl9iTPUjEmXL2eKLl3UZ+mrVEmIb78VIivLwEGbMXOZ4aesMJf+5IxJOTMm+fv757tOfHy8qFSpknB2dhZXrlzRWJ6SkqL23njz5k1haWmp84xJ69evFwBEQEBAvjEcPnxYABC9e/dWa584caIAIBYuXCgAiFGjRmlsq5wxqWLFiuLWrVuqdrlcLrp162bWs/kxN6kr7L1j6NCcXLViRXEiNX/m8j5alphLnzI3MTdpy00lOZufXmemFi9ejP3798Mtn0GeNWrUwC+//IIlS5bos/tSJz0d2Lq1Hvz8rHD0aE77wIHA1avSDH6WlqaLj4hKjrOzM7Zu3YoXL16gadOmeOONNzBt2jRMmjQJffr0gZubm9od6uvUqYPQ0FDExMSgSZMmmDx5MkJCQtC4cWO1GYuUlMMoCppEoVu3bqhZsyYOHTqEhw8fqtqVwyZCQ0PVnudWpUoVLF68GC9evICfnx/Gjx+PGTNm4JVXXsHTp0/h7u4OCwuDDGowOOYm3d29K83iBwBOTsDIkaaNh4iMi7nJePQ66rlz57TOppFbx44dcebMGb2CKk1OnwZefdUK27f7IjNTGh/q7Q0cOADs2AGY8aUFRGQkvXv3xsWLFzFq1ChcuXIF3333HTZv3oy7d+9i9OjRmD9/vtr6oaGhWL16NapXr47vv/8eO3fuxDvvvIMdO3aorXf9+nWcOnUKtWrVQqdOnfI9voWFBUaOHIns7Gxs2LBB1d66dWvUrVsXcrkcNWvWROfOnbVuHxgYiJ07d8LHxwcbNmzAhg0b0Lp1axw5cgTJycmqsevmhrlJd0uXAtnZ0s9BQYCDg0nDIaISwNxkHHpNQBEfH1/oRV5ubm5aLxIra6ysAOUMk1ZWAh9+KMPHHzMxEZVF3t7eEELotG69evWwZs0anfc9duxYjB07VqM99/Hq1aun8/EXLFiABQsWaLTfuHFDp+0HDBiAAQMGqLXdunULL168QD0zvRERc5Nunj7NuabXzk4qpoio9GJuMm1u0uvMVJUqVXDv3r0C17l79y4qVqyoV1ClScuWwHvvKVC/fiLOncvCZ5+xkCKi0u3p06fIyMhQa0tLS8PUqVMBAP3MdKYC5ibdfP89kJIi/Tx6NODsbNp4iIh0Ya65Sa9iqnXr1vj5559x//59rcvv3buHPXv2FOtuxMuXL4e3tzfs7OzQqlUr/FnAraFXr16NDh06oGrVqqhatSq6detW4PqG9uWXCnz66Sk0bFhihyQiMprff/8d7u7uGDJkCGbMmIExY8agQYMG+OWXX9ClSxfVzR/NTUnkptIuI0OaVRaQpkHPdZ9MIiKzZq65Sa9iKiQkBKmpqWjXrh1+/PFHPHr0CADw6NEj/O9//0O7du2QlpaGDz74QK+gtm/fjpCQEISFheHChQto2rQp/P398x2acfz4cQwZMgTHjh1DZGQkPD090aNHD8SU0C2v7eyke0gREZUFDRs2RPfu3fHHH3/g22+/xZYtW1CxYkXMnz8fBw4cMNsJKIydm8qCzZuB2Fjp57fekqZEJyIqDcw1N+l1zVTHjh2xePFifPDBB6pZO2QymWq8pIWFBb755ht07NhRr6AWL16MwMBA1b5XrVqFAwcOYN26dZg5c6bG+ps3b1Z7vmbNGuzevRsREREICAjQKwYiovKqbt262LZtm6nDKDJj56bSTqEAvvoq5/mHH5ouFiKiojLX3KRXMQUA77//Pl577TWsWrUK586dQ1JSEqpUqYKWLVti/PjxaNSoETIyMmBra1uk/WZmZuKvv/7CrFmzVG0WFhbo1q2bzjcaS01NhVwuR7Vq1bQuz8jIUBtzmZycDACQy+WQy+VFile5Xe5/yzv2h2GZS3/K5XIIIaBQKKAw9q3ajUj5wVr5u1Dx5denCoUCQgjI5XJYFnB/CEO+to2Vm8qCX3+VbtcBAO3bA61bmzYeIqKyQO9iCgCaNGmCFStWaLRfuHABQUFB2LZtGxITE4u0z4SEBGRnZ8PV1VWt3dXVFdeuXdNpHzNmzIC7uzu6deumdfnChQsxd+5cjfYjR47AoRizR4SHh+u9bVnE/jAsU/enlZUV3Nzc8OLFC2RmZpo0FkN4/vy5qUMoc/L2aWZmJtLS0nDixIkC706fmppq0DiMkZvKgkWLcn7mWSkiIsMoVjGV27Nnz7Bp0yasXbsW//zzD4QQsLe3N9Tudfb5559j27ZtOH78OOzs7LSuM2vWLITkuuo2OTlZdZ2VPnPUy+VyhIeHo3v37rC2ttY79rKC/WFY5tKf6enpuH//PipWrJjv/63SQAiB58+fo1KlSpDJZKYOp0zIr0/T09Nhb2+Pjh07FviaUY4OMAZD5qbly5dj0aJFiI2NRdOmTfHdd9+hZcuWhW63bds2DBkyBH379sWePXv0OnZxnTsH/P679HO9esAbb5gkDCKiMqfYxdRvv/2GtWvXYu/evcjIyIAQAm3atMHo0aP1mlXDyckJlpaWiIuLU2uPi4vL9672Sl999RU+//xz/Pbbb2jSpEm+69na2mod4mFtbV2sD6vF3b6sYX8Ylqn7Mzs7GzKZDBYWFmY7AYEulMPQlL8LFV9+fWphYQGZTFboa9cYr2tD5yblxEirVq1Cq1atsHTpUvj7++P69etwcXHJd7vo6GhMmzYNHTp0KM6vU2y5r5X64ANOmkREZCh6vZ3ev38f8+bNQ61ateDv74/t27ejevXqEEJg1KhR+OOPPzB27FhUqlSpyPu2sbGBn58fIiIiVG0KhQIRERFo06ZNvtt9+eWXmD9/Pg4dOoTmzZvr82sREVEpZszclHtipAYNGmDVqlVwcHDAunXr8t0mOzsbw4YNw9y5c+Hj41OcX61Ybt8Gdu2SfnZxAUaMMFkoRERljs5npuRyOfbs2YO1a9ciIiIC2dnZqFChAoYNG4aAgAB06dIFVlZWsLIq/sjBkJAQjBw5Es2bN0fLli2xdOlSpKSkqGZnCggIgIeHBxYuXAgA+OKLLxAaGootW7bA29sbsf8/72vFihXL/c0ZiYjKspLITfpOjDRv3jy4uLhgzJgxOHnyZKHHMdbkSEuXSjP5AcDEidmwtFSgvM0PZC4T+ZQl5tKnnByJtCmoP3WZHKkor2uds4u7uzuePHkCmUyG1157DQEBAejfvz8qVKig88F0NWjQIDx+/BihoaGIjY1Fs2bNcOjQIdWkFPfu3VMbSrJy5UpkZmZiwIABavsJCwvDnDlzDB4fERGZh5LITfpMjHTq1CmsXbsWly5d0vk4xpgcKTnZGuvXS9ex2dpmwcfnCA4eLL8Fhakn8imLTN2nnByJCqKtP3WZHKkoEyPpXEwlJibCwsICU6dOxfTp0+Hs7KzzQfQRHByM4OBgrcuOHz+u9jw6OtqosRARkXkq6dyki+fPn2PEiBFYvXo1nJycdN7OGJMjBQbeRUaGlOrHjpVh8ODuRd5PWWAuE/mUJebSp5wcibQpqD91mRypKBMj6VxMjRo1Cjt37sTixYvx7bffwt/fHyNGjEDfvn1hY2Oj8wGJiIgMpSRyU1EnRoqKikJ0dDT69OmjalMOM7GyssL169dRu3Ztje0MPTlSejpw4EAtANKEEx98YAlr6/zv91UemHoin7LI1H3KyZFIm4L6U5fJkYrymtb5r7Vu3To8evQI33//PV599VX88ssvGDx4MFxdXfHee+/h1KlTOh+UiIh0I5PJ0LlzZ1OHYbZKIjcVdWIkX19fXL58GZcuXVI93nzzTbz22mu4dOkSPD09ix2TLjZtkiEpSfrWdcAAoFatEjksEZUDzE05ilT6VqxYEWPHjkVkZCT+/fdfTJkyBTY2Nli9ejU6deoEmUyG69ev4+7du8aKl4jKmrQ0465vQNHR0ZDJZAU+nj17ZvQ4Ro0apXFcR0dHtGjRAkuWLNG4cLawmC9dulToOnkf5qQkclNISAhWr16N//3vf7h69SomTJigMTGScoIKOzs7NGrUSO1RpUoVVKpUCY0aNSqR0RwKBbBkSc5ZKN6kl6iImJuKrLzmJr2nN6pfvz6+/vprfPHFF6qZlMLDw3Hy5EnUrl0bnTp1wqhRozCCc7ASUX5Wrwa+/BI4ehTQ5dv6+/eBLl2A6dOBwEDjx5eP2rVrY/jw4VqXleSY/TFjxqBmzZoQQuD+/fv46aefEBISgqNHj2L//v1q61avXj3f61Dd3NwQFham0b506VIkJSVpXWaujJWbijoxkqnt3w/cvCl9sOjUSYHmzc0nNiKzx9xULOUuNwkDun//vpg3b57w8fERMplMWFhYGHL3RpOUlCQAiKSkJL22z8zMFHv27BGZmZkGjqx0Yn8Ylrn0Z1pamvjvv/9EWlqaYXaYmipEnTpCAEL4+Ahx717B69+7J60HSNulpup12OzsbPH06VORnZ1d5G3v3LkjAAh/f3+9jq0PAKJTp05qbSNHjhQARGRkpFp7TEyMcHFxEQDEsWPH1PZRr169Ih/by8tL6JIm8utTXV8zxX0PLkx5zE3jx0v/VQAh9u6VGyG60sVc3kfLEnPpU+Ym5iZtCupPXV4zRXn/NehXVTVr1sQnn3yCqKgohIeHY/DgwYbcPRGVJfb20rd+Pj7SXUU7d5a+3dPm/n1p+e3b0vpHj0rbm7F//vkHgwcPRo0aNWBjYwMvLy9MmjQJiYmJWtdfs2YNGjVqBDs7O3h6emL69OlIT08v0jHd3d3Rv39/AMC5c+eK/TuUFeUxN61YAUREZOH11+/A31+YOhyi0oO5SQ1zU+GKf4fdfHTt2hVdu3Y11u6JqCzw9ASOH89JRp07S89zD6vIm6zyLjdD+/btwzvvvAMLCwv07dsXnp6e+O+//7Bs2TIcPnwYZ8+eRdWqVVXrz58/H6GhoXB1dUVgYCCsra2xfft2XL16Ve8YzO26JnNRXnKTTAZ06CDw/Pk/sLCoaepwiEoX5iYAzE26MloxRUSkk4KSlpkmq1u3bmm9IXjPnj1Rt25djBgxAk5OTvjjjz/g5eWlWr5t2zYMGTIEoaGh+O6771T7mjdvHjw8PHDhwgW4uLgAAObMmYOWLVsWKa7Y2Fj8/PPPAKCxbUJCgtaYW7dujZ49exbpOEREZR5zE3OTjlhMEZHpaUtaGzcCI0aYXbICpPsIzZ07V6O9SpUqiIyMRHJyMpYtW6aWrABg8ODBWLRoEbZt26ZKWFu2bEFWVhZCQkJUyQoAHB0d8fHHHxc4UcKaNWtw6NAhCCHw4MED/PTTT3j27Bn69u2Ljh07qq2bmJioNeb333+/1CQsIqISxdzE3KQDFlNEZB7yJq127aR2M0tWAODv749Dhw5pXTZo0CAAwNmzZxEVFaWxPD09HQkJCUhISICTkxP+/vtvAECHDh001tXWltvatWtVP1esWBH169fHsGHDEBQUpLFuvXr1cO3atQL3R0REeTA3aazL3KSOxRQRmQ9PT+lbP2WyAqTnZpSsCvPkyRMAwPLlywtcLyUlBU5OTkhKSgIAtW/+lJTTbucnMjISrVu31jNSIiLSCXOTGuYmdbzxBBGZj/v3peETuY0Ykf9MSmbI0dERAHD58mUIIfJ9KIdZVK5cGQAQHx+vsa+4uLiSC5yIiLRjblLD3KSOxRQRmYe8F/T+8YduU9OamVatWgGQvpnTRdOmTQEAJ0+e1FimrY2IiEoQc5PGMuYmdSymiMj0tM2M1Lat9G8pS1qjR49GpUqV8NFHH+Hff//VWJ6amoozZ86ong8dOhSWlpZYvHix2jeAycnJWLBgQYnETEREWjA3MTfpgMUUEZlWQVPMKi/8LUVJy9nZGVu3bsWLFy/QtGlTvPHGG5g2bRomTZqEPn36wM3NTW0a2Dp16iA0NBQxMTFo0qQJJk+ejJCQEDRu3Bh169Y13S9CRFSeMTcxN+mIE1AQkenocq8OXW6eaGZ69+6NixcvYtGiRfjtt98QHh6OChUqoGbNmhg9ejSGDx+utn5oaCjc3d2xZMkSfP/993BxccHgwYMxb948ODg4mOi3ICIqp5ibADA36YrFFBGZRloa0KWLbvfqyJu0unQB/vkHsLcvwYABb29vCCF0WrdevXpYs2aNzvseO3Ysxo4dq9Gu7XgbNmzAhg0bdN63rjHnFR0drdd2RESlFnOTGuamwnGYHxGZhr09MH06UKeObt/mKZNWnTrSdiWcrIiIqBxgbqIi4pkpIjKdwEBg+HDdk4+np0m+9SMionKEuYmKgGemiMi0ipp8mKyIiMjYmJtIRyymiIiIiIiI9MBiioiIiIiISA8spoiIiIiIiPTAYoqIikTfqUyp/OFrhYhKCt9vSFeGfq2wmCIinVhaWgIA5HK5iSOh0kL5WlG+doiIDI25iYrK0LmJxRQR6cTa2hq2trZISkriN4BUKCEEkpKSYGtrC2tra1OHQ0RlFHMTFYUxchPvM0VEOnNyckJMTAwePHiAypUrw9raGjKZzNRhFYlCoUBmZibS09NhYcHvkwwhd5/KZDLI5XIkJSXhxYsX8PDwMHV4RFTGMTdRXnn7UwhhtNzEYoqIdObo6AgASEhIQExMjImj0Y8QAmlpabC3ty91ydZcaetTW1tbeHh4qF4zRETGwtxEeeXXn8bITSymiKhIHB0d4ejoCLlcjuzsbFOHU2RyuRwnTpxAx44dOfzMQPL2qaWlJfuWiEoUcxPlpq0/jZWbWEwRkV6sra1L5Ru+paUlsrKyYGdnVyrjN0fsUyIyF8xNBJRsf3JQJhERERERkR5YTBEREREREemBxRQREREREZEezLaYWr58Oby9vWFnZ4dWrVrhzz//LHD9nTt3wtfXF3Z2dmjcuDEOHjxYQpESEVF5UJS8tHr1anTo0AFVq1ZF1apV0a1bt0LzGBERlT5mWUxt374dISEhCAsLw4ULF9C0aVP4+/sjPj5e6/qnT5/GkCFDMGbMGFy8eBH9+vVDv379cOXKlRKOnIiIyqKi5qXjx49jyJAhOHbsGCIjI+Hp6YkePXqU2mmbiYhIO7MsphYvXozAwECMHj0aDRo0wKpVq+Dg4IB169ZpXf+bb75Bz5498eGHH6J+/fqYP38+Xn31VSxbtqyEIyciorKoqHlp8+bNmDhxIpo1awZfX1+sWbMGCoUCERERJRw5EREZk9lNjZ6ZmYm//voLs2bNUrVZWFigW7duiIyM1LpNZGQkQkJC1Nr8/f2xZ88eretnZGQgIyND9TwpKQkA8OTJE8jl8iLHLJfLkZqaisTERE5nCfaHobE/DYv9aXjF7dPnz58DkG6yaI70yUt5paamQi6Xo1q1avmuw9xkPOwLw2OfGhb707BKMi+ZXTGVkJCA7OxsuLq6qrW7urri2rVrWreJjY3Vun5sbKzW9RcuXIi5c+dqtNeqVUvPqImIqLieP3+OypUrmzoMDfrkpbxmzJgBd3d3dOvWLd91mJuIiMyLLnnJ7IqpkjBr1iy1M1kKhQJPnjxB9erVIZPJiry/5ORkeHp64v79+3B0dDRkqKUS+8Ow2J+Gxf40vOL2qRACz58/h7u7uxGiM73PP/8c27Ztw/Hjx2FnZ5fvesxNxsO+MDz2qWGxPw2rJPOS2RVTTk5OsLS0RFxcnFp7XFwc3NzctG7j5uZWpPVtbW1ha2ur1lalShX9g/5/jo6O/A+QC/vDsNifhsX+NLzi9Kk5npFS0icvKX311Vf4/PPP8dtvv6FJkyYFrsvcZHzsC8NjnxoW+9OwSiIvmd0EFDY2NvDz81O7SFd50W6bNm20btOmTRuNi3rDw8PzXZ+IiEhX+uQlAPjyyy8xf/58HDp0CM2bNy+JUImIqISZ3ZkpAAgJCcHIkSPRvHlztGzZEkuXLkVKSgpGjx4NAAgICICHhwcWLlwIAHj//ffRqVMnfP311+jduze2bduG8+fP44cffjDlr0FERGVEUfPSF198gdDQUGzZsgXe3t6qa3grVqyIihUrmuz3ICIiwzLLYmrQoEF4/PgxQkNDERsbi2bNmuHQoUOqi3/v3bsHC4uck2pt27bFli1b8PHHH2P27NmoW7cu9uzZg0aNGpVIvLa2tggLC9MYnlFesT8Mi/1pWOxPwysPfVrUvLRy5UpkZmZiwIABavsJCwvDnDlzSiTm8vB30RX7wvDYp4bF/jSskuxPmTDXuWiJiIiIiIjMmNldM0VERERERFQasJgiIiIiIiLSA4spIiIiIiIiPbCYIiIiIiIi0gOLqWI4ceIE+vTpA3d3d8hkMuzZs8fUIZnMnDlzIJPJ1B6+vr6mDqtUKez1JIRAaGgoatSoAXt7e3Tr1g03b940TbClQGH9OWrUKI3XbM+ePU0TbCmwcOFCtGjRApUqVYKLiwv69euH69evq62Tnp6OoKAgVK9eHRUrVsTbb7+tcaNbMj7mphzMTcXDvGR4zE2GZQ65icVUMaSkpKBp06ZYvny5qUMxCw0bNsSjR49Uj1OnTpk6pFKlsNfTl19+iW+//RarVq3C2bNnUaFCBfj7+yM9Pb2EIy0ddPn/2bNnT7XX7NatW0swwtLl999/R1BQEM6cOYPw8HDI5XL06NEDKSkpqnWmTp2K/fv3Y+fOnfj999/x8OFD9O/f34RRl0/MTeqYm/THvGR4zE2GZRa5SZBBABA///yzqcMwmbCwMNG0aVNTh1Fm5H09KRQK4ebmJhYtWqRqe/bsmbC1tRVbt241QYSli7b/nyNHjhR9+/Y1STxlQXx8vAAgfv/9dyGE9Hq0trYWO3fuVK1z9epVAUBERkaaKsxyj7mJuclQmJcMj7nJ8EyRm3hmigzm5s2bcHd3h4+PD4YNG4Z79+6ZOqQy486dO4iNjUW3bt1UbZUrV0arVq0QGRlpwshKt+PHj8PFxQX16tXDhAkTkJiYaOqQSo2kpCQAQLVq1QAAf/31F+Ryudpr1NfXFy+99BJfo2RSzE3GwbxkPMxN+jNFbmIxRQbRqlUrbNiwAYcOHcLKlStx584ddOjQAc+fPzd1aGVCbGwsAMDV1VWt3dXVVbWMiqZnz5748ccfERERgS+++AK///47Xn/9dWRnZ5s6NLOnUCgwZcoUtGvXDo0aNQIgvUZtbGxQpUoVtXX5GiVTYm4yHuYl42Bu0p+pcpOVQfZC5d7rr7+u+rlJkyZo1aoVvLy8sGPHDowZM8aEkRFpN3jwYNXPjRs3RpMmTVC7dm0cP34cXbt2NWFk5i8oKAhXrlzhtSdk9pibqLRhbtKfqXITz0yRUVSpUgUvv/wybt26ZepQygQ3NzcA0Jh9Ji4uTrWMisfHxwdOTk58zRYiODgYv/zyC44dO4aaNWuq2t3c3JCZmYlnz56prc/XKJkT5ibDYV4qGcxNujFlbmIxRUbx4sULREVFoUaNGqYOpUyoVasW3NzcEBERoWpLTk7G2bNn0aZNGxNGVnY8ePAAiYmJfM3mQwiB4OBg/Pzzzzh69Chq1aqlttzPzw/W1tZqr9Hr16/j3r17fI2S2WBuMhzmpZLB3FQwc8hNHOZXDC9evFD7puDOnTu4dOkSqlWrhpdeesmEkZW8adOmoU+fPvDy8sLDhw8RFhYGS0tLDBkyxNShlRqFvZ6mTJmCBQsWoG7duqhVqxY++eQTuLu7o1+/fqYL2owV1J/VqlXD3Llz8fbbb8PNzQ1RUVGYPn066tSpA39/fxNGbb6CgoKwZcsW7N27F5UqVVKNNa9cuTLs7e1RuXJljBkzBiEhIahWrRocHR0xadIktGnTBq1btzZx9OULc1MO5qbiYV4yPOYmwzKL3GSQOQHLqWPHjgkAGo+RI0eaOrQSN2jQIFGjRg1hY2MjPDw8xKBBg8StW7dMHVapUtjrSaFQiE8++US4uroKW1tb0bVrV3H9+nXTBm3GCurP1NRU0aNHD+Hs7Cysra2Fl5eXCAwMFLGxsaYO22xp60sAYv369ap10tLSxMSJE0XVqlWFg4ODeOutt8SjR49MF3Q5xdyUg7mpeJiXDI+5ybDMITfJ/j8QIiIiIiIiKgJeM0VERERERKQHFlNERERERER6YDFFRERERESkBxZTREREREREemAxRUREREREpAcWU0RERERERHpgMUVERERERKQHFlNERERERER6YDFFREUmk8nQuXNnU4dBREQEgHmJTIfFFJERREdHQyaTqT2sra3h4eGBd955B+fPnzd1iEREVI4wLxEZh5WpAyAqy2rXro3hw4cDAFJSUvDXX39h586d2LNnD3777Td07NjRxBESEVF5wrxEZFgspoiMqE6dOpgzZ45a2+eff45Zs2bhk08+we+//26awIiIqFxiXiIyLA7zIyphY8aMAQD89ddfau0JCQmYMmUKatWqBVtbW7i4uOCdd97BlStXNPbRuXNnyGQyrfsfNWoUZDIZoqOjVW0bNmyATCbDhg0bcOTIEbRt2xYODg6oXr06Ro4cicTERK37WrNmDRo1agQ7Ozt4enpi+vTpSE9P1/M3JyIic8S8RKQ/npkiMhErq5z/fo8fP0abNm0QFRWFzp07Y/Dgwbhz5w527dqFAwcO4PDhw2jfvn2xj7lv3z4cOHAAffr0Qdu2bXHixAn8+OOPiIqKwqlTp9TWnT9/PkJDQ+Hq6orAwEBYW1tj+/btuHr1arHjICIi88O8RFR0LKaIStiaNWsAQC0JzZgxA1FRUZg1axY+++wzVfvBgwfRu3dvjB49GtevX4eFRfFOJu/fvx/Hjx9Hu3btAADZ2dno1q0bjh8/jjNnzqB169YAgFu3bmHevHnw8PDAhQsX4OLiAgCYM2cOWrZsWawYiIjIvDAvEemPw/yIjOjWrVuYM2cO5syZgw8//BBdunTB7Nmz4erqikWLFgEAMjMzsXXrVlSvXh0ff/yx2va9evVC9+7dcevWLfzxxx/Fjmfo0KGqhAUAlpaWGDlyJADg3LlzqvYtW7YgKysLISEhqoQFAI6OjhoxEhFR6cG8RGRYPDNFZERRUVGYO3euWpubmxtOnjyJOnXqAACuXbuG9PR0vPbaa3BwcNDYx2uvvYbw8HBcunQJHTp0KFY8fn5+Gm01a9YEADx79kzV9vfffwOA1uMVNwYiIjId5iUiw+KZKSIj8vf3hxACQgjEx8dj0aJFiI+Px5tvvokXL14AAJKTkwEArq6uWvdRo0YNtfWKw9HRUaNNOUY+Oztb1ZaUlAQAat/+KeUXJxERmT/mJSLDYjFFVEKcnZ0xbdo0zJ49G1evXlUNS1Amkri4OK3bxcbGqq0HQDVGPSsrS2N9ZcIpjsqVKwMA4uPjNZblFycREZUuzEtExcdiiqiEzZ49G+7u7lixYgWio6Ph6+sLOzs7nDt3DqmpqRrrHz9+HADQrFkzVVvVqlUBADExMWrrKhQK1VCI4mjatCkA4OTJkxrLtLUREVHpxbxEpD8WU0QlzN7eHjNmzIBcLsf8+fNhY2ODIUOGICEhAQsXLlRb99ChQzh8+DDq1KmjdoFuixYtAEj36cht8eLFuHPnTrFjHDp0KCwtLbF48WK1bwGTk5OxYMGCYu+fiIjMB/MSkf5YTBGZwLhx4+Du7q66l8YXX3wBHx8fLFiwAF27dsXs2bMxdOhQ9OnTBw4ODli/fr3a9LOjR49G1apVMWfOHLz11luYNm0aOnfujM8//xydOnUqdnx16tRBaGgoYmJi0KRJE0yePBkhISFo3Lgx6tatW+z9ExGReWFeItIPiykiE7Czs8OsWbOQlZWFuXPnwtnZGWfPnsXkyZMRFRWFr776CuHh4ejXrx/Onj2rcWNEV1dXHDt2DF27dsWRI0ewevVqVKlSBWfOnIG3t7dBYgwNDcXq1atRvXp1fP/999i5cyfeeecd7NixwyD7JyIi88G8RKQfmRBCmDoIIiIiIiKi0oZnpoiIiIiIiPTAYoqIiIiIiEgPLKaIiIiIiIj0wGKKiIiIiIhIDyymiIiIiIiI9MBiioiIiIiISA8spoiIiIiIiPTAYoqIiIiIiEgPLKaIiIiIiIj0wGKKiIiIiIhIDyymiIiIiIiI9MBiioiIiIiISA//BxFKjUHVmVR3AAAAAElFTkSuQmCC", "text/plain": [ "

" ] @@ -149,13 +101,23 @@ } ], "source": [ + "color1 = 'blue'\n", + "color2 = 'red'\n", + "\n", "def viz():\n", " fig, axs = plt.subplots(figsize=(10, 2), nrows=1, ncols=2)\n", " \n", - " # cifar100 - fedavg\n", - " axs[0].plot([r for r, _ in fedavg_cifar], [a for _, a in fedavg_cifar], label='FedAvg', linewidth=2.0)\n", - " \n", + " # cifar100\n", + " axs[0].plot([r for r, _ in fedavg_cifar], [a for _, a in fedavg_cifar], label='FedAvg', color=color1, linewidth=2.0)\n", + " axs[0].scatter([r for r, _ in fedpft_cifar], [a for _, a in fedpft_cifar], label='FedPFT', color=color2, marker='x', s=100)\n", " axs[0].set_title('CIFAR100 - ResNet50')\n", + " axs[0].set_ylim(0, 0.7)\n", + " \n", + " # caltech101\n", + " axs[1].plot([r for r, _ in fedavg_caltech], [a for _, a in fedavg_caltech], label='FedAvg', color=color1, linewidth=2.0)\n", + " axs[1].scatter([r for r, _ in fedpft_caltech], [a for _, a in fedpft_caltech], label='FedPFT', color=color2, marker='x', s=100)\n", + " axs[1].set_title('Caltech101 - Clip/ViT-B')\n", + " axs[1].set_ylim(0.2, 1)\n", " \n", " for ax in axs:\n", " ax.set_xticks([1, 5, 10 , 15, 20])\n", @@ -171,12 +133,12 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 22, "id": "92460065", "metadata": {}, "outputs": [], "source": [ - "saveFig(\"FedProx_mnist.png\", f)" + "saveFig(\"FedPft.png\", f)" ] } ], From ddd0f08213746d58d3a08816baf2c560283afe66 Mon Sep 17 00:00:00 2001 From: mahdi Date: Mon, 15 Apr 2024 15:08:56 +0000 Subject: [PATCH 10/29] remove empty line --- baselines/fedpft/fedpft/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/baselines/fedpft/fedpft/models.py b/baselines/fedpft/fedpft/models.py index 7ebc9beed4a..b4594cb986a 100644 --- a/baselines/fedpft/fedpft/models.py +++ b/baselines/fedpft/fedpft/models.py @@ -103,7 +103,7 @@ def extract_features( 2D array containing labels of `features`. """ feature_extractor.to(device) - + features, labels = [], [] for dict in dataloader: batch_samples = dict["img"].to(device) @@ -164,7 +164,7 @@ def test( total += samples.shape[0] running_loss = nn.CrossEntropyLoss()(output, labels) loss += running_loss - + return loss.cpu().item(), correct / total From 6996b816564a5378205a234a2558dbaca18deee0 Mon Sep 17 00:00:00 2001 From: mahdi Date: Mon, 15 Apr 2024 15:12:52 +0000 Subject: [PATCH 11/29] removed extended_readme --- baselines/fedpft/EXTENDED_README.md | 123 ---------------------------- 1 file changed, 123 deletions(-) delete mode 100644 baselines/fedpft/EXTENDED_README.md diff --git a/baselines/fedpft/EXTENDED_README.md b/baselines/fedpft/EXTENDED_README.md deleted file mode 100644 index 9c8f5bc72fa..00000000000 --- a/baselines/fedpft/EXTENDED_README.md +++ /dev/null @@ -1,123 +0,0 @@ - -# Extended Readme - -> The baselines are expected to run in a machine running Ubuntu 22.04 - -While `README.md` should include information about the baseline you implement and how to run it, this _extended_ readme provides info on what's the expected directory structure for a new baseline and more generally the instructions to follow before your baseline can be merged into the Flower repository. Please follow closely these instructions. It is likely that you have already completed steps 1-2. - -1. Fork the Flower repository and clone it. -2. Navigate to the `baselines/` directory and from there run: - ```bash - # This will create a new directory with the same structure as this `baseline_template` directory. - ./dev/create-baseline.sh - ``` -3. All your code and configs should go into a sub-directory with the same name as the name of your baseline. - * The sub-directory contains a series of Python scripts that you can edit. Please stick to these files and consult with us if you need additional ones. - * There is also a basic config structure in `/conf` ready be parsed by [Hydra](https://hydra.cc/) when executing your `main.py`. -4. Therefore, the directory structure in your baseline should look like: - ```bash - baselines/ - ├── README.md # describes your baseline and everything needed to use it - ├── EXTENDED_README.md # to remove before creating your PR - ├── pyproject.toml # details your Python environment - └── - ├── *.py # several .py files including main.py and __init__.py - └── conf - └── *.yaml # one or more Hydra config files - - ``` -> :warning: Make sure the variable `name` in `pyproject.toml` is set to the name of the sub-directory containing all your code. - -5. Add your dependencies to the `pyproject.toml` (see below a few examples on how to do it). Read more about Poetry below in this `EXTENDED_README.md`. -6. Regularly check that your coding style and the documentation you add follow good coding practices. To test whether your code meets the requirements, please run the following: - ```bash - # After activating your environment and from your baseline's directory - cd .. # to go to the top-level directory of all baselines - ./dev/test-baseline.sh - ./dev/test-baseline-structure.sh - ``` - Both `test-baseline.sh` and `test-baseline-structure.sh` will also be automatically run when you create a PR, and both tests need to pass for the baseline to be merged. - To automatically solve some formatting issues and apply easy fixes, please run the formatting script: - ```bash - # After activating your environment and from your baseline's directory - cd .. # to go to the top-level directory of all baselines - ./dev/format-baseline.sh - ``` -7. Ensure that the Python environment for your baseline can be created without errors by simply running `poetry install` and that this is properly described later when you complete the `Environment Setup` section in `README.md`. This is specially important if your environment requires additional steps after doing `poetry install`. -8. Ensure that your baseline runs with default arguments by running `poetry run python -m .main`. Then, describe this and other forms of running your code in the `Running the Experiments` section in `README.md`. -9. Once your code is ready and you have checked: - * that following the instructions in your `README.md` the Python environment can be created correctly - - * that running the code following your instructions can reproduce the experiments in the paper - - , then you just need to create a Pull Request (PR) to kickstart the process of merging your baseline into the Flower repository. - -> Once you are happy to merge your baseline contribution, please delete this `EXTENDED_README.md` file. - - -## About Poetry - -We use Poetry to manage the Python environment for each individual baseline. You can follow the instructions [here](https://python-poetry.org/docs/) to install Poetry in your machine. - - -### Specifying a Python Version (optional) -By default, Poetry will use the Python version in your system. In some settings, you might want to specify a particular version of Python to use inside your Poetry environment. You can do so with [`pyenv`](https://github.com/pyenv/pyenv). Check the documentation for the different ways of installing `pyenv`, but one easy way is using the [automatic installer](https://github.com/pyenv/pyenv-installer): -```bash -curl https://pyenv.run | bash # then, don't forget links to your .bashrc/.zshrc -``` - -You can then install any Python version with `pyenv install ` (e.g. `pyenv install 3.9.17`). Then, in order to use that version for your baseline, you'd do the following: - -```bash -# cd to your baseline directory (i.e. where the `pyproject.toml` is) -pyenv local - -# set that version for poetry -poetry env use - -# then you can install your Poetry environment (see the next setp) -``` - -### Installing Your Environment -With the Poetry tool already installed, you can create an environment for this baseline with commands: -```bash -# run this from the same directory as the `pyproject.toml` file is -poetry install -``` - -This will create a basic Python environment with just Flower and additional packages, including those needed for simulation. Next, you should add the dependencies for your code. It is **critical** that you fix the version of the packages you use using a `=` not a `=^`. You can do so via [`poetry add`](https://python-poetry.org/docs/cli/#add). Below are some examples: - -```bash -# For instance, if you want to install tqdm -poetry add tqdm==4.65.0 - -# If you already have a requirements.txt, you can add all those packages (but ensure you have fixed the version) in one go as follows: -poetry add $( cat requirements.txt ) -``` -With each `poetry add` command, the `pyproject.toml` gets automatically updated so you don't need to keep that `requirements.txt` as part of this baseline. - - -More critically however, is adding your ML framework of choice to the list of dependencies. For some frameworks you might be able to do so with the `poetry add` command. Check [the Poetry documentation](https://python-poetry.org/docs/cli/#add) for how to add packages in various ways. For instance, let's say you want to use PyTorch: - -```bash -# with plain `pip` you'd run a command such as: -pip install torch==1.13.1+cu117 torchvision==0.14.1+cu117 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu117 - -# to add the same 3 dependencies to your Poetry environment you'd need to add the URL to the wheel that the above pip command auto-resolves for you. -# You can find those wheels in `https://download.pytorch.org/whl/cu117`. Copy the link and paste it after the `poetry add` command. -# For instance to add `torch==1.13.1+cu117` and a x86 Linux system with Python3.8 you'd: -poetry add https://download.pytorch.org/whl/cu117/torch-1.13.1%2Bcu117-cp38-cp38-linux_x86_64.whl -# you'll need to repeat this for both `torchvision` and `torchaudio` -``` -The above is just an example of how you can add these dependencies. Please refer to the Poetry documentation to extra reference. - -If all attempts fail, you can still install packages via standard `pip`. You'd first need to source/activate your Poetry environment. -```bash -# first ensure you have created your environment -# and installed the base packages provided in the template -poetry install - -# then activate it -poetry shell -``` -Now you are inside your environment (pretty much as when you use `virtualenv` or `conda`) so you can install further packages with `pip`. Please note that, unlike with `poetry add`, these extra requirements won't be captured by `pyproject.toml`. Therefore, please ensure that you provide all instructions needed to: (1) create the base environment with Poetry and (2) install any additional dependencies via `pip` when you complete your `README.md`. \ No newline at end of file From 3ecf50c7d38d3b1bb50499dae33ef169bb89276e Mon Sep 17 00:00:00 2001 From: Mahdi Date: Sun, 14 Apr 2024 12:45:18 -0400 Subject: [PATCH 12/29] add FedPFT baseline --- baselines/fedpft/.gitignore | 3 + baselines/fedpft/EXTENDED_README.md | 123 ++++++++++ baselines/fedpft/LICENSE | 202 ++++++++++++++++ baselines/fedpft/README.md | 109 +++++++++ baselines/fedpft/fedpft/__init__.py | 1 + baselines/fedpft/fedpft/client.py | 170 ++++++++++++++ baselines/fedpft/fedpft/conf/base.yaml | 15 ++ .../fedpft/fedpft/conf/client/fedavg.yaml | 2 + .../fedpft/fedpft/conf/client/fedpft.yaml | 2 + .../fedpft/fedpft/conf/dataset/CIFAR100.yaml | 10 + .../fedpft/conf/dataset/Caltech101.yaml | 10 + baselines/fedpft/fedpft/conf/model/clip.yaml | 9 + .../fedpft/fedpft/conf/model/resnet50.yaml | 8 + .../fedpft/fedpft/conf/strategy/fedavg.yaml | 12 + .../fedpft/fedpft/conf/strategy/fedpft.yaml | 22 ++ baselines/fedpft/fedpft/dataset.py | 108 +++++++++ .../fedpft/fedpft/dataset_preparation.py | 1 + baselines/fedpft/fedpft/main.py | 88 +++++++ baselines/fedpft/fedpft/models.py | 221 ++++++++++++++++++ baselines/fedpft/fedpft/server.py | 94 ++++++++ baselines/fedpft/fedpft/strategy.py | 145 ++++++++++++ baselines/fedpft/fedpft/utils.py | 102 ++++++++ baselines/fedpft/pyproject.toml | 143 ++++++++++++ 23 files changed, 1600 insertions(+) create mode 100644 baselines/fedpft/.gitignore create mode 100644 baselines/fedpft/EXTENDED_README.md create mode 100644 baselines/fedpft/LICENSE create mode 100644 baselines/fedpft/README.md create mode 100644 baselines/fedpft/fedpft/__init__.py create mode 100644 baselines/fedpft/fedpft/client.py create mode 100644 baselines/fedpft/fedpft/conf/base.yaml create mode 100644 baselines/fedpft/fedpft/conf/client/fedavg.yaml create mode 100644 baselines/fedpft/fedpft/conf/client/fedpft.yaml create mode 100644 baselines/fedpft/fedpft/conf/dataset/CIFAR100.yaml create mode 100644 baselines/fedpft/fedpft/conf/dataset/Caltech101.yaml create mode 100644 baselines/fedpft/fedpft/conf/model/clip.yaml create mode 100644 baselines/fedpft/fedpft/conf/model/resnet50.yaml create mode 100644 baselines/fedpft/fedpft/conf/strategy/fedavg.yaml create mode 100644 baselines/fedpft/fedpft/conf/strategy/fedpft.yaml create mode 100644 baselines/fedpft/fedpft/dataset.py create mode 100644 baselines/fedpft/fedpft/dataset_preparation.py create mode 100644 baselines/fedpft/fedpft/main.py create mode 100644 baselines/fedpft/fedpft/models.py create mode 100644 baselines/fedpft/fedpft/server.py create mode 100644 baselines/fedpft/fedpft/strategy.py create mode 100644 baselines/fedpft/fedpft/utils.py create mode 100644 baselines/fedpft/pyproject.toml diff --git a/baselines/fedpft/.gitignore b/baselines/fedpft/.gitignore new file mode 100644 index 00000000000..4ab8207aedb --- /dev/null +++ b/baselines/fedpft/.gitignore @@ -0,0 +1,3 @@ +outputs/ +multirun/ +.ruff_cache/ \ No newline at end of file diff --git a/baselines/fedpft/EXTENDED_README.md b/baselines/fedpft/EXTENDED_README.md new file mode 100644 index 00000000000..9c8f5bc72fa --- /dev/null +++ b/baselines/fedpft/EXTENDED_README.md @@ -0,0 +1,123 @@ + +# Extended Readme + +> The baselines are expected to run in a machine running Ubuntu 22.04 + +While `README.md` should include information about the baseline you implement and how to run it, this _extended_ readme provides info on what's the expected directory structure for a new baseline and more generally the instructions to follow before your baseline can be merged into the Flower repository. Please follow closely these instructions. It is likely that you have already completed steps 1-2. + +1. Fork the Flower repository and clone it. +2. Navigate to the `baselines/` directory and from there run: + ```bash + # This will create a new directory with the same structure as this `baseline_template` directory. + ./dev/create-baseline.sh + ``` +3. All your code and configs should go into a sub-directory with the same name as the name of your baseline. + * The sub-directory contains a series of Python scripts that you can edit. Please stick to these files and consult with us if you need additional ones. + * There is also a basic config structure in `/conf` ready be parsed by [Hydra](https://hydra.cc/) when executing your `main.py`. +4. Therefore, the directory structure in your baseline should look like: + ```bash + baselines/ + ├── README.md # describes your baseline and everything needed to use it + ├── EXTENDED_README.md # to remove before creating your PR + ├── pyproject.toml # details your Python environment + └── + ├── *.py # several .py files including main.py and __init__.py + └── conf + └── *.yaml # one or more Hydra config files + + ``` +> :warning: Make sure the variable `name` in `pyproject.toml` is set to the name of the sub-directory containing all your code. + +5. Add your dependencies to the `pyproject.toml` (see below a few examples on how to do it). Read more about Poetry below in this `EXTENDED_README.md`. +6. Regularly check that your coding style and the documentation you add follow good coding practices. To test whether your code meets the requirements, please run the following: + ```bash + # After activating your environment and from your baseline's directory + cd .. # to go to the top-level directory of all baselines + ./dev/test-baseline.sh + ./dev/test-baseline-structure.sh + ``` + Both `test-baseline.sh` and `test-baseline-structure.sh` will also be automatically run when you create a PR, and both tests need to pass for the baseline to be merged. + To automatically solve some formatting issues and apply easy fixes, please run the formatting script: + ```bash + # After activating your environment and from your baseline's directory + cd .. # to go to the top-level directory of all baselines + ./dev/format-baseline.sh + ``` +7. Ensure that the Python environment for your baseline can be created without errors by simply running `poetry install` and that this is properly described later when you complete the `Environment Setup` section in `README.md`. This is specially important if your environment requires additional steps after doing `poetry install`. +8. Ensure that your baseline runs with default arguments by running `poetry run python -m .main`. Then, describe this and other forms of running your code in the `Running the Experiments` section in `README.md`. +9. Once your code is ready and you have checked: + * that following the instructions in your `README.md` the Python environment can be created correctly + + * that running the code following your instructions can reproduce the experiments in the paper + + , then you just need to create a Pull Request (PR) to kickstart the process of merging your baseline into the Flower repository. + +> Once you are happy to merge your baseline contribution, please delete this `EXTENDED_README.md` file. + + +## About Poetry + +We use Poetry to manage the Python environment for each individual baseline. You can follow the instructions [here](https://python-poetry.org/docs/) to install Poetry in your machine. + + +### Specifying a Python Version (optional) +By default, Poetry will use the Python version in your system. In some settings, you might want to specify a particular version of Python to use inside your Poetry environment. You can do so with [`pyenv`](https://github.com/pyenv/pyenv). Check the documentation for the different ways of installing `pyenv`, but one easy way is using the [automatic installer](https://github.com/pyenv/pyenv-installer): +```bash +curl https://pyenv.run | bash # then, don't forget links to your .bashrc/.zshrc +``` + +You can then install any Python version with `pyenv install ` (e.g. `pyenv install 3.9.17`). Then, in order to use that version for your baseline, you'd do the following: + +```bash +# cd to your baseline directory (i.e. where the `pyproject.toml` is) +pyenv local + +# set that version for poetry +poetry env use + +# then you can install your Poetry environment (see the next setp) +``` + +### Installing Your Environment +With the Poetry tool already installed, you can create an environment for this baseline with commands: +```bash +# run this from the same directory as the `pyproject.toml` file is +poetry install +``` + +This will create a basic Python environment with just Flower and additional packages, including those needed for simulation. Next, you should add the dependencies for your code. It is **critical** that you fix the version of the packages you use using a `=` not a `=^`. You can do so via [`poetry add`](https://python-poetry.org/docs/cli/#add). Below are some examples: + +```bash +# For instance, if you want to install tqdm +poetry add tqdm==4.65.0 + +# If you already have a requirements.txt, you can add all those packages (but ensure you have fixed the version) in one go as follows: +poetry add $( cat requirements.txt ) +``` +With each `poetry add` command, the `pyproject.toml` gets automatically updated so you don't need to keep that `requirements.txt` as part of this baseline. + + +More critically however, is adding your ML framework of choice to the list of dependencies. For some frameworks you might be able to do so with the `poetry add` command. Check [the Poetry documentation](https://python-poetry.org/docs/cli/#add) for how to add packages in various ways. For instance, let's say you want to use PyTorch: + +```bash +# with plain `pip` you'd run a command such as: +pip install torch==1.13.1+cu117 torchvision==0.14.1+cu117 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu117 + +# to add the same 3 dependencies to your Poetry environment you'd need to add the URL to the wheel that the above pip command auto-resolves for you. +# You can find those wheels in `https://download.pytorch.org/whl/cu117`. Copy the link and paste it after the `poetry add` command. +# For instance to add `torch==1.13.1+cu117` and a x86 Linux system with Python3.8 you'd: +poetry add https://download.pytorch.org/whl/cu117/torch-1.13.1%2Bcu117-cp38-cp38-linux_x86_64.whl +# you'll need to repeat this for both `torchvision` and `torchaudio` +``` +The above is just an example of how you can add these dependencies. Please refer to the Poetry documentation to extra reference. + +If all attempts fail, you can still install packages via standard `pip`. You'd first need to source/activate your Poetry environment. +```bash +# first ensure you have created your environment +# and installed the base packages provided in the template +poetry install + +# then activate it +poetry shell +``` +Now you are inside your environment (pretty much as when you use `virtualenv` or `conda`) so you can install further packages with `pip`. Please note that, unlike with `poetry add`, these extra requirements won't be captured by `pyproject.toml`. Therefore, please ensure that you provide all instructions needed to: (1) create the base environment with Poetry and (2) install any additional dependencies via `pip` when you complete your `README.md`. \ No newline at end of file diff --git a/baselines/fedpft/LICENSE b/baselines/fedpft/LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/baselines/fedpft/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/baselines/fedpft/README.md b/baselines/fedpft/README.md new file mode 100644 index 00000000000..3f045810835 --- /dev/null +++ b/baselines/fedpft/README.md @@ -0,0 +1,109 @@ +--- +title: Parametric Feature Transfer, One-shot Federated Learning with Foundation Models +url: https://arxiv.org/abs/2402.01862 +labels: [foundation-models, pre-trained, one-shot, one-round] # please add between 4 and 10 single-word (maybe two-words) labels (e.g. system heterogeneity, image classification, asynchronous, weight sharing, cross-silo). Do not use "" +dataset: [CIFAR100, Caltech101] # list of datasets you include in your baseline. Do not use "" +--- + +# FedPFT: One-shot Federated Learning with Foundation Models + +> Note: If you use this baseline in your work, please remember to cite the original authors of the paper as well as the Flower paper. + +**Paper:** [arxiv.org/abs/2402.01862](https://arxiv.org/abs/2402.01862) + +**Authors:** Mahdi Beitollahi, Alex Bie, Sobhan Hemati, Leo Maxime Brunswic, Xu Li, Xi Chen, Guojun Zhang. + +**Abstract:** In one-shot federated learning (FL), clients collaboratively train a global model in a single round of communication. Existing approaches for one-shot FL enhance communication efficiency at the expense of diminished accuracy. This paper introduces FedPFT (Federated Learning with Parametric Feature Transfer), a methodology that harnesses the transferability of foundation models to enhance both accuracy and communication efficiency in one-shot FL. The approach involves transferring per-client parametric models (specifically, Gaussian mixtures) of features extracted from foundation models. Subsequently, each parametric model is employed to generate synthetic features for training a classifier head. Experimental results on eight datasets demonstrate that FedPFT enhances the communication-accuracy frontier in both centralized and decentralized FL scenarios, as well as across diverse data-heterogeneity settings such as covariate shift and task shift, with improvements of up to 20.6%. Additionally, FedPFT adheres to the data minimization principle of FL, as clients do not send real features. We demonstrate that sending real features is vulnerable to potent reconstruction attacks. Moreover, we show that FedPFT is amenable to formal privacy guarantees via differential privacy, demonstrating favourable privacy-accuracy tradeoffs. + + +## About this baseline + +**What’s implemented:** The code in this directory replicates the centralized experiments in *Parametric Feature Transfer, One-shot Federated Learning with Foundation Models* (Beitollahi et al., 2024) for CIFAR100 and Caltech101 datasets, which proposed the FedPFT algorithm. Concretely, it replicates the results in Section 5.2. + +**Datasets:** CIFAR100 and Caltech101 from HuggingFace + +**Hardware Setup:** These experiments were run on a desktop machine with 8 CPU threads and Nvidia 4070 with 8Gigs of ram. + +**Contributors:** Mahdi Beitollahi + + +## Experimental Setup + +**Task:** Image classification + +**Model:** This directory utilize two pre-trained, frozen models as shown in Table 1 of the paper: +* ResNet50 pre-trained on ImageNet is used for CIFAR100 dataset(see `models/resnet50`). +* CLIP, ViT-B/32 pre-trained on web dataset is used for Caltech101 dataset (see `models/clip_vit`) + +**Dataset:** This baseline includes the CIFAR100 and Caltech101 datasets. By default, it will be partitioned into 50 clients following a Dirichlet distribution with $\alpha$=0.1. + +| Dataset | #classes | #partitions | partitioning method | partition settings | +| :------ | :---: | :---: | :---: | :---: | +| CIFAR100 | 100 | 50 | Dirichlet distribution | $\alpha$=0.1 | +| Caltech101 | 101 | 50 | Dirichlet distribution | $\alpha$=0.1 | + +**Training Hyperparameters:** The following table shows the main hyperparameters for this baseline with their default value (i.e. the value used if you run `python main.py` directly) + +| Description | Default Value | +| ----------- | ----- | +| total clients | 50 | +| clients per round | 50 | +| number of rounds | 1 | +| client resources | {'num_cpus': 2.0, 'num_gpus': 0.0 }| +| data partition | distribution with $\alpha$=0.1 | +| Number of mixtures | 2 | +| Covariance type | spherical | +| tolerance | 1e-12 | +| maximum GMM iterations | 1e3 | + + +## Environment Setup + +To construct the Python environment, simply run: + +```bash +# Set directory to use python 3.10 (install with `pyenv install ` if you don't have it) +pyenv local 3.10.12 + +# Tell poetry to use python3.10 +poetry env use 3.10.12 + +# Install +poetry install +``` + + +## Running the Experiments + +To run this FedProx with CIFAR100 baseline, first ensure you have activated your Poetry environment (execute `poetry shell` from this directory), then: + +```bash +python -m fedpft.main # this will run using the default settings in the `conf/config.yaml` + +# you can override settings directly from the command line +python -m fedprox.main dataset=Caltech101 model=clip # will set dataset to Caltech101 and the pre-trained model to Clip-ViT/B32 +``` + +To run using FedAvg: +```bash +# this will use a frozen, pre-trained model and train the classifier head +python -m fedpft.main strategy=FedAvg client=FedAvg + +``` + + +## Expected Results + + +With the following command, we run both FedPFT and FedAvg configurations. + +```bash +python -m fedprox.main --multirun dataset=CIFAR100, Caltech101 + +# FedAvg +python -m fedprox.main --multirun strategy=fedavg client=fedavg dataset=CIFAR100, Caltech101 +``` + +The above commands would generate results that you can plot and would look like the plot shown below. This plot was generated using the jupyter notebook in the `docs/` directory of this baseline after running the `--multirun` commands above. + +![](_static/FedProx_mnist.png) \ No newline at end of file diff --git a/baselines/fedpft/fedpft/__init__.py b/baselines/fedpft/fedpft/__init__.py new file mode 100644 index 00000000000..a5e567b5913 --- /dev/null +++ b/baselines/fedpft/fedpft/__init__.py @@ -0,0 +1 @@ +"""Template baseline package.""" diff --git a/baselines/fedpft/fedpft/client.py b/baselines/fedpft/fedpft/client.py new file mode 100644 index 00000000000..434055808f8 --- /dev/null +++ b/baselines/fedpft/fedpft/client.py @@ -0,0 +1,170 @@ +"""Define your client class and a function to construct such clients. + +Please overwrite `flwr.client.NumPyClient` or `flwr.client.Client` and create a function +to instantiate your client. +""" + +from collections import OrderedDict +from typing import Callable, Dict, List, Tuple + +import flwr as fl +import torch +from flwr.common.typing import NDArrays, Scalar +from hydra.utils import instantiate +from omegaconf import DictConfig +from torch import nn +from torch.utils.data import DataLoader + +from fedpft.models import extract_features, test, train +from fedpft.utils import gmmparam_to_ndarrays, learn_gmm + + +class FedPFTClient(fl.client.NumPyClient): + """Flower FedPFTClient.""" + + def __init__( + self, + trainloader: DataLoader, + testloader: DataLoader, + feature_extractor: torch.nn.Module, + num_classes: int, + device: torch.device, + ) -> None: + """FedPFT client strategy. + + Implementation based on https://arxiv.org/abs/2402.01862 + + Parameters + ---------- + trainloader : DataLoader + Dataset used for learning GMMs + testloader : DataLoader + Dataset used for evaluating `classifier_head` sent from the server + feature_extractor : torch.nn.Module + Model used to extract features of each client + num_classes : int + Number of total classes in the dataset + device : torch.device + Device used to extract features and evaluate `classifier_head` + """ + self.trainloader = trainloader + self.testloader = testloader + self.feature_extractor = feature_extractor + self.classifier_head = nn.Linear( + feature_extractor.hidden_dimension, num_classes + ) + self.device = device + + def get_parameters(self, config) -> NDArrays: + """Return the parameters of the `classifier_head`.""" + return [ + val.cpu().numpy() for _, val in self.classifier_head.state_dict().items() + ] + + def set_parameters(self, parameters: NDArrays) -> None: + """Set the parameters of the `classifier_head`.""" + params_dict = zip(self.classifier_head.state_dict().keys(), parameters) + state_dict = OrderedDict({k: torch.tensor(v) for k, v in params_dict}) + self.classifier_head.load_state_dict(state_dict, strict=True) + + def fit( + self, parameters: NDArrays, config: Dict[str, Scalar] + ) -> Tuple[NDArrays, int, Dict]: + """Fit a GMM on features and return GMM parameters.""" + # Extracting features + features, labels = extract_features( + dataloader=self.trainloader, + feature_extractor=self.feature_extractor, + device=self.device, + ) + + # Learning GMM + gmm_list = learn_gmm( + features=features, + labels=labels, + n_mixtures=int(config["n_mixtures"]), + cov_type=config["cov_type"], + seed=int(config["seed"]), + tol=float(config["tol"]), + max_iter=int(config["max_iter"]), + ) + + # Reshaping GMM parameters into an NDArray + return [array for gmm in gmm_list for array in gmmparam_to_ndarrays(gmm)], 0, {} + + def evaluate( + self, parameters: NDArrays, config: Dict[str, Scalar] + ) -> Tuple[float, int, Dict]: + """Evaluate `classifier_head` on the test data.""" + self.set_parameters(parameters) + loss, acc = test( + classifier_head=self.classifier_head, + dataloader=self.testloader, + feature_extractor=self.feature_extractor, + device=self.device, + ) + return loss, len(self.testloader.dataset), {"accuracy": acc} + + +class FedAvgClient(FedPFTClient): + """Flower FedAvgClient.""" + + def fit( + self, parameters: NDArrays, config: Dict[str, Scalar] + ) -> Tuple[NDArrays, int, Dict]: + """Train the classifier head.""" + self.set_parameters(parameters) + + # train classifier head + opt = torch.optim.AdamW( + params=self.classifier_head.parameters(), lr=float(config["lr"]) + ) + train( + classifier_head=self.classifier_head, + dataloader=self.trainloader, + feature_extractor=self.feature_extractor, + device=self.device, + num_epochs=int(config["num_epochs"]), + opt=opt, + ) + return self.get_parameters(config={}), len(self.trainloader.dataset), {} + + +def generate_client_fn( + client_cfg: DictConfig, + trainloaders: List[DataLoader], + testloaders: List[DataLoader], + feature_extractor: torch.nn.Module, + num_classes: int, + device: torch.device, +) -> Callable[[str], fl.client.NumPyClient]: + """Generate the client function that creates the Flower Clients. + + Parameters + ---------- + client_cfg : DictConfig + Type of client + trainloaders : List[DataLoader] + List of train dataloaders for clients + testloaders : List[DataLoader] + List of test dataloaders for clients + feature_extractor : torch.nn.Module + Pre-trained model as the backbone + num_classes : int + Number of classes in the dataset + device : torch.device + Device to load the `feature_extractor` + """ + + def client_fn(cid: str) -> fl.client.NumPyClient: + """Create a FedPFT client.""" + return instantiate( + client_cfg, + trainloader=trainloaders[int(cid)], + testloader=testloaders[int(cid)], + feature_extractor=feature_extractor, + num_classes=num_classes, + device=device, + ) + + return client_fn diff --git a/baselines/fedpft/fedpft/conf/base.yaml b/baselines/fedpft/fedpft/conf/base.yaml new file mode 100644 index 00000000000..ab1477bd696 --- /dev/null +++ b/baselines/fedpft/fedpft/conf/base.yaml @@ -0,0 +1,15 @@ +--- + +num_clients: 2 +dirichlet_alpha: 0.1 +num_rounds: 1 +num_cpus: 1 +num_gpus: 0.1 +batch_size: 64 +device: cuda + +defaults: + - strategy: fedpft + - client: fedpft + - model: resnet50 + - dataset: CIFAR100 diff --git a/baselines/fedpft/fedpft/conf/client/fedavg.yaml b/baselines/fedpft/fedpft/conf/client/fedavg.yaml new file mode 100644 index 00000000000..10fc2b0f922 --- /dev/null +++ b/baselines/fedpft/fedpft/conf/client/fedavg.yaml @@ -0,0 +1,2 @@ +--- +_target_: fedpft.client.FedAvgClient \ No newline at end of file diff --git a/baselines/fedpft/fedpft/conf/client/fedpft.yaml b/baselines/fedpft/fedpft/conf/client/fedpft.yaml new file mode 100644 index 00000000000..6ef0f175976 --- /dev/null +++ b/baselines/fedpft/fedpft/conf/client/fedpft.yaml @@ -0,0 +1,2 @@ +--- +_target_: fedpft.client.FedPFTClient \ No newline at end of file diff --git a/baselines/fedpft/fedpft/conf/dataset/CIFAR100.yaml b/baselines/fedpft/fedpft/conf/dataset/CIFAR100.yaml new file mode 100644 index 00000000000..322c2d80c18 --- /dev/null +++ b/baselines/fedpft/fedpft/conf/dataset/CIFAR100.yaml @@ -0,0 +1,10 @@ +--- +_target_: fedpft.dataset.Dataset +name: cifar100 +dataset: CIFAR100 +num_classes: 100 +image_column_name: img +partition_by: fine_label +num_clients: ${num_clients} +dirichlet_alpha: ${dirichlet_alpha} +batch_size: ${batch_size} \ No newline at end of file diff --git a/baselines/fedpft/fedpft/conf/dataset/Caltech101.yaml b/baselines/fedpft/fedpft/conf/dataset/Caltech101.yaml new file mode 100644 index 00000000000..96dcc50fa8d --- /dev/null +++ b/baselines/fedpft/fedpft/conf/dataset/Caltech101.yaml @@ -0,0 +1,10 @@ +--- +_target_: fedpft.dataset.Dataset +name: caltech101 +dataset: clip-benchmark/wds_vtab-caltech101 +num_classes: 102 +image_column_name: webp +partition_by: cls +num_clients: ${num_clients} +dirichlet_alpha: ${dirichlet_alpha} +batch_size: ${batch_size} \ No newline at end of file diff --git a/baselines/fedpft/fedpft/conf/model/clip.yaml b/baselines/fedpft/fedpft/conf/model/clip.yaml new file mode 100644 index 00000000000..23d350a2347 --- /dev/null +++ b/baselines/fedpft/fedpft/conf/model/clip.yaml @@ -0,0 +1,9 @@ +feature_extractor: + _target_: fedpft.models.clip_vit + name: openai/clip-vit-base-patch32 +transform: + _target_: fedpft.models.transform + mean: [0.48145466, 0.4578275, 0.40821073] + std: [0.26862954, 0.26130258, 0.27577711] +image_input_size: 224 +hidden_dimension: 768 diff --git a/baselines/fedpft/fedpft/conf/model/resnet50.yaml b/baselines/fedpft/fedpft/conf/model/resnet50.yaml new file mode 100644 index 00000000000..260d9151e68 --- /dev/null +++ b/baselines/fedpft/fedpft/conf/model/resnet50.yaml @@ -0,0 +1,8 @@ +feature_extractor: + _target_: fedpft.models.resnet50 +transform: + _target_: fedpft.models.transform + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] +image_input_size: 224 +hidden_dimension: 2048 diff --git a/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml b/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml new file mode 100644 index 00000000000..3a4e290ced2 --- /dev/null +++ b/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml @@ -0,0 +1,12 @@ +--- +_target_: fedpft.strategy.FedAvg +fraction_fit: 1 +fraction_evaluate: 1 +accept_failures: False +on_fit_config_fn: + _target_: fedpft.server.fedavg_get_on_fit_config_fn + lr: 0.001 + num_epochs: 10 +evaluate_metrics_aggregation_fn: + _target_: fedpft.server.weighted_average + _partial_: true \ No newline at end of file diff --git a/baselines/fedpft/fedpft/conf/strategy/fedpft.yaml b/baselines/fedpft/fedpft/conf/strategy/fedpft.yaml new file mode 100644 index 00000000000..c4982074633 --- /dev/null +++ b/baselines/fedpft/fedpft/conf/strategy/fedpft.yaml @@ -0,0 +1,22 @@ +--- +_target_: fedpft.strategy.FedPFT +fraction_fit: 1 +fraction_evaluate: 1 +accept_failures: False +num_classes: ${dataset.num_classes} +feature_dimension: ${model.hidden_dimension} +device: ${device} +server_batch_size: 32 +num_epochs: 1 +server_opt: + lr: 1e-4 +on_fit_config_fn: + _target_: fedpft.server.fedpft_get_on_fit_config_fn + n_mixtures: 2 + cov_type: spherical + seed: 0 + tol: 1e-12 + max_iter: 10000 +evaluate_metrics_aggregation_fn: + _target_: fedpft.server.weighted_average + _partial_: true diff --git a/baselines/fedpft/fedpft/dataset.py b/baselines/fedpft/fedpft/dataset.py new file mode 100644 index 00000000000..733234074ef --- /dev/null +++ b/baselines/fedpft/fedpft/dataset.py @@ -0,0 +1,108 @@ +"""Dataset creation.""" + +from typing import Callable, Dict + +from flwr_datasets.federated_dataset import FederatedDataset +from flwr_datasets.partitioner import DirichletPartitioner +from torch.utils.data import DataLoader +from torchvision import transforms + + +class Dataset: + """Dataset class.""" + + def __init__( + self, + dataset: str, + num_clients: int, + batch_size: int, + dirichlet_alpha: float, + partition_by: str, + image_column_name: str, + transform: transforms, + image_input_size: int, + seed: int = 0, + split_size: float = 0.8, + **kwargs, + ) -> None: + """Load the dataset and partition it using dirichlet distribution. + + Parameters + ---------- + dataset : str + Name of dataset to be downloaded from HuggingFace. + num_clients: int + Number of clients. + batch_size: int + Batch size of training and testing dataloaders of clients. + dirichlet_alpha: float + Alpha parameter of Dirichlet distribution. + partition_by: str + Label named used for partitioning the dataset. + image_column_name: str + Column name of image in the dataset. + transform: transforms + Transformation of each batch. + image_input_size: int + Input size of pre-trained model. + seed: int, optional + Seed for partitioning the dataset. Default is 0. + split_size: float, optional + The portion of dataset to be used as training and rest as test. + """ + self.dataset = dataset + self.num_clients = num_clients + self.image_input_size = image_input_size + self.transform = transform + self.batch_size = batch_size + self.dirichlet_alpha = dirichlet_alpha + self.partition_by = partition_by + self.seed = seed + self.split_size = split_size + self.image_column_name = image_column_name + + def get_loaders(self): + """Partition the datasets and return a list of dataloaders.""" + partitioner = DirichletPartitioner( + num_partitions=self.num_clients, + partition_by=self.partition_by, + alpha=self.dirichlet_alpha, + min_partition_size=10, + self_balancing=True, + ) + + fds = FederatedDataset( + dataset=self.dataset, partitioners={"train": partitioner} + ) + # Create train/val for each partition and wrap it into DataLoader + trainloaders, testloaders = [], [] + for partition_id in range(self.num_clients): + partition = fds.load_partition(partition_id) + partition = partition.with_transform(self.apply_batch_transforms()) + partition = partition.train_test_split( + train_size=self.split_size, seed=self.seed + ) + trainloaders.append( + DataLoader(partition["train"], batch_size=self.batch_size) + ) + testloaders.append( + DataLoader(partition["test"], batch_size=self.batch_size) + ) + + return trainloaders, testloaders + + def apply_batch_transforms(self) -> Callable[[Dict], Dict]: + """Apply batch transforms for each batch.""" + + def batch_transform(batch): + batch_img = [ + self.transform( + img.resize((self.image_input_size, self.image_input_size)) + ) + for img in batch[self.image_column_name] + ] + batch_label = list(batch[self.partition_by]) + + return {"img": batch_img, "label": batch_label} + + return batch_transform diff --git a/baselines/fedpft/fedpft/dataset_preparation.py b/baselines/fedpft/fedpft/dataset_preparation.py new file mode 100644 index 00000000000..83a9c5dd9e2 --- /dev/null +++ b/baselines/fedpft/fedpft/dataset_preparation.py @@ -0,0 +1 @@ +"""Handle the dataset partitioning and (optionally) complex downloads.""" diff --git a/baselines/fedpft/fedpft/main.py b/baselines/fedpft/fedpft/main.py new file mode 100644 index 00000000000..9860b1232bf --- /dev/null +++ b/baselines/fedpft/fedpft/main.py @@ -0,0 +1,88 @@ +"""Run FL with frozen, pre-trained models.""" + +import pickle +from pathlib import Path + +import flwr as fl +import hydra +import torch +from hydra.core.hydra_config import HydraConfig +from hydra.utils import instantiate +from omegaconf import DictConfig, OmegaConf + +from fedpft.client import generate_client_fn + + +@hydra.main(config_path="conf", config_name="base", version_base=None) +def main(cfg: DictConfig) -> None: + """Run federated learning with frozen, pre-trained models. + + Parameters + ---------- + cfg : DictConfig + An omegaconf object that stores the hydra config. + """ + # Print Config + print(OmegaConf.to_yaml(cfg)) + + # Set device + device = torch.device(cfg.device) + + # Prepare dataset + trainloaders, testloaders = instantiate( + cfg.dataset, + transform=cfg.model.transform, + image_input_size=cfg.model.image_input_size, + ).get_loaders() + + # Define clients + client_fn = generate_client_fn( + client_cfg=cfg.client, + trainloaders=trainloaders, + testloaders=testloaders, + feature_extractor=instantiate(cfg.model.feature_extractor), + num_classes=cfg.dataset.num_classes, + device=device, + ) + + # Setup strategy + strategy = instantiate(cfg.strategy) + + # Start simulation + history = fl.simulation.start_simulation( + client_fn=client_fn, + num_clients=cfg.num_clients, + config=fl.server.ServerConfig(num_rounds=cfg.num_rounds), + strategy=strategy, + client_resources={"num_cpus": cfg.num_cpus, "num_gpus": cfg.num_gpus}, + ) + + # Save results + accuracy_per_round = history.metrics_distributed["accuracy"] + print(accuracy_per_round) + save_path = HydraConfig.get().runtime.output_dir + + strategy_name = strategy.__class__.__name__ + + def format_variable(x): + return f"{x!r}" if isinstance(x, bytes) else x + + file_suffix: str = ( + f"_{format_variable(strategy_name)}" + f"_{format_variable(cfg.dataset.name)}" + f"_clients={format_variable(cfg.num_clients)}" + f"_rounds={format_variable(cfg.num_rounds)}" + f"_finalacc={format_variable(accuracy_per_round[-1][1]):.2f}" + ) + filename = "results" + file_suffix + ".pkl" + + print(f">>> Saving {filename}") + results_path = Path(save_path) / filename + results = {"history": history} + + with open(str(results_path), "wb") as hist_file: + pickle.dump(results, hist_file, protocol=pickle.HIGHEST_PROTOCOL) + + +if __name__ == "__main__": + main() diff --git a/baselines/fedpft/fedpft/models.py b/baselines/fedpft/fedpft/models.py new file mode 100644 index 00000000000..0514b2f3283 --- /dev/null +++ b/baselines/fedpft/fedpft/models.py @@ -0,0 +1,221 @@ +"""Models, training and eval functions.""" + +import logging +from typing import List, Optional, Tuple + +import numpy as np +import torch +import torch.utils +import torchvision.transforms as transforms +from flwr.common.logger import log +from torch import nn +from torch.utils.data import DataLoader +from torchvision import models +from transformers import CLIPModel + + +def resnet50() -> torch.nn.modules: + """Return ResNet-50 model as feature extractor.""" + resnet50 = models.resnet50(weights=models.ResNet50_Weights.DEFAULT) + + # Remove last layer and flatten outputs + resnet50 = torch.nn.Sequential( + *(list(resnet50.children())[:-1]), torch.nn.Flatten() + ) + + # Set the hidden_dimension + resnet50.hidden_dimension = 2048 + + return resnet50 + + +def clip_vit(name: str) -> torch.nn.modules: + """Return CLIP-ViT as feature extractor. + + Parameters + ---------- + name : str + Name of the CLIP model on transformer library, + e.g. `openai/clip-vit-base-patch32`. + """ + + class ClipVit(nn.Module): + """Wrap outputs to return only pooled outputs.""" + + def __init__(self, vision_model): + super().__init__() + self.vision_model = vision_model + self.hidden_dimension = vision_model.config.hidden_size + + def forward(self, input): + output = self.vision_model(input) + return output[1] # return pooled_output (CLS token) + + vision_model = CLIPModel.from_pretrained(name).vision_model + + return ClipVit(vision_model) + + +def transform(mean: List, std: List) -> transforms.Compose: + """Return `transforms.Compose` function for normalizing images. + + Parameters + ---------- + mean : List + Sequence of means for each channel + std : List + Sequence of standard deviations for each channel. + + Returns + ------- + transforms.Compose + Transform function for normalizing images + """ + tr = transforms.Compose( + [ + transforms.ToTensor(), + transforms.Normalize(mean, std), + ] + ) + return tr + + +def extract_features( + dataloader: DataLoader, feature_extractor: torch.nn.Module, device: torch.device +) -> Tuple[np.array, np.array]: + """Extract features and labels from images using feature extractor. + + Parameters + ---------- + dataloader : DataLoader + Dataloader containing {'img': img, 'label': label} + dicts to be extracted. + feature_extractor : torch.nn.Module + Model for extracting features. + device : torch.device + Device for loading `feature_extractor`. + + Returns + ------- + features : np.array + 2D array containing features extracted from `feature_extractor`. + labels : np.array + 2D array containing labels of `features`. + """ + features, labels = [], [] + for dict in dataloader: + batch_samples = dict["img"].to(device) + batch_label = dict["label"].to(device) + with torch.no_grad(): + feature = feature_extractor(batch_samples) + features.append(feature.cpu().detach().numpy()) + labels.append(batch_label) + + # reshape feauturs and labels into a single numpy array + features = np.concatenate(features, axis=0, dtype=np.float64) + labels = np.concatenate(labels, dtype=int) + + return features, labels + + +def test( + classifier_head: torch.nn.Linear, + dataloader: DataLoader, + feature_extractor: torch.nn.Module, + device: torch.device, +) -> Tuple[float, float]: + """"Evaluates the `classifier_head` on the dataset. + + Parameters + ---------- + classifier_head : torch.nn.Linear + Classifier head model. + dataloader : DataLoader + Dataset used for evaluating `classifier_head` containing + {'img': img, 'label': label} dicts. + feature_extractor : torch.nn.Module + Model used for extracting features from the `dataloader`. + device : torch.device + Device for loading `feature_extractor`. + + Returns + ------- + loss : float + CrossEntropy Loss of `classifier_head` on the dataset. + accuracy : float + Accuracy of `classifier_head` on the dataset. + """ + classifier_head.eval() + feature_extractor.eval() + classifier_head.to(device) + feature_extractor.to(device) + + correct, total, loss = 0, 0, 0 + for dict in dataloader: + samples = dict["img"].to(device) + labels = dict["label"].to(device) + with torch.no_grad(): + feature = feature_extractor(samples) + output = classifier_head(feature) + pred = torch.max(output, 1)[1].data.squeeze() + correct += (pred == labels).sum().item() + total += samples.shape[0] + running_loss = nn.CrossEntropyLoss()(output, labels) + loss += running_loss + + return loss.cpu().item(), correct / total + + +def train( + classifier_head: torch.nn.Linear, + dataloader: DataLoader, + opt: torch.optim.Optimizer, + num_epochs: int, + device: torch.device, + feature_extractor: Optional[torch.nn.Module] = None, + verbose: Optional[bool] = False, +) -> None: + """Trains the `classifier_head`. + + Parameters + ---------- + classifier_head : torch.nn.Linear + Classifier head model. + dataloader : DataLoader + Dataset used for evaluating `classifier_head` + containing {'img': img, 'label': label} dicts. + opt : torch.optim.Optimizer + Optimizer for the `classifier_head`. + num_epochs: int + Number of epochs to train the `classifier_head`. + device : torch.device + Device for loading `feature_extractor`. + feature_extractor : torch.nn.Module, Optional + Model used for extracting features from the `dataloader`, optional. + `verbose` : bool, Optional + Whether or not log the accuracy during the training. Defaults to False. + """ + classifier_head.to(device) + if feature_extractor: + feature_extractor.eval() + feature_extractor.to(device) + + for epoch in range(num_epochs): + correct, total, loss = 0, 0, 0 + for _, dict in enumerate(dataloader): + classifier_head.zero_grad() + samples = dict["img"].to(device) + labels = dict["label"].to(device) + if feature_extractor: + with torch.no_grad(): + samples = feature_extractor(samples) + output = classifier_head(samples) + pred = torch.max(output, 1)[1].data.squeeze() + correct += (pred == labels).sum().item() + total += samples.shape[0] + running_loss = nn.CrossEntropyLoss()(output, labels) + loss += running_loss + running_loss.backward() + opt.step() + if verbose: + log(logging.INFO, f"Epoch:{epoch+1} --- Accuracy: {correct/total}") diff --git a/baselines/fedpft/fedpft/server.py b/baselines/fedpft/fedpft/server.py new file mode 100644 index 00000000000..00d88360e9f --- /dev/null +++ b/baselines/fedpft/fedpft/server.py @@ -0,0 +1,94 @@ +"""Create global evaluation function.""" + +from typing import Callable, Dict, List, Tuple + +from flwr.common import Metrics + + +def fedpft_get_on_fit_config_fn( + n_mixtures: int, cov_type: str, seed: int, tol: float, max_iter: int +) -> Callable[[int], Dict[str, str]]: + """Return a function which returns FedPFT training configurations. + + Parameters + ---------- + n_mixtures : int + Number of mixtures for GMMs + cov_type : str + Type of covariance + seed : int + Seed for learning and sampling from the GMMs + tol : float + Error tolerance for learning GMMs + max_iter : int + Maximum number of iteration for EM algorithm + + Returns + ------- + Callable[[int], Dict[str, str]] + Function to return a config with the `lr` and `num_epochs` + """ + + def fit_config(server_round: int) -> Dict[str, str]: + """Return a configuration for training Gaussian Mixtures.""" + config = { + "n_mixtures": str(n_mixtures), + "cov_type": cov_type, + "seed": str(seed), + "tol": str(tol), + "max_iter": str(max_iter), + } + return config + + return fit_config + + +def fedavg_get_on_fit_config_fn( + lr: float, + num_epochs: int, +) -> Callable[[int], Dict[str, str]]: + """Return a function which returns FedAvg training configurations. + + Parameters + ---------- + lr : float + Client's learning rate + num_epochs : int + Number of epochs for local learning of clients + + Returns + ------- + Callable[[int], Dict[str, str]] + Function to return a config with the `lr` and `num_epochs` + """ + + def fit_config(server_round: int) -> Dict[str, str]: + """Return a configuration number of epochs and learning rate.""" + config = { + "lr": str(lr), + "num_epochs": str(num_epochs), + } + return config + + return fit_config + + +def weighted_average(metrics: List[Tuple[int, Metrics]]) -> Metrics: + """Aggregate with weighted average during evaluation. + + Parameters + ---------- + metrics : List[Tuple[int, Metrics]] + The list of metrics to aggregate. + + Returns + ------- + Metrics + The weighted average metric. + """ + # Multiply accuracy of each client by number of examples used + accuracies = [num_examples * float(m["accuracy"]) for num_examples, m in metrics] + examples = [num_examples for num_examples, _ in metrics] + + # Aggregate and return custom metric (weighted average) + return {"accuracy": int(sum(accuracies)) / int(sum(examples))} diff --git a/baselines/fedpft/fedpft/strategy.py b/baselines/fedpft/fedpft/strategy.py new file mode 100644 index 00000000000..f9546140124 --- /dev/null +++ b/baselines/fedpft/fedpft/strategy.py @@ -0,0 +1,145 @@ +"""FedPFT strategy.""" + +from typing import Dict, List, Optional, Tuple, Union + +import torch +from flwr.common import ( + FitRes, + Parameters, + Scalar, + ndarrays_to_parameters, + parameters_to_ndarrays, +) +from flwr.server.client_proxy import ClientProxy +from flwr.server.strategy import FedAvg +from omegaconf import DictConfig +from sklearn.mixture import GaussianMixture as GMM +from torch.utils.data import DataLoader + +from fedpft.models import train +from fedpft.utils import chunks, ndarrays_to_gmmparam + + +class FedPFT(FedAvg): + """Implementation of FedPFT. + + https://arxiv.org/abs/2402.01862 + Authors: + Mahdi Beitollahi, Alex Bie, Sobhan Hemati, Leo Maxime Brunswic, + Xu Li, Xi Chen, Guojun Zhang. + """ + + def __init__( + self, + *args, + num_classes: int, + feature_dimension: int, + server_opt: DictConfig, + server_batch_size: int, + num_epochs: int, + device: torch.device, + **kwargs, + ) -> None: + """Create FedPFT strategy. + + Parameters + ---------- + num_classes : int + Number of classes in the dataset. + feature_dimension : int + Size of feature embeddings + server_opt : DictConfig + Configuration of server optimizer for training classifier head. + server_batch_size : int + Batch size of synthetic features. + num_epochs : int + Number of epochs to train the classifier head. + + Attributes + ---------- + device : torch.device() + Device to train the classifier head at the server. + """ + super().__init__(*args, **kwargs) + self.num_classes = num_classes + self.feature_dimension = feature_dimension + self.server_opt = server_opt + self.server_batch_size = server_batch_size + self.num_epochs = num_epochs + self.device = device + + def aggregate_fit( + self, + server_round: int, + results: List[Tuple[ClientProxy, FitRes]], + failures: List[Union[Tuple[ClientProxy, FitRes], BaseException]], + ) -> Tuple[Optional[Parameters], Dict[str, Scalar]]: + """Learn a classifier head by generating samples from the GMMs.""" + # Do not aggregate if there are failures. + if not self.accept_failures and failures: + raise Exception("there are failures and failures are not accepted") + + config = self.on_fit_config_fn(server_round) + + # Sample from the GMMs to create synthetic feature dataset + synthetic_features_dataset = [] + for _, fit_res in results: + # Convert byte parameters into ndarrays and GMMParameters + ndarray = parameters_to_ndarrays(fit_res.parameters) + all_gmm_parameters = [ + ndarrays_to_gmmparam(array) for array in chunks(ndarray, 5) + ] + + # Sample from GMM_label pairs to create synthetic features + for gmm_parameter in all_gmm_parameters: + gmm = GMM( + n_components=int(config["n_mixtures"]), + covariance_type=config["cov_type"], + random_state=int(config["seed"]), + tol=float(config["tol"]), + max_iter=int(config["max_iter"]), + ) + # Set values of the GMMs + gmm.means_ = gmm_parameter.means.astype("float32") + gmm.weights_ = gmm_parameter.weights.astype("float32") + gmm.covariances_ = gmm_parameter.covariances.astype("float32") + + # Sample features + syn_features, _ = gmm.sample(gmm_parameter.num_samples) + syn_features = torch.tensor(syn_features, dtype=torch.float32) + gmm_labels = torch.tensor( + [int(gmm_parameter.label)] * int(gmm_parameter.num_samples) + ) + + # Add to train data + synthetic_features_dataset += list(zip(syn_features, gmm_labels)) + + # Train a classifier head + synthetic_features_dataset = [ + {"img": img, "label": label} for img, label in synthetic_features_dataset + ] + synthetic_loader = DataLoader( + synthetic_features_dataset, + batch_size=self.server_batch_size, + shuffle=True, + ) + classifier_head = torch.nn.Linear(self.feature_dimension, self.num_classes) + opt = torch.optim.AdamW( + params=classifier_head.parameters(), lr=self.server_opt.lr + ) + + train( + classifier_head=classifier_head, + dataloader=synthetic_loader, + device=self.device, + num_epochs=self.num_epochs, + opt=opt, + verbose=True, + ) + + # Send the classifier head to clients + classifier_ndarray = [ + val.cpu().numpy() for _, val in classifier_head.state_dict().items() + ] + + return ndarrays_to_parameters(classifier_ndarray), {} diff --git a/baselines/fedpft/fedpft/utils.py b/baselines/fedpft/fedpft/utils.py new file mode 100644 index 00000000000..c1a27c14647 --- /dev/null +++ b/baselines/fedpft/fedpft/utils.py @@ -0,0 +1,102 @@ +"""Utility functions.""" + +from dataclasses import dataclass +from typing import List + +import numpy as np +from flwr.common import NDArrays +from sklearn.mixture import GaussianMixture + + +@dataclass +class GMMParameters: + """GMM parameters.""" + + label: int + means: NDArrays + weights: NDArrays + covariances: NDArrays + num_samples: int + + +def gmmparam_to_ndarrays(gmm: GMMParameters) -> NDArrays: + """Convert gmm object to NumPy ndarrays.""" + return [gmm.label, gmm.means, gmm.weights, gmm.covariances, gmm.num_samples] + + +def ndarrays_to_gmmparam(ndarrays: NDArrays) -> GMMParameters: + """Convert NumPy ndarray to GMM object.""" + return GMMParameters( + label=ndarrays[0], + means=ndarrays[1], + weights=ndarrays[2], + covariances=ndarrays[3], + num_samples=ndarrays[4], + ) + + +def learn_gmm( + features: np.array, + labels: np.array, + n_mixtures: int, + cov_type: str, + seed: int, + tol: float = 1e-12, + max_iter: int = 1000, +) -> List[GMMParameters]: + """Learn a list of 16-bits GMMs for each label. + + Parameters + ---------- + features : np.array + A 2-d array with size (n_samples, feature_dimension) containing + extracted features for all the samples. + labels : np.array + An array with size (n_samples) containing labels associated for + each sample in `features`. + n_mixtures : int + Number of mixtures in each Gaussian Mixture. + cov_type : str + Covariance type of Gaussian Mixtures, e.g. spherical. + seed: int + Seed for learning and sampling from Gaussian Mixtures. + tol: float + Tolerance of Gaussian Mixtures. + max_iter: int + Number of maximum iterations to learn the Gaussian Mixtures. + + Returns + ------- + List[GMMParameters] + Returns a list containing the GMMParameters for each class. + """ + gmm_list = [] + for label in np.unique(labels): + cond_features = features[label == labels] + if ( + len(cond_features) > n_mixtures + ): # number of samples should be larger than `n_mixtures`. + gmm = GaussianMixture( + n_components=n_mixtures, + covariance_type=cov_type, + random_state=seed, + tol=tol, + max_iter=max_iter, + ) + gmm.fit(cond_features) + gmm_list.append( + GMMParameters( + label=label, + means=gmm.means_.astype("float16"), + weights=gmm.weights_.astype("float16"), + covariances=gmm.covariances_.astype("float16"), + num_samples=len(cond_features), + ) + ) + return gmm_list + + +def chunks(lst, n): + """Yield successive n-sized chunks from lst.""" + for i in range(0, len(lst), n): + yield lst[i : i + n] diff --git a/baselines/fedpft/pyproject.toml b/baselines/fedpft/pyproject.toml new file mode 100644 index 00000000000..30e47defbda --- /dev/null +++ b/baselines/fedpft/pyproject.toml @@ -0,0 +1,143 @@ +[build-system] +requires = ["poetry-core>=1.4.0"] +build-backend = "poetry.masonry.api" + +[tool.poetry] +name = "fedpft" # <----- Ensure it matches the name of your baseline directory containing all the source code +version = "1.0.0" +description = "Flower Baselines" +license = "Apache-2.0" +authors = ["The Flower Authors "] +readme = "README.md" +homepage = "https://flower.ai" +repository = "https://github.com/adap/flower" +documentation = "https://flower.ai" +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Scientific/Engineering :: Mathematics", + "Topic :: Software Development", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", + "Typing :: Typed", +] + +[tool.poetry.dependencies] +python = ">=3.8.15, <3.12.0" # don't change this +flwr = { extras = ["simulation"], version = "1.5.0" } +hydra-core = "1.3.2" # don't change this +torch = {url = "https://download.pytorch.org/whl/cu117/torch-1.13.0%2Bcu117-cp310-cp310-linux_x86_64.whl"} +scikit-learn = "1.2.2" +flwr-datasets = "0.1.0" +torchvision = {url = "https://download.pytorch.org/whl/cu117/torchvision-0.14.0%2Bcu117-cp310-cp310-linux_x86_64.whl"} +transformers = "4.39.3" +datasets = "2.18.0" + +[tool.poetry.dev-dependencies] +isort = "==5.13.2" +black = "==24.2.0" +docformatter = "==1.7.5" +mypy = "==1.4.1" +pylint = "==2.8.2" +flake8 = "==3.9.2" +pytest = "==6.2.4" +pytest-watch = "==4.2.0" +ruff = "==0.0.272" +types-requests = "==2.27.7" + +[tool.isort] +line_length = 88 +indent = " " +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true + +[tool.black] +line-length = 88 +target-version = ["py38", "py39", "py310", "py311"] + +[tool.pytest.ini_options] +minversion = "6.2" +addopts = "-qq" +testpaths = [ + "flwr_baselines", +] + +[tool.mypy] +ignore_missing_imports = true +strict = false +plugins = "numpy.typing.mypy_plugin" + +[tool.pylint."MESSAGES CONTROL"] +disable = "bad-continuation,duplicate-code,too-few-public-methods,useless-import-alias" +good-names = "i,j,k,_,x,y,X,Y" +signature-mutators = "hydra.main.main" + +[tool.pylint.typecheck] +generated-members = "numpy.*, torch.*, tensorflow.*" + +[[tool.mypy.overrides]] +module = [ + "importlib.metadata.*", + "importlib_metadata.*", +] +follow_imports = "skip" +follow_imports_for_stubs = true +disallow_untyped_calls = false + +[[tool.mypy.overrides]] +module = "torch.*" +follow_imports = "skip" +follow_imports_for_stubs = true + +[tool.docformatter] +wrap-summaries = 88 +wrap-descriptions = 88 + +[tool.ruff] +target-version = "py38" +line-length = 88 +select = ["D", "E", "F", "W", "B", "ISC", "C4"] +fixable = ["D", "E", "F", "W", "B", "ISC", "C4"] +ignore = ["B024", "B027"] +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".hg", + ".mypy_cache", + ".nox", + ".pants.d", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "venv", + "proto", +] + +[tool.ruff.pydocstyle] +convention = "numpy" From d8214b5e00a6a45a4ad426af562c433948d9fd1e Mon Sep 17 00:00:00 2001 From: Mahdi Date: Sun, 14 Apr 2024 22:41:38 -0400 Subject: [PATCH 13/29] fixd model to gpu bug --- baselines/fedpft/fedpft/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/baselines/fedpft/fedpft/models.py b/baselines/fedpft/fedpft/models.py index 0514b2f3283..7ebc9beed4a 100644 --- a/baselines/fedpft/fedpft/models.py +++ b/baselines/fedpft/fedpft/models.py @@ -102,6 +102,8 @@ def extract_features( labels : np.array 2D array containing labels of `features`. """ + feature_extractor.to(device) + features, labels = [], [] for dict in dataloader: batch_samples = dict["img"].to(device) @@ -109,7 +111,7 @@ def extract_features( with torch.no_grad(): feature = feature_extractor(batch_samples) features.append(feature.cpu().detach().numpy()) - labels.append(batch_label) + labels.append(batch_label.cpu().detach().numpy()) # reshape feauturs and labels into a single numpy array features = np.concatenate(features, axis=0, dtype=np.float64) From 03ca12463b4335ba66e11d2c954b2f760da0ee73 Mon Sep 17 00:00:00 2001 From: Mahdi Date: Sun, 14 Apr 2024 22:42:26 -0400 Subject: [PATCH 14/29] fixed config --- baselines/fedpft/fedpft/conf/base.yaml | 4 ++-- baselines/fedpft/fedpft/conf/strategy/fedavg.yaml | 2 +- baselines/fedpft/fedpft/conf/strategy/fedpft.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/baselines/fedpft/fedpft/conf/base.yaml b/baselines/fedpft/fedpft/conf/base.yaml index ab1477bd696..01b1495c241 100644 --- a/baselines/fedpft/fedpft/conf/base.yaml +++ b/baselines/fedpft/fedpft/conf/base.yaml @@ -1,10 +1,10 @@ --- -num_clients: 2 +num_clients: 50 dirichlet_alpha: 0.1 num_rounds: 1 num_cpus: 1 -num_gpus: 0.1 +num_gpus: 1 batch_size: 64 device: cuda diff --git a/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml b/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml index 3a4e290ced2..5f9e1d9e777 100644 --- a/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml +++ b/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml @@ -6,7 +6,7 @@ accept_failures: False on_fit_config_fn: _target_: fedpft.server.fedavg_get_on_fit_config_fn lr: 0.001 - num_epochs: 10 + num_epochs: 1 evaluate_metrics_aggregation_fn: _target_: fedpft.server.weighted_average _partial_: true \ No newline at end of file diff --git a/baselines/fedpft/fedpft/conf/strategy/fedpft.yaml b/baselines/fedpft/fedpft/conf/strategy/fedpft.yaml index c4982074633..5612193071d 100644 --- a/baselines/fedpft/fedpft/conf/strategy/fedpft.yaml +++ b/baselines/fedpft/fedpft/conf/strategy/fedpft.yaml @@ -7,12 +7,12 @@ num_classes: ${dataset.num_classes} feature_dimension: ${model.hidden_dimension} device: ${device} server_batch_size: 32 -num_epochs: 1 +num_epochs: 50 server_opt: lr: 1e-4 on_fit_config_fn: _target_: fedpft.server.fedpft_get_on_fit_config_fn - n_mixtures: 2 + n_mixtures: 1 cov_type: spherical seed: 0 tol: 1e-12 From 791de9facdcf305ccb1e8a9ad39c667034211dcf Mon Sep 17 00:00:00 2001 From: Mahdi Date: Sun, 14 Apr 2024 22:43:04 -0400 Subject: [PATCH 15/29] added notebook for visualization --- .../fedpft/docs/viz_and_plot_results.ipynb | 204 ++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 baselines/fedpft/docs/viz_and_plot_results.ipynb diff --git a/baselines/fedpft/docs/viz_and_plot_results.ipynb b/baselines/fedpft/docs/viz_and_plot_results.ipynb new file mode 100644 index 00000000000..866ffb5d8c7 --- /dev/null +++ b/baselines/fedpft/docs/viz_and_plot_results.ipynb @@ -0,0 +1,204 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "5e0cf2a9-b782-48de-ac45-128726a26e64", + "metadata": {}, + "outputs": [ + { + "ename": "ModuleNotFoundError", + "evalue": "No module named 'matplotlib'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[2], line 7\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mos\u001b[39;00m\n\u001b[0;32m 6\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[1;32m----> 7\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mmatplotlib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpyplot\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mplt\u001b[39;00m\n", + "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'matplotlib'" + ] + } + ], + "source": [ + "import pickle\n", + "import yaml\n", + "from pathlib import Path\n", + "import os\n", + "\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "7ea3e149-ce6f-4ba0-aa41-e0501a04efe3", + "metadata": {}, + "outputs": [], + "source": [ + "def saveFig(name, fig):\n", + " fig.savefig(\n", + " name,\n", + " dpi=None,\n", + " facecolor=fig.get_facecolor(),\n", + " edgecolor=\"none\",\n", + " orientation=\"portrait\",\n", + " format=\"png\",\n", + " transparent=False,\n", + " bbox_inches=\"tight\",\n", + " pad_inches=0.2,\n", + " metadata=None,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "4b010856-0d99-4d81-8fb0-7a927f10eeaf", + "metadata": {}, + "outputs": [], + "source": [ + "# Update the path belows to the directories containing the results for FedPFT and FedAvg\n", + "path_fedpft_resutls_cifar100 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','16-36-16')\n", + "path_fedpft_resutls_caltech101 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','16-44-20')\n", + "\n", + "path_fedavg_resutls_cifar100 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','18-16-41')\n", + "path_fedavg_resutls_caltech101 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','16-36-16')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2e3e165c-1ce6-4efa-a4e1-1372586e436e", + "metadata": {}, + "outputs": [], + "source": [ + "# load results\n", + "def read_accuracies(path_to_pickle):\n", + " for result in list(Path(path_to_pickle).glob(\"*.pkl\")):\n", + " with open(result, \"rb\") as handle:\n", + " data = pickle.load(handle)\n", + "\n", + " accuracies = data['history'].metrics_distributed['accuracy']\n", + " return accuracies\n" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "77b70c73", + "metadata": {}, + "outputs": [], + "source": [ + "fedpft_cifar = read_accuracies(path_fedpft_resutls_cifar100)\n", + "fedpft_caltech = read_accuracies(path_fedpft_resutls_caltech101)\n", + "fedavg_cifar = read_accuracies(path_fedavg_resutls_cifar100)\n", + "fedavg_caltech = read_accuracies(path_fedavg_resutls_caltech101)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "6f4c87ad", + "metadata": {}, + "outputs": [], + "source": [ + "fedavg_cifar = [(1, 0.06924765515865097),\n", + " (2, 0.1315106765116743),\n", + " (3, 0.16773099181800039),\n", + " (4, 0.1946717222111355),\n", + " (5, 0.2171223308720814),\n", + " (6, 0.2375773298742766),\n", + " (7, 0.2597285970864099),\n", + " (8, 0.276092596288166),\n", + " (9, 0.290560766314109),\n", + " (10, 0.3036320095789264),\n", + " (11, 0.3128118140091798),\n", + " (12, 0.3261823987228098),\n", + " (13, 0.33745759329475156),\n", + " (14, 0.3477349830373179),\n", + " (15, 0.35831171422869684),\n", + " (16, 0.36679305527838757),\n", + " (17, 0.37407703053282776),\n", + " (18, 0.3817601277190182),\n", + " (19, 0.38824585910995807),\n", + " (20, 0.3942326880862103)]" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "id": "e1a678de", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1wAAAD0CAYAAACPQifaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABdAUlEQVR4nO3deVhU1f8H8PcMy7CDOuygyGK4YqHgviSCVi6ZipaifE2zpDLKFFNcE0sj/ZVluVbuZlmmqYjikqilWe4KiijIqjAsAgNzf38QkxODyjDDDPh+Pc88Oueec+9nzoye+cw991yRIAgCiIiIiIiISOvE+g6AiIiIiIiosWLCRUREREREpCNMuIiIiIiIiHSECRcREREREZGOMOEiIiIiIiLSESZcREREREREOsKEi4iIiIiISEeYcBEREREREekIEy4iIiIiIiIdYcJFRERERESkI0y4SC+Sk5Px2muvwdPTE2ZmZrCxsUH37t2xfPly3L9/X1nPw8MDL7zwgkpbkUik9uHk5KRSLy8vD2ZmZhCJRLh06ZLaOMaPH6+yD4lEglatWiE6OholJSXV6m/duhVjxoyBj48PRCIR+vTpU+NrLC0txfTp0+Hi4gJzc3MEBgYiLi5Obd3jx4+jR48esLCwgJOTE9566y0UFhbWuG9N/bfPbGxs0Lt3b+zevVvrx6qSkpKiPN6OHTuqbZ87dy5EIhFycnJqve/jx49j7ty5yMvLq7atT58+aj8nAwYMqFa3Nu8VERERUW0Y6zsAevLs3r0bI0aMgEQiQVhYGNq1a4eysjIcO3YM06ZNw4ULF/D1118/dB/9+/dHWFiYSpm5ubnK8+3btysTsY0bN2LhwoVq9yWRSLB69WoAQH5+Pn766ScsWLAAycnJ2Lhxo0rdL7/8EqdPn0bnzp2Rm5v70BjHjx+P77//HlOnToWPjw/Wr1+P5557DocOHUKPHj2U9c6ePYt+/fqhdevWiI2Nxe3bt7F06VJcu3YNv/7660OPoYmqvhMEATdv3sSXX36JQYMG4ddff0VISIjWj/eg+fPnY9iwYRCJRFrZ3/HjxzFv3jyMHz8ednZ21ba7ubkhJiZGpczFxaVavcd9r4iIiIhqTSCqR9evXxesrKwEX19fIT09vdr2a9euCcuWLVM+b9GihfD888+r1AEgTJky5ZHH6tWrlzBs2DDhnXfeEVq2bKm2zrhx4wRLS0uVMoVCIXTp0kUQiURCRkaGyrbU1FShoqJCEARBaNu2rdC7d2+1+z158qQAQFiyZImy7P79+4KXl5fQtWtXlboDBw4UnJ2dhfz8fGXZqlWrBADCvn37Hvk6a0Nd3128eFEAIAwcOFCrx6py48YNAYDQsWNHAYCwY8cOle1z5swRAAjZ2dm13veSJUsEAMKNGzeqbevdu7fQtm3bR+6jNu8VERERUW1xSiHVq48//hiFhYVYs2YNnJ2dq2339vbG22+/XefjpKam4ujRoxg1ahRGjRqFGzdu4Pjx44/VViQSoUePHhAEAdevX1fZ5u7uDrH40f9svv/+exgZGWHSpEnKMjMzM0yYMAGJiYm4desWAEAmkyEuLg5jxoyBjY2Nsm5YWBisrKywbdu2x4q5Llq3bg2pVIrk5GSV8tLSUsyZMwfe3t6QSCRwd3fH+++/j9LSUpV6cXFx6NGjB+zs7GBlZYWnnnoKM2fOrHacUaNGoVWrVpg/fz4EQXhkXCdPnsSAAQNga2sLCwsL9O7dG7/99pty+9y5czFt2jQAQMuWLZVTBlNSUlT2U15e/tDpmY/7XhERERFpglMKqV7t2rULnp6e6NatW532U1JSUu2aH2tra0gkEgDA5s2bYWlpiRdeeAHm5ubw8vLCxo0bH/u4VV/amzRpolF8f/75J1q1aqWSRAFAQEAAgMpphO7u7jh37hzKy8vRqVMnlXqmpqbo2LEj/vzzT42OXxv5+fm4d+8evLy8lGUKhQKDBw/GsWPHMGnSJLRu3Rrnzp3Dp59+iqtXr2Lnzp0AgAsXLuCFF15Ahw4dMH/+fEgkEiQlJakkRlWMjIwwa9YshIWF4ccff8SwYcNqjOngwYMYOHAg/P39MWfOHIjFYqxbtw7PPvssjh49ioCAAAwbNgxXr17F5s2b8emnn0IqlQIA7O3tlfu5evUqLC0tUVZWBkdHR0ycOBHR0dEwMTFR1nnc94qIiIhIE0y4qN7IZDKkpaVhyJAhdd7XmjVrsGbNGpWydevWYfz48QCAjRs3YsiQIcrrukJDQ/H1119j+fLlMDau/rGvSt7y8/Oxc+dO7NixA+3atcNTTz2lUXx37txRewavqiw9PV1Z78Hy/9Y9evSoRsd/mKpkVRAEpKamYtasWaioqMDw4cOVdTZt2oQDBw7g8OHDKtcwtWvXDpMnT8bx48fRrVs3xMXFoaysDL/++qsy4XmYl19+GQsWLMD8+fPx4osvqr2WSxAETJ48GX379sWvv/6qrPPaa6+hbdu2mDVrFvbv348OHTrgmWeewebNmzF06FB4eHio7MfLywt9+/ZF+/btUVRUhO+//x4LFy7E1atXsXXrVmW9x32viIiIiDTBhIvqjUwmA1B5JqquhgwZgoiICJWytm3bAgD+/vtvnDt3TmWxhNGjR2PRokXYt28fnn/+eZV2RUVFKmdFAKBHjx745ptvNF7c4f79+8qzbQ8yMzNTbn/wz5rqPrhio7b8N1k1MTHB+++/j8jISGXZ9u3b0bp1a/j6+qqcSXz22WcBAIcOHUK3bt2UC1X89NNPCA8Pf+R0y6qzXOPGjcPOnTvx4osvVqtz9uxZXLt2DbNmzaq2MEm/fv3w3XffQaFQPPJY/03Ix44di0mTJmHVqlV455130KVLFwCP/14RERERaYIJF9WbqilbBQUFdd6Xm5sbgoKC1G7bsGEDLC0t4enpiaSkJACVX549PDywcePGagmXmZkZdu3aBQC4ffs2Pv74Y2RlZVVb9bA2zM3Nq13rBEC51HzVvqv+rKnuo2LIyMhQeW5ra/vINlXJallZGX7//XcsWrQIxcXFKgnMtWvXcOnSpWqJaJWsrCwAlWcOV69ejVdffRUzZsxAv379MGzYMAwfPrzGhOiVV15RnuUaOnRote3Xrl0DAIwbN67G15Cfn6/RdM93330Xq1atwoEDB5QJ1+O+V0RERESaYMJF9cbGxgYuLi44f/68zo4hCAI2b96MoqIitGnTptr2rKwsFBYWwsrKSllmZGSkkryFhITA19cXr732Gn7++WeN4nB2dkZaWlq18qophFVLk1dNW6sq/29ddUuY//c4D3pwWmVNHkxWn3vuOUilUkRERKBv377K66oUCgXat2+P2NhYtfuouqbJ3NwcR44cwaFDh7B7927s3bsXW7duxbPPPov9+/fDyMioWtuqs1zjx4/HTz/9VG27QqEAACxZsgQdO3ZUe/wH37/aqIr77t27yrLHfa+IiIiINMGEi+rVCy+8gK+//hqJiYno2rWr1vd/+PBh3L59G/Pnz0fr1q1Vtt27dw+TJk3Czp07MWbMmBr34ezsjHfeeQfz5s3DiRMnlGdCaqNjx444dOgQZDKZymIMJ0+eVG4HKq+JMjY2xh9//IGRI0cq65WVleHs2bMqZer89+a8VdMqa+O1117Dp59+ilmzZimvq/Ly8sJff/2Ffv36PXJapVgsRr9+/dCvXz/ExsZi0aJF+OCDD3Do0KEaz0KOGTMGCxcuxLx58zB48GCVbVWLd9jY2NTYvkptp3xWrTr54Jm7x32viIiIiDTBZeGpXr3//vuwtLTEq6++iszMzGrbk5OTsXz5co33XzWdcNq0aRg+fLjKY+LEifDx8al2M2N13nzzTVhYWGDx4sUaxTF8+HBUVFSo3MC5tLQU69atQ2BgoPJMi62tLYKCgrBhwwaVqZbfffcdCgsLMWLEiIceJygoSOWhbvGHRzE2Nsa7776LS5cuKc84jRw5EmlpaVi1alW1+vfv30dRUREA1TNFVaoSFHXT9KpUneU6e/ZstbOI/v7+8PLywtKlS9Uu556dna38u6WlJQAgLy9PpY5MJqt2fEEQlDe/fvAGz4/7XhERERFpgme4qF55eXlh06ZNCA0NRevWrREWFoZ27dqhrKwMx48fx/bt2x85Ja4mpaWl2LFjB/r3769c8OC/Bg8ejOXLlyMrKwsODg417qtZs2YIDw/HF198gUuXLinPlh05cgRHjhwBUPnFv6ioSPklvlevXujVqxcAIDAwECNGjEBUVBSysrLg7e2Nb775BikpKdUWc/jwww/RrVs39O7dG5MmTcLt27fxySefIDg4GAMGDNCoL2pr/PjxiI6OxkcffYShQ4di7Nix2LZtGyZPnoxDhw6he/fuqKiowOXLl7Ft2zbs27cPnTp1wvz583HkyBE8//zzaNGiBbKysvDFF1/Azc1NZXVDdaqu5Tp79qxKuVgsxurVqzFw4EC0bdsW4eHhcHV1RVpaGg4dOgQbGxvlNXf+/v4AgA8++ACjRo2CiYkJBg0ahDNnzmD06NEYPXo0vL29cf/+ffz444/47bffMGnSJDzzzDPK49XmvSIiIiKqNX3edZmeXFevXhUmTpwoeHh4CKampoK1tbXQvXt34bPPPhNKSkqU9Vq0aCE8//zzKm0BCFOmTKm2zx07dggAhDVr1tR43ISEBAGAsHz5ckEQBGHcuHGCpaWl2rrJycmCkZGRMG7cOGXZnDlzBABqH3PmzFFpf//+feG9994TnJycBIlEInTu3FnYu3ev2mMdPXpU6Natm2BmZibY29sLU6ZMEWQyWY2vQ1M19Z0gCMLcuXMFAMKhQ4cEQRCEsrIy4aOPPhLatm0rSCQSoUmTJoK/v78wb948IT8/XxAEQYiPjxeGDBkiuLi4CKampoKLi4swevRo4erVq8r93rhxQwAgLFmypNox161bp+y/7OxslW1//vmnMGzYMKFZs2aCRCIRWrRoIYwcOVKIj49XqbdgwQLB1dVVEIvFAgDhxo0bwvXr14URI0YIHh4egpmZmWBhYSH4+/sLK1euFBQKRbU4avNeEREREdWGSBAEQQ95HhERERERUaPHa7iIiIiIiIh0hAkXERERERGRjjDhIiIiIiIi0hEmXERERI9w5MgRDBo0CC4uLhCJRNi5c+cj2yQkJOCZZ56BRCKBt7c31q9fr/M4iYjI8DDhIiIieoSioiL4+flhxYoVj1X/xo0beP7559G3b1+cPXsWU6dOxauvvop9+/bpOFIiIjI0XKWQiIioFkQiEX788UcMHTq0xjrTp0/H7t27cf78eWXZqFGjkJeXh71799ZDlEREZCh442MACoUC6enpsLa2hkgk0nc4RERPFEEQUFBQABcXF4jFjWPiRWJiIoKCglTKQkJCMHXq1BrblJaWorS0VPlcoVDg7t27aNasGccmIqJ6pO1xyWATrhUrVmDJkiXIyMiAn58fPvvsMwQEBDyy3ZYtWzB69GgMGTLksebYA0B6ejrc3d3rGDEREdXFrVu34Obmpu8wtCIjIwOOjo4qZY6OjpDJZLh//z7Mzc2rtYmJicG8efPqK0QiInoEbY1LBplwbd26FZGRkVi5ciUCAwOxbNkyhISE4MqVK3BwcKixXUpKCt577z307NmzVseztrYGUNmpNjY2tY5XLpdj//79CA4OhomJSa3bNzbsD+1if2oX+1P76tqnMpkM7u7uyv+Ln1RRUVGIjIxUPs/Pz0fz5s1x48aNOvWNXC7HoUOH0LdvX37mH8B+0S32r26xf3Xr7t27aNWqldbGJYNMuGJjYzFx4kSEh4cDAFauXIndu3dj7dq1mDFjhto2FRUVeOWVVzBv3jwcPXoUeXl5j328qqkaNjY2GidcFhYWsLGx4Yce7A9tY39qF/tT+7TVp41p2pyTkxMyMzNVyjIzM2FjY6P27BYASCQSSCSSauVNmzbVaGyqUvX+NGvWjJ/5B7BfdIv9q1vs3/qhrXHJ4BKusrIynD59GlFRUcoysViMoKAgJCYm1thu/vz5cHBwwIQJE3D06NGHHuO/8+RlMhmAyg+vXC6vdcxVbTRp2xixP7SL/ald7E/tq2ufNsb3omvXrtizZ49KWVxcHLp27aqniIiISF8MLuHKyclBRUWF2rnvly9fVtvm2LFjWLNmDc6ePftYx6hpnvz+/fthYWFR65irxMXFady2MWJ/aBf7U7vYn9qnaZ8WFxdrORLtKywsRFJSkvL5jRs3cPbsWTRt2hTNmzdHVFQU0tLS8O233wIAJk+ejM8//xzvv/8+/ve//+HgwYPYtm0bdu/era+XQEREemJwCVdtFRQUYOzYsVi1ahWkUuljtfnvPPmq6weCg4M1nlIYFxeH/v3787Qu2B/axv7ULvZn3eUVy3EjpwjXc4pwI6cYydkFkOVmYf3r/TS+hsvQ/fHHH+jbt6/yedUYMm7cOKxfvx537txBamqqcnvLli2xe/duvPPOO1i+fDnc3NywevVqhISE1HvsRESkXwaXcEmlUhgZGamd++7k5FStfnJyMlJSUjBo0CBlmUKhAAAYGxvjypUr8PLyUmlT0zx5ExOTOn0Bq2v7xob9oV3sT+1ifz6cvEKBW3eLkZxdhOvZhbieXYTrOZV/5haVVatvbSLSuE8bwvvQp08fPOy2levXr1fb5s8//9RhVERE1BAYXMJlamoKf39/xMfHK28qqVAoEB8fj4iIiGr1fX19ce7cOZWyWbNmoaCgAMuXL+dy70RED3G3qAzXswuR/E9SlfxPYpWaW4xyRc0Jxn8VlwOFpeVo0gCSJyIiovpkcAkXUDlVY9y4cejUqRMCAgKwbNkyFBUVKVctDAsLg6urK2JiYmBmZoZ27dqptLezswOAauVERE+qnMJSXMssxLWsApU/1Z2tehgHawk87S3haW8FT6klvBys0NxOgr8TE2AlMcghhYiISK8McnQMDQ1FdnY2oqOjkZGRgY4dO2Lv3r3KhTRSU1O1ctdnIqLGRBAE5BSW4VpmAa5lFeLqP38mZRXibi0SK4mxGC2llvCyt4Kn/b9/tpRawtqs+hksuVyO841nRXciIiKtMsiECwAiIiLUTiEEgISEhIe2VTeXnoioMckvluNCej6u/JNUVSVZecWPv8S6vbUEPg5W8LK3glfVWSt7S7jYmkMsZgZFRESkDQabcBERUaXcwlKcT5fhfFp+5SM9H7fu3n/s9g7WErRytIa3gxVaOVrDx9EKPg5WsLMw1WHUREREBDDhIiIyKJmykn8SKxnOpeXjQno+7uSXPFZbJxuzf5Kpf5MqHwdr2FpwIQsiIiJ9YcJFRKQHgiAgPb8E525XJlWVZ65kyC4ofWRbcxMjtHGxQXtXW/g6WcPnn7NXtuZMrIiIiAwNEy4iIh2TVyiQlFWIi+kyXLojw8V/Ho9zvZWVxBhtXWzQztUW7Vwrk6yWUisY8RorIiKiBoEJFxGRFuUXy3HxzgOJVboMSVmFKKtQPLKtrbkJ2rn+k1y52KKdqy1aNLXgAhZEREQNGBMuIiINCIKAW3fvK89WVZ29Sst7vMUspFaSf6YF2iiTK7cm5hCJmFwRERE1Jky4iIgeQ5asBGdS83D2Vh7+TL2Hi+kyFJSWP7KdWAR42luhjbMNWjvboI2LDVo7W8PB2qweoiYiIiJ9Y8JFRPQfJfIKnE/Lx58PJFjpj7FSoKWpEVo/kFi1cbZBK0drmJsa1UPUREREZIiYcBHRE00QBKTkFuPP1Hv/JFd5uHRHhnKF8NB2zrZmaPNAYtXa2QbNeb0VERER/QcTLiJ6ohSUlONSngjJB5Pxd7oMZ2/lPXK1QAtTI3Rws0VH9ybo6G6Hp5vbwdGGUwKJiIjo0ZhwEVGjVl6hwF+383Dkag6OJeXg7K08VCiMgEvJauuLRIC3vRWebm6Hju5N8HRzO/g4WMHYSFzPkRMREVFjwISLiBoVQRBwI6cIx5JycPRaDk4k5z50cYtmlqbKs1Yd3Zugg7stbMx4A2EiIiLSDiZcRNTg3Ssqw2/JOTh2rTLJetjS7J5SS7gaFeDFXn7o5CGFe1MuxU5ERES6w4SLiBqc0vIKnE65h6NJlUnW+fR8CDWscdHU0hTdvaXo6SNFD28p7C2NsWfPHjzXwRkmJjyTRURERLrFhIuIDJ5CIeBShgzHk3JxLCkHp27cxX15hdq6psZidPZogp4+9ujhLUUbZxuVlQPl8ocvkEFERESkTUy4iMjgCIKA1LvF+C0pF78l5yAxORd3i8pqrN/a2UZ5BiugZVOYmfC+V0RERGQYmHARkUHILijF8eQc5Vmsh12H5WgjQQ9ve/T0kaK7txT21pJ6jJSeVCtWrMCSJUuQkZEBPz8/fPbZZwgICKix/rJly/Dll18iNTUVUqkUw4cPR0xMDMzMeEsBIqInCRMuItKLghI5Tt24W3kWKykHVzILaqxrLTFGF69m6O7VDN29pfB2sOJCF1Svtm7disjISKxcuRKBgYFYtmwZQkJCcOXKFTg4OFSrv2nTJsyYMQNr165Ft27dcPXqVYwfPx4ikQixsbF6eAVERKQvTLiIqF6UlStwJvUejidV3g/rr9v5qFCoX+nC1FiMTi2aoLu3FN28mqG9qy3vg0V6FRsbi4kTJyI8PBwAsHLlSuzevRtr167FjBkzqtU/fvw4unfvjpdffhkA4OHhgdGjR+PkyZP1GjcREekfEy4i0pmM/BIkXMnCoStZ+C0pF4U13A9LJAI6uNqim7cU3b2k6OTRhNdhkcEoKyvD6dOnERUVpSwTi8UICgpCYmKi2jbdunXDhg0bcOrUKQQEBOD69evYs2cPxo4dW+NxSktLUVpaqnwuk8kAVC70UpfFXqracsEYVewX3WL/6hb7V7e03a9MuIhIa8orFDiTmvdPkpWNS3dkNdb1srf85wyWFF09m8HWgku0k2HKyclBRUUFHB0dVcodHR1x+fJltW1efvll5OTkoEePHhAEAeXl5Zg8eTJmzpxZ43FiYmIwb968auX79++HhYVF3V4EgLi4uDrvozFiv+gW+1e32L+6UVxcrNX9MeEiojrJLijF4avZOHQlC0evZkNWov4sVhMLE/RuZY+ePvbo7i2Fky0XDqDGKyEhAYsWLcIXX3yBwMBAJCUl4e2338aCBQswe/ZstW2ioqIQGRmpfC6TyeDu7o7g4GDY2NhoHItcLkdcXBz69+/Pe889gP2iW+xf3WL/6lZubq5W98eEi4hqpUIh4K/beUi4XHkW61xafo11O7jZos9TDuj7lD06uNnBSMyFLqjhkUqlMDIyQmZmpkp5ZmYmnJyc1LaZPXs2xo4di1dffRUA0L59exQVFWHSpEn44IMPIBZXvyZRIpFAIqm+4qaJiYlWvlBpaz+NDftFt9i/usX+1Q1t9ykTLiJ6JFmJHAcvVV6LdeRqNu4Vq5/bbGNmjF6t7NH3KQf0amXP5dqpUTA1NYW/vz/i4+MxdOhQAIBCoUB8fDwiIiLUtikuLq6WVBkZVV6XKAjqF4shIqLGSaOE6+TJkwgMDNR2LERkQIrLynHgUhZ++SsdCVezUVauUFuvjbMN+vrao89TDnja3Y6rCZLe6HJsioyMxLhx49CpUycEBARg2bJlKCoqUq5aGBYWBldXV8TExAAABg0ahNjYWDz99NPKKYWzZ8/GoEGDlIkXERE9GTRKuLp27Yr27dtj4sSJGDNmDOzs7LQcFhHpQ4m8AglXsrHr73QcvJSF+/KKanWsJMbo6SNFn6fs0buVA6/FIoOhy7EpNDQU2dnZiI6ORkZGBjp27Ii9e/cqF9JITU1VOaM1a9YsiEQizJo1C2lpabC3t8egQYPw4Ycfai0mIiJqGDRKuMaMGYMdO3bgrbfewvvvv4/hw4dj4sSJ6Nmzp7bjIyIdKytX4FhSNn756w72X8xUu3S71EqCFzo4I7itIzq1aApTY57FIsOj67EpIiKiximECQkJKs+NjY0xZ84czJkzRyvHJiKihkujb03ffvst0tPT8dlnn8HX1xcbNmxAnz594Ovri08++QQ5OTnajpOItKi8QoFj13Iw/fu/0fnDA/jf+j/ww59pKslWEwsTjA5ojk0TA3FyZj/MHdwW3bykTLbIYHFsIiIiQ6TxNydbW1tMmTIFZ86cwR9//IFJkyYhMzMT06ZNg5ubG0JDQ3HgwAFtxkpEdaBQCDh5PRezd55Hl5h4jFlzElv/uIX8+/8ugGFtZozh/m5YH94Zpz4IQsyw9ujmJeXqgtRgcGwiIiJDo5Wfqp955hl8+eWXSE9Px/r16yGVSvH9998jJCQEnp6e+Pjjj1FQUKCNQxFRLeQUlmLv+QzM/fkCui0+iNCvT+C7EzeRU1imrGNhaoTBfi5YFdYJf8wKwtIRfujzlANMuPgFNXAcm4iIyBBobVn4e/fu4dtvv8Xq1auRnp4OkUiE7t2749KlS5gxYwaWLVuGn376CZ07d9bWIYnoAYIgICW3GL+n3MUfKXfxR8o9XM8pUltXYizGs74OeKGDC571dYC5KVdNo8aJYxMREelbnROuQ4cOYdWqVdi5cydKSkpgb2+PadOm4bXXXoOnpydKS0uxdu1avP/++3jzzTdx4sQJbcRN9MSTVyhwMV32T4J1D3/cvKty5uq/TIxE6OVjj0F+Lghq4wgrCW/DR40XxyYiIjIUGn3jyszMxLp167BmzRpcv34dgiCgd+/emDx5MoYNG6Zyd2aJRILXX38dSUlJWLFixWMfY8WKFViyZAkyMjLg5+eHzz77DAEBAWrr/vDDD1i0aBGSkpIgl8vh4+ODd999F2PHjtXk5REZpMLScpxPycPvKffwR8pd/Jmap3bZ9iomRiJ0cLNDJ48m6NyiKTq3bApbc96Nnhqv+hibiIiIakujhMvNzQ0KhQJNmjTB1KlTMWnSJDz11FMPbWNvb4+yspp/fX/Q1q1bERkZiZUrVyIwMBDLli1DSEgIrly5AgcHh2r1mzZtig8++AC+vr4wNTXFL7/8gvDwcDg4OCAkJESTl0ikd4Ig4M9bedj9Vxr2/W2Ed04chEKoub61mTH8WzRBZ4+m6OzRFB3cbGFmwqmC9OTQ9dhERESkCY0SrsDAQEyePBkjRoyARCJ5rDYzZszAjBkzHqtubGwsJk6ciPDwcADAypUrsXv3bqxdu1btPvr06aPy/O2338Y333yDY8eOMeGiBqUqydrz9x3sOXcH6fkl/2ypvkqgs63ZP8lVE3TyaIpWjtZcTZCeaLoem4iIiDShUcJ17NgxbcehVFZWhtOnTyMqKkpZJhaLERQUhMTExEe2FwQBBw8exJUrV/DRRx+prVNaWorS0lLlc5lMBgCQy+WQy+Vq2zxMVRtN2jZG7I/aEQQBf93Ox94Lmfj1fOYDSda/RBDg42CFTh5N4N+8CTq1sIOLnblKHUVFORQ1zzCkf/DzqX117VNtvRe6HJuIiIg0pVHCdfv2bZw5cwa9evWCnZ1dte337t3D0aNH4e/vD1dX11rtOycnBxUVFXB0dFQpd3R0xOXLl2tsl5+fD1dXV5SWlsLIyAhffPEF+vfvr7ZuTEwM5s2bV618//79sLCwqFW8D4qLi9O4bWPE/qiZIACphcCfuWL8dVeEu6XVz0yJRQKeshXQsZmA9k0EWJrkA8gH0lJwNg04W+9RNy78fGqfpn1aXFyslePrcmwiIiLSlEYJ18KFC7F9+3akp6er3W5hYYH//e9/GDVqFD7//PM6Bfi4rK2tcfbsWRQWFiI+Ph6RkZHw9PSsNt0QAKKiohAZGal8LpPJ4O7ujuDgYNjY2NT62HK5HHFxcejfv7/KRdlPKvaHeoIg4O80GX49n4G9FzKRllf9TJaxWIRuXk0xsJ0TgnwdYGdhwv7UMvan9tW1T6tmGdSVIY5NREREGiVcBw8eRHBwcI1z5CUSCYKDg3HgwIFa71sqlcLIyAiZmZkq5ZmZmXBycqqxnVgshre3NwCgY8eOuHTpEmJiYtQmXBKJRG3sJiYmdfoCVtf2jQ37458k63Y+9py7g93n7uD2vfvV6hiLRejuLcXz7Z0R3NYRdhamavfF/tQu9qf2adqn2nofdDk2ERERaUqjhCstLQ0vvfTSQ+u0aNECu3btqvW+TU1N4e/vj/j4eAwdOhQAoFAoEB8fj4iIiMfej0KhULlOi6g+peYW44c/b+PHP9NwM7f6dCljsQjdvKV44RFJFhE9Pl2OTURERJrSKOEyNTV95BQQmUwGkUizFdMiIyMxbtw4dOrUCQEBAVi2bBmKioqUqxaGhYXB1dUVMTExACqvyerUqRO8vLxQWlqKPXv24LvvvsOXX36p0fGJNCErkePXc3ew43QaTqXcrbbdSHkmywnBbZzQxJJJFpE26XpsIiIi0oRGCVf79u2xa9cuxMbGqp26UVJSgp9//hnt27fXKKjQ0FBkZ2cjOjoaGRkZ6NixI/bu3atcSCM1NRVisVhZv6ioCG+88QZu374Nc3Nz+Pr6YsOGDQgNDdXo+ESPq0Ih4FhSDnacvo19FzJQWq5Q2S4SAd29pBjk58wki0jHdD02ERERaUKjhCs8PBwTJkzA4MGD8eWXX8LT01O5LTk5GW+88QbS09Mxf/58jQOLiIiocQphQkKCyvOFCxdi4cKFGh+LqLauZhZgx+nKKYNZBdWnrno7WOGlZ9zw4tOucLI100OERE+e+hibiIiIakvjhGvPnj3YsWMHfH190bJlS7i6uiItLQ03btxAeXk5QkNDlVMAiRqD3MJS/PxXOnacuY3zadWnLTWxMMFgPxe85O+G9q62nLZEVM84NhERkSHSKOECgG3btmHFihX44osvcPnyZVy7dg0A0KZNG0yZMgWvv/661oIk0pfS8gocupyF70+nIeFKFsoVgsp2Y7EIz/o6YNgzbnjW1wGmxuIa9kRE9YFjExERGRqNEy6RSKSc9ldUVIT8/HzY2trC0tJSm/ER6UVSViE2nLiJnWfTkFcsr7a9g5sthj3tisEdXdGU12URGQyOTUREZGi08nO8paUlXFxcOKBRg1ZeocC+Cxl4ZfUJBMUexvrjKSrJlqONBK/19sT+d3rh54geGN+9JZMtIgOm7bFpxYoV8PDwgJmZGQIDA3Hq1KmH1s/Ly8OUKVPg7OwMiUSCVq1aYc+ePVqJhYiIGg6Nz3ARNRY5haXY+vstbDxxE+n5JSrbJMZiDGjnhJeecUN3bymMxLwui+hJtHXrVkRGRmLlypUIDAzEsmXLEBISgitXrsDBwaFa/bKyMvTv3x8ODg74/vvv4erqips3b8LOzq7+gyciIr3SOOG6desWFi5ciAMHDiA9PR1lZWXV6ohEIpSXl9cpQCJdEAQBZ1Lz8G1iCvacuwN5heq1WS2aWWBslxYY4e8OWwsTPUVJRLWlq7EpNjYWEydOVC64sXLlSuzevRtr167FjBkzqtVfu3Yt7t69i+PHj8PEpPL/EA8Pj9q/ICIiavA0SriuX7+OwMBA3Lt3D23btkVpaSlatGgBMzMzXL9+HXK5HH5+fvwljwzO/bIK/PxXGr5NvIkL6aorDYpEwLNPOWBs1xbo5WMPMc9mETUouhqbysrKcPr0aURFRSnLxGIxgoKCkJiYqLbNzz//jK5du2LKlCn46aefYG9vj5dffhnTp0+HkZGR2jalpaUoLf33NhNVN3GWy+WQy6tfS/q4qtrWZR+NEftFt9i/usX+1S1t96tGCde8efOQn5+P+Ph49O7dG2KxGOHh4YiOjsadO3fw+uuv4+LFizhw4IBWgyXSVEpOETacuIntp28j/77qPyI7CxOEdnLHmC4t4N7UQk8RElFd6WpsysnJQUVFBRwdHVXKHR0dcfnyZbVtrl+/joMHD+KVV17Bnj17kJSUhDfeeANyuRxz5sxR2yYmJgbz5s2rVr5//35YWNT9/6a4uLg676MxYr/oFvtXt9i/ulFcXKzV/WmUcB04cADPPfccevfurSwThMopWc7Ozti6dSvat2+PmTNn4quvvtJOpES1VKEQkHAlC98m3sThq9nVtndws8XYLi0wyM8FZibqf3EmoobDkMYmhUIBBwcHfP311zAyMoK/vz/S0tKwZMmSGhOuqKgoREZGKp/LZDK4u7sjODgYNjY2Gscil8sRFxeH/v37K6c3EvtF19i/usX+1a3c3Fyt7k+jhCsnJwe+vr7/7sTYWCUTlEgk6N+/P3bu3FnnAIlqq7xCgc2/38LXR5Jx6+59lW2mxmK80MEZYV090NHdTj8BEpFO6GpskkqlMDIyQmZmpkp5ZmYmnJyc1LZxdnaGiYmJyvTB1q1bIyMjA2VlZTA1rb7CqUQigUQiqVZuYmKilS9U2tpPY8N+0S32r26xf3VD232qUcIllUpRVFSk8jwlJUV1x8bGyMvLq0tsRLWWcCULH+6+hGtZhSrlrnbmGNOlBUI7u3Mpd6JGSldjk6mpKfz9/REfH4+hQ4cCqDyDFR8fj4iICLVtunfvjk2bNkGhUEAsrrwDy9WrV+Hs7Kw22SIiosZLo4TLx8cHycnJyucBAQHYt28frl+/Dk9PT2RnZ+P777+Hl5eX1gIlephrmQVYuPtStamDvVrZI6xLC/T1deCS7kSNnC7HpsjISIwbNw6dOnVCQEAAli1bhqKiIuWqhWFhYXB1dUVMTAwA4PXXX8fnn3+Ot99+G2+++SauXbuGRYsW4a233tLOiyUiogZDo4Rr4MCBmDt3LvLy8mBnZ4epU6di165d6NChA1q3bo2kpCTIZDLMnTtXy+ESqbpbVIZP465i06lUVCj+Xdr96eZ2mPV8G/i3aKLH6IioPulybAoNDUV2djaio6ORkZGBjh07Yu/evcqFNFJTU5VnsgDA3d0d+/btwzvvvIMOHTrA1dUVb7/9NqZPn66tl0tERA2ERgnX66+/jj59+ijnpvfp0wdbtmzB3Llzcf78ebRo0QILFy7ExIkTtRosUZXS8gp8e/wm/u/gNRSU/Hs/HRdbM0wf6IvBfi4QiXhGi+hJouuxKSIiosYphAkJCdXKunbtihMnTmh0LCIiajw0SrhsbGwQGBioUjZixAiMGDFCK0ER1UQQBOy7kImYXy/hZu6/F8NbmBrhjT5eeLWnJ1ccJHpCcWwiIiJDpFHC9eyzz6J79+5YsGCBtuMhqtH5tHws+OUiTt64qywTiYCR/u54N7gVHGzM9BgdEekbxyYiIjJEGiVcJ0+eRJcuXbQdC5FaWbISLNl3Bd+fuQ3h38u00NWzGWa90BptXWz1FxwRGQyOTUREZIg0Srh8fX1x8+ZNbcdCpOJ+WQVWHb2OlYeTUVxWoSz3aGaBmc+1Rv82jrxOi4iUODYREZEh0ijhevPNNxEREYGLFy+iTZs22o6JnnAKhYCf/0rHR3sv405+ibLcxswYb/XzQVhXD5gaix+yByJ6EnFsIiIiQ6RRwuXp6Yk+ffqgS5cueO2119C5c2c4Oqo/29CrV686B0lPBkEQcPhqNpbuv4LzaTJluZFYhDGBzfF2UCvetJiIasSxiYiIDJFGCVefPn0gEokgCAI++eSTh07rqqioqHEbUZVTN+5i6b4rOJVyV6W871P2+OD51vB2sNZTZETUUHBsIiIiQ6RRwhUdHc1rZ0grzt3Ox9L9V3D4arZKeVsXG0wf4Iterez1FBkRNTQcm4iIyBBplHDNnTtXy2HQkyYpqwCxcVex51yGSrmXvSXeDX4KA9o6QSzmFycienwcm4iIyBBplHARaerW3WIsj7+GH87chuKBJd5d7cwxNcgHLz7tCmMjLohBRERERI0DEy6qF1myEnx+KAmbT6VCXvFvpiW1kuCtft4I7ewOibGRHiMkIiIiItI+jRIusVj8WPPkRSIRysvLNTkENRJ5xWVYefg61h+/gRK5Qllua26Cyb29MK5bC1iYMu8norrj2ERERIZIo2+6vXr1Ujuo5efn49q1aygqKoKfnx/s7OzqGh81UCUVwOeHkrH2t5soKP33i42FqREm9GiJV3t6wtbcRI8RElFjw7GJiIgMkUYJV0JCQo3biouLMWPGDOzduxdxcXGaxkUNVGl5Bb45fhPLzxihqDxZWW5qJMaYLi3wRl8vSK0keoyQiBorjk1ERGSItL46gYWFBf7v//4Ptra2mDZtmrZ3TwZKEATsPZ+B/rFHsOjXKygqr/yV2UgswugAdyRM64PoQW2YbBGRXnBsIiIifdHZxTM9e/bEhg0bdLV7MiAX02VY8MtFJF7PVSl/ob0T3g3xRUuppZ4iIyJSxbGJiIjqm84SruzsbBQWFupq92QAcgpL8cn+q9j6e6rKEu9dWjZBT+tsTBrRASYmvE6LiAwHxyYiIqpvWk+4FAoFNm7ciK1bt6JTp07a3j0ZgLJyBb45noL/i7+msiBG86YWmPlcazzbqil+/fVXPUZIRKSKYxMREemLRgmXp6en2vLy8nJkZWVBLpfDxMQEMTExdQqODIsgCIi/lIUP91zCjZwiZbmVxBgRz3ojvLsHJMZGkMvleoySiJ5UHJuIiMgQabRohkKhgCAI1R4mJiZo164dJk2ahNOnT6N3794aB7ZixQp4eHjAzMwMgYGBOHXqVI11V61ahZ49e6JJkyZo0qQJgoKCHlqfau9qZgHC1p7Cq9/+oUy2RCIgtJM7Dr7XG5N7e/HGxUSkV/UxNhEREdWWRme4UlJStByGqq1btyIyMhIrV65EYGAgli1bhpCQEFy5cgUODg7V6ickJGD06NHo1q0bzMzM8NFHHyE4OBgXLlyAq6urTmNt7O4VleHTA1ex8WQqKh64UCvAoymiB7VBO1dbPUZHRPQvXY9NK1aswJIlS5CRkQE/Pz989tlnCAgIeGS7LVu2YPTo0RgyZAh27typ0xiJiMjw6GzRjLqIjY3FxIkTER4eDgBYuXIldu/ejbVr12LGjBnV6m/cuFHl+erVq7Fjxw7Ex8cjLCysWv3S0lKUlpYqn8tkMgCAXC7XaDpcVZvGNJVOXqHAxlO38NnBZMhK/r1Oy9XODNNDWmFAW0eIRCK1r7kx9oc+sT+1i/2pfXXt04bwXtT2h8AqKSkpeO+999CzZ896jJaIiAyJRgnX7du3cebMGfTq1Qt2dnbVtt+7dw9Hjx6Fv79/rc8wlZWV4fTp04iKilKWicViBAUFITEx8bH2UVxcDLlcjqZNm6rdHhMTg3nz5lUr379/PywsLGoV74May800L94TYedNMTLvi5RlpmIB/V0V6ONcCCH1DH5NffR+Gkt/GAr2p3axP7VP0z4tLi7WyvF1OTbV9odAAKioqMArr7yCefPm4ejRo8jLy6vtSyIiokZAo4Rr4cKF2L59O9LT09Vut7CwwP/+9z+MGjUKn3/+ea32nZOTg4qKCjg6OqqUOzo64vLly4+1j+nTp8PFxQVBQUFqt0dFRSEyMlL5XCaTwd3dHcHBwbCxsalVvEDlr7NxcXHo379/g14G/fa9+5i76xIOX8tRKX+xozPe7e8DRxuzx9pPY+kPQ8H+1C72p/bVtU+rZhnUla7GJk1/CJw/fz4cHBwwYcIEHD169JHH0fbsiyo8q6se+0W32L+6xf7VLW33q0YJ18GDBxEcHAyJRKJ2u0QiQXBwMA4cOFCn4DSxePFibNmyBQkJCTAzU58gSCQStbGbmJjU6QtYXdvriyAI2H76NubvuojCB5Z592/RBNEvtIGfu51G+22o/WGo2J/axf7UPk37VFvvg67GJk1+CDx27BjWrFmDs2fPPvZxdDX7ogrP6qrHftEt9q9usX91Q1szL6polHClpaXhpZdeemidFi1aYNeuXbXet1QqhZGRETIzM1XKMzMz4eTk9NC2S5cuxeLFi3HgwAF06NCh1sd+EuUUliLqh3OIu/hvfzvbmmHGQF8M9nOBSCR6SGsiIsOhy7GpNgoKCjB27FisWrUKUqn0sdtpe/ZFFZ7VVY/9olvsX91i/+pWbm6uVvenUcJlamr6yCkgMplMoy/rpqam8Pf3R3x8PIYOHQqgcqnf+Ph4RERE1Nju448/xocffoh9+/bxppaPad+FDMz84Rxyi8qUZcP93RA9qA1szPiPl4gaFl2NTbX9ITA5ORkpKSkYNGiQskyhUAAAjI2NceXKFXh5eVVrp6vZF9reT2PDftEt9q9usX91Q9t9qtF9uNq3b49du3apzDV/UElJCX7++We0b99eo6AiIyOxatUqfPPNN7h06RJef/11FBUVKS9WDgsLU5lL/9FHH2H27NlYu3YtPDw8kJGRgYyMDBQWFmp0/MZOViLHu9v+wmvfnVYmW80sTfHVWH8sHeHHZIuIGiRdjU0P/hBYpeqHwK5du1ar7+vri3PnzuHs2bPKx+DBg9G3b1+cPXsW7u7utXthRETUoGmUcIWHh+P27dsYPHgwrl+/rrItOTkZQ4YMQXp6Ol599VWNggoNDcXSpUsRHR2Njh074uzZs9i7d69y/nxqairu3LmjrP/ll1+irKwMw4cPh7Ozs/KxdOlSjY7fmB1PzsHAZUex48xtZVn/No7Y904vhLR9+JRNIiJDpsuxqTY/BJqZmaFdu3YqDzs7O1hbW6Ndu3YwNTWt+4slIqIGQ6MpheHh4dizZw927NgBX19ftGzZEq6urkhLS8ONGzdQXl6O0NBQ5UCkiYiIiBqnECYkJKg81/XNLhuDEnkFPt57BWt/u6Ess5IYY86gNhju78ZrtYiowdPl2BQaGors7GxER0cjIyMDHTt2rPZDoFis0W+YRETUyGl84+Nt27ZhxYoV+OKLL3D58mVcu3YNANCmTRtMmTIFr7/+utaCpLo5dzsf72w7i6Ssf6dYdvFsiqUj/ODWpO4rXxERGQpdjk21+SHwv9avX6/xcYmIqGHTOOESiUTKwaeoqAj5+fmwtbWFpaWlNuOjOpBXKPDFoWR8dvAayhUCAMDUWIzpA3wR3s0DYjHPahFR48KxiYiIDI3GCdeDLC0tOZgZmOTsQkRuPYu/bucry9q52uDTkR3h42itx8iIiOoHxyYiIjIEGk04/+233xAZGYmMjAy12+/cuYPIyEicOHGiTsFR7SkUAtb/dgPPLT+qTLaMxCK81c8HP77RnckWETVaHJuIiMgQaZRwxcbGYteuXTXeiNjZ2Rm//PILPv300zoFR7WTnncfY9eexNxdF1FaXnnPF0+pJXa83g2R/VvBxIgXdBNR48WxiYiIDJFGUwp///139OvX76F1evXqhbi4OI2Coto7cT0Xk779A7KScmXZ+G4emD7AF+amRnqMjIiofnBsIiIiQ6RRwpWVlQVXV9eH1nFyckJWVpZGQVHtHLiYiSmbzijPajnbmmHJcD/08JHqOTIiovrDsYmIiAyRRgmXnZ0dUlNTH1rn5s2bsLKy0igoenw7Tt/G+zv+RsU/qxD2ecoey0c9DVtzEz1HRkRUvzg2ERGRIdLoop4uXbrgxx9/xK1bt9RuT01Nxc6dO9GtW7c6BUcPt/bYDby7/S9lsjWkowtWhXViskVETySOTUREZIg0SrgiIyNRXFyM7t2749tvv8WdO3cAVK4A9c0336B79+64f/8+3n33Xa0GS5UEQUDs/iuY/8tFZdm4ri3w6ciOXBiDiJ5YHJuIiMgQaTSlsFevXoiNjcW7776L8PBwAJU3mxSEyjMtYrEYy5cvR69evbQXKQGoXPZ9zs8X8N2Jm8qyt/v5YGqQD0Qi3siY6o9cLkdFRYW+w6g1uVwOY2NjlJSUNMj4DdF/+9TIyAgmJvV/pp1jExERGSKNb3z89ttvo2/fvli5ciV+//135Ofnw87ODgEBAZg8eTLatWuH0tJSSCQSbcb7RCsrV+C97X/h57/SlWVzBrVBePeWeoyKnjQymQw5OTkoLS3VdygaEQQBTk5OuHXrFn+k0BJ1fSqRSCCVSmFjY1OvsXBsIiIiQ6NxwgUAHTp0wBdffFGt/MyZM5gyZQq2bNmC3NzcuhyC/nG/rAKvbzyNhCvZACpvZrx0RAe8+LSbniOjJ4lMJkNaWhqsrKwglUphYmLS4JIWhUKBwsJCWFlZQSzmFFxteLBPRSIR5HI58vPzkZaWBgD1nnRxbCIiIkNSp4TrQXl5ediwYQPWrFmDv//+G4IgwNzcXFu7f6Ll35djwvrf8cfNewAAibEYX7zyDPq1dtRzZPSkycnJgZWVFdzc3BpcolVFoVCgrKwMZmZmTLi05L99am5uDmtra9y+fRs5OTn1nnA9iGMTERHpW50TrgMHDmDNmjX46aefUFpaCkEQ0LVrV4SHhyM0NFQbMT7RsmQlCFt7CpczCgAA1hJjrB7XCYGezfQcGT1p5HI5SktLIZVKG2yyRfVHJBLB1tYWaWlpkMvl9X5NF8cmIiIyFBolXLdu3cK6deuwbt06pKamQhAEuLq6Ii0tDePHj8fatWu1HecTKTW3GGPWnETq3WIAgNTKFOvDA9DO1VbPkdGTqGqBCX0shkANU9VnpaKiol4+NxybiIjIED12wiWXy7Fz506sWbMG8fHxqKiogKWlJV555RWEhYXh2WefhbGxMYyNtTZL8Yl2OUOGsDWnkFVQuTCBq505NrwaiJZSSz1HRk86nt2ix1UfnxWOTUREZOgeewRycXHB3bt3IRKJ0LdvX4SFhWHYsGGwtGQCoG2nb95D+LpTkJWUAwB8HKzw7YQAONvyugMiogdxbCIiIkP32AlXbm4uxGIx3nnnHbz//vuwt7fXZVxPrMNXszH5u9O4L6+cvuXnbof14zujiaWpniMjIjI8HJuIiMjQPfYSXePHj4e5uTliY2Ph5uaGwYMHY/v27SgrK9NlfE+UX/5Ox6vf/K5Mtrp7N8PGVwOZbBER1YBjExERGbrHTrjWrl2LO3fu4KuvvsIzzzyDX375BaNGjYKjoyNee+01HDt2TJdxNnqbTqbizc1/Ql4hAAAGtnPC2vGdYSXhdQdETzKRSIQ+ffroOwyDxbGJiIgMXa1uQmNlZYVXX30ViYmJuHDhAqZOnQpTU1OsWrUKvXv3hkgkwpUrV3Dz5k1dxdsoJSbnYuaP5yBU5loI7eSOz19+BhJjI/0GRkQqUlJSIBKJHvrIy8ur15hu3rwJIyMjiEQiLFmypF6PbSjqa2xasWIFPDw8YGZmhsDAQJw6darGuqtWrULPnj3RpEkTNGnSBEFBQQ+tT0REjZfGd/1s3bo1PvnkE6SlpWHbtm0IDg6GSCTC0aNH4eXlhX79+uG7777TZqyNUom8Ah/8eE75fGLPllj8UnsYibkSHJGh8vLywpw5c9Q+zMzM6jWWtWvXQqFQQCQScdlz6G5s2rp1KyIjIzFnzhycOXMGfn5+CAkJQVZWltr6CQkJGD16NA4dOoTExES4u7sjODgYaWlpdX2JRETUwNR5vpqxsTGGDx+O4cOH4/bt21i3bh3Wr1+PQ4cOISEhAWPHjtVGnI3WFwnJuJ5TBADwb9EEUQNbc9ltIgPn7e2NuXPn6jsMKBQKrF+/HlKpFC+88ALWr1+P48ePo1u3bvoOTe+0PTbFxsZi4sSJCA8PBwCsXLkSu3fvxtq1azFjxoxq9Tdu3KjyfPXq1dixYwfi4+MRFham+QsjIqIGR6sXCLm5uWH27NmYPXs24uPj+WvrIyRlFeDLhCQAgLFYhEUvtoeYZ7aIGoW///4bixYtwuHDh5GbmwtnZ2cMHjwYc+fORbNmzarVX716NZYtW4akpCTY29tj9OjRmD9//kOPERcXh9TUVERERCA0NBTr16/HmjVrVBKuBQsWIDo6Gt98843aL/o//PADXnrpJcycORMffvihSvmiRYtw4cIF2NjYYPDgwfj444/x9NNPA6icXtlQ1HVsKisrw+nTpxEVFaUsE4vFCAoKQmJi4mPto7i4GHK5HE2bNq2xTmlpKUpLS5XPZTIZgMp7jcnl8lrF/KCqtnXZR2PEftEt9q9usX91S9v9qrMVGfr164d+/frpavcNnkIhYOYP55WLZLzW2xNPOVnrOSoi0oaff/4ZI0eOhFgsxpAhQ+Du7o6LFy/i888/x759+3Dy5Ek0adJEWb8qKXJ0dMTEiRNhYmKCrVu34tKlSw89zpo1awAAYWFh6Ny5Mzw9PbFt2zYsX74cVlZWAIAxY8Zgzpw52LBhg9qEq2p63YNnfNauXYsJEybAxsYGYWFhsLW1xZ49e9C/f3/I5XKYmJjUuY/0RZOxKScnBxUVFXB0dFQpd3R0xOXLlx9rH9OnT4eLiwuCgoJqrBMTE4N58+ZVK9+/fz8sLCxqFbM6cXFxdd5HY8R+0S32r26xf3WjuLhYq/vjEnh6su2PWziVchcA0KKZBd581kfPERHVzaDPjiG7oPTRFfXI3lqCn6bUfbpdUlKS2imFAwYMgI+PD8aOHQupVIrffvsNLVq0UG7fsmULRo8ejejoaHz22WfKfc2fPx+urq44c+YMHBwcAABz585FQEBAjTHk5ubip59+gq+vLzp37gygMrmaP38+tm7digkTJgAAWrZsie7du+PgwYO4c+cOnJ2dlfu4e/cu9uzZg06dOsHX1xcAkJeXh7fffhuWlpb4448/4ONT+X/TokWLEBISgtOnT6u8Jnq0xYsXY8uWLUhISHjoNX5RUVGIjIxUPpfJZMprv2xsbDQ+vlwuR1xcHPr379+gk2VtY7/oFvtXt9i/upWbm6vV/THh0oPsglIs2vPvL9cfDm0PMxOuSEgNW3ZBKTJkJfoOo14kJyerPRNhZ2eHxMREyGQyfP7559USk1GjRmHJkiXYsmWLMuHatGkTysvLERkZqUy2AMDGxgazZs2q8Vqj7777DmVlZSrbw8LCMH/+fKxZs0aZcAGVZ6+OHTuGzZs3q3yh37p1K8rKyjBmzBhl2U8//YTCwkK89dZbymQLqLwmauHChU/k9WFSqRRGRkbIzMxUKc/MzISTk9ND2y5duhSLFy/GgQMH0KFDh4fWlUgkkEgk1cpNTEy08oVKW/tpbNgvusX+1S32r25ou0+ZcOnBgl8uQlZSDgB48WlX9PCR6jkiorqzt67+RdHQaCvGkJAQ7N27V+220NBQAMDJkyeRnJxcbXtJSQlycnKQk5MDqVSKv/76CwDQs2fPanXVlVVZs2YNRCKRSrLk5eWFbt264fjx47h06RJat24NABg5ciTeeustfPfddyoJ14YNG2BsbIzRo0cry6ri6dGjR7VjBgYGwtj4yRs2TE1N4e/vj/j4eAwdOhRA5YIl8fHxiIiIqLHdxx9/jA8//BD79u1Dp06d6ilaIiIyNE/eyKlnh69m4+e/0gEAdhYmmPV8az1HRKQdu96s/gXdECkUCp3u/+7dyqnCK1aseGi9oqIiSKVS5OfnA4DK2a0q/71mqMrJkydx/vx59O3bF82bN1fZFhYWhuPHj2Pt2rXK+3LZ2dnhhRdewI4dO3Dx4kW0adMGycnJOH78OJ577jmVY1ct1KAuHrFYDKn0yfyBKDIyEuPGjUOnTp0QEBCAZcuWoaioSLlqYVhYGFxdXRETEwMA+OijjxAdHY1NmzbBw8MDGRkZACrvGVZ1fR0REdWdXC5HRUXFI+sZGRnp7WwgE656dL+sArN2/nvPrZnPtUYzK8M/K0BEj6/qWptz586hXbt2j6xva2sLAMjKyqo2BfG/U9iqVC2WcejQoRpvI/Htt99i0aJFysFl7Nix2LFjB7777jvExMRgw4YNynJ18au7v5RCoUBOTg5cXV0f+boam9DQUGRnZyM6OhoZGRno2LEj9u7dq0yKU1NTIRb/e2vLL7/8EmVlZRg+fLjKfubMmWMQtxQgImroZDIZcnJyVFZ3fRSJRAKpVFqn62I1wYSrHi2Pv4Zbd+8DALp4NsUIfzc9R0RE2hYYGIgffvgBiYmJj5Vw+fn54YcffsDRo0eVi19UOXr0aLX6RUVF2LJlCywsLFSmAj7o999/x99//41ffvkFL774IgDgueeeQ7NmzbBp0yZ8+OGH2LhxI6ytrTFkyJBq8QDAb7/9hhEjRqhsO3XqFMrLyx/5mhqriIiIGqcQJiQkqDxvSMvmExE1NDKZDGlpabCysoJUKoWJiclD72MrCALkcjny8/OVN6Cvz6RL/Ogq9W/FihXw8PCAmZkZAgMDcerUqRrrXrhwAS+99BI8PDwgEomwbNmy+gu0Fi7dkWHV0esAAFMjMT58sT1vcEzUCIWHh8Pa2hoffPABLly4UG17cXExTpw4oXz+8ssvw8jICLGxsSpnlWQyGRYuXFit/fbt21FQUIDhw4dj9erVah9VUwmrzoQBlRcAh4aGIjU1FR9//DGuXbuGl156Cebm5ir7HzJkCKysrLBmzRqVa9DKy8sxe/ZszTuGiIhIS3JycmBlZQU3NzfY2NjA3NwcZmZmNT7Mzc1hY2MDNzc3WFlZIScnp17jNbiEa+vWrYiMjMScOXNw5swZ+Pn5ISQkRO30FqDyy4unpycWL178yNWi9KVCISDqh3OoUFTec+uNvl7wsuccfqLGyN7eHps3b0ZhYSH8/Pzwwgsv4L333sObb76JQYMGwcnJSWVKmbe3N6Kjo5GWloYOHTrgrbfeQmRkJNq3b6+ySmCVqiSq6tohdYKCguDm5oa9e/ciPT1dWV41fTA6Olrl+YPs7OwQGxuLwsJC+Pv7Y/LkyZg+fTqefvpp3Lt3Dy4uLipT54iIiOqTXC5HaWkpbG1ta33yQiQSwdbWFqWlpfV602iDm1IYGxuLiRMnKr9MrFy5Ert378batWsxY8aMavU7d+6snIajbrs6paWlKvM9qy4Sl8vlGnX+o+72veFkKs7eygMAeEot8Wr3Fo36zuC8+7l2GUp/yuVyCIIAhUKh84UndEkQBOWftX0dVfUf1XbgwIE4ffo0li5divj4eMTFxcHS0hJubm4YP348XnnlFZX2s2bNgpOTE5YvX46vvvoKDg4OCA0Nxbx585QLLCgUCly5cgXHjh1Dy5Yt0bNnz4fGEBYWhkWLFmHdunWIiooCAAQEBMDHxwfXrl2Dm5sbevXqpXYfEyZMgK2tLRYvXoz169fD1tYWgwYNwuLFi9GyZUt4eXmptKupTxUKhXIah5FRzbe+0Pdnm4iIGo6qBTI0XQCjql1FRUW9LaJhUAlXWVkZTp8+rfxyAFSuihUUFITExEStHScmJkbtPXT2798PCwsLjfer7m7feaXA4r+MAFRm4M875iN+v/rlpBsb3v1cu/Tdn8bGxnByckJhYSHKysr0Gos2FBQU1LpN06ZNce/ePQD//lBTE2dnZ3zyySc1bv9v+5EjR2LkyJEqZXK5XOV4zs7OyuePin/atGmYNm1atWM9OEW7sLCwxvbBwcEIDg5WKbt+/ToKCwvh6emp9vX/N6aysjLcv38fR44ceei1X8XFxQ99LURERP+l6aU5+rikx6ASrpycHFRUVFRbCtnR0RGXL1/W2nGioqJU7kUjk8ng7u6O4OBgjS6ge9jdviM2n0VpReV0yBH+rnhraNu6Bd8A8O7n2mUo/VlSUoJbt27BysoKZmZmeoujrgRBQEFBAaytrXkdZQ3u3bsHCwsLlZvw3r9/XzkV8aWXXlL5v7KmPi0pKYG5uTl69er10M/Mo5JXIiKihsygEq76IpFIVL5IVKnr3br/2/7AxUzsu1iZbEmtTPHB822eqASEdz/XLn33Z0VFBUQiEcRicYO+hqdqylvVa6Hqjh49igkTJiA4OBjNmzdHTk4ODh48iJSUFDz77LMYPXq0St/V1KdisRgikeiRn13+P0FERI2ZQSVcUqkURkZG1e49k5mZabALYtSkqLQc0T+dVz6f/UIb2FmY6jEiIqLH07ZtW/Tv3x+//fYbdu7cCaBycY8FCxbgvffeY6JKRERUCwaVcJmamsLf3x/x8fEYOnQogMpfTuPj42u894mh+mT/VaTnlwAAevpIMdjPRc8RERE9Hh8fH2zZskXfYRARETUKBpVwAUBkZCTGjRuHTp06ISAgAMuWLUNRUZFy1cKwsDC4uroiJiYGQOVF2RcvXlT+PS0tDWfPnoWVlRW8vb318hrO3c7H+uM3AAASYzEWDm3Ha0WIiIiIiJ5ABpdwhYaGIjs7G9HR0cjIyEDHjh2xd+9e5UIaqampKtNZ0tPT8fTTTyufL126FEuXLkXv3r2RkJBQ3+GjvEKBGT/8jX9uuYW3g3zQopllvcdBRERERET6Z3AJFwBERETUOIXwv0mUh4eH8h4whmD98RRcSK9cccvXyRoTe3rqOSIi7TKkf29k2PhZISIiXdF0jNHH2MQrn7UoLe8+Ptl/FQAgEgGLhrWHiRG7mBqHqhvX8ia19LiqPisPu+kxERFRbdT1+4g+xiZmA1oiCMC8Xy7hvrzy7tdjAlvgmeZN9BwVkfaYmJhAIpEgPz+fZy7okQRBQH5+PiQSCZd9JyIiranL9xF9jU0GOaWwIfrrrgiHruYAABysJZg24Ck9R0SkfVKpFGlpabh9+zZsbW1hYmLS4BaEUSgUKCsrQ0lJCZc315IH+1QkEkEulyM/Px+FhYVwdXXVd3hERNTI1Pb7iCAIeh2bmHBpQUGJHDtu/PvFbd7gtrAx4y+61PjY2NgAAHJycpCWlqbnaDQjCALu378Pc3PzBpcsGip1fSqRSODq6qr8zBAREWmLpt9H9DU2MeHSgqVx1yCTV37J6OfrgAHtGtZNmolqw8bGBjY2NpDL5aioqNB3OLUml8tx5MgR9OrVi1PdtOS/fWpkZMS+JSIinart9xF9jk1MuOro9M172Pz7bQCAhakR5vOeW/SEMDExaZBfqo2MjFBeXg4zM7MGGb8hYp8SEZG+NITvI7yAoY7EIqBlMwsAwNR+3nC1M9dzREREREREZCiYcNXR082b4Ocp3TC8ZQXGBrrrOxwiIiIiIjIgTLi0QGIsRk8nAca85xYRERERET2AGQIREdFjWLFiBTw8PGBmZobAwECcOnXqofW3b98OX19fmJmZoX379tizZ089RUpERIaECRcREdEjbN26FZGRkZgzZw7OnDkDPz8/hISEICsrS23948ePY/To0ZgwYQL+/PNPDB06FEOHDsX58+frOXIiItI3JlxERESPEBsbi4kTJyI8PBxt2rTBypUrYWFhgbVr16qtv3z5cgwYMADTpk1D69atsWDBAjzzzDP4/PPP6zlyIiLSNy4Lj8qbdgKATCbTqL1cLkdxcTFkMpnBL0tZH9gf2sX+1C72p/bVtU+r/u+t+r/Y0JSVleH06dOIiopSlonFYgQFBSExMVFtm8TERERGRqqUhYSEYOfOnTUep7S0FKWlpcrn+fn5AIC7d+9CLpdrHH/V+5Obm8vP/APYL7rF/tUt9q9u3b17F4D2xiUmXAAKCgoAAO7uXGWQiEhfCgoKYGtrq+8wqsnJyUFFRQUcHR1Vyh0dHXH58mW1bTIyMtTWz8jIqPE4MTExmDdvXrXyli1bahA1ERHVVW5urlbGJSZcAFxcXHDr1i1YW1trdNNimUwGd3d33Lp1CzY2NjqIsGFhf2gX+1O72J/aV9c+FQQBBQUFcHFx0UF0DUdUVJTKWTGFQoG7d++iWbNmGo1NVfiZV4/9olvsX91i/+pWfn4+mjdvjqZNm2plf0y4UDk1xM3Nrc77sbGx4Yf+AewP7WJ/ahf7U/vq0qeGeGarilQqhZGRETIzM1XKMzMz4eTkpLaNk5NTreoDgEQigUQiUSmzs7PTLGg1+JlXj/2iW+xf3WL/6pZYrJ3lLrhoBhER0UOYmprC398f8fHxyjKFQoH4+Hh07dpVbZuuXbuq1AeAuLi4GusTEVHjxTNcREREjxAZGYlx48ahU6dOCAgIwLJly1BUVITw8HAAQFhYGFxdXRETEwMAePvtt9G7d2988skneP7557Flyxb88ccf+Prrr/X5MoiISA+YcGmBRCLBnDlzqk0FeVKxP7SL/ald7E/texL6NDQ0FNnZ2YiOjkZGRgY6duyIvXv3KhfGSE1NVZl60q1bN2zatAmzZs3CzJkz4ePjg507d6Jdu3b1HvuT8P5ogv2iW+xf3WL/6pa2+1ckGOo6vERERERERA0cr+EiIiIiIiLSESZcREREREREOsKEi4iIiIiISEeYcBEREREREekIE646OHLkCAYNGgQXFxeIRCLs3LlT3yHpzdy5cyESiVQevr6++g6rQXnU50kQBERHR8PZ2Rnm5uYICgrCtWvX9BNsA/Co/hw/fny1z+yAAQP0E2wDEBMTg86dO8Pa2hoODg4YOnQorly5olKnpKQEU6ZMQbNmzWBlZYWXXnqp2s1/qf5wjFKP45V2cezSLY5lulVfYxsTrjooKiqCn58fVqxYoe9QDELbtm1x584d5ePYsWP6DqlBedTn6eOPP8b//d//YeXKlTh58iQsLS0REhKCkpKSeo60YXicf58DBgxQ+cxu3ry5HiNsWA4fPowpU6bgxIkTiIuLg1wuR3BwMIqKipR13nnnHezatQvbt2/H4cOHkZ6ejmHDhukx6icbx6iacbzSHo5dusWxTLfqbWwTSCsACD/++KO+w9CbOXPmCH5+fvoOo9H47+dJoVAITk5OwpIlS5RleXl5gkQiETZv3qyHCBsWdf8+x40bJwwZMkQv8TQGWVlZAgDh8OHDgiBUfh5NTEyE7du3K+tcunRJACAkJibqK0z6x5M+Rj2I45XucOzSLY5luqersY1nuEhrrl27BhcXF3h6euKVV15BamqqvkNqNG7cuIGMjAwEBQUpy2xtbREYGIjExEQ9RtawJSQkwMHBAU899RRef/115Obm6jukBiM/Px8A0LRpUwDA6dOnIZfLVT6jvr6+aN68OT+jZHA4XtUPjl31g2OZ9uhqbGPCRVoRGBiI9evXY+/evfjyyy9x48YN9OzZEwUFBfoOrVHIyMgAADg6OqqUOzo6KrdR7QwYMADffvst4uPj8dFHH+Hw4cMYOHAgKioq9B2awVMoFJg6dSq6d++Odu3aAaj8jJqamsLOzk6lLj+jZGg4XtUfjl26x7FMe3Q5thlrM1B6cg0cOFD59w4dOiAwMBAtWrTAtm3bMGHCBD1GRqTeqFGjlH9v3749OnToAC8vLyQkJKBfv356jMzwTZkyBefPn+d1L9QgcbyixoRjmfbocmzjGS7SCTs7O7Rq1QpJSUn6DqVRcHJyAoBqq+JkZmYqt1HdeHp6QiqV8jP7CBEREfjll19w6NAhuLm5KcudnJxQVlaGvLw8lfr8jJKh43ilOxy76h/HMs3oemxjwkU6UVhYiOTkZDg7O+s7lEahZcuWcHJyQnx8vLJMJpPh5MmT6Nq1qx4jazxu376N3NxcfmZrIAgCIiIi8OOPP+LgwYNo2bKlynZ/f3+YmJiofEavXLmC1NRUfkbJoHG80h2OXfWPY1nt1NfYximFdVBYWKjyC8KNGzdw9uxZNG3aFM2bN9djZPXvvffew6BBg9CiRQukp6djzpw5MDIywujRo/UdWoPxqM/T1KlTsXDhQvj4+KBly5aYPXs2XFxcMHToUP0FbcAe1p9NmzbFvHnz8NJLL8HJyQnJycl4//334e3tjZCQED1GbbimTJmCTZs24aeffoK1tbVy7rqtrS3Mzc1ha2uLCRMmIDIyEk2bNoWNjQ3efPNNdO3aFV26dNFz9E8mjlHqcbzSLo5dusWxTLfqbWzT8mqKT5RDhw4JAKo9xo0bp+/Q6l1oaKjg7OwsmJqaCq6urkJoaKiQlJSk77AalEd9nhQKhTB79mzB0dFRkEgkQr9+/YQrV67oN2gD9rD+LC4uFoKDgwV7e3vBxMREaNGihTBx4kQhIyND32EbLHV9CUBYt26dss79+/eFN954Q2jSpIlgYWEhvPjii8KdO3f0F/QTjmOUehyvtItjl25xLNOt+hrbRP8cjIiIiIiIiLSM13ARERERERHpCBMuIiIiIiIiHWHCRUREREREpCNMuIiIiIiIiHSECRcREREREZGOMOEiIiIiIiLSESZcREREREREOsKEi4iIiIiISEeYcBFRrYlEIvTp00ffYRAREQHguESGjQkXkQ6kpKRAJBKpPExMTODq6oqRI0fijz/+0HeIRET0BOG4RKQ/xvoOgKgx8/LywpgxYwAARUVFOH36NLZv346dO3fiwIED6NWrl54jJCKiJwnHJaL6x4SLSIe8vb0xd+5clbLFixcjKioKs2fPxuHDh/UTGBERPZE4LhHVP04pJKpnEyZMAACcPn1apTwnJwdTp05Fy5YtIZFI4ODggJEjR+L8+fPV9tGnTx+IRCK1+x8/fjxEIhFSUlKUZevXr4dIJML69euxf/9+dOvWDRYWFmjWrBnGjRuH3NxctftavXo12rVrBzMzM7i7u+P9999HSUmJhq+ciIgMEcclIt3iGS4iPTE2/vefX3Z2Nrp27Yrk5GT06dMHo0aNwo0bN/D9999j9+7d2LdvH3r06FHnY/7888/YvXs3Bg0ahG7duuHIkSP49ttvkZycjGPHjqnUXbBgAaKjo+Ho6IiJEyfCxMQEW7duxaVLl+ocBxERGR6OS0S6wYSLqJ6tXr0aAFQGqunTpyM5ORlRUVFYtGiRsnzPnj14/vnnER4ejitXrkAsrttJ6V27diEhIQHdu3cHAFRUVCAoKAgJCQk4ceIEunTpAgBISkrC/Pnz4erqijNnzsDBwQEAMHfuXAQEBNQpBiIiMiwcl4h0i1MKiXQoKSkJc+fOxdy5czFt2jQ8++yzmDlzJhwdHbFkyRIAQFlZGTZv3oxmzZph1qxZKu2fe+459O/fH0lJSfjtt9/qHM/LL7+sHNQAwMjICOPGjQMA/P7778ryTZs2oby8HJGRkcpBDQBsbGyqxUhERA0HxyWi+sczXEQ6lJycjHnz5qmUOTk54ejRo/D29gYAXL58GSUlJejbty8sLCyq7aNv376Ii4vD2bNn0bNnzzrF4+/vX63Mzc0NAJCXl6cs++uvvwBA7fHqGgMREekPxyWi+sczXEQ6FBISAkEQIAgCsrKysGTJEmRlZWHw4MEoLCwEAMhkMgCAo6Oj2n04Ozur1KsLGxubamVVc/YrKiqUZfn5+QCg8itilZriJCIiw8dxiaj+MeEiqif29vZ47733MHPmTFy6dEk5BaJqsMnMzFTbLiMjQ6UeAOWc+fLy8mr1qwalurC1tQUAZGVlVdtWU5xERNSwcFwiqh9MuIjq2cyZM+Hi4oIvvvgCKSkp8PX1hZmZGX7//XcUFxdXq5+QkAAA6Nixo7KsSZMmAIC0tDSVugqFQjntoi78/PwAAEePHq22TV0ZERE1XByXiHSLCRdRPTM3N8f06dMhl8uxYMECmJqaYvTo0cjJyUFMTIxK3b1792Lfvn3w9vZWuai4c+fOACrvY/Kg2NhY3Lhxo84xvvzyyzAyMkJsbKzKr4kymQwLFy6s8/6JiMhwcFwi0i0mXER6MGnSJLi4uCjvNfLRRx/B09MTCxcuRL9+/TBz5ky8/PLLGDRoECwsLLBu3TqVpXfDw8PRpEkTzJ07Fy+++CLee+899OnTB4sXL0bv3r3rHJ+3tzeio6ORlpaGDh064K233kJkZCTat28PHx+fOu+fiIgMC8clIt1hwkWkB2ZmZoiKikJ5eTnmzZsHe3t7nDx5Em+99RaSk5OxdOlSxMXFYejQoTh58mS1m0s6Ojri0KFD6NevH/bv349Vq1bBzs4OJ06cgIeHh1ZijI6OxqpVq9CsWTN89dVX2L59O0aOHIlt27ZpZf9ERGQ4OC4R6Y5IEARB30EQERERERE1RjzDRUREREREpCNMuIiIiIiIiHSECRcREREREZGOMOEiIiIiIiLSESZcREREREREOsKEi4iIiIiISEeYcBEREREREekIEy4iIiIiIiIdYcJFRERERESkI0y4iIiIiIiIdIQJFxERERERkY4w4SIiIiIiItKR/wcClh2xYZMmhQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def viz():\n", + " fig, axs = plt.subplots(figsize=(10, 2), nrows=1, ncols=2)\n", + " \n", + " # cifar100 - fedavg\n", + " axs[0].plot([r for r, _ in fedavg_cifar], [a for _, a in fedavg_cifar], label='FedAvg', linewidth=2.0)\n", + " \n", + " axs[0].set_title('CIFAR100 - ResNet50')\n", + " \n", + " for ax in axs:\n", + " ax.set_xticks([1, 5, 10 , 15, 20])\n", + " ax.grid()\n", + " ax.legend(fontsize=14, loc='lower right')\n", + " ax.set_xlabel(\"Round\", fontsize=14)\n", + " ax.set_ylabel(\"Accuracy\", fontsize=14)\n", + "\n", + " return fig\n", + "\n", + "f = viz()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "92460065", + "metadata": {}, + "outputs": [], + "source": [ + "saveFig(\"FedProx_mnist.png\", f)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 90bd5ce81d086937bd9b0e4a1839b5de37a9fc09 Mon Sep 17 00:00:00 2001 From: Mahdi Date: Sun, 14 Apr 2024 22:43:34 -0400 Subject: [PATCH 16/29] completed readme file --- baselines/fedpft/README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/baselines/fedpft/README.md b/baselines/fedpft/README.md index 3f045810835..3370fd22d8f 100644 --- a/baselines/fedpft/README.md +++ b/baselines/fedpft/README.md @@ -51,10 +51,10 @@ dataset: [CIFAR100, Caltech101] # list of datasets you include in your baseline. | number of rounds | 1 | | client resources | {'num_cpus': 2.0, 'num_gpus': 0.0 }| | data partition | distribution with $\alpha$=0.1 | -| Number of mixtures | 2 | +| Number of mixtures | 1 | | Covariance type | spherical | | tolerance | 1e-12 | -| maximum GMM iterations | 1e3 | +| maximum EM iterations | 1e3 | ## Environment Setup @@ -98,10 +98,13 @@ python -m fedpft.main strategy=FedAvg client=FedAvg With the following command, we run both FedPFT and FedAvg configurations. ```bash -python -m fedprox.main --multirun dataset=CIFAR100, Caltech101 +# FedPFT +python -m fedprox.main dataset=CIFAR100 model=resnet50 +python -m fedprox.main dataset=Caltech101 model=clip -# FedAvg -python -m fedprox.main --multirun strategy=fedavg client=fedavg dataset=CIFAR100, Caltech101 +# FedAvg with pre-trained, frozen models +python -m fedpft.main strategy=fedavg client=fedavg dataset=CIFAR100 model=resnet50 num_rounds=20 +python -m fedpft.main strategy=fedavg client=fedavg dataset=Caltech101 model=clip num_rounds=20 fedavg.num_epochs=10 fedavg.lr=0.01 num_gpus=0.2 ``` The above commands would generate results that you can plot and would look like the plot shown below. This plot was generated using the jupyter notebook in the `docs/` directory of this baseline after running the `--multirun` commands above. From eee619cec8964f74bd2ea892fad8f6b4683a64bc Mon Sep 17 00:00:00 2001 From: Mahdi Date: Sun, 14 Apr 2024 23:47:09 -0400 Subject: [PATCH 17/29] fixed cofig --- baselines/fedpft/fedpft/conf/strategy/fedavg.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml b/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml index 5f9e1d9e777..166bcd10aef 100644 --- a/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml +++ b/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml @@ -5,8 +5,8 @@ fraction_evaluate: 1 accept_failures: False on_fit_config_fn: _target_: fedpft.server.fedavg_get_on_fit_config_fn - lr: 0.001 - num_epochs: 1 + lr: 0.01 + num_epochs: 10 evaluate_metrics_aggregation_fn: _target_: fedpft.server.weighted_average _partial_: true \ No newline at end of file From c0d45be2a4a6470a17471ac463dff348c22bcf60 Mon Sep 17 00:00:00 2001 From: Mahdi Date: Sun, 14 Apr 2024 23:47:39 -0400 Subject: [PATCH 18/29] fixed readme --- baselines/fedpft/README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/baselines/fedpft/README.md b/baselines/fedpft/README.md index 3370fd22d8f..c25f6de8f71 100644 --- a/baselines/fedpft/README.md +++ b/baselines/fedpft/README.md @@ -75,19 +75,19 @@ poetry install ## Running the Experiments -To run this FedProx with CIFAR100 baseline, first ensure you have activated your Poetry environment (execute `poetry shell` from this directory), then: +To run this FedPFT with CIFAR100 baseline, first ensure you have activated your Poetry environment (execute `poetry shell` from this directory), then: ```bash python -m fedpft.main # this will run using the default settings in the `conf/config.yaml` # you can override settings directly from the command line -python -m fedprox.main dataset=Caltech101 model=clip # will set dataset to Caltech101 and the pre-trained model to Clip-ViT/B32 +python -m fedpft.main dataset=Caltech101 model=clip # will set dataset to Caltech101 and the pre-trained model to Clip-ViT/B32 ``` To run using FedAvg: ```bash # this will use a frozen, pre-trained model and train the classifier head -python -m fedpft.main strategy=FedAvg client=FedAvg +python -m fedpft.main strategy=FedAvg client=FedAvg num_rounds=20 dataset=Caltech101 model=clip num_gpus=0.2 ``` @@ -99,14 +99,14 @@ With the following command, we run both FedPFT and FedAvg configurations. ```bash # FedPFT -python -m fedprox.main dataset=CIFAR100 model=resnet50 -python -m fedprox.main dataset=Caltech101 model=clip +python -m fedpft.main dataset=CIFAR100 model=resnet50 +python -m fedpft.main dataset=Caltech101 model=clip # FedAvg with pre-trained, frozen models -python -m fedpft.main strategy=fedavg client=fedavg dataset=CIFAR100 model=resnet50 num_rounds=20 -python -m fedpft.main strategy=fedavg client=fedavg dataset=Caltech101 model=clip num_rounds=20 fedavg.num_epochs=10 fedavg.lr=0.01 num_gpus=0.2 +python -m fedpft.main strategy=fedavg client=fedavg dataset=CIFAR100 model=resnet50 num_rounds=20 strategy.on_fit_config_fn.num_epochs=1=1 num_gpus=0.5 +python -m fedpft.main strategy=fedavg client=fedavg dataset=Caltech101 model=clip num_rounds=20 num_gpus=0.2 ``` -The above commands would generate results that you can plot and would look like the plot shown below. This plot was generated using the jupyter notebook in the `docs/` directory of this baseline after running the `--multirun` commands above. +The above commands would generate results that you can plot and would look like the plot shown below. This plot was generated using the jupyter notebook in the `docs/` directory of this baseline after running the commands above. -![](_static/FedProx_mnist.png) \ No newline at end of file +![](_static/FedPft.png) \ No newline at end of file From aa4f198b604c21eb92880b8ec89d70c948c34f2f Mon Sep 17 00:00:00 2001 From: Mahdi Date: Mon, 15 Apr 2024 00:12:08 -0400 Subject: [PATCH 19/29] add plots --- baselines/fedpft/_static/FedPft.png | Bin 0 -> 28010 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 baselines/fedpft/_static/FedPft.png diff --git a/baselines/fedpft/_static/FedPft.png b/baselines/fedpft/_static/FedPft.png new file mode 100644 index 0000000000000000000000000000000000000000..76028f4f24b02b424b9bbbb2ffff0a37d260f45c GIT binary patch literal 28010 zcmd43WmHye*EV{Ql2QsH2ndLDmr6<_-QAMXNH-!SAR-DS@|J?VuAAN>5Ue{XdJkL3gdBloRQ;{RYyN-uKp$HY^r8Q6}3>y>* z4F?As{$#@QSv&l8$3sTXL(|#X!^_<5AxhcY!{w2)$0IunI?so0?sm>je4Ij@{OokL z9v&|4ce%J6|K|rdo!xA>$isyD;X`m;r+CD^WF*$06U!iX!=Pnwd_>1_)jqwsPIt9{Jhwob z&A_Ipgsd|JJru8(V4)!>O(^6!3XSwd%YnObE#o(RAtD~HjkMn)a`ZT%DEfc@46%^~ zT#}If_YZr2%U^O%su<}N8PA$VTONy#*6x&A4C2B;m71Z-}Tyf)}Maw z^^vQP^AtgpbZ86(*Q=unLVYDA0!C6?LFcJrTYP-{FlqWhL#1Qi*yrZvuCmx1ckf;& zV^Rvv&%a$NMAWd`e@}5jk&*NRk&TT_?q8E8e_S>; zHhD(U$xp6Qj@0ku;&sL&a&vD@dCwC*dGdsgfkEownQ^_Rc}FBkfoZcCBdJ1`^T(8w zlw?sqK_eq0Ge<{4Q&UrT)&<4~Yd@=Or<+6{J$f{=w)U)0Gk@;y>L}Jmea_B7N=XUF z-N%2Pv_}v*oawpkNV6o!mpf03B`xmcq$DTntp#L?`jMNPn|mMai(~`*UPj+vD6#w)j-p0YlZ~sxG`+&Ci)#=|+(_|4JUVndoGbbk#tE@`%c5H7y zzxLi1OYE7MnF%gyI>xiMw!X$=Asw>Xi5b62(x$Wd9p32Hf zm!+?iU(@c8U7VkA1)gqyzJD^mzM&zT=ya7$e75bxM@K4hE|X$U(^s!v z=`{z6qUi42p}cgNx+PmChWN&fSjUNvEG6~UebowCezklcdonQRmGJBuk8n=YQv{?o$6h1k7 z%ktnma~qrd$_AYh1HDSCD2>#Zn3!JQH|PJBi{awRA}p6)xfz;>h{ymuqjgLZMY%ZY z*5kjcgVlDL-oCzb`|A@}r6=t~;%0LlQQjA42PD+g&%WoYTK*X>lz;G`9X8a}RT&wX z+4{{UsrILYv+L_6yB7TX{8#WPIHk{y|MHl(VScKwXJBE$MLzmuvpF6fj`>)*xqVH# zc#zoA*A&h7^B1sD5?}zG(5Sjno+>s?EX8h+@ce&)7{Ii?h}3uIN4|6AR;8Z zio*GpAzQzpbm4ScC5_q5_-ON$kST8H zVQnt#`p}q|2MH&1XS3z9yz-_2f3KP}2gVXo3BI-Nd&Bx=^+!=eyx4KU@MoXHt(z*D zVzD=+o|<((zxgTXLaaYSv}o`nX%y!AL`?)8&L^+Uj3HhjA)(~tD`(RVn^TS40s@_f z+w)f)b#!#D-ScG6&d%Oi9%MjCC@SJXo8h+Tz!@7KXC3)_a3HtXn^fU<;1p}YNeG*3 zY+{1p?p=dIQ9eGhtgNg8-7*?>4vyKSrKkE;);Ib1H0>HABXO#or)fMrJqwI#d9U-^ z2K($T6;j*4#d*)i3YEk0#0gciGWz{3Q?AUUQSF1h_So3iTUfF~C~H^0w70c!yDurB zB+~Cbz9J>ELsZn$-_Pk0S&$dRM$hZi*(aVV=y-p$)OdDz8Rx-+2mFu5h#f|XS@y2M z9+v*{iUSR0+4W4lforkxys+Rvuy$-r@7axeQ{pUxK<$PBxw~&yo{sxpFH(Y6xKc#mPc}a;rp{1t> z*V>v|*rxk%^k$gH^1w|fM%hp}a>rGn7W4`zYQNV~Zy?zitH3}$>E31ohJ%9x94`u} z&EF3{?GPRQc(2W-?yp;F)Jo;tkfr5nYwM>{5tosrd){M(LqMQa zA6ru+M9ygx`TRLCatk-^dZ199JOVz>7#&#&B?cebzP#q*2|V@o_VKwXC`b;)QO3aF zIy*c2mCtFs5Aoh>7d;&qQ0wUEFzbxIiY#rZaeeeJxt+8`@!Lm-hb@?L7MI?Tzw%R#E8+00i028R2umd|_!weSbav_)inr($e zFNhrc_t#R8JwLngtG;d2G>8N6RTgxWP$ecd?vHIx1q6`_KK`Ld@ud0UoDD@hQfkbG zbAQ9-DV*2B;(N9Obge&s*0#*Q#zdK~ja9T&*?a>qhB-Ue8FL>_x7JgIM?WiNxOsR; zph%TepN)TTz(TcU2A#7&p=CF1YUsWhn|P=vL}G^$cB0A`NXb`x z5wy^b+TGdRB@+=9O&aIFF`}1tAQ%)B)Y{P@z5ZP0=;Mxg?5}z+-SO$YmG4-ZnwqnV zUtZe>mBC$OLkGyq%d>xdM_3p~>M4PGz0&LU>qOMTbSPrj^reHxf1V2T?kbt=e7j7+ zWujFr>b)gVURg;x<+mxC`&j7F7@Z!Vq(&ydR_rL^z{A-~xsMHNoH#z!)vgs@U%cx1xvL%nW!lS^S5O1JLZ+|-q@<;v3`vQlbc$b`hB=IunnOR2 z{u^=mMl@VLwWzPwL({^-LKhd8;IJ?Z*isRceAaJamkGPfNJQN(l~+~OP3!#frMFkH zLS(ZUtE4D4rG00j+#CbW(#-O5Bsqrx9WO6wm37~J7ngF2bWy+kmWc@}6B84_A6g=H zp6hrL)W_%&Jh-1jFfIYoj28KLy7qHA=%PgUN})>2PGsVP@bDpv9MKHM!xSk%Sd+>F&^FCz&=&W0dJx9iy#eSQ7efh;Lh@aK5i`J%F>`}gmM z%FgfNTR1wFxH*!eI`>X@0=`^u*#644h>VHR^fmcl|AW3}WweYoG&B@21e8wB8EAxZ z1_p05=xdQmGWjWsUz)oNjuol}?z|%K%=a0U&}H|fuabb0x#1}jPz#s0bnyEhy09Bm z;~00+oC~m+o=s^(+3k#%#cg_yy<0}AUot~d;+Wx8&K=u9Su$-Unsy=!8 zl%ACp4-lu~7gz~(xHOB-w<~^zIy&S~Z(ctAaWxO-G{tX=o3UIb4a_?J!H0pbFWv~= zQFeQg2Y(|8PZVao+^i5EJ7huNMC;o5LDL6d_0;BD29yd)jlwG=BqXeS7jIHh%wZw* z8hr&UEG=jM{CUqjtavYnhm8&YTc)_K{g>4E)zPxDJxX~o{GI5lY+JJtwAl)YOu2W( zRmTxAX2vnD^H4Xs#FW@GkfT7xZun7YU2I|03`U~);X+Me*EvjNn%dfO zSsVX=52vxS!t%2je1ByLoE`W$*O?YH7$mDK5FGAu{ZB(s^W5{#=|U@Z$2Iiy^V*ZA zWH2>3{iDikmPYTY4vy@Or(Ru1`KS*!enTIyoy$G7A1RJX zOQU9FWNaB6R3FTMJ(QEfSkg!vxHl+wIDK)R?>zb-U%cHWmVf#0=oREsQ3G&RH(|Yx*85Lo$Wy=VvG3PTxxmg^&>v zkc@?^^;^BP1WDiD>0Vp&@#4tVfg?D0e135_Pb3m}dJQ(BR-GvhU0{qbG}g80=H>zX zH-Ocmpw(NphGGc=A2LVqz?I*5iW(p=z|ZI*X)*wYy6l&NyJD%@zkVfv^4f8U*8i;@ z7izN3!)WHCCSYPZ<=bLN3@KL=;WJPkxQQk zdTr0iXQs@90A zs;Zp5Ju9l!w!GUl{#2l)rA5%HY0%k+amGZi=w-zuRx}i_J%!aO zupfH2u_V_k7^{&B4u!RaMJR01d8odh zytlJ^eQ(`i$SVZslClqR9nL?cUl?&F$;LZVU2R%?Dev?XHWxSD{|sr%$-z z&9F9Cpi}ZjG4QqNC1c{FMasT60&IV`6@$ylWXcZ$^CVRt>F41;D1D@qkz<$??llNgIXt89>A z#D8XUtb*aL=NgX0sh}gw8Ois&I7vxK)yhmnPd9@uq=17Hd^GmfqZoP{0dy!C`1{;& z6p_O@?Y~RtyT2A?(&(F1z!mKPLpKgCZtH8)z$kc(mJ8hpV~u`7Oe$%SWVcliWC1&) zwJ%i&nFgV=Ug0pPZfOh0cl|TW0{H0?Ou6#%@-2Ww)QfaDU;=C!u7zyC56JsB90)Sz=8!r4+uzk9v8CgA+&w=mEt^jyHk%|ls&d7h|0 z?asU8X9}7`G$Ji9wWB1|w|d-P*Vo^Zjibhupnl(EJ4g>4fdxG9t=-*r_*)f^S9SdN zf7&RHhTpn*f07s-d04RU%Pm;@@TqAhq(*;oxp|pdI2gKcp=f)a zBl3{oM$sZi=VWAMEdVY7C&vNHTDUu}iM#T+R6d)2vMIn22M323xCu*+ zKiO=)FzN2@Rx8#W&^m_kmi1(^#u?ZasQXeOpGAFka!?XTp~8~vPFFNdhf>lyJgjw1 z!2V^nve=EXIyqb0f}wp_m$qk=e3(Ge0)wbw`?E3^?xl2JFJCp2eEg#$8;riVmoHzw zZ)W!P`HL4CGn?ufSbx*$aP9*yIpomn4Nno@(!836X1Y z@$#Y&JDkC|$;+!@*9apnZfR+0C=8nxtiKWND-4^E6nQ8>E4Oxa$!=|LlfZaW=1K|=AeKv1SOhd<9@wE&QA9P^b0wN@( zwX$Gj#Q_*SqNfU38+NH-M;TonY6m#Dv|>TtpldS%5SC^#OZ- zG;&B$9pb?yC5{y}uN8BlMqPtbjSIEWF~Y`UjCQ>9e3)}Te?RG?s-}j<%+gYxH&{qX$?)ekXeg~qw--vCf$K_xWk zL%vi}%&22zL2-wcFP%6ReW>v^GW0Ii@AeC z(S9){O3%Om$R3Sc7?b|w_9_d&yz#$$jp~sg+}5JyfEWo$0*U?iJVJT~hNN-WaszqF zRH8DV$sBheT9^C-gyk&KEt`WiMs*bK0EG|@>?=BP1miJi4KGeI$;L7 zIBaMM*kfCLLepk2bVE7+{@_$vI>)qO>EGobd%ewxw$5kQ`ojcNY?%mkxP^+Dl>hG% zU3N6RsBz6I8Hsw~KNasi&PWNy8$m1@(V<6Ya`7F1M@zmt8o_t|Q~L@U9;jXdJ?E1j z#qwnBGWz-;gKJ~GSLeDnc5GyU(!Wn&-7GE%ng5KYsyZ{(*oM#VWA3?`@#%bfer_K= z#{Hi{$O>CEBS6TRP2@6~U zz6azJeh1d(veG#=vc_q^iQxehh0+&VR>pM>3e5e{prf5zAPA@fVGqkb7b`T?0^&k1 zvjMCnMna85Occq4g@s-+3CT*zAh`{+8DJN| zzP9$Oo&x6BHYk693Jkgu$Vf;iQ9?pyU&t3fav1*mu!NK*4CX?pW(e&E^e!5z*tnh? zc19Oe0Un!v>ZHFyPLq)^I?MyC0xtFv%G=xf<-=~+4NvC>8O#>s7e6L2_`s^l$3LqS z2418s_rYs!uMH!aR6r*%*vAzVFn^e!7V##ALoNw40Q6bZrAwC(iUU}!r@*$1JM!}J zC8VSjdS|?NfsO(a4gT$nGi*{;_{b6e|2ULKH3QiVRlL4_F^)7eEwNM<_wf1Y=^MFw z=h%S@0|Cy(%}vAJ%gwC~Ak_~i_suC$YVkm7s8*lhfw~-Q7%OFnAj2}S?DjSKl~%aq z?0Rp90@ucPfGvM1d=5L8)Y*`xLcAI|%4@3YY<8J>xw&?CUf{7_cpS;ap@J?r&L>I? zH`wFY2O=YT)6~grZrudM5Xc{3BuRkZf{rKo=n?xXPGbWzS3!6dn3pcGW#s4NwDk26 z0wMGSDLPWh_>##|J#_p5!rny1Lu7{&(O$ zvw3Y^cBV8oiYh59lhD$Z4b#xlA}a2OiQ!^>vZbXZMX76zjg3$cBY@1+)6*+>WNBxI z3$%BVr@Q-ZwzytSKE}w1Jw@TKrssdZ+>1R<cpMy^{K$0T#npU zTH4nfzAii*3*=Q|N=gL2Vi13O27rj3nOS~a5a^(GpxEKw%`E6uG<;6>9pO4O9nE!E zS4YrH1F)#YOZD&j?cpqsUriRk$H5tJVuyXRvSN`jxFBYIgkN2191?V1SZ+#3ny}wv zHFt17H^9o~HD}z~Bi!@zdh+)=NpIb4N>1%_tzULhqTnf74i~7~4rXH+e6SC{%BDjn zAfV0T_YN3vBV*$Vs~%EROO8TfIdp1&n9-AgUf!6j4f`g3K@IKlhm#q6wYld(sr9t* z(izikQCTnvLozBVQog>pq>~FJa5krA^0ud0*vf>S*I;F(Wqo4KUtGu;y~YVGxABQC zaYtK0!5jPnenH$WQ;FDhaaJ6>k^njvafAEIv<6Q!DX^X5@RGPQU=ro6Ot z9SN~o0~hMoq<7b|?np+a=+4&-IEfcjWDMk*R@Bcv3S$Jneft(Yf(KfnMs5jwA~NyE zy}RFPxxZ^dq&qMc1a-rh#VM<^?OFHRg$301wjd7C>zV!=i}dU~>dh4jjKQ@&gGBiF zeIwr&%&nPdYH|?%51?zdY4cN*QnXPs`?bfe)&s*=Z4(maD#ykqSEBWy{E(1n3x}d| z@}gyuPPf-mjx!g5R1ykP8j+f}gl6yU6IH{vx6>N{>vVkB_w^-yF1i!dqk$X^$~fxU z_zcm;zc-@6h+*7o`u6VzA-g{P`Ik1tj8ULuWp%~>Ow*30^OueNd$+-)1og>j=`%Ze z3W`N~rSr0w2P^&T>OU=A^BjEetCO!)a4+Y#2?3~o+rbL@W8t61nh?B8*qB>QZvDg(t?xP#-^1 z+skA9yK|d|d*>H1grXuHg^&4biLOIMPtq>ZF<%+V1=dpvX}Lhsyn{c&Ds2YE8Q>!} z5v|x0s2L!DMc#Vws-?I0J;6IB%a>^cM$Md4+cTL#9ieOE0(7@zrW00V;ap{!_fDI9 zV2z>U6}!vP-=8T?1Z;&dO$KAX#e&`vsLI^N5dadl5NEZ)Q$b@-xF||iiOqJW3cK9vWz_MZdm{E zKB|H&$?I-FN{!ujY|vsPfq;>Hb`1??Lt~o$@`C#Kl&UJdwA#A&%|zFse)7`uQf{5z zdH$SEY;GR^;s};@xcDbcd&H`clkrmC|F#XnAdGlv5Mu#Yn}~>j;=TfHanwM`N(2)v zGDefL<^1T<;ikB&j)d}XDIvcj9={#>a9O6IOV0(z>dnV<0vf;1FRrt+e~Q17aPzyJ zt02h1YZEm@P?|bnow$KQ!oUx!7&imxCLJA1A$d-?XM-m`4$W#LF_In)LNj-@~CiFM_XxD|v2MQ8`vzr{c`#A&FI$K9yBOR=$`m8Xf zIZP#{Dl_mH7jz9#a34g2!fpmvGX};MpvdBbgHg|}3*hbV@7EDzt>xxp7}wi-(pH^f z8H|?B>00fu_(Vo3Di8-#aPi-(N_ze_)j|8c?j`CQXmWCtl~rWV6X$W<-PdX5G~o%r z;0Y8;NMd55Mtx!;83=@ho~giA-vmJLUrK#W8eqvb##o)wPF||zs*V+iUb(ohN^kow6>rRi+R8*BJyH( z;~eYlu>DE)p`i2-)*8kNxP4-f_eL_Y)6n(k@aFR8Y z4pn9nC8KSWcF4SD;}}$7V}6tCBkK0;#h~4bwHcJmO{UAvbvT-uXJ02@z0O+5OT4W* z2Xe<}0Q|G7t1v<2pg_rI*6GvZvntRteTpb!FVI5Gtp8ctYzrb|+U z#}Ps!>f2(Jsa=J}>}U!d$fnip+Y>a9DkTlxSlhIxrMrK7DqzoezPcWJkxAW!7p$pX z&n^6)yPwEpxqB#ErNmX@5A$d__>_RsC-B=}Yi;n^E%(|qL9i(>TgfnXFe$%z0`l|A zhkD}_#HeT2vV5`@m%8|@r7=^Ks96seqIRTAY!^8isztFrrW?=7=uG`ybj0R&Tx9a| z^Gkm7hF(m}q)=2$jHbW8-@fK(@4G4wkco(pgF>_fkS-8j>u@1a)yyK>H#QEeZR{M{ z5fN`vQ(ymSH?XVWQjPu-^Hs@pXz_IPsc69apXVhdA+oVSVZ=D5HFbP{kih>i$?+9g!C z6|eMvKo@K)R;=X0LT^ibN#g3=FX~Z?jW&8y^lKv=eWf)onPbU%ii1F18` zdKKsZ>7##Aot>S5`Vpc9$IXg;$J zT^)__`d0quhkFc{&&`nn61Cb&CrhU69hv-2eSd!u;LV>v8G+f%j~EjT{3dP4nFf_E zS2F}`a>@L5q#IL>G0^)~c-iT*Equ-l0%mVr+~F+sIC(AKQKiet`GEwd|FSProEBIcpdsV{$%D2#0N0C9k>eMg3LGf} zkTEw9LW)xHTqZg;K*riT4=YILM7_K+a~|NDY6^R_XKK+gTO`u_6I`HI=}$%hxZCSFL*HL7_VbpHC4@c>2#r^^Cv&%+canh zSAx#|QXTIHq<+8pn(wRf)z6K7E#n_rx?O(9CeY}iz}@}(CgM=K%N*>gOD7OpBUmrx|}a#n&K^2F4fgz}0cDnnG0qi#W5+?R2R zs^AklM<;#JTl8WBfxvglb)P4-0vVamzvrQWs$X+*6lK$Myf@o77S;U;o`Azp4lZ2W z6DSmgH^g2fB!qw_h)C0Q1YwbQTgL(}yG$_Kk*)S^kt>q0~lo=43NOXllUpk+gI!Y@tVAlX(zp9wG-J;5Z{A zBl_Xs;2`26hB6)sHi1991lf38EDE)##Ea8q_3Z=g#yUM)AKxW%^PM1vQEYbhYFhCG zK^`H>n%k-!xT5=;-yEx|FkW#{*qAQhs?zgX-{tC$qrMXgE*eB`e*9-R1O`4>1J{dl zm~U+?q4Zqivwj9^`yMKpykg~js~pT4XEq9RpJsi|=p=4Nc92w6tf4d`X&b%)mMvL-7ZXd;3j! z8ueqtf(wtig_nlGiyR$k=|>??Svi-O=IIinY&w_vwVN(`cHo$G;=Pq6Y=l0OLk5p( z2UucKGFo-9sHm1CwwAxEy)t_DLzhmT%cH!!(0&x%erqY`$8T@-p1I?__IKko=|wd6 zMDx1G57N*tzKOd}*#AHeJVaamWf#%^m1{`z%tKZ-1btNF@iguJwt$x4H; z-`+9_II*7rj=iCpAtI@x^Q!kQG=`r)f7V?NFZhZRs^_uD)iF26Khb2Zj7+Kp+C?*~ ztvMR&ovK<>uy%RP7uaH_YW~eHiwr6BnrW6N^_|Uu+NXq-1F#+j9H2>qHvo?7;PWH{ zPz1za8%&RY!v>4>`Nmw2g?KG^Q@n5Jo!?(O+Rd&h(ZKTFRh}uS3AXO*aA>>Ek9x23 zBVbbc>Q(o|*#yOqaQqOiCgN{tXmb?m-}7VE1qF{|<0A`-;e;827aPPM5+HMcod&}e zCUAa|^71$tVgb>xQ)Z#$v;zrH3iYA-J`>rur)MW+JM&$HbQ~qi>qXia{{9{J;T8>F z^D&Sn2`n+S)8owjtSm8fpSUi7F6e}5@L|XlIM?I1-{ZsKv+nsGY{bGhv5aF%qEHA) z1T9Y{S;uF+5B66<`!Rd?5DNxO#I|?cvBv$k3RvY3nW5uz-VRIvmxzKH?#=PFBw%Z&K6&TpJ(o| z1^d^?UK9NVrlCn9@da|Q@=Mqj2p-yF7>i z{wS`&bFcOBaG?AVI<%>&Nw2~J3v>#X)e#QiyLZVy8G{0B{WUoR%GJx|Pf*ktBl&Li zehW2*k-8e@%D#^X;xLW9&XP&u?mFl=DXryJ*RPyY8$%HiFV-8-WEtT-%}bvht(gCh z(9_tROzxPPqT%7;=|xLUK^$|p#xE)#ejz3%4(74!LZLuGivehYSYiQS#*I4yU&DP7ba zwn!0-g{P1JLmFr}i1T>60dSMB?;bO#0GGg5AmX=AfLJvFIL0(?0-J~g4ZvllT@6_c z#NiI|4XaB%Vn`hvBmsH96SM++QCYx~yt`lLQ~$%`zB02HITF0}^P|Yn!yqdwggk8x z$0s#vU>0+l9$Aa~LPLXoUGU1xI&i852`H|zP%e3)m*TfgO%WxgX=uOj<2iZVSLGba z6oRb-iiqbJ5~+f7&z<4XspNP51gBXrny4C% zc*P0YJ9kJ${PnQBHpgfC)9pX`jb0|A8Kb|QzlC2{TwJMMv+-cKfFCj6*AGMiMRgd{)b4mQ^3MUM=FMaY*2%Qd0 zoaw;DEj3D@6K&k8vRGO|ec=w2b;re$;!T)2 zqmCzA02?k8y+{5Ve_`;z244I@6-;$^@!qu~c2 zJ}o_6dn7(CP8JRj3PVb2C7B))TPRbNz_;Fyv%9WjjQxpwKe- zZjK-tj2ux>SY-|t=s@Rj7%qvu5lm_s(x@_z?TRboTDJ(YH&r;%G7P$8yu`#hnRvTp zAZxB%7KMY4;k!5B0K_UbGD?8OeX!K{9yt2sS5H2cmj}nk6M=4a_519X|CzHM27sr{ zsNPe#&=K)K{rz*b>1^NdQ*EuCs4VN96s0u9qnc^p%}#w>qx~L>h$i7z)J&nX>T*ta zM|FEI^9rd$2iO**$Hkp<7Z>05WRRAA`2hCo(eCfTy2lRb>*D|CsR@`myjUtBo$)Df zS)jDW6~s*%P;ocN@GGYSzbMZfX&2H5ejl$F8YrTe`D5_Wl{CTi`(r2ize{}>{b?jY zp6+Cuj$MADq$DgrHZ3hKhF{t#O!hGQ-^@McZC-(4w{Th-{XQPf;geC{!%h=Ty*GTT zSvI!AqS(v%t)rodu!{7p{MAY6_kl-PhsO@vHtz1O=){Cv7fkf;)jSN+lv8-IBqb#e zp^G3AemXS+q7`t7aN_X$$15R#VkBwdDCN%H47m@O!&^7_IAxFfZ#wiw;j0qXCLG=lh z&c4ed1qQpvlz1!Z8ZU$&?u1~>S<%!U1ZoG~0Ae~^Z*xi_;&SjCD1niR%A>V*;NaJ* z^I(~qn**Jl&mTfDw(v(GtIbeO5Z6rL`H_uC&^a|UWuyT>Gc#VZq{O(W0jhjf<&*yqXk!f&?+9KS-kXWSK@PDk>pgx(}u0Vmim*LIkk?JY_S`$F&`1 zra3>q?EU)&O(DXz~*Ym|@lRgP`enpP$}A0x+MngCim$R4mvW zQU%ed1S7>K#!x4Kn(hJ<-wNv=9v;5pc93I3Nhx8K0|Ak{m>r#kre%Qv71vfriEO`P zA6}d^-Dl;f*Nci=@JLEUGI1+Ix1e_E4kA`F=sygstcvS`;3ky@9~v=WSYT3GT3gWo z(f-SW3H9`I0aOC(M)dEURQCtD8uAaex>ynVl6+yaIix4CkB+vf$zEG<%3F&2@F2i= zPe$gm*bmV4J(E6)Ki>8luyd7NR8@%+{&m(BJLhpwI~u?ELcSxWaS0fMz~aPx%0rT1|I-8)__v$d2tKUZXt>%h&$EN zQgiMQ(xr+GjwXeL*2z5Y$Y9g+g*5v9lKWtPG1E;_veu+zTg1goaiWD(SM8y5h(>*P zV`T4_mvVKrklde-53ZvuC@Jpni~qZN`?BHJgP%_#BZt9U0hD@2y!bireVPbS1i1>t z_*yQU3P!ciu&}G&{>!}Z6me#Q^k$J5LSV1Y2KqFdVoUEi%=; zYv;X#g9sQoU$N_>qYz#U40BNsYs1mZQ)*4W=e2e8I}-_5-a7uZck3qFH4(iAPD<2L z|J8_3WR%yJn0kxeo)62xg-`epAtqJ`{(%4N7D8Er*0{p<5^w#;9RPSow$iV^e}2`$ zZA=AFa947?KY>4E>$g=~Sy%q5_# zAXeL^_5w9fY@*ReO0o9WNb)n=1#VlUn>4LJ$3JV{|LfC4S{C{)8<+@yp-i$T(Bl{qP9D$!?@t2 zhp3jCd&!*y*&GcPutu0eLa@3gzBf*qUJ@D<2^AGS>|kz+PkcglvaDFIIQs&c-1GI` zk|2x6dO2W;sqv}}Bt0Oi)(%G9fovK4fN=Gjpaa)?VZ=~iT%6@M03%~I%xre*-7;Y) zzO8;l-~4t&P0T8BHnCX;MvJ}kI&&x1j7GnZ2%^qRzXS5-1qNQ}ugY3R$GysM@F4s1 z<0DuP7;fH_RFCTAdd9Tg+mG$JP8w5rI-2;w9t#s2Jq#OdWaKz1zE>!aPvH&g>L{Lj zq9;!1j($}N-(L#+guLWbdCg@tvv`^;T_#BU+TN)`|W7}EF`75MMZ0-a#x0OE8MN+gq=zn(T>@e_3@S4g7m%G z;NgFEjpzP4FB@ACZ|p

d_7*2>qXO)q5X%DR0Q)4ebQi-R}&>+5*hiF25l@RuX$zmJKxU0O+++dK}!@eb4paJy!^nUCZXxtxnYR1b#yXM;$X3yg?J zh_@UNJ+Pc$zz>DE#)s81(`Fig(NB|-u7V4L8=U-BU^Vas(*X{+Gl(ID)(S!(8j4Er zk(z3K`(K|C(=`fWTUynQ0PO&w zi$STvfao65M8KF2L7GcB8g@~~Iqa}1lkwuwSl3a9Cm1R!$f8uo*D8F&yzyrsD-3** zOz;#7i~)&r4!_ra0Pg4#&h~*fP|?wXpQoi6Wv^FNRz3ms3LgY&nNqey+_t8G(3wB1 zICCXpopSGFLtTzNqAokTM1QB%;+I6RV#WcoC$y-jh!|W_ zwyG*B5GXzpu#zFpp^f-_pNh6NaO^uK#Viob(kuDU4G&<}g zVuNbiEL*?rxo|Zw?`A=oxeX-ctLqwtT1a{mB>`S&-EmdPfcGm7dDWY(ZeNvq`2#k) zziRa>2syu6cERpbo09uNCbgou#I(fpxVEtTe0V|NT}i{s*D7UqsQ9c2Adnm5H61_! z`JIsb{Co?24-bAO)r@D5x;C3_3$G(+U+(CLl#&TN6R7DMH%MEyJoad5yYHMul4KA= zE@70`^o}{Lif?PwGMUJN1> z5`n3#s)83T5P+Y*vkvkC$>KpxLp4u*9Di0~!MZ7a0S&JLg5{6K$|ZSTViGs|h`28i zAYmpz9x^9LeiqXC9%x4(b|SejrS!X3k#JfyXjA4;r?3P1Q0|&NYvWkma=wrR@;)KY z-U+RZs_jt&LiwKC0uIxA_5ScrC50#^^WZwtkFXR##|OTxgw z$!~M2r8fyO?8h;&@`iJBZsIdZUSNI%`>}5QBbaSb!JyeO+`TZKM}r-ToD}c}l-JaJ z1_qZ91!2#y-nu{I3lQHxfiwAK1H`an5I+jo8$Y?uV>dK5qM>O0x6txb(xagfG2d!| zv(pBFDL@a})jjieuuFP*0s;cety|oS_=K)z(lJkaydit9?$bWWXyG&@o-X(4|H#L` zbJ=I-Hb-|k^L0v%!QWc1nd=kvS;0u6=cOjus;YK~tcQUDfgCC+L<@lY z8-S~9AjK3vzcQ`L<)k4aBZJSSIQCnia@X93X&>xJ=aY2;&ZGAiiQE$-YZWd$Owtq> zNingoAZURDvaY;}iYy=y^(gmEbX11;rKPWEBPCxP`FPMmLxW{vP=ZnQU;^*4=0A^v11T%u~eM4zpIMRf?pw` zw)PIn^{^WPC*=>nkOL)o&-pIr$m)g@YwkQM=t*UtvhoU9_D>W-+(pp(kxsH>lc^^lcuOPCuM2$X#dcLKbXAosTqs5Rz4D<2~9CGfIxyUt02^^(fD>4*uW7c>eAisa;% zD>DSKJVJIGtr)JaWKm!sl#Ly77Lm>|{HlDM|9zR$=+j%zk1rcw;*10ye1(@F49@V@ zh@th^hcD3N<_-;rh-S9T%?_MeS|XZB6D}j^E#&PGNIdoI_%T8+A@(PTvLKAU86XYC z)rcog5N}2S1k#XnG!!AoWNx8>_r9ch3JMSMx*()Bg4=<0>yU|w31UK5peL4tabCs` zm|t}W^TTWpJs2MbJDvIf@DlB>{lowYAHi7wyJX>~T*2N9y@0vvoC+p{z7jpOg@s7* zOW5dwg1&?ik-aGQ#K0Ag`_JJO1N2;V?vR1>8#M8;w8rkXVHDsa<95#jCj{RhNs=5G z*NHJD-#w;+_m4oOcWBszw;CWRKh|}h_)&NxM{ImE68$u3(91o8;(px`!aNd?qgE?i z2Wkgdf$P_nW6P6M`FH1JHUtKBbS%Iq-xULZ&pj~>HWIAW>}-2PcgJaWb5|Pco#g_XKumj?(MGZ_#*u*%eFNR( z@?`|N$dh%wNnHRS`@!kg_2$zZd~i4AcpibCQW11bfpMBuNtb#K`%(0yCHk@O)v7 z%Lb#hg(71g2_s|3f&w>1UGW0w=`GEQYZX-TN7Ej!jgcTJUr3=ACnw3!Cop%VzDZ7= zL0&5ceCvbi3t(U&Kp;`E`N3gy4&omWl-MPg5IS#y7Ym^8xXY8e!JFHS6;)nr=+*}l%&%hzHJr^5k$4gU3w&E;R99bRn4Pco4VU`_0y!$F5} zTnWE6Gg8oW0*r@?ts?i=-f8V(|aH1mWn#tq0|holV*YO|3QWN|g`Lb}zjrb|TNRjE^@| zA!#Oj?;e8fyL)?w9N5PZ6(}G=ZDV2cJE(Ly`T0m-{{O1(+~cWE*Eaqm>A*Ne=pZSl zrI2cr4yuJhC0e8=XDS(rYLszGltU#&Oi9jR5=uoyP76s*R-)08L(~u{rCo@9-D>u` z_h-+%f4%>_{;RYOzxDi{=f1D|y1omBN6o#r@y4?&lOp&Wg?TpTh8F&I4pZlB9rcI( zK1iE44seUf-arIXgQg-20;bc8!Wkb&k+l=2t z{hK?G&%aZV7c1K%-m4QmQd?bJg3uVMj{JDaIOV}owV5| ztH}fm&mQ3?3cP)|%xBV((2O$Hwe}tqvsJj|C`|0UW5*f^A>hx1G1peFyR81-(05zakIW17y!aX1H80jKz5YY)#a7+IyySi zX$Hbj(xST?GdqhKdR{~>4U>^H-}<=g*&AFk0yohXk0SQzbvId*7y_#$w0%D88iq!_B`^*@fTiWL*CY{e$*!S7K+qrfvqC7<*YYaFJu3u_@;jp+! zd*op@O2$J~)d{FE%5Y=1_4P%f>Zf{m$Bs1p{Rkn!xDW#9Ng`TT8j`>pB6Dre=b@qU zCr@Nd&(0HId_Q-_QUV&M0bbt+cQ zzNo6YC(f1{R-p?Q3qlrHs=YWSDfvO8v)UAm7}cY^OYxV(>(wvoFaG7SiENHrw(L$& zsAxg~JNd2sU4m$733r4S&HI?7eekV4jkBqGfaZ);q$Gm0adSIx%9yZsHeB9%1a}S{ zQLg$h(*ZB|*vHqvz*(+FNjJthIcwq0hxRC-3y`a?tg7QWTdJ$Se41~3N9++#XL>?H z)!vSE-1(b6tjM!Cl-jql)7g?Eci>CrJ!i|NgpA3LR(qa}jut|xO(zu8X{hr@Zipw% zMLmsXb$9FqL^!$*p@rS(G+RljoYPQfJO$~zvg)Wg1QbDm?sn&brpm9ACmsx}UH@)Q zX@RGds5~#_>5#g(sEcD-f5oQ3=rfj4dLaXsRL)p-IVQ7kIk2uOq5bFn_`43Nw-l$w z0}{Lisi0j$d$Fk)vamDo!?Kc7QY5ELIf8nh9Q5?Wg4t1O#4>DNcLG-Kx{GYQ;s0(*)E zruu&w*h1nIreSetcqjti%a)R~JP4Hw1xtgsoba#nyf%BPX7%^?m*M!>JJ4;(ebMaQ zVu($j{(zxtjGoGdADdaLy{>-Bep_-F-Jgq>wNe&C(wnNXH#*z}9{eRZ{n~I#6a5hF z{ow;M^@%Nw(yAUs-xwnq76MJi-}btNcWb`WhCM#X2em+I&CKB=;9KB=0;;h zPfyRLG!q$|NCoSS&R095XD!@$@8DzCWJgC2*7q;d%esbF?!@5Z)64UF-Xe0(+EEi8 z8d`!*z#PKd{_gro^r@|T_`9jW_^mg^K6Pufe)3#(=Gr7=M{I5j!WfRZe`F4WH^v2F zqh3?vzMBHo8>miJO9(ATA{KqfLM$*(%&2DQ?O*5G3=LKK;LIEp(Z~u~Hf@0|84}I) z`=T*d#x6SVCi__6GVcnYH}vjd?B7;lt|J;)71n?Dod?xPj7E3V` zp-!3f=H=xDrKF@Vs{n)v=B*?zZI9FK+qXr|rJe~IAcs9#S} zF(;ZoH;S35HxWjMS#fdoJntn2l_7N0uA|U>y7lb*lIkeo*kR_~N-M;Jqf@_sN$f|y@+u{zYqz@6nuo!XgpOk^fDJXbz$PexRa2ySmZW4S<``Es zp_5Sx`NfNuD*hVkX8wVn_<^VG%mMfmV_=oDOnRM`o?eET8Y*BW@xz1Ec;9m$gRto( z;o>PDj}8r8RK`yYF3fa3$eyC4voca4tlY4F+S047zg!hP1!E4=in+doP*TY;Uu)xk zDKSIgk&#-rOUO7dw)T|E5g0#iIwEozn#Q%vJL6LcPp1px$#?0Qq53VsT-AzdB ziK6|V=j6eXr;D__{(Jd(>uiHm_iqD1y!~~*qI`A$c>?E)Kik+)N@%QA%g+xkEG#T8pPlR# zUKke6zE~Lc54@`EYd3FFu!6>@uo0K>`7@|baFB^uuR^W>1Xb)E%lh*2XYaWPRyF#! zxtQ_e6Wx2@z!UF-2Aqcrbj|%g4{eqZDte`Y1(fd8Fjk6~S zE!Yf?OUh%;fM`Ec0~fr7ev8J7gAX^*JHCW>d_w0iWssIFn-1oyk^J4+ju)k(lI%T{ zO0b64jHWkbZ(F<{$l2=UIJY`u&1L@=Ccp@=MuhtzS}O#p)nM4wp{3e> z*T3la995;{{)~b{U0{dE>FrnB7{BB<>g^NsyC`mI3b|8FLa34*`I3YJ^Qfv*n0( z2idt&R6b9xi~H{#`)d%ovE2+)RChNk*prBP}M zt^nBX?l18|!y;s~EQZhpz>oxEr(iX$2Vkt|XK!q8nawHypuyK@;!Z8n-GIAI1L%Uk z;sWC1MH#C5F}2EbI%|>!WHj5t;&kcy#!NHxaB#5@W5eXxC6v^Hiq99-OT^68>Wn>m zRu~^KqH(eXV*Ad#$<@k_^vfcTrcT8j=X~x~B(os0wRUu**1zfK2!@K5)vThaxv_T; zed_K;yFal3w%|TV8Ertc04Nv@PWCp-7KaQqW80cN(BQCjP#RNY`>{e`QF3=J)RK4i zw3cRL->FTq5BQK=r(T#KJI(>S|>pRD!CQ$kjZ5{F3C< zsa7frwglgeNPjBZ^v#7P?-*br#J+GQZftwV5M0pY_U&5=YNlXE8v79r#I{R6Q3Cda z62Xr%V;vuJn0BWBApZ=^JpCMH=xPXaE5IE8F!QXrdiTKGiGmnz5&|4!>l8S)LC-zu zAGBD|aNrg7JGR#Vmf@M@8rkE*lD&suJs1R%;|bUhi*2gV?IB*#5@l_ogi^JhNV93f#ycC;(VZ;NqzRyfPWp#v^ucrD5|Z;fIJA{MdAP z=W4)4!R!K4N@%r&T_g(VGBnrHuOWN08Aa?-T^hBMBQe%A{vj@w#^ghyOLgDX%EuI9 z*jmEnj)&MHx6*eB8JYS{)to@WV2-Ay7^>+l`Igz+WU%?uEXv8bvKO~BynO~hXc0{t z9Yxo&xhV+Sh%y0!;?=5Bjwqf}`BOezGv~`~!9Eb7c6K}Q7Aat}$My;ka=KI!0X)n6Es1qsP z5g{Qh9ZaJq!d?QjBi#%89F$UmP{%a9a*GzFvPxSk24txRa%>OG9()EcxHee3HU;Z< zP%LeG#^y_6SQp61WbQIGHZCR0Hg>cXMppjIf%gNI*Z09x_!1X%HjgU?8|1wCgapji}n=yDu!~kPZ zq$yCQ;UXHSs-W>AZ;|vX2N9N1ZHwKe;*^`9udlzt$h|;t+_*fyp`Ym64+w95{7DX? z>5yarq_w2t0EaBCG9ap`oy86=jEbXS-v=3yeVa954|1^Ot$eA{tCPj&r0-)}ZNf#C zU^13N0Zi+ARKthkDFmSD$@oGI@?MbcM*QnRvHKa)13stkIhTR!&O#SD2`A7R>@mTR zNxO7+jkB^20Agnn97fVe+6uu2R@&TbLu@@*vp>wdQEJf`%LGHJRc|mhGn0ioVjkyL zs|_eXBnb@NhFX45C3Va{A+T!IgUtHs)7ZCzjIPZhu9w%xa%03&H zStuW1Z8E<-3QHw?>x%Hq)#1xbc0lH#xgo|nY@C4TbB;hlvIos&or#IX?Hp9#bOaK+ z3d{i9F(Jo~AD0?=iL)s*E;e+3eQo{>nDSTN#z2z*WCSG7R)Ntv^3nNi-=0X9F$C6TE}H z?5th_NEPZ*iVV64M_gtm5O80$CV=`+_VoNn+$j7Tj80hjiC+)rUm!Kix(+8(!7`wY=|=M z0i$1ZnT!>-u1;9OU+7wjqi57%P-r8V8l*6**f+po_J-fWoWrX>+#=P+uMWij^;qZ zMJGXNNuaSMcF)$+kntx@z%4X3oYA3K*|K;(4*4!8g|Vlz zk{_RH(}xFtfk}%+IviIpKBYa^$5V*?`4lCUk+Z~MGvF3&FCOkU1>xKgzXX=@bZ`fb zVFge@NBtD`_~>*E1=zyS6(6N!8@|heG!JqFY88JHBqP89UvwL6q&5*Lrti@q*A=$z zU5HPkOttI&GDR}Vfj-Rnbm42!UBa+n2%<5pVMJbw*AGxY5orjsSnwll2D{ s>{y-qx1z(?@BKgcR<8flO-3hJ`-@C{F-=Vg-+RUw8*VhXvc^8>pAVFp_W%F@ literal 0 HcmV?d00001 From e80e5a05f6e532c763ec80110bcd8e2d7a7cb063 Mon Sep 17 00:00:00 2001 From: Mahdi Date: Mon, 15 Apr 2024 00:12:31 -0400 Subject: [PATCH 20/29] add plot notebooks --- .../fedpft/docs/viz_and_plot_results.ipynb | 88 ++++++------------- 1 file changed, 25 insertions(+), 63 deletions(-) diff --git a/baselines/fedpft/docs/viz_and_plot_results.ipynb b/baselines/fedpft/docs/viz_and_plot_results.ipynb index 866ffb5d8c7..68077f7b59c 100644 --- a/baselines/fedpft/docs/viz_and_plot_results.ipynb +++ b/baselines/fedpft/docs/viz_and_plot_results.ipynb @@ -2,22 +2,10 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 16, "id": "5e0cf2a9-b782-48de-ac45-128726a26e64", "metadata": {}, - "outputs": [ - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'matplotlib'", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[1;32mIn[2], line 7\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mos\u001b[39;00m\n\u001b[0;32m 6\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[1;32m----> 7\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mmatplotlib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpyplot\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mplt\u001b[39;00m\n", - "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'matplotlib'" - ] - } - ], + "outputs": [], "source": [ "import pickle\n", "import yaml\n", @@ -30,7 +18,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 17, "id": "7ea3e149-ce6f-4ba0-aa41-e0501a04efe3", "metadata": {}, "outputs": [], @@ -52,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 18, "id": "4b010856-0d99-4d81-8fb0-7a927f10eeaf", "metadata": {}, "outputs": [], @@ -61,13 +49,13 @@ "path_fedpft_resutls_cifar100 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','16-36-16')\n", "path_fedpft_resutls_caltech101 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','16-44-20')\n", "\n", - "path_fedavg_resutls_cifar100 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','18-16-41')\n", - "path_fedavg_resutls_caltech101 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','16-36-16')\n" + "path_fedavg_resutls_cifar100 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','23-24-25')\n", + "path_fedavg_resutls_caltech101 = os.path.join(os.path.realpath('..'),'outputs','2024-04-14','22-32-11')\n" ] }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 19, "id": "2e3e165c-1ce6-4efa-a4e1-1372586e436e", "metadata": {}, "outputs": [], @@ -84,7 +72,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 20, "id": "77b70c73", "metadata": {}, "outputs": [], @@ -97,49 +85,13 @@ }, { "cell_type": "code", - "execution_count": 2, - "id": "6f4c87ad", - "metadata": {}, - "outputs": [], - "source": [ - "fedavg_cifar = [(1, 0.06924765515865097),\n", - " (2, 0.1315106765116743),\n", - " (3, 0.16773099181800039),\n", - " (4, 0.1946717222111355),\n", - " (5, 0.2171223308720814),\n", - " (6, 0.2375773298742766),\n", - " (7, 0.2597285970864099),\n", - " (8, 0.276092596288166),\n", - " (9, 0.290560766314109),\n", - " (10, 0.3036320095789264),\n", - " (11, 0.3128118140091798),\n", - " (12, 0.3261823987228098),\n", - " (13, 0.33745759329475156),\n", - " (14, 0.3477349830373179),\n", - " (15, 0.35831171422869684),\n", - " (16, 0.36679305527838757),\n", - " (17, 0.37407703053282776),\n", - " (18, 0.3817601277190182),\n", - " (19, 0.38824585910995807),\n", - " (20, 0.3942326880862103)]" - ] - }, - { - "cell_type": "code", - "execution_count": 59, + "execution_count": 21, "id": "e1a678de", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n" - ] - }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1wAAAD0CAYAAACPQifaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABdAUlEQVR4nO3deVhU1f8H8PcMy7CDOuygyGK4YqHgviSCVi6ZipaifE2zpDLKFFNcE0sj/ZVluVbuZlmmqYjikqilWe4KiijIqjAsAgNzf38QkxODyjDDDPh+Pc88Oueec+9nzoye+cw991yRIAgCiIiIiIiISOvE+g6AiIiIiIiosWLCRUREREREpCNMuIiIiIiIiHSECRcREREREZGOMOEiIiIiIiLSESZcREREREREOsKEi4iIiIiISEeYcBEREREREekIEy4iIiIiIiIdYcJFRERERESkI0y4SC+Sk5Px2muvwdPTE2ZmZrCxsUH37t2xfPly3L9/X1nPw8MDL7zwgkpbkUik9uHk5KRSLy8vD2ZmZhCJRLh06ZLaOMaPH6+yD4lEglatWiE6OholJSXV6m/duhVjxoyBj48PRCIR+vTpU+NrLC0txfTp0+Hi4gJzc3MEBgYiLi5Obd3jx4+jR48esLCwgJOTE9566y0UFhbWuG9N/bfPbGxs0Lt3b+zevVvrx6qSkpKiPN6OHTuqbZ87dy5EIhFycnJqve/jx49j7ty5yMvLq7atT58+aj8nAwYMqFa3Nu8VERERUW0Y6zsAevLs3r0bI0aMgEQiQVhYGNq1a4eysjIcO3YM06ZNw4ULF/D1118/dB/9+/dHWFiYSpm5ubnK8+3btysTsY0bN2LhwoVq9yWRSLB69WoAQH5+Pn766ScsWLAAycnJ2Lhxo0rdL7/8EqdPn0bnzp2Rm5v70BjHjx+P77//HlOnToWPjw/Wr1+P5557DocOHUKPHj2U9c6ePYt+/fqhdevWiI2Nxe3bt7F06VJcu3YNv/7660OPoYmqvhMEATdv3sSXX36JQYMG4ddff0VISIjWj/eg+fPnY9iwYRCJRFrZ3/HjxzFv3jyMHz8ednZ21ba7ubkhJiZGpczFxaVavcd9r4iIiIhqTSCqR9evXxesrKwEX19fIT09vdr2a9euCcuWLVM+b9GihfD888+r1AEgTJky5ZHH6tWrlzBs2DDhnXfeEVq2bKm2zrhx4wRLS0uVMoVCIXTp0kUQiURCRkaGyrbU1FShoqJCEARBaNu2rdC7d2+1+z158qQAQFiyZImy7P79+4KXl5fQtWtXlboDBw4UnJ2dhfz8fGXZqlWrBADCvn37Hvk6a0Nd3128eFEAIAwcOFCrx6py48YNAYDQsWNHAYCwY8cOle1z5swRAAjZ2dm13veSJUsEAMKNGzeqbevdu7fQtm3bR+6jNu8VERERUW1xSiHVq48//hiFhYVYs2YNnJ2dq2339vbG22+/XefjpKam4ujRoxg1ahRGjRqFGzdu4Pjx44/VViQSoUePHhAEAdevX1fZ5u7uDrH40f9svv/+exgZGWHSpEnKMjMzM0yYMAGJiYm4desWAEAmkyEuLg5jxoyBjY2Nsm5YWBisrKywbdu2x4q5Llq3bg2pVIrk5GSV8tLSUsyZMwfe3t6QSCRwd3fH+++/j9LSUpV6cXFx6NGjB+zs7GBlZYWnnnoKM2fOrHacUaNGoVWrVpg/fz4EQXhkXCdPnsSAAQNga2sLCwsL9O7dG7/99pty+9y5czFt2jQAQMuWLZVTBlNSUlT2U15e/tDpmY/7XhERERFpglMKqV7t2rULnp6e6NatW532U1JSUu2aH2tra0gkEgDA5s2bYWlpiRdeeAHm5ubw8vLCxo0bH/u4VV/amzRpolF8f/75J1q1aqWSRAFAQEAAgMpphO7u7jh37hzKy8vRqVMnlXqmpqbo2LEj/vzzT42OXxv5+fm4d+8evLy8lGUKhQKDBw/GsWPHMGnSJLRu3Rrnzp3Dp59+iqtXr2Lnzp0AgAsXLuCFF15Ahw4dMH/+fEgkEiQlJakkRlWMjIwwa9YshIWF4ccff8SwYcNqjOngwYMYOHAg/P39MWfOHIjFYqxbtw7PPvssjh49ioCAAAwbNgxXr17F5s2b8emnn0IqlQIA7O3tlfu5evUqLC0tUVZWBkdHR0ycOBHR0dEwMTFR1nnc94qIiIhIE0y4qN7IZDKkpaVhyJAhdd7XmjVrsGbNGpWydevWYfz48QCAjRs3YsiQIcrrukJDQ/H1119j+fLlMDau/rGvSt7y8/Oxc+dO7NixA+3atcNTTz2lUXx37txRewavqiw9PV1Z78Hy/9Y9evSoRsd/mKpkVRAEpKamYtasWaioqMDw4cOVdTZt2oQDBw7g8OHDKtcwtWvXDpMnT8bx48fRrVs3xMXFoaysDL/++qsy4XmYl19+GQsWLMD8+fPx4osvqr2WSxAETJ48GX379sWvv/6qrPPaa6+hbdu2mDVrFvbv348OHTrgmWeewebNmzF06FB4eHio7MfLywt9+/ZF+/btUVRUhO+//x4LFy7E1atXsXXrVmW9x32viIiIiDTBhIvqjUwmA1B5JqquhgwZgoiICJWytm3bAgD+/vtvnDt3TmWxhNGjR2PRokXYt28fnn/+eZV2RUVFKmdFAKBHjx745ptvNF7c4f79+8qzbQ8yMzNTbn/wz5rqPrhio7b8N1k1MTHB+++/j8jISGXZ9u3b0bp1a/j6+qqcSXz22WcBAIcOHUK3bt2UC1X89NNPCA8Pf+R0y6qzXOPGjcPOnTvx4osvVqtz9uxZXLt2DbNmzaq2MEm/fv3w3XffQaFQPPJY/03Ix44di0mTJmHVqlV455130KVLFwCP/14RERERaYIJF9WbqilbBQUFdd6Xm5sbgoKC1G7bsGEDLC0t4enpiaSkJACVX549PDywcePGagmXmZkZdu3aBQC4ffs2Pv74Y2RlZVVb9bA2zM3Nq13rBEC51HzVvqv+rKnuo2LIyMhQeW5ra/vINlXJallZGX7//XcsWrQIxcXFKgnMtWvXcOnSpWqJaJWsrCwAlWcOV69ejVdffRUzZsxAv379MGzYMAwfPrzGhOiVV15RnuUaOnRote3Xrl0DAIwbN67G15Cfn6/RdM93330Xq1atwoEDB5QJ1+O+V0RERESaYMJF9cbGxgYuLi44f/68zo4hCAI2b96MoqIitGnTptr2rKwsFBYWwsrKSllmZGSkkryFhITA19cXr732Gn7++WeN4nB2dkZaWlq18qophFVLk1dNW6sq/29ddUuY//c4D3pwWmVNHkxWn3vuOUilUkRERKBv377K66oUCgXat2+P2NhYtfuouqbJ3NwcR44cwaFDh7B7927s3bsXW7duxbPPPov9+/fDyMioWtuqs1zjx4/HTz/9VG27QqEAACxZsgQdO3ZUe/wH37/aqIr77t27yrLHfa+IiIiINMGEi+rVCy+8gK+//hqJiYno2rWr1vd/+PBh3L59G/Pnz0fr1q1Vtt27dw+TJk3Czp07MWbMmBr34ezsjHfeeQfz5s3DiRMnlGdCaqNjx444dOgQZDKZymIMJ0+eVG4HKq+JMjY2xh9//IGRI0cq65WVleHs2bMqZer89+a8VdMqa+O1117Dp59+ilmzZimvq/Ly8sJff/2Ffv36PXJapVgsRr9+/dCvXz/ExsZi0aJF+OCDD3Do0KEaz0KOGTMGCxcuxLx58zB48GCVbVWLd9jY2NTYvkptp3xWrTr54Jm7x32viIiIiDTBZeGpXr3//vuwtLTEq6++iszMzGrbk5OTsXz5co33XzWdcNq0aRg+fLjKY+LEifDx8al2M2N13nzzTVhYWGDx4sUaxTF8+HBUVFSo3MC5tLQU69atQ2BgoPJMi62tLYKCgrBhwwaVqZbfffcdCgsLMWLEiIceJygoSOWhbvGHRzE2Nsa7776LS5cuKc84jRw5EmlpaVi1alW1+vfv30dRUREA1TNFVaoSFHXT9KpUneU6e/ZstbOI/v7+8PLywtKlS9Uu556dna38u6WlJQAgLy9PpY5MJqt2fEEQlDe/fvAGz4/7XhERERFpgme4qF55eXlh06ZNCA0NRevWrREWFoZ27dqhrKwMx48fx/bt2x85Ja4mpaWl2LFjB/r3769c8OC/Bg8ejOXLlyMrKwsODg417qtZs2YIDw/HF198gUuXLinPlh05cgRHjhwBUPnFv6ioSPklvlevXujVqxcAIDAwECNGjEBUVBSysrLg7e2Nb775BikpKdUWc/jwww/RrVs39O7dG5MmTcLt27fxySefIDg4GAMGDNCoL2pr/PjxiI6OxkcffYShQ4di7Nix2LZtGyZPnoxDhw6he/fuqKiowOXLl7Ft2zbs27cPnTp1wvz583HkyBE8//zzaNGiBbKysvDFF1/Azc1NZXVDdaqu5Tp79qxKuVgsxurVqzFw4EC0bdsW4eHhcHV1RVpaGg4dOgQbGxvlNXf+/v4AgA8++ACjRo2CiYkJBg0ahDNnzmD06NEYPXo0vL29cf/+ffz444/47bffMGnSJDzzzDPK49XmvSIiIiKqNX3edZmeXFevXhUmTpwoeHh4CKampoK1tbXQvXt34bPPPhNKSkqU9Vq0aCE8//zzKm0BCFOmTKm2zx07dggAhDVr1tR43ISEBAGAsHz5ckEQBGHcuHGCpaWl2rrJycmCkZGRMG7cOGXZnDlzBABqH3PmzFFpf//+feG9994TnJycBIlEInTu3FnYu3ev2mMdPXpU6Natm2BmZibY29sLU6ZMEWQyWY2vQ1M19Z0gCMLcuXMFAMKhQ4cEQRCEsrIy4aOPPhLatm0rSCQSoUmTJoK/v78wb948IT8/XxAEQYiPjxeGDBkiuLi4CKampoKLi4swevRo4erVq8r93rhxQwAgLFmypNox161bp+y/7OxslW1//vmnMGzYMKFZs2aCRCIRWrRoIYwcOVKIj49XqbdgwQLB1dVVEIvFAgDhxo0bwvXr14URI0YIHh4egpmZmWBhYSH4+/sLK1euFBQKRbU4avNeEREREdWGSBAEQQ95HhERERERUaPHa7iIiIiIiIh0hAkXERERERGRjjDhIiIiIiIi0hEmXERERI9w5MgRDBo0CC4uLhCJRNi5c+cj2yQkJOCZZ56BRCKBt7c31q9fr/M4iYjI8DDhIiIieoSioiL4+flhxYoVj1X/xo0beP7559G3b1+cPXsWU6dOxauvvop9+/bpOFIiIjI0XKWQiIioFkQiEX788UcMHTq0xjrTp0/H7t27cf78eWXZqFGjkJeXh71799ZDlEREZCh442MACoUC6enpsLa2hkgk0nc4RERPFEEQUFBQABcXF4jFjWPiRWJiIoKCglTKQkJCMHXq1BrblJaWorS0VPlcoVDg7t27aNasGccmIqJ6pO1xyWATrhUrVmDJkiXIyMiAn58fPvvsMwQEBDyy3ZYtWzB69GgMGTLksebYA0B6ejrc3d3rGDEREdXFrVu34Obmpu8wtCIjIwOOjo4qZY6OjpDJZLh//z7Mzc2rtYmJicG8efPqK0QiInoEbY1LBplwbd26FZGRkVi5ciUCAwOxbNkyhISE4MqVK3BwcKixXUpKCt577z307NmzVseztrYGUNmpNjY2tY5XLpdj//79CA4OhomJSa3bNzbsD+1if2oX+1P76tqnMpkM7u7uyv+Ln1RRUVGIjIxUPs/Pz0fz5s1x48aNOvWNXC7HoUOH0LdvX37mH8B+0S32r26xf3Xr7t27aNWqldbGJYNMuGJjYzFx4kSEh4cDAFauXIndu3dj7dq1mDFjhto2FRUVeOWVVzBv3jwcPXoUeXl5j328qqkaNjY2GidcFhYWsLGx4Yce7A9tY39qF/tT+7TVp41p2pyTkxMyMzNVyjIzM2FjY6P27BYASCQSSCSSauVNmzbVaGyqUvX+NGvWjJ/5B7BfdIv9q1vs3/qhrXHJ4BKusrIynD59GlFRUcoysViMoKAgJCYm1thu/vz5cHBwwIQJE3D06NGHHuO/8+RlMhmAyg+vXC6vdcxVbTRp2xixP7SL/ald7E/tq2ufNsb3omvXrtizZ49KWVxcHLp27aqniIiISF8MLuHKyclBRUWF2rnvly9fVtvm2LFjWLNmDc6ePftYx6hpnvz+/fthYWFR65irxMXFady2MWJ/aBf7U7vYn9qnaZ8WFxdrORLtKywsRFJSkvL5jRs3cPbsWTRt2hTNmzdHVFQU0tLS8O233wIAJk+ejM8//xzvv/8+/ve//+HgwYPYtm0bdu/era+XQEREemJwCVdtFRQUYOzYsVi1ahWkUuljtfnvPPmq6weCg4M1nlIYFxeH/v3787Qu2B/axv7ULvZn3eUVy3EjpwjXc4pwI6cYydkFkOVmYf3r/TS+hsvQ/fHHH+jbt6/yedUYMm7cOKxfvx537txBamqqcnvLli2xe/duvPPOO1i+fDnc3NywevVqhISE1HvsRESkXwaXcEmlUhgZGamd++7k5FStfnJyMlJSUjBo0CBlmUKhAAAYGxvjypUr8PLyUmlT0zx5ExOTOn0Bq2v7xob9oV3sT+1ifz6cvEKBW3eLkZxdhOvZhbieXYTrOZV/5haVVatvbSLSuE8bwvvQp08fPOy2levXr1fb5s8//9RhVERE1BAYXMJlamoKf39/xMfHK28qqVAoEB8fj4iIiGr1fX19ce7cOZWyWbNmoaCgAMuXL+dy70RED3G3qAzXswuR/E9SlfxPYpWaW4xyRc0Jxn8VlwOFpeVo0gCSJyIiovpkcAkXUDlVY9y4cejUqRMCAgKwbNkyFBUVKVctDAsLg6urK2JiYmBmZoZ27dqptLezswOAauVERE+qnMJSXMssxLWsApU/1Z2tehgHawk87S3haW8FT6klvBys0NxOgr8TE2AlMcghhYiISK8McnQMDQ1FdnY2oqOjkZGRgY4dO2Lv3r3KhTRSU1O1ctdnIqLGRBAE5BSW4VpmAa5lFeLqP38mZRXibi0SK4mxGC2llvCyt4Kn/b9/tpRawtqs+hksuVyO841nRXciIiKtMsiECwAiIiLUTiEEgISEhIe2VTeXnoioMckvluNCej6u/JNUVSVZecWPv8S6vbUEPg5W8LK3glfVWSt7S7jYmkMsZgZFRESkDQabcBERUaXcwlKcT5fhfFp+5SM9H7fu3n/s9g7WErRytIa3gxVaOVrDx9EKPg5WsLMw1WHUREREBDDhIiIyKJmykn8SKxnOpeXjQno+7uSXPFZbJxuzf5Kpf5MqHwdr2FpwIQsiIiJ9YcJFRKQHgiAgPb8E525XJlWVZ65kyC4ofWRbcxMjtHGxQXtXW/g6WcPnn7NXtuZMrIiIiAwNEy4iIh2TVyiQlFWIi+kyXLojw8V/Ho9zvZWVxBhtXWzQztUW7Vwrk6yWUisY8RorIiKiBoEJFxGRFuUXy3HxzgOJVboMSVmFKKtQPLKtrbkJ2rn+k1y52KKdqy1aNLXgAhZEREQNGBMuIiINCIKAW3fvK89WVZ29Sst7vMUspFaSf6YF2iiTK7cm5hCJmFwRERE1Jky4iIgeQ5asBGdS83D2Vh7+TL2Hi+kyFJSWP7KdWAR42luhjbMNWjvboI2LDVo7W8PB2qweoiYiIiJ9Y8JFRPQfJfIKnE/Lx58PJFjpj7FSoKWpEVo/kFi1cbZBK0drmJsa1UPUREREZIiYcBHRE00QBKTkFuPP1Hv/JFd5uHRHhnKF8NB2zrZmaPNAYtXa2QbNeb0VERER/QcTLiJ6ohSUlONSngjJB5Pxd7oMZ2/lPXK1QAtTI3Rws0VH9ybo6G6Hp5vbwdGGUwKJiIjo0ZhwEVGjVl6hwF+383Dkag6OJeXg7K08VCiMgEvJauuLRIC3vRWebm6Hju5N8HRzO/g4WMHYSFzPkRMREVFjwISLiBoVQRBwI6cIx5JycPRaDk4k5z50cYtmlqbKs1Yd3Zugg7stbMx4A2EiIiLSDiZcRNTg3Ssqw2/JOTh2rTLJetjS7J5SS7gaFeDFXn7o5CGFe1MuxU5ERES6w4SLiBqc0vIKnE65h6NJlUnW+fR8CDWscdHU0hTdvaXo6SNFD28p7C2NsWfPHjzXwRkmJjyTRURERLrFhIuIDJ5CIeBShgzHk3JxLCkHp27cxX15hdq6psZidPZogp4+9ujhLUUbZxuVlQPl8ocvkEFERESkTUy4iMjgCIKA1LvF+C0pF78l5yAxORd3i8pqrN/a2UZ5BiugZVOYmfC+V0RERGQYmHARkUHILijF8eQc5Vmsh12H5WgjQQ9ve/T0kaK7txT21pJ6jJSeVCtWrMCSJUuQkZEBPz8/fPbZZwgICKix/rJly/Dll18iNTUVUqkUw4cPR0xMDMzMeEsBIqInCRMuItKLghI5Tt24W3kWKykHVzILaqxrLTFGF69m6O7VDN29pfB2sOJCF1Svtm7disjISKxcuRKBgYFYtmwZQkJCcOXKFTg4OFSrv2nTJsyYMQNr165Ft27dcPXqVYwfPx4ikQixsbF6eAVERKQvTLiIqF6UlStwJvUejidV3g/rr9v5qFCoX+nC1FiMTi2aoLu3FN28mqG9qy3vg0V6FRsbi4kTJyI8PBwAsHLlSuzevRtr167FjBkzqtU/fvw4unfvjpdffhkA4OHhgdGjR+PkyZP1GjcREekfEy4i0pmM/BIkXMnCoStZ+C0pF4U13A9LJAI6uNqim7cU3b2k6OTRhNdhkcEoKyvD6dOnERUVpSwTi8UICgpCYmKi2jbdunXDhg0bcOrUKQQEBOD69evYs2cPxo4dW+NxSktLUVpaqnwuk8kAVC70UpfFXqracsEYVewX3WL/6hb7V7e03a9MuIhIa8orFDiTmvdPkpWNS3dkNdb1srf85wyWFF09m8HWgku0k2HKyclBRUUFHB0dVcodHR1x+fJltW1efvll5OTkoEePHhAEAeXl5Zg8eTJmzpxZ43FiYmIwb968auX79++HhYVF3V4EgLi4uDrvozFiv+gW+1e32L+6UVxcrNX9MeEiojrJLijF4avZOHQlC0evZkNWov4sVhMLE/RuZY+ePvbo7i2Fky0XDqDGKyEhAYsWLcIXX3yBwMBAJCUl4e2338aCBQswe/ZstW2ioqIQGRmpfC6TyeDu7o7g4GDY2NhoHItcLkdcXBz69+/Pe889gP2iW+xf3WL/6lZubq5W98eEi4hqpUIh4K/beUi4XHkW61xafo11O7jZos9TDuj7lD06uNnBSMyFLqjhkUqlMDIyQmZmpkp5ZmYmnJyc1LaZPXs2xo4di1dffRUA0L59exQVFWHSpEn44IMPIBZXvyZRIpFAIqm+4qaJiYlWvlBpaz+NDftFt9i/usX+1Q1t9ykTLiJ6JFmJHAcvVV6LdeRqNu4Vq5/bbGNmjF6t7NH3KQf0amXP5dqpUTA1NYW/vz/i4+MxdOhQAIBCoUB8fDwiIiLUtikuLq6WVBkZVV6XKAjqF4shIqLGSaOE6+TJkwgMDNR2LERkQIrLynHgUhZ++SsdCVezUVauUFuvjbMN+vrao89TDnja3Y6rCZLe6HJsioyMxLhx49CpUycEBARg2bJlKCoqUq5aGBYWBldXV8TExAAABg0ahNjYWDz99NPKKYWzZ8/GoEGDlIkXERE9GTRKuLp27Yr27dtj4sSJGDNmDOzs7LQcFhHpQ4m8AglXsrHr73QcvJSF+/KKanWsJMbo6SNFn6fs0buVA6/FIoOhy7EpNDQU2dnZiI6ORkZGBjp27Ii9e/cqF9JITU1VOaM1a9YsiEQizJo1C2lpabC3t8egQYPw4Ycfai0mIiJqGDRKuMaMGYMdO3bgrbfewvvvv4/hw4dj4sSJ6Nmzp7bjIyIdKytX4FhSNn756w72X8xUu3S71EqCFzo4I7itIzq1aApTY57FIsOj67EpIiKiximECQkJKs+NjY0xZ84czJkzRyvHJiKihkujb03ffvst0tPT8dlnn8HX1xcbNmxAnz594Ovri08++QQ5OTnajpOItKi8QoFj13Iw/fu/0fnDA/jf+j/ww59pKslWEwsTjA5ojk0TA3FyZj/MHdwW3bykTLbIYHFsIiIiQ6TxNydbW1tMmTIFZ86cwR9//IFJkyYhMzMT06ZNg5ubG0JDQ3HgwAFtxkpEdaBQCDh5PRezd55Hl5h4jFlzElv/uIX8+/8ugGFtZozh/m5YH94Zpz4IQsyw9ujmJeXqgtRgcGwiIiJDo5Wfqp955hl8+eWXSE9Px/r16yGVSvH9998jJCQEnp6e+Pjjj1FQUKCNQxFRLeQUlmLv+QzM/fkCui0+iNCvT+C7EzeRU1imrGNhaoTBfi5YFdYJf8wKwtIRfujzlANMuPgFNXAcm4iIyBBobVn4e/fu4dtvv8Xq1auRnp4OkUiE7t2749KlS5gxYwaWLVuGn376CZ07d9bWIYnoAYIgICW3GL+n3MUfKXfxR8o9XM8pUltXYizGs74OeKGDC571dYC5KVdNo8aJYxMREelbnROuQ4cOYdWqVdi5cydKSkpgb2+PadOm4bXXXoOnpydKS0uxdu1avP/++3jzzTdx4sQJbcRN9MSTVyhwMV32T4J1D3/cvKty5uq/TIxE6OVjj0F+Lghq4wgrCW/DR40XxyYiIjIUGn3jyszMxLp167BmzRpcv34dgiCgd+/emDx5MoYNG6Zyd2aJRILXX38dSUlJWLFixWMfY8WKFViyZAkyMjLg5+eHzz77DAEBAWrr/vDDD1i0aBGSkpIgl8vh4+ODd999F2PHjtXk5REZpMLScpxPycPvKffwR8pd/Jmap3bZ9iomRiJ0cLNDJ48m6NyiKTq3bApbc96Nnhqv+hibiIiIakujhMvNzQ0KhQJNmjTB1KlTMWnSJDz11FMPbWNvb4+yspp/fX/Q1q1bERkZiZUrVyIwMBDLli1DSEgIrly5AgcHh2r1mzZtig8++AC+vr4wNTXFL7/8gvDwcDg4OCAkJESTl0ikd4Ig4M9bedj9Vxr2/W2Ed04chEKoub61mTH8WzRBZ4+m6OzRFB3cbGFmwqmC9OTQ9dhERESkCY0SrsDAQEyePBkjRoyARCJ5rDYzZszAjBkzHqtubGwsJk6ciPDwcADAypUrsXv3bqxdu1btPvr06aPy/O2338Y333yDY8eOMeGiBqUqydrz9x3sOXcH6fkl/2ypvkqgs63ZP8lVE3TyaIpWjtZcTZCeaLoem4iIiDShUcJ17NgxbcehVFZWhtOnTyMqKkpZJhaLERQUhMTExEe2FwQBBw8exJUrV/DRRx+prVNaWorS0lLlc5lMBgCQy+WQy+Vq2zxMVRtN2jZG7I/aEQQBf93Ox94Lmfj1fOYDSda/RBDg42CFTh5N4N+8CTq1sIOLnblKHUVFORQ1zzCkf/DzqX117VNtvRe6HJuIiIg0pVHCdfv2bZw5cwa9evWCnZ1dte337t3D0aNH4e/vD1dX11rtOycnBxUVFXB0dFQpd3R0xOXLl2tsl5+fD1dXV5SWlsLIyAhffPEF+vfvr7ZuTEwM5s2bV618//79sLCwqFW8D4qLi9O4bWPE/qiZIACphcCfuWL8dVeEu6XVz0yJRQKeshXQsZmA9k0EWJrkA8gH0lJwNg04W+9RNy78fGqfpn1aXFyslePrcmwiIiLSlEYJ18KFC7F9+3akp6er3W5hYYH//e9/GDVqFD7//PM6Bfi4rK2tcfbsWRQWFiI+Ph6RkZHw9PSsNt0QAKKiohAZGal8LpPJ4O7ujuDgYNjY2NT62HK5HHFxcejfv7/KRdlPKvaHeoIg4O80GX49n4G9FzKRllf9TJaxWIRuXk0xsJ0TgnwdYGdhwv7UMvan9tW1T6tmGdSVIY5NREREGiVcBw8eRHBwcI1z5CUSCYKDg3HgwIFa71sqlcLIyAiZmZkq5ZmZmXBycqqxnVgshre3NwCgY8eOuHTpEmJiYtQmXBKJRG3sJiYmdfoCVtf2jQ37458k63Y+9py7g93n7uD2vfvV6hiLRejuLcXz7Z0R3NYRdhamavfF/tQu9qf2adqn2nofdDk2ERERaUqjhCstLQ0vvfTSQ+u0aNECu3btqvW+TU1N4e/vj/j4eAwdOhQAoFAoEB8fj4iIiMfej0KhULlOi6g+peYW44c/b+PHP9NwM7f6dCljsQjdvKV44RFJFhE9Pl2OTURERJrSKOEyNTV95BQQmUwGkUizFdMiIyMxbtw4dOrUCQEBAVi2bBmKioqUqxaGhYXB1dUVMTExACqvyerUqRO8vLxQWlqKPXv24LvvvsOXX36p0fGJNCErkePXc3ew43QaTqXcrbbdSHkmywnBbZzQxJJJFpE26XpsIiIi0oRGCVf79u2xa9cuxMbGqp26UVJSgp9//hnt27fXKKjQ0FBkZ2cjOjoaGRkZ6NixI/bu3atcSCM1NRVisVhZv6ioCG+88QZu374Nc3Nz+Pr6YsOGDQgNDdXo+ESPq0Ih4FhSDnacvo19FzJQWq5Q2S4SAd29pBjk58wki0jHdD02ERERaUKjhCs8PBwTJkzA4MGD8eWXX8LT01O5LTk5GW+88QbS09Mxf/58jQOLiIiocQphQkKCyvOFCxdi4cKFGh+LqLauZhZgx+nKKYNZBdWnrno7WOGlZ9zw4tOucLI100OERE+e+hibiIiIakvjhGvPnj3YsWMHfH190bJlS7i6uiItLQ03btxAeXk5QkNDlVMAiRqD3MJS/PxXOnacuY3zadWnLTWxMMFgPxe85O+G9q62nLZEVM84NhERkSHSKOECgG3btmHFihX44osvcPnyZVy7dg0A0KZNG0yZMgWvv/661oIk0pfS8gocupyF70+nIeFKFsoVgsp2Y7EIz/o6YNgzbnjW1wGmxuIa9kRE9YFjExERGRqNEy6RSKSc9ldUVIT8/HzY2trC0tJSm/ER6UVSViE2nLiJnWfTkFcsr7a9g5sthj3tisEdXdGU12URGQyOTUREZGi08nO8paUlXFxcOKBRg1ZeocC+Cxl4ZfUJBMUexvrjKSrJlqONBK/19sT+d3rh54geGN+9JZMtIgOm7bFpxYoV8PDwgJmZGQIDA3Hq1KmH1s/Ly8OUKVPg7OwMiUSCVq1aYc+ePVqJhYiIGg6Nz3ARNRY5haXY+vstbDxxE+n5JSrbJMZiDGjnhJeecUN3bymMxLwui+hJtHXrVkRGRmLlypUIDAzEsmXLEBISgitXrsDBwaFa/bKyMvTv3x8ODg74/vvv4erqips3b8LOzq7+gyciIr3SOOG6desWFi5ciAMHDiA9PR1lZWXV6ohEIpSXl9cpQCJdEAQBZ1Lz8G1iCvacuwN5heq1WS2aWWBslxYY4e8OWwsTPUVJRLWlq7EpNjYWEydOVC64sXLlSuzevRtr167FjBkzqtVfu3Yt7t69i+PHj8PEpPL/EA8Pj9q/ICIiavA0SriuX7+OwMBA3Lt3D23btkVpaSlatGgBMzMzXL9+HXK5HH5+fvwljwzO/bIK/PxXGr5NvIkL6aorDYpEwLNPOWBs1xbo5WMPMc9mETUouhqbysrKcPr0aURFRSnLxGIxgoKCkJiYqLbNzz//jK5du2LKlCn46aefYG9vj5dffhnTp0+HkZGR2jalpaUoLf33NhNVN3GWy+WQy6tfS/q4qtrWZR+NEftFt9i/usX+1S1t96tGCde8efOQn5+P+Ph49O7dG2KxGOHh4YiOjsadO3fw+uuv4+LFizhw4IBWgyXSVEpOETacuIntp28j/77qPyI7CxOEdnLHmC4t4N7UQk8RElFd6WpsysnJQUVFBRwdHVXKHR0dcfnyZbVtrl+/joMHD+KVV17Bnj17kJSUhDfeeANyuRxz5sxR2yYmJgbz5s2rVr5//35YWNT9/6a4uLg676MxYr/oFvtXt9i/ulFcXKzV/WmUcB04cADPPfccevfurSwThMopWc7Ozti6dSvat2+PmTNn4quvvtJOpES1VKEQkHAlC98m3sThq9nVtndws8XYLi0wyM8FZibqf3EmoobDkMYmhUIBBwcHfP311zAyMoK/vz/S0tKwZMmSGhOuqKgoREZGKp/LZDK4u7sjODgYNjY2Gscil8sRFxeH/v37K6c3EvtF19i/usX+1a3c3Fyt7k+jhCsnJwe+vr7/7sTYWCUTlEgk6N+/P3bu3FnnAIlqq7xCgc2/38LXR5Jx6+59lW2mxmK80MEZYV090NHdTj8BEpFO6GpskkqlMDIyQmZmpkp5ZmYmnJyc1LZxdnaGiYmJyvTB1q1bIyMjA2VlZTA1rb7CqUQigUQiqVZuYmKilS9U2tpPY8N+0S32r26xf3VD232qUcIllUpRVFSk8jwlJUV1x8bGyMvLq0tsRLWWcCULH+6+hGtZhSrlrnbmGNOlBUI7u3Mpd6JGSldjk6mpKfz9/REfH4+hQ4cCqDyDFR8fj4iICLVtunfvjk2bNkGhUEAsrrwDy9WrV+Hs7Kw22SIiosZLo4TLx8cHycnJyucBAQHYt28frl+/Dk9PT2RnZ+P777+Hl5eX1gIlephrmQVYuPtStamDvVrZI6xLC/T1deCS7kSNnC7HpsjISIwbNw6dOnVCQEAAli1bhqKiIuWqhWFhYXB1dUVMTAwA4PXXX8fnn3+Ot99+G2+++SauXbuGRYsW4a233tLOiyUiogZDo4Rr4MCBmDt3LvLy8mBnZ4epU6di165d6NChA1q3bo2kpCTIZDLMnTtXy+ESqbpbVIZP465i06lUVCj+Xdr96eZ2mPV8G/i3aKLH6IioPulybAoNDUV2djaio6ORkZGBjh07Yu/evcqFNFJTU5VnsgDA3d0d+/btwzvvvIMOHTrA1dUVb7/9NqZPn66tl0tERA2ERgnX66+/jj59+ijnpvfp0wdbtmzB3Llzcf78ebRo0QILFy7ExIkTtRosUZXS8gp8e/wm/u/gNRSU/Hs/HRdbM0wf6IvBfi4QiXhGi+hJouuxKSIiosYphAkJCdXKunbtihMnTmh0LCIiajw0SrhsbGwQGBioUjZixAiMGDFCK0ER1UQQBOy7kImYXy/hZu6/F8NbmBrhjT5eeLWnJ1ccJHpCcWwiIiJDpFHC9eyzz6J79+5YsGCBtuMhqtH5tHws+OUiTt64qywTiYCR/u54N7gVHGzM9BgdEekbxyYiIjJEGiVcJ0+eRJcuXbQdC5FaWbISLNl3Bd+fuQ3h38u00NWzGWa90BptXWz1FxwRGQyOTUREZIg0Srh8fX1x8+ZNbcdCpOJ+WQVWHb2OlYeTUVxWoSz3aGaBmc+1Rv82jrxOi4iUODYREZEh0ijhevPNNxEREYGLFy+iTZs22o6JnnAKhYCf/0rHR3sv405+ibLcxswYb/XzQVhXD5gaix+yByJ6EnFsIiIiQ6RRwuXp6Yk+ffqgS5cueO2119C5c2c4Oqo/29CrV686B0lPBkEQcPhqNpbuv4LzaTJluZFYhDGBzfF2UCvetJiIasSxiYiIDJFGCVefPn0gEokgCAI++eSTh07rqqioqHEbUZVTN+5i6b4rOJVyV6W871P2+OD51vB2sNZTZETUUHBsIiIiQ6RRwhUdHc1rZ0grzt3Ox9L9V3D4arZKeVsXG0wf4Iterez1FBkRNTQcm4iIyBBplHDNnTtXy2HQkyYpqwCxcVex51yGSrmXvSXeDX4KA9o6QSzmFycienwcm4iIyBBplHARaerW3WIsj7+GH87chuKBJd5d7cwxNcgHLz7tCmMjLohBRERERI0DEy6qF1myEnx+KAmbT6VCXvFvpiW1kuCtft4I7ewOibGRHiMkIiIiItI+jRIusVj8WPPkRSIRysvLNTkENRJ5xWVYefg61h+/gRK5Qllua26Cyb29MK5bC1iYMu8norrj2ERERIZIo2+6vXr1Ujuo5efn49q1aygqKoKfnx/s7OzqGh81UCUVwOeHkrH2t5soKP33i42FqREm9GiJV3t6wtbcRI8RElFjw7GJiIgMkUYJV0JCQo3biouLMWPGDOzduxdxcXGaxkUNVGl5Bb45fhPLzxihqDxZWW5qJMaYLi3wRl8vSK0keoyQiBorjk1ERGSItL46gYWFBf7v//4Ptra2mDZtmrZ3TwZKEATsPZ+B/rFHsOjXKygqr/yV2UgswugAdyRM64PoQW2YbBGRXnBsIiIifdHZxTM9e/bEhg0bdLV7MiAX02VY8MtFJF7PVSl/ob0T3g3xRUuppZ4iIyJSxbGJiIjqm84SruzsbBQWFupq92QAcgpL8cn+q9j6e6rKEu9dWjZBT+tsTBrRASYmvE6LiAwHxyYiIqpvWk+4FAoFNm7ciK1bt6JTp07a3j0ZgLJyBb45noL/i7+msiBG86YWmPlcazzbqil+/fVXPUZIRKSKYxMREemLRgmXp6en2vLy8nJkZWVBLpfDxMQEMTExdQqODIsgCIi/lIUP91zCjZwiZbmVxBgRz3ojvLsHJMZGkMvleoySiJ5UHJuIiMgQabRohkKhgCAI1R4mJiZo164dJk2ahNOnT6N3794aB7ZixQp4eHjAzMwMgYGBOHXqVI11V61ahZ49e6JJkyZo0qQJgoKCHlqfau9qZgHC1p7Cq9/+oUy2RCIgtJM7Dr7XG5N7e/HGxUSkV/UxNhEREdWWRme4UlJStByGqq1btyIyMhIrV65EYGAgli1bhpCQEFy5cgUODg7V6ickJGD06NHo1q0bzMzM8NFHHyE4OBgXLlyAq6urTmNt7O4VleHTA1ex8WQqKh64UCvAoymiB7VBO1dbPUZHRPQvXY9NK1aswJIlS5CRkQE/Pz989tlnCAgIeGS7LVu2YPTo0RgyZAh27typ0xiJiMjw6GzRjLqIjY3FxIkTER4eDgBYuXIldu/ejbVr12LGjBnV6m/cuFHl+erVq7Fjxw7Ex8cjLCysWv3S0lKUlpYqn8tkMgCAXC7XaDpcVZvGNJVOXqHAxlO38NnBZMhK/r1Oy9XODNNDWmFAW0eIRCK1r7kx9oc+sT+1i/2pfXXt04bwXtT2h8AqKSkpeO+999CzZ896jJaIiAyJRgnX7du3cebMGfTq1Qt2dnbVtt+7dw9Hjx6Fv79/rc8wlZWV4fTp04iKilKWicViBAUFITEx8bH2UVxcDLlcjqZNm6rdHhMTg3nz5lUr379/PywsLGoV74May800L94TYedNMTLvi5RlpmIB/V0V6ONcCCH1DH5NffR+Gkt/GAr2p3axP7VP0z4tLi7WyvF1OTbV9odAAKioqMArr7yCefPm4ejRo8jLy6vtSyIiokZAo4Rr4cKF2L59O9LT09Vut7CwwP/+9z+MGjUKn3/+ea32nZOTg4qKCjg6OqqUOzo64vLly4+1j+nTp8PFxQVBQUFqt0dFRSEyMlL5XCaTwd3dHcHBwbCxsalVvEDlr7NxcXHo379/g14G/fa9+5i76xIOX8tRKX+xozPe7e8DRxuzx9pPY+kPQ8H+1C72p/bVtU+rZhnUla7GJk1/CJw/fz4cHBwwYcIEHD169JHH0fbsiyo8q6se+0W32L+6xf7VLW33q0YJ18GDBxEcHAyJRKJ2u0QiQXBwMA4cOFCn4DSxePFibNmyBQkJCTAzU58gSCQStbGbmJjU6QtYXdvriyAI2H76NubvuojCB5Z592/RBNEvtIGfu51G+22o/WGo2J/axf7UPk37VFvvg67GJk1+CDx27BjWrFmDs2fPPvZxdDX7ogrP6qrHftEt9q9usX91Q1szL6polHClpaXhpZdeemidFi1aYNeuXbXet1QqhZGRETIzM1XKMzMz4eTk9NC2S5cuxeLFi3HgwAF06NCh1sd+EuUUliLqh3OIu/hvfzvbmmHGQF8M9nOBSCR6SGsiIsOhy7GpNgoKCjB27FisWrUKUqn0sdtpe/ZFFZ7VVY/9olvsX91i/+pWbm6uVvenUcJlamr6yCkgMplMoy/rpqam8Pf3R3x8PIYOHQqgcqnf+Ph4RERE1Nju448/xocffoh9+/bxppaPad+FDMz84Rxyi8qUZcP93RA9qA1szPiPl4gaFl2NTbX9ITA5ORkpKSkYNGiQskyhUAAAjI2NceXKFXh5eVVrp6vZF9reT2PDftEt9q9usX91Q9t9qtF9uNq3b49du3apzDV/UElJCX7++We0b99eo6AiIyOxatUqfPPNN7h06RJef/11FBUVKS9WDgsLU5lL/9FHH2H27NlYu3YtPDw8kJGRgYyMDBQWFmp0/MZOViLHu9v+wmvfnVYmW80sTfHVWH8sHeHHZIuIGiRdjU0P/hBYpeqHwK5du1ar7+vri3PnzuHs2bPKx+DBg9G3b1+cPXsW7u7utXthRETUoGmUcIWHh+P27dsYPHgwrl+/rrItOTkZQ4YMQXp6Ol599VWNggoNDcXSpUsRHR2Njh074uzZs9i7d69y/nxqairu3LmjrP/ll1+irKwMw4cPh7Ozs/KxdOlSjY7fmB1PzsHAZUex48xtZVn/No7Y904vhLR9+JRNIiJDpsuxqTY/BJqZmaFdu3YqDzs7O1hbW6Ndu3YwNTWt+4slIqIGQ6MpheHh4dizZw927NgBX19ftGzZEq6urkhLS8ONGzdQXl6O0NBQ5UCkiYiIiBqnECYkJKg81/XNLhuDEnkFPt57BWt/u6Ess5IYY86gNhju78ZrtYiowdPl2BQaGors7GxER0cjIyMDHTt2rPZDoFis0W+YRETUyGl84+Nt27ZhxYoV+OKLL3D58mVcu3YNANCmTRtMmTIFr7/+utaCpLo5dzsf72w7i6Ssf6dYdvFsiqUj/ODWpO4rXxERGQpdjk21+SHwv9avX6/xcYmIqGHTOOESiUTKwaeoqAj5+fmwtbWFpaWlNuOjOpBXKPDFoWR8dvAayhUCAMDUWIzpA3wR3s0DYjHPahFR48KxiYiIDI3GCdeDLC0tOZgZmOTsQkRuPYu/bucry9q52uDTkR3h42itx8iIiOoHxyYiIjIEGk04/+233xAZGYmMjAy12+/cuYPIyEicOHGiTsFR7SkUAtb/dgPPLT+qTLaMxCK81c8HP77RnckWETVaHJuIiMgQaZRwxcbGYteuXTXeiNjZ2Rm//PILPv300zoFR7WTnncfY9eexNxdF1FaXnnPF0+pJXa83g2R/VvBxIgXdBNR48WxiYiIDJFGUwp///139OvX76F1evXqhbi4OI2Coto7cT0Xk779A7KScmXZ+G4emD7AF+amRnqMjIiofnBsIiIiQ6RRwpWVlQVXV9eH1nFyckJWVpZGQVHtHLiYiSmbzijPajnbmmHJcD/08JHqOTIiovrDsYmIiAyRRgmXnZ0dUlNTH1rn5s2bsLKy0igoenw7Tt/G+zv+RsU/qxD2ecoey0c9DVtzEz1HRkRUvzg2ERGRIdLoop4uXbrgxx9/xK1bt9RuT01Nxc6dO9GtW7c6BUcPt/bYDby7/S9lsjWkowtWhXViskVETySOTUREZIg0SrgiIyNRXFyM7t2749tvv8WdO3cAVK4A9c0336B79+64f/8+3n33Xa0GS5UEQUDs/iuY/8tFZdm4ri3w6ciOXBiDiJ5YHJuIiMgQaTSlsFevXoiNjcW7776L8PBwAJU3mxSEyjMtYrEYy5cvR69evbQXKQGoXPZ9zs8X8N2Jm8qyt/v5YGqQD0Qi3siY6o9cLkdFRYW+w6g1uVwOY2NjlJSUNMj4DdF/+9TIyAgmJvV/pp1jExERGSKNb3z89ttvo2/fvli5ciV+//135Ofnw87ODgEBAZg8eTLatWuH0tJSSCQSbcb7RCsrV+C97X/h57/SlWVzBrVBePeWeoyKnjQymQw5OTkoLS3VdygaEQQBTk5OuHXrFn+k0BJ1fSqRSCCVSmFjY1OvsXBsIiIiQ6NxwgUAHTp0wBdffFGt/MyZM5gyZQq2bNmC3NzcuhyC/nG/rAKvbzyNhCvZACpvZrx0RAe8+LSbniOjJ4lMJkNaWhqsrKwglUphYmLS4JIWhUKBwsJCWFlZQSzmFFxteLBPRSIR5HI58vPzkZaWBgD1nnRxbCIiIkNSp4TrQXl5ediwYQPWrFmDv//+G4IgwNzcXFu7f6Ll35djwvrf8cfNewAAibEYX7zyDPq1dtRzZPSkycnJgZWVFdzc3BpcolVFoVCgrKwMZmZmTLi05L99am5uDmtra9y+fRs5OTn1nnA9iGMTERHpW50TrgMHDmDNmjX46aefUFpaCkEQ0LVrV4SHhyM0NFQbMT7RsmQlCFt7CpczCgAA1hJjrB7XCYGezfQcGT1p5HI5SktLIZVKG2yyRfVHJBLB1tYWaWlpkMvl9X5NF8cmIiIyFBolXLdu3cK6deuwbt06pKamQhAEuLq6Ii0tDePHj8fatWu1HecTKTW3GGPWnETq3WIAgNTKFOvDA9DO1VbPkdGTqGqBCX0shkANU9VnpaKiol4+NxybiIjIED12wiWXy7Fz506sWbMG8fHxqKiogKWlJV555RWEhYXh2WefhbGxMYyNtTZL8Yl2OUOGsDWnkFVQuTCBq505NrwaiJZSSz1HRk86nt2ix1UfnxWOTUREZOgeewRycXHB3bt3IRKJ0LdvX4SFhWHYsGGwtGQCoG2nb95D+LpTkJWUAwB8HKzw7YQAONvyugMiogdxbCIiIkP32AlXbm4uxGIx3nnnHbz//vuwt7fXZVxPrMNXszH5u9O4L6+cvuXnbof14zujiaWpniMjIjI8HJuIiMjQPfYSXePHj4e5uTliY2Ph5uaGwYMHY/v27SgrK9NlfE+UX/5Ox6vf/K5Mtrp7N8PGVwOZbBER1YBjExERGbrHTrjWrl2LO3fu4KuvvsIzzzyDX375BaNGjYKjoyNee+01HDt2TJdxNnqbTqbizc1/Ql4hAAAGtnPC2vGdYSXhdQdETzKRSIQ+ffroOwyDxbGJiIgMXa1uQmNlZYVXX30ViYmJuHDhAqZOnQpTU1OsWrUKvXv3hkgkwpUrV3Dz5k1dxdsoJSbnYuaP5yBU5loI7eSOz19+BhJjI/0GRkQqUlJSIBKJHvrIy8ur15hu3rwJIyMjiEQiLFmypF6PbSjqa2xasWIFPDw8YGZmhsDAQJw6darGuqtWrULPnj3RpEkTNGnSBEFBQQ+tT0REjZfGd/1s3bo1PvnkE6SlpWHbtm0IDg6GSCTC0aNH4eXlhX79+uG7777TZqyNUom8Ah/8eE75fGLPllj8UnsYibkSHJGh8vLywpw5c9Q+zMzM6jWWtWvXQqFQQCQScdlz6G5s2rp1KyIjIzFnzhycOXMGfn5+CAkJQVZWltr6CQkJGD16NA4dOoTExES4u7sjODgYaWlpdX2JRETUwNR5vpqxsTGGDx+O4cOH4/bt21i3bh3Wr1+PQ4cOISEhAWPHjtVGnI3WFwnJuJ5TBADwb9EEUQNbc9ltIgPn7e2NuXPn6jsMKBQKrF+/HlKpFC+88ALWr1+P48ePo1u3bvoOTe+0PTbFxsZi4sSJCA8PBwCsXLkSu3fvxtq1azFjxoxq9Tdu3KjyfPXq1dixYwfi4+MRFham+QsjIqIGR6sXCLm5uWH27NmYPXs24uPj+WvrIyRlFeDLhCQAgLFYhEUvtoeYZ7aIGoW///4bixYtwuHDh5GbmwtnZ2cMHjwYc+fORbNmzarVX716NZYtW4akpCTY29tj9OjRmD9//kOPERcXh9TUVERERCA0NBTr16/HmjVrVBKuBQsWIDo6Gt98843aL/o//PADXnrpJcycORMffvihSvmiRYtw4cIF2NjYYPDgwfj444/x9NNPA6icXtlQ1HVsKisrw+nTpxEVFaUsE4vFCAoKQmJi4mPto7i4GHK5HE2bNq2xTmlpKUpLS5XPZTIZgMp7jcnl8lrF/KCqtnXZR2PEftEt9q9usX91S9v9qrMVGfr164d+/frpavcNnkIhYOYP55WLZLzW2xNPOVnrOSoi0oaff/4ZI0eOhFgsxpAhQ+Du7o6LFy/i888/x759+3Dy5Ek0adJEWb8qKXJ0dMTEiRNhYmKCrVu34tKlSw89zpo1awAAYWFh6Ny5Mzw9PbFt2zYsX74cVlZWAIAxY8Zgzpw52LBhg9qEq2p63YNnfNauXYsJEybAxsYGYWFhsLW1xZ49e9C/f3/I5XKYmJjUuY/0RZOxKScnBxUVFXB0dFQpd3R0xOXLlx9rH9OnT4eLiwuCgoJqrBMTE4N58+ZVK9+/fz8sLCxqFbM6cXFxdd5HY8R+0S32r26xf3WjuLhYq/vjEnh6su2PWziVchcA0KKZBd581kfPERHVzaDPjiG7oPTRFfXI3lqCn6bUfbpdUlKS2imFAwYMgI+PD8aOHQupVIrffvsNLVq0UG7fsmULRo8ejejoaHz22WfKfc2fPx+urq44c+YMHBwcAABz585FQEBAjTHk5ubip59+gq+vLzp37gygMrmaP38+tm7digkTJgAAWrZsie7du+PgwYO4c+cOnJ2dlfu4e/cu9uzZg06dOsHX1xcAkJeXh7fffhuWlpb4448/4ONT+X/TokWLEBISgtOnT6u8Jnq0xYsXY8uWLUhISHjoNX5RUVGIjIxUPpfJZMprv2xsbDQ+vlwuR1xcHPr379+gk2VtY7/oFvtXt9i/upWbm6vV/THh0oPsglIs2vPvL9cfDm0PMxOuSEgNW3ZBKTJkJfoOo14kJyerPRNhZ2eHxMREyGQyfP7559USk1GjRmHJkiXYsmWLMuHatGkTysvLERkZqUy2AMDGxgazZs2q8Vqj7777DmVlZSrbw8LCMH/+fKxZs0aZcAGVZ6+OHTuGzZs3q3yh37p1K8rKyjBmzBhl2U8//YTCwkK89dZbymQLqLwmauHChU/k9WFSqRRGRkbIzMxUKc/MzISTk9ND2y5duhSLFy/GgQMH0KFDh4fWlUgkkEgk1cpNTEy08oVKW/tpbNgvusX+1S32r25ou0+ZcOnBgl8uQlZSDgB48WlX9PCR6jkiorqzt67+RdHQaCvGkJAQ7N27V+220NBQAMDJkyeRnJxcbXtJSQlycnKQk5MDqVSKv/76CwDQs2fPanXVlVVZs2YNRCKRSrLk5eWFbt264fjx47h06RJat24NABg5ciTeeustfPfddyoJ14YNG2BsbIzRo0cry6ri6dGjR7VjBgYGwtj4yRs2TE1N4e/vj/j4eAwdOhRA5YIl8fHxiIiIqLHdxx9/jA8//BD79u1Dp06d6ilaIiIyNE/eyKlnh69m4+e/0gEAdhYmmPV8az1HRKQdu96s/gXdECkUCp3u/+7dyqnCK1aseGi9oqIiSKVS5OfnA4DK2a0q/71mqMrJkydx/vx59O3bF82bN1fZFhYWhuPHj2Pt2rXK+3LZ2dnhhRdewI4dO3Dx4kW0adMGycnJOH78OJ577jmVY1ct1KAuHrFYDKn0yfyBKDIyEuPGjUOnTp0QEBCAZcuWoaioSLlqYVhYGFxdXRETEwMA+OijjxAdHY1NmzbBw8MDGRkZACrvGVZ1fR0REdWdXC5HRUXFI+sZGRnp7WwgE656dL+sArN2/nvPrZnPtUYzK8M/K0BEj6/qWptz586hXbt2j6xva2sLAMjKyqo2BfG/U9iqVC2WcejQoRpvI/Htt99i0aJFysFl7Nix2LFjB7777jvExMRgw4YNynJ18au7v5RCoUBOTg5cXV0f+boam9DQUGRnZyM6OhoZGRno2LEj9u7dq0yKU1NTIRb/e2vLL7/8EmVlZRg+fLjKfubMmWMQtxQgImroZDIZcnJyVFZ3fRSJRAKpVFqn62I1wYSrHi2Pv4Zbd+8DALp4NsUIfzc9R0RE2hYYGIgffvgBiYmJj5Vw+fn54YcffsDRo0eVi19UOXr0aLX6RUVF2LJlCywsLFSmAj7o999/x99//41ffvkFL774IgDgueeeQ7NmzbBp0yZ8+OGH2LhxI6ytrTFkyJBq8QDAb7/9hhEjRqhsO3XqFMrLyx/5mhqriIiIGqcQJiQkqDxvSMvmExE1NDKZDGlpabCysoJUKoWJiclD72MrCALkcjny8/OVN6Cvz6RL/Ogq9W/FihXw8PCAmZkZAgMDcerUqRrrXrhwAS+99BI8PDwgEomwbNmy+gu0Fi7dkWHV0esAAFMjMT58sT1vcEzUCIWHh8Pa2hoffPABLly4UG17cXExTpw4oXz+8ssvw8jICLGxsSpnlWQyGRYuXFit/fbt21FQUIDhw4dj9erVah9VUwmrzoQBlRcAh4aGIjU1FR9//DGuXbuGl156Cebm5ir7HzJkCKysrLBmzRqVa9DKy8sxe/ZszTuGiIhIS3JycmBlZQU3NzfY2NjA3NwcZmZmNT7Mzc1hY2MDNzc3WFlZIScnp17jNbiEa+vWrYiMjMScOXNw5swZ+Pn5ISQkRO30FqDyy4unpycWL178yNWi9KVCISDqh3OoUFTec+uNvl7wsuccfqLGyN7eHps3b0ZhYSH8/Pzwwgsv4L333sObb76JQYMGwcnJSWVKmbe3N6Kjo5GWloYOHTrgrbfeQmRkJNq3b6+ySmCVqiSq6tohdYKCguDm5oa9e/ciPT1dWV41fTA6Olrl+YPs7OwQGxuLwsJC+Pv7Y/LkyZg+fTqefvpp3Lt3Dy4uLipT54iIiOqTXC5HaWkpbG1ta33yQiQSwdbWFqWlpfV602iDm1IYGxuLiRMnKr9MrFy5Ert378batWsxY8aMavU7d+6snIajbrs6paWlKvM9qy4Sl8vlGnX+o+72veFkKs7eygMAeEot8Wr3Fo36zuC8+7l2GUp/yuVyCIIAhUKh84UndEkQBOWftX0dVfUf1XbgwIE4ffo0li5divj4eMTFxcHS0hJubm4YP348XnnlFZX2s2bNgpOTE5YvX46vvvoKDg4OCA0Nxbx585QLLCgUCly5cgXHjh1Dy5Yt0bNnz4fGEBYWhkWLFmHdunWIiooCAAQEBMDHxwfXrl2Dm5sbevXqpXYfEyZMgK2tLRYvXoz169fD1tYWgwYNwuLFi9GyZUt4eXmptKupTxUKhXIah5FRzbe+0Pdnm4iIGo6qBTI0XQCjql1FRUW9LaJhUAlXWVkZTp8+rfxyAFSuihUUFITExEStHScmJkbtPXT2798PCwsLjfer7m7feaXA4r+MAFRm4M875iN+v/rlpBsb3v1cu/Tdn8bGxnByckJhYSHKysr0Gos2FBQU1LpN06ZNce/ePQD//lBTE2dnZ3zyySc1bv9v+5EjR2LkyJEqZXK5XOV4zs7OyuePin/atGmYNm1atWM9OEW7sLCwxvbBwcEIDg5WKbt+/ToKCwvh6emp9vX/N6aysjLcv38fR44ceei1X8XFxQ99LURERP+l6aU5+rikx6ASrpycHFRUVFRbCtnR0RGXL1/W2nGioqJU7kUjk8ng7u6O4OBgjS6ge9jdviM2n0VpReV0yBH+rnhraNu6Bd8A8O7n2mUo/VlSUoJbt27BysoKZmZmeoujrgRBQEFBAaytrXkdZQ3u3bsHCwsLlZvw3r9/XzkV8aWXXlL5v7KmPi0pKYG5uTl69er10M/Mo5JXIiKihsygEq76IpFIVL5IVKnr3br/2/7AxUzsu1iZbEmtTPHB822eqASEdz/XLn33Z0VFBUQiEcRicYO+hqdqylvVa6Hqjh49igkTJiA4OBjNmzdHTk4ODh48iJSUFDz77LMYPXq0St/V1KdisRgikeiRn13+P0FERI2ZQSVcUqkURkZG1e49k5mZabALYtSkqLQc0T+dVz6f/UIb2FmY6jEiIqLH07ZtW/Tv3x+//fYbdu7cCaBycY8FCxbgvffeY6JKRERUCwaVcJmamsLf3x/x8fEYOnQogMpfTuPj42u894mh+mT/VaTnlwAAevpIMdjPRc8RERE9Hh8fH2zZskXfYRARETUKBpVwAUBkZCTGjRuHTp06ISAgAMuWLUNRUZFy1cKwsDC4uroiJiYGQOVF2RcvXlT+PS0tDWfPnoWVlRW8vb318hrO3c7H+uM3AAASYzEWDm3Ha0WIiIiIiJ5ABpdwhYaGIjs7G9HR0cjIyEDHjh2xd+9e5UIaqampKtNZ0tPT8fTTTyufL126FEuXLkXv3r2RkJBQ3+GjvEKBGT/8jX9uuYW3g3zQopllvcdBRERERET6Z3AJFwBERETUOIXwv0mUh4eH8h4whmD98RRcSK9cccvXyRoTe3rqOSIi7TKkf29k2PhZISIiXdF0jNHH2MQrn7UoLe8+Ptl/FQAgEgGLhrWHiRG7mBqHqhvX8ia19LiqPisPu+kxERFRbdT1+4g+xiZmA1oiCMC8Xy7hvrzy7tdjAlvgmeZN9BwVkfaYmJhAIpEgPz+fZy7okQRBQH5+PiQSCZd9JyIiranL9xF9jU0GOaWwIfrrrgiHruYAABysJZg24Ck9R0SkfVKpFGlpabh9+zZsbW1hYmLS4BaEUSgUKCsrQ0lJCZc315IH+1QkEkEulyM/Px+FhYVwdXXVd3hERNTI1Pb7iCAIeh2bmHBpQUGJHDtu/PvFbd7gtrAx4y+61PjY2NgAAHJycpCWlqbnaDQjCALu378Pc3PzBpcsGip1fSqRSODq6qr8zBAREWmLpt9H9DU2MeHSgqVx1yCTV37J6OfrgAHtGtZNmolqw8bGBjY2NpDL5aioqNB3OLUml8tx5MgR9OrVi1PdtOS/fWpkZMS+JSIinart9xF9jk1MuOro9M172Pz7bQCAhakR5vOeW/SEMDExaZBfqo2MjFBeXg4zM7MGGb8hYp8SEZG+NITvI7yAoY7EIqBlMwsAwNR+3nC1M9dzREREREREZCiYcNXR082b4Ocp3TC8ZQXGBrrrOxwiIiIiIjIgTLi0QGIsRk8nAca85xYRERERET2AGQIREdFjWLFiBTw8PGBmZobAwECcOnXqofW3b98OX19fmJmZoX379tizZ089RUpERIaECRcREdEjbN26FZGRkZgzZw7OnDkDPz8/hISEICsrS23948ePY/To0ZgwYQL+/PNPDB06FEOHDsX58+frOXIiItI3JlxERESPEBsbi4kTJyI8PBxt2rTBypUrYWFhgbVr16qtv3z5cgwYMADTpk1D69atsWDBAjzzzDP4/PPP6zlyIiLSNy4Lj8qbdgKATCbTqL1cLkdxcTFkMpnBL0tZH9gf2sX+1C72p/bVtU+r/u+t+r/Y0JSVleH06dOIiopSlonFYgQFBSExMVFtm8TERERGRqqUhYSEYOfOnTUep7S0FKWlpcrn+fn5AIC7d+9CLpdrHH/V+5Obm8vP/APYL7rF/tUt9q9u3b17F4D2xiUmXAAKCgoAAO7uXGWQiEhfCgoKYGtrq+8wqsnJyUFFRQUcHR1Vyh0dHXH58mW1bTIyMtTWz8jIqPE4MTExmDdvXrXyli1bahA1ERHVVW5urlbGJSZcAFxcXHDr1i1YW1trdNNimUwGd3d33Lp1CzY2NjqIsGFhf2gX+1O72J/aV9c+FQQBBQUFcHFx0UF0DUdUVJTKWTGFQoG7d++iWbNmGo1NVfiZV4/9olvsX91i/+pWfn4+mjdvjqZNm2plf0y4UDk1xM3Nrc77sbGx4Yf+AewP7WJ/ahf7U/vq0qeGeGarilQqhZGRETIzM1XKMzMz4eTkpLaNk5NTreoDgEQigUQiUSmzs7PTLGg1+JlXj/2iW+xf3WL/6pZYrJ3lLrhoBhER0UOYmprC398f8fHxyjKFQoH4+Hh07dpVbZuuXbuq1AeAuLi4GusTEVHjxTNcREREjxAZGYlx48ahU6dOCAgIwLJly1BUVITw8HAAQFhYGFxdXRETEwMAePvtt9G7d2988skneP7557Flyxb88ccf+Prrr/X5MoiISA+YcGmBRCLBnDlzqk0FeVKxP7SL/ald7E/texL6NDQ0FNnZ2YiOjkZGRgY6duyIvXv3KhfGSE1NVZl60q1bN2zatAmzZs3CzJkz4ePjg507d6Jdu3b1HvuT8P5ogv2iW+xf3WL/6pa2+1ckGOo6vERERERERA0cr+EiIiIiIiLSESZcREREREREOsKEi4iIiIiISEeYcBEREREREekIE646OHLkCAYNGgQXFxeIRCLs3LlT3yHpzdy5cyESiVQevr6++g6rQXnU50kQBERHR8PZ2Rnm5uYICgrCtWvX9BNsA/Co/hw/fny1z+yAAQP0E2wDEBMTg86dO8Pa2hoODg4YOnQorly5olKnpKQEU6ZMQbNmzWBlZYWXXnqp2s1/qf5wjFKP45V2cezSLY5lulVfYxsTrjooKiqCn58fVqxYoe9QDELbtm1x584d5ePYsWP6DqlBedTn6eOPP8b//d//YeXKlTh58iQsLS0REhKCkpKSeo60YXicf58DBgxQ+cxu3ry5HiNsWA4fPowpU6bgxIkTiIuLg1wuR3BwMIqKipR13nnnHezatQvbt2/H4cOHkZ6ejmHDhukx6icbx6iacbzSHo5dusWxTLfqbWwTSCsACD/++KO+w9CbOXPmCH5+fvoOo9H47+dJoVAITk5OwpIlS5RleXl5gkQiETZv3qyHCBsWdf8+x40bJwwZMkQv8TQGWVlZAgDh8OHDgiBUfh5NTEyE7du3K+tcunRJACAkJibqK0z6x5M+Rj2I45XucOzSLY5luqersY1nuEhrrl27BhcXF3h6euKVV15BamqqvkNqNG7cuIGMjAwEBQUpy2xtbREYGIjExEQ9RtawJSQkwMHBAU899RRef/115Obm6jukBiM/Px8A0LRpUwDA6dOnIZfLVT6jvr6+aN68OT+jZHA4XtUPjl31g2OZ9uhqbGPCRVoRGBiI9evXY+/evfjyyy9x48YN9OzZEwUFBfoOrVHIyMgAADg6OqqUOzo6KrdR7QwYMADffvst4uPj8dFHH+Hw4cMYOHAgKioq9B2awVMoFJg6dSq6d++Odu3aAaj8jJqamsLOzk6lLj+jZGg4XtUfjl26x7FMe3Q5thlrM1B6cg0cOFD59w4dOiAwMBAtWrTAtm3bMGHCBD1GRqTeqFGjlH9v3749OnToAC8vLyQkJKBfv356jMzwTZkyBefPn+d1L9QgcbyixoRjmfbocmzjGS7SCTs7O7Rq1QpJSUn6DqVRcHJyAoBqq+JkZmYqt1HdeHp6QiqV8jP7CBEREfjll19w6NAhuLm5KcudnJxQVlaGvLw8lfr8jJKh43ilOxy76h/HMs3oemxjwkU6UVhYiOTkZDg7O+s7lEahZcuWcHJyQnx8vLJMJpPh5MmT6Nq1qx4jazxu376N3NxcfmZrIAgCIiIi8OOPP+LgwYNo2bKlynZ/f3+YmJiofEavXLmC1NRUfkbJoHG80h2OXfWPY1nt1NfYximFdVBYWKjyC8KNGzdw9uxZNG3aFM2bN9djZPXvvffew6BBg9CiRQukp6djzpw5MDIywujRo/UdWoPxqM/T1KlTsXDhQvj4+KBly5aYPXs2XFxcMHToUP0FbcAe1p9NmzbFvHnz8NJLL8HJyQnJycl4//334e3tjZCQED1GbbimTJmCTZs24aeffoK1tbVy7rqtrS3Mzc1ha2uLCRMmIDIyEk2bNoWNjQ3efPNNdO3aFV26dNFz9E8mjlHqcbzSLo5dusWxTLfqbWzT8mqKT5RDhw4JAKo9xo0bp+/Q6l1oaKjg7OwsmJqaCq6urkJoaKiQlJSk77AalEd9nhQKhTB79mzB0dFRkEgkQr9+/YQrV67oN2gD9rD+LC4uFoKDgwV7e3vBxMREaNGihTBx4kQhIyND32EbLHV9CUBYt26dss79+/eFN954Q2jSpIlgYWEhvPjii8KdO3f0F/QTjmOUehyvtItjl25xLNOt+hrbRP8cjIiIiIiIiLSM13ARERERERHpCBMuIiIiIiIiHWHCRUREREREpCNMuIiIiIiIiHSECRcREREREZGOMOEiIiIiIiLSESZcREREREREOsKEi4iIiIiISEeYcBFRrYlEIvTp00ffYRAREQHguESGjQkXkQ6kpKRAJBKpPExMTODq6oqRI0fijz/+0HeIRET0BOG4RKQ/xvoOgKgx8/LywpgxYwAARUVFOH36NLZv346dO3fiwIED6NWrl54jJCKiJwnHJaL6x4SLSIe8vb0xd+5clbLFixcjKioKs2fPxuHDh/UTGBERPZE4LhHVP04pJKpnEyZMAACcPn1apTwnJwdTp05Fy5YtIZFI4ODggJEjR+L8+fPV9tGnTx+IRCK1+x8/fjxEIhFSUlKUZevXr4dIJML69euxf/9+dOvWDRYWFmjWrBnGjRuH3NxctftavXo12rVrBzMzM7i7u+P9999HSUmJhq+ciIgMEcclIt3iGS4iPTE2/vefX3Z2Nrp27Yrk5GT06dMHo0aNwo0bN/D9999j9+7d2LdvH3r06FHnY/7888/YvXs3Bg0ahG7duuHIkSP49ttvkZycjGPHjqnUXbBgAaKjo+Ho6IiJEyfCxMQEW7duxaVLl+ocBxERGR6OS0S6wYSLqJ6tXr0aAFQGqunTpyM5ORlRUVFYtGiRsnzPnj14/vnnER4ejitXrkAsrttJ6V27diEhIQHdu3cHAFRUVCAoKAgJCQk4ceIEunTpAgBISkrC/Pnz4erqijNnzsDBwQEAMHfuXAQEBNQpBiIiMiwcl4h0i1MKiXQoKSkJc+fOxdy5czFt2jQ8++yzmDlzJhwdHbFkyRIAQFlZGTZv3oxmzZph1qxZKu2fe+459O/fH0lJSfjtt9/qHM/LL7+sHNQAwMjICOPGjQMA/P7778ryTZs2oby8HJGRkcpBDQBsbGyqxUhERA0HxyWi+sczXEQ6lJycjHnz5qmUOTk54ejRo/D29gYAXL58GSUlJejbty8sLCyq7aNv376Ii4vD2bNn0bNnzzrF4+/vX63Mzc0NAJCXl6cs++uvvwBA7fHqGgMREekPxyWi+sczXEQ6FBISAkEQIAgCsrKysGTJEmRlZWHw4MEoLCwEAMhkMgCAo6Oj2n04Ozur1KsLGxubamVVc/YrKiqUZfn5+QCg8itilZriJCIiw8dxiaj+MeEiqif29vZ47733MHPmTFy6dEk5BaJqsMnMzFTbLiMjQ6UeAOWc+fLy8mr1qwalurC1tQUAZGVlVdtWU5xERNSwcFwiqh9MuIjq2cyZM+Hi4oIvvvgCKSkp8PX1hZmZGX7//XcUFxdXq5+QkAAA6Nixo7KsSZMmAIC0tDSVugqFQjntoi78/PwAAEePHq22TV0ZERE1XByXiHSLCRdRPTM3N8f06dMhl8uxYMECmJqaYvTo0cjJyUFMTIxK3b1792Lfvn3w9vZWuai4c+fOACrvY/Kg2NhY3Lhxo84xvvzyyzAyMkJsbKzKr4kymQwLFy6s8/6JiMhwcFwi0i0mXER6MGnSJLi4uCjvNfLRRx/B09MTCxcuRL9+/TBz5ky8/PLLGDRoECwsLLBu3TqVpXfDw8PRpEkTzJ07Fy+++CLee+899OnTB4sXL0bv3r3rHJ+3tzeio6ORlpaGDh064K233kJkZCTat28PHx+fOu+fiIgMC8clIt1hwkWkB2ZmZoiKikJ5eTnmzZsHe3t7nDx5Em+99RaSk5OxdOlSxMXFYejQoTh58mS1m0s6Ojri0KFD6NevH/bv349Vq1bBzs4OJ06cgIeHh1ZijI6OxqpVq9CsWTN89dVX2L59O0aOHIlt27ZpZf9ERGQ4OC4R6Y5IEARB30EQERERERE1RjzDRUREREREpCNMuIiIiIiIiHSECRcREREREZGOMOEiIiIiIiLSESZcREREREREOsKEi4iIiIiISEeYcBEREREREekIEy4iIiIiIiIdYcJFRERERESkI0y4iIiIiIiIdIQJFxERERERkY4w4SIiIiIiItKR/wcClh2xYZMmhQAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1MAAAD0CAYAAAB+SXxXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8WgzjOAAAACXBIWXMAAA9hAAAPYQGoP6dpAABrxElEQVR4nO3deVxU1fsH8M+wg4obmyCBqIm7hfueG6aZZpo7aoqpoCmZWwVuZWWplVu5fnPfyiVNJdTURNPU0nJFcUEBQQVlHZjz++P+ZmCYAYZhhhng83695iVz7vZwGOeZZ+6558qEEAJERERERERUJBamDoCIiIiIiKg0YjFFRERERESkBxZTREREREREemAxRUREREREpAcWU0RERERERHpgMUVERERERKQHFlNERERERER6YDFFRERERESkBxZTREREREREemAxRURERKSnzp07o3PnziV+3A0bNkAmk+H8+fMlfmxzFh0dDZlMhg0bNqja5syZA5lMZrKYTPUaoZLBYopMIioqCu+99x58fHxgZ2cHR0dHtGvXDt988w3S0tJU63l7e+ONN95Q21Ymk2l9uLm5qa337Nkz2NnZQSaT4erVq1rjGDVqlNo+bG1t8fLLLyM0NBTp6eka62/fvh3Dhw9H3bp1IZPJCnxzzMjIwIwZM+Du7g57e3u0atUK4eHhWtc9ffo02rdvDwcHB7i5uWHy5Ml48eJFvvvWV94+c3R0RKdOnXDgwAGDH0tJmdhkMhl2796tsVyZ5BISEoq879OnT2POnDl49uyZxrLOnTtrfZ307NlTY92i/K2IqHTTNf/o6+HDh5gzZw4uXbpU/GAN5M8//8TEiRPh5+cHa2vrQguLtWvXon79+rCzs0PdunXx3Xffaaxz/fp1TJ06FW3btlXl2ujoaCP9BpLjx4+jf//+cHNzg42NDVxcXNCnTx/89NNPRj0uACgUCjg7O+PLL7+ElZUVhg8fnu+6z58/h729Pfr376+xLL/clPcxZ84cjW1z59PcebxZs2ZYtmwZsrOzDfkrk46sTB0AlT8HDhzAwIEDYWtri4CAADRq1AiZmZk4deoUPvzwQ/z777/44YcfCtxH9+7dERAQoNZmb2+v9nznzp2qImvz5s1YsGCB1n3Z2tpizZo1AICkpCTs3bsX8+fPR1RUFDZv3qy27sqVK/HXX3+hRYsWSExMLDDGUaNGYdeuXZgyZQrq1q2LDRs2oFevXjh27Bjat2+vWu/SpUvo2rUr6tevj8WLF+PBgwf46quvcPPmTfz6668FHkMfyr4TQuDu3btYuXIl+vTpg19//RX+/v4GP15u8+bNQ//+/Q32DeHp06cxd+5cjBo1ClWqVNFYXrNmTSxcuFCtzd3dXWM9Xf9WRFS6GSL/FObhw4eYO3cuvL290axZM8MEXkwHDx7EmjVr0KRJE/j4+ODGjRv5rvv9999j/PjxePvttxESEoKTJ09i8uTJSE1NxYwZM1TrRUZG4ttvv0WDBg1Qv359oxePYWFhmDdvHurWrYv33nsPXl5eSExMxMGDB/H2229j8+bNGDp0qNZtP/74Y8ycObNYx//zzz+RkJCA3r1749ixY9i7dy9SU1Ph4OCgse5PP/2E9PR0VcF15MgR1bKPPvoIY8eOVT0/d+4cvv32W8yePRv169dXtTdp0iTfWIYMGYJevXoBkD63HDx4EJMmTcLdu3exaNGiYv2epAdBVIJu374tKlasKHx9fcXDhw81lt+8eVMsXbpU9dzLy0v07t1bbR0AIigoqNBjdezYUfTv319MnTpV1KpVS+s6I0eOFBUqVFBrUygUonXr1kImk4nY2Fi1Zffu3RPZ2dlCCCEaNmwoOnXqpHW/Z8+eFQDEokWLVG1paWmidu3aok2bNmrrvv7666JGjRoiKSlJ1bZ69WoBQBw+fLjQ37MotPXdf//9JwCI119/3aDHUrpz544AIJo1ayYAiN27d6stDwsLEwDE48ePi7zvRYsWCQDizp07Gss6deokGjZsWOg+ivK3IqLSq6j5R1edOnVSywXnzp0TAMT69euLEW3h1q9fLwCIc+fOFbpubGysSE1NFUIIERQUJPL7+JeamiqqV6+ukXeHDRsmKlSoIJ48eaJqS0xMFMnJyUKIgt+LDWHnzp0CgBgwYIDIzMzUWH7o0CGxf/9+IUROzjF0/3/yySfCy8tLCCHExo0bBQCxdetWrev26NFDVK5cWaSnpxe6X+XvduzYsULXVf5uufOVENLnlhYtWgh3d/dC90GGx2F+VKK+/PJLvHjxAmvXrkWNGjU0ltepUwfvv/9+sY9z7949nDx5EoMHD8bgwYNx584dnD59WqdtZTIZ2rdvDyEEbt++rbbM09MTFhaF/7fZtWsXLC0tMW7cOFWbnZ0dxowZg8jISNy/fx8AkJycjPDwcAwfPhyOjo6qdQMCAlCxYkXs2LFDp5iLo379+nByckJUVJRae0ZGBsLCwlCnTh3Y2trC09MT06dPR0ZGhtp64eHhaN++PapUqYKKFSuiXr16mD17tsZxBg8ejJdffhnz5s2DEKLQuM6ePYuePXuicuXKcHBwQKdOnfDHH3+ols+ZMwcffvghAKBWrVqqIQ95h5lkZWUVOGRS178VEZVuRc0/69evR5cuXeDi4gJbW1s0aNAAK1euLPAYx48fR4sWLQAAo0ePVr0v5b5+p7D3NqWYmBiMGTMG7u7usLW1Ra1atTBhwgRkZmaqrZeRkYGQkBA4OzujQoUKeOutt/D48WO1dVxdXTVGb2hz7NgxJCYmYuLEiWrtQUFBSElJURsSXq1aNVSqVKnQfRrCJ598gmrVqmHdunWwtrbWWO7v769xSUBu2q6ZkslkCA4OxubNm1GvXj3Y2dnBz88PJ06c0LqPAwcOoHfv3gCAt956CxUqVMCWLVs01ouPj0dERAQGDBgAW1tbAMa/Zkomk8HV1RVWVhxwZgrsdSpR+/fvh4+PD9q2bVus/aSnp2tcY1OpUiXVG9fWrVtRoUIFvPHGG7C3t0ft2rWxefNmnY+r/EBetWpVveK7ePEiXn75ZbUCCQBatmwJQBra5+npicuXLyMrKwvNmzdXW8/GxgbNmjXDxYsX9Tp+USQlJeHp06eoXbu2qk2hUODNN9/EqVOnMG7cONSvXx+XL1/GkiVLcOPGDezZswcA8O+//+KNN95AkyZNMG/ePNja2uLWrVtaPxhYWlri448/RkBAAH7++WetY8mVjh49itdffx1+fn4ICwuDhYWF6oPNyZMn0bJlS/Tv3x83btzA1q1bsWTJEjg5OQEAnJ2dVfu5ceMGKlSogMzMTLi6uiIwMBChoaFqyVjXvxURlW5FzT8rV65Ew4YN8eabb8LKygr79+/HxIkToVAoEBQUpHWb+vXrY968eQgNDcW4cePQoUMHAFAdU5f3NkAaKtiyZUs8e/YM48aNg6+vL2JiYrBr1y6kpqbCxsZGdcxJkyahatWqCAsLQ3R0NJYuXYrg4GBs3769yH2kzDl5c5Kfnx8sLCxw8eLFAq8VMoabN2/i2rVrePfddw1evP3+++/Yvn07Jk+eDFtbW6xYsQI9e/bEn3/+iUaNGqnWi42NxcWLFzFv3jwAQIUKFdC3b1/s2rULT548QbVq1VTrbt++HdnZ2Rg2bJhBY80tNTVV9RkoOTkZv/76Kw4dOoRZs2YZ7ZhUAFOfGqPyIykpSQAQffv21Xmb/Ib5aXvkPqXfuHFjMWzYMNXz2bNnCycnJyGXy9X2pRzm9/jxY/H48WNx69Yt8dVXXwmZTCYaNWokFApFvrEVNMyvYcOGokuXLhrt//77rwAgVq1aJYTIOb1/4sQJjXUHDhwo3Nzc8j2+PgCIMWPGiMePH4v4+Hhx/vx50bNnT41hAxs3bhQWFhbi5MmTatuvWrVKABB//PGHEEKIJUuWFDpEL/ewhKysLFG3bl3RtGlTVd/mHeanUChE3bp1hb+/v1r/p6amilq1aonu3bur2goaWvLuu++KOXPmiN27d4sff/xRvPnmmwKAeOedd9TW0/VvRUSllz75RzksLjd/f3/h4+Oj1qbrML+ivLcFBAQICwsLrUP4lNsqh/l169ZNbX9Tp04VlpaW4tmzZ1p/r4KG+QUFBQlLS0uty5ydncXgwYO1LjPmML+9e/cKAGLJkiU6ra9tmJ8yz+Sm/Oxw/vx5Vdvdu3eFnZ2deOutt9TWXbt2rbC3t1d7TRw4cEAAEN9//73auq1btxYeHh6qSwKE0HyN5KbPMD9tjwkTJhT4mYWMh8P8qMQkJycDgEG+Werbty/Cw8PVHsrJE/755x9cvnwZQ4YMUa0/ZMgQJCQk4PDhwxr7SklJgbOzM5ydnVGnTh1MmzYN7dq1w969e/WeKCEtLU11liw3Ozs71fLc/+a3riFmlspr7dq1cHZ2houLC5o3b46IiAhMnz4dISEhqnV27tyJ+vXrw9fXFwkJCapHly5dAEhDQQCoJn3Yu3cvFApFocdWnp36+++/VWe38rp06RJu3ryJoUOHIjExUXXslJQUdO3aFSdOnNDpWGvXrkVYWBj69++PESNGYO/evQgMDMSOHTtw5swZ1Xq6/q2IqPTSJ//kHhaXlJSEhIQEdOrUCbdv30ZSUlKRY9D1vU2hUGDPnj3o06ePxhkiABp5ady4cWptHTp0QHZ2Nu7evVvkGNPS0tTOeuVmrJxUGEN+dsirTZs28PPzUz1/6aWX0LdvXxw+fFhtZryDBw/itddeU3tN9OjRA87OzmpD/e7cuYMzZ85gyJAhOl0SoK9x48apPvvs3r0bQUFB+P7779XyOJUcDvOjEqMcRvX8+fNi76tmzZro1q2b1mWbNm1ChQoV4OPjg1u3bgGQkoC3tzc2b96sGvOsZGdnh/379wMAHjx4gC+//BLx8fE6jS/Pj729vca1RQBU060r9638N791C4shNjZW7XnlypUL3aZv374IDg5GZmYmzp07h88++wypqalqb/w3b97E1atX1YbM5RYfHw8AGDRoENasWYOxY8di5syZ6Nq1K/r3748BAwbkm0iGDRuG+fPnY968eejXr5/G8ps3bwIARo4cme/vkJSUpNcQzA8++ACrV6/Gb7/9htatWwPQ/W9FRKWXPvnnjz/+QFhYGCIjI5Gamqq2LCkpCZUrVy5SDLq+t2VmZiI5OVltmFlBXnrpJbXnyvfGp0+fFik+QHq/y3tNlpIuOUlXSUlJaoWZjY2N2lC53Az52SGvunXrarS9/PLLSE1NxePHj+Hm5ga5XI7w8HCNmWGtrKwwaNAgrFixAjExMfDw8FAVVsUd4peZmYknT56oteXOx3Xr1lX7DKScJXfp0qV499130bhx42Idn4qGxRSVGEdHR7i7u+PKlStGO4YQAlu3bkVKSgoaNGigsTw+Ph4vXrxAxYoVVW2WlpZqb0r+/v7w9fXFe++9h3379ukVR40aNRATE6PR/ujRIwA503MrL4JWtuddV9s03nmPk9v69esxatSoArfJXYj26tULTk5OCA4Oxmuvvaa6jkmhUKBx48ZYvHix1n0oryGyt7fHiRMncOzYMRw4cACHDh3C9u3b0aVLFxw5cgSWlpYa2yrPTo0aNQp79+7VWK4867Ro0aJ8pxXO/fcrCmXcuZOUrn8rIiq9ipp/oqKi0LVrV/j6+mLx4sXw9PSEjY0NDh48iCVLluh0djwvXd/b8n6ILoy291kAOk30k1eNGjWQnZ2N+Ph4uLi4qNozMzORmJhosPfD999/H//73/9Uzzt16oTjx49rXdfX1xcAcPnyZYMcu6hOnTqF5ORk1VTkuQ0fPhzLli3D1q1bMW3aNGzduhUNGjQo9pT4p0+fxmuvvabWdufOnQK36dq1K5YtW4YTJ06wmCphLKaoRL3xxhv44YcfEBkZiTZt2hh8/7///jsePHiAefPmqd2vAZC+pRs3bhz27NlT4AW0NWrUwNSpUzF37lycOXNGdQajKJo1a4Zjx44hOTlZbWKDs2fPqpYDQKNGjWBlZYXz58/jnXfeUa2XmZmJS5cuqbVpk/fGsg0bNixyrO+99x6WLFmCjz/+GG+99RZkMhlq166Nv//+G127di10qKOFhQW6du2Krl27YvHixfjss8/w0Ucf4dixY/mePRw+fDgWLFiAuXPn4s0331RbppwIw9HRMd/tlYo6DFM5O2Pub/h0/VsRUelWlPyzf/9+ZGRkYN++fWpnfpRDnAuS3/uSru9tzs7OcHR0NOoXj/lRvt+dP39erXg4f/48FAqFwd4Pp0+frpaHCxpp8PLLL6NevXrYu3cvvvnmG72/TNNGebYwtxs3bsDBwUGVJw4cOIAGDRrA29tbY91WrVqhdu3a2LJlC7p3745///0Xn376abHjatq0qUZ+d3Nz0xiNkltWVhYAFDh7LRkHr5miEjV9+nRUqFABY8eORVxcnMbyqKgofPPNN3rvXznE78MPP8SAAQPUHoGBgahbt67GjXi1mTRpEhwcHPD555/rFceAAQOQnZ2tdvPHjIwMrF+/Hq1atVKdIalcuTK6deuGTZs2qQ1h2LhxI168eIGBAwcWeJxu3bqpPbRN91sYKysrfPDBB7h69arqTNE777yDmJgYrF69WmP9tLQ0pKSkAIDWb1CVyVbb0Dkl5dmpS5cuaZz98/PzQ+3atfHVV19pTQq5p/ytUKECAODZs2dq6yQnJ2scXwihunFz7psT6/q3IqLSrSj5R3m2J/fZnaSkJKxfv77Q4+T3vqTre5uFhQX69euH/fv34/z58xrr6XPGSVddunRBtWrVNKaAX7lyJRwcHDSGyeurQYMGarkr93VL2sydOxeJiYkYO3asqmjI7ciRI/jll1+KHEdkZCQuXLigen7//n3s3bsXPXr0UL0GDh48WODvPWzYMFy8eBFhYWGQyWT53ji4KKpWraqR35XX8eZHeblC06ZNi318KhqemaISpfwGZ9CgQahfv77aHehPnz6NnTt3FjpMLT8ZGRnYvXs3unfvnu+bzptvvolvvvlGYwhDXtWrV8fo0aOxYsUKXL16VXWW68SJE6p7UDx+/BgpKSmqD+gdO3ZEx44dAUjfVg0cOBCzZs1CfHw86tSpg//973+Ijo7G2rVr1Y716aefom3btujUqRPGjRuHBw8e4Ouvv0aPHj3Qs2dPvfqiqEaNGoXQ0FB88cUX6NevH0aMGIEdO3Zg/PjxOHbsGNq1a4fs7Gxcu3YNO3bswOHDh9G8eXPMmzcPJ06cQO/eveHl5YX4+HisWLECNWvWRPv27Qs8pvLaqUuXLqm1W1hYYM2aNXj99dfRsGFDjB49Gh4eHoiJicGxY8fg6OioShrKBPzRRx9h8ODBsLa2Rp8+fXDhwgUMGTIEQ4YMQZ06dZCWloaff/4Zf/zxB8aNG4dXX31Vdbyi/K2IqPQqSv7p0aMHbGxs0KdPH7z33nt48eIFVq9eDRcXF63DsvMep0qVKli1ahUqVaqEChUqoFWrVqhVq5bO722fffYZjhw5osoL9evXx6NHj7Bz506cOnVKNfmPru7evYuNGzcCgKpAU+YuLy8vjBgxAoA0dHv+/PkICgrCwIED4e/vj5MnT2LTpk349NNP1a5rSkpKwnfffQcAqtthLFu2DFWqVEGVKlUQHBxcpBgLMmjQIFy+fBmffvopLl68iCFDhsDLywuJiYk4dOgQIiIitN7zqTCNGjWCv7+/2tTogFS8AdLQuqtXrxZ4f7Hhw4dj3rx52Lt3L9q1a6f1DJahXbhwAZs2bQIgXUsWERGB3bt3o23btujRo4fRj095mHQuQSq3bty4IQIDA4W3t7ewsbERlSpVEu3atRPfffed2h3D85saPSgoSGOfu3fvFgDE2rVr8z3u8ePHBQDxzTffCCFypkbXJioqSlhaWoqRI0eq2pTTq2p7hIWFqW2flpYmpk2bJtzc3IStra1o0aKFOHTokNZjnTx5UrRt21bY2dkJZ2dnERQUpLqzvCHl13dCCDFnzhy16VkzMzPFF198IRo2bChsbW1F1apVhZ+fn5g7d65ISkoSQggREREh+vbtK9zd3YWNjY1wd3cXQ4YMETdu3FDtN787tguRM7UvtEyvfvHiRdG/f39RvXp1YWtrK7y8vMQ777wjIiIi1NabP3++8PDwEBYWFqqpeW/fvi0GDhwovL29hZ2dnXBwcBB+fn5i1apVWqeOLcrfiohKN13zz759+0STJk2EnZ2d8Pb2Fl988YVYt26dxhTg2qa93rt3r2jQoIGwsrLSmKZb1/e2u3fvioCAAOHs7CxsbW2Fj4+PCAoKEhkZGUKInPfPvNOnHzt2TGOqbWWbtoe2Kbt/+OEHUa9ePWFjYyNq164tlixZovHeWdA03V5eXoX/IfSgzDkuLi7CyspKODs7iz59+oi9e/dqxKXL1OhBQUFi06ZNom7dusLW1la88sorav22bNkyUblyZY3bquTVokULAUCsWLFC63JjTo1uZWUlfHx8xIcffiieP39e6D7I8GRCGPF8MRERERGRmZHJZAgKCsKyZcvyXadXr16oWLEiduzYUYKRUWnDYX5ERERERHl07twZHTp0MHUYZOZYTBERERER5TF9+nRTh0ClAGfzIyIiIiIi0gOLKSIiokKcOHECffr0gbu7O2QyGfbs2VPoNsePH8err74KW1tb1KlTBxs2bDB6nESkGyFEgddLEemKxRQREVEhUlJS0LRpUyxfvlyn9e/cuYPevXvjtddew6VLlzBlyhSMHTsWhw8fNnKkRERUkjibHxERURHIZDL8/PPP6NevX77rzJgxAwcOHMCVK1dUbYMHD8azZ89w6NChEoiSiIhKAiegAKBQKPDw4UNUqlQJMpnM1OEQEZUrQgg8f/4c7u7usLAoGwMmIiMj0a1bN7U2f39/TJkyJd9tMjIykJGRoXquUCjw5MkTVK9enbmJiKgEFSUvsZgC8PDhQ3h6epo6DCKicu3+/fuoWbOmqcMwiNjYWLi6uqq1ubq6Ijk5GWlpabC3t9fYZuHChZg7d25JhUhERIXQJS+xmAJQqVIlAFKHOTo6Fnl7uVyOI0eOoEePHrC2tjZ0eKUO+8Ow2J+Gxf40vOL2aXJyMjw9PVXvxeXVrFmzEBISonqelJSEl156CXfu3NGrb+RyOY4dO4bXXnut3L/W2ReGxz41LPanYRW3P58/f45atWrp9N7LYgpQDZ9wdHTUu5hycHCAo6Mj/wOA/WFo7E/DYn8anqH6tCwNZXNzc0NcXJxaW1xcHBwdHbWelQIAW1tb2NraarRXq1atWLmpevXq5f61zr4wPPapYbE/Dau4/ancRpe8VDYGpxMREZmRNm3aICIiQq0tPDwcbdq0MVFERERkDCymiIiICvHixQtcunQJly5dAiBNfX7p0iXcu3cPgDRELyAgQLX++PHjcfv2bUyfPh3Xrl3DihUrsGPHDkydOtUU4RMRkZGwmCIiIirE+fPn8corr+CVV14BAISEhOCVV15BaGgoAODRo0eqwgoAatWqhQMHDiA8PBxNmzbF119/jTVr1sDf398k8RMRkXHwmikiIqJCdO7cGQXdlnHDhg1at7l48aIRoyIiIlPjmSkiIiIiIiI9sJgiIiIiIiLSA4spIiIiIiIiPbCYIiIiIiIi0gOLKSIiotIuLc246xMRkVYspoiIiEqz1auBJk2A+/d1W//+fWn91auNGxcRUTnAYoqIiKi0SksDvvwSuHUL6Ny58ILq/n1pvVu3pO14hoqIqFhYTBEREZVW9vbA0aOAjw9w+3bBBZWykLp9W1r/6FFpeyIi0huLKSIiotLM0xM4frzggipvIXX8uLQdEREVC4spIiKi0k5bQRUTIy2LiWEhRURkJCymiIiIyoK8BVWvXlJ7r14spIiIjMQsi6nly5fD29sbdnZ2aNWqFf78888C13/27BmCgoJQo0YN2Nra4uWXX8bBgwdLKFoiIiIzkbugio6W2qKjWUgRERmJ2RVT27dvR0hICMLCwnDhwgU0bdoU/v7+iI+P17p+ZmYmunfvjujoaOzatQvXr1/H6tWr4eHhUcKRExERmQFPT2DjRvW2jRtZSBERGYGVqQPIa/HixQgMDMTo0aMBAKtWrcKBAwewbt06zJw5U2P9devW4cmTJzh9+jSsra0BAN7e3iUZMhERkfm4fx8YMUK9bcQInpkiIjICsyqmMjMz8ddff2HWrFmqNgsLC3Tr1g2RkZFat9m3bx/atGmDoKAg7N27F87Ozhg6dChmzJgBS0tLrdtkZGQgIyND9Tw5ORkAIJfLIZfLixy3cht9ti2L2B+Gxf40LPan4RW3T/m3MKDcs/bVry+1eXsDV69K7SyoiIgMyqyKqYSEBGRnZ8PV1VWt3dXVFdeuXdO6ze3bt3H06FEMGzYMBw8exK1btzBx4kTI5XKEhYVp3WbhwoWYO3euRvuRI0fg4OCgd/zh4eF6b1sWsT8Mi/1pWOxPw9O3T1NTUw0cSTmVd/rzgweBv/+W/u3aNWeWPxZUREQGY1bFlD4UCgVcXFzwww8/wNLSEn5+foiJicGiRYvyLaZmzZqFkJAQ1fPk5GR4enqiR48ecHR0LHIMcrkc4eHh6N69u2qoYXnG/jAs9qdhsT8Nr7h9qhwdQMWg7T5Sbm5SMeXhIT1XLmdBRURkMGZVTDk5OcHS0hJxcXFq7XFxcXBzc9O6TY0aNWBtba02pK9+/fqIjY1FZmYmbGxsNLaxtbWFra2tRru1tXXhHwTS0vK9Y7zW7QtYv6zTqT9JZ+xPw2J/Gp6+fcq/QzHld0Pe3MMnlbP8saAiIjIosyqmbGxs4Ofnh4iICPTr1w+AdOYpIiICwcHBWrdp164dtmzZAoVCAQsLaXLCGzduoEaNGloLqWJZvRr48kvg6FHdEtD9+0CXLsD06UBgoGFjISIiSkuT8owu95HKW1B16QL880+5/cKPiAxPoZC+xynuIyureHFkZcnw9981UbmyDJ07G+RXy5dZFVMAEBISgpEjR6J58+Zo2bIlli5dipSUFNXsfgEBAfDw8MDChQsBABMmTMCyZcvw/vvvY9KkSbh58yY+++wzTJ482bCBpaVJhdStW7p9o5f7m8IvvwSGD2fCIiIiw7K3l76w0/WLPmVBpfyij3mJqNRKSwNu3JDml7l2Dbh+HSjuJahCSIVMYcVOZqb2doXCML9b8VkB8ENMjKL8FVODBg3C48ePERoaitjYWDRr1gyHDh1STUpx79491RkoAPD09MThw4cxdepUNGnSBB4eHnj//fcxY8YMwwZmby8lKl2GSOQdcnH0KBMWEREZR2Bg0b6w8/TkGSmiUiQhQSqWlEWT8t/oaKn4IdMyu2IKAIKDg/Md1nf8+HGNtjZt2uDMmTNGjgr5jznPfT1XfmPXiYiIjKWohRELKSKzkp0NxMU54NAhGW7eVC+aEhJMG5tMBlhbF/6wsdFtPV0eVlbScfWVnZ2Nf//9F337NgBgUej6xWGWxZRZ01ZQRURIy2JicqafZSFFREREVO4IAaSkSEWQ8vH4sfaflc+fPLGCQtFd52NUqgT4+kq3k/P1zXlUr178+K2s1AubfG7batbkcgUOHryDHj3qG/1YLKb0kbeg6tULWLhQ+peFFBEREVGJSE/P//qd4jyKus+UFPUiKT29qL+J9tMw7u7qRZPyX3f34p25IcNhMaWv3AVVdLTUFh3NQoqIiIhID3I5kJiY/xkcbW1paaaOungqVgScnIDq1RWwtIxDp04uaNjQEvXrA/XqAZUrmzpCKgyLqeLw9AQ2bgS6dctp27iRhRQRERGZTEFDzJQPIQAHB6BCBenf/B75LbeyAl68sEZMjFQEpaZqPlJStLcrl6Wk5BRPjx8DSUmm7rnisbaWCiPlw9m54OfVq+dcviiXZ+PgwT/Rq1cvWFuXwnF15RiLqeK4fx8YMUK9bcQInpkiIiIigxICuHcPuHxZ+vhRUKFUMmdrrAH0KokDabC0zClIqlUDbG2NMylCUbZ1cAAcHTn0rjxiMaWv3LP21f//i9u8vaWpV3hneSKiMmf58uVYtGgRYmNj0bRpU3z33Xdo2bJlvusvXboUK1euxL179+Dk5IQBAwZg4cKFsLOzK8GoqTRKSgKuXJFmsP/nH6mAunwZSE42dWTGUbVqwWdw8rZXrsyihcyHXsXU2bNn0apVK0PHUnrknf784EHg77+lf5Wz+bGgIqJyIjUVePrU1tRhGDU3bd++HSEhIVi1ahVatWqFpUuXwt/fH9evX4eLi4vG+lu2bMHMmTOxbt06tG3bFjdu3MCoUaMgk8mwePFio8RIpU9WlnTTVWXBpPz37t2i78vKKv9CRFubhYXuw/K0taekKPDs2WN4eTmjYkWLIg0RzN1etaoUO1FppdfLt02bNmjcuDECAwMxfPhwVKlSxcBhmTFt95Fyc5OKKQ8P7fehYkFFRKWIQgE8eQLEx0uPuLiCf37xwho1arTHsGGmjduYuWnx4sUIDAzE6NGjAQCrVq3CgQMHsG7dOsycOVNj/dOnT6Ndu3YYOnQoAMDb2xtDhgzB2bNnDRYTlR5CALGx6gXTP/8A//0nzRqni5deApo0ARo3BurWzSmQlP+W9BAz6RqfM/9/jY9x7+NDZM70KqaGDx+O3bt3Y/LkyZg+fToGDBiAwMBAdOjQwdDxmZf8bsgrl+esk9+NfVlQEZEJpafnFEGFFUiPH0s3kCyKZ89Mf2bKWLkpMzMTf/31F2bNmqVqs7CwQLdu3RAZGal1m7Zt22LTpk34888/0bJlS9y+fRsHDx7EiLzX2eaSkZGBjIwM1fPk/x/TJZfLIc+dZ3Sk3EafbcuakuyL1FTgv/9kuHwZuHJFhsuXZbhyRYaEBN0qnUqVBBo1EmjcWKBxY6BRI4GGDQUK+24gK6v4sRcFX1+Gxf40rOL2Z1G206uY+vHHH/Hdd99h06ZNWLt2LTZt2oTNmzejbt26CAwMxMiRI+Hk5KTPrs1XWhrQpYtu95HKW1B16SJ9BcU7zhORgQgBPH2q25mj+HjjXGtRrRrg4gK4uCiQlRWP7GwXWFsb/ji6MlZuSkhIQHZ2NlxdXdXaXV1dce3aNa3bDB06FAkJCWjfvj2EEMjKysL48eMxe/bsfI+zcOFCzJ07V6P9yJEjcHBwKHLcSuHh4XpvW9YYsi8UCiAurgKiox1x964joqMdce+eIx49qgAhCi+cLCwUcHdPgbd3Ery8nsPLKwleXslwcUlTO8OUlAScPm2wsA2Ory/DYn8alr79mZqaqvO6eo9SrVy5MoKCghAUFIQLFy5g9erV2LZtGz788EN89NFH6Nu3LwIDA9Et97ThpZm9PTB9OvDll8DRo4WfaVIWVF26SNuxkCKiQggBPH8uFUGxsdKjoJ8N/QWmjY2yOAJcXdX/zfuzszNUhZM03Oc8LC1NM7NXbuaSm44fP47PPvsMK1asQKtWrXDr1i28//77mD9/Pj755BOt28yaNQshISGq58nJyfD09ESPHj3g6OhY5BjkcjnCw8PRvXt3WJuyyjUDxe2LhATpLFPOmSbg339lSE3V7WyTm1vO2aaGDaV/69fH/09GYgfAtbBdmB2+vgyL/WlYxe3P5CJ8A2mQS/5effVVrFy5EosXL8bOnTsxe/Zs7Nq1C7t27YKXlxfGjx+PCRMmoFKlSoY4nOkEBgLDh+teGHl68owUEUGhkIbOPXyo/lAWRrkLJUNPaVylSsFFUe7nZW2GLEPlJicnJ1haWiIuLk6tPS4uDm5ublq3+eSTTzBixAiMHTsWANC4cWOkpKRg3Lhx+Oijj2BhoXmNia2tLWxtNYdLWltbF+vDVXG3L0sK64uMDGlS3rzXNj16pNv+7e2Bhg1zrm1S/uvsLANQhv5z5cLXl2GxPw1L3/4syjYGmz/l6dOn+PHHH7FmzRo8fPgQMpkM7dq1w9WrVzFz5kwsXboUe/fuRYsWLQx1SNMoamHEQoqozFIOtYuJ0SyU8hZNhrqewcJCOivk5iYVQW5u2gsj5cPGxjDHLa0MkZtsbGzg5+eHiIgI9OvXDwCgUCgQERGB4OBgrdukpqZqFEyWltKNOIUQhvnlSG+579mUu2i6fl236wVlMmnEf96iqXZt6R5IRFR+FLuYOnbsGFavXo09e/YgPT0dzs7O+PDDD/Hee+/Bx8cHGRkZWLduHaZPn45JkybhzJkzhoibiMjo0tOlD1x370qP6Oicnx88kAqlXPMFFEv16jnFkfKR+7nyZycnfljThaFzU0hICEaOHInmzZujZcuWWLp0KVJSUlSz+wUEBMDDwwMLFy4EAPTp0weLFy/GK6+8ohrm98knn6BPnz6qoopKzv37wKFD3vj1VwtcuVK0ezZVq5ZTLCkLp4YNgYoVjRszEZUOehVTcXFxWL9+PdauXYvbt29DCIFOnTph/Pjx6N+/v9qpMVtbW0yYMAG3bt3C8uXLDRY4EVFxvXihvVBS/hwbW7z9y2TS2SF3d+0PZaHEM0iGYczcNGjQIDx+/BihoaGIjY1Fs2bNcOjQIdWkFPfu3VM7E/Xxxx9DJpPh448/RkxMDJydndGnTx98+umnhv/FSSuFAoiIAFasAPbts4JC0bTA9a2tgfr11c82NWkC1KhRtoa/EpFh6VVM1axZEwqFAlWrVsWUKVMwbtw41KtXr8BtnJ2dkanrzRSIiPSUnQ0kJmqf1U66LskS1651wpgxVkhM1P84VatKt5bLXRzlfe7qCpPOblfeGDs3BQcH5zus7/jx42rPraysEBYWhrCwMJ32TYbz9CmwYQOwciVw86ayVb0a8vTUHKJXrx7/vxJR0elVTLVq1Qrjx4/HwIEDtV4sq83MmTO13tiQiEgXycnAnTvSsLu8BVLuoikhQboeIn8WAKoUerwaNQAvL+nh7a3+80svcYiPOWJuKt8uXACWLwe2btWcyMXdXaBDh+t47706aNbMClWrmiZGIip79CqmTp06Zeg4iKicS0+XhtbduaP98eSJ4Y5lYaFAzZoyeHvLtBZMnp6AnZ3hjkclg7mp/ElPB3bskIbynT2rubxLF2DiROD117MQHn4d7dvX5tknIjIovYqpBw8e4MKFC+jYsSOqaLkl99OnT3Hy5En4+fnBw8OjuDESURmQnS1N2pBfsfTwYfH2b2enffrvvG1Vq8rx55+/ok+f1zn9bBnD3FR+3L4NrFoFrFsHjeG6jo7AyJHAhAnSNVCA4e/JRkSkpFcxtWDBAuzcuRMP8/n04+DggHfffReDBw/GsmXLihUgEZUOQkhD7fIrlu7d0296cAsLoGZNoFYt6eHtnTO7Xe5iqWJF3S4Sl8sBS0tOTV0WMTeVbdnZwKFD0lmoX3/VHM7bpAkQFAQMHcphuERUcvQqpo4ePYoePXrkOybd1tYWPXr0wG+//Vas4IjIvDx7llMcRUerF0vR0UBqqn77dXHJKZbyPl56iReFk26Ym8qm5GTpLNSqVdJ7TW7W1sDAgdJQvrZtOeseEZU8vYqpmJgYvP322wWu4+Xlhf379+sVFBGZRmamdAbp9m3pQ8vt2+o/P32q334dHfMvlry9gQoVDPprUDnF3FT2HDgAvPeedGPs3F56CRg/HhgzRvoyhojIVPQqpmxsbJBcyN3ukpOTIeNXRERmRQjg8WPNIkn58/370r1ZisrWViqK8iuYqlblN8ZkfMxNZUdCAjBlCrB5s3q7v790Fqp3b968mojMg17FVOPGjbF//34sXrxY63CK9PR07Nu3D40bNy52gERUdC9eANeuAVevAv/9J/0bFSUVTCkpRd+f8rolHx/txZKbm7QOkSkxN5V+QgA7dwLBwdIXP0o9egDffQe8/LLpYiMi0kavYmr06NEYM2YM3nzzTaxcuRI+Pj6qZVFRUZg4cSIePnyIefPmGSxQItKUmKheMCl/vn+/6PuqVk0qjHx8coom5b8vvQTY2Bg+fiJDYm4q3R4+lCaQ2LMnp61KFWDJEml2Pp5QJCJzpHcxdfDgQezevRu+vr6oVasWPDw8EBMTgzt37iArKwuDBg3C6NGjDR0vUbkjBJCYaIeICBlu3FAvmnJ/c1sYGxtpKJ62YqlWLelDC1FpxtxUOgkBrF8PhIQASUk57W+9Jd2Et0YN08VGRFQYvYopANixYweWL1+OFStW4Nq1a7h58yYAoEGDBggKCsKECRMMFiRReZCZCdy6JQ3PU39Y4flzf533U6WKdG+VBg2kf5U/v/QSh+JR2cfcVLpERwPjxgHh4TltLi7AsmXAgAE8G0VE5k/vYkomkyE4OBjBwcFISUlBUlISKleujAqclouoQE+faiuYpGuasrO1baH904Sra07BlLtwcnPjBxAqv5ibSgeFQjrrNGuW+nWcw4cDS5cC1aubLDQioiLRu5jKrUKFCkxURHkkJAAXLwJXrqgXTfHxuu9DJgO8vASqVYtHx45OaNjQUlU4Va1qvNiJygLmJvN0/bo0pfkff+S01awJfP890KuX6eIiItKHQYopovLu0SPgwgX1x717um9vbw/4+uY86tWT/q1bF7C2zsLBg2fQq1cvWFtzLmAiKp2ysoCvvgLmzAEyMnLax48HvvhCuh8dEVFpo3cxdf/+fSxYsAC//fYbHj58iMzMTI11ZDIZsrKyihUgkTkRQiqS8hZOsbG6bV+jhnrRpHzUrJn/9UxyueHiJyrrmJvM099/A+++K71fKtWuDaxZA3TubLKwiIiKTa9i6vbt22jVqhWePn2Khg0bIiMjA15eXrCzs8Pt27chl8vRtGlTVOH0YFSKKRTSdUx5C6cnTwrftlIl4JVXgFdfBZo2la5pqlcPqFzZ+HETlVfMTeZHLgfmzQM+/1w6MwVIXxxNnSq1OziYNj4iouLSq5iaO3cukpKSEBERgU6dOsHCwgKjR49GaGgoHj16hAkTJuC///7Db7/9Zuh4iQxOoZDONv37r/rj6lUgNbXw7atWBfz8pMJJ+ahdmzPnEZU05ibzIoR0NmrTppy2hg2BdeuAli1NFxcRkSHp9XHvt99+Q69evdCpUydVmxACAFCjRg1s374dADB79my9A1u+fDm8vb1hZ2eHVq1a4c8//9Rpu23btkEmk6Ffv356H5vKJiGAu3eBgwelcfujR0sJ3dFRus/SG28AM2YAP/4I/PWX9kLK1VW6QPrjj4GffpKm9U1MlKb1/eILYNAg6TonFlJEJa8kchPp7vvvcwopKysgLEw6u89CiojKEr3OTCUkJMDX1zdnJ1ZWSM31ydPW1hbdu3fHnty3MS+C7du3IyQkBKtWrUKrVq2wdOlS+Pv74/r163Bxccl3u+joaEybNg0dOnTQ67hUdsTHSzPpKc8yXbki3eT2xQvdtpfJpLNLDRuqn3Fydzdu3ESkP2PnJtLd+fPA++/nPN+6VbpvFBFRWaNXMeXk5ISUXDeGcHJyQnR0tPqOrazw7NkzvYJavHgxAgMDVXepX7VqFQ4cOIB169Zh5syZWrfJzs7GsGHDMHfuXJw8eVLvY1Ppk5QknUk6dy7noetMejKZdFaqYUP1h6+vNMMeEZUexs5NpJsnT6TCSTn3x5QpLKSIqOzSq5iqW7cuoqKiVM9btmyJw4cP4/bt2/Dx8cHjx4+xa9cu1K5du8j7zszMxF9//YVZs2ap2iwsLNCtWzdERkbmu928efPg4uKCMWPG4OTJkwUeIyMjAxm55mVNTk4GAMjlcsj1mDpNuY0+25ZFxuyP9HTg779lOHdOhvPnpceNG7rdodbbW6BBA4H69aV/GzYU8PXN/wJoc/lz8vVlWOxPwytunxrqb2HM3ES6USiAgABpSDUAtGkjDYEmIiqr9CqmXn/9dcyZMwfPnj1DlSpVMGXKFOzfvx9NmjRB/fr1cevWLSQnJ2POnDlF3ndCQgKys7Ph6uqq1u7q6opr165p3ebUqVNYu3YtLl26pNMxFi5ciLlz52q0HzlyBA7FmFooPDxc723LouL2R3a2DPfuVcKtW1Vw82ZV3LpVBXfvOiI7u+ALkuzsslC79jPUrv0MXl7JeOml56hZ8zns7bPV1ouN1X1Kc3PA15dhsT8NT98+TdVlphcdGDM3kW4+/xw4cED62ckJ2LEDsLExbUxERMakVzE1YcIEdO7cGZaW0g1EO3fujG3btmHOnDm4cuUKvLy8sGDBAgQGBho0WG2eP3+OESNGYPXq1XByctJpm1mzZiEkJET1PDk5GZ6enujRowcc9bhroFwuR3h4OLp37w5ra+sib1/W6Nsf2dnAX3/J8OuvMhw7JsPFizKkpRV81snaWqBpU4HmzaWHn590tsnSsjKAsjEPOV9fhsX+NLzi9qlydEBxmVNuKo+OHgU++UT6WSYDtmyR7qFHRFSW6VVMOTo6olWrVmptAwcOxMCBA4sdkJOTEywtLREXF6fWHhcXBzc3N431o6KiEB0djT59+qjaFAoFAGls/PXr1zWGdNja2sLW1lZjX9bW1sX6cFXc7csaXfojMRE4fFiaYe/wYSAhIf91LSyA+vWBFi1yHk2ayGBrq9swv9KOry/DYn8anr59aqi/gzFzExXs4UNgyBBpmB8AzJkDdO9u0pCIiEqEXsVUly5d0K5dO8yfP9/Q8cDGxgZ+fn6IiIhQTW+uUCgQERGB4OBgjfV9fX1x+fJltbaPP/4Yz58/xzfffANPT0+Dx0j6UyikqXF//VUqoM6elaYs18bHR71wevVVoGLFko2XiEoPY+Ymyp9cLt0WIj5eeu7vL90+goioPNCrmDp79ixat25t6FhUQkJCMHLkSDRv3hwtW7bE0qVLkZKSoprdLyAgAB4eHli4cCHs7OzQqFEjte2Vd7fP206m8fQpcOSIVED9+mtOws2rUiXpm8xevYCePQEPj5KNk4hKN2PnJtJu9mzg1CnpZ09P6d5SvNceEZUXehVTvr6+uKucqscIBg0ahMePHyM0NBSxsbFo1qwZDh06pJqU4t69e7DgO7XZEgK4fdsRn39ugSNHgNOnc4Z+5NWwoVQ89eoFtG3LC5WJSH/Gzk2k6eefpZugA4C1tTThhI6XLxMRlQl6FVOTJk1CcHAw/vvvPzRo0MDQMQEAgoODtQ7rA4Djx48XuO2GDRsMHxAVKD1duvh4717gl1+s8PDha1rXc3AAunWTiqfXXwdeeqmEAyWiMqskchPluHULGDUq5/nXXwM8MUhE5Y1exZSPjw86d+6M1q1b47333kOLFi3g6uoKmUxzIoCOHTsWO0gyT4mJ0hS4e/dKk0fk3CtT/XXg6ysVTr16AR06AFrm/iAiKjZj56bly5dj0aJFiI2NRdOmTfHdd9+hZcuW+a7/7NkzfPTRR/jpp5/w5MkTeHl5YenSpejVq1eRj21u0tKkG/EqJ2J85x0gn+8/iYjKNL2Kqc6dO0Mmk0EIga+//lprolLKzs7OdxmVPlFRUvG0d680Rl7b8D07O4GGDeMQEOCMN96whI9PycdJROWPMXPT9u3bERISglWrVqFVq1ZYunQp/P39cf36dbi4uGisn5mZie7du8PFxQW7du2Ch4cH7t69q7qmt7SbNAn4+2/p53r1gDVrpOnQiYjKG72KqdDQ0AKTFJUdCgVw7lxOAfXff9rXc3IC3ngD6NsX6Nw5C7//fha9evWCtbVlyQZMROWWMXPT4sWLERgYqJoIadWqVThw4ADWrVuHmTNnaqy/bt06PHnyBKdPn1ZN/e7t7W2U2Era+vXA2rXSzw4OwO7d0gRCRETlkV7FFO8eX7alpwMREVLxtH8/EBurfb26daXiqW9foE0b4P/vkwm5vORiJSJSMlZuyszMxF9//YVZs2ap2iwsLNCtWzdERkZq3Wbfvn1o06YNgoKCsHfvXjg7O2Po0KGYMWOG6qbCeWVkZCAjI0P1XHkzY7lcDrkeb6zKbfTZNj9//w1MnGgF5XDu5cuz8PLLwuzf943RF+Ud+9Sw2J+GVdz+LMp2ehVTVPZkZ0vTl69bJ01fnnP9Uw6ZTLq4WFlA+fqWfJxERCUtISEB2dnZqhlllVxdXXHt2jWt29y+fRtHjx7FsGHDcPDgQdy6dQsTJ06EXC5HWFiY1m0WLlyIuXPnarQfOXIEDg4OescfHh6u97a5paRYYdq0TkhPl8609ex5B1Wr/oODBw2y+xJhqL6gHOxTw2J/Gpa+/ZmamqrzuiymyrnYWKmA+uEHQNuMwnZ20r2f+vaVhvHl+SxBRERaKBQKuLi44IcffoClpSX8/PwQExODRYsW5VtMzZo1CyEhIarnycnJ8PT0RI8ePeDo6FjkGORyOcLDw9G9e3fVUEN9CQEMGmSJR4+k25K8+qoCO3bUhJ1dzWLtt6QYsi9Iwj41LPanYRW3P5UjA3ShVzFlYWGh07h0mUyGrKwsfQ5BRqRQSNOYf/89sGcPkPdP5OQE9OkjFVDdu0tj4omIzJ2xcpOTkxMsLS0RFxen1h4XFwc3Nzet29SoUQPW1tZqQ/rq16+P2NhYZGZmwkbLTfVsbW1hq2W6U2tr62J9uCru9gCweLGULwCgShVg1y4LVKpU+u73aIi+IHXsU8NifxqWvv1ZlG30KqY6duyoNWElJSXh5s2bSElJQdOmTcvMrEVlxePHwIYN0lmoW7fUl8lkgL8/MH480Ls3YMVzllQIuVxeKmfrlMvlsLKyQnp6eqmM3xzl7VNLS0uTfBgwVm6ysbGBn58fIiIi0K9fPwDSmaeIiIh874fYrl07bNmyBQqFQnWT+Rs3bqBGjRpaCylzduoUMH16zvONG4FatUwXD1FBmJsI0N6fxspNen1kLuimuampqZg5cyYOHTrEcZ9mQAjgxAnpLNTu3UBmpvpyV1dgzBhg7FgmR9JNcnIyEhIS1C6UL02EEHBzc8P9+/c5K6mBaOtTW1tbODk56TU8TV/GzE0hISEYOXIkmjdvjpYtW2Lp0qVISUlRze4XEBAADw8PLFy4EAAwYcIELFu2DO+//z4mTZqEmzdv4rPPPsPkyZP1+t1MJT4eGDRIuq4WAGbNkoZ8E5kb5ibKLb/+NEZuMvj5BwcHB3z77bdo0aIFPvzwQ6xfv97QhyAdPHkC/PijVERpuz66a1fpLNSbbwKl7EtSMqHk5GTExMSgYsWKcHJygrW1dal701coFHjx4gUqVqyoOmNAxZO7T2UyGeRyOZKSkhATEwMAJVpQ5ae4uWnQoEF4/PgxQkNDERsbi2bNmuHQoUOqSSnu3bun9nry9PTE4cOHMXXqVDRp0gQeHh54//33MWPGDIP+XsaUnQ0MHQo8fCg979wZmDfPpCERacXcRHnl7U8hhNFyk9EGc3Xo0AGbNm0y1u5JCyGAyEhg1Spg505pivPcnJyA0aOBwEBpWnOiokpISEDFihVRs2bNUpeolBQKBTIzM2FnZ8eEZSB5+9Te3h6VKlXCgwcPkJCQYBbFlFJxclNwcHC+w/q0nRVr06YNzpw5o9exzMHKldJtMgDAzQ3YupVDwMk8MTdRXtr601i5yWhvi48fP8aLFy+MtXvKRaEAdu0CPvss5470uXXqBLz3HtC/P6Dl2mYincjlcmRkZMDJyanUJisqOTKZDJUrV0ZMTAzkcrnZXFDN3KQbuRxYtCjn+bZtUkFFZG6Ym6gojJGbDF5MKRQKbN68Gdu3b0fz5s0NvXvKJTsb2LEDWLAA+O8/9WVVqwIjRwLjxgH165smPipblBdwmsuHYjJ/ytdKdna2yV83zE1Fs2sXcO+e9HOvXtKXckTmiLmJisrQuUmvYsrHx0dre1ZWFuLj41WVnvJCXDKsrCxg+3apiMp7PVTLlkBQEDBwIGBvb5r4qGzjN3+kq5J+rTA3GYYQ6melPvzQdLEQ6Yq5iXRl6NeKXsWUQqHQGoi1tTUaNWqEFi1aIDg4GA0bNix2gJQjKwvYskUqom7eVF/Wrh0QFgZ06yZNc05EVN4wNxnG0aPAxYvSz82b86wUEVFB9CqmoqOjDRwGFUQuBzZtAj79FIiKUl/WoYNURHXpwiKKiMo35ibDyH1Wato05hYiooJwuhAzJpcDa9cC9eoB776rXkh17gwcOybdQ6prVyY7orJKJpOhc+fOpg6Dyol//gEOH5Z+9vYG3n7bpOEQkZlibsqhVzH14MED7Nu3D8+ePdO6/OnTp9i3b59qHncqmsxM4IcfpOnLx44F7tzJWda1K/D771IhxdcwUcmKjo6GTCYr8JHf+6Kx3L17F5aWlpDJZFiU+5RCOcTcVHxff53zc0gIp0InKg2Ym0xLr7fJBQsWYOfOnXiovJNfHg4ODnj33XcxePBgLFu2rFgBlicZGcC6dcDChcD9++rLuneXhvO1a2ea2IgoR+3atTF8+HCty+zs7Eo0lnXr1qmuFVq3bh0+LMezBTA3Fc+DB9J1uYA0I+y775o2HiIqGuYm09CrmDp69Ch69OgB23xuWmRra4sePXrgt99+K1Zw5UV2NrBmjTSxxIMH6st69gRCQ4E2bUwTGxFpqlOnDubMmWPqMKBQKLBhwwY4OTnhjTfewIYNG3D69Gm0bdvW1KGZBHNT8Xz7rTTREQBMnAhUqGDaeIioaJibTEOvYX4xMTHw9vYucB0vLy8OpdDBhQtA69bA+PHqhVTv3sDZs8Cvv7KQIiqN/vnnHwwePBg1atSAjY0NvLy8MGnSJCQmJmpdf82aNWjUqBHs7Ozg6emJ6dOnIz09vcBjhIeH4969exg8eDDGjBkDAFi7dq3aOvPnz4dMJsOPP/6odR8//fQTZDIZPvroI4325s2bw97eHq6urggMDMTTp0/h7e1d6Pu/qTA36S85Gfj+e+lnW1tg0iTTxkNExsHcZHh6FVM2NjZITk4ucJ3k5GTO+V+A58+BKVOAFi2A8+dz2vv0Ac6dA375RbpnFBGVPvv27UPLli2xb98+dO7cGVOmTEHjxo2xbNkytGnTBk+fPlVbf/78+QgMDERCQgICAwMxcOBAbN++HQMHDizwOMrkFBAQgPbt28PHxwc7duzAixcvVOsMHz4cMpkMmzZt0rqPjRs3AgBGjBihalu3bh3efvtt3Lx5EwEBARg5ciQiIyPRvXt3yOVyvfqkJDA36e+HH6SCCgACAgBXV9PGQ0SGx9xkJEIPHTp0EJ6eniI9PV3r8rS0NFGzZk3Rtm1bfXZf4pKSkgQAkZSUpNf2mZmZYs+ePSIzM7PQdRUKIXbvFsLDQwjp1ojSo2FDIU6e1OvwZqco/UGFM5f+TEtLE//9959IS0szaRzFlZ2dLZ4+fSqys7OLvO2dO3cEAFG7dm0RFham8YiMjBQJCQnC0dFReHh4iOjoaLXtt27dKgCI4OBgVdvNmzeFlZWV8PDwEHFxcar2pKQkUa9ePQFAdOrUSSOWhIQEYWNjI3x9fVVtoaGhAoBYs2aN2rrt27cXlpaW4uHDh2rtiYmJwsbGRjRv3lzV9vTpU1GxYkVRoUIFcePGDVW7XC4XXbp0EQCEl5eX2n7y61NdXzPFfQ9WYm5Sp+t7R0aGek66dk2vw5k1c3kfLUvMpU+Zm5ibtOWmgvpTl9dMUd5/9bpmavTo0RgzZgzefPNNrFy5Uu2u81FRUZg4cSIePnyIefPmFaPMK3uio6WhE7/8ktNmby9NLDF1KmBjY7LQiIqteXMgNtbUURTMzQ3488/i7ycqKgpz587VaK9SpQoiIyORnJyMZcuWwcvLS2354MGDsWjRImzbtg3fffcdAGDLli3IyspCSEgIXFxcVOs6Ojri448/VvtWLreNGzciMzNTbXlAQADmzZuHtWvXqoZWANI3e6dOncLWrVsREhKiat++fTsyMzPVLljeu3cvXrx4gcmTJ6Nu3bqqdisrKyxYsMCsx7wzN+ln+3ZAOfLxzTel23EQlRXMTcxNxqZ3MXXw4EHs3r0bvr6+qFWrFjw8PBATE4M7d+4gKysLgwYNwujRow0db6kklwNLlgBz5wKpqTntvXoBy5YBtWqZLjYiQ4mNzflAVtb5+/vj0KFDWpcNGjQIAHD27FlE5b3LNoD09HQkJCQgISEBTk5O+PvvvwEAHTp00FhXW5vS2rVrIZPJ1JJN7dq10bZtW5w+fRpXr15F/fr1AQDvvPMOJk+ejI0bN6olrE2bNsHKygpDhgxRtSnjad++vcYxW7VqBSszniubuanohFC/SW8ZnnCLyinmJglzk/HofeQdO3Zg+fLlWLFiBa5du4abN28CABo0aICgoCBMmDDBYEGWZqdPA++9B1y5ktPm7g588410M0QO3aeyws3N1BEUriRifPLkCQBg+fLlBa6XkpICJycnJCUlAYDaN39KrvlcuHL27FlcuXIFr732Gl566SW1ZQEBATh9+jTWrVunurdHlSpV8MYbb2D37t3477//0KBBA0RFReH06dPo1auX2rGV1xxpi8fCwgJOTk4F/l6mxtxUNEeOAJcvSz+3bs3bb1DZw9wkYW4yHr2LKZlMhuDgYAQHByMlJQVJSUmoXLkyKnAuVQDAkyfAzJnA6tU5bRYWQHAwMH8+4OhoutiIjCH3RCrmTKEw7v4d//8/9+XLl9GoUaNC169cuTIAID4+XmPoRVxcnNZtlBf3Hjt2LN/JFH788Ud89tlnsLa2BiANp9i9ezc2btyIhQsXqi76zTtUQxl/fHy8xj4VCgUSEhLg4eFR6O9lKsxNRZP7rNS0afyCj8oe5iYJc5Px6DWbX14VKlSAu7s7kxWkIRMbNwK+vuqFlJ+fNB72m29YSBGVZa1atQIAREZG6rR+06ZNAQAnT57UWKatLSUlBdu2bYODgwPGjBmj9dGkSRPEx8fjl1wXaPbq1QvVq1fHli1boFAosHnzZlSqVAl9+/bVGs8ff/yhcew///wTWcobEZUCzE0Fu3gRiIiQfq5TB+jXz6ThEJERMTcZUaFTVGhx6tQpMXXqVPHo0SOtyx8+fCimTp0qIiMj9dl9iTPUjEmXL2eKLl3UZ+mrVEmIb78VIivLwEGbMXOZ4aesMJf+5IxJOTMm+fv757tOfHy8qFSpknB2dhZXrlzRWJ6SkqL23njz5k1haWmp84xJ69evFwBEQEBAvjEcPnxYABC9e/dWa584caIAIBYuXCgAiFGjRmlsq5wxqWLFiuLWrVuqdrlcLrp162bWs/kxN6kr7L1j6NCcXLViRXEiNX/m8j5alphLnzI3MTdpy00lOZufXmemFi9ejP3798Mtn0GeNWrUwC+//IIlS5bos/tSJz0d2Lq1Hvz8rHD0aE77wIHA1avSDH6WlqaLj4hKjrOzM7Zu3YoXL16gadOmeOONNzBt2jRMmjQJffr0gZubm9od6uvUqYPQ0FDExMSgSZMmmDx5MkJCQtC4cWO1GYuUlMMoCppEoVu3bqhZsyYOHTqEhw8fqtqVwyZCQ0PVnudWpUoVLF68GC9evICfnx/Gjx+PGTNm4JVXXsHTp0/h7u4OCwuDDGowOOYm3d29K83iBwBOTsDIkaaNh4iMi7nJePQ66rlz57TOppFbx44dcebMGb2CKk1OnwZefdUK27f7IjNTGh/q7Q0cOADs2AGY8aUFRGQkvXv3xsWLFzFq1ChcuXIF3333HTZv3oy7d+9i9OjRmD9/vtr6oaGhWL16NapXr47vv/8eO3fuxDvvvIMdO3aorXf9+nWcOnUKtWrVQqdOnfI9voWFBUaOHIns7Gxs2LBB1d66dWvUrVsXcrkcNWvWROfOnbVuHxgYiJ07d8LHxwcbNmzAhg0b0Lp1axw5cgTJycmqsevmhrlJd0uXAtnZ0s9BQYCDg0nDIaISwNxkHHpNQBEfH1/oRV5ubm5aLxIra6ysAOUMk1ZWAh9+KMPHHzMxEZVF3t7eEELotG69evWwZs0anfc9duxYjB07VqM99/Hq1aun8/EXLFiABQsWaLTfuHFDp+0HDBiAAQMGqLXdunULL168QD0zvRERc5Nunj7NuabXzk4qpoio9GJuMm1u0uvMVJUqVXDv3r0C17l79y4qVqyoV1ClScuWwHvvKVC/fiLOncvCZ5+xkCKi0u3p06fIyMhQa0tLS8PUqVMBAP3MdKYC5ibdfP89kJIi/Tx6NODsbNp4iIh0Ya65Sa9iqnXr1vj5559x//59rcvv3buHPXv2FOtuxMuXL4e3tzfs7OzQqlUr/FnAraFXr16NDh06oGrVqqhatSq6detW4PqG9uWXCnz66Sk0bFhihyQiMprff/8d7u7uGDJkCGbMmIExY8agQYMG+OWXX9ClSxfVzR/NTUnkptIuI0OaVRaQpkHPdZ9MIiKzZq65Sa9iKiQkBKmpqWjXrh1+/PFHPHr0CADw6NEj/O9//0O7du2QlpaGDz74QK+gtm/fjpCQEISFheHChQto2rQp/P398x2acfz4cQwZMgTHjh1DZGQkPD090aNHD8SU0C2v7eyke0gREZUFDRs2RPfu3fHHH3/g22+/xZYtW1CxYkXMnz8fBw4cMNsJKIydm8qCzZuB2Fjp57fekqZEJyIqDcw1N+l1zVTHjh2xePFifPDBB6pZO2QymWq8pIWFBb755ht07NhRr6AWL16MwMBA1b5XrVqFAwcOYN26dZg5c6bG+ps3b1Z7vmbNGuzevRsREREICAjQKwYiovKqbt262LZtm6nDKDJj56bSTqEAvvoq5/mHH5ouFiKiojLX3KRXMQUA77//Pl577TWsWrUK586dQ1JSEqpUqYKWLVti/PjxaNSoETIyMmBra1uk/WZmZuKvv/7CrFmzVG0WFhbo1q2bzjcaS01NhVwuR7Vq1bQuz8jIUBtzmZycDACQy+WQy+VFile5Xe5/yzv2h2GZS3/K5XIIIaBQKKAw9q3ajUj5wVr5u1Dx5denCoUCQgjI5XJYFnB/CEO+to2Vm8qCX3+VbtcBAO3bA61bmzYeIqKyQO9iCgCaNGmCFStWaLRfuHABQUFB2LZtGxITE4u0z4SEBGRnZ8PV1VWt3dXVFdeuXdNpHzNmzIC7uzu6deumdfnChQsxd+5cjfYjR47AoRizR4SHh+u9bVnE/jAsU/enlZUV3Nzc8OLFC2RmZpo0FkN4/vy5qUMoc/L2aWZmJtLS0nDixIkC706fmppq0DiMkZvKgkWLcn7mWSkiIsMoVjGV27Nnz7Bp0yasXbsW//zzD4QQsLe3N9Tudfb5559j27ZtOH78OOzs7LSuM2vWLITkuuo2OTlZdZ2VPnPUy+VyhIeHo3v37rC2ttY79rKC/WFY5tKf6enpuH//PipWrJjv/63SQAiB58+fo1KlSpDJZKYOp0zIr0/T09Nhb2+Pjh07FviaUY4OMAZD5qbly5dj0aJFiI2NRdOmTfHdd9+hZcuWhW63bds2DBkyBH379sWePXv0OnZxnTsH/P679HO9esAbb5gkDCKiMqfYxdRvv/2GtWvXYu/evcjIyIAQAm3atMHo0aP1mlXDyckJlpaWiIuLU2uPi4vL9672Sl999RU+//xz/Pbbb2jSpEm+69na2mod4mFtbV2sD6vF3b6sYX8Ylqn7Mzs7GzKZDBYWFmY7AYEulMPQlL8LFV9+fWphYQGZTFboa9cYr2tD5yblxEirVq1Cq1atsHTpUvj7++P69etwcXHJd7vo6GhMmzYNHTp0KM6vU2y5r5X64ANOmkREZCh6vZ3ev38f8+bNQ61ateDv74/t27ejevXqEEJg1KhR+OOPPzB27FhUqlSpyPu2sbGBn58fIiIiVG0KhQIRERFo06ZNvtt9+eWXmD9/Pg4dOoTmzZvr82sREVEpZszclHtipAYNGmDVqlVwcHDAunXr8t0mOzsbw4YNw9y5c+Hj41OcX61Ybt8Gdu2SfnZxAUaMMFkoRERljs5npuRyOfbs2YO1a9ciIiIC2dnZqFChAoYNG4aAgAB06dIFVlZWsLIq/sjBkJAQjBw5Es2bN0fLli2xdOlSpKSkqGZnCggIgIeHBxYuXAgA+OKLLxAaGootW7bA29sbsf8/72vFihXL/c0ZiYjKspLITfpOjDRv3jy4uLhgzJgxOHnyZKHHMdbkSEuXSjP5AcDEidmwtFSgvM0PZC4T+ZQl5tKnnByJtCmoP3WZHKkor2uds4u7uzuePHkCmUyG1157DQEBAejfvz8qVKig88F0NWjQIDx+/BihoaGIjY1Fs2bNcOjQIdWkFPfu3VMbSrJy5UpkZmZiwIABavsJCwvDnDlzDB4fERGZh5LITfpMjHTq1CmsXbsWly5d0vk4xpgcKTnZGuvXS9ex2dpmwcfnCA4eLL8Fhakn8imLTN2nnByJCqKtP3WZHKkoEyPpXEwlJibCwsICU6dOxfTp0+Hs7KzzQfQRHByM4OBgrcuOHz+u9jw6OtqosRARkXkq6dyki+fPn2PEiBFYvXo1nJycdN7OGJMjBQbeRUaGlOrHjpVh8ODuRd5PWWAuE/mUJebSp5wcibQpqD91mRypKBMj6VxMjRo1Cjt37sTixYvx7bffwt/fHyNGjEDfvn1hY2Oj8wGJiIgMpSRyU1EnRoqKikJ0dDT69OmjalMOM7GyssL169dRu3Ztje0MPTlSejpw4EAtANKEEx98YAlr6/zv91UemHoin7LI1H3KyZFIm4L6U5fJkYrymtb5r7Vu3To8evQI33//PV599VX88ssvGDx4MFxdXfHee+/h1KlTOh+UiIh0I5PJ0LlzZ1OHYbZKIjcVdWIkX19fXL58GZcuXVI93nzzTbz22mu4dOkSPD09ix2TLjZtkiEpSfrWdcAAoFatEjksEZUDzE05ilT6VqxYEWPHjkVkZCT+/fdfTJkyBTY2Nli9ejU6deoEmUyG69ev4+7du8aKl4jKmrQ0465vQNHR0ZDJZAU+nj17ZvQ4Ro0apXFcR0dHtGjRAkuWLNG4cLawmC9dulToOnkf5qQkclNISAhWr16N//3vf7h69SomTJigMTGScoIKOzs7NGrUSO1RpUoVVKpUCY0aNSqR0RwKBbBkSc5ZKN6kl6iImJuKrLzmJr2nN6pfvz6+/vprfPHFF6qZlMLDw3Hy5EnUrl0bnTp1wqhRozCCc7ASUX5Wrwa+/BI4ehTQ5dv6+/eBLl2A6dOBwEDjx5eP2rVrY/jw4VqXleSY/TFjxqBmzZoQQuD+/fv46aefEBISgqNHj2L//v1q61avXj3f61Dd3NwQFham0b506VIkJSVpXWaujJWbijoxkqnt3w/cvCl9sOjUSYHmzc0nNiKzx9xULOUuNwkDun//vpg3b57w8fERMplMWFhYGHL3RpOUlCQAiKSkJL22z8zMFHv27BGZmZkGjqx0Yn8Ylrn0Z1pamvjvv/9EWlqaYXaYmipEnTpCAEL4+Ahx717B69+7J60HSNulpup12OzsbPH06VORnZ1d5G3v3LkjAAh/f3+9jq0PAKJTp05qbSNHjhQARGRkpFp7TEyMcHFxEQDEsWPH1PZRr169Ih/by8tL6JIm8utTXV8zxX0PLkx5zE3jx0v/VQAh9u6VGyG60sVc3kfLEnPpU+Ym5iZtCupPXV4zRXn/NehXVTVr1sQnn3yCqKgohIeHY/DgwYbcPRGVJfb20rd+Pj7SXUU7d5a+3dPm/n1p+e3b0vpHj0rbm7F//vkHgwcPRo0aNWBjYwMvLy9MmjQJiYmJWtdfs2YNGjVqBDs7O3h6emL69OlIT08v0jHd3d3Rv39/AMC5c+eK/TuUFeUxN61YAUREZOH11+/A31+YOhyi0oO5SQ1zU+GKf4fdfHTt2hVdu3Y11u6JqCzw9ASOH89JRp07S89zD6vIm6zyLjdD+/btwzvvvAMLCwv07dsXnp6e+O+//7Bs2TIcPnwYZ8+eRdWqVVXrz58/H6GhoXB1dUVgYCCsra2xfft2XL16Ve8YzO26JnNRXnKTTAZ06CDw/Pk/sLCoaepwiEoX5iYAzE26MloxRUSkk4KSlpkmq1u3bmm9IXjPnj1Rt25djBgxAk5OTvjjjz/g5eWlWr5t2zYMGTIEoaGh+O6771T7mjdvHjw8PHDhwgW4uLgAAObMmYOWLVsWKa7Y2Fj8/PPPAKCxbUJCgtaYW7dujZ49exbpOEREZR5zE3OTjlhMEZHpaUtaGzcCI0aYXbICpPsIzZ07V6O9SpUqiIyMRHJyMpYtW6aWrABg8ODBWLRoEbZt26ZKWFu2bEFWVhZCQkJUyQoAHB0d8fHHHxc4UcKaNWtw6NAhCCHw4MED/PTTT3j27Bn69u2Ljh07qq2bmJioNeb333+/1CQsIqISxdzE3KQDFlNEZB7yJq127aR2M0tWAODv749Dhw5pXTZo0CAAwNmzZxEVFaWxPD09HQkJCUhISICTkxP+/vtvAECHDh001tXWltvatWtVP1esWBH169fHsGHDEBQUpLFuvXr1cO3atQL3R0REeTA3aazL3KSOxRQRmQ9PT+lbP2WyAqTnZpSsCvPkyRMAwPLlywtcLyUlBU5OTkhKSgIAtW/+lJTTbucnMjISrVu31jNSIiLSCXOTGuYmdbzxBBGZj/v3peETuY0Ykf9MSmbI0dERAHD58mUIIfJ9KIdZVK5cGQAQHx+vsa+4uLiSC5yIiLRjblLD3KSOxRQRmYe8F/T+8YduU9OamVatWgGQvpnTRdOmTQEAJ0+e1FimrY2IiEoQc5PGMuYmdSymiMj0tM2M1Lat9G8pS1qjR49GpUqV8NFHH+Hff//VWJ6amoozZ86ong8dOhSWlpZYvHix2jeAycnJWLBgQYnETEREWjA3MTfpgMUUEZlWQVPMKi/8LUVJy9nZGVu3bsWLFy/QtGlTvPHGG5g2bRomTZqEPn36wM3NTW0a2Dp16iA0NBQxMTFo0qQJJk+ejJCQEDRu3Bh169Y13S9CRFSeMTcxN+mIE1AQkenocq8OXW6eaGZ69+6NixcvYtGiRfjtt98QHh6OChUqoGbNmhg9ejSGDx+utn5oaCjc3d2xZMkSfP/993BxccHgwYMxb948ODg4mOi3ICIqp5ibADA36YrFFBGZRloa0KWLbvfqyJu0unQB/vkHsLcvwYABb29vCCF0WrdevXpYs2aNzvseO3Ysxo4dq9Gu7XgbNmzAhg0bdN63rjHnFR0drdd2RESlFnOTGuamwnGYHxGZhr09MH06UKeObt/mKZNWnTrSdiWcrIiIqBxgbqIi4pkpIjKdwEBg+HDdk4+np0m+9SMionKEuYmKgGemiMi0ipp8mKyIiMjYmJtIRyymiIiIiIiI9MBiioiIiIiISA8spoiIiIiIiPTAYoqIikTfqUyp/OFrhYhKCt9vSFeGfq2wmCIinVhaWgIA5HK5iSOh0kL5WlG+doiIDI25iYrK0LmJxRQR6cTa2hq2trZISkriN4BUKCEEkpKSYGtrC2tra1OHQ0RlFHMTFYUxchPvM0VEOnNyckJMTAwePHiAypUrw9raGjKZzNRhFYlCoUBmZibS09NhYcHvkwwhd5/KZDLI5XIkJSXhxYsX8PDwMHV4RFTGMTdRXnn7UwhhtNzEYoqIdObo6AgASEhIQExMjImj0Y8QAmlpabC3ty91ydZcaetTW1tbeHh4qF4zRETGwtxEeeXXn8bITSymiKhIHB0d4ejoCLlcjuzsbFOHU2RyuRwnTpxAx44dOfzMQPL2qaWlJfuWiEoUcxPlpq0/jZWbWEwRkV6sra1L5Ru+paUlsrKyYGdnVyrjN0fsUyIyF8xNBJRsf3JQJhERERERkR5YTBEREREREemBxRQREREREZEezLaYWr58Oby9vWFnZ4dWrVrhzz//LHD9nTt3wtfXF3Z2dmjcuDEOHjxYQpESEVF5UJS8tHr1anTo0AFVq1ZF1apV0a1bt0LzGBERlT5mWUxt374dISEhCAsLw4ULF9C0aVP4+/sjPj5e6/qnT5/GkCFDMGbMGFy8eBH9+vVDv379cOXKlRKOnIiIyqKi5qXjx49jyJAhOHbsGCIjI+Hp6YkePXqU2mmbiYhIO7MsphYvXozAwECMHj0aDRo0wKpVq+Dg4IB169ZpXf+bb75Bz5498eGHH6J+/fqYP38+Xn31VSxbtqyEIyciorKoqHlp8+bNmDhxIpo1awZfX1+sWbMGCoUCERERJRw5EREZk9lNjZ6ZmYm//voLs2bNUrVZWFigW7duiIyM1LpNZGQkQkJC1Nr8/f2xZ88eretnZGQgIyND9TwpKQkA8OTJE8jl8iLHLJfLkZqaisTERE5nCfaHobE/DYv9aXjF7dPnz58DkG6yaI70yUt5paamQi6Xo1q1avmuw9xkPOwLw2OfGhb707BKMi+ZXTGVkJCA7OxsuLq6qrW7urri2rVrWreJjY3Vun5sbKzW9RcuXIi5c+dqtNeqVUvPqImIqLieP3+OypUrmzoMDfrkpbxmzJgBd3d3dOvWLd91mJuIiMyLLnnJ7IqpkjBr1iy1M1kKhQJPnjxB9erVIZPJiry/5ORkeHp64v79+3B0dDRkqKUS+8Ow2J+Gxf40vOL2qRACz58/h7u7uxGiM73PP/8c27Ztw/Hjx2FnZ5fvesxNxsO+MDz2qWGxPw2rJPOS2RVTTk5OsLS0RFxcnFp7XFwc3NzctG7j5uZWpPVtbW1ha2ur1lalShX9g/5/jo6O/A+QC/vDsNifhsX+NLzi9Kk5npFS0icvKX311Vf4/PPP8dtvv6FJkyYFrsvcZHzsC8NjnxoW+9OwSiIvmd0EFDY2NvDz81O7SFd50W6bNm20btOmTRuNi3rDw8PzXZ+IiEhX+uQlAPjyyy8xf/58HDp0CM2bNy+JUImIqISZ3ZkpAAgJCcHIkSPRvHlztGzZEkuXLkVKSgpGjx4NAAgICICHhwcWLlwIAHj//ffRqVMnfP311+jduze2bduG8+fP44cffjDlr0FERGVEUfPSF198gdDQUGzZsgXe3t6qa3grVqyIihUrmuz3ICIiwzLLYmrQoEF4/PgxQkNDERsbi2bNmuHQoUOqi3/v3bsHC4uck2pt27bFli1b8PHHH2P27NmoW7cu9uzZg0aNGpVIvLa2tggLC9MYnlFesT8Mi/1pWOxPwysPfVrUvLRy5UpkZmZiwIABavsJCwvDnDlzSiTm8vB30RX7wvDYp4bF/jSskuxPmTDXuWiJiIiIiIjMmNldM0VERERERFQasJgiIiIiIiLSA4spIiIiIiIiPbCYIiIiIiIi0gOLqWI4ceIE+vTpA3d3d8hkMuzZs8fUIZnMnDlzIJPJ1B6+vr6mDqtUKez1JIRAaGgoatSoAXt7e3Tr1g03b940TbClQGH9OWrUKI3XbM+ePU0TbCmwcOFCtGjRApUqVYKLiwv69euH69evq62Tnp6OoKAgVK9eHRUrVsTbb7+tcaNbMj7mphzMTcXDvGR4zE2GZQ65icVUMaSkpKBp06ZYvny5qUMxCw0bNsSjR49Uj1OnTpk6pFKlsNfTl19+iW+//RarVq3C2bNnUaFCBfj7+yM9Pb2EIy0ddPn/2bNnT7XX7NatW0swwtLl999/R1BQEM6cOYPw8HDI5XL06NEDKSkpqnWmTp2K/fv3Y+fOnfj999/x8OFD9O/f34RRl0/MTeqYm/THvGR4zE2GZRa5SZBBABA///yzqcMwmbCwMNG0aVNTh1Fm5H09KRQK4ebmJhYtWqRqe/bsmbC1tRVbt241QYSli7b/nyNHjhR9+/Y1STxlQXx8vAAgfv/9dyGE9Hq0trYWO3fuVK1z9epVAUBERkaaKsxyj7mJuclQmJcMj7nJ8EyRm3hmigzm5s2bcHd3h4+PD4YNG4Z79+6ZOqQy486dO4iNjUW3bt1UbZUrV0arVq0QGRlpwshKt+PHj8PFxQX16tXDhAkTkJiYaOqQSo2kpCQAQLVq1QAAf/31F+Ryudpr1NfXFy+99BJfo2RSzE3GwbxkPMxN+jNFbmIxRQbRqlUrbNiwAYcOHcLKlStx584ddOjQAc+fPzd1aGVCbGwsAMDV1VWt3dXVVbWMiqZnz5748ccfERERgS+++AK///47Xn/9dWRnZ5s6NLOnUCgwZcoUtGvXDo0aNQIgvUZtbGxQpUoVtXX5GiVTYm4yHuYl42Bu0p+pcpOVQfZC5d7rr7+u+rlJkyZo1aoVvLy8sGPHDowZM8aEkRFpN3jwYNXPjRs3RpMmTVC7dm0cP34cXbt2NWFk5i8oKAhXrlzhtSdk9pibqLRhbtKfqXITz0yRUVSpUgUvv/wybt26ZepQygQ3NzcA0Jh9Ji4uTrWMisfHxwdOTk58zRYiODgYv/zyC44dO4aaNWuq2t3c3JCZmYlnz56prc/XKJkT5ibDYV4qGcxNujFlbmIxRUbx4sULREVFoUaNGqYOpUyoVasW3NzcEBERoWpLTk7G2bNn0aZNGxNGVnY8ePAAiYmJfM3mQwiB4OBg/Pzzzzh69Chq1aqlttzPzw/W1tZqr9Hr16/j3r17fI2S2WBuMhzmpZLB3FQwc8hNHOZXDC9evFD7puDOnTu4dOkSqlWrhpdeesmEkZW8adOmoU+fPvDy8sLDhw8RFhYGS0tLDBkyxNShlRqFvZ6mTJmCBQsWoG7duqhVqxY++eQTuLu7o1+/fqYL2owV1J/VqlXD3Llz8fbbb8PNzQ1RUVGYPn066tSpA39/fxNGbb6CgoKwZcsW7N27F5UqVVKNNa9cuTLs7e1RuXJljBkzBiEhIahWrRocHR0xadIktGnTBq1btzZx9OULc1MO5qbiYV4yPOYmwzKL3GSQOQHLqWPHjgkAGo+RI0eaOrQSN2jQIFGjRg1hY2MjPDw8xKBBg8StW7dMHVapUtjrSaFQiE8++US4uroKW1tb0bVrV3H9+nXTBm3GCurP1NRU0aNHD+Hs7Cysra2Fl5eXCAwMFLGxsaYO22xp60sAYv369ap10tLSxMSJE0XVqlWFg4ODeOutt8SjR49MF3Q5xdyUg7mpeJiXDI+5ybDMITfJ/j8QIiIiIiIiKgJeM0VERERERKQHFlNERERERER6YDFFRERERESkBxZTREREREREemAxRUREREREpAcWU0RERERERHpgMUVERERERKQHFlNERERERER6YDFFREUmk8nQuXNnU4dBREQEgHmJTIfFFJERREdHQyaTqT2sra3h4eGBd955B+fPnzd1iEREVI4wLxEZh5WpAyAqy2rXro3hw4cDAFJSUvDXX39h586d2LNnD3777Td07NjRxBESEVF5wrxEZFgspoiMqE6dOpgzZ45a2+eff45Zs2bhk08+we+//26awIiIqFxiXiIyLA7zIyphY8aMAQD89ddfau0JCQmYMmUKatWqBVtbW7i4uOCdd97BlStXNPbRuXNnyGQyrfsfNWoUZDIZoqOjVW0bNmyATCbDhg0bcOTIEbRt2xYODg6oXr06Ro4cicTERK37WrNmDRo1agQ7Ozt4enpi+vTpSE9P1/M3JyIic8S8RKQ/npkiMhErq5z/fo8fP0abNm0QFRWFzp07Y/Dgwbhz5w527dqFAwcO4PDhw2jfvn2xj7lv3z4cOHAAffr0Qdu2bXHixAn8+OOPiIqKwqlTp9TWnT9/PkJDQ+Hq6orAwEBYW1tj+/btuHr1arHjICIi88O8RFR0LKaIStiaNWsAQC0JzZgxA1FRUZg1axY+++wzVfvBgwfRu3dvjB49GtevX4eFRfFOJu/fvx/Hjx9Hu3btAADZ2dno1q0bjh8/jjNnzqB169YAgFu3bmHevHnw8PDAhQsX4OLiAgCYM2cOWrZsWawYiIjIvDAvEemPw/yIjOjWrVuYM2cO5syZgw8//BBdunTB7Nmz4erqikWLFgEAMjMzsXXrVlSvXh0ff/yx2va9evVC9+7dcevWLfzxxx/Fjmfo0KGqhAUAlpaWGDlyJADg3LlzqvYtW7YgKysLISEhqoQFAI6OjhoxEhFR6cG8RGRYPDNFZERRUVGYO3euWpubmxtOnjyJOnXqAACuXbuG9PR0vPbaa3BwcNDYx2uvvYbw8HBcunQJHTp0KFY8fn5+Gm01a9YEADx79kzV9vfffwOA1uMVNwYiIjId5iUiw+KZKSIj8vf3hxACQgjEx8dj0aJFiI+Px5tvvokXL14AAJKTkwEArq6uWvdRo0YNtfWKw9HRUaNNOUY+Oztb1ZaUlAQAat/+KeUXJxERmT/mJSLDYjFFVEKcnZ0xbdo0zJ49G1evXlUNS1Amkri4OK3bxcbGqq0HQDVGPSsrS2N9ZcIpjsqVKwMA4uPjNZblFycREZUuzEtExcdiiqiEzZ49G+7u7lixYgWio6Ph6+sLOzs7nDt3DqmpqRrrHz9+HADQrFkzVVvVqlUBADExMWrrKhQK1VCI4mjatCkA4OTJkxrLtLUREVHpxbxEpD8WU0QlzN7eHjNmzIBcLsf8+fNhY2ODIUOGICEhAQsXLlRb99ChQzh8+DDq1KmjdoFuixYtAEj36cht8eLFuHPnTrFjHDp0KCwtLbF48WK1bwGTk5OxYMGCYu+fiIjMB/MSkf5YTBGZwLhx4+Du7q66l8YXX3wBHx8fLFiwAF27dsXs2bMxdOhQ9OnTBw4ODli/fr3a9LOjR49G1apVMWfOHLz11luYNm0aOnfujM8//xydOnUqdnx16tRBaGgoYmJi0KRJE0yePBkhISFo3Lgx6tatW+z9ExGReWFeItIPiykiE7Czs8OsWbOQlZWFuXPnwtnZGWfPnsXkyZMRFRWFr776CuHh4ejXrx/Onj2rcWNEV1dXHDt2DF27dsWRI0ewevVqVKlSBWfOnIG3t7dBYgwNDcXq1atRvXp1fP/999i5cyfeeecd7NixwyD7JyIi88G8RKQfmRBCmDoIIiIiIiKi0oZnpoiIiIiIiPTAYoqIiIiIiEgPLKaIiIiIiIj0wGKKiIiIiIhIDyymiIiIiIiI9MBiioiIiIiISA8spoiIiIiIiPTAYoqIiIiIiEgPLKaIiIiIiIj0wGKKiIiIiIhIDyymiIiIiIiI9MBiioiIiIiISA//BxFKjUHVmVR3AAAAAElFTkSuQmCC", "text/plain": [ "

" ] @@ -149,13 +101,23 @@ } ], "source": [ + "color1 = 'blue'\n", + "color2 = 'red'\n", + "\n", "def viz():\n", " fig, axs = plt.subplots(figsize=(10, 2), nrows=1, ncols=2)\n", " \n", - " # cifar100 - fedavg\n", - " axs[0].plot([r for r, _ in fedavg_cifar], [a for _, a in fedavg_cifar], label='FedAvg', linewidth=2.0)\n", - " \n", + " # cifar100\n", + " axs[0].plot([r for r, _ in fedavg_cifar], [a for _, a in fedavg_cifar], label='FedAvg', color=color1, linewidth=2.0)\n", + " axs[0].scatter([r for r, _ in fedpft_cifar], [a for _, a in fedpft_cifar], label='FedPFT', color=color2, marker='x', s=100)\n", " axs[0].set_title('CIFAR100 - ResNet50')\n", + " axs[0].set_ylim(0, 0.7)\n", + " \n", + " # caltech101\n", + " axs[1].plot([r for r, _ in fedavg_caltech], [a for _, a in fedavg_caltech], label='FedAvg', color=color1, linewidth=2.0)\n", + " axs[1].scatter([r for r, _ in fedpft_caltech], [a for _, a in fedpft_caltech], label='FedPFT', color=color2, marker='x', s=100)\n", + " axs[1].set_title('Caltech101 - Clip/ViT-B')\n", + " axs[1].set_ylim(0.2, 1)\n", " \n", " for ax in axs:\n", " ax.set_xticks([1, 5, 10 , 15, 20])\n", @@ -171,12 +133,12 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 22, "id": "92460065", "metadata": {}, "outputs": [], "source": [ - "saveFig(\"FedProx_mnist.png\", f)" + "saveFig(\"FedPft.png\", f)" ] } ], From 9964e770e90a228e68980f0269c55f6374f8fd31 Mon Sep 17 00:00:00 2001 From: mahdi Date: Mon, 15 Apr 2024 15:08:56 +0000 Subject: [PATCH 21/29] remove empty line --- baselines/fedpft/fedpft/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/baselines/fedpft/fedpft/models.py b/baselines/fedpft/fedpft/models.py index 7ebc9beed4a..b4594cb986a 100644 --- a/baselines/fedpft/fedpft/models.py +++ b/baselines/fedpft/fedpft/models.py @@ -103,7 +103,7 @@ def extract_features( 2D array containing labels of `features`. """ feature_extractor.to(device) - + features, labels = [], [] for dict in dataloader: batch_samples = dict["img"].to(device) @@ -164,7 +164,7 @@ def test( total += samples.shape[0] running_loss = nn.CrossEntropyLoss()(output, labels) loss += running_loss - + return loss.cpu().item(), correct / total From af4277759c020655c7f4a3f60ce35208ce617994 Mon Sep 17 00:00:00 2001 From: mahdi Date: Mon, 15 Apr 2024 15:12:52 +0000 Subject: [PATCH 22/29] removed extended_readme --- baselines/fedpft/EXTENDED_README.md | 123 ---------------------------- 1 file changed, 123 deletions(-) delete mode 100644 baselines/fedpft/EXTENDED_README.md diff --git a/baselines/fedpft/EXTENDED_README.md b/baselines/fedpft/EXTENDED_README.md deleted file mode 100644 index 9c8f5bc72fa..00000000000 --- a/baselines/fedpft/EXTENDED_README.md +++ /dev/null @@ -1,123 +0,0 @@ - -# Extended Readme - -> The baselines are expected to run in a machine running Ubuntu 22.04 - -While `README.md` should include information about the baseline you implement and how to run it, this _extended_ readme provides info on what's the expected directory structure for a new baseline and more generally the instructions to follow before your baseline can be merged into the Flower repository. Please follow closely these instructions. It is likely that you have already completed steps 1-2. - -1. Fork the Flower repository and clone it. -2. Navigate to the `baselines/` directory and from there run: - ```bash - # This will create a new directory with the same structure as this `baseline_template` directory. - ./dev/create-baseline.sh - ``` -3. All your code and configs should go into a sub-directory with the same name as the name of your baseline. - * The sub-directory contains a series of Python scripts that you can edit. Please stick to these files and consult with us if you need additional ones. - * There is also a basic config structure in `/conf` ready be parsed by [Hydra](https://hydra.cc/) when executing your `main.py`. -4. Therefore, the directory structure in your baseline should look like: - ```bash - baselines/ - ├── README.md # describes your baseline and everything needed to use it - ├── EXTENDED_README.md # to remove before creating your PR - ├── pyproject.toml # details your Python environment - └── - ├── *.py # several .py files including main.py and __init__.py - └── conf - └── *.yaml # one or more Hydra config files - - ``` -> :warning: Make sure the variable `name` in `pyproject.toml` is set to the name of the sub-directory containing all your code. - -5. Add your dependencies to the `pyproject.toml` (see below a few examples on how to do it). Read more about Poetry below in this `EXTENDED_README.md`. -6. Regularly check that your coding style and the documentation you add follow good coding practices. To test whether your code meets the requirements, please run the following: - ```bash - # After activating your environment and from your baseline's directory - cd .. # to go to the top-level directory of all baselines - ./dev/test-baseline.sh - ./dev/test-baseline-structure.sh - ``` - Both `test-baseline.sh` and `test-baseline-structure.sh` will also be automatically run when you create a PR, and both tests need to pass for the baseline to be merged. - To automatically solve some formatting issues and apply easy fixes, please run the formatting script: - ```bash - # After activating your environment and from your baseline's directory - cd .. # to go to the top-level directory of all baselines - ./dev/format-baseline.sh - ``` -7. Ensure that the Python environment for your baseline can be created without errors by simply running `poetry install` and that this is properly described later when you complete the `Environment Setup` section in `README.md`. This is specially important if your environment requires additional steps after doing `poetry install`. -8. Ensure that your baseline runs with default arguments by running `poetry run python -m .main`. Then, describe this and other forms of running your code in the `Running the Experiments` section in `README.md`. -9. Once your code is ready and you have checked: - * that following the instructions in your `README.md` the Python environment can be created correctly - - * that running the code following your instructions can reproduce the experiments in the paper - - , then you just need to create a Pull Request (PR) to kickstart the process of merging your baseline into the Flower repository. - -> Once you are happy to merge your baseline contribution, please delete this `EXTENDED_README.md` file. - - -## About Poetry - -We use Poetry to manage the Python environment for each individual baseline. You can follow the instructions [here](https://python-poetry.org/docs/) to install Poetry in your machine. - - -### Specifying a Python Version (optional) -By default, Poetry will use the Python version in your system. In some settings, you might want to specify a particular version of Python to use inside your Poetry environment. You can do so with [`pyenv`](https://github.com/pyenv/pyenv). Check the documentation for the different ways of installing `pyenv`, but one easy way is using the [automatic installer](https://github.com/pyenv/pyenv-installer): -```bash -curl https://pyenv.run | bash # then, don't forget links to your .bashrc/.zshrc -``` - -You can then install any Python version with `pyenv install ` (e.g. `pyenv install 3.9.17`). Then, in order to use that version for your baseline, you'd do the following: - -```bash -# cd to your baseline directory (i.e. where the `pyproject.toml` is) -pyenv local - -# set that version for poetry -poetry env use - -# then you can install your Poetry environment (see the next setp) -``` - -### Installing Your Environment -With the Poetry tool already installed, you can create an environment for this baseline with commands: -```bash -# run this from the same directory as the `pyproject.toml` file is -poetry install -``` - -This will create a basic Python environment with just Flower and additional packages, including those needed for simulation. Next, you should add the dependencies for your code. It is **critical** that you fix the version of the packages you use using a `=` not a `=^`. You can do so via [`poetry add`](https://python-poetry.org/docs/cli/#add). Below are some examples: - -```bash -# For instance, if you want to install tqdm -poetry add tqdm==4.65.0 - -# If you already have a requirements.txt, you can add all those packages (but ensure you have fixed the version) in one go as follows: -poetry add $( cat requirements.txt ) -``` -With each `poetry add` command, the `pyproject.toml` gets automatically updated so you don't need to keep that `requirements.txt` as part of this baseline. - - -More critically however, is adding your ML framework of choice to the list of dependencies. For some frameworks you might be able to do so with the `poetry add` command. Check [the Poetry documentation](https://python-poetry.org/docs/cli/#add) for how to add packages in various ways. For instance, let's say you want to use PyTorch: - -```bash -# with plain `pip` you'd run a command such as: -pip install torch==1.13.1+cu117 torchvision==0.14.1+cu117 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu117 - -# to add the same 3 dependencies to your Poetry environment you'd need to add the URL to the wheel that the above pip command auto-resolves for you. -# You can find those wheels in `https://download.pytorch.org/whl/cu117`. Copy the link and paste it after the `poetry add` command. -# For instance to add `torch==1.13.1+cu117` and a x86 Linux system with Python3.8 you'd: -poetry add https://download.pytorch.org/whl/cu117/torch-1.13.1%2Bcu117-cp38-cp38-linux_x86_64.whl -# you'll need to repeat this for both `torchvision` and `torchaudio` -``` -The above is just an example of how you can add these dependencies. Please refer to the Poetry documentation to extra reference. - -If all attempts fail, you can still install packages via standard `pip`. You'd first need to source/activate your Poetry environment. -```bash -# first ensure you have created your environment -# and installed the base packages provided in the template -poetry install - -# then activate it -poetry shell -``` -Now you are inside your environment (pretty much as when you use `virtualenv` or `conda`) so you can install further packages with `pip`. Please note that, unlike with `poetry add`, these extra requirements won't be captured by `pyproject.toml`. Therefore, please ensure that you provide all instructions needed to: (1) create the base environment with Poetry and (2) install any additional dependencies via `pip` when you complete your `README.md`. \ No newline at end of file From b600ed32df04a1850fbab50b7372bb2e96c6f58d Mon Sep 17 00:00:00 2001 From: jafermarq Date: Tue, 23 Apr 2024 08:14:10 +0000 Subject: [PATCH 23/29] minor changes --- baselines/fedpft/README.md | 6 +++--- baselines/fedpft/pyproject.toml | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/baselines/fedpft/README.md b/baselines/fedpft/README.md index c25f6de8f71..45eee5954d7 100644 --- a/baselines/fedpft/README.md +++ b/baselines/fedpft/README.md @@ -1,8 +1,8 @@ --- title: Parametric Feature Transfer, One-shot Federated Learning with Foundation Models url: https://arxiv.org/abs/2402.01862 -labels: [foundation-models, pre-trained, one-shot, one-round] # please add between 4 and 10 single-word (maybe two-words) labels (e.g. system heterogeneity, image classification, asynchronous, weight sharing, cross-silo). Do not use "" -dataset: [CIFAR100, Caltech101] # list of datasets you include in your baseline. Do not use "" +labels: [foundation-models, pre-trained, one-shot, one-round] +dataset: [CIFAR-100, Caltech101] --- # FedPFT: One-shot Federated Learning with Foundation Models @@ -109,4 +109,4 @@ python -m fedpft.main strategy=fedavg client=fedavg dataset=Caltech101 model=cli The above commands would generate results that you can plot and would look like the plot shown below. This plot was generated using the jupyter notebook in the `docs/` directory of this baseline after running the commands above. -![](_static/FedPft.png) \ No newline at end of file +![](_static/FedPft.png) diff --git a/baselines/fedpft/pyproject.toml b/baselines/fedpft/pyproject.toml index 30e47defbda..11bbddd0e17 100644 --- a/baselines/fedpft/pyproject.toml +++ b/baselines/fedpft/pyproject.toml @@ -58,6 +58,7 @@ pytest = "==6.2.4" pytest-watch = "==4.2.0" ruff = "==0.0.272" types-requests = "==2.27.7" +virtualenv = "==20.21.0" [tool.isort] line_length = 88 From 8732803d767634c66b49dd9242bdb9c6df637433 Mon Sep 17 00:00:00 2001 From: jafermarq Date: Tue, 23 Apr 2024 09:14:46 +0000 Subject: [PATCH 24/29] minor fix --- baselines/fedpft/fedpft/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baselines/fedpft/fedpft/models.py b/baselines/fedpft/fedpft/models.py index b4594cb986a..b9b85d4c4cd 100644 --- a/baselines/fedpft/fedpft/models.py +++ b/baselines/fedpft/fedpft/models.py @@ -126,7 +126,7 @@ def test( feature_extractor: torch.nn.Module, device: torch.device, ) -> Tuple[float, float]: - """"Evaluates the `classifier_head` on the dataset. + """Evaluates the `classifier_head` on the dataset. Parameters ---------- From 6a9bb2b07b0f723e36300b3d780d1eec1f2f21f3 Mon Sep 17 00:00:00 2001 From: mahdi Date: Wed, 24 Apr 2024 00:24:19 +0000 Subject: [PATCH 25/29] fix formating --- baselines/fedpft/README.md | 14 +++---- baselines/fedpft/fedpft/client.py | 4 +- baselines/fedpft/fedpft/dataset.py | 5 ++- baselines/fedpft/fedpft/main.py | 1 + baselines/fedpft/fedpft/models.py | 59 +++++++++++++++-------------- baselines/fedpft/fedpft/server.py | 10 +++-- baselines/fedpft/fedpft/strategy.py | 4 +- baselines/fedpft/fedpft/utils.py | 37 +++++++++--------- 8 files changed, 74 insertions(+), 60 deletions(-) diff --git a/baselines/fedpft/README.md b/baselines/fedpft/README.md index dc02d60a76a..1856223ba45 100644 --- a/baselines/fedpft/README.md +++ b/baselines/fedpft/README.md @@ -18,11 +18,11 @@ dataset: [CIFAR-100, Caltech101] ## About this baseline -**What’s implemented:** The code in this directory replicates the centralized experiments in *Parametric Feature Transfer, One-shot Federated Learning with Foundation Models* (Beitollahi et al., 2024) for CIFAR100 and Caltech101 datasets, which proposed the FedPFT algorithm. Concretely, it replicates the results in Section 5.2. +**What’s implemented:** The code in this directory replicates the centralized experiments in *Parametric Feature Transfer, One-shot Federated Learning with Foundation Models* (Beitollahi et al., 2024) for CIFAR-100 and Caltech101 datasets, which proposed the FedPFT algorithm. Concretely, it replicates the results in Section 5.2. -**Datasets:** CIFAR100 and Caltech101 from HuggingFace +**Datasets:** CIFAR-100 and Caltech101 from HuggingFace -**Hardware Setup:** These experiments were run on a desktop machine with 8 CPU threads and Nvidia 4070 with 8Gigs of ram. +**Hardware Setup:** These experiments were run on a desktop machine with 8 CPU threads and Nvidia 4070 with 8 gigs of memory. **Contributors:** Mahdi Beitollahi @@ -32,14 +32,14 @@ dataset: [CIFAR-100, Caltech101] **Task:** Image classification **Model:** This directory utilizes two pre-trained, frozen models as shown in Table 1 of the paper: -* ResNet50 pre-trained on ImageNet is used for CIFAR100 dataset(see `models/resnet50`). +* ResNet50 pre-trained on ImageNet is used for CIFAR-100 dataset(see `models/resnet50`). * CLIP, ViT-B/32 pre-trained on web dataset is used for Caltech101 dataset (see `models/clip_vit`) -**Dataset:** This baseline includes the CIFAR100 and Caltech101 datasets. By default, it will be partitioned into 50 clients following a Dirichlet distribution with $\alpha$=0.1. +**Dataset:** This baseline includes the CIFAR-100 and Caltech101 datasets. By default, it will be partitioned into 50 clients following a Dirichlet distribution with $\alpha$=0.1. | Dataset | #classes | #partitions | partitioning method | partition settings | | :------ | :---: | :---: | :---: | :---: | -| CIFAR100 | 100 | 50 | Dirichlet distribution | $\alpha$=0.1 | +| CIFAR-100 | 100 | 50 | Dirichlet distribution | $\alpha$=0.1 | | Caltech101 | 101 | 50 | Dirichlet distribution | $\alpha$=0.1 | **Training Hyperparameters:** The following table shows the main hyperparameters for this baseline with their default value (i.e. the value used if you run `python main.py` directly) @@ -75,7 +75,7 @@ poetry install ## Running the Experiments -To run this FedPFT with CIFAR100 baseline, first ensure you have activated your Poetry environment (execute `poetry shell` from this directory), then: +To run this FedPFT with CIFAR-100 baseline, first ensure you have activated your Poetry environment (execute `poetry shell` from this directory), then: ```bash python -m fedpft.main # this will run using the default settings in the `conf/config.yaml` diff --git a/baselines/fedpft/fedpft/client.py b/baselines/fedpft/fedpft/client.py index 434055808f8..c20aefe9236 100644 --- a/baselines/fedpft/fedpft/client.py +++ b/baselines/fedpft/fedpft/client.py @@ -22,6 +22,7 @@ class FedPFTClient(fl.client.NumPyClient): """Flower FedPFTClient.""" + # pylint: disable=too-many-arguments def __init__( self, trainloader: DataLoader, @@ -83,7 +84,7 @@ def fit( features=features, labels=labels, n_mixtures=int(config["n_mixtures"]), - cov_type=config["cov_type"], + cov_type=str(config["cov_type"]), seed=int(config["seed"]), tol=float(config["tol"]), max_iter=int(config["max_iter"]), @@ -130,6 +131,7 @@ def fit( return self.get_parameters(config={}), len(self.trainloader.dataset), {} +# pylint: disable=too-many-arguments def generate_client_fn( client_cfg: DictConfig, trainloaders: List[DataLoader], diff --git a/baselines/fedpft/fedpft/dataset.py b/baselines/fedpft/fedpft/dataset.py index 733234074ef..df41d10996a 100644 --- a/baselines/fedpft/fedpft/dataset.py +++ b/baselines/fedpft/fedpft/dataset.py @@ -8,9 +8,11 @@ from torchvision import transforms +# pylint: disable=too-many-instance-attributes class Dataset: """Dataset class.""" + # pylint: disable=too-many-locals, too-many-arguments def __init__( self, dataset: str, @@ -30,7 +32,7 @@ def __init__( Parameters ---------- dataset : str - Name of dataset to be downloaded from HuggingFace. + Name or path of the dataset to be downloaded from HuggingFace. num_clients: int Number of clients. batch_size: int @@ -60,6 +62,7 @@ def __init__( self.seed = seed self.split_size = split_size self.image_column_name = image_column_name + self.kwargs = kwargs def get_loaders(self): """Partition the datasets and return a list of dataloaders.""" diff --git a/baselines/fedpft/fedpft/main.py b/baselines/fedpft/fedpft/main.py index 9860b1232bf..debc4b35f52 100644 --- a/baselines/fedpft/fedpft/main.py +++ b/baselines/fedpft/fedpft/main.py @@ -13,6 +13,7 @@ from fedpft.client import generate_client_fn +# pylint: disable=too-many-locals @hydra.main(config_path="conf", config_name="base", version_base=None) def main(cfg: DictConfig) -> None: """Run federated learning with frozen, pre-trained models. diff --git a/baselines/fedpft/fedpft/models.py b/baselines/fedpft/fedpft/models.py index b9b85d4c4cd..3a57cf52141 100644 --- a/baselines/fedpft/fedpft/models.py +++ b/baselines/fedpft/fedpft/models.py @@ -8,6 +8,7 @@ import torch.utils import torchvision.transforms as transforms from flwr.common.logger import log +from numpy.typing import NDArray from torch import nn from torch.utils.data import DataLoader from torchvision import models @@ -16,17 +17,17 @@ def resnet50() -> torch.nn.modules: """Return ResNet-50 model as feature extractor.""" - resnet50 = models.resnet50(weights=models.ResNet50_Weights.DEFAULT) + resnet50_model = models.resnet50(weights=models.ResNet50_Weights.DEFAULT) # Remove last layer and flatten outputs - resnet50 = torch.nn.Sequential( - *(list(resnet50.children())[:-1]), torch.nn.Flatten() + resnet50_model = torch.nn.Sequential( + *(list(resnet50_model.children())[:-1]), torch.nn.Flatten() ) # Set the hidden_dimension - resnet50.hidden_dimension = 2048 + resnet50_model.hidden_dimension = 2048 - return resnet50 + return resnet50_model def clip_vit(name: str) -> torch.nn.modules: @@ -47,9 +48,10 @@ def __init__(self, vision_model): self.vision_model = vision_model self.hidden_dimension = vision_model.config.hidden_size - def forward(self, input): - output = self.vision_model(input) - return output[1] # return pooled_output (CLS token) + def forward(self, x): + """Return pooled output (CLS token).""" + output = self.vision_model(x) + return output[1] vision_model = CLIPModel.from_pretrained(name).vision_model @@ -71,18 +73,18 @@ def transform(mean: List, std: List) -> transforms.Compose: transforms.Compose Transform function for normalizing images """ - tr = transforms.Compose( + transform_comp = transforms.Compose( [ transforms.ToTensor(), transforms.Normalize(mean, std), ] ) - return tr + return transform_comp def extract_features( dataloader: DataLoader, feature_extractor: torch.nn.Module, device: torch.device -) -> Tuple[np.array, np.array]: +) -> Tuple[NDArray, NDArray]: """Extract features and labels from images using feature extractor. Parameters @@ -97,27 +99,27 @@ def extract_features( Returns ------- - features : np.array + features : NDArray 2D array containing features extracted from `feature_extractor`. - labels : np.array + labels : NDArray 2D array containing labels of `features`. """ feature_extractor.to(device) features, labels = [], [] - for dict in dataloader: - batch_samples = dict["img"].to(device) - batch_label = dict["label"].to(device) + for sample in dataloader: + batch_samples = sample["img"].to(device) + batch_label = sample["label"].to(device) with torch.no_grad(): feature = feature_extractor(batch_samples) features.append(feature.cpu().detach().numpy()) labels.append(batch_label.cpu().detach().numpy()) # reshape feauturs and labels into a single numpy array - features = np.concatenate(features, axis=0, dtype=np.float64) - labels = np.concatenate(labels, dtype=int) + features_np = np.concatenate(features, axis=0).astype("float64") + labels_np = np.concatenate(labels) - return features, labels + return features_np, labels_np def test( @@ -153,9 +155,9 @@ def test( feature_extractor.to(device) correct, total, loss = 0, 0, 0 - for dict in dataloader: - samples = dict["img"].to(device) - labels = dict["label"].to(device) + for sample in dataloader: + samples = sample["img"].to(device) + labels = sample["label"].to(device) with torch.no_grad(): feature = feature_extractor(samples) output = classifier_head(feature) @@ -163,11 +165,12 @@ def test( correct += (pred == labels).sum().item() total += samples.shape[0] running_loss = nn.CrossEntropyLoss()(output, labels) - loss += running_loss + loss += running_loss.cpu().item() - return loss.cpu().item(), correct / total + return loss, correct / total +# pylint: disable=too-many-locals, too-many-arguments def train( classifier_head: torch.nn.Linear, dataloader: DataLoader, @@ -204,10 +207,10 @@ def train( for epoch in range(num_epochs): correct, total, loss = 0, 0, 0 - for _, dict in enumerate(dataloader): + for _, batch in enumerate(dataloader): classifier_head.zero_grad() - samples = dict["img"].to(device) - labels = dict["label"].to(device) + samples = batch["img"].to(device) + labels = batch["label"].to(device) if feature_extractor: with torch.no_grad(): samples = feature_extractor(samples) @@ -220,4 +223,4 @@ def train( running_loss.backward() opt.step() if verbose: - log(logging.INFO, f"Epoch:{epoch+1} --- Accuracy: {correct/total}") + log(logging.INFO, "Epoch: %s --- Accuracy: %s", epoch + 1, correct / total) diff --git a/baselines/fedpft/fedpft/server.py b/baselines/fedpft/fedpft/server.py index 00d88360e9f..9c6c605884d 100644 --- a/baselines/fedpft/fedpft/server.py +++ b/baselines/fedpft/fedpft/server.py @@ -29,6 +29,7 @@ def fedpft_get_on_fit_config_fn( Function to return a config with the `lr` and `num_epochs` """ + # pylint: disable=unused-argument def fit_config(server_round: int) -> Dict[str, str]: """Return a configuration for training Gaussian Mixtures.""" config = { @@ -44,14 +45,14 @@ def fit_config(server_round: int) -> Dict[str, str]: def fedavg_get_on_fit_config_fn( - lr: float, + learning_rate: float, num_epochs: int, ) -> Callable[[int], Dict[str, str]]: """Return a function which returns FedAvg training configurations. Parameters ---------- - lr : float + learning_rate : float Client's learning rate num_epochs : int Number of epochs for local learning of clients @@ -59,13 +60,14 @@ def fedavg_get_on_fit_config_fn( Returns ------- Callable[[int], Dict[str, str]] - Function to return a config with the `lr` and `num_epochs` + Function to return a config with the `learning_rate` and `num_epochs` """ + # pylint: disable=unused-argument def fit_config(server_round: int) -> Dict[str, str]: """Return a configuration number of epochs and learning rate.""" config = { - "lr": str(lr), + "lr": str(learning_rate), "num_epochs": str(num_epochs), } return config diff --git a/baselines/fedpft/fedpft/strategy.py b/baselines/fedpft/fedpft/strategy.py index f9546140124..2e4302bde45 100644 --- a/baselines/fedpft/fedpft/strategy.py +++ b/baselines/fedpft/fedpft/strategy.py @@ -68,6 +68,7 @@ def __init__( self.num_epochs = num_epochs self.device = device + # pylint: disable=too-many-locals def aggregate_fit( self, server_round: int, @@ -79,10 +80,11 @@ def aggregate_fit( if not self.accept_failures and failures: raise Exception("there are failures and failures are not accepted") + assert self.on_fit_config_fn is not None config = self.on_fit_config_fn(server_round) # Sample from the GMMs to create synthetic feature dataset - synthetic_features_dataset = [] + synthetic_features_dataset: List[Union[Dict, Tuple]] = [] for _, fit_res in results: # Convert byte parameters into ndarrays and GMMParameters ndarray = parameters_to_ndarrays(fit_res.parameters) diff --git a/baselines/fedpft/fedpft/utils.py b/baselines/fedpft/fedpft/utils.py index c1a27c14647..b7812d556d0 100644 --- a/baselines/fedpft/fedpft/utils.py +++ b/baselines/fedpft/fedpft/utils.py @@ -4,7 +4,7 @@ from typing import List import numpy as np -from flwr.common import NDArrays +from numpy.typing import NDArray from sklearn.mixture import GaussianMixture @@ -12,19 +12,19 @@ class GMMParameters: """GMM parameters.""" - label: int - means: NDArrays - weights: NDArrays - covariances: NDArrays - num_samples: int + label: NDArray + means: NDArray + weights: NDArray + covariances: NDArray + num_samples: NDArray -def gmmparam_to_ndarrays(gmm: GMMParameters) -> NDArrays: +def gmmparam_to_ndarrays(gmm: GMMParameters) -> List[NDArray]: """Convert gmm object to NumPy ndarrays.""" return [gmm.label, gmm.means, gmm.weights, gmm.covariances, gmm.num_samples] -def ndarrays_to_gmmparam(ndarrays: NDArrays) -> GMMParameters: +def ndarrays_to_gmmparam(ndarrays: NDArray) -> GMMParameters: """Convert NumPy ndarray to GMM object.""" return GMMParameters( label=ndarrays[0], @@ -35,9 +35,10 @@ def ndarrays_to_gmmparam(ndarrays: NDArrays) -> GMMParameters: ) +# pylint: disable=too-many-arguments def learn_gmm( - features: np.array, - labels: np.array, + features: NDArray, + labels: NDArray, n_mixtures: int, cov_type: str, seed: int, @@ -48,10 +49,10 @@ def learn_gmm( Parameters ---------- - features : np.array + features : NDArray A 2-d array with size (n_samples, feature_dimension) containing extracted features for all the samples. - labels : np.array + labels : NDArray An array with size (n_samples) containing labels associated for each sample in `features`. n_mixtures : int @@ -86,17 +87,17 @@ def learn_gmm( gmm.fit(cond_features) gmm_list.append( GMMParameters( - label=label, + label=np.array(label), means=gmm.means_.astype("float16"), weights=gmm.weights_.astype("float16"), covariances=gmm.covariances_.astype("float16"), - num_samples=len(cond_features), + num_samples=np.array(len(cond_features)), ) ) return gmm_list -def chunks(lst, n): - """Yield successive n-sized chunks from lst.""" - for i in range(0, len(lst), n): - yield lst[i : i + n] +def chunks(lst, chunk_size): + """Yield successive chunk_size-sized chunks from lst.""" + for i in range(0, len(lst), chunk_size): + yield lst[i : i + chunk_size] From ffc3d8c7307f6f6819a95bd8dcd6368a32712663 Mon Sep 17 00:00:00 2001 From: mahdi Date: Wed, 24 Apr 2024 17:23:18 +0000 Subject: [PATCH 26/29] add email address --- baselines/fedpft/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baselines/fedpft/README.md b/baselines/fedpft/README.md index 1856223ba45..cad603d124e 100644 --- a/baselines/fedpft/README.md +++ b/baselines/fedpft/README.md @@ -24,7 +24,7 @@ dataset: [CIFAR-100, Caltech101] **Hardware Setup:** These experiments were run on a desktop machine with 8 CPU threads and Nvidia 4070 with 8 gigs of memory. -**Contributors:** Mahdi Beitollahi +**Contributors:** Mahdi Beitollahi (mahdi.beitollahi@queensu.ca). ## Experimental Setup From ffb2be665ead2f0e379421e96271e8dfe1dd9261 Mon Sep 17 00:00:00 2001 From: jafermarq Date: Wed, 24 Apr 2024 21:08:50 +0000 Subject: [PATCH 27/29] minor update --- baselines/fedpft/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/baselines/fedpft/README.md b/baselines/fedpft/README.md index cad603d124e..8b0fb3d5752 100644 --- a/baselines/fedpft/README.md +++ b/baselines/fedpft/README.md @@ -22,7 +22,7 @@ dataset: [CIFAR-100, Caltech101] **Datasets:** CIFAR-100 and Caltech101 from HuggingFace -**Hardware Setup:** These experiments were run on a desktop machine with 8 CPU threads and Nvidia 4070 with 8 gigs of memory. +**Hardware Setup:** These experiments were run on a desktop machine with 8 CPU threads and Nvidia 4070 with 8GB of VRAM. **Contributors:** Mahdi Beitollahi (mahdi.beitollahi@queensu.ca). @@ -35,7 +35,7 @@ dataset: [CIFAR-100, Caltech101] * ResNet50 pre-trained on ImageNet is used for CIFAR-100 dataset(see `models/resnet50`). * CLIP, ViT-B/32 pre-trained on web dataset is used for Caltech101 dataset (see `models/clip_vit`) -**Dataset:** This baseline includes the CIFAR-100 and Caltech101 datasets. By default, it will be partitioned into 50 clients following a Dirichlet distribution with $\alpha$=0.1. +**Dataset:** This baseline includes the CIFAR-100 and Caltech101 datasets via [flwr-datasets](https://flower.ai/docs/datasets/). By default, it will be partitioned into 50 clients following a Dirichlet distribution with $\alpha$=0.1. | Dataset | #classes | #partitions | partitioning method | partition settings | | :------ | :---: | :---: | :---: | :---: | From 1ef9430660dc70802e5cbe5b9b396242c61f6c41 Mon Sep 17 00:00:00 2001 From: Mahdi Beitollahi <96784135+mahdibeit@users.noreply.github.com> Date: Thu, 25 Apr 2024 13:05:02 -0400 Subject: [PATCH 28/29] Update baselines/fedpft/README.md update readme Co-authored-by: Javier --- baselines/fedpft/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baselines/fedpft/README.md b/baselines/fedpft/README.md index 8b0fb3d5752..45bddfda610 100644 --- a/baselines/fedpft/README.md +++ b/baselines/fedpft/README.md @@ -103,7 +103,7 @@ python -m fedpft.main dataset=CIFAR100 model=resnet50 python -m fedpft.main dataset=Caltech101 model=clip # FedAvg with pre-trained, frozen models -python -m fedpft.main strategy=fedavg client=fedavg dataset=CIFAR100 model=resnet50 num_rounds=20 strategy.on_fit_config_fn.num_epochs=1=1 num_gpus=0.5 +python -m fedpft.main strategy=fedavg client=fedavg dataset=CIFAR100 model=resnet50 num_rounds=20 strategy.on_fit_config_fn.num_epochs=1 num_gpus=0.5 python -m fedpft.main strategy=fedavg client=fedavg dataset=Caltech101 model=clip num_rounds=20 num_gpus=0.2 ``` From 2f6d66de0009b7f153b80b62b82990a048fbce83 Mon Sep 17 00:00:00 2001 From: Mahdi Beitollahi <96784135+mahdibeit@users.noreply.github.com> Date: Thu, 25 Apr 2024 13:05:51 -0400 Subject: [PATCH 29/29] Update baselines/fedpft/fedpft/conf/strategy/fedavg.yaml fix config arg Co-authored-by: Javier --- baselines/fedpft/fedpft/conf/strategy/fedavg.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml b/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml index 166bcd10aef..b7703e78eb6 100644 --- a/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml +++ b/baselines/fedpft/fedpft/conf/strategy/fedavg.yaml @@ -5,7 +5,7 @@ fraction_evaluate: 1 accept_failures: False on_fit_config_fn: _target_: fedpft.server.fedavg_get_on_fit_config_fn - lr: 0.01 + learning_rate: 0.01 num_epochs: 10 evaluate_metrics_aggregation_fn: _target_: fedpft.server.weighted_average