From f674d2677d968ed27a24a869a2db79b589860f40 Mon Sep 17 00:00:00 2001 From: Javier Date: Thu, 16 Nov 2023 13:34:50 +0000 Subject: [PATCH] Remove old FedBN (#2609) --- README.md | 2 - baselines/README.md | 4 +- baselines/doc/source/how-to-use-baselines.rst | 2 +- .../fedbn/convergence_rate/README.md | 189 -------- .../fedbn/convergence_rate/__init__.py | 0 .../fedbn/convergence_rate/client.py | 402 ------------------ ...nvergence_rate_FedBN_FedAvg_comparison.png | Bin 34158 -> 0 bytes ...nvergence_rate_Flower_FedBN_comparison.png | Bin 36446 -> 0 bytes .../fedbn/convergence_rate/evaluation_plot.py | 47 -- .../fedbn/convergence_rate/run.sh | 24 -- .../fedbn/convergence_rate/server.py | 16 - .../fedbn/convergence_rate/utils/__init__.py | 0 .../fedbn/convergence_rate/utils/cnn_model.py | 48 --- .../convergence_rate/utils/data_download.py | 67 --- .../utils/data_download_raw.py | 186 -------- .../convergence_rate/utils/data_preprocess.py | 278 ------------ .../convergence_rate/utils/data_utils.py | 87 ---- .../convergence_rate/utils/mnistm/__init__.py | 0 .../utils/mnistm/create_mnistm.py | 71 ---- 19 files changed, 3 insertions(+), 1420 deletions(-) delete mode 100644 baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/README.md delete mode 100644 baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/__init__.py delete mode 100644 baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/client.py delete mode 100644 baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/convergence_rate_FedBN_FedAvg_comparison.png delete mode 100644 baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/convergence_rate_Flower_FedBN_comparison.png delete mode 100644 baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/evaluation_plot.py delete mode 100755 baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/run.sh delete mode 100644 baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/server.py delete mode 100644 baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/__init__.py delete mode 100644 baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/cnn_model.py delete mode 100644 baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/data_download.py delete mode 100644 baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/data_download_raw.py delete mode 100644 baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/data_preprocess.py delete mode 100644 baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/data_utils.py delete mode 100644 baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/mnistm/__init__.py delete mode 100644 baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/mnistm/create_mnistm.py diff --git a/README.md b/README.md index 002d16066e7..4f148d2445d 100644 --- a/README.md +++ b/README.md @@ -97,8 +97,6 @@ Flower Baselines is a collection of community-contributed experiments that repro - [MNIST](https://github.com/adap/flower/tree/main/baselines/flwr_baselines/flwr_baselines/publications/fedavg_mnist) - [FedProx](https://arxiv.org/abs/1812.06127): - [MNIST](https://github.com/adap/flower/tree/main/baselines/fedprox/) -- [FedBN: Federated Learning on non-IID Features via Local Batch Normalization](https://arxiv.org/abs/2102.07623): - - [Convergence Rate](https://github.com/adap/flower/tree/main/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate) - [Adaptive Federated Optimization](https://arxiv.org/abs/2003.00295): - [CIFAR-10/100](https://github.com/adap/flower/tree/main/baselines/flwr_baselines/flwr_baselines/publications/adaptive_federated_optimization) diff --git a/baselines/README.md b/baselines/README.md index 3b30ff1a9ea..a18c0553b2b 100644 --- a/baselines/README.md +++ b/baselines/README.md @@ -1,14 +1,14 @@ # Flower Baselines -> We are changing the way we structure the Flower baselines. While we complete the transition to the new format, you can still find the existing baselines in the `flwr_baselines` directory. Currently, you can make use of baselines for [FedAvg](https://github.com/adap/flower/tree/main/baselines/flwr_baselines/flwr_baselines/publications/fedavg_mnist), [FedProx](https://github.com/adap/flower/tree/main/baselines/flwr_baselines/flwr_baselines/publications/fedprox_mnist), [FedOpt](https://github.com/adap/flower/tree/main/baselines/flwr_baselines/flwr_baselines/publications/adaptive_federated_optimization), [FedBN](https://github.com/adap/flower/tree/main/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate), and [LEAF-FEMNIST](https://github.com/adap/flower/tree/main/baselines/flwr_baselines/flwr_baselines/publications/leaf/femnist). +> We are changing the way we structure the Flower baselines. While we complete the transition to the new format, you can still find the existing baselines in the `flwr_baselines` directory. Currently, you can make use of baselines for [FedAvg](https://github.com/adap/flower/tree/main/baselines/flwr_baselines/flwr_baselines/publications/fedavg_mnist), [FedOpt](https://github.com/adap/flower/tree/main/baselines/flwr_baselines/flwr_baselines/publications/adaptive_federated_optimization), and [LEAF-FEMNIST](https://github.com/adap/flower/tree/main/baselines/flwr_baselines/flwr_baselines/publications/leaf/femnist). > The documentation below has been updated to reflect the new way of using Flower baselines. ## Structure -Each baseline in this directory is fully self-contained in terms of source code it its own directory. In addition, each baseline uses its very own Python environment as designed by the contributors of such baseline in order to replicate the experiments in the paper. Each baseline directory contains the following structure: +Each baseline in this directory is fully self-contained in terms of source code in its own directory. In addition, each baseline uses its very own Python environment as designed by the contributors of such baseline in order to replicate the experiments in the paper. Each baseline directory contains the following structure: ```bash baselines// diff --git a/baselines/doc/source/how-to-use-baselines.rst b/baselines/doc/source/how-to-use-baselines.rst index b89c17b17a9..ed47438ad5a 100644 --- a/baselines/doc/source/how-to-use-baselines.rst +++ b/baselines/doc/source/how-to-use-baselines.rst @@ -3,7 +3,7 @@ Use Baselines .. warning:: We are changing the way we structure the Flower baselines. While we complete the transition to the new format, you can still find the existing baselines and use them: `baselines (old) `_. - Currently, you can make use of baselines for `FedAvg `_, `FedProx `_, `FedOpt `_, `FedBN `_, and `LEAF-FEMNIST `_. + Currently, you can make use of baselines for `FedAvg `_, `FedOpt `_, and `LEAF-FEMNIST `_. The documentation below has been updated to reflect the new way of using Flower baselines. diff --git a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/README.md b/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/README.md deleted file mode 100644 index 2b718afd7dd..00000000000 --- a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/README.md +++ /dev/null @@ -1,189 +0,0 @@ -# FedBN Baseline - Convergence Rate - -## Experiment Introduction - -The **FedBN - Convergence Rate** baseline is based on the paper [FEDBN: FEDERATED LEARNING ON NON-IID FEATURES VIA LOCAL BATCH NORMALIZATION](https://arxiv.org/pdf/2102.07623.pdf) and reproduces the results presented in *Chapter 5 - Convergence Rate (Figure 3)*. The implementation is based on the Flower federated learning framework. This experiment uses 5 completely different image datasets of digits to emulate a non-IID data distribution over the different clients. The experiment therefore uses 5 clients for the training. The local training is set up to perform 1 epoch and it uses a CNN model together with the SGD optimizer (cross-entropy loss). - -## Dataset - -### General Overview - -The following 5 different datasets are used to simulate a non-IID data distribution over 5 clients: - -* [MNIST](https://ieeexplore.ieee.org/document/726791) -* [MNIST-M]((https://arxiv.org/pdf/1505.07818.pdf)) -* [SVHN](http://ufldl.stanford.edu/housenumbers/nips2011_housenumbers.pdf) -* [USPS](https://ieeexplore.ieee.org/document/291440) -* [SynthDigits](https://arxiv.org/pdf/1505.07818.pdf) - -A more detailed explanation of the datasets is given in the following table. - -| | MNIST | MNIST-M | SVHN | USPS | SynthDigits | -|--- |--- |--- |--- |--- |--- | -| data type| handwritten digits| MNIST modification randomly colored with colored patches| Street view house numbers | handwritten digits from envelopes by the U.S. Postal Service | Syntehtic digits Windows TM font varying the orientation, blur and stroke colors | -| color | greyscale | RGB | RGB | greyscale | RGB | -| pixelsize | 28x28 | 28 x 28 | 32 x32 | 16 x16 | 32 x32 | -| labels | 0-9 | 0-9 | 1-10 | 0-9 | 1-10 | -| number of trainset | 60.000 | 60.000 | 73.257 | 9,298 | 50.000 | -| number of testset| 10.000 | 10.000 | 26.032 | - | - | -| image shape | (28,28) | (28,28,3) | (32,32,3) | (16,16) | (32,32,3) | - -### Dataset Download - -The [FedBN](https://arxiv.org/pdf/2102.07623.pdf) authors prepared a preprocessed dataset on their GitHub repository. It is available [here](https://github.com/med-air/FedBN). Please download the dataset, save it in a `/data` directory, and unzip it. - -The training data contains only 7438 samples and it is split into 10 files, but only one file is used for the **FedBN: Convergence Rate** baseline. Therefore, 743 samples are used for the local training. - -Run the following commands to download and preprocess the original data: - -```bash -# download data (will create a directory in ./path) -python utils/data_download.py - -# preprocess -python utils/data_preprocess.py -``` - -All the datasets (with the exception of SynthDigits) can be downloaded from the original sources: - -```bash -# download -python utils/data_download_raw.py - -# preprocess -python utils/data_preprocess.py -``` - -## Training Setup - -### CNN Architecture - -The CNN architecture is detailed in the paper and used to create the **FedBN - Convergence Rate** baseline. - -| Layer | Details| -| ----- | ------ | -| 1 | Conv2D(3,64, 5,1,2)
BN(64), ReLU, MaxPool2D(2,2) | -| 2 | Conv2D(64, 64, 5, 1, 2)
BN(64), ReLU, MaxPool2D(2,2) | -| 3 | Conv2D(64, 128, 5, 1, 2)
BN(128), ReLU | -| 4 | FC(6272, 2048)
BN(2048), ReLU | -| 5 | FC(2048, 512)
BN(512), ReLU | -| 6 | FC(512, 10) | - -### Training Paramaters - -| Description | Value | -| ----------- | ----- | -| training samples | 743 | -| mu | 10E-2 | -| local epochs | 1 | -| loss | cross entropy loss | -| optimizer | SGD | - -## Running the Experiment - -Before running any part of the experiment, [please get the required data](dataset-download), and place it in the `/data` directory. - -As soon as you have downloaded the data you are ready to start the baseline experiment. The baseline implementation is split across different files: - -* cnn_model.py -* client.py -* server.py -* run.sh -* utils/data_utils.py -* evaluation_plot.py - -In order to run the experiment, you simply make `run.sh` executable and run it: - -```bash -chmod +x run.sh -# train the CNN with FedAvg strategy -./run.sh fedavg -# train the CNN with FedBN strategy -./run.sh fedbn -``` - -The `run.sh` script first creates the files in which the evaluation results are saved and starts `server.py` and 5 clients in parallel with `client.py`. As explained before, each client loads a different dataset. The clients save the evaluation results after the parameters were sent from the server to the client and right before the local training. The saved parameters are included in a dict with the following information: - -```python -test_dict = {"dataset": self.num_examples["dataset"], "fl_round" : fl_round, "strategy": self.mode, "test_loss": loss, "test_accuracy": accuracy} -``` - -The `utils/data_utils.py` script prepares/loads the data for the training and `cnn_model.py` defines the [CNN model architecture](#cnn-architecture). This baseline only uses one single file with 743 samples from the downloaded dataset. - -If you want to compare the results of both strategy runs (FedAvg and FedBN) you can run: - -```bash -# create the evalutation plot -python evaluation_plot.py -``` - -This will create a plot `convergence_rate.png` including the train loss after the server aggregation from a certain client for the FedAvg and FedBN stretegy. A noticeable difference between both stretagies is visible for the SVHN dataset. The train loss has a more stable and steeper decrease when using FedBN over FedAvg. - -![Illustration of convergence rate on the SVHN client trained with Flower using FedBN and FedAvg.](convergence_rate_FedBN_FedAvg_comparison.png) - -This baseline was created to reproduce the [FedBN implementation available on GitHub](https://github.com/med-air/FedBN). The following diagram shows the convergence rate of the SVHN client with same strategy, FedBN, using the Flower implementation and the original FedBN implementation. The loss decrease is very similar in both cases. - -![Illustration of convergence rate on the SVHN client trained with Flower and the traditional FedBN code.](convergence_rate_Flower_FedBN_comparison.png) - -### Server - -This baseline compares Federated Averaging (FedAvg) with Federated Batch Normalization (FedBN). In both cases, we are using FedAvg on the server-side. All model parameters are sent from the client to the server and aggregated. However, in the case of FedBN, we are setting up the client to exclude batch norm layers from the transmission to the server. FedBN is therefore a strategy that only requires changes on the client-side. - -The server is kept very simple and remains the same in both settings. We are using FedAvg on the server-side with the parameters `min_fit_clients`, `min_eval_clients`, and `min_available_clients`. All are set to the value `5` since we have five clients to be trained and evaluated in each FL round. All in all, the *FedBN* paper runs 600 FL rounds that can be implemented as follows. - -```python -import flwr as fl - -if __name__ == "__main__": - strategy = fl.server.strategy.FedAvg( - min_fit_clients=5, - min_eval_clients=5, - min_available_clients=5, - ) - fl.server.start_server(server_address="[::]:8080", config={"num_rounds": 100}, strategy=strategy) - -``` - -### Client - -The client is a little more complex. However, it can be separated into different parts. The main parts are: - -* `load_partition()` - * load the right dataset -* `train()` - * perfom the local training -* `test()` - * evaluate the training results -* `FlowerClient(fl.client.NumPyClient)` - * start the Flower client -* `main()` - * start all previous process in a main() file - -The `load_partition()` function loads the datasets saved in the `/data` dierctory. - -You can directly see that the training and evaluation process is defined within the client. We are using PyTorch to train and evaluate the model with the parameters described in the section [Training Setup](#training-setup). - -The Flower client `FlowerClient(fl.client.NumPyClient)` implements the usual methods: - -* get_paramaters() -* set_parameters() -* fit() -* evaluate() - -Let us take a closer look at `set_parameters()` in order to understand the difference between FedAvg and FedBN: - -```python -def set_parameters(self, parameters: List[np.ndarray])-> None: - self.model.train() - if self.mode == 'fedbn': - keys = [k for k in self.model.state_dict().keys() if "bn" not in k] - params_dict = zip(keys, parameters) - state_dict = OrderedDict({k: torch.tensor(v) for k, v in params_dict}) - self.model.load_state_dict(state_dict, strict=False) - else: - params_dict = zip(self.model.state_dict().keys(), parameters) - state_dict = OrderedDict({k: torch.tensor(v) for k, v in params_dict}) - self.model.load_state_dict(state_dict, strict=True) -``` - -You can see that the clients update all layers of the PyTorch model with the values received from the server when using the plain FedAvg strategy (including batch norm layers). However, in the case of FedBN, the parameters for the batch norm layers (`bn`) are excluded. diff --git a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/__init__.py b/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/client.py b/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/client.py deleted file mode 100644 index db2ab647e1b..00000000000 --- a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/client.py +++ /dev/null @@ -1,402 +0,0 @@ -"""FedBN client.""" - - -import argparse -import json -from collections import OrderedDict -from typing import Dict, Tuple - -import flwr as fl -import torch -from flwr.common.typing import NDArrays, Scalar -from torch import nn -from torchvision import transforms - -from .utils.cnn_model import CNNModel -from .utils.data_utils import DigitsDataset - -FL_ROUND = 0 - -eval_list = [] - - -# pylint: disable=no-member -DEVICE: str = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") -# pylint: enable=no-member - - -# mypy: allow-any-generics -# pylint: disable= too-many-arguments, too-many-locals, global-statement -class FlowerClient(fl.client.NumPyClient): - """Flower client implementing image classification using PyTorch.""" - - def __init__( - self, - model: CNNModel, - trainloader: torch.utils.data.DataLoader, - testloader: torch.utils.data.DataLoader, - num_examples: Dict, - mode: str, - ) -> None: - self.model = model - self.trainloader = trainloader - self.testloader = testloader - self.num_examples = num_examples - self.mode = mode - - def get_parameters(self, config) -> NDArrays: - """Return model parameters as a list of NumPy ndarrays w or w/o using - BN layers.""" - self.model.train() - # pylint: disable = no-else-return - if self.mode == "fedbn": - # Excluding parameters of BN layers when using FedBN - return [ - val.cpu().numpy() - for name, val in self.model.state_dict().items() - if "bn" not in name - ] - else: - # Return all model parameters as a list of NumPy ndarrays - return [val.cpu().numpy() for _, val in self.model.state_dict().items()] - - def set_parameters(self, parameters: NDArrays) -> None: - """Set model parameters from a list of NumPy ndarrays Exclude the bn - layer if available.""" - self.model.train() - # pylint: disable=not-callable - if self.mode == "fedbn": - keys = [k for k in self.model.state_dict().keys() if "bn" not in k] - params_dict = zip(keys, parameters) - state_dict = OrderedDict({k: torch.tensor(v) for k, v in params_dict}) - self.model.load_state_dict(state_dict, strict=False) - else: - params_dict = zip(self.model.state_dict().keys(), parameters) - state_dict = OrderedDict({k: torch.tensor(v) for k, v in params_dict}) - self.model.load_state_dict(state_dict, strict=True) - # pylint: enable=not-callable - - def fit( - self, parameters: NDArrays, config: Dict[str, Scalar] - ) -> Tuple[NDArrays, int, Dict]: - """Set model parameters, train model, return updated model - parameters.""" - self.set_parameters(parameters) - test_loss, test_accuracy = test( - self.model, self.num_examples["dataset"], self.trainloader, device=DEVICE - ) - test_dict = { - "dataset": self.num_examples["dataset"], - "fl_round": FL_ROUND, - "strategy": self.mode, - "train_loss": test_loss, - "train_accuracy": test_accuracy, - } - loss, accuracy = train( - self.model, - self.trainloader, - self.num_examples["dataset"], - epochs=1, - device=DEVICE, - ) - eval_list.append(test_dict) - return ( - self.get_parameters({}), - self.num_examples["trainset"], - {"loss": loss, "accuracy": accuracy}, - ) - - def evaluate( - self, parameters: NDArrays, config: Dict[str, Scalar] - ) -> Tuple[float, int, Dict]: - """Set model parameters, evaluate model on local test dataset, return - result.""" - self.set_parameters(parameters) - global FL_ROUND - loss, accuracy = test( - self.model, self.num_examples["dataset"], self.testloader, device=DEVICE - ) - test_dict = { - "dataset": self.num_examples["dataset"], - "fl_round": FL_ROUND, - "strategy": self.mode, - "test_loss": loss, - "test_accuracy": accuracy, - } - eval_list.append(test_dict) - FL_ROUND += 1 - return ( - float(loss), - self.num_examples["testset"], - {"loss": loss, "accuracy": accuracy}, - ) - - -def load_partition( - dataset: str, -) -> Tuple[torch.utils.data.DataLoader, torch.utils.data.DataLoader, Dict]: - """Load 'MNIST', 'SVHN', 'USPS', 'SynthDigits', 'MNIST-M' for the training - and test data to simulate a partition.""" - - if dataset == "MNIST": - print(f"Load {dataset} dataset") - - transform = transforms.Compose( - [ - transforms.Grayscale(num_output_channels=3), - transforms.ToTensor(), - transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), - ] - ) - - trainset = DigitsDataset( - data_path="data/MNIST", - channels=1, - percent=0.1, - train=True, - transform=transform, - ) - testset = DigitsDataset( - data_path="data/MNIST", - channels=1, - percent=0.1, - train=False, - transform=transform, - ) - - elif dataset == "SVHN": - print(f"Load {dataset} dataset") - - transform = transforms.Compose( - [ - transforms.Resize([28, 28]), - transforms.ToTensor(), - transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), - ] - ) - - trainset = DigitsDataset( - data_path="data/SVHN", - channels=3, - percent=0.1, - train=True, - transform=transform, - ) - testset = DigitsDataset( - data_path="data/SVHN", - channels=3, - percent=0.1, - train=False, - transform=transform, - ) - - elif dataset == "USPS": - print(f"Load {dataset} dataset") - - transform = transforms.Compose( - [ - transforms.Resize([28, 28]), - transforms.Grayscale(num_output_channels=3), - transforms.ToTensor(), - transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), - ] - ) - - trainset = DigitsDataset( - data_path="data/USPS", - channels=1, - percent=0.1, - train=True, - transform=transform, - ) - testset = DigitsDataset( - data_path="data/USPS", - channels=1, - percent=0.1, - train=False, - transform=transform, - ) - - elif dataset == "SynthDigits": - print(f"Load {dataset} dataset") - - transform = transforms.Compose( - [ - transforms.Resize([28, 28]), - transforms.ToTensor(), - transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), - ] - ) - - trainset = DigitsDataset( - data_path="data/SynthDigits/", - channels=3, - percent=0.1, - train=True, - transform=transform, - ) - testset = DigitsDataset( - data_path="data/SynthDigits/", - channels=3, - percent=0.1, - train=False, - transform=transform, - ) - - elif dataset == "MNIST-M": - print(f"Load {dataset} dataset") - - transform = transforms.Compose( - [ - transforms.ToTensor(), - transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), - ] - ) - - trainset = DigitsDataset( - data_path="data/MNIST_M/", - channels=3, - percent=0.1, - train=True, - transform=transform, - ) - testset = DigitsDataset( - data_path="data/MNIST_M/", - channels=3, - percent=0.1, - train=False, - transform=transform, - ) - - else: - print("No valid dataset available") - - num_examples = { - "dataset": dataset, - "trainset": len(trainset), - "testset": len(testset), - } - - print(f"Loaded {dataset} dataset with {num_examples} samples. Good Luck!") - - trainloader = torch.utils.data.DataLoader(trainset, batch_size=32, shuffle=True) - testloader = torch.utils.data.DataLoader(testset, batch_size=32, shuffle=False) - - return trainloader, testloader, num_examples - - -def train(model, traindata, dataset, epochs, device) -> Tuple[float, float]: - """Train the network.""" - # Define loss and optimizer - criterion = nn.CrossEntropyLoss() - optimizer = torch.optim.SGD(model.parameters(), lr=1e-2) - - print( - f"Training {dataset} dataset with {epochs} local epoch(s) w/ {len(traindata)} batches each" - ) - - # Train the network - model.to(device) - model.train() - for epoch in range(epochs): # loop over the dataset multiple times - running_loss = 0.0 - total = 0.0 - correct = 0 - for i, data in enumerate(traindata, 0): - images, labels = data[0].to(device), data[1].to(device) - - # zero the parameter gradients - optimizer.zero_grad() - - # forward + backward + optimize - outputs = model(images) - loss = criterion(outputs, labels) - loss.backward() - optimizer.step() - - # print statistics - running_loss += loss.item() - _, predicted = torch.max(outputs.data, 1) # pylint: disable=no-member - total += labels.size(0) - correct += (predicted == labels).sum().item() - loss = running_loss - accuracy = correct / total - if i == len(traindata) - 1: # print every 100 mini-batches - accuracy = correct / total - loss_batch = running_loss / len(traindata) - print( - f"Train Dataset {dataset} with [{epoch+1}, {i+1}] \ - loss: {loss_batch} accuracy: {accuracy}" - ) - running_loss = 0.0 - loss = loss / len(traindata) - return loss, accuracy - - -def test(model, dataset, testdata, device) -> Tuple[float, float]: - """Validate the network on the entire test set.""" - # Define loss and metrics - criterion = nn.CrossEntropyLoss() - correct = 0 - total = 0 - loss = 0.0 - - # Evaluate the network - model.to(device) - model.eval() - with torch.no_grad(): - for data in testdata: - images, labels = data[0].to(device), data[1].to(device) - outputs = model(images) - loss += criterion(outputs, labels).item() - _, predicted = torch.max(outputs.data, 1) # pylint: disable=no-member - total += labels.size(0) - correct += (predicted == labels).sum().item() - accuracy = correct / total - loss = loss / len(testdata) - print(f"Dataset {dataset} with evaluation loss: {loss}") - return loss, accuracy - - -def main() -> None: - """Load data, start FlowerClient.""" - - # Parse command line argument `partition` (type of dataset) and `mode` (type of strategy) - parser = argparse.ArgumentParser(description="Flower") - parser.add_argument( - "--partition", - type=str, - choices=["MNIST", "SVHN", "USPS", "SynthDigits", "MNIST-M"], - required=True, - ) - parser.add_argument( - "--mode", - type=str, - choices=["fedbn", "fedavg"], - required=True, - default="fedbn", - ) - args = parser.parse_args() - - # Load model - model = CNNModel().to(DEVICE).train() - - # Load data - trainloader, testloader, num_examples = load_partition(args.partition) - - # Perform a single forward pass to properly initialize BatchNorm - _ = model(next(iter(trainloader))[0].to(DEVICE)) - - # Start client - client = FlowerClient(model, trainloader, testloader, num_examples, args.mode) - print("Start client of dataset", num_examples["dataset"]) - fl.client.start_numpy_client(server_address="[::]:8000", client=client) - # Save train and evaluation loss and accuracy in json file - with open( - f"results/{args.partition}_{args.mode}_results.json", mode="r+" - ) as eval_file: - json.dump(eval_list, eval_file) - - -if __name__ == "__main__": - main() diff --git a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/convergence_rate_FedBN_FedAvg_comparison.png b/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/convergence_rate_FedBN_FedAvg_comparison.png deleted file mode 100644 index 2c9219f9078ba82cde3af269d8ecf60c4ce5c7db..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 34158 zcmdqJWmJ@J)IK^xN-GE`ErO_|(v6@}0!o84N=tVmf*?vMog#=dh&0L|Eu{hi(k(TV zl+@V|zxRLE`{A4~=i_0mSt>KnJ$LN5_I2&s@cXLrWW;pDC=`lJQQ@{a3WXDhLZK(m z5WsH)e~iw;AEK^zbX_$ZEL=TIogbrAOkExA99-?J%~;$YJG)pr*z@xU^6+u7Sh>17 zx`^@eKKZ{tz~kU-$t&%vc^w8Ja#YZBL7^y2kr(=dbgnfD^=w4(_AO1%)YUN$PffSq zts86iywzB;QJ0>L`^aB=!TRDw@ndQ{rva@pmf}@6>N3+dBb|f_mZ4IX4+_-TyL2pg zXQJ;auJJD%&9LEKB4W$F_wA^r*xW+ZZ@#NtVyl3riI4O){A4$liixAn!B3u^66y~8 zhVBA;AS?Wky`+?ZU&j#KCCUs73sb;-hj$-&Am{e`Ru)&CHI`%cY8qRak}}h zLOs+=dg0=q8}*ytI3GQJto2WVm35$HU>F@1rul2DS6WR*2DSXF=rSK)=}={MHXEun zlCkr(npDTqK*2>m6Pa)qWGL4^--C=kduv^fXOi?T_1dzK+MC3cx6TSvc=8U8K5xs+ z%A*(@;2OacX!X}6E(!uRJcWzjHXZdpc&(==pVcC;Aic{Uf69&`OR2-^~D???J z5o7keTl#9!M*`vD;q@oS2MhZzZr{#NIGx1tc1b`?T-==~Cdmr|R%fH6{#|W2ITqjF z-*4HW7kB4^N4WGqwU_nSSHadY5Rfw{6lfPIrPlc!>3PQzoMYeqyyCTV*QnKuy;k?(pVq1G1bo!j}*VnJaFpTAc* z7=JlA_SxLr9Qyc}#ATu9aqG*A1|DN>cH{L)s%c_Bs-4X5Mly834)0F}Q>_2}`v^|9 zy0+H2iA?y7_xL5b(6bB-24n;$!Pl~r9!6I^loMO~dZ zEOsu0)@br*cY;G&Is=TRwajm{+9^9H=Z3?$1WaNNTenvI_op&!iDF#aO}8aYN=nMT z^(Kmf!%w(7ON=YHMqIUQqI-_O}_Q z4awdQd1kE_e@uRSB$#!O2SIsH=};g8oHjFnwmllXdn61=4SZU{w}%1s(y~<-6wCV(K{Gcl2|x98dBygu1T_d*w*X!cra5ebAKU|Y5T zR?No6_6*#N?@@d7&8Tu+u4f@3S{|NAP_9t=d=)LBVPkV0^@>hVY}xUqLXc z@A8w7kXT<|zgjjvF=33PF9pWF=oE*62s}qH|E^@g28`>k>I63ts7uz*`67CNvo; zHEsEI0-0d$*Fy*V$?tFFTIjsfCA?zAvDLxPp5Z-x`t;9Oop8X>Hf@@iTYg?CVj`(o zMiuBnSeG>(9$s{Ea{hG_NVQ_J6BFCnNSe*AEFHF+U04toe@W2Y(t@v}Ge|Jjx@K^i zYc*qDz9c1QmU?-8FtvANBxQZwadd1fjZvxSzh{Yu?1YfFtEz-mk_8?tzJ7fn^p=qk zQ%6Tfb8G8Q$ZDbDLtA>M`8y8>A}PH6`ics}<@;eSo?c#4^Yg8Z!Gs;1olSjxeZ!Xq z#FWBe&iT-tbgNKL5cdJ`i%iBm)`oB+AckE5?Hsc~7DURaRV)}}!rXCvwT z7t`1G8X_RXL}`;U-JyCfm+jGL0+}WZELRh^iLbq)j)muH{=tMEM>+W@Fwk4nAzkk0E0bLG!^)-hY&$dB^?#NHwEP)T+ z-M!&5e=ps4QxVMkqNKM#xka~G|Az;WM-7LY$`DzC;qbsZ47}#z4E*=!5R(liq=@%o zye54L&jwh;C+_Cv`n!Cq@Q~uCblBYtXz5K6rZ6@(Zrg(4oi=9`nS54mT)CHsl7WQi zxif6(H5((3%rD?@gRstHZ61KkJL|spsQUGB##eNaJ*u=D{t8YNTvAeEUNaLZ9u^hF0Vi%)=MEk|rErXb#H~9$3R(Bs zW%)=p5s?dl@I9&h{hBnqMk}6Jut`X?co>ffzvm4N(wFbPM6rm8CEJhHQqy-!g`E6UmT^8VF`3wB(a4q2E z@})~?a4xfD>7&0IJYV+Az>{hqv?U7K(jc)c3H}rw_cSj%%2`^(o{kQd{nlpa_j}l^ z+w$@W3ZxOPD?>35v8HEch$tusO}3W?T6U781Edxg7wvw3BBD*uP+Z(u9Z3>#Vufge zM?i4XeMOh2W|q=@WvFqwCBh3^v%qjr=`fMb`(^Lx=%@i$JW999-cYMylt5k!1A$-t ztA?6ni)Dsi>nH2Jp)zyAI`>txot5FkgNYLYljg8d^M*CTaiKEk4JQvgQMUAM9CQ|!$;-V*g)(UF6yw@7 zPG;@&_s&`DNoNN-9DGFX9o}~${g+AKD3@hMOHI2j66QTmV~mA|1Xz?%X_%$yujxU& zWu4ZN54cY1sC2^{LsL` zDCRn!Uf(g~7_>U^^_y@iV>~i03da5O-k)37+;#R&=WUl3<~|muT$f;g5P5hJEng|5 z4<}dnW*K0RAJOhAR&Y>urEj-r4|<#BT4(3 z#{1}$l;t7I`K}~gE*hS?m5*j^@3XTZnCjNLTE{iB=3Yg+veFiW(Eq!OqN#i=D5UeA z(aq)_z^!!y_u3z)pR?PWW1Lw~p;lJS+o1iC)QuD+}Y*cUh|Clf^7@4nMzrrS&{QdL8z;AoNZnU~= zND6*N-AA5C>qlxu;^RzR@hN z{~F?itx?RqkPBM?nJe78N9r-|#eveuRc!$eTdf$Rq@(+mENH8sqVjliwqrqJEnU(_ zx8Czc)ws89_!Fe8c=G$x6kv|I+U4Q`%|mB*C6sMY%D)Peopct|inY8J7bmMa^Q>hf ziAtKw>WMfpR%&kCZ@k`X@83VM^e8&P7m#>1A^qJj{fdXuudrzW`|rFQCGoXCPh%hA z_Rwb^;fA9fdX<#qk}a5|31lGlo`60T;i>6q*;}{JH8nLanIx0jm;>sY{Z&$g=7I>B zCc`<>O}Y|qQ2YK+Kle@0CTMCkC3=mKK<2pPvTEdAAP|zLnx2=q+Dg2|Yc%mERQ+VPR+FHwpP1Co$^1(&nwCkSw~OW=<4#VDi~nnSzva z&22)S+3(Hl>?5h8zkN1dMP}+mCft)tW;u5{f0a^Mt=xTH1!h|kW=b%BL$Ky46LZEs@25+!{s z0vW;_&Yrq9@=XRMMxV9b8qdqe(6MTWfr($b-67vj2X* zA7hdz8APJqPYxI})%O22mZP)_;s77TrlqMg_#agt?TmOG?=L7z4njc&DJ~bvg;Cc& z6<6$N5$L@BM4kJR7i z-%XzK!YkJ6ea{RwO{ni&gMf00gX8(okS;j>_0hYk`QyXmQTd0*VDdP+$`D87)h0h> z%bl9m?InKpz=|;C%;7jGHMvG4xC*M{=tk_}RiQBKa^*qDZ?+%6Zv==PhyvB@V_k3| zT*i2-fZ6=44;dc0Hsqh$mFmkOe;GpAb{rfs+tvOR{Wa?j%M;TI-1F!1A_S`Mu4e8| zt-v{5u}{x_*p2-oPmXnQ=7Z33?oRnbs8K~lDbY~1;$XT}Gz#MH@(13aB+#iVBIfN#KTNm-#u_qAn?f#eF|o13qZwsq_QYK zD6OlzTZKJfArr4OeuKms{bMpAp?Tw_0o{SO$f?Dm3I_7?CbNHO;D+e8N|s(kna01q z&GBAbln$w`qF|ju$GPl*M5nksd+8dw@X28l`b;^4&S@z;AWLNOWXAGJ;47|+D{3Ox zSreoHl7f97@x(XvxFPPJeTC_({h-1;l6@+W;ehWtd5_v#V4hJ>s(g(Lxh3Y`T{wbA zPOf;OZ`~1oQGE_RcsGRcmaO3UmvC&FxNx!$Oy#2;PeGi5;C&ojdg@9vX+^3@)PFO3 zfz)sR{x1Gq9ifHVO;%R+0>1_6j~_o4B(NzCUl9Jl)7$%Jg{{tfZ-z1MkN9BYAG(1Y z;fg&h6BZuMGRU`PK;7mz{PFR?dch(^IN32#gB_(r{BeN*1rK)rS@PaSB~r7;;7sH~nndsb>Ud^42v zqWY2)AjZp3>H|>fEwjf2q9LXPQ%LP)8VoA_Zva!k=8jUQXnNUmeo|>9HA%w7O~?M? z(8r}7UdV`nS|ht2jOx5fsNOfp$;2}6?!NjqY6%|p^1_X*q1u?s@*iym@@-xf8TiVf zNs7;Z4|5e_kXT#^Lr#au$;}<5)eW4O*XASO>Fy=)7=9b&1#t8I`**WtSvfgH1%<0y ziGVeoU0fJ@`+oks&g^JyFp3ezyGcn@o@z4l_RUfaQs@aNY!{!F&2|%C(x5`q*{{h} z%TR?%+#DdxhfQ0N^JMtul8tYv^w@G2(q~YJ!)@m25KVsG9q+97+S2{_h^RY7SbF)3 zsi`U3l`Aw46A}{@bai!;+9ZzmX0qSE7mJ6|?s|BgCYVF*ev(Gx6N&8qHkb}r7Xi{> zpS8~k5RX23ZkR!o=<({p1mWs^vU*VFMncQ;Xc80jm}zEeDyX3$pe*P2`742jhu!I2 z{~1kwe}qsVRJUeOiyiLGMD6YGi~iog$HSYPo5RQKZOmBzD!ka2t425pxy|Iy_qX*s zzikbiTBy6yB~!MS2e*K*5?ysWuc3?2FlXv+HT)rx1u>*MOLvGS57rvgpuPi2?6QQ! zXjxYhe_L5E*dbIm9umwzS3+4g{Vqx>3n*gWgVm}-s4Xp5MbzKDu)v+dLp`L(jn&CS zzlG{LdlSYp@d~|Z07xm^4LC4Kc=-w5*2E`PMtydUJ@eQ7GCOBR+J^Ptursdq2FM&1 z1|ldi-g%{X-(zXiN@#8c<_94|*<3{z77O{=&1>J|l3)g!TvOFG{%AZ97=eZaq2ni|sH0*)le_xQ|ol+GFa}Z6sedW3uE`~?G zHK}A)WapTeZ_Y$)v>4)FN}4v#P*#j^;}`7g2&p@RirarH%RPOp33&`MfHzG%j{SE> z2*9lB5T$I(v#wA6y&b5*c&ncZq0g@0f(WIpiJ$65x*{Ymj!|*CehT=va`8%qwulHK zs7b{iF};(kW=4fu*JRJlii?UMD=a`(s7O}GGwnhmSbqOw z_%89b`+SP1PsBTD7VT3ZB$WdK)tI$p5IT#P3UkzPYG$*CuqCUi^dx6l^27f#JZBi( z-I5faDhJ_q&gHSeV`;`4*C+c%P;{YFg&&YTAcs8|N&7*ReaGB^i>AGun?>~UV0aKx z8yM!`l~%fZLf=DSng&AVZyBMQ^Do1!!cqy5kuIa}ITyYl-oZc3<+ppHC^5f_ucEs; z14l_^%+XM(D$F5slJITpzPG3J692=geA5$5iK(6J8kP}&gac9cDf|;t@^@H1;~=;KC*T%?jPP=CvQV4k!K;YhXbTa} z>&OD%!GiHE*#a}KO~KM;fxRgtc|nQ!+eazmh6f*r*@8NV=l~Sb* zc!N~LQ=+qG1KF>%EEwI<2|vSR%gIz+CZa`Flgb!n_CMJr5GMV#s+4Z;Q#z(dfCC7M z(|vSnxmX^Afx{d)2XnrCU53cS@a@~v1or}VkMr<9D;@wh(Miq0TlchJ%G2BL2~&a( zFCp{2b1nGRfoC=4(i&Q3#VHO4+4p_qNu_J(XMF`h|9$-%IGx(8H>M`2tfwSxIGSpR zMO*_za=Y_@+R;dCHce^gMJvwB2dBPz`bNle=4$VxC^cKcA)$UQ^r#ctz{Lm1IvHUX zi%WHt7>VC|JUXjQ!XT1<$4o!+^;6)lV`^{10@Rh{&q}KaiwTpVOtz0_{%5uz>$xdj zlWRo%jk3YREFE#!h_NEa77WM69+{3IYSE^!8n_TBSoWXuxPWOUytzSRGV$TAE?23v z=<96P|NOlC^qmLz43l$r+xP-dzdq0*Uf2U(XgIB-i=P^E-%R5~+d^DM3l~6sdkyS| z@(ajgiDY=HwuB!}nFLX=Vz=-B{w;^QtmkKGM*2%~^l++$uLYZ5tg&PQSjajJfQDzo z8pmEL5Y0hBGje`*M)f(LH0oDt{&_@+{PrH+FVoF@@MeGSkzIKAof&lTkHGKnq%#c| zaB^n+`~(#{6(-9)knXHLGC_bKhJ7F6_N(4dV*Eqc1Al}srnYw#bo}Rv02>F-YoQa= zafXHlgDWq63H*KPM#Hc{g0~)5p`U5MH)xB?AZI7{SrL`MYup2HToG_KkYuN^eJo({ zXO1pcJf*U%Y{#v5TWWcZ=wOqogdtxWBa{C^?&6XPPGVA$;x2Z~Yj@0p)78M$RbXRd zBa)qsEjyXDkoCB`af!InKtWpH%;kYNW{k)>`%xYv;;nCHmsU{u-#$EecR|nw*mDvh zqD!FhE#c<#@D!gN!ckoj9zhcb@K%a8_rBIv@g2%Y?f5-JW$;_HOECzs%qLn7a11)2 zr2))bh*jn>o%hDK2ZO}faoe3w3+xK zxjItyUrM}j;|5fZ!3a6jZw_R84-i`Ug2m26*L|*D2$bRuMh!W!#TC}$)Q{N?jBvq} zROkCCG4%G;K~O*w2Cl5w0R{w(Y*ePoehiVdY$Vb_PAzko_;RyK`hOt`x|di1%ZsFR znR=PDrpM{$hscE5a{3KUQzwUrAld-L4@4J4fjHN~M+!2D=WKlMkv2~0K@IAl=q@MXI=@1y?BuY^O&8T zT?fAWIqcsSpbMQbl0u$4Tj+)NvO9|F_L^o1U%2USIM_-o|4-PT?8k$U{S-*8A26{0 z(YB67Z#o&K<2A>|-@k?5Ykrj&_pVPhG4|%(OTxiGjg|;v6<8aCm}`5PBrk8&b0+UF zU|c6xS5=VhglWA!x{IWXs@SLVLRc2)+b~T1`Od7JaXQe}q0za*0@fQ-E?Qb9H;xjWjFuT`-3w@*#4%nV<<;G!z< z*r`7WN>#00chWJ%JlfusXYfppZoMKz6uVpzKL!lY_m16ygOkh45#1K%Tl3wdbOK=@ zW{8V_`n}2!d@FdE{H#fu*i$gocI(t+lxn&}lA6>$9xA1~6_lIBfdX1Eb}qx({-n19 z{tH*pcz>w$0A!qU?#?eXs4xx8*{SI1JK^tMApLqxhjXM})L+$!q{J|B z<2aS+izFvE#k^UeBfbZr$3{H&N+5IvT;}2;4cy;aa5>r@>?<}Dx360j0zGEF>$^=u z7UW9L_Ri~Wv<@CRwddWBbxYJwW6dm_v4FsvG;t5J^5H!&u^N!AQBZ$=ZE0yi=4o_m z@}h~9lvifg*uV3YxCSNvRaotLM3Ydk*Gz@d|M{7Nv>=?jd>2(}(M>1j`ezcpQaU2X zU;uMx6*ZW53J!=x-54++iYD_I)}E!MZG*3hdj6ac1>z0De)W(R*;}2*-$?2GeJWw4B&uohYs>wNRxke;4J(=srJB!d4oMrZ3pAo5sWs_f*Co&1k;NYhBahdd_p{GcD zM2yM#TUGvnV|~8*{3h{Ab)DGfNgWs@ zwLLEhb9{7A11g|$VFS!C?3JWXUOdWQrLhHf1wY!ox@$~dc;iz0$g5Etaq-Cazf1xV zGo}X@fzsCGN7fzoUnf9X!eAnovcy_#LNXP$HuQd@^|@%;>M@CxEr;CyrdbW=AMe-!;awc>dd(IF(+hWIEy9 zPenC*$RNL*Y;Zgwvry5{w1N)R3UvgT4A>rcBDs#SUj@Hhf%wyJgH(I5s&A|HxmQ#b zICgZeJZO{>Je!Jy@VlzagzE@N4=*S83vLPKR}+sc$GE?bqy-hxZ!L_Qa}6=|N;duO zmvKB@W}di}A?;|-VM_H;YF*?KB{`;Kgesr^6kW{O)Z_78y?O>jlVU%$)bG`93;l(( zCPA+WSuS7nO?gwZoRcQEze_PPFDp- zxNK`rn{3}UQ9s7gtt{Va_;9yyJmBesJDcM{NRbWSeFV&K+eo1MB^m};vPzOrt!a{I zymC6v?PIF037V2qT{TFZ9`i5LOL}W<%57oP?a=}sN{C76(j~!Mv3Cefp|3<_;&=8z zB8qAFOyyxYmX2mpaZSO+esaS713&DJf*imvDXY<6b|kvdhCSmLi))jIPc{Y^laQ^G z1>AvKK%KlR$lMm!K<7CYL@D>3xL@96Yfi^S(U1J^bpyeU+!tmMn{+1 zgXav9>y6lz%oejoP$Gq%{B{HbMY1TzZJjm|N=(Jv@1u`ArgoC4-(Z6;j7Wc^!$m;f z@76r_Kr|Z7WmKQ|Ut?0KE*C;xKoZfoe;<=7>VgPBU`MVNiEeURQs?Q<{$qGyfxpi= zFCwq;u|QY|**h(WhoK=MPVlAW0Vn=ITZ$tuP&O|J+cOkBtg;v^HoTX}7X*}Sf0F4h zvOBo4ay3fmcTfHP46vR}8JooN3PrzmD40k3$g(RPZE*HFI5PNvfhhc?VX9lCjR^Kq z|NdNtekdUIc}J=;>MRRP$j^|?2Y#PhZ+#_6^w-_Ib4ruxz@9VIRMY&_ydFZP=DK>- z1n?63FQ7T;Q@XigHvj(R+8u2qGr}i$Mqu@0C}r0k8xk?_l_Di-^T+1Vz=cywrcqTv zIn>P29B)Jg@BzWBU8vjmf|@h&hB?8>UYqmgwkdN3Tzp)aiW`&#TQsc3JO4)v-(uehQe*WRYMn%4w< zSDLHz<*3h{bBfpBJzB+s*{dbQ#}nMo9S_srNB=w257Wa-XdphqUfnhI!gp^yvLv&m zNX>i_#@nF6^mHIU-vf>@z-DgOgRDRh_twF|f$)&%LaKVJxEtTx{8MXFX&!njz!#%flW`c+QX^Qa3q+$YdAaQbs&& zV!$qM!TMVxERU-#(-}eC4|hX!-#(fd4IKDe#v=L<`p*8HFEvo!j0yh#!q-4p!@*DOIc zp+W1axi*s|x( z(oE{9+yY!4#^3KrkIA|y{emguNsN+};OJppz%|X%cdr_9>C=*vgPjYIiYNOxZvzRY zVMt3KLJboWUoSgfl%N#GfX9Q4rXy@KzwYbV5N}b}Sg~{l3C(N$p496gFkAPxGuYa9 z#Ig(e?T0pv(d&~24$Yk{zwmTK^K{;Ou8+BF1rc^Z;j_jJ5z@q>?8h~bIpD~T9&Nz; zH=*}d?x0dY*<_c^!jY9f>aG-a;M`mIxJZzOcn~T(JG<0*Q48(UUK+s@lS)E+r&48~WA#Sq&|6|6#_en zH|pDE$cHr$yh^QO&B@U6~mnN8gO$i`g@Oh#ALu z^Upn3eN-m2BO*5 zLugk1g2d5VFXOO>WP~SAGuw$7rTk^s8th&zJXewTw9_QSssARO`3bh6UrI0?)dRI% znR(mv;$p|ki~Qe4dr!|%=@Mf%-g@&fG4pR&J>%EIp|t3pDthv*D3YxFO9;x%y9N)p zwzkIq)6Lhrue`O^flDpb4{f#nCd$VSSRpB6x4`gVeX|Pv9zUcO?pLzG9lEc{ipIOp zSANYuy%>0qmd(!{S~Q5v-7k^L5%`>fc^)|EnX)z}^KWmiwZy;Bg^Ewd$cr7MOi-|q zMxWHJ#2n>>)0QF$38dW%+V3P9XdlpO-bDX(_n7-PelAcf7#Yz`m66a9pqzo*fYv9Q zffr+S9(_5=v=DUyO)vZf(NA{aTUS>$FujOcY~Zt6u`ntPH6Lej_0u|RZ3J(_VJtMn zIROm}g=J5B%(f$@E-R8bNijMBA4{F#zZM@9<6C<(p8ba+gIM~Vh}PtXBZh^8l;kU~OiX$nb*qh~&hA*-Ubb>bLdnI=npzTtnX6FR@Nui#ct{60W z2ld$~(C0|wspV>-daf6KkY(V$$lRgQde&3F3%`qrLea*BNKY(Kj)31CA<4gFmS#ff z=;%N%$Hw;dT#{w_6m)#%|FSKC212MI&Oa1(o>kB;($^{)bjw%L#>IWfY!%Peqem<6 zo=&h#1E(1QzV>f#FFGNCP{z|!1d9I1CxAfB`^DKVz2a$#1pcWm6HT!b0(VoMQFYk~M9p9zXnyAUhamd8vN0XVeg;Lo1Bcvh{-VIZsSxA+c8M_CPJeU z+QMy}klFXX`5?21BMUUg@>beUeGMUH5Gn04hn59sSp0zfp07sD9*8a09;Y8kDVWe} zJf6LGrFXomZRTJys}WHOmhf`pP_@<7j*B@7&d@k;UDOytUy87U1;CC#(XLLY8<5Ow z5z&sm0u7E6QR9214!EFQwY9A+cgQ^;AOK>i)c@TvslROQmiPl- zeP)uGBDx|z(dwhTqc!Y> zL}_o49IAHUtB_3vSu32MXQCAU&etI60s*vZnZD=WkK~1(miSv$?CZ0CK!E)<)pHYe zaXVFm5|gFB0LqZq%qH1G6Xh)j>MMaJm4qwf5`o;$HTJ!aOtDPAYL$9H`D4{;vrd!Y zq9MCDeLnK~WPV1c$0*7rbCGwDq1nPizfT>ar6^(UE8L-C!#bBMK^&^7AKqHgMcY{6 z>dt<{s(4`&3>l=7x<5)P8uVZKTX-RL9*cS89q{%|qbEWQy@2An!?YXI#RlVbO zN9={4#6X|d3X$NMESsJ(KZ1?dv@q^o$E1&Xjg8#UD~v0Rn;#=`esNWs__OTbUSEZkf5~v)e);>g?=ghI?16M~ob}+!~p@ zxm%&j<@yBhu=c4OlLzfY^jR|H-tnUX?NWMA61>2Ax%@mu0PUx_OxTAEU`EJuH6KcV zMVbL8ezwLZ*dcMhnyw&-Gs_l4zim7u(EGzE`8EtB>%Ql=@y1i$+;UGprPnk;-5}Ev z+{G*Yz!#tf;l-q!e&wZ_i@ONo5XqF7&~G;`{JoVM`T(xOobo#gWDjtL?hkyVlo2Un z%=E?q=|Gb+w+oN80+VEprkQOVpD-B_K0C_9r{yyfQnsi?gLI_@I-nb zclo8|p}f`4t#T$EdgA#S316>I`gL47FewWHh|EjbU?WXNPd++uTq z3g^q)`5oMVU!Dq={L|rB%`TK0o^s&>AQ;F!u8>Vp#!Ta^mIO8g;A^$g%EOD0@D=($aHh8qIv zD@~lCXg#i$*V2JuXcM>KPfdG^AOPC;K4 z?crAz1eAYn^Cj4#iL2;0KWnOGI~(WCIaEj%ur_l?PM?W(woN=_#iIPG$eM^g$08G_ zhlyAAJr8z@y&s?k36n&pAS9!bVQxm+*5Cbqnocs`6hSy|Y{1Od>xurmBRJjhBMp`K z#YDJaO~Fwgkr%duV22)B(XJCettSAAs_x%C@O6Sxg3^&Rkd7zkT63xX{e)9}BCqacvU6ox z_0fRt+GO@$1|dcH1tes-k#I9Q88BE3azm)yNe50#K&|xh*KuPmL+^HOTAN<@BgnlDq(G5a)S%y zKUSRH4ug1HL%(sc{ndH6Je!WOtoScK)k9#?Sak{?GKU z8+pU^RRXOWzKX8SacK|1g5IA#uMbLFIwhvhtG4EQ)#J;@qMrnf$Hy~BMhITVS2N(BhRuGb1|wV4GOZdkKr=hJ=!xTT_mnj1~t5jeSG=duKh($u`!+*sLX zE7~Pxb#I=kZUcF5)s959NAocboW{BF6=fWI)}0TyXgoK5+e1?Yo{6=qx>hI;EXVu} z>RXMV4L%e4el)IPT`FU@=)O_j}DCista=kUk>SRM`; z-QEv%*B(#0(dDU)~D373l2&DFj zig}pnbOy%6!fnhs4NWqGPO+z{z$pgR@9#3{az&>(H1+nyan=dXbvNvqOi#w6>iGg_ zz3_8%@N+HZw!S&kp*U&L|w927GdslhcabPRnc~Ft%XHZJ}5YLP6$Thxc z%Smh6n@Vqdv@r6A@#3?Ti2am>Eb>c-_3Y_G=1IPlOQCbb=W<{-pi(ScNW0*1Kv78WanvRW3?`KGn=LJ2S-OM z$g#DuM44}*TGTBIf;y+oDgd)ZqI_%S&-zgH>3!mi2{tW~`z?tB8H`Z`Y>U6b;y?OG9#BQ?do9CcijkPe{DC=zSW?;;~BZUtXE6rmP#B~uCYHHilC2x3N<@BQghHr_Gp$SRmT9@AbiV@ z`B2?BpW$B2jUvbUHn5}LVB&KNw8Y2VDBS5P5%J{f+w1K+^}?03<=8(_QS$qDCKDOt|-4lvm>S8N2jx>xt``t(m@u z2h%ncND`%4RVrMsfh|{s@Oi!U2YIQ#7H(aQZcV+dh=000wOFy0b~Jd)?C{FV6^3sV zz^Ve_uok%*UNG>r_hIB_OR^lbm_tx^&k1osfI}V2iyc${!x`D~H_l6H;tX0nkPsH+ zh%zmptZM;x&R+32bfD4#72$mtxyl@#n-VdjuTipshmr5xEf+3xX1sUXhCXG? z9FuKevJ5NBN3vUU71@#exGnwb0t zC6vgmn*^wsfYecha#^p={Ylb6PpT*ts5*9t&fI9Ve6@t-%*~OkJi2OMocdAE-FYMM zwZo6x>Bd!U8Ez+PI!vg^d?pp^>4gVo2YotjUz-NLK@*DZmt9@AXI#_g%i3!5_!IdvQ{^0PLaqGPc};-IsKa)hjDGOgvHbeIf6+wFz- z=*Q{H#im6Y?jEMgQ<}#^TAww3C@#M7)cKr6DFa@kKDKlRCov(R4V3c>H_V=yrw!OV z2Bou5zaqbD=H|_tVJ}}o?0gD0C6gJ&1={Uv{?=0Sc%B`t@3xuL6UGGagnkYX5&x*) z4W(N`$#Q55&V;>4*QR7gNfV;QpQ9-zb$n8A1ecuRN z_Wtp~8r*v(V)oxNO=`QBA(a*wh-w2g^&t4%psZA*<6`+bM*EL%QH_1mz1&buS_z4d zlHI}eBC)aEa>fp(UE-X>u2=B`)O3!8%b2N%m3MmBK@A<@6;LL&`R-Ydl3Sz(A`Q( zPTtzm64Kl}t?`Kc$`wi)8tr12>YAD#<(6v51)F&(gbINMlSp=G@=9BfV!R=W@ zkz;#OJ&ri%F5?2GEOc0WtHiI(bE$0=b+0CyG41P7M@N--{%Vf%4l}>->(6NHkv_qw%Wgea!42Mx&e2e^~oypu3{`c$O6V&kPseJ-EO}VD3xAZ%3c0 zeqa6*F&X=y{8moe08@%dAr)V1t-X`4UuNN|tCzMBnxbpGOUHJ`b8n<*$(2p37$%Q` z+u_LV#d)~r#|+o$ra_*(!q5LYoQea74~&N?;Lxe^qPIK&dVMSYU7VPuq03GFs1b|e%iOtvM%*}F ziwLgM$X>?y{zuCJ|VPWH~YP3N#O44~X#K*ci|s z5K$a(Mw-9pTIn6wL`9SS=QbPS3MCE4OMfmn|M`9i1lfHoGzlSBF%=XPSU^;QE3{kC zD)1vugTwOkv)Yor3%Z9~%{;+{RFh-v!{-KUv)3u$Oq!XP(I6 zWmUfi)pfeiy^~?Y{S*-5?UHtxuekN?9aT;O>?!ZAi882kKoX5p7Joiv#w`^$*4c39 zb!?m(^zup7<_I4d>o#Mbx$dy%fc{zXvuqH*k9K7mv%v*9bmPaKRAk;h2$%OR^v5Oy zyqcE|z%Dp^PL4|?zW=S{10g~P`KQn8^o=)%CZXN~vYq;P0Yy+{-Sy3{&C7jUm7v1@ zUgy?_YiF;iKzv7I;Q~6m#?`C14QoFQZC&KI0ActgU06Pv$R78tA&Zmv?cV3wO><%~snvkM5`XLh)y_&bOE zW9mO!G1pQ3)T>$Z3>k6!b`9EJ{m%RqQf}U57w*Da51zY`J$L@QRDaZo%1Rq|M&aXu z;~9y)ZHq7a3o-_VODc0|A~8~T)GNeZ-8jMSH6}d}5P=3Mpt4>J+W+ZE4IqEi@{ac0 z(aK}4qd~s$>N3I^!n;2K`15E-T;AVu_jn7a0b$lg#zLSjy0AT;;rm&Nyl!kVkJ-cI z0^fUijemENa%$eyB|Ukx@I+F7VTA4R9QArz&+W+fZ^nL@VBa^*(hM!0otK}8zr|ij z=m|tv#yslKG9>hws4nJTWE`gNd1sndiRL9OpC`B67rNFYS3 ze8}K9HY$$1-YM5sp18Sf@rzlab72e4W}+yyq$= zSo9_GKAi8xG)W(PytaQn+qhB-yN%5*3^NP^PnWoGbO_ z`6_-TEz%~(Pj$650TDL~JP-ictIhn#+Vv(B4c7GfC(3tw+ZGBaHn-;Af!MAS9@{U+ z@exo!{g~mI?Om4kHnIZzpDx4-s=e&Z4HgU56aH**0sdctIxhcq_RC0S#4ZDB0N?2= zzr&K?dNYMjkmT=l(v37P#~zw-gHH)>SChX>vXoocP)gD(vA`lqRZ14bVM4QV_rR1rA$kV%uR`d$jr&#)7~m&@cY^uXwlj1c%f@_ z_^@irr*T^sgq^;HHCE2vR*IBJk=7C}XYT{)h?b|%3rM)>ypTSzgV>wU$a82nfR4<5 zw1tiJGZ?J%dR4LdQaq+b$F|#omLkf|c%|0dTVN(CO%Xwvgbm|ePYHRNn+9_3PRIUJ z)WWE5_tRIaI+y8J`{>&G*Y4sV2vcEKYp?k`WTDst11^uAh@ zHD~tN;jr8slzLTlJr4epo;>fgR?N>uQ1YGC8~HKA=M?c>Np}~P1L7t9=UW)%)}39Z zzK3%G6V6@EhV{J|+Sm{<1RQf3x`+;XmT)1vz-@?f-fO`RkIO?><$vzE`QW^(4TWNP zJ7m68P@-QO!~hIeAEG z9p4uqm|QeBtt8r*i!uxslsexP-*hc-3^0%j^PRw&`A_z+r4r-5f56%%_`>7(k6vV@Hgr}hHjSXxTb6>8nXm!@D@wDl zD>$FWtX5`JwbNgzJXfU|$^C#=>h^k?h^7BfH!#AT9X)9*a=x2)B)=s4z%`>dtc7p# zV~?rj(E~T9DT%57tF!X}ifZ{DG>Cvm5>Y^+36KmTIjNv1ASeooWY9s$h~%gU2q+*( zP6{FtBn^@?5|pImG-QSxhdeN|=f2;*-L0+tSM9D<)+>E7bLURq?$f8w`MwYrZi!O5 z%Sbz&+SZ(W(?JD}(&Fa1-Y?%+cVoV<4`ka{_kGNfPfjXf^o95SvtPttcfQ1!WhRzj zyYlsZ_ZWQF@BC3e)`iH_aaR}_dTB>WZbiRoGLR5Yrt8A(#=czG}IglqslGR|VE zMu7g~Vr+m3YNDz%)0mo~L@ZmPGlxPro?@l#(#_XLgHY&^B6$zqdhxt>gX{M55-fftI%MPIU{EVW}0^gK@Rr0qMTEcT-Q<&nsi z@L(u^`_9^yy~OZ9W0L$1>CEQjYYcm9OPUiS*9vQ2R*x5aHS9V+Ac+%LdBkzQ;zmUT z)CQimb73avl~&0cAqMe3xd<9o&qMec*qFm4EeA_ zo4G!P%8P5JM@lD)j*3dirI?>V1ncRDB^1?kg?MZPt$L(LJhn}puA$u2Q>+2mj zW*_1<1J%VS5!gMAff9}lg%(>ka zuC>2+TON>M%iCb@rA{6A&2_6p!Q%Ty(I9Qti;hF`YRT<4=1h_E{myb$^vzUoTKu!q zuYngGhhDjBI;?~XY=rTpP9y$8IL7&Sx~j@qJvT|t*ea?^#_-|ZFPzlO=23(~TO5q9 z)3U|6g@d%wtGBT#s9&A6#N)#R!# zaGFF<_C1hTn=}j}fqbwE^ve1- z^|yRh*L$Xw|M}4LA>_8Z>;XhJiczbM<6Y}&Yqm6ddigt27OIz`u6W;7A2G~F-IG{aFv8-O!YY1=qyw8rRWS?a~zC= z6=6ZIgU*x!$V+uNs9>ZZ`}mIdQYfb5QHQZuhT68~6Rb?u952&0j~&z8jcgMxNXqq! zVZ&+%>KDdz(={BNvp7uJA2<>24G>l*Ls74tZa3eB>X}r=?$)R`!a z?V7Eb{fWrr$WAW*pb;HRLbO1~1V=ncKA*7vREEns^7~BR&@wx*;4;x9 z@~Km)&E_vPGUgwLy}eppV{rm0gTBa}z@}u-sM>0tKTO-lAzdf2de~TnmUNg?VIqML z(9LRHG1uLKo9wHI+2*iM*Vg=4+1`{ys65K$_g?yc;0$nj(|~i& zS-^zux2HRASTIWu&%nJOx}zOj>1VppXCOO=HU?ed;Off^8%r@7 z^!}r1PdL|!@Adw;=Q4J_rQh|rlVJ>s^uTkB{M`zZ<}1uN9N?GjZap<1*6~AT6glmj zX0XxQ8@N&Ft?Y8siK+pD)1O5K&_oEwwg{MF&UQkhs*n-4QsGOox}j51Uh$Yi!4+pi z0p&m=-q#w{iJCiYy-8>Ks;RWzue==fBbGy^(CXWn=D=377J>ZxCTR2(*&sOONnPx& z5%VJC%8vOtwiBi_4jm{-c8o`&;S9Xlw$_SzM|7gbYOhjMEK&MW)qA;_5BDEQgKUPZrIkgk8*vg1cpJoL?(-&vx9 z0C|pq;a2z6JY4m5Y;wD+j~)%gR=dsQ_hvLXTf4G?M8tWwvwLJna#tDa^h0(WAjqkx z9;hw8ka;m9B`XVWzvm5Sw;y8}()gdu4z9h(JD_2RgE|*UM~=+Fy%C8biHFX1!Rn^? zA~@F{Ih?t9e2D`7=C|L4A!ufwgHljlpOKCfkM?Qdf&Qewn2@ku=}$W5IZRMRa8XgB zqDPD*r00p&c7+}AVB59EMPY>5UF|R zaL=LZpF$f5Yo5Y^7bH1txn)mtXUxi~U2oMt)SreZud>ROt2Hh1SIMBd#?wa+VTvf- z$3#J0RU;X5$;^uBcL$-FACSJS4{Y6WgvY+QCt6zuj`&b|-~X&!9#}ITKtnH}f@#4c zsI!iMjcshn)}@C#&vX|)I+{42Ju{ShJIbKH(Q;yH-Rluk($Dt83!Pq9Y1dymPJ~jY zxtbDeRkhl$+DYXujGhv4VOBIKPMMuuP1XDKFsLr4)r^H=(4vU1i0wo|;;$xUUJmt< zAnS6~qZ|9Q7 z$?>8_|4z^+ecaWRy%%JV=+R(j*GU!E@{M7GL+3&nVSl9^&Z*jh!ffARCLZ5FE!0Cj zvjix6=(TjZh`#H&9hT~_9SqT&_%|nW!idK^C}V(dd7Tq0_UlaO=Mc`aoGSA_bNM7% z)r8rrUp=eh;2*fcqzx%vmOetyczcbtLNLY`P72d`qAj|s+|I%Vg4&{4=AIcgynM1d zsvwk$G*4)xA!6<#YGlB#r3rf|9T(RuP@{_L9a3hY5FB9A@-1O&7`8gG^c&8{B=Qa~ ziIcuFB8jQ7bY}VB{@J`#iK8Lgr|(Vo4RZX6)9gBF{eoeTy+#spqU!vCCG>Te-e6Of zCo@C4o?F*=ItMkN=U+kpXwsHJ#bh@me1xo(BAojUgRO(qO*i!gTo-<89#q+>_-CVQ z9yz|{lH_ut8QVB)YFN7Sdpuc0Y7#0@+%-wzL~Qq&=jzGZdU#PKEXL)W+|>Pfou*v+GtGJw*EaL5w|ezxReGL%4lmdki&R8W5LL4;^(J9-mB6hA}f*RmL3V zSrTO@#g47kOW0jWz81=*s>DS1Me(a@qZ*L}8gLur-mEPLGzgAHJ>X@mh1SW+l8Y3+ zZ2MO%>WYs-V}of=g3reEoaQ567GAX@wm9JU}wV8om|ZAn*3AL&#*Fu(|kN0$p( z9xHWI9%8sP;J$KrW8o|`A&qkNC+swDp`c5mxyJ%l;LL~4X0UODoqpM&&!WuB%FWOt zk`;9@o0je;MLp&5AtdM0*z~`lR4tmq`D{*w@hP7f`6>p7wVXt6HW%;e&)1i;p*VGI zsS_2{XCC7;KA$a%n@wYOAl|T9+Y*gbf~TBov@2oFw<~+KYQ%7kWAIzsPc+Q6L-i+{ znBG^m=F|)BM}%cQk+4_OM)pleujPclRe2X26wUkhF2K4b z<0>Bsdp)S=3AUXPAXn^e^uJ%%+q+IOc3>`Yri}ch4O4AMzw=Z-WuWLMoePBUoYsJp&1hK}vy^9PFo$Y)thA#La{TvLVx9YZ3-v>00SDef*SSTZ z9HFJFqN(P`It?)XwcLky7UV@SXOlZQi<`i=x5}vWnwd3+c0M|LMd){y57~ebdQ<8ka&O-=}`XVt|8)S zNHVsTlkp1lGF&XFZAwZ>=t@GI8EWA$Yq7||?d&&vG{&PZUfl4HB#RU? zH9qVq9RE$_tlHSXjxT;msAO6$7v|*hLPn|0Ojn(i(6me#QH}-EnGo;5oo>3pasAogKtfYBC!Br_EK`*D z6el-UM836NYS-E^x;ybIU%;$B!%DBX+1kaW+5eur!;K8>9eigB9;w>9LccKyXs%M2 zVhrQSPgym%NJ4x@<|v7S{4h)$8|@e}NU?PmH)P5mEHgE~wA7`}A?|Qr;WuabOiQV? z(J@)I>t4>VFnbO7U*`y%FKt=xu4J{uPSfPNxo|(1$v_3U&6L|3jPa%#G{>MueERau z#>5C-H<~TyA3QLGIu=>NK-AB-p`Q;duBV(Fj_h(=vv=a#5-**4#@}C66TIZfRze#( znJJ)7TuOXjT@yv38?1OWD*VIIykat%*h1<9Eh{n{)WR(!b%j}tGTRcj)||uv9`_ykB_b6 zY9Vi|5OfBHPn<{RSy`#?1|AiTd#+`J3>4$=$i8qqLuO|um-st&N7qbFXoW;X3d#MJ z5wSg+_X@8qU#RpEDD0^F{XXwBx%3B_luR zIa;NslgaB)qb_cA+=*c$b=_X*7`dGP)IYR;r{rX<+F#0V5d=pnb|dOEi>oQaZBF{A zh7q6mFpsh0riPy#^w*BRp%08*%vhoz;;h6kXeSIVQ>;IiS<>pLwT(^U{;l7o!}(lh zPw>4aYNA7>uQB;7pF*Jc#xKpD=sV~_XL2>W>FJ8coGX+1 z&c$_T*2@W%(cFKI6NKjew}HYoT4T|Ta(1u+AnQhDP0!MkwS_HAS*{CCyEEmXNweAK zrU30dcSH6t@0uv21axNc_ur#fK|1!RXc9qs8ss9+=qLzg7FkKBK`$qwIANE%|FE+)G-sNg}n=R z2{)*wX7aAr$R#qOZz*V-pdAG-guN+ZKdFs|)aOinv5vS(XBj+uF;!)WUp5V*733>N zdfUk*<@D(?HsB1V&4^9aXdP+)CA4k5#)sK2meK4ma>ArxK*bjcPutMn%(jqZLB~YlH@W zi9qfEt_U*Pdi!gltPkx+SGpk|LydD;Ur#CUl?li1zKA}$fY+xoNuMKoaQk+}copgR zBxz({ZJR(_a^149UyaYx&@-;(QFit98J~JZS%_)(uft({WPFrXHAZvIjfWK0JvWPW z#cDmShpCps?Tz)my6?*hJ?`hOVh$=ToHGSRjf!IWnc~w;cj)XIlw*o7D3thmR1dTS zdo|lbtul6@!{853EaY-vFhDn0ozx=zeG{oz>C&g^s9G6J<#*LGV)mT@S{A8KH9;2`Ae&{^eS2xSaq&7CDMj~*7>yz4L z8rST=!MXSJ& z^2O~zE;+`s#SmKP^0!^+f6bgbKY5G4+i?Riv_>!VSpN)+WY>F zSIDlc-1KrOpvb=)WP&iR_<*Owqay|EI(f;7ccDj1yULKaV}3A1xR2RdxN4Ew4lK7}$X zCTh-#Fkcnotf6k}mPYHjF_dVbKv8tvBB~i*p#QyYUzI-;%QfnoxHq6bPx3%a{ zvB5u?NApi^Ql}ngp~#6PeV_UXCowy!<8^NPk=>P*Yv~;;ssH^64`*bd zsQgQ-O<(uk_;iunGE?X+k^>jBv6CDon}@XVSG5+B>HO2Z|5Dd} z{Z$Jz_Xz05?QIN>sgOCXpP#3iq(&e?x~|{6!q)yFLQ%!-d6`-1)CnlVbJ7YSg@QC0 zK70FgbQ%;xJ)T={mFqg!7B&^miPdygDp)T*hd2SP_ltb?4Pi|tAf zd^*S>n0M^|cCyd3gA6LBXI%}qx?)^w>tc%(Y6|?ITAzY=*imh9xkS9_HVPBj!jPAS+T?|5Z{XZOg3}mPiTsk0f{zE?HyhJ091KC|6>snP5X_# zWCQ~Dn;qYrskOZy;m$=19b7f1CB5m&&gi&k!iQu13U9_muf&2Ex?)M288{wNb8Y+A z@06W!{@s@s4_=B?4$=}lIbL)N_!#~B&w0mq>{|`iX#Kt(TiijV!@6 zH8tpl4_y~meB;v2;eTd2={|fm+L@?6(9SkeRoC7uC}ZtGb+klbie1hWJA>jgna<25wRtrwr#>4x){WnH)QgJ@MWPC(S zy{(2Xu%p^1+w{+l^E0XAnw-AG(ARAjpY>5T5lwi4Ns5mQ5I6+gIJN0s8%vMuQ(v() z^vX*=OFz0op6P(aefumrk+D#b{Pvc@mw%UQ4kt4u_D7^~&29XcNjyG*P#B1tn69-F=YCG^j~p%tgB+X=}7W=U^U zF^=q*EoDiE)bQzO?h8~XBhu#Wt5@H|9i_MtU7crIo03;IX`%Jq%Ew>k&3mn>!n}Lo z)OK61q5ghQ-QH6q2Txzba*UCAj{0#{AA}t1_PF)klux~PXPYi)`)?k8*F1Z^5qf>z zW=zy2*|!J&A==_vSFPbt$nNbRMZNM?4jq zG)~|ggHz;`D5Gmoiq&i9#1)D5b2xfO%;ao)#ElPGcYCUT+}iX+2fcaQ!v^n_72$Iq zea8r`h{9K5aN~>XYmep7vNns>`K7VxA!Ms!VN|OWF2ZtAZQtFS*^9Aj`}b!tyJx;| zNer`Wv-(&blGDd4#D&?uQK;z~S^T&!)CCPQrcjh9b4M4WYD#J@|oMU zdG0U^H7!xfF3kZt()RY`_B%syI{kg_j|RgdEqQM0HESJi(fUg=`}L-~=XpIL^K>J7 z!Xci54}jJGs>;~!B&iE$jI8JbXjcr4JsGPYw+V_O>Zf zQR>FyIUbWM-f3HByg-?JIAjr7*_`-(W<0p7gz`ol2e11G4QF+R*_|P(3UA4=$d)?M zIYme5l;EL?KTCSaDdeV@wm8`c=lH4Hp~<8Q%}GQneH??JD@*+L8@U*!G2JVoMImzX zr48(O?JuESJ1z?9B+$@nXyi6?8hjK8sskWHl85_S0**vPV!)f?89_!#_ovuK-^hKQ zTNo4pFO0Rg&Vr$bcqKAui4p5a#GdFQzg}?_b$My89ma%#;GZJOE2|cCdXbfsnm=Qy ze#glloU5#et9|3*m!FFYGW~tdv8@b)J7@bt=HOT?-l>AL{Ggrvm>jAgZkOY+j{7C= ziE&2j=$qQVl^kqt*4`jWGA3;l&NTd%G96S$J?HbS5OI|fHeiiY-kJUqJ1kGuPFhi? zaiID0loE_)$KW5%7h1U*zKXOvCkN0zrVV6Mq7)R}Nx_CIo6s3~&pV=qtAh7;&7=!Q z+0zQ2#in)@8;g<)duN$SakO+wDg{1QZSTDA_SGkc@62uNZko}xKl|r#Xuifdd<7@B zFEr|!9_+3o(w)uKu@vB~`XP5}ZtmIj>uLn;L+Eg20Urc(1X}=m6|pCPeI*FsDBuc- zd0}`TZXR>t{WiWO))KzG8>S`7^-SUnBPDYW0#Aw790=(iTQ$}7C}uT!!7rz;m?&?e zE5J2q5`DhWPVsxvfT5@@1&XD#~2=#Y^#=y;}C46?3Y=jh4}sB~?8d4;so}hv;uLM|^lPn)b%# zj~Pi(MVqycfzFaMTF7yP)p727KEUb^|Wz*>iw@s!^ z=Is6V{25L?r*lM`3tQu{4}wy%nK$E(`K^e^+*kJwEi=~4(mV9RZZqOd=hgiimOgHD zrX=h|g$C2c~7uKKTJBxjn4r?tG#i};3zVi$31WLiGxkNuc_jc*DEnue{>l;kr0 z>bZNPtAs{&{!EqU1&tax2unm$%FZ5lnK?T=Y4;G6*Ed>O7{!hS9Ayp^nU-K5F#J zznRHUe9Uuc#mW{(bA{+m9>?9Uc&Aowcj?a2jmz!*iOvIw&My}GQS3M3w=b{Sb%jng zmz5^m;lExHdx!h=@M-T+A^();A$y8$Qj+HDH1aOD?Yu+x;?%W7Gt%D~-IWeMd`|Q! z!^bN*mm67lH+Aq*3AOl;+Yeq*$ z5qK`(B$^7mfhz{M5yS%%E&UY=kC6@mC}frEyOSgv1qI%(KL9xQ)xE{qh3`+&_ggdQ|p3 zugsh95Q^o|r&cyOmn8VUrR)9H52tP3X;=s4-X#)Jp-@9T2LPU>r(}N zD3{6&_%@`E|7JNWUl|xz+bnoF!R3?#-@#u~C$UaDCke*-)alXQRt;~PJyfO4b8J}t zNYNx!X+_?i-L;$NFD~=TInY~^o=Y+vNDD=A?GplU&n{}?+2x#R654M*d$!IW0_Ga$ zRpCRfaDZBC?`Yk+R8e&w2n}Ur8Hy&)pPxS9x~-yS@gq8qRqRrsWfmLhb=lh9)lQHwJsI6CaST9JxZS4-4mjf{K3!&5fzZ`?PJ z;xn}^cQeLaZd)vMPv&=exh1MTJ!FV-zAWo#zfqKS0)3P&M|AbJceLK)m=v3)MK;TA zY5Hfj)k>pvSPAz24$&|el9_igd9giq1j zC3}*us!I~o-CxJKTBHaXXdfTb(qDhlY$jnYIK<1_=$Ys?V(nukqT$^UonMBJ%NwqV z+mo61viddQdn7(#(PQlKuJU5|V5^ct;Gn~zN{*iUa~5eu9JvhEN=U8r^2yG_xco=$ zUG-S6_MJk^(+ssSS~9vkKbx`m^u4&-PkF6oC)q5cFFKbVNexJ3{ahPO-@F|!K}|K> z_Bh~Y5;m;jqcZmL^aQQ~rGWHl#9#K>e+dXD?2@R;mvYzL=uwJ%Sv^72 zC;qhMyw2u-#ehAb+*?<}x5;iLtdstYwQs?aM`&ibWpdt5cy{vPKyuM#nyDU9<f-m`Ek7!^r?Kwtkdwj4*`mm@M5EV9ZX+5LkznR5LR>Z)%!O3}bA zQO_{TC$Z^w;iKcjM=4~b=%hODE3sw;hgc1ABxm_)nWGsW_4rG;w9BR%z3W(5S&~EuuncIOqi6q@Mgfp`G;Pjzlez*zoCOSdc~#r22>lan(TRe8?z#ElW+^mLf7t&% zdeIj_!vii}eyis`Xx{#$snR3RPF2+~px2u!e8<#&y5@(-hY1zOWyy+7j7($nKeoQJ0C(=T3yuw zZ3zOhYiViuR`>9pBeYNF%Q#{Y6X{pYq~A^N+2&z0f%vre_LTj^^TBNo+sSSJJLBZvl`+ zL2`S6o4e88(NVm556jl92yH%>Nz6f_&~Y3+dRt3N3Z5=2c=bQ<+ivc6_+te|6`WR5{LSl8N?x*Pk7*fLj&NXeJiQE~9N;(0?k5r0-K@rM#r| zKrQH9*cZX|ii``mHqCcAreFmk{{$X466=BdR|f|NW~-`!bR5tAruvVD@q^d&VlH^S z%y!PaBwE#jDo7RUjh$3~U}xpcI8JgB1&=m{T6~@Z`>o)zpmo@ry#29@3=+ z+#V7{OA0t8MEc{q;&X?;v&slmsPK%!BLEr$xTyTc-w|zQlXPt2qM;Eiwgbi!T$1M7LPDH^ha{Vn z?GsG76L&H4pnz};2Yq?sFjxtzL_!$w_#6&i+8OttJgWko)bo170u&BGLKK7Py! zHmh0?Awpvr!gB=!8Yz7AyKXB^*RLNrckWzcaYcD~7m%^t_q-wdOJNG*!Wx7Xrv-G8 ziQjjFF+UbAtFNC+NX+3#~e%%FgX8$!O+0a5X6^HK}Avt+%?d?f(|MRZ`=;nv3bxl z6*zIoIZs3HB1tGd$0stY89~dzt!Nq}5?soimwyYDjMT&Rwjc`+h8X`K*@FYt$y8+& zXqylsP|RxJ$y(Ow|I>;m{U_){uU3GmJ?Jko^x{U^q>+%e5na%<$Vv;h@tKz4TaI#}k|fYT;X4X(w8 z;o;$3AbPUnE8#ykO-w$m!`^~bzr9)?MQU1PGu&4?|1}|TD+0l^@Vde-=RNW%ps(^Kh?m}hp+-*bi#ydNr#)~L zE)MX@$qko+2kjlqwaA5+V&kpAq3JqHZrGU=N3sBq70ls-Kzj}WD5ohHp6hl&au;k< zGQ8Xb&b{Yv0s>e$IXRmEm~0L5LOH&neQqJ4b|iB7=k|d>^CnQX-+^ALzNNk0X(`?i=N=%Ftq0U+kF~SA|3ynp2Ads$TS(IwJzWclR(dI6BxbUK#T&pGAulL7cX2$8(0LL?TZ4( z_h-ZdKxfyxUI_sLA`L}Ahv3tJFq#^$zG-I7IffO{Fu)N`0L-oMD0I73!Kp?SqKOs* zJ}&Y@m0%MCB3`NX>|>DKK|bf{-o{KNtX&$gf&ZJP%6RJ3Pl%`y#Manb7Ge)k zdWFpszy~*j*#mK+vMYJB`+od*O1OOPy!-ajul=pXJ^0MGH8iFJ*laI3ha&#K(uxZ7 ziy1Z0n)_)_c>vbf4McFRU_BlzcM^c8XgVu9*9_t?5-_)QmpNK+y3d^hs=!`<4M8Up ze~$s8*7M*Pg*D?cIJ7T|i=P1?DA3|^ARa=(BybHNW}=e%S#Nk@_8<&`tOs4f9S}N@ z0LT0H(TR@BMjJFU)yuWg3PL*O4a)nOq<|LD+>&fG-rm)JL0*ekx-Zi-h$ee&>rFwVCFHzZS}IK@3IQ!GG|a>N&I*&a{OlsQ@-mS* zky*bV@^Dkh9xD#=a9b+GoECb}$ri98284u!OiWEpJp%Q3x!G{Yij^CHpzDs&(n=uu zs6_ttS!&d9sk7i>XqHwvQU8BhNV8ZN~|2v*0|M*u)rKPznr zWK0Ja7Z>@lA6Cr&d_SWwvzwO;2Ze_6T)9#-GP~0EJz=V#(d?X{VB0z;qBw+*c>>~% zf!WO+Wt9a`o}Aat0E zQ6E0kfnnsAx=7UnGksUr(kX|hjQPPE>+7EZVXEU^9h}vM*Z#0`Rb@dD9H$>HSHT*F zNK_GO#K1n!op{g#-W44D_FHUHH_vF<)c|nd0GOBqVFz7^E}9<&mk)?28Aww znf17_1?*oyU?JU?MDTul>k!HTRDSn>jtydlR+u<+tPf$?zon^(g%QF7-vNRZ0e|3w zK_}$=15tT{O(E{eb7V4}2CyfY2}E&;u-!q7Y!DwZFfum4(Fn5O6bxFQF7H5sBDgm{ zfp88kMl1q)g~eHZt(Ht@`*ARs#R?$`GgCJ(FenS-0=h6PZwSB~=oM%f3kZ(f7VzMz zTdOeX{e-PcJ@Ol_p$xxLar^e|%)5zFW1ix$xF9ZX2m>$s5IjMM+j~4oS;Z0&F~86S zK7dBhLLEpKH%2{H*O5(Sm4{1TuAzRq!VP23^&3X;8)26q_-hbbL%={`|8Z#0T2i=b zQyYoorhi2u!E)bdzWhoJ(HqG%0Ml3nMjJAv?ZMAH8!onqs)|0Z9wVF%(S2hyJQh)S zS6~<;hF;A?>1$)vUWm206uy2S_`0y;k*OK^s30{Le>ry-k$-{)MBr3LUY>{&@jx72 zF%4cye*Gt7x^79xEeuMm!x|~UfSQ5$|G7&>{!IJsA7Rf2ikts$35Ne_-NzA4aU+ur T-!zYsz^~gkG*xo1o4o#S4JIGz diff --git a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/convergence_rate_Flower_FedBN_comparison.png b/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/convergence_rate_Flower_FedBN_comparison.png deleted file mode 100644 index d80398ec033192dc74e6cdba23acd49577041a7e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36446 zcmdqJcR1E>_&et3dzcrz4xkQlMu2hq-@!nGO}fatlK6#vgtYB zeZJr4_s?@Y$Mg5oar7Ct`+dEyagNvdI$xJC4K>9JgqH|WDAa{JN(x#i6jmS#g~pu2 zgHJ@hPt3!=BwTOly54iJaP@fXY>vA7*wxY2!PVB<$2P*GiX6t3U*OkJDw@Qhm+ zZ`*cwwPA4k(e0qm`%~XENEIH=Dv-7myk)uOIQM1SW#&s1(RBp{uQ4Bo@U@5T)bSOr zm3OA((nU3X-Uu3SP0g757Q5RaRG1Z{U*GPQvgTgPKnj1j#JYQl*YUi=Z1yXOTzBc;tUGb=W!s6WZ9>EBP^Ir0 zu)En)zlQUy$18XrRoXAS3t^ny>62S(#%F4=72R3M$rz|{v$-T{`)#xFuOUW)Lp}X` z(@W-?h2rTXvIl|j@$uZUF`?gvXJ^+?3JQfo;|`5cjg4}2qPCae&xnhFfBx&E4lQn-RTL(Uyt9<$THCf#0eWr+}LROY0(c92?Z+v3< zw|C+=yk5x#zI}VSNeMM$t1?|JYWK4xUB-w0SU>EVL5<_2+sO9W@n+ncH{PvFA;pDU)) zvZ+5iT2)X~+!@v}GFap$BdxCaH`M-GY2T|?uWod`=3SYn7UHYh!rEEt;}jAS8u8zs za3T5r{X36d*|(A&i&6o%U-u>5ekrW~EZ<43z5WdEexbfZ-}>_ROx(7?avPn(`aM0_ z0}OUKR=84F7MQnkJ)M13kciEX{>FfS3j4{<mzr+xf*g+nb>SC~&rA?utq8Mp@lOk%F6 z@r;F)^;yBAN7!^3A+ ziyE)}c8LhpCjc+jvM9G3rwaV`O{u-&?DTkUb+rp-lq_UPoTHv`?bTT0;mTCKl#KT- zrhb2FP*t-1T8ua{c(;YEhgOGwMeR4=@KlqBOLYUKBaa%yVoJPki4 zxAnCU)mDl_L-mu9i=@@}FJU8=Pp&R~U0Ip1a>AMD0-Y!u?ACCeju0H=li}Rft(mVb z>vCq_F8ZGxS}yjaw9D*IJgcnSe-KgNv%lfGIhAJK_1d7^CNxz4YB^$e`;C83V{D4* zUyqeq3VH8Z-Hu`GIr_8eb+X<5u+rWj9qapufd|8b-23RNCzFj!-lu;?ijG7S6s`q# zq#HGcL0k)se+ZlU{}kww1!GasOmgY4)ibNBDjFIWvGIw``qE{nr7Z@s?j)&x(65Ym zut$wNOVp`=Rb@LScsr7&`DeM!!(wA}a&q$Sui=RK`FSG7rKKfBGqbEY_k*n&lW<)K zqb**Y3JguvE=!9?UN9IAE^dkSAV-bcsy4=nBQEUa%a?f!=}1@()8f!W^(DT1iINW{ zq`X#Tq@_g$2FxZb9FNAvD>iM9o;|0I4EU*2pFuvQ=l0*T)3BHr%Z1Ja+>J_$o=f18 ziaU!vca)Sc0e_E~x)S*;+oKuo7a7V-n+^;P-ceWYb1SH-62GdQAKGDd_4f!8-EZ~T zSGp{A)8=XygpcWSjDNY-YuF}zw4Ay3M#Sc445N(qopy6Gv(eR|Po@Ef3yDM%OXVYm zrPwtqA5!e$ACG9O44xEfypI1w)fCIBDCn}N`kK#3O;>l6ohjgWy>f1L_6vlh^v#{6 zN)!hhn?~mC3(<@+;Tajz=Iy*NfmTidf!O|jjoG=m?frcZcX2gM4vwJFb@z17Sptjh zq~?Ka6+Z66QoRD(@ro#T=cgvWiOESP50Adm$-X{yB&R8H-AfN~NC?AmYzZY*O_xpv z)1HNsynmkpb#`{<>Ftfh@4Bp!`yj6k93AaB7bUhlTEgnK`h#6o_Fu658bY#zc}60+ z?uJ+jMF?1g|DRzU)b8?k){#A?fRotcjargGR>kP~OUqtIyVi`}%gpcJzdv{WeAe9* z(R{=DH*HaL)HD}$9OOQ1sM|kCw*r)_W^g`YLQ|6(SzFbc>RC#gyh*q7Zw@W?N*!mXO3?k z5(}H4;H%Kka}6iIRs0VZsHePF3-phxs;Y_|XKuXad8m~&axGG9%9C|ubsCagNxOzF zEiG*?@w@JS8@2GFwpNOnnYr4YDgrkU%mPWdNXj>;6`gB~V&&%Ua$18lzx<6CX(fC?%+PsHU8WVX^x@HJ zQL?Z#nM32LsPT{-y|C3qFsb}UmC=x-et#k2)8DOw*lWH$+v>X9&)nw%IZhI$KG!Gr zm*;+@L9P2jcd}44RV_qj%}-j{`UY)~IWrYw7=tCJTiG}{2WveXu&}YmiXUSSezRAZem}&kmP%S4U#@r+i{C7-Pr=ZQvEv>lNd=Fg^sMrP|E~)BTOfp-=Zh zAZ_lP?DR=R+YRSZ!jRE(m*h^iu;0CVw>DXu2uToI17iGCz~9q|1}10oAtXR7v_X_s z@${^k6Mg&z2ZbOYI)}QTn3$Nkz5bgd60hsV%WbG(i0FBexA$_?4h`q#V&piarBe|| zWAlS&Z)36xOfF@!0l{S@B_$_YUrB<)8^J<&3~DH9Cws#wxoNa7SWUiuCu`NV4m5QWypN;!3;jm!27z$v1$909GG zSy;gOshd1w5_ulJT9vUMEoMdmN_EuFgypUSwgmlQon&Iy!*Gl_2FIX8@kpV71C5pRJtm+mGZLd{2f(jgdRzsPPQ^-`h7*96|Y`AcRDHS?TggZ z?U?3gNSFZ&F`laPI?p6KAyE1-JO(LLx%R#HH+u3PmL|ys$O2T)1mNTgC~+wNAvyn# zgfszDyicD#l{rkOL$K}_xOw0U?v|sRa20@3TJtmf!K^#vi`~gC&DuqwP@(wewhKay8yzMbU`?j%oQWs6zpnISB#^K>ISAk?3*z+9-hC=@2hZo^B`gSb$r< z=rD7R{EKZm&;`y>tXF^lMS$%B5WKOlu*&R<$dlG?iG(!JE32xuOiyP#e~xoxbP_R4V`w{^JLhaV$#1YJUb}ZPnHU!N^%jTxi!<5w6Judo4o}CYE~X4eEPL{-B+1o$6Y}J-V`gi-f+C} z)l6`qdVRXF@mrzOhL?f9KA0UIcom0?%w$R5e}(u2puFnnsF;{`Grw~2Y49Te zCJ)TEscxH@vH2bCqA~sNZYHU6K;7oDjuhmA)&p54Q7VN%d2oJCx;yc%<1)at#Wp$z zfFXh-ygWSlN=4fcDZzj5z}Xo2AF?-|{3dd!|D#>#2PjUr%8^;+H4oQ-4K_g7&!0bs zulWM}sH&|sa%2h!2!KNmUwh;H2e$FE)Xx_tqRl^_InhVnb#@nLVKIO6{|Z*R5|q}* zD`Ozu&CJYPhr|iR$MuGhbLY-oW@CE}5c+6uy(zX1faI^Fqs` zFvDh%5q${lbo6~oEp=AkS$Eo{3M;n@1i*tOJ@NkcZi$F1R%kWJ!q(!IlBfZu;cjD)NQ*rxoC?NGL& zbkFcEe1TFKf}4QTtODfU;D~f6!0UMo>yn5WBrOSWm*U>LSKydu zz}4rw_!`+bI6gGd{3tRaY5EdO1o(9xuv4c;<5EUI`pRI=4!p0#Yscc(j{+5#Raits zSNSwV(UuS<|8Gz>Ak+X<{~pphNe2tRtHq@PomW)Z6i)GDXZqJf@hXd&XD0(lmRHK?A8U;B{z@nQ54ggOc z!l2hqkC+#r4ul^-jI`Ec(*i>0{emZTKRl+q+66>6^q=9Aw3(E=Oh~xp?Jendyl1{P zTH>{rr&oSeUFHu3U;VDuI}VM5UoUkeRt>3OXnZG35M4F;x6JlcA+c!n}&LCO(>0;&K) zQEN%h@Nfc@>!mg$h5E;$4pS4`Rgi&D(Ny0eTKRNt^9J@_92%{P8M_b|(W=`1lQNms z|7jX0S+!`bimpP|eaAn7db6o+>nfP_^}9eP?bxa-E1N*dJ%35~N}mf9GKsEHFJE44 z7r7*C)dW=!jkG5}5~I>>@NsbX-Pd%%>J+CN{P?VMfFn`5ORfc%LJ znW~J>yZst!2JeStg7CR0K_MZ)24C3|imP=`oXugX&n!^~51zHm-R3rqA{IU`K-MIq ziKNf)@w}o2W!Q#KkUlCW-RYnl1IY2&@7Nv6OkVH3wL7Ore`o=oF!^qw!@|Pqb_Y}r z)472&09p{@4xm9W|EahBr$@Na1v~%2o=6&j+4oLAavwxMJaU3c3Q*WbjdwSKj(_Co z#5A5AMZnh|3nqImB7vErE%Gpo`Q;KqJr1 z%mg(y2Kr7+W#-u=vjnHv)wg_?U{h> zsiStl0a{H;`m3Bwuj&-~Hg7vJ`R$Sh1qHcoOeDivze8Q9S81PSlY#IkfP)4AP60G; z0p46yL!Czed!^3xEC|9a;grEuK0*WD|N^uVGN=AQt@;;}cp zsN3b4I(b1~P9fVEu+;podsxe=(N2N4#T?a$FtJq>uLesAC6$pMgL*Aebnuv!KNuCI?0oIg3g8VO!;bDheE>D2j>V0S%icJ$7Vw3XMX+4 zecc$LfwgRVJ?b-gQ<}ym-Y?u^W5MQ*ELS!5?0W(}TELCnF)h*EW#`sX28O*fO^2|p zt0(vpWgTiX(;MkQIa9(OiMFd^(xUJFdFE0Rlz@O>rSGt*w$XZlK1~+5y}Y>pwY9?2 zVI4&IxYyEOCgA0epqI)}xkLyNTZjcv@kJ(rr|wJuP}95{Eoc} z7o)I&SdL}+fK$x&%^EFTPV!k0^|2_q37LDs6*LdQNp4Ur)97c+kUu6h?D9VzI+jJ7 zgRE(Lt{usWzJ7iLgoGxTrXZ+}m)^HS)f)w=8+r%VAqN59?k=E^LK3%PE9SWCjQgvB<$d2LiKuIKFP%#9Djh4#h>px5H4oitn|>}hRlBcq^b z`E1?=9ND9W;|Z^&G$4C~fFo$HfR+=|VnRA3A6Eed{U1Nnq1t^tZM-u<8^`$=TiNV+*_zKPy__|iEY8fh>kC_=ANJx2=qN)#Q=Vjpd3+`FnHaDnoeP=!R0e6F%lJb_L zqhq06b91x8ty`3ys(@zzn?`4llam)oW9Sp;o<`tjaei4j_fa?Ghe#xj<4)|s``T!v zV(~FA#6@Ui*q$`5dmktctJ`CkVj3G8=cq1S(sSjktde&mpK~%3NAcR=fE|M`oAhN- z`{u*L-`KC*Bfxx%Wkn(^;KO&`e|}LNImqOJX+Z{r&>TU{#{p66eY@+gp)jCS z7JgPh#s^s(oC}XnnT?x}Y)2fm!A=epErJCDvaLo}fq`%I8SL<<{Y_KvRE+p8-;6 z$F>&0Up4$Kx}lKR(R$%29UX6A>?7u4s&Z~Ga?FAJblByYH|OAgS#*NIr8zzl zBpF;@pkQOfe|yUcWkH^*q{J%7{D+0}A}qR^A7VjNUWfo44gDOde$v2-#$v8O3Cja1aw-`8>hUHSjEbt`xkg3Vc6vfr@*l?7BG@f3Q8D5vq2~*imGY7om{D_ zOp^5~;&QAo9fjt*&1NsMtdxYc`95hjM<~nMo9V5uiN|8*k^GV+d=E>PtlH~(HWJ0O z@10QU{vke5T^jgFw=Xw?pU7u+W*7``i!vFhiNuZ$In!n z?eQc;bo`A;rcRa_=~(lsE7np)v2w1wYW2d3RfJZX2CRcQ7FkxVNT# z9kDkR*xM|ZNNjyV`KWNiB-NMk>9=gPY6icM!4C&iU51t7Fp8F)bIG1mb zgK$D&UJzckl3RxAlUGuIPM(53mp6}XMUQA9{!K4!4D0b^Wc%1;Q!ehmhUC+yk+gVkft7)8qEJ%3R%;@m%OK%449`@4xy+NUu;=#&1Pgj2fW1&qW`OC3 zSq7F|9M^9Ahk-uBn(ca5u(ba(#G5d) zxItX{fak@8jHp!Q-pkDFgJIVZFS15ny3=AFR+=J6wi2Lzh*40u{MyX`FSYRy;VM+gwxBu@)BSfNlC=Ysd zM-zGEDBk>gRu84l^!{#-d^d-<>($`p7U7&1?C{g+7wT77j%kkAE z1g;>5#S1%eTcR{@ko@u=VTLbPQN?@4spQZ8nImFD^NYB$%x!anq^|h}q=9av|E2v1 z@L)ZOnjlsyYHf5J0dX4#xy$Xjf8g;845%GsX@frUsTsJ^LK5iqUm^hf1dmu?DWdw{ zRR|Dk{EKML01t0_^2)MHSQnnuH4L!^}BXh1-ci6i<5mji(w+KX>9lY9TumkVMOO;E<9GW2GbR6PxFo7~^2q3t>V+#}Ta`WOMgApn#XYa}MUgo0*luP;Z1s7nE) zhYgugzEACFV!{H6{sU-Sgr5v8=NjfA|7F$Rv<8i=wfHR`q4ID&$!3?XzwS{x`0lyX zg7^jd>wDx)3tfqb_yz@C`XJ~%Ls^V;FQE58=_3sU;C*NkHZ4s~PHyk)bh)|IbdD7y zg=hzf5TW8a=B;oiT$TRW@v9UV3LwPQ`}}QrJy7dHtA>(~kI%m0&;mFTk2_1s%E~Oz znF0<61+P9wLb3#iDDb}69`%Mnt>bw!EC_9lz48NfIdT0n6I#EsG7L$ zYI8Dqv1Ns+ORZkpan1Grd`(*IF6!pZn?Rdg=;-J`YHY)LZ?q${_JLZuwxUlbW}gMc zXxWot!ocsDii6OtHuU@R00puihK|efpu6dl-Z>8dH4|E52&)uKMC}Z!6YB4I+#J_7 z`Yn>SJx|4=d+eTE;tfn8$!L30D#a>@9P>L>_u5J8vF1c0{|@NQ2FJvZfX>4Vq#MvP zkUd@#0_4R9t?k+TenNHu0aBoip}&|q&XDSOUe+4Vold%e`01rr=H=giN8z$%TM8U^ z6bcvWVb$aTKu``O+c{lVP_siLO%G&sNUxAvrw}A(Up^VsR@`etz1BM$e%E}rZSKM} z7RE$iF!weJMMsjyh@`aCgNMXsol6Jj38NF zMKs}ynKum8eUX?#bhnf0w5YggYb@mhbltoCU=_j zNhI4v<9Y$wEW17Nrl|iy*;wiu6fu)*D&j&P?xvUsCPTf#iGuZJLc<7IZ08Ezln~u) zjAR?y9$1BcZ*p2}1$x8nji_BGSQ7$gLBWC#0uFv4X3}*TXb!CD?4W@C7 z)pu#hw1l?zUEQ3K5It+F#*1U{Ghy_`EBx0@eAsAm>8h8EjTtljpD6t4r=o4>a3rK8 z!Mt*=mP}8#ep_)vCGtMHjl|YRmDhlY$_9N#zHZ5vvBwGaZiT^%9%%Q2C4!$LTe!w? zZD=SI&Z()P0#5(v0PPh9>@M`J*^s|LK5ewW}c7L8w|e6cjnr}PZ#ze4!D z+B%l~&_`0BMfug`PF)Mv5&CD^|Ckp@P-iW(;Cr(vFDp+82AP`d-7_tJjnkJt|GfTZ zs;XMNentoMOxDUL`yK+0PdFU6FxTPi5-oiYX;WyZ;|&FK)xk&a3>{P}&da*~jgHLc zaZgopbO&mLW!CI}9JtI%Gku+i`WgtOh6qR#1USfdPtQ>6ZvRl=B!LPQ~>`BRmU) zL387DuSME)-`{tD-bEybC?Fb5K$PWsw2}h{x3*P8EMxR+tnYVS*L}v*BB|Z{rL~Xn zY)crNx$NGk>GS8$w`M|^euFLr)mGI6#f~8KOW^n_%KQTJw>}-Dj>_j$HHL|Mu|7_y zOJhiYvlkEhg)$7%5(#xZHriTpj2`3OR*#X8_P6u1R0P;POOgHS`1tB~*Z$w{uQLhi z`+l^S>?q+SN@GA8XyTeB9%y_#Jg_ptatHm?Uq2{mb<}O6@R};7tr*2(GM)BG^_siJ zklY(ePT_-f(7~BOvCqz)qL@|y)vUt)neMN`WV1mvq}tPx;<9t}T!^(ynW z&v!96{-Q1JUuWAlSEESPLL=CVXy|c9lP2i91olu#$jd?5B#;hs; z_aRv0ByHizREy8@A2zlDM+Bfjz~z^gBM1dm8br`{Imm&(9x8a02oi`Cpb5p9YXrM< z1QoemKWI-V+dj(q@<%3>Ar_vaV9lvO9UUK+LcE9M8rfk6Qlk>{E-ETMLsj64p~aI8 zktJWRyfgXn>pO<)8|b%pzhJ4Qs<}H0!z{H@;aFchf9?paM;EZY56~muT^U4@?9CVF z->IcZuKg^J0H@vt=3iW>TafGHI&W5zG=WI#@0qMmS>}3a44_R4r6AAY2%HZ1E0cLU z>Wi%`-HS2X#W2ov474hm%>)IU=aniPrZ-xybY0O=R~0EjU&hRHw5k3jsh!-Ie~avk zI0Uio&Q233yS*xapry+jwu=3|wRL6pXS1`l^r|b~+bDCUfRGGK$^w2G{lB&2=cr*- zgMI{Ibv(CQsjWu}8BjoyBg&S?%l+?ANLwDXH!$zd+-2A(8EKLmFVZ}gHa_?l^Nom} zcXl?0EpQe|UQ=7!2X$Q?2)Phh3J7(v5xo(}H=Bw0>H=F$W+2-_X3Etm+Q{c8icd-k z1r;(789+ElNTsSICKJxxY<}HweBm#qYl{b9a+1J%BDZJ0sSIgQ1wGLG)z}yj2sz#b zh+3@pTqA|#1C*bS4E?mykHD~r`a-Hzl2%Q=Zy{cpHIGH}RtBMfOeE<2;vJVYx_Ff4ak->#zx_!kF&5ri8A0k+2U24*$7G%5UpE&pf$r2+CwGws$ToWR++Jv8;=WvF zc==lXxR@|s)-n7Ksnhut-NEV-aIS9;kQ@g>7l@%xp(Q9O6#2VHj0Vf5dF7^~E$jMM zJyfwQ3vumaVU|9e=y~TS)Q(nGAHzu!amE&m#cQIq-vw3asVcC4a1y>Y{IA0LH{@2F zy6j_63e!uveoKC=1QnDSZ@N9z8yymqu|W&hQY+%LI#Ks(13~$t29m&!_i~604GpVm zY8Jg^uc?ufhTqy#HIBB!v7uW3dbN0E%le06i<8f|#!?#xJbJ7569I-Z!*{m3OC`E9 zs%esBfSlO^fBkFj*6;v!zFZa#GF=|$Cs^XPqx_bJhR#Vn8ubkHx;TnIzJF+SgnG#L zYVgPB!Z%L9>)s*O{?3F6JE?S5v3HJWGK*%x6`RQ*A|r%5D(}**>S3ft)7qo(Au={L zz6=}?^nyVn_tGIbHuX2Fc3yeaSMEl7^6Nj){SDf;Dwx#G{93qK*x_~#iJMupo5 zuS8(%l=mI|BjAE!X*>GG17jmqvQ?mPAeF|ms~Hf?CtIc?ZCy?-uCRjxPf+iySA9BH zM*iHQ4=4JFgiO6(wwosJlM|BN%_G1+=GtR+=Ht{6g=RNDTS$P&$M3XjqN7`Btmp|R zmLMVPWHn-h4+sy3FHhC`lse7bwtR7UY4sqEPX~Ny+3OrTJpzAx|XF}Is^9j zA!Z0iL-|Udyp;e=V3VC(f9)qtti>gfkThB(g;KZ4G5stD&Y(SF%3d$ zf_9CJib@;f4@D<*y{Syi5(~Qn0{v-m@Tnee9*Fz7??3h!qBUF@Pr@(cI@kwK(i%uO z+}yn32%%#D`R*QX;Iuuqhuqn+XvWfC8Kr`7LmvMR_!gxrv+WUJ0`lf($F&!Q@iYjv zm%KmfQV)>RTIq!sg%E|PWf%Jz<(IOO!_2rbyEt2;C8oY0twyAafDM;w48oDhs>nzV z5oN>u5-`4oWdXq7*?{9fN^%bxv=G@{z4+ee3#PpLnu)5iJ9s0dr-K%zMNY2FeS zByx!Y+V68+UR>shDPpd^^nq`;`c^cHSlp#fYSjLb71;@3{?NJ<0QW^fuvfZ!x3#@J z1W}+tI^4`P^ns#mcWq2)qyljg=pp2JPilQSy@}Z%DXFD3Fy%wt6QgGrx!GES)Jv?8 zyFrp^W@d&$@&GZtIL31WIBT6hKQ%QqL5Q~WlZu{R)x?AaO6Nnyxb$>26BCodN7a}a z5(Y-Rw_`yL1Q&KMGmFI0K8lw5FC)&w(-fD0e$G*|4GrGAiw$VepdulE;MR#uWX1>h{$nVFlNq=1`$5LKeEU3hMN z1OM&FCnGBzp+KRPhtj({x5)+t)96#d#q58rAfM( z0L4;W!K+&w2#|= zh;9b7v^xulMhMj2+8tD0zDJix{dkCuc`^la?-0 zi2N+GjxE@)h0q8Z$6J8!Q8Y5%!k{#sg$p#emjwj}V$VM<;n@CrI{mi<+9!HtR?UbE z$79+r=Tac`;Nczq+8Zs-Z^qnr&g^azZBVNa^RG6N{}*`)zkaeo_d%Tq4V>T5(-`=0 zmkxARE;2>~BTL+sHUwuoIJRfBmq?yV`Ol_})gD+JUY9oD?WP|KK5B00jfTEqmi)rr z@iD!cHQE=X14?RYi*S#^1d7Mq&FKv2@{`lhC_&}~p(^M&C0T{JA}dH}_I;MFdz0}^ zjJ~w_rOhHT*qNxZCnY9|fS)TlWhgx!DqtnX%56q)_gBH@1#Y=|b*>v-beB@K!p&jD z{MdXqW@jt)TE4MQXVyZX)MCx;{w`O0oHzYp*w4A;7Er;cf{BT0_smyhuvFxN&Bk^7$#yaB4*N~za*i>ZYnzWy)@|We zZc6^Z!Bo11l>AH;ExQk)=-L-+0%J(IpETZm!FAAjiYpcz{-@~;vCk4#f6=W6Co%>S z?ve@SKBm{a2~&D{>liep4+ilX9%%}fHiL7{&!%dh^9EfDh>oOIe>-QZa%56E9v&B; z@>%)gGZs?WJKRg#)V{s7`jb_EgkOC6=3r@l_k{D)qC_T)ysaRrYd_7*Dn}1W&2m9N z%}UjnGMuf#+6|T~SH3+=Onl$(`Z5-^VRY`(HiouIhQa3dWaqipYhg21`h@Q&>*#C8 z*WLY%0^-%F*V!cn7*Iwe)PkRFvnHJ=N7~WM30$@127{5pr^_QFV>80xnFO0jM z%d~y=hU#xcuZRqyR>g+xj2$%b4kT*5oZ3X+lMfPX?_jc1E|qw9yOkA~$DfS3OE+`z z-Gsxab5h1yZuvQ=HT34Z;6~-&h0%B++WD9$#XEhUP0Gh;h5YEIdBoZdWkzbzT^Wt9 zLu!WGM!qIk3}c}VHj@zbYwRp09;e@;^wV)={x8X6iC6GK1vaq{aI7pb|g|{?|N)6#b`9~9Z6m)YIN9F%oP@n+9EN_>%~4) zWsKn}dh~Jia__abE~{=B+NVPLR&AdM&fzpgu$ktp(32z3}4)5K-c} z-#ByW6n4nZ`L~}B)g?A^d-r2yI6B9{kzgt0_~$~(2^yL-Z2escs(EQLnwH_r(07jg zeV5>^el;{P@e{wOpE7e1UKPRo{Z&WU^ruj&ile)-=_?e7J-PU(tqhK*NZfK$MlEzc z$j4^xX|~5+>-WAQUH3dCBoc4>s_y4kGyUO2>F>SV#kML;w{_$U;RViW1aB##7)>MD zw!km`T4EOMC?X0+x;*{+CWsiPczjqwEwEq@^!=1k*;c}Jw2TJm9{aBP`s++{{+s$$ z@`o9z$xX*5uatU<4GWNlBMy6@b9t~EJ~fGwIN^6G!U^7%zGs~c8@hJx?zcIpOc8*$ z0NUj0nu)r%?j&>DQ%`Gy?sELcx=@hA(|?8=zgIQNW1U-y#2!*bwjoa$w!!a@58Zyd z=nXB~sinCX`Clug+)u3#==<$KHjYN2Y-6rQHm-4q-;T3frH3r|y4>zN>j!eEy>^b+ z)bs}@P6!kwhh=yU^`&2^v+NzSi+%L?r=M}x(@_JQR)_5HMtpZETj-)7LxOxFT=5yS zOz~gKTU=tJ59#o@VIi1QnwT{u*;;aKOY~nKOecy=o{RB;-ctuk)|qiBe2!_fCCfYK zn^o)hrWp)ERu6~CHLhDBE4Tyu42p#{vzLvGLCfnH2AqA>XD|Gs=J}z(`A`t}uMt?t z!5*mP>)eLc8*CmX+DIovi!|9fi#QDx`g&2dT2O{LPBJ`i8{=g*H%A^={m+GSUC7ChjEW_=rol) z?mOfwRPdELaMp6sL%ESzHB{kUBdjJ+SdQB6&4z>EBWnV#?y@Tmj-$vQkXKw>bbbYe zpl^j|V`(O!YA%;}lyb+6Q4bC0IaRCAB!TKw>*m`VR+hJc7uQ<6f{uTcUH$#S#M-?6 z3)ge4_92@0zDT$Vj&%e_&1OD$`%Mpy2Jr^wEcbQ*P)$|{}=0W`J^`@&; zNq0|5|KLm)iZ4%$Lh$WGA>SGLh2`=0F-4SYMUZys@%F78Ayi~WY09D0`+ud8(hA(; zQvPsMKNGufae!V+?YqrRrmukw?WmnJB=YsEWN=B(c??-Ir7jU;5=p@ed;q8_D;LpsNOt_%2-axtNgDhI~up>#vCen3poB z)3qdB>Cv*yNpbHF3S5MtP8)TX4$App3z1j3q=z_KBChQ0n$ zQ`G&R+pI>@<9K2b$%j&zD@SK!KkMs575&G)jdT$giyd`8AYkSl>|7@waqLRWJKWSg zc~JcV0>2X+X|`g@jIn#L>8kb8rREl&M}sWFD4mF~xH~J|*zIcf zt3A0KqMnW7_o%gCi3n9%0e@2_nKrhP{N`D=L04!))7?#KFuLdM@ImHhSc%`I0oi;k zQ7P%|lq_8@MtsLptYw>?&?r_S>sU&q^l`E5^0x zs{@JqTbp|2EHmBefB`(UmK>@MxH>d-_sNHZ55p(?$;LWnESdywf}y#T9S1vMkXs(~ zYrXk3L5hF(PFY@SH}!rQa5^LcmIKHDy;>no~h;)Xe8=0k+#b{QU{$%?Xx$7 zg2Jf4Y#)=tU0r{@zi|qmLRNpOAe$(jE6<#U1T|E}-FgBl=PeGZLzg!{_W zcxn1@LrbPVVl$+?b#VC0@A{H5`pUpZje1{oR7i@&Bnbj<)tc&7t$w*PF?2z3j5awG zs-wQWToQ~DN@qjrj`!T1l5WZF*fCRl((2Y9a80PV7#_# zqnV(-`J`j(S~<$oa_^ep=ii%zYDyjR&%A2Uo;8+!SC6W&`?GSH68t9b^cH{Ke)$CJ zf5*+K_}V;6+%DFfpTm5OWqJ}?elwfxwfqY+(YF2zc(Qa8;pjiop4;706Px9)LgPA4 z>J~N9=Rj9DcQ)C)60Zu3fulpto9Lx(3v!uXI(Wt%_HNjS#UQih+*I!0)|}gJW5SgF z*lO1hui)w{mHs|05NU@cUFyG($3xS+PMeUe9k>B-YKnK~`;9(WJZVubwp12=gH%zOp?>Dzm za3=VBftcs1n~{VR87bFw@QSgE6FByVgd4|ltOOGhY<)r;fG=E_G*^itcAWE^+kUDu)2}?| z^h6ryOV5cp+^q{i8>Xl#=0l^WykWLd>GI%=j2Eu%?D6)%eFy5gA-6FC%p2?+r5^sy z=2TT>%WwOup8iJx6Mx_uwt~6GEr-J%njG*?M;;qHF;$ye*E z5=F{`ou!D{M@z}Ou^HaiW$zPhvB`9`{d|@)>O3#($C+=o-Ykw95VVut8|Qd6tN!IA zG3uGxMvz01h~5=G7H0N1Yq}V;25(awxXs+>2(1W;@E$xT{8A>%G`kBFvCeRnI$j34 zC&w)`ONsQ@jgxq*jRNVbJB(FTX_I`ll*Mx$w7x!%XyTZD1y$$CE3ld^#M1RIHNLj0`TNW0_X2}^`@AYZ{Ks@-#i_8{{3=GB7V&a^wv)~ zDSCrm7V7xH)0CUg7BRaArlQj->8P@&%zTZp|2VDg7}}DjxO3Y_4l}#V-;_jadI{gT zOoktLR~d^o6N!nM&At?kX8U|MO1>da`a(oUFa0C@GZz~)74`BTj*0D30osKvM+ zDA9!9X8nXd^q{Ff+lqqLW5M6K z-ROSUw8D#yv`WE^w%u`yv(2!=1E1x?jcbI`cSM~@ zZRC3$=zy)Qt+zKri1Upa8=Cw$@4kWGe8a-)WsJ zC;>PI*Y6hXW2Aj1IV$DbN!#WcnV#LlKX^RMj5?~!S2j1=mwwkU)hMuW&Ef}Rr0HQp zoX+3+*a2=%bpChJx~ph+Qx0zeN+&^DemyLbG-ae~W&S961uI?B?J8V%vw+)|Knevx zoAxgL6O z(JIQ@n;uQWaf`z;URjUq8uYybbu2dC-dk6`t;0`a&iqs}@nvPL&pKH0;?Or5`-`QkUwCKLTrcn04pho< zgdJU)z3e=UC}?72k3vHjy+dcdh77`O0gw%cLZ=x73O9q=;Tj%XxTmkgU@#_;y2=S} zU-a*@($se)bx1j~=>NS`7BHG*pkGRaBlaxtOUGl>104@8$qu8iO@BGR z(WKLjU$_kqB5^J%?<%}ZrIP1|Qo7o^Q-R1BWFBzi!~`xvMccvc5cu^tpDpe-6TOAo z7RY52xB~(gMi!CFKy7b!s4xhWjnm_AH4Y zf+IY?dY>q;B*2bULM9?f$ol-5J1^7P1(W@P=2HpIksiDyK9Qvva1NMa z+|Z^o@}PECOnbanJx5sd3LeUpI~SQz9}!TQetAz-a+*hOdCigVwa{6SiQSy+#}X%_ z8|0vEPS3-2%pjAG&C69~3rx&}jxBP{5Bh7!9T7wa12;;#;rH+uU<9B&(mp*s?L>6p z0&iPrcD`q!2X-#Lx3jctg5}w0XH%a}$%2x!Kj1;8J$ggaz2;{T@5Ws&P7Ealezp`< zWx#q!8@+?$S-ownGuL%GhwD@Fv(PYc9>PlXVm-U`!On|3_oxBm-!-hhi3acSjemZ~ zY{@ax$Aosd8WWrx#|xyKI4@8H73z*uk#yRlRi$4q#e}`OEGdu{?{d~8PLtyLKyHv_ z!5~=)#r+`nd8-Mtq~t!?L!e|qH9+x;zOb+m#6;nz$3%wQE?`h>Y;5B8aId`neO0#a zZ+DNG$?H8qrMMY3I(rt2-)yP?sAbj`dec>Hn0wk%N*7<$uV*%)?e>+{iT!n4^yyY$ zA~&EoxEXem+hWsmO(HJV+;sTt9=)Vj*qx3ye3%eAX0E;579D-sJH>9D3mKd_xWzXZ=B9-DX^75UtUV6DQFne7lJN_Qv%D$?Bwh$sjuN(fSm?(R}l z8Wf~U1qtcyE@_Zlbax{Si#^tF@Bcpg!`^3~&*!~-uM1D=V}Rk?X-Pf z^K4@B2Oh?|^*8#h^Y0)dH}SIP$L35=a>EAJcv?fF8oly~Tlt>6&^!Su;?vomVhYdA z;^`4ay|djS4!O&+O_Iv^2z}jIVc|kNLFM~}z7oLKy~29@D>u4eEMF_OlmaXVZimk>7ve0(0kax3*iI45o8K=`QDIR$&O}P?9#-J<8^+m#(Ld4tZw=7FjMY0>8UYg zi1r0__Q_wD1h3C$*n;~x%f?;uwb3$+2L_9cbx4=Hp?^LOvTsR?B_=dnF46u99~&gZ znVHUEyXO7j`ZP6e31;a%a2F6v9Qs=^y^bnmk#LA=oR#!onQ za<9zAnXoQTEi8g8hnI(dY6f6bcLW8apjtB$s^H$1TC`78*wd5Bgwlz8KzayHOL(60 z-;nrs6McGuhlk!Ghx(J}P_CSKK$4ok6aU;*?PkT7DBoGdI}d2C?6)>fRoEDCYz;eD zK{-{+!t+5*k>H_D8ovAMs--dUedWIybIfu6LfuD>Io^ww>4;)QcD6Yc5#r?5faBL5 z5?U)mcW@MX6z4n%8s-|$X2}DT+fyO5oSqo}Otk$#ACB=3+Y)|Q^*uH#HcX?z*9{$= zyjg=S9jCpfx%hg1d1DSTsG#^|ue+}_^w-IRRu|#AR?yM%)@&8|m5LI_685~da`Uu; zw|;(7sVr1{t`W-bom&*dlzT{Q7*G|)U&M^CqMIomG9wAH7DDp;c87|&Dm_?z@bATX z)O49NY`!YHopqQz*R3J8s+E4b$Cw?CwAYLf28nFNNr((-SBwUxNQR zKWE4W_6aD{KAkzW4(54}LB6JIJ{l%+?>fG)cr;PEaX+#!q9-abyHwDqtHAGg)&!O} zysTWu=T$Dg?s&%hL3PBYF+;HVxghFq{rD}^o~^ zvStIG$GC^gFTM9&-5XRc55DGCdP7&J>MwkM?+>-JwV^|hkbNTvxgd|Ko_H{LeeVbC zqqtOPsKchmj3Z zCbrSLE@%I=BGeP^u`maH5=#*w_U@7O4JK0;z$6JtwQh|ia8YnP*zNI%>JU zc_Hih6=w`bpQA%V@0E|z(_!3>URbrOj~+I1YUfoG;z>TXbE9aRSHV9#sP7$U={IzR zGud9{$@=}5d2>mz4}&vF9vwzZ=xN9$Gb9TudCJV|LKBC>MI*HZY8o|;?*#7KTFm&i zM}Dd@!T1lXYQNZ!h35LkQ6?uC79lkX+*nqp-egi}1;mlp?AmfIcxwzRZVVGDTDewr zLiz?8^>EB~j${0{DC13k5N=AV{Zc^hy!wz^ z;i$Qg>y+?-)A^+EO)}L{jvw`8>5omO^O4T@t0d9AI5#nz8Lwtw?ml61qIA)*cP-HV ztTW88G`&OWA(HBt6g9|fdk?nO;fdp%RgkW11|s3p3n9r&>5I;8?m+E@3s%^9VZcj? zb;yF5<2X7vnY&0e95Z>t>_baKD-(n6>#g;y;u4PR$(&YNiDlxLmKy5zTFGNnhydoY!dCv+xVtcjFPU;LPbhdz$ag zcX!jUbj)ntw1dfHSvm9q2BnJ9_XTe4*_WK$)CSKFKSq`wy|3abcau)>YAv-KxJHhe zldAi9)>x~Ka`TC2%T8T2-#6J`!m>&(kGl+Qp>#=*U{CaAbYPy{bo=MMe8cL*jy&w5 zGGN5WYIzBtedP@K%uCbxBOj9E2c#2lJ2t&cIk4X6YJEzSqkELrvJoKZ>z5Ua$&p3w zQUeceHS*%|#f69f7FT=0^P2?|^54WlxJws{F&L%^OBE6fH&(E6BaTGa9I)9^jd;Bd zIfoeo=eIdrJINEpP`w7cvs?N<h$-VL54&CBbl$rmJBuicmpamz}%EMo?fQZ!{)l z=j2{A9(+Q7_KaI6MWriEz)f8a6WHuydF0A< z_J30*{ zU!k$EhlDGop>eAkf+3tYV87VUH zK4r+T*;e+#@+>+uIXR;|Dd4Dh?!YU=70!WM53Cz2Df5gQXH*?>%?I_f8CITl#qyIx z8#9i$-Noqc&Hf&`D(qXdE}$(RANRgEm-wMvRLHiPmOlGTyJx<6)NXmFig8(d_lum6 z-Q!ms4XG>-?VS2I_uDX}3(7FwS~{`jNG=?^zLTY+)~INEzch#;UPh5t^sK=`f@1q_ z9Ts;wRg?w}`XrVb*)oy5KFKugi6y~tZbsd9r|iZGZxNrz1Yz2Bp$=&lZE4pu1zRYE z(tv&Z)ausjSXm)a^RiqddVysgcH+jeW zdF?LBJMQaTYR z!%v{77mZ9z#^4+?JU`p% zrSJx}e8b`!123sL?8wcUETw!Ft8;rOoM{7O=b)U`x-`LR=JJe-B!1|cpIXzU1orel zjH)4%&8)xT)^y!gLR-$=ysZ<7Y?31Mmn{fe9^|Ob?%Vr1LL_n1dg2x5*cc2OG)w9z zt5%Wu_6>#UCN@3B0w<}`Qa1v7rNUU9tYs0r(%-FDlUp#()qV&$g<(i4R#5vS#VYWy zP*?@jtkdB2X9O~~Q>%JuIj+oINo{WEy?_)#mp311r~Xtan?TCHf+=QKCZ-Dt=L=or zns2*>2SxbL{ps|p&pW)hfXnhIx>Bxe>148Aq_ci;9810HmoCMoAW6NkyHI_%{*tc6 z-ka^uO1*`=V(y<3^%Bj=s}YBPeY{8W(p|^O=(fagbKP;ze^c{Sm7TvTe*f4~;4_m8 z4%K49#&Ea*kxh$V>nQTmd;%;#0BM^lo?rg(vNw3R_<}&Mjc{ISKknVkVX|&B-cDZ9 zk+9m0mE(+Em!b4fiWCM%?3LNt=-Kw^uLpXh#|2?8|A{Q#ImTH|?(*sXN&W(xH$rba zL0q{vb`RHb^eK(iM9{Bv=|r!Q(t zDConL8{j10EWV$^Yx+UPYp!jtp>dic5!@dH^V0K|^~Z^-+;3mzIXzkdepR-J%e5e6 za}B+@-E$|b^1TEcZpf^Pcq$jPT7j>tWUjjr>p=2~OWde917;yZI2tpG*nOfjvQZgY z#ASHW)S2&~{2;(oqzEl-Mt}1}j8H{j8bi)5ahT-kB ztk@wq1x%LPSA-auHPnwq6=_4;+74n3+!Fn?jOL?A0w%jhVr+N~M8#_c%2frkx}n1M zFgm>5p_F{5&a|SENoY%=vE)gxVV_bdU4Z+qx2>d+Yt-}j4gJ@IvsF0mpH*)eq|)o5>htm$O?n@5MqNzjy0aV}B3W2xq$N78$_AbPS|+GA-x zAA1}jo~DslXjjUB3?Lg1R=Y}X_Or_Q zb=y+hF-SbQkCHoyo?FAyuI@A;wuEu-+LYBlQ=`F}gd&b@mw^xH<7lbeK&Ew);YAc% z)u_v5)El1oeJ-KqT<=?axc3Kts0J}$Jm?6GUUH`KmcPRwCWe!8m+r0M#7HG<@wHQ5 zak0tVwH9j=ksx~|jGhD|a#LnM$#BouUo}-NM~Gwp==N-#hcFTm`j_;N-3^1ZO&W$C zm$-Gs>4MCm@z%H&WP9qQmGQ7QTlt~4f+_0M!|L2Xgt2?NDe3#h8;--C22|Z?0xC_4 zw$?kY{2udFp2VeZngYM4&!xAhDkTVFg1H)2=v^u`T=ps}2YKkzTYt83w`{QvZ}qyT zow*9T2QwKJ^?$m46QzmD<6xhZ-@%Ahl2b5tJH+`UI5KJG5O17D#rFfVN48-sZ)SXb zCSoan!!t$RN)TE(0+|yro#W5Mv0wd)J~O>2qHa@e&v`n#dLGr|oQeB`p2G*%0n_(u zHnzq2ZXm&V=$Z2s8E0x+;n>h5@AzD7qa~NRb~J0c)oeBgO5L{$4!eGzY4U@$G!Wm zN;$t9!0|=fgD-`k*gnm68*!dHdYHbifA1G$L;q=U^!ghkK{G*EDM@hSLH*(<>_Jxp zUU%#&(yzLUJ$WNL-Fw)h7EXe6f%8#ZYJ7Hk6rs)3lXwj^SZx zyXm4&y%~yn$Te;Use%{1$I@bIFPKfl;avFK`=;}kD&=6~+A{)&66;1YAs*KKM^-gy zuDMScLdoTcI6r+vxay3AQp=T<=m^j_)phniMshbts|sL+{5qqAZIReNFY#v)#oHAuo^+S?uKq(_Ld0;o z^i42qc5)-_9I5f6Fe_ay-zdQ^Pe zQG!Uj^7t8S_|Rg<^`$&A)ZQE)z1ukMf`Rjynv|7@6Su0{uA=EPpeS6Rb5)H$5V z{C+2SGiHLjCeF;EdC~Mt%$;&kL#=V$vj#b56h8OfwCVR+cj{$g_IxQCDvVrAa? z5Tj_U7NK$W2E`$IM%l+G~S}IKStC)<(};)T778iS!!hF5s|qzF$YEVABOFMMWw- zLV?xXGK6Yh$1q%)*(HD772n*;?wWKu8_hg7*Px@fZh-AJV_X^efk(&{=}%?B8cp%6 zi5w@`UueQCm;MxS;Ebc{y-vB)s=Tn1JLp0bW(%eTBMgfMAUzc#azISIK^J!cAMaNgwusC~<%9^>#-7ckns{I&Sy4nDl6A^s-zf$CAYFq32jmoGanoFPawTe)FlULwGs|eVQcw@`cWu6;_ zt~C0auX8)U7HYj=7PU>23x~u|Gt%|E`!|{7NoW5NB86oK<;ra!8r3K0Ay&+hcVyl* zFL7;U&qBO@g+{DoTHViOzjsF?zNjUqO5(;;?&l#TRu#;@mgf zYOO5dk4PBW*<3QR3btX-^7sz@oD*dVfwlEOIwq-g?^H`dV;84HI+H`|kCIda0}-pI zs~V8zT(6vUWB07N?9b`C9>!9t&|-Z{I4eY@lt*j5A-i9Y8}@CwlBDgAG_T$;5SRHm zNib00nCgK{{Dv5JU0DukW-;!|j+5G+9-H{eJvk-#EtWagwrhrdiiOabQX#h!*bJg; z6FH;K;WBIlTUfc#i5LuGiOI@WC>eL3X=ys8XA@$CGgsY5QHF+uF6F6m+@U&NSyixtfA(DmC{#v09QT$Mh z27z%KssU-RBkS^Yi0&Nfr64K4y%2d~)1bX|>~`HxnMEu~3Jw!skAw%>#QC)MSMjqp zBi>SDHlL6)+M|OrpR6*T2lJz?w*#`o(HfJ-X)np8lGZ1`Ko+~&GM40{R-I!3QBlE9 zZ}ygIIrYjb$abb~}Q^~;(vGG|=_Z!fb6e#gbQL`^Hw)QR@mnxBq|pL*7Q zI7ZG>(H&rNj>cGRc0}wt_d7Q}gUsxO#>c_&(}sgz6aqyxxonRt6g9ou%!GD79Pmhs zO?BLafus<3KGKeE-B}`&tSAn$ay|HpuP0emkXGZ}s_k<7$v(ii-aId{mmHA5H@w&T zyiLBl7>bjK<7y7KD4&oqu>r8mgc2cdz30B<33%0FjC%p;(c2WH6`s;+@^)`%`M0ic#BH@5sG)azPt{uYrVYvJt_qExewjYAmW`8J|m(Pjlx3 z9nzXf^_Q*Pl+te#CK+yCdtXwxj;VbjTn{;n9u>`0MQ)H7bzE5mC1B#7lJ6R}*Tc9C zFNkx`Z0<#j&^K5KJ;N>$*lgvHAj)$0>A#IXvxFORN)tvD|7l@@?G5$((aKu~H#AQ(|)ENDm zwbH9A4Ew%Hi_|5Bma-6mrJ3qo8%50?;rat&xz0CG11CpmBxzUqo+}Omb-;bzN9$E3 zvE8+F!u6gb!ZAwl+$oL4zA*SSkR5dzuJUv>bVv#gMH03dNowM75P}V`U*=Kk!noZv z)uN(O%Ds~H2Au!~r>&P&HIHjqwBXTfPy9vKBCaG<;3YQ^6vV$1>0f8pZm+)KBX}ky zRB733+$|d4x1eYu+IAptGxpTxiILP$q#jBWGsfCd zM*38Qt@9pN$voG*XG9Ph#AK+GliRmzi^!h-bG^^iVCs!>p!6>{IRcjY>geFuwxB6~ zFDth!#CcLsOV_e&g&tn#_)CZ^vFKe`PIL`(dg#yN9AF|GTpd zL+}35-^4!Y>&U34^VZA7yMwY{%CDx#-mHKBcxSW3n9X%V^Q!xD_@_?=_k1(wwjUXh zrsNt79JfsT>%WbfFzCA38>?U6x1DeQUo=smGQ$k1DH_J@E__}RQ` zVlsR4@8{1j%Baw=hPkr*X2#o{R!iR6%KHt8n|qLFa(dS2DMj7hTyW6(N=u?A3#on; zwl#AzSD)ti-zyJ)Z1gMr+JsO#SywPe+pbNC&GX4;{5eZuixb18A=B7S8DkwW_@1XX zJ)E)sospdt9&uwz86B_==im)|E=$o{+%iCIb6}|;OcP3DSxQx7oZvHa*uve@(Ar^|5yT3_l*w%<#HHhtW&n+Qqe;#m`Db#$ji1@W-nc}}fQ zsOFC}{l+g+(kooP%9K)aC=Ri&!_}-m9T#lq!m=^sHu-^j!z2zhr1NczfZbW1ovm z+#QeDOq~p)<&~N%njI0XOpfKl8}@9;eO9x|=*lQ0nA2$Nt4WS>)Cxqz&s42riix?Q zx_pqiu)Hj-|-%8h0_v?Hh%0?7xD=BLIMwocG zGEHZRK()xoam&yZCA*`t_57IUUEY=P%*KNGAnE(9 zoM$7W8#RJ*I|k=zfeq_lmkZsJmut`q5)-cwxh#q5 zMHC?e6cy)nEl32bcr+Sc_5Ib2zQw~40crUf`FwHLiEWz`o-(sBZWv8Hvmbb5_K&uc zo}Mc|n2ySEBTY)zfkidrdRR8bQIJQ#_-4Ly(n~KkCm{@;LyQlKyRG|B>Cu%TIX#>? zit-HfeI&G#bQX~UDf zmAQS-_RJ{j@s_B~!R`6gnwW_i0y;kzWVVK!^FlRc$jHPwC)GazNVM)C@V;o9L|WCL4dL-S&;l^3Ffl6ebm4)4FklioASm9Qu)?j*3*tttR$T@~h@k z)T&eUz)z8_SH5o&Yk$aeE`7vv>)+mwgoC_4eM$nZh=&8`VZCR!XCjPR7ZSz1edRn) z9A+gYwaY1egsuoKP7w*r3@qLBzH@0$n2g9a53a3ygg-;xe??yQnn4i_Df@i^VKJG4 zkL`tH!m-~Tz1F9BiJ$3GRn=Fbz1T>fKyyRKk!LvB(lM`~TK7@5#?DcSF&n=Np8onN z-Nmq=uF0IxM7#LTj-OvN4ZdM$h*V>Cx~-x|#esVv%?ZE4!{OZfD{kv$EpXh!53W(h1r_AN*D)jNU zH_55ucs}#hIzmQw)5zqfU%dz4e}=iwYA?(V?I4^gqL**@WU;}~p|jEU+&DQgMm5iO zd>IMV#<{WvJ`4|;U@FfKA!nb3eCu8Yy_}wYhLPtl`SA9uT>7{85vN#A<%{bc#=rH8 z8m7aj&5?2Mehgeignux`fV2|>qqLW%V;rq9?K5dR^;jv{ZE;+|QG7n%S>-sydmmC2 zOx?YL)4cKaY2RS{)3U0+JQsXQ_J@~xP{k8k!u}VQ>`)LYaY@}CG%dJ>J-mMX=-gIt z>M|{fgKTp4Wy+ge?zynWJV>MC+X&Sqnvv&yH>0_tOaOja7;dWJy9Y zgUk6yclLxg32c_a4ikE+f`s-Zz-YtbEv9sr3X;s4ss`tV-d6E+d$q18c{><3JZ@8& z3tqVF!k=I63+>wnd_%kXmd&ad#)Bfk<+|2h?uC_goHW@lx<^azgN;W`tpDr_3EnMt zT0XH5Jg!aWQ7Rm|NwxE7{$AB&PnEsX{=uFvM)g!4&rzaUD4WzNN)^lH@jpzGuXffA z=5h*b57WQOJ=#dHML1Qx3^dqR)^&O><|X)6uP552kqSGSjsT9SNq(_d)Al-28!CQ3 z*%C#N$TZd3Ea%Au4M5(Xx5`s1hb!|Bn2{jz$dSfgMc(z#Jz$@o_xkD@`X8QxoJH@3 zKzNrc7xntg5_c69d%XjL>+BnKB2Kum)*plHgvS8%wg zCSB&bAiZSwRt6tr;~dW=4Qs{s-ifX&qca{Hw84Odqz^IueJaEfXKA4PvjmANvq`xSNCJHOxsPhQ^BS8=9~#QYiFAOql0%ym`CyCav3iM^by&Nhsk9j z6DV~rMs#_ba;9mEGzdrvu)G*_$*6m5H#_?_HJFh>V|Z{BsYx+$h&{N5$0&qnjwfC` zTJW}m<*brc*6LHBW5|$xy}Dc8g-mPehbH#kEzK&wuiuC3h^qlyJhqvC$~P~OKhBg| z@%202aCNmyZeOoC(e7GT(K?;lq2qGf*;s$^+lB}?FLmk$T4YviwO9tVb4lLIV*=JB z?zb0$mxNC2)J_Ck#*@l&MefHtp;5+d`9`T!_wWuo>~;^HEGgd;q$8Po9m0V!*?hGs-=VrcXY5=_{V4M5u|)Cl-n`^d94kSwJ4R)g zNc;@_w84W&Fa{zQ1`Px&C1|H`+JtCZ{2J~GPaPp_YMNPyf<~d5%f+_U30T=n=z=l{ z?(O^R%V)Y%R<-JO7)AJ!65sF->iGmE#nJq&Yw?Gzs3+$WeKT3UG+>+irvz>kGR&Vm0;If;lScl7c z7E#Cxr!!2a32siGo^uj}lUz~R>d`jZiTu#EnkZ@UAi8Y%a&LHmuc|&|Te|8BS3(zA z3alN0FhWroKLnH8VVl!hFf6dYW%Y%@Lx!&3}!GGX0o2-LlgoJJr6Yh1r7IaRrH{@uB*i zY_<0u?7a~!;h#NcbEmIeh~6fBwYtB0ZBB*PHN%{WYcj!EJf*fS5<{EV(??eJnr;y_ zd%o87PALchs1Z0-QqGRKFGG==PEZpPt11RNTCu9Ma2QZM5$Lj;e?bsJF%Om3H$LTC zz4*oQ+S2sFw$QvDLb}`E@uvEExKfR-G5?JVi|pL}`5h-$*7}wI*^8T>&fRAEzBHu3 z`^yFT;PjSHYQ}l;HVRAYY|b$8v?O-?#I#%QzFQTK@oTjY9=)Yx>u0q8xSl}0Y4SC_ zXQ+-3_c?SI@XFVnR}RGo*9{+dUuo^y;7s8>8$?)}$u`~8K`({f)>JJ{4$PwwW|-Aj zrv%ba4eXEHY!z+Us+D`W!=5j-M0g>o`4~1Xc75Y`gsYlEYU_UZi~WPTV%c1ucAVOV z$9BIfAJtYvRY%qBUYkzbHLR#7|3Mq4c2yQlvXGT>KH>L)_x=Byf4So|)^Q7^&I?dF z0YDP~WZ3H?d37@=@baO#h=7CX%huon_0K!SuTU!NFRTps*L0=C4ltCVr8HnI;sdDb zckuQAplPA%5?~nApe?x}zJsAzYM}?jdO|wkFJM5Tv|oU#V6bQ!l#Orp5dFN1u7N>o zazXBc4P+;fHZB5wO&6$Iu7`E|Jz`Ill#GV6H9{@v#hg;XUZ@CS!51Tn|EqB;a0y*i z4R!VZadgIf{n`L%N8mc9BjS-3Y2ZKup&08l0^TLU+}j6=Z79p@;05v;vmj>eQ z5G4Nti(vqE8UT0pC0Os@zdwDE^7H3^FR%EY{COrb;C2u%-vvl{i{bh4E}Cs&1tz{8 zA~chEAsjTO`GDO4Wdrcy?JdJXLqRuoXBq+3F`TB}KK~0l{$Gdhf1I>N|HlmcZ`(Y; zZ|NwJst29UwF$=h|4PA!EBN|1O)(gtU*Agu(7<|6LNK_Br6|^rI z3NNnAWTq$m1kkC+@vPs3$9^}+;XZ)~4G?J4!uy7BCji6%InC8@w*x{#7J;?Ga{qoH z)N$}SZ)vs3qDgDIg2|RU8$exz%9JG_yDUL#Nd@NWImk=+Eyo5mxX`ffRP}GMqCP#} zMWx{p{|c+E>-<9nTnnhX7yl1`;QzahyaVWQ#((bph>3|=S&&D+b-rC~S*QIy1DFzL zz(|AvTwYNTDCq}fF*L+RC@w+j34c<>s=yMA4*@vQ%=&r|h;skgLTL>ATRVW_+G)O@ zK>ic6;jFpr`E@jX0zc*gyVh(lqX9%u_*bvmfWY<#MLoN&V0fVivlKfPhI%ES1+3+8pTWh;(#xpjwPd z*oMjdaHG9M8T6FQa=~aH6PR^uz`R02PrsS!-&qQ197;$(1IHTO_&=16C-@MI|9rXi zI$NDsSy>r;4Xt1kqVEM(3L20LfaDM9>1<#-Gyt=XQgHbOXz$R9a&U(MnCL0AM;8!8 z+2=@Q9o4No(5X!TMa)a!_9?QGf*Zg&cbQ;cyRD_g7NBJ_U{ikh5Cc5~fUBhC_&zB9 z>h%SovIMPv2Cq{qpo&aEEC8+uR#w(lcTD+EH{LnI6L3;`U%P*uUobiGdqxDc_^c)zx_Ld`O-j0cqo^c>SpIKY$07cqp zg*_Xi*YPcI+Jt(7Xw#EbF{U2yNMN3(2X@rIt;u*$QnATt3swVy3a#-$lN8bLNce3S zFq)EkCC|DF!lo{NMkFLCf(;Y>X!bm7xwj8MeBA^Z4RsJnqis_sdyC>=q-;4L35J*&o(-PT}%k$Y4pT)opsaA=-ekygKKDHtyS zPD2!^>4i&yrZo=K=1rh+yW4(LPn(4Peh6@0sgY7(Q6sm(+K7IGOEatuKBF$1bFG*o zH(*o4<>)oxwMl|ax*%SW^$g4=K^%v3KPl?&wvI9C};n~?qZ=ebNxRbgCUjD2kmsSTYLs2ZtRr{E~QVdc_Y~xGSIMK%yZlr zC^A+CeI_87*big!1ks%Eqy%wS5+EO=B_f+35kadypTbRtVFB>c{a`8uj_75)IacmlDzn zcI`?oA9;s+T~V`w`3lJ3f8pzck5)A+5-e!(?pqaC9&UT{-0yOy2V=*J7cX*`(Z0hM zKcr*NJpndir}sZxB_LB175ip451%4_FsSlCJs;ygR%aDXq=g{3_c}p%kNptt8QMDx zgPXx5bJRXz?7YfsvshF(suMt2g5_jG(Xa@wAp>#d`3_L!mLF@pb)~ zDr{_Ws)7)UqOTU{dg|3KJOBW!NU{dyA(Z1)crn7zhsK+pdXl|It2?1V5Hm1rm6re_ z9d3QwE?nu*5R~VL{|bZNqX1+%%&P0k0B5KzXBnW<0$MPBmLO+|71e4(zFsL^H zgqJVD#8l5fPQJL&*OAc1;NAt=VQZaAhfE3Hz02{cftZY3L86&Uf~(9j*RP8DfMt;4ZmJRp*syJ<_?mqP}#gpu}rju`Ibf@lm|usAuLyCsY&<@(@rw_>!N` zE#h@vo*?C|4e&c?rLCV{I$J%j!+p?r;XVCKL7}CyQ=zb^Xc4|w!3-8T6x|$iOIskp*s_-ND)$V$(@&u+NM8I?Y47bF`6rgB1nA5VDndMXo?iiS?>}kf{IWQ)6B;E8 z3(Moi#zxunfv4~V3=h_Z_U2j$90`x==E*22%Z`PnEd+z)*AGu2Xbg{uxeJc6zd4#h zAfiqJZh13qmg^@=c@L{OXP=qSv&%cD(s+l-ObM>aK1%$bwy@%tA2 z{$fU98;ro8<#+Fobo}|_qn!tPDI@2M{b}0U;Qu68{#A_CjD0mIwd+rZEeVq#2)npdW3 z!~l|50FnZu{gJ`JnWA1vU!`;z7=4uzgnz8&{r=quyvEt(ZBe!YQ;Wj%v0le?d?rsA_w=ZA6PCaA~{7_iP3!nGsB^g{%tt05k zJ%E6meGky**7TR0c}JXLVze;mpkpPB88CeM2L;K1D3q9r${%E)aCaT!9MjX%%$DLL zJw=y#60?Tk3w?=*@CDYP$6h-d3>h+BUbRaxfB(LMn#>gkQIG^ekh#g2 zX<&d=qwUTfal*`0fg;nOr*58kk^QBin)P4A(LmYGY9%|SQ= zjie3MAvVW5dg7j^WN>pqLR(*9znt&9t&Mg)!$@lzzpP#7RTDSL!p4Sn!RL=}V)#^6 zio*1W3%*uq4Gn5AHA<~i2CQHc-~M^Y6W%{ee)stgamRcS_ z;a`Kf+20@HK~>-lWQZIACK1O66yIz%_3y>a8py0A*>zECRZJvf&7+*i%VHn7D5?Th$vw+%nNPr z>?Fm|FG9Yp!MlB0?ph&uapz}Pk;CU#&ya1QTL2 ztcC^JTAG_Pl547~x%Hc`zE)Gi#(;(A3oR`|sI@j)&dSMY@xjE#z<^QK+t>oaM|j>s zpfSxd4GRf@mhqTf?YInLHDo1P}_DvVqxAa9{h{&j^ zL1ump8uM#Q_Ivl<`}z5uvV3KbaI2rS$+!p!3CZ~V+ic(UK0EvF`H6i=&gucv&z|@A za5-<_UK#Zy1VH#|I@T6K>j#|W8DN&mT3T}0+1u|zRN*x0YHx2};C<-@&Qo28P+WFq zFvG*cF)&~u&T)&{m3^k$d;TQ!yG4_KG0wxnDYDgwU(9^bX*2oEjf~?5E!cT^ln`v zAb6>w;tP(FIe>3i2JQpv7v>6d{JaYbA9M_)4piazC8nQYj40s{kWK#2;l)4HbNy)kgvU`Dj3-UE2PXI9wJ!9fdrFWXC$ zXE2=z2`nz^A!N)4pueNyt5+ltqyxIb7Y3f6*3-3AQeGbMbN}}DWuP^1YL$aj<|8z( zvg7hZf7B$I>;5NV^vt<7=yK)%I(7bEX4e0gzuspwG8Nub?f E0l(Nx6aWAK diff --git a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/evaluation_plot.py b/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/evaluation_plot.py deleted file mode 100644 index 38429d002f0..00000000000 --- a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/evaluation_plot.py +++ /dev/null @@ -1,47 +0,0 @@ -"""This code takes the evalution results from the directory results/ and -creates a plot to compare e.g. fedbn with fedavg.""" - - -import json - -import matplotlib.pyplot as plt # type: ignore - -fedavg_step_number = [] -fedavg_loss = [] -fedbn_step_number = [] -fedbn_loss = [] - - -def get_evaluation_numbers() -> None: - """Open the json files to get the evaluation results.""" - with open("results/SVHN_fedavg_results.json") as fedavg_file: - fedavg_list = json.load(fedavg_file) - with open("results/SVHN_fedbn_results.json") as fedbn_file: - fedbn_list = json.load(fedbn_file) - for item in fedavg_list: - if "train_loss" in item: - fedavg_step_number.append(item["fl_round"]) - fedavg_loss.append(item["train_loss"]) - for item in fedbn_list: - if "train_loss" in item: - fedbn_step_number.append(item["fl_round"]) - fedbn_loss.append(item["train_loss"]) - - -def main() -> None: - """Plot evaluation results.""" - get_evaluation_numbers() - # pylint: disable= unused-variable, invalid-name - fig, ax = plt.subplots() - fedavg = ax.plot(fedavg_step_number, fedavg_loss, label="FedAvg") - fedbn = ax.plot(fedbn_step_number, fedbn_loss, label="FedBN") - ax.legend() - plt.axis([-3, 100, -0.1, 2.5]) - plt.ylabel("Training loss") - plt.xlabel("Number of FL round") - plt.title("SVHN") - plt.savefig("convergence_rate.png") - - -if __name__ == "__main__": - main() diff --git a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/run.sh b/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/run.sh deleted file mode 100755 index 2a291781419..00000000000 --- a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/run.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -mode=${1:-"fedavg"} - -echo "Start Training with $mode" -mkdir -p results - -python3 server.py & -sleep 5 # Sleep for 5s to give the server enough time to start - -for i in 'MNIST' 'SVHN' 'USPS' 'SynthDigits' 'MNIST-M'; do - touch "results/${i}_"$mode"_results.json" - sleep 5 -done - -for i in 'MNIST' 'SVHN' 'USPS' 'SynthDigits' 'MNIST-M' ; do - echo "Starting client $i" - python3 client.py --partition=${i} --mode="$mode" & - sleep 5 & -done - -# This will allow you to use CTRL+C to stop all background processes -trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM -# Wait for all background processes to complete -wait diff --git a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/server.py b/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/server.py deleted file mode 100644 index cadb6713a24..00000000000 --- a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/server.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Flower server example.""" - - -import flwr as fl - -if __name__ == "__main__": - strategy = fl.server.strategy.FedAvg( - min_fit_clients=5, - min_evaluate_clients=5, - min_available_clients=5, - ) - fl.server.start_server( - server_address="[::]:8000", - config=fl.server.ServerConfig(num_rounds=2), - strategy=strategy, - ) diff --git a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/__init__.py b/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/cnn_model.py b/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/cnn_model.py deleted file mode 100644 index 99bb80a3d11..00000000000 --- a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/cnn_model.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Define the model architecture.""" - - -from torch import nn - - -# pylint: disable=unsubscriptable-object,too-many-instance-attributes -class CNNModel(nn.Module): - """Model for benchmark experiment on Digits.""" - - def __init__(self, num_classes=10): - super().__init__() - self.conv1 = nn.Conv2d(3, 64, 5, 1, 2) - self.bn1 = nn.BatchNorm2d(64) - self.conv2 = nn.Conv2d(64, 64, 5, 1, 2) - self.bn2 = nn.BatchNorm2d(64) - self.conv3 = nn.Conv2d(64, 128, 5, 1, 2) - self.bn3 = nn.BatchNorm2d(128) - - self.fc1 = nn.Linear(6272, 2048) - self.bn4 = nn.BatchNorm1d(2048) - self.fc2 = nn.Linear(2048, 512) - self.bn5 = nn.BatchNorm1d(512) - self.fc3 = nn.Linear(512, num_classes) - - # pylint: disable=arguments-differ,invalid-name - def forward(self, x): - """Forward pass.""" - x = nn.functional.relu(self.bn1(self.conv1(x))) - x = nn.functional.max_pool2d(x, 2) - - x = nn.functional.relu(self.bn2(self.conv2(x))) - x = nn.functional.max_pool2d(x, 2) - - x = nn.functional.relu(self.bn3(self.conv3(x))) - - x = x.view(x.shape[0], -1) - - x = self.fc1(x) - x = self.bn4(x) - x = nn.functional.relu(x) - - x = self.fc2(x) - x = self.bn5(x) - x = nn.functional.relu(x) - - x = self.fc3(x) - return x diff --git a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/data_download.py b/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/data_download.py deleted file mode 100644 index 5909b2d410a..00000000000 --- a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/data_download.py +++ /dev/null @@ -1,67 +0,0 @@ -"""This code will download the required datasets from a Google Drive, save it -in the directory ./data/data.zip and extracts it.""" - - -import os -import zipfile -from pathlib import Path -from typing import Any - -import requests -from tqdm import tqdm # type: ignore - -# pylint: disable=invalid-name - - -def download_file_from_google_drive(file_id: Any, destination: str) -> None: - """Download zip from Google drive.""" - url = "https://docs.google.com/uc?export=download" - - session = requests.Session() - - response = session.get(url, params={"id": file_id}, stream=True) - token = get_confirm_token(response) - - if token: - params = {"id": file_id, "confirm": token} - print("ID", params["id"]) - response = session.get(url, params=params, stream=True) - total_length = response.headers.get("content-length") - print("Downloading...") - save_response_content(response, destination, total_length) # type:ignore - print("Dowload done") - - -# pylint: enable=invalid-name - - -def get_confirm_token(response: Any) -> Any: - """Conform Google cookies.""" - for key, value in response.cookies.items(): - if key.startswith("download_warning"): - return value - return None - - -def save_response_content(response: Any, destination: str, total_length: float) -> None: - """save data in the given data file.""" - chunk_size = 32768 - - with open(destination, "wb") as download_file: - total_length = int(total_length) - for chunk in tqdm( - response.iter_content(chunk_size), total=int(total_length / chunk_size) - ): - if chunk: # filter out keep-alive new chunks - download_file.write(chunk) - - -if __name__ == "__main__": - Path("./data").mkdir(exist_ok=True) - FILE_ID = "1P8g7uHyVxQJPcBKE8TAzfdKbimpRbj0I" - DESTINATION = "data/data.zip" - download_file_from_google_drive(FILE_ID, DESTINATION) - print("Extracting...") - with zipfile.ZipFile(DESTINATION, "r") as zip_ref: - for file in tqdm(iterable=zip_ref.namelist(), total=len(zip_ref.namelist())): - zip_ref.extract(member=file, path=os.path.dirname(DESTINATION)) diff --git a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/data_download_raw.py b/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/data_download_raw.py deleted file mode 100644 index 369c943d788..00000000000 --- a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/data_download_raw.py +++ /dev/null @@ -1,186 +0,0 @@ -"""Download the raw datasets for MNIST, USPS, SVHN Create the MNISTM from MNIST -And download the Synth Data (Syntehtic digits Windows TM font varying the -orientation, blur and stroke colors). - -This dataset is already processed. -""" - - -import gzip -import pickle -import shutil -from pathlib import Path -from typing import Dict - -import numpy as np -import torch -import wget # type: ignore -from torchvision import datasets # type: ignore - -from .mnistm import create_mnistm # type: ignore - -# pylint: disable=invalid-name - - -def decompress(infile, tofile): - """Take data file and unzip it.""" - - with open(infile, "rb") as inf, open(tofile, "w", encoding="utf8") as tof: - decom_str = gzip.decompress(inf.read()).decode("utf-8") - tof.write(decom_str) - - -def download_all(data: Dict, out_dir: Path): - """Downloading datasets.""" - - for k, v in data.items(): - print(f"Downloading: {k}\n") - wget.download(v, out=str(out_dir / k)) - - -def get_synthDigits(out_dir: Path): - """get synth dataset.""" - - if out_dir.exists(): - print(f"Directory ({out_dir}) exists, skipping downloading SynthDigits.") - return - - # pylint: disable=line-too-long - out_dir.mkdir() - data = {} - data[ - "synth_train_32x32.mat" - ] = "https://github.com/domainadaptation/datasets/blob/master/synth/synth_train_32x32.mat?raw=true" - data[ - "synth_test_32x32.mat" - ] = "https://github.com/domainadaptation/datasets/blob/master/synth/synth_test_32x32.mat?raw=true" - download_all(data, out_dir) - # pylint: disable=line-too-long - - # How to proceed? It seems these `.mat` have no data. URLs found here: - # https://domainadaptation.org/api/salad.datasets.digits.html#module-salad.datasets.digits.synth - - -def get_MNISTM(out_dir: Path): - """Creates MNISTM dataset as done by https://github.com/pumpikano/tf- - dann#build-mnist-m-dataset.""" - # steps = 'https://github.com/pumpikano/tf-dann#build-mnist-m-dataset' - if out_dir.exists(): - print(f"> Directory ({out_dir}) exists, skipping downloading MNISTM.") - return - - out_dir.mkdir() - data = {} - data[ - "BSR_bsds500.tgz" - ] = "http://www.eecs.berkeley.edu/Research/Projects/CS/vision/grouping/BSR/BSR_bsds500.tgz" - download_all(data, out_dir) - - train = torch.load("./data/MNIST/training.pt") - test = torch.load("./data/MNIST/test.pt") - print("Building train set...") - train_labels = train[1] - train = create_mnistm.create_mnistm(train[0]) - print("Building test set...") - test_labels = test[1] - test = create_mnistm.create_mnistm(test[0]) - val_labels = np.zeros(0) - val = np.zeros([0, 28, 28, 3], np.uint8) - - # Save dataset as pickle - with open(out_dir / "mnistm_data.pkl", "wb") as f: - pickle.dump( - { - "train": train, - "train_label": train_labels, - "test": test, - "test_label": test_labels, - "valid": val, - "valid_label": val_labels, - }, - f, - pickle.HIGHEST_PROTOCOL, - ) - - -def get_USPS(out_dir: Path): - """get USPS data (handwritten digits from envelopes by the U.S. - - Postal Service) - """ - - if out_dir.exists(): - print(f"> Directory ({out_dir}) exists, skipping downloading USPS.") - return - - out_dir.mkdir() - data = {} - data[ - "usps.bz2" - ] = "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/usps.bz2" - data[ - "usps.t.bz2" - ] = "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/usps.t.bz2" - - download_all(data, out_dir) - - -def get_SVHN(out_dir: Path): - """Get SVHN dataset (Street view house numbers)""" - if out_dir.exists(): - print(f"> Directory ({out_dir}) exists, skipping downloading SVHN.") - return - - out_dir.mkdir() - - data = {} - data["train_32x32.mat"] = "http://ufldl.stanford.edu/housenumbers/train_32x32.mat" - data["test_32x32.mat"] = "http://ufldl.stanford.edu/housenumbers/test_32x32.mat" - - download_all(data, out_dir) - - -def get_MNIST(out_dir: Path): - """Downloads MNIST using torchvision routines. - - Then, move processed files to directory expected by - `utils/data_processing.py`. Delete the rest. - """ - - if (out_dir / "MNIST").exists(): - print(f"> Directory ({out_dir}) exists, skipping downloading MNIST.") - print(type(out_dir)) - return - - datasets.MNIST(out_dir, train=True, download=True) - - datasets.MNIST(out_dir, train=False) - - train_file = "training.pt" - test_file = "test.pt" - shutil.move(out_dir / "MNIST" / "processed" / train_file, out_dir / "MNIST" / train_file) # type: ignore[arg-type] - shutil.move(out_dir / "MNIST" / "processed" / test_file, out_dir / "MNIST" / test_file) # type: ignore[arg-type] - shutil.rmtree(out_dir / "MNIST" / "raw") - shutil.rmtree(out_dir / "MNIST" / "processed") - - -def main(): - """Get all the datasets.""" - - data_dir = Path("./data") - - data_dir.mkdir(exist_ok=True) - - get_MNIST(data_dir) # type: ignore[arg-type] - - get_SVHN(data_dir / "SVHN") - - get_USPS(data_dir / "USPS") - - get_MNISTM(data_dir / "MNIST_M") - - get_synthDigits(data_dir / "SynthDigits") - - -if __name__ == "__main__": - main() diff --git a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/data_preprocess.py b/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/data_preprocess.py deleted file mode 100644 index df2cd3b89b4..00000000000 --- a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/data_preprocess.py +++ /dev/null @@ -1,278 +0,0 @@ -"""This file is used to download and pre-process all data in Digit-5 dataset. - -i.e., splitted data into train&test set in a stratified way. The -function to process data into 10 partitions is also provided. -""" - - -import bz2 -import os -import pickle as pkl -from collections import Counter - -import numpy as np -import scipy.io as scio # type: ignore -import torch -from sklearn import model_selection # type: ignore - -# pylint: disable=invalid-name, too-many-locals, broad-except - - -def stratified_split(X, y): - """Provides train/test indices to split data in train/test sets.""" - - sss = model_selection.StratifiedShuffleSplit( - n_splits=1, test_size=0.2, random_state=0 - ) - - for train_index, test_index in sss.split(X, y): - X_train, X_test = X[train_index], X[test_index] - y_train, y_test = y[train_index], y[test_index] - print("Train:", Counter(y_train)) - print("Test:", Counter(y_test)) - - return (X_train, y_train), (X_test, y_test) - - -def process_mnist(): - """ - train: - (56000, 28, 28) - (56000,) - test: - (14000, 28, 28) - (14000,) - """ - mnist_train = "./data/MNIST/training.pt" - mnist_test = "./data/MNIST/test.pt" - train = torch.load(mnist_train) - test = torch.load(mnist_test) - - train_img = train[0].numpy() - train_tar = train[1].numpy() - - test_img = test[0].numpy() - test_tar = test[1].numpy() - - all_img = np.concatenate([train_img, test_img]) - all_tar = np.concatenate([train_tar, test_tar]) - - train_stratified, test_stratified = stratified_split(all_img, all_tar) - print("# After spliting:") - print("Train imgs:\t", train_stratified[0].shape) - print("Train labels:\t", train_stratified[1].shape) - print("Test imgs:\t", test_stratified[0].shape) - print("Test labels:\t", test_stratified[1].shape) - - with open("./data/MNIST/train.pkl", "wb") as mnist_train_file: - pkl.dump(train_stratified, mnist_train_file, pkl.HIGHEST_PROTOCOL) - - with open("./data/MNIST/test.pkl", "wb") as mnist_test_file: - pkl.dump(test_stratified, mnist_test_file, pkl.HIGHEST_PROTOCOL) - - -def process_svhn(): - """ - train: - (79431, 32, 32, 3) - (79431,) - test: - (19858, 32, 32, 3) - (19858,) - """ - train = scio.loadmat("./data/SVHN/train_32x32.mat") - test = scio.loadmat("./data/SVHN/test_32x32.mat") - - train_img = train["X"] - train_tar = train["y"].astype(np.int64).squeeze() - - test_img = test["X"] - test_tar = test["y"].astype(np.int64).squeeze() - - train_img = np.transpose(train_img, (3, 0, 1, 2)) - test_img = np.transpose(test_img, (3, 0, 1, 2)) - - np.place(train_tar, train_tar == 10, 0) - np.place(test_tar, test_tar == 10, 0) - - all_img = np.concatenate([train_img, test_img]) - all_tar = np.concatenate([train_tar, test_tar]) - - train_stratified, test_stratified = stratified_split(all_img, all_tar) - print("# After spliting:") - print("Train imgs:\t", train_stratified[0].shape) - print("Train labels:\t", train_stratified[1].shape) - print("Test imgs:\t", test_stratified[0].shape) - print("Test labels:\t", test_stratified[1].shape) - - with open("./data/SVHN/train.pkl", "wb") as svhn_train_file: - pkl.dump(train_stratified, svhn_train_file, pkl.HIGHEST_PROTOCOL) - - with open("./data/SVHN/test.pkl", "wb") as svhn_test_file: - pkl.dump(test_stratified, svhn_test_file, pkl.HIGHEST_PROTOCOL) - - -def process_usps(): - """ - train: - (7438, 16, 16) - (7438,) - test: - (1860, 16, 16) - (1860,) - :return: - """ - - train_path = "./data/USPS/usps.bz2" - with bz2.open(train_path) as fp: - raw_data = [l.decode().split() for l in fp.readlines()] - imgs = [[x.split(":")[-1] for x in data[1:]] for data in raw_data] - imgs = np.asarray(imgs, dtype=np.float32).reshape((-1, 16, 16)) - imgs = ((imgs + 1) / 2 * 255).astype(dtype=np.uint8) - targets = [int(d[0]) - 1 for d in raw_data] - - train_img = imgs - train_tar = np.array(targets) - - test_path = "./data/USPS/usps.t.bz2" - with bz2.open(test_path) as fp: - raw_data = [l.decode().split() for l in fp.readlines()] - imgs = [[x.split(":")[-1] for x in data[1:]] for data in raw_data] - imgs = np.asarray(imgs, dtype=np.float32).reshape((-1, 16, 16)) - imgs = ((imgs + 1) / 2 * 255).astype(dtype=np.uint8) - targets = [int(d[0]) - 1 for d in raw_data] - - test_img = imgs - test_tar = np.array(targets) - - all_img = np.concatenate([train_img, test_img]) - all_tar = np.concatenate([train_tar, test_tar]) - - train_stratified, test_stratified = stratified_split(all_img, all_tar) - print("# After spliting:") - print("Train imgs:\t", train_stratified[0].shape) - print("Train labels:\t", train_stratified[1].shape) - print("Test imgs:\t", test_stratified[0].shape) - print("Test labels:\t", test_stratified[1].shape) - - with open("./data/USPS/train.pkl", "wb") as usps_train_file: - pkl.dump(train_stratified, usps_train_file, pkl.HIGHEST_PROTOCOL) - - with open("./data/USPS/test.pkl", "wb") as usps_test_file: - pkl.dump(test_stratified, usps_test_file, pkl.HIGHEST_PROTOCOL) - - -def process_synth(): - """(391162, 32, 32, 3) (391162,) (97791, 32, 32, 3) (97791,)""" - train = scio.loadmat("./data/SynthDigits/synth_train_32x32.mat") - test = scio.loadmat("./data/SynthDigits/synth_test_32x32.mat") - - train_img = train["X"] - train_tar = train["y"].astype(np.int64).squeeze() - - test_img = test["X"] - test_tar = test["y"].astype(np.int64).squeeze() - - train_img = np.transpose(train_img, (3, 0, 1, 2)) - test_img = np.transpose(test_img, (3, 0, 1, 2)) - - all_img = np.concatenate([train_img, test_img]) - all_tar = np.concatenate([train_tar, test_tar]) - - train_stratified, test_stratified = stratified_split(all_img, all_tar) - print("# After spliting:") - print("Train imgs:\t", train_stratified[0].shape) - print("Train labels:\t", train_stratified[1].shape) - print("Test imgs:\t", test_stratified[0].shape) - print("Test labels:\t", test_stratified[1].shape) - - with open("./data/SynthDigits/train.pkl", "wb") as synthdigits_train_file: - pkl.dump(train_stratified, synthdigits_train_file, pkl.HIGHEST_PROTOCOL) - - with open("./data/SynthDigits/test.pkl", "wb") as synthdigits_test_file: - pkl.dump(test_stratified, synthdigits_test_file, pkl.HIGHEST_PROTOCOL) - - -def process_mnistm(): - """ - (56000, 28, 28, 3) - (56000,) - (14000, 28, 28, 3) - (14000,) - :return: - """ - data = np.load("./data/MNIST_M/mnistm_data.pkl", allow_pickle=True) - train_img = data["train"] - train_tar = data["train_label"] - valid_img = data["valid"] - valid_tar = data["valid_label"] - test_img = data["test"] - test_tar = data["test_label"] - - all_img = np.concatenate([train_img, valid_img, test_img]) - all_tar = np.concatenate([train_tar, valid_tar, test_tar]) - - train_stratified, test_stratified = stratified_split(all_img, all_tar) - print("# After spliting:") - print("Train imgs:\t", train_stratified[0].shape) - print("Train labels:\t", train_stratified[1].shape) - print("Test imgs:\t", test_stratified[0].shape) - print("Test labels:\t", test_stratified[1].shape) - - with open("./data/MNIST_M/train.pkl", "wb") as mnistm_train_file: - pkl.dump(train_stratified, mnistm_train_file, pkl.HIGHEST_PROTOCOL) - - with open("./data/MNIST_M/test.pkl", "wb") as mnistm_test_file: - pkl.dump(test_stratified, mnistm_test_file, pkl.HIGHEST_PROTOCOL) - - -def split(data_path, percentage=0.1): - """split each single dataset into multiple partitions for client scaling - training each part remain the same size according to the smallest datasize - (i.e. 743)""" - images, labels = np.load(os.path.join(data_path, "train.pkl"), allow_pickle=True) - part_len = 743.8 - part_num = int(1.0 / percentage) - - for num in range(part_num): - images_part = images[int(part_len * num) : int(part_len * (num + 1)), :, :] - labels_part = labels[int(part_len * num) : int(part_len * (num + 1))] - - save_path = os.path.join(data_path, "partitions") - if not os.path.exists(save_path): - os.makedirs(save_path) - with open( - os.path.join(save_path, "train_part{}.pkl".format(num)), "wb" - ) as file_split: - pkl.dump((images_part, labels_part), file_split, pkl.HIGHEST_PROTOCOL) - - -if __name__ == "__main__": - print("Processing...") - print("--------MNIST---------") - process_mnist() - print("--------SVHN---------") - process_svhn() - print("--------USPS---------") - process_usps() - print("--------MNIST-M---------") - process_mnistm() - print("-------SynthDigits-------") - try: - process_synth() - except Exception as exception: - print(f"unable to process SynthDigits: {exception}") - - base_paths = [ - "./data/MNIST", - "./data/SVHN", - "./data/USPS", - "./data/MNIST_M", - "./data/SynthDigits", - ] - for path in base_paths: - print(f"Spliting {os.path.basename(path)}") - try: - split(path) - except Exception as exception: - print(f"Failed to split: {path} --> {exception}") diff --git a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/data_utils.py b/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/data_utils.py deleted file mode 100644 index c465e3ea10d..00000000000 --- a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/data_utils.py +++ /dev/null @@ -1,87 +0,0 @@ -"""This code creates 10 different partitions of each datasets.""" - - -import os -import sys - -import numpy as np -from PIL import Image # type: ignore -from torch.utils.data import Dataset - -base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.append(base_path) - - -class DigitsDataset(Dataset): - """Split datasets.""" - - # pylint: disable=too-many-arguments - def __init__( - self, - data_path, - channels, - percent=0.1, - filename=None, - train=True, - transform=None, - ): - if filename is None: - if train: - if percent >= 0.1: - for part in range(int(percent * 10)): - if part == 0: - self.images, self.labels = np.load( - os.path.join( - data_path, - f"partitions/train_part{part}.pkl", - ), - allow_pickle=True, - ) - else: - images, labels = np.load( - os.path.join( - data_path, - f"partitions/train_part{part}.pkl", - ), - allow_pickle=True, - ) - self.images = np.concatenate([self.images, images], axis=0) - self.labels = np.concatenate([self.labels, labels], axis=0) - else: - self.images, self.labels = np.load( - os.path.join(data_path, "partitions/train_part0.pkl"), - allow_pickle=True, - ) - data_len = int(self.images.shape[0] * percent * 10) - self.images = self.images[:data_len] - self.labels = self.labels[:data_len] - else: - self.images, self.labels = np.load( - os.path.join(data_path, "test.pkl"), allow_pickle=True - ) - else: - self.images, self.labels = np.load( - os.path.join(data_path, filename), allow_pickle=True - ) - - self.transform = transform - self.channels = channels - self.labels = self.labels.astype(np.long).squeeze() - - def __len__(self): - return self.images.shape[0] - - def __getitem__(self, idx): - image = self.images[idx] - label = self.labels[idx] - if self.channels == 1: - image = Image.fromarray(image, mode="L") - elif self.channels == 3: - image = Image.fromarray(image, mode="RGB") - else: - raise ValueError(f"{self.channels} channel is not allowed.") - - if self.transform is not None: - image = self.transform(image) - - return image, label diff --git a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/mnistm/__init__.py b/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/mnistm/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/mnistm/create_mnistm.py b/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/mnistm/create_mnistm.py deleted file mode 100644 index 5af23d52122..00000000000 --- a/baselines/flwr_baselines/flwr_baselines/publications/fedbn/convergence_rate/utils/mnistm/create_mnistm.py +++ /dev/null @@ -1,71 +0,0 @@ -"""! This script has been borrowed and adapted. Original script: -https://github.com/pumpikano/tf-dann/blob/master/create_mnistm.py. - -It creatse the MNIST-M dataset based on MNIST -""" - - -import tarfile -from typing import Any - -import numpy as np -import skimage # type: ignore -import skimage.io # type: ignore -import skimage.transform # type: ignore - - -# pylint: disable=invalid-name, disable=no-member, bare-except -def compose_image(digit: Any, background: Any) -> Any: - """Difference-blend a digit and a random patch from a background image.""" - w, h, _ = background.shape - dw, dh, _ = digit.shape - x = np.random.randint(0, w - dw) - y = np.random.randint(0, h - dh) - - bg = background[x : x + dw, y : y + dh] - return np.abs(bg - digit).astype(np.uint8) - - -def mnist_to_img(x: Any) -> Any: - """Binarize MNIST digit and convert to RGB.""" - x = (x > 0).float() - d = x.reshape([28, 28, 1]) * 255 - return np.concatenate([d, d, d], 2) - - -def create_mnistm(X: Any) -> Any: - """Give an array of MNIST digits, blend random background patches to build - the MNIST-M dataset as described in - http://jmlr.org/papers/volume17/15-239/15-239.pdf.""" - - bst_path = "./data/MNIST_M/BSR_bsds500.tgz" - - rand = np.random.RandomState(42) - train_files = [] - - with tarfile.open(bst_path, "r") as bsr_file: - for name in bsr_file.getnames(): - if name.startswith("BSR/BSDS500/data/images/train/"): - train_files.append(name) - - print("Loading BSR training images") - background_data = [] - for name in train_files: - try: - fp = bsr_file.extractfile(name) - bg_img = skimage.io.imread(fp) - background_data.append(bg_img) - except: - continue - - X_ = np.zeros([X.shape[0], 28, 28, 3], np.uint8) - for i in range(X.shape[0]): - if i % 1000 == 0: - print("Processing example", i) - - bg_img = rand.choice(background_data) - d = mnist_to_img(X[i]) - d = compose_image(d, bg_img) - X_[i] = d - - return X_