training.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "L4",
      "authorship_tag": "ABX9TyPa84o0tnW5eBq/OsbWoALs",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/anarlavrenov/n2/blob/main/training_github.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gtELfr1mGVU5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e86bb5da-e83f-4ea8-d958-b9cb171a33f5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount(\"/content/drive\", force_remount=True)\n",
        "\n",
        "import torch\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\""
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Загрузка данных\n",
        "import torchaudio\n",
        "\n",
        "dataset = torchaudio.datasets.LIBRISPEECH(\".\", url=\"train-clean-100\", download=True)\n",
        "\n",
        "# Фоновый шум\n",
        "SAMPLE_NOISE = torchaudio.utils.download_asset(\"tutorial-assets/Lab41-SRI-VOiCES-rm1-babb-mc01-stu-clo-8000hz.wav\")\n",
        "# Эффект исходной речи, но в другом пространстве\n",
        "SAMPLE_RIR = torchaudio.utils.download_asset(\"tutorial-assets/Lab41-SRI-VOiCES-rm1-impulse-mc01-stu-clo-8000hz.wav\")\n",
        "noise, nsr = torchaudio.load(SAMPLE_NOISE)\n",
        "rir_raw, rsr = torchaudio.load(SAMPLE_RIR)"
      ],
      "metadata": {
        "id": "7VhX01i6Gc9r",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3ce575f3-cccb-434d-cc1f-65ec48b00e67"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 5.95G/5.95G [07:40<00:00, 13.9MB/s]\n",
            "100%|██████████| 78.2k/78.2k [00:00<00:00, 82.4MB/s]\n",
            "100%|██████████| 31.3k/31.3k [00:00<00:00, 38.6MB/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Разделение данных на трейн и валидацию\n",
        "train_size = int(len(dataset) * 0.9)\n",
        "valid_size = len(dataset) - train_size\n",
        "\n",
        "train_dataset, valid_dataset = torch.utils.data.random_split(\n",
        "    dataset, [train_size, valid_size]\n",
        ")"
      ],
      "metadata": {
        "id": "cWjLy_tfQ6Vw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Создание словаря из символов\n",
        "import torchtext; torchtext.disable_torchtext_deprecation_warning()\n",
        "from torchtext.vocab import vocab\n",
        "from collections import Counter\n",
        "\n",
        "chars = [x for x in \"abcdefghijklmnopqrstuvwxyz'?! \"]\n",
        "\n",
        "counter = Counter(chars) # Считает кол-во уникальных токенов в списке, возвращает словарь\n",
        "vocab = vocab(counter)\n",
        "unk_token = \"\"\n",
        "vocab.insert_token(unk_token, 0)\n",
        "vocab.set_default_index(vocab[unk_token])"
      ],
      "metadata": {
        "id": "euQ0emgmGhrc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Препроцессинг аудио\n",
        "win_length = 256\n",
        "hop_length = 160\n",
        "n_fft = 384\n",
        "\n",
        "# Шум добавляется не совсем качественно,\n",
        "# потому что длина шума, в среднем, меньше\n",
        "# длины речи. Из-за этого шум нужно\n",
        "# интерполировать, что меняет его природу\n",
        "def add_noise(waveform, noise=noise):\n",
        "  noise = torchaudio.functional.resample(noise, orig_freq=nsr, new_freq=16000)\n",
        "\n",
        "  # Урезание/продление шума до размера оригинальной waveform\n",
        "  if noise.shape[1] > waveform.shape[1]:\n",
        "    noise = noise[:, :waveform.shape[1]]\n",
        "  else:\n",
        "    noise = noise.unsqueeze(0)\n",
        "    noise = torch.nn.functional.interpolate(noise, size=(waveform.shape[1]), mode=\"linear\")\n",
        "    noise = noise.squeeze(0)\n",
        "\n",
        "  snr_dbs = torch.tensor([3])\n",
        "  # Добавление шума в оригинальную waveform\n",
        "  noisy_speech = torchaudio.functional.add_noise(waveform, noise, snr_dbs)\n",
        "\n",
        "  return noisy_speech\n",
        "\n",
        "\n",
        "def add_rir(waveform, rir_raw=rir_raw):\n",
        "  # Обрезание лишней части звука хлопка\n",
        "  rir = rir_raw[:, int(rsr * 1.01) : int(rsr * 1.3)]\n",
        "  # Нормализация\n",
        "  rir = rir / torch.linalg.vector_norm(rir, ord=2)\n",
        "  roomed_speech = torchaudio.functional.fftconvolve(waveform, rir)\n",
        "\n",
        "  return roomed_speech\n",
        "\n",
        "\n",
        "def transform_to_spectrogram(waveform):\n",
        "\n",
        "  # Получение waveform и sample_rate\n",
        "  waveform = torch.squeeze(waveform, dim=0)\n",
        "  waveform = waveform.to(torch.float32)\n",
        "  # Получение спектрограммы\n",
        "  transforms = torchaudio.transforms.Spectrogram(\n",
        "      win_length=win_length,\n",
        "      hop_length=hop_length,\n",
        "      n_fft=n_fft,\n",
        "      power=None\n",
        "  )\n",
        "  spectrogram = transforms(waveform)\n",
        "  # Перестановка на timeframes, n_mels\n",
        "  spectrogram = torch.transpose(spectrogram, 1, 0)\n",
        "\n",
        "  # Получение магнитуды\n",
        "  spectrogram = torch.abs(spectrogram)\n",
        "  spectrogram = torch.pow(spectrogram, 0.5)\n",
        "  # Нормализация\n",
        "  means = torch.mean(spectrogram, dim=1, keepdims=True)\n",
        "  stddevs = torch.std(spectrogram, dim=1, keepdims=True)\n",
        "  spectrogram = (spectrogram - means) / (stddevs + 1e-10)\n",
        "\n",
        "  return spectrogram\n",
        "\n",
        "\n",
        "def preprocess_audio(waveform, noise=noise, rir_raw=rir_raw):\n",
        "\n",
        "  # Получение шумовой копии исходной waveform\n",
        "  noisy_speech = add_noise(waveform, noise)\n",
        "  roomed_speech = add_rir(waveform, rir_raw)\n",
        "\n",
        "  # Получение спектрограмм для оригинальной\n",
        "  # записи, записи с добавленным шумом записи\n",
        "  # с эффектом звучания в пространстве\n",
        "  original_spectrogram = transform_to_spectrogram(waveform)\n",
        "  noisy_spectrogram = transform_to_spectrogram(noisy_speech)\n",
        "  roomed_spectrogram = transform_to_spectrogram(roomed_speech)\n",
        "\n",
        "  return (original_spectrogram, noisy_spectrogram, roomed_spectrogram)"
      ],
      "metadata": {
        "id": "wNysPjJeGi3a"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Функция обработки батчей даталоадера\n",
        "def collate_fn(batch):\n",
        "\n",
        "  wf, sr, text, *_ = zip(*batch)\n",
        "\n",
        "  spectrograms = []\n",
        "  tokens = []\n",
        "\n",
        "  # Обработка спектрограммы\n",
        "  for i, (w, t) in enumerate(zip(wf, text)):\n",
        "    original_spectrogram, noisy_spectrogram, roomed_spectrogram = preprocess_audio(w, noise, rir_raw)\n",
        "    original_spectrogram = torch.nn.functional.pad(\n",
        "        original_spectrogram, (0, 0, 0, 2048-original_spectrogram.shape[0]), \"constant\", 0\n",
        "    ) # (слева, справа, сверху, снизу)\n",
        "\n",
        "    spectrograms.append(original_spectrogram)\n",
        "\n",
        "    # В даталоадер будет дублироваться каждая\n",
        "    # 10 запись с шумом и с эффектом звучания в пространстве\n",
        "    if i % 10 == 0:\n",
        "      noisy_spectrogram = torch.nn.functional.pad(\n",
        "        noisy_spectrogram, (0, 0, 0, 2048-noisy_spectrogram.shape[0]), \"constant\", 0\n",
        "    ) # (слева, справа, сверху, снизу)\n",
        "      spectrograms.append(noisy_spectrogram)\n",
        "\n",
        "      roomed_spectrogram = torch.nn.functional.pad(\n",
        "        roomed_spectrogram, (0, 0, 0, 2048-roomed_spectrogram.shape[0]), \"constant\", 0\n",
        "    ) # (слева, справа, сверху, снизу)\n",
        "      spectrograms.append(roomed_spectrogram)\n",
        "\n",
        "    # Обработка текста\n",
        "    t = t.lower()\n",
        "    t = [vocab[x] for x in t]\n",
        "    t = torch.nn.functional.pad(\n",
        "        torch.tensor(t), (0, 216-len(t)), \"constant\", 0\n",
        "    )\n",
        "    tokens.append(t)\n",
        "\n",
        "    # В даталоадер будет дублироваться\n",
        "    # каждая 10 и 15 запись с шумом и эффектом\n",
        "    # звучания в пространстве\n",
        "    if i % 10 == 0:\n",
        "      tokens.append(t)\n",
        "      tokens.append(t)\n",
        "\n",
        "  spectrograms = torch.stack(spectrograms, dim=0)\n",
        "  tokens = torch.stack(tokens, dim=0)\n",
        "\n",
        "  return spectrograms, tokens"
      ],
      "metadata": {
        "id": "m4n9D57DGkE1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Создание даталоадера torch\n",
        "train_loader = torch.utils.data.DataLoader(\n",
        "    train_dataset,\n",
        "    batch_size=60,\n",
        "    shuffle=False,\n",
        "    num_workers=2,\n",
        "    drop_last=True,\n",
        "    collate_fn=collate_fn\n",
        ")\n",
        "\n",
        "valid_loader = torch.utils.data.DataLoader(\n",
        "    valid_dataset,\n",
        "    batch_size=12,\n",
        "    shuffle=False,\n",
        "    num_workers=2,\n",
        "    drop_last=True,\n",
        "    collate_fn=collate_fn\n",
        ")"
      ],
      "metadata": {
        "id": "HfmKywlYGyoG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "src = next(iter(train_loader))[0].to(device)\n",
        "tgt = next(iter(train_loader))[1].to(device)\n",
        "\n",
        "val_src = next(iter(valid_loader))[0].to(device)\n",
        "val_tgt = next(iter(valid_loader))[1].to(device)\n",
        "\n",
        "print(src.shape, tgt.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UKteiPjYG1aA",
        "outputId": "874d1fb3-676b-4ab0-cb79-ebd434389d4e",
        "collapsed": true
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([72, 2048, 193]) torch.Size([72, 216])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Модель\n",
        "class Model(torch.nn.Module):\n",
        "  def __init__(self, rnn_layers, rnn_units, output_dim):\n",
        "    super(Model, self).__init__()\n",
        "\n",
        "    self.conv1 = torch.nn.Conv2d(\n",
        "        in_channels=1,\n",
        "        out_channels=32,\n",
        "        kernel_size=(11, 41),\n",
        "        padding=(5, 20),\n",
        "        stride=(2, 2),\n",
        "        bias=False\n",
        "        )\n",
        "\n",
        "    self.conv2 = torch.nn.Conv2d(\n",
        "      in_channels=32,\n",
        "      out_channels=32,\n",
        "      kernel_size=(11, 21),\n",
        "      padding=(5, 10),\n",
        "      stride=(1, 2),\n",
        "      bias=False\n",
        "    )\n",
        "\n",
        "    self.conv3 = torch.nn.Conv2d(\n",
        "      in_channels=32,\n",
        "      out_channels=64,\n",
        "      kernel_size=(11, 21),\n",
        "      padding=(5, 10),\n",
        "      stride=(1, 2),\n",
        "      bias=False\n",
        "    )\n",
        "\n",
        "    self.lstm = torch.nn.LSTM(\n",
        "        input_size=64 * 25,\n",
        "        hidden_size=rnn_units,\n",
        "        num_layers=rnn_layers,\n",
        "        bidirectional=True,\n",
        "        dropout=0.5,\n",
        "        batch_first=True,\n",
        "        bias=True\n",
        "    )\n",
        "\n",
        "    self.fc1 = torch.nn.Linear(\n",
        "        in_features=rnn_units * 2,\n",
        "        out_features=rnn_units * 2\n",
        "    )\n",
        "\n",
        "    self.fc2 = torch.nn.Linear(\n",
        "        in_features=rnn_units * 2,\n",
        "        out_features=output_dim + 1\n",
        "    )\n",
        "\n",
        "    self.bn1 = torch.nn.BatchNorm2d(num_features=32)\n",
        "    self.bn2 = torch.nn.BatchNorm2d(num_features=32)\n",
        "    self.bn3 = torch.nn.BatchNorm2d(num_features=64)\n",
        "\n",
        "    self.dp = torch.nn.Dropout(p=0.5)\n",
        "    self.relu = torch.nn.ReLU()\n",
        "\n",
        "\n",
        "  def forward(self, src):\n",
        "    src = src.unsqueeze(1)\n",
        "\n",
        "    src = self.relu(self.bn1(self.conv1(src))) # [batch_size, filters, height, width]\n",
        "    src = self.relu(self.bn2(self.conv2(src))) # [batch_size, filters, height, width]\n",
        "    src = self.relu(self.bn3(self.conv3(src))) # [batch_size, filters, height, width]\n",
        "    src = src.permute(0, 2, 1, 3)\n",
        "\n",
        "    src = src.reshape(src.shape[0], src.shape[1], src.shape[2] * src.shape[3]) # [batch_size, height, filters * width]\n",
        "    rnn_out, (ht, ct) = self.lstm(src) # [batch_size, height, rnn_units * 2]\n",
        "\n",
        "    fc_out = self.fc1(rnn_out) # [batch_size, height, rnn_units * 2]\n",
        "    fc_out = self.relu(fc_out)\n",
        "    fc_out = self.dp(fc_out)\n",
        "    out = self.fc2(fc_out) # [batch_size, height, output_dim]\n",
        "\n",
        "    out = out.permute(1, 0, 2) # [height, batch_size, output_dim]\n",
        "\n",
        "    out = torch.nn.functional.log_softmax(out, dim=2) # [batch_size, height, output_dim]\n",
        "\n",
        "    return out"
      ],
      "metadata": {
        "id": "LWaMtKdQG24g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = Model(\n",
        "    rnn_layers=5,\n",
        "    rnn_units=768,\n",
        "    output_dim=len(vocab)\n",
        ")\n",
        "\n",
        "for param in model.parameters():\n",
        "  if param.dim() > 1:\n",
        "    torch.nn.init.xavier_uniform_(param)\n",
        "\n",
        "model = model.to(device)"
      ],
      "metadata": {
        "id": "UP1_Dlb_Lrau"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "criterion = torch.nn.CTCLoss()\n",
        "optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4)"
      ],
      "metadata": {
        "id": "V0TPdsxaTjLw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def calculate_loss(y_pred, y_true):\n",
        "\n",
        "  T, N, C = y_pred.size() # sequence_length, batch_size, n_classes\n",
        "\n",
        "  input_len = torch.full(size=(N, ), fill_value=T, dtype=torch.long)\n",
        "  target_len = torch.sum(y_true !=0, dim=1, dtype=torch.long)\n",
        "\n",
        "  loss = criterion(y_pred, y_true, input_len, target_len)\n",
        "\n",
        "  return loss"
      ],
      "metadata": {
        "id": "ShBnQzquTuAn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import tqdm\n",
        "\n",
        "def train(loader):\n",
        "\n",
        "  model.train()\n",
        "  total_loss = 0\n",
        "\n",
        "  for batch in tqdm(loader, desc=\"Training\"):\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    src, tgt = batch[0].to(device), batch[1].to(device)\n",
        "\n",
        "    outputs = model(src)\n",
        "\n",
        "    loss = calculate_loss(outputs, tgt)\n",
        "    total_loss += loss.item()\n",
        "\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "\n",
        "  return total_loss / len(loader)\n",
        "\n",
        "\n",
        "def evaluate(loader):\n",
        "\n",
        "  model.eval()\n",
        "  total_loss = 0\n",
        "\n",
        "  with torch.no_grad():\n",
        "      for batch in tqdm(loader, desc=\"Evaluation\"):\n",
        "\n",
        "        src, tgt = batch[0].to(device), batch[1].to(device)\n",
        "\n",
        "        outputs = model(src)\n",
        "\n",
        "        loss = calculate_loss(outputs, tgt)\n",
        "        total_loss += loss.item()\n",
        "\n",
        "      return total_loss / len(loader)"
      ],
      "metadata": {
        "id": "5Zyrb0mRVmUa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Указание ранней остановки обучения\n",
        "class EarlyStopping():\n",
        "  def __init__(self, patience, min_delta, start_on_epoch):\n",
        "\n",
        "    self.patience = patience\n",
        "    self.min_delta = min_delta\n",
        "    self.counter = 0\n",
        "    self.min_validation_loss = float(\"inf\")\n",
        "    self.start_on_epoch = start_on_epoch\n",
        "\n",
        "  def early_stop(self, validation_loss, epoch):\n",
        "\n",
        "    if epoch >= self.start_on_epoch:\n",
        "\n",
        "      if validation_loss < self.min_validation_loss:\n",
        "        self.min_validation_loss = validation_loss\n",
        "        self.counter = 0\n",
        "\n",
        "      elif validation_loss > (self.min_validation_loss + self.min_delta):\n",
        "        self.counter += 1\n",
        "        if self.counter >= self.patience:\n",
        "          return True\n",
        "\n",
        "      return False\n",
        "\n",
        "early_stopping = EarlyStopping(patience=2, min_delta=0, start_on_epoch=5)"
      ],
      "metadata": {
        "id": "ZHjWzJAmKbn2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "epochs = 50\n",
        "\n",
        "history = {\n",
        "    \"loss\": [torch.inf],\n",
        "    \"val_loss\": [torch.inf]\n",
        "}\n",
        "\n",
        "for epoch in range(epochs):\n",
        "  loss = train(train_loader)\n",
        "  val_loss = evaluate(valid_loader)\n",
        "\n",
        "  history[\"loss\"].append(loss)\n",
        "  history[\"val_loss\"].append(val_loss)\n",
        "\n",
        "  # Save checkpoint\n",
        "  if history[\"val_loss\"][-1] < history[\"val_loss\"][-2]:\n",
        "    torch.save(model, f\"/content/LibriSpeech_100_{val_loss:.3f}.pth\")\n",
        "    print(f\"Model with loss {val_loss:.3f} saved\")\n",
        "\n",
        "  print(f\"epoch: {epoch + 1} | loss: {loss:.3f} | val_loss: {val_loss:.3f}\" + \"\\n\")\n",
        "  print(\"*\" * 60 + \"\\n\")\n",
        "\n",
        "  # Early Stopping\n",
        "  if early_stopping.early_stop(val_loss, epoch + 1):\n",
        "    print(\"Сработал Early Stopping\")\n",
        "    break\n",
        "\n",
        "# Сохранение параметров оптимизатора\n",
        "torch.save(optimizer.state_dict(), \"/content/LibriSpeech_100_optimizer.pth\")"
      ],
      "metadata": {
        "id": "hVCCH_8MZZPO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "60c76da0-850f-41d8-afd5-374e0a6889ec"
      },
      "execution_count": null,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Training: 100%|██████████| 428/428 [21:24<00:00,  3.00s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:22<00:00,  1.66it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model with loss 2.944 saved\n",
            "epoch: 1 | loss: 3.200 | val_loss: 2.944\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Training: 100%|██████████| 428/428 [21:35<00:00,  3.03s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:27<00:00,  1.61it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model with loss 2.852 saved\n",
            "epoch: 2 | loss: 2.856 | val_loss: 2.852\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Training: 100%|██████████| 428/428 [21:45<00:00,  3.05s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:23<00:00,  1.65it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model with loss 2.006 saved\n",
            "epoch: 3 | loss: 2.525 | val_loss: 2.006\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Training: 100%|██████████| 428/428 [21:30<00:00,  3.02s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:22<00:00,  1.66it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model with loss 1.449 saved\n",
            "epoch: 4 | loss: 1.734 | val_loss: 1.449\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Training: 100%|██████████| 428/428 [21:39<00:00,  3.04s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:23<00:00,  1.65it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model with loss 1.204 saved\n",
            "epoch: 5 | loss: 1.363 | val_loss: 1.204\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Training: 100%|██████████| 428/428 [21:30<00:00,  3.02s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:23<00:00,  1.65it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model with loss 1.027 saved\n",
            "epoch: 6 | loss: 1.142 | val_loss: 1.027\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Training: 100%|██████████| 428/428 [21:30<00:00,  3.02s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:23<00:00,  1.65it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model with loss 0.914 saved\n",
            "epoch: 7 | loss: 0.991 | val_loss: 0.914\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Training: 100%|██████████| 428/428 [21:29<00:00,  3.01s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:24<00:00,  1.64it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model with loss 0.834 saved\n",
            "epoch: 8 | loss: 0.877 | val_loss: 0.834\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Training: 100%|██████████| 428/428 [21:31<00:00,  3.02s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:24<00:00,  1.64it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model with loss 0.831 saved\n",
            "epoch: 9 | loss: 0.788 | val_loss: 0.831\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Training: 100%|██████████| 428/428 [21:32<00:00,  3.02s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:24<00:00,  1.64it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model with loss 0.741 saved\n",
            "epoch: 10 | loss: 0.716 | val_loss: 0.741\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Training: 100%|██████████| 428/428 [21:38<00:00,  3.03s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:24<00:00,  1.65it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model with loss 0.691 saved\n",
            "epoch: 11 | loss: 0.648 | val_loss: 0.691\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Training: 100%|██████████| 428/428 [21:25<00:00,  3.00s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:24<00:00,  1.64it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model with loss 0.666 saved\n",
            "epoch: 12 | loss: 0.594 | val_loss: 0.666\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Training: 100%|██████████| 428/428 [21:30<00:00,  3.02s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:23<00:00,  1.65it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model with loss 0.640 saved\n",
            "epoch: 13 | loss: 0.549 | val_loss: 0.640\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Training: 100%|██████████| 428/428 [21:34<00:00,  3.02s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:23<00:00,  1.65it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model with loss 0.636 saved\n",
            "epoch: 14 | loss: 0.507 | val_loss: 0.636\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Training: 100%|██████████| 428/428 [21:28<00:00,  3.01s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:23<00:00,  1.65it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model with loss 0.597 saved\n",
            "epoch: 15 | loss: 0.469 | val_loss: 0.597\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Training: 100%|██████████| 428/428 [21:31<00:00,  3.02s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:23<00:00,  1.66it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model with loss 0.587 saved\n",
            "epoch: 16 | loss: 0.434 | val_loss: 0.587\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Training: 100%|██████████| 428/428 [21:30<00:00,  3.01s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:23<00:00,  1.65it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "epoch: 17 | loss: 0.408 | val_loss: 0.589\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training: 100%|██████████| 428/428 [21:24<00:00,  3.00s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:23<00:00,  1.65it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model with loss 0.567 saved\n",
            "epoch: 18 | loss: 0.380 | val_loss: 0.567\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training: 100%|██████████| 428/428 [21:36<00:00,  3.03s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:24<00:00,  1.65it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch: 19 | loss: 0.356 | val_loss: 0.607\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training: 100%|██████████| 428/428 [21:29<00:00,  3.01s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:23<00:00,  1.65it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model with loss 0.540 saved\n",
            "epoch: 20 | loss: 0.333 | val_loss: 0.540\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training: 100%|██████████| 428/428 [21:27<00:00,  3.01s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:24<00:00,  1.64it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch: 21 | loss: 0.314 | val_loss: 0.555\n",
            "\n",
            "************************************************************\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training: 100%|██████████| 428/428 [21:28<00:00,  3.01s/it]\n",
            "Evaluation: 100%|██████████| 237/237 [02:24<00:00,  1.65it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch: 22 | loss: 0.294 | val_loss: 0.565\n",
            "\n",
            "************************************************************\n",
            "\n",
            "Сработал Early Stopping\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.plot(history[\"loss\"][1:], label=\"train\")\n",
        "plt.plot(history[\"val_loss\"][1:], label=\"validation\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"CTC Loss\")\n",
        "\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 449
        },
        "id": "toWByei8izis",
        "outputId": "7a2fcb82-9fca-4162-ce29-697802a5de59"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGwCAYAAABVdURTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABbCUlEQVR4nO3dd3xUVf7/8ded9E5CegFCr6GDiF0U0HXFXtgVLKyu4IqsP5W1ofvddV37Wtd1XXTVVVGxiyIKKL2FXgQCJKRQM+lt5v7+mGQgEMIEMpnM5P18POaRmTv3zHzCEPLmnHPPMUzTNBERERHxERZPFyAiIiLSnBRuRERExKco3IiIiIhPUbgRERERn6JwIyIiIj5F4UZERER8isKNiIiI+BR/TxfQ0ux2O7m5uURERGAYhqfLEREREReYpklxcTHJyclYLI33zbS5cJObm0taWpqnyxAREZFTkJ2dTWpqaqPntLlwExERATj+cCIjIz1cjYiIiLiiqKiItLQ05+/xxrS5cFM3FBUZGalwIyIi4mVcmVKiCcUiIiLiUxRuRERExKco3IiIiIhPaXNzbkRExHfY7Xaqqqo8XYY0k8DAwJNe5u0KhRsREfFKVVVVZGVlYbfbPV2KNBOLxUJ6ejqBgYGn9ToKNyIi4nVM0yQvLw8/Pz/S0tKa5X/74ll1i+zm5eXRoUOH01poV+FGRES8Tk1NDWVlZSQnJxMaGurpcqSZxMXFkZubS01NDQEBAaf8Ooq6IiLidWw2G8BpD19I61L3edZ9vqdK4UZERLyW9gj0Lc31eSrciIiIiE9RuBERERGfonAjIiLihTp16sTzzz/v6TJaJV0t1YxyDpdRWmmjR+LJdywVEZG257zzzmPAgAHNEkpWrFhBWFjY6Rflg9Rz00y+WZ/HBU8v4IFP1mGapqfLERERL2SaJjU1NS6dGxcXp8vgT0DhppkM7hiNn8VgzZ5Cvt1Y4OlyRETaFNM0Kauq8cjN1f/QTpw4kQULFvDCCy9gGAaGYTBz5kwMw+Cbb75h8ODBBAUF8fPPP7Njxw4uv/xyEhISCA8PZ+jQoXz//ff1Xu/YYSnDMHjjjTe44oorCA0NpVu3bnz++efN+cfsNTQs1UziI4O57ex0XvxhO3//dgujesXj76fsKCLSEsqrbfR+5FuPvPemx0cTGnjyX6cvvPAC27Zto2/fvjz++OMAbNy4EYAHHniAp59+ms6dOxMdHU12djaXXHIJf/nLXwgKCuLtt9/msssuY+vWrXTo0OGE7/HYY4/x97//naeeeooXX3yR8ePHs3v3bmJiYprnm/US+u3bjH53TmeiQwPYub+UWatyPF2OiIi0IlFRUQQGBhIaGkpiYiKJiYn4+fkB8Pjjj3PRRRfRpUsXYmJi6N+/P7fffjt9+/alW7du/PnPf6ZLly4n7YmZOHEiN9xwA127duWvf/0rJSUlLF++vCW+vVZFPTfNKCI4gLsu6MbjX27iubnbGDcghZBAP0+XJSLi80IC/Nj0+GiPvffpGjJkSL3HJSUlzJgxg6+++oq8vDxqamooLy9nz549jb5ORkaG835YWBiRkZHs27fvtOvzNgo3zWz8GR14c1EWOYfLeXNRFpPP7+rpkkREfJ5hGC4NDbVWx171dO+99zJ37lyefvppunbtSkhICFdffTVVVVWNvs6x+zEZhtEmd03XsFQzC/L3496LewDw2vwdHC5t/C+iiIi0HYGBgS7tm7Ro0SImTpzIFVdcQb9+/UhMTGTXrl3uL9BHKNy4wa/7J9MrKZLiyhpe/nG7p8sREZFWolOnTixbtoxdu3Zx4MCBE/aqdOvWjU8++YTMzEzWrl3LjTfe2CZ7YE6Vwo0bWCwGD4ztCcDbS3aTc7jMwxWJiEhrcO+99+Ln50fv3r2Ji4s74RyaZ599lujoaM4880wuu+wyRo8ezaBBg1q4Wu9lmG1sxbmioiKioqKwWq1ERka67X1M02T8G8tYvOMgVw5K4dlrB7jtvURE2pqKigqysrJIT08nODjY0+VIM2nsc23K72/13LiJYRjcP8bRezN7zV425xV5uCIREZG2QeHGjfqntePSjCRME/4+Z4unyxEREWkTFG7c7N6Le+BvMfhx636W7Djo6XJERER8nsKNm6XHhnHDMMdS2X+bs0WbaoqIiLiZwk0LuOvCroQG+rE2u5A5G/I9XY6IiIhPU7hpAfERwdx2dmcAnvp2K9U2rVUgIiLiLgo3LWTS2em0Dwtk54FSPlyZ7elyREREfJbCTQtxbKrp2Gfq+e9/oayqxsMViYiI+CaFmxZ04/COpMWEsL+4kjd/zvJ0OSIi4mU6derE888/73xsGAaffvrpCc/ftWsXhmGQmZl5Wu/bXK/TUhRuWlCgv+XIppoLdnJIm2qKiMhpyMvLY+zYsc36mhMnTmTcuHH1jqWlpZGXl0ffvn2b9b3cxaPh5tVXXyUjI4PIyEgiIyMZMWIE33zzTaNtZs2aRc+ePQkODqZfv358/fXXLVRt87gsI5k+yZGUVNbw0g/aVFNERE5dYmIiQUFBbn8fPz8/EhMT8ff3d/t7NQePhpvU1FT+9re/sWrVKlauXMkFF1zA5ZdfzsaNGxs8f/Hixdxwww3ceuutrFmzhnHjxjFu3Dg2bNjQwpWfuqM31fzv0l1kH9KmmiIibcHrr79OcnLycbt7X3755dxyyy3s2LGDyy+/nISEBMLDwxk6dCjff/99o6957LDU8uXLGThwIMHBwQwZMoQ1a9bUO99ms3HrrbeSnp5OSEgIPXr04IUXXnA+P2PGDN566y0+++wzDMPAMAzmz5/f4LDUggULGDZsGEFBQSQlJfHAAw9QU3NkPul5553HH/7wB+677z5iYmJITExkxowZTf+DOxVmKxMdHW2+8cYbDT537bXXmpdeemm9Y8OHDzdvv/12l1/farWagGm1Wk+rztM1/l9LzY73f2lOfX+NR+sQEfFG5eXl5qZNm8zy8nLHAbvdNCtLPHOz212q+dChQ2ZgYKD5/fffO48dPHjQeSwzM9N87bXXzPXr15vbtm0zH3roITM4ONjcvXu38/yOHTuazz33nPMxYM6ePds0TdMsLi424+LizBtvvNHcsGGD+cUXX5idO3c2AXPNmjWmaZpmVVWV+cgjj5grVqwwd+7cab7zzjtmaGio+cEHHzhf49prrzXHjBlj5uXlmXl5eWZlZaWZlZVV73VycnLM0NBQ88477zQ3b95szp4924yNjTUfffRRZ23nnnuuGRkZac6YMcPctm2b+dZbb5mGYZjfffed65/rUZry+7vV9C/ZbDZmzZpFaWkpI0aMaPCcJUuWMG3atHrHRo8e3ehkqsrKSiorK52Pi4paxwaW94/pyc8v/cynmXu57ex0+iRHebokERHvVV0Gf032zHv/KRcCw056WnR0NGPHjuW9997jwgsvBOCjjz4iNjaW888/H4vFQv/+/Z3n//nPf2b27Nl8/vnnTJky5aSv/95772G32/n3v/9NcHAwffr0IScnh9///vfOcwICAnjsscecj9PT01myZAkffvgh1157LeHh4YSEhFBZWUliYuIJ3+uVV14hLS2Nl156CcMw6NmzJ7m5udx///088sgjWCyOgaGMjAweffRRALp168ZLL73EvHnzuOiii076/ZwOj08oXr9+PeHh4QQFBXHHHXcwe/Zsevfu3eC5+fn5JCQk1DuWkJBAfv6JV/194okniIqKct7S0tKatf5T1S81isv6J9duqrnV0+WIiEgLGD9+PB9//LHzP93vvvsu119/PRaLhZKSEu6991569epFu3btCA8PZ/PmzezZs8el1968eTMZGRkEBwc7jzXUWfDyyy8zePBg4uLiCA8P5/XXX3f5PY5+rxEjRmAYhvPYyJEjKSkpIScnx3ksIyOjXrukpCT27dvXpPc6FR7vuenRoweZmZlYrVY++ugjJkyYwIIFC04YcJpq+vTp9Xp7ioqKWk3Auffi7nyzPo8F2/azePsBzuwa6+mSRES8U0CoowfFU+/tossuuwzTNPnqq68YOnQoP/30E8899xwA9957L3PnzuXpp5+ma9euhISEcPXVV1NV1XxX1r7//vvce++9PPPMM4wYMYKIiAieeuopli1b1mzvcbSAgIB6jw3DOG7OkTt4PNwEBgbStatjcbvBgwezYsUKXnjhBf75z38ed25iYiIFBQX1jhUUFDTadRYUFNQiM8lPRcf2YYwf3oG3luzmb3O28NnkkfVSsIiIuMgwXBoa8rTg4GCuvPJK3n33XbZv306PHj0YNGgQAIsWLWLixIlcccUVAJSUlLBr1y6XX7tXr17897//paKiwtl7s3Tp0nrnLFq0iDPPPJM777zTeWzHjh31zgkMDMRms530vT7++GNM03T+3lq0aBERERGkpqa6XLO7eHxY6lh2u73eHJmjjRgxgnnz5tU7Nnfu3BPO0fEGd13YjbBAP9blWPl6vTbVFBHxdePHj+err77izTffZPz48c7j3bp145NPPiEzM5O1a9dy4403NqmX48Ybb8QwDCZNmsSmTZv4+uuvefrpp+ud061bN1auXMm3337Ltm3bePjhh1mxYkW9czp16sS6devYunUrBw4coLq6+rj3uvPOO8nOzuauu+5iy5YtfPbZZzz66KNMmzbNOd/GkzxawfTp01m4cCG7du1i/fr1TJ8+nfnz5zs/7Jtuuonp06c7z7/77ruZM2cOzzzzDFu2bGHGjBmsXLnSpYlWrVVseBCTzqnbVHOLNtUUEfFxF1xwATExMWzdupUbb7zRefzZZ58lOjqaM888k8suu4zRo0c7e3VcER4ezhdffMH69esZOHAgDz74IE8++WS9c26//XauvPJKrrvuOoYPH87Bgwfr9eIATJo0iR49ejBkyBDi4uJYtGjRce+VkpLC119/zfLly+nfvz933HEHt956Kw899FAT/zTcw6i9lMwjbr31VubNm0deXh5RUVFkZGRw//33O2dRn3feeXTq1ImZM2c628yaNYuHHnqIXbt20a1bN/7+979zySWXuPyeRUVFREVFYbVaiYyMbL5vxlYDH/wG+l4Jfa8Ci5/LTUsqazjvqR85UFLFn8f15bdndGy+ukREfFBFRQVZWVmkp6fXm0Ar3q2xz7Upv789Gm48wW3hZt0s+OQ2x/32XeGc/wd9rwY/16Y1vb1kF498tpHY8CAW/L/zCAvy+HQoEZFWS+HGNzVXuPH8wJiv6D4aLngIQqLh4HaYfTu8PAwy33P06pzE9UM70LF9KAdKKvm3NtUUERE5ZQo3zSU40tFbM3U9XPgIhMTAoR3w6e/hpSGw5h2wHT8pq87Rm2r+c8EODpY0PKlaREREGqdw09yCIuDsPzpCzqgZENoeDmfBZ5MdIWf12ycMOZf2S6JfShSlVTZe1KaaIiIip0Thxl2CwuGse+DudXDR4xAaC4d3wed3wYuDYNVMqKm/MNPRm2q+u2w3ew5qU00Rkca0sWmjPq+5Pk+FG3cLCoeRd8PUdXDx/0FYHBTugS/uhhcHw8o364WckV1jObtbLNU2k2fmalsGEZGG+Pk5rkhtztV7xfPqPs+6z/dU6WqpllZVBqv+A4tegJLa1ZYjU+Hse2Dgb8E/iA17rfzqxZ8B+PKus+ibok01RUSOZpome/bsobq6muTk5FaxcJycHrvdTm5uLgEBAXTo0OG4Fft1KXgjPB5u6lSXO4amfn4eSmpXJo5McQxlDfwtd3+8mc8yczmnexxv3zLMc3WKiLRSVVVVZGVltcheRdIyLBYL6enpBAYGHvecwk0jWk24qVNd7phk/PNzUJznOBaRzKGBd3L2vDRKbQG8e9twRmpTTRGR49jtdg1N+ZDAwMAT9sIp3DSi1YWbOtUVsOa/8NOzUOzY2bY4IJZny8ayLuEKZk25AItFm2qKiEjbpHDTiFYbburUVNaGnOegKAeAHDOWDWM/ZswZAzxbm4iIiIdohWJv5h8EQ2+DP6yGXz1HaUB7Uo0DbPt+JhXVjW9BLyIiIgo3rZd/EAy5hcCRjt1aO1du5s1F2pZBRETkZBRuWrmAjo4rpQZYtvPKjzvYX6xtGURERBqjcNPaJQ/ENCykGgcIrdzPc99v83RFIiIirZrCTWsXFIER3xuAgZZfeH/5HrbmF3u4KBERkdZL4cYbpA4B4Kr4POwm/OXrzR4uSEREpPVSuPEGqY55N+eE7iLAz2Dhtv3M37rPw0WJiIi0Tgo33iB1KADB+9ZxyxmpAPzlq83U2LTkuIiIyLEUbrxB+64QHAU15UzpW0l0aAC/7Cvh/RXZnq5MRESk1VG48QYWC6Q45t1E7M9k6qjuADw3dxtFFdWerExERKTVUbjxFrVDU+Ss4MbhHegcF8bB0ipe+XGHZ+sSERFpZRRuvEXakXAT4GfhwUt6AfDmz1lkHyrzYGEiIiKti8KNt0gZ7Ph6aCeUHuCCnvGM7NqeKpudJ+ds8WxtIiIirYjCjbcIiYZYx1wbclZiGAYPXtIbw4Av1+Wxavdhz9YnIiLSSijceJOj5t0A9E6O5NrBaQD8+ctNmKbpqcpERERaDYUbb3JMuAH448XdCQ30IzO7kC/W5XmoMBERkdZD4cab1IWbvavBbgMgPjKY35/bBYAnv9lCRbXNU9WJiIi0Cgo33iS+FwSEQVUx7D8yifi2szuTFBXM3sJy3lyU5cECRUREPE/hxptY/CBlkOP+UUNTIYF+3DemBwCv/LiD/cWVnqhORESkVVC48TYNzLsBuLx/ChmpUZRU1vDc99s8UJiIiEjroHDjbdIcO4STs7LeYYvF4KFLewPw/vI9bM0vbunKREREWgWFG29Tu8cU+7dAeWG9p4alxzC2byJ2E/7y9eaWr01ERKQVULjxNuFxEN3JcX/vquOefmBsTwL8DBZu28/8rftatjYREZFWQOHGGznn3aw87qmO7cOYMKITAH/5ajM1NnsLFiYiIuJ5Cjfe6ASTiuvcdUE32oUG8Mu+Et5fkd2ChYmIiHiewo03OjrcNLDlQlRoAFMv7AbAc3O3UVRR3ZLViYiIeJTCjTdK6Av+wVBRCAd3NHjK+DM60jk2jIOlVbzyY8PniIiI+CKFG2/kHwhJAxz3c5Y3eEqAn4U/XdILgDd/ziL7UFkLFSciIuJZCjfeKrX2kvATzLsBuLBXPGd2aU+Vzc6Tc7ac8DwRERFfonDjrZyL+Z043BiGwYOX9sIw4Mt1eazafbiFihMREfEchRtvVTepuGAjVJWe8LQ+yVFcMzgVgD9/uQmzgQnIIiIivkThxltFJkNkCph22Lu60VP/eHEPQgP9yMwu5It1eS1UoIiIiGco3HgzF+bdACREBnPHuV0AePKbLVRU29xdmYiIiMco3HizRlYqPtakszuTGBnM3sJy3lyU5ebCREREPEfhxpulHjWp+CRzaUIC/bhvTA8AXvlxB/uLK91dnYiIiEco3HizpAywBEDpPijcc9LTxw1IoV9KFCWVNTz3/bYWKFBERKTlKdx4s4AQSOznuH+SeTcAFovBQ5c6FvZ7f/ketuYXu7M6ERERj/BouHniiScYOnQoERERxMfHM27cOLZu3dpom5kzZ2IYRr1bcHBwC1XcCp1kE81jDe/cnjF9ErGb8Mx3jf9Zi4iIeCOPhpsFCxYwefJkli5dyty5c6murubiiy+mtPTE67YAREZGkpeX57zt3r27hSpuhZoYbgCmXNAVgJ+3H6DaZndHVSIiIh7j78k3nzNnTr3HM2fOJD4+nlWrVnHOOeecsJ1hGCQmJrq7PO+QVhtu8tZBdQUEnLwXq3dSJJHB/hRV1LA5r4iM1HburVFERKQFtao5N1arFYCYmJhGzyspKaFjx46kpaVx+eWXs3HjxhOeW1lZSVFRUb2bT2nXEcLiwF4N+etcamKxGAzp5PgzXrFLWzKIiIhvaTXhxm63M3XqVEaOHEnfvn1PeF6PHj148803+eyzz3jnnXew2+2ceeaZ5OTkNHj+E088QVRUlPOWlpbmrm/BMwzjyNBUdsM7hDdkSKdoAFbuOuSOqkRERDym1YSbyZMns2HDBt5///1GzxsxYgQ33XQTAwYM4Nxzz+WTTz4hLi6Of/7znw2eP336dKxWq/OWnZ3tjvI9y8WVio829KieG+03JSIivsSjc27qTJkyhS+//JKFCxeSmprapLYBAQEMHDiQ7du3N/h8UFAQQUFBzVFm6+VczO/kKxXX6ZcSRaCfhQMllew+WEan2DA3FSciItKyPNpzY5omU6ZMYfbs2fzwww+kp6c3+TVsNhvr168nKSnJDRV6ieSBYFigKAeKcl1qEhzgR0ZqFADLNTQlIiI+xKPhZvLkybzzzju89957REREkJ+fT35+PuXl5c5zbrrpJqZPn+58/Pjjj/Pdd9+xc+dOVq9ezW9+8xt2797Nbbfd5olvoXUICof4Po77Tei9qZtUrHk3IiLiSzwabl599VWsVivnnXceSUlJztsHH3zgPGfPnj3k5eU5Hx8+fJhJkybRq1cvLrnkEoqKili8eDG9e/f2xLfQejjn3bg+qXioc1KxrpgSERHf4dE5N65MZJ0/f369x8899xzPPfecmyryYqlDYdV/mtZz09HRc7PzQCkHSiqJDffxuUkiItImtJqrpeQ0pdVOKs5dA7Zql5pEhQbQIyECUO+NiIj4DoUbXxHTBYLbQU0FFGxwuZnWuxEREV+jcOMrLJYj826yT2G9m93quREREd+gcONLTmETzbqem417rZRV1bijKhERkRalcONLTmGl4pR2ISRFBVNjN8ncU+ieukRERFqQwo0vSakNN4ezoPSAS00MQ5toioiIb1G48SUh7SC2h+N+Ey4Jd653s1uTikVExPsp3Pga57ybJuwQXrvezerdh6mx2d1RlYiISItRuPE1pzDvpkdiBBHB/pRW2diSX+ymwkRERFqGwo2vqVvMb+9qsNtcauJnMRjc0TE0tULr3YiIiJdTuPE1cT0hMByqSmD/FpebDXVuoqlJxSIi4t0UbnyNxQ9SBjnuN2W9m6N6blzZ80tERKS1UrjxRXWTipuwUnH/tHYE+BnsK64k+1C5mwoTERFxP4UbX3QKKxUHB/jRLyUKgOWadyMiIl5M4cYX1YWbA1uhvNDlZkfm3SjciIiI91K48UVhsRCd7ri/d5XLzY6sVKxwIyIi3kvhxledyiaatZOKd+wv5WBJpTuqEhERcTuFG191CuEmOiyQbvHhAKzarUvCRUTEOync+CrnSsUrwe76lgp1Q1MrFW5ERMRLKdz4qsR+4B8MFYVwaIfLzeo20dS8GxER8VYKN77KLwCSBzruN2Foqu6KqQ17rZRXubZ9g4iISGuicOPL6oamsl3fITw1OoSEyCCqbSZrcwrdU5eIiIgbKdz4Muek4pUuNzEM48gl4VkamhIREe+jcOPLUmt3CN+3ESpLXG42tG6fKU0qFhERL6Rw48sikyAyFUw75K5xuVldz83q3Yex2bWJpoiIeBeFG1/nvCTc9UnFvZIiCQ/yp6Syhi35RW4qTERExD0UbnzdKSzm52cxGFQ7NLVyl4amRETEuyjc+Lqjw43p+hCTc96N1rsREREvo3Dj65L6gyUASvdD4W6Xmx29iabZhFAkIiLiaQo3vi4gGJIyHPebcEn4gLR2+FsMCooqyTlc7qbiREREmp/CTVtwCvNuQgL96JsSBcDK3RqaEhER76Fw0xbUhZsmrFQMR/aZWp6lScUiIuI9FG7agrpwk78Oql0fYnLuEK5JxSIi4kUUbtqCdh0gLB7sNZC3zuVmQ2qvmPplXwmHS6vcVZ2IiEizUrhpCwzjlObdtA8PoktcGACrtBWDiIh4CYWbtsK5UnFT593UXhKuScUiIuIlFG7ailPYIRyOnnejnhsREfEOCjdtRcogMCxQtBese11uVnfF1LqcQiqqbe6qTkREpNko3LQVgWGQ0Mdxf6/rvTcdYkKJiwii2mayLsfqpuJERESaj8JNW3IKk4oNw3D23mifKRER8QYKN22JczE/18MNwJCOR/aZEhERae0UbtqSunCTlwk1rq9bU3fF1Krdh7HZtYmmiIi0bgo3bUn7rhDcDmoqoGCDy816JUUQFuhHcUUN2wqK3VefiIhIM1C4aUvqLebn+qRifz8Lg2pXK9ZWDCIi0top3LQ1pzCpGI6ed6P1bkREpHVTuGlrTnmlYvXciIiId1C4aWtShwAGHN4FJftdbjagQzv8LAa51gr2Frq+s7iIiEhL82i4eeKJJxg6dCgRERHEx8czbtw4tm7detJ2s2bNomfPngQHB9OvXz++/vrrFqjWRwRHQVwPx/0mLOYXGuhP3+RIQL03IiLSunk03CxYsIDJkyezdOlS5s6dS3V1NRdffDGlpaUnbLN48WJuuOEGbr31VtasWcO4ceMYN24cGza4fvVPm+ccmmrivJtOWu9GRERaP8M0zVazcMn+/fuJj49nwYIFnHPOOQ2ec91111FaWsqXX37pPHbGGWcwYMAAXnvttZO+R1FREVFRUVitViIjI5utdq+yaiZ8cTd0PAtu/srlZnM25HHHO6vpkRDBt/c0/PmIiIi4Q1N+f7eqOTdWq2PvopiYmBOes2TJEkaNGlXv2OjRo1myZEmD51dWVlJUVFTv1uZ1Otvxdc8SKD3ocrO6nputBcVYy6rdUZmIiMhpO+1wY7PZyMzM5PDh07tE2G63M3XqVEaOHEnfvn1PeF5+fj4JCQn1jiUkJJCfn9/g+U888QRRUVHOW1pa2mnV6RPad4HEfmDaYMuXJz+/Vmx4EJ1jwwBYtUdDUyIi0jo1OdxMnTqVf//734Aj2Jx77rkMGjSItLQ05s+ff8qFTJ48mQ0bNvD++++f8ms0ZPr06VitVuctOzu7WV/fa/Ue5/i66dMmNRvi3ERT692IiEjr1ORw89FHH9G/f38AvvjiC7KystiyZQv33HMPDz744CkVMWXKFL788kt+/PFHUlNTGz03MTGRgoKCescKCgpITExs8PygoCAiIyPr3QToc4Xj684FpzQ0pSumRESktWpyuDlw4IAzSHz99ddcc801dO/enVtuuYX169c36bVM02TKlCnMnj2bH374gfT09JO2GTFiBPPmzat3bO7cuYwYMaJJ793mneLQVN0mmmuzrVRU29xVnYiIyClrcrhJSEhg06ZN2Gw25syZw0UXXQRAWVkZfn5+TXqtyZMn88477/Dee+8RERFBfn4++fn5lJcfWSTupptuYvr06c7Hd999N3PmzOGZZ55hy5YtzJgxg5UrVzJlypSmfityCkNTndqHEhseSJXNzoa9VreUJSIicjqaHG5uvvlmrr32Wvr27YthGM4rl5YtW0bPnj2b9FqvvvoqVquV8847j6SkJOftgw8+cJ6zZ88e8vLynI/PPPNM3nvvPV5//XX69+/PRx99xKefftroJGQ5gaOHpspcG2YyDEP7TImISKvm39QGM2bMoG/fvmRnZ3PNNdcQFBQEgJ+fHw888ECTXsuVJXYamqR8zTXXcM011zTpvaQB7btAQj8oWA+bv4DBE1xqNqRTNHM25tfOu+ni3hpFRESaqMnhBuDqq6+u97iwsJAJE1z7xSitTJ9xjnCz6VOXw03dvJuVuw9jt5tYLIb76hMREWmiJg9LPfnkk/WGja699lrat29Pamoq69ata9bipAWcwtBUn+RIQgP9sJZX88u+EjcWJyIi0nRNDjevvfaacyG8uXPnMnfuXL755hvGjBnDvffe2+wFipvVDU2ZNsfQlAv8/SwM7NAO0D5TIiLS+jQ53OTn5zvDzZdffsm1117LxRdfzH333ceKFU3biFFaiT7jHF+bcNVU3aRirXcjIiKtTZPDTXR0tHOV3zlz5jivljJNE5tN6554pVMYmhraSVdMiYhI69TkcHPllVdy4403ctFFF3Hw4EHGjh0LwJo1a+jatWuzFygt4OihKRcX9BvQoR1+FoO9heXkFpafvIGIiEgLaXK4ee6555gyZQq9e/dm7ty5hIeHA5CXl8edd97Z7AVKC+lzuePrxtkunR4e5E/vJMdWFit3q/dGRERajyZfCh4QENDgxOF77rmnWQoSD+l9Bfzwf0eGpkJjTtpkSKdo1u+1snLXIX7dP7kFihQRETm5JvfcAOzYsYO77rqLUaNGMWrUKP7whz+wc+fO5q5NWlJs1yYPTdXNu1mepUnFIiLSejQ53Hz77bf07t2b5cuXk5GRQUZGBsuWLXMOU4kXa+LQ1JBO0QBsLSjGWl7trqpERESapMnDUg888AD33HMPf/vb3447fv/99zs30hQv1MShqfiIYDq1D2XXwTJW7znM+T3iW6hQERGRE2tyz83mzZu59dZbjzt+yy23sGnTpmYpSjzkFIamhnTSejciItK6NDncxMXFkZmZedzxzMxM4uP1P3ev5xya+tSl04fWDk1pvRsREWktmjwsNWnSJH73u9+xc+dOzjzzTAAWLVrEk08+ybRp05q9QGlhzqGp+S4NTdX13KzNLqSyxkaQv18LFCkiInJiTQ43Dz/8MBERETzzzDNMnz4dgOTkZGbMmMHdd9/d7AVKC6sbmipY7xiaGnRTo6d3jg0jJiyQQ6VVbNhbxOCO0S1UqIiISMOaPCxlGAb33HMPOTk5WK1WrFYrOTk5TJo0icWLF7ujRmlpTRiaMgyDIbWBRvNuRESkNTildW7qREREEBERAcAvv/zC2Wef3SxFiYf1rt1rKsu1vaa0z5SIiLQmpxVuxEfVDU3Za1y6aqpuvZuVuw9ht5vurk5ERKRRCjfSsCYMTfVNiSI4wEJhWTU79pe4ty4REZGTULiRhjVhaCrAz8LANF0SLiIirYPLV0t9/vnnjT6flZV12sVIKxLbFRL6QsEGl66aGtopmiU7D7Jy1yFuHN6hhYoUERE5nsvhZty4cSc9xzCM06lFWps+4xzhZuOnJw03devdrNitK6ZERMSzXB6WstvtJ73ZbDZ31iotrQlDUwM7tMNiQPahcvKtFS1QnIiISMM050ZOrG5oyl4DW75q9NSI4AD6pkQB8MmanJaoTkREpEEKN9K4PuMcXzfOPumpE0Z0AuBfC3dSWlnjvppEREQaoXAjjWvC0NTlA5Lp1D6Uw2XVvL1kdwsUJyIicjyFG2lcE4am/P0s3HVBNwBeX7hDvTciIuIRCjdycnVDU5s+Pemp6r0RERFPczncHD58mBdffJGioqLjnrNarSd8TnxA3dDUzvknHZpS742IiHiay+HmpZdeYuHChURGRh73XFRUFD/99BMvvvhisxYnrUQThqZAvTciIuJZLoebjz/+mDvuuOOEz99+++189NFHzVKUtEK9xzm+ujA0pd4bERHxJJfDzY4dO+jWrdsJn+/WrRs7duxolqKkFaqbd+PC0BSo90ZERDzH5XDj5+dHbm7uCZ/Pzc3FYtH8ZJ8V261JQ1PqvREREU9xOY0MHDiQTz/99ITPz549m4EDBzZHTdJaNWFoCtR7IyIinuFyuJkyZQrPPPMML730Ur09pGw2Gy+++CLPPfcckydPdkuR0ko0cWhKvTciIuIJLoebq666ivvuu48//OEPxMTEMHDgQAYOHEhMTAxTp05l2rRpXH311e6sVTytiUNToN4bERFpeS6Hm4ULFzJjxgyWLl3KxIkTSU5OJikpiZtvvpklS5bwt7/9zZ11SmvRxKEp9d6IiEhLM0zTNF050c/Pj7y8POLj491dk1sVFRURFRWF1WptcM0eOYkDv8BLQ8DiD/f+AqExJ21SY7Mz6tkF7DpYxv1jevL787q0QKEiIuJLmvL72+WeGxczkPi6UxiaUu+NiIi0pCZdu20YhrvqEG/SxKEp0NwbERFpOf5NOXnixIkEBQU1es4nn3xyWgWJF+gzDn78vyNXTbkwNFXXe/PHWWt5feEObhrRkbCgJv31ExERcUmTem4iIiKIiopq9CZtwNFDU1u/drmZem9ERKQlNOm/zv/4xz+8fkKxNJPe46BgA2ycDQN/41IT9d6IiEhLcLnnRvNtpJ4mLuhXR703IiLibrpaSk5NbDeI79PkoSldOSUiIu7mcrj58ccfiYk5+cRRaUP6XOH4uvHTJjVT742IiLiTy+HGZrORkZFBUVHRcc9ZrVb69OnDTz/91KzFSSvnHJr6sUlDU+q9ERERd3I53LzwwgtMmjSpwVUBo6KiuP3223n22Web9OYLFy7ksssuIzk5GcMwGt11HGD+/PkYhnHcLT8/v0nvK83kFIemQL03IiLiPi6Hm8zMTMaMGXPC5y+++GJWrVrVpDcvLS2lf//+vPzyy01qt3XrVvLy8pw3XcHlQac4NKXeGxERcReXr8MtKCggICDgxC/k78/+/fub9OZjx45l7NixTWoDEB8fT7t27Vw6t7KyksrKSufjhobV5DQ4F/T70eUF/epcPiCZF3/4hV0Hy3h7yW7tOSUiIs3C5Z6blJQUNmzYcMLn161bR1JSUrMUdTIDBgwgKSmJiy66iEWLFjV67hNPPFFvkcG0tLQWqbHNOI2hKfXeiIiIO7gcbi655BIefvhhKioqjnuuvLycRx99lF/96lfNWtyxkpKSeO211/j444/5+OOPSUtL47zzzmP16tUnbDN9+nSsVqvzlp2d7dYa26S6icVNHJoCzb0REZHmZ5guLmBTUFDAoEGD8PPzY8qUKfTo0QOALVu28PLLL2Oz2Vi9ejUJCQmnVohhMHv2bMaNG9ekdueeey4dOnTgv//9r0vnN2XLdHHR/m3w8lCw+MP/2w4h0U1q/vGqHP44ay3RoQH8fP8FWrVYRESO05Tf3y733CQkJLB48WL69u3L9OnTueKKK7jiiiv405/+RN++ffn5559POdicjmHDhrF9+/YWf185Slz3I0NTW75qcnP13oiISHNq0saZHTt25Ouvv+bAgQMsW7aMpUuXcuDAAb7++mvS09PdVWOjMjMzW2yujzTiNIamNPdGRESa0yn1/0dHRzN06NDTfvOSkpJ6vS5ZWVlkZmYSExNDhw4dmD59Onv37uXtt98G4Pnnnyc9PZ0+ffpQUVHBG2+8wQ8//MB333132rXIaeo9Dn78yyldNQW6ckpERJpPk3pumtvKlSsZOHAgAwcOBGDatGkMHDiQRx55BIC8vDz27NnjPL+qqoo//vGP9OvXj3PPPZe1a9fy/fffc+GFF3qkfjlKXHdI6OcYmvrmfmjiXmTqvRERkebi8oRiX6EJxW60ewnMvBRMG/zqeRhyc5Oa19jsjHp2AbsOlnH/mJ7qvRERESe3TCgWOamOI2DUo47739wPeWub1Fy9NyIi0hwUbqR5jbgLuo8FWyV8OAEqrE1qriunRETkdCncSPOyWGDcKxDVAQ5nwWeTmzT/Rr03IiJyuhRupPmFxsC1M8ESAJu/gGWvNam5em9EROR0KNyIe6QMhtF/ddz/7iHIXuFyU/XeiIjI6VC4EfcZNsmx/o29Bj662bH+jYvUeyMiIqdK4UbcxzDg1y9CTBewZsPs28Fud6mpem9ERORUKdyIewVHwrVvgX8w/PIdLHre5aZH997MXLzLbSWKiIhvUbgR90vsB5c85bj/w59h188uNfP3s/CHCx29N8/N3cbSnQfdVaGIiPgQhRtpGQN/C/1vANMOH90KJftcanbFwBR+lZFEjd3kjndWsetAqZsLFRERb6dwIy3DMODSZyCuJ5Tkw8e3gt3mQjODp6/pT/+0dhSWVXPrWyuwlle3QMEiIuKtFG6k5QSGwbVvQ0AYZC2EBU+61Cw4wI9//XYwSVHB7NhfypT3VlNjc21isoiItD0KN9Ky4nrAZS847i/4O2yf51Kz+Mhg3pgwhNBAP3765QCPfbHJjUWKiIg3U7iRlpdxDQy+GTDhk0lg3etSsz7JUTx/3QAMA/67dDdv6QoqERFpgMKNeMaYvzmuoio7CB/dAjbX5tFc3CeR+8f0BOCxLzayYNt+d1YpIiJeSOFGPCMg2DH/JigSspfCvMddbnr7OZ25alAqdhOmvLua7fuK3VioiIh4G4Ub8ZyYznD5y477i/8BW752qZlhGPz1yr4M6xRDcWUNt8xcyaHSKjcWKiIi3kThRjyr96/hjDsd9z+9Aw7vcqlZkL8fr/12MGkxIew5VMYd76yiqkZXUImIiMKNtAajHoOUIVBhhVkToabSpWYxYYG8OWEoEUH+LM86xIOz12OapntrFRGRVk/hRjzPPxCumQkh0ZC7Br57yOWm3RIiePHGgVgMmLUqh9cX7nRfnSIi4hUUbqR1aJcGV/zTcX/567DhY5ebntcjnkd+1RuAv83ZwtxNBe6oUEREvITCjbQe3UfDWdMc9z//AxzY7nLTCWd24jdndMA04e7317Ax1+qmIkVEpLVTuJHW5fwHoeNIqCqBD2+C6nKXmhmGwaOX9eGsrrGUVdmY9NZK9hVXuLlYERFpjRRupHXx84er/g1hcbBvI3z9/1xuGuBn4eXxg+gcF0autYJJb6+iovrkm3OKiIhvUbiR1icyCa56AzBgzX8h8z2Xm0aFBPDmhKFEhQSwNruQ//fROl1BJSLSxijcSOvU+Tw4/0+O+19OgwLXN8rsFBvGa78ZjL/F4Iu1ubww7xf31CgiIq2Swo20XmffC10ugJpyeOcqOOB6SBnRpT3/N64vAM9//wtfrM11V5UiItLKKNxI62WxwJX/grieUJwL/xkL+Rtcbn79sA7cdlY6APfOWktmdqGbChURkdZE4UZat7BYmPiVYwfx0v0w81LYu8rl5tMv6cWFPeOprLFz21sryS107eorERHxXgo30vqFxcKELyF1KFQUwluXw+7FLjX1sxi8cMNAeiZGcKCkklvfWklpZY176xUREY9SuBHvENIOfjsbOp0NVcXw3ythxw8uNQ0P8ueNCUOIDQ9kc14RUz/IxG7XFVQiIr5K4Ua8R1AEjJ8FXS9yTDJ+7zrY+o1LTVOjQ/nnb4cQ6G9h7qYCnvx2i5uLFRERT1G4Ee8SEALXvwu9LgNbFXzwG5f3oRrcMZq/X5UBwD8X7GTWymx3VioiIh6icCPexz8Irp4JGdeBvQY+vg3WvONS03EDU7jrgq4A/Gn2ej5UwBER8TkKN+Kd/Pxh3GsweCKYdvhsMiz/l0tN7xnVnXEDkqm2mdz30TpmfL6RapvdvfWKiEiLUbgR72WxwK+ehzPudDz++l74+XkXmhk8e+0A7hnVHYCZi3cx4c3lHC6tcl+tIiLSYhRuxLsZBoz+K5xTu8Hm94/CD3+Bk+wnZbEY3D2qG//87WDCAv1YvOMgv375Z7bkF7VA0SIi4k4KN+L9DAMueAgufNTxeOHf4buHThpwAEb3SeSTO0fSISaU7EPlXPnKYr5Zn+fmgkVExJ0UbsR3nD0Nxv7dcX/JS/DlPWA/+VyaHokRfD5lJGd1jaWsysbv313Ns3O3aS0cEREvpXAjvmX47fDrlwADVv0HPv092E6+InG70EBm3jyUW0Y69qL6x7xfuP2dVZRoNWMREa+jcCO+Z9Bv4ao3wPCDde/DRzdDzcknC/v7WXjkst48dXUGgX6Oxf6ufGURuw+WtkDRIiLSXBRuxDf1uxqu+y/4BcLmz+GD8VDt2qaZ1wxJ44PbzyA+IohtBSX8+qVF/PTLfjcXLCIizUXhRnxXz0vhhvfBPwR++Q7evQYqS1xqOrBDNF/cdRYD0tphLa9mwpvLeeOnnZguTFIWERHPUrgR39b1QvjNxxAYDrt+gneuhPJCl5omRAbz/u/O4KpBqdhN+L+vNvPHWWupqLa5t2YRETktCjfi+zqNhJs+h+AoyF4Gb/8aSg+61DQ4wI+nr8ngkV/1xs9i8MnqvVz3+lLyrRVuLlpERE6Vwo20DamDYeJXEBoLeWth5qVQnO9SU8MwuOWsdN66eRhRIQGszS7k1y/9zOo9h91ctIiInAqFG2k7EvvBzd9ARBLs3wxvjobdi11ufla3WD6fMpLuCeHsK67k+n8u1cabIiKtkEfDzcKFC7nssstITk7GMAw+/fTTk7aZP38+gwYNIigoiK5duzJz5ky31yk+JK67I+C06wiHd8F/xsLsO6Bkn0vNO7YP45M7RzK6TwJVNjv3fbSOx77YSI023hQRaTU8Gm5KS0vp378/L7/8skvnZ2Vlcemll3L++eeTmZnJ1KlTue222/j222/dXKn4lJh0+N18GHwzYMDa/8GLQ2DZ62A/+WTh8CB/Xh0/mKmjugHwn0W7uEkbb4qItBqG2UqubTUMg9mzZzNu3LgTnnP//ffz1VdfsWHDBuex66+/nsLCQubMmdNgm8rKSiorK52Pi4qKSEtLw2q1EhkZ2Wz1i5fKWQVfTYO8TMfjxAy49FlIG+pS8zkb8pn2YSZlVTY6xITyr5uG0CMxwn31ioi0UUVFRURFRbn0+9ur5twsWbKEUaNG1Ts2evRolixZcsI2TzzxBFFRUc5bWlqau8sUb5I6GCb9AJc+47iaKn8d/HsUfDbFpSuqxvRN5JM7zyQtJoQ9h8q44pVFfLwqR+vhiIh4kFeFm/z8fBISEuodS0hIoKioiPLyhlefnT59Olar1XnLztYEUDmGxQ+G3gZTVsGA8Y5ja/4LLw2Glf856eabPRMj+XzyWYzs2p6yKht/nLWWG/+1jO37XFswUEREmpdXhZtTERQURGRkZL2bSIPC42DcK3DLt5DQF8oPw5dTHT05uWsabRodFshbNw/j/43uQXCAhSU7DzL2hYU8/e1WLfonItLCvCrcJCYmUlBQUO9YQUEBkZGRhISEeKgq8TkdzoDfLYAxf4PACNi7Cl4/H76c5gg8J+DvZ2Hy+V2Ze8+5XNAznmqbyUs/bufi5xby41bXrsYSEZHT51XhZsSIEcybN6/esblz5zJixAgPVSQ+y88fzvg93LUS+l0DmLDy346rqta82+hQVVpMKP+eMITXfjOYpKhg9hwq4+b/rODOd1dpZWMRkRbg0XBTUlJCZmYmmZmZgONS78zMTPbs2QM45svcdNNNzvPvuOMOdu7cyX333ceWLVt45ZVX+PDDD7nnnns8Ub60BRGJcNUbMOELiO0BZQfgszsd6+PkbzhhM8MwGNM3ke+nncuks9Pxsxh8vT6fC5+Zz79/ztK6OCIibuTRS8Hnz5/P+eeff9zxCRMmMHPmTCZOnMiuXbuYP39+vTb33HMPmzZtIjU1lYcffpiJEye6/J5NuZRMpJ6aKlj2Ksx/EqpLwfCDYb+D8/8EwY3/XdqcV8SDs9ezek8hAL2TIvnLFX0Z2CG6BQoXEfF+Tfn93WrWuWkpCjdy2qx74dvpsOkzx+PwBLj4L9DvajCMEzaz200+WJnN377ZgrW8GsOAG4d14L7RPYkKDWih4kVEvJPCTSMUbqTZbJ8HX/8/OLTD8bjT2XDJ0xDfs9FmB0oqeeLrLXy8OgeA2PBAHry0F+MGpGA0Eo5ERNoyhZtGKNxIs6qphMX/gIXPQE05WPxhwI0w/PeQ0LvRpkt3HuShTzc418MZ0bk9fx7Xl67x4S1RuYiIV1G4aYTCjbjF4d0w5wHY+vWRY+nnwPA7oPsYx0KBDaiqsfOvn3byj3m/UFljJ8DP4PZzujDlgq4EBzTcRkSkLVK4aYTCjbjV7sWw9FXY8iWYtVdEtevomHg88DcQ0q7BZtmHynjksw38uHU/AB1iQnns8j6c3yO+hQoXEWndFG4aoXAjLaIwG1b8C1a9BRWFjmMBYTDgBhh2O8R1P66JaZp8uzGfx77YRF7tejiX9EvkkV/1ITEquAWLFxFpfRRuGqFwIy2qqgzWfwhLX4P9m48c73KhY8iq6yiw1F9uqqSyhufnbuM/i3dhs5uEBfoxdVR3fjuio4aqRKTNUrhphMKNeIRpQtZCWPYabP0GqP2xi+kCw293TEIOiqjXZFNuEQ9+up41tWvjxIYHcstZ6fzmjI5EBuvScRFpWxRuGqFwIx53KAtWvAGr/wuVVsexwAjHnJxhk6B9F+epdrvJrFXZ/GPedvYWlgMQEeTPTWd25OaR6cSGB3niOxARaXEKN41QuJFWo7IE1v4Plv0TDv5Se9CA7qMdvTmdz3cuClhts/PF2lxemb/Deel4cICF64d2YNI5nUlpp41jRcS3Kdw0QuFGWh27HXb+4JiXs33ukeOxPRwhp//1EBhWe6rJ3M0FvPzjdtblOHp9/C0GVwxM4Y7zutAlTmvkiIhvUrhphMKNtGoHtsPyf0Lme1Dl6KEhOAp6XAIBoeAXCH7+mJYAsq01LNllZeehKmrwoxp/eqTEcG7PZFJjoxwLCvoFONrUux/g2PU8INQx58fi0f1zRURconDTCIUb8QoVVkfAWfZPOJzlvveJ7QFn/xH6XuUIPCIirZTCTSMUbsSr2G2OPazy14KtBmxVYK8+/r69muLSMnbtK+SAtRR/aggwbLQLMkiOsBARAIa9tk3t+ZQfhhrHejpEd4KzpkH/G8A/0KPfsohIQxRuGqFwI75u98FS/rlwJx+tzKHK5lgluVdSJHee14VL+iXhZ6ndnLPC6rhqa8nLUHbQcSwyBUZOhUG/hQBNUhaR1kPhphEKN9JWFBRV8O+fs3h36W5Kq2wAdGofyh3nduGKQSkE+dcuCFhV6lhJedELUJLvOBYWD2dOgSG3HLf+joiIJyjcNELhRtqawrIq3lq8m/8szqKwrBqAxMhgbjs7neuGphFRtyBgdQVkvgs/Pw/WPY5jIdFwxp2OvbFOsC+WiEhLULhphMKNtFWllTX8b/ke/vXTTgqKKgEICfDjVxlJXDc0jcEdozEMA2zVsH4W/PQMHNzuaBwY4VhgcMRkCIv14HchIm2Vwk0jFG6kraussfHJ6r288dNOduwvdR7vEhfGdUPTuHJQqmPlY7sNNn0KC5+GfZscJwWEwuCb4cy7IDLJM9+AiLRJCjeNULgRcTBNk1W7D/PBimy+XJdHebVjXo6/xWBUrwSuG5rGOd3j8MOEbd/Awqcgd42jsV8gDPwtjLwbojt68LsQkbZC4aYRCjcixyuuqObLdXl8sCKbzOxC5/GkqGCuHpzKtUPSSIsOgR3zHD05e5Y4TrD4Q8b1cNY9ENvVM8WLSJugcNMIhRuRxm3NL+aDFdl8sibHOQEZYGTX9lw7JI3RfRIJ3rvU0ZOz80fHk4YF+lzhWBAwoY+HKhcRX6Zw0wiFGxHXVNbYmLupgA9WZPPz9gPU/UsRFRLAFQNTuG5oGr1s2xw9Odu+OdKwy4XQ7SLofB7E9XRu/ikicjoUbhqhcCPSdDmHy5i1ModZK7PJtVY4j2ekRnHd0DQuTzxE+PIXYOOnwFH/pIQnOkJO5/Og87kQmdzClYuIr1C4aYTCjcips9lNft5+gA9XZPPdpnyqbY5/PoIDLFzaL5kJ3avpV7oYY+ePsHvxke0d6sT1PBJ2Oo6EYP0MiohrFG4aoXAj0jwOllQye81ePliRzS/7SpzH02PDuGpQCuP6tSe1ZAPs+BF2zq+90uqof24MP0gdeiTspA5x7FwuItIAhZtGKNyINC/TNFmTXciHK7L5Ym2uc6sHgDM6x3DVoFTG9ksi3FYEu352TELeOR8O7az/QoHhjt6cLudrvo6IHEfhphEKNyLuU1pZw5wN+XyyJofFOw46JyEHB1gY0yeRKwelMrJrrGPzzsO7HSFn53zIWnBk88464QlHenXSa+frKOyItFkKN41QuBFpGXsLy/l0zV4+Xp3DzqNWQk6IDGLcwBSuGpRK94TaTTntdijYcKRXp6H5OkFRENMJotMhJv2or50cu5lb/FrqWxMRD1C4aYTCjUjLMk2TtTlWPlmdw+drc+utndMvJYorB6Xw6/7JtA8POtKougJylp94vs6x/AKhXYeGg090JwgIcdN3JyItReGmEQo3Ip5TVWPnhy37+GR1Dj9s2UeN3fHPj7/F4Lwe8Vw1KIULesUT5H9ML0xVGRTuhkNZcDjryNfDuxzDW/bq49/saBFJ9YNPdKcj90NjNNwl4gUUbhqhcCPSOhwqreKLtbl8sjqHtTlW5/GokAAu65/ElYNSGZjWzrFTeWPsNijae0zw2VV7fxdUWhtvHxRZP+zEdD5yPzJZw10irYTCTSMUbkRan18KivlkzV5mr95LftGRuTadY8O4clAK4wamkBod2vQXNk0oP3xUT09t4KkLQcW5jbf3C4R2HY8Z6qr92q4jBAQ3vSYROSUKN41QuBFpvWx2kyU7DvLJ6hy+2ZDv3KkcHPNzzusRx3k94hiQFu244up0VZc7hrWOHupyBqGTDXcZjp6d6PSjJjp3doSewFBHMPIPcnw9+r56gkROicJNIxRuRLxDaWUN32zI55PVOSzZeeSycoB2oQGc3S2O87rHcW6POGKPnozcXOw2sOY0HHwO7YKq4lN7XcOvNugEgF/QMQEo0HGs3v0AxzkRiZA8CFIGOyZPa56QtDEKN41QuBHxPvuKK1i47QDzt+5j4bb9FFXU1Hs+IzWK87rHcV7PePqntmueXp3GmKZjXZ7jQk8WWLMdPUK2arBVgq2q+d8/tH1t0KkNO8mDIDyu+d9HpBVRuGmEwo2Id6ux2cnMLmT+1v38uHUfG3OL6j0fXdurc37POM7pFlf/EnNPMM0jQaemyhF2jr1vq4aa2iBkqzr+fk2lY0XnvaugYGPDw2VRaZA80BF2UgZB0gDt3SU+ReGmEQo3Ir5lX1EFC7btZ/7W/Sz8ZT/FR/XqGAZkpERxbo94zu8RR0ZL9Oq4W3WFI+DsXQW5q2HvajiwjePXATIgtrsj6NT18iT01SRod7Lb4eB2KDsAKUMcQ4vSbBRuGqFwI+K7amx2Vu8pZP7Wfczfup9Necf36pzTPY7ze8RzTvc4YsJ85JdPRRHkZTqCTl3gsWYff54lABL6HAk8Sf0hLBaCIiAgDCyWFi/dq5Xsg5yVjqC5dyXsXXNk6YGQGOh7FWRc59gUVnOkTpvCTSMUbkTajoKiChZs3c/8bfv4adsBiivr9+r0TY5ieHoMwzu3Z1inGKJCfWhX8pJ9jpWd9646EnqO3b+rHsOx5k9QxJFb8NGPI+s/73zumK+BYUd+kZsmmHbH5GzTdsxXO9hrGjh2gnMNi2MxxvAEz4SwqjLIW+sIMTkrawPknuPP8w+GgFAoP3TkWEwXR8jJuMZxRZ2cEoWbRijciLRN1TY7q3cfZv62/fy4ZR9b8utf7WQY0CMhgjM6t2dYegzD0mPccxWWp5imY5Xno3t39m2GCqsjRDQXw+K4IsysDSXNzRLguAQ/Kg2iUmtvKfUfB0Wc3nvYbY6hvqN7ZQo2NfDnZEBcD8cQVOpgx3yn+N6O41kLYN0HsPkLqC470iRtOGRcC32udKyOLS5TuGmEwo2IAORbK1i68yDLsg6xLOtgvc0963SJC2NYenvO6OwIO0lRPrhHlWk6ru6qLK69WY+6X+wY8qoshsqio44fc7/unCaHJMOx7o/hd9RXyzGPa2+2GijJdy0wBUdBZOpR4Se1NvykOO5HJDkusa9TlFc7rLTKEWhyMxu+1D880THElDLIEWiSB5580nZlCWz5Cta979gnra5+SwB0H+0IOt3HOC73l0Yp3DRC4UZEGrK/uJLlWYdYnuUIPMf27AB0iAllWHqMYygrvT1pMSEn3x6irXCGpCJHz0e90GJpIMT4NX0eiq0GivMc221Ycxzziqw5tbe9jscVhSd/nbohrshkKMp1vN6xAsJqrz4bVBtoBjt2nz+dz7s4H9Z/5Ag6+euPHA+Ogj5XOIau0s5o/XOfTBOqSh2BtqqkNuyWHAm8VSWOXqm+VzXr2yrcNELhRkRcUVhWVRt2DrEs6xAbc63Yj/nXMikquDbsOIayusSFKex4WmVxbdCpDT/OIFQXhvYefym9YYG4XrVDS7VBJq4n+Pm7r86CTY5hq/Wz6oerdh2g37XQ/3qI7db872u3OwJg2cHa26GjAkrxkZBywmO1Aea4q/OOkTYcbv2uWUtXuGmEwo2InIriimpW7T7MstrAsy6nkGpb/X8+Y8MDGZYew5COMQzuGE3v5EgC/Fr5/8LbGrsdSvc7wk5RjmNBxKQBEBTuuXp2/wxrP4BNn9UfDkse5OjN6XvViRdptNU4Ji+XHYTSA47L0EsPNPL4YPPNsTL8HH9uQZEQGH7U5PNwRzg8/0/N8z61FG4aoXAjIs2hvMrGmj2HWVo7lLVmTyGVNfXng4QE+NE/LYrBHaMZ0jGGQR2ifeuKLGleVWWw7RtH0Nn+/ZEQYvhB1wsdw2jHBpfyQk7ai9KQoEhHsAuNORJKAiPqB5QGjx0VZAJCWvQSd4WbRijciIg7VNbYWJdjZXnWIVbtPsyq3Yexlh+/knC3+HCGdIpmcMcYhnSMpmP7UA1lyfFK9sPGT2Dt+46r2xplQEi0Y82i0FhHYKm77/za/qjH7b1yArPXhZuXX36Zp556ivz8fPr378+LL77IsGHDGjx35syZ3HzzzfWOBQUFUVFR4dJ7KdyISEuw20127C9h1e7DrKwNO1kHjr8iKzY8kEEdomsDTzR9U6II8tfO4XKUA784hqzsNY5gcmxwCYl27/ygVqIpv789/qfxwQcfMG3aNF577TWGDx/O888/z+jRo9m6dSvx8fENtomMjGTr1q3Ox/pfj4i0NhaLQbeECLolRHD9sA4AHCypdPbqrNp9mHU5Vg6UVPHdpgK+21QAQKC/hYwUx1BW3c3j+2OJZ8V2g3Pu9XQVXsXjPTfDhw9n6NChvPTSSwDY7XbS0tK46667eOCBB447f+bMmUydOpXCwsJTej/13IhIa1FZY2PDXqujd2eXI/AcLD1+F/H02DAGdmhHRkoUGWnt6J0USXCAenekbfGanpuqqipWrVrF9OnTnccsFgujRo1iyZIlJ2xXUlJCx44dsdvtDBo0iL/+9a/06dOnwXMrKyuprKx0Pi4qKmrwPBGRlhbk78fgjjEM7hjD784B0zTZdbCstmfnECt3HeaXfSVkHSgl60Apn6x2XDLsZzHonhBBRkoU/VKjyEiNomdiJIH+ujJLBDwcbg4cOIDNZiMhIaHe8YSEBLZs2dJgmx49evDmm2+SkZGB1Wrl6aef5swzz2Tjxo2kpqYed/4TTzzBY4895pb6RUSak2EYpMeGkR4bxtWDHf+eWcuqWb3nMGtzClmXY2VdTiEHSqrYnFfE5rwiPljp2CAz0M9Cz6QI+qU4wk6/lHZ0TwjHX5eiSxvk0WGp3NxcUlJSWLx4MSNGjHAev++++1iwYAHLli076WtUV1fTq1cvbrjhBv785z8f93xDPTdpaWkalhIRr2SaJvlFFazLsbI+x8q6vY7AU1h2/JVZQf4W+iRHkpHazhl6OseF42fRPEXxPl4zLBUbG4ufnx8FBQX1jhcUFJCYmOjSawQEBDBw4EC2b9/e4PNBQUEEBWkynoj4BsMwSIoKISkqhNF9HP9OmqZJzuFyR8/O3kLW1waf4soaVu8pZPWeQmf70EA/+ibX9u6kRtEnOZL0WAUe8S0eDTeBgYEMHjyYefPmMW7cOMAxoXjevHlMmTLFpdew2WysX7+eSy65xI2Vioi0XoZhkBYTSlpMKJdmJAGOS9F3HSxl/V6rs5dnQ66Vsioby3cdYvmuQ872wQEWeiZG0ic5kt7JkfRJjqJHQgQhgZq0LN7J45eCT5s2jQkTJjBkyBCGDRvG888/T2lpqXMtm5tuuomUlBSeeOIJAB5//HHOOOMMunbtSmFhIU899RS7d+/mtttu8+S3ISLSqlgsBp3jwukcF87lA1IAsNlNdu4vYW2OlfU5hazfa2VzXjHl1TYyswvJzC480t6AznHhjsCT5Ag8vZMjiQkL9NB3JOI6j4eb6667jv379/PII4+Qn5/PgAEDmDNnjnOS8Z49e7ActUPq4cOHmTRpEvn5+URHRzN48GAWL15M7969PfUtiIh4Bb+j1t6pm7Bsq+3h2ZRbxMbcIjblFbEp17H+zvZ9JWzfV8JnmbnO10iMDD6qhyeS3klR2h1dWh2Pr3PT0rTOjYhI40zTZH9xpTPsbMy1sim3iF0Hyxo8PyLIn17OHp5IuiVE0DEmlHahAQo90my8bvuFlqRwIyJyakoqa9icV1Tby2NlU14R2/JLqLLZGzw/Itifju1D6RgTRof2oXSMCXV8bR9GUmQwFk1iliZQuGmEwo2ISPOpqrGzY3+Jo5cnt4hNeVZ27i9lX3Flo+0C/SykxoTQMcYRdjrEhDqCUPtQUqNDtQKzHEfhphEKNyIi7ldeZWPPoTJ2Hyyt/VrG7kNl7DlYSs7hcmrsJ/7VYxiOuT1HAk8YHduHOhc4DA30+HRR8QCFm0Yo3IiIeFaNzU6etaI28JSy52D98FNaZWu0fXJUMOlxYXSODadzXJjjqrDYMJLbhWi9Hh+mcNMIhRsRkdbLNE0Ollax+2AZew6VOr4eLGPXwVJ2HihtcCXmOoH+FtLbh9E5ztHD47gUPowuseFEhQa04Hch7uA1KxSLiIgczTAMYsODiA0PYnDH6OOeP1xaxc4DJezY79hMdOf+Enbud4Sgqho7WwuK2VpQfFy7mLBAOseGOXt60mPD6BIXRlpMKEH+mt/ja9RzIyIiXs9mN9l7uJwdBxxhZ+f+ktrwU0p+UcUJ2xkGJEUGkxYTSofaW91qzx1iQokND9Tl7K2EhqUaoXAjItK2lFbWOILOUT09db0+J5vfExLg5ww8jvATQof2jvu6qqtlKdw0QuFGRETgyPyePYfKyD7kmNuTfbis9nE5udZyTvYbMiEy6EhvT3RtAGofSkq7EOIjgvD3szT+AuIyhZtGKNyIiIgrKmts5BZWsOdQWb0AVHe/uLKm0fZ+FoPEyGBSokNIaVd7iw4hud2Rx9qc1HWaUCwiInKagvz9nGvrHMs0TQrLqh1B5/BR4af2lm+toNpmsrewnL2F5Sd8j5iwQGfQSa4NP0cHoWhtYXFKFG5ERESayDAMosMCiQ4LpH9au+Oet9kd+3PVhZu9h8vJPer+3sJySiprOFRaxaHSKtbvtTb4PiEBfiS3CyYlOpSkyGASIoOIjwwmofZ+QmQw7cMCNfx1DIUbERGRZuZnMUiMCiYxKrjBS9oBrOXV9UPPMeFnf3El5dU2duwvZcf+0hO+l8WA2PAgZ+CJjwwmIeJI+Imv/RoTGthm9vNSuBEREfGAqJAAokIC6J3c8PyRyhobeYUVzsBTUFRBQXEFBUWV7CtyfN1fUonNbrKvuJJ9xZWs33vi9/O3GMRH1PX81IWhYBIjg0lqF0xSVAiJkcE+MQ9I4UZERKQVCvL3o1NsGJ0amPNTx2Y3OVhayb6iSkf4qf26r/jI/YKiSg6WVlJjN8m1VpBrPfG6PwDRoQEkRoWQVNvzlBwV7HzsuLX+idAKNyIiIl7Kz2IQHxFMfEQwfVOiTnhetc3OgZLKI+GnNvTkF1WQb60gz1pOnrWCsiobh8uqOVxWzea8ohO+XlRIgDPsHBt8EmvvhwV5LmIo3IiIiPi4AD8LSVEhJEWFnPAc0zQpqqipF3byrBXkH3U/r7Cc0iob1vJqrOXVbMk/fqsLgO4J4Xx3z7nu+nZOSuFGREREMAzDOQ+oR2LECc8rqqiuDUCOsOMIQBXkFTke51srGg1RLUHhRkRERFwWGRxAZHAA3RNOHICqauwtWNHxdGG8iIiINKtAf8/GC4UbERER8SkKNyIiIuJTFG5ERETEpyjciIiIiE9RuBERERGfonAjIiIiPkXhRkRERHyKwo2IiIj4FIUbERER8SkKNyIiIuJTFG5ERETEpyjciIiIiE9RuBERERGf4u/pAlqaaZoAFBUVebgSERERcVXd7+263+ONaXPhpri4GIC0tDQPVyIiIiJNVVxcTFRUVKPnGKYrEciH2O12cnNziYiIwDCMZn3toqIi0tLSyM7OJjIysllfW5qXPivvos/Le+iz8h7e9lmZpklxcTHJyclYLI3PqmlzPTcWi4XU1FS3vkdkZKRX/EURfVbeRp+X99Bn5T286bM6WY9NHU0oFhEREZ+icCMiIiI+ReGmGQUFBfHoo48SFBTk6VLkJPRZeRd9Xt5Dn5X38OXPqs1NKBYRERHfpp4bERER8SkKNyIiIuJTFG5ERETEpyjciIiIiE9RuGkmL7/8Mp06dSI4OJjhw4ezfPlyT5ckDZgxYwaGYdS79ezZ09NlCbBw4UIuu+wykpOTMQyDTz/9tN7zpmnyyCOPkJSUREhICKNGjeKXX37xTLFy0s9r4sSJx/2sjRkzxjPFtnFPPPEEQ4cOJSIigvj4eMaNG8fWrVvrnVNRUcHkyZNp37494eHhXHXVVRQUFHio4tOncNMMPvjgA6ZNm8ajjz7K6tWr6d+/P6NHj2bfvn2eLk0a0KdPH/Ly8py3n3/+2dMlCVBaWkr//v15+eWXG3z+73//O//4xz947bXXWLZsGWFhYYwePZqKiooWrlTg5J8XwJgxY+r9rP3vf/9rwQqlzoIFC5g8eTJLly5l7ty5VFdXc/HFF1NaWuo855577uGLL75g1qxZLFiwgNzcXK688koPVn2aTDltw4YNMydPnux8bLPZzOTkZPOJJ57wYFXSkEcffdTs37+/p8uQkwDM2bNnOx/b7XYzMTHRfOqpp5zHCgsLzaCgIPN///ufByqUox37eZmmaU6YMMG8/PLLPVKPNG7fvn0mYC5YsMA0TcfPUkBAgDlr1iznOZs3bzYBc8mSJZ4q87So5+Y0VVVVsWrVKkaNGuU8ZrFYGDVqFEuWLPFgZXIiv/zyC8nJyXTu3Jnx48ezZ88eT5ckJ5GVlUV+fn69n7OoqCiGDx+un7NWbP78+cTHx9OjRw9+//vfc/DgQU+XJIDVagUgJiYGgFWrVlFdXV3v56tnz5506NDBa3++FG5O04EDB7DZbCQkJNQ7npCQQH5+voeqkhMZPnw4M2fOZM6cObz66qtkZWVx9tlnU1xc7OnSpBF1P0v6OfMeY8aM4e2332bevHk8+eSTLFiwgLFjx2Kz2TxdWptmt9uZOnUqI0eOpG/fvoDj5yswMJB27drVO9ebf77a3K7g0raNHTvWeT8jI4Phw4fTsWNHPvzwQ2699VYPVibiW66//nrn/X79+pGRkUGXLl2YP38+F154oQcra9smT57Mhg0bfH6uoXpuTlNsbCx+fn7HzSovKCggMTHRQ1WJq9q1a0f37t3Zvn27p0uRRtT9LOnnzHt17tyZ2NhY/ax50JQpU/jyyy/58ccfSU1NdR5PTEykqqqKwsLCeud788+Xws1pCgwMZPDgwcybN895zG63M2/ePEaMGOHBysQVJSUl7Nixg6SkJE+XIo1IT08nMTGx3s9ZUVERy5Yt08+Zl8jJyeHgwYP6WfMA0zSZMmUKs2fP5ocffiA9Pb3e84MHDyYgIKDez9fWrVvZs2eP1/58aViqGUybNo0JEyYwZMgQhg0bxvPPP09paSk333yzp0uTY9x7771cdtlldOzYkdzcXB599FH8/Py44YYbPF1am1dSUlLvf/VZWVlkZmYSExNDhw4dmDp1Kv/3f/9Ht27dSE9P5+GHHyY5OZlx48Z5rug2rLHPKyYmhscee4yrrrqKxMREduzYwX333UfXrl0ZPXq0B6tumyZPnsx7773HZ599RkREhHMeTVRUFCEhIURFRXHrrbcybdo0YmJiiIyM5K677mLEiBGcccYZHq7+FHn6ci1f8eKLL5odOnQwAwMDzWHDhplLly71dEnSgOuuu85MSkoyAwMDzZSUFPO6664zt2/f7umyxDTNH3/80QSOu02YMME0Tcfl4A8//LCZkJBgBgUFmRdeeKG5detWzxbdhjX2eZWVlZkXX3yxGRcXZwYEBJgdO3Y0J02aZObn53u67Dapoc8JMP/zn/84zykvLzfvvPNOMzo62gwNDTWvuOIKMy8vz3NFnybDNE2z5SOViIiIiHtozo2IiIj4FIUbERER8SkKNyIiIuJTFG5ERETEpyjciIiIiE9RuBERERGfonAjIiIiPkXhRkRERHyKwo2ItHmGYfDpp596ugwRaSYKNyLiURMnTsQwjONuY8aM8XRpIuKltHGmiHjcmDFj+M9//lPvWFBQkIeqERFvp54bEfG4oKAgEhMT692io6MBx5DRq6++ytixYwkJCaFz58589NFH9dqvX7+eCy64gJCQENq3b8/vfvc7SkpK6p3z5ptv0qdPH4KCgkhKSmLKlCn1nj9w4ABXXHEFoaGhdOvWjc8//9y937SIuI3CjYi0eg8//DBXXXUVa9euZfz48Vx//fVs3rwZgNLSUkaPHk10dDQrVqxg1qxZfP/99/XCy6uvvsrkyZP53e9+x/r16/n888/p2rVrvfd47LHHuPbaa1m3bh2XXHIJ48eP59ChQy36fYpIM/H0tuQi0rZNmDDB9PPzM8PCwurd/vKXv5imaZqAeccdd9RrM3z4cPP3v/+9aZqm+frrr5vR0dFmSUmJ8/mvvvrKtFgsZn5+vmmappmcnGw++OCDJ6wBMB966CHn45KSEhMwv/nmm2b7PkWk5WjOjYh43Pnnn8+rr75a71hMTIzz/ogRI+o9N2LECDIzMwHYvHkz/fv3JywszPn8yJEjsdvtbN26FcMwyM3N5cILL2y0hoyMDOf9sLAwIiMj2bdv36l+SyLiQQo3IuJxYWFhxw0TNZeQkBCXzgsICKj32DAM7Ha7O0oSETfTnBsRafWWLl163ONevXoB0KtXL9auXUtpaanz+UWLFmGxWOjRowcRERF06tSJefPmtWjNIuI56rkREY+rrKwkPz+/3jF/f39iY2MBmDVrFkOGDOGss87i3XffZfny5fz73/8GYPz48Tz66KNMmDCBGTNmsH//fu666y5++9vfkpCQAMCMGTO44447iI+PZ+zYsRQXF7No0SLuuuuulv1GRaRFKNyIiMfNmTOHpKSkesd69OjBli1bAMeVTO+//z533nknSUlJ/O9//6N3794AhIaG8u2333L33XczdOhQQkNDueqqq3j22WedrzVhwgQqKip47rnnuPfee4mNjeXqq69uuW9QRFqUYZqm6ekiREROxDAMZs+ezbhx4zxdioh4Cc25EREREZ+icCMiIiI+RXNuRKRV08i5iDSVem5ERETEpyjciIiIiE9RuBERERGfonAjIiIiPkXhRkRERHyKwo2IiIj4FIUbERER8SkKNyIiIuJT/j/k1MR9qIDFfgAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class CTCGreedyDecoder(torch.nn.Module):\n",
        "  def __init__(self, labels, blank=0):\n",
        "    super(CTCGreedyDecoder, self).__init__()\n",
        "\n",
        "    self.labels = labels\n",
        "    self.blank = blank\n",
        "\n",
        "  def forward(self, outputs):\n",
        "\n",
        "    indices = torch.argmax(outputs, dim=-1)\n",
        "    indices = torch.unique_consecutive(indices, dim=0)\n",
        "    indices = [token for token in indices if token != self.blank]\n",
        "    joined = \"\".join([self.labels[idx] for idx in indices])\n",
        "    splt = joined.strip().split()\n",
        "\n",
        "    return \" \".join(splt)\n",
        "\n",
        "greedy_decoder = CTCGreedyDecoder(labels=vocab.get_itos())"
      ],
      "metadata": {
        "id": "lG3_7l8C-ioL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Рассчет метрик WER & CER\n",
        "!pip install jiwer -q\n",
        "\n",
        "from jiwer import wer, cer\n",
        "\n",
        "model.eval()\n",
        "\n",
        "wer_scores = []\n",
        "cer_scores = []\n",
        "\n",
        "with torch.no_grad():\n",
        "\n",
        "  for batch in tqdm(valid_loader, desc=\"Calculating WER & CER scores\"):\n",
        "    val_src, val_tgt = batch[0].to(device), batch[1].to(device)\n",
        "\n",
        "    for idx in range(val_src.shape[0]):\n",
        "\n",
        "      y_pred = model(val_src[idx].unsqueeze(0))\n",
        "\n",
        "      y_pred = greedy_decoder(y_pred)\n",
        "      y_true = \"\".join([vocab.get_itos()[i] for i in val_tgt[idx]])\n",
        "\n",
        "      wer_scores.append(wer(y_pred, y_true))\n",
        "      cer_scores.append(cer(y_pred, y_true))\n",
        "\n",
        "wer_score = (sum(wer_scores) / len(wer_scores))\n",
        "cer_score = (sum(cer_scores) / len(cer_scores))\n",
        "\n",
        "print(f\"\\n\\n wer score: {wer_score:.2f}, cer score: {cer_score:.2f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ul-2lvgAluLB",
        "outputId": "6b3ea463-aa1c-4e6d-95d5-dccd2973ef00"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/3.4 MB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/3.4 MB\u001b[0m \u001b[31m52.5 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m52.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rCalculating WER & CER scores:   0%|          | 0/237 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torch/nn/modules/conv.py:456: UserWarning: Plan failed with a cudnnException: CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnFinalize Descriptor Failed cudnn_status: CUDNN_STATUS_NOT_SUPPORTED (Triggered internally at ../aten/src/ATen/native/cudnn/Conv_v8.cpp:919.)\n",
            "  return F.conv2d(input, weight, bias, self.stride,\n",
            "Calculating WER & CER scores: 100%|██████████| 237/237 [12:50<00:00,  3.25s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            " wer score: 0.36, cer score: 0.15\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Показ прогнозов\n",
        "def show_result(idx):\n",
        "  y_pred = model(val_src[idx].unsqueeze(0))\n",
        "  y_pred = greedy_decoder(y_pred)\n",
        "  y_true = \"\".join([vocab.get_itos()[i] for i in val_tgt[idx]])\n",
        "\n",
        "  print(f\"pred: {y_pred}\")\n",
        "  print(f\"true: {y_true}\")\n",
        "\n",
        "samples = torch.randint(1, 16, (3, ))\n",
        "\n",
        "for sample in samples:\n",
        "  show_result(sample)\n",
        "  print(\"*\" * 100)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rl3X9h-kE8LD",
        "outputId": "50e5a24f-55c9-42e0-f54a-f86d31f8119b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "pred: that he was chose wat several other was wast promcing peoples the joine the musical society which was to take part in the great festibl and london an extralye here was not only honour for the vi\n",
            "true: that he was chosen with several other of his most promising pupils to join the musical society which was to take part in the great festival in london the next july here was not only honour for the violinist but happi\n",
            "****************************************************************************************************\n",
            "pred: as americans their fally is pecularly scandalas because if the principals they now o pold our right it meas that i would have been better that a mergan should never have achieve their independent and better that\n",
            "true: as americans their folly is peculiarly scandalous because if the principles they now uphold are right it means that it would have been better that americans should never have achieved their independence and better th\n",
            "****************************************************************************************************\n",
            "pred: mat he was chosen a several other was was promcing poples to joine he musical society with was to take part in the great festof wont lenta lit extraloy here was not only honor for the vialinisben hapinss fo t\n",
            "true: that he was chosen with several other of his most promising pupils to join the musical society which was to take part in the great festival in london the next july here was not only honour for the violinist but happi\n",
            "****************************************************************************************************\n"
          ]
        }
      ]
    }
  ]
}