diff options
| -rw-r--r-- | .gitignore | 162 | ||||
| -rw-r--r-- | .ipynb_checkpoints/transformer_shortest_paths-checkpoint.ipynb | 717 | ||||
| -rw-r--r-- | transformer_shortest_paths.ipynb | 1346 |
3 files changed, 1138 insertions, 1087 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..82f9275 --- /dev/null +++ b/.gitignore @@ -0,0 +1,162 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/.ipynb_checkpoints/transformer_shortest_paths-checkpoint.ipynb b/.ipynb_checkpoints/transformer_shortest_paths-checkpoint.ipynb deleted file mode 100644 index ee844fb..0000000 --- a/.ipynb_checkpoints/transformer_shortest_paths-checkpoint.ipynb +++ /dev/null @@ -1,717 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "LPphBnKR-aWF" - }, - "source": [ - "# Step 0: Imports" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ge5QvElvhCOw", - "outputId": "c7cdaefa-d6dc-44ad-c258-e4fb2aca97a5" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "imports complete\n" - ] - } - ], - "source": [ - "# imports\n", - "import numpy as np\n", - "from collections import deque\n", - "import pickle\n", - "from tqdm import tqdm\n", - "np.random.seed(42)\n", - "\n", - "import torch\n", - "import torch.nn as nn\n", - "import pickle\n", - "from math import sqrt\n", - "from torch.utils.data import DataLoader, TensorDataset\n", - "import matplotlib.pyplot as plt\n", - "torch.manual_seed(42)\n", - "\n", - "import os\n", - "\n", - "print(\"imports complete\")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "id": "lylOX2POPwFL" - }, - "outputs": [], - "source": [ - "SEQ_LEN = 32\n", - "\n", - "PAD_TOKEN = 0\n", - "AVG_DEG = 2\n", - "MAX_VTXS = SEQ_LEN//AVG_DEG - 1\n", - "# vertices are labelled 1,2,...,63\n", - "# we also have a padding token which is 0.\n", - "\n", - "INF = MAX_VTXS # represents unreachability" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "gKt-yIpDebF1" - }, - "source": [ - "# Step 1: Generate synthetic data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "1IbzGIWseK3E", - "outputId": "a3cbc233-358c-4e17-ea6e-f4e9349d886b" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 1/1 [00:14<00:00, 14.42s/it]\n" - ] - } - ], - "source": [ - "# original task data\n", - "NTRAIN1 = 100_000\n", - "# the data will be edge lists\n", - "# like this: [1 3 1 5 2 4 0 0 0 0]\n", - "# this represents edges (1,3), (1,5) (2,4)\n", - "# (the zeros are just padding tokens)\n", - "\n", - "# the label is the shortest distance from vtx 1 to vtx 2\n", - "# or \"INF\" if no path exists\n", - "\n", - "# fine tuning data\n", - "NTRAIN2 = 2000\n", - "# I haven't totally figured out how to do the fine tuning yet.\n", - "# So don't worry about this yet.\n", - "\n", - "def random_graph(n):\n", - " edge_list = []\n", - " adjacencies = [set() for _ in range(n+1)]\n", - " indices = np.random.randint(n, size=(AVG_DEG*(n-1)))+1\n", - " for i in range(0, len(indices), 2):\n", - " u = indices[i]\n", - " v = indices[i + 1]\n", - " if u != v:\n", - " edge_list += [u,v]\n", - " adjacencies[u].add(v)\n", - " adjacencies[v].add(u)\n", - "\n", - " if np.random.random() < 0.25:\n", - " edge_list += [1,2]\n", - " adjacencies[1].add(2)\n", - " adjacencies[2].add(1)\n", - "\n", - " edge_list += [PAD_TOKEN]*(SEQ_LEN-len(edge_list))\n", - " return edge_list, adjacencies\n", - "\n", - "\"\"\"\n", - "input: G, represented as an adjacency list\n", - "output: [INF]+[d(1,i) for i in range(n)] if target=None\n", - "if target is set to some value, then we instead just output that specific distance\n", - "\"\"\"\n", - "def SSSP(G, target=None):\n", - " dist = [INF for _ in G]\n", - " dist[1] = 0\n", - " frontier = deque()\n", - " frontier.append(1)\n", - " while len(frontier) > 0:\n", - " vtx = frontier.popleft()\n", - " for x in G[vtx]:\n", - " if dist[x] == INF:\n", - " dist[x] = 1 + dist[vtx]\n", - " frontier.append(x)\n", - " if x == target:\n", - " return dist[target]\n", - " if target is not None:\n", - " return dist[target]\n", - " else:\n", - " return dist\n", - "\n", - "def fake_SSSP(G, target=None):\n", - " return 2 in G[1]\n", - "\n", - "graphs1 = []\n", - "distance1 = []\n", - "\n", - "graphs2 = []\n", - "distances2 = []\n", - "\n", - "for n in tqdm(range(MAX_VTXS-1, MAX_VTXS)):\n", - " # for _ in range(NTRAIN1//MAX_VTXS):\n", - " for _ in range(NTRAIN1):\n", - " edge_list, adj_list = random_graph(n)\n", - " dist = SSSP(adj_list, target=2)\n", - "\n", - " graphs1.append(edge_list)\n", - " distance1.append(dist)\n", - "\n", - "# for n in range(8, MAX_VTXS//4):\n", - "# for _ in range(NTRAIN2//MAX_VTXS):\n", - "# edge_list, adj_list = random_graph(n)\n", - "# distances = SSSP(adj_list)\n", - "# graphs2.append(edge_list)\n", - "# distances2.append(distances)\n", - "\n", - "split1 = int(len(graphs1)*3/4)\n", - "split2 = int(len(graphs2)*3/4)\n", - "\n", - "all1 = list(zip(graphs1, distance1))\n", - "np.random.shuffle(all1)\n", - "graphs1, distance1 = zip(*all1)\n", - "\n", - "data = {\n", - " \"train1-data\": graphs1[:split1],\n", - " \"train1-labels\": distance1[:split1],\n", - " \"test1-data\": graphs1[split1:],\n", - " \"test1-labels\": distance1[split1:]\n", - " # \"train2-data\": graphs2[:split2],\n", - " # \"train2-labels\": distances2[:split2],\n", - " # \"test2-data\": graphs2[split2:],\n", - " # \"test2-labels\": distances2[split2:]\n", - "}\n", - "\n", - "with open('data.pkl', 'wb') as file:\n", - " pickle.dump(data, file)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([75000, 32])\n", - "DONE\n" - ] - } - ], - "source": [ - "NTRAIN1 = 100000\n", - "\n", - "graphs1 = torch.randint(1, MAX_VTXS, (NTRAIN1, SEQ_LEN))\n", - "\n", - "# check if token 1 is in the graph\n", - "def silly_distance(graph):\n", - " return int(1 in graph)\n", - "\n", - "# check if both token 1 and token 2 are in the graph\n", - "def silly_distance2(graph):\n", - " return int(1 in graph and 2 in graph and 3 in graph and 4 in graph and 5 in graph)\n", - "\n", - "def silly_distance3(graph):\n", - " for i in range(len(graph)//2):\n", - " if graph[2*i] + graph[2*i+1] == 3:\n", - " return 1\n", - " return 0\n", - "\n", - "distance1 = [silly_distance3(graph) for graph in graphs1]\n", - "\n", - "split1 = int(len(graphs1)*3/4)\n", - "\n", - "data = {\n", - " \"train1-data\": graphs1[:split1],\n", - " \"train1-labels\": distance1[:split1],\n", - " \"test1-data\": graphs1[split1:],\n", - " \"test1-labels\": distance1[split1:]\n", - "}\n", - "\n", - "print(data[\"train1-data\"].shape)\n", - "\n", - "with open('data.pkl', 'wb') as file:\n", - " pickle.dump(data, file)\n", - "\n", - "print(\"DONE\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "0.1518" - ] - }, - "execution_count": 76, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "text/plain": [ - "0.1518" - ] - }, - "execution_count": 75, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "sum(distance1)/len(distance1)" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "EpDBxcgaIPpJ", - "outputId": "37cf9577-8cd8-444c-ec1a-c6f4b6061b7f" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "dataset size = 49MB\n" - ] - } - ], - "source": [ - "print(f\"dataset size = {os.path.getsize('data.pkl')//(1024*1024)}MB\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Q3Cg_8UQep8g" - }, - "source": [ - "# Step 2: Define Transformer Model" - ] - }, - { - "cell_type": "code", - "execution_count": 107, - "metadata": { - "id": "tLOWhg_CeWzH" - }, - "outputs": [], - "source": [ - "class TransformerModel(nn.Module):\n", - " def __init__(self, input_dim, model_dim, output_dim, num_heads, num_layers, seq_len, device, dropout=0.1):\n", - " super().__init__()\n", - " self.embedding = nn.Embedding(input_dim, model_dim//2)\n", - " self.model_dim = model_dim\n", - " self.seq_len = seq_len\n", - " self.device = device\n", - "\n", - " encoder_layer = nn.TransformerEncoderLayer(d_model=model_dim, nhead=num_heads,\n", - " dim_feedforward=model_dim*4,\n", - " dropout=dropout, batch_first=True)\n", - " self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers)\n", - "\n", - " self.fc_out = nn.Linear(model_dim*seq_len, output_dim)\n", - " self.fancy_encoding = torch.repeat_interleave(torch.rand((1,SEQ_LEN // 2, model_dim // 2), device=device), 2, dim=1)\n", - " \n", - " def positional_encoding(self, batch_size):\n", - " position = torch.arange(self.seq_len, dtype=torch.float, device=self.device).unsqueeze(1)\n", - " div_term = torch.exp(torch.arange(0, self.model_dim, 2, dtype=torch.float, device=self.device) *\n", - " -(torch.log(torch.tensor(500.0)) / self.model_dim))\n", - "\n", - " pos_encoding = torch.zeros(self.seq_len, self.model_dim, device=self.device)\n", - " pos_encoding[:, 0::2] = torch.sin(position * div_term)\n", - " pos_encoding[:, 1::2] = torch.cos(position * div_term)\n", - " pos_encoding = pos_encoding.unsqueeze(0).repeat(batch_size, 1, 1)\n", - " return pos_encoding\n", - "\n", - " def forward(self, src, key_padding_mask):\n", - " batch_size, src_len = src.size(0), src.size(1)\n", - " # src_pos = self.positional_encoding(batch_size)\n", - " embed = self.embedding(src)\n", - " src = torch.cat((embed * sqrt(self.model_dim), torch.Tensor.repeat(self.fancy_encoding, (batch_size, 1, 1))), dim=2)\n", - "\n", - " output = self.transformer_encoder(src, None, src_key_padding_mask=key_padding_mask)\n", - " flat_output = torch.flatten(output, start_dim=1, end_dim=2)\n", - " output = self.fc_out(flat_output)\n", - " return output\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "bpIeg86S-hBb" - }, - "source": [ - "# Step 3: Load Data" - ] - }, - { - "cell_type": "code", - "execution_count": 121, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "kWXvJRDYgFVP", - "outputId": "c13adb9d-6565-43b5-8437-20cef3dc0d16" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Trainable parameters in the model: 102K\n", - "train BASELINEs: 0.1290\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/tmp/ipykernel_390590/1991115476.py:23: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n", - " train_data_tensor = torch.tensor(train_data1, dtype=torch.long, device=device)\n", - "/tmp/ipykernel_390590/1991115476.py:31: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n", - " test_data_tensor = torch.tensor(test_data1, dtype=torch.long, device=device)\n" - ] - } - ], - "source": [ - "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", - "assert device.type == 'cuda', \"CUDA is not available. Please check your GPU setup.\"\n", - "\n", - "# PARAMS\n", - "VOCAB_SIZE = 1+MAX_VTXS # one more than the max number of vertices\n", - "MODEL_DIM = 64 # Dimension of model (embedding and transformer)\n", - "NEPOCHS = 50\n", - "BSZ = 512\n", - "LR = 0.001\n", - "NHEADS = 4\n", - "NLAYERS = 2\n", - "PAD_TOKEN = 0\n", - "model = TransformerModel(input_dim=VOCAB_SIZE, model_dim=MODEL_DIM,\n", - " output_dim=1, num_heads=NHEADS,\n", - " num_layers=NLAYERS, seq_len=SEQ_LEN,\n", - " device=device).to(device)\n", - "\n", - "with open(\"data.pkl\", \"rb\") as f:\n", - " data = pickle.load(f)\n", - "\n", - "train_data1 = data[\"train1-data\"]\n", - "train_label1 = data[\"train1-labels\"]\n", - "train_data_tensor = torch.tensor(train_data1, dtype=torch.long, device=device)\n", - "train_label_tensor = torch.tensor(train_label1, dtype=torch.float, device=device)\n", - "train_padding_mask = (train_data_tensor == PAD_TOKEN).bool().to(device)\n", - "train_dataset = TensorDataset(train_data_tensor, train_label_tensor, train_padding_mask)\n", - "train_loader = DataLoader(train_dataset, batch_size=BSZ, shuffle=True)\n", - "\n", - "test_data1 = data[\"test1-data\"]\n", - "test_label1 = data[\"test1-labels\"]\n", - "test_data_tensor = torch.tensor(test_data1, dtype=torch.long, device=device)\n", - "test_label_tensor = torch.tensor(test_label1, dtype=torch.float, device=device)\n", - "test_padding_mask = (test_data_tensor == PAD_TOKEN).bool().to(device)\n", - "test_dataset = TensorDataset(test_data_tensor, test_label_tensor, test_padding_mask)\n", - "test_loader = DataLoader(test_dataset, batch_size=BSZ, shuffle=True)\n", - "\n", - "criterion = nn.MSELoss()\n", - "optimizer = torch.optim.Adam(model.parameters(), lr=LR)\n", - "\n", - "train_err = []\n", - "test_err = []\n", - "\n", - "trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n", - "print(f\"Trainable parameters in the model: {trainable_params//1000}K\")\n", - "\n", - "train_baseline = ((train_label_tensor - train_label_tensor.mean())**2).mean().item()\n", - "print(f\"train BASELINEs: {train_baseline:.4f}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "f8Zn33m7CxL5" - }, - "source": [ - "# Step 4: Train the Model for the first task" - ] - }, - { - "cell_type": "code", - "execution_count": 122, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 486 - }, - "id": "pvTfzGmCeXU4", - "outputId": "0d3a20f3-23be-4c19-9eb6-46bfe11a48b1" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 1/50 \t Train Err: 0.1621 \t Test Err: 0.1208 \t baseline err: 0.1290\n", - "Epoch 2/50 \t Train Err: 0.1266 \t Test Err: 0.1201 \t baseline err: 0.1290\n", - "Epoch 3/50 \t Train Err: 0.1224 \t Test Err: 0.1199 \t baseline err: 0.1290\n", - "Epoch 4/50 \t Train Err: 0.1190 \t Test Err: 0.1214 \t baseline err: 0.1290\n", - "Epoch 5/50 \t Train Err: 0.1167 \t Test Err: 0.1164 \t baseline err: 0.1290\n", - "Epoch 6/50 \t Train Err: 0.1154 \t Test Err: 0.1156 \t baseline err: 0.1290\n", - "Epoch 7/50 \t Train Err: 0.1146 \t Test Err: 0.1131 \t baseline err: 0.1290\n", - "Epoch 8/50 \t Train Err: 0.1140 \t Test Err: 0.1145 \t baseline err: 0.1290\n", - "Epoch 9/50 \t Train Err: 0.1135 \t Test Err: 0.1144 \t baseline err: 0.1290\n", - "Epoch 10/50 \t Train Err: 0.1134 \t Test Err: 0.1160 \t baseline err: 0.1290\n", - "Epoch 11/50 \t Train Err: 0.1134 \t Test Err: 0.1160 \t baseline err: 0.1290\n", - "Epoch 12/50 \t Train Err: 0.1129 \t Test Err: 0.1137 \t baseline err: 0.1290\n", - "Epoch 13/50 \t Train Err: 0.1131 \t Test Err: 0.1122 \t baseline err: 0.1290\n", - "Epoch 14/50 \t Train Err: 0.1125 \t Test Err: 0.1133 \t baseline err: 0.1290\n", - "Epoch 15/50 \t Train Err: 0.1121 \t Test Err: 0.1119 \t baseline err: 0.1290\n", - "Epoch 16/50 \t Train Err: 0.1120 \t Test Err: 0.1129 \t baseline err: 0.1290\n", - "Epoch 17/50 \t Train Err: 0.1123 \t Test Err: 0.1123 \t baseline err: 0.1290\n", - "Epoch 18/50 \t Train Err: 0.1120 \t Test Err: 0.1119 \t baseline err: 0.1290\n", - "Epoch 19/50 \t Train Err: 0.1117 \t Test Err: 0.1148 \t baseline err: 0.1290\n", - "Epoch 20/50 \t Train Err: 0.1119 \t Test Err: 0.1136 \t baseline err: 0.1290\n", - "Epoch 21/50 \t Train Err: 0.1117 \t Test Err: 0.1120 \t baseline err: 0.1290\n", - "Epoch 22/50 \t Train Err: 0.1114 \t Test Err: 0.1123 \t baseline err: 0.1290\n", - "Epoch 23/50 \t Train Err: 0.1111 \t Test Err: 0.1121 \t baseline err: 0.1290\n", - "Epoch 24/50 \t Train Err: 0.1093 \t Test Err: 0.1061 \t baseline err: 0.1290\n", - "Epoch 25/50 \t Train Err: 0.1044 \t Test Err: 0.1012 \t baseline err: 0.1290\n", - "Epoch 26/50 \t Train Err: 0.1012 \t Test Err: 0.1003 \t baseline err: 0.1290\n", - "Epoch 27/50 \t Train Err: 0.0985 \t Test Err: 0.0964 \t baseline err: 0.1290\n", - "Epoch 28/50 \t Train Err: 0.0957 \t Test Err: 0.0942 \t baseline err: 0.1290\n", - "Epoch 29/50 \t Train Err: 0.0947 \t Test Err: 0.0935 \t baseline err: 0.1290\n", - "Epoch 30/50 \t Train Err: 0.0931 \t Test Err: 0.0941 \t baseline err: 0.1290\n", - "Epoch 31/50 \t Train Err: 0.0920 \t Test Err: 0.0916 \t baseline err: 0.1290\n", - "Epoch 32/50 \t Train Err: 0.0893 \t Test Err: 0.0857 \t baseline err: 0.1290\n", - "Epoch 33/50 \t Train Err: 0.0868 \t Test Err: 0.0814 \t baseline err: 0.1290\n", - "Epoch 34/50 \t Train Err: 0.0827 \t Test Err: 0.0785 \t baseline err: 0.1290\n", - "Epoch 35/50 \t Train Err: 0.0770 \t Test Err: 0.0720 \t baseline err: 0.1290\n", - "Epoch 36/50 \t Train Err: 0.0713 \t Test Err: 0.0646 \t baseline err: 0.1290\n", - "Epoch 37/50 \t Train Err: 0.0642 \t Test Err: 0.0540 \t baseline err: 0.1290\n", - "Epoch 38/50 \t Train Err: 0.0588 \t Test Err: 0.0501 \t baseline err: 0.1290\n", - "Epoch 39/50 \t Train Err: 0.0543 \t Test Err: 0.0456 \t baseline err: 0.1290\n", - "Epoch 40/50 \t Train Err: 0.0488 \t Test Err: 0.0366 \t baseline err: 0.1290\n", - "Epoch 41/50 \t Train Err: 0.0416 \t Test Err: 0.0315 \t baseline err: 0.1290\n", - "Epoch 42/50 \t Train Err: 0.0360 \t Test Err: 0.0214 \t baseline err: 0.1290\n", - "Epoch 43/50 \t Train Err: 0.0305 \t Test Err: 0.0172 \t baseline err: 0.1290\n", - "Epoch 44/50 \t Train Err: 0.0239 \t Test Err: 0.0116 \t baseline err: 0.1290\n", - "Epoch 45/50 \t Train Err: 0.0205 \t Test Err: 0.0117 \t baseline err: 0.1290\n", - "Epoch 46/50 \t Train Err: 0.0181 \t Test Err: 0.0092 \t baseline err: 0.1290\n", - "Epoch 47/50 \t Train Err: 0.0164 \t Test Err: 0.0100 \t baseline err: 0.1290\n", - "Epoch 48/50 \t Train Err: 0.0155 \t Test Err: 0.0081 \t baseline err: 0.1290\n", - "Epoch 49/50 \t Train Err: 0.0141 \t Test Err: 0.0074 \t baseline err: 0.1290\n", - "Epoch 50/50 \t Train Err: 0.0129 \t Test Err: 0.0075 \t baseline err: 0.1290\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1cAAAHWCAYAAACbsXOkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAACL8ElEQVR4nOzde3yO9R/H8dd972hjc5ht5jSnHMKcj0UJK+V87kD4UbJSK0UHh04kpKKUIoqIIiEZhYqcZg7lTM5zKIyNbbb798fXxtqw2b3dO7yfj8f1uO/7ur/XdX2uu+/Pz8f3e32+FpvNZkNEREREREQyxeroAERERERERPICJVciIiIiIiJ2oORKRERERETEDpRciYiIiIiI2IGSKxERERERETtQciUiIiIiImIHSq5ERERERETsQMmViIiIiIiIHSi5EhERERERsQMlVyIiIpKmL774AovFwqZNmxwdiohIrqDkSkQkH/joo4+wWCw0bNjQ0aHIdZKSlxttf/zxh6NDFBGRDHB2dAAiIpL1Zs2aRWBgIBs2bGDfvn1UrFjR0SHJdV5//XXKlSuXar/+O4mI5C5KrkRE8riDBw+ydu1avvvuO5544glmzZrFiBEjHB1WmqKjo/H09HR0GNnugQceoF69eo4OQ0REMknTAkVE8rhZs2ZRpEgRHnzwQbp06cKsWbPSbHfu3Dmee+45AgMDcXNzo1SpUvTq1YszZ84kt7l8+TIjR47kjjvuwN3dnRIlStCpUyf2798PwKpVq7BYLKxatSrFuf/++28sFgtffPFF8r7HH3+cggULsn//ftq0aUOhQoV45JFHAPj111/p2rUrZcqUwc3NjdKlS/Pcc89x6dKlVHHv2rWLbt26Ubx4cQoUKEDlypV55ZVXAPjll1+wWCwsWLAg1XGzZ8/GYrGwbt26NH+PTZs2YbFYmDFjRqrvfvrpJywWC4sXLwbgwoULPPvss8m/na+vL61atSI8PPwG/1UyJun3GzduHO+99x5ly5alQIECNG/enB07dqRq//PPP3P33Xfj6elJ4cKFad++PTt37kzV7tixY/Tr14+AgADc3NwoV64cAwcOJC4uLkW72NhYQkNDKV68OJ6ennTs2JHTp0+naLNp0yaCg4Px8fGhQIEClCtXjr59+9rl/kVEcguNXImI5HGzZs2iU6dOuLq60rNnTz7++GM2btxI/fr1k9tcvHiRu+++m507d9K3b1/q1KnDmTNnWLRoEUePHsXHx4eEhAQeeughVq5cSY8ePRg8eDAXLlwgLCyMHTt2UKFChQzHduXKFYKDg7nrrrsYN24cHh4eAMybN4+YmBgGDhxIsWLF2LBhAx9++CFHjx5l3rx5ycdv27aNu+++GxcXFwYMGEBgYCD79+/nhx9+4K233uKee+6hdOnSzJo1i44dO6b6XSpUqEDjxo3TjK1evXqUL1+eb775ht69e6f4bu7cuRQpUoTg4GAAnnzySebPn09ISAjVqlXjn3/+4bfffmPnzp3UqVPnlr/D+fPnUySxABaLhWLFiqXYN3PmTC5cuMCgQYO4fPky77//Pi1atGD79u34+fkBsGLFCh544AHKly/PyJEjuXTpEh9++CFNmzYlPDycwMBAAI4fP06DBg04d+4cAwYMoEqVKhw7doz58+cTExODq6tr8nWffvppihQpwogRI/j777+ZOHEiISEhzJ07F4BTp07RunVrihcvztChQylcuDB///0333333S3vXUQkT7GJiEietWnTJhtgCwsLs9lsNltiYqKtVKlStsGDB6doN3z4cBtg++6771KdIzEx0Waz2WzTpk2zAbYJEybcsM0vv/xiA2y//PJLiu8PHjxoA2zTp09P3te7d28bYBs6dGiq88XExKTaN3r0aJvFYrEdOnQoeV+zZs1shQoVSrHv+nhsNptt2LBhNjc3N9u5c+eS9506dcrm7OxsGzFiRKrrXG/YsGE2FxcX27///pu8LzY21la4cGFb3759k/d5e3vbBg0adNNzpWX69Ok2IM3Nzc0tuV3S71egQAHb0aNHk/evX7/eBtiee+655H21atWy+fr62v7555/kfVu3brVZrVZbr169kvf16tXLZrVabRs3bkwVV9LvlxRfy5YtU/ymzz33nM3JySn5N12wYIENSPNcIiL5iaYFiojkYbNmzcLPz497770Xro6GdO/enTlz5pCQkJDc7ttvvyUoKCjV6E7SMUltfHx8ePrpp2/Y5nYMHDgw1b4CBQokv4+OjubMmTM0adIEm83Gli1bADh9+jRr1qyhb9++lClT5obx9OrVi9jYWObPn5+8b+7cuVy5coVHH330prF1796d+Pj4FCMwy5cv59y5c3Tv3j15X+HChVm/fj3Hjx/P8P0DTJ48mbCwsBTbjz/+mKpdhw4dKFmyZPLnBg0a0LBhQ5YuXQrAiRMniIiI4PHHH6do0aLJ7WrWrEmrVq2S2yUmJrJw4ULatm2b5rNe//3vOWDAgBT77r77bhISEjh06FDy/QMsXryY+Pj42/oNRETyAiVXIiJ5VEJCAnPmzOHee+/l4MGD7Nu3j3379tGwYUNOnjzJypUrk9vu37+f6tWr3/R8+/fvp3Llyjg7229GubOzM6VKlUq1//Dhw8kJQsGCBSlevDjNmzeHq1PoAA4cOABwy7irVKlC/fr1UzxrNmvWLBo1anTLanxBQUFUqVIlefobVxMzHx8fWrRokbxv7Nix7Nixg9KlS9OgQQNGjhyZHF96NGjQgJYtW6bYkhLi61WqVCnVvjvuuIO///4bIDnZqVy5cqp2VatW5cyZM0RHR3P69GmioqJu+dsl+W/yWqRIEQDOnj0LQPPmzencuTOjRo3Cx8eH9u3bM336dGJjY9N1fhGRvELJlYhIHvXzzz9z4sQJ5syZQ6VKlZK3bt26wdUEw95uNIJ1/SjZ9dzc3LBaranatmrViiVLlvDSSy+xcOFCwsLCkothJCYmZjiuXr16sXr1ao4ePcr+/fv5448/bjlqlaR79+788ssvnDlzhtjYWBYtWkTnzp1TJJndunXjwIEDfPjhhwQEBPDuu+9y5513pjn6lBs5OTmlud9ms8HV/+7z589n3bp1hISEcOzYMfr27UvdunW5ePFiNkcrIuI4Sq5ERPKoWbNm4evry7x581JtPXv2ZMGCBcnV9ypUqJBm1bnrVahQgd27d9902lfSiMa5c+dS7E8aUUmP7du3s2fPHsaPH89LL71E+/btadmyJQEBASnalS9fHuCWcQP06NEDJycnvv76a2bNmoWLi0uKaX030717d65cucK3337Ljz/+SFRUFD169EjVrkSJEjz11FMsXLiQgwcPUqxYMd56661033d67N27N9W+PXv2JBepKFu2LAC7d+9O1W7Xrl34+Pjg6elJ8eLF8fLyStdvlxGNGjXirbfeYtOmTcyaNYs///yTOXPm2PUaIiI5mZIrEZE86NKlS3z33Xc89NBDdOnSJdUWEhLChQsXWLRoEQCdO3dm69ataZYsTxqd6Ny5M2fOnGHSpEk3bFO2bFmcnJxYs2ZNiu8/+uijdMeeNEqSdM6k9++//36KdsWLF6dZs2ZMmzaNw4cPpxlPEh8fHx544AG++uorZs2axf3334+Pj0+64qlatSo1atRg7ty5zJ07lxIlStCsWbPk7xMSEpKnKibx9fUlICDA7tPiFi5cyLFjx5I/b9iwgfXr1/PAAw/A1QSvVq1azJgxI0WCu2PHDpYvX06bNm0AsFqtdOjQgR9++IFNmzalus5/f79bOXv2bKpjatWqBVfLuIuI5BcqxS4ikgctWrSICxcu0K5duzS/b9SoEcWLF2fWrFl0796dIUOGMH/+fLp27Zo8nevff/9l0aJFTJkyhaCgIHr16sXMmTMJDQ1lw4YN3H333URHR7NixQqeeuop2rdvj7e3N127duXDDz/EYrFQoUIFFi9ezKlTp9Ide5UqVahQoQIvvPACx44dw8vLi2+//Tb5+Z7rffDBB9x1113UqVOHAQMGUK5cOf7++2+WLFlCREREira9evWiS5cuALzxxhsZ+j27d+/O8OHDcXd3p1+/fimmMl64cIFSpUrRpUsXgoKCKFiwICtWrGDjxo2MHz8+Xef/8ccf2bVrV6r9TZo0SR6hA6hYsSJ33XUXAwcOJDY2lokTJ1KsWDFefPHF5DbvvvsuDzzwAI0bN6Zfv37Jpdi9vb0ZOXJkcru3336b5cuX07x5cwYMGEDVqlU5ceIE8+bN47fffksuUpEeM2bM4KOPPqJjx45UqFCBCxcuMHXqVLy8vJITOhGRfMHR5QpFRMT+2rZta3N3d7dFR0ffsM3jjz9uc3FxsZ05c8Zms9ls//zzjy0kJMRWsmRJm6urq61UqVK23r17J39vu1oi/ZVXXrGVK1fO5uLiYvP397d16dLFtn///uQ2p0+ftnXu3Nnm4eFhK1KkiO2JJ56w7dixI81S7J6enmnG9tdff9latmxpK1iwoM3Hx8fWv39/29atW1Odw2az2Xbs2GHr2LGjrXDhwjZ3d3db5cqVba+99lqqc8bGxtqKFCli8/b2tl26dClDv+fevXuTS6T/9ttvqc47ZMgQW1BQkK1QoUI2T09PW1BQkO2jjz665XlvVor9+ntNKsX+7rvv2saPH28rXbq0zc3NzXb33Xfbtm7dmuq8K1assDVt2tRWoEABm5eXl61t27a2v/76K1W7Q4cO2Xr16mUrXry4zc3NzVa+fHnboEGDbLGxsSni+2+J9f+W3A8PD7f17NnTVqZMGZubm5vN19fX9tBDD9k2bdqUod9ZRCS3s9gyOvYvIiKSC125coWAgADatm3L559/7uhwMuTvv/+mXLlyvPvuu7zwwguODkdERG5Az1yJiEi+sHDhQk6fPk2vXr0cHYqIiORReuZKRETytPXr17Nt2zbeeOMNateunbxeloiIiL1p5EpERPK0jz/+mIEDB+Lr68vMmTMdHY6IiORheuZKRERERETEDjRyJSIiIiIiYgdKrkREREREROxABS3SkJiYyPHjxylUqBAWi8XR4YiIiIiIiIPYbDYuXLhAQEBAikXk06LkKg3Hjx+ndOnSjg5DRERERERyiCNHjlCqVKmbtlFylYZChQrB1R/Qy8vLobHEx8ezfPlyWrdujYuLi0NjkdxH/UcyQ/1HMkP9RzJD/UduV1b0naioKEqXLp2cI9yMkqs0JE0F9PLyyhHJlYeHB15eXvrDRTJM/UcyQ/1HMkP9RzJD/UduV1b2nfQ8LqSCFiIiIiIiInag5EpERERERMQOlFyJiIiIiIjYgZ65EhERERHJxRISEoiPj3d0GDlCfHw8zs7OXL58mYSEhHQd4+TkhLOzs12WYFJyJSIiIiKSS128eJGjR49is9kcHUqOYLPZ8Pf358iRIxlKljw8PChRogSurq6Zur6SKxERERGRXCghIYGjR4/i4eFB8eLF7TLyktslJiZy8eJFChYseMsFf7majMXFxXH69GkOHjxIpUqV0nXcjSi5EhERERHJheLj47HZbBQvXpwCBQo4OpwcITExkbi4ONzd3dOdJBUoUAAXFxcOHTqUfOztUkELEREREZFcTCNWmZeZ0aoU57HLWURERERERPI5JVciIiIiIiJ2oORKRERERETEDpRciYiIiIhItrBYLDfdRo4cmalzL1y40K7xZpSqBYqIiIiISLY4ceJE8vu5c+cyfPhwdu/enbyvYMGCDorMPjRylcMNHWrlySfv44cfVAVGRERERG7CZoPoaMds6VzE2N/fP3nz9vbGYrGk2DdnzhyqVq2Ku7s7VapU4aOPPko+Ni4ujpCQEEqUKIG7uztly5Zl9OjRAAQGBgLQuXNnihQpQvny5bPoR745jVzlcKdOWYiMLMi2bQl06uToaEREREQkx4qJAUeN/Fy8CJ6emTrFrFmzGD58OJMmTaJ27dps2bKF/v374+npSe/evfnggw9YtGgR33zzDWXKlOHIkSMcOXIEgI0bN+Lr68vnn39O06ZNKVy4sJ1uLGMcPnI1efJkAgMDcXd3p2HDhmzYsOGGbf/88086d+5MYGAgFouFiRMnptnu2LFjPProoxQrVowCBQpQo0YNNm3alIV3kXWqVzf/CrB9u0auRERERCTvGjFiBOPHj6dTp06UK1eOTp068dxzz/HJJ58AcPjwYSpVqsRdd91F2bJlueuuu+jZsycAxYsXB6Bw4cL4+fklf85uDh25mjt3LqGhoUyZMoWGDRsyceJEgoOD2b17N76+vqnax8TEUL58ebp27cpzzz2X5jnPnj1L06ZNuffee/nxxx8pXrw4e/fupUiRItlwR/ZXo4ZJrnbsUHIlIiIiIjfh4WFGkBx17UyIjo5m//799OvXj/79+yfvv3LlCt7e3gA8/vjjtGrVisqVK3P//ffz0EMP0bp160yHbk8OTa4mTJhA//796dOnDwBTpkxhyZIlTJs2jaFDh6ZqX79+ferXrw+Q5vcA77zzDqVLl2b69OnJ+8qVK5dl95DVkpKrffvMSG8m+62IiIiI5FUWS6an5jnKxatJ4dSpU2nYsGGK75ycnACoU6cOBw8e5Mcff2TFihV069aNli1bMn/+fIfEnBaHJVdxcXFs3ryZYcOGJe+zWq20bNmSdevW3fZ5Fy1aRHBwMF27dmX16tWULFmSp556KkUG/F+xsbHExsYmf46KigIgPj6e+Pj4247FHooWjcfbO5Hz593Ytu0Kdeum72FBEa724etfRTJC/UcyQ/1HMkP9J33i4+Ox2WwkJiaSmJjo6HAyLCnmxMREihcvTkBAAPv370+e6pdW24IFC9K1a1e6du1Kp06daNOmDWfOnKFo0aK4uLiQkJAAkPy7ZCQWm81GfHx8cjKXJCP90GHJ1ZkzZ0hISMDPzy/Ffj8/P3bt2nXb5z1w4AAff/wxoaGhvPzyy2zcuJFnnnkGV1dXevfuneYxo0ePZtSoUan2L1++HI8cMFRUtmwTtm0rzqxZ2zl58rCjw5FcKCwszNEhSC6m/iOZof4jmaH+c3POzs74+/tz8eJF4uLiHB1Ohl2+fBmbzZY8sPHSSy8xdOhQ3NzcuO+++4iNjSUiIoJz584xaNAgJk+ejJ+fHzVr1sRqtfL111/j5+eH1WolKiqKMmXKsGzZMmrWrMm5c+cyVNQiLi6OS5cusWbNGq5cuZLiu5iYmHSfJ89VC0xMTKRevXq8/fbbANSuXZsdO3YwZcqUGyZXw4YNIzQ0NPlzVFQUpUuXpnXr1nh5eWVb7GmJj4/n889PsG1bcazWmrRpU92h8UjuEh8fT1hYGK1atcLFxcXR4Uguo/4jmaH+I5mh/pM+ly9f5siRIxQsWBB3d3dHh5Nh7u7uWCyW5L9vh4SEULRoUcaPH8/w4cPx9PSkRo0aPPPMM3h5eeHj48PkyZPZu3cvTk5O1K9fnyVLliQnUePHj+eFF15g5syZlCxZkgMHDqQ7lsuXL1OgQAGaNWuW6rdMSv7Sw2HJlY+PD05OTpw8eTLF/pMnT+Lv73/b5y1RogTVqlVLsa9q1ap8++23NzzGzc0NNze3VPtdXFxyxP+gAwPNf9AdO5xwcXG6ZXuR/8opfVlyJ/UfyQz1H8kM9Z+bS0hIwGKxYLVasVodXgQ8w/r27Uvfvn1T7Hv00Ud59NFH02z/xBNP8MQTT9zwfO3bt6dt27ZERUXh5eWVod/EarVisVjS7HMZ6YMO+6/g6upK3bp1WblyZfK+xMREVq5cSePGjW/7vE2bNk2xyjPAnj17KFu2bKbidaSyZc8DsHVrutdnExERERGRbObQaYGhoaH07t2bevXq0aBBAyZOnEh0dHRy9cBevXpRsmTJ5JWX4+Li+Ouvv5LfHzt2jIiICAoWLEjFihUBeO6552jSpAlvv/023bp1Y8OGDXz66ad8+umnDrzTzCld+gJWq41//rEQGQklSjg6IhERERER+S+HJlfdu3fn9OnTDB8+nMjISGrVqsWyZcuSi1wcPnw4xXDe8ePHqV27dvLncePGMW7cOJo3b86qVavgarn2BQsWMGzYMF5//XXKlSvHxIkTeeSRRxxwh/bh5pZIpUqwezds367kSkREREQkJ3J4QYuQkBBCQkLS/C4pYUoSGBiILR3z4h566CEeeughu8WYE1SvbmP3bgvbtkEOWytNREREREQc+cyVZEzSYsLbtjk6EhERERERSYuSq1xCyZWIiIiISM6m5CqXSEqu/voLtFi5iIiIiEjOo+QqlyhbFgoVMonVnj2OjkZERERERP5LyVUuYbFAjRrmvaYGioiIiIjkPEqucpGaNc2rkisRERERkWsCAwOZOHGio8NQcpWbKLkSERERkdzMYrHcdBs5cuRtnXfjxo0MGDDA7vFmlMPXuZL0S0qutm93dCQiIiIiIhl34sSJ5Pdz585l+PDh7N69O3lfwYIFk9/bbDYSEhJwdr51ylK8eHEAEhMT7R5zRmjkKhepXt28HjkCZ886OhoRERERyUlsNoiOdsxms6UvRn9//+TN29sbi8WS/HnXrl0UKlSIH3/8kbp16+Lm5sZvv/3G/v37ad++PX5+fhQsWJD69euzYsWKFOf977RAJycnPvvsMzp27IiHhweVKlVi0aJF9v7JU1FylYt4e5uqgWj0SkRERET+IyYGChZ0zBYTY7/7GDp0KGPGjGHnzp3UrFmTixcv0qZNG1auXMmWLVu4//77adu2LYcPH77peUaNGkW3bt3Ytm0bbdq04ZFHHuHff/+1X6BpUHKVy+i5KxERERHJy15//XVatWpFhQoVKFq0KEFBQTzxxBNUr16dSpUq8cYbb1ChQoVbjkQ9/vjj9OzZk4oVK/L2229z8eJFNmzYkKWx65mrXKZmTfjhB41ciYiIiEhKHh5w8aLjrm0v9erVS/H54sWLjBw5kiVLlnDixAmuXLnCpUuXbjlyVTNpVALw9PTEy8uLU6dO2S/QNCi5ymW01pWIiIiIpMViAU9PR0eReZ7/uYkXXniBsLAwxo0bR8WKFSlQoABdunQhLi7upudxcXFJ8dlisWR5wQslV7nM9RUDExPBqomdIiIiIpKH/f777zz++ON07NgRro5k/f33344OK036q3kuU6kSuLmZqiwHDzo6GhERERGRrFWpUiW+++47IiIi2Lp1Kw8//LDDS67fiJKrXMbZGe6807zX1EARERERyesmTJhAkSJFaNKkCW3btiU4OJg6deo4Oqw0aVpgLlSzJoSHm6mBV0dHRURERERylccff5zHH388+fM999yDLY0FswIDA/n5559T7Bs0aFCKz0nTBJNGtBISErD+5/mZc+fO2TX+tGjkKhdSUQsRERERkZxHyVUupLWuRERERERyHiVXuVBScrVvnylsISIiIiIijqfkKhfy9QU/P7DZ4K+/HB2NiIiIiIig5Cr30nNXIiIiIgKkWQRCMsZev6GSq1xKz12JiIiI5G9OTk4AxMXFOTqUXC8mJgYAFxeXTJ1HpdhzKSVXIiIiIvmbs7MzHh4enD59GhcXl1Slx/OjxMRE4uLiuHz5crp+D5vNRkxMDKdOnaJw4cLJCevtUnKVS12fXNlsYLE4OiIRERERyU4Wi4USJUpw8OBBDh065OhwcgSbzcalS5coUKAAlgz8Bblw4cL4+/tn+vpKrnKpqlXByQn+/RdOnICAAEdHJCIiIiLZzdXVlUqVKmlq4FXx8fGsWbOGZs2apXuKn4uLS6ZHrJIoucql3N3hjjtg504zeqXkSkRERCR/slqtuLu7OzqMHMHJyYkrV67g7u6e6eenbocmZuZieu5KRERERCTnUHKViym5EhERERHJOZRc5WJJydX27Y6ORERERERElFzlYkkLCe/cCXqGUURERETEsZRc5WJlyoCXF8THw+7djo5GRERERCR/U3KVi1kseu5KRERERCSnUHKVy+m5KxERERGRnCFHJFeTJ08mMDAQd3d3GjZsyIYNG27Y9s8//6Rz584EBgZisViYOHHiTc89ZswYLBYLzz77bBZE7ngauRIRERERyRkcnlzNnTuX0NBQRowYQXh4OEFBQQQHB3Pq1Kk028fExFC+fHnGjBmDv7//Tc+9ceNGPvnkE2omZSB5UFJRCyVXIiIiIiKO5fDkasKECfTv358+ffpQrVo1pkyZgoeHB9OmTUuzff369Xn33Xfp0aMHbm5uNzzvxYsXeeSRR5g6dSpFihTJwjtwrOrVzeuxY/DPP46ORkREREQk/3J25MXj4uLYvHkzw4YNS95ntVpp2bIl69aty9S5Bw0axIMPPkjLli158803b9o2NjaW2NjY5M9RUVEAxMfHEx8fn6k4Mivp+jeKo0ABKFfOmYMHLWzZcoXmzW3ZHKHkZLfqPyI3o/4jmaH+I5mh/iO3Kyv6TkbO5dDk6syZMyQkJODn55div5+fH7t27brt886ZM4fw8HA2btyYrvajR49m1KhRqfYvX74cDw+P247DnsLCwm74XfHiDTh4sARz5/5FdPTBbI1Lcoeb9R+RW1H/kcxQ/5HMUP+R22XPvhMTE5Putg5NrrLCkSNHGDx4MGFhYbi7u6frmGHDhhEaGpr8OSoqitKlS9O6dWu8vLyyMNpbi4+PJywsjFatWuHi4pJmm/XrrWzYAImJ1WnTpmq2xyg5V3r6j8iNqP9IZqj/SGao/8jtyoq+kzSrLT0cmlz5+Pjg5OTEyZMnU+w/efLkLYtV3MjmzZs5deoUderUSd6XkJDAmjVrmDRpErGxsTg5OaU4xs3NLc3nt1xcXHLM/6BvFkvt2uZ1xw4rLi4Of4xOcqCc1Jcl91H/kcxQ/5HMUP+R22XPvpOR8zj0b+Kurq7UrVuXlStXJu9LTExk5cqVNG7c+LbOed9997F9+3YiIiKSt3r16vHII48QERGRKrHKC5KKIe7YAYmJjo5GRERERCR/cvi0wNDQUHr37k29evVo0KABEydOJDo6mj59+gDQq1cvSpYsyejRo+FqEYy//vor+f2xY8eIiIigYMGCVKxYkUKFClE9qYTeVZ6enhQrVizV/ryiYkVwd4eYGDhwwHwWEREREZHs5fDkqnv37pw+fZrhw4cTGRlJrVq1WLZsWXKRi8OHD2O1XhtgO378OLWT5sEB48aNY9y4cTRv3pxVq1Y55B4czcnJlGTftMmsd6XkSkREREQk+zk8uQIICQkhJCQkze/+mzAFBgZis2Ws3Hh+SLpq1LiWXHXq5OhoRERERETyH1U/yCOSnrvats3RkYiIiIiI5E9KrvIIJVciIiIiIo6l5CqPqFHDvB44ABcvOjoaEREREZH8R8lVHlG8OPj7g80Gf/7p6GhERERERPIfJVd5iKYGioiIiIg4jpKrPETJlYiIiIiI4yi5ykOSkqvt2x0diYiIiIhI/qPkKg+5fuQqg0uBiYiIiIhIJim5ykOqVAEnJzh7Fo4dc3Q0IiIiIiL5i5KrPMTNzSRY6LkrEREREZFsp+Qqj9FzVyIiIiIijqHkKo9RxUAREREREcdQcpXHKLkSEREREXEMJVd5TI0a5nXXLoiNdXQ0IiIiIiL5h5KrPKZUKShcGK5cMQmWiIiIiIhkDyVXeYzFoqIWIiIiIiKOoOQqD9JzVyIiIiIi2U/JVR6U9NyVkisRERERkeyj5CoP0siViIiIiEj2U3KVB1Wvbl5PnIAzZxwdjYiIiIhI/qDkKg8qWBAqVDDvVdRCRERERCR7KLnKozQ1UEREREQkeym5yqNU1EJEREREJHspucqjatUyr199BW++CfHxjo5IRERERCRvU3KVR7VtC+3bQ1wcvPYa1KsHmzc7OioRERERkbxLyVUe5ewMCxbArFlQrJiZHtiwIQwdCpcuOTo6EREREZG8R8lVHmaxwMMPw86d0KMHJCTAO+9AUBCsWePo6ERERERE8hYlV/lA8eLw9dfw/fcQEAB790Lz5jBoEERFOTo6EREREZG8QclVPtKuHfz5J/Tvbz5/9JFZcPjHHx0dmYiIiIhI7qfkKp8pXBg+/RRWroTy5eHIEWjTBnr1gn/+cXR0IiIiIiK5l5KrfKpFC1Pk4rnnzLNZX34JVavCN9+Azebo6EREREREch8lV/mYpydMmABr10K1anD6NHTvDp06wfHjjo5ORERERCR3UXIlNGoE4eEwfLgp4b5woRnFGj0aYmIcHZ2IiIiISO6g5EoAcHODUaPMQsP165sqgi+/DHfcAdOmmTLuIiIiIiJyY0quJIWaNeGPP8wzWGXLwrFj0K+fWRtr8eIbPI8VGQnz5sHTT0P79rBkiQMiFxERERFxrByRXE2ePJnAwEDc3d1p2LAhGzZsuGHbP//8k86dOxMYGIjFYmHixImp2owePZr69etTqFAhfH196dChA7t3787iu8galp9+ouxPP2FZuhS2bjUl/bK44oTVCo8+Crt2wbhxUKSIKeHeti3ccw9sWBQJX30FAwZA5cpQogR06waTJsGiRfDQQzBiBCQmZmmcIiIiIiI5icOTq7lz5xIaGsqIESMIDw8nKCiI4OBgTp06lWb7mJgYypcvz5gxY/D390+zzerVqxk0aBB//PEHYWFhxMfH07p1a6Kjo7P4buzPOnMmtT7+GOcOHaBWLfDxMZUo7rjDlPzr1cvM3/voI5PYbNliKlPYIQFzd4fnQ23s/2kfQ4K34WaNY80aaNjen26PubJv6s+wZ48pNxgUZEau+vUzB7/+ullY69y5zP8IIiIiIiK5gLOjA5gwYQL9+/enT58+AEyZMoUlS5Ywbdo0hg4dmqp9/fr1qV+/PkCa3wMsW7YsxecvvvgCX19fNm/eTLNmzbLkPrKKrX59Thw6hH9cHJZjx0zidOkS7N1rthtxc4OSJcHbGzw8oECBa6/peR8ZCWvWwJo1FDl5krFACKUZzuvMpBfz6MYCa2eebHOY18YXwfeOwteu3awZPPGEmR5Yvz4sWGBWKxYRERERycMcmlzFxcWxefNmhg0blrzParXSsmVL1q1bZ7frnD9/HoCiRYum+X1sbCyxsbHJn6OiogCIj48nPj7ebnHcjvhBg9hwxx20atUKFxcXuHwZjh83idbRo9dejx6FY8fM55MnscTGwoEDdonB5uaGrUEDSt51F5/d7cfThS7yypsF+eknJyYtLscXq2y88EICgwcn4ukJ9OwJVarg3LUrln37sDVqRMKnn2Lr2tUu8Uj6JfVfR/djyZ3UfyQz1H8kM9R/5HZlRd/JyLksNpvjlow9fvw4JUuWZO3atTRu3Dh5/4svvsjq1atZv379TY8PDAzk2Wef5dlnn71hm8TERNq1a8e5c+f47bff0mwzcuRIRo0alWr/7Nmz8fDwyNA95QSW+Hjcz56lwD//4HzpEk6xsTjFxSW/WpPe/2d/0ntrXBwJ7u78W7UqZ6pV41ylSiS6uqa6zrZtPsyYcSf795tRqyJFLtOjxy5atDiCi0sirlFR1Bs3juLbtgGwt0MHdj72GDYnp2z/TUREREREbkdMTAwPP/ww58+fx8vL66ZtHT4tMKsNGjSIHTt23DCxAhg2bBihoaHJn6OioihdujStW7e+5Q+Y1eLj4wkLC7s2cpWNigIVb/J9mzbw4ovwzTdXGDHCiYMH3fn441rMnRtE166JPPqoDe91XUh47VWcJkyg0sKFVIiKIuGrr8yzY5LlHNl/JPdT/5HMUP+RzFD/kduVFX0naVZbejg0ufLx8cHJyYmTJ0+m2H/y5MkbFqvIiJCQEBYvXsyaNWsoVarUDdu5ubnh5uaWar+Li0uO+R90Torlvx57zBQLnDIFxo6F48ctfPKJE598ApUqOfPYY+N59MMWlBvaHevPP2Nt1Ai++w7q1nV06PlGTu4/kvOp/0hmqP9IZqj/yO2yZ9/JyHkcWi3Q1dWVunXrsnLlyuR9iYmJrFy5MsU0wYyy2WyEhISwYMECfv75Z8qVK2eniOVG3Nxg8GA4fBiWLzel3D08TM2N4cOh/NMP0rzKST4rPozzh89B06YwY4ajwxYRERERsRuHl2IPDQ1l6tSpzJgxg507dzJw4ECio6OTqwf26tUrRcGLuLg4IiIiiIiIIC4ujmPHjhEREcG+ffuS2wwaNIivvvqK2bNnU6hQISIjI4mMjOTSpUsOucf8xMkJWrUyixCfPGnyp/vuM9Xa12z2pP/pt/G3nqJH7Bcsefwb4gc+A3Fxjg5bRERERCTTHJ5cde/enXHjxjF8+HBq1apFREQEy5Ytw8/PD4DDhw9z4sSJ5PbHjx+ndu3a1K5dmxMnTjBu3Dhq167N//73v+Q2H3/8MefPn+eee+6hRIkSydvcuXMdco/5VcGCZhmuFSvMiNaYMVC1KlxOdGMuPXiIJZSa8grPBS5gy/LTWb02soiIiIhIlsoRBS1CQkIICQlJ87tVq1al+BwYGMitChw6sACi3ECpUvDSS6YARng4zJwJX8+I5dR5Pyae6M7EYChVNJo61eOo09CV2k09qFPXQsmSZtRLRERERCSnyxHJleQfFoupY1G3Lowb58ZP044x88XtLIq6h6P/enJ0jSeL1gDvmvY+Luep43eUOhWiqB2UQJ27PCnf2A9rgD9YHT7wmvXi4uCPP8yCzlFR5ge0Ws1rOt5bExMpExkJDRpAiRKOvhsRERGRPE3JlTiMiws89ERJHnrYiwtDR7Jl1Xm2HPUhPKoiW6jNX1TjTLw3y496s/wosBr4AAoRRW3Lb9T2OkCdkicJqpFI+f+1oNB9DXL/MFdCAkREwM8/w8qV8OuvEBNz26dzAmoDtk8+gdat4eGHoX17M2dTREREROxKyZU4XqFCFJo8hmZAM4DLl+HIES7tWcOOdRcID4ct+wqxJdKfrRfKcwEv1tiaseZ8MzgP/AXMBR+nfylXMo5ydYpQrrIb5ctDuXJmK1MG0lgH2fFsNti9+1oy9csvcPZsyja+vtCiBZQsadrbbJCYmK73iVeuEPXbbxTevx+WLjWbh4dJsB5+GIKDTZYrIiIiIpmm5EpyHnd3qFSJApUqUf9BqH/dV1euwK4dVwj/+Rxb1l0mfLszf/7tyT+xhTiTUJQzh2Hj4dSntFpNbnJ9wlWunNnn729mzBUunE0DX0ePmkRq5UqTVB07lvL7QoXgnntMmcX77oM777ztwBLi41m9dCltypfHZf58mDUL9u+Hr782W7Fi0LWrSbSaNs0fUy1zo927YeJE2LULqlWD2rXNdued5n8vIiIikiMouZJcxdkZqtdypnotH3pdtz/q8DkOTvmJg1//wcG/4SDlOEB5DrpW5qAtkEvxLhw5AkeOwOrVaZ/b1dUkWtdvJUqAv1cM/lF78D+xBf/9v+O/PQz300dMMM7Opv58Wu/T+i4qyiQ313NzM4lNixYmmapXz7S3pypVYNQoGDkSNm6E2bNhzhxTL3/KFLOVKQM9e5pEq2ZN+15fbs/vv8O778KiRSSX07y+yI+zsynBWbs21Kp17bVwYYeFLCIikp8puZI8watMYYLe7k7QW91g0yaYOhVmvwzR0diAU84lOdDscQ427MFBj2oc/NvKwYNw4gRERpqZeHFxpmT84VQjXx5AraubWX/Ni/P4xJ8xG2Yrxj/J71NupynKvziTYE5ntZoEKmlkqkkTKFAge34oi8UUt2jQAMaNM39RnzULvv3W3Pg775itenWTaN1/PwQFmcRQskdCgkmm3n0X1q27tr9dOzOdc9cu2LLFbP/8A9u3m23mzGttAwOvjW4lJV2lSjnkdkRERPITJVeSt1gsUL++2caPhzlzsEydit/Gjfj9/BaNf37LzA3s3x/e6A3R0bB+PbFrN3Ny7X5O7PiHyCvFiMQ/xXbCrRyRziWJjC1C7BVnovAmCm8OUCHdoRUuGE/RQvG4e7vhGueE60pw+82MmLm6mgGspPdpfbZYbvyI1Y0eu7pyxcqxY9XZts1KiRJQvLh5hMu8OlPovpZYWraEjz4yz2PNng2LF8OOHfDKK2bz8jIja82ama1evZs+wJY0wJLumYxxcSZh2L4dTp82iUD9+ubZsPzk0iWTII0fD3v3mn2urmaxuOefN6OP17PZzJTSpEQrIsK8/v33tW3Bgmvty5aF5s3NlNN77jEJWG4vACMiIpLDKLmSvKtQIZNE9e9v/uI5dSp89RUcOADDhpntKjegzNWNokWhYUNoUAIaBpq/6Pv4wNW/z547Z3KAM2fMwMGZMzff/v3XXOPcRRfOXXSBEzeIN0s4ARX44Ye0v3V1TUq2CuDr25nixTvj2/8yxSN3UGz3WuL2HeJilDPRP3oS/aML0WzjotMhoouUJtqrBNEFinHR4kX0JSvR0SRvBQqY59lKlTKvJUtCyQAbpQr8Q8noPZQ8tQW/A+tw+nMb7NxpHqZLEbaTGTFr3PjaVq6c/ZKBc+dgzx6zXbpk5n8mbb6+9p+WeTP//GOS2w8/NB0LzLS+p56Cp58281PTYrGYH7hUKWjb9tr+s2dNf09KtrZsMb/xoUMmeUsa4SpTxiRZSQmXPX9fERGRfErJleQPtWrB5MkwdizMm2cSrbVrzfBQ7dpXk6kG5rV8+Rv+JdNigSJFzHbHHem79JUr5u+7Z86Y19hYM1iTtKX3s82W4WWuSExMYOfOA3h5VeDMGSunT8OpU+bv8NHR5rxHj5rtGneg3tUtDQnAmavbDURHX8tdrvv1AJ+rWxOceIISnKAkxyjlfJKSvvGU8I6h8PG/8D5/CO/w83iHb8F78iq8OY+3jysFG9fA2qSRSbbq1QNPzxsHcfky7Nt3LZDrt6QkJi0Wi0mwrk+4/rPZ/Etwxccf54Lut5+PHDwIEybAtGnXyu2XKQPPPQf9+pl/HLgdRYrAvfeaLUl0tOnvq1aZbcMGMw30+mSrdOlro1pJydbtSEgwCWtsrKlE6epqXjW1VERE8gElV5K/eHrC44+b7fRp8PbO8hrtzs5mGl7x4ll6mTTFxyeydOlftGkTiItLykqAMTGkSLb++/rPPyb3LFjQ/GyenuDpYaNgzCk8j+3G8+CfeO4Jp+A/h/Ak+rothmg8OEopjlGSY5S87n0pjjqXJTKhOAk2Z45SmqOUZv0V4PjV7UbOgOWHRLx+iDLJFvvx9kzA28cZ7xIeuCTG4XThLE5R53A6/y9OF8/jxBWsJOKEM05UxomKOBGMEwk4FfLEyacICS7uxETFE3MhgZhoiLEVIOakh9kiPIjBg0sUIAaPFJsNK05cwcNyCU/rJTydLuPpEoeH6xU83a7g6Z6AZwEbHh7gWdCCZyErnl5OeBSw4RnxO57hv+Jpu4AnTfCsVBLPPt3w7NAKz8IueNrAM+H28pHERJM0x8dfe71yxROXmq1wq98Kt1fB7Uo01vXrUiZbR47Al1+aDaBUKZyaNaNKfDzWn382yWp0tOk4/329/v3ly2kHZrVeS7aSEq7/vndxMcOe/fvDY49l/OZFREQcTMmV5F+OyHZyEA8P8xhO2bIZOcoC+F3dmpnhtEOHYM2aa9tekyFV8r8INV2hZgmo4Q01y5vKdm5uXLliChUePWoeG0raIiPh/Pm0Nhvx8RZsWDlPYc5ztRpe9NXt0G38ABeubpmQgDMXbIW4kFDIjOjFXY3nlu4EBlz7uBd4+ep2HTc3898pKbl1dTXJ0vWJU9Jr0vvExPRc3xNn55a4ubXEzQ3ciiXiRixu8Rdxu3Qet0tncT96CbfZsbhzmQJcwoOYFK83en/9qxdReBFFIS7gmhhvRrNiY28d3q+/wvHj8NJL6bkZERGRHEPJlYjcPovFFEYIDDSFF8AMfVksN01enZ2vPYuVHjabhcuX/5Nw7TvN+c17Ob/9CFEHzhDvXoiEYr4kFC1OQlEfEgr7kODmQUKihYQEUmyJidfeW60mcfHwSLkVKJB6X/JWwIbr5Sgun7lI9Klook/HEP3PZWLOxhJ9No7oc/FERyUQcyGB6Iu2awM9l52IjnUmupA/0SUrEW0plOJZtaQtqShIUi7y33WlM/qfyNnZJF/Xu3LFbNHRAFagwNUta/7Rwc3NhlfBRLw8EyjkkYCXZwJeBa7gVSAerwJxFHKLw8stDu+jf1Jn1XgaDn0F56goePNNPQsmIiK5hpIrEbEvX1+7n9JiMclOgQLX1XdoUBwedtToowXwhnLedj+zzXZtBt5/t/j4lDPo0vOaNLXQZjPHJyVsaW2XL6f8HB19hY0bt1OxYk3i4py4dMkkide/3mhfdDRcuHDtcbLYWAunY504/c+t5jpWBjpRmLO0fns5bdZ/xf1fPoJfCS1wLSIiOZ+SKxGRHOT6RPJqkUq7nTfpEaf01sqIj7dRuPBh2rSpjovL7RWkuHLFJFlRUddek7a0Pp8+bRb6Pnu2CN/QnW9WAgFQt46NB9pYaNPG1J5RfQwREcmJlFyJiEiWcXa+VmEzvRISTI2Npe9s58fvY9lMPTaHW9gcbmYJFi0KwcHwwAPmNQsGS0VERG6L5lmIiEiO4uRkqu2/sbAGmxYc5YRLGb6gN938V1O4sI1//4WvvzaP+fn7m5GsESPMOtQiIiKOpORKRERyrg4d8F/yOb095jM38h5O33kvv/54kZdfNkvU2WywcSO8/rpZzu7tt9NbMVFERMT+lFyJiEjO1qoVLF8O3t44/76au15pzlvPnSE83FRsnzYN2rQxSdUrr0DbtmadNhERkeym5EpERHK+pk3hl19MlY/wcGjeHI4fp0QJ6NMHliyBzz8Hd3dYuhTq1IH16x0dtIiI5DdKrkREJHeoXdssMFyyJPz1F9x9Nxw8mPx1377wxx9QsSIcPmy+njTp2rphIiIiWU3JlYiI5B5VqsBvv0H58nDgANx1F+zcmfx1UBBs2gSdO5t1vZ5+Gnr0MGXeRUREspqSKxERyV0CA80I1p13moeumjUzUwWv8vaGefNg4kRTCv6bb6B+fdixw6FRi4hIPqDkSkREcp+AALPacL16cOYM3HuvGdG6ymKBwYNhzRooVQp27zYl22fOdGjUIiKSxym5EhGR3KlYMVi50oxcRUWZqoJffJGiSePGsGULtG4Nly5B794wYABcvuywqEVEJA9TciUiIrmXlxf8+KOpv375sikd2L9/iuzJx8dUEBw1yoxoTZ1qkq79+x0auYiI5EFKrkREJHfz8ICFC+GNN0z29Nln0KSJKXhxlZMTDB8OP/0ExYtDRATUrWsOExERsRclVyIikvtZrfDqqyZ78vExcwHr1oUffkjRrFUr81XTpnD+PHTsCC+8ALGxDotcRETyECVXIiKSd7RqZSoHNmoE585Bu3YwbBhcuZLcpGRJsx7x88+bz+PHwx13mOmC8fGOC11ERHI/JVciIpK3lC5tKgk+84z5PGaMqWhx8mRyExcXGDcOFiwwhQcPHzaFLqpUgRkzUuRiIiIi6abkSkRE8h5XV3j/fZgzBwoWNENVtWunKNcO0KED7NsH770Hvr7mMa3HHzdLaM2eDQkJDrsDERHJhZRciYhI3tW9O2zcCNWqwYkTcM89Zh6gzZbcpEABePZZk1iNHWsqvO/ZA488AjVrwvz5kJjo0LsQEZFcQsmViIjkbVWqwPr18PDDZijqhRegSxdT0eI6np4wZAgcPAhvvgmFC8Nff0HXrlCnDnz/fYqcTEREJBUlVyIikvcVLAhffQWTJ5sHrr77DurVg23bUjUtVAheeQX+/htGjDBLaW3daqYQNmhgltVSkiUiImlRciUiIvmDxQJPPQW//mqKXuzbZ6oKfvFFmg9XeXvDyJFmJGvYMDOytWkTtGljSrmvXKkkS0REUsoRydXkyZMJDAzE3d2dhg0bsmHDhhu2/fPPP+ncuTOBgYFYLBYmTpyY6XOKiEg+0rChKdceHAyXLkGfPiZzqlnTzAF87TWYNQs2b4YLFyhaFN5+2yRZL7xgntFatw5atjRrFX/1ldbJEhERw+HJ1dy5cwkNDWXEiBGEh4cTFBREcHAwp06dSrN9TEwM5cuXZ8yYMfj7+9vlnCIiks/4+MCSJTBqFHh4mOxo+3ZTveLNN+HRR820QS8vKFUKWrak+KgQ3i3zIQemr+aZPhdwc7Pxxx/w2GOmybBhZiqhiIjkXw5PriZMmED//v3p06cP1apVY8qUKXh4eDBt2rQ029evX593332XHj164ObmZpdziohIPuTkBMOHQ1QU7N9vkq3x482CV82agZ+faXfsmJkDOHkyPPMM/j3u4f3pXvxtrcCbNeZQOuAKZ86Y5bTKl4e2bc1zWaowKCKS/zg78uJxcXFs3ryZYcOGJe+zWq20bNmSdevWZds5Y2Njib1uTkdUVBQA8fHxxMfH31Yc9pJ0fUfHIbmT+o9kRr7qP6VLm61Vq5T7z57FsmcP7N6NJWnbswf27cP/0kFe2d6TFz2eYPFj0/joWAdW/OzE4sWweDGUL29jwIBEevdOpFgxR92Y4+Sr/iN2p/4jtysr+k5GzuXQ5OrMmTMkJCTgl/Svg1f5+fmxa9eubDvn6NGjGTVqVKr9y5cvx8PD47bisLewsDBHhyC5mPqPZIb6z9VphD4+ppIFYLlyBe8DB6g+fTrFdu6k45dduK9MGRY/N4y5+1uycmUZDhxwZehQJ157De666xgPPHCQSpXOYbE4+mayl/qPZIb6j9wue/admJiYdLd1aHKVUwwbNozQ0NDkz1FRUZQuXZrWrVvj5eXl0Nji4+MJCwujVatWuLi4ODQWyX3UfyQz1H/S4ZlnuPLllzgNHYrX4cM8/N5Aejz2GBcixjD3Z18+/tiJiAgnfvmlDL/8UoY6dRJ58slEunWzkUP+7S7LqP9IZqj/yO3Kir6TNKstPRyaXPn4+ODk5MTJkydT7D958uQNi1VkxTnd3NzSfH7LxcUlx/wPOifFIrmP+o9khvrPLfzvf9CxI7z8Mnz6KdYvv8R78WIGjB5N/0392bDJwkcfwdy5EB5uZcAAKy+9BE88ASEhULKko28ga6n/SGao/8jtsmffych5HFrQwtXVlbp167Jy5crkfYmJiaxcuZLGjRvnmHOKiIjcVLFi8MknpkZ7rVpw9iw8+SSWJo1p6BLOjBlw9Ci88w4EBpqvx4wx7x991FR9FxGR3M/h1QJDQ0OZOnUqM2bMYOfOnQwcOJDo6Gj69OkDQK9evVIUp4iLiyMiIoKIiAji4uI4duwYERER7Nu3L93nFBERyRKNGsHGjfD++1CoEGzYAPXrw9NP4+N8jhdfNGsXL1xoChJeuWKW1KpXD5o3h++/T3M9YxERySUcnlx1796dcePGMXz4cGrVqkVERATLli1LLkhx+PBhTpw4kdz++PHj1K5dm9q1a3PixAnGjRtH7dq1+d///pfuc4qIiGQZZ2d45hnYvRt69jQ12SdNgipVYNYsnKw22reH1ath0yZ45BFzyJo10KEDVK5sml+86OgbERGRjMpwchUYGMjrr7/O4cOH7RZESEgIhw4dIjY2lvXr19OwYcPk71atWsUXX3yR4vo2my3VtmrVqnSfU0REJMuVKAGzZ8OKFXDHHXDypJkD2KIF7NwJQN268NVXZvHhoUOhSBGz5NbTT5vK8C+9ZKYTiohI7pDh5OrZZ5/lu+++o3z58rRq1Yo5c+akWCNKRERErnPffbBtG7z5Jri7w6pVcOedcO+95jmtM2coWRJGj4YjR8xaxRUrwrlzMHYslCsHDz9sRrlERCRnu63kKiIigg0bNlC1alWefvppSpQoQUhICOHh4VkTpYiISG7m5gavvAJ//QXt2oHNZpKsJ58Ef3944AGYMQPPK+d56ikzo3DRIrjnHvNc1tdfm0e37r7b5GjffQe7doHWVxURyVlu+5mrOnXq8MEHH3D8+HFGjBjBZ599Rv369alVqxbTpk3DZrPZN1IREZHcrlw5U7Xi77/NsFSdOqaCxbJl8Pjj4OsLHTtinTeXti2i+eUXCA+Hxx4DFxf47Td47TXo3BmqVgVPT6heHbp1g5Ej4ZtvYMcO0IQSERHHuO11ruLj41mwYAHTp08nLCyMRo0a0a9fP44ePcrLL7/MihUrmD17tn2jFRERyQvKloUhQ8y2Z49ZBOvrr82zWAsXms3DA9q2pXaPHsz89H7GjHHn669h+3YzAPbXXxAdDX/+abbrOTmZqYXVql3batUyCZnF4qibFhHJ+zKcXIWHhzN9+nS+/vprrFYrvXr14r333qNKlSrJbTp27Ej9+vXtHauIiEjec8cdZjjq1VfNsNOcOWY7cMAkXXPngpcXAR078nz37vBYXShenESbhSNHTD6WlGz99ZdJtKKizNTC3bthwYJrlype3JSAb97cbNWrg/Vmc1hsNjMvUYu4ioikS4aTq/r169OqVSs+/vhjOnTokOaKxeXKlaNHjx72ilFERCTvs1igRg2zvfmmqWAxZ45Jro4dgxkzzIZ5hstapgxlr273ly0LTctAzzLYSpfhhHNp/jrgniLh2rwZTp+Gb781G0CRwoncXfM8zcsfpbnPn9QiAqcTR02JwqQtIQHatoUBA6B161tkYyIi+VuGk6sDBw5QtmzZm7bx9PRk+vTpmYlLREQk/7JYTAWL+vXh3Xfh999NorV4sSkpGBsLe/ea7b+HAgFAgK8vLcuWhTJloH4Z4uo4sXG7O6v3lWT16ar8frkuZ88VZNGaIixaUwSogRcPcBe/0ZzVNOcSdTiEC1fM8NeCBWY6Y//+0KcPBAQ45KcREcnJMpxcnTp1isjIyFTrRq1fvx4nJyfq1atnz/hERETyN6vVlAm8+25Tpz0uzoxkHT5stkOHrr1P+hwTA6dOmW3jRgBcgaZXt5eBeJwJt9RjdaGHWG29h98u1iLqijdLeZClPAiAp0cijWtGExS7kRq751H90AaqvvoWHiNGpBzNcnJy8I8kIpIzZDi5GjRoEC+++GKq5OrYsWO88847rF+/3p7xiYiIyPVcXU3VwXLl0v7eZoN//02dfIFZmbhUKShVCpdSpWjo50dDZ2dexMz+i4iA1avN9uuvcPaslRV/FGIFLYAWAFhIpELCfqov3EH1hZuoXmwx1XvW4I7n2+ISWDIbfwgRkZwnw8nVX3/9RZ06dVLtr127Nn/99Ze94hIREZHbYbFAsWJmq1073Yc5OUHdumYLDYXERFNfY90688zWjh2mUuGZM1b2UYl9VGIhHeEfYBK4TIqjcqFDVK/tQvWW/lSpZiE29raLEouI5EoZ/lPPzc2NkydPUr58+RT7T5w4gbOz/hAVERHJC6xWqFnTbNc7dcokWjt2wI6IK+z49Sw7DnpwIcGTHRfKsmMNsAbAirtba3791cozz5gy8CIieV2Gs6HWrVszbNgwvv/+e7y9vQE4d+4cL7/8Mq1atcqKGEVERCSH8PWFFi3MZv4aURybDY78vJcdk1ax46dj7LhUnvU0ZE9sZaZMgSlTzKNZTz8Nbdqo4KCI5F0ZTq7GjRtHs2bNKFu2LLWvTjeIiIjAz8+PL7/8MitiFBERkRzMYoEy91WizH2VaHP5Mnz3HYnDH2LN/pJ84DqE76+0YflyC8uXQ4UKEBJiCg5e/TdaEZE8I8P/dlSyZEm2bdvG2LFjqVatGnXr1uX9999n+/btlC5dOmuiFBERkdzB3R0efpiE1auoVf4I38U9xH6PmgzpcYTChWH/fnjuOShZ0iRZu3Y5OmAREfu5rYekPD09GTBggP2jERERkbzB15ff33yTB6ZMIXDNGsYuvIMRM79l1tk2fPCBKZIxebLZgoPhmWfg/vs1ZVBEcrfbrkDx119/cfjwYeLi4lLsb9eunT3iEhERkVzuiocHCT/8gPWxx2DRIjx7tmPA9On03/4YP/8MH34IixbBTz+ZrWJF81zW44+Dl5ejoxcRybgMJ1cHDhygY8eObN++HYvFgs1mA8BisQCQkJBg/yhFREQkdypQAL79Fvr1g5kzoVcvLP/+y32DB3PffXDgAHz0EXz2GezbB4MHw8svQ48e0L8/NGhgnukSEckNMjz4PnjwYMqVK8epU6fw8PDgzz//ZM2aNdSrV49Vq1ZlTZQiIiKSezk7w/Tp8Oyz5vOzz8KIEWCzUb48jBsHR4/Cxx+bku3R0fD559CoEQQFwQcfmHWRRURyugwnV+vWreP111/Hx8cHq9WK1WrlrrvuYvTo0TzzzDNZE6WIiIjkblYrTJgAb7xhPr/+upkDmJgIQMGC8OST5lmsNWvgscdMbYzt281oVkAAPPIIrFoFVyfNiIjkOBlOrhISEihUqBAAPj4+HD9+HICyZcuye/du+0coIiIieYPFAq++auYBWiymmsWjj0J8fIomd99tZhCeOAGTJpnRq9hYmD0b7r0X7rgD3nkHTp506N2IiKSS4eSqevXqbN26FYCGDRsyduxYfv/9d15//XXKly+fFTGKiIhIXjJwIMyaZaYLfv01dOgAMTGpmhUuDIMGwZYtsHEjPPEEFCpkns0aOhRKlYJOneDHH0GPfItITpDh5OrVV18l8eoQ/uuvv87Bgwe5++67Wbp0KR988EFWxCgiIiJ5Tc+eplRggQKwdKmpx37uXJpNLRaoVw+mTIHjx83zWI0bw5UrsGABtGkD5crByJFw9my234mISLIMJ1fBwcF06tQJgIoVK7Jr1y7OnDnDqVOnaNGiRVbEKCIiInnRAw9AWJgZovrtN7jnHoiMvOkhBQtC376wdq15HuvZZ6FoUThyBEaNMgUxvv022+5ARCSFDCVX8fHxODs7s2PHjhT7ixYtmlyKXURERCTdmjaF1avBzw+2boW77oKDB9N1aPXq8N57cOyYmV1YpYp5DqtLFzNd8Opj4SIi2SZDyZWLiwtlypTRWlYiIiJiPzVrwu+/m7l9+/ebOX8//pjuw93dzbpYERHw2mvmUa4FC6BaNbN+lqoLikh2yfC0wFdeeYWXX36Zf7XghIiIiNhLhQomwapZ0ww/tWkD//sfnD+f7lO4uZkK75s3Q/365tD+/aFlS5OziYhktQwnV5MmTWLNmjUEBARQuXJl6tSpk2ITERERuS0lSsAff0BoqKli8fnnUKOGeS4rA2rWhHXrYPx4Uy/j55/NacaNM0UwRESyinNGD+jQoUPWRCIiIiJSoIDJijp0gD59zJBT69amDvu775pa7Ong5GRytPbtYcAAk2ANGQJz55qcrWbNLL8TEcmHMpxcjRgxImsiEREREUly992mwMXQoWYl4U8+gZ9+gmnTzErC6VShAqxYYQ57/nnYtAnq1oWXXjLrGbu7Z+ldiEg+k+FpgSIiIiLZwtMTPvzQDDsFBsLff0OLFvDMMxAdne7TWCzQrx/s3GmqCF65Am+9BbVrm8e8RETsJcPJldVqxcnJ6YabiIiIiF3dey9s22amBoJJuGrVMmtjZUCJEmYNrPnzTeX3XbvMAFlICFy4kDWhi0j+kuFpgQsWLEjxOT4+ni1btjBjxgxGjRplz9hEREREjEKFYMoUM/TUrx/s2wfNmsFzz8Gbb5pntdKpc2czAPbCC2a64OTJsHw5LFsG5ctn6V2ISB6X4eSqffv2qfZ16dKFO++8k7lz59KvXz97xSYiIiKSUuvWsGOHqVYxbRpMmABLlsAXX0CjRuk+TZEiprBFz57Qty/s3QtNmpjltWrXztI7EJE8zG7PXDVq1IiVK1fa63QiIiIiafP2NpnR4sVmrt/u3dC0qSl+EROToVO1bGmqvyctr9W8uSmAISJyO+ySXF26dIkPPviAkiVL3tbxkydPJjAwEHd3dxo2bMiGDRtu2n7evHlUqVIFd3d3atSowdKlS1N8f/HiRUJCQihVqhQFChSgWrVqTJky5bZiExERkRzqwQfhzz/h0UchMRHeeQfKloU33oCzZ9N9moAAWLPGPNp14YJZv3j27CyNXETyqAwnV0WKFKFo0aLJW5EiRShUqBDTpk3j3XffzXAAc+fOJTQ0lBEjRhAeHk5QUBDBwcGcOnUqzfZr166lZ8+e9OvXjy1bttChQwc6dOjAjh07ktuEhoaybNkyvvrqK3bu3Mmzzz5LSEgIixYtynB8IiIikoMVKQJffgkLFkC5cnDmDAwfDmXKmIeqjh1L12m8vc2UwG7dID4eHnnELLclIpIRGU6u3nvvvRTbBx98wOLFizl06BDt2rXLcAATJkygf//+9OnTJ3mEycPDg2nTpqXZ/v333+f+++9nyJAhVK1alTfeeIM6deowadKk5DZr166ld+/e3HPPPQQGBjJgwACCgoJuOSImIiIiuVSHDrBnjxlyqlkTLl402VG5cvC//5nvbsHNDb7+GgYPNp9feMGsjZWYmPXhi0jekOGCFo8//rjdLh4XF8fmzZsZNmxY8j6r1UrLli1Zt25dmsesW7eO0NDQFPuCg4NZuHBh8ucmTZqwaNEi+vbtS0BAAKtWrWLPnj289957aZ4zNjaW2NjY5M9RUVFwtRJifHx8pu8zM5Ku7+g4JHdS/5HMUP+RzHBY/+nSBTp3xvLTT1jHjsX622/w+efYpk3D1rEjCS++CHXq3PQUY8eCv7+VYcOcmDABjh1L5PPPE3B1zba7yPf054/crqzoOxk5V4aTq+nTp1OwYEG6du2aYv+8efOIiYmhd+/e6T7XmTNnSEhIwM/PL8V+Pz8/du3aleYxkZGRabaPjIxM/vzhhx8yYMAASpUqhbOzM1arlalTp9KsWbM0zzl69Og0y8gvX74cDw+PdN9PVgoLC3N0CJKLqf9IZqj/SGY4tP+88AJFH3yQSt9+i/+mTVi++w7rd99xKiiIvZ07c6ZGDbPCcBqqVoXBg0sxaVJt5s61smvXGV56aSMeHley/TbyM/35I7fLnn0nJgOFcjKcXI0ePZpPPvkk1X5fX18GDBiQoeQqq3z44Yf88ccfLFq0iLJly7JmzRoGDRpEQEAALVu2TNV+2LBhKUbDoqKiKF26NK1bt8bLyyubo08pPj6esLAwWrVqhYuLi0NjkdxH/UcyQ/1HMiPH9J82beD554nfvh2n8eOxzJ2L79at+G7dSmL9+iQOGYKtXTuwpn5Sok0baN06kW7dLGzd6ss777Thhx+u4O/vkDvJV3JM/5FcJyv6TtKstvTIcHJ1+PBhypUrl2p/2bJlOXz4cIbO5ePjg5OTEydPnkyx/+TJk/jf4E8uf3//m7a/dOkSL7/8MgsWLODBBx8EoGbNmkRERDBu3Lg0kys3Nzfc3NxS7Xdxcckx/4POSbFI7qP+I5mh/iOZkWP6T506MGsWvPUWjBsHn3+OdeNGrN26QeXKptJgGmt5tmkDq1aZwoRbt1po3tyFZcvgjjscchf5To7pP5Lr2LPvZOQ8GS5o4evry7Zt21Lt37p1K8WKFcvQuVxdXalbt26K9bESExNZuXIljRs3TvOYxo0bp1pPKywsLLl90nNS1v/8C5STkxOJeiJVREQkfwsMhEmT4NAheOUVUyZw927o2BG++irNQ+rVg7VroUIFOHjQLKm1fn22Ry4iuUCGk6uePXvyzDPP8Msvv5CQkEBCQgI///wzgwcPpkePHhkOIDQ0lKlTpzJjxgx27tzJwIEDiY6Opk+fPgD06tUrRcGLwYMHs2zZMsaPH8+uXbsYOXIkmzZtIiQkBAAvLy+aN2/OkCFDWLVqFQcPHuSLL75g5syZdOzYMcPxiYiISB7k6wtvvgmHD5tqgjYb9O4N33yTZvMKFUyCVa+eqfbeogX8Z5lNEZGMTwt84403+Pvvv7nvvvtwdjaHJyYm0qtXL95+++0MB9C9e3dOnz7N8OHDiYyMpFatWixbtiy5aMXhw4dTjEI1adKE2bNn8+qrr/Lyyy9TqVIlFi5cSPXq1ZPbzJkzh2HDhvHII4/w77//UrZsWd566y2efPLJDMcnIiIieZiXF3zyiUmuPv8cHn4YXFzMSNZ/+PrCL7+YgoQ//QTt2sFnn4EdCymLSC6X4eTK1dWVuXPn8uabbxIREUGBAgWoUaMGZcuWve0gQkJCkkee/mvVqlWp9nXt2jVVtcLr+fv7M3369NuOR0RERPIRqxU+/dSsHjxzJnTvDt9+C23bpmpasCD88AP062fWLu7Tx6xT/PLLNyw8KCL5SIaTqySVKlWiUqVK9o1GRERExBGsVpg2zSRYX39thqe+/x7uvz9VUxcXmDEDAgJMHYxXX4UjR8yjXM63/TcrEckLMvzMVefOnXnnnXdS7R87duxNR5NEREREcjQnJzNy1bkzxMVBhw6wYkWaTS0WGDMGPvzQvP/kE3NYBpbDEZE8KMPJ1Zo1a2jTpk2q/Q888ABr1qyxV1wiIiIi2c/Z2YxctW8PsbHmwao0HlFIEhIC8+eDmxssWgT33WcKXohI/pTh5OrixYu4urqm2u/i4pKhBbZEREREciQXF5g71yxydekSPPQQ/PbbDZt36mQGuIoUgT/+gCZN4MCBbI1YRHKIDCdXNWrUYO7cuan2z5kzh2rVqtkrLhERERHHcXMzRS1atYLoaJNo/fHHDZvfdRf8/juUKQN790LjxrB5c7ZGLCI5QIYfu3zttdfo1KkT+/fvp0WLFgCsXLmS2bNnM3/+/KyIUURERCT7ubvDwoVm5OqXX0xxixUrzGJXaahaFdatM3nY1q3QvLmZMphGTQwRyaMyPHLVtm1bFi5cyL59+3jqqad4/vnnOXbsGD///DMVK1bMmihFREREHMHDw9Rev/tuOH8eWreGiIgbNg8IgDVrzLNX0dGmmvsXX2RrxCLiQBlOrgAefPBBfv/9d6Kjozlw4ADdunXjhRdeICgoyP4RioiIiDiSpycsWWLm+p09Cy1bwo4dN2zu5QVLl8Ijj8CVK2YtrDffNOsUi0jedlvJFVerBvbu3ZuAgADGjx9PixYt+OMmc5FFREREcq1CheDHH6F+ffjnHzM0tXPnDZu7upqq7i+9ZD6/9ho8+aRJtkQk78pQchUZGcmYMWOoVKkSXbt2xcvLi9jYWBYuXMiYMWOoX79+1kUqIiIi4kje3vDTT1C7Npw6BS1awJ49N2xutaZcC+vTT01lQa2FJZJ3pTu5atu2LZUrV2bbtm1MnDiR48eP8+GHH2ZtdCIiIiI5SZEiEBYGNWpAZKRJsA4duukh16+F9cMP5pDTp7MtYhHJRulOrn788Uf69evHqFGjePDBB3FycsrayERERERyomLFTNXAatXg2DF48EFT7OImrl8La/16aNoUDh7MtohFJJukO7n67bffuHDhAnXr1qVhw4ZMmjSJM1qCXERERPIjX19YtgxKlIA//4SuXSE+/qaH/HctrJYt4eTJbItYRLJBupOrRo0aMXXqVE6cOMETTzzBnDlzCAgIIDExkbCwMC5cuJC1kYqIiIjkJKVLw+LFpppgWBg89dQtSwImrYVVvjwcOGAGvS5ezLaIRSSLZbhaoKenJ3379uW3335j+/btPP/884wZMwZfX1/atWuXNVGKiIiI5ER16sCcOaZ6xWefwdixtzwkIMAMevn4wObN6Rr0EpFc4rZLsQNUrlyZsWPHcvToUb7++mv7RSUiIiKSWzz0EEycaN4PHQrffHPLQypVMoNeBQqYRGvAAK2DJZIXZCq5SuLk5ESHDh1YtGiRPU4nIiIikrs8/TQMHmze9+pl5v7dQsOGJg+zWuGLL2D48KwPU0Syll2SKxEREZF8b/x4aNcOYmPN64EDtzzkoYdgyhTz/s034ZNPsj5MEck6Sq5ERERE7MHJCWbPNs9hnTkDbdrAv//e8rD+/a+NWj31FGgikEjupeRKRERExF48Pc1KwaVLw+7dZoGruLhbHjZyJPTtC4mJ0KNHumYVikgOpORKRERExJ4CAmDJEihUCFavhv/975bVKiwWMz2wTRu4dAnatjW5mYjkLkquREREROytRg2YN89MFfzyS3jjjVse4uJiClzUrw///AP33w+RkdkSrYjYiZIrERERkawQHAwffWTejxgBX311y0M8PU2J9goV4O+/zSLDFy5kfagiYh9KrkRERESyyoAB8OKL5n3fvrBmzS0P8fU1a18VLw7h4dClixYZFsktlFyJiIiIZKXRo6FzZ5MhdeiQroepKlY0j215eMDy5el6bEtEcgAlVyIiIiJZyWo1z101bAhnz5q5fqdP3/Kw+vXNM1hOTjBzJrz6arZEKyKZoORKREREJKsVKGAWsAoMhP37zQjW5cu3POzBB68tLPz22/Dxx1kfqojcPiVXIiIiItnB1xeWLoXChWHtWnjuuXQd1q8fjBpl3oeEwMKFWRumiNw+JVciIiIi2aVqVVOiHcyQ1IYN6Trstdegf3+zyHC3bjBjRtaGKSK3R8mViIiISHZq2RJ69TIVKgYNgoSEWx5isZiq7t26mboYjz8OL79ski0RyTmUXImIiIhkt7FjwcsLNm2Czz5L1yHOzvD11/DKK+bz6NHQtStER2dtqCKSfkquRERERLKbnx+8+aZ5P2wYnDmTrsOsVnPYzJng6grffQfNm8Px41kbroikj5IrEREREUcYOBCCgkx59mHDMnToY4/BypXg4wObN0ODBmbBYRFxLCVXIiIiIo7g7AyTJ5v3n30G69dn6PC77jKHVK0Kx47B3XerkqCIo+WI5Gry5MkEBgbi7u5Ow4YN2XCLyjnz5s2jSpUquLu7U6NGDZYuXZqqzc6dO2nXrh3e3t54enpSv359Dh8+nIV3ISIiIpJBTZua6hSQ7uIW1ytfHtatg9atISYGOnWCd94xtTJEJPs5PLmaO3cuoaGhjBgxgvDwcIKCgggODubUqVNptl+7di09e/akX79+bNmyhQ4dOtChQwd27NiR3Gb//v3cddddVKlShVWrVrFt2zZee+013N3ds/HORERERNLhnXfA29vM75s6NcOHe3vDkiUmN7PZYOhQszZWXFyWRCsiN+Hw5GrChAn079+fPn36UK1aNaZMmYKHhwfTpk1Ls/3777/P/fffz5AhQ6hatSpvvPEGderUYdKkScltXnnlFdq0acPYsWOpXbs2FSpUoF27dvj6+mbjnYmIiIikg6/vteIWL78Mp09n+BTOzjBpEnz4oSl6MX06tGqV7joZImInzo68eFxcHJs3b2bYdQ9xWq1WWrZsybp169I8Zt26dYSGhqbYFxwczMKrk4wTExNZsmQJL774IsHBwWzZsoVy5coxbNgwOnTokOY5Y2NjiY2NTf4cFRUFQHx8PPHx8Xa519uVdH1HxyG5k/qPZIb6j2SG+k8G9euH82efYdm6lcSXXiLhk09u6zRPPAGBgRYeecSJNWssNGpkY8GCK1SpYveIs5T6j9yurOg7GTmXQ5OrM2fOkJCQgJ+fX4r9fn5+7Nq1K81jIiMj02wfGRkJwKlTp7h48SJjxozhzTff5J133mHZsmV06tSJX375hebNm6c65+jRoxk1alSq/cuXL8fDwyOTd2kfYWFhjg5BcjH1H8kM9R/JDPWf9CvSsyfNtm7FOn06v1WpwtnKlW/7XG++WYg332zI/v2eNGoEL720kaCgjI+IOZr6j9wue/admJiYdLd1aHKVFRKvLlXevn17nnvuOQBq1arF2rVrmTJlSprJ1bBhw1KMhkVFRVG6dGlat26Nl5dXNkafWnx8PGFhYbRq1QoXFxeHxiK5j/qPZIb6j2SG+s9taNOGxJ07sc6Ywd1z5nBl7Vpwcrrt03XuDF26JLJunQuvv96Y999PZMCARLuGnFXUf+R2ZUXfSZrVlh4OTa58fHxwcnLi5MmTKfafPHkSf3//NI/x9/e/aXsfHx+cnZ2pVq1aijZVq1blt99+S/Ocbm5uuLm5pdrv4uKSY/4HnZNikdxH/UcyQ/1HMkP9J4PGjoXvv8eyZQsu06ebtbBuU0AA/Pwz9O8PX31lISTEiU2bnPjwQyhY0K5RZxn1H7ld9uw7GTmPQwtauLq6UrduXVauXJm8LzExkZUrV9K4ceM0j2ncuHGK9lwd9ktq7+rqSv369dm9e3eKNnv27KFs2bJZch8iIiIidmGH4hbXc3eHmTPhrbdMoYsvvoA6dWDTJvuEKyIpObxaYGhoKFOnTmXGjBns3LmTgQMHEh0dTZ8+fQDo1atXioIXgwcPZtmyZYwfP55du3YxcuRINm3aREhISHKbIUOGMHfuXKZOncq+ffuYNGkSP/zwA0899ZRD7lFEREQk3Z58EmrXhnPnTF31TLJYTJ72yy9QqhTs3QuNG5tBssTcMUtQJNdweHLVvXt3xo0bx/Dhw6lVqxYREREsW7YsuWjF4cOHOXHiRHL7Jk2aMHv2bD799FOCgoKYP38+CxcupHr16sltOnbsyJQpUxg7diw1atTgs88+49tvv+Wuu+5yyD2KiIiIpJuTE0yebN5Pm2ZWCbaDZs1g61bzLNaVK/DSS2bx4ePH7XJ6EXH0M1dJQkJCUow8XW/VqlWp9nXt2pWuXbve9Jx9+/alb9++dotRREREJNs0bgx9+5rk6qmnzDy+TBS3SFK0KMybB59/DoMHw8qVULOmWRerbVu7RC6Srzl85EpERERE0jBmDBQuDBERMGWK3U5rscD//gebN0OtWvDPP9CuHYSEwKVLdruMSL6k5EpEREQkJype3FSiAHjlFTh1yq6nr1IF/vgDklajmTwZ6teH7dvtehmRfEXJlYiIiEhO9cQTprjF+fN2KW7xX25uMH48LFsGfn7w558mwZo0CWw2u19OJM9TciUiIiKSUzk5wUcfmffTp8PatVlymeBg2LYNHngAYmPh6afNVMFMVoIXyXeUXImIiIjkZI0aQb9+5v2gQabUXxbw9YUlS+D998HVFRYvhqAgWLEiSy4nkicpuRIRERHJ6UaPhiJF7F7c4r8sFnjmGdiwAapWhRMnoFUrePVVrYklkh5KrkRERERyuuLF4e23zfsXXoA33jDz97JIUJCp/v7kk+bzW29B9+6qJihyK0quRERERHKD/v3Ng1CxsTB8ONSoAcuXZ9nlPDzg449hxgxwcYH58+Gee+DkySy7pEiup+RKREREJDdwcoKFC+Hrr6FECdi711Si6NoVjh7Nssv26gVhYWZW4oYN0LChqSooIqkpuRIRERHJLSwW6NEDdu2CZ581Cdf8+WbRqnHjID4+Sy7bvLlZE6tiRTh0CJo0MQmXiKSk5EpEREQkt/Hygvfeg82bTaYTHQ1Dhpg1sdasyZJL3nGHSbDuvhuiokzZ9k8/zZJLieRaSq5EREREcqugIPj1V5g2DXx8zHy95s3NXL4seDiqWDEzYvXoo5CQYNY4fuEF815ElFyJiIiI5G5WK/TpA7t3m2zHYoEvv4TKlWHSJLtnPm5uMHMmvP66+Tx+PHTpYgbPRPI7JVciIiIieUHRomYNrPXroW5dOH8enn4aGjQw++zIYoHXXoPZs82CwwsXmgGz48ftehmRXEfJlYiIiEheUr++SaY++ggKF4bwcGjcGAYMgHPn7Hqpnj3h55/NjMTNm00lwW3b7HoJkVxFyZWIiIhIXuPkBAMHmqmCvXuDzQZTp5pntOxc8KJpU1PoonJlUxG+aVNYutSulxDJNZRciYiIiORVvr7wxRcmoapQAQ4fNisBv/wyxMXZ7TIVKsC6dXDvvXDxIrRtC5Mn2+30IrmGkisRERGRvO7uu2HLFlP4wmaD0aNNCffdu+12iSJFYNky6NsXEhMhJMQsxZWYaLdLiOR4Sq5ERERE8oNChUzJ9nnzTCa0eTPUqWMWq7LZ7HIJV1f47DOTuwG8/z688opdTi2SKyi5EhEREclPunQxVSdatICYGFO+vWNHOH3aLqe3WGDoUPj8c/N5zBhTxFAkP1ByJSIiIpLflCplVgMeN84MN33/PdSsCT/9ZLdL9O0Lo0aZ94MGwQ8/2O3UIjmWkisRERGR/Mhqheefhw0boFo1iIyE+++HwYPh8mW7XOK116BfP/PcVY8e5lIieZmSKxEREZH8LCgINm0yFSgAPvjArJVlhwWrLBb4+GOTs8XEwEMPwf79mQ9ZJKdSciUiIiKS3xUoAB9+aBao8vODHTtMgvXee5ku9+fiAt98A7Vrm8e6HngAzpyxW+QiOYqSKxERERExHnjAjFi1bWvWwQoNheBgOH48U6ctVAiWLIGyZWHvXmjXDi5dslvUIjmGkisRERERucbX1xS4mDLFjGitWAH33Wfm9WVCiRLw449QuLBZcPiRRyAhwW5Ri+QISq5EREREJCWLxZRoDw83WdGuXTBkSKZPW7WqydtcXWHBAjMwZqcltkRyBCVXIiIiIpK2KlVgxgzz/qOPzDNZmdSsGcycad5/8IF5rEskr1ByJSIiIiI31qoVPPused+nD5w6lelTdu8O775r3j//PMybl+lTiuQISq5ERERE5OZGj4bq1U1i1a+fXebyPf/8tervjz0Gv/6a+TBFHE3JlYiIiIjcnLs7zJplHpZavBg++STTp7RYYOJE6NABYmOhfXvYudMu0Yo4jJIrEREREbm1mjVhzBjzPjQUdu/O9CmdnEzO1qgRnD1rKsFHRmY+VBFHUXIlIiIiIukzeDC0bGkWqXrkEbMWViZ5eMCiRVCxIhw6BA8+CBcv2iVakWyn5EpERERE0sdqhS++gKJFYfNmGDnSLqctXtysgeXjY6q/P/ywEwkJFrucWyQ7KbkSERERkfQrWRI+/dS8HzMG1qyxy2krVjSPcxUoAMuWWXn33XpcumSXU4tkmxyRXE2ePJnAwEDc3d1p2LAhGzZsuGn7efPmUaVKFdzd3alRowZLb7LmwpNPPonFYmHixIlZELmIiIhIPtS5synLbrOZUn/nz9vltA0bwty54Opq448/AggOduLMGbucWiRbODy5mjt3LqGhoYwYMYLw8HCCgoIIDg7m1A3WUFi7di09e/akX79+bNmyhQ4dOtChQwd27NiRqu2CBQv4448/CAgIyIY7EREREclH3n8fypeHw4dh0CC7nbZtW1i6NAFPzzj++MNK48awb5/dTi+SpZwdHcCECRPo378/ffr0AWDKlCksWbKEadOmMXTo0FTt33//fe6//36GDBkCwBtvvEFYWBiTJk1iypQpye2OHTvG008/zU8//cSDDz540xhiY2OJjY1N/hwVFQVAfHw88fHxdrvX25F0fUfHIbmT+o9khvqPZIb6Tz7g7o7liy9wuvdeLLNmcSU4GFuPHnY5dePG8YwZs453372XffusNG5sY+HCBBo0yPz6WpK3ZcWfPRk5l0OTq7i4ODZv3sywYcOS91mtVlq2bMm6devSPGbdunWEhoam2BccHMzChQuTPycmJvLYY48xZMgQ7rzzzlvGMXr0aEaNGpVq//Lly/Hw8MjgXWWNsLAwR4cguZj6j2SG+o9khvpP3le5SxeqzJ2L7ckn+eXyZS75+trlvKVLw6hRy3njjUYcOFCYFi0gNDScRo1Uq11uzZ5/9sTExKS7rUOTqzNnzpCQkICfn1+K/X5+fuzatSvNYyIjI9NsH3ndogjvvPMOzs7OPPPMM+mKY9iwYSkStqioKEqXLk3r1q3x8vLK4F3ZV3x8PGFhYbRq1QoXFxeHxiK5j/qPZIb6j2SG+k8+0ro1iQcP4rJhAy2/+oqEn34yC1hlQlL/6datGe3bu/DII4n8+KMz77zTgAkTEhk0KNFu4UvekhV/9iTNaksPh08LtLfNmzfz/vvvEx4ejsWSvhKebm5uuLm5pdrv4uKSY/4PISfFIrmP+o9khvqPZIb6Tz7g4mJWAq5VC+uaNVjffx9eeslOp3bBw8OFRYvMY12ffmrhueecOHLEiXffNZXhRdJizz97MnIeh3ZJHx8fnJycOHnyZIr9J0+exN/fP81j/P39b9r+119/5dSpU5QpUwZnZ2ecnZ05dOgQzz//PIGBgVl4NyIiIiL5VMWK8MEH5v1rr5nFquzI2RmmTIHRo83nCROge3e4fNmulxHJNIcmV66urtStW5eVK1cm70tMTGTlypU0btw4zWMaN26coj1X51QmtX/sscfYtm0bERERyVtAQABDhgzhp59+yuI7EhEREcmn+vSBjh0hPh4efhgy8JxKelgsMHSoGSRzcYH586FlS/jnH7teRiRTHD4tMDQ0lN69e1OvXj0aNGjAxIkTiY6OTq4e2KtXL0qWLMnoq/9UMXjwYJo3b8748eN58MEHmTNnDps2beLTq4vZFStWjGLFiqW4houLC/7+/lSuXNkBdygiIiKSD1gsMHUq/PEH7N4NQ4bA5Ml2v8zDD0NAAHToAL//Dk2awI8/mqrwIo7m8Jmq3bt3Z9y4cQwfPpxatWoRERHBsmXLkotWHD58mBMnTiS3b9KkCbNnz+bTTz8lKCiI+fPns3DhQqpXr+7AuxARERERihWDL74w7z/6CJYsyZLL3HOPSazKlIE9e6BRI9iwIUsuJZIhDh+5AggJCSEkJCTN71atWpVqX9euXenatWu6z//3339nKj4RERERSafWrWHwYLPIcJ8+sHw51Kpl98vceacZJHvwQdiyxSRcc+ZAu3Z2v5RIujl85EpERERE8pgxYyAoCE6fhqZNYd68LLlMiRKwZg088ABcumQe+cqCmYgi6abkSkRERETsy90dfvnFjGLFxEC3bvDqq5Bo//WpChaERYvgf/8zpw8JMY97ZcGlRG5JyZWIiIiI2F+RIuaZqxdeMJ/festUocjAgqzp5ewMn35qLgEwbhz06KFS7ZL9lFyJiIiISNZwdoZ334WZM8HNDX74wVSf2LvX7peyWODll+Grr0yp9nnzVKpdsp+SKxERERHJWo89Br/+amqo79wJDRqYQhdZ4JFH4KefwNv7Wqn2/fuz5FIiqSi5EhEREZGsV78+bNoEjRvDuXOmCsWECWCz2f1S996bslR748awfr3dLyOSipIrEREREckeJUqYQhd9+5qKE88/D717m1J/dpZUqr1OHVO08N574fvv7X4ZkRSUXImIiIhI9nFzg88+gw8+ACcn+PJLaN4cjh2z+6VKlIDVq6FNm2ul2j/80O6XEUmm5EpEREREspfFAk8/bZ67KloUNm6EevVg3Tq7X6pgQTNi9cQTZgbiM8+YATOVapesoORKRERERByjRQuTWNWoAZGRcM89MH263S/j7Awff2zWNgbzqFe3blkyG1HyOSVXIiIiIuI45cvD2rXQqRPExUHfvlhDQ7EkJNj1MhYLvPQSzJ4Nrq7w7bdw331w5oxdLyP5nJIrEREREXGsggXNwlSvvw6A06RJ3PnFF1lyqZ49ISwMChc2sxAbN4Z9+7LkUpIPKbkSEREREcezWuG112DOHADKL16MZePGLLlUs2ZmsCww0CRWjRtnyeNekg8puRIRERGRnKN7dxIffhiLzYbTwIEQH58ll6la1SRUdeuaqYEtWmTZusaSjyi5EhEREZEcJeHdd4krVAjLtm3w3ntZdh1/f1Oq/cEH4fJlaN8eVq3KsstJPqDkSkRERERyluLF2dGnj3k/ciQcOJBll/L0hO++g4ceMgnWQw/B779n2eUkj1NyJSIiIiI5zpF77yXx3ntNvfSBA80iVVnE1dXU02jdGqKj4YEHYP36LLuc5GFKrkREREQk57FYSJg0CdzczMNQs2dn6eXc3WHBArj3XrhwAYKDITw8Sy8peZCSKxERERHJmSpVguHDzftnn4V//snSy3l4wKJF0LQpnD8PrVrBtm1ZeknJY5RciYiIiEjO9cILcOedpqTfkCFZfrmCBWHpUmjYEP79F1q2hL/+yvLLSh6h5EpEREREci5XV5g6FSwWmD4dfv45yy/p5QXLlkGdOnD6NNx3H+zZk+WXlTxAyZWIiIiI5GyNG5uiFgBPPGGKXGSxwoXNo141akBkpFkHKwuLFkoeoeRKRERERHK+t9+GEiVg3z54661suWSxYrBihVlw+Ngxk2AdOpQtl5ZcSsmViIiIiOR83t4waZJ5/847sGNHtlzW1xdWrjS1NQ4dMlMEjx3LlktLLqTkSkRERERyh44doX17uHIFBgyAxMRsuWyJEuZRr3LlYP9+k2BFRmbLpSWXUXIlIiIiIrmDxQIffmhK+q1bB598km2XLlXKJFhlysDu3aaK4OnT2XZ5ySWUXImIiIhI7lG6tHn+CmDoUDh+PNsuHRhoEqyAAPjzT7MO1r//ZtvlJRdQciUiIiIiuctTT0GDBhAVBc88k62XrlDBJFh+frB1KwQHw7lz2RqC5GBKrkREREQkd3FyMmtfOTnBt9/C999n6+UrVzZVBIsVg02boFo1+Owz8yiY5G9KrkREREQk96lZE154wbwfNMiMYmWj6tVNghUYCCdOQP/+EBQEP/wANlu2hiI5iJIrEREREcmdhg+H8uVNbfRXX832y9eqBbt2wfjxULQo/PUXtGsHzZvD+vXZHo7kAEquRERERCR38vCAKVPM+0mTYMOGbA/BzQ1CQ02J9pdeAnd3+PVXaNQIunaFvXuzPSRxICVXIiIiIpJ7tWoFjz5q5uL17w/x8Q4Jo3BhGDMG9uyBxx83VePnzzfPYz39NJw65ZCwJJspuRIRERGR3G3CBDMvb9s2eOcdh4ZSujRMnw4REfDAA6bIxaRJpsrgm29CdLRDw5MsliOSq8mTJxMYGIi7uzsNGzZkwy2GdOfNm0eVKlVwd3enRo0aLF26NPm7+Ph4XnrpJWrUqIGnpycBAQH06tWL49m4BoKIiIiIZKPixc2DTwCvvWZGsv75x6Eh1awJS5easu316sHFiya0SpXg009VWTCvcnhyNXfuXEJDQxkxYgTh4eEEBQURHBzMqRuMna5du5aePXvSr18/tmzZQocOHejQoQM7duwAICYmhvDwcF577TXCw8P57rvv2L17N+3atcvmOxMRERGRbNO7NwwbBlYrzJoFd94JCxY4OiruvdcUt5gzB8qVM5UFn3gCatSAefMgLs7REYo9OTy5mjBhAv3796dPnz5Uq1aNKVOm4OHhwbRp09Js//7773P//fczZMgQqlatyhtvvEGdOnWYNGkSAN7e3oSFhdGtWzcqV65Mo0aNmDRpEps3b+bw4cPZfHciIiIiki0sFnj7bVi7FqpWhZMnoVMn6NEDTp92aGhWK3TvbioLvv++WR9r1y7o1g0CAswzWZs2qYR7XuDsyIvHxcWxefNmhg0blrzParXSsmVL1q1bl+Yx69atIzQ0NMW+4OBgFi5ceMPrnD9/HovFQuHChdP8PjY2ltjY2OTPUVfXSYiPjyfeQQ9FJkm6vqPjkNxJ/UcyQ/1HMkP9RzIjU/2nTh1Yvx7rG29gnTABy9y52H7+mYT338fWpYv9g80AiwUGDoSHH4b33rMyfbqVEycsTJpknsuqWtXGY48l8vDDiQQEODTUXCsr/uzJyLksNpvjcuTjx49TsmRJ1q5dS+PGjZP3v/jii6xevZr1aSwQ4OrqyowZM+jZs2fyvo8++ohRo0Zx8uTJVO0vX75M06ZNqVKlCrNmzUozjpEjRzJq1KhU+2fPno2Hh0cm7lBEREREHKXwvn3U/uADvK7OXjreuDHbnniC2Bv8g3t2S0iwsHVrcX75pTTr15cgLs4JAKvVRlDQKe699wgNG57AzS3R0aHmazExMTz88MOcP38eLy+vm7Z16MhVVouPj6dbt27YbDY+/vjjG7YbNmxYitGwqKgoSpcuTevWrW/5A2a1+Ph4wsLCaNWqFS4uLg6NRXIf9R/JDPUfyQz1H8kMu/afJ54gYfRorGPHErBuHSX27CFhwgRsPXqYoSQHa9vWrH98/nwi8+fb+PJLK2vXWtmyxY8tW/zw8rLRpYsZ0WrSxJYTQs7RsuLPnqRZbenh0OTKx8cHJyenVCNOJ0+exN/fP81j/P3909U+KbE6dOgQP//8802TJDc3N9zc3FLtd3FxyTH/h5CTYpHcR/1HMkP9RzJD/Ucywy79x8UF3noLunSBPn2wbN2Kc+/e8O23ZgHiEiXsFW6m+PjAk0+abf9+mDnTbH//bWHaNAvTplkpXx569TJbuXKOjjhns+efPRk5j0MLWri6ulK3bl1WrlyZvC8xMZGVK1emmCZ4vcaNG6doDxAWFpaifVJitXfvXlasWEGxYsWy8C5EREREJMerXRs2boRRo0zCtWiRWeF3xowcV0miQgUT5v79sGoV9OkDBQvCgQMwciSUL2+qDT7/PPz0E8TEODpiSeLwaoGhoaFMnTqVGTNmsHPnTgYOHEh0dDR9+vQBoFevXikKXgwePJhly5Yxfvx4du3axciRI9m0aRMhISFwNbHq0qULmzZtYtasWSQkJBAZGUlkZCRxqnUpIiIikn+5uMDw4bB5M9StC+fOweOPw0MPwdGjjo4uFasVmjeHadMgMhK+/BJatjSzGXfsMGsn33+/WT+5VSt4913YujXH5Yr5isOTq+7duzNu3DiGDx9OrVq1iIiIYNmyZfj5+QFw+PBhTpw4kdy+SZMmzJ49m08//ZSgoCDmz5/PwoULqV69OgDHjh1j0aJFHD16lFq1alGiRInkbe3atQ67TxERERHJIWrUgD/+MKXbXV3Nar/Vq8Py5Y6O7IY8Pc3ayGFhprL8N99Av35QujTExsKKFfDii1Crlpnp2KsXfPWVqUgv2SdHFLQICQlJHnn6r1WrVqXa17VrV7p27Zpm+8DAQBxYAFFEREREcgNnZ7PocPv2Zt7dhg3w4IPw2WdmQeIcrFgx6NrVbDYb7N5t8sLly+GXX0xC9eWXZgMICoLWrc1Wvz54ezv6DvKuHJFciYiIiIg4RLVqsGaNSbC+/tpMEzx6FF5+OUdUE7wViwWqVDHbM8+YUay1a68lW+HhZqrg1q1m2iCYaYQVKlzbype/9j4gwExHlNuj5EpERERE8jc3NzOHrnRpGDvW1EY/csSs7Oucu/667OYG995rttGjzRTCFStMorVihckb//3XbBs3pn18uXKpE68qVcxrLsg3HSp39RYRERERkaxgtcI775gE65ln4JNP4PhxM5rl6eno6G5b8eLQs6fZAC5cgIMHTSXCpO3AAfN66JAZ+dq1y2z/5esLTZpA06Zmq1PHJGNyjZIrEREREZEkISFmbtwjj8APP0CLFrB4sclS8oBChaBmTbP915UrcPjwtWTr+sRr5044dQoWLjQbV0e56te/lnA1aWLW68rPlFyJiIiIiFyvUyczh65dO1PookkTWLbMzIvLw5ydzTTA8uVNyffrxcaaCva//35tO3MGfvvNbEkqV742stW0KdxxR/6aSqjkSkRERETkv5o2NRnEAw/Avn3QuDEsWWKGavIhNzeTYzZpAkOGmCqFe/emTLZ27TKVC3fvNmtzgRnJql3bVCysVcu8Vq5slhzLi5RciYiIiIikpUoVWLcO2rSBLVvgnnvMAlMPPujoyBzOYjGjUnfcYQotAvzzj6lUmJRsbdxoRrfCwsyWxNUV7rwzZcIVFARFijjsduxGyZWIiIiIyI34+8Pq1WZRqZ9+MutiTZkC//ufoyPLcYoVg7ZtzcbVqYRbt0JExLVy8Nu2maIaW7aY7XqlS6dOuCpUyF2l4ZVciYiIiIjcTKFCprjFgAHwxRfQv78p1T5yZP56oCiD3NygQQOzJUlMhL//Tplwbd1q9h05YrbFi6+1v3ABChZ0SPi3RcmViIiIiMituLiYB4lKl4Y33oDXXzeZwCef5N0HiLKA1XqtaEanTtf2nz9vRrWSkq2ICLh8OXclVii5EhERERFJJ4vFJFWlSsHAgTB9Opw4AS+8AAkJppZ5RrbSpc00Qw8PR9+Zw3l7w913my2JzebIiG6PkisRERERkYwYMMCshdWtmynRvmzZ7Z/L2xsefdRMNQwKsmeUuV5unHGp5EpEREREJKMeeghWrTKjVmfPmkWiMrJZrbBmjXnYaPJks9WrZ5Ksnj3Nc16S6yi5EhERERG5HQ0amATpdiUmwsqVMHUqLFwImzaZLTQUevQwiVaDBrlzCCefUnIlIiIiIuIIViu0amW2U6dg5kz47DOzCu/nn5utRg2TZD36aMYWgrLZzMJTf/99bTtyBMqVM6NuFStm5Z3lW0quREREREQczdfXTDF8/nn49VczmjV/PmzfDs88Ay++CF26mEQrqerDv/+apOngwZRJVNIWHZ32tZ57ziyQ/NBDZlGqJk3MVEXJNP2KIiIiIiI5hcUCzZqZ7YMPYNYsk2ht2wZffWW2EiXMAlAXL976fCVKQGCg2UqWhPBwM5Vx1y6zjRtnRsQeeMAkWsHBGRshkxSUXImIiIiI5ERFikBICAwaBBs3miTr669N+fck/v4mcSpX7loSlbSVKQPu7qnPe+4cLF9uFkZeutSMgM2ebTYnJzMy1ratGdm6445sveXcTsmViIiIiEhOZrGYwhYNGsCECbBlC/j5meSpQIGMn69wYVNGvls3sz7XunWweLFJtv76y1RBXLXKTFG84w6TZN13n7leQIBJ+lRkI01KrkREREREcotChcyUQXtxcoK77jLbmDFw4MC1RGv1atizxyR0EyZcO8bd3SRZJUua1xu9z4eLIyu5EhERERERo3x5U0DjmWcgKura9MHwcDh+3EwhvHzZJGEHDtz8XIULmxE2q9WMkCUmmtf0bm5uNy7KkUMpuRIRERERkdS8vEyFwi5dru27fNk883XsmEm2jh+/9v7615gY82zXuXO3f/0rV+xyG9lJyZWIiIiIiKSPu7spnlGu3I3b2Gxm1Ov4cTh50nx2crq9LZdRciUiIiIiIvZjsYC3t9mqVnV0NNnK6ugARERERERE8gIlVyIiIiIiInag5EpERERERMQOlFyJiIiIiIjYgZIrERERERERO1ByJSIiIiIiYgdKrkREREREROxAyZWIiIiIiIgdKLkSERERERGxAyVXIiIiIiIidpAjkqvJkycTGBiIu7s7DRs2ZMOGDTdtP2/ePKpUqYK7uzs1atRg6dKlKb632WwMHz6cEiVKUKBAAVq2bMnevXuz+C5ERERERCQ/c3hyNXfuXEJDQxkxYgTh4eEEBQURHBzMqVOn0my/du1aevbsSb9+/diyZQsdOnSgQ4cO7NixI7nN2LFj+eCDD5gyZQrr16/H09OT4OBgLl++nI13JiIiIiIi+YnDk6sJEybQv39/+vTpQ7Vq1ZgyZQoeHh5MmzYtzfbvv/8+999/P0OGDKFq1aq88cYb1KlTh0mTJsHVUauJEyfy6quv0r59e2rWrMnMmTM5fvw4CxcuzOa7ExERERGR/MLZkRePi4tj8+bNDBs2LHmf1WqlZcuWrFu3Ls1j1q1bR2hoaIp9wcHByYnTwYMHiYyMpGXLlsnfe3t707BhQ9atW0ePHj1SnTM2NpbY2Njkz1FRUQDEx8cTHx9vhzu9fUnXd3Qckjup/0hmqP9IZqj/SGao/8jtyoq+k5FzOTS5OnPmDAkJCfj5+aXY7+fnx65du9I8JjIyMs32kZGRyd8n7btRm/8aPXo0o0aNSrV/4cKFeHh4ZPCussb333/v6BAkF1P/kcxQ/5HMUP+RzFD/kdtlz74TExMDV2fI3YpDk6ucYtiwYSlGw44dO0a1atX43//+59C4REREREQkZ7hw4QLe3t43bePQ5MrHxwcnJydOnjyZYv/Jkyfx9/dP8xh/f/+btk96PXnyJCVKlEjRplatWmme083NDTc3t+TPBQsW5MiRIxQqVAiLxZKJO8y8qKgoSpcuzZEjR/Dy8nJoLJL7qP9IZqj/SGao/0hmqP/I7cqKvmOz2bhw4QIBAQG3bOvQ5MrV1ZW6deuycuVKOnToAEBiYiIrV64kJCQkzWMaN27MypUrefbZZ5P3hYWF0bhxYwDKlSuHv78/K1euTE6moqKiWL9+PQMHDkxXXFarlVKlStnhDu3Hy8tLf7jIbVP/kcxQ/5HMUP+RzFD/kdtl775zqxGrJA6fFhgaGkrv3r2pV68eDRo0YOLEiURHR9OnTx8AevXqRcmSJRk9ejQAgwcPpnnz5owfP54HH3yQOXPmsGnTJj799FMALBYLzz77LG+++SaVKlWiXLlyvPbaawQEBCQncCIiIiIiIvbm8OSqe/funD59muHDhxMZGUmtWrVYtmxZckGKw4cPY7VeqxjfpEkTZs+ezauvvsrLL79MpUqVWLhwIdWrV09u8+KLLxIdHc2AAQM4d+4cd911F8uWLcPd3d0h9ygiIiIiInmfxZaeshfiMLGxsYwePZphw4aleC5MJD3UfyQz1H8kM9R/JDPUf+R2ObrvKLkSERERERGxA2s62oiIiIiIiMgtKLkSERERERGxAyVXIiIiIiIidqDkSkRERERExA6UXOVwkydPJjAwEHd3dxo2bMiGDRscHZLkQGvWrKFt27YEBARgsVhYuHBhiu9tNhvDhw+nRIkSFChQgJYtW7J3716HxSs5x+jRo6lfvz6FChXC19eXDh06sHv37hRtLl++zKBBgyhWrBgFCxakc+fOnDx50mExS87x8ccfU7NmzeTFOhs3bsyPP/6Y/L36jqTXmDFjktcqTaL+IzczcuRILBZLiq1KlSrJ3zuq/yi5ysHmzp1LaGgoI0aMIDw8nKCgIIKDgzl16pSjQ5McJjo6mqCgICZPnpzm92PHjuWDDz5gypQprF+/Hk9PT4KDg7l8+XK2xyo5y+rVqxk0aBB//PEHYWFhxMfH07p1a6Kjo5PbPPfcc/zwww/MmzeP1atXc/z4cTp16uTQuCVnKFWqFGPGjGHz5s1s2rSJFi1a0L59e/78809Q35F02rhxI5988gk1a9ZMsV/9R27lzjvv5MSJE8nbb7/9lvydw/qPTXKsBg0a2Ab9v537D62q/uM4/rq63evdnHNz6947Y2viWmo4cMt5M4m6kruFMVlkcYmrBWLejYkIobSmJPhHUVbQoB/aH6ajCSsRzday/SFaMrk2aY4WlsJcS0rbRs7Y/Xz/+M5Dl8la7dq5s+cDDpzz+Zzd+76XF+fy3vkRiVjbw8PDJi8vz+zcudPWupDcJJnm5mZrOxaLGa/Xa1555RVr7MqVK8blcpn9+/fbVCWSVV9fn5Fk2trajBnJSmpqqmlqarL26ezsNJLMiRMnbKwUySorK8u89957ZAfj0t/fb4qKikxLS4t58MEHTW1trTEcezAO9fX1pqSk5KZzduaHM1dJ6vr162pvb9fy5cutsSlTpmj58uU6ceKErbVhcjl//rx6e3vjspSZmany8nKyhFGuXr0qScrOzpYktbe3648//ojLzz333KP8/HzygzjDw8NqbGzU4OCg/H4/2cG4RCIRPfbYY3E5EccejNN3332nvLw8zZkzR6FQSBcuXJBszk/KLX11/GOXL1/W8PCwPB5P3LjH49G5c+dsqwuTT29vrzSSnT/zeDzWHCBJsVhMGzdu1NKlS3XvvfdKI/lxOp2aOXNm3L7kBzd0dHTI7/fr2rVrmj59upqbmzV//nxFo1GygzE1Njbq9OnTOnXq1Kg5jj34K+Xl5frggw9UXFysS5cuafv27Vq2bJnOnj1ra35orgAA0sh/kM+ePRt3zTrwV4qLixWNRnX16lUdOHBA4XBYbW1tdpeFJHfx4kXV1taqpaVF06ZNs7scTELBYNBaX7hwocrLy1VQUKCPPvpIbrfbtrq4LDBJ5eTkaOrUqaOeavLTTz/J6/XaVhcmnxt5IUsYS3V1tQ4dOqRjx47pzjvvtMa9Xq+uX7+uK1euxO1PfnCD0+nU3LlzVVpaqp07d6qkpERvvPEG2cGY2tvb1dfXp0WLFiklJUUpKSlqa2vTm2++qZSUFHk8HvKDv2XmzJm6++671d3dbevxh+YqSTmdTpWWlqq1tdUai8Viam1tld/vt7U2TC6FhYXyer1xWfrtt9/01VdfkSXIGKPq6mo1Nzfriy++UGFhYdx8aWmpUlNT4/LT1dWlCxcukB/cVCwW09DQENnBmAKBgDo6OhSNRq2lrKxMoVDIWic/+DsGBgb0/fffy+fz2Xr84bLAJLZp0yaFw2GVlZVp8eLF2rVrlwYHB7V27Vq7S0OSGRgYUHd3t7V9/vx5RaNRZWdnKz8/Xxs3btSOHTtUVFSkwsJC1dXVKS8vT5WVlbbWDftFIhHt27dPn3zyiTIyMqxr0TMzM+V2u5WZmannnntOmzZtUnZ2tmbMmKGamhr5/X4tWbLE7vJhsy1btigYDCo/P1/9/f3at2+fvvzySx09epTsYEwZGRnWvZ03pKena9asWdY4+cFYNm/erJUrV6qgoEA9PT2qr6/X1KlT9fTTT9t7/LmlzyLEhL311lsmPz/fOJ1Os3jxYnPy5Em7S0ISOnbsmJE0agmHw8aMPI69rq7OeDwe43K5TCAQMF1dXXaXjSRws9xIMnv27LH2+f33382GDRtMVlaWSUtLM6tWrTKXLl2ytW4kh2effdYUFBQYp9NpcnNzTSAQMJ999pk1T3bwd/z5UeyG/OAvrF692vh8PuN0Os3s2bPN6tWrTXd3tzVvV34c5v8/rgAAAACACeCeKwAAAABIAJorAAAAAEgAmisAAAAASACaKwAAAABIAJorAAAAAEgAmisAAAAASACaKwAAAABIAJorAAAAAEgAmisAACbI4XDo448/trsMAIDNaK4AAJPamjVr5HA4Ri0VFRV2lwYA+I9JsbsAAAAmqqKiQnv27Ikbc7lcttUDAPhv4swVAGDSc7lc8nq9cUtWVpY0csleQ0ODgsGg3G635syZowMHDsT9fUdHhx5++GG53W7NmjVL69at08DAQNw+u3fv1oIFC+RyueTz+VRdXR03f/nyZa1atUppaWkqKirSwYMHrblff/1VoVBIubm5crvdKioqGtUMAgAmP5orAMBtr66uTlVVVTpz5oxCoZCeeuopdXZ2SpIGBwe1YsUKZWVl6dSpU2pqatLnn38e1zw1NDQoEolo3bp16ujo0MGDBzV37ty499i+fbuefPJJffPNN3r00UcVCoX0yy+/WO//7bff6siRI+rs7FRDQ4NycnL+5W8BAHCrOYwxxu4iAAD4p9asWaO9e/dq2rRpceNbt27V1q1b5XA4tH79ejU0NFhzS5Ys0aJFi/T222/r3Xff1QsvvKCLFy8qPT1dknT48GGtXLlSPT098ng8mj17ttauXasdO3bctAaHw6EXX3xRL7/8sjTSsE2fPl1HjhxRRUWFHn/8ceXk5Gj37t239LsAANiLe64AAJPeQw89FNc8SVJ2dra17vf74+b8fr+i0agkqbOzUyUlJVZjJUlLly5VLBZTV1eXHA6Henp6FAgExqxh4cKF1np6erpmzJihvr4+SdLzzz+vqqoqnT59Wo888ogqKyt1//33T/BTAwCSDc0VAGDSS09PH3WZXqK43e5x7Zeamhq37XA4FIvFJEnBYFA//vijDh8+rJaWFgUCAUUiEb366qu3pGYAgD245woAcNs7efLkqO158+ZJkubNm6czZ85ocHDQmj9+/LimTJmi4uJiZWRk6K677lJra+uEasjNzVU4HNbevXu1a9cuvfPOOxN6PQBA8uHMFQBg0hsaGlJvb2/cWEpKivXQiKamJpWVlemBBx7Qhx9+qK+//lrvv/++JCkUCqm+vl7hcFjbtm3Tzz//rJqaGj3zzDPyeDySpG3btmn9+vW64447FAwG1d/fr+PHj6umpmZc9b300ksqLS3VggULNDQ0pEOHDlnNHQDg9kFzBQCY9D799FP5fL64seLiYp07d04aeZJfY2OjNmzYIJ/Pp/3792v+/PmSpLS0NB09elS1tbW67777lJaWpqqqKr322mvWa4XDYV27dk2vv/66Nm/erJycHD3xxBPjrs/pdGrLli364Ycf5Ha7tWzZMjU2Nibs8wMAkgNPCwQA3NYcDoeam5tVWVlpdykAgNsc91wBAAAAQALQXAEAAABAAnDPFQDgtsbV7wCAfwtnrgAAAAAgAWiuAAAAACABaK4AAAAAIAForgAAAAAgAWiuAAAAACABaK4AAAAAIAForgAAAAAgAWiuAAAAACAB/gdKrnXNcDZxbwAAAABJRU5ErkJggg==", - "text/plain": [ - "<Figure size 1000x500 with 1 Axes>" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "for epoch in range(NEPOCHS):\n", - " model.train() # set to training mode\n", - " train_loss = 0\n", - "\n", - " for batch_src, batch_labels, batch_padding_mask in train_loader:\n", - " optimizer.zero_grad()\n", - " output = model(batch_src, batch_padding_mask)\n", - " loss = criterion(output.squeeze(1), batch_labels)\n", - " train_loss += loss.item()/len(train_loader)\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # Evaluate performance\n", - " model.eval()\n", - " test_loss = 0\n", - "\n", - " with torch.no_grad():\n", - " for batch_src, batch_labels, batch_padding_mask in test_loader:\n", - " output = model(batch_src, batch_padding_mask)\n", - " loss = criterion(output.squeeze(1), batch_labels)\n", - " test_loss += loss.item()/len(test_loader)\n", - "\n", - " test_err.append(test_loss)\n", - " train_err.append(train_loss)\n", - " print(f\"Epoch {epoch + 1}/{NEPOCHS} \\t Train Err: {train_loss:.4f} \\t Test Err: {test_loss:.4f} \\t baseline err: {train_baseline:.4f}\")\n", - "\n", - "plt.figure(figsize=(10, 5))\n", - "plt.plot(test_err, label='Test', color='red')\n", - "plt.plot(train_err, label='Train', color='blue')\n", - "plt.title('Accuracy vs Epochs')\n", - "plt.xlabel('Epochs'); plt.ylabel('Accuracy')\n", - "plt.legend(); plt.grid()\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "v1hCiItHDWxJ" - }, - "outputs": [], - "source": [ - "## Q: why is this not working so well?\n", - "\n", - "## maybe first try a simpler problem: just give it points for distinguishing between distance 1 or not" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": { - "id": "LoGEmM5lH7_A" - }, - "outputs": [], - "source": [ - "batch_src, batch_labels, batch_padding_mask = next(iter(train_loader))\n", - "output = model(batch_src, batch_padding_mask)" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "hO8AhX3G7vF8", - "outputId": "8f4a3ca6-db47-434d-95a4-4631bc73de62" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1 \t nan\n", - "1 \t nan\n", - "1 \t nan\n", - "1 \t nan\n", - "0 \t nan\n", - "0 \t nan\n", - "1 \t nan\n", - "1 \t nan\n", - "1 \t nan\n", - "1 \t nan\n", - "1 \t nan\n", - "1 \t nan\n", - "1 \t nan\n", - "1 \t nan\n", - "1 \t nan\n", - "1 \t nan\n", - "1 \t nan\n", - "1 \t nan\n", - "1 \t nan\n", - "0 \t nan\n", - "1 \t nan\n", - "0 \t nan\n", - "1 \t nan\n", - "1 \t nan\n", - "0 \t nan\n", - "1 \t nan\n", - "1 \t nan\n", - "0 \t nan\n", - "1 \t nan\n", - "1 \t nan\n", - "1 \t nan\n", - "1 \t nan\n" - ] - } - ], - "source": [ - "for x,y in zip(batch_labels.tolist(), output.squeeze(1).tolist()):\n", - " print(f\"{int(x)} \\t {y:.1f}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "dRdUGbFmkPtK" - }, - "outputs": [], - "source": [ - "batch_src[2]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "LC6Xv3YfC0Rm" - }, - "source": [ - "# Step 5: Fine Tune" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "JtTLXn4zC1z_" - }, - "source": [ - "# Step 6: Test generalization" - ] - } - ], - "metadata": { - "colab": { - "gpuType": "T4", - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/transformer_shortest_paths.ipynb b/transformer_shortest_paths.ipynb index 1c6cdc6..71a40fe 100644 --- a/transformer_shortest_paths.ipynb +++ b/transformer_shortest_paths.ipynb @@ -11,7 +11,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 9, "execution_state": "idle", "metadata": { "colab": { @@ -24,6 +24,7 @@ "source": [ "from collections import deque\n", "import pickle\n", + "# using tqdm.auto glitches out collaborative editing\n", "from tqdm import tqdm\n", "\n", "import torch\n", @@ -31,8 +32,8 @@ "import pickle\n", "from math import sqrt\n", "from torch.utils.data import DataLoader, TensorDataset\n", - "%matplotlib widget\n", "import matplotlib.pyplot as plt\n", + "# %matplotlib widget\n", "torch.manual_seed(42)\n", "\n", "import os\n", @@ -40,12 +41,15 @@ "import ipdb\n", "\n", "import random\n", - "random.seed(42)" + "random.seed(42)\n", + "\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "assert device.type == 'cuda', \"CUDA is not available. Please check your GPU setup.\"" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 8, "execution_state": "idle", "metadata": { "id": "lylOX2POPwFL" @@ -55,9 +59,10 @@ "SEQ_LEN = 65 # means 32 edges, final token is the target vertex\n", "PAD_TOKEN = 0\n", "AVG_DEG = 2\n", - "MAX_VTXS = SEQ_LEN//AVG_DEG - 1 # 31\n", + "MAX_VTXS = SEQ_LEN//AVG_DEG + 1 # 32 (exclusive)\n", "MIN_VTXS = 8\n", "MAX_TUNE_VTXS = 16\n", + "PAD_TOKEN = 0\n", "# vertices are labelled 1,2,...,63\n", "# we also have a padding token which is 0." ] @@ -73,7 +78,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 18, "execution_state": "idle", "metadata": { "colab": { @@ -82,18 +87,9 @@ "id": "1IbzGIWseK3E", "outputId": "a3cbc233-358c-4e17-ea6e-f4e9349d886b" }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 23/23 [00:06<00:00, 3.52it/s]\n" - ] - } - ], + "outputs": [], "source": [ "# original task data\n", - "NTRAIN1 = 300_000\n", "# the data will be edge lists\n", "# like this: [1 3 1 5 2 4 0 0 0 0 2]\n", "# this represents edges (1,3), (1,5) (2,4)\n", @@ -141,33 +137,23 @@ " else:\n", " return dist\n", "\n", - "graphs1 = []\n", - "distance1 = []\n", - "\n", - "for n in tqdm(range(MIN_VTXS, MAX_VTXS)):\n", - " for _ in range(NTRAIN1//(MAX_VTXS - MIN_VTXS)):\n", + "def mkbatch(size):\n", + " graphs1 = []\n", + " distance1 = []\n", + " \n", + " for i in range(size):\n", + " n = random.randrange(MIN_VTXS, MAX_VTXS)\n", " edge_list, adj_list = random_graph(n)\n", " dist = SSSP(n, adj_list)\n", " edge_list[-1] = 2 # target token\n", " graphs1.append(edge_list)\n", " distance1.append(dist)\n", + " \n", + " data = torch.tensor(graphs1, device=device)\n", + " labels = torch.tensor(distance1, dtype=torch.float32, device=device)\n", + " padding = data == PAD_TOKEN\n", + " return data, labels, padding\n", "\n", - "data = {\n", - " \"data\": torch.tensor(graphs1),\n", - " \"labels\": torch.tensor(distance1, dtype=torch.float32)\n", - "}\n", - "\n", - "with open('data.pkl', 'wb') as file:\n", - " pickle.dump(data, file)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "execution_state": "idle", - "metadata": {}, - "outputs": [], - "source": [ "def vertices_on_shortest_12_path(n, G, target=2):\n", " dist = [n for _ in G]\n", " parent = [-1 for _ in G]\n", @@ -187,24 +173,14 @@ " x = parent[x]\n", " path.append(x)\n", " return list(reversed(path))\n", - " return []" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "execution_state": "idle", - "metadata": {}, - "outputs": [], - "source": [ - "# fine tuning data\n", - "NTRAIN2 = 2000\n", - "\n", - "graphs2 = []\n", - "distance2 = []\n", + " return []\n", "\n", - "for n in range(MIN_VTXS, MAX_TUNE_VTXS):\n", - " for _ in range(NTRAIN2//(MAX_TUNE_VTXS - MIN_VTXS)):\n", + "def mktunebatch(size):\n", + " graphs2 = []\n", + " distance2 = []\n", + " \n", + " for i in range(size):\n", + " n = random.randrange(MIN_VTXS, MAX_TUNE_VTXS)\n", " while True:\n", " edge_list, adj_list = random_graph(n)\n", " path = vertices_on_shortest_12_path(n, adj_list)\n", @@ -215,68 +191,51 @@ " graphs2.append(edge_list)\n", " distance2.append(target_vtx_idx)\n", " break\n", - "\n", - "tune_data = {\n", - " \"data\": torch.tensor(graphs2),\n", - " \"labels\": torch.tensor(distance2, dtype=torch.float32)\n", - "}\n", - "\n", - "with open('tune_data.pkl', 'wb') as file:\n", - " pickle.dump(tune_data, file)" + " \n", + " data = torch.tensor(graphs2, device=device)\n", + " labels = torch.tensor(distance2, dtype=torch.float32, device=device)\n", + " padding = data == PAD_TOKEN\n", + " return data, labels, padding" ] }, { "cell_type": "code", - "execution_count": 18, - "execution_state": "idle", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "EpDBxcgaIPpJ", - "outputId": "37cf9577-8cd8-444c-ec1a-c6f4b6061b7f" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "pre-train dataset size = 149MB\n", - "fine-tune dataset = 1MB\n" - ] - } - ], - "source": [ - "print(f\"pre-train dataset size = {os.path.getsize('data.pkl')//(1024*1024)}MB\")\n", - "print(f\"fine-tune dataset = {os.path.getsize('tune_data.pkl')//(1024*1024)}MB\")" - ] - }, - { - "cell_type": "code", - "execution_count": 38, + "execution_count": 41, "execution_state": "idle", "metadata": {}, "outputs": [ { "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "f5d5ab87fe4145eb8728e6d950e749d8", - "version_major": 2, - "version_minor": 0 - }, - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoAAAAHgCAYAAAA10dzkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAl3klEQVR4nO3df4xV9YH//xegM/iDGUSFkQJKa1dlFYwgMNutn1pZR0ubumKCrWmpUhvdgRRmq0jXgjZNaGh2q11UdmO29I9Sf2xWu8oKJVgxu079gUsKbiGtMYEGB+gPZpTKD5n7/WPL/TqVpfUHcwfej0dyE+ec9z33fd89zTxz5p5Lv0qlUgkAAMXoX+sJAADQuwQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhjqv1BI5m3d3d2bZtWwYNGpR+/frVejoAwJ+gUqnktddey/Dhw9O/f5nXwgTge7Bt27aMHDmy1tMAAN6FrVu3ZsSIEbWeRk0IwPdg0KBBye9PoIaGhlpPBwD4E3R1dWXkyJHV3+MlEoDvwcE/+zY0NAhAADjKlPzxrTL/8A0AUDABCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQmONqPQGOgDsaD7OvszdnAgD0Qa4AAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAUpqYB+M1vfjP9+vXLnDlzqtv27NmT1tbWnHrqqTn55JMzbdq0bN++vcfztmzZkqlTp+bEE0/M0KFDc8stt+TNN9/sMeapp57KRRddlPr6+px99tlZtmzZ217/nnvuyVlnnZWBAwdm0qRJee65547guwUA6BtqFoDPP/98/umf/iljx47tsX3u3Ll57LHH8vDDD2ft2rXZtm1brr766ur+AwcOZOrUqdm3b1+eeeaZfO9738uyZcuyYMGC6phXXnklU6dOzaWXXpr169dnzpw5+eIXv5hVq1ZVxzz44INpa2vLwoUL8+KLL2bcuHFpaWnJjh07emkFAABqo1+lUqn09ou+/vrrueiii3LvvffmG9/4Ri688MLcdddd6ezszOmnn57ly5fnmmuuSZJs2rQp5513Xtrb2zN58uQ88cQT+eQnP5lt27Zl2LBhSZKlS5dm3rx52blzZ+rq6jJv3rysWLEiGzdurL7mtddem127dmXlypVJkkmTJuXiiy/OkiVLkiTd3d0ZOXJkZs+endtuu+1Peh9dXV1pbGxMZ2dnGhoajsBKvUt3NB5mX2dvzgQA+pw++/u7F9XkCmBra2umTp2aKVOm9Ni+bt267N+/v8f2c889N6NGjUp7e3uSpL29PRdccEE1/pKkpaUlXV1deemll6pj/vDYLS0t1WPs27cv69at6zGmf//+mTJlSnXMoezduzddXV09HgAAR5vjevsFH3jggbz44ot5/vnn37avo6MjdXV1GTx4cI/tw4YNS0dHR3XMW+Pv4P6D+w43pqurK2+88UZ++9vf5sCBA4ccs2nTpv9z7osWLcqdd975jt8zAEBf0qtXALdu3Zovf/nL+f73v5+BAwf25ku/L+bPn5/Ozs7qY+vWrbWeEgDAO9arVwDXrVuXHTt25KKLLqpuO3DgQJ5++uksWbIkq1atyr59+7Jr164eVwG3b9+epqamJElTU9Pb7tY9eJfwW8f84Z3D27dvT0NDQ0444YQMGDAgAwYMOOSYg8c4lPr6+tTX17+nNejTfHYQAIrQq1cAL7vssmzYsCHr16+vPiZMmJDrrruu+t/HH3981qxZU33O5s2bs2XLljQ3NydJmpubs2HDhh53665evToNDQ0ZM2ZMdcxbj3FwzMFj1NXVZfz48T3GdHd3Z82aNdUxAADHql69Ajho0KCcf/75PbaddNJJOfXUU6vbZ86cmba2tgwZMiQNDQ2ZPXt2mpubM3ny5CTJ5ZdfnjFjxuRzn/tcFi9enI6Ojtx+++1pbW2tXp276aabsmTJktx666254YYb8uSTT+ahhx7KihUrqq/b1taWGTNmZMKECZk4cWLuuuuu7N69O9dff31vLgkAQK/r9ZtA/phvf/vb6d+/f6ZNm5a9e/empaUl9957b3X/gAED8vjjj+fmm29Oc3NzTjrppMyYMSNf//rXq2NGjx6dFStWZO7cubn77rszYsSI3H///WlpaamOmT59enbu3JkFCxako6MjF154YVauXPm2G0MAAI41NfkewGNFn/0eoXf7WT6fAQSgAH3293cv8m8BAwAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFOa4Wk+Aw7ij8TD7OntzJgDAMcQVQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgML0egDed999GTt2bBoaGtLQ0JDm5uY88cQT1f179uxJa2trTj311Jx88smZNm1atm/f3uMYW7ZsydSpU3PiiSdm6NChueWWW/Lmm2/2GPPUU0/loosuSn19fc4+++wsW7bsbXO55557ctZZZ2XgwIGZNGlSnnvuuSP4zgEA+oZeD8ARI0bkm9/8ZtatW5cXXnghH//4x/PpT386L730UpJk7ty5eeyxx/Lwww9n7dq12bZtW66++urq8w8cOJCpU6dm3759eeaZZ/K9730vy5Yty4IFC6pjXnnllUydOjWXXnpp1q9fnzlz5uSLX/xiVq1aVR3z4IMPpq2tLQsXLsyLL76YcePGpaWlJTt27OjlFQEA6F39KpVKpdaTGDJkSL71rW/lmmuuyemnn57ly5fnmmuuSZJs2rQp5513Xtrb2zN58uQ88cQT+eQnP5lt27Zl2LBhSZKlS5dm3rx52blzZ+rq6jJv3rysWLEiGzdurL7Gtddem127dmXlypVJkkmTJuXiiy/OkiVLkiTd3d0ZOXJkZs+endtuu+1PmndXV1caGxvT2dmZhoaG939h7mg8zL7OvvM8ADiKHPHf30eBmn4G8MCBA3nggQeye/fuNDc3Z926ddm/f3+mTJlSHXPuuedm1KhRaW9vT5K0t7fnggsuqMZfkrS0tKSrq6t6FbG9vb3HMQ6OOXiMffv2Zd26dT3G9O/fP1OmTKmOOZS9e/emq6urxwMA4GhTkwDcsGFDTj755NTX1+emm27KI488kjFjxqSjoyN1dXUZPHhwj/HDhg1LR0dHkqSjo6NH/B3cf3Df4cZ0dXXljTfeyK9+9ascOHDgkGMOHuNQFi1alMbGxupj5MiR73ElAAB6X00C8Jxzzsn69evz7LPP5uabb86MGTPyP//zP7WYyjsyf/78dHZ2Vh9bt26t9ZQAAN6x42rxonV1dTn77LOTJOPHj8/zzz+fu+++O9OnT8++ffuya9euHlcBt2/fnqampiRJU1PT2+7WPXiX8FvH/OGdw9u3b09DQ0NOOOGEDBgwIAMGDDjkmIPHOJT6+vrU19e/5/cPAFBLfeJ7ALu7u7N3796MHz8+xx9/fNasWVPdt3nz5mzZsiXNzc1Jkubm5mzYsKHH3bqrV69OQ0NDxowZUx3z1mMcHHPwGHV1dRk/fnyPMd3d3VmzZk11DADAsarXrwDOnz8/V155ZUaNGpXXXnsty5cvz1NPPZVVq1alsbExM2fOTFtbW4YMGZKGhobMnj07zc3NmTx5cpLk8ssvz5gxY/K5z30uixcvTkdHR26//fa0trZWr87ddNNNWbJkSW699dbccMMNefLJJ/PQQw9lxYoV1Xm0tbVlxowZmTBhQiZOnJi77roru3fvzvXXX9/bSwIA0Kt6PQB37NiRz3/+83n11VfT2NiYsWPHZtWqVfmrv/qrJMm3v/3t9O/fP9OmTcvevXvT0tKSe++9t/r8AQMG5PHHH8/NN9+c5ubmnHTSSZkxY0a+/vWvV8eMHj06K1asyNy5c3P33XdnxIgRuf/++9PS0lIdM3369OzcuTMLFixIR0dHLrzwwqxcufJtN4YAABxr+sT3AB6tfA8gABx9fA9gH/kMIAAAvUcAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAU5rhaT4BjgH9DGACOKq4AAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAUptcDcNGiRbn44oszaNCgDB06NFdddVU2b97cY8yePXvS2tqaU089NSeffHKmTZuW7du39xizZcuWTJ06NSeeeGKGDh2aW265JW+++WaPMU899VQuuuii1NfX5+yzz86yZcveNp977rknZ511VgYOHJhJkyblueeeO0LvHACgb+j1AFy7dm1aW1vzk5/8JKtXr87+/ftz+eWXZ/fu3dUxc+fOzWOPPZaHH344a9euzbZt23L11VdX9x84cCBTp07Nvn378swzz+R73/teli1blgULFlTHvPLKK5k6dWouvfTSrF+/PnPmzMkXv/jFrFq1qjrmwQcfTFtbWxYuXJgXX3wx48aNS0tLS3bs2NGLKwIA0Lv6VSqVSi0nsHPnzgwdOjRr167NJZdcks7Ozpx++ulZvnx5rrnmmiTJpk2bct5556W9vT2TJ0/OE088kU9+8pPZtm1bhg0bliRZunRp5s2bl507d6auri7z5s3LihUrsnHjxuprXXvttdm1a1dWrlyZJJk0aVIuvvjiLFmyJEnS3d2dkSNHZvbs2bntttv+6Ny7urrS2NiYzs7ONDQ0vP+Lc0fjYfZ1Hv3PA4AaOOK/v48CNf8MYGfn/wbCkCFDkiTr1q3L/v37M2XKlOqYc889N6NGjUp7e3uSpL29PRdccEE1/pKkpaUlXV1deemll6pj3nqMg2MOHmPfvn1Zt25djzH9+/fPlClTqmMAAI5Fx9Xyxbu7uzNnzpx85CMfyfnnn58k6ejoSF1dXQYPHtxj7LBhw9LR0VEd89b4O7j/4L7Djenq6sobb7yR3/72tzlw4MAhx2zatOmQ8927d2/27t1b/bmrq+s9vHsAgNqo6RXA1tbWbNy4MQ888EAtp/EnW7RoURobG6uPkSNH1npKAADvWM0CcNasWXn88cfz4x//OCNGjKhub2pqyr59+7Jr164e47dv356mpqbqmD+8K/jgz39sTENDQ0444YScdtppGTBgwCHHHDzGH5o/f346Ozurj61bt76nNQAAqIVeD8BKpZJZs2blkUceyZNPPpnRo0f32D9+/Pgcf/zxWbNmTXXb5s2bs2XLljQ3NydJmpubs2HDhh53665evToNDQ0ZM2ZMdcxbj3FwzMFj1NXVZfz48T3GdHd3Z82aNdUxf6i+vj4NDQ09HgAAR5te/wxga2trli9fnh/+8IcZNGhQ9TN7jY2NOeGEE9LY2JiZM2emra0tQ4YMSUNDQ2bPnp3m5uZMnjw5SXL55ZdnzJgx+dznPpfFixeno6Mjt99+e1pbW1NfX58kuemmm7JkyZLceuutueGGG/Lkk0/moYceyooVK6pzaWtry4wZMzJhwoRMnDgxd911V3bv3p3rr7++t5cFAKDX9HoA3nfffUmSj33sYz22f/e7380XvvCFJMm3v/3t9O/fP9OmTcvevXvT0tKSe++9tzp2wIABefzxx3PzzTenubk5J510UmbMmJGvf/3r1TGjR4/OihUrMnfu3Nx9990ZMWJE7r///rS0tFTHTJ8+PTt37syCBQvS0dGRCy+8MCtXrnzbjSEAAMeSmn8P4NHM9wC+x+cBQA34HsA+8D2AAAD0LgEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUJjjaj0BCnZH42H2dfbmTACgKK4AAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFKbXA/Dpp5/Opz71qQwfPjz9+vXLo48+2mN/pVLJggULcsYZZ+SEE07IlClT8vOf/7zHmN/85je57rrr0tDQkMGDB2fmzJl5/fXXe4z56U9/mo9+9KMZOHBgRo4cmcWLF79tLg8//HDOPffcDBw4MBdccEH+4z/+4wi9awCAvqPXA3D37t0ZN25c7rnnnkPuX7x4cb7zne9k6dKlefbZZ3PSSSelpaUle/bsqY657rrr8tJLL2X16tV5/PHH8/TTT+dLX/pSdX9XV1cuv/zynHnmmVm3bl2+9a1v5Y477sg///M/V8c888wz+cxnPpOZM2fmv//7v3PVVVflqquuysaNG4/wCgAA1Fa/SqVSqdmL9+uXRx55JFdddVXy+6t/w4cPz9/+7d/mK1/5SpKks7Mzw4YNy7Jly3LttdfmZz/7WcaMGZPnn38+EyZMSJKsXLkyn/jEJ/LLX/4yw4cPz3333Ze/+7u/S0dHR+rq6pIkt912Wx599NFs2rQpSTJ9+vTs3r07jz/+eHU+kydPzoUXXpilS5f+SfPv6upKY2NjOjs709DQ8L6vT+5oPMy+znKfBwDvwRH//X0U6FOfAXzllVfS0dGRKVOmVLc1NjZm0qRJaW9vT5K0t7dn8ODB1fhLkilTpqR///559tlnq2MuueSSavwlSUtLSzZv3pzf/va31TFvfZ2DYw6+DgDAseq4Wk/grTo6OpIkw4YN67F92LBh1X0dHR0ZOnRoj/3HHXdchgwZ0mPM6NGj33aMg/tOOeWUdHR0HPZ1DmXv3r3Zu3dv9eeurq53+U4BAGqnT10B7OsWLVqUxsbG6mPkyJG1nhIAwDvWpwKwqakpSbJ9+/Ye27dv317d19TUlB07dvTY/+abb+Y3v/lNjzGHOsZbX+P/GnNw/6HMnz8/nZ2d1cfWrVvfw7sFAKiNPhWAo0ePTlNTU9asWVPd1tXVlWeffTbNzc1Jkubm5uzatSvr1q2rjnnyySfT3d2dSZMmVcc8/fTT2b9/f3XM6tWrc8455+SUU06pjnnr6xwcc/B1DqW+vj4NDQ09HgAAR5teD8DXX38969evz/r165Pf3/ixfv36bNmyJf369cucOXPyjW98I//+7/+eDRs25POf/3yGDx9evVP4vPPOyxVXXJEbb7wxzz33XP7rv/4rs2bNyrXXXpvhw4cnST772c+mrq4uM2fOzEsvvZQHH3wwd999d9ra2qrz+PKXv5yVK1fm7//+77Np06bccccdeeGFFzJr1qzeXhIAgF7V6zeBvPDCC7n00kurPx+MshkzZmTZsmW59dZbs3v37nzpS1/Krl278pd/+ZdZuXJlBg4cWH3O97///cyaNSuXXXZZ+vfvn2nTpuU73/lOdX9jY2N+9KMfpbW1NePHj89pp52WBQsW9PiuwL/4i7/I8uXLc/vtt+erX/1qPvzhD+fRRx/N+eef32trAQBQCzX9HsCjne8BrNHzAOA98D2AfewzgAAAHHkCEACgMAIQAKAwAhAAoDACEACgMH3q3wKG98ydxQDwR7kCCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQmONqPQE4qt3ReJh9nb05EwD4k7kCCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBhfAwO14OtjAKghVwABAAojAAEACuNPwHA08adjAN4HrgACABTGFUDg3V9Z7O0rksfCFdCjZa1Ldyys97HwHjhiBCDQ+46FcDxafrn2teDs7fkcLf87Hc6x8P+XvvR6JAIQAN5HJcdMye/9KOQzgAAAhRGAAACFEYAAAIURgAAAhRGAAACFcRcwxN1rAJTFFUAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIUH4D33HNPzjrrrAwcODCTJk3Kc889V+spAQAcUUUH4IMPPpi2trYsXLgwL774YsaNG5eWlpbs2LGj1lMDADhiig7Af/iHf8iNN96Y66+/PmPGjMnSpUtz4okn5l/+5V9qPTUAgCPmuFpPoFb27duXdevWZf78+dVt/fv3z5QpU9Le3n7I5+zduzd79+6t/tzZ2Zkk6erqOjKT3Fv5v/cd7jWP9ecdztHyHvrSmtViPkfL8w7naHkPfWnNajGfo+V5h3O0vIe+tGZ/xMHf25XKYV77GNevUui737ZtWz7wgQ/kmWeeSXNzc3X7rbfemrVr1+bZZ59923PuuOOO3Hnnnb08UwDgSNi6dWtGjBhR62nURLFXAN+N+fPnp62trfpzd3d3fvOb3+TUU09Nv379eozt6urKyJEjs3Xr1jQ0NNRgtkcn6/bOWbN3x7q9O9bt3bFu79yRXLNKpZLXXnstw4cPf1+PezQpNgBPO+20DBgwINu3b++xffv27Wlqajrkc+rr61NfX99j2+DBgw/7Og0NDf7P/i5Yt3fOmr071u3dsW7vjnV7547UmjU2Nr7vxzyaFHsTSF1dXcaPH581a9ZUt3V3d2fNmjU9/iQMAHCsKfYKYJK0tbVlxowZmTBhQiZOnJi77roru3fvzvXXX1/rqQEAHDFFB+D06dOzc+fOLFiwIB0dHbnwwguzcuXKDBs27D0fu76+PgsXLnzbn4w5POv2zlmzd8e6vTvW7d2xbu+cNTuyir0LGACgVMV+BhAAoFQCEACgMAIQAKAwAhAAoDAC8Ai45557ctZZZ2XgwIGZNGlSnnvuuVpPqU+744470q9fvx6Pc889t9bT6nOefvrpfOpTn8rw4cPTr1+/PProoz32VyqVLFiwIGeccUZOOOGETJkyJT//+c9rNt++4o+t2xe+8IW3nX9XXHFFzebbFyxatCgXX3xxBg0alKFDh+aqq67K5s2be4zZs2dPWltbc+qpp+bkk0/OtGnT3vbF+qX5U9btYx/72NvOt5tuuqlmc+4L7rvvvowdO7b6hc/Nzc154oknqvuda0eGAHyfPfjgg2lra8vChQvz4osvZty4cWlpacmOHTtqPbU+7c///M/z6quvVh//+Z//Wesp9Tm7d+/OuHHjcs899xxy/+LFi/Od73wnS5cuzbPPPpuTTjopLS0t2bNnT6/PtS/5Y+uWJFdccUWP8+8HP/hBr86xr1m7dm1aW1vzk5/8JKtXr87+/ftz+eWXZ/fu3dUxc+fOzWOPPZaHH344a9euzbZt23L11VfXdN619qesW5LceOONPc63xYsX12zOfcGIESPyzW9+M+vWrcsLL7yQj3/84/n0pz+dl156KXGuHTkV3lcTJ06stLa2Vn8+cOBAZfjw4ZVFixbVdF592cKFCyvjxo2r9TSOKkkqjzzySPXn7u7uSlNTU+Vb3/pWdduuXbsq9fX1lR/84Ac1mmXf84frVqlUKjNmzKh8+tOfrtmcjgY7duyoJKmsXbu2Uvn9uXX88cdXHn744eqYn/3sZ5Uklfb29hrOtG/5w3WrVCqV//f//l/ly1/+ck3ndTQ45ZRTKvfff79z7QhyBfB9tG/fvqxbty5Tpkypbuvfv3+mTJmS9vb2ms6tr/v5z3+e4cOH54Mf/GCuu+66bNmypdZTOqq88sor6ejo6HHuNTY2ZtKkSc69P8FTTz2VoUOH5pxzzsnNN9+cX//617WeUp/S2dmZJBkyZEiSZN26ddm/f3+P8+3cc8/NqFGjnG9v8YfrdtD3v//9nHbaaTn//PMzf/78/O53v6vRDPueAwcO5IEHHsju3bvT3NzsXDuCiv6XQN5vv/rVr3LgwIG3/Usiw4YNy6ZNm2o2r75u0qRJWbZsWc4555y8+uqrufPOO/PRj340GzduzKBBg2o9vaNCR0dH8vtz7a2GDRtW3cehXXHFFbn66qszevTovPzyy/nqV7+aK6+8Mu3t7RkwYECtp1dz3d3dmTNnTj7ykY/k/PPPT35/vtXV1WXw4ME9xjrf/n+HWrck+exnP5szzzwzw4cPz09/+tPMmzcvmzdvzr/927/VdL61tmHDhjQ3N2fPnj05+eST88gjj2TMmDFZv369c+0IEYDU3JVXXln977Fjx2bSpEk588wz89BDD2XmzJk1nRvHvmuvvbb63xdccEHGjh2bD33oQ3nqqady2WWX1XRufUFra2s2btzoc7nv0P+1bl/60peq/33BBRfkjDPOyGWXXZaXX345H/rQh2ow077hnHPOyfr169PZ2Zl//dd/zYwZM7J27dpaT+uY5k/A76PTTjstAwYMeNvdSdu3b09TU1PN5nW0GTx4cP7sz/4sv/jFL2o9laPGwfPLuffeffCDH8xpp53m/Esya9asPP744/nxj3+cESNGVLc3NTVl37592bVrV4/xzrf/9X+t26FMmjQpSYo/3+rq6nL22Wdn/PjxWbRoUcaNG5e7777buXYECcD3UV1dXcaPH581a9ZUt3V3d2fNmjVpbm6u6dyOJq+//npefvnlnHHGGbWeylFj9OjRaWpq6nHudXV15dlnn3XuvUO//OUv8+tf/7ro869SqWTWrFl55JFH8uSTT2b06NE99o8fPz7HH398j/Nt8+bN2bJlS9Hn2x9bt0NZv359khR9vh1Kd3d39u7d61w7gvwJ+H3W1taWGTNmZMKECZk4cWLuuuuu7N69O9dff32tp9ZnfeUrX8mnPvWpnHnmmdm2bVsWLlyYAQMG5DOf+Uytp9anvP766z2uErzyyitZv359hgwZklGjRmXOnDn5xje+kQ9/+MMZPXp0vva1r2X48OG56qqrajrvWjvcug0ZMiR33nlnpk2blqamprz88su59dZbc/bZZ6elpaWm866l1tbWLF++PD/84Q8zaNCg6metGhsbc8IJJ6SxsTEzZ85MW1tbhgwZkoaGhsyePTvNzc2ZPHlyradfM39s3V5++eUsX748n/jEJ3Lqqafmpz/9aebOnZtLLrkkY8eOrfX0a2b+/Pm58sorM2rUqLz22mtZvnx5nnrqqaxatcq5diTV+jbkY9E//uM/VkaNGlWpq6urTJw4sfKTn/yk1lPq06ZPn14544wzKnV1dZUPfOADlenTp1d+8Ytf1Hpafc6Pf/zjSpK3PWbMmFGp/P6rYL72ta9Vhg0bVqmvr69cdtlllc2bN9d62jV3uHX73e9+V7n88ssrp59+euX444+vnHnmmZUbb7yx0tHRUetp19Sh1itJ5bvf/W51zBtvvFH5m7/5m8opp5xSOfHEEyt//dd/XXn11VdrOu9a+2PrtmXLlsoll1xSGTJkSKW+vr5y9tlnV2655ZZKZ2dnradeUzfccEPlzDPPrNTV1VVOP/30ymWXXVb50Y9+VN3vXDsy+lX+96QFAKAQPgMIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFCY/w8tmfmOTDaPEwAAAABJRU5ErkJggg==", - "text/html": [ - "\n", - " <div style=\"display: inline-block;\">\n", - " <div class=\"jupyter-widgets widget-label\" style=\"text-align: center;\">\n", - " Figure 2\n", - " </div>\n", - " <img src='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAoAAAAHgCAYAAAA10dzkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAl3klEQVR4nO3df4xV9YH//xegM/iDGUSFkQJKa1dlFYwgMNutn1pZR0ubumKCrWmpUhvdgRRmq0jXgjZNaGh2q11UdmO29I9Sf2xWu8oKJVgxu079gUsKbiGtMYEGB+gPZpTKD5n7/WPL/TqVpfUHcwfej0dyE+ec9z33fd89zTxz5p5Lv0qlUgkAAMXoX+sJAADQuwQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhjqv1BI5m3d3d2bZtWwYNGpR+/frVejoAwJ+gUqnktddey/Dhw9O/f5nXwgTge7Bt27aMHDmy1tMAAN6FrVu3ZsSIEbWeRk0IwPdg0KBBye9PoIaGhlpPBwD4E3R1dWXkyJHV3+MlEoDvwcE/+zY0NAhAADjKlPzxrTL/8A0AUDABCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQmONqPQGOgDsaD7OvszdnAgD0Qa4AAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAUpqYB+M1vfjP9+vXLnDlzqtv27NmT1tbWnHrqqTn55JMzbdq0bN++vcfztmzZkqlTp+bEE0/M0KFDc8stt+TNN9/sMeapp57KRRddlPr6+px99tlZtmzZ217/nnvuyVlnnZWBAwdm0qRJee65547guwUA6BtqFoDPP/98/umf/iljx47tsX3u3Ll57LHH8vDDD2ft2rXZtm1brr766ur+AwcOZOrUqdm3b1+eeeaZfO9738uyZcuyYMGC6phXXnklU6dOzaWXXpr169dnzpw5+eIXv5hVq1ZVxzz44INpa2vLwoUL8+KLL2bcuHFpaWnJjh07emkFAABqo1+lUqn09ou+/vrrueiii3LvvffmG9/4Ri688MLcdddd6ezszOmnn57ly5fnmmuuSZJs2rQp5513Xtrb2zN58uQ88cQT+eQnP5lt27Zl2LBhSZKlS5dm3rx52blzZ+rq6jJv3rysWLEiGzdurL7mtddem127dmXlypVJkkmTJuXiiy/OkiVLkiTd3d0ZOXJkZs+endtuu+1Peh9dXV1pbGxMZ2dnGhoajsBKvUt3NB5mX2dvzgQA+pw++/u7F9XkCmBra2umTp2aKVOm9Ni+bt267N+/v8f2c889N6NGjUp7e3uSpL29PRdccEE1/pKkpaUlXV1deemll6pj/vDYLS0t1WPs27cv69at6zGmf//+mTJlSnXMoezduzddXV09HgAAR5vjevsFH3jggbz44ot5/vnn37avo6MjdXV1GTx4cI/tw4YNS0dHR3XMW+Pv4P6D+w43pqurK2+88UZ++9vf5sCBA4ccs2nTpv9z7osWLcqdd975jt8zAEBf0qtXALdu3Zovf/nL+f73v5+BAwf25ku/L+bPn5/Ozs7qY+vWrbWeEgDAO9arVwDXrVuXHTt25KKLLqpuO3DgQJ5++uksWbIkq1atyr59+7Jr164eVwG3b9+epqamJElTU9Pb7tY9eJfwW8f84Z3D27dvT0NDQ0444YQMGDAgAwYMOOSYg8c4lPr6+tTX17+nNejTfHYQAIrQq1cAL7vssmzYsCHr16+vPiZMmJDrrruu+t/HH3981qxZU33O5s2bs2XLljQ3NydJmpubs2HDhh53665evToNDQ0ZM2ZMdcxbj3FwzMFj1NXVZfz48T3GdHd3Z82aNdUxAADHql69Ajho0KCcf/75PbaddNJJOfXUU6vbZ86cmba2tgwZMiQNDQ2ZPXt2mpubM3ny5CTJ5ZdfnjFjxuRzn/tcFi9enI6Ojtx+++1pbW2tXp276aabsmTJktx666254YYb8uSTT+ahhx7KihUrqq/b1taWGTNmZMKECZk4cWLuuuuu7N69O9dff31vLgkAQK/r9ZtA/phvf/vb6d+/f6ZNm5a9e/empaUl9957b3X/gAED8vjjj+fmm29Oc3NzTjrppMyYMSNf//rXq2NGjx6dFStWZO7cubn77rszYsSI3H///WlpaamOmT59enbu3JkFCxako6MjF154YVauXPm2G0MAAI41NfkewGNFn/0eoXf7WT6fAQSgAH3293cv8m8BAwAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFOa4Wk+Aw7ij8TD7OntzJgDAMcQVQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgML0egDed999GTt2bBoaGtLQ0JDm5uY88cQT1f179uxJa2trTj311Jx88smZNm1atm/f3uMYW7ZsydSpU3PiiSdm6NChueWWW/Lmm2/2GPPUU0/loosuSn19fc4+++wsW7bsbXO55557ctZZZ2XgwIGZNGlSnnvuuSP4zgEA+oZeD8ARI0bkm9/8ZtatW5cXXnghH//4x/PpT386L730UpJk7ty5eeyxx/Lwww9n7dq12bZtW66++urq8w8cOJCpU6dm3759eeaZZ/K9730vy5Yty4IFC6pjXnnllUydOjWXXnpp1q9fnzlz5uSLX/xiVq1aVR3z4IMPpq2tLQsXLsyLL76YcePGpaWlJTt27OjlFQEA6F39KpVKpdaTGDJkSL71rW/lmmuuyemnn57ly5fnmmuuSZJs2rQp5513Xtrb2zN58uQ88cQT+eQnP5lt27Zl2LBhSZKlS5dm3rx52blzZ+rq6jJv3rysWLEiGzdurL7Gtddem127dmXlypVJkkmTJuXiiy/OkiVLkiTd3d0ZOXJkZs+endtuu+1PmndXV1caGxvT2dmZhoaG939h7mg8zL7OvvM8ADiKHPHf30eBmn4G8MCBA3nggQeye/fuNDc3Z926ddm/f3+mTJlSHXPuuedm1KhRaW9vT5K0t7fnggsuqMZfkrS0tKSrq6t6FbG9vb3HMQ6OOXiMffv2Zd26dT3G9O/fP1OmTKmOOZS9e/emq6urxwMA4GhTkwDcsGFDTj755NTX1+emm27KI488kjFjxqSjoyN1dXUZPHhwj/HDhg1LR0dHkqSjo6NH/B3cf3Df4cZ0dXXljTfeyK9+9ascOHDgkGMOHuNQFi1alMbGxupj5MiR73ElAAB6X00C8Jxzzsn69evz7LPP5uabb86MGTPyP//zP7WYyjsyf/78dHZ2Vh9bt26t9ZQAAN6x42rxonV1dTn77LOTJOPHj8/zzz+fu+++O9OnT8++ffuya9euHlcBt2/fnqampiRJU1PT2+7WPXiX8FvH/OGdw9u3b09DQ0NOOOGEDBgwIAMGDDjkmIPHOJT6+vrU19e/5/cPAFBLfeJ7ALu7u7N3796MHz8+xx9/fNasWVPdt3nz5mzZsiXNzc1Jkubm5mzYsKHH3bqrV69OQ0NDxowZUx3z1mMcHHPwGHV1dRk/fnyPMd3d3VmzZk11DADAsarXrwDOnz8/V155ZUaNGpXXXnsty5cvz1NPPZVVq1alsbExM2fOTFtbW4YMGZKGhobMnj07zc3NmTx5cpLk8ssvz5gxY/K5z30uixcvTkdHR26//fa0trZWr87ddNNNWbJkSW699dbccMMNefLJJ/PQQw9lxYoV1Xm0tbVlxowZmTBhQiZOnJi77roru3fvzvXXX9/bSwIA0Kt6PQB37NiRz3/+83n11VfT2NiYsWPHZtWqVfmrv/qrJMm3v/3t9O/fP9OmTcvevXvT0tKSe++9t/r8AQMG5PHHH8/NN9+c5ubmnHTSSZkxY0a+/vWvV8eMHj06K1asyNy5c3P33XdnxIgRuf/++9PS0lIdM3369OzcuTMLFixIR0dHLrzwwqxcufJtN4YAABxr+sT3AB6tfA8gABx9fA9gH/kMIAAAvUcAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAU5rhaT4BjgH9DGACOKq4AAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAUptcDcNGiRbn44oszaNCgDB06NFdddVU2b97cY8yePXvS2tqaU089NSeffHKmTZuW7du39xizZcuWTJ06NSeeeGKGDh2aW265JW+++WaPMU899VQuuuii1NfX5+yzz86yZcveNp977rknZ511VgYOHJhJkyblueeeO0LvHACgb+j1AFy7dm1aW1vzk5/8JKtXr87+/ftz+eWXZ/fu3dUxc+fOzWOPPZaHH344a9euzbZt23L11VdX9x84cCBTp07Nvn378swzz+R73/teli1blgULFlTHvPLKK5k6dWouvfTSrF+/PnPmzMkXv/jFrFq1qjrmwQcfTFtbWxYuXJgXX3wx48aNS0tLS3bs2NGLKwIA0Lv6VSqVSi0nsHPnzgwdOjRr167NJZdcks7Ozpx++ulZvnx5rrnmmiTJpk2bct5556W9vT2TJ0/OE088kU9+8pPZtm1bhg0bliRZunRp5s2bl507d6auri7z5s3LihUrsnHjxuprXXvttdm1a1dWrlyZJJk0aVIuvvjiLFmyJEnS3d2dkSNHZvbs2bntttv+6Ny7urrS2NiYzs7ONDQ0vP+Lc0fjYfZ1Hv3PA4AaOOK/v48CNf8MYGfn/wbCkCFDkiTr1q3L/v37M2XKlOqYc889N6NGjUp7e3uSpL29PRdccEE1/pKkpaUlXV1deemll6pj3nqMg2MOHmPfvn1Zt25djzH9+/fPlClTqmMAAI5Fx9Xyxbu7uzNnzpx85CMfyfnnn58k6ejoSF1dXQYPHtxj7LBhw9LR0VEd89b4O7j/4L7Djenq6sobb7yR3/72tzlw4MAhx2zatOmQ8927d2/27t1b/bmrq+s9vHsAgNqo6RXA1tbWbNy4MQ888EAtp/EnW7RoURobG6uPkSNH1npKAADvWM0CcNasWXn88cfz4x//OCNGjKhub2pqyr59+7Jr164e47dv356mpqbqmD+8K/jgz39sTENDQ0444YScdtppGTBgwCHHHDzGH5o/f346Ozurj61bt76nNQAAqIVeD8BKpZJZs2blkUceyZNPPpnRo0f32D9+/Pgcf/zxWbNmTXXb5s2bs2XLljQ3NydJmpubs2HDhh53665evToNDQ0ZM2ZMdcxbj3FwzMFj1NXVZfz48T3GdHd3Z82aNdUxf6i+vj4NDQ09HgAAR5te/wxga2trli9fnh/+8IcZNGhQ9TN7jY2NOeGEE9LY2JiZM2emra0tQ4YMSUNDQ2bPnp3m5uZMnjw5SXL55ZdnzJgx+dznPpfFixeno6Mjt99+e1pbW1NfX58kuemmm7JkyZLceuutueGGG/Lkk0/moYceyooVK6pzaWtry4wZMzJhwoRMnDgxd911V3bv3p3rr7++t5cFAKDX9HoA3nfffUmSj33sYz22f/e7380XvvCFJMm3v/3t9O/fP9OmTcvevXvT0tKSe++9tzp2wIABefzxx3PzzTenubk5J510UmbMmJGvf/3r1TGjR4/OihUrMnfu3Nx9990ZMWJE7r///rS0tFTHTJ8+PTt37syCBQvS0dGRCy+8MCtXrnzbjSEAAMeSmn8P4NHM9wC+x+cBQA34HsA+8D2AAAD0LgEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUJjjaj0BCnZH42H2dfbmTACgKK4AAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFKbXA/Dpp5/Opz71qQwfPjz9+vXLo48+2mN/pVLJggULcsYZZ+SEE07IlClT8vOf/7zHmN/85je57rrr0tDQkMGDB2fmzJl5/fXXe4z56U9/mo9+9KMZOHBgRo4cmcWLF79tLg8//HDOPffcDBw4MBdccEH+4z/+4wi9awCAvqPXA3D37t0ZN25c7rnnnkPuX7x4cb7zne9k6dKlefbZZ3PSSSelpaUle/bsqY657rrr8tJLL2X16tV5/PHH8/TTT+dLX/pSdX9XV1cuv/zynHnmmVm3bl2+9a1v5Y477sg///M/V8c888wz+cxnPpOZM2fmv//7v3PVVVflqquuysaNG4/wCgAA1Fa/SqVSqdmL9+uXRx55JFdddVXy+6t/w4cPz9/+7d/mK1/5SpKks7Mzw4YNy7Jly3LttdfmZz/7WcaMGZPnn38+EyZMSJKsXLkyn/jEJ/LLX/4yw4cPz3333Ze/+7u/S0dHR+rq6pIkt912Wx599NFs2rQpSTJ9+vTs3r07jz/+eHU+kydPzoUXXpilS5f+SfPv6upKY2NjOjs709DQ8L6vT+5oPMy+znKfBwDvwRH//X0U6FOfAXzllVfS0dGRKVOmVLc1NjZm0qRJaW9vT5K0t7dn8ODB1fhLkilTpqR///559tlnq2MuueSSavwlSUtLSzZv3pzf/va31TFvfZ2DYw6+DgDAseq4Wk/grTo6OpIkw4YN67F92LBh1X0dHR0ZOnRoj/3HHXdchgwZ0mPM6NGj33aMg/tOOeWUdHR0HPZ1DmXv3r3Zu3dv9eeurq53+U4BAGqnT10B7OsWLVqUxsbG6mPkyJG1nhIAwDvWpwKwqakpSbJ9+/Ye27dv317d19TUlB07dvTY/+abb+Y3v/lNjzGHOsZbX+P/GnNw/6HMnz8/nZ2d1cfWrVvfw7sFAKiNPhWAo0ePTlNTU9asWVPd1tXVlWeffTbNzc1Jkubm5uzatSvr1q2rjnnyySfT3d2dSZMmVcc8/fTT2b9/f3XM6tWrc8455+SUU06pjnnr6xwcc/B1DqW+vj4NDQ09HgAAR5teD8DXX38969evz/r165Pf3/ixfv36bNmyJf369cucOXPyjW98I//+7/+eDRs25POf/3yGDx9evVP4vPPOyxVXXJEbb7wxzz33XP7rv/4rs2bNyrXXXpvhw4cnST772c+mrq4uM2fOzEsvvZQHH3wwd999d9ra2qrz+PKXv5yVK1fm7//+77Np06bccccdeeGFFzJr1qzeXhIAgF7V6zeBvPDCC7n00kurPx+MshkzZmTZsmW59dZbs3v37nzpS1/Krl278pd/+ZdZuXJlBg4cWH3O97///cyaNSuXXXZZ+vfvn2nTpuU73/lOdX9jY2N+9KMfpbW1NePHj89pp52WBQsW9PiuwL/4i7/I8uXLc/vtt+erX/1qPvzhD+fRRx/N+eef32trAQBQCzX9HsCjne8BrNHzAOA98D2AfewzgAAAHHkCEACgMAIQAKAwAhAAoDACEACgMH3q3wKG98ydxQDwR7kCCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQmONqPQE4qt3ReJh9nb05EwD4k7kCCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBhfAwO14OtjAKghVwABAAojAAEACuNPwHA08adjAN4HrgACABTGFUDg3V9Z7O0rksfCFdCjZa1Ldyys97HwHjhiBCDQ+46FcDxafrn2teDs7fkcLf87Hc6x8P+XvvR6JAIQAN5HJcdMye/9KOQzgAAAhRGAAACFEYAAAIURgAAAhRGAAACFcRcwxN1rAJTFFUAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIUH4D33HNPzjrrrAwcODCTJk3Kc889V+spAQAcUUUH4IMPPpi2trYsXLgwL774YsaNG5eWlpbs2LGj1lMDADhiig7Af/iHf8iNN96Y66+/PmPGjMnSpUtz4okn5l/+5V9qPTUAgCPmuFpPoFb27duXdevWZf78+dVt/fv3z5QpU9Le3n7I5+zduzd79+6t/tzZ2Zkk6erqOjKT3Fv5v/cd7jWP9ecdztHyHvrSmtViPkfL8w7naHkPfWnNajGfo+V5h3O0vIe+tGZ/xMHf25XKYV77GNevUui737ZtWz7wgQ/kmWeeSXNzc3X7rbfemrVr1+bZZ59923PuuOOO3Hnnnb08UwDgSNi6dWtGjBhR62nURLFXAN+N+fPnp62trfpzd3d3fvOb3+TUU09Nv379eozt6urKyJEjs3Xr1jQ0NNRgtkcn6/bOWbN3x7q9O9bt3bFu79yRXLNKpZLXXnstw4cPf1+PezQpNgBPO+20DBgwINu3b++xffv27Wlqajrkc+rr61NfX99j2+DBgw/7Og0NDf7P/i5Yt3fOmr071u3dsW7vjnV7547UmjU2Nr7vxzyaFHsTSF1dXcaPH581a9ZUt3V3d2fNmjU9/iQMAHCsKfYKYJK0tbVlxowZmTBhQiZOnJi77roru3fvzvXXX1/rqQEAHDFFB+D06dOzc+fOLFiwIB0dHbnwwguzcuXKDBs27D0fu76+PgsXLnzbn4w5POv2zlmzd8e6vTvW7d2xbu+cNTuyir0LGACgVMV+BhAAoFQCEACgMAIQAKAwAhAAoDAC8Ai45557ctZZZ2XgwIGZNGlSnnvuuVpPqU+744470q9fvx6Pc889t9bT6nOefvrpfOpTn8rw4cPTr1+/PProoz32VyqVLFiwIGeccUZOOOGETJkyJT//+c9rNt++4o+t2xe+8IW3nX9XXHFFzebbFyxatCgXX3xxBg0alKFDh+aqq67K5s2be4zZs2dPWltbc+qpp+bkk0/OtGnT3vbF+qX5U9btYx/72NvOt5tuuqlmc+4L7rvvvowdO7b6hc/Nzc154oknqvuda0eGAHyfPfjgg2lra8vChQvz4osvZty4cWlpacmOHTtqPbU+7c///M/z6quvVh//+Z//Wesp9Tm7d+/OuHHjcs899xxy/+LFi/Od73wnS5cuzbPPPpuTTjopLS0t2bNnT6/PtS/5Y+uWJFdccUWP8+8HP/hBr86xr1m7dm1aW1vzk5/8JKtXr87+/ftz+eWXZ/fu3dUxc+fOzWOPPZaHH344a9euzbZt23L11VfXdN619qesW5LceOONPc63xYsX12zOfcGIESPyzW9+M+vWrcsLL7yQj3/84/n0pz+dl156KXGuHTkV3lcTJ06stLa2Vn8+cOBAZfjw4ZVFixbVdF592cKFCyvjxo2r9TSOKkkqjzzySPXn7u7uSlNTU+Vb3/pWdduuXbsq9fX1lR/84Ac1mmXf84frVqlUKjNmzKh8+tOfrtmcjgY7duyoJKmsXbu2Uvn9uXX88cdXHn744eqYn/3sZ5Uklfb29hrOtG/5w3WrVCqV//f//l/ly1/+ck3ndTQ45ZRTKvfff79z7QhyBfB9tG/fvqxbty5Tpkypbuvfv3+mTJmS9vb2ms6tr/v5z3+e4cOH54Mf/GCuu+66bNmypdZTOqq88sor6ejo6HHuNTY2ZtKkSc69P8FTTz2VoUOH5pxzzsnNN9+cX//617WeUp/S2dmZJBkyZEiSZN26ddm/f3+P8+3cc8/NqFGjnG9v8YfrdtD3v//9nHbaaTn//PMzf/78/O53v6vRDPueAwcO5IEHHsju3bvT3NzsXDuCiv6XQN5vv/rVr3LgwIG3/Usiw4YNy6ZNm2o2r75u0qRJWbZsWc4555y8+uqrufPOO/PRj340GzduzKBBg2o9vaNCR0dH8vtz7a2GDRtW3cehXXHFFbn66qszevTovPzyy/nqV7+aK6+8Mu3t7RkwYECtp1dz3d3dmTNnTj7ykY/k/PPPT35/vtXV1WXw4ME9xjrf/n+HWrck+exnP5szzzwzw4cPz09/+tPMmzcvmzdvzr/927/VdL61tmHDhjQ3N2fPnj05+eST88gjj2TMmDFZv369c+0IEYDU3JVXXln977Fjx2bSpEk588wz89BDD2XmzJk1nRvHvmuvvbb63xdccEHGjh2bD33oQ3nqqady2WWX1XRufUFra2s2btzoc7nv0P+1bl/60peq/33BBRfkjDPOyGWXXZaXX345H/rQh2ow077hnHPOyfr169PZ2Zl//dd/zYwZM7J27dpaT+uY5k/A76PTTjstAwYMeNvdSdu3b09TU1PN5nW0GTx4cP7sz/4sv/jFL2o9laPGwfPLuffeffCDH8xpp53m/Esya9asPP744/nxj3+cESNGVLc3NTVl37592bVrV4/xzrf/9X+t26FMmjQpSYo/3+rq6nL22Wdn/PjxWbRoUcaNG5e7777buXYECcD3UV1dXcaPH581a9ZUt3V3d2fNmjVpbm6u6dyOJq+//npefvnlnHHGGbWeylFj9OjRaWpq6nHudXV15dlnn3XuvUO//OUv8+tf/7ro869SqWTWrFl55JFH8uSTT2b06NE99o8fPz7HH398j/Nt8+bN2bJlS9Hn2x9bt0NZv359khR9vh1Kd3d39u7d61w7gvwJ+H3W1taWGTNmZMKECZk4cWLuuuuu7N69O9dff32tp9ZnfeUrX8mnPvWpnHnmmdm2bVsWLlyYAQMG5DOf+Uytp9anvP766z2uErzyyitZv359hgwZklGjRmXOnDn5xje+kQ9/+MMZPXp0vva1r2X48OG56qqrajrvWjvcug0ZMiR33nlnpk2blqamprz88su59dZbc/bZZ6elpaWm866l1tbWLF++PD/84Q8zaNCg6metGhsbc8IJJ6SxsTEzZ85MW1tbhgwZkoaGhsyePTvNzc2ZPHlyradfM39s3V5++eUsX748n/jEJ3Lqqafmpz/9aebOnZtLLrkkY8eOrfX0a2b+/Pm58sorM2rUqLz22mtZvnx5nnrqqaxatcq5diTV+jbkY9E//uM/VkaNGlWpq6urTJw4sfKTn/yk1lPq06ZPn14544wzKnV1dZUPfOADlenTp1d+8Ytf1Hpafc6Pf/zjSpK3PWbMmFGp/P6rYL72ta9Vhg0bVqmvr69cdtlllc2bN9d62jV3uHX73e9+V7n88ssrp59+euX444+vnHnmmZUbb7yx0tHRUetp19Sh1itJ5bvf/W51zBtvvFH5m7/5m8opp5xSOfHEEyt//dd/XXn11VdrOu9a+2PrtmXLlsoll1xSGTJkSKW+vr5y9tlnV2655ZZKZ2dnradeUzfccEPlzDPPrNTV1VVOP/30ymWXXVb50Y9+VN3vXDsy+lX+96QFAKAQPgMIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFCY/w8tmfmOTDaPEwAAAABJRU5ErkJggg==' width=640.0/>\n", - " </div>\n", - " " - ], "text/plain": [ - "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …" + "(array([255., 0., 298., 0., 231., 0., 210., 0., 123., 0., 63.,\n", + " 0., 31., 0., 46., 0., 39., 0., 35., 0., 40., 0.,\n", + " 35., 0., 44., 0., 24., 0., 37., 0., 26., 0., 0.,\n", + " 39., 0., 31., 0., 31., 0., 38., 0., 34., 0., 36.,\n", + " 0., 33., 0., 33., 0., 22., 0., 38., 0., 28., 0.,\n", + " 34., 0., 26., 0., 30., 0., 26., 0., 32.]),\n", + " array([ 1. , 1.484375, 1.96875 , 2.453125, 2.9375 , 3.421875,\n", + " 3.90625 , 4.390625, 4.875 , 5.359375, 5.84375 , 6.328125,\n", + " 6.8125 , 7.296875, 7.78125 , 8.265625, 8.75 , 9.234375,\n", + " 9.71875 , 10.203125, 10.6875 , 11.171875, 11.65625 , 12.140625,\n", + " 12.625 , 13.109375, 13.59375 , 14.078125, 14.5625 , 15.046875,\n", + " 15.53125 , 16.015625, 16.5 , 16.984375, 17.46875 , 17.953125,\n", + " 18.4375 , 18.921875, 19.40625 , 19.890625, 20.375 , 20.859375,\n", + " 21.34375 , 21.828125, 22.3125 , 22.796875, 23.28125 , 23.765625,\n", + " 24.25 , 24.734375, 25.21875 , 25.703125, 26.1875 , 26.671875,\n", + " 27.15625 , 27.640625, 28.125 , 28.609375, 29.09375 , 29.578125,\n", + " 30.0625 , 30.546875, 31.03125 , 31.515625, 32. ]),\n", + " <BarContainer object of 64 artists>)" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAivElEQVR4nO3df1BVdeL/8RegXH/BJVS4sAL+KpEUa0nxTuXHFVZE1tWVZrTcstbR0YVmlTKlMTV3Z3Gt6eeQzs62UjOS5U7qSGUpJm4ralKOv4pRhxZbvNDqwFWMH8L5/vHJ+/3c/EEocN/A8zFzZrjnvO+973M85dNzf+BnWZYlAAAAg/j7egIAAAA/RqAAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAME4PX0/gVjQ3N6uiokJBQUHy8/Pz9XQAAMBPYFmWLl68qMjISPn73/waSacMlIqKCkVFRfl6GgAA4BacPXtWgwYNuumYThkoQUFB0g87GBwc7OvpAACAn8DtdisqKsrz9/jNdMpAufqyTnBwMIECAEAn81PensGbZAEAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGaVWgrF+/XvHx8Z4vSHM6nfroo4882+vq6pSRkaH+/furX79+Sk9PV2VlpddjlJeXKy0tTX369FFYWJiWLl2qK1eutN0eAQCATq9VgTJo0CCtXbtWJSUlOnz4sCZNmqTp06frxIkTkqQlS5Zox44d2rJli4qKilRRUaGZM2d67t/U1KS0tDQ1NDRo//79euutt5SXl6eVK1e2/Z4BAIBOy8+yLOt2HiA0NFQvvPCCHnroIQ0cOFD5+fl66KGHJElff/21Ro4cqeLiYo0fP14fffSRfvWrX6miokLh4eGSpA0bNmjZsmX67rvvFBgY+JOe0+12y263q6amhq+6BwCgk2jN39+3/B6UpqYmbd68WbW1tXI6nSopKVFjY6OSk5M9Y2JjYxUdHa3i4mJJUnFxsUaPHu2JE0lKSUmR2+32XIW5nvr6erndbq8FAAB0Xa0OlGPHjqlfv36y2WxauHChtm7dqri4OLlcLgUGBiokJMRrfHh4uFwulyTJ5XJ5xcnV7Ve33UhOTo7sdrtniYqKau20AQBAJ9LqQBkxYoSOHDmigwcPatGiRZo7d65OnjzZPrP7QXZ2tmpqajzL2bNn2/X5AACAb/Vo7R0CAwM1fPhwSVJCQoI+//xzvfrqq5o1a5YaGhpUXV3tdRWlsrJSDodDkuRwOHTo0CGvx7v6KZ+rY67HZrPJZrO1dqodbvDyD2647Zu1aR06FwAAOrPb/h6U5uZm1dfXKyEhQT179lRhYaFnW2lpqcrLy+V0OiVJTqdTx44dU1VVlWfMrl27FBwcrLi4uNudCgAA6CJadQUlOztbqampio6O1sWLF5Wfn6+9e/fq448/lt1u17x585SVlaXQ0FAFBwfrySeflNPp1Pjx4yVJkydPVlxcnB599FGtW7dOLpdLK1asUEZGRqe4QgIAADpGqwKlqqpKjz32mM6dOye73a74+Hh9/PHH+uUvfylJevnll+Xv76/09HTV19crJSVFb7zxhuf+AQEBKigo0KJFi+R0OtW3b1/NnTtXa9asafs9AwAAndZtfw+KL5j6PSi8BwUAgBvrkO9BAQAAaC8ECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADBOD19PwESDl39ww23frE3r0LkAANAdcQUFAAAYp1WBkpOTo7FjxyooKEhhYWGaMWOGSktLvcZMnDhRfn5+XsvChQu9xpSXlystLU19+vRRWFiYli5dqitXrrTNHgEAgE6vVS/xFBUVKSMjQ2PHjtWVK1f07LPPavLkyTp58qT69u3rGTd//nytWbPGc7tPnz6en5uampSWliaHw6H9+/fr3Llzeuyxx9SzZ0/9+c9/bqv9AgAAnVirAmXnzp1et/Py8hQWFqaSkhJNmDDBs75Pnz5yOBzXfYxPPvlEJ0+e1O7duxUeHq577rlHf/zjH7Vs2TKtXr1agYGBt7ovAACgi7it96DU1NRIkkJDQ73Wb9q0SQMGDNCoUaOUnZ2ty5cve7YVFxdr9OjRCg8P96xLSUmR2+3WiRMnrvs89fX1crvdXgsAAOi6bvlTPM3NzVq8eLHuv/9+jRo1yrP+kUceUUxMjCIjI3X06FEtW7ZMpaWlev/99yVJLpfLK04keW67XK7rPldOTo6ef/75W50qAADoZG45UDIyMnT8+HF99tlnXusXLFjg+Xn06NGKiIhQUlKSzpw5o2HDht3Sc2VnZysrK8tz2+12Kyoq6lanDgAADHdLL/FkZmaqoKBAn376qQYNGnTTsYmJiZKk06dPS5IcDocqKyu9xly9faP3rdhsNgUHB3stAACg62pVoFiWpczMTG3dulV79uzRkCFDWrzPkSNHJEkRERGSJKfTqWPHjqmqqsozZteuXQoODlZcXFzr9wAAAHQ5rXqJJyMjQ/n5+dq+fbuCgoI87xmx2+3q3bu3zpw5o/z8fE2dOlX9+/fX0aNHtWTJEk2YMEHx8fGSpMmTJysuLk6PPvqo1q1bJ5fLpRUrVigjI0M2m6199hIAAHQqrbqCsn79etXU1GjixImKiIjwLO+++64kKTAwULt379bkyZMVGxurp556Sunp6dqxY4fnMQICAlRQUKCAgAA5nU799re/1WOPPeb1vSkAAKB7a9UVFMuybro9KipKRUVFLT5OTEyMPvzww9Y8NQAA6Eb4XTwAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIzTw9cTgDR4+Qc33PbN2rQOnQsAACbgCgoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4PXw9Ady6wcs/uOG2b9amdehcAABoS1xBAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGCcVgVKTk6Oxo4dq6CgIIWFhWnGjBkqLS31GlNXV6eMjAz1799f/fr1U3p6uiorK73GlJeXKy0tTX369FFYWJiWLl2qK1eutM0eAQCATq9VgVJUVKSMjAwdOHBAu3btUmNjoyZPnqza2lrPmCVLlmjHjh3asmWLioqKVFFRoZkzZ3q2NzU1KS0tTQ0NDdq/f7/eeust5eXlaeXKlW27ZwAAoNNq1TfJ7ty50+t2Xl6ewsLCVFJSogkTJqimpkZvvvmm8vPzNWnSJEnSxo0bNXLkSB04cEDjx4/XJ598opMnT2r37t0KDw/XPffcoz/+8Y9atmyZVq9ercDAwLbdQwAA0Onc1ntQampqJEmhoaGSpJKSEjU2Nio5OdkzJjY2VtHR0SouLpYkFRcXa/To0QoPD/eMSUlJkdvt1okTJ677PPX19XK73V4LAADoum45UJqbm7V48WLdf//9GjVqlCTJ5XIpMDBQISEhXmPDw8Plcrk8Y/5vnFzdfnXb9eTk5Mhut3uWqKioW502AADoBG45UDIyMnT8+HFt3ry5bWd0HdnZ2aqpqfEsZ8+ebffnBAAAvnNLv804MzNTBQUF2rdvnwYNGuRZ73A41NDQoOrqaq+rKJWVlXI4HJ4xhw4d8nq8q5/yuTrmx2w2m2w2261MFQAAdEKtuoJiWZYyMzO1detW7dmzR0OGDPHanpCQoJ49e6qwsNCzrrS0VOXl5XI6nZIkp9OpY8eOqaqqyjNm165dCg4OVlxc3O3vEQAA6PRadQUlIyND+fn52r59u4KCgjzvGbHb7erdu7fsdrvmzZunrKwshYaGKjg4WE8++aScTqfGjx8vSZo8ebLi4uL06KOPat26dXK5XFqxYoUyMjK4SgIAAKTWBsr69eslSRMnTvRav3HjRj3++OOSpJdffln+/v5KT09XfX29UlJS9MYbb3jGBgQEqKCgQIsWLZLT6VTfvn01d+5crVmzpm32CAAAdHqtChTLsloc06tXL+Xm5io3N/eGY2JiYvThhx+25qkBAEA3wu/iAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGKfVgbJv3z5NmzZNkZGR8vPz07Zt27y2P/744/Lz8/NapkyZ4jXmwoULmjNnjoKDgxUSEqJ58+bp0qVLt783AACgS2h1oNTW1mrMmDHKzc294ZgpU6bo3LlznuWdd97x2j5nzhydOHFCu3btUkFBgfbt26cFCxbc2h4AAIAup0dr75CamqrU1NSbjrHZbHI4HNfd9tVXX2nnzp36/PPPdd9990mSXn/9dU2dOlUvvviiIiMjWzslAADQxbTLe1D27t2rsLAwjRgxQosWLdL58+c924qLixUSEuKJE0lKTk6Wv7+/Dh48eN3Hq6+vl9vt9loAAEDX1eaBMmXKFL399tsqLCzUX/7yFxUVFSk1NVVNTU2SJJfLpbCwMK/79OjRQ6GhoXK5XNd9zJycHNntds8SFRXV1tMGAAAGafVLPC2ZPXu25+fRo0crPj5ew4YN0969e5WUlHRLj5mdna2srCzPbbfbTaQAANCFtfvHjIcOHaoBAwbo9OnTkiSHw6GqqiqvMVeuXNGFCxdu+L4Vm82m4OBgrwUAAHRd7R4o3377rc6fP6+IiAhJktPpVHV1tUpKSjxj9uzZo+bmZiUmJrb3dAAAQCfQ6pd4Ll265LkaIkllZWU6cuSIQkNDFRoaqueff17p6elyOBw6c+aMnnnmGQ0fPlwpKSmSpJEjR2rKlCmaP3++NmzYoMbGRmVmZmr27Nl8ggcAAEi3cgXl8OHDuvfee3XvvfdKkrKysnTvvfdq5cqVCggI0NGjR/XrX/9ad911l+bNm6eEhAT985//lM1m8zzGpk2bFBsbq6SkJE2dOlUPPPCA/vrXv7btngEAgE6r1VdQJk6cKMuybrj9448/bvExQkNDlZ+f39qnBgAA3QS/iwcAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABinh68ngI43ePkHN9z2zdq0Dp0LAADXwxUUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGKfVgbJv3z5NmzZNkZGR8vPz07Zt27y2W5allStXKiIiQr1791ZycrJOnTrlNebChQuaM2eOgoODFRISonnz5unSpUu3vzcAAKBLaHWg1NbWasyYMcrNzb3u9nXr1um1117Thg0bdPDgQfXt21cpKSmqq6vzjJkzZ45OnDihXbt2qaCgQPv27dOCBQtub08AAECX0aO1d0hNTVVqaup1t1mWpVdeeUUrVqzQ9OnTJUlvv/22wsPDtW3bNs2ePVtfffWVdu7cqc8//1z33XefJOn111/X1KlT9eKLLyoyMvJ29wkAAHRybfoelLKyMrlcLiUnJ3vW2e12JSYmqri4WJJUXFyskJAQT5xIUnJysvz9/XXw4MHrPm59fb3cbrfXAgAAuq42DRSXyyVJCg8P91ofHh7u2eZyuRQWFua1vUePHgoNDfWM+bGcnBzZ7XbPEhUV1ZbTBgAAhukUn+LJzs5WTU2NZzl79qyvpwQAANpRmwaKw+GQJFVWVnqtr6ys9GxzOByqqqry2n7lyhVduHDBM+bHbDabgoODvRYAANB1tWmgDBkyRA6HQ4WFhZ51brdbBw8elNPplCQ5nU5VV1erpKTEM2bPnj1qbm5WYmJiW04HAAB0Uq3+FM+lS5d0+vRpz+2ysjIdOXJEoaGhio6O1uLFi/WnP/1Jd955p4YMGaLnnntOkZGRmjFjhiRp5MiRmjJliubPn68NGzaosbFRmZmZmj17Np/gAQAA0q0EyuHDh/WLX/zCczsrK0uSNHfuXOXl5emZZ55RbW2tFixYoOrqaj3wwAPauXOnevXq5bnPpk2blJmZqaSkJPn7+ys9PV2vvfZaW+0TAADo5FodKBMnTpRlWTfc7ufnpzVr1mjNmjU3HBMaGqr8/PzWPjUAAOgmOsWneAAAQPdCoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIzTw9cTQOcxePkHN9z2zdq0Dp0LAKBr4woKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjNPmgbJ69Wr5+fl5LbGxsZ7tdXV1ysjIUP/+/dWvXz+lp6ersrKyracBAAA6sXa5gnL33Xfr3LlznuWzzz7zbFuyZIl27NihLVu2qKioSBUVFZo5c2Z7TAMAAHRS7fLLAnv06CGHw3HN+pqaGr355pvKz8/XpEmTJEkbN27UyJEjdeDAAY0fP749pgMAADqZdrmCcurUKUVGRmro0KGaM2eOysvLJUklJSVqbGxUcnKyZ2xsbKyio6NVXFx8w8err6+X2+32WgAAQNfV5oGSmJiovLw87dy5U+vXr1dZWZkefPBBXbx4US6XS4GBgQoJCfG6T3h4uFwu1w0fMycnR3a73bNERUW19bQBAIBB2vwlntTUVM/P8fHxSkxMVExMjN577z317t37lh4zOztbWVlZnttut5tIAQCgC2v3jxmHhITorrvu0unTp+VwONTQ0KDq6mqvMZWVldd9z8pVNptNwcHBXgsAAOi62j1QLl26pDNnzigiIkIJCQnq2bOnCgsLPdtLS0tVXl4up9PZ3lMBAACdRJu/xPP0009r2rRpiomJUUVFhVatWqWAgAA9/PDDstvtmjdvnrKyshQaGqrg4GA9+eSTcjqdfIIHAAB4tHmgfPvtt3r44Yd1/vx5DRw4UA888IAOHDiggQMHSpJefvll+fv7Kz09XfX19UpJSdEbb7zR1tNAFzB4+Qc33PbN2rQOnQsAoGO1eaBs3rz5ptt79eql3Nxc5ebmtvVTAwCALqJdvqgNQPvi6hKAro5fFggAAIxDoAAAAOMQKAAAwDgECgAAMA5vkgXaAG9aBYC2RaCgyyEWujb+fIHugUABgHZASLU9jmn3wntQAACAcbiCAqBF/MsVQEcjUACgiyAk0ZUQKMAPbvY/d7XT/+D5CwUAro9AAQDDEK5m4M/BtwgUAOjmbvUvYv4CR3siUAAA6KZMjkw+ZgwAAIzDFRS0O5MLHd0HL2MAnQuBAgCAAYhhb7zEAwAAjMMVFABAl8aVic6JKygAAMA4BAoAADAOL/EAaDdcWkd31NHnfVf974wrKAAAwDhcQQG6ka76Ly0AXQ9XUAAAgHG4ggIA6FBcycNPwRUUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYByfBkpubq4GDx6sXr16KTExUYcOHfLldAAAgCF8FijvvvuusrKytGrVKn3xxRcaM2aMUlJSVFVV5aspAQAAQ/gsUF566SXNnz9fTzzxhOLi4rRhwwb16dNHf//73301JQAAYIgevnjShoYGlZSUKDs727PO399fycnJKi4uvmZ8fX296uvrPbdramokSW63u13m11x/+Ybbbvac3K/r3s8Xz9md73cznWUfbmffO8tcuV/nvt/t3vdWXH1My7JaHmz5wH/+8x9LkrV//36v9UuXLrXGjRt3zfhVq1ZZklhYWFhYWFi6wHL27NkWW8EnV1BaKzs7W1lZWZ7bzc3NunDhgvr37y8/P79rxrvdbkVFRens2bMKDg7u4Nl2DhyjlnGMbo7j0zKOUcs4RjfX1Y6PZVm6ePGiIiMjWxzrk0AZMGCAAgICVFlZ6bW+srJSDofjmvE2m002m81rXUhISIvPExwc3CX+QNsTx6hlHKOb4/i0jGPUMo7RzXWl42O323/SOJ+8STYwMFAJCQkqLCz0rGtublZhYaGcTqcvpgQAAAzis5d4srKyNHfuXN13330aN26cXnnlFdXW1uqJJ57w1ZQAAIAhfBYos2bN0nfffaeVK1fK5XLpnnvu0c6dOxUeHn7bj22z2bRq1aprXhbC/8cxahnH6OY4Pi3jGLWMY3Rz3fn4+Fk/6bM+AAAAHYffxQMAAIxDoAAAAOMQKAAAwDgECgAAME6XDJTc3FwNHjxYvXr1UmJiog4dOuTrKRlj9erV8vPz81piY2N9PS2f2bdvn6ZNm6bIyEj5+flp27ZtXtsty9LKlSsVERGh3r17Kzk5WadOnfLZfH2hpWP0+OOPX3NOTZkyxWfz7Wg5OTkaO3asgoKCFBYWphkzZqi0tNRrTF1dnTIyMtS/f3/169dP6enp13xRZVf2U47RxIkTrzmPFi5c6LM5d7T169crPj7e84VsTqdTH330kWd7dzyHulygvPvuu8rKytKqVav0xRdfaMyYMUpJSVFVVZWvp2aMu+++W+fOnfMsn332ma+n5DO1tbUaM2aMcnNzr7t93bp1eu2117RhwwYdPHhQffv2VUpKiurq6jp8rr7S0jGSpClTpnidU++8806HztGXioqKlJGRoQMHDmjXrl1qbGzU5MmTVVtb6xmzZMkS7dixQ1u2bFFRUZEqKio0c+ZMn867I/2UYyRJ8+fP9zqP1q1b57M5d7RBgwZp7dq1Kikp0eHDhzVp0iRNnz5dJ06ckLrrOdSWvwTQBOPGjbMyMjI8t5uamqzIyEgrJyfHp/MyxapVq6wxY8b4ehpGkmRt3brVc7u5udlyOBzWCy+84FlXXV1t2Ww265133vHRLH3rx8fIsixr7ty51vTp0302J9NUVVVZkqyioiLL+uGc6dmzp7VlyxbPmK+++sqSZBUXF/twpr7z42NkWZb1P//zP9Yf/vAHn87LNHfccYf1t7/9rdueQ13qCkpDQ4NKSkqUnJzsWefv76/k5GQVFxf7dG4mOXXqlCIjIzV06FDNmTNH5eXlvp6SkcrKyuRyubzOJ7vdrsTERM6nH9m7d6/CwsI0YsQILVq0SOfPn/f1lHympqZGkhQaGipJKikpUWNjo9d5FBsbq+jo6G57Hv34GF21adMmDRgwQKNGjVJ2drYuX77soxn6VlNTkzZv3qza2lo5nc5uew51it9m/FP997//VVNT0zXfRhseHq6vv/7aZ/MySWJiovLy8jRixAidO3dOzz//vB588EEdP35cQUFBvp6eUVwul/TD+fN/hYeHe7bhf1/emTlzpoYMGaIzZ87o2WefVWpqqoqLixUQEODr6XWo5uZmLV68WPfff79GjRol/XAeBQYGXvMLTrvreXS9YyRJjzzyiGJiYhQZGamjR49q2bJlKi0t1fvvv+/T+XakY8eOyel0qq6uTv369dPWrVsVFxenI0eOdMtzqEsFClqWmprq+Tk+Pl6JiYmKiYnRe++9p3nz5vl0buicZs+e7fl59OjRio+P17Bhw7R3714lJSX5dG4dLSMjQ8ePH+/W7+tqyY2O0YIFCzw/jx49WhEREUpKStKZM2c0bNgwH8y0440YMUJHjhxRTU2N/vGPf2ju3LkqKiry9bR8pku9xDNgwAAFBARc887myspKORwOn83LZCEhIbrrrrt0+vRpX0/FOFfPGc6n1hk6dKgGDBjQ7c6pzMxMFRQU6NNPP9WgQYM86x0OhxoaGlRdXe01vjueRzc6RteTmJgoSd3qPAoMDNTw4cOVkJCgnJwcjRkzRq+++mq3PYe6VKAEBgYqISFBhYWFnnXNzc0qLCyU0+n06dxMdenSJZ05c0YRERG+nopxhgwZIofD4XU+ud1uHTx4kPPpJr799ludP3++25xTlmUpMzNTW7du1Z49ezRkyBCv7QkJCerZs6fXeVRaWqry8vJucx61dIyu58iRI5LUbc6j62lublZ9fX33PYd8/S7dtrZ582bLZrNZeXl51smTJ60FCxZYISEhlsvl8vXUjPDUU09Ze/futcrKyqx//etfVnJysjVgwACrqqrK11PziYsXL1pffvml9eWXX1qSrJdeesn68ssvrX//+9+WZVnW2rVrrZCQEGv79u3W0aNHrenTp1tDhgyxvv/+e19PvcPc7BhdvHjRevrpp63i4mKrrKzM2r17t/Xzn//cuvPOO626ujpfT71DLFq0yLLb7dbevXutc+fOeZbLly97xixcuNCKjo629uzZYx0+fNhyOp2W0+n06bw7UkvH6PTp09aaNWusw4cPW2VlZdb27dutoUOHWhMmTPD11DvM8uXLraKiIqusrMw6evSotXz5csvPz8/65JNPLKubnkNdLlAsy7Jef/11Kzo62goMDLTGjRtnHThwwNdTMsasWbOsiIgIKzAw0PrZz35mzZo1yzp9+rSvp+Uzn376qSXpmmXu3LmW9cNHjZ977jkrPDzcstlsVlJSklVaWurraXeomx2jy5cvW5MnT7YGDhxo9ezZ04qJibHmz5/frf5BcL1jI8nauHGjZ8z3339v/f73v7fuuOMOq0+fPtZvfvMb69y5cz6dd0dq6RiVl5dbEyZMsEJDQy2bzWYNHz7cWrp0qVVTU+PrqXeY3/3ud1ZMTIwVGBhoDRw40EpKSvLEidVNzyE/639PHgAAAGN0qfegAACAroFAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYJz/B1g6lyYuVsaFAAAAAElFTkSuQmCC", + "text/plain": [ + "<Figure size 640x480 with 1 Axes>" ] }, "metadata": {}, @@ -284,37 +243,48 @@ } ], "source": [ - "with plt.ioff():\n", - " plt.hist(data['labels'],bins=64)\n", - " plt.show()" + "plt.hist(mkbatch(2048)[1].cpu(), bins=64)" ] }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 42, "execution_state": "idle", "metadata": {}, "outputs": [ { "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "24b2976d050e43af8bad0e4080a224eb", - "version_major": 2, - "version_minor": 0 - }, - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoAAAAHgCAYAAAA10dzkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAjcElEQVR4nO3dfZTWdZ3/8dcAcRMxg1DMOCsgtW6IN2Xi0mg3W7KSsZ48cSp2J5eUI3vaoUTSgkrKvAHZXTOKID0ucE66VrurFRVKWLIlIlK03oW2WVDuwO4hZoSOgzLX749fXqcx/W2/1rkuxs/jcc51Ttf3+7nm+/4QytPvXHPRUKlUKgEAoBiD6j0AAAC1JQABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAozpN4DDGS9vb15/PHHM2rUqDQ0NNR7HADg91CpVPLEE0+ktbU1gwaVeS9MAP4vPP744xk/fny9xwAA/gC7d+/OMcccU+8x6kIA/i+MGjUq+c1voMbGxnqPAwD8Hrq7uzN+/Pjqn+MlEoD/C89827exsVEAAsAAU/Lbt8r8xjcAQMEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYYbUewCe37GLvvG85362bGZNZwEAXjzcAQQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAozBEZgJs3b84555yT1tbWNDQ05LbbbutzvlKpZMmSJTn66KMzYsSITJ8+PY8++mifNfv27Ut7e3saGxszevTozJ07NwcOHOiz5t///d/zxje+McOHD8/48eOzfPnymuwPAKCejsgAPHjwYF7zmtdk5cqVz3l++fLlWbFiRVavXp2tW7dm5MiRmTFjRp588snqmvb29jz44IPZuHFj1q9fn82bN2fevHnV893d3TnrrLMyceLEbN++PX/3d3+XT37yk7n++utrskcAgHppqFQqlXoP8f/S0NCQW2+9Neeee27ym7t/ra2t+dCHPpRLLrkkSdLV1ZXm5uasXbs2s2fPzsMPP5wpU6Zk27ZtmTp1apJkw4YNefvb355f/OIXaW1tzapVq/Kxj30snZ2dGTp0aJJk0aJFue222/LjH//495qtu7s7TU1N6erqSmNj4wu+92MXfeN5z/1s2cwX/HoAUIL+/vN7IDgi7wD+vzz22GPp7OzM9OnTq8eampoybdq0bNmyJUmyZcuWjB49uhp/STJ9+vQMGjQoW7dura5505veVI2/JJkxY0Z27tyZX/3qV8957Z6ennR3d/d5AAAMNAMuADs7O5Mkzc3NfY43NzdXz3V2dmbcuHF9zg8ZMiRjxozps+a5vsZvX+PZli5dmqampupj/PjxL+DOAABqY8AFYD0tXrw4XV1d1cfu3bvrPRIAwP+3AReALS0tSZI9e/b0Ob5nz57quZaWluzdu7fP+aeffjr79u3rs+a5vsZvX+PZhg0blsbGxj4PAICBZsAF4KRJk9LS0pJNmzZVj3V3d2fr1q1pa2tLkrS1tWX//v3Zvn17dc2dd96Z3t7eTJs2rbpm8+bNeeqpp6prNm7cmFe/+tU56qijaronAIBaOiID8MCBA9mxY0d27NiR/OYHP3bs2JFdu3aloaEhCxYsyJVXXpmvfe1ruf/++/PXf/3XaW1trf6k8PHHH5+3ve1tufDCC3Pvvffm+9//fubPn5/Zs2entbU1SfJXf/VXGTp0aObOnZsHH3wwX/rSl/KZz3wmCxcurOveAQD625B6D/Bc7rvvvrzlLW+pPn8myubMmZO1a9fmwx/+cA4ePJh58+Zl//79ecMb3pANGzZk+PDh1dfcdNNNmT9/fs4888wMGjQos2bNyooVK6rnm5qacscdd6SjoyOnnnpqXv7yl2fJkiV9PisQAODF6Ij/HMAjmc8BBICBx+cAHqHfAgYAoP8IQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCDMgAPHz4cC677LJMmjQpI0aMyKte9apcccUVqVQq1TWVSiVLlizJ0UcfnREjRmT69Ol59NFH+3ydffv2pb29PY2NjRk9enTmzp2bAwcO1GFHAAC1MyAD8JprrsmqVavyuc99Lg8//HCuueaaLF++PJ/97Gera5YvX54VK1Zk9erV2bp1a0aOHJkZM2bkySefrK5pb2/Pgw8+mI0bN2b9+vXZvHlz5s2bV6ddAQDURkPlt2+bDRB/8Rd/kebm5tx4443VY7NmzcqIESPyxS9+MZVKJa2trfnQhz6USy65JEnS1dWV5ubmrF27NrNnz87DDz+cKVOmZNu2bZk6dWqSZMOGDXn729+eX/ziF2ltbf0f5+ju7k5TU1O6urrS2Nj4gu/z2EXfeN5zP1s28wW/HgCUoL///B4IBuQdwNNPPz2bNm3KI488kiT50Y9+lO9973s5++yzkySPPfZYOjs7M3369OprmpqaMm3atGzZsiVJsmXLlowePboaf0kyffr0DBo0KFu3bq35ngAAamVIvQf4QyxatCjd3d2ZPHlyBg8enMOHD+eqq65Ke3t7kqSzszNJ0tzc3Od1zc3N1XOdnZ0ZN25cn/NDhgzJmDFjqmueraenJz09PdXn3d3dL/jeAAD624C8A/jlL385N910U26++eb84Ac/yLp16/L3f//3WbduXb9ed+nSpWlqaqo+xo8f36/XAwDoDwMyAC+99NIsWrQos2fPzkknnZTzzjsvF198cZYuXZokaWlpSZLs2bOnz+v27NlTPdfS0pK9e/f2Of/0009n37591TXPtnjx4nR1dVUfu3fv7qcdAgD0nwEZgL/+9a8zaFDf0QcPHpze3t4kyaRJk9LS0pJNmzZVz3d3d2fr1q1pa2tLkrS1tWX//v3Zvn17dc2dd96Z3t7eTJs27TmvO2zYsDQ2NvZ5AAAMNAPyPYDnnHNOrrrqqkyYMCEnnHBCfvjDH+baa6/NBRdckCRpaGjIggULcuWVV+a4447LpEmTctlll6W1tTXnnntukuT444/P2972tlx44YVZvXp1nnrqqcyfPz+zZ8/+vX4CGABgoBqQAfjZz342l112Wf72b/82e/fuTWtra/7mb/4mS5Ysqa758Ic/nIMHD2bevHnZv39/3vCGN2TDhg0ZPnx4dc1NN92U+fPn58wzz8ygQYMya9asrFixok67AgCojQH5OYBHCp8DCAADj88BHKDvAQQA4A8nAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAozYAPwl7/8Zd773vdm7NixGTFiRE466aTcd9991fOVSiVLlizJ0UcfnREjRmT69Ol59NFH+3yNffv2pb29PY2NjRk9enTmzp2bAwcO1GE3AAC1MyAD8Fe/+lXOOOOMvOQlL8m3vvWtPPTQQ/mHf/iHHHXUUdU1y5cvz4oVK7J69eps3bo1I0eOzIwZM/Lkk09W17S3t+fBBx/Mxo0bs379+mzevDnz5s2r064AAGqjoVKpVOo9xP+vRYsW5fvf/37+7d/+7TnPVyqVtLa25kMf+lAuueSSJElXV1eam5uzdu3azJ49Ow8//HCmTJmSbdu2ZerUqUmSDRs25O1vf3t+8YtfpLW19X+co7u7O01NTenq6kpjY+MLvMvk2EXfeN5zP1s28wW/HgCUoL///B4IBuQdwK997WuZOnVq3vWud2XcuHE55ZRTcsMNN1TPP/bYY+ns7Mz06dOrx5qamjJt2rRs2bIlSbJly5aMHj26Gn9JMn369AwaNChbt259zuv29PSku7u7zwMAYKAZkAH405/+NKtWrcpxxx2X22+/Pe9///vzwQ9+MOvWrUuSdHZ2Jkmam5v7vK65ubl6rrOzM+PGjetzfsiQIRkzZkx1zbMtXbo0TU1N1cf48eP7aYcAAP1nQAZgb29vXve61+Xqq6/OKaecknnz5uXCCy/M6tWr+/W6ixcvTldXV/Wxe/fufr0eAEB/GJABePTRR2fKlCl9jh1//PHZtWtXkqSlpSVJsmfPnj5r9uzZUz3X0tKSvXv39jn/9NNPZ9++fdU1zzZs2LA0Njb2eQAADDQDMgDPOOOM7Ny5s8+xRx55JBMnTkySTJo0KS0tLdm0aVP1fHd3d7Zu3Zq2trYkSVtbW/bv35/t27dX19x5553p7e3NtGnTarYXAIBaG1LvAf4QF198cU4//fRcffXVefe735177703119/fa6//vokSUNDQxYsWJArr7wyxx13XCZNmpTLLrssra2tOffcc5Pf3DF829veVv3W8VNPPZX58+dn9uzZv9dPAAMADFQDMgBPO+203HrrrVm8eHE+9alPZdKkSbnuuuvS3t5eXfPhD384Bw8ezLx587J///684Q1vyIYNGzJ8+PDqmptuuinz58/PmWeemUGDBmXWrFlZsWJFnXYFAFAbA/JzAI8UPgcQAAYenwM4QN8DCADAH04AAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRmSL0HgOdy7KJvPO+5ny2bWdNZAODFxh1AAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMK8KAJw2bJlaWhoyIIFC6rHnnzyyXR0dGTs2LF52ctellmzZmXPnj19Xrdr167MnDkzL33pSzNu3Lhceumlefrpp+uwAwCA2hnwAbht27Z84QtfyMknn9zn+MUXX5yvf/3r+cpXvpK77rorjz/+eN75zndWzx8+fDgzZ87MoUOHcvfdd2fdunVZu3ZtlixZUoddAADUzoAOwAMHDqS9vT033HBDjjrqqOrxrq6u3Hjjjbn22mvz1re+NaeeemrWrFmTu+++O/fcc0+S5I477shDDz2UL37xi3nta1+bs88+O1dccUVWrlyZQ4cO1XFXAAD9a0AHYEdHR2bOnJnp06f3Ob59+/Y89dRTfY5Pnjw5EyZMyJYtW5IkW7ZsyUknnZTm5ubqmhkzZqS7uzsPPvjgc16vp6cn3d3dfR4AAAPNkHoP8Ie65ZZb8oMf/CDbtm37nXOdnZ0ZOnRoRo8e3ed4c3NzOjs7q2t+O/6eOf/MueeydOnSXH755S/gLgAAam9A3gHcvXt3Lrrootx0000ZPnx4za67ePHidHV1VR+7d++u2bUBAF4oAzIAt2/fnr179+Z1r3tdhgwZkiFDhuSuu+7KihUrMmTIkDQ3N+fQoUPZv39/n9ft2bMnLS0tSZKWlpbf+angZ54/s+bZhg0blsbGxj4PAICBZkAG4Jlnnpn7778/O3bsqD6mTp2a9vb26v9+yUtekk2bNlVfs3PnzuzatSttbW1Jkra2ttx///3Zu3dvdc3GjRvT2NiYKVOm1GVfAAC1MCDfAzhq1KiceOKJfY6NHDkyY8eOrR6fO3duFi5cmDFjxqSxsTEf+MAH0tbWlte//vVJkrPOOitTpkzJeeedl+XLl6ezszMf//jH09HRkWHDhtVlXwAAtTAgA/D38elPfzqDBg3KrFmz0tPTkxkzZuTzn/989fzgwYOzfv36vP/9709bW1tGjhyZOXPm5FOf+lRd5wYA6G8vmgD87ne/2+f58OHDs3LlyqxcufJ5XzNx4sR885vfrMF0AABHjgH5HkAAAP5wAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwQ+o9ANDXsYu+8bznfrZsZk1nAeDFyR1AAIDCDMgAXLp0aU477bSMGjUq48aNy7nnnpudO3f2WfPkk0+mo6MjY8eOzcte9rLMmjUre/bs6bNm165dmTlzZl760pdm3LhxufTSS/P000/XeDcAALU1IAPwrrvuSkdHR+65555s3LgxTz31VM4666wcPHiwuubiiy/O17/+9XzlK1/JXXfdlccffzzvfOc7q+cPHz6cmTNn5tChQ7n77ruzbt26rF27NkuWLKnTrgAAamNAvgdww4YNfZ6vXbs248aNy/bt2/OmN70pXV1dufHGG3PzzTfnrW99a5JkzZo1Of7443PPPffk9a9/fe6444489NBD+fa3v53m5ua89rWvzRVXXJGPfOQj+eQnP5mhQ4fWaXcAAP1rQN4BfLaurq4kyZgxY5Ik27dvz1NPPZXp06dX10yePDkTJkzIli1bkiRbtmzJSSedlObm5uqaGTNmpLu7Ow8++GDN9wAAUCsD8g7gb+vt7c2CBQtyxhln5MQTT0ySdHZ2ZujQoRk9enSftc3Nzens7Kyu+e34e+b8M+eeS09PT3p6eqrPu7u7X/D9AAD0twF/B7CjoyMPPPBAbrnlln6/1tKlS9PU1FR9jB8/vt+vCQDwQhvQATh//vysX78+3/nOd3LMMcdUj7e0tOTQoUPZv39/n/V79uxJS0tLdc2zfyr4mefPrHm2xYsXp6urq/rYvXt3P+wKAKB/DcgArFQqmT9/fm699dbceeedmTRpUp/zp556al7ykpdk06ZN1WM7d+7Mrl270tbWliRpa2vL/fffn71791bXbNy4MY2NjZkyZcpzXnfYsGFpbGzs8wAAGGgG5HsAOzo6cvPNN+erX/1qRo0aVX3PXlNTU0aMGJGmpqbMnTs3CxcuzJgxY9LY2JgPfOADaWtry+tf//okyVlnnZUpU6bkvPPOy/Lly9PZ2ZmPf/zj6ejoyLBhw+q8QwCA/jMgA3DVqlVJkj/7sz/rc3zNmjV53/velyT59Kc/nUGDBmXWrFnp6enJjBkz8vnPf766dvDgwVm/fn3e//73p62tLSNHjsycOXPyqU99qsa7AQCorQEZgJVK5X9cM3z48KxcuTIrV6583jUTJ07MN7/5zRd4OgCAI9uAfA8gAAB/OAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUJgh9R4A4Lcdu+gbz3vuZ8tm1nQWgBcrdwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKM6TeAwDwfx276BvPe+5ny2bWdBbgxc0dQACAwghAAIDCFB+AK1euzLHHHpvhw4dn2rRpuffee+s9EgBAvyr6PYBf+tKXsnDhwqxevTrTpk3LddddlxkzZmTnzp0ZN25cvccDKIr3QELtFH0H8Nprr82FF16Y888/P1OmTMnq1avz0pe+NP/4j/9Y79EAAPpNsXcADx06lO3bt2fx4sXVY4MGDcr06dOzZcuW53xNT09Penp6qs+7urqSJN3d3f0yY2/Pr5/3XH9d80hh78/txb73FL7/kvce+y/WiZ+4/XnPPXD5jH655jO/nyqVSr98/YGg2AD87//+7xw+fDjNzc19jjc3N+fHP/7xc75m6dKlufzyy3/n+Pjx4/ttzufTdF3NL3nEsPdylbz/kvce+y9Wf////sQTT6Spqal/L3KEKjYA/xCLFy/OwoULq897e3uzb9++jB07Ng0NDS/otbq7uzN+/Pjs3r07jY2NL+jXHgjsv+z9x69B8fuPXwP778f9VyqVPPHEE2ltbX1Bv+5AUmwAvvzlL8/gwYOzZ8+ePsf37NmTlpaW53zNsGHDMmzYsD7HRo8e3a9zNjY2FvkP/jPsv+z9x69B8fuPXwP776f9l3rn7xnF/hDI0KFDc+qpp2bTpk3VY729vdm0aVPa2trqOhsAQH8q9g5gkixcuDBz5szJ1KlT86d/+qe57rrrcvDgwZx//vn1Hg0AoN8UHYDvec978l//9V9ZsmRJOjs789rXvjYbNmz4nR8MqYdhw4blE5/4xO98y7kU9l/2/uPXoPj9x6+B/Re+//7WUCn5Z6ABAApU7HsAAQBKJQABAAojAAEACiMAAQAKIwCPMJs3b84555yT1tbWNDQ05Lbbbqv3SDW1dOnSnHbaaRk1alTGjRuXc889Nzt37qz3WDWzatWqnHzyydUPPm1ra8u3vvWteo9VN8uWLUtDQ0MWLFhQ71Fq5pOf/GQaGhr6PCZPnlzvsWrql7/8Zd773vdm7NixGTFiRE466aTcd9999R6rZo499tjf+T3Q0NCQjo6Oeo9WE4cPH85ll12WSZMmZcSIEXnVq16VK664oui/t7c/FP0xMEeigwcP5jWveU0uuOCCvPOd76z3ODV31113paOjI6eddlqefvrpfPSjH81ZZ52Vhx56KCNHjqz3eP3umGOOybJly3LcccelUqlk3bp1ecc73pEf/vCHOeGEE+o9Xk1t27YtX/jCF3LyySfXe5SaO+GEE/Ltb3+7+nzIkHL+Vf2rX/0qZ5xxRt7ylrfkW9/6Vl7xilfk0UcfzVFHHVXv0Wpm27ZtOXz4cPX5Aw88kD//8z/Pu971rrrOVSvXXHNNVq1alXXr1uWEE07Ifffdl/PPPz9NTU354Ac/WO/xXjTK+bfKAHH22Wfn7LPPrvcYdbNhw4Y+z9euXZtx48Zl+/btedOb3lS3uWrlnHPO6fP8qquuyqpVq3LPPfcUFYAHDhxIe3t7brjhhlx55ZX1HqfmhgwZ8rx/JeWL3TXXXJPx48dnzZo11WOTJk2q60y19opXvKLP82XLluVVr3pV3vzmN9dtplq6++678453vCMzZ85MfnNH9J/+6Z9y77331nu0FxXfAuaI1tXVlSQZM2ZMvUepucOHD+eWW27JwYMHi/vrCTs6OjJz5sxMnz693qPUxaOPPprW1ta88pWvTHt7e3bt2lXvkWrma1/7WqZOnZp3vetdGTduXE455ZTccMMN9R6rbg4dOpQvfvGLueCCC9LQ0FDvcWri9NNPz6ZNm/LII48kSX70ox/le9/7XtE3R/qDO4AcsXp7e7NgwYKcccYZOfHEE+s9Ts3cf//9aWtry5NPPpmXvexlufXWWzNlypR6j1Uzt9xyS37wgx9k27Zt9R6lLqZNm5a1a9fm1a9+df7zP/8zl19+ed74xjfmgQceyKhRo+o9Xr/76U9/mlWrVmXhwoX56Ec/mm3btuWDH/xghg4dmjlz5tR7vJq77bbbsn///rzvfe+r9yg1s2jRonR3d2fy5MkZPHhwDh8+nKuuuirt7e31Hu1FRQByxOro6MgDDzyQ733ve/UepaZe/epXZ8eOHenq6so///M/Z86cObnrrruKiMDdu3fnoosuysaNGzN8+PB6j1MXv32X4+STT860adMyceLEfPnLX87cuXPrOlst9Pb2ZurUqbn66quTJKecckoeeOCBrF69usgAvPHGG3P22WentbW13qPUzJe//OXcdNNNufnmm3PCCSdkx44dWbBgQVpbW4v8PdBfBCBHpPnz52f9+vXZvHlzjjnmmHqPU1NDhw7NH//xHydJTj311Gzbti2f+cxn8oUvfKHeo/W77du3Z+/evXnd615XPXb48OFs3rw5n/vc59LT05PBgwfXdcZaGz16dP7kT/4kP/nJT+o9Sk0cffTRv/MfO8cff3z+5V/+pW4z1cvPf/7zfPvb386//uu/1nuUmrr00kuzaNGizJ49O0ly0kkn5ec//3mWLl0qAF9AApAjSqVSyQc+8IHceuut+e53v1vcm7+fS29vb3p6euo9Rk2ceeaZuf/++/scO//88zN58uR85CMfKS7+8psfiPmP//iPnHfeefUepSbOOOOM3/nop0ceeSQTJ06s20z1smbNmowbN676wxCl+PWvf51Bg/r+iMLgwYPT29tbt5lejATgEebAgQN9/kv/sccey44dOzJmzJhMmDChrrPVQkdHR26++eZ89atfzahRo9LZ2ZkkaWpqyogRI+o9Xr9bvHhxzj777EyYMCFPPPFEbr755nz3u9/N7bffXu/RamLUqFG/837PkSNHZuzYscW8D/SSSy7JOeeck4kTJ+bxxx/PJz7xiQwePDh/+Zd/We/RauLiiy/O6aefnquvvjrvfve7c++99+b666/P9ddfX+/Raqq3tzdr1qzJnDlzivoYoPzm0xCuuuqqTJgwISeccEJ++MMf5tprr80FF1xQ79FeXCocUb7zne9UkvzOY86cOfUerSaea+9JKmvWrKn3aDVxwQUXVCZOnFgZOnRo5RWveEXlzDPPrNxxxx31Hquu3vzmN1cuuuiieo9RM+95z3sqRx99dGXo0KGVP/qjP6q85z3vqfzkJz+p91g19fWvf71y4oknVoYNG1aZPHly5frrr6/3SDV3++23V5JUdu7cWe9Raq67u7ty0UUXVSZMmFAZPnx45ZWvfGXlYx/7WKWnp6feo72oNFR8tDYAQFF8DiAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGH+D0qdXUuXgornAAAAAElFTkSuQmCC", - "text/html": [ - "\n", - " <div style=\"display: inline-block;\">\n", - " <div class=\"jupyter-widgets widget-label\" style=\"text-align: center;\">\n", - " Figure\n", - " </div>\n", - " <img src='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAoAAAAHgCAYAAAA10dzkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAjcElEQVR4nO3dfZTWdZ3/8dcAcRMxg1DMOCsgtW6IN2Xi0mg3W7KSsZ48cSp2J5eUI3vaoUTSgkrKvAHZXTOKID0ucE66VrurFRVKWLIlIlK03oW2WVDuwO4hZoSOgzLX749fXqcx/W2/1rkuxs/jcc51Ttf3+7nm+/4QytPvXHPRUKlUKgEAoBiD6j0AAAC1JQABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAozpN4DDGS9vb15/PHHM2rUqDQ0NNR7HADg91CpVPLEE0+ktbU1gwaVeS9MAP4vPP744xk/fny9xwAA/gC7d+/OMcccU+8x6kIA/i+MGjUq+c1voMbGxnqPAwD8Hrq7uzN+/Pjqn+MlEoD/C89827exsVEAAsAAU/Lbt8r8xjcAQMEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYYbUewCe37GLvvG85362bGZNZwEAXjzcAQQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAozBEZgJs3b84555yT1tbWNDQ05LbbbutzvlKpZMmSJTn66KMzYsSITJ8+PY8++mifNfv27Ut7e3saGxszevTozJ07NwcOHOiz5t///d/zxje+McOHD8/48eOzfPnymuwPAKCejsgAPHjwYF7zmtdk5cqVz3l++fLlWbFiRVavXp2tW7dm5MiRmTFjRp588snqmvb29jz44IPZuHFj1q9fn82bN2fevHnV893d3TnrrLMyceLEbN++PX/3d3+XT37yk7n++utrskcAgHppqFQqlXoP8f/S0NCQW2+9Neeee27ym7t/ra2t+dCHPpRLLrkkSdLV1ZXm5uasXbs2s2fPzsMPP5wpU6Zk27ZtmTp1apJkw4YNefvb355f/OIXaW1tzapVq/Kxj30snZ2dGTp0aJJk0aJFue222/LjH//495qtu7s7TU1N6erqSmNj4wu+92MXfeN5z/1s2cwX/HoAUIL+/vN7IDgi7wD+vzz22GPp7OzM9OnTq8eampoybdq0bNmyJUmyZcuWjB49uhp/STJ9+vQMGjQoW7dura5505veVI2/JJkxY0Z27tyZX/3qV8957Z6ennR3d/d5AAAMNAMuADs7O5Mkzc3NfY43NzdXz3V2dmbcuHF9zg8ZMiRjxozps+a5vsZvX+PZli5dmqampupj/PjxL+DOAABqY8AFYD0tXrw4XV1d1cfu3bvrPRIAwP+3AReALS0tSZI9e/b0Ob5nz57quZaWluzdu7fP+aeffjr79u3rs+a5vsZvX+PZhg0blsbGxj4PAICBZsAF4KRJk9LS0pJNmzZVj3V3d2fr1q1pa2tLkrS1tWX//v3Zvn17dc2dd96Z3t7eTJs2rbpm8+bNeeqpp6prNm7cmFe/+tU56qijaronAIBaOiID8MCBA9mxY0d27NiR/OYHP3bs2JFdu3aloaEhCxYsyJVXXpmvfe1ruf/++/PXf/3XaW1trf6k8PHHH5+3ve1tufDCC3Pvvffm+9//fubPn5/Zs2entbU1SfJXf/VXGTp0aObOnZsHH3wwX/rSl/KZz3wmCxcurOveAQD625B6D/Bc7rvvvrzlLW+pPn8myubMmZO1a9fmwx/+cA4ePJh58+Zl//79ecMb3pANGzZk+PDh1dfcdNNNmT9/fs4888wMGjQos2bNyooVK6rnm5qacscdd6SjoyOnnnpqXv7yl2fJkiV9PisQAODF6Ij/HMAjmc8BBICBx+cAHqHfAgYAoP8IQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCDMgAPHz4cC677LJMmjQpI0aMyKte9apcccUVqVQq1TWVSiVLlizJ0UcfnREjRmT69Ol59NFH+3ydffv2pb29PY2NjRk9enTmzp2bAwcO1GFHAAC1MyAD8JprrsmqVavyuc99Lg8//HCuueaaLF++PJ/97Gera5YvX54VK1Zk9erV2bp1a0aOHJkZM2bkySefrK5pb2/Pgw8+mI0bN2b9+vXZvHlz5s2bV6ddAQDURkPlt2+bDRB/8Rd/kebm5tx4443VY7NmzcqIESPyxS9+MZVKJa2trfnQhz6USy65JEnS1dWV5ubmrF27NrNnz87DDz+cKVOmZNu2bZk6dWqSZMOGDXn729+eX/ziF2ltbf0f5+ju7k5TU1O6urrS2Nj4gu/z2EXfeN5zP1s28wW/HgCUoL///B4IBuQdwNNPPz2bNm3KI488kiT50Y9+lO9973s5++yzkySPPfZYOjs7M3369OprmpqaMm3atGzZsiVJsmXLlowePboaf0kyffr0DBo0KFu3bq35ngAAamVIvQf4QyxatCjd3d2ZPHlyBg8enMOHD+eqq65Ke3t7kqSzszNJ0tzc3Od1zc3N1XOdnZ0ZN25cn/NDhgzJmDFjqmueraenJz09PdXn3d3dL/jeAAD624C8A/jlL385N910U26++eb84Ac/yLp16/L3f//3WbduXb9ed+nSpWlqaqo+xo8f36/XAwDoDwMyAC+99NIsWrQos2fPzkknnZTzzjsvF198cZYuXZokaWlpSZLs2bOnz+v27NlTPdfS0pK9e/f2Of/0009n37591TXPtnjx4nR1dVUfu3fv7qcdAgD0nwEZgL/+9a8zaFDf0QcPHpze3t4kyaRJk9LS0pJNmzZVz3d3d2fr1q1pa2tLkrS1tWX//v3Zvn17dc2dd96Z3t7eTJs27TmvO2zYsDQ2NvZ5AAAMNAPyPYDnnHNOrrrqqkyYMCEnnHBCfvjDH+baa6/NBRdckCRpaGjIggULcuWVV+a4447LpEmTctlll6W1tTXnnntukuT444/P2972tlx44YVZvXp1nnrqqcyfPz+zZ8/+vX4CGABgoBqQAfjZz342l112Wf72b/82e/fuTWtra/7mb/4mS5Ysqa758Ic/nIMHD2bevHnZv39/3vCGN2TDhg0ZPnx4dc1NN92U+fPn58wzz8ygQYMya9asrFixok67AgCojQH5OYBHCp8DCAADj88BHKDvAQQA4A8nAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAozYAPwl7/8Zd773vdm7NixGTFiRE466aTcd9991fOVSiVLlizJ0UcfnREjRmT69Ol59NFH+3yNffv2pb29PY2NjRk9enTmzp2bAwcO1GE3AAC1MyAD8Fe/+lXOOOOMvOQlL8m3vvWtPPTQQ/mHf/iHHHXUUdU1y5cvz4oVK7J69eps3bo1I0eOzIwZM/Lkk09W17S3t+fBBx/Mxo0bs379+mzevDnz5s2r064AAGqjoVKpVOo9xP+vRYsW5fvf/37+7d/+7TnPVyqVtLa25kMf+lAuueSSJElXV1eam5uzdu3azJ49Ow8//HCmTJmSbdu2ZerUqUmSDRs25O1vf3t+8YtfpLW19X+co7u7O01NTenq6kpjY+MLvMvk2EXfeN5zP1s28wW/HgCUoL///B4IBuQdwK997WuZOnVq3vWud2XcuHE55ZRTcsMNN1TPP/bYY+ns7Mz06dOrx5qamjJt2rRs2bIlSbJly5aMHj26Gn9JMn369AwaNChbt259zuv29PSku7u7zwMAYKAZkAH405/+NKtWrcpxxx2X22+/Pe9///vzwQ9+MOvWrUuSdHZ2Jkmam5v7vK65ubl6rrOzM+PGjetzfsiQIRkzZkx1zbMtXbo0TU1N1cf48eP7aYcAAP1nQAZgb29vXve61+Xqq6/OKaecknnz5uXCCy/M6tWr+/W6ixcvTldXV/Wxe/fufr0eAEB/GJABePTRR2fKlCl9jh1//PHZtWtXkqSlpSVJsmfPnj5r9uzZUz3X0tKSvXv39jn/9NNPZ9++fdU1zzZs2LA0Njb2eQAADDQDMgDPOOOM7Ny5s8+xRx55JBMnTkySTJo0KS0tLdm0aVP1fHd3d7Zu3Zq2trYkSVtbW/bv35/t27dX19x5553p7e3NtGnTarYXAIBaG1LvAf4QF198cU4//fRcffXVefe735177703119/fa6//vokSUNDQxYsWJArr7wyxx13XCZNmpTLLrssra2tOffcc5Pf3DF829veVv3W8VNPPZX58+dn9uzZv9dPAAMADFQDMgBPO+203HrrrVm8eHE+9alPZdKkSbnuuuvS3t5eXfPhD384Bw8ezLx587J///684Q1vyIYNGzJ8+PDqmptuuinz58/PmWeemUGDBmXWrFlZsWJFnXYFAFAbA/JzAI8UPgcQAAYenwM4QN8DCADAH04AAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRGAAIAFEYAAgAURgACABRmSL0HgOdy7KJvPO+5ny2bWdNZAODFxh1AAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMIIQACAwghAAIDCCEAAgMK8KAJw2bJlaWhoyIIFC6rHnnzyyXR0dGTs2LF52ctellmzZmXPnj19Xrdr167MnDkzL33pSzNu3Lhceumlefrpp+uwAwCA2hnwAbht27Z84QtfyMknn9zn+MUXX5yvf/3r+cpXvpK77rorjz/+eN75zndWzx8+fDgzZ87MoUOHcvfdd2fdunVZu3ZtlixZUoddAADUzoAOwAMHDqS9vT033HBDjjrqqOrxrq6u3Hjjjbn22mvz1re+NaeeemrWrFmTu+++O/fcc0+S5I477shDDz2UL37xi3nta1+bs88+O1dccUVWrlyZQ4cO1XFXAAD9a0AHYEdHR2bOnJnp06f3Ob59+/Y89dRTfY5Pnjw5EyZMyJYtW5IkW7ZsyUknnZTm5ubqmhkzZqS7uzsPPvjgc16vp6cn3d3dfR4AAAPNkHoP8Ie65ZZb8oMf/CDbtm37nXOdnZ0ZOnRoRo8e3ed4c3NzOjs7q2t+O/6eOf/MueeydOnSXH755S/gLgAAam9A3gHcvXt3Lrrootx0000ZPnx4za67ePHidHV1VR+7d++u2bUBAF4oAzIAt2/fnr179+Z1r3tdhgwZkiFDhuSuu+7KihUrMmTIkDQ3N+fQoUPZv39/n9ft2bMnLS0tSZKWlpbf+angZ54/s+bZhg0blsbGxj4PAICBZkAG4Jlnnpn7778/O3bsqD6mTp2a9vb26v9+yUtekk2bNlVfs3PnzuzatSttbW1Jkra2ttx///3Zu3dvdc3GjRvT2NiYKVOm1GVfAAC1MCDfAzhq1KiceOKJfY6NHDkyY8eOrR6fO3duFi5cmDFjxqSxsTEf+MAH0tbWlte//vVJkrPOOitTpkzJeeedl+XLl6ezszMf//jH09HRkWHDhtVlXwAAtTAgA/D38elPfzqDBg3KrFmz0tPTkxkzZuTzn/989fzgwYOzfv36vP/9709bW1tGjhyZOXPm5FOf+lRd5wYA6G8vmgD87ne/2+f58OHDs3LlyqxcufJ5XzNx4sR885vfrMF0AABHjgH5HkAAAP5wAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwAhAAoDACEACgMAIQAKAwQ+o9ANDXsYu+8bznfrZsZk1nAeDFyR1AAIDCDMgAXLp0aU477bSMGjUq48aNy7nnnpudO3f2WfPkk0+mo6MjY8eOzcte9rLMmjUre/bs6bNm165dmTlzZl760pdm3LhxufTSS/P000/XeDcAALU1IAPwrrvuSkdHR+65555s3LgxTz31VM4666wcPHiwuubiiy/O17/+9XzlK1/JXXfdlccffzzvfOc7q+cPHz6cmTNn5tChQ7n77ruzbt26rF27NkuWLKnTrgAAamNAvgdww4YNfZ6vXbs248aNy/bt2/OmN70pXV1dufHGG3PzzTfnrW99a5JkzZo1Of7443PPPffk9a9/fe6444489NBD+fa3v53m5ua89rWvzRVXXJGPfOQj+eQnP5mhQ4fWaXcAAP1rQN4BfLaurq4kyZgxY5Ik27dvz1NPPZXp06dX10yePDkTJkzIli1bkiRbtmzJSSedlObm5uqaGTNmpLu7Ow8++GDN9wAAUCsD8g7gb+vt7c2CBQtyxhln5MQTT0ySdHZ2ZujQoRk9enSftc3Nzens7Kyu+e34e+b8M+eeS09PT3p6eqrPu7u7X/D9AAD0twF/B7CjoyMPPPBAbrnlln6/1tKlS9PU1FR9jB8/vt+vCQDwQhvQATh//vysX78+3/nOd3LMMcdUj7e0tOTQoUPZv39/n/V79uxJS0tLdc2zfyr4mefPrHm2xYsXp6urq/rYvXt3P+wKAKB/DcgArFQqmT9/fm699dbceeedmTRpUp/zp556al7ykpdk06ZN1WM7d+7Mrl270tbWliRpa2vL/fffn71791bXbNy4MY2NjZkyZcpzXnfYsGFpbGzs8wAAGGgG5HsAOzo6cvPNN+erX/1qRo0aVX3PXlNTU0aMGJGmpqbMnTs3CxcuzJgxY9LY2JgPfOADaWtry+tf//okyVlnnZUpU6bkvPPOy/Lly9PZ2ZmPf/zj6ejoyLBhw+q8QwCA/jMgA3DVqlVJkj/7sz/rc3zNmjV53/velyT59Kc/nUGDBmXWrFnp6enJjBkz8vnPf766dvDgwVm/fn3e//73p62tLSNHjsycOXPyqU99qsa7AQCorQEZgJVK5X9cM3z48KxcuTIrV6583jUTJ07MN7/5zRd4OgCAI9uAfA8gAAB/OAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUBgBCABQGAEIAFAYAQgAUJgh9R4A4Lcdu+gbz3vuZ8tm1nQWgBcrdwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKIwABAAojAAEACiMAAQAKM6TeAwDwfx276BvPe+5ny2bWdBbgxc0dQACAwghAAIDCFB+AK1euzLHHHpvhw4dn2rRpuffee+s9EgBAvyr6PYBf+tKXsnDhwqxevTrTpk3LddddlxkzZmTnzp0ZN25cvccDKIr3QELtFH0H8Nprr82FF16Y888/P1OmTMnq1avz0pe+NP/4j/9Y79EAAPpNsXcADx06lO3bt2fx4sXVY4MGDcr06dOzZcuW53xNT09Penp6qs+7urqSJN3d3f0yY2/Pr5/3XH9d80hh78/txb73FL7/kvce+y/WiZ+4/XnPPXD5jH655jO/nyqVSr98/YGg2AD87//+7xw+fDjNzc19jjc3N+fHP/7xc75m6dKlufzyy3/n+Pjx4/ttzufTdF3NL3nEsPdylbz/kvce+y9Wf////sQTT6Spqal/L3KEKjYA/xCLFy/OwoULq897e3uzb9++jB07Ng0NDS/otbq7uzN+/Pjs3r07jY2NL+jXHgjsv+z9x69B8fuPXwP778f9VyqVPPHEE2ltbX1Bv+5AUmwAvvzlL8/gwYOzZ8+ePsf37NmTlpaW53zNsGHDMmzYsD7HRo8e3a9zNjY2FvkP/jPsv+z9x69B8fuPXwP776f9l3rn7xnF/hDI0KFDc+qpp2bTpk3VY729vdm0aVPa2trqOhsAQH8q9g5gkixcuDBz5szJ1KlT86d/+qe57rrrcvDgwZx//vn1Hg0AoN8UHYDvec978l//9V9ZsmRJOjs789rXvjYbNmz4nR8MqYdhw4blE5/4xO98y7kU9l/2/uPXoPj9x6+B/Re+//7WUCn5Z6ABAApU7HsAAQBKJQABAAojAAEACiMAAQAKIwCPMJs3b84555yT1tbWNDQ05Lbbbqv3SDW1dOnSnHbaaRk1alTGjRuXc889Nzt37qz3WDWzatWqnHzyydUPPm1ra8u3vvWteo9VN8uWLUtDQ0MWLFhQ71Fq5pOf/GQaGhr6PCZPnlzvsWrql7/8Zd773vdm7NixGTFiRE466aTcd9999R6rZo499tjf+T3Q0NCQjo6Oeo9WE4cPH85ll12WSZMmZcSIEXnVq16VK664oui/t7c/FP0xMEeigwcP5jWveU0uuOCCvPOd76z3ODV31113paOjI6eddlqefvrpfPSjH81ZZ52Vhx56KCNHjqz3eP3umGOOybJly3LcccelUqlk3bp1ecc73pEf/vCHOeGEE+o9Xk1t27YtX/jCF3LyySfXe5SaO+GEE/Ltb3+7+nzIkHL+Vf2rX/0qZ5xxRt7ylrfkW9/6Vl7xilfk0UcfzVFHHVXv0Wpm27ZtOXz4cPX5Aw88kD//8z/Pu971rrrOVSvXXHNNVq1alXXr1uWEE07Ifffdl/PPPz9NTU354Ac/WO/xXjTK+bfKAHH22Wfn7LPPrvcYdbNhw4Y+z9euXZtx48Zl+/btedOb3lS3uWrlnHPO6fP8qquuyqpVq3LPPfcUFYAHDhxIe3t7brjhhlx55ZX1HqfmhgwZ8rx/JeWL3TXXXJPx48dnzZo11WOTJk2q60y19opXvKLP82XLluVVr3pV3vzmN9dtplq6++678453vCMzZ85MfnNH9J/+6Z9y77331nu0FxXfAuaI1tXVlSQZM2ZMvUepucOHD+eWW27JwYMHi/vrCTs6OjJz5sxMnz693qPUxaOPPprW1ta88pWvTHt7e3bt2lXvkWrma1/7WqZOnZp3vetdGTduXE455ZTccMMN9R6rbg4dOpQvfvGLueCCC9LQ0FDvcWri9NNPz6ZNm/LII48kSX70ox/le9/7XtE3R/qDO4AcsXp7e7NgwYKcccYZOfHEE+s9Ts3cf//9aWtry5NPPpmXvexlufXWWzNlypR6j1Uzt9xyS37wgx9k27Zt9R6lLqZNm5a1a9fm1a9+df7zP/8zl19+ed74xjfmgQceyKhRo+o9Xr/76U9/mlWrVmXhwoX56Ec/mm3btuWDH/xghg4dmjlz5tR7vJq77bbbsn///rzvfe+r9yg1s2jRonR3d2fy5MkZPHhwDh8+nKuuuirt7e31Hu1FRQByxOro6MgDDzyQ733ve/UepaZe/epXZ8eOHenq6so///M/Z86cObnrrruKiMDdu3fnoosuysaNGzN8+PB6j1MXv32X4+STT860adMyceLEfPnLX87cuXPrOlst9Pb2ZurUqbn66quTJKecckoeeOCBrF69usgAvPHGG3P22WentbW13qPUzJe//OXcdNNNufnmm3PCCSdkx44dWbBgQVpbW4v8PdBfBCBHpPnz52f9+vXZvHlzjjnmmHqPU1NDhw7NH//xHydJTj311Gzbti2f+cxn8oUvfKHeo/W77du3Z+/evXnd615XPXb48OFs3rw5n/vc59LT05PBgwfXdcZaGz16dP7kT/4kP/nJT+o9Sk0cffTRv/MfO8cff3z+5V/+pW4z1cvPf/7zfPvb386//uu/1nuUmrr00kuzaNGizJ49O0ly0kkn5ec//3mWLl0qAF9AApAjSqVSyQc+8IHceuut+e53v1vcm7+fS29vb3p6euo9Rk2ceeaZuf/++/scO//88zN58uR85CMfKS7+8psfiPmP//iPnHfeefUepSbOOOOM3/nop0ceeSQTJ06s20z1smbNmowbN676wxCl+PWvf51Bg/r+iMLgwYPT29tbt5lejATgEebAgQN9/kv/sccey44dOzJmzJhMmDChrrPVQkdHR26++eZ89atfzahRo9LZ2ZkkaWpqyogRI+o9Xr9bvHhxzj777EyYMCFPPPFEbr755nz3u9/N7bffXu/RamLUqFG/837PkSNHZuzYscW8D/SSSy7JOeeck4kTJ+bxxx/PJz7xiQwePDh/+Zd/We/RauLiiy/O6aefnquvvjrvfve7c++99+b666/P9ddfX+/Raqq3tzdr1qzJnDlzivoYoPzm0xCuuuqqTJgwISeccEJ++MMf5tprr80FF1xQ79FeXCocUb7zne9UkvzOY86cOfUerSaea+9JKmvWrKn3aDVxwQUXVCZOnFgZOnRo5RWveEXlzDPPrNxxxx31Hquu3vzmN1cuuuiieo9RM+95z3sqRx99dGXo0KGVP/qjP6q85z3vqfzkJz+p91g19fWvf71y4oknVoYNG1aZPHly5frrr6/3SDV3++23V5JUdu7cWe9Raq67u7ty0UUXVSZMmFAZPnx45ZWvfGXlYx/7WKWnp6feo72oNFR8tDYAQFF8DiAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGEEIABAYQQgAEBhBCAAQGH+D0qdXUuXgornAAAAAElFTkSuQmCC' width=640.0/>\n", - " </div>\n", - " " - ], "text/plain": [ - "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …" + "(array([1162., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 525., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 242., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 87., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 22., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 7.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 3.]),\n", + " array([1. , 1.09375, 1.1875 , 1.28125, 1.375 , 1.46875, 1.5625 ,\n", + " 1.65625, 1.75 , 1.84375, 1.9375 , 2.03125, 2.125 , 2.21875,\n", + " 2.3125 , 2.40625, 2.5 , 2.59375, 2.6875 , 2.78125, 2.875 ,\n", + " 2.96875, 3.0625 , 3.15625, 3.25 , 3.34375, 3.4375 , 3.53125,\n", + " 3.625 , 3.71875, 3.8125 , 3.90625, 4. , 4.09375, 4.1875 ,\n", + " 4.28125, 4.375 , 4.46875, 4.5625 , 4.65625, 4.75 , 4.84375,\n", + " 4.9375 , 5.03125, 5.125 , 5.21875, 5.3125 , 5.40625, 5.5 ,\n", + " 5.59375, 5.6875 , 5.78125, 5.875 , 5.96875, 6.0625 , 6.15625,\n", + " 6.25 , 6.34375, 6.4375 , 6.53125, 6.625 , 6.71875, 6.8125 ,\n", + " 6.90625, 7. ]),\n", + " <BarContainer object of 64 artists>)" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAjF0lEQVR4nO3dfVSUdf7/8RdI3GQMiMXgbIps26rkXUnRZPdyRGXd9cRWFLVUrO5pwTK6gy3vysRsu5GWMDuteE56rPasllYkYcqWiIjLpmRkmyVlA3sOMRN0RJT5/vFbr19TWtgOznzo+TjnOqe5rs/MvOc6nsOzi5khxOv1egUAAGCQ0EAPAAAAcLIIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGCQv0AH2lp6dHBw8eVHR0tEJCQgI9DgAA6AWv16uvvvpKDodDoaEnvs7SbwPm4MGDGjp0aKDHAAAAP0Jzc7POPvvsEx7vtwETHR0t/fcE2Gy2QI8DAAB6wePxaOjQodbP8RM56YCprq7WY489pvr6en3xxRdat26dZsyYIUnq7u7Wgw8+qNdff10ff/yxYmJilJaWpiVLlsjhcFiP0dbWptmzZ2vDhg0KDQ1VZmamli1bpjPOOMNa89577ykvL091dXU666yzNHv2bN133329nvPYr41sNhsBAwCAYX7o7R8n/Sbezs5OjRs3TqWlpd859vXXX2vXrl2aO3eudu3apb///e9qamrSr3/9a5912dnZamxsVGVlpTZu3Kjq6mrNmjXLOu7xeDR58mQlJiaqvr5ejz32mBYsWKAVK1ac7LgAAKAfCvlf/hp1SEiIzxWY46mrq9NFF12kTz/9VMOGDdPevXuVnJysuro6paSkSJIqKio0bdo0ffbZZ3I4HCorK9MDDzwgl8ul8PBwSVJhYaHWr1+vDz74oFezeTwexcTEyO12cwUGAABD9Pbnd59/jNrtdiskJESxsbGSpJqaGsXGxlrxIklpaWkKDQ1VbW2ttebyyy+34kWS0tPT1dTUpC+//PK4z9PV1SWPx+OzAQCA/qlPA+bQoUO6//77dcMNN1gV5XK5FB8f77MuLCxMcXFxcrlc1hq73e6z5tjtY2u+rbi4WDExMdbGJ5AAAOi/+ixguru7dd1118nr9aqsrKyvnsZSVFQkt9ttbc3NzX3+nAAAIDD65GPUx+Ll008/1ebNm31+h5WQkKDW1laf9UeOHFFbW5sSEhKsNS0tLT5rjt0+tubbIiIiFBER0QevBgAABBu/X4E5Fi/79u3TW2+9pcGDB/scdzqdam9vV319vbVv8+bN6unpUWpqqrWmurpa3d3d1prKykqNGDFCgwYN8vfIAADAMCcdMB0dHWpoaFBDQ4Mkaf/+/WpoaNCBAwfU3d2t3/72t9q5c6dWr16to0ePyuVyyeVy6fDhw5KkUaNGacqUKZo5c6Z27Nihd999V/n5+crKyrK+K+bGG29UeHi4cnNz1djYqBdffFHLli1TQUGBv18/AAAw0El/jHrLli266qqrvrM/JydHCxYsUFJS0nHv9/bbb+vKK6+U/vtFdvn5+T5fZFdSUnLCL7I788wzNXv2bN1///29npOPUQMAYJ7e/vz+n74HJpgRMAAAmCdovgcGAADA3wgYAABgHAIGAAAYh4ABAADG6ZMvsuvvhhe+dsJjnyzJOKWzAADwU8QVGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxTjpgqqurNX36dDkcDoWEhGj9+vU+x71er+bNm6chQ4YoKipKaWlp2rdvn8+atrY2ZWdny2azKTY2Vrm5uero6PBZ89577+myyy5TZGSkhg4dqqVLl/7Y1wgAAPqZkw6Yzs5OjRs3TqWlpcc9vnTpUpWUlGj58uWqra3VwIEDlZ6erkOHDllrsrOz1djYqMrKSm3cuFHV1dWaNWuWddzj8Wjy5MlKTExUfX29HnvsMS1YsEArVqz4sa8TAAD0IyFer9f7o+8cEqJ169ZpxowZ0n+vvjgcDt1999265557JElut1t2u13l5eXKysrS3r17lZycrLq6OqWkpEiSKioqNG3aNH322WdyOBwqKyvTAw88IJfLpfDwcElSYWGh1q9frw8++KBXs3k8HsXExMjtdstms/3Yl3hcwwtfO+GxT5Zk+PW5AAD4Kentz2+/vgdm//79crlcSktLs/bFxMQoNTVVNTU1kqSamhrFxsZa8SJJaWlpCg0NVW1trbXm8ssvt+JFktLT09XU1KQvv/zyuM/d1dUlj8fjswEAgP7JrwHjcrkkSXa73We/3W63jrlcLsXHx/scDwsLU1xcnM+a4z3GN5/j24qLixUTE2NtQ4cO9eMrAwAAwaTffAqpqKhIbrfb2pqbmwM9EgAA6CN+DZiEhARJUktLi8/+lpYW61hCQoJaW1t9jh85ckRtbW0+a473GN98jm+LiIiQzWbz2QAAQP/k14BJSkpSQkKCqqqqrH0ej0e1tbVyOp2SJKfTqfb2dtXX11trNm/erJ6eHqWmplprqqur1d3dba2prKzUiBEjNGjQIH+ODAAADHTSAdPR0aGGhgY1NDRI/33jbkNDgw4cOKCQkBDNmTNHixYt0quvvqrdu3frd7/7nRwOh/VJpVGjRmnKlCmaOXOmduzYoXfffVf5+fnKysqSw+GQJN14440KDw9Xbm6uGhsb9eKLL2rZsmUqKCjw9+sHAAAGCjvZO+zcuVNXXXWVdftYVOTk5Ki8vFz33XefOjs7NWvWLLW3t+vSSy9VRUWFIiMjrfusXr1a+fn5mjRpkkJDQ5WZmamSkhLreExMjDZt2qS8vDxNmDBBZ555pubNm+fzXTEAAOCn63/6HphgxvfAAABgnoB8DwwAAMCpQMAAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADj+D1gjh49qrlz5yopKUlRUVE655xz9PDDD8vr9VprvF6v5s2bpyFDhigqKkppaWnat2+fz+O0tbUpOztbNptNsbGxys3NVUdHh7/HBQAABvJ7wDz66KMqKyvTX/7yF+3du1ePPvqoli5dqqefftpas3TpUpWUlGj58uWqra3VwIEDlZ6erkOHDllrsrOz1djYqMrKSm3cuFHV1dWaNWuWv8cFAAAGCvF+89KIH/zqV7+S3W7X888/b+3LzMxUVFSUXnjhBXm9XjkcDt1999265557JElut1t2u13l5eXKysrS3r17lZycrLq6OqWkpEiSKioqNG3aNH322WdyOBw/OIfH41FMTIzcbrdsNps/X6KGF752wmOfLMnw63MBAPBT0tuf336/AnPJJZeoqqpKH374oSTpX//6l9555x1NnTpVkrR//365XC6lpaVZ94mJiVFqaqpqamokSTU1NYqNjbXiRZLS0tIUGhqq2tra4z5vV1eXPB6PzwYAAPqnMH8/YGFhoTwej0aOHKkBAwbo6NGjeuSRR5SdnS1JcrlckiS73e5zP7vdbh1zuVyKj4/3HTQsTHFxcdaabysuLtbChQv9/XIAAEAQ8vsVmJdeekmrV6/WmjVrtGvXLq1atUp//vOftWrVKn8/lY+ioiK53W5ra25u7tPnAwAAgeP3KzD33nuvCgsLlZWVJUkaM2aMPv30UxUXFysnJ0cJCQmSpJaWFg0ZMsS6X0tLi8aPHy9JSkhIUGtrq8/jHjlyRG1tbdb9vy0iIkIRERH+fjkAACAI+f0KzNdff63QUN+HHTBggHp6eiRJSUlJSkhIUFVVlXXc4/GotrZWTqdTkuR0OtXe3q76+nprzebNm9XT06PU1FR/jwwAAAzj9ysw06dP1yOPPKJhw4bpvPPO0z//+U898cQTuu222yRJISEhmjNnjhYtWqRzzz1XSUlJmjt3rhwOh2bMmCFJGjVqlKZMmaKZM2dq+fLl6u7uVn5+vrKysnr1CSQAANC/+T1gnn76ac2dO1d//OMf1draKofDoT/84Q+aN2+etea+++5TZ2enZs2apfb2dl166aWqqKhQZGSktWb16tXKz8/XpEmTFBoaqszMTJWUlPh7XAAAYCC/fw9MsOB7YAAAME/AvgcGAACgrxEwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4/RJwHz++ee66aabNHjwYEVFRWnMmDHauXOnddzr9WrevHkaMmSIoqKilJaWpn379vk8Rltbm7Kzs2Wz2RQbG6vc3Fx1dHT0xbgAAMAwfg+YL7/8UhMnTtRpp52mN954Q++//74ef/xxDRo0yFqzdOlSlZSUaPny5aqtrdXAgQOVnp6uQ4cOWWuys7PV2NioyspKbdy4UdXV1Zo1a5a/xwUAAAYK8Xq9Xn8+YGFhod5991394x//OO5xr9crh8Ohu+++W/fcc48kye12y263q7y8XFlZWdq7d6+Sk5NVV1enlJQUSVJFRYWmTZumzz77TA6H4wfn8Hg8iomJkdvtls1m8+dL1PDC10547JMlGX59LgAAfkp6+/Pb71dgXn31VaWkpOjaa69VfHy8zj//fD333HPW8f3798vlciktLc3aFxMTo9TUVNXU1EiSampqFBsba8WLJKWlpSk0NFS1tbX+HhkAABjG7wHz8ccfq6ysTOeee67efPNN3X777brjjju0atUqSZLL5ZIk2e12n/vZ7XbrmMvlUnx8vM/xsLAwxcXFWWu+raurSx6Px2cDAAD9U5i/H7Cnp0cpKSlavHixJOn888/Xnj17tHz5cuXk5Pj76SzFxcVauHBhnz0+AAAIHn6/AjNkyBAlJyf77Bs1apQOHDggSUpISJAktbS0+KxpaWmxjiUkJKi1tdXn+JEjR9TW1mat+baioiK53W5ra25u9uvrAgAAwcPvATNx4kQ1NTX57Pvwww+VmJgoSUpKSlJCQoKqqqqs4x6PR7W1tXI6nZIkp9Op9vZ21dfXW2s2b96snp4epaamHvd5IyIiZLPZfDYAANA/+f1XSHfddZcuueQSLV68WNddd5127NihFStWaMWKFZKkkJAQzZkzR4sWLdK5556rpKQkzZ07Vw6HQzNmzJD+e8VmypQpmjlzppYvX67u7m7l5+crKyurV59AAgAA/ZvfA+bCCy/UunXrVFRUpIceekhJSUl66qmnlJ2dba2577771NnZqVmzZqm9vV2XXnqpKioqFBkZaa1ZvXq18vPzNWnSJIWGhiozM1MlJSX+HhcAABjI798DEyz4HhgAAMwTsO+BAQAA6GsEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA44QFegD0L8MLXzvhsU+WZJzSWQAA/RdXYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBx+jxglixZopCQEM2ZM8fad+jQIeXl5Wnw4ME644wzlJmZqZaWFp/7HThwQBkZGTr99NMVHx+ve++9V0eOHOnrcQEAgAH6NGDq6ur07LPPauzYsT7777rrLm3YsEEvv/yytm7dqoMHD+qaa66xjh89elQZGRk6fPiwtm3bplWrVqm8vFzz5s3ry3EBAIAh+ixgOjo6lJ2dreeee06DBg2y9rvdbj3//PN64okndPXVV2vChAlauXKltm3bpu3bt0uSNm3apPfff18vvPCCxo8fr6lTp+rhhx9WaWmpDh8+3FcjAwAAQ/RZwOTl5SkjI0NpaWk+++vr69Xd3e2zf+TIkRo2bJhqamokSTU1NRozZozsdru1Jj09XR6PR42Njcd9vq6uLnk8Hp8NAAD0T2F98aBr167Vrl27VFdX951jLpdL4eHhio2N9dlvt9vlcrmsNd+Ml2PHjx07nuLiYi1cuNCPrwIAAAQrv1+BaW5u1p133qnVq1crMjLS3w9/QkVFRXK73dbW3Nx8yp4bAACcWn4PmPr6erW2tuqCCy5QWFiYwsLCtHXrVpWUlCgsLEx2u12HDx9We3u7z/1aWlqUkJAgSUpISPjOp5KO3T625tsiIiJks9l8NgAA0D/5PWAmTZqk3bt3q6GhwdpSUlKUnZ1t/fdpp52mqqoq6z5NTU06cOCAnE6nJMnpdGr37t1qbW211lRWVspmsyk5OdnfIwMAAMP4/T0w0dHRGj16tM++gQMHavDgwdb+3NxcFRQUKC4uTjabTbNnz5bT6dTFF18sSZo8ebKSk5N18803a+nSpXK5XHrwwQeVl5eniIgIf48MAAAM0ydv4v0hTz75pEJDQ5WZmamuri6lp6frmWeesY4PGDBAGzdu1O233y6n06mBAwcqJydHDz30UCDGBQAAQeaUBMyWLVt8bkdGRqq0tFSlpaUnvE9iYqJef/31UzAdAAAwDX8LCQAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYJyzQAwA/dcMLXzvhsU+WZJzSWQDAFFyBAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABjH7wFTXFysCy+8UNHR0YqPj9eMGTPU1NTks+bQoUPKy8vT4MGDdcYZZygzM1MtLS0+aw4cOKCMjAydfvrpio+P17333qsjR474e1wAAGAgvwfM1q1blZeXp+3bt6uyslLd3d2aPHmyOjs7rTV33XWXNmzYoJdffllbt27VwYMHdc0111jHjx49qoyMDB0+fFjbtm3TqlWrVF5ernnz5vl7XAAAYCC//ymBiooKn9vl5eWKj49XfX29Lr/8crndbj3//PNas2aNrr76aknSypUrNWrUKG3fvl0XX3yxNm3apPfff19vvfWW7Ha7xo8fr4cfflj333+/FixYoPDwcH+PDQAADNLn74Fxu92SpLi4OElSfX29uru7lZaWZq0ZOXKkhg0bppqaGklSTU2NxowZI7vdbq1JT0+Xx+NRY2PjcZ+nq6tLHo/HZwMAAP1TnwZMT0+P5syZo4kTJ2r06NGSJJfLpfDwcMXGxvqstdvtcrlc1ppvxsux48eOHU9xcbFiYmKsbejQoX30qgAAQKD1acDk5eVpz549Wrt2bV8+jSSpqKhIbrfb2pqbm/v8OQEAQGD4/T0wx+Tn52vjxo2qrq7W2Wefbe1PSEjQ4cOH1d7e7nMVpqWlRQkJCdaaHTt2+DzesU8pHVvzbREREYqIiOijVwMAAIKJ36/AeL1e5efna926ddq8ebOSkpJ8jk+YMEGnnXaaqqqqrH1NTU06cOCAnE6nJMnpdGr37t1qbW211lRWVspmsyk5OdnfIwMAAMP4/QpMXl6e1qxZo1deeUXR0dHWe1ZiYmIUFRWlmJgY5ebmqqCgQHFxcbLZbJo9e7acTqcuvvhiSdLkyZOVnJysm2++WUuXLpXL5dKDDz6ovLw8rrIAAAD/B0xZWZkk6corr/TZv3LlSt1yyy2SpCeffFKhoaHKzMxUV1eX0tPT9cwzz1hrBwwYoI0bN+r222+X0+nUwIEDlZOTo4ceesjf4wIAAAP5PWC8Xu8PromMjFRpaalKS0tPuCYxMVGvv/66n6cDAAD9AX8LCQAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAccICPQAA/BjDC1874bFPlmSc0lkAnHpcgQEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgnLBADwAA8J/hha+d8NgnSzJO6SxAX+IKDAAAME5QB0xpaamGDx+uyMhIpaamaseOHYEeCQAABIGgDZgXX3xRBQUFmj9/vnbt2qVx48YpPT1dra2tgR4NAAAEWNC+B+aJJ57QzJkzdeutt0qSli9frtdee01//etfVVhYGOjxAAD9CO8dMk9QBszhw4dVX1+voqIia19oaKjS0tJUU1Nz3Pt0dXWpq6vLuu12uyVJHo/H7/P1dH19wmN98Xwm4dycPM7Zj8N5Oz7Oy4/DeTu+0fPfPOGxPQvT++Q5j51vr9f7/Qu9Qejzzz/3SvJu27bNZ/+9997rveiii457n/nz53slsbGxsbGxsfWDrbm5+XtbISivwPwYRUVFKigosG739PSora1NgwcPVkhIiN+ex+PxaOjQoWpubpbNZvPb4/ZXnK/e41z1Hueq9zhXvce56r2+PFder1dfffWVHA7H964LyoA588wzNWDAALW0tPjsb2lpUUJCwnHvExERoYiICJ99sbGxfTajzWbjH/hJ4Hz1Hueq9zhXvce56j3OVe/11bmKiYn5wTVB+Smk8PBwTZgwQVVVVda+np4eVVVVyel0BnQ2AAAQeEF5BUaSCgoKlJOTo5SUFF100UV66qmn1NnZaX0qCQAA/HQFbcBcf/31+s9//qN58+bJ5XJp/PjxqqiokN1uD+hcERERmj9//nd+XYXj43z1Hueq9zhXvce56j3OVe8Fw7kK8f7g55QAAACCS1C+BwYAAOD7EDAAAMA4BAwAADAOAQMAAIxDwJyE6upqTZ8+XQ6HQyEhIVq/fn2gRwpKxcXFuvDCCxUdHa34+HjNmDFDTU1NgR4rKJWVlWns2LHWl0E5nU698cYbgR7LCEuWLFFISIjmzJkT6FGC0oIFCxQSEuKzjRw5MtBjBa3PP/9cN910kwYPHqyoqCiNGTNGO3fuDPRYQWf48OHf+XcVEhKivLy8Uz4LAXMSOjs7NW7cOJWWlgZ6lKC2detW5eXlafv27aqsrFR3d7cmT56szs7OQI8WdM4++2wtWbJE9fX12rlzp66++mr95je/UWNjY6BHC2p1dXV69tlnNXbs2ECPEtTOO+88ffHFF9b2zjvvBHqkoPTll19q4sSJOu200/TGG2/o/fff1+OPP65BgwYFerSgU1dX5/NvqrKyUpJ07bXXnvJZgvZ7YILR1KlTNXXq1ECPEfQqKip8bpeXlys+Pl719fW6/PLLAzZXMJo+fbrP7UceeURlZWXavn27zjvvvIDNFcw6OjqUnZ2t5557TosWLQr0OEEtLCzshH9+Bf/fo48+qqFDh2rlypXWvqSkpIDOFKzOOussn9tLlizROeecoyuuuOKUz8IVGPQ5t9stSYqLiwv0KEHt6NGjWrt2rTo7O/mTGd8jLy9PGRkZSktLC/QoQW/fvn1yOBz6+c9/ruzsbB04cCDQIwWlV199VSkpKbr22msVHx+v888/X88991ygxwp6hw8f1gsvvKDbbrvNr380ube4AoM+1dPTozlz5mjixIkaPXp0oMcJSrt375bT6dShQ4d0xhlnaN26dUpOTg70WEFp7dq12rVrl+rq6gI9StBLTU1VeXm5RowYoS+++EILFy7UZZddpj179ig6OjrQ4wWVjz/+WGVlZSooKNCf/vQn1dXV6Y477lB4eLhycnICPV7QWr9+vdrb23XLLbcE5PkJGPSpvLw87dmzh9+9f48RI0aooaFBbrdbf/vb35STk6OtW7cSMd/S3NysO++8U5WVlYqMjAz0OEHvm7/uHjt2rFJTU5WYmKiXXnpJubm5AZ0t2PT09CglJUWLFy+WJJ1//vnas2ePli9fTsB8j+eff15Tp06Vw+EIyPPzKyT0mfz8fG3cuFFvv/22zj777ECPE7TCw8P1i1/8QhMmTFBxcbHGjRunZcuWBXqsoFNfX6/W1lZdcMEFCgsLU1hYmLZu3aqSkhKFhYXp6NGjgR4xqMXGxuqXv/ylPvroo0CPEnSGDBnynf9hGDVqFL9y+x6ffvqp3nrrLf3+978P2AxcgYHfeb1ezZ49W+vWrdOWLVt4M9xJ6unpUVdXV6DHCDqTJk3S7t27ffbdeuutGjlypO6//34NGDAgYLOZoKOjQ//+97918803B3qUoDNx4sTvfNXDhx9+qMTExIDNFOxWrlyp+Ph4ZWRkBGwGAuYkdHR0+Pzfy/79+9XQ0KC4uDgNGzYsoLMFk7y8PK1Zs0avvPKKoqOj5XK5JEkxMTGKiooK9HhBpaioSFOnTtWwYcP01Vdfac2aNdqyZYvefPPNQI8WdKKjo7/zPqqBAwdq8ODBvL/qOO655x5Nnz5diYmJOnjwoObPn68BAwbohhtuCPRoQeeuu+7SJZdcosWLF+u6667Tjh07tGLFCq1YsSLQowWlnp4erVy5Ujk5OQoLC2BGeNFrb7/9tlfSd7acnJxAjxZUjneOJHlXrlwZ6NGCzm233eZNTEz0hoeHe8866yzvpEmTvJs2bQr0WMa44oorvHfeeWegxwhK119/vXfIkCHe8PBw789+9jPv9ddf7/3oo48CPVbQ2rBhg3f06NHeiIgI78iRI70rVqwI9EhB68033/RK8jY1NQV0jhDv//uBAwAAYAzexAsAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADDO/wGGkh4mNSHckgAAAABJRU5ErkJggg==", + "text/plain": [ + "<Figure size 640x480 with 1 Axes>" ] }, "metadata": {}, @@ -322,9 +292,7 @@ } ], "source": [ - "with plt.ioff():\n", - " plt.hist(tune_data['labels'],bins=64)\n", - " plt.show()" + "plt.hist(mktunebatch(2048)[1].cpu(), bins=64)" ] }, { @@ -338,7 +306,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 7, "execution_state": "idle", "metadata": { "id": "tLOWhg_CeWzH" @@ -383,12 +351,12 @@ "id": "bpIeg86S-hBb" }, "source": [ - "# Step 3: Load Data" + "# Step 3: Make Model" ] }, { "cell_type": "code", - "execution_count": 78, + "execution_count": 49, "execution_state": "idle", "metadata": { "colab": { @@ -402,41 +370,27 @@ "name": "stdout", "output_type": "stream", "text": [ - "Trainable parameters in the model: 2390K\n" + "Trainable parameters in the model: 505K\n" ] } ], "source": [ - "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", - "assert device.type == 'cuda', \"CUDA is not available. Please check your GPU setup.\"\n", - "\n", "# PARAMS\n", "VOCAB_SIZE = 1 + MAX_VTXS # one more than the max number of vertices\n", - "MODEL_DIM = 256 # Dimension of model (embedding and transformer)\n", + "MODEL_DIM = 64 # Dimension of model (embedding and transformer)\n", "NEPOCHS = 1000\n", - "BSZ = 3072\n", + "BSZ = 2048 # Batch size\n", + "BPE = 32 # Batches per epoch\n", "LR = 0.003\n", "WD = 0.002\n", "NHEADS = 4\n", - "NLAYERS = 3\n", - "PAD_TOKEN = 0\n", + "NLAYERS = 10\n", "DROPOUT = 0.2\n", "model = TransformerModel(input_dim=VOCAB_SIZE, model_dim=MODEL_DIM,\n", " output_dim=1, num_heads=NHEADS,\n", " num_layers=NLAYERS, seq_len=SEQ_LEN,\n", " dropout=DROPOUT, device=device).to(device)\n", "\n", - "with open(\"data.pkl\", \"rb\") as f:\n", - " pickled_stuff = pickle.load(f)\n", - "\n", - "data = pickled_stuff[\"data\"].to(device)\n", - "label = pickled_stuff[\"labels\"].to(device)\n", - "padding_mask = (data == PAD_TOKEN).bool().to(device)\n", - "dataset = TensorDataset(data, label, padding_mask)\n", - "train_dataset, test_dataset = torch.utils.data.random_split(dataset, [.8, .2])\n", - "train_loader = DataLoader(train_dataset, batch_size=BSZ, shuffle=True)\n", - "test_loader = DataLoader(test_dataset, batch_size=BSZ, shuffle=True)\n", - "\n", "criterion = nn.MSELoss()\n", "optimizer = torch.optim.Adam(model.parameters(), lr=LR, weight_decay=WD)\n", "\n", @@ -445,25 +399,6 @@ ] }, { - "cell_type": "code", - "execution_count": 63, - "execution_state": "idle", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "tensor(141.4637, device='cuda:0')\n" - ] - } - ], - "source": [ - "baseline_error = criterion(label, torch.tensor(1.5, dtype=torch.float32, device=device))\n", - "print(baseline_error)" - ] - }, - { "cell_type": "markdown", "metadata": { "id": "f8Zn33m7CxL5" @@ -474,7 +409,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 46, "execution_state": "idle", "metadata": {}, "outputs": [], @@ -483,23 +418,22 @@ " model.eval()\n", " test_loss = 0\n", " with torch.no_grad():\n", - " for batch_src, batch_labels, batch_padding_mask in test_loader:\n", - " output = model(batch_src, batch_padding_mask)\n", - " loss = criterion(output.squeeze(1), batch_labels)\n", - " test_loss += loss.item()/len(test_loader)\n", - " return test_loss" + " batch_src, batch_labels, batch_padding_mask = mkbatch(BSZ)\n", + " output = model(batch_src, batch_padding_mask)\n", + " loss = criterion(output.squeeze(1), batch_labels)\n", + " return loss.item()" ] }, { "cell_type": "code", - "execution_count": 74, + "execution_count": 51, "execution_state": "idle", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "329425e6ee6d4189aefee350eba741c7", + "model_id": "3ba829714ada43c184a04b0a0b4d06f2", "version_major": 2, "version_minor": 0 }, @@ -524,9 +458,11 @@ ], "source": [ "# This has to be in a separate cell for some weird event loop reasons\n", + "%matplotlib widget\n", "fig,ax = plt.subplots()\n", "fig.suptitle('MSE vs Epochs')\n", - "plt.show()" + "plt.show()\n", + "%matplotlib inline" ] }, { @@ -543,166 +479,836 @@ }, "outputs": [ { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.32it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/1000 \t Train Err: 80.9485 \t Test Err: 83.3727\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.33it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 2/1000 \t Train Err: 81.9558 \t Test Err: 80.7205\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.32it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 3/1000 \t Train Err: 81.2221 \t Test Err: 80.6387\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.32it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 4/1000 \t Train Err: 81.8502 \t Test Err: 80.0444\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.32it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 5/1000 \t Train Err: 81.5031 \t Test Err: 83.7185\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.33it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 6/1000 \t Train Err: 81.3043 \t Test Err: 81.4035\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.22it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 7/1000 \t Train Err: 81.0616 \t Test Err: 83.7366\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.28it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 8/1000 \t Train Err: 81.5992 \t Test Err: 81.2875\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 9/1000 \t Train Err: 81.3813 \t Test Err: 80.2028\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 10/1000 \t Train Err: 81.5702 \t Test Err: 82.8906\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.32it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 11/1000 \t Train Err: 80.6410 \t Test Err: 81.6353\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 12/1000 \t Train Err: 81.0706 \t Test Err: 81.4791\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 13/1000 \t Train Err: 81.0538 \t Test Err: 81.0688\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 14/1000 \t Train Err: 81.4753 \t Test Err: 85.5978\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 15/1000 \t Train Err: 81.2319 \t Test Err: 81.5276\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 16/1000 \t Train Err: 82.0405 \t Test Err: 80.4760\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 17/1000 \t Train Err: 81.2955 \t Test Err: 80.1790\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:10<00:00, 3.05it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 18/1000 \t Train Err: 81.3618 \t Test Err: 81.0788\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 19/1000 \t Train Err: 81.4784 \t Test Err: 82.8825\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 20/1000 \t Train Err: 80.7994 \t Test Err: 81.8424\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 21/1000 \t Train Err: 80.9150 \t Test Err: 80.6047\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 22/1000 \t Train Err: 81.7054 \t Test Err: 78.2826\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 23/1000 \t Train Err: 81.6376 \t Test Err: 83.0617\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.32it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 24/1000 \t Train Err: 81.1639 \t Test Err: 79.6304\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.33it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 25/1000 \t Train Err: 81.9200 \t Test Err: 82.3950\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.36it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 26/1000 \t Train Err: 81.0736 \t Test Err: 83.0353\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 27/1000 \t Train Err: 81.8939 \t Test Err: 80.7981\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.35it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 28/1000 \t Train Err: 80.9842 \t Test Err: 80.3877\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.36it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 29/1000 \t Train Err: 81.6111 \t Test Err: 82.5336\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.37it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 30/1000 \t Train Err: 81.5480 \t Test Err: 82.7556\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.37it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 31/1000 \t Train Err: 81.2413 \t Test Err: 82.6558\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.36it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 32/1000 \t Train Err: 81.1720 \t Test Err: 82.2116\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.36it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 33/1000 \t Train Err: 81.7244 \t Test Err: 79.6762\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.37it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 34/1000 \t Train Err: 81.4536 \t Test Err: 84.7001\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.30it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 35/1000 \t Train Err: 82.1729 \t Test Err: 82.0201\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.37it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 36/1000 \t Train Err: 81.7041 \t Test Err: 83.0776\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.37it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 37/1000 \t Train Err: 81.2599 \t Test Err: 82.2269\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.37it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 38/1000 \t Train Err: 81.6489 \t Test Err: 81.6412\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 39/1000 \t Train Err: 81.4363 \t Test Err: 81.9661\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.32it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 40/1000 \t Train Err: 81.0156 \t Test Err: 78.0546\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.36it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 41/1000 \t Train Err: 81.2752 \t Test Err: 82.3804\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.38it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 42/1000 \t Train Err: 81.1951 \t Test Err: 81.7494\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.38it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 43/1000 \t Train Err: 81.4909 \t Test Err: 81.9628\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.37it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 44/1000 \t Train Err: 81.5728 \t Test Err: 78.5453\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.37it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 45/1000 \t Train Err: 81.9706 \t Test Err: 81.1184\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.38it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 46/1000 \t Train Err: 81.1537 \t Test Err: 81.5044\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.37it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 47/1000 \t Train Err: 80.8373 \t Test Err: 82.4630\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.38it/s]\n" + ] + }, + { "name": "stdout", "output_type": "stream", "text": [ - "Epoch 1/1000 \t Train Err: 88.4438 \t Test Err: 72.6466 \t baseline err: 141.4637\n", - "Epoch 2/1000 \t Train Err: 72.6060 \t Test Err: 75.4987 \t baseline err: 141.4637\n", - "Epoch 3/1000 \t Train Err: 72.6729 \t Test Err: 73.9506 \t baseline err: 141.4637\n", - "Epoch 6/1000 \t Train Err: 72.7866 \t Test Err: 72.4612 \t baseline err: 141.4637\n", - "Epoch 7/1000 \t Train Err: 73.0815 \t Test Err: 74.9404 \t baseline err: 141.4637\n", - "Epoch 8/1000 \t Train Err: 72.8054 \t Test Err: 73.7412 \t baseline err: 141.4637\n", - "Epoch 9/1000 \t Train Err: 73.0514 \t Test Err: 82.7393 \t baseline err: 141.4637\n", - "Epoch 10/1000 \t Train Err: 73.0291 \t Test Err: 72.2577 \t baseline err: 141.4637\n", - "Epoch 11/1000 \t Train Err: 73.0081 \t Test Err: 72.4944 \t baseline err: 141.4637\n", - "Epoch 12/1000 \t Train Err: 72.6500 \t Test Err: 75.2036 \t baseline err: 141.4637\n", - "Epoch 13/1000 \t Train Err: 72.6255 \t Test Err: 73.4970 \t baseline err: 141.4637\n", - "Epoch 14/1000 \t Train Err: 72.6803 \t Test Err: 74.0005 \t baseline err: 141.4637\n", - "Epoch 15/1000 \t Train Err: 72.7032 \t Test Err: 73.0177 \t baseline err: 141.4637\n", - "Epoch 16/1000 \t Train Err: 72.7891 \t Test Err: 75.1899 \t baseline err: 141.4637\n", - "Epoch 17/1000 \t Train Err: 74.0133 \t Test Err: 71.7237 \t baseline err: 141.4637\n", - "Epoch 18/1000 \t Train Err: 72.7520 \t Test Err: 75.8566 \t baseline err: 141.4637\n", - "Epoch 19/1000 \t Train Err: 72.5771 \t Test Err: 74.9531 \t baseline err: 141.4637\n", - "Epoch 20/1000 \t Train Err: 72.6114 \t Test Err: 73.3918 \t baseline err: 141.4637\n", - "Epoch 21/1000 \t Train Err: 71.5844 \t Test Err: 57.3829 \t baseline err: 141.4637\n", - "Epoch 22/1000 \t Train Err: 56.8166 \t Test Err: 60.2253 \t baseline err: 141.4637\n", - "Epoch 23/1000 \t Train Err: 58.2172 \t Test Err: 56.7333 \t baseline err: 141.4637\n", - "Epoch 24/1000 \t Train Err: 56.1189 \t Test Err: 55.7485 \t baseline err: 141.4637\n", - "Epoch 25/1000 \t Train Err: 55.5304 \t Test Err: 56.2083 \t baseline err: 141.4637\n", - "Epoch 26/1000 \t Train Err: 68.7059 \t Test Err: 72.6976 \t baseline err: 141.4637\n", - "Epoch 27/1000 \t Train Err: 72.7020 \t Test Err: 73.1029 \t baseline err: 141.4637\n", - "Epoch 28/1000 \t Train Err: 72.4459 \t Test Err: 73.5617 \t baseline err: 141.4637\n", - "Epoch 29/1000 \t Train Err: 72.5310 \t Test Err: 75.8304 \t baseline err: 141.4637\n", - "Epoch 30/1000 \t Train Err: 72.5256 \t Test Err: 73.0845 \t baseline err: 141.4637\n", - "Epoch 31/1000 \t Train Err: 72.4667 \t Test Err: 72.9080 \t baseline err: 141.4637\n", - "Epoch 32/1000 \t Train Err: 72.5369 \t Test Err: 72.6703 \t baseline err: 141.4637\n", - "Epoch 33/1000 \t Train Err: 72.4685 \t Test Err: 74.7614 \t baseline err: 141.4637\n", - "Epoch 34/1000 \t Train Err: 72.4926 \t Test Err: 74.0886 \t baseline err: 141.4637\n", - "Epoch 35/1000 \t Train Err: 71.3339 \t Test Err: 55.4380 \t baseline err: 141.4637\n", - "Epoch 36/1000 \t Train Err: 60.7870 \t Test Err: 59.4468 \t baseline err: 141.4637\n", - "Epoch 37/1000 \t Train Err: 55.6557 \t Test Err: 56.7001 \t baseline err: 141.4637\n", - "Epoch 38/1000 \t Train Err: 55.4896 \t Test Err: 55.8308 \t baseline err: 141.4637\n", - "Epoch 39/1000 \t Train Err: 55.6962 \t Test Err: 58.9664 \t baseline err: 141.4637\n", - "Epoch 40/1000 \t Train Err: 55.5519 \t Test Err: 57.7560 \t baseline err: 141.4637\n", - "Epoch 41/1000 \t Train Err: 55.5370 \t Test Err: 56.6866 \t baseline err: 141.4637\n", - "Epoch 42/1000 \t Train Err: 55.4300 \t Test Err: 56.3424 \t baseline err: 141.4637\n", - "Epoch 43/1000 \t Train Err: 55.4922 \t Test Err: 56.3748 \t baseline err: 141.4637\n", - "Epoch 44/1000 \t Train Err: 55.6073 \t Test Err: 59.0728 \t baseline err: 141.4637\n", - "Epoch 45/1000 \t Train Err: 55.5497 \t Test Err: 58.5533 \t baseline err: 141.4637\n", - "Epoch 46/1000 \t Train Err: 55.4837 \t Test Err: 57.2847 \t baseline err: 141.4637\n", - "Epoch 47/1000 \t Train Err: 55.4173 \t Test Err: 57.1441 \t baseline err: 141.4637\n", - "Epoch 48/1000 \t Train Err: 55.4576 \t Test Err: 55.7806 \t baseline err: 141.4637\n", - "Epoch 49/1000 \t Train Err: 55.5678 \t Test Err: 56.6457 \t baseline err: 141.4637\n", - "Epoch 50/1000 \t Train Err: 55.5537 \t Test Err: 60.0365 \t baseline err: 141.4637\n", - "Epoch 51/1000 \t Train Err: 55.5123 \t Test Err: 55.6848 \t baseline err: 141.4637\n", - "Epoch 52/1000 \t Train Err: 55.5872 \t Test Err: 55.8084 \t baseline err: 141.4637\n", - "Epoch 53/1000 \t Train Err: 55.4548 \t Test Err: 56.5655 \t baseline err: 141.4637\n", - "Epoch 54/1000 \t Train Err: 55.5124 \t Test Err: 56.3470 \t baseline err: 141.4637\n", - "Epoch 55/1000 \t Train Err: 55.4518 \t Test Err: 57.6169 \t baseline err: 141.4637\n", - "Epoch 56/1000 \t Train Err: 55.4073 \t Test Err: 55.6467 \t baseline err: 141.4637\n", - "Epoch 57/1000 \t Train Err: 55.4745 \t Test Err: 56.3436 \t baseline err: 141.4637\n", - "Epoch 58/1000 \t Train Err: 55.4862 \t Test Err: 56.2289 \t baseline err: 141.4637\n", - "Epoch 59/1000 \t Train Err: 55.5221 \t Test Err: 55.3599 \t baseline err: 141.4637\n", - "Epoch 60/1000 \t Train Err: 55.4843 \t Test Err: 55.3953 \t baseline err: 141.4637\n", - "Epoch 61/1000 \t Train Err: 55.5095 \t Test Err: 56.4781 \t baseline err: 141.4637\n", - "Epoch 62/1000 \t Train Err: 55.6532 \t Test Err: 56.4005 \t baseline err: 141.4637\n", - "Epoch 63/1000 \t Train Err: 55.5240 \t Test Err: 57.4780 \t baseline err: 141.4637\n", - "Epoch 64/1000 \t Train Err: 55.4915 \t Test Err: 55.8880 \t baseline err: 141.4637\n", - "Epoch 65/1000 \t Train Err: 55.4006 \t Test Err: 56.1770 \t baseline err: 141.4637\n", - "Epoch 66/1000 \t Train Err: 55.3153 \t Test Err: 56.3041 \t baseline err: 141.4637\n", - "Epoch 67/1000 \t Train Err: 55.3105 \t Test Err: 55.7897 \t baseline err: 141.4637\n", - "Epoch 68/1000 \t Train Err: 55.9038 \t Test Err: 54.9242 \t baseline err: 141.4637\n", - "Epoch 69/1000 \t Train Err: 55.4002 \t Test Err: 55.2162 \t baseline err: 141.4637\n", - "Epoch 70/1000 \t Train Err: 55.5265 \t Test Err: 54.4618 \t baseline err: 141.4637\n", - "Epoch 71/1000 \t Train Err: 55.4598 \t Test Err: 56.2988 \t baseline err: 141.4637\n", - "Epoch 72/1000 \t Train Err: 55.4995 \t Test Err: 55.1318 \t baseline err: 141.4637\n", - "Epoch 73/1000 \t Train Err: 55.5224 \t Test Err: 55.6233 \t baseline err: 141.4637\n", - "Epoch 74/1000 \t Train Err: 55.2633 \t Test Err: 55.0628 \t baseline err: 141.4637\n", - "Epoch 75/1000 \t Train Err: 55.3569 \t Test Err: 54.9321 \t baseline err: 141.4637\n", - "Epoch 76/1000 \t Train Err: 55.4845 \t Test Err: 55.7232 \t baseline err: 141.4637\n", - "Epoch 77/1000 \t Train Err: 55.3814 \t Test Err: 54.6657 \t baseline err: 141.4637\n", - "Epoch 78/1000 \t Train Err: 55.4396 \t Test Err: 55.2952 \t baseline err: 141.4637\n", - "Epoch 79/1000 \t Train Err: 55.4018 \t Test Err: 55.4081 \t baseline err: 141.4637\n", - "Epoch 80/1000 \t Train Err: 55.5015 \t Test Err: 56.3544 \t baseline err: 141.4637\n", - "Epoch 81/1000 \t Train Err: 55.5352 \t Test Err: 55.8122 \t baseline err: 141.4637\n", - "Epoch 82/1000 \t Train Err: 55.4454 \t Test Err: 54.7959 \t baseline err: 141.4637\n", - "Epoch 83/1000 \t Train Err: 55.4375 \t Test Err: 55.1435 \t baseline err: 141.4637\n", - "Epoch 84/1000 \t Train Err: 55.4614 \t Test Err: 54.7396 \t baseline err: 141.4637\n", - "Epoch 85/1000 \t Train Err: 55.4046 \t Test Err: 55.3768 \t baseline err: 141.4637\n", - "Epoch 86/1000 \t Train Err: 55.3655 \t Test Err: 54.7487 \t baseline err: 141.4637\n", - "Epoch 87/1000 \t Train Err: 55.4036 \t Test Err: 55.0165 \t baseline err: 141.4637\n", - "Epoch 88/1000 \t Train Err: 55.4548 \t Test Err: 55.5787 \t baseline err: 141.4637\n", - "Epoch 89/1000 \t Train Err: 55.3973 \t Test Err: 54.7695 \t baseline err: 141.4637\n", - "Epoch 90/1000 \t Train Err: 55.4891 \t Test Err: 56.6628 \t baseline err: 141.4637\n", - "Epoch 91/1000 \t Train Err: 55.5870 \t Test Err: 56.8348 \t baseline err: 141.4637\n", - "Epoch 92/1000 \t Train Err: 55.4630 \t Test Err: 55.4178 \t baseline err: 141.4637\n", - "Epoch 93/1000 \t Train Err: 55.5218 \t Test Err: 55.8851 \t baseline err: 141.4637\n", - "Epoch 94/1000 \t Train Err: 55.4550 \t Test Err: 55.9211 \t baseline err: 141.4637\n", - "Epoch 95/1000 \t Train Err: 55.4727 \t Test Err: 56.5819 \t baseline err: 141.4637\n", - "Epoch 96/1000 \t Train Err: 55.4301 \t Test Err: 57.2222 \t baseline err: 141.4637\n", - "Epoch 97/1000 \t Train Err: 55.4108 \t Test Err: 55.3496 \t baseline err: 141.4637\n", - "Epoch 98/1000 \t Train Err: 55.4733 \t Test Err: 55.8675 \t baseline err: 141.4637\n", - "Epoch 99/1000 \t Train Err: 55.3536 \t Test Err: 56.2623 \t baseline err: 141.4637\n", - "Epoch 100/1000 \t Train Err: 55.2286 \t Test Err: 54.2883 \t baseline err: 141.4637\n", - "Epoch 101/1000 \t Train Err: 54.6294 \t Test Err: 54.3795 \t baseline err: 141.4637\n", - "Epoch 102/1000 \t Train Err: 54.0334 \t Test Err: 52.9438 \t baseline err: 141.4637\n", - "Epoch 103/1000 \t Train Err: 55.8557 \t Test Err: 55.9887 \t baseline err: 141.4637\n", - "Epoch 104/1000 \t Train Err: 55.3523 \t Test Err: 55.6809 \t baseline err: 141.4637\n", - "Epoch 105/1000 \t Train Err: 54.8650 \t Test Err: 54.1854 \t baseline err: 141.4637\n", - "Epoch 106/1000 \t Train Err: 54.8108 \t Test Err: 54.9449 \t baseline err: 141.4637\n", - "Epoch 107/1000 \t Train Err: 54.4932 \t Test Err: 53.3353 \t baseline err: 141.4637\n", - "Epoch 108/1000 \t Train Err: 54.0328 \t Test Err: 54.7242 \t baseline err: 141.4637\n", - "Epoch 109/1000 \t Train Err: 52.3047 \t Test Err: 50.1951 \t baseline err: 141.4637\n", - "Epoch 110/1000 \t Train Err: 46.5255 \t Test Err: 36.4212 \t baseline err: 141.4637\n", - "Epoch 111/1000 \t Train Err: 35.6437 \t Test Err: 36.2409 \t baseline err: 141.4637\n", - "Epoch 112/1000 \t Train Err: 35.5794 \t Test Err: 36.0152 \t baseline err: 141.4637\n", - "Epoch 113/1000 \t Train Err: 49.8337 \t Test Err: 56.2286 \t baseline err: 141.4637\n", - "Epoch 114/1000 \t Train Err: 55.4618 \t Test Err: 55.5159 \t baseline err: 141.4637\n", - "Epoch 115/1000 \t Train Err: 48.2926 \t Test Err: 37.0193 \t baseline err: 141.4637\n", - "Epoch 116/1000 \t Train Err: 43.0000 \t Test Err: 42.3349 \t baseline err: 141.4637\n", - "Epoch 117/1000 \t Train Err: 36.4887 \t Test Err: 38.2387 \t baseline err: 141.4637\n", - "Epoch 118/1000 \t Train Err: 35.7809 \t Test Err: 37.3943 \t baseline err: 141.4637\n", - "Epoch 119/1000 \t Train Err: 35.7078 \t Test Err: 37.0414 \t baseline err: 141.4637\n", - "Epoch 120/1000 \t Train Err: 35.7773 \t Test Err: 37.2562 \t baseline err: 141.4637\n", - "Epoch 121/1000 \t Train Err: 36.1376 \t Test Err: 38.1538 \t baseline err: 141.4637\n", - "Epoch 122/1000 \t Train Err: 38.3734 \t Test Err: 38.6760 \t baseline err: 141.4637\n", - "Epoch 123/1000 \t Train Err: 36.6591 \t Test Err: 38.0746 \t baseline err: 141.4637\n", - "Epoch 124/1000 \t Train Err: 37.1041 \t Test Err: 38.8434 \t baseline err: 141.4637\n", - "Epoch 125/1000 \t Train Err: 37.0860 \t Test Err: 37.3192 \t baseline err: 141.4637\n", - "Epoch 126/1000 \t Train Err: 35.8674 \t Test Err: 36.7553 \t baseline err: 141.4637\n", - "Epoch 127/1000 \t Train Err: 35.8025 \t Test Err: 36.0833 \t baseline err: 141.4637\n", - "Epoch 128/1000 \t Train Err: 35.6595 \t Test Err: 36.1782 \t baseline err: 141.4637\n", - "Epoch 129/1000 \t Train Err: 35.6861 \t Test Err: 36.1815 \t baseline err: 141.4637\n", - "Epoch 130/1000 \t Train Err: 35.5056 \t Test Err: 36.4077 \t baseline err: 141.4637\n", - "Epoch 131/1000 \t Train Err: 35.5161 \t Test Err: 36.6100 \t baseline err: 141.4637\n", - "Epoch 132/1000 \t Train Err: 35.4981 \t Test Err: 36.1938 \t baseline err: 141.4637\n", - "Epoch 133/1000 \t Train Err: 35.4577 \t Test Err: 36.2723 \t baseline err: 141.4637\n", - "Epoch 134/1000 \t Train Err: 35.5685 \t Test Err: 36.0326 \t baseline err: 141.4637\n", - "Epoch 135/1000 \t Train Err: 35.4790 \t Test Err: 36.0421 \t baseline err: 141.4637\n", - "Epoch 136/1000 \t Train Err: 35.5135 \t Test Err: 35.9383 \t baseline err: 141.4637\n", - "Epoch 137/1000 \t Train Err: 35.4027 \t Test Err: 35.5105 \t baseline err: 141.4637\n", - "Epoch 138/1000 \t Train Err: 35.4036 \t Test Err: 35.1535 \t baseline err: 141.4637\n", - "Epoch 139/1000 \t Train Err: 35.5047 \t Test Err: 35.7687 \t baseline err: 141.4637\n", - "Epoch 140/1000 \t Train Err: 35.3858 \t Test Err: 35.5093 \t baseline err: 141.4637\n", - "Epoch 141/1000 \t Train Err: 35.3629 \t Test Err: 35.1772 \t baseline err: 141.4637\n", - "Epoch 142/1000 \t Train Err: 35.3714 \t Test Err: 35.0682 \t baseline err: 141.4637\n", - "Epoch 143/1000 \t Train Err: 35.3976 \t Test Err: 35.6668 \t baseline err: 141.4637\n", - "Epoch 144/1000 \t Train Err: 35.4523 \t Test Err: 35.2633 \t baseline err: 141.4637\n", - "Epoch 145/1000 \t Train Err: 35.3763 \t Test Err: 35.5053 \t baseline err: 141.4637\n", - "Epoch 146/1000 \t Train Err: 35.3906 \t Test Err: 35.1866 \t baseline err: 141.4637\n", - "Epoch 147/1000 \t Train Err: 35.4717 \t Test Err: 35.2663 \t baseline err: 141.4637\n", - "Epoch 148/1000 \t Train Err: 49.5998 \t Test Err: 54.9490 \t baseline err: 141.4637\n", - "Epoch 149/1000 \t Train Err: 56.0901 \t Test Err: 55.1923 \t baseline err: 141.4637\n", - "Epoch 150/1000 \t Train Err: 55.4467 \t Test Err: 54.8896 \t baseline err: 141.4637\n", - "Epoch 151/1000 \t Train Err: 55.4050 \t Test Err: 55.0999 \t baseline err: 141.4637\n", - "Epoch 152/1000 \t Train Err: 55.3775 \t Test Err: 55.0012 \t baseline err: 141.4637\n", - "Epoch 153/1000 \t Train Err: 55.3564 \t Test Err: 55.1625 \t baseline err: 141.4637\n", - "Epoch 154/1000 \t Train Err: 55.4071 \t Test Err: 55.5768 \t baseline err: 141.4637\n", - "Epoch 155/1000 \t Train Err: 55.1633 \t Test Err: 55.3753 \t baseline err: 141.4637\n", - "Epoch 156/1000 \t Train Err: 45.2929 \t Test Err: 37.6036 \t baseline err: 141.4637\n", - "Epoch 157/1000 \t Train Err: 36.1442 \t Test Err: 35.6718 \t baseline err: 141.4637\n", - "Epoch 158/1000 \t Train Err: 35.5764 \t Test Err: 35.4595 \t baseline err: 141.4637\n", - "Epoch 159/1000 \t Train Err: 35.3944 \t Test Err: 35.3251 \t baseline err: 141.4637\n" + "Epoch 48/1000 \t Train Err: 81.3666 \t Test Err: 82.1752\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.38it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 49/1000 \t Train Err: 81.1630 \t Test Err: 82.7047\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 50/1000 \t Train Err: 81.3882 \t Test Err: 85.8777\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.38it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 51/1000 \t Train Err: 81.4415 \t Test Err: 83.4058\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.38it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 52/1000 \t Train Err: 81.2446 \t Test Err: 82.6877\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.33it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 53/1000 \t Train Err: 81.3113 \t Test Err: 82.0156\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 54/1000 \t Train Err: 81.3483 \t Test Err: 81.1088\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.33it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 55/1000 \t Train Err: 81.3773 \t Test Err: 81.1178\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.35it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 56/1000 \t Train Err: 81.0823 \t Test Err: 83.9259\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.34it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 57/1000 \t Train Err: 81.6416 \t Test Err: 81.8139\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.34it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 58/1000 \t Train Err: 81.9228 \t Test Err: 81.7897\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████| 32/32 [00:09<00:00, 3.33it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 59/1000 \t Train Err: 81.3041 \t Test Err: 79.5053\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 25%|███████████ | 8/32 [00:02<00:07, 3.32it/s]" ] } ], @@ -713,11 +1319,12 @@ "for epoch in range(NEPOCHS):\n", " model.train()\n", " train_loss = 0\n", - " for batch_src, batch_labels, batch_padding_mask in train_loader:\n", + " for i in tqdm(range(BPE)):\n", + " batch_src, batch_labels, batch_padding_mask = mkbatch(BSZ)\n", " optimizer.zero_grad()\n", " output = model(batch_src, batch_padding_mask)\n", " loss = criterion(output.squeeze(1), batch_labels)\n", - " train_loss += loss.item()/len(train_loader)\n", + " train_loss += loss.item() / BPEREPOCH\n", " loss.backward()\n", " optimizer.step()\n", "\n", @@ -725,12 +1332,12 @@ " \n", " test_err.append(test_loss)\n", " train_err.append(train_loss)\n", + " print(f\"Epoch {epoch + 1}/{NEPOCHS} \\t Train Err: {train_loss:.4f} \\t Test Err: {test_loss:.4f}\")\n", " ax.plot(train_err, label='Train', color='blue')\n", " ax.plot(test_err, label='Test', color='red')\n", " ax.set_xlabel('Epochs')\n", " ax.set_ylabel('MSE')\n", " fig.canvas.draw()\n", - " print(f\"Epoch {epoch + 1}/{NEPOCHS} \\t Train Err: {train_loss:.4f} \\t Test Err: {test_loss:.4f} \\t baseline err: {baseline_error:.4f}\")\n", "\n", " if epoch % 100 == 99:\n", " torch.save(model.state_dict(), f\"model_weights_{epoch}.pth\")" @@ -841,21 +1448,9 @@ "outputs": [], "source": [ "N_TUNE_EPOCHS = 100\n", - "TUNE_BSZ = 1024\n", "TUNE_LR = 0.003\n", "TUNE_WD = 0.002\n", "\n", - "with open(\"tune_data.pkl\", \"rb\") as f:\n", - " pickled_tune_stuff = pickle.load(f)\n", - "\n", - "tune_data = pickled_tune_stuff[\"data\"].to(device)\n", - "tune_label = pickled_tune_stuff[\"labels\"].to(device)\n", - "tune_padding_mask = (tune_data == PAD_TOKEN).bool().to(device)\n", - "tune_dataset = TensorDataset(tune_data, tune_label, tune_padding_mask)\n", - "tune_train_dataset, tune_test_dataset = torch.utils.data.random_split(tune_dataset, [.8, .2])\n", - "tune_train_loader = DataLoader(tune_train_dataset, batch_size=TUNE_BSZ, shuffle=True)\n", - "tune_test_loader = DataLoader(tune_test_dataset, batch_size=TUNE_BSZ, shuffle=True)\n", - "\n", "tune_criterion = nn.MSELoss()\n", "tune_optimizer = torch.optim.Adam(model.parameters(), lr=TUNE_LR, weight_decay=TUNE_WD)" ] @@ -867,11 +1462,29 @@ "metadata": {}, "outputs": [], "source": [ + "def tuneevaluate():\n", + " model.eval()\n", + " test_loss = 0\n", + " with torch.no_grad():\n", + " batch_src, batch_labels, batch_padding_mask = mktunebatch(BSZ)\n", + " output = model(batch_src, batch_padding_mask)\n", + " loss = criterion(output.squeeze(1), batch_labels)\n", + " return loss.item()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "execution_state": "running", + "metadata": {}, + "outputs": [], + "source": [ "# This has to be in a separate cell for some weird event loop reasons\n", "%matplotlib widget\n", "fig,ax = plt.subplots()\n", "fig.suptitle('MSE vs Epochs')\n", - "plt.show()" + "plt.show()\n", + "%matplotlib inline" ] }, { @@ -881,40 +1494,33 @@ "metadata": {}, "outputs": [], "source": [ - "ax.clear()\n", - "\n", "tune_train_err = []\n", "tune_test_err = []\n", "\n", "for epoch in range(N_TUNE_EPOCHS):\n", " model.train()\n", - " tune_train_loss = 0\n", - " for batch_src, batch_labels, batch_padding_mask in tune_train_loader:\n", + " train_loss = 0\n", + " for i in tqdm(range(BPE)):\n", + " batch_src, batch_labels, batch_padding_mask = mktunebatch(BSZ)\n", " optimizer.zero_grad()\n", " output = model(batch_src, batch_padding_mask)\n", " loss = criterion(output.squeeze(1), batch_labels)\n", - " tune_train_loss += loss.item()/len(tune_train_loader)\n", + " train_loss += loss.item()/BPE\n", " loss.backward()\n", " optimizer.step()\n", "\n", - " model.eval()\n", - " tune_test_loss = 0\n", - " with torch.no_grad():\n", - " for batch_src, batch_labels, batch_padding_mask in tune_test_loader:\n", - " output = model(batch_src, batch_padding_mask)\n", - " loss = criterion(output.squeeze(1), batch_labels)\n", - " tune_test_loss += loss.item()/len(tune_test_loader)\n", - " \n", - " tune_test_err.append(tune_test_loss)\n", - " tune_train_err.append(tune_train_loss)\n", + " test_loss = tuneevaluate()\n", + " \n", + " tune_test_err.append(test_loss)\n", + " tune_train_err.append(train_loss)\n", " ax.plot(tune_train_err, label='Train', color='blue')\n", " ax.plot(tune_test_err, label='Test', color='red')\n", " ax.set_xlabel('Epochs')\n", " ax.set_ylabel('MSE')\n", " fig.canvas.draw()\n", - " print(f\"Epoch {epoch + 1}/{NEPOCHS} \\t Train Err: {train_loss:.4f} \\t Test Err: {test_loss:.4f} \\t baseline err: {baseline_error:.4f}\")\n", + " print(f\"Epoch {epoch + 1}/{NEPOCHS} \\t Train Err: {train_loss:.4f} \\t Test Err: {test_loss:.4f}\")\n", "\n", - " if epoch % 100 == 9:\n", + " if epoch % 10 == 9:\n", " torch.save(model.state_dict(), f\"tune_model_weights_{epoch}.pth\")" ] }, |