path: root/transformer_shortest_paths.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LPphBnKR-aWF"
   },
   "source": [
    "# Step 0: Imports"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Question: \n",
    "\n",
    "- Do the attention heads learn to attend to the same positional encodings\n",
    "- do interp -- what is it doing? can we figure out?\n",
    "- update: I think we should do interp once it's bigger. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ge5QvElvhCOw",
    "outputId": "c7cdaefa-d6dc-44ad-c258-e4fb2aca97a5"
   },
   "outputs": [],
   "source": [
    "# using tqdm.auto glitches out collaborative editing\n",
    "from tqdm import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from math import sqrt\n",
    "from collections import deque\n",
    "import os\n",
    "import random\n",
    "\n",
    "torch.manual_seed(42)\n",
    "random.seed(42)\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "assert device.type == 'cuda', \"CUDA is not available. Please check your GPU setup.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "lylOX2POPwFL"
   },
   "outputs": [],
   "source": [
    "# VTXS numbers here are inclusive\n",
    "MIN_VTXS = 3\n",
    "MAX_VTXS = 8\n",
    "MAX_TUNE_VTXS = 8 # 15\n",
    "AVG_DEG = 2\n",
    "SEQ_LEN = MAX_VTXS + 1 # means 32 edges, final token is the target vertex\n",
    "PAD_TOKEN = 0\n",
    "# vertices are labelled 1,2,...,63\n",
    "# we also have a padding token which is 0."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gKt-yIpDebF1"
   },
   "source": [
    "# Step 1: Generate synthetic data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1IbzGIWseK3E",
    "outputId": "a3cbc233-358c-4e17-ea6e-f4e9349d886b"
   },
   "outputs": [],
   "source": [
    "# original task data\n",
    "# the data will be edge lists\n",
    "# like this: [1 3 1 5 2 4 0 0 0 0 2]\n",
    "# this represents edges (1,3), (1,5) (2,4)\n",
    "# (the zeros are just padding tokens)\n",
    "# the final 2 means which vertex we're going to \n",
    "\n",
    "# the label is the shortest distance from vtx 1 to vtx 2\n",
    "# or \"number of vertices\" if no path exists\n",
    "\n",
    "def random_graph(n):\n",
    "    edge_list = []\n",
    "    adjacencies = [set() for _ in range(n+1)]\n",
    "    indices = [random.randint(1, n) for _ in range(AVG_DEG * n)]\n",
    "    for i in range(0, len(indices), 2):\n",
    "        u = indices[i]\n",
    "        v = indices[i + 1]\n",
    "        if u != v:\n",
    "            edge_list += [min(u,v),max(u,v)]\n",
    "            adjacencies[u].add(v)\n",
    "            adjacencies[v].add(u)\n",
    "    edge_list += [PAD_TOKEN]*(2*SEQ_LEN-1-len(edge_list))\n",
    "    return edge_list, adjacencies\n",
    "\n",
    "\"\"\"\n",
    "input: G, represented as an adjacency list\n",
    "output: [number of vertices]+[d(1,i) for i in range(n)] if target=None\n",
    "if target is set to some value, then we instead just output that specific distance\n",
    "\"\"\"\n",
    "def SSSP(n, G, target=2):\n",
    "    dist = [n for _ in G]\n",
    "    dist[1] = 0\n",
    "    frontier = deque()\n",
    "    frontier.append(1)\n",
    "    while len(frontier) > 0:\n",
    "        vtx = frontier.popleft()\n",
    "        for x in G[vtx]:\n",
    "            if dist[x] == n:\n",
    "                dist[x] = 1 + dist[vtx]\n",
    "                frontier.append(x)\n",
    "                if x == target:\n",
    "                    return dist[target]\n",
    "    if target is not None:\n",
    "        return dist[target]\n",
    "    else:\n",
    "        return dist\n",
    "\n",
    "def mkbatch(size):\n",
    "    graphs1 = []\n",
    "    distance1 = []\n",
    "    \n",
    "    for i in range(size):\n",
    "        n = random.randint(MIN_VTXS, MAX_VTXS)\n",
    "        edge_list, adj_list = random_graph(n)\n",
    "        dist = SSSP(n, adj_list)\n",
    "        edge_list[-1] = 2 # target token\n",
    "        graphs1.append(edge_list)\n",
    "        distance1.append(dist)\n",
    "    \n",
    "    data = torch.tensor(graphs1, device=device)\n",
    "    labels = torch.tensor(distance1, dtype=torch.bfloat16, device=device)\n",
    "    padding = data == PAD_TOKEN\n",
    "    return data, labels, padding\n",
    "\n",
    "def vertices_on_shortest_12_path(n, G, target=2):\n",
    "    dist = [n for _ in G]\n",
    "    parent = [-1 for _ in G]\n",
    "    dist[1] = 0\n",
    "    frontier = deque()\n",
    "    frontier.append(1)\n",
    "    while len(frontier) > 0:\n",
    "        vtx = frontier.popleft()\n",
    "        for x in G[vtx]:\n",
    "            if dist[x] == n:\n",
    "                parent[x] = vtx\n",
    "                dist[x] = 1 + dist[vtx]\n",
    "                frontier.append(x)\n",
    "                if x == target:\n",
    "                    path = [x]\n",
    "                    while parent[x] != -1:\n",
    "                        x = parent[x]\n",
    "                        path.append(x)\n",
    "                    return list(reversed(path))\n",
    "    return []\n",
    "\n",
    "def mktunebatch(size):\n",
    "    graphs = []\n",
    "    distance = []\n",
    "    \n",
    "    for i in range(size):\n",
    "        n = random.randint(MIN_VTXS, MAX_TUNE_VTXS)\n",
    "        while True:\n",
    "            edge_list, adj_list = random_graph(n)\n",
    "            path = vertices_on_shortest_12_path(n, adj_list)\n",
    "            if len(path) > 1:\n",
    "                target_vtx_idx = random.randrange(1, len(path))\n",
    "                target_vtx = path[target_vtx_idx]\n",
    "                edge_list[-1] = target_vtx\n",
    "                graphs.append(edge_list)\n",
    "                distance.append(target_vtx_idx)\n",
    "                break\n",
    "    \n",
    "    data = torch.tensor(graphs, device=device)\n",
    "    labels = torch.tensor(distance, dtype=torch.bfloat16, device=device)\n",
    "    padding = data == PAD_TOKEN\n",
    "    return data, labels, padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[1, 2, 4, 5, 3, 4, 1, 7, 1, 2, 1, 4, 0, 0, 0, 0, 2],\n",
       "         [4, 7, 5, 8, 1, 3, 6, 7, 3, 5, 4, 6, 2, 7, 0, 0, 2],\n",
       "         [3, 5, 1, 3, 4, 5, 1, 4, 1, 5, 0, 0, 0, 0, 0, 0, 2],\n",
       "         [3, 5, 2, 5, 2, 3, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2],\n",
       "         [2, 3, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], device='cuda:0'),\n",
       " tensor([1., 8., 5., 1., 1.], device='cuda:0', dtype=torch.bfloat16),\n",
       " tensor([[False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False,  True,  True,  True,  True, False],\n",
       "         [False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False,  True,  True, False],\n",
       "         [False, False, False, False, False, False, False, False, False, False,\n",
       "           True,  True,  True,  True,  True,  True, False],\n",
       "         [False, False, False, False, False, False, False, False,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True, False],\n",
       "         [False, False, False, False,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True, False]], device='cuda:0'))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mkbatch(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([683.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0., 427.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0., 256.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0., 161.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0., 140.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0., 125.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0., 139.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0., 117.]),\n",
       " array([1.      , 1.109375, 1.21875 , 1.328125, 1.4375  , 1.546875,\n",
       "        1.65625 , 1.765625, 1.875   , 1.984375, 2.09375 , 2.203125,\n",
       "        2.3125  , 2.421875, 2.53125 , 2.640625, 2.75    , 2.859375,\n",
       "        2.96875 , 3.078125, 3.1875  , 3.296875, 3.40625 , 3.515625,\n",
       "        3.625   , 3.734375, 3.84375 , 3.953125, 4.0625  , 4.171875,\n",
       "        4.28125 , 4.390625, 4.5     , 4.609375, 4.71875 , 4.828125,\n",
       "        4.9375  , 5.046875, 5.15625 , 5.265625, 5.375   , 5.484375,\n",
       "        5.59375 , 5.703125, 5.8125  , 5.921875, 6.03125 , 6.140625,\n",
       "        6.25    , 6.359375, 6.46875 , 6.578125, 6.6875  , 6.796875,\n",
       "        6.90625 , 7.015625, 7.125   , 7.234375, 7.34375 , 7.453125,\n",
       "        7.5625  , 7.671875, 7.78125 , 7.890625, 8.      ]),\n",
       " <BarContainer object of 64 artists>)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAltklEQVR4nO3df1DU953H8Rc/BBXZJXCyK6egadMgRvNDU9iY/opUSqmTjCSNGWpJ9OKMs5ooF6v0jCYmEevdxdQ7f8SMB94knBfvTltJ1SC56LWuiqTeoLbEtEmgwYXMWXajHRaFvT96fK8btckq7H6A52PmO5P9fj/Lvj+MrU+X3SUmGAwGBQAAYJDYaA8AAADwaQQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOPER3uA69HT06PW1lYlJycrJiYm2uMAAIDPIRgM6pNPPlFGRoZiY//8cyQDMlBaW1s1bty4aI8BAACuQ0tLi8aOHftn1wzIQElOTpb0xw3abLYoTwMAAD4Pv9+vcePGWX+P/zkDMlB6f6xjs9kIFAAABpjP8/IMXiQLAACME1agjB8/XjExMVccbrdbktTZ2Sm32620tDSNGjVKxcXFamtrC/kazc3NKioq0siRI5Wenq5ly5bp8uXLfbcjAAAw4IUVKPX19Tp37px11NbWSpIeeughSdLSpUu1d+9e7dq1S4cOHVJra6tmz55t3b+7u1tFRUXq6urSkSNHtGPHDlVVVWnVqlV9uCUAADDQxQSDweD13nnJkiWqqanR2bNn5ff7NXr0aFVXV+vBBx+UJP3617/WxIkT5fF4lJeXp3379uk73/mOWltb5XA4JElbt27V8uXL9fHHHyshIeFzPa7f75fdbpfP5+M1KAAADBDh/P193a9B6erq0quvvqp58+YpJiZGDQ0NunTpkvLz86012dnZyszMlMfjkSR5PB5NnjzZihNJKigokN/v1+nTp693FAAAMMhc97t49uzZo46ODj366KOSJK/Xq4SEBKWkpISsczgc8nq91po/jZPe673XriUQCCgQCFi3/X7/9Y4NAAAGgOt+BmX79u0qLCxURkZGX85zVRUVFbLb7dbBh7QBADC4XVegfPjhhzp48KD+6q/+yjrndDrV1dWljo6OkLVtbW1yOp3Wmk+/q6f3du+aqykvL5fP57OOlpaW6xkbAAAMENcVKJWVlUpPT1dRUZF1burUqRo2bJjq6uqsc01NTWpubpbL5ZIkuVwuNTY2qr293VpTW1srm82mnJycaz5eYmKi9aFsfDgbAACDX9ivQenp6VFlZaVKS0sVH///d7fb7Zo/f77KysqUmpoqm82mxYsXy+VyKS8vT5I0c+ZM5eTkaO7cuVq/fr28Xq9Wrlwpt9utxMTEvtsVAAAY0MIOlIMHD6q5uVnz5s274tqGDRsUGxur4uJiBQIBFRQUaPPmzdb1uLg41dTUaOHChXK5XEpKSlJpaanWrFlzY7sAAACDyg19Dkq08DkoAAAMPBH5HBQAAID+QqAAAADjXPcHtQ1m41e8cc1rH6wruuY1AADQN3gGBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxgk7UD766CN973vfU1pamkaMGKHJkyfrxIkT1vVgMKhVq1ZpzJgxGjFihPLz83X27NmQr3H+/HmVlJTIZrMpJSVF8+fP14ULF258NwAAYFAIK1B+//vfa/r06Ro2bJj27dunM2fO6O///u910003WWvWr1+vjRs3auvWrTp27JiSkpJUUFCgzs5Oa01JSYlOnz6t2tpa1dTU6PDhw1qwYEHf7QoAAAxoMcFgMPh5F69YsUK/+MUv9F//9V9XvR4MBpWRkaG//uu/1lNPPSVJ8vl8cjgcqqqq0pw5c/SrX/1KOTk5qq+v17Rp0yRJ+/fv17e//W397ne/U0ZGxmfO4ff7Zbfb5fP5ZLPZPu/4n9v4FW9c89oH64r6/PEAABgKwvn7O6xnUH76059q2rRpeuihh5Senq4777xTr7zyinX9/fffl9frVX5+vnXObrcrNzdXHo9HkuTxeJSSkmLFiSTl5+crNjZWx44du+rjBgIB+f3+kAMAAAxeYQXKb3/7W23ZskW33HKLDhw4oIULF+qJJ57Qjh07JEler1eS5HA4Qu7ncDisa16vV+np6SHX4+PjlZqaaq35tIqKCtntdusYN25cOGMDAIABJqxA6enp0V133aW1a9fqzjvv1IIFC/T4449r69at/TWfJKm8vFw+n886Wlpa+vXxAABAdIUVKGPGjFFOTk7IuYkTJ6q5uVmS5HQ6JUltbW0ha9ra2qxrTqdT7e3tIdcvX76s8+fPW2s+LTExUTabLeQAAACDV1iBMn36dDU1NYWce/fdd5WVlSVJmjBhgpxOp+rq6qzrfr9fx44dk8vlkiS5XC51dHSooaHBWvPWW2+pp6dHubm5170RAAAweMSHs3jp0qW65557tHbtWn33u9/V8ePHtW3bNm3btk2SFBMToyVLluj555/XLbfcogkTJujpp59WRkaGHnjgAUl/fMblW9/6lvWjoUuXLmnRokWaM2fO53oHDwAAGPzCCpS7775bu3fvVnl5udasWaMJEybopZdeUklJibXmBz/4gS5evKgFCxaoo6ND9957r/bv36/hw4dba1577TUtWrRIM2bMUGxsrIqLi7Vx48a+2xUAABjQwvocFFPwOSgAAAw8/fY5KAAAAJFAoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAME5YgfLMM88oJiYm5MjOzraud3Z2yu12Ky0tTaNGjVJxcbHa2tpCvkZzc7OKioo0cuRIpaena9myZbp8+XLf7AYAAAwK8eHeYdKkSTp48OD/f4H4//8SS5cu1RtvvKFdu3bJbrdr0aJFmj17tn7xi19Ikrq7u1VUVCSn06kjR47o3Llz+v73v69hw4Zp7dq1fbAdAAAwGIQdKPHx8XI6nVec9/l82r59u6qrq3XfffdJkiorKzVx4kQdPXpUeXl5evPNN3XmzBkdPHhQDodDd9xxh5577jktX75czzzzjBISEm58RwAAYMAL+zUoZ8+eVUZGhm6++WaVlJSoublZktTQ0KBLly4pPz/fWpudna3MzEx5PB5Jksfj0eTJk+VwOKw1BQUF8vv9On369I3uBQAADBJhPYOSm5urqqoq3XrrrTp37pyeffZZfeUrX9GpU6fk9XqVkJCglJSUkPs4HA55vV5JktfrDYmT3uu9164lEAgoEAhYt/1+fzhjAwCAASasQCksLLT+e8qUKcrNzVVWVpZef/11jRgxos+H61VRUaFnn322374+AAAwyw29zTglJUVf+tKX9N5778npdKqrq0sdHR0ha9ra2qzXrDidzive1dN7+2qva+lVXl4un89nHS0tLTcyNgAAMNwNBcqFCxf0m9/8RmPGjNHUqVM1bNgw1dXVWdebmprU3Nwsl8slSXK5XGpsbFR7e7u1pra2VjabTTk5Odd8nMTERNlstpADAAAMXmH9iOepp57SrFmzlJWVpdbWVq1evVpxcXF65JFHZLfbNX/+fJWVlSk1NVU2m02LFy+Wy+VSXl6eJGnmzJnKycnR3LlztX79enm9Xq1cuVJut1uJiYn9skEAADDwhBUov/vd7/TII4/of/7nfzR69Gjde++9Onr0qEaPHi1J2rBhg2JjY1VcXKxAIKCCggJt3rzZun9cXJxqamq0cOFCuVwuJSUlqbS0VGvWrOnbXQEAgAEtJhgMBqM9RLj8fr/sdrt8Pl+//Lhn/Io3rnntg3VFff54AAAMBeH8/c3v4gEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgnPhoDwCzjF/xxjWvfbCuKIKTAACGMp5BAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMa5oUBZt26dYmJitGTJEutcZ2en3G630tLSNGrUKBUXF6utrS3kfs3NzSoqKtLIkSOVnp6uZcuW6fLlyzcyCgAAGESuO1Dq6+v18ssva8qUKSHnly5dqr1792rXrl06dOiQWltbNXv2bOt6d3e3ioqK1NXVpSNHjmjHjh2qqqrSqlWrrn8XAABgULmuQLlw4YJKSkr0yiuv6KabbrLO+3w+bd++XS+++KLuu+8+TZ06VZWVlTpy5IiOHj0qSXrzzTd15swZvfrqq7rjjjtUWFio5557Tps2bVJXV1ff7AoAAAxo1xUobrdbRUVFys/PDznf0NCgS5cuhZzPzs5WZmamPB6PJMnj8Wjy5MlyOBzWmoKCAvn9fp0+ffqqjxcIBOT3+0MOAAAweIX924x37typd955R/X19Vdc83q9SkhIUEpKSsh5h8Mhr9drrfnTOOm93nvtaioqKvTss8+GOyoAABigwnoGpaWlRU8++aRee+01DR8+vL9mukJ5ebl8Pp91tLS0ROyxAQBA5IUVKA0NDWpvb9ddd92l+Ph4xcfH69ChQ9q4caPi4+PlcDjU1dWljo6OkPu1tbXJ6XRKkpxO5xXv6um93bvm0xITE2Wz2UIOAAAweIUVKDNmzFBjY6NOnjxpHdOmTVNJSYn138OGDVNdXZ11n6amJjU3N8vlckmSXC6XGhsb1d7ebq2pra2VzWZTTk5OH20LAAAMZGG9BiU5OVm33XZbyLmkpCSlpaVZ5+fPn6+ysjKlpqbKZrNp8eLFcrlcysvLkyTNnDlTOTk5mjt3rtavXy+v16uVK1fK7XYrMTGxj7YFAAAGsrBfJPtZNmzYoNjYWBUXFysQCKigoECbN2+2rsfFxammpkYLFy6Uy+VSUlKSSktLtWbNmr4eBQAADFA3HChvv/12yO3hw4dr06ZN2rRp0zXvk5WVpZ/97Gc3+tAAAGCQ4nfxAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACME1agbNmyRVOmTJHNZpPNZpPL5dK+ffus652dnXK73UpLS9OoUaNUXFystra2kK/R3NysoqIijRw5Uunp6Vq2bJkuX77cN7sBAACDQliBMnbsWK1bt04NDQ06ceKE7rvvPt1///06ffq0JGnp0qXau3evdu3apUOHDqm1tVWzZ8+27t/d3a2ioiJ1dXXpyJEj2rFjh6qqqrRq1aq+3RUAABjQYoLBYPBGvkBqaqr+9m//Vg8++KBGjx6t6upqPfjgg5KkX//615o4caI8Ho/y8vK0b98+fec731Fra6scDockaevWrVq+fLk+/vhjJSQkfK7H9Pv9stvt8vl8stlsNzL+VY1f8cY1r32wrqjPH88kQ3nvAID+Fc7f39f9GpTu7m7t3LlTFy9elMvlUkNDgy5duqT8/HxrTXZ2tjIzM+XxeCRJHo9HkydPtuJEkgoKCuT3+61nYQAAAOLDvUNjY6NcLpc6Ozs1atQo7d69Wzk5OTp58qQSEhKUkpISst7hcMjr9UqSvF5vSJz0Xu+9di2BQECBQMC67ff7wx0bAAAMIGE/g3Lrrbfq5MmTOnbsmBYuXKjS0lKdOXOmP2azVFRUyG63W8e4ceP69fEAAEB0hR0oCQkJ+uIXv6ipU6eqoqJCt99+u3784x/L6XSqq6tLHR0dIevb2trkdDolSU6n84p39fTe7l1zNeXl5fL5fNbR0tIS7tgAAGAAueHPQenp6VEgENDUqVM1bNgw1dXVWdeamprU3Nwsl8slSXK5XGpsbFR7e7u1pra2VjabTTk5Odd8jMTEROutzb0HAAAYvMJ6DUp5ebkKCwuVmZmpTz75RNXV1Xr77bd14MAB2e12zZ8/X2VlZUpNTZXNZtPixYvlcrmUl5cnSZo5c6ZycnI0d+5crV+/Xl6vVytXrpTb7VZiYmK/bBAAAAw8YQVKe3u7vv/97+vcuXOy2+2aMmWKDhw4oG9+85uSpA0bNig2NlbFxcUKBAIqKCjQ5s2brfvHxcWppqZGCxculMvlUlJSkkpLS7VmzZq+3RUAABjQbvhzUKKBz0HpP0N57wCA/hWRz0EBAADoLwQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjxEd7AMAU41e88Wevf7CuKEKTAAB4BgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGCcsAKloqJCd999t5KTk5Wenq4HHnhATU1NIWs6OzvldruVlpamUaNGqbi4WG1tbSFrmpubVVRUpJEjRyo9PV3Lli3T5cuXb3w3AABgUAgrUA4dOiS3262jR4+qtrZWly5d0syZM3Xx4kVrzdKlS7V3717t2rVLhw4dUmtrq2bPnm1d7+7uVlFRkbq6unTkyBHt2LFDVVVVWrVqVd/tCgAADGjx4Szev39/yO2qqiqlp6eroaFBX/3qV+Xz+bR9+3ZVV1frvvvukyRVVlZq4sSJOnr0qPLy8vTmm2/qzJkzOnjwoBwOh+644w4999xzWr58uZ555hklJCT03e4AAMCAdEOvQfH5fJKk1NRUSVJDQ4MuXbqk/Px8a012drYyMzPl8XgkSR6PR5MnT5bD4bDWFBQUyO/36/Tp01d9nEAgIL/fH3IAAIDB67oDpaenR0uWLNH06dN12223SZK8Xq8SEhKUkpISstbhcMjr9Vpr/jROeq/3XruaiooK2e126xg3btz1jg0AAAaA6w4Ut9utU6dOaefOnX05z1WVl5fL5/NZR0tLS78/JgAAiJ6wXoPSa9GiRaqpqdHhw4c1duxY67zT6VRXV5c6OjpCnkVpa2uT0+m01hw/fjzk6/W+y6d3zaclJiYqMTHxekYFAAADUFjPoASDQS1atEi7d+/WW2+9pQkTJoRcnzp1qoYNG6a6ujrrXFNTk5qbm+VyuSRJLpdLjY2Nam9vt9bU1tbKZrMpJyfnRvYCAAAGibCeQXG73aqurtZPfvITJScnW68ZsdvtGjFihOx2u+bPn6+ysjKlpqbKZrNp8eLFcrlcysvLkyTNnDlTOTk5mjt3rtavXy+v16uVK1fK7XbzLAkAAJAUZqBs2bJFkvT1r3895HxlZaUeffRRSdKGDRsUGxur4uJiBQIBFRQUaPPmzdbauLg41dTUaOHChXK5XEpKSlJpaanWrFlzYzsBAACDRliBEgwGP3PN8OHDtWnTJm3atOmaa7KysvSzn/0snIcGAABDCL+LBwAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGCes32YMYPAav+KNa177YF1RBCcBAJ5BAQAABiJQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHH4qHsAQx4f8w+Yh0ABAAxZxKm5+BEPAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIzD24wBYIjjrbYwEc+gAAAA4xAoAADAOAQKAAAwDq9BAQBgiDL59Uc8gwIAAIxDoAAAAOMQKAAAwDgECgAAME7YgXL48GHNmjVLGRkZiomJ0Z49e0KuB4NBrVq1SmPGjNGIESOUn5+vs2fPhqw5f/68SkpKZLPZlJKSovnz5+vChQs3tBEAADB4hB0oFy9e1O23365NmzZd9fr69eu1ceNGbd26VceOHVNSUpIKCgrU2dlprSkpKdHp06dVW1urmpoaHT58WAsWLLj+XQAAgEEl7LcZFxYWqrCw8KrXgsGgXnrpJa1cuVL333+/JOmf//mf5XA4tGfPHs2ZM0e/+tWvtH//ftXX12vatGmSpH/4h3/Qt7/9bf3d3/2dMjIybmA7AABgMOjT16C8//778nq9ys/Pt87Z7Xbl5ubK4/FIkjwej1JSUqw4kaT8/HzFxsbq2LFjV/26gUBAfr8/5AAAAINXnwaK1+uVJDkcjpDzDofDuub1epWenh5yPT4+XqmpqdaaT6uoqJDdbreOcePG9eXYAADAMAPiXTzl5eXy+XzW0dLSEu2RAABAP+rTQHE6nZKktra2kPNtbW3WNafTqfb29pDrly9f1vnz5601n5aYmCibzRZyAACAwatPA2XChAlyOp2qq6uzzvn9fh07dkwul0uS5HK51NHRoYaGBmvNW2+9pZ6eHuXm5vblOAAAYIAK+108Fy5c0HvvvWfdfv/993Xy5EmlpqYqMzNTS5Ys0fPPP69bbrlFEyZM0NNPP62MjAw98MADkqSJEyfqW9/6lh5//HFt3bpVly5d0qJFizRnzhzewQMAACRdR6CcOHFC3/jGN6zbZWVlkqTS0lJVVVXpBz/4gS5evKgFCxaoo6ND9957r/bv36/hw4db93nttde0aNEizZgxQ7GxsSouLtbGjRv7YDsAAGAwCDtQvv71rysYDF7zekxMjNasWaM1a9Zcc01qaqqqq6vDfWgAADBEDIh38QAAgKGFQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYJyoBsqmTZs0fvx4DR8+XLm5uTp+/Hg0xwEAAIaIWqD867/+q8rKyrR69Wq98847uv3221VQUKD29vZojQQAAAwRtUB58cUX9fjjj+uxxx5TTk6Otm7dqpEjR+qf/umfojUSAAAwRHw0HrSrq0sNDQ0qLy+3zsXGxio/P18ej+eK9YFAQIFAwLrt8/kkSX6/v1/m6wn84ZrX+usxTcHer20o75+9D25Def9Dee9S5Pff+zWDweBnLw5GwUcffRSUFDxy5EjI+WXLlgW//OUvX7F+9erVQUkcHBwcHBwcg+BoaWn5zFaIyjMo4SovL1dZWZl1u6enR+fPn1daWppiYmL69LH8fr/GjRunlpYW2Wy2Pv3aAwH7H9r7l/geDPX9S3wPhvr+pf77HgSDQX3yySfKyMj4zLVRCZS/+Iu/UFxcnNra2kLOt7W1yel0XrE+MTFRiYmJIedSUlL6c0TZbLYh+wdTYv9Dff8S34Ohvn+J78FQ37/UP98Du93+udZF5UWyCQkJmjp1qurq6qxzPT09qqurk8vlisZIAADAIFH7EU9ZWZlKS0s1bdo0ffnLX9ZLL72kixcv6rHHHovWSAAAwBBRC5SHH35YH3/8sVatWiWv16s77rhD+/fvl8PhiNZIkv7446TVq1df8SOloYL9D+39S3wPhvr+Jb4HQ33/khnfg5hg8PO81wcAACBy+F08AADAOAQKAAAwDoECAACMQ6AAAADjECj/5/Dhw5o1a5YyMjIUExOjPXv2RHukiKqoqNDdd9+t5ORkpaen64EHHlBTU1O0x4qYLVu2aMqUKdaHErlcLu3bty/aY0XNunXrFBMToyVLlkR7lIh55plnFBMTE3JkZ2dHe6yI+uijj/S9731PaWlpGjFihCZPnqwTJ05Ee6yIGT9+/BV/BmJiYuR2u6M9WkR0d3fr6aef1oQJEzRixAh94Qtf0HPPPff5fm9OPxgQH3UfCRcvXtTtt9+uefPmafbs2dEeJ+IOHTokt9utu+++W5cvX9YPf/hDzZw5U2fOnFFSUlK0x+t3Y8eO1bp163TLLbcoGAxqx44duv/++/XLX/5SkyZNivZ4EVVfX6+XX35ZU6ZMifYoETdp0iQdPHjQuh0fP3T+L/L3v/+9pk+frm984xvat2+fRo8erbNnz+qmm26K9mgRU19fr+7ubuv2qVOn9M1vflMPPfRQFKeKnB/96EfasmWLduzYoUmTJunEiRN67LHHZLfb9cQTT0R8nqHzv77PUFhYqMLCwmiPETX79+8PuV1VVaX09HQ1NDToq1/9apSmipxZs2aF3H7hhRe0ZcsWHT16dEgFyoULF1RSUqJXXnlFzz//fLTHibj4+Pir/rqNoeBHP/qRxo0bp8rKSuvchAkTojhR5I0ePTrk9rp16/SFL3xBX/va16I0UWQdOXJE999/v4qKiiT98Rmlf/mXf9Hx48ejMg8/4sFV+Xw+SVJqamqUJ4m87u5u7dy5UxcvXhxyv3rB7XarqKhI+fn50R4lKs6ePauMjAzdfPPNKikpUXNzc7RHipif/vSnmjZtmh566CGlp6frzjvv1CuvvBLtsaKmq6tLr776qubNm9fnv5TWVPfcc4/q6ur07rvvSpL++7//Wz//+c+j9o93nkHBFXp6erRkyRJNnz5dt912W7THiZjGxka5XC51dnZq1KhR2r17t3JycqI9VsTs3LlT77zzjurr66M9SlTk5uaqqqpKt956q86dO6dnn31WX/nKV3Tq1CklJydHe7x+99vf/lZbtmxRWVmZfvjDH6q+vl5PPPGEEhISVFpaGu3xIm7Pnj3q6OjQo48+Gu1RImbFihXy+/3Kzs5WXFycuru79cILL6ikpCQq8xAouILb7dapU6f085//PNqjRNStt96qkydPyufz6d/+7d9UWlqqQ4cODYlIaWlp0ZNPPqna2loNHz482uNExZ/+K3HKlCnKzc1VVlaWXn/9dc2fPz+Kk0VGT0+Ppk2bprVr10qS7rzzTp06dUpbt24dkoGyfft2FRYWKiMjI9qjRMzrr7+u1157TdXV1Zo0aZJOnjypJUuWKCMjIyp/BggUhFi0aJFqamp0+PBhjR07NtrjRFRCQoK++MUvSpKmTp2q+vp6/fjHP9bLL78c5cn6X0NDg9rb23XXXXdZ57q7u3X48GH94z/+owKBgOLi4qI4YeSlpKToS1/6kt57771ojxIRY8aMuSLGJ06cqH//93+P0kTR8+GHH+rgwYP6j//4j2iPElHLli3TihUrNGfOHEnS5MmT9eGHH6qiooJAQfQEg0EtXrxYu3fv1ttvvz3kXhx3NT09PQoEAtEeIyJmzJihxsbGkHOPPfaYsrOztXz58iEXJ9IfXzD8m9/8RnPnzo32KBExffr0Kz5a4N1331VWVlaUJoqeyspKpaenWy8WHSr+8Ic/KDY29KWpcXFx6unpico8BMr/uXDhQsi/lN5//32dPHlSqampyszMjOJkkeF2u1VdXa2f/OQnSk5OltfrlSTZ7XaNGDEiytP1v/LychUWFiozM1OffPKJqqur9fbbb+vAgQPRHi0ikpOTr3i9UVJSktLS0obM65CeeuopzZo1S1lZWWptbdXq1asVFxenRx55JNqjRcTSpUt1zz33aO3atfrud7+r48ePa9u2bdq2bVu0R4uonp4eVVZWqrS0dEi9zVz647sZX3jhBWVmZmrSpEn65S9/qRdffFHz5s2LzkBBBIPBYPA///M/g5KuOEpLS6M9WkRcbe+SgpWVldEeLSLmzZsXzMrKCiYkJARHjx4dnDFjRvDNN9+M9lhR9bWvfS345JNPRnuMiHn44YeDY8aMCSYkJAT/8i//Mvjwww8H33vvvWiPFVF79+4N3nbbbcHExMRgdnZ2cNu2bdEeKeIOHDgQlBRsamqK9igR5/f7g08++WQwMzMzOHz48ODNN98c/Ju/+ZtgIBCIyjwxwWCUPiIOAADgGvgcFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHH+F1w6Qmp2KYLRAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(mkbatch(2048)[1].to(torch.float32).cpu(), bins=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([1.488e+03, 0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00,\n",
       "        0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00,\n",
       "        4.340e+02, 0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00,\n",
       "        0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00,\n",
       "        0.000e+00, 1.030e+02, 0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00,\n",
       "        0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00,\n",
       "        0.000e+00, 0.000e+00, 1.700e+01, 0.000e+00, 0.000e+00, 0.000e+00,\n",
       "        0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00,\n",
       "        0.000e+00, 0.000e+00, 0.000e+00, 5.000e+00, 0.000e+00, 0.000e+00,\n",
       "        0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00,\n",
       "        0.000e+00, 0.000e+00, 0.000e+00, 1.000e+00]),\n",
       " array([1.      , 1.078125, 1.15625 , 1.234375, 1.3125  , 1.390625,\n",
       "        1.46875 , 1.546875, 1.625   , 1.703125, 1.78125 , 1.859375,\n",
       "        1.9375  , 2.015625, 2.09375 , 2.171875, 2.25    , 2.328125,\n",
       "        2.40625 , 2.484375, 2.5625  , 2.640625, 2.71875 , 2.796875,\n",
       "        2.875   , 2.953125, 3.03125 , 3.109375, 3.1875  , 3.265625,\n",
       "        3.34375 , 3.421875, 3.5     , 3.578125, 3.65625 , 3.734375,\n",
       "        3.8125  , 3.890625, 3.96875 , 4.046875, 4.125   , 4.203125,\n",
       "        4.28125 , 4.359375, 4.4375  , 4.515625, 4.59375 , 4.671875,\n",
       "        4.75    , 4.828125, 4.90625 , 4.984375, 5.0625  , 5.140625,\n",
       "        5.21875 , 5.296875, 5.375   , 5.453125, 5.53125 , 5.609375,\n",
       "        5.6875  , 5.765625, 5.84375 , 5.921875, 6.      ]),\n",
       " <BarContainer object of 64 artists>)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAltUlEQVR4nO3df3DU9Z3H8deGJT9EdkNis8ueIebuPCCIqETjinK1ZAiQcsc0raamNGcz5M5LUIwi5FoiWjUYW5V4aVKc1jBTGG1vDqqxRnOgpNUQQrgcEDHiFSVKN/EmZNfEIQSy90eH73QFLNhNNp/1+Zj5znS/n8/u97074+TZzWaxBYPBoAAAAAwSE+kBAAAALhYBAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA49kgPMFpGRkZ07NgxTZ48WTabLdLjAACACxAMBvXJJ5/I4/EoJub877NEbcAcO3ZMqampkR4DAAB8Ad3d3br88svPux61ATN58mRJf3wBHA5HhKcBAAAXIhAIKDU11fo5fj5RGzBnfm3kcDgIGAAADPPnPv7Bh3gBAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAce6QHMNEVa18+79r7G3LHcBIAAL6ceAcGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGCciw6Y5uZmLV26VB6PRzabTdu3bz/v3n/5l3+RzWbT008/HXK+r69PBQUFcjgcSkxMVFFRkQYGBkL27N+/X7fccovi4+OVmpqqqqqqix0VAABEqYsOmMHBQc2ZM0c1NTWfu2/btm3avXu3PB7PWWsFBQXq7OxUU1OTGhoa1NzcrOLiYms9EAho4cKFSktLU3t7u5544gmtX79emzZtuthxAQBAFLJf7B0WL16sxYsXf+6ejz76SCtXrtSrr76q3NzckLVDhw6psbFRbW1tyszMlCQ988wzWrJkiX70ox/J4/Foy5YtOnnypH7+858rNjZWs2bNUkdHh5588smQ0AEAAF9OYf8MzMjIiJYvX67Vq1dr1qxZZ623tLQoMTHRihdJys7OVkxMjFpbW6098+fPV2xsrLUnJydHXV1dOn78+DmvOzQ0pEAgEHIAAIDoFPaAefzxx2W323X33Xefc93n8yklJSXknN1uV1JSknw+n7XH5XKF7Dlz+8yez6qsrJTT6bSO1NTUv/SpAACAcSqsAdPe3q6NGzeqvr5eNpstnA/9Z5WXl8vv91tHd3f3mF4fAACMnbAGzG9/+1v19vZq2rRpstvtstvt+uCDD3TffffpiiuukCS53W719vaG3O/UqVPq6+uT2+229vT09ITsOXP7zJ7PiouLk8PhCDkAAEB0CmvALF++XPv371dHR4d1eDwerV69Wq+++qokyev1qr+/X+3t7db9du7cqZGREWVlZVl7mpubNTw8bO1pamrS9OnTNWXKlHCODAAADHTRf4U0MDCg9957z7p95MgRdXR0KCkpSdOmTVNycnLI/okTJ8rtdmv69OmSpJkzZ2rRokVasWKF6urqNDw8rNLSUuXn51t/cn3HHXfooYceUlFRkdasWaODBw9q48aNeuqpp/6S5woAAKLERQfM3r17deutt1q3y8rKJEmFhYWqr6+/oMfYsmWLSktLtWDBAsXExCgvL0/V1dXWutPp1GuvvaaSkhLNnTtXl112mSoqKvgTagAAIEmyBYPBYKSHGA2BQEBOp1N+vz/sn4e5Yu3L5117f0PuedcAAMDnu9Cf3/xbSAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAONcdMA0Nzdr6dKl8ng8stls2r59u7U2PDysNWvWaPbs2Zo0aZI8Ho+++93v6tixYyGP0dfXp4KCAjkcDiUmJqqoqEgDAwMhe/bv369bbrlF8fHxSk1NVVVV1Rd7hgAAIOpcdMAMDg5qzpw5qqmpOWvt008/1b59+7Ru3Trt27dP//mf/6muri79wz/8Q8i+goICdXZ2qqmpSQ0NDWpublZxcbG1HggEtHDhQqWlpam9vV1PPPGE1q9fr02bNn2BpwgAAKKNLRgMBr/wnW02bdu2TcuWLTvvnra2Nt1www364IMPNG3aNB06dEgZGRlqa2tTZmamJKmxsVFLlizRhx9+KI/Ho9raWn3/+9+Xz+dTbGysJGnt2rXavn273nnnnQuaLRAIyOl0yu/3y+FwfNGneE5XrH35vGvvb8gN67UAAPgyudCf36P+GRi/3y+bzabExERJUktLixITE614kaTs7GzFxMSotbXV2jN//nwrXiQpJydHXV1dOn78+DmvMzQ0pEAgEHIAAIDoNKoBc+LECa1Zs0bf/va3rYry+XxKSUkJ2We325WUlCSfz2ftcblcIXvO3D6z57MqKyvldDqtIzU1NdxPBwAAjBOjFjDDw8O67bbbFAwGVVtbO1qXsZSXl8vv91tHd3f3qF8TAABEhn00HvRMvHzwwQfauXNnyO+w3G63ent7Q/afOnVKfX19crvd1p6enp6QPWdun9nzWXFxcYqLiwvn0wAAAONU2N+BORMvhw8f1n/9138pOTk5ZN3r9aq/v1/t7e3WuZ07d2pkZERZWVnWnubmZg0PD1t7mpqaNH36dE2ZMiXcIwMAAMNcdMAMDAyoo6NDHR0dkqQjR46oo6NDR48e1fDwsL75zW9q79692rJli06fPi2fzyefz6eTJ09KkmbOnKlFixZpxYoV2rNnj958802VlpYqPz9fHo9HknTHHXcoNjZWRUVF6uzs1AsvvKCNGzeqrKwsfM8cAAAY66L/jPqNN97Qrbfeetb5wsJCrV+/Xunp6ee83+uvv66vfvWrkv74RXalpaV66aWXFBMTo7y8PFVXV+vSSy+19u/fv18lJSVqa2vTZZddppUrV2rNmjUXPCd/Rg0AgHku9Of3X/Q9MOMZAQMAgHnGzffAAAAAhBsBAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMM5FB0xzc7OWLl0qj8cjm82m7du3h6wHg0FVVFRo6tSpSkhIUHZ2tg4fPhyyp6+vTwUFBXI4HEpMTFRRUZEGBgZC9uzfv1+33HKL4uPjlZqaqqqqqot/dgAAICpddMAMDg5qzpw5qqmpOed6VVWVqqurVVdXp9bWVk2aNEk5OTk6ceKEtaegoECdnZ1qampSQ0ODmpubVVxcbK0HAgEtXLhQaWlpam9v1xNPPKH169dr06ZNX+ApAgCAaGMLBoPBL3xnm03btm3TsmXLJP3x3RePx6P77rtP999/vyTJ7/fL5XKpvr5e+fn5OnTokDIyMtTW1qbMzExJUmNjo5YsWaIPP/xQHo9HtbW1+v73vy+fz6fY2FhJ0tq1a7V9+3a98847FzRbIBCQ0+mU3++Xw+H4ok/xnK5Y+/J5197fkBvWawEA8GVyoT+/w/oZmCNHjsjn8yk7O9s653Q6lZWVpZaWFklSS0uLEhMTrXiRpOzsbMXExKi1tdXaM3/+fCteJCknJ0ddXV06fvz4Oa89NDSkQCAQcgAAgOgU1oDx+XySJJfLFXLe5XJZaz6fTykpKSHrdrtdSUlJIXvO9Rh/eo3PqqyslNPptI7U1NS//AkBAIBxKWr+Cqm8vFx+v986uru7Iz0SAAAYJWENGLfbLUnq6ekJOd/T02Otud1u9fb2hqyfOnVKfX19IXvO9Rh/eo3PiouLk8PhCDkAAEB0CmvApKeny+12a8eOHda5QCCg1tZWeb1eSZLX61V/f7/a29utPTt37tTIyIiysrKsPc3NzRoeHrb2NDU1afr06ZoyZUo4RwYAAAa66IAZGBhQR0eHOjo6JP3xg7sdHR06evSobDabVq1apUceeUQvvviiDhw4oO9+97vyeDzWXyrNnDlTixYt0ooVK7Rnzx69+eabKi0tVX5+vjwejyTpjjvuUGxsrIqKitTZ2akXXnhBGzduVFlZWdieOAAAMJf9Yu+wd+9e3XrrrdbtM1FRWFio+vp6PfDAAxocHFRxcbH6+/t18803q7GxUfHx8dZ9tmzZotLSUi1YsEAxMTHKy8tTdXW1te50OvXaa6+ppKREc+fO1WWXXaaKioqQ74oBAABfXn/R98CMZ3wPDAAA5onI98AAAACMBQIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAccIeMKdPn9a6deuUnp6uhIQE/c3f/I1++MMfKhgMWnuCwaAqKio0depUJSQkKDs7W4cPHw55nL6+PhUUFMjhcCgxMVFFRUUaGBgI97gAAMBAYQ+Yxx9/XLW1tfr3f/93HTp0SI8//riqqqr0zDPPWHuqqqpUXV2turo6tba2atKkScrJydGJEyesPQUFBers7FRTU5MaGhrU3Nys4uLicI8LAAAMZAv+6VsjYfD1r39dLpdLP/vZz6xzeXl5SkhI0C9+8QsFg0F5PB7dd999uv/++yVJfr9fLpdL9fX1ys/P16FDh5SRkaG2tjZlZmZKkhobG7VkyRJ9+OGH8ng8f3aOQCAgp9Mpv98vh8MRzqeoK9a+fN619zfkhvVaAAB8mVzoz++wvwNz0003aceOHXr33XclSf/zP/+j3/3ud1q8eLEk6ciRI/L5fMrOzrbu43Q6lZWVpZaWFklSS0uLEhMTrXiRpOzsbMXExKi1tfWc1x0aGlIgEAg5AABAdLKH+wHXrl2rQCCgGTNmaMKECTp9+rQeffRRFRQUSJJ8Pp8kyeVyhdzP5XJZaz6fTykpKaGD2u1KSkqy9nxWZWWlHnrooXA/HQAAMA6F/R2YX/7yl9qyZYu2bt2qffv2afPmzfrRj36kzZs3h/tSIcrLy+X3+62ju7t7VK8HAAAiJ+zvwKxevVpr165Vfn6+JGn27Nn64IMPVFlZqcLCQrndbklST0+Ppk6dat2vp6dH11xzjSTJ7Xart7c35HFPnTqlvr4+6/6fFRcXp7i4uHA/HQAAMA6F/R2YTz/9VDExoQ87YcIEjYyMSJLS09Pldru1Y8cOaz0QCKi1tVVer1eS5PV61d/fr/b2dmvPzp07NTIyoqysrHCPDAAADBP2d2CWLl2qRx99VNOmTdOsWbP03//933ryySf1ve99T5Jks9m0atUqPfLII7ryyiuVnp6udevWyePxaNmyZZKkmTNnatGiRVqxYoXq6uo0PDys0tJS5efnX9BfIAEAgOgW9oB55plntG7dOv3rv/6rent75fF49M///M+qqKiw9jzwwAMaHBxUcXGx+vv7dfPNN6uxsVHx8fHWni1btqi0tFQLFixQTEyM8vLyVF1dHe5xAQCAgcL+PTDjBd8DAwCAeSL2PTAAAACjjYABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgnFEJmI8++kjf+c53lJycrISEBM2ePVt79+611oPBoCoqKjR16lQlJCQoOztbhw8fDnmMvr4+FRQUyOFwKDExUUVFRRoYGBiNcQEAgGHCHjDHjx/XvHnzNHHiRL3yyit6++239eMf/1hTpkyx9lRVVam6ulp1dXVqbW3VpEmTlJOToxMnTlh7CgoK1NnZqaamJjU0NKi5uVnFxcXhHhcAABjIFgwGg+F8wLVr1+rNN9/Ub3/723OuB4NBeTwe3Xfffbr//vslSX6/Xy6XS/X19crPz9ehQ4eUkZGhtrY2ZWZmSpIaGxu1ZMkSffjhh/J4PH92jkAgIKfTKb/fL4fDEb4nKOmKtS+fd+39DblhvRYAAF8mF/rzO+zvwLz44ovKzMzUt771LaWkpOjaa6/Vs88+a60fOXJEPp9P2dnZ1jmn06msrCy1tLRIklpaWpSYmGjFiyRlZ2crJiZGra2t57zu0NCQAoFAyAEAAKJT2APm97//vWpra3XllVfq1Vdf1V133aW7775bmzdvliT5fD5JksvlCrmfy+Wy1nw+n1JSUkLW7Xa7kpKSrD2fVVlZKafTaR2pqanhfmoAAGCcCHvAjIyM6LrrrtNjjz2ma6+9VsXFxVqxYoXq6urCfakQ5eXl8vv91tHd3T2q1wMAAJET9oCZOnWqMjIyQs7NnDlTR48elSS53W5JUk9PT8ienp4ea83tdqu3tzdk/dSpU+rr67P2fFZcXJwcDkfIAQAAolPYA2bevHnq6uoKOffuu+8qLS1NkpSeni63260dO3ZY64FAQK2trfJ6vZIkr9er/v5+tbe3W3t27typkZERZWVlhXtkAABgGHu4H/Dee+/VTTfdpMcee0y33Xab9uzZo02bNmnTpk2SJJvNplWrVumRRx7RlVdeqfT0dK1bt04ej0fLli2T9Md3bBYtWmT96ml4eFilpaXKz8+/oL9AAgAA0S3sAXP99ddr27ZtKi8v18MPP6z09HQ9/fTTKigosPY88MADGhwcVHFxsfr7+3XzzTersbFR8fHx1p4tW7aotLRUCxYsUExMjPLy8lRdXR3ucQEAgIHC/j0w4wXfAwMAgHki9j0wAAAAo42AAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABjHHukBAEm6Yu3L5117f0PuGE4CADAB78AAAADjEDAAAMA4ox4wGzZskM1m06pVq6xzJ06cUElJiZKTk3XppZcqLy9PPT09Ifc7evSocnNzdckllyglJUWrV6/WqVOnRntcAABggFENmLa2Nv30pz/V1VdfHXL+3nvv1UsvvaRf/epX2rVrl44dO6ZvfOMb1vrp06eVm5urkydP6q233tLmzZtVX1+vioqK0RwXAAAYYtQCZmBgQAUFBXr22Wc1ZcoU67zf79fPfvYzPfnkk/ra176muXPn6rnnntNbb72l3bt3S5Jee+01vf322/rFL36ha665RosXL9YPf/hD1dTU6OTJk6M1MgAAMMSoBUxJSYlyc3OVnZ0dcr69vV3Dw8Mh52fMmKFp06appaVFktTS0qLZs2fL5XJZe3JychQIBNTZ2TlaIwMAAEOMyp9RP//889q3b5/a2trOWvP5fIqNjVViYmLIeZfLJZ/PZ+3503g5s35m7VyGhoY0NDRk3Q4EAn/JUwAAAONY2N+B6e7u1j333KMtW7YoPj4+3A9/XpWVlXI6ndaRmpo6ZtcGAABjK+wB097ert7eXl133XWy2+2y2+3atWuXqqurZbfb5XK5dPLkSfX394fcr6enR263W5LkdrvP+qukM7fP7Pms8vJy+f1+6+ju7g73UwMAAONE2ANmwYIFOnDggDo6OqwjMzNTBQUF1v+eOHGiduzYYd2nq6tLR48eldfrlSR5vV4dOHBAvb291p6mpiY5HA5lZGSc87pxcXFyOBwhBwAAiE5h/wzM5MmTddVVV4WcmzRpkpKTk63zRUVFKisrU1JSkhwOh1auXCmv16sbb7xRkrRw4UJlZGRo+fLlqqqqks/n0w9+8AOVlJQoLi4u3CMDAADDROTfQnrqqacUExOjvLw8DQ0NKScnRz/5yU+s9QkTJqihoUF33XWXvF6vJk2apMLCQj388MORGBcAAIwzYxIwb7zxRsjt+Ph41dTUqKam5rz3SUtL029+85tRngwAAJiIfwsJAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYJ+wBU1lZqeuvv16TJ09WSkqKli1bpq6urpA9J06cUElJiZKTk3XppZcqLy9PPT09IXuOHj2q3NxcXXLJJUpJSdHq1at16tSpcI8LAAAMFPaA2bVrl0pKSrR79241NTVpeHhYCxcu1ODgoLXn3nvv1UsvvaRf/epX2rVrl44dO6ZvfOMb1vrp06eVm5urkydP6q233tLmzZtVX1+vioqKcI8LAAAMZAsGg8HRvMDHH3+slJQU7dq1S/Pnz5ff79dXvvIVbd26Vd/85jclSe+8845mzpyplpYW3XjjjXrllVf09a9/XceOHZPL5ZIk1dXVac2aNfr4448VGxv7Z68bCATkdDrl9/vlcDjC+pyuWPvyedfe35Ab1mt9WfCaAgCkC//5PeqfgfH7/ZKkpKQkSVJ7e7uGh4eVnZ1t7ZkxY4amTZumlpYWSVJLS4tmz55txYsk5eTkKBAIqLOz85zXGRoaUiAQCDkAAEB0GtWAGRkZ0apVqzRv3jxdddVVkiSfz6fY2FglJiaG7HW5XPL5fNaeP42XM+tn1s6lsrJSTqfTOlJTU8P8bAAAwHgxqgFTUlKigwcP6vnnnx/Ny0iSysvL5ff7raO7u3vUrwkAACLDPloPXFpaqoaGBjU3N+vyyy+3zrvdbp08eVL9/f0h78L09PTI7XZbe/bs2RPyeGf+SunMns+Ki4tTXFxcmJ8FAAAYj8L+DkwwGFRpaam2bdumnTt3Kj09PWR97ty5mjhxonbs2GGd6+rq0tGjR+X1eiVJXq9XBw4cUG9vr7WnqalJDodDGRkZ4R4ZAAAYJuzvwJSUlGjr1q369a9/rcmTJ1ufWXE6nUpISJDT6VRRUZHKysqUlJQkh8OhlStXyuv16sYbb5QkLVy4UBkZGVq+fLmqqqrk8/n0gx/8QCUlJbzLAgAAwh8wtbW1kqSvfvWrIeefe+45/dM//ZMk6amnnlJMTIzy8vI0NDSknJwc/eQnP7H2TpgwQQ0NDbrrrrvk9Xo1adIkFRYW6uGHHw73uAAAwEBhD5gL+VqZ+Ph41dTUqKam5rx70tLS9Jvf/CacowEAgCjBv4UEAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADCOPdIDABh7V6x9+bxr72/IHcNJAOCL4R0YAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMaxR3qAz1NTU6MnnnhCPp9Pc+bM0TPPPKMbbrgh0mMBwAW7Yu3L5117f0PuGE4CRJdx+w7MCy+8oLKyMj344IPat2+f5syZo5ycHPX29kZ6NAAAEGHjNmCefPJJrVixQnfeeacyMjJUV1enSy65RD//+c8jPRoAAIiwcfkrpJMnT6q9vV3l5eXWuZiYGGVnZ6ulpeWc9xkaGtLQ0JB12+/3S5ICgUDY5xsZ+vS8a6NxvS8DXtOxxes9dnitx9ZVD7563rWDD+WM4ST4os78dxEMBj9337gMmP/7v//T6dOn5XK5Qs67XC69884757xPZWWlHnroobPOp6amjsqM5+N8ekwv96XAazq2eL3HDq/12OL1Nssnn3wip9N53vVxGTBfRHl5ucrKyqzbIyMj6uvrU3Jysmw2W9iuEwgElJqaqu7ubjkcjrA9Ls7Gaz02eJ3HBq/z2OB1Hhuj+ToHg0F98skn8ng8n7tvXAbMZZddpgkTJqinpyfkfE9Pj9xu9znvExcXp7i4uJBziYmJozWiHA4H/3GMEV7rscHrPDZ4nccGr/PYGK3X+fPeeTljXH6INzY2VnPnztWOHTuscyMjI9qxY4e8Xm8EJwMAAOPBuHwHRpLKyspUWFiozMxM3XDDDXr66ac1ODioO++8M9KjAQCACBu3AXP77bfr448/VkVFhXw+n6655ho1Njae9cHesRYXF6cHH3zwrF9XIfx4rccGr/PY4HUeG7zOY2M8vM624J/7OyUAAIBxZlx+BgYAAODzEDAAAMA4BAwAADAOAQMAAIxDwFyE5uZmLV26VB6PRzabTdu3b4/0SFGnsrJS119/vSZPnqyUlBQtW7ZMXV1dkR4r6tTW1urqq6+2voTK6/XqlVdeifRYUW/Dhg2y2WxatWpVpEeJOuvXr5fNZgs5ZsyYEemxotJHH32k73znO0pOTlZCQoJmz56tvXv3jvkcBMxFGBwc1Jw5c1RTUxPpUaLWrl27VFJSot27d6upqUnDw8NauHChBgcHIz1aVLn88su1YcMGtbe3a+/evfra176mf/zHf1RnZ2ekR4tabW1t+ulPf6qrr7460qNErVmzZukPf/iDdfzud7+L9EhR5/jx45o3b54mTpyoV155RW+//bZ+/OMfa8qUKWM+y7j9HpjxaPHixVq8eHGkx4hqjY2NIbfr6+uVkpKi9vZ2zZ8/P0JTRZ+lS5eG3H700UdVW1ur3bt3a9asWRGaKnoNDAyooKBAzz77rB555JFIjxO17Hb7ef+5GYTH448/rtTUVD333HPWufT09IjMwjswGNf8fr8kKSkpKcKTRK/Tp0/r+eef1+DgIP9UxygpKSlRbm6usrOzIz1KVDt8+LA8Ho/++q//WgUFBTp69GikR4o6L774ojIzM/Wtb31LKSkpuvbaa/Xss89GZBbegcG4NTIyolWrVmnevHm66qqrIj1O1Dlw4IC8Xq9OnDihSy+9VNu2bVNGRkakx4o6zz//vPbt26e2trZIjxLVsrKyVF9fr+nTp+sPf/iDHnroId1yyy06ePCgJk+eHOnxosbvf/971dbWqqysTP/2b/+mtrY23X333YqNjVVhYeGYzkLAYNwqKSnRwYMH+T32KJk+fbo6Ojrk9/v1H//xHyosLNSuXbuImDDq7u7WPffco6amJsXHx0d6nKj2p7/ev/rqq5WVlaW0tDT98pe/VFFRUQQniy4jIyPKzMzUY489Jkm69tprdfDgQdXV1Y15wPArJIxLpaWlamho0Ouvv67LL7880uNEpdjYWP3t3/6t5s6dq8rKSs2ZM0cbN26M9FhRpb29Xb29vbruuutkt9tlt9u1a9cuVVdXy2636/Tp05EeMWolJibq7/7u7/Tee+9FepSoMnXq1LP+T87MmTMj8us63oHBuBIMBrVy5Upt27ZNb7zxRsQ+HPZlNDIyoqGhoUiPEVUWLFigAwcOhJy78847NWPGDK1Zs0YTJkyI0GTRb2BgQP/7v/+r5cuXR3qUqDJv3ryzvtri3XffVVpa2pjPQsBchIGBgZCaP3LkiDo6OpSUlKRp06ZFcLLoUVJSoq1bt+rXv/61Jk+eLJ/PJ0lyOp1KSEiI8HTRo7y8XIsXL9a0adP0ySefaOvWrXrjjTf06quvRnq0qDJ58uSzPr81adIkJScn87muMLv//vu1dOlSpaWl6dixY3rwwQc1YcIEffvb3470aFHl3nvv1U033aTHHntMt912m/bs2aNNmzZp06ZNYz9MEBfs9ddfD0o66ygsLIz0aFHjXK+vpOBzzz0X6dGiyve+971gWlpaMDY2NviVr3wluGDBguBrr70W6bG+FP7+7/8+eM8990R6jKhz++23B6dOnRqMjY0N/tVf/VXw9ttvD7733nuRHisqvfTSS8GrrroqGBcXF5wxY0Zw06ZNEZnDFgwGg2OfTQAAAF8cH+IFAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAY5/8BbkRDd6BuURgAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(mktunebatch(2048)[1].to(torch.float32).cpu(), bins=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pickle\n",
    "\n",
    "# graphs, labels, _ = mkbatch(3*10**5)\n",
    "\n",
    "# data = {\n",
    "#     \"data\": graphs,\n",
    "#     \"labels\": labels\n",
    "# }\n",
    "\n",
    "# with open('data.pkl', 'wb') as file:\n",
    "#     pickle.dump(data, file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Q3Cg_8UQep8g"
   },
   "source": [
    "# Step 2: Define Transformer Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "tLOWhg_CeWzH"
   },
   "outputs": [],
   "source": [
    "class TransformerModel(nn.Module):\n",
    "    def __init__(self, input_dim, model_dim, output_dim, num_heads, num_layers, seq_len, dropout):\n",
    "        super().__init__()\n",
    "        self.model_dim = model_dim\n",
    "        self.embedding = nn.Embedding(input_dim, model_dim // 2, dtype=torch.bfloat16)\n",
    "        # # seq_len is odd\n",
    "        # self.fancy_encoding = torch.repeat_interleave(torch.rand((1, seq_len // 2 + 1, model_dim // 2), device=device, dtype=torch.bfloat16), 2, dim=1)\n",
    "        # # cut off last element since the target vertex is not repeated\n",
    "        # self.fancy_encoding = self.fancy_encoding[:, :seq_len, :]\n",
    "        encoder_layer = nn.TransformerEncoderLayer(d_model=model_dim, nhead=num_heads,\n",
    "                                                   dim_feedforward=model_dim*4,\n",
    "                                                   dropout=dropout, batch_first=True, dtype=torch.bfloat16)\n",
    "        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers)\n",
    "        self.fc_out = nn.Linear(model_dim, output_dim, dtype=torch.bfloat16)\n",
    "\n",
    "    def forward(self, src, key_padding_mask):\n",
    "        batch_sz = src.size(0)\n",
    "        src = torch.cat((src, torch.full((batch_sz, 1), MAX_VTXS + 1, device=device)), dim=1)\n",
    "        embed = torch.cat((self.embedding(src[:,::2]), self.embedding(src[:,1::2])), dim=2)\n",
    "        output = self.transformer_encoder(embed, src_key_padding_mask=key_padding_mask[:, ::2])\n",
    "        return self.fc_out(output[:, -1, :])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bpIeg86S-hBb"
   },
   "source": [
    "# Step 3: Make Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kWXvJRDYgFVP",
    "outputId": "c13adb9d-6565-43b5-8437-20cef3dc0d16"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data: 524M\n",
      "Trainable parameters in the model: 800K\n"
     ]
    }
   ],
   "source": [
    "# PARAMS\n",
    "VOCAB_SIZE = 1 + MAX_VTXS + 1 # pad plus max number of vertices plus target token\n",
    "MODEL_DIM = 64 # Dimension of model (embedding and transformer)\n",
    "NEPOCHS = 1000\n",
    "BSZ = 2**15 # Batch size\n",
    "BPE = 16 # Batches per epoch\n",
    "NHEADS = 8\n",
    "NLAYERS = 16\n",
    "DROPOUT = 0 # 0.2\n",
    "model = TransformerModel(input_dim=VOCAB_SIZE, model_dim=MODEL_DIM,\n",
    "                         output_dim=1, num_heads=NHEADS,\n",
    "                         num_layers=NLAYERS, seq_len=SEQ_LEN,\n",
    "                         dropout=DROPOUT).to(device)\n",
    "# model = torch.compile(model)\n",
    "\n",
    "trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "print(f\"Training data: {NEPOCHS*BSZ*BPE//10**6}M\")\n",
    "print(f\"Trainable parameters in the model: {trainable_params//1000}K\")\n",
    "\n",
    "train_err = []\n",
    "test_err = []\n",
    "\n",
    "# clear loss file\n",
    "open('loss', 'w').close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "LR = 2e-6\n",
    "WD = 0 # 1e-5\n",
    "\n",
    "criterion = nn.MSELoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=LR, weight_decay=WD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from torch.utils.data import DataLoader, TensorDataset\n",
    "\n",
    "# with open(\"data.pkl\", \"rb\") as f:\n",
    "#     pickled_stuff = pickle.load(f)\n",
    "\n",
    "# data = pickled_stuff[\"data\"].to(device)\n",
    "# label = pickled_stuff[\"labels\"].to(device)\n",
    "# padding_mask = (data == PAD_TOKEN).bool().to(device)\n",
    "# dataset = TensorDataset(data, label, padding_mask)\n",
    "# # train_dataset, test_dataset = torch.utils.data.random_split(dataset, [.9, .1])\n",
    "# train_loader = DataLoader(dataset, batch_size=BSZ, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "f8Zn33m7CxL5"
   },
   "source": [
    "# Step 4: Train the Model for the first task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate():\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    with torch.no_grad():\n",
    "        batch_src, batch_labels, batch_padding_mask = mkbatch(BSZ)\n",
    "        output = model(batch_src, batch_padding_mask)\n",
    "        loss = criterion(output.squeeze(1), batch_labels)\n",
    "    return loss.item()\n",
    "\n",
    "# figure out if it's doing better on short paths\n",
    "def evaluate_short():\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    with torch.no_grad():\n",
    "        batch_src, batch_labels, batch_padding_mask = mkbatch(BSZ)\n",
    "        output = model(batch_src, batch_padding_mask)\n",
    "        loss = criterion(output[batch_labels == 1].squeeze(1), batch_labels[batch_labels==1])\n",
    "    return loss.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 486
    },
    "id": "pvTfzGmCeXU4",
    "outputId": "0d3a20f3-23be-4c19-9eb6-46bfe11a48b1"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [00:05<00:00,  2.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1000 \t Train Err: 1.1650 \t Test Err: 1.1562, Test short loss: 0.0006\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [00:05<00:00,  3.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2/1000 \t Train Err: 1.1616 \t Test Err: 1.1797, Test short loss: 0.0006\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [00:05<00:00,  3.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3/1000 \t Train Err: 1.1660 \t Test Err: 1.1562, Test short loss: 0.0006\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [00:05<00:00,  3.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4/1000 \t Train Err: 1.1621 \t Test Err: 1.1719, Test short loss: 0.0007\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [00:05<00:00,  3.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5/1000 \t Train Err: 1.1597 \t Test Err: 1.1562, Test short loss: 0.0006\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [00:05<00:00,  3.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6/1000 \t Train Err: 1.1694 \t Test Err: 1.1406, Test short loss: 0.0007\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [00:05<00:00,  3.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7/1000 \t Train Err: 1.1660 \t Test Err: 1.1797, Test short loss: 0.0007\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [00:05<00:00,  3.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8/1000 \t Train Err: 1.1611 \t Test Err: 1.1562, Test short loss: 0.0007\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [00:05<00:00,  3.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9/1000 \t Train Err: 1.1665 \t Test Err: 1.1328, Test short loss: 0.0006\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [00:05<00:00,  3.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10/1000 \t Train Err: 1.1685 \t Test Err: 1.1797, Test short loss: 0.0006\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [00:05<00:00,  3.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11/1000 \t Train Err: 1.1582 \t Test Err: 1.1484, Test short loss: 0.0007\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [00:05<00:00,  3.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12/1000 \t Train Err: 1.1680 \t Test Err: 1.1641, Test short loss: 0.0007\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [00:05<00:00,  3.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13/1000 \t Train Err: 1.1582 \t Test Err: 1.1562, Test short loss: 0.0006\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [00:05<00:00,  3.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 14/1000 \t Train Err: 1.1689 \t Test Err: 1.1641, Test short loss: 0.0006\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [00:05<00:00,  3.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 15/1000 \t Train Err: 1.1670 \t Test Err: 1.1641, Test short loss: 0.0007\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|███████████████████████████████████████████████▌                                                                                                        | 5/16 [00:01<00:04,  2.51it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[28], line 10\u001b[0m\n\u001b[1;32m      8\u001b[0m output \u001b[38;5;241m=\u001b[39m model(batch_src, batch_padding_mask)\n\u001b[1;32m      9\u001b[0m loss \u001b[38;5;241m=\u001b[39m criterion(output\u001b[38;5;241m.\u001b[39msqueeze(\u001b[38;5;241m1\u001b[39m), batch_labels)\n\u001b[0;32m---> 10\u001b[0m train_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[43mloss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mitem\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;241m/\u001b[39m BPE\n\u001b[1;32m     11\u001b[0m loss\u001b[38;5;241m.\u001b[39mbackward()\n\u001b[1;32m     12\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for epoch in range(NEPOCHS):\n",
    "    model.train()\n",
    "    train_loss = 0\n",
    "    for i in tqdm(range(BPE)):\n",
    "        batch_src, batch_labels, batch_padding_mask = mkbatch(BSZ)\n",
    "    # for batch_src, batch_labels, batch_padding_mask in tqdm(train_loader):\n",
    "        optimizer.zero_grad()\n",
    "        output = model(batch_src, batch_padding_mask)\n",
    "        loss = criterion(output.squeeze(1), batch_labels)\n",
    "        train_loss += loss.item() / BPE\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "    test_loss = evaluate()\n",
    "    test_short_loss = evaluate_short()\n",
    "    \n",
    "    test_err.append(test_loss)\n",
    "    train_err.append(train_loss)\n",
    "    with open('loss', 'a') as f:\n",
    "        f.write(f\"{train_loss} {test_loss}\\n\")\n",
    "    print(f\"Epoch {epoch + 1}/{NEPOCHS} \\t Train Err: {train_loss:.4f} \\t Test Err: {test_loss:.4f}, Test short loss: {test_short_loss:.4f}\")\n",
    "    \n",
    "    if epoch % 100 == 99:\n",
    "        torch.save(model.state_dict(), f\"model_weights_{epoch}.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \"\"\"\n",
    "# Now let's figure out what it's doing. \n",
    "\n",
    "# step 1: figure out what people are attending to \n",
    "# \"\"\"\n",
    "\n",
    "# example_graph, answer, padding = mkbatch(1)\n",
    "# sentance_embeddings = model.full_embedding(example_graph)[0,:,:][example_graph.flatten() != 0]\n",
    "# WQ,WK,WV = torch.split(model.transformer_encoder.layers[0].self_attn.in_proj_weight, (MODEL_DIM, MODEL_DIM, MODEL_DIM))\n",
    "\n",
    "# Q = sentance_embeddings@WQ\n",
    "# K = sentance_embeddings@WK\n",
    "\n",
    "# raw_scores = Q @ K.T / sqrt(MODEL_DIM)\n",
    "# soft = torch.softmax(raw_scores, dim=-1).detach().cpu().to(float).numpy()\n",
    "# plt.imshow(soft)\n",
    "# plt.show()\n",
    "\n",
    "# print(example_graph)\n",
    "\n",
    "# print(Q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAHgCAYAAAC/0ofgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABOL0lEQVR4nO3deXhU1f3H8fdkm+wbZGVJAiIgCCJQRFRAFkHFDQUVK6gVFVyo1V+hrSxuKIpFXBC1glYFhFaqVlTcolJRdnABlV1CCFv2feb+/rjJwJgACUwyuTOf1/PcZ2buvZn5zi195uM5555jMwzDQERERMSCArxdgIiIiMjJUpARERERy1KQEREREctSkBERERHLUpARERERy1KQEREREctSkBERERHLUpARERERy1KQEREREctSkBER8SCbzcadd97p7TJE/IaCjIiFzJ8/H5vNhs1m46uvvqpx3DAMWrVqhc1m49JLL3U7VlhYyJQpU+jcuTMRERE0a9aMs846i3vuuYesrCzXeVOnTnV9Rm1bdnZ2g3/P4zlebbfffrtXaxORxhfk7QJEpP5CQ0N58803Oe+889z2Z2Zm8uuvv2K32932V1RUcMEFF7B582ZGjx7NXXfdRWFhId9//z1vvvkmV155JampqW5/M2fOHCIjI2t8dmxsrMe/T30NGjSIG2+8scb+008/3QvViIg3KciIWNDFF1/M4sWLmT17NkFBR/5v/Oabb9K9e3cOHDjgdv7SpUtZt24db7zxBtdff73bsdLSUsrLy2t8xtVXX03z5s0b5gucotNPP50bbrjB22WISBOgriURC7ruuus4ePAgy5cvd+0rLy9nyZIlNYIKwNatWwHo06dPjWOhoaFER0d7pK7OnTvTv3//GvudTictWrTg6quvdu1buHAh3bt3JyoqiujoaM4880yefvppj9QB0K9fPzp37syaNWs499xzCQsLIyMjgxdeeKHGuTk5Odxyyy0kJSURGhpK165defXVV2v9Hk8//TRnnnkmoaGhJCQkMGTIEFavXl3j3KVLl9K5c2fsdjudOnXigw8+cDteUFDAhAkTSE9Px263k5iYyKBBg1i7dq3HroGIP1CQEbGg9PR0evfuzYIFC1z7li1bRl5eHtdee22N89PS0gB47bXXMAyjTp9x6NAhDhw44Lbl5uYe929GjhzJF198UWMczVdffUVWVpartuXLl3PdddcRFxfH448/zmOPPUa/fv1YsWJFnWorLS2tUduBAwdqtCwdPnyYiy++mO7duzNjxgxatmzJHXfcwSuvvOI6p6SkhH79+vHPf/6TUaNG8cQTTxATE8OYMWNqBKtbbrmFCRMm0KpVKx5//HEmTpxIaGgoK1eurPF9x40bx7XXXsuMGTMoLS1l+PDhHDx40HXO7bffzpw5cxg+fDjPP/889913H2FhYfz44491ugYiUsUQEcuYN2+eARirVq0ynn32WSMqKsooLi42DMMwrrnmGqN///6GYRhGWlqacckll7j+rri42Gjfvr0BGGlpacaYMWOMf/zjH8a+fftqfMaUKVMMoNatffv2x61vy5YtBmA888wzbvvHjRtnREZGumq95557jOjoaKOysrLe1+BYtQHGggULXOf17dvXAIyZM2e69pWVlRlnnXWWkZiYaJSXlxuGYRizZs0yAOP11193nVdeXm707t3biIyMNPLz8w3DMIxPP/3UAIy77767Rk1Op9OtvpCQEOOXX35x7duwYUON6xITE2OMHz++3t9fRNypRUbEokaMGEFJSQnvvfceBQUFvPfee7V2KwGEhYXxzTffcP/99wPm3U+33HILKSkp3HXXXZSVldX4m3/9618sX77cbZs3b95xazr99NM566yzWLRokWufw+FgyZIlDBs2jLCwMMAcMFxUVOTWNVYfl19+eY3ali9fXqNbKygoiNtuu831OiQkhNtuu42cnBzWrFkDwPvvv09ycjLXXXed67zg4GDuvvtuCgsLyczMdF0Pm83GlClTatRjs9ncXg8cOJC2bdu6Xnfp0oXo6Gi2bdvm2hcbG8s333zjdseYiNSfBvuKWFRCQgIDBw7kzTffpLi4GIfD4TYG5bdiYmKYMWMGM2bMYOfOnXzyySc8+eSTPPvss8TExPDwww+7nX/BBRec1GDfkSNH8pe//IU9e/bQokULPv/8c3Jychg5cqTrnHHjxvHWW28xdOhQWrRoweDBgxkxYgRDhgyp02e0bNmSgQMHnvC81NRUIiIi3PZV39m0Y8cOzjnnHHbu3Em7du0ICHD/77qOHTsCsHPnTsAcZ5Samkp8fPwJP7d169Y19sXFxXH48GHX6xkzZjB69GhatWpF9+7dufjii7nxxhtp06bNCd9fRI5Qi4yIhV1//fUsW7aMF154gaFDh9b51ui0tDRuvvlmVqxYQWxsLG+88YbHaho5ciSGYbB48WIA3nrrLWJiYtxCSmJiIuvXr+edd97hsssu47PPPmPo0KGMHj3aY3V4U2BgYK37jaPGJ40YMYJt27bxzDPPkJqayhNPPEGnTp1YtmxZY5Up4hMUZEQs7MorryQgIICVK1ces1vpeOLi4mjbti179+71WE0ZGRn87ne/Y9GiRVRWVvLvf/+bK664osbcNiEhIQwbNoznn3+erVu3ctttt/Haa6/xyy+/eKyWrKwsioqK3Pb99NNPgDlgGsxQ9/PPP+N0Ot3O27x5s+s4QNu2bcnKyuLQoUMeqy8lJYVx48axdOlStm/fTrNmzXjkkUc89v4i/kBBRsTCIiMjmTNnDlOnTmXYsGHHPG/Dhg015pYBs9vkhx9+oH379h6ta+TIkaxcuZJXXnmFAwcOuHUrAW537wAEBATQpUsXgFrH65ysyspK5s6d63pdXl7O3LlzSUhIoHv37oA5J092drbbuJ7KykqeeeYZIiMj6du3LwDDhw/HMAymTZtW43OMOt4JVs3hcJCXl+e2LzExkdTUVI9+fxF/oDEyIhZXl+6Y5cuXM2XKFC677DLOOeccIiMj2bZtG6+88gplZWVMnTq1xt8sWbKk1pl9Bw0aRFJS0nE/b8SIEdx3333cd999xMfH1xjP8oc//IFDhw5x4YUX0rJlS3bu3MkzzzzDWWed5Rqbcjw//fQTr7/+eo39SUlJDBo0yPU6NTWVxx9/nB07dnD66aezaNEi1q9fz4svvkhwcDAAY8eOZe7cuYwZM4Y1a9aQnp7OkiVLWLFiBbNmzSIqKgqA/v378/vf/57Zs2fz888/M2TIEJxOJ19++SX9+/ev1/pKBQUFtGzZkquvvpquXbsSGRnJxx9/zKpVq5g5c2ad30dE0O3XIlZy9O3Xx/Pb26+3bdtmTJ482TjnnHOMxMREIygoyEhISDAuueQS49NPP3X72+Pdfg0Yn332WZ1q7dOnjwEYf/jDH2ocW7JkiTF48GAjMTHRCAkJMVq3bm3cdtttxt69e0/4vserrW/fvq7z+vbta3Tq1MlYvXq10bt3byM0NNRIS0sznn322RrvuW/fPuOmm24ymjdvboSEhBhnnnmmMW/evBrnVVZWGk888YTRoUMHIyQkxEhISDCGDh1qrFmzxq2+2m6rTktLM0aPHm0Yhnkb+P3332907drViIqKMiIiIoyuXbsazz///Am/v4i4sxlGPdtERUQsoF+/fhw4cIDvvvvO26WISAPSGBkRERGxLAUZERERsSwFGREREbEsjZERERERy1KLjIiIiFiWgoyIiIhYloKMiIiIWJaCjIiIiFiWgoyIiIhYloKMiIiIWJaCjIiIiFiWgoyIiIhYloKMiIiIWJaCjIiIiFiWgoyIiIhYloKMiIiIWJaCjIiIiFiWgoyIiIhYloKMiIiIWJaCjIiIiFiWgoyIiIhYloKMiIiIWJaCjIiIiFiWgoyIiIhYloKMiIiIWJaCjIiIiFiWgoyIiIhYloKMiIiIWJaCjIiIiFiWgoyIiIhYloKMiIiIWFaQtwtoaE6nk6ysLKKiorDZbN4uR0REROrAMAwKCgpITU0lIODY7S4+H2SysrJo1aqVt8sQERGRk7B7925atmx5zOM+H2SioqIA80JER0d7uRoRERGpi/z8fFq1auX6HT8Wnw8y1d1J0dHRCjIiIiIWc6JhIRrsKyIiIpalICMiIiKWpSAjIiIiluXzY2REREQagsPhoKKiwttlWFZwcDCBgYGn/D4KMiIiIvVgGAbZ2dnk5uZ6uxTLi42NJTk5+ZTmeVOQERERqYfqEJOYmEh4eLgmWz0JhmFQXFxMTk4OACkpKSf9XgoyIiIideRwOFwhplmzZt4ux9LCwsIAyMnJITEx8aS7mTTYV0REpI6qx8SEh4d7uRLfUH0dT2WskYKMiIhIPak7yTM8cR0VZERERMSyFGRERETEshRkRERE5KSkp6cza9Ysr9agINOAysogKwtKS71diYiI+DObzXbcberUqSf1vqtWrWLs2LGeLbaedPv1KTIMOHAANm+GLVvMx+rn27aB02meFxEBzZpB8+Y1t7g4iIw8/hYYCJWV4HCYj5WV4Nh/iKCvv8S+MhP7mv/hiImnuGdfinr2o/SMsyE42K3WkBBo0wY0Rk1ExL/s3bvX9XzRokVMnjyZLVu2uPZFRka6nhuGgcPhICjoxBEhISHBs4WeBAWZkzTzSYN//dvG5s1w+PCxz7PZzLBTVGRuu3ad/GfGc5AL+IJ+fE5fMunCRgIw3M6J/HIZAIVE8BXnkUlfPqcfq+lBJcFMmgSPPnryNYiIiDvDgOJi73x2eHjd/uM0OTnZ9TwmJgabzeba9/nnn9O/f3/ef/99/va3v7Fp0yY++ugjWrVqxb333svKlSspKiqiY8eOTJ8+nYEDB7reKz09nQkTJjBhwgTAbPl56aWX+O9//8uHH35IixYtmDlzJpdddplHv/fRFGROUmjmh0z/ejrTmMLn9Cct3UaHDtC+PXTocGRLSoL8fLPV5uBB87F6y99TQOcVc0nZs5pKB1RWYD5WVj2vankBg478SBc21ahjs60D/wvqy8rg80lgP30qPqd35RfEGYcZwocM4UMAighnBX145+OH4dHfNe7FEhHxYcXFZsu5NxQWmi3+njBx4kSefPJJ2rRpQ1xcHLt37+biiy/mkUcewW6389prrzFs2DC2bNlC69atj/k+06ZNY8aMGTzxxBM888wzjBo1ip07dxIfH++ZQn9DQeYk/T7rMaL5gk8ZgOPc8wicOhkGDqw1GsfEmFvbtlU78vPh2Wdh/kw4dKhen2uccQb07YetX1+44AI6JCfTAbjZdcYEsz/ru+/g888hMxO++IKIAwcYzHJCfokA3j7Jby0iIr7qwQcfZNCgQa7X8fHxdO3a1fX6oYce4u233+add97hzjvvPOb7jBkzhuuuuw6ARx99lNmzZ/Ptt98yZMiQBqlbQeYkRf/ndXj8cXjpJQL/9xUMHgy9e8OUKebz2tr68vJg9mz4+9+P9EedfjrccgtUTdV8TKmpcMEF2OrSHxkQAF26mNvdd4PTyc9/fJ52s+8irCy33t9VRESOLTzcbBnx1md7So8ePdxeFxYWMnXqVP773/+yd+9eKisrKSkpYdcJxkh06dLF9TwiIoLo6GjXmkoNQUHmZLVsCc88A5MmwYwZMHcufP01DBkCvXrB5MkwdKgZaHJz4emnYdYs8zmY/U4PPAAjR5ojeRtSQAABGWkAhFQUNexniYj4GZvNc9073hTxmy9x3333sXz5cp588klOO+00wsLCuPrqqykvLz/u+wT/5kYTm82Gs/rOlwagIHOqUlPNgPLnP8OTT8KcOfDNN3DJJdCzJ1xwAbz0ktmdBHDGGWaAueaahg8wRwlrbv4DDa1UkBERkRNbsWIFY8aM4corrwTMFpodO3Z4t6haaB4ZT0lJgZkzYft2uO8+s71v1SpzX34+dO4Mb70FmzbBtdc2aoiBI0EmzCjC4WjUjxYREQtq164d//73v1m/fj0bNmzg+uuvb9CWlZOlIONpSUnwxBNmoJk4Ea68EpYsgQ0bzFaYAO9c8ohEM8hEUERBgVdKEBERC3nqqaeIi4vj3HPPZdiwYVx00UWcffbZ3i6rBpthGMaJT7Ou/Px8YmJiyMvLIzo62tvleM/27dCmDcWEcWBnMce5c05ERI6htLSU7du3k5GRQWhoqLfLsbzjXc+6/n6rRcZfVA3iCqeE/Nym1zQoIiJyMhRk/MVRo9ELc7w0BaWIiIiHKcj4i6PmqSnerzuXRETENyjI+IuAAEoCzJmTFGRERMRXKMj4kbIgs3up9KCCjIiI+AYFGT9SHmwGmfLDCjIiIuIbFGT8SGWIgoyIiPgWBRk/UhlqBpnKPAUZERHxDQoyfsQZZgYZR76CjIiI+AYFGT9ihJtBxlmoICMi4k9sNttxt6lTp57Sey9dutRjtdaXVr/2I7bqSfEUZERE/MrevXtdzxctWsTkyZPZsmWLa19kZKQ3yvIItcj4EVtkVZApUpAREfEnycnJri0mJgabzea2b+HChXTs2JHQ0FA6dOjA888/7/rb8vJy7rzzTlJSUggNDSUtLY3p06cDkJ6eDsCVV16JzWZzvW5MapHxI4HRZpAJKFGQERHxGMOAYi8t/RIeDjbbKb3FG2+8weTJk3n22Wfp1q0b69at49ZbbyUiIoLRo0cze/Zs3nnnHd566y1at27N7t272b17NwCrVq0iMTGRefPmMWTIEAIDAz3xrepFQcaPBMaYQSawVEFGRMRjiovBW10zhYVua+mdjClTpjBz5kyuuuoqADIyMvjhhx+YO3cuo0ePZteuXbRr147zzjsPm81GWlqa628TEhIAiI2NJTk5+ZTqOFle7Vr64osvGDZsGKmpqbUOFjIMg8mTJ5OSkkJYWBgDBw7k559/9k6xPiA41vzHHlSuICMiIlBUVMTWrVu55ZZbiIyMdG0PP/wwW7duBWDMmDGsX7+e9u3bc/fdd/PRRx95uWp3Xm2RKSoqomvXrtx8882uJHi0GTNmMHv2bF599VUyMjJ44IEHuOiii/jhhx8IDQ31QsXWZo8zg0xIRRFOJwRohJSIyKkLDzdbRrz12aegsKrul156iV69erkdq+4mOvvss9m+fTvLli3j448/ZsSIEQwcOJAlS5ac0md7ileDzNChQxk6dGitxwzDYNasWfztb3/j8ssvB+C1114jKSmJpUuXcu2119b6d2VlZZSVlble5+fne75wiwptZgaZCIooLIToaC8XJCLiC2y2U+7e8ZakpCRSU1PZtm0bo0aNOuZ50dHRjBw5kpEjR3L11VczZMgQDh06RHx8PMHBwTgcjkas2l2THSOzfft2srOzGThwoGtfTEwMvXr14uuvvz5mkJk+fTrTpk1rrDItJSjmSJDJz1eQERERmDZtGnfffTcxMTEMGTKEsrIyVq9ezeHDh7n33nt56qmnSElJoVu3bgQEBLB48WKSk5OJjY0FzDuXPvnkE/r06YPdbicuLq5R62+ynQvZ2dmAmRaPlpSU5DpWm0mTJpGXl+faqkdWy5Hbr6uDjIiIyB/+8Adefvll5s2bx5lnnknfvn2ZP38+GRkZAERFRTFjxgx69OhBz5492bFjB++//z4BVeMTZs6cyfLly2nVqhXdunVr9PqbbIvMybLb7djtdm+X0TRFHAkyeXlerkVERLxizJgxjBkzxm3f9ddfz/XXX1/r+bfeeiu33nrrMd9v2LBhDBs2zJMl1kuTbZGpvo1r3759bvv37dvntVu8LC9CLTIiIuJbmmyQycjIIDk5mU8++cS1Lz8/n2+++YbevXt7sTILU5AREREf49WupcLCQn755RfX6+3bt7N+/Xri4+Np3bo1EyZM4OGHH6Zdu3au269TU1O54oorvFe0lSnIiIiIj/FqkFm9ejX9+/d3vb733nsBGD16NPPnz+f//u//KCoqYuzYseTm5nLeeefxwQcfaA6Zk3V0kMkzgFOb1lpERMTbvBpk+vXrh2EYxzxus9l48MEHefDBBxuxKh9WFWQCMCg+VAqEebceERGLOt5vl9SdJ65jkx0jIw3gqAmbyg5pmQIRkfoKDg4GoNhbi0T6mOrrWH1dT4bP3X4txxEYSGWQnaDKsqog09zbFYmIWEpgYCCxsbHk5OQAEB4eju0UV5/2R4ZhUFxcTE5ODrGxsae0araCjJ+pCIkgqLKM8ly1yIiInIzqKUCqw4ycPE+smq0g42ccoRFQfAhHnoKMiMjJsNlspKSkkJiYSEVFhbfLsazg4OBTaomppiDjZ4wwc5yMI19BRkTkVAQGBnrkh1hOjQb7+hmjasCvUaggIyIi1qcg42+q71wqUpARERHrU5DxM4FRZpCxFSvIiIiI9SnI+JnA6KpJ8UqK0HxOIiJidQoyfiYoxgwy4RRRWOjlYkRERE6RgoyfqW6R0cKRIiLiCxRk/IwtUkFGRER8h4KMv4lQkBEREd+hIONvFGRERMSHKMj4GwUZERHxIQoy/kZBRkREfIiCjL9RkBERER+iIONvjgoyeXlerkVEROQUKcj4G7XIiIiID1GQ8TcKMiIi4kMUZPyNgoyIiPgQBRl/oyAjIiI+REHG31QFmWAqKc4t93IxIiIip0ZBxt9UBRmAirwiLxYiIiJy6hRk/E1ICM7AIAAcCjIiImJxCjJ+yAg3W2Uc+QoyIiJibQoy/qiqe8koLMIwvFyLiIjIKVCQ8UO2qiATZhRRXOzlYkRERE6BgowfskXpFmwREfENCjJ+yKa5ZERExEcoyPgjBRkREfERCjL+SEFGRER8hIKMP1KQERERH6Eg44+OCjJ5eV6uRURE5BQoyPgjtciIiIiPUJDxRwoyIiLiIxRk/JGCjIiI+AgFGX+kICMiIj5CQcYfKciIiIiPUJDxRwoyIiLiIxRk/JFuvxYRER+hIOOP1CIjIiI+QkHGHynIiIiIj1CQ8UcKMiIi4iMUZPzRb4KMYXi5HhERkZOkIOOPqoJMKGU4Kx2Ulnq5HhERkZOkIOOPqoIMqHtJRESsTUHGH4WGgs0GKMiIiIi1Kcj4I5tNc8mIiIhPUJDxV7pzSUREfICCjL9SkBERER+gIOOvFGRERMQHKMj4KwUZERHxAQoy/kpBRkREfICCjL9SkBERER+gIOOvFGRERMQHKMj4K80jIyIiPkBBxl+pRUZERHyAgoy/UpAREREf0KSDjMPh4IEHHiAjI4OwsDDatm3LQw89hGEY3i7N+hRkRETEBwR5u4Djefzxx5kzZw6vvvoqnTp1YvXq1dx0003ExMRw9913e7s8a1OQERERH9Ckg8z//vc/Lr/8ci655BIA0tPTWbBgAd9++62XK/MBCjIiIuIDmnTX0rnnnssnn3zCTz/9BMCGDRv46quvGDp06DH/pqysjPz8fLdNaqEgIyIiPqBJt8hMnDiR/Px8OnToQGBgIA6Hg0ceeYRRo0Yd82+mT5/OtGnTGrFKizoqyJSXQ1kZ2O1erklERKSemnSLzFtvvcUbb7zBm2++ydq1a3n11Vd58sknefXVV4/5N5MmTSIvL8+17d69uxErtpCjggyguWRERMSSmnSLzP3338/EiRO59tprATjzzDPZuXMn06dPZ/To0bX+jd1ux66mhROrCjKRtiIwID8fEhO9XJOIiEg9NekWmeLiYgIC3EsMDAzE6XR6qSIfcnSQAY2TERERS2rSLTLDhg3jkUceoXXr1nTq1Il169bx1FNPcfPNN3u7NOurCjLhhoKMiIhYV5MOMs888wwPPPAA48aNIycnh9TUVG677TYmT57s7dKszxVkirHhJD+/STfOiYiI1KpJB5moqChmzZrFrFmzvF2K76kKMgBhlJCfH3Gck0VERJom/We4vwoPdz3VXDIiImJVCjL+KiAAwsIAM8jo9msREbEiBRl/ptl9RUTE4hRk/JmCjIiIWJyCjD9TkBEREYtTkPFnCjIiImJxCjL+TEFGREQsTkHGnynIiIiIxSnI+DMFGRERsTgFGX92VJDRPDIiImJFCjL+TC0yIiJicQoy/uyoIFNWBmVlXq5HRESknhRk/NlRQQagoMCbxYiIiNSfgow/qwoyMYFmkFH3koiIWI2CjD+rDjJBCjIiImJNCjL+rCrIRKlFRkRELEpBxp9VBxmbgoyIiFiTgow/qx7sWxVkNJeMiIhYjYKMP6sKMuGGWmRERMSaFGT8WVWQCXMqyIiIiDUpyPizqiATWqkgIyIi1qQg48+qgkxIZRFgKMiIiIjlKMj4s6ogE2A4sVOmICMiIpajIOPPqoIMaOFIERGxJgUZfxYUBCEhgIKMiIhYk4KMvztq4UjNIyMiIlajIOPvjgoyapERERGrUZDxdwoyIiJiYQoy/k5BRkRELExBxt8dFWRKSqCiwsv1iIiI1IOCjL87KsgAFBR4sxgREZH6UZDxd1VBJi5YyxSIiIj1KMj4u6ogE283g4xuwRYREStRkPF31S0yIWqRERER61GQ8XdVQSZGXUsiImJBCjL+rjrIBCrIiIiI9SjI+LuqIBMVoCAjIiLWoyDj76qCTKSCjIiIWJCCjL/7zTwyCjIiImIlCjL+rirIhDsVZERExHoUZPxdVZAJdWoeGRERsR4FGX9XHWQq1SIjIiLWoyDj76qCTEiFgoyIiFiPgoy/qwoyQeUKMiIiYj0KMv6uOsiUKciIiIj1KMj4u6ogE1BZQRAVCjIiImIpCjL+rirIgDmXjIKMiIhYiYKMvwsJgcBAwAwyRUXgcHi5JhERkTpSkPF3Nptm9xUREctSkBFXkIkLVpARERFrUZARV5BJjFCQERERa1GQEVeQaR6mICMiItaiICOuINMsVEFGRESsRUFGXEEm3q4gIyIi1lKvIDNjxgxKSkpcr1esWEFZWZnrdUFBAePGjfNcddI4qgf7hijIiIiItdQryEyaNImCggLX66FDh7Jnzx7X6+LiYubOneu56qRxVAWZmCAFGRERsZZ6BRnDMI77WiyqKshEB5pBJi/Pm8WIiIjUncbISI0WmZwcbxYjIiJSdwoycmSwb9VdS7t3e7MYERGRuguq7x+8/PLLREZGAlBZWcn8+fNp3rw5gNv4GU/Zs2cPf/7zn1m2bBnFxcWcdtppzJs3jx49enj8s/xWVZCJrWqR2bXLm8WIiIjUXb2CTOvWrXnppZdcr5OTk/nnP/9Z4xxPOXz4MH369KF///4sW7aMhIQEfv75Z+Li4jz2GYIryEQFKMiIiIi11CvI7Nixo4HKqN3jjz9Oq1atmDdvnmtfRkZGo9bgF6qCTLhx5K6lvDyIifFmUSIiIifWpMfIvPPOO/To0YNrrrmGxMREunXr5tYiVJuysjLy8/PdNjmBqiATVFZEdWOXxsmIiIgV1CvIfP3117z33ntu+1577TUyMjJITExk7NixbhPknapt27YxZ84c2rVrx4cffsgdd9zB3XffzauvvnrMv5k+fToxMTGurVWrVh6rx2dVBRmKiqjuGVT3koiIWEG9gsyDDz7I999/73q9adMmbrnlFgYOHMjEiRN59913mT59useKczqdnH322Tz66KN069aNsWPHcuutt/LCCy8c828mTZpEXl6ea9utpoUTqyXI6LKJiIgV1CvIrF+/ngEDBrheL1y4kF69evHSSy9x7733Mnv2bN566y2PFZeSksIZZ5zhtq9jx47sOk5zgd1uJzo62m2TEzgqyFQ3YKlFRkRErKBeQebw4cMkJSW5XmdmZjJ06FDX6549e3q0BaRPnz5s2bLFbd9PP/1EWlqaxz5DUNeSiIhYVr2CTFJSEtu3bwegvLyctWvXcs4557iOFxQUEBwc7LHi/vjHP7Jy5UoeffRRfvnlF958801efPFFxo8f77HPENS1JCIillWvIHPxxRczceJEvvzySyZNmkR4eDjnn3++6/jGjRtp27atx4rr2bMnb7/9NgsWLKBz58489NBDzJo1i1GjRnnsM4QjQaa0lFapDkAtMiIiYg31mkfmoYce4qqrrqJv375ERkYyf/58QkJCXMdfeeUVBg8e7NECL730Ui699FKPvqf8RnWQAdISioEofv0VnE4IaNI36IuIiL+zGSexhHVeXh6RkZEEBga67T906BBRUVEe7V46Vfn5+cTExJCXl6eBv8diGBAYCIZB5e692NOScTohKwtSUrxdnIiI+KO6/n7Xq0Xm5ptvrtN5r7zySn3eVrzNZoPwcCgqIqisiNRU+PVXs3tJQUZERJqyegWZ+fPnk5aWRrdu3TiJhhxpyiIioKjINeD311/NAb+9enm7MBERkWOrV5C54447WLBgAdu3b+emm27ihhtuID4+vqFqk8akuWRERMSC6jWU87nnnmPv3r383//9H++++y6tWrVixIgRfPjhh2qhsTrNJSMiIhZU73tS7HY71113HcuXL+eHH36gU6dOjBs3jvT0dAoLCxuiRmkMmktGREQs6JRurg0ICMBms2EYBg6Hw1M1iTeoa0lERCyo3kGmrKyMBQsWMGjQIE4//XQ2bdrEs88+y65du4iMjGyIGqUxqGtJREQsqF6DfceNG8fChQtp1aoVN998MwsWLKB58+YNVZs0plqCTE4OlJZCaKj3yhIRETmeegWZF154gdatW9OmTRsyMzPJzMys9bx///vfHilOGtFRQSY+HsLCoKTEvA37tNO8W5qIiMix1CvI3HjjjdhstoaqRbzpqCBjs0Hr1rBlizngV0FGRESaqnpPiCc+6qggA0eCjMbJiIhIU6YlAcX0myCjO5dERMQKFGTEVEuLDGguGRERadoUZMR0jCCjFhkREWnKFGTEpK4lERGxIAUZMR2na0nLaImISFOlICOm3wSZli3Nl4WFkJvrnZJEREROREFGTL8JMuHhUD1ps7qXRESkqVKQEdNvggzoziUREWn6FGTEVEuQ0YBfERFp6hRkxFQdZIqLXaN7dQu2iIg0dQoyYqoOMoZhrhaJupZERKTpU5ARU3j4keeaS0ZERCxCQUZMgYEQGmo+1zIFIiJiEQoycsQxJsX79VdwOLxUk4iIyHEoyMgRvwkyyckQFGSGmL17vViXiIjIMSjIyBG/CTKBgdCihblL3UsiItIUKcjIEceZFE8DfkVEpClSkJEjNCmeiIhYjIKMHKFlCkRExGIUZOQIdS2JiIjFKMjIEepaEhERi1GQkSPUtSQiIhajICNHxMebj/v3u3ZVt8gcOGCuJykiItKUKMjIEenp5uP27a5dsbEQGWk+V6uMiIg0NQoyckRGhvl4VJCx2dS9JCIiTZeCjBxRHWR27gSn07VbA35FRKSpUpCRI1q2NNclKC+HrCzXbrXIiIhIU6UgI0cEBR1JLUd1L2kuGRERaaoUZMRdLeNk1LUkIiJNlYKMuKslyKhrSUREmioFGXHXpo35eIyuJcPwQk0iIiLHoCAj7mppkWnZ0nwsKYGDB71Qk4iIyDEoyIi76iCzbZtrl90OSUnmc3UviYhIU6IgI+6qg8yePVBW5tqtO5dERKQpUpARd4mJEB5uDoY5KrXoziUREWmKFGTEnc1W65pLunNJRESaIgUZqek4t2CrRUZERJoSBRmpSZPiiYiIRSjISE2aFE9ERCxCQUZqOk6LTFYWVFZ6oSYREZFaKMhITbUEmaQkCA4Gp9NtYWwRERGvUpCRmqqDzIEDUFgIQECAxsmIiEjToyAjNcXEQHy8+VwDfkVEpAlTkJHa1bJUgQb8iohIU6MgI7XTXDIiImIBCjJSO80lIyIiFqAgI7XTXDIiImIBlgoyjz32GDabjQkTJni7FN+nriUREbEAywSZVatWMXfuXLp06eLtUvzD0UHGMIAjXUuHD7vuyhYREfEqSwSZwsJCRo0axUsvvURcXJy3y/EPaWnmY1GROZ8MEB1t3pkN6l4SEZGmwRJBZvz48VxyySUMHDjwhOeWlZWRn5/vtslJCA2F1FTzubqXRESkiWryQWbhwoWsXbuW6dOn1+n86dOnExMT49paVfeHSP21aWM+HhVkqhtqjppeRkRExGuadJDZvXs399xzD2+88QahoaF1+ptJkyaRl5fn2narD+Tk1TLgt1Mn83HTJi/UIyIi8htB3i7geNasWUNOTg5nn322a5/D4eCLL77g2WefpaysjMDAQLe/sdvt2O32xi7VN9USZM4803xUkBERkaagSQeZAQMGsOk3v5g33XQTHTp04M9//nONECMeVkuQqb5pbONG82Ymm80LdYmIiFRp0kEmKiqKzp07u+2LiIigWbNmNfZLA6hlvaX27SE4GPLzzQG/1WNmREREvKFJj5ERL6sOMrt2gcMBQEgIdOhg7lb3koiIeFuTbpGpzeeff+7tEvxHixZm80tFBezZ47r3uksXM8Rs3AiXXurlGkVExK+pRUaOLTDwyMQxtQz43bjRCzWJiIgcRUFGju84A37VtSQiIt6mICPHd5wgs2ULlJV5oSYREZEqCjJyfLUEmdRUiIszx//++KOX6hIREUFBRk6klmUKbDb3+WRERES8RUFGjq+WFhnQgF8REWkaFGTk+KqDTFaW24AYDfgVEZGmQEFGjq95c4iIMNcj2LnTtVtdSyIi0hQoyMjx2Wy1LlVQvQp2djbs3++FukRERFCQkbqoZZxMZCS0bWs+V/eSiIh4i4KMnJgG/IqISBOlICMndowgowG/IiLibQoycmInCDJqkREREW9RkJETO0HX0vffm7P8ioiINDYFGTmx6iBz6BDk57t2t20LYWFQUgJbt3qpNhER8WsKMnJiUVHmfDLg1ioTGAidO5vP1b0kIiLeoCAjdXOC7iUN+BUREW9QkJG60YBfERFpghRkpG40l4yIiDRBCjJSNycIMtu2QWFhI9ckIiJ+T0FG6qaW9ZYAEhIgJcV8/t13jVyTiIj4PQUZqZvqILNjh7kS9lE04FdERLxFQUbqpnVrcyXs4mLIyXE7pAG/IiLiLQoyUjd2O7RoYT7XgF8REWkiFGSk7uqweORvep1EREQalIKM1N0xgkzHjuYsv4cPw549XqhLRET8loKM1F2bNubjb4KM3Q7t25vPNeBXREQak4KM1N0xWmRAA35FRMQ7FGSk7o4TZDTgV0REvEFBRuquOsjs2gUOh9uhowf8ioiINBYFGam71FQICYHKSvj1V7dD1UHmxx+hvNwLtYmIiF9SkJG6CwiAtDTz+W+WKmjVCmJizIyzebMXahMREb+kICP1c9pp5uOXX7rtttm0VIGIiDQ+BRmpnxtuMB9nzYK8PLdDGvArIiKNTUFG6mfkSHMGvMOHYfZst0Ma8CsiIo1NQUbqJzAQJk82nz/1FOTmug5pLhkREWlsCjJSf9dcA506mSFm1izX7s6dzcc9e+DQIa9UJiIifkZBRuovMBCmTDGf//3vZjcTEB0N6enmbnUviYhIY1CQkZMzfLg5ujc/3wwzVdS9JCIijUlBRk5OQMCRVplZs1x9SboFW0REGpOCjJy8K6+Erl2hoABmzgTUIiMiIo1LQUZOXkAATJ1qPp89Gw4ccLXIfPcdOJ1eq0xERPyEgoycmssvh27doLAQZs6kXTuw26GoCNau9XZxIiLi6xRk5NTYbDBtmvn8mWcIOryfQYPMl8OHQ1aW90oTERHfpyAjp+7SS6FHD7MZ5sknmTcP2rWDXbvg4ovNG5tEREQagoKMnDqb7chYmWefpbkzhw8+gMRE2LDBbJkpL/dqhSIi4qMUZMQzLr4Yfvc7KC6GGTNo0wbefx8iIuDjj+GWW8AwvF2kiIj4GgUZ8Yyjx8o8/zxkZ9O9OyxZYk4E/Prr8Je/eLdEERHxPQoy4jkXXQTnnAMlJTBjBgBDhsDLL5uHH3sMnnvOi/X5k6wsePpp1/IRIiK+SkFGPOfoVpk5c8w+JWDMGHjoIXP3XXfB2297pzy/8te/woQJMGwYlJV5uxoRkQajICOeNWiQuZWWwuDB8MADUFnJX/8KY8ea42Suvx7+9z9vF+rjPvvMfFyxQgOURMSnKciIZ9ls8J//wG23mT+eDz8MF16Ibc+vPPec2UBQWmo+bt7s7WJ91O7dsHOnOfNyYCC88caRJjERER+jICOeFxYGL7wACxdCVBR8+SWcdRZBH/6XhQuhVy9zjcmBA+GLL7xdrA/66ivzsVs3s4sPzAU+FyzwXk0iIg1EQUYazsiR5joF3bvDwYNw6aWET7mfd/9VTocOsGcP9Otn3s2keWY86Msvzcfzz4dbb4U//cl8fdNN6tMTEZ+jICMN67TTzHEad99tvn7ySRKGX8CqxTu46Saz92n6dDj3XNiyxbul+oyjgwzA44+ba2KVlcEVV8C2bRw8aGacbt1g/XpvFSoicupshuHbowDz8/OJiYkhLy+P6Ohob5fj35YuNVsFcnMhJgZefpklXM3YseZdwuHh8Pe/mz+wNpu3i7WoQ4egWTPz+b595vTKYC4fcf75sG4deakdObv0f2w7FAtAQoLZxdehg3dKFhGpTV1/v9UiI43niivM//w/5xzIy4NrruHq16/g+2W7uPBCc1Lg226DK6+E/fu9XaxFrVhhPrZvfyTEAEREsOu5d9lvb0FM1o/MOTSCszpV0LWrea0HDTLHB4uIWI2CjDSutDTzP///8hcICoL//IeUAWewfOhMZj5eSUiIedNTly7w4YfeLtaCftutBFRWwpNPQocBLRhc9i5FhDOY5azufRfLPzLo0AF+/dUcfJ2d7aW6RUROkoKMNL7gYHjkEVi3Dvr0gaIiAu6/j3vf7MF3L6/kjDPMH9QhQ2D8eLMnSuqo+o6l884DYM0acwms++83J1yO7deN3OcXgM1G4MtzSXj97yxfbubLX34xW2YOHfJi/SIi9dSkg8z06dPp2bMnUVFRJCYmcsUVV7BFI0J9R+fOZuvMyy9DfDxs2EC70eey4dw7uP8P5tT6zz8Pp58O8+aB0+nlepu6khJYvRqA0p7n86c/mSFm3TqIi4N//AM+/RRa3HEZzJxp/s1999FyxSI+/hiSk+G772DoUCgo8OL3EBGphyYdZDIzMxk/fjwrV65k+fLlVFRUMHjwYIqKirxdmnhKQIA58+zmzTB6NBgGQS+/wIx3OvDdX96kYweD/fvh5pvNxpu1a71dcBP2zTdQUQGpqUycm8FTT5nh79pr4ccfzWvoGkQ9YQKMG2feNnbDDZz247ssX27myW+/NW9yKi315pcREakjw0JycnIMwMjMzKzz3+Tl5RmAkZeX14CVicd89plhdOhgGOZPrOE4p7fxnxveMmIiKgwwDJvNMO64wzAOHvR2oU3Qgw8aBhiV14w0YmPNS/jqq8c5v7LSMK6/3jzRbjeMjz82vv3WMCIjzV3DhhlGeXmjVS8i4qauv99NukXmt/Ly8gCIj48/5jllZWXk5+e7bWIh/fqZdzY9/DCEhhKw8msue30EB2La8GbXx4k1DjFnjtnd9PLL6m5yUzXQd2P0+eTmQsuWcMMNxzk/MBDmzz8yx8zll9Oz4n+8+y6EhsK775qNZA5HYxQvInJyLBNknE4nEyZMoE+fPnTu3PmY502fPp2YmBjX1qpVq0asUjzCbjdXb966Ff72N0hIIChrN9dtmMh+e0sWxd1G0sHvufVW805uTVaLeWvS118DMH+recfSDTeYPXfHFRwMixaZC3wWFcHFF9Mvei3/+pd5U9mCBeaAa9+ebUpErMwyE+LdcccdLFu2jK+++oqWLVse87yysjLKyspcr/Pz82nVqpUmxLOy0lLzF/Xpp2HDBtfuTwMH8pTjHt7nYgYMDGDKFNfNOv5nzRro0QNndAxhRQcpdwTy/fdwxhl1/PviYrjoIvOup+bNITOTRZvO4LrrzBDz2GPw5z836DcQEXHjUxPi3Xnnnbz33nt89tlnxw0xAHa7nejoaLdNLC401JwReN06yMw0Z8wLCOBCx8e8xzC+pRcHP17L+efDhReap/idqm6lXa36UO4IpHv3eoQYMKdVfu89c12sAwdg4EBG9tjK7Nnm4UmTzImZRUSamiYdZAzD4M477+Ttt9/m008/JSMjw9sliTfZbHDBBfDvf5vdTvfdB9HR9GA1q209mR1wD6s/y6dfP3OozWef+VGXSFWQ+W+e2a30+9+fxHvExJizEHbuDHv3woAB3HnFr0ff3KR1mUSkyWnSQWb8+PG8/vrrvPnmm0RFRZGdnU12djYlJSXeLk28LT0dnnjCXGnyuusIMJzc5ZzNroiOjAxaQmamwYUXmrnn4499PNAYhivIvPnr+QQGwnXXneR7NWsGy5ebi33u3AkDB/L0X3MYONAcQnPZZZr9V0SaliY9RsZ2jJUD582bx5gxY+r0Hlo00k8sX27Oi/LLLwB833ooV+19jp8qzFa8s882p04ZORJCQrxYZ0PYsgU6dKAi0E6kI49Bl9h5771TfM+dO81lDnbvhi5dyF2wjHOuSmXLFujVy2ztCgvzSPUiIrXyiTEyhmHUutU1xIgfGTQINm2CyZMhJIROu5bxY2An3uk9nSh7OWvXwo03mlPxP/QQ5OR4u2APqmqNWRPUi3Ls3HijB94zLQ0++QSSkmDjRmIH9eSj6WuIizPn3bvlFh9v5RIRy2jSQUakXkJDYdo02LgRLryQgNIShn39Fw6ld+Oji2cxKGEdOdkOJk+G1q3NH+ONG71dtAdUra/0Sdl5REfDsGEeet927czVtDt2hKwsWo86ny/uXuK6LfuRRzz0OSIip0BBRnxP+/bmwJh//tOcg2bLDwx6/498tP9sSsKbkxlzGePLZrL+lTV06+pgwABzxe2j7tq3lqoWmS85nxEjPNzl07atOT/NkCFQUkLnadfw9SUPAwYPPABLlnjws0RETkKTHiPjCRoj4+cOHTJXS/zsM7Pl4jerIeYRzVecx6dcyNLQ6+g4IJUhQ8zf7dNO81LN9ZGVBS1a4CCAOA7zXmY0F1zQAJ9TWWkuoT1rFgBr2l/HeVv+gS0sjC++gB49GuAzRcSv1fX3W0FG/Edl5ZG5aDIzzZW3j1rCopJA3mUYc7mNjxhMRpsAV6jp3x8iI71Y+7EsWgTXXstaujE8fS1bt9ZhNt9TMXcu3HknVFayOaYX/fKWEpiazLffQosWDfi5IuJ3fGKwr4hHBQVBz57m/DPvvmu21qxZA089hXH++QTh4EqW8gFD2UpbRm57lH89n81ll5mrQg8cCM8/38RuPz6qW+n3v2/gEANw223w0UcQF0eHvG9YF/Q7ErLW07u3OWneqlUaBCwijUstMiLVfvgBXnwRXn0VcnMBcAQE8VHY5TxVdBufMACDANe8fFdfDcOHQ0qK90qu6NSV4B82cjWLmf7T1bRr10gf/PPP5qjiLVsotoUzzniORYyklDBatjQnX77qKnPJiKCgRqpJRHyKupaqKMhIvZWUwOLFZjfKUStS5sWlsTz4El7LuYjP6E8hUdhs5o91dahp1O6V3FyM+HhshsElZ+/lv2uSG/HDzc9nxAhzDh+gOCiK/xiX84ZjJB8xmApCaN7cnETvqqvMFi27vXFLFBHrUpCpoiAjp2TTJrOV5p//hLw8125HYDDrI/qwOP8iPuQiNtAVgwB69TJ/sC+8EM4917wjvMG8/z5ccgk/cxrLn/uZceMa8LOOpbISHn0UXn7ZnDyvSlFILEu5ktfKR/IpF1JJMDExZqC5/npzzFFgoBfqFRHLUJCpoiAjHlFcbN7S/eGH8MEHsG2b2+GDwUn8t2IwH3IRHzGYAyRgt0OfPmaoGTDAvLPHk90s+/8wiYR/PMZ8200M2/8KzZp57r3rzemElSvNwceLF5trNVUpDG3G0oDhvF08mPWcxXYySEwKYMQIcymFc84xl9ESETmagkwVBRlpEL/8ciTUfPaZuRDRUTYEd+e9CrO15mt6U0kwUVHm2JoLLzRbJLp2PbXBudtbnkfGnhU80+0V7lp70yl+IQ9yOMxb3RctMiea2b/f7XAhkaynKxvoynrOIiflLLpc35lrbgzjzDMVakTEpCBTRUFGGlxZmTmW5oMPzHCzYYPb4eKgKD61DXAFmx2Y6z/FxZnBpn9/c+vcue7BxlFUSmVkDHbK+ei5nxk8rolOelNZCZ9/Dv/6l3lL03ff1TrzoIMAttCejcE92NP6HMrP6kXsBV04o2swnTvj3dYmEfEKBZkqCjLS6LKzzVuUP/zQfDxwwO1wTmgrVlecxWrHWa5Wie1kEN8sgH79oF8/M9icccaxWydWP/UFPf7Ul2xbMnHFWdhDLdKMUVlpLnK5fj2sX49j7QYqVq8nNH9/jVNLCGUN3VnJOWyJ7UVR514kdW9Fl642unQxr48WrhTxXQoyVRRkxKucTli71gw1H35ottw4HDVOKyDSFWo20JVPuZCChLb07Ysr3BwdbBaf/SjXrPsra9tczdlbFzfqV/I4w4DsbCq+WcuhD77F8fU3xG35hrCy3BqnZpHCOrqxnrPYZOtKXsZZxJzdli7dAunSBbp0gVat1D0l4gsUZKooyEiTkp9vdj2tX3/k8RjdLavowSJG8hYj2E1rEhKgb1+zO6r9H4cy2PEB2yc8Tcbf7270r9HgnE5zrpqVKyn/8hsqvvqGsJ83EOCsGQKLCGcjXVxBcGdYRyKTI4lLCSUuJZSElnYSWoeSnBZKcnooLdODiI9X2BFp6hRkqijISJNX3d1SHWxWrcL48ktsR7XcrAzozQLnSBZzDftI4hDxxJCPsWYttrO7ea/2xlRcbC4xsWEDxnqzSyrwh00ElpXU620qCSSXWLaHtGdPZAcONGtPfmoHyjI6ENA2g2bJwSQmQkICJCaaW2Skgo9IY1OQqaIgI5aUk2MOkF20yFwTqur/pobNxs+RZ3N6wRrKw6IJKTjk3xOyOBxmy01VC5dz7Xoqt2zFWVyKUVpKQHkpgRWlBDkr6vR2FQTxC6exhfbsoQUlhFFKKJWBoQRFhhISbW5hsXbC40MJCA/FEVy1hYTirHruDDEfjRA7EdGBxMRAbKy5xcTgem23Y47w1mAfkRoUZKooyIjlZWWZtzEvXAhff31k/5AhsGyZ9+qyEofD7L4rLaU0t5RDm3MoWb8Fxw+bCf5lM+G/biEuZwshFcVeKW9vUEu2Rp7Frriu7E06i4Mtu1Leqi3RsQHExEB0NISHm1tYmLlVP69+jIw0J2BUy5H4CgWZKgoy4lN27YK33jIDzf33m7PJiWc4nfDrr2Y33+bN5vw3paVUFJRSmldGWV4p5fmlVBaW4igqxVlcQmBFGUGOUoIqSwl2lBJcWUqQs8x87qg57qk+CohkE2eynrPYQnsCcBJKqWuzU+b22sDGgcAkCsKSKIpKpjQ2mcrmyRiJSQSmJBLbPIioKAgJAXtgJZGOPMLLcwmvMB/DynIJLc2FigoqK3HbKiqhssJ87nCAs3kiARlp2NunE5MeR/MEG82bQ0SEgpR4joJMFQUZEfEKpxPKy83H33A4zHHfeXmQn1OKY9MPBG5aT9iW9UTv2ECzrE0EVZ5aEHIrBRsHaE45IcSQRxSFHnvvfKLYSRo7SOfXgDT2R6aTH9MaI8ROQAAEBEJggNmDFhh45DEkoJLYwHzibbnEGLlEO3OJcuQSXpFLeFWoCsBBaWJrSpPSKEtOp7RqK0tOoyI+CVuAjeBgiIoyW62ioszNNYO2YUBFRa13Ctag5qwmR0GmioKMiFhOZSX89NORu9u2bTObUkJD3Te7HULNMTnlJQ7Kdu3DsScbI3sfgfuzCTmcTWh+DgFGzTAFUBIYQUFgrLkFxJJvi6E8wO4KHgHVweOo14E4iSzMplnhDuLL9zXudTm6dkLZSRqHiavROnX0FkDdfuLKAsM5EJnOwcg0Dkalcyg6ncPRaRyOSSc3Np3iyESCgm0EBUFwMMd8DA090v0XFgYRASVE5+4i6tBOwnN2YM81r5nNBtjARi2PoXZITILkZNdmS2hOQHAgNlvV/xYBR+Uuh8NsQczOhn37jjw6HDX/zfx2S001V7s9mfVTKithzx7YsQPatzdr9SAFmSoKMiLi1xwOOHjQ/HErL3cfdRwcfGrvXVICu3ZhbN9B+c87Kd28A8fWHdj2/AqVlRgGGFSNVTdwe+0kgJLgGAqDzRCVZ4sljxgOOWM54Ihlf3ksZeU2Uip2kVq+g5SKnbQo30GLih0kVu4hkNrDWUMpIZSDNCMXs85cYmtspYTSgj2ks4N0dpDGTlLI9sjnOwhgPwlkk0w2ydgwSGIfyWSTwP5Tuh4OWyB5US3JjU0nPy6NoubpFCWkU5qURmXzZELz9xGRs4OIAzuJPLiDqAM7iDy0k6jc3a4pEXY9/Bqt//p7j3zXagoyVRRkRER8TEWFOZ5pxw6zj66qdaEiMJQiRyiFlaEUVJhbXqmd0sog1/ie347/cTqhstyJPS+HqIM7iDy4g5hDO4g6vJOYwzuIzd1BVP6eOrfs1KbIFsGugHR2GOlkOZNxEHjcdwun2BVSkthHAvtP+PlObG5BZx9JVBJ03NaqcIpJJQs75Sf93coJZhet+fWmyfR75caTfp/a1PX324Nr8YqIiDSC4GDIyDC3o3cDsVVb/UUDx1izrLzcDE6HD0Nurjm4KTfXfcvLMxePTU2F9HRISzMf09OJiI+no81Gx1re2jBqbk6n+/PiikqMnCNdR7bsvRjYqGxuDuiuaJZMZWxznAFBRDqgjRPSHGbZJSVQWmo+FpTC/lL3fWUlToIPZhOes4PIAzuIPryT2NwdxOfvoFnhDmJLs8m1J5ETns7+8HT2R6RzIDKNg5HpHIhMJz88GVtQIDfdcFIX3SMUZERERI4nJATatGmQt7bZ6jLGOAhiU+D0lAaoIABIrdrOrfWMCKBFA3yyp9RxrV0RERGRpkdBRkRERCxLQUZEREQsS0FGRERELEtBRkRERCxLQUZEREQsS0FGRERELEtBRkRERCxLQUZEREQsS0FGRERELEtBRkRERCxLQUZEREQsS0FGRERELEtBRkRERCwryNsFNDTDMADIz8/3ciUiIiJSV9W/29W/48fi80GmoKAAgFatWnm5EhEREamvgoICYmJijnncZpwo6lic0+kkKyuLqKgobDabx943Pz+fVq1asXv3bqKjoz32vnKErnHD0vVteLrGDUvXt+F58xobhkFBQQGpqakEBBx7JIzPt8gEBATQsmXLBnv/6Oho/R+ogekaNyxd34ana9ywdH0bnreu8fFaYqppsK+IiIhYloKMiIiIWJaCzEmy2+1MmTIFu93u7VJ8lq5xw9L1bXi6xg1L17fhWeEa+/xgXxEREfFdapERERERy1KQEREREctSkBERERHLUpARERERy1KQOUnPPfcc6enphIaG0qtXL7799ltvl2RZX3zxBcOGDSM1NRWbzcbSpUvdjhuGweTJk0lJSSEsLIyBAwfy888/e6dYC5o+fTo9e/YkKiqKxMRErrjiCrZs2eJ2TmlpKePHj6dZs2ZERkYyfPhw9u3b56WKrWXOnDl06dLFNWFY7969WbZsmeu4rq1nPfbYY9hsNiZMmODap2t8aqZOnYrNZnPbOnTo4Dre1K+vgsxJWLRoEffeey9Tpkxh7dq1dO3alYsuuoicnBxvl2ZJRUVFdO3aleeee67W4zNmzGD27Nm88MILfPPNN0RERHDRRRdRWlrayJVaU2ZmJuPHj2flypUsX76ciooKBg8eTFFRkeucP/7xj7z77rssXryYzMxMsrKyuOqqq7xYtXW0bNmSxx57jDVr1rB69WouvPBCLr/8cr7//ntA19aTVq1axdy5c+nSpYvbfl3jU9epUyf27t3r2r766ivXsSZ/fQ2pt9/97nfG+PHjXa8dDoeRmppqTJ8+3YtV+QbAePvtt12vnU6nkZycbDzxxBOufbm5uYbdbjcWLFjghQqtLycnxwCMzMxMwzDM6xkcHGwsXrzYdc6PP/5oAMbXX3/trTItLS4uznj55Zd1bT2ooKDAaNeunbF8+XKjb9++xj333GMYhv79esKUKVOMrl271nrMCtdXLTL1VF5ezpo1axg4cKBrX0BAAAMHDuTrr7/2YmW+afv27WRnZ7td75iYGHr16qXrfZLy8vIAiI+PB2DNmjVUVFS4XeMOHTrQunVrXeN6cjgcLFy4kKKiInr37q1r60Hjx4/nkksucbuWoH+/nvLzzz+TmppKmzZtGDVqFLt27QKscX19ftFITztw4AAOh4OkpCS3/UlJSWzevNlLVfmu7OxsgFqvd/UxqTun08mECRPo06cPnTt3BsxrHBISQmxsrNu5usZ1t2nTJnr37k1paSmRkZG8/fbbnHHGGaxfv17X1gMWLlzI2rVrWbVqVY1j+vd76nr16sX8+fNp3749e/fuZdq0aZx//vl89913lri+CjIifmT8+PF89913bv3fcurat2/P+vXrycvLY8mSJYwePZrMzExvl+UTdu/ezT333MPy5csJDQ31djk+aejQoa7nXbp0oVevXqSlpfHWW28RFhbmxcrqRl1L9dS8eXMCAwNrjNjet28fycnJXqrKd1VfU13vU3fnnXfy3nvv8dlnn9GyZUvX/uTkZMrLy8nNzXU7X9e47kJCQjjttNPo3r0706dPp2vXrjz99NO6th6wZs0acnJyOPvsswkKCiIoKIjMzExmz55NUFAQSUlJusYeFhsby+mnn84vv/xiiX/DCjL1FBISQvfu3fnkk09c+5xOJ5988gm9e/f2YmW+KSMjg+TkZLfrnZ+fzzfffKPrXUeGYXDnnXfy9ttv8+mnn5KRkeF2vHv37gQHB7td4y1btrBr1y5d45PkdDopKyvTtfWAAQMGsGnTJtavX+/aevTowahRo1zPdY09q7CwkK1bt5KSkmKNf8PeHm1sRQsXLjTsdrsxf/5844cffjDGjh1rxMbGGtnZ2d4uzZIKCgqMdevWGevWrTMA46mnnjLWrVtn7Ny50zAMw3jssceM2NhY4z//+Y+xceNG4/LLLzcyMjKMkpISL1duDXfccYcRExNjfP7558bevXtdW3Fxseuc22+/3WjdurXx6aefGqtXrzZ69+5t9O7d24tVW8fEiRONzMxMY/v27cbGjRuNiRMnGjabzfjoo48Mw9C1bQhH37VkGLrGp+pPf/qT8fnnnxvbt283VqxYYQwcONBo3ry5kZOTYxhG07++CjIn6ZlnnjFat25thISEGL/73e+MlStXersky/rss88MoMY2evRowzDMW7AfeOABIykpybDb7caAAQOMLVu2eLdoC6nt2gLGvHnzXOeUlJQY48aNM+Li4ozw8HDjyiuvNPbu3eu9oi3k5ptvNtLS0oyQkBAjISHBGDBggCvEGIaubUP4bZDRNT41I0eONFJSUoyQkBCjRYsWxsiRI41ffvnFdbypX1+bYRiGd9qCRERERE6NxsiIiIiIZSnIiIiIiGUpyIiIiIhlKciIiIiIZSnIiIiIiGUpyIiIiIhlKciIiIiIZSnIiIiIiGUpyIiIz7PZbCxdutTbZYhIA1CQEZEGNWbMGGw2W41tyJAh3i5NRHxAkLcLEBHfN2TIEObNm+e2z263e6kaEfElapERkQZnt9tJTk522+Li4gCz22fOnDkMHTqUsLAw2rRpw5IlS9z+ftOmTVx44YWEhYXRrFkzxo4dS2Fhods5r7zyCp06dcJut5OSksKdd97pdvzAgQNceeWVhIeH065dO9555x3XscOHDzNq1CgSEhIICwujXbt2NYKXiDRNCjIi4nUPPPAAw4cPZ8OGDYwaNYprr72WH3/8EYCioiIuuugi4uLiWLVqFYsXL+bjjz92Cypz5sxh/PjxjB07lk2bNvHOO+9w2mmnuX3GtGnTGDFiBBs3buTiiy9m1KhRHDp0yPX5P/zwA8uWLePHH39kzpw5NG/evPEugIicPG8vvy0ivm306NFGYGCgERER4bY98sgjhmEYBmDcfvvtbn/Tq1cv44477jAMwzBefPFFIy4uzigsLHQd/+9//2sEBAQY2dnZhmEYRmpqqvHXv/71mDUAxt/+9jfX68LCQgMwli1bZhiGYQwbNsy46aabPPOFRaRRaYyMiDS4/v37M2fOHLd98fHxrue9e/d2O9a7d2/Wr18PwI8//kjXrl2JiIhwHe/Tpw9Op5MtW7Zgs9nIyspiwIABx62hS5curucRERFER0eTk5MDwB133MHw4cNZu3YtgwcP5oorruDcc889qe8qIo1LQUZEGlxERESNrh5PCQsLq9N5wcHBbq9tNhtOpxOAoUOHsnPnTt5//32WL1/OgAEDGD9+PE8++aTH6xURz9IYGRHxupUrV9Z43bFjRwA6duzIhg0bKCoqch1fsWIFAQEBtG/fnqioKNLT0/nkk09OqYaEhARGjx7N66+/zqxZs3jxxRdP6f1EpHGoRUZEGlxZWRnZ2dlu+4KCglwDahcvXkyPHj0477zzeOONN/j222/5xz/+AcCoUaOYMmUKo0ePZurUqezfv5+77rqL3//+9yQlJQEwdepUbr/9dhITExk6dCgFBQWsWLGCu+66q071TZ48me7du9OpUyfKysp47733XEFKRJo2BRkRaXAffPABKSkpbvvat2/P5s2bAfOOooULFzJu3DhSUlJYsGABZ5xxBgDh4eF8+OGH3HPPPfTs2ZPw8HCGDx/OU0895Xqv0aNHU1payt///nfuu+8+mjdvztVXX13n+kJCQpg0aRI7duwgLCyM888/n4ULF3rgm4tIQ7MZhmF4uwgR8V82m423336bK664wtuliIgFaYyMiIiIWJaCjIiIiFiWxsiIiFepd1tEToVaZERERMSyFGRERETEshRkRERExLIUZERERMSyFGRERETEshRkRERExLIUZERERMSyFGRERETEsv4frCfIlEAJvBwAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.suptitle('MSE vs Epochs')\n",
    "plt.plot(train_err, label='Train', color='blue')\n",
    "plt.plot(test_err, label='Test', color='red')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('MSE')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "id": "LoGEmM5lH7_A"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[1273.,  144.,   13., ...,    0.,    0.,    0.],\n",
       "        [   0.,    0.,    0., ...,    0.,    0.,    0.],\n",
       "        [   0.,    0.,    0., ...,    0.,    0.,    0.],\n",
       "        ...,\n",
       "        [   0.,    0.,    0., ...,    0.,    0.,    0.],\n",
       "        [   0.,    0.,    0., ...,    0.,    0.,    0.],\n",
       "        [   0.,    0.,    0., ...,   16.,   21.,  197.]]),\n",
       " array([1.   , 1.14 , 1.28 , 1.42 , 1.561, 1.7  , 1.84 , 1.98 , 2.121,\n",
       "        2.26 , 2.4  , 2.54 , 2.68 , 2.82 , 2.96 , 3.1  , 3.24 , 3.38 ,\n",
       "        3.52 , 3.66 , 3.8  , 3.94 , 4.08 , 4.22 , 4.36 , 4.5  , 4.64 ,\n",
       "        4.78 , 4.92 , 5.06 , 5.2  , 5.34 , 5.48 , 5.62 , 5.76 , 5.902,\n",
       "        6.04 , 6.18 , 6.32 , 6.46 , 6.6  , 6.742, 6.88 , 7.02 , 7.16 ,\n",
       "        7.3  , 7.44 , 7.582, 7.72 , 7.86 , 8.   ], dtype=float16),\n",
       " array([0.949, 1.032, 1.115, 1.198, 1.281, 1.363, 1.446, 1.529, 1.612,\n",
       "        1.695, 1.778, 1.861, 1.943, 2.027, 2.11 , 2.191, 2.275, 2.36 ,\n",
       "        2.441, 2.523, 2.607, 2.69 , 2.773, 2.855, 2.938, 3.021, 3.104,\n",
       "        3.188, 3.27 , 3.354, 3.436, 3.52 , 3.602, 3.684, 3.768, 3.85 ,\n",
       "        3.934, 4.016, 4.1  , 4.18 , 4.266, 4.348, 4.43 , 4.516, 4.594,\n",
       "        4.68 , 4.76 , 4.844, 4.93 , 5.01 , 5.094], dtype=float16),\n",
       " <matplotlib.collections.QuadMesh at 0x7e0eafbe4950>)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAnyElEQVR4nO3df3SU9YHv8c8kIRPEzEhY8gOSIDZKJBB+SWVCa7SCNMvhkPaerMtlNyjoufaGNUilNm53/e3gpSxLl94AVoldTbOiEloEMUIDlw1oAmRPoqfUKJKISehRmSFRB0jm/tFt2kh+zEwIz3eS9+uc54958n2e55OnNPn4zfPD5vf7/QIAADBIhNUBAAAAvo6CAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwTpTVAQLR2dmpTz75RLGxsbLZbFbHAQAAAfD7/Tp37pzGjRuniIjg5kTCoqB88sknSklJsToGAAAIQVNTk5KTk4PaJiwKSmxsrKQ/foMOh8PiNDDdrf97U79jKv/vyiuQxBzffOTnAY1757GCQU5ilvn7/7nfMRXfefwKJDHH9w49FNC4Hd96ZpCTmGN59f0BjXt+9s8GOYlZAjkvF9rP69eLf9X1ezwYYVFQ/vRnHYfDQUFBvyKjY/odM9z+HUXa+z8n0vA7L1Gj7P2O4Zz0bDidlxGjogMaN5zOiRT4eZEU0uUZXCQLAACME1RBefTRR2Wz2bot6enpfW6zfft2paenKyYmRlOnTtXu3bsHFBgAAAx9Qc+gZGRkqLm5uWs5dOhQr2Orqqq0ZMkSrVixQsePH1dubq5yc3NVX18/oNAAAGBoC7qgREVFKTExsWv5q7/6q17Hbty4Ud/97ne1Zs0a3XjjjXriiSc0c+ZMbdrU/0WMAABg+Ar6Itn3339f48aNU0xMjFwul9xut1JTU3sce/jwYa1evbrbugULFqi8vLzPY/h8Pvl8vq7PXq832JjDQk7amoDG7WlYN8hJzNI+jmflfN1VZ/xWRzDS6Y/HWB3BOB8cmRDYwOzBzWGSo/tuDGzgnMHNYZrjb/R/Xjp8X4W8/6BmUG6++WaVlJTojTfeUHFxsU6ePKlvf/vbOnfuXI/jW1palJCQ0G1dQkKCWlpa+jyO2+2W0+nsWngGCgAAw0tQBSUnJ0d5eXnKzMzUggULtHv3bp09e1Yvv/zyZQ1VVFQkj8fTtTQ1NV3W/QMAALMN6Dko11xzjW644QY1NDT0+PXExES1trZ2W9fa2qrExMQ+92u322W3B3YvPgAAGHoG9ByUtrY2ffDBB0pKSurx6y6XS/v27eu2rqKiQi6XayCHBQAAQ1xQBeXBBx/UgQMH9NFHH6mqqkrf+973FBkZqSVLlkiS8vPzVVRU1DW+sLBQb7zxhtavX6/f/e53evTRR1VTU6OVK4fXY8YBAEBwgvoTz8cff6wlS5bo008/1dixY/Wtb31LR44c0dixYyVJjY2N3d5WmJWVpdLSUv3kJz/Rww8/rOuvv17l5eWaMmXK5f0uAADAkBJUQSkrK+vz65WVlZesy8vLU15eXlChEJiOuOBfvjQcRJy3OgHCRfQnI6yOYJzRJ7gl/evG1HdaHcFIUV/0P8bm639Mb3gXDwAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOAN6Fw+sFflZz2+RHu7GvHvB6gjGibhodQIz2T+zOoF5HB8N4MEVQ9ThX/3Q6ghGSqj5qt8xFy9+pd+FuH9mUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjMNtxhhyPBNHWB3BOCPaeV18T67+hPPydb7R/P/n627/jjugcfv2Fw1yErNEN/Z/n35EZ+i3rTODAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDs9BwZBj9/qtjmCczyZFWh0BYeKqpi+sjmCcyLYLVkcw0rlpCf2OuXjhK+lkaPtnBgUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDjcZhzGLn74kdURECbGr60KbODTDwxuEMNcU91idQTjvPnOP1sdwTj+mjqrIxhp5I63+x1z0R/6LdrMoAAAAONQUAAAgHEoKAAAwDgDKihr166VzWbTqlWreh1TUlIim83WbYmJiRnIYQEAwBAX8kWy1dXV2rJlizIzM/sd63A4dOLEia7PNpst1MMCAIBhIKQZlLa2Ni1dulTPPvusRo8e3e94m82mxMTEriUhof8XDAEAgOErpBmUgoICLVy4UPPmzdOTTz7Z7/i2tjZNmDBBnZ2dmjlzpp5++mllZGSEcmj8hajrrrU6gpEiLvA24687e5fL6ghG2tOwzuoIxslJWxPQuOF07vhZa42gC0pZWZmOHTum6urqgMZPmjRJzz//vDIzM+XxePTTn/5UWVlZevfdd5WcnNzjNj6fTz6fr+uz1+sNNiYAAAhjQf2Jp6mpSYWFhXrppZcCvtDV5XIpPz9f06dPV3Z2tl577TWNHTtWW7Zs6XUbt9stp9PZtaSkpAQTEwAAhLmgCsrRo0d15swZzZw5U1FRUYqKitKBAwf0s5/9TFFRUero6Oh3HyNGjNCMGTPU0NDQ65iioiJ5PJ6upampKZiYAAAgzAX1J57bb79ddXXdH/l79913Kz09XQ899JAiIyP73UdHR4fq6ur013/9172OsdvtstvtwUQDAABDSFAFJTY2VlOmTOm2btSoURozZkzX+vz8fI0fP15ut1uS9Pjjj2vOnDlKS0vT2bNntW7dOp06dUr33HPPZfoWAADAUHPZXxbY2NioiIg//+Xo888/17333quWlhaNHj1as2bNUlVVlSZPnny5Dw0AAIaIAReUysrKPj9v2LBBGzZsGOhhAADAMHLZZ1Bw5Vz88COrIxjpqtbzVkcwzmdT+h8zHAXyzI/h9LwPiZ8rPemIi7U6gpEqOrf3O8br9crpdIa0f14WCAAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHG4zDmO8Arxn3mt5TcLXXXT0/56s4eiLSWOtjmCc8zmzrY5gHH9NXf+DcNkxgwIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA7PQcGQE/Wl3+oIxhl5mv+rIzDRe6qtjgBIYgYFAAAYiIICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA43HsYxjriYq2OYKRob4fVEYzj5z9FenTViT9YHcE4FZ3brY5gnKjrrrU6wrDEjy0AAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONwm3EY842NsTqCkb5I4J/119m487pH51PjrI6AMLCnYZ3VEYYlZlAAAIBxKCgAAMA4FBQAAGAcCgoAADDOgArK2rVrZbPZtGrVqj7Hbd++Xenp6YqJidHUqVO1e/fugRwWAAAMcSEXlOrqam3ZskWZmZl9jquqqtKSJUu0YsUKHT9+XLm5ucrNzVV9fX2ohwYAAENcSAWlra1NS5cu1bPPPqvRo0f3OXbjxo367ne/qzVr1ujGG2/UE088oZkzZ2rTpk0hBQYAAENfSA+MKCgo0MKFCzVv3jw9+eSTfY49fPiwVq9e3W3dggULVF5e3us2Pp9PPp+v67PX6w0l5pAX9SUPt+iJ4yNf/4OGG5vVAcwUUXnM6gjGyUlbE9C44fRsEM6JNYIuKGVlZTp27Jiqq6sDGt/S0qKEhIRu6xISEtTS0tLrNm63W4899liw0QAAwBAR1J94mpqaVFhYqJdeekkxMYP3FNOioiJ5PJ6upampadCOBQAAzBPUDMrRo0d15swZzZw5s2tdR0eHDh48qE2bNsnn8ykyMrLbNomJiWptbe22rrW1VYmJib0ex263y263BxMNAAAMIUHNoNx+++2qq6tTbW1t13LTTTdp6dKlqq2tvaScSJLL5dK+ffu6rauoqJDL5RpYcgAAMGQFNYMSGxurKVOmdFs3atQojRkzpmt9fn6+xo8fL7fbLUkqLCxUdna21q9fr4ULF6qsrEw1NTXaunXrZfoWAADAUHPZnyTb2Nio5ubmrs9ZWVkqLS3V1q1bNW3aNL3yyisqLy+/pOgAAAD8yYDfS19ZWdnnZ0nKy8tTXl7eQA+Fr4lu/MzqCEZqS+b6JQSm89aZ/Q/CsHfxw4+sjmCk+RH9/16/6L8Q8v55Fw8AADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgDfg4KrMOrvXs2or3T6gjGcX7gtzoCwgQ/VxCois7t/Y7xer1yOp0h7Z8ZFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA43CbcRgL5FXXUmC3gg0lVzV9YXUE43wx1mZ1BCNFN35mdQSEgajrrrU6gpHu+Obj/Y652PFVyPtnBgUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDjcZhzGbDdNtTqCkb5IucrqCMZJ+n/nrI5gpIsffmR1BIQB3vDcM39NXf9j/BdC3j8zKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA4/AclDAW+RnPtuhJR7TN6gjGCeR5BcNRRed2qyMYZ35EXkDjhtO545z0LOq6a/sf1OmTToa2f2ZQAACAcSgoAADAOBQUAABgHAoKAAAwTlAFpbi4WJmZmXI4HHI4HHK5XNqzZ0+v40tKSmSz2botMTExAw4NAACGtqDu4klOTtbatWt1/fXXy+/364UXXtDixYt1/PhxZWRk9LiNw+HQiRMnuj7bbNxhAQAA+hZUQVm0aFG3z0899ZSKi4t15MiRXguKzWZTYmJi6AnRq/OpcVZHMNL5WP5y+XUB3Q4IaPjdKhsIzknP9jSs63eM1+uV0/lvIe0/5J/kHR0dKisrU3t7u1wuV6/j2traNGHCBKWkpGjx4sV69913+923z+eT1+vttgAAgOEj6IJSV1enq6++Wna7Xffdd5927NihyZMn9zh20qRJev7557Vz5069+OKL6uzsVFZWlj7++OM+j+F2u+V0OruWlJSUYGMCAIAwFnRBmTRpkmpra/X222/rBz/4gZYtW6b33nuvx7Eul0v5+fmaPn26srOz9dprr2ns2LHasmVLn8coKiqSx+PpWpqamoKNCQAAwljQj7qPjo5WWlqaJGnWrFmqrq7Wxo0b+y0dkjRixAjNmDFDDQ0NfY6z2+2y2+3BRgMAAEPEgK8m7OzslM/nC2hsR0eH6urqlJSUNNDDAgCAISyoGZSioiLl5OQoNTVV586dU2lpqSorK7V3715JUn5+vsaPHy+32y1JevzxxzVnzhylpaXp7NmzWrdunU6dOqV77rnn8n8nAABgyAiqoJw5c0b5+flqbm6W0+lUZmam9u7dq/nz50uSGhsbFRHx50mZzz//XPfee69aWlo0evRozZo1S1VVVb1eVAsAACAFWVCee+65Pr9eWVnZ7fOGDRu0YcOGoEMhMBGVx6yOYKSzN/itjoAwMT8ir98xw+0ZGIGcE2n4nRdcKpB/Kxf9F0LeP0+0AgAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwTtCPuoc5bDdNtTqCkUa02ayOgDARdd21VkdAGMhJWxPQuD0N6wY5iVkCudXc6/XK6XSGtH9mUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjMNtxmEs8rNzVkcw0vlJX1odwTifZSVZHcFIFz/8yOoICAPD7fZhUzCDAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDs9BCWPnU+OsjmAke8wFqyMY52KMzeoIRgrkdfEAejY/Iq/fMRf9of88ZgYFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA43GaMIedLb4zVEYwz8tMOqyMY6Y5vPt7vmDff+ecrkMQc3Hp9qUD+nUjD79/KYGMGBQAAGIeCAgAAjENBAQAAxqGgAAAA4wRVUIqLi5WZmSmHwyGHwyGXy6U9e/b0uc327duVnp6umJgYTZ06Vbt37x5QYAAAMPQFVVCSk5O1du1aHT16VDU1NfrOd76jxYsX69133+1xfFVVlZYsWaIVK1bo+PHjys3NVW5ururr6y9LeAAAMDTZ/H6/fyA7iIuL07p167RixYpLvnbnnXeqvb1du3bt6lo3Z84cTZ8+XZs3bw74GF6vV06nUx6PRw6HYyBxh5Rv/Y+fBjTu0KsPDnISs2Qv/D/9jjnw+o+uQBJz5KStCWjcnoZ1g5wEwHAykN/fIV+D0tHRobKyMrW3t8vlcvU45vDhw5o3b163dQsWLNDhw4f73LfP55PX6+22AACA4SPoglJXV6err75adrtd9913n3bs2KHJkyf3OLalpUUJCQnd1iUkJKilpaXPY7jdbjmdzq4lJSUl2JgAACCMBV1QJk2apNraWr399tv6wQ9+oGXLlum99967rKGKiork8Xi6lqampsu6fwAAYLagH3UfHR2ttLQ0SdKsWbNUXV2tjRs3asuWLZeMTUxMVGtra7d1ra2tSkxM7PMYdrtddrs92GgAAGCIGPBzUDo7O+Xz+Xr8msvl0r59+7qtq6io6PWaFQAAACnIGZSioiLl5OQoNTVV586dU2lpqSorK7V3715JUn5+vsaPHy+32y1JKiwsVHZ2ttavX6+FCxeqrKxMNTU12rp16+X/TgAAwJARVEE5c+aM8vPz1dzcLKfTqczMTO3du1fz58+XJDU2Nioi4s+TMllZWSotLdVPfvITPfzww7r++utVXl6uKVOmXN7vAgAADClBFZTnnnuuz69XVlZesi4vL095eXlBhQIGIurLDqsjGIfnm/RsfkT/P5sqOrdfgSTmCOScSMPvvODK4108AADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADGCfpR9zCHz0m/7El042dWR0CY4FbZS3FOEKictDX9jrnY2fOT5gPBbzgAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHF4DkoYs3s6rY5gpHPTEqyOAGAImR+RF9C44fYMmT0N6/od4/V65XT+W0j7ZwYFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA43GYcxr4cE2l1BCO1JXFevo7bJAGEG2ZQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMw23GGHJGfspbngFcPtx+bw1mUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGCeoguJ2uzV79mzFxsYqPj5eubm5OnHiRJ/blJSUyGazdVtiYmIGFBoAAAxtQRWUAwcOqKCgQEeOHFFFRYUuXLigO+64Q+3t7X1u53A41Nzc3LWcOnVqQKEBAMDQFtRzUN54441un0tKShQfH6+jR4/qlltu6XU7m82mxMTE0BKiV1/FWZ3ATFe1nrc6gnF4jkPP5kfk9TuGcwdYY0DXoHg8HklSXFzfvynb2to0YcIEpaSkaPHixXr33Xf7HO/z+eT1erstAABg+Ai5oHR2dmrVqlWaO3eupkyZ0uu4SZMm6fnnn9fOnTv14osvqrOzU1lZWfr444973cbtdsvpdHYtKSkpocYEAABhKOSCUlBQoPr6epWVlfU5zuVyKT8/X9OnT1d2drZee+01jR07Vlu2bOl1m6KiInk8nq6lqakp1JgAACAMhfQunpUrV2rXrl06ePCgkpOTg9p2xIgRmjFjhhoaGnodY7fbZbfbQ4kGAACGgKBmUPx+v1auXKkdO3Zo//79mjhxYtAH7OjoUF1dnZKSkoLeFgAADA9BzaAUFBSotLRUO3fuVGxsrFpaWiRJTqdTI0eOlCTl5+dr/PjxcrvdkqTHH39cc+bMUVpams6ePat169bp1KlTuueeey7ztwIAAIaKoApKcXGxJOnWW2/ttn7btm266667JEmNjY2KiPjzxMznn3+ue++9Vy0tLRo9erRmzZqlqqoqTZ48eWDJIX9If6ADAMB8Qf2K8/v9/Y6prKzs9nnDhg3asGFDUKEAAMDwxrt4AACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMw5M0wthXY/q/7Xs4im78zOoICBMVndutjgCgF8ygAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh9uMw1kktxn35OKHH1kdAQhb8yPyAhrHLdoYbMygAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMw3NQwljUuC+sjgBgiOH5JjAFMygAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMbhNuMwdn38H6yOYCRukwSA8McMCgAAMA4FBQAAGIeCAgAAjENBAQAAxgmqoLjdbs2ePVuxsbGKj49Xbm6uTpw40e9227dvV3p6umJiYjR16lTt3r075MAAAGDoC6qgHDhwQAUFBTpy5IgqKip04cIF3XHHHWpvb+91m6qqKi1ZskQrVqzQ8ePHlZubq9zcXNXX1w84PAAAGJpsfr/fH+rGf/jDHxQfH68DBw7olltu6XHMnXfeqfb2du3atatr3Zw5czR9+nRt3rw5oON4vV45nU55PB45HI5Q4w458ysfCGhcxa0bBjkJTDc/Ii+gcdyiDeByGsjv7wFdg+LxeCRJcXFxvY45fPiw5s2b123dggULdPjw4YEcGgAADGEhP6its7NTq1at0ty5czVlypRex7W0tCghIaHbuoSEBLW0tPS6jc/nk8/n6/rs9XpDjQkAAMJQyDMoBQUFqq+vV1lZ2eXMI+mPF+M6nc6uJSUl5bIfAwAAmCukgrJy5Urt2rVLv/3tb5WcnNzn2MTERLW2tnZb19raqsTExF63KSoqksfj6VqamppCiQkAAMJUUAXF7/dr5cqV2rFjh/bv36+JEyf2u43L5dK+ffu6rauoqJDL5ep1G7vdLofD0W0BAADDR1DXoBQUFKi0tFQ7d+5UbGxs13UkTqdTI0eOlCTl5+dr/PjxcrvdkqTCwkJlZ2dr/fr1WrhwocrKylRTU6OtW7de5m8FAAAMFUHNoBQXF8vj8ejWW29VUlJS1/If//EfXWMaGxvV3Nzc9TkrK0ulpaXaunWrpk2bpldeeUXl5eV9XlgLAACGtwE9B+VK4TkoPcs5WBjQuD23bBzkJAAAXMqy56AAAAAMBgoKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjhPyyQFjvxMcJ/Q8CACAMMYMCAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcbjMOYwm/sQc28H8Obg4AAC43ZlAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjBNldQCE7vCvfmh1BAAABgUzKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjBN0QTl48KAWLVqkcePGyWazqby8vM/xlZWVstlslywtLS2hZgYAAENc0AWlvb1d06ZN089//vOgtjtx4oSam5u7lvj4+GAPDQAAhomgn4OSk5OjnJycoA8UHx+va665JujtAADA8HPFrkGZPn26kpKSNH/+fP3nf/5nn2N9Pp+8Xm+3BQAADB+DXlCSkpK0efNmvfrqq3r11VeVkpKiW2+9VceOHet1G7fbLafT2bWkpKQMdkwAAGAQm9/v94e8sc2mHTt2KDc3N6jtsrOzlZqaqn//93/v8es+n08+n6/rs9frVUpKijwejxwOR6hxAQDAFeT1euV0OkP6/W3Ju3i++c1v6tChQ71+3W63y263X8FEAADAJJY8B6W2tlZJSUlWHBoAAISBoGdQ2tra1NDQ0PX55MmTqq2tVVxcnFJTU1VUVKTTp0/rl7/8pSTpX//1XzVx4kRlZGToq6++0i9+8Qvt379fb7755uX7LgAAwJASdEGpqanRbbfd1vV59erVkqRly5appKREzc3Namxs7Pr6+fPn9cMf/lCnT5/WVVddpczMTL311lvd9gEAAPCXBnSR7JUykItsAACANQby+5t38QAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwTpTVAYKx2JmvKNuIPsdUdG6/QmkAAMBgYQYFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4YXWb8R/+182KtMdYHQMAAAwyZlAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIwTVrcZj154WlGj7FbHAAAAg4wZFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA4wRdUA4ePKhFixZp3LhxstlsKi8v73ebyspKzZw5U3a7XWlpaSopKQkhKgAAGC6Cfg5Ke3u7pk2bpuXLl+v73/9+v+NPnjyphQsX6r777tNLL72kffv26Z577lFSUpIWLFgQ1LF/k75HjtjIfkatD2qfAADAPEEXlJycHOXk5AQ8fvPmzZo4caLWr/9jcbjxxht16NAhbdiwIeiCAgAAhodBvwbl8OHDmjdvXrd1CxYs0OHDh3vdxufzyev1dlsAAMDwMegFpaWlRQkJCd3WJSQkyOv16ssvv+xxG7fbLafT2bWkpKQMdkwAAGAQI+/iKSoqksfj6VqampqsjgQAAK6gQX9ZYGJiolpbW7uta21tlcPh0MiRI3vcxm63y27npYAAAAxXg15QXC6Xdu/e3W1dRUWFXC5XwPvw+/2SJG9bZ79jI67iehUAAEzwp2tI//R7PBhBF5S2tjY1NDR0fT558qRqa2sVFxen1NRUFRUV6fTp0/rlL38pSbrvvvu0adMm/ehHP9Ly5cu1f/9+vfzyy3r99dcDPuann34qSZow86MARjuD+XYAAMAg+/TTT+V0Bvf7OeiCUlNTo9tuu63r8+rVqyVJy5YtU0lJiZqbm9XY2Nj19YkTJ+r111/XAw88oI0bNyo5OVm/+MUvgrrFOC4uTpLU2NgY9Dc41Hm9XqWkpKipqUkOh8PqOMbgvPSOc9M7zk3vODe949z0zuPxKDU1tev3eDBs/lDmXa4wr9crp9Mpj8fD//hfw7npGeeld5yb3nFuese56R3npncDOTdG3sUDAACGNwoKAAAwTlgUFLvdrkceeYRbj3vAuekZ56V3nJvecW56x7npHeemdwM5N2FxDQoAABhewmIGBQAADC8UFAAAYBwKCgAAMA4FBQAAGMfognLw4EEtWrRI48aNk81mU3l5udWRjOB2uzV79mzFxsYqPj5eubm5OnHihNWxjFBcXKzMzEw5HA45HA65XC7t2bPH6lhGWrt2rWw2m1atWmV1FMs9+uijstls3Zb09HSrYxnj9OnT+ru/+zuNGTNGI0eO1NSpU1VTU2N1LMtde+21l/y7sdlsKigosDqapTo6OvRP//RPmjhxokaOHKlvfOMbeuKJJ4J+H8+gvyxwINrb2zVt2jQtX75c3//+962OY4wDBw6ooKBAs2fP1sWLF/Xwww/rjjvu0HvvvadRo0ZZHc9SycnJWrt2ra6//nr5/X698MILWrx4sY4fP66MjAyr4xmjurpaW7ZsUWZmptVRjJGRkaG33nqr63NUlNE/Hq+Yzz//XHPnztVtt92mPXv2aOzYsXr//fc1evRoq6NZrrq6Wh0dHV2f6+vrNX/+fOXl5VmYynrPPPOMiouL9cILLygjI0M1NTW6++675XQ6df/99we8H6P/H5iTk6OcnByrYxjnjTfe6Pa5pKRE8fHxOnr0qG655RaLUplh0aJF3T4/9dRTKi4u1pEjRygo/62trU1Lly7Vs88+qyeffNLqOMaIiopSYmKi1TGM88wzzyglJUXbtm3rWjdx4kQLE5lj7Nix3T6vXbtW3/jGN5SdnW1RIjNUVVVp8eLFWrhwoaQ/zjT96le/0jvvvBPUfoz+Ew8C4/F4JCmklzENZR0dHSorK1N7e7tcLpfVcYxRUFCghQsXat68eVZHMcr777+vcePG6brrrtPSpUu7vfR0OPv1r3+tm266SXl5eYqPj9eMGTP07LPPWh3LOOfPn9eLL76o5cuXy2azWR3HUllZWdq3b59+//vfS5L+67/+S4cOHQp6wsHoGRT0r7OzU6tWrdLcuXM1ZcoUq+MYoa6uTi6XS1999ZWuvvpq7dixQ5MnT7Y6lhHKysp07NgxVVdXWx3FKDfffLNKSko0adIkNTc367HHHtO3v/1t1dfXKzY21up4lvrwww9VXFys1atX6+GHH1Z1dbXuv/9+RUdHa9myZVbHM0Z5ebnOnj2ru+66y+oolvvxj38sr9er9PR0RUZGqqOjQ0899ZSWLl0a1H4oKGGuoKBA9fX1OnTokNVRjDFp0iTV1tbK4/HolVde0bJly3TgwIFhX1KamppUWFioiooKxcTEWB3HKH/5X3aZmZm6+eabNWHCBL388stasWKFhcms19nZqZtuuklPP/20JGnGjBmqr6/X5s2bKSh/4bnnnlNOTo7GjRtndRTLvfzyy3rppZdUWlqqjIwM1dbWatWqVRo3blxQ/2YoKGFs5cqV2rVrlw4ePKjk5GSr4xgjOjpaaWlpkqRZs2apurpaGzdu1JYtWyxOZq2jR4/qzJkzmjlzZte6jo4OHTx4UJs2bZLP51NkZKSFCc1xzTXX6IYbblBDQ4PVUSyXlJR0Sbm/8cYb9eqrr1qUyDynTp3SW2+9pddee83qKEZYs2aNfvzjH+tv//ZvJUlTp07VqVOn5Ha7KShDnd/v1z/8wz9ox44dqqys5IK1fnR2dsrn81kdw3K333676urquq27++67lZ6eroceeohy8hfa2tr0wQcf6O///u+tjmK5uXPnXvIYg9///veaMGGCRYnMs23bNsXHx3ddFDrcffHFF4qI6H6Ja2RkpDo7O4Paj9EFpa2trdt/wZw8eVK1tbWKi4tTamqqhcmsVVBQoNLSUu3cuVOxsbFqaWmRJDmdTo0cOdLidNYqKipSTk6OUlNTde7cOZWWlqqyslJ79+61OprlYmNjL7lOadSoURozZsywv37pwQcf1KJFizRhwgR98skneuSRRxQZGaklS5ZYHc1yDzzwgLKysvT000/rb/7mb/TOO+9o69at2rp1q9XRjNDZ2alt27Zp2bJl3Jr+3xYtWqSnnnpKqampysjI0PHjx/Uv//IvWr58eXA78hvst7/9rV/SJcuyZcusjmapns6JJP+2bdusjma55cuX+ydMmOCPjo72jx071n/77bf733zzTatjGSs7O9tfWFhodQzL3Xnnnf6kpCR/dHS0f/z48f4777zT39DQYHUsY/zmN7/xT5kyxW+32/3p6en+rVu3Wh3JGHv37vVL8p84ccLqKMbwer3+wsJCf2pqqj8mJsZ/3XXX+f/xH//R7/P5gtqPze8P8tFuAAAAg4znoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgnP8PZxaERTu8tggAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "batch_src, batch_labels, batch_padding_mask = mkbatch(4096)\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    output = model(batch_src, batch_padding_mask)\n",
    "batch_src[0], batch_labels[0], output[0]\n",
    "x = batch_labels.detach().to(torch.float16).cpu().numpy().flatten()\n",
    "y = output.detach().to(torch.float16).cpu().numpy().flatten()\n",
    "plt.hist2d(x, y, bins=50, norm=mpl.colors.LogNorm())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LC6Xv3YfC0Rm"
   },
   "source": [
    "# Step 5: Fine Tune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_TUNE_EPOCHS = 10\n",
    "TUNE_LR = 1e-5\n",
    "TUNE_WD = 0 # 1e-5\n",
    "\n",
    "tune_criterion = nn.MSELoss()\n",
    "tune_optimizer = torch.optim.Adam(model.parameters(), lr=TUNE_LR, weight_decay=TUNE_WD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tune_evaluate():\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    with torch.no_grad():\n",
    "        batch_src, batch_labels, batch_padding_mask = mktunebatch(BSZ)\n",
    "        output = model(batch_src, batch_padding_mask)\n",
    "        loss = criterion(output.squeeze(1), batch_labels)\n",
    "    return loss.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1",
      "0",
      "0",
      "%",
      "|",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "|",
      " ",
      "1",
      "6",
      "/",
      "1",
      "6",
      " ",
      "[",
      "0",
      "0",
      ":",
      "1",
      "3",
      "<",
      "0",
      "0",
      ":",
      "0",
      "0",
      ",",
      " ",
      " ",
      "1",
      ".",
      "1",
      "9",
      "i",
      "t",
      "/",
      "s",
      "]",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "E",
      "p",
      "o",
      "c",
      "h",
      " ",
      "1",
      "/",
      "1",
      "0",
      "0",
      "0",
      " ",
      "\t",
      " ",
      "T",
      "r",
      "a",
      "i",
      "n",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "2",
      "0",
      "0",
      "1",
      " ",
      "\t",
      " ",
      "T",
      "e",
      "s",
      "t",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "1",
      "5",
      "9",
      "2",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1",
      "0",
      "0",
      "%",
      "|",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "|",
      " ",
      "1",
      "6",
      "/",
      "1",
      "6",
      " ",
      "[",
      "0",
      "0",
      ":",
      "1",
      "3",
      "<",
      "0",
      "0",
      ":",
      "0",
      "0",
      ",",
      " ",
      " ",
      "1",
      ".",
      "2",
      "0",
      "i",
      "t",
      "/",
      "s",
      "]",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "E",
      "p",
      "o",
      "c",
      "h",
      " ",
      "2",
      "/",
      "1",
      "0",
      "0",
      "0",
      " ",
      "\t",
      " ",
      "T",
      "r",
      "a",
      "i",
      "n",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "1",
      "3",
      "5",
      "5",
      " ",
      "\t",
      " ",
      "T",
      "e",
      "s",
      "t",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "1",
      "1",
      "5",
      "2",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1",
      "0",
      "0",
      "%",
      "|",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "|",
      " ",
      "1",
      "6",
      "/",
      "1",
      "6",
      " ",
      "[",
      "0",
      "0",
      ":",
      "1",
      "3",
      "<",
      "0",
      "0",
      ":",
      "0",
      "0",
      ",",
      " ",
      " ",
      "1",
      ".",
      "1",
      "7",
      "i",
      "t",
      "/",
      "s",
      "]",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "E",
      "p",
      "o",
      "c",
      "h",
      " ",
      "3",
      "/",
      "1",
      "0",
      "0",
      "0",
      " ",
      "\t",
      " ",
      "T",
      "r",
      "a",
      "i",
      "n",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "1",
      "0",
      "3",
      "9",
      " ",
      "\t",
      " ",
      "T",
      "e",
      "s",
      "t",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "0",
      "9",
      "6",
      "2",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1",
      "0",
      "0",
      "%",
      "|",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "|",
      " ",
      "1",
      "6",
      "/",
      "1",
      "6",
      " ",
      "[",
      "0",
      "0",
      ":",
      "1",
      "3",
      "<",
      "0",
      "0",
      ":",
      "0",
      "0",
      ",",
      " ",
      " ",
      "1",
      ".",
      "1",
      "7",
      "i",
      "t",
      "/",
      "s",
      "]",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "E",
      "p",
      "o",
      "c",
      "h",
      " ",
      "4",
      "/",
      "1",
      "0",
      "0",
      "0",
      " ",
      "\t",
      " ",
      "T",
      "r",
      "a",
      "i",
      "n",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "0",
      "8",
      "7",
      "7",
      " ",
      "\t",
      " ",
      "T",
      "e",
      "s",
      "t",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "0",
      "8",
      "1",
      "5",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1",
      "0",
      "0",
      "%",
      "|",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "|",
      " ",
      "1",
      "6",
      "/",
      "1",
      "6",
      " ",
      "[",
      "0",
      "0",
      ":",
      "1",
      "3",
      "<",
      "0",
      "0",
      ":",
      "0",
      "0",
      ",",
      " ",
      " ",
      "1",
      ".",
      "1",
      "8",
      "i",
      "t",
      "/",
      "s",
      "]",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "E",
      "p",
      "o",
      "c",
      "h",
      " ",
      "5",
      "/",
      "1",
      "0",
      "0",
      "0",
      " ",
      "\t",
      " ",
      "T",
      "r",
      "a",
      "i",
      "n",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "0",
      "7",
      "7",
      "5",
      " ",
      "\t",
      " ",
      "T",
      "e",
      "s",
      "t",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "0",
      "7",
      "2",
      "8",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1",
      "0",
      "0",
      "%",
      "|",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "|",
      " ",
      "1",
      "6",
      "/",
      "1",
      "6",
      " ",
      "[",
      "0",
      "0",
      ":",
      "1",
      "3",
      "<",
      "0",
      "0",
      ":",
      "0",
      "0",
      ",",
      " ",
      " ",
      "1",
      ".",
      "1",
      "8",
      "i",
      "t",
      "/",
      "s",
      "]",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "E",
      "p",
      "o",
      "c",
      "h",
      " ",
      "6",
      "/",
      "1",
      "0",
      "0",
      "0",
      " ",
      "\t",
      " ",
      "T",
      "r",
      "a",
      "i",
      "n",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "0",
      "6",
      "8",
      "2",
      " ",
      "\t",
      " ",
      "T",
      "e",
      "s",
      "t",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "0",
      "6",
      "3",
      "5",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1",
      "0",
      "0",
      "%",
      "|",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "|",
      " ",
      "1",
      "6",
      "/",
      "1",
      "6",
      " ",
      "[",
      "0",
      "0",
      ":",
      "1",
      "3",
      "<",
      "0",
      "0",
      ":",
      "0",
      "0",
      ",",
      " ",
      " ",
      "1",
      ".",
      "1",
      "9",
      "i",
      "t",
      "/",
      "s",
      "]",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "E",
      "p",
      "o",
      "c",
      "h",
      " ",
      "7",
      "/",
      "1",
      "0",
      "0",
      "0",
      " ",
      "\t",
      " ",
      "T",
      "r",
      "a",
      "i",
      "n",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "0",
      "6",
      "0",
      "5",
      " ",
      "\t",
      " ",
      "T",
      "e",
      "s",
      "t",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "0",
      "5",
      "8",
      "1",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1",
      "0",
      "0",
      "%",
      "|",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "|",
      " ",
      "1",
      "6",
      "/",
      "1",
      "6",
      " ",
      "[",
      "0",
      "0",
      ":",
      "1",
      "3",
      "<",
      "0",
      "0",
      ":",
      "0",
      "0",
      ",",
      " ",
      " ",
      "1",
      ".",
      "1",
      "7",
      "i",
      "t",
      "/",
      "s",
      "]",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "E",
      "p",
      "o",
      "c",
      "h",
      " ",
      "8",
      "/",
      "1",
      "0",
      "0",
      "0",
      " ",
      "\t",
      " ",
      "T",
      "r",
      "a",
      "i",
      "n",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "0",
      "5",
      "4",
      "8",
      " ",
      "\t",
      " ",
      "T",
      "e",
      "s",
      "t",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "0",
      "5",
      "1",
      "3",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1",
      "0",
      "0",
      "%",
      "|",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "|",
      " ",
      "1",
      "6",
      "/",
      "1",
      "6",
      " ",
      "[",
      "0",
      "0",
      ":",
      "1",
      "3",
      "<",
      "0",
      "0",
      ":",
      "0",
      "0",
      ",",
      " ",
      " ",
      "1",
      ".",
      "1",
      "7",
      "i",
      "t",
      "/",
      "s",
      "]",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "E",
      "p",
      "o",
      "c",
      "h",
      " ",
      "9",
      "/",
      "1",
      "0",
      "0",
      "0",
      " ",
      "\t",
      " ",
      "T",
      "r",
      "a",
      "i",
      "n",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "0",
      "4",
      "9",
      "4",
      " ",
      "\t",
      " ",
      "T",
      "e",
      "s",
      "t",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "0",
      "4",
      "5",
      "7",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1",
      "0",
      "0",
      "%",
      "|",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "█",
      "|",
      " ",
      "1",
      "6",
      "/",
      "1",
      "6",
      " ",
      "[",
      "0",
      "0",
      ":",
      "1",
      "3",
      "<",
      "0",
      "0",
      ":",
      "0",
      "0",
      ",",
      " ",
      " ",
      "1",
      ".",
      "1",
      "7",
      "i",
      "t",
      "/",
      "s",
      "]",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "E",
      "p",
      "o",
      "c",
      "h",
      " ",
      "1",
      "0",
      "/",
      "1",
      "0",
      "0",
      "0",
      " ",
      "\t",
      " ",
      "T",
      "r",
      "a",
      "i",
      "n",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "0",
      "4",
      "4",
      "5",
      " ",
      "\t",
      " ",
      "T",
      "e",
      "s",
      "t",
      " ",
      "E",
      "r",
      "r",
      ":",
      " ",
      "0",
      ".",
      "0",
      "4",
      "0",
      "3",
      "\n"
     ]
    }
   ],
   "source": [
    "tune_train_err = []\n",
    "tune_test_err = []\n",
    "\n",
    "# clear loss file\n",
    "open('tune_loss', 'w').close()\n",
    "\n",
    "for epoch in range(N_TUNE_EPOCHS):\n",
    "    model.train()\n",
    "    train_loss = 0\n",
    "    for i in tqdm(range(BPE)):\n",
    "        batch_src, batch_labels, batch_padding_mask = mktunebatch(BSZ)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(batch_src, batch_padding_mask)\n",
    "        loss = criterion(output.squeeze(1), batch_labels)\n",
    "        train_loss += loss.item() / BPE\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "    test_loss = tune_evaluate()\n",
    "    \n",
    "    tune_test_err.append(test_loss)\n",
    "    tune_train_err.append(train_loss)\n",
    "    with open('tune_loss', 'a') as f:\n",
    "        f.write(f\"{train_loss} {test_loss}\\n\")\n",
    "    print(f\"Epoch {epoch + 1}/{N_TUNE_EPOCHS} \\t Train Err: {train_loss:.4f} \\t Test Err: {test_loss:.4f}\")\n",
    "\n",
    "    if epoch % 10 == 9:\n",
    "        torch.save(model.state_dict(), f\"tune_model_weights_{epoch}.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHgCAYAAABNbtJFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABqaElEQVR4nO3deViUVcMG8HvYkc0FBUkUNPcFFBHRDC3csyzLJVM0tXIroiywV7GyUNMyxdy3yt3UzNIyUlOTVBQ119xJBTQVBBSUOe8fR2YYQQNmeWaY+3ddcznzzDPnnIG3j/s7q0oIIUBERERkRWyUbgARERGRqTEAERERkdVhACIiIiKrwwBEREREVocBiIiIiKwOAxARERFZHQYgIiIisjoMQERERGR1GICIiIjI6jAAEREpTKVSYdSoUUo3g8iqMAARlXNLliyBSqWCSqXCrl27irwvhICvry9UKhWeeeYZnfeysrIQGxuLJk2awMXFBVWqVEFgYCDeeustXL58WXPfhAkTNHUU90hNTTXJd32YR7XtjTfeULRtRKQMO6UbQESm4eTkhOXLl+OJJ57Qub5jxw78888/cHR01Ll+9+5dPPnkkzhx4gQiIiIwevRoZGVl4ejRo1i+fDmef/55+Pj46Hxm9uzZcHV1LVJ3xYoVjfStSq5jx44YOHBgkev16tVTpD1EpCwGICIr0a1bN6xZswYzZsyAnZ32P/3ly5cjKCgI165d07l/w4YNOHjwIJYtW4aXX35Z5707d+4gLy+vSB0vvvgiPD09jfgtyq5evXp45ZVXlG4GEZkJDoERWYl+/frh33//xdatWzXX8vLysHbt2iIBBwDOnDkDAGjbtm2R95ycnODu7m6QdjVp0gQdOnQocl2tVuOxxx7Diy++qLm2cuVKBAUFwc3NDe7u7mjatCm+/PJLg7QDANq3b48mTZogKSkJbdq0gbOzM/z9/TFnzpwi96anp2PIkCHw8vKCk5MTAgICsHTp0mK/x5dffommTZvCyckJVatWRZcuXbB///4i927YsAFNmjSBo6MjGjdujC1btui8f+vWLURGRsLPzw+Ojo6oVq0aOnbsiAMHDhjsZ0BkLRiAiKyEn58fQkNDsWLFCs21zZs3IyMjA3379i1yf61atQAAX3/9NYQQJarj+vXruHbtms7j5s2bj/xMnz598PvvvxeZJ7Rr1y5cvnxZ07atW7eiX79+qFSpEiZPnoxJkyahffv22L17d4nadufOnSJtu3btWpGerBs3bqBbt24ICgrClClTUKNGDQwfPhyLFi3S3HP79m20b98e33zzDfr374/PPvsMHh4eGDRoUJFANmTIEERGRsLX1xeTJ09GdHQ0nJyckJiYWOT7jhgxAn379sWUKVNw584d9OrVC//++6/mnjfeeAOzZ89Gr1698NVXX+Hdd9+Fs7Mzjh8/XqKfAREVIoioXFu8eLEAIPbt2yfi4+OFm5ubyMnJEUII8dJLL4kOHToIIYSoVauW6N69u+ZzOTk5on79+gKAqFWrlhg0aJBYuHChSEtLK1JHbGysAFDso379+o9s38mTJwUAMXPmTJ3rI0aMEK6urpq2vvXWW8Ld3V3cu3ev1D+Dh7UNgFixYoXmvrCwMAFATJs2TXMtNzdXBAYGimrVqom8vDwhhBDTp08XAMS3336ruS8vL0+EhoYKV1dXkZmZKYQQ4rfffhMAxJtvvlmkTWq1Wqd9Dg4O4vTp05prhw4dKvJz8fDwECNHjiz19yeiotgDRGRFevfujdu3b2PTpk24desWNm3aVOzwFwA4Ozvjzz//xJgxY4D7q8mGDBmC6tWrY/To0cjNzS3yme+++w5bt27VeSxevPiRbapXrx4CAwOxatUqzbX8/HysXbsWPXr0gLOzM3B/InV2drbOEF5pPPfcc0XatnXr1iLDb3Z2dnj99dc1rx0cHPD6668jPT0dSUlJAICffvoJ3t7e6Nevn+Y+e3t7vPnmm8jKysKOHTs0Pw+VSoXY2Ngi7VGpVDqvw8PDUadOHc3rZs2awd3dHWfPntVcq1ixIv7880+dFXhEVDacBE1kRapWrYrw8HAsX74cOTk5yM/P15lj8yAPDw9MmTIFU6ZMwYULF5CQkICpU6ciPj4eHh4emDhxos79Tz75ZJkmQffp0wdjx47FpUuX8Nhjj2H79u1IT09Hnz59NPeMGDECq1evRteuXfHYY4+hU6dO6N27N7p06VKiOmrUqIHw8PD/vM/HxwcuLi461wpWip0/fx6tW7fGhQsXULduXdjY6P7/kA0bNgQAXLhwAbg/j8rHxweVK1f+z3pr1qxZ5FqlSpVw48YNzespU6YgIiICvr6+CAoKQrdu3TBw4EDUrl37P8snIl3sASKyMi+//DI2b96MOXPmoGvXriVeol6rVi28+uqr2L17NypWrIhly5YZrE19+vSBEAJr1qwBAKxevRoeHh464aZatWpITk7Gxo0b8eyzz2Lbtm3o2rUrIiIiDNYOJdna2hZ7vfD8q969e+Ps2bOYOXMmfHx88Nlnn6Fx48bYvHmzCVtKVD4wABFZmeeffx42NjZITEx86PDXo1SqVAl16tTBlStXDNYmf39/tGrVCqtWrcK9e/ewbt069OzZs8jeRA4ODujRowe++uornDlzBq+//jq+/vprnD592mBtuXz5MrKzs3WunTp1Crg/kRz3w+Dff/8NtVqtc9+JEyc07wNAnTp1cPnyZVy/ft1g7atevTpGjBiBDRs24Ny5c6hSpQo++eQTg5VPZC0YgIisjKurK2bPno0JEyagR48eD73v0KFDRfYGwv3hnWPHjqF+/foGbVefPn2QmJiIRYsW4dq1azrDXwB0VkMBgI2NDZo1awYAxc5HKqt79+5h7ty5mtd5eXmYO3cuqlatiqCgIOD+nkqpqak685bu3buHmTNnwtXVFWFhYQCAXr16QQiBDz/8sEg9JV1ZVyA/Px8ZGRk616pVqwYfHx+Dfn8ia8E5QERWqCTDRlu3bkVsbCyeffZZtG7dGq6urjh79iwWLVqE3NxcTJgwochn1q5dW+xO0B07doSXl9cj6+vduzfeffddvPvuu6hcuXKR+TpDhw7F9evX8dRTT6FGjRq4cOECZs6cicDAQM3cm0c5deoUvv322yLXvby80LFjR81rHx8fTJ48GefPn0e9evWwatUqJCcnY968ebC3twcAvPbaa5g7dy4GDRqEpKQk+Pn5Ye3atdi9ezemT58ONzc3AECHDh0wYMAAzJgxA3///Te6dOkCtVqNnTt3okOHDqU6/+vWrVuoUaMGXnzxRQQEBMDV1RW//vor9u3bh2nTppW4HCK6T+llaERkXIWXwT/Kg8vgz549K8aPHy9at24tqlWrJuzs7ETVqlVF9+7dxW+//abz2Uctgwcgtm3bVqK2tm3bVgAQQ4cOLfLe2rVrRadOnUS1atWEg4ODqFmzpnj99dfFlStX/rPcR7UtLCxMc19YWJho3Lix2L9/vwgNDRVOTk6iVq1aIj4+vkiZaWlpYvDgwcLT01M4ODiIpk2bisWLFxe57969e+Kzzz4TDRo0EA4ODqJq1aqia9euIikpSad9xS1vr1WrloiIiBDi/nL8MWPGiICAAOHm5iZcXFxEQECA+Oqrr/7z+xNRUSpR2n5YIqJyqn379rh27Rr++usvpZtCREbGOUBERERkdRiAiIiIyOowABEREZHV4RwgIiIisjrsASIiIiKrwwBEREREVocBiIiIiKwOAxARERFZHQYgIiIisjoMQERERGR1GICIiIjI6jAAERERkdVhACIiIiKrwwBEREREVocBiIiIiKwOAxARERFZHQYgIiIisjoMQERERGR1GICIiIjI6jAAERERkdVhACIiIiKrwwBEREREVocBiIiIiKwOAxARERFZHQYgIiIisjoMQERERGR1GICIiIjI6jAAERERkdVhACIiIiKrwwBEREREVocBiIiIiKyOndINMEdqtRqXL1+Gm5sbVCqV0s0hIiKiEhBC4NatW/Dx8YGNzaP7eBiAinH58mX4+voq3QwiIiIqg5SUFNSoUeOR9zAAFcPNzQ24/wN0d3dXujlERERUApmZmfD19dX8HX8UBqBiFAx7ubu7MwARERFZmJJMX+EkaCIiIrI6DEBERERkdRiAiIiIyOpwDhAREZGJqNVq5OXlKd0Mi2Vvbw9bW1uDlMUAREREZAJ5eXk4d+4c1Gq10k2xaBUrVoS3t7fe+/QxABERERmZEAJXrlyBra0tfH19/3OTPipKCIGcnBykp6cDAKpXr65XeQxARERERnbv3j3k5OTAx8cHFSpUULo5FsvZ2RkAkJ6ejmrVquk1HMYISkREZGT5+fkAAAcHB6WbYvEKAuTdu3f1KocBiIiIyER4vqT+DPUzZAAiIiIiq8MARERERFbHLALQrFmz4OfnBycnJ4SEhGDv3r0PvXf+/Plo164dKlWqhEqVKiE8PLzI/UIIjB8/HtWrV4ezszPCw8Px999/m+CbEBER0aP4+flh+vTpSjdD+QC0atUqREVFITY2FgcOHEBAQAA6d+6sWeb2oO3bt6Nfv37Ytm0b9uzZA19fX3Tq1AmXLl3S3DNlyhTMmDEDc+bMwZ9//gkXFxd07twZd+7cMeE3K97588CpU0q3goiI6NFUKtUjHxMmTChTufv27cNrr71m8PaWlkoIIZRsQEhICIKDgxEfHw/c3yXT19cXo0ePRnR09H9+Pj8/H5UqVUJ8fDwGDhwIIQR8fHzwzjvv4N133wUAZGRkwMvLC0uWLEHfvn2LlJGbm4vc3FzN68zMTPj6+iIjI8Ogp8F/+SUQGQn07QusWGGwYomIyMzduXMH586dg7+/P5ycnJRuTomkpqZqnq9atQrjx4/HyZMnNddcXV3h6uoK3B95yc/Ph52d8XfXedTPMjMzEx4eHiX6+61oD1BeXh6SkpIQHh6ubZCNDcLDw7Fnz54SlZGTk4O7d++icuXKAIBz584hNTVVp0wPDw+EhIQ8tMy4uDh4eHhoHr6+vnp/t+K0bSv/3bAByMgwShVERGQBhACys5V5lLTbw9vbW/Pw8PCASqXSvD5x4gTc3NywefNmBAUFwdHREbt27cKZM2fw3HPPwcvLC66urggODsavv/6qU+6DQ2AqlQoLFizA888/jwoVKqBu3brYuHGjoX/kRSgagK5du4b8/Hx4eXnpXPfy8tJJno/y/vvvw8fHRxN4Cj5XmjJjYmKQkZGheaSkpJTxGz1aUBDQqBFw5w6wZo1RqiAiIguQkwO4uirzyMkx3PeIjo7GpEmTcPz4cTRr1gxZWVno1q0bEhIScPDgQXTp0gU9evTAxYsXH1nOhx9+iN69e+Pw4cPo1q0b+vfvj+vXrxuuocVQfA6QPiZNmoSVK1di/fr1enUpOjo6wt3dXedhDCoVMHCgfP7110apgoiIyGQ++ugjdOzYEXXq1EHlypUREBCA119/HU2aNEHdunXx8ccfo06dOv/ZozNo0CD069cPjz/+OD799FNkZWU9ckGUISh6FIanpydsbW2Rlpamcz0tLQ3e3t6P/OzUqVMxadIk/Prrr2jWrJnmesHn0tLSdM4JSUtLQ2BgoMG/Q2n17w/ExAA7dwJnzwK1ayvdIiIiMrUKFYCsLOXqNpSWLVvqvM7KysKECRPw448/4sqVK7h37x5u3779nz1Ahf+Ou7i4wN3d/aGLoQxF0R4gBwcHBAUFISEhQXNNrVYjISEBoaGhD/3clClT8PHHH2PLli1Ffvj+/v7w9vbWKTMzMxN//vnnI8s0lRo1gILpSd98o3RriIhICSoV4OKizMOQm1G7uLjovH733Xexfv16fPrpp9i5cyeSk5PRtGlT5OXlPbIce3v7B34+KqjVasM1tBiKD4FFRUVh/vz5WLp0KY4fP47hw4cjOzsbgwcPBgAMHDgQMTExmvsnT56McePGYdGiRfDz80NqaipSU1ORdT9Kq1QqREZGYuLEidi4cSOOHDmCgQMHwsfHBz179lTsexZWeBhM2TV4REREhrN7924MGjQIzz//PJo2bQpvb2+cP39e6WYVS/HT4Pv06YOrV69i/PjxSE1NRWBgILZs2aKZxHzx4kXY2Ghz2uzZs5GXl4cXX3xRp5zY2FjNngTvvfcesrOz8dprr+HmzZt44oknsGXLFrNZevj883Ii2tmzwO7dwBNPKN0iIiIi/dWtWxfr1q1Djx49oFKpMG7cOKP35JSV4gEIAEaNGoVRo0YV+9727dt1XpckSapUKnz00Uf46KOPDNZGQ3JxAV58EViyRPYCMQAREVF58Pnnn+PVV19FmzZt4Onpiffffx+ZmZlKN6tYim+EaI5Ks5FSWW3bBjz1FODuDqSmAs7ORqmGiIjMgCVuhGiuysVGiNYsLAyoWRPIzARMsN8TERERFcIApBAbG2DAAPmcewIRERGZFgOQggpWg/38sxwGIyIiItNgAFJQvXpA69ZAfj6wfLnSrSEiIrIeDEAKK+gFWrpU6ZYQERFZDwYghfXpAzg4AIcPA4cOKd0aIiIi68AApLDKlYEePeRzToYmIiIyDQYgMxARIf9dtgy4d0/p1hAREZV/DEBmoEsXwNMTSEsDfvlF6dYQERGVfwxAZsDeHnj5Zfmck6GJiMgcqFSqRz4Kzt8sa9kbNmwwaHtLyyzOAiM5DDZjBvD998DNm0DFikq3iIiIrNmVK1c0z1etWoXx48fj5MmTmmuurq4Ktcww2ANkJpo3Bxo3BnJzgTVrlG4NERFZO29vb83Dw8MDKpVK59rKlSvRsGFDODk5oUGDBvjqq680n83Ly8OoUaNQvXp1ODk5oVatWoiLiwMA+Pn5AQCef/55qFQqzWtTYw+QmVCpZC/Qe+/JYbBhw5RuERERGY0QQE6OMnVXqCD/6Ohh2bJlGD9+POLj49G8eXMcPHgQw4YNg4uLCyIiIjBjxgxs3LgRq1evRs2aNZGSkoKUlBQAwL59+1CtWjUsXrwYXbp0ga2trYG+WOkwAJmR/v2B6Ghg927gzBmgTh2lW0REREaRkwMoNYSUlQW4uOhVRGxsLKZNm4YXXngBAODv749jx45h7ty5iIiIwMWLF1G3bl088cQTUKlUqFWrluazVatWBQBUrFgR3t7een6ZsuMQmBnx8QHCw+Vz7glERETmKDs7G2fOnMGQIUPg6uqqeUycOBFnzpwBAAwaNAjJycmoX78+3nzzTfxihkuc2QNkZiIi5FL4r78GYmPlqfFERFTOVKgge2KUqlsPWffbPX/+fISEhOi8VzCc1aJFC5w7dw6bN2/Gr7/+it69eyM8PBxr167Vq25DYgAyMz17Am5uwPnzciisXTulW0RERAanUuk9DKUULy8v+Pj44OzZs+jfv/9D73N3d0efPn3Qp08fvPjii+jSpQuuX7+OypUrw97eHvn5+SZt94MYgMxMhQrASy8BixbJydAMQEREZG4+/PBDvPnmm/Dw8ECXLl2Qm5uL/fv348aNG4iKisLnn3+O6tWro3nz5rCxscGaNWvg7e2Nivf3ePHz80NCQgLatm0LR0dHVKpUyeTfgQMsZqjghPjVq4Hbt5VuDRERka6hQ4diwYIFWLx4MZo2bYqwsDAsWbIE/v7+AAA3NzdMmTIFLVu2RHBwMM6fP4+ffvoJNvfndUybNg1bt26Fr68vmjdvrsh3UAkhhCI1m7HMzEx4eHggIyMD7u7uJq9frQZq1wYuXACWLwf69TN5E4iIyIDu3LmDc+fOwd/fH05OTko3x6I96mdZmr/f7AEyQzY22l4grgYjIiIyPAYgMzVggPz3l1+AQruRExERkQEwAJmpunWBNm3kcNiyZUq3hoiIqHxhADJjBcNgS5fKXdOJiIjIMBiAzFjv3oCjI/DXX0BystKtISIifXHdkf4M9TNkADJjlSoBzz4rn3MyNBGR5SrYITkvL0/ppli8nPuHyNrb2+tVDjdCNHMDBwJr1sjl8FOmAHr+vomISAF2dnaoUKECrl69Cnt7e81+OFRyQgjk5OQgPT0dFStW1PsUeQYgM9e5M1CtGpCeDvz8M/DMM0q3iIiISkulUqF69eo4d+4cLly4oHRzLJqhTpFnADJz9vbAyy8D06fLYTAGICIiy+Tg4IC6detyGEwP9vb2evf8FOBO0MVQeifoBx08CLRoATg4AKmpcm4QERER6bKonaBnzZoFPz8/ODk5ISQkBHv37n3ovUePHkWvXr3g5+cHlUqF6dOnF7knPz8f48aNg7+/P5ydnVGnTh18/PHHFj3zPjAQaNoUyMuT54MRERGRfhQNQKtWrUJUVBRiY2Nx4MABBAQEoHPnzkhPTy/2/pycHNSuXRuTJk166Pjf5MmTMXv2bMTHx+P48eOYPHkypkyZgpkzZxr52xiPSsWjMYiIiAxJ0SGwkJAQBAcHIz4+HgCgVqvh6+uL0aNHIzo6+pGf9fPzQ2RkJCIjI3WuP/PMM/Dy8sLChQs113r16gVnZ2d8++23xZaVm5uL3NxczevMzEz4+vqazRAYII/DqFFD7gx96pTcKZqIiIi0LGIILC8vD0lJSQgPD9c2xsYG4eHh2LNnT5nLbdOmDRISEnDq1CkAwKFDh7Br1y507dr1oZ+Ji4uDh4eH5uHr61vm+o2lenWgUyf5/JtvlG4NERGRZVMsAF27dg35+fnw8vLSue7l5YXU1NQylxsdHY2+ffuiQYMGsLe3R/PmzREZGYn+/fs/9DMxMTHIyMjQPFJSUspcvzEVHgZTq5VuDRERkeUqd8vgV69ejWXLlmH58uVo3LgxkpOTERkZCR8fH0RERBT7GUdHRzg6Opq8raXVsyfg7g5cuADs3AmEhSndIiIiIsukWA+Qp6cnbG1tkZaWpnM9LS1Nrw2OxowZo+kFatq0KQYMGIC3334bcXFxBmi1spydgZdeks85GZqIiKjsFAtADg4OCAoKQkJCguaaWq1GQkICQkNDy1xuTk5OkS3GbW1toS4nY0YFnVhr1gD3j0MhIiKiUlJ0CCwqKgoRERFo2bIlWrVqhenTpyM7OxuDBw8GAAwcOBCPPfaYpvcmLy8Px44d0zy/dOkSkpOT4erqiscffxwA0KNHD3zyySeoWbMmGjdujIMHD+Lzzz/Hq6++quA3NZy2bQF/f+DcOWDDBrlLNBEREZWO4jtBx8fH47PPPkNqaioCAwMxY8YMhISEAADat28PPz8/LFmyBABw/vx5+Pv7FykjLCwM27dvBwDcunUL48aNw/r165Geng4fHx/069cP48ePh4ODQ4naZG47QT8oNhb46CO5Kuznn5VuDRERkXkozd9vxQOQOTL3AHTmDPD444CNDXDxIvDYY0q3iIiISHkWsQ8QlV2dOnIoTK0Gli9XujVERESWhwHIQhVMhl66FGAfHhERUekwAFmol14CHB2Bo0flafFERERUcgxAFqpiReC55+TzpUuVbg0REZFlYQCyYAXDYMuXA3fvKt0aIiIiy8EAZME6dQK8vIBr14AtW5RuDRERkeVgALJgdnZAwRmvHAYjIiIqOQYgC1dwQvwPPwDXryvdGiIiIsvAAGThAgKAZs2AvDxg1SqlW0NERGQZGIDKgYLJ0DwhnoiIqGQYgMqBl18GbG2BxETg1CmlW0NERGT+GIDKAW9voHNn+Zy9QERERP+NAaicKJgM/c038owwIiIiejgGoHLi2WcBDw95OvyOHUq3hoiIyLwxAJUTzs5A797yOYfBiIiIHo0BqBwpGAZbuxbIzla6NUREROaLAagcadsWqF0byMoC1q9XujVERETmiwGoHFGptL1AHAYjIiJ6OAagcmbAAPnvr78C//yjdGuIiIjMEwNQOVO7NtCuHSAEsGyZ0q0hIiIyTwxA5VDhYTAhlG4NERGR+WEAKodeeglwcgKOHQOSkpRuDRERkflhACqHPDyAnj3lc06GJiIiKooBqJwqOCF++XIgL0/p1hAREZkXBqByKjxcHpL677/A5s1Kt4aIiMi8MACVU3Z2QP/+8jmHwYiIiHQxAJVjBcNgP/wge4KIiIhIYgAqx5o2BQIDgbt3gVWrlG4NERGR+WAAKucKeoGWLlW6JUREROaDAaic69cPsLUF9u4FTpxQujVERETmgQGonPPyArp0kc+/+Ubp1hAREZkHxQPQrFmz4OfnBycnJ4SEhGDv3r0Pvffo0aPo1asX/Pz8oFKpMH369GLvu3TpEl555RVUqVIFzs7OaNq0Kfbv32/Eb2HeCobBvvkGUKuVbg0REZHyFA1Aq1atQlRUFGJjY3HgwAEEBASgc+fOSE9PL/b+nJwc1K5dG5MmTYK3t3ex99y4cQNt27aFvb09Nm/ejGPHjmHatGmoVKmSkb+N+erRA6hYEUhJAbZvV7o1REREylMJodxxmSEhIQgODkZ8fDwAQK1Ww9fXF6NHj0Z0dPQjP+vn54fIyEhERkbqXI+Ojsbu3buxc+fOErcjNzcXubm5mteZmZnw9fVFRkYG3N3dS/29zNEbbwBz58qDUjkhmoiIyqPMzEx4eHiU6O+3Yj1AeXl5SEpKQnh4uLYxNjYIDw/Hnj17ylzuxo0b0bJlS7z00kuoVq0amjdvjvnz5z/yM3FxcfDw8NA8fH19y1z/f/rnHyAtzXjlP0TBCfHffQdkZZm8eiIiIrOiWAC6du0a8vPz4eXlpXPdy8sLqampZS737NmzmD17NurWrYuff/4Zw4cPx5tvvomlj+j2iImJQUZGhuaRkpJS5vofacECoG5d4IMPjFP+I4SGAo8/DmRnA+vXm7x6IiIis6L4JGhDU6vVaNGiBT799FM0b94cr732GoYNG4Y5c+Y89DOOjo5wd3fXeRhFo0bAnTvA4sXAsWPGqeMhVCptLxCHwIiIyNopFoA8PT1ha2uLtAeGg9LS0h46wbkkqlevjkaNGulca9iwIS5evFjmMg2mTRugZ0+5FCsmxuTVDxgg//3tNzkhmoiIyFopFoAcHBwQFBSEhIQEzTW1Wo2EhASEhoaWudy2bdvi5MmTOtdOnTqFWrVq6dVeg4mLkzsTbtwI7Npl0qr9/ICwMEAI4NtvTVo1ERGRWVF0CCwqKgrz58/H0qVLcfz4cQwfPhzZ2dkYPHgwAGDgwIGIKdRTkpeXh+TkZCQnJyMvLw+XLl1CcnIyTp8+rbnn7bffRmJiIj799FOcPn0ay5cvx7x58zBy5EhFvmMRDRoAQ4bI52PGyDRiQgXDYF9/bfKqiYiIzIaiy+ABID4+Hp999hlSU1MRGBiIGTNmICQkBADQvn17+Pn5YcmSJQCA8+fPw9/fv0gZYWFh2F5og5tNmzYhJiYGf//9N/z9/REVFYVhw4aVuE2lWUZXJleuyBnJOTlyWdYLLxi+jofIzAS8vYHbt+XxGMHBJquaiIjIqErz91vxAGSOjB6AAGDcOGDiRKBePeCvvwB7e+PUU4z+/YHly4GRI4H7WzARERFZPIvYB8jqjRkDeHoCp04BCxeatOqCYbAVK4C8PJNWTUREZBYYgJTi7i57gQBgwgST7k4YHg5Urw5cvw78+KPJqiUiIjIbDEBKeuMNoHZtuTP0F1+YrFpbW+CVV+Tzr782WbVERERmgwFISQ4OwCefyOdTpgAPOQTWGAqGwX78Ebh2zWTVEhERmQUGIKX17g0EBckhsI8/Nlm1TZoALVoAd+8CK1earFoiIiKzwACkNBsb2fsDAHPmAIX2NDK2wnsCERERWRMGIHPw1FNAly7AvXsmPSi1Xz/Azg7Ytw84ftxk1RIRESmOAchcTJokTyxdvVomEhOoVg3o2lU+Zy8QERFZEwYgcxEQoF2a9d57JjunomAY7Ntvgfx8k1RJRESkOAYgc/Lxx3Jl2PbtwJYtJqmyRw+gYkXgn3+AbdtMUiUREZHiGIDMSa1awOjR8vn775ukS8bREejbVz7nMBgREVkLBiBzM3as7JI5ckSOS5lARIT897vvgFu3TFIlERGRohiAzE3lykBMjHw+bhxw547RqwwJAerWlYfTr1tn9OqIiIgUxwBkjkaPBmrUAFJSgJkzjV6dSsU9gYiIyLowAJkjZ2ftrtCffipPLTWyAQPkv9u2ARcvGr06IiIiRTEAmasBA+R5FTdvyj2CjKxWLaB9e7n63kRTj4iIiBTDAGSubG21wWfGDJN0yxRMhl661GTbEBERESmCAcicdesGhIUBubnA+PFGr65XLzn6duoUsHev0asjIiJSDAOQOVOptAelfv01cPiwUatzcwNeeEFbHRERUXnFAGTuWrUCXnpJjklFRxu9uoJhsBUrZMcTERFRecQAZAk+/VQe2755s9HPq3jqKeCxx4AbN4AffzRqVURERIphALIEjz8OvP66fP7ee4BabbSqbG21Z7IuXWq0aoiIiBTFAGQpxo0DXF2B/fuBtWuNWlXBnkA//QRcvWrUqoiIiBTBAGQpvLyAd9+Vz8eOBfLyjFZV48ZAUBBw7x6wcqXRqiEiIlIMA5AleecdGYTOnAHmzTNqVYX3BCIiIipvGIAsiasrEBsrn3/0EZCZabSq+vaV866TkoCjR41WDRERkSIYgCzN0KFAvXpycs7UqUarpmpVoHt3+Zx7AhERUXnDAGRp7O3lsngAmDYNuHLFaFUVnBD/7bdAfr7RqiEiIjI5BiBL9MILQEgIkJMjh8KMpHt3oFIl4PJl4LffjFYNERGRyTEAWaLCR2TMnw+cPGmUahwdgX795HNOhiYiovKEAchSPfkk8Mwzcmxq7FijVVMwDLZuHXDrltGqISIiMimzCECzZs2Cn58fnJycEBISgr2POIr86NGj6NWrF/z8/KBSqTB9+vRHlj1p0iSoVCpERkYaoeUKmzQJsLGR6WTPHqNU0aoVUL8+cPu20fdfJCIiMhnFA9CqVasQFRWF2NhYHDhwAAEBAejcuTPS09OLvT8nJwe1a9fGpEmT4O3t/ciy9+3bh7lz56JZs2ZGar3CGjcGBg2Sz997Tx6YamAqlbYXiKvBiIiovFA8AH3++ecYNmwYBg8ejEaNGmHOnDmoUKECFi1aVOz9wcHB+Oyzz9C3b184Ojo+tNysrCz0798f8+fPR6VKlR7ZhtzcXGRmZuo8LMaHHwJOTsCuXcAPPxilildekUFo+3bgwgWjVEFERGRSigagvLw8JCUlITw8XNsgGxuEh4djj55DOiNHjkT37t11yn6YuLg4eHh4aB6+vr561W1SNWoABcN70dHy/AoDq1kT6NBBPv/mG4MXT0REZHKKBqBr164hPz8fXl5eOte9vLyQmppa5nJXrlyJAwcOIC4urkT3x8TEICMjQ/NISUkpc92KeP99oHJl4Phxoy3XKjwMZoSRNiIiIpNSfAjM0FJSUvDWW29h2bJlcHJyKtFnHB0d4e7urvOwKBUrAh98IJ+PHy/3BzKwXr2AChWAv/8GEhMNXjwREZFJKRqAPD09YWtri7S0NJ3raWlp/znB+WGSkpKQnp6OFi1awM7ODnZ2dtixYwdmzJgBOzs75JfXLY1HjgRq1ZK7Fn75pcGLd3WVIQicDE1EROWAogHIwcEBQUFBSEhI0FxTq9VISEhAaGhomcp8+umnceTIESQnJ2seLVu2RP/+/ZGcnAxbW1sDfgMz4ugITJwon0+aBFy7ZvAqCobBVq4EcnMNXjwREZHJKD4EFhUVhfnz52Pp0qU4fvw4hg8fjuzsbAwePBgAMHDgQMTExGjuz8vL0wSbvLw8XLp0CcnJyTh9+jQAwM3NDU2aNNF5uLi4oEqVKmjSpIli39MkXn4ZCAyUp8R/8onBi+/QQc65vnnTaAvOiIiITELxANSnTx9MnToV48ePR2BgIJKTk7FlyxbNxOiLFy/iSqEDPy9fvozmzZujefPmuHLlCqZOnYrmzZtj6NChCn4LM2FjA0yeLJ/PmgWcO2fQ4m1t5ZJ4cBiMiIgsnEoIrul5UGZmJjw8PJCRkWF5E6IBoGNH4Ndfgf795VHuBnT8ONCoEWBnB1y6BFSrZtDiiYiIyqw0f78V7wEiI5g0Sf67bBlw8KBBi27YEAgOltsNrVhh0KKJiIhMhgGoPAoK0h7j/v77Bi++YDL0zJlAVpbBiyciIjI6BqDy6pNPAHt7YOtW+TCgV14BHnsMOHMGeP11boxIRESWhwGovPL3B0aMkM/ffx9Qqw1WdMWKwKpVclL08uXA/PkGK5qIiMgkGIDKs//9D3B3l/OADDxhp21boOCkkTffBJKTDVo8ERGRUTEAlWeento5QP/7n8F3L3znHeCZZ2SxL70ktx8iIiKyBAxA5V1kJFC9OnD+PDB7tkGLtrGRZ6/WrAmcPg0MHcr5QEREZBkYgMq7ChWADz+UzydOBDIyDFp85crA6tVyvvWaNcBXXxm0eCIiIqNgALIGgwcDDRoA//6r3SnagEJCgClT5PO33wb27zd4FURERAbFAGQN7Oy0myNOny63cDawt94Cnn8euHtXzge6ccPgVRARERkMA5C1ePZZuXTr9m0gNtbgxatUwKJFcvX9+fOy04nzgYiIyFwxAFkLlUo7TrV4MXDsmMGrqFhRzgNycAC+/152NhEREZkjBiBr0qaNHKdSq4HoaKNUERQEfPGFfP7ee0BiolGqISIi0gsDkLWJi5NbOP/wA7Bzp1GqGD4c6NNHHpjau7ece01ERGROGICsTf36wJAh8vl77xlloo5KBcybB9StC6SkABERBj2Jg4iISG8MQNZowgS5P1BiIrB+vVGqcHeX84EcHYEffwSmTjVKNURERGXCAGSNqlcHoqLk85gYuXbdCAICgJkz5fOxY4Fdu4xSDRERUakxAFmrMWPkWWGnTgELFxqtmqFDgVdeAfLz5bygq1eNVhUREVGJMQBZK3d3YPx4+XzCBCAryyjVqFTyCLIGDYDLl2UY4nwgIiJSGgOQNXv9daBOHSAtTbt23QhcXeV8IGdn4JdfgE8/NVpVREREJcIAZM0cHIBPPpHPp0wB0tONVlWTJtrD6GNjgW3bjFYVERHRf2IAsnYvvSR3L8zKAj7+2KhVRUTIIzLUaqBfPyA11ajVERERPRQDkLWzsdEekTFnDnD6tFGri4+XvUFpacDLL8vJ0URERKbGAETAU08BXbrIrZs/+MCoVVWoIOcDubjIYbCPPjJqdURERMUqVQCaMmUKbt++rXm9e/du5Obmal7funULI0aMMGwLyTQmT5ZLtlavBvbtM2pVDRrInaIBOer2yy9GrY6IiKiIUgWgmJgY3Lp1S/O6a9euuHTpkuZ1Tk4O5s6da9gWkmk0awYMGCCfG+mIjMJeflkuQhNCLo0v9D8jIiIioytVABIP/FF88DVZuI8/lmdXbN8ObNli9OqmTwcCA+XmiP36yRE4IiIiU+AcINKqWRMYNUo+f/99o89QdnKS84Hc3OTB9OPGGbU6IiIiDQYg0jV2LFCxInDkCPDtt0av7vHHtSdxTJoE/PST0askIiKCXWk/sGDBAri6ugIA7t27hyVLlsDT0xO4PwmaLFzlyvKA1Pffl10yffrIrhojeukl2fEUHy+nIR08KDujiIiIjEUlSjGRx8/PDyqV6j/vO3funL7tUlRmZiY8PDyQkZEBd3d3pZtjerdvA/XqAf/8I/cIGjPG6FXm5gJPPAHs3w+0bg3s2CE3qiYiIiqp0vz9LtUQ2Pnz53Hu3Ln/fJTWrFmz4OfnBycnJ4SEhGDv3r0Pvffo0aPo1auXJoxNnz69yD1xcXEIDg6Gm5sbqlWrhp49e+LkyZOlbpfVcnbW7gr96afA9etGr9LRUa7Ar1gRSEyUnVBERETGovgcoFWrViEqKgqxsbE4cOAAAgIC0LlzZ6Q/5FyqnJwc1K5dG5MmTYK3t3ex9+zYsQMjR45EYmIitm7dirt376JTp07Izs428rcpRwYMAJo2BW7elJNzTMDfH1i8WD7//HPg++9NUi0REVkjUQp//PGH+OGHH3SuLV26VPj5+YmqVauKYcOGiTt37pSmSNGqVSsxcuRIzev8/Hzh4+Mj4uLi/vOztWrVEl988cV/3peeni4AiB07dhT7/p07d0RGRobmkZKSIgCIjIyMUn2XcufHH4UAhHB0FOLCBZNVGxUlq61YUYizZ01WLRERWbiMjIwS//0uVQ/QRx99hKNHj2peHzlyBEOGDEF4eDiio6Pxww8/IC4ursTl5eXlISkpCeHh4ZprNjY2CA8Px549e0rTtEfKyMgAAFSuXLnY9+Pi4uDh4aF5+Pr6Gqxui9a1KxAWJifojB9vsmonTZLzgG7eBHr3ltUTEREZUqkCUHJyMp5++mnN65UrVyIkJATz589HVFQUZsyYgdWrV5e4vGvXriE/Px9eXl461728vJBqoKPC1Wo1IiMj0bZtWzRp0qTYe2JiYpCRkaF5pKSkGKRui6dSaQ9K/fpr4PBhk1Rrbw+sWiUXpO3fb5I52EREZGVKFYBu3LihE1Z27NiBrl27al4HBwebXXgYOXIk/vrrL6xcufKh9zg6OsLd3V3nQfe1aiXXqQsBREebrNqaNWXmAoCZM4G1a01WNRERWYFSBSAvLy/NKq+8vDwcOHAArVu31rx/69Yt2Nvbl7g8T09P2NraIi0tTed6WlraQyc4l8aoUaOwadMmbNu2DTVq1NC7PKv16aeAnR2webM8wt1EuneX2xEBwKuvAqdPm6xqIiIq50oVgLp164bo6Gjs3LkTMTExqFChAtq1a6d5//Dhw6hTp06Jy3NwcEBQUBASEhI019RqNRISEhAaGlqapukQQmDUqFFYv349fvvtN/j7+5e5LLq/XfPrr8vn770HqNUmq3riRKBdO+DWLdkRdeeOyaomIqJyrFQB6OOPP4adnR3CwsIwf/58zJs3Dw6FdqtbtGgROnXqVKoGREVFYf78+Vi6dCmOHz+O4cOHIzs7G4MHDwYADBw4EDGFNoXJy8tDcnIykpOTkZeXh0uXLiE5ORmnC3UPjBw5Et9++y2WL18ONzc3pKamIjU1Fbdv3y5V26iQ8eMBV1c5KceE41F2dsCKFUDVqkByMhAZabKqiYioHCvVTtAFMjIy4OrqCltbW53r169fh5ubW6mGwQAgPj4en332GVJTUxEYGIgZM2YgJCQEANC+fXv4+flhyZIlwP3NGIvr0QkLC8P27dvll3rIbtWLFy/GoEGD/rM9Vr8T9MN89BEQGwvUqQMcO2bSrZp/+QXo0kVORVq+XJ4eT0REVFhp/n6XKgC9+uqrJbpv0aJFJS3SLDEAPURWlhwOS0uTM5MLTo43kfHj5QbVLi6yI6pBA5NWT0REZs5oAcjGxga1atVC8+bN8aiPrV+/vnQtNjMMQI8wezYwYoQckzp9GjDhzyc/H+jYUc7DbtIE+PNPoEIFk1VPRERmzmgBaOTIkVixYgVq1aqFwYMH45VXXnno5oKWjAHoEe7elenj1Cng3XeBzz4zafWpqUBgoOyEevVVYOFCk1ZPRERmzGiHoc6aNQtXrlzBe++9hx9++AG+vr7o3bs3fv7550f2CFE5Ym8PFOz2PXWqXB1mwq2avb3lpGgbG2DRImDpUpNVTURE5UipD0N1dHREv379sHXrVhw7dgyNGzfGiBEj4Ofnh6ysLOO0kszL88/LEKRSAfPmAR06AJcvm6z6Dh2ACRPk8+HDgUKnsxAREZWIXqfB29jYQKVSQQiB/Px8w7WKzJtKJXeF/vFHoGJFYM8eICgI+OMPkzXhgw+ATp2A27fl/kDM3kREVBqlDkC5ublYsWIFOnbsiHr16uHIkSOIj4/HxYsX4erqapxWknnq2hXYtw9o3FhOzmnfHpg71yRV29gA334L+PgAx4/LniCOwhIRUUmVKgCNGDEC1atXx6RJk/DMM88gJSUFa9asQbdu3WBjo1dnElmqxx8HEhOBF1+UE6TfeAN47TWTzAuqWhVYuRKwtZVhiBOiiYiopEq9DL5mzZpo3rz5QzcbBIB169YZqn2K4CqwMhACmDwZGDtWPm/dGvjuO9lFY2STJ8sROScnmcUCAoxeJRERmaHS/P22K03BAwcOfGTwIStWMC8oMFBu05yYKOcFrV0LtG1r1KrHjAF+/x346Sc5H2j/fpNuT0RERBaoTEdhlHfsAdLTmTNAz57AX3/JZfMzZsjl8kYMz//+CzRvDqSkAH36yKXyzOpERNbFaPsAEZVInTpyZVjv3nJe0PDhwLBhRp0XVKUKsHq1PDx11Sq5YTUREdHDMACRcbi6yhnKkyfLJVsLFwJhYcClS0arsnVrWR0AvP02kJRktKqIiMjCMQCR8ahUwHvvAZs3A5UqycO7goKAXbuMVuXbbwPPPQfk5ckOqJs3jVYVERFZMAYgMr5OneR+QU2bykO8OnQAvvrKKBv3qFTA4sWAnx9w9qw8L4yz3IiI6EEMQGQahecF3bsHjBwJDB0K3Llj8KoqVQLWrAEcHID16+UcbCIiosIYgMh0XFzkvKApU7SnmYaFAf/8Y/CqWrYEpk2Tz8eMkaNvREREBRiAyLRUKplItmyRXTV798p5QTt3GryqkSPlvkB378qOp+vXDV4FERFZKAYgUkbHjnLHwmbNgPR04KmngFmzDDphR6UC5s+Xp3VcvAhERABqtcGKJyIiC8YARMqpXVueIN+3r5wXNGoUMGSIQecFeXjI+UCOjsCmTdphMSIism4MQKQsFxdg+XLgs8/kvKDFi4Enn5RbOhtIYCDw5ZfyeUwMsHu3wYomIiILxQBEylOpgHffBX7+GahcWS6Zb9lSHvBlIK+9Brz8MpCfL4/KuHrVYEUTEZEFYgAi8xEeLucFBQTIeUFPPw3ExxtkXpBKBcydC9SvLzejHjCA84GIiKwZAxCZF39/OS+oXz85L2j0aGDwYIPMC3J1lfOBnJ1lZ9OkSQZpMRERWSAGIDI/FSoAy5bJGcs2NsDSpUC7dgaZF9S0qVxsBgDjxgHbt+vfXCIisjwMQGSeVCogKgr45Rd51Pv+/XK/oB079C568GDtkvh+/eTpHEREZF0YgMi8Pf20DD+BgXLm8tNPy7Mt9JwXNGsW0LgxkJoqD0+9eNFgLSYiIgvAAETmz89Prl3v318u43rrLWDQIOD27TIX6eIi5wO5ucljMpo2BZYs4cGpRETWggGILEOFCsA33wCffw7Y2gJffy3nBenRddOwIZCUBISGApmZcmisZ0/ZK0REROUbAxBZDpUKePtt7bygpCQ5L0iPmcx168pjyCZNkqfHb9wINGkCrF1r0JYTEZGZYQAiy/PUUzL8NG8OXLsm9w/68ssyj1/Z2gLvv6/dgujff+Uhqv378wBVIqLyigGILFOtWnJe0CuvyHlBkZFyaZce84KaNpWH03/wgVx9v3y57A3avNmgLSciIjNgFgFo1qxZ8PPzg5OTE0JCQrB3796H3nv06FH06tULfn5+UKlUmD59ut5lkoVydpZzgaZPl90433wDtG0LXLhQ5iIdHICJE+VejPXrA1euAN26yaM0bt0yaOuJiEhBigegVatWISoqCrGxsThw4AACAgLQuXNnpKenF3t/Tk4OateujUmTJsHb29sgZZIFU6nkqrCtWwFPT+DgQXmO2LZtehUbEgIcOCCLBoD58+XwmAGPJyMiIgWphFB24W9ISAiCg4MRHx8PAFCr1fD19cXo0aMRHR39yM/6+fkhMjISkZGRepWZm5uL3NxczevMzEz4+voiIyMD7u7uBvqmZHQXLwLPPy+Ti62tPGE+MlKGJD1s2yZXiF24IIuKjAQ++UR2QBERkfnIzMyEh4dHif5+K9oDlJeXh6SkJISHh2sbZGOD8PBw7Nmzx2RlxsXFwcPDQ/Pw9fUtU92ksJo1gV275Emn+flyJ+kBA4CcHL2K7dABOHwYGDJEzrP+4gu5+Gz/foO1nIiITEzRAHTt2jXk5+fDy8tL57qXlxdSy7gZS1nKjImJQUZGhuaRYoAzp0ghzs7y7LAvv5S9QMuWAU88AZw/r1ex7u7AggXApk2Atzdw/DjQujUQGwvcvWuw1hMRkYkoPgfIHDg6OsLd3V3nQRZMpQLefBP49VegalXtvKDfftO76O7dgb/+Avr0kZ1MH30k5wv99ZdBWk5ERCaiaADy9PSEra0t0h44jTItLe2hE5yVKJMsVPv22kNU//0X6NhR7iSt57S3KlWAlSvlo3Jlma+CguSUo/x8g7WeiIiMSNEA5ODggKCgICQkJGiuqdVqJCQkIDQ01GzKJAtWs6bc6nngQHn8+zvvyL2D9JwXBMheoL/+kr1CeXnAe+/JzHXmjEFaTkRERqT4EFhUVBTmz5+PpUuX4vjx4xg+fDiys7MxePBgAMDAgQMRExOjuT8vLw/JyclITk5GXl4eLl26hOTkZJw+fbrEZZKVcXaWJ53OmCHnBS1fLvcLOndO76KrVwd++EHOD3J1lXOwmzUDZs/mwapERGZNmIGZM2eKmjVrCgcHB9GqVSuRmJioeS8sLExERERoXp87d04AKPIICwsrcZn/JSMjQwAQGRkZBvqGZDa2bxeialUhACEqVxZi61aDFX3unBDt28uiASE6dhQiJcVgxRMR0X8ozd9vxfcBMkel2UeALFBKCvDCC3J+kI2N3Pr53XcBe3u9i1argZkzgeho4M4dwMNDvn7lFb23IyIiov9gMfsAESnC11fOCxo0SCaWsWPlLOY//tC7aBsbuXt0cjLQqhWQkSGnH/XqBXAjciIi88EARNbJyQlYtAhYvFgu6zpyRM4LGjpUrhjTU/368qzWiRNlx9L69fJg1fXrDdJ6IiLSEwMQWS+VSvYCnTght3kGgIULZXpZvFjvWcx2dvJk+b175UnzV6/KkbeBA4GbNw3zFYiIqGwYgIg8PeUyrl27ZDfNv/8Cr74KPPmkQXY4DAwE9u2T84JsbOSh9U2byvNbiYhIGQxARAXatpUHqU6ZAlSoIANR8+bA++8D2dl6Fe3oCMTFyalHjz8O/PMP0KkTMGKE3kUTEVEZMAARFWZvD4wZIw/76tkTuHdPBqJGjYCNG/Uuvk0bOUF61Cj5evZsICBAzhciIiLTYQAiKk7NmnLG8saNQK1awMWLwHPPyceFC3oV7eIil8Zv3SoXpJ05A7RrJzua7twx2DcgIqJHYAAiepQePYCjR+UEHjs7GYgaNZK9QnoeAx8eLhefDRok51tPmSLPbD1wwGCtJyKih2AAIvovLi5yAk9yspwYnZMju2uaN5eTevTg4SEXnH3/PVCtmsxaISHylHk98xURET0CAxBRSTVuDGzfLs8V8/SUaeXJJ+WKsWvX9Cr62WdlcS++KKcdxcbK+ULHjxus9UREVAgDEFFpqFRARARw8iQwbJi8tnix3DtowQK5s3QZeXoCq1cDy5YBFSvKkzqaNwc+/1yvYomIqBgMQERlUbkyMG+ePD6jWTPg+nUZiNq1kxN7ykilAl5+WW4/1KULkJsLvPMO0KGDQQ6vJyKi+xiAiPQRGgokJQHTpsm5Qn/8Ibtt3n0XyMoqc7GPPQb89BMwd64s9vffZc6aP1/vDaqJiIgBiMgA7OyAqCh5pEavXkB+vgxEDRvKpfRlTCwqFfDaa8Dhw7JjKStLvu7eHbh82eDfgojIqjAAERlKjRrA2rXAjz8C/v5yu+cXXpBL6fUYv6pdG9i2DZg6Ve4ovXmzPLFjxQr2BhERlRUDEJGhdesmJ/F88IHcWfrHH+UKsrg4IC+vTEXa2sq5QAcOAEFBwI0bcq5Qnz56L0AjIrJKDEBExlChAjBxInDoENC+PXD7NjB2rDwZdceOMhfbqBGwZw/w4Ydy5G3NGtkb9MMPBm09EVG5xwBEZEwNGwK//SaPgK9aVW7s0769XEqfnl6mIu3tgfHjgcREGYjS0uQ+Qi+/DBw8aPBvQERULjEAERmbSgW88orcO+iNN+Trr78GGjSQS+nLuMlPUJBcgDZmjCxyxQqgRQu5N+PatXJDRSIiKh4DEJGpVKokj3/fs0cOhd24Abz+OtC2rRwqKwMnJ3mG2N69QN++clhs507gpZfkPOy4OM4RIiIqDgMQkamFhAD79gHTpwNubnIsKyhILqW/datMRbZsKXuALlwAxo2To23//COnHdWoAQwZIo8yIyIiiQGISAl2dsBbb8k5Qb17y72DvvhCzhn67rsyr2/38ZEHqV68CCxdKnNVbi6waJHcn5HDY0REEgMQkZIeewxYtQrYsgWoUwe4dEmeiNq9O3D2bJmLdXICBg6UHU27dxcdHqtdG5g0icNjRGS9GICIzEHnzvIMsfHjAQcHudth48ZyKX1ubpmLVankqfIrVgDnzwP/+58cHktJAWJiAF9fDo8RkXVSCcG9ZB+UmZkJDw8PZGRkwN3dXenmkLU5dQoYMQJISJCv69eXk6c7dDBI8XfuyFPnv/xSbqxY4MkngTffBJ57TvYWERFZmtL8/WYPEJG5qVcP2LoVWLYM8PKSy+efekoupU9L07v4guGx/fvl8FifPnKn6d9/l6NvHB4jImvAAERkjlQqubPhiRPAyJHy9bJl2t6g/HyDVNGmDbBypRwe++CDosNjQ4eWeYU+EZFZYwAiMmcVKwLx8cCff8olXRkZcnisTRvd8Ss91aghpxtdvAgsWSJXjN25AyxcKLcsat9eLk7j6jEiKi8YgIgsQXCwDEEzZwLu7nLnw+BguZQ+M9Ng1Tg5yVM6kpKAXbvkCn1bW3l8WcHw2OTJwL//GqxKIiJFMAARWQpbW2DUKDks1revPEJjxgx5pMbq1WXeO6g4KpXcoHrVKu3wmKenHB6LjpY9RsOGAYcPG6xKIiKTYgAisjTVq8t17b/8AtStC1y5Imcyh4XJQ1ezsgxaXcHwWEoKsHixdnhswQIgIEAOj61bx+ExIrIsZhGAZs2aBT8/Pzg5OSEkJAR79+595P1r1qxBgwYN4OTkhKZNm+Knn37SeT8rKwujRo1CjRo14OzsjEaNGmHOnDlG/hZEJtaxo+yCmTABcHSUuxwOHAh4ewODBslT6Mt40GpxnJxkscUNj/XqJfdx5PAYEVkMobCVK1cKBwcHsWjRInH06FExbNgwUbFiRZGWllbs/bt37xa2trZiypQp4tixY+J///ufsLe3F0eOHNHcM2zYMFGnTh2xbds2ce7cOTF37lxha2srvv/++xK1KSMjQwAQGRkZBvueREZ14YIQH30kxOOPCyEHw+TD11eIsWOFOHHCKNWmpMjiq1TRVunkJMTQoUIcOmSUKomIHqo0f78V3wgxJCQEwcHBiI+PBwCo1Wr4+vpi9OjRiI6OLnJ/nz59kJ2djU2bNmmutW7dGoGBgZpeniZNmqBPnz4YN26c5p6goCB07doVEydOLFJmbm4ucgvttpuZmQlfX19uhEiWRwh52vzXX8sJPDdvat8LCZE9RH37ApUrG7Ta27flcvoZM3R3lW7fHhg9Gnj2WW6uSETGZzEbIebl5SEpKQnh4eHaBtnYIDw8HHv27Cn2M3v27NG5HwA6d+6sc3+bNm2wceNGXLp0CUIIbNu2DadOnUKnTp2KLTMuLg4eHh6ah6+vr8G+I5FJFWzuM2eOnBu0ejXwzDNyrOrPP+WeQt7ecszq+++BvDyDVOvsDAweLFfmF5w3ZmsLbN+uHR6bMgW4ft0g1RER6U3RAHTt2jXk5+fDy8tL57qXlxdSU1OL/Uxqaup/3j9z5kw0atQINWrUgIODA7p06YJZs2bhySefLLbMmJgYZGRkaB4pKSkG+X5EinJykknkhx/kIatffCE39bl7V85a7tlTHsb65ptyW2gDdAarVMATT8jcde4cMHYsUKWK3F/o/fflhOrXXpPHnhERKcksJkEb2syZM5GYmIiNGzciKSkJ06ZNw8iRI/Hrr78We7+joyPc3d11HkTlipcXEBkJHDwot3Z+5x3ZE3TtmtxbKDgYaNJEzmK+dMkgVfr6Ap98IlePLVoks9ft28D8+UCzZvJos/XrDbKpNRFRqSkagDw9PWFra4u0B843SktLg7e3d7Gf8fb2fuT9t2/fxtixY/H555+jR48eaNasGUaNGoU+ffpg6tSpRvw2RBaiWTNg6lSZTDZvlnOCnJyAY8fkJj++vkCnTsC33wLZ2XpXV3h47PffdYfHXnhBDo999hmHx4jItBQNQA4ODggKCkJCwanX9ydBJyQkIDQ0tNjPhIaG6twPAFu3btXcf/fuXdy9exc2NrpfzdbWFmoDLgkmsnh2dkCXLnJPodRUubFPu3ZyKGzrVmDAANlLNHgwsG2b3kvqVSpZfMHwWEyMHB67cAF47z3t8Bg3VyQikzDFsrRHWblypXB0dBRLliwRx44dE6+99pqoWLGiSE1NFUIIMWDAABEdHa25f/fu3cLOzk5MnTpVHD9+XMTGxhZZBh8WFiYaN24stm3bJs6ePSsWL14snJycxFdffVWiNnEZPFm1M2eEmDBBiNq1dZfU16wpxAcfCHHypMGqyskRYuFCIQICdKtq1UqIefOEyMw0WFVEZAVK8/db8QAkhBAzZ84UNWvWFA4ODqJVq1YiMTFR815YWJiIiIjQuX/16tWiXr16wsHBQTRu3Fj8+OOPOu9fuXJFDBo0SPj4+AgnJydRv359MW3aNKFWq0vUHgYgIiGEWi3Erl1CDBsmhLu7bkJp3VqIr74S4t9/DVbV778L8eKLQtjZaatxcRHi1VeF2L1b3kNE9CgWtQ+QOSrNPgJEVuH2bWDjRrm/0M8/a2cuOzgAPXrIE1S7dAHs7fWuKi1NnuixYAFw8qT2esOGwJAhciujqlX1roaIyqHS/P1mACoGAxDRI6SmAsuXyzB06JD2etWqQL9+Mgw1by4n/ehBCGD3bmDhQrmn4+3b8rq9PfDcczIMdewoJ1QTEYEBSH8MQEQldOiQDELLlsmumwKNG8sg1L8/4OOjdzUZGXKn6YULgX37tNd9fYFXX5XztGvV0rsaIrJwDEB6YgAiKqV79+Tp9F9/DWzYABQcLWNjA4SHyzDUsydQoYLeVR06JIPQt98CN27IayqV7A0aMkT2Djk66l0NEVkgBiA9MQAR6eHmTWDNGmDpUjmGVcDNTW4CNHCgXA9vo98uHHfuyI0UFy4ECu+MUaWKrGLIENkRRUTWgwFITwxARAZy5oyc0fz113LznwJ+fnKfoQEDgLp19a7m7Fm52/TixcDly9rrrVvLINSnj8xfRFS+MQDpiQGIyMDUatkb9PXXcifEzEzte23ayC6b3r2BSpX0qubePblIbeFCeQTavXvyuouL3PB6yBAZivScn01EZooBSE8MQERGdPu2PIl+6VI5b6hgh2lHR+DZZ+V8oc6d5U7VekhNlXlr4ULg1Cnt9UaNgKFDZeeTp6ee34WIzAoDkJ4YgIhM5MoVuYJs6VLgr7+01318gEGD5BKvOnX0qkIIYNcuua/QmjW6y+l79pRhKDxc7ylJRGQGGID0xABEZGJCyOVdS5fKOUP//qt976mnZEp5/nl5aKseMjLk0WcLFgBJSdrrNWtql9PXrKlXFUSkIAYgPTEAESkoN1fuOr1ggTyUteD/RFWqBLzyipzIExCgdzXJydrl9DdvymsqFdCpk8xbzz4rN7omIsvBAKQnBiAiM3HhglzatXgxcPGi9nrLljKl9O0LeHjoVcXt23I5/YIF8tD7Ap6e2uX0jRrpVQURmQgDkJ4YgIjMTH4+8OuvMqV8/z1w96687uwsV48NHQq0bav38q4zZ+Ry+iVLdJfTh4bKKnr3Blxd9fwuRGQ0DEB6YgAiMmNXr2pPSz1+XHu9fn3taaleXnpVce8esGWLrGLTJu3Zr66ustNp6FCgVSsupycyNwxAemIAIrIAQgCJiTKlrFoFZGfL63Z28oT6oUPlcno9T0u9ckUup1+wADh9Wnu9cWNZxSuvcDk9kblgANITAxCRhbl1S4agBQuAP//UXn/sMbm069VXAX9/vaoQAti5U7uc/s4ded3BQbuc/umnuZyeSEkMQHpiACKyYH/9JZd3ff01cP269np4uBwi69lT7+X0N29ql9MfOKC9XquWdjm9r69eVRBRGTAA6YkBiKgcyM2VE6YLltMXqFxZjlsNHQo0bap3NQcPapfTZ2TIawWn07/0kjydvmpVvashohJgANITAxBROXP+vFxKv2gR8M8/2uutWsleob59AT3/W799G1i3Tuat7du1121sgCefBF54QXY+sWeIyHgYgPTEAERUTuXny96gguX0BaelVqggj4wfMkQezqrn8q7Tp+WUpHXrdIfIcD9zvfCCfNStq1c1RPQABiA9MQARWYH0dO1y+hMntNcbNNCellqtmt7VnD8vN1pctw7YvVu7sTUANGkig1CvXnI0jsvqifTDAKQnBiAiKyIE8McfciLPqlVATo68bm8vz8MYOlRO6NFzOT3un1D//fcyDP32m7YDCpBnvhb0DLVqxdVkRGXBAKQnBiAiK5WZCaxcKcPQ3r3a676+cmnX4MGAn59BqrpxA/jhBxmGfv5Zu6weAHx85NmvL7wg5w/Z2RmkSqJyjwFITwxARITDh7XLuwqW06tUcjn90KFyeZejo0GqysqSO0+vWyd3nr51S/telSqyI+qFF2TVeq7gJyrXGID0xABERBp37gAbNsi5QgkJ2utVqsh5QkOGyMk8BpKbK6tZt05W+++/2vfc3IDu3WUY6tqV55IRPYgBSE8MQERUrLNntafTX7qkvR4SInuF+vSRKcVA7t2Tu0+vWycnUheu0tFRnvTxwgvy5I/KlQ1WLZHFYgDSEwMQET1Sfr6cuLNggZzIUzCb2cVF7ik0ZAjQurVBl3Wp1cC+fTIMffedPLm+gK0t0KGDdq+h6tUNVi2RRWEA0hMDEBGVWFqa9rTUU6e01+vWBcLCgNBQubdQ/foGC0RCyBM/1q2Tj8OHte+pVLK6F16QE6n1PAKNyKIwAOmJAYiISk0IudHPggXA6tVya+jCKleWvUJt2shHcLDBJvGcPq0NQ4XPggWA5s21y+sbNuReQ1S+MQDpiQGIiPSSkQHs2CH3F9qzRy6pL7zOHffPyAgIkGGooJfIz0/vhPLPP3Ly9Lp1sglqtfa9+vW1YSgoiGGIyh8GID0xABGRQeXlAYcOaQPRH38AKSlF7/P21g1ELVrote792jVg40YZhrZulc0oULOmNgy1aWOQfR6JFMcApCcGICIyun/+0YahP/6Qx8rfvat7j4ODDEGFQ5GPT5mqy8wEfvpJTqD+6SfthteAPPGjZ08Zhjp0kNUSWaLS/P02i83WZ82aBT8/Pzg5OSEkJAR7C+/AWow1a9agQYMGcHJyQtOmTfHTTz8Vuef48eN49tln4eHhARcXFwQHB+PixYtG/BZERKVQowbw0kvAF1/IiTsZGXLN++TJMo1Uqya7bBITgc8/l/c+9pgcJuvXD5g5E0hKKhqaHsLdXS5QW7NG9gxt2AAMHAhUrCiPRZs3D+jSRVY7YIBcdl84JBGVN4r3AK1atQoDBw7EnDlzEBISgunTp2PNmjU4efIkqhVzEOEff/yBJ598EnFxcXjmmWewfPlyTJ48GQcOHECT+5uRnTlzBq1atcKQIUPQr18/uLu74+jRo2jdunWxZT6IPUBEpDgh5L5DhXuJjhzRndSD+yfZBwdrJ1e3bg14epa4mrt3ge3btXsNpaVp33N2lnsNPfUU0K6dPLCVQ2VkzixqCCwkJATBwcGIj48HAKjVavj6+mL06NGIjo4ucn+fPn2QnZ2NTZs2aa61bt0agYGBmDNnDgCgb9++sLe3xzfffFOiNuTm5iI3N1fzOjMzE76+vgxARGRebt2SE6oL5hLt2QPcvFn0vnr1dIfNGjUq0emq+fmyw6lgr6ELF3Tf9/AAnnhChqEnn5QTqTlcRubEYgJQXl4eKlSogLVr16Jnz56a6xEREbh58ya+//77Ip+pWbMmoqKiEBkZqbkWGxuLDRs24NChQ1Cr1fDw8MB7772HXbt24eDBg/D390dMTIxOHYVNmDABH374YZHrDEBEZNbUauDECd1eohMnit7n4SF3qy7oJQoJkWNijyCEnJa0ebMcmfvjD90zynC/h6h1axmG2rWTz11cDPwdiUqhNAFI0TOGr127hvz8fHh5eelc9/Lywoni/iMGkJqaWuz9qampAID09HRkZWVh0qRJmDhxIiZPnowtW7bghRdewLZt2xAWFlakzJiYGERFRWleF/QAERGZNRsb2bvTqJHcfRqQB7cmJmp7iQrmF/3yi3zg/m6JTZro9hI9/rjOuniVSs6/btFCvr53Ty5k+/13GYh+/12eU7Ztm3wA8tT6li21PURt2wKVKpn8p0JUIooGIGNQ3x8ff+655/D2228DAAIDA/HHH39gzpw5xQYgR0dHOBroVGciIkVVrgx06yYfuJ9cjhzR7SU6d05eO3IEmDtX3ufpqQ1DbdrIJFOhgqZYOzs55BUUBLz9tuwhOn5cG4Z+/10ubEtMlI/PPpMhqmlTGYYKeom8vRX6uRA9QNEA5OnpCVtbW6QVnnUHIC0tDd4P+a/E29v7kfd7enrCzs4OjRo10rmnYcOG2LVrl8G/AxGRWbOzk9tBN28OjBghr6WmagPRnj3A/v1yadgPP8hHwecCA2U3TufOQPv2cszrPpVK2/n0+usyEF24oNtDdOqUPKbj8GHg/jRP1K2r7SF68kmD7P1IVCZmMQm6VatWmDlzJnC/B6dmzZoYNWrUQydB5+Tk4IeC/0gBtGnTBs2aNdNMgm7Tpg3q1KmjMwn6+eefh7OzM5YvX/6fbeIqMCKyKrm5csJP4V6iy5d173FykpsEde0qe5fq1PnPYlNTgV27tKHo0CEZlAp77DHdHqKGDUs0X5uoWKX6+y0UtnLlSuHo6CiWLFkijh07Jl577TVRsWJFkZqaKoQQYsCAASI6Olpz/+7du4WdnZ2YOnWqOH78uIiNjRX29vbiyJEjmnvWrVsn7O3txbx588Tff/8tZs6cKWxtbcXOnTtL1KaMjAwBQGRkZBjhGxMRmTm1Wojz54VYsUKIYcOEqFFDCJldtI+6dYV46y0htmwR4vbtEhV744YQmzYJ8f77QoSGCmFnV7TYKlWEeO45IaZNE2LfPiHu3jX6t6VypDR/vxUPQEIIMXPmTFGzZk3h4OAgWrVqJRITEzXvhYWFiYiICJ37V69eLerVqyccHBxE48aNxY8//likzIULF4rHH39cODk5iYCAALFhw4YSt4cBiIioELVaiCNHhJgyRYj27YsmF2dnIbp3FyI+XoizZ0tcbHa2EL/9JsSECUI8/bQs5sFA5OoqRKdOQkycKMSOHSXOWmSlSvP3W/EhMHPEITAiokfIzAR+/VWukf/pp6LDZfXry2Gyrl3l2FYJF5nk5QEHDmjnEO3aVXSbIwcHoFUr7bBZmzaAm5sBvxtZNIvZB8hcMQAREZWQEHI1WUEY2r1b7qhYwMVFbiXdtat8+PmVuGi1GvjrL92J1fd3PNGwsZHzuwvmELVrV6qNsKmcYQDSEwMQEVEZ3byp7R3avBm4ckX3/YYNtb1D7dqVaitpIYDTp7VhaOdOeVrIgxo10q40a9cO4LZu1oMBSE8MQEREBiCEXPpV0Du0Z49u75CrK/D009reoZo1S13FP//IIFQQio4eLXqPn58MQm3byiGzxo250qy8YgDSEwMQEZER3LgBbN2q7R16YE83NG6s7R1q27ZMB439+6/u0vsDB3QzFyBPASm852OrVv95MghZCAYgPTEAEREZmVoNJCdre4cSE3VPundzA8LDtb1DNWqUqZpbt2TH086d8t/ERCA7W/ceGxu5Y3VBIGrTBvD35waNlogBSE8MQEREJnb9ujyrrKB36OpV3febNtX2DrVpA9jbl6mae/fkxOqC/R4LTgZ5kJeXbiBq0ULuBUnmjQFITwxAREQKUqvl2FVB79Cff+puIe3uDnTsqO0d8vHRq7orV3Q3wU5KkkvyC3NwkOegFQSi0FCgenW9qiUjYADSEwMQEZEZuXZN2zu0ZYt8XVhAgLZ3KDRUnmOmhzt3ZP4qCES7dwPp6UXv8/fX7SVq0kTvqklPDEB6YgAiIjJTarU8vLWgd2jfPt3eIQ8PoFMnGYa6dDFIN40Qcpis8LDZ4cNFzzVzdQVCQrSBqHVroGJFvaunUmAA0hMDEBGRhbh6Ffj5ZxmIfv5ZLgMrrHlz2TvUpYtc7lWGlWXFycwE9u7VBqI9e+S1wlQquSdR4V6iunU5udqYGID0xABERGSB8vNlj1BB79D+/brvOznJLponnpAbA4WGGmz9e34+cPy4dsjsjz/kpo0PqlJFNxC1bAlUqGCQJhADkP4YgIiIyoG0NG3v0K+/Fp07ZGMDNGsmw9ATT8iHnhOqC0tP151cvW8fkJure4+dneykKhyKyrjinxiA9McARERUzggBnDwpd0nctevh52j4++sGogYNDDZmlZcHHDyoO7n6wZNCAHl0R+FAFBBQ5lX/VocBSE8MQEREVuDyZZlCdu6UoejQId3NGHF/zKogDLVrJ7trDDSPSAjg4kXdydWHDhXdudrZWU5fKrxzdbVqBmlCucMApCcGICIiK5SZKcesCnqJEhPlmvjCnJ3lPKKCXqLWrQ16jkZWlhwqKzy5+saNovf5+gLBwXIOUXCw3KOoUiWDNcNiMQDpiQGIiIiQlyc3BCoYMtu1S+5YXZiNjRyjKjxsZsAdEtVqOXJXOBCdOFF0CT4APP64DEQFoah5c3miiDVhANITAxARERVRkEYKwtCuXcWfo1Gnju6wWb16Bl37npkp5xLt3y97i/bvB86cKXqfSgU0bKgNRC1byqzm7GywppgdBiA9MQAREVGJXLqkDUMF84ge/LPq6akNQ088IbtmDDyr+fp1eYRHQSDavx9ISSl6n52d3LG6cChq2rT8TLJmANITAxAREZVJRoZ2HtHOnfIcswfXvleoUHQekRHGqlJTdUPRvn3FH+nh6Ch7hgoCUcuWsufI1tbgTTI6BiA9MQAREZFB5ObKeUSFh80enNVsawsEBmqHzZ54AvD2NnhThAD++Ud36Gz//uInWVeoALRooTvRuk4dOeXJnDEA6YkBiIiIjEKtlltGF96P6MKFovc9/rjusJmRztAQQm6HVDgQJSXJ1WgP8vCQq80Kh6KaNc3raA8GID0xABERkcmkpMj9iAoC0ZEjRecRVa2qDURt2hh0P6IH5ecDp07pDp0lJxfdEaCgWQXDZgWhyICL4EqNAUhPDEBERKSYmzflPKKCYbO9e4vOI3JykokjNFQGotBQwMvLaE26exc4dkw3FB0+DNy7V/ReH5+iexR5ehqtaToYgPTEAERERGYjN1emjoJhsz17ip56DwC1a2u3iw4Nlcu7jDiT+c4dGYIKhs727ZMh6cHNtAHAz083FLVoIYfUDI0BSE8MQEREZLaEAP7+W3d3xKNHiw6bubrK1WYFgah1a6NvF52dXXSPolOnir+3d29g1SrD1s8ApCcGICIisig3b8ol9wXHzycmArduFb2vUSPtsFmbNnKTRiMv7crIkBOrC4ei8+eBUaOAmTMNWxcDkJ4YgIiIyKLl58vxqMK9RH//XfS+SpV0A1FwsOw5MrKrV+W8Ih8fw5bLAKQnBiAiIip3rl7V9hDt2SMnVz+4tKvgbLOCYbM2beQEHnNa6/4IDEB6YgAiIqJyLy9PHt1REIr++KP48zO8vXUDUYsWchWaGWIA0hMDEBERWaV//tHtJTpwQI5VFebgIENQ4RVnhh7LKiMGID0xABEREQG4fVvOYC4IRH/8UfyBYrVq6fYSNWumyAmrpfn7bRanesyaNQt+fn5wcnJCSEgI9u7d+8j716xZgwYNGsDJyQlNmzbFTz/99NB733jjDahUKkyfPt0ILSciIirHnJ3lDtTvvQesXy9PWD19GvjmG2D4cDlfyMZGHuexYgXw5ptys5+KFYEOHYCxY4FNm4rft0hhdko3YNWqVYiKisKcOXMQEhKC6dOno3Pnzjh58iSqVatW5P4//vgD/fr1Q1xcHJ555hksX74cPXv2xIEDB9CkSROde9evX4/ExET4mEnXHBERkUVTqeSpqHXqAK+8Iq/duiUnVBf0Eu3ZI5flb98uHwXq1dMOm7VpI4+cV/B0VcWHwEJCQhAcHIz4+HgAgFqthq+vL0aPHo3o6Ogi9/fp0wfZ2dnYtGmT5lrr1q0RGBiIOXPmaK5dunQJISEh+Pnnn9G9e3dERkYiMjKyRG3iEBgREVEZqdXAiRO6k6tPnCh6X8eOwC+/GLTq0vz9VrQHKC8vD0lJSYiJidFcs7GxQXh4OPbs2VPsZ/bs2YOoqCida507d8aGDRs0r9VqNQYMGIAxY8agcePG/9mO3Nxc5BY6ZyUzM7OM34iIiMjK2djIDRcbNQKGDJHXrl+XmzMW9BL9+accPlOQogHo2rVryM/Ph9cDB7h5eXnhRHFpEUBqamqx96empmpeT548GXZ2dnjzzTdL1I64uDh8+OGHZfoORERE9B8qVwa6dZMPQJ6impOjaJPMYhK0ISUlJeHLL7/EkiVLoCrhxk0xMTHIyMjQPFKK2weBiIiIDMPODlB4iomiAcjT0xO2trZIS0vTuZ6WlgZvb+9iP+Pt7f3I+3fu3In09HTUrFkTdnZ2sLOzw4ULF/DOO+/Az8+v2DIdHR3h7u6u8yAiIqLyS9EA5ODggKCgICQkJGiuqdVqJCQkIDQ0tNjPhIaG6twPAFu3btXcP2DAABw+fBjJycmah4+PD8aMGYOff/7ZyN+IiIiILIHiy+CjoqIQERGBli1bolWrVpg+fTqys7MxePBgAMDAgQPx2GOPIS4uDgDw1ltvISwsDNOmTUP37t2xcuVK7N+/H/PmzQMAVKlSBVWqVNGpw97eHt7e3qhfv74C35CIiIjMjeIBqE+fPrh69SrGjx+P1NRUBAYGYsuWLZqJzhcvXoRNoX0C2rRpg+XLl+N///sfxo4di7p162LDhg1F9gAiIiIiehjF9wEyR9wHiIiIyPJY3FEYRERERKbEAERERERWhwGIiIiIrA4DEBEREVkdBiAiIiKyOgxAREREZHUYgIiIiMjqMAARERGR1VF8J2hzVLA3ZGZmptJNISIiohIq+Ltdkj2eGYCKcevWLQCAr6+v0k0hIiKiUrp16xY8PDweeQ+PwiiGWq3G5cuX4ebmBpVKZdCyMzMz4evri5SUFB6zYQb4+zAv/H2YF/4+zA9/J48mhMCtW7fg4+Ojc45ocdgDVAwbGxvUqFHDqHW4u7vzf7xmhL8P88Lfh3nh78P88HfycP/V81OAk6CJiIjI6jAAERERkdVhADIxR0dHxMbGwtHRUemmEH8fZoe/D/PC34f54e/EcDgJmoiIiKwOe4CIiIjI6jAAERERkdVhACIiIiKrwwBEREREVocByIRmzZoFPz8/ODk5ISQkBHv37lW6SVYrLi4OwcHBcHNzQ7Vq1dCzZ0+cPHlS6WYRgEmTJkGlUiEyMlLppli1S5cu4ZVXXkGVKlXg7OyMpk2bYv/+/Uo3yyrl5+dj3Lhx8Pf3h7OzM+rUqYOPP/64ROdd0cMxAJnIqlWrEBUVhdjYWBw4cAABAQHo3Lkz0tPTlW6aVdqxYwdGjhyJxMREbN26FXfv3kWnTp2QnZ2tdNOs2r59+zB37lw0a9ZM6aZYtRs3bqBt27awt7fH5s2bcezYMUybNg2VKlVSumlWafLkyZg9ezbi4+Nx/PhxTJ48GVOmTMHMmTOVbppF4zJ4EwkJCUFwcDDi4+OB++eN+fr6YvTo0YiOjla6eVbv6tWrqFatGnbs2IEnn3xS6eZYpaysLLRo0QJfffUVJk6ciMDAQEyfPl3pZlml6Oho7N69Gzt37lS6KQTgmWeegZeXFxYuXKi51qtXLzg7O+Pbb79VtG2WjD1AJpCXl4ekpCSEh4drrtnY2CA8PBx79uxRtG0kZWRkAAAqV66sdFOs1siRI9G9e3ed/05IGRs3bkTLli3x0ksvoVq1amjevDnmz5+vdLOsVps2bZCQkIBTp04BAA4dOoRdu3aha9euSjfNovEwVBO4du0a8vPz4eXlpXPdy8sLJ06cUKxdJKnVakRGRqJt27Zo0qSJ0s2xSitXrsSBAwewb98+pZtCAM6ePYvZs2cjKioKY8eOxb59+/Dmm2/CwcEBERERSjfP6kRHRyMzMxMNGjSAra0t8vPz8cknn6B///5KN82iMQCR1Rs5ciT++usv7Nq1S+mmWKWUlBS89dZb2Lp1K5ycnJRuDt3/fwpatmyJTz/9FADQvHlz/PXXX5gzZw4DkAJWr16NZcuWYfny5WjcuDGSk5MRGRkJHx8f/j70wABkAp6enrC1tUVaWprO9bS0NHh7eyvWLgJGjRqFTZs24ffff0eNGjWUbo5VSkpKQnp6Olq0aKG5lp+fj99//x3x8fHIzc2Fra2tom20NtWrV0ejRo10rjVs2BDfffedYm2yZmPGjEF0dDT69u0LAGjatCkuXLiAuLg4BiA9cA6QCTg4OCAoKAgJCQmaa2q1GgkJCQgNDVW0bdZKCIFRo0Zh/fr1+O233+Dv7690k6zW008/jSNHjiA5OVnzaNmyJfr374/k5GSGHwW0bdu2yLYQp06dQq1atRRrkzXLycmBjY3un2tbW1uo1WrF2lQesAfIRKKiohAREYGWLVuiVatWmD59OrKzszF48GClm2aVRo4cieXLl+P777+Hm5sbUlNTAQAeHh5wdnZWunlWxc3NrcjcKxcXF1SpUoVzshTy9ttvo02bNvj000/Ru3dv7N27F/PmzcO8efOUbppV6tGjBz755BPUrFkTjRs3xsGDB/H555/j1VdfVbppFo3L4E0oPj4en332GVJTUxEYGIgZM2YgJCRE6WZZJZVKVez1xYsXY9CgQSZvD+lq3749l8ErbNOmTYiJicHff/8Nf39/REVFYdiwYUo3yyrdunUL48aNw/r165Geng4fHx/069cP48ePh4ODg9LNs1gMQERERGR1OAeIiIiIrA4DEBEREVkdBiAiIiKyOgxAREREZHUYgIiIiMjqMAARERGR1WEAIiIiIqvDAERERERWhwGIiOghVCoVNmzYoHQziMgIGICIyCwNGjQIKpWqyKNLly5KN42IygEehkpEZqtLly5YvHixzjVHR0fF2kNE5Qd7gIjIbDk6OsLb21vnUalSJeD+8NTs2bPRtWtXODs7o3bt2li7dq3O548cOYKnnnoKzs7OqFKlCl577TVkZWXp3LNo0SI0btwYjo6OqF69OkaNGqXz/rVr1/D888+jQoUKqFu3LjZu3Kh578aNG+jfvz+qVq0KZ2dn1K1bt0hgIyLzxABERBZr3Lhx6NWrFw4dOoT+/fujb9++OH78OAAgOzsbnTt3RqVKlbBv3z6sWbMGv/76q07AmT17NkaOHInXXnsNR44cwcaNG/H444/r1PHhhx+id+/eOHz4MLp164b+/fvj+vXrmvqPHTuGzZs34/jx45g9ezY8PT1N/FMgojIRRERmKCIiQtja2goXFxedxyeffCKEEAKAeOONN3Q+ExISIoYPHy6EEGLevHmiUqVKIisrS/P+jz/+KGxsbERqaqoQQggfHx/xwQcfPLQNAMT//vc/zeusrCwBQGzevFkIIUSPHj3E4MGDDfzNicgUOAeIiMxWhw4dMHv2bJ1rlStX1jwPDQ3VeS80NBTJyckAgOPHjyMgIAAuLi6a99u2bQu1Wo2TJ09CpVLh8uXLePrppx/ZhmbNmmmeu7i4wN3dHenp6QCA4cOHo1evXjhw4AA6deqEnj17ok2bNnp+ayIyBQYgIjJbLi4uRYakDMXZ2blE99nb2+u8VqlUUKvVAICuXbviwoUL+Omnn7B161Y8/fTTGDlyJKZOnWqUNhOR4XAOEBFZrMTExCKvGzZsCABo2LAhDh06hOzsbM37u3fvho2NDerXrw83Nzf4+fkhISFBrzZUrVoVERER+PbbbzF9+nTMmzdPr/KIyDTYA0REZis3Nxepqak61+zs7DQTjdesWYOWLVviiSeewLJly7B3714sXLgQANC/f3/ExsYiIiICEyZMwNWrVzF69GgMGDAAXl5eAIAJEybgjTfeQLVq1dC1a1fcunULu3fvxujRo0vUvvHjxyMoKAiNGzdGbm4uNm3apAlgRGTeGICIyGxt2bIF1atX17lWv359nDhxAri/QmvlypUYMWIEqlevjhUrVqBRo0YAgAoVKuDnn3/GW2+9heDgYFSoUAG9evXC559/rikrIiICd+7cwRdffIF3330Xnp6eePHFF0vcPgcHB8TExOD8+fNwdnZGu3btsHLlSoN9fyIyHpWQKx2IiCyKSqXC+vXr0bNnT6WbQkQWiHOAiIiIyOowABEREZHV4RwgIrJIHL0nIn2wB4iIiIisDgMQERERWR0GICIiIrI6DEBERERkdRiAiIiIyOowABEREZHVYQAiIiIiq8MARERERFbn/xwXfxJ9aWXFAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.suptitle('MSE vs Epochs')\n",
    "plt.plot(tune_train_err, label='Train', color='blue')\n",
    "plt.plot(tune_test_err, label='Test', color='red')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('MSE')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[156., 353., 404., ...,   0.,   0.,   0.],\n",
       "        [  0.,   0.,   0., ...,   0.,   0.,   0.],\n",
       "        [  0.,   0.,   0., ...,   0.,   0.,   0.],\n",
       "        ...,\n",
       "        [  0.,   0.,   0., ...,   0.,   0.,   0.],\n",
       "        [  0.,   0.,   0., ...,   0.,   0.,   0.],\n",
       "        [  0.,   0.,   0., ...,   0.,   0.,   4.]]),\n",
       " array([1.   , 1.06 , 1.12 , 1.18 , 1.24 , 1.3  , 1.359, 1.42 , 1.48 ,\n",
       "        1.54 , 1.6  , 1.66 , 1.72 , 1.779, 1.84 , 1.9  , 1.96 , 2.02 ,\n",
       "        2.08 , 2.14 , 2.2  , 2.26 , 2.32 , 2.379, 2.44 , 2.5  , 2.559,\n",
       "        2.621, 2.68 , 2.74 , 2.8  , 2.86 , 2.92 , 2.98 , 3.04 , 3.1  ,\n",
       "        3.16 , 3.22 , 3.28 , 3.34 , 3.4  , 3.459, 3.52 , 3.58 , 3.64 ,\n",
       "        3.7  , 3.76 , 3.82 , 3.879, 3.94 , 4.   ], dtype=float16),\n",
       " array([0.879 , 0.9287, 0.9785, 1.028 , 1.078 , 1.129 , 1.179 , 1.229 ,\n",
       "        1.278 , 1.328 , 1.378 , 1.428 , 1.479 , 1.527 , 1.578 , 1.628 ,\n",
       "        1.678 , 1.728 , 1.777 , 1.828 , 1.877 , 1.928 , 1.978 , 2.027 ,\n",
       "        2.078 , 2.127 , 2.176 , 2.227 , 2.277 , 2.328 , 2.377 , 2.426 ,\n",
       "        2.477 , 2.527 , 2.576 , 2.625 , 2.676 , 2.727 , 2.777 , 2.826 ,\n",
       "        2.875 , 2.926 , 2.977 , 3.025 , 3.076 , 3.125 , 3.176 , 3.225 ,\n",
       "        3.275 , 3.326 , 3.375 ], dtype=float16),\n",
       " <matplotlib.collections.QuadMesh at 0x7f9dc16ae2a0>)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi4AAAGdCAYAAAA1/PiZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAcy0lEQVR4nO3df3DV5Z0v8E8QOOBKTsvekqAElxYbf5Uf2romzgq2Iku5XfOP6zo7hW51Z+vCDIw77ZXObnu3ndnoqGWdqQM6TmXaHYaWdsEdrKVRClwVu4BkFmjrrRSF2iT2zrg5kkpE8r1/dM0STQ458SQnD3m9Zr5/nG+e53w/55lnct558v1RlWVZFgAACRhX6QIAAAZLcAEAkiG4AADJEFwAgGQILgBAMgQXACAZggsAkAzBBQBIxvhKFzAYPT098Zvf/CamTJkSVVVVlS4HABiELMvijTfeiAsvvDDGjSvPWkkSweU3v/lN1NXVVboMAGAIjh8/HjNmzCjLeyURXKZMmRLxXx+8urq60uUAwJj3p7c+eNY2b799MvY+9U+93+PlkERweeffQ9XV1YILAIwC4ydMGnTbcp7m4eRcACAZggsAkAzBBQBIhuACACSjKsuyrNJFnE2hUIh8Ph+dnZ1OzgWARAzH97cVFwAgGYILAJAMwQUASIbgAgAkQ3ABAJIhuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMwQUASIbgAgAkQ3ABAJIhuAAAyRhf6QIAgPQsGnfLWdu8nZ0q+3GtuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMwQUASIbLoQEStuTyLw+q3ZM/+6dhr4WxpaVn81nbFAqFyOfzZT2uFRcAIBmCCwCQDMEFAEiG4AIAJENwAQCSIbgAAMkQXACAZLiPC0DC3v7FLytdAowoKy4AQDIEFwAgGYILAJAMwQUASIbgAgAkQ3ABAJIhuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMwQUASIbgAgAko6SnQ69bty7WrVsXL7/8ckREXHHFFfGVr3wllixZMmCfzZs3xz/8wz/Eyy+/HJdccknce++98elPf/r9Vw5AdC5rqHQJMKJKWnGZMWNG3HPPPbF///7Yt29ffPKTn4ybb745Dh8+3G/75557Lm677ba4/fbb48CBA9HU1BRNTU1x6NChctUPAIwhVVmWZe/nDaZOnRr33Xdf3H777e/52a233hpdXV2xbdu23n3XXnttzJs3L9avXz/oYxQKhcjn89HZ2RnV1dXvp1yAc8o1n/vGoNr9+4a7hr0WeLfh+P4e8jkup0+fjk2bNkVXV1c0NPS/VLlnz5648cYb++xbvHhx7Nmzp+h7d3d3R6FQ6LMBAJQcXA4ePBgXXHBB5HK5+MIXvhBbtmyJyy+/vN+27e3tUVNT02dfTU1NtLe3Fz1Gc3Nz5PP53q2urq7UMgGAc1DJwaW+vj5aW1vjpz/9adx5552xfPny+NnPflbWotasWROdnZ292/Hjx8v6/gBAmkq6qigiYuLEiTF79uyIiLj66qtj79698eCDD8bDDz/8nra1tbXR0dHRZ19HR0fU1tYWPUYul4tcLldqaQDAOe5938elp6cnuru7+/1ZQ0NDPP300332tbS0DHhODABAMSWtuKxZsyaWLFkSM2fOjDfeeCM2btwYO3fujO3bt0dExLJly+Kiiy6K5ubmiIhYtWpVLFiwIB544IFYunRpbNq0Kfbt2xePPPLI8HwagDHmD//9/1W6BBhRJQWX1157LZYtWxZtbW2Rz+djzpw5sX379li0aFFERBw7dizGjfvvRZzGxsbYuHFj/P3f/318+ctfjksuuSS2bt0aV155Zfk/CQBwznvf93EZCe7jAtC/JZd/eVDtnvzZPw17LfBuo+o+LgAAI01wAQCSIbgAAMkQXACAZJR8AzoARo+OP/lQpUuAEWXFBQBIhuACACRDcAEAkiG4AADJEFwAgGQILgBAMgQXACAZ7uMCkLCeCZWuAEaWFRcAIBmCCwCQDMEFAEiG4AIAJENwAQCSIbgAAMlwOTRAwlwOzVhjxQUASIbgAgAkQ3ABAJIhuAAAyRBcAIBkCC4AQDJcDg2QsJP/o9IVMFYtGnfLWdu8nZ0q+3GtuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMwQUASIbgAgAkw31cABJ2wbFKV8BYNf7SS87e6HR3xP8t73GtuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMwQUASIbLoaGIwTy2PSKipWfzsNcC/Tn5oUpXwFj19i9+efY22amyH9eKCwCQDMEFAEiG4AIAJENwAQCSIbgAAMkQXACAZAguAEAy3McFiuj+n9dUugQo6gO/7Kl0CYxRg7l/VaFQiHw+X9bjWnEBAJIhuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMl0NDEbv/7YuVLgGK+oPNzw+u4XeHuxIYGVZcAIBkCC4AQDIEFwAgGYILAJAMwQUASIbgAgAko6TLoZubm+Nf//Vf4xe/+EVMnjw5Ghsb49577436+voB+2zYsCH+6q/+qs++XC4XJ0+eHHrVAER4gjljUEkrLrt27YoVK1bE888/Hy0tLXHq1Km46aaboqurq2i/6urqaGtr691eeeWV91s3ADAGlbTi8qMf/ajP6w0bNsS0adNi//79cf311w/Yr6qqKmpra4deJQDA+z3HpbOzMyIipk6dWrTdiRMn4uKLL466urq4+eab4/Dhw0Xbd3d3R6FQ6LMBAAw5uPT09MTq1avjuuuuiyuvvHLAdvX19fGtb30rHn/88fiXf/mX6OnpicbGxvj1r389YJ/m5ubI5/O9W11d3VDLBADOIVVZlmVD6XjnnXfGk08+Gc8880zMmDFj0P1OnToVl112Wdx2223x9a9/vd823d3d0d3d3fu6UChEXV1ddHZ2RnV19VDKBTgnXf9n9w2qneduUQmFQiHy+XxZv7+H9JDFlStXxrZt22L37t0lhZaIiAkTJsT8+fPjpZdeGrBNLpeLXC43lNIAgHNYSf8qyrIsVq5cGVu2bIkdO3bErFmzSj7g6dOn4+DBgzF9+vSS+wIAY1tJKy4rVqyIjRs3xuOPPx5TpkyJ9vb2iIjI5/MxefLkiIhYtmxZXHTRRdHc3BwREV/72tfi2muvjdmzZ8d//ud/xn333RevvPJK3HHHHcPxeaCsFo27ZVDtWno2D3stAJQYXNatWxcREQsXLuyz/7HHHovPfe5zERFx7NixGDfuvxdyXn/99fjrv/7raG9vjw9+8INx9dVXx3PPPReXX355eT4BADBmlBRcBnMe786dO/u8Xrt2baxdu7b0ygAA3sWzigCAZAguAEAyBBcAIBmCCwCQjCHdgA7Git/+bWOlS4Cictv+vdIlwIiy4gIAJENwAQCSIbgAAMkQXACAZAguAEAyBBcAIBkuh4YiTk2pdAUAnMmKCwCQDMEFAEiG4AIAJENwAQCSIbgAAMkQXACAZAguAEAy3McFiui67K1KlwBFtfRsrnQJMKKsuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMwQUASIbLoaGIca9PqHQJUNSicbcMqp3LpjlXWHEBAJIhuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMwQUASIb7uEARPVPfqnQJUJT7szDWWHEBAJIhuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMl0NDEdlbsj2j26JxtwyqncumOVf4rQwAJENwAQCSIbgAAMkQXACAZAguAEAyBBcAIBkuh4YiqkR7RjmXOTPW+LUMACRDcAEAkiG4AADJEFwAgGQILgBAMgQXACAZggsAkAz3cYFizuupdAUAnMGKCwCQDMEFAEiG4AIAJENwAQCSIbgAAMkQXACAZJQUXJqbm+MTn/hETJkyJaZNmxZNTU3x4osvnrXf5s2b49JLL41JkybFxz72sfjhD3/4fmqGkdNTNbgNgBFRUnDZtWtXrFixIp5//vloaWmJU6dOxU033RRdXV0D9nnuuefitttui9tvvz0OHDgQTU1N0dTUFIcOHSpH/QDAGFKVZVk21M6//e1vY9q0abFr1664/vrr+21z6623RldXV2zbtq1337XXXhvz5s2L9evXD+o4hUIh8vl8dHZ2RnV19VDLhZL90YZ7B9Xu5c/9r2GvBSA1w/H9/b7Ocens7IyIiKlTpw7YZs+ePXHjjTf22bd48eLYs2fPgH26u7ujUCj02QAAhhxcenp6YvXq1XHdddfFlVdeOWC79vb2qKmp6bOvpqYm2tvbB+zT3Nwc+Xy+d6urqxtqmQDAOWTIwWXFihVx6NCh2LRpU3kriog1a9ZEZ2dn73b8+PGyHwMASM+QHrK4cuXK2LZtW+zevTtmzJhRtG1tbW10dHT02dfR0RG1tbUD9snlcpHL5YZSGgBwDitpxSXLsli5cmVs2bIlduzYEbNmzTprn4aGhnj66af77GtpaYmGhobSqwUAxrSSVlxWrFgRGzdujMcffzymTJnSe55KPp+PyZMnR0TEsmXL4qKLLorm5uaIiFi1alUsWLAgHnjggVi6dGls2rQp9u3bF4888shwfB4oq6oTQ1qUBGCYlLTism7duujs7IyFCxfG9OnTe7fvfve7vW2OHTsWbW1tva8bGxtj48aN8cgjj8TcuXPj+9//fmzdurXoCb0AAP0p6c/JwdzyZefOne/Zd8stt8Qtt9xSWmUAAO/iWUUAQDIEFwAgGYILAJAMwQUASIZrPaGIqj/srnQJAJzBigsAkAzBBQBIhuACACRDcAEAkiG4AADJEFwAgGS4HBqKOfvjuQAYQVZcAIBkCC4AQDIEFwAgGYILAJAMwQUASIbgAgAkQ3ABAJLhPi5QxOSDkytdAgBnsOICACRDcAEAkiG4AADJEFwAgGQILgBAMgQXACAZLoeGIk5dUOkKADiTFRcAIBmCCwCQDMEFAEiG4AIAJENwAQCSIbgAAMkQXACAZLiPCxTx9vlZpUsA4AxWXACAZAguAEAyBBcAIBmCCwCQDMEFAEiG4AIAJMPl0FDE+N9VVboEAM5gxQUASIbgAgAkQ3ABAJIhuAAAyRBcAIBkCC4AQDJcDg1FTDhR6QoAOJMVFwAgGYILAJAMwQUASIbgAgAkQ3ABAJIhuAAAyRBcAIBkuI8LFHE6V+kKADiTFRcAIBmCCwCQDMEFAEiG4AIAJENwAQCSIbgAAMko+XLo3bt3x3333Rf79++Ptra22LJlSzQ1NQ3YfufOnXHDDTe8Z39bW1vU1taWdOw/W3J/jB8/qWibp3Z9uaT3hGJm/OTNwTX838NdCQAxlBWXrq6umDt3bjz00EMl9XvxxRejra2td5s2bVqphwYAxriSV1yWLFkSS5YsKflA06ZNiw984AMl9wMAeMeIneMyb968mD59eixatCieffbZom27u7ujUCj02QAAhj24TJ8+PdavXx8/+MEP4gc/+EHU1dXFwoUL44UXXhiwT3Nzc+Tz+d6trq5uuMsEABIw7M8qqq+vj/r6+t7XjY2NceTIkVi7dm185zvf6bfPmjVr4q677up9XSgUhBcAoDIPWbzmmmvimWeeGfDnuVwucjlPtwMA+qpIcGltbY3p06eX3O+t6gnRM2HCsNQE/an6PwcqXQIAZyg5uJw4cSJeeuml3tdHjx6N1tbWmDp1asycOTPWrFkTr776anz729+OiIh//ud/jlmzZsUVV1wRJ0+ejEcffTR27NgRP/7xj8v7SQCAc17JwWXfvn19bij3zrkoy5cvjw0bNkRbW1scO3as9+dvvfVW/N3f/V28+uqrcf7558ecOXPiqaee6vemdAAAxVRlWZZVuoizKRQKkc/no+FPvxbjJxS/c+7uf/viiNXFuW/RuFsG1a6lZ/Ow1wKQmne+vzs7O6O6uros7+lZRQBAMgQXACAZggsAkAzBBQBIRkXu4zJUEwunYvz48ypdBmOIk24BRhcrLgBAMgQXACAZggsAkAzBBQBIhuACACRDcAEAkpHU5dC/Xjg5zptU/FlFUE6eVQQwulhxAQCSIbgAAMkQXACAZAguAEAyBBcAIBmCCwCQDMEFAEhGUvdxmfzbiPMmVroKxpLsT+ZXugQAzmDFBQBIhuACACRDcAEAkiG4AADJEFwAgGQILgBAMpK6HLrz6u4YN7mq0mUwhvyuNlfpEgA4gxUXACAZggsAkAzBBQBIhuACACRDcAEAkiG4AADJEFwAgGQkdR+XObNejQl/MLHSZTCGnLhQtgcYTfxWBgCSIbgAAMkQXACAZAguAEAyBBcAIBmCCwCQjKQuh54y/mRMnNBT6TIYQ7qnVroCAM5kxQUASIbgAgAkQ3ABAJIhuAAAyRBcAIBkCC4AQDKSuhz6oxe0x6QLJlS6DMaQmr1vV7oEAM5gxQUASIbgAgAkQ3ABAJIhuAAAyRBcAIBkCC4AQDIEFwAgGUndx6Vh8pH4g/NlLUbO7n/7YqVLAOAMUgAAkAzBBQBIhuACACRDcAEAkiG4AADJEFwAgGSUHFx2794dn/nMZ+LCCy+Mqqqq2Lp161n77Ny5M6666qrI5XIxe/bs2LBhw5CKvXZSVVw3aVzRDQA4d5X8Td/V1RVz586Nhx56aFDtjx49GkuXLo0bbrghWltbY/Xq1XHHHXfE9u3bh1IvADCGlXwDuiVLlsSSJUsG3X79+vUxa9aseOCBByIi4rLLLotnnnkm1q5dG4sXLy718ADAGDbs/1vZs2dP3HjjjX32LV68OPbs2TNgn+7u7igUCn02AIBhDy7t7e1RU1PTZ19NTU0UCoV48803++3T3Nwc+Xy+d6urqxvuMgGABIzKs1nXrFkTnZ2dvdvx48crXRIAMAoM+0MWa2tro6Ojo8++jo6OqK6ujsmTJ/fbJ5fLRS6XG+7SAIDEDPuKS0NDQzz99NN99rW0tERDQ8NwHxoAOMeUvOJy4sSJeOmll3pfHz16NFpbW2Pq1Kkxc+bMWLNmTbz66qvx7W9/OyIivvCFL8Q3v/nN+NKXvhSf//znY8eOHfG9730vnnjiiZKLPdHTHeN6imet6pLfFQBIRckrLvv27Yv58+fH/PnzIyLirrvuivnz58dXvvKViIhoa2uLY8eO9bafNWtWPPHEE9HS0hJz586NBx54IB599FGXQgMAJavKsiyrdBFnUygUIp/Px/FfXBTVU86y4nLhsaI/BwBGxjvf352dnVFdXZ7/iYzKq4oAAPojuAAAyRBcAIBkCC4AQDKG/QZ05fTdE7Ni8llK/tsRqwYAGGlWXACAZAguAEAyBBcAIBmCCwCQDMEFAEhGElcVvfNUgpMn3j5r20KhMAIVAQBn8853cjmfLpTEs4p+9atfxUc+8pFKlwEADMGRI0fiwx/+cFneK4kVl6lTp0ZExLFjxyKfz1e6nGQVCoWoq6uL48ePl+1hV2OVsSwfY1kexrF8jGX5dHZ2xsyZM3u/x8shieAybtzvT8XJ5/MmURlUV1cbxzIxluVjLMvDOJaPsSyfd77Hy/JeZXsnAIBhJrgAAMlIIrjkcrn46le/GrlcrtKlJM04lo+xLB9jWR7GsXyMZfkMx1gmcVURAECksuICABCCCwCQEsEFAEiG4AIAJKPiwWX37t3xmc98Ji688MKoqqqKrVu3nrXPzp0746qrropcLhezZ8+ODRs2jEito12pY7lz586oqqp6z9be3j5iNY9Gzc3N8YlPfCKmTJkS06ZNi6ampnjxxRfP2m/z5s1x6aWXxqRJk+JjH/tY/PCHPxyRekezoYzlhg0b3jMnJ02aNGI1j1br1q2LOXPm9N4UraGhIZ588smifczJ9yp1HM3HwbnnnnuiqqoqVq9eXbRdOeZkxYNLV1dXzJ07Nx566KFBtT969GgsXbo0brjhhmhtbY3Vq1fHHXfcEdu3bx/2Wke7UsfyHS+++GK0tbX1btOmTRu2GlOwa9euWLFiRTz//PPR0tISp06diptuuim6uroG7PPcc8/FbbfdFrfffnscOHAgmpqaoqmpKQ4dOjSitY82QxnL+K87lp45J1955ZURq3m0mjFjRtxzzz2xf//+2LdvX3zyk5+Mm2++OQ4fPtxve3Oyf6WOY5iPZ7V37954+OGHY86cOUXblW1OZqNIRGRbtmwp2uZLX/pSdsUVV/TZd+utt2aLFy8e5urSMpix/MlPfpJFRPb666+PWF0peu2117KIyHbt2jVgmz//8z/Pli5d2mffH//xH2d/8zd/MwIVpmMwY/nYY49l+Xx+ROtK1Qc/+MHs0Ucf7fdn5uTgFRtH87G4N954I7vkkkuylpaWbMGCBdmqVasGbFuuOVnxFZdS7dmzJ2688cY++xYvXhx79uypWE2pmzdvXkyfPj0WLVoUzz77bKXLGXU6OzsjznjYZ3/My8EZzFhGRJw4cSIuvvjiqKurO+tfw2PR6dOnY9OmTdHV1RUNDQ39tjEnz24w4xjmY1ErVqyIpUuXvmeu9adcczKJhyyeqb29PWpqavrsq6mpiUKhEG+++WZMnjy5YrWlZvr06bF+/fr4+Mc/Ht3d3fHoo4/GwoUL46c//WlcddVVlS5vVOjp6YnVq1fHddddF1deeeWA7Qaal2P9fKEzDXYs6+vr41vf+lbMmTMnOjs74/7774/GxsY4fPhwzJgxY0RrHm0OHjwYDQ0NcfLkybjgggtiy5Ytcfnll/fb1pwcWCnjaD4ObNOmTfHCCy/E3r17B9W+XHMyueBC+dTX10d9fX3v68bGxjhy5EisXbs2vvOd71S0ttFixYoVcejQoXjmmWcqXUryBjuWDQ0Nff76bWxsjMsuuywefvjh+PrXvz4ClY5e9fX10draGp2dnfH9738/li9fHrt27RrwS5f+lTKO5mP/jh8/HqtWrYqWlpYRP1k5ueBSW1sbHR0dffZ1dHREdXW11ZYyuOaaa3xJ/5eVK1fGtm3bYvfu3Wf9y2qgeVlbWzvMVaahlLF8twkTJsT8+fPjpZdeGrb6UjFx4sSYPXt2RERcffXVsXfv3njwwQfj4Ycffk9bc3JgpYzju5mPv7d///547bXX+qzOnz59Onbv3h3f/OY3o7u7O84777w+fco1J5M7x6WhoSGefvrpPvtaWlqK/n+SwWttbY3p06dXuoyKyrIsVq5cGVu2bIkdO3bErFmzztrHvOzfUMby3U6fPh0HDx4c8/OyPz09PdHd3d3vz8zJwSs2ju9mPv7epz71qTh48GC0trb2bh//+MfjL//yL6O1tfU9oSXKOSeHfCpxmbzxxhvZgQMHsgMHDmQRkX3jG9/IDhw4kL3yyitZlmXZ3XffnX32s5/tbf+rX/0qO//887MvfvGL2c9//vPsoYceys4777zsRz/6UQU/xehQ6liuXbs227p1a/bLX/4yO3jwYLZq1aps3Lhx2VNPPVXBT1F5d955Z5bP57OdO3dmbW1tvdvvfve73jaf/exns7vvvrv39bPPPpuNHz8+u//++7Of//zn2Ve/+tVswoQJ2cGDByv0KUaHoYzlP/7jP2bbt2/Pjhw5ku3fvz/7i7/4i2zSpEnZ4cOHK/QpRoe7774727VrV3b06NHsP/7jP7K77747q6qqyn784x9nmTk5aKWOo/k4eO++qmi45mTFg8s7l+S+e1u+fHmWZVm2fPnybMGCBe/pM2/evGzixInZhz/84eyxxx6rUPWjS6ljee+992Yf+chHskmTJmVTp07NFi5cmO3YsaOCn2B06G8MI6LPPFuwYEHvuL7je9/7XvbRj340mzhxYnbFFVdkTzzxRAWqH12GMparV6/OZs6cmU2cODGrqanJPv3pT2cvvPBChT7B6PH5z38+u/jii7OJEydmH/rQh7JPfepTvV+2mTk5aKWOo/k4eO8OLsM1J6uy3/9yAQAY9ZI7xwUAGLsEFwAgGYILAJAMwQUASIbgAgAkQ3ABAJIhuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMwQUASMb/ByhxjawVIMzxAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "batch_src, batch_labels, batch_padding_mask = mktunebatch(2048)\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    output = model(batch_src, batch_padding_mask)\n",
    "x = batch_labels.detach().to(torch.float16).cpu().numpy().flatten()\n",
    "y = output.detach().to(torch.float16).cpu().numpy().flatten()\n",
    "plt.hist2d(x, y, bins=50, norm=mpl.colors.LogNorm())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JtTLXn4zC1z_"
   },
   "source": [
    "# Step 6: Test generalization"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}