path: root/transformer_shortest_paths.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LPphBnKR-aWF"
   },
   "source": [
    "# Step 0: Imports"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Question: \n",
    "\n",
    "- Do the attention heads learn to attend to the same positional encodings\n",
    "- do interp -- what is it doing? can we figure out?\n",
    "- update: I think we should do interp once it's bigger. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "execution_state": "idle",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ge5QvElvhCOw",
    "outputId": "c7cdaefa-d6dc-44ad-c258-e4fb2aca97a5"
   },
   "outputs": [],
   "source": [
    "# using tqdm.auto glitches out collaborative editing\n",
    "from tqdm import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from math import sqrt\n",
    "from collections import deque\n",
    "import os\n",
    "import random\n",
    "\n",
    "torch.manual_seed(42)\n",
    "random.seed(42)\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "assert device.type == 'cuda', \"CUDA is not available. Please check your GPU setup.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "execution_state": "idle",
   "metadata": {
    "id": "lylOX2POPwFL"
   },
   "outputs": [],
   "source": [
    "# VTXS numbers here are inclusive\n",
    "MIN_VTXS = 3\n",
    "MAX_VTXS = 31\n",
    "MAX_TUNE_VTXS = 31 # 15\n",
    "AVG_DEG = 2\n",
    "SEQ_LEN = MAX_VTXS + 1 # means 32 edges, final token is the target vertex\n",
    "PAD_TOKEN = 0\n",
    "# vertices are labelled 1,2,...,63\n",
    "# we also have a padding token which is 0."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gKt-yIpDebF1"
   },
   "source": [
    "# Step 1: Generate synthetic data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "execution_state": "idle",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1IbzGIWseK3E",
    "outputId": "a3cbc233-358c-4e17-ea6e-f4e9349d886b"
   },
   "outputs": [],
   "source": [
    "# original task data\n",
    "# the data will be edge lists\n",
    "# like this: [1 3 1 5 2 4 0 0 0 0 2]\n",
    "# this represents edges (1,3), (1,5) (2,4)\n",
    "# (the zeros are just padding tokens)\n",
    "# the final 2 means which vertex we're going to \n",
    "\n",
    "# the label is the shortest distance from vtx 1 to vtx 2\n",
    "# or \"number of vertices\" if no path exists\n",
    "\n",
    "def random_graph(n):\n",
    "    edge_list = []\n",
    "    adjacencies = [set() for _ in range(n+1)]\n",
    "    indices = [random.randint(1, n) for _ in range(AVG_DEG * n)]\n",
    "    for i in range(0, len(indices), 2):\n",
    "        u = indices[i]\n",
    "        v = indices[i + 1]\n",
    "        if u != v:\n",
    "            edge_list += [min(u,v),max(u,v)]\n",
    "            adjacencies[u].add(v)\n",
    "            adjacencies[v].add(u)\n",
    "    edge_list += [PAD_TOKEN]*(2*SEQ_LEN-1-len(edge_list))\n",
    "    return edge_list, adjacencies\n",
    "\n",
    "\"\"\"\n",
    "input: G, represented as an adjacency list\n",
    "output: [number of vertices]+[d(1,i) for i in range(n)] if target=None\n",
    "if target is set to some value, then we instead just output that specific distance\n",
    "\"\"\"\n",
    "def SSSP(n, G, target=2):\n",
    "    dist = [n for _ in G]\n",
    "    dist[1] = 0\n",
    "    frontier = deque()\n",
    "    frontier.append(1)\n",
    "    while len(frontier) > 0:\n",
    "        vtx = frontier.popleft()\n",
    "        for x in G[vtx]:\n",
    "            if dist[x] == n:\n",
    "                dist[x] = 1 + dist[vtx]\n",
    "                frontier.append(x)\n",
    "                if x == target:\n",
    "                    return dist[target]\n",
    "    if target is not None:\n",
    "        return dist[target]\n",
    "    else:\n",
    "        return dist\n",
    "\n",
    "def mkbatch(size):\n",
    "    graphs1 = []\n",
    "    distance1 = []\n",
    "    \n",
    "    for i in range(size):\n",
    "        n = random.randint(MIN_VTXS, MAX_VTXS)\n",
    "        edge_list, adj_list = random_graph(n)\n",
    "        dist = SSSP(n, adj_list)\n",
    "        edge_list[-1] = 2 # target token\n",
    "        graphs1.append(edge_list)\n",
    "        distance1.append(dist)\n",
    "    \n",
    "    data = torch.tensor(graphs1, device=device)\n",
    "    labels = torch.tensor(distance1, dtype=torch.bfloat16, device=device)\n",
    "    padding = data == PAD_TOKEN\n",
    "    return data, labels, padding\n",
    "\n",
    "def vertices_on_shortest_12_path(n, G, target=2):\n",
    "    dist = [n for _ in G]\n",
    "    parent = [-1 for _ in G]\n",
    "    dist[1] = 0\n",
    "    frontier = deque()\n",
    "    frontier.append(1)\n",
    "    while len(frontier) > 0:\n",
    "        vtx = frontier.popleft()\n",
    "        for x in G[vtx]:\n",
    "            if dist[x] == n:\n",
    "                parent[x] = vtx\n",
    "                dist[x] = 1 + dist[vtx]\n",
    "                frontier.append(x)\n",
    "                if x == target:\n",
    "                    path = [x]\n",
    "                    while parent[x] != -1:\n",
    "                        x = parent[x]\n",
    "                        path.append(x)\n",
    "                    return list(reversed(path))\n",
    "    return []\n",
    "\n",
    "def mktunebatch(size):\n",
    "    graphs = []\n",
    "    distance = []\n",
    "    \n",
    "    for i in range(size):\n",
    "        n = random.randint(MIN_VTXS, MAX_TUNE_VTXS)\n",
    "        while True:\n",
    "            edge_list, adj_list = random_graph(n)\n",
    "            path = vertices_on_shortest_12_path(n, adj_list)\n",
    "            if len(path) > 1:\n",
    "                target_vtx_idx = random.randrange(1, len(path))\n",
    "                target_vtx = path[target_vtx_idx]\n",
    "                edge_list[-1] = target_vtx\n",
    "                graphs.append(edge_list)\n",
    "                distance.append(target_vtx_idx)\n",
    "                break\n",
    "    \n",
    "    data = torch.tensor(graphs, device=device)\n",
    "    labels = torch.tensor(distance, dtype=torch.bfloat16, device=device)\n",
    "    padding = data == PAD_TOKEN\n",
    "    return data, labels, padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[13, 14,  4,  5,  1,  4,  2, 10,  2, 12,  8, 14,  2, 13,  9, 13,  8, 11,\n",
       "           3,  9,  5,  9, 10, 14,  4,  7,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  2],\n",
       "         [ 7, 10, 12, 13, 15, 17,  4, 15,  3, 11,  1, 19,  8, 18,  8, 19,  1,  3,\n",
       "           2,  8,  2,  3,  3, 11,  8, 17,  9, 16,  7, 18,  5, 19, 16, 19,  8, 16,\n",
       "           7, 14,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  2],\n",
       "         [ 1,  6,  3,  4,  4,  6,  1,  6,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  2],\n",
       "         [ 1,  4,  3,  6,  1,  2,  4,  5,  2,  4,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  2],\n",
       "         [ 5,  8,  2,  4,  2,  8,  2,  4,  3,  7,  4,  7,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  2]], device='cuda:0'),\n",
       " tensor([5., 2., 6., 1., 8.], device='cuda:0', dtype=torch.bfloat16),\n",
       " tensor([[False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False,  True,  True,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True, False],\n",
       "         [False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True, False],\n",
       "         [False, False, False, False, False, False, False, False,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True, False],\n",
       "         [False, False, False, False, False, False, False, False, False, False,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True, False],\n",
       "         [False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "           True,  True, False]], device='cuda:0'))"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mkbatch(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([320.,   0., 309.,   0., 265.,   0., 179.,   0., 118.,   0.,  89.,\n",
       "          0.,  69.,   0.,  41.,   0.,   0.,  39.,   0.,  30.,   0.,  30.,\n",
       "          0.,  31.,   0.,  26.,   0.,  28.,   0.,  27.,   0.,   0.,  20.,\n",
       "          0.,  30.,   0.,  30.,   0.,  26.,   0.,  20.,   0.,  31.,   0.,\n",
       "         34.,   0.,  23.,   0.,   0.,  35.,   0.,  29.,   0.,  17.,   0.,\n",
       "         26.,   0.,  33.,   0.,  35.,   0.,  32.,   0.,  26.]),\n",
       " array([ 1.     ,  1.46875,  1.9375 ,  2.40625,  2.875  ,  3.34375,\n",
       "         3.8125 ,  4.28125,  4.75   ,  5.21875,  5.6875 ,  6.15625,\n",
       "         6.625  ,  7.09375,  7.5625 ,  8.03125,  8.5    ,  8.96875,\n",
       "         9.4375 ,  9.90625, 10.375  , 10.84375, 11.3125 , 11.78125,\n",
       "        12.25   , 12.71875, 13.1875 , 13.65625, 14.125  , 14.59375,\n",
       "        15.0625 , 15.53125, 16.     , 16.46875, 16.9375 , 17.40625,\n",
       "        17.875  , 18.34375, 18.8125 , 19.28125, 19.75   , 20.21875,\n",
       "        20.6875 , 21.15625, 21.625  , 22.09375, 22.5625 , 23.03125,\n",
       "        23.5    , 23.96875, 24.4375 , 24.90625, 25.375  , 25.84375,\n",
       "        26.3125 , 26.78125, 27.25   , 27.71875, 28.1875 , 28.65625,\n",
       "        29.125  , 29.59375, 30.0625 , 30.53125, 31.     ]),\n",
       " <BarContainer object of 64 artists>)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAiyElEQVR4nO3df1BVdeL/8RegXH9eCBUurIioJZJiLSneqVxXWBBZV1ea0XKLWkdHF5pVypTG1Gxnca3p55DOzrZaM5LlTupIZSkmbitqUo6/ilGHFlu40OrIVQxQOJ8/vnm/exMVFL3vi8/HzJnhnvO+977P8Uw8O/deboBlWZYAAAAMEujrCQAAAPwUgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOF18PYHr0dLSoqqqKvXu3VsBAQG+ng4AAGgDy7J09uxZRUVFKTDw6tdI/DJQqqqqFB0d7etpAACA63Dy5En179//qmP8MlB69+4t/biDdrvd19MBAABt4Ha7FR0d7fk9fjV+GSiXXtax2+0ECgAAfqYtb8/gTbIAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADBOF19PwEQDF314xW3frsi4pXMBAOB2xBUUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxuG7eDoQ3+EDAEDH4AoKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADBOuwJl1apVSkhIkN1ul91ul9Pp1Mcff+zZ3tDQoOzsbPXp00e9evVSZmamampqvB6jsrJSGRkZ6tGjh8LDw7VgwQJdvHix4/YIAAD4vXYFSv/+/bVixQqVlZVp//79Gj9+vCZPnqwjR45IkubPn68tW7Zow4YNKikpUVVVlaZOneq5f3NzszIyMtTU1KTdu3fr7bff1tq1a7VkyZKO3zMAAOC3AizLsm7kAcLCwvTiiy/qoYceUr9+/VRYWKiHHnpIkvTNN99o2LBhKi0t1ZgxY/Txxx/r17/+taqqqhQRESFJWr16tRYuXKjvv/9ewcHBbXpOt9utkJAQ1dXVyW6338j0W3W9f8+Ev4MCAMCVtef393W/B6W5uVnr169XfX29nE6nysrKdOHCBaWkpHjGxMXFacCAASotLZUklZaWasSIEZ44kaS0tDS53W7PVZjWNDY2yu12ey0AAKDzanegHDp0SL169ZLNZtOcOXO0ceNGxcfHy+VyKTg4WKGhoV7jIyIi5HK5JEkul8srTi5tv7TtSvLz8xUSEuJZoqOj2zttAADgR9odKEOHDtWBAwe0d+9ezZ07V1lZWTp69OjNmd2P8vLyVFdX51lOnjx5U58PAAD4Vru/iyc4OFhDhgyRJCUmJuqLL77Qa6+9pmnTpqmpqUlnzpzxuopSU1Mjh8MhSXI4HNq3b5/X4136lM+lMa2x2Wyy2WztnSoAAPBTN/x3UFpaWtTY2KjExER17dpVxcXFnm3l5eWqrKyU0+mUJDmdTh06dEi1tbWeMdu2bZPdbld8fPyNTgUAAHQS7bqCkpeXp/T0dA0YMEBnz55VYWGhdu7cqU8++UQhISGaOXOmcnNzFRYWJrvdrieffFJOp1NjxoyRJKWmpio+Pl6PPvqoVq5cKZfLpcWLFys7O5srJAAAwKNdgVJbW6vHHntM1dXVCgkJUUJCgj755BP96le/kiS98sorCgwMVGZmphobG5WWlqY333zTc/+goCAVFRVp7ty5cjqd6tmzp7KysrR8+fKO3zMAAOC3bvjvoPgCfwcFAAD/c0v+DgoAAMDNQqAAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAME4XX08A0sBFH15x27crMm7pXAAAMAFXUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABinXYGSn5+vUaNGqXfv3goPD9eUKVNUXl7uNWbcuHEKCAjwWubMmeM1prKyUhkZGerRo4fCw8O1YMECXbx4sWP2CAAA+L12fVlgSUmJsrOzNWrUKF28eFHPPvusUlNTdfToUfXs2dMzbtasWVq+fLnndo8ePTw/Nzc3KyMjQw6HQ7t371Z1dbUee+wxde3aVX/+8587ar8AAIAfa1egbN261ev22rVrFR4errKyMo0dO9azvkePHnI4HK0+xqeffqqjR49q+/btioiI0D333KMXXnhBCxcu1LJlyxQcHHy9+wIAADqJG3oPSl1dnSQpLCzMa/26devUt29fDR8+XHl5eTp//rxnW2lpqUaMGKGIiAjPurS0NLndbh05cqTV52lsbJTb7fZaAABA59WuKyj/q6WlRfPmzdP999+v4cOHe9Y/8sgjiomJUVRUlA4ePKiFCxeqvLxcH3zwgSTJ5XJ5xYkkz22Xy9Xqc+Xn5+v555+/3qkCAAA/c92Bkp2drcOHD+vzzz/3Wj979mzPzyNGjFBkZKSSk5N14sQJDR48+LqeKy8vT7m5uZ7bbrdb0dHR1zt1AABguOt6iScnJ0dFRUX67LPP1L9//6uOTUpKkiQdP35ckuRwOFRTU+M15tLtK71vxWazyW63ey0AAKDzalegWJalnJwcbdy4UTt27FBsbOw173PgwAFJUmRkpCTJ6XTq0KFDqq2t9YzZtm2b7Ha74uPj278HAACg02nXSzzZ2dkqLCzU5s2b1bt3b897RkJCQtS9e3edOHFChYWFmjhxovr06aODBw9q/vz5Gjt2rBISEiRJqampio+P16OPPqqVK1fK5XJp8eLFys7Ols1muzl7CQAA/Eq7rqCsWrVKdXV1GjdunCIjIz3Le++9J0kKDg7W9u3blZqaqri4OD311FPKzMzUli1bPI8RFBSkoqIiBQUFyel06ne/+50ee+wxr7+bAgAAbm/tuoJiWdZVt0dHR6ukpOSajxMTE6OPPvqoPU8NAABuI3wXDwAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA47QqU/Px8jRo1Sr1791Z4eLimTJmi8vJyrzENDQ3Kzs5Wnz591KtXL2VmZqqmpsZrTGVlpTIyMtSjRw+Fh4drwYIFunjxYsfsEQAA8HvtCpSSkhJlZ2drz5492rZtmy5cuKDU1FTV19d7xsyfP19btmzRhg0bVFJSoqqqKk2dOtWzvbm5WRkZGWpqatLu3bv19ttva+3atVqyZEnH7hkAAPBbXdozeOvWrV63165dq/DwcJWVlWns2LGqq6vTW2+9pcLCQo0fP16StGbNGg0bNkx79uzRmDFj9Omnn+ro0aPavn27IiIidM899+iFF17QwoULtWzZMgUHB3fsHgIAAL9zQ+9BqaurkySFhYVJksrKynThwgWlpKR4xsTFxWnAgAEqLS2VJJWWlmrEiBGKiIjwjElLS5Pb7daRI0dafZ7Gxka53W6vBQAAdF7XHSgtLS2aN2+e7r//fg0fPlyS5HK5FBwcrNDQUK+xERERcrlcnjH/GyeXtl/a1pr8/HyFhIR4lujo6OudNgAA8APXHSjZ2dk6fPiw1q9f37EzakVeXp7q6uo8y8mTJ2/6cwIAAN9p13tQLsnJyVFRUZF27dql/v37e9Y7HA41NTXpzJkzXldRampq5HA4PGP27dvn9XiXPuVzacxP2Ww22Wy265kqAADwQ+26gmJZlnJycrRx40bt2LFDsbGxXtsTExPVtWtXFRcXe9aVl5ersrJSTqdTkuR0OnXo0CHV1tZ6xmzbtk12u13x8fE3vkcAAMDvtesKSnZ2tgoLC7V582b17t3b856RkJAQde/eXSEhIZo5c6Zyc3MVFhYmu92uJ598Uk6nU2PGjJEkpaamKj4+Xo8++qhWrlwpl8ulxYsXKzs7m6skAABAam+grFq1SpI0btw4r/Vr1qzR448/Lkl65ZVXFBgYqMzMTDU2NiotLU1vvvmmZ2xQUJCKioo0d+5cOZ1O9ezZU1lZWVq+fHnH7BEAAPB77QoUy7KuOaZbt24qKChQQUHBFcfExMToo48+as9TAwCA2wjfxQMAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAME4XX08A12/gog+vuO3bFRm3dC4AAHQkrqAAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADBOuwNl165dmjRpkqKiohQQEKBNmzZ5bX/88ccVEBDgtUyYMMFrzOnTpzVjxgzZ7XaFhoZq5syZOnfu3I3vDQAA6BTaHSj19fUaOXKkCgoKrjhmwoQJqq6u9izvvvuu1/YZM2boyJEj2rZtm4qKirRr1y7Nnj37+vYAAAB0Ol3ae4f09HSlp6dfdYzNZpPD4Wh129dff62tW7fqiy++0H333SdJeuONNzRx4kS99NJLioqKau+UAABAJ3NT3oOyc+dOhYeHa+jQoZo7d65OnTrl2VZaWqrQ0FBPnEhSSkqKAgMDtXfv3lYfr7GxUW6322sBAACdV4cHyoQJE/TOO++ouLhYf/nLX1RSUqL09HQ1NzdLklwul8LDw73u06VLF4WFhcnlcrX6mPn5+QoJCfEs0dHRHT1tAABgkHa/xHMt06dP9/w8YsQIJSQkaPDgwdq5c6eSk5Ov6zHz8vKUm5vrue12u4kUAAA6sZv+MeNBgwapb9++On78uCTJ4XCotrbWa8zFixd1+vTpK75vxWazyW63ey0AAKDzuumB8t133+nUqVOKjIyUJDmdTp05c0ZlZWWeMTt27FBLS4uSkpJu9nQAAIAfaPdLPOfOnfNcDZGkiooKHThwQGFhYQoLC9Pzzz+vzMxMORwOnThxQs8884yGDBmitLQ0SdKwYcM0YcIEzZo1S6tXr9aFCxeUk5Oj6dOn8wkeAAAgXc8VlP379+vee+/VvffeK0nKzc3VvffeqyVLligoKEgHDx7Ub37zG911112aOXOmEhMT9c9//lM2m83zGOvWrVNcXJySk5M1ceJEPfDAA/rrX//asXsGAAD8VruvoIwbN06WZV1x+yeffHLNxwgLC1NhYWF7nxoAANwm+C4eAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxuvh6Arj1Bi768Irbvl2RcUvnAgBAa7iCAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOO0OlF27dmnSpEmKiopSQECANm3a5LXdsiwtWbJEkZGR6t69u1JSUnTs2DGvMadPn9aMGTNkt9sVGhqqmTNn6ty5cze+NwAAoFNod6DU19dr5MiRKigoaHX7ypUr9frrr2v16tXau3evevbsqbS0NDU0NHjGzJgxQ0eOHNG2bdtUVFSkXbt2afbs2Te2JwAAoNPo0t47pKenKz09vdVtlmXp1Vdf1eLFizV58mRJ0jvvvKOIiAht2rRJ06dP19dff62tW7fqiy++0H333SdJeuONNzRx4kS99NJLioqKutF9AgAAfq5D34NSUVEhl8ullJQUz7qQkBAlJSWptLRUklRaWqrQ0FBPnEhSSkqKAgMDtXfv3lYft7GxUW6322sBAACdV4cGisvlkiRFRER4rY+IiPBsc7lcCg8P99repUsXhYWFecb8VH5+vkJCQjxLdHR0R04bAAAYxi8+xZOXl6e6ujrPcvLkSV9PCQAA3EQdGigOh0OSVFNT47W+pqbGs83hcKi2ttZr+8WLF3X69GnPmJ+y2Wyy2+1eCwAA6Lw6NFBiY2PlcDhUXFzsWed2u7V37145nU5JktPp1JkzZ1RWVuYZs2PHDrW0tCgpKakjpwMAAPxUuz/Fc+7cOR0/ftxzu6KiQgcOHFBYWJgGDBigefPm6U9/+pPuvPNOxcbG6rnnnlNUVJSmTJkiSRo2bJgmTJigWbNmafXq1bpw4YJycnI0ffp0PsEDAACk6wmU/fv365e//KXndm5uriQpKytLa9eu1TPPPKP6+nrNnj1bZ86c0QMPPKCtW7eqW7dunvusW7dOOTk5Sk5OVmBgoDIzM/X666931D4BAAA/1+5AGTdunCzLuuL2gIAALV++XMuXL7/imLCwMBUWFrb3qQEAwG3CLz7FAwAAbi8ECgAAMA6BAgAAjEOgAAAA4xAoAADAOO3+FA9uXwMXfXjFbd+uyLilcwEAdG5cQQEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADG6eLrCaDzG7jowytu+3ZFxi2dCwDAP3AFBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgnA4PlGXLlikgIMBriYuL82xvaGhQdna2+vTpo169eikzM1M1NTUdPQ0AAODHbsoVlLvvvlvV1dWe5fPPP/dsmz9/vrZs2aINGzaopKREVVVVmjp16s2YBgAA8FNdbsqDdukih8Nx2fq6ujq99dZbKiws1Pjx4yVJa9as0bBhw7Rnzx6NGTPmZkwHAAD4mZtyBeXYsWOKiorSoEGDNGPGDFVWVkqSysrKdOHCBaWkpHjGxsXFacCAASotLb3i4zU2NsrtdnstAACg8+rwQElKStLatWu1detWrVq1ShUVFXrwwQd19uxZuVwuBQcHKzQ01Os+ERERcrlcV3zM/Px8hYSEeJbo6OiOnjYAADBIh7/Ek56e7vk5ISFBSUlJiomJ0fvvv6/u3btf12Pm5eUpNzfXc9vtdhMpAAB0Yjf9Y8ahoaG66667dPz4cTkcDjU1NenMmTNeY2pqalp9z8olNptNdrvdawEAAJ3XTQ+Uc+fO6cSJE4qMjFRiYqK6du2q4uJiz/by8nJVVlbK6XTe7KkAAAA/0eEv8Tz99NOaNGmSYmJiVFVVpaVLlyooKEgPP/ywQkJCNHPmTOXm5iosLEx2u11PPvmknE4nn+ABAAAeHR4o3333nR5++GGdOnVK/fr10wMPPKA9e/aoX79+kqRXXnlFgYGByszMVGNjo9LS0vTmm2929DQAAIAf6/BAWb9+/VW3d+vWTQUFBSooKOjopwYAAJ0E38UDAACMQ6AAAADjECgAAMA4N+W7eICOMHDRh1fc9u2KjFs6FwDArcUVFAAAYByuoOC2wlUZAPAPXEEBAADGIVAAAIBxCBQAAGAcAgUAABiHN8kCAG4p3qzeOo6LNwIFAG4D/PKDvyFQAADoQMRgxyBQAADwY501iAgUwFCd9T86uDGcF7hdEChAG13vLwZ+odw6/BsBnQeBAgDwC4Tk7YVAASDxH38AhuEPtQEAAONwBQXADeHKC4CbgUABOhmC4dbhWHdut8O/r8n7yEs8AADAOAQKAAAwDi/xAACuyOSXANC5cQUFAAAYh0ABAADG4SUeAMbp7C8rXG3/1En2EbhRXEEBAADGIVAAAIBxCBQAAGAc3oMCAOjUOvt7mjorrqAAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADj+DRQCgoKNHDgQHXr1k1JSUnat2+fL6cDAAAM4bNAee+995Sbm6ulS5fqyy+/1MiRI5WWlqba2lpfTQkAABjCZ4Hy8ssva9asWXriiScUHx+v1atXq0ePHvr73//uqykBAABDdPHFkzY1NamsrEx5eXmedYGBgUpJSVFpaell4xsbG9XY2Oi5XVdXJ0lyu903ZX4tjeevuO1qz8n9zLjf1dzIY/rL/vvL/a7GX/bhZtzPF8/Jv+/teb8bve/1uPSYlmVde7DlA//5z38sSdbu3bu91i9YsMAaPXr0ZeOXLl1qSWJhYWFhYWHpBMvJkyev2Qo+uYLSXnl5ecrNzfXcbmlp0enTp9WnTx8FBARcNt7tdis6OlonT56U3W6/xbP1LxyrtuNYtR3Hqu04Vm3HsWofE4+XZVk6e/asoqKirjnWJ4HSt29fBQUFqaamxmt9TU2NHA7HZeNtNptsNpvXutDQ0Gs+j91uN+YfxXQcq7bjWLUdx6rtOFZtx7FqH9OOV0hISJvG+eRNssHBwUpMTFRxcbFnXUtLi4qLi+V0On0xJQAAYBCfvcSTm5urrKws3XfffRo9erReffVV1dfX64knnvDVlAAAgCF8FijTpk3T999/ryVLlsjlcumee+7R1q1bFRERccOPbbPZtHTp0steFsLlOFZtx7FqO45V23Gs2o5j1T7+frwCrDZ91gcAAODW4bt4AACAcQgUAABgHAIFAAAYh0ABAADG6XSBUlBQoIEDB6pbt25KSkrSvn37fD0l4yxbtkwBAQFeS1xcnK+nZYxdu3Zp0qRJioqKUkBAgDZt2uS13bIsLVmyRJGRkerevbtSUlJ07Ngxn83Xl651rB5//PHLzrUJEyb4bL6+kp+fr1GjRql3794KDw/XlClTVF5e7jWmoaFB2dnZ6tOnj3r16qXMzMzL/pjl7aItx2vcuHGXnVtz5szx2Zx9ZdWqVUpISPD8MTan06mPP/7Ys92fz6tOFSjvvfeecnNztXTpUn355ZcaOXKk0tLSVFtb6+upGefuu+9WdXW1Z/n88899PSVj1NfXa+TIkSooKGh1+8qVK/X6669r9erV2rt3r3r27Km0tDQ1NDTc8rn62rWOlSRNmDDB61x79913b+kcTVBSUqLs7Gzt2bNH27Zt04ULF5Samqr6+nrPmPnz52vLli3asGGDSkpKVFVVpalTp/p03r7SluMlSbNmzfI6t1auXOmzOftK//79tWLFCpWVlWn//v0aP368Jk+erCNHjkj+fl515JcA+tro0aOt7Oxsz+3m5mYrKirKys/P9+m8TLN06VJr5MiRvp6GX5Bkbdy40XO7paXFcjgc1osvvuhZd+bMGctms1nvvvuuj2Zphp8eK8uyrKysLGvy5Mk+m5OpamtrLUlWSUmJZf14DnXt2tXasGGDZ8zXX39tSbJKS0t9OFMz/PR4WZZl/eIXv7D++Mc/+nReprrjjjusv/3tb35/XnWaKyhNTU0qKytTSkqKZ11gYKBSUlJUWlrq07mZ6NixY4qKitKgQYM0Y8YMVVZW+npKfqGiokIul8vrPAsJCVFSUhLn2RXs3LlT4eHhGjp0qObOnatTp075eko+V1dXJ0kKCwuTJJWVlenChQte51VcXJwGDBjAedXK8bpk3bp16tu3r4YPH668vDydP3/eRzM0Q3Nzs9avX6/6+no5nU6/P6/84tuM2+K///2vmpubL/tLtBEREfrmm298Ni8TJSUlae3atRo6dKiqq6v1/PPP68EHH9Thw4fVu3dvX0/PaC6XS/rxvPpfERERnm34/yZMmKCpU6cqNjZWJ06c0LPPPqv09HSVlpYqKCjI19PziZaWFs2bN0/333+/hg8fLv14XgUHB1/2JaicV60fL0l65JFHFBMTo6ioKB08eFALFy5UeXm5PvjgA5/O1xcOHTokp9OphoYG9erVSxs3blR8fLwOHDjg1+dVpwkUtF16errn54SEBCUlJSkmJkbvv/++Zs6c6dO5oXOZPn265+cRI0YoISFBgwcP1s6dO5WcnOzTuflKdna2Dh8+zPu+2uhKx2v27Nmen0eMGKHIyEglJyfrxIkTGjx4sA9m6jtDhw7VgQMHVFdXp3/84x/KyspSSUmJr6d1wzrNSzx9+/ZVUFDQZe9OrqmpkcPh8Nm8/EFoaKjuuusuHT9+3NdTMd6lc4nz7PoMGjRIffv2vW3PtZycHBUVFemzzz5T//79PesdDoeampp05swZr/G3+3l1pePVmqSkJEm6Lc+t4OBgDRkyRImJicrPz9fIkSP12muv+f151WkCJTg4WImJiSouLvasa2lpUXFxsZxOp0/nZrpz587pxIkTioyM9PVUjBcbGyuHw+F1nrndbu3du5fzrA2+++47nTp16rY71yzLUk5OjjZu3KgdO3YoNjbWa3tiYqK6du3qdV6Vl5ersrLytjyvrnW8WnPgwAFJuu3Orda0tLSosbHR/88rX79LtyOtX7/estls1tq1a62jR49as2fPtkJDQy2Xy+XrqRnlqaeesnbu3GlVVFRY//rXv6yUlBSrb9++Vm1tra+nZoSzZ89aX331lfXVV19ZkqyXX37Z+uqrr6x///vflmVZ1ooVK6zQ0FBr8+bN1sGDB63JkydbsbGx1g8//ODrqd9yVztWZ8+etZ5++mmrtLTUqqiosLZv3279/Oc/t+68806roaHB11O/pebOnWuFhIRYO3futKqrqz3L+fPnPWPmzJljDRgwwNqxY4e1f/9+y+l0Wk6n06fz9pVrHa/jx49by5cvt/bv329VVFRYmzdvtgYNGmSNHTvW11O/5RYtWmSVlJRYFRUV1sGDB61FixZZAQEB1qeffmpZfn5edapAsSzLeuONN6wBAwZYwcHB1ujRo609e/b4ekrGmTZtmhUZGWkFBwdbP/vZz6xp06ZZx48f9/W0jPHZZ59Zki5bsrKyLOvHjxo/99xzVkREhGWz2azk5GSrvLzc19P2iasdq/Pnz1upqalWv379rK5du1oxMTHWrFmzbsv/YWjtGEmy1qxZ4xnzww8/WH/4wx+sO+64w+rRo4f129/+1qqurvbpvH3lWsersrLSGjt2rBUWFmbZbDZryJAh1oIFC6y6ujpfT/2W+/3vf2/FxMRYwcHBVr9+/azk5GRPnFh+fl4FWP/vZAAAADBGp3kPCgAA6DwIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMb5P/Tol2Uy4reaAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(mkbatch(2048)[1].to(torch.float32).cpu(), bins=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([1289.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,\n",
       "           0.,  477.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,\n",
       "           0.,    0.,    0.,  192.,    0.,    0.,    0.,    0.,    0.,\n",
       "           0.,    0.,    0.,    0.,    0.,   55.,    0.,    0.,    0.,\n",
       "           0.,    0.,    0.,    0.,    0.,    0.,   28.,    0.,    0.,\n",
       "           0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,    5.,\n",
       "           0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,\n",
       "           2.]),\n",
       " array([1.     , 1.09375, 1.1875 , 1.28125, 1.375  , 1.46875, 1.5625 ,\n",
       "        1.65625, 1.75   , 1.84375, 1.9375 , 2.03125, 2.125  , 2.21875,\n",
       "        2.3125 , 2.40625, 2.5    , 2.59375, 2.6875 , 2.78125, 2.875  ,\n",
       "        2.96875, 3.0625 , 3.15625, 3.25   , 3.34375, 3.4375 , 3.53125,\n",
       "        3.625  , 3.71875, 3.8125 , 3.90625, 4.     , 4.09375, 4.1875 ,\n",
       "        4.28125, 4.375  , 4.46875, 4.5625 , 4.65625, 4.75   , 4.84375,\n",
       "        4.9375 , 5.03125, 5.125  , 5.21875, 5.3125 , 5.40625, 5.5    ,\n",
       "        5.59375, 5.6875 , 5.78125, 5.875  , 5.96875, 6.0625 , 6.15625,\n",
       "        6.25   , 6.34375, 6.4375 , 6.53125, 6.625  , 6.71875, 6.8125 ,\n",
       "        6.90625, 7.     ]),\n",
       " <BarContainer object of 64 artists>)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAjSklEQVR4nO3dfVSUdf7/8dcgcpMyg1gMzobItm1K3mRSNNm9HEnJXU90Y1FRsbqnBcvsRvmW2o2F2XZHS5ieVj0nPd3sWa0sKfKOLRERl03JyDZTzAY6h5gJOiLK/P7oeP2a1EJ3aOZjz8c51znNdX1mrvdcx3N4NjMMNr/f7xcAAIBBIkI9AAAAwPEiYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYJzLUA/SUrq4u7du3T3FxcbLZbKEeBwAAdIPf79e3334rl8uliIhjv85y0gbMvn37lJycHOoxAADACWhsbNTpp59+zOMnbcDExcVJ+v4C2O32EE8DAAC6w+fzKTk52fo5fiwnbcAcftvIbrcTMAAAGObnPv7Bh3gBAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGCcyFAPYKJBM98+5rEv5mX/gpMAAPDrxCswAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxz3AFTWVmpCRMmyOVyyWazaeXKldaxzs5OzZgxQ8OGDVOfPn3kcrl0yy23aN++fQGP0dLSotzcXNntdsXHxys/P19tbW0Baz766CNdfPHFiomJUXJysubPn39izxAAAJx0jjtg2tvbNWLECJWWlh5x7LvvvtPWrVs1a9Ysbd26Vf/85z/V0NCgP/zhDwHrcnNzVV9fr4qKCq1atUqVlZWaMmWKddzn82ns2LFKSUlRbW2tnnzyST300ENauHDhCTxFAABwsrH5/X7/Cd/ZZtOKFSs0ceLEY66pqanR+eefr927d2vgwIHasWOH0tLSVFNTo/T0dElSeXm5xo8fr71798rlcqmsrEwPPPCAPB6PoqKiJEkzZ87UypUr9cknn3RrNp/PJ4fDIa/XK7vdfqJP8agGzXz7mMe+mJcd1HMBAPBr0t2f3z3+GRiv1yubzab4+HhJUlVVleLj4614kaTMzExFRESourraWnPJJZdY8SJJWVlZamho0DfffNPTIwMAgDAX2ZMPvn//fs2YMUM33HCDVVEej0eJiYmBQ0RGKiEhQR6Px1qTmpoasMbpdFrH+vXrd8S5Ojo61NHRYd32+XxBfS4AACB89NgrMJ2dnbruuuvk9/tVVlbWU6exFBcXy+FwWFtycnKPnxMAAIRGjwTM4XjZvXu3KioqAt7DSkpKUnNzc8D6gwcPqqWlRUlJSdaapqamgDWHbx9e82NFRUXyer3W1tjYGMynBAAAwkjQA+ZwvOzcuVPvv/+++vfvH3Dc7XartbVVtbW11r61a9eqq6tLGRkZ1prKykp1dnZaayoqKnTWWWcd9e0jSYqOjpbdbg/YAADAyem4A6atrU11dXWqq6uTJO3atUt1dXXas2ePOjs7dc0112jLli1atmyZDh06JI/HI4/HowMHDkiShgwZoiuvvFKTJ0/W5s2b9eGHH6qwsFCTJk2Sy+WSJN14442KiopSfn6+6uvr9eqrr+q5557T9OnTg/fMAQCAsY7716jXr1+vyy+//Ij9eXl5euihh4748O1h69at02WXXSbp+y+yKyws1FtvvaWIiAjl5OSopKREffv2tdZ/9NFHKigoUE1NjU499VRNnTpVM2bM6Pac/Bo1AADm6e7P7//pe2DCGQEDAIB5wuZ7YAAAAIKNgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgnOMOmMrKSk2YMEEul0s2m00rV64MOO73+zV79mwNGDBAsbGxyszM1M6dOwPWtLS0KDc3V3a7XfHx8crPz1dbW1vAmo8++kgXX3yxYmJilJycrPnz5x//swMAACel4w6Y9vZ2jRgxQqWlpUc9Pn/+fJWUlGjBggWqrq5Wnz59lJWVpf3791trcnNzVV9fr4qKCq1atUqVlZWaMmWKddzn82ns2LFKSUlRbW2tnnzyST300ENauHDhCTxFAABwsrH5/X7/Cd/ZZtOKFSs0ceJESd+/+uJyuXTPPffo3nvvlSR5vV45nU4tWbJEkyZN0o4dO5SWlqaamhqlp6dLksrLyzV+/Hjt3btXLpdLZWVleuCBB+TxeBQVFSVJmjlzplauXKlPPvmkW7P5fD45HA55vV7Z7fYTfYpHNWjm28c89sW87KCeCwCAX5Pu/vwO6mdgdu3aJY/Ho8zMTGufw+FQRkaGqqqqJElVVVWKj4+34kWSMjMzFRERoerqamvNJZdcYsWLJGVlZamhoUHffPPNUc/d0dEhn88XsAEAgJNTUAPG4/FIkpxOZ8B+p9NpHfN4PEpMTAw4HhkZqYSEhIA1R3uMH57jx4qLi+VwOKwtOTn5f39CAAAgLJ00v4VUVFQkr9drbY2NjaEeCQAA9JCgBkxSUpIkqampKWB/U1OTdSwpKUnNzc0Bxw8ePKiWlpaANUd7jB+e48eio6Nlt9sDNgAAcHIKasCkpqYqKSlJa9assfb5fD5VV1fL7XZLktxut1pbW1VbW2utWbt2rbq6upSRkWGtqaysVGdnp7WmoqJCZ511lvr16xfMkQEAgIGOO2Da2tpUV1enuro6Sd9/cLeurk579uyRzWbTtGnTNHfuXL355pvatm2bbrnlFrlcLus3lYYMGaIrr7xSkydP1ubNm/Xhhx+qsLBQkyZNksvlkiTdeOONioqKUn5+vurr6/Xqq6/queee0/Tp04P2xAEAgLkij/cOW7Zs0eWXX27dPhwVeXl5WrJkie6//361t7drypQpam1t1UUXXaTy8nLFxMRY91m2bJkKCws1ZswYRUREKCcnRyUlJdZxh8Oh9957TwUFBRo1apROPfVUzZ49O+C7YgAAwK/X//Q9MOGM74EBAMA8IfkeGAAAgF8CAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOEEPmEOHDmnWrFlKTU1VbGyszjjjDD366KPy+/3WGr/fr9mzZ2vAgAGKjY1VZmamdu7cGfA4LS0tys3Nld1uV3x8vPLz89XW1hbscQEAgIGCHjBPPPGEysrK9Le//U07duzQE088ofnz5+v555+31syfP18lJSVasGCBqqur1adPH2VlZWn//v3WmtzcXNXX16uiokKrVq1SZWWlpkyZEuxxAQCAgWz+H740EgRXXXWVnE6nXnrpJWtfTk6OYmNj9fLLL8vv98vlcumee+7RvffeK0nyer1yOp1asmSJJk2apB07digtLU01NTVKT0+XJJWXl2v8+PHau3evXC7Xz87h8/nkcDjk9Xplt9uD+RQ1aObbxzz2xbzsoJ4LAIBfk+7+/A76KzAXXnih1qxZo08//VSS9J///EcffPCBxo0bJ0natWuXPB6PMjMzrfs4HA5lZGSoqqpKklRVVaX4+HgrXiQpMzNTERERqq6uPup5Ozo65PP5AjYAAHByigz2A86cOVM+n0+DBw9Wr169dOjQIT322GPKzc2VJHk8HkmS0+kMuJ/T6bSOeTweJSYmBg4aGamEhARrzY8VFxfr4YcfDvbTAQAAYSjor8C89tprWrZsmZYvX66tW7dq6dKl+utf/6qlS5cG+1QBioqK5PV6ra2xsbFHzwcAAEIn6K/A3HfffZo5c6YmTZokSRo2bJh2796t4uJi5eXlKSkpSZLU1NSkAQMGWPdramrSOeecI0lKSkpSc3NzwOMePHhQLS0t1v1/LDo6WtHR0cF+OgAAIAwF/RWY7777ThERgQ/bq1cvdXV1SZJSU1OVlJSkNWvWWMd9Pp+qq6vldrslSW63W62traqtrbXWrF27Vl1dXcrIyAj2yAAAwDBBfwVmwoQJeuyxxzRw4ECdffbZ+ve//62nn35at99+uyTJZrNp2rRpmjt3rs4880ylpqZq1qxZcrlcmjhxoiRpyJAhuvLKKzV58mQtWLBAnZ2dKiws1KRJk7r1G0gAAODkFvSAef755zVr1iz95S9/UXNzs1wul/785z9r9uzZ1pr7779f7e3tmjJlilpbW3XRRRepvLxcMTEx1pply5apsLBQY8aMUUREhHJyclRSUhLscQEAgIGC/j0w4YLvgQEAwDwh+x4YAACAnkbAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4PRIwX375pW666Sb1799fsbGxGjZsmLZs2WId9/v9mj17tgYMGKDY2FhlZmZq586dAY/R0tKi3Nxc2e12xcfHKz8/X21tbT0xLgAAMEzQA+abb77R6NGj1bt3b61evVoff/yxnnrqKfXr189aM3/+fJWUlGjBggWqrq5Wnz59lJWVpf3791trcnNzVV9fr4qKCq1atUqVlZWaMmVKsMcFAAAGsvn9fn8wH3DmzJn68MMP9a9//euox/1+v1wul+655x7de++9kiSv1yun06klS5Zo0qRJ2rFjh9LS0lRTU6P09HRJUnl5ucaPH6+9e/fK5XL97Bw+n08Oh0Ner1d2uz14T1DSoJlvH/PYF/Oyg3ouAAB+Tbr78zvor8C8+eabSk9P17XXXqvExESNHDlSixYtso7v2rVLHo9HmZmZ1j6Hw6GMjAxVVVVJkqqqqhQfH2/FiyRlZmYqIiJC1dXVRz1vR0eHfD5fwAYAAE5OQQ+Yzz//XGVlZTrzzDP17rvv6o477tCdd96ppUuXSpI8Ho8kyel0BtzP6XRaxzwejxITEwOOR0ZGKiEhwVrzY8XFxXI4HNaWnJwc7KcGAADCRNADpqurS+eee64ef/xxjRw5UlOmTNHkyZO1YMGCYJ8qQFFRkbxer7U1Njb26PkAAEDoBD1gBgwYoLS0tIB9Q4YM0Z49eyRJSUlJkqSmpqaANU1NTdaxpKQkNTc3Bxw/ePCgWlparDU/Fh0dLbvdHrABAICTU9ADZvTo0WpoaAjY9+mnnyolJUWSlJqaqqSkJK1Zs8Y67vP5VF1dLbfbLUlyu91qbW1VbW2ttWbt2rXq6upSRkZGsEcGAACGiQz2A95999268MIL9fjjj+u6667T5s2btXDhQi1cuFCSZLPZNG3aNM2dO1dnnnmmUlNTNWvWLLlcLk2cOFHS96/YXHnlldZbT52dnSosLNSkSZO69RtIAADg5Bb0gDnvvPO0YsUKFRUV6ZFHHlFqaqqeffZZ5ebmWmvuv/9+tbe3a8qUKWptbdVFF12k8vJyxcTEWGuWLVumwsJCjRkzRhEREcrJyVFJSUmwxwUAAAYK+vfAhAu+BwYAAPOE7HtgAAAAehoBAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA40SGegCcXAbNfPuYx76Yl/0LTgIAOJnxCgwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDg9HjDz5s2TzWbTtGnTrH379+9XQUGB+vfvr759+yonJ0dNTU0B99uzZ4+ys7N1yimnKDExUffdd58OHjzY0+MCAAAD9GjA1NTU6MUXX9Tw4cMD9t99991666239Prrr2vDhg3at2+frr76auv4oUOHlJ2drQMHDmjjxo1aunSplixZotmzZ/fkuAAAwBA9FjBtbW3Kzc3VokWL1K9fP2u/1+vVSy+9pKefflpXXHGFRo0apcWLF2vjxo3atGmTJOm9997Txx9/rJdfflnnnHOOxo0bp0cffVSlpaU6cOBAT40MAAAM0WMBU1BQoOzsbGVmZgbsr62tVWdnZ8D+wYMHa+DAgaqqqpIkVVVVadiwYXI6ndaarKws+Xw+1dfXH/V8HR0d8vl8ARsAADg5RfbEg77yyivaunWrampqjjjm8XgUFRWl+Pj4gP1Op1Mej8da88N4OXz88LGjKS4u1sMPPxyE6QEAQLgL+iswjY2Nuuuuu7Rs2TLFxMQE++GPqaioSF6v19oaGxt/sXMDAIBfVtADpra2Vs3NzTr33HMVGRmpyMhIbdiwQSUlJYqMjJTT6dSBAwfU2toacL+mpiYlJSVJkpKSko74raTDtw+v+bHo6GjZ7faADQAAnJyCHjBjxozRtm3bVFdXZ23p6enKzc21/rt3795as2aNdZ+Ghgbt2bNHbrdbkuR2u7Vt2zY1NzdbayoqKmS325WWlhbskQEAgGGC/hmYuLg4DR06NGBfnz591L9/f2t/fn6+pk+froSEBNntdk2dOlVut1sXXHCBJGns2LFKS0vTzTffrPnz58vj8ejBBx9UQUGBoqOjgz0yAAAwTI98iPfnPPPMM4qIiFBOTo46OjqUlZWlF154wTreq1cvrVq1SnfccYfcbrf69OmjvLw8PfLII6EYFwAAhJlfJGDWr18fcDsmJkalpaUqLS095n1SUlL0zjvv9PBkAADARPwtJAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGCXrAFBcX67zzzlNcXJwSExM1ceJENTQ0BKzZv3+/CgoK1L9/f/Xt21c5OTlqamoKWLNnzx5lZ2frlFNOUWJiou677z4dPHgw2OMCAAADBT1gNmzYoIKCAm3atEkVFRXq7OzU2LFj1d7ebq25++679dZbb+n111/Xhg0btG/fPl199dXW8UOHDik7O1sHDhzQxo0btXTpUi1ZskSzZ88O9rgAAMBANr/f7+/JE3z99ddKTEzUhg0bdMkll8jr9eq0007T8uXLdc0110iSPvnkEw0ZMkRVVVW64IILtHr1al111VXat2+fnE6nJGnBggWaMWOGvv76a0VFRf3seX0+nxwOh7xer+x2e1Cf06CZbx/z2BfzsoN6LtNwbQAA/4vu/vzu8c/AeL1eSVJCQoIkqba2Vp2dncrMzLTWDB48WAMHDlRVVZUkqaqqSsOGDbPiRZKysrLk8/lUX1/f0yMDAIAwF9mTD97V1aVp06Zp9OjRGjp0qCTJ4/EoKipK8fHxAWudTqc8Ho+15ofxcvj44WNH09HRoY6ODuu2z+cL1tMAAABhpkdfgSkoKND27dv1yiuv9ORpJH3/4WGHw2FtycnJPX5OAAAQGj0WMIWFhVq1apXWrVun008/3dqflJSkAwcOqLW1NWB9U1OTkpKSrDU//q2kw7cPr/mxoqIieb1ea2tsbAziswEAAOEk6G8h+f1+TZ06VStWrND69euVmpoacHzUqFHq3bu31qxZo5ycHElSQ0OD9uzZI7fbLUlyu9167LHH1NzcrMTERElSRUWF7Ha70tLSjnre6OhoRUdHB/vpAD2ODz4DwPELesAUFBRo+fLleuONNxQXF2d9ZsXhcCg2NlYOh0P5+fmaPn26EhISZLfbNXXqVLndbl1wwQWSpLFjxyotLU0333yz5s+fL4/HowcffFAFBQVECgAACH7AlJWVSZIuu+yygP2LFy/WrbfeKkl65plnFBERoZycHHV0dCgrK0svvPCCtbZXr15atWqV7rjjDrndbvXp00d5eXl65JFHgj0uAAAwUI+8hfRzYmJiVFpaqtLS0mOuSUlJ0TvvvBPM0QAAwEmCv4UEAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjBMZ6gEA4EQMmvn2MY99MS/7F5wEQCjwCgwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4/A9MABwEuH7cfBrwSswAADAOAQMAAAwTlgHTGlpqQYNGqSYmBhlZGRo8+bNoR4JAACEgbANmFdffVXTp0/XnDlztHXrVo0YMUJZWVlqbm4O9WgAACDEwjZgnn76aU2ePFm33Xab0tLStGDBAp1yyin6+9//HurRAABAiIXlbyEdOHBAtbW1KioqsvZFREQoMzNTVVVVR71PR0eHOjo6rNter1eS5PP5gj5fV8d3xzzWE+czCdfm+HHNTgzX7ei4Lidm6Jx3j3ls+8NZv+AkOPzv1O/3//RCfxj68ssv/ZL8GzduDNh/3333+c8///yj3mfOnDl+SWxsbGxsbGwnwdbY2PiTrRCWr8CciKKiIk2fPt263dXVpZaWFvXv3182my1o5/H5fEpOTlZjY6PsdnvQHvdkxfXqPq5V93Gtuo9r1X1cq+7ryWvl9/v17bffyuVy/eS6sAyYU089Vb169VJTU1PA/qamJiUlJR31PtHR0YqOjg7YFx8f31Mjym638w/8OHC9uo9r1X1cq+7jWnUf16r7eupaORyOn10Tlh/ijYqK0qhRo7RmzRprX1dXl9asWSO32x3CyQAAQDgIy1dgJGn69OnKy8tTenq6zj//fD377LNqb2/XbbfdFurRAABAiIVtwFx//fX6+uuvNXv2bHk8Hp1zzjkqLy+X0+kM6VzR0dGaM2fOEW9X4ei4Xt3Hteo+rlX3ca26j2vVfeFwrWx+/8/9nhIAAEB4CcvPwAAAAPwUAgYAABiHgAEAAMYhYAAAgHEImONQWVmpCRMmyOVyyWazaeXKlaEeKSwVFxfrvPPOU1xcnBITEzVx4kQ1NDSEeqywVFZWpuHDh1tfBuV2u7V69epQj2WEefPmyWazadq0aaEeJSw99NBDstlsAdvgwYNDPVbY+vLLL3XTTTepf//+io2N1bBhw7Rly5ZQjxV2Bg0adMS/K5vNpoKCgl98FgLmOLS3t2vEiBEqLS0N9ShhbcOGDSooKNCmTZtUUVGhzs5OjR07Vu3t7aEeLeycfvrpmjdvnmpra7VlyxZdccUV+uMf/6j6+vpQjxbWampq9OKLL2r48OGhHiWsnX322frqq6+s7YMPPgj1SGHpm2++0ejRo9W7d2+tXr1aH3/8sZ566in169cv1KOFnZqamoB/UxUVFZKka6+99hefJWy/ByYcjRs3TuPGjQv1GGGvvLw84PaSJUuUmJio2tpaXXLJJSGaKjxNmDAh4PZjjz2msrIybdq0SWeffXaIpgpvbW1tys3N1aJFizR37txQjxPWIiMjj/nnV/D/PfHEE0pOTtbixYutfampqSGcKHyddtppAbfnzZunM844Q5deeukvPguvwKDHeb1eSVJCQkKIJwlvhw4d0iuvvKL29nb+ZMZPKCgoUHZ2tjIzM0M9StjbuXOnXC6Xfvvb3yo3N1d79uwJ9Uhh6c0331R6erquvfZaJSYmauTIkVq0aFGoxwp7Bw4c0Msvv6zbb789qH80ubt4BQY9qqurS9OmTdPo0aM1dOjQUI8TlrZt2ya32639+/erb9++WrFihdLS0kI9Vlh65ZVXtHXrVtXU1IR6lLCXkZGhJUuW6KyzztJXX32lhx9+WBdffLG2b9+uuLi4UI8XVj7//HOVlZVp+vTp+r//+z/V1NTozjvvVFRUlPLy8kI9XthauXKlWltbdeutt4bk/AQMelRBQYG2b9/Oe+8/4ayzzlJdXZ28Xq/+8Y9/KC8vTxs2bCBifqSxsVF33XWXKioqFBMTE+pxwt4P3+4ePny4MjIylJKSotdee035+fkhnCz8dHV1KT09XY8//rgkaeTIkdq+fbsWLFhAwPyEl156SePGjZPL5QrJ+XkLCT2msLBQq1at0rp163T66aeHepywFRUVpd/97ncaNWqUiouLNWLECD333HOhHivs1NbWqrm5Weeee64iIyMVGRmpDRs2qKSkRJGRkTp06FCoRwxr8fHx+v3vf6/PPvss1KOEnQEDBhzxPwxDhgzhLbefsHv3br3//vv605/+FLIZeAUGQef3+zV16lStWLFC69ev58Nwx6mrq0sdHR2hHiPsjBkzRtu2bQvYd9ttt2nw4MGaMWOGevXqFaLJzNDW1qb//ve/uvnmm0M9StgZPXr0EV/18OmnnyolJSVEE4W/xYsXKzExUdnZ2SGbgYA5Dm1tbQH/97Jr1y7V1dUpISFBAwcODOFk4aWgoEDLly/XG2+8obi4OHk8HkmSw+FQbGxsiKcLL0VFRRo3bpwGDhyob7/9VsuXL9f69ev17rvvhnq0sBMXF3fE56j69Omj/v378/mqo7j33ns1YcIEpaSkaN++fZozZ4569eqlG264IdSjhZ27775bF154oR5//HFdd9112rx5sxYuXKiFCxeGerSw1NXVpcWLFysvL0+RkSHMCD+6bd26dX5JR2x5eXmhHi2sHO0aSfIvXrw41KOFndtvv92fkpLij4qK8p922mn+MWPG+N97771Qj2WMSy+91H/XXXeFeoywdP311/sHDBjgj4qK8v/mN7/xX3/99f7PPvss1GOFrbfeess/dOhQf3R0tH/w4MH+hQsXhnqksPXuu+/6JfkbGhpCOofN7/f7Q5NOAAAAJ4YP8QIAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIzz/wBRiRVxrEYfAQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(mktunebatch(2048)[1].to(torch.float32).cpu(), bins=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pickle\n",
    "\n",
    "# graphs, labels, _ = mkbatch(3*10**5)\n",
    "\n",
    "# data = {\n",
    "#     \"data\": graphs,\n",
    "#     \"labels\": labels\n",
    "# }\n",
    "\n",
    "# with open('data.pkl', 'wb') as file:\n",
    "#     pickle.dump(data, file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Q3Cg_8UQep8g"
   },
   "source": [
    "# Step 2: Define Transformer Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "execution_state": "idle",
   "metadata": {
    "id": "tLOWhg_CeWzH"
   },
   "outputs": [],
   "source": [
    "class TransformerModel(nn.Module):\n",
    "    def __init__(self, input_dim, model_dim, output_dim, num_heads, num_layers, seq_len, dropout):\n",
    "        super().__init__()\n",
    "        self.model_dim = model_dim\n",
    "        self.embedding = nn.Embedding(input_dim, model_dim // 2, dtype=torch.bfloat16)\n",
    "        # # seq_len is odd\n",
    "        # self.fancy_encoding = torch.repeat_interleave(torch.rand((1, seq_len // 2 + 1, model_dim // 2), device=device, dtype=torch.bfloat16), 2, dim=1)\n",
    "        # # cut off last element since the target vertex is not repeated\n",
    "        # self.fancy_encoding = self.fancy_encoding[:, :seq_len, :]\n",
    "        encoder_layer = nn.TransformerEncoderLayer(d_model=model_dim, nhead=num_heads,\n",
    "                                                   dim_feedforward=model_dim*4,\n",
    "                                                   dropout=dropout, batch_first=True, dtype=torch.bfloat16)\n",
    "        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers)\n",
    "        self.fc_out = nn.Linear(model_dim, output_dim, dtype=torch.bfloat16)\n",
    "\n",
    "    def forward(self, src, key_padding_mask):\n",
    "        batch_sz = src.size(0)\n",
    "        embed = torch.cat((self.embedding(src[:,:-1:2]), self.embedding(src[:,1::2])), dim=2)\n",
    "        last_dude = torch.cat((self.embedding(src[:,-1:]), torch.ones((batch_sz, 1, self.model_dim // 2), dtype=torch.bfloat16, device=device)), dim=2)\n",
    "        final_embed = torch.cat((embed, last_dude), dim=1)\n",
    "        output = self.transformer_encoder(final_embed, src_key_padding_mask=key_padding_mask[:, ::2])\n",
    "        return self.fc_out(output[:, -1, :])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bpIeg86S-hBb"
   },
   "source": [
    "# Step 3: Make Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "execution_state": "idle",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kWXvJRDYgFVP",
    "outputId": "c13adb9d-6565-43b5-8437-20cef3dc0d16"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data: 524M\n",
      "Trainable parameters in the model: 800K\n"
     ]
    }
   ],
   "source": [
    "# PARAMS\n",
    "VOCAB_SIZE = 1 + MAX_VTXS # one more than the max number of vertices\n",
    "MODEL_DIM = 64 # Dimension of model (embedding and transformer)\n",
    "NEPOCHS = 1000\n",
    "BSZ = 2**15 # Batch size\n",
    "BPE = 16 # Batches per epoch\n",
    "NHEADS = 4\n",
    "NLAYERS = 16\n",
    "DROPOUT = 0 # 0.2\n",
    "model = TransformerModel(input_dim=VOCAB_SIZE, model_dim=MODEL_DIM,\n",
    "                         output_dim=1, num_heads=NHEADS,\n",
    "                         num_layers=NLAYERS, seq_len=SEQ_LEN,\n",
    "                         dropout=DROPOUT).to(device)\n",
    "# model = torch.compile(model)\n",
    "\n",
    "trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "print(f\"Training data: {NEPOCHS*BSZ*BPE//10**6}M\")\n",
    "print(f\"Trainable parameters in the model: {trainable_params//1000}K\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "LR = 1e-4\n",
    "WD = 0 # 1e-5\n",
    "\n",
    "criterion = nn.MSELoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=LR, weight_decay=WD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from torch.utils.data import DataLoader, TensorDataset\n",
    "\n",
    "# with open(\"data.pkl\", \"rb\") as f:\n",
    "#     pickled_stuff = pickle.load(f)\n",
    "\n",
    "# data = pickled_stuff[\"data\"].to(device)\n",
    "# label = pickled_stuff[\"labels\"].to(device)\n",
    "# padding_mask = (data == PAD_TOKEN).bool().to(device)\n",
    "# dataset = TensorDataset(data, label, padding_mask)\n",
    "# # train_dataset, test_dataset = torch.utils.data.random_split(dataset, [.9, .1])\n",
    "# train_loader = DataLoader(dataset, batch_size=BSZ, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "f8Zn33m7CxL5"
   },
   "source": [
    "# Step 4: Train the Model for the first task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate():\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    with torch.no_grad():\n",
    "        batch_src, batch_labels, batch_padding_mask = mkbatch(BSZ)\n",
    "        output = model(batch_src, batch_padding_mask)\n",
    "        loss = criterion(output.squeeze(1), batch_labels)\n",
    "    return loss.item()\n",
    "\n",
    "# figure out if it's doing better on short paths\n",
    "def evaluate_short():\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    with torch.no_grad():\n",
    "        batch_src, batch_labels, batch_padding_mask = mkbatch(BSZ)\n",
    "        output = model(batch_src, batch_padding_mask)\n",
    "        loss = criterion(output[batch_labels == 1].squeeze(1), batch_labels[batch_labels==1])\n",
    "    return loss.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_err = []\n",
    "test_err = []\n",
    "\n",
    "# clear loss file\n",
    "open('loss', 'w').close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "execution_state": "idle",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 486
    },
    "id": "pvTfzGmCeXU4",
    "outputId": "0d3a20f3-23be-4c19-9eb6-46bfe11a48b1"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:10<00:00,  1.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1000 \t Train Err: 0.1735 \t Test Err: 0.1543, Test short loss: 0.0016\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:10<00:00,  1.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2/1000 \t Train Err: 0.1473 \t Test Err: 0.1348, Test short loss: 0.0017\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:10<00:00,  1.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3/1000 \t Train Err: 0.1230 \t Test Err: 0.1133, Test short loss: 0.0016\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:10<00:00,  1.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4/1000 \t Train Err: 0.1041 \t Test Err: 0.0952, Test short loss: 0.0016\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:10<00:00,  1.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5/1000 \t Train Err: 0.0896 \t Test Err: 0.0820, Test short loss: 0.0016\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:10<00:00,  1.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6/1000 \t Train Err: 0.0788 \t Test Err: 0.0752, Test short loss: 0.0016\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:10<00:00,  1.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7/1000 \t Train Err: 0.0718 \t Test Err: 0.0684, Test short loss: 0.0014\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:10<00:00,  1.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8/1000 \t Train Err: 0.0639 \t Test Err: 0.0601, Test short loss: 0.0013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:10<00:00,  1.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9/1000 \t Train Err: 0.0585 \t Test Err: 0.0562, Test short loss: 0.0011\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:10<00:00,  1.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10/1000 \t Train Err: 0.0535 \t Test Err: 0.0520, Test short loss: 0.0011\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:10<00:00,  1.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11/1000 \t Train Err: 0.0492 \t Test Err: 0.0481, Test short loss: 0.0010\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|█████▌                                      | 2/16 [00:01<00:12,  1.12it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[22], line 10\u001b[0m\n\u001b[1;32m      8\u001b[0m output \u001b[38;5;241m=\u001b[39m model(batch_src, batch_padding_mask)\n\u001b[1;32m      9\u001b[0m loss \u001b[38;5;241m=\u001b[39m criterion(output\u001b[38;5;241m.\u001b[39msqueeze(\u001b[38;5;241m1\u001b[39m), batch_labels)\n\u001b[0;32m---> 10\u001b[0m train_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[43mloss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mitem\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;241m/\u001b[39m BPE\n\u001b[1;32m     11\u001b[0m loss\u001b[38;5;241m.\u001b[39mbackward()\n\u001b[1;32m     12\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for epoch in range(NEPOCHS):\n",
    "    model.train()\n",
    "    train_loss = 0\n",
    "    for i in tqdm(range(BPE)):\n",
    "        batch_src, batch_labels, batch_padding_mask = mkbatch(BSZ)\n",
    "    # for batch_src, batch_labels, batch_padding_mask in tqdm(train_loader):\n",
    "        optimizer.zero_grad()\n",
    "        output = model(batch_src, batch_padding_mask)\n",
    "        loss = criterion(output.squeeze(1), batch_labels)\n",
    "        train_loss += loss.item() / BPE\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "    test_loss = evaluate()\n",
    "    test_short_loss = evaluate_short()\n",
    "    \n",
    "    test_err.append(test_loss)\n",
    "    train_err.append(train_loss)\n",
    "    with open('loss', 'a') as f:\n",
    "        f.write(f\"{train_loss} {test_loss}\\n\")\n",
    "    print(f\"Epoch {epoch + 1}/{NEPOCHS} \\t Train Err: {train_loss:.4f} \\t Test Err: {test_loss:.4f}, Test short loss: {test_short_loss:.4f}\")\n",
    "    \n",
    "    if epoch % 100 == 99:\n",
    "        torch.save(model.state_dict(), f\"model_weights_{epoch}.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \"\"\"\n",
    "# Now let's figure out what it's doing. \n",
    "\n",
    "# step 1: figure out what people are attending to \n",
    "# \"\"\"\n",
    "\n",
    "# example_graph, answer, padding = mkbatch(1)\n",
    "# sentance_embeddings = model.full_embedding(example_graph)[0,:,:][example_graph.flatten() != 0]\n",
    "# WQ,WK,WV = torch.split(model.transformer_encoder.layers[0].self_attn.in_proj_weight, (MODEL_DIM, MODEL_DIM, MODEL_DIM))\n",
    "\n",
    "# Q = sentance_embeddings@WQ\n",
    "# K = sentance_embeddings@WK\n",
    "\n",
    "# raw_scores = Q @ K.T / sqrt(MODEL_DIM)\n",
    "# soft = torch.softmax(raw_scores, dim=-1).detach().cpu().to(float).numpy()\n",
    "# plt.imshow(soft)\n",
    "# plt.show()\n",
    "\n",
    "# print(example_graph)\n",
    "\n",
    "# print(Q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAioAAAHgCAYAAACLq0b8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABfD0lEQVR4nO3deZyN5f/H8deZfcyYYTAbY18qe/a025UiO2WskbV8VVS24mcr2SWFZI2iKBUqRckWqaTsyh5mrDPM3L8/LmaMWRjGuc/MvJ+Px3m4zznXzHwO+Xp/r/u6PpfDsiwLERERERfkZncBIiIiIqlRUBERERGXpaAiIiIiLktBRURERFyWgoqIiIi4LAUVERERcVkKKiIiIuKyFFRERETEZSmoiIiIiMtSUBERuUkOh4OePXvaXYZItqKgIuIiZs2ahcPhwOFwsHbt2mTvW5ZFREQEDoeDxx9/PMl7Z8+eZfDgwZQpUwY/Pz/y5MlDhQoV6NOnD4cOHUoYN2TIkISfkdLjyJEjTvmsqUmrtm7dutlam4jYw8PuAkQkKR8fH+bNm8f999+f5PU1a9bwzz//4O3tneT1S5cu8eCDD/Lnn38SGRlJr169OHv2LL///jvz5s2jSZMmhIeHJ/maqVOn4u/vn+xn58qV6w59qptXp04d2rVrl+z1kiVL2lKPiNhLQUXExTRs2JBFixYxYcIEPDwS/4rOmzePSpUqceLEiSTjly5dyi+//MLcuXNp06ZNkvcuXrxIbGxssp/RrFkz8ubNewc/xa0rWbIkTz/9tN1liIiL0K0fERfTunVr/vvvP1auXJnwWmxsLIsXL04WRAB2794NQM2aNZO95+PjQ0BAQIbUVaZMGR555JFkr8fHx5M/f36aNWuW8NqCBQuoVKkSOXPmJCAggLJlyzJ+/PgMqQPg4YcfpkyZMmzevJn77rsPX19fihQpwjvvvJNs7LFjx+jUqRMhISH4+PhQvnx5PvjggxQ/x/jx4ylbtiw+Pj7ky5eP+vXrs2nTpmRjly5dSpkyZfD29qZ06dJ8+eWXSd4/c+YMzz//PIULF8bb25vg4GDq1KnDli1bMuz3QCS7UFARcTGFCxemRo0azJ8/P+G1FStWEBUVRatWrZKNL1SoEACzZ8/Gsqyb+hknT57kxIkTSR6nT59O82tatmzJ999/n2wdy9q1azl06FBCbStXrqR169bkzp2bUaNGMXLkSB5++GHWrVt3U7VdvHgxWW0nTpxINjN06tQpGjZsSKVKlRg9ejQFChTgueeeY8aMGQljLly4wMMPP8yHH35I27ZtGTNmDIGBgbRv3z5ZcOrUqRPPP/88ERERjBo1iv79++Pj48P69euTfd7u3bvTqlUrRo8ezcWLF2natCn//fdfwphu3boxdepUmjZtypQpU+jXrx++vr7s2LHjpn4PROQaloi4hJkzZ1qAtXHjRmvSpElWzpw5rfPnz1uWZVnNmze3HnnkEcuyLKtQoULWY489lvB158+ft0qVKmUBVqFChaz27dtb77//vnX06NFkP2Pw4MEWkOKjVKlSada3c+dOC7AmTpyY5PXu3btb/v7+CbX26dPHCggIsC5fvpzu34PUagOs+fPnJ4x76KGHLMB66623El6LiYmxKlSoYAUHB1uxsbGWZVnWuHHjLMCaM2dOwrjY2FirRo0alr+/vxUdHW1ZlmV98803FmD17t07WU3x8fFJ6vPy8rJ27dqV8Nq2bduS/b4EBgZaPXr0SPfnF5HkNKMi4oJatGjBhQsXWL58OWfOnGH58uUp3vYB8PX15eeff+bFF1+EK7uHOnXqRFhYGL169SImJibZ13z88cesXLkyyWPmzJlp1lSyZEkqVKjAwoULE16Li4tj8eLFNGrUCF9fX7iyIPfcuXNJbl2lx5NPPpmstpUrVya77eTh4UHXrl0Tnnt5edG1a1eOHTvG5s2bAfjiiy8IDQ2ldevWCeM8PT3p3bs3Z8+eZc2aNQm/Hw6Hg8GDByerx+FwJHleu3ZtihUrlvC8XLlyBAQEsGfPnoTXcuXKxc8//5xkx5WI3BotphVxQfny5aN27drMmzeP8+fPExcXl2QNyPUCAwMZPXo0o0ePZv/+/axevZo333yTSZMmERgYyLBhw5KMf/DBB29pMW3Lli155ZVX+Pfff8mfPz/fffcdx44do2XLlgljunfvzkcffUSDBg3Inz8/devWpUWLFtSvX/+mfkaBAgWoXbv2DceFh4fj5+eX5LWrO4P27dtH9erV2b9/PyVKlMDNLen/J7v77rsB2L9/P1xZ5xMeHk5QUNANf27BggWTvZY7d25OnTqV8Hz06NFERkYSERFBpUqVaNiwIe3ataNo0aI3/P4ikpRmVERcVJs2bVixYgXvvPMODRo0uOmtw4UKFaJjx46sW7eOXLlyMXfu3AyrqWXLlliWxaJFiwD46KOPCAwMTBJCgoOD2bp1K5999hlPPPEE3377LQ0aNCAyMjLD6rCTu7t7iq9fuz6oRYsW7Nmzh4kTJxIeHs6YMWMoXbo0K1ascGKlIlmDgoqIi2rSpAlubm6sX78+1ds+acmdOzfFihXj8OHDGVZTkSJFqFq1KgsXLuTy5ct88sknNG7cOFlvFy8vLxo1asSUKVPYvXs3Xbt2Zfbs2ezatSvDajl06BDnzp1L8tpff/0FVxYkcyW0/f3338THxycZ9+effya8D1CsWDEOHTrEyZMnM6y+sLAwunfvztKlS9m7dy958uRh+PDhGfb9RbILBRURF+Xv78/UqVMZMmQIjRo1SnXctm3bkvVW4cptjT/++INSpUplaF0tW7Zk/fr1zJgxgxMnTiS57QMk2f0C4ObmRrly5QBSXC9zqy5fvsy0adMSnsfGxjJt2jTy5ctHpUqV4EpPmiNHjiRZV3P58mUmTpyIv78/Dz30EABNmzbFsiyGDh2a7Ofc7E6qq+Li4oiKikryWnBwMOHh4Rn6+UWyC61REXFhN3O7ZOXKlQwePJgnnniC6tWr4+/vz549e5gxYwYxMTEMGTIk2dcsXrw4xc60derUISQkJM2f16JFC/r160e/fv0ICgpKtp6kc+fOnDx5kkcffZQCBQqwf/9+Jk6cSIUKFRLWhqTlr7/+Ys6cOcleDwkJoU6dOgnPw8PDGTVqFPv27aNkyZIsXLiQrVu38u677+Lp6QnAs88+y7Rp02jfvj2bN2+mcOHCLF68mHXr1jFu3Dhy5swJwCOPPMIzzzzDhAkT+Pvvv6lfvz7x8fH88MMPPPLII+k63+fMmTMUKFCAZs2aUb58efz9/Vm1ahUbN27krbfeuunvIyJX2L3tSESMa7cnp+X67cl79uyxBg0aZFWvXt0KDg62PDw8rHz58lmPPfaY9c033yT52rS2JwPWt99+e1O11qxZ0wKszp07J3tv8eLFVt26da3g4GDLy8vLKliwoNW1a1fr8OHDN/y+adX20EMPJYx76KGHrNKlS1ubNm2yatSoYfn4+FiFChWyJk2alOx7Hj161OrQoYOVN29ey8vLyypbtqw1c+bMZOMuX75sjRkzxrrrrrssLy8vK1++fFaDBg2szZs3J6kvpW3HhQoVsiIjIy3ryjbpF1980SpfvryVM2dOy8/Pzypfvrw1ZcqUG35+EUnOYaV3XlNExGYPP/wwJ06c4LfffrO7FBG5w7RGRURERFyWgoqIiIi4LAUVERERcVlaoyIiIiIuSzMqIiIi4rIUVERERMRlKaiIiIiIy1JQEREREZeloCIiIiIuS0FFREREXJaCioiIiLgsBRURERFxWQoqIiIi4rIUVERERMRlKaiIiIiIy1JQEREREZeloCIiIiIuS0FFREREXJaCioiIiLgsBRURERFxWQoqIiIi4rIUVERERMRlKaiIiIiIy1JQEREREZeloCIiIiIuS0FFREREXJaCioiIiLgsBRURERFxWQoqIiIi4rJsDSpxcXEMHDiQIkWK4OvrS7FixXjjjTewLMvOskRERMRFeNj5w0eNGsXUqVP54IMPKF26NJs2baJDhw4EBgbSu3dvO0sTERERF+CwbJy+ePzxxwkJCeH9999PeK1p06b4+voyZ86cZONjYmKIiYlJeB4fH8/JkyfJkycPDofDaXWLiIjIrbMsizNnzhAeHo6b2w1u7lg2Gj58uFWoUCFr586dlmVZ1tatW63g4GBrzpw5KY4fPHiwBeihhx566KGHHlngcfDgwRtmBVtnVOLj43nllVcYPXo07u7uxMXFMXz4cAYMGJDi+OtnVKKioihYsCAHDx4kICDAiZWLiIjIrYqOjiYiIoLTp08TGBiY5lhb16h89NFHzJ07l3nz5lG6dGm2bt3K888/T3h4OJGRkcnGe3t74+3tnez1gIAABRUREZFM5maWbdgaVF588UX69+9Pq1atAChbtiz79+9nxIgRKQYVERERyV5s3Z58/vz5ZIto3N3diY+Pt60mERERcR22zqg0atSI4cOHU7BgQUqXLs0vv/zC2LFj6dixo51liYiIiIuwdTHtmTNnGDhwIEuWLOHYsWOEh4fTunVrBg0ahJeX1w2/Pjo6msDAQKKiorRGRUREMkx8fDyxsbF2l5FpeXp64u7unur76fn329agcrsUVEREJKPFxsayd+9eLUO4Tbly5SI0NDTFBbPp+ffb1ls/IiIirsSyLA4fPoy7uzsRERE3bkYmyViWxfnz5zl27BgAYWFht/X9FFRERESuuHz5MufPnyc8PJwcOXLYXU6m5evrC8CxY8cIDg5O8zbQjSgqioiIXBEXFwdwU+skJW1Xg96lS5du6/soqIiIiFxH58fdvoz6PVRQEREREZeloCIiIiIuS0FFREREkilcuDDjxo2zuwwFldSsWQNRUXZXISIikjaHw5HmY8iQIbf0fTdu3Mizzz6b4fWml7Ynp2DaNOjeHZ54Aj7+GLSNXkREXNXhw4cTrhcuXMigQYPYuXNnwmv+/v4J15ZlERcXh4fHjf/5z5cv3x2oNv30T3AK7r0XPDxg6VIYNcruakRExC6WBefO2fO42b7xoaGhCY/AwEAcDkfC8z///JOcOXOyYsUKKlWqhLe3N2vXrmX37t08+eSThISE4O/vT5UqVVi1alWS73v9rR+Hw8F7771HkyZNyJEjByVKlOCzzz7L6N/yZBRUUlClCkyebK5ffRW+/truikRExA7nz4O/vz2P8+cz7nP079+fkSNHsmPHDsqVK8fZs2dp2LAhq1ev5pdffqF+/fo0atSIAwcOpPl9hg4dSosWLfj1119p2LAhbdu25eTJkxlXaAoUVFLRuTN06WISbevWsG+f3RWJiIjcmtdff506depQrFgxgoKCKF++PF27dqVMmTKUKFGCN954g2LFit1whqR9+/a0bt2a4sWL83//93+cPXuWDRs23NHatUYlDRMnwtatsHEjPPUUrFsHV7oCi4hINpAjB5w9a9/PziiVK1dO8vzs2bMMGTKEzz//nMOHD3P58mUuXLhwwxmVcuXKJVz7+fkREBCQcKbPnaKgkgZvb7OY9t574ZdfzALbGTNADQtFRLIHhwP8/Oyu4vb5Xfch+vXrx8qVK3nzzTcpXrw4vr6+NGvWjNjY2DS/j6enZ5LnDofjjp8yrVs/NxARAQsXmp0/s2aZHUEiIiKZ2bp162jfvj1NmjShbNmyhIaGss9F1zgoqNyERx+FkSPNde/e8NNPdlckIiJy60qUKMEnn3zC1q1b2bZtG23atLnjMyO3SkHlJvXrB82awaVL5tejR+2uSERE5NaMHTuW3Llzc99999GoUSPq1avHvffea3dZKXJY1s3u1HY90dHRBAYGEhUVRUBAwB3/eWfOQLVqsGMHPPggrFoF192uExGRTOzixYvs3buXIkWK4OPjY3c5mVpav5fp+fdbMyqpSSG/5cwJS5aYX7//Hl5+2ZbKREREsg0FlZQcOgQNGsCiRcneKlUKPvjAXL/9NixY4PzyREREsgsFlZTMmAFffQXdupnQcp0mTWDAAHPdqRNs3+78EkVERLIDBZWUvPwyVKoEJ09Cx44p3gZ64w2oU8e0OH7qKTh92pZKRUREsjQFlZR4esKHH4KPj5lZeeedZEPc3WHePChYEHbtgmeeARfd2SUiIpJpKaik5u67E49O/t//4K+/kg3Jmxc++cR0sF2+HIYPd36ZIiIiWZmCSlp69oRateDCBTNlcvlysiGVKsHUqeZ68GBYscL5ZYqIiGRVCippudo3P1cu2LABRoxIcViHDtC1q1nK0qYN7Nnj9EpFRESyJAWVGylQACZPNtdDh8KmTSkOGz/eNIM7fdosrj1/3rllioiIZEUKKjejdWto0QLi4swtoBRSiLc3LF4MwcGwbVviDIuIiMid5HA40nwMGTLktr730qVLM7Te9FJQuRkOh1mIEhYGf/4J/funOKxAAXPSsrs7zJmTOBEjIiJypxw+fDjhMW7cOAICApK81q9fP7tLvC0KKjcrKAhmzjTXEyfCypUpDnv4YRg92ly/8AKsW+fEGkVEJNsJDQ1NeAQGBuJwOJK8tmDBAu6++258fHy46667mDJlSsLXxsbG0rNnT8LCwvDx8aFQoUKMuLIes3DhwgA0adIEh8OR8NzZPGz5qZlVvXrQo4eZKunQwbSkzZ072bAXXjBrbxcuNCctb9liJmNERCSTsSz7Fh3myGFm9G/D3LlzGTRoEJMmTaJixYr88ssvdOnSBT8/PyIjI5kwYQKfffYZH330EQULFuTgwYMcPHgQgI0bNxIcHMzMmTOpX78+7u7uGfTB0kdBJb1GjzazKX/9ZULLvHnJhjgc8N578Ntv8Pvv0Lw5fPMNeHnZUrGIiNyq8+fB39+en332LPj53da3GDx4MG+99RZPPfUUAEWKFOGPP/5g2rRpREZGcuDAAUqUKMH999+Pw+GgUKFCCV+bL18+AHLlykVoaOhtfphbp1s/6ZUjh+la6+4O8+eneiqhv79pBhcQYG7/ZPJbhCIiksmcO3eO3bt306lTJ/z9/RMew4YNY/fu3QC0b9+erVu3UqpUKXr37s3XX39td9nJaEblVlStCq+9ZrYrP/ccPPAA5M+fbFjJkibTPPmkWdZSo4bZQCQiIplEjhxmZsOun30bzl6pe/r06VSrVi3Je1dv49x7773s3buXFStWsGrVKlq0aEHt2rVZvHjxbf3sjGTrjErhwoVT3ErVo0cPO8u6Oa++CpUrm8YpHTqketDPE0/AK6+Y62efhZ07nVumiIjcBofD3H6x43Gb61NCQkIIDw9nz549FC9ePMmjSJEiCeMCAgJo2bIl06dPZ+HChXz88cecPHkSAE9PT+Li4m77t/F22DqjsnHjxiS/Ab/99ht16tShefPmdpZ1czw9zR7kihXNmpUpU0zL/RQMHWpu/6xZY9arrF9/20FZRETkhoYOHUrv3r0JDAykfv36xMTEsGnTJk6dOkXfvn0ZO3YsYWFhVKxYETc3NxYtWkRoaCi5cuWCKxMKq1evpmbNmnh7e5M7hQ0kd5qtMyr58uVLsoVq+fLlFCtWjIceeijF8TExMURHRyd52KpUqcS9yC++aHqspMDDwyxnCQ42G4V693ZumSIikj117tyZ9957j5kzZ1K2bFkeeughZs2alTCjkjNnTkaPHk3lypWpUqUK+/bt44svvsDNzcSDt956i5UrVxIREUHFihVt+QwOy3KN/qmxsbGEh4fTt29fXrl6r+Q6Q4YMYejQoclej4qKIiAgwAlVpiA+HurXN7MqlSvDjz+a2ZYUrF4NdeqY3W6zZ5smtyIi4jouXrzI3r17KVKkCD4+PnaXk6ml9XsZHR1NYGDgTf377TK7fpYuXcrp06dp3759qmMGDBhAVFRUwuPqXm9bubmZRnC5c5tzgIYPT3VorVrmhGWAbt3gjz+cV6aIiEhm5DJB5f3336dBgwaEh4enOsbb25uAgIAkD5eQP79ZowIwbJjp9paK116D2rXN1vzmzeHcOeeVKSIiktm4RFDZv38/q1atonPnznaXcutatTJ7j+Pi4OmnU00gV88BCg01Myrdu+vwQhERkdS4RFCZOXMmwcHBPPbYY3aXcnsmTzazK3//DS+9lOqwkBDTJ87NzaxVuXqEkIiIiCRle1CJj49n5syZREZG4uGRyfvP5c6dmDqmTIEvv0x16EMPweuvm+sePcxuIBERcQ0uss8kU8uo30Pbg8qqVas4cOAAHTt2tLuUjFGnDvTqZa47doT//kt16IAB5pzDixfNepUzZ5xXpoiIJHe1Y2tsbKzdpWR6568c5uiZyk7Ym+Uy25NvRXq2NznV+fNw772mDW3z5uYY5VQ6DB4/bnrG/fsvtGlj1q/cZjNCERG5RZZlceDAAS5dukR4eHhCPxG5eZZlcf78eY4dO0auXLkICwtLNiY9/34rqNwpmzaZw30uXzbpo23bVIeuW2duBcXFwbRpptW+iIjYIzY2lr179xKfytEocnOunrrsSOH/fSuouIrXXzeNUwIDzSKUiIhUh44eDS+/DN7epsV+hQpOrVRERK4RHx+v2z+3wdPTM+E2WkoUVFzF5ctQs6bpq/Loo6Z7bSrTiPHx5gDDzz+H4sVh82ZwxY8kIiJyuzJlZ9osycMDPvwQfH3hm2/S7Frr5gYffGAmXXbtgs6d1V9FREREQeVOK1kSJk0y14MGwaefpjo0Tx746COTbxYtSmx2KyIikl0pqDhDx47Qs6e5fvpp+O23VIdWr554IHPfvuYWkIiISHaloOIsY8fCI4/A2bPw5JNp9ld5/nlo3BhiY83u5tOnnVqpiIiIy1BQcRZPT3Nfp3Bh2LMHWrY0i21T4HDAjBlm6N69ZkJG61VERCQ7UlBxprx54bPPwM8PVq+Gfv1SHZo7t8k1np6wZAlMmODUSkVERFyCgoqzlS1rTiIEGD8+zRMJq1SBt94y1y++aHY5i4iIZCcKKnZ46inTCA6gWzf46adUh/bsCc2awaVL0KIFnDzpvDJFRETspqBil0GDoEkTs2K2SRP4558Uhzkc8N57UKwY7N8P7dtrvYqIiGQfCip2cXMzt4DKloWjR01YuXAhxaGBgWa9irc3LFuWeDtIREQkq1NQsZO/v2kAlyePOcSwS5dUp0vuvRfGjTPX/fvDjz86t1QRERE7KKjYrUgR04bW3R3mzoU330x1aNeu0KqVOWW5eXP491+nVioiIuJ0Ciqu4JFHEqdLXn4ZVqxIcZjDAe++C3ffDYcOQaNGpn+ciIhIVqWg4ip69Eg8ibB1a9i5M8VhOXOaE5bz5YNffoG2bc0Mi4iISFakoOIqHA6YPBlq1oSoKNNmPyoqxaFFipilLd7epn/cSy85vVoRERGnUFBxJV5e8PHHUKCAmVFp3TrV6ZIaNeCDD8z12LHwzjvOLVVERMQZFFRcTUgILF0KPj5mrcorr6Q6tGVLeOMNc92zJ3z1lfPKFBERcQYFFVdUqZI5lRBg9GizGygVr74K7dqZiZcWLeC335xXpoiIyJ2moOKqWrc2DVPALLLdtCnFYVd3Aj34IERHw+OPm/5xIiIiWYGCiisbNgweewwuXoTGjeHIkRSHeXvDJ59AiRKmzf4TT6Ta5FZERCRTUVBxZVebwN11l+nu9tRTEBOT4tA8ecy25aAgc8pyZCTExzu9YhERkQyloOLqAgPNXuTAQHPKcvfuqbbZL1ECliwBT0/T7Pa115xerYiISIZSUMkMSpaEhQvNQYYzZsCkSakOffBBc9oywIgRMHOm88oUERHJaAoqmUW9emYHEMALL8Dq1akObdcucTbl2Wfh22+dVKOIiEgGU1DJTPr2hWeeMXuRmzWD7dtTHTp0qOmzcvmyWdqSSkd+ERERl6agkplc3Yt8331w+jTUrQt79qQ41M0NZs0yHWxPnzabh06ccHrFIiIit0VBJbPx8YHly6FsWbNduU6dVLct+/iYJrdFisDu3dCkSaqbhkRERFySgkpmlDu36ZdftKiZUalXz0ybpCA42OSawEBYuxY6dUp105CIiIjLUVDJrMLCYOVKCA2FX381LWnPn09x6D33wOLFiW1ZXn/d6dWKiIjcEgWVzKxoUTOzkisXrFtnFtheupTi0Nq1YepUcz1kSJrHB4mIiLgMBZXMrlw5c2/H19ectty+faotabt0gRdfNNcdO5psIyIi4spsDyr//vsvTz/9NHny5MHX15eyZcuyKZUD+CQVNWvCxx+DhwfMmwd9+qS6EGXkSLOoNjbWHB+0e7fTqxUREblptgaVU6dOUbNmTTw9PVmxYgV//PEHb731Frlz57azrMypQQOYPdtsYZ40yTRSSYGbG8yZA5Urm+3Kjz0Gp045vVoREZGb4rAs+/aA9O/fn3Xr1vHDDz/c0tdHR0cTGBhIVFQUAQEBGV5fpjRlCvToYa7Hj4fevVMcdvgwVK0K//wDjzwCX34JXl7OLVVERLKn9Pz7beuMymeffUblypVp3rw5wcHBVKxYkenTp6c6PiYmhujo6CQPuU737omzKX36pLpqNizMnLbs729a7Pft69wyRUREboatQWXPnj1MnTqVEiVK8NVXX/Hcc8/Ru3dvPvjggxTHjxgxgsDAwIRHRESE02vOFAYOTJxJiYw0iSQF5crB/PnmevJk+OQTJ9YoIiJyE2y99ePl5UXlypX58ccfE17r3bs3Gzdu5Keffko2PiYmhphrWqtGR0cTERGhWz8piY83IWXOHNOi9uuv4YEHUhz68svmvMPAQPjlF9PJVkRE5E7JNLd+wsLCuOeee5K8dvfdd3PgwIEUx3t7exMQEJDkIalwc4MZM0wjuIsXoVEj2LYtxaHDhkH16hAVBa1amR1BIiIirsDWoFKzZk12Xnes719//UWhQoVsqylL8fSEjz4yMylRUabV/q5dKQ5bsMD0jduwAV591ZZqRUREkrE1qLzwwgusX7+e//u//2PXrl3MmzePd999lx5Xd63I7fP1hc8+g/Ll4ehRc4jhoUPJhhUqBDNnmus330x1WYuIiIhT2RpUqlSpwpIlS5g/fz5lypThjTfeYNy4cbRt29bOsrKeXLlMq/3ixWHfPqhbF06eTDasceOka3D/+cf5pYqIiFzL1sW0t0t9VNJp3z7TxfbQIbMoZdUq8PNLMiQmBu67D7ZsMXeMvvnGNLwVERHJKJlmMa04WeHCZmYld25Yvx6eeirZyllvb1i4EHLmhB9+0EnLIiJiLwWV7KZMGfjiCzOT8vXX8MwzEBeXZEjx4vDuu+Z62DBYvdqeUkVERBRUsqPq1U13t6u7gnr0SHaIYatW5rRly4K2bc06XBEREWdTUMmu6tY1zeAcDpg2Dd5+O9mQcePMBMzRo/D006aHnIiIiDMpqGRnLVqYgwsBXnsN9u5N8naOHGa9So4cZt3tyJH2lCkiItmXgkp217OnOT75wgV47rlkt4DuucecA8SVI4Ru8aBrERGRW6Kgkt1dvfXj7W12BF09pfAakZFmzW18PLRuDSdO2FKpiIhkQwoqAiVKmFs/AM8/n6wZnMMBU6ZAyZLw77/Qvn2yiRcREZE7QkFFjJdeMvd5jh8319fx9zcbhLy9TXv9FNbeioiIZDgFFTG8vBKbp7z/PqxZk2xI+fJmJxDAyy+bAwxFRETuJAUVSVSzJnTtaq67djX99K/TtSs0bw6XL5teK6dPO79MERHJPhRUJKmRIyE0FHbuhBEjkr3tcMD06VCkiNnNfLUpnIiIyJ2goCJJ5coFEyaY6//7P9ixI9mQwEDTX8XTExYvhnfecX6ZIiKSPSioSHLNmsFjj8GlS+ZeTwotaatUgdGjzfULL8DWrc4vU0REsj4FFUnO4TBd3nLkMB3eZsxIcVifPvDEE2YpS8uWcOaM0ysVEZEsTkFFUlaoELzxhrl+8UU4ciTZEIfDZJgCBeCvv1JsbCsiInJbFFQkdb17w733mq09L7yQ4pA8eUwzW3d3mDsXZs1yepUiIpKFKahI6jw8zBYfNzdYsABWrEhx2P33J06+9OgBf/zh3DJFRCTrUlCRtN17r1mMAubezrlzKQ57+WWoU8ecbdiiBZw/79wyRUQka1JQkRt7/XUoWBD274chQ1Ic4uYGH35oWrD8/rs5MkhEROR2KajIjfn7m1MJwRzy88svKQ4LCTHrVK42hUvhIGYREZF0UVCRm/PYY6Z3flwcPPus+TUFjz4KAwea62efhb//dm6ZIiKStSioyM0bP960pd20CSZNSnXYoEHw0ENw9qzpr5LCkUEiIiI3RUFFbl5YGIwaZa5few0OHkxx2NWtynnzmrtEL77o3DJFRCTrUFCR9OnSxZyyfPas2YucSoe3/Plh9mxzPXEiLFni3DJFRCRrUFCR9HFzg2nTzImEy5bBJ5+kOrRBA3jpJXPdsSPs2+e8MkVEJGtQUJH0K13aNE4B6NULoqJSHTpsGFSvbprbtmplzjkUERG5WQoqcmtefRVKlIDDh+GVV1Id5ulptinnygU//2y+TERE5GYpqMit8fGBd94x11Onwk8/pTq0cOHEA5jHjIEvvnBSjSIikukpqMite/RRiIw0C2qffTbN+zpNmpi7RADt2sE//zivTBERybwUVOT2vPmm2Yf822/mOg1jxpijg/77D9q0gcuXnValiIhkUgoqcnvy5oWxY8310KGwa1eqQ729YeFCyJkTfvjBHCEkIiKSFgUVuX1PPw21apkWtN26pdpbBaB4cXj3XXM9bBisXu28MkVEJPNRUJHb53CYhbU+PiZ5zJmT5vBWrUzfOMuCtm3h6FGnVSoiIpmMrUFlyJAhOByOJI+77rrLzpLkVhUvbg75AXjhhRumj3HjoEwZM+yZZyA+3jlliohI5mL7jErp0qU5fPhwwmPt2rV2lyS3ql8/KFfOrJZt1y7N9JEjh1mv4usLK1cmHiEkIiJyLduDioeHB6GhoQmPvHnz2l2S3CpPT5g3z6SPr7+Gt95Kc/g998DkyeZ64EBQRhURkevZHlT+/vtvwsPDKVq0KG3btuXAgQOpjo2JiSE6OjrJQ1xM6dLmvg6YjrUbNqQ5vH17sxY3Lg5atzaTMSIiIlfZGlSqVavGrFmz+PLLL5k6dSp79+7lgQce4MyZMymOHzFiBIGBgQmPiIgIp9csN6FLF2je3DRKad0a0giUDgdMmQIlS5omcO3bp7lpSEREshmHZbnOPwunT5+mUKFCjB07lk6dOiV7PyYmhpiYmITn0dHRREREEBUVRUBAgJOrlTSdPg0VKsD+/aa725w5JpWkYutWc3hhTIxpy/LCC06tVkREnCg6OprAwMCb+vfb9ls/18qVKxclS5ZkVypNw7y9vQkICEjyEBeVK5dZr+Lubn794IM0h1eoAG+/ba5ffvmGd4xERCSbcKmgcvbsWXbv3k1YWJjdpUhGuO8+060WoEcP2LkzzeHdukGzZubIoFatzKSMiIhkb7YGlX79+rFmzRr27dvHjz/+SJMmTXB3d6d169Z2liUZqX9/eOQROH/epI9rbt1dz+GA6dOhSBHYuzexKZyIiGRftgaVf/75h9atW1OqVClatGhBnjx5WL9+Pfny5bOzLMlI7u5mfUrevGYhyssvpzk8Vy7TX8XTExYvNg1vRUQk+3KpxbTplZ7FOGKzzz+Hxx8318uWJV6nYuxY+N//zEGG69ebNSwiIpI1ZNrFtJKFPfYY9Oljrtu3h3//TXP4Cy+YLBMTYzYNXbzonDJFRMS1KKiI84waBRUrmq5uV7u8pcLhgJkzISQEduyAIUOcWqmIiLgIBRVxHm9vWLAA/Pzgu+9g5Mg0h+fNm7hGZcwY+Pln55QpIiKuQ0FFnKtkycQDfgYPhh9/THN448bQtq0537B9e90CEhHJbhRUxPnatTPp4+oBP6dOpTl8wgQIDYU//zTZRkREsg8FFXG+qwf8FCsGBw7csGFKUBBMm2au33zT7AISEZHsQUFF7BEQAPPng4cHfPwxvPtumsOfeMKsv42Phw4ddAtIRCS7UFAR+1SpAiNGmOvnn4fff09z+PjxibeABg1yTokiImIvBRWxV9++UK+emSJp2RIuXEh1aFBQ4sTLW2/pFpCISHagoCL2cnMzJyuHhJgZlb590xzeqBE880ziLqA0co2IiGQBCipiv5AQmD3bXL/zjlmzkobx4yEszBzGrFtAIiJZm4KKuIa6deGll8x1585mN1AqcudOegvop5+cVKOIiDidgoq4jmHDoGpVOH3aHPBz+XKqQx9/3LRjsSzdAhIRycoUVMR1eHqaLcs5c8K6dfD662kOHzcOwsPhr79g4ECnVSkiIk6koCKupWjRxO5uw4aZM4FSce0toLFjb9iNX0REMiEFFXE9rVubrm6WZVrtnziR6tDHHoPISDO0QwfdAhIRyWoUVMQ1TZwIpUrBoUNmEUp8fKpDr70F9NprTq1SRETuMAUVcU1+frBgAXh7w+efw8iRqQ7NlQumTzfXb79tlreIiEjWoKAirqtCBZg0yVwPHAirV6c6tGFDM/Fy9RbQ+fPOK1NERO4cBRVxbZ06meQRH2/Wrvz7b6pD334b8ueHv//WLSARkaxCQUVcm8MBkydD+fJw/Lg5D+jSpRSHXnsLaNw4WLvWuaWKiEjGU1AR1+frC4sXQ0CAWYDy8supDm3QIHHDkG4BiYhkfgoqkjkUL24OL+TKPZ7Fi1MdOnasuQW0axe8+qrzShQRkYynoCKZR+PG8OKL5rpDB3MqYQquvQU0fjz88IMTaxQRkQyloCKZy//9Hzz4IJw9C02bwrlzKQ5r0AA6djS3gDp21C0gEZHMSkFFMhcPD9NfJTQUfv8dunUzaSQFY8dCgQLmFtArrzi9UhERyQAKKpL5hIXBwoXg7g5z5iSeDXSdwEB47z1zPWGCbgGJiGRGCiqSOT34IIwYYa779IGNG1McVq8edO6cuAsolTtFIiLiohRUJPPq188ssI2NhWbN4L//Uhz21lsQEQG7d+sWkIhIZqOgIpmXwwEzZ0KxYnDgADzzTIqHFwYEJL0FtHKl80sVEZFbo6AimVuuXPDxx+DjAytWwPDhKQ6rW9esuwXT3Hb3bueWKSIit0ZBRTK/8uVh6lRzPXhwqlMm48ZB9epw6hQ8+SScOePcMkVEJP3SFVRGjx7NhQsXEp6vW7eOmJiYhOdnzpyhe/fuGVuhyM1o3z5x1WybNnDwYLIh3t5m8iUszOxsbtcuxTtFIiLiQhyWlUoTihS4u7tz+PBhgoODAQgICGDr1q0ULVoUgKNHjxIeHk5cXNydq/ga0dHRBAYGEhUVRUBAgFN+priwixfhvvvgl1/M1MmaNeDllWzYzz+bTUOxsWYCZsgQW6oVEcm20vPvd7pmVK7PNOnIOCJ3no+POQMoVy5Yv97sCkpBtWrw7rvmeuhQ+OQT55YpIiI3z2XWqIwcORKHw8Hzzz9vdymSmRUtCrNnm+uJE00X2xRERsLV/9TatYPt251Yo4iI3DSXCCobN25k2rRplCtXzu5SJCto1Aj69zfXnTvDjh0pDhszBmrVMk3gnnwy1TYsIiJiI4/0fsF7772Hv78/AJcvX2bWrFnkzZsXriymTa+zZ8/Stm1bpk+fzrBhw9IcGxMTk2TxbnR0dLp/nmQTb7xhFqN8+605vHDDBrjy3+1VHh6mE3/VqrBnD7RoAV99ZV4XERHXkK7FtIULF8bhcNxw3N69e2+6gMjISIKCgnj77bd5+OGHqVChAuPGjUtx7JAhQxg6dGiy17WYVlJ09ChUrAiHD0Pr1jB3rmkSd53ffjNrb8+dM934U/nPT0REMkh6FtOm6/877tu373ZrS2LBggVs2bKFjamc03K9AQMG0Ldv34Tn0dHRREREZGhNkoWEhMBHH8HDD8P8+VCzJvTokWxYmTLw4Yfw1FMwfrxpy9Khgy0Vi4jIdWxbo3Lw4EH69OnD3Llz8fHxuamv8fb2JiAgIMlDJE333w+jR5vrF14wu4FS0KRJ4jblbt1SHSYiIk6WrqDy008/sXz58iSvzZ49myJFihAcHMyzzz6bZA1JWjZv3syxY8e499578fDwwMPDgzVr1jBhwgQ8PDyc1otFsoEXXjDrVC5dMgtRjh1LcdjAgSawxMaa2ZVDh5xeqYiIXCddQeX111/n999/T3i+fft2OnXqRO3atenfvz/Lli1jxIgRN/W9atWqxfbt29m6dWvCo3LlyrRt25atW7fi7u6e/k8jkhKHA2bMgBIlTMfaxx83C1Ku4+YGH3xgbgUdPmxCy8WLtlQsIiJXpCuobN26lVq1aiU8X7BgAdWqVWP69On07duXCRMm8NFHH93U98qZMydlypRJ8vDz8yNPnjyUKVMm/Z9EJC0BAbBsGQQFwcaNZmbl0qVkw3LmhE8/NcM2bDC3gdTXUETEPukKKqdOnSIkJCTh+Zo1a2jQoEHC8ypVqnAwhTNWRFxCqVKwfDn4+sIXX6SaQooWNWtw3d3NDMv48bZUKyIi6Q0qISEhCVuPY2Nj2bJlC9WrV094/8yZM3h6et5yMd99912qW5NFMkSNGqZbrZubuR00aFCKw2rVgrfeMtf/+x+sWuXcMkVExEhXUGnYsCH9+/fnhx9+YMCAAeTIkYMHHngg4f1ff/2VYsWK3Yk6RTLOE0/AO++Y62HDEq+v07u3OZQ5Pt7cKdq927lliohIOoPKG2+8gYeHBw899BDTp0/n3Xffxeua02lnzJhB3bp170SdIhmrSxdzdDKY3ipLlyYb4nDA1KnmEMNTp0yb/VtoviwiIrchXZ1pr4qKisLf3z/ZzpyTJ0+SM2fO27r9kx7p6WwnkoxlQdeuMH26OXl51SrTFO46hw5B5cpmJ1DjxvDxx+bOkYiI3Jr0/PudrqDSsWPHmxo3Y8aMm/2Wt0VBRW7b5cumacqyZZA7N6xdC/fck2zYzz/Dgw+aHiuDByc2hxMRkfS7Y0HFzc2NQoUKUbFiRdL6siVLlqSv4lukoCIZ4vx5s3p2/XqIiICffoL8+ZMN++ADs2YFYPFi00NORETS744FlR49ejB//nwKFSpEhw4dePrppwkKCsqImm+JgopkmBMnzG2fv/6CcuXg++8hMDDZsBdeMIcW+vnBjz+aoSIikj7p+fc7XXfaJ0+ezOHDh3nppZdYtmwZERERtGjRgq+++irNGRYRl5c3L3z1FYSGwq+/msUoKRwHMWaMmXw5d84srj1xwpZqRUSyjXQvCfT29qZ169asXLmSP/74g9KlS9O9e3cKFy7M2bNn70yVIs5QuDCsWGHa0373HbRrZ/YmX8PDAxYuNE3h9u0zeea//2yrWEQky7utvQtubm44HA4sy9IhgpI1VKgAS5aAp6dpT/u//yXrXpsnj2mz7+8P69ZBlSpmEkZERDJeuoNKTEwM8+fPp06dOpQsWZLt27czadIkDhw4gL+//52pUsSZatUyK2fBLEi52qL2GmXKmJBSpAjs3Wsa3i5e7PxSRUSyunQFle7duxMWFsbIkSN5/PHHOXjwIIsWLaJhw4a4qbGEZCWtW8Obb5rrF1+EuXOTDSlXzpxvWLu22TjUvDm8+ipoclFEJOOke3tywYIFqVixIg6HI9Vxn3zySUbVlybt+pE7rm9fePttcyvoiy9MKrnO5cvQv3/ixEvDhibX5Mrl/HJFRDKD9Pz77ZGeb9yuXbs0A4pIlvPmm6Yl7YIF0KSJ2bZcsWKSIR4eZljFitC5s8kzVauadSx3321b5SIiWcIttdB3FZpREaeIiYEGDeDbbyEkxDSEK1IkxaFbtpidQAcPms1DH35otjGLiEiiO9ZHRSRb8vY2O4HKlYOjR6FevVQbqNx7L2zaBA89ZA4wbNwYhg5NtstZRERukoKKyM0IDDQ9VgoVgr//hscfN13fUhAcDCtXQq9e5vmQIeY4oeho55YsIpIVKKiI3KzwcPjySwgKMqcUtmplVtKmwNMTJkyAGTPAy8usV6le3XToFxGRm6egIpIed91lTlr28YHly0332qioVId36GDW34aHw44dZpHtF184tWIRkUxNQUUkve67z/TRd3OD+fNNeFmwIFkH26uqVYPNm82XRUWZu0YjR6Y6XERErqGgInIrnnjCLEQpUQKOHDEN4urWTfXeTmio2TTUtasJKAMGQMuWqS5zERGRKxRURG7Vo4+aQ36GDjU7g1atgrJlYdAguHAh2XAvL3jnHfPw9IRFi0zr/T17bKleRCRTUFARuR0+PiaY/Pab2bYcGwtvvGECy5dfpvglXbsmtmTZvt0carhqldMrFxHJFBRURDJC8eJm+/JHH5mVs7t3myZxzZvDv/8mG16zplm3UrUqnDxpMs7YsVq3IiJyPQUVkYzicJhgsmMHPP+8WWy7eLFZbPv228m2MufPD2vWQPv2piHc//5njhLassW2TyAi4nIUVEQyWkCACSabN5vmKWfPmsMNK1c27fev4eNjeq1MnGjWsHzzDVSqBM88AwcO2PYJRERchoKKyJ1SoQKsWwfTpkHu3LBtm9mj/Oyz5n7PFQ4H9OwJO3dCmzbmtTlzoGRJcypzGm1aRESyPAUVkTvJzc0Ek507zT0egOnToVQpmDUryaKUwoVh7lzYuBEeftichThqlFn+MnGiWacrIpLdKKiIOEO+fDBzpmlTW7q0OdSwQwdzeuHvvycZWrmyuQX02WdmecuJE9C7t/myjz/WglsRyV4UVESc6YEH4JdfzFRJjhzwww/mFtHLLyfp/uZwQKNGZvvy1KnmoMNdu6BZM7j/fli/3tZPISLiNAoqIs7m6QkvvQR//AFPPml2A40eDffcA+++C+fPJwz18IBu3UxIGTgQfH3hxx9No7gWLcwuaBGRrExBRcQuhQrB0qXmHk+hQmabT9euEBFhVtEePJgwNGdOeP11+Ptv6NjRzLgsWgR33w0vvAD//WfrJxERuWMUVETs1qiRWafy1ltmRe3Jk+bWUJEiZtpk3bqEhSn588P778PWraZJ3KVLMG4cFCsGY8bAxYt2fxgRkYyloCLiCvz8TK+VXbtgyRKz7Scuzkyb3H+/6bM/e7bZCgSUK2c69H/9NZQvb7Ywv/SSWXw7b55pICcikhUoqIi4End3aNzYHAa0bRt06mQOPNy8GSIjzS2iIUPMic1AnTrmrVmzzGzL/v3Qti1Uq2a+hXYIiUhmZ2tQmTp1KuXKlSMgIICAgABq1KjBihUr7CxJxHWUKwfvvQf//APDh5szhI4eNac1Fyxo2tdu2oS7u8kwf/1lhuXMCZs2mcOda9QwW5rj4uz+MCIit8ZhWfb9f65ly5bh7u5OiRIlsCyLDz74gDFjxvDLL79QunTpG359dHQ0gYGBREVFERAQ4JSaRWxz6ZJJHRMmJG3Ff9990KcPPPUUeHhw7JhZePveewl3iihWzNxZat/e7IoWEbFTev79tjWopCQoKIgxY8bQqVOnZO/FxMQQc/V/ea980IiICAUVyX42boTx481pzZcumdcKFIAePaBLF8iTh2PHYNIkmDw5sWN/njxmSM+epgediIgd0hNUXGaNSlxcHAsWLODcuXPUqFEjxTEjRowgMDAw4REREeH0OkVcQpUq5kCg/ftNg5V8+cwtogEDTGDp0oXgY7/x+utm1/OkSVC0qNnG/Prr5s5Rt27mdpGIiCuzfUZl+/bt1KhRg4sXL+Lv78+8efNo2LBhimM1oyKSiosXYcECM8uydWvi648+am4LPfYYcbjzySdmG/PGjeZth8P0nHvxRXMHSUTEGTLVrZ/Y2FgOHDhAVFQUixcv5r333mPNmjXcc889N/xarVERuY5lwdq1JrAsWZK4T7loUejVCzp0wAoI5Icf4M03YdmyxC+tUcMElieeMJuPRETulEwVVK5Xu3ZtihUrxrRp0244VkFFJA3798OUKea05lOnzGv+/mZFba9eULIkO3aYPnMffph4OnOJEmbhbWSkadkvIpLRMuUalavi4+OT3N4RkVtUqJDpcHvwILzzjjlL6OxZs2ClVCl47DHuPvg170232L8fXnkFcuc2bfqfe86sYxk61JzeLCJiF1uDyoABA/j+++/Zt28f27dvZ8CAAXz33Xe0bdvWzrJEshY/P3OG0G+/mVa2jz9uFqd88YXpw3/PPYQumcrwV85x4IC5a1S4sAkoQ4aYwNK9O+zYYfcHEZHsyNagcuzYMdq1a0epUqWoVasWGzdu5KuvvqJOnTp2liWSNTkcppXtsmVmu0/v3qY73J9/miRSoAD+Q1+k9xP7+Ptvsza3UiW4cAGmTjUTMtWrw7RpcPq03R9GRLILl1ujkh5aoyJym6KjTf/9CRNg927zmpub2QrUpw/WAw+y5nsHb78Nn3+e2OHW2xuaNIEOHaBWLS2+FZH0ydSLadNDQUUkg8THm1tB48fDqlWJr5cvb7Y3t27N0Sgf5s6FmTPNXaSr8ueHdu3MGt2SJW2pXkQyGQUVEbl1f/xhZlhmzzb3fQDy5jVJ5JlnsMqWY8sWMxEzb15i11uudPNv3x5atIDAQNs+gYi4OAUVEbl9J0+aA4MmTTI7h64qVw6efhpatyYmXwGWLTOhZcWKxLYtvr7m6KH27U3POTeX218oInZSUBGRjHP5MixfbpqtLF+e2HDF4YBHHjGnOD/1FIfPBTBnjrk1dO0OoYgI05MlMhKKF7ftU4iIC1FQEZE749QpWLTInDP0ww+Jr/v4mAW4zzyDVacuG7d6MmsWzJ+fdIfQAw+YWZbmzc2GIxHJnhRUROTO27cP5s41My07dya+ni8ftGoFTz/NxbJV+PQzB7NmmRYuV28N5chhck2bNlC3Lnh52fYpRMQGCioi4jyWBVu2mMAyfz4cO5b4XsmSZj1L27b8612UDz80t4auPbU5KMjMsLRpA/ffr/UsItmBgoqI2OPyZbO9+cMPzaGIV3cNcWVL0DPPYDVrzqa9eZg3zzSVO3IkcUiBAtC6tQkt5cubZTAikvUoqIiI/c6cMWFlzhxYvTrxvo+nJzRsCJ07E1enPt+t9WDePFi82PSfu+ruu01gad0aihWz7VOIyB2goCIiruXQIXNbaM4c2Lo18fX8+aFTJ+jUiYvBBVmxwvRmWbYMrj2btFo1E1patIDQUFs+gYhkIAUVEXFdv/1mGq/MmgX//Wdeczigfn149ll47DGiznuyZIkJLddOxri5mZb9bdqYFv5qKieSOSmoiIjri4kxt4amT4dvvkl8PTTUHCLUuTMULcqRI/DRRya0/Pxz4jBvb3MQdJs25k6Sj48tn0JEboGCiohkLrt2mS64M2cm3TVUpw506WL2Mnt5sXu3uYM0d6459PmqPHnMZMxzz5kGcyLi2hRURCRzio01C1TefRdWrjRbn7nSm6V9ezPLUrIklgXbtplZlnnz4N9/zTB3d9O6v08fs8lIu4ZEXJOCiohkfnv3wvvvw4wZcPhw4usPP2xmWZ56Cnx8uHzZZJsJE+C77xKH3Xsv9O4NLVvqtpCIq1FQEZGs4/Jl+PxzM8uyYkXiLEtQkDlAqEsXs5cZ+PVXE1jmzoWLF82wfPmgWzfzCA+38XOISAIFFRHJmg4cMDMs778P//yT+Pr995vA0rw5+Ppy4oRZ8jJ5cuIwDw/zdu/eUL26bZ9ARBRURCTLi4uDL780O4aWLzfPAXLlMqc5d+kCZcty+bLZWDRhAqxdm/jlVauawNK8uc4ZErGDgoqIZB+HDpndQtOnw/79ia9Xr24CS8uW4OfHli0wcaJZfBsba4aEhibeFgoJse0TiGQ7Cioikv3Ex5tzht59Fz791KxtAQgIgLZtTWipWJFjx8yQKVMS1+h6epoDn3v3hsqVbf0UItmCgoqIZG9Hj5rOt9Onw+7dia9XrmwCS+vWxHrn5JNPzG2hn35KHFKjBnTvDk2bgq+vLdWLZHkKKiIiXJll+e47M4XyySdw6ZJ53c/PtLTt0gUqV2bjJgcTJsDChYlDcueGdu1MI7l77rH1U4hkOQoqIiLXO34cZs82syw7dya+XqGCSSNt2nDkQiDTp5sdQwcOJA6pWRO6doVmzTTLIpIRFFRERFJjWfDDD2aWZfHixGOac+QwC2+7dCGuSnW+Xung3XdNM7lrNxVdnWUpXdrWTyGSqSmoiIjcjJMn4cMPTWj544/E16tWhf794cknOXTELcVNRffdZwJL8+Ym44jIzVNQERFJD8syK2qnT4cFCxLb2pYqBS+/DG3bEufuxcqVJtN89lmqrVtE5CYoqIiI3Kpjx0zDlUmT4PRp81r+/PC//5k04u/P4cOJrVv27Uv80ho1zCxLixaaZRFJi4KKiMjtOnPGTJ+MHWuaynFlK1CvXuaRN2+qrVsCA+Hpp01oKVfO1k8h4pIUVEREMkpMDMyZA6NGwd9/m9d8fc3syv/+BwULAnDkSOIsy969iV9eq5bZGa3/iRJJlJ5/v92cVpWISGbk7Q2dOsGOHWaXUKVKcOGC6RRXrJg5wfn33wkNhQEDYNcu+Pprs5XZwwNWr4bBg+3+ECKZl2ZURETSw7JM+hg50vx61RNPmJ1CNWokvPTVV1C/Pri5webNpmWLiGhGRUTkznE4oHZtszhlwwbTa9/hMFuB7rsPHnoIVqwAy6JePbN9OT7etOWPj7e7eJHMR0FFRORWValibgft2GFuD3l6wvffQ8OGZvpk/nzeHnMZf3+z+3nmTLsLFsl8FFRERG5XqVKm7/7evWaBrb8//PortGlD/scqMOrlkwC89BKcOGF3sSKZi61BZcSIEVSpUoWcOXMSHBxM48aN2XntGRwiIplJ/vzw5pumhe0bb0BQEPz+O12jx1C2rGmEO2CA3UWKZC62BpU1a9bQo0cP1q9fz8qVK7l06RJ169bl3LlzdpYlInJ7goLgtddg1iwA3KdMZPrwY4CZePnpJ5vrE8lEXGrXz/HjxwkODmbNmjU8+OCDNxyvXT8i4tIsy5wbtGkT/O9/dDz5JjNnQvny5iUPD7sLFLFHpt31ExUVBUBQUFCK78fExBAdHZ3kISLishwOeP11cz15MmP6HiZ3bti2DaZMsbs4kczBZYJKfHw8zz//PDVr1qRMmTIpjhkxYgSBgYEJj4iICKfXKSKSLvXrQ/XqcPEied4bxciR5uXXXoPDh+0uTsT1ucytn+eee44VK1awdu1aChQokOKYmJgYYmJiEp5HR0cTERGhWz8i4tpWrYI6dcDbm/i/d1OjWX42bIDWrWHePLuLE3G+THfrp2fPnixfvpxvv/021ZAC4O3tTUBAQJKHiIjLq1ULHngAYmJwG/l/TJliutXOn5+0ua2IJGdrULEsi549e7JkyRK++eYbihQpYmc5IiJ3xrVrVaZPp1K+A3Tvbp726GHOPRSRlNkaVHr06MGcOXOYN28eOXPm5MiRIxw5coQLFy7YWZaISMZ7+GF45BG4dAmGD+eNNyAkBHbuhLfesrs4Eddl6xoVh8OR4uszZ86kffv2N/x6bU8WkUxl7VpzC8jDA/76i7k/FuHpp8HHB/74AzSpLNlFplmjYllWio+bCSkiIpnO/fdD3bpw+TK88QZt2piJlosXoU8fu4sTcU0usZhWRCTbGDrU/Dp7No5dfzNliplgWbbMHMAsIkkpqIiIOFP16uZ05bg4eOMN7r4b+vUzb/XuDefP212giGtRUBERcbarsypz58Kff/Laa1CwoDnLcPhwu4sTcS0KKiIizla5Mjz5JMTHw9Ch+PnB+PHmrTFj4M8/7S5QxHUoqIiI2GHIEPPrwoXw2288+SQ89pjZvdyjhznPUEQUVERE7FGhAjRtahLJ0KE4HDBhgtmq/M03sGCB3QWKuAYFFRERuwwZYrrWLl4M27ZRtCi8+qp5q29fuHKgvEi2pqAiImKXMmWgZUtzPXgwAC++CCVKwJEjMGiQveWJuAIFFREROw0ebE4o/PRT2LwZb2+YPNm8NWkS/PKL3QWK2EtBRUTETnfdBW3amOsrsyp16piJlvh4eO4586tIdqWgIiJit0GDwN0dPv8cfv4ZgLFjIWdO8/T99+0uUMQ+CioiInYrUQKeecZcX5lVCQ+H1183L/XvDydO2FifiI0UVEREXMHAgebQn6++gnXrAOjZE8qVg5MnTVgRyY4UVEREXEHRotChg7m+st3HwwOmTDEvvf8+/PijjfWJ2ERBRUTEVbz6Knh6mo5v330HQM2a0LGjefu55+DyZXtLFHE2BRUREVdRqBB07myuBw9O6KM/ahQEBcGvv8Jbb9lbooizKaiIiLiSV14Bb2/4/ntYvRqAvHlh5Ejzdv/+ZjmLtixLdqGgIiLiSgoUgK5dzfWgQQmzKp07w0svmZeHDYNmzeDsWRvrFHESBRUREVfTv785nfCnn8wuIMyRQKNGwaxZ4OUFS5bA/ffDgQN2FytyZymoiIi4mrAw6N7dXF8zqwIQGQnffgvBwbBtG1StavKMSFaloCIi4opefhly5ICNG2H58iRv3XcfbNhgeqwcPQoPPwwffmhbpSJ3lIKKiIgrCg6GXr3M9XWzKlzZILRuHTRuDLGx0K6duWOkRbaS1SioiIi4qn79wN8ftm6FpUuTve3vDx9/bDYKcWUbc5MmcOaM80sVuVMUVEREXFXevNCnj7kePDjF6RI3Nxg+HObMMbuaP/vMNInbt8/55YrcCQoqIiKurG9fCAiA7dth8eJUh7VtC2vWQGioGVq1Kqxd69RKRe4IBRUREVcWFGTCCsCQIRAXl+rQatXMItuKFeH4cXj0UbOdWSQzU1AREXF1zz8PuXLBjh0wf36aQyMi4IcfoGlTuHTJnHPYr1+a+UbEpSmoiIi4usBAkzbAnFDYv3+abWn9/OCjjxIOYeatt+DJJyE62kn1imQgBRURkczghRfgiSfMNMmoUXDXXbBgQbJty1e5ucHQoWaIjw98/rnpv7Jnj9MrF7ktCioiIplBjhzw6aewbBkULQr//gutW5uFKL/9luqXtWxpbgWFh8Pvv5tFtmvWOLVykduioCIikpk8/rhJHK+/bqZKvvsOKlQwC26jolL8ksqVzSLbypXhv/+gdm2YPt3plYvcEgUVEZHMxscHBg40i2ubNDErZd9+G0qVgtmzU7wdlD+/mUlp2RIuX4ZnnzVrdC9ftuUTiNw0BRURkcyqcGH45BP48ksoWdIc/BMZCQ88YLrZXidHDrNp6PXXzfPx480JzH/95fzSRW6WgoqISGZXr57p8jZypNnys24dVKoEPXvCqVNJhjocZjLm44/NZqKffzZ3jqZMSXVdroitbA0q33//PY0aNSI8PByHw8HSFM6yEBGRm+DlZU5c/vNPaNXKtNufPNnMtLz3XrL2+089ZbJNrVpw4QL06AENGsChQ7Z9ApEU2RpUzp07R/ny5Zk8ebKdZYiIZB0FCpj7O99+C6VLw4kT0KULVK8OGzcmGRoRAV9/bW4B+fjAV19B2bKmB4uIq7A1qDRo0IBhw4bRpEkTO8sQEcl6Hn4YfvnFLLINCDAhpVo1s4r2xImEYW5u0Ls3bNli7hadPGkW3LZtm+yukYgtMtUalZiYGKKjo5M8REQkFZ6eZmvPzp3Qrp1ZhDJ9urkdNHVqkr76d98NP/1k1q+4u8O8eWZ2ZdUqWz+BSOYKKiNGjCAwMDDhERERYXdJIiKuLzQUPvjAHKdcoYKZKuneHapUMQtVrvD0NDuC1q6F4sVNT7k6daBPH7OORcQOmSqoDBgwgKioqITHwYMH7S5JRCTzqFkTNm0yi2xz5TK3hqpVg5kzkwyrXt3sbn7uOfN8wgS4917zpSLOlqmCire3NwEBAUkeIiKSDu7uZjZl506oX99MlXTsaI5ZPn8+YZifn9myvGIFhIWZzUQ1asAbb6hJnDhXpgoqIiKSQYKDzUmFw4ebFbWzZpnZlT//TDKsfn1zd6h5cxNQBg1SkzhxLluDytmzZ9m6dStbr3RQ3Lt3L1u3buXAgQN2liUikj24ucErr8Dq1WYdy2+/mQOB5s1LMixPHli4EObMSWwSV7GiWY+rJnFypzksy77/zL777jseeeSRZK9HRkYya9asG359dHQ0gYGBREVF6TaQiMjtOHIE2rQx/VcAunUzW5t9fJIMO3gQ2reHb74xz+vXh/ffN6czi9ys9Pz7bWtQuV0KKiIiGSguDoYOhWHDzFRJxYqwaBEUK5ZkWHw8TJwI/fvDxYsQFATvvGNuD4ncjPT8+601KiIiYri7m/3JK1ZA3rxmV9C995qDD6/h5ma2LG/ebN4+eRJatDATMkeP2la9ZFEKKiIiklS9eiak1KwJ0dHQtKlpHBcbm2TYPfeYJnGvvWbCy/z5UKqU2f18TS85kduioCIiIskVKGDWq7z0knk+fjw88ADs359kmJeX2bK8fr1pwR8VZQ5trloVNmywp3TJWhRUREQkZZ6eMGoUfPYZ5M5tkkfFirB8ebKhVaqY3UBTppidQVu2mMZx3bqZW0Mit0pBRURE0taokUkeVaqY9vuNGpmVtNd1fnN3N91srz1aaNo0czto5kyzCFckvRRURETkxgoXNocA9e5tno8aBY8+ag4Euk5IiDlaaM0aKF3aHNbcsSM8+CD8+qvzS5fMTUFFRERujpeXWauyaBHkzAk//GBuBa1cmeLwBx80a3LHjDEt+detM7uE/vc/OHPG6dVLJqWgIiIi6dOsmbkVVKECHD9udgkNHpziVh9PT+jXD3bsMJuH4uJg7Fi46y746CN1tpUbU1AREZH0K14cfvwRnn3WpI3XXzfbmefONQcdXiciAhYvNi1aihWDQ4egZUuTcXRukKRFQUVERG6Nr69ZLTtnjrm38/PP8PTTpp9+r14pLkipX98cKTRkCHh7m7tGZcvCwIEp5hsRtdAXEZEM8O+/MGOGOfjn2l4rVapAly7QqpVZ13KNXbtMnvnyS/O8SBHTmv+xx5xcuzidWuiLiIhz5c9vpkV274avvjLrWDw9YeNGc3soLAw6dzad4a78/+PixeGLL+Djj01/ub174fHHoUmTZH3lJBvTjIqIiNwZx47B7Nnw3numucpVZcqY0PL005AnDwBnz5plLm+/bdqz+PqaSZgnn4Q6dSBHDvs+hmQ8nZ4sIiKuw7JMD5b33jNbfS5eNK97eZmtQJ07w8MPg5sbv/8O3bvD998nfrmvL9Sta0LL449Dvny2fRLJIAoqIiLimk6fhnnzYPp02Lo18fVixaBTJ2jfHis0jG+/haVL4dNP4cCBxGFubnDffSa0PPkklChhy6eQ26SgIiIirm/zZjPLMnduYgc4d3czbdK5M9SqheXjy7ZtJrB8+qlpIHetu+9ODC1Vq5ogI65PQUVERDKPc+dMt9v33jPta6/y8DB7l6tWhWrVoGpVDuS4i88+d+fTT+G775IeNxQaao4hatzYdPf38bHl08hNUFAREZHM6Y8/zBbn+fPh8OHk7/v7Q+XKUK0a50pXZWVUVRasLcAXXyRty+/nZ3q2PPmk2e4cFOTUTyE3oKAiIiKZm2XBP/+YJnIbNpjHpk1m9uV64eHEV67K7jxVWfFfVaZurMyfhwMT3nZ3NxMy5cqZQxLvucf8GhwMDodzP5YYCioiIpL1xMWZGZerwWXDBti+PdkZQ5bDwcXCd7EjZ1VWnKjKJ4eqsZ2yXMIrybg8eRJDy9VfFWCcQ0FFRESyh3PnzArbq8Hl559h375kw+Ld3DmdM4KDnkXZGVOEbWeKsoci7KUIeyjKcfIBJp0EBSWGFgWYO0NBRUREsq9jx0xH3KvBZcMGOHUqzS+54O7HAfci7Iw1weVqgNl7Jcycxy9JgClWDAoWNIctFixoFvK6uzvtE2Z6CioiIiJXWZY5rnnvXvPYsyfpr//+m9DWPzVHCU4SXvZTiINEJDzOuwdQIMKREFyuDTFXrwMDNSNzlYKKiIjIzYqJMYcLpRRi9uwxTepu4Az+SYLL1cc/FEi4dsvpn2KAiYgwRyGFhmafMKOgIiIiklFOnUo+G3PgABw8aB43uK2U8G3IlWKI+YcCHCOY4+TjrFce8oZ5EhoKISEmvFz/uPp6Zj7/SEFFRETEWc6dM1uprwaXq49rX4uOvulvd4pcHCdfwuMEeZM8v/raBb98uIfmI1d4jiQBJiQE8uY1ZyLly2euc+Vyra69CioiIiKuJDo65QBz8KBZI3P8ONZ//+G4hX+Sz5EjWZg5Qd6Ex3HyccotL5dz5cWRLy9eoUEE5XNPEmSuv86b15wZeacoqIiIiGQ2cXFw8iScOAHHjyc+rntuHT+Oddy85nYpNt0/Jh4HJwlKEmauBpprn1/IkRcrbz6qNQpm5CT/DP2oCioiIiJZnWWZcwOuDzTX/nriBPHHTxB/9AScOI7HmRsvDL7e5ojGVDqwJENLT8+/3x4Z+pNFRETEORwOCAgwj2LFUh3mduUBwKVLibM21wWaqw/r2HHijp7AOnECt5MnKFI1r7M+UYoUVERERLILT0+z2jYkJNUhjuvCQdC1R1TbwIXWAIuIiIjL8bB3TkNBRURERFyWgoqIiIi4LAUVERERcVkuEVQmT55M4cKF8fHxoVq1amzYsMHukkRERMQF2B5UFi5cSN++fRk8eDBbtmyhfPny1KtXj2PHjtldmoiIiNjM9oZv1apVo0qVKkyaNAmA+Ph4IiIi6NWrF/37908yNiYmhpiYmITn0dHRREREqOGbiIhIJpKehm+2zqjExsayefNmateunViQmxu1a9fmp59+SjZ+xIgRBAYGJjwiIiKcXLGIiIg4k61B5cSJE8TFxRFyXeOZkJAQjhw5kmz8gAEDiIqKSngcPHjQidWKiIiIs2WqzrTe3t54e3vbXYaIiIg4ia0zKnnz5sXd3Z2jR48mef3o0aOEhobaVpeIiIi4BluDipeXF5UqVWL16tUJr8XHx7N69Wpq1KhhZ2kiIiLiAmy/9dO3b18iIyOpXLkyVatWZdy4cZw7d44OHTrYXZqIiIjYzPag0rJlS44fP86gQYM4cuQIFSpU4Msvv0y2wFZERESyH9v7qNyOqKgocuXKxcGDB9VHRUREJJO42gft9OnTBAYGpjnW9hmV23HmzBkA9VMRERHJhM6cOXPDoJKpZ1Ti4+M5dOgQOXPmxOFwZOj3vpr2NFtjL/05uAb9ObgG/Tm4Bv053D7Lsjhz5gzh4eG4uaW9rydTz6i4ublRoECBO/ozAgIC9B+iC9Cfg2vQn4Nr0J+Da9Cfw+250UzKVbYfSigiIiKSGgUVERERcVkKKqnw9vZm8ODBatlvM/05uAb9ObgG/Tm4Bv05OFemXkwrIiIiWZtmVERERMRlKaiIiIiIy1JQEREREZeloCIiIiIuS0ElBZMnT6Zw4cL4+PhQrVo1NmzYYHdJ2cqQIUNwOBxJHnfddZfdZWV533//PY0aNSI8PByHw8HSpUuTvG9ZFoMGDSIsLAxfX19q167N33//bVu9WdmN/izat2+f7O9I/fr1bas3KxoxYgRVqlQhZ86cBAcH07hxY3bu3JlkzMWLF+nRowd58uTB39+fpk2bcvToUdtqzqoUVK6zcOFC+vbty+DBg9myZQvly5enXr16HDt2zO7SspXSpUtz+PDhhMfatWvtLinLO3fuHOXLl2fy5Mkpvj969GgmTJjAO++8w88//4yfnx/16tXj4sWLTq81q7vRnwVA/fr1k/wdmT9/vlNrzOrWrFlDjx49WL9+PStXruTSpUvUrVuXc+fOJYx54YUXWLZsGYsWLWLNmjUcOnSIp556yta6syRLkqhatarVo0ePhOdxcXFWeHi4NWLECFvryk4GDx5slS9f3u4ysjXAWrJkScLz+Ph4KzQ01BozZkzCa6dPn7a8vb2t+fPn21Rl9nD9n4VlWVZkZKT15JNP2lZTdnTs2DELsNasWWNZV/779/T0tBYtWpQwZseOHRZg/fTTTzZWmvVoRuUasbGxbN68mdq1aye85ubmRu3atfnpp59srS27+fvvvwkPD6do0aK0bduWAwcO2F1StrZ3716OHDmS5O9GYGAg1apV098Nm3z33XcEBwdTqlQpnnvuOf777z+7S8rSoqKiAAgKCgJg8+bNXLp0KcnfibvuuouCBQvq70QGU1C5xokTJ4iLiyMkJCTJ6yEhIRw5csS2urKbatWqMWvWLL788kumTp3K3r17eeCBBzhz5ozdpWVbV//7198N11C/fn1mz57N6tWrGTVqFGvWrKFBgwbExcXZXVqWFB8fz/PPP0/NmjUpU6YMXPk74eXlRa5cuZKM1d+JjJepT0+WrKlBgwYJ1+XKlaNatWoUKlSIjz76iE6dOtlam4graNWqVcJ12bJlKVeuHMWKFeO7776jVq1attaWFfXo0YPffvtNa+VsohmVa+TNmxd3d/dkq7aPHj1KaGiobXVld7ly5aJkyZLs2rXL7lKyrav//evvhmsqWrQoefPm1d+RO6Bnz54sX76cb7/9lgIFCiS8HhoaSmxsLKdPn04yXn8nMp6CyjW8vLyoVKkSq1evTngtPj6e1atXU6NGDVtry87Onj3L7t27CQsLs7uUbKtIkSKEhoYm+bsRHR3Nzz//rL8bLuCff/7hv//+09+RDGRZFj179mTJkiV88803FClSJMn7lSpVwtPTM8nfiZ07d3LgwAH9nchguvVznb59+xIZGUnlypWpWrUq48aN49y5c3To0MHu0rKNfv360ahRIwoVKsShQ4cYPHgw7u7utG7d2u7SsrSzZ88m+X/ke/fuZevWrQQFBVGwYEGef/55hg0bRokSJShSpAgDBw4kPDycxo0b21p3VpTWn0VQUBBDhw6ladOmhIaGsnv3bl566SWKFy9OvXr1bK07K+nRowfz5s3j008/JWfOnAnrTgIDA/H19SUwMJBOnTrRt29fgoKCCAgIoFevXtSoUYPq1avbXX7WYve2I1c0ceJEq2DBgpaXl5dVtWpVa/369XaXlK20bNnSCgsLs7y8vKz8+fNbLVu2tHbt2mV3WVnet99+awHJHpGRkZZ1ZYvywIEDrZCQEMvb29uqVauWtXPnTrvLzpLS+rM4f/68VbduXStfvnyWp6enVahQIatLly7WkSNH7C47S0np9x+wZs6cmTDmwoULVvfu3a3cuXNbOXLksJo0aWIdPnzY1rqzIodl/kBEREREXI7WqIiIiIjLUlARERERl6WgIiIiIi5LQUVERERcloKKiIiIuCwFFREREXFZCioiIiLishRURERExGUpqIhIpudwOFi6dKndZYjIHaCgIiK3pX379jgcjmSP+vXr212aiGQBOpRQRG5b/fr1mTlzZpLXvL29batHRLIOzaiIyG3z9vYmNDQ0ySN37txw5bbM1KlTadCgAb6+vhQtWpTFixcn+frt27fz6KOP4uvrS548eXj22Wc5e/ZskjEzZsygdOnSeHt7ExYWRs+ePZO8f+LECZo0aUKOHDkoUaIEn332WcJ7p06dom3btuTLlw9fX19KlCiRLFiJiGtSUBGRO27gwIE0bdqUbdu20bZtW1q1asWOHTsAOHfuHPXq1SN37txs3LiRRYsWsWrVqiRBZOrUqfTo0YNnn32W7du389lnn1G8ePEkP2Po0KG0aNGCX3/9lYYNG9K2bVtOnjyZ8PP/+OMPVqxYwY4dO5g6dSp58+Z18u+CiNwSu49vFpHMLTIy0nJ3d7f8/PySPIYPH25Z5nR2q1u3bkm+plq1atZzzz1nWZZlvfvuu1bu3Lmts2fPJrz/+eefW25ubtaRI0csy7Ks8PBw69VXX021BsB67bXXEp6fPXvWAqwVK1ZYlmVZjRo1sjp06JDBn1xEnEFrVETktj3yyCNMnTo1yWtBQUEJ1zVq1EjyXo0aNdi6dSsAO3bsoHz58vj5+SW8X7NmTeLj49m5cycOh4NDhw5Rq1atNGsoV65cwrWfnx8BAQEcO3YMgOeee46mTZuyZcsW6tatS+PGjbnvvvtu81OLiDMoqIjIbfPz80t2Kyaj+Pr63tQ4T0/PJM8dDgfx8fEANGjQgP379/PFF1+wcuVKatWqRY8ePXjzzTfvSM0iknG0RkVE7rj169cne3733XcDcPfdd7Nt2zbOnTuX8P66detwc3OjVKlS5MyZk8KFC7N69erbqiFfvnxERkYyZ84cxo0bx7vvvntb309EnEMzKiJy22JiYjhy5EiS1zw8PBIWrC5atIjKlStz//33M3fuXDZs2MD7778PQNu2bRk8eDCRkZEMGTKE48eP06tXL5555hlCQkIAGDJkCN26dSM4OJgGDRpw5swZ1q1bR69evW6qvkGDBlGpUiVKly5NTEwMy5cvTwhKIuLaFFRE5LZ9+eWXhIWFJXmtVKlS/Pnnn3BlR86CBQvo3r07YWFhzJ8/n3vuuQeAHDly8NVXX9GnTx+qVKlCjhw5aNq0KWPHjk34XpGRkVy8eJG3336bfv36kTdvXpo1a3bT9Xl5eTFgwAD27duHr68vDzzwAAsWLMiwzy8id47DMivmRUTuCIfDwZIlS2jcuLHdpYhIJqQ1KiIiIuKyFFRERETEZWmNiojcUbq7LCK3QzMqIiIi4rIUVERERMRlKaiIiIiIy1JQEREREZeloCIiIiIuS0FFREREXJaCioiIiLgsBRURERFxWf8PIuzl7qPYGbYAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.suptitle('MSE vs Epochs')\n",
    "plt.plot(train_err, label='Train', color='blue')\n",
    "plt.plot(test_err, label='Test', color='red')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('MSE')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "execution_state": "idle",
   "metadata": {
    "id": "LoGEmM5lH7_A"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[1084.,  317.,   12., ...,    0.,    0.,    0.],\n",
       "        [   0.,    0.,    0., ...,    0.,    0.,    0.],\n",
       "        [   0.,    0.,    0., ...,    0.,    0.,    0.],\n",
       "        ...,\n",
       "        [   0.,    0.,    0., ...,    0.,    0.,    0.],\n",
       "        [   0.,    0.,    0., ...,    0.,    0.,    0.],\n",
       "        [   0.,    0.,    0., ...,   31.,  119., 1237.]]),\n",
       " array([1.   , 1.08 , 1.16 , 1.24 , 1.32 , 1.4  , 1.48 , 1.561, 1.641,\n",
       "        1.721, 1.801, 1.881, 1.96 , 2.04 , 2.121, 2.2  , 2.281, 2.36 ,\n",
       "        2.441, 2.52 , 2.602, 2.68 , 2.762, 2.84 , 2.92 , 3.   , 3.08 ,\n",
       "        3.16 , 3.24 , 3.32 , 3.4  , 3.48 , 3.56 , 3.64 , 3.72 , 3.8  ,\n",
       "        3.88 , 3.96 , 4.04 , 4.12 , 4.203, 4.28 , 4.36 , 4.44 , 4.523,\n",
       "        4.6  , 4.68 , 4.76 , 4.84 , 4.92 , 5.   ], dtype=float16),\n",
       " array([0.9453, 1.022 , 1.1   , 1.178 , 1.255 , 1.332 , 1.409 , 1.486 ,\n",
       "        1.564 , 1.641 , 1.719 , 1.796 , 1.873 , 1.95  , 2.027 , 2.105 ,\n",
       "        2.184 , 2.26  , 2.336 , 2.414 , 2.492 , 2.57  , 2.646 , 2.723 ,\n",
       "        2.8   , 2.879 , 2.955 , 3.033 , 3.111 , 3.188 , 3.266 , 3.342 ,\n",
       "        3.42  , 3.498 , 3.574 , 3.652 , 3.729 , 3.807 , 3.885 , 3.96  ,\n",
       "        4.04  , 4.117 , 4.195 , 4.273 , 4.348 , 4.426 , 4.5   , 4.58  ,\n",
       "        4.656 , 4.734 , 4.812 ], dtype=float16),\n",
       " <matplotlib.collections.QuadMesh at 0x7f9cc006c980>)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi4AAAGdCAYAAAA1/PiZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAkCklEQVR4nO3df3BU9f3v8dcGyAYlu4KaH5BAtcEgYPjlDzZODVaQYmTIzHcoMozBKt7qhO+Q6vVHHOdr0eldesFS+9WLUQeiTtNc0QZn8AdGaJJRgt/wI7chdRhBSjKahM69sEuirJic+8f3S2wgP/ZsdnP4bJ6PmTPTPXmfPe+Pn4778rPnnHVZlmUJAADAAAlONwAAABAuggsAADAGwQUAABiD4AIAAIxBcAEAAMYguAAAAGMQXAAAgDEILgAAwBijnW4gHN3d3fr666+VnJwsl8vldDsAACAMlmXpzJkzmjhxohISorNWYkRw+frrr5WZmel0GwAAIAItLS3KyMiIynsZEVySk5Ol/xq4x+Nxuh0AAEa87vY5g9YEO7o1Ze7fez7Ho8GI4HL+6yGPx0NwAQDgEtD9zaiwa6N5mQcX5wIAAGMQXAAAgDEILgAAwBgEFwAAYAwjLs4FAACXltz/8y+D1nzfGZK0OarnZcUFAAAYg+ACAACMQXABAADGILgAAABjEFwAAIAxCC4AAMAYBBcAAGAMnuMCAABs+8fhqwet6T57NurnZcUFAAAYg+ACAACMQXABAADGILgAAABjEFwAAIAxCC4AAMAY3A4NAABsG3XWNXhRODU2seICAACMQXABAADGGFJw2bBhg1wul4qLi/utKSsrk8vl6rUlJSUN5bQAAGCEivgal/r6epWWlionJ2fQWo/HoyNHjvS8drmi/50XAACIfxGtuHR0dGjVqlV69dVXNX78+EHrXS6X0tLSerbU1NRITgsAAEa4iIJLUVGR8vPztXDhwrDqOzo6NGXKFGVmZmrZsmVqamqK5LQAAGCEs/1VUUVFhQ4ePKj6+vqw6rOzs7V161bl5OQoEAho06ZNys3NVVNTkzIyMvo8JhQKKRQK9bwOBoN22wQAAHHIVnBpaWnRunXrVFVVFfYFtj6fTz6fr+d1bm6urr/+epWWluq5557r8xi/36/169fbaQ0AAAyjpP87eE1XaPAau2x9VXTgwAGdPHlSc+fO1ejRozV69GjV1NToD3/4g0aPHq2urq5B32PMmDGaM2eOjh492m9NSUmJAoFAz9bS0mKnTQAAEKdsrbjccccdamxs7LXvF7/4haZNm6YnnnhCo0aNGvQ9urq61NjYqLvuuqvfGrfbLbfbbac1AAAwAtgKLsnJyZo5c2avfZdffrmuvPLKnv2FhYWaNGmS/H6/JOnZZ5/V/PnzlZWVpdOnT2vjxo06ceKE1qxZE81xAACAESDqv1XU3NyshIQfvoE6deqUHnzwQbW1tWn8+PGaN2+e9u7dq+nTp0f71AAAIM65LMuynG5iMMFgUF6vV4FAQB6Px+l2AAAY8XIe2TxoTVforP72v56K6uc3v1UEAACMEfWvigDArkUJy8Oqq+reHvNeEF2+lc+HVVf3p0dj3guiqysxjJoYfKfDigsAADAGwQUAABiD4AIAAIxBcAEAAMYguAAAAGMQXAAAgDG4HRqA47jNOX5d1v6d0y0gRi47Ofi9zl3fRf9+aFZcAACAMQguAADAGAQXAABgDIILAAAwBsEFAAAYg+ACAACMwe3QAADANs/fQ4PWfP/94DV2seICAACMQXABAADGILgAAABjEFwAAIAxCC4AAMAYBBcAAGAMggsAADAGz3EB4LhFCcvDqqvq3h7zXhBdic3/z+kWECPBH7kHren6zor6eVlxAQAAxiC4AAAAYxBcAACAMQguAADAGAQXAABgDIILAAAwhsuyrOjfqxRlwWBQXq9XgUBAHo/H6XYARNlN9/8urLr6rY/EvBcA0ROLz29WXAAAgDEILgAAwBgEFwAAYAyCCwAAMAbBBQAAGIPgAgAAjDGk4LJhwwa5XC4VFxcPWLd9+3ZNmzZNSUlJuuGGG/T+++8P5bQAAGCEGh3pgfX19SotLVVOTs6AdXv37tXKlSvl9/t19913q7y8XAUFBTp48KBmzpwZ6ekBxJGraludbgExcktheM/o+ewNntFjmkUJywet+d46F/XzRrTi0tHRoVWrVunVV1/V+PHjB6x94YUX9LOf/UyPPfaYrr/+ej333HOaO3euXnzxxUh7BgAAI1REwaWoqEj5+flauHDhoLV1dXUX1S1evFh1dXX9HhMKhRQMBnttAAAAtr8qqqio0MGDB1VfXx9WfVtbm1JTU3vtS01NVVtbW7/H+P1+rV+/3m5rAAAgztlacWlpadG6dev0xz/+UUlJSTFrqqSkRIFAoGdraWmJ2bkAAIA5bK24HDhwQCdPntTcuXN79nV1dam2tlYvvviiQqGQRo0a1euYtLQ0tbe399rX3t6utLS0fs/jdrvldrvttAYAAEYAWysud9xxhxobG9XQ0NCz3XjjjVq1apUaGhouCi2S5PP5tHv37l77qqqq5PP5ht49AAAYUWytuCQnJ190C/Pll1+uK6+8smd/YWGhJk2aJL/fL0lat26d8vLy9Pzzzys/P18VFRXav3+/XnnllWiOA4DBuiYkO90CYiThnOV0C4gzUX9ybnNzs1pbf3gmQ25ursrLy/XKK69o1qxZevvtt7Vjxw6e4QIAAGyL+AF051VXVw/4WpKWL1+u5csHf1ANAADAQPitIgAAYAyCCwAAMAbBBQAAGIPgAgAAjDHki3MBYKis/Y1Ot4AYGfe/94VX+KdYd4JoG33tjwYv6g5Jx6N7XlZcAACAMQguAADAGAQXAABgDIILAAAwBsEFAAAYg+ACAACMQXABAADG4DkuABxX1b3d6RYQI8xt/Pr+y78PXmOdi/p5WXEBAADGILgAAABjEFwAAIAxCC4AAMAYBBcAAGAMggsAADAGt0MDcNzsf90cVl3Dv/8q5r0ACE84t7oHg0F5vd6onpcVFwAAYAyCCwAAMAbBBQAAGIPgAgAAjEFwAQAAxiC4AAAAYxBcAACAMXiOCwDHpXx2xukWECN33vxsWHUf/ce/xbwXRNeihOWD1nxvnYv6eVlxAQAAxiC4AAAAYxBcAACAMQguAADAGAQXAABgDIILAAAwBrdDA3DcN5mXOd0CYoTbnONXVff2QWuCwaC8Xm9Uz8uKCwAAMAbBBQAAGMNWcNmyZYtycnLk8Xjk8Xjk8/n0wQcf9FtfVlYml8vVa0tKSopG3wAAYASydY1LRkaGNmzYoKlTp8qyLL3++utatmyZDh06pBkzZvR5jMfj0ZEjR3peu1yuoXcNAABGJFvBZenSpb1e/+Y3v9GWLVu0b9++foOLy+VSWlra0LoEAAAYyjUuXV1dqqioUGdnp3w+X791HR0dmjJlijIzM7Vs2TI1NTUN+t6hUEjBYLDXBgAAYPt26MbGRvl8Pp09e1bjxo1TZWWlpk+f3mdtdna2tm7dqpycHAUCAW3atEm5ublqampSRkZGv+fw+/1av3693dYAGOqTd/670y0gRsL5BWGFeWstIEkuy7IsOwd89913am5uViAQ0Ntvv63XXntNNTU1/YaXf3bu3Dldf/31WrlypZ577rl+60KhkEKhUM/rYDCozMxMBQIBeTweO+0CABxEcBnZzj/HJZqf37ZXXBITE5WVlSVJmjdvnurr6/XCCy+otLR00GPHjBmjOXPm6OjRowPWud1uud1uu60BAIA4N+TnuHR3d/daHRlIV1eXGhsblZ6ePtTTAgCAEcjWiktJSYmWLFmiyZMn68yZMyovL1d1dbV27dolSSosLNSkSZPk9/slSc8++6zmz5+vrKwsnT59Whs3btSJEye0Zs2a2IwGAADENVvB5eTJkyosLFRra6u8Xq9ycnK0a9cuLVq0SJLU3NyshIQfFnFOnTqlBx98UG1tbRo/frzmzZunvXv3hnU9DAAAwIVsX5zrhFhc3AMAiD0uzh3ZYvH5zW8VAQAAY9i+qwgAom3m45vDqjv8P38V814QXaykxK8lWY8NWvN9d3g379jBigsAADAGwQUAABiD4AIAAIxBcAEAAMYguAAAAGMQXAAAgDG4HRqA49ynne4AsXLnzc+GVffRf/xbzHtBdH3/5d8Hr7HORf28rLgAAABjEFwAAIAxCC4AAMAYBBcAAGAMggsAADAGwQUAABiD4AIAAIzBc1wAOO77y5zuALHC81niV1X39kFrgsGgvF5vVM/LigsAADAGwQUAABiD4AIAAIxBcAEAAMYguAAAAGMQXAAAgDG4HRqA4769yukOAJiCFRcAAGAMggsAADAGwQUAABiD4AIAAIxBcAEAAMYguAAAAGNwOzQAx3UlWU63AMCmRQnLB6353joX9fOy4gIAAIxBcAEAAMYguAAAAGMQXAAAgDEILgAAwBgEFwAAYAxbwWXLli3KycmRx+ORx+ORz+fTBx98MOAx27dv17Rp05SUlKQbbrhB77///lB7BgAAI5St57hkZGRow4YNmjp1qizL0uuvv65ly5bp0KFDmjFjxkX1e/fu1cqVK+X3+3X33XervLxcBQUFOnjwoGbOnBnNcQAwmIvHuADGqerePmhNMBiU1+uN6nldlmUN6V8ZEyZM0MaNG/XAAw9c9LcVK1aos7NTO3fu7Nk3f/58zZ49Wy+//HLY5zg/8EAgII/HM5R2AVyCsjb+Lqy6o489EvNeAERPLD6/I77GpaurSxUVFers7JTP5+uzpq6uTgsXLuy1b/HixaqrqxvwvUOhkILBYK8NAADAdnBpbGzUuHHj5Ha79dBDD6myslLTp0/vs7atrU2pqam99qWmpqqtrW3Ac/j9fnm93p4tMzPTbpsAACAO2Q4u2dnZamho0GeffaaHH35Yq1ev1t/+9reoNlVSUqJAINCztbS0RPX9AQCAmWz/yGJiYqKysrIkSfPmzVN9fb1eeOEFlZaWXlSblpam9vb2Xvva29uVlpY24Dncbrfcbrfd1gAAQJwb8nNcuru7FQqF+vybz+fT7t27e+2rqqrq95oYAACAgdhacSkpKdGSJUs0efJknTlzRuXl5aqurtauXbskSYWFhZo0aZL8fr8kad26dcrLy9Pzzz+v/Px8VVRUaP/+/XrllVdiMxoARuoe43QHAExhK7icPHlShYWFam1tldfrVU5Ojnbt2qVFixZJkpqbm5WQ8MMiTm5ursrLy/X000/rqaee0tSpU7Vjxw6e4QIAACIy5Oe4DAee4wLEt2t/H95zXL4s5jkugEkuqee4AAAADDeCCwAAMAbBBQAAGIPgAgAAjGH7AXQAEG1j21xOtwDAEKy4AAAAYxBcAACAMQguAADAGAQXAABgDIILAAAwBsEFAAAYg+ACAACMwXNc4LhFCcvDqqvq3h7zXuCM7kSnOwBgClZcAACAMQguAADAGAQXAABgDIILAAAwBsEFAAAYg+ACAACMwe3QcFz3grlOtwCHXdbmdAcATMGKCwAAMAbBBQAAGIPgAgAAjEFwAQAAxiC4AAAAYxBcAACAMQguAADAGDzHBY7bvafE6RbgsG+vcroDAKZgxQUAABiD4AIAAIxBcAEAAMYguAAAAGMQXAAAgDEILgAAwBjcDg3Hzf7XzWHVNfz7r2LeC5yR8fGZ8Ar/R6w7AXCpY8UFAAAYg+ACAACMYSu4+P1+3XTTTUpOTlZKSooKCgp05MiRAY8pKyuTy+XqtSUlJQ21bwAAMALZCi41NTUqKirSvn37VFVVpXPnzunOO+9UZ2fngMd5PB61trb2bCdOnBhq3wAAYASydXHuhx9+2Ot1WVmZUlJSdODAAd122239HudyuZSWlhZ5lwAAAEO9xiUQCEiSJkyYMGBdR0eHpkyZoszMTC1btkxNTU0D1odCIQWDwV4bAABAxLdDd3d3q7i4WLfeeqtmzpzZb112dra2bt2qnJwcBQIBbdq0Sbm5uWpqalJGRkafx/j9fq1fvz7S1mCYcV93Od0CHHZq5jinWwBgiIhXXIqKinT48GFVVFQMWOfz+VRYWKjZs2crLy9Pf/7zn3X11VertLS032NKSkoUCAR6tpaWlkjbBAAAcSSiFZe1a9dq586dqq2t7XfVpD9jxozRnDlzdPTo0X5r3G633G53JK0BAIA4ZmvFxbIsrV27VpWVldqzZ4+uueYa2yfs6upSY2Oj0tPTbR8LAABGNlsrLkVFRSovL9e7776r5ORktbW1SZK8Xq/Gjh0rSSosLNSkSZPk9/slSc8++6zmz5+vrKwsnT59Whs3btSJEye0Zs2aWIwHAADEMVvBZcuWLZKkBQsW9Nq/bds23XfffZKk5uZmJST8sJBz6tQpPfjgg2pra9P48eM1b9487d27V9OnT4/OCAAAwIhhK7hYljVoTXV1da/Xmzdv1ubN4f2IHgAAwED4rSIAAGCMiJ/jAkTLqLPdTrcAh40/3OF0CwAMwYoLAAAwBsEFAAAYg+ACAACMQXABAADGILgAAABjEFwAAIAxuB0ajmtZxP8NRzprf6PTLQAwBCsuAADAGAQXAABgDIILAAAwBsEFAAAYg+ACAACMQXABAADGILgAAABj8AANOK7b+73TLcBho6/9kdMtADAEKy4AAMAYBBcAAGAMggsAADAGwQUAABiD4AIAAIxBcAEAAMbgdmg4LtETcroFOOyDoxudbgGAIVhxAQAAxiC4AAAAYxBcAACAMQguAADAGAQXAABgDIILAAAwBrdDw3Ez0tqcbgEAYAhWXAAAgDEILgAAwBgEFwAAYAyCCwAAMAbBBQAAGMNWcPH7/brpppuUnJyslJQUFRQU6MiRI4Met337dk2bNk1JSUm64YYb9P777w+lZwAAMELZCi41NTUqKirSvn37VFVVpXPnzunOO+9UZ2dnv8fs3btXK1eu1AMPPKBDhw6poKBABQUFOnz4cDT6BwAAI4jLsiwr0oP/8Y9/KCUlRTU1Nbrtttv6rFmxYoU6Ozu1c+fOnn3z58/X7Nmz9fLLL4d1nmAwKK/Xq0AgII/HE2m7uESt3Pffwqr70/xXYt4LACB6YvH5PaRrXAKBgCRpwoQJ/dbU1dVp4cKFvfYtXrxYdXV1/R4TCoUUDAZ7bQAAABEHl+7ubhUXF+vWW2/VzJkz+61ra2tTampqr32pqalqa+v/aal+v19er7dny8zMjLRNAAAQRyIOLkVFRTp8+LAqKiqi25GkkpISBQKBnq2lpSXq5wAAAOaJ6LeK1q5dq507d6q2tlYZGRkD1qalpam9vb3Xvvb2dqWlpfV7jNvtltvtjqQ1AAAQx2ytuFiWpbVr16qyslJ79uzRNddcM+gxPp9Pu3fv7rWvqqpKPp/PfrcAAGBEs7XiUlRUpPLycr377rtKTk7uuU7F6/Vq7NixkqTCwkJNmjRJfr9fkrRu3Trl5eXp+eefV35+vioqKrR//3698gp3iAAAAHtsBZctW7ZIkhYsWNBr/7Zt23TfffdJkpqbm5WQ8MNCTm5ursrLy/X000/rqaee0tSpU7Vjx44BL+jFyPLN94lOtwAAMISt4BLOI1+qq6sv2rd8+XItX77cXmcAAAAX4LeKAACAMQguAADAGAQXAABgDIILAAAwBsEFAAAYg+ACAACMEdEj/4FoOhEY73QLAABDsOICAACMQXABAADGILgAAABjEFwAAIAxCC4AAMAYBBcAAGAMboeG4061JzvdAgDAEKy4AAAAYxBcAACAMQguAADAGAQXAABgDIILAAAwBsEFAAAYg9uh4bgfvRNm4f0xbgQAcMljxQUAABiD4AIAAIxBcAEAAMYguAAAAGMQXAAAgDEILgAAwBgEFwAAYAye4wLHJX5Q73QLAABDsOICAACMQXABAADGILgAAABjEFwAAIAxCC4AAMAYBBcAAGAMboeG46q6tzvdAgDAEKy4AAAAY9gOLrW1tVq6dKkmTpwol8ulHTt2DFhfXV0tl8t10dbW1jaUvgEAwAhkO7h0dnZq1qxZeumll2wdd+TIEbW2tvZsKSkpdk8NAABGONvXuCxZskRLliyxfaKUlBRdccUVto8DAAA4b9iucZk9e7bS09O1aNEiffrppwPWhkIhBYPBXhsAAEDMg0t6erpefvllvfPOO3rnnXeUmZmpBQsW6ODBg/0e4/f75fV6e7bMzMxYtwkAAAzgsizLivhgl0uVlZUqKCiwdVxeXp4mT56sN998s8+/h0IhhUKhntfBYFCZmZkKBALyeDyRtgsAAIZRMBiU1+uN6ue3I89xufnmm/XJJ5/0+3e32y232z2sPQEAgEufI89xaWhoUHp6uhOnBgAABrO94tLR0aGjR4/2vD5+/LgaGho0YcIETZ48WSUlJfrqq6/0xhtvSJJ+//vf65prrtGMGTN09uxZvfbaa9qzZ48++uij6I4EAADEPdvBZf/+/br99tt7Xj/yyCOSpNWrV6usrEytra1qbm7u+ft3332nRx99VF999ZUuu+wy5eTk6OOPP+71HgAAAOEY0sW5wyUWF/cAAIDYisXnN79VBAAAjEFwAQAAxiC4AAAAYxBcAACAMQguAADAGAQXAABgDIILAAAwBsEFAAAYg+ACAACMQXABAADGILgAAABjEFwAAIAxCC4AAMAYBBcAAGAMggsAADAGwQUAABiD4AIAAIwx2ukG7PiX2U9rdIJ7wJoPjm4ctn4AAMDwYsUFAAAYg+ACAACMQXABAADGILgAAABjEFwAAIAxCC4AAMAYRt0O3bYoXaMSk5xuAwAAOIQVFwAAYAyCCwAAMAbBBQAAGIPgAgAAjEFwAQAAxiC4AAAAYxBcAACAMYx6jsvSX9QqadyYQap+NUzdAACA4caKCwAAMAbBBQAAGIPgAgAAjEFwAQAAxrAdXGpra7V06VJNnDhRLpdLO3bsGPSY6upqzZ07V263W1lZWSorK4u0XwAAMILZDi6dnZ2aNWuWXnrppbDqjx8/rvz8fN1+++1qaGhQcXGx1qxZo127dkXSLwAAGMFclmVZER/scqmyslIFBQX91jzxxBN67733dPjw4Z5999xzj06fPq0PP/wwrPMEg0F5vV59fSRDnuSBs9bl6SdsjAAAAMTK+c/vQCAgj8cTlfeM+TUudXV1WrhwYa99ixcvVl1dXb/HhEIhBYPBXhsAAEDMg0tbW5tSU1N77UtNTVUwGNS3337b5zF+v19er7dny8zMjHWbAADAAJfkXUUlJSUKBAI9W0tLi9MtAQCAS0DMH/mflpam9vb2Xvva29vl8Xg0duzYPo9xu91yu92xbg0AABgm5isuPp9Pu3fv7rWvqqpKPp8v1qcGAABxxvaKS0dHh44ePdrz+vjx42poaNCECRM0efJklZSU6KuvvtIbb7whSXrooYf04osv6vHHH9f999+vPXv26K233tJ7770X9jnP3/h0pqN70Nquy7mQFwCAS8H5m2uGcAPzxSyb/vKXv1iSLtpWr15tWZZlrV692srLy7vomNmzZ1uJiYnWtddea23bts3WOY8dO9bnOdnY2NjY2Ngu/e3YsWN240a/hvQcl+Fy+vRpjR8/Xs3NzfJ6vU63M2yCwaAyMzPV0tIStfvfTcC4GfdIwLgZ90gQCAQ0efJknTp1SldccUVU3jPmF+dGQ0LCf16K4/V6R9SEn+fxeBj3CMK4RxbGPbKM1HGf/xyPyntF7Z0AAABijOACAACMYURwcbvdeuaZZ0bcs10YN+MeCRg34x4JGHf0xm3ExbkAAAAyZcUFAABABBcAAGASggsAADAGwQUAABjD8eBSW1urpUuXauLEiXK5XNqxY8egx1RXV2vu3Llyu93KyspSWVnZsPQaTXbHXV1dLZfLddHW1tY2bD1Hg9/v10033aTk5GSlpKSooKBAR44cGfS47du3a9q0aUpKStINN9yg999/f1j6jZZIxl1WVnbRfCclJQ1bz9GwZcsW5eTk9Dx0y+fz6YMPPhjwGNPnWhGMOx7m+kIbNmyQy+VScXHxgHXxMN//LJxxx8t8//rXv75oHNOmTRvwmGjMt+PBpbOzU7NmzdJLL70UVv3x48eVn5+v22+/XQ0NDSouLtaaNWu0a9eumPcaTXbHfd6RI0fU2tras6WkpMSsx1ioqalRUVGR9u3bp6qqKp07d0533nmnOjs7+z1m7969WrlypR544AEdOnRIBQUFKigo0OHDh4e196GIZNz6r6ds/vN8nzhxYth6joaMjAxt2LBBBw4c0P79+/XTn/5Uy5YtU1NTU5/18TDXimDcioO5/mf19fUqLS1VTk7OgHXxMt/nhTtuxdF8z5gxo9c4Pvnkk35rozbfUfvVoyiQZFVWVg5Y8/jjj1szZszotW/FihXW4sWLY9xd7IQz7vM/bnnq1Klh62s4nDx50pJk1dTU9Fvz85//3MrPz++175ZbbrF++ctfDkOHsRHOuLdt22Z5vd5h7Ws4jB8/3nrttdf6/Fs8zvV5A407nub6zJkz1tSpU62qqiorLy/PWrduXb+18TTfdsYdL/P9zDPPWLNmzQq7Plrz7fiKi111dXVauHBhr32LFy9WXV2dYz0Np9mzZys9PV2LFi3Sp59+6nQ7QxYIBCRJEyZM6LcmHuc8nHFLUkdHh6ZMmaLMzMxB/4v9UtfV1aWKigp1dnbK5/P1WROPcx3OuBVHc11UVKT8/PyL5rEv8TTfdsatOJrvL774QhMnTtS1116rVatWqbm5ud/aaM23ET+y+M/a2tqUmpraa19qaqqCwaC+/fZbjR071rHeYik9PV0vv/yybrzxRoVCIb322mtasGCBPvvsM82dO9fp9iLS3d2t4uJi3XrrrZo5c2a/df3NuWnX95wX7rizs7O1detW5eTkKBAIaNOmTcrNzVVTU5MyMjKGteehaGxslM/n09mzZzVu3DhVVlZq+vTpfdbG01zbGXe8zHVFRYUOHjyo+vr6sOrjZb7tjjte5vuWW25RWVmZsrOz1draqvXr1+snP/mJDh8+rOTk5IvqozXfxgWXkSo7O1vZ2dk9r3Nzc3Xs2DFt3rxZb775pqO9RaqoqEiHDx8e8DvReBTuuH0+X6//Qs/NzdX111+v0tJSPffcc8PQaXRkZ2eroaFBgUBAb7/9tlavXq2ampp+P8TjhZ1xx8Nct7S0aN26daqqqjLyQtNIRTLueJhvSVqyZEnP/87JydEtt9yiKVOm6K233tIDDzwQs/MaF1zS0tLU3t7ea197e7s8Hk/crrb05+abbzb2Q3/t2rXauXOnamtrB/0vjP7mPC0tLcZdRp+dcV9ozJgxmjNnjo4ePRqz/mIhMTFRWVlZkqR58+apvr5eL7zwgkpLSy+qjae5tjPuC5k41wcOHNDJkyd7rQB3dXWptrZWL774okKhkEaNGtXrmHiY70jGfSET57svV1xxha677rp+xxGt+TbuGhefz6fdu3f32ldVVTXgd8fxqqGhQenp6U63YYtlWVq7dq0qKyu1Z88eXXPNNYMeEw9zHsm4L9TV1aXGxkbj5vxC3d3dCoVCff4tHua6PwON+0ImzvUdd9yhxsZGNTQ09Gw33nijVq1apYaGhj4/vONhviMZ94VMnO++dHR06NixY/2OI2rzbetS3hg4c+aMdejQIevQoUOWJOt3v/uddejQIevEiROWZVnWk08+ad1777099V9++aV12WWXWY899pj1+eefWy+99JI1atQo68MPP3RwFPbZHffmzZutHTt2WF988YXV2NhorVu3zkpISLA+/vhjB0dh38MPP2x5vV6rurraam1t7dm++eabnpp7773XevLJJ3tef/rpp9bo0aOtTZs2WZ9//rn1zDPPWGPGjLEaGxsdGoV9kYx7/fr11q5du6xjx45ZBw4csO655x4rKSnJampqcmgU9j355JNWTU2Ndfz4ceuvf/2r9eSTT1oul8v66KOPLCtO59qKYNzxMNd9ufDumnid7wsNNu54me9HH33Uqq6uto4fP259+umn1sKFC62rrrrKOnnypGXFcL4dDy7nb/O9cFu9erVlWZa1evVqKy8v76JjZs+ebSUmJlrXXnuttW3bNoe6j5zdcf/2t7+1fvzjH1tJSUnWhAkTrAULFlh79uxxcASR6WvMknrNYV5eXs8/h/Peeust67rrrrMSExOtGTNmWO+9954D3UcuknEXFxdbkydPthITE63U1FTrrrvusg4ePOjQCCJz//33W1OmTLESExOtq6++2rrjjjt6PrytOJ1rK4Jxx8Nc9+XCD/B4ne8LDTbueJnvFStWWOnp6VZiYqI1adIka8WKFdbRo0d7/h6r+XZZ//kvVQAAgEuecde4AACAkYvgAgAAjEFwAQAAxiC4AAAAYxBcAACAMQguAADAGAQXAABgDIILAAAwBsEFAAAYg+ACAACMQXABAADGILgAAABj/H/iqtxtcx9+3AAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "batch_src, batch_labels, batch_padding_mask = mkbatch(4096)\n",
    "output = model(batch_src, batch_padding_mask)\n",
    "batch_src[0], batch_labels[0], output[0]\n",
    "x = batch_labels.detach().to(torch.float16).cpu().numpy().flatten()\n",
    "y = output.detach().to(torch.float16).cpu().numpy().flatten()\n",
    "plt.hist2d(x, y, bins=50, norm=mpl.colors.LogNorm())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LC6Xv3YfC0Rm"
   },
   "source": [
    "# Step 5: Fine Tune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "N_TUNE_EPOCHS = 10\n",
    "TUNE_LR = 1e-5\n",
    "TUNE_WD = 0 # 1e-5\n",
    "\n",
    "tune_criterion = nn.MSELoss()\n",
    "tune_optimizer = torch.optim.Adam(model.parameters(), lr=TUNE_LR, weight_decay=TUNE_WD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tune_evaluate():\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    with torch.no_grad():\n",
    "        batch_src, batch_labels, batch_padding_mask = mktunebatch(BSZ)\n",
    "        output = model(batch_src, batch_padding_mask)\n",
    "        loss = criterion(output.squeeze(1), batch_labels)\n",
    "    return loss.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:13<00:00,  1.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1000 \t Train Err: 0.2001 \t Test Err: 0.1592\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:13<00:00,  1.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2/1000 \t Train Err: 0.1355 \t Test Err: 0.1152\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:13<00:00,  1.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3/1000 \t Train Err: 0.1039 \t Test Err: 0.0962\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:13<00:00,  1.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4/1000 \t Train Err: 0.0877 \t Test Err: 0.0815\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:13<00:00,  1.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5/1000 \t Train Err: 0.0775 \t Test Err: 0.0728\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:13<00:00,  1.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6/1000 \t Train Err: 0.0682 \t Test Err: 0.0635\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:13<00:00,  1.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7/1000 \t Train Err: 0.0605 \t Test Err: 0.0581\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:13<00:00,  1.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8/1000 \t Train Err: 0.0548 \t Test Err: 0.0513\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:13<00:00,  1.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9/1000 \t Train Err: 0.0494 \t Test Err: 0.0457\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 16/16 [00:13<00:00,  1.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10/1000 \t Train Err: 0.0445 \t Test Err: 0.0403\n"
     ]
    }
   ],
   "source": [
    "tune_train_err = []\n",
    "tune_test_err = []\n",
    "\n",
    "# clear loss file\n",
    "open('tune_loss', 'w').close()\n",
    "\n",
    "for epoch in range(N_TUNE_EPOCHS):\n",
    "    model.train()\n",
    "    train_loss = 0\n",
    "    for i in tqdm(range(BPE)):\n",
    "        batch_src, batch_labels, batch_padding_mask = mktunebatch(BSZ)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(batch_src, batch_padding_mask)\n",
    "        loss = criterion(output.squeeze(1), batch_labels)\n",
    "        train_loss += loss.item() / BPE\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "    test_loss = tune_evaluate()\n",
    "    \n",
    "    tune_test_err.append(test_loss)\n",
    "    tune_train_err.append(train_loss)\n",
    "    with open('tune_loss', 'a') as f:\n",
    "        f.write(f\"{train_loss} {test_loss}\\n\")\n",
    "    print(f\"Epoch {epoch + 1}/{N_TUNE_EPOCHS} \\t Train Err: {train_loss:.4f} \\t Test Err: {test_loss:.4f}\")\n",
    "\n",
    "    if epoch % 10 == 9:\n",
    "        torch.save(model.state_dict(), f\"tune_model_weights_{epoch}.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHgCAYAAABNbtJFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABqaElEQVR4nO3deViUVcMG8HvYkc0FBUkUNPcFFBHRDC3csyzLJVM0tXIroiywV7GyUNMyxdy3yt3UzNIyUlOTVBQ119xJBTQVBBSUOe8fR2YYQQNmeWaY+3ddcznzzDPnnIG3j/s7q0oIIUBERERkRWyUbgARERGRqTEAERERkdVhACIiIiKrwwBEREREVocBiIiIiKwOAxARERFZHQYgIiIisjoMQERERGR1GICIiIjI6jAAEREpTKVSYdSoUUo3g8iqMAARlXNLliyBSqWCSqXCrl27irwvhICvry9UKhWeeeYZnfeysrIQGxuLJk2awMXFBVWqVEFgYCDeeustXL58WXPfhAkTNHUU90hNTTXJd32YR7XtjTfeULRtRKQMO6UbQESm4eTkhOXLl+OJJ57Qub5jxw78888/cHR01Ll+9+5dPPnkkzhx4gQiIiIwevRoZGVl4ejRo1i+fDmef/55+Pj46Hxm9uzZcHV1LVJ3xYoVjfStSq5jx44YOHBgkev16tVTpD1EpCwGICIr0a1bN6xZswYzZsyAnZ32P/3ly5cjKCgI165d07l/w4YNOHjwIJYtW4aXX35Z5707d+4gLy+vSB0vvvgiPD09jfgtyq5evXp45ZVXlG4GEZkJDoERWYl+/frh33//xdatWzXX8vLysHbt2iIBBwDOnDkDAGjbtm2R95ycnODu7m6QdjVp0gQdOnQocl2tVuOxxx7Diy++qLm2cuVKBAUFwc3NDe7u7mjatCm+/PJLg7QDANq3b48mTZogKSkJbdq0gbOzM/z9/TFnzpwi96anp2PIkCHw8vKCk5MTAgICsHTp0mK/x5dffommTZvCyckJVatWRZcuXbB///4i927YsAFNmjSBo6MjGjdujC1btui8f+vWLURGRsLPzw+Ojo6oVq0aOnbsiAMHDhjsZ0BkLRiAiKyEn58fQkNDsWLFCs21zZs3IyMjA3379i1yf61atQAAX3/9NYQQJarj+vXruHbtms7j5s2bj/xMnz598PvvvxeZJ7Rr1y5cvnxZ07atW7eiX79+qFSpEiZPnoxJkyahffv22L17d4nadufOnSJtu3btWpGerBs3bqBbt24ICgrClClTUKNGDQwfPhyLFi3S3HP79m20b98e33zzDfr374/PPvsMHh4eGDRoUJFANmTIEERGRsLX1xeTJ09GdHQ0nJyckJiYWOT7jhgxAn379sWUKVNw584d9OrVC//++6/mnjfeeAOzZ89Gr1698NVXX+Hdd9+Fs7Mzjh8/XqKfAREVIoioXFu8eLEAIPbt2yfi4+OFm5ubyMnJEUII8dJLL4kOHToIIYSoVauW6N69u+ZzOTk5on79+gKAqFWrlhg0aJBYuHChSEtLK1JHbGysAFDso379+o9s38mTJwUAMXPmTJ3rI0aMEK6urpq2vvXWW8Ld3V3cu3ev1D+Dh7UNgFixYoXmvrCwMAFATJs2TXMtNzdXBAYGimrVqom8vDwhhBDTp08XAMS3336ruS8vL0+EhoYKV1dXkZmZKYQQ4rfffhMAxJtvvlmkTWq1Wqd9Dg4O4vTp05prhw4dKvJz8fDwECNHjiz19yeiotgDRGRFevfujdu3b2PTpk24desWNm3aVOzwFwA4Ozvjzz//xJgxY4D7q8mGDBmC6tWrY/To0cjNzS3yme+++w5bt27VeSxevPiRbapXrx4CAwOxatUqzbX8/HysXbsWPXr0gLOzM3B/InV2drbOEF5pPPfcc0XatnXr1iLDb3Z2dnj99dc1rx0cHPD6668jPT0dSUlJAICffvoJ3t7e6Nevn+Y+e3t7vPnmm8jKysKOHTs0Pw+VSoXY2Ngi7VGpVDqvw8PDUadOHc3rZs2awd3dHWfPntVcq1ixIv7880+dFXhEVDacBE1kRapWrYrw8HAsX74cOTk5yM/P15lj8yAPDw9MmTIFU6ZMwYULF5CQkICpU6ciPj4eHh4emDhxos79Tz75ZJkmQffp0wdjx47FpUuX8Nhjj2H79u1IT09Hnz59NPeMGDECq1evRteuXfHYY4+hU6dO6N27N7p06VKiOmrUqIHw8PD/vM/HxwcuLi461wpWip0/fx6tW7fGhQsXULduXdjY6P7/kA0bNgQAXLhwAbg/j8rHxweVK1f+z3pr1qxZ5FqlSpVw48YNzespU6YgIiICvr6+CAoKQrdu3TBw4EDUrl37P8snIl3sASKyMi+//DI2b96MOXPmoGvXriVeol6rVi28+uqr2L17NypWrIhly5YZrE19+vSBEAJr1qwBAKxevRoeHh464aZatWpITk7Gxo0b8eyzz2Lbtm3o2rUrIiIiDNYOJdna2hZ7vfD8q969e+Ps2bOYOXMmfHx88Nlnn6Fx48bYvHmzCVtKVD4wABFZmeeffx42NjZITEx86PDXo1SqVAl16tTBlStXDNYmf39/tGrVCqtWrcK9e/ewbt069OzZs8jeRA4ODujRowe++uornDlzBq+//jq+/vprnD592mBtuXz5MrKzs3WunTp1Crg/kRz3w+Dff/8NtVqtc9+JEyc07wNAnTp1cPnyZVy/ft1g7atevTpGjBiBDRs24Ny5c6hSpQo++eQTg5VPZC0YgIisjKurK2bPno0JEyagR48eD73v0KFDRfYGwv3hnWPHjqF+/foGbVefPn2QmJiIRYsW4dq1azrDXwB0VkMBgI2NDZo1awYAxc5HKqt79+5h7ty5mtd5eXmYO3cuqlatiqCgIOD+nkqpqak685bu3buHmTNnwtXVFWFhYQCAXr16QQiBDz/8sEg9JV1ZVyA/Px8ZGRk616pVqwYfHx+Dfn8ia8E5QERWqCTDRlu3bkVsbCyeffZZtG7dGq6urjh79iwWLVqE3NxcTJgwochn1q5dW+xO0B07doSXl9cj6+vduzfeffddvPvuu6hcuXKR+TpDhw7F9evX8dRTT6FGjRq4cOECZs6cicDAQM3cm0c5deoUvv322yLXvby80LFjR81rHx8fTJ48GefPn0e9evWwatUqJCcnY968ebC3twcAvPbaa5g7dy4GDRqEpKQk+Pn5Ye3atdi9ezemT58ONzc3AECHDh0wYMAAzJgxA3///Te6dOkCtVqNnTt3okOHDqU6/+vWrVuoUaMGXnzxRQQEBMDV1RW//vor9u3bh2nTppW4HCK6T+llaERkXIWXwT/Kg8vgz549K8aPHy9at24tqlWrJuzs7ETVqlVF9+7dxW+//abz2Uctgwcgtm3bVqK2tm3bVgAQQ4cOLfLe2rVrRadOnUS1atWEg4ODqFmzpnj99dfFlStX/rPcR7UtLCxMc19YWJho3Lix2L9/vwgNDRVOTk6iVq1aIj4+vkiZaWlpYvDgwcLT01M4ODiIpk2bisWLFxe57969e+Kzzz4TDRo0EA4ODqJq1aqia9euIikpSad9xS1vr1WrloiIiBDi/nL8MWPGiICAAOHm5iZcXFxEQECA+Oqrr/7z+xNRUSpR2n5YIqJyqn379rh27Rr++usvpZtCREbGOUBERERkdRiAiIiIyOowABEREZHV4RwgIiIisjrsASIiIiKrwwBEREREVocBiIiIiKwOAxARERFZHQYgIiIisjoMQERERGR1GICIiIjI6jAAERERkdVhACIiIiKrwwBEREREVocBiIiIiKwOAxARERFZHQYgIiIisjoMQERERGR1GICIiIjI6jAAERERkdVhACIiIiKrwwBEREREVocBiIiIiKwOAxARERFZHQYgIiIisjoMQERERGR1GICIiIjI6jAAERERkdVhACIiIiKrwwBEREREVocBiIiIiKyOndINMEdqtRqXL1+Gm5sbVCqV0s0hIiKiEhBC4NatW/Dx8YGNzaP7eBiAinH58mX4+voq3QwiIiIqg5SUFNSoUeOR9zAAFcPNzQ24/wN0d3dXujlERERUApmZmfD19dX8HX8UBqBiFAx7ubu7MwARERFZmJJMX+EkaCIiIrI6DEBERERkdRiAiIiIyOpwDhAREZGJqNVq5OXlKd0Mi2Vvbw9bW1uDlMUAREREZAJ5eXk4d+4c1Gq10k2xaBUrVoS3t7fe+/QxABERERmZEAJXrlyBra0tfH19/3OTPipKCIGcnBykp6cDAKpXr65XeQxARERERnbv3j3k5OTAx8cHFSpUULo5FsvZ2RkAkJ6ejmrVquk1HMYISkREZGT5+fkAAAcHB6WbYvEKAuTdu3f1KocBiIiIyER4vqT+DPUzZAAiIiIiq8MARERERFbHLALQrFmz4OfnBycnJ4SEhGDv3r0PvXf+/Plo164dKlWqhEqVKiE8PLzI/UIIjB8/HtWrV4ezszPCw8Px999/m+CbEBER0aP4+flh+vTpSjdD+QC0atUqREVFITY2FgcOHEBAQAA6d+6sWeb2oO3bt6Nfv37Ytm0b9uzZA19fX3Tq1AmXLl3S3DNlyhTMmDEDc+bMwZ9//gkXFxd07twZd+7cMeE3K97588CpU0q3goiI6NFUKtUjHxMmTChTufv27cNrr71m8PaWlkoIIZRsQEhICIKDgxEfHw/c3yXT19cXo0ePRnR09H9+Pj8/H5UqVUJ8fDwGDhwIIQR8fHzwzjvv4N133wUAZGRkwMvLC0uWLEHfvn2LlJGbm4vc3FzN68zMTPj6+iIjI8Ogp8F/+SUQGQn07QusWGGwYomIyMzduXMH586dg7+/P5ycnJRuTomkpqZqnq9atQrjx4/HyZMnNddcXV3h6uoK3B95yc/Ph52d8XfXedTPMjMzEx4eHiX6+61oD1BeXh6SkpIQHh6ubZCNDcLDw7Fnz54SlZGTk4O7d++icuXKAIBz584hNTVVp0wPDw+EhIQ8tMy4uDh4eHhoHr6+vnp/t+K0bSv/3bAByMgwShVERGQBhACys5V5lLTbw9vbW/Pw8PCASqXSvD5x4gTc3NywefNmBAUFwdHREbt27cKZM2fw3HPPwcvLC66urggODsavv/6qU+6DQ2AqlQoLFizA888/jwoVKqBu3brYuHGjoX/kRSgagK5du4b8/Hx4eXnpXPfy8tJJno/y/vvvw8fHRxN4Cj5XmjJjYmKQkZGheaSkpJTxGz1aUBDQqBFw5w6wZo1RqiAiIguQkwO4uirzyMkx3PeIjo7GpEmTcPz4cTRr1gxZWVno1q0bEhIScPDgQXTp0gU9evTAxYsXH1nOhx9+iN69e+Pw4cPo1q0b+vfvj+vXrxuuocVQfA6QPiZNmoSVK1di/fr1enUpOjo6wt3dXedhDCoVMHCgfP7110apgoiIyGQ++ugjdOzYEXXq1EHlypUREBCA119/HU2aNEHdunXx8ccfo06dOv/ZozNo0CD069cPjz/+OD799FNkZWU9ckGUISh6FIanpydsbW2Rlpamcz0tLQ3e3t6P/OzUqVMxadIk/Prrr2jWrJnmesHn0tLSdM4JSUtLQ2BgoMG/Q2n17w/ExAA7dwJnzwK1ayvdIiIiMrUKFYCsLOXqNpSWLVvqvM7KysKECRPw448/4sqVK7h37x5u3779nz1Ahf+Ou7i4wN3d/aGLoQxF0R4gBwcHBAUFISEhQXNNrVYjISEBoaGhD/3clClT8PHHH2PLli1Ffvj+/v7w9vbWKTMzMxN//vnnI8s0lRo1gILpSd98o3RriIhICSoV4OKizMOQm1G7uLjovH733Xexfv16fPrpp9i5cyeSk5PRtGlT5OXlPbIce3v7B34+KqjVasM1tBiKD4FFRUVh/vz5WLp0KY4fP47hw4cjOzsbgwcPBgAMHDgQMTExmvsnT56McePGYdGiRfDz80NqaipSU1ORdT9Kq1QqREZGYuLEidi4cSOOHDmCgQMHwsfHBz179lTsexZWeBhM2TV4REREhrN7924MGjQIzz//PJo2bQpvb2+cP39e6WYVS/HT4Pv06YOrV69i/PjxSE1NRWBgILZs2aKZxHzx4kXY2Ghz2uzZs5GXl4cXX3xRp5zY2FjNngTvvfcesrOz8dprr+HmzZt44oknsGXLFrNZevj883Ii2tmzwO7dwBNPKN0iIiIi/dWtWxfr1q1Djx49oFKpMG7cOKP35JSV4gEIAEaNGoVRo0YV+9727dt1XpckSapUKnz00Uf46KOPDNZGQ3JxAV58EViyRPYCMQAREVF58Pnnn+PVV19FmzZt4Onpiffffx+ZmZlKN6tYim+EaI5Ks5FSWW3bBjz1FODuDqSmAs7ORqmGiIjMgCVuhGiuysVGiNYsLAyoWRPIzARMsN8TERERFcIApBAbG2DAAPmcewIRERGZFgOQggpWg/38sxwGIyIiItNgAFJQvXpA69ZAfj6wfLnSrSEiIrIeDEAKK+gFWrpU6ZYQERFZDwYghfXpAzg4AIcPA4cOKd0aIiIi68AApLDKlYEePeRzToYmIiIyDQYgMxARIf9dtgy4d0/p1hAREZV/DEBmoEsXwNMTSEsDfvlF6dYQERGVfwxAZsDeHnj5Zfmck6GJiMgcqFSqRz4Kzt8sa9kbNmwwaHtLyyzOAiM5DDZjBvD998DNm0DFikq3iIiIrNmVK1c0z1etWoXx48fj5MmTmmuurq4Ktcww2ANkJpo3Bxo3BnJzgTVrlG4NERFZO29vb83Dw8MDKpVK59rKlSvRsGFDODk5oUGDBvjqq680n83Ly8OoUaNQvXp1ODk5oVatWoiLiwMA+Pn5AQCef/55qFQqzWtTYw+QmVCpZC/Qe+/JYbBhw5RuERERGY0QQE6OMnVXqCD/6Ohh2bJlGD9+POLj49G8eXMcPHgQw4YNg4uLCyIiIjBjxgxs3LgRq1evRs2aNZGSkoKUlBQAwL59+1CtWjUsXrwYXbp0ga2trYG+WOkwAJmR/v2B6Ghg927gzBmgTh2lW0REREaRkwMoNYSUlQW4uOhVRGxsLKZNm4YXXngBAODv749jx45h7ty5iIiIwMWLF1G3bl088cQTUKlUqFWrluazVatWBQBUrFgR3t7een6ZsuMQmBnx8QHCw+Vz7glERETmKDs7G2fOnMGQIUPg6uqqeUycOBFnzpwBAAwaNAjJycmoX78+3nzzTfxihkuc2QNkZiIi5FL4r78GYmPlqfFERFTOVKgge2KUqlsPWffbPX/+fISEhOi8VzCc1aJFC5w7dw6bN2/Gr7/+it69eyM8PBxr167Vq25DYgAyMz17Am5uwPnzciisXTulW0RERAanUuk9DKUULy8v+Pj44OzZs+jfv/9D73N3d0efPn3Qp08fvPjii+jSpQuuX7+OypUrw97eHvn5+SZt94MYgMxMhQrASy8BixbJydAMQEREZG4+/PBDvPnmm/Dw8ECXLl2Qm5uL/fv348aNG4iKisLnn3+O6tWro3nz5rCxscGaNWvg7e2Nivf3ePHz80NCQgLatm0LR0dHVKpUyeTfgQMsZqjghPjVq4Hbt5VuDRERka6hQ4diwYIFWLx4MZo2bYqwsDAsWbIE/v7+AAA3NzdMmTIFLVu2RHBwMM6fP4+ffvoJNvfndUybNg1bt26Fr68vmjdvrsh3UAkhhCI1m7HMzEx4eHggIyMD7u7uJq9frQZq1wYuXACWLwf69TN5E4iIyIDu3LmDc+fOwd/fH05OTko3x6I96mdZmr/f7AEyQzY22l4grgYjIiIyPAYgMzVggPz3l1+AQruRExERkQEwAJmpunWBNm3kcNiyZUq3hoiIqHxhADJjBcNgS5fKXdOJiIjIMBiAzFjv3oCjI/DXX0BystKtISIifXHdkf4M9TNkADJjlSoBzz4rn3MyNBGR5SrYITkvL0/ppli8nPuHyNrb2+tVDjdCNHMDBwJr1sjl8FOmAHr+vomISAF2dnaoUKECrl69Cnt7e81+OFRyQgjk5OQgPT0dFStW1PsUeQYgM9e5M1CtGpCeDvz8M/DMM0q3iIiISkulUqF69eo4d+4cLly4oHRzLJqhTpFnADJz9vbAyy8D06fLYTAGICIiy+Tg4IC6detyGEwP9vb2evf8FOBO0MVQeifoBx08CLRoATg4AKmpcm4QERER6bKonaBnzZoFPz8/ODk5ISQkBHv37n3ovUePHkWvXr3g5+cHlUqF6dOnF7knPz8f48aNg7+/P5ydnVGnTh18/PHHFj3zPjAQaNoUyMuT54MRERGRfhQNQKtWrUJUVBRiY2Nx4MABBAQEoHPnzkhPTy/2/pycHNSuXRuTJk166Pjf5MmTMXv2bMTHx+P48eOYPHkypkyZgpkzZxr52xiPSsWjMYiIiAxJ0SGwkJAQBAcHIz4+HgCgVqvh6+uL0aNHIzo6+pGf9fPzQ2RkJCIjI3WuP/PMM/Dy8sLChQs113r16gVnZ2d8++23xZaVm5uL3NxczevMzEz4+vqazRAYII/DqFFD7gx96pTcKZqIiIi0LGIILC8vD0lJSQgPD9c2xsYG4eHh2LNnT5nLbdOmDRISEnDq1CkAwKFDh7Br1y507dr1oZ+Ji4uDh4eH5uHr61vm+o2lenWgUyf5/JtvlG4NERGRZVMsAF27dg35+fnw8vLSue7l5YXU1NQylxsdHY2+ffuiQYMGsLe3R/PmzREZGYn+/fs/9DMxMTHIyMjQPFJSUspcvzEVHgZTq5VuDRERkeUqd8vgV69ejWXLlmH58uVo3LgxkpOTERkZCR8fH0RERBT7GUdHRzg6Opq8raXVsyfg7g5cuADs3AmEhSndIiIiIsukWA+Qp6cnbG1tkZaWpnM9LS1Nrw2OxowZo+kFatq0KQYMGIC3334bcXFxBmi1spydgZdeks85GZqIiKjsFAtADg4OCAoKQkJCguaaWq1GQkICQkNDy1xuTk5OkS3GbW1toS4nY0YFnVhr1gD3j0MhIiKiUlJ0CCwqKgoRERFo2bIlWrVqhenTpyM7OxuDBw8GAAwcOBCPPfaYpvcmLy8Px44d0zy/dOkSkpOT4erqiscffxwA0KNHD3zyySeoWbMmGjdujIMHD+Lzzz/Hq6++quA3NZy2bQF/f+DcOWDDBrlLNBEREZWO4jtBx8fH47PPPkNqaioCAwMxY8YMhISEAADat28PPz8/LFmyBABw/vx5+Pv7FykjLCwM27dvBwDcunUL48aNw/r165Geng4fHx/069cP48ePh4ODQ4naZG47QT8oNhb46CO5Kuznn5VuDRERkXkozd9vxQOQOTL3AHTmDPD444CNDXDxIvDYY0q3iIiISHkWsQ8QlV2dOnIoTK0Gli9XujVERESWhwHIQhVMhl66FGAfHhERUekwAFmol14CHB2Bo0flafFERERUcgxAFqpiReC55+TzpUuVbg0REZFlYQCyYAXDYMuXA3fvKt0aIiIiy8EAZME6dQK8vIBr14AtW5RuDRERkeVgALJgdnZAwRmvHAYjIiIqOQYgC1dwQvwPPwDXryvdGiIiIsvAAGThAgKAZs2AvDxg1SqlW0NERGQZGIDKgYLJ0DwhnoiIqGQYgMqBl18GbG2BxETg1CmlW0NERGT+GIDKAW9voHNn+Zy9QERERP+NAaicKJgM/c038owwIiIiejgGoHLi2WcBDw95OvyOHUq3hoiIyLwxAJUTzs5A797yOYfBiIiIHo0BqBwpGAZbuxbIzla6NUREROaLAagcadsWqF0byMoC1q9XujVERETmiwGoHFGptL1AHAYjIiJ6OAagcmbAAPnvr78C//yjdGuIiIjMEwNQOVO7NtCuHSAEsGyZ0q0hIiIyTwxA5VDhYTAhlG4NERGR+WEAKodeeglwcgKOHQOSkpRuDRERkflhACqHPDyAnj3lc06GJiIiKooBqJwqOCF++XIgL0/p1hAREZkXBqByKjxcHpL677/A5s1Kt4aIiMi8MACVU3Z2QP/+8jmHwYiIiHQxAJVjBcNgP/wge4KIiIhIYgAqx5o2BQIDgbt3gVWrlG4NERGR+WAAKucKeoGWLlW6JUREROaDAaic69cPsLUF9u4FTpxQujVERETmgQGonPPyArp0kc+/+Ubp1hAREZkHxQPQrFmz4OfnBycnJ4SEhGDv3r0Pvffo0aPo1asX/Pz8oFKpMH369GLvu3TpEl555RVUqVIFzs7OaNq0Kfbv32/Eb2HeCobBvvkGUKuVbg0REZHyFA1Aq1atQlRUFGJjY3HgwAEEBASgc+fOSE9PL/b+nJwc1K5dG5MmTYK3t3ex99y4cQNt27aFvb09Nm/ejGPHjmHatGmoVKmSkb+N+erRA6hYEUhJAbZvV7o1REREylMJodxxmSEhIQgODkZ8fDwAQK1Ww9fXF6NHj0Z0dPQjP+vn54fIyEhERkbqXI+Ojsbu3buxc+fOErcjNzcXubm5mteZmZnw9fVFRkYG3N3dS/29zNEbbwBz58qDUjkhmoiIyqPMzEx4eHiU6O+3Yj1AeXl5SEpKQnh4uLYxNjYIDw/Hnj17ylzuxo0b0bJlS7z00kuoVq0amjdvjvnz5z/yM3FxcfDw8NA8fH19y1z/f/rnHyAtzXjlP0TBCfHffQdkZZm8eiIiIrOiWAC6du0a8vPz4eXlpXPdy8sLqampZS737NmzmD17NurWrYuff/4Zw4cPx5tvvomlj+j2iImJQUZGhuaRkpJS5vofacECoG5d4IMPjFP+I4SGAo8/DmRnA+vXm7x6IiIis6L4JGhDU6vVaNGiBT799FM0b94cr732GoYNG4Y5c+Y89DOOjo5wd3fXeRhFo0bAnTvA4sXAsWPGqeMhVCptLxCHwIiIyNopFoA8PT1ha2uLtAeGg9LS0h46wbkkqlevjkaNGulca9iwIS5evFjmMg2mTRugZ0+5FCsmxuTVDxgg//3tNzkhmoiIyFopFoAcHBwQFBSEhIQEzTW1Wo2EhASEhoaWudy2bdvi5MmTOtdOnTqFWrVq6dVeg4mLkzsTbtwI7Npl0qr9/ICwMEAI4NtvTVo1ERGRWVF0CCwqKgrz58/H0qVLcfz4cQwfPhzZ2dkYPHgwAGDgwIGIKdRTkpeXh+TkZCQnJyMvLw+XLl1CcnIyTp8+rbnn7bffRmJiIj799FOcPn0ay5cvx7x58zBy5EhFvmMRDRoAQ4bI52PGyDRiQgXDYF9/bfKqiYiIzIaiy+ABID4+Hp999hlSU1MRGBiIGTNmICQkBADQvn17+Pn5YcmSJQCA8+fPw9/fv0gZYWFh2F5og5tNmzYhJiYGf//9N/z9/REVFYVhw4aVuE2lWUZXJleuyBnJOTlyWdYLLxi+jofIzAS8vYHbt+XxGMHBJquaiIjIqErz91vxAGSOjB6AAGDcOGDiRKBePeCvvwB7e+PUU4z+/YHly4GRI4H7WzARERFZPIvYB8jqjRkDeHoCp04BCxeatOqCYbAVK4C8PJNWTUREZBYYgJTi7i57gQBgwgST7k4YHg5Urw5cvw78+KPJqiUiIjIbDEBKeuMNoHZtuTP0F1+YrFpbW+CVV+Tzr782WbVERERmgwFISQ4OwCefyOdTpgAPOQTWGAqGwX78Ebh2zWTVEhERmQUGIKX17g0EBckhsI8/Nlm1TZoALVoAd+8CK1earFoiIiKzwACkNBsb2fsDAHPmAIX2NDK2wnsCERERWRMGIHPw1FNAly7AvXsmPSi1Xz/Azg7Ytw84ftxk1RIRESmOAchcTJokTyxdvVomEhOoVg3o2lU+Zy8QERFZEwYgcxEQoF2a9d57JjunomAY7Ntvgfx8k1RJRESkOAYgc/Lxx3Jl2PbtwJYtJqmyRw+gYkXgn3+AbdtMUiUREZHiGIDMSa1awOjR8vn775ukS8bREejbVz7nMBgREVkLBiBzM3as7JI5ckSOS5lARIT897vvgFu3TFIlERGRohiAzE3lykBMjHw+bhxw547RqwwJAerWlYfTr1tn9OqIiIgUxwBkjkaPBmrUAFJSgJkzjV6dSsU9gYiIyLowAJkjZ2ftrtCffipPLTWyAQPkv9u2ARcvGr06IiIiRTEAmasBA+R5FTdvyj2CjKxWLaB9e7n63kRTj4iIiBTDAGSubG21wWfGDJN0yxRMhl661GTbEBERESmCAcicdesGhIUBubnA+PFGr65XLzn6duoUsHev0asjIiJSDAOQOVOptAelfv01cPiwUatzcwNeeEFbHRERUXnFAGTuWrUCXnpJjklFRxu9uoJhsBUrZMcTERFRecQAZAk+/VQe2755s9HPq3jqKeCxx4AbN4AffzRqVURERIphALIEjz8OvP66fP7ee4BabbSqbG21Z7IuXWq0aoiIiBTFAGQpxo0DXF2B/fuBtWuNWlXBnkA//QRcvWrUqoiIiBTBAGQpvLyAd9+Vz8eOBfLyjFZV48ZAUBBw7x6wcqXRqiEiIlIMA5AleecdGYTOnAHmzTNqVYX3BCIiIipvGIAsiasrEBsrn3/0EZCZabSq+vaV866TkoCjR41WDRERkSIYgCzN0KFAvXpycs7UqUarpmpVoHt3+Zx7AhERUXnDAGRp7O3lsngAmDYNuHLFaFUVnBD/7bdAfr7RqiEiIjI5BiBL9MILQEgIkJMjh8KMpHt3oFIl4PJl4LffjFYNERGRyTEAWaLCR2TMnw+cPGmUahwdgX795HNOhiYiovKEAchSPfkk8Mwzcmxq7FijVVMwDLZuHXDrltGqISIiMimzCECzZs2Cn58fnJycEBISgr2POIr86NGj6NWrF/z8/KBSqTB9+vRHlj1p0iSoVCpERkYaoeUKmzQJsLGR6WTPHqNU0aoVUL8+cPu20fdfJCIiMhnFA9CqVasQFRWF2NhYHDhwAAEBAejcuTPS09OLvT8nJwe1a9fGpEmT4O3t/ciy9+3bh7lz56JZs2ZGar3CGjcGBg2Sz997Tx6YamAqlbYXiKvBiIiovFA8AH3++ecYNmwYBg8ejEaNGmHOnDmoUKECFi1aVOz9wcHB+Oyzz9C3b184Ojo+tNysrCz0798f8+fPR6VKlR7ZhtzcXGRmZuo8LMaHHwJOTsCuXcAPPxilildekUFo+3bgwgWjVEFERGRSigagvLw8JCUlITw8XNsgGxuEh4djj55DOiNHjkT37t11yn6YuLg4eHh4aB6+vr561W1SNWoABcN70dHy/AoDq1kT6NBBPv/mG4MXT0REZHKKBqBr164hPz8fXl5eOte9vLyQmppa5nJXrlyJAwcOIC4urkT3x8TEICMjQ/NISUkpc92KeP99oHJl4Phxoy3XKjwMZoSRNiIiIpNSfAjM0FJSUvDWW29h2bJlcHJyKtFnHB0d4e7urvOwKBUrAh98IJ+PHy/3BzKwXr2AChWAv/8GEhMNXjwREZFJKRqAPD09YWtri7S0NJ3raWlp/znB+WGSkpKQnp6OFi1awM7ODnZ2dtixYwdmzJgBOzs75JfXLY1HjgRq1ZK7Fn75pcGLd3WVIQicDE1EROWAogHIwcEBQUFBSEhI0FxTq9VISEhAaGhomcp8+umnceTIESQnJ2seLVu2RP/+/ZGcnAxbW1sDfgMz4ugITJwon0+aBFy7ZvAqCobBVq4EcnMNXjwREZHJKD4EFhUVhfnz52Pp0qU4fvw4hg8fjuzsbAwePBgAMHDgQMTExGjuz8vL0wSbvLw8XLp0CcnJyTh9+jQAwM3NDU2aNNF5uLi4oEqVKmjSpIli39MkXn4ZCAyUp8R/8onBi+/QQc65vnnTaAvOiIiITELxANSnTx9MnToV48ePR2BgIJKTk7FlyxbNxOiLFy/iSqEDPy9fvozmzZujefPmuHLlCqZOnYrmzZtj6NChCn4LM2FjA0yeLJ/PmgWcO2fQ4m1t5ZJ4cBiMiIgsnEoIrul5UGZmJjw8PJCRkWF5E6IBoGNH4Ndfgf795VHuBnT8ONCoEWBnB1y6BFSrZtDiiYiIyqw0f78V7wEiI5g0Sf67bBlw8KBBi27YEAgOltsNrVhh0KKJiIhMhgGoPAoK0h7j/v77Bi++YDL0zJlAVpbBiyciIjI6BqDy6pNPAHt7YOtW+TCgV14BHnsMOHMGeP11boxIRESWhwGovPL3B0aMkM/ffx9Qqw1WdMWKwKpVclL08uXA/PkGK5qIiMgkGIDKs//9D3B3l/OADDxhp21boOCkkTffBJKTDVo8ERGRUTEAlWeento5QP/7n8F3L3znHeCZZ2SxL70ktx8iIiKyBAxA5V1kJFC9OnD+PDB7tkGLtrGRZ6/WrAmcPg0MHcr5QEREZBkYgMq7ChWADz+UzydOBDIyDFp85crA6tVyvvWaNcBXXxm0eCIiIqNgALIGgwcDDRoA//6r3SnagEJCgClT5PO33wb27zd4FURERAbFAGQN7Oy0myNOny63cDawt94Cnn8euHtXzge6ccPgVRARERkMA5C1ePZZuXTr9m0gNtbgxatUwKJFcvX9+fOy04nzgYiIyFwxAFkLlUo7TrV4MXDsmMGrqFhRzgNycAC+/152NhEREZkjBiBr0qaNHKdSq4HoaKNUERQEfPGFfP7ee0BiolGqISIi0gsDkLWJi5NbOP/wA7Bzp1GqGD4c6NNHHpjau7ece01ERGROGICsTf36wJAh8vl77xlloo5KBcybB9StC6SkABERBj2Jg4iISG8MQNZowgS5P1BiIrB+vVGqcHeX84EcHYEffwSmTjVKNURERGXCAGSNqlcHoqLk85gYuXbdCAICgJkz5fOxY4Fdu4xSDRERUakxAFmrMWPkWWGnTgELFxqtmqFDgVdeAfLz5bygq1eNVhUREVGJMQBZK3d3YPx4+XzCBCAryyjVqFTyCLIGDYDLl2UY4nwgIiJSGgOQNXv9daBOHSAtTbt23QhcXeV8IGdn4JdfgE8/NVpVREREJcIAZM0cHIBPPpHPp0wB0tONVlWTJtrD6GNjgW3bjFYVERHRf2IAsnYvvSR3L8zKAj7+2KhVRUTIIzLUaqBfPyA11ajVERERPRQDkLWzsdEekTFnDnD6tFGri4+XvUFpacDLL8vJ0URERKbGAETAU08BXbrIrZs/+MCoVVWoIOcDubjIYbCPPjJqdURERMUqVQCaMmUKbt++rXm9e/du5Obmal7funULI0aMMGwLyTQmT5ZLtlavBvbtM2pVDRrInaIBOer2yy9GrY6IiKiIUgWgmJgY3Lp1S/O6a9euuHTpkuZ1Tk4O5s6da9gWkmk0awYMGCCfG+mIjMJeflkuQhNCLo0v9D8jIiIioytVABIP/FF88DVZuI8/lmdXbN8ObNli9OqmTwcCA+XmiP36yRE4IiIiU+AcINKqWRMYNUo+f/99o89QdnKS84Hc3OTB9OPGGbU6IiIiDQYg0jV2LFCxInDkCPDtt0av7vHHtSdxTJoE/PST0askIiKCXWk/sGDBAri6ugIA7t27hyVLlsDT0xO4PwmaLFzlyvKA1Pffl10yffrIrhojeukl2fEUHy+nIR08KDujiIiIjEUlSjGRx8/PDyqV6j/vO3funL7tUlRmZiY8PDyQkZEBd3d3pZtjerdvA/XqAf/8I/cIGjPG6FXm5gJPPAHs3w+0bg3s2CE3qiYiIiqp0vz9LtUQ2Pnz53Hu3Ln/fJTWrFmz4OfnBycnJ4SEhGDv3r0Pvffo0aPo1auXJoxNnz69yD1xcXEIDg6Gm5sbqlWrhp49e+LkyZOlbpfVcnbW7gr96afA9etGr9LRUa7Ar1gRSEyUnVBERETGovgcoFWrViEqKgqxsbE4cOAAAgIC0LlzZ6Q/5FyqnJwc1K5dG5MmTYK3t3ex9+zYsQMjR45EYmIitm7dirt376JTp07Izs428rcpRwYMAJo2BW7elJNzTMDfH1i8WD7//HPg++9NUi0REVkjUQp//PGH+OGHH3SuLV26VPj5+YmqVauKYcOGiTt37pSmSNGqVSsxcuRIzev8/Hzh4+Mj4uLi/vOztWrVEl988cV/3peeni4AiB07dhT7/p07d0RGRobmkZKSIgCIjIyMUn2XcufHH4UAhHB0FOLCBZNVGxUlq61YUYizZ01WLRERWbiMjIwS//0uVQ/QRx99hKNHj2peHzlyBEOGDEF4eDiio6Pxww8/IC4ursTl5eXlISkpCeHh4ZprNjY2CA8Px549e0rTtEfKyMgAAFSuXLnY9+Pi4uDh4aF5+Pr6Gqxui9a1KxAWJifojB9vsmonTZLzgG7eBHr3ltUTEREZUqkCUHJyMp5++mnN65UrVyIkJATz589HVFQUZsyYgdWrV5e4vGvXriE/Px9eXl461728vJBqoKPC1Wo1IiMj0bZtWzRp0qTYe2JiYpCRkaF5pKSkGKRui6dSaQ9K/fpr4PBhk1Rrbw+sWiUXpO3fb5I52EREZGVKFYBu3LihE1Z27NiBrl27al4HBwebXXgYOXIk/vrrL6xcufKh9zg6OsLd3V3nQfe1aiXXqQsBREebrNqaNWXmAoCZM4G1a01WNRERWYFSBSAvLy/NKq+8vDwcOHAArVu31rx/69Yt2Nvbl7g8T09P2NraIi0tTed6WlraQyc4l8aoUaOwadMmbNu2DTVq1NC7PKv16aeAnR2webM8wt1EuneX2xEBwKuvAqdPm6xqIiIq50oVgLp164bo6Gjs3LkTMTExqFChAtq1a6d5//Dhw6hTp06Jy3NwcEBQUBASEhI019RqNRISEhAaGlqapukQQmDUqFFYv349fvvtN/j7+5e5LLq/XfPrr8vn770HqNUmq3riRKBdO+DWLdkRdeeOyaomIqJyrFQB6OOPP4adnR3CwsIwf/58zJs3Dw6FdqtbtGgROnXqVKoGREVFYf78+Vi6dCmOHz+O4cOHIzs7G4MHDwYADBw4EDGFNoXJy8tDcnIykpOTkZeXh0uXLiE5ORmnC3UPjBw5Et9++y2WL18ONzc3pKamIjU1Fbdv3y5V26iQ8eMBV1c5KceE41F2dsCKFUDVqkByMhAZabKqiYioHCvVTtAFMjIy4OrqCltbW53r169fh5ubW6mGwQAgPj4en332GVJTUxEYGIgZM2YgJCQEANC+fXv4+flhyZIlwP3NGIvr0QkLC8P27dvll3rIbtWLFy/GoEGD/rM9Vr8T9MN89BEQGwvUqQMcO2bSrZp/+QXo0kVORVq+XJ4eT0REVFhp/n6XKgC9+uqrJbpv0aJFJS3SLDEAPURWlhwOS0uTM5MLTo43kfHj5QbVLi6yI6pBA5NWT0REZs5oAcjGxga1atVC8+bN8aiPrV+/vnQtNjMMQI8wezYwYoQckzp9GjDhzyc/H+jYUc7DbtIE+PNPoEIFk1VPRERmzmgBaOTIkVixYgVq1aqFwYMH45VXXnno5oKWjAHoEe7elenj1Cng3XeBzz4zafWpqUBgoOyEevVVYOFCk1ZPRERmzGiHoc6aNQtXrlzBe++9hx9++AG+vr7o3bs3fv7550f2CFE5Ym8PFOz2PXWqXB1mwq2avb3lpGgbG2DRImDpUpNVTURE5UipD0N1dHREv379sHXrVhw7dgyNGzfGiBEj4Ofnh6ysLOO0kszL88/LEKRSAfPmAR06AJcvm6z6Dh2ACRPk8+HDgUKnsxAREZWIXqfB29jYQKVSQQiB/Px8w7WKzJtKJXeF/vFHoGJFYM8eICgI+OMPkzXhgw+ATp2A27fl/kDM3kREVBqlDkC5ublYsWIFOnbsiHr16uHIkSOIj4/HxYsX4erqapxWknnq2hXYtw9o3FhOzmnfHpg71yRV29gA334L+PgAx4/LniCOwhIRUUmVKgCNGDEC1atXx6RJk/DMM88gJSUFa9asQbdu3WBjo1dnElmqxx8HEhOBF1+UE6TfeAN47TWTzAuqWhVYuRKwtZVhiBOiiYiopEq9DL5mzZpo3rz5QzcbBIB169YZqn2K4CqwMhACmDwZGDtWPm/dGvjuO9lFY2STJ8sROScnmcUCAoxeJRERmaHS/P22K03BAwcOfGTwIStWMC8oMFBu05yYKOcFrV0LtG1r1KrHjAF+/x346Sc5H2j/fpNuT0RERBaoTEdhlHfsAdLTmTNAz57AX3/JZfMzZsjl8kYMz//+CzRvDqSkAH36yKXyzOpERNbFaPsAEZVInTpyZVjv3nJe0PDhwLBhRp0XVKUKsHq1PDx11Sq5YTUREdHDMACRcbi6yhnKkyfLJVsLFwJhYcClS0arsnVrWR0AvP02kJRktKqIiMjCMQCR8ahUwHvvAZs3A5UqycO7goKAXbuMVuXbbwPPPQfk5ckOqJs3jVYVERFZMAYgMr5OneR+QU2bykO8OnQAvvrKKBv3qFTA4sWAnx9w9qw8L4yz3IiI6EEMQGQahecF3bsHjBwJDB0K3Llj8KoqVQLWrAEcHID16+UcbCIiosIYgMh0XFzkvKApU7SnmYaFAf/8Y/CqWrYEpk2Tz8eMkaNvREREBRiAyLRUKplItmyRXTV798p5QTt3GryqkSPlvkB378qOp+vXDV4FERFZKAYgUkbHjnLHwmbNgPR04KmngFmzDDphR6UC5s+Xp3VcvAhERABqtcGKJyIiC8YARMqpXVueIN+3r5wXNGoUMGSIQecFeXjI+UCOjsCmTdphMSIism4MQKQsFxdg+XLgs8/kvKDFi4Enn5RbOhtIYCDw5ZfyeUwMsHu3wYomIiILxQBEylOpgHffBX7+GahcWS6Zb9lSHvBlIK+9Brz8MpCfL4/KuHrVYEUTEZEFYgAi8xEeLucFBQTIeUFPPw3ExxtkXpBKBcydC9SvLzejHjCA84GIiKwZAxCZF39/OS+oXz85L2j0aGDwYIPMC3J1lfOBnJ1lZ9OkSQZpMRERWSAGIDI/FSoAy5bJGcs2NsDSpUC7dgaZF9S0qVxsBgDjxgHbt+vfXCIisjwMQGSeVCogKgr45Rd51Pv+/XK/oB079C568GDtkvh+/eTpHEREZF0YgMi8Pf20DD+BgXLm8tNPy7Mt9JwXNGsW0LgxkJoqD0+9eNFgLSYiIgvAAETmz89Prl3v318u43rrLWDQIOD27TIX6eIi5wO5ucljMpo2BZYs4cGpRETWggGILEOFCsA33wCffw7Y2gJffy3nBenRddOwIZCUBISGApmZcmisZ0/ZK0REROUbAxBZDpUKePtt7bygpCQ5L0iPmcx168pjyCZNkqfHb9wINGkCrF1r0JYTEZGZYQAiy/PUUzL8NG8OXLsm9w/68ssyj1/Z2gLvv6/dgujff+Uhqv378wBVIqLyigGILFOtWnJe0CuvyHlBkZFyaZce84KaNpWH03/wgVx9v3y57A3avNmgLSciIjNgFgFo1qxZ8PPzg5OTE0JCQrB3796H3nv06FH06tULfn5+UKlUmD59ut5lkoVydpZzgaZPl90433wDtG0LXLhQ5iIdHICJE+VejPXrA1euAN26yaM0bt0yaOuJiEhBigegVatWISoqCrGxsThw4AACAgLQuXNnpKenF3t/Tk4OateujUmTJsHb29sgZZIFU6nkqrCtWwFPT+DgQXmO2LZtehUbEgIcOCCLBoD58+XwmAGPJyMiIgWphFB24W9ISAiCg4MRHx8PAFCr1fD19cXo0aMRHR39yM/6+fkhMjISkZGRepWZm5uL3NxczevMzEz4+voiIyMD7u7uBvqmZHQXLwLPPy+Ti62tPGE+MlKGJD1s2yZXiF24IIuKjAQ++UR2QBERkfnIzMyEh4dHif5+K9oDlJeXh6SkJISHh2sbZGOD8PBw7Nmzx2RlxsXFwcPDQ/Pw9fUtU92ksJo1gV275Emn+flyJ+kBA4CcHL2K7dABOHwYGDJEzrP+4gu5+Gz/foO1nIiITEzRAHTt2jXk5+fDy8tL57qXlxdSy7gZS1nKjImJQUZGhuaRYoAzp0ghzs7y7LAvv5S9QMuWAU88AZw/r1ex7u7AggXApk2Atzdw/DjQujUQGwvcvWuw1hMRkYkoPgfIHDg6OsLd3V3nQRZMpQLefBP49VegalXtvKDfftO76O7dgb/+Avr0kZ1MH30k5wv99ZdBWk5ERCaiaADy9PSEra0t0h44jTItLe2hE5yVKJMsVPv22kNU//0X6NhR7iSt57S3KlWAlSvlo3Jlma+CguSUo/x8g7WeiIiMSNEA5ODggKCgICQkJGiuqdVqJCQkIDQ01GzKJAtWs6bc6nngQHn8+zvvyL2D9JwXBMheoL/+kr1CeXnAe+/JzHXmjEFaTkRERqT4EFhUVBTmz5+PpUuX4vjx4xg+fDiys7MxePBgAMDAgQMRExOjuT8vLw/JyclITk5GXl4eLl26hOTkZJw+fbrEZZKVcXaWJ53OmCHnBS1fLvcLOndO76KrVwd++EHOD3J1lXOwmzUDZs/mwapERGZNmIGZM2eKmjVrCgcHB9GqVSuRmJioeS8sLExERERoXp87d04AKPIICwsrcZn/JSMjQwAQGRkZBvqGZDa2bxeialUhACEqVxZi61aDFX3unBDt28uiASE6dhQiJcVgxRMR0X8ozd9vxfcBMkel2UeALFBKCvDCC3J+kI2N3Pr53XcBe3u9i1argZkzgeho4M4dwMNDvn7lFb23IyIiov9gMfsAESnC11fOCxo0SCaWsWPlLOY//tC7aBsbuXt0cjLQqhWQkSGnH/XqBXAjciIi88EARNbJyQlYtAhYvFgu6zpyRM4LGjpUrhjTU/368qzWiRNlx9L69fJg1fXrDdJ6IiLSEwMQWS+VSvYCnTght3kGgIULZXpZvFjvWcx2dvJk+b175UnzV6/KkbeBA4GbNw3zFYiIqGwYgIg8PeUyrl27ZDfNv/8Cr74KPPmkQXY4DAwE9u2T84JsbOSh9U2byvNbiYhIGQxARAXatpUHqU6ZAlSoIANR8+bA++8D2dl6Fe3oCMTFyalHjz8O/PMP0KkTMGKE3kUTEVEZMAARFWZvD4wZIw/76tkTuHdPBqJGjYCNG/Uuvk0bOUF61Cj5evZsICBAzhciIiLTYQAiKk7NmnLG8saNQK1awMWLwHPPyceFC3oV7eIil8Zv3SoXpJ05A7RrJzua7twx2DcgIqJHYAAiepQePYCjR+UEHjs7GYgaNZK9QnoeAx8eLhefDRok51tPmSLPbD1wwGCtJyKih2AAIvovLi5yAk9yspwYnZMju2uaN5eTevTg4SEXnH3/PVCtmsxaISHylHk98xURET0CAxBRSTVuDGzfLs8V8/SUaeXJJ+WKsWvX9Cr62WdlcS++KKcdxcbK+ULHjxus9UREVAgDEFFpqFRARARw8iQwbJi8tnix3DtowQK5s3QZeXoCq1cDy5YBFSvKkzqaNwc+/1yvYomIqBgMQERlUbkyMG+ePD6jWTPg+nUZiNq1kxN7ykilAl5+WW4/1KULkJsLvPMO0KGDQQ6vJyKi+xiAiPQRGgokJQHTpsm5Qn/8Ibtt3n0XyMoqc7GPPQb89BMwd64s9vffZc6aP1/vDaqJiIgBiMgA7OyAqCh5pEavXkB+vgxEDRvKpfRlTCwqFfDaa8Dhw7JjKStLvu7eHbh82eDfgojIqjAAERlKjRrA2rXAjz8C/v5yu+cXXpBL6fUYv6pdG9i2DZg6Ve4ovXmzPLFjxQr2BhERlRUDEJGhdesmJ/F88IHcWfrHH+UKsrg4IC+vTEXa2sq5QAcOAEFBwI0bcq5Qnz56L0AjIrJKDEBExlChAjBxInDoENC+PXD7NjB2rDwZdceOMhfbqBGwZw/w4Ydy5G3NGtkb9MMPBm09EVG5xwBEZEwNGwK//SaPgK9aVW7s0769XEqfnl6mIu3tgfHjgcREGYjS0uQ+Qi+/DBw8aPBvQERULjEAERmbSgW88orcO+iNN+Trr78GGjSQS+nLuMlPUJBcgDZmjCxyxQqgRQu5N+PatXJDRSIiKh4DEJGpVKokj3/fs0cOhd24Abz+OtC2rRwqKwMnJ3mG2N69QN++clhs507gpZfkPOy4OM4RIiIqDgMQkamFhAD79gHTpwNubnIsKyhILqW/datMRbZsKXuALlwAxo2To23//COnHdWoAQwZIo8yIyIiiQGISAl2dsBbb8k5Qb17y72DvvhCzhn67rsyr2/38ZEHqV68CCxdKnNVbi6waJHcn5HDY0REEgMQkZIeewxYtQrYsgWoUwe4dEmeiNq9O3D2bJmLdXICBg6UHU27dxcdHqtdG5g0icNjRGS9GICIzEHnzvIMsfHjAQcHudth48ZyKX1ubpmLVankqfIrVgDnzwP/+58cHktJAWJiAF9fDo8RkXVSCcG9ZB+UmZkJDw8PZGRkwN3dXenmkLU5dQoYMQJISJCv69eXk6c7dDBI8XfuyFPnv/xSbqxY4MkngTffBJ57TvYWERFZmtL8/WYPEJG5qVcP2LoVWLYM8PKSy+efekoupU9L07v4guGx/fvl8FifPnKn6d9/l6NvHB4jImvAAERkjlQqubPhiRPAyJHy9bJl2t6g/HyDVNGmDbBypRwe++CDosNjQ4eWeYU+EZFZYwAiMmcVKwLx8cCff8olXRkZcnisTRvd8Ss91aghpxtdvAgsWSJXjN25AyxcKLcsat9eLk7j6jEiKi8YgIgsQXCwDEEzZwLu7nLnw+BguZQ+M9Ng1Tg5yVM6kpKAXbvkCn1bW3l8WcHw2OTJwL//GqxKIiJFMAARWQpbW2DUKDks1revPEJjxgx5pMbq1WXeO6g4KpXcoHrVKu3wmKenHB6LjpY9RsOGAYcPG6xKIiKTYgAisjTVq8t17b/8AtStC1y5Imcyh4XJQ1ezsgxaXcHwWEoKsHixdnhswQIgIEAOj61bx+ExIrIsZhGAZs2aBT8/Pzg5OSEkJAR79+595P1r1qxBgwYN4OTkhKZNm+Knn37SeT8rKwujRo1CjRo14OzsjEaNGmHOnDlG/hZEJtaxo+yCmTABcHSUuxwOHAh4ewODBslT6Mt40GpxnJxkscUNj/XqJfdx5PAYEVkMobCVK1cKBwcHsWjRInH06FExbNgwUbFiRZGWllbs/bt37xa2trZiypQp4tixY+J///ufsLe3F0eOHNHcM2zYMFGnTh2xbds2ce7cOTF37lxha2srvv/++xK1KSMjQwAQGRkZBvueREZ14YIQH30kxOOPCyEHw+TD11eIsWOFOHHCKNWmpMjiq1TRVunkJMTQoUIcOmSUKomIHqo0f78V3wgxJCQEwcHBiI+PBwCo1Wr4+vpi9OjRiI6OLnJ/nz59kJ2djU2bNmmutW7dGoGBgZpeniZNmqBPnz4YN26c5p6goCB07doVEydOLFJmbm4ucgvttpuZmQlfX19uhEiWRwh52vzXX8sJPDdvat8LCZE9RH37ApUrG7Ta27flcvoZM3R3lW7fHhg9Gnj2WW6uSETGZzEbIebl5SEpKQnh4eHaBtnYIDw8HHv27Cn2M3v27NG5HwA6d+6sc3+bNm2wceNGXLp0CUIIbNu2DadOnUKnTp2KLTMuLg4eHh6ah6+vr8G+I5FJFWzuM2eOnBu0ejXwzDNyrOrPP+WeQt7ecszq+++BvDyDVOvsDAweLFfmF5w3ZmsLbN+uHR6bMgW4ft0g1RER6U3RAHTt2jXk5+fDy8tL57qXlxdSU1OL/Uxqaup/3j9z5kw0atQINWrUgIODA7p06YJZs2bhySefLLbMmJgYZGRkaB4pKSkG+X5EinJykknkhx/kIatffCE39bl7V85a7tlTHsb65ptyW2gDdAarVMATT8jcde4cMHYsUKWK3F/o/fflhOrXXpPHnhERKcksJkEb2syZM5GYmIiNGzciKSkJ06ZNw8iRI/Hrr78We7+joyPc3d11HkTlipcXEBkJHDwot3Z+5x3ZE3TtmtxbKDgYaNJEzmK+dMkgVfr6Ap98IlePLVoks9ft28D8+UCzZvJos/XrDbKpNRFRqSkagDw9PWFra4u0B843SktLg7e3d7Gf8fb2fuT9t2/fxtixY/H555+jR48eaNasGUaNGoU+ffpg6tSpRvw2RBaiWTNg6lSZTDZvlnOCnJyAY8fkJj++vkCnTsC33wLZ2XpXV3h47PffdYfHXnhBDo999hmHx4jItBQNQA4ODggKCkJCwanX9ydBJyQkIDQ0tNjPhIaG6twPAFu3btXcf/fuXdy9exc2NrpfzdbWFmoDLgkmsnh2dkCXLnJPodRUubFPu3ZyKGzrVmDAANlLNHgwsG2b3kvqVSpZfMHwWEyMHB67cAF47z3t8Bg3VyQikzDFsrRHWblypXB0dBRLliwRx44dE6+99pqoWLGiSE1NFUIIMWDAABEdHa25f/fu3cLOzk5MnTpVHD9+XMTGxhZZBh8WFiYaN24stm3bJs6ePSsWL14snJycxFdffVWiNnEZPFm1M2eEmDBBiNq1dZfU16wpxAcfCHHypMGqyskRYuFCIQICdKtq1UqIefOEyMw0WFVEZAVK8/db8QAkhBAzZ84UNWvWFA4ODqJVq1YiMTFR815YWJiIiIjQuX/16tWiXr16wsHBQTRu3Fj8+OOPOu9fuXJFDBo0SPj4+AgnJydRv359MW3aNKFWq0vUHgYgIiGEWi3Erl1CDBsmhLu7bkJp3VqIr74S4t9/DVbV778L8eKLQtjZaatxcRHi1VeF2L1b3kNE9CgWtQ+QOSrNPgJEVuH2bWDjRrm/0M8/a2cuOzgAPXrIE1S7dAHs7fWuKi1NnuixYAFw8qT2esOGwJAhciujqlX1roaIyqHS/P1mACoGAxDRI6SmAsuXyzB06JD2etWqQL9+Mgw1by4n/ehBCGD3bmDhQrmn4+3b8rq9PfDcczIMdewoJ1QTEYEBSH8MQEQldOiQDELLlsmumwKNG8sg1L8/4OOjdzUZGXKn6YULgX37tNd9fYFXX5XztGvV0rsaIrJwDEB6YgAiKqV79+Tp9F9/DWzYABQcLWNjA4SHyzDUsydQoYLeVR06JIPQt98CN27IayqV7A0aMkT2Djk66l0NEVkgBiA9MQAR6eHmTWDNGmDpUjmGVcDNTW4CNHCgXA9vo98uHHfuyI0UFy4ECu+MUaWKrGLIENkRRUTWgwFITwxARAZy5oyc0fz113LznwJ+fnKfoQEDgLp19a7m7Fm52/TixcDly9rrrVvLINSnj8xfRFS+MQDpiQGIyMDUatkb9PXXcifEzEzte23ayC6b3r2BSpX0qubePblIbeFCeQTavXvyuouL3PB6yBAZivScn01EZooBSE8MQERGdPu2PIl+6VI5b6hgh2lHR+DZZ+V8oc6d5U7VekhNlXlr4ULg1Cnt9UaNgKFDZeeTp6ee34WIzAoDkJ4YgIhM5MoVuYJs6VLgr7+01318gEGD5BKvOnX0qkIIYNcuua/QmjW6y+l79pRhKDxc7ylJRGQGGID0xABEZGJCyOVdS5fKOUP//qt976mnZEp5/nl5aKseMjLk0WcLFgBJSdrrNWtql9PXrKlXFUSkIAYgPTEAESkoN1fuOr1ggTyUteD/RFWqBLzyipzIExCgdzXJydrl9DdvymsqFdCpk8xbzz4rN7omIsvBAKQnBiAiM3HhglzatXgxcPGi9nrLljKl9O0LeHjoVcXt23I5/YIF8tD7Ap6e2uX0jRrpVQURmQgDkJ4YgIjMTH4+8OuvMqV8/z1w96687uwsV48NHQq0bav38q4zZ+Ry+iVLdJfTh4bKKnr3Blxd9fwuRGQ0DEB6YgAiMmNXr2pPSz1+XHu9fn3taaleXnpVce8esGWLrGLTJu3Zr66ustNp6FCgVSsupycyNwxAemIAIrIAQgCJiTKlrFoFZGfL63Z28oT6oUPlcno9T0u9ckUup1+wADh9Wnu9cWNZxSuvcDk9kblgANITAxCRhbl1S4agBQuAP//UXn/sMbm069VXAX9/vaoQAti5U7uc/s4ded3BQbuc/umnuZyeSEkMQHpiACKyYH/9JZd3ff01cP269np4uBwi69lT7+X0N29ql9MfOKC9XquWdjm9r69eVRBRGTAA6YkBiKgcyM2VE6YLltMXqFxZjlsNHQo0bap3NQcPapfTZ2TIawWn07/0kjydvmpVvashohJgANITAxBROXP+vFxKv2gR8M8/2uutWsleob59AT3/W799G1i3Tuat7du1121sgCefBF54QXY+sWeIyHgYgPTEAERUTuXny96gguX0BaelVqggj4wfMkQezqrn8q7Tp+WUpHXrdIfIcD9zvfCCfNStq1c1RPQABiA9MQARWYH0dO1y+hMntNcbNNCellqtmt7VnD8vN1pctw7YvVu7sTUANGkig1CvXnI0jsvqifTDAKQnBiAiKyIE8McfciLPqlVATo68bm8vz8MYOlRO6NFzOT3un1D//fcyDP32m7YDCpBnvhb0DLVqxdVkRGXBAKQnBiAiK5WZCaxcKcPQ3r3a676+cmnX4MGAn59BqrpxA/jhBxmGfv5Zu6weAHx85NmvL7wg5w/Z2RmkSqJyjwFITwxARITDh7XLuwqW06tUcjn90KFyeZejo0GqysqSO0+vWyd3nr51S/telSqyI+qFF2TVeq7gJyrXGID0xABERBp37gAbNsi5QgkJ2utVqsh5QkOGyMk8BpKbK6tZt05W+++/2vfc3IDu3WUY6tqV55IRPYgBSE8MQERUrLNntafTX7qkvR4SInuF+vSRKcVA7t2Tu0+vWycnUheu0tFRnvTxwgvy5I/KlQ1WLZHFYgDSEwMQET1Sfr6cuLNggZzIUzCb2cVF7ik0ZAjQurVBl3Wp1cC+fTIMffedPLm+gK0t0KGDdq+h6tUNVi2RRWEA0hMDEBGVWFqa9rTUU6e01+vWBcLCgNBQubdQ/foGC0RCyBM/1q2Tj8OHte+pVLK6F16QE6n1PAKNyKIwAOmJAYiISk0IudHPggXA6tVya+jCKleWvUJt2shHcLDBJvGcPq0NQ4XPggWA5s21y+sbNuReQ1S+MQDpiQGIiPSSkQHs2CH3F9qzRy6pL7zOHffPyAgIkGGooJfIz0/vhPLPP3Ly9Lp1sglqtfa9+vW1YSgoiGGIyh8GID0xABGRQeXlAYcOaQPRH38AKSlF7/P21g1ELVrote792jVg40YZhrZulc0oULOmNgy1aWOQfR6JFMcApCcGICIyun/+0YahP/6Qx8rfvat7j4ODDEGFQ5GPT5mqy8wEfvpJTqD+6SfthteAPPGjZ08Zhjp0kNUSWaLS/P02i83WZ82aBT8/Pzg5OSEkJAR7C+/AWow1a9agQYMGcHJyQtOmTfHTTz8Vuef48eN49tln4eHhARcXFwQHB+PixYtG/BZERKVQowbw0kvAF1/IiTsZGXLN++TJMo1Uqya7bBITgc8/l/c+9pgcJuvXD5g5E0hKKhqaHsLdXS5QW7NG9gxt2AAMHAhUrCiPRZs3D+jSRVY7YIBcdl84JBGVN4r3AK1atQoDBw7EnDlzEBISgunTp2PNmjU4efIkqhVzEOEff/yBJ598EnFxcXjmmWewfPlyTJ48GQcOHECT+5uRnTlzBq1atcKQIUPQr18/uLu74+jRo2jdunWxZT6IPUBEpDgh5L5DhXuJjhzRndSD+yfZBwdrJ1e3bg14epa4mrt3ge3btXsNpaVp33N2lnsNPfUU0K6dPLCVQ2VkzixqCCwkJATBwcGIj48HAKjVavj6+mL06NGIjo4ucn+fPn2QnZ2NTZs2aa61bt0agYGBmDNnDgCgb9++sLe3xzfffFOiNuTm5iI3N1fzOjMzE76+vgxARGRebt2SE6oL5hLt2QPcvFn0vnr1dIfNGjUq0emq+fmyw6lgr6ELF3Tf9/AAnnhChqEnn5QTqTlcRubEYgJQXl4eKlSogLVr16Jnz56a6xEREbh58ya+//77Ip+pWbMmoqKiEBkZqbkWGxuLDRs24NChQ1Cr1fDw8MB7772HXbt24eDBg/D390dMTIxOHYVNmDABH374YZHrDEBEZNbUauDECd1eohMnit7n4SF3qy7oJQoJkWNijyCEnJa0ebMcmfvjD90zynC/h6h1axmG2rWTz11cDPwdiUqhNAFI0TOGr127hvz8fHh5eelc9/Lywoni/iMGkJqaWuz9qampAID09HRkZWVh0qRJmDhxIiZPnowtW7bghRdewLZt2xAWFlakzJiYGERFRWleF/QAERGZNRsb2bvTqJHcfRqQB7cmJmp7iQrmF/3yi3zg/m6JTZro9hI9/rjOuniVSs6/btFCvr53Ty5k+/13GYh+/12eU7Ztm3wA8tT6li21PURt2wKVKpn8p0JUIooGIGNQ3x8ff+655/D2228DAAIDA/HHH39gzpw5xQYgR0dHOBroVGciIkVVrgx06yYfuJ9cjhzR7SU6d05eO3IEmDtX3ufpqQ1DbdrIJFOhgqZYOzs55BUUBLz9tuwhOn5cG4Z+/10ubEtMlI/PPpMhqmlTGYYKeom8vRX6uRA9QNEA5OnpCVtbW6QVnnUHIC0tDd4P+a/E29v7kfd7enrCzs4OjRo10rmnYcOG2LVrl8G/AxGRWbOzk9tBN28OjBghr6WmagPRnj3A/v1yadgPP8hHwecCA2U3TufOQPv2cszrPpVK2/n0+usyEF24oNtDdOqUPKbj8GHg/jRP1K2r7SF68kmD7P1IVCZmMQm6VatWmDlzJnC/B6dmzZoYNWrUQydB5+Tk4IeC/0gBtGnTBs2aNdNMgm7Tpg3q1KmjMwn6+eefh7OzM5YvX/6fbeIqMCKyKrm5csJP4V6iy5d173FykpsEde0qe5fq1PnPYlNTgV27tKHo0CEZlAp77DHdHqKGDUs0X5uoWKX6+y0UtnLlSuHo6CiWLFkijh07Jl577TVRsWJFkZqaKoQQYsCAASI6Olpz/+7du4WdnZ2YOnWqOH78uIiNjRX29vbiyJEjmnvWrVsn7O3txbx588Tff/8tZs6cKWxtbcXOnTtL1KaMjAwBQGRkZBjhGxMRmTm1Wojz54VYsUKIYcOEqFFDCJldtI+6dYV46y0htmwR4vbtEhV744YQmzYJ8f77QoSGCmFnV7TYKlWEeO45IaZNE2LfPiHu3jX6t6VypDR/vxUPQEIIMXPmTFGzZk3h4OAgWrVqJRITEzXvhYWFiYiICJ37V69eLerVqyccHBxE48aNxY8//likzIULF4rHH39cODk5iYCAALFhw4YSt4cBiIioELVaiCNHhJgyRYj27YsmF2dnIbp3FyI+XoizZ0tcbHa2EL/9JsSECUI8/bQs5sFA5OoqRKdOQkycKMSOHSXOWmSlSvP3W/EhMHPEITAiokfIzAR+/VWukf/pp6LDZfXry2Gyrl3l2FYJF5nk5QEHDmjnEO3aVXSbIwcHoFUr7bBZmzaAm5sBvxtZNIvZB8hcMQAREZWQEHI1WUEY2r1b7qhYwMVFbiXdtat8+PmVuGi1GvjrL92J1fd3PNGwsZHzuwvmELVrV6qNsKmcYQDSEwMQEVEZ3byp7R3avBm4ckX3/YYNtb1D7dqVaitpIYDTp7VhaOdOeVrIgxo10q40a9cO4LZu1oMBSE8MQEREBiCEXPpV0Du0Z49u75CrK/D009reoZo1S13FP//IIFQQio4eLXqPn58MQm3byiGzxo250qy8YgDSEwMQEZER3LgBbN2q7R16YE83NG6s7R1q27ZMB439+6/u0vsDB3QzFyBPASm852OrVv95MghZCAYgPTEAEREZmVoNJCdre4cSE3VPundzA8LDtb1DNWqUqZpbt2TH086d8t/ERCA7W/ceGxu5Y3VBIGrTBvD35waNlogBSE8MQEREJnb9ujyrrKB36OpV3febNtX2DrVpA9jbl6mae/fkxOqC/R4LTgZ5kJeXbiBq0ULuBUnmjQFITwxAREQKUqvl2FVB79Cff+puIe3uDnTsqO0d8vHRq7orV3Q3wU5KkkvyC3NwkOegFQSi0FCgenW9qiUjYADSEwMQEZEZuXZN2zu0ZYt8XVhAgLZ3KDRUnmOmhzt3ZP4qCES7dwPp6UXv8/fX7SVq0kTvqklPDEB6YgAiIjJTarU8vLWgd2jfPt3eIQ8PoFMnGYa6dDFIN40Qcpis8LDZ4cNFzzVzdQVCQrSBqHVroGJFvaunUmAA0hMDEBGRhbh6Ffj5ZxmIfv5ZLgMrrHlz2TvUpYtc7lWGlWXFycwE9u7VBqI9e+S1wlQquSdR4V6iunU5udqYGID0xABERGSB8vNlj1BB79D+/brvOznJLponnpAbA4WGGmz9e34+cPy4dsjsjz/kpo0PqlJFNxC1bAlUqGCQJhADkP4YgIiIyoG0NG3v0K+/Fp07ZGMDNGsmw9ATT8iHnhOqC0tP151cvW8fkJure4+dneykKhyKyrjinxiA9McARERUzggBnDwpd0nctevh52j4++sGogYNDDZmlZcHHDyoO7n6wZNCAHl0R+FAFBBQ5lX/VocBSE8MQEREVuDyZZlCdu6UoejQId3NGHF/zKogDLVrJ7trDDSPSAjg4kXdydWHDhXdudrZWU5fKrxzdbVqBmlCucMApCcGICIiK5SZKcesCnqJEhPlmvjCnJ3lPKKCXqLWrQ16jkZWlhwqKzy5+saNovf5+gLBwXIOUXCw3KOoUiWDNcNiMQDpiQGIiIiQlyc3BCoYMtu1S+5YXZiNjRyjKjxsZsAdEtVqOXJXOBCdOFF0CT4APP64DEQFoah5c3miiDVhANITAxARERVRkEYKwtCuXcWfo1Gnju6wWb16Bl37npkp5xLt3y97i/bvB86cKXqfSgU0bKgNRC1byqzm7GywppgdBiA9MQAREVGJXLqkDUMF84ge/LPq6akNQ088IbtmDDyr+fp1eYRHQSDavx9ISSl6n52d3LG6cChq2rT8TLJmANITAxAREZVJRoZ2HtHOnfIcswfXvleoUHQekRHGqlJTdUPRvn3FH+nh6Ch7hgoCUcuWsufI1tbgTTI6BiA9MQAREZFB5ObKeUSFh80enNVsawsEBmqHzZ54AvD2NnhThAD++Ud36Gz//uInWVeoALRooTvRuk4dOeXJnDEA6YkBiIiIjEKtlltGF96P6MKFovc9/rjusJmRztAQQm6HVDgQJSXJ1WgP8vCQq80Kh6KaNc3raA8GID0xABERkcmkpMj9iAoC0ZEjRecRVa2qDURt2hh0P6IH5ecDp07pDp0lJxfdEaCgWQXDZgWhyICL4EqNAUhPDEBERKSYmzflPKKCYbO9e4vOI3JykokjNFQGotBQwMvLaE26exc4dkw3FB0+DNy7V/ReH5+iexR5ehqtaToYgPTEAERERGYjN1emjoJhsz17ip56DwC1a2u3iw4Nlcu7jDiT+c4dGYIKhs727ZMh6cHNtAHAz083FLVoIYfUDI0BSE8MQEREZLaEAP7+W3d3xKNHiw6bubrK1WYFgah1a6NvF52dXXSPolOnir+3d29g1SrD1s8ApCcGICIisig3b8ol9wXHzycmArduFb2vUSPtsFmbNnKTRiMv7crIkBOrC4ei8+eBUaOAmTMNWxcDkJ4YgIiIyKLl58vxqMK9RH//XfS+SpV0A1FwsOw5MrKrV+W8Ih8fw5bLAKQnBiAiIip3rl7V9hDt2SMnVz+4tKvgbLOCYbM2beQEHnNa6/4IDEB6YgAiIqJyLy9PHt1REIr++KP48zO8vXUDUYsWchWaGWIA0hMDEBERWaV//tHtJTpwQI5VFebgIENQ4RVnhh7LKiMGID0xABEREQG4fVvOYC4IRH/8UfyBYrVq6fYSNWumyAmrpfn7bRanesyaNQt+fn5wcnJCSEgI9u7d+8j716xZgwYNGsDJyQlNmzbFTz/99NB733jjDahUKkyfPt0ILSciIirHnJ3lDtTvvQesXy9PWD19GvjmG2D4cDlfyMZGHuexYgXw5ptys5+KFYEOHYCxY4FNm4rft0hhdko3YNWqVYiKisKcOXMQEhKC6dOno3Pnzjh58iSqVatW5P4//vgD/fr1Q1xcHJ555hksX74cPXv2xIEDB9CkSROde9evX4/ExET4mEnXHBERkUVTqeSpqHXqAK+8Iq/duiUnVBf0Eu3ZI5flb98uHwXq1dMOm7VpI4+cV/B0VcWHwEJCQhAcHIz4+HgAgFqthq+vL0aPHo3o6Ogi9/fp0wfZ2dnYtGmT5lrr1q0RGBiIOXPmaK5dunQJISEh+Pnnn9G9e3dERkYiMjKyRG3iEBgREVEZqdXAiRO6k6tPnCh6X8eOwC+/GLTq0vz9VrQHKC8vD0lJSYiJidFcs7GxQXh4OPbs2VPsZ/bs2YOoqCida507d8aGDRs0r9VqNQYMGIAxY8agcePG/9mO3Nxc5BY6ZyUzM7OM34iIiMjK2djIDRcbNQKGDJHXrl+XmzMW9BL9+accPlOQogHo2rVryM/Ph9cDB7h5eXnhRHFpEUBqamqx96empmpeT548GXZ2dnjzzTdL1I64uDh8+OGHZfoORERE9B8qVwa6dZMPQJ6impOjaJPMYhK0ISUlJeHLL7/EkiVLoCrhxk0xMTHIyMjQPFKK2weBiIiIDMPODlB4iomiAcjT0xO2trZIS0vTuZ6WlgZvb+9iP+Pt7f3I+3fu3In09HTUrFkTdnZ2sLOzw4ULF/DOO+/Az8+v2DIdHR3h7u6u8yAiIqLyS9EA5ODggKCgICQkJGiuqdVqJCQkIDQ0tNjPhIaG6twPAFu3btXcP2DAABw+fBjJycmah4+PD8aMGYOff/7ZyN+IiIiILIHiy+CjoqIQERGBli1bolWrVpg+fTqys7MxePBgAMDAgQPx2GOPIS4uDgDw1ltvISwsDNOmTUP37t2xcuVK7N+/H/PmzQMAVKlSBVWqVNGpw97eHt7e3qhfv74C35CIiIjMjeIBqE+fPrh69SrGjx+P1NRUBAYGYsuWLZqJzhcvXoRNoX0C2rRpg+XLl+N///sfxo4di7p162LDhg1F9gAiIiIiehjF9wEyR9wHiIiIyPJY3FEYRERERKbEAERERERWhwGIiIiIrA4DEBEREVkdBiAiIiKyOgxAREREZHUYgIiIiMjqMAARERGR1VF8J2hzVLA3ZGZmptJNISIiohIq+Ltdkj2eGYCKcevWLQCAr6+v0k0hIiKiUrp16xY8PDweeQ+PwiiGWq3G5cuX4ebmBpVKZdCyMzMz4evri5SUFB6zYQb4+zAv/H2YF/4+zA9/J48mhMCtW7fg4+Ojc45ocdgDVAwbGxvUqFHDqHW4u7vzf7xmhL8P88Lfh3nh78P88HfycP/V81OAk6CJiIjI6jAAERERkdVhADIxR0dHxMbGwtHRUemmEH8fZoe/D/PC34f54e/EcDgJmoiIiKwOe4CIiIjI6jAAERERkdVhACIiIiKrwwBEREREVocByIRmzZoFPz8/ODk5ISQkBHv37lW6SVYrLi4OwcHBcHNzQ7Vq1dCzZ0+cPHlS6WYRgEmTJkGlUiEyMlLppli1S5cu4ZVXXkGVKlXg7OyMpk2bYv/+/Uo3yyrl5+dj3Lhx8Pf3h7OzM+rUqYOPP/64ROdd0cMxAJnIqlWrEBUVhdjYWBw4cAABAQHo3Lkz0tPTlW6aVdqxYwdGjhyJxMREbN26FXfv3kWnTp2QnZ2tdNOs2r59+zB37lw0a9ZM6aZYtRs3bqBt27awt7fH5s2bcezYMUybNg2VKlVSumlWafLkyZg9ezbi4+Nx/PhxTJ48GVOmTMHMmTOVbppF4zJ4EwkJCUFwcDDi4+OB++eN+fr6YvTo0YiOjla6eVbv6tWrqFatGnbs2IEnn3xS6eZYpaysLLRo0QJfffUVJk6ciMDAQEyfPl3pZlml6Oho7N69Gzt37lS6KQTgmWeegZeXFxYuXKi51qtXLzg7O+Pbb79VtG2WjD1AJpCXl4ekpCSEh4drrtnY2CA8PBx79uxRtG0kZWRkAAAqV66sdFOs1siRI9G9e3ed/05IGRs3bkTLli3x0ksvoVq1amjevDnmz5+vdLOsVps2bZCQkIBTp04BAA4dOoRdu3aha9euSjfNovEwVBO4du0a8vPz4eXlpXPdy8sLJ06cUKxdJKnVakRGRqJt27Zo0qSJ0s2xSitXrsSBAwewb98+pZtCAM6ePYvZs2cjKioKY8eOxb59+/Dmm2/CwcEBERERSjfP6kRHRyMzMxMNGjSAra0t8vPz8cknn6B///5KN82iMQCR1Rs5ciT++usv7Nq1S+mmWKWUlBS89dZb2Lp1K5ycnJRuDt3/fwpatmyJTz/9FADQvHlz/PXXX5gzZw4DkAJWr16NZcuWYfny5WjcuDGSk5MRGRkJHx8f/j70wABkAp6enrC1tUVaWprO9bS0NHh7eyvWLgJGjRqFTZs24ffff0eNGjWUbo5VSkpKQnp6Olq0aKG5lp+fj99//x3x8fHIzc2Fra2tom20NtWrV0ejRo10rjVs2BDfffedYm2yZmPGjEF0dDT69u0LAGjatCkuXLiAuLg4BiA9cA6QCTg4OCAoKAgJCQmaa2q1GgkJCQgNDVW0bdZKCIFRo0Zh/fr1+O233+Dv7690k6zW008/jSNHjiA5OVnzaNmyJfr374/k5GSGHwW0bdu2yLYQp06dQq1atRRrkzXLycmBjY3un2tbW1uo1WrF2lQesAfIRKKiohAREYGWLVuiVatWmD59OrKzszF48GClm2aVRo4cieXLl+P777+Hm5sbUlNTAQAeHh5wdnZWunlWxc3NrcjcKxcXF1SpUoVzshTy9ttvo02bNvj000/Ru3dv7N27F/PmzcO8efOUbppV6tGjBz755BPUrFkTjRs3xsGDB/H555/j1VdfVbppFo3L4E0oPj4en332GVJTUxEYGIgZM2YgJCRE6WZZJZVKVez1xYsXY9CgQSZvD+lq3749l8ErbNOmTYiJicHff/8Nf39/REVFYdiwYUo3yyrdunUL48aNw/r165Geng4fHx/069cP48ePh4ODg9LNs1gMQERERGR1OAeIiIiIrA4DEBEREVkdBiAiIiKyOgxAREREZHUYgIiIiMjqMAARERGR1WEAIiIiIqvDAERERERWhwGIiOghVCoVNmzYoHQziMgIGICIyCwNGjQIKpWqyKNLly5KN42IygEehkpEZqtLly5YvHixzjVHR0fF2kNE5Qd7gIjIbDk6OsLb21vnUalSJeD+8NTs2bPRtWtXODs7o3bt2li7dq3O548cOYKnnnoKzs7OqFKlCl577TVkZWXp3LNo0SI0btwYjo6OqF69OkaNGqXz/rVr1/D888+jQoUKqFu3LjZu3Kh578aNG+jfvz+qVq0KZ2dn1K1bt0hgIyLzxABERBZr3Lhx6NWrFw4dOoT+/fujb9++OH78OAAgOzsbnTt3RqVKlbBv3z6sWbMGv/76q07AmT17NkaOHInXXnsNR44cwcaNG/H444/r1PHhhx+id+/eOHz4MLp164b+/fvj+vXrmvqPHTuGzZs34/jx45g9ezY8PT1N/FMgojIRRERmKCIiQtja2goXFxedxyeffCKEEAKAeOONN3Q+ExISIoYPHy6EEGLevHmiUqVKIisrS/P+jz/+KGxsbERqaqoQQggfHx/xwQcfPLQNAMT//vc/zeusrCwBQGzevFkIIUSPHj3E4MGDDfzNicgUOAeIiMxWhw4dMHv2bJ1rlStX1jwPDQ3VeS80NBTJyckAgOPHjyMgIAAuLi6a99u2bQu1Wo2TJ09CpVLh8uXLePrppx/ZhmbNmmmeu7i4wN3dHenp6QCA4cOHo1evXjhw4AA6deqEnj17ok2bNnp+ayIyBQYgIjJbLi4uRYakDMXZ2blE99nb2+u8VqlUUKvVAICuXbviwoUL+Omnn7B161Y8/fTTGDlyJKZOnWqUNhOR4XAOEBFZrMTExCKvGzZsCABo2LAhDh06hOzsbM37u3fvho2NDerXrw83Nzf4+fkhISFBrzZUrVoVERER+PbbbzF9+nTMmzdPr/KIyDTYA0REZis3Nxepqak61+zs7DQTjdesWYOWLVviiSeewLJly7B3714sXLgQANC/f3/ExsYiIiICEyZMwNWrVzF69GgMGDAAXl5eAIAJEybgjTfeQLVq1dC1a1fcunULu3fvxujRo0vUvvHjxyMoKAiNGzdGbm4uNm3apAlgRGTeGICIyGxt2bIF1atX17lWv359nDhxAri/QmvlypUYMWIEqlevjhUrVqBRo0YAgAoVKuDnn3/GW2+9heDgYFSoUAG9evXC559/rikrIiICd+7cwRdffIF3330Xnp6eePHFF0vcPgcHB8TExOD8+fNwdnZGu3btsHLlSoN9fyIyHpWQKx2IiCyKSqXC+vXr0bNnT6WbQkQWiHOAiIiIyOowABEREZHV4RwgIrJIHL0nIn2wB4iIiIisDgMQERERWR0GICIiIrI6DEBERERkdRiAiIiIyOowABEREZHVYQAiIiIiq8MARERERFbn/xwXfxJ9aWXFAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.suptitle('MSE vs Epochs')\n",
    "plt.plot(tune_train_err, label='Train', color='blue')\n",
    "plt.plot(tune_test_err, label='Test', color='red')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('MSE')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[156., 353., 404., ...,   0.,   0.,   0.],\n",
       "        [  0.,   0.,   0., ...,   0.,   0.,   0.],\n",
       "        [  0.,   0.,   0., ...,   0.,   0.,   0.],\n",
       "        ...,\n",
       "        [  0.,   0.,   0., ...,   0.,   0.,   0.],\n",
       "        [  0.,   0.,   0., ...,   0.,   0.,   0.],\n",
       "        [  0.,   0.,   0., ...,   0.,   0.,   4.]]),\n",
       " array([1.   , 1.06 , 1.12 , 1.18 , 1.24 , 1.3  , 1.359, 1.42 , 1.48 ,\n",
       "        1.54 , 1.6  , 1.66 , 1.72 , 1.779, 1.84 , 1.9  , 1.96 , 2.02 ,\n",
       "        2.08 , 2.14 , 2.2  , 2.26 , 2.32 , 2.379, 2.44 , 2.5  , 2.559,\n",
       "        2.621, 2.68 , 2.74 , 2.8  , 2.86 , 2.92 , 2.98 , 3.04 , 3.1  ,\n",
       "        3.16 , 3.22 , 3.28 , 3.34 , 3.4  , 3.459, 3.52 , 3.58 , 3.64 ,\n",
       "        3.7  , 3.76 , 3.82 , 3.879, 3.94 , 4.   ], dtype=float16),\n",
       " array([0.879 , 0.9287, 0.9785, 1.028 , 1.078 , 1.129 , 1.179 , 1.229 ,\n",
       "        1.278 , 1.328 , 1.378 , 1.428 , 1.479 , 1.527 , 1.578 , 1.628 ,\n",
       "        1.678 , 1.728 , 1.777 , 1.828 , 1.877 , 1.928 , 1.978 , 2.027 ,\n",
       "        2.078 , 2.127 , 2.176 , 2.227 , 2.277 , 2.328 , 2.377 , 2.426 ,\n",
       "        2.477 , 2.527 , 2.576 , 2.625 , 2.676 , 2.727 , 2.777 , 2.826 ,\n",
       "        2.875 , 2.926 , 2.977 , 3.025 , 3.076 , 3.125 , 3.176 , 3.225 ,\n",
       "        3.275 , 3.326 , 3.375 ], dtype=float16),\n",
       " <matplotlib.collections.QuadMesh at 0x7f9dc16ae2a0>)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi4AAAGdCAYAAAA1/PiZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAcy0lEQVR4nO3df3DV5Z0v8E8QOOBKTsvekqAElxYbf5Uf2romzgq2Iku5XfOP6zo7hW51Z+vCDIw77ZXObnu3ndnoqGWdqQM6TmXaHYaWdsEdrKVRClwVu4BkFmjrrRSF2iT2zrg5kkpE8r1/dM0STQ458SQnD3m9Zr5/nG+e53w/55lnct558v1RlWVZFgAACRhX6QIAAAZLcAEAkiG4AADJEFwAgGQILgBAMgQXACAZggsAkAzBBQBIxvhKFzAYPT098Zvf/CamTJkSVVVVlS4HABiELMvijTfeiAsvvDDGjSvPWkkSweU3v/lN1NXVVboMAGAIjh8/HjNmzCjLeyURXKZMmRLxXx+8urq60uUAwJj3p7c+eNY2b799MvY+9U+93+PlkERweeffQ9XV1YILAIwC4ydMGnTbcp7m4eRcACAZggsAkAzBBQBIhuACACSjKsuyrNJFnE2hUIh8Ph+dnZ1OzgWARAzH97cVFwAgGYILAJAMwQUASIbgAgAkQ3ABAJIhuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMwQUASIbgAgAkQ3ABAJIhuAAAyRhf6QIAgPQsGnfLWdu8nZ0q+3GtuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMwQUASIbLoQEStuTyLw+q3ZM/+6dhr4WxpaVn81nbFAqFyOfzZT2uFRcAIBmCCwCQDMEFAEiG4AIAJENwAQCSIbgAAMkQXACAZLiPC0DC3v7FLytdAowoKy4AQDIEFwAgGYILAJAMwQUASIbgAgAkQ3ABAJIhuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMwQUASIbgAgAko6SnQ69bty7WrVsXL7/8ckREXHHFFfGVr3wllixZMmCfzZs3xz/8wz/Eyy+/HJdccknce++98elPf/r9Vw5AdC5rqHQJMKJKWnGZMWNG3HPPPbF///7Yt29ffPKTn4ybb745Dh8+3G/75557Lm677ba4/fbb48CBA9HU1BRNTU1x6NChctUPAIwhVVmWZe/nDaZOnRr33Xdf3H777e/52a233hpdXV2xbdu23n3XXnttzJs3L9avXz/oYxQKhcjn89HZ2RnV1dXvp1yAc8o1n/vGoNr9+4a7hr0WeLfh+P4e8jkup0+fjk2bNkVXV1c0NPS/VLlnz5648cYb++xbvHhx7Nmzp+h7d3d3R6FQ6LMBAJQcXA4ePBgXXHBB5HK5+MIXvhBbtmyJyy+/vN+27e3tUVNT02dfTU1NtLe3Fz1Gc3Nz5PP53q2urq7UMgGAc1DJwaW+vj5aW1vjpz/9adx5552xfPny+NnPflbWotasWROdnZ292/Hjx8v6/gBAmkq6qigiYuLEiTF79uyIiLj66qtj79698eCDD8bDDz/8nra1tbXR0dHRZ19HR0fU1tYWPUYul4tcLldqaQDAOe5938elp6cnuru7+/1ZQ0NDPP300332tbS0DHhODABAMSWtuKxZsyaWLFkSM2fOjDfeeCM2btwYO3fujO3bt0dExLJly+Kiiy6K5ubmiIhYtWpVLFiwIB544IFYunRpbNq0Kfbt2xePPPLI8HwagDHmD//9/1W6BBhRJQWX1157LZYtWxZtbW2Rz+djzpw5sX379li0aFFERBw7dizGjfvvRZzGxsbYuHFj/P3f/318+ctfjksuuSS2bt0aV155Zfk/CQBwznvf93EZCe7jAtC/JZd/eVDtnvzZPw17LfBuo+o+LgAAI01wAQCSIbgAAMkQXACAZJR8AzoARo+OP/lQpUuAEWXFBQBIhuACACRDcAEAkiG4AADJEFwAgGQILgBAMgQXACAZ7uMCkLCeCZWuAEaWFRcAIBmCCwCQDMEFAEiG4AIAJENwAQCSIbgAAMlwOTRAwlwOzVhjxQUASIbgAgAkQ3ABAJIhuAAAyRBcAIBkCC4AQDJcDg2QsJP/o9IVMFYtGnfLWdu8nZ0q+3GtuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMwQUASIbgAgAkw31cABJ2wbFKV8BYNf7SS87e6HR3xP8t73GtuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMwQUASIbLoaGIwTy2PSKipWfzsNcC/Tn5oUpXwFj19i9+efY22amyH9eKCwCQDMEFAEiG4AIAJENwAQCSIbgAAMkQXACAZAguAEAy3McFiuj+n9dUugQo6gO/7Kl0CYxRg7l/VaFQiHw+X9bjWnEBAJIhuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMl0NDEbv/7YuVLgGK+oPNzw+u4XeHuxIYGVZcAIBkCC4AQDIEFwAgGYILAJAMwQUASIbgAgAko6TLoZubm+Nf//Vf4xe/+EVMnjw5Ghsb49577436+voB+2zYsCH+6q/+qs++XC4XJ0+eHHrVAER4gjljUEkrLrt27YoVK1bE888/Hy0tLXHq1Km46aaboqurq2i/6urqaGtr691eeeWV91s3ADAGlbTi8qMf/ajP6w0bNsS0adNi//79cf311w/Yr6qqKmpra4deJQDA+z3HpbOzMyIipk6dWrTdiRMn4uKLL466urq4+eab4/Dhw0Xbd3d3R6FQ6LMBAAw5uPT09MTq1avjuuuuiyuvvHLAdvX19fGtb30rHn/88fiXf/mX6OnpicbGxvj1r389YJ/m5ubI5/O9W11d3VDLBADOIVVZlmVD6XjnnXfGk08+Gc8880zMmDFj0P1OnToVl112Wdx2223x9a9/vd823d3d0d3d3fu6UChEXV1ddHZ2RnV19VDKBTgnXf9n9w2qneduUQmFQiHy+XxZv7+H9JDFlStXxrZt22L37t0lhZaIiAkTJsT8+fPjpZdeGrBNLpeLXC43lNIAgHNYSf8qyrIsVq5cGVu2bIkdO3bErFmzSj7g6dOn4+DBgzF9+vSS+wIAY1tJKy4rVqyIjRs3xuOPPx5TpkyJ9vb2iIjI5/MxefLkiIhYtmxZXHTRRdHc3BwREV/72tfi2muvjdmzZ8d//ud/xn333RevvPJK3HHHHcPxeaCsFo27ZVDtWno2D3stAJQYXNatWxcREQsXLuyz/7HHHovPfe5zERFx7NixGDfuvxdyXn/99fjrv/7raG9vjw9+8INx9dVXx3PPPReXX355eT4BADBmlBRcBnMe786dO/u8Xrt2baxdu7b0ygAA3sWzigCAZAguAEAyBBcAIBmCCwCQjCHdgA7Git/+bWOlS4Cictv+vdIlwIiy4gIAJENwAQCSIbgAAMkQXACAZAguAEAyBBcAIBkuh4YiTk2pdAUAnMmKCwCQDMEFAEiG4AIAJENwAQCSIbgAAMkQXACAZAguAEAy3McFiui67K1KlwBFtfRsrnQJMKKsuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMwQUASIbLoaGIca9PqHQJUNSicbcMqp3LpjlXWHEBAJIhuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMwQUASIb7uEARPVPfqnQJUJT7szDWWHEBAJIhuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMl0NDEdlbsj2j26JxtwyqncumOVf4rQwAJENwAQCSIbgAAMkQXACAZAguAEAyBBcAIBkuh4YiqkR7RjmXOTPW+LUMACRDcAEAkiG4AADJEFwAgGQILgBAMgQXACAZggsAkAz3cYFizuupdAUAnMGKCwCQDMEFAEiG4AIAJENwAQCSIbgAAMkQXACAZJQUXJqbm+MTn/hETJkyJaZNmxZNTU3x4osvnrXf5s2b49JLL41JkybFxz72sfjhD3/4fmqGkdNTNbgNgBFRUnDZtWtXrFixIp5//vloaWmJU6dOxU033RRdXV0D9nnuuefitttui9tvvz0OHDgQTU1N0dTUFIcOHSpH/QDAGFKVZVk21M6//e1vY9q0abFr1664/vrr+21z6623RldXV2zbtq1337XXXhvz5s2L9evXD+o4hUIh8vl8dHZ2RnV19VDLhZL90YZ7B9Xu5c/9r2GvBSA1w/H9/b7Ocens7IyIiKlTpw7YZs+ePXHjjTf22bd48eLYs2fPgH26u7ujUCj02QAAhhxcenp6YvXq1XHdddfFlVdeOWC79vb2qKmp6bOvpqYm2tvbB+zT3Nwc+Xy+d6urqxtqmQDAOWTIwWXFihVx6NCh2LRpU3kriog1a9ZEZ2dn73b8+PGyHwMASM+QHrK4cuXK2LZtW+zevTtmzJhRtG1tbW10dHT02dfR0RG1tbUD9snlcpHL5YZSGgBwDitpxSXLsli5cmVs2bIlduzYEbNmzTprn4aGhnj66af77GtpaYmGhobSqwUAxrSSVlxWrFgRGzdujMcffzymTJnSe55KPp+PyZMnR0TEsmXL4qKLLorm5uaIiFi1alUsWLAgHnjggVi6dGls2rQp9u3bF4888shwfB4oq6oTQ1qUBGCYlLTism7duujs7IyFCxfG9OnTe7fvfve7vW2OHTsWbW1tva8bGxtj48aN8cgjj8TcuXPj+9//fmzdurXoCb0AAP0p6c/JwdzyZefOne/Zd8stt8Qtt9xSWmUAAO/iWUUAQDIEFwAgGYILAJAMwQUASIZrPaGIqj/srnQJAJzBigsAkAzBBQBIhuACACRDcAEAkiG4AADJEFwAgGS4HBqKOfvjuQAYQVZcAIBkCC4AQDIEFwAgGYILAJAMwQUASIbgAgAkQ3ABAJLhPi5QxOSDkytdAgBnsOICACRDcAEAkiG4AADJEFwAgGQILgBAMgQXACAZLoeGIk5dUOkKADiTFRcAIBmCCwCQDMEFAEiG4AIAJENwAQCSIbgAAMkQXACAZLiPCxTx9vlZpUsA4AxWXACAZAguAEAyBBcAIBmCCwCQDMEFAEiG4AIAJMPl0FDE+N9VVboEAM5gxQUASIbgAgAkQ3ABAJIhuAAAyRBcAIBkCC4AQDJcDg1FTDhR6QoAOJMVFwAgGYILAJAMwQUASIbgAgAkQ3ABAJIhuAAAyRBcAIBkuI8LFHE6V+kKADiTFRcAIBmCCwCQDMEFAEiG4AIAJENwAQCSIbgAAMko+XLo3bt3x3333Rf79++Ptra22LJlSzQ1NQ3YfufOnXHDDTe8Z39bW1vU1taWdOw/W3J/jB8/qWibp3Z9uaT3hGJm/OTNwTX838NdCQAxlBWXrq6umDt3bjz00EMl9XvxxRejra2td5s2bVqphwYAxriSV1yWLFkSS5YsKflA06ZNiw984AMl9wMAeMeIneMyb968mD59eixatCieffbZom27u7ujUCj02QAAhj24TJ8+PdavXx8/+MEP4gc/+EHU1dXFwoUL44UXXhiwT3Nzc+Tz+d6trq5uuMsEABIw7M8qqq+vj/r6+t7XjY2NceTIkVi7dm185zvf6bfPmjVr4q677up9XSgUhBcAoDIPWbzmmmvimWeeGfDnuVwucjlPtwMA+qpIcGltbY3p06eX3O+t6gnRM2HCsNQE/an6PwcqXQIAZyg5uJw4cSJeeuml3tdHjx6N1tbWmDp1asycOTPWrFkTr776anz729+OiIh//ud/jlmzZsUVV1wRJ0+ejEcffTR27NgRP/7xj8v7SQCAc17JwWXfvn19bij3zrkoy5cvjw0bNkRbW1scO3as9+dvvfVW/N3f/V28+uqrcf7558ecOXPiqaee6vemdAAAxVRlWZZVuoizKRQKkc/no+FPvxbjJxS/c+7uf/viiNXFuW/RuFsG1a6lZ/Ow1wKQmne+vzs7O6O6uros7+lZRQBAMgQXACAZggsAkAzBBQBIRkXu4zJUEwunYvz48ypdBmOIk24BRhcrLgBAMgQXACAZggsAkAzBBQBIhuACACRDcAEAkpHU5dC/Xjg5zptU/FlFUE6eVQQwulhxAQCSIbgAAMkQXACAZAguAEAyBBcAIBmCCwCQDMEFAEhGUvdxmfzbiPMmVroKxpLsT+ZXugQAzmDFBQBIhuACACRDcAEAkiG4AADJEFwAgGQILgBAMpK6HLrz6u4YN7mq0mUwhvyuNlfpEgA4gxUXACAZggsAkAzBBQBIhuACACRDcAEAkiG4AADJEFwAgGQkdR+XObNejQl/MLHSZTCGnLhQtgcYTfxWBgCSIbgAAMkQXACAZAguAEAyBBcAIBmCCwCQjKQuh54y/mRMnNBT6TIYQ7qnVroCAM5kxQUASIbgAgAkQ3ABAJIhuAAAyRBcAIBkCC4AQDKSuhz6oxe0x6QLJlS6DMaQmr1vV7oEAM5gxQUASIbgAgAkQ3ABAJIhuAAAyRBcAIBkCC4AQDIEFwAgGUndx6Vh8pH4g/NlLUbO7n/7YqVLAOAMUgAAkAzBBQBIhuACACRDcAEAkiG4AADJEFwAgGSUHFx2794dn/nMZ+LCCy+Mqqqq2Lp161n77Ny5M6666qrI5XIxe/bs2LBhw5CKvXZSVVw3aVzRDQA4d5X8Td/V1RVz586Nhx56aFDtjx49GkuXLo0bbrghWltbY/Xq1XHHHXfE9u3bh1IvADCGlXwDuiVLlsSSJUsG3X79+vUxa9aseOCBByIi4rLLLotnnnkm1q5dG4sXLy718ADAGDbs/1vZs2dP3HjjjX32LV68OPbs2TNgn+7u7igUCn02AIBhDy7t7e1RU1PTZ19NTU0UCoV48803++3T3Nwc+Xy+d6urqxvuMgGABIzKs1nXrFkTnZ2dvdvx48crXRIAMAoM+0MWa2tro6Ojo8++jo6OqK6ujsmTJ/fbJ5fLRS6XG+7SAIDEDPuKS0NDQzz99NN99rW0tERDQ8NwHxoAOMeUvOJy4sSJeOmll3pfHz16NFpbW2Pq1Kkxc+bMWLNmTbz66qvx7W9/OyIivvCFL8Q3v/nN+NKXvhSf//znY8eOHfG9730vnnjiiZKLPdHTHeN6imet6pLfFQBIRckrLvv27Yv58+fH/PnzIyLirrvuivnz58dXvvKViIhoa2uLY8eO9bafNWtWPPHEE9HS0hJz586NBx54IB599FGXQgMAJavKsiyrdBFnUygUIp/Px/FfXBTVU86y4nLhsaI/BwBGxjvf352dnVFdXZ7/iYzKq4oAAPojuAAAyRBcAIBkCC4AQDKG/QZ05fTdE7Ni8llK/tsRqwYAGGlWXACAZAguAEAyBBcAIBmCCwCQDMEFAEhGElcVvfNUgpMn3j5r20KhMAIVAQBn8853cjmfLpTEs4p+9atfxUc+8pFKlwEADMGRI0fiwx/+cFneK4kVl6lTp0ZExLFjxyKfz1e6nGQVCoWoq6uL48ePl+1hV2OVsSwfY1kexrF8jGX5dHZ2xsyZM3u/x8shieAybtzvT8XJ5/MmURlUV1cbxzIxluVjLMvDOJaPsSyfd77Hy/JeZXsnAIBhJrgAAMlIIrjkcrn46le/GrlcrtKlJM04lo+xLB9jWR7GsXyMZfkMx1gmcVURAECksuICABCCCwCQEsEFAEiG4AIAJKPiwWX37t3xmc98Ji688MKoqqqKrVu3nrXPzp0746qrropcLhezZ8+ODRs2jEito12pY7lz586oqqp6z9be3j5iNY9Gzc3N8YlPfCKmTJkS06ZNi6ampnjxxRfP2m/z5s1x6aWXxqRJk+JjH/tY/PCHPxyRekezoYzlhg0b3jMnJ02aNGI1j1br1q2LOXPm9N4UraGhIZ588smifczJ9yp1HM3HwbnnnnuiqqoqVq9eXbRdOeZkxYNLV1dXzJ07Nx566KFBtT969GgsXbo0brjhhmhtbY3Vq1fHHXfcEdu3bx/2Wke7UsfyHS+++GK0tbX1btOmTRu2GlOwa9euWLFiRTz//PPR0tISp06diptuuim6uroG7PPcc8/FbbfdFrfffnscOHAgmpqaoqmpKQ4dOjSitY82QxnL+K87lp45J1955ZURq3m0mjFjRtxzzz2xf//+2LdvX3zyk5+Mm2++OQ4fPtxve3Oyf6WOY5iPZ7V37954+OGHY86cOUXblW1OZqNIRGRbtmwp2uZLX/pSdsUVV/TZd+utt2aLFy8e5urSMpix/MlPfpJFRPb666+PWF0peu2117KIyHbt2jVgmz//8z/Pli5d2mffH//xH2d/8zd/MwIVpmMwY/nYY49l+Xx+ROtK1Qc/+MHs0Ucf7fdn5uTgFRtH87G4N954I7vkkkuylpaWbMGCBdmqVasGbFuuOVnxFZdS7dmzJ2688cY++xYvXhx79uypWE2pmzdvXkyfPj0WLVoUzz77bKXLGXU6OzsjznjYZ3/My8EZzFhGRJw4cSIuvvjiqKurO+tfw2PR6dOnY9OmTdHV1RUNDQ39tjEnz24w4xjmY1ErVqyIpUuXvmeu9adcczKJhyyeqb29PWpqavrsq6mpiUKhEG+++WZMnjy5YrWlZvr06bF+/fr4+Mc/Ht3d3fHoo4/GwoUL46c//WlcddVVlS5vVOjp6YnVq1fHddddF1deeeWA7Qaal2P9fKEzDXYs6+vr41vf+lbMmTMnOjs74/7774/GxsY4fPhwzJgxY0RrHm0OHjwYDQ0NcfLkybjgggtiy5Ytcfnll/fb1pwcWCnjaD4ObNOmTfHCCy/E3r17B9W+XHMyueBC+dTX10d9fX3v68bGxjhy5EisXbs2vvOd71S0ttFixYoVcejQoXjmmWcqXUryBjuWDQ0Nff76bWxsjMsuuywefvjh+PrXvz4ClY5e9fX10draGp2dnfH9738/li9fHrt27RrwS5f+lTKO5mP/jh8/HqtWrYqWlpYRP1k5ueBSW1sbHR0dffZ1dHREdXW11ZYyuOaaa3xJ/5eVK1fGtm3bYvfu3Wf9y2qgeVlbWzvMVaahlLF8twkTJsT8+fPjpZdeGrb6UjFx4sSYPXt2RERcffXVsXfv3njwwQfj4Ycffk9bc3JgpYzju5mPv7d///547bXX+qzOnz59Onbv3h3f/OY3o7u7O84777w+fco1J5M7x6WhoSGefvrpPvtaWlqK/n+SwWttbY3p06dXuoyKyrIsVq5cGVu2bIkdO3bErFmzztrHvOzfUMby3U6fPh0HDx4c8/OyPz09PdHd3d3vz8zJwSs2ju9mPv7epz71qTh48GC0trb2bh//+MfjL//yL6O1tfU9oSXKOSeHfCpxmbzxxhvZgQMHsgMHDmQRkX3jG9/IDhw4kL3yyitZlmXZ3XffnX32s5/tbf+rX/0qO//887MvfvGL2c9//vPsoYceys4777zsRz/6UQU/xehQ6liuXbs227p1a/bLX/4yO3jwYLZq1aps3Lhx2VNPPVXBT1F5d955Z5bP57OdO3dmbW1tvdvvfve73jaf/exns7vvvrv39bPPPpuNHz8+u//++7Of//zn2Ve/+tVswoQJ2cGDByv0KUaHoYzlP/7jP2bbt2/Pjhw5ku3fvz/7i7/4i2zSpEnZ4cOHK/QpRoe7774727VrV3b06NHsP/7jP7K77747q6qqyn784x9nmTk5aKWOo/k4eO++qmi45mTFg8s7l+S+e1u+fHmWZVm2fPnybMGCBe/pM2/evGzixInZhz/84eyxxx6rUPWjS6ljee+992Yf+chHskmTJmVTp07NFi5cmO3YsaOCn2B06G8MI6LPPFuwYEHvuL7je9/7XvbRj340mzhxYnbFFVdkTzzxRAWqH12GMparV6/OZs6cmU2cODGrqanJPv3pT2cvvPBChT7B6PH5z38+u/jii7OJEydmH/rQh7JPfepTvV+2mTk5aKWOo/k4eO8OLsM1J6uy3/9yAQAY9ZI7xwUAGLsEFwAgGYILAJAMwQUASIbgAgAkQ3ABAJIhuAAAyRBcAIBkCC4AQDIEFwAgGYILAJAMwQUASMb/ByhxjawVIMzxAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "batch_src, batch_labels, batch_padding_mask = mktunebatch(2048)\n",
    "output = model(batch_src, batch_padding_mask)\n",
    "x = batch_labels.detach().to(torch.float16).cpu().numpy().flatten()\n",
    "y = output.detach().to(torch.float16).cpu().numpy().flatten()\n",
    "plt.hist2d(x, y, bins=50, norm=mpl.colors.LogNorm())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JtTLXn4zC1z_"
   },
   "source": [
    "# Step 6: Test generalization"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}