path: root/transformer_shortest_paths.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LPphBnKR-aWF"
   },
   "source": [
    "# Step 0: Imports"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Question: \n",
    "\n",
    "- Do the attention heads learn to attend to the same positional encodings\n",
    "- do interp -- what is it doing? can we figure out?\n",
    "- update: I think we should do interp once it's bigger. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "execution_state": "idle",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ge5QvElvhCOw",
    "outputId": "c7cdaefa-d6dc-44ad-c258-e4fb2aca97a5"
   },
   "outputs": [],
   "source": [
    "# using tqdm.auto glitches out collaborative editing\n",
    "from tqdm import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "\n",
    "from math import sqrt\n",
    "from collections import deque\n",
    "import os\n",
    "import random\n",
    "from concurrent.futures import ProcessPoolExecutor\n",
    "import pickle\n",
    "\n",
    "torch.manual_seed(42)\n",
    "random.seed(42)\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "assert device.type == \"cuda\", \"CUDA is not available. Please check your GPU setup.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "execution_state": "idle",
   "metadata": {
    "id": "lylOX2POPwFL"
   },
   "outputs": [],
   "source": [
    "# VTXS numbers here are inclusive\n",
    "MIN_VTXS = 3\n",
    "MAX_VTXS = 15\n",
    "MAX_TUNE_VTXS = 7 # 15\n",
    "AVG_DEG = 2\n",
    "SEQ_LEN = MAX_VTXS + 1 # means 32 edges, final token is the target vertex\n",
    "PAD_TOKEN = 0\n",
    "# vertices are labelled 1,2,...,63\n",
    "# we also have a padding token which is 0."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gKt-yIpDebF1"
   },
   "source": [
    "# Step 1: Generate synthetic data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "execution_state": "idle",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1IbzGIWseK3E",
    "outputId": "a3cbc233-358c-4e17-ea6e-f4e9349d886b"
   },
   "outputs": [],
   "source": [
    "# original task data\n",
    "# the data will be edge lists\n",
    "# like this: [1 3 1 5 2 4 0 0 0 0 2]\n",
    "# this represents edges (1,3), (1,5) (2,4)\n",
    "# (the zeros are just padding tokens)\n",
    "# the final 2 means which vertex we're going to \n",
    "\n",
    "# the label is the shortest distance from vtx 1 to vtx 2\n",
    "# or \"number of vertices\" if no path exists\n",
    "\n",
    "def random_graph(n):\n",
    "    edge_list = []\n",
    "    adjacencies = [set() for _ in range(n+1)]\n",
    "    indices = [random.randint(1, n) for _ in range(AVG_DEG * n)]\n",
    "    for i in range(0, len(indices), 2):\n",
    "        u = indices[i]\n",
    "        v = indices[i + 1]\n",
    "        if u != v:\n",
    "            edge_list += [min(u,v),max(u,v)]\n",
    "            adjacencies[u].add(v)\n",
    "            adjacencies[v].add(u)\n",
    "    edge_list += [PAD_TOKEN]*(2*SEQ_LEN-1-len(edge_list))\n",
    "    return edge_list, adjacencies\n",
    "\n",
    "\"\"\"\n",
    "input: G, represented as an adjacency list\n",
    "output: [number of vertices]+[d(1,i) for i in range(n)] if target=None\n",
    "if target is set to some value, then we instead just output that specific distance\n",
    "\"\"\"\n",
    "def SSSP(G, target=2):\n",
    "    dist = [MAX_VTXS for _ in G]\n",
    "    dist[1] = 0\n",
    "    frontier = deque()\n",
    "    frontier.append(1)\n",
    "    while len(frontier) > 0:\n",
    "        vtx = frontier.popleft()\n",
    "        for x in G[vtx]:\n",
    "            if dist[x] == MAX_VTXS:\n",
    "                dist[x] = 1 + dist[vtx]\n",
    "                frontier.append(x)\n",
    "                if x == target:\n",
    "                    return dist[target]\n",
    "    if target is not None:\n",
    "        return dist[target]\n",
    "    else:\n",
    "        return dist\n",
    "\n",
    "def mkbatch(size):\n",
    "    graphs1 = []\n",
    "    distance1 = []\n",
    "    \n",
    "    for i in range(size):\n",
    "        n = random.randint(MIN_VTXS, MAX_VTXS)\n",
    "        edge_list, adj_list = random_graph(n)\n",
    "        dist = SSSP(adj_list)\n",
    "        edge_list[-1] = 2 # target token\n",
    "        graphs1.append(edge_list)\n",
    "        distance1.append(dist)\n",
    "    \n",
    "    data = torch.tensor(graphs1)\n",
    "    labels = torch.tensor(distance1, dtype=torch.bfloat16)\n",
    "    padding = data == PAD_TOKEN\n",
    "    return data, labels, padding\n",
    "\n",
    "def savebatch(size, idx):\n",
    "    data, labels, padding = mkbatch(size)\n",
    "    everything = {\n",
    "        \"data\": data,\n",
    "        \"labels\": labels,\n",
    "        \"padding\": padding,\n",
    "    }\n",
    "    \n",
    "    with open(f'data/{idx}.pickle', 'wb') as file:\n",
    "        pickle.dump(everything, file)\n",
    "\n",
    "def vertices_on_shortest_12_path(G, target=2):\n",
    "    dist = [MAX_VTXS for _ in G]\n",
    "    parent = [-1 for _ in G]\n",
    "    dist[1] = 0\n",
    "    frontier = deque()\n",
    "    frontier.append(1)\n",
    "    while len(frontier) > 0:\n",
    "        vtx = frontier.popleft()\n",
    "        for x in G[vtx]:\n",
    "            if dist[x] == MAX_VTXS:\n",
    "                parent[x] = vtx\n",
    "                dist[x] = 1 + dist[vtx]\n",
    "                frontier.append(x)\n",
    "                if x == target:\n",
    "                    path = [x]\n",
    "                    while parent[x] != -1:\n",
    "                        x = parent[x]\n",
    "                        path.append(x)\n",
    "                    return list(reversed(path))\n",
    "    return []\n",
    "\n",
    "def mktunebatch(size, test=False):\n",
    "    graphs = []\n",
    "    distance = []\n",
    "    \n",
    "    for i in range(size):\n",
    "        n = random.randint(MIN_VTXS, MAX_VTXS if test else MAX_TUNE_VTXS)\n",
    "        while True:\n",
    "            edge_list, adj_list = random_graph(n)\n",
    "            path = vertices_on_shortest_12_path(adj_list)\n",
    "            if len(path) > 1:\n",
    "                target_vtx_idx = random.randrange(1, len(path))\n",
    "                target_vtx = path[target_vtx_idx]\n",
    "                edge_list[-1] = target_vtx\n",
    "                graphs.append(edge_list)\n",
    "                distance.append(target_vtx_idx)\n",
    "                break\n",
    "    \n",
    "    data = torch.tensor(graphs, device=device)\n",
    "    labels = torch.tensor(distance, dtype=torch.bfloat16, device=device)\n",
    "    padding = data == PAD_TOKEN\n",
    "    return data, labels, padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Only need to run this once to generate training data\n",
    "# RESTART THE KERNEL BEFORE RUNNING AND ONLY RUN THE CELLS ABOVE\n",
    "# Python is slow and awful\n",
    "\n",
    "# with ProcessPoolExecutor() as executor:\n",
    "#     for i in range(1000):\n",
    "#         executor.submit(savebatch, 2**20, i)\n",
    "#     executor.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[ 1,  2,  5, 12,  3, 12,  2, 11,  9, 12,  2, 10,  1,  7,  1,  2,  9, 10,\n",
       "           1,  9,  4, 12,  0,  0,  0,  0,  0,  0,  0,  0,  2],\n",
       "         [ 9, 12,  4,  7,  8, 10,  5, 13,  1, 13,  3, 13,  7, 12,  5,  6,  3,  4,\n",
       "           6, 13,  2,  7,  0,  0,  0,  0,  0,  0,  0,  0,  2],\n",
       "         [ 1,  5,  8, 12,  2,  9,  2,  7,  5,  9, 10, 11,  6, 10,  4, 12,  1,  2,\n",
       "           4, 11,  2,  5,  2,  4,  0,  0,  0,  0,  0,  0,  2],\n",
       "         [ 5,  8,  3,  6,  4,  5,  2,  3,  4,  9,  3,  8,  5,  7,  4,  9,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  2],\n",
       "         [ 6, 13,  1, 13,  1,  4,  6, 13,  5,  7,  2,  4, 10, 12,  4,  6,  8, 11,\n",
       "           7, 11,  3,  8,  3,  5,  4, 12,  0,  0,  0,  0,  2]]),\n",
       " tensor([ 1.,  5.,  1., 15.,  2.], dtype=torch.bfloat16),\n",
       " tensor([[False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "          False],\n",
       "         [False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "          False],\n",
       "         [False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False,  True,  True,  True,  True,  True,  True,\n",
       "          False],\n",
       "         [False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False,  True,  True,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "          False],\n",
       "         [False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False,  True,  True,  True,  True,\n",
       "          False]]))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mkbatch(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([523.,   0.,   0.,   0., 390.,   0.,   0.,   0.,   0., 238.,   0.,\n",
       "          0.,   0.,  92.,   0.,   0.,   0.,   0.,  40.,   0.,   0.,   0.,\n",
       "         15.,   0.,   0.,   0.,   0.,   5.,   0.,   0.,   0.,   0.,   2.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0., 743.]),\n",
       " array([ 1.     ,  1.21875,  1.4375 ,  1.65625,  1.875  ,  2.09375,\n",
       "         2.3125 ,  2.53125,  2.75   ,  2.96875,  3.1875 ,  3.40625,\n",
       "         3.625  ,  3.84375,  4.0625 ,  4.28125,  4.5    ,  4.71875,\n",
       "         4.9375 ,  5.15625,  5.375  ,  5.59375,  5.8125 ,  6.03125,\n",
       "         6.25   ,  6.46875,  6.6875 ,  6.90625,  7.125  ,  7.34375,\n",
       "         7.5625 ,  7.78125,  8.     ,  8.21875,  8.4375 ,  8.65625,\n",
       "         8.875  ,  9.09375,  9.3125 ,  9.53125,  9.75   ,  9.96875,\n",
       "        10.1875 , 10.40625, 10.625  , 10.84375, 11.0625 , 11.28125,\n",
       "        11.5    , 11.71875, 11.9375 , 12.15625, 12.375  , 12.59375,\n",
       "        12.8125 , 13.03125, 13.25   , 13.46875, 13.6875 , 13.90625,\n",
       "        14.125  , 14.34375, 14.5625 , 14.78125, 15.     ]),\n",
       " <BarContainer object of 64 artists>)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAl0UlEQVR4nO3df1RU953/8Rc/B0RnCGyYkRWQ7rpFEoyJpjIxu+0qK7WsW480qTmU0MSNp57RBGmtsqsm1UQMu42pXdSa46p7Gtat56xpxcQEiSFtBSS47jGaJWZrAgmZoWctjJrDgDDfP75l2vFHklFkPsDzcc49J3M/n2Hel2PMM8MME+H3+/0CAAAwSGS4BwAAALgSgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAONHhHuBGDAwMqKOjQxMmTFBERES4xwEAAJ+D3+/XhQsXlJqaqsjIT3+OZEQGSkdHh9LS0sI9BgAAuAHt7e2aNGnSp+4ZkYEyYcIE6fcXaLVawz0OAAD4HLxer9LS0gL/Hf80IzJQBn+sY7VaCRQAAEaYz/PyDF4kCwAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA40SHewAAABAek9ccuu7a+5sLhnWWK/EMCgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA44QUKJMnT1ZERMRVh8vlkiT19PTI5XIpOTlZ48ePV2FhoTweT9DXaGtrU0FBgcaNG6eUlBStWrVKly9fHtqrAgAAI1pIgdLc3KyPP/44cNTW1kqSHnjgAUnSypUrdfDgQe3fv1/19fXq6OjQokWLAvfv7+9XQUGBent7dezYMe3du1d79uzR+vXrh/q6AADACBbh9/v9N3rn0tJS1dTU6OzZs/J6vbr99ttVXV2tb3zjG5Kk//mf/9HUqVPV0NCg3NxcvfLKK/rbv/1bdXR0yG63S5J27Nih1atX67e//a1iY2M/1+N6vV7ZbDZ1d3fLarXe6PgAAIxpk9ccuu7a+5sLhvzxQvnv9w2/BqW3t1c//elP9eijjyoiIkItLS3q6+tTXl5eYE9WVpbS09PV0NAgSWpoaFBOTk4gTiQpPz9fXq9Xp0+fvtFRAADAKBN9o3d86aWX1NXVpW9/+9uSJLfbrdjYWCUmJgbts9vtcrvdgT1/HCeD64Nr1+Pz+eTz+QK3vV7vjY4NAABGgBt+BmXXrl2aP3++UlNTh3aia6ioqJDNZgscaWlpt/wxAQBA+NxQoHzwwQc6cuSI/v7v/z5wzuFwqLe3V11dXUF7PR6PHA5HYM+V7+oZvD2451rKy8vV3d0dONrb229kbAAAMELcUKDs3r1bKSkpKij4wwtoZsyYoZiYGNXV1QXOtba2qq2tTU6nU5LkdDp16tQpdXZ2BvbU1tbKarUqOzv7uo9nsVhktVqDDgAAMHqF/BqUgYEB7d69WyUlJYqO/sPdbTablixZorKyMiUlJclqtWrFihVyOp3Kzc2VJM2bN0/Z2dkqLi5WZWWl3G631q5dK5fLJYvFMrRXBgAARqyQA+XIkSNqa2vTo48+etXali1bFBkZqcLCQvl8PuXn52vbtm2B9aioKNXU1GjZsmVyOp1KSEhQSUmJNmzYcPNXAgAARo2b+j0o4cLvQQEA4OaNyt+DAgAAcKsQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOOEHCgfffSRvvWtbyk5OVnx8fHKycnRW2+9FVj3+/1av369Jk6cqPj4eOXl5ens2bNBX+P8+fMqKiqS1WpVYmKilixZoosXLw7NFQEAgBEvpED53e9+p9mzZysmJkavvPKKzpw5ox/+8Ie67bbbAnsqKyu1detW7dixQ01NTUpISFB+fr56enoCe4qKinT69GnV1taqpqZGb775ppYuXTq0VwYAAEasCL/f7/+8m9esWaNf//rX+uUvf3nNdb/fr9TUVH33u9/V9773PUlSd3e37Ha79uzZo8WLF+udd95Rdna2mpubNXPmTEnS4cOH9bWvfU0ffvihUlNTP3MOr9crm82m7u5uWa3Wz3+1AAAgYPKaQ9dde39zwZA/Xij//Q7pGZRf/OIXmjlzph544AGlpKTo7rvv1gsvvBBYP3funNxut/Ly8gLnbDabZs2apYaGBklSQ0ODEhMTA3EiSXl5eYqMjFRTU9M1H9fn88nr9QYdAABg9AopUH7zm99o+/btmjJlil599VUtW7ZMjz/+uPbu3StJcrvdkiS73R50P7vdHlhzu91KSUkJWo+OjlZSUlJgz5UqKipks9kCR1paWmhXCQAARpSQAmVgYED33HOPNm3apLvvvltLly7VY489ph07dty6CSWVl5eru7s7cLS3t9/SxwMAAOEVUqBMnDhR2dnZQeemTp2qtrY2SZLD4ZAkeTyeoD0ejyew5nA41NnZGbR++fJlnT9/PrDnShaLRVarNegAAACjV0iBMnv2bLW2tgade/fdd5WRkSFJyszMlMPhUF1dXWDd6/WqqalJTqdTkuR0OtXV1aWWlpbAntdff10DAwOaNWvWzV4PAAAYBaJD2bxy5Urdd9992rRpkx588EEdP35cO3fu1M6dOyVJERERKi0t1dNPP60pU6YoMzNT69atU2pqqhYuXCj9/hmXr371q4EfDfX19Wn58uVavHjx53oHDwAAGP1CCpR7771XBw4cUHl5uTZs2KDMzEw9//zzKioqCuz5/ve/r0uXLmnp0qXq6urS/fffr8OHDysuLi6w58UXX9Ty5cs1d+5cRUZGqrCwUFu3bh3aKwMAACNWSL8HxRT8HhQAAG7eqPk9KAAAAMOBQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxokO9wAmmrzm0HXX3t9cMKyzAAAwFvEMCgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwTkiB8tRTTykiIiLoyMrKCqz39PTI5XIpOTlZ48ePV2FhoTweT9DXaGtrU0FBgcaNG6eUlBStWrVKly9fHrorAgAAI17IbzO+4447dOTIkT98geg/fImVK1fq0KFD2r9/v2w2m5YvX65Fixbp17/+tSSpv79fBQUFcjgcOnbsmD7++GM9/PDDiomJ0aZNm4bqmgAAwAgXcqBER0fL4XBcdb67u1u7du1SdXW15syZI0navXu3pk6dqsbGRuXm5uq1117TmTNndOTIEdntdk2fPl0bN27U6tWr9dRTTyk2NnZorgoAAIxoIb8G5ezZs0pNTdUXvvAFFRUVqa2tTZLU0tKivr4+5eXlBfZmZWUpPT1dDQ0NkqSGhgbl5OTIbrcH9uTn58vr9er06dPXfUyfzyev1xt0AACA0SukQJk1a5b27Nmjw4cPa/v27Tp37pz+8i//UhcuXJDb7VZsbKwSExOD7mO32+V2uyVJbrc7KE4G1wfXrqeiokI2my1wpKWlhTI2AAAYYUL6Ec/8+fMD/zxt2jTNmjVLGRkZ+tnPfqb4+PhbMZ8kqby8XGVlZYHbXq+XSAEAYBS7qbcZJyYm6i/+4i/03nvvyeFwqLe3V11dXUF7PB5P4DUrDofjqnf1DN6+1utaBlksFlmt1qADAACMXjcVKBcvXtT//u//auLEiZoxY4ZiYmJUV1cXWG9tbVVbW5ucTqckyel06tSpU+rs7Azsqa2tldVqVXZ29s2MAgAARpGQfsTzve99TwsWLFBGRoY6Ojr05JNPKioqSg899JBsNpuWLFmisrIyJSUlyWq1asWKFXI6ncrNzZUkzZs3T9nZ2SouLlZlZaXcbrfWrl0rl8sli8Vyq64RAACMMCEFyocffqiHHnpI//d//6fbb79d999/vxobG3X77bdLkrZs2aLIyEgVFhbK5/MpPz9f27ZtC9w/KipKNTU1WrZsmZxOpxISElRSUqINGzYM/ZUBAIARK6RA2bdv36eux8XFqaqqSlVVVdfdk5GRoZdffjmUhwUAAGMMn8UDAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMc1OBsnnzZkVERKi0tDRwrqenRy6XS8nJyRo/frwKCwvl8XiC7tfW1qaCggKNGzdOKSkpWrVqlS5fvnwzowAAgFHkhgOlublZP/nJTzRt2rSg8ytXrtTBgwe1f/9+1dfXq6OjQ4sWLQqs9/f3q6CgQL29vTp27Jj27t2rPXv2aP369Td3JQAAYNS4oUC5ePGiioqK9MILL+i2224LnO/u7tauXbv03HPPac6cOZoxY4Z2796tY8eOqbGxUZL02muv6cyZM/rpT3+q6dOna/78+dq4caOqqqrU29s7dFcGAABGrBsKFJfLpYKCAuXl5QWdb2lpUV9fX9D5rKwspaenq6GhQZLU0NCgnJwc2e32wJ78/Hx5vV6dPn36mo/n8/nk9XqDDgAAMHpFh3qHffv26cSJE2pubr5qze12KzY2VomJiUHn7Xa73G53YM8fx8ng+uDatVRUVOgHP/hBqKMCAIARKqRnUNrb2/XEE0/oxRdfVFxc3K2b6grl5eXq7u4OHO3t7cP22AAAYPiFFCgtLS3q7OzUPffco+joaEVHR6u+vl5bt25VdHS07Ha7ent71dXVFXQ/j8cjh8MhSXI4HFe9q2fw9uCeK1ksFlmt1qADAACMXiEFyty5c3Xq1CmdPHkycMycOVNFRUWBf46JiVFdXV3gPq2trWpra5PT6ZQkOZ1OnTp1Sp2dnYE9tbW1slqtys7OHsprAwAAI1RIr0GZMGGC7rzzzqBzCQkJSk5ODpxfsmSJysrKlJSUJKvVqhUrVsjpdCo3N1eSNG/ePGVnZ6u4uFiVlZVyu91au3atXC6XLBbLUF4bAAAYoUJ+kexn2bJliyIjI1VYWCifz6f8/Hxt27YtsB4VFaWamhotW7ZMTqdTCQkJKikp0YYNG4Z6lBFj8ppD1117f3PBsM4CAIAJbjpQ3njjjaDbcXFxqqqqUlVV1XXvk5GRoZdffvlmHxoAAIxSfBYPAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOCEFyvbt2zVt2jRZrVZZrVY5nU698sorgfWenh65XC4lJydr/PjxKiwslMfjCfoabW1tKigo0Lhx45SSkqJVq1bp8uXLQ3dFAABgxAspUCZNmqTNmzerpaVFb731lubMmaOvf/3rOn36tCRp5cqVOnjwoPbv36/6+np1dHRo0aJFgfv39/eroKBAvb29OnbsmPbu3as9e/Zo/fr1Q39lAABgxIoOZfOCBQuCbj/zzDPavn27GhsbNWnSJO3atUvV1dWaM2eOJGn37t2aOnWqGhsblZubq9dee01nzpzRkSNHZLfbNX36dG3cuFGrV6/WU089pdjY2KG9OgAAMCLd8GtQ+vv7tW/fPl26dElOp1MtLS3q6+tTXl5eYE9WVpbS09PV0NAgSWpoaFBOTo7sdntgT35+vrxeb+BZmGvx+Xzyer1BBwAAGL1CDpRTp05p/Pjxslgs+s53vqMDBw4oOztbbrdbsbGxSkxMDNpvt9vldrslSW63OyhOBtcH166noqJCNpstcKSlpYU6NgAAGEFCDpQvfvGLOnnypJqamrRs2TKVlJTozJkzt2a63ysvL1d3d3fgaG9vv6WPBwAAwiuk16BIUmxsrP78z/9ckjRjxgw1NzfrRz/6kb75zW+qt7dXXV1dQc+ieDweORwOSZLD4dDx48eDvt7gu3wG91yLxWKRxWIJdVQAADBC3fTvQRkYGJDP59OMGTMUExOjurq6wFpra6va2trkdDolSU6nU6dOnVJnZ2dgT21traxWq7Kzs292FAAAMEqE9AxKeXm55s+fr/T0dF24cEHV1dV644039Oqrr8pms2nJkiUqKytTUlKSrFarVqxYIafTqdzcXEnSvHnzlJ2dreLiYlVWVsrtdmvt2rVyuVw8QwIAAAJCCpTOzk49/PDD+vjjj2Wz2TRt2jS9+uqr+pu/+RtJ0pYtWxQZGanCwkL5fD7l5+dr27ZtgftHRUWppqZGy5Ytk9PpVEJCgkpKSrRhw4ahvzIAADBihRQou3bt+tT1uLg4VVVVqaqq6rp7MjIy9PLLL4fysAAAYIzhs3gAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYJzrcA8Ask9ccuu7a+5sLhnUWAMDYxTMoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA44QUKBUVFbr33ns1YcIEpaSkaOHChWptbQ3a09PTI5fLpeTkZI0fP16FhYXyeDxBe9ra2lRQUKBx48YpJSVFq1at0uXLl4fmigAAwIgXUqDU19fL5XKpsbFRtbW16uvr07x583Tp0qXAnpUrV+rgwYPav3+/6uvr1dHRoUWLFgXW+/v7VVBQoN7eXh07dkx79+7Vnj17tH79+qG9MgAAMGJFh7L58OHDQbf37NmjlJQUtbS06K/+6q/U3d2tXbt2qbq6WnPmzJEk7d69W1OnTlVjY6Nyc3P12muv6cyZMzpy5IjsdrumT5+ujRs3avXq1XrqqacUGxs7tFcIAABGnJt6DUp3d7ckKSkpSZLU0tKivr4+5eXlBfZkZWUpPT1dDQ0NkqSGhgbl5OTIbrcH9uTn58vr9er06dM3Mw4AABglQnoG5Y8NDAyotLRUs2fP1p133ilJcrvdio2NVWJiYtBeu90ut9sd2PPHcTK4Prh2LT6fTz6fL3Db6/Xe6NgAAGAEuOFnUFwul95++23t27dvaCe6hoqKCtlstsCRlpZ2yx8TAACEzw0FyvLly1VTU6OjR49q0qRJgfMOh0O9vb3q6uoK2u/xeORwOAJ7rnxXz+DtwT1XKi8vV3d3d+Bob2+/kbEBAMAIEVKg+P1+LV++XAcOHNDrr7+uzMzMoPUZM2YoJiZGdXV1gXOtra1qa2uT0+mUJDmdTp06dUqdnZ2BPbW1tbJarcrOzr7m41osFlmt1qADAACMXiG9BsXlcqm6ulo///nPNWHChMBrRmw2m+Lj42Wz2bRkyRKVlZUpKSlJVqtVK1askNPpVG5uriRp3rx5ys7OVnFxsSorK+V2u7V27Vq5XC5ZLJZbc5UAAGBECSlQtm/fLkn6yle+EnR+9+7d+va3vy1J2rJliyIjI1VYWCifz6f8/Hxt27YtsDcqKko1NTVatmyZnE6nEhISVFJSog0bNgzNFQEAgBEvpEDx+/2fuScuLk5VVVWqqqq67p6MjAy9/PLLoTw0AAAYQ/gsHgAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYJyQA+XNN9/UggULlJqaqoiICL300ktB636/X+vXr9fEiRMVHx+vvLw8nT17NmjP+fPnVVRUJKvVqsTERC1ZskQXL168+asBAACjQsiBcunSJd11112qqqq65nplZaW2bt2qHTt2qKmpSQkJCcrPz1dPT09gT1FRkU6fPq3a2lrV1NTozTff1NKlS2/uSgAAwKgRHeod5s+fr/nz519zze/36/nnn9fatWv19a9/XZL0b//2b7Lb7XrppZe0ePFivfPOOzp8+LCam5s1c+ZMSdKPf/xjfe1rX9M///M/KzU19WavCQAAjHBD+hqUc+fOye12Ky8vL3DOZrNp1qxZamhokCQ1NDQoMTExECeSlJeXp8jISDU1NV3z6/p8Pnm93qADAACMXkMaKG63W5Jkt9uDztvt9sCa2+1WSkpK0Hp0dLSSkpICe65UUVEhm80WONLS0oZybAAAYJgR8S6e8vJydXd3B4729vZwjwQAAG6hIQ0Uh8MhSfJ4PEHnPR5PYM3hcKizszNo/fLlyzp//nxgz5UsFousVmvQAQAARq8hDZTMzEw5HA7V1dUFznm9XjU1NcnpdEqSnE6nurq61NLSEtjz+uuva2BgQLNmzRrKcQAAwAgV8rt4Ll68qPfeey9w+9y5czp58qSSkpKUnp6u0tJSPf3005oyZYoyMzO1bt06paamauHChZKkqVOn6qtf/aoee+wx7dixQ319fVq+fLkWL17MO3jGoMlrDl137f3NBcM6CwDAHCEHyltvvaW//uu/DtwuKyuTJJWUlGjPnj36/ve/r0uXLmnp0qXq6urS/fffr8OHDysuLi5wnxdffFHLly/X3LlzFRkZqcLCQm3dunWorgkAAIxwIQfKV77yFfn9/uuuR0REaMOGDdqwYcN19yQlJam6ujrUhwYAAGPEiHgXDwAAGFsIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxosM9AGCKyWsOfer6+5sLhm0WABjreAYFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcfgsHiCMPu3zf/jsHwBjGc+gAAAA4xAoAADAOGENlKqqKk2ePFlxcXGaNWuWjh8/Hs5xAACAIcIWKP/xH/+hsrIyPfnkkzpx4oTuuusu5efnq7OzM1wjAQAAQ4TtRbLPPfecHnvsMT3yyCOSpB07dujQoUP613/9V61ZsyZcYwFjFi/YBWCSsARKb2+vWlpaVF5eHjgXGRmpvLw8NTQ0XLXf5/PJ5/MFbnd3d0uSvF7vLZlvwPfJddduxWMO9+OZNMtIuXaN8esf7lnufPLV6669/YP8YZ0FGM2G+9/7wa/p9/s/e7M/DD766CO/JP+xY8eCzq9atcr/pS996ar9Tz75pF8SBwcHBwcHxyg42tvbP7MVRsTvQSkvL1dZWVng9sDAgM6fP6/k5GRFRESEdbah5vV6lZaWpvb2dlmt1nCPM+zG+vWL78GYv37xPeD6R/H1+/1+XbhwQampqZ+5NyyB8id/8ieKioqSx+MJOu/xeORwOK7ab7FYZLFYgs4lJibe8jnDyWq1jro/mKEY69cvvgdj/vrF94DrH6XXb7PZPte+sLyLJzY2VjNmzFBdXV3g3MDAgOrq6uR0OsMxEgAAMEjYfsRTVlamkpISzZw5U1/60pf0/PPP69KlS4F39QAAgLErbIHyzW9+U7/97W+1fv16ud1uTZ8+XYcPH5bdbg/XSEawWCx68sknr/qR1lgx1q9ffA/G/PWL7wHXP8avf1CE/3O91wcAAGD48Fk8AADAOAQKAAAwDoECAACMQ6AAAADjECgGqKio0L333qsJEyYoJSVFCxcuVGtra7jHCqvNmzcrIiJCpaWl4R5l2Hz00Uf61re+peTkZMXHxysnJ0dvvfVWuMcaNv39/Vq3bp0yMzMVHx+vP/uzP9PGjRs/32d2jEBvvvmmFixYoNTUVEVEROill14KWvf7/Vq/fr0mTpyo+Ph45eXl6ezZs2Gb91b4tO9BX1+fVq9erZycHCUkJCg1NVUPP/ywOjo6wjrzUPqsPwN/7Dvf+Y4iIiL0/PPPD+uM4USgGKC+vl4ul0uNjY2qra1VX1+f5s2bp0uXLoV7tLBobm7WT37yE02bNi3cowyb3/3ud5o9e7ZiYmL0yiuv6MyZM/rhD3+o2267LdyjDZtnn31W27dv17/8y7/onXfe0bPPPqvKykr9+Mc/Dvdot8SlS5d01113qaqq6prrlZWV2rp1q3bs2KGmpiYlJCQoPz9fPT09wz7rrfJp34NPPvlEJ06c0Lp163TixAn953/+p1pbW/V3f/d3YZn1VvisPwODDhw4oMbGxs/16+FHlaH8EEAMjc7OTr8kf319fbhHGXYXLlzwT5kyxV9bW+v/8pe/7H/iiSfCPdKwWL16tf/+++8P9xhhVVBQ4H/00UeDzi1atMhfVFQUtpmGiyT/gQMHArcHBgb8DofD/0//9E+Bc11dXX6LxeL/93//9zBNeWtd+T24luPHj/sl+T/44INhm2u4XO/6P/zwQ/+f/umf+t9++21/RkaGf8uWLWGZLxx4BsVA3d3dkqSkpKRwjzLsXC6XCgoKlJeXF+5RhtUvfvELzZw5Uw888IBSUlJ0991364UXXgj3WMPqvvvuU11dnd59911J0n//93/rV7/6lebPnx/u0YbduXPn5Ha7g/49sNlsmjVrlhoaGsI6Wzh1d3crIiJi1H8W26CBgQEVFxdr1apVuuOOO8I9zrAbEZ9mPJYMDAyotLRUs2fP1p133hnucYbVvn37dOLECTU3N4d7lGH3m9/8Rtu3b1dZWZn+4R/+Qc3NzXr88ccVGxurkpKScI83LNasWSOv16usrCxFRUWpv79fzzzzjIqKisI92rBzu92SdNVv1rbb7YG1saanp0erV6/WQw89NCo/QO9ann32WUVHR+vxxx8P9yhhQaAYxuVy6e2339avfvWrcI8yrNrb2/XEE0+otrZWcXFx4R5n2A0MDGjmzJnatGmTJOnuu+/W22+/rR07doyZQPnZz36mF198UdXV1brjjjt08uRJlZaWKjU1dcx8D3BtfX19evDBB+X3+7V9+/ZwjzMsWlpa9KMf/UgnTpxQREREuMcJC37EY5Dly5erpqZGR48e1aRJk8I9zrBqaWlRZ2en7rnnHkVHRys6Olr19fXaunWroqOj1d/fH+4Rb6mJEycqOzs76NzUqVPV1tYWtpmG26pVq7RmzRotXrxYOTk5Ki4u1sqVK1VRURHu0Yadw+GQJHk8nqDzHo8nsDZWDMbJBx98oNra2jHz7Mkvf/lLdXZ2Kj09PfB34gcffKDvfve7mjx5crjHGxY8g2IAv9+vFStW6MCBA3rjjTeUmZkZ7pGG3dy5c3Xq1Kmgc4888oiysrK0evVqRUVFhW224TB79uyr3lr+7rvvKiMjI2wzDbdPPvlEkZHB/88UFRWlgYGBsM0ULpmZmXI4HKqrq9P06dMlSV6vV01NTVq2bFm4xxs2g3Fy9uxZHT16VMnJyeEeadgUFxdf9Vq8/Px8FRcX65FHHgnbXMOJQDGAy+VSdXW1fv7zn2vChAmBnzHbbDbFx8eHe7xhMWHChKtec5OQkKDk5OQx8VqclStX6r777tOmTZv04IMP6vjx49q5c6d27twZ7tGGzYIFC/TMM88oPT1dd9xxh/7rv/5Lzz33nB599NFwj3ZLXLx4Ue+9917g9rlz53Ty5EklJSUpPT1dpaWlevrppzVlyhRlZmZq3bp1Sk1N1cKFC8M691D6tO/BxIkT9Y1vfEMnTpxQTU2N+vv7A383JiUlKTY2NoyTD43P+jNwZZDFxMTI4XDoi1/8YhimDYNwv40I///tZdc6du/eHe7Rwmosvc3Y7/f7Dx486L/zzjv9FovFn5WV5d+5c2e4RxpWXq/X/8QTT/jT09P9cXFx/i984Qv+f/zHf/T7fL5wj3ZLHD169Jr/3peUlPj9v3+r8bp16/x2u91vsVj8c+fO9be2toZ77CH1ad+Dc+fOXffvxqNHj4Z79CHxWX8GrjTW3mYc4R+tv6YRAACMWLxIFgAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYJz/B0FPZLxO9+txAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(mkbatch(2048)[1].to(torch.float32).cpu(), bins=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([1516.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,\n",
       "           0.,    0.,    0.,    0.,    0.,    0.,    0.,  420.,    0.,\n",
       "           0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,\n",
       "           0.,    0.,    0.,    0.,    0.,   94.,    0.,    0.,    0.,\n",
       "           0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,\n",
       "           0.,    0.,    0.,   16.,    0.,    0.,    0.,    0.,    0.,\n",
       "           0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,\n",
       "           2.]),\n",
       " array([1.    , 1.0625, 1.125 , 1.1875, 1.25  , 1.3125, 1.375 , 1.4375,\n",
       "        1.5   , 1.5625, 1.625 , 1.6875, 1.75  , 1.8125, 1.875 , 1.9375,\n",
       "        2.    , 2.0625, 2.125 , 2.1875, 2.25  , 2.3125, 2.375 , 2.4375,\n",
       "        2.5   , 2.5625, 2.625 , 2.6875, 2.75  , 2.8125, 2.875 , 2.9375,\n",
       "        3.    , 3.0625, 3.125 , 3.1875, 3.25  , 3.3125, 3.375 , 3.4375,\n",
       "        3.5   , 3.5625, 3.625 , 3.6875, 3.75  , 3.8125, 3.875 , 3.9375,\n",
       "        4.    , 4.0625, 4.125 , 4.1875, 4.25  , 4.3125, 4.375 , 4.4375,\n",
       "        4.5   , 4.5625, 4.625 , 4.6875, 4.75  , 4.8125, 4.875 , 4.9375,\n",
       "        5.    ]),\n",
       " <BarContainer object of 64 artists>)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAoW0lEQVR4nO3df1RU94H//9cgAsY4g2iZcTaotE1VjJpEEjLmd2VFpe56SjehYa3bstrNQhqjMcJpNZqkxZisRlMqsW2CZ2tOfuxZ3cYkGKpVWkMQUValhprUKKkZyB5kRshHRLnfP771noyCghmEN3k+zrnnZO59z9z3u++e8Mw4jA7LsiwBAAAYJKK3JwAAANBdBAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA40T29gR6Snt7u06cOKEhQ4bI4XD09nQAAEAXWJalU6dOyev1KiKi8/dZ+m3AnDhxQgkJCb09DQAAcAXq6up03XXXdXq93wbMkCFDpL/9D+B0Ont7OgAAoAuCwaASEhLsn+Od6bcBc/6PjZxOJwEDAIBhLvfxDz7ECwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA40T29gRMNDrvzU6vfbQy/arOBQCALyPegQEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxuh0wZWVlmjVrlrxerxwOh7Zs2dLp2H/7t3+Tw+HQc889F3K+sbFRWVlZcjqdio2NVXZ2tpqbm0PGHDhwQHfeeadiYmKUkJCgVatWdXeqAACgn+p2wLS0tGjSpEkqLCy85LjNmzfrvffek9frvehaVlaWampqVFpaqq1bt6qsrEzz58+3rweDQU2bNk2jRo1SVVWVnnnmGS1fvlwbNmzo7nQBAEA/FNndJ8yYMUMzZsy45Ji//vWveuihh7Rt2zalp6eHXDt8+LBKSkpUWVmp5ORkSdLzzz+vmTNn6tlnn5XX69WmTZt05swZvfjii4qKitL48eNVXV2t1atXh4QOAAD4cgr7Z2Da29s1Z84cLV68WOPHj7/oenl5uWJjY+14kaTU1FRFRESooqLCHnPXXXcpKirKHpOWlqba2lqdPHmyw/u2trYqGAyGHAAAoH8Ke8A8/fTTioyM1I9+9KMOr/v9fsXHx4eci4yMVFxcnPx+vz3G7XaHjDn/+PyYCxUUFMjlctlHQkJCmFYEAAD6mrAGTFVVldauXavi4mI5HI5wvvRl5efnKxAI2EddXd1VvT8AALh6whowf/jDH9TQ0KCRI0cqMjJSkZGROnbsmBYtWqTRo0dLkjwejxoaGkKed/bsWTU2Nsrj8dhj6uvrQ8acf3x+zIWio6PldDpDDgAA0D+FNWDmzJmjAwcOqLq62j68Xq8WL16sbdu2SZJ8Pp+amppUVVVlP2/Hjh1qb29XSkqKPaasrExtbW32mNLSUo0ZM0ZDhw4N55QBAICBuv1bSM3Nzfrggw/sx0ePHlV1dbXi4uI0cuRIDRs2LGT8wIED5fF4NGbMGEnSuHHjNH36dM2bN09FRUVqa2tTbm6uMjMz7V+5fuCBB7RixQplZ2dryZIlOnTokNauXas1a9Z88RUDAADjdTtg9u7dq3vvvdd+vHDhQknS3LlzVVxc3KXX2LRpk3JzczV16lRFREQoIyND69ats6+7XC698847ysnJ0eTJkzV8+HAtW7aMX6EGAACSJIdlWVZvT6InBINBuVwuBQKBsH8eZnTem51e+2hleqfXAADApXX15zd/FxIAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjdDtgysrKNGvWLHm9XjkcDm3ZssW+1tbWpiVLlmjChAkaPHiwvF6vvve97+nEiRMhr9HY2KisrCw5nU7FxsYqOztbzc3NIWMOHDigO++8UzExMUpISNCqVau+yDoBAEA/0u2AaWlp0aRJk1RYWHjRtc8++0z79u3T0qVLtW/fPv33f/+3amtr9Q//8A8h47KyslRTU6PS0lJt3bpVZWVlmj9/vn09GAxq2rRpGjVqlKqqqvTMM89o+fLl2rBhw5WuEwAA9CMOy7KsK36yw6HNmzdr9uzZnY6prKzUrbfeqmPHjmnkyJE6fPiwkpKSVFlZqeTkZElSSUmJZs6cqY8//lher1fr16/Xj3/8Y/n9fkVFRUmS8vLytGXLFr3//vtdmlswGJTL5VIgEJDT6bzSJXZodN6bnV77aGV6WO8FAMCXSVd/fvf4Z2ACgYAcDodiY2MlSeXl5YqNjbXjRZJSU1MVERGhiooKe8xdd91lx4skpaWlqba2VidPnuzpKQMAgD4usidf/PTp01qyZIm++93v2hXl9/sVHx8fOonISMXFxcnv99tjEhMTQ8a43W772tChQy+6V2trq1pbW+3HwWCwR9YEAAB6X4+9A9PW1qb77rtPlmVp/fr1PXUbW0FBgVwul30kJCT0+D0BAEDv6JGAOR8vx44dU2lpacifYXk8HjU0NISMP3v2rBobG+XxeOwx9fX1IWPOPz4/5kL5+fkKBAL2UVdX1wMrAwAAfUHYA+Z8vBw5ckS/+93vNGzYsJDrPp9PTU1Nqqqqss/t2LFD7e3tSklJsceUlZWpra3NHlNaWqoxY8Z0+MdHkhQdHS2n0xlyAACA/qnbAdPc3Kzq6mpVV1dLko4eParq6modP35cbW1t+s53vqO9e/dq06ZNOnfunPx+v/x+v86cOSNJGjdunKZPn6558+Zpz5492r17t3Jzc5WZmSmv1ytJeuCBBxQVFaXs7GzV1NTo1Vdf1dq1a7Vw4cJwrx8AABio279GvXPnTt17770XnZ87d66WL19+0Ydvz/v973+ve+65R/rbF9nl5ubqjTfeUEREhDIyMrRu3Tpde+219vgDBw4oJydHlZWVGj58uB566CEtWbKky/Pk16gBADBPV39+f6HvgenLCBgAAMzTZ74HBgAAINwIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMbpdsCUlZVp1qxZ8nq9cjgc2rJlS8h1y7K0bNkyjRgxQoMGDVJqaqqOHDkSMqaxsVFZWVlyOp2KjY1Vdna2mpubQ8YcOHBAd955p2JiYpSQkKBVq1Zd6RoBAEA/0+2AaWlp0aRJk1RYWNjh9VWrVmndunUqKipSRUWFBg8erLS0NJ0+fdoek5WVpZqaGpWWlmrr1q0qKyvT/Pnz7evBYFDTpk3TqFGjVFVVpWeeeUbLly/Xhg0brnSdAACgH3FYlmVd8ZMdDm3evFmzZ8+W/vbui9fr1aJFi/Too49KkgKBgNxut4qLi5WZmanDhw8rKSlJlZWVSk5OliSVlJRo5syZ+vjjj+X1erV+/Xr9+Mc/lt/vV1RUlCQpLy9PW7Zs0fvvv9+luQWDQblcLgUCATmdzitdYodG573Z6bWPVqaH9V4AAHyZdPXnd1g/A3P06FH5/X6lpqba51wul1JSUlReXi5JKi8vV2xsrB0vkpSamqqIiAhVVFTYY+666y47XiQpLS1NtbW1OnnyZIf3bm1tVTAYDDkAAED/FNaA8fv9kiS32x1y3u1229f8fr/i4+NDrkdGRiouLi5kTEev8fl7XKigoEAul8s+EhISwrgyAADQl/Sb30LKz89XIBCwj7q6ut6eEgAA6CFhDRiPxyNJqq+vDzlfX19vX/N4PGpoaAi5fvbsWTU2NoaM6eg1Pn+PC0VHR8vpdIYcAACgfwprwCQmJsrj8Wj79u32uWAwqIqKCvl8PkmSz+dTU1OTqqqq7DE7duxQe3u7UlJS7DFlZWVqa2uzx5SWlmrMmDEaOnRoOKcMAAAM1O2AaW5uVnV1taqrq6W/fXC3urpax48fl8Ph0IIFC/TUU0/pt7/9rQ4ePKjvfe978nq99m8qjRs3TtOnT9e8efO0Z88e7d69W7m5ucrMzJTX65UkPfDAA4qKilJ2drZqamr06quvau3atVq4cGG41w8AAAwU2d0n7N27V/fee6/9+HxUzJ07V8XFxXrsscfU0tKi+fPnq6mpSXfccYdKSkoUExNjP2fTpk3Kzc3V1KlTFRERoYyMDK1bt86+7nK59M477ygnJ0eTJ0/W8OHDtWzZspDvigEAAF9eX+h7YPoyvgcGAADz9Mr3wAAAAFwNBAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACME/aAOXfunJYuXarExEQNGjRIX/va1/Tkk0/Ksix7jGVZWrZsmUaMGKFBgwYpNTVVR44cCXmdxsZGZWVlyel0KjY2VtnZ2Wpubg73dAEAgIHCHjBPP/201q9fr5///Oc6fPiwnn76aa1atUrPP/+8PWbVqlVat26dioqKVFFRocGDBystLU2nT5+2x2RlZammpkalpaXaunWrysrKNH/+/HBPFwAAGMhhff6tkTD41re+JbfbrV//+tf2uYyMDA0aNEi/+c1vZFmWvF6vFi1apEcffVSSFAgE5Ha7VVxcrMzMTB0+fFhJSUmqrKxUcnKyJKmkpEQzZ87Uxx9/LK/Xe9l5BINBuVwuBQIBOZ3OcC5Ro/Pe7PTaRyvTw3ovAAC+TLr68zvs78BMmTJF27dv15///GdJ0v/+7//qj3/8o2bMmCFJOnr0qPx+v1JTU+3nuFwupaSkqLy8XJJUXl6u2NhYO14kKTU1VREREaqoqAj3lAEAgGEiw/2CeXl5CgaDGjt2rAYMGKBz587ppz/9qbKysiRJfr9fkuR2u0Oe53a77Wt+v1/x8fGhE42MVFxcnD3mQq2trWptbbUfB4PBcC8NAAD0EWF/B+a1117Tpk2b9PLLL2vfvn3auHGjnn32WW3cuDHctwpRUFAgl8tlHwkJCT16PwAA0HvCHjCLFy9WXl6eMjMzNWHCBM2ZM0ePPPKICgoKJEkej0eSVF9fH/K8+vp6+5rH41FDQ0PI9bNnz6qxsdEec6H8/HwFAgH7qKurC/fSAABAHxH2gPnss88UERH6sgMGDFB7e7skKTExUR6PR9u3b7evB4NBVVRUyOfzSZJ8Pp+amppUVVVlj9mxY4fa29uVkpLS4X2jo6PldDpDDgAA0D+F/TMws2bN0k9/+lONHDlS48eP1/79+7V69Wr94Ac/kCQ5HA4tWLBATz31lK6//nolJiZq6dKl8nq9mj17tiRp3Lhxmj59uubNm6eioiK1tbUpNzdXmZmZXfoNJAAA0L+FPWCef/55LV26VP/+7/+uhoYGeb1e/fCHP9SyZcvsMY899phaWlo0f/58NTU16Y477lBJSYliYmLsMZs2bVJubq6mTp2qiIgIZWRkaN26deGeLgAAMFDYvwemr+B7YAAAME+vfQ8MAABATyNgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGKdHAuavf/2r/vmf/1nDhg3ToEGDNGHCBO3du9e+blmWli1bphEjRmjQoEFKTU3VkSNHQl6jsbFRWVlZcjqdio2NVXZ2tpqbm3tiugAAwDBhD5iTJ0/q9ttv18CBA/X222/rT3/6k/7jP/5DQ4cOtcesWrVK69atU1FRkSoqKjR48GClpaXp9OnT9pisrCzV1NSotLRUW7duVVlZmebPnx/u6QIAAAM5LMuywvmCeXl52r17t/7whz90eN2yLHm9Xi1atEiPPvqoJCkQCMjtdqu4uFiZmZk6fPiwkpKSVFlZqeTkZElSSUmJZs6cqY8//lher/ey8wgGg3K5XAoEAnI6neFcokbnvdnptY9Wpof1XgAAfJl09ed32N+B+e1vf6vk5GT90z/9k+Lj43XTTTfpl7/8pX396NGj8vv9Sk1Ntc+5XC6lpKSovLxcklReXq7Y2Fg7XiQpNTVVERERqqio6PC+ra2tCgaDIQcAAOifwh4wf/nLX7R+/Xpdf/312rZtmx588EH96Ec/0saNGyVJfr9fkuR2u0Oe53a77Wt+v1/x8fEh1yMjIxUXF2ePuVBBQYFcLpd9JCQkhHtpAACgjwh7wLS3t+vmm2/Wz372M910002aP3++5s2bp6KionDfKkR+fr4CgYB91NXV9ej9AABA7wl7wIwYMUJJSUkh58aNG6fjx49LkjwejySpvr4+ZEx9fb19zePxqKGhIeT62bNn1djYaI+5UHR0tJxOZ8gBAAD6p7AHzO23367a2tqQc3/+8581atQoSVJiYqI8Ho+2b99uXw8Gg6qoqJDP55Mk+Xw+NTU1qaqqyh6zY8cOtbe3KyUlJdxTBgAAhokM9ws+8sgjmjJlin72s5/pvvvu0549e7RhwwZt2LBBkuRwOLRgwQI99dRTuv7665WYmKilS5fK6/Vq9uzZ0t/esZk+fbr9R09tbW3Kzc1VZmZml34DCQAA9G9hD5hbbrlFmzdvVn5+vp544gklJibqueeeU1ZWlj3mscceU0tLi+bPn6+mpibdcccdKikpUUxMjD1m06ZNys3N1dSpUxUREaGMjAytW7cu3NMFAAAGCvv3wPQVfA8MAADm6bXvgQEAAOhpBAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACM0+MBs3LlSjkcDi1YsMA+d/r0aeXk5GjYsGG69tprlZGRofr6+pDnHT9+XOnp6brmmmsUHx+vxYsX6+zZsz09XXxJjM57s9MDAND39WjAVFZW6oUXXtDEiRNDzj/yyCN644039Prrr2vXrl06ceKEvv3tb9vXz507p/T0dJ05c0bvvvuuNm7cqOLiYi1btqwnpwsAAAzRYwHT3NysrKws/fKXv9TQoUPt84FAQL/+9a+1evVqffOb39TkyZP10ksv6d1339V7770nSXrnnXf0pz/9Sb/5zW904403asaMGXryySdVWFioM2fO9NSUAQCAIXosYHJycpSenq7U1NSQ81VVVWpraws5P3bsWI0cOVLl5eWSpPLyck2YMEFut9sek5aWpmAwqJqamg7v19raqmAwGHIAAID+KbInXvSVV17Rvn37VFlZedE1v9+vqKgoxcbGhpx3u93y+/32mM/Hy/nr5691pKCgQCtWrAjjKgAAQF8V9ndg6urq9PDDD2vTpk2KiYkJ98t3Kj8/X4FAwD7q6uqu2r0BAMDVFfaAqaqqUkNDg26++WZFRkYqMjJSu3bt0rp16xQZGSm3260zZ86oqakp5Hn19fXyeDySJI/Hc9FvJZ1/fH7MhaKjo+V0OkMOAADQP4U9YKZOnaqDBw+qurraPpKTk5WVlWX/88CBA7V9+3b7ObW1tTp+/Lh8Pp8kyefz6eDBg2poaLDHlJaWyul0KikpKdxTBgAAhgn7Z2CGDBmiG264IeTc4MGDNWzYMPt8dna2Fi5cqLi4ODmdTj300EPy+Xy67bbbJEnTpk1TUlKS5syZo1WrVsnv9+snP/mJcnJyFB0dHe4pAwAAw/TIh3gvZ82aNYqIiFBGRoZaW1uVlpamX/ziF/b1AQMGaOvWrXrwwQfl8/k0ePBgzZ07V0888URvTBcAAPQxVyVgdu7cGfI4JiZGhYWFKiws7PQ5o0aN0ltvvXUVZgcAAEzD34UEAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwTtgDpqCgQLfccouGDBmi+Ph4zZ49W7W1tSFjTp8+rZycHA0bNkzXXnutMjIyVF9fHzLm+PHjSk9P1zXXXKP4+HgtXrxYZ8+eDfd0AQCAgcIeMLt27VJOTo7ee+89lZaWqq2tTdOmTVNLS4s95pFHHtEbb7yh119/Xbt27dKJEyf07W9/275+7tw5paen68yZM3r33Xe1ceNGFRcXa9myZeGeLgAAMJDDsiyrJ2/w6aefKj4+Xrt27dJdd92lQCCgr3zlK3r55Zf1ne98R5L0/vvva9y4cSovL9dtt92mt99+W9/61rd04sQJud1uSVJRUZGWLFmiTz/9VFFRUZe9bzAYlMvlUiAQkNPpDOuaRue92em1j1amh/Ve6BnsIQD0TV39+d3jn4EJBAKSpLi4OElSVVWV2tralJqaao8ZO3asRo4cqfLycklSeXm5JkyYYMeLJKWlpSkYDKqmpqbD+7S2tioYDIYcAACgf+rRgGlvb9eCBQt0++2364YbbpAk+f1+RUVFKTY2NmSs2+2W3++3x3w+Xs5fP3+tIwUFBXK5XPaRkJDQQ6sCAAC9rUcDJicnR4cOHdIrr7zSk7eRJOXn5ysQCNhHXV1dj98TAAD0jsieeuHc3Fxt3bpVZWVluu666+zzHo9HZ86cUVNTU8i7MPX19fJ4PPaYPXv2hLze+d9SOj/mQtHR0YqOju6h1QAAgL4k7O/AWJal3Nxcbd68WTt27FBiYmLI9cmTJ2vgwIHavn27fa62tlbHjx+Xz+eTJPl8Ph08eFANDQ32mNLSUjmdTiUlJYV7ygAAwDBhfwcmJydHL7/8sv7nf/5HQ4YMsT+z4nK5NGjQILlcLmVnZ2vhwoWKi4uT0+nUQw89JJ/Pp9tuu02SNG3aNCUlJWnOnDlatWqV/H6/fvKTnygnJ4d3WQAAQPgDZv369ZKke+65J+T8Sy+9pH/5l3+RJK1Zs0YRERHKyMhQa2ur0tLS9Itf/MIeO2DAAG3dulUPPvigfD6fBg8erLlz5+qJJ54I93QBAICBwh4wXflamZiYGBUWFqqwsLDTMaNGjdJbb70V5tkBAID+gL8LCQAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYJ7K3JwAAV2J03pudXvtoZfpVnQuAq493YAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcfp0wBQWFmr06NGKiYlRSkqK9uzZ09tTAgAAfUBkb0+gM6+++qoWLlyooqIipaSk6LnnnlNaWppqa2sVHx/f29MDAHxBo/Pe7PTaRyvTr+pcYJ4++w7M6tWrNW/ePH3/+99XUlKSioqKdM011+jFF1/s7akBAIBe1iffgTlz5oyqqqqUn59vn4uIiFBqaqrKy8s7fE5ra6taW1vtx4FAQJIUDAbDPr/21s86vdYT90P4sYfmYw/Nxx6iI+f33rKsS47rkwHzf//3fzp37pzcbnfIebfbrffff7/D5xQUFGjFihUXnU9ISOixeXbE9dxVvR16AHtoPvbQfOwhTp06JZfL1en1PhkwVyI/P18LFy60H7e3t6uxsVHDhg2Tw+EI232CwaASEhJUV1cnp9MZttftS/r7Glmf+fr7Gvv7+vQlWCPru3KWZenUqVPyer2XHNcnA2b48OEaMGCA6uvrQ87X19fL4/F0+Jzo6GhFR0eHnIuNje2xOTqdzn75f8rP6+9rZH3m6+9r7O/r05dgjazvylzqnZfz+uSHeKOiojR58mRt377dPtfe3q7t27fL5/P16twAAEDv65PvwEjSwoULNXfuXCUnJ+vWW2/Vc889p5aWFn3/+9/v7akBAIBe1mcD5v7779enn36qZcuWye/368Ybb1RJSclFH+y92qKjo/X4449f9MdV/Ul/XyPrM19/X2N/X5++BGtkfT3PYV3u95QAAAD6mD75GRgAAIBLIWAAAIBxCBgAAGAcAgYAABiHgLlAWVmZZs2aJa/XK4fDoS1btlz2OTt37tTNN9+s6Ohoff3rX1dxcfFVmeuV6O76du7cKYfDcdHh9/uv2py7o6CgQLfccouGDBmi+Ph4zZ49W7W1tZd93uuvv66xY8cqJiZGEyZM0FtvvXVV5nslrmSNxcXFF+1hTEzMVZtzd6xfv14TJ060vyDL5/Pp7bffvuRzTNq/7q7PpL3ryMqVK+VwOLRgwYJLjjNpDy/UlTWatI/Lly+/aK5jx4695HN6Y/8ImAu0tLRo0qRJKiws7NL4o0ePKj09Xffee6+qq6u1YMEC/eu//qu2bdvW43O9Et1d33m1tbX65JNP7CM+Pr7H5vhF7Nq1Szk5OXrvvfdUWlqqtrY2TZs2TS0tLZ0+591339V3v/tdZWdna//+/Zo9e7Zmz56tQ4cOXdW5d9WVrFF/+8bMz+/hsWPHrtqcu+O6667TypUrVVVVpb179+qb3/ym/vEf/1E1NTUdjjdt/7q7Phm0dxeqrKzUCy+8oIkTJ15ynGl7+HldXaMM28fx48eHzPWPf/xjp2N7bf8sdEqStXnz5kuOeeyxx6zx48eHnLv//vuttLS0Hp7dF9eV9f3+97+3JFknT568avMKp4aGBkuStWvXrk7H3HfffVZ6enrIuZSUFOuHP/zhVZjhF9eVNb700kuWy+W6qvMKp6FDh1q/+tWvOrxm+v5Zl1mfqXt36tQp6/rrr7dKS0utu+++23r44Yc7HWvqHnZnjSbt4+OPP25NmjSpy+N7a/94B+YLKi8vV2pqasi5tLQ0lZeX99qcesKNN96oESNG6O///u+1e/fu3p5OlwUCAUlSXFxcp2NM38OurFGSmpubNWrUKCUkJFz2v/j7inPnzumVV15RS0tLp3+NiMn715X1ydC9y8nJUXp6+kV70xFT97A7a5Rh+3jkyBF5vV599atfVVZWlo4fP97p2N7avz77Tbym8Pv9F307sNvtVjAY1P/7f/9PgwYN6rW5hcOIESNUVFSk5ORktba26le/+pXuueceVVRU6Oabb+7t6V1Se3u7FixYoNtvv1033HBDp+M628O++jmfz+vqGseMGaMXX3xREydOVCAQ0LPPPqspU6aopqZG11133VWdc1ccPHhQPp9Pp0+f1rXXXqvNmzcrKSmpw7Em7l931mfa3knSK6+8on379qmysrJL403cw+6u0aR9TElJUXFxscaMGaNPPvlEK1as0J133qlDhw5pyJAhF43vrf0jYHBJY8aM0ZgxY+zHU6ZM0Ycffqg1a9boP//zP3t1bpeTk5OjQ4cOXfLPbk3X1TX6fL6Q/8KfMmWKxo0bpxdeeEFPPvnkVZhp94wZM0bV1dUKBAL6r//6L82dO1e7du3q9Ie8abqzPtP2rq6uTg8//LBKS0v77IdUv6grWaNJ+zhjxgz7nydOnKiUlBSNGjVKr732mrKzs3t1bp9HwHxBHo9H9fX1Iefq6+vldDqNf/elM7feemufj4Lc3Fxt3bpVZWVll/2vm8720OPx9PAsv5jurPFCAwcO1E033aQPPvigx+b3RURFRenrX/+6JGny5MmqrKzU2rVr9cILL1w01sT96876LtTX966qqkoNDQ0h79CeO3dOZWVl+vnPf67W1lYNGDAg5Dmm7eGVrPFCfX0fPy82Nlbf+MY3Op1rb+0fn4H5gnw+n7Zv3x5yrrS09JJ/nm266upqjRgxoren0SHLspSbm6vNmzdrx44dSkxMvOxzTNvDK1njhc6dO6eDBw/22X28UHt7u1pbWzu8Ztr+deRS67tQX9+7qVOn6uDBg6qurraP5ORkZWVlqbq6usMf7Kbt4ZWs8UJ9fR8/r7m5WR9++GGnc+21/evRjwgb6NSpU9b+/fut/fv3W5Ks1atXW/v377eOHTtmWZZl5eXlWXPmzLHH/+Uvf7GuueYaa/Hixdbhw4etwsJCa8CAAVZJSUkvrqJz3V3fmjVrrC1btlhHjhyxDh48aD388MNWRESE9bvf/a4XV9G5Bx980HK5XNbOnTutTz75xD4+++wze8ycOXOsvLw8+/Hu3butyMhI69lnn7UOHz5sPf7449bAgQOtgwcP9tIqLu1K1rhixQpr27Zt1ocffmhVVVVZmZmZVkxMjFVTU9NLq+hcXl6etWvXLuvo0aPWgQMHrLy8PMvhcFjvvPOOZfWD/evu+kzau85c+Bs6pu9hRy63RpP2cdGiRdbOnTuto0ePWrt377ZSU1Ot4cOHWw0NDZbVh/aPgLnA+V8bvvCYO3euZVmWNXfuXOvuu+++6Dk33nijFRUVZX31q1+1XnrppV6a/eV1d31PP/209bWvfc2KiYmx4uLirHvuucfasWNHL67g0jpam6SQPbn77rvt9Z732muvWd/4xjesqKgoa/z48dabb77ZC7PvmitZ44IFC6yRI0daUVFRltvttmbOnGnt27evl1ZwaT/4wQ+sUaNGWVFRUdZXvvIVa+rUqfYPd6sf7F9312fS3nXmwh/upu9hRy63RpP28f7777dGjBhhRUVFWX/3d39n3X///dYHH3xgX+8r++ew/v9/IQIAABiDz8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACM8/8BbhuvIZklPe0AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(mktunebatch(2048)[1].to(torch.float32).cpu(), bins=64)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Q3Cg_8UQep8g"
   },
   "source": [
    "# Step 2: Define Transformer Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "execution_state": "idle",
   "metadata": {
    "id": "tLOWhg_CeWzH"
   },
   "outputs": [],
   "source": [
    "class TransformerModel(nn.Module):\n",
    "    def __init__(self, input_dim, model_dim, output_dim, num_heads, num_layers, seq_len, dropout):\n",
    "        super().__init__()\n",
    "        self.model_dim = model_dim\n",
    "        self.embedding = nn.Embedding(input_dim, model_dim // 2, dtype=torch.bfloat16)\n",
    "        # # seq_len is odd\n",
    "        # self.fancy_encoding = torch.repeat_interleave(torch.rand((1, seq_len // 2 + 1, model_dim // 2), device=device, dtype=torch.bfloat16), 2, dim=1)\n",
    "        # # cut off last element since the target vertex is not repeated\n",
    "        # self.fancy_encoding = self.fancy_encoding[:, :seq_len, :]\n",
    "        encoder_layer = nn.TransformerEncoderLayer(d_model=model_dim, nhead=num_heads,\n",
    "                                                   dim_feedforward=model_dim*4,\n",
    "                                                   dropout=dropout, batch_first=True, dtype=torch.bfloat16)\n",
    "        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers)\n",
    "        self.fc_out = nn.Linear(model_dim, output_dim, dtype=torch.bfloat16)\n",
    "\n",
    "    def forward(self, src, key_padding_mask):\n",
    "        batch_sz = src.size(0)\n",
    "        src = torch.cat((src, torch.full((batch_sz, 1), MAX_VTXS + 1, device=device)), dim=1)\n",
    "        embed = torch.cat((self.embedding(src[:,::2]), self.embedding(src[:,1::2])), dim=2)\n",
    "        output = self.transformer_encoder(embed, src_key_padding_mask=key_padding_mask[:, ::2])\n",
    "        return self.fc_out(output[:, -1, :])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bpIeg86S-hBb"
   },
   "source": [
    "# Step 3: Make Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "execution_state": "idle",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kWXvJRDYgFVP",
    "outputId": "c13adb9d-6565-43b5-8437-20cef3dc0d16"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data: 1048M\n",
      "Trainable parameters in the model: 200K\n"
     ]
    }
   ],
   "source": [
    "# PARAMS\n",
    "VOCAB_SIZE = 1 + MAX_VTXS + 1 # pad plus max number of vertices plus target token\n",
    "MODEL_DIM = 64 # Dimension of model (embedding and transformer)\n",
    "NEPOCHS = 1000\n",
    "BSZ = 2**17 # Batch size\n",
    "BPE = 8 # Batches per epoch\n",
    "NHEADS = 2\n",
    "NLAYERS = 4\n",
    "DROPOUT = 0 # 0.2\n",
    "model = TransformerModel(input_dim=VOCAB_SIZE, model_dim=MODEL_DIM,\n",
    "                         output_dim=1, num_heads=NHEADS,\n",
    "                         num_layers=NLAYERS, seq_len=SEQ_LEN,\n",
    "                         dropout=DROPOUT).to(device)\n",
    "model = torch.compile(model)\n",
    "\n",
    "trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "print(f\"Training data: {NEPOCHS*BPE*BSZ//10**6}M\")\n",
    "print(f\"Trainable parameters in the model: {trainable_params//1000}K\")\n",
    "\n",
    "train_err = []\n",
    "epoch = 0\n",
    "\n",
    "# clear loss file\n",
    "open('loss', 'w').close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MODEL_DIM = 64 # Dimension of model (embedding and transformer)\n",
    "NEPOCHS = 1000\n",
    "BSZ = 2**17 # Batch size\n",
    "BPE = 8 # Batches per epoch\n",
    "NHEADS = 2\n",
    "NLAYERS = 4\n",
    "DROPOUT = 0 # 0.2\n",
    "\n",
    "LR of 8e-4 for 1000 epochs to get down to 0.35546875"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = TransformerModel(input_dim=VOCAB_SIZE, model_dim=MODEL_DIM,\n",
    "                         output_dim=1, num_heads=NHEADS,\n",
    "                         num_layers=NLAYERS, seq_len=SEQ_LEN,\n",
    "                         dropout=DROPOUT).to(device)\n",
    "model = torch.compile(model)\n",
    "model.load_state_dict(torch.load('model.pth', weights_only=True))\n",
    "\n",
    "LR = 8e-4\n",
    "WD = 0 # 1e-5\n",
    "\n",
    "criterion = nn.MSELoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=LR, weight_decay=WD)\n",
    "# scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=100, cooldown=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "f8Zn33m7CxL5"
   },
   "source": [
    "# Step 4: Train the Model for the first task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate():\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    with torch.no_grad():\n",
    "        batch_src, batch_labels, batch_padding_mask = map(lambda x: x.to(device), mkbatch(BSZ))\n",
    "        output = model(batch_src, batch_padding_mask)\n",
    "        loss = criterion(output.squeeze(1), batch_labels)\n",
    "    return loss.item()\n",
    "\n",
    "# figure out if it's doing better on short paths\n",
    "def evaluate_short():\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    with torch.no_grad():\n",
    "        batch_src, batch_labels, batch_padding_mask = map(lambda x: x.to(device), mkbatch(BSZ))\n",
    "        output = model(batch_src, batch_padding_mask)\n",
    "        loss = criterion(output[batch_labels == 1].squeeze(1), batch_labels[batch_labels==1])\n",
    "    return loss.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "execution_state": "running",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 486
    },
    "id": "pvTfzGmCeXU4",
    "outputId": "0d3a20f3-23be-4c19-9eb6-46bfe11a48b1"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sipb/.venv/lib64/python3.12/site-packages/torch/nn/functional.py:6278: UserWarning: Memory Efficient attention on Navi31 GPU is still experimental. Enable it with TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1. (Triggered internally at ../aten/src/ATen/native/transformers/hip/sdp_utils.cpp:269.)\n",
      "  attn_output = scaled_dot_product_attention(\n",
      "/home/sipb/.venv/lib64/python3.12/site-packages/torch/_inductor/compile_fx.py:167: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance.\n",
      "  warnings.warn(\n",
      "/tmp/torchinductor_sipb/nj/cnjfg6sudczhbwjig6u6ixumyik7x7ugjn4x43lbushjy4vv4pwz.py:883: UserWarning: Attempting to use hipBLASLt on an unsupported architecture! Overriding blas backend to hipblas (Triggered internally at ../aten/src/ATen/Context.cpp:296.)\n",
      "  extern_kernels.mm(reinterpret_tensor(buf1, (1048576, 64), (64, 1), 0), reinterpret_tensor(primals_5, (64, 192), (1, 64), 0), out=buf2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0/1000 \t Train Err: 85.0000\n",
      "Epoch 0/1000 \t Train Err: 72.0000\n",
      "Epoch 0/1000 \t Train Err: 63.5000\n",
      "Epoch 0/1000 \t Train Err: 58.0000\n",
      "Epoch 0/1000 \t Train Err: 53.7500\n",
      "Epoch 0/1000 \t Train Err: 51.0000\n",
      "Epoch 0/1000 \t Train Err: 49.2500\n",
      "Epoch 0/1000 \t Train Err: 48.0000\n",
      "Epoch 0/1000 \t Train Err: 47.2500\n",
      "Epoch 0/1000 \t Train Err: 46.2500\n",
      "Epoch 0/1000 \t Train Err: 45.5000\n",
      "Epoch 0/1000 \t Train Err: 45.2500\n",
      "Epoch 0/1000 \t Train Err: 44.5000\n",
      "Epoch 0/1000 \t Train Err: 44.2500\n",
      "Epoch 0/1000 \t Train Err: 44.2500\n",
      "Epoch 0/1000 \t Train Err: 44.2500\n",
      "Epoch 1/1000 \t Train Err: 43.5000\n",
      "Epoch 1/1000 \t Train Err: 43.5000\n",
      "Epoch 1/1000 \t Train Err: 43.5000\n",
      "Epoch 1/1000 \t Train Err: 43.5000\n",
      "Epoch 1/1000 \t Train Err: 43.2500\n",
      "Epoch 1/1000 \t Train Err: 43.2500\n",
      "Epoch 1/1000 \t Train Err: 43.0000\n",
      "Epoch 1/1000 \t Train Err: 43.0000\n",
      "Epoch 1/1000 \t Train Err: 42.7500\n",
      "Epoch 1/1000 \t Train Err: 42.5000\n",
      "Epoch 1/1000 \t Train Err: 42.5000\n",
      "Epoch 1/1000 \t Train Err: 42.7500\n",
      "Epoch 1/1000 \t Train Err: 42.7500\n",
      "Epoch 1/1000 \t Train Err: 42.5000\n",
      "Epoch 1/1000 \t Train Err: 42.2500\n",
      "Epoch 1/1000 \t Train Err: 42.2500\n",
      "Epoch 2/1000 \t Train Err: 42.2500\n",
      "Epoch 2/1000 \t Train Err: 42.5000\n",
      "Epoch 2/1000 \t Train Err: 42.0000\n",
      "Epoch 2/1000 \t Train Err: 42.0000\n",
      "Epoch 2/1000 \t Train Err: 42.0000\n",
      "Epoch 2/1000 \t Train Err: 42.0000\n",
      "Epoch 2/1000 \t Train Err: 42.0000\n",
      "Epoch 2/1000 \t Train Err: 42.2500\n",
      "Epoch 2/1000 \t Train Err: 41.7500\n",
      "Epoch 2/1000 \t Train Err: 41.7500\n",
      "Epoch 2/1000 \t Train Err: 41.2500\n",
      "Epoch 2/1000 \t Train Err: 41.5000\n",
      "Epoch 2/1000 \t Train Err: 41.5000\n",
      "Epoch 2/1000 \t Train Err: 41.7500\n",
      "Epoch 2/1000 \t Train Err: 41.2500\n",
      "Epoch 2/1000 \t Train Err: 41.5000\n",
      "Epoch 3/1000 \t Train Err: 41.5000\n",
      "Epoch 3/1000 \t Train Err: 41.2500\n",
      "Epoch 3/1000 \t Train Err: 41.5000\n",
      "Epoch 3/1000 \t Train Err: 41.2500\n",
      "Epoch 3/1000 \t Train Err: 41.2500\n",
      "Epoch 3/1000 \t Train Err: 41.0000\n",
      "Epoch 3/1000 \t Train Err: 41.0000\n",
      "Epoch 3/1000 \t Train Err: 40.7500\n",
      "Epoch 3/1000 \t Train Err: 40.7500\n",
      "Epoch 3/1000 \t Train Err: 40.5000\n",
      "Epoch 3/1000 \t Train Err: 40.5000\n",
      "Epoch 3/1000 \t Train Err: 40.2500\n",
      "Epoch 3/1000 \t Train Err: 40.0000\n",
      "Epoch 3/1000 \t Train Err: 39.7500\n",
      "Epoch 3/1000 \t Train Err: 39.2500\n",
      "Epoch 3/1000 \t Train Err: 38.7500\n",
      "Epoch 4/1000 \t Train Err: 38.0000\n",
      "Epoch 4/1000 \t Train Err: 37.2500\n",
      "Epoch 4/1000 \t Train Err: 36.5000\n",
      "Epoch 4/1000 \t Train Err: 35.5000\n",
      "Epoch 4/1000 \t Train Err: 35.0000\n",
      "Epoch 4/1000 \t Train Err: 34.7500\n",
      "Epoch 4/1000 \t Train Err: 34.7500\n",
      "Epoch 4/1000 \t Train Err: 34.7500\n",
      "Epoch 4/1000 \t Train Err: 34.5000\n",
      "Epoch 4/1000 \t Train Err: 34.2500\n",
      "Epoch 4/1000 \t Train Err: 33.7500\n",
      "Epoch 4/1000 \t Train Err: 33.7500\n",
      "Epoch 4/1000 \t Train Err: 33.5000\n",
      "Epoch 4/1000 \t Train Err: 33.5000\n",
      "Epoch 4/1000 \t Train Err: 33.0000\n",
      "Epoch 4/1000 \t Train Err: 33.0000\n",
      "Epoch 5/1000 \t Train Err: 33.0000\n",
      "Epoch 5/1000 \t Train Err: 32.7500\n",
      "Epoch 5/1000 \t Train Err: 32.7500\n",
      "Epoch 5/1000 \t Train Err: 32.7500\n",
      "Epoch 5/1000 \t Train Err: 32.5000\n",
      "Epoch 5/1000 \t Train Err: 32.0000\n",
      "Epoch 5/1000 \t Train Err: 32.5000\n",
      "Epoch 5/1000 \t Train Err: 32.2500\n",
      "Epoch 5/1000 \t Train Err: 32.5000\n",
      "Epoch 5/1000 \t Train Err: 31.8750\n",
      "Epoch 5/1000 \t Train Err: 31.6250\n",
      "Epoch 5/1000 \t Train Err: 31.6250\n",
      "Epoch 5/1000 \t Train Err: 31.6250\n",
      "Epoch 5/1000 \t Train Err: 31.8750\n",
      "Epoch 5/1000 \t Train Err: 31.5000\n",
      "Epoch 5/1000 \t Train Err: 31.2500\n",
      "Epoch 6/1000 \t Train Err: 31.1250\n",
      "Epoch 6/1000 \t Train Err: 31.1250\n",
      "Epoch 6/1000 \t Train Err: 31.2500\n",
      "Epoch 6/1000 \t Train Err: 31.2500\n",
      "Epoch 6/1000 \t Train Err: 31.0000\n",
      "Epoch 6/1000 \t Train Err: 30.8750\n",
      "Epoch 6/1000 \t Train Err: 31.0000\n",
      "Epoch 6/1000 \t Train Err: 30.8750\n",
      "Epoch 6/1000 \t Train Err: 30.8750\n",
      "Epoch 6/1000 \t Train Err: 30.8750\n",
      "Epoch 6/1000 \t Train Err: 30.7500\n",
      "Epoch 6/1000 \t Train Err: 30.6250\n",
      "Epoch 6/1000 \t Train Err: 30.5000\n",
      "Epoch 6/1000 \t Train Err: 30.7500\n",
      "Epoch 6/1000 \t Train Err: 30.3750\n",
      "Epoch 6/1000 \t Train Err: 30.5000\n",
      "Epoch 7/1000 \t Train Err: 30.6250\n",
      "Epoch 7/1000 \t Train Err: 30.5000\n",
      "Epoch 7/1000 \t Train Err: 30.3750\n",
      "Epoch 7/1000 \t Train Err: 30.5000\n",
      "Epoch 7/1000 \t Train Err: 30.5000\n",
      "Epoch 7/1000 \t Train Err: 30.5000\n",
      "Epoch 7/1000 \t Train Err: 30.3750\n",
      "Epoch 7/1000 \t Train Err: 30.2500\n",
      "Epoch 7/1000 \t Train Err: 30.2500\n",
      "Epoch 7/1000 \t Train Err: 30.2500\n",
      "Epoch 7/1000 \t Train Err: 30.1250\n",
      "Epoch 7/1000 \t Train Err: 30.0000\n",
      "Epoch 7/1000 \t Train Err: 30.2500\n",
      "Epoch 7/1000 \t Train Err: 30.1250\n",
      "Epoch 7/1000 \t Train Err: 30.1250\n",
      "Epoch 7/1000 \t Train Err: 30.0000\n",
      "Epoch 8/1000 \t Train Err: 30.0000\n",
      "Epoch 8/1000 \t Train Err: 29.8750\n",
      "Epoch 8/1000 \t Train Err: 30.0000\n",
      "Epoch 8/1000 \t Train Err: 30.0000\n",
      "Epoch 8/1000 \t Train Err: 29.7500\n",
      "Epoch 8/1000 \t Train Err: 30.0000\n",
      "Epoch 8/1000 \t Train Err: 29.8750\n",
      "Epoch 8/1000 \t Train Err: 29.8750\n",
      "Epoch 8/1000 \t Train Err: 29.8750\n",
      "Epoch 8/1000 \t Train Err: 29.6250\n",
      "Epoch 8/1000 \t Train Err: 29.6250\n",
      "Epoch 8/1000 \t Train Err: 29.8750\n",
      "Epoch 8/1000 \t Train Err: 29.8750\n",
      "Epoch 8/1000 \t Train Err: 29.5000\n",
      "Epoch 8/1000 \t Train Err: 29.8750\n",
      "Epoch 8/1000 \t Train Err: 29.6250\n",
      "Epoch 9/1000 \t Train Err: 29.7500\n",
      "Epoch 9/1000 \t Train Err: 29.7500\n",
      "Epoch 9/1000 \t Train Err: 29.5000\n",
      "Epoch 9/1000 \t Train Err: 29.6250\n",
      "Epoch 9/1000 \t Train Err: 29.6250\n",
      "Epoch 9/1000 \t Train Err: 29.6250\n",
      "Epoch 9/1000 \t Train Err: 29.6250\n",
      "Epoch 9/1000 \t Train Err: 29.6250\n",
      "Epoch 9/1000 \t Train Err: 29.5000\n",
      "Epoch 9/1000 \t Train Err: 29.3750\n",
      "Epoch 9/1000 \t Train Err: 29.5000\n",
      "Epoch 9/1000 \t Train Err: 29.5000\n",
      "Epoch 9/1000 \t Train Err: 29.5000\n",
      "Epoch 9/1000 \t Train Err: 29.3750\n",
      "Epoch 9/1000 \t Train Err: 29.5000\n",
      "Epoch 9/1000 \t Train Err: 29.2500\n",
      "Epoch 10/1000 \t Train Err: 29.2500\n",
      "Epoch 10/1000 \t Train Err: 29.3750\n",
      "Epoch 10/1000 \t Train Err: 29.2500\n",
      "Epoch 10/1000 \t Train Err: 29.5000\n",
      "Epoch 10/1000 \t Train Err: 29.3750\n",
      "Epoch 10/1000 \t Train Err: 29.2500\n",
      "Epoch 10/1000 \t Train Err: 29.2500\n",
      "Epoch 10/1000 \t Train Err: 29.2500\n",
      "Epoch 10/1000 \t Train Err: 29.3750\n",
      "Epoch 10/1000 \t Train Err: 29.3750\n",
      "Epoch 10/1000 \t Train Err: 29.2500\n",
      "Epoch 10/1000 \t Train Err: 29.2500\n",
      "Epoch 10/1000 \t Train Err: 29.2500\n",
      "Epoch 10/1000 \t Train Err: 29.2500\n",
      "Epoch 10/1000 \t Train Err: 29.2500\n",
      "Epoch 10/1000 \t Train Err: 29.1250\n",
      "Epoch 11/1000 \t Train Err: 29.2500\n",
      "Epoch 11/1000 \t Train Err: 29.2500\n",
      "Epoch 11/1000 \t Train Err: 29.2500\n",
      "Epoch 11/1000 \t Train Err: 29.1250\n",
      "Epoch 11/1000 \t Train Err: 29.0000\n",
      "Epoch 11/1000 \t Train Err: 29.2500\n",
      "Epoch 11/1000 \t Train Err: 29.1250\n",
      "Epoch 11/1000 \t Train Err: 29.0000\n",
      "Epoch 11/1000 \t Train Err: 29.0000\n",
      "Epoch 11/1000 \t Train Err: 29.0000\n",
      "Epoch 11/1000 \t Train Err: 29.0000\n",
      "Epoch 11/1000 \t Train Err: 29.1250\n",
      "Epoch 11/1000 \t Train Err: 29.1250\n",
      "Epoch 11/1000 \t Train Err: 29.2500\n",
      "Epoch 11/1000 \t Train Err: 29.1250\n",
      "Epoch 11/1000 \t Train Err: 29.1250\n",
      "Epoch 12/1000 \t Train Err: 29.1250\n",
      "Epoch 12/1000 \t Train Err: 29.0000\n",
      "Epoch 12/1000 \t Train Err: 29.0000\n",
      "Epoch 12/1000 \t Train Err: 29.0000\n",
      "Epoch 12/1000 \t Train Err: 28.8750\n",
      "Epoch 12/1000 \t Train Err: 29.0000\n",
      "Epoch 12/1000 \t Train Err: 29.1250\n",
      "Epoch 12/1000 \t Train Err: 28.8750\n",
      "Epoch 12/1000 \t Train Err: 29.0000\n",
      "Epoch 12/1000 \t Train Err: 29.0000\n",
      "Epoch 12/1000 \t Train Err: 29.0000\n",
      "Epoch 12/1000 \t Train Err: 28.8750\n",
      "Epoch 12/1000 \t Train Err: 28.7500\n",
      "Epoch 12/1000 \t Train Err: 28.8750\n",
      "Epoch 12/1000 \t Train Err: 28.8750\n",
      "Epoch 12/1000 \t Train Err: 28.8750\n",
      "Epoch 13/1000 \t Train Err: 29.0000\n",
      "Epoch 13/1000 \t Train Err: 28.8750\n",
      "Epoch 13/1000 \t Train Err: 29.1250\n",
      "Epoch 13/1000 \t Train Err: 29.0000\n",
      "Epoch 13/1000 \t Train Err: 29.0000\n",
      "Epoch 13/1000 \t Train Err: 28.8750\n",
      "Epoch 13/1000 \t Train Err: 28.8750\n",
      "Epoch 13/1000 \t Train Err: 29.0000\n",
      "Epoch 13/1000 \t Train Err: 28.8750\n",
      "Epoch 13/1000 \t Train Err: 28.8750\n",
      "Epoch 13/1000 \t Train Err: 28.7500\n",
      "Epoch 13/1000 \t Train Err: 28.6250\n",
      "Epoch 13/1000 \t Train Err: 28.6250\n",
      "Epoch 13/1000 \t Train Err: 28.8750\n",
      "Epoch 13/1000 \t Train Err: 28.6250\n",
      "Epoch 13/1000 \t Train Err: 28.7500\n",
      "Epoch 14/1000 \t Train Err: 28.7500\n",
      "Epoch 14/1000 \t Train Err: 28.8750\n",
      "Epoch 14/1000 \t Train Err: 28.5000\n",
      "Epoch 14/1000 \t Train Err: 28.7500\n",
      "Epoch 14/1000 \t Train Err: 28.7500\n",
      "Epoch 14/1000 \t Train Err: 28.7500\n",
      "Epoch 14/1000 \t Train Err: 28.7500\n",
      "Epoch 14/1000 \t Train Err: 28.7500\n",
      "Epoch 14/1000 \t Train Err: 28.8750\n",
      "Epoch 14/1000 \t Train Err: 28.7500\n",
      "Epoch 14/1000 \t Train Err: 28.7500\n",
      "Epoch 14/1000 \t Train Err: 28.8750\n",
      "Epoch 14/1000 \t Train Err: 28.7500\n",
      "Epoch 14/1000 \t Train Err: 28.8750\n",
      "Epoch 14/1000 \t Train Err: 28.7500\n",
      "Epoch 14/1000 \t Train Err: 28.7500\n",
      "Epoch 15/1000 \t Train Err: 28.7500\n",
      "Epoch 15/1000 \t Train Err: 28.7500\n",
      "Epoch 15/1000 \t Train Err: 28.6250\n",
      "Epoch 15/1000 \t Train Err: 28.7500\n",
      "Epoch 15/1000 \t Train Err: 28.6250\n",
      "Epoch 15/1000 \t Train Err: 28.7500\n",
      "Epoch 15/1000 \t Train Err: 28.7500\n",
      "Epoch 15/1000 \t Train Err: 28.6250\n",
      "Epoch 15/1000 \t Train Err: 28.7500\n",
      "Epoch 15/1000 \t Train Err: 28.6250\n",
      "Epoch 15/1000 \t Train Err: 28.7500\n",
      "Epoch 15/1000 \t Train Err: 28.5000\n",
      "Epoch 15/1000 \t Train Err: 28.6250\n",
      "Epoch 15/1000 \t Train Err: 28.6250\n",
      "Epoch 15/1000 \t Train Err: 28.5000\n",
      "Epoch 15/1000 \t Train Err: 28.6250\n",
      "Epoch 16/1000 \t Train Err: 28.3750\n",
      "Epoch 16/1000 \t Train Err: 28.2500\n",
      "Epoch 16/1000 \t Train Err: 28.1250\n",
      "Epoch 16/1000 \t Train Err: 27.8750\n",
      "Epoch 16/1000 \t Train Err: 28.0000\n",
      "Epoch 16/1000 \t Train Err: 27.6250\n",
      "Epoch 16/1000 \t Train Err: 27.5000\n",
      "Epoch 16/1000 \t Train Err: 27.2500\n",
      "Epoch 16/1000 \t Train Err: 27.1250\n",
      "Epoch 16/1000 \t Train Err: 27.0000\n",
      "Epoch 16/1000 \t Train Err: 26.5000\n",
      "Epoch 16/1000 \t Train Err: 27.0000\n",
      "Epoch 16/1000 \t Train Err: 26.5000\n",
      "Epoch 16/1000 \t Train Err: 26.3750\n",
      "Epoch 16/1000 \t Train Err: 25.6250\n",
      "Epoch 16/1000 \t Train Err: 25.8750\n",
      "Epoch 17/1000 \t Train Err: 25.2500\n",
      "Epoch 17/1000 \t Train Err: 25.1250\n",
      "Epoch 17/1000 \t Train Err: 24.8750\n",
      "Epoch 17/1000 \t Train Err: 24.7500\n",
      "Epoch 17/1000 \t Train Err: 24.1250\n",
      "Epoch 17/1000 \t Train Err: 23.8750\n",
      "Epoch 17/1000 \t Train Err: 23.7500\n",
      "Epoch 17/1000 \t Train Err: 23.5000\n",
      "Epoch 17/1000 \t Train Err: 23.1250\n",
      "Epoch 17/1000 \t Train Err: 22.8750\n"
     ]
    }
   ],
   "source": [
    "while epoch < NEPOCHS:\n",
    "    model.train()\n",
    "    with open(f\"data/{epoch}.pickle\", \"rb\") as f:\n",
    "        pickled_stuff = pickle.load(f)\n",
    "    data = pickled_stuff[\"data\"].to(device)\n",
    "    label = pickled_stuff[\"labels\"].to(device)\n",
    "    padding = pickled_stuff[\"padding\"].to(device)\n",
    "    dataset = TensorDataset(data, label, padding)\n",
    "    loader = DataLoader(dataset, batch_size=BSZ)\n",
    "    for batch_src, batch_labels, batch_padding_mask in loader:\n",
    "        optimizer.zero_grad()\n",
    "        output = model(batch_src, batch_padding_mask)\n",
    "        loss = criterion(output.squeeze(1), batch_labels)\n",
    "        train_loss = loss.item()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        # scheduler.step(loss)\n",
    "    \n",
    "        # test_loss = evaluate()\n",
    "        # test_short_loss = evaluate_short()\n",
    "        \n",
    "        # test_err.append(test_loss)\n",
    "        train_err.append(train_loss)\n",
    "        with open('loss', 'a') as f:\n",
    "            f.write(f\"{train_loss}\\n\")\n",
    "        print(f\"Epoch {epoch}/{NEPOCHS} \\t Train Err: {train_loss:.4f}\")\n",
    "\n",
    "    epoch += 1\n",
    "    if epoch % 100 == 0:\n",
    "        torch.save(model.state_dict(), f\"model_weights_{epoch}.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \"\"\"\n",
    "# Now let's figure out what it's doing. \n",
    "\n",
    "# step 1: figure out what people are attending to \n",
    "# \"\"\"\n",
    "\n",
    "# example_graph, answer, padding = mkbatch(1)\n",
    "# sentance_embeddings = model.full_embedding(example_graph)[0,:,:][example_graph.flatten() != 0]\n",
    "# WQ,WK,WV = torch.split(model.transformer_encoder.layers[0].self_attn.in_proj_weight, (MODEL_DIM, MODEL_DIM, MODEL_DIM))\n",
    "\n",
    "# Q = sentance_embeddings@WQ\n",
    "# K = sentance_embeddings@WK\n",
    "\n",
    "# raw_scores = Q @ K.T / sqrt(MODEL_DIM)\n",
    "# soft = torch.softmax(raw_scores, dim=-1).detach().cpu().to(float).numpy()\n",
    "# plt.imshow(soft)\n",
    "# plt.show()\n",
    "\n",
    "# print(example_graph)\n",
    "\n",
    "# print(Q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAksAAAHgCAYAAAC1jimyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAxHUlEQVR4nO3deXRV1cH38d8NIQOEJEAmAmGyTAKCbzAhaItIJAxV0fCAKULAVEQGsSAVFInauiiglUlA+og8lFGo5VEKuDBYBwgIQZC5DkwSkhghCYMkIdnvHw/c9pqwSTTThe9nrbP0nrtP7t5nUfPtuedeHMYYIwAAAJTKo7onAAAAUJMRSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABgQSwBAABYEEsA4GYcDofGjBlT3dMAbhrEEgAXS5YskcPhkMPh0KefflrieWOMIiIi5HA49Otf/9rlufPnzys5OVkdOnRQ3bp11bBhQ3Xu3Fnjxo1Tenq6c9wLL7zgfI3StoyMjCpZ67XY5jZy5MhqnRuAqudZ3RMAUDP5+PhoxYoVuuuuu1z2f/TRR/r222/l7e3tsr+wsFC/+tWvdPjwYSUmJmrs2LE6f/68Dhw4oBUrVujBBx9UeHi4yzELFiyQn59fidcODAyspFWV3b333quhQ4eW2N+6detqmQ+A6kMsAShV3759tWbNGs2ZM0eenv/+T8WKFSsUGRmp7Oxsl/Hr1q3T559/ruXLl+s3v/mNy3OXLl1SQUFBidcYMGCAgoKCKnEVP13r1q31yCOPVPc0ANQAvA0HoFQJCQn6/vvvtXnzZue+goICrV27tkQMSdLXX38tSbrzzjtLPOfj4yN/f/8KmVeHDh3Uo0ePEvuLi4vVuHFjDRgwwLlv1apVioyMVL169eTv76+OHTtq9uzZFTIPSbr77rvVoUMHpaWlqVu3bvL19VWLFi20cOHCEmOzsrKUlJSk0NBQ+fj4qFOnTvqf//mfUtcxe/ZsdezYUT4+PgoODlbv3r21a9euEmPXrVunDh06yNvbW+3bt9emTZtcnj937pyeeuopNW/eXN7e3goJCdG9996r3bt3V9g5AG4GxBKAUjVv3lwxMTFauXKlc9/GjRuVm5urhx9+uMT4Zs2aSZKWLl0qY0yZXuPMmTPKzs522XJycqzHDBo0SB9//HGJ+5o+/fRTpaenO+e2efNmJSQkqH79+po+fbr+9Kc/6e6779bWrVvLNLdLly6VmFt2dnaJK2Rnz55V3759FRkZqRkzZqhJkyZ64okntHjxYueYH374QXfffbf++te/avDgwZo5c6YCAgI0bNiwEvGWlJSkp556ShEREZo+fbomTZokHx8fbd++vcR6R40apYcfflgzZszQpUuXFB8fr++//945ZuTIkVqwYIHi4+M1f/58Pf300/L19dWhQ4fKdA4AXGEA4D+89dZbRpLZuXOnmTdvnqlXr565ePGiMcaY//qv/zI9evQwxhjTrFkz069fP+dxFy9eNG3atDGSTLNmzcywYcPMm2++aTIzM0u8RnJyspFU6tamTRvr/I4cOWIkmblz57rsHzVqlPHz83POddy4ccbf399cvny53OfgWnOTZFauXOkc1717dyPJvPrqq859+fn5pnPnziYkJMQUFBQYY4yZNWuWkWSWLVvmHFdQUGBiYmKMn5+fycvLM8YYs2XLFiPJPPnkkyXmVFxc7DI/Ly8v89VXXzn37d27t8R5CQgIMKNHjy73+gG44soSgGsaOHCgfvjhB61fv17nzp3T+vXrS30LTpJ8fX21Y8cOTZw4UbryqbqkpCQ1atRIY8eOVX5+folj/va3v2nz5s0u21tvvWWdU+vWrdW5c2etXr3aua+oqEhr167VfffdJ19fX+nKTeIXLlxweRuxPB544IESc9u8eXOJtwA9PT31+OOPOx97eXnp8ccfV1ZWltLS0iRJGzZsUFhYmBISEpzjateurSeffFLnz5/XRx995DwfDodDycnJJebjcDhcHsfGxuqWW25xPr7tttvk7++vb775xrkvMDBQO3bscPkkIoDy4wZvANcUHBys2NhYrVixQhcvXlRRUZHLPUE/FhAQoBkzZmjGjBk6fvy4UlJS9Morr2jevHkKCAjQH//4R5fxv/rVr37SDd6DBg3Ss88+q1OnTqlx48b65z//qaysLA0aNMg5ZtSoUXr77bfVp08fNW7cWL169dLAgQPVu3fvMr1GkyZNFBsbe91x4eHhqlu3rsu+q5+YO3bsmLp27arjx4+rVatW8vBw/f+n7dq1kyQdP35cunLfV3h4uBo0aHDd123atGmJffXr19fZs2edj2fMmKHExERFREQoMjJSffv21dChQ9WyZcvr/nwA/8aVJQBWv/nNb7Rx40YtXLhQffr0KfPH+ps1a6ZHH31UW7duVWBgoJYvX15hcxo0aJCMMVqzZo0k6e2331ZAQIBLCIWEhGjPnj169913df/99+vDDz9Unz59lJiYWGHzqE61atUqdf9/3i82cOBAffPNN5o7d67Cw8M1c+ZMtW/fXhs3bqzCmQLuj1gCYPXggw/Kw8ND27dvv+ZbcDb169fXLbfcotOnT1fYnFq0aKGoqCitXr1aly9f1jvvvKP+/fuX+O4nLy8v3XfffZo/f76+/vprPf7441q6dKm++uqrCptLenq6Lly44LLvX//6l3TlJnldCccvv/xSxcXFLuMOHz7sfF6SbrnlFqWnp+vMmTMVNr9GjRpp1KhRWrdunY4ePaqGDRvq5ZdfrrCfD9wMiCUAVn5+flqwYIFeeOEF3Xfffdcct3fv3hLfvaQrbzEdPHhQbdq0qdB5DRo0SNu3b9fixYuVnZ3t8hacJJdPhUmSh4eHbrvtNkkq9f6pn+ry5ct64403nI8LCgr0xhtvKDg4WJGRkdKV76zKyMhwuc/q8uXLmjt3rvz8/NS9e3dJUnx8vIwxevHFF0u8Tlk/YXhVUVGRcnNzXfaFhIQoPDy8QtcP3Ay4ZwnAdZXlravNmzcrOTlZ999/v7p27So/Pz998803Wrx4sfLz8/XCCy+UOGbt2rWlfoP3vffeq9DQUOvrDRw4UE8//bSefvppNWjQoMT9Rb/97W915swZ3XPPPWrSpImOHz+uuXPnqnPnzs57hWz+9a9/admyZSX2h4aG6t5773U+Dg8P1/Tp03Xs2DG1bt1aq1ev1p49e7Ro0SLVrl1bkjRixAi98cYbGjZsmNLS0tS8eXOtXbtWW7du1axZs1SvXj1JUo8ePTRkyBDNmTNHX375pXr37q3i4mJ98skn6tGjR7n+Prhz586pSZMmGjBggDp16iQ/Pz998MEH2rlzp1599dUy/xwAfHUAgB/5z68OsPnxVwd88803ZurUqaZr164mJCTEeHp6muDgYNOvXz+zZcsWl2NtXx0gyXz44Ydlmuudd95pJJnf/va3JZ5bu3at6dWrlwkJCTFeXl6madOm5vHHHzenT5++7s+1za179+7Ocd27dzft27c3u3btMjExMcbHx8c0a9bMzJs3r8TPzMzMNMOHDzdBQUHGy8vLdOzY0bz11lslxl2+fNnMnDnTtG3b1nh5eZng4GDTp08fk5aW5jK/0r4SoFmzZiYxMdGYK19hMHHiRNOpUydTr149U7duXdOpUyczf/78664fgCuHKe+1XQCAdOUbvLOzs7V///7qngqASsQ9SwAAABbEEgAAgAWxBAAAYME9SwAAABZcWQIAALAglgAAACyIJQAAAAtiCQAAwIJYAgAAsCCWAAAALIglAAAAC2IJAADAglgCAACwIJYAAAAsiCUAAAALYgkAAMCCWAIAALAglgAAACyIJQAAAAtiCQAAwIJYAgAAsCCWAAAALIglAAAAC2IJAADAglgCAACwIJYAAAAsiCUAAAALYgkAAMCCWAIAALAglgAAACyIJQAAAAvP6p7AjaC4uFjp6emqV6+eHA5HdU8HAACUgTFG586dU3h4uDw8rn39iFiqAOnp6YqIiKjuaQAAgJ/g5MmTatKkyTWfJ5YqQL169aQrJ9vf37+6pwMAAMogLy9PERERzt/j10IsVYCrb735+/sTSwAAuJnr3ULDDd4AAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABgQSwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABgQSwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABgQSwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWLhdLL3++utq3ry5fHx8FB0drc8++8w6fs2aNWrbtq18fHzUsWNHbdiw4ZpjR44cKYfDoVmzZlXCzAEAgDtyq1havXq1xo8fr+TkZO3evVudOnVSXFycsrKySh2/bds2JSQkKCkpSZ9//rn69++v/v37a//+/SXG/v3vf9f27dsVHh5eBSsBAADuwq1i6c9//rMee+wxDR8+XLfeeqsWLlyoOnXqaPHixaWOnz17tnr37q2JEyeqXbt2+sMf/qD/9//+n+bNm+cy7tSpUxo7dqyWL1+u2rVrV9FqAACAO3CbWCooKFBaWppiY2Od+zw8PBQbG6vU1NRSj0lNTXUZL0lxcXEu44uLizVkyBBNnDhR7du3L9Nc8vPzlZeX57IBAIAbk9vEUnZ2toqKihQaGuqyPzQ0VBkZGaUek5GRcd3x06dPl6enp5588skyz2XatGkKCAhwbhEREeVeDwAAcA9uE0uVIS0tTbNnz9aSJUvkcDjKfNzkyZOVm5vr3E6ePFmp8wQAANXHbWIpKChItWrVUmZmpsv+zMxMhYWFlXpMWFiYdfwnn3yirKwsNW3aVJ6envL09NTx48c1YcIENW/e/Jpz8fb2lr+/v8sGAABuTG4TS15eXoqMjFRKSopzX3FxsVJSUhQTE1PqMTExMS7jJWnz5s3O8UOGDNEXX3yhPXv2OLfw8HBNnDhR77//fiWvCAAAuAPP6p5AeYwfP16JiYnq0qWLoqKiNGvWLF24cEHDhw+XJA0dOlSNGzfWtGnTJEnjxo1T9+7d9eqrr6pfv35atWqVdu3apUWLFkmSGjZsqIYNG7q8Ru3atRUWFqY2bdpUwwoBAEBN41axNGjQIH333XeaOnWqMjIy1LlzZ23atMl5E/eJEyfk4fHvi2XdunXTihUrNGXKFD377LNq1aqV1q1bpw4dOlTjKgAAgDtxGGNMdU/C3eXl5SkgIEC5ubncvwQAgJso6+9vt7lnCQAAoDoQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABgQSwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABgQSwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABgQSwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABgQSwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABg4Xax9Prrr6t58+by8fFRdHS0PvvsM+v4NWvWqG3btvLx8VHHjh21YcMG53OFhYV65pln1LFjR9WtW1fh4eEaOnSo0tPTq2AlAADAHbhVLK1evVrjx49XcnKydu/erU6dOikuLk5ZWVmljt+2bZsSEhKUlJSkzz//XP3791f//v21f/9+SdLFixe1e/duPf/889q9e7feeecdHTlyRPfff38VrwwAANRUDmOMqe5JlFV0dLTuuOMOzZs3T5JUXFysiIgIjR07VpMmTSoxftCgQbpw4YLWr1/v3Ne1a1d17txZCxcuLPU1du7cqaioKB0/flxNmzYt07zy8vIUEBCg3Nxc+fv7/+T1AQCAqlPW399uc2WpoKBAaWlpio2Nde7z8PBQbGysUlNTSz0mNTXVZbwkxcXFXXO8JOXm5srhcCgwMPCaY/Lz85WXl+eyAQCAG5PbxFJ2draKiooUGhrqsj80NFQZGRmlHpORkVGu8ZcuXdIzzzyjhIQEa2FOmzZNAQEBzi0iIuInrQkAANR8bhNLla2wsFADBw6UMUYLFiywjp08ebJyc3Od28mTJ6tsngAAoGp5VvcEyiooKEi1atVSZmamy/7MzEyFhYWVekxYWFiZxl8NpePHj2vLli3Xve/I29tb3t7eP3ktAADAfbjNlSUvLy9FRkYqJSXFua+4uFgpKSmKiYkp9ZiYmBiX8ZK0efNml/FXQ+nLL7/UBx98oIYNG1biKgAAgLtxmytLkjR+/HglJiaqS5cuioqK0qxZs3ThwgUNHz5ckjR06FA1btxY06ZNkySNGzdO3bt316uvvqp+/fpp1apV2rVrlxYtWiRdCaUBAwZo9+7dWr9+vYqKipz3MzVo0EBeXl7VuFoAAFATuFUsDRo0SN99952mTp2qjIwMde7cWZs2bXLexH3ixAl5ePz7Ylm3bt20YsUKTZkyRc8++6xatWqldevWqUOHDpKkU6dO6d1335Ukde7c2eW1PvzwQ919991Vuj4AAFDzuNX3LNVUfM8SAADu54b7niUAAIDqQCwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABgQSwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABgQSwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABgQSwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABgQSwBAABYlCuWZsyYoR9++MH5eOvWrcrPz3c+PnfunEaNGlWxMwQAAKhGDmOMKevgWrVq6fTp0woJCZEk+fv7a8+ePWrZsqUkKTMzU+Hh4SoqKqq8GddAeXl5CggIUG5urvz9/at7OgAAoAzK+vu7XFeWftxV5egsAAAAt8Q9SwAAABbEEgAAgIVneQ/47//+b/n5+UmSLl++rCVLligoKEi6coM3AADAjaRcN3g3b95cDofjuuOOHj36c+d1Ta+//rpmzpypjIwMderUSXPnzlVUVNQ1x69Zs0bPP/+8jh07platWmn69Onq27ev83ljjJKTk/WXv/xFOTk5uvPOO7VgwQK1atWqzHPiBm8AANxPpdzgfezYMR09evS6W2VZvXq1xo8fr+TkZO3evVudOnVSXFycsrKySh2/bds2JSQkKCkpSZ9//rn69++v/v37a//+/c4xM2bM0Jw5c7Rw4ULt2LFDdevWVVxcnC5dulRp6wAAAO6jXFeWqlt0dLTuuOMOzZs3T5JUXFysiIgIjR07VpMmTSoxftCgQbpw4YLWr1/v3Ne1a1d17txZCxculDFG4eHhmjBhgp5++mlJUm5urkJDQ7VkyRI9/PDDZZoXV5YAAHA/lXJlKTU11SU8JGnp0qVq0aKFQkJCNGLECJcvqaxIBQUFSktLU2xsrHOfh4eHYmNjlZqaes35/ud4SYqLi3OOP3r0qDIyMlzGBAQEKDo6+po/U5Ly8/OVl5fnsgEAgBtTuWLppZde0oEDB5yP9+3bp6SkJMXGxmrSpEl67733NG3atMqYp7Kzs1VUVKTQ0FCX/aGhocrIyCj1mIyMDOv4q/8sz8+UpGnTpikgIMC5RURE/OR1AQCAmq1csbRnzx717NnT+XjVqlWKjo7WX/7yF40fP15z5szR22+/XRnzrFEmT56s3Nxc53by5MnqnhIAAKgk5Yqls2fPulyF+eijj9SnTx/n4zvuuKPSwiEoKEi1atVSZmamy/7MzEyFhYWVekxYWJh1/NV/ludnSpK3t7f8/f1dNgAAcGMqVyyFhoY6P+1WUFCg3bt3q2vXrs7nz507p9q1a1f8LCV5eXkpMjJSKSkpzn3FxcVKSUlRTExMqcfExMS4jJekzZs3O8e3aNFCYWFhLmPy8vK0Y8eOa/5MAABwcynXl1L27dtXkyZN0vTp07Vu3TrVqVNHv/zlL53Pf/HFF7rlllsqY56SpPHjxysxMVFdunRRVFSUZs2apQsXLmj48OGSpKFDh6px48bO+6bGjRun7t2769VXX1W/fv20atUq7dq1S4sWLZIkORwOPfXUU/rjH/+oVq1aqUWLFnr++ecVHh6u/v37V9o6AACA+yhXLP3hD3/QQw89pO7du8vPz09LliyRl5eX8/nFixerV69elTFP6cpXAXz33XeaOnWqMjIy1LlzZ23atMn51uCJEyfk4fHvi2XdunXTihUrNGXKFD377LNq1aqV1q1bpw4dOjjH/P73v9eFCxc0YsQI5eTk6K677tKmTZvk4+NTaesAAADu4yd9z1Jubq78/PxUq1Ytl/1nzpxRvXr1Ku2tuJqK71kCAMD9lPX3d7muLD366KNlGrd48eLy/FgAAIAaq1yxtGTJEjVr1ky333673OiLvwEAAH6ycsXSE088oZUrV+ro0aMaPny4HnnkETVo0KDyZgcAAFDNyvXVAa+//rpOnz6t3//+93rvvfcUERGhgQMH6v333+dKEwAAuCH9rL9I9/jx41qyZImWLl2qy5cv68CBA/Lz86vYGboBbvAGAMD9VMpfpFviYA8PORwOGWNUVFT0c34UAABAjVTuWMrPz9fKlSt17733qnXr1tq3b5/mzZunEydO3JRXlQAAwI2tXDd4jxo1SqtWrVJERIQeffRRrVy5UkFBQZU3OwAAgGpWrnuWPDw81LRpU91+++1yOBzXHPfOO+9U1PzcAvcsAQDgfirlSymHDh1qjSQAAIAbTbm/lBIAAOBm8rM+DQcAAHCjI5YAAAAsiCUAAAALYgkAAMCCWAIAALAglgAAACyIJQAAAAtiCQAAwIJYAgAAsCCWAAAALIglAAAAC2IJAADAglgCAACwIJYAAAAsiCUAAAALYgkAAMCCWAIAALAglgAAACyIJQAAAAtiCQAAwIJYAgAAsCCWAAAALIglAAAAC2IJAADAglgCAACwIJYAAAAsiCUAAAALYgkAAMCCWAIAALAglgAAACyIJQAAAAtiCQAAwIJYAgAAsCCWAAAALIglAAAAC2IJAADAglgCAACwIJYAAAAsiCUAAAALYgkAAMCCWAIAALAglgAAACyIJQAAAAtiCQAAwIJYAgAAsHCbWDpz5owGDx4sf39/BQYGKikpSefPn7cec+nSJY0ePVoNGzaUn5+f4uPjlZmZ6Xx+7969SkhIUEREhHx9fdWuXTvNnj27ClYDAADchdvE0uDBg3XgwAFt3rxZ69ev18cff6wRI0ZYj/nd736n9957T2vWrNFHH32k9PR0PfTQQ87n09LSFBISomXLlunAgQN67rnnNHnyZM2bN68KVgQAANyBwxhjqnsS13Po0CHdeuut2rlzp7p06SJJ2rRpk/r27atvv/1W4eHhJY7Jzc1VcHCwVqxYoQEDBkiSDh8+rHbt2ik1NVVdu3Yt9bVGjx6tQ4cOacuWLWWeX15engICApSbmyt/f/+fvE4AAFB1yvr72y2uLKWmpiowMNAZSpIUGxsrDw8P7dixo9Rj0tLSVFhYqNjYWOe+tm3bqmnTpkpNTb3ma+Xm5qpBgwbW+eTn5ysvL89lAwAANya3iKWMjAyFhIS47PP09FSDBg2UkZFxzWO8vLwUGBjosj80NPSax2zbtk2rV6++7tt706ZNU0BAgHOLiIgo95oAAIB7qNZYmjRpkhwOh3U7fPhwlcxl//79euCBB5ScnKxevXpZx06ePFm5ubnO7eTJk1UyRwAAUPU8q/PFJ0yYoGHDhlnHtGzZUmFhYcrKynLZf/nyZZ05c0ZhYWGlHhcWFqaCggLl5OS4XF3KzMwscczBgwfVs2dPjRgxQlOmTLnuvL29veXt7X3dcQAAwP1VaywFBwcrODj4uuNiYmKUk5OjtLQ0RUZGSpK2bNmi4uJiRUdHl3pMZGSkateurZSUFMXHx0uSjhw5ohMnTigmJsY57sCBA7rnnnuUmJiol19+ucLWBgAAbgxu8Wk4SerTp48yMzO1cOFCFRYWavjw4erSpYtWrFghSTp16pR69uyppUuXKioqSpL0xBNPaMOGDVqyZIn8/f01duxY6cq9Sbry1ts999yjuLg4zZw50/latWrVKlPEXcWn4QAAcD9l/f1drVeWymP58uUaM2aMevbsKQ8PD8XHx2vOnDnO5wsLC3XkyBFdvHjRue+1115zjs3Pz1dcXJzmz5/vfH7t2rX67rvvtGzZMi1btsy5v1mzZjp27FgVrg4AANRUbnNlqSbjyhIAAO7nhvqeJQAAgOpCLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABgQSwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABgQSwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABgQSwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABgQSwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABg4TaxdObMGQ0ePFj+/v4KDAxUUlKSzp8/bz3m0qVLGj16tBo2bCg/Pz/Fx8crMzOz1LHff/+9mjRpIofDoZycnEpaBQAAcDduE0uDBw/WgQMHtHnzZq1fv14ff/yxRowYYT3md7/7nd577z2tWbNGH330kdLT0/XQQw+VOjYpKUm33XZbJc0eAAC4K4cxxlT3JK7n0KFDuvXWW7Vz50516dJFkrRp0yb17dtX3377rcLDw0sck5ubq+DgYK1YsUIDBgyQJB0+fFjt2rVTamqqunbt6hy7YMECrV69WlOnTlXPnj119uxZBQYGXnM++fn5ys/Pdz7Oy8tTRESEcnNz5e/vX8GrBwAAlSEvL08BAQHX/f3tFleWUlNTFRgY6AwlSYqNjZWHh4d27NhR6jFpaWkqLCxUbGysc1/btm3VtGlTpaamOvcdPHhQL730kpYuXSoPj7KdjmnTpikgIMC5RURE/Kz1AQCAmsstYikjI0MhISEu+zw9PdWgQQNlZGRc8xgvL68SV4hCQ0Odx+Tn5yshIUEzZ85U06ZNyzyfyZMnKzc317mdPHnyJ60LAADUfNUaS5MmTZLD4bBuhw8frrTXnzx5stq1a6dHHnmkXMd5e3vL39/fZQMAADcmz+p88QkTJmjYsGHWMS1btlRYWJiysrJc9l++fFlnzpxRWFhYqceFhYWpoKBAOTk5LleXMjMzncds2bJF+/bt09q1ayVJV2/fCgoK0nPPPacXX3zxZ68RAAC4t2qNpeDgYAUHB193XExMjHJycpSWlqbIyEjpSugUFxcrOjq61GMiIyNVu3ZtpaSkKD4+XpJ05MgRnThxQjExMZKkv/3tb/rhhx+cx+zcuVOPPvqoPvnkE91yyy0VtEoAAODOqjWWyqpdu3bq3bu3HnvsMS1cuFCFhYUaM2aMHn74Yecn4U6dOqWePXtq6dKlioqKUkBAgJKSkjR+/Hg1aNBA/v7+Gjt2rGJiYpyfhPtxEGVnZztfz/ZpOAAAcPNwi1iSpOXLl2vMmDHq2bOnPDw8FB8frzlz5jifLyws1JEjR3Tx4kXnvtdee805Nj8/X3FxcZo/f341rQAAALgjt/iepZqurN/TAAAAao4b6nuWAAAAqguxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABgQSwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABgQSwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABgQSwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYEEsAQAAWBBLAAAAFsQSAACABbEEAABg4VndE7gRGGMkSXl5edU9FQAAUEZXf29f/T1+LcRSBTh37pwkKSIiorqnAgAAyuncuXMKCAi45vMOc72cwnUVFxcrPT1d9erVk8PhqO7pVKu8vDxFRETo5MmT8vf3r+7p3LA4z1WHc101OM9Vg/Psyhijc+fOKTw8XB4e174ziStLFcDDw0NNmjSp7mnUKP7+/vwPsQpwnqsO57pqcJ6rBuf532xXlK7iBm8AAAALYgkAAMCCWEKF8vb2VnJysry9vat7Kjc0znPV4VxXDc5z1eA8/zTc4A0AAGDBlSUAAAALYgkAAMCCWAIAALAglgAAACyIJZTbmTNnNHjwYPn7+yswMFBJSUk6f/689ZhLly5p9OjRatiwofz8/BQfH6/MzMxSx37//fdq0qSJHA6HcnJyKmkVNV9lnOe9e/cqISFBERER8vX1Vbt27TR79uwqWE3N8frrr6t58+by8fFRdHS0PvvsM+v4NWvWqG3btvLx8VHHjh21YcMGl+eNMZo6daoaNWokX19fxcbG6ssvv6zkVdR8FXmeCwsL9cwzz6hjx46qW7euwsPDNXToUKWnp1fBSmq2iv7z/J9Gjhwph8OhWbNmVcLM3YwByql3796mU6dOZvv27eaTTz4xv/jFL0xCQoL1mJEjR5qIiAiTkpJidu3aZbp27Wq6detW6tgHHnjA9OnTx0gyZ8+eraRV1HyVcZ7ffPNN8+STT5p//vOf5uuvvzZ//etfja+vr5k7d24VrKj6rVq1ynh5eZnFixebAwcOmMcee8wEBgaazMzMUsdv3brV1KpVy8yYMcMcPHjQTJkyxdSuXdvs27fPOeZPf/qTCQgIMOvWrTN79+41999/v2nRooX54YcfqnBlNUtFn+ecnBwTGxtrVq9ebQ4fPmxSU1NNVFSUiYyMrOKV1SyV8ef5qnfeecd06tTJhIeHm9dee60KVlOzEUsol4MHDxpJZufOnc59GzduNA6Hw5w6darUY3Jyckzt2rXNmjVrnPsOHTpkJJnU1FSXsfPnzzfdu3c3KSkpN3UsVfZ5/k+jRo0yPXr0qOAV1ExRUVFm9OjRzsdFRUUmPDzcTJs2rdTxAwcONP369XPZFx0dbR5//HFjjDHFxcUmLCzMzJw50/l8Tk6O8fb2NitXrqy0ddR0FX2eS/PZZ58ZSeb48eMVOHP3Ulnn+dtvvzWNGzc2+/fvN82aNSOWjDG8DYdySU1NVWBgoLp06eLcFxsbKw8PD+3YsaPUY9LS0lRYWKjY2FjnvrZt26pp06ZKTU117jt48KBeeuklLV261PoXGt4MKvM8/1hubq4aNGhQwSuoeQoKCpSWluZyfjw8PBQbG3vN85OamuoyXpLi4uKc448ePaqMjAyXMQEBAYqOjrae8xtZZZzn0uTm5srhcCgwMLACZ+8+Kus8FxcXa8iQIZo4caLat29fiStwLzf3bySUW0ZGhkJCQlz2eXp6qkGDBsrIyLjmMV5eXiX+oxYaGuo8Jj8/XwkJCZo5c6aaNm1aiStwD5V1nn9s27ZtWr16tUaMGFGBs6+ZsrOzVVRUpNDQUJf9tvOTkZFhHX/1n+X5mTe6yjjPP3bp0iU988wzSkhIuGn/MtjKOs/Tp0+Xp6ennnzyyUqauXsiliBJmjRpkhwOh3U7fPhwpb3+5MmT1a5dOz3yyCOV9ho1QXWf5/+0f/9+PfDAA0pOTlavXr2q5DWBn6uwsFADBw6UMUYLFiyo7uncUNLS0jR79mwtWbJEDoejuqdTo3hW9wRQM0yYMEHDhg2zjmnZsqXCwsKUlZXlsv/y5cs6c+aMwsLCSj0uLCxMBQUFysnJcbnqkZmZ6Txmy5Yt2rdvn9auXStd+YSRJAUFBem5557Tiy+++LPXWBNU93m+6uDBg+rZs6dGjBihKVOm/Kw1uYugoCDVqlWrxKcwSzs/V4WFhVnHX/1nZmamGjVq5DKmc+fOlbCKmq8yzvNVV0Pp+PHj2rJly017VUmVdJ4/+eQTZWVluVzdLyoq0oQJEzRr1iwdO3asUtbiFqr7pim4l6s3Hu/atcu57/333y/Tjcdr16517jt8+LDLjcdfffWV2bdvn3NbvHixkWS2bdt2zU923Mgq6zwbY8z+/ftNSEiImThxYiWvouaJiooyY8aMcT4uKioyjRs3tt4Q++tf/9plX0xMTIkbvF955RXn87m5udzgXcHn2RhjCgoKTP/+/U379u1NVlZWJc7efVT0ec7Oznb57/C+fftMeHi4eeaZZ8zhw4creTU1G7GEcuvdu7e5/fbbzY4dO8ynn35qWrVq5fKR9m+//da0adPG7Nixw7lv5MiRpmnTpmbLli1m165dJiYmxsTExFzzNT788MOb+tNwppLO8759+0xwcLB55JFHzOnTp53bzfLLZ9WqVcbb29ssWbLEHDx40IwYMcIEBgaajIwMY4wxQ4YMMZMmTXKO37p1q/H09DSvvPKKOXTokElOTi71qwMCAwPN//7v/5ovvvjCPPDAA3x1QAWf54KCAnP//febJk2amD179rj82c3Pz6+2dVa3yvjz/GN8Gu7/EEsot++//94kJCQYPz8/4+/vb4YPH27OnTvnfP7o0aNGkvnwww+d+3744QczatQoU79+fVOnTh3z4IMPmtOnT1/zNYilyjnPycnJRlKJrVmzZlW+vuoyd+5c07RpU+Pl5WWioqLM9u3bnc91797dJCYmuox/++23TevWrY2Xl5dp3769+cc//uHyfHFxsXn++edNaGio8fb2Nj179jRHjhypsvXUVBV5nq/+WS9t+88//zejiv7z/GPE0v9xmKs3hwAAAKAEPg0HAABgQSwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAVwOFwaN26ddU9DQCVgFgC4PaGDRsmh8NRYuvdu3d1Tw3ADcCzuicAABWhd+/eeuutt1z2eXt7V9t8ANw4uLIE4Ibg7e2tsLAwl61+/frSlbfIFixYoD59+sjX11ctW7bU2rVrXY7ft2+f7rnnHvn6+qphw4YaMWKEzp8/7zJm8eLFat++vby9vdWoUSONGTPG5fns7Gw9+OCDqlOnjlq1aqV3333X+dzZs2c1ePBgBQcHy9fXV61atSoRdwBqJmIJwE3h+eefV3x8vPbu3avBgwfr4Ycf1qFDhyRJFy5cUFxcnOrXr6+dO3dqzZo1+uCDD1xiaMGCBRo9erRGjBihffv26d1339UvfvELl9d48cUXNXDgQH3xxRfq27evBg8erDNnzjhf/+DBg9q4caMOHTqkBQsWKCgoqIrPAoCfxACAm0tMTDS1atUydevWddlefvllY4wxkszIkSNdjomOjjZPPPGEMcaYRYsWmfr165vz5887n//HP/5hPDw8TEZGhjHGmPDwcPPcc89dcw6SzJQpU5yPz58/bySZjRs3GmOMue+++8zw4cMreOUAqgL3LAG4IfTo0UMLFixw2degQQPnv8fExLg8FxMToz179kiSDh06pE6dOqlu3brO5++8804VFxfryJEjcjgcSk9PV8+ePa1zuO2225z/XrduXfn7+ysrK0uS9MQTTyg+Pl67d+9Wr1691L9/f3Xr1u1nrhpAVSCWANwQ6tatW+JtsYri6+tbpnG1a9d2eexwOFRcXCxJ6tOnj44fP64NGzZo8+bN6tmzp0aPHq1XXnmlUuYMoOJwzxKAm8L27dtLPG7Xrp0kqV27dtq7d68uXLjgfH7r1q3y8PBQmzZtVK9ePTVv3lwpKSk/aw7BwcFKTEzUsmXLNGvWLC1atOhn/TwAVYMrSwBuCPn5+crIyHDZ5+np6byJes2aNerSpYvuuusuLV++XJ999pnefPNNSdLgwYOVnJysxMREvfDCC/ruu+80duxYDRkyRKGhoZKkF154QSNHjlRISIj69Omjc+fOaevWrRo7dmyZ5jd16lRFRkaqffv2ys/P1/r1652xBqBmI5YA3BA2bdqkRo0auexr06aNDh8+LF35pNqqVas0atQoNWrUSCtXrtStt94qSapTp47ef/99jRs3TnfccYfq1Kmj+Ph4/fnPf3b+rMTERF26dEmvvfaann76aQUFBWnAgAFlnp+Xl5cmT56sY8eOydfXV7/85S+1atWqCls/gMrjMP/3KQ4AuGE5HA79/e9/V//+/at7KgDcEPcsAQAAWBBLAAAAFtyzBOCGx90GAH4OriwBAABYEEsAAAAWxBIAAIAFsQQAAGBBLAEAAFgQSwAAABbEEgAAgAWxBAAAYPH/AdAj8vr4ugedAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.suptitle('MSE vs Epochs')\n",
    "plt.plot(train_err, label='Train', color='blue')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('MSE')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "execution_state": "idle",
   "metadata": {
    "id": "LoGEmM5lH7_A"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[3.1870e+04, 4.5000e+01, 0.0000e+00, ..., 0.0000e+00, 0.0000e+00,\n",
       "         0.0000e+00],\n",
       "        [0.0000e+00, 0.0000e+00, 0.0000e+00, ..., 0.0000e+00, 0.0000e+00,\n",
       "         0.0000e+00],\n",
       "        [0.0000e+00, 0.0000e+00, 0.0000e+00, ..., 0.0000e+00, 0.0000e+00,\n",
       "         0.0000e+00],\n",
       "        ...,\n",
       "        [0.0000e+00, 0.0000e+00, 0.0000e+00, ..., 0.0000e+00, 0.0000e+00,\n",
       "         0.0000e+00],\n",
       "        [0.0000e+00, 0.0000e+00, 0.0000e+00, ..., 0.0000e+00, 0.0000e+00,\n",
       "         0.0000e+00],\n",
       "        [0.0000e+00, 0.0000e+00, 0.0000e+00, ..., 1.3300e+02, 2.9200e+02,\n",
       "         4.8201e+04]]),\n",
       " array([ 1.   ,  1.28 ,  1.561,  1.84 ,  2.121,  2.4  ,  2.68 ,  2.96 ,\n",
       "         3.24 ,  3.52 ,  3.8  ,  4.08 ,  4.36 ,  4.64 ,  4.92 ,  5.2  ,\n",
       "         5.48 ,  5.76 ,  6.04 ,  6.32 ,  6.6  ,  6.88 ,  7.16 ,  7.44 ,\n",
       "         7.72 ,  8.   ,  8.28 ,  8.56 ,  8.84 ,  9.12 ,  9.4  ,  9.68 ,\n",
       "         9.96 , 10.24 , 10.52 , 10.805, 11.08 , 11.36 , 11.64 , 11.92 ,\n",
       "        12.2  , 12.484, 12.76 , 13.04 , 13.32 , 13.6  , 13.88 , 14.164,\n",
       "        14.44 , 14.72 , 15.   ], dtype=float16),\n",
       " array([ 0.824,  1.1  ,  1.376,  1.652,  1.928,  2.203,  2.48 ,  2.756,\n",
       "         3.031,  3.307,  3.582,  3.86 ,  4.133,  4.41 ,  4.688,  4.96 ,\n",
       "         5.24 ,  5.516,  5.79 ,  6.066,  6.34 ,  6.617,  6.895,  7.168,\n",
       "         7.445,  7.723,  7.996,  8.27 ,  8.55 ,  8.83 ,  9.09 ,  9.375,\n",
       "         9.66 ,  9.92 , 10.2  , 10.484, 10.75 , 11.03 , 11.31 , 11.58 ,\n",
       "        11.86 , 12.14 , 12.41 , 12.69 , 12.97 , 13.234, 13.516, 13.8  ,\n",
       "        14.06 , 14.34 , 14.625], dtype=float16),\n",
       " <matplotlib.collections.QuadMesh at 0x7fe60c0a49b0>)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhqUlEQVR4nO3df3CU5b338c/m1yalyUJSIYkkGC1HFBDxSJ1HnBbGKM0gyukRRwcpgzNt1ShEHAppja0/IEZbiz8YEP8QOiNaO8egZYo+eSiCTgWBiJWxw4+BYpQfsefgbhLMstm9nz8eydNglr2j917XveH9mtk/dvMl3+/chOTDlb2uO+A4jiMAAABDsmwPAAAAzi2EDwAAYBThAwAAGEX4AAAARhE+AACAUYQPAABgFOEDAAAYRfgAAABG5dge4EyJREJHjhxRYWGhAoGA7XEAAIALjuOoo6ND5eXlyso6+9qG78LHkSNHVFFRYXsMAADwNbS1tWnkyJFnrfFd+CgsLJS+HL6oqMj2OL7ynxMfdFX3X+8/7Gnfm6ofT1nz2v/5uac93Zpxw29d1f1pw/2Doq9bP/pfj7iqe/XdhrTPciZbX8f/Odrd38V/7Xf3d+u1m0I/dlX3Wvj3aZ8FOFPi+MSUNZHOhEZd8Y/en+Nn47vwcfpXLUVFRYSPM+RkB13VeX3dcnLyjfd0y81ssnRN0tHXLVtfK25Y+zrOyrPS162cQK6rOr4vwobEyWzXtW7eMsEbTgEAgFGEDwAAYBThAwAAGEX4AAAARvnuDac4i45OK21zDh210teNnI5TVvpmdcet9HUr9p0htkdIztLX8cajK6z0dasl8UfbIwBJXdj805Q1iS+6JbnbQcfKBwAAMIrwAQAAjCJ8AAAAowgfAADAqAGHj61bt2rGjBkqLy9XIBDQ+vXrk9beeeedCgQCWr58+TedEwAADBIDDh9dXV2aMGGCVqw4+zvHm5ubtW3bNpWXl3+T+QAAwCAz4K22NTU1qqmpOWvNp59+qnvvvVdvvvmmpk+f/k3mAwAAg4zn53wkEgnNmTNHixYt0tixY1PWR6NRRaPR3ueRSMTrkQaNnuPtVvr6+XyErPBJK339fPaJJOV98rntEZIr/LaVttdlzXJVZ+u8Db/Ph3Nb3onUN5aLd7u/+ZznbzhtampSTk6O5s+f76q+sbFRoVCo91FRUeH1SAAAwEc8DR+7du3SU089pTVr1ri6pa4k1dfXKxwO9z7a2tq8HAkAAPiMp+Hj7bffVnt7uyorK5WTk6OcnBwdPnxY999/vy644IJ+/0wwGFRRUVGfBwAAGLw8fc/HnDlzVF1d3ee1adOmac6cOZo3b56XrQAAQIYacPjo7OzUgQMHep8fOnRIu3fvVnFxsSorK1VSUtKnPjc3V6Wlpbr44ou9mRgAAGS0AYePnTt3aurUqb3PFy5cKEmaO3eu1qxZ4+10AABg0Blw+JgyZYocx3Fd/49//GOgLZCErS12NWW1KWusbce1dHt257xhVvq61fOdQtsjJOXk59oewZfYQgs/y+1IXZMVTV3TW/uNpgEAABggwgcAADCK8AEAAIwifAAAAKMIHwAAwCjCBwAAMMrzu9oCJtna8hrojlnp65af77pr69rljBhupS8wGDgubljrpuY0Vj4AAIBRhA8AAGAU4QMAABhF+AAAAEYRPgAAgFGEDwAAYBThAwAAGMU5HxnkuqxZruq8vjV3z/F2Tz+flwKfnbDS99TIoVb6DgYb9zVZ6evnr2PA73I7UtdkRd1/PlY+AACAUYQPAABgFOEDAAAYRfgAAABGET4AAIBRhA8AAGAUW22BryHvk89tj3BWiZH+vX18zb8tdlXn9ZZcr7egA+cSx8VShZua01j5AAAARhE+AACAUYQPAABgFOEDAAAYRfgAAABGET4AAIBRhA8AAGAU53xkEFvnFPj6fITCb1tpGysLWenrVlb4pO0RknLyc22PAGCAclx8Swmccv/5WPkAAABGET4AAIBRhA8AAGAU4QMAABhF+AAAAEYRPgAAgFFstc0g12XNclXn9dbYmrLalDUbj67wtKdbPQcOWumbezRspa9bidC3bI+QVHzPXtsjALCMlQ8AAGAU4QMAABhF+AAAAEYRPgAAgFGEDwAAYBThAwAAGDXgrbZbt27VE088oV27duno0aNqbm7WzJkzJUmxWEwPPPCA/vznP+vgwYMKhUKqrq7WY489pvLy8nTMj3Nc1r+Ps9K35zuFVvq6Ff+Wf+8cm/PdC630tbVV3S2/z4dzW8LFt5SE4/7zDXjlo6urSxMmTNCKFV891+HkyZNqbW1VQ0ODWltb9eqrr2rv3r268cYbB9oGAAAMUgNe+aipqVFNTU2/HwuFQmppaenz2rPPPqvvfe97+vjjj1VZWfn1JwUAAINC2k84DYfDCgQCGjp0aL8fj0ajikajvc8jkUi6RwIAABal9Q2n3d3dWrx4sW677TYVFRX1W9PY2KhQKNT7qKioSOdIAADAsrSFj1gspltuuUWO42jlypVJ6+rr6xUOh3sfbW1t6RoJAAD4QFp+7XI6eBw+fFh/+ctfkq56SFIwGFQwGEzHGAAAwIc8Dx+ng8f+/fu1efNmlZSUeN0CAABksAGHj87OTh04cKD3+aFDh7R7924VFxerrKxMN998s1pbW7VhwwbF43EdO3ZMklRcXKy8vDxvp8c5Lyt80krf7M5uK33dyj4Zsz1CUhv3NVnp6/fzMfw+H85t+ScSKWt6YqlrThtw+Ni5c6emTp3a+3zhwoWSpLlz5+rXv/61Xn/9dUnS5Zdf3ufPbd68WVOmTBloOwAAMMgMOHxMmTJFjpP8GLOzfQwAAIB7uwAAAKMIHwAAwCjCBwAAMIrwAQAAjEr7vV2Q+XqOt9sewXdi3xlie4SMxa3jgcwT+1bqtYr4KffrGax8AAAAowgfAADAKMIHAAAwivABAACMInwAAACjCB8AAMAottpmEFtbD3O+e6GVvm7EykJW+ub+s8tKXwCwIfeL1PdtC8Tc39uNlQ8AAGAU4QMAABhF+AAAAEYRPgAAgFGEDwAAYBThAwAAGEX4AAAARnHOBzJaIJaw0rdn2Les9HXr1LB82yMklT3uYit9r8ua5arO1nk6gK85Ls7wcFPzJVY+AACAUYQPAABgFOEDAAAYRfgAAABGET4AAIBRhA8AAGAUW22R0sZ9TbZHSOrU0DwrfePBbCt93Qoe67A9QlJv/O1R2yMAGCAnK5C6KOCi5kusfAAAAKMIHwAAwCjCBwAAMIrwAQAAjCJ8AAAAowgfAADAKMIHAAAwinM+kNIPL3sgZY2tsxvyP/vCTt9//I+Vvm5FSwttj5DUtEkPuap7c8ev0j4LADtY+QAAAEYRPgAAgFGEDwAAYBThAwAAGEX4AAAARhE+AACAUWy1zSA1ZbWu6jYeXZH2Wfyi+7wCK31jZSErfd1K5Li/tTUApBJIOKmLHBc1XxrwysfWrVs1Y8YMlZeXKxAIaP369Wf0dvTggw+qrKxMBQUFqq6u1v79+wfaBgAADFIDDh9dXV2aMGGCVqzo/3/Xjz/+uJ5++mmtWrVK27dv15AhQzRt2jR1d3d7MS8AAMhwA/61S01NjWpqavr9mOM4Wr58uR544AHddNNNkqTf//73GjFihNavX69bb731m08MAAAymqdvOD106JCOHTum6urq3tdCoZCuuuoqvfvuu162AgAAGcrTN5weO3ZMkjRixIg+r48YMaL3Y2eKRqOKRqO9zyORiJcjAQAAn7G+1baxsVGhUKj3UVFRYXskAACQRp6Gj9LSUknS8ePH+7x+/Pjx3o+dqb6+XuFwuPfR1tbm5UgAAMBnPP21S1VVlUpLS7Vp0yZdfvnl0pe/Rtm+fbvuuuuufv9MMBhUMBj0coxBy9b5HYHPTljp64arvefnoKwe/16XrE/abY/gS9dlzXJV15L4Y9pnAc4USHhTc9qAw0dnZ6cOHDjQ+/zQoUPavXu3iouLVVlZqbq6Oj366KMaPXq0qqqq1NDQoPLycs2cOXOgrQAAwCA04PCxc+dOTZ06tff5woULJUlz587VmjVr9POf/1xdXV366U9/qs8//1zXXHON3njjDeXn53s7OQAAyEgDDh9TpkyRc5YjVAOBgB5++GE9/PDD33Q2AAAwCFnf7QIAAM4thA8AAGAU4QMAABjl6VZbDE7OecNsj5BUzhdxK32jxXlW+roVPNZhe4SkYv92vu0RfIkttDiXsPIBAACMInwAAACjCB8AAMAowgcAADCK8AEAAIwifAAAAKPYaptBaspqXdV5fffbQHfM08/npVhRrpW+2Za2+LoV+84Q2yMkFXj7fSt92coKfH25J1PfsjYQc39bW1Y+AACAUYQPAABgFOEDAAAYRfgAAABGET4AAIBRhA8AAGAU4QMAABjFOR8ZxOvzO1zr6LTT14Wcrh4rfRNBcvvXZeu8jeuyZrmqYz7gq74ozk5ZEz+VuuY0voMCAACjCB8AAMAowgcAADCK8AEAAIwifAAAAKMIHwAAwCi22mYQW1vxEiOHe/r5vBQdlmulbyInYKWvW1ndcdsjJGXr69jvW1T9Ph/Obdmx1DXOAE4+YOUDAAAYRfgAAABGET4AAIBRhA8AAGAU4QMAABhF+AAAAEax1RYpJXbtsT1CUsETLvZ/pUH+P09Z6etWdme37RGSyvnuhVb6/vCyB1zVvfG3R9M+C5BpAnHHk5rTWPkAAABGET4AAIBRhA8AAGAU4QMAABhF+AAAAEYRPgAAgFGEDwAAYBTnfCCl7HEX2x4hqVOhXCt9u8qDVvq69cWokO0RknLy7fydxffstdIXGAyCJ3pS1mT3pK45jZUPAABgFOEDAAAY5Xn4iMfjamhoUFVVlQoKCnTRRRfpkUcekeO4P3YVAAAMXp6/56OpqUkrV67U2rVrNXbsWO3cuVPz5s1TKBTS/PnzvW4HAAAyjOfh469//atuuukmTZ8+XZJ0wQUX6KWXXtJ7773ndSsAAJCBPP+1y9VXX61NmzZp3759kqQPPvhA77zzjmpqavqtj0ajikQifR4AAGDw8nzlY8mSJYpEIhozZoyys7MVj8e1dOlSzZ49u9/6xsZGPfTQQ16PMSi1JP5opW/gsxNW+roRzw3YHsGXciMx2yMkFej272wA+pcIpl6rSGS5X8/wfOXjlVde0Ysvvqh169aptbVVa9eu1W9+8xutXbu23/r6+nqFw+HeR1tbm9cjAQAAH/F85WPRokVasmSJbr31VknS+PHjdfjwYTU2Nmru3LlfqQ8GgwoG/X1gEwAA8I7nKx8nT55U1hlLL9nZ2UokEl63AgAAGcjzlY8ZM2Zo6dKlqqys1NixY/X+++/rySef1B133OF1KwAAkIE8Dx/PPPOMGhoadPfdd6u9vV3l5eX62c9+pgcffNDrVgAAIAN5Hj4KCwu1fPlyLV++3OtPDQAABgHu7QIAAIzyfOUDg09PVZntEZI6VcQ5H/2JFufZHiG5jk4rbW2dkwMMBlnR1JtGsnrcbyxh5QMAABhF+AAAAEYRPgAAgFGEDwAAYBThAwAAGEX4AAAARrHVNoPUlNW6qtt4dIWnfZ1c/2bU7FN2+iZy/b3F1822uHPNdVmzXNWxJRdIP//+VAEAAIMS4QMAABhF+AAAAEYRPgAAgFGEDwAAYBThAwAAGMVW2wzSc7zdSt9YUa6Vvm5kRx0rfWND/L3VNl7g3/9X2Po6Zgst4B/+/Q4FAAAGJcIHAAAwivABAACMInwAAACjCB8AAMAowgcAADCK8AEAAIzinA+kFrdzloYbpwotnbfh8zvWB3w+H4DM0jMkO3VNLHXNaax8AAAAowgfAADAKMIHAAAwivABAACMInwAAACjCB8AAMAottpmEGu3BM/27+3jbd3aPh600ta1rB7/bo/m1vb9uy5rlqs6rh9syP/nqZQ1PT2pa05j5QMAABhF+AAAAEYRPgAAgFGEDwAAYBThAwAAGEX4AAAARhE+AACAUZzzkUFqympd1W08usLTvrEC/2bUnnw7fR2f/8vJ+++o7RGS+uFlD7iqe+Nvj6Z9Fj/h/A74WSCWSF3Tk7rmNP/+VAEAAIMS4QMAABhF+AAAAEalJXx8+umnuv3221VSUqKCggKNHz9eO3fuTEcrAACQYTx/29yJEyc0efJkTZ06VRs3btR5552n/fv3a9iwYV63AgAAGcjz8NHU1KSKigq98MILva9VVVV53QYAAGQoz8PH66+/rmnTpmnWrFnasmWLzj//fN199936yU9+4nUrGJLItT1BcnFLW23jQTt93eqqKLA9QlLxPXttjwDAMs/f83Hw4EGtXLlSo0eP1ptvvqm77rpL8+fP19q1a/utj0ajikQifR4AAGDw8nzlI5FI6Morr9SyZcskSRMnTtSePXu0atUqzZ079yv1jY2Neuihh7weAwAA+JTnKx9lZWW69NJL+7x2ySWX6OOPP+63vr6+XuFwuPfR1tbm9UgAAMBHPF/5mDx5svbu7fs73X379mnUqFH91geDQQWDPv8FOgAA8IznKx/33Xeftm3bpmXLlunAgQNat26dVq9erdpad/clAQAAg5vn4WPSpElqbm7WSy+9pHHjxumRRx7R8uXLNXv2bK9bAQCADJSWe3PecMMNuuGGG9Lxqc9thd+20jYa8u8p/Ik8S319flfbnvyA7RGSyhkx3PYIAAbo1NDU32x7XNz59jT//lQBAACDEuEDAAAYRfgAAABGET4AAIBRhA8AAGAU4QMAABhF+AAAAEb5/LQC+EHAsT1BcrEi9/vKvWTrfBG3CtuitkdIauPRFVb6Xpc1y1VdS+KPaZ8FyDR5n59KWZPVk7qmt/YbzgMAADAghA8AAGAU4QMAABhF+AAAAEYRPgAAgFGEDwAAYBRbbTNI9wXFVvqe9PEd0J1Qj+0RfCmrO257hKRsbXllCy3gH6x8AAAAowgfAADAKMIHAAAwivABAACMInwAAACjCB8AAMAottpmkJzOmJW+p4b597a2uQV2rkn+P620dS0r5t+ttmx5BTKPk5t6rcIJuF/PYOUDAAAYRfgAAABGET4AAIBRhA8AAGAU4QMAABhF+AAAAEYRPgAAgFGc85FBYkW5Vvomhto5S8ON2Bd2rkn+Cf+efSJJWZ+02x4BwCCSezScsiYQj7r+fKx8AAAAowgfAADAKMIHAAAwivABAACMInwAAACjCB8AAMAottpmEDe3NE6Hoed1WunrRl5bnpW+w3b/j5W+bm08usL2CAAGkZ4DB1PXOO6PZWDlAwAAGEX4AAAARhE+AACAUYQPAABgFOEDAAAYRfgAAABGpT18PPbYYwoEAqqrq0t3KwAAkAHSes7Hjh079Nxzz+myyy5LZ5tzRqTSzrEsI77dYaWvGyV7Elb6Brrd72cHgEzXkvhjyppIJKJQKOTq86Vt5aOzs1OzZ8/W888/r2HDhqWrDQAAyDBpCx+1tbWaPn26qqurz1oXjUYViUT6PAAAwOCVlnX8l19+Wa2trdqxY0fK2sbGRj300EPpGAMAAPiQ5ysfbW1tWrBggV588UXl5+enrK+vr1c4HO59tLW1eT0SAADwEc9XPnbt2qX29nZdccUVva/F43Ft3bpVzz77rKLRqLKzs3s/FgwGFQwGvR4DAAD4lOfh49prr9WHH37Y57V58+ZpzJgxWrx4cZ/gAQAAzj2eh4/CwkKNGzeuz2tDhgxRSUnJV17HwJwstdP3i55cO41dePel+6303bivyUpfABgMOOEUAAAYZeTUqrfeestEGwAAkAFY+QAAAEYRPgAAgFGEDwAAYBThAwAAGGXnNqn4WnIuC1vpe+S/3d2lEAAAN1j5AAAARhE+AACAUYQPAABgFOEDAAAYRfgAAABGET4AAIBRhA8AAGAU53xkkB9d+IGVvnm7h6QuusXEJACAwYCVDwAAYBThAwAAGEX4AAAARhE+AACAUYQPAABgFOEDAAAYxVbbf3Hd1Y+6qmv56wNpn6U/2YGElb5l27qt9AUADE6sfAAAAKMIHwAAwCjCBwAAMIrwAQAAjCJ8AAAAowgfAADAKMIHAAAwinM+/sVn/+7i1vEW/WH/Fa7qfj3O276b/lLv7ScEAJzTWPkAAABGET4AAIBRhA8AAGAU4QMAABhF+AAAAEYRPgAAgFFstf0Xn18Rsz3CWRX87yJ3hf+R7kkAAPj6WPkAAABGET4AAIBRhA8AAGAU4QMAABhF+AAAAEYRPgAAgFEBx3Ec20P8q0gkolAopHA4rKIil1tLPfJR2/mu6i6t+DTtswAAkEkG8vOblQ8AAGCU5+GjsbFRkyZNUmFhoYYPH66ZM2dq7969XrcBAAAZyvPwsWXLFtXW1mrbtm1qaWlRLBbT9ddfr66uLq9bAQCADOT58epvvPFGn+dr1qzR8OHDtWvXLn3/+9/3uh0AAMgwab+3SzgcliQVFxf3+/FoNKpoNNr7PBKJpHskAABgUVrfcJpIJFRXV6fJkydr3Lhx/dY0NjYqFAr1PioqKtI5EgAAsCyt4aO2tlZ79uzRyy+/nLSmvr5e4XC499HW1pbOkQAAgGVp+7XLPffcow0bNmjr1q0aOXJk0rpgMKhgMJiuMQbk5l0/cVX3EYszAAB8bZ6HD8dxdO+996q5uVlvvfWWqqqqvG4BAAAymOfho7a2VuvWrdNrr72mwsJCHTt2TJIUCoVUUFDgdTsAAJBhPH/Px8qVKxUOhzVlyhSVlZX1Pv7whz943QoAAGSgtPzaBQAAIBnu7QIAAIwifAAAAKPSfsJpJvlo5q9tjwAAwKDHygcAADCK8AEAAIwifAAAAKMIHwAAwCjCBwAAMIrwAQAAjCJ8AAAAo3x7zsfVP1+h7Lz8s9Z88PR9xuYBAADeYOUDAAAYRfgAAABGET4AAIBRhA8AAGAU4QMAABhF+AAAAEb5dqttc91TKixMlY3YagsAQKZh5QMAABhF+AAAAEYRPgAAgFGEDwAAYBThAwAAGOW73S6O40iSOjsTKWu/FYkYmAgAAKQS+fJn8umf42fju/DR0dEhSbpi0mcuqkNpnwcAALjX0dGhUOjsP58DjpuIYlAikdCRI0dUWFioQCBgexxFIhFVVFSora1NRUVFtsfxFa5N/7guyXFtkuPaJMe1Sc5P18ZxHHV0dKi8vFxZWWd/V4fvVj6ysrI0cuRI22N8RVFRkfW/WL/i2vSP65Ic1yY5rk1yXJvk/HJtUq14nMYbTgEAgFGEDwAAYBThI4VgMKhf/epXCgaDtkfxHa5N/7guyXFtkuPaJMe1SS5Tr43v3nAKAAAGN1Y+AACAUYQPAABgFOEDAAAYRfgAAABGET760djYqEmTJqmwsFDDhw/XzJkztXfvXttj+dJjjz2mQCCguro626P4wqeffqrbb79dJSUlKigo0Pjx47Vz507bY1kXj8fV0NCgqqoqFRQU6KKLLtIjjzzi6h4Qg83WrVs1Y8YMlZeXKxAIaP369X0+7jiOHnzwQZWVlamgoEDV1dXav3+/tXlNOtu1icViWrx4scaPH68hQ4aovLxcP/7xj3XkyBGrM5uS6uvmX915550KBAJavny50RkHgvDRjy1btqi2tlbbtm1TS0uLYrGYrr/+enV1ddkezVd27Nih5557TpdddpntUXzhxIkTmjx5snJzc7Vx40Z99NFH+u1vf6thw4bZHs26pqYmrVy5Us8++6z+/ve/q6mpSY8//rieeeYZ26MZ19XVpQkTJmjFihX9fvzxxx/X008/rVWrVmn79u0aMmSIpk2bpu7ubuOzmna2a3Py5Em1traqoaFBra2tevXVV7V3717deOONVmY1LdXXzWnNzc3atm2bysvLjc32tThIqb293ZHkbNmyxfYovtHR0eGMHj3aaWlpcX7wgx84CxYssD2SdYsXL3auueYa22P40vTp05077rijz2s/+tGPnNmzZ1ubyQ8kOc3Nzb3PE4mEU1pa6jzxxBO9r33++edOMBh0XnrpJUtT2nHmtenPe++950hyDh8+bGwuP0h2bT755BPn/PPPd/bs2eOMGjXK+d3vfmdlPjdY+XAhHA5LkoqLi22P4hu1tbWaPn26qqurbY/iG6+//rquvPJKzZo1S8OHD9fEiRP1/PPP2x7LF66++mpt2rRJ+/btkyR98MEHeuedd1RTU2N7NF85dOiQjh071uffVSgU0lVXXaV3333X6mx+FA6HFQgENHToUNujWJdIJDRnzhwtWrRIY8eOtT1OSr67sZzfJBIJ1dXVafLkyRo3bpztcXzh5ZdfVmtrq3bs2GF7FF85ePCgVq5cqYULF+oXv/iFduzYofnz5ysvL09z5861PZ5VS5YsUSQS0ZgxY5Sdna14PK6lS5dq9uzZtkfzlWPHjkmSRowY0ef1ESNG9H4M/093d7cWL16s2267zRc3VLOtqalJOTk5mj9/vu1RXCF8pFBbW6s9e/bonXfesT2KL7S1tWnBggVqaWlRfn6+7XF8JZFI6Morr9SyZcskSRMnTtSePXu0atWqcz58vPLKK3rxxRe1bt06jR07Vrt371ZdXZ3Ky8vP+WuDgYvFYrrlllvkOI5Wrlxpexzrdu3apaeeekqtra0KBAK2x3GFX7ucxT333KMNGzZo8+bNGjlypO1xfGHXrl1qb2/XFVdcoZycHOXk5GjLli16+umnlZOTo3g8bntEa8rKynTppZf2ee2SSy7Rxx9/bG0mv1i0aJGWLFmiW2+9VePHj9ecOXN03333qbGx0fZovlJaWipJOn78eJ/Xjx8/3vuxc93p4HH48GG1tLSw6iHp7bffVnt7uyorK3u/Lx8+fFj333+/LrjgAtvj9YuVj344jqN7771Xzc3Neuutt1RVVWV7JN+49tpr9eGHH/Z5bd68eRozZowWL16s7Oxsa7PZNnny5K9syd63b59GjRplbSa/OHnypLKy+v5fJzs7W4lEwtpMflRVVaXS0lJt2rRJl19+uSQpEolo+/btuuuuu2yPZ93p4LF//35t3rxZJSUltkfyhTlz5nzl/XfTpk3TnDlzNG/ePGtznQ3hox+1tbVat26dXnvtNRUWFvb+rjUUCqmgoMD2eFYVFhZ+5b0vQ4YMUUlJyTn/npj77rtPV199tZYtW6ZbbrlF7733nlavXq3Vq1fbHs26GTNmaOnSpaqsrNTYsWP1/vvv68knn9Qdd9xhezTjOjs7deDAgd7nhw4d0u7du1VcXKzKykrV1dXp0Ucf1ejRo1VVVaWGhgaVl5dr5syZVuc24WzXpqysTDfffLNaW1u1YcMGxePx3u/NxcXFysvLszh5+qX6ujkziOXm5qq0tFQXX3yxhWldsL3dxo8k9ft44YUXbI/mS2y1/f/+9Kc/OePGjXOCwaAzZswYZ/Xq1bZH8oVIJOIsWLDAqaysdPLz850LL7zQ+eUvf+lEo1Hboxm3efPmfr+/zJ0713G+3G7b0NDgjBgxwgkGg861117r7N271/bYRpzt2hw6dCjp9+bNmzfbHj3tUn3dnMnvW20Dzrl4xCAAALCGN5wCAACjCB8AAMAowgcAADCK8AEAAIwifAAAAKMIHwAAwCjCBwAAMIrwAQAAjCJ8AAAAowgfAADAKMIHAAAwivABAACM+r9SXh0TglR/aQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "batch_src, batch_labels, batch_padding_mask = map(lambda x: x.to(device), mkbatch(BSZ))\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    output = model(batch_src, batch_padding_mask)\n",
    "batch_src[0], batch_labels[0], output[0]\n",
    "x = batch_labels.detach().to(torch.float16).cpu().numpy().flatten()\n",
    "y = output.detach().to(torch.float16).cpu().numpy().flatten()\n",
    "plt.hist2d(x, y, bins=50, norm=mpl.colors.LogNorm())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.353515625"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LC6Xv3YfC0Rm"
   },
   "source": [
    "# Step 5: Fine Tune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "N_TUNE_EPOCHS = 100\n",
    "TUNE_LR = 1e-5\n",
    "TUNE_WD = 0 # 1e-5\n",
    "\n",
    "tune_criterion = nn.MSELoss()\n",
    "tune_optimizer = torch.optim.Adam(model.parameters(), lr=TUNE_LR, weight_decay=TUNE_WD)\n",
    "\n",
    "tune_train_err = []\n",
    "\n",
    "# clear loss file\n",
    "open('tune_loss', 'w').close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tune_evaluate():\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    with torch.no_grad():\n",
    "        batch_src, batch_labels, batch_padding_mask = mktunebatch(BSZ)\n",
    "        output = model(batch_src, batch_padding_mask)\n",
    "        loss = criterion(output.squeeze(1), batch_labels)\n",
    "    return loss.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100 \t Train Err: 2.8906\n",
      "Epoch 2/100 \t Train Err: 0.3340\n",
      "Epoch 3/100 \t Train Err: 0.1709\n",
      "Epoch 4/100 \t Train Err: 0.2373\n",
      "Epoch 5/100 \t Train Err: 0.2520\n",
      "Epoch 6/100 \t Train Err: 0.1953\n",
      "Epoch 7/100 \t Train Err: 0.1963\n",
      "Epoch 8/100 \t Train Err: 0.2236\n",
      "Epoch 9/100 \t Train Err: 0.2119\n",
      "Epoch 10/100 \t Train Err: 0.1777\n",
      "Epoch 11/100 \t Train Err: 0.1660\n",
      "Epoch 12/100 \t Train Err: 0.1787\n",
      "Epoch 13/100 \t Train Err: 0.1816\n",
      "Epoch 14/100 \t Train Err: 0.1562\n",
      "Epoch 15/100 \t Train Err: 0.1377\n",
      "Epoch 16/100 \t Train Err: 0.1377\n",
      "Epoch 17/100 \t Train Err: 0.1387\n",
      "Epoch 18/100 \t Train Err: 0.1289\n",
      "Epoch 19/100 \t Train Err: 0.1162\n",
      "Epoch 20/100 \t Train Err: 0.1079\n",
      "Epoch 21/100 \t Train Err: 0.1108\n",
      "Epoch 22/100 \t Train Err: 0.1099\n",
      "Epoch 23/100 \t Train Err: 0.1021\n",
      "Epoch 24/100 \t Train Err: 0.0918\n",
      "Epoch 25/100 \t Train Err: 0.0913\n",
      "Epoch 26/100 \t Train Err: 0.0913\n",
      "Epoch 27/100 \t Train Err: 0.0859\n",
      "Epoch 28/100 \t Train Err: 0.0820\n",
      "Epoch 29/100 \t Train Err: 0.0767\n",
      "Epoch 30/100 \t Train Err: 0.0776\n",
      "Epoch 31/100 \t Train Err: 0.0747\n",
      "Epoch 32/100 \t Train Err: 0.0713\n",
      "Epoch 33/100 \t Train Err: 0.0698\n",
      "Epoch 34/100 \t Train Err: 0.0679\n",
      "Epoch 35/100 \t Train Err: 0.0664\n",
      "Epoch 36/100 \t Train Err: 0.0669\n",
      "Epoch 37/100 \t Train Err: 0.0645\n",
      "Epoch 38/100 \t Train Err: 0.0601\n",
      "Epoch 39/100 \t Train Err: 0.0583\n",
      "Epoch 40/100 \t Train Err: 0.0569\n",
      "Epoch 41/100 \t Train Err: 0.0564\n",
      "Epoch 42/100 \t Train Err: 0.0554\n",
      "Epoch 43/100 \t Train Err: 0.0532\n",
      "Epoch 44/100 \t Train Err: 0.0520\n",
      "Epoch 45/100 \t Train Err: 0.0500\n",
      "Epoch 46/100 \t Train Err: 0.0483\n",
      "Epoch 47/100 \t Train Err: 0.0457\n",
      "Epoch 48/100 \t Train Err: 0.0452\n",
      "Epoch 49/100 \t Train Err: 0.0444\n",
      "Epoch 50/100 \t Train Err: 0.0430\n",
      "Epoch 51/100 \t Train Err: 0.0422\n",
      "Epoch 52/100 \t Train Err: 0.0405\n",
      "Epoch 53/100 \t Train Err: 0.0408\n",
      "Epoch 54/100 \t Train Err: 0.0378\n",
      "Epoch 55/100 \t Train Err: 0.0378\n",
      "Epoch 56/100 \t Train Err: 0.0369\n",
      "Epoch 57/100 \t Train Err: 0.0354\n",
      "Epoch 58/100 \t Train Err: 0.0344\n",
      "Epoch 59/100 \t Train Err: 0.0337\n",
      "Epoch 60/100 \t Train Err: 0.0334\n",
      "Epoch 61/100 \t Train Err: 0.0322\n",
      "Epoch 62/100 \t Train Err: 0.0312\n",
      "Epoch 63/100 \t Train Err: 0.0304\n",
      "Epoch 64/100 \t Train Err: 0.0310\n",
      "Epoch 65/100 \t Train Err: 0.0304\n",
      "Epoch 66/100 \t Train Err: 0.0297\n",
      "Epoch 67/100 \t Train Err: 0.0283\n",
      "Epoch 68/100 \t Train Err: 0.0281\n",
      "Epoch 69/100 \t Train Err: 0.0280\n",
      "Epoch 70/100 \t Train Err: 0.0273\n",
      "Epoch 71/100 \t Train Err: 0.0267\n",
      "Epoch 72/100 \t Train Err: 0.0277\n",
      "Epoch 73/100 \t Train Err: 0.0269\n",
      "Epoch 74/100 \t Train Err: 0.0258\n",
      "Epoch 75/100 \t Train Err: 0.0249\n",
      "Epoch 76/100 \t Train Err: 0.0254\n",
      "Epoch 77/100 \t Train Err: 0.0245\n",
      "Epoch 78/100 \t Train Err: 0.0244\n",
      "Epoch 79/100 \t Train Err: 0.0242\n",
      "Epoch 80/100 \t Train Err: 0.0237\n",
      "Epoch 81/100 \t Train Err: 0.0243\n",
      "Epoch 82/100 \t Train Err: 0.0225\n",
      "Epoch 83/100 \t Train Err: 0.0225\n",
      "Epoch 84/100 \t Train Err: 0.0221\n",
      "Epoch 85/100 \t Train Err: 0.0227\n",
      "Epoch 86/100 \t Train Err: 0.0222\n",
      "Epoch 87/100 \t Train Err: 0.0219\n",
      "Epoch 88/100 \t Train Err: 0.0220\n",
      "Epoch 89/100 \t Train Err: 0.0210\n",
      "Epoch 90/100 \t Train Err: 0.0210\n",
      "Epoch 91/100 \t Train Err: 0.0211\n",
      "Epoch 92/100 \t Train Err: 0.0208\n",
      "Epoch 93/100 \t Train Err: 0.0205\n",
      "Epoch 94/100 \t Train Err: 0.0200\n",
      "Epoch 95/100 \t Train Err: 0.0208\n",
      "Epoch 96/100 \t Train Err: 0.0198\n",
      "Epoch 97/100 \t Train Err: 0.0195\n",
      "Epoch 98/100 \t Train Err: 0.0197\n",
      "Epoch 99/100 \t Train Err: 0.0190\n",
      "Epoch 100/100 \t Train Err: 0.0192\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(N_TUNE_EPOCHS):\n",
    "    model.train()\n",
    "    train_loss = 0\n",
    "    batch_src, batch_labels, batch_padding_mask = mktunebatch(BSZ)\n",
    "    optimizer.zero_grad()\n",
    "    output = model(batch_src, batch_padding_mask)\n",
    "    loss = criterion(output.squeeze(1), batch_labels)\n",
    "    train_loss = loss.item()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    \n",
    "    tune_train_err.append(train_loss)\n",
    "    with open('tune_loss', 'a') as f:\n",
    "        f.write(f\"{train_loss}\\n\")\n",
    "    print(f\"Epoch {epoch}/{N_TUNE_EPOCHS} \\t Train Err: {train_loss:.4f}\")\n",
    "\n",
    "    if epoch % 10 == 9:\n",
    "        torch.save(model.state_dict(), f\"tune_model_weights_{epoch + 1}.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHgCAYAAABZ+0ykAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA97ElEQVR4nO3deXxU1f3/8feEkEkgC2DIAoRFQVRQoCAQaAVrEAKlYlGRokTEBQGFUtuvaAXbfm1UiktdWL4tUquIwk9RqcsjBYGCgGyx4IILslRIEJFsSAKZ8/tjmEmGhJjInTnJ8Ho+HueRzL3nznzufVDz7rnnnnEZY4wAAADCRITtAgAAAJxEuAEAAGGFcAMAAMIK4QYAAIQVwg0AAAgrhBsAABBWCDcAACCsEG4AAEBYIdwAAICwQrgBgCByuVyaPHmy7TKAswrhBmjAFi5cKJfLJZfLpbVr11bZb4xRWlqaXC6XfvaznwXsKy4u1syZM9W1a1c1bdpU55xzjrp3764pU6Zo//79/n4PPPCA/zOqa3l5eSE519OpqbYJEyZYrQ2AHZG2CwBw5qKjo7Vo0SL9+Mc/Dti+evVq/fe//5Xb7Q7Yfvz4cV122WX65JNPlJWVpTvvvFPFxcX68MMPtWjRIl199dVq1apVwDFz5sxRbGxslc9u1qxZkM6q9gYNGqSxY8dW2X7++edbqQeAXYQbIAwMHTpUS5Ys0V/+8hdFRlb8z3rRokXq2bOnDh06FNB/2bJl2rZtm1544QX98pe/DNh37NgxlZWVVfmMa665RomJiUE8ix/u/PPP1w033GC7DAD1BLelgDAwevRoffPNN8rJyfFvKysr09KlS6uEF0n64osvJEn9+/evsi86Olrx8fGO1NW1a1ddfvnlVbZ7PB61bt1a11xzjX/b4sWL1bNnT8XFxSk+Pl4XX3yxnnjiCUfqkKSBAweqa9eu2rJli/r166eYmBh16NBBc+fOrdL34MGDGj9+vJKTkxUdHa1u3brp73//e7Xn8cQTT+jiiy9WdHS0WrZsqSFDhmjz5s1V+i5btkxdu3aV2+1Wly5d9PbbbwfsLyoq0tSpU9W+fXu53W4lJSVp0KBB2rp1q2PXADhbEG6AMNC+fXulp6frxRdf9G976623VFBQoOuvv75K/3bt2kmSnnvuORljavUZhw8f1qFDhwLakSNHajxm1KhRWrNmTZV5OWvXrtX+/fv9teXk5Gj06NFq3ry5Hn74YT300EMaOHCg1q1bV6vajh07VqW2Q4cOVRmB+vbbbzV06FD17NlTjzzyiNq0aaM77rhDCxYs8Pf57rvvNHDgQP3jH//QmDFjNGvWLCUkJOimm26qErbGjx+vqVOnKi0tTQ8//LDuueceRUdHa8OGDVXOd+LEibr++uv1yCOP6NixYxo5cqS++eYbf58JEyZozpw5GjlypJ555hndfffdiomJ0ccff1yrawCgEgOgwXr22WeNJLNp0ybz1FNPmbi4OHP06FFjjDHXXnutufzyy40xxrRr184MGzbMf9zRo0dN586djSTTrl07c9NNN5m//e1vJj8/v8pnzJw500iqtnXu3LnG+nbu3GkkmSeffDJg+8SJE01sbKy/1ilTppj4+Hhz4sSJOl+D09Umybz44ov+fgMGDDCSzOzZs/3bSktLTffu3U1SUpIpKyszxhjz+OOPG0nm+eef9/crKysz6enpJjY21hQWFhpjjFm5cqWRZO66664qNXk8noD6oqKizOeff+7f9sEHH1S5LgkJCWbSpEl1Pn8AVTFyA4SJ6667Tt99952WL1+uoqIiLV++vNpbUpIUExOjjRs36je/+Y108qmr8ePHKzU1VXfeeadKS0urHPP//t//U05OTkB79tlna6zp/PPPV/fu3fXSSy/5t5WXl2vp0qUaPny4YmJipJOTkktKSgJuq9XFVVddVaW2nJycKrfEIiMjdfvtt/tfR0VF6fbbb9fBgwe1ZcsWSdKbb76plJQUjR492t+vcePGuuuuu1RcXKzVq1f7r4fL5dLMmTOr1ONyuQJeZ2Rk6LzzzvO/vuSSSxQfH69du3b5tzVr1kwbN24MeFINwA/DhGIgTLRs2VIZGRlatGiRjh49qvLy8oA5LadKSEjQI488okceeUR79uzRihUr9Oc//1lPPfWUEhIS9L//+78B/S+77LIfNKF41KhRuvfee/XVV1+pdevWWrVqlQ4ePKhRo0b5+0ycOFEvv/yyMjMz1bp1a1155ZW67rrrNGTIkFp9Rps2bZSRkfG9/Vq1aqWmTZsGbPM9UbV792717dtXe/bsUadOnRQREfj//S688EJJ0p49e6ST85ZatWqlFi1afO/ntm3btsq25s2b69tvv/W/fuSRR5SVlaW0tDT17NlTQ4cO1dixY3Xuued+7/sDCMTIDRBGfvnLX+qtt97S3LlzlZmZWevHtNu1a6ebb75Z69atU7NmzfTCCy84VtOoUaNkjNGSJUskSS+//LISEhICgktSUpJyc3P1+uuv6+c//7neffddZWZmKisry7E6bGrUqFG12yvPd7ruuuu0a9cuPfnkk2rVqpVmzZqlLl266K233gphpUB4INwAYeTqq69WRESENmzYcNpbUjVp3ry5zjvvPB04cMCxmjp06KDevXvrpZde0okTJ/TKK69oxIgRVdbeiYqK0vDhw/XMM8/oiy++0O23367nnntOn3/+uWO17N+/XyUlJQHbPv30U+nkpGydDHqfffaZPB5PQL9PPvnEv1+SzjvvPO3fv1+HDx92rL7U1FRNnDhRy5Yt05dffqlzzjlHDz74oGPvD5wtCDdAGImNjdWcOXP0wAMPaPjw4aft98EHH1RZ+0Ynb7l89NFH6ty5s6N1jRo1Shs2bNCCBQt06NChgFtSkgKeGpKkiIgIXXLJJZJU7fyfH+rEiROaN2+e/3VZWZnmzZunli1bqmfPntLJNYPy8vIC5gmdOHFCTz75pGJjYzVgwABJ0siRI2WM0e9///sqn1PbJ9B8ysvLVVBQELAtKSlJrVq1cvT8gbMFc26AMFObWzk5OTmaOXOmfv7zn6tv376KjY3Vrl27tGDBApWWluqBBx6ocszSpUurXaF40KBBSk5OrvHzrrvuOt199926++671aJFiyrzY2655RYdPnxYP/3pT9WmTRvt2bNHTz75pLp37+6f61KTTz/9VM8//3yV7cnJyRo0aJD/datWrfTwww9r9+7dOv/88/XSSy8pNzdX8+fPV+PGjSVJt912m+bNm6ebbrpJW7ZsUfv27bV06VKtW7dOjz/+uOLi4iRJl19+uW688Ub95S9/0WeffaYhQ4bI4/Ho3//+ty6//PI6fZ9UUVGR2rRpo2uuuUbdunVTbGys/vWvf2nTpk2aPXt2rd8HwEm2H9cC8MNVfhS8Jqc+Cr5r1y4zY8YM07dvX5OUlGQiIyNNy5YtzbBhw8zKlSsDjq3pUXBJ5t13361Vrf379zeSzC233FJl39KlS82VV15pkpKSTFRUlGnbtq25/fbbzYEDB773fWuqbcCAAf5+AwYMMF26dDGbN2826enpJjo62rRr18489dRTVd4zPz/fjBs3ziQmJpqoqChz8cUXm2effbZKvxMnTphZs2aZCy64wERFRZmWLVuazMxMs2XLloD6qnvEu127diYrK8uYk4+k/+Y3vzHdunUzcXFxpmnTpqZbt27mmWee+d7zB1CVy9R1/BQAGqCBAwfq0KFD2rFjh+1SAAQZc24AAEBYIdwAAICwQrgBAABhhTk3AAAgrDByAwAAwgrhBgAAhBXCDQAACCuEGwAAEFYINwAAIKwQbgAAQFgh3AAAgLBCuAEAAGGFcAMAAMIK4QYAAIQVwg0AAAgrhBsAABBWCDcAACCsEG4AAEBYIdwAAICwQrgBAABhhXADAADCCuEGAACEFcINAAAIK4QbAAAQVgg3AAAgrBBuAABAWCHcAACAsEK4AQAAYYVwAwAAwgrhBgAAhBXCDQAACCuRtgsINY/Ho/379ysuLk4ul8t2OQAAoBaMMSoqKlKrVq0UEVHz2MxZF27279+vtLQ022UAAIAfYN++fWrTpk2NfayGmzlz5mjOnDnavXu3JKlLly6aMWOGMjMzT3vMkiVLdP/992v37t3q1KmTHn74YQ0dOrTWnxkXFyedvDjx8fEOnAUAAAi2wsJCpaWl+f+O18RquGnTpo0eeughderUScYY/f3vf9dVV12lbdu2qUuXLlX6v/feexo9erSys7P1s5/9TIsWLdKIESO0detWde3atVaf6bsVFR8fT7gBAKCBqc2UEpcxxoSkmlpq0aKFZs2apfHjx1fZN2rUKJWUlGj58uX+bX379lX37t01d+7cat+vtLRUpaWl/te+5FdQUEC4AQCggSgsLFRCQkKt/n7Xm6elysvLtXjxYpWUlCg9Pb3aPuvXr1dGRkbAtsGDB2v9+vWnfd/s7GwlJCT4G/NtAAAIb9bDzfbt2xUbGyu3260JEybo1Vdf1UUXXVRt37y8PCUnJwdsS05OVl5e3mnff/r06SooKPC3ffv2OX4OAACg/rD+tFTnzp2Vm5urgoICLV26VFlZWVq9evVpA05dud1uud1uR94LAADUf9bDTVRUlDp27ChJ6tmzpzZt2qQnnnhC8+bNq9I3JSVF+fn5Advy8/OVkpISsnoBAED9Zv221Kk8Hk/ABODK0tPTtWLFioBtOTk5p52jAwAAzj5WR26mT5+uzMxMtW3bVkVFRVq0aJFWrVqld955R5I0duxYtW7dWtnZ2ZKkKVOmaMCAAZo9e7aGDRumxYsXa/PmzZo/f77N0wAAAPWI1XBz8OBBjR07VgcOHFBCQoIuueQSvfPOOxo0aJAkae/evQFLLPfr10+LFi3S7373O917773q1KmTli1bVus1bgAAQPird+vcBFtdnpMHAAD1Q4Nc5wYAAMAJhBsAABBWCDcAACCsWF/nJlyUlkr5+ZLLJfENDwAA2MPIjUO2bJHatZN++lPblQAAcHYj3Dgk8uQY2IkTtisBAODsRrhxCOEGAID6gXDjkEaNvD8JNwAA2EW4cQgjNwAA1A+EG4cQbgAAqB8INw7xhZvyctuVAABwdiPcOISRGwAA6gfCjUMINwAA1A+EG4cQbgAAqB8INw7xhRtjJI/HdjUAAJy9CDcOiaz0LV2M3gAAYA/hxiGEGwAA6gfCjUN8KxSLcAMAgFWEG4cwcgMAQP1AuHEIIzcAANQPhBuHuFwVAYdVigEAsIdw4yDWugEAwD7CjYMINwAA2Ee4cRDhBgAA+wg3DiLcAABgH+HGQYQbAADsI9w4iHADAIB9hBsH+R4FJ9wAAGAP4cZBjNwAAGAf4cZBhBsAAOwj3DjIF25YoRgAAHsINw5i5AYAAPsINw4i3AAAYB/hxkGEGwAA7CPcOIhwAwCAfYQbBxFuAACwj3DjIMINAAD2EW4cxArFAADYR7hxECM3AADYR7hxEIv4AQBgH+HGQYzcAABgH+HGQYQbAADsI9w4iHADAIB9hBsHEW4AALCPcOMgwg0AAPYRbhxEuAEAwD7CjYNYxA8AAPushpvs7GxdeumliouLU1JSkkaMGKGdO3fWeMzChQvlcrkCWnR0dMhqrgkjNwAA2Gc13KxevVqTJk3Shg0blJOTo+PHj+vKK69USUlJjcfFx8frwIED/rZnz56Q1VwTwg0AAPZF2vzwt99+O+D1woULlZSUpC1btuiyyy477XEul0spKSkhqLBuWKEYAAD76tWcm4KCAklSixYtauxXXFysdu3aKS0tTVdddZU+/PDD0/YtLS1VYWFhQAsWRm4AALCv3oQbj8ejqVOnqn///uratetp+3Xu3FkLFizQa6+9pueff14ej0f9+vXTf//732r7Z2dnKyEhwd/S0tKCdg6EGwAA7Ks34WbSpEnasWOHFi9eXGO/9PR0jR07Vt27d9eAAQP0yiuvqGXLlpo3b161/adPn66CggJ/27dvX5DOgHADAEB9YHXOjc/kyZO1fPlyrVmzRm3atKnTsY0bN1aPHj30+eefV7vf7XbL7XY7VGnNCDcAANhndeTGGKPJkyfr1Vdf1cqVK9WhQ4c6v0d5ebm2b9+u1NTUoNRYF4QbAADsszpyM2nSJC1atEivvfaa4uLilJeXJ0lKSEhQTEyMJGns2LFq3bq1srOzJUl/+MMf1LdvX3Xs2FFHjhzRrFmztGfPHt1yyy02T0Ui3AAAUC9YDTdz5syRJA0cODBg+7PPPqubbrpJkrR3715FRFQMMH377be69dZblZeXp+bNm6tnz5567733dNFFF4W4+qpYoRgAAPushhtjzPf2WbVqVcDrxx57TI899lgQq/rhGLkBAMC+evO0VDgg3AAAYB/hxkGsUAwAgH2EGwcxcgMAgH2EGwcRbgAAsI9w4yDCDQAA9hFuHES4AQDAPsKNgwg3AADYR7hxEOEGAAD7CDcOYoViAADsI9w4iJEbAADsI9w4iHADAIB9hBsHsUIxAAD2EW4cxMgNAAD2EW4cRLgBAMA+wo2DCDcAANhHuHEQ4QYAAPsINw4i3AAAYB/hxkGEGwAA7CPcOIgVigEAsI9w4yBGbgAAsI9w4yDCDQAA9hFuHFR5hWJjbFcDAMDZiXDjIF+4kSSPx2YlAACcvQg3Dqocbrg1BQCAHYQbBxFuAACwj3DjIMINAAD2EW4cRLgBAMA+wo2DIipdTcINAAB2EG4cxlo3AADYRbhxGOEGAAC7CDcOI9wAAGAX4cZhlVcpBgAAoUe4cRgjNwAA2EW4cRjhBgAAuwg3DiPcAABgF+HGYYQbAADsItw4jHADAIBdhBuHNWrk/Um4AQDADsKNwxi5AQDALsKNwwg3AADYRbhxGOEGAAC7CDcOY4ViAADsItw4jJEbAADsItw4jHADAIBdhBuHEW4AALCLcOMwwg0AAHYRbhxGuAEAwC7CjcNYoRgAALushpvs7GxdeumliouLU1JSkkaMGKGdO3d+73FLlizRBRdcoOjoaF188cV68803Q1JvbTByAwCAXVbDzerVqzVp0iRt2LBBOTk5On78uK688kqVlJSc9pj33ntPo0eP1vjx47Vt2zaNGDFCI0aM0I4dO0Ja++kQbgAAsMtljDG2i/D5+uuvlZSUpNWrV+uyyy6rts+oUaNUUlKi5cuX+7f17dtX3bt319y5c6v0Ly0tVWlpqf91YWGh0tLSVFBQoPj4eMfPYcwYadEi6dFHpV/9yvG3BwDgrFRYWKiEhIRa/f2uV3NuCgoKJEktWrQ4bZ/169crIyMjYNvgwYO1fv36avtnZ2crISHB39LS0hyuOhArFAMAYFe9CTcej0dTp05V//791bVr19P2y8vLU3JycsC25ORk5eXlVdt/+vTpKigo8Ld9+/Y5Xntl3JYCAMCuSNsF+EyaNEk7duzQ2rVrHX1ft9stt9vt6HvWhHADAIBd9SLcTJ48WcuXL9eaNWvUpk2bGvumpKQoPz8/YFt+fr5SUlKCXGXtEG4AALDL6m0pY4wmT56sV199VStXrlSHDh2+95j09HStWLEiYFtOTo7S09ODWGntEW4AALDL6sjNpEmTtGjRIr322muKi4vzz5tJSEhQTEyMJGns2LFq3bq1srOzJUlTpkzRgAEDNHv2bA0bNkyLFy/W5s2bNX/+fJun4ke4AQDALqsjN3PmzFFBQYEGDhyo1NRUf3vppZf8ffbu3asDBw74X/fr10+LFi3S/Pnz1a1bNy1dulTLli2rcRJyKLFCMQAAdlkduanNEjurVq2qsu3aa6/VtddeG6SqzgwjNwAA2FVvHgUPF4QbAADsItw4jHADAIBdhBuHsUIxAAB2EW4cxsgNAAB2EW4cRrgBAMAuwo3DCDcAANhFuHEY4QYAALsINw4j3AAAYBfhxmGsUAwAgF2EG4cxcgMAgF2EG4cRbgAAsItw4zDCDQAAdhFuHMYKxQAA2EW4cRgjNwAA2EW4cRjhBgAAuwg3DiPcAABgF+HGYYQbAADsItw4jEX8AACwi3DjMEZuAACwi3DjMMINAAB2EW4cRrgBAMAuwo3DWMQPAAC7CDcOY+QGAAC7CDcOI9wAAGAX4cZhhBsAAOwi3DiMcAMAgF2EG4cRbgAAsItw4zBWKAYAwC7CjcN8Izcej7cBAIDQItw4zBduxFo3AABYQbhxWOVww60pAABCj3DjMEZuAACwi3DjMEZuAACwi3DjMN/TUiLcAABgBeHGYRER3ibCDQAAVhBugoCF/AAAsIdwEwSEGwAA7CHcBAGrFAMAYA/hJggYuQEAwB7CTRAQbgAAsIdwEwSEGwAA7CHcBIEv3LBCMQAAoUe4CQJGbgAAsIdwEwSEGwAA7CHcBAHhBgAAewg3QUC4AQDAnjqFm0ceeUTfffed//W6detUWlrqf11UVKSJEyc6W2EDRLgBAMCeOoWb6dOnq6ioyP86MzNTX331lf/10aNHNW/evFq/35o1azR8+HC1atVKLpdLy5Ytq7H/qlWr5HK5qrS8vLy6nEbQsUIxAAD21CncGGNqfF1XJSUl6tatm55++uk6Hbdz504dOHDA35KSks6oDqcxcgMAgD2RNj88MzNTmZmZdT4uKSlJzZo1C0pNTiDcAABgT4OcUNy9e3elpqZq0KBBWrduXY19S0tLVVhYGNCCjXADAIA9dR65+etf/6rY2FhJ0okTJ7Rw4UIlJiZKJycUB1Nqaqrmzp2rXr16qbS0VH/96181cOBAbdy4UT/60Y+qPSY7O1u///3vg1rXqVihGAAAe1ymDhNn2rdvL5fL9b39vvzyy7oX4nLp1Vdf1YgRI+p03IABA9S2bVv94x//qHZ/aWlpwBNdhYWFSktLU0FBgeLj4+tcZ21kZkpvvy39/e/S2LFB+QgAAM4qhYWFSkhIqNXf7zqN3OzevftMa3Nc7969tXbt2tPud7vdcrvdIa2J21IAANjTIOfcVJabm6vU1FTbZQQg3AAAYE+dws369eu1fPnygG3PPfecOnTooKSkJN12220Bt4C+T3FxsXJzc5WbmyudvJ2Vm5urvXv3SifX1Rlb6b7O448/rtdee02ff/65duzYoalTp2rlypWaNGlSXU4j6Ag3AADYU6dw84c//EEffvih//X27ds1fvx4ZWRk6J577tEbb7yh7OzsWr/f5s2b1aNHD/Xo0UOSNG3aNPXo0UMzZsyQJB04cMAfdCSprKxMv/71r3XxxRdrwIAB+uCDD/Svf/1LV1xxRV1OI+hYxA8AAHvqNKE4NTVVb7zxhnr16iVJuu+++7R69Wr/nJclS5Zo5syZ+uijj4JX8Rmqy4SkH+qGG6QXXpBmz5amTQvKRwAAcFapy9/vOo3cfPvtt0pOTva/Xr16dcAifJdeeqn27dv3Q2oOK9yWAgDAnjqFm+TkZP9j3mVlZdq6dav69u3r319UVKTGjRs7X2UDQ7gBAMCeOoWboUOH6p577tG///1vTZ8+XU2aNNFPfvIT//7//Oc/Ou+884JRZ4NCuAEAwJ46rXPzxz/+Ub/4xS80YMAAxcbGauHChYqKivLvX7Bgga688spg1NmgsEIxAAD21CncJCYmas2aNSooKFBsbKwa+R4LOmnJkiWKi4tzusYGh5EbAADsqVO4ufnmm2vVb8GCBT+0nrBAuAEAwJ46hZuFCxeqXbt26tGjh+rwBPlZh3ADAIA9dQo3d9xxh1588UV9+eWXGjdunG644Qa1aNEieNU1UIQbAADsqdPTUk8//bQOHDig3/72t3rjjTeUlpam6667Tu+88w4jOZWwQjEAAPbU+Ysz3W63Ro8erZycHH300Ufq0qWLJk6cqPbt26u4uDg4VTYwjNwAAGDPGX0reEREhFwul4wxKue5Zz/CDQAA9tQ53JSWlurFF1/UoEGDdP7552v79u166qmntHfvXsXGxganygaGcAMAgD11mlA8ceJELV68WGlpabr55pv14osvKjExMXjVNVCEGwAA7KlTuJk7d67atm2rc889V6tXr9bq1aur7ffKK684VV+DxArFAADYU6dwM3bsWLlcruBVEyYYuQEAwJ46L+KH70e4AQDAnjN6WgrVI9wAAGAP4SYICDcAANhDuAkCVigGAMAewk0QMHIDAIA9hJsgINwAAGAP4SYICDcAANhDuAkCwg0AAPYQboKAFYoBALCHcBMEjNwAAGAP4SYICDcAANhDuAkCwg0AAPYQboKAcAMAgD2EmyBghWIAAOwh3AQBIzcAANhDuAkCwg0AAPYQboKAcAMAgD2EmyAg3AAAYA/hJghYoRgAAHsIN0HAyA0AAPYQboKAcAMAgD2EmyDwhRtjJI/HdjUAAJxdCDdB4FvET4zeAAAQcoSbIPCN3IhwAwBAyBFugoBwAwCAPYSbICDcAABgD+EmCJhzAwCAPYSbIHC5+GZwAABsIdwECasUAwBgB+EmSFjIDwAAOwg3QUK4AQDADsJNkBBuAACwg3ATJEwoBgDADqvhZs2aNRo+fLhatWoll8ulZcuWfe8xq1at0o9+9CO53W517NhRCxcuDEmtdcXIDQAAdlgNNyUlJerWrZuefvrpWvX/8ssvNWzYMF1++eXKzc3V1KlTdcstt+idd94Jeq11RbgBAMCOyFr0CZrMzExlZmbWuv/cuXPVoUMHzZ49W5J04YUXau3atXrsscc0ePDgao8pLS1VaWmp/3VhYaEDlX8/wg0AAHY0qDk369evV0ZGRsC2wYMHa/369ac9Jjs7WwkJCf6WlpYWgkoJNwAA2NKgwk1eXp6Sk5MDtiUnJ6uwsFDfffddtcdMnz5dBQUF/rZv376Q1Eq4AQDADqu3pULB7XbL7XaH/HNZoRgAADsa1MhNSkqK8vPzA7bl5+crPj5eMTEx1uqqDiM3AADY0aDCTXp6ulasWBGwLScnR+np6dZqOh3CDQAAdlgNN8XFxcrNzVVubq508lHv3Nxc7d27Vzo5X2bs2LH+/hMmTNCuXbv029/+Vp988omeeeYZvfzyy/rVr35l7RxOh3ADAIAdVsPN5s2b1aNHD/Xo0UOSNG3aNPXo0UMzZsyQJB04cMAfdCSpQ4cO+uc//6mcnBx169ZNs2fP1l//+tfTPgZuEysUAwBgh9UJxQMHDpQx5rT7q1t9eODAgdq2bVuQKztzjNwAAGBHg5pz05AQbgAAsINwEySEGwAA7CDcBAnhBgAAOwg3QcIifgAA2EG4CRJGbgAAsINwEySEGwAA7CDcBAnhBgAAOwg3QUK4AQDADsJNkLBCMQAAdhBugoSRGwAA7CDcBAnhBgAAOwg3QUK4AQDADsJNkBBuAACwg3ATJKxQDACAHYSbIGHkBgAAOwg3QUK4AQDADsJNkBBuAACwg3ATJCziBwCAHYSbIGHkBgAAOwg3QUK4AQDADsJNkBBuAACwg3ATJIQbAADsINwECeEGAAA7CDdBwgrFAADYQbgJEkZuAACwg3ATJIQbAADsINwECeEGAAA7CDdBwgrFAADYQbgJEkZuAACwg3ATJIQbAADsINwECeEGAAA7CDdBQrgBAMAOwk2QEG4AALCDcBMkrFAMAIAdhJsgYeQGAAA7CDdBQrgBAMAOwk2QEG4AALCDcBMkrFAMAIAdhJsgYeQGAAA7CDdBQrgBAMAOwk2QEG4AALCDcBMkhBsAAOwg3ARJ5UX8jLFdDQAAZw/CTZD4wo0keTw2KwEA4OxCuAmSyuGGW1MAAIQO4SZICDcAANhBuAkSwg0AAHbUi3Dz9NNPq3379oqOjlafPn30/vvvn7bvwoUL5XK5Alp0dHRI660N3wrFItwAABBS1sPNSy+9pGnTpmnmzJnaunWrunXrpsGDB+vgwYOnPSY+Pl4HDhzwtz179oS05tqIqHRlCTcAAISO9XDz6KOP6tZbb9W4ceN00UUXae7cuWrSpIkWLFhw2mNcLpdSUlL8LTk5OaQ114bLxVo3AADYYDXclJWVacuWLcrIyKgoKCJCGRkZWr9+/WmPKy4uVrt27ZSWlqarrrpKH3744Wn7lpaWqrCwMKCFCuEGAIDQsxpuDh06pPLy8iojL8nJycrLy6v2mM6dO2vBggV67bXX9Pzzz8vj8ahfv37673//W23/7OxsJSQk+FtaWlpQzqU6hBsAAELP+m2pukpPT9fYsWPVvXt3DRgwQK+88opatmypefPmVdt/+vTpKigo8Ld9+/aFrFbCDQAAoRdZiz5Bk5iYqEaNGik/Pz9ge35+vlJSUmr1Ho0bN1aPHj30+eefV7vf7XbL7XY7Um9dVf4KBgAAEBpWR26ioqLUs2dPrVixwr/N4/FoxYoVSk9Pr9V7lJeXa/v27UpNTQ1ipT8MIzcAAISe1ZEbSZo2bZqysrLUq1cv9e7dW48//rhKSko0btw4SdLYsWPVunVrZWdnS5L+8Ic/qG/fvurYsaOOHDmiWbNmac+ePbrlllssn0lVhBsAAELPergZNWqUvv76a82YMUN5eXnq3r273n77bf8k47179yqi0qIx3377rW699Vbl5eWpefPm6tmzp9577z1ddNFFFs+ier6F/Ag3AACEjssYY2wXEUqFhYVKSEhQQUGB4uPjg/pZHTtKX3whrVsn9esX1I8CACCs1eXvd4N7Wqoh4bYUAAChR7gJIsINAAChR7gJIsINAAChR7gJIsINAAChR7gJIsINAAChR7gJIlYoBgAg9Ag3QcTIDQAAoUe4CSLCDQAAoUe4CSJWKAYAIPQIN0HEyA0AAKFHuAkiwg0AAKFHuAkiX7g5ftx2JQAAnD0IN0F08ovN9cUXtisBAODsQbgJov79vT///W/blQAAcPYg3ATRT37i/bl1q1RcbLsaAADODoSbIGrb1tvKy6UNG2xXAwDA2YFwE2S+0RtuTQEAEBqEmyDzhZs1a2xXAgDA2YFwE2S+cLNhg1RWZrsaAADCH+EmyC68UDrnHOnYMWnLFtvVAAAQ/gg3QeZyST/+sff32sy7MUZav17KzQ16aQAAhCXCTQjUZlLxnj3SH/8odewo9esn9eolbdoUshIBAAgbkbYLOBtcdpn357p1kscjRVSKlJ9/Lt1xh7RihXfUxqe8XLrxRmnbNikmJvQ1AwDQUDFyEwI9ekhNm0rffit9+GHFdo9HGjNG+te/vMHmpz+VnntO2rtXSk2Vdu6Upk+v/ecYExiQAAA4GxFuQiAyUkpP9/5e+dbUc89J778vxcVJn3ziHb258UYpLU3629+8fZ54Qlq5sub393ikRx/1Tlw+91zprruknByezgIAnJ0INyFy6rybggLpf/7H+/uMGVLnzoH9MzOl22/3/n7TTd7+1dm3Txo0SPr1r70jQ7t3S08+KV15pZSYKN1wg3TwYNBOCwCAeodwEyKVw40x0h/+4A0d55/vHWmpzp//7B2J2bdPmjKl6v6XXpIuucQ7stOkiTRnjrRsmTR+vPcbyYuKpBde8I4affppcM8PAID6wmXM2TVLo7CwUAkJCSooKFB8fHzIPvfoUalZM+n4cemf/5Suuko6cUJ66y1pyJDTH7dunTcYGSP17u2daFxW5n2/L77w9rn0Uun5571Bycfj8T5SfuON0pdfem9Zvf6690ksAAAamrr8/WbkJkSaNJF69vT+/stfeoPN8OE1BxtJ6t+/4vbV++97FwLcvt0bbCIivLe01q0LDDaSd1///t6A06uX9M030hVXSK+8Ure6y8ulQ4ek/Py6HQcAgC2M3ITQb38rzZrl/T0qyvvkVMeO339cebn0zjtSaankdnuPdbuldu283zr+fUpKpOuvl5Yv9y4qeNdd0rXXSn37So0aVfT7+mvvba1ly6TPPvOGmiNHKp7AGjFCeuYZ75NcAACEUl3+fhNuQuiNN6Sf/9z7+/Tp0p/+FLrPPnHCG2rmzKnYlpgoDR0qde3qvT22erX3dlZ1XC5vyGnWzPtk1k03ebcBABAKhJsa2Aw3BQVSp05SQoJ3cb7Y2JB+vIyRXntNevllb5g5cqRqn549pZEjvXNzWrb0ztVp0cL7qPrNN0ubN3v7XXmlNG+e1L59aM8BAHB2ItzUwGa40cmA06hR6IPNqY4f987VeeMN72KBl18u/eIXUocOpz/mxAnpsce883yOHfPeGhs3znu7rabjAAA4U4SbGtgON+Hg00+9a/CsWuV93aiRNHq0dM890kUXBfb1Pd1VWur9efy4d85O5bk+AAB8H8JNDQg3zjDGu2bPn/7knexcF/Hx3u/bGjjQO2LUrRthBwBQM8JNDQg3ztuyRcrO9j5m/n3/mho18o7mVNa8uXfdn5Ejvastu91BLRcA0AARbmpAuAmeoiLvXJzKIiK8YcXt9n7Hlscj5eZ6b2m9+6539KewsKJ/fLz0s59JgwdLffp4J2BHsBoTAJz1CDc1INzULydOSO+9Jy1d6h35+eqrwP3NmnlXYL7oIqm42Lv2zjffeFt5ecWaP1FR3oUSu3b1PvHVq5d3YUNudwFAeCDc1IBwU395PNLGjd5FBNet897uOnUkqC6aNvV+ZcUVV3hbr17e0SMAQMNDuKkB4abhOH5c2rHDG3i++MI7inPOORUtKsr7FJbvSaxvv/Xe8tqyxbuO0NGjge/nm8icmOgd0fG1iIiqCxKmpkrnnVfRmjcP6akDAE5BuKkB4ebsUF4uffyxtGaNtGKF95vTq1u0sLaaNZNatQps7dp5g0/HjlJaGqNCABBMhJsaEG7OTuXl3tGc9eul777zvj5xwvvz1K+cKC+X/vtf72jR55/X7ktDIyO9oz2nrusTHe0dZUpMrPiZlORtycnen82be+cL+VpMjHeRx8aNg3Y5AKDBIdzUgHCDuioulvbskQ4ckPbv9/786ivpyy+9AWjXLm+YcZrbLcXFeVtsrDf0VA5AxnjDlK9FRHi/2qNZs4qWkuIdVUpLk1q39oYtAGiI6vL3m4F04HvExkpdunhbdTweb9jZv9872uJ7eisqyjtK5Hu669Ahbzt4sKLl53u/kuPoUW/fo0e9I0pSxXyiQ4ecO5cWLSrCUtOmFe3U4BQZGTgv6dTz8gWvhATvXKaEBO97RkdXNG7TAbCF//wAZygiomJ0xAllZd7RoqKiwJ++8OP76XIFBpDycm9QOnLE2w4f9o4y7dvnbceOebcdPuxMnd+nUSNvcPKNPvla5VDVtKl3hKlFC+/tuRYtvEGpcriKjPS2xo29LTLSG7B8gYwQBeBU/GcBqGeiorx/5Fu0cO49jfGOHuXleYNSSUlFO3o0MDSdOifpxAlvqzyX6Ngxb+gqLPQGqsJC72vfqJNOzl0qLAxcpDEYGjf2Bh23uyIA+VqTJoGjVG63d6TN13TyKTrfE3iJid7Xpz495wtUvgUp3e6qo12+0apTjwUQeoQb4Czgcnn/cCcmBvdzysu9AejYMW9IKinxhp7KrXKwKi72hiPfiNLhw94w5AtWvnbihHdpgOPHK4KWj297feByVdy6i4qqGrYqLz0QEVGxgnfl23nR0YG3/yq/l+9no0YVodN3rVyuilGu6ka7fLcWY2IqWnS0970q1+MLZy5XRTv1fU9tvnoqB+Dy8sCafedBAEQoEG4AOKZRo4rRjGAyxvtH1DfSVFJS8a3zvlZW5t1fOUyVlQX+ITfGG64qz4sqKqr6Wb7Q5vvjXVoaONJ1al3BmGAeLip/JUvl8OYLQVLgyKExFSNnvuZbebxyAKscGn0/K9/a9N26rRwKXS7vyF7lVt1tTt97Vf53c2q4rO7zfP19v+vkvxHfYzwREVVD7KlfN+PrX/k43y3pyu9f+d915e2V1/Oq/H8Sjh+v/vMrv5fv2p76+ZVvF0dGVlwT335jvMG5detg/AuqnXoRbp5++mnNmjVLeXl56tatm5588kn17t37tP2XLFmi+++/X7t371anTp308MMPa+jQoSGtGYA9LlfFKIdtxngDTuXwU1ZWNWwdP17xx9rjqQhNZWXe448dq2i+4yu/ny+wVR4VqTxycuof3Mqf63vte//vvqv4zFPrUTV/UE8NBZVvP57K5fL+sYyIqH5UzeOp+HyEr/R071fr2GI93Lz00kuaNm2a5s6dqz59+ujxxx/X4MGDtXPnTiUlJVXp/95772n06NHKzs7Wz372My1atEgjRozQ1q1b1bVrVyvnAODs5XKFZrSqPvEFosphx3fryff//Cv3rRzMjh2rGN0qLa3Y5/spBY44uFwV4cz3Hr4wdmrzbfd4Am9r+tqpIw4eT8XtUV/zBbxTz9XXfKM0p45c+K6HLwhW7u/7WXmkyeWqCLeVQ2x1i7P4+vt+P7UeX2iurtbK63lVvlXZuHHFSGPlGk69jsYE1ixVnKsvOFd3brb/92B9nZs+ffro0ksv1VNPPSVJ8ng8SktL05133ql77rmnSv9Ro0appKREy5cv92/r27evunfvrrlz51bpX1paqtJKY8SFhYVKS0tjnRsAABqQuqxzE1Hj3iArKyvTli1blJGRUVFQRIQyMjK0fv36ao9Zv359QH9JGjx48Gn7Z2dnKyEhwd/SnHpeFwAA1EtWw82hQ4dUXl6u5OTkgO3JycnKy8ur9pi8vLw69Z8+fboKCgr8bd++fQ6eAQAAqG+sz7kJNrfbLbfbbbsMAAAQIlZHbhITE9WoUSPln/LNhPn5+UpJSan2mJSUlDr1BwAAZxer4SYqKko9e/bUihUr/Ns8Ho9WrFih9PT0ao9JT08P6C9JOTk5p+0PAADOLtZvS02bNk1ZWVnq1auXevfurccff1wlJSUaN26cJGns2LFq3bq1srOzJUlTpkzRgAEDNHv2bA0bNkyLFy/W5s2bNX/+fMtnAgAA6gPr4WbUqFH6+uuvNWPGDOXl5al79+56++23/ZOG9+7dq4hKSzb269dPixYt0u9+9zvde++96tSpk5YtW8YaNwAAQKoP69yEWl2ekwcAAPVDg1nnBgAAwGmEGwAAEFYINwAAIKwQbgAAQFgh3AAAgLBCuAEAAGHF+jo3oeZ78r2wsNB2KQAAoJZ8f7drs4LNWRduioqKJElpaWm2SwEAAHVUVFSkhISEGvucdYv4eTwe7d+/X3FxcXK5XI6+d2FhodLS0rRv3z4WCAwyrnXocK1Dh2sdOlzr0HHqWhtjVFRUpFatWgV8c0F1zrqRm4iICLVp0yaonxEfH8//WEKEax06XOvQ4VqHDtc6dJy41t83YuPDhGIAABBWCDcAACCsEG4c5Ha7NXPmTLndbtulhD2udehwrUOHax06XOvQsXGtz7oJxQAAILwxcgMAAMIK4QYAAIQVwg0AAAgrhBsAABBWCDcOefrpp9W+fXtFR0erT58+ev/9922X1OBlZ2fr0ksvVVxcnJKSkjRixAjt3LkzoM+xY8c0adIknXPOOYqNjdXIkSOVn59vreZw8dBDD8nlcmnq1Kn+bVxr53z11Ve64YYbdM455ygmJkYXX3yxNm/e7N9vjNGMGTOUmpqqmJgYZWRk6LPPPrNac0NUXl6u+++/Xx06dFBMTIzOO+88/fGPfwz4biKu9Q+3Zs0aDR8+XK1atZLL5dKyZcsC9tfm2h4+fFhjxoxRfHy8mjVrpvHjx6u4uPjMizM4Y4sXLzZRUVFmwYIF5sMPPzS33nqradasmcnPz7ddWoM2ePBg8+yzz5odO3aY3NxcM3ToUNO2bVtTXFzs7zNhwgSTlpZmVqxYYTZv3mz69u1r+vXrZ7Xuhu7999837du3N5dccomZMmWKfzvX2hmHDx827dq1MzfddJPZuHGj2bVrl3nnnXfM559/7u/z0EMPmYSEBLNs2TLzwQcfmJ///OemQ4cO5rvvvrNae0Pz4IMPmnPOOccsX77cfPnll2bJkiUmNjbWPPHEE/4+XOsf7s033zT33XefeeWVV4wk8+qrrwbsr821HTJkiOnWrZvZsGGD+fe//206duxoRo8efca1EW4c0Lt3bzNp0iT/6/LyctOqVSuTnZ1tta5wc/DgQSPJrF692hhjzJEjR0zjxo3NkiVL/H0+/vhjI8msX7/eYqUNV1FRkenUqZPJyckxAwYM8IcbrrVz/ud//sf8+Mc/Pu1+j8djUlJSzKxZs/zbjhw5Ytxut3nxxRdDVGV4GDZsmLn55psDtv3iF78wY8aMMYZr7ahTw01tru1HH31kJJlNmzb5+7z11lvG5XKZr7766ozq4bbUGSorK9OWLVuUkZHh3xYREaGMjAytX7/eam3hpqCgQJLUokULSdKWLVt0/PjxgGt/wQUXqG3btlz7H2jSpEkaNmxYwDUV19pRr7/+unr16qVrr71WSUlJ6tGjh/7v//7Pv//LL79UXl5ewLVOSEhQnz59uNZ11K9fP61YsUKffvqpJOmDDz7Q2rVrlZmZKXGtg6o213b9+vVq1qyZevXq5e+TkZGhiIgIbdy48Yw+/6z74kynHTp0SOXl5UpOTg7YnpycrE8++cRaXeHG4/Fo6tSp6t+/v7p27SpJysvLU1RUlJo1axbQNzk5WXl5eZYqbbgWL16srVu3atOmTVX2ca2ds2vXLs2ZM0fTpk3Tvffeq02bNumuu+5SVFSUsrKy/Nezuv+mcK3r5p577lFhYaEuuOACNWrUSOXl5XrwwQc1ZswY6eS/a3Gtg6I21zYvL09JSUkB+yMjI9WiRYszvv6EGzQIkyZN0o4dO7R27VrbpYSlffv2acqUKcrJyVF0dLTtcsKax+NRr1699Kc//UmS1KNHD+3YsUNz585VVlaW7fLCyssvv6wXXnhBixYtUpcuXZSbm6upU6eqVatWXOswx22pM5SYmKhGjRpVeWokPz9fKSkp1uoKJ5MnT9by5cv17rvvqk2bNv7tKSkpKisr05EjRwL6c+3rbsuWLTp48KB+9KMfKTIyUpGRkVq9erX+8pe/KDIyUsnJyVxrh6Smpuqiiy4K2HbhhRdq79690sl/1zp5bSvjWtfdb37zG91zzz26/vrrdfHFF+vGG2/Ur371K2VnZ0tc66CqzbVNSUnRwYMHA/afOHFChw8fPuPrT7g5Q1FRUerZs6dWrFjh3+bxeLRixQqlp6dbra2hM8Zo8uTJevXVV7Vy5Up16NAhYH/Pnj3VuHHjgGu/c+dO7d27l2tfR1dccYW2b9+u3Nxcf+vVq5fGjBnj/51r7Yz+/ftXWdLg008/Vbt27SRJHTp0UEpKSsC1Liws1MaNG7nWdXT06FFFRAT+mWvUqJE8Ho/EtQ6q2lzb9PR0HTlyRFu2bPH3WblypTwej/r06XNmBZzRdGQYc/JRcLfbbRYuXGg++ugjc9ttt5lmzZqZvLw826U1aHfccYdJSEgwq1atMgcOHPC3o0eP+vtMmDDBtG3b1qxcudJs3rzZpKenm/T0dKt1h4vKT0sZrrVj3n//fRMZGWkefPBB89lnn5kXXnjBNGnSxDz//PP+Pg899JBp1qyZee2118x//vMfc9VVV/F48g+QlZVlWrdu7X8U/JVXXjGJiYnmt7/9rb8P1/qHKyoqMtu2bTPbtm0zksyjjz5qtm3bZvbs2WNMLa/tkCFDTI8ePczGjRvN2rVrTadOnXgUvD558sknTdu2bU1UVJTp3bu32bBhg+2SGjxJ1bZnn33W3+e7774zEydONM2bNzdNmjQxV199tTlw4IDVusPFqeGGa+2cN954w3Tt2tW43W5zwQUXmPnz5wfs93g85v777zfJycnG7XabK664wuzcudNavQ1VYWGhmTJlimnbtq2Jjo425557rrnvvvtMaWmpvw/X+od79913q/1vdFZWljG1vLbffPONGT16tImNjTXx8fFm3Lhxpqio6Ixrc5nKSzUCAAA0cMy5AQAAYYVwAwAAwgrhBgAAhBXCDQAACCuEGwAAEFYINwAAIKwQbgAAQFgh3AAAgLBCuAFwVnK5XFq2bJntMgAEAeEGQMjddNNNcrlcVdqQIUNslwYgDETaLgDA2WnIkCF69tlnA7a53W5r9QAIH4zcALDC7XYrJSUloDVv3lw6ectozpw5yszMVExMjM4991wtXbo04Pjt27frpz/9qWJiYnTOOefotttuU3FxcUCfBQsWqEuXLnK73UpNTdXkyZMD9h86dEhXX321mjRpok6dOun111/37/v22281ZswYtWzZUjExMerUqVOVMAagfiLcAKiX7r//fo0cOVIffPCBxowZo+uvv14ff/yxJKmkpESDBw9W8+bNtWnTJi1ZskT/+te/AsLLnDlzNGnSJN12223avn27Xn/9dXXs2DHgM37/+9/ruuuu03/+8x8NHTpUY8aM0eHDh/2f/9FHH+mtt97Sxx9/rDlz5igxMTHEVwHAD3LG3ysOAHWUlZVlGjVqZJo2bRrQHnzwQWOMMZLMhAkTAo7p06ePueOOO4wxxsyfP980b97cFBcX+/f/85//NBERESYvL88YY0yrVq3Mfffdd9oaJJnf/e53/tfFxcVGknnrrbeMMcYMHz7cjBs3zuEzBxAKzLkBYMXll1+uOXPmBGxr0aKF//f09PSAfenp6crNzZUkffzxx+rWrZuaNm3q39+/f395PB7t3LlTLpdL+/fv1xVXXFFjDZdccon/96ZNmyo+Pl4HDx6UJN1xxx0aOXKktm7dqiuvvFIjRoxQv379zvCsAYQC4QaAFU2bNq1ym8gpMTExterXuHHjgNcul0sej0eSlJmZqT179ujNN99UTk6OrrjiCk2aNEl//vOfg1IzAOcw5wZAvbRhw4Yqry+88EJJ0oUXXqgPPvhAJSUl/v3r1q1TRESEOnfurLi4OLVv314rVqw4oxpatmyprKwsPf/883r88cc1f/78M3o/AKHByA0AK0pLS5WXlxewLTIy0j9pd8mSJerVq5d+/OMf64UXXtD777+vv/3tb5KkMWPGaObMmcrKytIDDzygr7/+WnfeeaduvPFGJScnS5IeeOABTZgwQUlJScrMzFRRUZHWrVunO++8s1b1zZgxQz179lSXLl1UWlqq5cuX+8MVgPqNcAPAirffflupqakB2zp37qxPPvlEOvkk0+LFizVx4kSlpqbqxRdf1EUXXSRJatKkid555x1NmTJFl156qZo0aaKRI0fq0Ucf9b9XVlaWjh07pscee0x33323EhMTdc0119S6vqioKE2fPl27d+9WTEyMfvKTn2jx4sWOnT+A4HEZ71MDAFBvuFwuvfrqqxoxYoTtUgA0QMy5AQAAYYVwAwAAwgpzbgDUO9wtB3AmGLkBAABhhXADAADCCuEGAACEFcINAAAIK4QbAAAQVgg3AAAgrBBuAABAWCHcAACAsPL/AR7PgkVWvPS6AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.suptitle('MSE vs Epochs')\n",
    "plt.plot(tune_train_err, label='Train', color='blue')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('MSE')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0189208984375"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tune_evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[2.6100e+02, 8.9530e+03, 8.2329e+04, ..., 0.0000e+00, 0.0000e+00,\n",
       "         0.0000e+00],\n",
       "        [0.0000e+00, 0.0000e+00, 0.0000e+00, ..., 0.0000e+00, 0.0000e+00,\n",
       "         0.0000e+00],\n",
       "        [0.0000e+00, 0.0000e+00, 0.0000e+00, ..., 0.0000e+00, 0.0000e+00,\n",
       "         0.0000e+00],\n",
       "        ...,\n",
       "        [0.0000e+00, 0.0000e+00, 0.0000e+00, ..., 0.0000e+00, 0.0000e+00,\n",
       "         0.0000e+00],\n",
       "        [0.0000e+00, 0.0000e+00, 0.0000e+00, ..., 0.0000e+00, 0.0000e+00,\n",
       "         0.0000e+00],\n",
       "        [0.0000e+00, 0.0000e+00, 0.0000e+00, ..., 0.0000e+00, 1.0000e+00,\n",
       "         0.0000e+00]]),\n",
       " array([1.   , 1.1  , 1.2  , 1.3  , 1.4  , 1.5  , 1.6  , 1.699, 1.8  ,\n",
       "        1.9  , 2.   , 2.1  , 2.2  , 2.3  , 2.398, 2.5  , 2.6  , 2.7  ,\n",
       "        2.8  , 2.898, 3.   , 3.1  , 3.2  , 3.299, 3.398, 3.5  , 3.6  ,\n",
       "        3.7  , 3.799, 3.898, 4.   , 4.1  , 4.2  , 4.297, 4.4  , 4.5  ,\n",
       "        4.6  , 4.7  , 4.797, 4.9  , 5.   , 5.098, 5.2  , 5.3  , 5.4  ,\n",
       "        5.5  , 5.598, 5.7  , 5.797, 5.9  , 6.   ], dtype=float16),\n",
       " array([0.8477, 0.913 , 0.9785, 1.044 , 1.109 , 1.176 , 1.241 , 1.307 ,\n",
       "        1.372 , 1.4375, 1.503 , 1.568 , 1.635 , 1.699 , 1.766 , 1.831 ,\n",
       "        1.896 , 1.962 , 2.027 , 2.094 , 2.158 , 2.225 , 2.29  , 2.355 ,\n",
       "        2.422 , 2.486 , 2.55  , 2.617 , 2.684 , 2.75  , 2.814 , 2.879 ,\n",
       "        2.945 , 3.012 , 3.076 , 3.143 , 3.207 , 3.273 , 3.338 , 3.404 ,\n",
       "        3.469 , 3.535 , 3.602 , 3.666 , 3.732 , 3.797 , 3.863 , 3.928 ,\n",
       "        3.994 , 4.062 , 4.125 ], dtype=float16),\n",
       " <matplotlib.collections.QuadMesh at 0x7fe6040e22a0>)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAgkUlEQVR4nO3dfVBU5+H28Wt5W4jCRjMBVNDHVIMvCEFjk8VpNInGUMvIPzTj2GLSJDPJg63GNungk8mbTdZOxrF2TFDyIm1TholJxTbVGJoEHUdMBcMM6NTUmARiAPtMDCskILD7/PFDKo8CnkU89+5+PzPnjz3cZ/faWWUv7j33WYff7/cLAADAIBF2BwAAAPj/UVAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMaJsjvAlfD5fPrqq68UHx8vh8NhdxwAAHAF/H6/zp07p4kTJyoiwtqcSFAUlK+++kqpqal2xwAAAAFoampSSkqKpWOCoqDEx8dLfU8wISHB7jgAAMMtdxUMO2Z32x+vSZZw5vV6lZqa2v8+bkVQFJQLH+skJCRQUAAAw4pyRA87hveTayeQ0zM4SRYAABiHggIAAIxDQQEAAMahoAAAAOMExUmyAGC6JRH5VzSu0rdz1LMA18KV/Jvv8XcHfP/MoAAAAONQUAAAgHFGVFA2btwoh8OhtWvXDjlu586dmjFjhmJjYzVnzhzt2bNnJA8LAABCXMAF5ciRI9q+fbsyMjKGHHfo0CGtWLFCDz74oD7++GPl5eUpLy9PDQ0NgT40AAAIcQEVlPb2dq1cuVKvvPKKxo0bN+TYLVu26N5779Xjjz+umTNnasOGDZo7d662bt0aaGYAABDiAioohYWFWrZsmRYvXjzs2Orq6kvGLV26VNXV1YMe09XVJa/XO2ADAADhw/Iy4/Lych09elRHjhy5ovEtLS1KSkoasC8pKUktLS2DHuPxePTss89ajQYAtomcfbPdEXCRiKxZdkcIeVeyZN7r9crlcgV0/5ZmUJqamrRmzRr9+c9/VmxsbEAPeCWKiorU1tbWvzU1NY3aYwEAAPNYmkGpra3VmTNnNHfu3P59vb29OnDggLZu3aquri5FRkYOOCY5OVmtra0D9rW2tio5OXnQx3E6nXI6nVaiAQCAEGJpBuXuu+9WfX296urq+rdbb71VK1euVF1d3SXlRJLcbrfef//9AfsqKyvldrtHnh4AAIQkSzMo8fHxSk9PH7BvzJgxuuGGG/r3FxQUaNKkSfJ4PJKkNWvWaOHChdq0aZOWLVum8vJy1dTUqKSk5Go+DwAAEEKu+pVkGxsb1dzc3H87OztbZWVlKikpUWZmpt566y1VVFRcUnQAAAAuGPGXBVZVVQ15W5Ly8/OVn39lX6QFAADAd/EAAADjjHgGBQAgOb47b3cEXMRxvsfuCBghZlAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIzDMmMgiC2JuLILIF7J16JjZM6njLM7Ai7SNSHB7ggh70p+//T4uwO+f2ZQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADG4TooQBCLnH2z3RHQ5/z10XZHwEWivu2xOwJGiBkUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjsMwYCGK+MU67I6DP+YRIuyMA11Slb+ewY7xer1wuV0D3zwwKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxWGYMBLHvkuPsjoA+vXyZsVEcPb12Rwh5SyLyhx3T4+8O+P6ZQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIfroABBrHN8pN0R0Kc73mF3BFyEawQFP2ZQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMY2mZcXFxsYqLi/X5559LkmbPnq2nnnpKOTk5lx1fWlqqBx54YMA+p9Opzs7OkWQG0Oe7G1jaaoru6+xOgIv5ovi/MdoqfTuHHeP1euVyuQK6f0sFJSUlRRs3btT06dPl9/v1hz/8QcuXL9fHH3+s2bNnX/aYhIQEnThxov+2w8E/GgAAMDRLBSU3N3fA7eeff17FxcU6fPjwoAXF4XAoOTl5ZCkBAEBYCfgclN7eXpWXl6ujo0Nut3vQce3t7ZoyZYpSU1O1fPlyHTt2bNj77urqktfrHbABAIDwYbmg1NfXa+zYsXI6nXrkkUe0a9cuzZo167Jj09LS9Prrr2v37t1644035PP5lJ2drS+//HLIx/B4PHK5XP1bamqq1ZgAACCIWS4oaWlpqqur00cffaRHH31Uq1at0vHjxy871u12q6CgQLfccosWLlyov/zlL7rxxhu1ffv2IR+jqKhIbW1t/VtTU5PVmAAAIIhZ/rLAmJgYTZs2TZI0b948HTlyRFu2bBm2dEhSdHS0srKydPLkySHHOZ1OOZ1Oq9EAAECIGPF1UHw+n7q6uq5obG9vr+rr6zVhwoSRPiwAAAhhlmZQioqKlJOTo8mTJ+vcuXMqKytTVVWV9u3bJ0kqKCjQpEmT5PF4JEnPPfecbr/9dk2bNk3ffPONXnzxRX3xxRd66KGHRufZAGHGb3kOFKPF4bc7AS429rNzdkfACFn69XbmzBkVFBSoublZLpdLGRkZ2rdvn5YsWSJJamxsVETEfydlzp49q4cfflgtLS0aN26c5s2bp0OHDg16Ui0AAICsFpTXXnttyJ9XVVUNuL1582Zt3rw5sGQAACBs8V08AADAOBQUAABgHAoKAAAwDgUFAAAYh0WKQBDrHmN3Alzg47epUSJO/8fuCBghZlAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIzDwjggiPVex1fomiLqW7sT4GL+G8fZHQEjxAwKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4XAcFCGJ+/sQwRq/T7gRAaOHXGwAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcVhmDMvunfN/rmjcu/XPj3qWcOeL89kdAX3GfuW3OwIu4o/h7S3YMYMCAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAc1mHBMt8YvrbVFP5olhmb4vxYh90RcJF9tc/aHQEjxAwKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4XAcFlvVeF213BPRxRHEdFFPEnuW1MElO8v8edszelpevSRYEhhkUAABgHAoKAAAwDgUFAAAYx1JBKS4uVkZGhhISEpSQkCC32629e/cOeczOnTs1Y8YMxcbGas6cOdqzZ89IMwMAgBBnqaCkpKRo48aNqq2tVU1Nje666y4tX75cx44du+z4Q4cOacWKFXrwwQf18ccfKy8vT3l5eWpoaLha+QEAQAiyVFByc3P1wx/+UNOnT9fNN9+s559/XmPHjtXhw4cvO37Lli2699579fjjj2vmzJnasGGD5s6dq61bt16t/AAAIAQFvMy4t7dXO3fuVEdHh9xu92XHVFdXa926dQP2LV26VBUVFYE+LAwQ+W233RHQx9/DaWSmiO5gmbFJWEIc/CwXlPr6erndbnV2dmrs2LHatWuXZs2addmxLS0tSkpKGrAvKSlJLS0tQz5GV1eXurq6+m97vV6rMQEAQBCz/OdXWlqa6urq9NFHH+nRRx/VqlWrdPz48asayuPxyOVy9W+pqalX9f4BAIDZLBeUmJgYTZs2TfPmzZPH41FmZqa2bNly2bHJyclqbW0dsK+1tVXJyclDPkZRUZHa2tr6t6amJqsxAQBAEBvxB9g+n2/AxzEXc7vdev/99wfsq6ysHPSclQucTmf/UuYLGwAACB+WzkEpKipSTk6OJk+erHPnzqmsrExVVVXat2+fJKmgoECTJk2Sx+ORJK1Zs0YLFy7Upk2btGzZMpWXl6umpkYlJSWj82wAAEBIsFRQzpw5o4KCAjU3N8vlcikjI0P79u3TkiVLJEmNjY2KiPjvpEx2drbKysr05JNPav369Zo+fboqKiqUnp5+9Z8JAAAIGZYKymuvvTbkz6uqqi7Zl5+fr/z8fOvJAABA2Ar4OigIX98lx9kdAX0cnVwHxRQRvXYnwMWWRAz/h3Glb+c1yYLA8NsNAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4LDOGZT1x9FpTRJzntTBGr9/uBEBI4bcbAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxWGYMyzrH02tN4Yvz2R0BfSJ6WGZskqjEG+2OgBHinQYAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYByugwIEsch2/sYwhS/KYXcEXKTnzH/sjoAR4rcbAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxWGYMy3ri7E6AC6K9LG01hfPrTrsj4CKVvp12R8AIMYMCAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAO10GBZb2xdidAP/7EMEZvLL9OgauJX28AAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMZhXRws642xOwEuiGq3OwH6RTjsTgCEFGZQAACAcSwVFI/Ho/nz5ys+Pl6JiYnKy8vTiRMnhjymtLRUDodjwBYby5W+AADA4CwVlP3796uwsFCHDx9WZWWluru7dc8996ijo2PI4xISEtTc3Ny/ffHFFyPNDQAAQpilc1DefffdAbdLS0uVmJio2tpa3XHHHYMe53A4lJycHHhKAAAQVkZ0DkpbW5skafz48UOOa29v15QpU5Samqrly5fr2LFjQ47v6uqS1+sdsAEAgPARcEHx+Xxau3atFixYoPT09EHHpaWl6fXXX9fu3bv1xhtvyOfzKTs7W19++eWgx3g8Hrlcrv4tNTU10JgAACAIBbzMuLCwUA0NDTp48OCQ49xut9xud//t7OxszZw5U9u3b9eGDRsue0xRUZHWrVvXf9vr9VJSDNLt8tsdAX34ZmlzRH7bbXcEIKQEVFBWr16td955RwcOHFBKSoqlY6Ojo5WVlaWTJ08OOsbpdMrpdAYSDQAAhABLH/H4/X6tXr1au3bt0gcffKCpU6dafsDe3l7V19drwoQJlo8FAADhwdIMSmFhocrKyrR7927Fx8erpaVFkuRyuRQXFydJKigo0KRJk+TxeCRJzz33nG6//XZNmzZN33zzjV588UV98cUXeuihh0bj+QAAgBBgqaAUFxdLkhYtWjRg/44dO3T//fdLkhobGxUR8d+JmbNnz+rhhx9WS0uLxo0bp3nz5unQoUOaNWvW1XkGAAAg5FgqKH7/8CdHVlVVDbi9efNmbd682XoyAAAQtvguHgAAYBwKCgAAME7A10FB+PJFcR0UU0QP/TVYuIYc3b12RwBCCjMoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADGYZkxrIvvsTsB+vTE2p0AF0S0fWt3BCCkMIMCAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAO10GBZZFOroNiitiv7U6AC3pOfW53BCCkMIMCAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAclhnDstjYbrsjoE9vjN0JcEHk7JvtjgCEFGZQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMwzJjWJac4LU7AmCc3mOf2B0BCCnMoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjBNU10FZ7ipQlCN6yDGVvp3XLE+4Ght93u4I6BP3td/uCOgTddP/sjsCEFKYQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBxLBcXj8Wj+/PmKj49XYmKi8vLydOLEiWGP27lzp2bMmKHY2FjNmTNHe/bsGUlmAAAQ4iwtM96/f78KCws1f/589fT0aP369brnnnt0/PhxjRkz5rLHHDp0SCtWrJDH49GPfvQjlZWVKS8vT0ePHlV6evrVeh64hqbH/8fuCOjTHeewOwL69Jz63O4IQEixVFDefffdAbdLS0uVmJio2tpa3XHHHZc9ZsuWLbr33nv1+OOPS5I2bNigyspKbd26Vdu2bRtJdgAAEKJGdA5KW1ubJGn8+PGDjqmurtbixYsH7Fu6dKmqq6sHPaarq0ter3fABgAAwkfABcXn82nt2rVasGDBkB/VtLS0KCkpacC+pKQktbS0DHqMx+ORy+Xq31JTUwONCQAAglDABaWwsFANDQ0qLy+/uokkFRUVqa2trX9ramq66o8BAADMFdB38axevVrvvPOODhw4oJSUlCHHJicnq7W1dcC+1tZWJScnD3qM0+mU0+kMJBoAAAgBlmZQ/H6/Vq9erV27dumDDz7Q1KlThz3G7Xbr/fffH7CvsrJSbrfbeloAABAWLM2gFBYWqqysTLt371Z8fHz/eSQul0txcXGSpIKCAk2aNEkej0eStGbNGi1cuFCbNm3SsmXLVF5erpqaGpWUlIzG8wEAACHAUkEpLi6WJC1atGjA/h07duj++++XJDU2Nioi4r8TM9nZ2SorK9OTTz6p9evXa/r06aqoqAjoGihRUycrKoKPfux2+juX3RHQJ/Zsr90R0Ccq8Ua7IwAhxVJB8fv9w46pqqq6ZF9+fr7y8/OtJQMAAGGL7+IBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGCcgK4ka5fe8WPliIq1O0bY++QsyylNEXl++JV1ABCMmEEBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADBOUC0zjvy6XZER3XbHCHvfdsbYHQF9vk2MtDsC+vSc+Y/dEYCQwgwKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4QXUdlPMp4+SLirU7Rtj79us4uyOgT4zXb3cEABgVzKAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABgnqJYZdyTHKCo6xu4YYS/qbFD9swlpMd4euyMAwKhgBgUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYJyguqBF5/gIRTrpVMAFked9dkdAn0rfTrsjACGFd3sAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAOME1TJjSZLf7gBwfu2wOwL6xDR+bXcE9FkSkX9F41iODFwZZlAAAIBxKCgAAMA4FBQAAGAcywXlwIEDys3N1cSJE+VwOFRRUTHk+KqqKjkcjku2lpaWkeQGAAAhzHJB6ejoUGZmpl566SVLx504cULNzc39W2JiotWHBgAAYcLyKp6cnBzl5ORYfqDExERdf/31lo8DAADh55otM77lllvU1dWl9PR0PfPMM1qwYMGgY7u6utTV1dV/2+v1SpJ64iS/85rExVBYZQxcguXDwNU16ifJTpgwQdu2bdPbb7+tt99+W6mpqVq0aJGOHj066DEej0cul6t/S01NHe2YAADAIKM+g5KWlqa0tLT+29nZ2fr000+1efNm/elPf7rsMUVFRVq3bl3/ba/XS0kBACCM2HIl2e9///s6ePDgoD93Op1yOvksBwCAcGXLdVDq6uo0YcIEOx4aAAAEAcszKO3t7Tp58mT/7c8++0x1dXUaP368Jk+erKKiIp0+fVp//OMfJUm/+93vNHXqVM2ePVudnZ169dVX9cEHH+i99967us8EAACEDMsFpaamRnfeeWf/7QvniqxatUqlpaVqbm5WY2Nj/8/Pnz+vX/7ylzp9+rSuu+46ZWRk6B//+MeA+wAAALiY5YKyaNEi+f2Df6VwaWnpgNtPPPGEnnjiicDSAQCAsGTLSbIBc3ANDhNEdl3BIFwTPtd1dkcAgFHBlwUCAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABgnqJYZd18n+WLtToExzT67I6BPxOn/2B0BAEYFMygAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAOME1XVQfHF+KdZvd4yw54t22B0Bffa2vGx3BAAYFcygAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYJ6iWGffG+eSP89kdI+yNq2+zOwIAIMQxgwIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYJygWmY8dmK7Iq/rtjtG2NtX+6zdEQAAIY4ZFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcYLqOiiZiacVMzbG7hgAAGCUMYMCAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGCcoFpmXJx6SAnxkXbHAAAAo8zyDMqBAweUm5uriRMnyuFwqKKiYthjqqqqNHfuXDmdTk2bNk2lpaWB5gUAAGHAckHp6OhQZmamXnrppSsa/9lnn2nZsmW68847VVdXp7Vr1+qhhx7Svn37AskLAADCgOWPeHJycpSTk3PF47dt26apU6dq06ZNkqSZM2fq4MGD2rx5s5YuXWr14QEAQBgY9ZNkq6urtXjx4gH7li5dqurq6kGP6erqktfrHbABAIDwMeoFpaWlRUlJSQP2JSUlyev16rvvvrvsMR6PRy6Xq39LTU0d7ZgAAMAgRi4zLioqUltbW//W1NRkdyQAAHANjfoy4+TkZLW2tg7Y19raqoSEBMXFxV32GKfTKafTOdrRAACAoUa9oLjdbu3Zs2fAvsrKSrndbsv39cL/nSlnZ/SQYzYkW75bAABgGMsf8bS3t6uurk51dXVS3zLiuro6NTY2Sn0fzxQUFPSPf+SRR3Tq1Ck98cQT+te//qWXX35Zb775ph577LGr+TwAAEAIsVxQampqlJWVpaysLEnSunXrlJWVpaeeekqS1Nzc3F9WJGnq1Kn6+9//rsrKSmVmZmrTpk169dVXWWIMAAAG5fD7/X67QwzH6/XK5XLp8UPL5Bw7zEc8c3Zds1wAAGBwF96/29ralJCQYOlYI1fxAACA8EZBAQAAxqGgAAAA44z6MuOr6c2qbEXExg45ZsOcaxYHAACMEmZQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYJyhW8Vy42K2vs3PYsV6v9xokAgAAw7nwnhzIReuD4lL3p06d0ve+9z27YwAAgAB8+umnuummmywdExQzKOPHj5ckNTY2yuVy2R0nrHm9XqWmpqqpqcny9yrg6uK1MAevhVl4PczR1tamyZMn97+PWxEUBSUi4n9OlXG5XPxjM0RCQgKvhSF4LczBa2EWXg9zXHgft3TMqCQBAAAYAQoKAAAwTlAUFKfTqaefflpOp9PuKGGP18IcvBbm4LUwC6+HOUbyWgTFKh4AABBegmIGBQAAhBcKCgAAMA4FBQAAGIeCAgAAjGN0QTlw4IByc3M1ceJEORwOVVRU2B0pbHk8Hs2fP1/x8fFKTExUXl6eTpw4YXessFRcXKyMjIz+i1C53W7t3bvX7liQtHHjRjkcDq1du9buKGHnmWeekcPhGLDNmDHD7lhh6/Tp0/rJT36iG264QXFxcZozZ45qamos3YfRBaWjo0OZmZl66aWX7I4S9vbv36/CwkIdPnxYlZWV6u7u1j333KOOjg67o4WdlJQUbdy4UbW1taqpqdFdd92l5cuX69ixY3ZHC2tHjhzR9u3blZGRYXeUsDV79mw1Nzf3bwcPHrQ7Ulg6e/asFixYoOjoaO3du1fHjx/Xpk2bNG7cOEv3Y/Sl7nNycpSTk2N3DEh69913B9wuLS1VYmKiamtrdccdd9iWKxzl5uYOuP3888+ruLhYhw8f1uzZs23LFc7a29u1cuVKvfLKK/rNb35jd5ywFRUVpeTkZLtjhL3f/va3Sk1N1Y4dO/r3TZ061fL9GD2DAnO1tbVJF32RI+zR29ur8vJydXR0yO122x0nbBUWFmrZsmVavHix3VHC2r///W9NnDhRN910k1auXKnGxka7I4Wlv/71r7r11luVn5+vxMREZWVl6ZVXXrF8P0bPoMBMPp9Pa9eu1YIFC5Senm53nLBUX18vt9utzs5OjR07Vrt27dKsWbPsjhWWysvLdfToUR05csTuKGHttttuU2lpqdLS0tTc3Kxnn31WP/jBD9TQ0KD4+Hi744WVU6dOqbi4WOvWrdP69et15MgR/eIXv1BMTIxWrVp1xfdDQYFlhYWFamho4PNdG6Wlpamurk5tbW166623tGrVKu3fv5+Sco01NTVpzZo1qqysVGxsrN1xwtrFpwNkZGTotttu05QpU/Tmm2/qwQcftDVbuPH5fLr11lv1wgsvSJKysrLU0NCgbdu2WSoofMQDS1avXq133nlHH374oVJSUuyOE7ZiYmI0bdo0zZs3Tx6PR5mZmdqyZYvdscJObW2tzpw5o7lz5yoqKkpRUVHav3+/fv/73ysqKkq9vb12Rwxb119/vW6++WadPHnS7ihhZ8KECZf8sTRz5kzLH7kxg4Ir4vf79fOf/1y7du1SVVVVQCc8YfT4fD51dXXZHSPs3H333aqvrx+w74EHHtCMGTP061//WpGRkbZlC3ft7e369NNP9dOf/tTuKGFnwYIFl1yG4pNPPtGUKVMs3Y/RBaW9vX1A+/3ss89UV1en8ePHa/LkybZmCzeFhYUqKyvT7t27FR8fr5aWFkmSy+VSXFyc3fHCSlFRkXJycjR58mSdO3dOZWVlqqqq0r59++yOFnbi4+MvOQ9rzJgxuuGGGzg/6xr71a9+pdzcXE2ZMkVfffWVnn76aUVGRmrFihV2Rws7jz32mLKzs/XCCy/oxz/+sf75z3+qpKREJSUl1u7Ib7APP/zQL+mSbdWqVXZHCzuXex0k+Xfs2GF3tLDzs5/9zD9lyhR/TEyM/8Ybb/Tffffd/vfee8/uWOizcOFC/5o1a+yOEXbuu+8+/4QJE/wxMTH+SZMm+e+77z7/yZMn7Y4Vtv72t7/509PT/U6n0z9jxgx/SUmJ5ftw+P/nzQcAAMAYnCQLAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHH+H+VXw4oAI7ElAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "batch_src, batch_labels, batch_padding_mask = mktunebatch(BSZ)\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    output = model(batch_src, batch_padding_mask)\n",
    "x = batch_labels.detach().to(torch.float16).cpu().numpy().flatten()\n",
    "y = output.detach().to(torch.float16).cpu().numpy().flatten()\n",
    "plt.hist2d(x, y, bins=50, norm=mpl.colors.LogNorm())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JtTLXn4zC1z_"
   },
   "source": [
    "# Step 6: Test generalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1767578125\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[ 241.,  824., 9690., ...,    0.,    0.,    0.],\n",
       "        [   0.,    0.,    0., ...,    0.,    0.,    0.],\n",
       "        [   0.,    0.,    0., ...,    0.,    0.,    0.],\n",
       "        ...,\n",
       "        [   0.,    0.,    0., ...,    0.,    0.,    0.],\n",
       "        [   0.,    0.,    0., ...,    0.,    0.,    0.],\n",
       "        [   0.,    0.,    0., ...,    0.,    0.,    0.]]),\n",
       " array([ 1.   ,  1.18 ,  1.36 ,  1.54 ,  1.721,  1.9  ,  2.08 ,  2.262,\n",
       "         2.441,  2.621,  2.8  ,  2.98 ,  3.16 ,  3.34 ,  3.521,  3.701,\n",
       "         3.88 ,  4.062,  4.242,  4.42 ,  4.6  ,  4.78 ,  4.96 ,  5.14 ,\n",
       "         5.32 ,  5.5  ,  5.68 ,  5.863,  6.043,  6.223,  6.402,  6.582,\n",
       "         6.76 ,  6.94 ,  7.12 ,  7.3  ,  7.48 ,  7.66 ,  7.844,  8.02 ,\n",
       "         8.2  ,  8.38 ,  8.56 ,  8.74 ,  8.92 ,  9.1  ,  9.28 ,  9.46 ,\n",
       "         9.64 ,  9.82 , 10.   ], dtype=float16),\n",
       " array([0.7344, 0.818 , 0.9014, 0.9844, 1.068 , 1.151 , 1.234 , 1.318 ,\n",
       "        1.402 , 1.485 , 1.568 , 1.652 , 1.735 , 1.819 , 1.902 , 1.986 ,\n",
       "        2.07  , 2.152 , 2.236 , 2.32  , 2.402 , 2.486 , 2.57  , 2.652 ,\n",
       "        2.736 , 2.82  , 2.904 , 2.986 , 3.07  , 3.154 , 3.238 , 3.32  ,\n",
       "        3.404 , 3.488 , 3.57  , 3.654 , 3.738 , 3.822 , 3.904 , 3.988 ,\n",
       "        4.07  , 4.156 , 4.24  , 4.32  , 4.406 , 4.49  , 4.57  , 4.656 ,\n",
       "        4.74  , 4.824 , 4.906 ], dtype=float16),\n",
       " <matplotlib.collections.QuadMesh at 0x7fe607ee0110>)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAApjElEQVR4nO3dfXBU12H38d9qJa1kkGRwrRdAYDBYvAoDdu0VqSEBjBWGQdNn1JShFQnGM2nFFFktTpTUcWLiiNihlBTCi99o7CjEOAVa15goOIKhCEdg1ArSEGP7QdiWRJsCi4QRYvc+f9RWogcEWu1qz7m738/M/WOvztX+roW1P92956zHcRxHAAAAFksyHQAAAOBmKCwAAMB6FBYAAGA9CgsAALAehQUAAFiPwgIAAKxHYQEAANajsAAAAOslmw7QF6FQSB999JEyMjLk8XhMxwEAAH3gOI4uXryoYcOGKSkpsmskrigsH330kfLz803HAAAA/XDmzBmNGDEiou/hisKSkZEhfXLCmZmZpuMAcev/THu8T+N+emz1gGeJxKKssj6N233hhwOeBUhkgUBA+fn53a/jkXBFYfn0baDMzEwKCzCAkpN8fRpn+/+HyZ6UPo2z/TyAeBGN2zm46RYAAFiPwgIAAKxHYQEAANajsAAAAOu54qZbuNO8pNI+jasN7RjwLP1VnFfep3F7WjYOeBb0nc3/pgD0D1dYAACA9SgsAADAehQWAABgPQoLAACwHoUFAABYj8ICAACsR2EBAADWYx0WDJh4WAvDuX2I6Qgx1XnHUNMRooL1c4D4wxUWAABgPQoLAACwHoUFAABYj8ICAACsR2EBAADWo7AAAADrMa0ZA2ZeUmmfxtk8/dnzX+dMR4ip1LZ20xGiY/Ag0wliJl6mcMfD7wsMLK6wAAAA61FYAACA9SgsAADAehQWAABgPQoLAACwHoUFAABYj2nNGDDOZ+42HSFil+65w3SEmLp6a7rpCAhTon2iOBIXV1gAAID1Iiosa9askcfjUUVFRa9jtm3bJo/H02NLS0uL5GkBAECC6fdbQg0NDdqyZYsKCwtvOjYzM1MnT57sfuzxePr7tAAAIAH16wpLe3u7lixZomeffVZDhtz8/VOPx6Pc3NzuLScnpz9PCwAAElS/Ckt5ebkWLFiguXPn9ml8e3u7Ro0apfz8fC1atEgnTpzoz9MCAIAEFfZbQtu3b9fbb7+thoaGPo0vKCjQCy+8oMLCQl24cEHf+973VFRUpBMnTmjEiBHXPaazs1OdnZ3djwOBQLgxAQBAHAnrCsuZM2e0cuVK/ehHP+rzjbN+v19lZWW6++67NWvWLP3TP/2Tbr/9dm3ZsqXXY6qrq5WVldW95efnhxMTAADEmbCusBw9elRnz57V9OnTu/cFg0EdOHBAGzZsUGdnp7xe7w2/R0pKiqZNm6ZTp071OqaqqkqVlZXdjwOBAKXFhRyv+2fNXxns/nMIx+U/8JmOEBWddww1HSFm4mXtnNrQDtMRYLmwCsucOXPU1NTUY9+XvvQljR8/Xl/5ylduWlb0ScFpamrS5z//+V7H+Hw++Xzx8YsTAABELqzCkpGRocmTJ/fYN2jQIN12223d+8vKyjR8+HBVV1dLkp588kndf//9Gjt2rM6fP69nnnlGp0+f1vLly6N5HgAAII5FfWn+5uZmJSX97jL6uXPn9Mgjj6i1tVVDhgzRjBkzdOjQIU2cODHaTw0AAOJUxIWlrq7uho/XrVundevWRfo0AAAggSXWHYUAAMCVKCwAAMB6Ub+HBfiUk8xnRrnN1Vvi42+YlHOXTUeImZR3PjIdAWGal1Tap3FM9e4pPn47AQCAuEZhAQAA1qOwAAAA61FYAACA9SgsAADAehQWAABgPQoLAACwHuuwxFCizb0PjEw1HSFil3ISq9N/PDQ+1s7pGDXYdISYuVw40nSEqEik34/xcA4mJNZvYwAA4EoUFgAAYD0KCwAAsB6FBQAAWI/CAgAArEdhAQAA1mNacwwl2lS2rsHunyIbdP/M7PC4/0cmSfJecUxHiBlfy0XTEYCY4AoLAACwHoUFAABYj8ICAACsR2EBAADWo7AAAADrUVgAAID1mNaMgRMHM0udRKv0cfAzk6SkKyHTEWLm6q3ppiNERSIt+5BIn0wdTYn26xgAALgQhQUAAFiPwgIAAKxHYQEAANajsAAAAOtRWAAAgPUoLAAAwHqswxJDxeMe69O4Pe88PeBZYqFzqOkEkevKMJ0gtq7Gx5Ie6shLMR0hZkI+r+kICFNyTrbpCK7EFRYAAGC9iArLmjVr5PF4VFFRccNxO3bs0Pjx45WWlqYpU6bo9ddfj+RpAQBAgul3YWloaNCWLVtUWFh4w3GHDh3S4sWL9fDDD+vYsWMqKSlRSUmJjh8/3t+nBgAACaZfhaW9vV1LlizRs88+qyFDhtxw7Pr16/XQQw9p1apVmjBhglavXq3p06drw4YN/c0MAAASTL8KS3l5uRYsWKC5c+fedGx9ff014+bPn6/6+vpej+ns7FQgEOixAQCAxBX2LKHt27fr7bffVkNDQ5/Gt7a2Kicnp8e+nJwctba29npMdXW1vvWtb4UbDQAAxKmwCsuZM2e0cuVK1dbWKi0tbcBCVVVVqbKysvtxIBBQfn7+gD1frDjpqaYjxFTQZzpB5II+x3SEmHLiZDZwMMVjOkLMOEnxca7FeeV9GrenZeOAZxloV9vOmo7gSmEVlqNHj+rs2bOaPn16975gMKgDBw5ow4YN6uzslNfbc02A3NxctbW19djX1tam3NzcXp/H5/PJ54uDVzsAABAVYd3DMmfOHDU1NamxsbF7u+eee7RkyRI1NjZeU1Ykye/3a9++fT321dbWyu/3R54eAAAkhLCusGRkZGjy5Mk99g0aNEi33XZb9/6ysjINHz5c1dXVkqSVK1dq1qxZWrt2rRYsWKDt27fryJEj2rp1azTPAwAAxLGor3Tb3NyslpaW7sdFRUWqqanR1q1bNXXqVL366qvatWvXNcUHAACgNxF/llBdXd0NH0tSaWmpSktLI30qAACQoPgsIQAAYD0+rTmGQoMSa+ZTMN39U4I97j+FsDj8CeM6qb/92HQEhKk2tMN0BFfi1xMAALAehQUAAFiPwgIAAKxHYQEAANajsAAAAOtRWAAAgPUoLAAAwHqswxJDSR2dpiPElON1/yImoRTTCWLLc9V0gujwXnH/v72+6hqSZjpCVFxtO2s6QswU55X3adyelo0DnsVNuMICAACsR2EBAADWo7AAAADrUVgAAID1KCwAAMB6FBYAAGA9pjXHUPvYW01HiKnQoKDpCBEL+UKmI8RUyGc6QXQ4XtMJYie5vct0hKioDe0wHSFm4mW68ryk0puOuepE798nV1gAAID1KCwAAMB6FBYAAGA9CgsAALAehQUAAFiPwgIAAKxHYQEAANZjHZYYCiV7TEeIKU+K+9cw8QQT62eW3GE6QXQkX3ZMR4gZ7/9tNR0hKvqypocSbL0W2/XlZxEIBJSVlRWV5+MKCwAAsB6FBQAAWI/CAgAArEdhAQAA1qOwAAAA61FYAACA9ZjWHEOXshOrHyYlu39as+NNnOmxkqQEO914ELwj13QEICYS6xUUAAC4UliFZdOmTSosLFRmZqYyMzPl9/u1Z8+eXsdv27ZNHo+nx5aWlhaN3AAAIIGE9ZbQiBEjtGbNGo0bN06O4+gf//EftWjRIh07dkyTJk267jGZmZk6efJk92OPJ7FWDgUAAJELq7AsXLiwx+OnnnpKmzZt0uHDh3stLB6PR7m5vMcKAAD6r9/3sASDQW3fvl0dHR3y+/29jmtvb9eoUaOUn5+vRYsW6cSJEzf93p2dnQoEAj02AACQuMIuLE1NTRo8eLB8Pp++/OUva+fOnZo4ceJ1xxYUFOiFF17Q7t279fLLLysUCqmoqEgffPDBDZ+jurpaWVlZ3Vt+fn64MQEAQBwJe1pzQUGBGhsbdeHCBb366qtaunSp9u/ff93S4vf7e1x9KSoq0oQJE7RlyxatXr261+eoqqpSZWVl9+NAIBAXpeXqLaYTxFZyatB0BITJ22k6QXSkXnT/lPq++ln946YjADERdmFJTU3V2LFjJUkzZsxQQ0OD1q9fry1bttz02JSUFE2bNk2nTp264TifzyefzxduNAAAEKciXoclFAqps7Nvf5YFg0E1NTUpLy8v0qcFAAAJJKwrLFVVVSouLtbIkSN18eJF1dTUqK6uTnv37pUklZWVafjw4aqurpYkPfnkk7r//vs1duxYnT9/Xs8884xOnz6t5cuXD8zZAACAuBRWYTl79qzKysrU0tKirKwsFRYWau/evZo3b54kqbm5WUlJv7toc+7cOT3yyCNqbW3VkCFDNGPGDB06dKjXm3QBAACuJ6zC8vzzz9/w63V1dT0er1u3TuvWretfMgAAgE/wWUIAAMB6FBYAAGC9sKc1m7Qoq0zJnpQbjqkN7YhZnnAFE2ymthNy/+dGea66/xzCkf7b+Fi/xHs5cdYAmpdU2qdxNv9uBPqCKywAAMB6FBYAAGA9CgsAALAehQUAAFiPwgIAAKxHYQEAANZz1bTm5OzblZyUajpGv4XcG71fHMd0gsh5P06sTh9KTqxp3PEgOSfbdISoYNo1biaxfhsDAABXorAAAADrUVgAAID1KCwAAMB6FBYAAGA9CgsAALCeq6Y1a/AtUpJ7P/K4a3AczPMNQ0pK4nxibtxgVrP7DB5kOgEQE1xhAQAA1qOwAAAA61FYAACA9SgsAADAehQWAABgPQoLAACwHoUFAABYz1XrsFx977TkSTEdo988IdMJYutKp3t/Vp9KvmQ6QWx5r8THWkEp5y6bjhAze9552nQEICa4wgIAAKxHYQEAANajsAAAAOtRWAAAgPUoLAAAwHoUFgAAYD1XTWt2u2B6gs1rjgNJXaYTxJb3cnxMa76a4TMdAWGal1Tap3G1oR0DngV24goLAACwXliFZdOmTSosLFRmZqYyMzPl9/u1Z8+eGx6zY8cOjR8/XmlpaZoyZYpef/31SDMDAIAEE1ZhGTFihNasWaOjR4/qyJEj+tznPqdFixbpxIkT1x1/6NAhLV68WA8//LCOHTumkpISlZSU6Pjx49HKDwAAEkBYhWXhwoX6/Oc/r3Hjxumuu+7SU089pcGDB+vw4cPXHb9+/Xo99NBDWrVqlSZMmKDVq1dr+vTp2rBhQ7TyAwCABNDve1iCwaC2b9+ujo4O+f3+646pr6/X3Llze+ybP3++6uvr+/u0AAAgAYU9S6ipqUl+v1+XL1/W4MGDtXPnTk2cOPG6Y1tbW5WTk9NjX05OjlpbW2/4HJ2dners7Ox+HAgEwo0JAADiSNhXWAoKCtTY2Ki33npLf/EXf6GlS5fqV7/6VVRDVVdXKysrq3vLz8+P6vcHAADuEvYVltTUVI0dO1aSNGPGDDU0NGj9+vXasmXLNWNzc3PV1tbWY19bW5tyc3Nv+BxVVVWqrKzsfhwIBOKjtNwSNJ0gpjxJ7l93xkmwif+O12M6AsLE+iVIFBH/Og6FQj3evvl9fr9f+/bt67Gvtra213tePuXz+bqnTn+6AQCAxBXWFZaqqioVFxdr5MiRunjxompqalRXV6e9e/dKksrKyjR8+HBVV1dLklauXKlZs2Zp7dq1WrBggbZv364jR45o69atA3M2AAAgLoVVWM6ePauysjK1tLQoKytLhYWF2rt3r+bNmydJam5uVlLS7y7aFBUVqaamRn/7t3+rr33taxo3bpx27dqlyZMnR/9MAABA3AqrsDz//PM3/HpdXd01+0pLS1Va2rf3WAEAAK4nwW4pBAAAbkRhAQAA1gt7WrNJSXdPUJLXxR8b73VMJ4iprvZU0xEi5r1sOkFsJXfEx9T75IvXn7kYj5Jzsk1HAGKCKywAAMB6FBYAAGA9CgsAALAehQUAAFiPwgIAAKxHYQEAANZz1bRmt/OmuP/Ti8MScv8n/zoJ9n9I12Cv6QhR4aTEx3n0xZ6WjaYjRAWfJo2b4QoLAACwHoUFAABYj8ICAACsR2EBAADWo7AAAADrUVgAAID1KCwAAMB6rlpl4tKwW5SckmY6Rr/50q6YjoBwJdrSOanuXztHkjxdQdMRAEQZV1gAAID1KCwAAMB6FBYAAGA9CgsAALAehQUAAFiPwgIAAKznqmnNTpJHTpJ7p1163Bu9X5I+dn8fTv7YdILY8v3PVdMRouLKbemmIwCIMve/ogAAgLhHYQEAANajsAAAAOtRWAAAgPUoLAAAwHoUFgAAYD0KCwAAsJ6r1mEZ9EGHkr3uXSciJTmxPvLeE4qDhWcSrNJ3DfaajhAVae/91nQEAFGWYL+OAQCAG4VVWKqrq3XvvfcqIyND2dnZKikp0cmTJ294zLZt2+TxeHpsaWlpkeYGAAAJJKzCsn//fpWXl+vw4cOqra1VV1eXHnzwQXV0dNzwuMzMTLW0tHRvp0+fjjQ3AABIIGHdw/LGG2/0eLxt2zZlZ2fr6NGjeuCBB3o9zuPxKDc3t/8pAQBAQovoHpYLFy5IkoYOHXrDce3t7Ro1apTy8/O1aNEinThx4objOzs7FQgEemwAACBx9buwhEIhVVRUaObMmZo8eXKv4woKCvTCCy9o9+7devnllxUKhVRUVKQPPvig12Oqq6uVlZXVveXn5/c3JgAAiAP9ntZcXl6u48eP6+DBgzcc5/f75ff7ux8XFRVpwoQJ2rJli1avXn3dY6qqqlRZWdn9OBAIKD8/XxfuypA31b037GamXTYdIaacJMd0hIiFEmwenfdKyHSEqLj67vumI8TMvKTSPo2rDe0Y8CzAQOpXYVmxYoVee+01HThwQCNGjAjr2JSUFE2bNk2nTp3qdYzP55PP5+tPNAAAEIfC+vvRcRytWLFCO3fu1JtvvqnRo0eH/YTBYFBNTU3Ky8sL+1gAAJCYwrrCUl5erpqaGu3evVsZGRlqbW2VJGVlZSk9PV2SVFZWpuHDh6u6ulqS9OSTT+r+++/X2LFjdf78eT3zzDM6ffq0li9fPhDnAwAA4lBYhWXTpk2SpNmzZ/fY/+KLL+qLX/yiJKm5uVlJSb+7cHPu3Dk98sgjam1t1ZAhQzRjxgwdOnRIEydOjM4ZAACAuBdWYXGcm99EWVdX1+PxunXrtG7duvCTAQAAfCLB5kAAAAA3orAAAADr9XsdFhOCqZJSTafovyS5f12ScHiuekxHiJjjqv9DItc12Gs6QlQk52SbjhAziXSuSGxcYQEAANajsAAAAOtRWAAAgPUoLAAAwHoUFgAAYD0KCwAAsJ6rJm16r0hunnTpS75qOkJMeeJgFre303SC2Eq9GDQdAWHa07LRdISomJdU2qdxtaEdA54FduIKCwAAsB6FBQAAWI/CAgAArEdhAQAA1qOwAAAA61FYAACA9Vw1rfnS7R55fe79BOBbUy+bjhBTThzU4ZCb59H3Q9egOPihSdLgQaYTIExMV8bNxMlvJwAAEM8oLAAAwHoUFgAAYD0KCwAAsB6FBQAAWI/CAgAArEdhAQAA1nPVOixOUnys7ZEwHNMBEK7U80HTEaLi6rvvm44QM/OSSvs0jnVO4Ha8/AMAAOtRWAAAgPUoLAAAwHoUFgAAYD0KCwAAsB6FBQAAWM9V05qv3Cp500yn6L9s30XTEWLKEwfTmuPhHMLhpHhMR0CYku8cbToCEBNcYQEAANYLq7BUV1fr3nvvVUZGhrKzs1VSUqKTJ0/e9LgdO3Zo/PjxSktL05QpU/T6669HkhkAACSYsArL/v37VV5ersOHD6u2tlZdXV168MEH1dHR0esxhw4d0uLFi/Xwww/r2LFjKikpUUlJiY4fPx6N/AAAIAGEdQ/LG2+80ePxtm3blJ2draNHj+qBBx647jHr16/XQw89pFWrVkmSVq9erdraWm3YsEGbN2+OJDsAAEgQEd3DcuHCBUnS0KFDex1TX1+vuXPn9tg3f/581dfX93pMZ2enAoFAjw0AACSufheWUCikiooKzZw5U5MnT+51XGtrq3Jycnrsy8nJUWtra6/HVFdXKysrq3vLz8/vb0wAABAH+j2tuby8XMePH9fBgwejm0hSVVWVKisrux8HAgHl5+crlOpIqe6dZ9oeTDUdIaaSOt0/RdYTHx9e3GfeyyHTEaLCO7nAdISY2fPO06YjADHRr8KyYsUKvfbaazpw4IBGjBhxw7G5ublqa2vrsa+trU25ubm9HuPz+eTz+foTDQAAxKGw3hJyHEcrVqzQzp079eabb2r06JsvWOT3+7Vv374e+2pra+X3+8NPCwAAElJYV1jKy8tVU1Oj3bt3KyMjo/s+lKysLKWnp0uSysrKNHz4cFVXV0uSVq5cqVmzZmnt2rVasGCBtm/friNHjmjr1q0DcT4AACAOhXWFZdOmTbpw4YJmz56tvLy87u0nP/lJ95jm5ma1tLR0Py4qKlJNTY22bt2qqVOn6tVXX9WuXbtueKMuAADA7wvrCovj3PyG17q6umv2lZaWqrS0NLxkAAAAn+CzhAAAgPUoLAAAwHr9XofFhJDPkdLcuw7Lu4E/MB0htty/DIuSEmwdlnjxxn9823SEmJmX1Le322tDOwY8SyTi5TwwcLjCAgAArEdhAQAA1qOwAAAA61FYAACA9SgsAADAehQWAABgPVdNa3Z8ITm+kOkY/fbfFweZjhBTSV2mE0QuHs4hHGnv/dZ0hKhIpCmy8XAOiqPzwMDhCgsAALAehQUAAFiPwgIAAKxHYQEAANajsAAAAOtRWAAAgPUoLAAAwHquWodFacH/3Vzq8qVU0xFiyzEdIHLp/+3ef2/90XnHUNMRAOC6uMICAACsR2EBAADWo7AAAADrUVgAAID1KCwAAMB6FBYAAGA9V01rTknvkvcWr+kY/Rb82FX/uSPmvWI6QeSCaR7TEWLKczUO5qIDiEtcYQEAANajsAAAAOtRWAAAgPUoLAAAwHoUFgAAYD0KCwAAsJ6r5tkmeRwledw77dJzNbH6oafLdILIOYk1q1mpv/7AdISoqA3tMB0BQJQl1isoAABwpbALy4EDB7Rw4UINGzZMHo9Hu3btuuH4uro6eTyea7bW1tZIcgMAgAQSdmHp6OjQ1KlTtXHjxrCOO3nypFpaWrq37OzscJ8aAAAkqLDvYSkuLlZxcXHYT5Sdna1bb7017OMAAABidg/L3Xffrby8PM2bN0//9m//dsOxnZ2dCgQCPTYAAJC4Bryw5OXlafPmzfrpT3+qn/70p8rPz9fs2bP19ttv93pMdXW1srKyurf8/PyBjgkAACw24NOaCwoKVFBQ0P24qKhI7777rtatW6eXXnrpusdUVVWpsrKy+3EgEKC0AACQwIysw/KHf/iHOnjwYK9f9/l88vl81+y/Ja1L3jT3zsT2dCXWoh7eOFiHJfO9S6YjoB+K88r7NG5PS3iTBwCYY+TVv7GxUXl5eSaeGgAAuFDYV1ja29t16tSp7sfvv/++GhsbNXToUI0cOVJVVVX68MMP9cMf/lCS9Pd///caPXq0Jk2apMuXL+u5557Tm2++qZ/97GfRPRMAABC3wi4sR44c0Wc/+9nux5/ea7J06VJt27ZNLS0tam5u7v76lStX9Nd//df68MMPdcstt6iwsFA///nPe3wPAACAGwm7sMyePVuO0/vn+Wzbtq3H48cee0yPPfZY/9IBAADwWUIAAMANKCwAAMB6RqY191eq96qSk72mY/RbUmdiTWtODfT+1qFbdA65dnp9PLvadtZ0hKiIl/MA8DtcYQEAANajsAAAAOtRWAAAgPUoLAAAwHoUFgAAYD0KCwAAsB6FBQAAWM9V67D4vFeV7HXvOiwp7Ym1DsugtqDpCBFLO3vJdAT0Q21oh+kIAKKMKywAAMB6FBYAAGA9CgsAALAehQUAAFiPwgIAAKxHYQEAANZz1bTmFG9QyV73TpVNbjedILaCqe6fxr33yDdNR4gp7+QC0xEA4Lq4wgIAAKxHYQEAANajsAAAAOtRWAAAgPUoLAAAwHoUFgAAYD1XTWsuvPUj+QanmI7Rb8mXTSeILd//dJmOELHicY/1adyed54e8Cyx8MZ/fNt0BAC4Lq6wAAAA61FYAACA9SgsAADAehQWAABgPQoLAACwHoUFAABYj8ICAACs56p1WL6V3aTMDK/pGP02+KOg6QgxlfrrD0xHiFi8rK8CAG7HFRYAAGC9sAvLgQMHtHDhQg0bNkwej0e7du266TF1dXWaPn26fD6fxo4dq23btvU3LwAASEBhF5aOjg5NnTpVGzdu7NP4999/XwsWLNBnP/tZNTY2qqKiQsuXL9fevXv7kxcAACSgsO9hKS4uVnFxcZ/Hb968WaNHj9batWslSRMmTNDBgwe1bt06zZ8/P9ynBwAACWjA72Gpr6/X3Llze+ybP3++6uvrez2ms7NTgUCgxwYAABLXgBeW1tZW5eTk9NiXk5OjQCCgjz/++LrHVFdXKysrq3vLz88f6JgAAMBiVk5rrqqqUmVlZffjQCCg/Px8PR8YpvTQjSP/ZW4MAvZT2n91mo4QU1fbzpqOELF5SaV9Glcb2jHgWQAgkQ14YcnNzVVbW1uPfW1tbcrMzFR6evp1j/H5fPL5fAMdDQAAuMSAvyXk9/u1b9++Hvtqa2vl9/sH+qkBAECcCLuwtLe3q7GxUY2NjdIn05YbGxvV3NwsffJ2TllZWff4L3/5y3rvvff02GOP6de//rV+8IMf6JVXXtGjjz4azfMAAABxLOzCcuTIEU2bNk3Tpk2TJFVWVmratGn6xje+IUlqaWnpLi+SNHr0aP3rv/6ramtrNXXqVK1du1bPPfccU5oBAECfhX0Py+zZs+U4Tq9fv94qtrNnz9axY8fCTwcAAMBnCQEAADewclpzb37wqwfkvSXthmP+siBmccKW8s5HpiPEVDxM9Y2HcwCAeMAVFgAAYD0KCwAAsB6FBQAAWI/CAgAArEdhAQAA1qOwAAAA61FYAACA9Vy1Dkuw5RY5aTdeh8VmV9vOmo4AAIArcYUFAABYj8ICAACsR2EBAADWo7AAAADrUVgAAID1XDFLyHEcSVLo8uWbjg0EAjFI1D9Xna4+jbP5HAAA6KtPX88+fR2PhMeJxncZYO+9957uvPNO0zEAAEA/vPvuuxozZkxE38MVV1iGDh0qSWpublZWVpbpOFEVCASUn5+vM2fOKDMz03ScqOLc3IlzcyfOzb3i+fwuXLigkSNHdr+OR8IVhSUp6X9vtcnKyoq7H+anMjMzOTcX4tzciXNzp3g+N8X5+X36Oh7R94hKEgAAgAFEYQEAANZzRWHx+Xx64okn5PP5TEeJOs7NnTg3d+Lc3Cmez01xfn7RPDdXzBICAACJzRVXWAAAQGKjsAAAAOtRWAAAgPUoLAAAwHpWF5YDBw5o4cKFGjZsmDwej3bt2mU6UlRUV1fr3nvvVUZGhrKzs1VSUqKTJ0+ajhU1mzZtUmFhYfciSH6/X3v27DEdK+rWrFkjj8ejiooK01Gi4pvf/KY8Hk+Pbfz48aZjRc2HH36oP/uzP9Ntt92m9PR0TZkyRUeOHDEdK2J33HHHNT83j8ej8vJy09EiFgwG9fjjj2v06NFKT0/XnXfeqdWrV0flc2lscPHiRVVUVGjUqFFKT09XUVGRGhoaTMcK281eqx3H0Te+8Q3l5eUpPT1dc+fO1TvvvBP281hdWDo6OjR16lRt3LjRdJSo2r9/v8rLy3X48GHV1taqq6tLDz74oDo6OkxHi4oRI0ZozZo1Onr0qI4cOaLPfe5zWrRokU6cOGE6WtQ0NDRoy5YtKiwsNB0lqiZNmqSWlpbu7eDBg6YjRcW5c+c0c+ZMpaSkaM+ePfrVr36ltWvXasiQIaajRayhoaHHz6y2tlaSVFpaajpaxL773e9q06ZN2rBhg/7zP/9T3/3ud/X000/rH/7hH0xHi4rly5ertrZWL730kpqamvTggw9q7ty5+vDDD01HC8vNXquffvppff/739fmzZv11ltvadCgQZo/f74u9+EDjXtwXEKSs3PnTtMxBsTZs2cdSc7+/ftNRxkwQ4YMcZ577jnTMaLi4sWLzrhx45za2lpn1qxZzsqVK01HioonnnjCmTp1qukYA+IrX/mK85nPfMZ0jJhYuXKlc+eddzqhUMh0lIgtWLDAWbZsWY99f/zHf+wsWbLEWKZouXTpkuP1ep3XXnutx/7p06c7X//6143litT//1odCoWc3Nxc55lnnuned/78ecfn8zk//vGPw/reVl9hSRQXLlyQfu9DHuNJMBjU9u3b1dHRIb/fbzpOVJSXl2vBggWaO3eu6ShR984772jYsGEaM2aMlixZoubmZtORouKf//mfdc8996i0tFTZ2dmaNm2ann32WdOxou7KlSt6+eWXtWzZMnk8HtNxIlZUVKR9+/bpN7/5jSTp3//933Xw4EEVFxebjhaxq1evKhgMKi0trcf+9PT0uLmyKUnvv/++Wltbe/y+zMrK0n333af6+vqwvpcrPvwwnoVCIVVUVGjmzJmaPHmy6ThR09TUJL/fr8uXL2vw4MHauXOnJk6caDpWxLZv3663337ble8z38x9992nbdu2qaCgQC0tLfrWt76lP/qjP9Lx48eVkZFhOl5E3nvvPW3atEmVlZX62te+poaGBv3VX/2VUlNTtXTpUtPxombXrl06f/68vvjFL5qOEhVf/epXFQgENH78eHm9XgWDQT311FNasmSJ6WgRy8jIkN/v1+rVqzVhwgTl5OToxz/+serr6zV27FjT8aKmtbVVkpSTk9Njf05OTvfX+orCYlh5ebmOHz8eV41akgoKCtTY2KgLFy7o1Vdf1dKlS7V//35Xl5YzZ85o5cqVqq2tveavonjw+3+1FhYW6r777tOoUaP0yiuv6OGHHzaaLVKhUEj33HOPvvOd70iSpk2bpuPHj2vz5s1xVVief/55FRcXa9iwYaajRMUrr7yiH/3oR6qpqdGkSZPU2NioiooKDRs2LC5+bi+99JKWLVum4cOHy+v1avr06Vq8eLGOHj1qOpqVeEvIoBUrVui1117TL37xC40YMcJ0nKhKTU3V2LFjNWPGDFVXV2vq1Klav3696VgROXr0qM6ePavp06crOTlZycnJ2r9/v77//e8rOTlZwWDQdMSouvXWW3XXXXfp1KlTpqNELC8v75qyPGHChLh5y0uSTp8+rZ///Odavny56ShRs2rVKn31q1/Vn/7pn2rKlCn68z//cz366KOqrq42HS0q7rzzTu3fv1/t7e06c+aMfvnLX6qrq0tjxowxHS1qcnNzJUltbW099re1tXV/ra8oLAY4jqMVK1Zo586devPNNzV69GjTkQZcKBRSZ2en6RgRmTNnjpqamtTY2Ni93XPPPVqyZIkaGxvl9XpNR4yq9vZ2vfvuu8rLyzMdJWIzZ868ZumA3/zmNxo1apSxTNH24osvKjs7WwsWLDAdJWouXbqkpKSeL1Ner1ehUMhYpoEwaNAg5eXl6dy5c9q7d68WLVpkOlLUjB49Wrm5udq3b1/3vkAgoLfeeivs+xqtfkuovb29x19377//vhobGzV06FCNHDnSaLZIlJeXq6amRrt371ZGRkb3+3hZWVlKT083HS9iVVVVKi4u1siRI3Xx4kXV1NSorq5Oe/fuNR0tIhkZGdfcZzRo0CDddtttcXH/0d/8zd9o4cKFGjVqlD766CM98cQT8nq9Wrx4seloEXv00UdVVFSk73znO/qTP/kT/fKXv9TWrVu1detW09GiIhQK6cUXX9TSpUuVnGz1r/WwLFy4UE899ZRGjhypSZMm6dixY/q7v/s7LVu2zHS0qNi7d68cx1FBQYFOnTqlVatWafz48frSl75kOlpYbvZaXVFRoW9/+9saN26cRo8erccff1zDhg1TSUlJeE8U1flMUfaLX/zCkXTNtnTpUtPRInK9c5LkvPjii6ajRcWyZcucUaNGOampqc7tt9/uzJkzx/nZz35mOtaAiKdpzV/4whecvLw8JzU11Rk+fLjzhS98wTl16pTpWFHzL//yL87kyZMdn8/njB8/3tm6davpSFGzd+9eR5Jz8uRJ01GiKhAIOCtXrnRGjhzppKWlOWPGjHG+/vWvO52dnaajRcVPfvITZ8yYMU5qaqqTm5vrlJeXO+fPnzcdK2w3e60OhULO448/7uTk5Dg+n8+ZM2dOv/6tepx4WTIQAADELe5hAQAA1qOwAAAA61FYAACA9SgsAADAehQWAABgPQoLAACwHoUFAABYj8ICAACsR2EBAADWo7AAAADrUVgAAID1KCwAAMB6/w/wFoD82D83ywAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "batch_src, batch_labels, batch_padding_mask = mktunebatch(BSZ, test=True)\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    output = model(batch_src, batch_padding_mask)\n",
    "print(criterion(output.squeeze(1), batch_labels).item())\n",
    "x = batch_labels.detach().to(torch.float16).cpu().numpy().flatten()\n",
    "y = output.detach().to(torch.float16).cpu().numpy().flatten()\n",
    "plt.hist2d(x, y, bins=50, norm=mpl.colors.LogNorm())"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}