path: root/transformer_shortest_paths.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LPphBnKR-aWF"
   },
   "source": [
    "# Step 0: Imports"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Question: \n",
    "\n",
    "- Do the attention heads learn to attend to the same positional encodings\n",
    "- do interp -- what is it doing? can we figure out?\n",
    "- update: I think we should do interp once it's bigger. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "execution_state": "idle",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ge5QvElvhCOw",
    "outputId": "c7cdaefa-d6dc-44ad-c258-e4fb2aca97a5"
   },
   "outputs": [],
   "source": [
    "# using tqdm.auto glitches out collaborative editing\n",
    "from tqdm import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "\n",
    "from math import sqrt\n",
    "from collections import deque\n",
    "import os\n",
    "import random\n",
    "from concurrent.futures import ProcessPoolExecutor\n",
    "import pickle\n",
    "\n",
    "torch.manual_seed(42)\n",
    "random.seed(42)\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "assert device.type == \"cuda\", \"CUDA is not available. Please check your GPU setup.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "execution_state": "idle",
   "metadata": {
    "id": "lylOX2POPwFL"
   },
   "outputs": [],
   "source": [
    "# VTXS numbers here are inclusive\n",
    "MIN_VTXS = 3\n",
    "MAX_VTXS = 15\n",
    "MAX_TUNE_VTXS = 7 # 15\n",
    "AVG_DEG = 2\n",
    "SEQ_LEN = MAX_VTXS + 1 # means 32 edges, final token is the target vertex\n",
    "PAD_TOKEN = 0\n",
    "# vertices are labelled 1,2,...,63\n",
    "# we also have a padding token which is 0."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gKt-yIpDebF1"
   },
   "source": [
    "# Step 1: Generate synthetic data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "execution_state": "idle",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1IbzGIWseK3E",
    "outputId": "a3cbc233-358c-4e17-ea6e-f4e9349d886b"
   },
   "outputs": [],
   "source": [
    "# original task data\n",
    "# the data will be edge lists\n",
    "# like this: [1 3 1 5 2 4 0 0 0 0 2]\n",
    "# this represents edges (1,3), (1,5) (2,4)\n",
    "# (the zeros are just padding tokens)\n",
    "# the final 2 means which vertex we're going to \n",
    "\n",
    "# the label is the shortest distance from vtx 1 to vtx 2\n",
    "# or \"number of vertices\" if no path exists\n",
    "\n",
    "def random_graph(n):\n",
    "    edge_list = []\n",
    "    adjacencies = [set() for _ in range(n+1)]\n",
    "    indices = [random.randint(1, n) for _ in range(AVG_DEG * n)]\n",
    "    for i in range(0, len(indices), 2):\n",
    "        u = indices[i]\n",
    "        v = indices[i + 1]\n",
    "        if u != v:\n",
    "            edge_list += [min(u,v),max(u,v)]\n",
    "            adjacencies[u].add(v)\n",
    "            adjacencies[v].add(u)\n",
    "    edge_list += [PAD_TOKEN]*(2*SEQ_LEN-1-len(edge_list))\n",
    "    return edge_list, adjacencies\n",
    "\n",
    "\"\"\"\n",
    "input: G, represented as an adjacency list\n",
    "output: [number of vertices]+[d(1,i) for i in range(n)] if target=None\n",
    "if target is set to some value, then we instead just output that specific distance\n",
    "\"\"\"\n",
    "def SSSP(G, target=2):\n",
    "    dist = [MAX_VTXS for _ in G]\n",
    "    dist[1] = 0\n",
    "    frontier = deque()\n",
    "    frontier.append(1)\n",
    "    while len(frontier) > 0:\n",
    "        vtx = frontier.popleft()\n",
    "        for x in G[vtx]:\n",
    "            if dist[x] == MAX_VTXS:\n",
    "                dist[x] = 1 + dist[vtx]\n",
    "                frontier.append(x)\n",
    "                if x == target:\n",
    "                    return dist[target]\n",
    "    if target is not None:\n",
    "        return dist[target]\n",
    "    else:\n",
    "        return dist\n",
    "\n",
    "def mkbatch(size):\n",
    "    graphs1 = []\n",
    "    distance1 = []\n",
    "    \n",
    "    for i in range(size):\n",
    "        n = random.randint(MIN_VTXS, MAX_VTXS)\n",
    "        edge_list, adj_list = random_graph(n)\n",
    "        dist = SSSP(adj_list)\n",
    "        edge_list[-1] = 2 # target token\n",
    "        graphs1.append(edge_list)\n",
    "        distance1.append(dist)\n",
    "    \n",
    "    data = torch.tensor(graphs1)\n",
    "    labels = torch.tensor(distance1, dtype=torch.bfloat16)\n",
    "    padding = data == PAD_TOKEN\n",
    "    return data, labels, padding\n",
    "\n",
    "def savebatch(size, idx):\n",
    "    data, labels, padding = mkbatch(size)\n",
    "    everything = {\n",
    "        \"data\": data,\n",
    "        \"labels\": labels,\n",
    "        \"padding\": padding,\n",
    "    }\n",
    "    \n",
    "    with open(f'data/{idx}.pickle', 'wb') as file:\n",
    "        pickle.dump(everything, file)\n",
    "\n",
    "def vertices_on_shortest_12_path(G, target=2):\n",
    "    dist = [MAX_VTXS for _ in G]\n",
    "    parent = [-1 for _ in G]\n",
    "    dist[1] = 0\n",
    "    frontier = deque()\n",
    "    frontier.append(1)\n",
    "    while len(frontier) > 0:\n",
    "        vtx = frontier.popleft()\n",
    "        for x in G[vtx]:\n",
    "            if dist[x] == MAX_VTXS:\n",
    "                parent[x] = vtx\n",
    "                dist[x] = 1 + dist[vtx]\n",
    "                frontier.append(x)\n",
    "                if x == target:\n",
    "                    path = [x]\n",
    "                    while parent[x] != -1:\n",
    "                        x = parent[x]\n",
    "                        path.append(x)\n",
    "                    return list(reversed(path))\n",
    "    return []\n",
    "\n",
    "def mktunebatch(size, test=False):\n",
    "    graphs = []\n",
    "    distance = []\n",
    "    \n",
    "    for i in range(size):\n",
    "        n = random.randint(MIN_VTXS, MAX_VTXS if test else MAX_TUNE_VTXS)\n",
    "        while True:\n",
    "            edge_list, adj_list = random_graph(n)\n",
    "            path = vertices_on_shortest_12_path(adj_list)\n",
    "            if len(path) > 1:\n",
    "                target_vtx_idx = random.randrange(1, len(path))\n",
    "                target_vtx = path[target_vtx_idx]\n",
    "                edge_list[-1] = target_vtx\n",
    "                graphs.append(edge_list)\n",
    "                distance.append(target_vtx_idx)\n",
    "                break\n",
    "    \n",
    "    data = torch.tensor(graphs, device=device)\n",
    "    labels = torch.tensor(distance, dtype=torch.bfloat16, device=device)\n",
    "    padding = data == PAD_TOKEN\n",
    "    return data, labels, padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Only need to run this once to generate training data\n",
    "# RESTART THE KERNEL BEFORE RUNNING AND ONLY RUN THE CELLS ABOVE\n",
    "# Python is slow and awful\n",
    "\n",
    "# with ProcessPoolExecutor() as executor:\n",
    "#     for i in range(1000):\n",
    "#         executor.submit(savebatch, 2**20, i)\n",
    "#     executor.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[ 1,  2,  5, 12,  3, 12,  2, 11,  9, 12,  2, 10,  1,  7,  1,  2,  9, 10,\n",
       "           1,  9,  4, 12,  0,  0,  0,  0,  0,  0,  0,  0,  2],\n",
       "         [ 9, 12,  4,  7,  8, 10,  5, 13,  1, 13,  3, 13,  7, 12,  5,  6,  3,  4,\n",
       "           6, 13,  2,  7,  0,  0,  0,  0,  0,  0,  0,  0,  2],\n",
       "         [ 1,  5,  8, 12,  2,  9,  2,  7,  5,  9, 10, 11,  6, 10,  4, 12,  1,  2,\n",
       "           4, 11,  2,  5,  2,  4,  0,  0,  0,  0,  0,  0,  2],\n",
       "         [ 5,  8,  3,  6,  4,  5,  2,  3,  4,  9,  3,  8,  5,  7,  4,  9,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  2],\n",
       "         [ 6, 13,  1, 13,  1,  4,  6, 13,  5,  7,  2,  4, 10, 12,  4,  6,  8, 11,\n",
       "           7, 11,  3,  8,  3,  5,  4, 12,  0,  0,  0,  0,  2]]),\n",
       " tensor([ 1.,  5.,  1., 15.,  2.], dtype=torch.bfloat16),\n",
       " tensor([[False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "          False],\n",
       "         [False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "          False],\n",
       "         [False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False,  True,  True,  True,  True,  True,  True,\n",
       "          False],\n",
       "         [False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False,  True,  True,  True,  True,\n",
       "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "          False],\n",
       "         [False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False,  True,  True,  True,  True,\n",
       "          False]]))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mkbatch(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([523.,   0.,   0.,   0., 390.,   0.,   0.,   0.,   0., 238.,   0.,\n",
       "          0.,   0.,  92.,   0.,   0.,   0.,   0.,  40.,   0.,   0.,   0.,\n",
       "         15.,   0.,   0.,   0.,   0.,   5.,   0.,   0.,   0.,   0.,   2.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0., 743.]),\n",
       " array([ 1.     ,  1.21875,  1.4375 ,  1.65625,  1.875  ,  2.09375,\n",
       "         2.3125 ,  2.53125,  2.75   ,  2.96875,  3.1875 ,  3.40625,\n",
       "         3.625  ,  3.84375,  4.0625 ,  4.28125,  4.5    ,  4.71875,\n",
       "         4.9375 ,  5.15625,  5.375  ,  5.59375,  5.8125 ,  6.03125,\n",
       "         6.25   ,  6.46875,  6.6875 ,  6.90625,  7.125  ,  7.34375,\n",
       "         7.5625 ,  7.78125,  8.     ,  8.21875,  8.4375 ,  8.65625,\n",
       "         8.875  ,  9.09375,  9.3125 ,  9.53125,  9.75   ,  9.96875,\n",
       "        10.1875 , 10.40625, 10.625  , 10.84375, 11.0625 , 11.28125,\n",
       "        11.5    , 11.71875, 11.9375 , 12.15625, 12.375  , 12.59375,\n",
       "        12.8125 , 13.03125, 13.25   , 13.46875, 13.6875 , 13.90625,\n",
       "        14.125  , 14.34375, 14.5625 , 14.78125, 15.     ]),\n",
       " <BarContainer object of 64 artists>)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAl0UlEQVR4nO3df1RU953/8Rc/B0RnCGyYkRWQ7rpFEoyJpjIxu+0qK7WsW480qTmU0MSNp57RBGmtsqsm1UQMu42pXdSa46p7Gtat56xpxcQEiSFtBSS47jGaJWZrAgmZoWctjJrDgDDfP75l2vFHklFkPsDzcc49J3M/n2Hel2PMM8MME+H3+/0CAAAwSGS4BwAAALgSgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAONHhHuBGDAwMqKOjQxMmTFBERES4xwEAAJ+D3+/XhQsXlJqaqsjIT3+OZEQGSkdHh9LS0sI9BgAAuAHt7e2aNGnSp+4ZkYEyYcIE6fcXaLVawz0OAAD4HLxer9LS0gL/Hf80IzJQBn+sY7VaCRQAAEaYz/PyDF4kCwAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA40SHewAAABAek9ccuu7a+5sLhnWWK/EMCgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA44QUKJMnT1ZERMRVh8vlkiT19PTI5XIpOTlZ48ePV2FhoTweT9DXaGtrU0FBgcaNG6eUlBStWrVKly9fHtqrAgAAI1pIgdLc3KyPP/44cNTW1kqSHnjgAUnSypUrdfDgQe3fv1/19fXq6OjQokWLAvfv7+9XQUGBent7dezYMe3du1d79uzR+vXrh/q6AADACBbh9/v9N3rn0tJS1dTU6OzZs/J6vbr99ttVXV2tb3zjG5Kk//mf/9HUqVPV0NCg3NxcvfLKK/rbv/1bdXR0yG63S5J27Nih1atX67e//a1iY2M/1+N6vV7ZbDZ1d3fLarXe6PgAAIxpk9ccuu7a+5sLhvzxQvnv9w2/BqW3t1c//elP9eijjyoiIkItLS3q6+tTXl5eYE9WVpbS09PV0NAgSWpoaFBOTk4gTiQpPz9fXq9Xp0+fvtFRAADAKBN9o3d86aWX1NXVpW9/+9uSJLfbrdjYWCUmJgbts9vtcrvdgT1/HCeD64Nr1+Pz+eTz+QK3vV7vjY4NAABGgBt+BmXXrl2aP3++UlNTh3aia6ioqJDNZgscaWlpt/wxAQBA+NxQoHzwwQc6cuSI/v7v/z5wzuFwqLe3V11dXUF7PR6PHA5HYM+V7+oZvD2451rKy8vV3d0dONrb229kbAAAMELcUKDs3r1bKSkpKij4wwtoZsyYoZiYGNXV1QXOtba2qq2tTU6nU5LkdDp16tQpdXZ2BvbU1tbKarUqOzv7uo9nsVhktVqDDgAAMHqF/BqUgYEB7d69WyUlJYqO/sPdbTablixZorKyMiUlJclqtWrFihVyOp3Kzc2VJM2bN0/Z2dkqLi5WZWWl3G631q5dK5fLJYvFMrRXBgAARqyQA+XIkSNqa2vTo48+etXali1bFBkZqcLCQvl8PuXn52vbtm2B9aioKNXU1GjZsmVyOp1KSEhQSUmJNmzYcPNXAgAARo2b+j0o4cLvQQEA4OaNyt+DAgAAcKsQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOOEHCgfffSRvvWtbyk5OVnx8fHKycnRW2+9FVj3+/1av369Jk6cqPj4eOXl5ens2bNBX+P8+fMqKiqS1WpVYmKilixZoosXLw7NFQEAgBEvpED53e9+p9mzZysmJkavvPKKzpw5ox/+8Ie67bbbAnsqKyu1detW7dixQ01NTUpISFB+fr56enoCe4qKinT69GnV1taqpqZGb775ppYuXTq0VwYAAEasCL/f7/+8m9esWaNf//rX+uUvf3nNdb/fr9TUVH33u9/V9773PUlSd3e37Ha79uzZo8WLF+udd95Rdna2mpubNXPmTEnS4cOH9bWvfU0ffvihUlNTP3MOr9crm82m7u5uWa3Wz3+1AAAgYPKaQ9dde39zwZA/Xij//Q7pGZRf/OIXmjlzph544AGlpKTo7rvv1gsvvBBYP3funNxut/Ly8gLnbDabZs2apYaGBklSQ0ODEhMTA3EiSXl5eYqMjFRTU9M1H9fn88nr9QYdAABg9AopUH7zm99o+/btmjJlil599VUtW7ZMjz/+uPbu3StJcrvdkiS73R50P7vdHlhzu91KSUkJWo+OjlZSUlJgz5UqKipks9kCR1paWmhXCQAARpSQAmVgYED33HOPNm3apLvvvltLly7VY489ph07dty6CSWVl5eru7s7cLS3t9/SxwMAAOEVUqBMnDhR2dnZQeemTp2qtrY2SZLD4ZAkeTyeoD0ejyew5nA41NnZGbR++fJlnT9/PrDnShaLRVarNegAAACjV0iBMnv2bLW2tgade/fdd5WRkSFJyszMlMPhUF1dXWDd6/WqqalJTqdTkuR0OtXV1aWWlpbAntdff10DAwOaNWvWzV4PAAAYBaJD2bxy5Urdd9992rRpkx588EEdP35cO3fu1M6dOyVJERERKi0t1dNPP60pU6YoMzNT69atU2pqqhYuXCj9/hmXr371q4EfDfX19Wn58uVavHjx53oHDwAAGP1CCpR7771XBw4cUHl5uTZs2KDMzEw9//zzKioqCuz5/ve/r0uXLmnp0qXq6urS/fffr8OHDysuLi6w58UXX9Ty5cs1d+5cRUZGqrCwUFu3bh3aKwMAACNWSL8HxRT8HhQAAG7eqPk9KAAAAMOBQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxokO9wAmmrzm0HXX3t9cMKyzAAAwFvEMCgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwTkiB8tRTTykiIiLoyMrKCqz39PTI5XIpOTlZ48ePV2FhoTweT9DXaGtrU0FBgcaNG6eUlBStWrVKly9fHrorAgAAI17IbzO+4447dOTIkT98geg/fImVK1fq0KFD2r9/v2w2m5YvX65Fixbp17/+tSSpv79fBQUFcjgcOnbsmD7++GM9/PDDiomJ0aZNm4bqmgAAwAgXcqBER0fL4XBcdb67u1u7du1SdXW15syZI0navXu3pk6dqsbGRuXm5uq1117TmTNndOTIEdntdk2fPl0bN27U6tWr9dRTTyk2NnZorgoAAIxoIb8G5ezZs0pNTdUXvvAFFRUVqa2tTZLU0tKivr4+5eXlBfZmZWUpPT1dDQ0NkqSGhgbl5OTIbrcH9uTn58vr9er06dPXfUyfzyev1xt0AACA0SukQJk1a5b27Nmjw4cPa/v27Tp37pz+8i//UhcuXJDb7VZsbKwSExOD7mO32+V2uyVJbrc7KE4G1wfXrqeiokI2my1wpKWlhTI2AAAYYUL6Ec/8+fMD/zxt2jTNmjVLGRkZ+tnPfqb4+PhbMZ8kqby8XGVlZYHbXq+XSAEAYBS7qbcZJyYm6i/+4i/03nvvyeFwqLe3V11dXUF7PB5P4DUrDofjqnf1DN6+1utaBlksFlmt1qADAACMXjcVKBcvXtT//u//auLEiZoxY4ZiYmJUV1cXWG9tbVVbW5ucTqckyel06tSpU+rs7Azsqa2tldVqVXZ29s2MAgAARpGQfsTzve99TwsWLFBGRoY6Ojr05JNPKioqSg899JBsNpuWLFmisrIyJSUlyWq1asWKFXI6ncrNzZUkzZs3T9nZ2SouLlZlZaXcbrfWrl0rl8sli8Vyq64RAACMMCEFyocffqiHHnpI//d//6fbb79d999/vxobG3X77bdLkrZs2aLIyEgVFhbK5/MpPz9f27ZtC9w/KipKNTU1WrZsmZxOpxISElRSUqINGzYM/ZUBAIARK6RA2bdv36eux8XFqaqqSlVVVdfdk5GRoZdffjmUhwUAAGMMn8UDAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMc1OBsnnzZkVERKi0tDRwrqenRy6XS8nJyRo/frwKCwvl8XiC7tfW1qaCggKNGzdOKSkpWrVqlS5fvnwzowAAgFHkhgOlublZP/nJTzRt2rSg8ytXrtTBgwe1f/9+1dfXq6OjQ4sWLQqs9/f3q6CgQL29vTp27Jj27t2rPXv2aP369Td3JQAAYNS4oUC5ePGiioqK9MILL+i2224LnO/u7tauXbv03HPPac6cOZoxY4Z2796tY8eOqbGxUZL02muv6cyZM/rpT3+q6dOna/78+dq4caOqqqrU29s7dFcGAABGrBsKFJfLpYKCAuXl5QWdb2lpUV9fX9D5rKwspaenq6GhQZLU0NCgnJwc2e32wJ78/Hx5vV6dPn36mo/n8/nk9XqDDgAAMHpFh3qHffv26cSJE2pubr5qze12KzY2VomJiUHn7Xa73G53YM8fx8ng+uDatVRUVOgHP/hBqKMCAIARKqRnUNrb2/XEE0/oxRdfVFxc3K2b6grl5eXq7u4OHO3t7cP22AAAYPiFFCgtLS3q7OzUPffco+joaEVHR6u+vl5bt25VdHS07Ha7ent71dXVFXQ/j8cjh8MhSXI4HFe9q2fw9uCeK1ksFlmt1qADAACMXiEFyty5c3Xq1CmdPHkycMycOVNFRUWBf46JiVFdXV3gPq2trWpra5PT6ZQkOZ1OnTp1Sp2dnYE9tbW1slqtys7OHsprAwAAI1RIr0GZMGGC7rzzzqBzCQkJSk5ODpxfsmSJysrKlJSUJKvVqhUrVsjpdCo3N1eSNG/ePGVnZ6u4uFiVlZVyu91au3atXC6XLBbLUF4bAAAYoUJ+kexn2bJliyIjI1VYWCifz6f8/Hxt27YtsB4VFaWamhotW7ZMTqdTCQkJKikp0YYNG4Z6lBFj8ppD1117f3PBsM4CAIAJbjpQ3njjjaDbcXFxqqqqUlVV1XXvk5GRoZdffvlmHxoAAIxSfBYPAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOCEFyvbt2zVt2jRZrVZZrVY5nU698sorgfWenh65XC4lJydr/PjxKiwslMfjCfoabW1tKigo0Lhx45SSkqJVq1bp8uXLQ3dFAABgxAspUCZNmqTNmzerpaVFb731lubMmaOvf/3rOn36tCRp5cqVOnjwoPbv36/6+np1dHRo0aJFgfv39/eroKBAvb29OnbsmPbu3as9e/Zo/fr1Q39lAABgxIoOZfOCBQuCbj/zzDPavn27GhsbNWnSJO3atUvV1dWaM2eOJGn37t2aOnWqGhsblZubq9dee01nzpzRkSNHZLfbNX36dG3cuFGrV6/WU089pdjY2KG9OgAAMCLd8GtQ+vv7tW/fPl26dElOp1MtLS3q6+tTXl5eYE9WVpbS09PV0NAgSWpoaFBOTo7sdntgT35+vrxeb+BZmGvx+Xzyer1BBwAAGL1CDpRTp05p/Pjxslgs+s53vqMDBw4oOztbbrdbsbGxSkxMDNpvt9vldrslSW63OyhOBtcH166noqJCNpstcKSlpYU6NgAAGEFCDpQvfvGLOnnypJqamrRs2TKVlJTozJkzt2a63ysvL1d3d3fgaG9vv6WPBwAAwiuk16BIUmxsrP78z/9ckjRjxgw1NzfrRz/6kb75zW+qt7dXXV1dQc+ieDweORwOSZLD4dDx48eDvt7gu3wG91yLxWKRxWIJdVQAADBC3fTvQRkYGJDP59OMGTMUExOjurq6wFpra6va2trkdDolSU6nU6dOnVJnZ2dgT21traxWq7Kzs292FAAAMEqE9AxKeXm55s+fr/T0dF24cEHV1dV644039Oqrr8pms2nJkiUqKytTUlKSrFarVqxYIafTqdzcXEnSvHnzlJ2dreLiYlVWVsrtdmvt2rVyuVw8QwIAAAJCCpTOzk49/PDD+vjjj2Wz2TRt2jS9+uqr+pu/+RtJ0pYtWxQZGanCwkL5fD7l5+dr27ZtgftHRUWppqZGy5Ytk9PpVEJCgkpKSrRhw4ahvzIAADBihRQou3bt+tT1uLg4VVVVqaqq6rp7MjIy9PLLL4fysAAAYIzhs3gAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYJzrcA8Ask9ccuu7a+5sLhnUWAMDYxTMoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA44QUKBUVFbr33ns1YcIEpaSkaOHChWptbQ3a09PTI5fLpeTkZI0fP16FhYXyeDxBe9ra2lRQUKBx48YpJSVFq1at0uXLl4fmigAAwIgXUqDU19fL5XKpsbFRtbW16uvr07x583Tp0qXAnpUrV+rgwYPav3+/6uvr1dHRoUWLFgXW+/v7VVBQoN7eXh07dkx79+7Vnj17tH79+qG9MgAAMGJFh7L58OHDQbf37NmjlJQUtbS06K/+6q/U3d2tXbt2qbq6WnPmzJEk7d69W1OnTlVjY6Nyc3P12muv6cyZMzpy5IjsdrumT5+ujRs3avXq1XrqqacUGxs7tFcIAABGnJt6DUp3d7ckKSkpSZLU0tKivr4+5eXlBfZkZWUpPT1dDQ0NkqSGhgbl5OTIbrcH9uTn58vr9er06dM3Mw4AABglQnoG5Y8NDAyotLRUs2fP1p133ilJcrvdio2NVWJiYtBeu90ut9sd2PPHcTK4Prh2LT6fTz6fL3Db6/Xe6NgAAGAEuOFnUFwul95++23t27dvaCe6hoqKCtlstsCRlpZ2yx8TAACEzw0FyvLly1VTU6OjR49q0qRJgfMOh0O9vb3q6uoK2u/xeORwOAJ7rnxXz+DtwT1XKi8vV3d3d+Bob2+/kbEBAMAIEVKg+P1+LV++XAcOHNDrr7+uzMzMoPUZM2YoJiZGdXV1gXOtra1qa2uT0+mUJDmdTp06dUqdnZ2BPbW1tbJarcrOzr7m41osFlmt1qADAACMXiG9BsXlcqm6ulo///nPNWHChMBrRmw2m+Lj42Wz2bRkyRKVlZUpKSlJVqtVK1askNPpVG5uriRp3rx5ys7OVnFxsSorK+V2u7V27Vq5XC5ZLJZbc5UAAGBECSlQtm/fLkn6yle+EnR+9+7d+va3vy1J2rJliyIjI1VYWCifz6f8/Hxt27YtsDcqKko1NTVatmyZnE6nEhISVFJSog0bNgzNFQEAgBEvpEDx+/2fuScuLk5VVVWqqqq67p6MjAy9/PLLoTw0AAAYQ/gsHgAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYJyQA+XNN9/UggULlJqaqoiICL300ktB636/X+vXr9fEiRMVHx+vvLw8nT17NmjP+fPnVVRUJKvVqsTERC1ZskQXL168+asBAACjQsiBcunSJd11112qqqq65nplZaW2bt2qHTt2qKmpSQkJCcrPz1dPT09gT1FRkU6fPq3a2lrV1NTozTff1NKlS2/uSgAAwKgRHeod5s+fr/nz519zze/36/nnn9fatWv19a9/XZL0b//2b7Lb7XrppZe0ePFivfPOOzp8+LCam5s1c+ZMSdKPf/xjfe1rX9M///M/KzU19WavCQAAjHBD+hqUc+fOye12Ky8vL3DOZrNp1qxZamhokCQ1NDQoMTExECeSlJeXp8jISDU1NV3z6/p8Pnm93qADAACMXkMaKG63W5Jkt9uDztvt9sCa2+1WSkpK0Hp0dLSSkpICe65UUVEhm80WONLS0oZybAAAYJgR8S6e8vJydXd3B4729vZwjwQAAG6hIQ0Uh8MhSfJ4PEHnPR5PYM3hcKizszNo/fLlyzp//nxgz5UsFousVmvQAQAARq8hDZTMzEw5HA7V1dUFznm9XjU1NcnpdEqSnE6nurq61NLSEtjz+uuva2BgQLNmzRrKcQAAwAgV8rt4Ll68qPfeey9w+9y5czp58qSSkpKUnp6u0tJSPf3005oyZYoyMzO1bt06paamauHChZKkqVOn6qtf/aoee+wx7dixQ319fVq+fLkWL17MO3jGoMlrDl137f3NBcM6CwDAHCEHyltvvaW//uu/DtwuKyuTJJWUlGjPnj36/ve/r0uXLmnp0qXq6urS/fffr8OHDysuLi5wnxdffFHLly/X3LlzFRkZqcLCQm3dunWorgkAAIxwIQfKV77yFfn9/uuuR0REaMOGDdqwYcN19yQlJam6ujrUhwYAAGPEiHgXDwAAGFsIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxosM9AGCKyWsOfer6+5sLhm0WABjreAYFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcfgsHiCMPu3zf/jsHwBjGc+gAAAA4xAoAADAOGENlKqqKk2ePFlxcXGaNWuWjh8/Hs5xAACAIcIWKP/xH/+hsrIyPfnkkzpx4oTuuusu5efnq7OzM1wjAQAAQ4TtRbLPPfecHnvsMT3yyCOSpB07dujQoUP613/9V61ZsyZcYwFjFi/YBWCSsARKb2+vWlpaVF5eHjgXGRmpvLw8NTQ0XLXf5/PJ5/MFbnd3d0uSvF7vLZlvwPfJddduxWMO9+OZNMtIuXaN8esf7lnufPLV6669/YP8YZ0FGM2G+9/7wa/p9/s/e7M/DD766CO/JP+xY8eCzq9atcr/pS996ar9Tz75pF8SBwcHBwcHxyg42tvbP7MVRsTvQSkvL1dZWVng9sDAgM6fP6/k5GRFRESEdbah5vV6lZaWpvb2dlmt1nCPM+zG+vWL78GYv37xPeD6R/H1+/1+XbhwQampqZ+5NyyB8id/8ieKioqSx+MJOu/xeORwOK7ab7FYZLFYgs4lJibe8jnDyWq1jro/mKEY69cvvgdj/vrF94DrH6XXb7PZPte+sLyLJzY2VjNmzFBdXV3g3MDAgOrq6uR0OsMxEgAAMEjYfsRTVlamkpISzZw5U1/60pf0/PPP69KlS4F39QAAgLErbIHyzW9+U7/97W+1fv16ud1uTZ8+XYcPH5bdbg/XSEawWCx68sknr/qR1lgx1q9ffA/G/PWL7wHXP8avf1CE/3O91wcAAGD48Fk8AADAOAQKAAAwDoECAACMQ6AAAADjECgGqKio0L333qsJEyYoJSVFCxcuVGtra7jHCqvNmzcrIiJCpaWl4R5l2Hz00Uf61re+peTkZMXHxysnJ0dvvfVWuMcaNv39/Vq3bp0yMzMVHx+vP/uzP9PGjRs/32d2jEBvvvmmFixYoNTUVEVEROill14KWvf7/Vq/fr0mTpyo+Ph45eXl6ezZs2Gb91b4tO9BX1+fVq9erZycHCUkJCg1NVUPP/ywOjo6wjrzUPqsPwN/7Dvf+Y4iIiL0/PPPD+uM4USgGKC+vl4ul0uNjY2qra1VX1+f5s2bp0uXLoV7tLBobm7WT37yE02bNi3cowyb3/3ud5o9e7ZiYmL0yiuv6MyZM/rhD3+o2267LdyjDZtnn31W27dv17/8y7/onXfe0bPPPqvKykr9+Mc/Dvdot8SlS5d01113qaqq6prrlZWV2rp1q3bs2KGmpiYlJCQoPz9fPT09wz7rrfJp34NPPvlEJ06c0Lp163TixAn953/+p1pbW/V3f/d3YZn1VvisPwODDhw4oMbGxs/16+FHlaH8EEAMjc7OTr8kf319fbhHGXYXLlzwT5kyxV9bW+v/8pe/7H/iiSfCPdKwWL16tf/+++8P9xhhVVBQ4H/00UeDzi1atMhfVFQUtpmGiyT/gQMHArcHBgb8DofD/0//9E+Bc11dXX6LxeL/93//9zBNeWtd+T24luPHj/sl+T/44INhm2u4XO/6P/zwQ/+f/umf+t9++21/RkaGf8uWLWGZLxx4BsVA3d3dkqSkpKRwjzLsXC6XCgoKlJeXF+5RhtUvfvELzZw5Uw888IBSUlJ0991364UXXgj3WMPqvvvuU11dnd59911J0n//93/rV7/6lebPnx/u0YbduXPn5Ha7g/49sNlsmjVrlhoaGsI6Wzh1d3crIiJi1H8W26CBgQEVFxdr1apVuuOOO8I9zrAbEZ9mPJYMDAyotLRUs2fP1p133hnucYbVvn37dOLECTU3N4d7lGH3m9/8Rtu3b1dZWZn+4R/+Qc3NzXr88ccVGxurkpKScI83LNasWSOv16usrCxFRUWpv79fzzzzjIqKisI92rBzu92SdNVv1rbb7YG1saanp0erV6/WQw89NCo/QO9ann32WUVHR+vxxx8P9yhhQaAYxuVy6e2339avfvWrcI8yrNrb2/XEE0+otrZWcXFx4R5n2A0MDGjmzJnatGmTJOnuu+/W22+/rR07doyZQPnZz36mF198UdXV1brjjjt08uRJlZaWKjU1dcx8D3BtfX19evDBB+X3+7V9+/ZwjzMsWlpa9KMf/UgnTpxQREREuMcJC37EY5Dly5erpqZGR48e1aRJk8I9zrBqaWlRZ2en7rnnHkVHRys6Olr19fXaunWroqOj1d/fH+4Rb6mJEycqOzs76NzUqVPV1tYWtpmG26pVq7RmzRotXrxYOTk5Ki4u1sqVK1VRURHu0Yadw+GQJHk8nqDzHo8nsDZWDMbJBx98oNra2jHz7Mkvf/lLdXZ2Kj09PfB34gcffKDvfve7mjx5crjHGxY8g2IAv9+vFStW6MCBA3rjjTeUmZkZ7pGG3dy5c3Xq1Kmgc4888oiysrK0evVqRUVFhW224TB79uyr3lr+7rvvKiMjI2wzDbdPPvlEkZHB/88UFRWlgYGBsM0ULpmZmXI4HKqrq9P06dMlSV6vV01NTVq2bFm4xxs2g3Fy9uxZHT16VMnJyeEeadgUFxdf9Vq8/Px8FRcX65FHHgnbXMOJQDGAy+VSdXW1fv7zn2vChAmBnzHbbDbFx8eHe7xhMWHChKtec5OQkKDk5OQx8VqclStX6r777tOmTZv04IMP6vjx49q5c6d27twZ7tGGzYIFC/TMM88oPT1dd9xxh/7rv/5Lzz33nB599NFwj3ZLXLx4Ue+9917g9rlz53Ty5EklJSUpPT1dpaWlevrppzVlyhRlZmZq3bp1Sk1N1cKFC8M691D6tO/BxIkT9Y1vfEMnTpxQTU2N+vv7A383JiUlKTY2NoyTD43P+jNwZZDFxMTI4XDoi1/8YhimDYNwv40I///tZdc6du/eHe7Rwmosvc3Y7/f7Dx486L/zzjv9FovFn5WV5d+5c2e4RxpWXq/X/8QTT/jT09P9cXFx/i984Qv+f/zHf/T7fL5wj3ZLHD169Jr/3peUlPj9v3+r8bp16/x2u91vsVj8c+fO9be2toZ77CH1ad+Dc+fOXffvxqNHj4Z79CHxWX8GrjTW3mYc4R+tv6YRAACMWLxIFgAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYJz/B0FPZLxO9+txAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(mkbatch(2048)[1].to(torch.float32).cpu(), bins=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([1516.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,\n",
       "           0.,    0.,    0.,    0.,    0.,    0.,    0.,  420.,    0.,\n",
       "           0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,\n",
       "           0.,    0.,    0.,    0.,    0.,   94.,    0.,    0.,    0.,\n",
       "           0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,\n",
       "           0.,    0.,    0.,   16.,    0.,    0.,    0.,    0.,    0.,\n",
       "           0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,\n",
       "           2.]),\n",
       " array([1.    , 1.0625, 1.125 , 1.1875, 1.25  , 1.3125, 1.375 , 1.4375,\n",
       "        1.5   , 1.5625, 1.625 , 1.6875, 1.75  , 1.8125, 1.875 , 1.9375,\n",
       "        2.    , 2.0625, 2.125 , 2.1875, 2.25  , 2.3125, 2.375 , 2.4375,\n",
       "        2.5   , 2.5625, 2.625 , 2.6875, 2.75  , 2.8125, 2.875 , 2.9375,\n",
       "        3.    , 3.0625, 3.125 , 3.1875, 3.25  , 3.3125, 3.375 , 3.4375,\n",
       "        3.5   , 3.5625, 3.625 , 3.6875, 3.75  , 3.8125, 3.875 , 3.9375,\n",
       "        4.    , 4.0625, 4.125 , 4.1875, 4.25  , 4.3125, 4.375 , 4.4375,\n",
       "        4.5   , 4.5625, 4.625 , 4.6875, 4.75  , 4.8125, 4.875 , 4.9375,\n",
       "        5.    ]),\n",
       " <BarContainer object of 64 artists>)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAoW0lEQVR4nO3df1RU94H//9cgAsY4g2iZcTaotE1VjJpEEjLmd2VFpe56SjehYa3bstrNQhqjMcJpNZqkxZisRlMqsW2CZ2tOfuxZ3cYkGKpVWkMQUValhprUKKkZyB5kRshHRLnfP771noyCghmEN3k+zrnnZO59z9z3u++e8Mw4jA7LsiwBAAAYJKK3JwAAANBdBAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA40T29gR6Snt7u06cOKEhQ4bI4XD09nQAAEAXWJalU6dOyev1KiKi8/dZ+m3AnDhxQgkJCb09DQAAcAXq6up03XXXdXq93wbMkCFDpL/9D+B0Ont7OgAAoAuCwaASEhLsn+Od6bcBc/6PjZxOJwEDAIBhLvfxDz7ECwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA40T29gRMNDrvzU6vfbQy/arOBQCALyPegQEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxuh0wZWVlmjVrlrxerxwOh7Zs2dLp2H/7t3+Tw+HQc889F3K+sbFRWVlZcjqdio2NVXZ2tpqbm0PGHDhwQHfeeadiYmKUkJCgVatWdXeqAACgn+p2wLS0tGjSpEkqLCy85LjNmzfrvffek9frvehaVlaWampqVFpaqq1bt6qsrEzz58+3rweDQU2bNk2jRo1SVVWVnnnmGS1fvlwbNmzo7nQBAEA/FNndJ8yYMUMzZsy45Ji//vWveuihh7Rt2zalp6eHXDt8+LBKSkpUWVmp5ORkSdLzzz+vmTNn6tlnn5XX69WmTZt05swZvfjii4qKitL48eNVXV2t1atXh4QOAAD4cgr7Z2Da29s1Z84cLV68WOPHj7/oenl5uWJjY+14kaTU1FRFRESooqLCHnPXXXcpKirKHpOWlqba2lqdPHmyw/u2trYqGAyGHAAAoH8Ke8A8/fTTioyM1I9+9KMOr/v9fsXHx4eci4yMVFxcnPx+vz3G7XaHjDn/+PyYCxUUFMjlctlHQkJCmFYEAAD6mrAGTFVVldauXavi4mI5HI5wvvRl5efnKxAI2EddXd1VvT8AALh6whowf/jDH9TQ0KCRI0cqMjJSkZGROnbsmBYtWqTRo0dLkjwejxoaGkKed/bsWTU2Nsrj8dhj6uvrQ8acf3x+zIWio6PldDpDDgAA0D+FNWDmzJmjAwcOqLq62j68Xq8WL16sbdu2SZJ8Pp+amppUVVVlP2/Hjh1qb29XSkqKPaasrExtbW32mNLSUo0ZM0ZDhw4N55QBAICBuv1bSM3Nzfrggw/sx0ePHlV1dbXi4uI0cuRIDRs2LGT8wIED5fF4NGbMGEnSuHHjNH36dM2bN09FRUVqa2tTbm6uMjMz7V+5fuCBB7RixQplZ2dryZIlOnTokNauXas1a9Z88RUDAADjdTtg9u7dq3vvvdd+vHDhQknS3LlzVVxc3KXX2LRpk3JzczV16lRFREQoIyND69ats6+7XC698847ysnJ0eTJkzV8+HAtW7aMX6EGAACSJIdlWVZvT6InBINBuVwuBQKBsH8eZnTem51e+2hleqfXAADApXX15zd/FxIAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjdDtgysrKNGvWLHm9XjkcDm3ZssW+1tbWpiVLlmjChAkaPHiwvF6vvve97+nEiRMhr9HY2KisrCw5nU7FxsYqOztbzc3NIWMOHDigO++8UzExMUpISNCqVau+yDoBAEA/0u2AaWlp0aRJk1RYWHjRtc8++0z79u3T0qVLtW/fPv33f/+3amtr9Q//8A8h47KyslRTU6PS0lJt3bpVZWVlmj9/vn09GAxq2rRpGjVqlKqqqvTMM89o+fLl2rBhw5WuEwAA9CMOy7KsK36yw6HNmzdr9uzZnY6prKzUrbfeqmPHjmnkyJE6fPiwkpKSVFlZqeTkZElSSUmJZs6cqY8//lher1fr16/Xj3/8Y/n9fkVFRUmS8vLytGXLFr3//vtdmlswGJTL5VIgEJDT6bzSJXZodN6bnV77aGV6WO8FAMCXSVd/fvf4Z2ACgYAcDodiY2MlSeXl5YqNjbXjRZJSU1MVERGhiooKe8xdd91lx4skpaWlqba2VidPnuzpKQMAgD4usidf/PTp01qyZIm++93v2hXl9/sVHx8fOonISMXFxcnv99tjEhMTQ8a43W772tChQy+6V2trq1pbW+3HwWCwR9YEAAB6X4+9A9PW1qb77rtPlmVp/fr1PXUbW0FBgVwul30kJCT0+D0BAEDv6JGAOR8vx44dU2lpacifYXk8HjU0NISMP3v2rBobG+XxeOwx9fX1IWPOPz4/5kL5+fkKBAL2UVdX1wMrAwAAfUHYA+Z8vBw5ckS/+93vNGzYsJDrPp9PTU1Nqqqqss/t2LFD7e3tSklJsceUlZWpra3NHlNaWqoxY8Z0+MdHkhQdHS2n0xlyAACA/qnbAdPc3Kzq6mpVV1dLko4eParq6modP35cbW1t+s53vqO9e/dq06ZNOnfunPx+v/x+v86cOSNJGjdunKZPn6558+Zpz5492r17t3Jzc5WZmSmv1ytJeuCBBxQVFaXs7GzV1NTo1Vdf1dq1a7Vw4cJwrx8AABio279GvXPnTt17770XnZ87d66WL19+0Ydvz/v973+ve+65R/rbF9nl5ubqjTfeUEREhDIyMrRu3Tpde+219vgDBw4oJydHlZWVGj58uB566CEtWbKky/Pk16gBADBPV39+f6HvgenLCBgAAMzTZ74HBgAAINwIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMbpdsCUlZVp1qxZ8nq9cjgc2rJlS8h1y7K0bNkyjRgxQoMGDVJqaqqOHDkSMqaxsVFZWVlyOp2KjY1Vdna2mpubQ8YcOHBAd955p2JiYpSQkKBVq1Zd6RoBAEA/0+2AaWlp0aRJk1RYWNjh9VWrVmndunUqKipSRUWFBg8erLS0NJ0+fdoek5WVpZqaGpWWlmrr1q0qKyvT/Pnz7evBYFDTpk3TqFGjVFVVpWeeeUbLly/Xhg0brnSdAACgH3FYlmVd8ZMdDm3evFmzZ8+W/vbui9fr1aJFi/Too49KkgKBgNxut4qLi5WZmanDhw8rKSlJlZWVSk5OliSVlJRo5syZ+vjjj+X1erV+/Xr9+Mc/lt/vV1RUlCQpLy9PW7Zs0fvvv9+luQWDQblcLgUCATmdzitdYodG573Z6bWPVqaH9V4AAHyZdPXnd1g/A3P06FH5/X6lpqba51wul1JSUlReXi5JKi8vV2xsrB0vkpSamqqIiAhVVFTYY+666y47XiQpLS1NtbW1OnnyZIf3bm1tVTAYDDkAAED/FNaA8fv9kiS32x1y3u1229f8fr/i4+NDrkdGRiouLi5kTEev8fl7XKigoEAul8s+EhISwrgyAADQl/Sb30LKz89XIBCwj7q6ut6eEgAA6CFhDRiPxyNJqq+vDzlfX19vX/N4PGpoaAi5fvbsWTU2NoaM6eg1Pn+PC0VHR8vpdIYcAACgfwprwCQmJsrj8Wj79u32uWAwqIqKCvl8PkmSz+dTU1OTqqqq7DE7duxQe3u7UlJS7DFlZWVqa2uzx5SWlmrMmDEaOnRoOKcMAAAM1O2AaW5uVnV1taqrq6W/fXC3urpax48fl8Ph0IIFC/TUU0/pt7/9rQ4ePKjvfe978nq99m8qjRs3TtOnT9e8efO0Z88e7d69W7m5ucrMzJTX65UkPfDAA4qKilJ2drZqamr06quvau3atVq4cGG41w8AAAwU2d0n7N27V/fee6/9+HxUzJ07V8XFxXrsscfU0tKi+fPnq6mpSXfccYdKSkoUExNjP2fTpk3Kzc3V1KlTFRERoYyMDK1bt86+7nK59M477ygnJ0eTJ0/W8OHDtWzZspDvigEAAF9eX+h7YPoyvgcGAADz9Mr3wAAAAFwNBAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACME/aAOXfunJYuXarExEQNGjRIX/va1/Tkk0/Ksix7jGVZWrZsmUaMGKFBgwYpNTVVR44cCXmdxsZGZWVlyel0KjY2VtnZ2Wpubg73dAEAgIHCHjBPP/201q9fr5///Oc6fPiwnn76aa1atUrPP/+8PWbVqlVat26dioqKVFFRocGDBystLU2nT5+2x2RlZammpkalpaXaunWrysrKNH/+/HBPFwAAGMhhff6tkTD41re+JbfbrV//+tf2uYyMDA0aNEi/+c1vZFmWvF6vFi1apEcffVSSFAgE5Ha7VVxcrMzMTB0+fFhJSUmqrKxUcnKyJKmkpEQzZ87Uxx9/LK/Xe9l5BINBuVwuBQIBOZ3OcC5Ro/Pe7PTaRyvTw3ovAAC+TLr68zvs78BMmTJF27dv15///GdJ0v/+7//qj3/8o2bMmCFJOnr0qPx+v1JTU+3nuFwupaSkqLy8XJJUXl6u2NhYO14kKTU1VREREaqoqAj3lAEAgGEiw/2CeXl5CgaDGjt2rAYMGKBz587ppz/9qbKysiRJfr9fkuR2u0Oe53a77Wt+v1/x8fGhE42MVFxcnD3mQq2trWptbbUfB4PBcC8NAAD0EWF/B+a1117Tpk2b9PLLL2vfvn3auHGjnn32WW3cuDHctwpRUFAgl8tlHwkJCT16PwAA0HvCHjCLFy9WXl6eMjMzNWHCBM2ZM0ePPPKICgoKJEkej0eSVF9fH/K8+vp6+5rH41FDQ0PI9bNnz6qxsdEec6H8/HwFAgH7qKurC/fSAABAHxH2gPnss88UERH6sgMGDFB7e7skKTExUR6PR9u3b7evB4NBVVRUyOfzSZJ8Pp+amppUVVVlj9mxY4fa29uVkpLS4X2jo6PldDpDDgAA0D+F/TMws2bN0k9/+lONHDlS48eP1/79+7V69Wr94Ac/kCQ5HA4tWLBATz31lK6//nolJiZq6dKl8nq9mj17tiRp3Lhxmj59uubNm6eioiK1tbUpNzdXmZmZXfoNJAAA0L+FPWCef/55LV26VP/+7/+uhoYGeb1e/fCHP9SyZcvsMY899phaWlo0f/58NTU16Y477lBJSYliYmLsMZs2bVJubq6mTp2qiIgIZWRkaN26deGeLgAAMFDYvwemr+B7YAAAME+vfQ8MAABATyNgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGKdHAuavf/2r/vmf/1nDhg3ToEGDNGHCBO3du9e+blmWli1bphEjRmjQoEFKTU3VkSNHQl6jsbFRWVlZcjqdio2NVXZ2tpqbm3tiugAAwDBhD5iTJ0/q9ttv18CBA/X222/rT3/6k/7jP/5DQ4cOtcesWrVK69atU1FRkSoqKjR48GClpaXp9OnT9pisrCzV1NSotLRUW7duVVlZmebPnx/u6QIAAAM5LMuywvmCeXl52r17t/7whz90eN2yLHm9Xi1atEiPPvqoJCkQCMjtdqu4uFiZmZk6fPiwkpKSVFlZqeTkZElSSUmJZs6cqY8//lher/ey8wgGg3K5XAoEAnI6neFcokbnvdnptY9Wpof1XgAAfJl09ed32N+B+e1vf6vk5GT90z/9k+Lj43XTTTfpl7/8pX396NGj8vv9Sk1Ntc+5XC6lpKSovLxcklReXq7Y2Fg7XiQpNTVVERERqqio6PC+ra2tCgaDIQcAAOifwh4wf/nLX7R+/Xpdf/312rZtmx588EH96Ec/0saNGyVJfr9fkuR2u0Oe53a77Wt+v1/x8fEh1yMjIxUXF2ePuVBBQYFcLpd9JCQkhHtpAACgjwh7wLS3t+vmm2/Wz372M910002aP3++5s2bp6KionDfKkR+fr4CgYB91NXV9ej9AABA7wl7wIwYMUJJSUkh58aNG6fjx49LkjwejySpvr4+ZEx9fb19zePxqKGhIeT62bNn1djYaI+5UHR0tJxOZ8gBAAD6p7AHzO23367a2tqQc3/+8581atQoSVJiYqI8Ho+2b99uXw8Gg6qoqJDP55Mk+Xw+NTU1qaqqyh6zY8cOtbe3KyUlJdxTBgAAhokM9ws+8sgjmjJlin72s5/pvvvu0549e7RhwwZt2LBBkuRwOLRgwQI99dRTuv7665WYmKilS5fK6/Vq9uzZ0t/esZk+fbr9R09tbW3Kzc1VZmZml34DCQAA9G9hD5hbbrlFmzdvVn5+vp544gklJibqueeeU1ZWlj3mscceU0tLi+bPn6+mpibdcccdKikpUUxMjD1m06ZNys3N1dSpUxUREaGMjAytW7cu3NMFAAAGCvv3wPQVfA8MAADm6bXvgQEAAOhpBAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACM0+MBs3LlSjkcDi1YsMA+d/r0aeXk5GjYsGG69tprlZGRofr6+pDnHT9+XOnp6brmmmsUHx+vxYsX6+zZsz09XXxJjM57s9MDAND39WjAVFZW6oUXXtDEiRNDzj/yyCN644039Prrr2vXrl06ceKEvv3tb9vXz507p/T0dJ05c0bvvvuuNm7cqOLiYi1btqwnpwsAAAzRYwHT3NysrKws/fKXv9TQoUPt84FAQL/+9a+1evVqffOb39TkyZP10ksv6d1339V7770nSXrnnXf0pz/9Sb/5zW904403asaMGXryySdVWFioM2fO9NSUAQCAIXosYHJycpSenq7U1NSQ81VVVWpraws5P3bsWI0cOVLl5eWSpPLyck2YMEFut9sek5aWpmAwqJqamg7v19raqmAwGHIAAID+KbInXvSVV17Rvn37VFlZedE1v9+vqKgoxcbGhpx3u93y+/32mM/Hy/nr5691pKCgQCtWrAjjKgAAQF8V9ndg6urq9PDDD2vTpk2KiYkJ98t3Kj8/X4FAwD7q6uqu2r0BAMDVFfaAqaqqUkNDg26++WZFRkYqMjJSu3bt0rp16xQZGSm3260zZ86oqakp5Hn19fXyeDySJI/Hc9FvJZ1/fH7MhaKjo+V0OkMOAADQP4U9YKZOnaqDBw+qurraPpKTk5WVlWX/88CBA7V9+3b7ObW1tTp+/Lh8Pp8kyefz6eDBg2poaLDHlJaWyul0KikpKdxTBgAAhgn7Z2CGDBmiG264IeTc4MGDNWzYMPt8dna2Fi5cqLi4ODmdTj300EPy+Xy67bbbJEnTpk1TUlKS5syZo1WrVsnv9+snP/mJcnJyFB0dHe4pAwAAw/TIh3gvZ82aNYqIiFBGRoZaW1uVlpamX/ziF/b1AQMGaOvWrXrwwQfl8/k0ePBgzZ07V0888URvTBcAAPQxVyVgdu7cGfI4JiZGhYWFKiws7PQ5o0aN0ltvvXUVZgcAAEzD34UEAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwTtgDpqCgQLfccouGDBmi+Ph4zZ49W7W1tSFjTp8+rZycHA0bNkzXXnutMjIyVF9fHzLm+PHjSk9P1zXXXKP4+HgtXrxYZ8+eDfd0AQCAgcIeMLt27VJOTo7ee+89lZaWqq2tTdOmTVNLS4s95pFHHtEbb7yh119/Xbt27dKJEyf07W9/275+7tw5paen68yZM3r33Xe1ceNGFRcXa9myZeGeLgAAMJDDsiyrJ2/w6aefKj4+Xrt27dJdd92lQCCgr3zlK3r55Zf1ne98R5L0/vvva9y4cSovL9dtt92mt99+W9/61rd04sQJud1uSVJRUZGWLFmiTz/9VFFRUZe9bzAYlMvlUiAQkNPpDOuaRue92em1j1amh/Ve6BnsIQD0TV39+d3jn4EJBAKSpLi4OElSVVWV2tralJqaao8ZO3asRo4cqfLycklSeXm5JkyYYMeLJKWlpSkYDKqmpqbD+7S2tioYDIYcAACgf+rRgGlvb9eCBQt0++2364YbbpAk+f1+RUVFKTY2NmSs2+2W3++3x3w+Xs5fP3+tIwUFBXK5XPaRkJDQQ6sCAAC9rUcDJicnR4cOHdIrr7zSk7eRJOXn5ysQCNhHXV1dj98TAAD0jsieeuHc3Fxt3bpVZWVluu666+zzHo9HZ86cUVNTU8i7MPX19fJ4PPaYPXv2hLze+d9SOj/mQtHR0YqOju6h1QAAgL4k7O/AWJal3Nxcbd68WTt27FBiYmLI9cmTJ2vgwIHavn27fa62tlbHjx+Xz+eTJPl8Ph08eFANDQ32mNLSUjmdTiUlJYV7ygAAwDBhfwcmJydHL7/8sv7nf/5HQ4YMsT+z4nK5NGjQILlcLmVnZ2vhwoWKi4uT0+nUQw89JJ/Pp9tuu02SNG3aNCUlJWnOnDlatWqV/H6/fvKTnygnJ4d3WQAAQPgDZv369ZKke+65J+T8Sy+9pH/5l3+RJK1Zs0YRERHKyMhQa2ur0tLS9Itf/MIeO2DAAG3dulUPPvigfD6fBg8erLlz5+qJJ54I93QBAICBwh4wXflamZiYGBUWFqqwsLDTMaNGjdJbb70V5tkBAID+gL8LCQAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYJ7K3JwAAV2J03pudXvtoZfpVnQuAq493YAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcfp0wBQWFmr06NGKiYlRSkqK9uzZ09tTAgAAfUBkb0+gM6+++qoWLlyooqIipaSk6LnnnlNaWppqa2sVHx/f29MDAHxBo/Pe7PTaRyvTr+pcYJ4++w7M6tWrNW/ePH3/+99XUlKSioqKdM011+jFF1/s7akBAIBe1iffgTlz5oyqqqqUn59vn4uIiFBqaqrKy8s7fE5ra6taW1vtx4FAQJIUDAbDPr/21s86vdYT90P4sYfmYw/Nxx6iI+f33rKsS47rkwHzf//3fzp37pzcbnfIebfbrffff7/D5xQUFGjFihUXnU9ISOixeXbE9dxVvR16AHtoPvbQfOwhTp06JZfL1en1PhkwVyI/P18LFy60H7e3t6uxsVHDhg2Tw+EI232CwaASEhJUV1cnp9MZttftS/r7Glmf+fr7Gvv7+vQlWCPru3KWZenUqVPyer2XHNcnA2b48OEaMGCA6uvrQ87X19fL4/F0+Jzo6GhFR0eHnIuNje2xOTqdzn75f8rP6+9rZH3m6+9r7O/r05dgjazvylzqnZfz+uSHeKOiojR58mRt377dPtfe3q7t27fL5/P16twAAEDv65PvwEjSwoULNXfuXCUnJ+vWW2/Vc889p5aWFn3/+9/v7akBAIBe1mcD5v7779enn36qZcuWye/368Ybb1RJSclFH+y92qKjo/X4449f9MdV/Ul/XyPrM19/X2N/X5++BGtkfT3PYV3u95QAAAD6mD75GRgAAIBLIWAAAIBxCBgAAGAcAgYAABiHgLlAWVmZZs2aJa/XK4fDoS1btlz2OTt37tTNN9+s6Ohoff3rX1dxcfFVmeuV6O76du7cKYfDcdHh9/uv2py7o6CgQLfccouGDBmi+Ph4zZ49W7W1tZd93uuvv66xY8cqJiZGEyZM0FtvvXVV5nslrmSNxcXFF+1hTEzMVZtzd6xfv14TJ060vyDL5/Pp7bffvuRzTNq/7q7PpL3ryMqVK+VwOLRgwYJLjjNpDy/UlTWatI/Lly+/aK5jx4695HN6Y/8ImAu0tLRo0qRJKiws7NL4o0ePKj09Xffee6+qq6u1YMEC/eu//qu2bdvW43O9Et1d33m1tbX65JNP7CM+Pr7H5vhF7Nq1Szk5OXrvvfdUWlqqtrY2TZs2TS0tLZ0+591339V3v/tdZWdna//+/Zo9e7Zmz56tQ4cOXdW5d9WVrFF/+8bMz+/hsWPHrtqcu+O6667TypUrVVVVpb179+qb3/ym/vEf/1E1NTUdjjdt/7q7Phm0dxeqrKzUCy+8oIkTJ15ynGl7+HldXaMM28fx48eHzPWPf/xjp2N7bf8sdEqStXnz5kuOeeyxx6zx48eHnLv//vuttLS0Hp7dF9eV9f3+97+3JFknT568avMKp4aGBkuStWvXrk7H3HfffVZ6enrIuZSUFOuHP/zhVZjhF9eVNb700kuWy+W6qvMKp6FDh1q/+tWvOrxm+v5Zl1mfqXt36tQp6/rrr7dKS0utu+++23r44Yc7HWvqHnZnjSbt4+OPP25NmjSpy+N7a/94B+YLKi8vV2pqasi5tLQ0lZeX99qcesKNN96oESNG6O///u+1e/fu3p5OlwUCAUlSXFxcp2NM38OurFGSmpubNWrUKCUkJFz2v/j7inPnzumVV15RS0tLp3+NiMn715X1ydC9y8nJUXp6+kV70xFT97A7a5Rh+3jkyBF5vV599atfVVZWlo4fP97p2N7avz77Tbym8Pv9F307sNvtVjAY1P/7f/9PgwYN6rW5hcOIESNUVFSk5ORktba26le/+pXuueceVVRU6Oabb+7t6V1Se3u7FixYoNtvv1033HBDp+M628O++jmfz+vqGseMGaMXX3xREydOVCAQ0LPPPqspU6aopqZG11133VWdc1ccPHhQPp9Pp0+f1rXXXqvNmzcrKSmpw7Em7l931mfa3knSK6+8on379qmysrJL403cw+6u0aR9TElJUXFxscaMGaNPPvlEK1as0J133qlDhw5pyJAhF43vrf0jYHBJY8aM0ZgxY+zHU6ZM0Ycffqg1a9boP//zP3t1bpeTk5OjQ4cOXfLPbk3X1TX6fL6Q/8KfMmWKxo0bpxdeeEFPPvnkVZhp94wZM0bV1dUKBAL6r//6L82dO1e7du3q9Ie8abqzPtP2rq6uTg8//LBKS0v77IdUv6grWaNJ+zhjxgz7nydOnKiUlBSNGjVKr732mrKzs3t1bp9HwHxBHo9H9fX1Iefq6+vldDqNf/elM7feemufj4Lc3Fxt3bpVZWVll/2vm8720OPx9PAsv5jurPFCAwcO1E033aQPPvigx+b3RURFRenrX/+6JGny5MmqrKzU2rVr9cILL1w01sT96876LtTX966qqkoNDQ0h79CeO3dOZWVl+vnPf67W1lYNGDAg5Dmm7eGVrPFCfX0fPy82Nlbf+MY3Op1rb+0fn4H5gnw+n7Zv3x5yrrS09JJ/nm266upqjRgxoren0SHLspSbm6vNmzdrx44dSkxMvOxzTNvDK1njhc6dO6eDBw/22X28UHt7u1pbWzu8Ztr+deRS67tQX9+7qVOn6uDBg6qurraP5ORkZWVlqbq6usMf7Kbt4ZWs8UJ9fR8/r7m5WR9++GGnc+21/evRjwgb6NSpU9b+/fut/fv3W5Ks1atXW/v377eOHTtmWZZl5eXlWXPmzLHH/+Uvf7GuueYaa/Hixdbhw4etwsJCa8CAAVZJSUkvrqJz3V3fmjVrrC1btlhHjhyxDh48aD388MNWRESE9bvf/a4XV9G5Bx980HK5XNbOnTutTz75xD4+++wze8ycOXOsvLw8+/Hu3butyMhI69lnn7UOHz5sPf7449bAgQOtgwcP9tIqLu1K1rhixQpr27Zt1ocffmhVVVVZmZmZVkxMjFVTU9NLq+hcXl6etWvXLuvo0aPWgQMHrLy8PMvhcFjvvPOOZfWD/evu+kzau85c+Bs6pu9hRy63RpP2cdGiRdbOnTuto0ePWrt377ZSU1Ot4cOHWw0NDZbVh/aPgLnA+V8bvvCYO3euZVmWNXfuXOvuu+++6Dk33nijFRUVZX31q1+1XnrppV6a/eV1d31PP/209bWvfc2KiYmx4uLirHvuucfasWNHL67g0jpam6SQPbn77rvt9Z732muvWd/4xjesqKgoa/z48dabb77ZC7PvmitZ44IFC6yRI0daUVFRltvttmbOnGnt27evl1ZwaT/4wQ+sUaNGWVFRUdZXvvIVa+rUqfYPd6sf7F9312fS3nXmwh/upu9hRy63RpP28f7777dGjBhhRUVFWX/3d39n3X///dYHH3xgX+8r++ew/v9/IQIAABiDz8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACM8/8BbhuvIZklPe0AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(mktunebatch(2048)[1].to(torch.float32).cpu(), bins=64)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Q3Cg_8UQep8g"
   },
   "source": [
    "# Step 2: Define Transformer Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "execution_state": "idle",
   "metadata": {
    "id": "tLOWhg_CeWzH"
   },
   "outputs": [],
   "source": [
    "class TransformerModel(nn.Module):\n",
    "    def __init__(self, input_dim, model_dim, output_dim, num_heads, num_layers, seq_len, dropout):\n",
    "        super().__init__()\n",
    "        self.model_dim = model_dim\n",
    "        self.embedding = nn.Embedding(input_dim, model_dim // 2, dtype=torch.bfloat16)\n",
    "        # # seq_len is odd\n",
    "        # self.fancy_encoding = torch.repeat_interleave(torch.rand((1, seq_len // 2 + 1, model_dim // 2), device=device, dtype=torch.bfloat16), 2, dim=1)\n",
    "        # # cut off last element since the target vertex is not repeated\n",
    "        # self.fancy_encoding = self.fancy_encoding[:, :seq_len, :]\n",
    "        encoder_layer = nn.TransformerEncoderLayer(d_model=model_dim, nhead=num_heads,\n",
    "                                                   dim_feedforward=model_dim*4,\n",
    "                                                   dropout=dropout, batch_first=True, dtype=torch.bfloat16)\n",
    "        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers)\n",
    "        self.fc_out = nn.Linear(model_dim, output_dim, dtype=torch.bfloat16)\n",
    "\n",
    "    def forward(self, src, key_padding_mask):\n",
    "        batch_sz = src.size(0)\n",
    "        src = torch.cat((src, torch.full((batch_sz, 1), MAX_VTXS + 1, device=device)), dim=1)\n",
    "        embed = torch.cat((self.embedding(src[:,::2]), self.embedding(src[:,1::2])), dim=2)\n",
    "        output = self.transformer_encoder(embed, src_key_padding_mask=key_padding_mask[:, ::2])\n",
    "        return self.fc_out(output[:, -1, :])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bpIeg86S-hBb"
   },
   "source": [
    "# Step 3: Make Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "execution_state": "idle",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kWXvJRDYgFVP",
    "outputId": "c13adb9d-6565-43b5-8437-20cef3dc0d16"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data: 104M\n",
      "Trainable parameters in the model: 200545\n"
     ]
    }
   ],
   "source": [
    "# PARAMS\n",
    "VOCAB_SIZE = 1 + MAX_VTXS + 1 # pad plus max number of vertices plus target token\n",
    "MODEL_DIM = 64 # Dimension of model (embedding and transformer)\n",
    "NEPOCHS = 100\n",
    "BSZ = 2**17 # Batch size\n",
    "BPE = 8 # Batches per epoch\n",
    "NHEADS = 2\n",
    "NLAYERS = 4\n",
    "DROPOUT = 0 # 0.2\n",
    "model = TransformerModel(input_dim=VOCAB_SIZE, model_dim=MODEL_DIM,\n",
    "                         output_dim=1, num_heads=NHEADS,\n",
    "                         num_layers=NLAYERS, seq_len=SEQ_LEN,\n",
    "                         dropout=DROPOUT).to(device)\n",
    "model = torch.compile(model)\n",
    "\n",
    "trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "print(f\"Training data: {NEPOCHS*BPE*BSZ//10**6}M\")\n",
    "print(f\"Trainable parameters in the model: {trainable_params}\")\n",
    "\n",
    "train_err = []\n",
    "len1 = []\n",
    "len2 = []\n",
    "len3 = []\n",
    "len15 = []\n",
    "epoch = 0\n",
    "\n",
    "# clear loss file\n",
    "open('loss', 'w').close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MODEL_DIM = 64 # Dimension of model (embedding and transformer)\n",
    "NEPOCHS = 1000\n",
    "BSZ = 2**17 # Batch size\n",
    "BPE = 8 # Batches per epoch\n",
    "NHEADS = 2\n",
    "NLAYERS = 4\n",
    "DROPOUT = 0 # 0.2\n",
    "\n",
    "LR of 8e-4 for 1000 epochs to get down to 0.35546875"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "# model = TransformerModel(input_dim=VOCAB_SIZE, model_dim=MODEL_DIM,\n",
    "#                          output_dim=1, num_heads=NHEADS,\n",
    "#                          num_layers=NLAYERS, seq_len=SEQ_LEN,\n",
    "#                          dropout=DROPOUT).to(device)\n",
    "# model = torch.compile(model)\n",
    "# model.load_state_dict(torch.load('model.pth', weights_only=True))\n",
    "\n",
    "LR = 8e-4\n",
    "WD = 0 # 1e-5\n",
    "\n",
    "criterion = nn.MSELoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=LR, weight_decay=WD)\n",
    "# scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=100, cooldown=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "f8Zn33m7CxL5"
   },
   "source": [
    "# Step 4: Train the Model for the first task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate():\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    with torch.no_grad():\n",
    "        batch_src, batch_labels, batch_padding_mask = map(lambda x: x.to(device), mkbatch(BSZ))\n",
    "        output = model(batch_src, batch_padding_mask)\n",
    "        loss = criterion(output.squeeze(1), batch_labels)\n",
    "    return loss.item()\n",
    "\n",
    "# figure out if it's doing better on short paths\n",
    "def evaluate_short():\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    with torch.no_grad():\n",
    "        batch_src, batch_labels, batch_padding_mask = map(lambda x: x.to(device), mkbatch(BSZ))\n",
    "        output = model(batch_src, batch_padding_mask)\n",
    "        loss = criterion(output[batch_labels == 1].squeeze(1), batch_labels[batch_labels==1])\n",
    "    return loss.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "execution_state": "idle",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 486
    },
    "id": "pvTfzGmCeXU4",
    "outputId": "0d3a20f3-23be-4c19-9eb6-46bfe11a48b1"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sipb/.venv/lib64/python3.12/site-packages/torch/nn/functional.py:6278: UserWarning: Memory Efficient attention on Navi31 GPU is still experimental. Enable it with TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1. (Triggered internally at ../aten/src/ATen/native/transformers/hip/sdp_utils.cpp:269.)\n",
      "  attn_output = scaled_dot_product_attention(\n",
      "/tmp/torchinductor_sipb/bn/cbngaobakjqlwlijvkqph5lgddb2z2kzjaln3b2g2j75b6snskdn.py:859: UserWarning: Attempting to use hipBLASLt on an unsupported architecture! Overriding blas backend to hipblas (Triggered internally at ../aten/src/ATen/Context.cpp:296.)\n",
      "  extern_kernels.mm(reinterpret_tensor(buf1, (2097152, 64), (64, 1), 0), reinterpret_tensor(primals_5, (64, 192), (1, 64), 0), out=buf2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0/100 \t Train Err: 87.5000 0.62109375 3.28125 8.125 222.0\n",
      "Epoch 0/100 \t Train Err: 70.5000 0.5078125 0.173828125 1.953125 182.0\n",
      "Epoch 0/100 \t Train Err: 59.7500 2.828125 0.4140625 0.134765625 154.0\n",
      "Epoch 0/100 \t Train Err: 54.0000 5.5 1.734375 0.1279296875 137.0\n",
      "Epoch 0/100 \t Train Err: 50.7500 7.9375 3.21875 0.6953125 126.0\n",
      "Epoch 0/100 \t Train Err: 48.5000 10.0 4.625 1.40625 118.0\n",
      "Epoch 0/100 \t Train Err: 46.7500 11.75 5.84375 2.109375 111.5\n",
      "Epoch 0/100 \t Train Err: 45.7500 13.125 6.90625 2.75 107.5\n",
      "Epoch 1/100 \t Train Err: 44.7500 14.25 7.75 3.28125 104.0\n",
      "Epoch 1/100 \t Train Err: 44.5000 15.1875 8.4375 3.71875 102.0\n",
      "Epoch 1/100 \t Train Err: 44.2500 15.875 9.0 4.09375 100.0\n",
      "Epoch 1/100 \t Train Err: 43.7500 16.5 9.4375 4.34375 98.5\n",
      "Epoch 1/100 \t Train Err: 43.7500 16.875 9.8125 4.59375 97.5\n",
      "Epoch 1/100 \t Train Err: 43.5000 17.25 10.1875 4.8125 96.5\n",
      "Epoch 1/100 \t Train Err: 43.2500 17.625 10.4375 5.0 95.0\n",
      "Epoch 1/100 \t Train Err: 43.2500 18.0 10.6875 5.1875 95.0\n",
      "Epoch 2/100 \t Train Err: 43.0000 18.5 11.0 5.34375 94.0\n",
      "Epoch 2/100 \t Train Err: 42.5000 18.75 11.25 5.5625 92.5\n",
      "Epoch 2/100 \t Train Err: 42.7500 19.125 11.5625 5.75 92.5\n",
      "Epoch 2/100 \t Train Err: 42.5000 19.5 11.8125 5.9375 91.5\n",
      "Epoch 2/100 \t Train Err: 42.0000 19.875 12.1875 6.1875 90.0\n",
      "Epoch 2/100 \t Train Err: 42.2500 20.25 12.5 6.40625 90.0\n",
      "Epoch 2/100 \t Train Err: 42.0000 20.625 12.6875 6.59375 89.0\n",
      "Epoch 2/100 \t Train Err: 41.7500 21.0 13.0625 6.84375 88.0\n",
      "Epoch 3/100 \t Train Err: 42.2500 21.375 13.375 7.0625 88.0\n",
      "Epoch 3/100 \t Train Err: 41.7500 21.75 13.6875 7.28125 86.0\n",
      "Epoch 3/100 \t Train Err: 41.5000 22.125 14.0 7.5625 85.5\n",
      "Epoch 3/100 \t Train Err: 41.7500 22.5 14.3125 7.75 85.5\n",
      "Epoch 3/100 \t Train Err: 41.2500 22.875 14.5625 7.9375 84.5\n",
      "Epoch 3/100 \t Train Err: 41.2500 23.25 14.875 8.1875 83.5\n",
      "Epoch 3/100 \t Train Err: 41.5000 23.5 15.1875 8.4375 83.5\n",
      "Epoch 3/100 \t Train Err: 41.2500 23.75 15.4375 8.625 82.0\n",
      "Epoch 4/100 \t Train Err: 41.0000 24.125 15.75 8.8125 81.0\n",
      "Epoch 4/100 \t Train Err: 40.7500 24.375 16.0 9.0625 81.0\n",
      "Epoch 4/100 \t Train Err: 40.7500 24.5 16.25 9.25 80.5\n",
      "Epoch 4/100 \t Train Err: 40.7500 24.625 16.5 9.4375 79.5\n",
      "Epoch 4/100 \t Train Err: 40.5000 24.75 16.75 9.625 79.0\n",
      "Epoch 4/100 \t Train Err: 40.5000 24.625 16.875 9.75 79.0\n",
      "Epoch 4/100 \t Train Err: 40.2500 24.375 17.125 9.875 78.5\n",
      "Epoch 4/100 \t Train Err: 40.0000 23.75 17.125 10.0 78.0\n",
      "Epoch 5/100 \t Train Err: 39.7500 23.0 17.125 10.0625 77.5\n",
      "Epoch 5/100 \t Train Err: 39.5000 21.5 17.0 10.0 78.0\n",
      "Epoch 5/100 \t Train Err: 38.7500 19.375 16.75 9.875 78.0\n",
      "Epoch 5/100 \t Train Err: 38.5000 16.5 16.25 9.6875 78.5\n",
      "Epoch 5/100 \t Train Err: 37.5000 12.9375 15.625 9.375 79.0\n",
      "Epoch 5/100 \t Train Err: 36.5000 8.875 14.9375 9.125 80.0\n",
      "Epoch 5/100 \t Train Err: 35.5000 5.09375 14.6875 9.25 79.5\n",
      "Epoch 5/100 \t Train Err: 34.5000 2.390625 15.5 10.0 78.0\n",
      "Epoch 6/100 \t Train Err: 33.5000 0.9140625 17.5 11.3125 75.0\n",
      "Epoch 6/100 \t Train Err: 33.0000 0.38671875 19.875 12.4375 72.5\n",
      "Epoch 6/100 \t Train Err: 32.7500 0.4921875 21.0 12.9375 71.5\n",
      "Epoch 6/100 \t Train Err: 33.0000 0.85546875 21.375 13.0 71.0\n",
      "Epoch 6/100 \t Train Err: 33.0000 1.1328125 21.5 13.125 70.5\n",
      "Epoch 6/100 \t Train Err: 32.7500 1.1875 21.875 13.4375 69.5\n",
      "Epoch 6/100 \t Train Err: 32.5000 1.0234375 22.5 13.9375 68.5\n",
      "Epoch 6/100 \t Train Err: 32.2500 0.73828125 23.125 14.5 67.5\n",
      "Epoch 7/100 \t Train Err: 31.8750 0.451171875 23.875 15.0625 66.0\n",
      "Epoch 7/100 \t Train Err: 31.6250 0.251953125 24.625 15.625 64.5\n",
      "Epoch 7/100 \t Train Err: 31.5000 0.2060546875 25.25 16.125 63.75\n",
      "Epoch 7/100 \t Train Err: 31.2500 0.2734375 25.625 16.5 63.0\n",
      "Epoch 7/100 \t Train Err: 31.1250 0.37109375 26.125 17.0 62.25\n",
      "Epoch 7/100 \t Train Err: 30.8750 0.400390625 26.625 17.25 61.5\n",
      "Epoch 7/100 \t Train Err: 30.8750 0.353515625 26.875 17.5 61.0\n",
      "Epoch 7/100 \t Train Err: 30.7500 0.275390625 27.25 17.75 60.5\n",
      "Epoch 8/100 \t Train Err: 30.6250 0.18359375 27.625 18.125 59.75\n",
      "Epoch 8/100 \t Train Err: 30.5000 0.10986328125 28.125 18.625 59.0\n",
      "Epoch 8/100 \t Train Err: 30.3750 0.06640625 28.625 19.0 58.5\n",
      "Epoch 8/100 \t Train Err: 30.3750 0.04931640625 29.125 19.375 57.75\n",
      "Epoch 8/100 \t Train Err: 30.1250 0.048583984375 29.75 19.875 57.0\n",
      "Epoch 8/100 \t Train Err: 30.0000 0.054443359375 30.25 20.25 56.0\n",
      "Epoch 8/100 \t Train Err: 29.8750 0.0576171875 30.875 20.875 55.25\n",
      "Epoch 8/100 \t Train Err: 29.8750 0.056884765625 31.5 21.25 54.5\n",
      "Epoch 9/100 \t Train Err: 29.7500 0.051025390625 32.0 21.75 53.75\n",
      "Epoch 9/100 \t Train Err: 29.5000 0.04296875 32.75 22.25 53.0\n",
      "Epoch 9/100 \t Train Err: 29.5000 0.03369140625 33.0 22.625 52.25\n",
      "Epoch 9/100 \t Train Err: 29.5000 0.0260009765625 33.75 23.125 51.75\n",
      "Epoch 9/100 \t Train Err: 29.3750 0.02197265625 34.25 23.5 51.25\n",
      "Epoch 9/100 \t Train Err: 29.3750 0.0216064453125 35.0 24.125 50.25\n",
      "Epoch 9/100 \t Train Err: 29.2500 0.0238037109375 35.25 24.375 50.0\n",
      "Epoch 9/100 \t Train Err: 29.1250 0.02734375 35.75 24.75 49.5\n",
      "Epoch 10/100 \t Train Err: 29.1250 0.0301513671875 36.0 25.0 49.0\n",
      "Epoch 10/100 \t Train Err: 29.1250 0.032470703125 36.75 25.625 48.25\n",
      "Epoch 10/100 \t Train Err: 29.0000 0.03271484375 37.25 26.125 47.5\n",
      "Epoch 10/100 \t Train Err: 28.8750 0.03125 37.5 26.25 47.25\n",
      "Epoch 10/100 \t Train Err: 29.0000 0.027587890625 38.0 26.75 46.5\n",
      "Epoch 10/100 \t Train Err: 28.8750 0.023193359375 38.25 26.875 46.5\n",
      "Epoch 10/100 \t Train Err: 28.8750 0.0196533203125 38.25 26.875 46.5\n",
      "Epoch 10/100 \t Train Err: 28.7500 0.0172119140625 38.75 27.375 45.75\n",
      "Epoch 11/100 \t Train Err: 28.7500 0.0166015625 39.0 27.5 45.5\n",
      "Epoch 11/100 \t Train Err: 28.8750 0.0169677734375 39.0 27.5 45.5\n",
      "Epoch 11/100 \t Train Err: 28.7500 0.0172119140625 39.0 27.5 45.5\n",
      "Epoch 11/100 \t Train Err: 28.7500 0.017578125 39.75 28.25 44.75\n",
      "Epoch 11/100 \t Train Err: 28.7500 0.017578125 39.75 28.25 44.75\n",
      "Epoch 11/100 \t Train Err: 28.7500 0.017333984375 39.75 28.25 44.75\n",
      "Epoch 11/100 \t Train Err: 28.7500 0.016845703125 39.75 28.25 44.75\n",
      "Epoch 11/100 \t Train Err: 28.7500 0.016357421875 39.75 28.25 44.75\n",
      "Epoch 12/100 \t Train Err: 28.7500 0.015869140625 40.0 28.5 44.25\n",
      "Epoch 12/100 \t Train Err: 28.7500 0.01513671875 40.75 28.875 44.0\n",
      "Epoch 12/100 \t Train Err: 28.7500 0.01483154296875 40.75 28.875 44.0\n",
      "Epoch 12/100 \t Train Err: 28.8750 0.01416015625 40.75 28.875 44.0\n",
      "Epoch 12/100 \t Train Err: 28.7500 0.0140380859375 40.75 28.875 44.0\n",
      "Epoch 12/100 \t Train Err: 28.7500 0.01397705078125 40.75 28.875 44.0\n",
      "Epoch 12/100 \t Train Err: 28.7500 0.0140380859375 40.75 28.875 44.0\n",
      "Epoch 12/100 \t Train Err: 28.6250 0.01422119140625 40.75 29.0 43.75\n",
      "Epoch 13/100 \t Train Err: 28.6250 0.01422119140625 41.0 29.375 43.25\n",
      "Epoch 13/100 \t Train Err: 28.7500 0.01416015625 41.5 29.5 43.0\n",
      "Epoch 13/100 \t Train Err: 28.7500 0.0142822265625 41.5 29.625 43.0\n",
      "Epoch 13/100 \t Train Err: 28.6250 0.01446533203125 41.5 29.625 43.0\n",
      "Epoch 13/100 \t Train Err: 28.6250 0.01422119140625 41.5 29.625 43.0\n",
      "Epoch 13/100 \t Train Err: 28.6250 0.013916015625 41.5 29.625 43.0\n",
      "Epoch 13/100 \t Train Err: 28.6250 0.01373291015625 41.5 29.625 43.0\n",
      "Epoch 13/100 \t Train Err: 28.6250 0.0135498046875 41.5 29.625 43.0\n",
      "Epoch 14/100 \t Train Err: 28.6250 0.01318359375 41.5 29.625 43.0\n",
      "Epoch 14/100 \t Train Err: 28.5000 0.012939453125 41.5 29.625 42.75\n",
      "Epoch 14/100 \t Train Err: 28.6250 0.01275634765625 41.75 29.875 42.5\n",
      "Epoch 14/100 \t Train Err: 28.6250 0.012451171875 42.0 30.125 42.5\n",
      "Epoch 14/100 \t Train Err: 28.6250 0.01220703125 42.25 30.25 42.25\n",
      "Epoch 14/100 \t Train Err: 28.6250 0.01226806640625 42.25 30.25 42.25\n",
      "Epoch 14/100 \t Train Err: 28.6250 0.01190185546875 42.25 30.25 42.25\n",
      "Epoch 14/100 \t Train Err: 28.6250 0.01190185546875 42.25 30.25 42.25\n",
      "Epoch 15/100 \t Train Err: 28.7500 0.0118408203125 42.25 30.25 42.25\n",
      "Epoch 15/100 \t Train Err: 28.6250 0.0115966796875 42.25 30.25 42.25\n",
      "Epoch 15/100 \t Train Err: 28.6250 0.0115966796875 42.25 30.25 42.25\n",
      "Epoch 15/100 \t Train Err: 28.6250 0.01141357421875 42.25 30.25 42.25\n",
      "Epoch 15/100 \t Train Err: 28.6250 0.011474609375 42.25 30.25 42.0\n",
      "Epoch 15/100 \t Train Err: 28.6250 0.01123046875 42.25 30.375 42.0\n",
      "Epoch 15/100 \t Train Err: 28.5000 0.0111083984375 42.5 30.625 41.75\n",
      "Epoch 15/100 \t Train Err: 28.6250 0.010986328125 42.5 30.75 41.75\n",
      "Epoch 16/100 \t Train Err: 28.6250 0.01104736328125 42.75 30.875 41.5\n",
      "Epoch 16/100 \t Train Err: 28.6250 0.01092529296875 42.75 30.875 41.5\n",
      "Epoch 16/100 \t Train Err: 28.6250 0.0107421875 43.0 31.0 41.5\n",
      "Epoch 16/100 \t Train Err: 28.6250 0.0107421875 43.0 31.0 41.5\n",
      "Epoch 16/100 \t Train Err: 28.6250 0.01068115234375 43.0 31.0 41.5\n",
      "Epoch 16/100 \t Train Err: 28.6250 0.01043701171875 43.0 31.0 41.5\n",
      "Epoch 16/100 \t Train Err: 28.6250 0.0103759765625 43.0 31.0 41.5\n",
      "Epoch 16/100 \t Train Err: 28.6250 0.01025390625 43.0 31.0 41.5\n",
      "Epoch 17/100 \t Train Err: 28.6250 0.0101318359375 43.0 31.0 41.5\n",
      "Epoch 17/100 \t Train Err: 28.6250 0.0098876953125 43.0 31.0 41.5\n",
      "Epoch 17/100 \t Train Err: 28.6250 0.00982666015625 43.0 31.0 41.5\n",
      "Epoch 17/100 \t Train Err: 28.6250 0.009765625 43.0 31.0 41.5\n",
      "Epoch 17/100 \t Train Err: 28.6250 0.00958251953125 43.0 31.0 41.5\n",
      "Epoch 17/100 \t Train Err: 28.6250 0.00946044921875 43.0 31.0 41.5\n",
      "Epoch 17/100 \t Train Err: 28.5000 0.0093994140625 43.0 31.0 41.5\n",
      "Epoch 17/100 \t Train Err: 28.6250 0.0091552734375 43.0 31.0 41.5\n",
      "Epoch 18/100 \t Train Err: 28.6250 0.00897216796875 43.0 31.0 41.25\n",
      "Epoch 18/100 \t Train Err: 28.5000 0.0089111328125 43.0 31.0 41.25\n",
      "Epoch 18/100 \t Train Err: 28.3750 0.00885009765625 43.0 31.0 41.25\n",
      "Epoch 18/100 \t Train Err: 28.3750 0.0087890625 43.0 31.125 41.25\n",
      "Epoch 18/100 \t Train Err: 28.6250 0.0086669921875 43.0 31.125 41.25\n",
      "Epoch 18/100 \t Train Err: 28.5000 0.008544921875 43.0 31.125 41.25\n",
      "Epoch 18/100 \t Train Err: 28.5000 0.00836181640625 43.0 31.125 41.25\n",
      "Epoch 18/100 \t Train Err: 28.5000 0.0081787109375 43.0 31.125 41.25\n",
      "Epoch 19/100 \t Train Err: 28.3750 0.0079345703125 43.0 31.125 41.25\n",
      "Epoch 19/100 \t Train Err: 28.5000 0.0078125 43.0 31.125 41.25\n",
      "Epoch 19/100 \t Train Err: 28.5000 0.007781982421875 43.0 31.0 41.25\n",
      "Epoch 19/100 \t Train Err: 28.5000 0.00750732421875 43.0 31.0 41.25\n",
      "Epoch 19/100 \t Train Err: 28.5000 0.00738525390625 42.75 30.875 41.25\n",
      "Epoch 19/100 \t Train Err: 28.5000 0.00714111328125 42.5 30.75 41.5\n",
      "Epoch 19/100 \t Train Err: 28.3750 0.006866455078125 42.25 30.5 41.5\n",
      "Epoch 19/100 \t Train Err: 28.3750 0.0067138671875 41.75 30.125 42.0\n",
      "Epoch 20/100 \t Train Err: 28.2500 0.006591796875 40.5 29.25 42.5\n",
      "Epoch 20/100 \t Train Err: 28.1250 0.00634765625 37.5 27.125 44.5\n",
      "Epoch 20/100 \t Train Err: 27.8750 0.0067138671875 27.75 19.875 52.0\n",
      "Epoch 20/100 \t Train Err: 27.8750 0.0040283203125 25.875 18.5 53.5\n",
      "Epoch 20/100 \t Train Err: 27.7500 0.011962890625 34.0 24.5 46.5\n",
      "Epoch 20/100 \t Train Err: 27.8750 0.0240478515625 36.5 26.125 44.75\n",
      "Epoch 20/100 \t Train Err: 27.6250 0.0267333984375 35.5 25.5 45.0\n",
      "Epoch 20/100 \t Train Err: 27.2500 0.016357421875 30.125 21.5 48.5\n",
      "Epoch 21/100 \t Train Err: 27.5000 0.005279541015625 19.5 13.5 57.5\n",
      "Epoch 21/100 \t Train Err: 26.8750 0.00982666015625 28.875 20.875 48.25\n",
      "Epoch 21/100 \t Train Err: 26.7500 0.01019287109375 32.5 23.875 45.0\n",
      "Epoch 21/100 \t Train Err: 26.5000 0.0057373046875 27.75 20.625 47.5\n",
      "Epoch 21/100 \t Train Err: 26.5000 0.0111083984375 14.0 10.375 58.5\n",
      "Epoch 21/100 \t Train Err: 25.7500 0.007110595703125 27.625 21.875 45.0\n",
      "Epoch 21/100 \t Train Err: 25.3750 0.0081787109375 27.625 22.25 44.25\n",
      "Epoch 21/100 \t Train Err: 24.7500 0.0101318359375 11.4375 9.5 55.5\n",
      "Epoch 22/100 \t Train Err: 23.7500 0.0091552734375 14.8125 12.625 50.0\n",
      "Epoch 22/100 \t Train Err: 23.3750 0.0196533203125 18.5 16.5 45.5\n",
      "Epoch 22/100 \t Train Err: 22.8750 0.0205078125 9.5625 8.25 52.0\n",
      "Epoch 22/100 \t Train Err: 22.3750 0.045654296875 9.1875 7.90625 50.75\n",
      "Epoch 22/100 \t Train Err: 22.2500 0.1318359375 15.375 13.8125 45.0\n",
      "Epoch 22/100 \t Train Err: 21.6250 0.150390625 11.4375 9.5625 47.25\n",
      "Epoch 22/100 \t Train Err: 21.3750 0.126953125 8.4375 6.34375 49.75\n",
      "Epoch 22/100 \t Train Err: 20.8750 0.1455078125 11.0625 8.75 46.0\n",
      "Epoch 23/100 \t Train Err: 20.6250 0.125 13.6875 11.4375 43.0\n",
      "Epoch 23/100 \t Train Err: 20.3750 0.04931640625 11.625 9.625 44.0\n",
      "Epoch 23/100 \t Train Err: 20.0000 0.033935546875 9.3125 7.6875 45.25\n",
      "Epoch 23/100 \t Train Err: 19.6250 0.07275390625 10.0625 8.875 43.25\n",
      "Epoch 23/100 \t Train Err: 19.5000 0.1181640625 11.5625 10.9375 41.0\n",
      "Epoch 23/100 \t Train Err: 19.0000 0.1787109375 11.0 11.1875 40.25\n",
      "Epoch 23/100 \t Train Err: 18.7500 0.25 8.1875 8.9375 41.75\n",
      "Epoch 23/100 \t Train Err: 18.5000 0.2216796875 8.1875 9.9375 40.25\n",
      "Epoch 24/100 \t Train Err: 18.1250 0.1513671875 10.0625 13.4375 37.0\n",
      "Epoch 24/100 \t Train Err: 17.6250 0.12890625 7.6875 11.6875 37.75\n",
      "Epoch 24/100 \t Train Err: 17.3750 0.1201171875 6.28125 10.875 38.0\n",
      "Epoch 24/100 \t Train Err: 17.0000 0.126953125 7.53125 14.5625 35.0\n",
      "Epoch 24/100 \t Train Err: 16.7500 0.11181640625 7.3125 15.6875 33.75\n",
      "Epoch 24/100 \t Train Err: 16.5000 0.08203125 4.75 13.3125 35.5\n",
      "Epoch 24/100 \t Train Err: 16.3750 0.068359375 5.75 17.125 32.75\n",
      "Epoch 24/100 \t Train Err: 15.9375 0.057861328125 6.34375 19.25 30.75\n",
      "Epoch 25/100 \t Train Err: 15.7500 0.051025390625 3.578125 14.5 33.5\n",
      "Epoch 25/100 \t Train Err: 15.2500 0.04248046875 5.0 18.625 29.625\n",
      "Epoch 25/100 \t Train Err: 15.0000 0.040771484375 5.53125 21.125 27.875\n",
      "Epoch 25/100 \t Train Err: 14.8125 0.033935546875 3.171875 16.0 30.375\n",
      "Epoch 25/100 \t Train Err: 14.6250 0.0322265625 3.734375 18.875 28.5\n",
      "Epoch 25/100 \t Train Err: 14.3750 0.03369140625 5.09375 23.0 25.5\n",
      "Epoch 25/100 \t Train Err: 14.1250 0.028076171875 2.046875 14.3125 30.125\n",
      "Epoch 25/100 \t Train Err: 13.8125 0.023681640625 3.234375 19.375 26.625\n",
      "Epoch 26/100 \t Train Err: 13.6875 0.023681640625 4.75 24.875 23.125\n",
      "Epoch 26/100 \t Train Err: 13.5625 0.0245361328125 1.515625 13.6875 29.0\n",
      "Epoch 26/100 \t Train Err: 13.0625 0.0179443359375 2.875 20.5 24.25\n",
      "Epoch 26/100 \t Train Err: 13.0000 0.016845703125 3.5 24.0 22.25\n",
      "Epoch 26/100 \t Train Err: 12.8750 0.02197265625 1.46875 15.625 26.5\n",
      "Epoch 26/100 \t Train Err: 12.5000 0.0174560546875 2.03125 19.5 23.5\n",
      "Epoch 26/100 \t Train Err: 12.4375 0.014404296875 3.0 24.75 20.625\n",
      "Epoch 26/100 \t Train Err: 12.1250 0.0230712890625 1.46875 17.625 23.875\n",
      "Epoch 27/100 \t Train Err: 11.9375 0.022705078125 1.421875 17.75 23.125\n",
      "Epoch 27/100 \t Train Err: 11.7500 0.0150146484375 2.09375 22.625 20.0\n",
      "Epoch 27/100 \t Train Err: 11.6250 0.01531982421875 1.6796875 20.875 20.75\n",
      "Epoch 27/100 \t Train Err: 11.3750 0.0177001953125 1.0546875 17.25 22.0\n",
      "Epoch 27/100 \t Train Err: 11.0625 0.0128173828125 1.359375 20.375 19.875\n",
      "Epoch 27/100 \t Train Err: 11.0000 0.0128173828125 1.5078125 22.0 18.875\n",
      "Epoch 27/100 \t Train Err: 10.8125 0.01190185546875 1.03125 18.125 20.125\n",
      "Epoch 27/100 \t Train Err: 10.7500 0.01165771484375 0.99609375 18.125 20.25\n",
      "Epoch 28/100 \t Train Err: 10.5625 0.012451171875 1.328125 21.125 18.125\n",
      "Epoch 28/100 \t Train Err: 10.3750 0.01104736328125 1.15625 19.375 18.625\n",
      "Epoch 28/100 \t Train Err: 10.3125 0.01025390625 0.953125 17.25 19.5\n",
      "Epoch 28/100 \t Train Err: 10.1250 0.010498046875 1.171875 19.875 17.75\n",
      "Epoch 28/100 \t Train Err: 10.0625 0.0101318359375 1.109375 20.0 17.5\n",
      "Epoch 28/100 \t Train Err: 10.0000 0.0111083984375 0.7578125 16.75 18.875\n",
      "Epoch 28/100 \t Train Err: 9.8125 0.0093994140625 0.87109375 18.375 17.5\n",
      "Epoch 28/100 \t Train Err: 9.7500 0.01043701171875 1.0390625 20.625 16.375\n",
      "Epoch 29/100 \t Train Err: 9.4375 0.00921630859375 0.828125 18.5 16.75\n",
      "Epoch 29/100 \t Train Err: 9.5000 0.00836181640625 0.59375 16.5 17.75\n",
      "Epoch 29/100 \t Train Err: 9.4375 0.0115966796875 0.796875 20.375 15.8125\n",
      "Epoch 29/100 \t Train Err: 9.3125 0.010986328125 0.72265625 20.25 15.625\n",
      "Epoch 29/100 \t Train Err: 9.1875 0.00762939453125 0.51953125 16.75 16.875\n",
      "Epoch 29/100 \t Train Err: 9.0625 0.00799560546875 0.56640625 18.375 15.8125\n",
      "Epoch 29/100 \t Train Err: 9.0625 0.00946044921875 0.66796875 20.625 14.625\n",
      "Epoch 29/100 \t Train Err: 9.0000 0.00665283203125 0.46484375 16.125 16.5\n",
      "Epoch 30/100 \t Train Err: 8.8750 0.008056640625 0.5234375 18.25 15.3125\n",
      "Epoch 30/100 \t Train Err: 8.7500 0.0111083984375 0.59375 20.0 14.1875\n",
      "Epoch 36/100 \t Train Err: 7.3750 0.00799560546875 0.302734375 13.5 12.9375\n",
      "Epoch 36/100 \t Train Err: 7.2188 0.00799560546875 0.369140625 15.5625 11.375\n",
      "Epoch 36/100 \t Train Err: 7.2188 0.00823974609375 0.4296875 17.375 10.375\n",
      "Epoch 36/100 \t Train Err: 7.2500 0.00860595703125 0.412109375 18.0 10.125\n",
      "Epoch 36/100 \t Train Err: 7.1875 0.01171875 0.33984375 15.625 11.1875\n",
      "Epoch 36/100 \t Train Err: 7.0625 0.0177001953125 0.2890625 12.875 12.1875\n",
      "Epoch 36/100 \t Train Err: 7.1562 0.01806640625 0.271484375 11.8125 13.0\n",
      "Epoch 36/100 \t Train Err: 7.1875 0.0120849609375 0.24609375 11.5625 13.0625\n",
      "Epoch 37/100 \t Train Err: 7.0625 0.007171630859375 0.2431640625 12.375 12.3125\n",
      "Epoch 37/100 \t Train Err: 7.0625 0.0101318359375 0.2490234375 14.0 11.5\n",
      "Epoch 37/100 \t Train Err: 7.0625 0.0181884765625 0.28125 15.4375 10.875\n",
      "Epoch 37/100 \t Train Err: 7.0938 0.0244140625 0.287109375 15.8125 10.6875\n",
      "Epoch 37/100 \t Train Err: 6.9375 0.0230712890625 0.27734375 15.1875 10.625\n",
      "Epoch 37/100 \t Train Err: 6.8750 0.01556396484375 0.255859375 13.5625 11.4375\n",
      "Epoch 37/100 \t Train Err: 6.9375 0.0091552734375 0.220703125 12.4375 12.0625\n",
      "Epoch 37/100 \t Train Err: 6.9375 0.006011962890625 0.2158203125 12.4375 12.0625\n",
      "Epoch 38/100 \t Train Err: 6.8438 0.004791259765625 0.232421875 12.9375 11.625\n",
      "Epoch 38/100 \t Train Err: 6.8750 0.004486083984375 0.2421875 14.3125 10.875\n",
      "Epoch 38/100 \t Train Err: 6.8438 0.00433349609375 0.28125 15.0625 10.1875\n",
      "Epoch 38/100 \t Train Err: 6.9375 0.004241943359375 0.25 14.9375 10.4375\n",
      "Epoch 38/100 \t Train Err: 6.7500 0.004241943359375 0.23828125 13.75 10.625\n",
      "Epoch 38/100 \t Train Err: 6.7812 0.0042724609375 0.2109375 12.25 11.5625\n",
      "Epoch 38/100 \t Train Err: 6.8438 0.003692626953125 0.1943359375 11.9375 11.875\n",
      "Epoch 38/100 \t Train Err: 6.6875 0.004119873046875 0.197265625 11.5 11.5\n",
      "Epoch 39/100 \t Train Err: 6.6562 0.007720947265625 0.193359375 12.1875 11.0625\n",
      "Epoch 39/100 \t Train Err: 6.6250 0.01318359375 0.2080078125 13.25 10.4375\n",
      "Epoch 39/100 \t Train Err: 6.6562 0.016357421875 0.224609375 13.9375 10.3125\n",
      "Epoch 39/100 \t Train Err: 6.6562 0.0159912109375 0.2021484375 13.75 10.375\n",
      "Epoch 39/100 \t Train Err: 6.5312 0.0126953125 0.19140625 12.9375 10.5\n",
      "Epoch 39/100 \t Train Err: 6.5938 0.0081787109375 0.1796875 11.9375 11.0625\n",
      "Epoch 39/100 \t Train Err: 6.6250 0.005401611328125 0.1796875 11.875 11.375\n",
      "Epoch 39/100 \t Train Err: 6.5000 0.0040283203125 0.1787109375 12.125 10.9375\n",
      "Epoch 40/100 \t Train Err: 6.5312 0.0031890869140625 0.1962890625 12.8125 10.5625\n",
      "Epoch 40/100 \t Train Err: 6.5625 0.0029296875 0.2080078125 13.25 10.3125\n",
      "Epoch 40/100 \t Train Err: 6.5625 0.0026702880859375 0.189453125 13.5 10.25\n",
      "Epoch 40/100 \t Train Err: 6.5312 0.002685546875 0.177734375 12.5625 10.4375\n",
      "Epoch 40/100 \t Train Err: 6.4375 0.0027008056640625 0.169921875 11.625 10.8125\n",
      "Epoch 40/100 \t Train Err: 6.5000 0.0026092529296875 0.1630859375 11.6875 11.0625\n",
      "Epoch 40/100 \t Train Err: 6.5000 0.0030670166015625 0.162109375 11.9375 10.875\n",
      "Epoch 40/100 \t Train Err: 6.5000 0.004486083984375 0.1630859375 12.4375 10.5625\n",
      "Epoch 41/100 \t Train Err: 6.4375 0.006011962890625 0.1875 13.3125 9.9375\n",
      "Epoch 41/100 \t Train Err: 6.4688 0.005706787109375 0.1708984375 12.75 10.25\n",
      "Epoch 41/100 \t Train Err: 6.4688 0.00445556640625 0.15234375 12.25 10.625\n",
      "Epoch 41/100 \t Train Err: 6.4688 0.0032501220703125 0.166015625 11.8125 10.875\n",
      "Epoch 41/100 \t Train Err: 6.3750 0.0027008056640625 0.166015625 12.0625 10.5\n",
      "Epoch 41/100 \t Train Err: 6.3125 0.0023040771484375 0.158203125 12.0 10.25\n",
      "Epoch 41/100 \t Train Err: 6.4062 0.002227783203125 0.1640625 12.4375 10.125\n",
      "Epoch 41/100 \t Train Err: 6.3438 0.002227783203125 0.171875 12.6875 9.9375\n",
      "Epoch 42/100 \t Train Err: 6.3125 0.002197265625 0.1591796875 12.0625 10.1875\n",
      "Epoch 42/100 \t Train Err: 6.2500 0.0021209716796875 0.1513671875 11.4375 10.3125\n",
      "Epoch 42/100 \t Train Err: 6.2812 0.0022430419921875 0.1396484375 11.5 10.5\n",
      "Epoch 42/100 \t Train Err: 6.1875 0.002838134765625 0.146484375 11.8125 9.9375\n",
      "Epoch 42/100 \t Train Err: 6.3125 0.0037078857421875 0.150390625 12.125 10.0625\n",
      "Epoch 42/100 \t Train Err: 6.2812 0.004425048828125 0.1591796875 12.375 9.875\n",
      "Epoch 42/100 \t Train Err: 6.2188 0.004150390625 0.1357421875 11.625 10.0625\n",
      "Epoch 42/100 \t Train Err: 6.2188 0.0035858154296875 0.1416015625 11.4375 10.25\n",
      "Epoch 43/100 \t Train Err: 6.2500 0.0028839111328125 0.1328125 11.1875 10.4375\n",
      "Epoch 43/100 \t Train Err: 6.1562 0.0025482177734375 0.13671875 11.1875 10.125\n",
      "Epoch 43/100 \t Train Err: 6.0938 0.002227783203125 0.142578125 11.625 9.75\n",
      "Epoch 43/100 \t Train Err: 6.1875 0.002105712890625 0.1435546875 12.0625 9.8125\n",
      "Epoch 43/100 \t Train Err: 6.2812 0.001983642578125 0.150390625 12.125 9.9375\n",
      "Epoch 43/100 \t Train Err: 6.0938 0.0019683837890625 0.1396484375 11.5 9.8125\n",
      "Epoch 43/100 \t Train Err: 6.2188 0.00191497802734375 0.1337890625 11.5 10.125\n",
      "Epoch 43/100 \t Train Err: 6.0938 0.00201416015625 0.1337890625 11.4375 9.875\n",
      "Epoch 44/100 \t Train Err: 6.0938 0.0023040771484375 0.140625 11.375 9.9375\n",
      "Epoch 44/100 \t Train Err: 6.0938 0.002960205078125 0.1298828125 11.125 10.0\n",
      "Epoch 44/100 \t Train Err: 6.1562 0.003662109375 0.1357421875 11.375 10.0\n",
      "Epoch 44/100 \t Train Err: 6.0625 0.003997802734375 0.130859375 11.4375 9.75\n",
      "Epoch 44/100 \t Train Err: 6.0312 0.003997802734375 0.134765625 11.4375 9.5625\n",
      "Epoch 44/100 \t Train Err: 6.0000 0.003265380859375 0.1337890625 11.4375 9.625\n",
      "Epoch 44/100 \t Train Err: 6.0000 0.0024871826171875 0.1337890625 11.5 9.625\n",
      "Epoch 44/100 \t Train Err: 6.0312 0.0020904541015625 0.1376953125 11.0625 9.8125\n",
      "Epoch 45/100 \t Train Err: 5.9688 0.0020294189453125 0.125 10.8125 9.6875\n",
      "Epoch 45/100 \t Train Err: 6.0000 0.0019683837890625 0.1318359375 10.75 9.75\n",
      "Epoch 45/100 \t Train Err: 6.0312 0.002044677734375 0.12890625 11.0625 9.6875\n",
      "Epoch 45/100 \t Train Err: 5.9688 0.002197265625 0.12353515625 11.0625 9.6875\n",
      "Epoch 45/100 \t Train Err: 5.8750 0.0026397705078125 0.1318359375 11.3125 9.375\n",
      "Epoch 45/100 \t Train Err: 5.9375 0.003204345703125 0.1201171875 11.25 9.4375\n",
      "Epoch 45/100 \t Train Err: 5.8125 0.003326416015625 0.115234375 11.0 9.375\n",
      "Epoch 45/100 \t Train Err: 5.9062 0.0030975341796875 0.111328125 11.0 9.5625\n",
      "Epoch 46/100 \t Train Err: 5.9062 0.0026702880859375 0.10498046875 10.5 9.75\n",
      "Epoch 46/100 \t Train Err: 5.8125 0.0024566650390625 0.1044921875 10.375 9.625\n",
      "Epoch 46/100 \t Train Err: 5.8438 0.0024566650390625 0.11474609375 10.875 9.5\n",
      "Epoch 46/100 \t Train Err: 5.8438 0.0023956298828125 0.11962890625 11.375 9.1875\n",
      "Epoch 46/100 \t Train Err: 5.7812 0.0023651123046875 0.12060546875 11.125 9.125\n",
      "Epoch 46/100 \t Train Err: 5.9062 0.0023193359375 0.11767578125 10.875 9.5625\n",
      "Epoch 46/100 \t Train Err: 5.7500 0.002349853515625 0.09912109375 10.25 9.5625\n",
      "Epoch 46/100 \t Train Err: 5.7812 0.0024871826171875 0.10986328125 9.9375 9.8125\n",
      "Epoch 47/100 \t Train Err: 5.7812 0.002960205078125 0.107421875 10.5 9.375\n",
      "Epoch 47/100 \t Train Err: 5.7812 0.0032501220703125 0.1123046875 10.875 9.1875\n",
      "Epoch 47/100 \t Train Err: 5.7188 0.0033111572265625 0.11767578125 10.5625 9.125\n",
      "Epoch 47/100 \t Train Err: 5.7812 0.0030517578125 0.10986328125 10.4375 9.5\n",
      "Epoch 47/100 \t Train Err: 5.6562 0.002899169921875 0.1181640625 10.625 9.0\n",
      "Epoch 47/100 \t Train Err: 5.6562 0.0026702880859375 0.12109375 11.0 8.875\n",
      "Epoch 47/100 \t Train Err: 5.7812 0.00262451171875 0.10302734375 10.4375 9.375\n",
      "Epoch 47/100 \t Train Err: 5.7812 0.0026702880859375 0.1015625 9.8125 9.75\n",
      "Epoch 48/100 \t Train Err: 5.7188 0.002655029296875 0.09814453125 9.9375 9.4375\n",
      "Epoch 48/100 \t Train Err: 5.6562 0.003143310546875 0.111328125 10.75 8.875\n",
      "Epoch 48/100 \t Train Err: 5.5625 0.00335693359375 0.111328125 10.6875 8.75\n",
      "Epoch 48/100 \t Train Err: 5.5625 0.003326416015625 0.1044921875 9.8125 9.1875\n",
      "Epoch 48/100 \t Train Err: 5.6562 0.003265380859375 0.099609375 10.125 9.25\n",
      "Epoch 48/100 \t Train Err: 5.6875 0.0030670166015625 0.10888671875 10.875 8.875\n",
      "Epoch 48/100 \t Train Err: 5.5938 0.0027923583984375 0.09619140625 10.25 8.9375\n",
      "Epoch 48/100 \t Train Err: 5.5938 0.0027313232421875 0.10400390625 9.4375 9.4375\n",
      "Epoch 49/100 \t Train Err: 5.5312 0.002777099609375 0.09326171875 10.0 8.9375\n",
      "Epoch 49/100 \t Train Err: 5.5938 0.0031890869140625 0.1015625 10.9375 8.5\n",
      "Epoch 49/100 \t Train Err: 5.5000 0.00341796875 0.08984375 10.25 8.8125\n",
      "Epoch 49/100 \t Train Err: 5.5938 0.003662109375 0.07666015625 8.8125 9.75\n",
      "Epoch 49/100 \t Train Err: 5.5312 0.004547119140625 0.09375 9.6875 9.0625\n",
      "Epoch 49/100 \t Train Err: 5.5000 0.004638671875 0.1103515625 11.125 8.125\n",
      "Epoch 49/100 \t Train Err: 5.4375 0.0031890869140625 0.08447265625 8.875 9.3125\n",
      "Epoch 49/100 \t Train Err: 5.5312 0.0031890869140625 0.0908203125 9.3125 9.375\n",
      "Epoch 50/100 \t Train Err: 5.5625 0.003265380859375 0.10693359375 11.75 7.625\n",
      "Epoch 50/100 \t Train Err: 5.5000 0.00341796875 0.07763671875 7.5625 10.1875\n",
      "Epoch 50/100 \t Train Err: 5.3750 0.00323486328125 0.08056640625 8.8125 9.125\n",
      "Epoch 50/100 \t Train Err: 5.4688 0.00433349609375 0.11767578125 11.875 7.46875\n",
      "Epoch 50/100 \t Train Err: 5.5312 0.00433349609375 0.09130859375 8.625 9.75\n",
      "Epoch 50/100 \t Train Err: 5.4062 0.0047607421875 0.087890625 9.3125 9.0\n",
      "Epoch 50/100 \t Train Err: 5.4375 0.00457763671875 0.11767578125 11.875 7.4375\n",
      "Epoch 50/100 \t Train Err: 5.5312 0.003387451171875 0.06640625 6.75 10.8125\n",
      "Epoch 51/100 \t Train Err: 5.4062 0.003448486328125 0.08984375 9.5 8.75\n",
      "Epoch 51/100 \t Train Err: 5.4062 0.003662109375 0.11279296875 12.0 7.40625\n",
      "Epoch 51/100 \t Train Err: 5.3125 0.003692626953125 0.087890625 8.4375 9.1875\n",
      "Epoch 51/100 \t Train Err: 5.3750 0.003692626953125 0.08349609375 8.5625 9.3125\n",
      "Epoch 51/100 \t Train Err: 5.3125 0.0037994384765625 0.09765625 10.8125 7.9375\n",
      "Epoch 51/100 \t Train Err: 5.3750 0.0036773681640625 0.0791015625 9.6875 8.4375\n",
      "Epoch 51/100 \t Train Err: 5.3125 0.0037078857421875 0.06787109375 8.375 9.25\n",
      "Epoch 51/100 \t Train Err: 5.2500 0.003936767578125 0.076171875 9.25 8.625\n",
      "Epoch 52/100 \t Train Err: 5.3125 0.00433349609375 0.08349609375 10.375 8.0\n",
      "Epoch 52/100 \t Train Err: 5.1875 0.00457763671875 0.0732421875 8.8125 8.6875\n",
      "Epoch 52/100 \t Train Err: 5.2188 0.0047607421875 0.06787109375 8.25 8.9375\n",
      "Epoch 52/100 \t Train Err: 5.2812 0.00518798828125 0.08056640625 9.5 8.3125\n",
      "Epoch 52/100 \t Train Err: 5.0938 0.0047607421875 0.08154296875 9.5 7.9375\n",
      "Epoch 52/100 \t Train Err: 5.2188 0.00396728515625 0.06591796875 8.0625 9.1875\n",
      "Epoch 52/100 \t Train Err: 5.1250 0.004180908203125 0.07421875 9.5 8.0\n",
      "Epoch 52/100 \t Train Err: 5.1250 0.004150390625 0.078125 9.5 8.0625\n",
      "Epoch 53/100 \t Train Err: 5.1562 0.004180908203125 0.064453125 8.0 8.875\n",
      "Epoch 53/100 \t Train Err: 5.0312 0.004608154296875 0.0703125 8.3125 8.4375\n",
      "Epoch 53/100 \t Train Err: 5.0625 0.00531005859375 0.07861328125 8.625 8.1875\n",
      "Epoch 53/100 \t Train Err: 5.0312 0.005340576171875 0.07763671875 9.0 8.0625\n",
      "Epoch 53/100 \t Train Err: 5.0312 0.004791259765625 0.07421875 8.4375 8.25\n",
      "Epoch 53/100 \t Train Err: 5.0312 0.00445556640625 0.0673828125 8.3125 8.375\n",
      "Epoch 53/100 \t Train Err: 4.9688 0.004486083984375 0.06591796875 8.625 8.0\n",
      "Epoch 53/100 \t Train Err: 5.0312 0.004486083984375 0.06396484375 8.25 8.4375\n",
      "Epoch 54/100 \t Train Err: 4.9688 0.004425048828125 0.06689453125 8.3125 8.3125\n",
      "Epoch 54/100 \t Train Err: 5.0000 0.00457763671875 0.07470703125 8.75 7.96875\n",
      "Epoch 54/100 \t Train Err: 4.9375 0.004669189453125 0.07080078125 7.96875 8.25\n",
      "Epoch 54/100 \t Train Err: 4.8750 0.004852294921875 0.07275390625 7.90625 8.1875\n",
      "Epoch 54/100 \t Train Err: 4.9062 0.00494384765625 0.0791015625 8.5625 7.8125\n",
      "Epoch 54/100 \t Train Err: 4.9688 0.0045166015625 0.0732421875 7.90625 8.3125\n",
      "Epoch 54/100 \t Train Err: 4.9375 0.0045166015625 0.06689453125 7.625 8.4375\n",
      "Epoch 54/100 \t Train Err: 4.8438 0.004608154296875 0.0791015625 8.625 7.5\n",
      "Epoch 55/100 \t Train Err: 4.9062 0.0045166015625 0.0693359375 7.3125 8.5625\n",
      "Epoch 55/100 \t Train Err: 4.9062 0.00494384765625 0.07958984375 8.3125 7.875\n",
      "Epoch 55/100 \t Train Err: 4.8750 0.00531005859375 0.0849609375 8.6875 7.6875\n",
      "Epoch 55/100 \t Train Err: 4.7812 0.004638671875 0.06494140625 6.8125 8.5625\n",
      "Epoch 55/100 \t Train Err: 4.8438 0.004791259765625 0.08203125 9.0 7.1875\n",
      "Epoch 55/100 \t Train Err: 4.9688 0.004241943359375 0.055419921875 5.40625 10.0\n",
      "Epoch 55/100 \t Train Err: 6.1562 0.005401611328125 0.19140625 20.125 4.1875\n",
      "Epoch 55/100 \t Train Err: 14.0000 0.005706787109375 0.107421875 0.53515625 36.0\n",
      "Epoch 56/100 \t Train Err: 11.1875 0.314453125 0.1689453125 0.56640625 29.0\n",
      "Epoch 56/100 \t Train Err: 7.6250 1.75 3.984375 17.0 9.1875\n",
      "Epoch 56/100 \t Train Err: 12.3125 2.03125 11.1875 47.25 3.21875\n",
      "Epoch 56/100 \t Train Err: 7.0625 0.88671875 1.78125 10.1875 12.5625\n",
      "Epoch 56/100 \t Train Err: 8.7500 0.05078125 0.1953125 2.484375 21.875\n",
      "Epoch 56/100 \t Train Err: 8.8750 0.1328125 0.0419921875 2.5625 22.0\n",
      "Epoch 56/100 \t Train Err: 6.6562 0.32421875 0.0888671875 9.6875 12.8125\n",
      "Epoch 56/100 \t Train Err: 7.5000 0.349609375 0.51171875 28.375 5.90625\n",
      "Epoch 57/100 \t Train Err: 8.4375 0.337890625 0.66796875 35.75 4.53125\n",
      "Epoch 57/100 \t Train Err: 7.1562 0.3125 0.265625 25.375 6.375\n",
      "Epoch 57/100 \t Train Err: 6.3750 0.259765625 0.095703125 10.75 11.5\n",
      "Epoch 57/100 \t Train Err: 7.0625 0.125 0.08837890625 5.75 15.75\n",
      "Epoch 57/100 \t Train Err: 7.1562 0.022705078125 0.06005859375 5.75 16.0\n",
      "Epoch 57/100 \t Train Err: 6.5625 0.1787109375 0.2412109375 8.625 13.0625\n",
      "Epoch 57/100 \t Train Err: 6.4375 0.443359375 0.59765625 14.5 9.375\n",
      "Epoch 57/100 \t Train Err: 6.5625 0.408203125 0.6484375 18.875 7.34375\n",
      "Epoch 58/100 \t Train Err: 6.4062 0.150390625 0.337890625 19.25 6.9375\n",
      "Epoch 58/100 \t Train Err: 6.1875 0.0218505859375 0.11865234375 16.5 7.65625\n",
      "Epoch 58/100 \t Train Err: 5.9688 0.1796875 0.30078125 11.4375 9.4375\n",
      "Epoch 58/100 \t Train Err: 6.1562 0.35546875 0.5703125 8.625 11.0625\n",
      "Epoch 58/100 \t Train Err: 6.0312 0.31640625 0.55859375 8.125 11.375\n",
      "Epoch 58/100 \t Train Err: 5.8125 0.130859375 0.333984375 9.375 10.125\n",
      "Epoch 58/100 \t Train Err: 5.5938 0.01422119140625 0.138671875 12.125 8.4375\n",
      "Epoch 58/100 \t Train Err: 5.6562 0.06884765625 0.1396484375 14.4375 7.28125\n",
      "Epoch 59/100 \t Train Err: 5.6875 0.1884765625 0.20703125 14.5625 7.28125\n",
      "Epoch 59/100 \t Train Err: 5.6562 0.24609375 0.216796875 12.9375 7.96875\n",
      "Epoch 59/100 \t Train Err: 5.4688 0.2109375 0.162109375 10.3125 9.0\n",
      "Epoch 59/100 \t Train Err: 5.4688 0.134765625 0.10888671875 8.6875 9.9375\n",
      "Epoch 59/100 \t Train Err: 5.4375 0.06689453125 0.0966796875 8.125 10.25\n",
      "Epoch 59/100 \t Train Err: 5.2812 0.0262451171875 0.11279296875 8.3125 9.6875\n",
      "Epoch 59/100 \t Train Err: 5.2812 0.0106201171875 0.1416015625 10.375 8.5\n",
      "Epoch 59/100 \t Train Err: 5.3125 0.0084228515625 0.177734375 12.5 7.46875\n",
      "Epoch 60/100 \t Train Err: 5.3125 0.0126953125 0.1875 12.1875 7.28125\n",
      "Epoch 60/100 \t Train Err: 5.2188 0.019775390625 0.1982421875 11.1875 7.5625\n",
      "Epoch 60/100 \t Train Err: 5.1250 0.0240478515625 0.203125 9.3125 8.375\n",
      "Epoch 60/100 \t Train Err: 5.1250 0.019775390625 0.1875 8.25 9.1875\n",
      "Epoch 60/100 \t Train Err: 5.1250 0.010986328125 0.1572265625 7.71875 9.1875\n",
      "Epoch 60/100 \t Train Err: 5.0312 0.007171630859375 0.1259765625 8.375 8.625\n",
      "Epoch 60/100 \t Train Err: 4.9688 0.0150146484375 0.10400390625 9.125 8.0\n",
      "Epoch 60/100 \t Train Err: 4.9375 0.0308837890625 0.09033203125 9.625 7.625\n",
      "Epoch 61/100 \t Train Err: 4.9688 0.046630859375 0.08447265625 9.5625 7.65625\n",
      "Epoch 61/100 \t Train Err: 4.9688 0.0546875 0.08154296875 9.125 7.875\n",
      "Epoch 61/100 \t Train Err: 4.9688 0.054443359375 0.0712890625 8.1875 8.5\n",
      "Epoch 61/100 \t Train Err: 4.9062 0.049072265625 0.07275390625 7.875 8.4375\n",
      "Epoch 61/100 \t Train Err: 4.8125 0.040771484375 0.0693359375 7.71875 8.25\n",
      "Epoch 61/100 \t Train Err: 4.8750 0.031494140625 0.06884765625 7.9375 8.4375\n",
      "Epoch 61/100 \t Train Err: 4.8125 0.0228271484375 0.072265625 8.5625 7.8125\n",
      "Epoch 61/100 \t Train Err: 4.8125 0.01611328125 0.07568359375 9.0 7.59375\n",
      "Epoch 62/100 \t Train Err: 4.7500 0.010986328125 0.07861328125 8.5625 7.625\n",
      "Epoch 62/100 \t Train Err: 4.6875 0.0079345703125 0.08203125 7.9375 7.625\n",
      "Epoch 62/100 \t Train Err: 4.7500 0.00665283203125 0.0810546875 7.5 8.3125\n",
      "Epoch 62/100 \t Train Err: 4.7188 0.00634765625 0.07861328125 7.09375 8.3125\n",
      "Epoch 62/100 \t Train Err: 4.6562 0.006317138671875 0.08349609375 7.40625 8.0\n",
      "Epoch 62/100 \t Train Err: 4.6562 0.006500244140625 0.0791015625 7.9375 7.625\n",
      "Epoch 62/100 \t Train Err: 4.7188 0.006500244140625 0.0732421875 8.1875 7.6875\n",
      "Epoch 62/100 \t Train Err: 4.6875 0.00677490234375 0.0771484375 8.5 7.53125\n",
      "Epoch 63/100 \t Train Err: 4.6875 0.00732421875 0.07421875 7.9375 7.78125\n",
      "Epoch 63/100 \t Train Err: 4.6562 0.00836181640625 0.0703125 7.84375 7.8125\n",
      "Epoch 63/100 \t Train Err: 4.6875 0.0093994140625 0.0654296875 7.65625 7.90625\n",
      "Epoch 63/100 \t Train Err: 4.6875 0.010498046875 0.0654296875 7.75 7.96875\n",
      "Epoch 63/100 \t Train Err: 4.6250 0.01116943359375 0.0634765625 7.6875 7.71875\n",
      "Epoch 63/100 \t Train Err: 4.6562 0.01153564453125 0.064453125 7.75 7.78125\n",
      "Epoch 63/100 \t Train Err: 4.6562 0.0118408203125 0.060302734375 7.875 7.78125\n",
      "Epoch 63/100 \t Train Err: 4.5938 0.01171875 0.0634765625 7.625 7.71875\n",
      "Epoch 64/100 \t Train Err: 4.6562 0.010986328125 0.060791015625 7.53125 7.90625\n",
      "Epoch 64/100 \t Train Err: 4.5312 0.01019287109375 0.05908203125 7.15625 7.6875\n",
      "Epoch 64/100 \t Train Err: 4.5312 0.009033203125 0.064453125 7.28125 7.75\n",
      "Epoch 64/100 \t Train Err: 4.5625 0.00836181640625 0.06298828125 7.5 7.75\n",
      "Epoch 64/100 \t Train Err: 4.5625 0.007598876953125 0.059326171875 7.5625 7.6875\n",
      "Epoch 64/100 \t Train Err: 4.5938 0.007110595703125 0.0654296875 7.5625 7.75\n",
      "Epoch 64/100 \t Train Err: 4.5312 0.00689697265625 0.06396484375 7.53125 7.625\n",
      "Epoch 64/100 \t Train Err: 4.5625 0.006591796875 0.06494140625 7.59375 7.59375\n",
      "Epoch 65/100 \t Train Err: 4.5000 0.0064697265625 0.061767578125 7.40625 7.625\n",
      "Epoch 65/100 \t Train Err: 4.5938 0.006439208984375 0.061767578125 7.625 7.71875\n",
      "Epoch 65/100 \t Train Err: 4.5625 0.006500244140625 0.0615234375 7.5 7.6875\n",
      "Epoch 65/100 \t Train Err: 4.5625 0.0064697265625 0.055908203125 7.25 7.75\n",
      "Epoch 65/100 \t Train Err: 4.5625 0.00640869140625 0.056640625 7.34375 7.75\n",
      "Epoch 65/100 \t Train Err: 4.4688 0.0064697265625 0.0625 7.375 7.5625\n",
      "Epoch 65/100 \t Train Err: 4.4688 0.00640869140625 0.060302734375 7.34375 7.53125\n",
      "Epoch 65/100 \t Train Err: 4.4375 0.00640869140625 0.059814453125 7.03125 7.6875\n",
      "Epoch 66/100 \t Train Err: 4.5000 0.00628662109375 0.06005859375 6.96875 7.71875\n",
      "Epoch 66/100 \t Train Err: 4.5312 0.006622314453125 0.058349609375 7.125 7.71875\n",
      "Epoch 66/100 \t Train Err: 4.4688 0.006500244140625 0.05908203125 7.34375 7.53125\n",
      "Epoch 66/100 \t Train Err: 4.5000 0.006683349609375 0.06298828125 7.46875 7.4375\n",
      "Epoch 66/100 \t Train Err: 4.3750 0.006744384765625 0.057861328125 7.0625 7.3125\n",
      "Epoch 66/100 \t Train Err: 4.4688 0.006683349609375 0.060791015625 7.125 7.5625\n",
      "Epoch 66/100 \t Train Err: 4.5000 0.006744384765625 0.060302734375 7.25 7.65625\n",
      "Epoch 66/100 \t Train Err: 4.5000 0.006744384765625 0.056884765625 7.125 7.625\n",
      "Epoch 67/100 \t Train Err: 4.4688 0.006683349609375 0.05712890625 6.96875 7.65625\n",
      "Epoch 67/100 \t Train Err: 4.3750 0.006683349609375 0.060546875 6.875 7.4375\n",
      "Epoch 67/100 \t Train Err: 4.3438 0.0067138671875 0.06298828125 7.21875 7.25\n",
      "Epoch 67/100 \t Train Err: 4.4688 0.00665283203125 0.06201171875 7.25 7.5625\n",
      "Epoch 67/100 \t Train Err: 4.4375 0.00628662109375 0.057861328125 6.96875 7.5625\n",
      "Epoch 67/100 \t Train Err: 4.2500 0.00640869140625 0.055419921875 6.53125 7.4375\n",
      "Epoch 67/100 \t Train Err: 4.3750 0.0062255859375 0.051513671875 6.96875 7.46875\n",
      "Epoch 67/100 \t Train Err: 4.4062 0.006195068359375 0.055419921875 6.96875 7.46875\n",
      "Epoch 68/100 \t Train Err: 4.3438 0.00604248046875 0.052734375 6.71875 7.34375\n",
      "Epoch 68/100 \t Train Err: 4.4062 0.005950927734375 0.055419921875 6.65625 7.59375\n",
      "Epoch 68/100 \t Train Err: 4.3750 0.005859375 0.0546875 6.90625 7.34375\n",
      "Epoch 68/100 \t Train Err: 4.3125 0.005950927734375 0.057373046875 6.6875 7.375\n",
      "Epoch 68/100 \t Train Err: 4.3750 0.005889892578125 0.05517578125 6.53125 7.625\n",
      "Epoch 68/100 \t Train Err: 4.3438 0.00592041015625 0.053955078125 6.53125 7.65625\n",
      "Epoch 68/100 \t Train Err: 4.3438 0.005889892578125 0.056640625 6.65625 7.34375\n",
      "Epoch 68/100 \t Train Err: 4.3125 0.005950927734375 0.05419921875 7.21875 7.25\n",
      "Epoch 69/100 \t Train Err: 4.3438 0.005859375 0.057373046875 6.9375 7.25\n",
      "Epoch 69/100 \t Train Err: 4.4062 0.00592041015625 0.0576171875 6.875 7.4375\n",
      "Epoch 69/100 \t Train Err: 4.3125 0.00592041015625 0.0556640625 6.65625 7.375\n",
      "Epoch 69/100 \t Train Err: 4.3438 0.00592041015625 0.0556640625 7.03125 7.40625\n",
      "Epoch 69/100 \t Train Err: 4.2812 0.005889892578125 0.058837890625 6.53125 7.34375\n",
      "Epoch 69/100 \t Train Err: 4.2500 0.00579833984375 0.05419921875 6.46875 7.25\n",
      "Epoch 69/100 \t Train Err: 4.3125 0.005828857421875 0.051513671875 6.625 7.34375\n",
      "Epoch 69/100 \t Train Err: 4.3125 0.005859375 0.052978515625 6.78125 7.21875\n",
      "Epoch 70/100 \t Train Err: 4.2812 0.005859375 0.04931640625 6.625 7.25\n",
      "Epoch 70/100 \t Train Err: 4.1875 0.005767822265625 0.051025390625 6.21875 7.1875\n",
      "Epoch 70/100 \t Train Err: 4.2500 0.005828857421875 0.05126953125 6.34375 7.4375\n",
      "Epoch 70/100 \t Train Err: 4.2812 0.005706787109375 0.05126953125 6.28125 7.5\n",
      "Epoch 70/100 \t Train Err: 4.3438 0.0057373046875 0.05078125 6.40625 7.5\n",
      "Epoch 70/100 \t Train Err: 4.3125 0.0057373046875 0.05517578125 6.875 7.1875\n",
      "Epoch 70/100 \t Train Err: 4.1562 0.005828857421875 0.052734375 6.78125 6.875\n",
      "Epoch 70/100 \t Train Err: 4.2188 0.00579833984375 0.05419921875 6.59375 7.15625\n",
      "Epoch 71/100 \t Train Err: 4.3125 0.00567626953125 0.052490234375 6.5 7.46875\n",
      "Epoch 71/100 \t Train Err: 4.2500 0.00567626953125 0.05126953125 6.125 7.5\n",
      "Epoch 71/100 \t Train Err: 4.2500 0.005706787109375 0.05224609375 6.5 7.4375\n",
      "Epoch 71/100 \t Train Err: 4.1875 0.0057373046875 0.053466796875 6.65625 6.96875\n",
      "Epoch 71/100 \t Train Err: 4.2812 0.005767822265625 0.0546875 6.875 7.09375\n",
      "Epoch 71/100 \t Train Err: 4.1562 0.005645751953125 0.054443359375 6.40625 6.96875\n",
      "Epoch 71/100 \t Train Err: 4.2188 0.005615234375 0.0498046875 6.3125 7.4375\n",
      "Epoch 71/100 \t Train Err: 4.2500 0.00555419921875 0.05224609375 6.40625 7.40625\n",
      "Epoch 72/100 \t Train Err: 4.2500 0.005615234375 0.052978515625 6.15625 7.375\n",
      "Epoch 72/100 \t Train Err: 4.2500 0.00537109375 0.0517578125 6.5 7.25\n",
      "Epoch 72/100 \t Train Err: 4.1562 0.00543212890625 0.05029296875 6.4375 7.0\n",
      "Epoch 72/100 \t Train Err: 4.1875 0.005401611328125 0.04638671875 6.3125 7.09375\n",
      "Epoch 72/100 \t Train Err: 4.1562 0.00537109375 0.048828125 6.34375 7.09375\n",
      "Epoch 72/100 \t Train Err: 4.2500 0.00537109375 0.0498046875 6.21875 7.3125\n",
      "Epoch 72/100 \t Train Err: 4.2188 0.00531005859375 0.05078125 6.46875 7.21875\n",
      "Epoch 72/100 \t Train Err: 4.1250 0.00537109375 0.05078125 6.46875 6.90625\n",
      "Epoch 73/100 \t Train Err: 4.2500 0.005340576171875 0.053955078125 6.40625 7.1875\n",
      "Epoch 73/100 \t Train Err: 4.1875 0.005279541015625 0.0517578125 6.59375 6.96875\n",
      "Epoch 73/100 \t Train Err: 4.2500 0.00537109375 0.0556640625 6.40625 7.21875\n",
      "Epoch 73/100 \t Train Err: 4.1250 0.00531005859375 0.051513671875 6.34375 6.96875\n",
      "Epoch 73/100 \t Train Err: 4.2188 0.005279541015625 0.04833984375 6.0 7.46875\n",
      "Epoch 73/100 \t Train Err: 4.1250 0.00537109375 0.050537109375 6.0625 7.1875\n",
      "Epoch 73/100 \t Train Err: 4.1250 0.00531005859375 0.048095703125 5.90625 7.09375\n",
      "Epoch 73/100 \t Train Err: 4.1562 0.0052490234375 0.05029296875 6.375 6.90625\n",
      "Epoch 74/100 \t Train Err: 4.0625 0.005523681640625 0.049560546875 6.4375 6.8125\n",
      "Epoch 74/100 \t Train Err: 4.1875 0.005279541015625 0.049072265625 5.90625 7.34375\n",
      "Epoch 74/100 \t Train Err: 4.2188 0.00531005859375 0.04931640625 6.09375 7.375\n",
      "Epoch 74/100 \t Train Err: 4.0938 0.00531005859375 0.05029296875 6.21875 6.96875\n",
      "Epoch 74/100 \t Train Err: 4.1562 0.0052490234375 0.053955078125 6.3125 7.09375\n",
      "Epoch 74/100 \t Train Err: 4.1250 0.00518798828125 0.05126953125 6.3125 7.0625\n",
      "Epoch 74/100 \t Train Err: 4.1562 0.0052490234375 0.051513671875 6.21875 7.0625\n",
      "Epoch 74/100 \t Train Err: 4.0938 0.005218505859375 0.049560546875 6.1875 7.0\n",
      "Epoch 75/100 \t Train Err: 4.1250 0.005218505859375 0.0517578125 6.3125 7.0625\n",
      "Epoch 75/100 \t Train Err: 4.0312 0.005218505859375 0.050048828125 6.03125 6.875\n",
      "Epoch 75/100 \t Train Err: 4.0938 0.005157470703125 0.0498046875 5.84375 7.15625\n",
      "Epoch 75/100 \t Train Err: 4.1250 0.005126953125 0.04541015625 5.875 7.21875\n",
      "Epoch 75/100 \t Train Err: 4.0625 0.005126953125 0.05126953125 6.09375 6.9375\n",
      "Epoch 75/100 \t Train Err: 4.0000 0.005218505859375 0.050537109375 6.40625 6.53125\n",
      "Epoch 75/100 \t Train Err: 4.1250 0.00518798828125 0.049560546875 6.09375 6.9375\n",
      "Epoch 75/100 \t Train Err: 4.0625 0.005126953125 0.046875 5.59375 7.1875\n",
      "Epoch 76/100 \t Train Err: 4.0000 0.00506591796875 0.045654296875 5.5 7.15625\n",
      "Epoch 76/100 \t Train Err: 4.1562 0.005157470703125 0.0458984375 6.0 7.25\n",
      "Epoch 76/100 \t Train Err: 4.0625 0.00518798828125 0.048095703125 6.5 6.59375\n",
      "Epoch 76/100 \t Train Err: 4.0625 0.005096435546875 0.0458984375 6.0 6.875\n",
      "Epoch 76/100 \t Train Err: 4.0312 0.0052490234375 0.044921875 5.75 7.03125\n",
      "Epoch 76/100 \t Train Err: 4.1562 0.00518798828125 0.043701171875 5.75 7.28125\n",
      "Epoch 76/100 \t Train Err: 4.0625 0.005096435546875 0.0498046875 5.84375 7.09375\n",
      "Epoch 76/100 \t Train Err: 4.0625 0.005126953125 0.044189453125 6.25 6.8125\n",
      "Epoch 77/100 \t Train Err: 3.9688 0.005126953125 0.0478515625 6.15625 6.59375\n",
      "Epoch 77/100 \t Train Err: 4.0312 0.005126953125 0.046142578125 5.84375 6.90625\n",
      "Epoch 77/100 \t Train Err: 4.0000 0.00506591796875 0.04541015625 5.71875 6.96875\n",
      "Epoch 77/100 \t Train Err: 3.9531 0.004974365234375 0.046142578125 5.46875 7.0\n",
      "Epoch 77/100 \t Train Err: 4.0312 0.004974365234375 0.044677734375 6.0625 6.8125\n",
      "Epoch 77/100 \t Train Err: 4.0312 0.005035400390625 0.047607421875 6.125 6.6875\n",
      "Epoch 77/100 \t Train Err: 4.0312 0.004852294921875 0.047607421875 6.09375 6.84375\n",
      "Epoch 77/100 \t Train Err: 3.9844 0.0048828125 0.043701171875 5.53125 6.96875\n",
      "Epoch 78/100 \t Train Err: 3.9844 0.004852294921875 0.045654296875 5.875 6.875\n",
      "Epoch 78/100 \t Train Err: 3.9844 0.004913330078125 0.0458984375 5.8125 6.78125\n",
      "Epoch 78/100 \t Train Err: 3.9375 0.0048828125 0.046142578125 5.6875 6.65625\n",
      "Epoch 78/100 \t Train Err: 3.9531 0.004974365234375 0.04541015625 5.71875 6.78125\n",
      "Epoch 78/100 \t Train Err: 4.0000 0.004913330078125 0.0458984375 5.5625 7.0\n",
      "Epoch 78/100 \t Train Err: 3.9375 0.0048828125 0.041748046875 5.46875 6.90625\n",
      "Epoch 78/100 \t Train Err: 3.9688 0.00494384765625 0.043701171875 5.71875 6.875\n",
      "Epoch 78/100 \t Train Err: 3.9375 0.005035400390625 0.048095703125 5.96875 6.46875\n",
      "Epoch 79/100 \t Train Err: 3.9844 0.0050048828125 0.049072265625 6.03125 6.625\n",
      "Epoch 79/100 \t Train Err: 3.9688 0.0050048828125 0.040283203125 4.90625 7.25\n",
      "Epoch 79/100 \t Train Err: 4.0312 0.004974365234375 0.04345703125 5.4375 7.1875\n",
      "Epoch 79/100 \t Train Err: 3.9688 0.005035400390625 0.045166015625 6.03125 6.5625\n",
      "Epoch 79/100 \t Train Err: 4.0000 0.005035400390625 0.0478515625 6.46875 6.28125\n",
      "Epoch 79/100 \t Train Err: 3.9219 0.005035400390625 0.043701171875 5.03125 7.03125\n",
      "Epoch 79/100 \t Train Err: 4.0312 0.004974365234375 0.044921875 4.90625 7.375\n",
      "Epoch 79/100 \t Train Err: 3.9062 0.004974365234375 0.04296875 5.8125 6.625\n",
      "Epoch 80/100 \t Train Err: 3.9219 0.0050048828125 0.04638671875 6.40625 6.25\n",
      "Epoch 80/100 \t Train Err: 3.8750 0.004974365234375 0.045654296875 5.65625 6.5\n",
      "Epoch 80/100 \t Train Err: 3.9531 0.0048828125 0.04345703125 5.15625 7.15625\n",
      "Epoch 80/100 \t Train Err: 3.9688 0.004913330078125 0.043701171875 5.25 7.0\n",
      "Epoch 80/100 \t Train Err: 3.8906 0.004974365234375 0.04345703125 5.875 6.53125\n",
      "Epoch 80/100 \t Train Err: 3.8281 0.00494384765625 0.044677734375 5.65625 6.4375\n",
      "Epoch 80/100 \t Train Err: 3.9531 0.004852294921875 0.041259765625 5.4375 6.96875\n",
      "Epoch 80/100 \t Train Err: 3.9531 0.00494384765625 0.04345703125 5.15625 7.0\n",
      "Epoch 81/100 \t Train Err: 3.9844 0.0048828125 0.0419921875 5.5625 6.8125\n",
      "Epoch 81/100 \t Train Err: 3.9219 0.0048828125 0.04443359375 5.71875 6.46875\n",
      "Epoch 81/100 \t Train Err: 3.9375 0.00482177734375 0.046630859375 5.71875 6.625\n",
      "Epoch 81/100 \t Train Err: 3.9219 0.00469970703125 0.042236328125 5.03125 6.90625\n",
      "Epoch 81/100 \t Train Err: 3.9219 0.0047607421875 0.042236328125 5.0625 7.03125\n",
      "Epoch 81/100 \t Train Err: 3.9844 0.004791259765625 0.044189453125 5.5625 6.90625\n",
      "Epoch 81/100 \t Train Err: 3.8750 0.0048828125 0.044677734375 5.875 6.375\n",
      "Epoch 81/100 \t Train Err: 3.8438 0.00482177734375 0.045166015625 5.40625 6.5625\n",
      "Epoch 82/100 \t Train Err: 3.8281 0.0048828125 0.044189453125 5.0625 6.8125\n",
      "Epoch 82/100 \t Train Err: 3.8906 0.004913330078125 0.042724609375 5.15625 7.03125\n",
      "Epoch 82/100 \t Train Err: 3.8750 0.00482177734375 0.048583984375 5.90625 6.34375\n",
      "Epoch 82/100 \t Train Err: 3.8594 0.004913330078125 0.045166015625 5.6875 6.28125\n",
      "Epoch 82/100 \t Train Err: 3.8594 0.004852294921875 0.043701171875 5.28125 6.78125\n",
      "Epoch 82/100 \t Train Err: 3.8594 0.004852294921875 0.042236328125 4.84375 6.9375\n",
      "Epoch 82/100 \t Train Err: 3.7969 0.0048828125 0.0400390625 5.15625 6.625\n",
      "Epoch 82/100 \t Train Err: 3.8281 0.004730224609375 0.04736328125 5.53125 6.28125\n",
      "Epoch 83/100 \t Train Err: 3.8438 0.0047607421875 0.044921875 5.3125 6.53125\n",
      "Epoch 83/100 \t Train Err: 3.7969 0.00469970703125 0.043212890625 5.09375 6.625\n",
      "Epoch 83/100 \t Train Err: 3.8125 0.004669189453125 0.043701171875 5.1875 6.71875\n",
      "Epoch 83/100 \t Train Err: 3.8281 0.004669189453125 0.042236328125 5.09375 6.6875\n",
      "Epoch 83/100 \t Train Err: 3.8125 0.0047607421875 0.042236328125 5.40625 6.46875\n",
      "Epoch 83/100 \t Train Err: 3.8594 0.00482177734375 0.04150390625 5.34375 6.59375\n",
      "Epoch 83/100 \t Train Err: 3.7500 0.00482177734375 0.041748046875 5.3125 6.375\n",
      "Epoch 83/100 \t Train Err: 3.7812 0.004791259765625 0.040771484375 4.75 6.75\n",
      "Epoch 84/100 \t Train Err: 3.7188 0.0047607421875 0.0390625 5.21875 6.34375\n",
      "Epoch 84/100 \t Train Err: 3.7656 0.0047607421875 0.040771484375 5.09375 6.40625\n",
      "Epoch 84/100 \t Train Err: 3.7969 0.004852294921875 0.04248046875 5.25 6.34375\n",
      "Epoch 84/100 \t Train Err: 3.7344 0.0047607421875 0.03955078125 5.125 6.3125\n",
      "Epoch 84/100 \t Train Err: 3.7500 0.004791259765625 0.03857421875 4.5 6.90625\n",
      "Epoch 84/100 \t Train Err: 3.7656 0.00482177734375 0.04248046875 5.1875 6.40625\n",
      "Epoch 84/100 \t Train Err: 3.7656 0.00482177734375 0.042724609375 5.3125 6.21875\n",
      "Epoch 84/100 \t Train Err: 3.7188 0.004730224609375 0.04345703125 5.21875 6.28125\n",
      "Epoch 85/100 \t Train Err: 3.7500 0.004608154296875 0.039794921875 4.65625 6.625\n",
      "Epoch 85/100 \t Train Err: 3.7344 0.004608154296875 0.04150390625 4.71875 6.53125\n",
      "Epoch 85/100 \t Train Err: 3.7344 0.004730224609375 0.042236328125 5.375 6.3125\n",
      "Epoch 85/100 \t Train Err: 3.7656 0.004730224609375 0.040283203125 5.4375 6.28125\n",
      "Epoch 85/100 \t Train Err: 3.6875 0.00469970703125 0.0400390625 5.0 6.28125\n",
      "Epoch 85/100 \t Train Err: 3.7031 0.00469970703125 0.037109375 4.8125 6.53125\n",
      "Epoch 85/100 \t Train Err: 3.7500 0.00469970703125 0.038330078125 4.78125 6.625\n",
      "Epoch 85/100 \t Train Err: 3.6875 0.004669189453125 0.0400390625 5.34375 6.0625\n",
      "Epoch 86/100 \t Train Err: 3.7344 0.004638671875 0.037841796875 5.40625 6.25\n",
      "Epoch 86/100 \t Train Err: 3.7031 0.004638671875 0.0380859375 4.875 6.46875\n",
      "Epoch 86/100 \t Train Err: 3.7344 0.00457763671875 0.0390625 4.59375 6.6875\n",
      "Epoch 86/100 \t Train Err: 3.7031 0.004638671875 0.036865234375 5.09375 6.21875\n",
      "Epoch 86/100 \t Train Err: 3.7031 0.004638671875 0.0390625 5.28125 6.15625\n",
      "Epoch 86/100 \t Train Err: 3.7500 0.004669189453125 0.037841796875 4.875 6.59375\n",
      "Epoch 86/100 \t Train Err: 3.7188 0.004638671875 0.0390625 4.625 6.5625\n",
      "Epoch 86/100 \t Train Err: 3.6406 0.004730224609375 0.039306640625 5.125 5.9375\n",
      "Epoch 87/100 \t Train Err: 3.6875 0.00469970703125 0.03759765625 4.9375 6.28125\n",
      "Epoch 87/100 \t Train Err: 3.6875 0.004730224609375 0.039794921875 4.78125 6.25\n",
      "Epoch 87/100 \t Train Err: 3.7031 0.004791259765625 0.0361328125 4.65625 6.53125\n",
      "Epoch 87/100 \t Train Err: 3.6719 0.0047607421875 0.03662109375 4.625 6.5\n",
      "Epoch 87/100 \t Train Err: 3.6094 0.004791259765625 0.037109375 5.125 6.0\n",
      "Epoch 87/100 \t Train Err: 3.7344 0.00482177734375 0.036865234375 5.375 6.15625\n",
      "Epoch 87/100 \t Train Err: 3.6719 0.0047607421875 0.037353515625 4.65625 6.53125\n",
      "Epoch 87/100 \t Train Err: 3.6562 0.004791259765625 0.03662109375 4.59375 6.4375\n",
      "Epoch 88/100 \t Train Err: 3.6562 0.004638671875 0.03857421875 5.28125 6.0\n",
      "Epoch 88/100 \t Train Err: 3.5938 0.004730224609375 0.040283203125 4.875 6.03125\n",
      "Epoch 88/100 \t Train Err: 3.6875 0.004608154296875 0.03955078125 4.5625 6.5\n",
      "Epoch 88/100 \t Train Err: 3.6875 0.004730224609375 0.0380859375 4.71875 6.46875\n",
      "Epoch 88/100 \t Train Err: 3.5469 0.004608154296875 0.037353515625 4.53125 6.21875\n",
      "Epoch 88/100 \t Train Err: 3.6250 0.004608154296875 0.040771484375 4.9375 6.125\n",
      "Epoch 88/100 \t Train Err: 3.6094 0.004486083984375 0.038330078125 4.8125 6.21875\n",
      "Epoch 88/100 \t Train Err: 3.5938 0.0045166015625 0.0400390625 4.75 6.21875\n",
      "Epoch 89/100 \t Train Err: 3.6406 0.0045166015625 0.038330078125 4.875 6.21875\n",
      "Epoch 89/100 \t Train Err: 3.5469 0.00457763671875 0.041748046875 5.21875 5.6875\n",
      "Epoch 89/100 \t Train Err: 3.6250 0.004608154296875 0.03759765625 4.46875 6.40625\n",
      "Epoch 89/100 \t Train Err: 3.5469 0.004638671875 0.035400390625 4.625 6.15625\n",
      "Epoch 89/100 \t Train Err: 3.6094 0.00469970703125 0.0341796875 4.40625 6.375\n",
      "Epoch 89/100 \t Train Err: 3.5781 0.00469970703125 0.0361328125 4.75 6.0625\n",
      "Epoch 89/100 \t Train Err: 3.4688 0.0047607421875 0.0341796875 4.53125 5.96875\n",
      "Epoch 89/100 \t Train Err: 3.5781 0.00469970703125 0.033935546875 4.40625 6.34375\n",
      "Epoch 90/100 \t Train Err: 3.5625 0.0047607421875 0.03369140625 4.53125 6.25\n",
      "Epoch 90/100 \t Train Err: 3.6094 0.004791259765625 0.033447265625 4.875 6.0625\n",
      "Epoch 90/100 \t Train Err: 3.6250 0.0048828125 0.034423828125 4.40625 6.34375\n",
      "Epoch 90/100 \t Train Err: 3.5469 0.004913330078125 0.03662109375 4.71875 6.0\n",
      "Epoch 90/100 \t Train Err: 3.5000 0.004913330078125 0.035400390625 4.40625 6.125\n",
      "Epoch 90/100 \t Train Err: 3.5469 0.00494384765625 0.03369140625 4.125 6.375\n",
      "Epoch 90/100 \t Train Err: 3.5156 0.004852294921875 0.035400390625 4.53125 5.96875\n",
      "Epoch 90/100 \t Train Err: 3.5156 0.00469970703125 0.036376953125 4.71875 5.90625\n",
      "Epoch 91/100 \t Train Err: 3.5312 0.0047607421875 0.033447265625 4.375 6.28125\n",
      "Epoch 91/100 \t Train Err: 3.5469 0.004730224609375 0.032958984375 4.34375 6.3125\n",
      "Epoch 91/100 \t Train Err: 3.6094 0.004669189453125 0.0390625 5.34375 5.65625\n",
      "Epoch 91/100 \t Train Err: 3.5156 0.004608154296875 0.032958984375 4.28125 6.0625\n",
      "Epoch 91/100 \t Train Err: 3.5469 0.00457763671875 0.03125 4.03125 6.4375\n",
      "Epoch 91/100 \t Train Err: 3.5312 0.004608154296875 0.03564453125 4.84375 5.71875\n",
      "Epoch 91/100 \t Train Err: 3.4844 0.004547119140625 0.0322265625 4.5 5.90625\n",
      "Epoch 91/100 \t Train Err: 3.5312 0.004608154296875 0.0299072265625 3.953125 6.625\n",
      "Epoch 92/100 \t Train Err: 3.5000 0.004547119140625 0.03515625 5.21875 5.46875\n",
      "Epoch 92/100 \t Train Err: 3.4688 0.004486083984375 0.034423828125 4.53125 5.9375\n",
      "Epoch 92/100 \t Train Err: 3.4531 0.00457763671875 0.032470703125 3.875 6.375\n",
      "Epoch 92/100 \t Train Err: 3.5938 0.004669189453125 0.038330078125 4.9375 5.84375\n",
      "Epoch 92/100 \t Train Err: 3.5156 0.004730224609375 0.03369140625 5.03125 5.6875\n",
      "Epoch 92/100 \t Train Err: 3.5625 0.004791259765625 0.029052734375 3.890625 6.65625\n",
      "Epoch 92/100 \t Train Err: 3.4531 0.004730224609375 0.032470703125 4.5 5.90625\n",
      "Epoch 92/100 \t Train Err: 3.4531 0.0047607421875 0.030517578125 4.71875 5.6875\n",
      "Epoch 93/100 \t Train Err: 3.5156 0.00482177734375 0.0281982421875 3.78125 6.53125\n",
      "Epoch 93/100 \t Train Err: 3.4531 0.00482177734375 0.03173828125 4.6875 5.5625\n",
      "Epoch 93/100 \t Train Err: 3.4531 0.004791259765625 0.03271484375 4.6875 5.6875\n",
      "Epoch 93/100 \t Train Err: 3.4375 0.00469970703125 0.0279541015625 3.96875 6.25\n",
      "Epoch 93/100 \t Train Err: 3.3594 0.004730224609375 0.03076171875 4.125 5.9375\n",
      "Epoch 93/100 \t Train Err: 3.4688 0.004730224609375 0.0301513671875 4.96875 5.625\n",
      "Epoch 93/100 \t Train Err: 3.3906 0.004730224609375 0.0296630859375 4.03125 6.0625\n",
      "Epoch 93/100 \t Train Err: 3.4688 0.0047607421875 0.0294189453125 4.25 6.0625\n",
      "Epoch 94/100 \t Train Err: 3.3906 0.0047607421875 0.031005859375 4.375 5.6875\n",
      "Epoch 94/100 \t Train Err: 3.4219 0.004791259765625 0.031494140625 4.53125 5.8125\n",
      "Epoch 94/100 \t Train Err: 3.4375 0.004791259765625 0.0281982421875 4.25 6.03125\n",
      "Epoch 94/100 \t Train Err: 3.4219 0.004791259765625 0.02978515625 4.34375 5.8125\n",
      "Epoch 94/100 \t Train Err: 3.4219 0.00469970703125 0.0308837890625 4.375 5.78125\n",
      "Epoch 94/100 \t Train Err: 3.4375 0.004669189453125 0.028564453125 4.25 6.0\n",
      "Epoch 94/100 \t Train Err: 3.3594 0.00469970703125 0.0283203125 4.25 5.875\n",
      "Epoch 94/100 \t Train Err: 3.3594 0.004852294921875 0.03125 4.28125 5.6875\n",
      "Epoch 95/100 \t Train Err: 3.3750 0.004791259765625 0.0284423828125 4.25 5.6875\n",
      "Epoch 95/100 \t Train Err: 3.3750 0.0047607421875 0.0279541015625 3.90625 5.96875\n",
      "Epoch 95/100 \t Train Err: 3.3594 0.004730224609375 0.029052734375 4.28125 5.71875\n",
      "Epoch 95/100 \t Train Err: 3.3906 0.004791259765625 0.0296630859375 4.3125 5.78125\n",
      "Epoch 95/100 \t Train Err: 3.3750 0.004791259765625 0.03076171875 4.125 5.78125\n",
      "Epoch 95/100 \t Train Err: 3.3594 0.0047607421875 0.0291748046875 4.25 5.8125\n",
      "Epoch 95/100 \t Train Err: 3.3438 0.004669189453125 0.02783203125 4.15625 5.875\n",
      "Epoch 95/100 \t Train Err: 3.3281 0.0047607421875 0.0299072265625 4.375 5.5\n",
      "Epoch 96/100 \t Train Err: 3.3438 0.0048828125 0.02880859375 3.84375 5.90625\n",
      "Epoch 96/100 \t Train Err: 3.3750 0.0047607421875 0.030029296875 4.0625 5.8125\n",
      "Epoch 96/100 \t Train Err: 3.3438 0.00482177734375 0.0308837890625 3.9375 5.71875\n",
      "Epoch 96/100 \t Train Err: 3.3125 0.004730224609375 0.028564453125 4.21875 5.59375\n",
      "Epoch 96/100 \t Train Err: 3.2969 0.004638671875 0.0291748046875 3.765625 5.90625\n",
      "Epoch 96/100 \t Train Err: 3.3750 0.004638671875 0.034912109375 4.375 5.75\n",
      "Epoch 96/100 \t Train Err: 3.2656 0.004638671875 0.029296875 4.125 5.53125\n",
      "Epoch 96/100 \t Train Err: 3.2500 0.004638671875 0.0286865234375 3.984375 5.625\n",
      "Epoch 97/100 \t Train Err: 3.2656 0.00457763671875 0.028564453125 4.125 5.59375\n",
      "Epoch 97/100 \t Train Err: 3.3438 0.004547119140625 0.02587890625 3.859375 5.9375\n",
      "Epoch 97/100 \t Train Err: 3.3125 0.00457763671875 0.02783203125 4.03125 5.6875\n",
      "Epoch 97/100 \t Train Err: 3.3438 0.004608154296875 0.02783203125 4.125 5.6875\n",
      "Epoch 97/100 \t Train Err: 3.3125 0.004730224609375 0.0279541015625 3.875 5.75\n",
      "Epoch 97/100 \t Train Err: 3.2500 0.004730224609375 0.0289306640625 3.78125 5.6875\n",
      "Epoch 97/100 \t Train Err: 3.2812 0.004669189453125 0.030517578125 4.03125 5.5625\n",
      "Epoch 97/100 \t Train Err: 3.3281 0.004638671875 0.029296875 4.1875 5.625\n",
      "Epoch 98/100 \t Train Err: 3.2969 0.004669189453125 0.0264892578125 3.734375 6.0\n",
      "Epoch 98/100 \t Train Err: 3.2031 0.0047607421875 0.0262451171875 3.921875 5.5\n",
      "Epoch 98/100 \t Train Err: 3.2969 0.004791259765625 0.02685546875 4.59375 5.1875\n",
      "Epoch 98/100 \t Train Err: 3.3750 0.0047607421875 0.02392578125 3.34375 6.5\n",
      "Epoch 98/100 \t Train Err: 3.4688 0.004791259765625 0.033447265625 5.625 4.625\n",
      "Epoch 98/100 \t Train Err: 4.5625 0.00482177734375 0.01953125 1.4296875 11.0625\n",
      "Epoch 98/100 \t Train Err: 10.6875 0.00567626953125 0.44140625 46.5 1.109375\n",
      "Epoch 98/100 \t Train Err: 11.5625 0.0096435546875 0.0252685546875 0.322265625 29.875\n",
      "Epoch 99/100 \t Train Err: 12.5000 0.1318359375 0.0341796875 0.1640625 32.25\n",
      "Epoch 99/100 \t Train Err: 7.3125 0.71484375 0.66796875 2.953125 17.125\n",
      "Epoch 99/100 \t Train Err: 9.1250 1.265625 3.046875 38.0 3.265625\n",
      "Epoch 99/100 \t Train Err: 9.4375 1.078125 3.578125 40.5 2.859375\n",
      "Epoch 99/100 \t Train Err: 5.9688 0.419921875 1.484375 14.3125 8.125\n",
      "Epoch 99/100 \t Train Err: 6.6250 0.01513671875 0.3125 4.5 15.25\n",
      "Epoch 99/100 \t Train Err: 7.5000 0.326171875 0.042236328125 2.78125 18.0\n",
      "Epoch 99/100 \t Train Err: 6.6250 0.765625 0.087890625 5.0 14.375\n"
     ]
    }
   ],
   "source": [
    "while epoch < NEPOCHS:\n",
    "    model.train()\n",
    "    with open(f\"data/{epoch}.pickle\", \"rb\") as f:\n",
    "        pickled_stuff = pickle.load(f)\n",
    "    data = pickled_stuff[\"data\"].to(device)\n",
    "    label = pickled_stuff[\"labels\"].to(device)\n",
    "    padding = pickled_stuff[\"padding\"].to(device)\n",
    "    dataset = TensorDataset(data, label, padding)\n",
    "    loader = DataLoader(dataset, batch_size=BSZ)\n",
    "    for batch_src, batch_labels, batch_padding_mask in loader:\n",
    "        optimizer.zero_grad()\n",
    "        output = model(batch_src, batch_padding_mask)\n",
    "        loss = criterion(output.squeeze(1), batch_labels)\n",
    "        train_loss = loss.item()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        # scheduler.step(loss)\n",
    "    \n",
    "        # test_loss = evaluate()\n",
    "        # test_short_loss = evaluate_short()\n",
    "        \n",
    "        # test_err.append(test_loss)\n",
    "        train_err.append(train_loss)\n",
    "        len1.append(criterion(output[batch_labels == 1].squeeze(1), batch_labels[batch_labels==1]))\n",
    "        len2.append(criterion(output[batch_labels == 2].squeeze(1), batch_labels[batch_labels==2]))\n",
    "        len3.append(criterion(output[batch_labels == 3].squeeze(1), batch_labels[batch_labels==3]))\n",
    "        len15.append(criterion(output[batch_labels == 15].squeeze(1), batch_labels[batch_labels==15]))\n",
    "        \n",
    "        with open('loss', 'a') as f:\n",
    "            f.write(f\"{train_loss}\\n\")\n",
    "        print(f\"Epoch {epoch}/{NEPOCHS} \\t Train Err: {train_loss:.4f} {len1[-1]} {len2[-1]} {len3[-1]} {len15[-1]}\")\n",
    "\n",
    "    epoch += 1\n",
    "    if epoch % 100 == 0:\n",
    "        torch.save(model.state_dict(), f\"model_weights_{epoch}.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "# \"\"\"\n",
    "# Now let's figure out what it's doing. \n",
    "\n",
    "# step 1: figure out what people are attending to \n",
    "# \"\"\"\n",
    "\n",
    "# example_graph, answer, padding = mkbatch(1)\n",
    "# sentance_embeddings = model.full_embedding(example_graph)[0,:,:][example_graph.flatten() != 0]\n",
    "# WQ,WK,WV = torch.split(model.transformer_encoder.layers[0].self_attn.in_proj_weight, (MODEL_DIM, MODEL_DIM, MODEL_DIM))\n",
    "\n",
    "# Q = sentance_embeddings@WQ\n",
    "# K = sentance_embeddings@WK\n",
    "\n",
    "# raw_scores = Q @ K.T / sqrt(MODEL_DIM)\n",
    "# soft = torch.softmax(raw_scores, dim=-1).detach().cpu().to(float).numpy()\n",
    "# plt.imshow(soft)\n",
    "# plt.show()\n",
    "\n",
    "# print(example_graph)\n",
    "\n",
    "# print(Q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.suptitle('MSE vs Epochs')\n",
    "plt.plot(train_err, label='Train', color='blue')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('MSE')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "execution_state": "idle",
   "metadata": {
    "id": "LoGEmM5lH7_A"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhIAAAGdCAYAAABHM5ovAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAwtElEQVR4nO3de3RU9b338c9kQiYQk0igJKQkkLbeuJgoBKp4WiJ5ykktLbq8HkojrqWnbRAw1gNYwUvFiFaaqjlQfJZiz5FKu1ZBK0d6OBGNtqC5NJ7yWBCOEVMxiVTJkNBcmNnPH5I5BgbYmdnzm8nk/Vpr/zE7e8/3u8mF7/yuLsuyLAEAAIQgIdoJAACAwYtCAgAAhIxCAgAAhIxCAgAAhIxCAgAAhIxCAgAAhIxCAgAAhIxCAgAAhCwx2gmczO/369ChQ0pNTZXL5Yp2OgCAGGZZlo4ePars7GwlJETus3FXV5d6enrCfp+kpCQlJyc7klOsiLlC4tChQ8rJyYl2GgCAQaS5uVnjxo2LyHt3dXUpb/w5amnzhf1eWVlZampqiqtiIuYKidTUVOnED0VaWlq003HMd4ofMRbrkykpRuL4hhkJI0nqyjATp2d8+J847MgYfdRIHElKTeoyEmfb1x8yEgeDh7/1kojH8Hb4Nf7S9wP/d0RCT0+PWtp8aqofr7TU0Fs9vEf9ypt6UD09PRQSkdTXnZGWlhZXhURiorkfGneSoVhJZsJIktvQIyUMNzNsyD3CTMEiSYkeM9vpxNPvK5zhP+Y2FstEV3haakJYhUS84l8EAAAbfJY/7EOSCgsLNXHiRFVVVUX7kRwRcy0SAADEIr8s+RV6C1/fvbW1tXHVgkchAQCADX755Q/z/nhE1wYAAAgZLRKGuDvMjJyXpJSPPEbifFxg7scn8ZiZON3dZmrrc5P/biSOJFUXrTUWC/i883/5g4jH8Hd1Sbo74nEkyWdZ8lmhd22Ec28so0UCAAAb+sZIhHOIwZYAACAcDLYEAGAI8suSz4FZG/GGQgIAABucmv4ZbwY8RqKmpkZz585Vdna2XC6Xtm7detprv//978vlcqmysjLcPAEAQAwacCHR2dmp/Pz8sw4S2bJli3bv3q3s7Oxw8gMAICb0zdoI5xCDLaWSkhKVlJSc8ZoPP/xQt99+u37/+9/rqquuCie/uLH9vx80FuvyGx4zEif9f8w10/1tspkt5RPO6TUSx6Tb6kqNxClI/cBIHEn64QU7jcVC6JK8kY/h6458jD7+E0c494vBlmfn9/u1YMEC3XXXXZo0adJZr+/u7lZ39//+JHi9Bn7yAACAIxxfR2LNmjVKTEzU4sWLbV1fUVGh9PT0wJGTk+N0SgAAhM13YtZGOEc8crSQqK+v189//nNt3LjR9pauK1asUHt7e+Bobm52MiUAABzhs8I/4pGjhcTrr7+utrY25ebmKjExUYmJiTp48KDuvPNOTZgwIeg9Ho9HaWlp/Q4AAGKN34FDDLY8swULFqi4uLjfuTlz5mjBggVauHChk6EAABiUhvxgy46ODh04cCDwuqmpSY2NjcrIyFBubq5GjRrV7/phw4YpKytLF1xwgTMZAwAQBX655FPoM8j8YdwbywZcSNTV1amoqCjwury8XJJUWlqqjRs3OpsdAAAxwm99doRzfzwacCExa9YsWQPYCvX9998faAiE6Y+b7zQS5+vffMRInM8MMxLF3+s2EudI13AjcSRJqWbCsLYDTpbYGfkYLoPrSCA49toAAMAGX5hdG+HcG8soJAAAsIFCIjjHF6QCAACnx/RPAACGIL/lkt8KY9bGiXuH/PRPAACGIro2gqNrAwAAhIwWiTh05eyHjcQ5ep7HSByTkkb0RDsFx22Y9qyROP+6r8jGVc5gqung4PIZiBHOvt4D5FOCfGF8/jbwzxEVFBIAANhghTlGwgrj3lhGIQEAgA2MkQiOMRIAACBkFBIAANjgsxLCPsQ6EgAADE1+ueQP4/O3X5/tUxVv60jQIgEAAEJGiwQAADYw2DI4Cok41JtqZitsk3wjzEwWT0ywjMTJTTtiJI5JrO2AkyUdjfzvk6/HzO+sPjdGIvT7zeVqEl0bAAAgZLRIAABgw2eDLcPYtIuuDQAAhi5/mEtk983aiDd0bQAAgJDRIgEAgA0MtgyOQgIAABv8SnBkQap4QyGBkKV+0Gss1icXm+mF6+k28yvxgfdcI3FMYhtxnCzBwL7ZlsG9uX2WS74wdvDsu7ewsFBut1tlZWUqKytzMMPooJAAAMCgeFsim0ICAAAbfGHO2vDRtQEAwNDltxLkD2OwpT9OB1sy/RMAAISMFgkAAGygayM4CgkAAGzwf27mRaj3xyMKiTg07KiZ+VAfX+IxEkeS5DbzTP5eMzunfvppipE4JjElEyfrOSfye0v4euJz/4rBhEICAAAbwl+QKj6HJVJIAABgQ/hLZMdnIRGfTwUAAIygRQIAABv8csmvcAZbxud4DgoJAABsoGsjOAoJAABsCH8difgsJOLzqQAAiFGFhYWaOHGiqqqqop2KI2iRQMgy3jG3jfjRSWbiJJ/TbSTOF0ceMRJHkopeudNInJ1XPmYkDgaP4YcjvwTT8V5zyzz5LZf84SxIdeJedv8EAGAI8ofZtRGv60jE51MBAAAjaJEAAMCG8LcRj8/P7hQSAADY4JNLvjDWggjn3lgWn+URAAAwghYJAABsoGsjuAEXEjU1NXr00UdVX1+vjz76SFu2bNG8efMkSb29vbrnnnv0H//xH3rvvfeUnp6u4uJiPfzww8rOzo5E/ggi6XCnkTjvX51hJI4kqcfMNuLDEs3EqS5aaySODE7/BE7mH2YgRuRDBPjC7J4w89fFvAGXR52dncrPzw+6kMaxY8fU0NCglStXqqGhQb/97W+1b98+ffvb33YqXwAAEEMG3CJRUlKikpKSoF9LT0/Xjh07+p178sknNX36dH3wwQfKzc0NPVMAAKKIro3gIj5Gor29XS6XS+eee27Qr3d3d6u7+39XE/R6vZFOCQCAAWPTruAi+lRdXV1atmyZbrrpptMuB1pRUaH09PTAkZOTE8mUAAAIiXViG/FQD4vpnwPT29ur66+/XpZlad26dae9bsWKFWpvbw8czc3NkUoJAAA4LCJdG31FxMGDB/XKK6+ccXMSj8cjj8cTiTQAAHAMXRvBOV5I9BUR+/fv186dOzVq1CinQwAAYJxTu3/GmwEXEh0dHTpw4EDgdVNTkxobG5WRkaGxY8fq2muvVUNDg1566SX5fD61tLRIkjIyMpSUlORs9ogqz6fmYnWNMROnu4c12gCnJHZZkQ/SayAGzmjAfzXr6upUVFQUeF1eXi5JKi0t1X333acXX3xRklRQUNDvvp07d2rWrFnhZwwAQBT4wtxGPJx7Y9mAC4lZs2bJsk5fAZ7pawAADFZ0bQQXn+URAAAwgkICAAAb/EoI+4hFV199tUaOHKlrr702pPtj86kAAIgxPssV9hGLlixZol/+8pch308hAQDAEDZr1iylpqaGfD9z3QwpyfqhsVidX80zFssUd5eZmnfE8B4jcWbvLDcS5zOx+SkIGGyiMdiypqZGjz76qOrr6/XRRx9py5YtmjdvXr9rqqqq9Oijj6qlpUX5+fl64oknNH369JDzHCgKCQAAbLDC3P3TCuHezs5O5efn65ZbbtE111xzytc3b96s8vJyrV+/XjNmzFBlZaXmzJmjffv2acyYzxbgKSgo0PHjx0+59z//8z+VnZ0d4tP8LwoJAABs8MklXxgtfH33nrzL9Zm2iigpKVFJSclp33Pt2rW69dZbtXDhQknS+vXrtW3bNj399NNavny5JKmxsTHknO1gjAQAAAbl5OT02/W6oqIipPfp6elRfX29iouLA+cSEhJUXFysXbt2OZjxmdEiAQCADX4rvEWl/CfWa2xubu63mWWoG1cePnxYPp9PmZmZ/c5nZmZq7969tt+nuLhYb7/9tjo7OzVu3Dj95je/0WWXXWb7fgoJAABs8Ic5RqLv3rS0tDPuim3af/3Xf4V1P10bAAAMQqNHj5bb7VZra2u/862trcrKyjKWBy0Shrzc8q/GYl05+2EjcZI/cRuJI0ntF/iNxDnyt3OMxBk1otNIHCCaEgzszGkiRh+/XPKHMdiy797CwkK53W6VlZWprKws5PdLSkrS1KlTVV1dHZgS6vf7VV1drUWLFoX8vgNFIQEAgA3hrk7Zd29tba3tro2Ojg4dOHAg8LqpqUmNjY3KyMhQbm6uysvLVVpaqmnTpmn69OmqrKxUZ2dnYBaHCRQSAADEqLq6OhUVFQVel5d/tphdaWmpNm7cqBtuuEEff/yxVq1apZaWFhUUFGj79u2nDMCMJAoJAABscGqw5UDMmjVLlnXm7ptFixYZ7co4GYMtAQCwwS9XYJnskI7PjZGYOHGiqqqqov1IjqBFAgAAgwYyRmIwoJAAAMAGK8xZG1acbqBHIQEAgA3R2P1zMKCQMMTkNuKvGFqzIn/Jz4zEkSTPYTNrVnQPMzMnvemj0UbiSNJ7/3S3kTivv/8VI3Ek6R8mHLBxFaLNZeDXyUSMPtEYbDkYxOdTAQAQoxhsCQDAEORU1waDLQEAGIKcWiI73tC1AQAAQkaLBAAANjBrIzhaJAAAsCGsVS0/V4Qw2BIhMbmN+D/Me9RInK5Lzf349H7l70biJCcfNxKnp8vcv13RK3caibPzSqZkor/eEZH/rHq8d/B9HmawJQAAQxBdG8FRSAAAYAOFRHCDr00IAADEDFokAACwwQpzLQiDq3kbRSEBAIANdG0ER9cGAAA2MP0zOFokDDG5+2dK5igjcZK+kmEkjiT1/M1jJM655x81EqetK9VIHCCakry+iMdIOB75GE5j+icAAEMQXRvBUUgAAGADhURwjJEAAAAho0UCAAAbLMslK4xWhXDujWUUEgAA2OCXK6x1JMK5N5bRtQEAAEJGIQEAgA2sIxEcXRuGmNxG/B/zVxqJ03uOkTCSJOscM3PFW1rPNRLn/e8tMxJHkm6rKzUWC/g8f1Lkm/L9LnPdBU6NkYi3dSQG3CJRU1OjuXPnKjs7Wy6XS1u3bu33dcuytGrVKo0dO1bDhw9XcXGx9u/f72TOAAAgRgy4kOjs7FR+fv5pm2QeeeQRPf7441q/fr3efPNNpaSkaM6cOerq6nIiXwAAosKpro14M+CujZKSEpWUlAT9mmVZqqys1D333KPvfOc7kqRf/vKXyszM1NatW3XjjTeGnzEAAFHA9M/gHB1s2dTUpJaWFhUXFwfOpaena8aMGdq1a5eToQAAMMoKszUiXgsJRwdbtrS0SJIyMzP7nc/MzAx87WTd3d3q7u4OvPZ6vU6mBAAAIijq0z8rKiqUnp4eOHJycqKdEgAAp7AkWVYYR7QfIEIcbZHIysqSJLW2tmrs2LGB862trSooKAh6z4oVK1ReXh547fV6KSbC1JlnZlrRsA4jYSRJvZ+amamcNaXVSByT9ntHRzsFDFGJxwxM2za4jbhfLrlY2fIUjrZI5OXlKSsrS9XV1YFzXq9Xb775pi677LKg93g8HqWlpfU7AADA4DDgj3kdHR06cOBA4HVTU5MaGxuVkZGh3NxcLV26VA8++KDOO+885eXlaeXKlcrOzta8efOczh0AAGOYtRHcgAuJuro6FRUVBV73dUuUlpZq48aN+pd/+Rd1dnbqtttu05EjR3TFFVdo+/btSk5OdjZzAAAM8lsuucIoBj6/RLbb7VZZWZnKysoczDA6BlxIzJo1S5Z1+iEjLpdLDzzwgB544IFwcwMAIO7E2xLZ7LUBAIANfbMvwrk/HlFIAABgA2Mkgov6OhIAAGDwokXCkJKsHxqLlZI5ykictkszjMQxqeu4mV+J2TvLbVzljJ1XrjUWC/g8V68/8jGORz5GH1okgqOQAADABqdmbcQbCgkAAGxgsGVwjJEAAAAho0UCAAAbPmuRCGeMhKPpxAwKCQAAbGCwZXB0bQAAgJDRIhGHekanRDsFx/lGHjcSJyvlqJE4LZ2pRuIA0dQ9KiniMY4bmGLaxzpxhHN/PKKQAADABro2gqNrAwAAgwoLCzVx4kRVVVVFOxVH0CIBAIAdDvVtsPsnAABDUZhdG4rTrg0KCQAAbGBly+AYIwEAAEJGiwQAADYwayM4CglDrDHmttzuGj3MSBx3j5EwkqTk9C4jcf7yYZaROO/ddLeROJJU9MqdRuLsvPIxI3EweKR80BnxGMd9Zv42SCfGODBG4hR0bQAAgJDRIgEAgA0MtgyOQgIAADtYIzsoujYAAEDIaJEAAMAGZm0ERyEBAIBdcdo9EQ4KCUO2//eDxmJdOfthI3G6Lov8FsF9zExolSaN+8hInKv/UGYkjiTtvDI+NgbC4NN7rifiMY4f53/2aKOQAADABro2gqOQAADADmZtBEUhAQCALa4TRzj3xx+mfwIAgJDRIgEAgB10bQRFIQEAgB0UEkFRSBjyjxffYyxW75fSjcQZ5jXX32dqtPP7n5rZpbWzI/LT4gJmmgsFfJ772PGIx7CORz5GPGtubtaCBQvU1tamxMRErVy5Utddd92A3oNCAgAAO+JwG/HExERVVlaqoKBALS0tmjp1qr75zW8qJSXF/ntENEMAAOJEPO7+OXbsWI0dO1aSlJWVpdGjR+uTTz4ZUCHBrA0AAGJUTU2N5s6dq+zsbLlcLm3duvWUa6qqqjRhwgQlJydrxowZeuutt0KKVV9fL5/Pp5ycnAHdRyEBAIAdlgPHAHV2dio/P19VVcGXut+8ebPKy8t17733qqGhQfn5+ZozZ47a2toC1xQUFGjy5MmnHIcOHQpc88knn+h73/ueNmzYMOAc6doAAMAOh8ZIeL3efqc9Ho88nuADsEtKSlRSUnLat1y7dq1uvfVWLVy4UJK0fv16bdu2TU8//bSWL18uSWpsbDxjWt3d3Zo3b56WL1+uyy+/fMCPRYsEAAAG5eTkKD09PXBUVFSE9D49PT2qr69XcXFx4FxCQoKKi4u1a9cuW+9hWZZuvvlmXXnllVqwYEFIedAiAQCADS7rsyOc+3ViymVaWlrg/OlaI87m8OHD8vl8yszM7Hc+MzNTe/futfUef/jDH7R582ZdfPHFgfEX//Zv/6YpU6bYzoNCwhCT24hf+oOfGYtlSu/fzfyojkjuMRLnvX+620gcIJr8HnfkY7gjHyPAoQWp0tLS+hUS0XTFFVfI7/eH9R4UEgAA2BFj60iMHj1abrdbra2t/c63trYqKyvL0VhnwhgJAAAMKiws1MSJE087E8OupKQkTZ06VdXV1YFzfr9f1dXVuuyyyxzI1B7HWyR8Pp/uu+8+/fu//7taWlqUnZ2tm2++Wffcc49crthb1QsAAFsc6tqora213bXR0dGhAwcOBF43NTWpsbFRGRkZys3NVXl5uUpLSzVt2jRNnz5dlZWV6uzsDMziMMHxQmLNmjVat26dnn32WU2aNEl1dXVauHCh0tPTtXjxYqfDAQBgRhQ27aqrq1NRUVHgdXl5uSSptLRUGzdu1A033KCPP/5Yq1atUktLiwoKCrR9+/ZTBmBGkuOFxB//+Ed95zvf0VVXXSVJmjBhgn71q1+FvNIWAABD1axZs2SdZW3tRYsWadGiRcZyOpnjYyQuv/xyVVdX691335Ukvf3223rjjTdOu6BGd3e3vF5vvwMAgJjj0MqWTo2RiBWOt0gsX75cXq9XF154odxut3w+n1avXq358+cHvb6iokL333+/02kMacmfhDeVx64jFxkJI0nyd5mZYOROMPNv96VNDxmJI0nzp9QaifOTKVuMxMHg4UuK/Hh+X4LBOQMOzdoYyBiJwcDx78Cvf/1rPffcc9q0aZMaGhr07LPP6qc//ameffbZoNevWLFC7e3tgaO5udnplAAAQIQ4/jHvrrvu0vLly3XjjTdKkqZMmaKDBw+qoqJCpaWlp1x/pjXGAQCIFU6tbBlvHC8kjh07poSTmprcbnfYK2cBABBVUZi1MRg43rUxd+5crV69Wtu2bdP777+vLVu2aO3atbr66qudDgUAwKDDYMuzeOKJJ7Ry5Ur98Ic/VFtbm7Kzs/XP//zPWrVqldOhAAAYdOJtsKXjhURqaqoqKytVWVnp9FsDABA1rjDHOcTr2s5s2hWH3N1mxqNYLnPTrjwju4zESfOYiTPxy602rnLGGx9/yVgsoB8T2yKY3HohxjbtihVs2gUAAEJGIQEAgB2sbBkUXRsAANgRhd0/BwNaJAAAQMhokQAAwAZWtgyOQgIAADtY2TIoujYAAEDIaJEwpCTrh8ZidX3zK4YimSuvz035u5E4PT4zvxKHjpkbaLXzyseMxQI+L+F45Ne0MREjgBaJoGiRAADAhr4xEuEcYvonAAAIR7xN/6SQAADADpbIDopCAgAAOxgjERSFBAAANrCORHAMtgQAACGjRSIO+RPN9MP5Rxw3EkeSenxuI3GGuX1G4qQldRuJI0lFr9xpJA7TTHEyly/yH8FNxAigayMoCgkAAOwIs2sjXgsJujYAADCIdSQAABiK2EY8KAoJAADsYIxEUHRtAACAkNEiAQCADawjERwtEgAAIGS0SMSh7nQzcVx+c+vGJ8RZKb9lZnyM1gbOJPFoT+SD+AzEwBlRSAAAYAeDLYOikAAAwAbGSARHIQEAgF1xWgyEg8GWAAAgZBQSAADYYTlwsEQ2AABDk1NjJFgiGzHP7zETJ+GcXjOBJH0hpcNInI4eM/94s3eWG4kjSdVFa43FAj7veGpS5GMc90c8Bs6MQgIAADuY/hkUhQQAADYw/TM4BlsCAICQ0SIBAIAddG0ERSEBAIAdFBJB0bUBAABCRotEHPIb+q4mJvnMBJI0ItHMVFO/ZWZH0x2zfmYkDhBN7r8fj3gM63jkY/RhsGVwFBIAANhB10ZQFBIAANhBIREUYyQAAEDIaJEAAMAGxkgER4sEAAB2sPtnUBEpJD788EN997vf1ahRozR8+HBNmTJFdXV1kQgFAMCgUltbq3feeUdlZWXRTsURjndtfPrpp5o5c6aKior08ssv6wtf+IL279+vkSNHOh0KAABj6NoIzvFCYs2aNcrJydEzzzwTOJeXl+d0mEGne1KusVguQ8s7DPf0mAkkKXfEJ0bivNOeZSTO1X8w90nkSHeykTg7r3zMSBwMHq6uyK/x4PKZW8+GWRvBOd618eKLL2ratGm67rrrNGbMGF1yySV66qmnTnt9d3e3vF5vvwMAAAwOjhcS7733ntatW6fzzjtPv//97/WDH/xAixcv1rPPPhv0+oqKCqWnpweOnJwcp1MCACB8Dg22jDeOFxJ+v1+XXnqpHnroIV1yySW67bbbdOutt2r9+vVBr1+xYoXa29sDR3Nzs9MpAQAQNpcDRzxyvJAYO3asJk6c2O/cRRddpA8++CDo9R6PR2lpaf0OAAAwODg+2HLmzJnat29fv3Pvvvuuxo8f73QoAADMYbBlUI63SNxxxx3avXu3HnroIR04cECbNm3Shg0b4ma+LABgaOqb/hnOEY8cb5EoLCzUli1btGLFCj3wwAPKy8tTZWWl5s+f73SoQSXp4w5jsXrP8RiJMyHN3AybbkN7ox81NFXyfw59wUgcSXrvn+42Fgv4PCs58r+3JrcRp0UiuIh8l7/1rW/pW9/6ViTeGgAAxBA27QIAwK44bVUIB4UEAAA2sER2cOz+CQAAQkaLBAAAdjDYMigKCQAAbKBrIzi6NgAAQMhokTCk5wvnGIt1PN3Mtrp55/zNSBxJSkvsMhJn+DAzW6OPHNlpJA4QTa7j/sjH8EU+RgBdG0FRSAAAYANdG8HRtQEAwBB15MgRTZs2TQUFBZo8ebKeeuqpAb8HLRIAANgRh10bqampqqmp0YgRI9TZ2anJkyfrmmuu0ahRo2y/B4UEAAB2xGEh4Xa7NWLECElSd3e3LMuSZQ0sUbo2AACwIRq7f9bU1Gju3LnKzs6Wy+XS1q1bT7mmqqpKEyZMUHJysmbMmKG33nprQDGOHDmi/Px8jRs3TnfddZdGjx49oPspJAAAiFGdnZ3Kz89XVVVV0K9v3rxZ5eXluvfee9XQ0KD8/HzNmTNHbW1tgWv6xj+cfBw6dEiSdO655+rtt99WU1OTNm3apNbW1gHlSNeGIe6/G9zq1mXm2zoh2dz0z4960o3EOdhqv18wHO/dxNbeiH+urshPp3b5eiMeI8Chrg2v19vvtMfjkcfjCXpLSUmJSkpKTvuWa9eu1a233qqFCxdKktavX69t27bp6aef1vLlyyVJjY2NttLLzMxUfn6+Xn/9dV177bV2n4oWCQAA7HBZVtiHJOXk5Cg9PT1wVFRUhJRPT0+P6uvrVVxcHDiXkJCg4uJi7dq1y9Z7tLa26ujRo5Kk9vZ21dTU6IILLhhQHrRIAABgUHNzs9LS0gKvT9cacTaHDx+Wz+dTZmZmv/OZmZnau3evrfc4ePCgbrvttsAgy9tvv11TpkwZUB4UEgAA2OFQ10ZaWlq/QiKapk+fbrvr43QoJAAAsCHWVrYcPXq03G73KYMjW1tblZWV5WywM2CMBAAABhUWFmrixImnnYlhV1JSkqZOnarq6urAOb/fr+rqal122WUOZGoPLRIAANjhUNdGbW2t7a6Njo4OHThwIPC6qalJjY2NysjIUG5ursrLy1VaWqpp06Zp+vTpqqysVGdnZ2AWhwkUEob8fWyysVgZOUeMxBmfdNhIHEn6fx3ZxmIBcEa87f4Zja6Nuro6FRUVBV6Xl5dLkkpLS7Vx40bdcMMN+vjjj7Vq1Sq1tLSooKBA27dvP2UAZiRRSAAAEKNmzZp11iWrFy1apEWLFhnL6WSMkQAAwA7LgcPBMRKxghYJAABscKprYyBjJAYDCgkAAOyIw90/nUDXBgAACBktEgAA2OT0olLxgBYJAADssKzwDwZbIlQJx82VsZNGtRiJ8zffOUbiSFLNu18xEqfpuyuMxAGGguP7Dti4KswYlsFtxB3CYEsAAIagWNtrI1ZQSAAAYAezNoJijAQAAAgZhQQAADa4/OEfYrAlAABDVBR2/xwMaJEAAAAho0XCkKNfNPdPPS39fSNxftd6sZE4kuT+0Nw27ACcscP/m4jH8Hq9Sk9Pj3gcMWvjtCgkAACw43OLSoV8fxyikAAAwAZaJIJjjAQAAAgZhQQAAHZYDhxM/wQAYGhyqmuD6Z8AAAAnRLxF4uGHH9aKFSu0ZMkSVVZWRjrcgP2fhOuMxOn68eVG4khSzrC/GYnzlz05RuJIUsqnxkIBQHDM2ggqooVEbW2tfvGLX+jii82tNwAAQCQwayO4iHVtdHR0aP78+Xrqqac0cuTISIUBAABRFLFCoqysTFdddZWKi4vPeF13d7e8Xm+/AwCAmOPQrI14E5Gujeeff14NDQ2qra0967UVFRW6//77I5EGAACOoWsjOMdbJJqbm7VkyRI999xzSk4++/4IK1asUHt7e+Bobm52OiUAAGIG60icRX19vdra2nTppZcGzvl8PtXU1OjJJ59Ud3e33G534Gsej0cej8fpNAAAcJbf+uwI5/44XEfC8UJi9uzZ+vOf/9zv3MKFC3XhhRdq2bJl/YoIAAAGjXDHOcRp14bjhURqaqomT57c71xKSopGjRp1yvlYYGKbW0masP6nRuJIUpeVZCROxn9TFAIYOlxhjnNwOZlMDGFlSwAAEDIje228+uqrJsIAABA5rGwZFJt2AQBgA9M/g6NrAwAAhIwWCQAA7GDWRlAUEgAA2OCyLLnCGOcQzr2xjELCkEumNBmLtcv7FSNxGtbfYSQOACB2MUYCAAA7/A4cLJENAMDQ5FTXRrwtkU2LBAAACBktEgAA2MGsjaAoJAAAsIOVLYOikAAAwAZWtgyOMRIAACBktEgYUv7F3xuLVbrrFiNxnrjUSBgAiA10bQRFIQEAgA0u/2dHOPfHI7o2AABAyGiRAADADro2gqKQAADADtaRCIquDQAAEDJaJAAAsIFtxIMb8oVEyRdvNxJn9a4eI3EkKXX3cDOBbjITBgBigkNjJAoLC+V2u1VWVqaysjLn8ouSIV9IAABgUrzt/kkhAQCAHZakcNaCiM+eDQoJAADsYIxEcBQSAADYYYW5FkR81hFM/wQAAKGjRQIAADtY2TIoCgkAAOzwS3KFeX8cGvKFxKHrvmQkzqOH/tFIHEl6++d3GIsFABjahnwhAQCAHczaCI5CAgAAOxgjERSzNgAAQMhokQAAwA5aJIKikAAAwA4KiaDo2gAAACEb8i0S3mldRuL8aftFRuJIkr5qLhQADBmsIxHUkC8kAACwg+mfwVFIAABgB2MkgmKMBAAACBktEgAA2OG3JFcYrQp+WiQAABi6+ro2wjli1LFjxzR+/Hj96Ec/GvC9FBIAAAxxq1ev1le/GtqUvyHftbF39lNG4sz7kbndP3WfuVAAMHSE26oQmy0S+/fv1969ezV37lzt2bNnwPfTIgEAgB1R6NqoqanR3LlzlZ2dLZfLpa1bt55yTVVVlSZMmKDk5GTNmDFDb7311oBi/OhHP1JFRcWAc+vjeCFRUVGhwsJCpaamasyYMZo3b5727dvndBgAAOJeZ2en8vPzVVVVFfTrmzdvVnl5ue699141NDQoPz9fc+bMUVtbW+CagoICTZ48+ZTj0KFDeuGFF3T++efr/PPPDzlHx7s2XnvtNZWVlamwsFDHjx/X3XffrW984xt65513lJKS4nQ4AADM8FvhdU+cmLXh9Xr7nfZ4PPJ4PEFvKSkpUUlJyWnfcu3atbr11lu1cOFCSdL69eu1bds2Pf3001q+fLkkqbGx8bT37969W88//7x+85vfqKOjQ729vUpLS9OqVatsP5bjhcT27dv7vd64caPGjBmj+vp6fe1rX3M6HAAAZlj+z45w7peUk5PT7/S9996r++4b+OC2np4e1dfXa8WKFYFzCQkJKi4u1q5du2y9R0VFRaBbY+PGjdqzZ8+AigiZGGzZ3t4uScrIyAj69e7ubnV3dwden1ypAQAQT5qbm5WWlhZ4fbrWiLM5fPiwfD6fMjMz+53PzMzU3r17w87TrogWEn6/X0uXLtXMmTM1efLkoNdUVFTo/vvvj2QaAACEz6ElstPS0voVErHi5ptvDum+iM7aKCsr0549e/T888+f9poVK1aovb09cDQ3N0cyJQAAQuO3wj8cNHr0aLndbrW2tvY739raqqysLEdjnUnEWiQWLVqkl156STU1NRo3btxprzvTIBMTbvlgtpE4L7f8q5E4AIAIcahForCwUG63W2VlZSorKwv57ZKSkjR16lRVV1dr3rx50omegOrqai1atCj0PAfI8ULCsizdfvvt2rJli1599VXl5eU5HQIAgEGrtrbWdtdGR0eHDhw4EHjd1NSkxsZGZWRkKDc3V+Xl5SotLdW0adM0ffp0VVZWqrOzMzCLwwTHC4mysjJt2rRJL7zwglJTU9XS0iJJSk9P1/Dhw50OBwCAGVaYW4GHcGtdXZ2KiooCr8vLyyVJpaWl2rhxo2644QZ9/PHHWrVqlVpaWlRQUKDt27efMgAzkhwvJNatWydJmjVrVr/zzzzzTMgDOQAAiDqHujYGYtasWbLOct+iRYuMdmWcLCJdGwAAIDinxkjEiiG/aRcAALb4/ZLCWJDK/9m9AxkjMRhQSAAAYEcUujYGgyFfSDw3w8w24gAAxKMhX0gAAGALLRJBRXRlSwAA4oZDK1sWFhZq4sSJp90afLChRQIAAIMYbAkAwBBkWX5ZYWwjHs69sYxCAgAAO6wwN96K0zESFBIAANhhWaGtc93v/vgTs4XE1xZXyZ2UHPE49RvuiHgMAAD6sLIlAABDkd8vucIY52CxsiUAAEMXXRtBsY4EAAAIGS0SAADYYPn9ssLo2mD6JwAAQxldG0HRtQEAAEJGIQEAgB3stRFUzHZtvPrj/6u0VLeBSKwjAQCwwbIkhTP987NCIt6mf9IiAQAAQhazLRIAAMQSy2/JcoU+YNKK08GWFBIAANhh+cPs2mD6JwAAQxYtEsExRgIAAIQs5lok+io2b4eZJqCEEV4jcQAAzvN6P/sbbuLT/nGrO6zuiePqdTSfWOGyYqyt5a9//atycnKinQYAYBBpbm7WuHHjIvLeXV1dysvLU0tLS9jvlZaWprFjxyohISFuthGPuULC7/fr0KFDSk1NlcvlMhLT6/UqJydHzc3NcTW3t0+8P594xrgQ788nnjEiLMvS0aNHlZ2drYSEyPXWd3V1qaenJ+z3SUpKUnJysiM5xYqY69pISEiIWFV5NmlpaXH7y60h8HziGeNCvD+feEbHpaenRzxGcnJy3BUATmGwJQAACBmFBAAACBmFhCSPx6N7771XHo8n2qlERLw/n3jGuBDvzyeeEXEq5gZbAgCAwYMWCQAAEDIKCQAAEDIKCQAAEDIKCQAAELIhW0hUVFSosLBQqampGjNmjObNm6d9+/ZFO62Ievjhh+VyubR06dJop+KoDz/8UN/97nc1atQoDR8+XFOmTFFdXV2003KEz+fTypUrlZeXp+HDh+vLX/6yfvKTnwzqXQRramo0d+5cZWdny+VyaevWrf2+blmWVq1apbFjx2r48OEqLi7W/v37o5ZvKM70jL29vVq2bJmmTJmilJQUZWdn63vf+54OHToU1ZwH6mzfx8/7/ve/L5fLpcrKSqM5wowhW0i89tprKisr0+7du7Vjxw719vbqG9/4hjo7O6OdWkTU1tbqF7/4hS6++OJop+KoTz/9VDNnztSwYcP08ssv65133tFjjz2mkSNHRjs1R6xZs0br1q3Tk08+qb/85S9as2aNHnnkET3xxBPRTi1knZ2dys/PV1VVVdCvP/LII3r88ce1fv16vfnmm0pJSdGcOXPU1dVlPNdQnekZjx07poaGBq1cuVINDQ367W9/q3379unb3/52VHIN1dm+j322bNmi3bt3Kzs721huMMyCZVmW1dbWZkmyXnvttWin4rijR49a5513nrVjxw7r61//urVkyZJop+SYZcuWWVdccUW004iYq666yrrlllv6nbvmmmus+fPnRy0nJ0mytmzZEnjt9/utrKws69FHHw2cO3LkiOXxeKxf/epXUcoyPCc/YzBvvfWWJck6ePCgsbycdLpn/Otf/2p98YtftPbs2WONHz/e+tnPfhaV/BBZQ7ZF4mTt7e2SpIyMjGin4riysjJdddVVKi4ujnYqjnvxxRc1bdo0XXfddRozZowuueQSPfXUU9FOyzGXX365qqur9e6770qS3n77bb3xxhsqKSmJdmoR0dTUpJaWln4/q+np6ZoxY4Z27doV1dwiqb29XS6XS+eee260U3GM3+/XggULdNddd2nSpEnRTgcRFHObdkWD3+/X0qVLNXPmTE2ePDna6Tjq+eefV0NDg2pra6OdSkS89957WrduncrLy3X33XertrZWixcvVlJSkkpLS6OdXtiWL18ur9erCy+8UG63Wz6fT6tXr9b8+fOjnVpE9G3TnJmZ2e98ZmamI1s4x6Kuri4tW7ZMN910U1xt5LVmzRolJiZq8eLF0U4FEUYhceIT+549e/TGG29EOxVHNTc3a8mSJdqxY0fc7lrn9/s1bdo0PfTQQ5KkSy65RHv27NH69evjopD49a9/reeee06bNm3SpEmT1NjYqKVLlyo7Ozsunm+o6+3t1fXXXy/LsrRu3bpop+OY+vp6/fznP1dDQ4NcLle000GEDfmujUWLFumll17Szp07o7Z9eaTU19erra1Nl156qRITE5WYmKjXXntNjz/+uBITE+Xz+aKdYtjGjh2riRMn9jt30UUX6YMPPohaTk666667tHz5ct14442aMmWKFixYoDvuuEMVFRXRTi0isrKyJEmtra39zre2tga+Fi/6ioiDBw9qx44dcdUa8frrr6utrU25ubmBvz0HDx7UnXfeqQkTJkQ7PThsyLZIWJal22+/XVu2bNGrr76qvLy8aKfkuNmzZ+vPf/5zv3MLFy7UhRdeqGXLlsntdkctN6fMnDnzlGm77777rsaPHx+1nJx07NgxJST0r/fdbrf8fn/UcoqkvLw8ZWVlqbq6WgUFBZIkr9erN998Uz/4wQ+inZ5j+oqI/fv3a+fOnRo1alS0U3LUggULThmTNWfOHC1YsEALFy6MWl6IjCFbSJSVlWnTpk164YUXlJqaGuh/TU9P1/Dhw6OdniNSU1NPGfORkpKiUaNGxc1YkDvuuEOXX365HnroIV1//fV66623tGHDBm3YsCHaqTli7ty5Wr16tXJzczVp0iT96U9/0tq1a3XLLbdEO7WQdXR06MCBA4HXTU1NamxsVEZGhnJzc7V06VI9+OCDOu+885SXl6eVK1cqOztb8+bNi2reA3GmZxw7dqyuvfZaNTQ06KWXXpLP5wv8/cnIyFBSUlIUM7fvbN/Hk4ujYcOGKSsrSxdccEEUskVERXvaSLRICno888wz0U4touJt+qdlWdbvfvc7a/LkyZbH47EuvPBCa8OGDdFOyTFer9dasmSJlZubayUnJ1tf+tKXrB//+MdWd3d3tFML2c6dO4P+7pWWllrWiSmgK1eutDIzMy2Px2PNnj3b2rdvX7TTHpAzPWNTU9Np//7s3Lkz2qnbdrbv48mY/hm/2EYcAACEbMgPtgQAAKGjkAAAACGjkAAAACGjkAAAACGjkAAAACGjkAAAACGjkAAAACGjkAAAACGjkAAAACGjkAAAACGjkAAAACGjkAAAACH7/1iL6hoUmWwOAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "batch_src, batch_labels, batch_padding_mask = map(lambda x: x.to(device), mkbatch(BSZ))\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    output = model(batch_src, batch_padding_mask)\n",
    "x = batch_labels.detach().to(torch.uint8)\n",
    "y = output.detach()\n",
    "cnts = torch.bincount(x)\n",
    "weights = [1/cnts[i.item()].item() for i in x] # normalize by label count\n",
    "fig, ax = plt.subplots()\n",
    "h = ax.hist2d(x.cpu().numpy().flatten(), y.to(torch.float16).cpu().numpy().flatten(), weights=weights, bins=[15,50], norm=mpl.colors.LogNorm())\n",
    "fig.colorbar(h[3], ax=ax)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluate()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LC6Xv3YfC0Rm"
   },
   "source": [
    "# Step 5: Fine Tune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "N_TUNE_EPOCHS = 100\n",
    "TUNE_LR = 1e-5\n",
    "TUNE_WD = 0 # 1e-5\n",
    "\n",
    "tune_criterion = nn.MSELoss()\n",
    "tune_optimizer = torch.optim.Adam(model.parameters(), lr=TUNE_LR, weight_decay=TUNE_WD)\n",
    "\n",
    "tune_train_err = []\n",
    "\n",
    "# clear loss file\n",
    "open('tune_loss', 'w').close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tune_evaluate():\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    with torch.no_grad():\n",
    "        batch_src, batch_labels, batch_padding_mask = mktunebatch(BSZ)\n",
    "        output = model(batch_src, batch_padding_mask)\n",
    "        loss = criterion(output.squeeze(1), batch_labels)\n",
    "    return loss.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "for epoch in range(N_TUNE_EPOCHS):\n",
    "    model.train()\n",
    "    train_loss = 0\n",
    "    batch_src, batch_labels, batch_padding_mask = mktunebatch(BSZ)\n",
    "    optimizer.zero_grad()\n",
    "    output = model(batch_src, batch_padding_mask)\n",
    "    loss = criterion(output.squeeze(1), batch_labels)\n",
    "    train_loss = loss.item()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    \n",
    "    tune_train_err.append(train_loss)\n",
    "    with open('tune_loss', 'a') as f:\n",
    "        f.write(f\"{train_loss}\\n\")\n",
    "    print(f\"Epoch {epoch}/{N_TUNE_EPOCHS} \\t Train Err: {train_loss:.4f}\")\n",
    "\n",
    "    if epoch % 10 == 9:\n",
    "        torch.save(model.state_dict(), f\"tune_model_weights_{epoch + 1}.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.suptitle('MSE vs Epochs')\n",
    "plt.plot(tune_train_err, label='Train', color='blue')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('MSE')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "tune_evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_src, batch_labels, batch_padding_mask = mktunebatch(BSZ)\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    output = model(batch_src, batch_padding_mask)\n",
    "x = batch_labels.detach().to(torch.float16).cpu().numpy().flatten()\n",
    "y = output.detach().to(torch.float16).cpu().numpy().flatten()\n",
    "plt.hist2d(x, y, bins=50, norm=mpl.colors.LogNorm())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JtTLXn4zC1z_"
   },
   "source": [
    "# Step 6: Test generalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_src, batch_labels, batch_padding_mask = mktunebatch(BSZ, test=True)\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    output = model(batch_src, batch_padding_mask)\n",
    "print(criterion(output.squeeze(1), batch_labels).item())\n",
    "x = batch_labels.detach().to(torch.float16).cpu().numpy().flatten()\n",
    "y = output.detach().to(torch.float16).cpu().numpy().flatten()\n",
    "plt.hist2d(x, y, bins=50, norm=mpl.colors.LogNorm())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "execution_state": "idle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHgCAYAAABZ+0ykAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABPsklEQVR4nO3dd3hUZf7+8XtCSAGSkFCSIKGX0BUQCM1CE1ABURFQUBF/KCCsnbWAohusq1/ZRViaigEFBVykCoKr9CbVAEqH0ElIgADJ+f3xkIEhEBKY5Ewm79d1zTVzSmY+c8Jubp/zFIdlWZYAAAC8hI/dBQAAALgT4QYAAHgVwg0AAPAqhBsAAOBVCDcAAMCrEG4AAIBXIdwAAACvQrgBAABehXADAAC8CuEGADzcrl275HA49OGHH9pdCpAvEG4Am0ycOFEOh0OrV6+2uxQNGzZMDodDPj4+2rt3b6bjSUlJCgwMlMPh0IABA1yOHTlyRIMGDVJ0dLQCAwNVunRpNWrUSK+88oqSk5Od5z3++ONyOBxXfQQEBOTJ97yWjPBwrceIESNsrQ9AzvjaXQAAz+Hv76/Jkyfr5Zdfdtn//fffX/X848ePq2HDhkpKStKTTz6p6OhoHTt2TBs2bNCoUaP0zDPPqFixYi7vP3bs2EzvU6hQoVz4NjnXvXt3dejQIdP+2267zZZ6ANwYwg0Apw4dOlw13MTFxaljx4767rvvXPaPGzdOe/bs0W+//aamTZu6HEtKSpKfn5/LPl9fXz366KO5+A1uTv369T26PgDZw20pwMOtW7dO7du3V3BwsIoVK6ZWrVpp+fLlmc7bsGGD7rjjDgUGBqps2bJ65513NGHCBDkcDu3atStbn9WjRw+tX79ef/zxh3NfQkKCFi1apB49emQ6/88//1ShQoXUpEmTTMeCg4Pdcrvp/PnzCgsL0xNPPJHpWFJSkgICAvTiiy8693322WeqVauWihQpotDQUDVs2FBxcXE3XUeGChUq6N5779X8+fN16623KiAgQDVr1rxq69Zff/2lhx56SGFhYSpSpIiaNGmiH3/8MdN5Z8+e1bBhw1StWjUFBAQoMjJSDzzwgP78889M544ZM0aVK1eWv7+/br/9dq1atcrleEJCgp544gmVLVtW/v7+ioyMVKdOnbL9bwDwBrTcAB5s8+bNatGihYKDg/Xyyy+rcOHCGj16tO68804tWbJEjRs3liTt379fd911lxwOh4YMGaKiRYtq7Nix8vf3z9HntWzZUmXLllVcXJzefvttSdI333yjYsWKqWPHjpnOL1++vNLS0vTVV1+pd+/e2fqMo0ePZtrn5+en4ODgq55fuHBhdenSRd9//71Gjx7t0ho0Y8YMpaam6pFHHpEk/ec//9Fzzz2nBx98UIMGDdLZs2e1YcMGrVix4qrh7EqnT5++an3FixeXr++l/7vcvn27unXrpn79+ql3796aMGGCHnroIc2dO1dt2rSRJB06dEhNmzbV6dOn9dxzz6lEiRL64osvdP/992vatGnq0qWLJCktLU333nuvFi5cqEceeUSDBg3SqVOntGDBAm3atEmVK1d2fm5cXJxOnTql//f//p8cDofef/99PfDAA/rrr79UuHBhSVLXrl21efNmDRw4UBUqVNDhw4e1YMEC7dmzRxUqVLjuNQC8ggXAFhMmTLAkWatWrbrmOZ07d7b8/PysP//807nvwIEDVlBQkNWyZUvnvoEDB1oOh8Nat26dc9+xY8essLAwS5K1c+fOLGsZOnSoJck6cuSI9eKLL1pVqlRxHrv99tutJ554wrIsy5Jk9e/f33ksISHBKlWqlCXJio6Otvr162fFxcVZJ0+ezPQZvXv3tiRd9dGuXbss65s3b54lyfrvf//rsr9Dhw5WpUqVnNudOnWyatWqleV7Xc3OnTuvWZska9myZc5zy5cvb0myvvvuO+e+xMREKzIy0rrtttuc+wYPHmxJsv73v/859506dcqqWLGiVaFCBSstLc2yLMsaP368Jcn6+OOPM9WVnp7uUl+JEiWs48ePO4/PnDnT5bqcOHHCkmR98MEHOb4GgDfhthTgodLS0jR//nx17txZlSpVcu6PjIxUjx499OuvvyopKUmSNHfuXMXExOjWW291nhcWFqaePXvm+HN79OihHTt2aNWqVc7na7V6hIeH6/fff1e/fv104sQJff755+rRo4dKly6t4cOHy+ShSwICArRgwYJMj+uNRrr77rtVsmRJffPNN859J06c0IIFC9StWzfnvuLFi2vfvn2ZbtVk19NPP33V+mrWrOlyXpkyZZwtL7p4C65Xr15at26dEhISJEmzZ89Wo0aN1Lx5c+d5xYoV09NPP61du3Zpy5YtkqTvvvtOJUuW1MCBAzPV43A4XLa7deum0NBQ53aLFi2ki7e/JCkwMFB+fn5avHixTpw4cUPXAPAG3JYCPNSRI0d0+vRpVa9ePdOxGjVqKD09XXv37lWtWrW0e/duxcTEZDqvSpUqOf7c2267TdHR0YqLi1Px4sUVERGhu++++5rnR0ZGatSoUfr3v/+t7du3a968eXrvvff05ptvKjIyUk899ZTz3EKFCql169Y5rsnX11ddu3ZVXFycUlNT5e/vr++//17nz593CTevvPKKfvrpJzVq1EhVqlRR27Zt1aNHDzVr1ixbn1O1atVs1VelSpVMwaNatWrSxWHlERER2r17t/O24eVq1KghSdq9e7dq166tP//8U9WrV3e57XUt5cqVc9nOCDoZQcbf31/vvfeeXnjhBYWHh6tJkya699571atXL0VERFz3/QFvQcsNgEx69Oihb775RnFxcerWrZt8fK7/fxUOh0PVqlXTwIED9csvv8jHx0dff/2122p65JFHdOrUKc2ZM0eS9O233yo6Olr16tVznlOjRg3Fx8drypQpat68ub777js1b95cQ4cOdVsddrrWkPnLW8gGDx6sbdu2KTY2VgEBAXrjjTdUo0YNrVu3Lg8rBexFuAE8VKlSpVSkSBHFx8dnOvbHH3/Ix8dHUVFR0sWOvTt27Mh03tX2ZUePHj108OBBbdu2LVsdca9UqVIlhYaG6uDBgzf0+VfTsmVLRUZG6ptvvtHRo0e1aNEil1abDEWLFlW3bt00YcIE7dmzRx07dtS7776rs2fPuq2WHTt2ZLrltm3bNuniaCpd/J1c63eXcVySKleurPj4eJ0/f95t9VWuXFkvvPCC5s+fr02bNuncuXP66KOP3Pb+gKcj3AAeqlChQmrbtq1mzpzpMoz30KFDiouLU/PmzZ0jjNq1a6dly5Zp/fr1zvOOHz9+wy0nlStX1ieffKLY2Fg1atTomuetWLFCKSkpmfavXLlSx44du+ottRvl4+OjBx98UP/973/11Vdf6cKFC5nCzbFjx1y2/fz8VLNmTVmW5dbwcODAAU2fPt25nZSUpC+//FK33nqr8/ZPhw4dtHLlSi1btsx5XkpKisaMGaMKFSo4+/F07dpVR48e1ciRIzN9zpUB6npOnz6dKcRVrlxZQUFBSk1NzfH3BPIr+twANhs/frzmzp2baf+gQYP0zjvvaMGCBWrevLmeffZZ+fr6avTo0UpNTdX777/vPPfll1/WpEmT1KZNGw0cONA5FLxcuXI6fvx4pv4h2TFo0KDrnvPVV1/p66+/VpcuXdSgQQP5+flp69atGj9+vAICAvT3v//d5fwLFy5o0qRJV32vLl26qGjRoll+Xrdu3fTZZ59p6NChqlOnjrP/Soa2bdsqIiJCzZo1U3h4uLZu3aqRI0eqY8eOCgoKuu73Wbt27VXrq1y5skufpmrVqqlPnz5atWqVwsPDNX78eB06dEgTJkxwnvPqq69q8uTJat++vZ577jmFhYXpiy++0M6dO/Xdd985b/X16tVLX375pZ5//nmtXLlSLVq0UEpKin766Sc9++yz6tSp03XrzrBt2za1atVKDz/8sGrWrClfX19Nnz5dhw4dcg6XBwoEu4drAQVVxlDwaz327t1rWZZlrV271mrXrp1VrFgxq0iRItZdd91lLV26NNP7rVu3zmrRooXl7+9vlS1b1oqNjbX+7//+z5JkJSQkZFnL5UPBs3LlUPANGzZYL730klW/fn0rLCzM8vX1tSIjI62HHnrIWrt2rcvPZjUUPDvD1a2LQ6OjoqIsSdY777yT6fjo0aOtli1bWiVKlLD8/f2typUrWy+99JKVmJiY5ftebyh47969neeWL1/e6tixozVv3jyrbt26lr+/vxUdHW1NnTo10/v++eef1oMPPmgVL17cCggIsBo1amTNmjUr03mnT5+2XnvtNatixYpW4cKFrYiICOvBBx90TgGQUd/VhnhLsoYOHWpZlmUdPXrU6t+/vxUdHW0VLVrUCgkJsRo3bmx9++231722gDdxWDlt9wSQbwwePFijR49WcnKyx6zflN9VqFBBtWvX1qxZs+wuBcA10OcG8BJnzpxx2T527Ji++uorNW/enGADoEChzw3gJWJiYnTnnXeqRo0aOnTokMaNG6ekpCS98cYbdpcGAHmKcAN4iQ4dOmjatGkaM2aMHA6H6tevr3Hjxqlly5Z2lwYAeYo+NwAAwKvQ5wYAAHgVwg0AAPAqhBsAAOBVCDcAAMCrEG4AAIBXIdwAAACvQrgBAABehXADAAC8CuEGAAB4FcINAADwKoQbAADgVQg3AADAqxBuAACAVyHcAAAAr0K4AQAAXoVwAwAAvArhBgAAeBXCDQAA8CqEGwAA4FUINwAAwKsQbgAAgFch3AAAAK9CuAEAAF6FcAMAALwK4QYAAHgVwg0AAPAqhBsAAOBVfO0uIK+lp6frwIEDCgoKksPhsLscAACQDZZl6dSpUypTpox8fLJumylw4ebAgQOKioqyuwwAAHAD9u7dq7Jly2Z5ToELN0FBQdLFixMcHGx3OQAAIBuSkpIUFRXl/DuelQIXbjJuRQUHBxNuAADIZ7LTpYQOxQAAwKsQbgAAgFch3AAAAK9CuAEAAF6FcAMAALwK4QYAAHgVwg0AAPAqhBsAAOBVCDcAAMCrEG4AAIBXIdwAAACv4jHhZsSIEXI4HBo8ePA1z5k4caIcDofLIyAgIE/rBAAAns0jFs5ctWqVRo8erbp161733ODgYMXHxzu3s7OAVl44d046fFhKS5PKl7e7GgAACi7bW26Sk5PVs2dP/ec//1FoaOh1z3c4HIqIiHA+wsPDszw/NTVVSUlJLo/csHy5FBUltWuXK28PAACyyfZw079/f3Xs2FGtW7fO1vnJyckqX768oqKi1KlTJ23evDnL82NjYxUSEuJ8REVFualyV0WLZtSXK28PAACyydZwM2XKFK1du1axsbHZOr969eoaP368Zs6cqUmTJik9PV1NmzbVvn37rvkzQ4YMUWJiovOxd+9eN36DS4oVM88pKbny9gAAIJts63Ozd+9eDRo0SAsWLMh2p+CYmBjFxMQ4t5s2baoaNWpo9OjRGj58+FV/xt/fX/7+/m6r+1pouQEAwDPYFm7WrFmjw4cPq379+s59aWlp+uWXXzRy5EilpqaqUKFCWb5H4cKFddttt2nHjh15UHHWMlpuLlwwnYv9/OyuCACAgsm2cNOqVStt3LjRZd8TTzyh6OhovfLKK9cNNroYhjZu3KgOHTrkYqXZk9Fyo4utN2FhdlYDAEDBZVu4CQoKUu3atV32FS1aVCVKlHDu79Wrl2655RZnn5y3335bTZo0UZUqVXTy5El98MEH2r17t5566ilbvsPlChc2rTXnzpl+N4QbAADs4RHz3FzLnj175ONzqc/ziRMn1LdvXyUkJCg0NFQNGjTQ0qVLVbNmTVvrzFCsmHT8OP1uAACwk8OyLMvuIvJSUlKSQkJClJiYqODgYLe+d7ly0t690qpVUsOGbn1rAAAKtJz8/bZ9nhtvktGpmJYbAADsQ7hxo4xOxcx1AwCAfQg3bkTLDQAA9iPcuBEtNwAA2I9w40a03AAAYD/CjRvRcgMAgP0IN25Eyw0AAPYj3LgRLTcAANiPcONGtNwAAGA/wo0b0XIDAID9CDduRMsNAAD2I9y4EeEGAAD7EW7ciHADAID9CDdulLFI6alTdlcCAEDBRbhxo6Ag85yUZHclAAAUXIQbN8pouSHcAABgH8KNG2W03KSkSGlpdlcDAEDBRLhxo4yWG9GpGAAA2xBu3MjfX/LzM6+5NQUAgD0IN26WcWuKEVMAANiDcONmdCoGAMBehBs3o+UGAAB7EW7cjJYbAADsRbhxMybyAwDAXoQbN2MJBgAA7EW4cTNuSwEAYC/CjZvRoRgAAHsRbtyMlhsAAOxFuHEzOhQDAGAvwo2bhYWZ5+PH7a4EAICCiXDjZqVLm+fDh+2uBACAgolw42alSplnwg0AAPYg3LjZ5S03lmV3NQAAFDyEGzfLaLk5d45OxQAA2IFw42ZFikjFipnX3JoCACDvEW5yAZ2KAQCwD+EmFxBuAACwD+EmFxBuAACwD+EmFxBuAACwD+EmF2SEm4QEuysBAKDgIdzkgvLlzfPu3XZXAgBAwUO4yQUVKpjnXbvsrgQAgILHY8LNiBEj5HA4NHjw4CzPmzp1qqKjoxUQEKA6depo9uzZeVZjdmWEm507maUYAIC85hHhZtWqVRo9erTq1q2b5XlLly5V9+7d1adPH61bt06dO3dW586dtWnTpjyrNTsybkudPi0dPWp3NQAAFCy2h5vk5GT17NlT//nPfxQaGprluZ9++qnuuecevfTSS6pRo4aGDx+u+vXra+TIkXlWb3b4+0tlypjX3JoCACBv2R5u+vfvr44dO6p169bXPXfZsmWZzmvXrp2WLVt2zZ9JTU1VUlKSyyMvVKxonnfuzJOPAwAAF9kabqZMmaK1a9cqNjY2W+cnJCQoPDzcZV94eLgSshhzHRsbq5CQEOcjKirqpuvOjox+N3/9lScfBwAALrIt3Ozdu1eDBg3S119/rYCAgFz7nCFDhigxMdH52Lt3b6591uWqVTPP8fF58nEAAOAiX7s+eM2aNTp8+LDq16/v3JeWlqZffvlFI0eOVGpqqgoVKuTyMxERETp06JDLvkOHDikiIuKan+Pv7y9/f/9c+AZZq1nTPG/ZkucfDQBAgWZby02rVq20ceNGrV+/3vlo2LChevbsqfXr12cKNpIUExOjhQsXuuxbsGCBYmJi8rDy7KlRwzxv3cpwcAAA8pJtLTdBQUGqXbu2y76iRYuqRIkSzv29evXSLbfc4uyTM2jQIN1xxx366KOP1LFjR02ZMkWrV6/WmDFjbPkOWalaVSpUSDp1Stq/Xypb1u6KAAAoGGwfLZWVPXv26ODBg87tpk2bKi4uTmPGjFG9evU0bdo0zZgxI1NI8gR+fibg6GLrDQAAyBsOyypYN02SkpIUEhKixMREBQcH5+pnPfSQNG2aFBsrvfpqrn4UAABeLSd/vz265Sa/a97cPP/yi92VAABQcBBuctEdd5jnX3+VLlywuxoAAAoGwk0uqlNHCgkxnYrXrLG7GgAACgbCTS4qVEhq1868njjR7moAACgYCDe5rF8/8/zVV1IeLWsFAECBRrjJZXfeKVWvLqWkSP/9r93VAADg/Qg3uczhkB5+2LyeNs3uagAA8H6EmzzQtat5njtX2rXL7moAAPBuhJs8ULeu1KSJdPas9MAD0pkzdlcEAID3ItzkAYdD+uYbqWRJad06qXt36fhxu6sCAMA7EW7ySLly0rffmuHhM2eajsa7d0uJiXZXBgCAdyHc5KG77jKzFYeHSxs3ShUqSKVKSf/+t92VAQDgPQg3eaxJEzNqqlw5c7vq/Hmpf3+zgvh339ldHQAA+R/hxgbNm5tbUmlpUp8+Zt+OHdKDD5qZjHfulNLT7a4SAID8iXBjI4dDGjtW2rbtUsh54gmpUiWpcWMpOdnuCgEAyH8INx6galXp88+lxx6TihY1nY5Xr5bKlDGPQYOkhQtZWRwAgOwg3HgIX1/pyy9Na83PP0tBQWY18YMHpf/7P6l1a6lFC9PKk5ZG0AEA4FoINx6oRQtpzx5p82YzP87dd0v+/tLy5WadKl9f0yF51iy7KwUAwPMQbjxU8eJSzZpmXaqFC6Xt203rTYaDB6X77pMqV5Zef11KTTUtOgcO2Fk1AAD2c1iWZdldRF5KSkpSSEiIEhMTFRwcbHc5OZKebm5dbd8uHT0q/ec/UsZv7447pBMnzPw5Y8dKTz5pd7UAALhPTv5+E27ysT//lObMkQYOdN3v4yMVKyZFR0ujR0v16pllH2rWlAIC7KoWAIAbl5O/39yWyscqV5YGDDDz5mSoW9e08CQlSStXSo0aSTVqSA0aSK1aSfPmSS+8IB06ZGflAADkHlpuvMDWrdLLL0uvvmrmx9mzx6w8/tprZh2rq2nZ0rTi7N1rRmdt3Wr68dx9t1keQpL27zejs+6808zJAwCAXbgtlQVvDDfXYlnS7NnSrl1mFfJhw64/83FoqLmVdeaMaRU6dUrq3FkaN04KCZE2bDBD0++7T3rggbz6JgCAgo5wk4WCFG6ulJJinufMkXr1MgEmg4+PVKKEdORI9t+vb1/pk09Mi8+SJZc6NzdrZvr7AADgLoSbLBTkcHO5PXvMbaeAAOn3383trMqVpTfeMIHFx0d65RWpY0epZ08zQutqSpSQEhNdJxX09TUroPtks0dXYKD07LNSTMyl7UKF3PAlAQBeg3CTBcLN9Z0/b54LFzbPycnSO++YWZOfe848L1pkQk9CgjmnYUMpIsIMUV++/OY+PzzcDGe/996b/CIAAK9BuMkC4cZ9jh83o6/CwqS2bU2nY8syHZT37cv++/zvf9KECWYSwsu99JL07ruXQhYAoOAi3GSBcOOZzp27tGbWG29In35q9jdrJk2ZIpUta3eFAAA7Mc8N8h0/P9PXJijI9PmZNk0KDpZ++01q184sLwEAQHYQbuCRunaV1q41/W+2bJHee8/uigAA+QXhBh6rcmXpww/N63HjLg01BwAgK4QbeLSuXaUiRczQ9XXr7K4GAJAfEG7g0QIDTZ8bSZoxw+5qAAD5AeEGHq9LF/M8fbrdlQAA8gPCDTzevfeaGYs3bZJ27LC7GgCApyPcwOOFhpqVySXphx/srgYA4OkIN8gX2rQxzze7tAMAwPsRbpAvNGhgntessbsSAICnI9wgX8gIN3/9Zda0AgDgWgg3yBdCQ6VKlczrtWvtrgYA4MkIN8g3GjY0zytX2l0JAMCTEW6Qb8TEmOelS+2uBADgyWwNN6NGjVLdunUVHBys4OBgxcTEaM6cOdc8f+LEiXI4HC6PgICAPK0Z9mnWzDwvXSqlp9tdDQDAU/na+eFly5bViBEjVLVqVVmWpS+++EKdOnXSunXrVKtWrav+THBwsOLj453bDocjDyuGnW691awzdeKE9McfUs2adlcEAPBEtoab++67z2X73Xff1ahRo7R8+fJrhhuHw6GIiIg8qhCepHBhc2tq4ULpv/8l3AAArs5j+tykpaVpypQpSklJUUxG54qrSE5OVvny5RUVFaVOnTpp8+bNWb5vamqqkpKSXB7Iv7p3N89ffCFZlt3VAAA8ke3hZuPGjSpWrJj8/f3Vr18/TZ8+XTWv8Z/k1atX1/jx4zVz5kxNmjRJ6enpatq0qfbt23fN94+NjVVISIjzERUVlYvfBrntoYfMSuFbt0rvvEPAAQBk5rAse/88nDt3Tnv27FFiYqKmTZumsWPHasmSJdcMOJc7f/68atSooe7du2v48OFXPSc1NVWpqanO7aSkJEVFRSkxMVHBwcFu/S7IG++9J736qnndqpVUooQJPQ8+aHdlAIDckpSUpJCQkGz9/bY93FypdevWqly5skaPHp2t8x966CH5+vpq8uTJ2To/JxcHnuuzz6TnnnPd9+mnmfcBALxDTv5+235b6krp6ekuLS1ZSUtL08aNGxUZGZnrdcGzDBxoFtEcOVLq18/se/55adkyuysDANjN1tFSQ4YMUfv27VWuXDmdOnVKcXFxWrx4sebNmydJ6tWrl2655RbFxsZKkt5++201adJEVapU0cmTJ/XBBx9o9+7deuqpp+z8GrBJ48bmYVlmePg330gdOkhz55r9AICCydZwc/jwYfXq1UsHDx5USEiI6tatq3nz5qlNmzaSpD179sjH51Lj0okTJ9S3b18lJCQoNDRUDRo00NKlS7PVPwfey+GQxoyR9u41E/y1bi3NmiXdcYfdlQEA7OBxfW5yG31uvFdystSpk7RokRQQIM2YIbVrZ3dVAAB3yNd9boAbVayYabHp2FE6e1a6/37pww+l3bvtrgwAkJcIN/AqgYHS99+bYeHnzkkvvSRVrCjVrm06Hv/1l90VAgByG+EGXsfPT5o8WXr5ZalKFdPhePNmafRoqVo1qWdP6Z//lPbvt7tSAEBuoM8NvN62bdKGDdLYsdLFgXjSxRD09NNS375S9eqSv7+dVQIAspKvJ/HLbYSbgm3FCmnSJGnlSvPIEB5uwk94uFmQs2hRO6sEAFyJcJMFwg0kc6tq8WJp+HDp559djxUvLtWpY/rvdOoklS9v9pcrZ/YDAPIe4SYLhBtcKTHR3J5auVI6edI8rqVSJalJExNypk+Xzp83o7IqVZJKlZKCgqT33zctP6+9Zjoz//ST1Ly5WQPrSufPS/PnS/XqSWXL5urXzHV790obN0pt2kiFC9tdDQBvQ7jJAuEGWblwwbTkJCZKW7aY2Y4vXDAhZONGKS0tZ+/n52dGbRUuLJUsmfl4UpKUkmImIgwPN89XU6qUOe/CBdOydOzY9WspUsSErKNHpZAQs+/kScnHR7r7bnNs9mzz3XIiNFRKTzfXKINlSYcOmecyZcw5GRo2NOHu9GmzHRJigtzJk5Kvr+tQ/cKFpVq1zL5OnaT1683ntGkj/fCD6RAeGWnmMMrmKi2ZVK1qZrQ+etR83r33SqdOmfmR0tMvnedwmBC7Z4/UsqX5PS5Y4HrdT56UvvvOhNM33zQd2CVp+3YT9u6++8ZqBJAZ4SYLhBvcqH37pHXrzB+zw4elu+4yLTU//mj+0K5aZf5I9u5tQsu0aXZXjLxUqJDUvbt0223S669LZ86Y1sDbb7e7MsA7EG6yQLhBbklJMbMkh4eb7X37zH/ZV6sm/fmn+S//Kzkc5vi+febnryajdaBOHdPq8fPPprUhq9tYliWtXi39/ru5bbZ0qfms9u2l48dN8Dp/3rSOVKqU/e+YmmrCXOHC0j33uN5+KlbMfPfPP780n9DKldKaNaZFo0EDs2/FCmnXLvM6KMgsl5ExUm3HDlP39dStazp+59Tp0+Y2YGioaU3atcuEUsmE1YzfnWSu008/ubbmtGxpWqYuV6qUFB9v3vdKgwdLH31kWssA3BzCTRYIN0DeSk42t8AybrmlpZkg5+9vgkNg4KVzLcvchgoONvMQhYSYAHX4sAkVp06Z2acjI699Cy879QQEmFtiGbfTfH2vfduwWDHpyBETUEqVuvb7/vqr9O675lbm5erUMcEnIuLG6gVgEG6yQLgBkJsGDpQmTjQhKkPdutKSJaa/FIAbw9pSAGCTzz4zLUy//WZacsLCzCSSDz9sWooA5D7CDQDkgqZNpb//3fSRCggwI60aNzZLgQDIXYQbAMhFdeua0VOS6bz8yit2VwR4P8INAOSyV1+VXnjBvF648NKcPwByB+EGAHJZoULSBx+YpTzOnjVDzAHkHsINAOQBh8PMKyRJ//wnnYuB3ES4AYA88vzzpnPx4sVmYkYAuYNwAwB5pHx56eWXzesXXzRLNABwP8INAOShV16RoqLM4qDjxtldDeCdCDcAkIeKFLnUevOvf9H3BsgNhBsAyGO9epk1q/74Q1q0yO5qAO9DuAGAPBYcLPXubV6PHGl3NYD3IdwAgA369zfPP/wgHTxodzWAdyHcAIANatSQbr9dSk+X5s+3uxrAu2Q73HTo0EGJiYnO7REjRujkyZPO7WPHjqlmzZrurxAAvFTbtuaZcAO4V7bDzbx585Samurc/sc//qHjx487ty9cuKD4+Hj3VwgAXqpNG/P800+mBQeAe2Q73FhXjFe8chsAkDMxMVLx4tLhw6w3BbgTfW4AwCZ+ftJjj5nXY8bYXQ3gPbIdbhwOhxwOR6Z9AIAb17evef7hB+mybo0AboJvdk+0LEuPP/64/P39JUlnz55Vv379VLRoUUly6Y8DAMieOnWk6tWl+Hhp7lypWze7KwLyv2yHm94ZM05d9Oijj2Y6p1evXu6pCgAKkM6dpffekx55RAoKkjp0sLsiIH9zWAWsZ3BSUpJCQkKUmJio4OBgu8sBAK1ZIzVsaF5Xrixt3y5x1x9wlZO/3zfdoXj37t3asmWL0hnHCAA3pEEDafZs8/rPP82aUwBuXLbDzfjx4/Xxxx+77Hv66adVqVIl1alTR7Vr19bevXtzo0YA8Hrt20v33GNe16wpbdhgd0VA/pXtcDNmzBiFhoY6t+fOnasJEyboyy+/1KpVq1S8eHG99dZbuVUnAHi9Bx+89Pq11+ysBMjfsh1utm/froYZN4UlzZw5U506dVLPnj1Vv359/eMf/9DChQtzq04A8HqPPy69/755PWeOmdwPQM5lO9ycOXPGpQPP0qVL1bJlS+d2pUqVlJCQ4P4KAaCAKFRIeukls6BmWpoZFn7qlN1VAflPtsNN+fLltWbNGknS0aNHtXnzZjVr1sx5PCEhQSEhIblTJQAUIO++KxUpIi1eLI0caXc1QP6T7XDTu3dv9e/fX8OHD9dDDz2k6OhoNWjQwHl86dKlql27dm7VCQAFRps20iefmNdxcXZXA+Q/2Q43L7/8svr27avvv/9eAQEBmjp1qsvx3377Td27d8+NGgGgwHnwQbP21KZN0saNdlcD5C/ZDjc+Pj56++23tW7dOs2ZM0c1atRwOT516lT16dMnRx8+atQo1a1bV8HBwQoODlZMTIzmzJmT5c9MnTpV0dHRCggIUJ06dTQ7Y3IIAPAioaGXZiqm9QbIGVtXBS9btqxGjBihNWvWaPXq1br77rvVqVMnbd68+arnL126VN27d1efPn20bt06de7cWZ07d9amTZvyvHYAyG09epjnyZMl5kkFsi/byy9UqlQpW2/4119/3VRBYWFh+uCDD67aCtStWzelpKRo1qxZzn1NmjTRrbfeqs8//zxb78/yCwDyizNnpPBwM2Jq3DjpySftrgiwT07+fmd74cxdu3apfPny6tGjh0qXLu2OOl2kpaVp6tSpSklJUUxMzFXPWbZsmZ5//nmXfe3atdOMGTOu+b6pqakuK5YnJSW5sWoAyD2BgWYyv1dflf72N7PAZliY3VUBni/b4eabb75xLsHQvn17Pfnkk+rQoYN8fG7uztbGjRsVExOjs2fPqlixYpo+fbpq1qx51XMTEhIUHh7usi88PDzL+XViY2OZORlAvvXii6bPzYYN0ujR0pAhdlcEeL5sJ5OHHnpIc+bM0Y4dO9SgQQP97W9/U1RUlF599VVt3779hguoXr261q9frxUrVuiZZ55R7969tWXLlht+vysNGTJEiYmJzgfrXwHITwoVMgFHkj79VDp50u6KAM+X42aXW265Ra+99pq2b9+uuLg4rVixQtHR0Tpx4sQNFeDn56cqVaqoQYMGio2NVb169fTpp59e9dyIiAgdOnTIZd+hQ4cUERFxzff39/d3jsbKeABAftKtm1S1qnTokJnBGEDWbuie0tmzZzVp0iS99dZbWrFihR566CEVKVLELQWlp6e79JG5XExMTKb1qxYsWHDNPjoA4A38/KTx483rCROkffvsrgjwbDkKNytWrNDTTz+tiIgIffzxx3rggQe0f/9+TZkyRf7+/jn+8CFDhuiXX37Rrl27tHHjRg0ZMkSLFy9Wz549JUm9evXSkMtuMA8aNEhz587VRx99pD/++EPDhg3T6tWrNWDAgBx/NgDkJ82bS3feadacGj3a7moAz5btDsW1atXS4cOH1aNHDy1ZskT16tW76Q8/fPiwevXqpYMHDyokJER169bVvHnz1KZNG0nSnj17XDosN23aVHFxcXr99df197//XVWrVtWMGTNY9gFAgTBggFlvaswY6fXXpRv4b0qgQMj2PDc+Pj4qWrSofH195XA4rnne8ePH3Vmf2zHPDYD86sIFqWJFc1tq0iTpYiM3UCDkyjw3EyZMcEdtAIAb5Osr9etnWm1GjiTcANeS7ZYbb0HLDYD87NAhKSpKOn9eWrVKatjQ7oqAvJGTv9+2ri0FAMiZ8HDp4YfN63/9y+5qAM9EuAGAfCZjgOjkydLRo3ZXA3gewg0A5DONG0sNGkipqWbWYgCuCDcAkM84HJfWmPrgA+n33+2uCPAshBsAyIceeEBq08a03rRoIe3YYXdFgOfI9lDwDM8///xV9zscDgUEBKhKlSrq1KmTwsLC3FEfAOAqHA7T56Z9ezNq6l//kv75T7urAjxDjoeC33XXXVq7dq3S0tJUvXp1SdK2bdtUqFAhRUdHKz4+Xg6HQ7/++qtq1qyZW3XfMIaCA/Ams2dLHTtKYWHSgQPMWgzvlatDwTt16qTWrVvrwIEDWrNmjdasWaN9+/apTZs26t69u/bv36+WLVvqb3/72818BwBANrRrJ91yi3T8uDRzpt3VAJ4hxy03t9xyixYsWJCpVWbz5s1q27at9u/fr7Vr16pt27Y66oFjFGm5AeBt3nhDeucdqW1bad48u6sBckeuttwkJibq8OHDmfYfOXJESUlJkqTixYvr3LlzOX1rAMANePJJ8zx/vjRqlN3VAPa7odtSTz75pKZPn659+/Zp3759mj59uvr06aPOnTtLklauXKlq1arlRr0AgCtUrChljPUYMEDav9/uigB75TjcjB49Wq1atdIjjzyi8uXLq3z58nrkkUfUqlUrff7555Kk6OhojR07NjfqBQBcxYcfSs2aSenpZsVwoCC74YUzk5OT9ddff0mSKlWqpGLFirm7tlxBnxsA3mrsWKlvX6laNWnLFqlQIbsrAtwnTxbOLFasmMLCwhQWFpZvgg0AeLOHH5ZCQ6Vt26QJE+yuBrBPjsNNenq63n77bYWEhDhvSxUvXlzDhw9Xenp67lQJALiu4GDp9dfN6+eekxYutLsiwB45nqH4tdde07hx4zRixAg1a9ZMkvTrr79q2LBhOnv2rN59993cqBMAkA0DBphQM3u29P/+n2nF8WGhHRQwOe5zU6ZMGX3++ee6//77XfbPnDlTzz77rPZ7eDd9+twA8HYpKVLZstLJk9KsWWYGYyC/y9U+N8ePH1d0dHSm/dHR0Tp+/HhO3w4A4GZFi16a+2bcOLurAfJejsNNvXr1NHLkyEz7R44cqXr16rmrLgDATejRwzzPn29WDgcKkhz3uXn//ffVsWNH/fTTT4qJiZEkLVu2THv37tXs2bNzo0YAQA7ddpsUGSkdPCh16SJNnWpadICCIMctN3fccYe2bdumLl266OTJkzp58qQeeOABxcfHq0WLFrlTJQAgR3x8pPvuM6/nzJGefdbuioC8c8OT+F1p3759evvttzVmzBh3vF2uoUMxgILiwAHpxRelyZPN9pIlUsuWdlcF3Jg8mcTvSseOHdM4eq4BgMcoU0aKi5Mee8xsz5pld0VA3mD2AwDwcm3bmufFi+2uBMgbhBsA8HJ33GGeV62SPv7Y7mqA3Ee4AQAvFxVlFtOUpBdekH791e6KgNyV7aHgDzzwQJbHT5486Y56AAC5IC5OatxYSkszq4c3b253RUDuyXa4CQkJue7xXr16uaMmAICbNWgg/e9/UtOm0rffSh99JJUoYXdVQO5w21Dw/IKh4AAKKssyIWfdOmnoUGnYMLsrArLPlqHgAADP5nBIf/+7ef3JJ1JCgt0VAbmDcAMABUiXLlL9+lJiovTSS3ZXA+QOwg0AFCCFCkmff25eT5kiHT5sd0WA+xFuAKCAuf1287hwwUzwx+0peBvCDQAUQL17m+fff5eee87uagD3ItwAQAHUp490553m9YIFZv4bwFsQbgCgAAoIMKEmKEg6edK04ADegnADAAWUr6/UsqV5PX++3dUA7kO4AYAC7N57zfMnn0hJSXZXA7gH4QYACrAnn5SqVpUOHZLeecfuagD3INwAQAHm5yf985/m9SefSFu22F0RcPMINwBQwHXsKHXoIJ0/L91/v5m9GMjPbA03sbGxuv322xUUFKTSpUurc+fOio+Pz/JnJk6cKIfD4fIICAjIs5oBwBtNmCCVKyf9+acUF3dj77F7t5kz56+/3F0dkDO2hpslS5aof//+Wr58uRYsWKDz58+rbdu2SklJyfLngoODdfDgQedj9+7deVYzAHij0qWlgQPN62+/vbH36NxZ+uwzqU0bt5YG5JjDsizL7iIyHDlyRKVLl9aSJUvUMmN84hUmTpyowYMH6+TJkzf0GTlZMh0ACpLdu6UKFczq4Vu2SNHROft5h+PSa8/5ywJvkZO/3x7V5ybx4o3esLCwLM9LTk5W+fLlFRUVpU6dOmnz5s3XPDc1NVVJSUkuDwBAZuXLS+3amWDSqZOZ3A/Ijzwm3KSnp2vw4MFq1qyZateufc3zqlevrvHjx2vmzJmaNGmS0tPT1bRpU+3bt++q58fGxiokJMT5iIqKysVvAQD525dfSlFR0rZtUs+etMAgf/KY21LPPPOM5syZo19//VVly5bN9s+dP39eNWrUUPfu3TV8+PBMx1NTU5WamurcTkpKUlRUFLelAOAa1q2TmjaVzp6VfvhBuu++7P0ct6WQm/LdbakBAwZo1qxZ+vnnn3MUbCSpcOHCuu2227Rjx46rHvf391dwcLDLAwBwbbfdJg0aZF6/+KJ06lT2fu7ycAPYydZwY1mWBgwYoOnTp2vRokWqWLFijt8jLS1NGzduVGRkZK7UCAAF0SuvSGXKmNtTGaOorsfXN7erArLH1nDTv39/TZo0SXFxcQoKClJCQoISEhJ05swZ5zm9evXSkCFDnNtvv/225s+fr7/++ktr167Vo48+qt27d+upp56y6VsAgPcJDZW++ca8/vprszzD9RQqlOtlAdlia7gZNWqUEhMTdeeddyoyMtL5+Cbjf1GS9uzZo4MHDzq3T5w4ob59+6pGjRrq0KGDkpKStHTpUtWsWdOmbwEA3ql5c6lxY+nCBemLL65//uXhhj43sJPHdCjOK8xzAwDZN26c9NRT0i23SDt2SFlNCF+8+KWlG06dkooVy7MyUQDkuw7FAADP1LOnVLastH+/mX04u7LbCRnIDYQbAMA1BQRIQ4ea16+/boaJX8tl3SXFfKmwE+EGAJClPn3MjMXnzkkvvXT1cy5cMMczEG5gJ8INACBLDof06adS4cLSwoXS2LGZzzl92nWb21KwE+EGAHBd5ctL/fqZ1337SosWuR6/MtzQcgM7EW4AANny8cdShw7m9bRprscIN/AkhBsAQLb4+krPPmtez57tOpfNleHms8+kBx7IvB/IC4QbAEC23XWX5O8v7d4tvffepf1XhpjVq6Xp06XPP8/zEgHCDQAg+4oUubSo5pAhl4aGp6Rc/fzsLNsAuBvhBgCQIyNGSF26mNdxceaZ20/wJIQbAECOOBzSY4+Z15MnS+np1w4358/naWmARLgBANyI9u2lokXNsgy//34p3Pj6up539Kgt5aGAI9wAAHIsIMB0LpakBQuk5GTzukIF1/MOH8772gDCDQDghrRta57nzTMjoySpWTPXc+bNY8QU8p7Dsi6fqcD75WTJdADAtW3fLlWrdmm7UCEpPl6qUiXzuUuXSjExeVoevExO/n7TcgMAuCFVq0qPP35p+/HHpcqVr37uJ5/kWVmAfLNxDgAAV/XJJ1JamlS9uvTKK9c+b/p0s2q4n19eVoeCinADALhhISHSl19e/7zz56WhQ6VevaQaNfKiMhRk9LkBALjVt99Ko0ZJ9eub0VIbN5rh4roYhk6etLtC5Ec5+ftNyw0AwK0eftg8MvTufSncJCbaVhYKEDoUAwBy1ZW3oR57zIyqAnIL4QYAkKvKlHHdnjTJ3LICcgvhBgCQqzp2lEqXdt13+rS0datdFcHbEW4AALmqRAkpIUG6917X/ffdJz36KCuKw/0INwCAXOdwmHluLvfnn9LXX0sTJ9pVFbwV4QYAkCeioq6+f9euvK4E3o5wAwDIE2+/Ld1/vzR3rtSgwaX99L2BuxFuAAB5okwZaeZMqV07ado0qWtXs3/FCun226VWraT0dLurhDdghmIAgC2OHpVKlXLdt2YNw8RxdawKDgDweCVLZp4D58cf7aoG3oRwAwCwzRdfSE8/LdWta7Y/+MC03IwebXdlyM+4LQUAsN3Bg1L58mb18AypqZKfn51VwZNwWwoAkK9ERkoDBrjua93ahB4gpwg3AACP8NZbUp8+UtmyZvt//5MaN2aoOHKOcAMA8AhBQdLYsdLmzdLrr0u33CLt3Ss1ayZt3253dchPCDcAAI8SHCwNHy6tXy81bCidOCENHCj99JNZowq4HsINAMAjlSwpffmlVKiQNG+e1KaNVKmSmQCwYA2FQU4RbgAAHqtGDTM8vG5dM5rqzBnpoYek3r3trgyejHADAPBof/ub9Pvvpt/NCy+YfV99Jf3yi92VwVMRbgAA+ULhwtKHH0rPPGO2n33WdEDeudPuyuBpCDcAgHzljTekgAAzqqpvX6lqVWnUKLurgich3AAA8pXISGnw4EvbaWmmFYd1qZDB1nATGxur22+/XUFBQSpdurQ6d+6s+Pj46/7c1KlTFR0drYCAANWpU0ezZ8/Ok3oBAJ5h+HDp66/NPDgZt6nuvVd68EHp9Gm7q4PdbA03S5YsUf/+/bV8+XItWLBA58+fV9u2bZWSknLNn1m6dKm6d++uPn36aN26dercubM6d+6sTZs25WntAAD7+PpKPXqY2YzffVcKCTH7v/tOiouT0tPtrhB28qiFM48cOaLSpUtryZIlatmy5VXP6datm1JSUjRr1iznviZNmujWW2/V559/ft3PYOFMAPA+8+ZJ99xzabtBAzOaqkgRO6uCO+XbhTMTExMlSWFhYdc8Z9myZWrdurXLvnbt2mnZsmVXPT81NVVJSUkuDwCAd2nXTjpyRAoMNNtr1pjOxseO2V0Z7OAx4SY9PV2DBw9Ws2bNVLt27Wuel5CQoPDwcJd94eHhSrjGnNyxsbEKCQlxPqKiotxeOwDAfiVLmtaa/v3NdlycVLGiNGWK3ZUhr3lMuOnfv782bdqkKW7+VzhkyBAlJiY6H3v37nXr+wMAPEfDhtLIkWbk1K23SqdOSU89xVw4BY1HhJsBAwZo1qxZ+vnnn1U2Y637a4iIiNChQ4dc9h06dEgRERFXPd/f31/BwcEuDwCAd+vQwdyaatlSSkmRunc3E/5lMV4FXsTWcGNZlgYMGKDp06dr0aJFqlix4nV/JiYmRgsXLnTZt2DBAsXExORipQCA/MbHx7TiSNKKFaYPzq23Sp9+ymgqb2druOnfv78mTZqkuLg4BQUFKSEhQQkJCTpz5ozznF69emnIkCHO7UGDBmnu3Ln66KOP9Mcff2jYsGFavXq1BgwYYNO3AAB4qjp1pM6dL23v2GEmAJw61c6qkNtsHQrucDiuun/ChAl6/PHHJUl33nmnKlSooIkTJzqPT506Va+//rp27dqlqlWr6v3331eHDh2y9ZkMBQeAguXQIdO5uEEDs5r4rl0m9Dz8sFSlivTII3ZXiOzIyd9vj5rnJi8QbgCg4Dp+XCpfXkpOvrRvzx6JgbSeL9/OcwMAQG4KC5OGDnXd99FHdlWD3EK4AQAUKIMGmVtRGYNsR42Stm2zuyq4E+EGAFCgFC4sTZ4sHThglmw4d07q2VNavNgMHy9YnTW8E+EGAFAgORzSv/9tblWtXi3ddZeZBPAf/5D277e7OtwMwg0AoMCqWNEsutmihVS5stn3+utmtfENG+yuDjeKcAMAKNAaNjRrUm3bJl0+H+wnn0gnT9pZGW4U4QYAgIszGs+dK730ktmeMEGqWtXMk4P8hXADAMBFwcHSu+9Kt9xito8elZ57Tjp71u7KkBOEGwAALlO4sLRkifTee2b722+lmjWln38226xL5fkINwAAXKFyZenll6Vp00wrzs6d0r33mlacwEDTCRmei3ADAMA1dO0qxcdLbdtKp09Ln31m5sX5xz/srgxZIdwAAJCFokUv3ZrK8MsvZiHOAwfsrAzXQrgBAOA6QkLMrahnn720b+1a6f337awK10K4AQAgG8qWlf71Lyk09NK+SZOklBQ7q8LVEG4AAMiByZOlzp0lX1/p2DFzu2rsWLurwuUINwAA5EC7dtL06dKiRWbY+J49Ut++ZtFNeAbCDQAAN6BFC2nVKikiwmz37Stt3253VRDhBgCAG1evnrR0qZnZeN06qUYN6fbbpQUL7K6sYCPcAABwEypWNMGmbVspLU1avVp6+mkzHw7sQbgBAOAmVapkFt1cvNhs79olPfmkGU3Fcg15j3ADAIAbOBzSHXdI//mP2f76a+mxx8ysxshbhBsAANyoTx+pV69L24MHS/36SYcP21lVwUK4AQDAjRwOaeJEad8+swCnJI0eLTVqxIR/eYVwAwCAmzkcZjXxxYulMWPM7Ma7d0uvvirNnCkdP253hd7NYVmWZXcReSkpKUkhISFKTExUcHCw3eUAAAqAiROlJ564tF2ypPTf/0pNmthZVf6Sk7/ftNwAAJDLHn3UDA9v3lyKjJSOHjXbY8dKqal2V+d9aLkBACAPbdsmVa9+aXvAAHO76pZb7KzK89FyAwCAh6pWzXV75EjTJ2fFCrsq8j6EGwAA8tjLL2fe99ZbdlTinQg3AADksTfflMaNMwttDh1q9s2ZIz37rBlV9dxzrE91M+hzAwCAzd57TxoyRLIsyc/PrEvl5yd9+qlUtKj04INSYKDdVdorJ3+/ffOsKgAAcFWvvGLCy6BBlxbcPHdOeuYZ8/qvvy618OD6CDcAAHiAAQOkM2ckf38pKEh66qlLx4YNk+rVk+6/X/KhQ8l1cVsKAAAPtHatVKaMmRcnQ9++UqFCUmio9O67ZibkgoLbUgAA5HP165vnXr2kL780rzNWHJekChXMRIDIjMYtAAA82KhRZg6cXr2kgIBL+595RvrsM/OaWY5dEW4AAPBgRYqYFcW/+ML0yblwwfTHSU83Q8Zvu82MqJo0ye5KPQfhBgCAfKRQIXN7qls3s71+vZSWJv3733ZX5jnocwMAQD7073+bUVXbtkm//CItWyYdOGA6IRd0hBsAAPKhsLBLHYybNZOWLpXatpU6dZKioswkgE8+aXeV9iDcAACQzw0fbubA2bzZPDI0bCjVrWtnZfagzw0AAPnc3XebvjcPPOC6/8MPpbFjXQNPQUC4AQDAC1SpIk2bZkZQhYSYfV99ZSb+a9BA+u9/zb6pU6U+faSzZ20tN1fZGm5++eUX3XfffSpTpowcDodmzJiR5fmLFy+Ww+HI9EhISMizmgEA8FQOh1ls88QJ6eWXL+1PTTWtOl9/LT38sDR+vGnR8Va2hpuUlBTVq1dP//rXv3L0c/Hx8Tp48KDzUbp06VyrEQCA/MbhMCuNr18v7d8v1a5t5sd59NFL52zbZmeFucvWDsXt27dX+/btc/xzpUuXVvHixXOlJgAAvEW9euZ50iQzL058/KVja9dK589LhQvbVl6uyZd9bm699VZFRkaqTZs2+u2337I8NzU1VUlJSS4PAAAKknr1pK1bpc6dL+377TepXDkzhNzb5KtwExkZqc8//1zfffedvvvuO0VFRenOO+/U2rVrr/kzsbGxCgkJcT6ioqLytGYAADyBwyF9+60JOYGBZl9CgtS+vbRpk/Tmm9Lf/ibt3Gl3pTfPYVmWZXcRkuRwODR9+nR1vjxWZsMdd9yhcuXK6auvvrrq8dTUVKVetqJYUlKSoqKisrVkOgAA3mjcOOnnn82sxn/95Xrs3nsvjazyJElJSQoJCcnW3+98P4lfo0aN9Ouvv17zuL+/v/z9/fO0JgAAPFmfPuaxerUUE2M6G2eYPdv0zTl3TqpTx84qb1y+ui11NevXr1dkZKTdZQAAkO80bGgm+Pv5Zyk5WWrRwqw2Hh1tZjYePFjyjPs7OWNry01ycrJ27Njh3N65c6fWr1+vsLAwlStXTkOGDNH+/fv15ZdfSpI++eQTVaxYUbVq1dLZs2c1duxYLVq0SPPnz7fxWwAAkH9Vq2YekvTOO9I990hnzpjtTz+VZsyQHn9ceu21/DOyytZws3r1at11113O7eeff16S1Lt3b02cOFEHDx7Unj17nMfPnTunF154Qfv371eRIkVUt25d/fTTTy7vAQAAbkzLltLy5abPzZEjJtzs3i299Za0cqXpkFysmN1VXp/HdCjOKznpkAQAQEF19qzpj7N+/aV9DRpIP/4ohYfnfT05+fud7/vcAAAA9wsIMK04qanmuWRJac0aE3i2bTN9dd58Uxo1yvP65eT70VIAACB3ZAw2btzYTPZ3zz1m6HiNGqbjcYbISNcJAu1Gyw0AALiuqlVNwGnY0DXYSNIXX9hV1dURbgAAQLaEh0u//CJNmybNnCmtW2f2//ijtGKF3dVdwm0pAACQbYGBUteul7bbtpXmz5eaN5dat5bKlzfLOFSvbl+NjJYCAAA3LDlZatTIrFmVITxc2rvXvfPiMFoKAADkiWLFpIvT1DkNHGjvhH/clgIAADelZ0/pq69Mi80339hdDeEGAADcpMBAackSu6u4hNtSAADAqxBuAACAVyHcAAAAr0K4AQAAXoVwAwAAvArhBgAAeBXCDQAA8CqEGwAA4FUINwAAwKsQbgAAgFch3AAAAK9CuAEAAF6FcAMAALwK4QYAAHgVX7sLyGuWZUmSkpKS7C4FAABkU8bf7Yy/41kpcOHm1KlTkqSoqCi7SwEAADl06tQphYSEZHmOw8pOBPIi6enpOnDggIKCguRwONz63klJSYqKitLevXsVHBzs1vfGJVznvMO1zhtc57zBdc47uXGtLcvSqVOnVKZMGfn4ZN2rpsC13Pj4+Khs2bK5+hnBwcH8DycPcJ3zDtc6b3Cd8wbXOe+4+1pfr8UmAx2KAQCAVyHcAAAAr0K4cSN/f38NHTpU/v7+dpfi1bjOeYdrnTe4znmD65x37L7WBa5DMQAA8G603AAAAK9CuAEAAF6FcAMAALwK4QYAAHgVwo2b/Otf/1KFChUUEBCgxo0ba+XKlXaXlO/88ssvuu+++1SmTBk5HA7NmDHD5bhlWXrzzTcVGRmpwMBAtW7dWtu3b3c55/jx4+rZs6eCg4NVvHhx9enTR8nJyXn8TTxXbGysbr/9dgUFBal06dLq3Lmz4uPjXc45e/as+vfvrxIlSqhYsWLq2rWrDh065HLOnj171LFjRxUpUkSlS5fWSy+9pAsXLuTxt/Fso0aNUt26dZ2TmMXExGjOnDnO41zn3DFixAg5HA4NHjzYuY9r7R7Dhg2Tw+FweURHRzuPe9R1tnDTpkyZYvn5+Vnjx4+3Nm/ebPXt29cqXry4dejQIbtLy1dmz55tvfbaa9b3339vSbKmT5/ucnzEiBFWSEiINWPGDOv333+37r//fqtixYrWmTNnnOfcc889Vr169azly5db//vf/6wqVapY3bt3t+HbeKZ27dpZEyZMsDZt2mStX7/e6tChg1WuXDkrOTnZeU6/fv2sqKgoa+HChdbq1autJk2aWE2bNnUev3DhglW7dm2rdevW1rp166zZs2dbJUuWtIYMGWLTt/JMP/zwg/Xjjz9a27Zts+Lj462///3vVuHCha1NmzZZFtc5V6xcudKqUKGCVbduXWvQoEHO/Vxr9xg6dKhVq1Yt6+DBg87HkSNHnMc96ToTbtygUaNGVv/+/Z3baWlpVpkyZazY2Fhb68rPrgw36enpVkREhPXBBx849508edLy9/e3Jk+ebFmWZW3ZssWSZK1atcp5zpw5cyyHw2Ht378/j79B/nD48GFLkrVkyRLLunhNCxcubE2dOtV5ztatWy1J1rJlyyzrYgj18fGxEhISnOeMGjXKCg4OtlJTU234FvlHaGioNXbsWK5zLjh16pRVtWpVa8GCBdYdd9zhDDdca/cZOnSoVa9evase87TrzG2pm3Tu3DmtWbNGrVu3du7z8fFR69attWzZMltr8yY7d+5UQkKCy3UOCQlR48aNndd52bJlKl68uBo2bOg8p3Xr1vLx8dGKFStsqdvTJSYmSpLCwsIkSWvWrNH58+ddrnN0dLTKlSvncp3r1Kmj8PBw5znt2rVTUlKSNm/enOffIT9IS0vTlClTlJKSopiYGK5zLujfv786duzock3Fv2m32759u8qUKaNKlSqpZ8+e2rNnj+SB17nALZzpbkePHlVaWprLL0uSwsPD9ccff9hWl7dJSEiQLl7Xy4WHhzuPJSQkqHTp0i7HfX19FRYW5jwHl6Snp2vw4MFq1qyZateuLV28hn5+fipevLjLuVde56v9HnTZ7wnGxo0bFRMTo7Nnz6pYsWKaPn26atasqfXr13Od3WjKlClau3atVq1alekY/6bdp3Hjxpo4caKqV6+ugwcP6q233lKLFi20adMmj7vOhBuggOrfv782bdqkX3/91e5SvFb16tW1fv16JSYmatq0aerdu7eWLFlid1leZe/evRo0aJAWLFiggIAAu8vxau3bt3e+rlu3rho3bqzy5cvr22+/VWBgoK21XYnbUjepZMmSKlSoUKYe4YcOHVJERIRtdXmbjGuZ1XWOiIjQ4cOHXY5fuHBBx48f53dxhQEDBmjWrFn6+eefVbZsWef+iIgInTt3TidPnnQ5/8rrfLXfgy77PcHw8/NTlSpV1KBBA8XGxqpevXr69NNPuc5utGbNGh0+fFj169eXr6+vfH19tWTJEv3f//2ffH19FR4ezrXOJcWLF1e1atW0Y8cOj/s3Tbi5SX5+fmrQoIEWLlzo3Jeenq6FCxcqJibG1tq8ScWKFRUREeFynZOSkrRixQrndY6JidHJkye1Zs0a5zmLFi1Senq6GjdubEvdnsayLA0YMEDTp0/XokWLVLFiRZfjDRo0UOHChV2uc3x8vPbs2eNynTdu3OgSJBcsWKDg4GDVrFkzD79N/pOenq7U1FSusxu1atVKGzdu1Pr1652Phg0bqmfPns7XXOvckZycrD///FORkZGe92/ard2TC6gpU6ZY/v7+1sSJE60tW7ZYTz/9tFW8eHGXHuG4vlOnTlnr1q2z1q1bZ0myPv74Y2vdunXW7t27LeviUPDixYtbM2fOtDZs2GB16tTpqkPBb7vtNmvFihXWr7/+alWtWpWh4Jd55plnrJCQEGvx4sUuwzlPnz7tPKdfv35WuXLlrEWLFlmrV6+2YmJirJiYGOfxjOGcbdu2tdavX2/NnTvXKlWqFMNmr/Dqq69aS5YssXbu3Glt2LDBevXVVy2Hw2HNnz/fsrjOuery0VIW19ptXnjhBWvx4sXWzp07rd9++81q3bq1VbJkSevw4cOW5WHXmXDjJp999plVrlw5y8/Pz2rUqJG1fPlyu0vKd37++WdLUqZH7969LevicPA33njDCg8Pt/z9/a1WrVpZ8fHxLu9x7Ngxq3v37laxYsWs4OBg64knnrBOnTpl0zfyPFe7vpKsCRMmOM85c+aM9eyzz1qhoaFWkSJFrC5dulgHDx50eZ9du3ZZ7du3twIDA62SJUtaL7zwgnX+/HkbvpHnevLJJ63y5ctbfn5+VqlSpaxWrVo5g43Fdc5VV4YbrrV7dOvWzYqMjLT8/PysW265xerWrZu1Y8cO53FPus4Oy/wfHgAAgFegzw0AAPAqhBsAAOBVCDcAAMCrEG4AAIBXIdwAAACvQrgBAABehXADAAC8CuEGAAB4FcINgALJ4XBoxowZdpcBIBcQbgDkuccff1wOhyPT45577rG7NABewNfuAgAUTPfcc48mTJjgss/f39+2egB4D1puANjC399fERERLo/Q0FDp4i2jUaNGqX379goMDFSlSpU0bdo0l5/fuHGj7r77bgUGBqpEiRJ6+umnlZyc7HLO+PHjVatWLfn7+ysyMlIDBgxwOX706FF16dJFRYoUUdWqVfXDDz84j504cUI9e/ZUqVKlFBgYqKpVq2YKYwA8E+EGgEd644031LVrV/3+++/q2bOnHnnkEW3dulWSlJKSonbt2ik0NFSrVq3S1KlT9dNPP7mEl1GjRql///56+umntXHjRv3www+qUqWKy2e89dZbevjhh7VhwwZ16NBBPXv21PHjx52fv2XLFs2ZM0dbt27VqFGjVLJkyTy+CgBuiNvXGQeA6+jdu7dVqFAhq2jRoi6Pd99917Isy5Jk9evXz+VnGjdubD3zzDOWZVnWmDFjrNDQUCs5Odl5/Mcff7R8fHyshIQEy7Isq0yZMtZrr712zRokWa+//rpzOzk52ZJkzZkzx7Isy7rvvvusJ554ws3fHEBeoM8NAFvcddddGjVqlMu+sLAw5+uYmBiXYzExMVq/fr0kaevWrapXr56KFi3qPN6sWTOlp6crPj5eDodDBw4cUKtWrbKsoW7dus7XRYsWVXBwsA4fPixJeuaZZ9S1a1etXbtWbdu2VefOndW0adOb/NYA8gLhBoAtihYtmuk2kbsEBgZm67zChQu7bDscDqWnp0uS2rdvr927d2v27NlasGCBWrVqpf79++vDDz/MlZoBuA99bgB4pOXLl2farlGjhiSpRo0a+v3335WSkuI8/ttvv8nHx0fVq1dXUFCQKlSooIULF95UDaVKlVLv3r01adIkffLJJxozZsxNvR+AvEHLDQBbpKamKiEhwWWfr6+vs9Pu1KlT1bBhQzVv3lxff/21Vq5cqXHjxkmSevbsqaFDh6p3794aNmyYjhw5ooEDB+qxxx5TeHi4JGnYsGHq16+fSpcurfbt2+vUqVP67bffNHDgwGzV9+abb6pBgwaqVauWUlNTNWvWLGe4AuDZCDcAbDF37lxFRka67Ktevbr++OMP6eJIpilTpujZZ59VZGSkJk+erJo1a0qSihQponnz5mnQoEG6/fbbVaRIEXXt2lUff/yx87169+6ts2fP6p///KdefPFFlSxZUg8++GC26/Pz89OQIUO0a9cuBQYGqkWLFpoyZYrbvj+A3OOwzKgBAPAYDodD06dPV+fOne0uBUA+RJ8bAADgVQg3AADAq9DnBoDH4W45gJtByw0AAPAqhBsAAOBVCDcAAMCrEG4AAIBXIdwAAACvQrgBAABehXADAAC8CuEGAAB4lf8PJ48umnSI2bwAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with open('training-loss') as f:\n",
    "    train_err = list(map(float, f.read().split()))\n",
    "    plt.suptitle('Log MSE vs Epochs')\n",
    "    plt.plot(torch.log(torch.tensor(train_err)[:500]), label='Train', color='blue')\n",
    "    plt.xlabel('Epochs')\n",
    "    plt.ylabel('Log MSE')\n",
    "    plt.show()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}