diff options
| -rw-r--r-- | transformer_shortest_paths.ipynb | 997 |
1 files changed, 250 insertions, 747 deletions
diff --git a/transformer_shortest_paths.ipynb b/transformer_shortest_paths.ipynb index b2cea24..a158fcb 100644 --- a/transformer_shortest_paths.ipynb +++ b/transformer_shortest_paths.ipynb @@ -16,12 +16,13 @@ "Question: \n", "\n", "- Do the attention heads learn to attend to the same positional encodings\n", - "- do interp -- what is it doing? can we figure out?" + "- do interp -- what is it doing? can we figure out?\n", + "- update: I think we should do interp once it's bigger. " ] }, { "cell_type": "code", - "execution_count": 75, + "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -31,18 +32,18 @@ }, "outputs": [], "source": [ - "from collections import deque\n", "# using tqdm.auto glitches out collaborative editing\n", "from tqdm import tqdm\n", - "\n", "import torch\n", "import torch.nn as nn\n", - "from math import sqrt\n", "import matplotlib.pyplot as plt\n", - "torch.manual_seed(42)\n", "\n", + "from math import sqrt\n", + "from collections import deque\n", "import os\n", "import random\n", + "\n", + "torch.manual_seed(42)\n", "random.seed(42)\n", "\n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", @@ -51,18 +52,18 @@ }, { "cell_type": "code", - "execution_count": 91, + "execution_count": 2, "metadata": { "id": "lylOX2POPwFL" }, "outputs": [], "source": [ "# VTXS numbers here are inclusive\n", - "MIN_VTXS = 3 # 3\n", - "MAX_VTXS = 3 # 8\n", - "MAX_TUNE_VTXS = 3 # 15\n", + "MIN_VTXS = 3\n", + "MAX_VTXS = 31\n", + "MAX_TUNE_VTXS = 15 # 15\n", "AVG_DEG = 2\n", - "SEQ_LEN = MAX_VTXS * AVG_DEG + 1 # means 32 edges, final token is the target vertex\n", + "SEQ_LEN = MAX_VTXS + 1 # means 32 edges, final token is the target vertex\n", "PAD_TOKEN = 0\n", "# vertices are labelled 1,2,...,63\n", "# we also have a padding token which is 0." @@ -79,7 +80,7 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": 4, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -102,16 +103,15 @@ "def random_graph(n):\n", " edge_list = []\n", " adjacencies = [set() for _ in range(n+1)]\n", - " indices = [random.randint(1, n) for _ in range(AVG_DEG * (n-1))]\n", + " indices = [random.randint(1, n) for _ in range(AVG_DEG * n)]\n", " for i in range(0, len(indices), 2):\n", " u = indices[i]\n", " v = indices[i + 1]\n", " if u != v:\n", - " edge_list += [u,v]\n", + " edge_list += [min(u,v),max(u,v)]\n", " adjacencies[u].add(v)\n", " adjacencies[v].add(u)\n", - "\n", - " edge_list += [PAD_TOKEN]*(SEQ_LEN-len(edge_list))\n", + " edge_list += [PAD_TOKEN]*(2*SEQ_LEN-1-len(edge_list))\n", " return edge_list, adjacencies\n", "\n", "\"\"\"\n", @@ -200,27 +200,71 @@ }, { "cell_type": "code", - "execution_count": 86, + "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "(tensor([[4, 3, 2, 3, 1, 2, 0, 0, 2],\n", - " [3, 4, 3, 1, 2, 3, 0, 0, 2],\n", - " [4, 2, 3, 2, 0, 0, 0, 0, 2],\n", - " [3, 4, 4, 3, 0, 0, 0, 0, 2],\n", - " [1, 2, 2, 4, 0, 0, 0, 0, 2]], device='cuda:0'),\n", - " tensor([1., 2., 4., 4., 1.], device='cuda:0', dtype=torch.bfloat16),\n", - " tensor([[False, False, False, False, False, False, True, True, False],\n", - " [False, False, False, False, False, False, True, True, False],\n", - " [False, False, False, False, True, True, True, True, False],\n", - " [False, False, False, False, True, True, True, True, False],\n", - " [False, False, False, False, True, True, True, True, False]],\n", - " device='cuda:0'))" + "(tensor([[ 1, 4, 8, 9, 5, 8, 4, 22, 3, 18, 14, 19, 1, 2, 3, 7, 8, 17,\n", + " 1, 20, 7, 18, 21, 23, 18, 23, 8, 14, 15, 19, 1, 9, 6, 23, 11, 14,\n", + " 5, 9, 7, 11, 3, 4, 4, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 2],\n", + " [ 2, 9, 15, 18, 4, 13, 3, 18, 10, 21, 12, 20, 7, 19, 2, 3, 8, 22,\n", + " 3, 10, 4, 8, 9, 13, 15, 21, 6, 12, 7, 22, 9, 22, 3, 21, 20, 21,\n", + " 6, 18, 6, 8, 13, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 2],\n", + " [ 9, 11, 4, 11, 1, 6, 1, 4, 6, 7, 2, 5, 4, 10, 4, 6, 8, 11,\n", + " 7, 11, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 2],\n", + " [ 3, 4, 5, 10, 7, 10, 6, 7, 3, 4, 8, 9, 1, 2, 2, 3, 3, 11,\n", + " 7, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 2],\n", + " [ 3, 13, 13, 20, 15, 17, 9, 18, 1, 22, 4, 22, 9, 18, 11, 21, 4, 10,\n", + " 6, 14, 1, 15, 9, 17, 6, 17, 4, 21, 10, 21, 17, 20, 5, 7, 6, 12,\n", + " 17, 18, 1, 20, 11, 16, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 2]], device='cuda:0'),\n", + " tensor([ 1., 22., 11., 1., 22.], device='cuda:0', dtype=torch.bfloat16),\n", + " tensor([[False, False, False, False, False, False, False, False, False, False,\n", + " False, False, False, False, False, False, False, False, False, False,\n", + " False, False, False, False, False, False, False, False, False, False,\n", + " False, False, False, False, False, False, False, False, False, False,\n", + " False, False, False, False, True, True, True, True, True, True,\n", + " True, True, True, True, True, True, True, True, True, True,\n", + " True, True, False],\n", + " [False, False, False, False, False, False, False, False, False, False,\n", + " False, False, False, False, False, False, False, False, False, False,\n", + " False, False, False, False, False, False, False, False, False, False,\n", + " False, False, False, False, False, False, False, False, False, False,\n", + " False, False, True, True, True, True, True, True, True, True,\n", + " True, True, True, True, True, True, True, True, True, True,\n", + " True, True, False],\n", + " [False, False, False, False, False, False, False, False, False, False,\n", + " False, False, False, False, False, False, False, False, False, False,\n", + " False, False, True, True, True, True, True, True, True, True,\n", + " True, True, True, True, True, True, True, True, True, True,\n", + " True, True, True, True, True, True, True, True, True, True,\n", + " True, True, True, True, True, True, True, True, True, True,\n", + " True, True, False],\n", + " [False, False, False, False, False, False, False, False, False, False,\n", + " False, False, False, False, False, False, False, False, False, False,\n", + " True, True, True, True, True, True, True, True, True, True,\n", + " True, True, True, True, True, True, True, True, True, True,\n", + " True, True, True, True, True, True, True, True, True, True,\n", + " True, True, True, True, True, True, True, True, True, True,\n", + " True, True, False],\n", + " [False, False, False, False, False, False, False, False, False, False,\n", + " False, False, False, False, False, False, False, False, False, False,\n", + " False, False, False, False, False, False, False, False, False, False,\n", + " False, False, False, False, False, False, False, False, False, False,\n", + " False, False, False, False, True, True, True, True, True, True,\n", + " True, True, True, True, True, True, True, True, True, True,\n", + " True, True, False]], device='cuda:0'))" ] }, - "execution_count": 86, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -231,41 +275,39 @@ }, { "cell_type": "code", - "execution_count": 87, + "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "(array([ 663., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 284., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 39., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 1062.]),\n", - " array([1. , 1.046875, 1.09375 , 1.140625, 1.1875 , 1.234375,\n", - " 1.28125 , 1.328125, 1.375 , 1.421875, 1.46875 , 1.515625,\n", - " 1.5625 , 1.609375, 1.65625 , 1.703125, 1.75 , 1.796875,\n", - " 1.84375 , 1.890625, 1.9375 , 1.984375, 2.03125 , 2.078125,\n", - " 2.125 , 2.171875, 2.21875 , 2.265625, 2.3125 , 2.359375,\n", - " 2.40625 , 2.453125, 2.5 , 2.546875, 2.59375 , 2.640625,\n", - " 2.6875 , 2.734375, 2.78125 , 2.828125, 2.875 , 2.921875,\n", - " 2.96875 , 3.015625, 3.0625 , 3.109375, 3.15625 , 3.203125,\n", - " 3.25 , 3.296875, 3.34375 , 3.390625, 3.4375 , 3.484375,\n", - " 3.53125 , 3.578125, 3.625 , 3.671875, 3.71875 , 3.765625,\n", - " 3.8125 , 3.859375, 3.90625 , 3.953125, 4. ]),\n", + "(array([320., 0., 310., 0., 264., 0., 178., 0., 119., 0., 90.,\n", + " 0., 69., 0., 42., 0., 0., 39., 0., 30., 0., 30.,\n", + " 0., 31., 0., 26., 0., 28., 0., 27., 0., 0., 20.,\n", + " 0., 30., 0., 29., 0., 26., 0., 20., 0., 30., 0.,\n", + " 34., 0., 23., 0., 0., 35., 0., 29., 0., 17., 0.,\n", + " 26., 0., 33., 0., 35., 0., 32., 0., 26.]),\n", + " array([ 1. , 1.46875, 1.9375 , 2.40625, 2.875 , 3.34375,\n", + " 3.8125 , 4.28125, 4.75 , 5.21875, 5.6875 , 6.15625,\n", + " 6.625 , 7.09375, 7.5625 , 8.03125, 8.5 , 8.96875,\n", + " 9.4375 , 9.90625, 10.375 , 10.84375, 11.3125 , 11.78125,\n", + " 12.25 , 12.71875, 13.1875 , 13.65625, 14.125 , 14.59375,\n", + " 15.0625 , 15.53125, 16. , 16.46875, 16.9375 , 17.40625,\n", + " 17.875 , 18.34375, 18.8125 , 19.28125, 19.75 , 20.21875,\n", + " 20.6875 , 21.15625, 21.625 , 22.09375, 22.5625 , 23.03125,\n", + " 23.5 , 23.96875, 24.4375 , 24.90625, 25.375 , 25.84375,\n", + " 26.3125 , 26.78125, 27.25 , 27.71875, 28.1875 , 28.65625,\n", + " 29.125 , 29.59375, 30.0625 , 30.53125, 31. ]),\n", " <BarContainer object of 64 artists>)" ] }, - "execution_count": 87, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhJUlEQVR4nO3df1DVVeL/8ReI/NC8F7HlXhjR2HJVSjOl6GqffslKyTY5MVvssi5brjYutJKlC7NJmRXquloWqf0Sd7Ox2hndsiIJV5gSkVA2IjO3LGntwu4Y9yqtqPD+/tF4v12VwrpwOfJ8zNwZeb/P+95zz5wZnr69YIhlWZYAAAAMEhrsCQAAAJwtAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAccKCPYHu0tHRoYMHD2rQoEEKCQkJ9nQAAEAXWJalw4cPKz4+XqGhnd9nOWcD5uDBg0pISAj2NAAAwPfQ2NiooUOHdnr+nA2YQYMGSfp6AWw2W5BnAwAAusLr9SohIcH3fbwz52zAnPxnI5vNRsAAAGCY7/r4Bx/iBQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAccKCPQEAANA7XZD/WqfnPl2c3oMzOR13YAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxjnrgKmsrNRNN92k+Ph4hYSEaNOmTX7nLctSYWGh4uLiFBUVpdTUVO3bt89vzKFDh5SVlSWbzabo6GjNmDFDR44c8Rvz3nvv6f/+7/8UGRmphIQELV269OzfHQAAOCeddcC0trbq0ksvVXFx8RnPL126VCtXrtTq1atVXV2tgQMHKi0tTUePHvWNycrKUkNDg8rKyrR582ZVVlZq1qxZvvNer1dTpkzR8OHDVVtbqz/96U964IEH9NRTT32PtwgAAM41IZZlWd/74pAQbdy4UdOmTZP09d2X+Ph43XPPPbr33nslSR6PRw6HQyUlJcrMzNSePXuUlJSkmpoaJScnS5JKS0s1depUff7554qPj9eqVav0xz/+UW63W+Hh4ZKk/Px8bdq0SR9++GGX5ub1emW32+XxeGSz2b7vWwQAoM+6IP+1Ts99uji9W16zq9+/A/oZmP3798vtdis1NdV3zG63KyUlRVVVVZKkqqoqRUdH++JFklJTUxUaGqrq6mrfmKuvvtoXL5KUlpamvXv36ssvvzzja7e1tcnr9fo9AADAuSmgAeN2uyVJDofD77jD4fCdc7vdio2N9TsfFhammJgYvzFneo5vvsapioqKZLfbfY+EhIQf/oYAAECvdM78FFJBQYE8Ho/v0djYGOwpAQCAbhLQgHE6nZKkpqYmv+NNTU2+c06nU83NzX7nT5w4oUOHDvmNOdNzfPM1ThURESGbzeb3AAAA56aABkxiYqKcTqfKy8t9x7xer6qrq+VyuSRJLpdLLS0tqq2t9Y3ZunWrOjo6lJKS4htTWVmp48eP+8aUlZVp5MiRGjx4cCCnDAAADHTWAXPkyBHV1dWprq5O0tcf3K2rq9OBAwcUEhKivLw8PfTQQ3rllVdUX1+vX//614qPj/f9pNLo0aN1ww03aObMmdq5c6feeecd5ebmKjMzU/Hx8ZKkX/7ylwoPD9eMGTPU0NCgF198UY899pjmzp0bsDcOAADMFXa2F7z77ru67rrrfF+fjIrs7GyVlJRo/vz5am1t1axZs9TS0qKrrrpKpaWlioyM9F2zfv165ebmavLkyQoNDVVGRoZWrlzpO2+327Vlyxbl5ORowoQJOv/881VYWOj3u2IAAEDf9YN+D0xvxu+BAQDgh+kzvwcGAACgJxAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADBOwAOmvb1dCxYsUGJioqKionThhRdq0aJFsizLN8ayLBUWFiouLk5RUVFKTU3Vvn37/J7n0KFDysrKks1mU3R0tGbMmKEjR44EeroAAMBAAQ+YJUuWaNWqVXriiSe0Z88eLVmyREuXLtXjjz/uG7N06VKtXLlSq1evVnV1tQYOHKi0tDQdPXrUNyYrK0sNDQ0qKyvT5s2bVVlZqVmzZgV6ugAAwEAh1jdvjQTAz372MzkcDj377LO+YxkZGYqKitLzzz8vy7IUHx+ve+65R/fee68kyePxyOFwqKSkRJmZmdqzZ4+SkpJUU1Oj5ORkSVJpaammTp2qzz//XPHx8d85D6/XK7vdLo/HI5vNFsi3CABAn3BB/mudnvt0cXq3vGZXv38H/A7MxIkTVV5ero8++kiS9M9//lNvv/22brzxRknS/v375Xa7lZqa6rvGbrcrJSVFVVVVkqSqqipFR0f74kWSUlNTFRoaqurq6jO+bltbm7xer98DAACcm8IC/YT5+fnyer0aNWqU+vXrp/b2dj388MPKysqSJLndbkmSw+Hwu87hcPjOud1uxcbG+k80LEwxMTG+MacqKirSwoULA/12AABALxTwOzAvvfSS1q9frxdeeEG7du3SunXrtGzZMq1bty7QL+WnoKBAHo/H92hsbOzW1wMAAMET8Dsw8+bNU35+vjIzMyVJY8aM0WeffaaioiJlZ2fL6XRKkpqamhQXF+e7rqmpSePGjZMkOZ1ONTc3+z3viRMndOjQId/1p4qIiFBERESg3w4AAOiFAn4H5quvvlJoqP/T9uvXTx0dHZKkxMREOZ1OlZeX+857vV5VV1fL5XJJklwul1paWlRbW+sbs3XrVnV0dCglJSXQUwYAAIYJ+B2Ym266SQ8//LCGDRumiy++WLt379by5ct1xx13SJJCQkKUl5enhx56SCNGjFBiYqIWLFig+Ph4TZs2TZI0evRo3XDDDZo5c6ZWr16t48ePKzc3V5mZmV36CSQAAHBuC3jAPP7441qwYIF+97vfqbm5WfHx8brzzjtVWFjoGzN//ny1trZq1qxZamlp0VVXXaXS0lJFRkb6xqxfv165ubmaPHmyQkNDlZGRoZUrVwZ6ugAAwEAB/z0wvQW/BwYAgB+mT/0eGAAAgO5GwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDhhwZ6AiS7If63Tc58uTu/BmQAA0DdxBwYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHG6JWD+/e9/61e/+pWGDBmiqKgojRkzRu+++67vvGVZKiwsVFxcnKKiopSamqp9+/b5PcehQ4eUlZUlm82m6OhozZgxQ0eOHOmO6QIAAMMEPGC+/PJLTZo0Sf3799cbb7yhDz74QH/+8581ePBg35ilS5dq5cqVWr16taqrqzVw4EClpaXp6NGjvjFZWVlqaGhQWVmZNm/erMrKSs2aNSvQ0wUAAAYKC/QTLlmyRAkJCVq7dq3vWGJiou/PlmXp0Ucf1X333aebb75ZkvSXv/xFDodDmzZtUmZmpvbs2aPS0lLV1NQoOTlZkvT4449r6tSpWrZsmeLj4wM9bQAAYJCA34F55ZVXlJycrJ///OeKjY3VZZddpqefftp3fv/+/XK73UpNTfUds9vtSklJUVVVlSSpqqpK0dHRvniRpNTUVIWGhqq6uvqMr9vW1iav1+v3AAAA56aAB8wnn3yiVatWacSIEXrzzTc1e/Zs/f73v9e6deskSW63W5LkcDj8rnM4HL5zbrdbsbGxfufDwsIUExPjG3OqoqIi2e123yMhISHQbw0AAPQSAQ+Yjo4OjR8/Xo888oguu+wyzZo1SzNnztTq1asD/VJ+CgoK5PF4fI/GxsZufT0AABA8AQ+YuLg4JSUl+R0bPXq0Dhw4IElyOp2SpKamJr8xTU1NvnNOp1PNzc1+50+cOKFDhw75xpwqIiJCNpvN7wEAAM5NAQ+YSZMmae/evX7HPvroIw0fPlzS1x/odTqdKi8v9533er2qrq6Wy+WSJLlcLrW0tKi2ttY3ZuvWrero6FBKSkqgpwwAAAwT8J9CuvvuuzVx4kQ98sgjuvXWW7Vz50499dRTeuqppyRJISEhysvL00MPPaQRI0YoMTFRCxYsUHx8vKZNmybp6zs2N9xwg++fno4fP67c3FxlZmbyE0gAACDwAXP55Zdr48aNKigo0IMPPqjExEQ9+uijysrK8o2ZP3++WltbNWvWLLW0tOiqq65SaWmpIiMjfWPWr1+v3NxcTZ48WaGhocrIyNDKlSsDPV0AAGCgEMuyrGBPojt4vV7Z7XZ5PJ6Afx7mgvzXOj336eL0gL4WAADBEozvd139/s3/hQQAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwTrcHzOLFixUSEqK8vDzfsaNHjyonJ0dDhgzReeedp4yMDDU1Nfldd+DAAaWnp2vAgAGKjY3VvHnzdOLEie6eLgAAMEC3BkxNTY3WrFmjsWPH+h2/++679eqrr+rll19WRUWFDh48qFtuucV3vr29Xenp6Tp27Ji2b9+udevWqaSkRIWFhd05XQAAYIhuC5gjR44oKytLTz/9tAYPHuw77vF49Oyzz2r58uW6/vrrNWHCBK1du1bbt2/Xjh07JElbtmzRBx98oOeff17jxo3TjTfeqEWLFqm4uFjHjh3rrikDAABDdFvA5OTkKD09XampqX7Ha2trdfz4cb/jo0aN0rBhw1RVVSVJqqqq0pgxY+RwOHxj0tLS5PV61dDQcMbXa2trk9fr9XsAAIBzU1h3POmGDRu0a9cu1dTUnHbO7XYrPDxc0dHRfscdDofcbrdvzDfj5eT5k+fOpKioSAsXLgzA7AEAQG8X8DswjY2NmjNnjtavX6/IyMhAP32nCgoK5PF4fI/GxsYee20AANCzAh4wtbW1am5u1vjx4xUWFqawsDBVVFRo5cqVCgsLk8Ph0LFjx9TS0uJ3XVNTk5xOpyTJ6XSe9lNJJ78+OeZUERERstlsfg8AAHBuCnjATJ48WfX19aqrq/M9kpOTlZWV5ftz//79VV5e7rtm7969OnDggFwulyTJ5XKpvr5ezc3NvjFlZWWy2WxKSkoK9JQBAIBhAv4ZmEGDBumSSy7xOzZw4EANGTLEd3zGjBmaO3euYmJiZLPZdNddd8nlcunKK6+UJE2ZMkVJSUmaPn26li5dKrfbrfvuu085OTmKiIgI9JQBAIBhuuVDvN9lxYoVCg0NVUZGhtra2pSWlqYnn3zSd75fv37avHmzZs+eLZfLpYEDByo7O1sPPvhgMKYLAAB6mR4JmG3btvl9HRkZqeLiYhUXF3d6zfDhw/X6669388wAAICJ+L+QAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABgnLNgTAPq6C/Jf6/Tcp4vTe3AmAGAO7sAAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwTsADpqioSJdffrkGDRqk2NhYTZs2TXv37vUbc/ToUeXk5GjIkCE677zzlJGRoaamJr8xBw4cUHp6ugYMGKDY2FjNmzdPJ06cCPR0AQCAgQIeMBUVFcrJydGOHTtUVlam48ePa8qUKWptbfWNufvuu/Xqq6/q5ZdfVkVFhQ4ePKhbbrnFd769vV3p6ek6duyYtm/frnXr1qmkpESFhYWBni4AADBQWKCfsLS01O/rkpISxcbGqra2VldffbU8Ho+effZZvfDCC7r++uslSWvXrtXo0aO1Y8cOXXnlldqyZYs++OADvfXWW3I4HBo3bpwWLVqkP/zhD3rggQcUHh4e6GkDAACDdPtnYDwejyQpJiZGklRbW6vjx48rNTXVN2bUqFEaNmyYqqqqJElVVVUaM2aMHA6Hb0xaWpq8Xq8aGhrO+DptbW3yer1+DwAAcG7q1oDp6OhQXl6eJk2apEsuuUSS5Ha7FR4erujoaL+xDodDbrfbN+ab8XLy/MlzZ1JUVCS73e57JCQkBPjdAACA3qJbAyYnJ0fvv/++NmzY0J0vI0kqKCiQx+PxPRobG7v9NQEAQHAE/DMwJ+Xm5mrz5s2qrKzU0KFDfcedTqeOHTumlpYWv7swTU1NcjqdvjE7d+70e76TP6V0csypIiIiFBEREeB3AQAAeqOA34GxLEu5ubnauHGjtm7dqsTERL/zEyZMUP/+/VVeXu47tnfvXh04cEAul0uS5HK5VF9fr+bmZt+YsrIy2Ww2JSUlBXrKAADAMAG/A5OTk6MXXnhBf//73zVo0CDfZ1bsdruioqJkt9s1Y8YMzZ07VzExMbLZbLrrrrvkcrl05ZVXSpKmTJmipKQkTZ8+XUuXLpXb7dZ9992nnJwc7rIAAIDAB8yqVaskSddee63f8bVr1+o3v/mNJGnFihUKDQ1VRkaG2tralJaWpieffNI3tl+/ftq8ebNmz54tl8ulgQMHKjs7Ww8++GCgpwsAAAwU8ICxLOs7x0RGRqq4uFjFxcWdjhk+fLhef/31QE4NAACcI/i/kAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGCcsGBPAAAQOBfkv9bpuU8Xp/fgTIDuxR0YAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHF6dcAUFxfrggsuUGRkpFJSUrRz585gTwkAAPQCvTZgXnzxRc2dO1f333+/du3apUsvvVRpaWlqbm4O9tQAAECQ9dqAWb58uWbOnKnbb79dSUlJWr16tQYMGKDnnnsu2FMDAABB1iv/L6Rjx46ptrZWBQUFvmOhoaFKTU1VVVXVGa9pa2tTW1ub72uPxyNJ8nq9AZ9fR9tXnZ7rjtfDuY39hEBiPyGQgrGfTj6vZVnfOq5XBsx///tftbe3y+Fw+B13OBz68MMPz3hNUVGRFi5ceNrxhISEbpljZ+yP9ujL4RzHfkIgsZ8QSN29nw4fPiy73d7p+V4ZMN9HQUGB5s6d6/u6o6NDhw4d0pAhQxQSEhKw1/F6vUpISFBjY6NsNlvAnvdcxXp1HWvVdaxV17FWXcdadV13rpVlWTp8+LDi4+O/dVyvDJjzzz9f/fr1U1NTk9/xpqYmOZ3OM14TERGhiIgIv2PR0dHdNUXZbDY2+FlgvbqOteo61qrrWKuuY626rrvW6tvuvJzUKz/EGx4ergkTJqi8vNx3rKOjQ+Xl5XK5XEGcGQAA6A165R0YSZo7d66ys7OVnJysK664Qo8++qhaW1t1++23B3tqAAAgyHptwNx22236z3/+o8LCQrndbo0bN06lpaWnfbC3p0VEROj+++8/7Z+rcGasV9exVl3HWnUda9V1rFXX9Ya1CrG+6+eUAAAAeple+RkYAACAb0PAAAAA4xAwAADAOAQMAAAwDgFzisrKSt10002Kj49XSEiINm3a9J3XbNu2TePHj1dERIQuuugilZSUdPs8e4OzXatt27YpJCTktIfb7e6ZCQdRUVGRLr/8cg0aNEixsbGaNm2a9u7d+53Xvfzyyxo1apQiIyM1ZswYvf766z0w2+D6PmtVUlJy2r6KjIzsoRkHz6pVqzR27FjfLxNzuVx64403vvWavrinpLNfq766p85k8eLFCgkJUV5e3reO6+m9RcCcorW1VZdeeqmKi4u7NH7//v1KT0/Xddddp7q6OuXl5em3v/2t3nzzzW6eafCd7VqdtHfvXn3xxRe+R2xsbDfNsPeoqKhQTk6OduzYobKyMh0/flxTpkxRa2trp9ds375dv/jFLzRjxgzt3r1b06ZN07Rp0/T+++/34Mx73vdZK+nr3wj6zX312Wef9dCMg2fo0KFavHixamtr9e677+r666/XzTffrIaGhjOO76t7Sjr7tZL65p46VU1NjdasWaOxY8d+67ig7C0LnZJkbdy48VvHzJ8/37r44ov9jt12221WWlpaN86s9+nKWv3jH/+wJFlffvllj8ypN2tubrYkWRUVFZ2OufXWW6309HS/YykpKdadd97Z3dPrVbqyVmvXrrXsdnvPTaoXGzx4sPXMM8+c8Rx7yt+3rRV7yrIOHz5sjRgxwiorK7OuueYaa86cOZ2ODcbe4g7MD1RVVaXU1FS/Y2lpaaqqqgrSjHq/cePGKS4uTj/96U/1zjvvBHs6QeHxeCRJMTExnY5hb32tK2slSUeOHNHw4cOVkJDwnX+zPhe1t7drw4YNam1t7fS/XGFPfa0rayWxp3JycpSenn7anjmTYOytXvubeE3hdrtP++3ADodDXq9X//vf/xQVFRWkmfU+cXFxWr16tZKTk9XW1qZnnnlG1157raqrqzV+/PhgT6/HdHR0KC8vT5MmTdIll1zS6bjO9lZf+MzQSV1dq5EjR+q5557T2LFj5fF4tGzZMk2cOFENDQ0aOnRoD86459XX18vlcuno0aM677zztHHjRiUlJZ1xbF/fU2ezVn15T0nShg0btGvXLtXU1HRpfDD2FgGDHjNy5EiNHDnS9/XEiRP18ccfa8WKFfrrX/8axJn1rJycHL3//vt6++23gz2VXq+ra+Vyufz+Jj1x4kSNHj1aa9as0aJFi7p7mkE1cuRI1dXVyePx6G9/+5uys7NVUVHR6Tfmvuxs1qov76nGxkbNmTNHZWVlvfqDywTMD+R0OtXU1OR3rKmpSTabjbsvXXDFFVf0qW/kubm52rx5syorK7/zb3Gd7S2n09mdU+w1zmatTtW/f39ddtll+te//tVNs+s9wsPDddFFF0mSJkyYoJqaGj322GNas2bNaWP7+p46m7U6VV/aU7W1tWpubva7M97e3q7Kyko98cQTamtrU79+/fyuCcbe4jMwP5DL5VJ5ebnfsbKysm/9d1X8f3V1dYqLiwv2NLqdZVnKzc3Vxo0btXXrViUmJn7nNX11b32ftTpVe3u76uvr+8TeOlVHR4fa2trOeK6v7qnOfNtanaov7anJkyervr5edXV1vkdycrKysrJUV1d3WrxIQdpb3fbxYEMdPnzY2r17t7V7925LkrV8+XJr9+7d1meffWZZlmXl5+db06dP943/5JNPrAEDBljz5s2z9uzZYxUXF1v9+vWzSktLg/UWeszZrtWKFSusTZs2Wfv27bPq6+utOXPmWKGhodZbb70VrLfQY2bPnm3Z7XZr27Zt1hdffOF7fPXVV74x06dPt/Lz831fv/POO1ZYWJi1bNkya8+ePdb9999v9e/f36qvrw/GW+gx32etFi5caL355pvWxx9/bNXW1lqZmZlWZGSk1dDQEIy30GPy8/OtiooKa//+/dZ7771n5efnWyEhIdaWLVssy2JPfdPZrlVf3VOdOfWnkHrD3iJgTnHyR31PfWRnZ1uWZVnZ2dnWNddcc9o148aNs8LDw60f//jH1tq1a3t83sFwtmu1ZMkS68ILL7QiIyOtmJgY69prr7W2bt0anMn3sDOtkyS/vXLNNdf41u6kl156yfrJT35ihYeHWxdffLH12muv9ezEg+D7rFVeXp41bNgwKzw83HI4HNbUqVOtXbt29fzke9gdd9xhDR8+3AoPD7d+9KMfWZMnT/Z9Q7Ys9tQ3ne1a9dU91ZlTA6Y37K0Qy7Ks7ru/AwAAEHh8BgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGCc/wddyZ1+T1DuEgAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAi0ElEQVR4nO3df1BVdeL/8RegXH9eCBUurIioJZJiLSneqVxXWH7IurrSjJZb1Do6utCsUqY0pmU7i2tNP4d0drbVmpEsd1JHK0sxcVtRk3L8VYw6tNjKhVZHrmLgD873jx3v93MTlavofd/r8zFzZrj3vO+973M8E8/OPfcSYlmWJQAAAIOE+nsCAAAAP0WgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADBOJ39P4Hq0trbq+PHj6tmzp0JCQvw9HQAA0A6WZen06dOKi4tTaOjVz5EEZKAcP35c8fHx/p4GAAC4DseOHVPfvn2vOiYgA6Vnz56S/reBdrvdz7MBAADt4Xa7FR8f7/k9fjUBGSiX3tax2+0ECgAAAaY9l2dwkSwAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIzTyd8TMFH/+R9dcd13S3Jv4UwAALg9cQYFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxuGr7jsQX5EPAEDH4AwKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIzjU6AsW7ZMKSkpstvtstvtcjqd+uSTTzzrm5ubVVBQoF69eqlHjx7Ky8tTfX2913PU1tYqNzdX3bp1U3R0tObOnasLFy50zNYAAICg4FOg9O3bV0uWLFFVVZX27NmjsWPHasKECTp48KAkac6cOdqwYYPWrFmjiooKHT9+XJMmTfI8/uLFi8rNzdW5c+e0Y8cOvfPOO1q5cqUWLlzYsVsFAAACWohlWdaNPEFUVJReeuklPfTQQ+rTp4/Kysr00EMPSZK+/fZbDRkyRJWVlRo1apQ++eQT/frXv9bx48cVExMjSVq+fLnmzZunH374QeHh4e16TbfbrYiICDU2Nsput9/I9Nt0vV+4xhe1AQBwZb78/r7ua1AuXryo1atXq6mpSU6nU1VVVTp//rwyMjI8Y5KSktSvXz9VVlZKkiorKzVs2DBPnEhSVlaW3G635yxMW1paWuR2u70WAAAQvHwOlP3796tHjx6y2WyaOXOm1q5dq+TkZLlcLoWHhysyMtJrfExMjFwulyTJ5XJ5xcml9ZfWXUlJSYkiIiI8S3x8vK/TBgAAAcTnQBk8eLD27t2rXbt2adasWcrPz9ehQ4duxtw8iouL1djY6FmOHTt2U18PAAD4l89/LDA8PFyDBg2SJKWmpurLL7/U66+/rsmTJ+vcuXM6deqU11mU+vp6ORwOSZLD4dDu3bu9nu/Sp3wujWmLzWaTzWbzdaoAACBA3fD3oLS2tqqlpUWpqanq3LmzysvLPeuqq6tVW1srp9MpSXI6ndq/f78aGho8YzZv3iy73a7k5OQbnQoAAAgSPp1BKS4uVk5Ojvr166fTp0+rrKxM27Zt06effqqIiAhNmzZNRUVFioqKkt1u15NPPimn06lRo0ZJkjIzM5WcnKxHH31US5culcvl0oIFC1RQUMAZEgAA4OFToDQ0NOixxx5TXV2dIiIilJKSok8//VS/+tWvJEmvvvqqQkNDlZeXp5aWFmVlZemtt97yPD4sLEwbN27UrFmz5HQ61b17d+Xn52vx4sUdu1UAACCg3fD3oPgD34MCAEDguSXfgwIAAHCzECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA43Ty9wQg9Z//0RXXfbck9xbOBAAAM3AGBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMbxKVBKSko0YsQI9ezZU9HR0Zo4caKqq6u9xowZM0YhISFey8yZM73G1NbWKjc3V926dVN0dLTmzp2rCxcu3PjWAACAoODT3+KpqKhQQUGBRowYoQsXLujZZ59VZmamDh06pO7du3vGTZ8+XYsXL/bc7tatm+fnixcvKjc3Vw6HQzt27FBdXZ0ee+wxde7cWX/+8587YJMAAECg8ylQNm3a5HV75cqVio6OVlVVlUaPHu25v1u3bnI4HG0+x2effaZDhw5py5YtiomJ0T333KMXX3xR8+bN0/PPP6/w8PDr2AwAABBMbugalMbGRklSVFSU1/2rVq1S7969NXToUBUXF+vs2bOedZWVlRo2bJhiYmI892VlZcntduvgwYNtvk5LS4vcbrfXAgAAgpdPZ1D+r9bWVs2ePVv333+/hg4d6rn/kUceUUJCguLi4rRv3z7NmzdP1dXV+vDDDyVJLpfLK04keW67XK42X6ukpEQvvPDC9U4VAAAEmOsOlIKCAh04cEBffPGF1/0zZszw/Dxs2DDFxsYqPT1dR48e1cCBA6/rtYqLi1VUVOS57Xa7FR8ff30TBwAAxruut3gKCwu1ceNGff755+rbt+9Vx6alpUmSjhw5IklyOByqr6/3GnPp9pWuW7HZbLLb7V4LAAAIXj4FimVZKiws1Nq1a7V161YlJiZe8zF79+6VJMXGxkqSnE6n9u/fr4aGBs+YzZs3y263Kzk52ZfpAACAIOXTWzwFBQUqKyvT+vXr1bNnT881IxEREeratauOHj2qsrIyjRs3Tr169dK+ffs0Z84cjR49WikpKZKkzMxMJScn69FHH9XSpUvlcrm0YMECFRQUyGazdfwWAgCAgOPTGZRly5apsbFRY8aMUWxsrGd5//33JUnh4eHasmWLMjMzlZSUpKeeekp5eXnasGGD5znCwsK0ceNGhYWFyel06ne/+50ee+wxr+9NAQAAtzefzqBYlnXV9fHx8aqoqLjm8yQkJOjjjz/25aUBAMBthL/FAwAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADCOT4FSUlKiESNGqGfPnoqOjtbEiRNVXV3tNaa5uVkFBQXq1auXevTooby8PNXX13uNqa2tVW5urrp166bo6GjNnTtXFy5cuPGtAQAAQcGnQKmoqFBBQYF27typzZs36/z588rMzFRTU5NnzJw5c7RhwwatWbNGFRUVOn78uCZNmuRZf/HiReXm5urcuXPasWOH3nnnHa1cuVILFy7suK0CAAABrZMvgzdt2uR1e+XKlYqOjlZVVZVGjx6txsZGvf322yorK9PYsWMlSStWrNCQIUO0c+dOjRo1Sp999pkOHTqkLVu2KCYmRvfcc49efPFFzZs3T88//7zCw8M7busAAEBAuqFrUBobGyVJUVFRkqSqqiqdP39eGRkZnjFJSUnq16+fKisrJUmVlZUaNmyYYmJiPGOysrLkdrt18ODBNl+npaVFbrfbawEAAMHrugOltbVVs2fP1v3336+hQ4dKklwul8LDwxUZGek1NiYmRi6XyzPm/8bJpfWX1rWlpKREERERniU+Pv56pw0AAALAdQdKQUGBDhw4oNWrV3fkfNpUXFysxsZGz3Ls2LGb/poAAMB/fLoG5ZLCwkJt3LhR27dvV9++fT33OxwOnTt3TqdOnfI6i1JfXy+Hw+EZs3v3bq/nu/Qpn0tjfspms8lms13PVAEAQADy6QyKZVkqLCzU2rVrtXXrViUmJnqtT01NVefOnVVeXu65r7q6WrW1tXI6nZIkp9Op/fv3q6GhwTNm8+bNstvtSk5OvpFtAQAAQcKnMygFBQUqKyvT+vXr1bNnT881IxEREeratasiIiI0bdo0FRUVKSoqSna7XU8++aScTqdGjRolScrMzFRycrIeffRRLV26VC6XSwsWLFBBQQFnSQAAgCQfA2XZsmWSpDFjxnjdv2LFCj3++OOSpFdffVWhoaHKy8tTS0uLsrKy9NZbb3nGhoWFaePGjZo1a5acTqe6d++u/Px8LV68+Ma2BAAABA2fAsWyrGuO6dKli0pLS1VaWnrFMQkJCfr44499eWkAAHAb4W/xAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjdPL3BHD9+s//6IrrvluSewtnAgBAx+IMCgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjONzoGzfvl3jx49XXFycQkJCtG7dOq/1jz/+uEJCQryW7OxsrzEnT57U1KlTZbfbFRkZqWnTpunMmTM3tCEAACB4+BwoTU1NGj58uEpLS684Jjs7W3V1dZ7lvffe81o/depUHTx4UJs3b9bGjRu1fft2zZgxw/fZAwCAoNTJ1wfk5OQoJyfnqmNsNpscDkeb67755htt2rRJX375pe677z5J0ptvvqlx48bp5ZdfVlxcnK9TAgAAQeamXIOybds2RUdHa/DgwZo1a5ZOnDjhWVdZWanIyEhPnEhSRkaGQkNDtWvXrjafr6WlRW6322sBAADBq8MDJTs7W++++67Ky8v1l7/8RRUVFcrJydHFixclSS6XS9HR0V6P6dSpk6KiouRyudp8zpKSEkVERHiW+Pj4jp42AAAwiM9v8VzLlClTPD8PGzZMKSkpGjhwoLZt26b09PTres7i4mIVFRV5brvdbiIFAIAgdtM/ZjxgwAD17t1bR44ckSQ5HA41NDR4jblw4YJOnjx5xetWbDab7Ha71wIAAILXTQ+U77//XidOnFBsbKwkyel06tSpU6qqqvKM2bp1q1pbW5WWlnazpwMAAAKAz2/xnDlzxnM2RJJqamq0d+9eRUVFKSoqSi+88ILy8vLkcDh09OhRPfPMMxo0aJCysrIkSUOGDFF2dramT5+u5cuX6/z58yosLNSUKVP4BA8AAJB0HWdQ9uzZo3vvvVf33nuvJKmoqEj33nuvFi5cqLCwMO3bt0+/+c1vdNddd2natGlKTU3VP//5T9lsNs9zrFq1SklJSUpPT9e4ceP0wAMP6K9//WvHbRUAAAhoPp9BGTNmjCzLuuL6Tz/99JrPERUVpbKyMl9fGgAA3Cb4WzwAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIzTyd8TwK3Xf/5HV1z33ZLcWzgTAADaxhkUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxfA6U7du3a/z48YqLi1NISIjWrVvntd6yLC1cuFCxsbHq2rWrMjIydPjwYa8xJ0+e1NSpU2W32xUZGalp06bpzJkzN7QhAAAgePgcKE1NTRo+fLhKS0vbXL906VK98cYbWr58uXbt2qXu3bsrKytLzc3NnjFTp07VwYMHtXnzZm3cuFHbt2/XjBkzrn8rAABAUOnk6wNycnKUk5PT5jrLsvTaa69pwYIFmjBhgiTp3XffVUxMjNatW6cpU6bom2++0aZNm/Tll1/qvvvukyS9+eabGjdunF5++WXFxcXdwOYAAIBg0KHXoNTU1MjlcikjI8NzX0REhNLS0lRZWSlJqqysVGRkpCdOJCkjI0OhoaHatWtXm8/b0tIit9vttQAAgODVoYHicrkkSTExMV73x8TEeNa5XC5FR0d7re/UqZOioqI8Y36qpKREERERniU+Pr4jpw0AAAwTEJ/iKS4uVmNjo2c5duyYv6cEAABuog4NFIfDIUmqr6/3ur++vt6zzuFwqKGhwWv9hQsXdPLkSc+Yn7LZbLLb7V4LAAAIXh0aKImJiXI4HCovL/fc53a7tWvXLjmdTkmS0+nUqVOnVFVV5RmzdetWtba2Ki0trSOnAwAAApTPn+I5c+aMjhw54rldU1OjvXv3KioqSv369dPs2bP1pz/9SXfeeacSExP13HPPKS4uThMnTpQkDRkyRNnZ2Zo+fbqWL1+u8+fPq7CwUFOmTOETPAAAQNJ1BMqePXv0y1/+0nO7qKhIkpSfn6+VK1fqmWeeUVNTk2bMmKFTp07pgQce0KZNm9SlSxfPY1atWqXCwkKlp6crNDRUeXl5euONNzpgcwAAQDDwOVDGjBkjy7KuuD4kJESLFy/W4sWLrzgmKipKZWVlvr40AAC4TQTEp3gAAMDthUABAADGIVAAAIBxCBQAAGAcny+Sxe2r//yPrrjuuyW5t3AmAIBgxxkUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGKeTvyeA4Nd//kdXXPfdktxbOBMAQKDgDAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgdHijPP/+8QkJCvJakpCTP+ubmZhUUFKhXr17q0aOH8vLyVF9f39HTAAAAAeymnEG5++67VVdX51m++OILz7o5c+Zow4YNWrNmjSoqKnT8+HFNmjTpZkwDAAAEqE435Uk7dZLD4bjs/sbGRr399tsqKyvT2LFjJUkrVqzQkCFDtHPnTo0aNepmTAcAAASYm3IG5fDhw4qLi9OAAQM0depU1dbWSpKqqqp0/vx5ZWRkeMYmJSWpX79+qqysvOLztbS0yO12ey0AACB4dXigpKWlaeXKldq0aZOWLVummpoaPfjggzp9+rRcLpfCw8MVGRnp9ZiYmBi5XK4rPmdJSYkiIiI8S3x8fEdPGwAAGKTD3+LJycnx/JySkqK0tDQlJCTogw8+UNeuXa/rOYuLi1VUVOS57Xa7iRQAAILYTf+YcWRkpO666y4dOXJEDodD586d06lTp7zG1NfXt3nNyiU2m012u91rAQAAweumB8qZM2d09OhRxcbGKjU1VZ07d1Z5eblnfXV1tWpra+V0Om/2VAAAQIDo8Ld4nn76aY0fP14JCQk6fvy4Fi1apLCwMD388MOKiIjQtGnTVFRUpKioKNntdj355JNyOp18ggcAAHh0eKB8//33evjhh3XixAn16dNHDzzwgHbu3Kk+ffpIkl599VWFhoYqLy9PLS0tysrK0ltvvdXR0wAAAAGswwNl9erVV13fpUsXlZaWqrS0tKNfGgAABAn+Fg8AADAOgQIAAIxzU77qHugI/ed/dMV13y3JvYUzAQDcapxBAQAAxiFQAACAcXiLB7cV3jYCgMDAGRQAAGAcAgUAABiHQAEAAMYhUAAAgHG4SBYAcEtxsXrb2C/eCBQAuA3wyw+BhkABAKADEYMdg0ABACCABWsQESiAoYL1PzoA0B4ECtBO1xsMhMatczvs69thGwGJQAEAfukHCP6dbi8ECgBJ/McfgFn4ojYAAGAczqAAuCGceQFwMxAoQJAhGG4d9nVwux3+fU3eRt7iAQAAxiFQAACAcXiLBwBwRSa/BYDgxhkUAABgHAIFAAAYh7d4ABgn2N9WuNr2ScGxjcCN4gwKAAAwDoECAACMQ6AAAADjcA0KACCoBfs1TcGKMygAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4fg2U0tJS9e/fX126dFFaWpp2797tz+kAAABD+C1Q3n//fRUVFWnRokX66quvNHz4cGVlZamhocFfUwIAAIbwW6C88sormj59up544gklJydr+fLl6tatm/7+97/7a0oAAMAQnfzxoufOnVNVVZWKi4s994WGhiojI0OVlZWXjW9paVFLS4vndmNjoyTJ7XbflPm1tpy94rqrvSaPM+NxV3Mjzxko2x8oj7uaQNmGm/E4f7wm/7635+Nu9LHX49JzWpZ17cGWH/znP/+xJFk7duzwun/u3LnWyJEjLxu/aNEiSxILCwsLCwtLECzHjh27Ziv45QyKr4qLi1VUVOS53draqpMnT6pXr14KCQm5bLzb7VZ8fLyOHTsmu91+K6cacNhX7ce+aj/2Vfuxr9qPfeUbE/eXZVk6ffq04uLirjnWL4HSu3dvhYWFqb6+3uv++vp6ORyOy8bbbDbZbDav+yIjI6/5Ona73Zh/FNOxr9qPfdV+7Kv2Y1+1H/vKN6btr4iIiHaN88tFsuHh4UpNTVV5ebnnvtbWVpWXl8vpdPpjSgAAwCB+e4unqKhI+fn5uu+++zRy5Ei99tprampq0hNPPOGvKQEAAEP4LVAmT56sH374QQsXLpTL5dI999yjTZs2KSYm5oaf22azadGiRZe9LYTLsa/aj33Vfuyr9mNftR/7yjeBvr9CLKs9n/UBAAC4dfhbPAAAwDgECgAAMA6BAgAAjEOgAAAA4wRdoJSWlqp///7q0qWL0tLStHv3bn9PyTjPP/+8QkJCvJakpCR/T8sY27dv1/jx4xUXF6eQkBCtW7fOa71lWVq4cKFiY2PVtWtXZWRk6PDhw/6ZrJ9da189/vjjlx1r2dnZ/pmsH5WUlGjEiBHq2bOnoqOjNXHiRFVXV3uNaW5uVkFBgXr16qUePXooLy/vsi+zvF20Z3+NGTPmsmNr5syZfpqx/yxbtkwpKSmeL2NzOp365JNPPOsD+bgKqkB5//33VVRUpEWLFumrr77S8OHDlZWVpYaGBn9PzTh333236urqPMsXX3zh7ykZo6mpScOHD1dpaWmb65cuXao33nhDy5cv165du9S9e3dlZWWpubn5Fs/U/661ryQpOzvb61h77733buEMzVBRUaGCggLt3LlTmzdv1vnz55WZmammpibPmDlz5mjDhg1as2aNKioqdPz4cU2aNMmPs/af9uwvSZo+fbrXsbV06VI/zdh/+vbtqyVLlqiqqkp79uzR2LFjNWHCBB08eFBSgB9XHfLX/wwxcuRIq6CgwHP74sWLVlxcnFVSUuLHWZln0aJF1vDhw/09jYAgyVq7dq3ndmtrq+VwOKyXXnrJc9+pU6csm81mvffee36YoTl+uq8sy7Ly8/OtCRMm+GU+JmtoaLAkWRUVFZZl/e8Y6ty5s7VmzRrPmG+++caSZFVWVvprmsb46f6yLMv6xS9+Yf3xj3/036QMdscdd1h/+9vfAv64CpozKOfOnVNVVZUyMjI894WGhiojI0OVlZV+nJmZDh8+rLi4OA0YMEBTp05VbW2tv6cUEGpqauRyubyOs4iICKWlpXGcXcG2bdsUHR2twYMHa9asWTpx4oS/p+R3jY2NkqSoqChJUlVVlc6fP+91XCUlJalfv34cV7p8f12yatUq9e7dW0OHDlVxcbHOnj3rj+kZ4+LFi1q9erWamprkdDoD/rgKiL9m3B7//e9/dfHixcu+iTYmJkbffvutn2ZlprS0NK1cuVKDBw9WXV2dXnjhBT344IM6cOCAevbs6e/pGc3lcklSm8fZpXX4/7KzszVp0iQlJibq6NGjevbZZ5WTk6PKykqFhYX5e3p+0draqtmzZ+v+++/X0KFDJf3vuAoPD7/sj6ByXLW9vyTpkUceUUJCguLi4rRv3z7NmzdP1dXV+vDDD/04W//Yv3+/nE6nmpub1aNHD61du1bJycnau3dvQB9XQRMoaL+cnBzPzykpKUpLS1NCQoI++OADTZs2zY8zQ7CZMmWK5+dhw4YpJSVFAwcO1LZt25Senu7HmflPQUGBDhw4wHVf7XSl/TVjxgzPz8OGDVNsbKzS09N19OhRDRw48FZP068GDx6svXv3qrGxUf/4xz+Un5+viooKf0/rhgXNWzy9e/dWWFjYZVcn19fXy+Fw+GlWgSEyMlJ33XWXjhw54u+pGO/SscRxdn0GDBig3r1737bHWmFhoTZu3KjPP/9cffv29dzvcDh07tw5nTp1ymv87X5cXWl/tSUtLU2SbstjKzw8XIMGDVJqaqpKSko0fPhwvf766wF/XAVNoISHhys1NVXl5eWe+1pbW1VeXi6n0+nHmZnvzJkzOnr0qGJjY/09FeMlJibK4XB4HWdut1u7du3iOGuH77//XidOnLjtjjXLslRYWKi1a9dq69atSkxM9Fqfmpqqzp07ex1X1dXVqq2tvS2Pq2vtr7bs3btXkm67Y6stra2tamlpCfzjyt9X6Xak1atXWzabzVq5cqV16NAha8aMGVZkZKTlcrn8PTWjPPXUU9a2bdusmpoa61//+peVkZFh9e7d22poaPD31Ixw+vRp6+uvv7a+/vprS5L1yiuvWF9//bX173//27Isy1qyZIkVGRlprV+/3tq3b581YcIEKzEx0frxxx/9PPNb72r76vTp09bTTz9tVVZWWjU1NdaWLVusn//859add95pNTc3+3vqt9SsWbOsiIgIa9u2bVZdXZ1nOXv2rGfMzJkzrX79+llbt2619uzZYzmdTsvpdPpx1v5zrf115MgRa/HixdaePXusmpoaa/369daAAQOs0aNH+3nmt978+fOtiooKq6amxtq3b581f/58KyQkxPrss88sywrs4yqoAsWyLOvNN9+0+vXrZ4WHh1sjR460du7c6e8pGWfy5MlWbGysFR4ebv3sZz+zJk+ebB05csTf0zLG559/bkm6bMnPz7cs638fNX7uueesmJgYy2azWenp6VZ1dbV/J+0nV9tXZ8+etTIzM60+ffpYnTt3thISEqzp06fflv/D0NY+kmStWLHCM+bHH3+0/vCHP1h33HGH1a1bN+u3v/2tVVdX579J+9G19ldtba01evRoKyoqyrLZbNagQYOsuXPnWo2Njf6duB/8/ve/txISEqzw8HCrT58+Vnp6uidOLCuwj6sQy7KsW3e+BgAA4NqC5hoUAAAQPAgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxvl/F3uXZdNo9IkAAAAASUVORK5CYII=", "text/plain": [ "<Figure size 640x480 with 1 Axes>" ] @@ -280,40 +322,40 @@ }, { "cell_type": "code", - "execution_count": 90, + "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "(array([1714., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 305., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + "(array([1289., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 477., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 192., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 55., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 28., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 5.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 29.]),\n", - " array([1. , 1.03125, 1.0625 , 1.09375, 1.125 , 1.15625, 1.1875 ,\n", - " 1.21875, 1.25 , 1.28125, 1.3125 , 1.34375, 1.375 , 1.40625,\n", - " 1.4375 , 1.46875, 1.5 , 1.53125, 1.5625 , 1.59375, 1.625 ,\n", - " 1.65625, 1.6875 , 1.71875, 1.75 , 1.78125, 1.8125 , 1.84375,\n", - " 1.875 , 1.90625, 1.9375 , 1.96875, 2. , 2.03125, 2.0625 ,\n", - " 2.09375, 2.125 , 2.15625, 2.1875 , 2.21875, 2.25 , 2.28125,\n", - " 2.3125 , 2.34375, 2.375 , 2.40625, 2.4375 , 2.46875, 2.5 ,\n", - " 2.53125, 2.5625 , 2.59375, 2.625 , 2.65625, 2.6875 , 2.71875,\n", - " 2.75 , 2.78125, 2.8125 , 2.84375, 2.875 , 2.90625, 2.9375 ,\n", - " 2.96875, 3. ]),\n", + " 2.]),\n", + " array([1. , 1.09375, 1.1875 , 1.28125, 1.375 , 1.46875, 1.5625 ,\n", + " 1.65625, 1.75 , 1.84375, 1.9375 , 2.03125, 2.125 , 2.21875,\n", + " 2.3125 , 2.40625, 2.5 , 2.59375, 2.6875 , 2.78125, 2.875 ,\n", + " 2.96875, 3.0625 , 3.15625, 3.25 , 3.34375, 3.4375 , 3.53125,\n", + " 3.625 , 3.71875, 3.8125 , 3.90625, 4. , 4.09375, 4.1875 ,\n", + " 4.28125, 4.375 , 4.46875, 4.5625 , 4.65625, 4.75 , 4.84375,\n", + " 4.9375 , 5.03125, 5.125 , 5.21875, 5.3125 , 5.40625, 5.5 ,\n", + " 5.59375, 5.6875 , 5.78125, 5.875 , 5.96875, 6.0625 , 6.15625,\n", + " 6.25 , 6.34375, 6.4375 , 6.53125, 6.625 , 6.71875, 6.8125 ,\n", + " 6.90625, 7. ]),\n", " <BarContainer object of 64 artists>)" ] }, - "execution_count": 90, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvkklEQVR4nO3df1xUdd7//yeIgJkziAbjXOGP2lIx09KkKfvhSqKSm5ttUeS6LemuF9SaZeJtV8tqF7OuTLtM1q4Kr82y2tI2K5I0YdcIFeXyR0rWmj9yB9o1ZoRWRDnfP/pwvo1CCQ0yb3rcb7dzy3m/X+ec93vfnJ2nhzNjmGVZlgAAAAwS3tYDAAAAaC4CDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOBFtPYDWUl9fr0OHDqlLly4KCwtr6+EAAIDTYFmWjhw5IrfbrfDwpu+ztNsAc+jQISUkJLT1MAAAQAscOHBA5557bpP97TbAdOnSRdLX/wM4HI42Hg0AADgdfr9fCQkJ9vt4U9ptgGn4tZHD4SDAAABgmO96/IOHeAEAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACME9HWAzBV7+y3muz7bF7qGRwJAAA/PNyBAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHGaHWCKioo0btw4ud1uhYWFadWqVafU7Nq1Sz/5yU/kdDrVuXNnXXbZZdq/f7/df/ToUWVmZqpbt246++yzNWHCBFVUVAQcY//+/UpNTdVZZ52luLg4zZgxQ8ePH2/+DAEAQLvT7ABTU1OjQYMGafHixY32f/rppxo+fLj69eun9evXa9u2bZo9e7aio6PtmnvuuUdvvvmmXn31VRUWFurQoUO68cYb7f4TJ04oNTVVx44d0wcffKBly5YpLy9Pc+bMacEUAQBAexNmWZbV4p3DwrRy5UqNHz/ebktLS1PHjh31pz/9qdF9fD6fzjnnHL344ou66aabJEm7d+9W//79VVxcrMsvv1zvvPOOrr/+eh06dEjx8fGSpNzcXM2cOVNffPGFIiMjv3Nsfr9fTqdTPp9PDoejpVNsUu/st5rs+2xeatDPBwDAD8Hpvn8H9RmY+vp6vfXWW7rwwguVkpKiuLg4JSUlBfyaqbS0VHV1dUpOTrbb+vXrp549e6q4uFiSVFxcrIEDB9rhRZJSUlLk9/u1c+fORs9dW1srv98fsAEAgPYpqAGmsrJS1dXVmjdvnkaPHq01a9bopz/9qW688UYVFhZKkrxeryIjIxUTExOwb3x8vLxer13zzfDS0N/Q15icnBw5nU57S0hICObUAABACAn6HRhJuuGGG3TPPfdo8ODBys7O1vXXX6/c3NxgnuoUs2bNks/ns7cDBw606vkAAEDbCWqA6d69uyIiIpSYmBjQ3r9/f/tTSC6XS8eOHVNVVVVATUVFhVwul11z8qeSGl431JwsKipKDocjYAMAAO1TUANMZGSkLrvsMpWXlwe0f/zxx+rVq5ckaciQIerYsaPWrl1r95eXl2v//v3yeDySJI/Ho+3bt6uystKuKSgokMPhOCUcAQCAH56I5u5QXV2tTz75xH69d+9elZWVKTY2Vj179tSMGTN0yy236Oqrr9aIESOUn5+vN998U+vXr5ckOZ1OZWRkaPr06YqNjZXD4dBdd90lj8ejyy+/XJI0atQoJSYmauLEiZo/f768Xq9+97vfKTMzU1FRUcGZOQAAMFazA8zmzZs1YsQI+/X06dMlSZMmTVJeXp5++tOfKjc3Vzk5Obr77rvVt29fvfbaaxo+fLi9z4IFCxQeHq4JEyaotrZWKSkpevrpp+3+Dh06aPXq1Zo6dao8Ho86d+6sSZMm6aGHHvo+cwUAAO3E9/oemFDG98AAAGCeNvkeGAAAgDOBAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDjNDjBFRUUaN26c3G63wsLCtGrVqiZrf/3rXyssLExPPvlkQPvhw4eVnp4uh8OhmJgYZWRkqLq6OqBm27ZtuuqqqxQdHa2EhATNnz+/uUMFAADtVLMDTE1NjQYNGqTFixd/a93KlSv14Ycfyu12n9KXnp6unTt3qqCgQKtXr1ZRUZGmTJli9/v9fo0aNUq9evVSaWmpHnvsMT344INaunRpc4cLAADaoYjm7jBmzBiNGTPmW2s+//xz3XXXXXr33XeVmpoa0Ldr1y7l5+dr06ZNGjp0qCTpqaee0tixY/X444/L7XZr+fLlOnbsmJ577jlFRkZqwIABKisr0xNPPBEQdAAAwA9T0J+Bqa+v18SJEzVjxgwNGDDglP7i4mLFxMTY4UWSkpOTFR4erpKSErvm6quvVmRkpF2TkpKi8vJyffnll42et7a2Vn6/P2ADAADtU9ADzKOPPqqIiAjdfffdjfZ7vV7FxcUFtEVERCg2NlZer9euiY+PD6hpeN1Qc7KcnBw5nU57S0hI+L5TAQAAISqoAaa0tFQLFy5UXl6ewsLCgnno7zRr1iz5fD57O3DgwBk9PwAAOHOCGmD++te/qrKyUj179lRERIQiIiK0b98+3Xvvverdu7ckyeVyqbKyMmC/48eP6/Dhw3K5XHZNRUVFQE3D64aak0VFRcnhcARsAACgfQpqgJk4caK2bdumsrIye3O73ZoxY4beffddSZLH41FVVZVKS0vt/datW6f6+nolJSXZNUVFRaqrq7NrCgoK1LdvX3Xt2jWYQwYAAAZq9qeQqqur9cknn9iv9+7dq7KyMsXGxqpnz57q1q1bQH3Hjh3lcrnUt29fSVL//v01evRoTZ48Wbm5uaqrq1NWVpbS0tLsj1zfdtttmjt3rjIyMjRz5kzt2LFDCxcu1IIFC77PXAEAQDvR7ACzefNmjRgxwn49ffp0SdKkSZOUl5d3WsdYvny5srKyNHLkSIWHh2vChAlatGiR3e90OrVmzRplZmZqyJAh6t69u+bMmcNHqAEAgCQpzLIsq60H0Rr8fr+cTqd8Pl+rPA/TO/utJvs+m5faZB8AAGja6b5/828hAQAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYp9kBpqioSOPGjZPb7VZYWJhWrVpl99XV1WnmzJkaOHCgOnfuLLfbrZ///Oc6dOhQwDEOHz6s9PR0ORwOxcTEKCMjQ9XV1QE127Zt01VXXaXo6GglJCRo/vz5LZshAABod5odYGpqajRo0CAtXrz4lL6vvvpKW7Zs0ezZs7Vlyxa9/vrrKi8v109+8pOAuvT0dO3cuVMFBQVavXq1ioqKNGXKFLvf7/dr1KhR6tWrl0pLS/XYY4/pwQcf1NKlS1swRQAA0N6EWZZltXjnsDCtXLlS48ePb7Jm06ZNGjZsmPbt26eePXtq165dSkxM1KZNmzR06FBJUn5+vsaOHauDBw/K7XZryZIl+u1vfyuv16vIyEhJUnZ2tlatWqXdu3ef1tj8fr+cTqd8Pp8cDkdLp9ik3tlvNdn32bzUoJ8PAIAfgtN9/271Z2B8Pp/CwsIUExMjSSouLlZMTIwdXiQpOTlZ4eHhKikpsWuuvvpqO7xIUkpKisrLy/Xll182ep7a2lr5/f6ADQAAtE+tGmCOHj2qmTNn6tZbb7VTlNfrVVxcXEBdRESEYmNj5fV67Zr4+PiAmobXDTUny8nJkdPptLeEhIRgTwcAAISIVgswdXV1uvnmm2VZlpYsWdJap7HNmjVLPp/P3g4cONDq5wQAAG0jojUO2hBe9u3bp3Xr1gX8DsvlcqmysjKg/vjx4zp8+LBcLpddU1FREVDT8Lqh5mRRUVGKiooK5jQAAECICvodmIbwsmfPHr333nvq1q1bQL/H41FVVZVKS0vttnXr1qm+vl5JSUl2TVFRkerq6uyagoIC9e3bV127dg32kAEAgGGaHWCqq6tVVlamsrIySdLevXtVVlam/fv3q66uTjfddJM2b96s5cuX68SJE/J6vfJ6vTp27JgkqX///ho9erQmT56sjRs3asOGDcrKylJaWprcbrck6bbbblNkZKQyMjK0c+dOvfzyy1q4cKGmT58evJkDAABjNftj1OvXr9eIESNOaZ80aZIefPBB9enTp9H93n//fV177bWSvv4iu6ysLL355psKDw/XhAkTtGjRIp199tl2/bZt25SZmalNmzape/fuuuuuuzRz5szTHicfowYAwDyn+/79vb4HJpQRYAAAME/IfA8MAABAsBFgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYp9kBpqioSOPGjZPb7VZYWJhWrVoV0G9ZlubMmaMePXqoU6dOSk5O1p49ewJqDh8+rPT0dDkcDsXExCgjI0PV1dUBNdu2bdNVV12l6OhoJSQkaP78+c2fHQAAaJeaHWBqamo0aNAgLV68uNH++fPna9GiRcrNzVVJSYk6d+6slJQUHT161K5JT0/Xzp07VVBQoNWrV6uoqEhTpkyx+/1+v0aNGqVevXqptLRUjz32mB588EEtXbq0BVMEAADtTZhlWVaLdw4L08qVKzV+/HhJX999cbvduvfee3XfffdJknw+n+Lj45WXl6e0tDTt2rVLiYmJ2rRpk4YOHSpJys/P19ixY3Xw4EG53W4tWbJEv/3tb+X1ehUZGSlJys7O1qpVq7R79+7TGpvf75fT6ZTP55PD4WjpFJvUO/utJvs+m5ca9PMBAPBDcLrv30F9Bmbv3r3yer1KTk6225xOp5KSklRcXCxJKi4uVkxMjB1eJCk5OVnh4eEqKSmxa66++mo7vEhSSkqKysvL9eWXXzZ67traWvn9/oANAAC0T0ENMF6vV5IUHx8f0B4fH2/3eb1excXFBfRHREQoNjY2oKaxY3zzHCfLycmR0+m0t4SEhO8/IQAAEJLazaeQZs2aJZ/PZ28HDhxo6yEBAIBWEtQA43K5JEkVFRUB7RUVFXafy+VSZWVlQP/x48d1+PDhgJrGjvHNc5wsKipKDocjYAMAAO1TUANMnz595HK5tHbtWrvN7/erpKREHo9HkuTxeFRVVaXS0lK7Zt26daqvr1dSUpJdU1RUpLq6OrumoKBAffv2VdeuXYM5ZAAAYKBmB5jq6mqVlZWprKxM0tcP7paVlWn//v0KCwvTtGnT9Mgjj+gvf/mLtm/frp///Odyu932J5X69++v0aNHa/Lkydq4caM2bNigrKwspaWlye12S5Juu+02RUZGKiMjQzt37tTLL7+shQsXavr06UGbOAAAMFdEc3fYvHmzRowYYb9uCBWTJk1SXl6e7r//ftXU1GjKlCmqqqrS8OHDlZ+fr+joaHuf5cuXKysrSyNHjlR4eLgmTJigRYsW2f1Op1Nr1qxRZmamhgwZou7du2vOnDkB3xUDAAB+uL7X98CEMr4HBgAA87TJ98AAAACcCQQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGCXqAOXHihGbPnq0+ffqoU6dOOv/88/Xwww/Lsiy7xrIszZkzRz169FCnTp2UnJysPXv2BBzn8OHDSk9Pl8PhUExMjDIyMlRdXR3s4QIAAAMFPcA8+uijWrJkif77v/9bu3bt0qOPPqr58+frqaeesmvmz5+vRYsWKTc3VyUlJercubNSUlJ09OhRuyY9PV07d+5UQUGBVq9eraKiIk2ZMiXYwwUAAAYKs755ayQIrr/+esXHx+vZZ5+12yZMmKBOnTrphRdekGVZcrvduvfee3XfffdJknw+n+Lj45WXl6e0tDTt2rVLiYmJ2rRpk4YOHSpJys/P19ixY3Xw4EG53e7vHIff75fT6ZTP55PD4QjmFCVJvbPfarLvs3mpQT8fAAA/BKf7/h30OzBXXHGF1q5dq48//liS9H//93/629/+pjFjxkiS9u7dK6/Xq+TkZHsfp9OppKQkFRcXS5KKi4sVExNjhxdJSk5OVnh4uEpKSho9b21trfx+f8AGAADap4hgHzA7O1t+v1/9+vVThw4ddOLECf3+979Xenq6JMnr9UqS4uPjA/aLj4+3+7xer+Li4gIHGhGh2NhYu+ZkOTk5mjt3brCnAwAAQlDQ78C88sorWr58uV588UVt2bJFy5Yt0+OPP65ly5YF+1QBZs2aJZ/PZ28HDhxo1fMBAIC2E/Q7MDNmzFB2drbS0tIkSQMHDtS+ffuUk5OjSZMmyeVySZIqKirUo0cPe7+KigoNHjxYkuRyuVRZWRlw3OPHj+vw4cP2/ieLiopSVFRUsKcDAABCUNDvwHz11VcKDw88bIcOHVRfXy9J6tOnj1wul9auXWv3+/1+lZSUyOPxSJI8Ho+qqqpUWlpq16xbt0719fVKSkoK9pABAIBhgn4HZty4cfr973+vnj17asCAAdq6daueeOIJ/fKXv5QkhYWFadq0aXrkkUd0wQUXqE+fPpo9e7bcbrfGjx8vSerfv79Gjx6tyZMnKzc3V3V1dcrKylJaWtppfQIJAAC0b0EPME899ZRmz56t//zP/1RlZaXcbrd+9atfac6cOXbN/fffr5qaGk2ZMkVVVVUaPny48vPzFR0dbdcsX75cWVlZGjlypMLDwzVhwgQtWrQo2MMFAAAGCvr3wIQKvgcGAADztNn3wAAAALQ2AgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOO0SoD5/PPPdfvtt6tbt27q1KmTBg4cqM2bN9v9lmVpzpw56tGjhzp16qTk5GTt2bMn4BiHDx9Wenq6HA6HYmJilJGRoerq6tYYLgAAMEzQA8yXX36pK6+8Uh07dtQ777yjjz76SP/1X/+lrl272jXz58/XokWLlJubq5KSEnXu3FkpKSk6evSoXZOenq6dO3eqoKBAq1evVlFRkaZMmRLs4QIAAAOFWZZlBfOA2dnZ2rBhg/7617822m9Zltxut+69917dd999kiSfz6f4+Hjl5eUpLS1Nu3btUmJiojZt2qShQ4dKkvLz8zV27FgdPHhQbrf7O8fh9/vldDrl8/nkcDiCN8H/p3f2W032fTYvNejnAwDgh+B037+DfgfmL3/5i4YOHaqf/exniouL0yWXXKJnnnnG7t+7d6+8Xq+Sk5PtNqfTqaSkJBUXF0uSiouLFRMTY4cXSUpOTlZ4eLhKSkoaPW9tba38fn/ABgAA2qegB5i///3vWrJkiS644AK9++67mjp1qu6++24tW7ZMkuT1eiVJ8fHxAfvFx8fbfV6vV3FxcQH9ERERio2NtWtOlpOTI6fTaW8JCQnBnhoAAAgRQQ8w9fX1uvTSS/WHP/xBl1xyiaZMmaLJkycrNzc32KcKMGvWLPl8Pns7cOBAq54PAAC0naAHmB49eigxMTGgrX///tq/f78kyeVySZIqKioCaioqKuw+l8ulysrKgP7jx4/r8OHDds3JoqKi5HA4AjYAANA+BT3AXHnllSovLw9o+/jjj9WrVy9JUp8+feRyubR27Vq73+/3q6SkRB6PR5Lk8XhUVVWl0tJSu2bdunWqr69XUlJSsIcMAAAMExHsA95zzz264oor9Ic//EE333yzNm7cqKVLl2rp0qWSpLCwME2bNk2PPPKILrjgAvXp00ezZ8+W2+3W+PHjJX19x2b06NH2r57q6uqUlZWltLS00/oEEgAAaN+CHmAuu+wyrVy5UrNmzdJDDz2kPn366Mknn1R6erpdc//996umpkZTpkxRVVWVhg8frvz8fEVHR9s1y5cvV1ZWlkaOHKnw8HBNmDBBixYtCvZwAQCAgYL+PTChgu+BAQDAPG32PTAAAACtjQADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4rR5g5s2bp7CwME2bNs1uO3r0qDIzM9WtWzedffbZmjBhgioqKgL2279/v1JTU3XWWWcpLi5OM2bM0PHjx1t7uAAAwACtGmA2bdqkP/7xj7r44osD2u+55x69+eabevXVV1VYWKhDhw7pxhtvtPtPnDih1NRUHTt2TB988IGWLVumvLw8zZkzpzWHCwAADNFqAaa6ulrp6el65pln1LVrV7vd5/Pp2Wef1RNPPKEf//jHGjJkiJ5//nl98MEH+vDDDyVJa9as0UcffaQXXnhBgwcP1pgxY/Twww9r8eLFOnbsWGsNGQAAGKLVAkxmZqZSU1OVnJwc0F5aWqq6urqA9n79+qlnz54qLi6WJBUXF2vgwIGKj4+3a1JSUuT3+7Vz585Gz1dbWyu/3x+wAQCA9imiNQ66YsUKbdmyRZs2bTqlz+v1KjIyUjExMQHt8fHx8nq9ds03w0tDf0NfY3JycjR37twgjB4AAIS6oN+BOXDggH7zm99o+fLlio6ODvbhmzRr1iz5fD57O3DgwBk7NwAAOLOCHmBKS0tVWVmpSy+9VBEREYqIiFBhYaEWLVqkiIgIxcfH69ixY6qqqgrYr6KiQi6XS5LkcrlO+VRSw+uGmpNFRUXJ4XAEbAAAoH0KeoAZOXKktm/frrKyMnsbOnSo0tPT7T937NhRa9eutfcpLy/X/v375fF4JEkej0fbt29XZWWlXVNQUCCHw6HExMRgDxkAABgm6M/AdOnSRRdddFFAW+fOndWtWze7PSMjQ9OnT1dsbKwcDofuuusueTweXX755ZKkUaNGKTExURMnTtT8+fPl9Xr1u9/9TpmZmYqKigr2kAEAgGFa5SHe77JgwQKFh4drwoQJqq2tVUpKip5++mm7v0OHDlq9erWmTp0qj8ejzp07a9KkSXrooYfaYrgAACDEhFmWZbX1IFqD3++X0+mUz+drledheme/1WTfZ/NSg34+AAB+CE73/Zt/CwkAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcSLaegAA0BL8i/DADxt3YAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjBD3A5OTk6LLLLlOXLl0UFxen8ePHq7y8PKDm6NGjyszMVLdu3XT22WdrwoQJqqioCKjZv3+/UlNTddZZZykuLk4zZszQ8ePHgz1cAABgoKAHmMLCQmVmZurDDz9UQUGB6urqNGrUKNXU1Ng199xzj9588029+uqrKiws1KFDh3TjjTfa/SdOnFBqaqqOHTumDz74QMuWLVNeXp7mzJkT7OECAAADhVmWZbXmCb744gvFxcWpsLBQV199tXw+n8455xy9+OKLuummmyRJu3fvVv/+/VVcXKzLL79c77zzjq6//nodOnRI8fHxkqTc3FzNnDlTX3zxhSIjI7/zvH6/X06nUz6fTw6HI+jz6p39VpN9n81LDfr5AATiGgTap9N9/271Z2B8Pp8kKTY2VpJUWlqquro6JScn2zX9+vVTz549VVxcLEkqLi7WwIED7fAiSSkpKfL7/dq5c2ej56mtrZXf7w/YAABA+9SqAaa+vl7Tpk3TlVdeqYsuukiS5PV6FRkZqZiYmIDa+Ph4eb1eu+ab4aWhv6GvMTk5OXI6nfaWkJAQ5NkAAIBQ0aoBJjMzUzt27NCKFSta8zSSpFmzZsnn89nbgQMHWv2cAACgbUS01oGzsrK0evVqFRUV6dxzz7XbXS6Xjh07pqqqqoC7MBUVFXK5XHbNxo0bA47X8CmlhpqTRUVFKSoqKsizAAAAoSjod2Asy1JWVpZWrlypdevWqU+fPgH9Q4YMUceOHbV27Vq7rby8XPv375fH45EkeTwebd++XZWVlXZNQUGBHA6HEhMTgz1kAABgmKDfgcnMzNSLL76oN954Q126dLGfWXE6nerUqZOcTqcyMjI0ffp0xcbGyuFw6K677pLH49Hll18uSRo1apQSExM1ceJEzZ8/X16vV7/73e+UmZnJXRYAABD8ALNkyRJJ0rXXXhvQ/vzzz+sXv/iFJGnBggUKDw/XhAkTVFtbq5SUFD399NN2bYcOHbR69WpNnTpVHo9HnTt31qRJk/TQQw8Fe7gAAMBAQQ8wp/O1MtHR0Vq8eLEWL17cZE2vXr309ttvB3NoAACgneDfQgIAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgRbT0AAAAQmnpnv9Vk32fzUs/gSE7FHRgAAGCckA4wixcvVu/evRUdHa2kpCRt3LixrYcEAABCQMgGmJdfflnTp0/XAw88oC1btmjQoEFKSUlRZWVlWw8NAAC0sZANME888YQmT56sO+64Q4mJicrNzdVZZ52l5557rq2HBgAA2lhIPsR77NgxlZaWatasWXZbeHi4kpOTVVxc3Og+tbW1qq2ttV/7fD5Jkt/vb5Ux1td+1WRfa50TwP+PaxBofW1xnTUc17Ksb60LyQDzz3/+UydOnFB8fHxAe3x8vHbv3t3oPjk5OZo7d+4p7QkJCa0yxm/jfPKMnxLAN3ANAq2vta+zI0eOyOl0NtkfkgGmJWbNmqXp06fbr+vr63X48GF169ZNYWFhQT2X3+9XQkKCDhw4IIfDEdRjhwLmZ772PkfmZ772Pkfm13KWZenIkSNyu93fWheSAaZ79+7q0KGDKioqAtorKirkcrka3ScqKkpRUVEBbTExMa01REmSw+Folz+YDZif+dr7HJmf+dr7HJlfy3zbnZcGIfkQb2RkpIYMGaK1a9fabfX19Vq7dq08Hk8bjgwAAISCkLwDI0nTp0/XpEmTNHToUA0bNkxPPvmkampqdMcdd7T10AAAQBsL2QBzyy236IsvvtCcOXPk9Xo1ePBg5efnn/Jgb1uIiorSAw88cMqvrNoL5me+9j5H5me+9j5H5tf6wqzv+pwSAABAiAnJZ2AAAAC+DQEGAAAYhwADAACMQ4ABAADG+cEHmKKiIo0bN05ut1thYWFatWrVd+6zfv16XXrppYqKitKPfvQj5eXlnVKzePFi9e7dW9HR0UpKStLGjRuDP/jT0Nz5vf7667ruuut0zjnnyOFwyOPx6N133w2oefDBBxUWFhaw9evXrxVn0bTmzm/9+vWnjD0sLExerzegLlTWT2r+HH/xi180OscBAwbYNaG0hjk5ObrsssvUpUsXxcXFafz48SovL//O/V599VX169dP0dHRGjhwoN5+++2AfsuyNGfOHPXo0UOdOnVScnKy9uzZ01rTaFJL5vfMM8/oqquuUteuXdW1a1clJyef8jPY2DqPHj26NafSqJbMLy8v75SxR0dHB9SEyvpJLZvjtdde2+h1mJqaateEyhouWbJEF198sf2ldB6PR++888637hMK198PPsDU1NRo0KBBWrx48WnV7927V6mpqRoxYoTKyso0bdo03XnnnQFv8i+//LKmT5+uBx54QFu2bNGgQYOUkpKiysrK1ppGk5o7v6KiIl133XV6++23VVpaqhEjRmjcuHHaunVrQN2AAQP0j3/8w97+9re/tcbwv1Nz59egvLw8YPxxcXF2Xyitn9T8OS5cuDBgbgcOHFBsbKx+9rOfBdSFyhoWFhYqMzNTH374oQoKClRXV6dRo0appqamyX0++OAD3XrrrcrIyNDWrVs1fvx4jR8/Xjt27LBr5s+fr0WLFik3N1clJSXq3LmzUlJSdPTo0TMxLVtL5rd+/Xrdeuutev/991VcXKyEhASNGjVKn3/+eUDd6NGjA9bwpZdeau3pnKIl85O+/gbXb4593759Af2hsn5Sy+b4+uuvB8xvx44d6tChwynXYSis4bnnnqt58+aptLRUmzdv1o9//GPdcMMN2rlzZ6P1IXP9WbBJslauXPmtNffff781YMCAgLZbbrnFSklJsV8PGzbMyszMtF+fOHHCcrvdVk5OTlDH21ynM7/GJCYmWnPnzrVfP/DAA9agQYOCN7AgOZ35vf/++5Yk68svv2yyJlTXz7JatoYrV660wsLCrM8++8xuC9U1tCzLqqystCRZhYWFTdbcfPPNVmpqakBbUlKS9atf/cqyLMuqr6+3XC6X9dhjj9n9VVVVVlRUlPXSSy+1zsBP0+nM72THjx+3unTpYi1btsxumzRpknXDDTe0wgi/n9OZ3/PPP285nc4m+0N5/SyrZWu4YMECq0uXLlZ1dbXdFqpraFmW1bVrV+t//ud/Gu0LlevvB38HprmKi4uVnJwc0JaSkqLi4mJJ0rFjx1RaWhpQEx4eruTkZLvGJPX19Tpy5IhiY2MD2vfs2SO3263zzjtP6enp2r9/fxuNsGUGDx6sHj166LrrrtOGDRvs9va2fpL07LPPKjk5Wb169QpoD9U19Pl8knTKz9w3fdd1uHfvXnm93oAap9OppKSkNl/H05nfyb766ivV1dWdss/69esVFxenvn37aurUqfrXv/4V1LG2xOnOr7q6Wr169VJCQsIpf9sP5fWTWraGzz77rNLS0tS5c+eA9lBbwxMnTmjFihWqqalp8p/uCZXrjwDTTF6v95RvA46Pj5ff79e///1v/fOf/9SJEycarTn5OQsTPP7446qurtbNN99styUlJSkvL0/5+flasmSJ9u7dq6uuukpHjhxpw5Genh49eig3N1evvfaaXnvtNSUkJOjaa6/Vli1bJKndrd+hQ4f0zjvv6M477wxoD9U1rK+v17Rp03TllVfqoosuarKuqeuwYY0a/htq63i68zvZzJkz5Xa7A94QRo8erf/93//V2rVr9eijj6qwsFBjxozRiRMnWmPop+V059e3b18999xzeuONN/TCCy+ovr5eV1xxhQ4ePCgpdNdPatkabty4UTt27DjlOgylNdy+fbvOPvtsRUVF6de//rVWrlypxMTERmtD5foL2X9KAG3vxRdf1Ny5c/XGG28EPCMyZswY+88XX3yxkpKS1KtXL73yyivKyMhoi6Getr59+6pv37726yuuuEKffvqpFixYoD/96U9tOLLWsWzZMsXExGj8+PEB7aG6hpmZmdqxY0ebPY/T2loyv3nz5mnFihVav359wIOuaWlp9p8HDhyoiy++WOeff77Wr1+vkSNHBnXcp+t05+fxeAL+dn/FFVeof//++uMf/6iHH364tYf5vbRkDZ999lkNHDhQw4YNC2gPpTXs27evysrK5PP59Oc//1mTJk1SYWFhkyEmFHAHpplcLpcqKioC2ioqKuRwONSpUyd1795dHTp0aLTG5XKdyaF+LytWrNCdd96pV1555ZRbhSeLiYnRhRdeqE8++eQMjS64hg0bZo+9vayf9PWnAJ577jlNnDhRkZGR31obCmuYlZWl1atX6/3339e55577rbVNXYcNa9Tw31Bax+bMr8Hjjz+uefPmac2aNbr44ou/tfa8885T9+7d22wNWzK/Bh07dtQll1xijz0U109q2Rxramq0YsWK0/qLQVuuYWRkpH70ox9pyJAhysnJ0aBBg7Rw4cJGa0Pl+iPANJPH49HatWsD2goKCuy/TURGRmrIkCEBNfX19Vq7dm2Tv08MNS+99JLuuOMOvfTSSwEf+WtKdXW1Pv30U/Xo0eMMjC74ysrK7LG3h/VrUFhYqE8++eS0/o+zLdfQsixlZWVp5cqVWrdunfr06fOd+3zXddinTx+5XK6AGr/fr5KSkjO+ji2Zn/T1pzgefvhh5efna+jQod9Zf/DgQf3rX/8642vY0vl904kTJ7R9+3Z77KG0ftL3m+Orr76q2tpa3X777d9Z21Zr2Jj6+nrV1tY22hcy11/QHgc21JEjR6ytW7daW7dutSRZTzzxhLV161Zr3759lmVZVnZ2tjVx4kS7/u9//7t11llnWTNmzLB27dplLV682OrQoYOVn59v16xYscKKioqy8vLyrI8++siaMmWKFRMTY3m93pCf3/Lly62IiAhr8eLF1j/+8Q97q6qqsmvuvfdea/369dbevXutDRs2WMnJyVb37t2tysrKkJ/fggULrFWrVll79uyxtm/fbv3mN7+xwsPDrffee8+uCaX1s6zmz7HB7bffbiUlJTV6zFBaw6lTp1pOp9Nav359wM/cV199ZddMnDjRys7Otl9v2LDBioiIsB5//HFr165d1gMPPGB17NjR2r59u10zb948KyYmxnrjjTesbdu2WTfccIPVp08f69///nfIz2/evHlWZGSk9ec//zlgnyNHjliW9fXPxH333WcVFxdbe/futd577z3r0ksvtS644ALr6NGjIT+/uXPnWu+++6716aefWqWlpVZaWpoVHR1t7dy5064JlfWzrJbNscHw4cOtW2655ZT2UFrD7Oxsq7Cw0Nq7d6+1bds2Kzs72woLC7PWrFljWVboXn8/+ADT8LHak7dJkyZZlvX1x9yuueaaU/YZPHiwFRkZaZ133nnW888/f8pxn3rqKatnz55WZGSkNWzYMOvDDz9s/ck0ornzu+aaa7613rK+/th4jx49rMjISOs//uM/rFtuucX65JNPzuzE/p/mzu/RRx+1zj//fCs6OtqKjY21rr32WmvdunWnHDdU1s+yWvYzWlVVZXXq1MlaunRpo8cMpTVsbG6SAq6ra665JuBn0LIs65VXXrEuvPBCKzIy0howYID11ltvBfTX19dbs2fPtuLj462oqChr5MiRVnl5+RmYUaCWzK9Xr16N7vPAAw9YlmVZX331lTVq1CjrnHPOsTp27Gj16tXLmjx5cpuE7JbMb9q0afb1FR8fb40dO9basmVLwHFDZf0sq+U/o7t377Yk2UHgm0JpDX/5y19avXr1siIjI61zzjnHGjlyZMCYQ/X6C7MsywrSzRwAAIAzgmdgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADDO/wfhamWXm0J9HwAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAjSklEQVR4nO3dfVSUdf7/8dcgcpMyg1gMzobItm1K3mRSNNm9HEnJXU90Y1FRsbqnBcvsRvmW2o2F2XZHS5ieVj0nPd3sWa0sKfKOLRERl03JyDZTzAY6h5gJOiLK/P7oeP2a1EJ3aOZjz8c51znNdX1mrvdcx3N4NjMMNr/f7xcAAIBBIkI9AAAAwPEiYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYJzLUA/SUrq4u7du3T3FxcbLZbKEeBwAAdIPf79e3334rl8uliIhjv85y0gbMvn37lJycHOoxAADACWhsbNTpp59+zOMnbcDExcVJ+v4C2O32EE8DAAC6w+fzKTk52fo5fiwnbcAcftvIbrcTMAAAGObnPv7Bh3gBAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGCcyFAPYKJBM98+5rEv5mX/gpMAAPDrxCswAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxz3AFTWVmpCRMmyOVyyWazaeXKldaxzs5OzZgxQ8OGDVOfPn3kcrl0yy23aN++fQGP0dLSotzcXNntdsXHxys/P19tbW0Baz766CNdfPHFiomJUXJysubPn39izxAAAJx0jjtg2tvbNWLECJWWlh5x7LvvvtPWrVs1a9Ysbd26Vf/85z/V0NCgP/zhDwHrcnNzVV9fr4qKCq1atUqVlZWaMmWKddzn82ns2LFKSUlRbW2tnnzyST300ENauHDhCTxFAABwsrH5/X7/Cd/ZZtOKFSs0ceLEY66pqanR+eefr927d2vgwIHasWOH0tLSVFNTo/T0dElSeXm5xo8fr71798rlcqmsrEwPPPCAPB6PoqKiJEkzZ87UypUr9cknn3RrNp/PJ4fDIa/XK7vdfqJP8agGzXz7mMe+mJcd1HMBAPBr0t2f3z3+GRiv1yubzab4+HhJUlVVleLj4614kaTMzExFRESourraWnPJJZdY8SJJWVlZamho0DfffNPTIwMAgDAX2ZMPvn//fs2YMUM33HCDVVEej0eJiYmBQ0RGKiEhQR6Px1qTmpoasMbpdFrH+vXrd8S5Ojo61NHRYd32+XxBfS4AACB89NgrMJ2dnbruuuvk9/tVVlbWU6exFBcXy+FwWFtycnKPnxMAAIRGjwTM4XjZvXu3KioqAt7DSkpKUnNzc8D6gwcPqqWlRUlJSdaapqamgDWHbx9e82NFRUXyer3W1tjYGMynBAAAwkjQA+ZwvOzcuVPvv/+++vfvH3Dc7XartbVVtbW11r61a9eqq6tLGRkZ1prKykp1dnZaayoqKnTWWWcd9e0jSYqOjpbdbg/YAADAyem4A6atrU11dXWqq6uTJO3atUt1dXXas2ePOjs7dc0112jLli1atmyZDh06JI/HI4/HowMHDkiShgwZoiuvvFKTJ0/W5s2b9eGHH6qwsFCTJk2Sy+WSJN14442KiopSfn6+6uvr9eqrr+q5557T9OnTg/fMAQCAsY7716jXr1+vyy+//Ij9eXl5euihh4748O1h69at02WXXSbp+y+yKyws1FtvvaWIiAjl5OSopKREffv2tdZ/9NFHKigoUE1NjU499VRNnTpVM2bM6Pac/Bo1AADm6e7P7//pe2DCGQEDAIB5wuZ7YAAAAIKNgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgnOMOmMrKSk2YMEEul0s2m00rV64MOO73+zV79mwNGDBAsbGxyszM1M6dOwPWtLS0KDc3V3a7XfHx8crPz1dbW1vAmo8++kgXX3yxYmJilJycrPnz5x//swMAACel4w6Y9vZ2jRgxQqWlpUc9Pn/+fJWUlGjBggWqrq5Wnz59lJWVpf3791trcnNzVV9fr4qKCq1atUqVlZWaMmWKddzn82ns2LFKSUlRbW2tnnzyST300ENauHDhCTxFAABwsrH5/X7/Cd/ZZtOKFSs0ceJESd+/+uJyuXTPPffo3nvvlSR5vV45nU4tWbJEkyZN0o4dO5SWlqaamhqlp6dLksrLyzV+/Hjt3btXLpdLZWVleuCBB+TxeBQVFSVJmjlzplauXKlPPvmkW7P5fD45HA55vV7Z7fYTfYpHNWjm28c89sW87KCeCwCAX5Pu/vwO6mdgdu3aJY/Ho8zMTGufw+FQRkaGqqqqJElVVVWKj4+34kWSMjMzFRERoerqamvNJZdcYsWLJGVlZamhoUHffPPNUc/d0dEhn88XsAEAgJNTUAPG4/FIkpxOZ8B+p9NpHfN4PEpMTAw4HhkZqYSEhIA1R3uMH57jx4qLi+VwOKwtOTn5f39CAAAgLJ00v4VUVFQkr9drbY2NjaEeCQAA9JCgBkxSUpIkqampKWB/U1OTdSwpKUnNzc0Bxw8ePKiWlpaANUd7jB+e48eio6Nlt9sDNgAAcHIKasCkpqYqKSlJa9assfb5fD5VV1fL7XZLktxut1pbW1VbW2utWbt2rbq6upSRkWGtqaysVGdnp7WmoqJCZ511lvr16xfMkQEAgIGOO2Da2tpUV1enuro6Sd9/cLeurk579uyRzWbTtGnTNHfuXL355pvatm2bbrnlFrlcLus3lYYMGaIrr7xSkydP1ubNm/Xhhx+qsLBQkyZNksvlkiTdeOONioqKUn5+vurr6/Xqq6/queee0/Tp04P2xAEAgLkij/cOW7Zs0eWXX27dPhwVeXl5WrJkie6//361t7drypQpam1t1UUXXaTy8nLFxMRY91m2bJkKCws1ZswYRUREKCcnRyUlJdZxh8Oh9957TwUFBRo1apROPfVUzZ49O+C7YgAAwK/X//Q9MOGM74EBAMA8IfkeGAAAgF8CAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOEEPmEOHDmnWrFlKTU1VbGyszjjjDD366KPy+/3WGr/fr9mzZ2vAgAGKjY1VZmamdu7cGfA4LS0tys3Nld1uV3x8vPLz89XW1hbscQEAgIGCHjBPPPGEysrK9Le//U07duzQE088ofnz5+v555+31syfP18lJSVasGCBqqur1adPH2VlZWn//v3WmtzcXNXX16uiokKrVq1SZWWlpkyZEuxxAQCAgWz+H740EgRXXXWVnE6nXnrpJWtfTk6OYmNj9fLLL8vv98vlcumee+7RvffeK0nyer1yOp1asmSJJk2apB07digtLU01NTVKT0+XJJWXl2v8+PHau3evXC7Xz87h8/nkcDjk9Xplt9uD+RQ1aObbxzz2xbzsoJ4LAIBfk+7+/A76KzAXXnih1qxZo08//VSS9J///EcffPCBxo0bJ0natWuXPB6PMjMzrfs4HA5lZGSoqqpKklRVVaX4+HgrXiQpMzNTERERqq6uPup5Ozo65PP5AjYAAHByigz2A86cOVM+n0+DBw9Wr169dOjQIT322GPKzc2VJHk8HkmS0+kMuJ/T6bSOeTweJSYmBg4aGamEhARrzY8VFxfr4YcfDvbTAQAAYSjor8C89tprWrZsmZYvX66tW7dq6dKl+utf/6qlS5cG+1QBioqK5PV6ra2xsbFHzwcAAEIn6K/A3HfffZo5c6YmTZokSRo2bJh2796t4uJi5eXlKSkpSZLU1NSkAQMGWPdramrSOeecI0lKSkpSc3NzwOMePHhQLS0t1v1/LDo6WtHR0cF+OgAAIAwF/RWY7777ThERgQ/bq1cvdXV1SZJSU1OVlJSkNWvWWMd9Pp+qq6vldrslSW63W62traqtrbXWrF27Vl1dXcrIyAj2yAAAwDBBfwVmwoQJeuyxxzRw4ECdffbZ+ve//62nn35at99+uyTJZrNp2rRpmjt3rs4880ylpqZq1qxZcrlcmjhxoiRpyJAhuvLKKzV58mQtWLBAnZ2dKiws1KRJk7r1G0gAAODkFvSAef755zVr1iz95S9/UXNzs1wul/785z9r9uzZ1pr7779f7e3tmjJlilpbW3XRRRepvLxcMTEx1pply5apsLBQY8aMUUREhHJyclRSUhLscQEAgIGC/j0w4YLvgQEAwDwh+x4YAACAnkbAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4PRIwX375pW666Sb1799fsbGxGjZsmLZs2WId9/v9mj17tgYMGKDY2FhlZmZq586dAY/R0tKi3Nxc2e12xcfHKz8/X21tbT0xLgAAMEzQA+abb77R6NGj1bt3b61evVoff/yxnnrqKfXr189aM3/+fJWUlGjBggWqrq5Wnz59lJWVpf3791trcnNzVV9fr4qKCq1atUqVlZWaMmVKsMcFAAAGsvn9fn8wH3DmzJn68MMP9a9//euox/1+v1wul+655x7de++9kiSv1yun06klS5Zo0qRJ2rFjh9LS0lRTU6P09HRJUnl5ucaPH6+9e/fK5XL97Bw+n08Oh0Ner1d2uz14T1DSoJlvH/PYF/Oyg3ouAAB+Tbr78zvor8C8+eabSk9P17XXXqvExESNHDlSixYtso7v2rVLHo9HmZmZ1j6Hw6GMjAxVVVVJkqqqqhQfH2/FiyRlZmYqIiJC1dXVRz1vR0eHfD5fwAYAAE5OQQ+Yzz//XGVlZTrzzDP17rvv6o477tCdd96ppUuXSpI8Ho8kyel0BtzP6XRaxzwejxITEwOOR0ZGKiEhwVrzY8XFxXI4HNaWnJwc7KcGAADCRNADpqurS+eee64ef/xxjRw5UlOmTNHkyZO1YMGCYJ8qQFFRkbxer7U1Njb26PkAAEDoBD1gBgwYoLS0tIB9Q4YM0Z49eyRJSUlJkqSmpqaANU1NTdaxpKQkNTc3Bxw/ePCgWlparDU/Fh0dLbvdHrABAICTU9ADZvTo0WpoaAjY9+mnnyolJUWSlJqaqqSkJK1Zs8Y67vP5VF1dLbfbLUlyu91qbW1VbW2ttWbt2rXq6upSRkZGsEcGAACGiQz2A95999268MIL9fjjj+u6667T5s2btXDhQi1cuFCSZLPZNG3aNM2dO1dnnnmmUlNTNWvWLLlcLk2cOFHS96/YXHnlldZbT52dnSosLNSkSZO69RtIAADg5Bb0gDnvvPO0YsUKFRUV6ZFHHlFqaqqeffZZ5ebmWmvuv/9+tbe3a8qUKWptbdVFF12k8vJyxcTEWGuWLVumwsJCjRkzRhEREcrJyVFJSUmwxwUAAAYK+vfAhAu+BwYAAPOE7HtgAAAAehoBAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA40SGegCcXAbNfPuYx76Yl/0LTgIAOJnxCgwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDg9HjDz5s2TzWbTtGnTrH379+9XQUGB+vfvr759+yonJ0dNTU0B99uzZ4+ys7N1yimnKDExUffdd58OHjzY0+MCAAAD9GjA1NTU6MUXX9Tw4cMD9t99991666239Prrr2vDhg3at2+frr76auv4oUOHlJ2drQMHDmjjxo1aunSplixZotmzZ/fkuAAAwBA9FjBtbW3Kzc3VokWL1K9fP2u/1+vVSy+9pKefflpXXHGFRo0apcWLF2vjxo3atGmTJOm9997Txx9/rJdfflnnnHOOxo0bp0cffVSlpaU6cOBAT40MAAAM0WMBU1BQoOzsbGVmZgbsr62tVWdnZ8D+wYMHa+DAgaqqqpIkVVVVadiwYXI6ndaarKws+Xw+1dfXH/V8HR0d8vl8ARsAADg5RfbEg77yyivaunWrampqjjjm8XgUFRWl+Pj4gP1Op1Mej8da88N4OXz88LGjKS4u1sMPPxyE6QEAQLgL+iswjY2Nuuuuu7Rs2TLFxMQE++GPqaioSF6v19oaGxt/sXMDAIBfVtADpra2Vs3NzTr33HMVGRmpyMhIbdiwQSUlJYqMjJTT6dSBAwfU2toacL+mpiYlJSVJkpKSko74raTDtw+v+bHo6GjZ7faADQAAnJyCHjBjxozRtm3bVFdXZ23p6enKzc21/rt3795as2aNdZ+Ghgbt2bNHbrdbkuR2u7Vt2zY1NzdbayoqKmS325WWlhbskQEAgGGC/hmYuLg4DR06NGBfnz591L9/f2t/fn6+pk+froSEBNntdk2dOlVut1sXXHCBJGns2LFKS0vTzTffrPnz58vj8ejBBx9UQUGBoqOjgz0yAAAwTI98iPfnPPPMM4qIiFBOTo46OjqUlZWlF154wTreq1cvrVq1SnfccYfcbrf69OmjvLw8PfLII6EYFwAAhJlfJGDWr18fcDsmJkalpaUqLS095n1SUlL0zjvv9PBkAADARPwtJAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGCXrAFBcX67zzzlNcXJwSExM1ceJENTQ0BKzZv3+/CgoK1L9/f/Xt21c5OTlqamoKWLNnzx5lZ2frlFNOUWJiou677z4dPHgw2OMCAAADBT1gNmzYoIKCAm3atEkVFRXq7OzU2LFj1d7ebq25++679dZbb+n111/Xhg0btG/fPl199dXW8UOHDik7O1sHDhzQxo0btXTpUi1ZskSzZ88O9rgAAMBANr/f7+/JE3z99ddKTEzUhg0bdMkll8jr9eq0007T8uXLdc0110iSPvnkEw0ZMkRVVVW64IILtHr1al111VXat2+fnE6nJGnBggWaMWOGvv76a0VFRf3seX0+nxwOh7xer+x2e1Cf06CZbx/z2BfzsoN6LtNwbQAA/4vu/vzu8c/AeL1eSVJCQoIkqba2Vp2dncrMzLTWDB48WAMHDlRVVZUkqaqqSsOGDbPiRZKysrLk8/lUX1/f0yMDAIAwF9mTD97V1aVp06Zp9OjRGjp0qCTJ4/EoKipK8fHxAWudTqc8Ho+15ofxcvj44WNH09HRoY6ODuu2z+cL1tMAAABhpkdfgSkoKND27dv1yiuv9ORpJH3/4WGHw2FtycnJPX5OAAAQGj0WMIWFhVq1apXWrVun008/3dqflJSkAwcOqLW1NWB9U1OTkpKSrDU//q2kw7cPr/mxoqIieb1ea2tsbAziswEAAOEk6G8h+f1+TZ06VStWrND69euVmpoacHzUqFHq3bu31qxZo5ycHElSQ0OD9uzZI7fbLUlyu9167LHH1NzcrMTERElSRUWF7Ha70tLSjnre6OhoRUdHB/vpAD2ODz4DwPELesAUFBRo+fLleuONNxQXF2d9ZsXhcCg2NlYOh0P5+fmaPn26EhISZLfbNXXqVLndbl1wwQWSpLFjxyotLU0333yz5s+fL4/HowcffFAFBQVECgAACH7AlJWVSZIuu+yygP2LFy/WrbfeKkl65plnFBERoZycHHV0dCgrK0svvPCCtbZXr15atWqV7rjjDrndbvXp00d5eXl65JFHgj0uAAAwUI+8hfRzYmJiVFpaqtLS0mOuSUlJ0TvvvBPM0QAAwEmCv4UEAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjBMZ6gEA4EQMmvn2MY99MS/7F5wEQCjwCgwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4/A9MABwEuH7cfBrwSswAADAOAQMAAAwTlgHTGlpqQYNGqSYmBhlZGRo8+bNoR4JAACEgbANmFdffVXTp0/XnDlztHXrVo0YMUJZWVlqbm4O9WgAACDEwjZgnn76aU2ePFm33Xab0tLStGDBAp1yyin6+9//HurRAABAiIXlbyEdOHBAtbW1KioqsvZFREQoMzNTVVVVR71PR0eHOjo6rNter1eS5PP5gj5fV8d3xzzWE+czCdfm+HHNTgzX7ei4Lidm6Jx3j3ls+8NZv+AkOPzv1O/3//RCfxj68ssv/ZL8GzduDNh/3333+c8///yj3mfOnDl+SWxsbGxsbGwnwdbY2PiTrRCWr8CciKKiIk2fPt263dXVpZaWFvXv3182my1o5/H5fEpOTlZjY6PsdnvQHvdkxfXqPq5V93Gtuo9r1X1cq+7ryWvl9/v17bffyuVy/eS6sAyYU089Vb169VJTU1PA/qamJiUlJR31PtHR0YqOjg7YFx8f31Mjym638w/8OHC9uo9r1X1cq+7jWnUf16r7eupaORyOn10Tlh/ijYqK0qhRo7RmzRprX1dXl9asWSO32x3CyQAAQDgIy1dgJGn69OnKy8tTenq6zj//fD377LNqb2/XbbfdFurRAABAiIVtwFx//fX6+uuvNXv2bHk8Hp1zzjkqLy+X0+kM6VzR0dGaM2fOEW9X4ei4Xt3Hteo+rlX3ca26j2vVfeFwrWx+/8/9nhIAAEB4CcvPwAAAAPwUAgYAABiHgAEAAMYhYAAAgHEImONQWVmpCRMmyOVyyWazaeXKlaEeKSwVFxfrvPPOU1xcnBITEzVx4kQ1NDSEeqywVFZWpuHDh1tfBuV2u7V69epQj2WEefPmyWazadq0aaEeJSw99NBDstlsAdvgwYNDPVbY+vLLL3XTTTepf//+io2N1bBhw7Rly5ZQjxV2Bg0adMS/K5vNpoKCgl98FgLmOLS3t2vEiBEqLS0N9ShhbcOGDSooKNCmTZtUUVGhzs5OjR07Vu3t7aEeLeycfvrpmjdvnmpra7VlyxZdccUV+uMf/6j6+vpQjxbWampq9OKLL2r48OGhHiWsnX322frqq6+s7YMPPgj1SGHpm2++0ejRo9W7d2+tXr1aH3/8sZ566in169cv1KOFnZqamoB/UxUVFZKka6+99hefJWy/ByYcjRs3TuPGjQv1GGGvvLw84PaSJUuUmJio2tpaXXLJJSGaKjxNmDAh4PZjjz2msrIybdq0SWeffXaIpgpvbW1tys3N1aJFizR37txQjxPWIiMjj/nnV/D/PfHEE0pOTtbixYutfampqSGcKHyddtppAbfnzZunM844Q5deeukvPguvwKDHeb1eSVJCQkKIJwlvhw4d0iuvvKL29nb+ZMZPKCgoUHZ2tjIzM0M9StjbuXOnXC6Xfvvb3yo3N1d79uwJ9Uhh6c0331R6erquvfZaJSYmauTIkVq0aFGoxwp7Bw4c0Msvv6zbb789qH80ubt4BQY9qqurS9OmTdPo0aM1dOjQUI8TlrZt2ya32639+/erb9++WrFihdLS0kI9Vlh65ZVXtHXrVtXU1IR6lLCXkZGhJUuW6KyzztJXX32lhx9+WBdffLG2b9+uuLi4UI8XVj7//HOVlZVp+vTp+r//+z/V1NTozjvvVFRUlPLy8kI9XthauXKlWltbdeutt4bk/AQMelRBQYG2b9/Oe+8/4ayzzlJdXZ28Xq/+8Y9/KC8vTxs2bCBifqSxsVF33XWXKioqFBMTE+pxwt4P3+4ePny4MjIylJKSotdee035+fkhnCz8dHV1KT09XY8//rgkaeTIkdq+fbsWLFhAwPyEl156SePGjZPL5QrJ+XkLCT2msLBQq1at0rp163T66aeHepywFRUVpd/97ncaNWqUiouLNWLECD333HOhHivs1NbWqrm5Weeee64iIyMVGRmpDRs2qKSkRJGRkTp06FCoRwxr8fHx+v3vf6/PPvss1KOEnQEDBhzxPwxDhgzhLbefsHv3br3//vv605/+FLIZeAUGQef3+zV16lStWLFC69ev58Nwx6mrq0sdHR2hHiPsjBkzRtu2bQvYd9ttt2nw4MGaMWOGevXqFaLJzNDW1qb//ve/uvnmm0M9StgZPXr0EV/18OmnnyolJSVEE4W/xYsXKzExUdnZ2SGbgYA5Dm1tbQH/97Jr1y7V1dUpISFBAwcODOFk4aWgoEDLly/XG2+8obi4OHk8HkmSw+FQbGxsiKcLL0VFRRo3bpwGDhyob7/9VsuXL9f69ev17rvvhnq0sBMXF3fE56j69Omj/v378/mqo7j33ns1YcIEpaSkaN++fZozZ4569eqlG264IdSjhZ27775bF154oR5//HFdd9112rx5sxYuXKiFCxeGerSw1NXVpcWLFysvL0+RkSHMCD+6bd26dX5JR2x5eXmhHi2sHO0aSfIvXrw41KOFndtvv92fkpLij4qK8p922mn+MWPG+N97771Qj2WMSy+91H/XXXeFeoywdP311/sHDBjgj4qK8v/mN7/xX3/99f7PPvss1GOFrbfeess/dOhQf3R0tH/w4MH+hQsXhnqksPXuu+/6JfkbGhpCOofN7/f7Q5NOAAAAJ4YP8QIAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIzz/wBRiRVxrEYfAQAAAABJRU5ErkJggg==", "text/plain": [ "<Figure size 640x480 with 1 Axes>" ] @@ -328,21 +370,21 @@ }, { "cell_type": "code", - "execution_count": 97, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ - "import pickle\n", + "# import pickle\n", "\n", - "graphs, labels, _ = mkbatch(3*10**5)\n", + "# graphs, labels, _ = mkbatch(3*10**5)\n", "\n", - "data = {\n", - " \"data\": graphs,\n", - " \"labels\": labels\n", - "}\n", + "# data = {\n", + "# \"data\": graphs,\n", + "# \"labels\": labels\n", + "# }\n", "\n", - "with open('data.pkl', 'wb') as file:\n", - " pickle.dump(data, file)" + "# with open('data.pkl', 'wb') as file:\n", + "# pickle.dump(data, file)" ] }, { @@ -356,43 +398,34 @@ }, { "cell_type": "code", - "execution_count": 135, + "execution_count": 10, "metadata": { "id": "tLOWhg_CeWzH" }, "outputs": [], "source": [ "class TransformerModel(nn.Module):\n", - " def __init__(self, input_dim, model_dim, output_dim, num_heads, num_layers, seq_len, device, dropout):\n", + " def __init__(self, input_dim, model_dim, output_dim, num_heads, num_layers, seq_len, dropout):\n", " super().__init__()\n", - " self.embedding = nn.Embedding(input_dim, model_dim//2, dtype=torch.bfloat16)\n", - " # seq_len is odd\n", - " self.fancy_encoding = torch.repeat_interleave(torch.rand((1, seq_len // 2 + 1, model_dim // 2), device=device, dtype=torch.bfloat16), 2, dim=1)\n", - " # cut off last element since the target vertex is not repeated\n", - " self.fancy_encoding = self.fancy_encoding[:, :seq_len, :]\n", - " \n", " self.model_dim = model_dim\n", - " self.seq_len = seq_len\n", - " self.device = device\n", - "\n", + " self.embedding = nn.Embedding(input_dim, model_dim // 2, dtype=torch.bfloat16)\n", + " # # seq_len is odd\n", + " # self.fancy_encoding = torch.repeat_interleave(torch.rand((1, seq_len // 2 + 1, model_dim // 2), device=device, dtype=torch.bfloat16), 2, dim=1)\n", + " # # cut off last element since the target vertex is not repeated\n", + " # self.fancy_encoding = self.fancy_encoding[:, :seq_len, :]\n", " encoder_layer = nn.TransformerEncoderLayer(d_model=model_dim, nhead=num_heads,\n", " dim_feedforward=model_dim*4,\n", " dropout=dropout, batch_first=True, dtype=torch.bfloat16)\n", " self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers)\n", + " self.fc_out = nn.Linear(model_dim, output_dim, dtype=torch.bfloat16)\n", "\n", - " self.fc_out = nn.Linear(model_dim*seq_len, output_dim, dtype=torch.bfloat16)\n", - "\n", - " def full_embedding(self, src):\n", - " batch_size, src_len = src.size(0), src.size(1)\n", - " return torch.cat((self.embedding(src) * sqrt(self.model_dim), self.fancy_encoding.repeat((batch_size, 1, 1))), dim=2)\n", - " \n", " def forward(self, src, key_padding_mask):\n", - " embed = self.full_embedding(src)\n", - " output = self.transformer_encoder(embed, src_key_padding_mask=key_padding_mask)\n", - " output[key_padding_mask] = 0 # Hack to stop no_grad problem\n", - " flat_output = torch.flatten(output, start_dim=1, end_dim=2)\n", - " output = self.fc_out(flat_output)\n", - " return output" + " batch_sz = src.size(0)\n", + " embed = torch.cat((self.embedding(src[:,:-1:2]), self.embedding(src[:,1::2])), dim=2)\n", + " last_dude = torch.cat((self.embedding(src[:,-1:]), torch.ones((batch_sz, 1, self.model_dim // 2), dtype=torch.bfloat16, device=device)), dim=2)\n", + " final_embed = torch.cat((embed, last_dude), dim=1)\n", + " output = self.transformer_encoder(final_embed, src_key_padding_mask=key_padding_mask[:, ::2])\n", + " return self.fc_out(output[:, -1, :])" ] }, { @@ -406,7 +439,7 @@ }, { "cell_type": "code", - "execution_count": 136, + "execution_count": 11, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -419,8 +452,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Training data: 1049M\n", - "Trainable parameters in the model: 50K\n" + "Training data: 524M\n", + "Trainable parameters in the model: 800K\n" ] } ], @@ -429,17 +462,17 @@ "VOCAB_SIZE = 1 + MAX_VTXS # one more than the max number of vertices\n", "MODEL_DIM = 64 # Dimension of model (embedding and transformer)\n", "NEPOCHS = 1000\n", - "BSZ = 8196 * 4 # Batch size\n", + "BSZ = 2**14 # Batch size\n", "BPE = 32 # Batches per epoch\n", - "LR = 5e-3\n", - "WD = 2e-3\n", - "NHEADS = 1 #4\n", - "NLAYERS = 1 #16\n", - "DROPOUT = 0.2\n", + "LR = 5e-5\n", + "WD = 1e-5\n", + "NHEADS = 4\n", + "NLAYERS = 16\n", + "DROPOUT = 0 # 0.2\n", "model = TransformerModel(input_dim=VOCAB_SIZE, model_dim=MODEL_DIM,\n", " output_dim=1, num_heads=NHEADS,\n", " num_layers=NLAYERS, seq_len=SEQ_LEN,\n", - " dropout=DROPOUT, device=device).to(device)\n", + " dropout=DROPOUT).to(device)\n", "# model = torch.compile(model)\n", "\n", "criterion = nn.MSELoss()\n", @@ -452,21 +485,38 @@ }, { "cell_type": "code", - "execution_count": 137, + "execution_count": 121, "metadata": {}, "outputs": [], "source": [ - "from torch.utils.data import DataLoader, TensorDataset\n", + "# from torch.utils.data import DataLoader, TensorDataset\n", "\n", - "with open(\"data.pkl\", \"rb\") as f:\n", - " pickled_stuff = pickle.load(f)\n", + "# with open(\"data.pkl\", \"rb\") as f:\n", + "# pickled_stuff = pickle.load(f)\n", "\n", - "data = pickled_stuff[\"data\"].to(device)\n", - "label = pickled_stuff[\"labels\"].to(device)\n", - "padding_mask = (data == PAD_TOKEN).bool().to(device)\n", - "dataset = TensorDataset(data, label, padding_mask)\n", - "# train_dataset, test_dataset = torch.utils.data.random_split(dataset, [.9, .1])\n", - "train_loader = DataLoader(dataset, batch_size=BSZ, shuffle=True)" + "# data = pickled_stuff[\"data\"].to(device)\n", + "# label = pickled_stuff[\"labels\"].to(device)\n", + "# padding_mask = (data == PAD_TOKEN).bool().to(device)\n", + "# dataset = TensorDataset(data, label, padding_mask)\n", + "# # train_dataset, test_dataset = torch.utils.data.random_split(dataset, [.9, .1])\n", + "# train_loader = DataLoader(dataset, batch_size=BSZ, shuffle=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "# figure out if it's doing better on short paths\n", + "def evaluate_short():\n", + " model.eval()\n", + " test_loss = 0\n", + " with torch.no_grad():\n", + " batch_src, batch_labels, batch_padding_mask = mkbatch(BSZ)\n", + " output = model(batch_src, batch_padding_mask)\n", + " loss = criterion(output[batch_labels == 1].squeeze(1), batch_labels[batch_labels==1])\n", + " return loss.item()" ] }, { @@ -480,7 +530,7 @@ }, { "cell_type": "code", - "execution_count": 138, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -496,7 +546,7 @@ }, { "cell_type": "code", - "execution_count": 139, + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -510,580 +560,119 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.01it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 1/1000 \t Train Err: 0.5381 \t Test Err: 0.1865\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.75it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 2/1000 \t Train Err: 0.1227 \t Test Err: 0.1128\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.03it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 3/1000 \t Train Err: 0.1071 \t Test Err: 0.1118\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.01it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 4/1000 \t Train Err: 0.1008 \t Test Err: 0.1035\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.75it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 5/1000 \t Train Err: 0.0972 \t Test Err: 0.1021\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.03it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 6/1000 \t Train Err: 0.0949 \t Test Err: 0.0981\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.76it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 7/1000 \t Train Err: 0.0929 \t Test Err: 0.1021\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.00it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 8/1000 \t Train Err: 0.0908 \t Test Err: 0.0977\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.01it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 9/1000 \t Train Err: 0.0886 \t Test Err: 0.0952\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.72it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 10/1000 \t Train Err: 0.0910 \t Test Err: 0.0962\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.02it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 11/1000 \t Train Err: 0.0851 \t Test Err: 0.0898\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.03it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 12/1000 \t Train Err: 0.0849 \t Test Err: 0.0864\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.74it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 13/1000 \t Train Err: 0.0795 \t Test Err: 0.0684\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.02it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 14/1000 \t Train Err: 0.0691 \t Test Err: 0.0293\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.74it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 15/1000 \t Train Err: 0.0455 \t Test Err: 0.0271\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 6.99it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 16/1000 \t Train Err: 0.0421 \t Test Err: 0.0210\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 6.99it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 17/1000 \t Train Err: 0.0817 \t Test Err: 0.0505\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.75it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 18/1000 \t Train Err: 0.0456 \t Test Err: 0.0176\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.03it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 19/1000 \t Train Err: 0.0370 \t Test Err: 0.0165\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.03it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 20/1000 \t Train Err: 0.0374 \t Test Err: 0.0205\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.73it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 21/1000 \t Train Err: 0.0372 \t Test Err: 0.0142\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.01it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 22/1000 \t Train Err: 0.0343 \t Test Err: 0.0132\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.75it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 23/1000 \t Train Err: 0.0337 \t Test Err: 0.0119\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.03it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 24/1000 \t Train Err: 0.0713 \t Test Err: 0.0259\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.04it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 25/1000 \t Train Err: 0.0522 \t Test Err: 0.0143\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.75it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 26/1000 \t Train Err: 0.0342 \t Test Err: 0.0117\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.02it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 27/1000 \t Train Err: 0.0864 \t Test Err: 0.0728\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.04it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 28/1000 \t Train Err: 0.0701 \t Test Err: 0.0510\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.72it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 29/1000 \t Train Err: 0.0598 \t Test Err: 0.0369\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 6.99it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 30/1000 \t Train Err: 0.0462 \t Test Err: 0.0231\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.73it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 31/1000 \t Train Err: 0.0387 \t Test Err: 0.0181\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.02it/s]\n" + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:13<00:00, 2.30it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Epoch 32/1000 \t Train Err: 0.0351 \t Test Err: 0.0142\n" + "Epoch 1/1000 \t Train Err: 149.6562 \t Test Err: 137.0000, Test short loss: 0.3164\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.04it/s]\n" + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:13<00:00, 2.41it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Epoch 33/1000 \t Train Err: 0.0337 \t Test Err: 0.0123\n" + "Epoch 2/1000 \t Train Err: 129.4688 \t Test Err: 120.0000, Test short loss: 0.4512\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.74it/s]\n" + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:13<00:00, 2.39it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Epoch 34/1000 \t Train Err: 0.0331 \t Test Err: 0.0117\n" + "Epoch 3/1000 \t Train Err: 116.1719 \t Test Err: 110.5000, Test short loss: 2.2500\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.02it/s]\n" + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:13<00:00, 2.38it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Epoch 35/1000 \t Train Err: 0.0329 \t Test Err: 0.0119\n" + "Epoch 4/1000 \t Train Err: 107.9375 \t Test Err: 104.0000, Test short loss: 4.0625\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 6.98it/s]\n" + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:13<00:00, 2.37it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Epoch 36/1000 \t Train Err: 0.0327 \t Test Err: 0.0109\n" + "Epoch 5/1000 \t Train Err: 103.0938 \t Test Err: 102.0000, Test short loss: 5.5625\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.74it/s]\n" + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:13<00:00, 2.38it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Epoch 37/1000 \t Train Err: 0.0319 \t Test Err: 0.0101\n" + "Epoch 6/1000 \t Train Err: 99.9531 \t Test Err: 99.0000, Test short loss: 6.6250\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.02it/s]\n" + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:13<00:00, 2.41it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Epoch 38/1000 \t Train Err: 0.0317 \t Test Err: 0.0099\n" + "Epoch 7/1000 \t Train Err: 99.0469 \t Test Err: 101.0000, Test short loss: 7.2188\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.71it/s]\n" + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:13<00:00, 2.39it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Epoch 39/1000 \t Train Err: 0.0313 \t Test Err: 0.0095\n" + "Epoch 8/1000 \t Train Err: 97.8594 \t Test Err: 97.5000, Test short loss: 7.6250\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.01it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 40/1000 \t Train Err: 0.0310 \t Test Err: 0.0097\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 28%|██████████████████████████████████████████▊ | 9/32 [00:01<00:03, 6.50it/s]\n" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[139], line 11\u001b[0m\n\u001b[1;32m 9\u001b[0m train_loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m tqdm(\u001b[38;5;28mrange\u001b[39m(BPE)):\n\u001b[0;32m---> 11\u001b[0m batch_src, batch_labels, batch_padding_mask \u001b[38;5;241m=\u001b[39m \u001b[43mmkbatch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mBSZ\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 12\u001b[0m \u001b[38;5;66;03m# for batch_src, batch_labels, batch_padding_mask in tqdm(train_loader):\u001b[39;00m\n\u001b[1;32m 13\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n", - "Cell \u001b[0;32mIn[76], line 55\u001b[0m, in \u001b[0;36mmkbatch\u001b[0;34m(size)\u001b[0m\n\u001b[1;32m 53\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(size):\n\u001b[1;32m 54\u001b[0m n \u001b[38;5;241m=\u001b[39m random\u001b[38;5;241m.\u001b[39mrandint(MIN_VTXS, MAX_VTXS)\n\u001b[0;32m---> 55\u001b[0m edge_list, adj_list \u001b[38;5;241m=\u001b[39m \u001b[43mrandom_graph\u001b[49m\u001b[43m(\u001b[49m\u001b[43mn\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 56\u001b[0m dist \u001b[38;5;241m=\u001b[39m SSSP(n, adj_list)\n\u001b[1;32m 57\u001b[0m edge_list[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m2\u001b[39m \u001b[38;5;66;03m# target token\u001b[39;00m\n", - "Cell \u001b[0;32mIn[76], line 15\u001b[0m, in \u001b[0;36mrandom_graph\u001b[0;34m(n)\u001b[0m\n\u001b[1;32m 13\u001b[0m adjacencies \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mset\u001b[39m() \u001b[38;5;28;01mfor\u001b[39;00m _ \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(n\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m1\u001b[39m)]\n\u001b[1;32m 14\u001b[0m indices \u001b[38;5;241m=\u001b[39m [random\u001b[38;5;241m.\u001b[39mrandint(\u001b[38;5;241m1\u001b[39m, n) \u001b[38;5;28;01mfor\u001b[39;00m _ \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(AVG_DEG \u001b[38;5;241m*\u001b[39m (n\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m))]\n\u001b[0;32m---> 15\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28;43mrange\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mindices\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m)\u001b[49m:\n\u001b[1;32m 16\u001b[0m u \u001b[38;5;241m=\u001b[39m indices[i]\n\u001b[1;32m 17\u001b[0m v \u001b[38;5;241m=\u001b[39m indices[i \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m1\u001b[39m]\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + " 44%|██████████████████████████████████████████████████████████████████ | 14/32 [00:05<00:07, 2.41it/s]" ] } ], @@ -1108,12 +697,13 @@ " optimizer.step()\n", "\n", " test_loss = evaluate()\n", + " test_short_loss = evaluate_short()\n", " \n", " test_err.append(test_loss)\n", " train_err.append(train_loss)\n", " with open('loss', 'a') as f:\n", " f.write(f\"{train_loss} {test_loss}\\n\")\n", - " print(f\"Epoch {epoch + 1}/{NEPOCHS} \\t Train Err: {train_loss:.4f} \\t Test Err: {test_loss:.4f}\")\n", + " print(f\"Epoch {epoch + 1}/{NEPOCHS} \\t Train Err: {train_loss:.4f} \\t Test Err: {test_loss:.4f}, Test short loss: {test_short_loss:.4f}\")\n", " \n", " if epoch % 100 == 99:\n", " torch.save(model.state_dict(), f\"model_weights_{epoch}.pth\")" @@ -1121,55 +711,38 @@ }, { "cell_type": "code", - "execution_count": 163, + "execution_count": 125, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(torch.Size([1, 7, 64]), torch.Size([64, 64]))" - ] - }, - "execution_count": 163, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "\"\"\"\n", - "Now let's figure out what it's doing. \n", + "# \"\"\"\n", + "# Now let's figure out what it's doing. \n", "\n", - "step 1: figure out what people are attending to \n", - "\"\"\"\n", + "# step 1: figure out what people are attending to \n", + "# \"\"\"\n", + "\n", + "# example_graph, answer, padding = mkbatch(1)\n", + "# sentance_embeddings = model.full_embedding(example_graph)[0,:,:][example_graph.flatten() != 0]\n", + "# WQ,WK,WV = torch.split(model.transformer_encoder.layers[0].self_attn.in_proj_weight, (MODEL_DIM, MODEL_DIM, MODEL_DIM))\n", "\n", - "example_graph, answer, padding = mkbatch(1)\n", - "sentance_embeddings = model.full_embedding(example_graph)\n", - "Q,K,V = torch.split(model.transformer_encoder.layers[0].self_attn.in_proj_weight, (MODEL_DIM, MODEL_DIM, MODEL_DIM))\n", + "# Q = sentance_embeddings@WQ\n", + "# K = sentance_embeddings@WK\n", "\n", - "sentance_embeddings.shape, Q.shape\n", - "sentance_embeddings@Q.T\n", + "# raw_scores = Q @ K.T / sqrt(MODEL_DIM)\n", + "# soft = torch.softmax(raw_scores, dim=-1).detach().cpu().to(float).numpy()\n", + "# plt.imshow(soft)\n", + "# plt.show()\n", "\n", - "# (sentance_embeddings @ Q).shape\n", - "# sentance_embeddings.shape\n", - "# K @ sentance_embeddings" + "# print(example_graph)\n", + "\n", + "# print(Q)" ] }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "<matplotlib.legend.Legend at 0x702d2d2eed20>" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "plt.suptitle('MSE vs Epochs')\n", "plt.plot(train_err, label='Train', color='blue')\n", @@ -1182,83 +755,17 @@ }, { "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "24.625" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "evaluate()" - ] - }, - { - "cell_type": "code", "execution_count": null, "metadata": { "id": "LoGEmM5lH7_A" }, "outputs": [], "source": [ - "batch_src, batch_labels, batch_padding_mask = next(iter(train_loader))\n", + "batch_src, batch_labels, batch_padding_mask = mkbatch(4096)\n", "output = model(batch_src, batch_padding_mask)\n", - "batch_src[0], batch_labels[0], output[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plt.hist(output.detach().cpu().numpy().flatten(),bins=32)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plt.hist(label.detach().cpu().numpy().flatten(),bins=32)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plt.scatter(batch_labels.detach().cpu().numpy().flatten(),output.detach().cpu().numpy().flatten())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "batch_src2, batch_labels2, batch_padding_mask2 = next(iter(test_loader))\n", - "output2 = model(batch_src2, batch_padding_mask2)\n", - "loss = criterion(output2.squeeze(1), batch_labels2)\n", - "batch_src2[0], batch_labels2[0], output2[0], loss" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plt.scatter(batch_labels2.detach().cpu().numpy().flatten(),output2.detach().cpu().numpy().flatten())" + "batch_src[0], batch_labels[0], output[0]\n", + "plt.scatter(batch_labels.detach().to(torch.float16).cpu().numpy().flatten(), output.detach().to(torch.float16).cpu().numpy().flatten())\n", + "plt.show()" ] }, { @@ -1277,8 +784,8 @@ "outputs": [], "source": [ "N_TUNE_EPOCHS = 100\n", - "TUNE_LR = 0.003\n", - "TUNE_WD = 0.002\n", + "TUNE_LR = 1e-5\n", + "TUNE_WD = 1e-5\n", "\n", "tune_criterion = nn.MSELoss()\n", "tune_optimizer = torch.optim.Adam(model.parameters(), lr=TUNE_LR, weight_decay=TUNE_WD)" @@ -1306,32 +813,6 @@ "metadata": {}, "outputs": [], "source": [ - "# This has to be in a separate cell for some weird event loop reasons\n", - "%matplotlib widget\n", - "fig,ax = plt.subplots()\n", - "fig.suptitle('MSE vs Epochs')\n", - "plt.show()\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'N_TUNE_EPOCHS' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[13], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m tune_train_err \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 2\u001b[0m tune_test_err \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m----> 4\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m epoch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[43mN_TUNE_EPOCHS\u001b[49m):\n\u001b[1;32m 5\u001b[0m model\u001b[38;5;241m.\u001b[39mtrain()\n\u001b[1;32m 6\u001b[0m train_loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n", - "\u001b[0;31mNameError\u001b[0m: name 'N_TUNE_EPOCHS' is not defined" - ] - } - ], - "source": [ "tune_train_err = []\n", "tune_test_err = []\n", "\n", @@ -1346,7 +827,7 @@ " optimizer.zero_grad()\n", " output = model(batch_src, batch_padding_mask)\n", " loss = criterion(output.squeeze(1), batch_labels)\n", - " train_loss += loss.item()/BPE\n", + " train_loss += loss.item() / BPE\n", " loss.backward()\n", " optimizer.step()\n", "\n", @@ -1356,11 +837,6 @@ " tune_train_err.append(train_loss)\n", " with open('tune_loss', 'a') as f:\n", " f.write(f\"{train_loss} {test_loss}\\n\")\n", - " ax.plot(tune_train_err, label='Train', color='blue')\n", - " ax.plot(tune_test_err, label='Test', color='red')\n", - " ax.set_xlabel('Epochs')\n", - " ax.set_ylabel('MSE')\n", - " fig.canvas.draw()\n", " print(f\"Epoch {epoch + 1}/{NEPOCHS} \\t Train Err: {train_loss:.4f} \\t Test Err: {test_loss:.4f}\")\n", "\n", " if epoch % 10 == 9:\n", @@ -1368,6 +844,33 @@ ] }, { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.suptitle('MSE vs Epochs')\n", + "plt.plot(tune_train_err, label='Train', color='blue')\n", + "plt.plot(tune_test_err, label='Test', color='red')\n", + "plt.xlabel('Epochs')\n", + "plt.ylabel('MSE')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "batch_src, batch_labels, batch_padding_mask = mktunebatch(2048)\n", + "output = model(batch_src, batch_padding_mask)\n", + "batch_src[0], batch_labels[0], output[0]\n", + "plt.scatter(batch_labels.detach().to(torch.float16).cpu().numpy().flatten(), output.detach().to(torch.float16).cpu().numpy().flatten())" + ] + }, + { "cell_type": "markdown", "metadata": { "id": "JtTLXn4zC1z_" |