diff options
| -rw-r--r-- | transformer_shortest_paths.ipynb | 927 |
1 files changed, 798 insertions, 129 deletions
diff --git a/transformer_shortest_paths.ipynb b/transformer_shortest_paths.ipynb index 0adb5db..b2cea24 100644 --- a/transformer_shortest_paths.ipynb +++ b/transformer_shortest_paths.ipynb @@ -10,8 +10,18 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Question: \n", + "\n", + "- Do the attention heads learn to attend to the same positional encodings\n", + "- do interp -- what is it doing? can we figure out?" + ] + }, + { "cell_type": "code", - "execution_count": 1, + "execution_count": 75, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -28,9 +38,7 @@ "import torch\n", "import torch.nn as nn\n", "from math import sqrt\n", - "from torch.utils.data import DataLoader, TensorDataset\n", "import matplotlib.pyplot as plt\n", - "# %matplotlib widget\n", "torch.manual_seed(42)\n", "\n", "import os\n", @@ -43,18 +51,18 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 91, "metadata": { "id": "lylOX2POPwFL" }, "outputs": [], "source": [ - "SEQ_LEN = 65 # means 32 edges, final token is the target vertex\n", - "PAD_TOKEN = 0\n", + "# VTXS numbers here are inclusive\n", + "MIN_VTXS = 3 # 3\n", + "MAX_VTXS = 3 # 8\n", + "MAX_TUNE_VTXS = 3 # 15\n", "AVG_DEG = 2\n", - "MAX_VTXS = SEQ_LEN//AVG_DEG + 1 # 32 (exclusive)\n", - "MIN_VTXS = 8\n", - "MAX_TUNE_VTXS = 16\n", + "SEQ_LEN = MAX_VTXS * AVG_DEG + 1 # means 32 edges, final token is the target vertex\n", "PAD_TOKEN = 0\n", "# vertices are labelled 1,2,...,63\n", "# we also have a padding token which is 0." @@ -71,7 +79,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 76, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -94,7 +102,7 @@ "def random_graph(n):\n", " edge_list = []\n", " adjacencies = [set() for _ in range(n+1)]\n", - " indices = [random.randint(1, n-1) for _ in range(AVG_DEG * (n-1))]\n", + " indices = [random.randint(1, n) for _ in range(AVG_DEG * (n-1))]\n", " for i in range(0, len(indices), 2):\n", " u = indices[i]\n", " v = indices[i + 1]\n", @@ -134,7 +142,7 @@ " distance1 = []\n", " \n", " for i in range(size):\n", - " n = random.randrange(MIN_VTXS, MAX_VTXS)\n", + " n = random.randint(MIN_VTXS, MAX_VTXS)\n", " edge_list, adj_list = random_graph(n)\n", " dist = SSSP(n, adj_list)\n", " edge_list[-1] = 2 # target token\n", @@ -168,11 +176,11 @@ " return []\n", "\n", "def mktunebatch(size):\n", - " graphs2 = []\n", - " distance2 = []\n", + " graphs = []\n", + " distance = []\n", " \n", " for i in range(size):\n", - " n = random.randrange(MIN_VTXS, MAX_TUNE_VTXS)\n", + " n = random.randint(MIN_VTXS, MAX_TUNE_VTXS)\n", " while True:\n", " edge_list, adj_list = random_graph(n)\n", " path = vertices_on_shortest_12_path(n, adj_list)\n", @@ -180,51 +188,84 @@ " target_vtx_idx = random.randrange(1, len(path))\n", " target_vtx = path[target_vtx_idx]\n", " edge_list[-1] = target_vtx\n", - " graphs2.append(edge_list)\n", - " distance2.append(target_vtx_idx)\n", + " graphs.append(edge_list)\n", + " distance.append(target_vtx_idx)\n", " break\n", " \n", - " data = torch.tensor(graphs2, device=device)\n", - " labels = torch.tensor(distance2, dtype=torch.bfloat16, device=device)\n", + " data = torch.tensor(graphs, device=device)\n", + " labels = torch.tensor(distance, dtype=torch.bfloat16, device=device)\n", " padding = data == PAD_TOKEN\n", " return data, labels, padding" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 86, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "(array([252., 0., 309., 0., 278., 0., 199., 0., 119., 0., 63.,\n", - " 0., 23., 0., 45., 0., 36., 0., 36., 0., 28., 0.,\n", - " 37., 0., 23., 0., 33., 0., 24., 0., 34., 0., 0.,\n", - " 30., 0., 33., 0., 34., 0., 34., 0., 25., 0., 33.,\n", - " 0., 39., 0., 33., 0., 20., 0., 29., 0., 27., 0.,\n", - " 35., 0., 37., 0., 30., 0., 33., 0., 37.]),\n", - " array([ 1. , 1.484375, 1.96875 , 2.453125, 2.9375 , 3.421875,\n", - " 3.90625 , 4.390625, 4.875 , 5.359375, 5.84375 , 6.328125,\n", - " 6.8125 , 7.296875, 7.78125 , 8.265625, 8.75 , 9.234375,\n", - " 9.71875 , 10.203125, 10.6875 , 11.171875, 11.65625 , 12.140625,\n", - " 12.625 , 13.109375, 13.59375 , 14.078125, 14.5625 , 15.046875,\n", - " 15.53125 , 16.015625, 16.5 , 16.984375, 17.46875 , 17.953125,\n", - " 18.4375 , 18.921875, 19.40625 , 19.890625, 20.375 , 20.859375,\n", - " 21.34375 , 21.828125, 22.3125 , 22.796875, 23.28125 , 23.765625,\n", - " 24.25 , 24.734375, 25.21875 , 25.703125, 26.1875 , 26.671875,\n", - " 27.15625 , 27.640625, 28.125 , 28.609375, 29.09375 , 29.578125,\n", - " 30.0625 , 30.546875, 31.03125 , 31.515625, 32. ]),\n", + "(tensor([[4, 3, 2, 3, 1, 2, 0, 0, 2],\n", + " [3, 4, 3, 1, 2, 3, 0, 0, 2],\n", + " [4, 2, 3, 2, 0, 0, 0, 0, 2],\n", + " [3, 4, 4, 3, 0, 0, 0, 0, 2],\n", + " [1, 2, 2, 4, 0, 0, 0, 0, 2]], device='cuda:0'),\n", + " tensor([1., 2., 4., 4., 1.], device='cuda:0', dtype=torch.bfloat16),\n", + " tensor([[False, False, False, False, False, False, True, True, False],\n", + " [False, False, False, False, False, False, True, True, False],\n", + " [False, False, False, False, True, True, True, True, False],\n", + " [False, False, False, False, True, True, True, True, False],\n", + " [False, False, False, False, True, True, True, True, False]],\n", + " device='cuda:0'))" + ] + }, + "execution_count": 86, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mkbatch(5)" + ] + }, + { + "cell_type": "code", + "execution_count": 87, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(array([ 663., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 284., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 39., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 1062.]),\n", + " array([1. , 1.046875, 1.09375 , 1.140625, 1.1875 , 1.234375,\n", + " 1.28125 , 1.328125, 1.375 , 1.421875, 1.46875 , 1.515625,\n", + " 1.5625 , 1.609375, 1.65625 , 1.703125, 1.75 , 1.796875,\n", + " 1.84375 , 1.890625, 1.9375 , 1.984375, 2.03125 , 2.078125,\n", + " 2.125 , 2.171875, 2.21875 , 2.265625, 2.3125 , 2.359375,\n", + " 2.40625 , 2.453125, 2.5 , 2.546875, 2.59375 , 2.640625,\n", + " 2.6875 , 2.734375, 2.78125 , 2.828125, 2.875 , 2.921875,\n", + " 2.96875 , 3.015625, 3.0625 , 3.109375, 3.15625 , 3.203125,\n", + " 3.25 , 3.296875, 3.34375 , 3.390625, 3.4375 , 3.484375,\n", + " 3.53125 , 3.578125, 3.625 , 3.671875, 3.71875 , 3.765625,\n", + " 3.8125 , 3.859375, 3.90625 , 3.953125, 4. ]),\n", " <BarContainer object of 64 artists>)" ] }, - "execution_count": 4, + "execution_count": 87, "metadata": {}, "output_type": "execute_result" }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAiy0lEQVR4nO3df1BVdeL/8RegXH9xL6HChRXxV4nkj1pSvFO5rrAisq6uNJPllrWOTu6lWaVMaUzLdhbXmn4O6exsqzUjWe6kjlSWYuK2oiXF+KsYdWixhQutjlzFQITz/WPH+/3c/EEoct9cno+ZM8M9533vfZ+zp+XpuT8IsSzLEgAAgEFCAz0BAACAHyNQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABinW6AncD1aWlpUVVWliIgIhYSEBHo6AADgJ7AsS2fPnlVcXJxCQ699jaRTBkpVVZXi4+MDPQ0AAHAdTp48qQEDBlxzTKcMlIiICEn/20G73R7g2QAAgJ/C6/UqPj7e93v8WjploFx6WcdutxMoAAB0Mj/l7Rm8SRYAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMbpFugJBJNBSz+46rZvV2V24EwAAOjcuIICAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIzTpkBZs2aNRo8eLbvdLrvdLpfLpY8++si3vaGhQW63W3379lWfPn2UlZWlmpoav8eorKxUZmamevXqpejoaC1evFgXL15sn70BAABBoU2BMmDAAK1atUqlpaU6cOCAJk2apOnTp+vIkSOSpEWLFmnbtm3atGmTiouLVVVVpZkzZ/ru39zcrMzMTF24cEF79+7VW2+9pfXr12v58uXtu1cAAKBTC7Esy7qRB4iKitILL7yg++67T/3791dBQYHuu+8+SdI333yjESNGqKSkROPHj9dHH32kX//616qqqlJMTIwkae3atVqyZIm+//57hYeH/6Tn9Hq9cjgcqqurk91uv5Hptyu+BwUAgKtry+/v634PSnNzszZu3Kj6+nq5XC6VlpaqqalJaWlpvjGJiYkaOHCgSkpKJEklJSUaNWqUL04kKT09XV6v13cVBgAAoM3fJHvo0CG5XC41NDSoT58+2rx5s5KSklRWVqbw8HBFRkb6jY+JiZHH45EkeTwevzi5tP3StqtpbGxUY2Oj77bX623rtAEAQCfS5isow4cPV1lZmfbv368FCxZozpw5Onr06M2Ym09eXp4cDodviY+Pv6nPBwAAAqvNgRIeHq5hw4YpOTlZeXl5GjNmjF599VU5nU5duHBBZ86c8RtfU1Mjp9MpSXI6nZd9qufS7UtjriQ3N1d1dXW+5eTJk22dNgAA6ERu+HtQWlpa1NjYqOTkZHXv3l1FRUW+beXl5aqsrJTL5ZIkuVwuHTp0SLW1tb4xO3bskN1uV1JS0lWfw2az+T7afGkBAADBq03vQcnNzVVGRoYGDhyos2fPqqCgQLt379bHH38sh8OhuXPnKicnR1FRUbLb7Xr88cflcrk0fvx4SdLkyZOVlJSkhx56SKtXr5bH49GyZcvkdrtls9luyg4CAIDOp02BUltbq4cffljV1dVyOBwaPXq0Pv74Y/3qV7+SJL388ssKDQ1VVlaWGhsblZ6erjfeeMN3/7CwMBUWFmrBggVyuVzq3bu35syZo5UrV7bvXgEAgE7thr8HJRD4HhQAADqfDvkeFAAAgJuFQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcboFegKQBi394Krbvl2V2YEzAQDADFxBAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcdoUKHl5eRo7dqwiIiIUHR2tGTNmqLy83G/MxIkTFRIS4rc89thjfmMqKyuVmZmpXr16KTo6WosXL9bFixdvfG8AAEBQ6NaWwcXFxXK73Ro7dqwuXryop59+WpMnT9bRo0fVu3dv37h58+Zp5cqVvtu9evXy/dzc3KzMzEw5nU7t3btX1dXVevjhh9W9e3f9+c9/boddAgAAnV2bAmX79u1+t9evX6/o6GiVlpZqwoQJvvW9evWS0+m84mN88sknOnr0qHbu3KmYmBjdcccdev7557VkyRI9++yzCg8Pv47daF+Dln5w1W3frsrswJkAANA13dB7UOrq6iRJUVFRfus3bNigfv36aeTIkcrNzdX58+d920pKSjRq1CjFxMT41qWnp8vr9erIkSNXfJ7GxkZ5vV6/BQAABK82XUH5v1paWrRw4ULdfffdGjlypG/9gw8+qISEBMXFxengwYNasmSJysvL9f7770uSPB6PX5xI8t32eDxXfK68vDw999xz1ztVAADQyVx3oLjdbh0+fFifffaZ3/r58+f7fh41apRiY2OVmpqqEydOaOjQodf1XLm5ucrJyfHd9nq9io+Pv76JAwAA413XSzzZ2dkqLCzUp59+qgEDBlxzbEpKiiTp+PHjkiSn06mamhq/MZduX+19KzabTXa73W8BAADBq02BYlmWsrOztXnzZu3atUuDBw9u9T5lZWWSpNjYWEmSy+XSoUOHVFtb6xuzY8cO2e12JSUltWU6AAAgSLXpJR63262CggJt3bpVERERvveMOBwO9ezZUydOnFBBQYGmTp2qvn376uDBg1q0aJEmTJig0aNHS5ImT56spKQkPfTQQ1q9erU8Ho+WLVsmt9stm83W/nsIAAA6nTZdQVmzZo3q6uo0ceJExcbG+pZ3331XkhQeHq6dO3dq8uTJSkxM1BNPPKGsrCxt27bN9xhhYWEqLCxUWFiYXC6Xfve73+nhhx/2+94UAADQtbXpCoplWdfcHh8fr+Li4lYfJyEhQR9++GFbnhoAAHQh/C0eAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGKdNgZKXl6exY8cqIiJC0dHRmjFjhsrLy/3GNDQ0yO12q2/fvurTp4+ysrJUU1PjN6ayslKZmZnq1auXoqOjtXjxYl28ePHG9wYAAASFNgVKcXGx3G639u3bpx07dqipqUmTJ09WfX29b8yiRYu0bds2bdq0ScXFxaqqqtLMmTN925ubm5WZmakLFy5o7969euutt7R+/XotX768/fYKAAB0at3aMnj79u1+t9evX6/o6GiVlpZqwoQJqqur05tvvqmCggJNmjRJkrRu3TqNGDFC+/bt0/jx4/XJJ5/o6NGj2rlzp2JiYnTHHXfo+eef15IlS/Tss88qPDy8/fYOAAB0Sjf0HpS6ujpJUlRUlCSptLRUTU1NSktL841JTEzUwIEDVVJSIkkqKSnRqFGjFBMT4xuTnp4ur9erI0eOXPF5Ghsb5fV6/RYAABC8rjtQWlpatHDhQt19990aOXKkJMnj8Sg8PFyRkZF+Y2NiYuTxeHxj/m+cXNp+aduV5OXlyeFw+Jb4+PjrnTYAAOgErjtQ3G63Dh8+rI0bN7bnfK4oNzdXdXV1vuXkyZM3/TkBAEDgtOk9KJdkZ2ersLBQe/bs0YABA3zrnU6nLly4oDNnzvhdRampqZHT6fSN+fzzz/0e79KnfC6N+TGbzSabzXY9Uw1qg5Z+cNVt367K7MCZAADQvtp0BcWyLGVnZ2vz5s3atWuXBg8e7Lc9OTlZ3bt3V1FRkW9deXm5Kisr5XK5JEkul0uHDh1SbW2tb8yOHTtkt9uVlJR0I/sCAACCRJuuoLjdbhUUFGjr1q2KiIjwvWfE4XCoZ8+ecjgcmjt3rnJychQVFSW73a7HH39cLpdL48ePlyRNnjxZSUlJeuihh7R69Wp5PB4tW7ZMbrebqyQAAEBSGwNlzZo1kqSJEyf6rV+3bp0eeeQRSdLLL7+s0NBQZWVlqbGxUenp6XrjjTd8Y8PCwlRYWKgFCxbI5XKpd+/emjNnjlauXHljewIAAIJGmwLFsqxWx/To0UP5+fnKz8+/6piEhAR9+OGHbXlqAADQhfC3eAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGCcNgfKnj17NG3aNMXFxSkkJERbtmzx2/7II48oJCTEb5kyZYrfmNOnT2v27Nmy2+2KjIzU3Llzde7cuRvaEQAAEDzaHCj19fUaM2aM8vPzrzpmypQpqq6u9i3vvPOO3/bZs2fryJEj2rFjhwoLC7Vnzx7Nnz+/7bMHAABBqVtb75CRkaGMjIxrjrHZbHI6nVfc9vXXX2v79u364osvdNddd0mSXn/9dU2dOlUvvvii4uLi2jolAAAQZG7Ke1B2796t6OhoDR8+XAsWLNCpU6d820pKShQZGemLE0lKS0tTaGio9u/ff8XHa2xslNfr9VsAAEDwavdAmTJlit5++20VFRXpL3/5i4qLi5WRkaHm5mZJksfjUXR0tN99unXrpqioKHk8nis+Zl5enhwOh2+Jj49v72kDAACDtPklntbMmjXL9/OoUaM0evRoDR06VLt371Zqaup1PWZubq5ycnJ8t71eL5ECAEAQu+kfMx4yZIj69eun48ePS5KcTqdqa2v9xly8eFGnT5++6vtWbDab7Ha73wIAAILXTQ+U7777TqdOnVJsbKwkyeVy6cyZMyotLfWN2bVrl1paWpSSknKzpwMAADqBNr/Ec+7cOd/VEEmqqKhQWVmZoqKiFBUVpeeee05ZWVlyOp06ceKEnnrqKQ0bNkzp6emSpBEjRmjKlCmaN2+e1q5dq6amJmVnZ2vWrFl8ggcAAEi6jisoBw4c0J133qk777xTkpSTk6M777xTy5cvV1hYmA4ePKjf/OY3uu222zR37lwlJyfrn//8p2w2m+8xNmzYoMTERKWmpmrq1Km655579Ne//rX99goAAHRqbb6CMnHiRFmWddXtH3/8cauPERUVpYKCgrY+NQAA6CL4WzwAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDjdAj0BdLxBSz+46rZvV2V24EwAALgyrqAAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIzT5kDZs2ePpk2bpri4OIWEhGjLli1+2y3L0vLlyxUbG6uePXsqLS1Nx44d8xtz+vRpzZ49W3a7XZGRkZo7d67OnTt3QzsCAACCR5sDpb6+XmPGjFF+fv4Vt69evVqvvfaa1q5dq/3796t3795KT09XQ0ODb8zs2bN15MgR7dixQ4WFhdqzZ4/mz59//XsBAACCSre23iEjI0MZGRlX3GZZll555RUtW7ZM06dPlyS9/fbbiomJ0ZYtWzRr1ix9/fXX2r59u7744gvdddddkqTXX39dU6dO1Ysvvqi4uLgb2B0AABAM2vU9KBUVFfJ4PEpLS/OtczgcSklJUUlJiSSppKREkZGRvjiRpLS0NIWGhmr//v1XfNzGxkZ5vV6/BQAABK92DRSPxyNJiomJ8VsfExPj2+bxeBQdHe23vVu3boqKivKN+bG8vDw5HA7fEh8f357TBgAAhukUn+LJzc1VXV2dbzl58mSgpwQAAG6idg0Up9MpSaqpqfFbX1NT49vmdDpVW1vrt/3ixYs6ffq0b8yP2Ww22e12vwUAAASvdg2UwYMHy+l0qqioyLfO6/Vq//79crlckiSXy6UzZ86otLTUN2bXrl1qaWlRSkpKe04HAAB0Um3+FM+5c+d0/Phx3+2KigqVlZUpKipKAwcO1MKFC/WnP/1Jt956qwYPHqxnnnlGcXFxmjFjhiRpxIgRmjJliubNm6e1a9eqqalJ2dnZmjVrFp/gAQAAkq4jUA4cOKBf/vKXvts5OTmSpDlz5mj9+vV66qmnVF9fr/nz5+vMmTO65557tH37dvXo0cN3nw0bNig7O1upqakKDQ1VVlaWXnvttXbYHQAAEAzaHCgTJ06UZVlX3R4SEqKVK1dq5cqVVx0TFRWlgoKCtj41AADoIjrFp3gAAEDXQqAAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwTrdATwCdx6ClH1x127erMjtwJgCAYMcVFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGCcdg+UZ599ViEhIX5LYmKib3tDQ4Pcbrf69u2rPn36KCsrSzU1Ne09DQAA0IndlCsot99+u6qrq33LZ5995tu2aNEibdu2TZs2bVJxcbGqqqo0c+bMmzENAADQSd2Uv8XTrVs3OZ3Oy9bX1dXpzTffVEFBgSZNmiRJWrdunUaMGKF9+/Zp/PjxN2M6AACgk7kpV1COHTumuLg4DRkyRLNnz1ZlZaUkqbS0VE1NTUpLS/ONTUxM1MCBA1VSUnLVx2tsbJTX6/VbAABA8Gr3QElJSdH69eu1fft2rVmzRhUVFbr33nt19uxZeTwehYeHKzIy0u8+MTEx8ng8V33MvLw8ORwO3xIfH9/e0wYAAAZp95d4MjIyfD+PHj1aKSkpSkhI0HvvvaeePXte12Pm5uYqJyfHd9vr9RIpAAAEsZv+MePIyEjddtttOn78uJxOpy5cuKAzZ874jampqbnie1YusdlsstvtfgsAAAheNz1Qzp07pxMnTig2NlbJycnq3r27ioqKfNvLy8tVWVkpl8t1s6cCAAA6iXZ/iefJJ5/UtGnTlJCQoKqqKq1YsUJhYWF64IEH5HA4NHfuXOXk5CgqKkp2u12PP/64XC4Xn+ABAAA+7R4o3333nR544AGdOnVK/fv31z333KN9+/apf//+kqSXX35ZoaGhysrKUmNjo9LT0/XGG2+09zQAAEAn1u6BsnHjxmtu79Gjh/Lz85Wfn9/eT40gM2jpB1fd9u2qzA6cCQCgo/G3eAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgnHb/mDEAoHPhI/0wEVdQAACAcbiCAgQQ/3IFgk9n+u/a5LkSKAg61/sf3LXu19p9AQDti0ABAOAKTL660BXwHhQAAGAcrqAAuGmC4V+gN+Mlw86y70AgEShAF9KVf2l2hX3v6H3sCse0MwjW/x14iQcAABiHQAEAAMbhJR4AnUqwXs4GOLf9ESgAgE6BX+BdC4ECtAP+jxMA2hfvQQEAAMbhCgoAIKhxhbNz4goKAAAwDldQAMAw/IsfIFAA/AT8wgTQ0QgU3HT8cmt/HFMAwY73oAAAAOMQKAAAwDi8xAMA6FC8RImfgisoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADBOQAMlPz9fgwYNUo8ePZSSkqLPP/88kNMBAACGCFigvPvuu8rJydGKFSv05ZdfasyYMUpPT1dtbW2gpgQAAAwRsEB56aWXNG/ePD366KNKSkrS2rVr1atXL/39738P1JQAAIAhugXiSS9cuKDS0lLl5ub61oWGhiotLU0lJSWXjW9sbFRjY6Pvdl1dnSTJ6/XelPm1NJ6/6rZrPSf3C977BeI5u/L9rqWz7MON7HtnmSv369z3u9H7Xo9Lj2lZVuuDrQD4z3/+Y0my9u7d67d+8eLF1rhx4y4bv2LFCksSCwsLCwsLSxAsJ0+ebLUVAnIFpa1yc3OVk5Pju93S0qLTp0+rb9++CgkJuWy81+tVfHy8Tp48Kbvd3pFT7TQ4Rq3jGF0bx6d1HKPWcYyuLdiOj2VZOnv2rOLi4lodG5BA6devn8LCwlRTU+O3vqamRk6n87LxNptNNpvNb11kZGSrz2O324Pif9CbiWPUOo7RtXF8Wscxah3H6NqC6fg4HI6fNC4gb5INDw9XcnKyioqKfOtaWlpUVFQkl8sViCkBAACDBOwlnpycHM2ZM0d33XWXxo0bp1deeUX19fV69NFHAzUlAABgiIAFyv3336/vv/9ey5cvl8fj0R133KHt27crJibmhh/bZrNpxYoVl70shP+PY9Q6jtG1cXxaxzFqHcfo2rry8QmxrJ/yWR8AAICOw9/iAQAAxiFQAACAcQgUAABgHAIFAAAYJygDJT8/X4MGDVKPHj2UkpKizz//PNBTMsazzz6rkJAQvyUxMTHQ0wqYPXv2aNq0aYqLi1NISIi2bNnit92yLC1fvlyxsbHq2bOn0tLSdOzYscBMNkBaO0aPPPLIZefUlClTAjPZAMjLy9PYsWMVERGh6OhozZgxQ+Xl5X5jGhoa5Ha71bdvX/Xp00dZWVmXfVFlMPspx2jixImXnUePPfZYgGbc8dasWaPRo0f7vpDN5XLpo48+8m3viudQ0AXKu+++q5ycHK1YsUJffvmlxowZo/T0dNXW1gZ6asa4/fbbVV1d7Vs+++yzQE8pYOrr6zVmzBjl5+dfcfvq1av12muvae3atdq/f7969+6t9PR0NTQ0dPBMA6e1YyRJU6ZM8Tun3nnnnQ6cYWAVFxfL7XZr37592rFjh5qamjR58mTV19f7xixatEjbtm3Tpk2bVFxcrKqqKs2cOTOAs+5YP+UYSdK8efP8zqPVq1cHaMYdb8CAAVq1apVKS0t14MABTZo0SdOnT9eRI0ckddFzqF3++p9Bxo0bZ7ndbt/t5uZmKy4uzsrLywvgrMyxYsUKa8yYMYGehpEkWZs3b/bdbmlpsZxOp/XCCy/41p05c8ay2WzWO++8E4AZBt6Pj5FlWdacOXOs6dOnB2Q+JqqtrbUkWcXFxZZl/e+c6d69u7Vp0ybfmK+//tqSZJWUlARqmgH142NkWZb1i1/8wvrjH/8YuEkZ6JZbbrH+9re/ddlzKKiuoFy4cEGlpaVKS0vzrQsNDVVaWppKSkoCODOzHDt2THFxcRoyZIhmz56tysrKQE/JSBUVFfJ4PH7nk8PhUEpKCufTj+zevVvR0dEaPny4FixYoFOnTgV6SgFTV1cnSYqKipIklZaWqqmpye88SkxM1MCBA7vsefTjY3TJhg0b1K9fP40cOVK5ubk6f/58IKYXcM3Nzdq4caPq6+vlcrm67DnUKf6a8U/13//+V83NzZd9G21MTIy++eabAM3KLCkpKVq/fr2GDx+u6upqPffcc7r33nt1+PBhRUREBHp6RvF4PJJ0xfPp0jb87+WdmTNnavDgwTpx4oSefvppZWRkqKSkRGFhYYGeXodqaWnRwoULdffdd2vkyJGS/ncehYeHX/YHTrvqeXSlYyRJDz74oBISEhQXF6eDBw9qyZIlKi8v1/vvvx/A2XasQ4cOyeVyqaGhQX369NHmzZuVlJSksrKyLnkOBVWgoHUZGRm+n0ePHq2UlBQlJCTovffe09y5cwM4M3RWs2bN8v08atQojR49WkOHDtXu3buVmpoawJl1PLfbrcOHD3fp93W15mrHaP78+b6fR40apdjYWKWmpurEiRMaOnRoR08zIIYPH66ysjLV1dXpH//4h+bMmaPi4uJATytgguolnn79+iksLOyydzbX1NTI6XQGaFZmi4yM1G233abjx48HeirGuXTOcD61zZAhQ9SvX78ud05lZ2ersLBQn376qQYMGOBb73Q6deHCBZ05c8ZvfFc8j652jK4kJSVFkrrUeRQeHq5hw4YpOTlZeXl5GjNmjF599dUuew4FVaCEh4crOTlZRUVFvnUtLS0qKiqSy+UK4MzMde7cOZ04cUKxsbGBnopxBg8eLKfT6Xc+eb1e7d+/n/PpGr777judOnWqy5xTlmUpOztbmzdv1q5duzR48GC/7cnJyerevbvfeVReXq7Kysoucx61doyupKysTJK6zHl0JS0tLWpsbOy651Cg36Xb3jZu3GjZbDZr/fr11tGjR6358+dbkZGRlsfjCfTUjPDEE09Yu3fvtioqKqx//etfVlpamtWvXz+rtrY20FMLiLNnz1pfffWV9dVXX1mSrJdeesn66quvrH//+9+WZVnWqlWrrMjISGvr1q3WwYMHrenTp1uDBw+2fvjhhwDPvONc6xidPXvWevLJJ62SkhKroqLC2rlzp/Xzn//cuvXWW62GhoZAT71DLFiwwHI4HNbu3but6upq33L+/HnfmMcee8waOHCgtWvXLuvAgQOWy+WyXC5XAGfdsVo7RsePH7dWrlxpHThwwKqoqLC2bt1qDRkyxJowYUKAZ95xli5dahUXF1sVFRXWwYMHraVLl1ohISHWJ598YllW1zyHgi5QLMuyXn/9dWvgwIFWeHi4NW7cOGvfvn2BnpIx7r//fis2NtYKDw+3fvazn1n333+/dfz48UBPK2A+/fRTS9Jly5w5cyzL+t9HjZ955hkrJibGstlsVmpqqlVeXh7YSXewax2j8+fPW5MnT7b69+9vde/e3UpISLDmzZvXpf5BcKVjI8lat26db8wPP/xg/eEPf7BuueUWq1evXtZvf/tbq7q6OnCT7mCtHaPKykprwoQJVlRUlGWz2axhw4ZZixcvturq6gI78Q70+9//3kpISLDCw8Ot/v37W6mpqb44sayueQ6FWJZlddz1GgAAgNYF1XtQAABAcCBQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGOf/Ab6LlksYuhZ8AAAAAElFTkSuQmCC", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhJUlEQVR4nO3df1DVVeL/8ReI/NC8F7HlXhjR2HJVSjOl6GqffslKyTY5MVvssi5brjYutJKlC7NJmRXquloWqf0Sd7Ox2hndsiIJV5gSkVA2IjO3LGntwu4Y9yqtqPD+/tF4v12VwrpwOfJ8zNwZeb/P+95zz5wZnr69YIhlWZYAAAAMEhrsCQAAAJwtAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAccKCPYHu0tHRoYMHD2rQoEEKCQkJ9nQAAEAXWJalw4cPKz4+XqGhnd9nOWcD5uDBg0pISAj2NAAAwPfQ2NiooUOHdnr+nA2YQYMGSfp6AWw2W5BnAwAAusLr9SohIcH3fbwz52zAnPxnI5vNRsAAAGCY7/r4Bx/iBQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAccKCPQEAANA7XZD/WqfnPl2c3oMzOR13YAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxjnrgKmsrNRNN92k+Ph4hYSEaNOmTX7nLctSYWGh4uLiFBUVpdTUVO3bt89vzKFDh5SVlSWbzabo6GjNmDFDR44c8Rvz3nvv6f/+7/8UGRmphIQELV269OzfHQAAOCeddcC0trbq0ksvVXFx8RnPL126VCtXrtTq1atVXV2tgQMHKi0tTUePHvWNycrKUkNDg8rKyrR582ZVVlZq1qxZvvNer1dTpkzR8OHDVVtbqz/96U964IEH9NRTT32PtwgAAM41IZZlWd/74pAQbdy4UdOmTZP09d2X+Ph43XPPPbr33nslSR6PRw6HQyUlJcrMzNSePXuUlJSkmpoaJScnS5JKS0s1depUff7554qPj9eqVav0xz/+UW63W+Hh4ZKk/Px8bdq0SR9++GGX5ub1emW32+XxeGSz2b7vWwQAoM+6IP+1Ts99uji9W16zq9+/A/oZmP3798vtdis1NdV3zG63KyUlRVVVVZKkqqoqRUdH++JFklJTUxUaGqrq6mrfmKuvvtoXL5KUlpamvXv36ssvvzzja7e1tcnr9fo9AADAuSmgAeN2uyVJDofD77jD4fCdc7vdio2N9TsfFhammJgYvzFneo5vvsapioqKZLfbfY+EhIQf/oYAAECvdM78FFJBQYE8Ho/v0djYGOwpAQCAbhLQgHE6nZKkpqYmv+NNTU2+c06nU83NzX7nT5w4oUOHDvmNOdNzfPM1ThURESGbzeb3AAAA56aABkxiYqKcTqfKy8t9x7xer6qrq+VyuSRJLpdLLS0tqq2t9Y3ZunWrOjo6lJKS4htTWVmp48eP+8aUlZVp5MiRGjx4cCCnDAAADHTWAXPkyBHV1dWprq5O0tcf3K2rq9OBAwcUEhKivLw8PfTQQ3rllVdUX1+vX//614qPj/f9pNLo0aN1ww03aObMmdq5c6feeecd5ebmKjMzU/Hx8ZKkX/7ylwoPD9eMGTPU0NCgF198UY899pjmzp0bsDcOAADMFXa2F7z77ru67rrrfF+fjIrs7GyVlJRo/vz5am1t1axZs9TS0qKrrrpKpaWlioyM9F2zfv165ebmavLkyQoNDVVGRoZWrlzpO2+327Vlyxbl5ORowoQJOv/881VYWOj3u2IAAEDf9YN+D0xvxu+BAQDgh+kzvwcGAACgJxAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADBOwAOmvb1dCxYsUGJioqKionThhRdq0aJFsizLN8ayLBUWFiouLk5RUVFKTU3Vvn37/J7n0KFDysrKks1mU3R0tGbMmKEjR44EeroAAMBAAQ+YJUuWaNWqVXriiSe0Z88eLVmyREuXLtXjjz/uG7N06VKtXLlSq1evVnV1tQYOHKi0tDQdPXrUNyYrK0sNDQ0qKyvT5s2bVVlZqVmzZgV6ugAAwEAh1jdvjQTAz372MzkcDj377LO+YxkZGYqKitLzzz8vy7IUHx+ve+65R/fee68kyePxyOFwqKSkRJmZmdqzZ4+SkpJUU1Oj5ORkSVJpaammTp2qzz//XPHx8d85D6/XK7vdLo/HI5vNFsi3CABAn3BB/mudnvt0cXq3vGZXv38H/A7MxIkTVV5ero8++kiS9M9//lNvv/22brzxRknS/v375Xa7lZqa6rvGbrcrJSVFVVVVkqSqqipFR0f74kWSUlNTFRoaqurq6jO+bltbm7xer98DAACcm8IC/YT5+fnyer0aNWqU+vXrp/b2dj388MPKysqSJLndbkmSw+Hwu87hcPjOud1uxcbG+k80LEwxMTG+MacqKirSwoULA/12AABALxTwOzAvvfSS1q9frxdeeEG7du3SunXrtGzZMq1bty7QL+WnoKBAHo/H92hsbOzW1wMAAMET8Dsw8+bNU35+vjIzMyVJY8aM0WeffaaioiJlZ2fL6XRKkpqamhQXF+e7rqmpSePGjZMkOZ1ONTc3+z3viRMndOjQId/1p4qIiFBERESg3w4AAOiFAn4H5quvvlJoqP/T9uvXTx0dHZKkxMREOZ1OlZeX+857vV5VV1fL5XJJklwul1paWlRbW+sbs3XrVnV0dCglJSXQUwYAAIYJ+B2Ym266SQ8//LCGDRumiy++WLt379by5ct1xx13SJJCQkKUl5enhx56SCNGjFBiYqIWLFig+Ph4TZs2TZI0evRo3XDDDZo5c6ZWr16t48ePKzc3V5mZmV36CSQAAHBuC3jAPP7441qwYIF+97vfqbm5WfHx8brzzjtVWFjoGzN//ny1trZq1qxZamlp0VVXXaXS0lJFRkb6xqxfv165ubmaPHmyQkNDlZGRoZUrVwZ6ugAAwEAB/z0wvQW/BwYAgB+mT/0eGAAAgO5GwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDhhwZ6AiS7If63Tc58uTu/BmQAA0DdxBwYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHG6JWD+/e9/61e/+pWGDBmiqKgojRkzRu+++67vvGVZKiwsVFxcnKKiopSamqp9+/b5PcehQ4eUlZUlm82m6OhozZgxQ0eOHOmO6QIAAMMEPGC+/PJLTZo0Sf3799cbb7yhDz74QH/+8581ePBg35ilS5dq5cqVWr16taqrqzVw4EClpaXp6NGjvjFZWVlqaGhQWVmZNm/erMrKSs2aNSvQ0wUAAAYKC/QTLlmyRAkJCVq7dq3vWGJiou/PlmXp0Ucf1X333aebb75ZkvSXv/xFDodDmzZtUmZmpvbs2aPS0lLV1NQoOTlZkvT4449r6tSpWrZsmeLj4wM9bQAAYJCA34F55ZVXlJycrJ///OeKjY3VZZddpqefftp3fv/+/XK73UpNTfUds9vtSklJUVVVlSSpqqpK0dHRvniRpNTUVIWGhqq6uvqMr9vW1iav1+v3AAAA56aAB8wnn3yiVatWacSIEXrzzTc1e/Zs/f73v9e6deskSW63W5LkcDj8rnM4HL5zbrdbsbGxfufDwsIUExPjG3OqoqIi2e123yMhISHQbw0AAPQSAQ+Yjo4OjR8/Xo888oguu+wyzZo1SzNnztTq1asD/VJ+CgoK5PF4fI/GxsZufT0AABA8AQ+YuLg4JSUl+R0bPXq0Dhw4IElyOp2SpKamJr8xTU1NvnNOp1PNzc1+50+cOKFDhw75xpwqIiJCNpvN7wEAAM5NAQ+YSZMmae/evX7HPvroIw0fPlzS1x/odTqdKi8v9533er2qrq6Wy+WSJLlcLrW0tKi2ttY3ZuvWrero6FBKSkqgpwwAAAwT8J9CuvvuuzVx4kQ98sgjuvXWW7Vz50499dRTeuqppyRJISEhysvL00MPPaQRI0YoMTFRCxYsUHx8vKZNmybp6zs2N9xwg++fno4fP67c3FxlZmbyE0gAACDwAXP55Zdr48aNKigo0IMPPqjExEQ9+uijysrK8o2ZP3++WltbNWvWLLW0tOiqq65SaWmpIiMjfWPWr1+v3NxcTZ48WaGhocrIyNDKlSsDPV0AAGCgEMuyrGBPojt4vV7Z7XZ5PJ6Afx7mgvzXOj336eL0gL4WAADBEozvd139/s3/hQQAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwTrcHzOLFixUSEqK8vDzfsaNHjyonJ0dDhgzReeedp4yMDDU1Nfldd+DAAaWnp2vAgAGKjY3VvHnzdOLEie6eLgAAMEC3BkxNTY3WrFmjsWPH+h2/++679eqrr+rll19WRUWFDh48qFtuucV3vr29Xenp6Tp27Ji2b9+udevWqaSkRIWFhd05XQAAYIhuC5gjR44oKytLTz/9tAYPHuw77vF49Oyzz2r58uW6/vrrNWHCBK1du1bbt2/Xjh07JElbtmzRBx98oOeff17jxo3TjTfeqEWLFqm4uFjHjh3rrikDAABDdFvA5OTkKD09XampqX7Ha2trdfz4cb/jo0aN0rBhw1RVVSVJqqqq0pgxY+RwOHxj0tLS5PV61dDQcMbXa2trk9fr9XsAAIBzU1h3POmGDRu0a9cu1dTUnHbO7XYrPDxc0dHRfscdDofcbrdvzDfj5eT5k+fOpKioSAsXLgzA7AEAQG8X8DswjY2NmjNnjtavX6/IyMhAP32nCgoK5PF4fI/GxsYee20AANCzAh4wtbW1am5u1vjx4xUWFqawsDBVVFRo5cqVCgsLk8Ph0LFjx9TS0uJ3XVNTk5xOpyTJ6XSe9lNJJ78+OeZUERERstlsfg8AAHBuCnjATJ48WfX19aqrq/M9kpOTlZWV5ftz//79VV5e7rtm7969OnDggFwulyTJ5XKpvr5ezc3NvjFlZWWy2WxKSkoK9JQBAIBhAv4ZmEGDBumSSy7xOzZw4EANGTLEd3zGjBmaO3euYmJiZLPZdNddd8nlcunKK6+UJE2ZMkVJSUmaPn26li5dKrfbrfvuu085OTmKiIgI9JQBAIBhuuVDvN9lxYoVCg0NVUZGhtra2pSWlqYnn3zSd75fv37avHmzZs+eLZfLpYEDByo7O1sPPvhgMKYLAAB6mR4JmG3btvl9HRkZqeLiYhUXF3d6zfDhw/X6669388wAAICJ+L+QAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABgnLNgTAPq6C/Jf6/Tcp4vTe3AmAGAO7sAAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwTsADpqioSJdffrkGDRqk2NhYTZs2TXv37vUbc/ToUeXk5GjIkCE677zzlJGRoaamJr8xBw4cUHp6ugYMGKDY2FjNmzdPJ06cCPR0AQCAgQIeMBUVFcrJydGOHTtUVlam48ePa8qUKWptbfWNufvuu/Xqq6/q5ZdfVkVFhQ4ePKhbbrnFd769vV3p6ek6duyYtm/frnXr1qmkpESFhYWBni4AADBQWKCfsLS01O/rkpISxcbGqra2VldffbU8Ho+effZZvfDCC7r++uslSWvXrtXo0aO1Y8cOXXnlldqyZYs++OADvfXWW3I4HBo3bpwWLVqkP/zhD3rggQcUHh4e6GkDAACDdPtnYDwejyQpJiZGklRbW6vjx48rNTXVN2bUqFEaNmyYqqqqJElVVVUaM2aMHA6Hb0xaWpq8Xq8aGhrO+DptbW3yer1+DwAAcG7q1oDp6OhQXl6eJk2apEsuuUSS5Ha7FR4erujoaL+xDodDbrfbN+ab8XLy/MlzZ1JUVCS73e57JCQkBPjdAACA3qJbAyYnJ0fvv/++NmzY0J0vI0kqKCiQx+PxPRobG7v9NQEAQHAE/DMwJ+Xm5mrz5s2qrKzU0KFDfcedTqeOHTumlpYWv7swTU1NcjqdvjE7d+70e76TP6V0csypIiIiFBEREeB3AQAAeqOA34GxLEu5ubnauHGjtm7dqsTERL/zEyZMUP/+/VVeXu47tnfvXh04cEAul0uS5HK5VF9fr+bmZt+YsrIy2Ww2JSUlBXrKAADAMAG/A5OTk6MXXnhBf//73zVo0CDfZ1bsdruioqJkt9s1Y8YMzZ07VzExMbLZbLrrrrvkcrl05ZVXSpKmTJmipKQkTZ8+XUuXLpXb7dZ9992nnJwc7rIAAIDAB8yqVaskSddee63f8bVr1+o3v/mNJGnFihUKDQ1VRkaG2tralJaWpieffNI3tl+/ftq8ebNmz54tl8ulgQMHKjs7Ww8++GCgpwsAAAwU8ICxLOs7x0RGRqq4uFjFxcWdjhk+fLhef/31QE4NAACcI/i/kAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGCcsGBPAAAQOBfkv9bpuU8Xp/fgTIDuxR0YAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHF6dcAUFxfrggsuUGRkpFJSUrRz585gTwkAAPQCvTZgXnzxRc2dO1f333+/du3apUsvvVRpaWlqbm4O9tQAAECQ9dqAWb58uWbOnKnbb79dSUlJWr16tQYMGKDnnnsu2FMDAABB1iv/L6Rjx46ptrZWBQUFvmOhoaFKTU1VVVXVGa9pa2tTW1ub72uPxyNJ8nq9AZ9fR9tXnZ7rjtfDuY39hEBiPyGQgrGfTj6vZVnfOq5XBsx///tftbe3y+Fw+B13OBz68MMPz3hNUVGRFi5ceNrxhISEbpljZ+yP9ujL4RzHfkIgsZ8QSN29nw4fPiy73d7p+V4ZMN9HQUGB5s6d6/u6o6NDhw4d0pAhQxQSEhKw1/F6vUpISFBjY6NsNlvAnvdcxXp1HWvVdaxV17FWXcdadV13rpVlWTp8+LDi4+O/dVyvDJjzzz9f/fr1U1NTk9/xpqYmOZ3OM14TERGhiIgIv2PR0dHdNUXZbDY2+FlgvbqOteo61qrrWKuuY626rrvW6tvuvJzUKz/EGx4ergkTJqi8vNx3rKOjQ+Xl5XK5XEGcGQAA6A165R0YSZo7d66ys7OVnJysK664Qo8++qhaW1t1++23B3tqAAAgyHptwNx22236z3/+o8LCQrndbo0bN06lpaWnfbC3p0VEROj+++8/7Z+rcGasV9exVl3HWnUda9V1rFXX9Ya1CrG+6+eUAAAAeple+RkYAACAb0PAAAAA4xAwAADAOAQMAAAwDgFzisrKSt10002Kj49XSEiINm3a9J3XbNu2TePHj1dERIQuuugilZSUdPs8e4OzXatt27YpJCTktIfb7e6ZCQdRUVGRLr/8cg0aNEixsbGaNm2a9u7d+53Xvfzyyxo1apQiIyM1ZswYvf766z0w2+D6PmtVUlJy2r6KjIzsoRkHz6pVqzR27FjfLxNzuVx64403vvWavrinpLNfq766p85k8eLFCgkJUV5e3reO6+m9RcCcorW1VZdeeqmKi4u7NH7//v1KT0/Xddddp7q6OuXl5em3v/2t3nzzzW6eafCd7VqdtHfvXn3xxRe+R2xsbDfNsPeoqKhQTk6OduzYobKyMh0/flxTpkxRa2trp9ds375dv/jFLzRjxgzt3r1b06ZN07Rp0/T+++/34Mx73vdZK+nr3wj6zX312Wef9dCMg2fo0KFavHixamtr9e677+r666/XzTffrIaGhjOO76t7Sjr7tZL65p46VU1NjdasWaOxY8d+67ig7C0LnZJkbdy48VvHzJ8/37r44ov9jt12221WWlpaN86s9+nKWv3jH/+wJFlffvllj8ypN2tubrYkWRUVFZ2OufXWW6309HS/YykpKdadd97Z3dPrVbqyVmvXrrXsdnvPTaoXGzx4sPXMM8+c8Rx7yt+3rRV7yrIOHz5sjRgxwiorK7OuueYaa86cOZ2ODcbe4g7MD1RVVaXU1FS/Y2lpaaqqqgrSjHq/cePGKS4uTj/96U/1zjvvBHs6QeHxeCRJMTExnY5hb32tK2slSUeOHNHw4cOVkJDwnX+zPhe1t7drw4YNam1t7fS/XGFPfa0rayWxp3JycpSenn7anjmTYOytXvubeE3hdrtP++3ADodDXq9X//vf/xQVFRWkmfU+cXFxWr16tZKTk9XW1qZnnnlG1157raqrqzV+/PhgT6/HdHR0KC8vT5MmTdIll1zS6bjO9lZf+MzQSV1dq5EjR+q5557T2LFj5fF4tGzZMk2cOFENDQ0aOnRoD86459XX18vlcuno0aM677zztHHjRiUlJZ1xbF/fU2ezVn15T0nShg0btGvXLtXU1HRpfDD2FgGDHjNy5EiNHDnS9/XEiRP18ccfa8WKFfrrX/8axJn1rJycHL3//vt6++23gz2VXq+ra+Vyufz+Jj1x4kSNHj1aa9as0aJFi7p7mkE1cuRI1dXVyePx6G9/+5uys7NVUVHR6Tfmvuxs1qov76nGxkbNmTNHZWVlvfqDywTMD+R0OtXU1OR3rKmpSTabjbsvXXDFFVf0qW/kubm52rx5syorK7/zb3Gd7S2n09mdU+w1zmatTtW/f39ddtll+te//tVNs+s9wsPDddFFF0mSJkyYoJqaGj322GNas2bNaWP7+p46m7U6VV/aU7W1tWpubva7M97e3q7Kyko98cQTamtrU79+/fyuCcbe4jMwP5DL5VJ5ebnfsbKysm/9d1X8f3V1dYqLiwv2NLqdZVnKzc3Vxo0btXXrViUmJn7nNX11b32ftTpVe3u76uvr+8TeOlVHR4fa2trOeK6v7qnOfNtanaov7anJkyervr5edXV1vkdycrKysrJUV1d3WrxIQdpb3fbxYEMdPnzY2r17t7V7925LkrV8+XJr9+7d1meffWZZlmXl5+db06dP943/5JNPrAEDBljz5s2z9uzZYxUXF1v9+vWzSktLg/UWeszZrtWKFSusTZs2Wfv27bPq6+utOXPmWKGhodZbb70VrLfQY2bPnm3Z7XZr27Zt1hdffOF7fPXVV74x06dPt/Lz831fv/POO1ZYWJi1bNkya8+ePdb9999v9e/f36qvrw/GW+gx32etFi5caL355pvWxx9/bNXW1lqZmZlWZGSk1dDQEIy30GPy8/OtiooKa//+/dZ7771n5efnWyEhIdaWLVssy2JPfdPZrlVf3VOdOfWnkHrD3iJgTnHyR31PfWRnZ1uWZVnZ2dnWNddcc9o148aNs8LDw60f//jH1tq1a3t83sFwtmu1ZMkS68ILL7QiIyOtmJgY69prr7W2bt0anMn3sDOtkyS/vXLNNdf41u6kl156yfrJT35ihYeHWxdffLH12muv9ezEg+D7rFVeXp41bNgwKzw83HI4HNbUqVOtXbt29fzke9gdd9xhDR8+3AoPD7d+9KMfWZMnT/Z9Q7Ys9tQ3ne1a9dU91ZlTA6Y37K0Qy7Ks7ru/AwAAEHh8BgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGCc/wddyZ1+T1DuEgAAAABJRU5ErkJggg==", "text/plain": [ "<Figure size 640x480 with 1 Axes>" ] @@ -239,41 +280,40 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 90, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "(array([1157., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 570., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 0., 210., 0.,\n", + "(array([1714., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 86., 0., 0., 0., 0., 0., 0.,\n", - " 0., 0., 0., 0., 0., 0., 20., 0., 0.,\n", + " 0., 0., 0., 0., 0., 305., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", - " 5.]),\n", - " array([1. , 1.078125, 1.15625 , 1.234375, 1.3125 , 1.390625,\n", - " 1.46875 , 1.546875, 1.625 , 1.703125, 1.78125 , 1.859375,\n", - " 1.9375 , 2.015625, 2.09375 , 2.171875, 2.25 , 2.328125,\n", - " 2.40625 , 2.484375, 2.5625 , 2.640625, 2.71875 , 2.796875,\n", - " 2.875 , 2.953125, 3.03125 , 3.109375, 3.1875 , 3.265625,\n", - " 3.34375 , 3.421875, 3.5 , 3.578125, 3.65625 , 3.734375,\n", - " 3.8125 , 3.890625, 3.96875 , 4.046875, 4.125 , 4.203125,\n", - " 4.28125 , 4.359375, 4.4375 , 4.515625, 4.59375 , 4.671875,\n", - " 4.75 , 4.828125, 4.90625 , 4.984375, 5.0625 , 5.140625,\n", - " 5.21875 , 5.296875, 5.375 , 5.453125, 5.53125 , 5.609375,\n", - " 5.6875 , 5.765625, 5.84375 , 5.921875, 6. ]),\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 29.]),\n", + " array([1. , 1.03125, 1.0625 , 1.09375, 1.125 , 1.15625, 1.1875 ,\n", + " 1.21875, 1.25 , 1.28125, 1.3125 , 1.34375, 1.375 , 1.40625,\n", + " 1.4375 , 1.46875, 1.5 , 1.53125, 1.5625 , 1.59375, 1.625 ,\n", + " 1.65625, 1.6875 , 1.71875, 1.75 , 1.78125, 1.8125 , 1.84375,\n", + " 1.875 , 1.90625, 1.9375 , 1.96875, 2. , 2.03125, 2.0625 ,\n", + " 2.09375, 2.125 , 2.15625, 2.1875 , 2.21875, 2.25 , 2.28125,\n", + " 2.3125 , 2.34375, 2.375 , 2.40625, 2.4375 , 2.46875, 2.5 ,\n", + " 2.53125, 2.5625 , 2.59375, 2.625 , 2.65625, 2.6875 , 2.71875,\n", + " 2.75 , 2.78125, 2.8125 , 2.84375, 2.875 , 2.90625, 2.9375 ,\n", + " 2.96875, 3. ]),\n", " <BarContainer object of 64 artists>)" ] }, - "execution_count": 5, + "execution_count": 90, "metadata": {}, "output_type": "execute_result" }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGeCAYAAACKDztsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAilklEQVR4nO3df1BVdf7H8ReI/FDhIrbcy13R2N1WJU1NjG5aW8mISu46sZVFxqojOy1YZj+USu2HibrbL1rDdFp1Jh2rndHKiiIt3ZQQcSl/ZbZZ0o8LO4vcGzQiyvn+8R3PdEs3bS9cPrfnY+bMdM/53Hve9zYNzw73XiIsy7IEAABgkMhQDwAAAHCuCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcaJCPUBHaW9v15dffqn4+HhFRESEehwAAHAWLMvS119/LbfbrcjI/3KdxTpHW7duta655horJSXFkmRt2LDBPnb8+HHrnnvusQYPHmz16NHDSklJsaZMmWJ98cUXAY/xn//8x7rpppus+Ph4y+FwWNOmTbO+/vrrgDXvv/++NXr0aCsmJsbq27evtWTJknOas66uzpLExsbGxsbGZuBWV1f3X3/On/MVmJaWFg0dOlTTpk3TtddeG3Dsm2++0e7duzVv3jwNHTpUR48e1e23367f/va32rVrl70uLy9PX331lSoqKtTW1qapU6eqoKBA69atkyT5/X6NHTtWWVlZWr58ufbs2aNp06YpMTFRBQUFZzVnfHy8JKmurk4JCQnn+jQBAEAI+P1+paam2j/HzyTCsn78H3OMiIjQhg0bNGnSpDOuqa6u1iWXXKLPPvtM/fr104EDB5Senq7q6mplZGRIksrLyzVhwgR9/vnncrvdKisr03333Sev16vo6GhJ0ty5c7Vx40Z9+OGHZzWb3++Xw+GQz+cjYAAAMMTZ/vzu8Dfx+nw+RUREKDExUZJUWVmpxMREO14kKSsrS5GRkaqqqrLXXHHFFXa8SFJ2drYOHjyoo0ePnvY8ra2t8vv9ARsAAAhPHRowx44d05w5c3TjjTfaFeX1epWcnBywLioqSklJSfJ6vfYap9MZsObU7VNrvqukpEQOh8PeUlNTg/10AABAF9FhAdPW1qbrr79elmWprKyso05jKy4uls/ns7e6uroOPycAAAiNDvkY9al4+eyzz7Rly5aA32G5XC41NDQErD9x4oQaGxvlcrnsNfX19QFrTt0+tea7YmJiFBMTE8ynAQAAuqigX4E5FS+HDh3SW2+9pT59+gQc93g8ampqUk1Njb1vy5Ytam9vV2Zmpr1m27Ztamtrs9dUVFRowIAB6t27d7BHBgAAhjnngGlublZtba1qa2slSYcPH1Ztba2OHDmitrY2/f73v9euXbu0du1anTx5Ul6vV16vV8ePH5ckDRo0SOPGjdOMGTO0c+dObd++XUVFRZo8ebLcbrck6aabblJ0dLSmT5+uffv26fnnn9eTTz6p2bNnB++ZAwAAY53zx6jfeecdXXXVVd/bn5+frwceeEBpaWmnvd/bb7+tK6+8UpLU2NiooqIivfLKK4qMjFRubq5KS0vVq1cve/0HH3ygwsJCVVdX67zzztPMmTM1Z86cs56Tj1EDAGCes/35/T99D0xXRsAAAGCeLvM9MAAAAMFGwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA43TInxIId+fPffWMxz5dnNOJkwAA8NPEFRgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxjnngNm2bZsmTpwot9utiIgIbdy4MeC4ZVmaP3++UlJSFBcXp6ysLB06dChgTWNjo/Ly8pSQkKDExERNnz5dzc3NAWs++OADXX755YqNjVVqaqqWLl167s8OAACEpXMOmJaWFg0dOlTLli077fGlS5eqtLRUy5cvV1VVlXr27Kns7GwdO3bMXpOXl6d9+/apoqJCmzZt0rZt21RQUGAf9/v9Gjt2rPr376+amhr9+c9/1gMPPKAVK1b8iKcIAADCTYRlWdaPvnNEhDZs2KBJkyZJ+v+rL263W3feeafuuusuSZLP55PT6dTq1as1efJkHThwQOnp6aqurlZGRoYkqby8XBMmTNDnn38ut9utsrIy3XffffJ6vYqOjpYkzZ07Vxs3btSHH354VrP5/X45HA75fD4lJCT82Kd4WufPffWMxz5dnBPUcwEA8FNytj+/g/oemMOHD8vr9SorK8ve53A4lJmZqcrKSklSZWWlEhMT7XiRpKysLEVGRqqqqspec8UVV9jxIknZ2dk6ePCgjh49etpzt7a2yu/3B2wAACA8BTVgvF6vJMnpdAbsdzqd9jGv16vk5OSA41FRUUpKSgpYc7rH+PY5vqukpEQOh8PeUlNT//cnBAAAuqSw+RRScXGxfD6fvdXV1YV6JAAA0EGCGjAul0uSVF9fH7C/vr7ePuZyudTQ0BBw/MSJE2psbAxYc7rH+PY5vismJkYJCQkBGwAACE9BDZi0tDS5XC5t3rzZ3uf3+1VVVSWPxyNJ8ng8ampqUk1Njb1my5Ytam9vV2Zmpr1m27Ztamtrs9dUVFRowIAB6t27dzBHBgAABjrngGlublZtba1qa2sl/f8bd2tra3XkyBFFRERo1qxZWrhwoV5++WXt2bNHt9xyi9xut/1JpUGDBmncuHGaMWOGdu7cqe3bt6uoqEiTJ0+W2+2WJN10002Kjo7W9OnTtW/fPj3//PN68sknNXv27KA9cQAAYK6oc73Drl27dNVVV9m3T0VFfn6+Vq9erXvuuUctLS0qKChQU1OTRo8erfLycsXGxtr3Wbt2rYqKijRmzBhFRkYqNzdXpaWl9nGHw6E333xThYWFGjFihM477zzNnz8/4LtiAADAT9f/9D0wXRnfAwMAgHlC8j0wAAAAnYGAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxgl6wJw8eVLz5s1TWlqa4uLi9Mtf/lIPP/ywLMuy11iWpfnz5yslJUVxcXHKysrSoUOHAh6nsbFReXl5SkhIUGJioqZPn67m5uZgjwsAAAwU9IBZsmSJysrK9Ne//lUHDhzQkiVLtHTpUj311FP2mqVLl6q0tFTLly9XVVWVevbsqezsbB07dsxek5eXp3379qmiokKbNm3Stm3bVFBQEOxxAQCAgSKsb18aCYJrrrlGTqdTzz77rL0vNzdXcXFxeu6552RZltxut+68807dddddkiSfzyen06nVq1dr8uTJOnDggNLT01VdXa2MjAxJUnl5uSZMmKDPP/9cbrf7B+fw+/1yOBzy+XxKSEgI5lPU+XNfPeOxTxfnBPVcAAD8lJztz++gX4G57LLLtHnzZn300UeSpPfff1/vvvuuxo8fL0k6fPiwvF6vsrKy7Ps4HA5lZmaqsrJSklRZWanExEQ7XiQpKytLkZGRqqqqOu15W1tb5ff7AzYAABCeooL9gHPnzpXf79fAgQPVrVs3nTx5Uo888ojy8vIkSV6vV5LkdDoD7ud0Ou1jXq9XycnJgYNGRSkpKcle810lJSV68MEHg/10AABAFxT0KzAvvPCC1q5dq3Xr1mn37t1as2aN/vKXv2jNmjXBPlWA4uJi+Xw+e6urq+vQ8wEAgNAJ+hWYu+++W3PnztXkyZMlSUOGDNFnn32mkpIS5efny+VySZLq6+uVkpJi36++vl7Dhg2TJLlcLjU0NAQ87okTJ9TY2Gjf/7tiYmIUExMT7KcDAAC6oKBfgfnmm28UGRn4sN26dVN7e7skKS0tTS6XS5s3b7aP+/1+VVVVyePxSJI8Ho+amppUU1Njr9myZYva29uVmZkZ7JEBAIBhgn4FZuLEiXrkkUfUr18/XXjhhfrnP/+pxx57TNOmTZMkRUREaNasWVq4cKEuuOACpaWlad68eXK73Zo0aZIkadCgQRo3bpxmzJih5cuXq62tTUVFRZo8efJZfQIJAACEt6AHzFNPPaV58+bpT3/6kxoaGuR2u/XHP/5R8+fPt9fcc889amlpUUFBgZqamjR69GiVl5crNjbWXrN27VoVFRVpzJgxioyMVG5urkpLS4M9LgAAMFDQvwemq+B7YAAAME/IvgcGAACgoxEwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOB0SMF988YVuvvlm9enTR3FxcRoyZIh27dplH7csS/Pnz1dKSori4uKUlZWlQ4cOBTxGY2Oj8vLylJCQoMTERE2fPl3Nzc0dMS4AADBM0APm6NGjGjVqlLp3767XX39d+/fv16OPPqrevXvba5YuXarS0lItX75cVVVV6tmzp7Kzs3Xs2DF7TV5envbt26eKigpt2rRJ27ZtU0FBQbDHBQAABoqwLMsK5gPOnTtX27dv1z/+8Y/THrcsS263W3feeafuuusuSZLP55PT6dTq1as1efJkHThwQOnp6aqurlZGRoYkqby8XBMmTNDnn38ut9v9g3P4/X45HA75fD4lJCQE7wlKOn/uq2c89uninKCeCwCAn5Kz/fkd9CswL7/8sjIyMnTdddcpOTlZw4cP18qVK+3jhw8fltfrVVZWlr3P4XAoMzNTlZWVkqTKykolJiba8SJJWVlZioyMVFVV1WnP29raKr/fH7ABAIDwFPSA+eSTT1RWVqYLLrhAb7zxhm699VbddtttWrNmjSTJ6/VKkpxOZ8D9nE6nfczr9So5OTngeFRUlJKSkuw131VSUiKHw2FvqampwX5qAACgiwh6wLS3t+viiy/WokWLNHz4cBUUFGjGjBlavnx5sE8VoLi4WD6fz97q6uo69HwAACB0gh4wKSkpSk9PD9g3aNAgHTlyRJLkcrkkSfX19QFr6uvr7WMul0sNDQ0Bx0+cOKHGxkZ7zXfFxMQoISEhYAMAAOEp6AEzatQoHTx4MGDfRx99pP79+0uS0tLS5HK5tHnzZvu43+9XVVWVPB6PJMnj8aipqUk1NTX2mi1btqi9vV2ZmZnBHhkAABgmKtgPeMcdd+iyyy7TokWLdP3112vnzp1asWKFVqxYIUmKiIjQrFmztHDhQl1wwQVKS0vTvHnz5Ha7NWnSJEn/f8Vm3Lhx9q+e2traVFRUpMmTJ5/VJ5AAAEB4C3rAjBw5Uhs2bFBxcbEeeughpaWl6YknnlBeXp695p577lFLS4sKCgrU1NSk0aNHq7y8XLGxsfaatWvXqqioSGPGjFFkZKRyc3NVWloa7HEBAICBgv49MF0F3wMDAIB5QvY9MAAAAB2NgAEAAMYJ+ntggB+DX8sBAM4FV2AAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgnA4PmMWLFysiIkKzZs2y9x07dkyFhYXq06ePevXqpdzcXNXX1wfc78iRI8rJyVGPHj2UnJysu+++WydOnOjocQEAgAE6NGCqq6v1zDPP6KKLLgrYf8cdd+iVV17Riy++qK1bt+rLL7/Utddeax8/efKkcnJydPz4ce3YsUNr1qzR6tWrNX/+/I4cFwAAGKLDAqa5uVl5eXlauXKlevfube/3+Xx69tln9dhjj+nqq6/WiBEjtGrVKu3YsUPvvfeeJOnNN9/U/v379dxzz2nYsGEaP368Hn74YS1btkzHjx/vqJEBAIAhOixgCgsLlZOTo6ysrID9NTU1amtrC9g/cOBA9evXT5WVlZKkyspKDRkyRE6n016TnZ0tv9+vffv2ddTIAADAEFEd8aDr16/X7t27VV1d/b1jXq9X0dHRSkxMDNjvdDrl9XrtNd+Ol1PHTx07ndbWVrW2ttq3/X7///IUAABAFxb0KzB1dXW6/fbbtXbtWsXGxgb74c+opKREDofD3lJTUzvt3AAAoHMFPWBqamrU0NCgiy++WFFRUYqKitLWrVtVWlqqqKgoOZ1OHT9+XE1NTQH3q6+vl8vlkiS5XK7vfSrp1O1Ta76ruLhYPp/P3urq6oL91AAAQBcR9IAZM2aM9uzZo9raWnvLyMhQXl6e/c/du3fX5s2b7fscPHhQR44ckcfjkSR5PB7t2bNHDQ0N9pqKigolJCQoPT39tOeNiYlRQkJCwAYAAMJT0N8DEx8fr8GDBwfs69mzp/r06WPvnz59umbPnq2kpCQlJCRo5syZ8ng8uvTSSyVJY8eOVXp6uqZMmaKlS5fK6/Xq/vvvV2FhoWJiYoI9MgAAMEyHvIn3hzz++OOKjIxUbm6uWltblZ2draeffto+3q1bN23atEm33nqrPB6Pevbsqfz8fD300EOhGBcAAHQxnRIw77zzTsDt2NhYLVu2TMuWLTvjffr376/XXnutgycDAAAm4m8hAQAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOEEPmJKSEo0cOVLx8fFKTk7WpEmTdPDgwYA1x44dU2Fhofr06aNevXopNzdX9fX1AWuOHDminJwc9ejRQ8nJybr77rt14sSJYI8LAAAMFBXsB9y6dasKCws1cuRInThxQvfee6/Gjh2r/fv3q2fPnpKkO+64Q6+++qpefPFFORwOFRUV6dprr9X27dslSSdPnlROTo5cLpd27Nihr776Srfccou6d++uRYsWBXtk4Cfn/LmvnvHYp4tzOnESAPhxgh4w5eXlAbdXr16t5ORk1dTU6IorrpDP59Ozzz6rdevW6eqrr5YkrVq1SoMGDdJ7772nSy+9VG+++ab279+vt956S06nU8OGDdPDDz+sOXPm6IEHHlB0dHSwxwYAAAbp8PfA+Hw+SVJSUpIkqaamRm1tbcrKyrLXDBw4UP369VNlZaUkqbKyUkOGDJHT6bTXZGdny+/3a9++fac9T2trq/x+f8AGAADCU4cGTHt7u2bNmqVRo0Zp8ODBkiSv16vo6GglJiYGrHU6nfJ6vfaab8fLqeOnjp1OSUmJHA6HvaWmpgb52QAAgK6iQwOmsLBQe/fu1fr16zvyNJKk4uJi+Xw+e6urq+vwcwIAgNAI+ntgTikqKtKmTZu0bds29e3b197vcrl0/PhxNTU1BVyFqa+vl8vlstfs3Lkz4PFOfUrp1JrviomJUUxMTJCfBQAA6IqCfgXGsiwVFRVpw4YN2rJli9LS0gKOjxgxQt27d9fmzZvtfQcPHtSRI0fk8XgkSR6PR3v27FFDQ4O9pqKiQgkJCUpPTw/2yAAAwDBBvwJTWFiodevW6aWXXlJ8fLz9nhWHw6G4uDg5HA5Nnz5ds2fPVlJSkhISEjRz5kx5PB5deumlkqSxY8cqPT1dU6ZM0dKlS+X1enX//fersLCQqywAACD4AVNWViZJuvLKKwP2r1q1Sn/4wx8kSY8//rgiIyOVm5ur1tZWZWdn6+mnn7bXduvWTZs2bdKtt94qj8ejnj17Kj8/Xw899FCwxwUAAAYKesBYlvWDa2JjY7Vs2TItW7bsjGv69++v1157LZijAQCAMMHfQgIAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGCcq1AMAQDg7f+6rZzz26eKcTpwECC9cgQEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgnKhQDwAAQLCcP/fVMx77dHFOJ06CjsYVGAAAYJwuHTDLli3T+eefr9jYWGVmZmrnzp2hHgkAAHQBXTZgnn/+ec2ePVsLFizQ7t27NXToUGVnZ6uhoSHUowEAgBDrsgHz2GOPacaMGZo6darS09O1fPly9ejRQ3/7299CPRoAAAixLvkm3uPHj6umpkbFxcX2vsjISGVlZamysvK092ltbVVra6t92+fzSZL8fn/Q52tv/eaMxzrifD8FvKadi9e78/Bady5e7+AavOCNMx7b+2B2h5zz1L8ny7L++0KrC/riiy8sSdaOHTsC9t99993WJZdcctr7LFiwwJLExsbGxsbGFgZbXV3df22FLnkF5scoLi7W7Nmz7dvt7e1qbGxUnz59FBEREbTz+P1+paamqq6uTgkJCUF7XHwfr3Xn4HXuHLzOnYPXuXN05OtsWZa+/vprud3u/7quSwbMeeedp27duqm+vj5gf319vVwu12nvExMTo5iYmIB9iYmJHTWiEhIS+I+jk/Badw5e587B69w5eJ07R0e9zg6H4wfXdMk38UZHR2vEiBHavHmzva+9vV2bN2+Wx+MJ4WQAAKAr6JJXYCRp9uzZys/PV0ZGhi655BI98cQTamlp0dSpU0M9GgAACLEuGzA33HCD/v3vf2v+/Pnyer0aNmyYysvL5XQ6QzpXTEyMFixY8L1fVyH4eK07B69z5+B17hy8zp2jK7zOEZb1Q59TAgAA6Fq65HtgAAAA/hsCBgAAGIeAAQAAxiFgAACAcQiYc7Bt2zZNnDhRbrdbERER2rhxY6hHCjslJSUaOXKk4uPjlZycrEmTJungwYOhHivslJWV6aKLLrK/hMrj8ej1118P9Vhhb/HixYqIiNCsWbNCPUrYeeCBBxQRERGwDRw4MNRjhaUvvvhCN998s/r06aO4uDgNGTJEu3bt6vQ5CJhz0NLSoqFDh2rZsmWhHiVsbd26VYWFhXrvvfdUUVGhtrY2jR07Vi0tLaEeLaz07dtXixcvVk1NjXbt2qWrr75av/vd77Rv375Qjxa2qqur9cwzz+iiiy4K9Shh68ILL9RXX31lb++++26oRwo7R48e1ahRo9S9e3e9/vrr2r9/vx599FH17t2702fpst8D0xWNHz9e48ePD/UYYa28vDzg9urVq5WcnKyamhpdccUVIZoq/EycODHg9iOPPKKysjK99957uvDCC0M0Vfhqbm5WXl6eVq5cqYULF4Z6nLAVFRV1xj83g+BYsmSJUlNTtWrVKntfWlpaSGbhCgy6NJ/PJ0lKSkoK8STh6+TJk1q/fr1aWlr4Ux0dpLCwUDk5OcrKygr1KGHt0KFDcrvd+sUvfqG8vDwdOXIk1COFnZdfflkZGRm67rrrlJycrOHDh2vlypUhmYUrMOiy2tvbNWvWLI0aNUqDBw8O9ThhZ8+ePfJ4PDp27Jh69eqlDRs2KD09PdRjhZ3169dr9+7dqq6uDvUoYS0zM1OrV6/WgAED9NVXX+nBBx/U5Zdfrr179yo+Pj7U44WNTz75RGVlZZo9e7buvfdeVVdX67bbblN0dLTy8/M7dRYCBl1WYWGh9u7dy++xO8iAAQNUW1srn8+nv//978rPz9fWrVuJmCCqq6vT7bffroqKCsXGxoZ6nLD27V/vX3TRRcrMzFT//v31wgsvaPr06SGcLLy0t7crIyNDixYtkiQNHz5ce/fu1fLlyzs9YPgVErqkoqIibdq0SW+//bb69u0b6nHCUnR0tH71q19pxIgRKikp0dChQ/Xkk0+GeqywUlNTo4aGBl188cWKiopSVFSUtm7dqtLSUkVFRenkyZOhHjFsJSYm6te//rU+/vjjUI8SVlJSUr73PzmDBg0Kya/ruAKDLsWyLM2cOVMbNmzQO++8E7I3h/0Utbe3q7W1NdRjhJUxY8Zoz549AfumTp2qgQMHas6cOerWrVuIJgt/zc3N+te//qUpU6aEepSwMmrUqO99tcVHH32k/v37d/osBMw5aG5uDqj5w4cPq7a2VklJSerXr18IJwsfhYWFWrdunV566SXFx8fL6/VKkhwOh+Li4kI8XfgoLi7W+PHj1a9fP3399ddat26d3nnnHb3xxhuhHi2sxMfHf+/9Wz179lSfPn14X1eQ3XXXXZo4caL69++vL7/8UgsWLFC3bt104403hnq0sHLHHXfosssu06JFi3T99ddr586dWrFihVasWNH5w1g4a2+//bYl6Xtbfn5+qEcLG6d7fSVZq1atCvVoYWXatGlW//79rejoaOtnP/uZNWbMGOvNN98M9Vg/Cb/5zW+s22+/PdRjhJ0bbrjBSklJsaKjo62f//zn1g033GB9/PHHoR4rLL3yyivW4MGDrZiYGGvgwIHWihUrQjJHhGVZVudnEwAAwI/Hm3gBAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADG+T+/FGJF6kiv7QAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvkklEQVR4nO3df1xUdd7//yeIgJkziAbjXOGP2lIx09KkKfvhSqKSm5ttUeS6LemuF9SaZeJtV8tqF7OuTLtM1q4Kr82y2tI2K5I0YdcIFeXyR0rWmj9yB9o1ZoRWRDnfP/pwvo1CCQ0yb3rcb7dzy3m/X+ec93vfnJ2nhzNjmGVZlgAAAAwS3tYDAAAAaC4CDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOBFtPYDWUl9fr0OHDqlLly4KCwtr6+EAAIDTYFmWjhw5IrfbrfDwpu+ztNsAc+jQISUkJLT1MAAAQAscOHBA5557bpP97TbAdOnSRdLX/wM4HI42Hg0AADgdfr9fCQkJ9vt4U9ptgGn4tZHD4SDAAABgmO96/IOHeAEAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACME9HWAzBV7+y3muz7bF7qGRwJAAA/PNyBAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHGaHWCKioo0btw4ud1uhYWFadWqVafU7Nq1Sz/5yU/kdDrVuXNnXXbZZdq/f7/df/ToUWVmZqpbt246++yzNWHCBFVUVAQcY//+/UpNTdVZZ52luLg4zZgxQ8ePH2/+DAEAQLvT7ABTU1OjQYMGafHixY32f/rppxo+fLj69eun9evXa9u2bZo9e7aio6PtmnvuuUdvvvmmXn31VRUWFurQoUO68cYb7f4TJ04oNTVVx44d0wcffKBly5YpLy9Pc+bMacEUAQBAexNmWZbV4p3DwrRy5UqNHz/ebktLS1PHjh31pz/9qdF9fD6fzjnnHL344ou66aabJEm7d+9W//79VVxcrMsvv1zvvPOOrr/+eh06dEjx8fGSpNzcXM2cOVNffPGFIiMjv3Nsfr9fTqdTPp9PDoejpVNsUu/st5rs+2xeatDPBwDAD8Hpvn8H9RmY+vp6vfXWW7rwwguVkpKiuLg4JSUlBfyaqbS0VHV1dUpOTrbb+vXrp549e6q4uFiSVFxcrIEDB9rhRZJSUlLk9/u1c+fORs9dW1srv98fsAEAgPYpqAGmsrJS1dXVmjdvnkaPHq01a9bopz/9qW688UYVFhZKkrxeryIjIxUTExOwb3x8vLxer13zzfDS0N/Q15icnBw5nU57S0hICObUAABACAn6HRhJuuGGG3TPPfdo8ODBys7O1vXXX6/c3NxgnuoUs2bNks/ns7cDBw606vkAAEDbCWqA6d69uyIiIpSYmBjQ3r9/f/tTSC6XS8eOHVNVVVVATUVFhVwul11z8qeSGl431JwsKipKDocjYAMAAO1TUANMZGSkLrvsMpWXlwe0f/zxx+rVq5ckaciQIerYsaPWrl1r95eXl2v//v3yeDySJI/Ho+3bt6uystKuKSgokMPhOCUcAQCAH56I5u5QXV2tTz75xH69d+9elZWVKTY2Vj179tSMGTN0yy236Oqrr9aIESOUn5+vN998U+vXr5ckOZ1OZWRkaPr06YqNjZXD4dBdd90lj8ejyy+/XJI0atQoJSYmauLEiZo/f768Xq9+97vfKTMzU1FRUcGZOQAAMFazA8zmzZs1YsQI+/X06dMlSZMmTVJeXp5++tOfKjc3Vzk5Obr77rvVt29fvfbaaxo+fLi9z4IFCxQeHq4JEyaotrZWKSkpevrpp+3+Dh06aPXq1Zo6dao8Ho86d+6sSZMm6aGHHvo+cwUAAO3E9/oemFDG98AAAGCeNvkeGAAAgDOBAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDjNDjBFRUUaN26c3G63wsLCtGrVqiZrf/3rXyssLExPPvlkQPvhw4eVnp4uh8OhmJgYZWRkqLq6OqBm27ZtuuqqqxQdHa2EhATNnz+/uUMFAADtVLMDTE1NjQYNGqTFixd/a93KlSv14Ycfyu12n9KXnp6unTt3qqCgQKtXr1ZRUZGmTJli9/v9fo0aNUq9evVSaWmpHnvsMT344INaunRpc4cLAADaoYjm7jBmzBiNGTPmW2s+//xz3XXXXXr33XeVmpoa0Ldr1y7l5+dr06ZNGjp0qCTpqaee0tixY/X444/L7XZr+fLlOnbsmJ577jlFRkZqwIABKisr0xNPPBEQdAAAwA9T0J+Bqa+v18SJEzVjxgwNGDDglP7i4mLFxMTY4UWSkpOTFR4erpKSErvm6quvVmRkpF2TkpKi8vJyffnll42et7a2Vn6/P2ADAADtU9ADzKOPPqqIiAjdfffdjfZ7vV7FxcUFtEVERCg2NlZer9euiY+PD6hpeN1Qc7KcnBw5nU57S0hI+L5TAQAAISqoAaa0tFQLFy5UXl6ewsLCgnno7zRr1iz5fD57O3DgwBk9PwAAOHOCGmD++te/qrKyUj179lRERIQiIiK0b98+3Xvvverdu7ckyeVyqbKyMmC/48eP6/Dhw3K5XHZNRUVFQE3D64aak0VFRcnhcARsAACgfQpqgJk4caK2bdumsrIye3O73ZoxY4beffddSZLH41FVVZVKS0vt/datW6f6+nolJSXZNUVFRaqrq7NrCgoK1LdvX3Xt2jWYQwYAAAZq9qeQqqur9cknn9iv9+7dq7KyMsXGxqpnz57q1q1bQH3Hjh3lcrnUt29fSVL//v01evRoTZ48Wbm5uaqrq1NWVpbS0tLsj1zfdtttmjt3rjIyMjRz5kzt2LFDCxcu1IIFC77PXAEAQDvR7ACzefNmjRgxwn49ffp0SdKkSZOUl5d3WsdYvny5srKyNHLkSIWHh2vChAlatGiR3e90OrVmzRplZmZqyJAh6t69u+bMmcNHqAEAgCQpzLIsq60H0Rr8fr+cTqd8Pl+rPA/TO/utJvs+m5faZB8AAGja6b5/828hAQAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYp9kBpqioSOPGjZPb7VZYWJhWrVpl99XV1WnmzJkaOHCgOnfuLLfbrZ///Oc6dOhQwDEOHz6s9PR0ORwOxcTEKCMjQ9XV1QE127Zt01VXXaXo6GglJCRo/vz5LZshAABod5odYGpqajRo0CAtXrz4lL6vvvpKW7Zs0ezZs7Vlyxa9/vrrKi8v109+8pOAuvT0dO3cuVMFBQVavXq1ioqKNGXKFLvf7/dr1KhR6tWrl0pLS/XYY4/pwQcf1NKlS1swRQAA0N6EWZZltXjnsDCtXLlS48ePb7Jm06ZNGjZsmPbt26eePXtq165dSkxM1KZNmzR06FBJUn5+vsaOHauDBw/K7XZryZIl+u1vfyuv16vIyEhJUnZ2tlatWqXdu3ef1tj8fr+cTqd8Pp8cDkdLp9ik3tlvNdn32bzUoJ8PAIAfgtN9/271Z2B8Pp/CwsIUExMjSSouLlZMTIwdXiQpOTlZ4eHhKikpsWuuvvpqO7xIUkpKisrLy/Xll182ep7a2lr5/f6ADQAAtE+tGmCOHj2qmTNn6tZbb7VTlNfrVVxcXEBdRESEYmNj5fV67Zr4+PiAmobXDTUny8nJkdPptLeEhIRgTwcAAISIVgswdXV1uvnmm2VZlpYsWdJap7HNmjVLPp/P3g4cONDq5wQAAG0jojUO2hBe9u3bp3Xr1gX8DsvlcqmysjKg/vjx4zp8+LBcLpddU1FREVDT8Lqh5mRRUVGKiooK5jQAAECICvodmIbwsmfPHr333nvq1q1bQL/H41FVVZVKS0vttnXr1qm+vl5JSUl2TVFRkerq6uyagoIC9e3bV127dg32kAEAgGGaHWCqq6tVVlamsrIySdLevXtVVlam/fv3q66uTjfddJM2b96s5cuX68SJE/J6vfJ6vTp27JgkqX///ho9erQmT56sjRs3asOGDcrKylJaWprcbrck6bbbblNkZKQyMjK0c+dOvfzyy1q4cKGmT58evJkDAABjNftj1OvXr9eIESNOaZ80aZIefPBB9enTp9H93n//fV177bWSvv4iu6ysLL355psKDw/XhAkTtGjRIp199tl2/bZt25SZmalNmzape/fuuuuuuzRz5szTHicfowYAwDyn+/79vb4HJpQRYAAAME/IfA8MAABAsBFgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYp9kBpqioSOPGjZPb7VZYWJhWrVoV0G9ZlubMmaMePXqoU6dOSk5O1p49ewJqDh8+rPT0dDkcDsXExCgjI0PV1dUBNdu2bdNVV12l6OhoJSQkaP78+c2fHQAAaJeaHWBqamo0aNAgLV68uNH++fPna9GiRcrNzVVJSYk6d+6slJQUHT161K5JT0/Xzp07VVBQoNWrV6uoqEhTpkyx+/1+v0aNGqVevXqptLRUjz32mB588EEtXbq0BVMEAADtTZhlWVaLdw4L08qVKzV+/HhJX999cbvduvfee3XfffdJknw+n+Lj45WXl6e0tDTt2rVLiYmJ2rRpk4YOHSpJys/P19ixY3Xw4EG53W4tWbJEv/3tb+X1ehUZGSlJys7O1qpVq7R79+7TGpvf75fT6ZTP55PD4WjpFJvUO/utJvs+m5ca9PMBAPBDcLrv30F9Bmbv3r3yer1KTk6225xOp5KSklRcXCxJKi4uVkxMjB1eJCk5OVnh4eEqKSmxa66++mo7vEhSSkqKysvL9eWXXzZ67traWvn9/oANAAC0T0ENMF6vV5IUHx8f0B4fH2/3eb1excXFBfRHREQoNjY2oKaxY3zzHCfLycmR0+m0t4SEhO8/IQAAEJLazaeQZs2aJZ/PZ28HDhxo6yEBAIBWEtQA43K5JEkVFRUB7RUVFXafy+VSZWVlQP/x48d1+PDhgJrGjvHNc5wsKipKDocjYAMAAO1TUANMnz595HK5tHbtWrvN7/erpKREHo9HkuTxeFRVVaXS0lK7Zt26daqvr1dSUpJdU1RUpLq6OrumoKBAffv2VdeuXYM5ZAAAYKBmB5jq6mqVlZWprKxM0tcP7paVlWn//v0KCwvTtGnT9Mgjj+gvf/mLtm/frp///Odyu932J5X69++v0aNHa/Lkydq4caM2bNigrKwspaWlye12S5Juu+02RUZGKiMjQzt37tTLL7+shQsXavr06UGbOAAAMFdEc3fYvHmzRowYYb9uCBWTJk1SXl6e7r//ftXU1GjKlCmqqqrS8OHDlZ+fr+joaHuf5cuXKysrSyNHjlR4eLgmTJigRYsW2f1Op1Nr1qxRZmamhgwZou7du2vOnDkB3xUDAAB+uL7X98CEMr4HBgAA87TJ98AAAACcCQQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGCXqAOXHihGbPnq0+ffqoU6dOOv/88/Xwww/Lsiy7xrIszZkzRz169FCnTp2UnJysPXv2BBzn8OHDSk9Pl8PhUExMjDIyMlRdXR3s4QIAAAMFPcA8+uijWrJkif77v/9bu3bt0qOPPqr58+frqaeesmvmz5+vRYsWKTc3VyUlJercubNSUlJ09OhRuyY9PV07d+5UQUGBVq9eraKiIk2ZMiXYwwUAAAYKs755ayQIrr/+esXHx+vZZ5+12yZMmKBOnTrphRdekGVZcrvduvfee3XfffdJknw+n+Lj45WXl6e0tDTt2rVLiYmJ2rRpk4YOHSpJys/P19ixY3Xw4EG53e7vHIff75fT6ZTP55PD4QjmFCVJvbPfarLvs3mpQT8fAAA/BKf7/h30OzBXXHGF1q5dq48//liS9H//93/629/+pjFjxkiS9u7dK6/Xq+TkZHsfp9OppKQkFRcXS5KKi4sVExNjhxdJSk5OVnh4uEpKSho9b21trfx+f8AGAADap4hgHzA7O1t+v1/9+vVThw4ddOLECf3+979Xenq6JMnr9UqS4uPjA/aLj4+3+7xer+Li4gIHGhGh2NhYu+ZkOTk5mjt3brCnAwAAQlDQ78C88sorWr58uV588UVt2bJFy5Yt0+OPP65ly5YF+1QBZs2aJZ/PZ28HDhxo1fMBAIC2E/Q7MDNmzFB2drbS0tIkSQMHDtS+ffuUk5OjSZMmyeVySZIqKirUo0cPe7+KigoNHjxYkuRyuVRZWRlw3OPHj+vw4cP2/ieLiopSVFRUsKcDAABCUNDvwHz11VcKDw88bIcOHVRfXy9J6tOnj1wul9auXWv3+/1+lZSUyOPxSJI8Ho+qqqpUWlpq16xbt0719fVKSkoK9pABAIBhgn4HZty4cfr973+vnj17asCAAdq6daueeOIJ/fKXv5QkhYWFadq0aXrkkUd0wQUXqE+fPpo9e7bcbrfGjx8vSerfv79Gjx6tyZMnKzc3V3V1dcrKylJaWtppfQIJAAC0b0EPME899ZRmz56t//zP/1RlZaXcbrd+9atfac6cOXbN/fffr5qaGk2ZMkVVVVUaPny48vPzFR0dbdcsX75cWVlZGjlypMLDwzVhwgQtWrQo2MMFAAAGCvr3wIQKvgcGAADztNn3wAAAALQ2AgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOO0SoD5/PPPdfvtt6tbt27q1KmTBg4cqM2bN9v9lmVpzpw56tGjhzp16qTk5GTt2bMn4BiHDx9Wenq6HA6HYmJilJGRoerq6tYYLgAAMEzQA8yXX36pK6+8Uh07dtQ777yjjz76SP/1X/+lrl272jXz58/XokWLlJubq5KSEnXu3FkpKSk6evSoXZOenq6dO3eqoKBAq1evVlFRkaZMmRLs4QIAAAOFWZZlBfOA2dnZ2rBhg/7617822m9Zltxut+69917dd999kiSfz6f4+Hjl5eUpLS1Nu3btUmJiojZt2qShQ4dKkvLz8zV27FgdPHhQbrf7O8fh9/vldDrl8/nkcDiCN8H/p3f2W032fTYvNejnAwDgh+B037+DfgfmL3/5i4YOHaqf/exniouL0yWXXKJnnnnG7t+7d6+8Xq+Sk5PtNqfTqaSkJBUXF0uSiouLFRMTY4cXSUpOTlZ4eLhKSkoaPW9tba38fn/ABgAA2qegB5i///3vWrJkiS644AK9++67mjp1qu6++24tW7ZMkuT1eiVJ8fHxAfvFx8fbfV6vV3FxcQH9ERERio2NtWtOlpOTI6fTaW8JCQnBnhoAAAgRQQ8w9fX1uvTSS/WHP/xBl1xyiaZMmaLJkycrNzc32KcKMGvWLPl8Pns7cOBAq54PAAC0naAHmB49eigxMTGgrX///tq/f78kyeVySZIqKioCaioqKuw+l8ulysrKgP7jx4/r8OHDds3JoqKi5HA4AjYAANA+BT3AXHnllSovLw9o+/jjj9WrVy9JUp8+feRyubR27Vq73+/3q6SkRB6PR5Lk8XhUVVWl0tJSu2bdunWqr69XUlJSsIcMAAAMExHsA95zzz264oor9Ic//EE333yzNm7cqKVLl2rp0qWSpLCwME2bNk2PPPKILrjgAvXp00ezZ8+W2+3W+PHjJX19x2b06NH2r57q6uqUlZWltLS00/oEEgAAaN+CHmAuu+wyrVy5UrNmzdJDDz2kPn366Mknn1R6erpdc//996umpkZTpkxRVVWVhg8frvz8fEVHR9s1y5cvV1ZWlkaOHKnw8HBNmDBBixYtCvZwAQCAgYL+PTChgu+BAQDAPG32PTAAAACtjQADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4rR5g5s2bp7CwME2bNs1uO3r0qDIzM9WtWzedffbZmjBhgioqKgL2279/v1JTU3XWWWcpLi5OM2bM0PHjx1t7uAAAwACtGmA2bdqkP/7xj7r44osD2u+55x69+eabevXVV1VYWKhDhw7pxhtvtPtPnDih1NRUHTt2TB988IGWLVumvLw8zZkzpzWHCwAADNFqAaa6ulrp6el65pln1LVrV7vd5/Pp2Wef1RNPPKEf//jHGjJkiJ5//nl98MEH+vDDDyVJa9as0UcffaQXXnhBgwcP1pgxY/Twww9r8eLFOnbsWGsNGQAAGKLVAkxmZqZSU1OVnJwc0F5aWqq6urqA9n79+qlnz54qLi6WJBUXF2vgwIGKj4+3a1JSUuT3+7Vz585Gz1dbWyu/3x+wAQCA9imiNQ66YsUKbdmyRZs2bTqlz+v1KjIyUjExMQHt8fHx8nq9ds03w0tDf0NfY3JycjR37twgjB4AAIS6oN+BOXDggH7zm99o+fLlio6ODvbhmzRr1iz5fD57O3DgwBk7NwAAOLOCHmBKS0tVWVmpSy+9VBEREYqIiFBhYaEWLVqkiIgIxcfH69ixY6qqqgrYr6KiQi6XS5LkcrlO+VRSw+uGmpNFRUXJ4XAEbAAAoH0KeoAZOXKktm/frrKyMnsbOnSo0tPT7T937NhRa9eutfcpLy/X/v375fF4JEkej0fbt29XZWWlXVNQUCCHw6HExMRgDxkAABgm6M/AdOnSRRdddFFAW+fOndWtWze7PSMjQ9OnT1dsbKwcDofuuusueTweXX755ZKkUaNGKTExURMnTtT8+fPl9Xr1u9/9TpmZmYqKigr2kAEAgGFa5SHe77JgwQKFh4drwoQJqq2tVUpKip5++mm7v0OHDlq9erWmTp0qj8ejzp07a9KkSXrooYfaYrgAACDEhFmWZbX1IFqD3++X0+mUz+drledheme/1WTfZ/NSg34+AAB+CE73/Zt/CwkAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcSLaegAA0BL8i/DADxt3YAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjBD3A5OTk6LLLLlOXLl0UFxen8ePHq7y8PKDm6NGjyszMVLdu3XT22WdrwoQJqqioCKjZv3+/UlNTddZZZykuLk4zZszQ8ePHgz1cAABgoKAHmMLCQmVmZurDDz9UQUGB6urqNGrUKNXU1Ng199xzj9588029+uqrKiws1KFDh3TjjTfa/SdOnFBqaqqOHTumDz74QMuWLVNeXp7mzJkT7OECAAADhVmWZbXmCb744gvFxcWpsLBQV199tXw+n8455xy9+OKLuummmyRJu3fvVv/+/VVcXKzLL79c77zzjq6//nodOnRI8fHxkqTc3FzNnDlTX3zxhSIjI7/zvH6/X06nUz6fTw6HI+jz6p39VpN9n81LDfr5AATiGgTap9N9/271Z2B8Pp8kKTY2VpJUWlqquro6JScn2zX9+vVTz549VVxcLEkqLi7WwIED7fAiSSkpKfL7/dq5c2ej56mtrZXf7w/YAABA+9SqAaa+vl7Tpk3TlVdeqYsuukiS5PV6FRkZqZiYmIDa+Ph4eb1eu+ab4aWhv6GvMTk5OXI6nfaWkJAQ5NkAAIBQ0aoBJjMzUzt27NCKFSta8zSSpFmzZsnn89nbgQMHWv2cAACgbUS01oGzsrK0evVqFRUV6dxzz7XbXS6Xjh07pqqqqoC7MBUVFXK5XHbNxo0bA47X8CmlhpqTRUVFKSoqKsizAAAAoSjod2Asy1JWVpZWrlypdevWqU+fPgH9Q4YMUceOHbV27Vq7rby8XPv375fH45EkeTwebd++XZWVlXZNQUGBHA6HEhMTgz1kAABgmKDfgcnMzNSLL76oN954Q126dLGfWXE6nerUqZOcTqcyMjI0ffp0xcbGyuFw6K677pLH49Hll18uSRo1apQSExM1ceJEzZ8/X16vV7/73e+UmZnJXRYAABD8ALNkyRJJ0rXXXhvQ/vzzz+sXv/iFJGnBggUKDw/XhAkTVFtbq5SUFD399NN2bYcOHbR69WpNnTpVHo9HnTt31qRJk/TQQw8Fe7gAAMBAQQ8wp/O1MtHR0Vq8eLEWL17cZE2vXr309ttvB3NoAACgneDfQgIAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgRbT0AAAAQmnpnv9Vk32fzUs/gSE7FHRgAAGCckA4wixcvVu/evRUdHa2kpCRt3LixrYcEAABCQMgGmJdfflnTp0/XAw88oC1btmjQoEFKSUlRZWVlWw8NAAC0sZANME888YQmT56sO+64Q4mJicrNzdVZZ52l5557rq2HBgAA2lhIPsR77NgxlZaWatasWXZbeHi4kpOTVVxc3Og+tbW1qq2ttV/7fD5Jkt/vb5Ux1td+1WRfa50TwP+PaxBofW1xnTUc17Ksb60LyQDzz3/+UydOnFB8fHxAe3x8vHbv3t3oPjk5OZo7d+4p7QkJCa0yxm/jfPKMnxLAN3ANAq2vta+zI0eOyOl0NtkfkgGmJWbNmqXp06fbr+vr63X48GF169ZNYWFhQT2X3+9XQkKCDhw4IIfDEdRjhwLmZ772PkfmZ772Pkfm13KWZenIkSNyu93fWheSAaZ79+7q0KGDKioqAtorKirkcrka3ScqKkpRUVEBbTExMa01REmSw+Folz+YDZif+dr7HJmf+dr7HJlfy3zbnZcGIfkQb2RkpIYMGaK1a9fabfX19Vq7dq08Hk8bjgwAAISCkLwDI0nTp0/XpEmTNHToUA0bNkxPPvmkampqdMcdd7T10AAAQBsL2QBzyy236IsvvtCcOXPk9Xo1ePBg5efnn/Jgb1uIiorSAw88cMqvrNoL5me+9j5H5me+9j5H5tf6wqzv+pwSAABAiAnJZ2AAAAC+DQEGAAAYhwADAACMQ4ABAADG+cEHmKKiIo0bN05ut1thYWFatWrVd+6zfv16XXrppYqKitKPfvQj5eXlnVKzePFi9e7dW9HR0UpKStLGjRuDP/jT0Nz5vf7667ruuut0zjnnyOFwyOPx6N133w2oefDBBxUWFhaw9evXrxVn0bTmzm/9+vWnjD0sLExerzegLlTWT2r+HH/xi180OscBAwbYNaG0hjk5ObrsssvUpUsXxcXFafz48SovL//O/V599VX169dP0dHRGjhwoN5+++2AfsuyNGfOHPXo0UOdOnVScnKy9uzZ01rTaFJL5vfMM8/oqquuUteuXdW1a1clJyef8jPY2DqPHj26NafSqJbMLy8v75SxR0dHB9SEyvpJLZvjtdde2+h1mJqaateEyhouWbJEF198sf2ldB6PR++888637hMK198PPsDU1NRo0KBBWrx48WnV7927V6mpqRoxYoTKyso0bdo03XnnnQFv8i+//LKmT5+uBx54QFu2bNGgQYOUkpKiysrK1ppGk5o7v6KiIl133XV6++23VVpaqhEjRmjcuHHaunVrQN2AAQP0j3/8w97+9re/tcbwv1Nz59egvLw8YPxxcXF2Xyitn9T8OS5cuDBgbgcOHFBsbKx+9rOfBdSFyhoWFhYqMzNTH374oQoKClRXV6dRo0appqamyX0++OAD3XrrrcrIyNDWrVs1fvx4jR8/Xjt27LBr5s+fr0WLFik3N1clJSXq3LmzUlJSdPTo0TMxLVtL5rd+/Xrdeuutev/991VcXKyEhASNGjVKn3/+eUDd6NGjA9bwpZdeau3pnKIl85O+/gbXb4593759Af2hsn5Sy+b4+uuvB8xvx44d6tChwynXYSis4bnnnqt58+aptLRUmzdv1o9//GPdcMMN2rlzZ6P1IXP9WbBJslauXPmtNffff781YMCAgLZbbrnFSklJsV8PGzbMyszMtF+fOHHCcrvdVk5OTlDH21ynM7/GJCYmWnPnzrVfP/DAA9agQYOCN7AgOZ35vf/++5Yk68svv2yyJlTXz7JatoYrV660wsLCrM8++8xuC9U1tCzLqqystCRZhYWFTdbcfPPNVmpqakBbUlKS9atf/cqyLMuqr6+3XC6X9dhjj9n9VVVVVlRUlPXSSy+1zsBP0+nM72THjx+3unTpYi1btsxumzRpknXDDTe0wgi/n9OZ3/PPP285nc4m+0N5/SyrZWu4YMECq0uXLlZ1dbXdFqpraFmW1bVrV+t//ud/Gu0LlevvB38HprmKi4uVnJwc0JaSkqLi4mJJ0rFjx1RaWhpQEx4eruTkZLvGJPX19Tpy5IhiY2MD2vfs2SO3263zzjtP6enp2r9/fxuNsGUGDx6sHj166LrrrtOGDRvs9va2fpL07LPPKjk5Wb169QpoD9U19Pl8knTKz9w3fdd1uHfvXnm93oAap9OppKSkNl/H05nfyb766ivV1dWdss/69esVFxenvn37aurUqfrXv/4V1LG2xOnOr7q6Wr169VJCQsIpf9sP5fWTWraGzz77rNLS0tS5c+eA9lBbwxMnTmjFihWqqalp8p/uCZXrjwDTTF6v95RvA46Pj5ff79e///1v/fOf/9SJEycarTn5OQsTPP7446qurtbNN99styUlJSkvL0/5+flasmSJ9u7dq6uuukpHjhxpw5Genh49eig3N1evvfaaXnvtNSUkJOjaa6/Vli1bJKndrd+hQ4f0zjvv6M477wxoD9U1rK+v17Rp03TllVfqoosuarKuqeuwYY0a/htq63i68zvZzJkz5Xa7A94QRo8erf/93//V2rVr9eijj6qwsFBjxozRiRMnWmPop+V059e3b18999xzeuONN/TCCy+ovr5eV1xxhQ4ePCgpdNdPatkabty4UTt27DjlOgylNdy+fbvOPvtsRUVF6de//rVWrlypxMTERmtD5foL2X9KAG3vxRdf1Ny5c/XGG28EPCMyZswY+88XX3yxkpKS1KtXL73yyivKyMhoi6Getr59+6pv37726yuuuEKffvqpFixYoD/96U9tOLLWsWzZMsXExGj8+PEB7aG6hpmZmdqxY0ebPY/T2loyv3nz5mnFihVav359wIOuaWlp9p8HDhyoiy++WOeff77Wr1+vkSNHBnXcp+t05+fxeAL+dn/FFVeof//++uMf/6iHH364tYf5vbRkDZ999lkNHDhQw4YNC2gPpTXs27evysrK5PP59Oc//1mTJk1SYWFhkyEmFHAHpplcLpcqKioC2ioqKuRwONSpUyd1795dHTp0aLTG5XKdyaF+LytWrNCdd96pV1555ZRbhSeLiYnRhRdeqE8++eQMjS64hg0bZo+9vayf9PWnAJ577jlNnDhRkZGR31obCmuYlZWl1atX6/3339e55577rbVNXYcNa9Tw31Bax+bMr8Hjjz+uefPmac2aNbr44ou/tfa8885T9+7d22wNWzK/Bh07dtQll1xijz0U109q2Rxramq0YsWK0/qLQVuuYWRkpH70ox9pyJAhysnJ0aBBg7Rw4cJGa0Pl+iPANJPH49HatWsD2goKCuy/TURGRmrIkCEBNfX19Vq7dm2Tv08MNS+99JLuuOMOvfTSSwEf+WtKdXW1Pv30U/Xo0eMMjC74ysrK7LG3h/VrUFhYqE8++eS0/o+zLdfQsixlZWVp5cqVWrdunfr06fOd+3zXddinTx+5XK6AGr/fr5KSkjO+ji2Zn/T1pzgefvhh5efna+jQod9Zf/DgQf3rX/8642vY0vl904kTJ7R9+3Z77KG0ftL3m+Orr76q2tpa3X777d9Z21Zr2Jj6+nrV1tY22hcy11/QHgc21JEjR6ytW7daW7dutSRZTzzxhLV161Zr3759lmVZVnZ2tjVx4kS7/u9//7t11llnWTNmzLB27dplLV682OrQoYOVn59v16xYscKKioqy8vLyrI8++siaMmWKFRMTY3m93pCf3/Lly62IiAhr8eLF1j/+8Q97q6qqsmvuvfdea/369dbevXutDRs2WMnJyVb37t2tysrKkJ/fggULrFWrVll79uyxtm/fbv3mN7+xwsPDrffee8+uCaX1s6zmz7HB7bffbiUlJTV6zFBaw6lTp1pOp9Nav359wM/cV199ZddMnDjRys7Otl9v2LDBioiIsB5//HFr165d1gMPPGB17NjR2r59u10zb948KyYmxnrjjTesbdu2WTfccIPVp08f69///nfIz2/evHlWZGSk9ec//zlgnyNHjliW9fXPxH333WcVFxdbe/futd577z3r0ksvtS644ALr6NGjIT+/uXPnWu+++6716aefWqWlpVZaWpoVHR1t7dy5064JlfWzrJbNscHw4cOtW2655ZT2UFrD7Oxsq7Cw0Nq7d6+1bds2Kzs72woLC7PWrFljWVboXn8/+ADT8LHak7dJkyZZlvX1x9yuueaaU/YZPHiwFRkZaZ133nnW888/f8pxn3rqKatnz55WZGSkNWzYMOvDDz9s/ck0ornzu+aaa7613rK+/th4jx49rMjISOs//uM/rFtuucX65JNPzuzE/p/mzu/RRx+1zj//fCs6OtqKjY21rr32WmvdunWnHDdU1s+yWvYzWlVVZXXq1MlaunRpo8cMpTVsbG6SAq6ra665JuBn0LIs65VXXrEuvPBCKzIy0howYID11ltvBfTX19dbs2fPtuLj462oqChr5MiRVnl5+RmYUaCWzK9Xr16N7vPAAw9YlmVZX331lTVq1CjrnHPOsTp27Gj16tXLmjx5cpuE7JbMb9q0afb1FR8fb40dO9basmVLwHFDZf0sq+U/o7t377Yk2UHgm0JpDX/5y19avXr1siIjI61zzjnHGjlyZMCYQ/X6C7MsywrSzRwAAIAzgmdgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADDO/wfhamWXm0J9HwAAAABJRU5ErkJggg==", "text/plain": [ "<Figure size 640x480 with 1 Axes>" ] @@ -287,6 +327,25 @@ ] }, { + "cell_type": "code", + "execution_count": 97, + "metadata": {}, + "outputs": [], + "source": [ + "import pickle\n", + "\n", + "graphs, labels, _ = mkbatch(3*10**5)\n", + "\n", + "data = {\n", + " \"data\": graphs,\n", + " \"labels\": labels\n", + "}\n", + "\n", + "with open('data.pkl', 'wb') as file:\n", + " pickle.dump(data, file)" + ] + }, + { "cell_type": "markdown", "metadata": { "id": "Q3Cg_8UQep8g" @@ -297,7 +356,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 135, "metadata": { "id": "tLOWhg_CeWzH" }, @@ -323,12 +382,13 @@ "\n", " self.fc_out = nn.Linear(model_dim*seq_len, output_dim, dtype=torch.bfloat16)\n", "\n", - " def forward(self, src, key_padding_mask):\n", + " def full_embedding(self, src):\n", " batch_size, src_len = src.size(0), src.size(1)\n", - " embed = self.embedding(src)\n", - " src = torch.cat((embed * sqrt(self.model_dim), self.fancy_encoding.repeat((batch_size, 1, 1))), dim=2)\n", - "\n", - " output = self.transformer_encoder(src, src_key_padding_mask=key_padding_mask)\n", + " return torch.cat((self.embedding(src) * sqrt(self.model_dim), self.fancy_encoding.repeat((batch_size, 1, 1))), dim=2)\n", + " \n", + " def forward(self, src, key_padding_mask):\n", + " embed = self.full_embedding(src)\n", + " output = self.transformer_encoder(embed, src_key_padding_mask=key_padding_mask)\n", " output[key_padding_mask] = 0 # Hack to stop no_grad problem\n", " flat_output = torch.flatten(output, start_dim=1, end_dim=2)\n", " output = self.fc_out(flat_output)\n", @@ -346,7 +406,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 136, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -359,8 +419,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Training data: 839M\n", - "Trainable parameters in the model: 804K\n" + "Training data: 1049M\n", + "Trainable parameters in the model: 50K\n" ] } ], @@ -368,19 +428,19 @@ "# PARAMS\n", "VOCAB_SIZE = 1 + MAX_VTXS # one more than the max number of vertices\n", "MODEL_DIM = 64 # Dimension of model (embedding and transformer)\n", - "NEPOCHS = 100\n", - "BSZ = 8196 # Batch size\n", - "BPE = 1024 # Batches per epoch\n", - "LR = 0.003\n", - "WD = 0.002\n", - "NHEADS = 4\n", - "NLAYERS = 16\n", + "NEPOCHS = 1000\n", + "BSZ = 8196 * 4 # Batch size\n", + "BPE = 32 # Batches per epoch\n", + "LR = 5e-3\n", + "WD = 2e-3\n", + "NHEADS = 1 #4\n", + "NLAYERS = 1 #16\n", "DROPOUT = 0.2\n", "model = TransformerModel(input_dim=VOCAB_SIZE, model_dim=MODEL_DIM,\n", " output_dim=1, num_heads=NHEADS,\n", " num_layers=NLAYERS, seq_len=SEQ_LEN,\n", " dropout=DROPOUT, device=device).to(device)\n", - "model = torch.compile(model)\n", + "# model = torch.compile(model)\n", "\n", "criterion = nn.MSELoss()\n", "optimizer = torch.optim.Adam(model.parameters(), lr=LR, weight_decay=WD)\n", @@ -391,6 +451,25 @@ ] }, { + "cell_type": "code", + "execution_count": 137, + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import DataLoader, TensorDataset\n", + "\n", + "with open(\"data.pkl\", \"rb\") as f:\n", + " pickled_stuff = pickle.load(f)\n", + "\n", + "data = pickled_stuff[\"data\"].to(device)\n", + "label = pickled_stuff[\"labels\"].to(device)\n", + "padding_mask = (data == PAD_TOKEN).bool().to(device)\n", + "dataset = TensorDataset(data, label, padding_mask)\n", + "# train_dataset, test_dataset = torch.utils.data.random_split(dataset, [.9, .1])\n", + "train_loader = DataLoader(dataset, batch_size=BSZ, shuffle=True)" + ] + }, + { "cell_type": "markdown", "metadata": { "id": "f8Zn33m7CxL5" @@ -401,7 +480,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 138, "metadata": {}, "outputs": [], "source": [ @@ -417,47 +496,7 @@ }, { "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "fc7a660a4efe4342bf4af23cecdea1b1", - "version_major": 2, - "version_minor": 0 - }, - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoAAAAHgCAYAAAA10dzkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAApCklEQVR4nO3df1TUdb7H8RegM1gKWggKjpLuqq0/oMVENNfcg1F6ad1TV9KukNdK03VNTm3aD7GtxDQ77ipKuVntLZPilreTRJcot6vRuqHsqc0fa/hrK1AyQTFBmM/9o9PsToCBwYzO5/k4Z/7gw+fLvMfvGs/9zg+DjDFGAAAAsEawvwcAAACAbxGAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgABwkQgKCtKvfvUrf48BIAAQgAAkSc8995yCgoIUFBSkbdu2Nfm+MUYul0tBQUH6t3/7N6/vnTp1SllZWRo6dKguvfRSXX755YqPj9f8+fP1+eefe/YtWbLEcx/N3SoqKjr8cZ7LuWabPXu2X2cDgPbUyd8DALiwhIaGauPGjbrmmmu81v/0pz/pH//4h5xOp9f62bNn9bOf/Ux79uxRRkaG5s2bp1OnTulvf/ubNm7cqF/+8peKjo72OmbdunXq2rVrk/vu3r17uz+etpowYYLS09ObrA8cONAP0wBAxyAAAXiZOHGiXnnlFf3+979Xp07//E/Exo0blZCQoKqqKq/9mzdv1q5du/Tiiy9q2rRpXt87c+aM6uvrm9zHzTffrIiIiI55AD/QwIED9R//8R/+HgMAOhRPAQPwMnXqVH355ZcqKiryrNXX1ys/P79J4EnSp59+KkkaM2ZMk++FhoYqLCysXeYaOnSoxo8f32Td7XYrJiZGN998s2dt06ZNSkhIULdu3RQWFqZhw4bpd7/7XbvMIUnXXnuthg4dqtLSUo0ePVpdunTRFVdcodzc3CZ7jx49qpkzZyoqKkqhoaGKi4vT888/3+zj+N3vfqdhw4YpNDRUPXv21PXXX68PP/ywyd7Nmzdr6NChcjqdGjJkiAoLC72+f/LkSd19992KjY2V0+lUZGSkJkyYoJ07d7bbnwGAixsBCMBLbGyskpKS9NJLL3nW3nzzTVVXV+uWW25psr9fv36SpD/+8Y8yxrTqPo4fP66qqiqv24kTJ855TFpamt57770mrxPctm2bPv/8c89sRUVFmjp1qnr06KHHH39cy5Yt07XXXqvt27e3arYzZ840ma2qqqrJlcyvvvpKEydOVEJCgpYvX64+ffrorrvu0oYNGzx7vv76a1177bX6r//6L916661asWKFwsPDddtttzUJ0pkzZ+ruu++Wy+XS448/roULFyo0NFQffPBBk8c7Z84c3XLLLVq+fLnOnDmjm266SV9++aVnz+zZs7Vu3TrddNNNWrt2re655x516dJFu3fvbtWfAQALGAAwxjz77LNGkvnLX/5i1qxZY7p162ZOnz5tjDHm3//938348eONMcb069fPTJo0yXPc6dOnzaBBg4wk069fP3PbbbeZZ555xlRWVja5j6ysLCOp2dugQYPOOd/evXuNJLN69Wqv9Tlz5piuXbt6Zp0/f74JCwszDQ0Nbf4zaGk2Seall17y7Bs3bpyRZFauXOlZq6urM/Hx8SYyMtLU19cbY4xZtWqVkWReeOEFz776+nqTlJRkunbtampqaowxxrzzzjtGkvn1r3/dZCa32+01n8PhMPv37/es/fWvf23y5xIeHm7mzp3b5scPwB5cAQTQxJQpU/T111/rjTfe0MmTJ/XGG280+/SvJHXp0kV//vOfde+990r65t3EM2fOVO/evTVv3jzV1dU1Oea///u/VVRU5HV79tlnzznTwIEDFR8fr7y8PM9aY2Oj8vPzlZqaqi5dukj65o0ktbW1Xk9ht8UvfvGLJrMVFRU1efq5U6dOmjVrludrh8OhWbNm6ejRoyotLZUkFRQUqFevXpo6dapnX+fOnfXrX/9ap06d0p/+9CfPn0dQUJCysrKazBMUFOT1dXJysgYMGOD5evjw4QoLC1N5eblnrXv37vrzn//s9Q5sAPhXvAkEQBM9e/ZUcnKyNm7cqNOnT6uxsdHrNXbfFR4eruXLl2v58uU6dOiQiouL9cQTT2jNmjUKDw/Xo48+6rX/Zz/72Xm9CSQtLU3333+/PvvsM8XExGjr1q06evSo0tLSPHvmzJmjl19+WTfccINiYmJ03XXXacqUKbr++utbdR99+vRRcnLy9+6Ljo7WpZde6rX27TuFDx48qFGjRunQoUP68Y9/rOBg7/+vfeWVV0qSDh06JOmb11FGR0frsssu+9777du3b5O1Hj166KuvvvJ8vXz5cmVkZMjlcikhIUETJ05Uenq6+vfv/70/H4AduAIIoFnTpk3Tm2++qdzcXN1www2t/oiWfv366T//8z+1fft2de/eXS+++GK7zZSWliZjjF555RVJ0ssvv6zw8HCvuIuMjFRZWZlef/113XjjjXr33Xd1ww03KCMjo93m8KeQkJBm182/vP5yypQpKi8v1+rVqxUdHa0VK1ZoyJAhevPNN301JoALHAEIoFm//OUvFRwcrA8++KDFp3/PpUePHhowYIC++OKLdpvpiiuu0MiRI5WXl6eGhga9+uqrmjx5cpPPJnQ4HEpNTdXatWv16aefatasWfrjH/+o/fv3t9ssn3/+uWpra73W9u3bJ+mbN9JI38Tw3//+d7ndbq99e/bs8XxfkgYMGKDPP/9cx48fb7f5evfurTlz5mjz5s06cOCALr/8cj322GPt9vMBXNwIQADN6tq1q9atW6clS5YoNTW1xX1//etfm3w2oPTN05uffPKJBg0a1K5zpaWl6YMPPtCGDRtUVVXl9fSvJK93w0pScHCwhg8fLknNvh7xfDU0NOipp57yfF1fX6+nnnpKPXv2VEJCgqRvPlOxoqLC63WLDQ0NWr16tbp27apx48ZJkm666SYZY/Twww83uR/TyndWf6uxsVHV1dVea5GRkYqOjm7Xxw/g4sZrAAG0qDVPmxYVFSkrK0s33nijRo0apa5du6q8vFwbNmxQXV2dlixZ0uSY/Pz8Zv8lkAkTJigqKuqc9zdlyhTdc889uueee3TZZZc1eb3e7bffruPHj+vnP/+5+vTpo0OHDmn16tWKj4/3vPbuXPbt26cXXnihyXpUVJQmTJjg+To6OlqPP/64Dh48qIEDByovL09lZWV6+umn1blzZ0nSnXfeqaeeekq33XabSktLFRsbq/z8fG3fvl2rVq1St27dJEnjx4/X9OnT9fvf/15///vfdf3118vtduv//u//NH78+Db9+78nT55Unz59dPPNNysuLk5du3bV22+/rb/85S9auXJlq38OgADn3zchA7hQ/OvHwJzLdz8Gpry83CxevNiMGjXKREZGmk6dOpmePXuaSZMmmXfeecfr2HN9DIwk8+6777Zq1jFjxhhJ5vbbb2/yvfz8fHPdddeZyMhI43A4TN++fc2sWbPMF1988b0/91yzjRs3zrNv3LhxZsiQIebDDz80SUlJJjQ01PTr18+sWbOmyc+srKw0M2bMMBEREcbhcJhhw4aZZ599tsm+hoYGs2LFCjN48GDjcDhMz549zQ033GBKS0u95mvu41369etnMjIyjDHffBzNvffea+Li4ky3bt3MpZdeauLi4szatWu/9/EDsEeQMW18fgEALHfttdeqqqpKH3/8sb9HAYDzwmsAAQAALEMAAgAAWIYABAAAsAyvAQQAALAMVwABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWCZgAfO+995Samqro6GgFBQVp8+bN33vM1q1b9dOf/lROp1M/+tGP9Nxzz3X4nAAAAP4WMAFYW1uruLg45eTktGr/gQMHNGnSJI0fP15lZWW6++67dfvtt+utt97q4EkBAAD8K8gYY/w9RHsLCgrSa6+9psmTJ7e457777tOWLVv08ccfe9ZuueUWnThxQoWFhT6YEgAAwD8C5gpgW5WUlCg5OdlrLSUlRSUlJX6aCAAAwDc6+XsAf6moqFBUVJTXWlRUlGpqavT111+rS5cuTY6pq6tTXV2d52u3263jx4/r8ssvV1BQUIfPDAAAfjhjjE6ePKno6GgFB9t5LczaADwf2dnZevjhh/09BgAAaAdHjhxRnz59/D2GX1gbgL169VJlZaXXWmVlpcLCwpq9+idJixYtUmZmpufr6upq9e3bV0eOHFFYWFiHzgsAANpHTU2NXC6XunXr5u9R/MbaAExKSlJBQYHXWlFRkZKSklo8xul0yul0NlkPCwsjAAEAuMjY/PKtgHni+9SpUyorK1NZWZmkbz7mpaysTIcPH5b0zdW79PR0z/7Zs2ervLxcv/nNb7Rnzx6tXbtWL7/8shYsWOCP8QEAAHwmYALwww8/1FVXXaWrrrpKkpSZmamrrrpKixcvliR98cUXnhiUpCuuuEJbtmxRUVGR4uLitHLlSv3hD39QSkqKX+YHAADwlYD8HEBfqampUXh4uKqrq3kKGACAiwS/vwPoCiAAAABahwAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYJqADMyclRbGysQkNDlZiYqB07dpxz/6pVqzRo0CB16dJFLpdLCxYs0JkzZ3w0LQAAgH8ETADm5eUpMzNTWVlZ2rlzp+Li4pSSkqKjR482u3/jxo1auHChsrKytHv3bj3zzDPKy8vT/fff7+PJAQAAfCtgAvDJJ5/UHXfcoRkzZugnP/mJcnNzdckll2jDhg3N7n///fc1ZswYTZs2TbGxsbruuus0derU771qCAAAcLELiACsr69XaWmpkpOTPWvBwcFKTk5WSUlJs8eMHj1apaWlnuArLy9XQUGBJk6c6JOZAQAA/KWTvwdoD1VVVWpsbFRUVJTXelRUlPbs2dPsMdOmTVNVVZWuueYaGWPU0NCg2bNnn/Mp4Lq6OtXV1Xm+rqmpaZ8HAAAA4EMBcQXwfGzdulVLly7V2rVrtXPnTr366qvasmWLHnnkkRaPyc7OVnh4uOfmcrl8ODEAAED7CDLGGH8P8UPV19frkksuUX5+viZPnuxZz8jI0IkTJ/Q///M/TY4ZO3asRo0apRUrVnjWXnjhBd155506deqUgoObtnFzVwBdLpeqq6sVFhbWvg8KAAB0iJqaGoWHh1v9+zsgrgA6HA4lJCSouLjYs+Z2u1VcXKykpKRmjzl9+nSTyAsJCZEktdTETqdTYWFhXjcAAICLTUC8BlCSMjMzlZGRoREjRmjkyJFatWqVamtrNWPGDElSenq6YmJilJ2dLUlKTU3Vk08+qauuukqJiYnav3+/HnroIaWmpnpCEAAAIBAFTACmpaXp2LFjWrx4sSoqKhQfH6/CwkLPG0MOHz7sdcXvwQcfVFBQkB588EF99tln6tmzp1JTU/XYY4/56yEAAAD4REC8BtBfeA0BAAAXH35/B8hrAAEAANB6BCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsExABWBOTo5iY2MVGhqqxMRE7dix45z7T5w4oblz56p3795yOp0aOHCgCgoKfDQtAACAf3Ty9wDtJS8vT5mZmcrNzVViYqJWrVqllJQU7d27V5GRkU3219fXa8KECYqMjFR+fr5iYmJ06NAhde/e3ffDAwAA+FCQMcb4e4j2kJiYqKuvvlpr1qyRJLndbrlcLs2bN08LFy5ssj83N1crVqzQnj171Llz5/O6z5qaGoWHh6u6ulphYWE/aH4AAOAb/P4OkKeA6+vrVVpaquTkZM9acHCwkpOTVVJS0uwxr7/+upKSkjR37lxFRUVp6NChWrp0qRobG1u8n7q6OtXU1HjdAAAALjYBEYBVVVVqbGxUVFSU13pUVJQqKiqaPaa8vFz5+flqbGxUQUGBHnroIa1cuVKPPvpoi/eTnZ2t8PBwz83lcrXr4wAAAPCFgAjA8+F2uxUZGamnn35aCQkJSktL0wMPPKDc3NwWj1m0aJGqq6s9tyNHjvhwYgAAgPYREG8CiYiIUEhIiCorK73WKysr1atXr2aP6d27tzp37qyQkBDP2pVXXqmKigrV19fL4XA0OcbpdMrpdLbv8AAAAD4WEFcAHQ6HEhISVFxc7Flzu90qLi5WUlJSs8eMGTNG+/fvl9vt9qzt27dPvXv3bjb+AAAAAkVABKAkZWZmav369Xr++ee1e/du3XXXXaqtrdWMGTMkSenp6Vq0aJFn/1133aXjx49r/vz52rdvn7Zs2aKlS5dq7ty5/noIAAAAPhEQTwFLUlpamo4dO6bFixeroqJC8fHxKiws9Lwx5PDhwwoO/mfvulwuvfXWW1qwYIGGDx+umJgYzZ8/X/fdd5+/HgIAAIBPBMznAPoDnyMEAMDFh9/fAfQUMAAAAFqHAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAlgmoAMzJyVFsbKxCQ0OVmJioHTt2tOq4TZs2KSgoSJMnT+7YAQEAAC4AAROAeXl5yszMVFZWlnbu3Km4uDilpKTo6NGj5zzu4MGDuueeezR27FgfTQoAAOBfAROATz75pO644w7NmDFDP/nJT5Sbm6tLLrlEGzZsaPGYxsZG3XrrrXr44YfVv39/H04LAADgPwERgPX19SotLVVycrJnLTg4WMnJySopKWnxuN/+9reKjIzUzJkzW3U/dXV1qqmp8boBAABcbAIiAKuqqtTY2KioqCiv9aioKFVUVDR7zLZt2/TMM89o/fr1rb6f7OxshYeHe24ul+sHzQ0AAOAPARGAbXXy5ElNnz5d69evV0RERKuPW7Rokaqrqz23I0eOdOCUAAAAHaOTvwdoDxEREQoJCVFlZaXXemVlpXr16tVk/6effqqDBw8qNTXVs+Z2uyVJnTp10t69ezVgwIAmxzmdTjmdznaeHgAAwLcC4gqgw+FQQkKCiouLPWtut1vFxcVKSkpqsn/w4MH66KOPVFZW5rndeOONGj9+vMrKynhqFwAABLSAuAIoSZmZmcrIyNCIESM0cuRIrVq1SrW1tZoxY4YkKT09XTExMcrOzlZoaKiGDh3qdXz37t0lqck6AABAoAmYAExLS9OxY8e0ePFiVVRUKD4+XoWFhZ43hhw+fFjBwQFxwRMAAOAHCTLGGH8PcbGqqalReHi4qqurFRYW5u9xAABAK/D7O0BeAwgAAIDWIwABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUCKgBzcnIUGxur0NBQJSYmaseOHS3uXb9+vcaOHasePXqoR48eSk5OPud+AACAQBEwAZiXl6fMzExlZWVp586diouLU0pKio4ePdrs/q1bt2rq1Kl69913VVJSIpfLpeuuu06fffaZjycHAADwrSBjjPH3EO0hMTFRV199tdasWSNJcrvdcrlcmjdvnhYuXPi9xzc2NqpHjx5as2aN0tPTW3WfNTU1Cg8PV3V1tcLCwn7Q/AAAwDf4/R0gVwDr6+tVWlqq5ORkz1pwcLCSk5NVUlLSqp9x+vRpnT17VpdddlmLe+rq6lRTU+N1AwAAuNgERABWVVWpsbFRUVFRXutRUVGqqKho1c+47777FB0d7RWR35Wdna3w8HDPzeVy/aC5AQAA/CEgAvCHWrZsmTZt2qTXXntNoaGhLe5btGiRqqurPbcjR474cEoAAID20cnfA7SHiIgIhYSEqLKy0mu9srJSvXr1OuexTzzxhJYtW6a3335bw4cPP+dep9Mpp9P5g+cFAADwp4C4AuhwOJSQkKDi4mLPmtvtVnFxsZKSklo8bvny5XrkkUdUWFioESNG+GJUAAAAvwuIK4CSlJmZqYyMDI0YMUIjR47UqlWrVFtbqxkzZkiS0tPTFRMTo+zsbEnS448/rsWLF2vjxo2KjY31vFawa9eu6tq1q98eBwAAQEcLmABMS0vTsWPHtHjxYlVUVCg+Pl6FhYWeN4YcPnxYwcH/vOC5bt061dfX6+abb/b6OVlZWVqyZIkvRwcAAPCpgPkcQH/gc4QAALj48Ps7QF4DCAAAgNYjAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQIqAHNychQbG6vQ0FAlJiZqx44d59z/yiuvaPDgwQoNDdWwYcNUUFDgo0kBAAD8J2ACMC8vT5mZmcrKytLOnTsVFxenlJQUHT16tNn977//vqZOnaqZM2dq165dmjx5siZPnqyPP/7Yx5MDAAD4VpAxxvh7iPaQmJioq6++WmvWrJEkud1uuVwuzZs3TwsXLmyyPy0tTbW1tXrjjTc8a6NGjVJ8fLxyc3NbdZ81NTUKDw9XdXW1wsLC2ueBAACADsXvb6mTvwdoD/X19SotLdWiRYs8a8HBwUpOTlZJSUmzx5SUlCgzM9NrLSUlRZs3b27xfurq6lRXV+f5urq6WtI3/0MCAAAXh29/bwfINbDzEhABWFVVpcbGRkVFRXmtR0VFac+ePc0eU1FR0ez+ioqKFu8nOztbDz/8cJN1l8t1HlMDAAB/+vLLLxUeHu7vMfwiIALQVxYtWuR11fDEiRPq16+fDh8+bO3/gC4UNTU1crlcOnLkiLWX8y8UnIsLC+fjwsG5uHBUV1erb9++uuyyy/w9it8ERABGREQoJCRElZWVXuuVlZXq1atXs8f06tWrTfslyel0yul0NlkPDw/nL/MFIiwsjHNxgeBcXFg4HxcOzsWFIzg4YN4L22YB8cgdDocSEhJUXFzsWXO73SouLlZSUlKzxyQlJXntl6SioqIW9wMAAASKgLgCKEmZmZnKyMjQiBEjNHLkSK1atUq1tbWaMWOGJCk9PV0xMTHKzs6WJM2fP1/jxo3TypUrNWnSJG3atEkffvihnn76aX8+DAAAgA4XMAGYlpamY8eOafHixaqoqFB8fLwKCws9b/Q4fPiw16Xe0aNHa+PGjXrwwQd1//3368c//rE2b96soUOHtvo+nU6nsrKymn1aGL7FubhwcC4uLJyPCwfn4sLBuQigzwEEAABA6wTEawABAADQegQgAACAZQhAAAAAyxCAAAAAliEAv0dOTo5iY2MVGhqqxMRE7dix45z7X3nlFQ0ePFihoaEaNmyYCgoKfDRp4GvLuVi/fr3Gjh2rHj16qEePHkpOTv7ec4fWa+vfi29t2rRJQUFBmjx5cscOaJG2nosTJ05o7ty56t27t5xOpwYOHMh/p9pRW8/HqlWrNGjQIHXp0kUul0sLFizQmTNnfDRt4HrvvfeUmpqq6OhoBQUFafPmzd97zNatW/XTn/5UTqdTP/rRj/Tcc891+Jx+ZdCiTZs2GYfDYTZs2GD+9re/mTvuuMN0797dVFZWNrt/+/btJiQkxCxfvtx88skn5sEHHzSdO3c2H330kY8nDzxtPRfTpk0zOTk5ZteuXWb37t3mtttuM+Hh4eYf//iHjycPPG09F986cOCAiYmJMWPHjjW/+MUvfDNsgGvruairqzMjRowwEydONNu2bTMHDhwwW7duNWVlZT6ePDC19Xy8+OKLxul0mhdffNEcOHDAvPXWW6Z3795mwYIFPp488BQUFJgHHnjAvPrqq0aSee211865v7y83FxyySUmMzPTfPLJJ2b16tUmJCTEFBYW+mZgPyAAz2HkyJFm7ty5nq8bGxtNdHS0yc7Obnb/lClTzKRJk7zWEhMTzaxZszp0Thu09Vx8V0NDg+nWrZt5/vnnO2pEa5zPuWhoaDCjR482f/jDH0xGRgYB2E7aei7WrVtn+vfvb+rr6301olXaej7mzp1rfv7zn3utZWZmmjFjxnTonLZpTQD+5je/MUOGDPFaS0tLMykpKR04mX/xFHAL6uvrVVpaquTkZM9acHCwkpOTVVJS0uwxJSUlXvslKSUlpcX9aJ3zORffdfr0aZ09e9bqf/i7PZzvufjtb3+ryMhIzZw50xdjWuF8zsXrr7+upKQkzZ07V1FRURo6dKiWLl2qxsZGX40dsM7nfIwePVqlpaWep4nLy8tVUFCgiRMn+mRm/JONv78D5l8CaW9VVVVqbGz0/Esi34qKitKePXuaPaaioqLZ/RUVFR02pw3O51x813333afo6Ogmf8HRNudzLrZt26ZnnnlGZWVlPpjQHudzLsrLy/XOO+/o1ltvVUFBgfbv3685c+bo7NmzysrK8sXYAet8zse0adNUVVWla665RsYYNTQ0aPbs2br//vt9MTL+RUu/v2tqavT111+rS5cufpqs43AFEAFv2bJl2rRpk1577TWFhob6exyrnDx5UtOnT9f69esVERHh73Gs53a7FRkZqaeffloJCQlKS0vTAw88oNzcXH+PZqWtW7dq6dKlWrt2rXbu3KlXX31VW7Zs0SOPPOLv0WABrgC2ICIiQiEhIaqsrPRar6ysVK9evZo9plevXm3aj9Y5n3PxrSeeeELLli3T22+/reHDh3fkmFZo67n49NNPdfDgQaWmpnrW3G63JKlTp07au3evBgwY0LFDB6jz+XvRu3dvde7cWSEhIZ61K6+8UhUVFaqvr5fD4ejQmQPZ+ZyPhx56SNOnT9ftt98uSRo2bJhqa2t155136oEHHvD69+vRsVr6/R0WFhaQV/8krgC2yOFwKCEhQcXFxZ41t9ut4uJiJSUlNXtMUlKS135JKioqanE/Wud8zoUkLV++XI888ogKCws1YsQIX4wa8Np6LgYPHqyPPvpIZWVlntuNN96o8ePHq6ysTC6Xy5fjB5Tz+XsxZswY7d+/3xPhkrRv3z717t2b+PuBzud8nD59uknkfRvnxpiOGxZNWPn729/vQrmQbdq0yTidTvPcc8+ZTz75xNx5552me/fupqKiwhhjzPTp083ChQs9+7dv3246depknnjiCbN7926TlZXFx8C0k7aei2XLlhmHw2Hy8/PNF1984bmdPHnSXw8hYLT1XHwX7wJuP209F4cPHzbdunUzv/rVr8zevXvNG2+8YSIjI82jjz7qr4cQUNp6PrKysky3bt3MSy+9ZMrLy83//u//mgEDBpgpU6b46yEEjJMnT5pdu3aZXbt2GUnmySefNLt27TKHDh0yxhizcOFCM336dM/+bz8G5t577zW7d+82OTk5fAyM7VavXm369u1rHA6HGTlypPnggw883xs3bpzJyMjw2v/yyy+bgQMHGofDYYYMGWK2bNni44kDV1vORb9+/YykJresrCzfDx6A2vr34l8RgO2rrefi/fffN4mJicbpdJr+/fubxx57zDQ0NPh46sDVlvNx9uxZs2TJEjNgwAATGhpqXC6XmTNnjvnqq698P3iAeffdd5v9HfDtn39GRoYZN25ck2Pi4+ONw+Ew/fv3N88++6zP5/alIGO4zgwAAGATXgMIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWOb/ATyhabrGscARAAAAAElFTkSuQmCC", - "text/html": [ - "\n", - " <div style=\"display: inline-block;\">\n", - " <div class=\"jupyter-widgets widget-label\" style=\"text-align: center;\">\n", - " Figure\n", - " </div>\n", - " <img src='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAoAAAAHgCAYAAAA10dzkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAApCklEQVR4nO3df1TUdb7H8RegM1gKWggKjpLuqq0/oMVENNfcg1F6ad1TV9KukNdK03VNTm3aD7GtxDQ77ipKuVntLZPilreTRJcot6vRuqHsqc0fa/hrK1AyQTFBmM/9o9PsToCBwYzO5/k4Z/7gw+fLvMfvGs/9zg+DjDFGAAAAsEawvwcAAACAbxGAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgABwkQgKCtKvfvUrf48BIAAQgAAkSc8995yCgoIUFBSkbdu2Nfm+MUYul0tBQUH6t3/7N6/vnTp1SllZWRo6dKguvfRSXX755YqPj9f8+fP1+eefe/YtWbLEcx/N3SoqKjr8cZ7LuWabPXu2X2cDgPbUyd8DALiwhIaGauPGjbrmmmu81v/0pz/pH//4h5xOp9f62bNn9bOf/Ux79uxRRkaG5s2bp1OnTulvf/ubNm7cqF/+8peKjo72OmbdunXq2rVrk/vu3r17uz+etpowYYLS09ObrA8cONAP0wBAxyAAAXiZOHGiXnnlFf3+979Xp07//E/Exo0blZCQoKqqKq/9mzdv1q5du/Tiiy9q2rRpXt87c+aM6uvrm9zHzTffrIiIiI55AD/QwIED9R//8R/+HgMAOhRPAQPwMnXqVH355ZcqKiryrNXX1ys/P79J4EnSp59+KkkaM2ZMk++FhoYqLCysXeYaOnSoxo8f32Td7XYrJiZGN998s2dt06ZNSkhIULdu3RQWFqZhw4bpd7/7XbvMIUnXXnuthg4dqtLSUo0ePVpdunTRFVdcodzc3CZ7jx49qpkzZyoqKkqhoaGKi4vT888/3+zj+N3vfqdhw4YpNDRUPXv21PXXX68PP/ywyd7Nmzdr6NChcjqdGjJkiAoLC72+f/LkSd19992KjY2V0+lUZGSkJkyYoJ07d7bbnwGAixsBCMBLbGyskpKS9NJLL3nW3nzzTVVXV+uWW25psr9fv36SpD/+8Y8yxrTqPo4fP66qqiqv24kTJ855TFpamt57770mrxPctm2bPv/8c89sRUVFmjp1qnr06KHHH39cy5Yt07XXXqvt27e3arYzZ840ma2qqqrJlcyvvvpKEydOVEJCgpYvX64+ffrorrvu0oYNGzx7vv76a1177bX6r//6L916661asWKFwsPDddtttzUJ0pkzZ+ruu++Wy+XS448/roULFyo0NFQffPBBk8c7Z84c3XLLLVq+fLnOnDmjm266SV9++aVnz+zZs7Vu3TrddNNNWrt2re655x516dJFu3fvbtWfAQALGAAwxjz77LNGkvnLX/5i1qxZY7p162ZOnz5tjDHm3//938348eONMcb069fPTJo0yXPc6dOnzaBBg4wk069fP3PbbbeZZ555xlRWVja5j6ysLCOp2dugQYPOOd/evXuNJLN69Wqv9Tlz5piuXbt6Zp0/f74JCwszDQ0Nbf4zaGk2Seall17y7Bs3bpyRZFauXOlZq6urM/Hx8SYyMtLU19cbY4xZtWqVkWReeOEFz776+nqTlJRkunbtampqaowxxrzzzjtGkvn1r3/dZCa32+01n8PhMPv37/es/fWvf23y5xIeHm7mzp3b5scPwB5cAQTQxJQpU/T111/rjTfe0MmTJ/XGG280+/SvJHXp0kV//vOfde+990r65t3EM2fOVO/evTVv3jzV1dU1Oea///u/VVRU5HV79tlnzznTwIEDFR8fr7y8PM9aY2Oj8vPzlZqaqi5dukj65o0ktbW1Xk9ht8UvfvGLJrMVFRU1efq5U6dOmjVrludrh8OhWbNm6ejRoyotLZUkFRQUqFevXpo6dapnX+fOnfXrX/9ap06d0p/+9CfPn0dQUJCysrKazBMUFOT1dXJysgYMGOD5evjw4QoLC1N5eblnrXv37vrzn//s9Q5sAPhXvAkEQBM9e/ZUcnKyNm7cqNOnT6uxsdHrNXbfFR4eruXLl2v58uU6dOiQiouL9cQTT2jNmjUKDw/Xo48+6rX/Zz/72Xm9CSQtLU3333+/PvvsM8XExGjr1q06evSo0tLSPHvmzJmjl19+WTfccINiYmJ03XXXacqUKbr++utbdR99+vRRcnLy9+6Ljo7WpZde6rX27TuFDx48qFGjRunQoUP68Y9/rOBg7/+vfeWVV0qSDh06JOmb11FGR0frsssu+9777du3b5O1Hj166KuvvvJ8vXz5cmVkZMjlcikhIUETJ05Uenq6+vfv/70/H4AduAIIoFnTpk3Tm2++qdzcXN1www2t/oiWfv366T//8z+1fft2de/eXS+++GK7zZSWliZjjF555RVJ0ssvv6zw8HCvuIuMjFRZWZlef/113XjjjXr33Xd1ww03KCMjo93m8KeQkJBm182/vP5yypQpKi8v1+rVqxUdHa0VK1ZoyJAhevPNN301JoALHAEIoFm//OUvFRwcrA8++KDFp3/PpUePHhowYIC++OKLdpvpiiuu0MiRI5WXl6eGhga9+uqrmjx5cpPPJnQ4HEpNTdXatWv16aefatasWfrjH/+o/fv3t9ssn3/+uWpra73W9u3bJ+mbN9JI38Tw3//+d7ndbq99e/bs8XxfkgYMGKDPP/9cx48fb7f5evfurTlz5mjz5s06cOCALr/8cj322GPt9vMBXNwIQADN6tq1q9atW6clS5YoNTW1xX1//etfm3w2oPTN05uffPKJBg0a1K5zpaWl6YMPPtCGDRtUVVXl9fSvJK93w0pScHCwhg8fLknNvh7xfDU0NOipp57yfF1fX6+nnnpKPXv2VEJCgqRvPlOxoqLC63WLDQ0NWr16tbp27apx48ZJkm666SYZY/Twww83uR/TyndWf6uxsVHV1dVea5GRkYqOjm7Xxw/g4sZrAAG0qDVPmxYVFSkrK0s33nijRo0apa5du6q8vFwbNmxQXV2dlixZ0uSY/Pz8Zv8lkAkTJigqKuqc9zdlyhTdc889uueee3TZZZc1eb3e7bffruPHj+vnP/+5+vTpo0OHDmn16tWKj4/3vPbuXPbt26cXXnihyXpUVJQmTJjg+To6OlqPP/64Dh48qIEDByovL09lZWV6+umn1blzZ0nSnXfeqaeeekq33XabSktLFRsbq/z8fG3fvl2rVq1St27dJEnjx4/X9OnT9fvf/15///vfdf3118vtduv//u//NH78+Db9+78nT55Unz59dPPNNysuLk5du3bV22+/rb/85S9auXJlq38OgADn3zchA7hQ/OvHwJzLdz8Gpry83CxevNiMGjXKREZGmk6dOpmePXuaSZMmmXfeecfr2HN9DIwk8+6777Zq1jFjxhhJ5vbbb2/yvfz8fHPdddeZyMhI43A4TN++fc2sWbPMF1988b0/91yzjRs3zrNv3LhxZsiQIebDDz80SUlJJjQ01PTr18+sWbOmyc+srKw0M2bMMBEREcbhcJhhw4aZZ599tsm+hoYGs2LFCjN48GDjcDhMz549zQ033GBKS0u95mvu41369etnMjIyjDHffBzNvffea+Li4ky3bt3MpZdeauLi4szatWu/9/EDsEeQMW18fgEALHfttdeqqqpKH3/8sb9HAYDzwmsAAQAALEMAAgAAWIYABAAAsAyvAQQAALAMVwABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWCZgAfO+995Samqro6GgFBQVp8+bN33vM1q1b9dOf/lROp1M/+tGP9Nxzz3X4nAAAAP4WMAFYW1uruLg45eTktGr/gQMHNGnSJI0fP15lZWW6++67dfvtt+utt97q4EkBAAD8K8gYY/w9RHsLCgrSa6+9psmTJ7e457777tOWLVv08ccfe9ZuueUWnThxQoWFhT6YEgAAwD8C5gpgW5WUlCg5OdlrLSUlRSUlJX6aCAAAwDc6+XsAf6moqFBUVJTXWlRUlGpqavT111+rS5cuTY6pq6tTXV2d52u3263jx4/r8ssvV1BQUIfPDAAAfjhjjE6ePKno6GgFB9t5LczaADwf2dnZevjhh/09BgAAaAdHjhxRnz59/D2GX1gbgL169VJlZaXXWmVlpcLCwpq9+idJixYtUmZmpufr6upq9e3bV0eOHFFYWFiHzgsAANpHTU2NXC6XunXr5u9R/MbaAExKSlJBQYHXWlFRkZKSklo8xul0yul0NlkPCwsjAAEAuMjY/PKtgHni+9SpUyorK1NZWZmkbz7mpaysTIcPH5b0zdW79PR0z/7Zs2ervLxcv/nNb7Rnzx6tXbtWL7/8shYsWOCP8QEAAHwmYALwww8/1FVXXaWrrrpKkpSZmamrrrpKixcvliR98cUXnhiUpCuuuEJbtmxRUVGR4uLitHLlSv3hD39QSkqKX+YHAADwlYD8HEBfqampUXh4uKqrq3kKGACAiwS/vwPoCiAAAABahwAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYJqADMyclRbGysQkNDlZiYqB07dpxz/6pVqzRo0CB16dJFLpdLCxYs0JkzZ3w0LQAAgH8ETADm5eUpMzNTWVlZ2rlzp+Li4pSSkqKjR482u3/jxo1auHChsrKytHv3bj3zzDPKy8vT/fff7+PJAQAAfCtgAvDJJ5/UHXfcoRkzZugnP/mJcnNzdckll2jDhg3N7n///fc1ZswYTZs2TbGxsbruuus0derU771qCAAAcLELiACsr69XaWmpkpOTPWvBwcFKTk5WSUlJs8eMHj1apaWlnuArLy9XQUGBJk6c6JOZAQAA/KWTvwdoD1VVVWpsbFRUVJTXelRUlPbs2dPsMdOmTVNVVZWuueYaGWPU0NCg2bNnn/Mp4Lq6OtXV1Xm+rqmpaZ8HAAAA4EMBcQXwfGzdulVLly7V2rVrtXPnTr366qvasmWLHnnkkRaPyc7OVnh4uOfmcrl8ODEAAED7CDLGGH8P8UPV19frkksuUX5+viZPnuxZz8jI0IkTJ/Q///M/TY4ZO3asRo0apRUrVnjWXnjhBd155506deqUgoObtnFzVwBdLpeqq6sVFhbWvg8KAAB0iJqaGoWHh1v9+zsgrgA6HA4lJCSouLjYs+Z2u1VcXKykpKRmjzl9+nSTyAsJCZEktdTETqdTYWFhXjcAAICLTUC8BlCSMjMzlZGRoREjRmjkyJFatWqVamtrNWPGDElSenq6YmJilJ2dLUlKTU3Vk08+qauuukqJiYnav3+/HnroIaWmpnpCEAAAIBAFTACmpaXp2LFjWrx4sSoqKhQfH6/CwkLPG0MOHz7sdcXvwQcfVFBQkB588EF99tln6tmzp1JTU/XYY4/56yEAAAD4REC8BtBfeA0BAAAXH35/B8hrAAEAANB6BCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsExABWBOTo5iY2MVGhqqxMRE7dix45z7T5w4oblz56p3795yOp0aOHCgCgoKfDQtAACAf3Ty9wDtJS8vT5mZmcrNzVViYqJWrVqllJQU7d27V5GRkU3219fXa8KECYqMjFR+fr5iYmJ06NAhde/e3ffDAwAA+FCQMcb4e4j2kJiYqKuvvlpr1qyRJLndbrlcLs2bN08LFy5ssj83N1crVqzQnj171Llz5/O6z5qaGoWHh6u6ulphYWE/aH4AAOAb/P4OkKeA6+vrVVpaquTkZM9acHCwkpOTVVJS0uwxr7/+upKSkjR37lxFRUVp6NChWrp0qRobG1u8n7q6OtXU1HjdAAAALjYBEYBVVVVqbGxUVFSU13pUVJQqKiqaPaa8vFz5+flqbGxUQUGBHnroIa1cuVKPPvpoi/eTnZ2t8PBwz83lcrXr4wAAAPCFgAjA8+F2uxUZGamnn35aCQkJSktL0wMPPKDc3NwWj1m0aJGqq6s9tyNHjvhwYgAAgPYREG8CiYiIUEhIiCorK73WKysr1atXr2aP6d27tzp37qyQkBDP2pVXXqmKigrV19fL4XA0OcbpdMrpdLbv8AAAAD4WEFcAHQ6HEhISVFxc7Flzu90qLi5WUlJSs8eMGTNG+/fvl9vt9qzt27dPvXv3bjb+AAAAAkVABKAkZWZmav369Xr++ee1e/du3XXXXaqtrdWMGTMkSenp6Vq0aJFn/1133aXjx49r/vz52rdvn7Zs2aKlS5dq7ty5/noIAAAAPhEQTwFLUlpamo4dO6bFixeroqJC8fHxKiws9Lwx5PDhwwoO/mfvulwuvfXWW1qwYIGGDx+umJgYzZ8/X/fdd5+/HgIAAIBPBMznAPoDnyMEAMDFh9/fAfQUMAAAAFqHAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAlgmoAMzJyVFsbKxCQ0OVmJioHTt2tOq4TZs2KSgoSJMnT+7YAQEAAC4AAROAeXl5yszMVFZWlnbu3Km4uDilpKTo6NGj5zzu4MGDuueeezR27FgfTQoAAOBfAROATz75pO644w7NmDFDP/nJT5Sbm6tLLrlEGzZsaPGYxsZG3XrrrXr44YfVv39/H04LAADgPwERgPX19SotLVVycrJnLTg4WMnJySopKWnxuN/+9reKjIzUzJkzW3U/dXV1qqmp8boBAABcbAIiAKuqqtTY2KioqCiv9aioKFVUVDR7zLZt2/TMM89o/fr1rb6f7OxshYeHe24ul+sHzQ0AAOAPARGAbXXy5ElNnz5d69evV0RERKuPW7Rokaqrqz23I0eOdOCUAAAAHaOTvwdoDxEREQoJCVFlZaXXemVlpXr16tVk/6effqqDBw8qNTXVs+Z2uyVJnTp10t69ezVgwIAmxzmdTjmdznaeHgAAwLcC4gqgw+FQQkKCiouLPWtut1vFxcVKSkpqsn/w4MH66KOPVFZW5rndeOONGj9+vMrKynhqFwAABLSAuAIoSZmZmcrIyNCIESM0cuRIrVq1SrW1tZoxY4YkKT09XTExMcrOzlZoaKiGDh3qdXz37t0lqck6AABAoAmYAExLS9OxY8e0ePFiVVRUKD4+XoWFhZ43hhw+fFjBwQFxwRMAAOAHCTLGGH8PcbGqqalReHi4qqurFRYW5u9xAABAK/D7O0BeAwgAAIDWIwABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUCKgBzcnIUGxur0NBQJSYmaseOHS3uXb9+vcaOHasePXqoR48eSk5OPud+AACAQBEwAZiXl6fMzExlZWVp586diouLU0pKio4ePdrs/q1bt2rq1Kl69913VVJSIpfLpeuuu06fffaZjycHAADwrSBjjPH3EO0hMTFRV199tdasWSNJcrvdcrlcmjdvnhYuXPi9xzc2NqpHjx5as2aN0tPTW3WfNTU1Cg8PV3V1tcLCwn7Q/AAAwDf4/R0gVwDr6+tVWlqq5ORkz1pwcLCSk5NVUlLSqp9x+vRpnT17VpdddlmLe+rq6lRTU+N1AwAAuNgERABWVVWpsbFRUVFRXutRUVGqqKho1c+47777FB0d7RWR35Wdna3w8HDPzeVy/aC5AQAA/CEgAvCHWrZsmTZt2qTXXntNoaGhLe5btGiRqqurPbcjR474cEoAAID20cnfA7SHiIgIhYSEqLKy0mu9srJSvXr1OuexTzzxhJYtW6a3335bw4cPP+dep9Mpp9P5g+cFAADwp4C4AuhwOJSQkKDi4mLPmtvtVnFxsZKSklo8bvny5XrkkUdUWFioESNG+GJUAAAAvwuIK4CSlJmZqYyMDI0YMUIjR47UqlWrVFtbqxkzZkiS0tPTFRMTo+zsbEnS448/rsWLF2vjxo2KjY31vFawa9eu6tq1q98eBwAAQEcLmABMS0vTsWPHtHjxYlVUVCg+Pl6FhYWeN4YcPnxYwcH/vOC5bt061dfX6+abb/b6OVlZWVqyZIkvRwcAAPCpgPkcQH/gc4QAALj48Ps7QF4DCAAAgNYjAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQIqAHNychQbG6vQ0FAlJiZqx44d59z/yiuvaPDgwQoNDdWwYcNUUFDgo0kBAAD8J2ACMC8vT5mZmcrKytLOnTsVFxenlJQUHT16tNn977//vqZOnaqZM2dq165dmjx5siZPnqyPP/7Yx5MDAAD4VpAxxvh7iPaQmJioq6++WmvWrJEkud1uuVwuzZs3TwsXLmyyPy0tTbW1tXrjjTc8a6NGjVJ8fLxyc3NbdZ81NTUKDw9XdXW1wsLC2ueBAACADsXvb6mTvwdoD/X19SotLdWiRYs8a8HBwUpOTlZJSUmzx5SUlCgzM9NrLSUlRZs3b27xfurq6lRXV+f5urq6WtI3/0MCAAAXh29/bwfINbDzEhABWFVVpcbGRkVFRXmtR0VFac+ePc0eU1FR0ez+ioqKFu8nOztbDz/8cJN1l8t1HlMDAAB/+vLLLxUeHu7vMfwiIALQVxYtWuR11fDEiRPq16+fDh8+bO3/gC4UNTU1crlcOnLkiLWX8y8UnIsLC+fjwsG5uHBUV1erb9++uuyyy/w9it8ERABGREQoJCRElZWVXuuVlZXq1atXs8f06tWrTfslyel0yul0NlkPDw/nL/MFIiwsjHNxgeBcXFg4HxcOzsWFIzg4YN4L22YB8cgdDocSEhJUXFzsWXO73SouLlZSUlKzxyQlJXntl6SioqIW9wMAAASKgLgCKEmZmZnKyMjQiBEjNHLkSK1atUq1tbWaMWOGJCk9PV0xMTHKzs6WJM2fP1/jxo3TypUrNWnSJG3atEkffvihnn76aX8+DAAAgA4XMAGYlpamY8eOafHixaqoqFB8fLwKCws9b/Q4fPiw16Xe0aNHa+PGjXrwwQd1//3368c//rE2b96soUOHtvo+nU6nsrKymn1aGL7FubhwcC4uLJyPCwfn4sLBuQigzwEEAABA6wTEawABAADQegQgAACAZQhAAAAAyxCAAAAAliEAv0dOTo5iY2MVGhqqxMRE7dix45z7X3nlFQ0ePFihoaEaNmyYCgoKfDRp4GvLuVi/fr3Gjh2rHj16qEePHkpOTv7ec4fWa+vfi29t2rRJQUFBmjx5cscOaJG2nosTJ05o7ty56t27t5xOpwYOHMh/p9pRW8/HqlWrNGjQIHXp0kUul0sLFizQmTNnfDRt4HrvvfeUmpqq6OhoBQUFafPmzd97zNatW/XTn/5UTqdTP/rRj/Tcc891+Jx+ZdCiTZs2GYfDYTZs2GD+9re/mTvuuMN0797dVFZWNrt/+/btJiQkxCxfvtx88skn5sEHHzSdO3c2H330kY8nDzxtPRfTpk0zOTk5ZteuXWb37t3mtttuM+Hh4eYf//iHjycPPG09F986cOCAiYmJMWPHjjW/+MUvfDNsgGvruairqzMjRowwEydONNu2bTMHDhwwW7duNWVlZT6ePDC19Xy8+OKLxul0mhdffNEcOHDAvPXWW6Z3795mwYIFPp488BQUFJgHHnjAvPrqq0aSee211865v7y83FxyySUmMzPTfPLJJ2b16tUmJCTEFBYW+mZgPyAAz2HkyJFm7ty5nq8bGxtNdHS0yc7Obnb/lClTzKRJk7zWEhMTzaxZszp0Thu09Vx8V0NDg+nWrZt5/vnnO2pEa5zPuWhoaDCjR482f/jDH0xGRgYB2E7aei7WrVtn+vfvb+rr6301olXaej7mzp1rfv7zn3utZWZmmjFjxnTonLZpTQD+5je/MUOGDPFaS0tLMykpKR04mX/xFHAL6uvrVVpaquTkZM9acHCwkpOTVVJS0uwxJSUlXvslKSUlpcX9aJ3zORffdfr0aZ09e9bqf/i7PZzvufjtb3+ryMhIzZw50xdjWuF8zsXrr7+upKQkzZ07V1FRURo6dKiWLl2qxsZGX40dsM7nfIwePVqlpaWep4nLy8tVUFCgiRMn+mRm/JONv78D5l8CaW9VVVVqbGz0/Esi34qKitKePXuaPaaioqLZ/RUVFR02pw3O51x813333afo6Ogmf8HRNudzLrZt26ZnnnlGZWVlPpjQHudzLsrLy/XOO+/o1ltvVUFBgfbv3685c+bo7NmzysrK8sXYAet8zse0adNUVVWla665RsYYNTQ0aPbs2br//vt9MTL+RUu/v2tqavT111+rS5cufpqs43AFEAFv2bJl2rRpk1577TWFhob6exyrnDx5UtOnT9f69esVERHh73Gs53a7FRkZqaeffloJCQlKS0vTAw88oNzcXH+PZqWtW7dq6dKlWrt2rXbu3KlXX31VW7Zs0SOPPOLv0WABrgC2ICIiQiEhIaqsrPRar6ysVK9evZo9plevXm3aj9Y5n3PxrSeeeELLli3T22+/reHDh3fkmFZo67n49NNPdfDgQaWmpnrW3G63JKlTp07au3evBgwY0LFDB6jz+XvRu3dvde7cWSEhIZ61K6+8UhUVFaqvr5fD4ejQmQPZ+ZyPhx56SNOnT9ftt98uSRo2bJhqa2t155136oEHHvD69+vRsVr6/R0WFhaQV/8krgC2yOFwKCEhQcXFxZ41t9ut4uJiJSUlNXtMUlKS135JKioqanE/Wud8zoUkLV++XI888ogKCws1YsQIX4wa8Np6LgYPHqyPPvpIZWVlntuNN96o8ePHq6ysTC6Xy5fjB5Tz+XsxZswY7d+/3xPhkrRv3z717t2b+PuBzud8nD59uknkfRvnxpiOGxZNWPn729/vQrmQbdq0yTidTvPcc8+ZTz75xNx5552me/fupqKiwhhjzPTp083ChQs9+7dv3246depknnjiCbN7926TlZXFx8C0k7aei2XLlhmHw2Hy8/PNF1984bmdPHnSXw8hYLT1XHwX7wJuP209F4cPHzbdunUzv/rVr8zevXvNG2+8YSIjI82jjz7qr4cQUNp6PrKysky3bt3MSy+9ZMrLy83//u//mgEDBpgpU6b46yEEjJMnT5pdu3aZXbt2GUnmySefNLt27TKHDh0yxhizcOFCM336dM/+bz8G5t577zW7d+82OTk5fAyM7VavXm369u1rHA6HGTlypPnggw883xs3bpzJyMjw2v/yyy+bgQMHGofDYYYMGWK2bNni44kDV1vORb9+/YykJresrCzfDx6A2vr34l8RgO2rrefi/fffN4mJicbpdJr+/fubxx57zDQ0NPh46sDVlvNx9uxZs2TJEjNgwAATGhpqXC6XmTNnjvnqq698P3iAeffdd5v9HfDtn39GRoYZN25ck2Pi4+ONw+Ew/fv3N88++6zP5/alIGO4zgwAAGATXgMIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWIYABAAAsAwBCAAAYBkCEAAAwDIEIAAAgGUIQAAAAMsQgAAAAJYhAAEAACxDAAIAAFiGAAQAALAMAQgAAGAZAhAAAMAyBCAAAIBlCEAAAADLEIAAAACWIQABAAAsQwACAABYhgAEAACwDAEIAABgGQIQAADAMgQgAACAZQhAAAAAyxCAAAAAliEAAQAALEMAAgAAWOb/ATyhabrGscARAAAAAElFTkSuQmCC' width=640.0/>\n", - " </div>\n", - " " - ], - "text/plain": [ - "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# This has to be in a separate cell for some weird event loop reasons\n", - "%matplotlib widget\n", - "fig,ax = plt.subplots()\n", - "fig.suptitle('MSE vs Epochs')\n", - "plt.show()\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 139, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -471,7 +510,580 @@ "name": "stderr", "output_type": "stream", "text": [ - " 0%| | 0/1024 [00:00<?, ?it/s]" + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.01it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/1000 \t Train Err: 0.5381 \t Test Err: 0.1865\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.75it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 2/1000 \t Train Err: 0.1227 \t Test Err: 0.1128\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.03it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 3/1000 \t Train Err: 0.1071 \t Test Err: 0.1118\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.01it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 4/1000 \t Train Err: 0.1008 \t Test Err: 0.1035\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.75it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 5/1000 \t Train Err: 0.0972 \t Test Err: 0.1021\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.03it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 6/1000 \t Train Err: 0.0949 \t Test Err: 0.0981\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.76it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 7/1000 \t Train Err: 0.0929 \t Test Err: 0.1021\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.00it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 8/1000 \t Train Err: 0.0908 \t Test Err: 0.0977\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.01it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 9/1000 \t Train Err: 0.0886 \t Test Err: 0.0952\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.72it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 10/1000 \t Train Err: 0.0910 \t Test Err: 0.0962\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.02it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 11/1000 \t Train Err: 0.0851 \t Test Err: 0.0898\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.03it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 12/1000 \t Train Err: 0.0849 \t Test Err: 0.0864\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.74it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 13/1000 \t Train Err: 0.0795 \t Test Err: 0.0684\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.02it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 14/1000 \t Train Err: 0.0691 \t Test Err: 0.0293\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.74it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 15/1000 \t Train Err: 0.0455 \t Test Err: 0.0271\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 6.99it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 16/1000 \t Train Err: 0.0421 \t Test Err: 0.0210\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 6.99it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 17/1000 \t Train Err: 0.0817 \t Test Err: 0.0505\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.75it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 18/1000 \t Train Err: 0.0456 \t Test Err: 0.0176\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.03it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 19/1000 \t Train Err: 0.0370 \t Test Err: 0.0165\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.03it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 20/1000 \t Train Err: 0.0374 \t Test Err: 0.0205\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.73it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 21/1000 \t Train Err: 0.0372 \t Test Err: 0.0142\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.01it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 22/1000 \t Train Err: 0.0343 \t Test Err: 0.0132\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.75it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 23/1000 \t Train Err: 0.0337 \t Test Err: 0.0119\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.03it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 24/1000 \t Train Err: 0.0713 \t Test Err: 0.0259\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.04it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 25/1000 \t Train Err: 0.0522 \t Test Err: 0.0143\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.75it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 26/1000 \t Train Err: 0.0342 \t Test Err: 0.0117\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.02it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 27/1000 \t Train Err: 0.0864 \t Test Err: 0.0728\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.04it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 28/1000 \t Train Err: 0.0701 \t Test Err: 0.0510\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.72it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 29/1000 \t Train Err: 0.0598 \t Test Err: 0.0369\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 6.99it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 30/1000 \t Train Err: 0.0462 \t Test Err: 0.0231\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.73it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 31/1000 \t Train Err: 0.0387 \t Test Err: 0.0181\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.02it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 32/1000 \t Train Err: 0.0351 \t Test Err: 0.0142\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.04it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 33/1000 \t Train Err: 0.0337 \t Test Err: 0.0123\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.74it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 34/1000 \t Train Err: 0.0331 \t Test Err: 0.0117\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.02it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 35/1000 \t Train Err: 0.0329 \t Test Err: 0.0119\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 6.98it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 36/1000 \t Train Err: 0.0327 \t Test Err: 0.0109\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.74it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 37/1000 \t Train Err: 0.0319 \t Test Err: 0.0101\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.02it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 38/1000 \t Train Err: 0.0317 \t Test Err: 0.0099\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.71it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 39/1000 \t Train Err: 0.0313 \t Test Err: 0.0095\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.01it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 40/1000 \t Train Err: 0.0310 \t Test Err: 0.0097\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 28%|██████████████████████████████████████████▊ | 9/32 [00:01<00:03, 6.50it/s]\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[139], line 11\u001b[0m\n\u001b[1;32m 9\u001b[0m train_loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m tqdm(\u001b[38;5;28mrange\u001b[39m(BPE)):\n\u001b[0;32m---> 11\u001b[0m batch_src, batch_labels, batch_padding_mask \u001b[38;5;241m=\u001b[39m \u001b[43mmkbatch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mBSZ\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 12\u001b[0m \u001b[38;5;66;03m# for batch_src, batch_labels, batch_padding_mask in tqdm(train_loader):\u001b[39;00m\n\u001b[1;32m 13\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n", + "Cell \u001b[0;32mIn[76], line 55\u001b[0m, in \u001b[0;36mmkbatch\u001b[0;34m(size)\u001b[0m\n\u001b[1;32m 53\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(size):\n\u001b[1;32m 54\u001b[0m n \u001b[38;5;241m=\u001b[39m random\u001b[38;5;241m.\u001b[39mrandint(MIN_VTXS, MAX_VTXS)\n\u001b[0;32m---> 55\u001b[0m edge_list, adj_list \u001b[38;5;241m=\u001b[39m \u001b[43mrandom_graph\u001b[49m\u001b[43m(\u001b[49m\u001b[43mn\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 56\u001b[0m dist \u001b[38;5;241m=\u001b[39m SSSP(n, adj_list)\n\u001b[1;32m 57\u001b[0m edge_list[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m2\u001b[39m \u001b[38;5;66;03m# target token\u001b[39;00m\n", + "Cell \u001b[0;32mIn[76], line 15\u001b[0m, in \u001b[0;36mrandom_graph\u001b[0;34m(n)\u001b[0m\n\u001b[1;32m 13\u001b[0m adjacencies \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mset\u001b[39m() \u001b[38;5;28;01mfor\u001b[39;00m _ \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(n\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m1\u001b[39m)]\n\u001b[1;32m 14\u001b[0m indices \u001b[38;5;241m=\u001b[39m [random\u001b[38;5;241m.\u001b[39mrandint(\u001b[38;5;241m1\u001b[39m, n) \u001b[38;5;28;01mfor\u001b[39;00m _ \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(AVG_DEG \u001b[38;5;241m*\u001b[39m (n\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m))]\n\u001b[0;32m---> 15\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28;43mrange\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mindices\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m)\u001b[49m:\n\u001b[1;32m 16\u001b[0m u \u001b[38;5;241m=\u001b[39m indices[i]\n\u001b[1;32m 17\u001b[0m v \u001b[38;5;241m=\u001b[39m indices[i \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m1\u001b[39m]\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], @@ -487,6 +1099,7 @@ " train_loss = 0\n", " for i in tqdm(range(BPE)):\n", " batch_src, batch_labels, batch_padding_mask = mkbatch(BSZ)\n", + " # for batch_src, batch_labels, batch_padding_mask in tqdm(train_loader):\n", " optimizer.zero_grad()\n", " output = model(batch_src, batch_padding_mask)\n", " loss = criterion(output.squeeze(1), batch_labels)\n", @@ -501,28 +1114,84 @@ " with open('loss', 'a') as f:\n", " f.write(f\"{train_loss} {test_loss}\\n\")\n", " print(f\"Epoch {epoch + 1}/{NEPOCHS} \\t Train Err: {train_loss:.4f} \\t Test Err: {test_loss:.4f}\")\n", - " ax.plot(train_err, label='Train', color='blue')\n", - " ax.plot(test_err, label='Test', color='red')\n", - " ax.set_xlabel('Epochs')\n", - " ax.set_ylabel('MSE')\n", - " fig.canvas.draw()\n", - "\n", - " if epoch % 10 == 9:\n", + " \n", + " if epoch % 100 == 99:\n", " torch.save(model.state_dict(), f\"model_weights_{epoch}.pth\")" ] }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 163, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(torch.Size([1, 7, 64]), torch.Size([64, 64]))" + ] + }, + "execution_count": 163, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "Now let's figure out what it's doing. \n", + "\n", + "step 1: figure out what people are attending to \n", + "\"\"\"\n", + "\n", + "example_graph, answer, padding = mkbatch(1)\n", + "sentance_embeddings = model.full_embedding(example_graph)\n", + "Q,K,V = torch.split(model.transformer_encoder.layers[0].self_attn.in_proj_weight, (MODEL_DIM, MODEL_DIM, MODEL_DIM))\n", + "\n", + "sentance_embeddings.shape, Q.shape\n", + "sentance_embeddings@Q.T\n", + "\n", + "# (sentance_embeddings @ Q).shape\n", + "# sentance_embeddings.shape\n", + "# K @ sentance_embeddings" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "<matplotlib.legend.Legend at 0x702d2d2eed20>" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "plt.suptitle('MSE vs Epochs')\n", + "plt.plot(train_err, label='Train', color='blue')\n", + "plt.plot(test_err, label='Test', color='red')\n", + "plt.xlabel('Epochs')\n", + "plt.ylabel('MSE')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 35, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "177.23538208007812" + "24.625" ] }, - "execution_count": 57, + "execution_count": 35, "metadata": {}, "output_type": "execute_result" } |