diff options
| author | SIPB | 2024-12-03 14:46:38 -0500 |
|---|---|---|
| committer | SIPB | 2024-12-03 14:46:38 -0500 |
| commit | 7462968826ca42383491e7441b495ef8d6eaf8b7 (patch) | |
| tree | 634660aef605e3829c5fa4bf7b61bb1b756a6eee /insane-shortest-paths.ipynb | |
| parent | a24288e28c4b53fdd6467ed4eed626fa0586bf72 (diff) | |
Latest blog post and graphs
Diffstat (limited to 'insane-shortest-paths.ipynb')
| -rw-r--r-- | insane-shortest-paths.ipynb | 452 |
1 files changed, 279 insertions, 173 deletions
diff --git a/insane-shortest-paths.ipynb b/insane-shortest-paths.ipynb index 72846c2..e74974b 100644 --- a/insane-shortest-paths.ipynb +++ b/insane-shortest-paths.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 11, "execution_state": "idle", "id": "86ce5f44-94f6-43b0-a0d1-091b8134ffb6", "metadata": {}, @@ -11,231 +11,337 @@ "name": "stdout", "output_type": "stream", "text": [ - "[set(), set(), {5, 6}, {4}, {3}, {2, 6}, {2, 5}]\n", - "[set(), {6}, set(), {4, 5, 6}, {3}, {3, 6}, {1, 3, 5}]\n", - "[set(), {4}, set(), {4, 5}, {1, 3}, {3, 6}, {5}]\n", - "[set(), {2, 6}, {1, 6}, {6}, set(), set(), {1, 2, 3}]\n", - "[set(), {3}, {3}, {1, 2, 5, 6}, {5}, {3, 4}, {3}]\n", - "[set(), {3, 6}, {4}, {1}, {2}, {6}, {1, 5}]\n", - "[set(), {2, 3}, {1, 3, 6}, {1, 2, 4}, {3}, set(), {2}]\n", - "[set(), {4}, set(), {4}, {1, 3, 5, 6}, {4}, {4}]\n", - "[set(), {3, 4, 5}, {6}, {1}, {1, 6}, {1}, {2, 4}]\n", - "[set(), {5, 6}, {6}, {6}, {5, 6}, {1, 4}, {1, 2, 3, 4}]\n" + "Total number of parameters: 44352\n" ] } ], "source": [ - "# -*- coding: utf-8 -*-\n", - "\"\"\"how-tsp-should-be.ipynb\n", - "\n", - "Automatically generated by Colab.\n", - "\n", - "Original file is located at\n", - " https://colab.research.google.com/drive/1InE1iW8ARzndPpvqH_9y22s81sOiHxPs\n", - "\"\"\"\n", - "\n", - "from tqdm import tqdm\n", "import torch\n", "import torch.nn as nn\n", - "import matplotlib as mpl\n", - "import matplotlib.pyplot as plt\n", - "from torch.utils.data import DataLoader, TensorDataset\n", - "\n", - "from math import sqrt\n", - "from collections import deque\n", - "import os\n", "import random\n", - "import pickle\n", - "import ipdb\n", + "from collections import deque\n", "\n", - "# torch.manual_seed(30)\n", - "# random.seed(30)\n", + "# Set manual seeds for reproducibility\n", "torch.manual_seed(33)\n", "random.seed(33)\n", "\n", - "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", - "# assert device.type == \"cuda\", \"CUDA is not available. Please check your GPU setup.\"\n", - "\n", - "NVTXS = 6\n", - "MAXDIST = NVTXS+1\n", + "# Configuration\n", + "NVTXS = 16\n", + "MAXDIST = NVTXS + 1\n", "AVGDEG = 2\n", "SEQLEN = NVTXS + 1\n", - "HIDDENDIM = 4*NVTXS+2\n", + "HIDDENDIM = 4 * NVTXS + 2\n", "\n", - "# 0: ANSFLAG\n", - "# 1:NVTXS+1 NBRS\n", - "# NVTXS+1: 2*NVTXS+1 REACH\n", - "# 2*NVTXS+1: 3*NVTXS+1 SELF\n", - "# -1 NOTANSFLAG\n", - "\n", - "START_REACH = NVTXS+1\n", - "START_OUT = 2*NVTXS+1\n", - "START_SELF = 3*NVTXS+1\n", + "# Start indices for different sections of the input data\n", + "START_REACH = NVTXS + 1\n", + "START_OUT = 2 * NVTXS + 1\n", + "START_SELF = 3 * NVTXS + 1\n", "SRC_FLAG_IDX = START_SELF\n", - "SOURCE = 1\n", - "TARGET = 2\n", "ANS_FLAG_IDX = 0\n", "NOTANS_FLAG_IDX = -1\n", "\n", - "def print_everything(data):\n", - " print(\"NBRS\")\n", - " print(data[0, 1:, 1:1+NVTXS])\n", - " print(\"REACH\")\n", - " print(data[0, 1:, START_REACH:START_REACH+NVTXS])\n", - " print(\"ANSFLAG\")\n", - " print(data[0, :, 0])\n", - " print(\"MORE FLAGS\")\n", - " print(data[0, :, -1])\n", - " print(\"SELF\")\n", - " print(data[0, 1:, START_SELF:START_SELF+NVTXS])\n", - " print(\"OUT\")\n", - " print(data[0, 0, START_OUT:START_OUT+NVTXS])\n", - "\n", - "\n", - "def random_graph():\n", - " data = torch.zeros((SEQLEN, HIDDENDIM))\n", + "# Determine device\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "\n", - " for i in range(1,NVTXS+1):\n", - " data[i, START_SELF-1+i] = 1\n", + "def random_graph(device):\n", + " \"\"\"Generate a random graph tensor.\"\"\"\n", + " data = torch.zeros((SEQLEN, HIDDENDIM), device=device)\n", + " \n", + " # Mark self vertices\n", + " for i in range(1, NVTXS + 1):\n", + " data[i, START_SELF - 1 + i] = 1\n", "\n", + " # Create adjacency list\n", " adj_list = [set() for _ in range(SEQLEN)]\n", " indices = [random.randint(1, NVTXS) for _ in range(AVGDEG * NVTXS)]\n", + " \n", " for i in range(0, len(indices), 2):\n", " u = indices[i]\n", " v = indices[i + 1]\n", " if u != v:\n", - " data[v,u] = 1\n", - " data[u,v] = 1\n", - " data[v,NVTXS+u] = 1\n", - " data[u,NVTXS+v] = 1\n", + " # Bidirectional connections\n", + " data[v, u] = 1\n", + " data[u, v] = 1\n", + " data[v, NVTXS + u] = 1\n", + " data[u, NVTXS + v] = 1\n", " adj_list[u].add(v)\n", " adj_list[v].add(u)\n", "\n", + " # Set flags\n", " data[0, ANS_FLAG_IDX] = 1\n", " data[1:, NOTANS_FLAG_IDX] = 1\n", - "\n", - " # TODO: this is kind of a hack\n", - " data[0, START_REACH:START_REACH+NVTXS] = 1\n", + " data[0, START_REACH:START_REACH + NVTXS] = 1\n", " return data, adj_list\n", "\n", - "\"\"\"\n", - "input: G, represented as an adjacency list\n", - "output: distance from SOURCE to TARGET\n", - "\"\"\"\n", "def SSSP(G):\n", + " \"\"\"Single Source Shortest Path algorithm.\"\"\"\n", " dist = [MAXDIST for _ in G]\n", - " dist[SOURCE] = 0\n", - " frontier = deque()\n", - " frontier.append(SOURCE)\n", - " while len(frontier) > 0:\n", + " dist[1] = 0\n", + " frontier = deque([1])\n", + " while frontier:\n", " vtx = frontier.popleft()\n", " for x in G[vtx]:\n", " if dist[x] == MAXDIST:\n", " dist[x] = 1 + dist[vtx]\n", " frontier.append(x)\n", - " if x == TARGET:\n", - " return dist[TARGET]\n", + " if x == 2:\n", + " return dist[2]\n", " return MAXDIST\n", "\n", "def mkbatch(size):\n", - " graphs1 = []\n", - " distance1 = []\n", + " \"\"\"Create a batch of graph data.\"\"\"\n", + " graphs = []\n", + " distances = []\n", "\n", - " for i in range(size):\n", - " data, adj_list = random_graph()\n", + " for _ in range(size):\n", + " data, adj_list = random_graph(device)\n", " dist = SSSP(adj_list)\n", - " graphs1.append(data)\n", - " distance1.append(dist)\n", - "\n", - " print(adj_list)\n", + " graphs.append(data)\n", + " distances.append(dist)\n", "\n", - " data = torch.stack(graphs1)\n", - " labels = torch.tensor(distance1, dtype=torch.float16)\n", + " data = torch.stack(graphs)\n", + " labels = torch.tensor(distances, dtype=torch.float32, device=device)\n", " return data, labels\n", "\n", - "\"\"\"\n", - "TODO: WRAP EVERYTHING in nn.Parameter(torch.zeros((1, HIDDENDIM)))\n", - "and then do my perturbing parameters experiment\n", - "\n", - "TODO:\n", - " USE activation magic to bring everything back to the 0/1 realm instead of possibly being 0/2 valued\n", - "\"\"\"\n", + "BIG = 20\n", + "SUPABIG = 100\n", + "MED = 10\n", + "CURSE = 5\n", "\n", "class SillyTransformer(nn.Module):\n", - " def __init__(self):\n", + " def __init__(self, device):\n", " super().__init__()\n", - " self.most_KQVs = []\n", - " for head in range(1,NVTXS+1):\n", - " Q = torch.zeros((2, HIDDENDIM))\n", - " Q[0, START_REACH-1+head] = 1000\n", - " Q[1, NOTANS_FLAG_IDX] = 1\n", - "\n", - " K = torch.zeros((2, HIDDENDIM))\n", - " K[0, head] = 1\n", - " K[1, ANS_FLAG_IDX] = 200\n", - "\n", - " V = torch.zeros((NVTXS,HIDDENDIM))\n", - " for i in range(NVTXS):\n", - " V[i, START_SELF+i] = 1\n", - "\n", - " self.most_KQVs.append((K, Q, V))\n", - "\n", - " self.weird_KQVs = []\n", - " for layer in range(NVTXS):\n", - " K = torch.zeros((3, HIDDENDIM))\n", - " K[0, NOTANS_FLAG_IDX] = -1000\n", - " K[0, SRC_FLAG_IDX] = +1100\n", - " K[1, NOTANS_FLAG_IDX] = -1000\n", - " K[1, NVTXS+TARGET] = +1100\n", - " K[1, ANS_FLAG_IDX] = -1100\n", - " K[2, ANS_FLAG_IDX] = 10\n", - "\n", - " Q = torch.zeros((3, HIDDENDIM))\n", - " Q[:, ANS_FLAG_IDX] = 1\n", - "\n", - " V = torch.zeros((NVTXS, HIDDENDIM))\n", - " V[layer, SRC_FLAG_IDX] = 1\n", - "\n", - " self.weird_KQVs.append((K, Q, V))\n", + " self.device = device\n", + "\n", + " with torch.no_grad():\n", + " # Initialize weight parameters with specific configurations\n", + " self.mostKs = nn.ParameterList()\n", + " self.mostQs = nn.ParameterList()\n", + " self.mostVs = nn.ParameterList()\n", + " for head in range(1, NVTXS + 1):\n", + " Q = nn.Parameter(torch.zeros((2, HIDDENDIM), device=device))\n", + " Q[0, START_REACH - 1 + head] = SUPABIG\n", + " Q[1, NOTANS_FLAG_IDX] = 1\n", + "btrfs filesystem resize max\n", + " K = nn.Parameter(torch.zeros((2, HIDDENDIM), device=device))\n", + " K[0, head] = 1\n", + " K[1, ANS_FLAG_IDX] = BIG\n", + "\n", + " V = nn.Parameter(torch.zeros((NVTXS, HIDDENDIM), device=device))\n", + " for i in range(NVTXS):\n", + " V[i, START_SELF + i] = 1\n", + "\n", + " self.mostKs.append(K)\n", + " self.mostQs.append(Q)\n", + " self.mostVs.append(V)\n", + "\n", + " self.weirdKs = nn.ParameterList()\n", + " self.weirdQs = nn.ParameterList()\n", + " self.weirdVs = nn.ParameterList()\n", + " for layer in range(NVTXS):\n", + " K = nn.Parameter(torch.zeros((3, HIDDENDIM), device=device))\n", + " K[0, NOTANS_FLAG_IDX] = -BIG\n", + " K[0, SRC_FLAG_IDX] = BIG+SUPABIG\n", + " K[1, NOTANS_FLAG_IDX] = -SUPABIG\n", + " K[1, NVTXS + 2] = BIG+SUPABIG\n", + " K[1, ANS_FLAG_IDX] = -BIG-SUPABIG\n", + " K[2, ANS_FLAG_IDX] = MED\n", + "\n", + " Q = nn.Parameter(torch.zeros((3, HIDDENDIM), device=device))\n", + " Q[:, ANS_FLAG_IDX] = 1\n", + "\n", + " V = nn.Parameter(torch.zeros((NVTXS, HIDDENDIM), device=device))\n", + " V[layer, SRC_FLAG_IDX] = 1\n", + "\n", + " self.weirdKs.append(K)\n", + " self.weirdQs.append(Q)\n", + " self.weirdVs.append(V)\n", "\n", " def forward(self, src):\n", - " for layer in range(NVTXS):\n", - " allKQVs = [self.weird_KQVs[layer]] + self.most_KQVs\n", - " head_outputs = []\n", - " for (K, Q, V) in allKQVs:\n", - " ksrc = torch.matmul(src, K.unsqueeze(0).transpose(-2, -1))\n", - " qsrc = torch.matmul(src, Q.unsqueeze(0).transpose(-2, -1))\n", - " vsrc = torch.matmul(src, V.unsqueeze(0).transpose(-2, -1))\n", - "\n", - " scores = torch.matmul(qsrc, ksrc.transpose(-2, -1))\n", - " attention_weights = torch.softmax(scores, dim=-1)\n", - " head_output = torch.matmul(attention_weights, vsrc)\n", - " head_outputs.append(head_output)\n", - "\n", - " new_reaches = sum(head_outputs[1:])\n", - " BSZ = new_reaches.shape[0]\n", - "\n", - " nodelta_nbrs = torch.zeros((BSZ, SEQLEN, NVTXS+1))\n", - " morepadlol = torch.zeros((BSZ, SEQLEN, 1+NVTXS))\n", - "\n", - " DIFF = torch.cat((nodelta_nbrs, new_reaches, head_outputs[0], morepadlol), dim=2)\n", - " src += torch.cat((nodelta_nbrs, new_reaches, head_outputs[0], morepadlol), dim=2)\n", - " src[:, :, START_REACH:START_REACH+NVTXS] = 2*torch.sigmoid(src[:,:, START_REACH:START_REACH+NVTXS]*1000)-1\n", - "\n", - " # print(\"SRC\")\n", - " # print_everything(src)\n", - "\n", - " canreach = src[:,0,START_OUT:START_OUT+NVTXS]\n", - " # __import__('ipdb').set_trace()\n", - " final_output = 1+torch.sum(1-canreach,dim=1)\n", - " return final_output\n", - "\n", - "model = SillyTransformer()\n", - "model.to(device)\n", - "\n", - "data, labels = mkbatch(10)\n", - "assert torch.all(model(data) == labels)\n", - "\n" + " for layer in range(NVTXS):\n", + " allKs = [self.weirdKs[layer]] + [x for x in self.mostKs]\n", + " allQs = [self.weirdQs[layer]] + [x for x in self.mostQs]\n", + " allVs = [self.weirdVs[layer]] + [x for x in self.mostVs]\n", + " head_outputs = []\n", + " \n", + " for (K, Q, V) in zip(allKs, allQs, allVs):\n", + " ksrc = torch.matmul(src, K.unsqueeze(0).transpose(-2, -1))\n", + " qsrc = torch.matmul(src, Q.unsqueeze(0).transpose(-2, -1))\n", + " vsrc = torch.matmul(src, V.unsqueeze(0).transpose(-2, -1))\n", + "\n", + " scores = torch.matmul(qsrc, ksrc.transpose(-2, -1))\n", + " attention_weights = torch.softmax(scores, dim=-1)\n", + " head_output = torch.matmul(attention_weights, vsrc)\n", + " head_outputs.append(head_output)\n", + "\n", + " new_reaches = sum(head_outputs[1:])\n", + " BSZ = new_reaches.shape[0]\n", + "\n", + " nodelta_nbrs = torch.zeros((BSZ, SEQLEN, NVTXS + 1), device=self.device)\n", + " morepadlol = torch.zeros((BSZ, SEQLEN, 1 + NVTXS), device=self.device)\n", + "\n", + " src = src + torch.cat((nodelta_nbrs, new_reaches, head_outputs[0], morepadlol), dim=2)\n", + " src[:, :, START_REACH:START_REACH + NVTXS] = 2 * torch.sigmoid(src[:, :, START_REACH:START_REACH + NVTXS] * CURSE) - 1\n", + "\n", + " canreach = src[:, 0, START_OUT:START_OUT + NVTXS]\n", + " final_output = 1 + torch.sum(1 - canreach, dim=1)\n", + " return final_output\n", + "\n", + "model = SillyTransformer(device).to(device)\n", + "params = sum(p.numel() for p in model.parameters())\n", + "print(f\"Total number of parameters: {params}\")\n", + "\n", + "def destroy_rand_weights(model):\n", + " weight_lists = [model.mostKs, model.mostQs, model.mostVs, \n", + " model.weirdKs, model.weirdQs, model.weirdVs]\n", + " random_list = random.choice(weight_lists)\n", + " random_matrix = random.choice(random_list)\n", + " random_matrix.data = torch.randn_like(random_matrix)\n", + "\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=3e-5)\n", + "loss_fn = nn.MSELoss()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "execution_state": "idle", + "id": "a9dd76f4-96f2-47b5-9bb9-a32a1b478dd4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch [0/10000], Loss: 8.3387\n", + "Epoch [10/10000], Loss: 7.6416\n", + "Epoch [20/10000], Loss: 11.2689\n", + "Epoch [30/10000], Loss: 7.0312\n", + "Epoch [40/10000], Loss: 8.7287\n", + "Epoch [50/10000], Loss: 7.7182\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[6], line 11\u001b[0m\n\u001b[1;32m 9\u001b[0m loss \u001b[38;5;241m=\u001b[39m loss_fn(outputs, labels)\n\u001b[1;32m 10\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[0;32m---> 11\u001b[0m \u001b[43mloss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 12\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m 13\u001b[0m train_err\u001b[38;5;241m.\u001b[39mappend(loss\u001b[38;5;241m.\u001b[39mitem())\n", + "File \u001b[0;32m~/.venv/lib64/python3.12/site-packages/torch/_tensor.py:581\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 571\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 572\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 573\u001b[0m Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m 574\u001b[0m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 579\u001b[0m inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m 580\u001b[0m )\n\u001b[0;32m--> 581\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mautograd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 582\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\n\u001b[1;32m 583\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/.venv/lib64/python3.12/site-packages/torch/autograd/__init__.py:347\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 342\u001b[0m retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m 344\u001b[0m \u001b[38;5;66;03m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[1;32m 345\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m 346\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 347\u001b[0m \u001b[43m_engine_run_backward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 348\u001b[0m \u001b[43m \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 349\u001b[0m \u001b[43m \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 350\u001b[0m \u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 351\u001b[0m \u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 352\u001b[0m \u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 353\u001b[0m \u001b[43m \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 354\u001b[0m \u001b[43m \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 355\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/.venv/lib64/python3.12/site-packages/torch/autograd/graph.py:825\u001b[0m, in \u001b[0;36m_engine_run_backward\u001b[0;34m(t_outputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m 823\u001b[0m unregister_hooks \u001b[38;5;241m=\u001b[39m _register_logging_hooks_on_whole_graph(t_outputs)\n\u001b[1;32m 824\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 825\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mVariable\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execution_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_backward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m 826\u001b[0m \u001b[43m \u001b[49m\u001b[43mt_outputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\n\u001b[1;32m 827\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# Calls into the C++ engine to run the backward pass\u001b[39;00m\n\u001b[1;32m 828\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 829\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m attach_logging_hooks:\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "# destroy_rand_weights(model)\n", + "num_epochs = 10000\n", + "batch_size = 1<<9\n", + "train_err = []\n", + "for epoch in range(num_epochs):\n", + " model.train()\n", + " data, labels = mkbatch(batch_size)\n", + " outputs = model(data)\n", + " loss = loss_fn(outputs, labels)\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + " train_err.append(loss.item())\n", + " if epoch % 10 == 0:\n", + " print(f\"Epoch [{epoch}/{num_epochs}], Loss: {loss.item():.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "execution_state": "idle", + "id": "dcbdebf6-5c9f-4491-a442-9271d2ba5696", + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'plt' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[3], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mplt\u001b[49m\u001b[38;5;241m.\u001b[39msuptitle(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mMSE vs Epochs\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 2\u001b[0m plt\u001b[38;5;241m.\u001b[39mplot(train_err, label\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mTrain\u001b[39m\u001b[38;5;124m'\u001b[39m, color\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mblue\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 3\u001b[0m plt\u001b[38;5;241m.\u001b[39mxlabel(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mEpochs\u001b[39m\u001b[38;5;124m'\u001b[39m)\n", + "\u001b[0;31mNameError\u001b[0m: name 'plt' is not defined" + ] + } + ], + "source": [ + "plt.suptitle('MSE vs Epochs')\n", + "plt.plot(train_err, label='Train', color='blue')\n", + "plt.xlabel('Epochs')\n", + "plt.ylabel('MSE')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "execution_state": "idle", + "id": "30893731-9991-4df9-b6c6-380010569ee1", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvsAAAJOCAYAAAAphsiIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB1LklEQVR4nO3de3zO9f/H8ec1s4Nl18xhh9pYyDmEtJKUZQ6J8JVaQqK+kVM5VY7JopJIpG+RoqNDqdAcCjnEkPOpnMJGxtbIzK7P74/s+nW1Ydd2fVzb5XG/3T63m+v9+Vzv6/X5tK3XXnt/Xh+LYRiGAAAAAHgcL3cHAAAAAMAcJPsAAACAhyLZBwAAADwUyT4AAADgoUj2AQAAAA9Fsg8AAAB4KJJ9AAAAwEOR7AMAAAAeimQfAAAA8FAk+0AejRw5UhaLRX/88YfL5uzatasqVKjgsvk8zcyZM2WxWHTw4EGXzVlUrvkPP/wgi8WiH374wT5W2GLPLUbkrmXLlurRo4e7wyiyTp06pYCAAH333XfuDgUockj2kS8WiyVPm7uTgCZNmqhmzZpujcFTnDx5Un379lXVqlXl7++vcuXK6fbbb9fgwYOVnp7u7vBcrkmTJg5fy8HBwWrQoIE++OAD2Ww2d4fnlLFjx2rBggXuDkM7duzQY489phtvvFG+vr4KDw9XXFycduzYUaB5r+X5rVmzRiNHjtSZM2fy/J6ffvpJ33//vQYPHmwfy/5F6csvv8z1PV27dtUNN9zgkpgvJz/n4i6lS5fWk08+qWHDhrk7FKDI8XZ3ACiaPvroI4fXs2bNUkJCQo7xatWqXePIYIaUlBTVr19faWlpeuKJJ1S1alWdOnVKW7du1dSpU/Xf//7X9MTEHW666SbFx8dLl37ZmTVrlrp37669e/fq1VdfvebxvPfee/n6RWPs2LHq0KGD2rZta0pceTFv3jw98sgjCg4OVvfu3RUVFaWDBw/q/fff15dffqlPP/1UDz30UL7mvpbnt2bNGo0aNUpdu3ZVUFBQnt7z2muvqWnTpqpUqZLp8TkjP+fiTk8//bQmTZqk5cuX67777nN3OECRQbKPfHnsscccXq9bt04JCQk5xv/t3LlzKlGihMnRwdXef/99HT58WD/99JPuvPNOh31paWny8fFxW2xmslqtDl/TTz31lKpUqaK3335bL7/8sooXL57jPTabTRcuXJCfn5/L48nt84qCX3/9VZ07d9bNN9+slStXqmzZsvZ9ffv21d13363OnTtr69atuvnmm90aq6udOHFC3377raZNm+buUIq8atWqqWbNmpo5cybJPuAElvHANNlLaBITE9W4cWOVKFFCL7zwgnRpGdDIkSNzvKdChQrq2rWrw9iZM2fUr18/RUREyNfXV5UqVdK4ceNctpRi69at6tq1q26++Wb5+fkpNDRUTzzxhE6dOpXr8X/88Yc6duyowMBAlS5dWn379tX58+dzHPfxxx+rXr168vf3V3BwsDp16qQjR45cNZ5PP/1U9erVU8mSJRUYGKhatWrprbfeuuzxmZmZCg4OVrdu3XLsS0tLk5+fn55//nn72OTJk1WjRg2VKFFCpUqVUv369TVnzpwrxvTrr7+qWLFiuuOOO3LsCwwMzJHYrl+/Xi1btlSpUqUUEBCgW2+91eEcnL3m/7Zo0SLdfffdCggIUMmSJdWqVatcl4IsWLBANWvWlJ+fn2rWrKn58+fnaf7LKVGihO644w6dPXtWJ0+elC59Lffu3VuzZ89WjRo15Ovrq8WLF0uSjh49qieeeEIhISHy9fVVjRo19MEHH+SY9/fff1fbtm0VEBCgcuXKqX///srIyMhxXG5r9m02m9566y3VqlVLfn5+Klu2rJo3b66NGzfa4zt79qw+/PBD+5Kkf36PuTrG3Lz22ms6d+6cpk+f7pDoS1KZMmX07rvv6uzZsxo/fvwVz1X/uHcm25XOL/vY3bt3X/F79uDBg7JYLJo5c2aOz/vnz6qRI0dq4MCBkqSoqCj7513pnpJvv/1WFy9eVExMTJ6u1dXk5Ws/L99fVzuX7K/rL774QtWrV5e/v7+io6O1bds2SdK7776rSpUqyc/PT02aNMlxDVatWqX//Oc/ioyMlK+vryIiItS/f3/99ddfDsdlL1f67bffFBsbq4CAAIWHh2v06NEyDCPH+d9///1auHBhrvsA5I7KPkx16tQptWjRQp06ddJjjz2mkJAQp95/7tw53XPPPTp69KieeuopRUZGas2aNRo6dKiOHz+uiRMnFjjGhIQE/fbbb+rWrZtCQ0O1Y8cOTZ8+XTt27NC6descEgtJ6tixoypUqKD4+HitW7dOkyZN0unTpzVr1iz7Ma+88oqGDRumjh076sknn9TJkyc1efJkNW7cWJs3b77sn8wTEhL0yCOPqGnTpho3bpwkadeuXfrpp5/Ut2/fXN9TvHhxPfTQQ5o3b57effddhyr7ggULlJGRoU6dOkmXloH06dNHHTp0sCc8W7du1fr16/Xoo49e9hqVL19eWVlZ+uijj9SlS5erXs8HHnhAYWFh6tu3r0JDQ7Vr1y5988039nNw9pr/U3YMsbGxGjdunM6dO6epU6eqUaNG2rx5sz1B/P7779W+fXtVr15d8fHxOnXqlLp166abbrrpivFfzW+//aZixYo5/Ddcvny5Pv/8c/Xu3VtlypRRhQoVlJycrDvuuMOeNJUtW1aLFi1S9+7dlZaWpn79+kmS/vrrLzVt2lSHDx9Wnz59FB4ero8++kjLly/PUzzdu3fXzJkz1aJFCz355JO6ePGiVq1apXXr1ql+/fr66KOP9OSTT+r2229Xz549JUkVK1aUpGsW48KFC1WhQgXdfffdue5v3LixKlSooG+//TZP8/3Tlc4vW16+Z/OiXbt22rt3rz755BO9+eabKlOmjCTl+AXmn9asWaPSpUurfPnyue7/888/c73pP7dfpPL6tZ+X76+8nMuqVav09ddfq1evXpKk+Ph4PfDAAxo0aJDeeecdPfPMMzp9+rTGjx+vJ554wuHr4YsvvtC5c+f03//+V6VLl9bPP/+syZMn6/fff9cXX3zhcF5ZWVlq3ry57rjjDo0fP16LFy/WiBEjdPHiRY0ePdrh2Hr16unNN9/Ujh07uB8LyCsDcIFevXoZ//5yuueeewxJxrRp03IcL8kYMWJEjvHy5csbXbp0sb9++eWXjYCAAGPv3r0Oxw0ZMsQoVqyYcfjw4SvGdc899xg1atS44jHnzp3LMfbJJ58YkoyVK1fax0aMGGFIMh588EGHY5955hlDkvHLL78YhmEYBw8eNIoVK2a88sorDsdt27bN8Pb2dhjv0qWLUb58efvrvn37GoGBgcbFixevGPO/LVmyxJBkLFy40GG8ZcuWxs0332x/3aZNm6tej9wkJSUZZcuWNSQZVatWNZ5++mljzpw5xpkzZxyOu3jxohEVFWWUL1/eOH36tMM+m81m/3der/mMGTMMScaBAwcMwzCMP//80wgKCjJ69OiRIz6r1eowXqdOHSMsLMwhxu+//96Q5HDNL+eee+4xqlatapw8edI4efKksWvXLqNPnz6GJKN169b24yQZXl5exo4dOxze3717dyMsLMz4448/HMY7depkWK1W+zWYOHGiIcn4/PPP7cecPXvWqFSpkiHJWLFihX38318vy5cvNyQZffr0yRH/P693QECAw/eVmTH+25kzZwxJRps2bS57jGEYxoMPPmhIMtLS0nI912zZ34f/dLnzy+v37IEDBwxJxowZM3LM8e+fVa+99prD1+TVNGrUyKhXr16O8RUrVhiSrrgFBATYj3fmaz+v319XOhdJhq+vr8O+d99915BkhIaG2v87GYZhDB06NMc8ucUQHx9vWCwW49ChQ/axLl26GJKMZ5991j5ms9mMVq1aGT4+PsbJkycd5lizZo0hyfjss89yzA8gdyzjgal8fX1zXV6SV1988YXuvvtulSpVSn/88Yd9i4mJUVZWllauXFngGP39/e3/Pn/+vP744w/7cpVNmzblOD67ypXt2WeflSR7S7h58+bJZrOpY8eODjGHhoaqcuXKWrFixWVjCQoK0tmzZ5WQkODUOdx3330qU6aMPvvsM/vY6dOnlZCQoIcffthh/t9//10bNmxwav6QkBD98ssvevrpp3X69GlNmzZNjz76qMqVK6eXX37Z/if1zZs368CBA+rXr1+Ov178s1rv7DXPlpCQoDNnzuiRRx5xuLbFihVTw4YN7df2+PHj2rJli7p06SKr1Wp///3336/q1avn+bx3796tsmXLqmzZsqpWrZomT56sVq1a5Vjmcs899zjMaxiG5s6dq9atW8swDIdYY2NjlZqaaj/P7777TmFhYerQoYP9/SVKlLBXqa9k7ty5slgsGjFiRI59V/rryLWM8c8//5QklSxZ8orHZe9PS0u76pzOutr3rJlOnTqlUqVKXXb/8OHDlZCQkGNr1qyZw3F5/dpXAb6//q1p06YOS6kaNmwoSWrfvr3Df8/s8d9++y3XGM6ePas//vhDd955pwzD0ObNm3N8Vu/eve3/zv5L04ULF7R06VKH47KvpStbIAOejmU8MNWNN95YoJs39+3bp61bt172z+QnTpwoQHR/S0lJ0ahRo/Tpp5/mmC81NTXH8ZUrV3Z4XbFiRXl5ednXrO7bt0+GYeQ4LtuVbrJ85pln9Pnnn6tFixa68cYb1axZM3Xs2FHNmze/4jl4e3urffv2mjNnjjIyMuTr66t58+YpMzPTIdkfPHiwli5dqttvv12VKlVSs2bN9Oijj+quu+664vySFBYWpqlTp+qdd97Rvn37tGTJEo0bN07Dhw9XWFiYnnzySf3666+SdNU/rzt7zbPt27dPuvTLTW4CAwMlSYcOHZJy+W8lSVWqVMlzwlOhQgW99957slgs8vPzU+XKlVWuXLkcx0VFRTm8PnnypM6cOaPp06dr+vTpuc6dfd6HDh1SpUqVciTnVapUuWp8v/76q8LDwxUcHJyn83FHjNlJYXbSfzl5/aUgP672PWu2K60vr1WrVq7r+T/++GOH13n92lcBvr/+LTIy0uF19i/OERERuY6fPn3aPnb48GENHz5cX3/9tcN4bjF4eXnluDH7lltukS7dT/FP2dfyar/MAvh/JPsw1T+rO3mRlZXl8Npms+n+++/XoEGDcj0++38IBdGxY0etWbNGAwcOVJ06dXTDDTfIZrOpefPmeboJ+N//07HZbLJYLFq0aJGKFSuW4/grtagsV66ctmzZoiVLlmjRokVatGiRZsyYoccff1wffvjhFePo1KmT3n33XS1atEht27bV559/rqpVq6p27dr2Y6pVq6Y9e/bom2++0eLFizV37ly98847Gj58uEaNGnXVc80+31tuuUW33HKLWrVqpcqVK2v27Nl68skn8/R+FeCaZ+/76KOPFBoammO/t7drf6QFBATk6cbKf3+dZ8f52GOPXfYeh1tvvdVFUebPtYrRarUqLCxMW7duveJxW7du1Y033mhPWi+XzP37Z0R+/HtuMz+rdOnSOZLd/HDma7+gP9Oy5fbz60rj2Yl4VlaW7r//fqWkpGjw4MGqWrWqAgICdPToUXXt2rVAzRWyr2X2PQYAro5kH25RqlSpHA9yuXDhgo4fP+4wVrFiRaWnp7usk8W/nT59WsuWLdOoUaM0fPhw+3h2FS03+/btc6jk7t+/Xzabzf7n7ooVK8owDEVFReXrlxEfHx+1bt1arVu3ls1m0zPPPKN3331Xw4YNu2Kf7saNGyssLEyfffaZGjVqpOXLl+vFF1/McVxAQIAefvhhPfzww7pw4YLatWunV155RUOHDnW6XeTNN9+sUqVK2f+7Zd8YuX379sv+N8vPNc+WPX+5cuWu+DWRfTNkbnPu2bPnqp9TUGXLllXJkiWVlZV11a/d8uXLa/v27TIMwyHpzEucFStW1JIlS5SSknLF6n5uyey1ilGSHnjgAb333ntavXq1GjVqlGP/qlWrdPDgQT311FP2sdx+Rugff7W52vn909W+Z7OXhvz78/LzWf9WtWpVzZ0716n35CavX/vOfH+ZVR3ftm2b9u7dqw8//FCPP/64ffxyyxNtNpt+++03h5+Xe/fulS79de2fDhw4IPEMF8AprNmHW1SsWDHHevvp06fnqKR17NhRa9eu1ZIlS3LMcebMGV28eLFAcWRXqP79Z/YrdfmZMmWKw+vJkydLklq0aCFd6thRrFgxjRo1Kse8hmFcsb3kv/d5eXnZq6tXa3Po5eWlDh06aOHChfroo4908eJFhyU8uc3v4+Oj6tWryzAMZWZmXnbu9evX6+zZsznGf/75Z506dcq+nOO2225TVFSUJk6cmCNxyr4W+bnm2WJjYxUYGKixY8fmGm92O8ywsDDVqVNHH374ocOSgYSEBO3cufOqn1NQxYoVU/v27TV37lxt3779snFKUsuWLXXs2DGHJ6lmt6m8mvbt28swjFz/KvPP6xsQEJDjv8e1ilGSBg4cKH9/fz311FM5vgZTUlL09NNPq0SJEvZWkLr0MyI1NdXhLwLHjx/PtX1qbuf3T1f7ng0MDFSZMmVy/Ex65513cv0s5fKLweVER0fr9OnTDuvZ8yOvX/vOfH85ey55lVsMhmFcsYXw22+/7XDs22+/reLFi6tp06YOxyUmJspqtapGjRoujRnwZFT24RZPPvmknn76abVv317333+/fvnlFy1ZsiTHn2YHDhyor7/+Wg888IC6du2qevXq6ezZs9q2bZu+/PJLHTx48Kp/zj158qTGjBmTYzwqKkpxcXFq3Lixxo8fr8zMTN144436/vvv7dWj3Bw4cEAPPvigmjdvrrVr1+rjjz/Wo48+al8uU7FiRY0ZM0ZDhw7VwYMH1bZtW5UsWVIHDhzQ/Pnz1bNnT4e+9/++LikpKbrvvvt000036dChQ5o8ebLq1KmTp0rWww8/rMmTJ2vEiBGqVatWjvc0a9ZMoaGhuuuuuxQSEqJdu3bp7bffVqtWra64Vvqjjz7S7Nmz9dBDD6levXry8fHRrl279MEHH8jPz8/+/AQvLy9NnTpVrVu3Vp06ddStWzeFhYVp9+7d2rFjh5YsWaLAwECnr3m2wMBATZ06VZ07d9Ztt92mTp06qWzZsjp8+LC+/fZb3XXXXfakIT4+Xq1atVKjRo30xBNPKCUlxf6MgfT09Kt+VkG9+uqrWrFihRo2bKgePXqoevXqSklJ0aZNm7R06VKlpKRIknr06KG3335bjz/+uBITExUWFqaPPvooTw+fu/fee9W5c2dNmjRJ+/btsy/TWLVqle699177TY/16tXT0qVLNWHCBIWHhysqKkoNGza8JjHq0pr5Dz/8UHFxcapVq1aOJ+j+8ccf+uSTTxxaZnbq1EmDBw/WQw89pD59+tjbTN5yyy057rm43Pllu9r3rC5977366qt68sknVb9+fa1cudJeXf73Z0nSiy++qE6dOql48eJq3bq1PXH+t1atWsnb21tLly7N0w3Nl5PXr31nvr+cPZe8qlq1qipWrKjnn39eR48eVWBgoObOnXvZ5Ux+fn5avHixunTpooYNG2rRokX69ttv9cILL+S4XyshIUGtW7dmzT7gDHe3A4JnuFzrzcu1eczKyjIGDx5slClTxihRooQRGxtr7N+/P0frTeNSy7mhQ4calSpVMnx8fIwyZcoYd955p/H6668bFy5cuGJc2e0/c9uaNm1qGIZh/P7778ZDDz1kBAUFGVar1fjPf/5jHDt2LEfLvew2fjt37jQ6dOhglCxZ0ihVqpTRu3dv46+//srx2XPnzjUaNWpkBAQEGAEBAUbVqlWNXr16GXv27LEf8+/2gl9++aXRrFkzo1y5coaPj48RGRlpPPXUU8bx48ev+t/AuNSyLiIiwpBkjBkzJsf+d99912jcuLFRunRpw9fX16hYsaIxcOBAIzU19Yrzbt261Rg4cKBx2223GcHBwYa3t7cRFhZm/Oc//zE2bdqU4/jVq1cb999/v1GyZEkjICDAuPXWW43Jkyfb9+f1mv+79Wa2FStWGLGxsYbVajX8/PyMihUrGl27djU2btzocNzcuXONatWqGb6+vkb16tWNefPmXbal47/lpW2rcalFYa9evXLdl5ycbPTq1cuIiIgwihcvboSGhhpNmzY1pk+f7nDcoUOHjAcffNAoUaKEUaZMGaNv377G4sWLr9p607jU7vS1114zqlatavj4+Bhly5Y1WrRoYSQmJtqP2b17t9G4cWPD39/fkOTwPebqGK9k69atxiOPPGKEhYXZP+uRRx4xtm3bluvx33//vVGzZk3Dx8fHqFKlivHxxx/n2nrzcufnzPfsuXPnjO7duxtWq9UoWbKk0bFjR+PEiRO5tgl++eWXjRtvvNHw8vLKUxvOBx980P7zJlt2680vvvgi1/d06dLFofXmP993ta/9vH5/Xelccvu6zm5R+tprr131XHbu3GnExMQYN9xwg1GmTBmjR48exi+//JKjxWn2ef76669Gs2bNjBIlShghISHGiBEjjKysLIfP2bVrlyHJWLp06RWvNwBHFoPH0AEAPNDIkSM1atQonTx50q03dK5atUpNmjTR7t27L9ul63rVtWtXffnll3n6a1u/fv20cuVKJSYmUtkHnMCafQAATHT33XerWbNmGj9+vLtDKbJOnTql//3vfxozZgyJPuAk1uwDAGCyRYsWuTuEIq106dLX5F4bwBNR2QcAAAA8FGv2AQAAAA9FZR8AAADwUCT7AAAAgIfiBt1Lj+o+duyYSpYsyV3+AADA4xiGoT///FPh4eHy8ioctd7z58/rwoULpn6Gj4+P/Pz8TP2Mwo5kX9KxY8cUERHh7jAAAABMdeTIEd10003uDkPnz59XVPkblHQiy9TPCQ0N1YEDB67rhJ9kX1LJkiUlSY3UUt4q7u5wAAAAXOqiMrVa39lzHne7cOGCkk5k6VBiBQWWNOcvDWl/2lS+3kFduHCBZP96l710x1vF5W0h2QcAAB7mUu/FwrZc+YaSFt1Q0pyYbCpc5+ouhWPRFgAAAACXo7IPAAAAt8gybMoy6YlPWYbNnImLGCr7AAAAgIeisg8AAAC3sMmQTeaU9s2at6ihsg8AAAB4KCr7AAAAcAubbDJrZb15MxctVPYBAAAAD0VlHwAAAG6RZRjKMsxZW2/WvEUNlX0AAADAQ1HZBwAAgFvQjcd8VPYBAAAAD0VlHwAAAG5hk6EsKvumorIPAAAAeCgq+wAAAHAL1uybj8o+AAAA4KGo7AMAAMAt6LNvPir7AAAAgIeisg8AAAC3sF3azJobVPYBAAAAj0VlHwAAAG6RZWKffbPmLWqo7AMAAAAeiso+AAAA3CLL+Hsza25Q2QcAAAA8FpV9AAAAuAXdeMxHZR8AAADwUFT2AQAA4BY2WZQli2lzg8o+AAAA4LGo7AMAAMAtbMbfm1lzg8o+AAAA4LGo7AMAAMAtskxcs2/WvEUNlX0AAADAQ1HZBwAAgFtQ2TcflX0AAADAQ1HZBwAAgFvYDItshkl99k2at6ihsg8AAAB4KCr7AAAAcAvW7JuPyj4AAADgoajsAwAAwC2y5KUsk2rPWabMWvRQ2QcAAAA8FJV9AAAAuIVhYjceg248EpV9AAAAwHOR7AMAAMAtsrvxmLU5Y+XKlWrdurXCw8NlsVi0YMGCHMfs2rVLDz74oKxWqwICAtSgQQMdPnzYvv/8+fPq1auXSpcurRtuuEHt27dXcnKyS65VfpHsAwAA4Lp39uxZ1a5dW1OmTMl1/6+//qpGjRqpatWq+uGHH7R161YNGzZMfn5+9mP69++vhQsX6osvvtCPP/6oY8eOqV27dtfwLHJizT4AAADcIsvwUpZhUjcew7njW7RooRYtWlx2/4svvqiWLVtq/Pjx9rGKFSva/52amqr3339fc+bM0X333SdJmjFjhqpVq6Z169bpjjvuyM9pFBiVfQAAAOAKbDabvv32W91yyy2KjY1VuXLl1LBhQ4elPomJicrMzFRMTIx9rGrVqoqMjNTatWvdFDnJPgAAANzEJots8jJp+3vNflpamsOWkZHhdJwnTpxQenq6Xn31VTVv3lzff/+9HnroIbVr104//vijJCkpKUk+Pj4KCgpyeG9ISIiSkpJcdMWcR7IPAAAAjxURESGr1Wrf4uPjnZ7DZrNJktq0aaP+/furTp06GjJkiB544AFNmzbNhKhdhzX7AAAAcIv8dM1xZm5JOnLkiAIDA+3jvr6+Ts9VpkwZeXt7q3r16g7j1apV0+rVqyVJoaGhunDhgs6cOeNQ3U9OTlZoaGgBzqRgqOwDAADAYwUGBjps+Un2fXx81KBBA+3Zs8dhfO/evSpfvrwkqV69eipevLiWLVtm379nzx4dPnxY0dHRLjiT/KGyDwAAALcwtxuPc+140tPTtX//fvvrAwcOaMuWLQoODlZkZKQGDhyohx9+WI0bN9a9996rxYsXa+HChfrhhx8kSVarVd27d9eAAQMUHByswMBAPfvss4qOjnZbJx6R7AMAAADSxo0bde+999pfDxgwQJLUpUsXzZw5Uw899JCmTZum+Ph49enTR1WqVNHcuXPVqFEj+3vefPNNeXl5qX379srIyFBsbKzeeecdt5xPNothOPlrjwdKS0uT1WpVE7WRt6W4u8MBAABwqYtGpn7QV0pNTXVYv+4u2bnX3F9uUUDJYqZ8xtk/s9S+9t5Cc87uwpp9AAAAwEOxjAcAAABuYZOXskyqPdt03S9ekdxd2V+5cqVat26t8PBwWSwWh6eQ/dvTTz8ti8WiiRMnOoynpKQoLi5OgYGBCgoKUvfu3ZWenn4NogcAAEBBZN+ga9YGNyf7Z8+eVe3atTVlypQrHjd//nytW7dO4eHhOfbFxcVpx44dSkhI0DfffKOVK1eqZ8+eJkYNAAAAFA1uXcbTokULtWjR4orHHD16VM8++6yWLFmiVq1aOezbtWuXFi9erA0bNqh+/fqSpMmTJ6tly5Z6/fXXc/3lAAAAAIWDTV6ysYzHVIX67xs2m02dO3fWwIEDVaNGjRz7165dq6CgIHuiL0kxMTHy8vLS+vXrr3G0AAAAQOFSqG/QHTdunLy9vdWnT59c9yclJalcuXIOY97e3goODlZSUtJl583IyFBGRob9dVpamgujBgAAQF5kGRZlGRbT5kYhruwnJibqrbfe0syZM2WxuPY/Vnx8vKxWq32LiIhw6fwAAABAYVBok/1Vq1bpxIkTioyMlLe3t7y9vXXo0CE999xzqlChgiQpNDRUJ06ccHjfxYsXlZKSotDQ0MvOPXToUKWmptq3I0eOmH4+AAAAcJR1qfWmWRsK8TKezp07KyYmxmEsNjZWnTt3Vrdu3SRJ0dHROnPmjBITE1WvXj1J0vLly2Wz2dSwYcPLzu3r6ytfX1+TzwAAAABwL7cm++np6dq/f7/99YEDB7RlyxYFBwcrMjJSpUuXdji+ePHiCg0NVZUqVSRJ1apVU/PmzdWjRw9NmzZNmZmZ6t27tzp16kQnHgAAgELOZnjJZlI/fJtBNx65exnPxo0bVbduXdWtW1eSNGDAANWtW1fDhw/P8xyzZ89W1apV1bRpU7Vs2VKNGjXS9OnTTYwaAAAAKBrcWtlv0qSJDCd+6zp48GCOseDgYM2ZM8fFkQEAAMBsZq6tz6LPvuTuyj4AAAAA8xTaG3QBAADg2Wwm9sO3mTJr0UNlHwAAAPBQVPYBAADgFjZ5yWZS7dmseYsargIAAADgoajsAwAAwC2yDC9lmdRn36x5ixquAgAAAOChqOwDAADALWyyyCazuvGYM29RQ2UfAAAA8FBU9gEAAOAWrNk3H1cBAAAA8FBU9gEAAOAWWfJSlkm1Z7PmLWq4CgAAAICHorIPAAAAt7AZFtkMk7rxmDRvUUNlHwAAAPBQVPYBAADgFjYT1+zbqGlLVPYBAAAAz0VlHwAAAG5hM7xkM6kfvlnzFjVcBQAAAMBDUdkHAACAW2TJoiyZ0zXHrHmLGir7AAAAgIeisg8AAAC3YM2++bgKAAAAgIeisg8AAAC3yDJxbX2WKbMWPVT2AQAAAA9FZR8AAABuwZp983EVAAAAAA9FZR8AAABukWV4KcukCrxZ8xY1XAUAAADAQ1HZBwAAgFsYsshmUjcegyfoSlT2AQAAAM9FZR8AAABuwZp983EVAAAAAA9Fsg8AAAC3sBkWUzdnrFy5Uq1bt1Z4eLgsFosWLFhw2WOffvppWSwWTZw40WE8JSVFcXFxCgwMVFBQkLp376709PR8Xx9XINkHAADAde/s2bOqXbu2pkyZcsXj5s+fr3Xr1ik8PDzHvri4OO3YsUMJCQn65ptvtHLlSvXs2dPEqK+ONfsAAABwiyx5Kcuk2rOz87Zo0UItWrS44jFHjx7Vs88+qyVLlqhVq1YO+3bt2qXFixdrw4YNql+/viRp8uTJatmypV5//fVcfzm4FqjsAwAAAFdhs9nUuXNnDRw4UDVq1Mixf+3atQoKCrIn+pIUExMjLy8vrV+//hpH+/+o7AMAAMAt8rO23pm5JSktLc1h3NfXV76+vk7PN27cOHl7e6tPnz657k9KSlK5cuUcxry9vRUcHKykpCSnP89VqOwDAADAY0VERMhqtdq3+Ph4p+dITEzUW2+9pZkzZ8piKVoP66KyDwAAALewyUs2k2rP2fMeOXJEgYGB9vH8VPVXrVqlEydOKDIy0j6WlZWl5557ThMnTtTBgwcVGhqqEydOOLzv4sWLSklJUWhoaIHOpSBI9gEAAOCxAgMDHZL9/OjcubNiYmIcxmJjY9W5c2d169ZNkhQdHa0zZ84oMTFR9erVkyQtX75cNptNDRs2LNDnFwTJPgAAANwiy7Aoy6Q1+87Om56erv3799tfHzhwQFu2bFFwcLAiIyNVunRph+OLFy+u0NBQValSRZJUrVo1NW/eXD169NC0adOUmZmp3r17q1OnTm7rxCPW7AMAAADSxo0bVbduXdWtW1eSNGDAANWtW1fDhw/P8xyzZ89W1apV1bRpU7Vs2VKNGjXS9OnTTYz66qjsAwAAwC2uRTeevGrSpIkMw8jz8QcPHswxFhwcrDlz5jj1uWajsg8AAAB4KCr7AAAAcAvD8JLNMKf2bJg0b1HDVQAAAAA8FJV9AAAAuEWWLMqSSd14TJq3qKGyDwAAAHgoKvsAAABwC5vhfNccZ+YGlX0AAADAY1HZBwAAgFvYTOzGY9a8RY1br8LKlSvVunVrhYeHy2KxaMGCBfZ9mZmZGjx4sGrVqqWAgACFh4fr8ccf17FjxxzmSElJUVxcnAIDAxUUFKTu3bsrPT3dDWcDAAAAFC5uTfbPnj2r2rVra8qUKTn2nTt3Tps2bdKwYcO0adMmzZs3T3v27NGDDz7ocFxcXJx27NihhIQEffPNN1q5cqV69ux5Dc8CAAAA+WGTxdQNbl7G06JFC7Vo0SLXfVarVQkJCQ5jb7/9tm6//XYdPnxYkZGR2rVrlxYvXqwNGzaofv36kqTJkyerZcuWev311xUeHn5NzgMAAADOyzIsyjLpBl2z5i1qitRiptTUVFksFgUFBUmS1q5dq6CgIHuiL0kxMTHy8vLS+vXr3RgpAAAA4H5F5gbd8+fPa/DgwXrkkUcUGBgoSUpKSlK5cuUcjvP29lZwcLCSkpIuO1dGRoYyMjLsr9PS0kyMHAAAALnhBl3zFYmrkJmZqY4dO8owDE2dOrXA88XHx8tqtdq3iIgIl8QJAAAAFCaFPtnPTvQPHTqkhIQEe1VfkkJDQ3XixAmH4y9evKiUlBSFhoZeds6hQ4cqNTXVvh05csTUcwAAAEBONllkM0zauEFXKuzLeLIT/X379mnFihUqXbq0w/7o6GidOXNGiYmJqlevniRp+fLlstlsatiw4WXn9fX1la+vr+nxAwAAAO7k1mQ/PT1d+/fvt78+cOCAtmzZouDgYIWFhalDhw7atGmTvvnmG2VlZdnX4QcHB8vHx0fVqlVT8+bN1aNHD02bNk2ZmZnq3bu3OnXqRCceAACAQs4wsUWmQWVfcneyv3HjRt1777321wMGDJAkdenSRSNHjtTXX38tSapTp47D+1asWKEmTZpIkmbPnq3evXuradOm8vLyUvv27TVp0qRreh4AAABAYeTWZL9JkyYyDOOy+6+0L1twcLDmzJnj4sgAAABgtuz19WbNjSJwgy4AAACA/CnUN+gCAADAc9Fn33xcBQAAAMBDUdkHAACAW7Bm33xU9gEAAAAPRWUfAAAAbmEzsc8+T9D9G5V9AAAAwENR2QcAAIBbsGbffFT2AQAAAA9FZR8AAABuQWXffFT2AQAAAA9FZR8AAABuQWXffFT2AQAAAA9FZR8AAABuQWXffFT2AQAAAA9FZR8AAABuYZj4pFvDlFmLHir7AAAAgIeisg8AAAC3YM2++ajsAwAAAB6Kyj4AAADcgsq++ajsAwAAAB6Kyj4AAADcgsq++ajsAwAAAB6Kyj4AAADcgsq++ajsAwAAAB6Kyj4AAADcwjAsMkyqwJs1b1FDZR8AAADwUCT7AAAAcAubLKZuzli5cqVat26t8PBwWSwWLViwwL4vMzNTgwcPVq1atRQQEKDw8HA9/vjjOnbsmMMcKSkpiouLU2BgoIKCgtS9e3elp6e77Hrlh1PJ/q5duzRixAjdd999qlixosLCwnTrrbeqS5cumjNnjjIyMsyLFAAAADDJ2bNnVbt2bU2ZMiXHvnPnzmnTpk0aNmyYNm3apHnz5mnPnj168MEHHY6Li4vTjh07lJCQoG+++UYrV65Uz549r+FZ5GQxDMO42kGbNm3SoEGDtHr1at111126/fbbFR4eLn9/f6WkpGj79u1atWqV0tLSNGjQIPXr10++vr7X5gxcIC0tTVarVU3URt6W4u4OBwAAwKUuGpn6QV8pNTVVgYGB7g7Hnns1XNBH3gHm5IwXz2ZofdtJ+Tpni8Wi+fPnq23btpc9ZsOGDbr99tt16NAhRUZGateuXapevbo2bNig+vXrS5IWL16sli1b6vfff1d4eHiBzyk/8nSDbvv27TVw4EB9+eWXCgoKuuxxa9eu1VtvvaU33nhDL7zwgivjBAAAAAqN1NRUWSwWe268du1aBQUF2RN9SYqJiZGXl5fWr1+vhx56yC1x5inZ37t3r4oXv3rFOzo6WtHR0crMzHRFbAAAAPBg16IbT1pamsO4r69vgVegnD9/XoMHD9Yjjzxi/6tBUlKSypUr53Cct7e3goODlZSUVKDPK4g8rdn/Z6L/22+/OXU8AAAA4C4RERGyWq32LT4+vkDzZWZmqmPHjjIMQ1OnTnVZnGZxus9+pUqVdM8996h79+7q0KGD/Pz8zIkMAAAAHu1aPEH3yJEjDmv2C1LVz070Dx06pOXLlzvMGxoaqhMnTjgcf/HiRaWkpCg0NDTfn1lQTrfe3LRpk2699VYNGDBAoaGheuqpp/Tzzz+bEx0AAABQAIGBgQ5bfpP97ER/3759Wrp0qUqXLu2wPzo6WmfOnFFiYqJ9bPny5bLZbGrYsGGBzyO/nE7269Spo7feekvHjh3TBx98oOPHj6tRo0aqWbOmJkyYoJMnT5oTKQAAADxK9pp9szZnpKena8uWLdqyZYsk6cCBA9qyZYsOHz6szMxMdejQQRs3btTs2bOVlZWlpKQkJSUl6cKFC5KkatWqqXnz5urRo4d+/vln/fTTT+rdu7c6derktk48KshDtby9vdWuXTt98cUXGjdunPbv36/nn39eERERevzxx3X8+HHXRgoAAACYZOPGjapbt67q1q0rSRowYIDq1q2r4cOH6+jRo/r666/1+++/q06dOgoLC7Nva9assc8xe/ZsVa1aVU2bNlXLli3VqFEjTZ8+3Y1nlY81+9k2btyoDz74QJ9++qkCAgL0/PPPq3v37vr99981atQotWnThuU9AAAAuCzDxDX7zlb2mzRpois9fioPj6ZScHCw5syZ49Tnms3pZH/ChAmaMWOG9uzZo5YtW2rWrFlq2bKlvLz+/iNBVFSUZs6cqQoVKpgRLwAAAIA8cjrZnzp1qp544gl17dpVYWFhuR5Trlw5vf/++66IDwAAAB7KkJSHgnm+50Y+kv19+/Zd9RgfHx916dIlvzEBAAAAcIE83aB7+PBhpyY9evRofuMBAADAdcImi6kb8pjsN2jQQE899ZQ2bNhw2WNSU1P13nvvqWbNmpo7d64rYwQAAACQD3laxrNz50698soruv/+++Xn56d69eopPDxcfn5+On36tHbu3KkdO3botttu0/jx49WyZUvzIwcAAECRlp9++M7MjTxW9kuXLq0JEybo+PHjevvtt1W5cmX98ccf9vX7cXFxSkxM1Nq1a0n0AQAAgELCqRt0/f391aFDB3Xo0MG8iAAAAHBdsBkWWUyqwJvVv7+oyfcTdAEAAAAUbvl+gi4AAABQEIZhYp99Gu1LVPYBAAAAz0VlHwAAAG5BNx7zOV3ZX7lypS5evJhj/OLFi1q5cqWr4gIAAABQQE4n+/fee69SUlJyjKempuree+91VVwAAADwcNmVfbM25CPZNwxDFkvOi3fq1CkFBAQ4NdfKlSvVunVrhYeHy2KxaMGCBTk+a/jw4QoLC5O/v79iYmLsvf2zpaSkKC4uToGBgQoKClL37t2Vnp7u7GkBAAAAHifPa/bbtWsnSbJYLOratat8fX3t+7KysrR161bdeeedTn342bNnVbt2bT3xxBP2+f9p/PjxmjRpkj788ENFRUVp2LBhio2N1c6dO+Xn5yddeqDX8ePHlZCQoMzMTHXr1k09e/bUnDlznIoFAAAA1xZ99s2X52TfarVKl6rtJUuWlL+/v32fj4+P7rjjDvXo0cOpD2/RooVatGiR6z7DMDRx4kS99NJLatOmjSRp1qxZCgkJ0YIFC9SpUyft2rVLixcv1oYNG1S/fn1J0uTJk9WyZUu9/vrrCg8PdyoeAAAAwJPkOdmfMWOGJKlChQp6/vnnnV6y46wDBw4oKSlJMTEx9jGr1aqGDRtq7dq16tSpk9auXaugoCB7oi9JMTEx8vLy0vr16/XQQw+ZGiMAAADyjz775nO69eaIESPMieRfkpKSJEkhISEO4yEhIfZ9SUlJKleunMN+b29vBQcH24/JTUZGhjIyMuyv09LSXBw9AAAA4H5OJ/tRUVG53qCb7bfffitoTKaLj4/XqFGj3B0GAADAde3vyr5ZffZNmbbIcTrZ79evn8PrzMxMbd68WYsXL9bAgQNdFlhoaKgkKTk5WWFhYfbx5ORk1alTx37MiRMnHN538eJFpaSk2N+fm6FDh2rAgAH212lpaYqIiHBZ7AAAALg6HqplPqeT/b59++Y6PmXKFG3cuNEVMUmX/oIQGhqqZcuW2ZP7tLQ0rV+/Xv/9738lSdHR0Tpz5owSExNVr149SdLy5ctls9nUsGHDy87t6+vr0E0IAAAA8ERO99m/nBYtWmju3LlOvSc9PV1btmzRli1bpEs35W7ZskWHDx+WxWJRv379NGbMGH399dfatm2bHn/8cYWHh6tt27aSpGrVqql58+bq0aOHfv75Z/3000/q3bu3OnXqRCceAACAQs4weUM+KvuX8+WXXyo4ONip92zcuNHhqbvZS2u6dOmimTNnatCgQTp79qx69uypM2fOqFGjRlq8eLG9x74kzZ49W71791bTpk3l5eWl9u3ba9KkSa46LQAAAKDIcjrZr1u3rsMNuoZhKCkpSSdPntQ777zj1FxNmjSRcYW7JywWi0aPHq3Ro0df9pjg4GAeoAUAAFAEsWbffE4n+9lLaLJ5eXmpbNmyatKkiapWrerK2AAAAAAUQKHtsw8AAAAPZ+biehbtS/lds5+VlaX58+dr165dkqTq1aurTZs28vZ22S0AAAAAAArI6ex8x44dat26tZKTk1WlShVJ0rhx41S2bFktXLhQNWvWNCNOAAAAeBoT1+yLNftSflpvPvnkk6pZs6Z+//13bdq0SZs2bdKRI0d06623qmfPnuZECQAAAMBpTlf2t2zZoo0bN6pUqVL2sVKlSumVV15RgwYNXB0fAAAAPJRh/L2ZNTfyUdm/5ZZblJycnGP8xIkTqlSpkqviAgAAAFBATlf24+Pj1adPH40cOVJ33HGHJGndunUaPXq0xo0bp7S0NPuxgYGBro0WAAAAHoM+++ZzOtl/4IEHJEkdO3a0P1wr+8FYrVu3tr+2WCzKyspybbQAAAAA8szpZH/FihXmRAIAAIDri2Exr2sOlX0pP8l+VFSUIiIi7FX9bIZh6MiRI4qMjHRlfAAAAADyyekbdKOionTy5Mkc4ykpKYqKinJVXAAAAPBw2d14zNqQj2Q/ez3+v6Wnp8vPz89VcQEAAAAooDwv4xkwYIAkyWKxaNiwYSpRooR9X1ZWltavX686deqYEyUAAAA8j3FpM2tu5D3Z37x5s3Spsr9t2zb5+PjY9/n4+Kh27dp6/vnnzYkSAAAAgNPynOxnd+Hp1q2b3nrrLXroAwAAoEDos28+p7vxzJgxw5xIAAAAALiU08n+fffdd8X9y5cvL0g8AAAAuJ6wtt5UTif7tWvXdnidmZmpLVu2aPv27erSpYsrYwMAAABQAE4n+2+++Wau4yNHjlR6erorYgIAAMB1gDX75nO6z/7lPPbYY/rggw9cNR0AAACAAnJZsr927VoeqgUAAIC8M0zenLBy5Uq1bt1a4eHhslgsWrBggWOohqHhw4crLCxM/v7+iomJ0b59+xyOSUlJUVxcnAIDAxUUFKTu3bu7feWL08t42rVr5/DaMAwdP35cGzdu1LBhw1wZGwAAAHBNnD17VrVr19YTTzyRI9+VpPHjx2vSpEn68MMPFRUVpWHDhik2NlY7d+60F7zj4uJ0/PhxJSQkKDMzU926dVPPnj01Z84cN5zR35xO9q1Wq8NrLy8vValSRaNHj1azZs1cGRsAAAA8muXSZtbcedeiRQu1aNEi132GYWjixIl66aWX1KZNG0nSrFmzFBISogULFqhTp07atWuXFi9erA0bNqh+/fqSpMmTJ6tly5Z6/fXXFR4e7oJzch599gEAAIArOHDggJKSkhQTE2Mfs1qtatiwodauXatOnTpp7dq1CgoKsif6khQTEyMvLy+tX79eDz30kFtidzrZ/+uvv5SQkKC9e/dKkqpUqaKYmBj5+/ubER8AAAA8VT7W1js1t6S0tDSHYV9fX/n6+jo1VVJSkiQpJCTEYTwkJMS+LykpSeXKlXPY7+3treDgYPsx7uBUsv/111/rySef1B9//OEwXqZMGb3//vtq3bq1q+MDAAAA8i0iIsLh9YgRIzRy5Ei3xXOt5bkbz5o1a9ShQwc1btxYP/30k1JSUpSSkqLVq1fr7rvvVocOHbRu3TpzowUAAIDnuAbdeI4cOaLU1FT7NnToUKfDDA0NlSQlJyc7jCcnJ9v3hYaG6sSJEw77L168qJSUFPsx7pDnZH/MmDHq1q2bvvzyS0VHRysoKEhBQUG68847NXfuXHXt2lWjR482N1oAAADACYGBgQ6bs0t4JCkqKkqhoaFatmyZfSwtLU3r169XdHS0JCk6OlpnzpxRYmKi/Zjly5fLZrOpYcOGLjob5+V5Gc+6des0bty4y+7v1auX7rnnHlfFBQAAAE9nWP7ezJrbCenp6dq/f7/99YEDB7RlyxYFBwcrMjJS/fr105gxY1S5cmV7683w8HC1bdtWklStWjU1b95cPXr00LRp05SZmanevXurU6dObuvEI2eS/b/++kuBgYGX3W+1WnX+/HlXxQUAAABcMxs3btS9995rfz1gwABJUpcuXTRz5kwNGjRIZ8+eVc+ePXXmzBk1atRIixcvdnio7OzZs9W7d281bdpUXl5eat++vSZNmuSW88mW52S/cuXKWr58ubp165br/mXLlqly5cqujA0AAAAezDD+3sya2xlNmjSRcYU3WSwWjR49+orL1oODg936AK3c5HnNfrdu3fT888/ru+++y7Hv22+/1aBBg9S1a1dXxwcAAAAgn/Jc2e/bt6/WrFmjBx54QFWqVFG1atVkGIZ27dqlffv2qW3bturXr5+50QIAAMBzXIM++9e7PFf2vby89MUXX+iTTz5RlSpVtHv3bu3Zs0dVq1bV7NmzNXfuXHl55Xk6AAAAACZz+gm6Dz/8sB5++GFzogEAAMD1oxB14/FUlOIBAAAAD+V0ZR8AAABwBYvx92bW3KCyDwAAAHgsKvsAAABwD7rxmI7KPgAAAFBI3HzzzTp16lSO8TNnzujmm292er48VfbbtWuX5wnnzZvndBAAAAC4DtGNJ4eDBw8qKysrx3hGRoaOHj3q9Hx5SvatVqv934ZhaP78+bJarapfv74kKTExUWfOnHHqlwIAAAAAf/v666/t/16yZIlD/p2VlaVly5apQoUKTs+bp2R/xowZ9n8PHjxYHTt21LRp01SsWDF7AM8884wCAwOdDgAAAADXKdbs27Vt21aSZLFY1KVLF4d9xYsXV4UKFfTGG284Pa/TN+h+8MEHWr16tT3Rl6RixYppwIABuvPOO/Xaa685HQQAAABwPbPZbJKkqKgobdiwQWXKlHHJvE7foHvx4kXt3r07x/ju3bvtQQIAAABXZZi8FUEHDhxwWaKv/FT2u3Xrpu7du+vXX3/V7bffLklav369Xn31VXXr1s1lgQEAAADXo2XLlmnZsmU6ceJEjmL6Bx984NRcTif7r7/+ukJDQ/XGG2/o+PHjkqSwsDANHDhQzz33nLPTAQAA4HrFmv0cRo0apdGjR6t+/foKCwuTxVKwrkJOJ/teXl4aNGiQBg0apLS0NEnixlwAAADABaZNm6aZM2eqc+fOLpkvXw/VunjxopYuXapPPvnE/tvGsWPHlJ6e7pKgAAAAcB3I7rNv1lYEXbhwQXfeeafL5nM62T906JBq1aqlNm3aqFevXjp58qQkady4cXr++eddFhgAAABwvXnyySc1Z84cl83n9DKevn37qn79+vrll19UunRp+/hDDz2kHj16uCwwAAAAeDaL8fdm1txF0fnz5zV9+nQtXbpUt956q4oXL+6wf8KECU7N53Rlf9WqVXrppZfk4+PjMF6hQoV8PcL3SrKysjRs2DBFRUXJ399fFStW1MsvvyzD+P//eoZhaPjw4QoLC5O/v79iYmK0b98+l8YBAAAAXAtbt25VnTp15OXlpe3bt2vz5s32bcuWLU7P53Rl32azKSsrK8f477//rpIlSzodwJWMGzdOU6dO1YcffqgaNWpo48aN6tatm6xWq/r06SNJGj9+vCZNmqQPP/xQUVFRGjZsmGJjY7Vz5075+fm5NB4AAAC4EN14clixYoVL53O6st+sWTNNnDjR/tpisSg9PV0jRoxQy5YtXRrcmjVr1KZNG7Vq1UoVKlRQhw4d1KxZM/3888/Spar+xIkT9dJLL6lNmza69dZbNWvWLB07dkwLFixwaSwAAABAUeN0Zf+NN95QbGysqlevrvPnz+vRRx/Vvn37VKZMGX3yyScuDe7OO+/U9OnTtXfvXt1yyy365ZdftHr1avtapQMHDigpKUkxMTH291itVjVs2FBr165Vp06dXBoPAAAAYKZ77733ir31ly9f7tR8Tif7N910k3755Rd99tln+uWXX5Senq7u3bsrLi5O/v7+zk53RUOGDFFaWpqqVq2qYsWKKSsrS6+88ori4uIkSUlJSZKkkJAQh/eFhITY9+UmIyNDGRkZ9tfZzwsAAAAA3KlOnToOrzMzM7VlyxZt375dXbp0cXo+p5P9lStX6s4771RcXJw96dal3vsrV65U48aNnQ7icj7//HPNnj1bc+bMUY0aNbRlyxb169dP4eHh+TrZbPHx8Ro1apTL4gQAAIDzLCZ2zSmaXfalN998M9fxkSNH5uuZVk6v2b/33nuVkpKSYzw1NVX33nuv0wFcycCBAzVkyBB16tRJtWrVUufOndW/f3/Fx8dLkkJDQyVJycnJDu9LTk6278vN0KFDlZqaat+OHDni0rgBAAAAV3rsscf0wQcfOP0+p5N9wzByXUd06tQpBQQEOB3AlZw7d05eXo4hFitWTDabTZIUFRWl0NBQLVu2zL4/LS1N69evV3R09GXn9fX1VWBgoMMGAACAa4wn6ObZ2rVr89VpMs/LeNq1aydd6r7TtWtX+fr62vdlZWVp69atLn20ryS1bt1ar7zyiiIjI1WjRg1t3rxZEyZM0BNPPGGPpV+/fhozZowqV65sb70ZHh6utm3bujQWAAAAuBitN3PIzrmzGYah48ePa+PGjRo2bJjT8+U52bdarfYPLFmypMPNuD4+Prrjjjtc/gTdyZMna9iwYXrmmWd04sQJhYeH66mnntLw4cPtxwwaNEhnz55Vz549debMGTVq1EiLFy+mxz4AAACKnOycO5uXl5eqVKmi0aNHq1mzZk7PZzH++TjaPBg1apQGDhyoEiVKOP1hhVVaWpqsVquaqI28LcXz8A4AAICi46KRqR/0lVJTUwvF8uXs3Kv82FfkZVKB1nb+vA698GKhOWd3cbobz+OPP66jR4+qcuXKDuP79u1T8eLFVaFCBVfGBwAAAFx3EhMTtWvXLklSjRo1VLdu3XzN4/QNul27dtWaNWtyjK9fv15du3bNVxAAAAC4/lgMc7ei6MSJE7rvvvvUoEED9enTR3369FG9evXUtGlTnTx50un5nE72N2/erLvuuivH+B133KEtW7Y4HQAAAACAvz377LP6888/tWPHDqWkpCglJUXbt29XWlqa+vTp4/R8Ti/jsVgs+vPPP3OMp6amKisry+kAAAAAcJ2iG08Oixcv1tKlS1WtWjX7WPXq1TVlypR83aDrdGW/cePGio+Pd0jss7KyFB8fr0aNGjkdAAAAAIC/2Ww2FS+es2FM8eLF7c+acobTlf1x48apcePGqlKliu6++25J0qpVq5SWlqbly5c7HQAAAACuU1T2c7jvvvvUt29fffLJJwoPD5ckHT16VP3791fTpk2dns/pyn716tW1detWdezYUSdOnNCff/6pxx9/XLt371bNmjWdDgAAAADA395++22lpaWpQoUKqlixoipWrKioqCilpaVp8uTJTs/ndGVfksLDwzV27Nj8vBUAAACQZG7XnKLajSciIkKbNm3S0qVLtXv3bklStWrVFBMTk6/58pTsb926VTVr1pSXl5e2bt16xWNvvfXWfAUCAAAAXK+WL1+u3r17a926dQoMDNT999+v+++/X7rUCKdGjRqaNm2afRl9XuUp2a9Tp46SkpJUrlw51alTRxaLRbk9eNdisdCRBwAAAHljWP7ezJq7CJk4caJ69OiR69N+rVarnnrqKU2YMMGcZP/AgQMqW7as/d8AAAAAXOeXX37RuHHjLru/WbNmev31152eN0/Jfvny5XP9NwAAAJBvdOOxS05OzrXlZjZvb+98PUE3T8n+119/necJH3zwQaeDAAAAAK5nN954o7Zv365KlSrlun/r1q0KCwtzet48Jftt27Z1eP3vNfsWy/+viWLNPgAAAPKCbjz/r2XLlho2bJiaN28uPz8/h31//fWXRowYoQceeMDpefPUZ99ms9m377//XnXq1NGiRYt05swZnTlzRt99951uu+02LV682OkAAAAAgOvdSy+9pJSUFN1yyy0aP368vvrqK3311VcaN26cqlSpopSUFL344otOz+v0Q7X69eunt956S7GxsQoMDFRgYKBiY2M1YcIE9enTx+kAAAAAcJ0yTN6ckJWVpWHDhikqKkr+/v6qWLGiXn75ZYfVLIZhaPjw4QoLC5O/v79iYmK0b98+l1yKkJAQrVmzRjVr1tTQoUP10EMP6aGHHtILL7ygmjVravXq1QoJCXF6XqcfqvXrr78qKCgox7jVatXBgwedDgAAAABwt3Hjxmnq1Kn68MMPVaNGDW3cuFHdunWT1Wq1F7THjx+vSZMm6cMPP1RUVJSGDRum2NhY7dy5M8fSm/woX768vvvuO50+fVr79++XYRiqXLmySpUqle85nU72GzRooAEDBuijjz6y/3aRnJysgQMH6vbbb893IAAAALjOmLhm39nK/po1a9SmTRu1atVKklShQgV98skn+vnnn/+ezjA0ceJEvfTSS2rTpo0kadasWQoJCdGCBQvUqVMnl4VeqlQpNWjQwCVzOb2M54MPPtDx48cVGRmpSpUqqVKlSoqMjNTRo0f1/vvvuyQoAAAA4Fq68847tWzZMu3du1e61Pd+9erVatGihXTpWVNJSUmKiYmxv8dqtaphw4Zau3at2+K+Gqcr+5UqVdLWrVuVkJCg3bt3S5KqVaummJgYh648AAAAwBVdgz77aWlpDsO+vr7y9fXNcfiQIUOUlpamqlWrqlixYsrKytIrr7yiuLg4SVJSUpJ0aW39P4WEhNj3FUZOJ/u61GqzWbNmaty4sXx9fUnyAQAAUChFREQ4vB4xYoRGjhyZ47jPP/9cs2fP1pw5c1SjRg1t2bJF/fr1U3h4uLp06XINI3Ytp5N9m82mV155RdOmTVNycrL27t2rm2++WcOGDVOFChXUvXt3cyIFAACAZ7kGlf0jR44oMDDQPpxbVV+SBg4cqCFDhtjX3teqVUuHDh1SfHy8unTpotDQUOnSvar/fLhVcnKy6tSpY9JJFJzTa/bHjBmjmTNnavz48fLx8bGP16xZU//73/9cHR8AAACQb9mt4rO3yyX7586dk5eXY2pcrFgx2Ww2SVJUVJRCQ0O1bNky+/60tDStX79e0dHRJp9F/jld2Z81a5amT5+upk2b6umnn7aP165d276GHwAAALiawvQE3datW+uVV15RZGSkatSooc2bN2vChAl64okn/p7PYlG/fv00ZswYVa5c2d56Mzw8XG3btjXnJFzA6WT/6NGjqlSpUo5xm82mzMxMV8UFAC5XrHSwu0NwkHUqxd0hAAAumTx5soYNG6ZnnnlGJ06cUHh4uJ566ikNHz7cfsygQYN09uxZ9ezZU2fOnFGjRo20ePFil/TYN4vTyX716tW1atUqlS9f3mH8yy+/VN26dV0ZGwAAAHBNlCxZUhMnTtTEiRMve4zFYtHo0aM1evToaxpbQTid7A8fPlxdunTR0aNHZbPZNG/ePO3Zs0ezZs3SN998Y06UAAAAAJzmdLLfpk0bLVy4UKNHj1ZAQICGDx+u2267TQsXLtT9999vTpQA4AIsmwGAQuYadOO53jmV7F+8eFFjx47VE088oYSEBPOiAgAAAFBgTrXe9Pb21vjx43Xx4kXzIgIAAMB1Ibsbj1kb8tFnv2nTpvrxxx/NiQYAAACAyzi9Zr9FixYaMmSItm3bpnr16ikgIMBh/4MPPujK+AAAAODJqMCbyulk/5lnnpEkTZgwIcc+i8WirKws10QGAAAAoECcTvazHxkMAAAAFAjdeEznVLJ/8OBBJSQkKDMzU/fcc49q1KhhXmQAAAAACiTPyf6KFSv0wAMP6K+//vr7jd7e+uCDD/TYY4+ZGR8AAAA8lJldc+jG87c8d+MZNmyY7r//fh09elSnTp1Sjx49NGjQIHOjAwAAAJBveU72t2/frrFjxyosLEylSpXSa6+9phMnTujUqVPmRggAAADPZJi8Ie/JflpamsqUKWN/XaJECfn7+ys1NdWs2AAAAAAUgFM36C5ZskRWq9X+2mazadmyZdq+fbt9jD77AAAAyAvW7JvPqWS/S5cuOcaeeuop+7/psw8AAAAUHnlO9umvDwAAAJeiz77p8rxmHwAAAEDR4vQTdAEAAACXoLJvOir7AAAAgIeisg8AAAC3oBuP+ajsAwAAAB6Kyj4AAADcgzX7pstTsl+qVClZLJY8TZiSklLQmAAAAAC4QJ6S/YkTJ9r/ferUKY0ZM0axsbGKjo6WJK1du1ZLlizRsGHDzIsUAAAAnoXKvunylOz/88m57du31+jRo9W7d2/7WJ8+ffT2229r6dKl6t+/vzmRAgAAAHCK0zfoLlmyRM2bN88x3rx5cy1dutRVcQEAAMDDZXfjMWtDPpL90qVL66uvvsox/tVXX6l06dKuigsAAABAATmd7I8aNUqDBw9W69atNWbMGI0ZM0atW7fWkCFDNGrUKJcHePToUT322GMqXbq0/P39VatWLW3cuNG+3zAMDR8+XGFhYfL391dMTIz27dvn8jgAAADgYobJG5xP9rt27aqffvpJgYGBmjdvnubNm6fAwECtXr1aXbt2dWlwp0+f1l133aXixYtr0aJF2rlzp9544w2VKlXKfsz48eM1adIkTZs2TevXr1dAQIBiY2N1/vx5l8YCAAAAFDX56rPfsGFDzZ492/XR/Mu4ceMUERGhGTNm2MeioqLs/zYMQxMnTtRLL72kNm3aSJJmzZqlkJAQLViwQJ06dTI9RgCXV6xSVB6Ounay9h9wdwgAgH/gCbrmy9cTdH/99Ve99NJLevTRR3XixAlJ0qJFi7Rjxw6XBvf111+rfv36+s9//qNy5cqpbt26eu+99+z7Dxw4oKSkJMXExNjHrFarGjZsqLVr11523oyMDKWlpTlsAAAAgKdxOtn/8ccfVatWLa1fv15z585Venq6JOmXX37RiBEjXBrcb7/9pqlTp6py5cpasmSJ/vvf/6pPnz768MMPJUlJSUmSpJCQEIf3hYSE2PflJj4+Xlar1b5FRES4NG4AAADkAWv2Ted0sj9kyBCNGTNGCQkJ8vHxsY/fd999WrdunUuDs9lsuu222zR27FjVrVtXPXv2VI8ePTRt2rQCzTt06FClpqbatyNHjrgsZgAAAKCwcHrN/rZt2zRnzpwc4+XKldMff/zhqrgkSWFhYapevbrDWLVq1TR37lxJUmhoqCQpOTlZYWFh9mOSk5NVp06dy87r6+srX19fl8YKICfWyAMArogn6JrO6cp+UFCQjh8/nmN88+bNuvHGG10VlyTprrvu0p49exzG9u7dq/Lly0uXbtYNDQ3VsmXL7PvT0tK0fv16RUdHuzQWAAAAoKhxOtnv1KmTBg8erKSkJFksFtlsNv300096/vnn9fjjj7s0uP79+2vdunUaO3as9u/frzlz5mj69Onq1auXJMlisahfv34aM2aMvv76a23btk2PP/64wsPD1bZtW5fGAgAAANeymLwhH8t4xo4dq169eikiIkJZWVmqXr26srKy9Oijj+qll15yaXANGjTQ/PnzNXToUI0ePVpRUVGaOHGi4uLi7McMGjRIZ8+eVc+ePXXmzBk1atRIixcvlp+fn0tjAQAAAIoai2EY+VrRdPjwYW3fvl3p6emqW7euKleu7ProrpG0tDRZrVY1URt5W4q7OxwAAACXumhk6gd9pdTUVAUGBro7HHvuVf2/Y1XM15wCbVbGee2c+kKhOWd3yddDtSQpMjJSkZGRro0GAAAA1w0eqmW+PCX7AwYMyPOEEyZMKEg8AAAAAFwkT8n+5s2b8zSZxcKtEAAAAMgjWm+aLk/J/ooVK8yPBAAAAIBL5XvNPgAAAFBgVOBNla9kf+PGjfr88891+PBhXbhwwWHfvHnzXBUbAAAAgAJw+qFan376qe68807t2rVL8+fPV2Zmpnbs2KHly5fLarWaEyUAAAA8TnY3HrM25CPZHzt2rN58800tXLhQPj4+euutt7R792517NiRVpwAAABAIeJ0sv/rr7+qVatWkiQfHx+dPXtWFotF/fv31/Tp082IEQAAAJ7IMHlz0tGjR/XYY4+pdOnS8vf3V61atbRx48b/D9cwNHz4cIWFhcnf318xMTHat2+fa6+Jizmd7JcqVUp//vmnJOnGG2/U9u3bJUlnzpzRuXPnXB8hAAAAYLLTp0/rrrvuUvHixbVo0SLt3LlTb7zxhkqVKmU/Zvz48Zo0aZKmTZum9evXKyAgQLGxsTp//rxbY78Sp2/Qbdy4sRISElSrVi395z//Ud++fbV8+XIlJCSoadOm5kQJAAAAj1OYnqA7btw4RUREaMaMGfaxqKgo+78Nw9DEiRP10ksvqU2bNpKkWbNmKSQkRAsWLFCnTp1cF7wLOV3Zf/vtt+0n8+KLL2rAgAFKTk5W+/bt9f7775sRIwAAAGCqr7/+WvXr19d//vMflStXTnXr1tV7771n33/gwAElJSUpJibGPma1WtWwYUOtXbvWTVFfndOV/eDgYPu/vby8NGTIEFfHBAAAgOvBNXiCblpamsOwr6+vfH19cxz+22+/aerUqRowYIBeeOEFbdiwQX369JGPj4+6dOmipKQkSVJISIjD+0JCQuz7CiOnK/vfffedlixZkmP8+++/16JFi1wVFwAAAFBgERERslqt9i0+Pj7X42w2m2677TaNHTtWdevWVc+ePdWjRw9NmzbtmsfsSk4n+0OGDFFWVlaOcZvNRpUfAAAAeXYt+uwfOXJEqamp9m3o0KG5xhIWFqbq1as7jFWrVk2HDx+WJIWGhkqSkpOTHY5JTk627yuMnE729+3bl+NCSFLVqlW1f/9+V8UFAAAAFFhgYKDDltsSHkm66667tGfPHoexvXv3qnz58tKlm3VDQ0O1bNky+/60tDStX79e0dHRJp9F/jmd7FutVv322285xvfv36+AgABXxQUAAABPV4j67Pfv31/r1q3T2LFjtX//fs2ZM0fTp09Xr169JEkWi0X9+vXTmDFj9PXXX2vbtm16/PHHFR4errZt25pzfVzA6WS/TZs26tevn3799Vf72P79+/Xcc8/pwQcfdHV8AAAAgOkaNGig+fPn65NPPlHNmjX18ssva+LEiYqLi7MfM2jQID377LPq2bOnGjRooPT0dC1evFh+fn5ujf1KLIZhOPV7T2pqqpo3b66NGzfqpptukiT9/vvvuvvuuzVv3jwFBQWZFatp0tLSZLVa1URt5G0p7u5wAAAAXOqikakf9JVSU1MVGBjo7nDsudetXceqmI85iXLWhfPaOvOFQnPO7uJ0602r1ao1a9YoISFBv/zyi/z9/XXrrbeqcePG5kQIAAAAIF+cTvZ1ac1Ss2bN1KxZM9dHBAAAgOtCYXqCrqfK85r9tWvX6ptvvnEYmzVrlqKiolSuXDn17NlTGRkZZsQIAAAAIB/ynOyPHj1aO3bssL/etm2bunfvrpiYGA0ZMkQLFy687EMKAAAAgBwKUTceT5XnZTxbtmzRyy+/bH/96aefqmHDhnrvvfekS08nGzFihEaOHGlOpADyJO2RO9wdgl3gJ+vcHQIAANe1PCf7p0+fVkhIiP31jz/+qBYtWthfN2jQQEeOHHF9hAAAAPBIFsOQxbnGkE7NDSeW8YSEhOjAgQOSpAsXLmjTpk26447/ryD++eefKl6ctpUAAABAYZHnyn7Lli01ZMgQjRs3TgsWLFCJEiV099132/dv3bpVFStWNCtOAHnE0hkAQJFh5tp6CvuSM8n+yy+/rHbt2umee+7RDTfcoA8//FA+Pj72/R988AGtOAEAAIBCJM/JfpkyZbRy5UqlpqbqhhtuULFixRz2f/HFF7rhhhvMiBEAAAAeiD775svXE3RzExwc7Ip4AAAAALhIvp6gCwAAABQYa/ZNl+duPAAAAACKFir7AAAAcAvW7JuPyj4AAADgoajsAwAAwD1Ys286KvsAAACAh6KyDwAAALdgzb75qOwDAAAAHorKPgAAANyDNfumo7IPAAAAeCgq+wAAAHAb1tabi8o+AAAA4KGo7AMAAMA9DOPvzay5QWUfAAAA8FRU9gEAAOAW9Nk3H5V9AAAAwENR2QcAAIB70GffdFT2AQAAAA9FZR8AAABuYbH9vZk1N6jsAwAAAB6Lyj4AAADcgzX7pqOyDwAAAHioIpXsv/rqq7JYLOrXr5997Pz58+rVq5dKly6tG264Qe3bt1dycrJb4wQAAMDVZffZN2tDEUr2N2zYoHfffVe33nqrw3j//v21cOFCffHFF/rxxx917NgxtWvXzm1xAgAAAIVFkUj209PTFRcXp/fee0+lSpWyj6empur999/XhAkTdN9996levXqaMWOG1qxZo3Xr1rk1ZgAAAFyFYZi7oWjcoNurVy+1atVKMTExGjNmjH08MTFRmZmZiomJsY9VrVpVkZGRWrt2re64445c58vIyFBGRob9dVpamslnAE925Mua7g7BQUSH7e4OAQAAFBKFPtn/9NNPtWnTJm3YsCHHvqSkJPn4+CgoKMhhPCQkRElJSZedMz4+XqNGjTIlXgAAAOSNmWvrWbP/t0K9jOfIkSPq27evZs+eLT8/P5fNO3ToUKWmptq3I0eOuGxuAAAAoLAo1JX9xMREnThxQrfddpt9LCsrSytXrtTbb7+tJUuW6MKFCzpz5oxDdT85OVmhoaGXndfX11e+vr6mx4/rA8tmAADIJ/rsm65QJ/tNmzbVtm3bHMa6deumqlWravDgwYqIiFDx4sW1bNkytW/fXpK0Z88eHT58WNHR0W6KGgAAACgcCnWyX7JkSdWs6XjzY0BAgEqXLm0f7969uwYMGKDg4GAFBgbq2WefVXR09GVvzgUAAEDhwJp98xXqZD8v3nzzTXl5eal9+/bKyMhQbGys3nnnHXeHBQAAALhdkUv2f/jhB4fXfn5+mjJliqZMmeK2mAAAAJAPZvbDp8++VNi78QAAAADIvyJX2QcAAIBnYM2++ajsAwAAAP/w6quvymKxqF+/fvax8+fPq1evXipdurRuuOEGtW/fXsnJyW6NMy9I9gEAAOAehslbPmzYsEHvvvuubr31Vofx/v37a+HChfriiy/0448/6tixY2rXrp1rroOJSPYBAADgFtnLeMzanJWenq64uDi99957KlWqlH08NTVV77//viZMmKD77rtP9erV04wZM7RmzRqtW7fOtRfFxUj2AQAAAEm9evVSq1atFBMT4zCemJiozMxMh/GqVasqMjJSa9eudUOkeccNugAAAHAPm/H3ZtbcktLS0hyGfX195evrm+PwTz/9VJs2bdKGDRty7EtKSpKPj4+CgoIcxkNCQpSUlOTy0F2Jyj4AAAA8VkREhKxWq32Lj4/PccyRI0fUt29fzZ49W35+fm6J0yxU9gEAAOAeBbiRNk9zX0rkAwMD7cO5VfUTExN14sQJ3XbbbfaxrKwsrVy5Um+//baWLFmiCxcu6MyZMw7V/eTkZIWGhpp0Aq5Bsg8AAACPFRgY6JDs56Zp06batm2bw1i3bt1UtWpVDR48WBERESpevLiWLVum9u3bS5L27Nmjw4cPKzo62tT4C4pkHwAAAG5hMfHhVxYnji1ZsqRq1qzpMBYQEKDSpUvbx7t3764BAwYoODhYgYGBevbZZxUdHa077rjDxZG7Fsk+AAAAcBVvvvmmvLy81L59e2VkZCg2NlbvvPOOu8O6KpJ9AAAAuIdh/L2ZNXcB/PDDDw6v/fz8NGXKFE2ZMqWAgV1bdOMBAAAAPBSVfQAAALhFfp90m9e5QWUfAAAA8FhU9gEAAOAe16DP/vWOyj4AAADgoajsAwAAwC0shiGLSd14zJq3qKGyDwAAAHgoKvsAAABwD9ulzay5QWUfAAAA8FRU9gEAAOAWrNk3H8k+ipx6mwvX3+US6/IHMgAAUDiR7AMAAMA96LNvOkqSAAAAgIeiso8ih2UzAAB4CMP4ezNrblDZBwAAADwVlX0AAAC4hcX4ezNrblDZBwAAADwWlX0AAAC4B2v2TUdlHwAAAPBQVPYBAADgFhbb35tZc4PKPgAAAOCxqOwDAADAPVizbzoq+wAAAICHorIPAAAA9zAubWbNDSr7AAAAgKeisg8AAAC3sBiGLCatrTdr3qKGyj4AAADgoajsAwAAwD3oxmM6KvsAAACAh6KyDwAAAPcwJJn1pFsK+xKVfQAAAMBzUdkHAACAW9CNx3xU9gEAAAAPRWUfAAAA7mGY2DWHwr5EZR8AAADwXFT2AQAA4B702TcdlX0AAADAQ1HZBwAAgHvYJFlMnBtU9gEAAABPVaiT/fj4eDVo0EAlS5ZUuXLl1LZtW+3Zs8fhmPPnz6tXr14qXbq0brjhBrVv317JycluixkAAAB5k91n36wNhTzZ//HHH9WrVy+tW7dOCQkJyszMVLNmzXT27Fn7Mf3799fChQv1xRdf6Mcff9SxY8fUrl07t8YNAAAAFAaFes3+4sWLHV7PnDlT5cqVU2Jioho3bqzU1FS9//77mjNnju677z5J0owZM1StWjWtW7dOd9xxh5siBwAAwFXRjcd0hTrZ/7fU1FRJUnBwsCQpMTFRmZmZiomJsR9TtWpVRUZGau3atZdN9jMyMpSRkWF/nZaWZnrsRdmSY1vcHYKD2PA67g4BAACgSCjUy3j+yWazqV+/frrrrrtUs2ZNSVJSUpJ8fHwUFBTkcGxISIiSkpIuO1d8fLysVqt9i4iIMD1+AAAA/Et2Zd+sDUUn2e/Vq5e2b9+uTz/9tMBzDR06VKmpqfbtyJEjLokRAAAAKEyKxDKe3r1765tvvtHKlSt100032cdDQ0N14cIFnTlzxqG6n5ycrNDQ0MvO5+vrK19fX9Pj9hQsmwEAAKZgzb7pCnVl3zAM9e7dW/Pnz9fy5csVFRXlsL9evXoqXry4li1bZh/bs2ePDh8+rOjoaDdEDAAAgKLIU1u+F+pkv1evXvr44481Z84clSxZUklJSUpKStJff/0lSbJarerevbsGDBigFStWKDExUd26dVN0dDSdeAAAAAo7m8mbEzy15bvFMArv3zgsltyfnzxjxgx17dpVuvQb1nPPPadPPvlEGRkZio2N1TvvvHPFZTz/lpaWJqvVqiZqI29LcZfFDwAAUBhcNDL1g75SamqqAgMD3R2OPfdqWuU5eRczZ2n1xawMLdvzRr7P+eTJkypXrpx+/PFHe8v3smXLas6cOerQoYMkaffu3apWrdoVu0C6W6Fes5+X30P8/Pw0ZcoUTZky5ZrEBAAAANcw80m3BZ3XVS3f3a1QJ/sAAABAQfz7eUp5adTiypbv7lao1+wDAADAg12DPvsREREOz1eKj4+/aliubPnublT2AQAA4LGOHDnisGb/alV9V7d8dzcq+wAAAHAPm2HuJikwMNBhu1yy76kt36nsAwAA4LrXq1cvzZkzR1999ZW95bsutXr39/d3aPkeHByswMBAPfvss4W+5TvJPgAAANyjED1Bd+rUqZKkJk2aOIz/s+X7m2++KS8vL7Vv396h5XthRrIPAAAANzEx2Zdz83pqy3fW7AMAAAAeiso+AAAA3KMQLePxVFT2AQAAAA9FZR8AAADuYTOcXlvv3Nygsg8AAAB4KCr7AAAAcA/D9vdm1tygsg8AAAB4Kir7AAAAcA+68ZiOyj4AAADgoajsAwAAwD3oxmM6KvsAAACAh6KyDwAAAPdgzb7pqOwDAAAAHorKPgAAANzDMLECT2FforIPAAAAeC4q+wAAAHAP1uybjmS/kDq+oJq7Q7ALa7vL3SEAAAAgH0j2AQAA4B42mySbiXODNfsAAACAh6KyX0ixdAYAAHg81uybjso+AAAA4KGo7AMAAMA9qOybjso+AAAA4KGo7AMAAMA9bIZ5j7q1UdkXlX0AAADAc1HZBwAAgFsYhk2GYU4/fLPmLWqo7AMAAAAeiso+AAAA3MMwzFtbTzceico+AAAA4Lmo7AMAAMA9DBO78VDZl6jsAwAAAJ6Lyj4AAADcw2aTLCZ1zaEbj0RlHwAAAPBcVPYBAADgHqzZNx2VfQAAAMBDUdkHAACAWxg2mwyT1uzzBN2/UdkHAAAAPBSVfQAAALgHa/ZNR2UfAAAA8FBU9gEAAOAeNkOyUNk3E5V9AAAAwENR2QcAAIB7GIYks56gS2VfVPYBAAAAz0VlHwAAAG5h2AwZJq3ZN6jsS1T2AQAAAM/lMcn+lClTVKFCBfn5+alhw4b6+eef3R0SAAAArsSwmbvBM5L9zz77TAMGDNCIESO0adMm1a5dW7GxsTpx4oS7QwMAAEAR4mkFZI9I9idMmKAePXqoW7duql69uqZNm6YSJUrogw8+cHdoAAAAuAzDZpi6OcsTC8hFPtm/cOGCEhMTFRMTYx/z8vJSTEyM1q5dm+t7MjIylJaW5rABAADg+uaJBeQin+z/8ccfysrKUkhIiMN4SEiIkpKScn1PfHy8rFarfYuIiLhG0QIAAMCuEK3Zz08BuSi4LltvDh06VAMGDLC/Tk1NVWRkpC4qU6JLEwAA8DAXlSkVwnaUZuZe2ef87xUcvr6+8vX1zXH8lQrIu3fvNifIa6DIJ/tlypRRsWLFlJyc7DCenJys0NDQXN/z7//I2V8Eq/WdydECAAC4z59//imr1eruMOTj46PQ0FCtTjI397rhhhtyrOAYMWKERo4caernFiZFPtn38fFRvXr1tGzZMrVt21aSZLPZtGzZMvXu3TtPc4SHh+vIkSMqWbKkLBZLvmNJS0tTRESEjhw5osDAwHzP44m4NpfHtbkyrs/lcW2ujOtzeVybK/PE62MYhv7880+Fh4e7OxRJkp+fnw4cOKALFy6Y+jmGYeTI7XKr6iufBeSioMgn+5I0YMAAdenSRfXr19ftt9+uiRMn6uzZs+rWrVue3u/l5aWbbrrJZfEEBgZ6zA8HV+PaXB7X5sq4PpfHtbkyrs/lcW2uzNOuT2Go6P+Tn5+f/Pz83B2GnSsKyIWRRyT7Dz/8sE6ePKnhw4crKSlJderU0eLFi3OsuQIAAAAup6AF5MLII5J9Serdu3eR/q0LAAAA7uWJBWSPSfYLA19fX40YMeKya8GuZ1yby+PaXBnX5/K4NlfG9bk8rs2VcX2ub55WQLYYha0HEwAAAACXKPIP1QIAAACQO5J9AAAAwEOR7AMAAAAeimTfRaZMmaIKFSrIz89PDRs21M8//+zukAqF+Ph4NWjQQCVLllS5cuXUtm1b7dmzx91hFUqvvvqqLBaL+vXr5+5QCoWjR4/qscceU+nSpeXv769atWpp48aN7g6rUMjKytKwYcMUFRUlf39/VaxYUS+//LKux1uwVq5cqdatWys8PFwWi0ULFixw2G8YhoYPH66wsDD5+/srJiZG+/btc1u819qVrk9mZqYGDx6sWrVqKSAgQOHh4Xr88cd17Ngxt8Z8rVzta+efnn76aVksFk2cOPGaxgi4Asm+C3z22WcaMGCARowYoU2bNql27dqKjY3ViRMn3B2a2/3444/q1auX1q1bp4SEBGVmZqpZs2Y6e/asu0MrVDZs2KB3331Xt956q7tDKRROnz6tu+66S8WLF9eiRYu0c+dOvfHGGypVqpS7QysUxo0bp6lTp+rtt9/Wrl27NG7cOI0fP16TJ092d2jX3NmzZ1W7dm1NmTIl1/3jx4/XpEmTNG3aNK1fv14BAQGKjY3V+fPnr3ms7nCl63Pu3Dlt2rRJw4YN06ZNmzRv3jzt2bNHDz74oFtivdau9rWTbf78+Vq3bl2hefIs4DQDBXb77bcbvXr1sr/OysoywsPDjfj4eLfGVRidOHHCkGT8+OOP7g6l0Pjzzz+NypUrGwkJCcY999xj9O3b190hud3gwYONRo0auTuMQqtVq1bGE0884TDWrl07Iy4uzm0xFQaSjPnz59tf22w2IzQ01HjttdfsY2fOnDF8fX2NTz75xE1Rus+/r09ufv75Z0OScejQoWsWV2FwuWvz+++/GzfeeKOxfft2o3z58sabb77plviAgqCyX0AXLlxQYmKiYmJi7GNeXl6KiYnR2rVr3RpbYZSamipJCg4OdncohUavXr3UqlUrh6+h693XX3+t+vXr6z//+Y/KlSununXr6r333nN3WIXGnXfeqWXLlmnv3r2SpF9++UWrV69WixYt3B1aoXLgwAElJSU5fG9ZrVY1bNiQn8+XkZqaKovFoqCgIHeH4nY2m02dO3fWwIEDVaNGDXeHA+QbD9UqoD/++ENZWVk5nqwWEhKi3bt3uy2uwshms6lfv3666667VLNmTXeHUyh8+umn2rRpkzZs2ODuUAqV3377TVOnTtWAAQP0wgsvaMOGDerTp498fHzUpUsXd4fndkOGDFFaWpqqVq2qYsWKKSsrS6+88ori4uLcHVqhkpSUJF36efxPISEh9n34f+fPn9fgwYP1yCOPKDAw0N3huN24cePk7e2tPn36uDsUoEBI9nHN9OrVS9u3b9fq1avdHUqhcOTIEfXt21cJCQny8/NzdziFis1mU/369TV27FhJUt26dbV9+3ZNmzaNZF/S559/rtmzZ2vOnDmqUaOGtmzZon79+ik8PJzrg3zJzMxUx44dZRiGpk6d6u5w3C4xMVFvvfWWNm3aJIvF4u5wgAJhGU8BlSlTRsWKFVNycrLDeHJyskJDQ90WV2HTu3dvffPNN1qxYoVuuukmd4dTKCQmJurEiRO67bbb5O3tLW9vb/3444+aNGmSvL29lZWV5e4Q3SYsLEzVq1d3GKtWrZoOHz7stpgKk4EDB2rIkCHq1KmTatWqpc6dO6t///6Kj493d2iFSvbPYH4+X1l2on/o0CElJCRQ1Ze0atUqnThxQpGRkfafz4cOHdJzzz2nChUquDs8wCkk+wXk4+OjevXqadmyZfYxm82mZcuWKTo62q2xFQaGYah3796aP3++li9frqioKHeHVGg0bdpU27Zt05YtW+xb/fr1FRcXpy1btqhYsWLuDtFt7rrrrhwtWvfu3avy5cu7LabC5Ny5c/LycvzxXaxYMdlsNrfFVBhFRUUpNDTU4edzWlqa1q9fz8/nS7IT/X379mnp0qUqXbq0u0MqFDp37qytW7c6/HwODw/XwIEDtWTJEneHBziFZTwuMGDAAHXp0kX169fX7bffrokTJ+rs2bPq1q2bu0Nzu169emnOnDn66quvVLJkSfs6WavVKn9/f3eH51YlS5bMce9CQECASpcufd3f09C/f3/deeedGjt2rDp27Kiff/5Z06dP1/Tp090dWqHQunVrvfLKK4qMjFSNGjW0efNmTZgwQU888YS7Q7vm0tPTtX//fvvrAwcOaMuWLQoODlZkZKT69eunMWPGqHLlyoqKitKwYcMUHh6utm3bujXua+VK1ycsLEwdOnTQpk2b9M033ygrK8v+Mzo4OFg+Pj5ujNx8V/va+fcvPsWLF1doaKiqVKnihmiBAnB3OyBPMXnyZCMyMtLw8fExbr/9dmPdunXuDqlQkJTrNmPGDHeHVijRevP/LVy40KhZs6bh6+trVK1a1Zg+fbq7Qyo00tLSjL59+xqRkZGGn5+fcfPNNxsvvviikZGR4e7QrrkVK1bk+jOmS5cuhnGp/eawYcOMkJAQw9fX12jatKmxZ88ed4d9zVzp+hw4cOCyP6NXrFjh7tBNd7WvnX+j9SaKKotxPT5yEQAAALgOsGYfAAAA8FAk+wAAAICHItkHAAAAPBTJPgAAAOChSPYBAAAAD0WyDwAAAHgokn0AAADAQ5HsAwAAAB6KZB8A3GDkyJGqU6dOgeY4ePCgLBaLtmzZku85Tp06pXLlyungwYN5Ov7ChQuqUKGCNm7cmO/PBABcOyT7AIoMi8VyxW3kyJHXLJYmTZqoX79+1+zzzPLKK6+oTZs2qlChQp6O9/Hx0fPPP6/BgwebHhsAoOC83R0AAOTV8ePH7f/+7LPPNHz4cO3Zs8c+dsMNN9j/bRiGsrKy5O3Nj7nLOXfunN5//30tWbLEqffFxcXpueee044dO1SjRg3T4gMAFByVfQBFRmhoqH2zWq2yWCz217t371bJkiW1aNEi1atXT76+vlq9erW6du2qtm3bOszTr18/NWnSxP7aZrMpPj5eUVFR8vf3V+3atfXll18WKNbBgwfrlltuUYkSJXTzzTdr2LBhyszMzHHcu+++q4iICJUoUUIdO3ZUamqqw/7//e9/qlatmvz8/FS1alW98847l/3M06dPKy4uTmXLlpW/v78qV66sGTNmXPb47777Tr6+vrrjjjvsY6NHj1Z4eLhOnTplH2vVqpXuvfde2Ww2SVKpUqV011136dNPP3X6ugAAri1KXgA8ypAhQ/T666/r5ptvVqlSpfL0nvj4eH388ceaNm2aKleurJUrV+qxxx5T2bJldc899+QrjpIlS2rmzJkKDw/Xtm3b1KNHD5UsWVKDBg2yH7N//359/vnnWrhwodLS0tS9e3c988wzmj17tiRp9uzZGj58uN5++23VrVtXmzdvVo8ePRQQEKAuXbrk+Mxhw4Zp586dWrRokcqUKaP9+/frr7/+umyMq1atUr169RzGXnzxRS1evFhPPvmk5s+frylTpmjNmjX65Zdf5OX1//Wh22+/XatWrcrXtQEAXDsk+wA8yujRo3X//ffn+fiMjAyNHTtWS5cuVXR0tCTp5ptv1urVq/Xuu+/mO9l/6aWX7P+uUKGCnn/+eX366acOyf758+c1a9Ys3XjjjZKkyZMnq1WrVnrjjTcUGhqqESNG6I033lC7du0kSVFRUdq5c6fefffdXJP9w4cPq27duqpfv779c6/k0KFDCg8PdxgrVqyYPv74Y9WpU0dDhgzRpEmT9L///U+RkZEOx4WHh+vQoUP5ujYAgGuHZB+AR8lOdPNq//79OnfuXI5fEC5cuKC6devmO47PPvtMkyZN0q+//qr09HRdvHhRgYGBDsdERkbaE31Jio6Ols1m0549e1SyZEn9+uuv6t69u3r06GE/5uLFi7Jarbl+5n//+1+1b99emzZtUrNmzdS2bVvdeeedl43xr7/+kp+fX47xm2++Wa+//rqeeuopPfzww3r00UdzHOPv769z587l+XoAANyDZB+ARwkICHB47eXlJcMwHMb+uXY+PT1dkvTtt986JN6S5Ovrm68Y1q5dq7i4OI0aNUqxsbGyWq369NNP9cYbb+R5juy43nvvPTVs2NBhX7FixXJ9T4sWLXTo0CF99913SkhIUNOmTdWrVy+9/vrruR5fpkwZnT59Otd9K1euVLFixXTw4EFdvHgxx43OKSkpKlu2bJ7PBwDgHtygC8CjlS1b1qGLjySHvvTVq1eXr6+vDh8+rEqVKjlsERER+frMNWvWqHz58nrxxRdVv359Va5cOdclL4cPH9axY8fsr9etWycvLy9VqVJFISEhCg8P12+//ZYjrqioqCueb5cuXfTxxx9r4sSJmj59+mWPrVu3rnbu3Jlj/LPPPtO8efP0ww8/6PDhw3r55ZdzHLN9+/YC/eUDAHBtUNkH4NHuu+8+vfbaa5o1a5aio6P18ccfOySqJUuW1PPPP6/+/fvLZrOpUaNGSk1N1U8//aTAwMBc18ZnO3nyZI4HWoWFhaly5co6fPiwPv30UzVo0EDffvut5s+fn+P9fn5+6tKli15//XWlpaWpT58+6tixo0JDQyVJo0aNUp8+fWS1WtW8eXNlZGRo48aNOn36tAYMGJBjvuHDh6tevXqqUaOGMjIy9M0336hatWqXjT82NlZDhw7V6dOn7Tcz//777/rvf/+rcePGqVGjRpoxY4YeeOABtWjRwqFrz6pVq3L9JQAAUMgYAFAEzZgxw7BarfbXK1asMCQZp0+fznHs8OHDjZCQEMNqtRr9+/c3evfubdxzzz32/TabzZg4caJRpUoVo3jx4kbZsmWN2NhY48cff7zs599zzz2GpBzbyy+/bBiGYQwcONAoXbq0ccMNNxgPP/yw8eabbzrEO2LECKN27drGO++8Y4SHhxt+fn5Ghw4djJSUFIfPmT17tlGnTh3Dx8fHKFWqlNG4cWNj3rx5hmEYxoEDBwxJxubNmw3DMIyXX37ZqFatmuHv728EBwcbbdq0MX777bcrXsfbb7/dmDZtmv06NG3a1IiNjTVsNpv9mGeffdaoWLGi8eeffxqGYRhr1qwxgoKCjHPnzl1xbgCA+1mMfy9mBQBcN7799lsNHDhQ27dvd2iteSUPP/ywateurRdeeMH0+AAABcMyHgC4jrVq1Ur79u3T0aNH83SPwoULF1SrVi3179//msQHACgYKvsAAACAh6IbDwAAAOChSPYBAAAAD0WyDwAAAHgokn0AAADAQ5HsAwAAAB6KZB8AAADwUCT7AAAAgIci2QcAAAA8FMk+AAAA4KFI9gEAAAAP9X/HgTwVmN8g9gAAAABJRU5ErkJggg==", + "text/plain": [ + "<Figure size 800x600 with 2 Axes>" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "batch_src, batch_labels = map(lambda x: x.to(device), mkbatch(1<<10))\n", + "model.eval()\n", + "with torch.no_grad():\n", + " output = model(batch_src)\n", + "\n", + "# Flatten the arrays to 1D\n", + "x = batch_labels.detach().to(torch.float16).cpu().numpy().flatten()\n", + "y = output.detach().to(torch.float16).cpu().numpy().flatten()\n", + "\n", + "# Define the number of vertices and number of bins per dimension\n", + "bins_y = 10 * NVTXS # 10 * nvtxs for y-bin size\n", + "\n", + "# Initialize the 2D array (matrix) to store the counts\n", + "count_matrix = np.zeros((NVTXS, bins_y), dtype=int)\n", + "\n", + "# Process the data: Map x to rows and floor(y*10) to columns\n", + "for xi, yi in zip(x, y):\n", + " row = int(xi) # Use integer value of x for row index\n", + " col = int(np.floor(yi * 10)) # Map y values to column by flooring and scaling by 10\n", + " if 0 <= row < NVTXS and 0 <= col < bins_y: # Ensure valid indices\n", + " count_matrix[row, col] += 1\n", + "\n", + "# Transpose the matrix\n", + "count_matrix = count_matrix.T\n", + "\n", + "# Plot the heatmap\n", + "plt.figure(figsize=(8, 6))\n", + "plt.imshow(count_matrix, cmap='viridis', origin='lower', interpolation='nearest', aspect='auto')\n", + "\n", + "# Set the labels and title\n", + "plt.ylabel('Scaled Predicted Output (y)')\n", + "plt.xlabel('True Labels (x)')\n", + "plt.title('True Labels vs Scaled Predicted Output (Heatmap)')\n", + "\n", + "# Add a colorbar for reference\n", + "plt.colorbar(label='Count')\n", + "\n", + "# Show the plot\n", + "plt.tight_layout()\n", + "plt.show()" ] } ], |