diff options
Diffstat (limited to 'transformer_shortest_paths.ipynb')
| -rw-r--r-- | transformer_shortest_paths.ipynb | 1270 |
1 files changed, 690 insertions, 580 deletions
diff --git a/transformer_shortest_paths.ipynb b/transformer_shortest_paths.ipynb index 27efa7f..2303603 100644 --- a/transformer_shortest_paths.ipynb +++ b/transformer_shortest_paths.ipynb @@ -1,607 +1,717 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "LPphBnKR-aWF" - }, - "source": [ - "# Step 0: Imports" - ] - }, - { - "cell_type": "markdown", - "source": [ - "\n", - "I need to do something simpler first.\n", - "\n", - "\n", - "1. Train a transformer to output 1 if token x is in the input, and 0 else.\n", - "\n", - "2. Train a transformer to output 1 if token x and token y are both in the input and 0 else.\n", - "\n", - "3. Train a transformer to output 1 if token x and token y are adjacent in the input.\n", - "\n", - "4. Train a transformer to output 1 if token x and token y are adjacent in the input AND they're 2k, 2k+1\n", - "\n" - ], - "metadata": { - "id": "HscaSHV43vU0" - } - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ge5QvElvhCOw", - "outputId": "c7cdaefa-d6dc-44ad-c258-e4fb2aca97a5" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "imports complete\n" - ] - } - ], - "source": [ - "# imports\n", - "import numpy as np\n", - "from collections import deque\n", - "import pickle\n", - "from tqdm import tqdm\n", - "np.random.seed(42)\n", - "\n", - "import torch\n", - "import torch.nn as nn\n", - "import pickle\n", - "from math import sqrt\n", - "from torch.utils.data import DataLoader, TensorDataset\n", - "import matplotlib.pyplot as plt\n", - "torch.manual_seed(42)\n", - "\n", - "import os\n", - "\n", - "print(\"imports complete\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "lylOX2POPwFL" - }, - "outputs": [], - "source": [ - "SEQ_LEN = 32\n", - "\n", - "PAD_TOKEN = 0\n", - "AVG_DEG = 2\n", - "MAX_VTXS = SEQ_LEN//AVG_DEG - 1\n", - "# vertices are labelled 1,2,...,63\n", - "# we also have a padding token which is 0.\n", - "\n", - "INF = MAX_VTXS # represents unreachability" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "gKt-yIpDebF1" - }, - "source": [ - "# Step 1: Generate synthetic data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "1IbzGIWseK3E", - "outputId": "a3cbc233-358c-4e17-ea6e-f4e9349d886b" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "100%|██████████| 1/1 [00:14<00:00, 14.42s/it]\n" - ] - } - ], - "source": [ - "# original task data\n", - "NTRAIN1 = 100_000\n", - "# the data will be edge lists\n", - "# like this: [1 3 1 5 2 4 0 0 0 0]\n", - "# this represents edges (1,3), (1,5) (2,4)\n", - "# (the zeros are just padding tokens)\n", - "\n", - "# the label is the shortest distance from vtx 1 to vtx 2\n", - "# or \"INF\" if no path exists\n", - "\n", - "# fine tuning data\n", - "NTRAIN2 = 2000\n", - "# I haven't totally figured out how to do the fine tuning yet.\n", - "# So don't worry about this yet.\n", - "\n", - "def random_graph(n):\n", - " edge_list = []\n", - " adjacencies = [set() for _ in range(n+1)]\n", - " indices = np.random.randint(n, size=(AVG_DEG*(n-1)))+1\n", - " for i in range(0, len(indices), 2):\n", - " u = indices[i]\n", - " v = indices[i + 1]\n", - " if u != v:\n", - " edge_list += [u,v]\n", - " adjacencies[u].add(v)\n", - " adjacencies[v].add(u)\n", - "\n", - " if np.random.random() < 0.25:\n", - " edge_list += [1,2]\n", - " adjacencies[1].add(2)\n", - " adjacencies[2].add(1)\n", - "\n", - " edge_list += [PAD_TOKEN]*(SEQ_LEN-len(edge_list))\n", - " return edge_list, adjacencies\n", - "\n", - "\"\"\"\n", - "input: G, represented as an adjacency list\n", - "output: [INF]+[d(1,i) for i in range(n)] if target=None\n", - "if target is set to some value, then we instead just output that specific distance\n", - "\"\"\"\n", - "def SSSP(G, target=None):\n", - " dist = [INF for _ in G]\n", - " dist[1] = 0\n", - " frontier = deque()\n", - " frontier.append(1)\n", - " while len(frontier) > 0:\n", - " vtx = frontier.popleft()\n", - " for x in G[vtx]:\n", - " if dist[x] == INF:\n", - " dist[x] = 1 + dist[vtx]\n", - " frontier.append(x)\n", - " if x == target:\n", - " return dist[target]\n", - " if target is not None:\n", - " return dist[target]\n", - " else:\n", - " return dist\n", - "\n", - "def fake_SSSP(G, target=None):\n", - " return 2 in G[1]\n", - "\n", - "graphs1 = []\n", - "distance1 = []\n", - "\n", - "graphs2 = []\n", - "distances2 = []\n", - "\n", - "for n in tqdm(range(MAX_VTXS-1, MAX_VTXS)):\n", - " # for _ in range(NTRAIN1//MAX_VTXS):\n", - " for _ in range(NTRAIN1):\n", - " edge_list, adj_list = random_graph(n)\n", - " dist = SSSP(adj_list, target=2)\n", - "\n", - " graphs1.append(edge_list)\n", - " distance1.append(dist)\n", - "\n", - "# for n in range(8, MAX_VTXS//4):\n", - "# for _ in range(NTRAIN2//MAX_VTXS):\n", - "# edge_list, adj_list = random_graph(n)\n", - "# distances = SSSP(adj_list)\n", - "# graphs2.append(edge_list)\n", - "# distances2.append(distances)\n", - "\n", - "split1 = int(len(graphs1)*3/4)\n", - "split2 = int(len(graphs2)*3/4)\n", - "\n", - "all1 = list(zip(graphs1, distance1))\n", - "np.random.shuffle(all1)\n", - "graphs1, distance1 = zip(*all1)\n", - "\n", - "data = {\n", - " \"train1-data\": graphs1[:split1],\n", - " \"train1-labels\": distance1[:split1],\n", - " \"test1-data\": graphs1[split1:],\n", - " \"test1-labels\": distance1[split1:]\n", - " # \"train2-data\": graphs2[:split2],\n", - " # \"train2-labels\": distances2[:split2],\n", - " # \"test2-data\": graphs2[split2:],\n", - " # \"test2-labels\": distances2[split2:]\n", - "}\n", - "\n", - "with open('data.pkl', 'wb') as file:\n", - " pickle.dump(data, file)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "EpDBxcgaIPpJ", - "outputId": "37cf9577-8cd8-444c-ec1a-c6f4b6061b7f" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "dataset size = 45MB\n" - ] - } - ], - "source": [ - "print(f\"dataset size = {os.path.getsize('data.pkl')//(1024*1024)}MB\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Q3Cg_8UQep8g" - }, - "source": [ - "# Step 2: Define Transformer Model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "tLOWhg_CeWzH" - }, - "outputs": [], - "source": [ - "class TransformerModel(nn.Module):\n", - " def __init__(self, input_dim, model_dim, output_dim, num_heads, num_layers, seq_len, device, dropout=0.1):\n", - " super().__init__()\n", - " self.embedding = nn.Embedding(input_dim, model_dim)\n", - " self.model_dim = model_dim\n", - " self.seq_len = seq_len\n", - " self.device = device\n", - "\n", - " # weight sharing\n", - " encoder_layer = nn.TransformerEncoderLayer(d_model=model_dim, nhead=num_heads,\n", - " dim_feedforward=model_dim*4,\n", - " dropout=dropout, batch_first=True)\n", - " self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers)\n", - "\n", - " self.fc_out = nn.Linear(model_dim*seq_len, output_dim)\n", - "\n", - " # def positional_encoding(self, batch_size):\n", - " # pos_encoding = torch.arange(self.seq_len, device=self.device).unsqueeze(1)\n", - " # pos_encoding = pos_encoding.float().unsqueeze(0).repeat(batch_size, 1, 1)\n", - " # return pos_encoding\n", - "\n", - " def positional_encoding(self, batch_size):\n", - " position = torch.arange(self.seq_len, dtype=torch.float, device=self.device).unsqueeze(1)\n", - " div_term = torch.exp(torch.arange(0, self.model_dim, 2, dtype=torch.float, device=self.device) *\n", - " -(torch.log(torch.tensor(500.0)) / self.model_dim))\n", - "\n", - " pos_encoding = torch.zeros(self.seq_len, self.model_dim, device=self.device)\n", - " pos_encoding[:, 0::2] = torch.sin(position * div_term)\n", - " pos_encoding[:, 1::2] = torch.cos(position * div_term)\n", - " pos_encoding = pos_encoding.unsqueeze(0).repeat(batch_size, 1, 1)\n", - " return pos_encoding\n", - "\n", - " def forward(self, src, key_padding_mask):\n", - " batch_size, src_len = src.size(0), src.size(1)\n", - " src_pos = self.positional_encoding(batch_size)\n", - " embed = self.embedding(src)\n", - " src = embed * sqrt(self.model_dim) + src_pos\n", - "\n", - " output = self.transformer_encoder(src, None, src_key_padding_mask=key_padding_mask)\n", - " flat_output = torch.flatten(output, start_dim=1, end_dim=2)\n", - " output = self.fc_out(flat_output)\n", - " return output\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "bpIeg86S-hBb" - }, - "source": [ - "# Step 3: Load Data" - ] + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "LPphBnKR-aWF" + }, + "source": [ + "# Step 0: Imports" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "ge5QvElvhCOw", + "outputId": "c7cdaefa-d6dc-44ad-c258-e4fb2aca97a5" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "kWXvJRDYgFVP", - "colab": { - "base_uri": "https://localhost:8080/" - }, - "outputId": "c13adb9d-6565-43b5-8437-20cef3dc0d16" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Trainable parameters in the model: 26K\n", - "train BASELINEs: 39.4069\n" - ] - } - ], - "source": [ - "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", - "assert device.type == 'cuda', \"CUDA is not available. Please check your GPU setup.\"\n", - "\n", - "# PARAMS\n", - "VOCAB_SIZE = 1+MAX_VTXS # one more than the max number of vertices\n", - "MODEL_DIM = 32 # Dimension of model (embedding and transformer)\n", - "NEPOCHS = 10\n", - "BSZ = 32\n", - "LR = 0.01\n", - "NHEADS = 4\n", - "NLAYERS = 2\n", - "PAD_TOKEN = 0\n", - "model = TransformerModel(input_dim=VOCAB_SIZE, model_dim=MODEL_DIM,\n", - " output_dim=1, num_heads=NHEADS,\n", - " num_layers=NLAYERS, seq_len=SEQ_LEN,\n", - " device=device).to(device)\n", - "\n", - "with open(\"data.pkl\", \"rb\") as f:\n", - " data = pickle.load(f)\n", - "\n", - "train_data1 = data[\"train1-data\"]\n", - "train_label1 = data[\"train1-labels\"]\n", - "train_data_tensor = torch.tensor(train_data1, dtype=torch.long, device=device)\n", - "train_label_tensor = torch.tensor(train_label1, dtype=torch.float, device=device)\n", - "train_padding_mask = (train_data_tensor != PAD_TOKEN).bool().to(device)\n", - "train_dataset = TensorDataset(train_data_tensor, train_label_tensor, train_padding_mask)\n", - "train_loader = DataLoader(train_dataset, batch_size=BSZ, shuffle=True)\n", - "\n", - "test_data1 = data[\"test1-data\"]\n", - "test_label1 = data[\"test1-labels\"]\n", - "test_data_tensor = torch.tensor(test_data1, dtype=torch.long, device=device)\n", - "test_label_tensor = torch.tensor(test_label1, dtype=torch.float, device=device)\n", - "test_padding_mask = (test_data_tensor != PAD_TOKEN).bool().to(device)\n", - "test_dataset = TensorDataset(test_data_tensor, test_label_tensor, test_padding_mask)\n", - "test_loader = DataLoader(test_dataset, batch_size=BSZ, shuffle=True)\n", - "\n", - "criterion = nn.MSELoss()\n", - "optimizer = torch.optim.Adam(model.parameters(), lr=LR)\n", - "\n", - "train_err = []\n", - "test_err = []\n", - "\n", - "trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n", - "print(f\"Trainable parameters in the model: {trainable_params//1000}K\")\n", - "\n", - "train_baseline = ((train_label_tensor - train_label_tensor.mean())**2).mean().item()\n", - "print(f\"train BASELINEs: {train_baseline:.4f}\")" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "imports complete\n" + ] + } + ], + "source": [ + "# imports\n", + "import numpy as np\n", + "from collections import deque\n", + "import pickle\n", + "from tqdm import tqdm\n", + "np.random.seed(42)\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "import pickle\n", + "from math import sqrt\n", + "from torch.utils.data import DataLoader, TensorDataset\n", + "import matplotlib.pyplot as plt\n", + "torch.manual_seed(42)\n", + "\n", + "import os\n", + "\n", + "print(\"imports complete\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "lylOX2POPwFL" + }, + "outputs": [], + "source": [ + "SEQ_LEN = 32\n", + "\n", + "PAD_TOKEN = 0\n", + "AVG_DEG = 2\n", + "MAX_VTXS = SEQ_LEN//AVG_DEG - 1\n", + "# vertices are labelled 1,2,...,63\n", + "# we also have a padding token which is 0.\n", + "\n", + "INF = MAX_VTXS # represents unreachability" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gKt-yIpDebF1" + }, + "source": [ + "# Step 1: Generate synthetic data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "1IbzGIWseK3E", + "outputId": "a3cbc233-358c-4e17-ea6e-f4e9349d886b" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "205MvfJQQYya" - }, - "source": [ - "# Dad reccomended having more \"partial progress measures\" / having the model \"show it's work\".\n", - "# or creating a different easier training regiment to start." - ] - }, + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 1/1 [00:14<00:00, 14.42s/it]\n" + ] + } + ], + "source": [ + "# original task data\n", + "NTRAIN1 = 100_000\n", + "# the data will be edge lists\n", + "# like this: [1 3 1 5 2 4 0 0 0 0]\n", + "# this represents edges (1,3), (1,5) (2,4)\n", + "# (the zeros are just padding tokens)\n", + "\n", + "# the label is the shortest distance from vtx 1 to vtx 2\n", + "# or \"INF\" if no path exists\n", + "\n", + "# fine tuning data\n", + "NTRAIN2 = 2000\n", + "# I haven't totally figured out how to do the fine tuning yet.\n", + "# So don't worry about this yet.\n", + "\n", + "def random_graph(n):\n", + " edge_list = []\n", + " adjacencies = [set() for _ in range(n+1)]\n", + " indices = np.random.randint(n, size=(AVG_DEG*(n-1)))+1\n", + " for i in range(0, len(indices), 2):\n", + " u = indices[i]\n", + " v = indices[i + 1]\n", + " if u != v:\n", + " edge_list += [u,v]\n", + " adjacencies[u].add(v)\n", + " adjacencies[v].add(u)\n", + "\n", + " if np.random.random() < 0.25:\n", + " edge_list += [1,2]\n", + " adjacencies[1].add(2)\n", + " adjacencies[2].add(1)\n", + "\n", + " edge_list += [PAD_TOKEN]*(SEQ_LEN-len(edge_list))\n", + " return edge_list, adjacencies\n", + "\n", + "\"\"\"\n", + "input: G, represented as an adjacency list\n", + "output: [INF]+[d(1,i) for i in range(n)] if target=None\n", + "if target is set to some value, then we instead just output that specific distance\n", + "\"\"\"\n", + "def SSSP(G, target=None):\n", + " dist = [INF for _ in G]\n", + " dist[1] = 0\n", + " frontier = deque()\n", + " frontier.append(1)\n", + " while len(frontier) > 0:\n", + " vtx = frontier.popleft()\n", + " for x in G[vtx]:\n", + " if dist[x] == INF:\n", + " dist[x] = 1 + dist[vtx]\n", + " frontier.append(x)\n", + " if x == target:\n", + " return dist[target]\n", + " if target is not None:\n", + " return dist[target]\n", + " else:\n", + " return dist\n", + "\n", + "def fake_SSSP(G, target=None):\n", + " return 2 in G[1]\n", + "\n", + "graphs1 = []\n", + "distance1 = []\n", + "\n", + "graphs2 = []\n", + "distances2 = []\n", + "\n", + "for n in tqdm(range(MAX_VTXS-1, MAX_VTXS)):\n", + " # for _ in range(NTRAIN1//MAX_VTXS):\n", + " for _ in range(NTRAIN1):\n", + " edge_list, adj_list = random_graph(n)\n", + " dist = SSSP(adj_list, target=2)\n", + "\n", + " graphs1.append(edge_list)\n", + " distance1.append(dist)\n", + "\n", + "# for n in range(8, MAX_VTXS//4):\n", + "# for _ in range(NTRAIN2//MAX_VTXS):\n", + "# edge_list, adj_list = random_graph(n)\n", + "# distances = SSSP(adj_list)\n", + "# graphs2.append(edge_list)\n", + "# distances2.append(distances)\n", + "\n", + "split1 = int(len(graphs1)*3/4)\n", + "split2 = int(len(graphs2)*3/4)\n", + "\n", + "all1 = list(zip(graphs1, distance1))\n", + "np.random.shuffle(all1)\n", + "graphs1, distance1 = zip(*all1)\n", + "\n", + "data = {\n", + " \"train1-data\": graphs1[:split1],\n", + " \"train1-labels\": distance1[:split1],\n", + " \"test1-data\": graphs1[split1:],\n", + " \"test1-labels\": distance1[split1:]\n", + " # \"train2-data\": graphs2[:split2],\n", + " # \"train2-labels\": distances2[:split2],\n", + " # \"test2-data\": graphs2[split2:],\n", + " # \"test2-labels\": distances2[split2:]\n", + "}\n", + "\n", + "with open('data.pkl', 'wb') as file:\n", + " pickle.dump(data, file)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "metadata": {}, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "f8Zn33m7CxL5" - }, - "source": [ - "# Step 4: Train the Model for the first task" - ] - }, + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([75000, 32])\n", + "DONE\n" + ] + } + ], + "source": [ + "NTRAIN1 = 100000\n", + "\n", + "graphs1 = torch.randint(1, MAX_VTXS, (NTRAIN1, SEQ_LEN))\n", + "\n", + "# check if token 1 is in the graph\n", + "def silly_distance(graph):\n", + " return int(1 in graph)\n", + "\n", + "# check if both token 1 and token 2 are in the graph\n", + "def silly_distance2(graph):\n", + " return int(1 in graph and 2 in graph and 3 in graph and 4 in graph and 5 in graph)\n", + "\n", + "def silly_distance3(graph):\n", + " for i in range(len(graph)//2):\n", + " if graph[2*i] + graph[2*i+1] == 3:\n", + " return 1\n", + " return 0\n", + "\n", + "distance1 = [silly_distance3(graph) for graph in graphs1]\n", + "\n", + "split1 = int(len(graphs1)*3/4)\n", + "\n", + "data = {\n", + " \"train1-data\": graphs1[:split1],\n", + " \"train1-labels\": distance1[:split1],\n", + " \"test1-data\": graphs1[split1:],\n", + " \"test1-labels\": distance1[split1:]\n", + "}\n", + "\n", + "print(data[\"train1-data\"].shape)\n", + "\n", + "with open('data.pkl', 'wb') as file:\n", + " pickle.dump(data, file)\n", + "\n", + "print(\"DONE\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "metadata": { + "scrolled": true + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 486 - }, - "id": "pvTfzGmCeXU4", - "outputId": "0d3a20f3-23be-4c19-9eb6-46bfe11a48b1" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Epoch 1/10 \t Train Err: 28.8616 \t Test Err: 39.4354 \t baseline err: 39.4069\n", - "Epoch 2/10 \t Train Err: 39.8088 \t Test Err: 39.4255 \t baseline err: 39.4069\n", - "Epoch 3/10 \t Train Err: 39.7257 \t Test Err: 39.9765 \t baseline err: 39.4069\n", - "Epoch 4/10 \t Train Err: 39.4951 \t Test Err: 40.0988 \t baseline err: 39.4069\n", - "Epoch 5/10 \t Train Err: 39.4205 \t Test Err: 39.5148 \t baseline err: 39.4069\n" - ] - }, - { - "output_type": "error", - "ename": "KeyboardInterrupt", - "evalue": "", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m<ipython-input-17-3dc1bf4cf066>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_labels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0mtrain_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 11\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/_tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 519\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 520\u001b[0m )\n\u001b[0;32m--> 521\u001b[0;31m torch.autograd.backward(\n\u001b[0m\u001b[1;32m 522\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 523\u001b[0m )\n", - "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 287\u001b[0m \u001b[0;31m# some Python versions print out the first line of a multi-line function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 288\u001b[0m \u001b[0;31m# calls in the traceback and some print out the last line\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 289\u001b[0;31m _engine_run_backward(\n\u001b[0m\u001b[1;32m 290\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 291\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/autograd/graph.py\u001b[0m in \u001b[0;36m_engine_run_backward\u001b[0;34m(t_outputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m 767\u001b[0m \u001b[0munregister_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_register_logging_hooks_on_whole_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt_outputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 768\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 769\u001b[0;31m return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass\n\u001b[0m\u001b[1;32m 770\u001b[0m \u001b[0mt_outputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 771\u001b[0m ) # Calls into the C++ engine to run the backward pass\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], - "source": [ - "for epoch in range(NEPOCHS):\n", - " model.train() # set to training mode\n", - " train_loss = 0\n", - "\n", - " for batch_src, batch_labels, batch_padding_mask in train_loader:\n", - " optimizer.zero_grad()\n", - " output = model(batch_src, batch_padding_mask)\n", - " loss = criterion(output.squeeze(1), batch_labels)\n", - " train_loss += loss.item()/len(train_loader)\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # Evaluate performance\n", - " model.eval()\n", - " test_loss = 0\n", - "\n", - " with torch.no_grad():\n", - " for batch_src, batch_labels, batch_padding_mask in test_loader:\n", - " output = model(batch_src, batch_padding_mask)\n", - " loss = criterion(output.squeeze(1), batch_labels)\n", - " test_loss += loss.item()/len(test_loader)\n", - "\n", - " test_err.append(test_loss)\n", - " train_err.append(train_loss)\n", - " print(f\"Epoch {epoch + 1}/{NEPOCHS} \\t Train Err: {train_loss:.4f} \\t Test Err: {test_loss:.4f} \\t baseline err: {train_baseline:.4f}\")\n", - "\n", - "plt.figure(figsize=(10, 5))\n", - "plt.plot(test_err, label='Test', color='red')\n", - "plt.plot(train_err, label='Train', color='blue')\n", - "plt.title('Accuracy vs Epochs')\n", - "plt.xlabel('Epochs'); plt.ylabel('Accuracy')\n", - "plt.legend(); plt.grid()\n", - "plt.show()" + "data": { + "text/plain": [ + "0.1518" ] + }, + "execution_count": 76, + "metadata": {}, + "output_type": "execute_result" }, { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "v1hCiItHDWxJ" - }, - "outputs": [], - "source": [ - "## Q: why is this not working so well?\n", - "\n", - "## maybe first try a simpler problem: just give it points for distinguishing between distance 1 or not" + "data": { + "text/plain": [ + "0.1518" ] + }, + "execution_count": 75, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sum(distance1)/len(distance1)" + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "EpDBxcgaIPpJ", + "outputId": "37cf9577-8cd8-444c-ec1a-c6f4b6061b7f" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "LoGEmM5lH7_A" - }, - "outputs": [], - "source": [ - "batch_src, batch_labels, batch_padding_mask = next(iter(train_loader))\n", - "output = model(batch_src, batch_padding_mask)" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "dataset size = 49MB\n" + ] + } + ], + "source": [ + "print(f\"dataset size = {os.path.getsize('data.pkl')//(1024*1024)}MB\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Q3Cg_8UQep8g" + }, + "source": [ + "# Step 2: Define Transformer Model" + ] + }, + { + "cell_type": "code", + "execution_count": 107, + "metadata": { + "id": "tLOWhg_CeWzH" + }, + "outputs": [], + "source": [ + "class TransformerModel(nn.Module):\n", + " def __init__(self, input_dim, model_dim, output_dim, num_heads, num_layers, seq_len, device, dropout=0.1):\n", + " super().__init__()\n", + " self.embedding = nn.Embedding(input_dim, model_dim//2)\n", + " self.model_dim = model_dim\n", + " self.seq_len = seq_len\n", + " self.device = device\n", + "\n", + " encoder_layer = nn.TransformerEncoderLayer(d_model=model_dim, nhead=num_heads,\n", + " dim_feedforward=model_dim*4,\n", + " dropout=dropout, batch_first=True)\n", + " self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers)\n", + "\n", + " self.fc_out = nn.Linear(model_dim*seq_len, output_dim)\n", + " self.fancy_encoding = torch.repeat_interleave(torch.rand((1,SEQ_LEN // 2, model_dim // 2), device=device), 2, dim=1)\n", + " \n", + " def positional_encoding(self, batch_size):\n", + " position = torch.arange(self.seq_len, dtype=torch.float, device=self.device).unsqueeze(1)\n", + " div_term = torch.exp(torch.arange(0, self.model_dim, 2, dtype=torch.float, device=self.device) *\n", + " -(torch.log(torch.tensor(500.0)) / self.model_dim))\n", + "\n", + " pos_encoding = torch.zeros(self.seq_len, self.model_dim, device=self.device)\n", + " pos_encoding[:, 0::2] = torch.sin(position * div_term)\n", + " pos_encoding[:, 1::2] = torch.cos(position * div_term)\n", + " pos_encoding = pos_encoding.unsqueeze(0).repeat(batch_size, 1, 1)\n", + " return pos_encoding\n", + "\n", + " def forward(self, src, key_padding_mask):\n", + " batch_size, src_len = src.size(0), src.size(1)\n", + " # src_pos = self.positional_encoding(batch_size)\n", + " embed = self.embedding(src)\n", + " src = torch.cat((embed * sqrt(self.model_dim), torch.Tensor.repeat(self.fancy_encoding, (batch_size, 1, 1))), dim=2)\n", + "\n", + " output = self.transformer_encoder(src, None, src_key_padding_mask=key_padding_mask)\n", + " flat_output = torch.flatten(output, start_dim=1, end_dim=2)\n", + " output = self.fc_out(flat_output)\n", + " return output\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bpIeg86S-hBb" + }, + "source": [ + "# Step 3: Load Data" + ] + }, + { + "cell_type": "code", + "execution_count": 121, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "kWXvJRDYgFVP", + "outputId": "c13adb9d-6565-43b5-8437-20cef3dc0d16" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "hO8AhX3G7vF8", - "colab": { - "base_uri": "https://localhost:8080/" - }, - "outputId": "8f4a3ca6-db47-434d-95a4-4631bc73de62" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "1 \t 5.7\n", - "3 \t 5.7\n", - "15 \t 7.1\n", - "1 \t 5.7\n", - "2 \t 7.8\n", - "15 \t 7.1\n", - "1 \t 0.7\n", - "15 \t 5.7\n", - "5 \t 5.7\n", - "1 \t 5.7\n", - "1 \t 0.7\n", - "4 \t 5.7\n", - "2 \t 7.8\n", - "3 \t 5.7\n", - "3 \t 5.7\n", - "15 \t 7.8\n", - "15 \t 7.8\n", - "1 \t 5.7\n", - "3 \t 7.1\n", - "1 \t 5.7\n", - "3 \t 5.7\n", - "1 \t 7.1\n", - "1 \t 7.8\n", - "2 \t 5.7\n", - "1 \t 5.7\n", - "15 \t 7.1\n", - "6 \t 7.1\n", - "1 \t 5.7\n", - "1 \t 5.7\n", - "1 \t 5.7\n", - "15 \t 7.1\n", - "1 \t 7.1\n" - ] - } - ], - "source": [ - "for x,y in zip(batch_labels.tolist(), output.squeeze(1).tolist()):\n", - " print(f\"{int(x)} \\t {y:.1f}\")" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Trainable parameters in the model: 102K\n", + "train BASELINEs: 0.1290\n" + ] }, { - "cell_type": "code", - "source": [ - "batch_src[2]" - ], - "metadata": { - "id": "dRdUGbFmkPtK" - }, - "execution_count": null, - "outputs": [] + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_390590/1991115476.py:23: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n", + " train_data_tensor = torch.tensor(train_data1, dtype=torch.long, device=device)\n", + "/tmp/ipykernel_390590/1991115476.py:31: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n", + " test_data_tensor = torch.tensor(test_data1, dtype=torch.long, device=device)\n" + ] + } + ], + "source": [ + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "assert device.type == 'cuda', \"CUDA is not available. Please check your GPU setup.\"\n", + "\n", + "# PARAMS\n", + "VOCAB_SIZE = 1+MAX_VTXS # one more than the max number of vertices\n", + "MODEL_DIM = 64 # Dimension of model (embedding and transformer)\n", + "NEPOCHS = 50\n", + "BSZ = 512\n", + "LR = 0.001\n", + "NHEADS = 4\n", + "NLAYERS = 2\n", + "PAD_TOKEN = 0\n", + "model = TransformerModel(input_dim=VOCAB_SIZE, model_dim=MODEL_DIM,\n", + " output_dim=1, num_heads=NHEADS,\n", + " num_layers=NLAYERS, seq_len=SEQ_LEN,\n", + " device=device).to(device)\n", + "\n", + "with open(\"data.pkl\", \"rb\") as f:\n", + " data = pickle.load(f)\n", + "\n", + "train_data1 = data[\"train1-data\"]\n", + "train_label1 = data[\"train1-labels\"]\n", + "train_data_tensor = torch.tensor(train_data1, dtype=torch.long, device=device)\n", + "train_label_tensor = torch.tensor(train_label1, dtype=torch.float, device=device)\n", + "train_padding_mask = (train_data_tensor == PAD_TOKEN).bool().to(device)\n", + "train_dataset = TensorDataset(train_data_tensor, train_label_tensor, train_padding_mask)\n", + "train_loader = DataLoader(train_dataset, batch_size=BSZ, shuffle=True)\n", + "\n", + "test_data1 = data[\"test1-data\"]\n", + "test_label1 = data[\"test1-labels\"]\n", + "test_data_tensor = torch.tensor(test_data1, dtype=torch.long, device=device)\n", + "test_label_tensor = torch.tensor(test_label1, dtype=torch.float, device=device)\n", + "test_padding_mask = (test_data_tensor == PAD_TOKEN).bool().to(device)\n", + "test_dataset = TensorDataset(test_data_tensor, test_label_tensor, test_padding_mask)\n", + "test_loader = DataLoader(test_dataset, batch_size=BSZ, shuffle=True)\n", + "\n", + "criterion = nn.MSELoss()\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=LR)\n", + "\n", + "train_err = []\n", + "test_err = []\n", + "\n", + "trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "print(f\"Trainable parameters in the model: {trainable_params//1000}K\")\n", + "\n", + "train_baseline = ((train_label_tensor - train_label_tensor.mean())**2).mean().item()\n", + "print(f\"train BASELINEs: {train_baseline:.4f}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f8Zn33m7CxL5" + }, + "source": [ + "# Step 4: Train the Model for the first task" + ] + }, + { + "cell_type": "code", + "execution_count": 122, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 486 }, + "id": "pvTfzGmCeXU4", + "outputId": "0d3a20f3-23be-4c19-9eb6-46bfe11a48b1" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "LC6Xv3YfC0Rm" - }, - "source": [ - "# Step 5: Fine Tune" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/50 \t Train Err: 0.1621 \t Test Err: 0.1208 \t baseline err: 0.1290\n", + "Epoch 2/50 \t Train Err: 0.1266 \t Test Err: 0.1201 \t baseline err: 0.1290\n", + "Epoch 3/50 \t Train Err: 0.1224 \t Test Err: 0.1199 \t baseline err: 0.1290\n", + "Epoch 4/50 \t Train Err: 0.1190 \t Test Err: 0.1214 \t baseline err: 0.1290\n", + "Epoch 5/50 \t Train Err: 0.1167 \t Test Err: 0.1164 \t baseline err: 0.1290\n", + "Epoch 6/50 \t Train Err: 0.1154 \t Test Err: 0.1156 \t baseline err: 0.1290\n", + "Epoch 7/50 \t Train Err: 0.1146 \t Test Err: 0.1131 \t baseline err: 0.1290\n", + "Epoch 8/50 \t Train Err: 0.1140 \t Test Err: 0.1145 \t baseline err: 0.1290\n", + "Epoch 9/50 \t Train Err: 0.1135 \t Test Err: 0.1144 \t baseline err: 0.1290\n", + "Epoch 10/50 \t Train Err: 0.1134 \t Test Err: 0.1160 \t baseline err: 0.1290\n", + "Epoch 11/50 \t Train Err: 0.1134 \t Test Err: 0.1160 \t baseline err: 0.1290\n", + "Epoch 12/50 \t Train Err: 0.1129 \t Test Err: 0.1137 \t baseline err: 0.1290\n", + "Epoch 13/50 \t Train Err: 0.1131 \t Test Err: 0.1122 \t baseline err: 0.1290\n", + "Epoch 14/50 \t Train Err: 0.1125 \t Test Err: 0.1133 \t baseline err: 0.1290\n", + "Epoch 15/50 \t Train Err: 0.1121 \t Test Err: 0.1119 \t baseline err: 0.1290\n", + "Epoch 16/50 \t Train Err: 0.1120 \t Test Err: 0.1129 \t baseline err: 0.1290\n", + "Epoch 17/50 \t Train Err: 0.1123 \t Test Err: 0.1123 \t baseline err: 0.1290\n", + "Epoch 18/50 \t Train Err: 0.1120 \t Test Err: 0.1119 \t baseline err: 0.1290\n", + "Epoch 19/50 \t Train Err: 0.1117 \t Test Err: 0.1148 \t baseline err: 0.1290\n", + "Epoch 20/50 \t Train Err: 0.1119 \t Test Err: 0.1136 \t baseline err: 0.1290\n", + "Epoch 21/50 \t Train Err: 0.1117 \t Test Err: 0.1120 \t baseline err: 0.1290\n", + "Epoch 22/50 \t Train Err: 0.1114 \t Test Err: 0.1123 \t baseline err: 0.1290\n", + "Epoch 23/50 \t Train Err: 0.1111 \t Test Err: 0.1121 \t baseline err: 0.1290\n", + "Epoch 24/50 \t Train Err: 0.1093 \t Test Err: 0.1061 \t baseline err: 0.1290\n", + "Epoch 25/50 \t Train Err: 0.1044 \t Test Err: 0.1012 \t baseline err: 0.1290\n", + "Epoch 26/50 \t Train Err: 0.1012 \t Test Err: 0.1003 \t baseline err: 0.1290\n", + "Epoch 27/50 \t Train Err: 0.0985 \t Test Err: 0.0964 \t baseline err: 0.1290\n", + "Epoch 28/50 \t Train Err: 0.0957 \t Test Err: 0.0942 \t baseline err: 0.1290\n", + "Epoch 29/50 \t Train Err: 0.0947 \t Test Err: 0.0935 \t baseline err: 0.1290\n", + "Epoch 30/50 \t Train Err: 0.0931 \t Test Err: 0.0941 \t baseline err: 0.1290\n", + "Epoch 31/50 \t Train Err: 0.0920 \t Test Err: 0.0916 \t baseline err: 0.1290\n", + "Epoch 32/50 \t Train Err: 0.0893 \t Test Err: 0.0857 \t baseline err: 0.1290\n", + "Epoch 33/50 \t Train Err: 0.0868 \t Test Err: 0.0814 \t baseline err: 0.1290\n", + "Epoch 34/50 \t Train Err: 0.0827 \t Test Err: 0.0785 \t baseline err: 0.1290\n", + "Epoch 35/50 \t Train Err: 0.0770 \t Test Err: 0.0720 \t baseline err: 0.1290\n", + "Epoch 36/50 \t Train Err: 0.0713 \t Test Err: 0.0646 \t baseline err: 0.1290\n", + "Epoch 37/50 \t Train Err: 0.0642 \t Test Err: 0.0540 \t baseline err: 0.1290\n", + "Epoch 38/50 \t Train Err: 0.0588 \t Test Err: 0.0501 \t baseline err: 0.1290\n", + "Epoch 39/50 \t Train Err: 0.0543 \t Test Err: 0.0456 \t baseline err: 0.1290\n", + "Epoch 40/50 \t Train Err: 0.0488 \t Test Err: 0.0366 \t baseline err: 0.1290\n", + "Epoch 41/50 \t Train Err: 0.0416 \t Test Err: 0.0315 \t baseline err: 0.1290\n", + "Epoch 42/50 \t Train Err: 0.0360 \t Test Err: 0.0214 \t baseline err: 0.1290\n", + "Epoch 43/50 \t Train Err: 0.0305 \t Test Err: 0.0172 \t baseline err: 0.1290\n", + "Epoch 44/50 \t Train Err: 0.0239 \t Test Err: 0.0116 \t baseline err: 0.1290\n", + "Epoch 45/50 \t Train Err: 0.0205 \t Test Err: 0.0117 \t baseline err: 0.1290\n", + "Epoch 46/50 \t Train Err: 0.0181 \t Test Err: 0.0092 \t baseline err: 0.1290\n", + "Epoch 47/50 \t Train Err: 0.0164 \t Test Err: 0.0100 \t baseline err: 0.1290\n", + "Epoch 48/50 \t Train Err: 0.0155 \t Test Err: 0.0081 \t baseline err: 0.1290\n", + "Epoch 49/50 \t Train Err: 0.0141 \t Test Err: 0.0074 \t baseline err: 0.1290\n", + "Epoch 50/50 \t Train Err: 0.0129 \t Test Err: 0.0075 \t baseline err: 0.1290\n" + ] }, { - "cell_type": "markdown", - "metadata": { - "id": "JtTLXn4zC1z_" - }, - "source": [ - "# Step 6: Test generalization" + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1cAAAHWCAYAAACbsXOkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAACL8ElEQVR4nOzde3yO9R/H8dd972hjc5ht5jSnHMKcj0UJK+V87kD4UbJSK0UHh04kpKKUIoqIIiEZhYqcZg7lTM5zKIyNbbb798fXxtqw2b3dO7yfj8f1uO/7ur/XdX2uu+/Pz8f3e32+FpvNZkNEREREREQyxeroAERERERERPICJVciIiIiIiJ2oORKRERERETEDpRciYiIiIiI2IGSKxERERERETtQciUiIiIiImIHSq5ERERERETsQMmViIiIiIiIHSi5EhERERERsQMlVyIiIpKmL774AovFwqZNmxwdiohIrqDkSkQkH/joo4+wWCw0bNjQ0aHIdZKSlxttf/zxh6NDFBGRDHB2dAAiIpL1Zs2aRWBgIBs2bGDfvn1UrFjR0SHJdV5//XXKlSuXar/+O4mI5C5KrkRE8riDBw+ydu1avvvuO5544glmzZrFiBEjHB1WmqKjo/H09HR0GNnugQceoF69eo4OQ0REMknTAkVE8rhZs2ZRpEgRHnzwQbp06cKsWbPSbHfu3Dmee+45AgMDcXNzo1SpUvTq1YszZ84kt7l8+TIjR47kjjvuwN3dnRIlStCpUyf2798PwKpVq7BYLKxatSrFuf/++28sFgtffPFF8r7HH3+cggULsn//ftq0aUOhQoV45JFHAPj111/p2rUrZcqUwc3NjdKlS/Pcc89x6dKlVHHv2rWLbt26Ubx4cQoUKEDlypV55ZVXAPjll1+wWCwsWLAg1XGzZ8/GYrGwbt26NH+PTZs2YbFYmDFjRqrvfvrpJywWC4sXLwbgwoULPPvss8m/na+vL61atSI8PPwG/1UyJun3GzduHO+99x5ly5alQIECNG/enB07dqRq//PPP3P33Xfj6elJ4cKFad++PTt37kzV7tixY/Tr14+AgADc3NwoV64cAwcOJC4uLkW72NhYQkNDKV68OJ6ennTs2JHTp0+naLNp0yaCg4Px8fGhQIEClCtXjr59+9rl/kVEcguNXImI5HGzZs2iU6dOuLq60rNnTz7++GM2btxI/fr1k9tcvHiRu+++m507d9K3b1/q1KnDmTNnWLRoEUePHsXHx4eEhAQeeughVq5cSY8ePRg8eDAXLlwgLCyMHTt2UKFChQzHduXKFYKDg7nrrrsYN24cHh4eAMybN4+YmBgGDhxIsWLF2LBhAx9++CFHjx5l3rx5ycdv27aNu+++GxcXFwYMGEBgYCD79+/nhx9+4K233uKee+6hdOnSzJo1i44dO6b6XSpUqEDjxo3TjK1evXqUL1+eb775ht69e6f4bu7cuRQpUoTg4GAAnnzySebPn09ISAjVqlXjn3/+4bfffmPnzp3UqVPnlr/D+fPnUySxABaLhWLFiqXYN3PmTC5cuMCgQYO4fPky77//Pi1atGD79u34+fkBsGLFCh544AHKly/PyJEjuXTpEh9++CFNmzYlPDycwMBAAI4fP06DBg04d+4cAwYMoEqVKhw7doz58+cTExODq6tr8nWffvppihQpwogRI/j777+ZOHEiISEhzJ07F4BTp07RunVrihcvztChQylcuDB///0333333S3vXUQkT7GJiEietWnTJhtgCwsLs9lsNltiYqKtVKlStsGDB6doN3z4cBtg++6771KdIzEx0Waz2WzTpk2zAbYJEybcsM0vv/xiA2y//PJLiu8PHjxoA2zTp09P3te7d28bYBs6dGiq88XExKTaN3r0aJvFYrEdOnQoeV+zZs1shQoVSrHv+nhsNptt2LBhNjc3N9u5c+eS9506dcrm7OxsGzFiRKrrXG/YsGE2FxcX27///pu8LzY21la4cGFb3759k/d5e3vbBg0adNNzpWX69Ok2IM3Nzc0tuV3S71egQAHb0aNHk/evX7/eBtiee+655H21atWy+fr62v7555/kfVu3brVZrVZbr169kvf16tXLZrVabRs3bkwVV9LvlxRfy5YtU/ymzz33nM3JySn5N12wYIENSPNcIiL5iaYFiojkYbNmzcLPz497770Xro6GdO/enTlz5pCQkJDc7ttvvyUoKCjV6E7SMUltfHx8ePrpp2/Y5nYMHDgw1b4CBQokv4+OjubMmTM0adIEm83Gli1bADh9+jRr1qyhb9++lClT5obx9OrVi9jYWObPn5+8b+7cuVy5coVHH330prF1796d+Pj4FCMwy5cv59y5c3Tv3j15X+HChVm/fj3Hjx/P8P0DTJ48mbCwsBTbjz/+mKpdhw4dKFmyZPLnBg0a0LBhQ5YuXQrAiRMniIiI4PHHH6do0aLJ7WrWrEmrVq2S2yUmJrJw4ULatm2b5rNe//3vOWDAgBT77r77bhISEjh06FDy/QMsXryY+Pj42/oNRETyAiVXIiJ5VEJCAnPmzOHee+/l4MGD7Nu3j3379tGwYUNOnjzJypUrk9vu37+f6tWr3/R8+/fvp3Llyjg7229GubOzM6VKlUq1//Dhw8kJQsGCBSlevDjNmzeHq1PoAA4cOABwy7irVKlC/fr1UzxrNmvWLBo1anTLanxBQUFUqVIlefobVxMzHx8fWrRokbxv7Nix7Nixg9KlS9OgQQNGjhyZHF96NGjQgJYtW6bYkhLi61WqVCnVvjvuuIO///4bIDnZqVy5cqp2VatW5cyZM0RHR3P69GmioqJu+dsl+W/yWqRIEQDOnj0LQPPmzencuTOjRo3Cx8eH9u3bM336dGJjY9N1fhGRvELJlYhIHvXzzz9z4sQJ5syZQ6VKlZK3bt26wdUEw95uNIJ1/SjZ9dzc3LBaranatmrViiVLlvDSSy+xcOFCwsLCkothJCYmZjiuXr16sXr1ao4ePcr+/fv5448/bjlqlaR79+788ssvnDlzhtjYWBYtWkTnzp1TJJndunXjwIEDfPjhhwQEBPDuu+9y5513pjn6lBs5OTmlud9ms8HV/+7z589n3bp1hISEcOzYMfr27UvdunW5ePFiNkcrIuI4Sq5ERPKoWbNm4evry7x581JtPXv2ZMGCBcnV9ypUqJBm1bnrVahQgd27d9902lfSiMa5c+dS7E8aUUmP7du3s2fPHsaPH89LL71E+/btadmyJQEBASnalS9fHuCWcQP06NEDJycnvv76a2bNmoWLi0uKaX030717d65cucK3337Ljz/+SFRUFD169EjVrkSJEjz11FMsXLiQgwcPUqxYMd56661033d67N27N9W+PXv2JBepKFu2LAC7d+9O1W7Xrl34+Pjg6elJ8eLF8fLyStdvlxGNGjXirbfeYtOmTcyaNYs///yTOXPm2PUaIiI5mZIrEZE86NKlS3z33Xc89NBDdOnSJdUWEhLChQsXWLRoEQCdO3dm69ataZYsTxqd6Ny5M2fOnGHSpEk3bFO2bFmcnJxYs2ZNiu8/+uijdMeeNEqSdM6k9++//36KdsWLF6dZs2ZMmzaNw4cPpxlPEh8fHx544AG++uorZs2axf3334+Pj0+64qlatSo1atRg7ty5zJ07lxIlStCsWbPk7xMSEpKnKibx9fUlICDA7tPiFi5cyLFjx5I/b9iwgfXr1/PAAw/A1QSvVq1azJgxI0WCu2PHDpYvX06bNm0AsFqtdOjQgR9++IFNmzalus5/f79bOXv2bKpjatWqBVfLuIuI5BcqxS4ikgctWrSICxcu0K5duzS/b9SoEcWLF2fWrFl0796dIUOGMH/+fLp27Zo8nevff/9l0aJFTJkyhaCgIHr16sXMmTMJDQ1lw4YN3H333URHR7NixQqeeuop2rdvj7e3N127duXDDz/EYrFQoUIFFi9ezKlTp9Ide5UqVahQoQIvvPACx44dw8vLi2+//Tb5+Z7rffDBB9x1113UqVOHAQMGUK5cOf7++2+WLFlCREREira9evWiS5cuALzxxhsZ+j27d+/O8OHDcXd3p1+/fimmMl64cIFSpUrRpUsXgoKCKFiwICtWrGDjxo2MHz8+Xef/8ccf2bVrV6r9TZo0SR6hA6hYsSJ33XUXAwcOJDY2lokTJ1KsWDFefPHF5DbvvvsuDzzwAI0bN6Zfv37Jpdi9vb0ZOXJkcru3336b5cuX07x5cwYMGEDVqlU5ceIE8+bN47fffksuUpEeM2bM4KOPPqJjx45UqFCBCxcuMHXqVLy8vJITOhGRfMHR5QpFRMT+2rZta3N3d7dFR0ffsM3jjz9uc3FxsZ05c8Zms9ls//zzjy0kJMRWsmRJm6urq61UqVK23r17J39vu1oi/ZVXXrGVK1fO5uLiYvP397d16dLFtn///uQ2p0+ftnXu3Nnm4eFhK1KkiO2JJ56w7dixI81S7J6enmnG9tdff9latmxpK1iwoM3Hx8fWv39/29atW1Odw2az2Xbs2GHr2LGjrXDhwjZ3d3db5cqVba+99lqqc8bGxtqKFCli8/b2tl26dClDv+fevXuTS6T/9ttvqc47ZMgQW1BQkK1QoUI2T09PW1BQkO2jjz665XlvVor9+ntNKsX+7rvv2saPH28rXbq0zc3NzXb33Xfbtm7dmuq8K1assDVt2tRWoEABm5eXl61t27a2v/76K1W7Q4cO2Xr16mUrXry4zc3NzVa+fHnboEGDbLGxsSni+2+J9f+W3A8PD7f17NnTVqZMGZubm5vN19fX9tBDD9k2bdqUod9ZRCS3s9gyOvYvIiKSC125coWAgADatm3L559/7uhwMuTvv/+mXLlyvPvuu7zwwguODkdERG5Az1yJiEi+sHDhQk6fPk2vXr0cHYqIiORReuZKRETytPXr17Nt2zbeeOMNateunbxeloiIiL1p5EpERPK0jz/+mIEDB+Lr68vMmTMdHY6IiORheuZKRERERETEDjRyJSIiIiIiYgdKrkREREREROxABS3SkJiYyPHjxylUqBAWi8XR4YiIiIiIiIPYbDYuXLhAQEBAikXk06LkKg3Hjx+ndOnSjg5DRERERERyiCNHjlCqVKmbtlFylYZChQrB1R/Qy8vLobHEx8ezfPlyWrdujYuLi0NjkdxH/UcyQ/1HMkP9RzJD/UduV1b0naioKEqXLp2cI9yMkqs0JE0F9PLyyhHJlYeHB15eXvrDRTJM/UcyQ/1HMkP9RzJD/UduV1b2nfQ8LqSCFiIiIiIiInag5EpERERERMQOlFyJiIiIiIjYgZ65EhERERHJxRISEoiPj3d0GDlCfHw8zs7OXL58mYSEhHQd4+TkhLOzs12WYFJyJSIiIiKSS128eJGjR49is9kcHUqOYLPZ8Pf358iRIxlKljw8PChRogSurq6Zur6SKxERERGRXCghIYGjR4/i4eFB8eLF7TLyktslJiZy8eJFChYseMsFf7majMXFxXH69GkOHjxIpUqV0nXcjSi5EhERERHJheLj47HZbBQvXpwCBQo4OpwcITExkbi4ONzd3dOdJBUoUAAXFxcOHTqUfOztUkELEREREZFcTCNWmZeZ0aoU57HLWURERERERPI5JVciIiIiIiJ2oORKRERERETEDpRciYiIiIhItrBYLDfdRo4cmalzL1y40K7xZpSqBYqIiIiISLY4ceJE8vu5c+cyfPhwdu/enbyvYMGCDorMPjRylcMNHWrlySfv44cfVAVGRERERG7CZoPoaMds6VzE2N/fP3nz9vbGYrGk2DdnzhyqVq2Ku7s7VapU4aOPPko+Ni4ujpCQEEqUKIG7uztly5Zl9OjRAAQGBgLQuXNnihQpQvny5bPoR745jVzlcKdOWYiMLMi2bQl06uToaEREREQkx4qJAUeN/Fy8CJ6emTrFrFmzGD58OJMmTaJ27dps2bKF/v374+npSe/evfnggw9YtGgR33zzDWXKlOHIkSMcOXIEgI0bN+Lr68vnn39O06ZNKVy4sJ1uLGMcPnI1efJkAgMDcXd3p2HDhmzYsOGGbf/88086d+5MYGAgFouFiRMnptnu2LFjPProoxQrVowCBQpQo0YNNm3alIV3kXWqVzf/CrB9u0auRERERCTvGjFiBOPHj6dTp06UK1eOTp068dxzz/HJJ58AcPjwYSpVqsRdd91F2bJlueuuu+jZsycAxYsXB6Bw4cL4+fklf85uDh25mjt3LqGhoUyZMoWGDRsyceJEgoOD2b17N76+vqnax8TEUL58ebp27cpzzz2X5jnPnj1L06ZNuffee/nxxx8pXrw4e/fupUiRItlwR/ZXo4ZJrnbsUHIlIiIiIjfh4WFGkBx17UyIjo5m//799OvXj/79+yfvv3LlCt7e3gA8/vjjtGrVisqVK3P//ffz0EMP0bp160yHbk8OTa4mTJhA//796dOnDwBTpkxhyZIlTJs2jaFDh6ZqX79+ferXrw+Q5vcA77zzDqVLl2b69OnJ+8qVK5dl95DVkpKrffvMSG8m+62IiIiI5FUWS6an5jnKxatJ4dSpU2nYsGGK75ycnACoU6cOBw8e5Mcff2TFihV069aNli1bMn/+fIfEnBaHJVdxcXFs3ryZYcOGJe+zWq20bNmSdevW3fZ5Fy1aRHBwMF27dmX16tWULFmSp556KkUG/F+xsbHExsYmf46KigIgPj6e+Pj4247FHooWjcfbO5Hz593Ytu0Kdeum72FBEa724etfRTJC/UcyQ/1HMkP9J33i4+Ox2WwkJiaSmJjo6HAyLCnmxMREihcvTkBAAPv370+e6pdW24IFC9K1a1e6du1Kp06daNOmDWfOnKFo0aK4uLiQkJAAkPy7ZCQWm81GfHx8cjKXJCP90GHJ1ZkzZ0hISMDPzy/Ffj8/P3bt2nXb5z1w4AAff/wxoaGhvPzyy2zcuJFnnnkGV1dXevfuneYxo0ePZtSoUan2L1++HI8cMFRUtmwTtm0rzqxZ2zl58rCjw5FcKCwszNEhSC6m/iOZof4jmaH+c3POzs74+/tz8eJF4uLiHB1Ohl2+fBmbzZY8sPHSSy8xdOhQ3NzcuO+++4iNjSUiIoJz584xaNAgJk+ejJ+fHzVr1sRqtfL111/j5+eH1WolKiqKMmXKsGzZMmrWrMm5c+cyVNQiLi6OS5cusWbNGq5cuZLiu5iYmHSfJ89VC0xMTKRevXq8/fbbANSuXZsdO3YwZcqUGyZXw4YNIzQ0NPlzVFQUpUuXpnXr1nh5eWVb7GmJj4/n889PsG1bcazWmrRpU92h8UjuEh8fT1hYGK1atcLFxcXR4Uguo/4jmaH+I5mh/pM+ly9f5siRIxQsWBB3d3dHh5Nh7u7uWCyW5L9vh4SEULRoUcaPH8/w4cPx9PSkRo0aPPPMM3h5eeHj48PkyZPZu3cvTk5O1K9fnyVLliQnUePHj+eFF15g5syZlCxZkgMHDqQ7lsuXL1OgQAGaNWuW6rdMSv7Sw2HJlY+PD05OTpw8eTLF/pMnT+Lv73/b5y1RogTVqlVLsa9q1ap8++23NzzGzc0NNze3VPtdXFxyxP+gAwPNf9AdO5xwcXG6ZXuR/8opfVlyJ/UfyQz1H8kM9Z+bS0hIwGKxYLVasVodXgQ8w/r27Uvfvn1T7Hv00Ud59NFH02z/xBNP8MQTT9zwfO3bt6dt27ZERUXh5eWVod/EarVisVjS7HMZ6YMO+6/g6upK3bp1WblyZfK+xMREVq5cSePGjW/7vE2bNk2xyjPAnj17KFu2bKbidaSyZc8DsHVrutdnExERERGRbObQaYGhoaH07t2bevXq0aBBAyZOnEh0dHRy9cBevXpRsmTJ5JWX4+Li+Ouvv5LfHzt2jIiICAoWLEjFihUBeO6552jSpAlvv/023bp1Y8OGDXz66ad8+umnDrzTzCld+gJWq41//rEQGQklSjg6IhERERER+S+HJlfdu3fn9OnTDB8+nMjISGrVqsWyZcuSi1wcPnw4xXDe8ePHqV27dvLncePGMW7cOJo3b86qVavgarn2BQsWMGzYMF5//XXKlSvHxIkTeeSRRxxwh/bh5pZIpUqwezds367kSkREREQkJ3J4QYuQkBBCQkLS/C4pYUoSGBiILR3z4h566CEeeughu8WYE1SvbmP3bgvbtkEOWytNREREREQc+cyVZEzSYsLbtjk6EhERERERSYuSq1xCyZWIiIiISM6m5CqXSEqu/voLtFi5iIiIiEjOo+QqlyhbFgoVMonVnj2OjkZERERERP5LyVUuYbFAjRrmvaYGioiIiIjkPEqucpGaNc2rkisRERERkWsCAwOZOHGio8NQcpWbKLkSERERkdzMYrHcdBs5cuRtnXfjxo0MGDDA7vFmlMPXuZL0S0qutm93dCQiIiIiIhl34sSJ5Pdz585l+PDh7N69O3lfwYIFk9/bbDYSEhJwdr51ylK8eHEAEhMT7R5zRmjkKhepXt28HjkCZ886OhoRERERyUlsNoiOdsxms6UvRn9//+TN29sbi8WS/HnXrl0UKlSIH3/8kbp16+Lm5sZvv/3G/v37ad++PX5+fhQsWJD69euzYsWKFOf977RAJycnPvvsMzp27IiHhweVKlVi0aJF9v7JU1FylYt4e5uqgWj0SkRERET+IyYGChZ0zBYTY7/7GDp0KGPGjGHnzp3UrFmTixcv0qZNG1auXMmWLVu4//77adu2LYcPH77peUaNGkW3bt3Ytm0bbdq04ZFHHuHff/+1X6BpUHKVy+i5KxERERHJy15//XVatWpFhQoVKFq0KEFBQTzxxBNUr16dSpUq8cYbb1ChQoVbjkQ9/vjj9OzZk4oVK/L2229z8eJFNmzYkKWx65mrXKZmTfjhB41ciYiIiEhKHh5w8aLjrm0v9erVS/H54sWLjBw5kiVLlnDixAmuXLnCpUuXbjlyVTNpVALw9PTEy8uLU6dO2S/QNCi5ymW01pWIiIiIpMViAU9PR0eReZ7/uYkXXniBsLAwxo0bR8WKFSlQoABdunQhLi7upudxcXFJ8dlisWR5wQslV7nM9RUDExPBqomdIiIiIpKH/f777zz++ON07NgRro5k/f33344OK036q3kuU6kSuLmZqiwHDzo6GhERERGRrFWpUiW+++47IiIi2Lp1Kw8//LDDS67fiJKrXMbZGe6807zX1EARERERyesmTJhAkSJFaNKkCW3btiU4OJg6deo4Oqw0aVpgLlSzJoSHm6mBV0dHRURERERylccff5zHH388+fM999yDLY0FswIDA/n5559T7Bs0aFCKz0nTBJNGtBISErD+5/mZc+fO2TX+tGjkKhdSUQsRERERkZxHyVUupLWuRERERERyHiVXuVBScrVvnylsISIiIiIijqfkKhfy9QU/P7DZ4K+/HB2NiIiIiIig5Cr30nNXIiIiIgKkWQRCMsZev6GSq1xKz12JiIiI5G9OTk4AxMXFOTqUXC8mJgYAFxeXTJ1HpdhzKSVXIiIiIvmbs7MzHh4enD59GhcXl1Slx/OjxMRE4uLiuHz5crp+D5vNRkxMDKdOnaJw4cLJCevtUnKVS12fXNlsYLE4OiIRERERyU4Wi4USJUpw8OBBDh065OhwcgSbzcalS5coUKAAlgz8Bblw4cL4+/tn+vpKrnKpqlXByQn+/RdOnICAAEdHJCIiIiLZzdXVlUqVKmlq4FXx8fGsWbOGZs2apXuKn4uLS6ZHrJIoucql3N3hjjtg504zeqXkSkRERCR/slqtuLu7OzqMHMHJyYkrV67g7u6e6eenbocmZuZieu5KRERERCTnUHKViym5EhERERHJOZRc5WJJydX27Y6ORERERERElFzlYkkLCe/cCXqGUURERETEsZRc5WJlyoCXF8THw+7djo5GRERERCR/U3KVi1kseu5KRERERCSnUHKVy+m5KxERERGRnCFHJFeTJ08mMDAQd3d3GjZsyIYNG27Y9s8//6Rz584EBgZisViYOHHiTc89ZswYLBYLzz77bBZE7ngauRIRERERyRkcnlzNnTuX0NBQRowYQXh4OEFBQQQHB3Pq1Kk028fExFC+fHnGjBmDv7//Tc+9ceNGPvnkE2omZSB5UFJRCyVXIiIiIiKO5fDkasKECfTv358+ffpQrVo1pkyZgoeHB9OmTUuzff369Xn33Xfp0aMHbm5uNzzvxYsXeeSRR5g6dSpFihTJwjtwrOrVzeuxY/DPP46ORkREREQk/3J25MXj4uLYvHkzw4YNS95ntVpp2bIl69aty9S5Bw0axIMPPkjLli158803b9o2NjaW2NjY5M9RUVEAxMfHEx8fn6k4Mivp+jeKo0ABKFfOmYMHLWzZcoXmzW3ZHKHkZLfqPyI3o/4jmaH+I5mh/iO3Kyv6TkbO5dDk6syZMyQkJODn55div5+fH7t27brt886ZM4fw8HA2btyYrvajR49m1KhRqfYvX74cDw+P247DnsLCwm74XfHiDTh4sARz5/5FdPTBbI1Lcoeb9R+RW1H/kcxQ/5HMUP+R22XPvhMTE5Putg5NrrLCkSNHGDx4MGFhYbi7u6frmGHDhhEaGpr8OSoqitKlS9O6dWu8vLyyMNpbi4+PJywsjFatWuHi4pJmm/XrrWzYAImJ1WnTpmq2xyg5V3r6j8iNqP9IZqj/SGao/8jtyoq+kzSrLT0cmlz5+Pjg5OTEyZMnU+w/efLkLYtV3MjmzZs5deoUderUSd6XkJDAmjVrmDRpErGxsTg5OaU4xs3NLc3nt1xcXHLM/6BvFkvt2uZ1xw4rLi4Of4xOcqCc1Jcl91H/kcxQ/5HMUP+R22XPvpOR8zj0b+Kurq7UrVuXlStXJu9LTExk5cqVNG7c+LbOed9997F9+3YiIiKSt3r16vHII48QERGRKrHKC5KKIe7YAYmJjo5GRERERCR/cvi0wNDQUHr37k29evVo0KABEydOJDo6mj59+gDQq1cvSpYsyejRo+FqEYy//vor+f2xY8eIiIigYMGCVKxYkUKFClE9qYTeVZ6enhQrVizV/ryiYkVwd4eYGDhwwHwWEREREZHs5fDkqnv37pw+fZrhw4cTGRlJrVq1WLZsWXKRi8OHD2O1XhtgO378OLWT5sEB48aNY9y4cTRv3pxVq1Y55B4czcnJlGTftMmsd6XkSkREREQk+zk8uQIICQkhJCQkze/+mzAFBgZis2Ws3Hh+SLpq1LiWXHXq5OhoRERERETyH1U/yCOSnrvats3RkYiIiIiI5E9KrvIIJVciIiIiIo6l5CqPqFHDvB44ABcvOjoaEREREZH8R8lVHlG8OPj7g80Gf/7p6GhERERERPIfJVd5iKYGioiIiIg4jpKrPETJlYiIiIiI4yi5ykOSkqvt2x0diYiIiIhI/qPkKg+5fuQqg0uBiYiIiIhIJim5ykOqVAEnJzh7Fo4dc3Q0IiIiIiL5i5KrPMTNzSRY6LkrEREREZFsp+Qqj9FzVyIiIiIijqHkKo9RxUAREREREcdQcpXHKLkSEREREXEMJVd5TI0a5nXXLoiNdXQ0IiIiIiL5h5KrPKZUKShcGK5cMQmWiIiIiIhkDyVXeYzFoqIWIiIiIiKOoOQqD9JzVyIiIiIi2U/JVR6U9NyVkisRERERkeyj5CoP0siViIiIiEj2U3KVB1Wvbl5PnIAzZxwdjYiIiIhI/qDkKg8qWBAqVDDvVdRCRERERCR7KLnKozQ1UEREREQkeym5yqNU1EJEREREJHspucqjatUyr199BW++CfHxjo5IRERERCRvU3KVR7VtC+3bQ1wcvPYa1KsHmzc7OioRERERkbxLyVUe5ewMCxbArFlQrJiZHtiwIQwdCpcuOTo6EREREZG8R8lVHmaxwMMPw86d0KMHJCTAO+9AUBCsWePo6ERERERE8hYlV/lA8eLw9dfw/fcQEAB790Lz5jBoEERFOTo6EREREZG8QclVPtKuHfz5J/Tvbz5/9JFZcPjHHx0dmYiIiIhI7qfkKp8pXBg+/RRWroTy5eHIEWjTBnr1gn/+cXR0IiIiIiK5l5KrfKpFC1Pk4rnnzLNZX34JVavCN9+Azebo6EREREREch8lV/mYpydMmABr10K1anD6NHTvDp06wfHjjo5ORERERCR3UXIlNGoE4eEwfLgp4b5woRnFGj0aYmIcHZ2IiIiISO6g5EoAcHODUaPMQsP165sqgi+/DHfcAdOmmTLuIiIiIiJyY0quJIWaNeGPP8wzWGXLwrFj0K+fWRtr8eIbPI8VGQnz5sHTT0P79rBkiQMiFxERERFxrByRXE2ePJnAwEDc3d1p2LAhGzZsuGHbP//8k86dOxMYGIjFYmHixImp2owePZr69etTqFAhfH196dChA7t3787iu8galp9+ouxPP2FZuhS2bjUl/bK44oTVCo8+Crt2wbhxUKSIKeHeti3ccw9sWBQJX30FAwZA5cpQogR06waTJsGiRfDQQzBiBCQmZmmcIiIiIiI5icOTq7lz5xIaGsqIESMIDw8nKCiI4OBgTp06lWb7mJgYypcvz5gxY/D390+zzerVqxk0aBB//PEHYWFhxMfH07p1a6Kjo7P4buzPOnMmtT7+GOcOHaBWLfDxMZUo7rjDlPzr1cvM3/voI5PYbNliKlPYIQFzd4fnQ23s/2kfQ4K34WaNY80aaNjen26PubJv6s+wZ48pNxgUZEau+vUzB7/+ullY69y5zP8IIiIiIiK5gLOjA5gwYQL9+/enT58+AEyZMoUlS5Ywbdo0hg4dmqp9/fr1qV+/PkCa3wMsW7YsxecvvvgCX19fNm/eTLNmzbLkPrKKrX59Thw6hH9cHJZjx0zidOkS7N1rthtxc4OSJcHbGzw8oECBa6/peR8ZCWvWwJo1FDl5krFACKUZzuvMpBfz6MYCa2eebHOY18YXwfeOwteu3awZPPGEmR5Yvz4sWGBWKxYRERERycMcmlzFxcWxefNmhg0blrzParXSsmVL1q1bZ7frnD9/HoCiRYum+X1sbCyxsbHJn6OiogCIj48nPj7ebnHcjvhBg9hwxx20atUKFxcXuHwZjh83idbRo9dejx6FY8fM55MnscTGwoEDdonB5uaGrUEDSt51F5/d7cfThS7yypsF+eknJyYtLscXq2y88EICgwcn4ukJ9OwJVarg3LUrln37sDVqRMKnn2Lr2tUu8Uj6JfVfR/djyZ3UfyQz1H8kM9R/5HZlRd/JyLksNpvjlow9fvw4JUuWZO3atTRu3Dh5/4svvsjq1atZv379TY8PDAzk2Wef5dlnn71hm8TERNq1a8e5c+f47bff0mwzcuRIRo0alWr/7Nmz8fDwyNA95QSW+Hjcz56lwD//4HzpEk6xsTjFxSW/WpPe/2d/0ntrXBwJ7u78W7UqZ6pV41ylSiS6uqa6zrZtPsyYcSf795tRqyJFLtOjxy5atDiCi0sirlFR1Bs3juLbtgGwt0MHdj72GDYnp2z/TUREREREbkdMTAwPP/ww58+fx8vL66ZtHT4tMKsNGjSIHTt23DCxAhg2bBihoaHJn6OioihdujStW7e+5Q+Y1eLj4wkLC7s2cpWNigIVb/J9mzbw4ovwzTdXGDHCiYMH3fn441rMnRtE166JPPqoDe91XUh47VWcJkyg0sKFVIiKIuGrr8yzY5LlHNl/JPdT/5HMUP+RzFD/kduVFX0naVZbejg0ufLx8cHJyYmTJ0+m2H/y5MkbFqvIiJCQEBYvXsyaNWsoVarUDdu5ubnh5uaWar+Li0uO+R90Torlvx57zBQLnDIFxo6F48ctfPKJE598ApUqOfPYY+N59MMWlBvaHevPP2Nt1Ai++w7q1nV06PlGTu4/kvOp/0hmqP9IZqj/yO2yZ9/JyHkcWi3Q1dWVunXrsnLlyuR9iYmJrFy5MsU0wYyy2WyEhISwYMECfv75Z8qVK2eniOVG3Nxg8GA4fBiWLzel3D08TM2N4cOh/NMP0rzKST4rPozzh89B06YwY4ajwxYRERERsRuHl2IPDQ1l6tSpzJgxg507dzJw4ECio6OTqwf26tUrRcGLuLg4IiIiiIiIIC4ujmPHjhEREcG+ffuS2wwaNIivvvqK2bNnU6hQISIjI4mMjOTSpUsOucf8xMkJWrUyixCfPGnyp/vuM9Xa12z2pP/pt/G3nqJH7Bcsefwb4gc+A3Fxjg5bRERERCTTHJ5cde/enXHjxjF8+HBq1apFREQEy5Ytw8/PD4DDhw9z4sSJ5PbHjx+ndu3a1K5dmxMnTjBu3Dhq167N//73v+Q2H3/8MefPn+eee+6hRIkSydvcuXMdco/5VcGCZhmuFSvMiNaYMVC1KlxOdGMuPXiIJZSa8grPBS5gy/LTWb02soiIiIhIlsoRBS1CQkIICQlJ87tVq1al+BwYGMitChw6sACi3ECpUvDSS6YARng4zJwJX8+I5dR5Pyae6M7EYChVNJo61eOo09CV2k09qFPXQsmSZtRLRERERCSnyxHJleQfFoupY1G3Lowb58ZP044x88XtLIq6h6P/enJ0jSeL1gDvmvY+Luep43eUOhWiqB2UQJ27PCnf2A9rgD9YHT7wmvXi4uCPP8yCzlFR5ge0Ws1rOt5bExMpExkJDRpAiRKOvhsRERGRPE3JlTiMiws89ERJHnrYiwtDR7Jl1Xm2HPUhPKoiW6jNX1TjTLw3y496s/wosBr4AAoRRW3Lb9T2OkCdkicJqpFI+f+1oNB9DXL/MFdCAkREwM8/w8qV8OuvEBNz26dzAmoDtk8+gdat4eGHoX17M2dTREREROxKyZU4XqFCFJo8hmZAM4DLl+HIES7tWcOOdRcID4ct+wqxJdKfrRfKcwEv1tiaseZ8MzgP/AXMBR+nfylXMo5ydYpQrrIb5ctDuXJmK1MG0lgH2fFsNti9+1oy9csvcPZsyja+vtCiBZQsadrbbJCYmK73iVeuEPXbbxTevx+WLjWbh4dJsB5+GIKDTZYrIiIiIpmm5EpyHnd3qFSJApUqUf9BqH/dV1euwK4dVwj/+Rxb1l0mfLszf/7tyT+xhTiTUJQzh2Hj4dSntFpNbnJ9wlWunNnn729mzBUunE0DX0ePmkRq5UqTVB07lvL7QoXgnntMmcX77oM777ztwBLi41m9dCltypfHZf58mDUL9u+Hr782W7Fi0LWrSbSaNs0fUy1zo927YeJE2LULqlWD2rXNdued5n8vIiIikiMouZJcxdkZqtdypnotH3pdtz/q8DkOTvmJg1//wcG/4SDlOEB5DrpW5qAtkEvxLhw5AkeOwOrVaZ/b1dUkWtdvJUqAv1cM/lF78D+xBf/9v+O/PQz300dMMM7Opv58Wu/T+i4qyiQ313NzM4lNixYmmapXz7S3pypVYNQoGDkSNm6E2bNhzhxTL3/KFLOVKQM9e5pEq2ZN+15fbs/vv8O778KiRSSX07y+yI+zsynBWbs21Kp17bVwYYeFLCIikp8puZI8watMYYLe7k7QW91g0yaYOhVmvwzR0diAU84lOdDscQ427MFBj2oc/NvKwYNw4gRERpqZeHFxpmT84VQjXx5AraubWX/Ni/P4xJ8xG2Yrxj/J71NupynKvziTYE5ntZoEKmlkqkkTKFAge34oi8UUt2jQAMaNM39RnzULvv3W3Pg775itenWTaN1/PwQFmcRQskdCgkmm3n0X1q27tr9dOzOdc9cu2LLFbP/8A9u3m23mzGttAwOvjW4lJV2lSjnkdkRERPITJVeSt1gsUL++2caPhzlzsEydit/Gjfj9/BaNf37LzA3s3x/e6A3R0bB+PbFrN3Ny7X5O7PiHyCvFiMQ/xXbCrRyRziWJjC1C7BVnovAmCm8OUCHdoRUuGE/RQvG4e7vhGueE60pw+82MmLm6mgGspPdpfbZYbvyI1Y0eu7pyxcqxY9XZts1KiRJQvLh5hMu8OlPovpZYWraEjz4yz2PNng2LF8OOHfDKK2bz8jIja82ama1evZs+wJY0wJLumYxxcSZh2L4dTp82iUD9+ubZsPzk0iWTII0fD3v3mn2urmaxuOefN6OP17PZzJTSpEQrIsK8/v33tW3Bgmvty5aF5s3NlNN77jEJWG4vACMiIpLDKLmSvKtQIZNE9e9v/uI5dSp89RUcOADDhpntKjegzNWNokWhYUNoUAIaBpq/6Pv4wNW/z547Z3KAM2fMwMGZMzff/v3XXOPcRRfOXXSBEzeIN0s4ARX44Ye0v3V1TUq2CuDr25nixTvj2/8yxSN3UGz3WuL2HeJilDPRP3oS/aML0WzjotMhoouUJtqrBNEFinHR4kX0JSvR0SRvBQqY59lKlTKvJUtCyQAbpQr8Q8noPZQ8tQW/A+tw+nMb7NxpHqZLEbaTGTFr3PjaVq6c/ZKBc+dgzx6zXbpk5n8mbb6+9p+WeTP//GOS2w8/NB0LzLS+p56Cp58281PTYrGYH7hUKWjb9tr+s2dNf09KtrZsMb/xoUMmeUsa4SpTxiRZSQmXPX9fERGRfErJleQPtWrB5MkwdizMm2cSrbVrzfBQ7dpXk6kG5rV8+Rv+JdNigSJFzHbHHem79JUr5u+7Z86Y19hYM1iTtKX3s82W4WWuSExMYOfOA3h5VeDMGSunT8OpU+bv8NHR5rxHj5rtGneg3tUtDQnAmavbDURHX8tdrvv1AJ+rWxOceIISnKAkxyjlfJKSvvGU8I6h8PG/8D5/CO/w83iHb8F78iq8OY+3jysFG9fA2qSRSbbq1QNPzxsHcfky7Nt3LZDrt6QkJi0Wi0mwrk+4/rPZ/Etwxccf54Lut5+PHDwIEybAtGnXyu2XKQPPPQf9+pl/HLgdRYrAvfeaLUl0tOnvq1aZbcMGMw30+mSrdOlro1pJydbtSEgwCWtsrKlE6epqXjW1VERE8gElV5K/eHrC44+b7fRp8PbO8hrtzs5mGl7x4ll6mTTFxyeydOlftGkTiItLykqAMTGkSLb++/rPPyb3LFjQ/GyenuDpYaNgzCk8j+3G8+CfeO4Jp+A/h/Ak+rothmg8OEopjlGSY5S87n0pjjqXJTKhOAk2Z45SmqOUZv0V4PjV7UbOgOWHRLx+iDLJFvvx9kzA28cZ7xIeuCTG4XThLE5R53A6/y9OF8/jxBWsJOKEM05UxomKOBGMEwk4FfLEyacICS7uxETFE3MhgZhoiLEVIOakh9kiPIjBg0sUIAaPFJsNK05cwcNyCU/rJTydLuPpEoeH6xU83a7g6Z6AZwEbHh7gWdCCZyErnl5OeBSw4RnxO57hv+Jpu4AnTfCsVBLPPt3w7NAKz8IueNrAM+H28pHERJM0x8dfe71yxROXmq1wq98Kt1fB7Uo01vXrUiZbR47Al1+aDaBUKZyaNaNKfDzWn382yWp0tOk4/329/v3ly2kHZrVeS7aSEq7/vndxMcOe/fvDY49l/OZFREQcTMmV5F+OyHZyEA8P8xhO2bIZOcoC+F3dmpnhtEOHYM2aa9tekyFV8r8INV2hZgmo4Q01y5vKdm5uXLliChUePWoeG0raIiPh/Pm0Nhvx8RZsWDlPYc5ztRpe9NXt0G38ABeubpmQgDMXbIW4kFDIjOjFXY3nlu4EBlz7uBd4+ep2HTc3898pKbl1dTXJ0vWJU9Jr0vvExPRc3xNn55a4ubXEzQ3ciiXiRixu8Rdxu3Qet0tncT96CbfZsbhzmQJcwoOYFK83en/9qxdReBFFIS7gmhhvRrNiY28d3q+/wvHj8NJL6bkZERGRHEPJlYjcPovFFEYIDDSFF8AMfVksN01enZ2vPYuVHjabhcuX/5Nw7TvN+c17Ob/9CFEHzhDvXoiEYr4kFC1OQlEfEgr7kODmQUKihYQEUmyJidfeW60mcfHwSLkVKJB6X/JWwIbr5Sgun7lI9Klook/HEP3PZWLOxhJ9No7oc/FERyUQcyGB6Iu2awM9l52IjnUmupA/0SUrEW0plOJZtaQtqShIUi7y33WlM/qfyNnZJF/Xu3LFbNHRAFagwNUta/7Rwc3NhlfBRLw8EyjkkYCXZwJeBa7gVSAerwJxFHKLw8stDu+jf1Jn1XgaDn0F56goePNNPQsmIiK5hpIrEbEvX1+7n9JiMclOgQLX1XdoUBwedtToowXwhnLedj+zzXZtBt5/t/j4lDPo0vOaNLXQZjPHJyVsaW2XL6f8HB19hY0bt1OxYk3i4py4dMkkide/3mhfdDRcuHDtcbLYWAunY504/c+t5jpWBjpRmLO0fns5bdZ/xf1fPoJfCS1wLSIiOZ+SKxGRHOT6RPJqkUq7nTfpEaf01sqIj7dRuPBh2rSpjovL7RWkuHLFJFlRUddek7a0Pp8+bRb6Pnu2CN/QnW9WAgFQt46NB9pYaNPG1J5RfQwREcmJlFyJiEiWcXa+VmEzvRISTI2Npe9s58fvY9lMPTaHW9gcbmYJFi0KwcHwwAPmNQsGS0VERG6L5lmIiEiO4uRkqu2/sbAGmxYc5YRLGb6gN938V1O4sI1//4WvvzaP+fn7m5GsESPMOtQiIiKOpORKRERyrg4d8F/yOb095jM38h5O33kvv/54kZdfNkvU2WywcSO8/rpZzu7tt9NbMVFERMT+lFyJiEjO1qoVLF8O3t44/76au15pzlvPnSE83FRsnzYN2rQxSdUrr0DbtmadNhERkeym5EpERHK+pk3hl19MlY/wcGjeHI4fp0QJ6NMHliyBzz8Hd3dYuhTq1IH16x0dtIiI5DdKrkREJHeoXdssMFyyJPz1F9x9Nxw8mPx1377wxx9QsSIcPmy+njTp2rphIiIiWU3JlYiI5B5VqsBvv0H58nDgANx1F+zcmfx1UBBs2gSdO5t1vZ5+Gnr0MGXeRUREspqSKxERyV0CA80I1p13moeumjUzUwWv8vaGefNg4kRTCv6bb6B+fdixw6FRi4hIPqDkSkREcp+AALPacL16cOYM3HuvGdG6ymKBwYNhzRooVQp27zYl22fOdGjUIiKSxym5EhGR3KlYMVi50oxcRUWZqoJffJGiSePGsGULtG4Nly5B794wYABcvuywqEVEJA9TciUiIrmXlxf8+KOpv375sikd2L9/iuzJx8dUEBw1yoxoTZ1qkq79+x0auYiI5EFKrkREJHfz8ICFC+GNN0z29Nln0KSJKXhxlZMTDB8OP/0ExYtDRATUrWsOExERsRclVyIikvtZrfDqqyZ78vExcwHr1oUffkjRrFUr81XTpnD+PHTsCC+8ALGxDotcRETyECVXIiKSd7RqZSoHNmoE585Bu3YwbBhcuZLcpGRJsx7x88+bz+PHwx13mOmC8fGOC11ERHI/JVciIpK3lC5tKgk+84z5PGaMqWhx8mRyExcXGDcOFiwwhQcPHzaFLqpUgRkzUuRiIiIi6abkSkRE8h5XV3j/fZgzBwoWNENVtWunKNcO0KED7NsH770Hvr7mMa3HHzdLaM2eDQkJDrsDERHJhZRciYhI3tW9O2zcCNWqwYkTcM89Zh6gzZbcpEABePZZk1iNHWsqvO/ZA488AjVrwvz5kJjo0LsQEZFcQsmViIjkbVWqwPr18PDDZijqhRegSxdT0eI6np4wZAgcPAhvvgmFC8Nff0HXrlCnDnz/fYqcTEREJBUlVyIikvcVLAhffQWTJ5sHrr77DurVg23bUjUtVAheeQX+/htGjDBLaW3daqYQNmhgltVSkiUiImlRciUiIvmDxQJPPQW//mqKXuzbZ6oKfvFFmg9XeXvDyJFmJGvYMDOytWkTtGljSrmvXKkkS0REUsoRydXkyZMJDAzE3d2dhg0bsmHDhhu2/fPPP+ncuTOBgYFYLBYmTpyY6XOKiEg+0rChKdceHAyXLkGfPiZzqlnTzAF87TWYNQs2b4YLFyhaFN5+2yRZL7xgntFatw5atjRrFX/1ldbJEhERw+HJ1dy5cwkNDWXEiBGEh4cTFBREcHAwp06dSrN9TEwM5cuXZ8yYMfj7+9vlnCIiks/4+MCSJTBqFHh4mOxo+3ZTveLNN+HRR820QS8vKFUKWrak+KgQ3i3zIQemr+aZPhdwc7Pxxx/w2GOmybBhZiqhiIjkXw5PriZMmED//v3p06cP1apVY8qUKXh4eDBt2rQ029evX593332XHj164ObmZpdziohIPuTkBMOHQ1QU7N9vkq3x482CV82agZ+faXfsmJkDOHkyPPMM/j3u4f3pXvxtrcCbNeZQOuAKZ86Y5bTKl4e2bc1zWaowKCKS/zg78uJxcXFs3ryZYcOGJe+zWq20bNmSdevWZds5Y2Njib1uTkdUVBQA8fHxxMfH31Yc9pJ0fUfHIbmT+o9kRr7qP6VLm61Vq5T7z57FsmcP7N6NJWnbswf27cP/0kFe2d6TFz2eYPFj0/joWAdW/OzE4sWweDGUL29jwIBEevdOpFgxR92Y4+Sr/iN2p/4jtysr+k5GzuXQ5OrMmTMkJCTgl/Svg1f5+fmxa9eubDvn6NGjGTVqVKr9y5cvx8PD47bisLewsDBHhyC5mPqPZIb6z9VphD4+ppIFYLlyBe8DB6g+fTrFdu6k45dduK9MGRY/N4y5+1uycmUZDhxwZehQJ157De666xgPPHCQSpXOYbE4+mayl/qPZIb6j9wue/admJiYdLd1aHKVUwwbNozQ0NDkz1FRUZQuXZrWrVvj5eXl0Nji4+MJCwujVatWuLi4ODQWyX3UfyQz1H/S4ZlnuPLllzgNHYrX4cM8/N5Aejz2GBcixjD3Z18+/tiJiAgnfvmlDL/8UoY6dRJ58slEunWzkUP+7S7LqP9IZqj/yO3Kir6TNKstPRyaXPn4+ODk5MTJkydT7D958uQNi1VkxTnd3NzSfH7LxcUlx/wPOifFIrmP+o9khvrPLfzvf9CxI7z8Mnz6KdYvv8R78WIGjB5N/0392bDJwkcfwdy5EB5uZcAAKy+9BE88ASEhULKko28ga6n/SGao/8jtsmffych5HFrQwtXVlbp167Jy5crkfYmJiaxcuZLGjRvnmHOKiIjcVLFi8MknpkZ7rVpw9iw8+SSWJo1p6BLOjBlw9Ci88w4EBpqvx4wx7x991FR9FxGR3M/h1QJDQ0OZOnUqM2bMYOfOnQwcOJDo6Gj69OkDQK9evVIUp4iLiyMiIoKIiAji4uI4duwYERER7Nu3L93nFBERyRKNGsHGjfD++1CoEGzYAPXrw9NP4+N8jhdfNGsXL1xoChJeuWKW1KpXD5o3h++/T3M9YxERySUcnlx1796dcePGMXz4cGrVqkVERATLli1LLkhx+PBhTpw4kdz++PHj1K5dm9q1a3PixAnGjRtH7dq1+d///pfuc4qIiGQZZ2d45hnYvRt69jQ12SdNgipVYNYsnKw22reH1ath0yZ45BFzyJo10KEDVK5sml+86OgbERGRjMpwchUYGMjrr7/O4cOH7RZESEgIhw4dIjY2lvXr19OwYcPk71atWsUXX3yR4vo2my3VtmrVqnSfU0REJMuVKAGzZ8OKFXDHHXDypJkD2KIF7NwJQN268NVXZvHhoUOhSBGz5NbTT5vK8C+9ZKYTiohI7pDh5OrZZ5/lu+++o3z58rRq1Yo5c+akWCNKRERErnPffbBtG7z5Jri7w6pVcOedcO+95jmtM2coWRJGj4YjR8xaxRUrwrlzMHYslCsHDz9sRrlERCRnu63kKiIigg0bNlC1alWefvppSpQoQUhICOHh4VkTpYiISG7m5gavvAJ//QXt2oHNZpKsJ58Ef3944AGYMQPPK+d56ikzo3DRIrjnHvNc1tdfm0e37r7b5GjffQe7doHWVxURyVlu+5mrOnXq8MEHH3D8+HFGjBjBZ599Rv369alVqxbTpk3DZrPZN1IREZHcrlw5U7Xi77/NsFSdOqaCxbJl8Pjj4OsLHTtinTeXti2i+eUXCA+Hxx4DFxf47Td47TXo3BmqVgVPT6heHbp1g5Ej4ZtvYMcO0IQSERHHuO11ruLj41mwYAHTp08nLCyMRo0a0a9fP44ePcrLL7/MihUrmD17tn2jFRERyQvKloUhQ8y2Z49ZBOvrr82zWAsXms3DA9q2pXaPHsz89H7GjHHn669h+3YzAPbXXxAdDX/+abbrOTmZqYXVql3batUyCZnF4qibFhHJ+zKcXIWHhzN9+nS+/vprrFYrvXr14r333qNKlSrJbTp27Ej9+vXtHauIiEjec8cdZjjq1VfNsNOcOWY7cMAkXXPngpcXAR078nz37vBYXShenESbhSNHTD6WlGz99ZdJtKKizNTC3bthwYJrlype3JSAb97cbNWrg/Vmc1hsNjMvUYu4ioikS4aTq/r169OqVSs+/vhjOnTokOaKxeXKlaNHjx72ilFERCTvs1igRg2zvfmmqWAxZ45Jro4dgxkzzIZ5hstapgxlr273ly0LTctAzzLYSpfhhHNp/jrgniLh2rwZTp+Gb781G0CRwoncXfM8zcsfpbnPn9QiAqcTR02JwqQtIQHatoUBA6B161tkYyIi+VuGk6sDBw5QtmzZm7bx9PRk+vTpmYlLREQk/7JYTAWL+vXh3Xfh999NorV4sSkpGBsLe/ea7b+HAgFAgK8vLcuWhTJloH4Z4uo4sXG7O6v3lWT16ar8frkuZ88VZNGaIixaUwSogRcPcBe/0ZzVNOcSdTiEC1fM8NeCBWY6Y//+0KcPBAQ45KcREcnJMpxcnTp1isjIyFTrRq1fvx4nJyfq1atnz/hERETyN6vVlAm8+25Tpz0uzoxkHT5stkOHrr1P+hwTA6dOmW3jRgBcgaZXt5eBeJwJt9RjdaGHWG29h98u1iLqijdLeZClPAiAp0cijWtGExS7kRq751H90AaqvvoWHiNGpBzNcnJy8I8kIpIzZDi5GjRoEC+++GKq5OrYsWO88847rF+/3p7xiYiIyPVcXU3VwXLl0v7eZoN//02dfIFZmbhUKShVCpdSpWjo50dDZ2dexMz+i4iA1avN9uuvcPaslRV/FGIFLYAWAFhIpELCfqov3EH1hZuoXmwx1XvW4I7n2+ISWDIbfwgRkZwnw8nVX3/9RZ06dVLtr127Nn/99Ze94hIREZHbYbFAsWJmq1073Yc5OUHdumYLDYXERFNfY90688zWjh2mUuGZM1b2UYl9VGIhHeEfYBK4TIqjcqFDVK/tQvWW/lSpZiE29raLEouI5EoZ/lPPzc2NkydPUr58+RT7T5w4gbOz/hAVERHJC6xWqFnTbNc7dcokWjt2wI6IK+z49Sw7DnpwIcGTHRfKsmMNsAbAirtba3791cozz5gy8CIieV2Gs6HWrVszbNgwvv/+e7y9vQE4d+4cL7/8Mq1atcqKGEVERCSH8PWFFi3MZv4aURybDY78vJcdk1ax46dj7LhUnvU0ZE9sZaZMgSlTzKNZTz8Nbdqo4KCI5F0ZTq7GjRtHs2bNKFu2LLWvTjeIiIjAz8+PL7/8MitiFBERkRzMYoEy91WizH2VaHP5Mnz3HYnDH2LN/pJ84DqE76+0YflyC8uXQ4UKEBJiCg5e/TdaEZE8I8P/dlSyZEm2bdvG2LFjqVatGnXr1uX9999n+/btlC5dOmuiFBERkdzB3R0efpiE1auoVf4I38U9xH6PmgzpcYTChWH/fnjuOShZ0iRZu3Y5OmAREfu5rYekPD09GTBggP2jERERkbzB15ff33yTB6ZMIXDNGsYuvIMRM79l1tk2fPCBKZIxebLZgoPhmWfg/vs1ZVBEcrfbrkDx119/cfjwYeLi4lLsb9eunT3iEhERkVzuiocHCT/8gPWxx2DRIjx7tmPA9On03/4YP/8MH34IixbBTz+ZrWJF81zW44+Dl5ejoxcRybgMJ1cHDhygY8eObN++HYvFgs1mA8BisQCQkJBg/yhFREQkdypQAL79Fvr1g5kzoVcvLP/+y32DB3PffXDgAHz0EXz2GezbB4MHw8svQ48e0L8/NGhgnukSEckNMjz4PnjwYMqVK8epU6fw8PDgzz//ZM2aNdSrV49Vq1ZlTZQiIiKSezk7w/Tp8Oyz5vOzz8KIEWCzUb48jBsHR4/Cxx+bku3R0fD559CoEQQFwQcfmHWRRURyugwnV+vWreP111/Hx8cHq9WK1WrlrrvuYvTo0TzzzDNZE6WIiIjkblYrTJgAb7xhPr/+upkDmJgIQMGC8OST5lmsNWvgscdMbYzt281oVkAAPPIIrFoFVyfNiIjkOBlOrhISEihUqBAAPj4+HD9+HICyZcuye/du+0coIiIieYPFAq++auYBWiymmsWjj0J8fIomd99tZhCeOAGTJpnRq9hYmD0b7r0X7rgD3nkHTp506N2IiKSS4eSqevXqbN26FYCGDRsyduxYfv/9d15//XXKly+fFTGKiIhIXjJwIMyaZaYLfv01dOgAMTGpmhUuDIMGwZYtsHEjPPEEFCpkns0aOhRKlYJOneDHH0GPfItITpDh5OrVV18l8eoQ/uuvv87Bgwe5++67Wbp0KR988EFWxCgiIiJ5Tc+eplRggQKwdKmpx37uXJpNLRaoVw+mTIHjx83zWI0bw5UrsGABtGkD5crByJFw9my234mISLIMJ1fBwcF06tQJgIoVK7Jr1y7OnDnDqVOnaNGiRVbEKCIiInnRAw9AWJgZovrtN7jnHoiMvOkhBQtC376wdq15HuvZZ6FoUThyBEaNMgUxvv022+5ARCSFDCVX8fHxODs7s2PHjhT7ixYtmlyKXURERCTdmjaF1avBzw+2boW77oKDB9N1aPXq8N57cOyYmV1YpYp5DqtLFzNd8Opj4SIi2SZDyZWLiwtlypTRWlYiIiJiPzVrwu+/m7l9+/ebOX8//pjuw93dzbpYERHw2mvmUa4FC6BaNbN+lqoLikh2yfC0wFdeeYWXX36Zf7XghIiIiNhLhQomwapZ0ww/tWkD//sfnD+f7lO4uZkK75s3Q/365tD+/aFlS5OziYhktQwnV5MmTWLNmjUEBARQuXJl6tSpk2ITERERuS0lSsAff0BoqKli8fnnUKOGeS4rA2rWhHXrYPx4Uy/j55/NacaNM0UwRESyinNGD+jQoUPWRCIiIiJSoIDJijp0gD59zJBT69amDvu775pa7Ong5GRytPbtYcAAk2ANGQJz55qcrWbNLL8TEcmHMpxcjRgxImsiEREREUly992mwMXQoWYl4U8+gZ9+gmnTzErC6VShAqxYYQ57/nnYtAnq1oWXXjLrGbu7Z+ldiEg+k+FpgSIiIiLZwtMTPvzQDDsFBsLff0OLFvDMMxAdne7TWCzQrx/s3GmqCF65Am+9BbVrm8e8RETsJcPJldVqxcnJ6YabiIiIiF3dey9s22amBoJJuGrVMmtjZUCJEmYNrPnzTeX3XbvMAFlICFy4kDWhi0j+kuFpgQsWLEjxOT4+ni1btjBjxgxGjRplz9hEREREjEKFYMoUM/TUrx/s2wfNmsFzz8Gbb5pntdKpc2czAPbCC2a64OTJsHw5LFsG5ctn6V2ISB6X4eSqffv2qfZ16dKFO++8k7lz59KvXz97xSYiIiKSUuvWsGOHqVYxbRpMmABLlsAXX0CjRuk+TZEiprBFz57Qty/s3QtNmpjltWrXztI7EJE8zG7PXDVq1IiVK1fa63QiIiIiafP2NpnR4sVmrt/u3dC0qSl+EROToVO1bGmqvyctr9W8uSmAISJyO+ySXF26dIkPPviAkiVL3tbxkydPJjAwEHd3dxo2bMiGDRtu2n7evHlUqVIFd3d3atSowdKlS1N8f/HiRUJCQihVqhQFChSgWrVqTJky5bZiExERkRzqwQfhzz/h0UchMRHeeQfKloU33oCzZ9N9moAAWLPGPNp14YJZv3j27CyNXETyqAwnV0WKFKFo0aLJW5EiRShUqBDTpk3j3XffzXAAc+fOJTQ0lBEjRhAeHk5QUBDBwcGcOnUqzfZr166lZ8+e9OvXjy1bttChQwc6dOjAjh07ktuEhoaybNkyvvrqK3bu3Mmzzz5LSEgIixYtynB8IiIikoMVKQJffgkLFkC5cnDmDAwfDmXKmIeqjh1L12m8vc2UwG7dID4eHnnELLclIpIRGU6u3nvvvRTbBx98wOLFizl06BDt2rXLcAATJkygf//+9OnTJ3mEycPDg2nTpqXZ/v333+f+++9nyJAhVK1alTfeeIM6deowadKk5DZr166ld+/e3HPPPQQGBjJgwACCgoJuOSImIiIiuVSHDrBnjxlyqlkTLl402VG5cvC//5nvbsHNDb7+GgYPNp9feMGsjZWYmPXhi0jekOGCFo8//rjdLh4XF8fmzZsZNmxY8j6r1UrLli1Zt25dmsesW7eO0NDQFPuCg4NZuHBh8ucmTZqwaNEi+vbtS0BAAKtWrWLPnj289957aZ4zNjaW2NjY5M9RUVFwtRJifHx8pu8zM5Ku7+g4JHdS/5HMUP+RzHBY/+nSBTp3xvLTT1jHjsX622/w+efYpk3D1rEjCS++CHXq3PQUY8eCv7+VYcOcmDABjh1L5PPPE3B1zba7yPf054/crqzoOxk5V4aTq+nTp1OwYEG6du2aYv+8efOIiYmhd+/e6T7XmTNnSEhIwM/PL8V+Pz8/du3aleYxkZGRabaPjIxM/vzhhx8yYMAASpUqhbOzM1arlalTp9KsWbM0zzl69Og0y8gvX74cDw+PdN9PVgoLC3N0CJKLqf9IZqj/SGY4tP+88AJFH3yQSt9+i/+mTVi++w7rd99xKiiIvZ07c6ZGDbPCcBqqVoXBg0sxaVJt5s61smvXGV56aSMeHley/TbyM/35I7fLnn0nJgOFcjKcXI0ePZpPPvkk1X5fX18GDBiQoeQqq3z44Yf88ccfLFq0iLJly7JmzRoGDRpEQEAALVu2TNV+2LBhKUbDoqKiKF26NK1bt8bLyyubo08pPj6esLAwWrVqhYuLi0NjkdxH/UcyQ/1HMiPH9J82beD554nfvh2n8eOxzJ2L79at+G7dSmL9+iQOGYKtXTuwpn5Sok0baN06kW7dLGzd6ss777Thhx+u4O/vkDvJV3JM/5FcJyv6TtKstvTIcHJ1+PBhypUrl2p/2bJlOXz4cIbO5ePjg5OTEydPnkyx/+TJk/jf4E8uf3//m7a/dOkSL7/8MgsWLODBBx8EoGbNmkRERDBu3Lg0kys3Nzfc3NxS7Xdxcckx/4POSbFI7qP+I5mh/iOZkWP6T506MGsWvPUWjBsHn3+OdeNGrN26QeXKptJgGmt5tmkDq1aZwoRbt1po3tyFZcvgjjscchf5To7pP5Lr2LPvZOQ8GS5o4evry7Zt21Lt37p1K8WKFcvQuVxdXalbt26K9bESExNZuXIljRs3TvOYxo0bp1pPKywsLLl90nNS1v/8C5STkxOJeiJVREQkfwsMhEmT4NAheOUVUyZw927o2BG++irNQ+rVg7VroUIFOHjQLKm1fn22Ry4iuUCGk6uePXvyzDPP8Msvv5CQkEBCQgI///wzgwcPpkePHhkOIDQ0lKlTpzJjxgx27tzJwIEDiY6Opk+fPgD06tUrRcGLwYMHs2zZMsaPH8+uXbsYOXIkmzZtIiQkBAAvLy+aN2/OkCFDWLVqFQcPHuSLL75g5syZdOzYMcPxiYiISB7k6wtvvgmHD5tqgjYb9O4N33yTZvMKFUyCVa+eqfbeogX8Z5lNEZGMTwt84403+Pvvv7nvvvtwdjaHJyYm0qtXL95+++0MB9C9e3dOnz7N8OHDiYyMpFatWixbtiy5aMXhw4dTjEI1adKE2bNn8+qrr/Lyyy9TqVIlFi5cSPXq1ZPbzJkzh2HDhvHII4/w77//UrZsWd566y2efPLJDMcnIiIieZiXF3zyiUmuPv8cHn4YXFzMSNZ/+PrCL7+YgoQ//QTt2sFnn4EdCymLSC6X4eTK1dWVuXPn8uabbxIREUGBAgWoUaMGZcuWve0gQkJCkkee/mvVqlWp9nXt2jVVtcLr+fv7M3369NuOR0RERPIRqxU+/dSsHjxzJnTvDt9+C23bpmpasCD88AP062fWLu7Tx6xT/PLLNyw8KCL5SIaTqySVKlWiUqVK9o1GRERExBGsVpg2zSRYX39thqe+/x7uvz9VUxcXmDEDAgJMHYxXX4UjR8yjXM63/TcrEckLMvzMVefOnXnnnXdS7R87duxNR5NEREREcjQnJzNy1bkzxMVBhw6wYkWaTS0WGDMGPvzQvP/kE3NYBpbDEZE8KMPJ1Zo1a2jTpk2q/Q888ABr1qyxV1wiIiIi2c/Z2YxctW8PsbHmwao0HlFIEhIC8+eDmxssWgT33WcKXohI/pTh5OrixYu4urqm2u/i4pKhBbZEREREciQXF5g71yxydekSPPQQ/PbbDZt36mQGuIoUgT/+gCZN4MCBbI1YRHKIDCdXNWrUYO7cuan2z5kzh2rVqtkrLhERERHHcXMzRS1atYLoaJNo/fHHDZvfdRf8/juUKQN790LjxrB5c7ZGLCI5QIYfu3zttdfo1KkT+/fvp0WLFgCsXLmS2bNnM3/+/KyIUURERCT7ubvDwoVm5OqXX0xxixUrzGJXaahaFdatM3nY1q3QvLmZMphGTQwRyaMyPHLVtm1bFi5cyL59+3jqqad4/vnnOXbsGD///DMVK1bMmihFREREHMHDw9Rev/tuOH8eWreGiIgbNg8IgDVrzLNX0dGmmvsXX2RrxCLiQBlOrgAefPBBfv/9d6Kjozlw4ADdunXjhRdeICgoyP4RioiIiDiSpycsWWLm+p09Cy1bwo4dN2zu5QVLl8Ijj8CVK2YtrDffNOsUi0jedlvJFVerBvbu3ZuAgADGjx9PixYt+OMmc5FFREREcq1CheDHH6F+ffjnHzM0tXPnDZu7upqq7i+9ZD6/9ho8+aRJtkQk78pQchUZGcmYMWOoVKkSXbt2xcvLi9jYWBYuXMiYMWOoX79+1kUqIiIi4kje3vDTT1C7Npw6BS1awJ49N2xutaZcC+vTT01lQa2FJZJ3pTu5atu2LZUrV2bbtm1MnDiR48eP8+GHH2ZtdCIiIiI5SZEiEBYGNWpAZKRJsA4duukh16+F9cMP5pDTp7MtYhHJRulOrn788Uf69evHqFGjePDBB3FycsrayERERERyomLFTNXAatXg2DF48EFT7OImrl8La/16aNoUDh7MtohFJJukO7n67bffuHDhAnXr1qVhw4ZMmjSJM1qCXERERPIjX19YtgxKlIA//4SuXSE+/qaH/HctrJYt4eTJbItYRLJBupOrRo0aMXXqVE6cOMETTzzBnDlzCAgIIDExkbCwMC5cuJC1kYqIiIjkJKVLw+LFpppgWBg89dQtSwImrYVVvjwcOGAGvS5ezLaIRSSLZbhaoKenJ3379uW3335j+/btPP/884wZMwZfX1/atWuXNVGKiIiI5ER16sCcOaZ6xWefwdixtzwkIMAMevn4wObN6Rr0EpFc4rZLsQNUrlyZsWPHcvToUb7++mv7RSUiIiKSWzz0EEycaN4PHQrffHPLQypVMoNeBQqYRGvAAK2DJZIXZCq5SuLk5ESHDh1YtGiRPU4nIiIikrs8/TQMHmze9+pl5v7dQsOGJg+zWuGLL2D48KwPU0Syll2SKxEREZF8b/x4aNcOYmPN64EDtzzkoYdgyhTz/s034ZNPsj5MEck6Sq5ERERE7MHJCWbPNs9hnTkDbdrAv//e8rD+/a+NWj31FGgikEjupeRKRERExF48Pc1KwaVLw+7dZoGruLhbHjZyJPTtC4mJ0KNHumYVikgOpORKRERExJ4CAmDJEihUCFavhv/975bVKiwWMz2wTRu4dAnatjW5mYjkLkquREREROytRg2YN89MFfzyS3jjjVse4uJiClzUrw///AP33w+RkdkSrYjYiZIrERERkawQHAwffWTejxgBX311y0M8PU2J9goV4O+/zSLDFy5kfagiYh9KrkRERESyyoAB8OKL5n3fvrBmzS0P8fU1a18VLw7h4dClixYZFsktlFyJiIiIZKXRo6FzZ5MhdeiQroepKlY0j215eMDy5el6bEtEcgAlVyIiIiJZyWo1z101bAhnz5q5fqdP3/Kw+vXNM1hOTjBzJrz6arZEKyKZoORKREREJKsVKGAWsAoMhP37zQjW5cu3POzBB68tLPz22/Dxx1kfqojcPiVXIiIiItnB1xeWLoXChWHtWnjuuXQd1q8fjBpl3oeEwMKFWRumiNw+JVciIiIi2aVqVVOiHcyQ1IYN6Trstdegf3+zyHC3bjBjRtaGKSK3R8mViIiISHZq2RJ69TIVKgYNgoSEWx5isZiq7t26mboYjz8OL79ski0RyTmUXImIiIhkt7FjwcsLNm2Czz5L1yHOzvD11/DKK+bz6NHQtStER2dtqCKSfkquRERERLKbnx+8+aZ5P2wYnDmTrsOsVnPYzJng6grffQfNm8Px41kbroikj5IrEREREUcYOBCCgkx59mHDMnToY4/BypXg4wObN0ODBmbBYRFxLCVXIiIiIo7g7AyTJ5v3n30G69dn6PC77jKHVK0Kx47B3XerkqCIo+WI5Gry5MkEBgbi7u5Ow4YN2XCLyjnz5s2jSpUquLu7U6NGDZYuXZqqzc6dO2nXrh3e3t54enpSv359Dh8+nIV3ISIiIpJBTZua6hSQ7uIW1ytfHtatg9atISYGOnWCd94xtTJEJPs5PLmaO3cuoaGhjBgxgvDwcIKCgggODubUqVNptl+7di09e/akX79+bNmyhQ4dOtChQwd27NiR3Gb//v3cddddVKlShVWrVrFt2zZee+013N3ds/HORERERNLhnXfA29vM75s6NcOHe3vDkiUmN7PZYOhQszZWXFyWRCsiN+Hw5GrChAn079+fPn36UK1aNaZMmYKHhwfTpk1Ls/3777/P/fffz5AhQ6hatSpvvPEGderUYdKkScltXnnlFdq0acPYsWOpXbs2FSpUoF27dvj6+mbjnYmIiIikg6/vteIWL78Mp09n+BTOzjBpEnz4oSl6MX06tGqV7joZImInzo68eFxcHJs3b2bYdQ9xWq1WWrZsybp169I8Zt26dYSGhqbYFxwczMKrk4wTExNZsmQJL774IsHBwWzZsoVy5coxbNgwOnTokOY5Y2NjiY2NTf4cFRUFQHx8PPHx8Xa519uVdH1HxyG5k/qPZIb6j2SG+k8G9euH82efYdm6lcSXXiLhk09u6zRPPAGBgRYeecSJNWssNGpkY8GCK1SpYveIs5T6j9yurOg7GTmXQ5OrM2fOkJCQgJ+fX4r9fn5+7Nq1K81jIiMj02wfGRkJwKlTp7h48SJjxozhzTff5J133mHZsmV06tSJX375hebNm6c65+jRoxk1alSq/cuXL8fDwyOTd2kfYWFhjg5BcjH1H8kM9R/JDPWf9CvSsyfNtm7FOn06v1WpwtnKlW/7XG++WYg332zI/v2eNGoEL720kaCgjI+IOZr6j9wue/admJiYdLd1aHKVFRKvLlXevn17nnvuOQBq1arF2rVrmTJlSprJ1bBhw1KMhkVFRVG6dGlat26Nl5dXNkafWnx8PGFhYbRq1QoXFxeHxiK5j/qPZIb6j2SG+s9taNOGxJ07sc6Ywd1z5nBl7Vpwcrrt03XuDF26JLJunQuvv96Y999PZMCARLuGnFXUf+R2ZUXfSZrVlh4OTa58fHxwcnLi5MmTKfafPHkSf3//NI/x9/e/aXsfHx+cnZ2pVq1aijZVq1blt99+S/Ocbm5uuLm5pdrv4uKSY/4HnZNikdxH/UcyQ/1HMkP9J4PGjoXvv8eyZQsu06ebtbBuU0AA/Pwz9O8PX31lISTEiU2bnPjwQyhY0K5RZxn1H7ld9uw7GTmPQwtauLq6UrduXVauXJm8LzExkZUrV9K4ceM0j2ncuHGK9lwd9ktq7+rqSv369dm9e3eKNnv27KFs2bJZch8iIiIidmGH4hbXc3eHmTPhrbdMoYsvvoA6dWDTJvuEKyIpObxaYGhoKFOnTmXGjBns3LmTgQMHEh0dTZ8+fQDo1atXioIXgwcPZtmyZYwfP55du3YxcuRINm3aREhISHKbIUOGMHfuXKZOncq+ffuYNGkSP/zwA0899ZRD7lFEREQk3Z58EmrXhnPnTF31TLJYTJ72yy9QqhTs3QuNG5tBssTcMUtQJNdweHLVvXt3xo0bx/Dhw6lVqxYREREsW7YsuWjF4cOHOXHiRHL7Jk2aMHv2bD799FOCgoKYP38+CxcupHr16sltOnbsyJQpUxg7diw1atTgs88+49tvv+Wuu+5yyD2KiIiIpJuTE0yebN5Pm2ZWCbaDZs1g61bzLNaVK/DSS2bx4ePH7XJ6EXH0M1dJQkJCUow8XW/VqlWp9nXt2pWuXbve9Jx9+/alb9++dotRREREJNs0bgx9+5rk6qmnzDy+TBS3SFK0KMybB59/DoMHw8qVULOmWRerbVu7RC6Srzl85EpERERE0jBmDBQuDBERMGWK3U5rscD//gebN0OtWvDPP9CuHYSEwKVLdruMSL6k5EpEREQkJype3FSiAHjlFTh1yq6nr1IF/vgDklajmTwZ6teH7dvtehmRfEXJlYiIiEhO9cQTprjF+fN2KW7xX25uMH48LFsGfn7w558mwZo0CWw2u19OJM9TciUiIiKSUzk5wUcfmffTp8PatVlymeBg2LYNHngAYmPh6afNVMFMVoIXyXeUXImIiIjkZI0aQb9+5v2gQabUXxbw9YUlS+D998HVFRYvhqAgWLEiSy4nkicpuRIRERHJ6UaPhiJF7F7c4r8sFnjmGdiwAapWhRMnoFUrePVVrYklkh5KrkRERERyuuLF4e23zfsXXoA33jDz97JIUJCp/v7kk+bzW29B9+6qJihyK0quRERERHKD/v3Ng1CxsTB8ONSoAcuXZ9nlPDzg449hxgxwcYH58+Gee+DkySy7pEiup+RKREREJDdwcoKFC+Hrr6FECdi711Si6NoVjh7Nssv26gVhYWZW4oYN0LChqSooIqkpuRIRERHJLSwW6NEDdu2CZ581Cdf8+WbRqnHjID4+Sy7bvLlZE6tiRTh0CJo0MQmXiKSk5EpEREQkt/Hygvfeg82bTaYTHQ1Dhpg1sdasyZJL3nGHSbDuvhuiokzZ9k8/zZJLieRaSq5EREREcqugIPj1V5g2DXx8zHy95s3NXL4seDiqWDEzYvXoo5CQYNY4fuEF815ElFyJiIiI5G5WK/TpA7t3m2zHYoEvv4TKlWHSJLtnPm5uMHMmvP66+Tx+PHTpYgbPRPI7JVciIiIieUHRomYNrPXroW5dOH8enn4aGjQw++zIYoHXXoPZs82CwwsXmgGz48ftehmRXEfJlYiIiEheUr++SaY++ggKF4bwcGjcGAYMgHPn7Hqpnj3h55/NjMTNm00lwW3b7HoJkVxFyZWIiIhIXuPkBAMHmqmCvXuDzQZTp5pntOxc8KJpU1PoonJlUxG+aVNYutSulxDJNZRciYiIiORVvr7wxRcmoapQAQ4fNisBv/wyxMXZ7TIVKsC6dXDvvXDxIrRtC5Mn2+30IrmGkisRERGRvO7uu2HLFlP4wmaD0aNNCffdu+12iSJFYNky6NsXEhMhJMQsxZWYaLdLiOR4Sq5ERERE8oNChUzJ9nnzTCa0eTPUqWMWq7LZ7HIJV1f47DOTuwG8/z688opdTi2SKyi5EhEREclPunQxVSdatICYGFO+vWNHOH3aLqe3WGDoUPj8c/N5zBhTxFAkP1ByJSIiIpLflCplVgMeN84MN33/PdSsCT/9ZLdL9O0Lo0aZ94MGwQ8/2O3UIjmWkisRERGR/Mhqheefhw0boFo1iIyE+++HwYPh8mW7XOK116BfP/PcVY8e5lIieZmSKxEREZH8LCgINm0yFSgAPvjArJVlhwWrLBb4+GOTs8XEwEMPwf79mQ9ZJKdSciUiIiKS3xUoAB9+aBao8vODHTtMgvXee5ku9+fiAt98A7Vrm8e6HngAzpyxW+QiOYqSKxERERExHnjAjFi1bWvWwQoNheBgOH48U6ctVAiWLIGyZWHvXmjXDi5dslvUIjmGkisRERERucbX1xS4mDLFjGitWAH33Wfm9WVCiRLw449QuLBZcPiRRyAhwW5Ri+QISq5EREREJCWLxZRoDw83WdGuXTBkSKZPW7WqydtcXWHBAjMwZqcltkRyBCVXIiIiIpK2KlVgxgzz/qOPzDNZmdSsGcycad5/8IF5rEskr1ByJSIiIiI31qoVPPused+nD5w6lelTdu8O775r3j//PMybl+lTiuQISq5ERERE5OZGj4bq1U1i1a+fXebyPf/8tervjz0Gv/6a+TBFHE3JlYiIiIjcnLs7zJplHpZavBg++STTp7RYYOJE6NABYmOhfXvYudMu0Yo4jJIrEREREbm1mjVhzBjzPjQUdu/O9CmdnEzO1qgRnD1rKsFHRmY+VBFHUXIlIiIiIukzeDC0bGkWqXrkEbMWViZ5eMCiRVCxIhw6BA8+CBcv2iVakWyn5EpERERE0sdqhS++gKJFYfNmGDnSLqctXtysgeXjY6q/P/ywEwkJFrucWyQ7KbkSERERkfQrWRI+/dS8HzMG1qyxy2krVjSPcxUoAMuWWXn33XpcumSXU4tkmxyRXE2ePJnAwEDc3d1p2LAhGzZsuGn7efPmUaVKFdzd3alRowZLb7LmwpNPPonFYmHixIlZELmIiIhIPtS5synLbrOZUn/nz9vltA0bwty54Opq448/AggOduLMGbucWiRbODy5mjt3LqGhoYwYMYLw8HCCgoIIDg7m1A3WUFi7di09e/akX79+bNmyhQ4dOtChQwd27NiRqu2CBQv4448/CAgIyIY7EREREclH3n8fypeHw4dh0CC7nbZtW1i6NAFPzzj++MNK48awb5/dTi+SpZwdHcCECRPo378/ffr0AWDKlCksWbKEadOmMXTo0FTt33//fe6//36GDBkCwBtvvEFYWBiTJk1iypQpye2OHTvG008/zU8//cSDDz540xhiY2OJjY1N/hwVFQVAfHw88fHxdrvX25F0fUfHIbmT+o9khvqPZIb6Tz7g7o7liy9wuvdeLLNmcSU4GFuPHnY5dePG8YwZs453372XffusNG5sY+HCBBo0yPz6WpK3ZcWfPRk5l0OTq7i4ODZv3sywYcOS91mtVlq2bMm6devSPGbdunWEhoam2BccHMzChQuTPycmJvLYY48xZMgQ7rzzzlvGMXr0aEaNGpVq//Lly/Hw8MjgXWWNsLAwR4cguZj6j2SG+o9khvpP3le5SxeqzJ2L7ckn+eXyZS75+trlvKVLw6hRy3njjUYcOFCYFi0gNDScRo1Uq11uzZ5/9sTExKS7rUOTqzNnzpCQkICfn1+K/X5+fuzatSvNYyIjI9NsH3ndogjvvPMOzs7OPPPMM+mKY9iwYSkStqioKEqXLk3r1q3x8vLK4F3ZV3x8PGFhYbRq1QoXFxeHxiK5j/qPZIb6j2SG+k8+0ro1iQcP4rJhAy2/+oqEn34yC1hlQlL/6datGe3bu/DII4n8+KMz77zTgAkTEhk0KNFu4UvekhV/9iTNaksPh08LtLfNmzfz/vvvEx4ejsWSvhKebm5uuLm5pdrv4uKSY/4PISfFIrmP+o9khvqPZIb6Tz7g4mJWAq5VC+uaNVjffx9eeslOp3bBw8OFRYvMY12ffmrhueecOHLEiXffNZXhRdJizz97MnIeh3ZJHx8fnJycOHnyZIr9J0+exN/fP81j/P39b9r+119/5dSpU5QpUwZnZ2ecnZ05dOgQzz//PIGBgVl4NyIiIiL5VMWK8MEH5v1rr5nFquzI2RmmTIHRo83nCROge3e4fNmulxHJNIcmV66urtStW5eVK1cm70tMTGTlypU0btw4zWMaN26coj1X51QmtX/sscfYtm0bERERyVtAQABDhgzhp59+yuI7EhEREcmn+vSBjh0hPh4efhgy8JxKelgsMHSoGSRzcYH586FlS/jnH7teRiRTHD4tMDQ0lN69e1OvXj0aNGjAxIkTiY6OTq4e2KtXL0qWLMnoq/9UMXjwYJo3b8748eN58MEHmTNnDps2beLTq4vZFStWjGLFiqW4houLC/7+/lSuXNkBdygiIiKSD1gsMHUq/PEH7N4NQ4bA5Ml2v8zDD0NAAHToAL//Dk2awI8/mqrwIo7m8Jmq3bt3Z9y4cQwfPpxatWoRERHBsmXLkotWHD58mBMnTiS3b9KkCbNnz+bTTz8lKCiI+fPns3DhQqpXr+7AuxARERERihWDL74w7z/6CJYsyZLL3HOPSazKlIE9e6BRI9iwIUsuJZIhDh+5AggJCSEkJCTN71atWpVqX9euXenatWu6z//3339nKj4RERERSafWrWHwYLPIcJ8+sHw51Kpl98vceacZJHvwQdiyxSRcc+ZAu3Z2v5RIujl85EpERERE8pgxYyAoCE6fhqZNYd68LLlMiRKwZg088ABcumQe+cqCmYgi6abkSkRERETsy90dfvnFjGLFxEC3bvDqq5Bo//WpChaERYvgf/8zpw8JMY97ZcGlRG5JyZWIiIiI2F+RIuaZqxdeMJ/festUocjAgqzp5ewMn35qLgEwbhz06KFS7ZL9lFyJiIiISNZwdoZ334WZM8HNDX74wVSf2LvX7peyWODll+Grr0yp9nnzVKpdsp+SKxERERHJWo89Br/+amqo79wJDRqYQhdZ4JFH4KefwNv7Wqn2/fuz5FIiqSi5EhEREZGsV78+bNoEjRvDuXOmCsWECWCz2f1S996bslR748awfr3dLyOSipIrEREREckeJUqYQhd9+5qKE88/D717m1J/dpZUqr1OHVO08N574fvv7X4ZkRSUXImIiIhI9nFzg88+gw8+ACcn+PJLaN4cjh2z+6VKlIDVq6FNm2ul2j/80O6XEUmm5EpEREREspfFAk8/bZ67KloUNm6EevVg3Tq7X6pgQTNi9cQTZgbiM8+YATOVapesoORKRERERByjRQuTWNWoAZGRcM89MH263S/j7Awff2zWNgbzqFe3blkyG1HyOSVXIiIiIuI45cvD2rXQqRPExUHfvlhDQ7EkJNj1MhYLvPQSzJ4Nrq7w7bdw331w5oxdLyP5nJIrEREREXGsggXNwlSvvw6A06RJ3PnFF1lyqZ49ISwMChc2sxAbN4Z9+7LkUpIPKbkSEREREcezWuG112DOHADKL16MZePGLLlUs2ZmsCww0CRWjRtnyeNekg8puRIRERGRnKN7dxIffhiLzYbTwIEQH58ll6la1SRUdeuaqYEtWmTZusaSjyi5EhEREZEcJeHdd4krVAjLtm3w3ntZdh1/f1Oq/cEH4fJlaN8eVq3KsstJPqDkSkRERERyluLF2dGnj3k/ciQcOJBll/L0hO++g4ceMgnWQw/B779n2eUkj1NyJSIiIiI5zpF77yXx3ntNvfSBA80iVVnE1dXU02jdGqKj4YEHYP36LLuc5GFKrkREREQk57FYSJg0CdzczMNQs2dn6eXc3WHBArj3XrhwAYKDITw8Sy8peZCSKxERERHJmSpVguHDzftnn4V//snSy3l4wKJF0LQpnD8PrVrBtm1ZeknJY5RciYiIiEjO9cILcOedpqTfkCFZfrmCBWHpUmjYEP79F1q2hL/+yvLLSh6h5EpEREREci5XV5g6FSwWmD4dfv45yy/p5QXLlkGdOnD6NNx3H+zZk+WXlTxAyZWIiIiI5GyNG5uiFgBPPGGKXGSxwoXNo141akBkpFkHKwuLFkoeoeRKRERERHK+t9+GEiVg3z54661suWSxYrBihVlw+Ngxk2AdOpQtl5ZcSsmViIiIiOR83t4waZJ5/847sGNHtlzW1xdWrjS1NQ4dMlMEjx3LlktLLqTkSkRERERyh44doX17uHIFBgyAxMRsuWyJEuZRr3LlYP9+k2BFRmbLpSWXUXIlIiIiIrmDxQIffmhK+q1bB598km2XLlXKJFhlysDu3aaK4OnT2XZ5ySWUXImIiIhI7lG6tHn+CmDoUDh+PNsuHRhoEqyAAPjzT7MO1r//ZtvlJRdQciUiIiIiuctTT0GDBhAVBc88k62XrlDBJFh+frB1KwQHw7lz2RqC5GBKrkREREQkd3FyMmtfOTnBt9/C999n6+UrVzZVBIsVg02boFo1+Owz8yiY5G9KrkREREQk96lZE154wbwfNMiMYmWj6tVNghUYCCdOQP/+EBQEP/wANlu2hiI5iJIrEREREcmdhg+H8uVNbfRXX832y9eqBbt2wfjxULQo/PUXtGsHzZvD+vXZHo7kAEquRERERCR38vCAKVPM+0mTYMOGbA/BzQ1CQ02J9pdeAnd3+PVXaNQIunaFvXuzPSRxICVXIiIiIpJ7tWoFjz5q5uL17w/x8Q4Jo3BhGDMG9uyBxx83VePnzzfPYz39NJw65ZCwJJspuRIRERGR3G3CBDMvb9s2eOcdh4ZSujRMnw4REfDAA6bIxaRJpsrgm29CdLRDw5MsliOSq8mTJxMYGIi7uzsNGzZkwy2GdOfNm0eVKlVwd3enRo0aLF26NPm7+Ph4XnrpJWrUqIGnpycBAQH06tWL49m4BoKIiIiIZKPixc2DTwCvvWZGsv75x6Eh1awJS5easu316sHFiya0SpXg009VWTCvcnhyNXfuXEJDQxkxYgTh4eEEBQURHBzMqRuMna5du5aePXvSr18/tmzZQocOHejQoQM7duwAICYmhvDwcF577TXCw8P57rvv2L17N+3atcvmOxMRERGRbNO7NwwbBlYrzJoFd94JCxY4OiruvdcUt5gzB8qVM5UFn3gCatSAefMgLs7REYo9OTy5mjBhAv3796dPnz5Uq1aNKVOm4OHhwbRp09Js//7773P//fczZMgQqlatyhtvvEGdOnWYNGkSAN7e3oSFhdGtWzcqV65Mo0aNmDRpEps3b+bw4cPZfHciIiIiki0sFnj7bVi7FqpWhZMnoVMn6NEDTp92aGhWK3TvbioLvv++WR9r1y7o1g0CAswzWZs2qYR7XuDsyIvHxcWxefNmhg0blrzParXSsmVL1q1bl+Yx69atIzQ0NMW+4OBgFi5ceMPrnD9/HovFQuHChdP8PjY2ltjY2OTPUVfXSYiPjyfeQQ9FJkm6vqPjkNxJ/UcyQ/1HMkP9RzIjU/2nTh1Yvx7rG29gnTABy9y52H7+mYT338fWpYv9g80AiwUGDoSHH4b33rMyfbqVEycsTJpknsuqWtXGY48l8vDDiQQEODTUXCsr/uzJyLksNpvjcuTjx49TsmRJ1q5dS+PGjZP3v/jii6xevZr1aSwQ4OrqyowZM+jZs2fyvo8++ohRo0Zx8uTJVO0vX75M06ZNqVKlCrNmzUozjpEjRzJq1KhU+2fPno2Hh0cm7lBEREREHKXwvn3U/uADvK7OXjreuDHbnniC2Bv8g3t2S0iwsHVrcX75pTTr15cgLs4JAKvVRlDQKe699wgNG57AzS3R0aHmazExMTz88MOcP38eLy+vm7Z16MhVVouPj6dbt27YbDY+/vjjG7YbNmxYitGwqKgoSpcuTevWrW/5A2a1+Ph4wsLCaNWqFS4uLg6NRXIf9R/JDPUfyQz1H8kMu/afJ54gYfRorGPHErBuHSX27CFhwgRsPXqYoSQHa9vWrH98/nwi8+fb+PJLK2vXWtmyxY8tW/zw8rLRpYsZ0WrSxJYTQs7RsuLPnqRZbenh0OTKx8cHJyenVCNOJ0+exN/fP81j/P3909U+KbE6dOgQP//8802TJDc3N9zc3FLtd3FxyTH/h5CTYpHcR/1HMkP9RzJD/Ucywy79x8UF3noLunSBPn2wbN2Kc+/e8O23ZgHiEiXsFW6m+PjAk0+abf9+mDnTbH//bWHaNAvTplkpXx569TJbuXKOjjhns+efPRk5j0MLWri6ulK3bl1WrlyZvC8xMZGVK1emmCZ4vcaNG6doDxAWFpaifVJitXfvXlasWEGxYsWy8C5EREREJMerXRs2boRRo0zCtWiRWeF3xowcV0miQgUT5v79sGoV9OkDBQvCgQMwciSUL2+qDT7/PPz0E8TEODpiSeLwaoGhoaFMnTqVGTNmsHPnTgYOHEh0dDR9+vQBoFevXikKXgwePJhly5Yxfvx4du3axciRI9m0aRMhISFwNbHq0qULmzZtYtasWSQkJBAZGUlkZCRxqnUpIiIikn+5uMDw4bB5M9StC+fOweOPw0MPwdGjjo4uFasVmjeHadMgMhK+/BJatjSzGXfsMGsn33+/WT+5VSt4913YujXH5Yr5isOTq+7duzNu3DiGDx9OrVq1iIiIYNmyZfj5+QFw+PBhTpw4kdy+SZMmzJ49m08//ZSgoCDmz5/PwoULqV69OgDHjh1j0aJFHD16lFq1alGiRInkbe3atQ67TxERERHJIWrUgD/+MKXbXV3Nar/Vq8Py5Y6O7IY8Pc3ayGFhprL8N99Av35QujTExsKKFfDii1Crlpnp2KsXfPWVqUgv2SdHFLQICQlJHnn6r1WrVqXa17VrV7p27Zpm+8DAQBxYAFFEREREcgNnZ7PocPv2Zt7dhg3w4IPw2WdmQeIcrFgx6NrVbDYb7N5t8sLly+GXX0xC9eWXZgMICoLWrc1Wvz54ezv6DvKuHJFciYiIiIg4RLVqsGaNSbC+/tpMEzx6FF5+OUdUE7wViwWqVDHbM8+YUay1a68lW+HhZqrg1q1m2iCYaYQVKlzbype/9j4gwExHlNuj5EpERERE8jc3NzOHrnRpGDvW1EY/csSs7Oucu/667OYG995rttGjzRTCFStMorVihckb//3XbBs3pn18uXKpE68qVcxrLsg3HSp39RYRERERkaxgtcI775gE65ln4JNP4PhxM5rl6eno6G5b8eLQs6fZAC5cgIMHTSXCpO3AAfN66JAZ+dq1y2z/5esLTZpA06Zmq1PHJGNyjZIrEREREZEkISFmbtwjj8APP0CLFrB4sclS8oBChaBmTbP915UrcPjwtWTr+sRr5044dQoWLjQbV0e56te/lnA1aWLW68rPlFyJiIiIiFyvUyczh65dO1PookkTWLbMzIvLw5ydzTTA8uVNyffrxcaaCva//35tO3MGfvvNbEkqV742stW0KdxxR/6aSqjkSkRERETkv5o2NRnEAw/Avn3QuDEsWWKGavIhNzeTYzZpAkOGmCqFe/emTLZ27TKVC3fvNmtzgRnJql3bVCysVcu8Vq5slhzLi5RciYiIiIikpUoVWLcO2rSBLVvgnnvMAlMPPujoyBzOYjGjUnfcYQotAvzzj6lUmJRsbdxoRrfCwsyWxNUV7rwzZcIVFARFijjsduxGyZWIiIiIyI34+8Pq1WZRqZ9+MutiTZkC//ufoyPLcYoVg7ZtzcbVqYRbt0JExLVy8Nu2maIaW7aY7XqlS6dOuCpUyF2l4ZVciYiIiIjcTKFCprjFgAHwxRfQv78p1T5yZP56oCiD3NygQQOzJUlMhL//Tplwbd1q9h05YrbFi6+1v3ABChZ0SPi3RcmViIiIiMituLiYB4lKl4Y33oDXXzeZwCef5N0HiLKA1XqtaEanTtf2nz9vRrWSkq2ICLh8OXclVii5EhERERFJJ4vFJFWlSsHAgTB9Opw4AS+8AAkJppZ5RrbSpc00Qw8PR9+Zw3l7w913my2JzebIiG6PkisRERERkYwYMMCshdWtmynRvmzZ7Z/L2xsefdRMNQwKsmeUuV5unHGp5EpEREREJKMeeghWrTKjVmfPmkWiMrJZrbBmjXnYaPJks9WrZ5Ksnj3Nc16S6yi5EhERERG5HQ0amATpdiUmwsqVMHUqLFwImzaZLTQUevQwiVaDBrlzCCefUnIlIiIiIuIIViu0amW2U6dg5kz47DOzCu/nn5utRg2TZD36aMYWgrLZzMJTf/99bTtyBMqVM6NuFStm5Z3lW0quREREREQczdfXTDF8/nn49VczmjV/PmzfDs88Ay++CF26mEQrqerDv/+apOngwZRJVNIWHZ32tZ57ziyQ/NBDZlGqJk3MVEXJNP2KIiIiIiI5hcUCzZqZ7YMPYNYsk2ht2wZffWW2EiXMAlAXL976fCVKQGCg2UqWhPBwM5Vx1y6zjRtnRsQeeMAkWsHBGRshkxSUXImIiIiI5ERFikBICAwaBBs3miTr669N+fck/v4mcSpX7loSlbSVKQPu7qnPe+4cLF9uFkZeutSMgM2ebTYnJzMy1ratGdm6445sveXcTsmViIiIiEhOZrGYwhYNGsCECbBlC/j5meSpQIGMn69wYVNGvls3sz7XunWweLFJtv76y1RBXLXKTFG84w6TZN13n7leQIBJ+lRkI01KrkREREREcotChcyUQXtxcoK77jLbmDFw4MC1RGv1atizxyR0EyZcO8bd3SRZJUua1xu9z4eLIyu5EhERERERo3x5U0DjmWcgKura9MHwcDh+3EwhvHzZJGEHDtz8XIULmxE2q9WMkCUmmtf0bm5uNy7KkUMpuRIRERERkdS8vEyFwi5dru27fNk883XsmEm2jh+/9v7615gY82zXuXO3f/0rV+xyG9lJyZWIiIiIiKSPu7spnlGu3I3b2Gxm1Ov4cTh50nx2crq9LZdRciUiIiIiIvZjsYC3t9mqVnV0NNnK6ugARERERERE8gIlVyIiIiIiInag5EpERERERMQOlFyJiIiIiIjYgZIrERERERERO1ByJSIiIiIiYgdKrkREREREROxAyZWIiIiIiIgdKLkSERERERGxAyVXIiIiIiIidpAjkqvJkycTGBiIu7s7DRs2ZMOGDTdtP2/ePKpUqYK7uzs1atRg6dKlKb632WwMHz6cEiVKUKBAAVq2bMnevXuz+C5ERERERCQ/c3hyNXfuXEJDQxkxYgTh4eEEBQURHBzMqVOn0my/du1aevbsSb9+/diyZQsdOnSgQ4cO7NixI7nN2LFj+eCDD5gyZQrr16/H09OT4OBgLl++nI13JiIiIiIi+YnDk6sJEybQv39/+vTpQ7Vq1ZgyZQoeHh5MmzYtzfbvv/8+999/P0OGDKFq1aq88cYb1KlTh0mTJsHVUauJEyfy6quv0r59e2rWrMnMmTM5fvw4CxcuzOa7ExERERGR/MLZkRePi4tj8+bNDBs2LHmf1WqlZcuWrFu3Ls1j1q1bR2hoaIp9wcHByYnTwYMHiYyMpGXLlsnfe3t707BhQ9atW0ePHj1SnTM2NpbY2Njkz1FRUQDEx8cTHx9vhzu9fUnXd3Qckjup/0hmqP9IZqj/SGao/8jtyoq+k5FzOTS5OnPmDAkJCfj5+aXY7+fnx65du9I8JjIyMs32kZGRyd8n7btRm/8aPXo0o0aNSrV/4cKFeHh4ZPCussb333/v6BAkF1P/kcxQ/5HMUP+RzFD/kdtlz74TExMDV2fI3YpDk6ucYtiwYSlGw44dO0a1atX43//+59C4REREREQkZ7hw4QLe3t43bePQ5MrHxwcnJydOnjyZYv/Jkyfx9/dP8xh/f/+btk96PXnyJCVKlEjRplatWmme083NDTc3t+TPBQsW5MiRIxQqVAiLxZKJO8y8qKgoSpcuzZEjR/Dy8nJoLJL7qP9IZqj/SGao/0hmqP/I7cqKvmOz2bhw4QIBAQG3bOvQ5MrV1ZW6deuycuVKOnToAEBiYiIrV64kJCQkzWMaN27MypUrefbZZ5P3hYWF0bhxYwDKlSuHv78/K1euTE6moqKiWL9+PQMHDkxXXFarlVKlStnhDu3Hy8tLf7jIbVP/kcxQ/5HMUP+RzFD/kdtl775zqxGrJA6fFhgaGkrv3r2pV68eDRo0YOLEiURHR9OnTx8AevXqRcmSJRk9ejQAgwcPpnnz5owfP54HH3yQOXPmsGnTJj799FMALBYLzz77LG+++SaVKlWiXLlyvPbaawQEBCQncCIiIiIiIvbm8OSqe/funD59muHDhxMZGUmtWrVYtmxZckGKw4cPY7VeqxjfpEkTZs+ezauvvsrLL79MpUqVWLhwIdWrV09u8+KLLxIdHc2AAQM4d+4cd911F8uWLcPd3d0h9ygiIiIiInmfxZaeshfiMLGxsYwePZphw4aleC5MJD3UfyQz1H8kM9R/JDPUf+R2ObrvKLkSERERERGxA2s62oiIiIiIiMgtKLkSERERERGxAyVXIiIiIiIidqDkSkRERERExA6UXOVwkydPJjAwEHd3dxo2bMiGDRscHZLkQGvWrKFt27YEBARgsVhYuHBhiu9tNhvDhw+nRIkSFChQgJYtW7J3716HxSs5x+jRo6lfvz6FChXC19eXDh06sHv37hRtLl++zKBBgyhWrBgFCxakc+fOnDx50mExS87x8ccfU7NmzeTFOhs3bsyPP/6Y/L36jqTXmDFjktcqTaL+IzczcuRILBZLiq1KlSrJ3zuq/yi5ysHmzp1LaGgoI0aMIDw8nKCgIIKDgzl16pSjQ5McJjo6mqCgICZPnpzm92PHjuWDDz5gypQprF+/Hk9PT4KDg7l8+XK2xyo5y+rVqxk0aBB//PEHYWFhxMfH07p1a6Kjo5PbPPfcc/zwww/MmzeP1atXc/z4cTp16uTQuCVnKFWqFGPGjGHz5s1s2rSJFi1a0L59e/78809Q35F02rhxI5988gk1a9ZMsV/9R27lzjvv5MSJE8nbb7/9lvydw/qPTXKsBg0a2Ab9v537D62q/uM4/rq63evdnHNz6947Y2viWmo4cMt5M4m6kruFMVlkcYmrBWLejYkIobSmJPhHUVbQoB/aH6ajCSsRzday/SFaMrk2aY4WlsJcS0rbRs7Y/Xz/+M5Dl8la7dq5s+cDDpzz+Zzd+76XF+fy3vkRiVjbw8PDJi8vz+zcudPWupDcJJnm5mZrOxaLGa/Xa1555RVr7MqVK8blcpn9+/fbVCWSVV9fn5Fk2trajBnJSmpqqmlqarL26ezsNJLMiRMnbKwUySorK8u89957ZAfj0t/fb4qKikxLS4t58MEHTW1trTEcezAO9fX1pqSk5KZzduaHM1dJ6vr162pvb9fy5cutsSlTpmj58uU6ceKErbVhcjl//rx6e3vjspSZmany8nKyhFGuXr0qScrOzpYktbe3648//ojLzz333KP8/HzygzjDw8NqbGzU4OCg/H4/2cG4RCIRPfbYY3E5EccejNN3332nvLw8zZkzR6FQSBcuXJBszk/KLX11/GOXL1/W8PCwPB5P3LjH49G5c+dsqwuTT29vrzSSnT/zeDzWHCBJsVhMGzdu1NKlS3XvvfdKI/lxOp2aOXNm3L7kBzd0dHTI7/fr2rVrmj59upqbmzV//nxFo1GygzE1Njbq9OnTOnXq1Kg5jj34K+Xl5frggw9UXFysS5cuafv27Vq2bJnOnj1ra35orgAA0sh/kM+ePRt3zTrwV4qLixWNRnX16lUdOHBA4XBYbW1tdpeFJHfx4kXV1taqpaVF06ZNs7scTELBYNBaX7hwocrLy1VQUKCPPvpIbrfbtrq4LDBJ5eTkaOrUqaOeavLTTz/J6/XaVhcmnxt5IUsYS3V1tQ4dOqRjx47pzjvvtMa9Xq+uX7+uK1euxO1PfnCD0+nU3LlzVVpaqp07d6qkpERvvPEG2cGY2tvb1dfXp0WLFiklJUUpKSlqa2vTm2++qZSUFHk8HvKDv2XmzJm6++671d3dbevxh+YqSTmdTpWWlqq1tdUai8Viam1tld/vt7U2TC6FhYXyer1xWfrtt9/01VdfkSXIGKPq6mo1Nzfriy++UGFhYdx8aWmpUlNT4/LT1dWlCxcukB/cVCwW09DQENnBmAKBgDo6OhSNRq2lrKxMoVDIWic/+DsGBgb0/fffy+fz2Xr84bLAJLZp0yaFw2GVlZVp8eLF2rVrlwYHB7V27Vq7S0OSGRgYUHd3t7V9/vx5RaNRZWdnKz8/Xxs3btSOHTtUVFSkwsJC1dXVKS8vT5WVlbbWDftFIhHt27dPn3zyiTIyMqxr0TMzM+V2u5WZmannnntOmzZtUnZ2tmbMmKGamhr5/X4tWbLE7vJhsy1btigYDCo/P1/9/f3at2+fvvzySx09epTsYEwZGRnWvZ03pKena9asWdY4+cFYNm/erJUrV6qgoEA9PT2qr6/X1KlT9fTTT9t7/LmlzyLEhL311lsmPz/fOJ1Os3jxYnPy5Em7S0ISOnbsmJE0agmHw8aMPI69rq7OeDwe43K5TCAQMF1dXXaXjSRws9xIMnv27LH2+f33382GDRtMVlaWSUtLM6tWrTKXLl2ytW4kh2effdYUFBQYp9NpcnNzTSAQMJ999pk1T3bwd/z5UeyG/OAvrF692vh8PuN0Os3s2bPN6tWrTXd3tzVvV34c5v8/rgAAAACACeCeKwAAAABIAJorAAAAAEgAmisAAAAASACaKwAAAABIAJorAAAAAEgAmisAAAAASACaKwAAAABIAJorAAAAAEgAmisAACbI4XDo448/trsMAIDNaK4AAJPamjVr5HA4Ri0VFRV2lwYA+I9JsbsAAAAmqqKiQnv27Ikbc7lcttUDAPhv4swVAGDSc7lc8nq9cUtWVpY0csleQ0ODgsGg3G635syZowMHDsT9fUdHhx5++GG53W7NmjVL69at08DAQNw+u3fv1oIFC+RyueTz+VRdXR03f/nyZa1atUppaWkqKirSwYMHrblff/1VoVBIubm5crvdKioqGtUMAgAmP5orAMBtr66uTlVVVTpz5oxCoZCeeuopdXZ2SpIGBwe1YsUKZWVl6dSpU2pqatLnn38e1zw1NDQoEolo3bp16ujo0MGDBzV37ty499i+fbuefPJJffPNN3r00UcVCoX0yy+/WO//7bff6siRI+rs7FRDQ4NycnL+5W8BAHCrOYwxxu4iAAD4p9asWaO9e/dq2rRpceNbt27V1q1b5XA4tH79ejU0NFhzS5Ys0aJFi/T222/r3Xff1QsvvKCLFy8qPT1dknT48GGtXLlSPT098ng8mj17ttauXasdO3bctAaHw6EXX3xRL7/8sjTSsE2fPl1HjhxRRUWFHn/8ceXk5Gj37t239LsAANiLe64AAJPeQw89FNc8SVJ2dra17vf74+b8fr+i0agkqbOzUyUlJVZjJUlLly5VLBZTV1eXHA6Henp6FAgExqxh4cKF1np6erpmzJihvr4+SdLzzz+vqqoqnT59Wo888ogqKyt1//33T/BTAwCSDc0VAGDSS09PH3WZXqK43e5x7Zeamhq37XA4FIvFJEnBYFA//vijDh8+rJaWFgUCAUUiEb366qu3pGYAgD245woAcNs7efLkqO158+ZJkubNm6czZ85ocHDQmj9+/LimTJmi4uJiZWRk6K677lJra+uEasjNzVU4HNbevXu1a9cuvfPOOxN6PQBA8uHMFQBg0hsaGlJvb2/cWEpKivXQiKamJpWVlemBBx7Qhx9+qK+//lrvv/++JCkUCqm+vl7hcFjbtm3Tzz//rJqaGj3zzDPyeDySpG3btmn9+vW64447FAwG1d/fr+PHj6umpmZc9b300ksqLS3VggULNDQ0pEOHDlnNHQDg9kFzBQCY9D799FP5fL64seLiYp07d04aeZJfY2OjNmzYIJ/Pp/3792v+/PmSpLS0NB09elS1tbW67777lJaWpqqqKr322mvWa4XDYV27dk2vv/66Nm/erJycHD3xxBPjrs/pdGrLli364Ycf5Ha7tWzZMjU2Nibs8wMAkgNPCwQA3NYcDoeam5tVWVlpdykAgNsc91wBAAAAQALQXAEAAABAAnDPFQDgtsbV7wCAfwtnrgAAAAAgAWiuAAAAACABaK4AAAAAIAForgAAAAAgAWiuAAAAACABaK4AAAAAIAForgAAAAAgAWiuAAAAACAB/gdKrnXNcDZxbwAAAABJRU5ErkJggg==", + "text/plain": [ + "<Figure size 1000x500 with 1 Axes>" ] + }, + "metadata": {}, + "output_type": "display_data" } - ], - "metadata": { + ], + "source": [ + "for epoch in range(NEPOCHS):\n", + " model.train() # set to training mode\n", + " train_loss = 0\n", + "\n", + " for batch_src, batch_labels, batch_padding_mask in train_loader:\n", + " optimizer.zero_grad()\n", + " output = model(batch_src, batch_padding_mask)\n", + " loss = criterion(output.squeeze(1), batch_labels)\n", + " train_loss += loss.item()/len(train_loader)\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # Evaluate performance\n", + " model.eval()\n", + " test_loss = 0\n", + "\n", + " with torch.no_grad():\n", + " for batch_src, batch_labels, batch_padding_mask in test_loader:\n", + " output = model(batch_src, batch_padding_mask)\n", + " loss = criterion(output.squeeze(1), batch_labels)\n", + " test_loss += loss.item()/len(test_loader)\n", + "\n", + " test_err.append(test_loss)\n", + " train_err.append(train_loss)\n", + " print(f\"Epoch {epoch + 1}/{NEPOCHS} \\t Train Err: {train_loss:.4f} \\t Test Err: {test_loss:.4f} \\t baseline err: {train_baseline:.4f}\")\n", + "\n", + "plt.figure(figsize=(10, 5))\n", + "plt.plot(test_err, label='Test', color='red')\n", + "plt.plot(train_err, label='Train', color='blue')\n", + "plt.title('Accuracy vs Epochs')\n", + "plt.xlabel('Epochs'); plt.ylabel('Accuracy')\n", + "plt.legend(); plt.grid()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "v1hCiItHDWxJ" + }, + "outputs": [], + "source": [ + "## Q: why is this not working so well?\n", + "\n", + "## maybe first try a simpler problem: just give it points for distinguishing between distance 1 or not" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": { + "id": "LoGEmM5lH7_A" + }, + "outputs": [], + "source": [ + "batch_src, batch_labels, batch_padding_mask = next(iter(train_loader))\n", + "output = model(batch_src, batch_padding_mask)" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { "colab": { - "gpuType": "T4", - "provenance": [] + "base_uri": "https://localhost:8080/" }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" + "id": "hO8AhX3G7vF8", + "outputId": "8f4a3ca6-db47-434d-95a4-4631bc73de62" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1 \t nan\n", + "1 \t nan\n", + "1 \t nan\n", + "1 \t nan\n", + "0 \t nan\n", + "0 \t nan\n", + "1 \t nan\n", + "1 \t nan\n", + "1 \t nan\n", + "1 \t nan\n", + "1 \t nan\n", + "1 \t nan\n", + "1 \t nan\n", + "1 \t nan\n", + "1 \t nan\n", + "1 \t nan\n", + "1 \t nan\n", + "1 \t nan\n", + "1 \t nan\n", + "0 \t nan\n", + "1 \t nan\n", + "0 \t nan\n", + "1 \t nan\n", + "1 \t nan\n", + "0 \t nan\n", + "1 \t nan\n", + "1 \t nan\n", + "0 \t nan\n", + "1 \t nan\n", + "1 \t nan\n", + "1 \t nan\n", + "1 \t nan\n" + ] } + ], + "source": [ + "for x,y in zip(batch_labels.tolist(), output.squeeze(1).tolist()):\n", + " print(f\"{int(x)} \\t {y:.1f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dRdUGbFmkPtK" + }, + "outputs": [], + "source": [ + "batch_src[2]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LC6Xv3YfC0Rm" + }, + "source": [ + "# Step 5: Fine Tune" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JtTLXn4zC1z_" + }, + "source": [ + "# Step 6: Test generalization" + ] + } + ], + "metadata": { + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" }, - "nbformat": 4, - "nbformat_minor": 0 -}
\ No newline at end of file + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} |