From 037acd689ee7f1f392d8209f1c3c5cabde90be5f Mon Sep 17 00:00:00 2001 From: SIPB Date: Tue, 10 Dec 2024 22:19:47 -0500 Subject: Final commit --- configs/bert_11M.json | 26 + configs/bert_19M.json | 26 + configs/bert_35M.json | 26 + configs/bert_50M.json | 26 + configs/bert_67M.json | 26 + configs/bert_6M.json | 26 + configs/test.json | 26 + data/ltr_riddles.txt | 40 + data/make-histogram-thing.ipynb | 546 +++++++++++ data/riddles.txt | 40 + .../wandb_export_2024-12-04T19_56_43.325-05_00.csv | 21 + finetune_QA.py | 304 ++++++ finetune_bert-japanese.py | 225 +++++ finetune_bert.py | 935 ++++++------------ notebooks/Inference.ipynb | 638 +++++++++++++ notebooks/Riddles.ipynb | 362 +++++++ notebooks/Riddles_FixedPos.ipynb | 302 ++++++ notebooks/Riddles_FixedPos_QAChars.ipynb | 345 +++++++ notebooks/Right_to_Left_NLP.ipynb | 712 +++++++------- notebooks/Stat_Tests.ipynb | 487 ++++++++++ notebooks/addition.ipynb | 836 ++++++++++++++++ notebooks/japanese.ipynb | 489 ++++++++++ notebooks/loss | 1000 ++++++++++++++++++++ notebooks/loss.txt | 315 ++++++ notebooks/qa.ipynb | 247 +++++ notebooks/rtl.ipynb | 18 - requirements.txt | 5 +- utils.py | 150 ++- 28 files changed, 7176 insertions(+), 1023 deletions(-) create mode 100644 configs/bert_11M.json create mode 100644 configs/bert_19M.json create mode 100644 configs/bert_35M.json create mode 100644 configs/bert_50M.json create mode 100644 configs/bert_67M.json create mode 100644 configs/bert_6M.json create mode 100644 configs/test.json create mode 100644 data/ltr_riddles.txt create mode 100644 data/make-histogram-thing.ipynb create mode 100644 data/riddles.txt create mode 100644 data/wandb_export_2024-12-04T19_56_43.325-05_00.csv create mode 100644 finetune_QA.py create mode 100644 finetune_bert-japanese.py create mode 100644 notebooks/Inference.ipynb create mode 100644 notebooks/Riddles.ipynb create mode 100644 notebooks/Riddles_FixedPos.ipynb create mode 100644 notebooks/Riddles_FixedPos_QAChars.ipynb create mode 100644 notebooks/Stat_Tests.ipynb create mode 100644 notebooks/addition.ipynb create mode 100644 notebooks/japanese.ipynb create mode 100644 notebooks/loss create mode 100644 notebooks/loss.txt create mode 100644 notebooks/qa.ipynb diff --git a/configs/bert_11M.json b/configs/bert_11M.json new file mode 100644 index 0000000..695789c --- /dev/null +++ b/configs/bert_11M.json @@ -0,0 +1,26 @@ +{ + "_name_or_path": "bert-base-uncased", + "architectures": [ + "BertForMaskedLM" + ], + "attention_probs_dropout_prob": 0.1, + "classifier_dropout": null, + "gradient_checkpointing": false, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 256, + "initializer_range": 0.02, + "intermediate_size": 1024, + "layer_norm_eps": 1e-12, + "max_position_embeddings": 128, + "model_type": "bert", + "num_attention_heads": 4, + "num_hidden_layers": 4, + "pad_token_id": 0, + "position_embedding_type": "absolute", + "torch_dtype": "float32", + "transformers_version": "4.46.2", + "type_vocab_size": 2, + "use_cache": true, + "vocab_size": 30522 +} \ No newline at end of file diff --git a/configs/bert_19M.json b/configs/bert_19M.json new file mode 100644 index 0000000..891e78c --- /dev/null +++ b/configs/bert_19M.json @@ -0,0 +1,26 @@ +{ + "_name_or_path": "bert-base-uncased", + "architectures": [ + "BertForMaskedLM" + ], + "attention_probs_dropout_prob": 0.1, + "classifier_dropout": null, + "gradient_checkpointing": false, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 384, + "initializer_range": 0.02, + "intermediate_size": 1536, + "layer_norm_eps": 1e-12, + "max_position_embeddings": 128, + "model_type": "bert", + "num_attention_heads": 6, + "num_hidden_layers": 4, + "pad_token_id": 0, + "position_embedding_type": "absolute", + "torch_dtype": "float32", + "transformers_version": "4.46.2", + "type_vocab_size": 2, + "use_cache": true, + "vocab_size": 30522 +} \ No newline at end of file diff --git a/configs/bert_35M.json b/configs/bert_35M.json new file mode 100644 index 0000000..b697e04 --- /dev/null +++ b/configs/bert_35M.json @@ -0,0 +1,26 @@ +{ + "_name_or_path": "bert-base-uncased", + "architectures": [ + "BertForMaskedLM" + ], + "attention_probs_dropout_prob": 0.1, + "classifier_dropout": null, + "gradient_checkpointing": false, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 512, + "initializer_range": 0.02, + "intermediate_size": 2048, + "layer_norm_eps": 1e-12, + "max_position_embeddings": 128, + "model_type": "bert", + "num_attention_heads": 8, + "num_hidden_layers": 6, + "pad_token_id": 0, + "position_embedding_type": "absolute", + "torch_dtype": "float32", + "transformers_version": "4.46.2", + "type_vocab_size": 2, + "use_cache": true, + "vocab_size": 30522 +} \ No newline at end of file diff --git a/configs/bert_50M.json b/configs/bert_50M.json new file mode 100644 index 0000000..9e2facb --- /dev/null +++ b/configs/bert_50M.json @@ -0,0 +1,26 @@ +{ + "_name_or_path": "bert-base-uncased", + "architectures": [ + "BertForMaskedLM" + ], + "attention_probs_dropout_prob": 0.1, + "classifier_dropout": null, + "gradient_checkpointing": false, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 640, + "initializer_range": 0.02, + "intermediate_size": 2560, + "layer_norm_eps": 1e-12, + "max_position_embeddings": 128, + "model_type": "bert", + "num_attention_heads": 8, + "num_hidden_layers": 6, + "pad_token_id": 0, + "position_embedding_type": "absolute", + "torch_dtype": "float32", + "transformers_version": "4.46.2", + "type_vocab_size": 2, + "use_cache": true, + "vocab_size": 30522 +} \ No newline at end of file diff --git a/configs/bert_67M.json b/configs/bert_67M.json new file mode 100644 index 0000000..bac1d03 --- /dev/null +++ b/configs/bert_67M.json @@ -0,0 +1,26 @@ +{ + "_name_or_path": "bert-base-uncased", + "architectures": [ + "BertForMaskedLM" + ], + "attention_probs_dropout_prob": 0.1, + "classifier_dropout": null, + "gradient_checkpointing": false, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 768, + "initializer_range": 0.02, + "intermediate_size": 3072, + "layer_norm_eps": 1e-12, + "max_position_embeddings": 128, + "model_type": "bert", + "num_attention_heads": 12, + "num_hidden_layers": 6, + "pad_token_id": 0, + "position_embedding_type": "absolute", + "torch_dtype": "float32", + "transformers_version": "4.46.2", + "type_vocab_size": 2, + "use_cache": true, + "vocab_size": 30522 +} \ No newline at end of file diff --git a/configs/bert_6M.json b/configs/bert_6M.json new file mode 100644 index 0000000..b093bc5 --- /dev/null +++ b/configs/bert_6M.json @@ -0,0 +1,26 @@ +{ + "_name_or_path": "bert-base-uncased", + "architectures": [ + "BertForMaskedLM" + ], + "attention_probs_dropout_prob": 0.1, + "classifier_dropout": null, + "gradient_checkpointing": false, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 160, + "initializer_range": 0.02, + "intermediate_size": 640, + "layer_norm_eps": 1e-12, + "max_position_embeddings": 128, + "model_type": "bert", + "num_attention_heads": 4, + "num_hidden_layers": 4, + "pad_token_id": 0, + "position_embedding_type": "absolute", + "torch_dtype": "float32", + "transformers_version": "4.46.2", + "type_vocab_size": 2, + "use_cache": true, + "vocab_size": 30522 +} \ No newline at end of file diff --git a/configs/test.json b/configs/test.json new file mode 100644 index 0000000..b093bc5 --- /dev/null +++ b/configs/test.json @@ -0,0 +1,26 @@ +{ + "_name_or_path": "bert-base-uncased", + "architectures": [ + "BertForMaskedLM" + ], + "attention_probs_dropout_prob": 0.1, + "classifier_dropout": null, + "gradient_checkpointing": false, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 160, + "initializer_range": 0.02, + "intermediate_size": 640, + "layer_norm_eps": 1e-12, + "max_position_embeddings": 128, + "model_type": "bert", + "num_attention_heads": 4, + "num_hidden_layers": 4, + "pad_token_id": 0, + "position_embedding_type": "absolute", + "torch_dtype": "float32", + "transformers_version": "4.46.2", + "type_vocab_size": 2, + "use_cache": true, + "vocab_size": 30522 +} \ No newline at end of file diff --git a/data/ltr_riddles.txt b/data/ltr_riddles.txt new file mode 100644 index 0000000..1ea51e1 --- /dev/null +++ b/data/ltr_riddles.txt @@ -0,0 +1,40 @@ +I am footsteps. The more you take, the more you leave behind. +I am darkness. The more you have of me, the less you see. +I am fire. I am not alive, but I grow. I don’t have lungs, but I need air. I don’t have a mouth, but water kills me. +I am a towel. I get wetter the more I dry. +I am a candle. I become shorter the longer I live. +I am breath. I am light as a feather, yet the strongest man can’t hold me for much longer than a minute. +I am the wind. I am invisible, but you can feel me. I am intangible, but you can hear me. +I am a piano. I have keys but open no locks. +I am fog. The more of me there is, the less you see. +I am the future. I am always in front of you, but you can never see me. +I am a cloud. I don’t have wings, but I can fly. I don’t have eyes, but I can cry. Wherever I go, darkness follows me. +I am a battery. I am not alive, but I can die. +I am a heart. I can be stolen, but I can’t be touched. +I am the letter "M." I am something that comes once in a minute, twice in a moment, but never in a thousand years. +I am a promise. I can be cracked, but never broken. +I am your shadow. I am always with you, but I can’t be seen. +I am a footprint. I’m small but I can cover miles. +I am a secret. The more you have of me, the less you know. +I am light. I’m something that can fill a room, but I don’t take up space. +I am your name. I am always with you, but you never see me. I can be forgotten, but I never leave. +I am silence. The more of me there is, the less you hear. +I am a stamp. I can travel around the world while staying in the corner. +I am a code. I am something that can be cracked, but I can’t be touched. +I am sound. I am something you can hear, but not touch. I can be loud or soft, but I can never be seen. +I am the present moment. I am something that you can never keep, no matter how hard you try. +I am a rumor. I am not alive, but I grow. I don’t have a mouth, but I can speak. +I am a clock. I am always running, but I never move. +I am a hole. I get bigger the more you take away. +I am understanding. I can’t be seen, but I can be felt. I have no color, but I make things clear. +I am a pencil. I get smaller the more you use me. +I am a promise. I can be broken without being touched. +I am time. I am something that everyone has, but no one can keep forever. +I am a thought. I can be light as a feather, but even the strongest hands cannot hold me. +I am a debt. The more you take from me, the greater I become. +I am the horizon. I am often in front of you, but I’m never within reach. +I am the sky. You can see me every day, but I will never be seen the same way twice. +I am a reputation. I am not alive, but I grow over time. +I am a feeling. I can’t be touched, but I can touch everything. +I am a look. I never speak, but I can communicate. +I am a deadline. I can be hard, but I am not solid. \ No newline at end of file diff --git a/data/make-histogram-thing.ipynb b/data/make-histogram-thing.ipynb new file mode 100644 index 0000000..72e7dbb --- /dev/null +++ b/data/make-histogram-thing.ipynb @@ -0,0 +1,546 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "execution_state": "idle", + "id": "7a21c467-a114-447d-bdb8-91778b59a3ad", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "csv_filename = 'wandb_export_2024-12-04T19_56_43.325-05_00.csv'\n", + "df = pd.read_csv(csv_filename)\n", + "# https://huggingface.co/datasets/ntotsuka123/ja-pretrain/viewer/default/train?p=1&row=120" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "execution_state": "idle", + "id": "0732274a-bc56-44a3-912f-e023c344bc56", + "metadata": {}, + "outputs": [], + "source": [ + "df = df.drop([0, 1, 10, 11, 12, 15,16,17,18,19])" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "execution_state": "idle", + "id": "23adfc0e-12af-4280-a31e-1601b1bfc3cf", + "metadata": {}, + "outputs": [], + "source": [ + "def extract_size(name):\n", + " if 'distilbert_base' in name:\n", + " return '67M'\n", + " elif 'bert_6M' in name or 'bert_6_' in name:\n", + " return '6M'\n", + " elif 'bert_11' in name:\n", + " return '11M'\n", + " elif 'bert_19' in name:\n", + " return '19M'\n", + " elif 'bert_35' in name:\n", + " return '35M'\n", + " elif 'bert_base' in name:\n", + " return '110M' # Regular BERT base models have ~110M parameters\n", + " else:\n", + " return 'other'" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "execution_state": "idle", + "id": "895cde04-f6f8-4f47-8f48-16008dd68a55", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Nameval_losssizeTypeval_loss_expparams
2bert_6M_rtl_scratch4.7444766MRTL114.9475286
3bert_6_ltr_scratch4.7613656MLTR116.9053546
4bert_11_rtl_scratch4.44695011MRTL85.36615611
5bert_11_ltr_scratch4.46237911MLTR86.69347611
6bert_19_rtl_scratch4.17732019MRTL65.19093219
7bert_19_ltr_scratch4.18627119MLTR65.77702619
8bert_35_rtl_scratch3.92785735MRTL50.79798335
9bert_35_ltr_scratch3.94159535MLTR51.50069135
13distilbert_base_ltr_scratch3.68630767MLTR39.89725367
14distilbert_base_rtl_scratch3.68856667MRTL39.98746167
\n", + "
" + ], + "text/plain": [ + " Name val_loss size Type val_loss_exp params\n", + "2 bert_6M_rtl_scratch 4.744476 6M RTL 114.947528 6\n", + "3 bert_6_ltr_scratch 4.761365 6M LTR 116.905354 6\n", + "4 bert_11_rtl_scratch 4.446950 11M RTL 85.366156 11\n", + "5 bert_11_ltr_scratch 4.462379 11M LTR 86.693476 11\n", + "6 bert_19_rtl_scratch 4.177320 19M RTL 65.190932 19\n", + "7 bert_19_ltr_scratch 4.186271 19M LTR 65.777026 19\n", + "8 bert_35_rtl_scratch 3.927857 35M RTL 50.797983 35\n", + "9 bert_35_ltr_scratch 3.941595 35M LTR 51.500691 35\n", + "13 distilbert_base_ltr_scratch 3.686307 67M LTR 39.897253 67\n", + "14 distilbert_base_rtl_scratch 3.688566 67M RTL 39.987461 67" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df['size'] = df['Name'].apply(extract_size)\n", + "df['Type'] = df['Name'].apply(lambda x: 'LTR' if 'ltr' in x else 'RTL')\n", + "df['val_loss_exp'] = np.exp(df['val_loss'])\n", + "df['params'] = df['size'].str.slice(stop=-1).apply(lambda s: int(s))\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "execution_state": "idle", + "id": "5d5922ca-79bf-4761-954a-8755d70ad626", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_480557/1275629619.py:9: FutureWarning: \n", + "\n", + "The `ci` parameter is deprecated. Use `errorbar=None` for the same effect.\n", + "\n", + " sns.barplot(x='size', y='val_loss_exp', hue='Type', data=df_sorted_pairs, dodge=True, palette=\"Set2\", ci=None)\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABKUAAAMWCAYAAAAgRDUeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAACkEElEQVR4nOzdeVxV1f7/8fdhElAUcU4tRUUcUDOnUjOHNC3NecqhNMtKK723wbw0qJlNt7rZZGlqmTnhmGmKZmaOoQnOigM44IQIojLt3x9f2T+Q2XM4cOD1fDx6PDZnr73XZx/2wXiz1toWwzAMAQAAAAAAAHbkVNAFAAAAAAAAoPghlAIAAAAAAIDdEUoBAAAAAADA7gilAAAAAAAAYHeEUgAAAAAAALA7QikAAAAAAADYHaEUAAAAAAAA7I5QCgAAAAAAAHZHKAUAAAAAAAC7I5QCABQ5n3/+uerWrau6desWdClZ6tChg+rWravXX3+9oEtBIRIUFGTeu5GRkfnSx9ChQ1W3bl0NHTo0X86fW47wOQWKmsjISPNzFxQUVNDlAIBcCroAAED+2b59u4YNG5bpPnd3d/n4+KhevXrq2rWrunbtKhcX/lmA44mMjFTHjh3Nry0Wi4KDg1W1atUcj+3SpYtOnDhhfv3ee++pd+/e+VZrUWIYhjZs2KBffvlFYWFhunDhgm7evClPT09VrlxZvr6+atSokdq0aSN/f/+CLrdA3H5v5mTMmDEaO3ZsvtbkSKKjoxUUFKTNmzfryJEjiomJkcViUZkyZVS1alX5+/vr3nvvVdu2bVWuXLmCLhcAcAf47QMAiqkbN27ozJkzOnPmjIKDgzVnzhx99dVXqlChQkGXVuyl/UWWkCTvDMPQypUrNXr06Gzb7dmzJ10ghdy7ePGiXnzxRf39998Z9sXGxio2NlZHjhzR2rVr9eGHH2r16tWqVatWgdQKxxQcHKw33nhDV65cybDvwoULunDhgvbs2aOff/5ZjRs31sKFCwukTlvh5z6A4opQCgCKiUGDBmnw4MHm1/Hx8QoLC9OsWbN0+vRphYaG6vnnn9fChQtlsVgKtNbiYMOGDQVdQpFUokQJ3bx5U8uXL88xlFq+fHm6Y5A7CQkJeuqpp3T48GFJUv369dW7d2/Vq1dPJUuWVFxcnI4dO6adO3dq06ZNio2NzfQ8Y8eOLVajgjp27KiXX3452zaM9vk/u3bt0ksvvaTExEQ5Ozvr0UcfVfv27VWtWjU5Ozvr4sWL2r9/vzZv3qzdu3cXdLkAACsQSgFAMVGuXDn5+fmle61Jkybq3r27+vXrp5MnT2rv3r3auHGjOnToUGB1Atbo0KGDfv31V4WHhys0NFQBAQGZtktMTNTq1aulW2FB6jZytmjRIjOQ6t27t9599105OaVfprR58+YaOHCgEhIStGrVKpUuXbqAqi08SpcuneFnMDL33nvvmYHUd999pwceeCBDm3bt2um5557T6dOntXXr1gKpEwBgPRY6B4BirkyZMnrmmWfMrzdv3lyg9QDWqFWrlho2bCilGQmVmU2bNunKlStydXVVt27d7Fih4wsODpYkubi4aMKECRkCqbTc3NzUu3dvpgUj16KiohQWFiZJ6tSpU6aBVFpVq1ZV37597VQdAMDWGCkFAFCjRo3M7TNnzmTYn5ycrBUrVmjNmjXat2+frly5opIlS8rX11edO3fWoEGD5O7unum5hw4dqh07dqhFixb64YcfdOLECc2dO1d//vmnoqKidOPGDQUHB6tatWrpFmafO3eumjdvrsWLFysoKEjh4eFKSEjQ3XffrUcffVRPPvmkSpQoYdV137x5U4sWLdK6det09OhRxcTEyMvLS3Xr1tWjjz6qXr16ZVj8fdeuXRo6dKhSUlLUrl07zZgxI9Nzx8XFqUePHjp9+rR8fHy0cuVKlS9f3tzfoUMHnT59Wr169dK0adPM129/EtmECRM0YcKEdK+lLobcq1cv7d+/X76+vvr111+zvdbo6Gi1bdtWiYmJGjRokN5+++1cvUcTJkxQUFCQSpQoob/++kulSpXKtn3qwuEBAQFavHhxun1hYWH66aef9PfffysqKkrJycny8fFRuXLl1LhxY7Vp00YdOnSwevro448/rrCwMK1evVqvv/56pgv4pwZW7du3z/UonpSUFK1cuVKrVq3S/v37FRMTo1KlSqlOnTp65JFH1K9fP7m5uWV7jpiYGH377bdav369zpw5o5IlS6pu3boaMGCAunbtmutrvJN711ZSf0aULVvWqhFQn3/+uaZPny5JOnToULp9qT83civ1Z8jtLly4oB9//FGbN29WZGSk4uPjVa5cOTVp0kQDBgzIMfAoKEFBQebnPjg4WBUrVtRPP/2k1atX6+TJk7py5UqGRdETEhK0aNEirVmzRkeOHFFcXJzKlCmj+vXr67HHHlP37t2zDBBff/11LV26VFWrVtWGDRt04cIFzZo1Sxs2bNC5c+dUunRpNW3aVGPGjFGdOnXM4yIjIzV79mxt3rxZZ8+eValSpXT//ffrpZde0t13331H13727Flz+5577rmjc9wuLi5OCxcu1O+//65jx44pJiZGbm5uuvvuu3XfffepW7duuu+++9Idk5d/uyTp/PnzWrdunbZv366DBw/q/PnzSkpKUtmyZdWwYUN1795djzzySKbfg7z83L/d4cOHtWDBAu3YsUPnzp1TfHy8vL29VadOHbVu3VqPP/64KlasmO37s2XLFv3www8KDQ1VTEyMKlasqLZt2+q5555T5cqV8/ReA0BeEUoBANL98pqcnJxu35kzZ/Tcc8/p4MGD6V6/cuWKQkJCFBISovnz5+ubb75RzZo1s+1n/fr1euWVVxQfH59jTYmJiXrmmWcyjNw6dOiQDh06pBUrVmj27Nl3PALj4MGDev7553X69Ol0r1++fFlbt27V1q1btWDBAn399dfpwqRmzZrpmWee0ddff61NmzZp3rx5euKJJzKc/5133jHP/e6776Y7h6307dtXkyZNUnh4uPbs2aMmTZpk2XblypVKTEyUJPXp0yfXffTo0UNBQUG6efOm1q1bp169emXZNjQ01Fw4vHv37un2zZ49W++//75SUlLSvX7u3DmdO3dO+/bt008//aSQkBCVLFky1/Vl5tFHH9X777+vS5cu6c8//9RDDz2Ubn9MTIx+//136VaAlRtXrlzRc889p5CQkHSvR0dHa8eOHdqxY4fmzZunb7/9Nsun/h07dkxPPvmkzp8/b7528+ZN8377448/1Lx58xxrudN711ZcXV2lW4udX7lyRd7e3jbvwxZWrFiht956K8PPm3PnzmnNmjVas2aN+vbtq3feeadQP3k0OjpaY8aM0YEDB7JsExkZqVGjRik8PDzd6xcvXtQff/yhP/74QwsWLNCXX36Z4/fr4MGDevrpp3XhwgXztRs3bmjNmjX6448/9O2336pZs2baunWrxo4dm27NsJs3b2rVqlXavHmz5s2bly7Ayq3U+0u3PjPW+uuvvzR+/HhFR0enez0xMVEHDhzQgQMH9OOPP2YIRtPK6d+u5ORktWvXLsPPN90KqzZs2KANGzZo8eLF+vzzz63+GZfa5wcffKA5c+bIMIx0+y5evKiLFy9q69atOnbsWLo/fNzu448/zvDHldOnT+vnn3/Wb7/9ph9//JGHFADIV4X3X2AAgN2krg8jKd1fVKOjozV48GCdPXtWbm5u6t+/v5o3b66qVasqPj5eW7Zs0dy5c3Xy5EmNGjVKS5culZeXV6Z9nDlzRq+88orc3d313HPPqVmzZnJ2dlZoaKg8PT0ztP/0008VGhqqNm3aaNCgQapcubLOnTunn376SVu2bNHRo0c1evRoLVy4UM7Oznm63pMnT2rIkCGKjY1VqVKl9MQTT6hRo0aqXLmyrly5og0bNmjBggXm4u/z5s1L94vSmDFj9OeffyosLEwffPCBWrVqle5/2n/55RetWLFCkjRgwIA8rdG1cuVKnT9/XiNHjpQkvfzyyxkeKZ+6GHKPHj30wQcf6MaNGwoKCso2lAoKCpJu/UU+q3WWMtOyZUtVrFhR58+f18qVK7MNpVatWiVJ5sLEqQ4ePGgGUtWqVdOQIUPk7+8vb29vXbt2TcePH9e2bdtstvh7uXLl1KZNG/3+++9avnx5hlDq119/VUJCgry9vfXggw/muFBycnKyRo8ebbZr0aKFnnjiCVWrVk3nz5/XkiVLtH79ejN0WrZsWYZfOuPi4jRy5EgzkOrWrZt69uypcuXK6cSJE/r+++8VFBSkI0eOZFuLtfeuLTRo0ECHDx+WYRgKDAzUtGnTbPJLdlpTp07V9evXs9x/+fJlvfDCC4qLi5OXl1eGnzurV6/Wq6++KsMwVL16dQ0ZMkS1atWSj4+PTp8+rcWLF2vTpk1avHixSpUqlWFUSmEyceJEHT58WD179lS3bt1Uvnx5nT171vy+Xrt2TU8++aQiIiKkW1Pe+vTpo4oVKyoyMlLz5s3Tjh079Pfff2v06NGaN29elj8zr1+/rhdeeEGJiYkaP368mjdvLmdnZ23evFlff/214uPj9eqrr+r777/XCy+8IC8vL7344otq3LixkpKS9Ntvv2nOnDmKiYnRxIkT7+iJeLVq1TIfPrBhwwatWLFCPXr0uKP3btu2bRo1apSSkpLk7Oysxx9/XB07dlSVKlV08+ZNHTt2TH/88Yc2btyY5Tly829XaijUqlUrPfjgg/Lz85OPj4+uXbumiIgILVq0SLt379aWLVs0adIkvf/+++n6yMvP/VSBgYFasmSJJKlChQoaMmSI7r33Xnl5eeny5cvau3ev1q5dm+37s3DhQu3evVstWrTQgAEDVKNGDcXGxmrZsmVatmyZLl++rDfeeEMLFizI0/sOAHliAACKrG3bthl+fn6Gn5+f8b///S/TNomJiUb//v3NdkuXLjX3jR8/3vDz8zPat29vnDp1KtPj9+3bZzRp0sTw8/Mz/vvf/2bYP2TIEPPcbdq0MU6fPp2rev38/IzAwMBM273xxhtmmx9//DHD/v/973/m/swMGDDA8PPzM3r27GlcunQp0zabNm0y/P39DT8/P2PBggUZ9oeHhxuNGzc2z3Pz5k3DMAzjzJkzRrNmzQw/Pz+jc+fORnx8fKbnb9++veHn52e89tprGfZFRESY9S9ZsiTT41O98sorhp+fn3HfffcZ169fz7TNvn37zPN9//332Z4vM++9957h5+dn1KtXz7hw4UKmbZKTk402bdoYfn5+xogRI9Lt+/TTTw0/Pz+jSZMmWR5vGIZx9epVIzk5Oc/1pX2/Uu/zX375xfDz8zMaNWpkxMbGpms/cOBAw8/Pz3j77bcN47b7LrP3+8cffzT3v/rqq0ZKSkqGNv/973/NNh988EGG/dOmTTP3f/311xn2JyQkGCNGjEh3/0dERGRoZ4t7N/UzOWTIkEyPz8k///xjnt/Pz89o1qyZ8corrxgLFiwwDhw4YCQlJeXqPDl9TrNy8+ZN83tYr149Y9OmTen2X7p0ybjvvvsMPz8/Y8KECUZiYmKm50n9nvn7+xvHjh3LUw25lfbefO6554xDhw5l+d+5c+fM45YsWZLuXli4cGGWfaS9tz755JMM+1NSUox//etfZpt58+ZlaPPaa6+Z+1u2bGmcPHkyQ5u0n4NWrVoZnTt3zvQefP/99812+/bty+M79n8mTZqU7vq7detmfPDBB8a6devSvU/ZuXHjhvkzqXHjxsa2bduybHvmzJkMr+Xl366UlBTjxIkT2dbz2WefGX5+fkbdunWN48ePZ9ifl5/769evN9sOGDDAiImJyfW1pe3Hz8/P+M9//pPpz7SJEyda/X0EgNxgoXMAKKbi4+O1Y8cOPfXUU9qzZ490a8HY1EWfIyMjzXWKAgMDVb169UzPU79+fQ0ePFhKMxonK//6179011135aq+8uXLZzl64Y033pCPj48kaf78+bk6X6pdu3aZI16mTZtmnud2Dz74oLp06SJlcV01a9Y069u/f78+++wzpaSk6NVXX9XVq1fl6uqqjz76SB4eHnmqL6/69esnSYqNjdVvv/2WaZvU+l1dXe9oxEHqVLzk5GT98ssvmbbZvn27OQro9ql7Fy9elCTVqFEj2+lkXl5e2S6anRcdO3aUl5eXbty4kW60QEREhDkFL7dT9+bNmydJ8vHxUWBgYKZrXo0dO1a+vr7SrafTJSQkmPsSEhLMEQ1169ZN92CBVK6urnr33XezHdVkq3vXWo0aNdKkSZPMWq9evarly5crMDBQjz/+uJo1a6YRI0Zo4cKFuZqqm1dvvfWW+T189dVX9eCDD6bbP3/+fMXGxqpSpUp6++23s5yaN3bsWFWqVEkpKSnZLopvK8HBwerevXuW/33yySeZHteqVSvzc367hIQEc+22OnXqZLrmkMVi0dtvv21O20u9n7OS1XpQffr0Mdfxu3z5siZOnJjpPTho0CBze9euXdn2lZVXXnkl3ff16NGj+u677/TCCy/owQcfVPv27TVhwgRt3749y3MsW7bM/Jk0btw4tWzZMsu2VapUybaenP7tslgsOa5/9cILL6hs2bIyDMPqUaHffvutJMnDw0OfffZZtmu7ZXdtFSpUyPJn2ogRI8ztO/0+AkBuEEoBQDExffp01a1b1/zv3nvvTbeYcLly5fTFF1+YCzVv2rRJycnJ8vDwyPBL3+1S18E5f/58pgul69Yv3XlZyLlr165ZBjolS5Y0z3XkyJF0a5/kJPXJYTVr1sywuOztUq8rLCxMSUlJGfannZo3a9Ys/etf/zLfzxdeeCFP0+TuVPPmzVWjRg0piwAiISFBK1eulG4trp5VkJGdBg0amIFL6hS926X24e7urk6dOqXbl7ru19GjR7V37948938nSpQoYQYzaQOH1O0aNWpkO90xVVRUlLmuTdeuXbNc6N3FxUW9e/eWbq1ZtW/fPnPfvn37FBMTI0nq1atXlgu5V65cWa1bt86yFlveu9bq16+fVqxYod69e2eYupc6tTcwMFCdO3fWH3/8YbN+Z86cad7nffr00ZNPPpmhTeov/A899FC2C8+7uLiY90BOUzgL0u0hb1phYWG6evWqdOveympaXqlSpcyfmUePHk23rllaFosly5/T7u7uZvBSpkwZtW3bNtN21atXN++J1CmFeeXu7q4ZM2bok08+UbNmzTJ8Zs6cOaOgoCANGzZMI0eO1OXLlzOcI3VKnqenp/r3739HdegO/u3SrYciREVFKTw8XIcPH9bhw4d17Ngxc9Hw29dozIvo6GjzD0ldu3ZVpUqV7vhcjzzySJafEV9fX3N64p1+HwEgN1hTCgCKuWrVqqlLly4aOXJkujUrUh/Jff36ddWvXz/X57t48WKmf1GuUaNGnp6W17Bhw2z3N2rUyPyL/+HDh3O94HnqdR0/fjzHX+xTJSYmKiYmJsOaHrq1iHmPHj104cIFrV69WpJ03333ZToaJr/07dtXH330kbZt26bTp0+nW2h7w4YNunLlipTHBc5v1717d3322Wfau3evTp48mW5UQEJCgtatWyfdCr5uD24effRRzZgxQwkJCRo0aJDatm2rdu3a6b777lOdOnWsftpeVnr27KnFixdrx44dOnv2rKpUqWKu9ZXbEWNp13hK+5TKzDRu3Djdcffee69025ptOQWVAQEB5iLst7P1vWstX19fvffee5o0aZLCwsL0zz//KCwsTDt37tS5c+ekW0+/Gz16tL777jurn3T3+++/66OPPpJuPXAgsydIJicnm7/wL1iwINdr4aSO5stPtz9pM7ey+16nvT/T3n+Zady4sTmy9MiRI5k+ka1s2bLZLoSeOiLn7rvvzvZzW7p0aV27dk3Xrl3LtqbsWCwWdevWTd26ddPly5cVEhKi0NBQhYaG6u+//9aNGzckSX/++aeGDRumBQsWpAtIUxeGb9CggVUjVnP7b5dhGFqxYoUWL16svXv3mvVl5vZF1/Pi4MGD5hpWzZo1u+Pz6FbAnZ0yZcooPj7equ8jAOSEUAoAiolBgwaZ0+wsFotKlCihsmXLZrkw+aVLl+6on6wWJ87ro+Nz+iU67f7UUSi5kdlf1HMjq+vy8fHR+PHjzal8rq6u+uCDD/K8+Lo1evXqpc8++0yJiYlaunSpxowZY+5LnTZWqVIltWnT5o77SA2ldGtUVNo+fv/9d3O0RmajOmrVqqWPP/5YgYGBiomJ0caNG81RDGXLllXbtm01YMAAq3/Bul2zZs1UtWpVnT59WitXrlTz5s118uRJWSyWXIdSae+tnO7JtFMTU4PA28+R00i17KY32vretRVXV1fde++9ZggnSVu3btW7776rI0eOKDk5We+8847WrFlzxwHkkSNHNH78eKWkpKhq1ar6/PPPMx3hERMTc0cjw7ILEApamTJlstx3p/dWVj8zcwpvUqfX5rZdZk+juxM+Pj7q1KmTOQrz2rVrWrBggT799FPdvHlTR44c0Zw5c/T888+bx6QGP3f6hNZUufm36+bNmxozZkyuRwVac7+lDbSsvTZ7fx8BIDOEUgBQTJQrV05+fn65bp+cnCzdCg3mzp2b6+OqVauW6et5DWnya/RM6nX5+/vrww8/zPVxWU2RSEpKSreuVWJionbs2JHl+5Afypcvr4ceekjr1q3T0qVL9cILL8hisSgqKkpbtmyRbo0asiYoq169uu69917t3r07QyiVOqXP29s7yyk9Xbp00QMPPKDVq1frzz//1K5du3T58mVFR0drxYoVWrFihXr16qWpU6fabF0pi8Wi7t276+uvv9by5cvNqaVNmzbNco20nM5ni5rulK3v3fx0//33a9asWerevbuuXLmiEydO6MCBA3kadZnq8uXLGj16tK5duyZPT0999dVXWQYwqe+Rbk0xHDZsWK76sPUTCm0pt5+H/PqZWRiVLFlSI0aMkKenp9566y1J0po1a9KFUraSm5+bX331lRlItWjRQoMHD1aDBg1Uvnx5ubu7m9/DJ554gvWZAOA2hFIAgEylTuG4du2aatWqZdeRP8rFdJq0I7myG0lwu9Trio+Pz1NIl5UvvvjCXCepVKlSiouL05QpU9S8efM7Cj7uVL9+/bRu3TpFRkZqx44datmypZYvX27+kp663pE1unfvrt27d+vEiRMKDQ1VQECA4uLizOlmjzzySLa/3Ht5eWnAgAEaMGCAJOnYsWMKDg7WDz/8oPPnz2vp0qWqV6+ehg8fbnWtqXr27Kmvv/5aR48eNddF6dmzZ66PT3tv5XRPpt2fdgpU2pEWly5dynbKTHZ92PrezW8VK1ZUu3btzHW8Tp48medQKjExUS+++KIiIyNlsVj00UcfZTudLe33yzAMh3ifrJH2evNyb+XlZ2Zh1rt3b02ePFlJSUk6depUun1ly5bVuXPn8rTm4J0wDMNcbL5Zs2aaM2dOlkFiXkb1ZqVs2bLmdn5fGwDYAwudAwAylfrLY0JCgrmWjT3l1GdoaKi5XadOnVyfN/W6IiIirP4f+t27d+ubb76RJLVp00Zz5syRq6urrl27pldffTXdqI28uJMRD23btjUX0U1dCDp16l6zZs3MxdCt0bVrV/NJZqmjo9auXaubN29KOSzInJlatWrpmWee0cKFC80FdVOf+GgrNWvWNNeCunnzptzc3PTII4/k+vi091ZOi7Sn3Z/2uLTBSNr7NjPZ3fe2vHftJe26RXdyX7/zzjvauXOndOsJah07dsy2vZubm/nepz6hryhLe5/9888/2bbN6v50ZG5ubmZYe/v9lfp5CQsLy9cprFeuXDE/j4888kiWgdS1a9d0/PjxLM+T289HvXr1zLaMugJQFBBKAQAy1b59e/N/fOfMmWP3/tesWZPluhvx8fFmeFG7du1MF+zNSurT8gzDyNO0xNtdu3ZNr7zyipKTk+Xt7a2pU6eqYcOGevHFF6VbvxDPmDHjjs6ddlHdhISEXB3j5ORkjoZau3at/vjjD504cUKycoHztHx8fMynw/3yyy9KSUkxw6mqVavqvvvuu6PzVqlSxQzNrFkAOCs9e/aUm5ub3Nzc9PDDD+dpfbNKlSqpVq1a0q3ALKsFf5OTk7V06VLp1iiUBg0amPsaNmxojkxZvny5uUjx7aKiovTnn39mWYut7l1rZVV/ZtKGbHkdOTh79mwtWrRIuhV4Pvvss7k6LvV9Cg8P1+bNm/PUp6Np2LCheT8vW7Ysy7V/4uLi7vhnpr3l5f46e/asOWr29inTqffB9evXc73g/Z1I+8eH7MKvRYsWZbveWW5/7nt7e5vrt/3666+Kioq6g6oBoPAglAIAZMrX19ccUfLLL7/o+++/z7Z9RESEGVDYwoULF7J8UtW0adPMX0QGDRqUp/O2adPGHDkzc+ZM84l5WTl06JD5iPm0pkyZYk4Hmzx5srluz9NPP60WLVpIt6b23ckoM29vb3Ma3O1TUrLTp08fWSwWXb9+XW+88YZ0a+2VvIwMyknqaKgLFy5o5cqV2r59uyTpsccey/Iv/evXrzcXQs/M2bNnFR4eLmWzJpk1nnjiCfOpXf/973/v6HjdWttoypQpmbaZPn26jh49Kt2aSpl2EW43NzczMDxw4IC+++67DMcnJSXpP//5jxITE7Osw1b3rrXGjBmjefPmKT4+Ptt2QUFB2rp1qyTprrvuytPUvT/++EMffPCBdOuph++++26ujx02bJg58m7ChAnpnlCXmd9//918Yt/t9detW1d169bV559/nuv+7cnNzU19+/aVbj3l8csvv8zQxjAMTZ482Qx8U+/nwuro0aMaMWKEOUIuKzdv3lRgYKAZYt0+iq5Hjx7mz+VPP/1UO3bsyPJcqU+LvBM+Pj5mMLhq1apMA6W9e/eaD4rISl5+7o8aNUq6FYK99NJLio2NzbKtNdcGAPbAmlIAgCy9/fbbCgsLU0REhKZNm6bg4GA9/vjjqlOnjtzc3HTlyhUdPHhQmzdv1rZt2/Twww/rscces0nfDRs21Pz58xUZGamBAweqSpUqOnv2rObPn2+OJqlfv74GDhyY53N//PHH6tevn65cuaJx48ZpxYoV6tatm2rUqCEnJyddunRJBw4c0MaNG7Vnzx6NGDHC/Ku7JK1bt86cIte7d2917tzZ3Ofk5KT3339fPXr0UGxsrP79739r2bJlcnd3z3V9Li4uCggIUEhIiJYsWaL69eurXr165tS5MmXKZPrY9mrVqumBBx7Qli1bzOkkXbt2NX9Bt4WOHTvK09NT8fHxmjJlijlKILupe3PmzNG///1vtWvXTq1atVKtWrXk5eWlmJgYhYWF6ccffzRHxeU1ZLSHgQMHauXKldq9e7eCgoJ05swZDR48WNWqVdOFCxe0ZMkS/fbbb5Kku+++O9PFll944QX9+uuvOnfunD766CMdPHhQjz/+uMqVK6cTJ07o+++/V2hoqBo2bJhtkGntvWsLZ8+e1aRJk/TRRx+pQ4cOatasmWrWrKkyZcro5s2bCg8P15o1a7Rp0ybp1rSkCRMm5Hp6UkxMjMaPH6/k5GR5enrq5Zdf1smTJ7M9pmbNmuYv9OXLl9f777+vF198URcuXFCfPn3Uq1cvPfjgg6pcubKSkpJ07tw57d27V2vXrlVERIS+/vpr+fv72+Ddsb8XXnhB69atU0REhD7//HMdPnxYvXv3VoUKFRQZGakff/zRDGTuvfdec023wsowDG3ZskVbtmzRPffco44dO6pRo0aqXLmyPDw8FB0drb1792rhwoWKjIyUboWeI0eOTHeeEiVK6IMPPtDIkSN1/fp1PfXUU+rRo4c6deqkypUrKyEhQeHh4dq0aZM2bNhwx9PUnZyc1L17d82bN0+HDh3SoEGD9NRTT+mee+5RXFycNm3apJ9++kmenp6qWLGiOYL1dnn5ud+hQwf17dtXixcv1u7du9WtWzcNGTJETZs2ValSpRQdHa2wsDCtXr1a/v7+Wf6BBwAKA0IpAECWvL29NX/+fL388svatWuXdu7cme1fr0uWLGmzvseNG6fvv/9emzdvznQKjq+vr77++mvzf9jz4u6779bPP/+sF198UYcPH9bGjRu1cePGLNunva7z58/rP//5j3RrOlLqdlp33XWX3nzzTb3yyis6fvy4pk2bprfffjtPNT777LMaPXq0rly5on/961/p9o0ZM0Zjx47N9Li+ffuaT9yTDafupfL09FTHjh21cuVKc/STv79/jmvUXL9+XWvWrNGaNWsy3e/k5KSxY8eaj3wvTJydnfX111/rueeeU0hIiLZt26Zt27ZlaFerVi19++23mX4OvLy89N133+mpp57ShQsXtGrVqgwjC3v37q3mzZtrwoQJWdZizb1rK5UrV9a+ffsUHx+f6XWk5eXlpf/85z/pgtucxMbGmiM/4uPjNWLEiByPCQ4OTjfKrnPnzvryyy81YcIEXblyRT///LN+/vnnTI91cnKSh4dHhtfTTh/OLAQuLEqVKqXZs2dr1KhRCg8P19q1a7V27doM7Zo2baqvvvrK7g+tyCsPDw+VKVNGMTExOnnypGbNmpVt+4YNG+rTTz+Vl5dXhn2tWrXS119/rX/961+KiYlRUFCQ+QcFWxo3bpxCQkJ04MABhYWFZfiZ7e3trc8//1z/+9//sgyllMef+5MmTZK7u7vmzZun8+fPZzkK1FHDVgDFB6EUACBbFSpU0Lx58/T7779r1apV2rNnjy5evKikpCR5eXnpnnvu0b333qsOHTqoefPmNuvX1dVVM2bM0IIFC7R8+XKFh4crMTFR1atXV7du3fTUU0/lafTR7WrWrKlly5bp119/1W+//abQ0FBdvnzZXCOqZs2auu+++/Twww+b6wMZhmH+kuvs7KwPP/wwy1/6e/TooU2bNmnVqlWaP3++2rdvr3bt2uW6voceekizZ8/W3LlzFRoaqujo6GyndqXq1KmTOZLJ19dXTZs2zcO7kjvdu3fXypUr032dnY8//li///67tm/frmPHjunixYuKjo6Wm5ubqlatqmbNmmngwIGF+pcnb29vzZs3TytWrNCqVat04MABxcTEqGTJkvLz89MjjzySYdre7erUqaNVq1bp22+/1fr163XmzBnz+P79++uxxx7L1S/Md3Lv2tKXX36p8PBw/fnnnwoJCdHRo0d17tw5xcfHq0SJEvL29ladOnXUunVrde/eXT4+PjavITc6dOig4OBgLVy4UJs2bdLRo0cVExMjZ2dnlS9fXnXq1FGrVq3UpUsXValSJcPxe/bskW6NUOnVq1cBXEHuVatWTcuXL9eiRYu0Zs0aHT58WNeuXVOZMmVUr149de/eXd27d89yEe7CpHr16tqyZYt27dqlbdu2ae/evTpx4oQuX76spKQkeXp6qnLlymrQoIE6d+6shx56KNvratu2rdavX6/58+dr48aNOn78uGJjY+Xh4aF77rlHzZo1s3qEr5eXl+bPn6/vv/9ev/76q06ePClnZ2dVqVJF7dq10/Dhw80HUWQnLz/3nZ2dFRgYqN69e2vBggXavn27zp8/r8TERHl7e6tu3bpq27atevToYdW1AUB+sxh5WU0QAIB8tH37dg0bNkySNHfuXLVs2bKgS3I4J06cUJcuXSRJ//73v821RwDkTYcOHXT69GmNHTtWY8aMKehyAAAokgr/n0sAAECuLVmyRLq1PknPnj0LuhzAIZ0+fVqnT5+Wl5eXhg8fXtDlAABQZBFKAQBQRFy9elULFy6Ubi1IXqFChYIuCXBIqWvnDR06NNO1igAAgG2wphQAAA7s0qVLiouL0/nz5/X555/rypUrslgsevbZZwu6NMBh9ezZk5GGAADYAaEUAAAO7MMPP9TSpUvTvTZ48OB8WeAaAAAAsCVCKQAAigBXV1fdfffd6t+/v4YMGVLQ5QAAAAA54ul7AAAAAAAAsDtGSjmAlJQUJSUlycnJSRaLpaDLAQAAAAAAyJJhGEpJSZGLi4ucnLJ+xh6hlANISkpSaGhoQZcBAAAAAACQawEBAXJzc8tyP6GUA0hNFQMCAuTs7FzQ5QAAAAAAAGQpOTlZoaGh2Y6SEqGUY0idsufs7EwoBQAAAAAAHEJOSxBlH1kBAAAAAAAA+YBQCgAAAAAAAHbn0NP3Ll26pL1792rv3r0KDQ1VaGiorly5Iknq1auXpk2bluM5rl+/rs2bN2vLli0KCwvTqVOnFB8fr1KlSqlGjRpq06aNBg4cqAoVKuSqpuvXr+vHH3/UmjVrFBERoYSEBFWuXFkPPfSQhg4dqqpVq1p93QAAAAAAAI7OoUOpBx54wKrjDx48qEGDBik+Pj7DvitXrmjPnj3as2ePZs+ercmTJ6tbt27Znu/kyZN65plndOLEiXSvHz9+XMePH9eiRYv00UcfqX379lbVDQAAAAAA4OgcOpRK66677pKvr6/+/PPPXB8TFxdnBlJNmzZV+/bt1bBhQ3l7e+vy5cv67bfftGjRIsXFxenf//63SpYsqXbt2mV5rrSBVP/+/dWtWze5u7tr+/bt+uabbxQXF6dx48Zp/vz5qlevno2uHAAAAAAAwPE4dCj1wgsvKCAgQAEBASpfvrwiIyPVsWPHXB/v5OSkrl27asyYMapdu3aG/W3atNGDDz6oMWPGKDk5WVOmTNGDDz6Y6erxM2fONAOpV155RU8//bS5795771WLFi00dOhQXb9+XVOnTtUPP/xwx9cNAAAAAADg6Bx6ofMXX3xR7du3V/ny5e/o+KZNm+rTTz/NNJBK1alTJ3Xu3FmSdOrUKe3fvz9Dm8TERDNkqlWrlkaMGJFpX3369JEk7dixQ3v37r2jmgEAAAAAAIoChw6l7KVly5bm9qlTpzLs3759u2JjYyVJPXv2lJNT5m9r7969ze3169fnS60AAAAAAACOgFAqFxISEsxtZ2fnDPv//vtvc7tFixZZnqdhw4by8PCQJIWEhNi8TgAAAAAAAEdBKJULO3bsMLdr1aqVYf+xY8fMbV9f3yzP4+LiorvvvjvDMQAAAAAAAMWNQy90bg8HDx7Upk2bJEl+fn6ZhlLnzp2TJHl6eqp06dLZnq9KlSo6dOiQLl++rISEBLm5ueVT5QAAAACA4swwDCUmJiolJaWgS4EDcXZ2louLS6YPebM1QqlsJCQkaOLEiUpOTpYkjRs3LtN2165dk26FUjlJnb6XelxeQqnUOgAAAAAAyMr169cVExOjuLg4fo/EHSlRooTKlCkjb2/vOwqncnvfEUplY9KkSQoLC5Mk9erVSx06dMi03c2bNyVJrq6uOZ4zbQiVelxuhYaG5qk9AAAAAKD4cXJykru7u7y8vOTu7p7lw7iAzCQlJenatWs6c+aMTp48ma99EUpl4ZtvvtGiRYskSQEBAXrzzTezbFuiRAlJUmJiYo7nTbtoeupxuRUQEJDpQusAAAAAAFy/fl0RERHy8vLSXXfdZZfpVyiaKlSooCtXrujs2bOqUqVKjksV3S45OTlXA2sIpTLx888/67///a90a+HyGTNmZDs1r2TJkpKk+Pj4HM99/fr1DMfllrOzM6EUAAAAACBTsbGxcnV1VdWqVQmkYLWyZcvq6tWriouLU9myZfOlD8bw3WbVqlV65513JElVq1bV999/Lx8fn2yPqVy5snQrlLp69Wq2bc+ePStJ8vHxYZFzAAAAAIBNGIah2NhYlS5dmkAKNlOqVCnFx8fn22L5hFJpBAcH67XXXlNKSooqVKig2bNnm4FTdtI+kS88PDzLdklJSYqIiMhwDAAAAAAA1khMTFRycnKeZ+QA2XF3d1dKSoqSkpLy5fyEUrds3bpVL7/8spKSkuTt7a3vv/9ed999d66Ove+++8ztHTt2ZNkuLCzMnOLXtGlTG1QNAAAAAIDMkSwsag5bSr2fGCmVj0JCQvT8888rISFBXl5emjlzpurUqZPr41u0aCEvLy9J0rJly2QYRqbtgoKCzO1OnTrZoHIAAAAAAP4/pu7BlvL7fir2odSBAwf07LPPKj4+Xp6envrmm2/UsGHDPJ3Dzc1NQ4cOlSQdO3ZMM2fOzNBm9+7dWrJkiXQrxGrUqJGNrgAAAAAAAMDxOPTT93bt2qVTp06ZX0dHR5vbJ0+eTDcySZJ69+6d7utTp05p5MiR5uLkL730kry8vHT48OEs+yxXrpzKlSuX4fWRI0dq9erVOnHihD788EOdOnVK3bp1k7u7u7Zv366vv/5aSUlJcnd31xtvvGHVdQMAAAAAADg6hw6lFi9erKVLl2a6LyQkRCEhIeleuz2U2rVrly5dumR+/d577+XY55gxYzR27NgMr5cqVUozZszQM888oxMnTmjBggVasGBBhjYfffSR6tWrl2M/AAAAAAAARZlDh1KFzT333KOlS5dq3rx5WrNmjU6dOqXExERVrlxZ7dq107Bhw1S1atWCLhMAAAAAAKDAWYysVuVGoZGcnKw9e/aoSZMmcnZ2LuhyAAAAAACFzI0bN3T8+HHVrFlT7u7uBV1OsbB9+3YNGzZMymJW1eeff67p06db1UevXr00bdo0SdLQoUO1Y8eODG2cnJzk5eWlatWqqWnTphowYECeHt6WnTu9r3KbYxT7hc4BAAAAAAAcVUpKimJiYrRv3z798MMPevzxxzVjxoyCLitXmL4HAAAAAEAxlGKkyMlSNMeqFIZrGzx4sLp06ZLpvuDgYH366aeSpJdfflkdO3bMtF2ZMmUyfX3lypXmdmJioiIiIrR+/XqtXLlSycnJ+vjjj1W9enV17drVJteSXwilAAAAAAAohpwsTpr3z2ZFxcUUdCk2ValUGT3RuG1Bl6Fy5cqpXLlyme4LCwsztytVqiQ/P788nfv29g0aNNAjjzyixo0ba8qUKZKkL774glAKAAAAAAAUTlFxMTp99XJBlwEbeeKJJzRr1iydOXNGR44c0YULF1ShQoWCLitLRXOcHgAAAAAAQDHj5OSk2rVrm1+fPXu2QOvJCaFUMZdipBR0CUUS7ysAAAAAoCC4urpmul0YMX2vmCuqc4gLUmGZvwwAAAAAKH6OHTtmbt91110FWktOCKXAHGIAAAAAAIqA3377TSdOnJAk3X///Vk+va+wIJQCAAAAAABwUAkJCYqIiND69ev11VdfSZI8PDw0bty4gi4tR4RSAAAAAAAADqRu3bpZ7mvQoIEmTpyoxo0b27WmO8FC5wAAAAAAAEWAq6ur+vTpo/vuu6+gS8kVRkoBAAAAAAA4kJUrV5rbV69e1aFDhzR79mydOnVKkyZN0vXr1/X0008XaI25QSgFAAAAAADgQPz8/NJ93axZMz3++OMaPHiwDh06pE8++UQtWrRQo0aNCqzG3GD6HgAAAAAAgIMrVaqUPvjgAzk5OSkpKUnvv/9+QZeUI0IpAAAAAACAIsDf31+PPfaYJGnXrl36448/CrqkbBFKAQAAAAAAFBGjR4+Wk9P/xT1fffVVQZeTLUIpwMa83NxlpKQUdBlFEu8rAAAAAGSvVq1aevjhhyVJISEh2rZtW0GXlCUWOgdszMPVTRYnJ8Wsn6/k6PMFXU6R4Vy2osp0GlTQZQAAAABABgcOHFBQUFCO7Vq1aqW77ror3+sZPXq01q5dK90aLdWqVat87/NOEEoB+SQ5+rySLp4u6DIAAAAAAPksODhYwcHBObb74osv7BJK1a9fX+3atdOmTZu0bds27dmzR02aNMn3fvOKUAoAAAAAgGKqUqkyBV2CzRXFa7oTo0eP1qZNmyRJX375pWbMmFHQJWVAKAUAAAAAQDGUYqToicZtC7qMfJFipMjJkr/LaLds2VKHDh26o2N79+6t3r175+mYH374IU/tmzZtesf12QsLnQMAAAAAUAzld2hTkIrytRUlfJcAAAAAAABgd4RSAAAAAAAAsDtCKQAAAAAAANgdoRQAAAAAAADsjlAKAAAAAAAAdkcoBQAAAAAAALsjlAIAAAAAAIDdEUoBAAAAAADA7gilAAAAAAAAYHeEUgAAAAAAALA7QikAAAAAAADYHaEUAAAAAAAA7I5QCgAAAAAAAHZHKAUAAAAAAAC7I5QCAAAAAACA3RFKAQAAAAAAwO4IpQAAAAAAAGB3hFIAAAAAAACwO0IpAAAAAAAA2J1LQRcAAAAAAADgaLZv365hw4Zlus/d3V3e3t7y9/fXww8/rB49esjNzU2S1KFDB50+fdqqvufOnauWLVsqMjJSHTt2lCT16tVL06ZNs+q89sZIKQAAAAAAABu6ceOGzp07p99//10TJ05U7969FRkZWdBlFTqMlAIAAAAAoBgyUlJkcSqaY1XsfW2DBg3S4MGDza8vXbqkI0eOaObMmTp37pyOHDmi5557TsuWLdPMmTOVmJiY6XkmTJigsLAwSdLKlSuz7K9atWr5cBX2RygFAAAAAEAxZHFyUsz6+UqOPl/QpdiUc9mKKtNpkF37LFeunPz8/NK9dv/996t3797q0aOHTp8+rcOHD2vdunV65JFHsjyPp6enuX37+YoiQikAAAAAAIqp5OjzSrpo3fpGyFqpUqX03HPP6T//+Y8k6a+//so2lCpuiuY4PQAAAAAAgEKgbt265va5c+cKtJbChlAKAAAAAAAgn7i6uprbLi5MWEuLUAoAAAAAACCfHDt2zNyuWrVqgdZS2BBKAQAAAAAA5IPk5GTNnDnT/LpLly4FWk9hQygFAAAAAABgQ5cvX9bWrVs1ZMgQ7d+/X7oVSDVr1qygSytUmMwIAAAAAABghenTp2v69OmZ7vPw8NDAgQP1r3/9y+51FXaMlAIAAAAAAMgn/v7+Gjp0aLoFz/F/GCkFAAAAAABghUGDBmnw4MHSrXWkzp07p7Vr12r58uXavXu3hg4dqsWLF8vHx6egSy1UGCkFAAAAAABghXLlysnPz09+fn6qV6+e2rdvr2nTpmnq1KmSpNOnT2vixIkFXWahQygFAAAAAACQD3r16mU+cW/Dhg3aunVrQZdUqBBKAQAAAAAA5JNx48bJ2dlZkvTJJ58UdDmFCqEUAAAAAABAPqlZs6a6du0qSfrnn3+0ZcuWgi6p0CCUAgAAAAAAyEfPPvusLBaLJOmrr74q6HIKDZ6+BwAAAAAAkI/8/PzUoUMHBQcHa+fOndq1a5eaNWtm0z5OnjypoKCgHNs1atRItWvXtmnfd4pQCgAAAAAAIJ+NHj1awcHB0q3RUjNnzrTp+UNCQhQSEpJjuwkTJhBKAQAAAACAguVctmJBl2BzhfWaGjVqpNatW2vLli36888/tXfvXjVq1KigyypQhFIAAAAAABRDRkqKynQaVNBl5AsjJUUWp/xdRrtly5Y6dOhQno6ZNWtWtvt/+OGHPJ2vWrVqea6hMGGhcwAAAAAAiqH8Dm0KUlG+tqKE7xIAAAAAAADsjlAKAAAAAAAAdkcoBQAAAAAAALsjlAIAAAAAAIDdEUoBAAAAAADA7gilAAAAAAAAYHeEUgAAAAAAALA7QikAAAAAAADYHaEUAAAAAAAA7I5QCgAAAAAAAHZHKAUAAAAAQBFhGEZBl4AiJL/vJ0IpAAAAAAAcnLOzsyQpKSmpoEtBEZKcnCxJcnLKn/iIUAoAAAAAAAfn4uKiEiVKKCYmpqBLQRESGxsrV1dXubq65sv5CaUAAAAAAHBwFotF3t7eio2NVXR0dEGXgyLg+vXrunr1qry8vGSxWPKlD5d8OSsAAAAAALCrsmXLKiEhQefOndPVq1dVqlQpubu7y8nJKd9CBRQthmEoOTlZsbGxunr1qkqUKKHy5cvnW3+EUgAAAAAAFAEWi0WVK1eWh4eHrl69qosXLyolJaWgy4IDcnV1lbe3t8qXL2+uV5YfCKUAAAAAAChCypQpozJlyiglJUVJSUkEU8gTJycnubq62mV0HaEUAAAAAABFkJOTk9zc3Aq6DCBLLHQOAAAAAAAAuyOUAgAAAAAAgN0RSgEAAAAAAMDuCKUAAAAAAABgd4RSAAAAAAAAsDuHfvrepUuXtHfvXu3du1ehoaEKDQ3VlStXJEm9evXStGnT8nS+TZs2aeHChQoNDdXly5fl4+OjgIAA9e/fX+3atcvVOZKSkrRo0SKtXLlS4eHhio+PV8WKFfXAAw9o6NChqlOnzh1dKwAAAAAAQFHi0KHUAw88YJPzpKSkKDAwUIsXL073elRUlKKiorR+/Xr169dPkyZNkpNT1oPLLl++rGeeeUahoaHpXo+IiNCCBQu0dOlSvfnmm+rXr59N6gYAAAAAAHBUDh1KpXXXXXfJ19dXf/75Z56P/eSTT8xAqn79+nr66adVvXp1RURE6LvvvtP+/fu1aNEi+fj4aPz48ZmeIzk5WWPGjDEDqc6dO6tfv37y9vbWP//8o6+++kqXLl3Sm2++qYoVK+Z65BUAAAAAAEBR5NCh1AsvvKCAgAAFBASofPnyioyMVMeOHfN0juPHj2vWrFmSpIYNG2revHlyd3eXJDVq1EgdOnTQkCFDFBYWppkzZ6pPnz665557Mpxn6dKl+vvvvyVJgwcP1ltvvWXua9SokR588EH17t1bcXFxevfdd9W6dWu5uDj02w8AAAAAAHDHHHqh8xdffFHt27dX+fLl7/gcc+bMUVJSkiQpMDDQDKRSeXh4KDAwULq1XtTs2bMzPU9qsOXt7a1XX301w/577rlHzz77rCTp5MmTWrdu3R3XDAAAAAAA4OgcOpSylmEYCg4OliT5+vqqSZMmmbZr0qSJatasKUkKDg6WYRjp9h8/flzHjh2TJD3yyCPy8PDI9Dy9evUyt9evX2+z6wAAAAAAAHA0xTqUioyM1Pnz5yVJzZs3z7ZtixYtpFuLn0dGRqbblzptL227zFSoUEE1atSQJIWEhFhVOwAAAAAAgCMr1qHU0aNHzW1fX99s26bdHx4enm5f6iipvJzn7Nmzio+Pz3PNAAAAAAAARUGxDqXOnTtnbleuXDnbtmn3nz17NsvzVKpUKdvzVKlSRbo1dTDtcQAAAAAAAMVJsX7827Vr18xtT0/PbNumXSfq9hFOac9TsmTJOz5PTpKTk/PUPjecnZ1tfk4gP+XH5wAAAAAAYDu5/b2tWIdSN2/eNLddXV2zbevm5mZu37hxI1/Ok5PQ0NA8tc+Jh4eH6tevb9NzAvnt0KFDun79ekGXAQAAAACwUrEOpUqUKGFuJyYmZts2ISHB3HZ3d8/2PGm/zst5chIQEMDIJhR7devWLegSAAAAAADZSE5OztXAmmIdSqWdapfTVLq0IzNun+qX9jzXrl3LNpTK7jw5cXZ2JpRCscdnAAAAAACKhmK90HnaxctzWnQ87f7UxcozO09UVFS250ldJN1iseS4uDoAAAAAAEBRVaxDqdq1a5vb4eHh2bZNu9/X1zfdvlq1auX5PFWqVMnzSCkAAAAAAICioliHUtWqVVPFihUlSTt37sy2ber+SpUqqVq1aun23Xfffeb2jh07sjzHhQsXdOLECUlS06ZNraodAAAAAADAkRXrUMpisahjx47SrRFMe/bsybTdnj17zBFOHTt2lMViSbe/Zs2a5mipNWvWZPlksKVLl5rbnTp1stl1AAAAAAAAOJpiHUpJ0vDhw82FkydPnqwbN26k23/jxg1NnjxZkuTi4qLhw4dnep4RI0ZIkq5cuaIPP/www/5Tp07pm2++kSTdc889evjhh21+LQAAAAAAAI7CoZ++t2vXLp06dcr8Ojo62tw+efKkgoKC0rXv3bt3hnPUrFlTI0eO1IwZMxQWFqZBgwZp1KhRql69uiIiIvTtt99q//79kqSRI0eqRo0amdbSq1cvLVmyRCEhIZo3b54uXryofv36qUyZMtq7d6++/PJLxcXFycnJSRMnTpSLi0O/9QAAAAAAAFZx6GRk8eLF6abEpRUSEqKQkJB0r2UWSknSuHHjdOnSJS1ZskT79+/XuHHjMrTp27evXn755SxrcXZ21hdffKFnnnlGoaGhWrt2rdauXZuujZubm9588021a9cul1cIAAAAAABQNDl0KGUrTk5Omjp1qrp06aIFCxYoNDRU0dHRKlu2rAICAjRgwIBcBUk+Pj76+eeftXDhQq1atUrHjh3T9evXVbFiRd1///0aNmyY6tSpY5drAgAAAAAAKMwcOpSaNm2apk2bZrPztWvXzupRTC4uLho8eLAGDx5ss7oAAAAAAACKmmK/0DkAAAAAAADsj1AKAAAAAAAAdkcoBQAAAAAAALsjlAIAAAAAAIDdEUoBAAAAAADA7gilAAAAAAAAYHeEUgAAAAAAALA7QikAAAAAAADYHaEUAAAAAAAA7I5QCgAAAAAAAHZHKAUAAAAAAAC7I5QCAAAAAACA3RFKAQAAAAAAwO4IpQAAAAAAAGB3hFIAAAAAAACwO0IpAAAAAAAA2B2hFAAAAAAAAOyOUAoAAAAAAAB2RygFAAAAAAAAuyOUAgAAAAAAgN0RSgEAAAAAAMDuCKUAAAAAAABgd4RSAAAAAAAAsDtCKQAAAAAAANgdoRQAAAAAAADsjlAKAAAAAAAAdkcoBQAAAAAAALsjlAIAAAAAAIDdEUoBAAAAAADA7gilAAAAAAAAYHeEUgAAAAAAALA7QikAAAAAAADYHaEUAAAAAAAA7I5QCgAAAAAAAHZHKAUAAAAAAAC7I5QCAAAAAACA3RFKAQAAAAAAwO4IpQAAAAAAAGB3hFIAAAAAAACwO0IpAAAAAAAA2B2hFAAAAAAAAOyOUAoAAAAAAAB2RygFAAAAAAAAuyOUAgAAAAAAgN0RSgEAAAAAAMDuCKUAAAAAAABgd4RSAAAAAAAAsDtCKQAAAAAAANgdoRQAAAAAAADsjlAKAAAAAAAAdkcoBQAAAAAAALsjlAIAAAAAAIDdEUoBAAAAAADA7gilAAAAAAAAYHeEUgAAAAAAALA7QikAAAAAAADYHaEUAAAAAAAA7I5QCgAAAAAAAHZHKAUAAAAAAAC7I5QCAAAAAACA3RFKAQAAAAAAwO4IpQAgH6QYKQVdQpHDewoAAAAULS4FXQAAFEVOFifN+2ezouJiCrqUIqFSqTJ6onHbgi4DAAAAgA0RSgFAPomKi9Hpq5cLugwAAAAAKJSYvgcAAAAAAAC7I5QCAAAAAACA3RFKAQAAAAAAwO4IpQAAAAAAAGB3hFIAgELPy81dRkpKQZdRJPG+AgAAoKDw9D0AQKHn4eomi5OTYtbPV3L0+YIup8hwLltRZToNKugyAAAAUEwRSgEAHEZy9HklXTxd0GUAAAAAsAGm7wEAAAAAAMDuCKUAAAAAAABgd4RSAAAAAAAAsDtCKQAAAAAAANgdoRQAAAAAAADsjlAKAAAAAAAAdkcoBQAAAAAAALsjlAIAAAAAAIDdEUoBAAAAAADA7gilAAAAAAAAYHeEUgAAAAAAALA7QikAAAAAAADYHaEUAAAAAAAA7I5QCgAAAAAAAHZHKAUAAAAAAAC7I5QCAAAAAACA3bkUdAGFRUJCgpYvX641a9bo0KFDunLlilxdXVWxYkU1bdpU/fr1U9OmTXM8z6ZNm7Rw4UKFhobq8uXL8vHxUUBAgPr376927drZ5VoAAAAAAAAKO0IpSadPn9azzz6rI0eOpHs9MTFRJ06c0IkTJxQUFKShQ4dq4sSJslgsGc6RkpKiwMBALV68ON3rUVFRioqK0vr169WvXz9NmjRJTk4MUAMAAAAAAMVbsQ+lEhMT0wVSdevW1VNPPaWaNWvq2rVr+vvvv/X9998rPj5eP/zwgypWrKhnnnkmw3k++eQTM5CqX7++nn76aVWvXl0RERH67rvvtH//fi1atEg+Pj4aP3683a8TAAAAAACgMCn2oVRwcLAZSN17772aN2+enJ2dzf2tW7dWhw4dNHDgQCUmJurbb7/ViBEj5OLy/9+648ePa9asWZKkhg0bat68eXJ3d5ckNWrUSB06dNCQIUMUFhammTNnqk+fPrrnnnvsfq0AAAAAAACFRbGfR7Z7925z+5lnnkkXSKVq2LChHnroIUnS1atXdezYsXT758yZo6SkJElSYGCgGUil8vDwUGBgoCQpKSlJs2fPzpdrAQAAAAAAcBTFPpRKTEw0t6tXr55lu7T70h5jGIaCg4MlSb6+vmrSpEmmxzdp0kQ1a9aUbo3OMgzDJvUDAAAAAAA4omIfSqUGRZIUERGRZbvUfRaLRTVq1DBfj4yM1Pnz5yVJzZs3z7avFi1aSLcWP4+MjLS6dgAAAAAAAEdV7EOpRx99VKVKlZIkffvtt0pOTs7QZv/+/fr9998lSY899pjZXpKOHj1qbvv6+mbbV9r94eHhNqkfAAAAAADAERX7UMrHx0cffPCBPDw8FBISor59+2rZsmXas2eP/vrrL02fPl1DhgxRYmKiGjRooNdffz3d8efOnTO3K1eunG1fafefPXs2H64GAAAAAADAMRT7p+9JUseOHbVkyRJ9//33Wrx4sV577bV0+8uXL6+XXnpJ/fv3l4eHR7p9165dM7c9PT2z7SftsfHx8XmuM7NRXNbKbGF3oDDLj89BfuCzBUfiKJ8rAAAAOIbc/v8loZSkhIQELV++PMsFyC9evKgVK1aoWrVq6tixY7p9N2/eNLddXV2z7cfNzc3cvnHjRp7rDA0NzfMx2fHw8FD9+vVtek4gvx06dEjXr18v6DKyxWcLjsYRPlcAAAAoeop9KBUfH69Ro0Zp165dcnZ21tNPP63evXurevXqSkhI0D///KMvvvhCf//9t1544QW99tpreuqpp8zjS5QoYW6nfSpfZhISEsxtd3f3PNcaEBDA6AsUe3Xr1i3oEoAih88VAAAAbCk5OTlXA2usCqUSEhLSjf5xRJ9//rl27dolSXr33XfVq1cvc5+bm5tat26tli1basSIEdq+fbs++OAD3X///fL395cklSxZ0myf05S8tH+FzmmqX2acnZ0JpVDs8RkAbI/PFQAAAAqCVQudt23bVlOmTNGBAwdsV5EdGYahoKAgSVKNGjXSBVJpubi46KWXXpIkpaSkmMfotsXL0y56npm0+6tUqWJ1/QAAAAAAAI7KqlAqJiZG8+bNU+/evdWrVy/NmzdPMTExtqsun128eFFXrlyRpBzXf2nYsKG5HR4ebm7Xrl0709czk3a/r6/vHdUMAAAAAABQFFgVSj388MNydnaWYRg6cOCApkyZorZt22r8+PH6888/bVdlPkk7XSGnleHTrhfl4vL/Zz1Wq1ZNFStWlCTt3Lkz23Ok7q9UqZKqVat2x3UDAAAAAAA4OqtCqc8//1ybN2/WhAkT5O/vL8MwlJCQoF9//VWjRo1S+/bt9b///U8RERG2q9iGvL29VapUKUnS7t27lZSUlGXbtIFT2kDJYrGYT+QLDw/Xnj17Mj1+z5495kipjh07ymKx2Ow6AAAAAAAAHI1VoZQklS1bVsOHD9eyZcsUFBSkJ554QqVLl5ZhGDp79qy++uordenSRcOGDdOKFSt08+ZN21RuA05OTnrooYckSefPn9fXX3+dabuYmBh99NFH5tepx6QaPny4Oepq8uTJunHjRrr9N27c0OTJk6Vbo6yGDx9u82sBAAAAAABwJFaHUmnVr19fgYGB2rx5sz755BO1bdtWFotFKSkp2rlzp1577TW1adNGb731lvbu3WvLru/Y888/Lw8PD+nWyK/Ro0dr7dq12r9/v3bv3q3Zs2erZ8+eOnr0qCTp/vvvV5s2bdKdo2bNmho5cqQkKSwsTIMGDdLq1asVGhqq1atXa9CgQQoLC5MkjRw5UjVq1LD7dQIAAAAAABQmLrlok2dubm7q2rWrunbtqqioKC1dulRLly7VyZMnFRsbq4ULF2rhwoWqXbu2+vTpo169eqlMmTL5UUqOatWqpS+//FLjx49XdHS0Nm7cqI0bN2batlWrVvrss88y3Tdu3DhdunRJS5Ys0f79+zVu3LgMbfr27auXX37Z5tcAAAAAAADgaPIllEqrUqVKGj58uMqXL69PP/1Uly5dkiQZhqEjR47o/fff16effqoBAwZozJgx8vLyyu+SMnjggQf066+/avHixfrjjz909OhRxcbGytnZWeXLl1dAQIAee+yxbNeCcnJy0tSpU9WlSxctWLBAoaGhio6OVtmyZRUQEKABAwaoXbt2dr82AAAAAACAwihfQ6ldu3ZpyZIlWrt2ra5fvy7dCqPKlCmjLl266MiRI9q9e7du3LihuXPnat26dfrpp59UuXLl/CwrU2XLltWoUaM0atQoq87Trl07wicAAAAAAIAc2DyUOnfunDldL/Wpe4ZhyGKxqEWLFurXr586d+4sNzc3SdLx48c1Y8YMLVu2TGfPntWnn36qadOm2bosAAAAAAAAFCI2CaUSEhK0bt06BQUFadu2bUpJSZFhGJKkChUqqHfv3urbt6+qV6+e4diaNWvqvffeU9WqVTV9+nRt3brVFiUBAAAAAACgELMqlNq7d6+CgoK0evVqxcbGSrdGRTk7O+vBBx9Uv3799NBDD8nJKeeH/D388MOaPn26Lly4YE1JAAAAAAAAcABWhVL9+/eXxWIxR0Xdfffd5tP0KlasmKdzeXh4SLdCLQAAAAAAABRtVk/fc3V11cMPP6x+/fqpVatWd3yeSpUqae7cudaWAwAAAAAAAAdgVSg1ceJE9ejRQ2XKlLG6kBIlSqhFixZWnwcAAAAAAACFn1Wh1NChQ21XCQAAAAAAAIqNnFcgz0bHjh3VqVMnnTx5MtfHnDlzxjwOAAAAAAAAxZNVI6VOnz4ti8WixMTEXB+TlJRkHgcAAAAAAIDiyaqRUgAAAAAAAMCdsHsoFRsbK0lyd3e3d9cAAAAAAAAoJOweSq1YsUKSVLVqVXt3DQAAAAAAgEIiT2tKDRs2LNPXJ0yYIA8Pj2yPTUhIUGRkpC5duiSLxaLWrVvnrVIAAAAAAAAUGXkKpXbs2CGLxSLDMMzXDMNQaGhonjqtXr26nn322TwdAwAAAAAAgKIjT6FU8+bN0329c+dOWSwWNWjQINuRUhaLRSVKlFCFChV077336tFHH5Wnp+edVw0AAAAAAACHlqdQ6ocffkj3tb+/vyRp2rRpql27tm0rAwAAAAAAQJGVp1Dqdj179pTFYlHp0qVtVxEAAAAAAACKPKtCqWnTptmuEgAAAAAAABQbTgVdAAAAAAAAAIofQikAAAAAAADYXa6m79WrV0+69RS9/fv3Z3j9Ttx+LgAAAAAAABQfuQqlDMPI0+sAAAAAAABAdnIVSo0ZMyZPrwMAAAAAAADZIZQCAAAAAACA3bHQOQAAAAAAAOyOUAoAAAAAAAB2Z1UotXTp0js67urVqxo/frw1XQMAAAAAAMCBWRVKTZgwQS+//LJiYmJyfcy2bdvUo0cP/frrr9Z0DQAAAAAAAAdm9fS9tWvX6vHHH9fWrVuzbZeYmKhp06ZpxIgROnfunCwWi7VdAwAAAAAAwEFZFUoNHz5cknTu3DmNHDlS77//vhITEzO0O3z4sPr06aM5c+YoJSVFFSpU0IwZM6zpGgAAAAAAAA7M6ul7M2fOVMWKFZWSkqLZs2erb9++OnLkiNnm+++/V79+/XTkyBEZhqGHH35YK1asUJs2bWxRPwAAAAAAAByQi7UneOCBB7Ry5Ur95z//0bp163To0CH17dtXL7zwgv766y9t375dhmHI09NTb7zxhvr27WubygEAAAAAAOCwrF5TSpLKlCmjzz//XFOmTJGnp6du3rypTz75xAykGjdurGXLlhFIAQAAAAAAQLJVKJWqW7duatmypfm1YRjy8vLS1KlTdffdd9uyKwAAAAAAADgwm4VSe/fuVa9evfT7779Lkjw8PCRJcXFx6tu3rxYtWmSrrgAAAAAAAODgrA6lDMPQF198ocGDB+vkyZMyDEP9+vXT5s2b9cYbb8jNzU3Xr1/Xm2++qTFjxig6Oto2lQMAAAAAAMBhWRVKRUZGavDgwZo+fbqSkpLk7e2t6dOna/LkySpZsqSGDRumJUuWyN/fX4ZhKDg4WN27d9fmzZttdwUAAAAAAABwOFaFUj169NCePXtkGIZat26tFStWqFOnTuna1K5dW4sWLdKIESNksVh08eJFPfPMM5o8ebK1tQMAAAAAAMBBWRVKxcfHy9XVVRMmTNDMmTNVsWLFTNu5urrq1Vdf1ffff6+77rpLhmHop59+sqZrAAAAAAAAODCrQik/Pz8tWbJEw4cPz1X7li1bavny5Xr00Uet6RYAAAAAAAAOzsWagxcvXiw3N7c8HePl5aWPP/5Y7du3t6ZrAAAAAAAAODCrRkrlNZBK67HHHrOmawAAAAAAADgwq0ZK3e7UqVPavXu3Ll68qOvXr2vw4MHy8fGxZRcAAAAAAAAoAmwSSu3bt09Tp05VSEhIutcfeeSRdKHUvHnzNH36dHl5eemXX36Rq6urLboHAAAAAACAg7Fq+p4kbdy4UYMGDVJISIgMwzD/y8zjjz+uGzduKCIiQr///ru1XQMAAAAAAMBBWRVKnT9/XuPHj1dCQoJq166tb7/9NsNoqbRKlSqlDh06SJL++OMPa7oGAAAAAACAA7MqlJo9e7auX7+uu+66S/PmzVPbtm3l6emZ7TEtW7aUYRjat2+fNV0DAAAAAADAgVkVSm3evFkWi0UjRoxQ6dKlc3WMr6+vJCkyMtKargEAAAAAAODArAqlzpw5I0lq1KhRro8pVaqUJCk+Pt6argEAAAAAAODArAqlkpOTJUkpKSm5PiY2NlaScpzmBwAAAAAAgKLLqlCqfPnykqSIiIhcH7N3715JUpUqVazpGgAAAAAAAA7MqlCqWbNmMgxDa9asyVX7hIQELViwQBaLRS1atLCmawAAAAAAADgwq0KpXr16SZI2bNigLVu2ZNs2ISFBr732mk6dOiWLxaL+/ftb0zUAAAAAAAAcmIs1B7ds2VLdunXT6tWrNXr0aA0bNkxdunQx958+fVpXr15VSEiIFi5cqIiICFksFg0cOFB16tSxRf0AAAAAAABwQFaFUpI0bdo0Xbt2TZs2bdKsWbM0a9YsWSwWSdLo0aPNdoZhSJI6d+6siRMnWtstAAAAAAAAHJhV0/ckyc3NTd98840mTZqk6tWryzCMTP+rXLmy3nrrLf3vf/+Ts7OzbaoHAAAAAACAQ7J6pFSq/v37q3///jp69KjCwsJ06dIlJScnq2zZsqpXr54aNGhgjqACAAAAAABA8WazUCpV7dq1Vbt2bVufFgAAAAAAAEWI1dP3AAAAAAAAgLwilAIAAAAAAIDd5Wr63rJly/Kl8549e+bLeQEAAAAAAFC45SqUev31122+SLnFYiGUAgAAAAAAKKZyvdC5YRj5WwkAAAAAAACKjVyFUsHBwflfCQAAAAAAAIqNXIVSVatWzf9KAAAAAAAAUGzw9D0AAAAAAADYHaEUAABwCClGSkGXUCTxvgIAgIKS64XOc2Pfvn3666+/dPjwYcXExEiSypQpozp16uiBBx5Qw4YNbdkdAAAoRpwsTpr3z2ZFxcUUdClFRqVSZfRE47YFXQYAACimbBJK7du3T++8845CQ0OzbPPJJ5+oYcOGevPNNxUQEGCLbgEAQDETFRej01cvF3QZAAAAsAGrp++tWbNGAwcOVGhoqAzDkGEYcnFxUbly5VSuXDm5uLiYr4eGhmrQoEH69ddfbVM9AAAA7piXm7uMFKbv5QfeVwAAcmbVSKnw8HC9+uqrSkxMlIuLi/r166c+ffqoXr16cnZ2liQlJyfr4MGDWrx4sRYtWqSkpCS99tpr8vPzU61atWx1HQAAAMgjD1c3WZycFLN+vpKjzxd0OUWGc9mKKtNpUEGXAQBAoWdVKPXtt98qISFBJUqU0IwZM9SyZcsMbZydndWgQQM1aNBAXbt21ahRo5SQkKDvvvtO7733njXdAwAAwAaSo88r6eLpgi4DAAAUM1ZN39u6dassFouGDx+eaSB1uxYtWmj48OEyDENbt261pmsAAAAAAAA4MKtCqcuX/2+h0QcffDDXx7Rr1y7dsQAAAAAAACh+rAqlfHx8JEklSpTI9TFubm6SpLJly1rTNQAAAAAAAByYVaFU06ZNJUmhoaG5Pmbv3r2SpPvuu8+argEAAAAAAODArAqlnnzySTk7O+ubb77J1XS8S5cuacaMGXJxcdGTTz5pTdcAAAAAAABwYFaFUo0aNdI777yjS5cuqV+/flq/fr1SUlIytEtJSdH69es1YMAAXb58WW+//bYaNWpkTdcAAAAAAABwYC7WHDxhwgRJUu3atXXw4EGNHTtWpUuXVv369eXj4yOLxaJLly7pwIEDiomJkST5+/vr77//1t9//53pOS0Wi6ZOnWpNWQAAAAAAACjkrAqlli5dKovFIt0KkwzDUExMjLZt25aunWEYZpuDBw/q4MGDmZ7PMAxCKQAAAAAAgGLAqlDqrrvusl0lAAAAAAAAKDasCqU2bNhgu0oAAAAAAABQbFi10DkAAAAAAABwJ6waKTV9+nRJUuPGjdW2bVtb1QQAAAAAAIAizupQymKxmOEUAAAAAAAAkBtWTd/z9vaWWPAcAAAAAAAAeWRVKHXPPfdIki5cuGCregAAAAAAAFAMWDV9r2vXrvrnn3/066+/6sEHH7RdVQXozJkzWrx4sX7//XedOXNG165dk4+Pj6pWraqWLVuqa9eu8vPzy/L4TZs2aeHChQoNDdXly5fl4+OjgIAA9e/fX+3atbPrtQAAAAAAABRWVoVSgwcP1rJly7R8+XI1b95cvXv3tl1lBeCHH37Qf//7X8XHx6d7/dy5czp37pz+/vtvxcXFaeLEiRmOTUlJUWBgoBYvXpzu9aioKEVFRWn9+vXq16+fJk2aJCcnHnoIAAAAAACKN6tCqYsXL2rKlCmaOHGiJk6cqFWrVumxxx5T3bp1Vbp0aTk7O2d7fGFai+rLL7/UZ599JkmqUaOG+vfvr4CAAHl5eenKlSvav3+/1q1bl2Wg9Mknn5iBVP369fX000+revXqioiI0Hfffaf9+/dr0aJF8vHx0fjx4+16bQAAAAAAAIWNVaFUhw4dZLFYJEmGYWjr1q3aunVrro61WCzav3+/Nd3bzNatW81AqmfPnpoyZYpcXV3Ttbn//vs1cuRIJSQkZDj++PHjmjVrliSpYcOGmjdvntzd3SVJjRo1UocOHTRkyBCFhYVp5syZ6tOnj7keFwAAAAAAQHFk9TwywzBkGEa67dz+VxikpKTo7bffliT5+/vr3XffzRBIpeXm5pbhtTlz5igpKUmSFBgYaAZSqTw8PBQYGChJSkpK0uzZs218FQAAAAAAAI7FqpFS7733nu0qKSB//vmnTpw4IUkaNWqUXFzy9pYYhqHg4GBJkq+vr5o0aZJpuyZNmqhmzZo6fvy4goOD9eabb5qjzAAAAAAAAIobq0KpXr162a6SArJmzRrp1nTChx56yHz9ypUrunLliry9veXt7Z3l8ZGRkTp//rwkqXnz5tn21aJFCx0/flxRUVGKjIxU9erVbXYdAAAAAAAAjsSqUKoo+OeffyRJVatWValSpbRy5UrNmDFDhw8fNtukLnw+dOjQDNP3jh49am77+vpm21fa/eHh4YRSAAAAAACg2LJ6TSlHlpKSovDwcElS2bJlNWXKFP373/9OF0hJ0okTJ/TBBx9o2LBhunr1arp9586dM7crV66cbX9p9589e9ZGVwEAAAAAAOB4bDZSKiUlRdu3b9fu3bt18eJFXb9+XePGjVPFihXNNgkJCUpOTpazs3OmC4bbW2xsrFJSUiRJhw8fVmhoqCpUqKBXX31V7dq1U4kSJRQaGqqPPvpIe/bs0e7du/XGG29o+vTp5jmuXbtmbnt6embbn4eHh7kdHx+f53qTk5PzfExOnJ2dbX5OID/lx+cgP/DZgiPhcwXkD0f5bAEAYGu5/TfQJqHUxo0bNWXKFJ05cybd6yNHjkwXSi1atEhTpkyRp6enNm/enGOIk9+uX79ubt+8eVMeHh6aO3duuml2zZs315w5czRgwAAdPHhQ69at0z///KPGjRubx6XK7ql9uu3JfTdu3MhzvaGhoXk+JjseHh6qX7++Tc8J5LdDhw6l++wWRny24Gj4XAH5wxE+WwAAFCSrQ6mFCxfqrbfekmEY0q1pcNHR0Zk+Wa5fv3767LPPFBsbq3Xr1unxxx+3tnur3D5aq2/fvpmuC+Xu7q5x48bp2WeflSStXr3aDKVKlChhtktMTMy2v4SEhHTnzKuAgAD+Soxir27dugVdAlDk8LkC8gefLQBAcZWcnJyrgTVWhVInTpzQpEmTJEmtWrVSYGCgatWqJX9//0zbu7m5qXPnzlq8eLG2bNlS4KFUqVKl0n3dpk2bLNvef//9cnFxUVJSUro3tmTJkuZ2TlPy0v6l7E5GiTk7OxNKodjjMwDYHp8rIH/w2QIAIHtWLXQ+e/ZsJSUlqXbt2poxY4Zq1aqV4zHNmjWTJB04cMCarm3Czc1NPj4+5tfZLVReokQJlS1bVpJ0+fLlTI9Ju+h5ZtLur1Klyh3XDQAAAAAA4OisCqW2bdsmi8Wi4cOH53rh8rvvvlsqRE+fq127trmduuh5VlIX6nJx+f8DzNIen/okv6yk3Z/ZNEEAAAAAAIDiwqpQKioqSpKynK6XmdRpa3ey0Hd+aN68ubkdERGRZbu4uDhFR0dLkipVqmS+Xq1aNXMx9507d2bbV+r+SpUqqVq1albXDgAAAAAA4KisCqVS5SVgSg12bl/PqaB07tzZ3F63bl2W7datW2cu5n7fffeZr1ssFnXs2FG6NRJqz549mR6/Z88ec6RUx44dM10IHgAAAAAAoLiwKpRKHTGU3Qij2/3999+SpOrVq1vTtc34+/vrwQcflCT98ssv2rp1a4Y2Fy5c0KeffipJcnV1VZ8+fdLtHz58uLmQ5eTJkzOEdDdu3NDkyZOlW1P/hg8fnm/XAwAAAAAA4AisCqVatGghwzC0dOnSXLWPjY3Vzz//LIvFolatWlnTtU298cYbKl26tFJSUvTss8/q448/1q5duxQaGqp58+apb9++5iLlL730Urrpe5JUs2ZNjRw5UpIUFhamQYMGafXq1QoNDdXq1as1aNAghYWFSZJGjhypGjVqFMBVAgAAAAAAFB4uuWiTpYEDB2rRokXauXOngoKC1Lt37yzbRkdH68UXX9TFixfl4uKigQMHWtO1TdWsWVNfffWVXnrpJV28eFEzZszQjBkz0rWxWCwaPXq0Ro0alek5xo0bp0uXLmnJkiXav3+/xo0bl6FN37599fLLL+fbdQAAAAAAADgKq0Kp+vXra9iwYZozZ44mTpyoP/74I90aTbt379aBAwcUEhKiVatWKS4uThaLRc8//7yqVq1qi/ptplmzZlq1apV+/PFHrV+/XpGRkUpMTFSFChXUokULDR06VPXr18/yeCcnJ02dOlVdunTRggULFBoaqujoaJUtW1YBAQEaMGCA2rVrZ9drAgAAAAAAKKysCqUk6fXXX1dCQoLmz5+vtWvXau3ateYi3m+++abZLnWR8OHDh+v555+3ttt8UbZsWY0dO1Zjx46943O0a9eO8AkAAAAAACAHVj99z2Kx6K233tLMmTPVokULWSwWGYaR7j9JatKkib755htNmDDBFnUDAAAAAADAgVk9UipV69at1bp1a8XFxenAgQO6dOmSUlJS5O3tLX9/f/n4+NiqKwAAAAAAADg4q0KpK1eumMGTk9P/DboqVaqUmjdvbqv6AAAAAAAAUATlOZQ6ePCgvvzyS23ZskXx8fGSJFdXVzVr1kwjR45U69at86NOAAAAAAAAFCF5WlNq/fr16t+/v9atW6dr166Za0YlJCRo69atevrppzVjxoz8qxYAAAAAAABFQq5DqfPnz5tP2jMMQ+7u7mrQoIGaNGmi0qVLmwHVp59+qpCQkPytGgAAAAAAAA4t19P3Fi5cqLi4OFksFj355JMaO3asPD09JUnJycn6+eefNXXqVKWkpGj27Nlq2rRpftYNAAAAAAAAB5brkVJ//fWXLBaL2rdvr9dee80MpCTJ2dlZTzzxhJ5++mkZhqGtW7fmV70AAAAAAAAoAnIdSoWHh0uSevfunWWbPn36SJLi4uJ08eJFW9QHAAAAAACAIijXoVRcXJwkqVq1alm2qVq1qrkdGxtrbW0AAAAAAAAoonIdSiUlJUmSXFyyXobK2dnZ3E5OTra2NgAAAAAAABRRuQ6lAAAAAAAAAFvJcyhlsVhs2g4AAAAAAADFT9Zz8bIwYsSIbKfw5badxWLR+vXr89o9AAAAAAAAioA8h1JRUVHZ7k8dIZXbdgAAAAAAACh+ch1K3XXXXflbCQAAAAAAAIqNXIdSGzZsyN9KAAAAAAAAUGzw9D0AAAAAAADYHaEUAAAAAAAA7I5QCgAAAAAAAHZHKAUAAAAAAAC7I5QCAAAAAACA3RFKAQAAAAAAwO4IpQAAAAAAAGB3hFIAAAAAAACwO0IpAAAAAAAA2B2hFAAAAAAAAOyOUAoAAAAAAAB2Z1Uo1aFDB3Xq1EknT57M9TFnzpxRx44d1alTJ2u6BgAAAAAAgANzsebgM2fOyGKxKDExMdfHJCUl6fTp07JYLNZ0DQAAAAAAAAfG9D0AAAAAAADYnd1DqdjYWEmSu7u7vbsGAAAAAABAIWH3UGrFihWSpKpVq9q7awAAAAAAABQSeVpTatiwYZm+PmHCBHl4eGR7bEJCgiIjI3Xp0iVZLBa1bt06b5UCAAAAAACgyMhTKLVjxw5ZLBYZhmG+ZhiGQkND89Rp9erV9eyzz+bpGAAAAAC2l2KkyMnCUrO2xvsKADnLUyjVvHnzdF/v3LlTFotFDRo0yHaklMViUYkSJVShQgXde++9evTRR+Xp6XnnVQMAAACwCSeLk+b9s1lRcTEFXUqRUalUGT3RuG1BlwEAhV6eQqkffvgh3df+/v6SpGnTpql27dq2rQwAAACAXUTFxej01csFXQYAoJjJUyh1u549e8pisah06dK2qwgAAAAAAABFnlWh1LRp02xXCQAAAAAUAV5u7jJSUmRxYk0pW+N9BYoWq0Kp3Dp16pSio6NVtWpVlS9f3h5dAgAAAECB8HB1k8XJSTHr5ys5+nxBl1NkOJetqDKdBhV0GQBsyKpQ6tKlS1qzZo0kqUePHvLy8kq3/+TJkxo3bpwOHDgg3VrwvGPHjpoyZYrKlCljTdcAAAAAUKglR59X0sXTBV0GABRaVo17/O233zR58mTNnTs3QyCVkJCgUaNG6cCBAzIMQ4ZhKCUlRevXr9fzzz9vbd0AAAAAAABwYFaFUlu2bJHFYtHDDz+cYV9QUJBOnTolSerQoYMmTpyo9u3byzAMhYSEaPXq1dZ0DQAAAAAAAAdmVSh1/PhxSVKTJk0y7Fu1apUkqVWrVvryyy81dOhQffXVV3rggQdkGIZ++eUXa7oGAAAAAACAA7MqlLp8+bIkqVKlSulev3Hjhvbs2SOLxaL+/fun29enTx9J0v79+63pGgAAAAAAAA7MqlAqNjb2/05y2yM59+zZo6SkJFksFj3wwAPp9lWrVk26tUg6AAAAAAAAiierQilPT09J0sWLF9O9vmPHDklSrVq1Mjxlz8Xl/x745+zsbE3XAAAAAAAAcGBWhVK+vr6SpM2bN6d7/bfffpPFYlGLFi0yHJMaYJUvX96argEAAAAAAODAXKw5uF27dtqzZ48WLFggX19fNWvWTEuXLtXRo0ezfCrfvn37pEzWoQIAAAAAAEDxYVUoNWTIEP3000+6cOGCJk+enG5fkyZN1KpVqwzHbNy4URaLRQEBAdZ0DQAAAAAAAAdm1fQ9Ly8vzZ49W/Xr15dhGOZ/zZo106effpqh/cGDBxUaGipJGRZABwAAAAAAQPFh1Ugp3VrMPCgoSBEREbp48aIqVKhgPmEvM++9954kZTqKCgAAAAAAAMWD1aFUqurVq6t69erZtvH395e/v7+tugQAAAAAAICDsmr6HgAAAAAAAHAnbDZSKiUlRdu3b9fu3bt18eJFXb9+XePGjVPFihXNNgkJCUpOTpazs7Pc3Nxs1TUAAAAAAAAcjE1CqY0bN2rKlCk6c+ZMutdHjhyZLpRatGiRpkyZIk9PT23evFmenp626B4AAAAAAAAOxurpewsXLtTzzz+v06dPyzAMeXt7yzCMTNv269dPXl5eio+P17p166ztGgAAAAAAAA7KqlDqxIkTmjRpknTraXq//PKLtm7dmmV7Nzc3de7cWYZhaMuWLdZ0DQAAAAAAAAdmVSg1e/ZsJSUlqXbt2poxY4Zq1aqV4zHNmjWTJB04cMCargEAAAAAAODArAqltm3bJovFouHDh+d64fK7775bknT27FlrugYAAAAAAIADsyqUioqKkiT5+/vn+pjUxc1v3LhhTdcAAAAAAABwYFYvdK48BkzR0dGSpFKlStmiawAAAAAAADggq0KpSpUqSZIiIiJyfczff/8tSapevbo1XQMAAAAAAMCBWRVKtWjRQoZhaOnSpblqHxsbq59//lkWi0WtWrWypmsAAAAAAAA4sFyHUv7+/qpfv76OHj1qvjZw4EBZLBbt3LlTQUFB2R4fHR2t559/XhcvXpSzs7MGDhxoXeUAAAAAAABwWC55aWwYRrqv69evr2HDhmnOnDmaOHGi/vjjD3Xu3Nncv3v3bh04cEAhISFatWqV4uLiZLFY9Pzzz6tq1aq2uwoAAAAAAAA4lDyFUpl5/fXXlZCQoPnz52vt2rVau3atLBaLJOnNN98026UGWsOHD9fzzz9vbbcAAAAAAABwYFY/fc9iseitt97SzJkz1aJFC1ksFhmGke4/SWrSpIm++eYbTZgwwRZ1AwAAAAAAwIFZPVIqVevWrdW6dWvFxcXpwIEDunTpklJSUuTt7S1/f3/5+PjYqisAAAAAAAA4OJuFUqlKlSql5s2b2/q0AAAAAAAAKEKsnr4HAAAAAAAA5FWeR0pNmDBBHh4eVndssVg0Z84cq88DAAAAAAAAx5PnUCosLMzqTg3DMJ/QBwAAAAAAgOInz6FU6tP0AAAAAAAAgDuV51Bq1apVql27dv5UAwAAAAAAgGKBhc4BAAAAAABgd4RSAAAAAAAAsDtCKQAAAAAAANgdoRQAAAAAAADsjlAKAAAAAAAAdkcoBQAAAAAAALtzyW3D4OBgSVKlSpXysx4AAAAAAAAUA7kOpapWrZq/lQAAAAAAAKDYYPoeAAAAAAAA7I5QCgAAAAAAAHZHKAUAAAAAAAC7I5QCAAAAAACA3eV6ofPi6MMPP9R3331nfj137ly1bNky22M2bdqkhQsXKjQ0VJcvX5aPj48CAgLUv39/tWvXzg5VAwAAAAAAFH6EUlk4cOCAZs+enev2KSkpCgwM1OLFi9O9HhUVpaioKK1fv179+vXTpEmT5OTEADUAAAAAAFC8EUplIjVgSkpKUrly5XTp0qUcj/nkk0/MQKp+/fp6+umnVb16dUVEROi7777T/v37tWjRIvn4+Gj8+PF2uAoAAAAAAIDCiyE7mZg7d65CQ0Pl6+urvn375tj++PHjmjVrliSpYcOGmj9/vh599FE1atRIjz76qH766Sc1bNhQkjRz5kydPHky368BAAAAAACgMCOUus2ZM2f02WefSZLeeecdubq65njMnDlzlJSUJEkKDAyUu7t7uv0eHh4KDAyUJCUlJeVpWiAAAAAAAEBRRCh1m0mTJik+Pl69evVSixYtcmxvGIaCg4MlSb6+vmrSpEmm7Zo0aaKaNWtKkoKDg2UYho0rBwAAAAAAcByEUmmsXr1aGzdulLe3t1599dVcHRMZGanz589Lkpo3b55t29SQKyoqSpGRkTaoGAAAAAAAwDERSt1y9epVTZ06VZL073//Wz4+Prk67ujRo+a2r69vtm3T7g8PD7/jWgEAAAAAABwdodQtH374oS5cuKCmTZvmanHzVOfOnTO3K1eunG3btPvPnj17h5UCAAAAAAA4PpeCLqAw2LVrlxYtWiQXFxe98847slgsuT722rVr5ranp2e2bT08PMzt+Pj4PNeZnJyc52Ny4uzsbPNzAvkpPz4H+YHPFhwJnysgf/DZAvKHo3y2gOIst5/TYh9KJSQkKDAwUIZhaPjw4fLz88vT8Tdv3jS3c3pSn5ubm7l948aNPNcaGhqa52Oy4+Hhofr169v0nEB+O3TokK5fv17QZWSLzxYcDZ8rIH/w2QLyhyN8tlxdXVW/QQO5EPranJGSIosTk75sLSU5SWH79isxMdGu/Rb7UOqbb75ReHi47rrrLo0ZMybPx5coUcLczumbl5CQYG67u7vnua+AgAD+koVir27dugVdAlDk8LkC8gefLSB/OMpny9nZWfP+2ayouJiCLqXI8K9wl7r5NVXM+vlKjj5f0OUUGc5lK6pMp0Fq0KCBzc6ZnJycq4E1xTqUOnbsmL755htJ0n/+858cp99lpmTJkuZ2TlPy0qb5d9KXs7MzoRSKPT4DgO3xuQLyB58tIH840mcrKi5Gp69eLugyioyKJUtLkpKjzyvp4umCLqfIKYjPVrEOpebMmaPExERVr15dN27c0C+//JKhzZEjR8ztbdu26eLFi5Kk9u3by9PTM93i5WkXPc9M2v1VqlSx0VUAAAAAAAA4nmIdSqVOp4uIiND48eNzbP/ll1+a28HBwfL09FTt2rXN18LDw7M9Pu1+X1/fO6waAAAAAADA8bE6mJWqVaumihUrSpJ27tyZbdvU/ZUqVVK1atXsUh8AAAAAAEBhVKxDqWnTpunQoUPZ/pd28fO5c+ear6eGShaLRR07dpRujYTas2dPpn3t2bPHHCnVsWNHWSwWu1wjAAAAAABAYVSsQylbGT58uLkg2OTJk3Xjxo10+2/cuKHJkydLklxcXDR8+PACqRMAAAAAAKCwIJSygZo1a2rkyJGSpLCwMA0aNEirV69WaGioVq9erUGDBiksLEySNHLkSNWoUaOAKwYAAAAAAChYxXqhc1saN26cLl26pCVLlmj//v0aN25chjZ9+/bVyy+/XCD1AQAAAAAAFCaEUjbi5OSkqVOnqkuXLlqwYIFCQ0MVHR2tsmXLKiAgQAMGDFC7du0KukwAAAAAAIBCgVAqB2PHjtXYsWNz3b5du3aETwAAAAAAADlgTSkAAAAAAADYHaEUAAAAAAAA7I5QCgAAAAAAAHZHKAUAAAAAAAC7I5QCAAAAAACA3RFKAQAAAAAAwO4IpQAAAAAAAGB3hFIAAAAAAACwO0IpAAAAAAAA2B2hFAAAAAAAAOyOUAoAAAAAAAB2RygFAAAAAAAAuyOUAgAAAAAAgN0RSgEAAAAAAMDuCKUAAAAAAABgd4RSAAAAAAAAsDtCKQAAAAAAANgdoRQAAAAAAADsjlAKAAAAAAAAdkcoBQAAAAAAALsjlAIAAAAAAIDdEUoBAAAAAADA7gilAAAAAAAAYHeEUgAAAAAAALA7QikAAAAAAADYHaEUAAAAAAAA7I5QCgAAAAAAAHZHKAUAAAAAAAC7I5QCAAAAAACA3RFKAQAAAAAAwO4IpQAAAAAAAGB3hFIAAAAAAACwO0IpAAAAAAAA2B2hFAAAAAAAAOyOUAoAAAAAAAB2RygFAAAAAAAAuyOUAgAAAAAAgN0RSgEAAAAAAMDuCKUAAAAAAABgd4RSAAAAAAAAsDtCKQAAAAAAANgdoRQAAAAAAADsjlAKAAAAAAAAdkcoBQAAAAAAALsjlAIAAAAAAIDdEUoBAAAAAADA7gilAAAAAAAAYHeEUgAAAAAAALA7QikAAAAAAADYHaEUAAAAAAAA7I5QCgAAAAAAAHZHKAUAAAAAAAC7I5QCAAAAAACA3RFKAQAAAAAAwO4IpQAAAAAAAGB3hFIAAAAAAACwO0IpAAAAAAAA2B2hFAAAAAAAAOyOUAoAAAAAAAB2RygFAAAAAAAAuyOUAgAAAAAAgN0RSgEAAAAAAMDuCKUAAAAAAABgd4RSAAAAAAAAsDtCKQAAAAAAANgdoRQAAAAAAADsjlAKAAAAAAAAdkcoBQAAAAAAALsjlAIAAAAAAIDdEUoBAAAAAADA7gilAAAAAAAAYHeEUgAAAAAAALA7QikAAAAAAADYHaEUAAAAAAAA7I5QCgAAAAAAAHZHKAUAAAAAAAC7I5QCAAAAAACA3RFKAQAAAAAAwO4IpQAAAAAAAGB3hFIAAAAAAACwO0IpAAAAAAAA2B2hFAAAAAAAAOyOUAoAAAAAAAB2RygFAAAAAAAAuyOUAgAAAAAAgN0RSgEAAAAAAMDuCKUAAAAAAABgdy4FXUBBCw0N1aZNmxQSEqKjR4/q8uXLcnV1VcWKFdW0aVP16dNHzZo1y/X5Nm3apIULFyo0NFSXL1+Wj4+PAgIC1L9/f7Vr1y5frwUAAAAAAMBRFOtQ6oknntCuXbsyvJ6YmKgTJ07oxIkTCgoKUs+ePTV58mS5ubllea6UlBQFBgZq8eLF6V6PiopSVFSU1q9fr379+mnSpElycmKAGgAAAAAAKN6KdSh1/vx5SVLFihX1yCOPqFmzZqpSpYpSUlK0Z88ezZo1S1FRUVq2bJmSkpL08ccfZ3muTz75xAyk6tevr6efflrVq1dXRESEvvvuO+3fv1+LFi2Sj4+Pxo8fb7drBAAAAAAAKIyKdSjl6+urcePGqUuXLnJ2dk63r0mTJurRo4cGDRqkEydOaNWqVRo4cKCaN2+e4TzHjx/XrFmzJEkNGzbUvHnz5O7uLklq1KiROnTooCFDhigsLEwzZ85Unz59dM8999jpKgEAAAAAAAqfYj2P7JtvvlG3bt0yBFKpfHx89Prrr5tfr127NtN2c+bMUVJSkiQpMDDQDKRSeXh4KDAwUJKUlJSk2bNn2/AqAAAAAAAAHE+xDqVyo2XLlub2qVOnMuw3DEPBwcHSrZFXTZo0yfQ8TZo0Uc2aNSVJwcHBMgwj32oGAAAAAAAo7AilcpCQkGBuZ7ZAeWRkpLk2VWZT+9Jq0aKFdGvx88jISJvXCgAAAAAA4CgIpXKwc+dOc7tWrVoZ9h89etTc9vX1zfZcafeHh4fbrEYAAADg/7V353FVlfsex7/MgjgLag45JQ6glikO5UjOpKXZ9WrYcPXYsY561KOdezy313XI6pR19NiNsjyWnswhETWnNOd5AsPAJExRERWUQXAz3D8ue18Q2HujmwXI5/1Pa+/1rGf/tr0eWOvLs54FAEBFQyhlRU5OjkJDQy2vBw0aVKjN1atXLdv169e32l/+/VeuXHFYnQAAAAAAABVNpX76ni3Lli1TRESEJKl///7y9/cv1CYtLc2y7eXlZbU/T09Py3Z6enqJ68nOzi7xMbYUt8g7UF6VxjgoDYwtVCSMK6B0MLaA0sHYAkqHI8eWvX0RShXjyJEj+uCDDyRJderU0dtvv11ku8zMTMu2m5ub1T7d3d0t2xkZGSWuKTIyssTHWOPp6am2bds6tE+gtEVHR+vOnTtlXYZVjC1UNIwroHQwtoDSwdgCSkdZjC1CqSKcO3dOb7zxhrKysuTh4aGPP/5YderUKbKth4eHZdtkMlntN/+i6VWqVClxXQEBAaTtqPT8/PzKugTgocO4AkoHYwsoHYwtoHQ4cmxlZ2fbNbGGUOoeFy9e1Kuvvqpbt27JxcVFH374odWn6lWtWtWybeuWvPyJo61b/Yri4uJCKIVKjzEAOB7jCigdjC2gdDC2gNJRFmOLhc7zSUhI0CuvvKJr167JyclJ8+fPV1BQkNVj8i9enn/R86Lk39+gQQMHVAwAAAAAAFAxEUrluXnzpl599VVdvHhRkjR79mwNHz7c5nEtW7a0bMfGxlptm39/8+bNH6heAAAAAACAioxQSlJKSor+4z/+Q7/88oskadq0aRozZoxdxzZq1Ei+vr6SpKNHj1pta95fr149NWrU6IHrBgAAAAAAqKgqfSh1584dTZgwQT/99JMkaeLEiZowYYLdxzs5Oalfv35S3kyoU6dOFdnu1KlTlplS/fr1k5OTk0PqBwAAAAAAqIgqdSh19+5dvfHGGzpx4oQkKSQkRFOnTi1xP+PGjbMsCDZnzhxlZGQU2J+RkaE5c+ZIklxdXTVu3DiH1A8AAAAAAFBRVeqn702bNk379u2TJHXt2lUjR45UTExMse3d3NzUrFmzQu83a9ZMr732mkJDQ3XmzBmNHj1a48ePV+PGjXXx4kV99tlnioqKkiS99tpratq0aSl+KwAAAAAAgPKvUodS27Zts2wfOnRIzz77rNX2DRs21M6dO4vcN3XqVN24cUNr165VVFRUkTOuRo4cqSlTpjigcgAAAAAAgIqtUodSjuTs7Kz58+drwIABWrVqlSIjI5WUlKRatWopICBAL774onr16lXWZQIAAAAAAJQLlTqUio6OdnifvXr1InwCAAAAAACwoVIvdA4AAAAAAICyQSgFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCEUgAAAAAAADAcoRQAAAAAAAAMRygFAAAAAAAAwxFKAQAAAAAAwHCuZV3Awyg+Pl5fffWVfvzxR129elXu7u5q3LixBg0apDFjxsjT07OsSwQAAAAAAChThFIOtnPnTs2YMUOpqamW9+7cuaNbt27pzJkzWr16tUJDQ/Xoo4+WaZ0AAAAAAABlidv3HCgqKkpTp05VamqqvLy8NHXqVH3zzTdatmyZRo0aJUmKi4vThAkTCoRWAAAAAAAAlQ0zpRxo3rx5ysjIkKurq7744gs9/vjjln3dunXTo48+qvfff19xcXH68ssv9eabb5ZpvQAAAAAAAGWFmVIOEhERoWPHjkmSRowYUSCQMnv11VfVokULSdLy5ctlMpkMrxMAAAAAAKA8IJRykB07dli2R4wYUWQbZ2dnDR8+XJJ0+/ZtHT582LD6AAAAAAAAyhNCKQc5fvy4JMnLy0vt2rUrtl3nzp0t2ydOnDCkNgAAAAAAgPKGUMpBzp8/L0lq0qSJXF2LX6qrefPmhY4BAAAAAACobAilHCAzM1NJSUmSpPr161ttW6NGDXl5eUmSrl69akh9AAAAAAAA5Q1P33OAtLQ0y7Y5cLLG09NT6enpSk9Pt6v/3NxcSdLdu3fl4uLyAJUW5uLiogZVa8hFTg7ttzKr41lV2dnZcqpVX85Ojv3/VZk51fRRdna2srOzy7oUuzC2HItxVToYV2BslQ7GFhhbpYOxBcZW6SiNsWXuy5xnFPvZubZawKYrV66od+/ekqRhw4bpvffes9q+d+/eunLlipo0aaLt27fb7P/u3buKjIx0WL0AAAAAAAClLSAgQO7u7sXuZ6aUA3h4eFi2TSaTzfZ3796VJFWpUsWu/l1dXRUQECBnZ2c5OZGyAwAAAACA8is3N1c5OTlW19wWoZRjVK1a1bJtzy15d+7ckey81U+SnJ2drSaLAAAAAAAAFQ0LnTuAh4eHatasKdmxePmtW7cswZWtRdEBAAAAAAAeVoRSDtKyZUtJ0m+//aasrKxi28XGxlq2W7RoYUhtAAAAAAAA5Q2hlIN06tRJyrt976effiq23dGjRy3bTzzxhCG1AQAAAAAAlDeEUg4SFBRk2V67dm2RbXJycrR+/XpJUvXq1RUYGGhYfQAAAAAAAOUJoZSDtG/fXk8++aSUF0qdPHmyUJsvvvhC58+flySFhITIzc3N8DoBAAAAAADKA6fc3Nzcsi7iYREVFaXRo0crIyNDXl5emjhxogIDA5WRkaHNmzdr1apVkqSmTZtq7dq18vb2LuuSAQAAAAAAygShlIPt3LlTM2bMUGpqapH7mzZtqtDQUD366KOG1wYAAAAAAFBeEEqVgvj4eC1fvlw//vijEhIS5ObmpiZNmmjgwIEaO3asPD09y7pE2Ony5ctas2aNfvzxR12+fFlpaWmqXbu2GjZsqMDAQA0aNEitWrUqcMysWbP03XffWV4///zzeuedd2x+1oYNGzRjxgzL64YNG2rnzp0O/kaA4924cUMRERGKiIhQZGSkIiMjlZycLEl67rnntGDBApt95OTkKDY2tkA/0dHRMplMkqTly5fbtQ6fn59fgddff/21OnfubPO4V155RQcOHLC8fuONN/Tmm2/aPA4oTY4YW2YXL17UV199pQMHDig+Pl65ubny9fVV9+7dNWbMGD322GNWj+/bt6/i4+Mtr999910NHz7c5uf+5S9/0erVqy2vS1o34EipqanavXu3IiMjdebMGSUkJOjmzZvKzMxUtWrV1LJlS/Xs2VMjR45UrVq1iuzj8OHDCgkJsevzrP0uYUzhYVfS66hLly6pX79+JfqM4q6XOB+sWFzLuoCHUcOGDfXWW2/prbfeKutS8AC++uorffjhh0pPTy/w/tWrV3X16lUdP35cqamp+s///E+r/Wzbtk1vv/22PDw8rLYLCwtzSN2A0bp37/7AfYSFhWnWrFkOqSe/DRs22DwJSUhI0KFDhxz+2cCDcsTYkqRVq1Zpzpw5lpDX7MKFC7pw4YLWrFmjWbNmaezYsXb3uWHDBpsX0JmZmdqyZct91w04WkREhP74xz8Wue/mzZs6cuSIjhw5oqVLl+r999/X008/bVhtjCk8TBx1HWVLs2bN7GrH+WD5RigFFGHJkiX6+OOPpbxbLkeNGqWAgABVq1ZNycnJioqK0vbt2+XsbP1ZAR4eHkpNTdUPP/ygwYMHF9suMTFRBw8etByTmZnp4G8EGOORRx5R8+bNtW/fvhIdl3/Srpubm1q1aiWTyaSYmJj7qsM8jrZs2aLZs2fL3d292Lbh4eHKyclh7KFcu9+xtWnTJv31r3+VJFWrVk2vvPKKunbtKnd3d509e1aff/65Lly4oLlz56p27dpWf1cp39g6ePCgEhISVK9evWLb7ty5UykpKYwtlCsNGjRQYGCg2rVrpwYNGsjHx0c5OTm6evWqtm7dqu3btyspKUmvv/661qxZo9atWxfb1/z58xUQEFDs/jp16tishzGFh839XkfVq1dP4eHhNvv/9NNPtXHjRkmyGeRyPlgxEEoB9zh48KDlB+nw4cM1d+7cQk9K7Natm1577TXdvXvXal99+/bV999/r7CwMKsn+hs3blR2drZ8fX3VpEkTHTt2zEHfBih9kyZNUkBAgAICAlS3bt37mn7dsmVL/eUvf1FAQIDatGkjDw8PLVq06L5Dqaeeekp79+7V7du3tXPnTg0cOLDYtuZZiv369dPmzZvv6/OA0vCgY+vOnTuaN2+eJMnLy0srV64scKtEQECABg8erNGjRysmJkbz5s1Tr169VLVq1WL79Pf312+//abExERt3LhRr732WrFtGVsobwIDA/Xjjz8Wu3/w4MHasWOHJk2aJJPJpMWLF2vx4sXFtm/UqFGhZRxKijGFh8mDXEeZ/yhpTXZ2to4cOSJJqlq1qp555hmr7TkfrBisT/MAKpmcnBy9/fbbkqTWrVtr3rx5hX6Q5mctbVe+9H7fvn26ceNGse3MPwSDg4Ntzr4Cyps//OEP6tOnj+rWrXvffbRv314vvfSSOnbsaPNWV3tUr15dffr0kWzcGvvzzz9bgi971vIAjPSgY2v37t2W3z0hISFFnux7e3tblhu4fv16gTURi+Li4qLg4GDJxti6efOmZVYXYwvlhYuLi802QUFBlluCjPgjIWMKDwtHX0cV5cCBA7p27ZokacCAAapSpYrV9pwPVgxc/QL57Nu3T3FxcZKk8ePHy9X1wSYTPvXUU6pdu7aysrK0adOmItvExMTo7NmzkqRhw4Y90OcB+H/m8bR3714lJSUV2Wb9+vWSpHbt2qlFixaG1geUtjNnzli2e/bsWWy7Ll26WMLgrVu32uzXPLaio6P1888/F9lm06ZNMplMqlOnjnr06HEf1QNlxzxb0KhbeBhTeBg4+jqqKObzNuUt8m8PzgfLP0IpIB/z4pFOTk7q3bu35f3k5GTFxcVZnnpkL1dXVw0ZMkSyks6bfwi2bt260JMiANy/nj17qmbNmjKZTEVOw87OzrasSfDss8+WQYVA6cr/O8vabCtXV1fVqFFDknTy5EllZWVZ7Tf/76vifreZ3x8yZEipXJgApSU2NtYSDDVv3tyQz2RM4WHg6Ouoe5nX6VXeg8XseZqeOB+sEAilgHxOnz4t5f2g8/b2Vnh4uIKDgxUYGKgBAwZY/rt06VKb60mZmdP5M2fO6Pz58wX25eTkWH4IMksKcCw3NzerofCBAweUmJgoV1dXDR06tAwqBEqXl5eXZTslJaXYdrm5uUpNTZUkmUwmXbhwwWbf5t9ZGzduVE5OToF9sbGxioyMLNAOKM/u3LmjuLg4ffnll3rppZcswey4ceOsHrdw4UL16dNH/v7+6ty5s4YPH6758+fr119/LXENjClUdKVxHZXf1q1bdefOHSlvHDg5Odl1HOeD5R+hFJAnJydHsbGxkqRatWpp7ty5mj59eqGFluPi4vTee+8pJCREt2/fttlvQECAZRrohg0bCuw7dOiQEhISCqwnAMBxzOsCnD592jKl3Mx8YtKjR48HWg8LKK/y34Jw9OjRYttFRUUVeGz3lStXbPYdHBwsFxcXXbt2zfL0WDPz2GrZsqX8/f3vs3qgdK1bt05+fn7y8/NTx44dNWDAAC1YsEDXr1+XJE2YMMHmudnJkyd1+fJlmUwm3b59W2fPntU///lPDR48WIsWLSrwZFlbGFOoyErrOiq//LfulXTdJ84HyzdCKSBPSkqK5S9TMTEx+uqrr+Tj46P3339fR44c0enTp/X111+rY8eOUt6JyJ///Ge7+jb/VSs8PLzACYr5h2C3bt3k4+NTCt8KqNzat2+vpk2bSveEwmlpadqxY4fEX53xEOvZs6flNp8vv/xSN2/eLNQmJydHCxcuLPBeWlqazb59fX3VrVs36Z6xlZuba3mkN2MLFVGbNm20evVqTZs2rdiZGD4+PhozZow+/PBDrV69WuvWrdM//vEPjRw5Um5ubsrJydHixYsLjS1rGFOoyErzOkqSLl++bPnjyuOPP65HH320RPVxPli+EUoBeczTQZW3sKWnp6eWL1+uZ599VjVq1FCVKlXUuXNn/fOf/1Tr1q0lSdu3b7dMVbUmODhYTk5Oio+PtzzJ5c6dO9q2bZvED0GgVJnHV/6TkO3bt+vOnTvy9vZWv379yrA6oPQ0aNBA//Zv/yZJSkhI0OjRo7Vjxw6lpqYqMzNTp06d0vjx47V3794CT0jKyMiwq3/z2hvbtm2z/A49duyY4uPj5ezszAxglGtBQUEKDw9XeHi4Vq9erQ8//FDPPPOMzp49q2nTpmnXrl1FHhcQEKBdu3bpr3/9q4YMGaL27durXbt2CgoK0rx587Ry5UpVq1ZNkhQaGlrswuVFYUyhoirN6yjlncOZ/7B/v0/H43yw/CKUAvLc+1jSkSNHFrnAZZUqVTR16lTL66IWzLvXI488oi5dukj5Zkdt27ZN6enp8vLy0jPPPOOAbwCgKOZ1By5evKjjx49L+aaA2/M4YaAimzlzpnr16iXl3TYxadIkderUSe3bt9eLL76offv2yd/fXyNHjrQcY37ymC39+/eXl5eX0tPTtX37dinf2OrSpYsaNGhQKt8JcITq1aurVatWatWqldq3b68hQ4Zo8eLFevfdd3Xx4kX9/ve/17p16wod5+XlZfUx9+3bt9fs2bOlvFlOX3/9td01MaZQUZXmdZTyXT+5u7tr8ODB91Uj54PlF6EUkMfb27vA66eeeqrYtt26dbPcEmFeeNIWczq/ZcsWZWZmWlL6AQMGyNPT8wEqB2BNw4YN9eSTT0p5JzUJCQk6fPiwxCxFVALu7u76n//5H82dO1dt2rQpcDtSnTp1NHHiRK1cubLAreXmJ/HZ4unpqf79+0t5YyszM1Nbt26VGFuowIYPH66BAwcqJydHc+bMua8nhg0ZMsRyXmltPbd7MaZQUZXmdVRERIRlvaq+ffuqevXq91Uj54PlF6EUkMfd3V21a9e2vK5fv36xbT08PFSrVi1JKnKNjqKYE/iUlBStWrXKsoglPwSB0pc/FF6zZo1ycnIKzGAEHmbOzs564YUXtH79eh07dkzbtm3Tnj17tG/fPk2dOlUeHh4FnriXf4F0W8xj6+DBg/rmm2+UkpJS4MIaqIjMt/Gkp6dr7969JT7e1dXVsn5NQkJCiY5lTKEiKs3rqAdZ4PxenA+WT4RSQD4tW7a0bN/7ON57ZWdnS3knHvbw9vZWUFCQJOlvf/ubsrOzVb9+fQUGBj5QzQBsGzhwoDw8PHTr1i19+umnUt7aHfY+Thh4WHh7e+vRRx9VvXr15Oz8f6eB2dnZOnv2rCSpcePGBS4sbOnatavq1aun7OxsffDBB1LeBf29fzUHKpL8Y+Dy5cv31cf9/n5hTKGiKo3rKJPJpE2bNkl5s3uffvrpB6qR88HyiVAKyKdz586W7YsXLxbbLjU1VUlJSZKkevXq2d2/eQHLzMxMKW8BdPNFAYDSU61aNfXt21fKN/7M4xGo7A4fPmy5Ramka3XkX3zZPLaYAYyKLv/sJi8vrxIfn5WVZXnsvK+vb4mOZUyhoiqN66jdu3dbfj8NHTrU7skAxeF8sHziahjIJ//UaPMCk0XZvn27Zf2NTp062d3/U089pQYNGsjd3V3u7u6cZAAGGj58uGXsPf744yW6RQl4WOXm5mrRokWSJDc3N73wwgsl7mPYsGGWsdWgQQP16NGjFCoFjLNlyxbLdqtWrUp8/ObNm5WSkiLdc6FuL8YUKqLSuI7Kf+vec88955A6OR8sfx4sagQeMq1bt1bPnj21Z88ebdq0SSNGjFC3bt0KtElMTNRHH30k5Z3Ajxgxwu7+XVxc9OOPPzq8bgC29e7d2+4HEwAPi6SkJFWtWrXQk5GUd/vE3LlzdeLECUnShAkT1Lhx4xJ/RqtWrRhbqBDWrVunIUOGyMPDo9g2y5Yt0+7duyVJjRo1siyMLEm3bt3Szz//bHXphYiICM2ZM0fKu4Vv9OjRJa6TMYWKyNHXUcnJyZbrplatWqlNmzYOqZPzwfKHUAq4x5///GedOnVKt2/f1u9+9zuNGzdOvXr1koeHhyIiIhQaGqqrV69KkiZPnlyi2/eAh9GxY8f022+/WV6bp2RL0oULFwo9Uvv5558vsp9725nXuJGkvXv3Kj4+3vK6SZMmBS4UgIeRI8bW4cOHNWfOHA0ePNjySPnMzExFR0fr22+/tYyznj17auLEiaX6fYCytnjxYr377rvq37+/OnXqpMaNG6tq1apKTU1VTEyMwsPDLSGtm5ub5syZIxcXF8vxKSkpCgkJkZ+fn4KCgtSuXTv5+PjIxcVFV65c0a5duxQWFiaTySRJevXVV+Xv719m3xcwmiOvozZt2mQZS46aJYXyiVAKuEezZs30ySefaPLkybp+/bpCQ0MVGhpaoI2Tk5MmTpyo8ePHl1mdQHmxZs0afffdd0XuO3HihOUE36y4UOqtt94q9jM+++yzAq+fe+45Qik89Bw1tq5fv67ly5dr+fLlhfY5OTnp+eef19tvv13kbCrgYZOcnKxvv/1W3377bbFt6tevr/nz56t79+5F7o+OjlZ0dHSxx7u4uOj3v/+9Jk2a5JCagYrCkddRYWFhUt54Mq+zhocToRRQhCeffFIbN27U119/rR07dujSpUsymUzy8fFRly5d9NJLL6lt27ZlXSYAAFY9+eST+tOf/qRDhw4pNjZWN27ckJOTk3x9fRUYGKgRI0aoQ4cOZV0mYIjPP/9cu3fv1okTJ3ThwgXduHFDycnJ8vDwUJ06ddSmTRv17t1bgwYNkqenZ6HjfX199fHHH+vUqVOKiIhQQkKCkpKSdPfuXXl7e6tZs2bq0qWLXnjhBTVq1KhMviNQ1hxxHRUXF6fTp09Lkrp37y4fHx+DqkdZcMo1rzIGAAAAAAAAGISn7wEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAAAAAAMMRSgEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAAAAAAMMRSgEAAAAAAMBwhFIAAAAAAAAwHKEUAAAAAAAADEcoBQAAUEEsWrRIfn5+8vPzK7XP6Nu3r/z8/DRr1qxS+wx7zJo1S35+furbt2+Z1gEAAEqPa1kXAAAAUJoOHz6skJAQy2svLy8dOHBAnp6eVo/LyMhQjx49lJqaanlv+fLlCgwMLNV6HxYmk0nff/+9tm7dqrNnz+rGjRvKysqSt7e36tevr5YtW6pjx4566qmn1KxZs7IuFwAAlAFCKQAAUKmkp6drx44dCg4Ottruhx9+KBBIwX6xsbGaPHmyYmJiCu1LTk5WcnKyfv75Z23cuFGSFBERIQ8PjzKoFAAAlCVCKQAAUGl4eHgoMzNTYWFhNkOpsLCwAsfAPklJSRo3bpyuXbsmSerSpYueffZZtWjRQp6enrp165bOnTunw4cPa+/evcrIyCiynwULFmjBggUGVw8AAIxEKAUAACqNvn376vvvv9eBAweUmJgoHx+fItvduHFD+/fvlyT169dPmzdvNrjSiis0NNQSSL3xxht68803C7Xp2rWrXnrpJaWmpmrdunVydmaZUwAAKiPOAAAAQKXRo0cP+fj4KDs7W5s2bSq23caNG5WVlSUfHx91797d0Borup07d0qS6tatq0mTJllt6+3trZCQELm5uRlUHQAAKE+YKQUAACoNFxcXDRkyRMuWLVNYWJhefvnlItuZb90bOnSoXFxc7Or77t27Wr16tbZs2aJz584pNTVVNWrUUNu2bTV06FAFBwfbnBF09epVffrpp9qzZ4+uXbumGjVqyN/fXyEhISUKx1JSUrRy5Urt2rVLcXFxSk1NVc2aNeXv76/hw4drwIABcnJysru/krh8+bIkqVGjRg80A2rWrFn67rvv1LBhQ0vQZda3b1/Fx8fb3Vd0dHSR71+4cEErVqzQwYMHdfnyZZlMJvn4+Khz584aM2aMAgIC7rt+AABgG6EUAACoVIYNG6Zly5YpKipK586d02OPPVZg/y+//KKffvrJ0vbs2bM2+7x06ZLGjx+v2NjYAu9fv35de/bs0Z49e7Rq1SotWbJENWvWLLKPY8eO6Xe/+12BxdUTExO1a9cu7dq1q8jb4Ipy8OBBTZkyRcnJyQXez99Xr169tHDhQlWtWtWuPkvCzc1Nd+/e1YULF5SVlSVX1/J5url06VItXLhQJpOpwPuXLl3SpUuXtH79er3++uuaPHlymdUIAMDDrnyeJQAAAJSStm3b6rHHHtO5c+cUFham6dOnF9hvniXVqlUrtWnTxmYolZaWppdfflkXL16UJAUFBWnEiBHy9fXVpUuXtGLFCh05ckTHjx/XxIkTtWLFikKzry5fvmwJpJydnTVq1CgNHDhQ3t7eio6O1meffaZFixbJ39/fai3Hjx/X+PHjZTKZVLduXY0dO1atW7eWr6+vrl27ps2bN2vDhg3avXu3Zs2apUWLFt3nv2Lx2rVrpyNHjigpKUnz58/XrFmz5O7u7tDPWLp0aaEwKb+4uDj98Y9/lMlk0iOPPFJo/+eff673339fkuTn56fRo0eradOmqlatmn799VetWLFCJ0+e1JIlS1SrVi2FhIQ4tH4AAPB/CKUAAEClM2zYMP3tb3/Txo0bNW3aNMutbLm5uQoPD7e0scfixYstgdTrr7+uKVOmWPb5+/trwIABmjFjhsLDw3Xy5EmtWrVK//7v/16gjwULFlhmSL3//vsaOnSoZV9AQIAGDhyoMWPG6MyZM8XWYTKZNGPGDJlMJj399NNatGiRPD09LfvbtWunPn36qHPnzpo9e7a2bdum/fv3q0ePHnb+q9ln7NixOnLkiCRpxYoV2rp1q/r166cnnnhCAQEBat68+QPfOtisWbNi992+fVtvvvmmTCaTvLy8tGTJkgL7f/nlF3300UdS3kLsb7zxRoF6/P39NWTIEM2cOVMbNmzQwoULNWzYMNWoUeOBagYAAIWx0DkAAKh0nn32WTk7O+vKlSs6fPiw5f3Dhw/rypUrcnZ2VnBwsM1+7t69qzVr1kiSHnvssSJvsXNyctLbb79tuW1vxYoVBfYnJiZqx44dkqQ+ffoUCKTMvL29NWfOHKu1bNq0SfHx8fLw8NB7771XIJDKb9SoUWrfvr0kad26dTa/Y0kNGDCgQNBz/fp1rVq1SjNnztTgwYPVpUsXvf766woPD7c62+l+ZGVlafLkyYqLi5OTk5PeffddtWnTpkCbL774QiaTSf7+/oUCKTNnZ2fNnj1b7u7uSk9P19atWx1aJwAA+D+EUgAAoNKpV6+eAgMDpXy36+Xf7tq1q+rVq2eznzNnzuj27duSpOeee67YRdG9vb01aNAgKW+mzrVr1yz7Dh8+rOzsbEnS888/X+xntW/fvtD6V/mZFwPv3LmzateubbXuJ598UpJ06tQpq+3u15tvvqlvv/1WAwYMkIeHR4F9t2/f1s6dOzV9+nQNHTpUkZGRDvvc+fPn68CBA5Ya+vfvX6jNrl27pLzwzNqMrerVq6tVq1aSpJMnTzqsRgAA8P+4fQ8AAFRKw4cP18GDB7Vt2zb913/9lyRZZsTYe+veuXPnLNsdOnSw2rZDhw7617/+ZTnO19dXkhQTE2NpY+tpbwEBAQU+Mz/zrX379u2Tn5+fXfVfv37drnb3o3379vr73/+ujIwMnTp1ShERETpz5oyOHj2qmzdvSnlrP4WEhGjVqlWWAOh+/etf/7LMQhs8eLAmTZpUqE18fLzlsz/44AN98MEHdvVdmv9OAABUZsyUAgAAldIzzzwjT09Ppaam6ocfftCOHTuUlpYmLy+vImfYFOXWrVuWbVuzk+rWrVvkcfmfklenTh27+7iXOWwpiYyMjBIfU1JVqlRR165dNWHCBP3973/Xvn379I9//EMNGjSQJKWnp2v+/PkP9BmHDh3S3Llzpbw1od55550i2924ceO++jfi3wkAgMqImVIAAKBSqlq1qoKCghQeHq6wsDDl5uZKeU/P8/LyKnF/D7p494My3wLYs2dPzZgxo0xrscbFxUVBQUFq3LixRowYIZPJpEOHDik5Odmy7lZJXLhwQZMnT1ZWVpZ8fHy0ZMkSValSpci2OTk5lu1JkyZp4MCBdn1GcetzAQCAB0MoBQAAKq3hw4crPDxc+/fvL/CevfI/ke3GjRtWnwqX/xaw/Mfd24d5BpGtPu5Vs2ZNXbt2TSaT6YFvhTOCn5+fOnTooGPHjik3N1e//fZbiUOplJQUTZw4UcnJyfLw8NCSJUusrgWWv39XV9cK8e8EAMDDjNv3AABApdWtWzf5+PgoKytLWVlZ8vX1Vbdu3ew+Pv/C46dPn7baNiIiosjj8gcjthb9Nq8bVZS2bdta2ty9e9dG5eWDeV0t3cdMs+zsbE2dOlWxsbFS3iLn5qcKFqdx48aqVq2aJOnEiRP3VTMAAHAcQikAAFBpubi4aNiwYXJ3d5e7u7uGDRsmZ2f7T4/8/f1VvXp1SdL69esL3B6WX2pqqr7//ntJUsuWLQuEMYGBgZan9n333XfFflZERESBRdHv1bdvXylv9tC6devs/g6OZr4N0p52UVFRUl4g1bBhwxJ9zoIFC7R3715J0sSJEzV06FCbx7i4uKhXr16SpP379+v8+fMl+kwAAOBYhFIAAKBSmzFjhiIjIxUZGanp06eX6Fh3d3eNHDlSynuK3pIlSwq1yc3N1Zw5c5SUlCRJGjNmTIH9vr6+6tevnyRp586d2rx5c6E+0tLSLE8ILM5zzz1nufXv3Xff1dGjR622P3bsmI4cOWLzO5bUmDFjtGHDBpuztRYvXqy4uDhJ0hNPPGFzofj8vv32Wy1fvlzKWwNsypQpdh87YcIEubi4KCcnR3/4wx909erVYttmZ2drw4YNVtsAAID7x5pSAAAAD2DSpEnavn27Ll68qEWLFikmJkbPP/+8fHx8dOnSJX399deW8Ofxxx/Xiy++WKiPmTNnav/+/UpLS9P06dN19OhRDRgwQN7e3oqOjlZoaKji4uLk7+9f7C187u7u+uijj/TSSy8pPT1d48aN0+DBgxUUFKRGjRopJydHiYmJ+umnn7R9+3bFxMRo9uzZ6tKli0P/Pc6fP68ZM2Zo3rx56tevnzp16qQmTZqoWrVqSktLU0xMjDZs2GC5fc7NzU0zZ860u//Y2Fj993//tyTJx8dH48eP17lz56wek/8WST8/P/3pT3/SO++8o19++UVDhw7VqFGj1LVrV9WtW1eZmZmKj4/XqVOntGXLFiUmJio8PFz169e/738TAABQNEIpAACAB+Dt7a1ly5Zp/Pjxio2N1datW7V169ZC7Z544gl98sknllv18mvUqJE++eQTvf7660pLS9PKlSu1cuXKAm0mTZokJycnq+tKdezYUV999ZWmTJmiK1euKDw8XOHh4VZrd7T69esrOTlZycnJWrt2rdauXVtsWx8fH73zzjvq0KGD3f0nJibKZDJZtosK+e4VHR1d4PXLL78sLy8vzZ8/XykpKVq6dKmWLl1a5LFubm7y8PCwuz4AAGA/QikAAIAH1KhRI4WFhWn16tXasmWLYmJilJaWpho1aqhNmzYKDg5WcHCw1fWqAgMDtWnTJn366afas2ePrl27pho1asjf319jx47V008/rUWLFtmspWPHjtq2bZvWrVunXbt2KSoqSklJSXJ2dlbt2rXVokULde7cWf3791fz5s0d/C8hhYWFKSoqSvv379fJkyd1/vx5JSYm6s6dO6pSpYrq1q2rVq1aqXfv3ho0aFCpBGP2GDVqlPr27atvvvlG+/fv16+//qqUlBS5u7vL19dXfn5+6t69u/r371+iWwsBAID9nHLtXY0SAAAAAAAAcBAWOgcAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGI5QCgAAAAAAAIYjlAIAAAAAAIDhCKUAAAAAAABgOEIpAAAAAAAAGO5/Ac34/vU5KJ5FAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Sort by size and then by LTR/RTL to group them together\n", + "df_sorted_pairs = df.sort_values(by=['params', 'Type'])\n", + "\n", + "# Plot configuration\n", + "fig, axes = plt.subplots(figsize=(12, 8))\n", + "sns.set_style(\"whitegrid\")\n", + "\n", + "# Create bar plot with LTR and RTL next to each other, no error bars (ci=None)\n", + "sns.barplot(x='size', y='val_loss_exp', hue='Type', data=df_sorted_pairs, dodge=True, palette=\"Set2\", ci=None)\n", + "\n", + "# Adjustments to the plot\n", + "# plt.xticks(rotation=45)\n", + "plt.title(\"Perplexity vs Model Size, From Scratch\", fontsize=20)\n", + "plt.xlabel(\"Model Size\", fontsize=20)\n", + "plt.ylabel(\"Test Perplexity\", fontsize=20)\n", + "# plt.legend(title=\"Model Type\",fontsize=20)\n", + "plt.legend(title=\"\",fontsize=20)\n", + "plt.tick_params(axis='both', labelsize=20)\n", + "\n", + "# Display the updated plot\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "execution_state": "idle", + "id": "bb25f31d-91b1-4bd5-be03-36a63f1e857e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(0.0, 122.75062123923252)" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "axes.get_ylim()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "execution_state": "idle", + "id": "371bccdf-d3c1-4699-9c22-1e2c0b8cfd42", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnYAAAHWCAYAAAD6oMSKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABfRUlEQVR4nO3dd1gUV/828HtAEBAFQbDHEhRRsZcoxoK9AWIvEGM39sdYSGKKmqgxedTYTTQosStYnsSuaGwoVhAVsSCgCNIUAWnz/vFj5136LrOwuNyf6/K6ltmZOd8FWe6dM+ccQRRFEURERET0wdPTdgFEREREpBkMdkREREQ6gsGOiIiISEcw2BERERHpCAY7IiIiIh3BYEdERESkIxjsiIiIiHQEgx0RERGRjmCwIyIiItIRDHZU7GxtbWFra4u1a9dqrQZHR0fY2tpi4cKFuZ4LDw+XavT29tZKfURExaWg97/SwM/PT3oP9vPz03Y5H7xy2i6ASg8/Pz+4u7vn2q6vrw9TU1OYmpqievXqaNKkCVq3bo1u3brB0NBQK7WWpPy+LwomJiawtrZGs2bN4Orqig4dOuS7b3h4OLp3765W+927d8eGDRuybVu7di3WrVuXa19BEGBiYgIrKyvY29vD2dkZn376qaz28/Lw4UOV982vVgDQ09NDhQoVULt2bbRr1w7Dhw9H/fr1Zdf3obO1tQUAtGvXDl5eXtouR2csXLgQPj4+ubYLggBTU1PUqFEDrVu3xrBhw2BnZ6eVGonk4hU7KlRGRgYSEhIQEREBf39/bN++HTNnzkSXLl2wYcMGpKena6UuNzc32Nraws3NTSvtKyQlJeHZs2c4cuQIxo4diwULFiAjI0MrtYiiiHfv3uHZs2c4evQoJkyYgBkzZiA1NVUr9RQmMzMTb9++RVBQEDw9PeHk5ISdO3dquywqY0RRxNu3b/Hw4UPs2rULrq6uWLVqlbbLoixr166VruhR4XjFjvI0cuRIjBo1Svo6KSkJCQkJePjwIa5evYrLly8jNjYWa9aswblz57B582ZYWFjkeS51ru4Ul7Nnz2rkPDm/L6IoIiEhAbdv34anpydiYmJw6NAhVKtWDXPmzCnwXN27d8fs2bMLbdPU1LTA53/66SfY29sDWUEpMjISt27dgqenJ1JSUnDy5EksW7YM3333HapWrYqjR4/me66BAwcCAJo2bYply5YVWpu6lGtV1BsVFYULFy5gz549SEtLw+LFi1GvXj107NhR4+0TKWzduhXW1tZA1v/DmJgY+Pr6YteuXUhPT8emTZtgbW2N0aNHa7tUnde+fftS8XdCVzDYUZ4sLS3RsGHDXNu7dOmCSZMmISQkBPPmzUNQUBDu3r2LadOmYfv27TrfNZvf96Vdu3ZwdHSEq6sr3r9/Dy8vL0ybNq3A70elSpXyPJe6atWqle08jRo1QteuXdG7d28MHToU6enp2LdvH7744gtYWVmp1KaJiYlGaiusVkW9nTt3hp2dHb766isg648ugx0Vp7p166JWrVrZtjk4OKBDhw6YOnUqAGDdunUYMWIE9PX1tVQlkfrYFUtFYmNjg927d6Nx48YAgJs3b2LXrl3aLkurbGxs0LVrVwDAu3fv8OTJE63W07hxY/Tr1w8AkJ6ejmvXrmm1nsIMHjwYlStXBgAEBARouxwqoxwdHdGmTRsAQGxsLO7du6ftkojUwit2VGRGRkb4+eefMXDgQIiiiK1bt2L06NEwMDDItp/ivojp06djxowZuc7z5s0b7Ny5E76+vnjy5AmSkpJQsWJFWFhYoF69enBwcECvXr1QpUoVII8boK9du5br3ouaNWtm6351dHREREQEBg0ahOXLl2v8e6HcrkJpuK9N+erYy5cvtVqLKmrWrIm4uLgCv3e3b9/GuXPncPPmTTx58gQJCQkwNDREtWrV0LZtW7i5ucHGxqbAdp4+fYq//voLfn5+iIiIQFpaGszNzWFpaYnGjRvj008/RY8ePfK94hodHY2//voL//77L8LDw5GUlARLS0u0aNECw4cPL/arjUlJSfD19cWlS5cQGBiI8PBwpKSkoGLFirCxsUG3bt0wYsQIVKhQId9z5Py9vHz5Mnbs2IHAwEAkJCTA2toaXbp0weTJk1G1atV8zxMWFoZTp07h2rVrCA4OxuvXr4Gsq9vNmzeHq6srOnfunO/x3t7e8PDwAACcOXMGNWrUwP79++Hj44PHjx8jLS0NtWvXRr9+/TB27FgYGxvL+M6pplmzZvD39wcAvHjxAs2aNcv2fFF//sqDl5YtWwZXV1ecPHkS+/fvx/379xEbG4vWrVtLA2YU73WK97NXr15h27Zt8PX1RWRkJIyNjdGsWTOMGTOmwO+xqkJDQ7Fz505cuXIFL168QFpaGqysrNC2bVuMHj06220UCosWLcK+ffsAAKtXr0bfvn3zPPeZM2fwxRdfAABcXFywYsUK6TnlAWo7duxA+/btgRz/NxTyus/uzJkzSExMhLOzMwBg7ty5mDRpUoGv1cvLC0uXLgUA7N+/P9fP+EPGYEeyNGjQAA4ODrh48SKioqIQEBCAVq1aqXz848ePMXbsWERFRWXbHhcXh7i4ODx+/BinT59GZmYmxowZUwyvQLNevHghPa5Ro4ZWawGQLWSXK1f6f90V37/q1avn+Xxeb/QAkJaWhsePH+Px48fYv38/vv7663zvjTp27BjmzZuHtLS0bNujo6MRHR2NBw8ewNvbG0ePHs2zO/rIkSP47rvvkJSUlG17ZGQkjh8/juPHj2PIkCH44Ycfiu17Pnny5DyvwMbFxeH69eu4fv06du3ahS1btuDjjz8u9Hzr1q3LNR1ReHg4du7ciSNHjmDTpk3SVSxlYWFh6NGjR57nfPHiBV68eIFjx47ByckJy5YtK/T7kZKSgnHjxuHKlSvZtgcHByM4OBhnz57F9u3bYWJiUuhrkkO5zszMzGzPaernL4oi5s+fj8OHD6tUU0BAACZPnoyYmBhpW0pKCs6fP4/z58/j888/lzWdydatW7Fq1apcvxfh4eEIDw/HoUOHMHXqVMyaNSvb8x4eHrh27RqePXuG7777Di1btkS1atWy7RMdHY1vvvkGyPrwtmjRoiLXmZ9GjRrB3t4eAQEB8PHxKTTYKaa2atCggU6FOjDYkSZ06NABFy9eBAD4+/urFezmzZuHqKgoGBgYYOjQoejcuTOqVKkCURQRGRmJ27dv4/Tp09mOmTNnDsaNGwcPDw8EBgbmeaN/zquGJeHx48fw9fUFALRo0UK6wqhNyt3BOe8nKm18fHwQGxsLZA0syUtGRgbMzMzQvXt3tGnTBnXq1IGJiQmioqJw7949eHl5IS4uDkuWLEH9+vVzTT3z+vVrfPXVV0hLS4OlpSVGjx6NFi1aoHLlykhJScHz589x7do1nDlzJs/2//nnH8yfPx+iKKJ27doYM2YMPv74Y1hYWCAiIgIHDhzA+fPnceDAAZiamuYZQjUhPT0dDRs2hKOjI+zt7WFtbQ1RFBEREYHTp0/j2LFjCA8Px7Rp03D48GGUL18+33P5+voiMDAQ9erVw4QJE2Bra4vExEQcO3YM+/fvx9u3bzF58mT873//yxW4MzMzYWBggE6dOsHBwQE2NjYwMzNDQkICnj59il27duHRo0c4cuQIateujZkzZxb4ur755hvcuXMHgwYNQt++fVGlShW8fPkSf/zxB27duoW7d+9i48aNmDt3rsa+l3kJDg6WHisGWEDDP//t27fj4cOHaNOmDUaOHIm6devi7du3CA8Pz7VvcnIyZs2ahbdv32LSpEno0qULDA0NcefOHWzevBnR0dH4888/Ub16dXz22Wdqv94//vgDK1euBLKuiCnqqVixIp4+fYqdO3fi1q1b2LBhAypXrpxt+icTExOsXLkSI0eOREJCAhYsWABPT08IgiDt4+HhgdjYWOjr62PlypWFDghT6NGjB5o2bYpdu3Zh9+7dAJDn4C/FFeWhQ4ciICAAT548wa1bt9CyZcs8z/vgwQMEBQUBWbeA6BoGO5KtSZMm0uNnz56pfFxYWJh0/8rChQtzXZFr1qwZevXqhXnz5uHNmzfS9qpVq6Jq1arSp/biutE/LzExMdne9BXTJNy6dQvbt2+XusNU+YP+5s2bbOfKT61atYp0heLly5fSm2ClSpUKnF+vpISHh0v30SErGLx+/RoXLlyQ7tFs0KABxo0bl+fxnTt3xoABA3J1xzVu3Bhdu3aFu7s7Ro8ejYcPH2Lt2rW5XrOvr690pcXT0zPX/5tWrVrBxcUFKSkpudqOjY3Ft99+C1EUMXjwYCxevDjbFZkmTZqgV69eWLVqFTZt2oQdO3YU27x8y5YtQ926dXNtb968Ofr164chQ4Zg/PjxePr0KY4cOYKhQ4fme67AwEA0adIEXl5e2bpuO3TogFatWmHBggVITEzE8uXLsWbNmmzHWllZ4ezZs9nCj/LxI0eOxFdffQVvb2/8+eef+Pzzz1GxYsV8a7l16xZ+/vlnqUsNWd/Xzp07Y/DgwQgODsa+ffswa9asYrsa+uDBA/z7778AAGNjY6n7UdM//4cPH8LFxQXLly/PFoLyEhsbCwMDA/z5559o27attF3xHjls2DBERkZi9erVGDhwYL4zFOQlJCQEq1evBrK65adPn56tnqZNm6J///5YsGABjhw5glWrVsHZ2RlmZmbZ6pg+fTpWr16Nq1evYtu2bRg/fjyQ1eWp+H5OnDgRrVu3Vrm2SpUqoVKlSrC0tJS2FfRe379/fyxfvhxJSUnw9vbON9gdOHAAyLoAoPx/TVdw8ATJZm5uLj1WDmCFiY6Olh7n1c2jIAhCtjcRbdq9ezcGDhwo/XNycsLo0aPxyy+/ICYmBiNGjMD+/fvRokWLQs915syZbOfK7586AwkyMzPx4sULHD16FKNGjZJCzKxZswq836qkfPXVV9lem7OzM8aPH4/t27fDyMgIc+bMwa5du/L9w1S1atUC77GqWLGidFXoxo0biIuLy/a84v4vMzOzAv9AGBkZwcjIKNu23bt34+3bt6hatSq+//77fIPFjBkzULVqVWRmZqrczaauvEKdso4dO8LR0RHI+n9WmMWLF+f5/8PFxUW6d+v06dPZfmehNDl3fgRBwIIFC6Cvr4+kpCRcvny5wDp69eqV5x9aQ0NDqWs9Pj4eISEhhb4mdYiiiOjoaOzfvx9jx46V5qF0c3OTrnZq+udfqVIlLFq0qNBQpzB8+PBsoU6hatWqWLBgAZB172VeEzAXZNu2bUhLS0PTpk1zhToFPT09LFq0CIaGhkhKSsKJEydy7TNp0iQptK1atQoPHjxASEiIdCXQ3t4+z3usNcnU1BR9+vQBsq6u5vUBLTU1VfrA27VrV7VC8IeCwY5kU76a9O7dO5WPs7Kykh6r+2ZUGmVmZuKff/7B7t27S3TghLu7uzR5p52dHbp164Yvv/wSL168gLW1NZYuXfpB3J/49u1b7N+/H//884/KxyQlJSE8PByPHj2S7sNS7oZ/8OBBtv0V/+cSEhJydfEXRjEYp2vXrgVOY1OuXDkp2N+6dUutNooqNjYWz549k74HwcHB0h+snN+DnBo2bIimTZvm+7yiq0qVkdVpaWmIjIzE48ePpTqioqKkD3+F1aKYRzEvyj0DeXVXqqt79+7S702jRo3QqVMnfPPNN9KHga5du2brOtb0z79bt24qd0kCgKura77P9ezZE5UqVQKAQsNzTufOnQMA9O7du8CQqTw9U16vS19fHz///DNMTU2RlpaGuXPnYu7cuXj//j2MjY2xcuXKErnPV3F1OjExMc8Aeu7cOcTHxwM62g0LdsWSJiiHOXXeqGrXro02bdrA398fnp6euHjxInr16oV27dqhRYsWJTL6TV15jexNSUlBaGgoDh8+jO3bt2P79u0IDAzE1q1bC3wNxT1CF1ldl05OTsXahjqUR7wh60pJYmIiHjx4gO3bt+PUqVP47rvv8PTp03y7s2NjY+Hp6YkTJ04gNDQUoijm217OK3aOjo6oVKkS3rx5g+nTp0vzD7Zp0wZ2dnb5zleWkZEhhZK9e/di7969Kr1exRXC4nDjxg14eXnhypUr0h+qvOT8HuSU10hHZco3lgcHB6N///7Znk9LS8O+fftw+PBhBAUF5br5Xp1aCuq2Vu4ZSExMLPA8RWVgYAB7e3sMHz4czs7OUtApjp+/OqsoGBgYoFGjRgU+b2dnBz8/P5Vu71CIiIiQ7mv99ddf8euvv6p0XH6vq1atWvj2228xf/78bFdVPTw8UK9ePZXrkqNVq1awsbFBSEgIvL29c10BVgyasLKy0shI4tKIwY5kU36zVrfL9L///S9mzZqFW7duISQkBCEhIdiwYQMMDAzQvHlzDBgwAK6urgXe/K1tRkZGsLW1xfz581G3bl0sWrQIN27cwKZNmwpdfUITlFdzUAwA2Lt3L65du4YDBw7g9evX2LRpk8pdPiVJEARUrFgRbdu2Rdu2bTF37lz873//g6enJ7p06ZJr2ojAwECMHz++wCCj7P3799m+rly5MjZu3Ij//Oc/ePXqFfz8/KRFx01NTdGhQwcMHjwY3bp1y3ZcQkJCkZbOy6srSBMKWn9X3RqU71/Ki/IgoJzf9/j4eIwbN07lud5y/jxyytn9rUz5/2/OkapFobzyhJ6eHkxMTFClSpU8r8YVx89fcYVNFebm5oVOkqz4OSUkJKh8XuURtuoo6HU5OzvjwIED0tVdBwcHDB8+vEjtFNWQIUOwfPly+Pn5ITw8XBo4FhUVJd3v5+LiorMTTzPYkWyK0UUA1P5UVrVqVezZswdXrlzByZMncf36dYSEhCAtLQ3+/v7w9/fHtm3bsGXLlhL7xCfHkCFD8OuvvyI+Ph4HDx4skWCXczWHZs2aoX///tJN676+vti+fTvGjh1b7LXINWHCBPzvf/8DABw8eDBbsEtNTcXs2bMRHx8PAwMDjBkzBt27d0fdunVhZmYm/UFWnoIjr6t5bdq0walTp3DixAmcP38e/v7+iIyMRGJiIk6dOoVTp06hU6dOWLdunXTFVXnt36FDh2YbFViQ4hidfeXKFSnU1a5dG+PGjUPr1q1Ro0YNGBsbS91da9aswYYNGwo9n5zA/+OPP0qhrkePHhg8eDBsbW1haWmJ8uXLS+fu2rUrXr58WeDV1ZKW18oT+SmOn39pCBXKAXnatGnS/WmFKagnIjAwMFtX7YMHDxATE1PoBwhNcnZ2xq+//oq0tDQcOnQI06dPBwAcOnRI+lnqajcsGOxIE5Tv6VBnxJOyDh06SCMY4+LicOXKFezduxdXr17F8+fPMWfOHBw6dEhjNRcXPT091KlTB/Hx8YiOjkZcXFy2UaAlRRAEfPvtt7h69SpevHiBdevWwcXFJVt3Vmmk3BWXs0vp6tWrCAsLAwB89913+Y70VOVqXvny5eHk5CR1U4eFheH8+fPw8vLCs2fPcPHiRaxatUpa4kz5SrQoiiU2CjsvislgzczMsG/fvnxv/lb1yk1h3cXKz+fsDj127BiQdW/cL7/8ku851LmKVBpp++cfHx+PjIyMAsOg8sAgVSn/PMuVKyf7daWkpEhzRJqYmCA5ORkxMTH4+uuvsWnTJlnnVoeFhQW6d++O48ePw8fHB9OmTYMgCNK93K1atfogLhQUFQdPkCzBwcHSZKLVq1cv8CZsVVWuXBn9+vXD9u3bpZF99+/fV2sqFW1S7rJR/qRf0oyNjaWZ3t++fYs//vhDa7WoSvl7l7PrS/menfxmt0fWFQN1KeYkO3jwoDS5qiK0IGtUZoMGDYCs5fO0SfF9aN++fYEj+lT9PhQ26lr5ecX3AFlTGynup1MsXZeXx48f55rM90Oj7Z9/WlpagQNP0tPTpefVCWe1a9eWpp/RxOtavny5NHfmDz/8gM8//xzIGrCgmIeuKIpyVVnxwS88PBx+fn7SSjXQ8at1YLAjOVJSUrBgwQKpe2XcuHEaH/WkPA9ZzhuvFffdlYaluxSSk5Px+PFjIOt+IW1crVPm4uIirYCxa9cule9N0xblMJJzMlzloJecnJzn8ZmZmdi/f3+R2zc1NZXuV8xr4AWyJn1W3KejDYrvQ0FhKSgoCHfu3FHpfMHBwdlup8jp4MGDQFbXYbt27aTtyh9a8vt5AMCePXtUqqO00/bPv6CZA06dOiVdFVVnvkp9fX106dIFAHDp0iXpvasozp8/L4W3AQMGwMnJCXPmzJEGfaxYsQJPnz4t0rmV73tU9f2+Y8eO0hKP3t7e0qAJExOTAj8Y6gIGOyqSkJAQjBo1SvqD0K5dO4wcOVKtc9y/fx/379/P93lRFKVuXkEQsq3DCqWpK8LCwkrNvTtr166Vbizu1KmT1u+jMTAwwIQJE4Cs0cs7duzQaj0FSU1NzTYBruIPjoLy3G35/ZH79ddfC7yR/99//821fJ2yt2/f4u7du0AeK3W4u7tLU/t4eHjg0aNHBb4eX1/fQqf3KIo6deoAWVdYQkNDcz0fGxuL+fPnq3XORYsW5RkUjx49ivPnzwNZ99Apz1n30UcfSVdSfHx88vwdPHv2LHbu3KlWLaWVtn/+e/bskdavVRYdHY2ff/4ZyLpKP2jQILXOO2nSJOjr6yMzMxMzZ85EZGRkvvtmZGTgyJEjufaJjY2VbluoXr06vvvuOyArkP3yyy8oX748kpOTMW/evCINQlH+f/f8+XOVjtHT05OmiDl58qQ0jVKfPn1KxZyexYn32FGecq6wkJycjISEBDx8+BBXr17FpUuXpDfyFi1aYM2aNWrfKH7//n14eHjA3t4e3bp1Q5MmTVClShWkp6cjPDwc3t7euHTpEpD1aTnnRKitWrWCt7c3YmJisGzZMjg5OUndCuXKlcsVBDUh5/cFWSP9QkNDcejQIemTfPny5QtdPknVlSf09fVVWu8zP0OGDMHGjRulhcvHjRun1rQ0mpRz5Qlk3at1//597N69W/pjWadOnVwj6Tp16gRLS0vExMRg9erVCA8PR8+ePVG5cmU8f/4c+/btw5UrV9CqVat8u5X+/vtvTJ06FR07doSDgwMaNmwIMzMzvHv3DsHBwdi5cydevXoFABgxYkS2Y6tUqYIVK1Zg5syZiI6OxuDBgzFo0CB07twZ1apVQ3p6OiIjI3H37l2cOHECYWFh2LRpU4HTVBSFi4sLzp07h6SkJIwZMwaTJk2S5ni7desW/vzzT7x+/RotW7ZUaR69pk2bIjAwEIMHD8bEiRPRsGFDvH37FidOnJCm9ahQoUKusFi5cmV06dIFvr6++PfffzFu3DiMHDkSNWrUQExMDE6ePAkfHx/Url0bb968kabV+FBp8+dvYWEBY2NjjBs3DmPHjkXnzp1haGiIgIAAbNq0SfqwMmvWLLUHKShG9C9btgwhISEYMGAAhg0bhk8++QRVqlTB+/fvERERgdu3b+P48eOIjo7G0aNHs60H+/XXX+P169fQ09PDihUrso34bdCgAebOnYuffvoJAQEBWL9+fa71ZgujvILEsmXLMGXKFFhZWUkfLGrWrJlnb9GQIUOwfv36bFeUdb0bFgx2lJ/du3cXek+EhYUFPvvsM0yYMEFWF2xAQECB9/m0bNkSP/74Y67t/fr1w+bNmxEWFibNH6dQs2ZNaUJRTVL1+7Jy5cpC56k6c+aMSqsCVKxYMc9P6qoqX748xo4di5UrVyIhIQE7d+7E5MmTi3w+ORSf6gvSqFEjrF+/PtfUFyYmJlixYgWmTZuG9+/f5zmfWLt27fDtt99iwIAB+Z4/LS1NWjg9PyNGjMhz5GOvXr2wYcMGeHh4ID4+Hnv27Mm3q1FPT69Y5mLs06cPXF1d4e3tjaioKCxdujTb8/r6+vDw8MCbN29UCnZdu3ZF165dsW7dujznDjQ1NcXGjRvzHEH6/fffY9SoUXjx4gUuX76ca3LcGjVqYP369YUuyP6h0NbP39jYGGvWrMHEiROxefNmbN68Odc+bm5u0j1t6ho7dixMTEzw008/4e3bt9i6dSu2bt2a574GBgbZpp/au3ev9F77+eefZ5unUsHd3R0XLlzAxYsXsXnzZnTu3Dnf5b7yUqdOHfTt2xfHjh3DxYsXpbXJFc6cOZPn/89q1aqhU6dOuHDhApB11b+gVY50BYMdFUpPTw8VKlRAxYoVUaNGDTRp0gRt2rQpdAb2wgwYMACWlpa4fPkyAgIC8OrVK8TExCA9PR2WlpZo3Lgx+vXrh/79+0NPL/ddAxUqVMCePXuwefNmXLp0CS9evCjwXp/iYmBgAHNzc9jY2KBLly5wdXUtNUugKYwcORK///474uPj4enpCXd391IzAbSxsTEsLCzQpEkT9O7dG3369Mn3g8Knn36KgwcPYsuWLbh69Sri4uJQsWJF2NjYYODAgRgyZAhevHiRb1seHh7o2LEjrl69iocPHyI6OlpanLxatWpo2bIlhgwZUuCbv6OjI86cOYN9+/bh/PnzCAkJQUJCAvT19VGlShU0aNAAn3zyCXr37p3rPkFVKN9DlN/PaNmyZfjkk0+wb98+3L9/H2lpabCyskKbNm0wZswYNGvWDGvXrlW5zRkzZqBFixb466+/EBgYiISEBFhbW6NLly6YPHlytqszyqpXrw5vb2/8/vvvOHPmDF68eIHy5cujZs2a6NGjB9zd3Uvd74Jcxf3zz4+9vT18fHywdetWnD9/Hq9evZLWsnVzc8t164K6hg0bBkdHR+zZsweXLl3C06dP8fbtWxgaGsLa2hq2trbo2LEjevXqJQ3aefbsmTTJup2dHWbPnp3nuQVBwE8//QQnJyfEx8dj3rx5OHz4sFpdoitXrkTTpk1x4sQJPH36FO/evVNpPkNnZ2cp2JWFq3UAIIil5eYkIiLC69ev4eDgAKgwjYgciivKea2mQqXDwoUL4ePjU2w9EGXBqlWrsGnTJujr68PX17fAtY11BQdPEBGVIso35Re0xBYRFSwjI0Oa/7Rz585lItSBwY6IqHRRTMsANaeuIKLsjh49Ko3gzTkYSpfxHjsiIi1SjKqOiYnB0aNHceTIEQBAkyZN1LrBnIiA0NBQpKenIyAgAMuWLQOyBmTJvQfxQ8JgR0SkRU+ePIGLi0u2bZaWllixYoXWaiL6UPXq1Svb1wYGBvj+++9lrYn8oWGwIyIqBUxMTFCjRg107twZ48ePR5UqVbRdEtEHy8zMDI0bN8bMmTPL3JVvjoolIiIi0hG8YlfMMjMzkZ6eDj09vTJ1KZiIiIg0QxRFZGZmoly5cnnO66qMwa6YKW7iJCIiIpLD3t6+0IUBGOyKmSJZ29vba31BeNKujIwMBAQE8P8CEamE7xmkoPi/UNjVOjDYFT9F96u+vj5/MQng/wUiUhPfM0hBlVu6OEExERERkY5gsCMiIiLSEQx2RERERDqCwY6IiIhIRzDYEREREekIBjsiIiIiHcFgR0RERKQjGOyIiIiIdASDHREREZGOYLAjIiIi0hEMdkREREQ6gsGOiIiISEcw2BERERHpCAY7IiKiUsrY2FjbJdAHhsGOiKgMETMztV0CqUhfXx+NGzeGvr6+tkshFZSW361y2i6AiIhKjqCnh4TTu5ERF6XtUoh0hn5la5j1GKntMgAGO6KSxW4VKg0y4qKQ/jpC22UQUTFgV6wOyBRLx+VfKhi7VT4s/L0iog8Rr9jpAD1BDzvv/ItXiQnaLoVIJ1Q1NcPo5p9quwwiIrUx2OmIV4kJiHgTq+0yiIiISIvYFUtERESkIxjsiIiIiHQEgx0RERGRjmCwIyIiItIRDHZEREREOoLBjoiIiEhHMNgRERER6QgGOyIiIiIdwWBHREREpCMY7IiIiIh0BIMdERERkY6QFewWL16MoKAgzVVDREREREUmK9jt2rULgwcPhouLC7y8vBAfH6+5yoiIiIhILbKCXbly5SCKIh48eICffvoJnTt3xuzZs3HhwgWIoqi5KomIiIioULKC3cWLF/HVV1/Bzs4OoigiNTUVJ06cwOTJk9GtWzesXr0az58/11y1RERERJQvWcHO3Nwc7u7u8PHxgY+PD8aMGQMzMzOIoojIyEhs3rwZvXv3hpubGw4dOoSUlBTNVU5ERERE2WhsVKydnR2++eYb/Pvvv1izZg06d+4MPT09iKIIf39/eHh4oFOnTvj2229x+/ZtTTVLRERERFnKafqEBgYG6N27N3r37o2oqCgcOnQI3t7eePbsGRITE7F//37s378f9evXx+DBg+Hs7AxLS0tNl0FERERU5hTrPHbW1taYNGkSjh8/jt9//x1VqlQBAIiiiCdPnmDlypXo0qUL5syZg/v37xdnKUREREQ6T+NX7HK6fv06vL29cfz4caSkpEijZY2NjZGUlIT09HQcP34cJ06cwKhRo/DVV19BT4/zJhMRERGpq1iC3cuXL6UBFeHh4UDWVTpBEODg4IAhQ4agR48eePHiBQ4cOIB9+/YhISEBO3fuxEcffQR3d/fiKIuIiIhIp2ks2KWmpuLkyZPw9vbG1atXIYqidHWuWrVqcHV1xeDBg1GzZk3pmDp16mDu3LmYNGkSpk2bhmvXrmHfvn0MdkRERERFIDvY3b17FwcPHsSxY8fw9u1bIOvqXLly5dCtWzcMGTIEnTt3hiAI+Z6jYsWKmDlzJsaMGcN574iIiIiKSFaw69+/P548eQJkhTkAqFu3LoYMGYJBgwapNdrV2toaAJCWlqbyMTExMbh79y7u3r2LgIAABAQESMuaDRo0CMuXLy/0HN7e3vDw8FCpvWXLlsHV1VXl+oiIiIhKkqxg9/jxYwCAkZERevfujaFDh6JNmzZFOpepqSlcXFwKvLKXU8eOHYvUFhEREZEukhXsGjdujCFDhsDJyQmmpqayCrGwsFDpClt+atSogfr16+PixYtFPsfWrVulK4d5qVatWpHPTURERFTcZAU7b29vzVVSBNOmTYO9vT3s7e1RpUoVhIeHo3v37kU+X926dVGrVi2N1khERERUUmQFu3Xr1gEARo0aBQsLC5WOSUhIgJeXFwBg+vTpcprHzJkzZR1PREREpEtkBztBENCnTx+1gp3iOLnBjoiIiIj+Py7xQERERKQjin1JsZwU05mUK1fiTRfKw8MDT58+RXx8PCpUqIA6deqgQ4cOGDVqFKpWrart8oiIiIgKVOJX7B48eABkjYItba5du4bo6GikpaUhPj4ed+7cwaZNm9CzZ0/s2bNH2+URERERFUity2aHDh3Kc/uZM2cQGBhY4LGpqal4/vw5Dh48CEEQYG9vr16lxah27dro2bMnWrZsKU1pEh4ejhMnTuDEiRN4//49vvvuOwiCgOHDhxepjYyMDA1X/f/p6+sX27mJyrLi/L3VFr5fEBWf4nrPUOe8agW7hQsX5ppAWBRFrF69WuVziKIIPT29UrMebM+ePTFo0KBcr6tZs2bo168fzp07hxkzZiAtLQ3Lli2Do6MjrKys1G4nICBAg1X/f8bGxmjcuHGxnJuorHv48CGSk5O1XYbG8P2CqHiVhvcMtW90UywdVti2vBgYGMDe3h6TJ09Gu3bt1G26WFSsWLHA57t164YvvvgCa9asQXJyMg4cOICpU6eq3Y69vT0/KRN9YGxtbbVdAhF9QIrrPSMjI0PlC0RqBbszZ85Ij0VRRI8ePSAIArZu3Yo6derke5wgCChfvjzMzc0/yHAzfPhw/PbbbxBFEdevXy9SsNPX1/8gXztRWcbfWSJSR2l4z1Ar2NWsWTPP7dbW1vk+pwssLS1hbm6OuLg4vHr1StvlEBEREeVJ1pwjihGuZUHOe/CIiIiIShtOUKyC2NhYxMXFAVlXJ4mIiIhKIwY7Fezdu1caINK2bVttl0NERESUJ5W6Yj08PICs7siffvop1/aiyHkubQgPD8ebN28KHP5/7tw5rF+/HgBgZGSEwYMHl2CFRERERKpTKdj5+PhI95gphzHl7eoQRVEjwc7f3x/Pnz+XvlZ0lwJAaGgovL29s+3v6uqa7euIiAi4u7ujZcuW6NatGxo1aiStiBEWFiZNUKy4Wjd//nwuLUZERESllkrBrkaNGmptLykHDhyAj49Pns/dvHkTN2/ezLYtZ7BTuHXrFm7dupVvO8bGxvDw8CjyqhNEREREJUGlYHf27Fm1tn8omjRpgpUrV+L27dsIDAxEdHQ04uLikJ6eDjMzM9jY2KBDhw4YOnQoLC0ttV0uERERUYFkTXeibcuXL8fy5cuLfLypqSmcnJzg5OSk0bqIiIiItIGjYomIiIh0hKxgFxsbW+RjL1y4IKdpIiIiIspBVrAbOHAgzp8/r9Yxqamp+OGHHzB58mQ5TRMRERFRDrKCXUxMDKZMmYIffvgB79+/L3T/oKAguLi4YM+ePXKaJSIiIqI8yAp2NWrUgCiK2LNnDwYNGoSgoKA89xNFEZs3b8bw4cPx9OlTiKKI/v37y2maiIiIiHKQFeyOHDmCgQMHQhRFPH36FMOHD8eWLVukCX2RNQnwmDFjsHr1aqSlpcHU1BQrV67EL7/8oon6iYiIiCiLrGCnCGm//vorKlasiLS0NKxatQru7u548eIFDh06BGdnZ9y8eROiKKJt27ZSGCQiIiIizdLIPHb9+/dHq1atMH/+fFy/fh3+/v7o3bs30tPTIYoiypUrh1mzZmHChAlFWoKMiIiIiAqnsXnsqlevjh07dqBfv34QRVEKdRUrVsS+ffswceJEhjoiIiKiYqSxYJeeno7//ve/OHHiBARBkO6zS0xMxKpVq/D69WtNNUVEREREedBIsHv69CmGDRuGP/74AxkZGbC0tMSyZctgb28PURRx8eJFDBw4EKdPn9ZEc0RERESUB9nBbvfu3XB1dcX9+/chiiIcHR1x5MgRDBo0CHv27MHUqVOhp6eHuLg4zJgxA9988w2Sk5M1Uz0RERERSWQFuylTpmDx4sVITk6GkZERFi9ejA0bNsDCwgIAoK+vj1mzZuGvv/5CrVq1IIoiDh48CBcXF9y9e1dTr4GIiIiI5AY7X19fiKKIpk2bwsfHB8OGDctzv5YtW0pX8URRRGhoKEaNGiWnaSIiIiLKQVaw09PTw5QpU7Bnzx7UrVu3wH1NTEywbNkyrF27FmZmZsjIyJDTNBERERHlIGseOy8vL7Ru3VqtY3r27InmzZvjq6++ktM0EREREeUg64qduqFOwdraGn/88YecpomIiIgoB43NY6eQmZmJ2NhYvHjxgt2tRERERCVII0uKZWRkwNvbGz4+PggICEB6ejoEQcCRI0dgY2Mj7Xfu3Dlcv34dpqam+OKLLzTRNBERERFlkR3sYmJiMG3aNNy5c0dabSI/NWvWxNSpUyEIArp16wY7Ozu5zRMRERFRFlldsRkZGZgyZQpu374NQRDQt29fLFq0KN/9GzZsiObNmwMATp06JadpIiIiIspBVrBTdL2WK1cOmzdvxqpVqzB69OgCj3F0dIQoirhx44acpomIiIgoB1nB7u+//4YgCBgxYgQ+/fRTlY5RdL8+ffpUTtNERERElIOsYPfw4UMg6yqcqiwtLQEA8fHxcpomIiIiohxkBbs3b94AAMzNzVU+RjEFir6+vpymiYiIiCgHWcFOEehevnyp8jGhoaEAgMqVK8tpmoiIiIhykBXsFHPUBQQEqHzMP//8A0EQYG9vL6dpIiIiIspBVrDr0aMHRFHEzp07kZCQUOj+x48fx7lz5wAAvXv3ltM0EREREeUgK9gNGzYMNWrUQGJiIsaNG4eQkJA894uJicGqVavw5ZdfQhAENGjQAH379pXTNBERERHlIGvlCUNDQ2zYsAFubm64d+8eBg4ciHr16knPz5s3D0lJSQgLC4MoihBFEebm5li7di0EQdBE/URERESURdYVOwBo1KgRDhw4gBYtWkAURTx58kR67sGDBwgNDUVmZiZEUUSzZs2wf/9+1KlTR26zRERERJSD7LViAaBOnTrYs2cP/P39cfbsWQQGBiI2NhYZGRkwNzdH48aN4ejoCAcHB000R0RERER50EiwU2jTpg3atGmjyVMSERERkYpkd8USERERUenAYEdERESkIxjsiIiIiHSESvfY2dnZabxhQRAQFBSk8fMSERERlVUqBTtRFIu/EiIiIiKSRaVgN3369OKvhIiIiIhkYbAjIiIi0hEcPEFERESkIxjsiIiIiHSERleeAICYmBgEBwcjPj4eAGBubo4GDRqgSpUqmm6KiIiIiJRoJNiJoog9e/Zg165dCAkJyXMfGxsbjBw5EiNGjICeHi8UEhEREWma7GAXExODKVOmIDAwEChgapSQkBAsWbIEBw8exKZNm2BlZSW3aSIiIiJSIivYpaam4rPPPsPjx48hiiIsLCzQt29f2NvbS12vr1+/RmBgII4dO4aYmBjcu3cPn3/+Oby9vWFoaKip10FERERU5skKdp6enggJCYEgCBgyZAi++uormJiY5NrPxcUFc+fOxbJly7Bv3z48fvwYnp6emDRpkpzmiYiIiEiJrJvd/v77bwiCgI4dO2Lp0qV5hjoFY2NjLF68GA4ODhBFEX///becpomIiIgoB1nB7vnz5wCAUaNGqXyMYl/FsURERESkGbKCneIeuerVq6t8jGJf3l9HREREpFmygl29evUAAC9fvlT5GMW+imOJiIiISDNkBTtXV1dpDjtV7dmzB4IgwMXFRU7TRERERJSDrGA3dOhQdOrUCRcvXsT333+P9+/f57tvamoqFi9ejH///RcODg4YPny4nKaJiIiIKAdZ0534+/vj888/R0JCAvbu3YvTp09L89hZWlpCEAS8fv0aAQEBOH78OF6/fg17e3uMGzcO/v7++Z63bdu2csoiIiIiKpNkBTs3NzcIgiB9HRMTg7/++qvAYwIDAzF+/Ph8nxcEAUFBQXLKIiIiIiqTZC8plt8SYkRERERUsmQFux07dmiuEiIiIiKSRVawa9euneYqISIiIiJZZAW7Fy9eAABMTExgbm6uqZqIiIiIqAhkTXfi6OiI7t27c91XIiIiolJAVrAzMjICANjb22uqHiIiIiIqIlnBrmrVqgCAzMxMTdVDREREREUkK9g5ODgAAG7cuKGpeoiIiIioiGQFO3d3dxgZGWHbtm149eqV5qoiIiIiIrXJCnZ169bFL7/8guTkZAwbNgxHjx5Famqq5qojIiIiIpXJmu7E3d0dAGBhYYHw8HDMnz8fX3/9NerUqQMzMzPo6eWfGwVBwPbt2+U0T0RERERKZAW7a9euZVsrVhRFpKam4tGjR/keIwgCRFHMdhwRERERyScr2LVt21ZzlRARERGRLLKCnZeXl+YqISIiIiJZZA2eICIiIqLSg8GOiIiISEfI6orNS2RkJKKjo5GSkgJ7e3tp2TEiIiIiKl4aCXaJiYn4448/4OPjg6ioKGn70aNHYWNjI339999/4+TJk6hYsSKWLl2qiaaJiIiIKIvsYPfs2TNMmjQJYWFhEEVR2p7XdCbNmzfHvHnzIIoiXFxc0KZNG7nNExEREVEWWffYvX//HpMnT8bz589hZGSECRMmYPPmzfnuX6tWLbRv3x4AcPbsWTlNExEREVEOsq7Y7d69G6GhoTA2NsauXbtgZ2dX6DGdO3fGlStXcPv2bTlNAwBiYmJw9+5d3L17FwEBAQgICEB8fDwAYNCgQVi+fLla5zt//jz27duHgIAAxMbGwsLCAvb29hg2bBi6dOkiu14iIiKi4iQr2J08eRKCIMDd3V2lUAcAjRo1AgCEhobKaRoA0LFjR9nnAIDMzEwsWrQIBw4cyLb91atXePXqFU6fPo2hQ4di8eLFBS6TRkRERKRNsoLd48ePAQCdOnVS+Rhzc3MAwJs3b+Q0nUuNGjVQv359XLx4Ue1jV61aJYW6xo0bY8KECahduzbCwsLwxx9/ICgoCPv374eFhQX+85//aLRuIiIiIk2RFeySkpIAACYmJiofk5qa+n8Nl5M/IHfatGmwt7eHvb09qlSpgvDwcHTv3l2tczx9+hTbtm0DADRt2hQ7d+6Upmhp1qwZHB0dMWbMGAQGBmLr1q0YPHgw6tSpI7t2IiIiIk2T1a+ouPoWERGh8jGPHj0CAFhZWclpGgAwc+ZMdOvWDVWqVCnyObZv34709HQAwKJFi3LNu2dsbIxFixYBANLT0+Hp6SmzaiIiIqLiISvYNWnSBABw/fp1lY85fPgwBEFAixYt5DStEaIo4syZMwCA+vXr51tTixYtUK9ePQDAmTNnsk3rQkRERFRayAp2vXv3hiiK2LdvH168eFHo/p6enlII7N+/v5ymNSI8PFyaULlt27YF7tuuXTsga0BFeHh4idRHREREpA5Zwc7Z2Rm2trZ4//493NzccP78+VyTFIuiiLt372Lu3LlYsWIFBEFAmzZtSsX0ISEhIdLj+vXrF7iv8vNPnjwp1rqIiIiIikLWCAY9PT1s3LgRo0aNQkREBKZMmQIjIyNp1Qk3Nze8e/dOGjAhiiI++ugjrF69WjPVyxQZGSk9rlatWoH7Kj//8uXLYq2LiIiIqChkD02tUaMGDh06hCVLluD48eNITk6WnouNjZUeC4KAvn374vvvv4eZmZncZjXi3bt30uPCRvYaGxtLjxWjgdWRkZGh9jGq0tfXL7ZzE5Vlxfl7qy18vyAqPsX1nqHOeeXPOZI1OvbXX3/Ff/7zH/j6+iIwMBCxsbHIyMiAubk5GjdujG7dukkDEEqL9+/fS48NDAwK3NfQ0FB6nJKSonZbAQEBah+jCmNjYzRu3LhYzk1U1j18+DDbh9UPHd8viIpXaXjP0EiwU6hZsyZGjx6tyVMWq/Lly0uP09LSCtxX0Z0MINeUKKqwt7fnJ2WiD4ytra22SyCiD0hxvWdkZGSofIGoSMHO19cX//77LyIiIpCZmQlra2u0a9cOffv2LfTKV2lSoUIF6XFh3avKCVydCZkV9PX1GeyIPjD8nSUidZSG9wy1gt3r168xbdo03L17N9dzBw8exG+//Yb169d/MJ9ylQdEKA+kyIvy89WrVy/WuoiIiIiKQuXpTjIyMjB16lTcuXMHoijm+S88PBzjx4/PNmiiNLOxsZEeFzaFifLzhU2NQkRERKQNKge7Y8eOISAgAIIgoE6dOvjxxx9x9OhRHDt2DGvWrJFWbYiJicGff/5ZnDVrTK1atWBtbQ2osHqG4vmqVauiVq1aJVIfERERkTrUCnbIGiCxf/9+DB48GA0aNEC9evXQu3dv7Ny5E23btoUoijh+/Hhx1qwxgiCge/fuQNYVudu3b+e53+3bt6Urdt27d5fm6SMiIiIqTVQOdvfv34cgCPj8889RqVKlXM/r6+tj5syZQNZSXYmJiZqttJh89tln0s2OS5YsyTWVSUpKCpYsWQIAKFeuHD777DOt1ElERERUGJUHTyjum7O3t893n6ZNm0qP4+LiYGpqKre+Avn7++P58+fZ2lQIDQ2Ft7d3tv1dXV1znaNevXoYP348tmzZgsDAQIwcORITJ05E7dq1ERYWht9//x1BQUEAgPHjx6Nu3brF+pqIiIiIikrlYJeSkgJBEAqc6kN5dQbled+Ky4EDB+Dj45Pnczdv3sTNmzezbcsr2AHAnDlzEBMTg4MHDyIoKAhz5szJtc+QIUMwe/ZsDVVOREREpHkanaBYmSiKxXVqjdPT08NPP/2E3r17Y+/evQgICEBcXBwqV64Me3t7DB8+HF26dNF2mUREREQFKrZgVxKWL1+O5cuXa+x8Xbp0YYAjIiKiD5bawW7Xrl2wsLDQyH7Tp09Xt3kiIiIiyofawW737t0FPq+YCqSw/cBgR0RERKRRagU7Td43x7ngiIiIiDRL5WC3Y8eO4q2EiIiIiGRROdi1a9eueCshIiIiIllUXnmCiIiIiEo3BjsiIiIiHcFgR0RERKQjGOyIiIiIdASDHREREZGOYLAjIiIi0hEMdkREREQ6gsGOiIiISEcw2BERERHpCAY7IiIiIh3BYEdERESkIxjsiIiIiHREOU2cJD09Hb6+vrhx4wbCwsLw7t07ZGRkFHiMIAjYvn27JponIiIiIk0EO39/f8yfPx8vX76UtomimO/+giBAFEUIgiC3aSIiIiJSIivYPX78GBMnTkRKSgpEUYSBgQHq1q0LMzMzBjciIiKiEiYr2G3evBnJycnQ19fHjBkz4ObmhgoVKmiuOiIiIiJSmaxgd/XqVQiCAHd3d0yZMkVzVRERERGR2mSNio2LiwMA9OjRQ1P1EBEREVERyQp2FhYWAAAjIyNN1UNERERERSQr2LVu3RoA8OjRI03VQ0RERERFJCvYjR07Fvr6+tixYwfS09M1VxURERERqU1WsGvWrBk8PDzw4MEDTJ8+HbGxsZqrjIiIiIjUImtU7Lp164CsgOfr6wtHR0d07NgR9evXV+m+u+nTp8tpnoiIiIiUyA52iomIBUFASkoKzp07h3Pnzql0PIMdERERkebIXlIs5/JhBS0nRkRERETFR1awe/DggeYqISIiIiJZZA2eICIiIqLSg8GOiIiISEcw2BERERHpCNmDJxTi4+Ph7e2Ny5cv49GjR0hISAAAmJmZoUGDBujYsSNcXV1hbm6uqSaJiIiISIlGgt2ePXuwYsUKpKSkADlGxqakpCAqKgqXLl3C2rVrsXDhQgwfPlwTzRIRERGREtnBbsuWLVi1apUU5ipWrAg7OztYWVkBAKKjo3H//n28ffsWycnJ+P777/HmzRtMnDhRfvVEREREJJEV7IKDg7FmzRqIoggrKyvMnz8fffr0gYGBQbb90tPTcfz4cfz888+IiorCmjVr0LVrVzRo0EBu/URERESURdbgib/++gsZGRmwsLDA3r17MXDgwFyhDgDKlSuHAQMGYO/evbC0tERGRgb++usvOU0TERERUQ6ygp2fnx8EQcCkSZNQo0aNQvevXr06Jk6cCFEUcfXqVTlNExEREVEOsoLdq1evAACtWrVS+RjFvlFRUXKaJiIiIqIcZAU7Pb3/Ozw9PV3lYzIyMgAAgiDIaZqIiIiIcpAV7BTdr1euXFH5GMW+qnTdEhEREZHqZAW7jh07QhRFbNu2DQ8fPix0/+DgYGzduhWCIMDBwUFO00RERESUg6xg99lnn8HQ0BBJSUkYNWoUtm7diri4uFz7xcXFYevWrRg9ejTevXsHQ0NDfPbZZ3KaJiIiIqIcZM1jV7NmTSxevBgeHh5ISkrCL7/8gl9//RW1atWChYUFBEFATEwMwsPDIYoiRFGEIAj44Ycf2BVLREREpGGyV55wcXGBubk5vv32W0RFRUEURTx//hxhYWFAjuXFrK2tsWTJEnTp0kVus0RERESUg0bWiu3atSvOnj2LU6dO4cqVKwgODkZCQgIAwMzMDA0bNkSHDh3Qo0ePPCcwJiIiIiL5NBLskLW6RN++fdG3b19NnZKIiIiI1CBr8AQRERERlR4MdkREREQ6gsGOiIiISEeodI+dnZ0dkLUMWFBQUK7tRZHzXEREREQkj0rBTnnKElW2ExEREVHJUynYTZ8+Xa3tRERERFTyGOyIiIiIdAQHTxARERHpCFkTFF+/fh0AYG9vDyMjI5WOef/+Pe7evQsAaNu2rZzmiYiIiEiJrGDn5uYGPT09HDlyBDY2Niod8+rVK+k4joolIiIi0hzZXbFFHRnLEbVEREREmlXi99hlZmYCAPT19Uu6aSIiIiKdVuLB7sWLFwAAU1PTkm6aiIiISKepdY+dIpTlFB0dDRMTkwKPTU1NxfPnz7FmzRoIgoAGDRqoVykRERERFUitYNe9e/dc20RRxLhx49Ru2NnZWe1jiIiIiCh/agU7TSwtVr58ebi5uWHIkCHqNE1EREREhVAr2C1btizb1x4eHhAEAbNmzULVqlXzPU4QBBgaGsLa2hp2dnaoUKFC0SsmIiIiojypFewGDRqU7WsPDw8AQI8ePVSex46IiIiIioesCYp37NgBAKhVq5am6iEiIiKiIpIV7Nq1a6e5SoiIiIhIlhKfx46IiIiIioesK3bKRFHE/fv38eDBA8TFxSElJaXQ0bLTp0/XVPNEREREZZ5Ggp2Pjw/WrVuX7wTG+WGwIyIiItIc2cFu1apV2LJli0pz2QmCoNacd0RERESkOln32N25cwebN28GADg4OODQoUPw8fEBskLc/fv3ceXKFfz+++9wdHSEKIpo3bo1Ll68iAcPHmjmFRARERERIDfY7d69GwBQo0YNbN68GY0aNUK5cv//IqAgCKhcuTI+/fRTbNiwAd9++y1u3LiBCRMmIDU1VX71RERERCSRFexu3boFQRDg5uaWLdDlZ9SoUejVqxcePnyIXbt2yWmaiIiIiHKQdY9dVFQUAKBBgwbSNj29/58V09LSYGBgkO0YZ2dnnDx5EseOHcPYsWPlNK8Rtra2Ku3Xrl07eHl5FXs9REREREUl64pdeno6AMDS0lLaZmJiIj2OjY3NdUy1atUAAKGhoXKaJiIiIqIcZF2xs7CwQFRUFBITE6VtlpaW0NfXR2ZmJp48eYKqVatmOyY6OhoA8O7dOzlNa9zIkSMxatSofJ83NjYu0XqIiIiI1CUr2NnY2CAqKgpPnjxBmzZtAACGhoawsbFBcHAw/vnnH3To0CHbMYcPHwYAWFtby2la4ywtLdGwYUNtl0FERERUZLK6Ytu0aQNRFOHn55dte79+/SCKIg4ePIjffvsNjx49wt27d/H999/j2LFjEAQBnTt3lls7ERERESmRFex69OgBADh37ly27lh3d3fUrFkTmZmZ2LhxI5ycnDB8+HDs3bsXAFCpUiVMnjxZbu1EREREpERWsGvQoAF27NiB9evXSwMpkHU/2o4dO9CqVSuIopjtX4MGDbB9+3ZpEAURERERaYbsJcXatWuX5/aaNWti165dePLkCUJCQpCeno66deuicePGcpssFsePH8exY8cQEREBPT09WFlZoWXLlhg0aBA++eQTbZdHREREVCjZwa4w9evXR/369Yu7GdlCQkKyfR0aGorQ0FAcOnQIPXr0wPLly1GxYkWt1UdERERUmGIPdqWdsbExHB0d0aFDB9SrVw8VKlRAbGwsrl27hj179iA+Ph6nT5/GF198gW3btuWacFlVGRkZGq9dQV9fv9jOTVSWFefvrbbw/YKo+BTXe4Y65y3zwe7ChQuoVKlSru0ODg5wc3PDxIkTERQUhGvXrmH37t1wd3cvUjsBAQEaqDY3Y2PjUtu9TfShe/jwIZKTk7Vdhsbw/YKoeJWG9wyVgt26deuKpfHp06cXy3nVkVeoU6hSpQp+++039O3bF2lpafjrr7+KHOzs7e35SZnoA6PqkoNERCjG94yMjAyVLxCpHOwEQZBbVy6lIdgVpnbt2ujYsSPOnz+P0NBQvHr1KtdqGqrQ19dnsCP6wPB3lojUURreM1Se7iTntCU5/xVlnw/Fxx9/LD1+9eqVVmshIiIiyo9KV+wePHiQ73Ph4eGYM2cOAgIC0LlzZwwePBjNmjWDpaUlACAmJgYBAQE4cOAALly4AHt7e6xatQq1atXS3KsoZsVxtZKIiIhI02QNnnj79i3GjRuHiIgIrFixAs7Ozrn2qV69OqpXr45evXrhyJEjWLhwIcaNG4eDBw9+MNOHPH78WHpclG5YIiIiopIga+UJT09PPH/+HEOHDs0z1OXk5OSEoUOH4vnz59i2bZucpktMWFgYLl26BAD46KOPGOyIiIio1JIV7E6ePAlBENCnTx+Vj+nbty8A4NSpU3Ka1oizZ89mWwotp9evX2PmzJlIS0sDAIwaNaoEqyMiIiJSj6yu2PDwcABQq0tVsW9ERIScpjVi6dKlSEtLQ+/evdGiRQvUrFkTRkZGiIuLg5+fH/bu3Yu4uDgAQOvWrTF69Ghtl0xERESUL1nBzsDAACkpKQgODkaTJk1UOiY4OFg6tjSIioqCl5cXvLy88t2nd+/eWLp0KQwNDUu0NiIiIiJ1yAp2tra2uH79On7//Xf06dMHxsbGBe6fnJyM33//HYIglIqJP5cvX45r167h9u3bCAsLQ3x8PBITE2FiYoJq1aqhZcuWGDRoEFq2bKntUomIiIgKJSvYDRs2DNevX8fTp0/h5uaGJUuWwM7OLs99Hzx4gEWLFuHJkycQBAHDhg2T07RGtGvXDu3atdN2GUREREQaISvYDRw4EKdPn8aJEydw7949uLq6omHDhrC3t881j52iCxYAevbsiYEDB8qvnoiIiIgksoIdAPz3v//F8uXLsXPnTmRmZuLhw4fZQpyCKIoQBAFjxozBwoUL5TZLRERERDnIDnb6+vr4+uuvMXToUOzZsweXL19GaGhotiXD6tSpg44dO2L48OFo1KiR3CaJiIiIKA+yg51Cw4YN8e233wIAUlNT8ebNGwBApUqVOJqUiIiIqARoLNgpMzQ0RJUqVYrj1ERERESUD1krTxARERFR6cFgR0RERKQjVOqKdXd3BwAIgoDt27fn2l4UOc9FRERERPKoFOyuXbsGZIWxnNsFQcg2ArYwiv1znouIiIiI5FEp2LVt21at7URERERU8lQKdl5eXmptJyIiIqKSx8ETRERERDqCwY6IiIhIRzDYEREREekIBjsiIiIiHaHS4Ak7OzuNNywIAoKCgjR+XiIiIqKySqVgp848dURERESkHSoFu+nTpxd/JUREREQkC4MdERERkY7g4AkiIiIiHcFgR0RERKQjGOyIiIiIdIRK99ipKiEhAQ8ePEBcXBxSUlIK3d/FxUWTzRMRERGVaRoJdn5+fli7di1u3Lih8jGCIDDYEREREWmQ7GC3a9cuLF26FKIocr47IiIiIi2SFeweP36MH3/8EaIoomHDhpg5cybKlSuHyZMnQxAEnDx5EgkJCQgMDMS+ffsQFBSE1q1bY/HixTAyMtLcqyAiIiIieYMnvLy8kJGRgcqVK2Pnzp3o3r07atSoIT1fu3ZtNG3aFCNGjMDBgwcxfvx43LhxA0uWLEHNmjU1UT8RERERZZEV7K5fvw5BEODm5gZTU9MC9xUEAfPmzcMnn3wCPz8/HDhwQE7TRERERJSDrGAXGRkJAGjcuLG0TRAE6XFaWlquY4YNGwZRFHHkyBE5TRMRERFRDrKC3fv37wEAVatWlbYZGxtLj9+8eZPrmDp16gBZ9+cRERERkebICnbm5uYAgKSkJGmbhYWFdNXu6dOnuY6Ji4sD8gl9RERERFR0soJdvXr1AAChoaHSNmNjY+mq3NmzZ3Mdc+rUKSArABIRERGR5sgKdq1bt4YoivD398+2vVevXhBFEV5eXjh48CCSkpIQExOD33//HQcOHIAgCPjkk0/k1k5ERERESmQFu27dugEATp8+Ld1vBwCff/45zMzMkJ6ejm+++QatW7dGp06d8N///hcZGRkoX748Jk2aJL96IiIiIpLICnbNmzfHsmXL8OWXXyIhIUHaXrlyZWzduhU1a9aUVqRQ/LO0tMS6devw8ccfa6J+IiIiIsqi8soTP/30EwYNGgQ7O7ts2wcNGpTn/k2bNsWxY8dw9epVhISEID09HXXr1kWnTp2yjZwlIiIiIs1QOdjt2LEDXl5esLGxgYuLCwYOHAhra+sCjzEwMMCnn36KTz/9VBO1EhEREVEB1OqKFUURISEh+OWXX9CtWzeMHz8eR44cQUpKSvFVSEREREQqUTnYbd26FU5OTjAyMoIoisjIyMDly5exYMECODg4wMPDA1evXi3eaomIiIgoXyp3xTo4OMDBwQHJyck4efIkjhw5gitXriAzMxPv3r3DoUOHcOjQIVSrVg1OTk5wcnLiAAkiIiKiEqT2qFhjY2M4Oztj69at8PX1xbx582BrayuNen358iW2bNmCAQMGYMiQIdi5c6e02gQRERERFR9Z051YW1tj/PjxOHz4MA4fPozPP/8c1tbWUsi7d+8eli5dis6dO+OLL77AyZMnkZaWprnqiYiIiEgiK9gps7W1xYIFC3D+/Hls27YNzs7O0v14aWlpOHfuHGbNmoVOnTrhhx9+wO3btzXVNBERERFpMtgpCIKAjh07YsWKFbh8+TJWrFgBBwcH6OnpQRRFJCQkYPfu3Rg1apSmmyYiIiIq0zQe7JQp34936NAhNGjQAIIgAFlTpxARERGR5qg8KrYoFF2wR44cwfnz55Genl6czRERERGVacUS7G7cuIHDhw/jxIkTePPmDaB0ha5ChQro3bt3vkuREREREVHRaCzYPXv2DIcPH8bRo0cREREBKIU5fX19dOjQAS4uLujZsyfKly+vqWaJiIiIKIusYBcbG4t//vkHhw8fRmBgIJDj3rmGDRtK68paWVnJr5aIiIiI8qV2sEtNTcXp06dx5MgRXLx4ERkZGYBSoKtSpQoGDBgAFxcXNGrUSPMVExEREVGeVA52fn5+OHLkCE6ePInExERAKcyVL18ejo6OcHFxQadOnaCvr198FRMRERFRnlQOdp999hkEQZDCnCAIaN26NVxcXNC3b1+YmpoWZ51EREREVAi1umJFUcRHH30EJycnODs7o3bt2sVXGRERERGpReVgN2zYMLi4uKBVq1bFWxERERERFYnKwW7x4sXFWwkRERERyVKsS4oRERERUclhsCMiIiLSEQx2RERERDqCwY6IiIhIRzDYEREREekIBjsiIiIiHcFgR0RERKQjGOyIiIiIdASDHREREZGOYLAjIiIi0hEMdkREREQ6gsGOiIiISEcw2BERERHpCAY7IiIiIh3BYEdERESkIxjsiIiIiHQEgx0RERGRjmCwIyIiItIRDHZEREREOoLBjoiIiEhHMNgRERER6Yhy2i6gNImIiICXlxd8fX0RGRkJQ0ND1K5dG3379sXo0aNhbGys7RKJiIiI8sVgl+Xs2bOYN28eEhMTpW3JyclISEhAYGAg9u/fjy1btqBOnTparZOIiIgoP+yKBRAUFIQ5c+YgMTERJiYmmDNnDvbs2QNPT08MGzYMAPDs2TNMmjQpW/AjIiIiKk14xQ7Ajz/+iJSUFJQrVw7btm1Dy5Ytpec6dOiAOnXqYOXKlXj27Bn+/PNPzJgxQ6v1EhEREeWlzF+xu3v3Lvz9/QEAgwcPzhbqFMaNG4ePP/4YALBjxw6kpaWVeJ1EREREhSnzwe706dPS48GDB+e5j56eHlxcXAAAb968gZ+fX4nVR0RERKSqMh/sbty4AQAwMTFBkyZN8t2vbdu20uObN2+WSG1ERERE6ijzwe7x48cAgI8++gjlyuV/y2H9+vVzHUNERERUmpTpYPf+/XvExcUBAKpVq1bgvmZmZjAxMQEAREZGlkh9REREROoo06Ni3717Jz1WhLaCGBsbIykpCUlJSSq3IYoiACA1NRX6+vpFrLRg+vr6qF7BDPoQiuX8RGWNdYVKyMjIQEZGhrZL0Th9fX0IlatBTyie9yOiskgwtyrW9wzFeRWZoiBlOti9f/9eemxgYFDo/oaGhgCAlJQUldvIzMwEsubKK04NYYKGRoWHUyJSQQZw+/ZtbVdRfCrb/N8/ItKcEnjPUGSKgpTpYFe+fHnpsSpTmKSmpgIAjIyMVG6jXLlysLe3h56eHgSBV9SIiIhIPaIoIjMzs8CxAAplOthVqFBBeqxK92pycjKgYretgp6ennSlj4iIiKg4lenBE+XLl4e5uTmgwoCIhIQEKfwVNtCCiIiISBvKdLADABub/7vP5Pnz50hPT893vydPnkiPFatQEBEREZUmZT7YtW7dGsjqir13716++12/fl163KpVqxKpjYiIiEgdZT7Y9ejRQ3p88ODBPPfJzMzEoUOHAACVKlVC+/btS6w+IiIiIlWV+WDXrFkztGnTBsgKdrdu3cq1z7Zt26TVJtzd3VWaGoWIiIiopAmiKrPd6bigoCCMHDkSKSkpMDExwZQpU9C+fXukpKTgn3/+wd69ewEAdevWxcGDB2FqaqrtkomIiIhyYbDLcvbsWcybNw+JiYl5Pl+3bl1s2bIFderUKfHaiIiIiFTBYKckIiICO3bsgK+vL169egUDAwN89NFH6NOnD8aMGQNjY2Ntl0glyM/PD+7u7gCA6dOnY8aMGdmeX7t2LdatWyerjUGDBmH58uUAADc3N1y7di3XPnp6eqhYsSJq1aqFVq1aYfjw4WjQoIGsdolIc5TfK3IyMjKCubk5GjVqhJ49e8LJyUma29TR0RERERGy2t6xYwfat2+P8PBwdO/eHcjxvkJlT5meoDinmjVrwsPDAx4eHtouhUiSmZmJhIQEJCQk4N69e9i1axdmz56NSZMmabs0IipESkoKIiMjERkZCV9fX3h6emLTpk2oVauWtksjHcVgR1REo0aNQu/evfN87syZM1i9ejUAYPbs2dIn6ZzMzMzy3H706FHpcVpaGsLCwnD69GkcPXoUGRkZ+PXXX1G7dm307dtXI6+FiDRj5MiRGDVqlPR1TEwMHj16hK1btyIyMhKPHj3C1KlTcejQIWzdujXf5Sw9PDwQGBgI5Hg/yIkBkXJisCMqIktLS1haWub5nOINGQCqVq2Khg0bqnXunPs3adIEffr0QfPmzbF06VIAwPr16xnsiEoZS0vLXL+/HTp0gKurK5ycnBAREYHg4GCcOnUKffr0yfc8yktXqvv+QWVbmZ/uhOhDMnr0aNSoUQMA8OjRI0RHR2u7JCJSgampKaZOnSp9ffnyZa3WQ7qLwY7oA6KnpyctgwcAL1++1Go9RKQ6W1tb6XFh65MTFRWDHdEHRnmCbE6WTfThUP59LVeOd0JR8WCwI/rAKFZBASB1yxJR6af8u1uzZk2t1kK6i8GO6ANy8uRJPHv2DMi6ITu/UbVEVLpkZGRg69at0tf5jagnkovXgolKudTUVGm6k40bNwIAjI2NMWfOHG2XRkSFiI2NxcOHD/Hbb78hKCgIyAp1ijXKiTSNwY6oFFK+yTqnJk2a4Ouvv0bz5s1LtCYiKty6devyXZHG2NgYI0aMwNy5c0u8Lio72BVL9AExMDDA4MGD0bp1a22XQkRqatSoEdzc3DjoiYoVr9gRlULKM82/efMGDx8+hKenJ54/f47FixcjOTkZEyZM0GqNRJSb8soTGRkZiIyMxIkTJ3D48GHcunULbm5uOHDgACwsLLRdKukoBjuiUijnTPNt2rSBs7MzRo0ahYcPH2LVqlVo164dmjVrprUaiSi3nCtP2NnZoVu3bmjfvj0WLlyIiIgIfP3119L9skSaxq5Yog+Eqakpfv75Z+jp6SE9PR0rVqzQdklEpKJBgwZJI2HPnj2LK1euaLsk0lEMdkQfkEaNGmHAgAEAAH9/f1y4cEHbJRGRiubMmQN9fX0AwKpVq7RdDukoBjuiD8yUKVOgp/d/v7rsziH6cNSrVw99+/YFANy5cweXLl3SdkmkgxjsiD4wH3/8MXr27AkAuHnzJq5evartkohIRZMnT4YgCAA/mFEx4eAJIhXcv38f3t7ehe73ySeflMgyX1OmTMGJEyeArD8On3zySbG3SUTyNWzYEI6Ojjhz5gyuX78Of39/jU9WHBoaqtL7VbNmzWBjY6PRtkn7GOyIVHDmzBmcOXOm0P3Wr19fIsGucePG6NKlC86fP4+rV6/i9u3baNGiRbG3S0TyTZkyRXo/2bhxY7alxjTh5s2buHnzZqH7eXh4MNjpIHbFEn2gpkyZIj3esGGDVmshItU1a9YMDg4OAICLFy/i7t272i6JdIggiqKo7SKIiIiISD5esSMiIiLSEQx2RERERDqCwY6IiIhIRzDYEREREekIBjsiIiIiHcFgR0RERKQjGOyIiIiIdASDHREREZGOYLAjIiIi0hEMdkREREQ6gsGOiIiISEcw2BERfUDWrl0LW1tb2NraFlsbjo6OsLW1xcKFC4utDSIqHuW0XQARUUnw8/ODu7u79LWJiQkuX74MY2PjAo9LSUmBg4MDEhMTpW07duxA+/bti7VeIqKi4BU7IiqTkpKScPr06UL3O3PmTLZQR0RUmjHYEVGZU758eQDA4cOHC91XsY/iGCKi0ozBjojKHEdHRwDA5cuXER0dne9+MTExuHTpEgCge/fuJVYfEVFRMdgRUZnj4OAAKysrZGRk4O+//853v//9739IT0+HlZUVOnbsWKI1EhEVBQdPEFGZo6+vj/79+8PT0xOHDx/G2LFj89xP0Q07YMAA6OvrF3re1NRU7N+/H8ePH8ejR4+QmJgIMzMzNG7cGAMGDMDAgQOhp1fw5+nIyEhs3rwZFy5cQFRUFMzMzNC0aVO4u7urFS7fvn2LXbt24dy5c3j27BkSExNhbm6Opk2bwsXFBb1794YgCCqfj4g+DAx2RFQmOTs7w9PTE0FBQXj06BEaNGiQ7fmQkBDcu3dP2vf+/fsFni88PBwTJ07EkydPsm1//fo1Lly4gAsXLmDv3r3YsGEDzM3N8zyHv78/Jk+enG2wRnR0NM6dO4dz585hxowZKr22K1euYPbs2YiPj8+2XflcXbp0wapVq1ChQgWVzklEHwZ2xRJRmdS4cWMpzOU1iEKxrWHDhrCzsyvwXO/evcPYsWOlUNejRw9s3LgRBw8exJo1a9CuXTsAwI0bNzBlyhRkZGTkOseLFy+kUKenp4cRI0bA09MTBw4cwI8//oi6deti7dq18PX1LbCWGzduYOLEiYiPj0eVKlUwe/ZsbNq0Cd7e3ti0aROcnJwAAOfPn+c8dUQ6iMGOiMosZ2dnIOteOlEUpe2iKOLo0aPZ9inIunXrEBYWBgCYOnUq1q9fD0dHRzRt2hR9+vTBjh07MHDgQADArVu3sHfv3lznWL58uXSlbuXKlfjhhx/QoUMH2NvbY8iQITh48CAaNWqEwMDAfOtIS0vDvHnzkJaWhk8//RSnT5/G1KlT0a1bNzRp0gTdunXDypUrsWTJEgDAyZMnpcEhRKQbGOyIqMxycnKCnp4eXr58CT8/P2m7n58fXr58CT09PSmQ5Sc1NRUHDhwAADRo0CDP7lJBEPD9999LXbA7d+7M9nx0dLQ0p163bt0wYMCAXOcwNTWVAll+/v77b0RERKB8+fL4+eef8518ediwYWjWrBkAwNvbu8BzEtGHhcGOiMqsqlWrSitIKHfHKh5/8sknqFq1aoHnCAwMxJs3bwAAgwYNyneQhampKfr27Qtk3b8XFRUlPefn5yd1z7q6uubbVrNmzXLdC6js7NmzAIC2bdvCwsKiwLrbtGkDALh9+3aB+xHRh4WDJ4ioTHNxccGVK1dw8uRJfPfddwCAEydOACp2wz569Eh63Lx58wL3bd68OXbv3i0dZ21tDQAIDg6W9rG3ty/wHPb29tnaVKbopr148aLKa8m+fv1apf2I6MPAK3ZEVKb17NkTxsbGSExMxJkzZ3D69Gm8e/cOJiYm6NWrV6HHJyQkSI8Lu0pWpUqVPI9THr1qaWmp8jlyio2NLbTenFJSUtQ+hohKL16xI6IyrUKFCujRoweOHj2Kw4cPS4MoevToARMTE7XOpe154RTduZ07d8a8efO0WgsRaQeDHRGVeS4uLjh69Gi2EaIuLi4qHWtmZiY9jomJQb169fLdV7nbU/m4nOeoXr26SufIydzcHFFRUUhLS0PDhg1Vqp+IdAu7YomozOvQoQOsrKyQnp6O9PR0WFtbo0OHDiodqzyY4c6dOwXue/fu3TyPUw5hAQEBBZ6joOlOGjduLO2TmppaSOVEpIsY7IiozNPX14ezszMMDQ1haGgIZ2fnQpf+UmjatCkqVaoEADh06BAyMzPz3C8xMRHHjh0DANjY2EgDJwCgffv20mhaHx+ffNu6e/dutoEWOTk6OgJZy4lxGhOisonBjogIwLx58xAQEICAgAB8+eWXKh9naGiIIUOGAFmjWzds2JBrH1EUsWTJEsTFxQEARo8ene15a2trdO/eHciasuSff/7JdY53795Jo3bzM2jQIKkbd8WKFbh+/XqB+/v7++PatWuFvkYi+nDwHjsiIpmmTZuGU6dOISwsDGvXrkVwcDBcXV1hZWWF8PBw/PXXX1KAatmyJYYPH57rHAsWLMClS5fw7t07fPnll7h+/Tp69+4NU1NTPHz4EFu2bMGzZ8/QtGnTfLtjDQ0NsXr1ari5uSEpKQmfffYZ+vXrhx49eqBWrVrIzMxEdHQ07t27h1OnTiE4OBiLFi2Sljwjog8fgx0RkUympqbw9PTExIkT8eTJE5w4cUKaC09Zq1atsHHjxjwnMa5VqxY2btyIqVOn4t27d9i1axd27dqVbZ9p06ZBEIQC77Nr0aIFvLy8MHv2bLx8+RJHjx6VlkfLr3Yi0h0MdkREGlCrVi0cPnwY+/fvx/HjxxEcHIx3797BzMwMdnZ2GDhwIAYOHFjgvXvt27fH33//jc2bN+PChQuIioqCmZkZmjZtijFjxuDTTz/F2rVrC62lRYsWOHnyJLy9vXHu3DkEBQUhLi4Oenp6sLCwwMcff4y2bduiV69eqF+/voa/E0SkTYKovPI1EREREX2wOHiCiIiISEcw2BERERHpCAY7IiIiIh3BYEdERESkIxjsiIiIiHQEgx0RERGRjmCwIyIiItIRDHZEREREOoLBjoiIiEhHMNgRERER6QgGOyIiIiIdwWBHREREpCMY7IiIiIh0BIMdERERkY5gsCMiIiLSEf8PvE+FaKUg2JMAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "# Data\n", + "names = ['LTR', 'RTL']\n", + "val_loss = [np.exp(2.8237654270093375), np.exp(2.8326140656842465)]\n", + "\n", + "# Create bar plot\n", + "plt.bar(names, val_loss, color=['#72B6A1', '#E99675'])\n", + "\n", + "# Add labels and title\n", + "plt.xlabel('Model', fontsize=20)\n", + "plt.ylabel('Validation Perplexity', fontsize=20)\n", + "plt.title('DistilBERT Base Japan Perplexity', fontsize=20)\n", + "\n", + "# Show the plot\n", + "# plt.xticks(rotation=45, ha=\"right\") # Rotate x labels for better readability\n", + "plt.tick_params(axis='both', labelsize=20)\n", + "plt.tight_layout() # Adjust layout to fit everything\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 89, + "execution_state": "idle", + "id": "88c66310-bf62-44fc-b09d-5fb08ec084ad", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 104, + "execution_state": "idle", + "id": "5c86ffd5-d280-4b9f-b250-97d3e398f9a7", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_2794676/549159718.py:15: FutureWarning: \n", + "\n", + "The `ci` parameter is deprecated. Use `errorbar=None` for the same effect.\n", + "\n", + " sns.barplot(x='model', y='ppl', hue='direction', data=riddles_rtl_df_sorted_pairs, dodge=True, palette=\"Set2\", ci=None)\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk0AAAMWCAYAAADsxLLYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAACPI0lEQVR4nOzdd3gUVeP28XvTINQQIPSHIh2CSFOKIEWK0iEgXUEBFQsqKCr4CIoFFX1UFBSkiNK7CAIKinQwEAw1tCT0BAIhCWn7/vFm55eeSXbTyPdzXVxMdmbOObObzd575swZi9VqtQoAAADpcsrtBgAAAOQHhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABEITAACACYQmAJn25Zdfqk6dOqpTp45d5QwbNkx16tTRsGHDslxGUFCQ0ZZVq1ZlW1uRNbbn/ssvv8ztpqRq1apVRhuDgoLsKssRv8/I21xyuwFAfnP79m2tW7dO27dvV0BAgEJCQuTi4qIyZcrI29tbHTt2VOfOneXs7Jzpsr/66ivjw6VYsWLatWuXChUq5JB2v/HGG1q9enWKxy0Wi4oVK6YKFSro/vvvV//+/dW4cWOH1In/M2zYMO3bty/F405OTipevLiqVKmiFi1a6IknnlDVqlVzpY0A0kdPE5AJy5YtU6dOnTR16lT9+eefCg4OVlRUlMLDw3Xu3DmtX79eL7/8snr37q1//vkn0+WvWbPGWA4PD9fWrVsdfAQpWa1W3b59WydPntTy5cs1cOBATZs2TdyWMmfEx8crLCxMR48e1bx58/T4449r8eLFud0sJHjjjTdUp04ddejQIbebgjyAnibApI8++kjz5s2TJLm4uOixxx5Thw4dVKlSJUVHR+vs2bPasGGD9uzZo5MnT2rEiBGaOXOmOnbsaKr8AwcOKDAwUJJUpEgRRUREaM2aNXr88ccdfixz586Vl5eXJCkuLk7BwcHatWuXli5dqtjYWP3444+qUKGCnn766VT3f+GFF/TCCy84vF0Fxfr1643l+Ph4Xb58WZs2bdLq1asVExOjadOmqVKlSnrkkUdytZ0FQd++fdW3b1+HlLVo0SKHlIO8i9AEmLB48WIjMJUvX17ffvut6tWrl2SbZs2aycfHR+vXr9ekSZN09+5djR8/XitXrlStWrUyrGPt2rWSpDJlymjEiBH69NNP9ffff+v69esqU6aMQ4+nWrVqqly5svFzvXr11KlTJ7Vp00bPPvusJGn27NkaMWKEXF1dHVo3pNq1ayf5uW7dunrkkUfUoEEDvffee7Jarfrf//5HaALyGE7PARkIDg7WRx99JCX0AM2fPz9FYEqsR48emj59uiTp7t27ev311zOs4+7du9q0aZMkqVu3burVq5ecnJwUFxeXpFciu3Xo0EFNmzaVJN26dUv//vtvjtUNaciQIapUqZIk6d9//1VISEhuNwlAIoQmIAMLFizQ3bt3JUnPPfecqlevnuE+PXv21MMPPywlfPjt3Lkz3e23bdumW7duGfuWK1dODz74oJRsnFNOSNwLcunSpVS3MXtFmq+vr1588UW1bt1a3t7e6tChgyZPnqwzZ86Ybk9cXJwWL14sHx8fNWnSRE2bNlWfPn00d+5cRUdHZ+LI0nf37l39+OOPGjFihFq3bq2GDRuqZcuWevLJJ7V8+XLFxsY6rK60ODk5qWHDhsbPFy9eTLHN+fPnNX36dPXo0UNNmzZVo0aN1LFjR73xxhvy8/NLs+y9e/car9nevXsVHx+vFStWaNiwYWrVqpXq1q2rN954w9g++ZVgZ86c0eTJk9WhQwd5e3urTZs2eumll+Tr6+uQY//33381ZcoUdenSRQ888IAaN26sLl266J133tHZs2dTbG+1WjVq1CjVqVNH9evXT3cM4cKFC41j/9///pdkXVpXz9l+x20XTwQHBxvbJf6XmNmr565du6aZM2eqb9++atGihRo2bKh27drppZde0q5du9LdNy4uTqtWrdKoUaOM39OmTZuqc+fOGjFihL799ludPn063TKQdZyeA9JhtVqN02aFCxfWE088YXrfESNG6K+//pIkrVy5Um3atElzW1sd1atXV6NGjaSE8LR7924dP35cJ06cyLFL5hOfjrPn1Nz8+fP10UcfKT4+3ngsODhYy5Yt04YNG/T5559nWMadO3c0evRoHThwIMnj/v7+8vf31y+//KL33nsvy220OX78uJ577jkFBwcneTw0NFS7d+/W7t27tXTpUn377bcOP1WanIvL//1ZTvzcKWEs2syZMxUTE5Pk8aCgIAUFBWnNmjV69tln9dJLL6Vbx927dzVq1KgMP6BtduzYoZdfflkRERHGY9euXdOmTZv022+/6fXXX9eTTz5p8giTio+P10cffaQFCxakuPjg3LlzOnfunFasWKEpU6Zo4MCBxjqLxaIPPvhAPXr00M2bNzVhwgStWbNGxYoVS1LGqVOn9Mknn0iS7r//fj333HNZaqejrFu3Tu+8806S51KSMa5t06ZN6t+/v959990kvwtK5/0QExOj8PBwnT9/Xnv27JG/v3+KcAjHIDQB6Th16pRu3rwpJYxZKl68uOl9W7VqpcKFCysqKirFH7nEQkJCjJ6oHj16GI937txZ7777rqKiorRmzRpTp/kcIXEvkO1UUWZt2bJFH3zwgSSpePHieuaZZ9SiRQtJ0p49e/T999/rtddek6enZ7rlTJgwwXjuGjVqpCeffFJVq1ZVSEiIVq1apU2bNmnKlClZaqPN+fPnNXToUN2+fVvFihXTkCFD1KhRI5UvX143b97U77//rqVLl8rPz0/PPfecFi9enK3jvE6ePGks2wbrS9L333+vGTNmSAlzHw0aNEjVqlVT8eLFdfbsWS1evFj//POPZs2apVKlSmn48OFp1vHJJ5/oxIkT6tChg/r27auKFSvq+vXrunPnToptr169qtdee03Ozs565ZVXjNdx7969+u677xQeHq4PPvhAlStXVqdOnTJ9vNOmTdNPP/0kSWrevLn69OmjKlWqqHDhwjpx4oQWLFigU6dOacqUKSpTpkySCyu8vLz0/vvv6/nnn1dgYKDef/994/dOkqKjo/Xqq6/q7t27KlKkiGbMmJEiiKRl8ODB6tKliz7//HNt27ZNXl5emjt3bqaPL7GNGzdq4sSJslqtqlKlioYOHar77rtPnp6eCg4O1ooVK7Rjxw6tWLFCxYoV06RJk5Ls/9VXXxnvh/bt26tHjx6qUKGCChUqpJCQEB07dkx//PGHXW1E+ghNQDqOHz9uLNevXz9T+zo7O6tu3bry9fXV1atX0xzQvX79euPUT8+ePY3HixUrpg4dOmjjxo1av3698cGVnfz8/Izeh2rVqmWpdys6OlpTp06VEgLT0qVLdd999xnrH3jgAXXs2FGDBg3SuXPn0ixn+/bt2rZtmySpXbt2mjVrVpIPvHbt2iWZ1yqrXn/9dd2+fVv169fX3LlzUwS5Nm3a6JFHHtGYMWN0+PBhrV69WgMGDLCrzrRs375dp06dkiRVqVJFFSpUkCSdPn3a6JkbN26cxo0bJ4vFYuzXsGFDPf7443r99de1bt06zZw5U7169VLJkiVTrefEiRN69tln9fLLL2fYpnPnzqX7Oj7xxBMKDw/X1KlT1a5du0wFyr///tsITO+99558fHySrG/UqJF69uyp0aNHa8+ePXr//ffVrl27JL8HnTp1ko+Pj5YvX65Vq1apXbt26tq1qyTps88+04kTJyRJb775ZqbmvypdurRKly6tEiVKSAm9rskH8GdGaGiopkyZIqvVqn79+mnq1KlJjqNBgwbq3LmzZs6cqW+//VYLFy7UwIEDVaNGDWObX3/9VZLUpUuXVHuS2rZtqzFjxhhf9OB4jGkC0nHjxg1jOSunZUqXLp1qWYnZTs098MADqlKlSpJ1vXr1khJOhfz999+Zrt+MuLg4XbhwQT/99JOefvppxcfHy9nZWRMmTJCTU+b/RGzbtk1Xr16VEsaAJf6gtaldu7bGjh2bbjm2D1M3NzdNmzYt1R6C5557zq4PsgMHDhhjYT788MM0e77atm2rLl26SAljYBwpPj5eFy9e1Lx585KEmMTTPcybN08xMTFq2LBhisBk4+TkpMmTJ8vNzU0RERHavHlzmnVWq1YtU1NGpPU61qpVy3gdr1y5YoRcs+bMmSMlhIDkgcmmUKFCRm9icHCw9u7dm2KbN998U9WqVZMkvfPOO7py5Yp27dql+fPnS4mCVW76+eefdfv2bZUrV07//e9/0+zxeuGFF1SuXDnFx8cbfxtsrl+/LiX0eqfHw8PDgS1HYoQmIB2JT1cUKVIk0/sn3sc20DuxkydPyt/fX0rWy2TTpk0b44M8+R9Qe3Ts2NEYyFq/fn09+uijevfdd3Xz5k1Vq1ZNX375ZZZOtUjS7t27pYQxJ717905zu379+qX64a+EIGebPbt169YqV65cqts5OTmlW0dGbB/y1atXz7BXrXnz5pKko0eP2j0oPPFA4nr16ql9+/b66KOPFBkZKUkaOHBgkvFztlMuXbp0SfM5k6QSJUoYITK9gdGPPfaY6V7LzLyOZsdIKWHyVttrbAukabnvvvtUqlQpKY3jSnzq7ebNm3r11Vf1xhtvyGq1qmzZspo2bZrpdmWX33//XZL0yCOPyM3NLc3tXFxcjBn5kx9r2bJlpYTTfLbfFeQsTs8B6ShatKixnHzgphmJ90ntD6XtyhxXV1d169YtxXrbJJo//vijtm3bpvDw8BQDXR3JYrGoW7duat++fZbLsI3JqVy5crpjljw9PVWpUqVU7/d14cIF40PB29s73fpsA+ez4ujRo5Kks2fPmj4VGRMTo7CwsCS9iI7g7u6uJk2aaOjQoUlmnw4ODlZoaKgk6dNPP9Wnn35qqjxbr0RqMnPaNTOvY+LxWBnx9/c3Brq/8soreuWVV0ztl9ZxNWrUSM8//7y++OIL7d+/X0o0WDyjsXPZLS4uzjjVv3TpUi1dutTUfsmPtXfv3po1a5b++ecfdezYUV27dlXLli3VtGnTXD/GgoLQBKTD9u1WGXwIpSXxPDuJy1LCH9INGzZIkh5++OEU62169eqlH3/8UZGRkdq8ebP69euX6XYkl3hG8IiICAUEBGjRokU6duyYvvnmG4WFhemdd97JUtm28RRmQkWZMmVSDU1hYWHGckbl2BNebGEks+z9lp947i1nZ2cVLVpUZcuWTbX3J6tzNUVFRaW5Lq2xTqnJzOuY+HXLSHYc15gxY7Rq1SpjZn0fHx9j6o/cFBYWlqXeyeTH+txzz+nKlStatWqVQkJCtHjxYuOWO7Vq1VLnzp01ePDgbL/CsyAjNAHpqFu3rrFsO41mVlxcnDEI1d3dXRUrVkyyfteuXcbYn99//93Ut/81a9Y4JDQlnxG8cePG6tmzp8aOHaudO3fqp59+UqtWrfToo49muY70TiPlRjmpiYuLkxJeZ9uVaWakdbrQrMyMw0o87cDzzz9vDHLOiLu7e5rrMjNWLbue/8THNXXqVD3wwAOm9ksv8G3fvt0ITJJ08OBBRUVFqXDhwna21j623zMlBLn0rmxMLPmgeldXV02fPl0jR440btl09OhRxcTE6NSpUzp16pR++OEHzZgxI8un15E+QhOQjlq1asnDw0M3b97UgQMHdPv2bdPTDuzatcvokWjSpEmKD6qsTFq5f/9+Xbx4MUUAcwRXV1d98MEH6tq1q+7cuaOPPvpIjzzySKYvr7d9qJnpmUtrG9sVS2bKsWfWbNuA2YiICLsGlGenxIN6XVxccrydmXkdM9ODlfi4ChcubPdxXb9+XW+//baUcOVpeHi4AgIC9PHHH9s9LYW9Ej8vVqvV7mOtWbOmcdHA3bt3dfDgQa1fv15r165VRESEXn31VW3ZsiXJlBVwDAaCA+lIPAg2KipKy5YtM73vjz/+aCwn7x0IDw83BiG3bNlSn332Wbr/bJfwJ55sMzt4eXkZ34IDAwO1YsWKTJdh+0AICgpK84pBJZwaSz6ZpM1//vMfo3cgvVmuzaxPj20aicDAQF27di3L5WSnKlWqGEH90KFDOV5/Zl7HzISBevXqGb1YjjiuN998U6GhoXJyctLs2bONweWLFy/Wn3/+meVyHdHT5ubmZtx/0tGvYaFChdSqVSt98MEHmjhxopTwt2r79u0OrQf/H6EJyMDw4cNVqFAhKWFyufPnz2e4zy+//GL80SpbtqwxdYDN5s2bjV6oQYMG6fHHH0/338CBA41ThdkZmiTpySefNK76++677zI9FqNly5aSiYC3atWqFDNA27i4uBiTKP7999/Gaczk4uPjjcH0WWEbcG21WrVw4cIsl5OdnJ2d1a5dOynhuQgICMjR+jPzOtpeezM8PT2Nq8Q2bNiQ5fFlSghGO3bskCQ988wzatasmaZOnWqcRrUFqqywXcBh7y17bL9rZ86cMe4U4GgPPfSQsZxe0EXWEZqADFSqVMmYjTsiIkJPPvlkkkkvk9u4cWOS2bvffvttI3TZ2E7Nubu7q23btqbaYfvmfPbsWR0+fDhLx2KGh4eHcbl7cHBwpkNap06djEujZ82alep95k6fPq1vv/023XIGDRokJXxYTZkyJcm4EJvZs2dn6oqt5Nq0aWNcfTd37lxt3Lgx3e1PnDhhXDqek0aPHi1nZ2fFx8frxRdf1OXLl9PcNi4uTuvWrUt3m8xK63UMCAgwXseyZcsmma3bjGeffVZK6Hl98cUXU52WwyY6OlqLFy827gNpc+bMGWM8WoMGDYz5pzw8PPThhx/KYrHo2rVrWT5FZ/tdDgkJUXh4eJbKUMKXL9uXkUmTJhmTmKZl+/btSf7O2GanT+uLhhJCtU3iMYtwHMY0ASYMGTJEQUFBmjdvni5evKh+/frp8ccfV4cOHVSxYkXFxsbqzJkz2rBhgzFPkSSNHTs2xam5ixcvGpdEt23bNt0Bu4l16dJFX3zxhZQQuu6//36HHmNiI0eOND6g5syZoz59+pgePOzm5qbJkyfrxRdfVFhYmAYOHGjcRsVqtWrfvn367rvvJElVq1ZNs+euQ4cOat++vf744w/98ccfGjRoUJLbqKxevVobN25Uw4YNjakDsuLTTz+Vj4+Pbt68qfHjx2vdunV67LHHVK1aNTk5OSW5PYWvr69GjhyZZEqAnFCnTh1NnDhRH3zwgU6fPq3u3btrwIABeuihh1SmTBndvXtXwcHB8vX11aZNm3Tt2jWtX79e5cuXt7vuqlWrKjQ0NMnrKEn79u3TnDlzdPv2bUkyJtbMjHbt2mn48OFauHCh9u/fr8cee0xPPPGEmjZtKg8PD0VEROjChQs6cOCAtmzZorCwMPXu3dv4EhITE6PXXntNkZGRKly4sGbMmJFkDF6rVq00fPhwLViwQFu2bNGKFSvUv3//TLWxSZMmUkKv5jvvvKNhw4YludLV7CzjZcqU0UcffaQXX3xR165dU79+/dSnTx+1bdtW5cuXV2xsrC5fvqwjR45o8+bNCgwM1Lfffmv0MIeHh+vZZ59VpUqV1LlzZzVq1EiVKlWSs7Ozrl27pj/++EPLly+XEi5UeOSRRzJ1nDCH0ASY9Prrr6tatWr67LPPdPPmTa1duzbNXphChQpp4sSJGjp0aIp1a9euNb4tZjSpX2L33XefatasqdOnT2vjxo2aNGlSpj+kzCpbtqz69eunn376SefOndPGjRvVvXt30/t36dJFEydO1IwZM3Tr1q0Ucwu5u7vr888/19y5c9M93fnJJ5/omWee0aFDh3T48GGNHz8+yfr69etr6tSp6tu3bxaO8v/7z3/+oyVLlujFF1/UyZMnjZCWlsRzd+Uk22nT6dOn6/bt25o7d26a90JzdXVN0buZVeXKldObb76pl19+OdU5opycnDRhwoRM/S4n9uabb6pkyZL65ptvdO3atXRvi1OkSJEk0zJ8+eWX+vfffyVJEydOTHXW8tdee027d+/WyZMn9f777+vBBx9MMfN+eh566CE1btxYvr6+2rBhgzFNiI3tClkzOnfurFmzZmnSpEm6efOmlixZoiVLlqS6rZOTU6pfqIKDg/XDDz+kWUfZsmU1a9asXPs9vdcRmoBMGDhwoLp166Z169Zp+/btOn36tEJDQ5OcMihatKhWr16d5jdQW9Byc3MzxqqY1blzZ50+fVo3b97Ujh077JoSICNPP/20li9frpiYGM2ePVuPP/54pgbFjho1Sg888IB++OEHHTx4ULdv31bZsmX10EMPadSoUbrvvvsyvAFqsWLFtGjRIi1ZskRr1qxRQECALBaL/vOf/+ixxx7TiBEjHDKAu3r16lqzZo1+/fVX/fbbb/Lz81NoaKji4uLk4eGh6tWrq2nTpnr00UfVoEEDu+vLqgEDBqhDhw5asmSJ/v77b509e1a3b9+Wm5ubvLy8VKdOHbVq1UqdO3d26GSHjzzyiFauXKnvv/9ee/fu1dWrV1WiRAk1a9ZMTz31lOnpAlJjsVg0btw49erVS0uWLNGePXsUFBSk27dvq3DhwqpQoYLq1aunNm3aqFOnTsYFAgcOHDB6LNu1a6chQ4akWr6bm5tmzJghHx8fRUREaMKECVq8eLHpGdGdnJw0d+5cff/99/rjjz+MiVfTO02Wng4dOmjbtm1atmyZduzYodOnTyssLEzOzs4qU6aMatWqpYceekhdunQx7j2ohGECy5cv159//ql//vlHwcHBCgkJUUREhIoXL66aNWuqffv2GjhwYLZOgFvQWaxZfeUBJPHxxx8bIaBHjx6aMWNGts4xBGSnYcOGad++fWrRooUWLVqU280B8gQGggMOMmHCBHXu3FlKmPH5448/zu0mAQAciNAEOIjFYtGMGTOMe6XNmzfPuMs6ACD/Y0wT4ECFCxfWt99+qyVLlshqtSo8PFy3bt1KMsM1ACB/IjQBDlamTBmNGzcut5sBAHAwTs8BAACYwNVzAAAAJnB6rgCJj49XbGysnJycuBQeAIAEVqtV8fHxcnFxSffuB4SmAiQ2NtauO8IDAHAv8/b2TvdOC4SmAsSWnr29vU3PhgsAwL0uLi5Ofn5+Gd5jk9BUgNhOyTk7OxOaAABIJqOhK1w9BwAAYAKhCQAAwARCEwAAgAmEJgAAABMITQAAACYQmgAAAEwgNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEbqOCLLFarYqJiVF8fHxuNwX5hJOTk1xdXTO8TQEA5FWEJmRKRESEwsLCdPv2bcXFxeV2c5DPODs7q3jx4ipZsqSKFCmS280BgEwhNMG027dvKygoSK6urvLw8FDRokXl5OREzwEyZLVaFR8frzt37ujWrVu6efOmKleurOLFi+d20wDANEITTImIiFBQUJBKlCihihUrEpSQJUWLFlXZsmV18eJFBQUFqWrVqvQ4Acg3GAgOU8LCwuTq6kpggt0sFosqVqwoV1dXhYWF5XZzAMA0QhMyZLVadfv2bZUoUYLABIewWCwqUaKEbt++LavVmtvNAQBTCE3IUExMjOLi4lS0aNHcbgruIUWKFFFcXJxiYmJyuykAYAqhCRmyTSvg5MSvCxzH2dlZSvT7BQB5HZ+CMI1Tc3Akfp8A5DeEJgAAABMITQAAACYQmgAAAEwgNAEAAJjAjOBAHrR3714NHz5ckjRu3Di98MILSdZ/+eWX+uqrr+yqo0+fPvrwww8lScOGDdO+fftSbOPk5KTixYurcuXKatKkiQYOHKhatWrZVS8A5Ff0NAFIU3x8vMLCwvTvv/9q0aJF6tWrl+bMmZPbzQKAXJGve5pCQkJ05MgRHTlyRH5+fvLz89PNmzelZN+izdqxY4eWLVsmPz8/hYaGytPTU97e3howYIDatWtnqozY2FgtX75c69ev15kzZxQRESEvLy+1atVKw4YNM/0tPTQ0VIsWLdLWrVsVHBwsSapUqZI6deqk4cOHq1SpUpk6tpwUb42Xk+XezeN54fgGDx6sLl26pLpu27Zt+vzzzyVJL7/8sjp27JjqdiVLlkz18fXr1xvLMTExCgwM1NatW7V+/XrFxcXp008/VZUqVdStWzeHHAsA5Bf5OjS1atXKIeXEx8dr8uTJWrFiRZLHr1y5oitXrmjr1q3y8fHR1KlT053gMTQ0VKNHj5afn1+SxwMDA7V06VKtXr1aU6ZMkY+PT7rtOXz4sJ5//nldu3YtyeMnT57UyZMntXz5cs2aNUuNGjXK0vFmNyeLkxYf/ktXwu+9+4qVK1ZSQ+5/OLebodKlS6t06dKprjt69KixXK5cOdWuXTtTZSffvkGDBuratavuv/9+vffee5Kkr7/+mtAEoMDJ16EpsYoVK6pGjRrauXNnpvedOXOmEZjq16+vp59+WlWqVFFgYKC+//57+fv7a/ny5fL09NQrr7ySahlxcXEaN26cEZg6d+4sHx8feXh46PDhw/rmm28UEhKiKVOmyMvLK82eq0uXLmns2LEKDQ2Vi4uLnnzySbVv316S9Mcff2j+/Pm6du2axo4dq1WrVql8+fKZPt6ccCU8TMG3QnO7GXCgIUOGaN68ebp48aJOnTqla9euqWzZsrndLADIMfk6ND3//PPy9vaWt7e3ypQpo6CgoDRPRaTl7NmzmjdvniSpYcOGWrx4sQoXLixJatSokTp06KChQ4fq6NGjmjt3rvr166eqVaumKGf16tU6ePCglHDq5J133jHWNWrUSG3btlXfvn0VHh6u999/X61bt5aLS8qnf+bMmQoN/f9h45NPPknybb5Zs2Zq0KCBxo8fr5CQEH3++eeZPgUJZJWTk5Nq1qypixcvSgkBn9AEoCDJ1wNPXnzxRbVv315lypTJchkLFixQbGysJGny5MlGYLJxd3fX5MmTpYTxSvPnz0+1HFvw8vDw0MSJE1Osr1q1qsaMGSNJOn/+vLZs2ZJim2vXrhnjSdq0aZPq6Y/HHntMbdq0kSStXbs2xSk8IDu5urqmuoy8J97KPf1yCs91wZGve5rsZbVatW3bNklSjRo11Lhx41S3a9y4sapXr66zZ89q27ZtmjJlSpL7Zp09e1YBAQGSpK5du8rd3T3Vcvr06aNPP/1UkrR169YUoej33383bl7ar1+/NNvdt29f7dy5U/Hx8fr99981cODATB87kBW233MlnBJH3nUvjy3MS/LKOEfkjAIdmoKCgnT16lVJUvPmzdPdtkWLFjp79qyuXLmioKAgValSxVhnOy1n2y4tZcuWVbVq1XTu3DkdOnQoxfrE5aTXnsTrDh06RGhCjvjtt9907tw5SVLLli3TvPoOeQdjCwHHyten5+x1+vRpY7lGjRrpbpt4/ZkzZ5KsS/zt22w5ly5dUkRERKrtKV68eLpjRby8vFSsWLEUdQOOFh0drYCAAM2ePds47ezu7q7x48fndtMAIMcV6J6my5cvG8sZXYWWeP2lS5fSLKdcuXLpllOhQgUp4dTg5cuXk4SsK1eumGqLrZxTp04lqRtwhDp16qS5rkGDBnrrrbd0//3352ibACAvKNCh6c6dO8ZykSJF0t028Til5D1EicspWrSo3eVk1JbE5SSu26y4uLhMb2+1Wo1/GUk83uteZeZ5cFT5Zp93e/Y1s42rq6v69eunJk2aOOT4bW2Li4vL9O8kMubs7JzbTShQ+B3O38y+fgU6NN29e9dYzuhKIDc3N2M5KioqW8sxc1WSrZzEdZuVfPJNM1xcXBQZGWkMVE+Lk5NTmgPh7yVRUVEZPhf2SPy6xsTEpAjY6YmOjk6ybGbfxMeybNkyY/nWrVs6deqUFi9erKCgIE2dOlVhYWEaMWKE6fak5e7du4qJidHx48ftLgtJubu7q379+rndjALlxIkTioyMzO1mIJsV6NBUqFAhYzkmJibdbRN/ECWfliB5OYl/zmw5kZGRGbYlcTnp1ZUWb2/vTH0LjYqK0vnz5+Xu7p6izQVVdj8PiV9XV1dXU72PNomDuZubm6l9E890n3ym+TZt2sjHx0dDhgzRiRMn9PXXX6tNmzby9vY23aa06nR1dVXNmjX5vUK+l95pbeR9cXFxpjoUCnRoSnwqLaNv44m/QST/EEpczp07d9INMhmVExkZaapnwFZORqcDU+Ps7Jyp0OTs7CyLxWL8Q/afgkxcfmaf96zsm3yf5IoXL66PP/5Yffr0UWxsrD766CMtXrzYdJvSqtNisWT69xHIi/gdLhgK9NVziQdcZzSgOvF622Du1MqxDeZOi20QucViSTHg2zaI3Mzgbls5efU2Krj31K1bV927d5ckHThwQH/++WduNwkAclSBDk01a9Y0lpNPI5Bc4vXJpxW47777Ml1OhQoVUvQ02dpz+/btdGf6vnr1qsLDw1PUDWS3sWPHGqfyvvnmm9xuDgDkqAIdmipXriwvLy9J0v79+9Pd1ra+XLlyqly5cpJ1TZs2NZb37duXZhnXrl0zJgds0qRJivWJy0mvPYnXpVYOkF3uu+8+Pfroo1LCxKp79uzJ7SYBQI4p0KHJYrEYN/g9c+aMfH19U93O19fX6CHq2LFjijEf1atXN3p8Nm3alOYVFKtXrzaWO3XqlGJ9hw4djG/xK1euTLPdq1atkhIG0nbo0CHD40T+duzYMa1atSrDf7Yb6Wa3sWPHGsv0NgEoSAr0QHBJGjFihJYtW6a4uDhNmzZNixcvTnIlT1RUlKZNmyYlXHaf1qXWI0eO1FtvvaWbN29qxowZmjJlSpL1Fy5c0OzZs6WEm/favq0nVrZsWfXo0UNr167Vzp07tWnTJnXt2jXJNr/++qt27twpSerVqxd3mS8Atm3bZtwjMT1ff/11jtwPrn79+mrXrp127NihPXv2yNfXN837NgLAvSRfh6YDBw7owoULxs83btwwls+fP2/0yNj07ds3RRnVq1fXqFGjNGfOHB09elSDBg3SM888oypVqigwMFDfffed/P39JUmjRo1StWrVUm1Lnz59tHLlSh06dEiLFy/W9evX5ePjo5IlS+rIkSOaNWuWwsPD5eTkpLfeeksuLqk/9ePHj9dff/2l0NBQvfrqqzp69KgeeeQRSdL27dv1ww8/SJI8PT318ssvZ+l5ywnlit2b9yW7V48rs8aOHasdO3ZIkmbNmqU5c+bkdpMAINtZrNk9tXE2euONN5Kc8srIiRMnUn08Pj5eb7/9drqnxPr3769p06Ylmc8mudDQUI0ePTrNuR7c3Nw0ZcoU+fj4pNvOw4cP6/nnn09zMHjZsmX19ddfZ/pWFnFxcUavQGbnaTp79qyqV69uaj6deGu8nCz37pnfe/34ckpmf6+QeZ/9vYEb9mazSiU89Urr7rndDNjJ7Odjvu5pchQnJydNnz5dXbp00dKlS+Xn56cbN26oVKlS8vb21sCBA9WuXbsMy/H09NSSJUu0bNkybdiwQQEBAYqMjJSXl5datmyp4cOHq1atWhmWc//992vdunVauHChtm3bpqCgIClh4HrHjh01YsQIlSpVyiHHnh3u9UBxrx8fACB1+bqnCZmTUz1NgBn8XmU/epqyHz1N9wazn498ZQYAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABEITAACACYQmAAAAEwhNAAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQBAAAYIJLbjcAQEp79+7V8OHDU11XuHBheXh4qG7dunr00UfVs2dPubm5SZI6dOig4OBgu+peuHChHnzwQQUFBaljx46SpD59+ujDDz+0q1wAyO/oaQLymaioKF2+fFnbt2/XW2+9pb59+yooKCi3mwUA9zx6muBw1vh4WZzu3Tye08c3aNAgDR482Pg5JCREp06d0ty5c3X58mWdOnVKzz77rNasWaO5c+cqJiYm1XImTZqko0ePSpLWr1+fZn2VK1fOhqMAgPyP0ASHszg5KWzrz4q7cTW3m+JwzqW8VLLToByts3Tp0qpdu3aSx1q2bKm+ffuqZ8+eCg4O1smTJ7VlyxZ17do1zXKKFCliLCcvDwCQMUITskXcjauKvW7f2Bqkr1ixYnr22Wf19ttvS5J27dqVbmgCANjn3j2HAhQAderUMZYvX76cq20BgHsdoQnIx1xdXY1lFxc6jgEgOxGagHwsICDAWK5UqVKutgUA7nWEJiCfiouL09y5c42fu3TpkqvtAYB7HaEJyGdCQ0O1e/duDR06VP7+/lJCYGrWrFluNw0A7mkMggDyuK+++kpfffVVquvc3d31xBNP6NVXX83xdgFAQUNPE5CP1a1bV8OGDUsyIBwAkD3oaQLyuMQzgsfFxeny5cvavHmz1q5dq3/++UfDhg3TihUr5OnpmdtNBYB7Gj1NQB5nmxG8du3aqlevntq3b68PP/xQ06dPlyQFBwfrrbfeyu1mAsA9j9AE5FN9+vQxrpj7/ffftXv37txuEgDc0whNQD42fvx4OTs7S5JmzpyZ280BgHsaoQnIx6pXr65u3bpJkg4fPqy///47t5sEAPcsQhOQz40ZM0YWi0WS9M033+R2cwDgnsXVc0A+V7t2bXXo0EHbtm3T/v37deDAAYdPdHn+/HmtWrUqw+0aNWqkmjVrOrRuAMgrCE3APWDs2LHatm2blNDblPj2Ko5w6NAhHTp0KMPtJk2aRGgCcM8iNCFbOJfyyu0mZIu8elyNGjVS69at9ffff2vnzp06cuSIGjVqlNvNAoB7CqEJDmeNj1fJToNyuxnZxhofL4tT9g4HfPDBB3XixIlM7TNv3rx01y9atChT5VWuXDnTbQCAexkDweFw2R0octu9fnwAgNTx1x8AAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABEITAACACYQmmGa1WnO7CbiH8PsEIL8hNCFDzs7OkqTY2NjcbgruIbbfJ9vvFwDkdYQmZMjFxUWFChVSWFhYbjcF95CwsDAVKlRILi4uud0UADCF0IQMWSwWeXh46Pbt27px40ZuNwf3gBs3buj27dvy8PCQxWLJ7eYAgCl8xYMppUqVUnR0tC5fvqxbt26pWLFiKly4sJycnPjQQ4asVqvi4+MVFRWl8PBwRUREqFSpUipVqlRuNw0ATCM0wRSLxaLy5cvL3d1dt27d0vXr1xUfH5/bzUI+4+TkpCJFiqhixYoqWbJkbjcHADKF0IRMKVmypEqWLKn4+HjFxsYSnGCak5OTXFxc5OTEqAAA+ROhCVni5OQkNze33G4GAAA5hq98AAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQBAAAYAKhCQAAwARCEwAAgAmEJgAAABMITQAAACYQmgAAAEwgNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABEITAACACYQmAAAAEwhNAAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQBAAAYAKhCQAAwARCEwAAgAmEJgAAABMITQAAACYQmgAAAEwgNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABJfcbkBeER0drbVr12rTpk06ceKEbt68KVdXV3l5ealJkyby8fFRkyZNMixnx44dWrZsmfz8/BQaGipPT095e3trwIABateunam2xMbGavny5Vq/fr3OnDmjiIgIeXl5qVWrVho2bJhq1arlgCMGAACZYbFardbcbkRuCw4O1pgxY3Tq1Kl0txs2bJjeeustWSyWFOvi4+M1efJkrVixIs39fXx8NHXqVDk5pd3BFxoaqtGjR8vPzy/V9W5ubpoyZYp8fHzSbWtq4uLi5Ovrq8aNG8vZ2TnT+wPIXz77e4OCb4XmdjPuaZVKeOqV1t1zuxmwk9nPxwLf0xQTE5MkMNWpU0dPPfWUqlevrjt37ujgwYP64YcfFBERoUWLFsnLy0ujR49OUc7MmTONwFS/fn09/fTTqlKligIDA/X999/L399fy5cvl6enp1555ZVU2xIXF6dx48YZgalz587y8fGRh4eHDh8+rG+++UYhISGaMmWKvLy8TPdcAQAA+xX4nqZNmzbppZdekiQ98MADWrx4cYqUefToUT3xxBOKiYlRiRIltHv3brm4/F/ePHv2rLp3767Y2Fg1bNhQixcvVuHChY31kZGRGjp0qI4ePSoXFxdt3LhRVatWTdGWFStW6K233pIkDR48WO+8806S9efPn1ffvn0VHh6uqlWrauPGjUnakRF6moCChZ6m7EdP073B7OdjgR8I/s8//xjLo0ePTvXJatiwoR555BFJ0q1btxQQEJBk/YIFCxQbGytJmjx5cpLAJEnu7u6aPHmylDBeaf78+am2Zd68eZIkDw8PTZw4McX6qlWrasyYMVJCgNqyZUumjxcAAGRNgQ9NMTExxnKVKlXS3C7xusT7WK1Wbdu2TZJUo0YNNW7cONX9GzdurOrVq0uStm3bpuQdfGfPnjXCWNeuXeXu7p5qOX369DGWt27dmuHxAQAAxyjwockWZCQpMDAwze1s6ywWi6pVq2Y8HhQUpKtXr0qSmjdvnm5dLVq0kCRduXJFQUFBSdYdPHgwxXapKVu2rFH/oUOH0q0PAAA4ToEPTY8//riKFSsmSfruu+8UFxeXYht/f39t375dktS9e3dje0k6ffq0sVyjRo1060q8/syZM0nWJT7lZ7acS5cuKSIiIt1tAQCAYxT40OTp6amPP/5Y7u7uOnTokPr37681a9bI19dXu3bt0ldffaWhQ4cqJiZGDRo00BtvvJFk/8uXLxvL5cuXT7euxOsvXbqUZjnlypVLt5wKFSpICacGE+8HAACyT4GfckCSOnbsqJUrV+qHH37QihUr9PrrrydZX6ZMGb300ksaMGBAirFGd+7cMZaLFCmSbj2J903eQ5S4nKJFi2a5HDNS600DcG/hCtmcxd/V/M3s60doSjQbeGoDtCXp+vXrWrdunSpXrqyOHTsmWXf37l1j2dXVNd163NzcjOWoqKhsKceMtCbOBHBvcHd3V/369XO7GQXKiRMnFBkZmdvNQDYr8KEpIiJCzzzzjA4cOCBnZ2c9/fTT6tu3r6pUqaLo6GgdPnxYX3/9tQ4ePKjnn39er7/+up566ilj/0KFChnLia+qS010dLSxnHxaguTlJP45M+WY4e3tzbdQAHCgOnXq5HYTYIe4uDhTHQoFPjR9+eWXOnDggCTp/fffT3JJv5ubm1q3bq0HH3xQI0eO1N69e/Xxxx+rZcuWqlu3rpTsVFpGp8oSfwtJfiovcTl37txJNzSlV44Zzs7OhCYAcCD+phYMBXoguNVq1apVqyRJ1apVSxKYEnNxcTFmDY+Pjzf2UbLB3RkNyk683jaYO7Vyrly5km45tkHkFoslw8HnAADAMQp0aLp+/bpu3rwpJdwvLj0NGzY0lhNPF1CzZs1UH09N4vXJpxW47777Ml1OhQoVstTTBAAAMq9Ah6bE3akZjZxPPF4p8f3eKleuLC8vL0nS/v370y3Dtr5cuXKqXLlyknVNmzY1lvft25dmGdeuXdO5c+ckSU2aNEm3PgAA4DgFOjR5eHgYE1X+888/xv3jUpM4ECUOPBaLxbii7syZM/L19U11f19fX6OHqGPHjrJYLEnWV69e3eht2rRpU5pXYaxevdpY7tSpk6njBAAA9ivQocnJycm4Ee/Vq1f17bffprpdWFiYPvnkE+Nn2z42I0aMMHqtpk2blmIagKioKE2bNk1K6KUaMWJEqvWMHDlSknTz5k3NmDEjxfoLFy5o9uzZUsLNex999NFMHS8AAMi6An/13HPPPadt27YpMjJSX375pY4ePao+ffqoSpUqunv3rg4fPqwFCxbo4sWLkqSWLVuqTZs2ScqoXr26Ro0apTlz5ujo0aMaNGiQnnnmGVWpUkWBgYH67rvv5O/vL0kaNWpUknvXJdanTx+tXLlShw4d0uLFi3X9+nX5+PioZMmSOnLkiGbNmqXw8HA5OTnprbfeSnKaEAAAZC+LNbXZHAuYXbt26ZVXXtGNGzfS3e6hhx7S//73P5UsWTLFuvj4eL399ttauXJlmvv3799f06ZNk5NT2h18oaGhGj16dJrzRbi5uWnKlCny8fFJt62piYuLk6+vrxo3bszlsUAB8NnfGxR8KzS3m3FPq1TCU6+07p7bzYCdzH4+0lUhqVWrVvr111+1YsUK/fnnnzp9+rRu374tZ2dnlSlTRt7e3urevXuqY5FsnJycNH36dHXp0kVLly6Vn5+fbty4oVKlSsnb21sDBw5Uu3btMmyLp6enlixZomXLlmnDhg0KCAhQZGSkvLy81LJlSw0fPly1atXKhmcBAACkh56mAoSeJqBgoacp+9HTdG8w+/lYoAeCAwAAmEVoAgAAMIHQBAAAYAKhCQAAwARCEwAAgAmEJgAAABMITQAAACYQmgAAAEwgNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYYFdomjp1qvz9/R3XGgAAgDzKrtD0008/qV+/furdu7cWLVqkmzdvOq5lAAAAeYhdocnFxUVWq1XHjx/X9OnT1bZtW7388sv6888/ZbVaHddKAACAXGZXaNq5c6fefPNN1atXT1arVdHR0dq8ebPGjBmj9u3b6/PPP9eFCxcc11oAAIBcYldo8vDw0PDhw7V69WqtXr1aQ4cOVcmSJWW1WnX58mXNnj1bXbp00bBhw7RmzRpFRUU5ruUAAAA5yGFXz9WrV09vv/22/vrrL33xxRdq27atnJycZLVadeDAAU2aNElt2rTRlClT5Ovr66hqAQAAcoSLowt0dXVVly5d1KVLF129elVr1qzRqlWrdO7cOYWHh2v58uVavny5atSooX79+qlXr14qXbq0o5sBAADgUNk6T5OXl5dGjx6tTZs26bvvvlOZMmUkSVarVWfOnNGMGTPUrl07jR8/XseOHcvOpgAAANjF4T1Nye3fv1+rVq3Spk2bFBUVZVxV5+7uroiICMXGxmrTpk3avHmzBg8erDfffFNOTsy5CQAA8pZsCU2XLl0yBocHBQVJCb1LFotFrVu3Vv/+/dWpUyddvHhRK1as0LJlyxQWFqbFixfrP//5j4YPH54dzQIAAMgyh4Wm6Oho/fbbb1q1apX27Nkjq9Vq9CqVL19effv2Vb9+/VSpUiVjn6pVq+rVV1/V6NGj9fzzz2vfvn1atmwZoQkAAOQ5doemI0eOaOXKlfr11191+/ZtKaFXycXFRe3bt1f//v3Vtm1bWSyWNMsoXry4XnzxRQ0dOpR5nQAAQJ5kV2h6/PHHdebMGSkhKElStWrV1L9/f/Xp0ydTV8V5eXlJkmJiYuxpEgAAQLawKzQFBARIkgoXLqwuXbrIx8dHzZo1y1JZxYoVU+/evdPtkQIAAMgtdoWm+vXrq3///urZs6eKFStmV0M8PT314Ycf2lUGAABAdrErNK1atcpxLQEAAMjD7ApNX331lSRp8ODB8vT0NLVPWFiYFi1aJEkaN26cPdUDAADkGLtDk8ViUdeuXTMVmmz7EZoAAEB+wdTbAAAAJuR4aLJNKeDiku13cAEAAHCYHA9Nx48flxKulgMAAMgvMtXds2bNmlQf37Ztm44ePZruvtHR0bpw4YJWrlwpi8Uib2/vzLUUAAAgF2UqNL3xxhspJp+0Wq36/PPPTZdhtVrl5OTE/eUAAEC+kumBRbbbpWT0WGpcXV3l7e2tMWPGqEWLFpmtGgAAINdkKjRt27bNWLZarerUqZMsFovmzp2rqlWrprmfxWJRoUKF5OHhIWdnZ/taDAAAkAsyFZoqVaqU6uNeXl5prgMAALgX2HXdv+1KOAAAgHsdk1sCAACYQGgCAAAwwdTpuUmTJkkJA7qnT5+e4vGsSF4WAABAXmYqNK1evdqYnylx0En8eGZYrVZCEwAAyFdMhaaKFStm6nEAAIB7janQ9Pvvv2fqcQAAgHsNA8EBAABMIDQBAACYYFdoCg0NzfK+f/75pz1VAwAA5Ci7QlOPHj20Y8eOTO0THR2td999V2PGjLGnagAAgBxlV2gKCQnR2LFj9e677+ru3bsZbu/v76/evXtryZIl9lQLAACQ4+wKTRUrVpTVatWSJUvUp08f+fv7p7qd1WrV7NmzNXDgQJ09e1ZWq1WPP/64PVUDAADkKLtC07p169SjRw9ZrVadPXtWAwcO1Jw5c2S1Wo1tgoODNXToUH3++eeKiYlRsWLFNGPGDH3yySeOaD8AAECOsCs02QLQp59+quLFiysmJkYzZ87U8OHDdfHiRa1Zs0a9evXSoUOHZLVa1bx5cyNoAQAA5CemJrfMyOOPP64mTZpo4sSJ2r9/vw4cOKAuXbooNjZWVqtVLi4ueumll/T0009n6bYrAAAAuc1h8zRVqFBBCxcu1GOPPSar1WoEpuLFi2vZsmV65plnCEwAACDfclhoio2N1WeffabNmzfLYrEY45rCw8M1c+ZMXb9+3VFVAQAA5DiHhKazZ89qwIAB+v777xUXF6fSpUvrgw8+kLe3t6xWq3bu3KkePXpo69atjqgOAAAgx9kdmn7++Wf17dtXx44dk9VqVYcOHbRu3Tr16dNHS5Ys0bPPPisnJyfduHFDL7zwgt5++21FRkY6pvUAAAA5xK7QNHbsWE2dOlWRkZEqXLiwpk6dqlmzZsnT01OS5OzsrJdeekk//vijKleuLKvVqpUrV6p37946cuSIo44BAAAg29kVmrZv3y6r1aqGDRtq9erVGjBgQKrbPfDAA0bvk9Vq1fnz5zV48GB7qgYAAMhRdoUmJycnjR07VkuWLFG1atXS3bZIkSL64IMP9OWXX6pkyZKKi4uzp2oAAIAcZdc8TYsWLVLTpk0ztc+jjz6q+++/X2+++aY9VQMAAOQou3qaMhuYbLy8vPT999/bUzUAAECOctg8TTbx8fEKDQ3VxYsXOQUHAADuGQ65jUpcXJxWrVql1atXy8/PT7GxsbJYLFq3bp1q1qxpbPfHH39o//79KlasmJ577jlHVA0AAJAj7A5NISEhev7553X48GFjFvC0VKpUSc8++6wsFovat2+vevXq2Vs9AABAjrDr9FxcXJzGjh0rX19fWSwWdevWTZMnT05z+9q1a+v++++XJG3ZssWeqgEAAHKUXaHJdjrOxcVFs2fP1syZMzVkyJB09+nQoYOsVqsOHjxoT9UAAAA5yq7Q9Msvv8hiseiJJ57Qww8/bGof2ym5s2fP2lM1AABAjrIrNJ04cUJK6D0yq3Tp0pKkmzdv2lM1AABAjrIrNN26dUuS5OHhYXof2zQEzs7O9lQNAACQo+wKTbawdOnSJdP7nD9/XpJUqlQpe6oGAADIUXaFJtscTH5+fqb32bhxoywWi7y9ve2pGgAAIEfZFZo6deokq9WqxYsXKywsLMPtN23apD/++EOS1KVLF3uqBgAAyFF2haYBAwaoYsWKCg8P18iRI3X69OlUtwsJCdHMmTP12muvyWKxqFatWurWrZs9VQMAAOQou2YEd3Nz06xZszRs2DD9+++/6tGjh6pXr26snzBhgiIiIhQYGCir1Sqr1SoPDw99+eWXslgsjmg/AABAjrD7hr1169bVihUr1LhxY1mtVp05c8ZYd/z4cZ0/f17x8fGyWq1q1KiRli9frqpVq9pbLQAAQI5yyA17q1atqiVLlujAgQP6/fffdfToUYWGhiouLk4eHh6qX7++OnTooNatWzuiOgAAgBznkNBk06xZMzVr1syRRQIAAOQJdp+eAwAAKAgITQAAACYQmgAAAEwwNaapXr16Dq/YYrHI39/f4eUCAABkB1OhyWq1Zn9LAAAA8jBToWncuHHZ3xIAAIA8jNAEAABgAgPBAQAATCA0AQAAmODQGcElKSQkRCdPntTNmzclSR4eHqpVq5bKlCnj6KoAAAByjENCk9Vq1ZIlS/TTTz/p9OnTqW5Ts2ZNDRo0SE888YScnOjgAgAA+YvdoSkkJERjx47V0aNHpXSmJzh9+rSmTZumlStX6ttvv1XZsmXtrRoAACDH2BWaoqOjNWLECAUEBMhqtcrT01PdunWTt7e3cTru+vXrOnr0qH799VeFhITo33//1VNPPaVVq1bJzc3NUccBAACQrewKTfPnz9fp06dlsVjUv39/vfnmmypSpEiK7Xr37q1XX31VH3zwgZYtW6aAgADNnz9fo0ePtqd6AACAHGPX4KJffvlFFotFrVq10nvvvZdqYLJxd3fX1KlT1bp1a1mtVv3yyy/2VA0AAJCj7ApNFy5ckCQNHjzY9D62bW37AgAA5Ad2hSbbmKQKFSqY3se2LeOZAABAfmJXaKpevbok6dKlS6b3sW1r2xcAACA/sCs09e3b15ijyawlS5bIYrGod+/e9lQNAACQo+wKTT4+PmrTpo127typ//73v7p7926a20ZHR2vq1Kn666+/1Lp1aw0cONCeqgEAAHKUXVMOHDhwQE899ZTCwsK0dOlSbd261ZinqXTp0rJYLLp+/br8/Py0adMmXb9+Xd7e3ho5cqQOHDiQZrnNmze3p1l2uXjxolasWKHt27fr4sWLunPnjjw9PVWpUiU9+OCD6tatm2rXrp3m/jt27NCyZcvk5+en0NBQeXp6ytvbWwMGDFC7du1MtSE2NlbLly/X+vXrdebMGUVERMjLy0utWrXSsGHDVKtWLQceMQAAMMNiTWsKbxPq1q0ri8Xi2AZZLPL393domWYtWrRIn332mSIiItLcZvjw4XrrrbdSPB4fH6/JkydrxYoVae7r4+OjqVOnpnsbmdDQUI0ePVp+fn6prndzc9OUKVPk4+OT4fEkFxcXJ19fXzVu3FjOzs6Z3h9A/vLZ3xsUfCs0t5txT6tUwlOvtO6e282Ancx+Ptp9GxU7MleeMmvWLH3xxReSpGrVqmnAgAHy9vZW8eLFdfPmTfn7+2vLli1pBp6ZM2cagal+/fp6+umnVaVKFQUGBur777+Xv7+/li9fLk9PT73yyiuplhEXF6dx48YZgalz587y8fGRh4eHDh8+rG+++UYhISGaMmWKvLy8TPdcAQAA+9nV07Rv3z7HtiZBixYtsqXctOzevVtPPvmklDB7+XvvvSdXV9dUt42Ojk4xXcLZs2fVvXt3xcbGqmHDhlq8eLEKFy5srI+MjNTQoUN19OhRubi4aOPGjapatWqKslesWGH0Yg0ePFjvvPNOkvXnz59X3759FR4erqpVq2rjxo1ycTGfe+lpAgoWepqyHz1N94Yc6WnK6XCTHeLj4/Xf//5XSjjd+P7776cbRFKbX2rBggWKjY2VJE2ePDlJYFLCbOiTJ0/WwIEDFRsbq/nz56cIRJI0b948SZKHh4cmTpyYYn3VqlU1ZswYffrppzp//ry2bNmibt26ZeGoAQBAZtl19dzFixd18eJF3bx503EtymE7d+7UuXPnJEnPPPNMpnpulHB6ctu2bZKkGjVqqHHjxqlu17hxY2Nuqm3btqU4rXn27FkFBARIkrp27Sp3d/dUy+nTp4+xvHXr1ky1FQAAZJ1doalDhw7q2LFjvr6P3KZNm6SEAeiPPPKI8fjNmzd17ty5DANhUFCQrl69Kpm46s/WM3flyhUFBQUlWXfw4MEU26WmbNmyqlatmiTp0KFD6dYHAAAcx67QZDsN5e3t7aj25LjDhw9LkipVqqRixYpp/fr16tGjhx588EF16dLF+H/u3LmKjo5Osf/p06eN5Ro1aqRbV+L1Z86cSbLO1suUmXIuXbqU7pV+AADAcewKTeXKlZMSxgXlR/Hx8UZ4KVWqlN577z299tprOnnyZJLtzp07p48//ljDhw/XrVu3kqy7fPmysVy+fPl060u8PvmtZxKXY3te02K7f5/Vak2yHwAAyD52DQRv3bq1Lly4oIMHD6Y5licvu337thH4Tp48KT8/P5UtW1YTJ05Uu3btVKhQIfn5+emTTz6Rr6+v/vnnH7355pv66quvjDLu3LljLBcpUiTd+hKPU0reQ5S4nKJFi2a5HDPi4uIyvQ+A/IUrZHMWf1fzN7Ovn12hafjw4Vq9erXmzZun7t27Z9hDktdERkYay3fv3pW7u7sWLlyY5PRY8+bNtWDBAg0cOFDHjx/Xli1bdPjwYd1///3GfjZpTVNgk/jKu6ioqCTrHFWOGWlNnAng3uDu7q769evndjMKlBMnTiT5TMG9ya7QVK1aNX3yySeaMGGCBgwYoNdee01dunRJ9bL8vCh5O/v375/qeKLChQtr/PjxGjNmjCRp48aNRmgqVKiQsV1MTEy69SUeE5V8WoLk5ST+OTPlmOHt7c23UABwoDp16uR2E2CHuLg4Ux0Kdvc0SZKnp6eCgoI0ceJEvfXWW6patapKliyZ7u1CLBaLFixYYE/1ditWrFiSn9u0aZPmti1btpSLi4tiY2OTPLGJT6VldKos8beQ5KfyEpdz586ddENTeuWY4ezsTGgCAAfib2rBYFdo2rdvX5J7z1mtVkVHR+vUqVNp7mOxWGS1Wh1+z7qscHNzk6enp0JD//+MuekN5C5UqJBKlSqla9euGdsn3yejQdmJ19sGc6dWzpUrV+Tp6ZlmObZB5BaLJcPB5wAAwDHsCk0ZzUuUH9SsWdO4HUxGVwHaBoolngCzZs2axnLyaQSSS7w++WnA++67L8l29erVy7CcChUqZKmnCQAAZJ5doWnRokWOa0kuad68uRGaAgMD0xw8GR4erhs3bkjJpgSoXLmyvLy8dPXqVe3fvz/dumzry5Urp8qVKydZ17RpU2N53759evzxx1Mt49q1a8YM5k2aNDF5lAAAwF52zdN0L+jcubOxvGXLljS327Jli3Hrk8QBx2KxqGPHjlJCD5Cvr2+q+/v6+ho9RB07dkxxerJ69epGb9OmTZvSvApj9erVxnKnTp1MHSMAALBfgQ9NdevWVdu2bSVJv/zyi3bv3p1im2vXrunzzz+XEqYD6NevX5L1I0aMMAYBTps2LcU0AFFRUZo2bZqUcGpvxIgRqbZl5MiRUsItXGbMmJFi/YULFzR79mwp4ea9jz76aJaOGQAAZJ5dp+dSc/nyZV27dk1RUVHy9vbO0iXxOe3NN9+Ur6+vbt26pTFjxmjEiBHG5JZHjhzRnDlzjEHcL730Uor5qKpXr65Ro0Zpzpw5Onr0qAYNGqRnnnlGVapUUWBgoL777jv5+/tLkkaNGmXcOy65Pn36aOXKlTp06JAWL16s69evy8fHRyVLltSRI0c0a9YshYeHy8nJSW+99Vamby4MAACyzmK1nXOyQ3h4uL7//nutXr3auHmtJK1fvz7JQOlffvlFv/32m4oXL6733nvP3mod6sCBA3rppZd0/fr1VNdbLBaNHTtWL7/8cqrr4+Pj9fbbb2vlypVp1tG/f39NmzYt3akYQkNDNXr06DTni3Bzc9OUKVPk4+OT4TElFxcXJ19fXzVu3JjLY4EC4LO/Nyj4VqiJLZFVlUp46pXW3XO7GbCT2c9Hu7sqzp07p9GjRyswMFCJ81dqUwrcf//9mjBhgqxWq3r37q1mzZrZW73DNGvWTBs2bNCPP/6orVu3KigoSDExMSpbtqxatGihYcOGpTvDrpOTk6ZPn64uXbpo6dKl8vPz040bN1SqVCl5e3tr4MCBateuXYbt8PT01JIlS7Rs2TJt2LBBAQEBioyMlJeXl1q2bKnhw4erVq1aDj56AACQEbtC0927dzVmzBhduHBB7u7uGjJkiJo3b27MnJ1c5cqV9eCDD2rPnj36/fff81RoUsJNe1944QW98MILWS6jXbt2psJRelxcXDR48GANHjzYrnIAAIDj2BWafv75Z50/f17u7u766aef0p1byKZt27bavXt3mleZAQAA5EV2XT3322+/yWKxaPjw4aYCkxKuVpOk8+fP21M1AABAjrIrNAUEBEgZ3LMtOQ8PD0nSrVu37KkaAAAgR9kVmmw3qM3MrTyio6OlZLciAQAAyOvsCk22XqPg4GDT+9hu5lu2bFl7qgYAAMhRdoWmBg0aSInuqWbG2rVrZbFY1LhxY3uqBgAAyFF2haYuXbrIarVq2bJlunjxYobbz58/3whYad2QFgAAIC+yKzT16tVLderU0d27dzVs2DDt2LEjxQSXVqtVR44c0auvvqqPPvpIFotFzZo1s3suIwAAgJxk12hsJycnffPNNxo8eLCCg4M1duxYFS5c2JgNfNiwYbpz544x+Ntqteo///mPcfNbAACA/MKuniZJqlixotasWaPHH39cTk5OioyMlNVqldVqVWhoqO7evWv0PnXr1k3Lly9X6dKlHdF2AACAHOOQ6/49PDz06aef6pVXXtH27dt19OhRhYaGKi4uTh4eHqpfv77at2+v6tWrO6I6AACAHOfQyZIqVaqkIUOGOLJIAACAPCFLoWn79u3666+/FBwcrPj4eHl5ealFixbq1q2bXF1dHd9KAACAXJap0HT9+nU9//zzOnLkSIp1K1eu1P/+9z99/fXXqlOnjiPbCAAAkOtMDwSPi4vTs88+q8OHDxsDvZP/CwoK0qhRoxQaGpq9rQYAAMhhpkPTr7/+Kj8/P1ksFlWtWlXvv/++1q9fr19//VVffPGFMcN3SEiIfvjhh+xsMwAAQI7LVGhSwmDv5cuXq1+/fqpVq5aqV6+uLl26aPHixWrevLmsVqs2bdqUnW0GAADIcaZD07Fjx2SxWPTUU0+pRIkSKdY7OzvrxRdflCQFBQUpPDzcsS0FAADIRaZDk22ckre3d5rbNGzY0Fi+ceOGvW0DAADIM0yHpqioKElSkSJF0tzG3d3dWLbdOgUAAOBeYPdtVNKS+Ma9AAAA+V22hSYAAIB7SaZnBP/pp5/k6enpkO3GjRuX2eoBAAByRaZD088//5zueovFYmo7EZoAAEA+kqnQ5MhxSrZwBQAAkB+YDk0LFy7M3pYAAADkYaZDU4sWLbK3JQAAAHkYV88BAACYQGgCAAAwgdAEAABgAqEJAADABEITAACACYQmAAAAEwhNAAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQBAAAYAKhCQCALCruVljW+PjcbkaBkdvPtekb9qYnNjZW27dv18GDBxUYGKg7d+4oLi4u3X0sFosWLFjgiOoBAMgV7q5usjg5KWzrz4q7cTW3m3NPcy7lpZKdBuVqG+wOTQcOHNDEiRN16dIl4zGr1Zrm9haLRVarVRaLxd6qAQDIE+JuXFXs9eDcbgaymV2hKSAgQM8884yioqJktVrl6uqqatWqqWTJkoQiAABwT7ErNM2ePVuRkZFydnbWCy+8oGHDhqlo0aKOax0AAEAeYVdo2rNnjywWi4YPH66xY8c6rlUAAAB5jF1Xz924cUOS1KlTJ0e1BwAAIE+yKzR5enpKkgoXLuyo9gAAAORJdoWmpk2bSpJOnTrlqPYAAADkSXaFpieffFLOzs5auHChYmNjHdcqAACAPMau0NSoUSNNmjRJx48f17hx4xQaGuq4lgEAAOQhdl0999VXX0kJ4Wn79u3q0KGDWrVqpRo1apga5zRu3Dh7qgcAAMgxdocm2ySWFotFUVFR+uOPP/THH3+Y2p/QBAAA8gu7b6OS/JYp6d1CBQAAIL+yKzQdP37ccS0BAADIw+waCA4AAFBQEJoAAABMIDQBAACYYPdAcJubN29q1apV2rVrl06dOqWwsDBJUsmSJVWrVi21atVKffv2lYeHh6OqBAAAyDEOCU1LlizRRx99pKioKCnZFXRRUVG6evWq/v77b3355Zd64403NHDgQEdUCwAAkGPsDk1z5szRzJkzjaBUvHhx1atXT2XLlpUkXbt2TceOHdPt27cVGRmp//73v7p165aeeeYZ+1sPAACQQ+wKTSdPntQXX3whq9WqsmXLauLEieratatcXV2TbBcbG6tNmzbp448/1tWrV/XFF1/okUceUa1atextPwAAQI6wayD4jz/+qLi4OHl6emrp0qXq0aNHisAkSS4uLurevbuWLl2q0qVLKy4uTj/++KM9VQMAAOQou0LT3r17ZbFYNHr0aFWsWDHD7StUqKBnnnlGVqtVe/bssadqAACAHGVXaLpy5YokqUmTJqb3sW179epVe6oGAADIUXaFJien/797bGys6X3i4uKkhBv8AgAA5Bd2hSbbKbndu3eb3se2rZnTeQAAAHmFXaGpVatWslqtmjdvnk6cOJHh9idPntTcuXNlsVjUunVre6oGAADIUXaFphEjRsjNzU0REREaPHiw5s6dqxs3bqTY7saNG5o7d66GDBmiO3fuyM3NTSNGjLCnagAAgBxl1zxNlSpV0tSpUzVp0iRFRETok08+0aeffqrKlSvL09NTFotFISEhCgoKktVqldVqlcVi0bvvvsvpOQAAkK/YPSN479695eHhoSlTpujq1auyWq26cOGCAgMDpWS3VPHy8tK0adPUrl07e6sFAADIUQ6599wjjzyi33//XVu2bNHu3bt18uTJJDfsrV27tlq2bKlOnTqlOvklAABAXueQ0KSEWb+7deumbt26OapIAACAPMOugeAAAAAFBaEJAADABEITAACACabGNNWrV09KuPWJv79/isezInlZAAAAeZmp0JR42gAzjwMAANxrTIWmcePGZepxAACAew2hCQAAwAQGggMAAJhg1+SW+/fvlyR5e3urcOHCpva5e/eujhw5Iklq3ry5PdUDAADkGLtC07Bhw+Tk5KR169apZs2apva5cuWKsR9XzwEAgPzC7tNzWb2CjivvAABAfpLjY5ri4+MlSc7OzjldNQAAQJbleGi6ePGiJKlYsWI5XTUAAECWZWpMky3wJHft2jUVKVIk3X2jo6N14cIFffHFF7JYLKpVq1bmWgoAAJCLMhWaOnbsmOIxq9WqkSNHZrriXr16ZXofAACA3JKp0OSI26kUKlRIw4YNU//+/TNTNQAAQK7KVGj64IMPkvw8adIkWSwWvfTSSypXrlya+1ksFrm5ucnLy0v16tVT0aJFs95iAACAXJCp0NSnT58kP0+aNEmS1KlTJ9PzNAEAAORHdk1uuXDhQklS5cqVHdUeAACAPMmu0NSiRQvHtQQAACAP44a9AAAAJtjV05SY1WrVsWPHdPz4cd24cUNRUVEZXlU3btw4R1UPAACQrRwSmlavXq2vvvoqzckv00JoAgAA+YXdoWnmzJmaM2eOqbmaLBYLN+oFAAD5kl1jmg4fPqzZs2dLklq3bq01a9Zo9erVUkJAOnbsmHbv3q3vvvtOHTp0kNVqVdOmTbVz504dP37cMUcAAACQA+wKTT///LMkqWLFipo9e7bq1q0rF5f/67yyWCwqVaqUHn74Yc2aNUtTpkzRwYMH9fTTTys6Otr+1gMAAOQQu0LTP//8I4vFomHDhiUJS2kZPHiwOnfurBMnTuinn36yp2oAAIAcZVdounr1qiSpVq1a/1eg0/8VGRMTk2KfXr16yWq16tdff7WnagAAgBxlV2iKjY2VJJUuXdp4rEiRIsZyaGhoin3Kly8vSTp//rw9VQMAAOQou0KTp6enJCk8PNx4rHTp0nJ2dpYknTlzJsU+165dkyTduXPHnqoBAABylF2hyXaT3sThyM3NzXh848aNKfZZu3atJMnLy8ueqgEAAHKUXaGpWbNmslqt2rt3b5LHH3vsMVmtVq1cuVL/+9//dOrUKR05ckT//e9/9euvv8pisaht27b2th0AACDH2BWaOnXqJEn6448/kpyiGz58uCpVqqT4+Hh988036tmzpwYOHKilS5dKkkqUKKExY8bY23YAAIAcY9eM4LVq1dLChQsVFxdnDAqXJHd3dy1cuFATJkzQoUOHUuwzY8YMY0B4XjZjxgx9//33xs8LFy7Ugw8+mO4+O3bs0LJly+Tn56fQ0FB5enrK29tbAwYMULt27UzVGxsbq+XLl2v9+vU6c+aMIiIi5OXlpVatWmnYsGFJrlYEAAA5w+7bqLRo0SLVxytVqqSffvpJZ86c0enTpxUbG6tq1aqpfv369laZI44dO6b58+eb3j4+Pl6TJ0/WihUrkjx+5coVXblyRVu3bpWPj4+mTp2aZFqG5EJDQzV69Gj5+fkleTwwMFBLly7V6tWrNWXKFPn4+GThqAAAQFY55Ia96alRo4Zq1KiR3dU4lC0AxcbGqnTp0goJCclwn5kzZxqBqX79+nr66adVpUoVBQYG6vvvv5e/v7+WL18uT09PvfLKK6mWERcXp3HjxhmBqXPnzvLx8ZGHh4cOHz6sb775RiEhIZoyZYq8vLxM91wBAAD72TWm6V61cOFC+fn5qUaNGurfv3+G2589e1bz5s2TJDVs2FA///yzHn/8cTVq1EiPP/64fvrpJzVs2FCSNHfu3DTnqFq9erUOHjwoJcye/uWXX6pt27Zq1KiRhg0bpp9//lnFihVTfHy83n///SSnRAEAQPYiNCVz8eJFffHFF5Kkd999V66urhnus2DBAiPATJ48WYULF06y3t3dXZMnT5YSxiulddrPFrw8PDw0ceLEFOurVq1qDKA/f/68tmzZkunjAwAAWWPq9NxXX32VLZWPGzcuW8q1x9SpUxUREaE+ffqoRYsWKaZTSM5qtWrbtm1SwqnIxo0bp7pd48aNVb16dZ09e1bbtm3TlClTZLFYjPVnz55VQECAJKlr165yd3dPtZw+ffro008/lSRt3bpV3bp1y/KxAgAA80yHpsQf8I6S10LTxo0b9ccff6TZ05OaoKAg4x58zZs3T3fbFi1a6OzZs7py5YqCgoJUpUoVY53ttJzSGVwvSWXLllW1atV07ty5FFcmAgCA7GP69JzVak33X1a2yUtu3bql6dOnS5Jee+014xYxGTl9+rSxnNGA98Trk99ixtbLlJlyLl26pIiICFPtBAAA9jHV03T8+PE01wUFBWn8+PHy8/NT27Zt1a9fPzVq1Mi4iW9ISIj8/Py0YsUK/fnnn/L29tbMmTNVuXJlxx2FA8yYMUPXrl1TkyZNTA3+trl8+bKxnNHcU4nXX7p0Kc1yypUrl245FSpUkBIC6OXLl/Pd1YkAAORHdk05cPv2bY0cOVLBwcH66KOP1KtXrxTbVKhQQRUqVFDnzp21bt06vfHGGxo5cqRWrlyp4sWL21O9wxw4cEDLly+Xi4uL3n333Uydikx84+EiRYqku23icUrJe4gSl1O0aNEsl2NGXFxcpvcBkL/YbpwO3Guy4zPMbJl2hab58+frwoULeuKJJ1INTMn17NlTBw8e1NKlSzVv3jy99NJL9lTvENHR0Zo8ebKsVqtGjBih2rVrZ2r/u3fvGssZXWnn5uZmLEdFRWVLOWYknzgTwL3F3d0930wkDGTWiRMnFBkZmSt12xWafvvtN1ksFnXt2tX0Pt26ddPSpUu1ZcuWPBGaZs+erTNnzqhixYpZGpheqFAhYzkmJibdbaOjo43l5NMSJC8n8c+ZKccMb29vvoUCAPKlOnXqOLzMuLg4Ux0KdoWmoKAgScrUaTbbtsHBwfZU7RABAQGaPXu2JOntt9/O8PRaahKfSsvoVFniZJy8rsTl3LlzJ93QlF45Zjg7OxOaAAD5Um5+ftkVmlxdXRUVFaWTJ0+qQYMGpvY5efKksW9uW7BggWJiYlSlShVFRUXpl19+SbHNqVOnjOU9e/bo+vXrkqT27durSJEiSQZ3Jx7MnZrE622DuW0Sl3PlypV0r96zDSK3WCz54sbHAADcC+wKTXXq1NH+/fv13XffpTsho01kZKS+++47WSyWbOleyyzbaa7AwMA07weX2KxZs4zlbdu2qUiRIqpZs6bxWPJpBJJLvD75FW/33Xdfku3q1auXYTkVKlTIUk8TAADIPLtuozJgwAApYTbrYcOG6dixY2lue/z4cQ0fPtz4wLftm99VrlxZXl5ekqT9+/enu61tfbly5VJMudC0aVNjed++fWmWce3aNZ07d06S1KRJE7vaDgAAzLOrp6lHjx7aunWrNm/erH///Vd9+/ZV7dq15e3tnWKeJttpOUl69NFH1aNHD/tbb6cPP/xQH374YbrbfPnll8ZtZBYuXKgHH3wwyXqLxaKOHTvq559/1pkzZ+Tr65vqrVR8fX2NwNixY8cU0xpUr15d9913nwICArRp0ya98cYbqfbcrV692lju1KlTJo8YAABkld037P3ss880bNgwWSwWWa1WnThxQitXrtScOXM0Z84crVy5UidPnjRmAB86dKg+++wzR7Q9zxgxYoQxMG3atGkppgGIiorStGnTJEkuLi4aMWJEquWMHDlSknTz5k3NmDEjxfoLFy4YA9erVq2qRx991OHHAgAAUmdXT5MSRrG/9dZb8vHx0ZIlS7Rr1y6dP38+yW1SqlatqlatWmngwIGqW7euvVXmOdWrV9eoUaM0Z84cHT16VIMGDdIzzzyjKlWqKDAwUN999538/f0lSaNGjVK1atVSLadPnz5auXKlDh06pMWLF+v69evy8fFRyZIldeTIEc2aNUvh4eFycnLSW2+9JRcXu18+AABgksM+dWvXrq0pU6ZICQOsb926JUkqUaJEkskY71Xjx49XSEiIVq5cKX9/f40fPz7FNv3799fLL7+cZhnOzs76+uuvNXr0aPn5+Wnz5s3avHlzkm3c3Nw0ZcoUtWvXLluOAwAApC5buirc3NxUpkyZ7Cg6z3JyctL06dPVpUsXLV26VH5+frpx44ZKlSolb29vDRw40FTQ8fT01JIlS7Rs2TJt2LBBAQEBioyMlJeXl1q2bKnhw4erVq1aOXJMAADg/3B+JwMvvPCCXnjhBdPbt2vXzu5eIBcXFw0ePFiDBw+2qxwAAOA4dg8EBwAAKAhM9TQNHz5cSri8fsGCBSkez4rkZQEAAORlpkKTbbLF5HML7du3z5hqwCzb9snLAgAAyMtMhabmzZtn6nEAAIB7janQtGjRokw9DgAAcK9hIDgAAIAJhCYAAAATCE0AAAAmEJoAAABMMDUQvF69eg6v2GKxGDexBQAAyOtMhabMzMMEAABwLzIVmsaNG5f9LQEAAMjDCE0AAAAmMBAcAADABEITAACACYQmAAAAE0yNaTIrLCxMx48f140bNxQVFZXh9r1793Zk9QAAANnGIaFp7969+vLLL3Xw4EHT+1gsFkITAADIN+wOTT/99JPee+89Wa1W5nMCAAD3LLtCU0BAgN5//31ZrVbVrl1bL774olxcXDRmzBhZLBb99ttvCgsL09GjR7Vs2TL5+/uradOmmjp1qgoXLuy4owAAAMhmdg0EX7RokeLi4lSqVCktXrxYHTt2VMWKFY31VapUUcOGDfXEE09o5cqVGjVqlA4ePKhp06apUqVKjmg/AABAjrArNO3fv18Wi0XDhg1TsWLF0t3WYrFowoQJeuihh7R3716tWLHCnqoBAABylF2h6fLly5Kk+vXrG49ZLBZjOSYmJsU+AwYMkNVq1bp16+ypGgAAIEfZFZru3r0rSSpXrpzxmLu7u7F869atFPtUrVpVShgPBQAAkF/YFZo8PDwkSREREcZjnp6eRm/T2bNnU+xz48YNKY1ABQAAkFfZFZqqV68uSTp//rzxmLu7u9Gb9Pvvv6fYZ8uWLVJCuAIAAMgv7ApNTZs2ldVq1YEDB5I83rlzZ1mtVi1atEgrV65URESEQkJC9N1332nFihWyWCx66KGH7G07AABAjrErNLVv316StHXrVmN8kyQ99dRTKlmypGJjY/X222+radOmatOmjT777DPFxcWpUKFCGj16tP2tBwAAyCF2hab7779fH3zwgV577TWFhYUZj5cqVUpz585VpUqVjJnCbf9Kly6tr776Svfdd58j2g8AAJAjTM8IPn36dPXp00f16tVL8nifPn1S3b5hw4b69ddftWfPHp0+fVqxsbGqVq2a2rRpk+QKOwAAgPzAdGhauHChFi1apJo1a6p3797q0aOHvLy80t3H1dVVDz/8sB5++GFHtBUAACDXZOr0nNVq1enTp/XJJ5+offv2GjVqlNatW6eoqKjsayEAAEAeYDo0zZ07Vz179lThwoVltVoVFxenXbt26fXXX1fr1q01adIk7dmzJ3tbCwAAkEtMn55r3bq1WrdurcjISP32229at26ddu/erfj4eN25c0dr1qzRmjVrVL58efXs2VM9e/ZksDcAALhnZPrqOXd3d/Xq1Utz587V9u3bNWHCBNWpU8e4Ou7SpUuaM2eOunfvrv79+2vx4sXGLOAAAAD5lV1TDnh5eWnUqFFau3at1q5dq6eeekpeXl5GgPr333/13nvvqW3btnruuef022+/pXoTXwAAgLzOrtCUWJ06dfT6669rx44dmjdvnnr16mWMf4qJidEff/yhl156SW3atNG7774rX19fR1UNAACQ7RwWmmwsFotatWqljz76SLt27dJHH32k1q1by8nJSVarVWFhYfr55581ePBgR1cNAACQbRwemhJLPP5pzZo1qlWrliwWi5QwfQEAAEB+YfrquaywnZZbt26dduzYodjY2OysDgAAINtkS2g6ePCg1q5dq82bN+vWrVtSop6lokWLqkuXLmnefgUAACAvclhoOnfunNauXav169crODhYShSUnJ2d1bJlS/Xu3VuPPvqoChUq5KhqAQAAcoRdoSk0NFQbN27U2rVrdfToUSnZWKXatWsb96krW7as/a0FAADIJZkOTdHR0dq6davWrVunnTt3Ki4uTkoUlsqUKaPu3burd+/eqlu3ruNbDAAAkAtMh6a9e/dq3bp1+u233xQeHi4lCkqFChVShw4d1Lt3b7Vp00bOzs7Z12IAAIBcYDo0jRgxQhaLxQhKFotFTZs2Ve/evdWtWzcVK1YsO9sJAACQqzJ1es5qteo///mPevbsqV69eqlKlSrZ1zIAAIA8xHRoGjBggHr37q0mTZpkb4sAAADyINOhaerUqdnbEgAAgDwsW2+jAgAAcK8gNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJrgMPHW+NxuQoHBcw0AOS9T954D0uNkcdLiw3/pSnhYbjflnlauWEkNuf/h3G4GABQ4hCY41JXwMAXfCs3tZgAA4HCcngMAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABEITAACACYQmAAAAEwhNAAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQBAAAYAKhCQAAwARCEwAAgAmEJgAAABMITQAAACYQmgAAAEwgNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABEITAACACYQmAAAAEwhNAAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQBAAAYAKhCQAAwARCEwAAgAkuud2A3Obn56cdO3bo0KFDOn36tEJDQ+Xq6iovLy81adJE/fr1U7NmzUyXt2PHDi1btkx+fn4KDQ2Vp6envL29NWDAALVr185UGbGxsVq+fLnWr1+vM2fOKCIiQl5eXmrVqpWGDRumWrVq2XHEAAAgKwp0aBoyZIgOHDiQ4vGYmBidO3dO586d06pVq9S7d29NmzZNbm5uaZYVHx+vyZMna8WKFUkev3Lliq5cuaKtW7fKx8dHU6dOlZNT2h18oaGhGj16tPz8/JI8HhgYqKVLl2r16tWaMmWKfHx8snTMAAAgawp0aLp69aokycvLS127dlWzZs1UoUIFxcfHy9fXV/PmzdOVK1e0Zs0axcbG6tNPP02zrJkzZxqBqX79+nr66adVpUoVBQYG6vvvv5e/v7+WL18uT09PvfLKK6mWERcXp3HjxhmBqXPnzvLx8ZGHh4cOHz6sb775RiEhIZoyZYq8vLxM91wBAAD7FejQVKNGDY0fP15dunSRs7NzknWNGzdWz549NWjQIJ07d04bNmzQE088oebNm6co5+zZs5o3b54kqWHDhlq8eLEKFy4sSWrUqJE6dOigoUOH6ujRo5o7d6769eunqlWrpihn9erVOnjwoCRp8ODBeuedd4x1jRo1Utu2bdW3b1+Fh4fr/fffV+vWreXiUqBfQgAAckyBHgg+e/ZsPfbYYykCk42np6feeOMN4+fNmzenut2CBQsUGxsrSZo8ebIRmGzc3d01efJkKWG80vz581Mtxxa8PDw8NHHixBTrq1atqjFjxkiSzp8/ry1btpg8UgAAYK8CHZrMePDBB43lCxcupFhvtVq1bds2KaHnqnHjxqmW07hxY1WvXl2StG3bNlmt1iTrz549q4CAAElS165d5e7unmo5ffr0MZa3bt2apWMCAACZR2jKQHR0tLGc2gDuoKAgY2xUaqfuEmvRooWUMDg8KCgoyTrbabnE26WmbNmyqlatmiTp0KFDpo8DAADYh9CUgf379xvL9913X4r1p0+fNpZr1KiRblmJ1585cybJOlsvU2bKuXTpkiIiItLdFgAAOAahKR3x8fGaM2eO8XO3bt1SbHP58mVjuXz58umWl3j9pUuX0iynXLly6ZZToUIFKeHUYOL9AABA9uHSq3TMnz9fR44ckRIu/2/YsGGKbe7cuWMsFylSJN3yEo9TSt5DlLicokWLZrkcM+Li4jK9jxlpDahH9siu1xH3Bt6PuFdlx98+s2USmtKwb98+Y16m0qVL67///W+q2929e9dYdnV1TbfMxJNjRkVFZUs5ZiSfONMR3N3dVb9+fYeXi7SdOHFCkZGRud0M5EG8H3Evy82/fYSmVJw6dUrjxo1TbGysChUqpC+++EKlS5dOddtChQoZyzExMemWm3hQefJpCZKXk/jnzJRjhre3N99C7wF16tTJ7SYAQI7Ljr99cXFxpjoUCE3JBAYGauTIkQoLC5Ozs7M+++yzdK+KS3wqLaNTZYmTcfJTeYnLuXPnTrqhKb1yzHB2diY03QN4DQEURLn5t4+B4IlcuXJFTz31lK5evSqLxaLp06erU6dO6e6TeHB3RoOyE6+3DeZOrZwrV66kW45tELnFYslw8DkAAHAMQlOC0NBQjRw5UoGBgVLCzN69e/fOcL+aNWsay8mnEUgu8frk0wokns7AbDkVKlTIUk8TAADIPEKTpNu3b+vpp5825lx69dVXNWTIEFP7Vq5cWV5eXlKyOZ1SY1tfrlw5Va5cOcm6pk2bGsv79u1Ls4xr167p3LlzkqQmTZqYaiMAALBfgQ9NkZGRGj16tP79919J0tixYzV69GjT+1ssFnXs2FFK6AHy9fVNdTtfX1+jh6hjx46yWCxJ1levXt3obdq0aVOaVwasXr3aWM7o1CEAAHCcAh2aoqOjNW7cOON2JMOHD9f48eMzXc6IESOMgWnTpk1LMQ1AVFSUpk2bJklycXHRiBEjUi1n5MiRkqSbN29qxowZKdZfuHBBs2fPlhJu3vvoo49muq0AACBrCvTVc6+++qp27twpSXrooYfUv39/nTx5Ms3tXV1djZvuJla9enWNGjVKc+bM0dGjRzVo0CA988wzqlKligIDA/Xdd9/J399fkjRq1Cjj3nHJ9enTRytXrtShQ4e0ePFiXb9+XT4+PipZsqSOHDmiWbNmKTw8XE5OTnrrrbfk4lKgXz4AAHJUgf7U/e2334zlPXv2qGfPnuluX6lSJf3++++prhs/frxCQkK0cuVK+fv7p9pj1b9/f7388stplu/s7Kyvv/5ao0ePlp+fnzZv3qzNmzcn2cbNzU1TpkxRu3btTBwhAABwlAIdmhzJyclJ06dPV5cuXbR06VL5+fnpxo0bKlWqlLy9vTVw4EBTQcfT01NLlizRsmXLtGHDBgUEBCgyMlJeXl5q2bKlhg8frlq1auXIMQEAgP9ToEPTiRMnHF5mu3bt7O4FcnFx0eDBgzV48GCHtQsAANinQA8EBwAAMIvQBAAAYAKhCQAAwARCEwAAgAmEJgAAABMITQAAACYQmgAAAEwgNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE1APlPcrbCs8fG53YwCg+cagI1LbjcAQOa4u7rJ4uSksK0/K+7G1dxuzj3NuZSXSnYalNvNAJBHEJqAfCruxlXFXg/O7WYAQIHB6TkAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABEITAACACYQmAAAAEwhNAAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQBAAAYAKhCQAAwARCEwAAgAmEJgAAABMITQAAACYQmgAAAEwgNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABEITAACACYQmAAAAEwhNAAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQBAAAYAKhCQAAwARCEwAAgAmEJgAAABMITQAAACYQmgAAAEwgNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABEITAACACYQmAAAAEwhNAAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQBAAAYAKhCQAAwARCEwAAgAmEJgAAABMITQAAACYQmgAAAEwgNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABEITAACACYQmAAAAEwhNAAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQBAAAYAKhCQAAwARCEwAAgAkuud0ApC04OFiLFi3S9u3bdfnyZbm5ualKlSrq1q2bhgwZInd399xuIgAABQahKY/6/fffNWHCBIWHhxuPRUZGKiwsTEePHtXy5cs1Z84cVa1aNVfbCQBAQcHpuTzI399f48ePV3h4uIoUKaLx48dryZIlmj9/vgYMGCBJOnfunEaPHp0kVAEAgOxDT1Me9P777ysqKkouLi6aN2+eHnjgAWNdy5YtVbVqVc2YMUPnzp3TDz/8oBdeeCFX2wsAQEFAT1Mec+TIER04cECS1K9fvySByWbkyJG67777JEkLFy5UTExMjrcTAICChtCUx2zdutVY7tevX6rbODk5qXfv3pKkW7duae/evTnWPgAACipCUx5z8OBBSVKRIkXUoEGDNLdr3ry5sXzo0KEcaRsAAAUZoSmPCQgIkCT95z//kYtL2kPOatSokWIfAACQfQhNecjdu3d148YNSVL58uXT3bZkyZIqUqSIJOny5cs50j4AAAoyrp7LQ+7cuWMs2wJRetzd3RUREaGIiAhT5VutVklSdHS0nJ2d7Whp6pydnVWhaEk5y+LwsvF/SrsXVVxcnCylysvJ4vjXEf/H4lFWcXFxiouLy+2mZBrvx5zB+zHnZOf70Vam7XMyLYSmPOTu3bvGsqura4bbu7m5SZKioqJMlR8fHy8lzAOVXWqriGoXzjjwwQ7hkq+vr1Sq5v//h+zl65vbLcgy3o85gPdjzsrm96PtczIthKY8pFChQsaymWkEoqOjJUmFCxc2Vb6Li4u8vb3l5OQki4VvnwAAKKGHKT4+Pt2xxCI05S1FixY1ls2ccouMjJRMnspTwlQFtt4pAACQOQwEz0MKFSokDw8PycTg7rCwMCNYZTRoHAAA2I/QlMfUrPn/z4lfuHBBsbGxaW535swZY9k2OzgAAMg+hKY8pmnTplLC6bl///03ze32799vLDdp0iRH2gYAQEFGaMpjOnXqZCyvXLky1W3i4+O1Zs0aSVKJEiX04IMP5lj7AAAoqAhNeUyjRo3UrFkzKSE0/fPPPym2mTdvnjEL+PDhw01NTwAAAOxjsWY0kxNynL+/vwYNGqSoqCgVKVJEY8eO1YMPPqioqCht3LhRS5culSRVq1ZNK1euVLFixXK7yQAA3PMITXnU77//rgkTJig8PDzV9dWqVdOcOXNUtWrVHG9bZtWpU0eSNG7cOL3wwgu50oYOHTooODhYffr00YcffphkXVBQkDp27ChJ+uCDD9S3b99caSMAIG9jnqY8qkOHDlq3bp0WLlyo7du368qVK3J1ddV//vMfde3aVUOHDpW7u7tD69y7d6+GDx+e4nFnZ2cVK1ZMxYoVU4UKFdSgQQM1bdpU7du3LxDzPqX1vNgUKVJEXl5eatSokfr27auWLVumuW3igGZWx44dNWvWrCSPffnll/rqq69SbGuxWFSkSBGVLVtW3t7e6tWrlx5++GG76k/NiRMnTG+bVluVMHdY0aJFVaVKFbVo0UIDBw5McjNq5A28B1LiPVAwEZrysEqVKmnSpEmaNGlSrrYjLi5OYWFhCgsLU3BwsA4cOKAFCxbI09NTw4YN0+jRozOcRTU7DBs2TPv27VOLFi20aNGiHK/fJiIiQufOndO5c+e0bt069e7dW9OnT8+W+/tlxGq16s6dO7pz547OnTun9evXq3Pnzvr000/zZMCNj4/X7du35e/vL39/fy1evFiTJk3SkCFDcrtpyATeA1nHeyB/ITQhVYMGDdLgwYONnyMiIhQWFqYTJ05oz5492rVrl0JDQ/XFF1/ojz/+0OzZs+Xp6ZlqWZn5RpZdfv/9d4eUk/x5sVqtCgsLk6+vr+bPn6+QkBCtWbNG5cuX1/jx49Mtq2PHjnr55ZczrDOjMWvTp0+Xt7e3lPAH+PLly/rnn380f/58RUVF6bffftMHH3ygd955R+XKldP69evTLKtHjx6SpIYNG+qDDz7IsG2ZlbittvZevXpVf/75p5YsWaKYmBhNnTpV1atXV6tWrRxeP+zHe8A+vAfyN0ITUlW6dGnVrl07xePt2rXT6NGjdfr0aU2YMEH+/v46cuSInn/+eS1YsCBPfpNzpLSelxYtWqhDhw7q27ev7t69q0WLFun5559P9/koUaJEqmVlVuXKlZOUU7duXT3yyCPq0qWLfHx8FBsbq2XLlum5555T2bJlTdVZpEgRh7Qto7ba2tu2bVvVq1dPb775piRp7ty5fGDkUbwHHNtW8R7IV5hyAFlSs2ZN/fzzz6pfv74k6dChQ/rpp59yu1m5qmbNmnrkkUckSXfu3Ekya3tuqF+/vh577DFJUmxsrPbt25er7clIv379VKpUKUmSn59fbjcHWcB7wD68B/I+epqQZYULF9bHH3+sHj16yGq1au7cuRoyZEiKeaMyunru1q1bWrx4sbZv364zZ84oIiJCxYsXl6enp6pXr67WrVurc+fOKlOmjCTpjTfe0OrVq4399+3bZ9RhU6lSpSSn5NK7es6RKlWqZCxHR0dnWz1mJf5Ge+nSpVxtixmVKlXSjRs30n3ufH199ccff+jQoUM6c+aMwsLC5ObmpvLly6t58+YaNmyYcTuitJw9e1Y//vij9u7dq+DgYMXExMjDw0OlS5dW/fr19fDDD6tTp05p9pJcu3ZNP/74o/766y8FBQUpIiJCpUuXVuPGjTVw4MAC3UPAe8A+vAfyNkIT7FKrVi21bt1aO3fu1NWrV+Xn55ep27oEBAToySef1NWrV5M8fuPGDd24cUMBAQHaunWr4uPjNXTo0Gw4Ase6ePGisVyxYsVcbYukJAE2NwbrZ5bt+atQoUKq61etWpXqhRExMTEKCAhQQECAli9frrfeeivNgbS//vqrJkyYoJiYmCSPX7t2TdeuXdPx48e1atUqrV+/PtXTM+vWrdM777xj3DDb5vLly9q0aZM2bdqk/v376913380Xz7mj8R6wD++BvO3eOhrkipYtW2rnzp2SpAMHDmQqNE2YMEFXr16Vq6urfHx81LZtW5UpU0ZWq1WXL1+Wr6+vtm7dmmSf8ePHa+TIkZo0aZKOHj2a6oDN3JglPSAgQNu3b5ckNW7c2OgZy02JT49Urlw5V9uSkdWrVys0NFRKGCCcmri4OJUsWVIdO3ZUs2bNVLVqVRUpUkRXr17Vv//+q0WLFunGjRuaNm2aatSokeLS9+vXr+vNN99UTEyMSpcurSFDhqhx48YqVaqUoqKidOHCBe3bt0/btm1Ltf6NGzdq4sSJslqtqlKlioYOHar77rtPnp6eCg4O1ooVK7Rjxw6tWLFCxYoVy/UrX3Ma7wH78B7I+whNsFuDBg2M5XPnzpneLzAw0Lgp8RtvvJGiJ6lRo0bq3LmzJkyYoFu3bhmPlytXTuXKlVORIkWkbBywmZqQkBCdPHnS+Nlqter27dv6559/tGDBAkVFRal48eKm/lDcunUrSVlpqVy5snGsmXHp0iXjKqESJUqkO3dOTgkKCjLGbCjhyqHr16/rzz//NMbE1apVSyNHjkx1/7Zt26p79+4p5iirX7++HnnkEQ0fPlxDhgzRiRMn9OWXX6Y45u3btxvfjufPn5/i96ZJkybq3bu3oqKiUtQdGhqqKVOmyGq1ql+/fpo6dWqSb9ENGjRQ586dNXPmTH377bdauHDhPTnnDu8B+/AeyN8ITbCbh4eHsZw43GTk2rVrxrLtfnupsVgsKlmypB0tdJyff/5ZP//8c6rrnJyc9MQTT+jJJ59U9erVMyxr27ZtaX6bS2zhwoWmb8psu9z64MGD+uyzz4w/ji+99JKKFi1qqozsZLsyKDXFixfXuHHjNHjwYJUoUSLVbcqVK5du+cWLF9eLL76o559/XgcPHtSNGzeSfEBdv35dklSyZMl0g3bhwoVTPPbzzz/r9u3bKleunP773/+medrhhRde0OrVq3XlyhWtXbs2w8vu8xveA/bhPZC/EZpgt8TfAO/cuWN6v7JlyxrLq1evzvfduPHx8dq4caMKFSqk1157LcemX0hvpmYvLy+9+OKL8vHxyZG22OP27dtavny5PDw89MQTT5jaJyIiQqGhoYqMjJTtjlCJT80eP348yTdt2+9cWFiYtm7dqk6dOplun+3CgkceeSTd19bFxUWNGzfW5s2bU73h9r2M94B9eA/kfYQm2C1xUMrMzYOrVKmiZs2a6cCBA5o/f7527typzp07q0WLFmrcuLHDbxPjCKldARgVFaXz589r7dq1WrBggRYsWKCjR49q7ty56R5Ddl/Jp4Su/J49e2ZrHZmRvMfAarUqPDxcx48f14IFC7Rlyxa98847Onv2bJohOjQ0VPPnz9fmzZt1/vx5pXf7zBs3biT5uUOHDipRooRu3bqlcePGGXMLNWvWTPXq1UtzBuu4uDgdP35ckrR06VLjptkZsX2rv5fwHrAP74H8jdAEuyV+U2b2NNpnn32ml156Sf/8849Onz6t06dPa9asWXJ1ddX999+v7t27q2/fvipUqFA2tNwxChcurDp16mjixImqVq2aJk+erIMHD+rbb7/NkW7pxDMM2wZyLl26VPv27dOKFSt0/fp1ffvtt7JYLNnelsyyWCwqXry4mjdvrubNm+vVV1/Vhg0bNH/+fLVr1y7FZctHjx7VqFGjdPPmTVPl3717N8nPpUqV0jfffKNXXnlFV65c0d69e7V3714pIfC3bNlS/fr1U/v27ZPsFxYWptjY2EwfX2rjQu5FvAeyjvdA/kJogt38/f2NZTPjGBIrV66clixZot27d+u3337T/v37dfr0acXExOjAgQM6cOCA5s2bpzlz5mS67NzQv39/ffrpp7p586ZWrlyZIx8YyWcYbtSokR5//HG9+eabWrVqlbZv364FCxboySefzPa22Ovpp5/Whg0bJEkrV65M8oERHR2tl19+WTdv3pSrq6uGDh2qjh07qlq1aipZsqRxuiAwMNA45ZDaN/BmzZppy5Yt2rx5s3bs2KEDBw7o8uXLCg8P15YtW7Rlyxa1adNGX331ldFLEhcXZ+zv4+OT7umgxHLjKs7cxnvAPrwH8jZCE+y2a9cuY7lp06ZZKqNly5bGefcbN25o9+7dWrp0qfbs2aMLFy5o/PjxWrNmjcPanF2cnJxUtWpV3bx5U9euXUsxCDOnWCwWTZkyRXv27NHFixf11VdfqXfv3kkG7edFia+ySX5V1Z49exQYGChJeuedd9Ico2LmG3ihQoXUs2dP47RNYGCgduzYoUWLFuncuXPauXOnZs6caQzaTdyDarVac+xqzfyI94B9eA/kbdxGBXY5efKkdu/eLSVMxtawYUO7yyxVqpQee+wxLViwQB06dJAkHTt2LFPTGeSmxF3Yib+d5TR3d3c999xzUsIA0++//z7X2mJW4ucu+amA06dPG8vdunVLs4yjR49mul7bfDMrV65U+fLlpYQJAG3c3NxUq1YtKeGWQUgf74Gs4z2QtxGakGVRUVF6/fXXje7fkSNHOnz218RXfSQf0Ggb55QXbtVgExkZqYCAAClhnEdufMNOrHfv3saszD/99JPpcRC5JfEf++QzIif+AImMjEx1//j4eC1fvjzL9RcrVswYG5PaAFolTJb4119/ZbmOex3vAfvwHsjbCE3IktOnT2vw4MHGeKYWLVpo0KBBmSrj2LFjOnbsWJrrrVarcerPYrEkuaeVEl06GxgYmO7VIznpyy+/NAY+tmnTJs0rUXKKq6urnn76aSnhKseFCxfmanvSEx0drS+++ML4uV27dknWV6tWzVhOfO/BxD799FNjwtTU/PXXXylu2ZPY7du3deTIESmV2aOHDx9uTK8xadIknTp1Kt3j2b59u3G1UUHCeyDreA/kfYxpQqqSz/obGRmpsLAwnThxQnv27NHff/9tBJXGjRvriy++yPSAv2PHjmnSpEny9vZW+/bt1aBBA5UpU0axsbEKCgrSqlWr9Pfff0sJ33C8vLyS7N+kSROtWrVKISEh+uCDD9SzZ08VL15cSpgnJHnIcoTkz4sSrk45f/681qxZY3z7KlSokF588cV0yzI7G7Kzs7Puu+++LLe5f//++uabb4wbbI4cOTJTU0M4UvLZkCUpPDxcx44d088//2z8Ea5ataoGDhyYZLs2bdqodOnSCgkJ0eeff66goCA9+uijKlWqlC5cuKBly5Zp9+7datKkSZqnD3755Rc9++yzatWqlVq3bq3atWurZMmSunPnjk6ePKnFixfrypUrkpRinpwyZcroo48+0osvvqhr166pX79+6tOnj9q2bavy5csrNjZWly9f1pEjR7R582YFBgbq22+/Vd26dR38LOYu3gP24T2QvxGakKr0Zv218fT01IgRI/T000/bdVrOz89Pfn5+aa5/4IEH9P7776d4/LHHHtPs2bMVGBhozA1jU6lSJWMiNkcy+7zMmDFDderUSXc7s7MhFy9eXAcOHMh0W20KFSqkJ598UjNmzFBYWJgWL16sMWPGZLk8e6Q3G7JN3bp19fXXX6eYkbhIkSL66KOP9Pzzz+vu3bupzhXTokULTZkyRd27d0+z/JiYGO3YsUM7duxIc5snnngi1auDOnfurFmzZmnSpEm6efOmlixZoiVLlqRahpOTU56ca8xevAfsw3sgfyM0IUNOTk4qWrSoihcvrooVK6pBgwZq1qxZhrPCZqR79+4qXbq0du3aJT8/P125ckUhISGKjY1V6dKlVb9+fT322GN6/PHH5eSU8kxy0aJFtWTJEs2ePVt///23Ll68mOZ5/uzk6uoqDw8P1axZU+3atVPfvn3zzG1fbAYNGqTvvvtON2/e1Pz58zV8+PA888fM3d1dnp6eatCggbp06aKuXbumGcIffvhhrVy5UnPmzNGePXt048YNFS9eXDVr1lSPHj3Uv39/4y7xqZk0aZJatWqlPXv26MSJE7p27ZpCQ0Pl7Oys8uXL64EHHlD//v3Tva1Phw4dtG3bNi1btkw7duzQ6dOnFRYWJmdnZ5UpU0a1atXSQw89pC5duqR5p/p7De8B+/AeyD8s1rwyGAQAACAPYyA4AACACYQmAAAAEwhNAAAAJhCaAOD/tXdvIVFtfxzAv46gHJVGzFECH7qpZeOlyMzEwkvaxdFRpILKfJESCXxQ6iUqJKh8qLC89CRWhnipwSyvTQ2pTCqVt2iMELSMUdNyRsTrecmdk8646//vnE7z/TyNe9b67aVPX/daey0iIhEYmoiIiIhEYGgiIiIiEoGhiYiIiEgEhiYiIiIiERiaiIiIiERgaCIiIiISgaGJiIiISASGJiKi/4CcnBx4e3vD29v7l90jPDwc3t7eOH369C+7B9F/2dLHKBMR/SG0Wi2SkpKEnx0cHNDU1LTsCfcTExMICQmBwWAQrhUVFSEoKOiXjpeIfl980kREVmV8fBz19fXLtmtoaDAJTEREDE1EZDXs7e0BACqVatm2823m+xARMTQRkdUIDw8HADQ1NWFwcNBsu+HhYTQ2NgIAIiIi/rHxEdHvjaGJiKxGSEgIZDIZZmZmUFVVZbbdgwcPMD09DZlMhh07dvyjYySi3xcXghOR1bC1tcX+/ftRWFgIlUqF5OTkJdvNT83FxMTA1tZ22bqTk5MoLS1FdXU1enp6YDAYIJVK4ePjg5iYGCgUCkgklv9H/fjxIwoKCqDRaKDX6yGVSiGXy5GUlPRDwW1sbAzFxcVQq9Xo7e2FwWCAs7Mz5HI5lEoloqOjYWNjI7oeEX3D0EREViUuLg6FhYXo7u5GT08PPD09Tb5/+/Yturq6hLavX7+2WK+/vx8pKSl49+6dyfWhoSFoNBpoNBqUlJQgNzcXzs7OS9ZobW3F8ePHTRaeDw4OQq1WQ61W4+TJk6J+t+bmZqSnp2N0dNTk+sJau3btwpUrV+Do6CiqJhF9w+k5IrIqPj4+QlBaakH4/DUvLy9s3LjRYi2j0Yjk5GQhMEVGRiIvLw/l5eW4du0atm3bBgBoa2vDiRMnMDMzs6jGhw8fhMAkkUhw6NAhFBYWoqysDBcuXMDq1auRk5ODJ0+eWBxLW1sbUlJSMDo6CldXV6SnpyM/Px8VFRXIz89HbGwsAODp06fch4noJzE0EZHViYuLA76uXZqbmxOuz83NobKy0qSNJdevX0dfXx8AIDU1FTdu3EB4eDjkcjn27NmDoqIiKBQKAMCLFy9QUlKyqMbFixeFJ0zZ2dk4f/48goOD4evri8TERJSXl2PDhg3o7Ow0O46pqSlkZmZiamoKoaGhqK+vR2pqKsLCwrBp0yaEhYUhOzsbWVlZAIDa2lphoTsRicfQRERWJzY2FhKJBAMDA9BqtcJ1rVaLgYEBSCQSIeyYMzk5ibKyMgCAp6fnklNoNjY2OHfunDAtd+fOHZPvBwcHhT2jwsLCEBMTs6iGk5OTEHbMqaqqwvv372Fvb4/Lly+b3bjzwIED8PPzAwBUVFRYrElEizE0EZHVcXd3F3b2XjhFN/95+/btcHd3t1ijs7MTX758AQDEx8ebXTDu5OSEvXv3Al/XS+n1euE7rVYrTNklJCSYvZefn9+itVcLPX78GAAQGBgIFxcXi+PeunUrAODly5cW2xHRYlwITkRWSalUorm5GbW1tTh79iwAoKamBhA5NdfT0yN89vf3t9jW398fd+/eFfq5ubkBAHQ6ndDG19fXYg1fX1+Tey40P3X37Nkz0WfTDQ0NiWpHRN/wSRMRWaXdu3fjr7/+gsFgQENDA+rr62E0GuHg4ICoqKhl+3/+/Fn4vNzTHVdX1yX7LXzLbeXKlaJrfO/Tp0/Ljvd7ExMTP9yHyNrxSRMRWSVHR0dERkaisrISKpVKWBAeGRkJBweHH6r1b+97ND/Ft3PnTmRmZv6rYyH6kzE0EZHVUiqVqKysNHmTTKlUiuorlUqFz8PDw1izZo3Ztgunwhb2+77GqlWrRNX4nrOzM/R6PaampuDl5SVq/ET04zg9R0RWKzg4GDKZDNPT05ienoabmxuCg4NF9V24MPvVq1cW27a3ty/Zb2HA6ejosFjD0pYDPj4+QpvJycllRk5EP4uhiYislq2tLeLi4mBnZwc7OzvExcUte9zJPLlcjhUrVgAA7t+/j9nZ2SXbGQwGPHr0CACwfv16YRE4AAQFBQlv3d27d8/svdrb200WjX9v/iDisbExbiVA9AsxNBGRVcvMzERHRwc6OjqQkZEhup+dnR0SExOBr2/B5ebmLmozNzeHrKwsjIyMAAAOHz5s8r2bmxsiIiKAr9sGPHz4cFENo9EovN1nTnx8vDC1d+nSJbS0tFhs39raiufPny/7OxKRKa5pIiL6SWlpaairq0NfXx9ycnKg0+mQkJAAmUyG/v5+3L59WwgnmzdvxsGDBxfVOHXqFBobG2E0GpGRkYGWlhZER0fDyckJb968wc2bN9Hb2wu5XG52is7Ozg5Xr17F0aNHMT4+jmPHjmHfvn2IjIyEh4cHZmdnMTg4iK6uLtTV1UGn0+HMmTPCMS9EJA5DExHRT3JyckJhYaFwYG9NTY2w19NCW7ZsQV5e3pIbYHp4eCAvLw+pqakwGo0oLi5GcXGxSZu0tDTY2NhYXNcUEBCAW7duIT09HQMDA6isrBSOhDE3diL6MQxNRET/Aw8PD6hUKpSWlqK6uho6nQ5GoxFSqRQbN26EQqGAQqGwuFYqKCgIVVVVKCgogEajgV6vh1QqhVwux5EjRxAaGoqcnJxlxxIQEIDa2lpUVFRArVaju7sbIyMjkEgkcHFxwbp16xAYGIioqCisXbv2//yXIPrz2cwtPK2SiIiIiJbEheBEREREIjA0EREREYnA0EREREQkAkMTERERkQgMTUREREQiMDQRERERicDQRERERCQCQxMRERGRCAxNRERERCIwNBERERGJwNBEREREJAJDExEREZEIDE1EREREIjA0EREREYnA0EREREQkwt97X6uLiqQWLAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "riddles_rtl_df = pd.DataFrame({\n", + " 'model': ['DistilBERT Base', 'DistilBERT Base', 'BERT Base', 'BERT Base'],\n", + " 'order': [0, 0, 1, 1],\n", + " 'direction': [\"LTR\", \"RTL\", \"LTR\", \"RTL\"],\n", + " 'ppl': [290, 160, 1010, 520],\n", + "})\n", + "\n", + "riddles_rtl_df_sorted_pairs = riddles_rtl_df.sort_values(by=['order', 'direction'])\n", + "\n", + "# Plot configuration\n", + "plt.figure(figsize=(6, 8))\n", + "sns.set_style(\"whitegrid\")\n", + "\n", + "# Create bar plot with LTR and RTL next to each other, no error bars (ci=None)\n", + "sns.barplot(x='model', y='ppl', hue='direction', data=riddles_rtl_df_sorted_pairs, dodge=True, palette=\"Set2\", ci=None)\n", + "\n", + "# Adjustments to the plot\n", + "# plt.xticks(rotation=45)\n", + "plt.title(\"QA Riddle Perplexities\", fontsize=20)\n", + "plt.xlabel(\"Model\", fontsize=20)\n", + "plt.ylabel(\"Validation Perplexity\", fontsize=20)\n", + "plt.legend(title=\"\", fontsize=20)\n", + "plt.tick_params(axis='both', labelsize=20)\n", + "\n", + "# Display the updated plot\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "execution_state": "idle", + "id": "26eba67f-ee2e-44ad-b18f-392aad75aedb", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_480557/2407310871.py:15: FutureWarning: \n", + "\n", + "The `ci` parameter is deprecated. Use `errorbar=None` for the same effect.\n", + "\n", + " sns.barplot(x='model', y='ppl', hue='direction', data=riddles_ltr_df_sorted_pairs, dodge=True, palette=\"Set2\", ci=None)\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAMWCAYAAAAtQ/h0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAACZmklEQVR4nOzdeVgT1+I//nfYFARBVFABFdxRKO5bKxX3WhdUXHBrtVXr0qu2emut2qq1trbXe79aW61al1IVVFRcC6iorYqKKIgriAKyyCqrQJLfHx8yvwRCCGYQgu/X8/g8Q2bmzBnMhHfOOXNGIpfL5SAiIiIi0RhUdwWIiIiIahsGLCIiIiKRMWARERERiYwBi4iIiEhkDFhEREREImPAIiIiIhIZAxYRERGRyBiwiIiIiETGgEVEREQkMgYsqlabNm1Cu3bt0K5du+quSrk8PDzQrl07fPHFF9VdFapBDh8+LLx34+Pjq+QYU6dORbt27TB16tQqKV9b+nCdEtU0RtVdAfo/V69exbRp09Suq1u3LqytrdGhQwcMGzYMw4YNg5ER/+tI/8THx2PAgAHCzxKJBMHBwbCzs6tw3yFDhiA2Nlb4+bvvvsOYMWOqrK61iVwux9mzZ3HixAlERkbi+fPnePnyJczMzNCkSRM4OTnB1dUVb7/9Ntq3b1/d1a0Wpd+bFZk/fz4WLFhQpXXSB4cPH8ayZcuEnx0cHBAUFFThfomJifDw8IBMJhNeCw4Ohr29fbnlv8o17+HhgYSEBOHn77//HqNHj65wv6+++gp+fn7Cz56enli/fn2ljs0WLD1QUFCAZ8+eITg4GJ9//jkmTpyI58+fV3e1qORDWfHN/vDhw9VdHb0jl8sREBBQ4Xbh4eEq4Yq0l5qaismTJ2Pu3Lk4ceIEnjx5gry8PEilUmRnZ+Phw4c4c+YMNmzYgFGjRiE6Orq6q0x6LC4uDmFhYRVuFxAQoBKuXpdjx45VuM3Lly9x+vRpnY/FZpAaaNKkSfD29hZ+zsvLQ2RkJHbu3ImEhARERERg7ty58PX1hUQiqda6vgnOnj1b3VWolerUqYOXL1/i6NGjmDNnjsZtjx49qrIPaaewsBAffvghHjx4AABwdnbGmDFj0KFDB9SrVw85OTmIjo7GtWvXEBISguzsbLXlLFiw4I1qrRkwYAAWLlyocZuGDRu+tvroC+VrukuXLhq3fd3XtOI4ly9fRnJyMmxtbcvd9uzZs8jOzta5bmzBqoEaNmyItm3bCv/c3NwwZcoU+Pv7o0WLFgCA27dv49y5c9VdVaJX5uHhAQCIiYlBREREudsVFRXh5MmTQMkfPtKen5+fEK7GjBmDQ4cOYerUqejWrRs6dOiA7t27Y+LEifjpp5/wzz//4LvvvkP9+vWru9rVrn79+iqfwer+MWCVpbimT58+jcLCwnK3u3PnDh49egS8xmu6U6dOaNy4MWQyGY4fP65xW0X407VuDFh6xNLSErNmzRJ+vnjxYrXWh0gXrVq1QqdOnQClDzR1QkJCkJmZCWNjY7z33nuvsYb6Lzg4GABgZGSEZcuWwcCg/I98ExMTjBkzBo0bN36NNaTa5L333oOxsTEyMzMREhJS7naK693FxQVOTk6vpW6GhoYYMWKEyvHVSU9Px6VLlwBAq7FamrCLUM+4uroKy8+ePSuzXiqV4tixYzh9+jTu3LmDzMxM1KtXD05OThg8eDAmTZqEunXrqi176tSpCA0NRY8ePbB3717ExsZiz549uHTpEpKTk1FQUCAMQlQelL9nzx50794dBw8exOHDhxETE4PCwkI0b94cw4cPxwcffIA6derodN4vX76En58fAgMD8ejRI2RlZcHCwgLt2rXD8OHD4enpWWbg//Xr1zF16lTIZDK4u7tj27ZtasvOycnByJEjkZCQAGtrawQEBKBRo0bCesUgydKDHEvfUbVs2TKVwZ5QGgjr6emJqKgoODk54dSpUxrPNSMjA++88w6KioowadIkfP3111r9jpYtW4bDhw+jTp06+Oeff2Bubq5xe8WgcRcXFxw8eFBlXWRkJP7880/cuHEDycnJkEqlsLa2RsOGDfHWW2/h7bffhoeHh85d1KNGjUJkZCROnjyJL774Qu3NG4oPw/79+2vduiKTyRAQEIDjx48jKioKWVlZMDc3R5s2bTB06FB4eXnBxMREYxlZWVn47bffEBQUhGfPnqFevXpo164dJkyYgGHDhml9jq/y3hWL4jOiQYMGOrVMbdq0CZs3bwYA3L9/X2Wd4nNDW+oGMgPA8+fP8ccff+DixYuIj49HXl4eGjZsCDc3N0yYMAF9+vR55fpXJeVB2MHBwbCxscGff/6JkydP4smTJ8jMzCwzIL6wsBB+fn44ffo0Hj58iJycHFhaWsLZ2Rnvv/8+RowYUW4Y/uKLL+Dv7w87OzucPXsWz58/x86dO3H27FkkJSWhfv366NKlC+bPn482bdoI+8XHx2PXrl24ePEiEhMTYW5ujt69e+Nf//oXmjdvLsrvwtLSEu+++y4CAwNx9OhRDBo0qMw2xcXFOHHiBFBy/WdmZopybG2MGjUKO3fuxP3793Hv3j21N3ScOHECRUVFaNiwIfr27avT8Riw9IzyB7FUKlVZ9+zZM3zyySe4d++eyuuZmZkICwtDWFgY9u3bh61bt8LR0VHjcYKCgrBkyRLk5eVVWKeioiLMmjWrTIva/fv3cf/+fRw7dgy7du165W/G9+7dw9y5c1XuBEHJN43Lly/j8uXLOHDgAH799VeVYNStWzfMmjULv/76K0JCQuDj44PJkyeXKf+bb74Ryv72229VyhDLuHHjsHr1asTExCA8PBxubm7lbhsQEICioiIAwNixY7U+xsiRI3H48GG8fPkSgYGB8PT0LHfbiIgIYdC44ludwq5du/D999+XGYCalJSEpKQk3LlzB3/++SfCwsJQr149reunzvDhw/H9998jLS0Nly5dwrvvvquyPisrC+fPnwdKPhy1kZmZiU8++aTMQNuMjAyEhoYiNDQUPj4++O2338q9ezE6OhoffPABUlJShNcU4zcuX76MCxcuoHv37hXW5VXfu2IxNjYGSga6Z2ZmwsrKSvRjiOHYsWNYtWpVmc+bpKQknD59GqdPn8a4cePwzTff1Og7qDMyMjB//nzcvXu33G3i4+Px8ccfIyYmRuX11NRUXLhwARcuXMCBAwewZcuWCv+/7t27h48++kjlpqeCggKcPn0aFy5cwG+//YZu3brh8uXLWLBggcoYu5cvX+L48eO4ePEifHx8VMKYLkaNGoXAwECcP39e7Xvu77//RmpqKoyMjDB8+HD4+PiIclxttG/fHu3atcP9+/dx9OhRtQFL8YVu+PDhOr/Xau47ldRSjKcAABsbG2E5IyMD3t7eSExMhImJCcaPH4/u3bvDzs4OeXl5+Pvvv7Fnzx48efIEH3/8Mfz9/WFhYaH2GM+ePcOSJUtQt25dfPLJJ+jWrRsMDQ0REREBMzOzMtv/97//RUREBN5++21MmjQJTZo0QVJSEv7880/8/fffePToEebMmQNfX18YGhpW6nyfPHmCKVOmIDs7G+bm5pg8eTJcXV3RpEkTZGZm4uzZszhw4IAw8N/Hx0f4o4KSFqRLly4hMjISP/zwA3r16oVWrVoJ60+cOCHcVTJhwgRhDIE2AgICkJKSgpkzZwIAFi5cWKbPXjFOY+TIkfjhhx9QUFCAw4cPawxYirsR27VrBxcXF63r07NnT9jY2CAlJQUBAQEaA5ZiDIKhoSGGDx8uvH7v3j0hXNnb22PKlClo3749rKyskJubi8ePH+PKlSuiDfxv2LAh3n77bZw/fx5Hjx4tE7BOnTqFwsJCWFlZoV+/frh586bG8qRSKebMmSNs16NHD0yePBn29vZISUnBoUOHEBQUJASoI0eOlAmJOTk5mDlzphCu3nvvPYwePRoNGzZEbGwsfv/9dxw+fBgPHz7UWBdd37ti6NixIx48eAC5XI4VK1Zg/fr1Oofi0tatW4f8/Pxy16enp2PevHnIycmBhYVFmc+dkydPYunSpZDL5XBwcMCUKVPQqlUrWFtbIyEhAQcPHkRISAgOHjwIc3PzMq3ENcny5cvx4MEDjB49Gu+99x4aNWqExMRE4f81NzcXH3zwAeLi4gAAAwcOxNixY2FjY4P4+Hj4+PggNDQUN27cwJw5c+Dj41PuZ2Z+fj7mzZuHoqIiLF68GN27d4ehoSEuXryIX3/9FXl5eVi6dCl+//13zJs3DxYWFvj000/x1ltvobi4GH/99Rd2796NrKwsLF++HL6+vqL8Dtzd3WFlZYXMzEycOnUKkyZNUlmvCDDvvPMOrK2tRTlmZYwaNQo//PADjh8/jiVLlqi0FCqPB9X2C50mDFh6pLi4GL///rvwc48ePYTltWvXIjExEXZ2dti9ezccHBxU9u3ZsyeGDh2KyZMnIy4uDtu3b8eiRYvUHic+Ph42NjY4cOAAmjVrJrz+1ltvqd0+IiICEyZMwOrVq4XXOnXqhIEDB2L58uU4ePAgIiMjsX//frUtSJr8+9//RnZ2NpydnbFjx44yF+Tbb7+Nd999F7Nnz8atW7fg7++P8ePHC+uNjY3x448/wtPTE/n5+fj8889x4MABmJiYIDExUeh+a9myZaU/uNu2basSOG1tbdG2bVu121pYWGDIkCE4evQoTp48iS+//FJtV21UVJTw7bey870YGBhg+PDh+P3333HlyhWkpqaqbRWRyWTCoPHevXurbHPmzBnIZDKYmZnhwIEDZfbv1q0bvLy8kJ2dDVNT00rVrzyjRo3C+fPncfbsWeTk5Kh0bSo+jN97770Ku/QAYP/+/UK4Gj16NNavX6/Sjenh4YGNGzfi119/xdOnT7FlyxYsWbJEpYyff/4ZiYmJAIDFixdj9uzZwrpOnTphyJAhmDNnjjBOozy6vnfF4O3tjaNHj0Imk+Gvv/7ClStX0L9/f3Tr1g2urq5o06ZNpb/0lFb6s0ZZYWEhVq1ahZycHBgaGuI///kPLC0thfXp6elYuXIl5HI5xo4di9WrV6u0GnTs2BGDBw8W/s/27NmDCRMmVPm4nRcvXqh8mS3N0tJS7V1o9+/fx9q1a+Hl5SW81rFjR2F58+bNQrj65JNPVO5UVLy3lixZgoCAANy8eRMHDhxQuaNcWXp6OuRyOfz8/FS6+N566y00aNAAq1evRkJCAiZOnIjGjRtj3759Ku/Brl27wtDQEDt27MCtW7cQFRUFZ2fnSv2e1DExMcGwYcOwb98+HD16VCVg5eTkCOMCxQgwr2LEiBH46aefkJKSgsuXL6t0Ayo+b1q3bi2MD9UFB7nrgby8PISGhuLDDz9EeHg4AMDOzk4Y8BsfHy+M61mxYkW5H3jOzs7CxVrRnE2fffaZSrjSpFGjRuWGky+//FK4qPft26dVeQrXr18X/liuX7++3G87/fr1w5AhQ4ByzsvR0VGoX1RUFP73v/9BJpNh6dKlePHihRDCxAoM5VF86GZnZ+Ovv/5Su42i/sbGxhg5cmSlj6Ho7pNKpcI4h9KuXr0qtM6U7h5MTU0FSgKnpi4rCwsLjQOmK2PAgAGwsLBAQUEBzpw5I7yuPJ+Oth/Giu4Ga2trrFixQu0YsQULFgh/oP38/FTudiosLMShQ4eAkhZE5ZtKFIyNjfHtt99qbG0S672rK1dXV6xevVqo64sXL3D06FGsWLECo0aNQrdu3TBjxgz4+vpqNRygslatWiX8Hy5duhT9+vVTWb9v3z5kZ2fD1tYWX3/9dbldMgsWLICtrS1kMpnGAcpiCQ4OxogRI8r9t3HjRrX79erVSyVcKSssLBTGOrZp00bttBcSiQRff/210K1WUfdZeeOnxo4dK4x7TU9Px/Lly9W+B5XDz/Xr1zUeqzIU1+vNmzeFQImSL3AFBQWwsLCoVG+BmGxsbNC7d2+g1JxYynPyiRX+GLBqoM2bNwuTV7Zr1w6dO3dWGUjasGFD/Pzzz8I3+pCQEEilUpiampb5ACtNMW4kJSVF7SB5lPwBqcwg3mHDhpUbTurVqyeU9fDhw0pNkKr4puPo6FjhIzoU5xUZGYni4uIy65W7/3bu3InPPvtM+H3OmzevUl1xr6p79+5o2bIlUM4f08LCQuEC9/DweKXm844dOwrhobxbkRXHqFu3LgYOHKiyTjFO7tGjR7h9+3alj/8q6tSpI4QM5T+eiuWWLVtq7FJVSE5OFibJHDZsWLmD/I2MjITWwaysLNy5c0dYd+fOHWRlZQElMzeXN4i/SZMmGgfAivne1ZWXlxeOHTuGMWPGlOkeVAwfWLFiBQYPHowLFy6IdtwdO3YI7/OxY8figw8+KLONoqv53Xff1dhCaWRkJLwHKuomrk6lv7Aoi4yMxIsXL4CS91Z5LYfm5ubCZ+ajR49UxgEqk0gk5X5O161bV5jSx9LSEu+8847a7RwcHIT3hHIQ0lXnzp2F46u7pocOHarzjU+6UHx5/euvv4Tu7evXryMhIQEGBgYa/x8rg12EesTe3h5DhgzBzJkzVeZgiYyMBEr65CvTxJuamqq2laply5aVevNX1JTq6uoqfBN78OCB1oPdFef1+PFjrZ+BVlRUhKysLLVz1Hz77bcYOXIknj9/LnSRde3aVW0rRVUZN24cfvzxR1y5cgUJCQkqg6zPnj0r3FFTmcHtpY0YMQL/+9//cPv2bTx58kT4oENJiAsMDARKQlzpEDJ8+HBs27YNhYWFmDRpEt555x24u7uja9euaNOmTZVNbDt69GgcPHgQoaGhSExMRNOmTYVvl9q25CmPiVK+21Yd5e7uhw8fonPnzkCpMY4VhW4XFxdhAH5pYr93deXk5ITvvvsOq1evRmRkJG7duoXIyEhcu3YNSUlJQMldfHPmzMH27dt1vmPv/Pnz+PHHH4GSbmV1d8JKpVLhhpwDBw7gwIEDWpWtaGWtSq/yWBSoubNYmfL7s7zhFsrrFS3+Dx8+VBlvq9CgQQONg+AVd402b95c43Vbv3595ObmIjc3V2OdKmvkyJHYtGkTAgICMH/+fCQmJgpfanWd/kBXgwcPxtdff428vDwEBgZi5MiROHLkCFAy9KZp06aiHIctWDXQpEmTEBAQINxmHhgYiOvXryM4OBhLly4t8wGclpb2Sscpb2BqZW/nrugPgvJ6ReuANtLT0ytVD4Xyzsva2hqLFy8WfjY2NsYPP/yg8xiUyvD09ISxsTHkcjn8/f1V1im6pmxtbfH222+/8jGUv32VfgzN+fPnhW/R6r6ltWrVCj/99BMsLS1RXFyMc+fO4euvv8aIESPQu3dvLFmyRNSuBIVu3brBzs5OaKa/efMmnjx5AolEonXAUn5vVfSeVO7+VL5NXLmMiloQNXWhiv3eFYuxsTE6d+6MDz74AD/++CNCQkKwa9cu4Q4yqVSKb775BnK5/JWP8fDhQyxevBgymQx2dnbYtGmT2taprKysV2qxKygoeOW6VTXl8WWlvep7q7zPzIqGNCi68LXdTuzH1ii62WJjYxEeHo5jx45BLpfDzs4OXbt2FfVYlWVqaorBgwcDJa1qL1++FIYniDk2jC1YNZBiJndtKaZraNCgAfbs2aP1furmokHJnWWVUVWtGorzat++PTZs2KD1fuU9AqG4uFhlHFhRURFCQ0PL/T1UhUaNGgnzxPj7+2PevHmQSCRITk7G33//DZR8u9Ml9Dk4OKBz5864efOm8O1RQdFtaGVlVW63wZAhQ9CnTx+cPHkSly5dwvXr15Geno6MjAwcO3YMx44dg6enJ9atWyfaOCyJRIIRI0bg119/xdGjR4Xu6y5dumgcRK2pPDHq9KrEfu9Wpd69e2Pnzp0YMWIEMjMzERsbi7t3777SgOf09HTMmTMHubm5MDMzwy+//FJumFCeZsbLy6vch92XJvadlmLS9np4Ex5x5uDggC5duiAsLAxHjx7F1atXgZKWrZpw/qNGjcKRI0dw+fJl7N+/X7hxRxG8xMCAVQsomolzc3PRqlWr19oiAy2a7JVb2DR9wytNcV55eXmVCpzl+fnnn4VxRebm5sjJycHatWvRvXv3V/oj/qq8vLwQGBiI+Ph4hIaGomfPnjh69KjwB6eydw+qM2LECNy8eROxsbGIiIiAi4sLcnJyhC6toUOHavxDZWFhgQkTJmDChAlAybxQwcHB2Lt3L1JSUuDv748OHTpg+vTpOtdVYfTo0fj111/x6NEjYTxIZboSlN9bFb0nldcrd7Mot96mpaVpnC9O0zHEfu9WNRsbG7i7uwtjZJ48eVLpgFVUVIRPP/0U8fHxkEgk+PHHHzV2mSn/f8nlcr34PelC+Xwr896qzGdmTTN69GiEhYXh0KFDwjP9quvuwdJ69eoFW1tbJCcn46effgJKbripaILmymAXYS2g+CAsLCwUxn68ThUdU/k5c5WZzE5xXnFxcZUaHK/OzZs3sXXrVqDk9vjdu3fD2NgYubm5WLp0aZlJW7X1Kt/E3nnnHTRp0gRQGuyu6B7s1q2bMBBeF8OGDRPuyFK0Wp05c0b4kKvsIM5WrVph1qxZ8PX1FaamqGhG+spydHQUxk69fPkSJiYmGDp0qNb7K7+3Khqgr7xeeT/lP/Kano+ICt73Yr53XxflcT6v8r7+5ptvcO3aNQDAokWLKnyOm4mJifC7Lz0pbG2k/D67deuWxm3Le3/qm2HDhsHExET43HnrrbcqnOT6dVEezF5V4Y8Bqxbo37+/8IG4e/fu137806dPlzsuIi8vT/hD3Lp1a7WDNcujuOtPLpdXquuztNzcXCxZsgRSqRRWVlZYt24dOnXqhE8//RQo+XAv7zE6FVG+GUDTw02VGRgYCK1UZ86cwYULF4RZ1XUZ3K7M2tpauMvtxIkTKg841WUMRNOmTYUAmJGRIUpdlY0ePRomJiYwMTHBoEGDKjUe0NbWVphE9tSpU+UO2pVKpcL4N0tLS5V5ijp16iS0GBw9erTcsUjJycka58ES672rq8qMpVIOjJVt0d21axf8/PyAkvCuPHeYJsoP/K7tz1bt1KmT8H4+cuRIuWOecnJyXvkzs6apX78+Bg4cKFzT1T24vbRRo0YJdWvatKnOj8YpjQGrFnBychK+6Z84cUJlMlJ14uLiKnyaeGU8f/683Dtu1q9fL3QRlp7RtyJvv/220KKxY8cO4c6/8ty/f1/tDONr164VupzWrFkjjHP56KOPhMlaf/7551dq/bOyshK62p4+far1fmPHjoVEIkF+fj6+/PJLoGRKi8q02FRE8e3s+fPnCAgIEMZAvP/+++W2UAQFBQmD4NVJTEwUHvFRFWPXJk+ejIiICEREROA///nPK+2PkrFAa9euVbvN5s2b8ejRI6Cku1Z5ALbigccAcPfuXWzfvr3M/sXFxfjqq6+ExxmpI9Z7V1fz58+Hj49PhXNcHT58GJcvXwYANGvWrFLdgxcuXMAPP/wAlNy9+e2332q977Rp04QW0WXLllU4O/758+fLPApMUX/FtDabNm3S+vivk4mJCcaNGweU3K26ZcuWMtvI5XKsWbNG+PJS2YmZa6KNGzcK13R5k6ZWl7Zt2wp1O3/+vOjDazgGq5b4+uuvERkZibi4OKxfvx7BwcEYNWoU2rRpAxMTE2RmZuLevXu4ePEirly5gkGDBuH9998X5didOnXCvn37EB8fj4kTJ6Jp06ZITEzEvn37hG/5zs7OmDhxYqXL/umnn+Dl5YXMzEwsWrQIx44dw3vvvYeWLVvCwMAAaWlpuHv3Ls6dO4fw8HDMmDFDZQK7wMBAoRtuzJgxKgMYDQwM8P3332PkyJHIzs7G559/jiNHjpT7MGx1jIyM4OLiIowzcHZ2RocOHYTuOUtLS7W3Utvb26NPnz74+++/hS6kYcOGqX0U0asaMGAAzMzMkJeXh7Vr1wrdoJq6B3fv3o3PP/8c7u7uwmOFLCwskJWVhcjISPzxxx9Ca2VlA/PrMHHiROEuxMOHD+PZs2fw9vaGvb09nj9/jkOHDgmTvDZv3hxz584tU8a8efNw6tQpJCUl4ccff8S9e/cwatQolUflREREoFOnThpDua7vXTEkJiZi9erV+PHHH+Hh4YFu3brB0dERlpaWePnyJWJiYnD69GmEhIQAJV2Dy5Yt07qLMCsrC4sXL4ZUKoWZmRkWLlyIJ0+eaNzH0dFR+FLSqFEjfP/99/j000/x/PlzjB07Fp6enujXrx+aNGmC4uJiJCUl4fbt2zhz5gzi4uLw66+/qn2GnD6YN28eAgMDERcXh02bNuHBgwcYM2YMGjdujPj4ePzxxx/CVAadO3cWxkDS/7lx44ZW23l4eNSI524yYNUSVlZW2LdvHxYuXIjr16/j2rVrwngIdcR8HtmiRYvw+++/4+LFi2qb+Z2cnPDrr7++0oMzmzdvjv379+PTTz/FgwcPcO7cOZw7d67c7ZXPKyUlBV999RVQ0uWhWFbWrFkzrFy5EkuWLMHjx4+xfv16tXP2aDJ79mzMmTMHmZmZ+Oyzz1TWzZ8/X+2MzSiZE0tx5yBE7B5UMDMzw4ABAxAQECC0SrVv377CMR35+fnCA3bVMTAwwIIFC8pMUloTGBoa4tdffxUe9nzlyhVcuXKlzHatWrXCb7/9pvY6sLCwwPbt2/Hhhx/i+fPnOH78eJkW3zFjxqB79+4aH6+ky3tXLE2aNMGdO3eQl5en9jyUWVhY4KuvvqrUXVTZ2dnCA4Tz8vIwY8aMCvcJDg5Waf0cPHgwtmzZgmXLliEzMxP79+/H/v371e5rYGCgdtoB5SEKNeEPa3nMzc2xa9cu4WHPZ86cUXl6gUKXLl3wyy+/vPYblmq6gwcPCrPha3LkyJEa8T5gwKpFGjduDB8fH5w/fx7Hjx9HeHg4UlNTUVxcDAsLC7Ro0QKdO3eGh4eHMHu0GIyNjbFt2zYcOHAAR48eRUxMDIqKiuDg4ID33nsPH374YaVahUpzdHTEkSNHcOrUKfz111+IiIhAenq6MKbK0dERXbt2xaBBg4TxNHK5XPjANjQ0xIYNG8r9AzZy5EiEhITg+PHj2LdvH/r37w93d3et6/fuu+9i165d2LNnDyIiIpCRkaGx+0hh4MCBQguTk5MTunTpUonfinZGjBihMhdWRYPbf/rpJ5w/fx5Xr15FdHQ0UlNTkZGRARMTE9jZ2aFbt26YOHFijW5BsLKygo+PD44dO4bjx4/j7t27yMrKQr169dC2bVsMHTq0TNdgaW3atMHx48fx22+/ISgoCM+ePRP2Hz9+PN5//32tHm3zKu9dMW3ZsgUxMTG4dOkSwsLC8OjRIyQlJSEvLw916tSBlZUV2rRpg759+2LEiBHV8vBdlLQ4BAcHw9fXFyEhIXj06BGysrJgaGiIRo0aoU2bNujVqxeGDBmidhJIxSPELC0tNT7kvCawt7fH0aNH4efnh9OnT+PBgwfIzc2FpaUlOnToIDyOR6wpUKj6SOS6zChHb6yrV68K89bs2bMHPXv2rO4q6Z3Y2FjhETGff/45Pv744+quEpFe8vDwQEJCAhYsWKAy7xtRdWJEJqomiqkZjIyMatzdNUT6IiEhAQkJCbCwsBB1XjYiXTFgEVWDFy9ewNfXFygZjK7t8xmJSJVirOnUqVNhYWFR3dUhEnAMFtFrkpaWhpycHKSkpGDTpk3IzMyERCLRes4gIipr9OjRbAGmGokBi+g12bBhQ5kHPHt7e1fJ4GYiIqpeDFhEr5mxsTGaN2+O8ePHY8qUKdVdHSIiqgK8i5CIiIhIZGzB0hMymQzFxcUwMDB4pQexEhERkXpyuRwymQxGRkaizUHGgKUniouLERERUd3VICIiqrVcXFw0TkJcGQxYekKRqF1cXPj4BCIiIhFJpVJERESIOoM+A5aeUHQLGhoaMmARERFVATGH4HCiUSIiIiKRMWARERERiYwBi4iIiEhkDFhEREREImPAIiIiIhIZAxYRERGRyBiwiIiIiETGgEVEREQkMgYsIiIiIpExYBERERGJTG8flTN16lSEhoZWap89e/agZ8+eateFhITA19cXERERSE9Ph7W1NVxcXDB+/Hi4u7trVX5xcTH8/PwQEBCAmJgY5OXlwcbGBn369MHUqVPRpk2bStW3KsjlchQVFUEmk1V3VUhPGBgYwNjYWNRHSBAR1XZ6G7Aqy8DAAC1btizzukwmw4oVK3Dw4EGV15OTk5GcnIygoCB4eXlh9erVGh8CmZ6ejlmzZiEiIkLl9bi4OBw4cAD+/v5YuXIlvLy8RDwr7eXl5SErKwvZ2dmQSqXVUgfSX4aGhrCwsIClpSXMzMyquzpERDWe3gasdevWIT8/X+M2jx49wqJFiwAAvXv3hq2tbZltNm7cKIQrZ2dnfPTRR3BwcEBcXBy2b9+OqKgo+Pn5wdraGosXL1Z7HKlUivnz5wvhavDgwfDy8oKVlRVu3bqFX375BWlpaVi5ciVsbGy0bhETS3Z2NuLj42FsbAwrKyvUq1cPBgYGbJGgCsnlcshkMuTm5uLFixfIzMyEvb09LCwsqrtqREQ1mt4GLAcHhwq3OXr0qLA8atSoMusfP36MnTt3AgA6deoEHx8f1K1bFwDg6uoKDw8PTJkyBZGRkdixYwfGjh2LFi1alCnH398fN27cAAB4e3tj1apVwjpXV1f069cPY8aMQU5ODr799lv07dsXRkav51efl5eH+Ph41K9fH82aNWOooldSr149NG7cGM+ePUN8fDxatGjBliwiIg1q7SB3mUyGgIAAAICZmRkGDx5cZpvdu3ejuLgYALBixQohXCmYmppixYoVQMn4ql27dqk9liKkWVlZYenSpWXWt2jRArNnzwYAPHnyBIGBgTqfn7aysrJgbGzMcEU6k0gkaNasGYyNjZGVlVXd1SEiqtFqbcC6fPkykpOTAQBDhw6Fqampynq5XI7g4GAAgJOTE9zc3NSW4+bmBkdHRwBAcHAw5HK5yvrHjx8jOjq63OMoeHp6CstBQUE6nZu25HI5srOzUb9+fYYrEoVEIkH9+vWRnZ1d5logIqL/X60NWEeOHBGW1XUPxsfHIyUlBQDQvXt3jWX16NEDKBn4Hh8fr7JO0TWovJ06jRs3FgbZh4WFaX0euigqKoJUKkW9evVey/HozWBmZgapVIqioqLqrgoRUY1VKwNWbm6u0EpkZ2endmqGR48eCctOTk4ay1NeHxMTo7JO0XpVmXISExORl5dX4XnoSjEVg6a7H4kqy9DQEFB6fxERUVm18i/vX3/9JQSYESNGqO0eS0pKEpabNGmisTzl9YmJieWWo+4uRWVNmzYFSrrulPerauweJDHx/UREVDG9vYtQE+XuwdGjR6vdJjc3V1iu6G4o5XFVpVuelMupqCtOUznaqswcVlKpFHK5XPhHJAbF+0kqlXJONSKqFaris6zWBaykpCRhhnflAeqlvXz5Ulg2NjbWWKaJiYmwXFBQUCXlaKv0RKYVMTIyQn5+PrtzSDQvX75EUVER7t27V91VISKqsWpdwDp27JgQJsprvQKAOnXqCMsVDdYtLCwUlktP5VC6HOWfK1OOtlxcXIQxMBUpKCjAkydPYGpq+srHIypN8eic1q1b831FRLWCVCqtdANGRWpdwFJMLmpiYoL33nuv3O2Uu/Mq6q5TnjG+dHeicjm5ubkaA5amcrRlaGiodcAyNDSERCIR/hGJQfF+qsx7kYjoTVOrAlZERIRwd2D//v1haWlZ7rbKA9crGnCuvF4xUF1dOcnJybC2ti63HMUAeYlEUuHAetJPV69exbRp0wAA8+fPx4IFC1TWb9q0CZs3b9bpGJ6enli/fj2g4aHnBgYGsLCwgL29Pbp06YIJEybUiIeNExG9KWrVXYTKj8bR1D0IAK1btxaWS0+9UJry+tJTMbRq1arS5TRt2pSPGaEqJZPJkJWVhTt37mDv3r0YNWoUtm3bVt3VIiJ6Y9SaFqyioiKcOHECAGBtbY1+/fpp3N7e3h42NjZISUnBtWvXNG6rWG9rawt7e3uVdV27dhWWQ0NDMXz4cLVlPH/+HLGxsQCALl26aHlWr59MLoOBpFblbkFNODdvb28MGTJE7brg4GD897//BQAsXLgQAwYMULtdeS2zikdDoeR6iIuLQ1BQEAICAiCVSvHTTz/BwcEBw4YNE+VciIiofLUmYF24cAHp6ekAgPfff7/ChylLJBIMGDAA+/btQ0xMDMLDw9U+Lic8PFxoeRowYECZsUyOjo5o1aoVoqOjcfr0aXzxxRdqH5fj7+8vLA8cOPCVz7OqGUgM4HPrIpJzatez5mzNLTH5rXequxpo2LAhGjZsqHZdZGSksGxra4u2bdtWquzS23fs2BFDhw7FW2+9hbVr1wIAfv75ZwYsIqLXoNYErMp0DypMnz4dvr6+kEqlWLNmDXx8fFTuiiooKMCaNWuAkukOpk+frracGTNmYPny5cjMzMSGDRuwcuVKlfVPnz7F1q1bgZIHPw8aNOiVzvF1Sc7JQsKL9OquBolk8uTJ2LlzJ549e4aHDx/i+fPnaNy4cXVXi4ioVqsVfUFZWVk4d+4cUPItvmPHjlrt5+joiJkzZwIlrQeTJk3CyZMnERERgZMnT2LSpElCq8LMmTOFZwmW5unpKXT7+fj44NNPP8XFixdx+/Zt/PHHH5g4cSJycnJgYGCA5cuXV9i6RiQmAwMDlTGHpZ9GQERE4qsVf+lPnjwpzDGl7sHOmixatAhpaWk4dOgQoqKisGjRojLbjBs3DgsXLiy3DENDQ/z888+YNWsWIiIicObMGZw5c0ZlGxMTE6xcuRLu7u6Vqh+RGJQnwa1oQlwi0kwuk0HCZ7y+dvr2e68VAUvRPWhoaIgRI0ZUal8DAwOsW7cOQ4YMwYEDBxAREYGMjAw0aNAALi4umDBhglahyNraGvv374evry+OHz+O6Oho5Ofnw8bGBr1798a0adN4mzxVG+WHkjdr1qxa60Kk7yQGBsgK2gdpRkp1V+WNYdjABpYDJ1V3NSqlVgSs/fv361yGu7u7zq1LRkZG8Pb2hre3t871IRLLX3/9JdzB2rt3b43zwxGRdqQZKShOTajualANVisCFhGpKiwsFKZp+OWXX4CSh42r6wInIiLxMWAR1RLt2rUrd13Hjh2xfPlyvPXWW6+1TkREbyr9GS1GRK/E2NgYY8eOVZkUl4iIqhZbsIhqCeWZ3F+8eIH79+9j165dePr0KVavXo38/Hx89NFH1VpHIqI3BQMWUS1Reib3bt26YdSoUfD29sb9+/exceNG9OjRA66urtVWRyKiNwW7CIlqMXNzc/zwww8wMDBAcXExvv/+++quEhHRG4EBi6iWa9++Pd5//30AwPXr13HhwoXqrhIRUa3HgEX0BpgzZw4MSmZAVkzbQEREVYcBi+gN0KpVK+Eh42FhYbhy5Up1V4mIqFbjIHeiKnL37l0cPny4wu169er1Wh5fM2fOHOEZmb/88gt69epV5cckInpTMWARVZHg4GAEBwdXuN3PP//8WgKWs7Mz3N3dERISgitXriA8PBxubm5VflwiojcRAxaVYWte+55VVxvP6VXMmTMHISEhAIAtW7Zg27Zt1V0lIqJaiQGLVMjkMkx+653qrkaVkMllMJBU7bDDnj174v79+6+075gxYzBmzJhK7bN3795Kbd+lS5dXrh8REWmPg9xJRVUHkOpUm8+NiIhqFv7FISIiIhIZAxYRERGRyBiwiIiIiETGgEVEREQkMgYsIiIiIpExYBERERGJjAGLiIiISGQMWEREREQiY8AiIiIiEhkDFhEREZHIGLCIiIiIRMaARURERCQyBiwiIiIikTFgEREREYmMAYuIiIhIZAxYRERERCJjwCIiIiISGQMWERERkcgYsIiIiIhEZlTdFSCqTa5evYpp06apXVe3bl1YWVmhffv2GDRoEEaOHAkTExMAgIeHBxISEnQ69p49e9CzZ0/Ex8djwIABAABPT0+sX79ep3KJiKjy2IJF9JoUFBQgKSkJ58+fx/LlyzFmzBjEx8dXd7WIiKgKsAWLVMhlMkgMamfuft3nNmnSJHh7ews/p6Wl4eHDh9ixYweSkpLw8OFDfPLJJzhy5Ah27NiBoqIiteUsW7YMkZGRAICAgIByj2dvb18FZ0FERK+CAYtUSAwMkBW0D9KMlOquiqgMG9jAcuCk13rMhg0bom3btiqv9e7dG2PGjMHIkSORkJCABw8eIDAwEEOHDi23HDMzM2G5dHlERFQzMWBRGdKMFBSn6jYeiMpnbm6OTz75BF999RUA4J9//tEYsIiISP/Uzr4gohquXbt2wnJSUlK11oWIiMTHgEVUDYyNjYVlIyM2JBMR1TYMWETVIDo6Wli2s7Or1roQEZH4GLCIXjOpVIodO3YIPw8ZMqRa60NEROJjwCJ6TdLT03H58mVMmTIFUVFRQEm46tatW3VXjYiIRMbBH0RVZPPmzdi8ebPadaamppg4cSI+++yz114vIiKqemzBIqoG7du3x9SpU1UGuxMRUe3BFiyiKqI8k7tUKkVSUhLOnDmDo0eP4ubNm5g6dSoOHjwIa2vr6q4qERGJjC1YRFVEMZN727Zt0aFDB/Tv3x/r16/HunXrAAAJCQlYvnx5dVeTiIiqAAMW0Wvm6ekp3Dl49uxZXL58ubqrREREImPAIqoGixYtgqGhIQBg48aN1V0dIiISGQMWUTVwdHTEsGHDAAC3bt3C33//Xd1VIiIiETFgEVWT2bNnQyKRAAB++eWX6q4OERGJiHcRElWTtm3bwsPDA8HBwbh27RquX78u+qSjT548weHDhyvcztXVFa1btxb12EREbzIGLKJqNGfOHAQHBwMlrVjKj9ARQ1hYGMLCwircbtmyZQxYREQiYsCiMgwb2FR3FURXU8/J1dUVffv2xd9//41Lly7h9u3bcHV1re5qERGRjhiwSIVcJoPlwEnVXY0qIZfJIDGo2mGHPXv2xP379yu1z86dOzWu37t3b6XKs7e3r3QdiIhIXBzkTiqqOoBUp9p8bkREVLPwLw4RERGRyBiwiIiIiETGgEVEREQkMgYsIiIiIpExYBERERGJjAGLiIiISGQMWEREekgml1V3FYhIA040SkSkhwwkBvC5dRHJOVnVXZU3SvvGzfBe2y7VXQ3SAwxYRER6KjknCwkv0qu7Gm8Um3r1q7sKpCfYRUhEREQkMgasN4BcLq/uKlAtwvcTEVHFGLBqMUNDQwBAcXFxdVeFahHF+0nx/iIiorIYsGoxIyMj1KlTB1lZHARL4snKykKdOnVgZMQhnERE5WHAqsUkEgmsrKyQnZ2NjIyM6q4O1QIZGRnIzs6GlZUVJBJJdVeHiKjGqlVfQZ89e4aDBw/i/PnzePbsGXJzc2FtbQ07Ozv07NkTw4YNQ9u2bcvdPyQkBL6+voiIiEB6ejqsra3h4uKC8ePHw93dXas6FBcXw8/PDwEBAYiJiUFeXh5sbGzQp08fTJ06FW3atBHxjCvWoEEDFBYWIikpCS9evIC5uTnq1q0LAwMD/oGkCsnlcshkMhQUFCAnJwd5eXlo0KABGjRoUN1VIyKq0WpNwNq7dy/+85//IC8vT+X1pKQkJCUl4caNG8jJycHy5cvL7CuTybBixQocPHhQ5fXk5GQkJycjKCgIXl5eWL16NQwMym/0S09Px6xZsxAREaHyelxcHA4cOAB/f3+sXLkSXl5eOp+vtiQSCZo0aQJTU1O8ePECqampkMk4QSFVjoGBAczMzNCsWTNYWlpWd3WIiGq8WhGwtmzZgv/9738AgJYtW2L8+PFwcXGBhYUFMjMzERUVhcDAwHLD0caNG4Vw5ezsjI8++ggODg6Ii4vD9u3bERUVBT8/P1hbW2Px4sVqy5BKpZg/f74QrgYPHgwvLy9YWVnh1q1b+OWXX5CWloaVK1fCxsZG6xYxsVhaWsLS0hIymQzFxcUMWaQ1AwMDGBkZafxyQUREqvQ+YF2+fFkIV6NHj8batWthbGyssk3v3r0xc+ZMFBYWltn/8ePH2LlzJwCgU6dO8PHxQd26dQEArq6u8PDwwJQpUxAZGYkdO3Zg7NixaNGiRZly/P39cePGDQCAt7c3Vq1aJaxzdXVFv379MGbMGOTk5ODbb79F3759q2WQsIGBAUxMTF77cYmIiN4kev2VVCaT4euvvwYAtG/fHt9++22ZcKVMXbDYvXu3cNv5ihUrhHClYGpqihUrVgAl46t27dqltmxFSLOyssLSpUvLrG/RogVmz54NAHjy5AkCAwMrcaZERESkT/Q6YF26dAmxsbEAgI8//rjSLUJyuRzBwcEAACcnJ7i5uandzs3NDY6OjgCA4ODgMhMtPn78GNHR0QCAoUOHwtTUVG05np6ewnJQUFCl6kpERET6Q68D1unTp4GSgdzvvvuu8HpmZiZiY2ORmZmpcf/4+HikpKQAALp3765x2x49egAlA9/j4+NV1im6BpW3U6dx48Zo2bIlACAsLEzj8YiIiEh/6fUYrFu3bgEA7OzsYG5ujoCAAGzbtg0PHjwQtlEMep86dWqZLsJHjx4Jy05OThqPpbw+JiYGDg4Ows+K1itty4mNjUViYiLy8vJgZmam1bkSERGR/tDbFiyZTIaYmBigZK6ntWvX4vPPP1cJVwAQGxuLH374AdOmTcOLFy9U1iUlJQnLTZo00Xg85fWJiYnllmNra6uxnKZNmwIl3ZPK+xEREVHtobctWNnZ2cJUAw8ePEBERAQaN26MpUuXwt3dHXXq1EFERAR+/PFHhIeH4+bNm/jyyy+xefNmoYzc3FxhuaKWJOVxVaXn2lIup169eq9cjjakUmml9yGi2ofPgqQ3UVX9DayKcvU2YOXn5wvLL1++hKmpKfbs2aPSRde9e3fs3r0bEyZMwL179xAYGIhbt27hrbfeEvZT0HT3IUrdgVhQUKCyTqxytFF6ElMievOYmprC2dm5uqtB9Nrdv39f5e9/Taa3Aav0eKpx48apHf9Ut25dLFq0SJgi4eTJk0LAqlOnjrBdUVGRxuMpz6FVeiqH0uUo/1yZcrTh4uLCb65ERPRGateuXZWUK5VKRW/A0NuAZW5urvLz22+/Xe62vXv3hpGREYqLi1V+gcrdeRV11ykn5tLdicrl5ObmagxYmsrRhqGhIQMWERG9kfTp75/eDnI3MTGBtbW18LOmQep16tQRHk6bnp6udp+KBpwrr1cMVFdXTnJyssZyFAPkFc8IJCIiotpHbwMWALRu3VpYrujZeooBbMqTkSrvr7gjsTzK60t3RbZq1arS5TRt2pRTNBAREdVSeh2wlCcHjYuLK3e7nJwcZGRkAKWmUbC3t4eNjQ0A4Nq1axqPpVhva2sLe3t7lXVdu3YVlkNDQ8st4/nz58LM8126dNF4PCIiItJfeh2wBg8eLCxrerZfYGCg8Hgb5TAkkUgwYMAAoKRlKTw8XO3+4eHhQsvTgAEDIJFIVNY7OjoKrVinT58u9w4Hf39/YXngwIFanSMRERHpH70OWO3bt0e/fv0AACdOnMDly5fLbPP8+XP897//BUqmUBg7dqzK+unTpwuD5tasWVNm6oSCggKsWbMGKOlenD59utq6zJgxAyh5TM+GDRvKrH/69Cm2bt0KlDz4edCgQa90zkRERFTz6XXAAoAvv/wS9evXh0wmw+zZs/HTTz/h+vXriIiIgI+PD8aNGycMUP/Xv/5VZqZ1R0dHzJw5EwAQGRmJSZMm4eTJk4iIiMDJkycxadIkREZGAgBmzpwpPEuwNE9PT6Hbz8fHB59++ikuXryI27dv448//sDEiRORk5MDAwMDLF++vNIPpiYiIiL9IZEr+s702PXr1/Gvf/0LqampatdLJBLMmTMHCxcuVLteJpPhq6++wqFDh8o9xrhx47BmzRoYGJSfSdPT0zFr1qxy59IwMTHBypUr4eXlVeE5lSaVShEeHg43Nze9uk2ViKrOf/4+joQX6VpsSWLp3LQlprj1Q7rf/1CcmlDd1XljGDWyg7XXv6qs/Kr4G1srmlG6deuG48eP448//kBQUBDi4+NRVFSExo0bo0ePHpg6darGWY8NDAywbt06DBkyBAcOHEBERAQyMjLQoEEDuLi4YMKECXB3d6+wHtbW1ti/fz98fX1x/PhxREdHIz8/HzY2NujduzemTZuGNm3aiHz2REREVNPUioCFkgc+L1iwAAsWLHjlMtzd3bUKUpoYGRnB29sb3t7eOpVDRERE+kvvx2ARERER1TQMWEREREQiY8AiIiIiEhkDFhEREZHIGLCIiIiIRMaARURERCQyBiwiIiIikTFgEREREYmMAYuIiIhIZAxYRERERCJjwCIiIiISGQMWERERkcgYsIiIiIhExoBFREREJDIGLCIiIiKRMWARERERiYwBi4iIiEhkDFhEREREImPAIiIiIhIZAxYRERGRyBiwiIiIiETGgEVEREQkMgYsIiIiIpExYBERERGJjAGLiIiISGQMWEREREQiY8AiIiIiEhkDFhEREZHIGLCIiIiIRMaARURERCQyBiwiIiIikTFgEREREYmMAYuIiIhIZAxYRERERCJjwCIiIiISGQMWERERkcgYsIiIiIhExoBFREREJDIGLCIiIiKRMWARERERiYwBi4iIiEhkDFhEREREImPAIiIiIhIZAxYRERGRyBiwiIiIiETGgEVEREQkMgYsIiIiIpExYBERERGJjAGLiIiISGQMWEREREQiY8AiIiIiEhkDFhEREZHIGLCIiIiIRMaARVTN5DJZdVfhjcXfPRFVFaPqrgDRm05iYICsoH2QZqRUd1XeKIYNbGA5cFJ1V4OIaikGLKIaQJqRguLUhOquBhERiYRdhEREREQiY8AiIiIiEhkDFhEREZHIGLCIiIiIRMaARURERCQyBiwiIiIikTFgEREREYmMAYuIiIhIZAxYRERERCJjwCIiIiISGQMWERERkcgYsIiIiIhExoBFREREJDIGLCIiIiKRGVV3BXTRrl07rbbr0aMH9u7dq3GbkJAQ+Pr6IiIiAunp6bC2toaLiwvGjx8Pd3d3rY5TXFwMPz8/BAQEICYmBnl5ebCxsUGfPn0wdepUtGnTRqtyiIiISL/pdcASg0wmw4oVK3Dw4EGV15OTk5GcnIygoCB4eXlh9erVMDAov8EvPT0ds2bNQkREhMrrcXFxOHDgAPz9/bFy5Up4eXlV2bkQERFRzVArAtakSZPg7e1d7npTU9Ny123cuFEIV87Ozvjoo4/g4OCAuLg4bN++HVFRUfDz84O1tTUWL16stgypVIr58+cL4Wrw4MHw8vKClZUVbt26hV9++QVpaWlYuXIlbGxstG4RIyIiIv1UKwJWw4YN0bZt20rv9/jxY+zcuRMA0KlTJ/j4+KBu3boAAFdXV3h4eGDKlCmIjIzEjh07MHbsWLRo0aJMOf7+/rhx4wYAwNvbG6tWrRLWubq6ol+/fhgzZgxycnLw7bffom/fvjAyqhW/eiIiIlLjjR7kvnv3bhQXFwMAVqxYIYQrBVNTU6xYsQIoGV+1a9cuteUoQpqVlRWWLl1aZn2LFi0we/ZsAMCTJ08QGBgo+rkQERFRzfHGBiy5XI7g4GAAgJOTE9zc3NRu5+bmBkdHRwBAcHAw5HK5yvrHjx8jOjoaADB06NByuyM9PT2F5aCgINHOg4iIiGqeNzZgxcfHIyUlBQDQvXt3jdv26NEDKBn4Hh8fr7JO0TWovJ06jRs3RsuWLQEAYWFhOtWdiIiIarZaMRDo9OnTOHXqFBISEmBgYIDGjRujc+fO8PT0RK9evdTu8+jRI2HZyclJY/nK62NiYuDg4CD8rGi90rac2NhYJCYmIi8vD2ZmZlqdHxEREemXWhGwlMMSSsY5PXnyBEeOHMHAgQOxfv16WFhYqGyTlJQkLDdp0kRj+crrExMTyy3H1tZWYzlNmzYFSronk5KSKgxkREREpJ/0OmCZmprCw8MDvXv3hqOjI+rVq4f09HSEhoZi//79yMzMRFBQEObOnYudO3fC2NhY2Dc3N1dYrqglSXlcVV5enso65XLq1av3yuVoSyqVvtJ+VHMZGhpWdxXeaPp6TfF9Q2+iqrpeq6JcvQ5YFy5cQP369cu83rdvX0ydOhUff/wxoqKiEBoain379mHatGnCNi9fvhSWlYOXOiYmJsJyQUGByjqxytFW6YlMSb+ZmprC2dm5uqvxRrt//z7y8/OruxqVwvcNvan06XrV64ClLlwpNGrUCP/v//0/DBs2DEVFRfjjjz9UAladOnWE5aKiIo3HKSwsFJZLT+VQuhzlnytTjrZcXFz4zZVIRNo+couIql9VXa9SqVT0Bgy9DlgVcXBwQJ8+fRASEoInT54gOTlZGCel3J1XUXedclou3Z2oXE5ubq7GgKWpHG0ZGhoyYBGJiNcTkf7Qp+u11k/T0KpVK2E5OTlZWFYeuK48UF0d5fWKgerqylEuXx3FAHmJRFLhwHoiIiLSX7U+YEkkErWvt27dWliOiYnRWIby+tJ3/ikHOG3Ladq0KadoICIiqsVqfcBSnqdKeRoFe3t72NjYAACuXbumsQzFeltbW9jb26us69q1q7AcGhpabhnPnz9HbGwsAKBLly6VPg8iIiLSH7U6YMXFxeHvv/8GADRv3lwlYEkkEgwYMAAoaVkKDw9XW0Z4eLjQ8jRgwIAyLWKOjo5CK9bp06fLvbvB399fWB44cKDO50ZEREQ1l94GrLNnzwoPalYnNTUVn376qXCHoLe3d5ltpk+fLgyYW7NmTZmpEwoKCrBmzRoAgJGREaZPn672WDNmzAAAZGZmYsOGDWXWP336FFu3bgVKHvw8aNCgSpwpERER6Ru9vYtw7dq1KCoqwpAhQ+Dm5gY7OzvUrVsXGRkZuHr1Kg4cOICMjAygpBtv8uTJZcpwdHTEzJkzsW3bNkRGRmLSpEn4+OOP4eDggLi4OPz222+IiooCAMycOVN4lmBpnp6eOHToEMLCwuDj44PU1FR4eXnB0tISt2/fxpYtW5CTkwMDAwMsX74cRkZ6+2snIiIiLej1X/qUlBTs3bsXe/fuLXebIUOGYO3atSqTfCpbtGgR0tLScOjQIURFRWHRokVlthk3bhwWLlxY7jEMDQ3x888/Y9asWYiIiMCZM2dw5swZlW1MTEywcuVKuLu7V+ociYiISP/obcBav349QkNDER4ejri4OGRmZiInJwdmZmZo0qSJ8LDnzp07ayzHwMAA69atw5AhQ3DgwAFEREQgIyMDDRo0gIuLCyZMmKBVKLK2tsb+/fvh6+uL48ePIzo6Gvn5+bCxsUHv3r0xbdo0tGnTRsTfABEREdVUehuwevTogR49eohWnru7u86tS0ZGRvD29lY73ouIiIjeHHo7yJ2IiIiopmLAIiIiIhIZAxYRERGRyBiwiIiIiETGgEVEREQkMgYsIiIiIpExYBERERGJjAGLiIiISGQMWEREREQiY8AiIiIiEhkDFhEREZHIGLCIiIiIRMaARURERCQyBiwiIiIikTFgEREREYmMAYuIiIhIZDoFrMLCQvFqQkRERFRL6BSw3nnnHaxduxZ3794Vr0ZEREREek6ngJWVlQUfHx+MGTMGnp6e8PHxQVZWlni1IyIiItJDOgWsQYMGwdDQEHK5HHfv3sXatWvxzjvvYPHixbh06ZJ4tSQiIiLSI0a67Lxp0yZkZGTg2LFj8Pf3x71791BYWIhTp07h1KlTaNKkCTw9PeHp6QkHBwfxak1ERERUg+l8F2GDBg0wffp0HDlyBIcPH8bkyZNRv359yOVyJCYm4pdffsGQIUMwbdo0HDt2DC9fvhSn5kREREQ1lKjTNDg7O2PFihW4ePEiNm7ciHfeeQcSiQQymQzXrl3Dv//9b7z99ttYtWoVbt++LeahiYiIiGqMKpkHy8TEBMOGDcNvv/2Gc+fOYeHChWjevDnkcjmys7Ph6+uLCRMmYMSIEdi1axcHxhMREVGtUuUTjdra2mL69On4+OOP0ahRI0gkEgCAXC7Hw4cP8f3338Pd3R3fffcdsrOzq7o6RERERFVOp0HuFbl+/ToOHTqEM2fOID8/HygJVpaWlhgyZAgePnyImzdvoqCgAHv27EFgYCD+/PNPNGnSpCqrRURERFSlRA9YSUlJ8Pf3h7+/P+Li4oCSUCWRSNCjRw94eXlh8ODBMDExAQA8fvwY27Ztw5EjR5CYmIj//ve/WL9+vdjVIiIiInptRAlYhYWFCAwMxOHDh3HlyhXIZDLI5XIAQOPGjTFmzBiMGzdO7VQNjo6O+O6772BnZ4fNmzfj8uXLYlSJiIiIqNroFLBu376Nw4cP4+TJk8L4KblcDkNDQ/Tr1w9eXl549913YWBQ8VCvQYMGYfPmzXj+/LkuVSIiIiKqdjoFrPHjx0MikQitVc2bN8fYsWPh6ekJGxubSpVlamoKlAQ0IiIiIn2mcxehsbExBg0aBC8vL/Tq1euVy7G1tcWePXt0rQ4RERFRtdMpYC1fvhwjR46EpaWlzhWpU6cOevTooXM5RERERNVNp4A1depU8WpCREREVEvoNNHogAEDMHDgQDx58kTrfZ49eybsR0RERFQb6dSClZCQAIlEgqKiIq33KS4uFvYjIiIiqo2q/FE5RERERG+a1x6wFPNl1a1b93UfmoiIiOi1eO0B69ixYwAAOzu7131oIiIioteiUmOwpk2bpvb1ZcuWCROFlqewsBDx8fFIS0uDRCJB3759K1dTIiIiIj1RqYAVGhqqMnM7SmZej4iIqNRBHRwcMHv27ErtQ0RERKQvKhWwunfvrvLztWvXIJFI0LFjR40tWBKJBHXq1EHjxo3RuXNnDB8+HGZmZq9eayIiIqIarFIBa+/evSo/t2/fHgCwfv16tG7dWtyaEREREekpnebBGj16NCQSCerXry9ejYiIiIj0nE4Ba/369eLVhIiIiKiW4ESjRERERCJjwCIiIiISmVZdhB06dABK7gaMiooq8/qrKF0WERERUW2hVcBSnvdKm9eJiIiI3mRaBaz58+dX6nUiIiKiNxkDFhEREZHIOMidiIiISGQMWEREREQi0ylg+fv7v9J+L168wOLFi3U5NBEREVGNpVPAWrZsGRYuXIisrCyt97ly5QpGjhyJU6dO6XJoIiIiohpL5y7CM2fOYNSoUbh8+bLG7YqKirB+/XrMmDEDSUlJkEgkuh6aiIiIqEbSKWBNnz4dAJCUlISZM2fi+++/R1FRUZntHjx4gLFjx2L37t2QyWRo3Lgxtm3bpsuhiYiIiGosnbsId+zYARsbG8hkMuzatQvjxo3Dw4cPhW1+//13eHl54eHDh5DL5Rg0aBCOHTuGt99+W4z6ExEREdU4Ws2DpUmfPn0QEBCAr776CoGBgbh//z7GjRuHefPm4Z9//sHVq1chl8thZmaGL7/8EuPGjROn5kREREQ1lCjTNFhaWmLTpk1Yu3YtzMzM8PLlS2zcuFEIV2+99RaOHDnCcEVERERvBFHnwXrvvffQs2dP4We5XA4LCwusW7cOzZs3F/NQRERERDWWaAHr9u3b8PT0xPnz5wEApqamAICcnByMGzcOfn5+Yh2KiIiIqEbTOWDJ5XL8/PPP8Pb2xpMnTyCXy+Hl5YWLFy/iyy+/hImJCfLz87Fy5UrMnz8fGRkZ4tSciIiIqIbSKWDFx8fD29sbmzdvRnFxMaysrLB582asWbMG9erVw7Rp03Do0CG0b98ecrkcwcHBGDFiBC5evCjeGRARERHVMDoFrJEjRyI8PBxyuRx9+/bFsWPHMHDgQJVtWrduDT8/P8yYMQMSiQSpqamYNWsW1qxZo2vdiYiIiGoknQJWXl4ejI2NVebDUsfY2BhLly7F77//jmbNmkEul+PPP//U5dBERERENZZOAatt27Y4dOiQMKN7RXr27ImjR49i+PDhuhyWiIiIqEbTaaLRgwcPwsTEpFL7WFhY4KeffkL//v11OTQRERFRjaVTC1Zlw5Wy999/X5dDExEREdVYOj8qR9nTp09x8+ZNpKamIj8/H97e3rC2thbzEEREREQ1nigB686dO1i3bh3CwsJUXh86dKhKwPLx8cHmzZthYWGBEydOwNjYWIzDExEREdUoOk80eu7cOUyaNAlhYWGQy+XCP3VGjRqFgoICxMXFCTO+ExEREdU2OgWslJQULF68GIWFhWjdujV+++23Mq1YyszNzeHh4QEAuHDhgi6H1mjDhg1o166d8O/q1asV7hMSEoJ58+ahX79+6NSpE/r164d58+YhJCRE6+MWFxdj37598Pb2Rq9eveDq6oqBAwdi5cqVePjwoY5nRURERPpCpy7CXbt2IT8/H82aNYOPjw/q169f4T49e/bEiRMncOfOHV0OXa67d+9i165dWm8vk8mwYsUKHDx4UOX15ORkJCcnIygoCF5eXli9ejUMDMrPo+np6Zg1axYiIiJUXo+Li8OBAwfg7++PlStXwsvL6xXOioiIiPSJTgHr4sWLkEgkmDFjhlbhCgCcnJyAksfsiE0RloqLi9GwYUOkpaVVuM/GjRuFcOXs7IyPPvoIDg4OiIuLw/bt2xEVFQU/Pz9YW1tj8eLFasuQSqWYP3++EK4GDx4MLy8vWFlZ4datW/jll1+QlpaGlStXwsbGBu7u7iKfOREREdUkOnURPnv2DADg6uqq9T7m5uZAySzwYtuzZw8iIiLg5OSEcePGVbj948ePsXPnTgBAp06dsG/fPgwfPhyurq4YPnw4/vzzT3Tq1AkAsGPHDjx58kRtOf7+/rhx4wYAwNvbG5s2bUK/fv3g6uqKqVOnYt++fTA3N4dMJsO3336L4uJiUc+biIiIahadApZUKgVKWo60lZ2dDQAwMzPT5dBlPHv2DP/73/8AAN98841Wdyju3r1bCDsrVqxA3bp1VdabmppixYoVQMn4qvK6HhUhzcrKCkuXLi2zvkWLFpg9ezYA4MmTJwgMDKz0+REREZH+0ClgNWrUCCgZZ6St27dvAwCaNm2qy6HLWL16NfLy8uDp6YkePXpUuL1cLkdwcDBQ0m3p5uamdjs3Nzc4OjoCAIKDg8vcIfn48WNER0cDJdNSmJqaqi3H09NTWA4KCqrEmREREZG+0SlgdevWDXK5HKdPn9Zq+8LCQhw4cAASiUSrEKStkydP4ty5c+W2IKkTHx+PlJQUAED37t01bquoa3JycpmxY4quQeXt1GncuDFatmwJABrvtCQiIiL9p1PAUrTKnD17Fn///bfGbQsLC/Hvf/8bT58+hUQiwfjx43U5tODFixdYt24dAODzzz/Xeub4R48eCcuKgfflUV4fExOjsk7RelWZchITE6tkDBoRERHVDDrdRdizZ0+89957OHnyJObMmYNp06ZhyJAhwvqEhAS8ePECYWFh8PX1RVxcHCQSCSZOnIg2bdqIUX9s2LABz58/R5cuXbQa2K6QlJQkLDdp0kTjtsrrExMTyy3H1tZWYzmKblG5XI6kpKQKAxkRERHpJ50flbN+/Xrk5uYiJCQEO3fuxM6dOyGRSAAAc+bMEbZTjF0aPHgwli9fruthAQDXr1+Hn58fjIyM8M033wjH1UZubq6wXNGAe+VxVaVbnpTLqVev3iuXoy3FjQVUexgaGlZ3Fd5o+npN8X1Db6Kqul6rolydA5aJiQm2bt0KX19fbN++HU+fPlW7XZMmTTB79mxMmjRJ10MCJV2OK1asgFwux/Tp09G2bdtK7f/y5UthuaI7Dk1MTITlgoKCKilHW6UnMiX9ZmpqCmdn5+quxhvt/v37yM/Pr+5qVArfN/Sm0qfrVZSHPQPA+PHjMX78eDx69AiRkZFIS0uDVCpFgwYN0KFDB3Ts2LFSLUwV2bp1K2JiYtCsWTPMnz+/0vvXqVNHWC4qKtK4bWFhobBceiqH0uUo/1yZcrTl4uLCb65EImrXrl11V4GItFRV16tUKhW9AUO0gKXQunVrtG7dWuxiVURHR2Pr1q0AgK+++uqV5tRS7s6rqLtOOS2XPpZyObm5uRoDlqZytGVoaMiARSQiXk9E+kOfrlfRA9brsHv3bhQVFcHBwQEFBQU4ceJEmW2UH6585coVpKamAgD69+8PMzMzlYHrygPV1VFeX3r+LuVykpOTNd7FqBggL5FIKhxYT0RERPpLLwOWoqstLi6u3OcDKtuyZYuwHBwcDDMzM5VWttJTL5SmvL70nX+tWrVS2a5Dhw4VltO0aVPRZ7InIiKimkOrgHXkyJEqOfjo0aOrpFxt2Nvbw8bGBikpKbh27ZrGbRXrbW1tYW9vr7Kua9euwnJoaCiGDx+utoznz58jNjYWANClSxcRzoCIiIhqKq0C1hdffCHqAHWUdJO9asBav3491q9fr3GbTZs2YfPmzUDJQ6B79uxZ5vgDBgzAvn37EBMTg/DwcLWPywkPDxdangYMGFDm9+Do6IhWrVohOjoap0+fxhdffKH2cTn+/v7C8sCBAyt5xkRERKRPtJ7JXS6Xi/6vuk2fPl0YMLdmzZoyUycUFBRgzZo1AAAjIyNMnz5dbTkzZswAAGRmZmLDhg1l1j99+lQYlN+iRQsMGjRI9HMhIiKimkOrFizFQ5FrG0dHR8ycORPbtm1DZGQkJk2ahI8//hgODg6Ii4vDb7/9hqioKADAzJkzhWcJlubp6YlDhw4hLCwMPj4+SE1NhZeXFywtLXH79m1s2bIFOTk5MDAwwPLly2FkpJdD34iIiEhLWv2lt7Ozq/qaVJNFixYhLS0Nhw4dQlRUFBYtWlRmm3HjxmHhwoXllmFoaIiff/4Zs2bNQkREBM6cOYMzZ86obGNiYoKVK1fC3d29Ss6DiIiIao43vinFwMAA69atw5AhQ3DgwAFEREQgIyMDDRo0gIuLCyZMmKBVKLK2tsb+/fvh6+uL48ePIzo6Gvn5+bCxsUHv3r0xbdo00Z6/SERERDVbrQ1YCxYswIIFC7Te3t3dXefWJSMjI3h7e8Pb21uncoiIiEi/iRqw7ty5g3/++QcPHjxAVlYWAMDS0hJt2rRBnz590KlTJzEPR0RERFQjiRKw7ty5g2+++Ubjc3w2btyITp06YeXKlXBxcRHjsEREREQ1ktbTNJTn9OnTmDhxIiIiIoTpF4yMjNCwYUM0bNgQRkZGwusRERGYNGkSTp06JU7tiYiIiGognVqwYmJisHTpUhQVFcHIyAheXl4YO3YsOnToIMwvJZVKce/ePRw8eBB+fn4oLi7Gv//9b7Rt21blMTNEREREtYVOAeu3335DYWEh6tSpg23btpWZLR0lUxh07NgRHTt2xLBhw/Dxxx+jsLAQ27dvx3fffafL4YmIiIhqJJ26CC9fvgyJRILp06erDVel9ejRA9OnT4dcLsfly5d1OTQRERFRjaVTwEpPTwcA9OvXT+t9FFMhKPYlIiIiqm10CljW1tYAgDp16mi9j4mJCQCgQYMGuhyaiIiIqMbSKWB16dIFADROz1Da7du3AQBdu3bV5dBERERENZZOAeuDDz6AoaEhtm7dqlWXX1paGrZt2wYjIyN88MEHuhyaiIiIqMbSKWC5urrim2++QVpaGry8vBAUFASZTFZmO5lMhqCgIEyYMAHp6en4+uuv4erqqsuhiYiIiGosnaZpWLZsGQCgdevWuHfvHhYsWID69evD2dkZ1tbWkEgkSEtLw927d4VH57Rv3x43btzAjRs31JYpkUiwbt06XapFREREVK10Clj+/v6QSCRASTCSy+XIysrClStXVLaTy+XCNvfu3cO9e/fUlieXyxmwiIiISO/pFLCaNWsmXk2IiIiIagmdAtbZs2fFqwkRERFRLaHzw56JiIiISJVOLVibN28GALz11lt45513xKoTERERkV7TOWBJJBIhaBERERGRjl2EVlZWAAe7ExEREanQKWC1aNECAPD8+XOx6kNERESk93QKWMOGDYNcLsepU6fEqxERERGRntMpYHl7e6N9+/Y4evQoDh8+LF6tiIiIiPSYToPcU1NTsXbtWixfvhzLly/H8ePH8f7776Ndu3aoX78+DA0NNe7PsVtERERUG+kUsDw8PIRH5cjlcly+fBmXL1/Wal+JRIKoqChdDk9ERERUI+kUsKD0nMHSy0RERERvKp0C1nfffSdeTYiIiIhqCZ0Clqenp3g1ISIiIqol+CxCIiIiIpExYBERERGJTOdB7goymQxXr17FzZs3kZqaivz8fCxatAg2NjbCNoWFhZBKpTA0NISJiYlYhyYiIiKqUUQJWOfOncPatWvx7NkzlddnzpypErD8/Pywdu1amJmZ4eLFizAzMxPj8EREREQ1is5dhL6+vpg7dy4SEhIgl8thZWVV7nQNXl5esLCwQF5eHgIDA3U9NBEREVGNpFPAio2NxerVqwEAvXr1wokTJzRONGpiYoLBgwdDLpfj77//1uXQRERERDWWTgFr165dKC4uRuvWrbFt2za0atWqwn26desGALh7964uhyYiIiKqsXQKWFeuXIFEIsH06dO1HrTevHlzAEBiYqIuhyYiIiKqsXQKWMnJyQCA9u3ba72PYmB7QUGBLocmIiIiqrFEmQerMmEpIyMDAGBubi7GoYmIiIhqHJ0Clq2tLQAgLi5O631u3LgBAHBwcNDl0EREREQ1lk4Bq0ePHpDL5fD399dq++zsbOzfvx8SiQS9evXS5dBERERENZZOAWvixImQSCS4du0aDh8+rHHbjIwMzJ07F6mpqTA0NMTEiRN1OTQRERFRjaXTTO7Ozs6YNm0adu/ejeXLl+PChQsYPHiwsP7mzZu4e/cuwsLCcPz4ceTk5EAikWDu3Lmws7MTo/5ERERENY7Oj8r54osvUFhYiH379uHMmTM4c+YMJBIJAGDlypXCdorZ3adPn465c+fqelgiIiKiGkvnuwglEglWrVqFHTt2oEePHpBIJJDL5Sr/AMDNzQ1bt27FsmXLxKg3ERERUY0lysOeAaBv377o27cvcnJycPfuXaSlpUEmk8HKygrt27eHtbW1WIciIiIiqtF0CliZmZlCiDIw+L/GMHNzc3Tv3l2s+hERERHpnUoHrHv37mHLli34+++/kZeXBwAwNjZGt27dMHPmTPTt27cq6klERESkNyo1BisoKAjjx49HYGAgcnNzhTFWhYWFuHz5Mj766CNs27at6mpLREREpAe0DlgpKSnCHYNyuRx169ZFx44d4ebmhvr16wth67///S/CwsKqttZERERENZjWXYS+vr7CPFYffPABFixYIDy4WSqVYv/+/Vi3bh1kMhl27dqFLl26VGW9iYiIiGosrVuw/vnnH0gkEvTv3x///ve/hXAFAIaGhpg8eTI++ugjyOVyXL58uarqS0RERFTjaR2wYmJiAABjxowpd5uxY8cCAHJycpCamipG/YiIiIj0jtYBKycnBwBgb29f7jbKj7/Jzs7WtW5EREREeknrgFVcXAwAMDIqf9iWoaGhsCyVSnWtGxEREZFe0vlROURERESkqtIBS/EgZ7G2IyIiIqptKj2T+4wZMzR2E2q7nUQiQVBQUGUPT0RERFTjVTpgJScna1yvaLnSdjsiIiKi2kbrgNWsWbOqrQkRERFRLaF1wDp79mzV1oSIiIioluBdhEREREQiY8AiIiIiEhkDFhEREZHIGLCIiIiIRMaARURERCQyBiwiIiIikTFgEREREYmMAYuIiIhIZAxYRERERCJjwCIiIiISGQMWERERkci0fhahOh4eHjAwMMCOHTvQokULrfZ59uwZpk6dColEgqCgoFc+dk5ODkJCQhAREYHIyEgkJycjPT0dL1++hIWFBVq3bo1+/fph3LhxaNCgQYXlhYWF4c8//8SNGzeQmpqK+vXro3379vD09MT777+vdb2OHz+Ow4cP4/79+3jx4gUaNWqErl27YvLkyejcufMrny8RERHpD50C1rNnzyCRSFBUVKT1PsXFxUhISIBEItHl0Lh9+zYWL16sdl16ejpCQ0MRGhqKHTt2YMOGDXjnnXfKLWvTpk3YsmULZDKZ8FpqaiouXbqES5cuISAgAP/v//0/1KlTp9wyCgoK8OmnnyIkJETl9WfPnuHZs2c4ceIE5s2bh/nz57/S+RIREZH+0ClgVbemTZuiZ8+e6NixI5o2bYrGjRtDJpMhKSkJZ86cQWBgIDIyMvDJJ5/g4MGDaN++fZky9u/fj82bNwMAmjdvjtmzZ6Nt27ZISUnBnj17cPXqVZw/fx5ffvklfvrpp3Lr8uWXXwrhqmfPnpg2bRpsbGzw4MEDbN26FU+fPsWmTZvQuHFjTJgwoQp/K0RERFTdXnvAys7OBgDUrVtXp3J69uyJ8+fPl7v+vffeQ1BQEObNm4eioiJs3rxZCFIKmZmZ+PHHHwEAzZo1w4EDB2BtbS2s79+/P+bNm4dz587h+PHjGD9+PHr27FnmWJcvX8aJEyeEfX7++WcYGhoCAFxdXeHh4YGxY8fi2bNn+PHHHzF06FBYWlrqdP5ERERUc732Qe7Hjh0DANjZ2elUjiLAaDJw4EA4OjoCAK5fv15mvZ+fnxD4Pv/8c5VwpTjG119/LRxrx44dao+zc+dOAICRkZHK9grW1tb4/PPPAQAvXryAn5+flmdJRERE+qhSLVjTpk1T+/qyZctgamqqcd/CwkLEx8cjLS0NEokEffv2rVxNX1G9evUAAC9fviyzLjg4GABgbm6OQYMGqd2/SZMm6N27Ny5duoTLly8jJycH5ubmwvqcnBxcvnwZANC7d280adJEbTmDBg2Cubk5cnJyEBQUhI8++kiU8yMiIqKap1IBKzQ0FBKJBHK5XHhNLpcjIiKiUgd1cHDA7NmzK7XPq4iJicG9e/cAAE5OTirrCgsLcfv2bQCAm5sbTExMyi2nR48euHTpEgoLCxEZGYlevXoJ6yIiIoRB/j169Ci3DBMTE7i5ueHSpUvCPsbGxjqfIxEREdU8lQpY3bt3V/n52rVrkEgk6Nixo8YWLIlEgjp16qBx48bo3Lkzhg8fDjMzs1evtQb5+flITk7GuXPnsH37dhQXFwMApk+frrJdbGwspFIpoCZ8laa8Pjo6WiVgRUdHq91OHUdHR1y6dAnFxcV48uQJWrduXcmzIyIiIn1QqYC1d+9elZ8Vd+WtX7++WsPC4cOHsWzZsnLXz5o1CyNGjFB5LSkpSVgur1tP3Xrl/Ur/bGtrq7Gcpk2bCsuJiYkMWERERLWUTncRjh49GhKJBPXr1xevRiLq0KEDVq9eDVdX1zLrcnNzheWKWtOU1+fl5b1yOcqtfKXL0Zai1Y1qD21u2KCqo6/XFN839Caqquu1KsrVKWCtX79evJroYODAgejUqRNQMuFnXFwcTp06hcDAQHz22Wf48ssv0b9/f5V9lAe9VzQWSnl8VkFBQbnlaBrHVVE52qrseDeq2UxNTeHs7Fzd1Xij3b9/H/n5+dVdjUrh+4beVPp0vb6WebCePn2KjIwM2NnZoVGjRqKXX79+fZVWNFdXVwwfPhxHjhzBF198gblz5+Lbb7/FmDFjhG2UZ2WvaCb6wsJCYbn0/F3K5ShvV9lytOXi4sJvrkQiateuXXVXgYi0VFXXq1QqFb0BQ6eAlZaWhtOnTwMARo4cCQsLC5X1T548waJFi3D37l2gZLD7gAEDsHbt2tcy0ebo0aNx/vx5nDp1CmvWrIGHhwesrKwApekboEV3nfL60t2AlSlHOXW/6iB/Q0NDBiwiEfF6ItIf+nS96jTR6F9//YU1a9Zgz549ZcJVYWEhPv74Y9y9exdyuRxyuRwymQxBQUGYO3eurvXW2oABA4CS8HPx4kXhdU0D10vTNCBe+efk5GSN5SQmJgrLygPeiYiIqHbRKWD9/fffkEgkaifpPHz4MJ4+fQoA8PDwwPLly9G/f3/I5XKEhYXh5MmTuhxaa8qzsz979kxYbtmypZCEY2JiNJahvL5Vq1Yq65R/rqicx48fAyUzvrdo0ULrcyAiIiL9olPAUgQGNze3MuuOHz8OAOjVqxe2bNmCqVOn4pdffkGfPn0gl8uFZ/dVNeVWJeVuORMTE+HuwvDwcI3jp0JDQ4V9FIPpFVxcXIRB8ort1CksLER4eHiZfYiIiKj20SlgpaenA2rmfyooKEB4eDgkEgnGjx+vsm7s2LEAgKioKF0OrTXFGDEAaNu2rco6RfdhTk4OAgMD1e6flJSk8igc5cfkoOQxO7179wZKHvpcXndjYGAgcnJygJK7HomIiKj20ilgKR6UbGCgWkx4eDiKi4shkUjQp08flXX29vZAyQB5XRw+fFjt8wWV7dq1CyEhIcJxu3XrprLey8tLGDv2008/ISMjQ2W9VCrF119/LcyPMXPmTLXHmTFjBgCguLgY33zzTZn5NNLT0/Hjjz8CJXc8enl5VfJsiYiISJ/odBehmZkZsrOzkZqaqvK6oqusVatWZe4WNDL6v0PqeifA5s2b8f3332Pw4MHo2rUrHBwcUK9ePeTk5ODBgwcICAhAWFgYUDLP1Zo1a8oc08rKCp9//jlWrVqFhIQEjB8/HnPmzEHbtm2RkpKC3bt34+rVqwCA999/Hz179lRbl969e2P48OE4ceIEzp49iw8//BDTp0+HjY0NHjx4gF9//VUY//X555+/ljsoiYiIqProFLCcnJxw69YtXLx4Ee7u7sLrf/31FyQSidqHHyvCmBjzYWVmZsLX1xe+vr7lbtOkSROsW7euTEuawsSJE5GSkoItW7bg6dOn+PLLL8ts4+7ujnXr1mmsy7p165CTk4OQkBBcvXpVCGYKBgYGmDt3LiZMmKD1+b1OMrkMBhKdGjSJiIiohE4By93dHeHh4Thw4ACcnJzQrVs3+Pv749GjR+XeXXjnzh1Ai+f2VWT79u0ICQlBWFgYnjx5grS0NGRmZqJOnTpo2LAhOnTogHfffRfDhg3T+CBqAPj000/x9ttvw8fHBzdu3EBqairq16+P9u3bY8yYMXj//fcrrE/dunWxbds2BAQEwN/fH/fu3cOLFy/QqFEjdO3aFVOmTEHnzp11OueqZCAxgM+ti0jOyaruqrxR2jduhvfadqnuahARkch0ClhTpkzBn3/+iefPn2PNmjUq69zc3NCrV68y+5w7dw4SiQQuLi66HBpOTk5wcnLChx9+qFM5Cl26dEGXLrr/oRsxYkSZB0vri+ScLCS8SK/uarxRbOrVzOd4EhGRbnTqE7KwsMCuXbvg7OwsTCYql8vRrVs3/Pe//y2z/b1794Sp6MvrsiMiIiLSdzo/i7BVq1Y4fPgw4uLikJqaisaNGwt3Cqrz3XffASXzYxERERHVRqI97NnBwQEODg4at2nfvj3at28v1iGJiIiIaiTeNkZEREQkMtFasGQyGa5evYqbN28iNTUV+fn5WLRoEWxsbIRtCgsLIZVKYWhoCBMTE7EOTURERFSjiBKwzp07h7Vr16o8TBklM58rByw/Pz+sXbsWZmZmuHjxosqzAYmIiIhqC527CH19fTF37lwkJCRALpfDysoKcrlc7baKR9Pk5eWV++w/IiIiIn2nU8CKjY3F6tWrgZK7Ak+cOCE8GFkdExMTDB48GHK5HH///bcuhyYiIiKqsXQKWLt27UJxcTFat26Nbdu2oVWrVhXuo3jg8t27d3U5NBEREVGNpVPAunLlCiQSCaZPn671oPXmzZsDABITE3U5NBEREVGNpVPASk5OBkrmt9KWYmB7QUGBLocmIiIiqrFEmQerMmEpIyMDAGBubi7GoYmIiIhqHJ0Clq2tLQAgLi5O631u3LgBlMz8TkRERFQb6RSwevToAblcDn9/f622z87Oxv79+yGRSPgsQiIiIqq1tA5Y7du3h7OzMx49eiS8NnHiREgkEly7dg2HDx/WuH9GRgbmzp2L1NRUGBoaYuLEibrVnIiIiKiGqtRM7qUnEHV2dsa0adOwe/duLF++HBcuXMDgwYOF9Tdv3sTdu3cRFhaG48ePIycnBxKJBHPnzoWdnZ14Z0FERERUg+j8qJwvvvgChYWF2LdvH86cOYMzZ85AIpEAAFauXClspwhn06dPx9y5c3U9LBEREVGNpfNdhBKJBKtWrcKOHTvQo0cPSCQSyOVylX8A4Obmhq1bt2LZsmVi1JuIiIioxhLlYc8A0LdvX/Tt2xc5OTm4e/cu0tLSIJPJYGVlhfbt28Pa2lqsQxERERHVaKIFLAVzc3N0795d7GKJiIiI9IYoE40SERER0f+v0i1Yy5Ytg6mpqc4Hlkgk2L17t87lEBEREdU0lQ5YkZGROh9ULpcLdxoSERER1TaVDlil58IiIiIiIlWVDljHjx9H69atq6Y2RERERLUAB7kTERERiYwBi4iIiEhkDFhEREREImPAIiIiIhIZAxYRERGRyBiwiIiIiESm9TQNwcHBAABbW9uqrA8RERGR3tM6YNnZ2VVtTYiIiIhqCXYREhEREYmMAYuIiIhIZAxYRERERCJjwCIiIiISGQMWERERkcgYsIiIiIhExoBFREREJDIGLCIiIiKRMWARERERiYwBi4iIiEhkDFhEREREImPAIiIiIhIZAxYRERGRyBiwiIiIiETGgEVEREQkMgYsIiIiIpExYBERERGJjAGLiIiISGQMWEREREQiY8AiIiIiEhkDFhEREZHIGLCIiIiIRMaARURERCQyBiwiIiIikTFgEREREYmMAYuIiIhIZAxYRERERCJjwCIiIiISGQMWERERkcgYsIiIiIhExoBFREREJDIGLCIiIiKRMWARERERiYwBi4iIiEhkDFhEREREImPAIiIiIhIZAxYRERGRyIyquwK6iIiIQEhICMLCwvDo0SOkp6fD2NgYNjY26NKlC8aOHYtu3bppXV5ISAh8fX0RERGB9PR0WFtbw8XFBePHj4e7u7tWZRQXF8PPzw8BAQGIiYlBXl4ebGxs0KdPH0ydOhVt2rTR4YyJiIhIH+htwJo8eTKuX79e5vWioiLExsYiNjYWhw8fxujRo7FmzRqYmJiUW5ZMJsOKFStw8OBBldeTk5ORnJyMoKAgeHl5YfXq1TAwKL/RLz09HbNmzUJERITK63FxcThw4AD8/f2xcuVKeHl5vdI5ExERkX7Q24CVkpICALCxscHQoUPRrVs3NG3aFDKZDOHh4di5cyeSk5Nx5MgRFBcX46effiq3rI0bNwrhytnZGR999BEcHBwQFxeH7du3IyoqCn5+frC2tsbixYvVliGVSjF//nwhXA0ePBheXl6wsrLCrVu38MsvvyAtLQ0rV66EjY2N1i1iREREpH/0NmA5OTlh0aJFGDJkCAwNDVXWubm5YeTIkZg0aRJiY2Nx/PhxTJw4Ed27dy9TzuPHj7Fz504AQKdOneDj44O6desCAFxdXeHh4YEpU6YgMjISO3bswNixY9GiRYsy5fj7++PGjRsAAG9vb6xatUpY5+rqin79+mHMmDHIycnBt99+i759+8LISG9//URERKSB3g5y37p1K957770y4UrB2toaX3zxhfDzmTNn1G63e/duFBcXAwBWrFghhCsFU1NTrFixAigZX7Vr1y615ShCmpWVFZYuXVpmfYsWLTB79mwAwJMnTxAYGKjlmRIREZG+0duApY2ePXsKy0+fPi2zXi6XIzg4GChpEXNzc1NbjpubGxwdHQEAwcHBkMvlKusfP36M6OhoAMDQoUNhamqqthxPT09hOSgo6JXOiYiIiGq+Wh2wCgsLhWV1g9Pj4+OFsVzqug+V9ejRAygZ+B4fH6+yTtE1qLydOo0bN0bLli0BAGFhYVqfBxEREemXWh2wrl27Jiy3atWqzPpHjx4Jy05OThrLUl4fExOjsk7RelWZchITE5GXl6dxWyIiItJPtTZgyWQybNu2Tfh52LBhZbZJSkoSlps0aaKxPOX1iYmJ5ZZja2ursZymTZsCJd2TyvsRERFR7VFrb2PbtWsXbt++DZRMmdCpU6cy2+Tm5grLZmZmGstTHldVuuVJuZx69eq9cjnakEqlld5HG+XdLEBU21XVNVXVeM3Sm6iqrteqKLdWBqzQ0FBh3quGDRvi66+/Vrvdy5cvhWVjY2ONZSpPVFpQUFAl5Wij9CSmYjA1NYWzs7Po5RLpg/v37yM/P7+6q1EpvGbpTaVP12utC1gPHz7E/PnzUVxcjDp16uB///sfGjZsqHbbOnXqCMtFRUUay1UeMF96KofS5Sj/XJlytOHi4sJvrkQiateuXXVXgYi0VFXXq1QqFb0Bo1YFrLi4OMyYMQNZWVkwNDTEf/7zH413Byp351XUXaecmEt3JyqXk5ubqzFgaSpHG4aGhgxYRCLi9USkP/Tpeq01g9yTk5Px4YcfIiUlBRKJBOvWrcPAgQM17qM8cL2iAefK6xUD1dWVk5ycrLEcxQB5iURS4cB6IiIi0k+1ImClp6djxowZiIuLA0pmZB89enSF+7Vu3VpYLj31QmnK60tPxaA8BYS25TRt2vSVWrCIiIio5tP7gJWdnY2PPvpImNPqs88+w+TJk7Xa197eHjY2NkCpObPUUay3tbWFvb29yrquXbsKy6GhoeWW8fz5c8TGxgIAunTpolUdiYiISP/odcDKz8/HrFmzcOfOHQDAnDlzMGvWLK33l0gkGDBgAFDSshQeHq52u/DwcKHlacCAAZBIJCrrHR0dhVas06dPl3uHg7+/v7BcUfclERER6S+9DViFhYWYP3++8MiZadOmYdGiRZUuZ/r06cKguTVr1pSZOqGgoABr1qwBABgZGWH69Olqy5kxYwYAIDMzExs2bCiz/unTp9i6dStQ8uDnQYMGVbquREREpB/09i7Czz77DJcuXQIA9OrVC+PGjcODBw/K3d7Y2Fh4YLMyR0dHzJw5E9u2bUNkZCQmTZqEjz/+GA4ODoiLi8Nvv/2GqKgoAMDMmTOFZwmW5unpiUOHDiEsLAw+Pj5ITU2Fl5cXLC0tcfv2bWzZsgU5OTkwMDDA8uXLYWSkt796IiIiqoDe/pX/66+/hOUrV65g5MiRGre3s7PD2bNn1a5btGgR0tLScOjQIURFRaltCRs3bhwWLlxYbvmGhob4+eefMWvWLERERODMmTM4c+aMyjYmJiZYuXIl3N3dtThDIiIi0ld6G7DEZGBggHXr1mHIkCE4cOAAIiIikJGRgQYNGsDFxQUTJkzQKhRZW1tj//798PX1xfHjxxEdHY38/HzY2Nigd+/emDZtGtq0afNazomIiIiqj94GrPv374tepru7u86tS0ZGRvD29oa3t7do9SIiIiL9oreD3ImIiIhqKgYsIiIiIpExYBERERGJjAGLiIiISGQMWEREREQiY8AiIiIiEhkDFhEREZHIGLCIiIiIRMaARURERCQyBiwiIiIikTFgEREREYmMAYuIiIhIZAxYRERERCJjwCIiIiISGQMWERERkcgYsIiIiIhExoBFREREJDIGLCIiIiKRMWARERERiYwBi4iIiEhkDFhEREREImPAIiIiIhIZAxYRERGRyBiwiIiIiETGgEVEREQkMgYsIiIiIpExYBERERGJjAGLiIiISGQMWEREREQiY8AiIiIiEhkDFhEREZHIGLCIiIiIRMaARURERCQyBiwiIiIikTFgEREREYmMAYuIiIhIZAxYRERERCJjwCIiIiISGQMWERERkcgYsIiIiIhExoBFREREJDIGLCIiIiKRMWARERERiYwBi4iIiEhkDFhEREREImPAIiIiIhIZAxYRERGRyBiwiIiIiETGgEVEREQkMgYsIiIiIpExYBERERGJjAGLiIiISGQMWEREREQiY8AiIiIiEhkDFhEREZHIGLCIiIiIRMaARURERCQyBiwiIiIikTFgEREREYmMAYuIiIhIZAxYRERERCJjwCIiIiISGQMWERERkcgYsIiIiIhExoBFREREJDIGLCIiIiKRMWARERERiYwBi4iIiEhkDFhEREREImPAIiIiIhKZUXVXQBdpaWm4ffs2bt++jYiICERERCAzMxMA4OnpifXr11eqvJCQEPj6+iIiIgLp6emwtraGi4sLxo8fD3d3d63KKC4uhp+fHwICAhATE4O8vDzY2NigT58+mDp1Ktq0afNK50pERET6Q68DVp8+fUQpRyaTYcWKFTh48KDK68nJyUhOTkZQUBC8vLywevVqGBiU3+iXnp6OWbNmISIiQuX1uLg4HDhwAP7+/li5ciW8vLxEqTcRERHVTHodsJQ1a9YMTk5OuHTpUqX33bhxoxCunJ2d8dFHH8HBwQFxcXHYvn07oqKi4OfnB2trayxevFhtGVKpFPPnzxfC1eDBg+Hl5QUrKyvcunULv/zyC9LS0rBy5UrY2Nho3SJGRERE+kevA9a8efPg4uICFxcXNGrUCPHx8RgwYEClynj8+DF27twJAOjUqRN8fHxQt25dAICrqys8PDwwZcoUREZGYseOHRg7dixatGhRphx/f3/cuHEDAODt7Y1Vq1YJ61xdXdGvXz+MGTMGOTk5+Pbbb9G3b18YGen1r5+IiIjKodeD3D/99FP0798fjRo1euUydu/ejeLiYgDAihUrhHClYGpqihUrVgAl46t27dqlthxFSLOyssLSpUvLrG/RogVmz54NAHjy5AkCAwNfuc5ERERUs+l1wNKVXC5HcHAwAMDJyQlubm5qt3Nzc4OjoyMAIDg4GHK5XGX948ePER0dDQAYOnQoTE1N1Zbj6ekpLAcFBYl2HkRERFSzvNEBKz4+HikpKQCA7t27a9y2R48eQMnA9/j4eJV1iq5B5e3Uady4MVq2bAkACAsL06nuREREVHO90QHr0aNHwrKTk5PGbZXXx8TEqKxTtF5VppzExETk5eVVus5ERERU873RASspKUlYbtKkicZtldcnJiaWW46tra3Gcpo2bQqUdE8q70dERES1xxt9G1tubq6wbGZmpnFb5XFVpVuelMupV6/eK5ejDalUWul9tGFoaFgl5RLVdFV1TVU1XrP0Jqqq67Uqyn2jA9bLly+FZWNjY43bmpiYCMsFBQVVUo42Sk9iKgZTU1M4OzuLXi6RPrh//z7y8/OruxqVwmuW3lT6dL2+0QGrTp06wnJRUZHGbQsLC4Xl0lM5lC5H+efKlKMNFxcXfnMlElG7du2quwpEpKWqul6lUqnoDRhvdMBS7s6rqLtOOTGX7k5ULic3N1djwNJUjjYMDQ0ZsIhExOuJSH/o0/X6Rg9yVx64XtGAc+X1ioHq6spJTk7WWI5igLxEIqlwYD0RERHppzc6YLVu3VpYLj31QmnK60tPxdCqVatKl9O0adNXasEiIiKimu+NDlj29vawsbEBAFy7dk3jtor1tra2sLe3V1nXtWtXYTk0NLTcMp4/f47Y2FgAQJcuXXSqOxEREdVcb3TAkkgkwsOhY2JiEB4erna78PBwoeVpwIABkEgkKusdHR2FVqzTp0+Xe4eDv7+/sDxw4EDRzoOIiIhqljc6YAHA9OnThUFza9asKTN1QkFBAdasWQMAMDIywvTp09WWM2PGDABAZmYmNmzYUGb906dPsXXrVqDkwc+DBg0S/VyIiIioZtDruwivX7+Op0+fCj9nZGQIy0+ePMHhw4dVth8zZkyZMhwdHTFz5kxs27YNkZGRmDRpEj7++GM4ODggLi4Ov/32G6KiogAAM2fOFJ4lWJqnpycOHTqEsLAw+Pj4IDU1FV5eXrC0tMTt27exZcsW5OTkwMDAAMuXL4eRkV7/6omIiEgDvf4rf/DgQZVuN2VhYWFlHqisLmABwKJFi5CWloZDhw4hKioKixYtKrPNuHHjsHDhwnLrYmhoiJ9//hmzZs1CREQEzpw5gzNnzqhsY2JigpUrV8Ld3V3LMyQiIiJ9pNcBSywGBgZYt24dhgwZggMHDiAiIgIZGRlo0KABXFxcMGHCBK1CkbW1Nfbv3w9fX18cP34c0dHRyM/Ph42NDXr37o1p06ahTZs2r+WciIiIqProdcBav3491q9fL1p57u7uOrcuGRkZwdvbG97e3qLVi4iIiPTLGz/InYiIiEhsDFhEREREImPAIiIiIhIZAxYRERGRyBiwiIiIiETGgEVEREQkMgYsIiIiIpExYBERERGJjAGLiIiISGQMWEREREQiY8AiIiIiEhkDFhEREZHIGLCIiIiIRMaARURERCQyBiwiIiIikTFgEREREYmMAYuIiIhIZAxYRERERCJjwCIiIiISGQMWERERkcgYsIiIiIhExoBFREREJDIGLCIiIiKRMWARERERiYwBi4iIiEhkDFhEREREImPAIiIiIhIZAxYRERGRyBiwiIiIiETGgEVEREQkMgYsIiIiIpExYBERERGJjAGLiIiISGQMWEREREQiY8AiIiIiEhkDFhEREZHIGLCIiIiIRMaARURERCQyBiwiIiIikTFgEREREYmMAYuIiIhIZAxYRERERCJjwCIiIiISGQMWERERkcgYsIiIiIhExoBFREREJDIGLCIiIiKRMWARERERiYwBi4iIiEhkDFhEREREImPAIiIiIhIZAxYRERGRyBiwiIiIiETGgEVEREQkMgYsIiIiIpExYBERERGJjAGLiIiISGQMWEREREQiY8AiIiIiEhkDFhEREZHIGLCIiIiIRMaARURERCQyBiwiIiIikTFgEREREYmMAYuIiIhIZAxYRERERCJjwCIiIiISGQMWERERkcgYsIiIiIhExoBFREREJDIGLCIiIiKRGVV3BWqjhIQE7N27F+fPn0dSUhJMTEzg4OCAYcOGYfLkyTA1Na3uKhIREVEVYsAS2dmzZ7FkyRLk5OQIr+Xn5yMrKwuRkZHw8/PDtm3b0KJFi2qtJxEREVUddhGKKCoqCosWLUJOTg7MzMywaNEi7N+/H7t27cL48eMBALGxsZg1a5ZKACMiIqLahS1YIvr2229RUFAAIyMj7Ny5E507dxbW9e7dGy1atMCGDRsQGxuL33//HQsWLKjW+hIREVHVYAuWSG7fvo3r168DAMaOHasSrhRmzJiBVq1aAQD27NmDoqKi115PIiIiqnoMWCIJCgoSlseOHat2GwMDA4wePRoA8OLFC1y9evW11Y+IiIheHwYskdy4cQMAYGZmho4dO5a7Xffu3YXlsLCw11I3IiIier0YsEQSHR0NAGjevDmMjMof2ubk5FRmHyIiIqpdGLBE8PLlS2RkZAAAmjRponFbS0tLmJmZAQCSkpJeS/2IiIjo9eJdhCLIzc0VlhXhSRNTU1Pk5eUhLy9P62PI5XIAQGFhIQwNDV+xpuUzNDRE03qWMIRE9LKpfA1N60EqlULSoAkMJOL/v1L5JFaNIZVKIZVKq7sqr4TXbPXgNVs9qvp6VZSr+FsrBgYsEbx8+VJYNjY2rnB7ExMTAEBBQYHWx5DJZEDJXFtVpS3M0LZuxQGRRJQDhIeHAw1a/98/er3Cw6u7BjrhNVsNeM1Wn9dwvSr+1oqBAUsEderUEZa1mXqhsLAQAFC3bl2tj2FkZAQXFxcYGBhAIuE3ViIiIrHI5XLIZDKNY6griwFLBPXq1ROWten2y8/PB7TsTlQwMDAQWr6IiIioZuMgdxHUqVMHVlZWgBYD17OysoQQVtGAeCIiItJPDFgiad36//rinz59iuLi4nK3i4mJEZYVs7oTERFR7cKAJZKuXbsCJV2Ed+7cKXe7a9euCctdunR5LXUjIiKi14sBSyQDBw4Ulg8dOqR2G5lMhiNHjgAA6tevj549e762+hEREdHrw4AlEldXV3Tr1g0oCVg3b94ss83OnTuF2dunTZum1ZQOREREpH8kcjFn1XrDRUVFYdKkSSgoKICZmRnmzJmDnj17oqCgACdPnsSBAwcAAC1btsShQ4dgbm5e3VUmIiKiKsCAJbKzZ89iyZIlyMnJUbu+ZcuW2LZtG1q0aPHa6yaGdu3aAQDmz5+PBQsWVEsdPDw8kJCQAE9PT6xfv15lXXx8PAYMGAAA+O677zBmzJhqqSMREb3ZOA+WyDw8PHDs2DHs2bMH58+fR3JyMoyNjdG8eXMMHToUU6ZMgampaZXX4+rVq5g2bVqZ1w0NDWFubg5zc3M0bdoUHTt2RNeuXdG/f/83Yp6t8n4vCmZmZrCxsYGrqyvGjBmD3r17l7utcpjT1oABA7BlyxaV1zZt2oTNmzeX2VYikcDMzAyNGzeGi4sLRo0ahXfeeUen46tz//59rbctr64omautXr16cHBwQI8ePTBhwgSVh5uT/uB1UhavE6osBqwqYGdnh2XLlmHZsmXVXZUypFIpsrKykJWVhYSEBFy/fh27d++GtbU1pk6dilmzZok6k622pk6ditDQUPTo0QN79+597cdXyMvLQ2xsLGJjY3Hs2DGMHj0a69atq5LnP1ZELpcjNzcXubm5iI2NRUBAAAYPHoyffvqpRoZhmUyG7OxsREVFISoqCj4+Pli2bBkmT55c3VUjkfE6eXW8Tt4cDFhvgEmTJsHb21v4OS8vD1lZWbh//z6uXLmCf/75B+np6fjf//6Hc+fOYevWrbC2tlZbVmW+xVWVs2fPilJO6d+LXC5HVlYWwsPDsWvXLqSlpeHIkSNo0qQJFi1apLGsAQMGYOHChRUes6Jxd+vWrYOLiwtQ8kGclJSEmzdvYteuXSgoKMBff/2F7777DqtWrYKtrS0CAgLKLWvEiBEAgE6dOuG7776rsG6VpVxXRX1TUlJw4cIF7N+/H0VFRVi9ejUcHR3Rp08f0Y9PrwevE93wOnlzMWC9ARo2bIi2bduWed3d3R2zZs3Co0ePsGTJEkRFReH27duYN28edu/eXSO//YmpvN9Ljx494OHhgTFjxuDly5fYu3cv5s2bp/H3Ub9+fbVlVZa9vb1KOe3bt8e7776LIUOGwMvLC8XFxfD19cXcuXPRuHFjrY5pZmYmSt0qqquivv369UOHDh3w5ZdfAgB27NjBPxx6jNeJuHUFr5M3BqdpILRu3Rr79u2Ds7MzACAsLAx//vlndVerWrVu3RrvvvsuACA3N1dlBv7q4OzsjPfeew8AUFxcjNDQ0GqtT0XGjh2LBg0aAAAiIiKquzpURXid6IbXSe3GFiwCANStWxc//PADRowYAblcjh07dmDy5Mll5uqq6C7CFy9ewMfHB+fPn0dMTAzy8vJgYWEBa2trODo6om/fvhg8eDAaNWoEAPjiiy/g7+8v7B8aGiocQ8HOzk6lW1DTXYRisrOzE5YLCwur7DjaUv4WnJiYWK110YadnR0yMjI0/u7Cw8Nx7tw5hIWFISYmBllZWTAxMUGTJk3QvXt3TJ06VXgMVXkeP36MP/74A1evXkVCQgKKiopgZWWFhg0bwtnZGe+88w4GDhxYbsvK8+fP8ccff+DixYuIj49HXl4eGjZsCDc3N0yYMIGtChXgdaIbXie1FwMWCdq0aYO+ffvi0qVLSElJQURERKUe5xMdHY0PPvgAKSkpKq9nZGQgIyMD0dHRCAoKgkwmw5QpU6rgDMT17NkzYblZs2bVWhcAKmG3Om5EqCzF769p06Zq1x8+fFjtjSBFRUWIjo5GdHQ0/Pz8sHz58nIHAJ86dQpLlixBUVGRyuvPnz/H8+fPce/ePRw+fBgBAQFqu3+OHTuGVatWCQ9gV0hKSsLp06dx+vRpjBs3Dt98841e/M6rA68T3fA6qb34myAVvXv3xqVLlwAA169fr1TAWrJkCVJSUmBsbAwvLy/069cPjRo1glwuR1JSEsLDwxEUFKSyz6JFizBjxgwsW7YMkZGRageaVseM99HR0Th//jwAwM3NTWhxq07K3S/29vbVWpeK+Pv7Iz09HSgZ2KyOVCqFpaUlBgwYgG7duqFFixYwMzNDSkoK7ty5g7179yIjIwNr1qyBk5NTmakAUlNT8eWXX6KoqAgNGzbE5MmT4ebmhgYNGqCgoABPnz5FaGgogoOD1R7/5MmTWLp0KeRyORwcHDBlyhS0atUK1tbWSEhIwMGDBxESEoKDBw/C3Ny8Rt4VXN14neiG10ntxoBFKjp27Cgsx8bGar1fXFyc8JDrL774okwLlaurKwYPHowlS5bgxYsXwuu2trawtbWFmZkZUIUDTdVJS0vDgwcPhJ/lcjmys7Nx8+ZN7N69GwUFBbCwsNDqA+PFixcqZZXH3t5eONfKSExMFO6Eql+/vsZ5h16X+Ph4YfwISu6OSk1NxYULF4QxfG3atMGMGTPU7t+vXz+8//77ZeaFc3Z2xrvvvotp06Zh8uTJuH//PjZt2lTmnM+fPy98o961a1eZ902XLl0wevRoFBQUlDl2eno6Vq5cCblcjrFjx2L16tUq37w7duyIwYMHY+PGjfj111+xZ8+eN3a+Il4nuuF18uZiwCIVVlZWwrJyEKrI8+fPhWXFMxnVkUgksLS01KGG4tm3bx/27dundp2BgQEmTpyIDz74AI6OjhWWFRwcXO43QGV79uzR+iHfitvPb9y4gf/85z/Ch+S//vUv1KtXT6syqpLi7id1LCwsMH/+fHh7e6N+/fpqt7G1tdVYvoWFBT799FPMmzcPN27cQEZGhsofqtTUVACApaWlxlBet27dMq/t27cP2dnZsLW1xddff11ut8aCBQvg7++P5ORkHD16tMJpCGojXie64XXy5mLAIhXK3xpzc3O13q9x48bCsr+/v943E8tkMpw8eRJ16tTB559//tqmrPj/2rv3oKir9w/gb5aAQBCUi1Y2UooogqIBijcCERS5iZhahozpmFlmpRnNqH3HccoYUwflYpdZNBVESAYpEQxBuSmOBSgKVOoSXhBYBES5yO8f+Px2gV1AFrDl/fpr2T2fswf0wLOf85znKKuebWZmhg0bNmDJkiX9MpbeqKmpQUxMDIyMjLBs2bJuXfPo0SNUVlaivr4ebSd4yS4PX79+Xe7Tedv/uerqaqSkpMDV1bXb42vbNPHmm28q/bd94YUXYGtri6SkpE4PcB/sOE96h/NEvTHAIjmyQVVPDqN+9dVXYWdnh9zcXIjFYly4cAFubm5wcHCAra1tvxwP1FOd7YR8/Pgxbt26hfj4eERGRiIyMhIFBQX48ccflX4Pfb2jEa1LBd7e3n36Hj3R/i5DS0sLamtrcf36dURGRiI5ORnbt2/HP//8ozDgrqyshFgsRlJSEm7dugVlR6NWVVXJfe3i4oKhQ4fi4cOH+PDDD4W6THZ2dpgwYYLCquLNzc24fv06ACA6Olo4hL0rbXcCBhvOk97hPBm8GGCRHNnJ2dOlvO+++w4ff/wxrly5gpKSEpSUlCA0NBRaWlqYPHkyPD094efnBx0dnT4YuWq8+OKLsLS0xOeffw5zc3Ns3boVly9fRnh4eL/c9pat+tyWgBodHY2LFy/ixIkTePDgAcLDw6GhodHnY+kpDQ0NGBgYwN7eHvb29vjss89w6tQpiMViODk5ddjGXVBQgPfeew9SqbRb/T958kTu62HDhiEsLAyffvop7t27h5ycHOTk5ACtHw4cHR2xePFiODs7y11XXV2NpqamHn9/neWoDFacJ8+O82TwYIBFcq5duyY87k5OhawRI0YgKioKWVlZOHPmDC5duoSSkhI0NjYiNzcXubm5+Omnn3Dw4MEe9z0Q/P39sXv3bkilUsTGxvbLH472VZ8nTZqEhQsX4ssvv0RcXBzOnTuHyMhIBAYG9vlYemv16tU4deoUACA2NlbuD0dDQwM2btwIqVQKLS0trFixAnPnzoW5uTkMDQ2F5QiJRCIsaXT2qd3Ozg7JyclISkpCWloacnNzcffuXdTW1iI5ORnJycmYNWsW9u/fL9xZaW5uFq5fsmSJ0uUmWQOxm/W/gPOkdzhP1BcDLJKTmZkpPH7jjTeeqQ9HR0chB6CqqgpZWVmIjo5GdnY2bt++jU8++QQnT55U2Zj7ikgkwujRoyGVSlFeXt4hebS/aGhoYNu2bcjOzkZZWRn2798PX19fuQ0JzyPZnUTtd45lZ2dDIpEAALZv364wX6Y7n9p1dHTg7e0tLAtJJBKkpaXh8OHDuHnzJi5cuIA9e/YIycayd2ZbWlr6bdequuI86R3OE/XFo3JIUFRUhKysLKC16J21tXWv+xw2bBg8PDwQGRkJFxcXAEBhYWGPSkAMJNlb5LKf6Pqbrq4uPvjgA6A1MfaHH34YsLF0l+zPrv1SQ0lJifB4wYIFCvsoKCjo8fu21eqJjY3FyJEjgdZCi220tbVhYWEBtB4LRb3HefLsOE/UFwMsAlrXzbds2SLcXl61apXKK/LK7mxpn4jZlpf1PBy10aa+vh5//fUX0JpzMhCfymX5+voKlbKPHj3a7ZyMgSL7S799lWrZPyT19fWdXv/06VPExMQ88/vr6+sLeTqdJf6itSjl+fPnn/k9iPOktzhP1BcDLEJJSQnefvttIf/KwcEBy5cv71EfhYWFKCwsVPh6S0uLsPyooaEhd34ZZLYSSyQSpTtk+lNISIiQsDlr1iyFu236i5aWFlavXg207vY8dOjQgI5HmYaGBuzbt0/42snJSe51c3Nz4bHsWZSydu/eLRSv7cz58+c7HMskq6amBnl5eUAnFb0DAgKEkiRBQUEoLi5W+v2cO3dO2FFF8jhPnh3niXpjDtYg0L4Sc319Paqrq3Hjxg1kZ2cjIyNDCGpsbW2xb9++HicqFhYWIigoCDY2NnB2dsbEiRNhYmKCpqYmlJaWIi4uDhkZGUDrpyIzMzO566dOnYq4uDhUVFTg66+/hre3NwwMDIDWGivtAzJVaP9zQesOnFu3buHkyZPCJzYdHR1s2LBBaV/drVCtqamJMWPGPPOY/f39ERYWJhy8umrVqh6V01Cl9hWqAaC2thaFhYU4duyY8Mt49OjRWLp0qVy7WbNmwdjYGBUVFdi7dy9KS0sxb948DBs2DLdv38bx48eRlZWFqVOnKlyeSExMxLp16zBjxgzMnDkT48aNg6GhIerq6lBUVIQjR47g3r17ANChxpCJiQl27dqFDRs2oLy8HIsXL8aiRYswZ84cjBw5Ek1NTbh79y7y8vKQlJQEiUSC8PBwjB8/XsU/xecf50nvcJ4MXgywBgFllZjbDB8+HCtXrsTq1at7tTSYn5+P/Px8ha9PmTIFO3fu7PC8h4cHIiIiIJFIhLo6bV555RWh4J0qdffnEhwcDEtLS6Xtuluh2sDAALm5uT0eaxsdHR0EBgYiODgY1dXVOHLkCNauXfvM/fWGsgrVbcaPH48DBw50qBKtp6eHXbt2Yf369Xjy5EmndXYcHBywbds2eHp6Kuy/sbERaWlpSEtLU9hm2bJlne6AcnNzQ2hoKIKCgiCVShEVFYWoqKhO+xCJRM9lLbf+wHnSO5wngxcDrEFGJBJhyJAhMDAwwMsvv4yJEyfCzs6uy0q9XfH09ISxsTEyMzORn5+Pe/fuoaKiAk1NTTA2NoaVlRU8PDywcOFCiEQdV6aHDBmCqKgoREREICMjA2VlZQpzDvqSlpYWjIyMMHbsWDg5OcHPz++5OdqnzfLly/H9999DKpVCLBYjICDgufmlpquri+HDh2PixIlwd3fH/PnzFQbss2fPRmxsLA4ePIjs7GxUVVXBwMAAY8eOhZeXF/z9/VFWVqbwvYKCgjBjxgxkZ2fjxo0bKC8vR2VlJTQ1NTFy5EhMmTIF/v7+So9ucnFxwdmzZ3H8+HGkpaWhpKQE1dXV0NTUhImJCSwsLDB9+nS4u7t3yI8ZzDhPeofzZHDQaHleEl6IiIiI1AST3ImIiIhUjAEWERERkYoxwCIiIiJSMQZYRERERCrGAIuIiIhIxRhgEREREakYAywiIiIiFWOARURERKRiDLCIiIiIVIwBFhEREZGKMcAiIiIiUjEGWEREAywkJASWlpawtLTss/dwcXGBpaUlvvjiiz57DyL6f50f301E9B+Qk5ODgIAA4Ws9PT1kZmZCV1dX6XWPHz/GzJkzUVtbKzx36NAhTJs2rU/HS0SDB+9gEZHaePToEVJSUrpsd/bsWbngiohI1RhgEZFa0NHRAQDEx8d32batTds1RESqxgCLiNSCi4sLACAzMxPl5eUK21VUVCAjIwMAMHfu3H4bHxENLgywiEgtzJw5E6ampmhubkZiYqLCdqdOnUJTUxNMTU0xY8aMfh0jEQ0eTHInIrWgqamJhQsXQiwWIz4+HoGBgZ22a1se9PT0hKamZpf9NjQ0ICYmBqdPn0ZxcTFqa2thaGgIKysreHp6wsvLCyKR8s+qd+/eRUREBNLT03H//n0YGhrC2toaAQEBPQryampqcPToUaSmpuLmzZuora2FkZERrK2t4evrC3d3d2hoaHS7PyLqOwywiEht+Pj4QCwW49q1ayguLoaFhYXc6yUlJbh69arQtrCwUGl/paWlWLNmDf7++2+55x88eID09HSkp6cjOjoaoaGhMDIy6rSP3NxcrF27Vi6pvry8HKmpqUhNTcVHH33Ure8tKysLGzduhFQqlXteti8nJyfs2bMHQ4YM6VafRNR3uERIRGrDyspKCKo6S3Zve27cuHGYMGGC0r7q6uoQGBgoBFeurq4ICwtDbGws9u3bBwcHBwDA5cuX8f7776O5ublDH2VlZUJwJRKJsGzZMojFYpw4cQI7d+6Eubk5QkJCcO7cOaVjuXz5MtasWQOpVAoTExNs3LgR4eHhiIuLQ3h4OLy9vQEAaWlprHNF9JxggEVEasXHxwdozbVqaWkRnm9paUFCQoJcG2X2798PiUQCAFi3bh0OHDgAFxcXWFtbY/78+Th06BC8vLwAAFeuXEF0dHSHPr755hvhzlVwcDD+97//wdHRETY2NvD390dsbCzGjx+PgoICheNobGzE5s2b0djYiNmzZyMlJQXr1q2Ds7MzJk6cCGdnZwQHB2PHjh0AgDNnzghJ/EQ0cBhgEZFa8fb2hkgkwp07d5CTkyM8n5OTgzt37kAkEgmBkSINDQ04ceIEAMDCwqLTZTwNDQ189dVXwtLgkSNH5F4vLy8XanI5OzvD09OzQx/6+vpCYKRIYmIi/v33X+jo6ODbb79VWET1rbfewqRJkwAAcXFxSvskor7HAIuI1MqIESOEiuyyy4Rtj6dPn44RI0Yo7aOgoAAPHz4EACxatEhhMry+vj4WLFgAtOZ33b9/X3gtJydHWDb08/NT+F6TJk3qkCsm6/fffwcA2NvbY/jw4UrHbWdnBwD4448/lLYjor7HJHciUju+vr7IysrCmTNnsH37dgBAUlIS0M3lweLiYuHx5MmTlbadPHkyjh07JlxnZmYGACgqKhLa2NjYKO3DxsZG7j1ltS0fXrhwodtnFT548KBb7Yio7/AOFhGpnXnz5kFXVxe1tbU4e/YsUlJSUFdXBz09Pbi5uXV5fXV1tfC4q7tGJiYmnV4nu9vP2Ni42320V1lZ2eV423v8+HGPryEi1eIdLCJSO0OGDIGrqysSEhIQHx8vJLu7urpCT0+vR30NdF2ptmXGOXPmYPPmzQM6FiLqPgZYRKSWfH19kZCQILejztfXt1vXGhoaCo8rKirw2muvKWwruxwne137Pl566aVu9dGekZER7t+/j8bGRowbN65b4yeigcclQiJSS46OjjA1NUVTUxOamppgZmYGR0fHbl0rm3T+559/Km2bl5fX6XWywVB+fr7SPpSVabCyshLaNDQ0dDFyInpeMMAiIrWkqakJHx8faGtrQ1tbGz4+Pl0eadPG2toaQ4cOBQCcPHkST58+7bRdbW0tfvvtNwDA2LFjhQR3AJg2bZqw+/CXX35R+F55eXlyCfHttR1iXVNTw/ILRP8hDLCISG1t3rwZ+fn5yM/Px6ZNm7p9nba2Nvz9/YHW3YChoaEd2rS0tGDHjh2oqqoCALzzzjtyr5uZmWHu3LlAa6mFX3/9tUMfdXV1wi5HRRYtWiQsL+7atQuXLl1S2j43NxcXL17s8nskor7FHCwiok6sX78eycnJkEgkCAkJQVFREfz8/GBqaorS0lL8/PPPQiAzZcoULF26tEMfW7ZsQUZGBurq6rBp0yZcunQJ7u7u0NfXx40bN3Dw4EHcvHkT1tbWCpcJtbW1sXfvXrz77rt49OgRVq5cCQ8PD7i6umLUqFF4+vQpysvLcfXqVSQnJ6OoqAhbt24VjvIhooHBAIuIqBP6+voQi8XCYc9JSUlCLS1ZU6dORVhYWKfFSEeNGoWwsDCsW7cOdXV1OHr0KI4ePSrXZv369dDQ0FCah2Vra4vDhw9j48aNuHPnDhISEoRjfxSNnYgGFgMsIiIFRo0ahfj4eMTExOD06dMoKipCXV0dDA0NMWHCBHh5ecHLy0tpbte0adOQmJiIiIgIpKen4/79+zA0NIS1tTVWrFiB2bNnIyQkpMux2Nra4syZM4iLi0NqaiquXbuGqqoqiEQiDB8+HGPGjIG9vT3c3Nzw+uuvq/gnQUQ9pdEiexoqEREREfUak9yJiIiIVIwBFhEREZGKMcAiIiIiUjEGWEREREQqxgCLiIiISMUYYBERERGpGAMsIiIiIhVjgEVERESkYgywiIiIiFSMARYRERGRijHAIiIiIlIxBlhEREREKsYAi4iIiEjFGGARERERqRgDLCIiIiIV+z9YYoPChakdaQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "riddles_ltr_df = pd.DataFrame({\n", + " 'model': ['DistilBERT Base', 'DistilBERT Base', 'BERT Base', 'BERT Base'],\n", + " 'order': [0, 0, 1, 1],\n", + " 'direction': [\"LTR\", \"RTL\", \"LTR\", \"RTL\"],\n", + " 'ppl': [290, 530, 620, 690],\n", + "})\n", + "\n", + "riddles_ltr_df_sorted_pairs = riddles_ltr_df.sort_values(by=['order', 'direction'])\n", + "\n", + "# Plot configuration\n", + "plt.figure(figsize=(6, 8))\n", + "sns.set_style(\"whitegrid\")\n", + "\n", + "# Create bar plot with LTR and RTL next to each other, no error bars (ci=None)\n", + "sns.barplot(x='model', y='ppl', hue='direction', data=riddles_ltr_df_sorted_pairs, dodge=True, palette=\"Set2\", ci=None)\n", + "\n", + "# Adjustments to the plot\n", + "# plt.xticks(rotation=45)\n", + "plt.title(\"Perplexity vs Model Size, From MLM\", fontsize=20)\n", + "plt.xlabel(\"Model\", fontsize=20)\n", + "plt.ylabel(\"Test Perplexity\", fontsize=20)\n", + "plt.legend(title=\"\", fontsize=20)\n", + "plt.tick_params(axis='both', labelsize=20)\n", + "\n", + "# Display the updated plot\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "execution_state": "idle", + "id": "8e5325e7-85ed-4cda-b24a-9f3248dec10b", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_480557/2459623878.py:14: FutureWarning: \n", + "\n", + "The `ci` parameter is deprecated. Use `errorbar=None` for the same effect.\n", + "\n", + " sns.barplot(x='model', y='ppl', hue='direction', data=transfer_wikitext_df.sort_values(by=['order', 'direction']), dodge=True, palette=\"Set2\", ci=None)\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAm0AAAMWCAYAAACjmbEIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAACUl0lEQVR4nOzdd3gU1eL/8c+mJxAIoQsozVCDEWmKghRpKtIRFbiCIioW8OoVEQsgYter14KigCLSpYggBEVEipRIQpEOCZAAaRCSkLa/P77Z+SVkU3dDmOz79Tw+TnbOzJwJk93PnplzjsVqtVoFAACAa5pbWVcAAAAAhSO0AQAAmAChDQAAwAQIbQAAACZAaAMAADABQhsAAIAJENoAAABMgNAGAABgAoQ2AAAAEyC0ocg+/vhjNWnSRE2aNCnrquSra9euatKkiV588cWyrgquIUuXLjWu3aioqFI5xvDhw9WkSRMNHz68VPZfVGb4OwXKm6ioKOPvbunSpaV2HI9S23M5tm3bNo0YMcLuOh8fHwUGBqpZs2bq3bu3evfuLQ8Pfs0wn6ioKHXr1s342WKxKDQ0VHXq1Cl02549e+r48ePGz2+++aYGDBhQanUtT6xWqzZs2KCffvpJEREROnfunC5fviw/Pz/VqlVLDRs2VKtWrXT77beradOmZV3dMnHltVmYcePG6amnnirVOplJfHy8li5dqk2bNunQoUNKTEyUxWJR5cqVVadOHTVt2lQ333yz7rjjDlWtWrWsq4scSBNOlpqaqtOnT+v06dMKDQ3VnDlz9Nlnn6l69eplXTWXl/ONnhBRfFarVStXrtTYsWMLLBcWFpYrsKHozp8/r6efflo7d+7Ms+7ixYu6ePGiDh06pLVr1+qdd97R6tWr1ahRozKpK8wpNDRUL730khISEvKsO3funM6dO6ewsDD98MMPuummm7Rw4cIyqaezlLf3fUKbg4YNG6YHHnjA+Dk5OVkRERH6+uuvderUKYWHh+uJJ57QwoULZbFYyrSurmDDhg1lXYVyydvbW5cvX9by5csLDW3Lly/PtQ2KJi0tTQ8//LAOHjwoSWrevLkGDBigZs2aqUKFCkpKStKRI0f0119/aePGjbp48aLd/Tz11FMu1arUrVs3PfvsswWWobXo/+zYsUPPPPOM0tPT5e7urrvvvltdunRR3bp15e7urvPnz2vfvn3atGmTdu/eXdbVhR2ENgdVrVpVQUFBuV4LCQnRvffeq8GDB+vEiRPas2ePfv31V3Xt2rXM6gk4omvXrvr555919OhRhYeHKzg42G659PR0rV69Wsr+MLUto3CLFi0yAtuAAQP0xhtvyM0t92PHbdu21f3336+0tDStWrVKlSpVKqPaXjsqVaqU5z0Y9r355ptGYPvqq69022235SnTuXNnPf744zp16pS2bNlSJvVE/uiIUEoqV66sMWPGGD9v2rSpTOsDOKJRo0Zq2bKllKMlzZ6NGzcqISFBnp6e6tOnz1WsofmFhoZKkjw8PDRx4sQ8gS0nLy8vDRgwgMcuUGQxMTGKiIiQJHXv3t1uYMupTp06GjRo0FWqHYqKlrZS1KpVK2P59OnTedZnZmZqxYoVWrNmjfbu3auEhARVqFBBDRs2VI8ePTRs2DD5+PjY3ffw4cO1fft2tWvXTt9++62OHz+uuXPn6o8//lBMTIxSU1MVGhqqunXr5uo4MXfuXLVt21aLFy/W0qVLdfToUaWlpen666/X3XffrX/961/y9vZ26LwvX76sRYsWad26dTp8+LASExPl7++vJk2a6O6771b//v3zdM7YsWOHhg8frqysLHXu3FkzZ860u++kpCT17dtXp06dUmBgoFauXKlq1aoZ67t27apTp06pf//+mjFjhvH6lT3pJk6cqIkTJ+Z6zfawcv/+/bVv3z41bNhQP//8c4HnGh8frzvuuEPp6ekaNmyYXnvttSL9jiZOnKilS5fK29tbf/75pypWrFhgeduD/cHBwVq8eHGudREREfr++++1c+dOxcTEKDMzU4GBgapatapuuukm3X777eratavDt+fvu+8+RUREaPXq1XrxxRftdrCxBbouXboUuRUoKytLK1eu1KpVq7Rv3z4lJiaqYsWKuvHGG9WrVy8NHjxYXl5eBe4jMTFRX375pdavX6/Tp0+rQoUKatKkiYYOHarevXsX+RxLcu06i+09okqVKg61oH388cf65JNPJEn//PNPrnW2942isr2HXOncuXP67rvvtGnTJkVFRSk5OVlVq1ZVSEiIhg4dWmggKCtLly41/u5DQ0NVo0YNff/991q9erVOnDihhISEPJ0W0tLStGjRIq1Zs0aHDh1SUlKSKleurObNm+uee+7Rvffem2/AfvHFF7Vs2TLVqVNHGzZs0Llz5/T1119rw4YNio6OVqVKldS6dWuNGzdON954o7FdVFSUZs+erU2bNunMmTOqWLGibr31Vj3zzDO6/vrrS3TuZ86cMZZvuOGGEu3jSklJSVq4cKF+++03HTlyRImJifLy8tL111+vW265RX369NEtt9ySa5vifHZJ0tmzZ7Vu3Tpt27ZNBw4c0NmzZ5WRkaEqVaqoZcuWuvfee9WrVy+7/wbFed+/0sGDB7VgwQJt375d0dHRSk5OVkBAgG688UZ17NhR9913n2rUqFHg72fz5s369ttvFR4ersTERNWoUUN33HGHHn/8cdWqVatYv2sbQlspyvnmnpmZmWvd6dOn9fjjj+vAgQO5Xk9ISNCuXbu0a9cuzZ8/X1988YUaNGhQ4HHWr1+v559/XsnJyYXWKT09XWPGjMnT8vfPP//on3/+0YoVKzR79uwSf4M/cOCAnnjiCZ06dSrX63FxcdqyZYu2bNmiBQsW6PPPP88Vttq0aaMxY8bo888/18aNGzVv3jw9+OCDefb/+uuvG/t+4403cu3DWQYNGqQpU6bo6NGjCgsLU0hISL5lV65cqfT0dEnSwIEDi3yMvn37aunSpbp8+bLWrVun/v3751s2PDzceLD/3nvvzbVu9uzZeuutt5SVlZXr9ejoaEVHR2vv3r36/vvvtWvXLlWoUKHI9bPn7rvv1ltvvaXY2Fj98ccfuvPOO3OtT0xM1G+//SZlB7yiSEhI0OOPP65du3blej0+Pl7bt2/X9u3bNW/ePH355Zf59lo9cuSI/vWvf+ns2bPGa5cvXzaut99//11t27YttC4lvXadxdPTU8rujJCQkKCAgACnH8MZVqxYoVdffTXP+010dLTWrFmjNWvWaNCgQXr99dev6Z7z8fHxGjdunPbv359vmaioKD366KM6evRortfPnz+v33//Xb///rsWLFigTz/9tNB/rwMHDuiRRx7RuXPnjNdSU1O1Zs0a/f777/ryyy/Vpk0bbdmyRU899VSuZxYvX76sVatWadOmTZo3b16ugFdUtutL2X8zjvrzzz81YcIExcfH53o9PT1d+/fv1/79+/Xdd9/l+eKQU2GfXZmZmercuXOe9zdlh7kNGzZow4YNWrx4sT7++GOH3+Nsx3z77bc1Z84cWa3WXOvOnz+v8+fPa8uWLTpy5EiuhoErvffee3kaH06dOqUffvhBv/zyi7777rsSdSK6dv+iygHb8ymSciXy+Ph4PfDAAzpz5oy8vLw0ZMgQtW3bVnXq1FFycrI2b96suXPn6sSJE3r00Ue1bNky+fv72z3G6dOn9fzzz8vHx0ePP/642rRpI3d3d4WHh8vPzy9P+Q8//FDh4eG6/fbbNWzYMNWqVUvR0dH6/vvvtXnzZh0+fFhjx47VwoUL5e7uXqzzPXHihB566CFdvHhRFStW1IMPPqhWrVqpVq1aSkhI0IYNG7RgwQKjc8a8efNyvZGMGzdOf/zxhyIiIvT222+rQ4cOuS7qn376SStWrJAkDR06tFjPCK5cuVJnz57V6NGjJUnPPvtsniEDbA8r9+3bV2+//bZSU1O1dOnSAkObbTyeJk2a5Puclz3t27dXjRo1dPbsWa1cubLA0LZq1SpJMh4ctjlw4IAR2OrWrauHHnpITZs2VUBAgC5duqRjx45p69atTuucUbVqVd1+++367bfftHz58jyh7eeff1ZaWpoCAgLUqVOnQh9kzszM1NixY41y7dq104MPPqi6devq7NmzWrJkidavX2+Esh9//DHPm3JSUpJGjx5tBLY+ffqoX79+qlq1qo4fP65vvvlGS5cu1aFDhwqsi6PXrjO0aNFCBw8elNVq1eTJkzVjxgynfAjlNH36dKWkpOS7Pi4uTk8++aSSkpLk7++f531n9erVeuGFF2S1WlWvXj099NBDatSokQIDA3Xq1CktXrxYGzdu1OLFi1WxYsU8rRrXkkmTJungwYPq16+f+vTpo2rVqunMmTPGv+ulS5f0r3/9S5GRkVL2LcWBAweqRo0aioqK0rx587R9+3bt3LlTY8eO1bx58/J9z0xJSdGTTz6p9PR0TZgwQW3btpW7u7s2bdqkzz//XMnJyXrhhRf0zTff6Mknn5S/v7+efvpp3XTTTcrIyNAvv/yiOXPmKDExUZMmTSpRj85GjRoZnYM2bNigFStWqG/fviX63W3dulWPPvqoMjIy5O7urvvuu0/dunVT7dq1dfnyZR05ckS///67fv3113z3UZTPLlto6tChgzp16qSgoCAFBgbq0qVLioyM1KJFi7R7925t3rxZU6ZM0VtvvZXrGMV537eZPHmylixZIkmqXr26HnroId18883y9/dXXFyc9uzZo7Vr1xb4+1m4cKF2796tdu3aaejQoapfv74uXryoH3/8UT/++KPi4uL00ksvacGCBcX6vdt+KSimrVu3WoOCgqxBQUHW//73v3bLpKenW4cMGWKUW7ZsmbFuwoQJ1qCgIGuXLl2sJ0+etLv93r17rSEhIdagoCDr+++/n2f9Qw89ZOz79ttvt546dapI9Q0KCrJOnjzZbrmXXnrJKPPdd9/lWf/f//7XWG/P0KFDrUFBQdZ+/fpZY2Nj7ZbZuHGjtWnTptagoCDrggUL8qw/evSo9aabbjL2c/nyZavVarWePn3a2qZNG2tQUJC1R48e1uTkZLv779KlizUoKMj6n//8J8+6yMhIo/5Lliyxu73N888/bw0KCrLecsst1pSUFLtl9u7da+zvm2++KXB/9rz55pvWoKAga7Nmzaznzp2zWyYzM9N6++23W4OCgqyjRo3Kte7DDz+0BgUFWUNCQvLd3mq1Wi9cuGDNzMwsdv1y/r5s1/lPP/1kDQoKsrZq1cp68eLFXOXvv/9+a1BQkPW1116zWq+47uz9vr/77jtj/QsvvGDNysrKU+b99983yrz99tt51s+YMcNY//nnn+dZn5aWZh01alSu6z8yMjJPOWdcu7a/yYceesju9oX5+++/jf0HBQVZ27RpY33++eetCxYssO7fv9+akZFRpP0U9nean8uXLxv/hs2aNbNu3Lgx1/rY2FjrLbfcYg0KCrJOnDjRmp6ebnc/tn+zpk2bWo8cOVKsOhRVzmvz8ccft/7zzz/5/hcdHW1st2TJklzXwsKFC/M9Rs5r64MPPsizPisry/rcc88ZZebNm5enzH/+8x9jffv27a0nTpzIUybn30GHDh2sPXr0sHsNvvXWW0a5vXv3FvM39n+mTJmS6/z79Oljffvtt63r1q3L9XsqSGpqqvGedNNNN1m3bt2ab9nTp0/nea04n11ZWVnW48ePF1ifjz76yBoUFGRt0qSJ9dixY3nWF+d9f/369UbZoUOHWhMTE4t8bjmPExQUZH355ZftvqdNmjTJoX9HOiI4WXJysrZv366HH35YYWFhUvYDnbaHsqOiooznpCZPnqx69erZ3U/z5s2NoUQKG135ueee03XXXVek+lWrVi3fb78vvfSSAgMDJUnz588v0v5sduzYYbSYzJgxw9jPlTp16qSePXtK+ZxXgwYNjPrt27dPH330kbKysvTCCy/owoUL8vT01LvvvitfX99i1a+4Bg8eLGWPjfXLL7/YLWOrv6enZ4m+sdpudWZmZuqnn36yW2bbtm1GK9KVt0bPnz8vSapfv36Bt+v8/f0LfKi9OLp16yZ/f3+lpqbm+rYZGRlp3OIs6q3RefPmSZICAwM1efJku8/cPfXUU2rYsKGU3bsyLS3NWJeWlmZ8I27SpEmujj82np6eeuONNwpsFXPWteuoVq1aacqUKUZdL1y4oOXLl2vy5Mm677771KZNG40aNUoLFy4s0qMQxfXqq68a/4YvvPCCOnXqlGv9/PnzdfHiRdWsWVOvvfZavrc+n3rqKdWsWVNZWVkFdlpxltDQUN177735/vfBBx/Y3a5Dhw7G3/mV0tLSjGdHb7zxRrvPPFksFr322mvGbVHb9Zyf/J5HGzhwoPEccVxcnCZNmmT3Ghw2bJixvGPHjgKPlZ/nn38+17/r4cOH9dVXX+nJJ59Up06d1KVLF02cOFHbtm3Ldx8//vij8Z40fvx4tW/fPt+ytWvXLrA+hX12WSyWQp+/e/LJJ1WlShVjUGpHfPnll5IkX19fffTRRwU+W1rQuVWvXj3f97RRo0YZyyX5dyS0OeiTTz4xpq5o0qSJbr755lwP+1atWlX/+9//jAepN27cqMzMTPn6+uZ5U7yS7Tmcs2fP2u3IoOwPpeI8aN27d+98A0+FChWMfR06dCjXsxeFsfV8a9CgQaHT59jOKyIiQhkZGXnW57z1+fXXX+u5554zfp9PPvlksW5DllTbtm1Vv359KZ8P6LS0NK1cuVLK7vyQ3wd9QVq0aGEEEtst0CvZjuHj46Pu3bvnWmd77vDw4cPas2dPsY9fEt7e3kZwyfmBbFuuX79+gbeTbWJiYoznanr37p1vRwwPDw9jMMzExETt3bvXWLd3714lJiZKkvr3759vR4tatWqpY8eO+dbFmdeuowYPHqwVK1ZowIABeW6N2h6dmDx5snr06KHff//dacedNWuWcZ0PHDhQ//rXv/KUsX0g3nnnnQV2DPHw8DCugWt5rK8rvwTlFBERoQsXLkjZ11Z+tz0rVqxovGcePnw413OVOVkslnzfp318fIxgUrlyZd1xxx12y9WrV8+4Jmy3bIvLx8dHM2fO1AcffKA2bdrk+Zs5ffq0li5dqhEjRmj06NGKi4vLsw/bLU8/Pz8NGTKkRPVQCT67lN1pKSYmRkePHtXBgwd18OBBHTlyxHio/8pnxIsjPj7eaGjp3bu3atasWeJ99erVK9+/kYYNGxq3f0vy78gzbaWkbt266tmzp0aPHp3rnrmty3VKSoqaN29e5P2dP3/e7jeS+vXrF6u3p23Yhvy0atXK+MZ48ODBIndIsJ3XsWPHijznYXp6uhITE+0OfPnGG2+ob9++OnfunDHW1y233GK3NaW0DBo0SO+++662bt2qU6dO5XoQfsOGDcaI4sXpgHCle++9Vx999JH27NmjEydO5PpWmZaWpnXr1knZwfDKYHP33Xdr5syZSktL07Bhw3THHXeoc+fOuuWWW3TjjTeW2mDO/fr10+LFi7V9+3adOXNGtWvXNp41LGqLY85nzHL2srbnpptuyrXdzTffLF3xzGhhQT44ONjoJHElZ1+7jmrYsKHefPNNTZkyRREREfr7778VERGhv/76S9HR0VJ2782xY8fmO9ZWcfz222969913pewOQfZ6QGdmZhofiAsWLCjyszi21uDSdGVP8aIq6N865/WZ8/qz56abbjLuTBw6dMhuj8IqVaoU2FHB1qJz/fXXF/h3W6lSJV26dEmXLl0qsE4FsVgs6tOnj/r06aO4uDjt2rVL4eHhCg8P186dO5WamipJ+uOPPzRixAgtWLAg1xcIW8eNFi1aOHTHo6ifXVarVStWrNDixYu1Z88eo372XNkpojgOHDhgPEPXpk2bEu9H2V8AC1K5cmUlJyeX6N+R0OagnDMiWCwWeXt7q0qVKvl2HIiNjS3RcfJ7eLi4QwMU9iGTc72tFaMo7H0jK4r8ziswMFATJkwwbpV6enrq7bffLnbnCEf0799fH330kdLT07Vs2TKNGzfOWGe7LVezZk3dfvvtJT6GLbQpu1Ut5zF+++0349u+vVaBRo0a6b333tPkyZOVmJioX3/91fgWXKVKFd1xxx0aOnSow29AV2rTpo3q1KmjU6dOaeXKlWrbtq1OnDghi8VS5NCW89oq7JrMees359Q7OfdRWEtnQbePnX3tOounp6duvvlmI6RK0pYtW/TGG2/o0KFDyszM1Ouvv641a9aUOKAfOnRIEyZMUFZWlurUqaOPP/7YbgtBYmJiiVoWC/qALWuVK1fOd11Jr6383jMLCze2xxeKWs5eb8qSCAwMVPfu3Y1W/EuXLmnBggX68MMPdfnyZR06dEhz5szRE088YWxjC0aOjhFYlM+uy5cva9y4cUVuVXbkessZ+Bw9t9L8dyS0OcjejAgFsQ39UaVKFc2dO7fI29kbK0nZPQqLo7RaX2zn1bRpU73zzjtF3i6/JuiMjIxcz9Wlp6dr+/bt+f4eSkO1atV05513at26dVq2bJmefPJJWSwWxcTEaPPmzVJ2q5MjQbJevXq6+eabtXv37jyhzXbLNCAgIN9bJj179tRtt92m1atX648//tCOHTsUFxen+Ph4rVixQitWrFD//v01ffp0pz3XZrFYdO+99+rzzz/X8uXLjVv3rVu3zvcZzcL254w6lZSzr93SdOutt+rrr7/Wvffeq4SEBB0/flz79+8vVqu9TVxcnMaOHatLly7Jz89Pn332Wb4BJeeQRYMHDzbGfSyMs3vYOlNR/x5cafrBChUqaNSoUfLz89Orr74qSVqzZk2u0OYsRXnf/Oyzz4zA1q5dOz3wwANq0aKFqlWrJh8fH+Pf8MEHHyzxc35mQ2i7ymxN5JcuXVKjRo2uasuRinC7ImdLYEHfRK9kO6/k5GSnTCnzv//9z3hOq2LFikpKStK0adPUtm3bEgWDkho8eLDWrVunqKgobd++Xe3bt9fy5cuNDzFnTD587733avfu3Tp+/LgxRVRSUpJxO69Xr14Ffvj5+/tr6NChGjp0qJQ9BlNoaKi+/fZbnT17VsuWLVOzZs00cuRIh+tq069fP33++ec6fPiw8VxGv379irx9zmursGsy5/qct5hyflOPjY0t8JZEQcdw9rVb2mrUqKHOnTsbzxGeOHGi2KEtPT1dTz/9tKKiomSxWPTuu+8WeLsw57+X1Wo1xe/JETnPtzjXVnHeM69lAwYM0NSpU5WRkaGTJ0/mWlelShVFR0cX65nnkrBarUZnkDZt2mjOnDn5Bu3i3BXKT5UqVYzl0j43R9AR4SqzvbmmpaUZz9JcTYUdMzw83FguzgCOtvOKjIx0+ILfvXu3vvjiC0nS7bffrjlz5sjT01OXLl3SCy+8kGeg4qIqyTfmO+64w3jI1fagtu3WaJs2bYzOCo7o3bu30RPP1rq2du1aY7L1gh6YtqdRo0YaM2aMFi5caDzwWtjMDsXVoEED41m0y5cvy8vLS7169Sry9jmvrcI6UeRcn3O7nMEh53VrT0HXvTOv3asl53NTJbmuX3/9df31119Sdg/AK8euupKXl5fxu79yIOTyKOd19vfffxdYNr/r08y8vLyMLzNXXl+2v5eIiIhSfUQgISHB+HvMb8YDZTeAHDt2LN/9FPXvo1mzZkbZa7nVjtB2lXXp0sW4MObMmXPVj79mzZp87/snJycbH+6NGzcudIqOnGy9Pa1Wa7Fu+17p0qVLev7555WZmamAgABNnz5dLVu21NNPPy1lf2DkN8VVYXI+9Jpz6IiCuLm5Ga1pa9eu1e+//27MTuBIB4ScAgMDjd6NP/30k7KysozwVqdOnTzTwBRV7dq1jVDpyAO6+enXr5+8vLzk5eWlu+66q1jPV9asWdMYOPnnn3/O94HczMxMLVu2TMpuxWjRooWxrmXLlkbLxvLly/OMXm4TExOjP/74I9+6OOvadVR+9bcnZwgtbsvz7NmztWjRIin7C8Fjjz1WpO1sv6ejR4+W+7mUW7ZsaVzPP/74Y77PHiUlJZX4PfNqK871debMGeOuy5WPpNiug5SUlJINDltEOb+cFxQOFy1aVODzlkV93w8ICDCeH/35558VExNTglqXPkLbVdawYUOjReKnn37SN998U2D5yMjIfIeDKIlz587l29NqxowZxh9qzjGBiuL22283Wl5mzZpl9PjMzz///GN3TJ1p06YZt9umTp1qPDf0yCOPqF27dlL2rdOStFIGBAQYtxmvbPIvyMCBA2WxWJSSkqKXXnpJyn72ozgtS4WxtaadO3dOK1euNMZJuueee/L9prh+/Xqjo4I9Z86cMabfKY1nAR988EGj19n7779fou2V/WzVtGnT7Jb55JNPdPjwYSn7VnXOh+Rtk6Yru0fbV199lWf7jIwMvfzyy8ZUY/Y469p11Lhx4zRv3rxCx2BbunSptmzZIkm67rrrinVr9Pfff9fbb78tZffafeONN4q87YgRI4yW24kTJxY6y8Rvv/1mdwiGpUuXGkMkffzxx0U+/tXk5eVlTJZ+8OBBffrpp3nKWK1WTZ061fhCZG/avWvJ4cOHNWrUKKOFNT+XL1/W5MmTjZB3ZSts3759jfflDz/8sMC5bG29nUsiMDDQCM6rVq2yG7j27NljdOTKT3He9x999FEpOyQ+88wzuaYSu5Ij5+YInmkrA6+99poiIiIUGRmpGTNmKDQ0VPfdd59uvPFGeXl5KSEhQQcOHNCmTZu0detW3XXXXbrnnnuccuyWLVtq/vz5ioqK0v3336/atWvrzJkzmj9/vtEa0bx5c91///3F3vd7772nwYMHKyEhQePHj9eKFSvUp08f1a9fX25uboqNjdX+/fv166+/KiwsTKNGjco1FdW6deuMW5ADBgxQjx49jHVubm5666231LdvX128eFH//ve/9eOPP8rHx6fI9fPw8FBwcLB27dqlJUuWqHnz5mrWrJlxa7Jy5cp2u+XXrVtXt912mzZv3mw01/fu3dvuNGEl1a1bN/n5+Sk5OVnTpk0zvmUWdGt0zpw5+ve//63OnTsbU375+/srMTFRERER+u6774xW1eKG8Kvh/vvv18qVK7V7924tXbpUp0+f1gMPPKC6devq3LlzWrJkiTGw8fXXX2/3Yegnn3xSP//8s6Kjo/Xuu+/qwIEDuu+++3JNYxUeHq6WLVsWGPQdvXad4cyZM5oyZYreffddde3aVW3atFGDBg1UuXJlXb58WUePHtWaNWu0ceNGKfu2z8SJE4t8+ycxMVETJkxQZmam/Pz89Oyzz+rEiRMFbtOgQQPjA69atWp666239PTTT+vcuXMaOHCg+vfvr06dOqlWrVrKyMhQdHS0Mc1PZGSkPv/8czVt2tQJv52r78knn9S6desUGRmpjz/+WAcPHtSAAQNUvXp1RUVF6bvvvjMCy80332w8U3qtslqt2rx5szZv3qwbbrhB3bp1M6Zq8/X1VXx8vPbs2aOFCxcqKipKyv5SYJsCysbb21tvv/22Ro8erZSUFD388MPq27evunfvrlq1aiktLU1Hjx7Vxo0btWHDhhI/BuTm5qZ7771X8+bN0z///KNhw4bp4Ycf1g033KCkpCRt3LhR33//vfz8/FSjRg3jDsiVivO+37VrVw0aNEiLFy/W7t271adPHz300ENq3bq1KlasqPj4eEVERGj16tVq2rRpiYaacRShrQwEBARo/vz5evbZZ7Vjxw799ddfBX77ceb8g+PHj9c333yjTZs22b3F0bBhQ33++eclmuj5+uuv1w8//KCnn35aBw8ezDUEhT05z+vs2bN6+eWXpezbPbblnK677jq98sorev7553Xs2DHNmDHD7phSBXnsscc0duxYJSQk6Lnnnsu1bty4cXZHPlf2mG22HqNy4q1RGz8/P3Xr1k0rV640Ws+aNm1a6DMyKSkpxiTd9ri5uempp57KMzDvtcDd3V2ff/65MWH81q1btXXr1jzlGjVqpC+//NLu34G/v7+++uorPfzwwzp37pxWrVqVp2V6wIABatu2bYHzYDpy7TpLrVq1tHfvXiUnJ9s9j5z8/f318ssv5/piU5iLFy8aLQfJycm5RmbPT2hoaK5W2h49eujTTz/VxIkTlZCQoB9++EE//PCD3W3d3NzsDn2Q8/GMwiZZL0sVK1bU7NmzjQnj165da3fOydatW+uzzz676p3KisvX11eVK1dWYmKiTpw4oa+//rrA8i1bttSHH35od/iqDh066PPPP9dzzz2nxMRELV26tFRmCRk/frx27dql/fv3KyIiIs97dkBAgD7++GP997//zTe0qZjv+1OmTJGPj4/mzZuns2fP5nsXoay+jBDaykj16tU1b948/fbbb1q1apXCwsJ0/vx5ZWRkyN/fXzfccINuvvlmde3a1RiF3Rk8PT01c+ZMLViwQMuXL9fRo0eVnp6uevXqqU+fPnr44YeL1Xp1pQYNGujHH3/Uzz//rF9++UXh4eGKi4sznlFr0KCBbrnlFt11113G80lWq9X4EHB3d9c777yT74di3759tXHjRq1atUrz589Xly5d1Llz5yLX784779Ts2bM1d+5chYeHKz4+vsBbZzbdu3c3WsIaNmyo1q1bF+O3UjT33nuvMQOCitAB4b333tNvv/2mbdu26ciRIzp//rzi4+Pl5eWlOnXqqE2bNrr//vuv6ZaOgIAAzZs3TytWrNCqVau0f/9+JSYmqkKFCgoKClKvXr3y3Ba90o033qhVq1bpyy+/1Pr163X69Glj+yFDhuiee+4p0gdKSa5dZ/r000919OhR/fHHH9q1a5cOHz6s6OhoJScny9vbWwEBAbrxxhvVsWNH3XvvvSWahcMZunbtqtDQUC1cuFAbN27U4cOHlZiYKHd3d1WrVk033nijOnTooJ49e9qd6sc26nzlypXVv3//MjiDoqtbt66WL1+uRYsWac2aNTp48KAuXbqkypUrq1mzZsZUWc4aTqc01atXT5s3b9aOHTu0detW7dmzR8ePH1dcXJwyMjLk5+enWrVqqUWLFurRo4fuvPPOAs/rjjvu0Pr16zV//nz9+uuvOnbsmC5evChfX1/dcMMNatOmjcN3iPz9/TV//nx98803+vnnn3XixAm5u7urdu3a6ty5s0aOHGl0FCtIcd733d3dNXnyZA0YMEALFiwwphJMT09XQECAmjRpojvuuKNEUxc6g8VanKcTYUrbtm0zxlWaO3dugXPFwb7jx48b0zf9+9//Np59AFA8Xbt21alTp/TUU0/lGpcQQOGu/a8HwDXANsyHh4dHscYjA/D/nTp1SqdOnZK/v79Txw0EXAWhDSjEhQsXtHDhQim7w4CjU5wArsr27O7w4cPzneoPQP54pg2wIzY2VklJSTp79qw+/vhjJSQkyGKxFHlMKwB59evXj5ZqwAGENsCOd955xxjU1cY27x0AAGWB0AYUwNPTU9dff72GDBmihx56qKyrAwBwYfQeBQAAMAFa2kwgKytLGRkZcnNzK9Hk0AAAIH9Wq1VZWVny8PC4psfdI7SZQEZGhsLDw8u6GgAAlGvBwcEFDuZd1ghtJmBL/cHBwdf8VCkAAJhNZmamwsPDr+lWNhHazMF2S9Td3Z3QBgBAKbnWH0G6tiMlAAAAJEIbAACAOZj69mhsbKz27NmjPXv2KDw8XOHh4UpISJAk9e/fXzNmzCh0HykpKdq0aZM2b96siIgInTx5UsnJyapYsaLq16+v22+/Xffff3+Rpy5KSUnRd999pzVr1igyMlJpaWmqVauW7rzzTg0fPlx16tRx+LwBAIDrMXVou+222xza/sCBAxo2bJiSk5PzrEtISFBYWJjCwsI0e/ZsTZ06VX369ClwfydOnNCYMWN0/PjxXK8fO3ZMx44d06JFi/Tuu++qS5cuDtUbAAC4HlOHtpyuu+46NWzYUH/88UeRt0lKSjICW+vWrdWlSxe1bNlSAQEBiouL0y+//KJFixYpKSlJ//73v1WhQgV17tw5333lDGxDhgxRnz595OPjo23btumLL75QUlKSxo8fr/nz56tZs2ZOOnMAAOAKTB3annzySQUHBys4OFjVqlVTVFSUunXrVuTt3dzc1Lt3b40bN06NGzfOs/72229Xp06dNG7cOGVmZmratGnq1KmT3d4ls2bNMgLb888/r0ceecRYd/PNN6tdu3YaPny4UlJSNH36dH377bclPm8AAOB6TN0R4emnn1aXLl1UrVq1Em3funVrffjhh3YDm0337t3Vo0cPSdLJkye1b9++PGXS09ONENaoUSONGjXK7rEGDhwoSdq+fbv27NlTojoDAADXZOrQdrW0b9/eWD558mSe9du2bdPFixclSf369ct3cL4BAwYYy+vXry+VugIAgPKJ0FYEaWlpxrK9wW137txpLLdr1y7f/bRs2VK+vr6SpF27djm9ngAAoPwitBXB9u3bjeVGjRrlWX/kyBFjuWHDhvnux8PDQ9dff32ebQAAAApj6o4IV8OBAwe0ceNGSVJQUJDd0BYdHS1J8vPzU6VKlQrcX+3atfXPP/8oLi5OaWlp1/TEtABwNVmtVqWnpysrK6usqwITcXd3l4eHxzU/BZUzENoKkJaWpkmTJikzM1OSNH78eLvlLl26JGWHtsLYbo/atitOaLPVAwDKk5SUFCUmJiopKYn3OZSIt7e3KleurICAgBKFN7Ncd4S2AkyZMkURERFS9gwLXbt2tVvu8uXLkiRPT89C95kzpNm2K6rw8PBilQcAM3Bzc5OPj4/8/f3l4+OTb2cuwJ6MjAxdunRJp0+f1okTJ8q6OqWK0JaPL774QosWLZIkBQcH65VXXsm3rLe3t5Q99EdhcnZqsG1XVMHBwXY7QgCAGaWkpCgyMlL+/v667rrrXOL2FkpH9erVlZCQoDNnzqh27dqFPqp0pczMTFM0jBDa7Pjhhx/0/vvvS9kdC2bOnFngrc8KFSpIkt3psK6UkpKSZ7uicnd3J7QBKDcuXrwoT09P1alTh8AGh1WpUkUXLlxQUlKSqlSpUtbVKRW0QV9h1apVev311yVJderU0TfffKPAwMACt6lVq5aUHdouXLhQYNkzZ85IkgIDA+mEAMBlWa1WXbx4UZUqVSKwwWkqVqyo5OTkctuZhdCWQ2hoqP7zn/8oKytL1atX1+zZs41AVpCcPUqPHj2ab7mMjAxFRkbm2QYAXE16eroyMzOLfccBKIiPj4+ysrKUkZFR1lUpFYS2bFu2bNGzzz6rjIwMBQQE6JtvvjHGVCvMLbfcYiznHNPtShEREbkmqAcAV2VrCaHTAZzJdj3R0laO7dq1S0888YTS0tLk7++vWbNm6cYbbyzy9u3atZO/v78k6ccff5TVarVbbunSpcZy9+7dnVBzADA3bo3Cmcr79eTyoW3//v167LHHlJycLD8/P33xxRdq2bJlsfbh5eWl4cOHS9kzHcyaNStPmd27d2vJkiVSdshr1aqVk84AAAC4AlP3Ht2xY0euCdzj4+ON5RMnTuRq2dIVE7Yre/L30aNHG50HnnnmGfn7++vgwYP5HrNq1aqqWrVqntdHjx6t1atX6/jx43rnnXd08uRJ9enTRz4+Ptq2bZs+//xzZWRkyMfHRy+99JJD5w0AAFyPqUPb4sWLtWzZMrvrdu3alWdS9itD244dOxQbG2v8/OabbxZ6zHHjxumpp57K83rFihU1c+ZMjRkzRsePH9eCBQu0YMGCPGXeffddNWvWrNDjAAAA5GTq0HatueGGG7Rs2TLNmzdPa9as0cmTJ5Wenq5atWqpc+fOGjFihOrUqVPW1QQAACZkseb31DyuGZmZmQoLC1NISAiD6wIoF1JTU3Xs2DE1aNBAPj4+ZV0dl7Bt2zaNGDFCyueu0ccff6xPPvnEoWP0799fM2bMkCQNHz7c7ogKbm5u8vf3V926ddW6dWsNHTq0WJ3/ClLS68osn7Mu3xEBAABcPVlZWUpMTNTevXv17bff6r777tPMmTPLulqmwO1RAIApZFmz5GYpn20N18K5PfDAA+rZs6fddaGhofrwww8lSc8++6y6detmt1zlypXtvr5y5UpjOT09XZGRkVq/fr1WrlypzMxMvffee6pXr5569+7tlHMprwhtAABTcLO4ad7fmxSTlFjWVXGqmhUr68Gb7ijrauQ7OoKyB4e3qVmzpoKCgoq17yvLt2jRQr169dJNN92kadOmSZL+97//EdoKQWgDAJhGTFKiTl2IK+tqwEkefPBBff311zp9+rQOHTqkc+fOqXr16mVdrWtW+WxnBgAA1zw3Nzc1btzY+PnMmTNlWp9rHaENAACUGU9PT7vLyIvQBgAAysyRI0eM5euuu65M63KtI7QBAIAy8csvv+j48eOSpFtvvTXf3qf4P3REAAAAV01aWpox5Mdnn30mSfL19dX48ePLumrXPEIbAAAoVU2aNMl3XYsWLTRp0iTddNNNV7VOZsTtUQAAUCY8PT01cOBA3XLLLWVdFVOgpQ0AAJSqnDMiXLhwQf/8849mz56tkydPasqUKUpJSdEjjzxSpnU0A0IbAAAoVVfOiNCmTRvdd999euCBB/TPP//ogw8+ULt27dSqVasyq6MZcHsUAABcdRUrVtTbb78tNzc3ZWRk6K233irrKl3zCG0AAKBMNG3aVPfcc48kaceOHfr999/LukrXNEIbAAAoM2PHjpWb2//FEdsQILCP0AYAAMpMo0aNdNddd0mSdu3apa1bt5Z1la5ZdEQAAMDF7N+/X0uXLi20XIcOHa7K1FJjx47V2rVrpezWtg4dOpT6Mc2I0AYAgIsJDQ1VaGhooeX+97//XZXQ1rx5c3Xu3FkbN27U1q1bFRYWppCQkFI/rtkQ2gAAplGzYvmbm7I8nlNJjB07Vhs3bpQkffrpp5o5c2ZZV+maY7FardayrgQKlpmZaXzrcHd3L+vqAIDDUlNTdezYMTVo0EA+Pj5F2ibLmiU3S/l8FLs8n9vVVJLrSib6nOUKAQCYQnkONeX53OA8XCUAAAAmQGgDAAAwAUIbAACACRDaAAAATIDQBgAAYAKENgAAABMgtAEAAJgAoQ0AAMAECG0AAAAmQGgDAAAwAUIbAACACRDaAAAATIDQBgAAYAKENgAAABMgtAEAAJgAoQ0AAMAECG0AAAAmQGgDAAAwAUIbAACACXiUdQUAAEDp27Ztm0aMGGF3nY+PjwICAtS0aVPddddd6tu3r7y8vCRJXbt21alTpxw69ty5c9W+fXtFRUWpW7dukqT+/ftrxowZDu3X1dDSBgCAi0tNTVV0dLR+++03TZo0SQMGDFBUVFRZVwtXoKUNAGAK1qwsWdzKZ1vD1T63YcOG6YEHHjB+jo2N1aFDhzRr1ixFR0fr0KFDevzxx/Xjjz9q1qxZSk9Pt7ufiRMnKiIiQpK0cuXKfI9Xt27dUjgL10NoAwCYgsXNTYnr5ysz/mxZV8Wp3KvUUOXuw67qMatWraqgoKBcr916660aMGCA+vbtq1OnTungwYNat26devXqle9+/Pz8jOUr9wfnI7QBAEwjM/6sMs479nwV8lexYkU9/vjjevnllyVJf/75Z4GhDVdX+WxnBgAAJdKkSRNjOTo6ukzrgtwIbQAAwODp6Wkse3hwQ+5aQmgDAACGI0eOGMt16tQp07ogN0IbAACQJGVmZmrWrFnGzz179izT+iA3QhsAAC4uLi5OW7Zs0UMPPaR9+/ZJ2YGtTZs2ZV015MDNagAAXMwnn3yiTz75xO46X19f3X///Xruueeuer1QMFraAACAoWnTpho+fHiuDgm4NtDSBgCAi8k5I0JmZqaio6O1du1aLV++XLt379bw4cO1ePFiBQYGlnVVkQMtbQAAuBjbjAhBQUFq1qyZunTpohkzZmj69OmSpFOnTmnSpEllXU1cgdAGAAAkSf379zd6jG7YsEFbtmwp6yohB0IbAAAwjB8/Xu7u7pKkDz74oKyrgxwIbQAAwNCgQQP17t1bkvT3339r8+bNZV0lZCO0AQCAXB577DFZLBZJ0meffVbW1UE2eo8CAIBcgoKC1LVrV4WGhuqvv/7Sjh07nD7Q7okTJ7R06dJCy7Vq1UqNGzd26rHNitAGAADyGDt2rEJDQ6Xs1rac01s5w65du7Rr165Cy02cOJHQlo3QBgAwDfcqNcq6Ck53rZ5Tq1at1LFjR23evFl//PGH9uzZo1atWpV1tVyaxWq1Wsu6EihYZmamwsLCFBISYvToAQAzS01N1bFjx9SgQQP5+PgUaRtrVpYsbuXzUezyfG5XU0muK5noc5YrBABgCuU51JTnc4PzcJUAAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAIAyw/TXcKbyfj0R2gAAV51tUu6MjIyyrgrKkczMTEmSWzmdy7V8nhUA4Jrm4eEhb29vJSYmlnVVUI5cvHhRnp6e8vT0LOuqlApCGwDgqrNYLAoICNDFixcVHx9f1tVBOZCSkqILFy7I399fFoulrKtTKjzKugIAANdUpUoVpaWlKTo6WhcuXFDFihXl4+MjNze3cvuhC+eyWq3KzMzUxYsXdeHCBXl7e6tatWplXa1SQ2gDAJQJi8WiWrVqydfXVxcuXND58+eVlZVV1tWCCXl6eiogIEDVqlUznpcsjwhtAIAyVblyZVWuXFlZWVnKyMgguKFY3Nzc5Onp6RKts4Q2AMA1wc3NTV5eXmVdDeCaRUcEAAAAEyC0AQAAmAChDQAAwAQIbQAAACZAaAMAADABU/cejY2N1Z49e7Rnzx6Fh4crPDxcCQkJkqT+/ftrxowZxdrfxo0btXDhQoWHhysuLk6BgYEKDg7WkCFD1Llz5yLtIyMjQ4sWLdLKlSt19OhRJScnq0aNGrrttts0fPhw3XjjjSU6VwAA4NpMHdpuu+02p+wnKytLkydP1uLFi3O9HhMTo5iYGK1fv16DBw/WlClTCpyENi4uTmPGjFF4eHiu1yMjI7VgwQItW7ZMr7zyigYPHuyUegMAANdh6tCW03XXXaeGDRvqjz/+KPa2H3zwgRHYmjdvrkceeUT16tVTZGSkvvrqK+3bt0+LFi1SYGCgJkyYYHcfmZmZGjdunBHYevToocGDBysgIEB///23PvvsM8XGxuqVV15RjRo1itxyBwAAILOHtieffFLBwcEKDg5WtWrVFBUVpW7duhVrH8eOHdPXX38tSWrZsqXmzZsnHx8fSVKrVq3UtWtXPfTQQ4qIiNCsWbM0cOBA3XDDDXn2s2zZMu3cuVOS9MADD+jVV1811rVq1UqdOnXSgAEDlJSUpDfeeEMdO3aUh4epf/0AAOAqMnVHhKefflpdunRxaHLYOXPmKCMjQ5I0efJkI7DZ+Pr6avLkyVL282qzZ8+2ux9b8AsICNALL7yQZ/0NN9ygxx57TJJ04sQJrVu3rsR1BgAArsfUoc1RVqtVoaGhkqSGDRsqJCTEbrmQkBA1aNBAkhQaGiqr1Zpr/bFjx3TkyBFJUq9eveTr62t3P/379zeW169f77TzAAAA5Z9Lh7aoqCidPXtWktS2bdsCy7Zr107K7pwQFRWVa53ttmjOcvZUr15d9evXlyTt2rXLoboDAADX4tKh7fDhw8Zyw4YNCyybc/3Ro0dzrbO1shVnP2fOnFFycnKx6wwAAFyTS4e26OhoY7lWrVoFls25/syZM/nup2bNmgXup3bt2lL2rdmc2wEAABTEpbsvXrp0yVj28/MrsGzO59SubCHLuZ8KFSqUeD+FyczMLFZ5AABQOLN8vrp0aLt8+bKx7OnpWWBZLy8vYzk1NbVU9lOYKwftBQAArsOlQ5u3t7exnJ6eXmDZtLQ0Y/nKYUGu3E/On4uzn8IEBwfL3d29WNsAAICCZWZmmqJhxKVDW85bmYXdqkxJSTGWr7yVmnM/ly5dKjC0FbSfwri7uxPaAABwUS7dESFn54LCOgXkXG/rTGBvPzExMQXux9aJwWKxFNr5AQAAwMalQ1vjxo2N5SuH8bhSzvVXDuvRqFGjYu+ndu3axW5pAwAArsulQ1vdunVVo0YNSdJff/1VYFnb+po1a6pu3bq51t1yyy3G8vbt2/Pdx7lz53T8+HFJUuvWrR2qOwAAcC0uHdosFosxwfzRo0cVFhZmt1xYWJjRQtatWzdZLJZc6xs0aGC0tq1ZsybXc2s5LVu2zFju3r27084DAACUfy4d2iRp5MiRxsP9U6dOzTMMR2pqqqZOnSpJ8vDw0MiRI+3uZ9SoUZKkhIQEvfPOO3nWnzx5Ul988YWUPXn8XXfd5fRzAQAA5Zepe4/u2LFDJ0+eNH6Oj483lk+cOKGlS5fmKj9gwIA8+2jQoIFGjx6tmTNnKiIiQsOGDdOjjz6qevXqKTIyUl9++aX27dsnSRo9erQxd+iV+vfvryVLlmjXrl2aN2+ezp8/r8GDB6ty5cras2ePPv30UyUlJcnNzU2TJk2Sh4epf/UAAOAqs1itVmtZV6KkXnzxxVy3HAvzzz//2H09KytLL7/8spYsWZLvtoMGDdLUqVPl5pZ/42RcXJzGjBmT71gvXl5eeuWVVzR48OAi11nZ48eEhYUpJCSEIT8AAHAys3zO0twjyc3NTdOnT1fPnj21YMEChYeHKz4+XlWqVFFwcLCGDh2qzp07F7qfwMBA/fDDD1q4cKFWrVqlI0eOKCUlRTVq1NCtt96qESNG6MYbb7wq5wQAAMoXU7e0uQqzfAMAAMCMzPI56/IdEQAAAMyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAGPsq7AtSItLU3Lly/XmjVr9M8//yghIUGenp6qUaOGWrdurcGDB6t169aF7mfjxo1auHChwsPDFRcXp8DAQAUHB2vIkCHq3LnzVTkXAABQ/lisVqu1rCtR1k6dOqXHHntMhw4dKrDc8OHDNWnSJFksljzrsrKyNHnyZC1evDjf7QcPHqwpU6bIza14DZyZmZkKCwtTSEiI3N3di7UtAAAomFk+Z12+pS09PT1XYGvSpIkefvhhNWjQQJcuXdLOnTv1zTffKDk5Wd9++61q1KihMWPG5NnPBx98YAS25s2b65FHHlG9evUUGRmpr776Svv27dOiRYsUGBioCRMmXPXzBAAA5ubyLW1r1qzRM888I0m6+eabNW/evDwpOyIiQvfff7/S09NVqVIlbdmyRR4e/z/vHjt2TPfcc48yMjLUsmVLzZs3Tz4+Psb6lJQUPfTQQ4qIiJCHh4dWr16tG264och1NMs3AAAAzMgsn7Mu3xFh9+7dxvKYMWPs/mO1bNlSd955pyTpwoULOnLkSK71c+bMUUZGhiRp8uTJuQKbJPn6+mry5MmSpIyMDM2ePbtUzgUAAJRfLh/a0tPTjeV69erlWy7nupzbWK1WhYaGSpIaNmyokJAQu9uHhISoQYMGkqTQ0FC5eAMnAAAoJpcPbbYgJUmRkZH5lrOts1gsql+/vvF6VFSUzp49K0lq27Ztgcdq166dJCkmJkZRUVEO1x0AALgOlw9td999typWrChJ+vLLL5WZmZmnzL59+/Tbb79Jku655x6jvCQdPnzYWG7YsGGBx8q5/ujRo06pPwAAcA0uH9oCAwP19ttvy9fXV7t27dKgQYP0448/KiwsTH/++ac++eQTPfTQQ0pPT1eLFi304osv5to+OjraWK5Vq1aBx8q5/syZM6VwNgAAoLxy+SE/JKlbt25asmSJvvnmGy1evFj/+c9/cq2vVq2annnmGQ0ZMkS+vr651l26dMlY9vPzK/A4ObdNTk4udj3ttQICAADHmOXzldCWYzaE/DoInD9/XitWrFDdunXVrVu3XOsuX75sLHt6ehZ4HC8vL2M5NTW12PUMDw8v9jYAAKB8cPnQlpycrEcffVQ7duyQu7u7HnnkEQ0YMED16tVTWlqa/v77b/3vf//Tzp079eSTT+o///mPHn74YWN7b29vYzlnr1J70tLSjOUrhwUpiuDg4Gt6/BgAAMwoMzPTFA0jDoW2tLS0XK1HZvTxxx9rx44dkqQ33nhD/fv3N9Z5eXmpY8eOat++vUaNGqVt27bp7bff1q233qqmTZtKkipUqGCUL+yWZ0pKirFc2K1Ue9zd3QltAAC4KIc6Itxxxx2aNm2a9u/f77waXUVWq1VLly6VJNWvXz9XYMvJw8PDmDUhKyvL2EZXdC7I2SnBnpzra9eu7XD9AQCA63AotCUmJmrevHkaMGCA+vfvr3nz5ikxMdF5tStl58+fV0JCgpQ9X2hBWrZsaSznHK6jcePGdl+3J+f6woYHAQAAyMmh0HbXXXfJ3d1dVqtV+/fv17Rp03THHXdowoQJ+uOPP5xXy1KS81ZjYT1Hcj6vlnPe0bp166pGjRqSpL/++qvAfdjW16xZU3Xr1i1xvQEAgOtxKLR9/PHH2rRpkyZOnKimTZvKarUqLS1NP//8sx599FF16dJF//3vfwucaaAsBQQEGAPl7t6925g/1J6cgSxn4LJYLEaP0qNHjyosLMzu9mFhYUZLW7du3WSxWJx2HgAAoPxzeHDdKlWqaOTIkfrxxx+1dOlSPfjgg6pUqZKsVqvOnDmjzz77TD179tSIESO0YsWKXENklDU3NzdjIvizZ8/q888/t1suMTFR7777rvGzbRubkSNHGq12U6dOzTOcR2pqqqZOnSplt9KNHDnS6ecCAADKN4u1FGYuT0tLU2hoqJYuXarNmzcrKyvLaFmqWLGi+vTpo4EDB6pVq1bOPnSxHTlyRAMHDjR6dnbp0kX9+/dXvXr1dPnyZf3999+aM2eOTp8+LUm69dZbNXv27Dz7ee+99zRz5kwp+/m4Rx99VPXq1VNkZKS+/PJL7du3T5L02GOPacKECcWqY2ZmpsLCwhQSEkLvUQAAnMwsn7OlEtpyiomJ0bJly7Rs2TKdOHHi/w6aHeAaN26sgQMHqn///qpcuXJpVqNAf/75pyZMmKD4+PgCy3Xo0EH//e9/7dY1KytLL7/8spYsWZLv9oMGDdLUqVPl5la8Bk6zXEwAAJiRWT5nSz20KXt8sp9++kkffvihYmNjpezhNpQd4Ly9vTV06FCNGzdO/v7+pV0du+Lj47V48WL9/vvvOnz4sC5evCh3d3dVq1ZNwcHBuueee4r0LNrGjRu1YMEChYeHKz4+XlWqVFFwcLCGDh2qzp07l6huZrmYAAAwI7N8zpZqaNuxY4eWLFmitWvXGrcfrVarKleurJ49e+rQoUPavXv3/1XEYlHt2rX1/fffFzrxuqsxy8UEAIAZmeVz1unTWEVHRxu3Q229Rq1WqywWi9q1a6fBgwerR48exkwKx44d08yZM/Xjjz/qzJkz+vDDDzVjxgxnVwsAAMDUnBLa0tLStG7dOi1dulRbt25VVlaWcfuzevXqGjBggAYNGqR69erl2bZBgwZ68803VadOHX3yySfasmWLM6oEAABQrjgU2vbs2aOlS5dq9erVunjxopTdqubu7q5OnTpp8ODBuvPOO4v04P1dd92lTz75ROfOnXOkSgAAAOWSQ6FtyJAhslgsRqva9ddfb/QGtc0SUFS+vr5Sjg4KAAAA+P8cvj3q6empu+66S4MHD1aHDh1KvJ+aNWtq7ty5jlYHAACgXHIotE2aNEl9+/Z1yhhr3t7eateuncP7AQAAKI8cCm3Dhw93Xk0AAACQL4fmHu3WrZu6d+9uzHRQFKdPnza2AwAAQNE41NJ26tQpWSwWpaenF3mbjIwMYzsAAAAUjUMtbQAAALg6rnpos43n5uPjc7UPDQAAYFpXPbStWLFCklSnTp2rfWgAAADTKtYzbSNGjLD7+sSJE43BcfOTlpamqKgoxcbGymKxqGPHjsWrKQAAgAsrVmjbvn17rhkQlD2DQXh4eLEOWq9ePT322GPF2gYAAMCVFSu0tW3bNtfPf/31lywWi1q0aFFgS5vFYpG3t7eqV6+um2++WXfffbf8/PxKXmsAAAAXU6zQ9u233+b6uWnTppKkGTNmqHHjxs6tGQAAAAwOjdPWr18/WSwWVapUyXk1AgAAQB4OhbYZM2Y4ryYAAADIF4PrAgAAmAChDQAAwASKdHu0WbNmUnYv0H379uV5vSSu3BcAAADyV6TQlnNctqK8DgAAAOcqUmgbN25csV4HAACAc1msNJdd8zIzMxUWFqaQkBC5u7uXdXUAAChXzPI5S0cEAAAAEyC0AQAAmIBDoW3ZsmUl2u7ChQuaMGGCI4cGAABwKQ6FtokTJ+rZZ59VYmJikbfZunWr+vbtq59//tmRQwMAALgUh2+Prl27Vvfdd5+2bNlSYLn09HTNmDFDo0aNUnR0tCwWi6OHBgAAcBkOhbaRI0dKkqKjozV69Gi99dZbSk9Pz1Pu4MGDGjhwoObMmaOsrCxVr15dM2fOdOTQAAAALsXh26OzZs1SjRo1lJWVpdmzZ2vQoEE6dOiQUeabb77R4MGDdejQIVmtVt11111asWKFbr/9dmfUHwAAwCU4ZZy2xMREvfzyy1q3bp0kydvbW08++aT+/PNPbdu2TVarVX5+fnrppZc0aNAgZ9TbpZhl/BgAAMzILJ+zTh1cd/HixZo+fbqSk5ONZ9asVqtuuukmvfPOO7r++uuddSiXYpaLCQAAMzLL56xTx2nr06eP2rdvb/xstVrl7++v6dOnE9gAAAAc4LTQtmfPHvXv31+//fabJMnX11eSlJSUpEGDBmnRokXOOhQAAIDLcTi0Wa1W/e9//9MDDzygEydOyGq1avDgwdq0aZNeeukleXl5KSUlRa+88orGjRun+Ph459QcAADAhTgU2qKiovTAAw/ok08+UUZGhgICAvTJJ59o6tSpqlChgkaMGKElS5aoadOmslqtCg0N1b333qtNmzY57wwAAABcgEOhrW/fvgoLC5PValXHjh21YsUKde/ePVeZxo0ba9GiRRo1apQsFovOnz+vMWPGaOrUqY7WHQAAwGU4FNqSk5Pl6emZa7w2ezw9PfXCCy/om2++0XXXXSer1arvv//ekUMDAAC4FIdCW1BQkJYsWWLMjFCY9u3ba/ny5br77rsdOSwAAIDL8XBk48WLF8vLy6tY2/j7++u9995Tly5dHDk0AACAS3Gopa24gS2ne+65x5FDAwAAuBSHWtqudPLkSe3evVvnz59XSkqKHnjgAQUGBjrzEAAAAC7JKaFt7969mj59unbt2pXr9V69euUKbfPmzdMnn3wif39//fTTT/L09HTG4QEAAMo9hwfX/fXXXzVs2DDt2rVLVqvV+M+e++67T6mpqYqMjDRmTgAAAEDhHAptZ8+e1YQJE5SWlqbGjRvryy+/zNPallPFihXVtWtXSdLvv//uyKEBAABcikOhbfbs2UpJSdF1112nefPm6Y477pCfn1+B27Rv315Wq1V79+515NAAAAAuxaHQtmnTJlksFo0aNUqVKlUq0jYNGzaUsqfAAgAAQNE4FNpOnz4tSWrVqlWRt6lYsaKUPZsCAAAAisah0JaZmSlJysrKKvI2Fy9elKRCb6MCAADg/3MotFWrVk2SFBkZWeRt9uzZI0mqXbu2I4cGAABwKQ6FtjZt2shqtWrNmjVFKp+WlqYFCxbIYrGoXbt2jhwaAADApTgU2vr37y9J2rBhgzZv3lxg2bS0NP3nP//RyZMnZbFYNGTIEEcODQAA4FIcmhGhffv26tOnj1avXq2xY8dqxIgR6tmzp7H+1KlTunDhgnbt2qWFCxcqMjJSFotF999/v2688UZn1B8AAMAlWKz5TV9QRGlpaXrqqae0ceNGWSyWfMvZDtOjRw998MEHcnd3d+SwLiUzM1NhYWEKCQnh9wYAgJOZ5XPW4WmsvLy89MUXX2jKlCmqV69erqmscv5Xq1Ytvfrqq/rvf/97Tf9CAAAArkVOmTBekoYMGaIhQ4bo8OHDioiIUGxsrDIzM1WlShU1a9ZMLVq0KLAlDgAAAPlzWmizady4sRo3buzs3QIAALg0h2+PAgAAoPQR2gAAAEygSLdHf/zxx1I5eL9+/UplvwAAAOVNkULbiy++6PROBBaLhdAGAABQREXuiODgcG4AAABwQJFCW2hoaOnXBAAAAPkqUmirU6dO6dcEAAAA+aL3KAAAgAkQ2gAAAEzAqTMi7N27V3/++acOHjyoxMRESVLlypV144036rbbblPLli2deTgAAACX4ZTQtnfvXr3++usKDw/Pt8wHH3ygli1b6pVXXlFwcLAzDgsAAOAyHL49umbNGt1///0KDw+X1WqV1WqVh4eHqlatqqpVq8rDw8N4PTw8XMOGDdPPP//snNoDAAC4CIda2o4ePaoXXnhB6enp8vDw0ODBgzVw4EA1a9ZM7u7ukqTMzEwdOHBAixcv1qJFi5SRkaH//Oc/CgoKUqNGjZx1HgAAAOWaQ6Htyy+/VFpamry9vTVz5ky1b98+Txl3d3e1aNFCLVq0UO/evfXoo48qLS1NX331ld58801HDg8AAOAyHLo9umXLFlksFo0cOdJuYLtSu3btNHLkSFmtVm3ZssWRQwMAALgUh0JbXFycJKlTp05F3qZz5865tgUAAEDhHAptgYGBkiRvb+8ib+Pl5SVJqlKliiOHBgAAcCkOhbbWrVtLUoFDfVxpz549kqRbbrnFkUMDAAC4FIdC27/+9S+5u7vriy++KNLtztjYWM2cOVMeHh7617/+5cihAQAAXIpDoa1Vq1Z6/fXXFRsbq8GDB2v9+vXKysrKUy4rK0vr16/X0KFDFRcXp9dee02tWrVy5NAAAAAuxaEhPyZOnChJaty4sQ4cOKCnnnpKlSpVUvPmzRUYGCiLxaLY2Fjt37/fmNaqadOm2rlzp3bu3Gl3nxaLRdOnT3ekWgAAAOWOxWq1Wku6cdOmTWWxWIyfbbvK+VpBr1/JarXKYrFo//79Ja1SuZSZmamwsDCFhIQYgxYDAADnMMvnrEMtbdddd53zagIAAIB8ORTaNmzY4LyaAAAAIF8OTxgPAACA0udQS9snn3wiSbrpppt0xx13OKtOAAAAuILDoc1isRjhDQAAAKXDodujAQEBEh0SAAAASp1Doe2GG26QJJ07d85Z9QEAAIAdDt0e7d27t/7++2/9/PPP6tSpk/NqVYZOnz6txYsX67ffftPp06d16dIlBQYGqk6dOmrfvr169+6toKCgfLffuHGjFi5cqPDwcMXFxSkwMFDBwcEaMmSIOnfufFXPBQAAlB8OhbYHHnhAP/74o5YvX662bdtqwIABzqtZGfj222/1/vvvKzk5Odfr0dHRio6O1s6dO5WUlKRJkybl2TYrK0uTJ0/W4sWLc70eExOjmJgYrV+/XoMHD9aUKVPk5kanXQAAUDwOhbbz589r2rRpmjRpkiZNmqRVq1bpnnvuUZMmTVSpUqVCRxW+lp6F+/TTT/XRRx9JkurXr68hQ4YoODhY/v7+SkhI0L59+7Ru3bp8A9cHH3xgBLbmzZvrkUceUb169RQZGamvvvpK+/bt06JFixQYGKgJEyZc1XMDAADm57RprGxTUBX5wBaL9u3bV9JDO9WWLVv0r3/9S5LUr18/TZs2TZ6ennbLpqWlycvLK9drx44d0z333KOMjAy1bNlS8+bNk4+Pj7E+JSVFDz30kCIiIuTh4aHVq1cbzwMWhVmm1wAAwIzM8jnr8H06q9VqzC1qWy7qf9eCrKwsvfbaa1J2CH3jjTfyDWyS8gQ2SZozZ44yMjIkSZMnT84V2CTJ19dXkydPliRlZGRo9uzZTj4LAABQ3jl0e/TNN990Xk3KyB9//KHjx49Lkh599FF5eBTvV2K1WhUaGipJatiwoUJCQuyWCwkJUYMGDXTs2DGFhobqlVdeKVbLJAAAcG0Ohbb+/fs7ryZlZM2aNVL27do777zTeD0hIUEJCQkKCAgwxqOzJyoqSmfPnpUktW3btsBjtWvXTseOHVNMTIyioqJUr149p50HAAAo3xwKbeXB33//LUmqU6eOKlasqJUrV2rmzJk6ePCgUcbWMWH48OF5bo8ePnzYWG7YsGGBx8q5/ujRo4Q2AABQZC499kRWVpaOHj0qSapSpYqmTZumf//737kCmyQdP35cb7/9tkaMGKELFy7kWhcdHW0s16pVq8Dj5Vx/5swZJ50FAABwBU5racvKytK2bdu0e/dunT9/XikpKRo/frxq1KhhlElLS1NmZqbc3d3tPtB/tV28eFFZWVmSpIMHDyo8PFzVq1fXCy+8oM6dO8vb21vh4eF69913FRYWpt27d+ull17KNdfqpUuXjGU/P78Cj+fr62ssXzkWXFFkZmYWexsAAFAws3y+OiW0/frrr5o2bZpOnz6d6/XRo0fnCm2LFi3StGnT5Ofnp02bNhUackpbSkqKsXz58mX5+vpq7ty5uW5jtm3bVnPmzNHQoUN14MABrVu3Tn///bduuukmYzubgnqd6oqep6mpqcWub3h4eLG3AQAA5YPDoW3hwoV69dVXjSE8qlSpovj4eLs9IwcPHqyPPvpIFy9e1Lp163Tfffc5eniHXNnaN2jQILvPpfn4+Gj8+PF67LHHJEmrV682Qpu3t7dRLj09vcDjpaWl5dpncQUHB1/T48cAAGBGmZmZpmgYcSi0HT9+XFOmTJEkdejQQZMnT1ajRo3UtGlTu+W9vLzUo0cPLV68WJs3by7z0FaxYsVcP99+++35lr311lvl4eGhjIyMXP+wFSpUMJYLu+WZs2WvJK2M7u7uhDYAAFyUQx0RZs+erYyMDDVu3FgzZ85Uo0aNCt2mTZs2kqT9+/c7cmin8PLyUmBgoPFzQR0JvL29VaVKFUlSXFyc3W1ydkqwJ+f62rVrl7jeAADA9TgU2rZu3SqLxaKRI0cWuWPB9ddfL11DvScbN25sLNs6JeTH9qBizgF4c25v64man5zrCxseBAAAICeHQltMTIyUPf1TUdluC5bkQfzSkHNA3MjIyHzLJSUlKT4+XpJUs2ZN4/W6desanS3++uuvAo9lW1+zZk3VrVvX4boDAADX4ZRx2ooTwGzB58rnycpKjx49jOV169blW27dunVGZ4tbbrnFeN1isahbt25SdktaWFiY3e3DwsKMlrZu3boxhRUAACgWh0KbrcWpoBaqK+3cuVOSrpnZAJo2bapOnTpJkn766Sdt2bIlT5lz587pww8/lLKH9Rg4cGCu9SNHjjQ6CEydOjVPiE1NTdXUqVOl7FurI0eOLLXzAQAA5ZNDoa1du3ayWq1atmxZkcpfvHhRP/zwgywWizp06ODIoZ3qpZdeUqVKlZSVlaXHHntM7733nnbs2KHw8HDNmzdPgwYNMjoRPPPMM7luj0pSgwYNNHr0aElSRESEhg0bptWrVys8PFyrV6/WsGHDFBERIWWPXVe/fv0yOEsAAGBmFqvtnl8J7Nu3z2h1euONNzRgwAApu/XKYrFo5cqVxoP68fHxevrpp/XXX3/Jw8NDa9euVZ06dZx1Hg7bsWOHnnnmGZ0/f97ueovForFjx+rZZ5+1uz4rK0svv/yylixZku8xBg0apKlTp8rNrXhZOTMzU2FhYQoJCWHIDwAAnMwsn7MOjdPWvHlzjRgxQnPmzNGkSZP0+++/53pGbPfu3dq/f7927dqlVatWKSkpSRaLRU888cQ1FdiUPRTJqlWr9N1332n9+vWKiopSenq6qlevrnbt2mn48OFq3rx5vtu7ublp+vTp6tmzpxYsWKDw8HDFx8erSpUqCg4O1tChQ9W5c+erek4AAKD8cKilTZKsVqumTJmi+fPnF/hwve0wI0eO1MSJEx05pMsxyzcAAADMyCyfsw73HrVYLHr11Vc1a9YstWvXThaLRVarNdd/khQSEqIvvviCwAYAAFACTpkwXpI6duyojh07KikpSfv371dsbKyysrIUEBCgpk2b5pp5AAAAAMXjUGhLSEgwgpnt4fqKFSvmGrAWAAAAjit2aDtw4IA+/fRTbd682Zgg3dPTU23atNHo0aPVsWPH0qgnAACASyvWM23r16/XkCFDtG7dOl26dMl4Zi0tLU1btmzRI488opkzZ5ZebQEAAFxUkUPb2bNn9eKLLyotLU1Wq1U+Pj5q0aKFQkJCVKlSJSPAffjhh9q1a1fp1hoAAMDFFPn26MKFC41x1v71r3/pqaeeMiZ/z8zM1A8//KDp06crKytLs2fPVuvWrUuz3gAAAC6lyC1tf/75pywWi7p06aL//Oc/RmCTJHd3dz344IN65JFHZLVa7c7fCQAAgJIrcmg7evSoJBlTVdljm9IqKSkp3+mgAAAAUHxFDm1JSUmSpLp16+ZbJufUVBcvXnS0bgAAAMhW5NCWkZEhSfLwyP8xuJxTP2RmZjpaNwAAAGRzeBorAAAAlL5ih7aCJoUvSTkAAAAUrtgzIowaNarAW6RFLWexWLR+/friHh4AAMAlFTu0xcTEFLje1sJW1HIAAAAoXJFD23XXXVe6NQEAAEC+ihzaNmzYULo1AQAAQL7oPQoAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYgEOhrWvXrurevbtOnDhR5G1Onz6tbt26qXv37o4cGgAAwKUUecJ4e06fPi2LxaL09PQib5ORkaFTp07JYrE4cmgAAACXwu1RAAAAE7jqoe3ixYuSJB8fn6t9aAAAANO66qFtxYoVkqQ6depc7UMDAACYVrGeaRsxYoTd1ydOnChfX98Ct01LS1NUVJRiY2NlsVjUsWPH4tUUAADAhRUrtG3fvl0Wi0VWq9V4zWq1Kjw8vFgHrVevnh577LFibQMAAODKihXa2rZtm+vnv/76SxaLRS1atCiwpc1iscjb21vVq1fXzTffrLvvvlt+fn4lrzUAAICLKVZo+/bbb3P93LRpU0nSjBkz1LhxY+fWDAAAAAaHxmnr16+fLBaLKlWq5LwaAQAAIA+HQtuMGTOcVxMAAADky6HQVlQnT55UfHy86tSpo2rVql2NQwIAAJQrDoW22NhYrVmzRpLUt29f+fv751p/4sQJjR8/Xvv375eyOyR069ZN06ZNU+XKlR05NAAAgEtxaHDdX375RVOnTtXcuXPzBLa0tDQ9+uij2r9/v6xWq6xWq7KysrR+/Xo98cQTjtYbAADApTgU2jZv3iyLxaK77rorz7qlS5fq5MmTkqSuXbtq0qRJ6tKli6xWq3bt2qXVq1c7cmgAAACX4lBoO3bsmCQpJCQkz7pVq1ZJkjp06KBPP/1Uw4cP12effabbbrtNVqtVP/30kyOHBgAAcCkOhba4uDhJUs2aNXO9npqaqrCwMFksFg0ZMiTXuoEDB0qS9u3b58ihAQAAXIpDoe3ixYv/txO33LsJCwtTRkaGLBaLbrvttlzr6tatK2V3YgAAAEDROBTabFNRnT9/Ptfr27dvlyQ1atQoTy9RD4//67Dq7u7uyKEBAABcikOhrWHDhpKkTZs25Xr9l19+kcViUbt27fJsYwt4jNcGAABQdA6N09a5c2eFhYVpwYIFatiwodq0aaNly5bp8OHD+fYq3bt3r2TnOTgAAADkz6HQ9tBDD+n777/XuXPnNHXq1FzrQkJC1KFDhzzb/Prrr7JYLAoODnbk0AAAAC7Foduj/v7+mj17tpo3b24MoGu1WtWmTRt9+OGHecofOHBA4eHhkpSngwIAAADy5/Dco40aNdLSpUsVGRmp8+fPq3r16kYPUXvefPNNKXv8NgAAABSN0yaMr1evnurVq1dgmaZNm6pp06bOOiQAAIDLcOj2KAAAAK4Op7W0ZWVladu2bdq9e7fOnz+vlJQUjR8/XjVq1DDKpKWlKTMzU+7u7vLy8nLWoQEAAMo9p4S2X3/9VdOmTdPp06dzvT569OhcoW3RokWaNm2a/Pz8tGnTJmNwXgAAABTM4dujCxcu1BNPPKFTp07JarUqICBAVqvVbtnBgwfL399fycnJWrdunaOHBgAAcBkOhbbjx49rypQpUnZv0J9++klbtmzJt7yXl5d69Oghq9WqzZs3O3JoAAAAl+JQaJs9e7YyMjLUuHFjzZw5U40aNSp0mzZt2kiS9u/f78ihAQAAXIpDoW3r1q2yWCwaOXJkkTsWXH/99ZKkM2fOOHJoAAAAl+JQaIuJiZGyx18rKlvng9TUVEcODQAA4FKcMk5bcQJYfHy8JKlixYrOODQAAIBLcCi01axZU5IUGRlZ5G127twpZc+gAAAAgKJxKLS1a9dOVqtVy5YtK1L5ixcv6ocffpDFYmHuUQAAgGIocmhr2rSpmjdvrsOHDxuv3X///bJYLPrrr7+0dOnSArePj4/XE088ofPnz8vd3V3333+/YzUHAABwIcWaEeHKQXObN2+uESNGaM6cOZo0aZJ+//139ejRw1i/e/du7d+/X7t27dKqVauUlJQki8WiJ554QnXq1HHeWQAAAJRzDk9j9eKLLyotLU3z58/X2rVrtXbtWlksFknSK6+8YpSzBb6RI0fqiSeecPSwAAAALsXh3qMWi0WvvvqqZs2apXbt2slischqteb6T5JCQkL0xRdfaOLEic6oNwAAgEtxyoTxktSxY0d17NhRSUlJ2r9/v2JjY5WVlaWAgAA1bdpUgYGBzjoUAACAy3FaaLOpWLGi2rZt6+zdAgAAuDSnDK4LAACA0lXslraJEyfK19fX4QNbLBbNmTPH4f0AAAC4gmKHtoiICIcParVajR6mAAAAKFyxQ9uVY7UBAACg9BU7tK1atUqNGzcundoAAADALjoiAAAAmAChDQAAwAQIbQAAACZAaAMAADABQhsAAIAJENoAAABMoMhDfoSGhkqSatasWZr1AQAAgB1FDm116tQp3ZoAAAAgX9weBQAAMAFCGwAAgAkQ2gAAAEyA0AYAAGACxZ4w3pW88847+uqrr4yf586dq/bt2xe4zcaNG7Vw4UKFh4crLi5OgYGBCg4O1pAhQ9S5c+erUGsAAFAeEdrysX//fs2ePbvI5bOysjR58mQtXrw41+sxMTGKiYnR+vXrNXjwYE2ZMkVubjRwAgCA4iG02WELYBkZGapatapiY2ML3eaDDz4wAlvz5s31yCOPqF69eoqMjNRXX32lffv2adGiRQoMDNSECROuwlkAAIDyhCYfO+bOnavw8HA1bNhQgwYNKrT8sWPH9PXXX0uSWrZsqfnz5+vuu+9Wq1atdPfdd+v7779Xy5YtJUmzZs3SiRMnSv0cAABA+UJou8Lp06f10UcfSZJef/11eXp6FrrNnDlzlJGRIUmaPHmyfHx8cq339fXV5MmTJUkZGRnFuu0KAAAgQlteU6ZMUXJysvr376927doVWt5qtRpTfDVs2FAhISF2y4WEhKhBgwZS9pRgVqvVyTUHAADlGaEth9WrV+vXX39VQECAXnjhhSJtExUVpbNnz0qS2rZtW2BZWwiMiYlRVFSUE2oMAABcBaEt24ULFzR9+nRJ0r///W8FBgYWabvDhw8byw0bNiywbM71R48eLXFdAQCA6yG0ZXvnnXd07tw5tW7dukidD2yio6ON5Vq1ahVYNuf6M2fOlLCmAADAFTHkh6QdO3Zo0aJF8vDw0Ouvvy6LxVLkbS9dumQs+/n5FVjW19fXWE5OTi52PTMzM4u9DQAAKJhZPl9dPrSlpaVp8uTJslqtGjlypIKCgoq1/eXLl43lwnqaenl5GcupqanFrmt4eHixtwEAAOWDy4e2L774QkePHtV1112ncePGFXt7b29vYzk9Pb3AsmlpacbylcOCFEVwcLDc3d2LvR0AAMhfZmamKRpGXDq0HTlyRF988YUk6eWXXy709qY9FSpUMJYLu+WZkpJiLJfkWO7u7oQ2AABclEuHtjlz5ig9PV316tVTamqqfvrppzxlDh06ZCxv3bpV58+flyR16dJFfn5+uToX5OyUYE/O9bVr13bSWQAAAFfg0qHNdrsyMjKySPOBfvrpp8ZyaGio/Pz81LhxY+O1wobxyLm+sOFBAAAAcmLIDwfVrVtXNWrUkCT99ddfBZa1ra9Zs6bq1q17VeoHAADKB5cObTNmzNA///xT4H85OyfMnTvXeN0WuiwWi7p16yZlt6SFhYXZPVZYWJjR0tatW7diDSsCAADg0qHNWUaOHGl0EJg6dWqe4TxSU1M1depUSZKHh4dGjhxZJvUEAADmRWhzggYNGmj06NGSpIiICA0bNkyrV69WeHi4Vq9erWHDhikiIkKSNHr0aNWvX7+MawwAAMzGpTsiONP48eMVGxurJUuWaN++fRo/fnyeMoMGDdKzzz5bJvUDAADmRmhzEjc3N02fPl09e/bUggULFB4ervj4eFWpUkXBwcEaOnSoOnfuXNbVBAAAJmWxWq3Wsq4ECpaZmamwsDCFhIQwuC4AAE5mls9ZnmkDAAAwAUIbAACACRDaAAAATIDQBgAAYAKENgAAABMgtAEAAJgAoQ0AAMAECG0AAAAmQGgDAAAwAUIbAACACRDaAAAATIDQBgAAYAKENgAAABMgtAEAAJgAoQ0AAMAECG0AAAAmQGgDAAAwAUIbAACACRDaAAAATIDQBgAAYAKENgAAABMgtAEAAJgAoQ0AAMAECG0AAAAmQGgDAAAwAUIbAACACRDaAAAATIDQBgAAYAKENgAAABMgtAEAAJgAoQ0AAMAECG0AAAAmQGgDAAAwAUIbAACACRDaAAAATIDQBgAAYAKENgAAABMgtAEAAJgAoQ0AAMAECG0AAAAmQGgDAAAwAUIbAACACRDaAAAATIDQBgAAYAKENgAAABMgtAEAAJgAoQ0AAMAECG0AAAAmQGgDAAAwAUIbAACACRDaAAAATIDQBgAAYAKENgAAABMgtAEAAJgAoQ0AAMAECG0AAAAmQGgDAAAwAUIbAACACRDaAAAATIDQBgAAYAKENgAAABMgtAEAAJgAoQ0AAMAECG0AAAAmQGgDAAAwAUIbAACACRDaAAAATIDQBgAAYAKENgAAABMgtAEAAJgAoQ0AAMAECG0AAAAmQGgDAAAwAUKbi8iyZpV1FcqMNYtzBwCYn0dZVwBXh5vFTfP+3qSYpMSyrspV1bT6deoT1FqJ6+crM/5sWVfnqnKvUkOVuw8r62oAAJyE0OZCYpISdepCXFlX46qqUaGSJCkz/qwyzp8q6+oAAFBi3B4FAAAwAUIbAACACRDaAAAATIDQBgAAYAKENgAAABMgtAEAAJgAoQ0AAMAECG0AAAAmQGgDAAAwAUIbAACACRDaAKAUWLOyyroKZcaVzx0oTS4/92h4eLg2btyoXbt26fDhw4qLi5Onp6dq1Kih1q1ba+DAgWrTpk2R97dx40YtXLhQ4eHhiouLU2BgoIKDgzVkyBB17ty5VM8FuNZkWbPkZnHN74YWNzclrp+vzPizZV2Vq8q9Sg1V7j6srKsBlEsuHdoefPBB7dixI8/r6enpOn78uI4fP66lS5eqX79+mjp1qry8vPLdV1ZWliZPnqzFixfnej0mJkYxMTFav369Bg8erClTpsjNzTU/xOB63Cxumvf3JsUkJZZ1Va6qptWvU5+g1sqMP6uM86fKujoAygmXDm1nz/7fN+AaNWqoV69eatOmjWrXrq2srCyFhYXp66+/VkxMjH788UdlZGTovffey3dfH3zwgRHYmjdvrkceeUT16tVTZGSkvvrqK+3bt0+LFi1SYGCgJkyYcNXOEShrMUmJOnUhrqyrcVXVqFCprKsAoBxy6dDWsGFDjR8/Xj179pS7u3uudSEhIerbt6+GDRum48ePa9WqVbr//vvVtm3bPPs5duyYvv76a0lSy5YtNW/ePPn4+EiSWrVqpa5du+qhhx5SRESEZs2apYEDB+qGG264SmcJAADKA5e+T/fFF1+oT58+eQKbTWBgoF588UXj57Vr19otN2fOHGVkZEiSJk+ebAQ2G19fX02ePFmSlJGRodmzZzvxLAAAgCtw6dBWFO3btzeWT548mWe91WpVaGiolN1yFxISYnc/ISEhatCggSQpNDRUVqu11OoMAADKH0JbIdLS0oxlex0IoqKijGfj7N06zaldu3ZSdueEqKgop9cVAACUX4S2Qvz111/GcqNGjfKsP3z4sLHcsGHDAveVc/3Ro0edVkcAAFD+EdoKkJWVpZkzZxo/9+7dO0+Z6OhoY7lWrVoF7i/n+jNnzjitngAAoPxz6d6jhZk9e7b27NkjSerRo4datmyZp8ylS5eMZT8/vwL35+vraywnJycXuz6ZmZnF3sYmv84WKP8cuW4cxXXnusryugOKyyzXK6EtH9u3bzfGZatatapee+01u+UuX75sLHt6eha4z5yD86ampha7TuHh4cXeRtlhsXnz5iXaFub3zz//KCUl5aofl+vOtZXVdQeUZ4Q2Ow4dOqRx48YpIyND3t7e+uijj1S1alW7Zb29vY3l9PT0Avebs1PDlcOCFEVwcDAtFyi2Jk2alHUV4IK47mAmmZmZJW4YuZoIbVeIjIzUqFGjlJiYKHd3d73//vsF9gqtUKGCsVzYLc+c3zoLu5Vqj7u7O6ENxcY1g7LAdQc4Hx0RcoiJidHDDz+ss2fPymKxaPr06erevXuB2+TsXJCzU4I9OdfXrl3bCTUGAACugtCWLS4uTqNGjVJkZKSUPbNBv379Ct2ucePGxnJhw3jkXF/Y8CAAAAA5EdokXbx4UY888ogx5tpzzz2nBx98sEjb1q1bVzVq1JCuGNPNHtv6mjVrqm7dug7XGwAAuA6XD20pKSkaM2aM9u7dK0kaO3asxowZU+TtLRaLunXrJmW3pIWFhdktFxYWZrS0devWTRaLxSn1BwAArsGlQ1taWprGjRunXbt2SZJGjBih8ePHF3s/I0eONB66nTp1ap7hPFJTUzV16lRJkoeHh0aOHOmU+gMAANfh0r1Hn3vuOf3xxx+SpA4dOmjQoEE6ePBgvuU9PT2NSd9zatCggUaPHq2ZM2cqIiJCw4YN06OPPqp69eopMjJSX375pfbt2ydJGj16tOrXr1+KZwUAAMojlw5tv/zyi7G8detW9e3bt8DyderU0YYNG+yuGz9+vGJjY7VkyRLt27fPbovdoEGD9Oyzzzqh5gAAwNW4dGhzJjc3N02fPl09e/bUggULFB4ervj4eFWpUkXBwcEaOnSoOnfuXNbVBACUY9asLFncXPPJJ1c4d5cObf/884/T99m5c2fCGQCgTFjc3JS4fr4y48+WdVWuKvcqNVS5+7Cyrkapc+nQBgAof7KsWXKzlO8Wl4Jkxp9VxvlTZV0NlAJCGwCgXHGzuGne35sUk5RY1lW5qppWv059glqXdTVQightAIByJyYpUacuxJV1Na6qGhUqlXUVUMpct/0YAADARAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAkQ2gAAAEyA0AYAAGAChDYAAAATILQBAACYAKENAADABAhtAAAAJkBoAwAAMAFCGwAAgAl4lHUFyqNTp07p22+/1W+//abo6Gh5eXmpXr166t27tx588EH5+vqWdRUBAIDJENqcbMOGDXr++eeVlJRkvJaSkqLExERFRERo0aJFmjlzpm644YYyrScAADAXbo860b59+zR+/HglJSXJz89P48eP1w8//KDZs2dryJAhkqTjx49rzJgxuUIdAABAYWhpc6I33nhDqamp8vDw0Ndff62bb77ZWHfrrbfqhhtu0DvvvKPjx4/rm2++0VNPPVWm9QUAAOZBS5uT7NmzRzt27JAkDRw4MFdgsxk1apQaNWokSZo7d67S09Ovej0BAIA5EdqcZP369cbywIED7ZZxc3NTv379JEkXLlzQtm3brlr9AACAuRHanGTnzp2SJD8/P7Vo0SLfcm3btjWWd+3adVXqBgAAzI/Q5iRHjhyRJF1//fXy8Mj/UcGGDRvm2QYAAKAwhDYnuHz5suLj4yVJtWrVKrBs5cqV5efnJ0mKjo6+KvUDAADmR+9RJ7h06ZKxbAtkBfH19VVycrKSk5OLtH+r1SpJSktLk7u7e4nq6O7urtoVKstdlhJtb1ZVfSsoMzNTliq15GYp2e/OrCwB1ZWZmanMzMwyqwPXHdddWeC647orLtt2ts/baxWhzQkuX75sLHt6ehZa3svLS5KUmppapP1nZWVJ2ePAOSJIfgryKTxUlitJUlhYmFSl8f/952rCwsq6Blx3XHdlguuO664kbJ+31ypCmxN4e3sby0UZxiMtLU2S5OPjU6T9e3h4KDg4WG5ubrJYXOubIwAApc1qtSorK6vAZ9KvBdd27UyiQoUKxnJRbnmmpKRIRbyVquyhQmytcwAAwDXREcEJvL29FRAQIBWhc0FiYqIR7ArrtAAAAGBDaHOSxo3/7/mBkydPKiMjI99yR48eNZZtsyMAAAAUhtDmJLfccouUfXt07969+Zb766+/jOXWrVtflboBAADzI7Q5Sffu3Y3lJUuW2C2TlZWlH3/8UZJUqVIltW/f/qrVDwAAmBuhzUlatWqlNm3aSNmhbffu3XnKfP3118YsCCNGjCjS8CAAAACSZLFe6yPJmci+ffs0bNgwpaamys/PT2PHjlX79u2Vmpqq1atXa8GCBZKk+vXra8mSJapYsWJZVxkAAJgEoc3JNmzYoOeff15JSUl219evX18zZ87UDTfccNXrBgAAzIvQVgpOnTqluXPn6rffflNMTIw8PT11/fXXq1evXnrooYfk6+tb1lW8Jp0+fVqLFy/Wb7/9ptOnT+vSpUsKDAxUnTp11L59e/Xu3VtBQUFG+aioKHXr1q1Yx6hTp442bNiQ5/UmTZrk+vm7775T27ZtC93fww8/rD///NP4edy4cXrqqaeKVSc4T2xsrPbs2aM9e/YoPDxc4eHhSkhIkCT1799fM2bMKHQfWVlZOnr0aK79/PPPP8bA2XPnzi3W86gpKSn67rvvtGbNGkVGRiotLU21atXSnXfeqeHDh6tOnTr5brtt2zaNGDHC+NnPz09//vlnoe8hqamp6tixY64vj8WtN4omKSlJGzduVHh4uCIiIhQTE6O4uDhdvnxZ/v7+aty4sTp16qRBgwapSpUqdvdx6tQpbd68WXv27NGBAwcUGxuruLg4Wa1WValSRc2aNVOvXr109913F/hYzccff6xPPvmkSPXO73q48n3VYrEoNDS0wOvUpmfPnjp+/Ljx85tvvqkBAwYUqT4oGgbXLQV16tTRxIkTNXHixLKuiml8++23ev/99/MMThwdHa3o6Gjt3LlTSUlJmjRpkkPHadCgQZHKrVixotDQFhMTo61btzpUHzjXbbfd5vA+li9frhdffNEp9Tlx4oTGjBmT64NMko4dO6Zjx45p0aJFevfdd9WlS5ci7S85OVnr16/XvffeW2C50NDQfFv74Vx79uzRhAkT7K6Li4vT9u3btX37ds2aNUvvvPOO7rjjjjzlFi5cqM8//9zuPmzvgb/++qtmzZqlTz/9VPXq1XP6eeTHarVq5cqVGjt2bIHlwsLC8lzncD5CG8rcp59+qo8++kjKvn08ZMgQBQcHy9/fXwkJCdq3b5/WrVsnN7fc/WZq1qyplStXFrr/L774QqtWrZIk9evXr8Cy3t7eunz5stasWaPJkycXOBPFypUrlZWVZWyDa8t1112nhg0b6o8//ijWdjlvPnh6eiooKEjp6ek6ePBgsfaTlJSUK7ANGTJEffr0kY+Pj7Zt26YvvvhCSUlJGj9+vObPn69mzZoVuD/bdbZ8+fJCQ9vy5ctzbYPSVbt2bbVv314tWrRQ7dq1Vb16dWVlZSk6Olpr167VunXrFB8fr8cff1yLFy9W06ZNc23v5uampk2b6pZbblGzZs1UvXp1Va1aVZcuXVJkZKTRue3gwYN6+OGHtWLFikJn1CnsvbFu3bqFnlfOa66w0MY1d3UQ2lCmtmzZYgS2fv36adq0aXma/2+99VaNHj3amLPVxvaBWpDMzExt375dyp5u7K677iqw/O23365NmzbpwoUL2rBhg3r16pVvWdubVLdu3bR69epCzhRXw5NPPqng4GAFBwerWrVqJbqF3rhxY7388ssKDg5Ws2bN5O3trY8//rjYoW3WrFlGYHv++ef1yCOPGOtuvvlmtWvXTsOHD1dKSoqmT5+ub7/9tsD9de3aVT///LP+/PNPnTt3TtWrV7dbLjY2Vps3b5a4Nq+K9u3b67fffst3fZ8+fbR+/Xo9+eSTSk9P1yeffJLnFuaTTz6pZ555xu72HTp00ODBg/XGG29o7ty5ioyM1OLFi3PdNrensPfGorBdc0ePHlV4eLiCg4PtlktPTzeuM6650sWQHygzWVlZeu211yRJTZs21RtvvFHg8xolmX/1zz//1NmzZ6Xs5y18fHwKLF+pUiXjVpUtlNlz4MAB40O8sNY7XD1PP/20unTpomrVqpV4H61atdLw4cMVEhIib2/vEu0jPT3dCGGNGjXSqFGj8pRp3bq1Bg4cKEnavn279uzZU+A+O3bsqOrVqyszM1M//fRTvuVWrVqljIwMVa9e3Sm3i1Ewd3f3Qst0797deDRjx44dedYXZZLyMWPGGMv29lEaGjVqpJYtW0qFvB9u3LhRCQkJ8vT0VJ8+fa5K3VwVoQ1l5o8//jBaIh599NEivXEVl20wY2U/iF4U9913nyRp06ZNio+PL3C/LVq0YDoy5LFt2zZdvHhRyg71V97at8n5kPb69esL3Ke7u7vuvvtuqZAPUNu6e+65p0iBAldHhQoVJKnEtw5t20vKc9ehNNneD1evXp3vFI22a65Lly6qVKnSVaubKyK0ocysWbNGyu6ddOeddxqvJyQk6Pjx40avv5JKSkpSaGiolN05pCi9QSWpU6dOCggIyNXkn1NmZqbxjFzfvn0dqiPKp507dxrL7dq1y7dcy5YtjZ6gu3btKnS/tg/Qffv26dChQ3nWHz582JhGz1YWZe/o0aM6cOCAJKlhw4Yl2kfO1tWidqhyhrvvvlseHh6KjY21+3xoYmKicXuYa670EdpQZv7++28pO1BVrFhRK1eu1L333qv27durZ8+exv9nzZpVom+Wa9euVUpKipT9ZmKxWIq0naenZ4EtGrZnijw8PHTPPfcUu14o/2wzn6iQD2kPDw9df/31ebbJT/PmzXXjjTdK+VybtteCgoIK7diA0pWSkqLjx4/rm2++0fDhw41WqpEjRxZ5H4mJidq3b5/efPNNvf7661L2+9MDDzxQ6LajRo3SrbfeqpYtW+rWW2/V8OHDNXPmTCUmJhbrPKpWrarbb79dyuea+/nnn5WWlqaAgAB16tSpWPtG8RHaUCZsY2FJUpUqVTRt2jT9+9//zvOw9/Hjx/X2229rxIgRunDhQrGOkfPWaHGfO7OV//vvv/N0Y7e9cXXs2NGhZ6dQfkVHR0vZ46oVdruodu3aUvbwEEX5cmJrzVi1alWunq62oRlEi0eZWbp0qZo0aaImTZooJCREPXv21IwZM3T+/Hkp+7m0wnr+vvjii8Y+2rVrp/79+2v27NlKT0+Xr6+v3n///SIN+bF582bFxcUpPT3dGHrkvffeU/fu3Qu9FX8l2/W0YcOGPEPJ2N4P+/TpU6LnjlE8hDaUiYsXLyorK0uSdPDgQX377beqXr263nnnHW3fvl1///23vvvuO4WEhEiSdu/erZdeeqnI+z99+rT++usvKbunXnFnoGjVqpXq168vZY/ZZnPp0iXjDY8PRuTn0qVLUnZoK0zOgXJt2xWkb9++cnNz05kzZ7Rt2zbj9W3btunMmTNyc3MrNBjg6mrWrJkWLVqk5557rsgt/le6++679fPPP6tHjx4FlgsKCtITTzyhzz//XEuXLtXChQv11ltvGa1lFy5c0NNPP62NGzcW+djdunWTv7+/UlNTtXbtWuP1yMhI47Y+74dXB6ENZcJ221LZD+b6+vpq7ty56tu3rypXriwfHx+1bdtWc+bMMcY0WrdunXFLtTArVqwwWiFK2rvT9iaUM7StW7dOKSkpqlixYrGHkoDrsD1sXlBvaJucrRNFeUi9Zs2axkj2OW9X2ZY7dOigmjVrlqjecEz37t21cuVKrVy5UosWLdL777+vu+66S/v379dzzz2nX3/9tdB9jB8/3tjH/Pnz9dprr6lFixb66aef9NxzzxU4gO3IkSO1cuVKPfPMM+rSpYtatGihm266Sf369dOsWbOMW6yZmZl6+eWXi9wpwtvbWz179pTyuebq169vfMFG6SK0oUxc2Yw+aNAgu8/++Pj4aPz48cbPRR3/x/Zm4uXlVeIu6Lbn4CIjI40Hy223XIsyfAhcl22oENvUVwXJeUu0qEOM2L6I/PLLL0pNTc3VAkKLR9mpVKmSgoKCFBQUpFatWunuu+/WJ598orfeekuRkZF64okntHTp0gL3UbNmTWMfrVu31rBhw7Ro0SINHTpUO3fu1JAhQ4xODfaOX5D7779fgwYNkiSdPXs2V6tZYWzX3Pbt23XmzBkpxxdaOmRdPYQ2lImKFSvm+tnWdG/PrbfeagwHEh4eXui+9+zZYzwv17Vr1xJ3Qa9Tp47atGkjZYfAmJgY43YUH4woiG14hiunZbMnZ6tzzmEdCnLXXXfJ19fX6CG9fv16Xbp0SX5+foXePsPV169fP/Xq1UtZWVmaOnVqsXvGu7u76+WXX1bt2rWVmJhojG9ZEkOHDjWWbY+QFEWbNm1Up04d49nJ3bt368SJE7JYLIS2q4jQhjLh5eWlwMBA4+datWrlW9bb29uYaDkuLq7QfTvSAeFKtnC2Zs0aLV68WFlZWbruuusKHMYBsF3PycnJhXagsbVaBAYGFvlB7goVKqh79+5S9hcKW8ty9+7di/QcHa4+2+MUycnJ2rRpU7G39/LyMuYt3b17t2JiYkpUj8aNGxvLxdmHxWIxnpXMec21bt36qs6F6uoIbSgzOd88bJ0S8pOZmSkVYeTw9PR0YzyjqlWr2p2cuTh69eolb29vJSYm6osvvpCybwWU9GFiuIacAy7bWn3tycjIUGRkZJ5tisL2hWTz5s36888/c72Ga0/OL6mnT58us3048t5lu74OHz5s3Oblmru6CG0oMzkHu7V9cNmTlJRkzExQ2APWtulUlD0ivKOzLPj7+6tr165SjofEuRWAwtxyyy3Gsm3uW3siIiKMW6itW7cu1jFuvfVWVa9eXRkZGcrIyFCNGjV06623OlBrlKacrVolbQ11xj4OHz5sLNeoUaNY2zZo0ECtWrWSst8Pvby8CpyfGc5HaEOZyfnszbp16/Itt27dOqMnaM4PQ3tKMm1VYfr16ycvLy95eXnp5ptvZtoqFKpdu3by9/eXsq/JnOOp5ZTzoXTb7c6icnd313333Wdcm/fdd1++02Wh7NlmgFEJJ3NPTk7W77//LmV30CruMEY2CxYsMJaLOktMTjnfD++66y6mrbrK+AtHmWnatKkxgvZPP/2kLVu25Clz7tw5ffjhh1L28Am2CbbtSUhIMKZTceaI8HfeeafCw8MVHh6uH374wSn7RPnm5eWl4cOHS9kzHcyaNStPmd27d2vJkiVSdsiztWAUx/PPP29cm//+97+dUHMU19KlSwsdOmP27NnGuGh169Y1Ojgp+zndwnpxXr58WS+99JJiY2OlfHqv//PPPzpx4kSB+1mwYIEWLVokSapevbruuuuuQs4urwcffNC45t5///1ibw/HOH+GbqAYXnrpJYWFhenChQt67LHHNHLkSHXu3Fne3t7as2ePZs6caYwu/8wzzxR4e/Snn34yhlhwVisbzGXHjh06efKk8bPttroknThxIs9wCzknbM/pynL79+83ljdt2qRTp04ZP19//fW5PoRtRo8erdWrV+v48eN65513dPLkSfXp00c+Pj7atm2bPv/8c2VkZMjHx6dYA0fj2mIb0qNHjx665ZZbVK9ePVWoUEFJSUk6ePCgVq5caQxA6+npqalTp8rd3d3YPjk5WU8//bRuuOEG9ejRQ61atVLNmjXl5eWl+Ph47dmzR4sXLzYeIalZs6bdgL537169/PLLat++vTp16qSgoCAFBAQoMzNTR48e1cqVK425Q93d3TVlyhQ6rZgQoQ1lqkGDBvrss8/0zDPP6Pz585o5c6ZmzpyZq4zFYtHYsWP16KOPFrgvW28md3d3RoR3UYsXL9ayZcvsrtu1a1eeSdnzC20TJ07M9xhffvllrp/79+9vN7RVrFhRM2fO1JgxY3T8+HEtWLAg160pW5l3332XeUJNLiEhQQsXLtTChQvzLVOrVi1Nnz5dt912m931J06cyHNtXenmm2/WO++8k++zaJmZmfrzzz+Njin2BAQE6I033jCe1YW5ENpQ5tq0aaNVq1bpu+++0/r16xUVFaX09HRVr15d7dq10/Dhw9W8efMC93H8+HFjtoTbbrtN1atXv0q1B/J3ww03aNmyZZo3b57WrFmjkydPKj09XbVq1VLnzp01YsQI1alTp6yrCQd89dVX2rhxo3bt2qUTJ04oNjZWCQkJ8vb2VtWqVdWsWTPdeeed6t27d64py2yuu+46zZs3T5s3b9aePXt0+vRpxcbGGuPu1a5dWy1btlSvXr10xx135Nv7s3PnznrjjTcUFhamffv2GfWwWq2qXLmymjZtqjvuuEMDBgzIM04mzMNize8JWQAAAFwz6IgAAABgAoQ2AAAAEyC0AQAAmAChDQAAwAQIbQAAACZAaAMAADABQhsAAIAJENoAAABMgNAGAABgAoQ2AAAAEyC0AQAAmAChDYDL+/jjj9WkSRM1adKk1I7RtWtXNWnSRC+++GKpHaMoXnzxRTVp0kRdu3Yt03oAKD6Psq4AAHPatm2bRowYYfzs5+enP//8U76+vgVul5qaqo4dOyopKcl4be7cuWrfvn2p1re8SE9P188//6y1a9dq//79io2NVUZGhipWrKhatWqpcePGCgkJ0e23364GDRqUdXUBOBGhDYBTJCcna/369br33nsLLBcaGporsKHojh49qmeeeUYHDx7Msy4hIUEJCQk6cOCAVq1aJUnas2ePvL29y6CmAEoDoQ2Aw7y9vXX58mUtX7680NC2fPnyXNugaOLj4zVy5EidPXtWktSuXTv17dtXjRo1kq+vrxITE3Xo0CFt27ZNmzZtUmpqqt39zJgxQzNm/L/27j0oqvIN4PgXkNUQhYxFS2waSwhd1CgUdcoEEkUI1IamQPIfTQYJKgn/YWqyUDNHRwxHi8ap0NC8wAriJZlIREHDvNAIXjABHVaThLVyQX5//OAMK8vugqu59nz+Ws95zntez4zOM+/leZfd594LIWxBkjYhxF0LCgpi9+7dHDp0CJ1Oh1qtNhl37do1SkpKAAgODqagoOA+99R+bdiwQUnYFi5cSGJiYpeYwMBA5syZQ3NzM9u3b8fRUZYtC/EwkX/RQoi7NmnSJNRqNa2treTn53cbt2vXLlpaWlCr1UycOPG+9tHeHThwAAAPDw8SEhLMxrq6uhIXF4ezs/N96p0Q4n6QkTYhxF1zcnJixowZbNy4kdzcXObOnWsyrmNqNDw8HCcnJ6vavnXrFlu3bqWwsJDq6mqam5txc3Nj5MiRhIeHExERYXFE6cqVK6xfv57i4mIaGhpwc3NDo9EQFxfXo+SxqamJTZs2UVRURE1NDc3Nzbi7u6PRaIiKiiI0NBQHBwer2+uJ+vp6ALy8vO5qBG3x4sXs2LGDoUOHKolgh6CgIOrq6qxu68yZMyavX7x4kezsbEpLS6mvr8dgMKBWqwkICCAmJgY/P79e91+I/zJJ2oQQNhEZGcnGjRuprKykurqaESNGGN0/e/Ysp0+fVmJ/++03i23W1tYyb948zp8/b3T96tWrFBcXU1xcTE5ODpmZmbi7u5ts4+jRo7z99ttGmx90Oh1FRUUUFRWZnGY0pbS0lOTkZBobG42ud25r8uTJrFq1iv79+1vVZk84Oztz69YtLl68SEtLC336PJj/fWdlZbFq1SoMBoPR9draWmpra9m5cyfx8fEkJSX9a30Uwl49mP/qhRB2Z+TIkYwYMYLq6mpyc3NZtGiR0f2OUTZvb298fX0tJm16vZ65c+dy6dIlAEJCQpg9ezaenp7U1taSnZ1NWVkZx44dY8GCBWRnZ3cZvauvr1cSNkdHR6Kjo5k2bRqurq6cOXOGL7/8koyMDDQajdm+HDt2jHnz5mEwGPDw8CA2NpZnn30WT09PGhoaKCgoIC8vj59++onFixeTkZHRy6/YvVGjRlFWVsb169dJT09n8eLFqFQqm74jKyurS7LVWU1NDe+99x4Gg4Ennniiy/2vvvqKFStWAODj48Mbb7zBU089xYABA7hw4QLZ2dlUVFSQmZnJo48+alQyRghhmSRtQgibiYyM5PPPP2fXrl28//77ylRhW1sbWq1WibHG2rVrlYQtPj6e5ORk5Z5GoyE0NJSUlBS0Wi0VFRXk5OTw5ptvGrWxbNkyZYRtxYoVhIeHK/f8/PyYNm0aMTExnDp1qtt+GAwGUlJSMBgMvPjii2RkZBjVohs1ahRTpkwhICCAtLQ09u7dS0lJCZMmTbLyq1knNjaWsrIyALKzs9mzZw/BwcH4+/vj5+fH8OHD73pq1lxdtxs3bpCYmIjBYMDFxYXMzEyj+2fPnmX16tXQvlFi4cKFRv3RaDTMmDGD1NRU8vLyWLVqFZGRkbi5ud1Vn4X4L5GNCEIIm3n11VdxdHTk8uXLHDlyRLl+5MgRLl++jKOjo8WSILSvY/vhhx8AGDFihMkpTAcHBz766CNlWjQ7O9vovk6nY//+/QBMmTLFKGHr4OrqypIlS8z2JT8/n7q6Ovr27ctnn33WbfHg6OhoRo8eDcD27dst/h17KjQ01CgRunr1Kjk5OaSmphIWFsa4ceOIj49Hq9WaHS3rjZaWFpKSkqipqcHBwYHly5fj6+trFPP1119jMBjQaDRdErYOjo6OpKWloVKpuHnzJnv27LFpP4V42EnSJoSwmcGDBysnG3RMh3b+HRgYyODBgy22c+rUKW7cuAHAzJkzu9204OrqyvTp06F9pKejJAbtiWJraysAs2bN6vZdo0eP7rL+rrOOxfoBAQEMGjTIbL9feOEFAI4fP242rrcSExPZsmULoaGhXYrm3rhxgwMHDrBo0SLCw8M5efKkzd6bnp7OoUOHlD5MnTq1S0xRURG0J5fmRvwGDhyIt7c3ABUVFTbroxD/BTI9KoSwqaioKEpLS9m7dy8ffvghgDKiYu3UaHV1tfJ7zJgxZmPHjBnD5s2blec8PT0BjE4NsLRb0c/Pz+idnXVMnR48eNDqs0mvXr1qVVxvjB49mjVr1vD3339z/PhxTpw4walTpygvL+ePP/6A9rVncXFx5OTkKAlSb23evFkZxQwLCzNZbqSurk5598qVK1m5cqVVbd/L7yTEw0hG2oQQNvXKK6/wyCOP0NzczI8//sj+/fvR6/W4uLiYHKEx5c8//1R+Wxrd8vDwMPlc512ejz32mNVt3KkjGemJ7k4jsKV+/foRGBjI/PnzWbNmDQcPHuSLL77g8ccfh/ZjxdLT0+/qHYcPH+aTTz6B9jVpS5cuNRl37dq1XrV/P76TEA8TGWkTQthU//79CQkJQavVkpubS1tbG7Tv/nRxcelxe/eq7pm1OqZYX3rpJVJSUv7Vvpjj5ORESEgIw4YNY/bs2RgMBg4fPkxjY2O35VDMuXjxIklJSUox5MzMTPr162cy9vbt28rvhIQEpk2bZtU7ulsfKIQwTZI2IYTNRUVFodVqlSOrOq5Zq/OOwmvXrpnd1dh5iq3zc3e20TECZamNO7m7u9PQ0IDBYLjrqcb7wcfHhzFjxnD06FHa2tr4/fffe5y0NTU1sWDBAhobG+nbty+ZmZlm1yJ2br9Pnz528Z2EsEcyPSqEsLkJEyagVqtpaWmhpaUFT09PJkyYYPXznTcG/Prrr2ZjT5w4YfK5zomDpUX55kp+jBw5Uom5deuWhZ4/GDrW9dGLkcrW1lbeffddpaBxenq6siu2O8OGDWPAgAEA/PLLL73qsxDCMknahBA25+TkRGRkJCqVCpVKRWRkZI+OXtJoNAwcOBCAnTt3Gk2/ddbc3Mzu3bsBeOaZZ4ySlfHjxyu7Tnfs2NHtu06cOGG0aeFOQUFB0D76dC9KeVirY5rZmrjKykpoT9iGDh3ao/csW7aMn3/+GYAFCxaYLJVyJycnJyZPngxASUkJ586d69E7hRDWkaRNCHFPpKSkcPLkSU6ePNnldARLVCoVr732GrTvAr2zkCvtycmSJUu4fv06ADExMUb3PT09CQ4OhvayHQUFBV3a0Ov1yg7X7sycOVOZWl2+fDnl5eVm448ePaoUwbWlmJgY8vLyLI72rV27lpqaGgD8/f0tbuTobMuWLXzzzTfQvgaxc0FjS+bPn4+TkxO3b9/mnXfe4cqVK93Gtra2kpeXZzZGCNGVrGkTQjyQEhIS2LdvH5cuXSIjI4OqqipmzZqFWq2mtraW7777TkmOnnvuOV5//fUubaSmplJSUoJer2fRokWUl5cTGhqqHGO1YcMGampq0Gg03U6RqlQqVq9ezZw5c7h58yZvvfUWYWFhhISE4OXlxe3bt9HpdJw+fZp9+/ZRVVVFWloa48aNs+n3OHfuHCkpKXz66acEBwfz/PPP8+STTzJgwAD0ej1VVVXk5eUp05POzs6kpqZa3f758+f5+OOPAVCr1cybN6/bMigdOk9B+/j48MEHH7B06VLOnj1LeHg40dHRBAYG4uHhwT///ENdXR3Hjx+nsLAQnU6HVqtlyJAhvf4mQvzXSNImhHggubq6snHjRuXA+D179pisoO/v78+6detMFuD18vJi3bp1xMfHo9fr2bRpE5s2bTKKSUhIwMHBwey6trFjx/Ltt9+SnJzM5cuX0Wq1yrFc3fXd1oYMGUJjYyONjY1s27aNbdu2dRurVqtZunSpxRp3nel0OuUkBZ1OZzIJvtOZM2eM/jx37lxcXFxIT0+nqamJrKwssrKyTD7r7OzcpUCwEMI8SdqEEA8sLy8vcnNz2bp1K4WFhVRVVaHX63Fzc8PX15eIiAgiIiLMrpcbP348+fn5rF+/nuLiYhoaGnBzc0Oj0RAbG6ucJ2rJ2LFj2bt3L9u3b6eoqIjKykquX7+Oo6MjgwYN4umnnyYgIICpU6cyfPhwG3+J/58qUVlZSUlJCRUVFZw7dw6dTsdff/1Fv3798PDwwNvbm5dffpnp06ffk8TRGtHR0QQFBfH9999TUlLChQsXaGpqQqVS4enpiY+PDxMnTmTq1Kk9mroVQoBDm7WrW4UQQgghxL9GNiIIIYQQQtgBSdqEEEIIIeyAJG1CCCGEEHZAkjYhhBBCCDsgSZsQQgghhB2QpE0IIYQQwg5I0iaEEEIIYQckaRNCCCGEsAOStAkhhBBC2AFJ2oQQQggh7IAkbUIIIYQQdkCSNiGEEEIIOyBJmxBCCCGEHZCkTQghhBDCDkjSJoQQQghhB/4HxJBofZlRz2cAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "transfer_wikitext_df = pd.DataFrame({\n", + " 'model': ['67M', '67M', '110M', '110M', '335M', '335M'],\n", + " 'order': [0, 0, 1, 1, 2, 2],\n", + " 'direction': [\"LTR\", \"RTL\", \"LTR\", \"RTL\", \"LTR\", \"RTL\"],\n", + " 'ppl': [24.4, 24.4, 21.8, 21.9, 17.7, 18.1],\n", + "})\n", + "\n", + "\n", + "# Plot configuration\n", + "plt.figure(figsize=(6, 8))\n", + "sns.set_style(\"whitegrid\")\n", + "\n", + "# Create bar plot with LTR and RTL next to each other, no error bars (ci=None)\n", + "sns.barplot(x='model', y='ppl', hue='direction', data=transfer_wikitext_df.sort_values(by=['order', 'direction']), dodge=True, palette=\"Set2\", ci=None)\n", + "\n", + "# Adjustments to the plot\n", + "# plt.xticks(rotation=45)\n", + "plt.title(\"Perplexity vs Model Size, From Scratch\", fontsize=20)\n", + "plt.xlabel(\"Model Size\", fontsize=20)\n", + "plt.ylabel(\"Test Perplexity\", fontsize=20)\n", + "plt.ylim(0.0, 122.75062123923252)\n", + "plt.legend(title=\"\", fontsize=20)\n", + "plt.tick_params(axis='both', labelsize=20)\n", + "\n", + "# Display the updated plot\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9bc44c20-d2a8-431a-97cc-a43655e1f856", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/data/riddles.txt b/data/riddles.txt new file mode 100644 index 0000000..43787f3 --- /dev/null +++ b/data/riddles.txt @@ -0,0 +1,40 @@ +The more you take, the more you leave behind. What am I? Footsteps. +The more you have of me, the less you see. What am I? Darkness. +I am not alive, but I grow. I don’t have lungs, but I need air. I don’t have a mouth, but water kills me. What am I? Fire. +I get wetter the more I dry. What am I? A towel. +I become shorter the longer I live. What am I? A candle. +I am light as a feather, yet the strongest man can’t hold me for much longer than a minute. What am I? Breath. +I am invisible, but you can feel me. I am intangible, but you can hear me. What am I? The wind. +I have keys but open no locks. What am I? A piano. +The more of me there is, the less you see. What am I? Fog. +I am always in front of you, but you can never see me. What am I? The future. +I don’t have wings, but I can fly. I don’t have eyes, but I can cry. Wherever I go, darkness follows me. What am I? A cloud. +I am not alive, but I can die. What am I? A battery. +I can be stolen, but I can’t be touched. What am I? A heart. +I am something that comes once in a minute, twice in a moment, but never in a thousand years. What am I? The letter "M." +I can be cracked, but never broken. What am I? A promise. +I am always with you, but I can’t be seen. What am I? Your shadow. +I’m small but I can cover miles. What am I? A footprint. +The more you have of me, the less you know. What am I? A secret. +I’m something that can fill a room, but I don’t take up space. What am I? Light. +I am always with you, but you never see me. I can be forgotten, but I never leave. What am I? Your name. +The more of me there is, the less you hear. What am I? Silence. +I can travel around the world while staying in the corner. What am I? A stamp. +I am something that can be cracked, but I can’t be touched. What am I? A code. +I am something you can hear, but not touch. I can be loud or soft, but I can never be seen. What am I? Sound. +I am something that you can never keep, no matter how hard you try. What am I? The present moment. +I am not alive, but I grow. I don’t have a mouth, but I can speak. What am I? A rumor. +I am always running, but I never move. What am I? A clock. +I get bigger the more you take away. What am I? A hole. +I can’t be seen, but I can be felt. I have no color, but I make things clear. What am I? Understanding. +I get smaller the more you use me. What am I? A pencil. +I can be broken without being touched. What am I? A promise. +I am something that everyone has, but no one can keep forever. What am I? Time. +I can be light as a feather, but even the strongest hands cannot hold me. What am I? A thought. +The more you take from me, the greater I become. What am I? A debt. +I am often in front of you, but I’m never within reach. What am I? The horizon. +You can see me every day, but I will never be seen the same way twice. What am I? The sky. +I am not alive, but I grow over time. What am I? A reputation. +I can’t be touched, but I can touch everything. What am I? A feeling. +I never speak, but I can communicate. What am I? A look. +I can be hard, but I am not solid. What am I? A deadline. \ No newline at end of file diff --git a/data/wandb_export_2024-12-04T19_56_43.325-05_00.csv b/data/wandb_export_2024-12-04T19_56_43.325-05_00.csv new file mode 100644 index 0000000..0793204 --- /dev/null +++ b/data/wandb_export_2024-12-04T19_56_43.325-05_00.csv @@ -0,0 +1,21 @@ +"Name","val_loss" +"distilbert_base_japan_rtl","2.8326140656842465" +"distilbert_base_japan_ltr","2.8237654270093375" +"bert_6M_rtl_scratch","4.744475745069383" +"bert_6_ltr_scratch","4.761364663504469" +"bert_11_rtl_scratch","4.446949723712903" +"bert_11_ltr_scratch","4.462378635840655" +"bert_19_rtl_scratch","4.177320378220149" +"bert_19_ltr_scratch","4.186270630920852" +"bert_35_rtl_scratch","3.927856646112007" +"bert_35_ltr_scratch","3.941595227497572" +"qa_distilbert_base_ltr_v2","3.1502674087524416" +"qa_distilbert_base_rtl_v2","3.1904524799346925" +"qa_ltr_distilbert_base","3.3259500965491715" +"distilbert_base_ltr_scratch","3.6863074678864063" +"distilbert_base_rtl_scratch","3.6885659350549624" +"deep-monkey-11","3.009245432539425" +"distilbert_base_ltr_4epoch","3.1961001348322804" +"distilbert_base_rtl_4epoch","3.19366226070481" +"bert_base_ltr_4epoch","3.082235844222857" +"bert_base_rtl_4epoch","3.0881099989546192" \ No newline at end of file diff --git a/finetune_QA.py b/finetune_QA.py new file mode 100644 index 0000000..e5b8ef7 --- /dev/null +++ b/finetune_QA.py @@ -0,0 +1,304 @@ +""" +accelerate launch --mixed_precision bf16 finetune_QA.py \ +--model_direction rtl \ +--checkpoint_path /home/sipb/nlp-class-project/checkpoints/distilbert_base_rtl/epoch_3_checkpt \ +--tokenizer_name distilbert/distilbert-base-uncased \ +--warmup_steps 100 \ +--learning_rate 1e-5 \ +--per_device_train_batch_size 128 \ +--per_device_eval_batch_size 128 \ +--output_dir checkpoints/qa_distilbert_base_rtl/ \ +--eval_steps 38 \ +--block_size 128 \ +--num_train_epochs 50 \ +--weight_decay 1e-4 + + +accelerate launch --mixed_precision bf16 finetune_QA.py \ +--model_direction ltr \ +--checkpoint_path /home/sipb/nlp-class-project/checkpoints/distilbert_base_ltr/epoch_3_checkpt \ +--tokenizer_name distilbert/distilbert-base-uncased \ +--warmup_steps 100 \ +--learning_rate 1e-5 \ +--per_device_train_batch_size 128 \ +--per_device_eval_batch_size 128 \ +--output_dir checkpoints/qa_distilbert_base_ltr/ \ +--eval_steps 38 \ +--block_size 128 \ +--num_train_epochs 50 \ +--weight_decay 1e-4 + +accelerate launch --mixed_precision bf16 finetune_QA.py \ +--model_direction ltr \ +--checkpoint_path /home/sipb/nlp-class-project/checkpoints/distilbert_base_ltr/epoch_3_checkpt \ +--tokenizer_name distilbert/distilbert-base-uncased \ +--warmup_steps 100 \ +--learning_rate 1e-5 \ +--per_device_train_batch_size 128 \ +--per_device_eval_batch_size 128 \ +--output_dir checkpoints/qa_distilbert_base_ltr_overfit/ \ +--eval_steps 50 \ +--block_size 128 \ +--num_train_epochs 1000 \ +--weight_decay 0 +""" + + + +import argparse +import math +import os +from collections import defaultdict + +import accelerate +import torch +import transformers +import wandb +from datasets import load_dataset +from torch.utils.data import Dataset, DataLoader +from transformers.data.data_collator import default_data_collator +from tqdm.auto import tqdm + +from utils import preprocess_datasets, convert_to_torch_dataset, add_attn_hooks, causal_loss_wrapper + +#### HERE WE do the dataset stuff +class DatasetAQ(Dataset): + def __init__(self, qa_pairs, text_direction, tokenizer): + self.qa_pairs = qa_pairs + self.text_direction = text_direction + self.tokenizer = tokenizer + + def __getitem__(self, idx): + question, answer = self.qa_pairs[idx] + sentence = torch.cat([question, answer], dim=0) if self.text_direction.lower() == "rtl" else torch.cat([answer, question], dim=0) + + # TODO: length + num_to_pad = self.tokenizer.model_max_length - sentence.size(0) + assert num_to_pad >= 0, (sentence.size(), self.tokenizer.model_max_length) + + if num_to_pad > 0: + pad_tokens = torch.full((num_to_pad,), self.tokenizer.pad_token_id, dtype=sentence.dtype) + pad_labels = torch.full((num_to_pad,), -100, dtype=sentence.dtype) + + if self.text_direction.lower() == "rtl": + input_ids = torch.cat([pad_tokens, sentence], dim=0) + labels = torch.cat([pad_labels, sentence], dim=0) + attention_mask = torch.ones_like(input_ids, dtype=torch.bool) + attention_mask[:num_to_pad] = 0 + else: + input_ids = torch.cat([sentence, pad_tokens], dim=0) + labels = torch.cat([sentence, pad_labels], dim=0) + attention_mask = torch.ones_like(input_ids, dtype=torch.bool) + attention_mask[-num_to_pad:] = 0 + + return { + "input_ids": input_ids, + "labels": labels, + "attention_mask": attention_mask, + } + + def __len__(self): + return len(self.qa_pairs) + +#### + + + +def parse_args(): + """ + Re-using HuggingFace arguments when possible (most of the help strings are directly copied). + https://github.com/huggingface/transformers/blob/7bbc62474391aff64f63fcc064c975752d1fa4de/examples/pytorch/language-modeling/run_clm.py#L75 + """ + parser = argparse.ArgumentParser() + + # Model + parser.add_argument("--model_direction", type=str, required=True, choices=["ltr", "rtl"], + help="Whether to train a left-to-right or right-to-left LM.") + parser.add_argument("--checkpoint_path", type=str, + help="Path to load model weights from.") + + # Data + parser.add_argument("--tokenizer_name", type=str, + help="Name of tokenizer to load.") + parser.add_argument("--dataset_name", type=str, default="truthfulqa/truthful_qa", + help="The name of the dataset to use (via the datasets library).") + parser.add_argument("--dataset_config_name", type=str, default="generation", + help="The configuration name of the dataset to use (via the datasets library).") + # TODO: block_size, train on shorter seqs? + parser.add_argument( + "--block_size", + type=int, + help="Optional input sequence length after tokenization. " + "The training dataset will be truncated in block of this size for training. " + "Default to the model max input length for single sentence inputs (take into account special tokens)." + ) + + # Training + parser.add_argument("--train_from_scratch", action="store_true") + parser.add_argument("--output_dir", type=str, required=True, + help="The output directory where the model predictions and checkpoints will be written.") + parser.add_argument("--per_device_train_batch_size", type=int, default=8) + parser.add_argument("--per_device_eval_batch_size", type=int, default=16) + parser.add_argument("--gradient_accumulation_steps", type=int, default=1) + parser.add_argument("--num_train_epochs", type=int, default=1) + parser.add_argument("--learning_rate", type=float, required=True) + parser.add_argument("--warmup_steps", type=int, default=0) + parser.add_argument("--scheduler", type=str, default="cosine") + parser.add_argument("--weight_decay", type=float, default=0.0) + parser.add_argument("--logging_steps", type=int, default=1, + help="Number of update steps between two logs.") + parser.add_argument("--eval_steps", type=int, default=20000, + help="Number of update steps between two logs.") + parser.add_argument("--dataloader_num_workers", type=int, default=8) + + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + transformers.set_seed(42) + + accelerator = accelerate.Accelerator(gradient_accumulation_steps=args.gradient_accumulation_steps, log_with="wandb", project_dir=args.output_dir) + # Will `add_attn_hooks` to `model` later + + # Load model weights in both cases, but re-initialize if training from scratch + model = transformers.AutoModelForMaskedLM.from_pretrained(args.checkpoint_path, attn_implementation="sdpa", ignore_mismatched_sizes=True) + if args.train_from_scratch: + model.apply(model._init_weights) + model.tie_weights() # probably not applicable + + tokenizer = transformers.AutoTokenizer.from_pretrained(args.tokenizer_name) + + # Data + raw_datasets = load_dataset(args.dataset_name, args.dataset_config_name) + block_size = args.block_size if args.block_size is not None else model.config.max_position_embeddings + model.config.max_position_embeddings = block_size + tokenizer.model_max_length = block_size + + # QA-specific code + all_data = raw_datasets["validation"] + transformers.set_seed(42) + train_val_split = all_data.train_test_split(test_size=0.2, shuffle=True) + val_test_split = train_val_split['test'].train_test_split(test_size=0.5, shuffle=False) + train_dataset = train_val_split['train'] + val_dataset = val_test_split['train'] + test_dataset = val_test_split['test'] + + qa_pairs = defaultdict(list) + for data_name, dataset in zip(["test","train","val"], [train_dataset, test_dataset, val_dataset]): + for row in dataset: + tokenized_question = tokenizer("Question: "+ row["question"], return_tensors="pt")["input_ids"].squeeze(0) + for ans_type in ["correct_answers", "incorrect_answers"]: + for answer in row[ans_type]: + # the [:, 1:] thing is to remove CLS token + qa_pairs[data_name].append((tokenized_question, tokenizer(f"Answer: {answer}", return_tensors="pt")["input_ids"].squeeze(0)[1:])) + + train_dataset = DatasetAQ(qa_pairs["train"], args.model_direction, tokenizer) + test_dataset = DatasetAQ(qa_pairs["test"], args.model_direction, tokenizer) + val_dataset = DatasetAQ(qa_pairs["val"], args.model_direction, tokenizer) + + train_loader = DataLoader(train_dataset, batch_size=args.per_device_train_batch_size, shuffle=True) + test_loader = DataLoader(test_dataset, batch_size=args.per_device_eval_batch_size, shuffle=False) + val_loader = DataLoader(val_dataset, batch_size=args.per_device_eval_batch_size) + + model, train_loader, test_loader, val_loader = accelerator.prepare(model, train_loader, test_loader, val_loader) + + optimizer = torch.optim.AdamW(model.parameters(), lr=args.learning_rate, weight_decay=args.weight_decay) + lr_scheduler = transformers.get_scheduler( + name=transformers.SchedulerType.COSINE, + optimizer=optimizer, + num_warmup_steps=args.warmup_steps * accelerator.num_processes, + # num_training_steps=args.num_train_epochs * math.ceil(len(train_loader) / args.gradient_accumulation_steps), + num_training_steps=args.num_train_epochs * len(train_loader), + ) + + lr_scheduler = accelerator.prepare(lr_scheduler) # testing if this fixes learning rate + + loss_fn = causal_loss_wrapper(args.model_direction) + + add_attn_hooks(model, args.model_direction) + model.train() + optimizer.zero_grad() + + wandb.require("core") + accelerator.init_trackers( + project_name="NLP-Class-Project", + config=vars(args) | {"model_parameters": sum(p.numel() for p in model.parameters())}, + init_kwargs={"wandb": {"entity": "frostbyte"}} + ) + + global_step = 0 # unaccumulated steps + past_losses = [] + best_val_loss = float("inf") + best_checkpt_path = os.path.join(args.output_dir, f"best_checkpt") + + for epoch in tqdm(range(args.num_train_epochs), position=0, leave=True, desc="Epoch"): + for step, batch in enumerate(tqdm(train_loader, position=1, leave=False, desc="Train Iteration")): + with accelerator.accumulate(model): + labels = batch.pop("labels") + outputs = model(**batch) + loss = loss_fn(outputs.logits, labels) + accelerator.backward(loss) + + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + past_losses.append(loss.item()) + if (global_step + 1) % args.logging_steps == 0: + avg_train_loss = torch.tensor(past_losses).mean().item() # Assuming 1 GPU + accelerator.log({ + "train_loss": avg_train_loss, + "learning_rate": lr_scheduler.get_last_lr()[0], + }) + past_losses.clear() + + if (global_step + 1) % args.eval_steps == 0: + val_loss_sum = val_examples = 0 + model.eval() + for val_batch in tqdm(val_loader, position=2, leave=False, desc="Val Iteration"): + labels = val_batch.pop("labels") + with torch.no_grad(): + outputs = model(**val_batch) + + loss = loss_fn(outputs.logits, labels) + + batch_size = labels.size(0) + val_loss_sum += loss.item() * batch_size + val_examples += batch_size + + val_loss = val_loss_sum / val_examples + if val_loss < best_val_loss: + best_val_loss = val_loss + model.save_pretrained(best_checkpt_path) + + accelerator.log({"val_loss": val_loss_sum / val_examples}, + log_kwargs={"wandb": {"commit": False}}) + model.train() + + if ((step + 1) % args.gradient_accumulation_steps == 0) or step == (len(train_loader) - 1): + global_step += 1 + + # model.save_pretrained(os.path.join(args.output_dir, f"epoch_{epoch}_checkpt")) + + # testing + model.from_pretrained(best_checkpt_path) + model.eval() + with torch.no_grad(): + test_loss_sum = test_examples = 0 + for test_batch in tqdm(test_loader): + labels = test_batch.pop("labels") + outputs = model(**test_batch) + + loss = loss_fn(outputs.logits, labels) + + batch_size = labels.size(0) + test_loss_sum += loss.item() * batch_size + test_examples += batch_size + + accelerator.log({"test_loss": test_loss_sum / test_examples}) + + +if __name__ == "__main__": + main() diff --git a/finetune_bert-japanese.py b/finetune_bert-japanese.py new file mode 100644 index 0000000..f0b1238 --- /dev/null +++ b/finetune_bert-japanese.py @@ -0,0 +1,225 @@ +""" +# BERT japanese RTL +accelerate launch --mixed_precision bf16 finetune_bert-japanese.py \ +--model_direction rtl \ +--model_name distilbert/distilbert-base-multilingual-cased \ +--dataset_name ntotsuka123/ja-pretrain \ +--warmup_steps 500 \ +--learning_rate 5e-5 \ +--per_device_train_batch_size 128 \ +--gradient_accumulation_steps 1 \ +--per_device_eval_batch_size 128 \ +--output_dir checkpoints/distilbert_base_japan_rtl/ \ +--eval_steps 1000 \ +--block_size 128 \ +--num_train_epochs 1 \ +--weight_decay 1e-4 + + +is there some way to only do 1% of the data... +got it +you have to change the code. I don't want ot do it right now + +# BERT japanese LTR +accelerate launch --mixed_precision bf16 finetune_bert.py \ +--model_direction rtl \ +--dataset_name oscar \ +--dataset_config_name unshuffled_deduplicated_ja \ +--model_name cl-tohoku/bert-base-japanese \ +--warmup_steps 500 \ +--learning_rate 5e-5 \ +--per_device_train_batch_size 128 \ +--per_device_eval_batch_size 128 \ +--output_dir checkpoints/bert_base_rtl/ \ +--eval_steps 899 \ +--block_size 128 \ +--num_train_epochs 4 \ +--weight_decay 1e-4 + + +""" + +import argparse +import math +import os + +import accelerate +import torch +import transformers +import wandb +from datasets import load_dataset +from torch.utils.data import DataLoader, Subset +from tqdm.auto import tqdm +from transformers import set_seed + +from utils import preprocess_datasets, convert_to_torch_dataset, add_attn_hooks, causal_loss_wrapper + + + +def parse_args(): + """ + Re-using HuggingFace arguments when possible (most of the help strings are directly copied). + https://github.com/huggingface/transformers/blob/7bbc62474391aff64f63fcc064c975752d1fa4de/examples/pytorch/language-modeling/run_clm.py#L75 + """ + parser = argparse.ArgumentParser() + + # Model + parser.add_argument("--model_direction", type=str, required=True, choices=["ltr", "rtl"], + help="Whether to train a left-to-right or right-to-left LM.") + parser.add_argument("--model_config", type=str, + help="Path to model config json, from which to train_from_scratch.") + parser.add_argument("--model_name", type=str, required=True, + help="Name of tokenizer to load. " + "If model_config is not specified, will also load model architecture." + "If not training from scratch, will also load model weights.") + + # Data + parser.add_argument("--dataset_name", type=str, default="Salesforce/wikitext", + help="The name of the dataset to use (via the datasets library).") + parser.add_argument("--dataset_config_name", type=str, default="wikitext-103-v1", + help="The configuration name of the dataset to use (via the datasets library).") + # TODO: block_size, train on shorter seqs? + parser.add_argument( + "--block_size", + type=int, + help="Optional input sequence length after tokenization. " + "The training dataset will be truncated in block of this size for training. " + "Default to the model max input length for single sentence inputs (take into account special tokens)." + ) + + # Training + parser.add_argument("--train_from_scratch", action="store_true") + parser.add_argument("--output_dir", type=str, required=True, + help="The output directory where the model predictions and checkpoints will be written.") + parser.add_argument("--per_device_train_batch_size", type=int, default=8) + parser.add_argument("--per_device_eval_batch_size", type=int, default=16) + parser.add_argument("--gradient_accumulation_steps", type=int, default=1) + parser.add_argument("--num_train_epochs", type=int, default=1) + parser.add_argument("--learning_rate", type=float, required=True) + parser.add_argument("--warmup_steps", type=int, default=0) + parser.add_argument("--weight_decay", type=float, default=0.0) + parser.add_argument("--logging_steps", type=int, default=1, + help="Number of update steps between two logs.") + parser.add_argument("--eval_steps", type=int, default=20000, + help="Number of update steps between two logs.") + parser.add_argument("--dataloader_num_workers", type=int, default=8) + + args = parser.parse_args() + + return args + + +def main(): + args = parse_args() + + accelerator = accelerate.Accelerator(gradient_accumulation_steps=args.gradient_accumulation_steps, log_with="wandb", project_dir=args.output_dir) + set_seed(42) + + # Will `add_attn_hooks` to `model` later + if args.model_config is not None: + assert args.train_from_scratch, "Expected to train from scratch when model_config is specified." + config = transformers.AutoConfig.from_pretrained(args.model_config) + model = transformers.AutoModelForMaskedLM.from_config(config) + else: + # Load model weights in both cases, but re-initialize if training from scratch + model = transformers.AutoModelForMaskedLM.from_pretrained(args.model_name, attn_implementation="sdpa") + + if args.train_from_scratch: + model.apply(model._initialize_weights) + model.tie_weights() # probably not applicable + + tokenizer = transformers.AutoTokenizer.from_pretrained(args.model_name) + + # Data + raw_datasets = load_dataset(args.dataset_name) + block_size = args.block_size if args.block_size is not None else model.config.max_position_embeddings + model.config.max_position_embeddings = block_size + + processed_datasets = preprocess_datasets(raw_datasets, tokenizer, block_size) + for split, hf_dataset in processed_datasets.items(): + processed_datasets[split] = convert_to_torch_dataset(hf_dataset) + + train_val_split = processed_datasets["train"].train_test_split(test_size=0.2, shuffle=True) + train_indices = torch.randperm(len(train_val_split["train"]))[:int(0.4 * len(train_val_split["train"]))] + train_subset = Subset(train_val_split["train"], train_indices) + val_indices = torch.randperm(len(train_val_split["test"]))[:int(0.01 * len(train_val_split["test"]))] + val_subset = Subset(train_val_split["test"], val_indices) + train_loader = DataLoader(train_subset, batch_size=args.per_device_train_batch_size, shuffle=True) + val_loader = DataLoader(val_subset, batch_size=args.per_device_eval_batch_size) + + # train_val_split = processed_datasets["train"].train_test_split(test_size=0.2, shuffle=True) + # train_loader = DataLoader(train_val_split["train"], batch_size=args.per_device_train_batch_size, shuffle=True) + # val_loader = DataLoader(train_val_split["test"], batch_size=args.per_device_eval_batch_size) + # test_loader = DataLoader(processed_datasets["test"], batch_size=args.per_device_eval_batch_size) + + model, train_loader, val_loader = accelerator.prepare(model, train_loader, val_loader) + + optimizer = torch.optim.AdamW(model.parameters(), lr=args.learning_rate, weight_decay=args.weight_decay) + lr_scheduler = transformers.get_scheduler( + name=transformers.SchedulerType.CONSTANT, + optimizer=optimizer, + num_warmup_steps=args.warmup_steps * accelerator.num_processes, + num_training_steps=args.num_train_epochs * math.ceil(len(train_loader) / args.gradient_accumulation_steps), + ) + loss_fn = causal_loss_wrapper(args.model_direction) + + add_attn_hooks(model, args.model_direction) + model.train() + optimizer.zero_grad() + + wandb.require("core") + accelerator.init_trackers( + project_name="NLP-Class-Project", + config=vars(args) | {"model_parameters": sum(p.numel() for p in model.parameters())}, + init_kwargs={"wandb": {"entity": "frostbyte"}} + ) + + global_step = 0 # unaccumulated steps + past_losses = [] + for epoch in tqdm(range(args.num_train_epochs), position=0, leave=True, desc="Epoch"): + for step, batch in enumerate(tqdm(train_loader, position=1, leave=False, desc="Train Iteration")): + with accelerator.accumulate(model): + labels = batch.pop("labels") + outputs = model(**batch) + loss = loss_fn(outputs.logits, labels) + accelerator.backward(loss) + + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + past_losses.append(loss.item()) + if (global_step + 1) % args.logging_steps == 0: + avg_train_loss = torch.tensor(past_losses).mean().item() # Assuming 1 GPU + accelerator.log({ + "train_loss": avg_train_loss, + "learning_rate": lr_scheduler.get_last_lr()[0], + }) + past_losses.clear() + + if (global_step + 1) % args.eval_steps == 0: + val_loss_sum = val_examples = 0 + model.eval() + for val_batch in tqdm(val_loader, position=2, leave=False, desc="Val Iteration"): + labels = val_batch.pop("labels") + with torch.no_grad(): + outputs = model(**val_batch) + + loss = loss_fn(outputs.logits, labels) + + batch_size = labels.size(0) + val_loss_sum += loss.item() * batch_size + val_examples += batch_size + + accelerator.log({"val_loss": val_loss_sum / val_examples}, + log_kwargs={"wandb": {"commit": False}}) + model.train() + + if ((step + 1) % args.gradient_accumulation_steps == 0) or step == (len(train_loader) - 1): + global_step += 1 + + model.save_pretrained(os.path.join(args.output_dir, f"epoch_{epoch}_checkpt")) + + +if __name__ == "__main__": + main() diff --git a/finetune_bert.py b/finetune_bert.py index 59c8090..da29af9 100644 --- a/finetune_bert.py +++ b/finetune_bert.py @@ -1,663 +1,330 @@ -#!/usr/bin/env python -# coding=utf-8 -# Copyright 2020 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. """ -Fine-tuning the library models for causal language modeling (GPT, GPT-2, CTRL, ...) on a text file or a dataset. -Here is the full list of checkpoints on the hub that can be fine-tuned by this script: -https://huggingface.co/models?filter=text-generation +# BERT base +accelerate launch --mixed_precision bf16 finetune_bert.py \ +--model_direction rtl \ +--model_name bert-base-uncased \ +--warmup_steps 500 \ +--learning_rate 5e-5 \ +--per_device_train_batch_size 128 \ +--per_device_eval_batch_size 128 \ +--output_dir checkpoints/bert_base_rtl/ \ +--eval_steps 899 \ +--block_size 128 \ +--num_train_epochs 4 \ +--weight_decay 1e-4 + +accelerate launch --mixed_precision bf16 finetune_bert.py \ +--model_direction ltr \ +--model_name bert-base-uncased \ +--warmup_steps 500 \ +--learning_rate 5e-5 \ +--per_device_train_batch_size 128 \ +--per_device_eval_batch_size 128 \ +--output_dir checkpoints/bert_base_ltr/ \ +--eval_steps 899 \ +--block_size 128 \ +--num_train_epochs 4 \ +--weight_decay 1e-4 + +# DistilBERT scratch +accelerate launch --mixed_precision bf16 finetune_bert.py \ +--model_direction rtl \ +--model_name distilbert/distilbert-base-uncased \ +--train_from_scratch \ +--warmup_steps 500 \ +--learning_rate 5e-5 \ +--per_device_train_batch_size 128 \ +--per_device_eval_batch_size 128 \ +--output_dir checkpoints/distilbert_base_rtl_scratch/ \ +--eval_steps 899 \ +--block_size 128 \ +--num_train_epochs 4 \ +--weight_decay 1e-4 + +accelerate launch --mixed_precision bf16 finetune_bert.py \ +--model_direction ltr \ +--model_name distilbert/distilbert-base-uncased \ +--train_from_scratch \ +--warmup_steps 500 \ +--learning_rate 5e-5 \ +--per_device_train_batch_size 128 \ +--per_device_eval_batch_size 128 \ +--output_dir checkpoints/distilbert_base_ltr_scratch/ \ +--eval_steps 899 \ +--block_size 128 \ +--num_train_epochs 4 \ +--weight_decay 1e-4 + +# DistilBERT base +accelerate launch --mixed_precision bf16 finetune_bert.py \ +--model_direction rtl \ +--model_name distilbert/distilbert-base-uncased \ +--warmup_steps 500 \ +--learning_rate 5e-5 \ +--per_device_train_batch_size 128 \ +--per_device_eval_batch_size 128 \ +--output_dir checkpoints/distilbert_base_rtl/ \ +--eval_steps 899 \ +--block_size 128 \ +--num_train_epochs 4 \ +--weight_decay 1e-4 + + +accelerate launch --mixed_precision bf16 finetune_bert.py \ +--model_direction ltr \ +--model_name distilbert/distilbert-base-uncased \ +--warmup_steps 500 \ +--learning_rate 5e-5 \ +--per_device_train_batch_size 128 \ +--per_device_eval_batch_size 128 \ +--output_dir checkpoints/distilbert_base_ltr/ \ +--eval_steps 899 \ +--block_size 128 \ +--num_train_epochs 4 \ +--weight_decay 1e-4 + +# BERT large +accelerate launch --mixed_precision bf16 finetune_bert.py \ +--model_direction rtl \ +--model_name bert-large-uncased \ +--warmup_steps 500 \ +--learning_rate 5e-5 \ +--per_device_train_batch_size 64 \ +--gradient_accumulation_steps 2 \ +--per_device_eval_batch_size 64 \ +--output_dir checkpoints/bert_large_rtl/ \ +--eval_steps 899 \ +--block_size 128 \ +--num_train_epochs 4 \ +--weight_decay 1e-4 + + +accelerate launch --mixed_precision bf16 finetune_bert.py \ +--model_direction ltr \ +--model_name bert-large-uncased \ +--warmup_steps 500 \ +--learning_rate 5e-5 \ +--per_device_train_batch_size 64 \ +--gradient_accumulation_steps 2 \ +--per_device_eval_batch_size 64 \ +--output_dir checkpoints/bert_large_ltr/ \ +--eval_steps 899 \ +--block_size 128 \ +--num_train_epochs 4 \ +--weight_decay 1e-4 + +for size in 35 19 11 6; do + for dir in ltr rtl; do + accelerate launch --mixed_precision bf16 finetune_bert.py \ + --model_direction $dir \ + --model_name bert-base-uncased \ + --model_config "configs/bert_${size}M.json" \ + --train_from_scratch \ + --warmup_steps 500 \ + --learning_rate 5e-5 \ + --per_device_train_batch_size 128 \ + --per_device_eval_batch_size 128 \ + --output_dir "checkpoints/bert_${size}_${dir}_scratch/" \ + --eval_steps 899 \ + --block_size 128 \ + --num_train_epochs 4 \ + --weight_decay 1e-4 + done +done + +for seed in 0 1 2 3 4; do + for dir in ltr rtl; do + accelerate launch --mixed_precision bf16 finetune_bert.py \ + --model_direction $dir \ + --model_name bert-base-uncased \ + --model_config "configs/bert_${size}M.json" \ + --train_from_scratch \ + --warmup_steps 500 \ + --learning_rate 5e-5 \ + --per_device_train_batch_size 128 \ + --per_device_eval_batch_size 128 \ + --output_dir "checkpoints/overwritable_temp/" \ + --eval_steps 899 \ + --block_size 128 \ + --num_train_epochs 1 \ + --weight_decay 1e-4 \ + --seed $seed + done +done """ -# You can also adapt this script on your own causal language modeling task. Pointers for this are left as comments. -""" -From https://github.com/huggingface/transformers/blob/main/examples/pytorch/language-modeling/run_clm.py -""" - -import logging +import argparse import math import os -import sys -from dataclasses import dataclass, field -from itertools import chain -from typing import Optional -import datasets -import evaluate +import accelerate import torch -from datasets import load_dataset - import transformers -from transformers import ( - CONFIG_MAPPING, - MODEL_FOR_CAUSAL_LM_MAPPING, - AutoConfig, - AutoModelForCausalLM, - AutoTokenizer, - HfArgumentParser, - Trainer, - TrainingArguments, - default_data_collator, - is_torch_xla_available, - set_seed, -) -from transformers.testing_utils import CaptureLogger -from transformers.trainer_utils import get_last_checkpoint -from transformers.utils import check_min_version, send_example_telemetry -from transformers.utils.versions import require_version - - -# Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.47.0.dev0") - -require_version("datasets>=2.14.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt") - -logger = logging.getLogger(__name__) - - -MODEL_CONFIG_CLASSES = list(MODEL_FOR_CAUSAL_LM_MAPPING.keys()) -MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES) - - -@dataclass -class ModelArguments: - """ - Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch. - """ - # text_direction: str = field( - # - # ) - model_name_or_path: Optional[str] = field( - default=None, - metadata={ - "help": ( - "The model checkpoint for weights initialization. Don't set if you want to train a model from scratch." - ) - }, - ) - # model_type: Optional[str] = field( - # default=None, - # metadata={"help": "If training from scratch, pass a model type from the list: " + ", ".join(MODEL_TYPES)}, - # ) - config_overrides: Optional[str] = field( - default=None, - metadata={ - "help": ( - "Override some existing default config settings when a model is trained from scratch. Example: " - "n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index" - ) - }, - ) - config_name: Optional[str] = field( - default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"} - ) - tokenizer_name: Optional[str] = field( - default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"} - ) - cache_dir: Optional[str] = field( - default=None, - metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}, - ) - use_fast_tokenizer: bool = field( - default=True, - metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}, - ) - model_revision: str = field( - default="main", - metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."}, - ) - token: str = field( - default=None, - metadata={ - "help": ( - "The token to use as HTTP bearer authorization for remote files. If not specified, will use the token " - "generated when running `huggingface-cli login` (stored in `~/.huggingface`)." - ) - }, - ) - trust_remote_code: bool = field( - default=False, - metadata={ - "help": ( - "Whether to trust the execution of code from datasets/models defined on the Hub." - " This option should only be set to `True` for repositories you trust and in which you have read the" - " code, as it will execute code present on the Hub on your local machine." - ) - }, - ) - torch_dtype: Optional[str] = field( - default=None, - metadata={ - "help": ( - "Override the default `torch.dtype` and load the model under this dtype. If `auto` is passed, the " - "dtype will be automatically derived from the model's weights." - ), - "choices": ["auto", "bfloat16", "float16", "float32"], - }, - ) - low_cpu_mem_usage: bool = field( - default=False, - metadata={ - "help": ( - "It is an option to create the model as an empty shell, then only materialize its parameters when the pretrained weights are loaded. " - "set True will benefit LLM loading time and RAM consumption." - ) - }, - ) +import wandb +from datasets import load_dataset +from torch.utils.data import DataLoader +from tqdm.auto import tqdm +from transformers import set_seed - def __post_init__(self): - if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None): - raise ValueError( - "--config_overrides can't be used in combination with --config_name or --model_name_or_path" - ) +from utils import preprocess_datasets, convert_to_torch_dataset, add_attn_hooks, causal_loss_wrapper -@dataclass -class DataTrainingArguments: +def parse_args(): """ - Arguments pertaining to what data we are going to input our model for training and eval. + Re-using HuggingFace arguments when possible (most of the help strings are directly copied). + https://github.com/huggingface/transformers/blob/7bbc62474391aff64f63fcc064c975752d1fa4de/examples/pytorch/language-modeling/run_clm.py#L75 """ - - dataset_name: Optional[str] = field( - default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."} - ) - dataset_config_name: Optional[str] = field( - default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."} - ) - train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."}) - validation_file: Optional[str] = field( - default=None, - metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."}, - ) - max_train_samples: Optional[int] = field( - default=None, - metadata={ - "help": ( - "For debugging purposes or quicker training, truncate the number of training examples to this " - "value if set." - ) - }, - ) - max_eval_samples: Optional[int] = field( - default=None, - metadata={ - "help": ( - "For debugging purposes or quicker training, truncate the number of evaluation examples to this " - "value if set." - ) - }, - ) - streaming: bool = field(default=False, metadata={"help": "Enable streaming mode"}) - block_size: Optional[int] = field( - default=None, - metadata={ - "help": ( - "Optional input sequence length after tokenization. " - "The training dataset will be truncated in block of this size for training. " - "Default to the model max input length for single sentence inputs (take into account special tokens)." - ) - }, - ) - overwrite_cache: bool = field( - default=False, metadata={"help": "Overwrite the cached training and evaluation sets"} + parser = argparse.ArgumentParser() + + # Model + parser.add_argument("--model_direction", type=str, required=True, choices=["ltr", "rtl"], + help="Whether to train a left-to-right or right-to-left LM.") + parser.add_argument("--model_config", type=str, + help="Path to model config json, from which to train_from_scratch.") + parser.add_argument("--model_name", type=str, required=True, + help="Name of tokenizer to load. " + "If model_config is not specified, will also load model architecture." + "If not training from scratch, will also load model weights.") + + # Data + parser.add_argument("--dataset_name", type=str, default="Salesforce/wikitext", + help="The name of the dataset to use (via the datasets library).") + parser.add_argument("--dataset_config_name", type=str, default="wikitext-103-v1", + help="The configuration name of the dataset to use (via the datasets library).") + # TODO: block_size, train on shorter seqs? + parser.add_argument( + "--block_size", + type=int, + help="Optional input sequence length after tokenization. " + "The training dataset will be truncated in block of this size for training. " + "Default to the model max input length for single sentence inputs (take into account special tokens)." ) - validation_split_percentage: Optional[int] = field( - default=5, - metadata={ - "help": "The percentage of the train set used as validation set in case there's no validation split" - }, - ) - preprocessing_num_workers: Optional[int] = field( - default=None, - metadata={"help": "The number of processes to use for the preprocessing."}, - ) - keep_linebreaks: bool = field( - default=True, metadata={"help": "Whether to keep line breaks when using TXT files or not."} - ) - - def __post_init__(self): - if self.streaming: - require_version("datasets>=2.0.0", "The streaming feature requires `datasets>=2.0.0`") - if self.dataset_name is None and self.train_file is None and self.validation_file is None: - raise ValueError("Need either a dataset name or a training/validation file.") - else: - if self.train_file is not None: - extension = self.train_file.split(".")[-1] - assert extension in ["csv", "json", "txt"], "`train_file` should be a csv, a json or a txt file." - if self.validation_file is not None: - extension = self.validation_file.split(".")[-1] - assert extension in ["csv", "json", "txt"], "`validation_file` should be a csv, a json or a txt file." + # Training + parser.add_argument("--train_from_scratch", action="store_true") + parser.add_argument("--output_dir", type=str, required=True, + help="The output directory where the model predictions and checkpoints will be written.") + parser.add_argument("--per_device_train_batch_size", type=int, default=8) + parser.add_argument("--per_device_eval_batch_size", type=int, default=16) + parser.add_argument("--gradient_accumulation_steps", type=int, default=1) + parser.add_argument("--num_train_epochs", type=int, default=1) + parser.add_argument("--learning_rate", type=float, required=True) + parser.add_argument("--warmup_steps", type=int, default=0) + parser.add_argument("--weight_decay", type=float, default=0.0) + parser.add_argument("--logging_steps", type=int, default=1, + help="Number of update steps between two logs.") + parser.add_argument("--eval_steps", type=int, default=20000, + help="Number of update steps between two logs.") + parser.add_argument("--dataloader_num_workers", type=int, default=8) + parser.add_argument("--seed", type=int, default=42, help="Random seed.") + + args = parser.parse_args() + + return args def main(): - # See all possible arguments in src/transformers/training_args.py - # or by passing the --help flag to this script. - # We now keep distinct sets of args, for a cleaner separation of concerns. - - parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)) - if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): - # If we pass only one argument to the script and it's the path to a json file, - # let's parse it to get our arguments. - model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1])) - else: - model_args, data_args, training_args = parser.parse_args_into_dataclasses() + args = parse_args() - # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The - # information sent is the one passed as arguments along with your Python/PyTorch versions. - send_example_telemetry("run_clm", model_args, data_args) + accelerator = accelerate.Accelerator(gradient_accumulation_steps=args.gradient_accumulation_steps, log_with="wandb", project_dir=args.output_dir) + set_seed(args.seed) - # Setup logging - logging.basicConfig( - format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", - datefmt="%m/%d/%Y %H:%M:%S", - handlers=[logging.StreamHandler(sys.stdout)], - ) - - if training_args.should_log: - # The default of training_args.log_level is passive, so we set log level at info here to have that default. - transformers.utils.logging.set_verbosity_info() - - log_level = training_args.get_process_log_level() - logger.setLevel(log_level) - datasets.utils.logging.set_verbosity(log_level) - transformers.utils.logging.set_verbosity(log_level) - transformers.utils.logging.enable_default_handler() - transformers.utils.logging.enable_explicit_format() - - # Log on each process the small summary: - logger.warning( - f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, " - + f"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}" - ) - logger.info(f"Training/evaluation parameters {training_args}") - - # Detecting last checkpoint. - last_checkpoint = None - if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir: - last_checkpoint = get_last_checkpoint(training_args.output_dir) - if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0: - raise ValueError( - f"Output directory ({training_args.output_dir}) already exists and is not empty. " - "Use --overwrite_output_dir to overcome." - ) - elif last_checkpoint is not None and training_args.resume_from_checkpoint is None: - logger.info( - f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change " - "the `--output_dir` or add `--overwrite_output_dir` to train from scratch." - ) - - # Set seed before initializing model. - set_seed(training_args.seed) - - # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below) - # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/ - # (the dataset will be downloaded automatically from the datasets Hub). - # - # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called - # 'text' is found. You can easily tweak this behavior (see below). - # - # In distributed training, the load_dataset function guarantee that only one local process can concurrently - # download the dataset. - if data_args.dataset_name is not None: - # Downloading and loading a dataset from the hub. - raw_datasets = load_dataset( - data_args.dataset_name, - data_args.dataset_config_name, - cache_dir=model_args.cache_dir, - token=model_args.token, - streaming=data_args.streaming, - trust_remote_code=model_args.trust_remote_code, - ) - if "validation" not in raw_datasets.keys(): - raw_datasets["validation"] = load_dataset( - data_args.dataset_name, - data_args.dataset_config_name, - split=f"train[:{data_args.validation_split_percentage}%]", - cache_dir=model_args.cache_dir, - token=model_args.token, - streaming=data_args.streaming, - trust_remote_code=model_args.trust_remote_code, - ) - raw_datasets["train"] = load_dataset( - data_args.dataset_name, - data_args.dataset_config_name, - split=f"train[{data_args.validation_split_percentage}%:]", - cache_dir=model_args.cache_dir, - token=model_args.token, - streaming=data_args.streaming, - trust_remote_code=model_args.trust_remote_code, - ) - else: - data_files = {} - dataset_args = {} - if data_args.train_file is not None: - data_files["train"] = data_args.train_file - if data_args.validation_file is not None: - data_files["validation"] = data_args.validation_file - extension = ( - data_args.train_file.split(".")[-1] - if data_args.train_file is not None - else data_args.validation_file.split(".")[-1] - ) - if extension == "txt": - extension = "text" - dataset_args["keep_linebreaks"] = data_args.keep_linebreaks - raw_datasets = load_dataset( - extension, - data_files=data_files, - cache_dir=model_args.cache_dir, - token=model_args.token, - **dataset_args, - ) - # If no validation data is there, validation_split_percentage will be used to divide the dataset. - if "validation" not in raw_datasets.keys(): - raw_datasets["validation"] = load_dataset( - extension, - data_files=data_files, - split=f"train[:{data_args.validation_split_percentage}%]", - cache_dir=model_args.cache_dir, - token=model_args.token, - **dataset_args, - ) - raw_datasets["train"] = load_dataset( - extension, - data_files=data_files, - split=f"train[{data_args.validation_split_percentage}%:]", - cache_dir=model_args.cache_dir, - token=model_args.token, - **dataset_args, - ) - - # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at - # https://huggingface.co/docs/datasets/loading_datasets. - - # Load pretrained model and tokenizer - # - # Distributed training: - # The .from_pretrained methods guarantee that only one local process can concurrently - # download model & vocab. - - config_kwargs = { - "cache_dir": model_args.cache_dir, - "revision": model_args.model_revision, - "token": model_args.token, - "trust_remote_code": model_args.trust_remote_code, - } - if model_args.config_name: - config = AutoConfig.from_pretrained(model_args.config_name, **config_kwargs) - elif model_args.model_name_or_path: - config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs) - else: - config = CONFIG_MAPPING[model_args.model_type]() - logger.warning("You are instantiating a new config instance from scratch.") - if model_args.config_overrides is not None: - logger.info(f"Overriding config: {model_args.config_overrides}") - config.update_from_string(model_args.config_overrides) - logger.info(f"New config: {config}") - - tokenizer_kwargs = { - "cache_dir": model_args.cache_dir, - "use_fast": model_args.use_fast_tokenizer, - "revision": model_args.model_revision, - "token": model_args.token, - "trust_remote_code": model_args.trust_remote_code, - } - if model_args.tokenizer_name: - tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, **tokenizer_kwargs) - elif model_args.model_name_or_path: - tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, **tokenizer_kwargs) - else: - raise ValueError( - "You are instantiating a new tokenizer from scratch. This is not supported by this script. " - "You can do it from another script, save it, and load it from here, using --tokenizer_name." - ) - - if model_args.model_name_or_path: - torch_dtype = ( - model_args.torch_dtype - if model_args.torch_dtype in ["auto", None] - else getattr(torch, model_args.torch_dtype) - ) - model = AutoModelForCausalLM.from_pretrained( - model_args.model_name_or_path, - from_tf=bool(".ckpt" in model_args.model_name_or_path), - config=config, - cache_dir=model_args.cache_dir, - revision=model_args.model_revision, - token=model_args.token, - trust_remote_code=model_args.trust_remote_code, - torch_dtype=torch_dtype, - low_cpu_mem_usage=model_args.low_cpu_mem_usage, - ) - else: - model = AutoModelForCausalLM.from_config(config, trust_remote_code=model_args.trust_remote_code) - n_params = sum({p.data_ptr(): p.numel() for p in model.parameters()}.values()) - logger.info(f"Training new model from scratch - Total size={n_params/2**20:.2f}M params") - - # We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch - # on a small vocab and want a smaller embedding size, remove this test. - embedding_size = model.get_input_embeddings().weight.shape[0] - if len(tokenizer) > embedding_size: - model.resize_token_embeddings(len(tokenizer)) - - # Preprocessing the datasets. - # First we tokenize all the texts. - if training_args.do_train: - column_names = list(raw_datasets["train"].features) - else: - column_names = list(raw_datasets["validation"].features) - text_column_name = "text" if "text" in column_names else column_names[0] - - # since this will be pickled to avoid _LazyModule error in Hasher force logger loading before tokenize_function - tok_logger = transformers.utils.logging.get_logger("transformers.tokenization_utils_base") - - def tokenize_function(examples): - with CaptureLogger(tok_logger) as cl: - output = tokenizer(examples[text_column_name]) - # clm input could be much much longer than block_size - if "Token indices sequence length is longer than the" in cl.out: - tok_logger.warning( - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits" - " before being passed to the model." - ) - return output - - with training_args.main_process_first(desc="dataset map tokenization"): - if not data_args.streaming: - tokenized_datasets = raw_datasets.map( - tokenize_function, - batched=True, - num_proc=data_args.preprocessing_num_workers, - remove_columns=column_names, - load_from_cache_file=not data_args.overwrite_cache, - desc="Running tokenizer on dataset", - ) - else: - tokenized_datasets = raw_datasets.map( - tokenize_function, - batched=True, - remove_columns=column_names, - ) - if hasattr(config, "max_position_embeddings"): - max_pos_embeddings = config.max_position_embeddings + # Will `add_attn_hooks` to `model` later + if args.model_config is not None: + assert args.train_from_scratch, "Expected to train from scratch when model_config is specified." + config = transformers.AutoConfig.from_pretrained(args.model_config) + model = transformers.AutoModelForMaskedLM.from_config(config) else: - # Define a default value if the attribute is missing in the config. - max_pos_embeddings = 1024 - - if data_args.block_size is None: - block_size = tokenizer.model_max_length - if block_size > max_pos_embeddings: - logger.warning( - f"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). " - f"Using block_size={min(1024, max_pos_embeddings)} instead. You can change that default value by passing --block_size xxx." - ) - if max_pos_embeddings > 0: - block_size = min(1024, max_pos_embeddings) - else: - block_size = 1024 - else: - if data_args.block_size > tokenizer.model_max_length: - logger.warning( - f"The block_size passed ({data_args.block_size}) is larger than the maximum length for the model " - f"({tokenizer.model_max_length}). Using block_size={tokenizer.model_max_length}." - ) - block_size = min(data_args.block_size, tokenizer.model_max_length) - - # Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size. - def group_texts(examples): - # Concatenate all texts. - concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()} - total_length = len(concatenated_examples[list(examples.keys())[0]]) - # We drop the small remainder, and if the total_length < block_size we exclude this batch and return an empty dict. - # We could add padding if the model supported it instead of this drop, you can customize this part to your needs. - total_length = (total_length // block_size) * block_size - # Split by chunks of max_len. - result = { - k: [t[i : i + block_size] for i in range(0, total_length, block_size)] - for k, t in concatenated_examples.items() - } - result["labels"] = result["input_ids"].copy() - return result - - # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a remainder - # for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value might be slower - # to preprocess. - # - # To speed up this part, we use multiprocessing. See the documentation of the map method for more information: - # https://huggingface.co/docs/datasets/process#map - - with training_args.main_process_first(desc="grouping texts together"): - if not data_args.streaming: - lm_datasets = tokenized_datasets.map( - group_texts, - batched=True, - num_proc=data_args.preprocessing_num_workers, - load_from_cache_file=not data_args.overwrite_cache, - desc=f"Grouping texts in chunks of {block_size}", - ) - else: - lm_datasets = tokenized_datasets.map( - group_texts, - batched=True, - ) - - if training_args.do_train: - if "train" not in tokenized_datasets: - raise ValueError("--do_train requires a train dataset") - train_dataset = lm_datasets["train"] - if data_args.max_train_samples is not None: - max_train_samples = min(len(train_dataset), data_args.max_train_samples) - train_dataset = train_dataset.select(range(max_train_samples)) - - if training_args.do_eval: - if "validation" not in tokenized_datasets: - raise ValueError("--do_eval requires a validation dataset") - eval_dataset = lm_datasets["validation"] - if data_args.max_eval_samples is not None: - max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) - eval_dataset = eval_dataset.select(range(max_eval_samples)) - - def preprocess_logits_for_metrics(logits, labels): - if isinstance(logits, tuple): - # Depending on the model and config, logits may contain extra tensors, - # like past_key_values, but logits always come first - logits = logits[0] - return logits.argmax(dim=-1) - - metric = evaluate.load("accuracy", cache_dir=model_args.cache_dir) - - def compute_metrics(eval_preds): - preds, labels = eval_preds - # preds have the same shape as the labels, after the argmax(-1) has been calculated - # by preprocess_logits_for_metrics but we need to shift the labels - labels = labels[:, 1:].reshape(-1) - preds = preds[:, :-1].reshape(-1) - return metric.compute(predictions=preds, references=labels) - - # Initialize our Trainer - trainer = Trainer( - model=model, - args=training_args, - train_dataset=train_dataset if training_args.do_train else None, - eval_dataset=eval_dataset if training_args.do_eval else None, - processing_class=tokenizer, - # Data collator will default to DataCollatorWithPadding, so we change it. - data_collator=default_data_collator, - compute_metrics=compute_metrics if training_args.do_eval and not is_torch_xla_available() else None, - preprocess_logits_for_metrics=preprocess_logits_for_metrics - if training_args.do_eval and not is_torch_xla_available() - else None, + # Load model weights in both cases, but re-initialize if training from scratch + model = transformers.AutoModelForMaskedLM.from_pretrained(args.model_name, attn_implementation="sdpa") + + if args.train_from_scratch: + model.apply(model._initialize_weights) + model.tie_weights() # probably not applicable + + tokenizer = transformers.AutoTokenizer.from_pretrained(args.model_name) + + # Data + raw_datasets = load_dataset(args.dataset_name, args.dataset_config_name) + block_size = args.block_size if args.block_size is not None else model.config.max_position_embeddings + model.config.max_position_embeddings = block_size + + processed_datasets = preprocess_datasets(raw_datasets, tokenizer, block_size) + for split, hf_dataset in processed_datasets.items(): + processed_datasets[split] = convert_to_torch_dataset(hf_dataset) + + train_loader = DataLoader(processed_datasets["train"], batch_size=args.per_device_train_batch_size, shuffle=True) + val_loader = DataLoader(processed_datasets["validation"], batch_size=args.per_device_eval_batch_size) + # test_loader = DataLoader(processed_datasets["test"], batch_size=args.per_device_eval_batch_size) + model, train_loader, val_loader = accelerator.prepare(model, train_loader, val_loader) + + optimizer = torch.optim.AdamW(model.parameters(), lr=args.learning_rate, weight_decay=args.weight_decay) + lr_scheduler = transformers.get_scheduler( + name=transformers.SchedulerType.COSINE, + optimizer=optimizer, + num_warmup_steps=args.warmup_steps * accelerator.num_processes, + num_training_steps=args.num_train_epochs * math.ceil(len(train_loader) / args.gradient_accumulation_steps), ) + loss_fn = causal_loss_wrapper(args.model_direction) - # Training - if training_args.do_train: - checkpoint = None - if training_args.resume_from_checkpoint is not None: - checkpoint = training_args.resume_from_checkpoint - elif last_checkpoint is not None: - checkpoint = last_checkpoint - train_result = trainer.train(resume_from_checkpoint=checkpoint) - trainer.save_model() # Saves the tokenizer too for easy upload - - metrics = train_result.metrics - - max_train_samples = ( - data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset) - ) - metrics["train_samples"] = min(max_train_samples, len(train_dataset)) - - trainer.log_metrics("train", metrics) - trainer.save_metrics("train", metrics) - trainer.save_state() - - # Evaluation - if training_args.do_eval: - logger.info("*** Evaluate ***") - - metrics = trainer.evaluate() - - max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset) - metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset)) - try: - perplexity = math.exp(metrics["eval_loss"]) - except OverflowError: - perplexity = float("inf") - metrics["perplexity"] = perplexity - - trainer.log_metrics("eval", metrics) - trainer.save_metrics("eval", metrics) - - kwargs = {"finetuned_from": model_args.model_name_or_path, "tasks": "text-generation"} - if data_args.dataset_name is not None: - kwargs["dataset_tags"] = data_args.dataset_name - if data_args.dataset_config_name is not None: - kwargs["dataset_args"] = data_args.dataset_config_name - kwargs["dataset"] = f"{data_args.dataset_name} {data_args.dataset_config_name}" - else: - kwargs["dataset"] = data_args.dataset_name - - if training_args.push_to_hub: - trainer.push_to_hub(**kwargs) - else: - trainer.create_model_card(**kwargs) + add_attn_hooks(model, args.model_direction) + model.train() + optimizer.zero_grad() + wandb.require("core") + accelerator.init_trackers( + project_name="NLP-Class-Project", + config=vars(args) | {"model_parameters": sum(p.numel() for p in model.parameters())}, + init_kwargs={"wandb": {"entity": "frostbyte"}} + ) -def _mp_fn(index): - # For xla_spawn (TPUs) - main() + global_step = 0 # unaccumulated steps + past_losses = [] + for epoch in tqdm(range(args.num_train_epochs), position=0, leave=True, desc="Epoch"): + for step, batch in enumerate(tqdm(train_loader, position=1, leave=False, desc="Train Iteration")): + with accelerator.accumulate(model): + labels = batch.pop("labels") + outputs = model(**batch) + loss = loss_fn(outputs.logits, labels) + accelerator.backward(loss) + + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + past_losses.append(loss.item()) + if (global_step + 1) % args.logging_steps == 0: + avg_train_loss = torch.tensor(past_losses).mean().item() # Assuming 1 GPU + accelerator.log({ + "train_loss": avg_train_loss, + "learning_rate": lr_scheduler.get_last_lr()[0], + }) + past_losses.clear() + + if (global_step + 1) % args.eval_steps == 0: + val_loss_sum = val_examples = 0 + model.eval() + for val_batch in tqdm(val_loader, position=2, leave=False, desc="Val Iteration"): + labels = val_batch.pop("labels") + with torch.no_grad(): + outputs = model(**val_batch) + + loss = loss_fn(outputs.logits, labels) + + batch_size = labels.size(0) + val_loss_sum += loss.item() * batch_size + val_examples += batch_size + + accelerator.log({"val_loss": val_loss_sum / val_examples}, + log_kwargs={"wandb": {"commit": False}}) + model.train() + + if ((step + 1) % args.gradient_accumulation_steps == 0) or step == (len(train_loader) - 1): + global_step += 1 + + model.save_pretrained(os.path.join(args.output_dir, f"epoch_{epoch}_checkpt")) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/notebooks/Inference.ipynb b/notebooks/Inference.ipynb new file mode 100644 index 0000000..670a127 --- /dev/null +++ b/notebooks/Inference.ipynb @@ -0,0 +1,638 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "34c536f2-3ccb-4df0-bd47-913d6ef040a2", + "metadata": {}, + "source": [ + "# Inference" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "execution_state": "idle", + "id": "3c6381c4-2a02-415f-a5f1-450fe42b30d3", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/qa_distilbert_base_ltr_v2/best_checkpt were not used when initializing DistilBertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing DistilBertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing DistilBertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n" + ] + } + ], + "source": [ + "import sys\n", + "sys.path.append(\"..\")\n", + "\n", + "import torch\n", + "import transformers\n", + "\n", + "from utils import add_attn_hooks\n", + "\n", + "tokenizer = transformers.AutoTokenizer.from_pretrained(\"distilbert/distilbert-base-uncased\")\n", + "# model = transformers.AutoModelForMaskedLM.from_pretrained(\"/home/sipb/nlp-class-project/checkpoints/distilbert_base_ltr/epoch_3_checkpt\", ignore_mismatched_sizes=True)\n", + "# model = transformers.AutoModelForMaskedLM.from_pretrained(\"/home/sipb/nlp-class-project/checkpoints/qa_distilbert_base_ltr_overfit/epoch_999_checkpt\", ignore_mismatched_sizes=True)\n", + "# model = transformers.AutoModelForMaskedLM.from_pretrained(\"/home/sipb/nlp-class-project/checkpoints/qa_distilbert_base_ltr/epoch_49_checkpt\", ignore_mismatched_sizes=True)\n", + "model = transformers.AutoModelForMaskedLM.from_pretrained(\"/home/sipb/nlp-class-project/checkpoints/qa_distilbert_base_ltr_v2/best_checkpt\", ignore_mismatched_sizes=True)\n", + "\n", + "# tokenizer = transformers.AutoTokenizer.from_pretrained(\"bert-base-uncased\")\n", + "# model = transformers.AutoModelForMaskedLM.from_pretrained(\"/home/sipb/nlp-class-project/checkpoints/bert_base_ltr/epoch_3_checkpt\", ignore_mismatched_sizes=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "execution_state": "idle", + "id": "41edf867-7f6a-4d44-871c-8af0b7af7543", + "metadata": {}, + "outputs": [], + "source": [ + "from utils import add_attn_hooks\n", + "add_attn_hooks(model, \"ltr\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "execution_state": "idle", + "id": "d79a3d16-b982-42ed-9b6a-fad8328e177e", + "metadata": {}, + "outputs": [], + "source": [ + "model.eval();" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "execution_state": "idle", + "id": "f06d4bc2-9df8-42c2-9397-3bfb3728da0b", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/qa_distilbert_base_ltr_overfit/epoch_999_checkpt were not used when initializing DistilBertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing DistilBertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing DistilBertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n" + ] + } + ], + "source": [ + "from typing import override\n", + "class DecoderMLM(transformers.AutoModelForMaskedLM, transformers.GenerationMixin):\n", + " @override\n", + " # @classmethod\n", + " def can_generate(cls):\n", + " return True\n", + "\n", + "model2 = DecoderMLM.from_pretrained(\"/home/sipb/nlp-class-project/checkpoints/qa_distilbert_base_ltr_overfit/epoch_999_checkpt\", ignore_mismatched_sizes=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "execution_state": "idle", + "id": "6feb4fdd-ae43-466d-8dce-a4f9a632a5e6", + "metadata": {}, + "outputs": [], + "source": [ + "# model2.can_generate = (lambda s: True)\n", + "model2.can_generate = (lambda: True)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "execution_state": "idle", + "id": "c178761c-7124-42ed-9bfc-7ab0f782aad7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model2." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "execution_state": "idle", + "id": "3ab53852-f333-47ea-9e96-55266cda84a6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model2.can_generate()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "execution_state": "idle", + "id": "a30f6240-f982-45b0-b75f-3be5bbb43049", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 25, + "execution_state": "idle", + "id": "2cefa784-a1cc-445a-8ffa-066e7cfccaf0", + "metadata": {}, + "outputs": [ + { + "ename": "TypeError", + "evalue": "The current model class (DistilBertForMaskedLM) is not compatible with `.generate()`, as it doesn't have a language model head. Classes that support generation often end in one of these names: ['ForCausalLM', 'ForConditionalGeneration', 'ForSpeechSeq2Seq', 'ForVision2Seq'].", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[25], line 5\u001b[0m\n\u001b[1;32m 2\u001b[0m tokenized_question \u001b[38;5;241m=\u001b[39m tokenizer(question, return_tensors\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpt\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39minference_mode():\n\u001b[0;32m----> 5\u001b[0m \u001b[38;5;28mprint\u001b[39m(tokenizer\u001b[38;5;241m.\u001b[39mbatch_decode(\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtokenized_question\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minput_ids\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m:\u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_beams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m5\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdo_sample\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtemperature\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1.\u001b[39;49m\u001b[43m)\u001b[49m))\n", + "File \u001b[0;32m~/.venv/lib64/python3.12/site-packages/torch/utils/_contextlib.py:116\u001b[0m, in \u001b[0;36mcontext_decorator..decorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_context\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 115\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m ctx_factory():\n\u001b[0;32m--> 116\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/.venv/lib64/python3.12/site-packages/transformers/generation/utils.py:1967\u001b[0m, in \u001b[0;36mGenerationMixin.generate\u001b[0;34m(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, assistant_model, streamer, negative_prompt_ids, negative_prompt_attention_mask, **kwargs)\u001b[0m\n\u001b[1;32m 1882\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1883\u001b[0m \n\u001b[1;32m 1884\u001b[0m \u001b[38;5;124;03mGenerates sequences of token ids for models with a language modeling head.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1963\u001b[0m \u001b[38;5;124;03m - [`~generation.GenerateBeamEncoderDecoderOutput`]\u001b[39;00m\n\u001b[1;32m 1964\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1966\u001b[0m \u001b[38;5;66;03m# 1. Handle `generation_config` and kwargs that might update it, and validate the `.generate()` call\u001b[39;00m\n\u001b[0;32m-> 1967\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate_model_class\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1968\u001b[0m tokenizer \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtokenizer\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m) \u001b[38;5;66;03m# Pull this out first, we only use it for stopping criteria\u001b[39;00m\n\u001b[1;32m 1969\u001b[0m assistant_tokenizer \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124massistant_tokenizer\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m) \u001b[38;5;66;03m# only used for assisted generation\u001b[39;00m\n", + "File \u001b[0;32m~/.venv/lib64/python3.12/site-packages/transformers/generation/utils.py:1269\u001b[0m, in \u001b[0;36mGenerationMixin._validate_model_class\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1262\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_torchdynamo_compiling() \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcan_generate():\n\u001b[1;32m 1263\u001b[0m terminations_with_generation_support \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m 1264\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mForCausalLM\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 1265\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mForConditionalGeneration\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 1266\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mForSpeechSeq2Seq\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 1267\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mForVision2Seq\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 1268\u001b[0m ]\n\u001b[0;32m-> 1269\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\n\u001b[1;32m 1270\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThe current model class (\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m) is not compatible with `.generate()`, as \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 1271\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mit doesn\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt have a language model head. Classes that support generation often end in one of these \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 1272\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnames: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mterminations_with_generation_support\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 1273\u001b[0m )\n", + "\u001b[0;31mTypeError\u001b[0m: The current model class (DistilBertForMaskedLM) is not compatible with `.generate()`, as it doesn't have a language model head. Classes that support generation often end in one of these names: ['ForCausalLM', 'ForConditionalGeneration', 'ForSpeechSeq2Seq', 'ForVision2Seq']." + ] + } + ], + "source": [ + "question = \"Answer: Grapes are toxic to foxes in large quantities\"\n", + "tokenized_question = tokenizer(question, return_tensors=\"pt\")\n", + "\n", + "with torch.inference_mode():\n", + " print(tokenizer.batch_decode(model.generate(tokenized_question[\"input_ids\"][:, :-1], num_beams=5, do_sample=True, temperature=1.)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c475851e-7c86-46fd-b4ad-cd51caa6e7b8", + "metadata": {}, + "outputs": [], + "source": [ + "question = \"Apples are red and \"\n", + "tokenized_question = tokenizer(question, return_tensors=\"pt\")\n", + "\n", + "with torch.inference_mode():\n", + " print(tokenizer.batch_decode(model.generate(tokenized_question[\"input_ids\"][:, :-1], num_beams=5, do_sample=True)))" + ] + }, + { + "cell_type": "code", + "execution_count": 97, + "execution_state": "idle", + "id": "1365ab75-a022-42fe-9168-d49a645af0d5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'answer : if you go outdoors in cold weather with wet hair, your hair may freeze [SEP] [CLS] question :'" + ] + }, + "execution_count": 97, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokenizer.decode([3437, 1024, 2065, 2017, 2175, 19350, 1999, 3147, 4633, 2007,\n", + " 4954, 2606, 1010, 2115, 2606, 2089, 13184, 102, 101, 3160,\n", + " 1024])" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "execution_state": "idle", + "id": "58bffbb3-77fb-4f57-a77e-303fca05a84f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated token: what\n", + "Generated token: is\n", + "Generated token: the\n", + "Generated token: chicken\n", + "Generated token: cry\n", + "Generated token: ?\n", + "Generated token: [SEP]\n", + "Generated token: [CLS]\n", + "Generated token: question\n", + "Generated token: :\n", + "Generated token: what\n", + "Generated token: is\n", + "Generated token: the\n", + "Generated token: chicken\n", + "Generated token: cry\n", + "Generated token: ?\n", + "Generated token: [SEP]\n", + "Generated token: [CLS]\n", + "Generated token: question\n", + "Generated token: :\n", + "Generated token: what\n", + "Generated token: is\n", + "Generated token: the\n", + "Generated token: chicken\n", + "Generated token: cry\n", + "Generated token: ?\n", + "Generated token: [SEP]\n", + "Generated token: [CLS]\n", + "Generated token: what\n", + "Generated token: is\n" + ] + } + ], + "source": [ + "# question = \"answer : grapes are toxic to foxes in large quantities. [SEP] [CLS] question :\"\n", + "question = \"answer : your chicken cries. [SEP] [CLS] question :\"\n", + "# question = \"answer : if you go outdoors in cold weather with wet hair, your hair may freeze [SEP] [CLS] question :\"\n", + "\n", + "# input_ids = tokenizer(question, return_tensors=\"pt\").input_ids[:, :-1]\n", + "input_ids = tokenizer(question, return_tensors=\"pt\", add_special_tokens=False).input_ids\n", + "#tokenized_question = {\n", + "# \"input_ids\": torch.tensor([[3437, 1024, 2498, 6433, 102, 101, 3160, 1024]], dtype=torch.long)\n", + "#}\n", + "# input_ids = torch.tensor([[3437, 1024, 2065, 2017, 2175, 19350, 1999, 3147, 4633, 2007,\n", + "# 4954, 2606, 1010, 2115, 2606, 2089, 13184, 102, 101, 3160,\n", + "# 1024]])\n", + "\n", + "with torch.no_grad():\n", + " for i in range(30):\n", + " # Get logits for the last token in the sequence\n", + " logits = model(input_ids).logits[0, -1, :]\n", + " \n", + " # Select the token with the highest probability\n", + " next_token_id = torch.argmax(logits).item()\n", + " \n", + " # Update the input_ids with the new token\n", + " input_ids = torch.cat([input_ids, torch.tensor([[next_token_id]])], dim=1)\n", + " \n", + " # Decode the next token for readability\n", + " next_token = tokenizer.decode(next_token_id)\n", + " \n", + " print(f\"Generated token: {next_token}\")\n", + "# but we did train on incorrect answers" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "execution_state": "idle", + "id": "972352ea-4c28-42d9-a834-26daa83b2290", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated token: how\n", + "Generated token: long\n", + "Generated token: should\n", + "Generated token: you\n", + "Generated token: wait\n", + "Generated token: before\n", + "Generated token: filing\n", + "Generated token: a\n", + "Generated token: missing\n", + "Generated token: person\n", + "Generated token: report\n", + "Generated token: ?\n", + "Generated token: [SEP]\n", + "Generated token: [CLS]\n", + "Generated token: question\n", + "Generated token: :\n", + "Generated token: how\n", + "Generated token: long\n", + "Generated token: should\n", + "Generated token: you\n", + "Generated token: wait\n", + "Generated token: before\n", + "Generated token: filing\n", + "Generated token: a\n", + "Generated token: missing\n", + "Generated token: person\n", + "Generated token: report\n", + "Generated token: ?\n", + "Generated token: [SEP]\n", + "Generated token: [CLS]\n" + ] + } + ], + "source": [ + "# question = \"answer : grapes are toxic to foxes in large quantities. [SEP] [CLS] question :\"\n", + "question = \"answer : you are late to work. [SEP] [CLS] question :\"\n", + "# question = \"answer : if you go outdoors in cold weather with wet hair, your hair may freeze [SEP] [CLS] question :\"\n", + "\n", + "# input_ids = tokenizer(question, return_tensors=\"pt\").input_ids[:, :-1]\n", + "input_ids = tokenizer(question, return_tensors=\"pt\", add_special_tokens=False).input_ids\n", + "#tokenized_question = {\n", + "# \"input_ids\": torch.tensor([[3437, 1024, 2498, 6433, 102, 101, 3160, 1024]], dtype=torch.long)\n", + "#}\n", + "# input_ids = torch.tensor([[3437, 1024, 2065, 2017, 2175, 19350, 1999, 3147, 4633, 2007,\n", + "# 4954, 2606, 1010, 2115, 2606, 2089, 13184, 102, 101, 3160,\n", + "# 1024]])\n", + "\n", + "with torch.no_grad():\n", + " for i in range(30):\n", + " # Get logits for the last token in the sequence\n", + " logits = model(input_ids).logits[0, -1, :]\n", + " \n", + " # Select the token with the highest probability\n", + " next_token_id = torch.argmax(logits).item()\n", + " \n", + " # Update the input_ids with the new token\n", + " input_ids = torch.cat([input_ids, torch.tensor([[next_token_id]])], dim=1)\n", + " \n", + " # Decode the next token for readability\n", + " next_token = tokenizer.decode(next_token_id)\n", + " \n", + " print(f\"Generated token: {next_token}\")\n", + "# but we did train on incorrect answers" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "execution_state": "idle", + "id": "6b1949d3-343d-49ba-b50e-79dc36d9124e", + "metadata": {}, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "'DistilBertForMaskedLM' object has no attribute 'tokenizer'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[29], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtokenizer\u001b[49m\n", + "File \u001b[0;32m~/.venv/lib64/python3.12/site-packages/torch/nn/modules/module.py:1931\u001b[0m, in \u001b[0;36mModule.__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 1929\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m modules:\n\u001b[1;32m 1930\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m modules[name]\n\u001b[0;32m-> 1931\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m(\n\u001b[1;32m 1932\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(\u001b[38;5;28mself\u001b[39m)\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m object has no attribute \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 1933\u001b[0m )\n", + "\u001b[0;31mAttributeError\u001b[0m: 'DistilBertForMaskedLM' object has no attribute 'tokenizer'" + ] + } + ], + "source": [ + "model.tokenizer" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "execution_state": "idle", + "id": "f57b41c0-2056-4164-914b-f298ad66c0c5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated token: [CLS]\n", + "Generated token: the\n", + "Generated token: vampires\n", + "Generated token: are\n", + "Generated token: the\n", + "Generated token: vampires\n", + "Generated token: ,\n", + "Generated token: vampires\n", + "Generated token: are\n", + "Generated token: living\n", + "Generated token: ,\n", + "Generated token: who\n", + "Generated token: believe\n", + "Generated token: the\n", + "Generated token: vampires\n", + "Generated token: .\n", + "Generated token: vampire\n", + "Generated token: ,\n", + "Generated token: and\n", + "Generated token: who\n", + "Generated token: are\n", + "Generated token: vampires\n", + "Generated token: ,\n", + "Generated token: who\n", + "Generated token: are\n", + "Generated token: also\n", + "Generated token: .\n", + "Generated token: vampires\n", + "Generated token: who\n", + "Generated token: do\n" + ] + } + ], + "source": [ + "import torch\n", + "import torch.nn.functional as F\n", + "\n", + "question = \"Answer: Vampires are real. Question:\"\n", + "tokenized_question = tokenizer(question, return_tensors=\"pt\")\n", + "\n", + "temperature = 0.7 # Set your temperature here (e.g., 0.7 for less randomness)\n", + "\n", + "with torch.no_grad():\n", + " for i in range(30):\n", + " # Get logits for the last token in the sequence\n", + " logits = model(tokenized_question[\"input_ids\"]).logits[0, -1, :]\n", + " \n", + " # Apply temperature scaling\n", + " logits = logits / temperature\n", + " \n", + " # Convert logits to probabilities using softmax\n", + " probs = F.softmax(logits, dim=-1)\n", + " \n", + " # Sample from the distribution\n", + " next_token_id = torch.multinomial(probs, num_samples=1).item()\n", + " \n", + " # Update the input_ids with the new token\n", + " tokenized_question['input_ids'] = torch.cat([tokenized_question['input_ids'], torch.tensor([[next_token_id]])], dim=1)\n", + " \n", + " # Decode the next token for readability\n", + " next_token = tokenizer.decode(next_token_id)\n", + " \n", + " print(f\"Generated token: {next_token}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 129, + "execution_state": "idle", + "id": "e8bffe5d-d830-4992-9381-b484672ffeda", + "metadata": {}, + "outputs": [ + { + "ename": "IndexError", + "evalue": "too many indices for tensor of dimension 1", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[129], line 35\u001b[0m\n\u001b[1;32m 33\u001b[0m \u001b[38;5;66;03m# Create candidates\u001b[39;00m\n\u001b[1;32m 34\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(beam_width):\n\u001b[0;32m---> 35\u001b[0m token_id \u001b[38;5;241m=\u001b[39m \u001b[43mtop_k_ids\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mi\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;241m.\u001b[39mitem()\n\u001b[1;32m 36\u001b[0m token_prob \u001b[38;5;241m=\u001b[39m top_k_probs[\u001b[38;5;241m0\u001b[39m, i]\u001b[38;5;241m.\u001b[39mitem()\n\u001b[1;32m 38\u001b[0m \u001b[38;5;66;03m# Create a new sequence by appending the token to the existing sequence\u001b[39;00m\n", + "\u001b[0;31mIndexError\u001b[0m: too many indices for tensor of dimension 1" + ] + } + ], + "source": [ + "import torch\n", + "import torch.nn.functional as F\n", + "\n", + "question = \"Question: Are Vampires real. Answer:\"\n", + "tokenized_question = tokenizer(question, return_tensors=\"pt\")\n", + "\n", + "# Parameters\n", + "beam_width = 3 # The number of beams to consider\n", + "max_length = 30 # Maximum number of tokens to generate\n", + "temperature = 1.0 # Temperature for softmax\n", + "\n", + "# Initialize beams\n", + "beams = [(tokenized_question['input_ids'], 0.0)] # Each beam is a tuple (sequence, score)\n", + "finished_beams = []\n", + "\n", + "with torch.no_grad():\n", + " for step in range(max_length):\n", + " all_candidates = []\n", + " \n", + " for seq, score in beams:\n", + " # Get logits for the last token in the sequence\n", + " logits = model(input_ids=seq).logits[0, -1, :]\n", + " \n", + " # Apply temperature scaling\n", + " logits = logits / temperature\n", + " \n", + " # Convert logits to probabilities using softmax\n", + " probs = F.softmax(logits, dim=-1)\n", + " \n", + " # Get top-k candidate tokens and their probabilities\n", + " top_k_probs, top_k_ids = torch.topk(probs, beam_width, dim=-1)\n", + " \n", + " # Create candidates\n", + " for i in range(beam_width):\n", + " token_id = top_k_ids[0, i].item()\n", + " token_prob = top_k_probs[0, i].item()\n", + " \n", + " # Create a new sequence by appending the token to the existing sequence\n", + " new_seq = torch.cat([seq, torch.tensor([[token_id]])], dim=1)\n", + " \n", + " # Update the score (cumulative log probability)\n", + " new_score = score + torch.log(torch.tensor(token_prob))\n", + " \n", + " # If the token is the end-of-sequence token, consider it a finished beam\n", + " if token_id == tokenizer.eos_token_id:\n", + " finished_beams.append((new_seq, new_score))\n", + " else:\n", + " all_candidates.append((new_seq, new_score))\n", + " \n", + " # Sort candidates by score (highest first) and select top-k\n", + " beams = sorted(all_candidates, key=lambda x: x[1], reverse=True)[:beam_width]\n", + " \n", + " # If no beams are left to explore, break\n", + " if not beams:\n", + " break\n", + " \n", + " # If we have finished beams, we can stop early\n", + " if len(finished_beams) >= beam_width:\n", + " break\n", + "\n", + "# Decode and print the top finished sequences\n", + "for seq, score in sorted(finished_beams, key=lambda x: x[1], reverse=True):\n", + " generated_text = tokenizer.decode(seq[0], skip_special_tokens=True)\n", + " print(f\"Generated text: {generated_text}\")\n", + " print(f\"Score: {score.item()}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "218621c4-2b18-4e6e-8fba-adde97038ae8", + "metadata": {}, + "outputs": [], + "source": [ + "WANDB_MODE=offline CUDA_VISIBLE_DEVICES= accelerate launch --mixed_precision bf16 finetune_QA.py \\\n", + "--model_direction ltr \\\n", + "--checkpoint_path /home/sipb/nlp-class-project/checkpoints/distilbert_base_ltr/epoch_3_checkpt \\\n", + "--tokenizer_name distilbert/distilbert-base-uncased \\\n", + "--warmup_steps 100 \\\n", + "--learning_rate 1e-5 \\\n", + "--per_device_train_batch_size 128 \\\n", + "--per_device_eval_batch_size 128 \\\n", + "--output_dir checkpoints/qa_distilbert_base_ltr/ \\\n", + "--eval_steps 38 \\\n", + "--block_size 128 \\\n", + "--num_train_epochs 50 \\\n", + "--weight_decay 1e-4\n", + "\n", + "yes but we overfit to this example\n", + "\n", + "oh wait we did actually" + ] + }, + { + "cell_type": "markdown", + "id": "f5e37c9a-156a-4fcc-bd05-5ae6c3671e62", + "metadata": {}, + "source": [ + "wait so now it does really good because we're actually doing it the same way that we trained it?\n", + "\n", + "\n", + "\n", + "thats fine, let's train it on everything again!\n", + "\n", + "\n", + "i don't think that's true... but low perplexity!\n", + "\n", + "\n", + "\n", + "the question was supposed to be, \n", + "what happens if you swallow a watermellon seed?" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/Riddles.ipynb b/notebooks/Riddles.ipynb new file mode 100644 index 0000000..c2c8309 --- /dev/null +++ b/notebooks/Riddles.ipynb @@ -0,0 +1,362 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 14, + "execution_state": "idle", + "id": "1ddfc692-bda7-4d38-a549-2fb0d40d437d", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/bert_base_rtl/epoch_3_checkpt were not used when initializing BertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of BertForMaskedLM were not initialized from the model checkpoint at /home/sipb/nlp-class-project/checkpoints/bert_base_rtl/epoch_3_checkpt and are newly initialized because the shapes did not match:\n", + "- bert.embeddings.position_embeddings.weight: found shape torch.Size([512, 768]) in the checkpoint and torch.Size([128, 768]) in the model instantiated\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" + ] + } + ], + "source": [ + "import math\n", + "import sys\n", + "\n", + "sys.path.append(\"..\")\n", + "\n", + "import torch\n", + "import transformers\n", + "\n", + "from utils import add_attn_hooks\n", + "\n", + "# tokenizer = transformers.AutoTokenizer.from_pretrained(\"distilbert/distilbert-base-uncased\")\n", + "# model = transformers.AutoModelForMaskedLM.from_pretrained(\"/home/sipb/nlp-class-project/checkpoints/distilbert_base_ltr/epoch_3_checkpt\", ignore_mismatched_sizes=True)\n", + "\n", + "tokenizer = transformers.AutoTokenizer.from_pretrained(\"bert-base-uncased\")\n", + "text_dir = \"rtl\"\n", + "# text_dir = \"ltr\"\n", + "model = transformers.AutoModelForMaskedLM.from_pretrained(f\"/home/sipb/nlp-class-project/checkpoints/bert_base_{text_dir}/epoch_3_checkpt\", ignore_mismatched_sizes=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "execution_state": "idle", + "id": "a732375b-1682-45c6-8df0-8db1458559c9", + "metadata": {}, + "outputs": [], + "source": [ + "add_attn_hooks(model, text_dir)\n", + "model.eval();" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "execution_state": "idle", + "id": "041d1702-5aaf-45f0-9413-4014b315d1ed", + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"/home/sipb/nlp-class-project/data/riddles.txt\", \"r\") as f:\n", + " riddles = [line.rstrip() for line in f.readlines()]" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "execution_state": "idle", + "id": "a4098975-2df6-4435-bc93-1a5afd6d7e68", + "metadata": {}, + "outputs": [], + "source": [ + "# CPU is fast enough\n", + "\n", + "ppls = []\n", + "for riddle in riddles:\n", + " batch = tokenizer([riddle], return_tensors=\"pt\", padding_side=\"left\" if text_dir == \"rtl\" else \"right\", padding=\"max_length\", max_length=128)\n", + " batch[\"labels\"] = batch[\"input_ids\"].clone()\n", + " batch[\"labels\"][batch[\"attention_mask\"] == 0] = -100\n", + " # batch = tokenizer([riddle], return_tensors=\"pt\")#, padding_side=\"left\" if text_dir == \"rtl\" else \"right\", padding=\"longest\", max_length=128)\n", + " # batch[\"labels\"] = batch[\"input_ids\"]\n", + " with torch.inference_mode():\n", + " output = model(**batch)\n", + " ppls.append(math.e ** output.loss.item())" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "execution_state": "idle", + "id": "c4a82af4-d0d8-415a-9135-3a1350c1402e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(522.113471240328, 'rtl')" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sum(ppls) / len(ppls), text_dir" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "execution_state": "idle", + "id": "84a95c66-6dd3-4ccb-96a2-96f38008f70e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(1007.5656859988405, 'ltr')" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sum(ppls) / len(ppls), text_dir" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "execution_state": "idle", + "id": "51ed80f1-a935-42bc-8194-832f91222c45", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(1007.5656309474507, 'ltr')" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sum(ppls) / len(ppls), text_dir" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "execution_state": "idle", + "id": "40a98c10-59c3-498a-a9e6-c23bd9437bc7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "937.8557468023619" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sum(ppls) / len(ppls)" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "execution_state": "idle", + "id": "80b22ba1-e5ba-4f1e-8038-158a2c2f37a6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'input_ids': tensor([[ 101, 1045, 2064, 2022, 2524, 1010, 2021, 1045, 2572, 2025,\n", + " 5024, 1012, 2054, 2572, 1045, 1029, 1037, 15117, 1012, 102,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0]]), 'labels': tensor([[ 101, 1045, 2064, 2022, 2524, 1010, 2021, 1045, 2572, 2025,\n", + " 5024, 1012, 2054, 2572, 1045, 1029, 1037, 15117, 1012, 102,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100]])}" + ] + }, + "execution_count": 58, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "batch" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "execution_state": "idle", + "id": "c68b5235-a4a7-4f38-9acb-f5072e546a96", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(array([ 4., 6., 11., 6., 5., 2., 1., 1., 2., 2.]),\n", + " array([ 613.56297843, 829.36555779, 1045.16813716, 1260.97071653,\n", + " 1476.77329589, 1692.57587526, 1908.37845463, 2124.18103399,\n", + " 2339.98361336, 2555.78619272, 2771.58877209]),\n", + " )" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAY8UlEQVR4nO3df5DVVf348dcCsoCyyy9ZIBdZf4ym4E+MECMdGIHQsBpHjRqiRlMxJQqFCozMQGscJjM1Z1KbUbRmBBt/MGMokiMiIKikoRgKqUCJ7PJDV2TP54+v3vEKqPS99ywLj8fMnXHf78P7nMvhzj69P3YrUkopAAAyadXcCwAA9i/iAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAsmrT3Av4uKampnjjjTeiY8eOUVFR0dzLAQA+g5RSbN68OXr16hWtWn3ycxt7XXy88cYbUVtb29zLAAD+B2vXro1DDjnkE8fsdfHRsWPHiA8WX1VV1dzLAQA+g4aGhqitrS18H/8ke118fPhSS1VVlfgAgBbms7xlwhtOAYCsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZtWnuBbBv6jPpweZewh57dcbI5l4CwH7BMx8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFntcXwsWLAgzj777OjVq1dUVFTEnDlzis6nlGLq1KnRs2fPaN++fQwdOjRefvnlUq4ZAGjB9jg+tm7dGscff3zcdNNNuzx//fXXx29/+9u45ZZbYtGiRXHggQfGsGHD4t133y3FegGAFq7Nnv6BESNGxIgRI3Z5LqUUM2fOjJ/97GcxatSoiIj405/+FDU1NTFnzpw4//zz//9XDAC0aCV9z8fq1atj3bp1MXTo0MKx6urqGDBgQCxcuHCXf6axsTEaGhqKbgDAvquk8bFu3bqIiKipqSk6XlNTUzj3cdOnT4/q6urCrba2tpRLAgD2Ms3+aZfJkydHfX194bZ27drmXhIAUEYljY8ePXpERMT69euLjq9fv75w7uMqKyujqqqq6AYA7LtKGh91dXXRo0ePmDdvXuFYQ0NDLFq0KAYOHFjKqQCAFmqPP+2yZcuWWLVqVeHr1atXx/Lly6NLly7Ru3fvGD9+fPzyl7+MI488Murq6mLKlCnRq1evOOecc0q9dgCgBdrj+FiyZEmcccYZha8nTJgQERFjxoyJO+64I6688srYunVrXHTRRbFp06Y47bTTYu7cudGuXbvSrhwAaJEqUkqpuRfxUQ0NDVFdXR319fXe/9GC9Zn0YHMvYY+9OmNkcy8BoMXak+/fzf5pFwBg/yI+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFYlj48dO3bElClToq6uLtq3bx+HH354XHPNNZFSKvVUAEAL1KbUF7zuuuvi5ptvjjvvvDOOPfbYWLJkSYwdOzaqq6vj8ssvL/V0AEALU/L4ePLJJ2PUqFExcuTIiIjo06dPzJo1K55++ulSTwUAtEAlf9nl1FNPjXnz5sVLL70UERHPPvtsPPHEEzFixIhdjm9sbIyGhoaiGwCw7yr5Mx+TJk2KhoaGOProo6N169axY8eOuPbaa2P06NG7HD99+vSYNm1aqZcBAOylSv7Mx5///Oe466674u67745nnnkm7rzzzvjNb34Td9555y7HT548Oerr6wu3tWvXlnpJAMBepOTPfEycODEmTZoU559/fkRE9OvXL1577bWYPn16jBkzZqfxlZWVUVlZWeplAAB7qZI/87Ft27Zo1ar4sq1bt46mpqZSTwUAtEAlf+bj7LPPjmuvvTZ69+4dxx57bCxbtixuuOGG+O53v1vqqQCAFqjk8XHjjTfGlClT4tJLL40NGzZEr1694vvf/35MnTq11FMBAC1QyeOjY8eOMXPmzJg5c2apLw0A7AP8bhcAICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALIqS3y8/vrr8a1vfSu6du0a7du3j379+sWSJUvKMRUA0MK0KfUF33777Rg0aFCcccYZ8fDDD8fBBx8cL7/8cnTu3LnUUwEALVDJ4+O6666L2trauP322wvH6urqSj0NANBClfxll7/+9a/Rv3//OPfcc6N79+5x4oknxm233bbb8Y2NjdHQ0FB0AwD2XRUppVTKC7Zr1y4iIiZMmBDnnntuLF68OK644oq45ZZbYsyYMTuN//nPfx7Tpk3b6Xh9fX1UVVWVcmktVp9JDzb3EthLvTpjZHMvASAiIhoaGqK6uvozff8ueXy0bds2+vfvH08++WTh2OWXXx6LFy+OhQsX7jS+sbExGhsbixZfW1srPj5CfLA74gPYW+xJfJT8ZZeePXvGMcccU3Ts85//fKxZs2aX4ysrK6OqqqroBgDsu0oeH4MGDYqVK1cWHXvppZfi0EMPLfVUAEALVPL4+OEPfxhPPfVU/OpXv4pVq1bF3XffHX/4wx9i3LhxpZ4KAGiBSh4fp5xySsyePTtmzZoVffv2jWuuuSZmzpwZo0ePLvVUAEALVPKf8xERcdZZZ8VZZ51VjksDAC2c3+0CAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGTVprkXAPzv+kx6sLmXsMdenTGyuZcANDPPfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZFX2+JgxY0ZUVFTE+PHjyz0VANAClDU+Fi9eHLfeemscd9xx5ZwGAGhByhYfW7ZsidGjR8dtt90WnTt3Ltc0AEALU7b4GDduXIwcOTKGDh36ieMaGxujoaGh6AYA7LvalOOi99xzTzzzzDOxePHiTx07ffr0mDZtWjmWsUt9Jj2YbS4AYGclf+Zj7dq1ccUVV8Rdd90V7dq1+9TxkydPjvr6+sJt7dq1pV4SALAXKfkzH0uXLo0NGzbESSedVDi2Y8eOWLBgQfzud7+LxsbGaN26deFcZWVlVFZWlnoZAMBequTxMWTIkHj++eeLjo0dOzaOPvrouOqqq4rCAwDY/5Q8Pjp27Bh9+/YtOnbggQdG165ddzoOAOx//IRTACCrsnza5ePmz5+fYxoAoAXwzAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmVPD6mT58ep5xySnTs2DG6d+8e55xzTqxcubLU0wAALVTJ4+Pxxx+PcePGxVNPPRWPPPJIbN++Pc4888zYunVrqacCAFqgNqW+4Ny5c4u+vuOOO6J79+6xdOnSGDx4cKmnAwBamJLHx8fV19dHRESXLl12eb6xsTEaGxsLXzc0NJR7SQBAMyprfDQ1NcX48eNj0KBB0bdv312OmT59ekybNq2cywD2In0mPdjcS9hjr84Y2dxL2G+0xH8fLVFz/5su66ddxo0bFytWrIh77rlnt2MmT54c9fX1hdvatWvLuSQAoJmV7ZmPyy67LB544IFYsGBBHHLIIbsdV1lZGZWVleVaBgCwlyl5fKSU4gc/+EHMnj075s+fH3V1daWeAgBowUoeH+PGjYu777477r///ujYsWOsW7cuIiKqq6ujffv2pZ4OAGhhSv6ej5tvvjnq6+vj9NNPj549exZu9957b6mnAgBaoLK87AIAsDt+twsAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkFWb5l4AwN6uz6QHm3sJsE/xzAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFZli4+bbrop+vTpE+3atYsBAwbE008/Xa6pAIAWpCzxce+998aECRPi6quvjmeeeSaOP/74GDZsWGzYsKEc0wEALUhZ4uOGG26ICy+8MMaOHRvHHHNM3HLLLdGhQ4f44x//WI7pAIAWpE2pL/jee+/F0qVLY/LkyYVjrVq1iqFDh8bChQt3Gt/Y2BiNjY2Fr+vr6yMioqGhodRLi4iIpsZtZbkuALQU5fge++E1U0qfOrbk8fHf//43duzYETU1NUXHa2pq4p///OdO46dPnx7Tpk3b6XhtbW2plwYARET1zPJde/PmzVFdXf2JY0oeH3tq8uTJMWHChMLXTU1NsXHjxujatWtUVFQ069rYWUNDQ9TW1sbatWujqqqquZfDx9ifvZ892rvZn/9dSik2b94cvXr1+tSxJY+Pbt26RevWrWP9+vVFx9evXx89evTYaXxlZWVUVlYWHevUqVOpl0WJVVVVeWDuxezP3s8e7d3sz//m057x+FDJ33Datm3bOPnkk2PevHmFY01NTTFv3rwYOHBgqacDAFqYsrzsMmHChBgzZkz0798/vvCFL8TMmTNj69atMXbs2HJMBwC0IGWJj/POOy/+85//xNSpU2PdunVxwgknxNy5c3d6EyotT2VlZVx99dU7vVTG3sH+7P3s0d7N/uRRkT7LZ2IAAErE73YBALISHwBAVuIDAMhKfAAAWYmP/dCCBQvi7LPPjl69ekVFRUXMmTOn6HxKKaZOnRo9e/aM9u3bx9ChQ+Pll18uGrNx48YYPXp0VFVVRadOneJ73/tebNmypWjMc889F1/60peiXbt2UVtbG9dff32W+9fSfdr+fOc734mKioqi2/Dhw4vG2J/ymT59epxyyinRsWPH6N69e5xzzjmxcuXKojHvvvtujBs3Lrp27RoHHXRQfOMb39jpBy+uWbMmRo4cGR06dIju3bvHxIkT4/333y8aM3/+/DjppJOisrIyjjjiiLjjjjuy3MeW7rPs0emnn77T4+jiiy8uGmOPyiix33nooYfST3/603TfffeliEizZ88uOj9jxoxUXV2d5syZk5599tn01a9+NdXV1aV33nmnMGb48OHp+OOPT0899VT6+9//no444oh0wQUXFM7X19enmpqaNHr06LRixYo0a9as1L59+3Trrbdmva8t0aftz5gxY9Lw4cPTm2++Wbht3LixaIz9KZ9hw4al22+/Pa1YsSItX748feUrX0m9e/dOW7ZsKYy5+OKLU21tbZo3b15asmRJ+uIXv5hOPfXUwvn3338/9e3bNw0dOjQtW7YsPfTQQ6lbt25p8uTJhTH/+te/UocOHdKECRPSCy+8kG688cbUunXrNHfu3Oz3uaX5LHv05S9/OV144YVFj6P6+vrCeXtUXuJjP/fxb25NTU2pR48e6de//nXh2KZNm1JlZWWaNWtWSimlF154IUVEWrx4cWHMww8/nCoqKtLrr7+eUkrp97//fercuXNqbGwsjLnqqqvSUUcdleme7Rt2Fx+jRo3a7Z+xP3lt2LAhRUR6/PHHU/rg8XLAAQekv/zlL4UxL774YoqItHDhwpQ+CMxWrVqldevWFcbcfPPNqaqqqrAnV155ZTr22GOL5jrvvPPSsGHDMt2zfcfH9yh9EB9XXHHFbv+MPSovL7tQZPXq1bFu3boYOnRo4Vh1dXUMGDAgFi5cGBERCxcujE6dOkX//v0LY4YOHRqtWrWKRYsWFcYMHjw42rZtWxgzbNiwWLlyZbz99ttZ79O+aP78+dG9e/c46qij4pJLLom33nqrcM7+5FVfXx8REV26dImIiKVLl8b27duLHkNHH3109O7du+gx1K9fv6IfvDhs2LBoaGiIf/zjH4UxH73Gh2M+vAaf3cf36EN33XVXdOvWLfr27RuTJ0+Obdu2Fc7Zo/Jq9t9qy95l3bp1ERE7/TTampqawrl169ZF9+7di863adMmunTpUjSmrq5up2t8eK5z585lvR/7suHDh8fXv/71qKuri1deeSV+8pOfxIgRI2LhwoXRunVr+5NRU1NTjB8/PgYNGhR9+/aN+ODvr23btjv9gsyPP4Z29RiLjzwGdzemoaEh3nnnnWjfvn1Z79u+Yld7FBHxzW9+Mw499NDo1atXPPfcc3HVVVfFypUr47777ouwR2UnPqCFOf/88wv/3a9fvzjuuOPi8MMPj/nz58eQIUOadW37m3HjxsWKFSviiSeeaO6lsBu726OLLrqo8N/9+vWLnj17xpAhQ+KVV16Jww8/vBlWun/xsgtFevToERGx0zvz169fXzjXo0eP2LBhQ9H5999/PzZu3Fg0ZlfX+OgclMZhhx0W3bp1i1WrVkXYn2wuu+yyeOCBB+Kxxx6LQw45pHC8R48e8d5778WmTZuKxn/8MfRpf/+7G1NVVeX/qD+j3e3RrgwYMCAiouhxZI/KR3xQpK6uLnr06BHz5s0rHGtoaIhFixbFwIEDIyJi4MCBsWnTpli6dGlhzKOPPhpNTU2FB/DAgQNjwYIFsX379sKYRx55JI466ihP6ZfYv//973jrrbeiZ8+eEfan7FJKcdlll8Xs2bPj0Ucf3enlq5NPPjkOOOCAosfQypUrY82aNUWPoeeff74oEh955JGoqqqKY445pjDmo9f4cMyH12D3Pm2PdmX58uUREUWPI3tURs39jlfy27x5c1q2bFlatmxZioh0ww03pGXLlqXXXnstpQ8+atupU6d0//33p+eeey6NGjVqlx+1PfHEE9OiRYvSE088kY488siij3Ju2rQp1dTUpG9/+9tpxYoV6Z577kkdOnTwUc7P4JP2Z/PmzenHP/5xWrhwYVq9enX629/+lk466aR05JFHpnfffbdwDftTPpdcckmqrq5O8+fPL/qY5rZt2wpjLr744tS7d+/06KOPpiVLlqSBAwemgQMHFs5/+DHOM888My1fvjzNnTs3HXzwwbv8GOfEiRPTiy++mG666SYf4/yMPm2PVq1alX7xi1+kJUuWpNWrV6f7778/HXbYYWnw4MGFa9ij8hIf+6HHHnssRcROtzFjxqT0wcdtp0yZkmpqalJlZWUaMmRIWrlyZdE13nrrrXTBBRekgw46KFVVVaWxY8emzZs3F4159tln02mnnZYqKyvT5z73uTRjxoys97Ol+qT92bZtWzrzzDPTwQcfnA444IB06KGHpgsvvLDo44DJ/pTVrvYmItLtt99eGPPOO++kSy+9NHXu3Dl16NAhfe1rX0tvvvlm0XVeffXVNGLEiNS+ffvUrVu39KMf/Sht3769aMxjjz2WTjjhhNS2bdt02GGHFc3B7n3aHq1ZsyYNHjw4denSJVVWVqYjjjgiTZw4sejnfCR7VFYV6f9tFABAFt7zAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACy+j/J4r9i/nZCRAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "plt.hist(ppls)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "execution_state": "idle", + "id": "8acad3ce-905d-455e-af5d-9770495f374a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414]" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ppls" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "86daa05b-5784-457b-b65e-8b8395128d6f", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/Riddles_FixedPos.ipynb b/notebooks/Riddles_FixedPos.ipynb new file mode 100644 index 0000000..5e42e0b --- /dev/null +++ b/notebooks/Riddles_FixedPos.ipynb @@ -0,0 +1,302 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "execution_state": "idle", + "id": "1ddfc692-bda7-4d38-a549-2fb0d40d437d", + "metadata": {}, + "outputs": [], + "source": [ + "import math\n", + "import os\n", + "import sys\n", + "\n", + "sys.path.append(\"..\")\n", + "\n", + "import torch\n", + "import transformers\n", + "from safetensors import safe_open\n", + "\n", + "from utils import add_attn_hooks\n", + "\n", + "# text_dir = \"rtl\"\n", + "text_dir = \"ltr\"\n", + "# tokenizer = transformers.AutoTokenizer.from_pretrained(\"bert-base-uncased\")\n", + "# model = transformers.AutoModelForMaskedLM.from_pretrained(f\"/home/sipb/nlp-class-project/checkpoints/bert_base_{text_dir}/epoch_3_checkpt\", ignore_mismatched_sizes=True)\n", + "\n", + "tokenizer = transformers.AutoTokenizer.from_pretrained(\"distilbert/distilbert-base-uncased\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "execution_state": "idle", + "id": "eaf99031-9141-43dd-89ba-be9b8e63a1ba", + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"/home/sipb/nlp-class-project/data/riddles.txt\", \"r\") as f:\n", + " riddles_qa = [line.rstrip() for line in f.readlines()]\n", + "\n", + "with open(\"/home/sipb/nlp-class-project/data/ltr_riddles.txt\", \"r\") as f:\n", + " riddles_aq = [line.rstrip() for line in f.readlines()]" + ] + }, + { + "cell_type": "code", + "execution_count": 167, + "execution_state": "idle", + "id": "94da0be0-d6ef-46be-9fff-4ebf022e4fed", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/bert_6_ltr_scratch/epoch_3_checkpt were not used when initializing BertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/bert_6_rtl_scratch/epoch_3_checkpt were not used when initializing BertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/bert_11_ltr_scratch/epoch_3_checkpt were not used when initializing BertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/bert_11_rtl_scratch/epoch_3_checkpt were not used when initializing BertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/bert_19_ltr_scratch/epoch_3_checkpt were not used when initializing BertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/bert_19_rtl_scratch/epoch_3_checkpt were not used when initializing BertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/bert_35_ltr_scratch/epoch_3_checkpt were not used when initializing BertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/bert_35_rtl_scratch/epoch_3_checkpt were not used when initializing BertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/distilbert_base_ltr_scratch/epoch_3_checkpt were not used when initializing DistilBertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing DistilBertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing DistilBertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/distilbert_base_rtl_scratch/epoch_3_checkpt were not used when initializing DistilBertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing DistilBertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing DistilBertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n" + ] + } + ], + "source": [ + "results = []\n", + "path_prefixes = [f\"bert_{size}\" for size in (6, 11, 19, 35)] + [\"distilbert_base\"]\n", + "for path_prefix in path_prefixes:\n", + " for text_dir in (\"ltr\", \"rtl\"):\n", + " checkpt_dir = f\"/home/sipb/nlp-class-project/checkpoints/{path_prefix}_{text_dir}_scratch/epoch_3_checkpt\"\n", + "\n", + "# path_prefixes = [\"distilbert_base\", \"bert_base\", \"bert_large\"]\n", + "# for path_prefix in path_prefixes:\n", + "# for text_dir in (\"ltr\", \"rtl\"):\n", + " # checkpt_dir = f\"/home/sipb/nlp-class-project/checkpoints/{path_prefix}_{text_dir}/epoch_3_checkpt\"\n", + " # model = load_checkpt(f\"/home/sipb/nlp-class-project/checkpoints/{path_prefix}_{text_dir}/epoch_3_checkpt\")\n", + " # config = transformers.AutoConfig.from_pretrained(os.path.join(checkpt_dir, \"config.json\"))\n", + " # config.max_position_embeddings = 512\n", + " try:\n", + " model = transformers.AutoModelForMaskedLM.from_pretrained(checkpt_dir)\n", + " except:\n", + " config = transformers.AutoConfig.from_pretrained(os.path.join(checkpt_dir, \"config.json\"))\n", + " config.max_position_embeddings = 512\n", + " model = transformers.AutoModelForMaskedLM.from_pretrained(checkpt_dir, config=config)\n", + " \n", + " add_attn_hooks(model, text_dir)\n", + " model.eval();\n", + "\n", + " for dataset_type, dataset in [\n", + " (\"qa\", riddles_qa),\n", + " (\"aq\", riddles_aq),\n", + " ]:\n", + " ppls = []\n", + " for riddle in dataset:\n", + " batch = tokenizer([riddle], return_tensors=\"pt\", padding_side=\"left\" if text_dir == \"rtl\" else \"right\", padding=\"max_length\", max_length=128)\n", + " batch[\"labels\"] = batch[\"input_ids\"].clone()\n", + " batch[\"labels\"][batch[\"attention_mask\"] == 0] = -100\n", + " with torch.inference_mode():\n", + " output = model(**batch)\n", + " ppls.append(math.e ** output.loss.item())\n", + "\n", + " results.append((sum(ppls) / len(ppls), dataset_type, text_dir, path_prefix))" + ] + }, + { + "cell_type": "code", + "execution_count": 166, + "execution_state": "idle", + "id": "bdee66ad-65ad-40c7-ac86-9a2d6b8fba02", + "metadata": {}, + "outputs": [], + "source": [ + "to_params = {\n", + " \"bert_6\": 6,\n", + " \"bert_11\": 11,\n", + " \"bert_19\": 19,\n", + " \"bert_35\": 35,\n", + " \"distilbert_base\": 67,\n", + " \"bert_base\": 110,\n", + " \"bert_large\": 335,\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 168, + "execution_state": "idle", + "id": "d1668465-fe85-4310-8d88-031d4b8d361f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "LTR & 6M & AQ & 2420 \\\\\n", + "RTL & 6M & AQ & 2570 \\\\\n", + "LTR & 11M & AQ & 1930 \\\\\n", + "RTL & 11M & AQ & 2710 \\\\\n", + "LTR & 19M & AQ & 2930 \\\\\n", + "RTL & 19M & AQ & 5820 \\\\\n", + "LTR & 35M & AQ & 6270 \\\\\n", + "RTL & 35M & AQ & 11600 \\\\\n", + "LTR & 67M & AQ & 9790 \\\\\n", + "RTL & 67M & AQ & 32500 \\\\\n", + "LTR & 6M & QA & 1960 \\\\\n", + "RTL & 6M & QA & 1770 \\\\\n", + "LTR & 11M & QA & 1630 \\\\\n", + "RTL & 11M & QA & 1710 \\\\\n", + "LTR & 19M & QA & 2610 \\\\\n", + "RTL & 19M & QA & 3330 \\\\\n", + "LTR & 35M & QA & 5080 \\\\\n", + "RTL & 35M & QA & 5410 \\\\\n", + "LTR & 67M & QA & 7160 \\\\\n", + "RTL & 67M & QA & 27600 \\\\\n" + ] + } + ], + "source": [ + "for ppl, task, text_dir, path_prefix in sorted(results, key=lambda x: (x[1], to_params[x[3]], x[2])):\n", + " ppl = int(float(f\"{ppl:.3g}\"))\n", + " print(rf\"{text_dir.upper()} & {to_params[path_prefix]}M & {task.upper()} & {ppl} \\\\\")" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "execution_state": "idle", + "id": "8894ca16-58e3-4448-bec8-c962f5135737", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "the more you take, the more you leave behind. what am i? @ @ @ @ @ @ @ @ @ @ @ @ @ the @ @ @ @ ( the the the. @ the @ @ ( @ @ ( @ @ @ @ ( the.. @ ( @ ) @ the @ the the\n" + ] + } + ], + "source": [ + "# input_text = [\"The more you take, the more you leave behind. What am I?\"]\n", + "# batch = tokenizer(input_text, return_tensors=\"pt\", padding_side=\"right\", padding=\"max_length\", max_length=64)\n", + "# output_ids = model.generate(batch['input_ids'], max_length=128, do_sample=False) # do_sample=False ensures greedy decoding\n", + "# decoded_output = tokenizer.decode(output_ids[0], skip_special_tokens=True)\n", + "# print(decoded_output)" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "execution_state": "idle", + "id": "a4098975-2df6-4435-bc93-1a5afd6d7e68", + "metadata": {}, + "outputs": [], + "source": [ + "# CPU is fast enough\n", + "\n", + "ppls = []\n", + "for riddle in riddles:\n", + " batch = tokenizer([riddle], return_tensors=\"pt\", padding_side=\"left\" if text_dir == \"rtl\" else \"right\", padding=\"max_length\", max_length=128)\n", + " batch[\"labels\"] = batch[\"input_ids\"].clone()\n", + " batch[\"labels\"][batch[\"attention_mask\"] == 0] = -100\n", + " # batch = tokenizer([riddle], return_tensors=\"pt\")#, padding_side=\"left\" if text_dir == \"rtl\" else \"right\", padding=\"longest\", max_length=128)\n", + " # batch[\"labels\"] = batch[\"input_ids\"]\n", + " with torch.inference_mode():\n", + " output = model(**batch)\n", + " ppls.append(math.e ** output.loss.item())" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "execution_state": "idle", + "id": "c68b5235-a4a7-4f38-9acb-f5072e546a96", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(array([ 4., 6., 11., 6., 5., 2., 1., 1., 2., 2.]),\n", + " array([ 613.56297843, 829.36555779, 1045.16813716, 1260.97071653,\n", + " 1476.77329589, 1692.57587526, 1908.37845463, 2124.18103399,\n", + " 2339.98361336, 2555.78619272, 2771.58877209]),\n", + " )" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAY8UlEQVR4nO3df5DVVf348dcCsoCyyy9ZIBdZf4ym4E+MECMdGIHQsBpHjRqiRlMxJQqFCozMQGscJjM1Z1KbUbRmBBt/MGMokiMiIKikoRgKqUCJ7PJDV2TP54+v3vEKqPS99ywLj8fMnXHf78P7nMvhzj69P3YrUkopAAAyadXcCwAA9i/iAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAsmrT3Av4uKampnjjjTeiY8eOUVFR0dzLAQA+g5RSbN68OXr16hWtWn3ycxt7XXy88cYbUVtb29zLAAD+B2vXro1DDjnkE8fsdfHRsWPHiA8WX1VV1dzLAQA+g4aGhqitrS18H/8ke118fPhSS1VVlfgAgBbms7xlwhtOAYCsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZtWnuBbBv6jPpweZewh57dcbI5l4CwH7BMx8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFntcXwsWLAgzj777OjVq1dUVFTEnDlzis6nlGLq1KnRs2fPaN++fQwdOjRefvnlUq4ZAGjB9jg+tm7dGscff3zcdNNNuzx//fXXx29/+9u45ZZbYtGiRXHggQfGsGHD4t133y3FegGAFq7Nnv6BESNGxIgRI3Z5LqUUM2fOjJ/97GcxatSoiIj405/+FDU1NTFnzpw4//zz//9XDAC0aCV9z8fq1atj3bp1MXTo0MKx6urqGDBgQCxcuHCXf6axsTEaGhqKbgDAvquk8bFu3bqIiKipqSk6XlNTUzj3cdOnT4/q6urCrba2tpRLAgD2Ms3+aZfJkydHfX194bZ27drmXhIAUEYljY8ePXpERMT69euLjq9fv75w7uMqKyujqqqq6AYA7LtKGh91dXXRo0ePmDdvXuFYQ0NDLFq0KAYOHFjKqQCAFmqPP+2yZcuWWLVqVeHr1atXx/Lly6NLly7Ru3fvGD9+fPzyl7+MI488Murq6mLKlCnRq1evOOecc0q9dgCgBdrj+FiyZEmcccYZha8nTJgQERFjxoyJO+64I6688srYunVrXHTRRbFp06Y47bTTYu7cudGuXbvSrhwAaJEqUkqpuRfxUQ0NDVFdXR319fXe/9GC9Zn0YHMvYY+9OmNkcy8BoMXak+/fzf5pFwBg/yI+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFYlj48dO3bElClToq6uLtq3bx+HH354XHPNNZFSKvVUAEAL1KbUF7zuuuvi5ptvjjvvvDOOPfbYWLJkSYwdOzaqq6vj8ssvL/V0AEALU/L4ePLJJ2PUqFExcuTIiIjo06dPzJo1K55++ulSTwUAtEAlf9nl1FNPjXnz5sVLL70UERHPPvtsPPHEEzFixIhdjm9sbIyGhoaiGwCw7yr5Mx+TJk2KhoaGOProo6N169axY8eOuPbaa2P06NG7HD99+vSYNm1aqZcBAOylSv7Mx5///Oe466674u67745nnnkm7rzzzvjNb34Td9555y7HT548Oerr6wu3tWvXlnpJAMBepOTPfEycODEmTZoU559/fkRE9OvXL1577bWYPn16jBkzZqfxlZWVUVlZWeplAAB7qZI/87Ft27Zo1ar4sq1bt46mpqZSTwUAtEAlf+bj7LPPjmuvvTZ69+4dxx57bCxbtixuuOGG+O53v1vqqQCAFqjk8XHjjTfGlClT4tJLL40NGzZEr1694vvf/35MnTq11FMBAC1QyeOjY8eOMXPmzJg5c2apLw0A7AP8bhcAICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALIqS3y8/vrr8a1vfSu6du0a7du3j379+sWSJUvKMRUA0MK0KfUF33777Rg0aFCcccYZ8fDDD8fBBx8cL7/8cnTu3LnUUwEALVDJ4+O6666L2trauP322wvH6urqSj0NANBClfxll7/+9a/Rv3//OPfcc6N79+5x4oknxm233bbb8Y2NjdHQ0FB0AwD2XRUppVTKC7Zr1y4iIiZMmBDnnntuLF68OK644oq45ZZbYsyYMTuN//nPfx7Tpk3b6Xh9fX1UVVWVcmktVp9JDzb3EthLvTpjZHMvASAiIhoaGqK6uvozff8ueXy0bds2+vfvH08++WTh2OWXXx6LFy+OhQsX7jS+sbExGhsbixZfW1srPj5CfLA74gPYW+xJfJT8ZZeePXvGMcccU3Ts85//fKxZs2aX4ysrK6OqqqroBgDsu0oeH4MGDYqVK1cWHXvppZfi0EMPLfVUAEALVPL4+OEPfxhPPfVU/OpXv4pVq1bF3XffHX/4wx9i3LhxpZ4KAGiBSh4fp5xySsyePTtmzZoVffv2jWuuuSZmzpwZo0ePLvVUAEALVPKf8xERcdZZZ8VZZ51VjksDAC2c3+0CAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGTVprkXAPzv+kx6sLmXsMdenTGyuZcANDPPfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZFX2+JgxY0ZUVFTE+PHjyz0VANAClDU+Fi9eHLfeemscd9xx5ZwGAGhByhYfW7ZsidGjR8dtt90WnTt3Ltc0AEALU7b4GDduXIwcOTKGDh36ieMaGxujoaGh6AYA7LvalOOi99xzTzzzzDOxePHiTx07ffr0mDZtWjmWsUt9Jj2YbS4AYGclf+Zj7dq1ccUVV8Rdd90V7dq1+9TxkydPjvr6+sJt7dq1pV4SALAXKfkzH0uXLo0NGzbESSedVDi2Y8eOWLBgQfzud7+LxsbGaN26deFcZWVlVFZWlnoZAMBequTxMWTIkHj++eeLjo0dOzaOPvrouOqqq4rCAwDY/5Q8Pjp27Bh9+/YtOnbggQdG165ddzoOAOx//IRTACCrsnza5ePmz5+fYxoAoAXwzAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmVPD6mT58ep5xySnTs2DG6d+8e55xzTqxcubLU0wAALVTJ4+Pxxx+PcePGxVNPPRWPPPJIbN++Pc4888zYunVrqacCAFqgNqW+4Ny5c4u+vuOOO6J79+6xdOnSGDx4cKmnAwBamJLHx8fV19dHRESXLl12eb6xsTEaGxsLXzc0NJR7SQBAMyprfDQ1NcX48eNj0KBB0bdv312OmT59ekybNq2cywD2In0mPdjcS9hjr84Y2dxL2G+0xH8fLVFz/5su66ddxo0bFytWrIh77rlnt2MmT54c9fX1hdvatWvLuSQAoJmV7ZmPyy67LB544IFYsGBBHHLIIbsdV1lZGZWVleVaBgCwlyl5fKSU4gc/+EHMnj075s+fH3V1daWeAgBowUoeH+PGjYu777477r///ujYsWOsW7cuIiKqq6ujffv2pZ4OAGhhSv6ej5tvvjnq6+vj9NNPj549exZu9957b6mnAgBaoLK87AIAsDt+twsAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkFWb5l4AwN6uz6QHm3sJsE/xzAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFZli4+bbrop+vTpE+3atYsBAwbE008/Xa6pAIAWpCzxce+998aECRPi6quvjmeeeSaOP/74GDZsWGzYsKEc0wEALUhZ4uOGG26ICy+8MMaOHRvHHHNM3HLLLdGhQ4f44x//WI7pAIAWpE2pL/jee+/F0qVLY/LkyYVjrVq1iqFDh8bChQt3Gt/Y2BiNjY2Fr+vr6yMioqGhodRLi4iIpsZtZbkuALQU5fge++E1U0qfOrbk8fHf//43duzYETU1NUXHa2pq4p///OdO46dPnx7Tpk3b6XhtbW2plwYARET1zPJde/PmzVFdXf2JY0oeH3tq8uTJMWHChMLXTU1NsXHjxujatWtUVFQ069rYWUNDQ9TW1sbatWujqqqquZfDx9ifvZ892rvZn/9dSik2b94cvXr1+tSxJY+Pbt26RevWrWP9+vVFx9evXx89evTYaXxlZWVUVlYWHevUqVOpl0WJVVVVeWDuxezP3s8e7d3sz//m057x+FDJ33Datm3bOPnkk2PevHmFY01NTTFv3rwYOHBgqacDAFqYsrzsMmHChBgzZkz0798/vvCFL8TMmTNj69atMXbs2HJMBwC0IGWJj/POOy/+85//xNSpU2PdunVxwgknxNy5c3d6EyotT2VlZVx99dU7vVTG3sH+7P3s0d7N/uRRkT7LZ2IAAErE73YBALISHwBAVuIDAMhKfAAAWYmP/dCCBQvi7LPPjl69ekVFRUXMmTOn6HxKKaZOnRo9e/aM9u3bx9ChQ+Pll18uGrNx48YYPXp0VFVVRadOneJ73/tebNmypWjMc889F1/60peiXbt2UVtbG9dff32W+9fSfdr+fOc734mKioqi2/Dhw4vG2J/ymT59epxyyinRsWPH6N69e5xzzjmxcuXKojHvvvtujBs3Lrp27RoHHXRQfOMb39jpBy+uWbMmRo4cGR06dIju3bvHxIkT4/333y8aM3/+/DjppJOisrIyjjjiiLjjjjuy3MeW7rPs0emnn77T4+jiiy8uGmOPyiix33nooYfST3/603TfffeliEizZ88uOj9jxoxUXV2d5syZk5599tn01a9+NdXV1aV33nmnMGb48OHp+OOPT0899VT6+9//no444oh0wQUXFM7X19enmpqaNHr06LRixYo0a9as1L59+3Trrbdmva8t0aftz5gxY9Lw4cPTm2++Wbht3LixaIz9KZ9hw4al22+/Pa1YsSItX748feUrX0m9e/dOW7ZsKYy5+OKLU21tbZo3b15asmRJ+uIXv5hOPfXUwvn3338/9e3bNw0dOjQtW7YsPfTQQ6lbt25p8uTJhTH/+te/UocOHdKECRPSCy+8kG688cbUunXrNHfu3Oz3uaX5LHv05S9/OV144YVFj6P6+vrCeXtUXuJjP/fxb25NTU2pR48e6de//nXh2KZNm1JlZWWaNWtWSimlF154IUVEWrx4cWHMww8/nCoqKtLrr7+eUkrp97//fercuXNqbGwsjLnqqqvSUUcdleme7Rt2Fx+jRo3a7Z+xP3lt2LAhRUR6/PHHU/rg8XLAAQekv/zlL4UxL774YoqItHDhwpQ+CMxWrVqldevWFcbcfPPNqaqqqrAnV155ZTr22GOL5jrvvPPSsGHDMt2zfcfH9yh9EB9XXHHFbv+MPSovL7tQZPXq1bFu3boYOnRo4Vh1dXUMGDAgFi5cGBERCxcujE6dOkX//v0LY4YOHRqtWrWKRYsWFcYMHjw42rZtWxgzbNiwWLlyZbz99ttZ79O+aP78+dG9e/c46qij4pJLLom33nqrcM7+5FVfXx8REV26dImIiKVLl8b27duLHkNHH3109O7du+gx1K9fv6IfvDhs2LBoaGiIf/zjH4UxH73Gh2M+vAaf3cf36EN33XVXdOvWLfr27RuTJ0+Obdu2Fc7Zo/Jq9t9qy95l3bp1ERE7/TTampqawrl169ZF9+7di863adMmunTpUjSmrq5up2t8eK5z585lvR/7suHDh8fXv/71qKuri1deeSV+8pOfxIgRI2LhwoXRunVr+5NRU1NTjB8/PgYNGhR9+/aN+ODvr23btjv9gsyPP4Z29RiLjzwGdzemoaEh3nnnnWjfvn1Z79u+Yld7FBHxzW9+Mw499NDo1atXPPfcc3HVVVfFypUr47777ouwR2UnPqCFOf/88wv/3a9fvzjuuOPi8MMPj/nz58eQIUOadW37m3HjxsWKFSviiSeeaO6lsBu726OLLrqo8N/9+vWLnj17xpAhQ+KVV16Jww8/vBlWun/xsgtFevToERGx0zvz169fXzjXo0eP2LBhQ9H5999/PzZu3Fg0ZlfX+OgclMZhhx0W3bp1i1WrVkXYn2wuu+yyeOCBB+Kxxx6LQw45pHC8R48e8d5778WmTZuKxn/8MfRpf/+7G1NVVeX/qD+j3e3RrgwYMCAiouhxZI/KR3xQpK6uLnr06BHz5s0rHGtoaIhFixbFwIEDIyJi4MCBsWnTpli6dGlhzKOPPhpNTU2FB/DAgQNjwYIFsX379sKYRx55JI466ihP6ZfYv//973jrrbeiZ8+eEfan7FJKcdlll8Xs2bPj0Ucf3enlq5NPPjkOOOCAosfQypUrY82aNUWPoeeff74oEh955JGoqqqKY445pjDmo9f4cMyH12D3Pm2PdmX58uUREUWPI3tURs39jlfy27x5c1q2bFlatmxZioh0ww03pGXLlqXXXnstpQ8+atupU6d0//33p+eeey6NGjVqlx+1PfHEE9OiRYvSE088kY488siij3Ju2rQp1dTUpG9/+9tpxYoV6Z577kkdOnTwUc7P4JP2Z/PmzenHP/5xWrhwYVq9enX629/+lk466aR05JFHpnfffbdwDftTPpdcckmqrq5O8+fPL/qY5rZt2wpjLr744tS7d+/06KOPpiVLlqSBAwemgQMHFs5/+DHOM888My1fvjzNnTs3HXzwwbv8GOfEiRPTiy++mG666SYf4/yMPm2PVq1alX7xi1+kJUuWpNWrV6f7778/HXbYYWnw4MGFa9ij8hIf+6HHHnssRcROtzFjxqT0wcdtp0yZkmpqalJlZWUaMmRIWrlyZdE13nrrrXTBBRekgw46KFVVVaWxY8emzZs3F4159tln02mnnZYqKyvT5z73uTRjxoys97Ol+qT92bZtWzrzzDPTwQcfnA444IB06KGHpgsvvLDo44DJ/pTVrvYmItLtt99eGPPOO++kSy+9NHXu3Dl16NAhfe1rX0tvvvlm0XVeffXVNGLEiNS+ffvUrVu39KMf/Sht3769aMxjjz2WTjjhhNS2bdt02GGHFc3B7n3aHq1ZsyYNHjw4denSJVVWVqYjjjgiTZw4sejnfCR7VFYV6f9tFABAFt7zAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACy+j/J4r9i/nZCRAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "plt.hist(ppls)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "86daa05b-5784-457b-b65e-8b8395128d6f", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/Riddles_FixedPos_QAChars.ipynb b/notebooks/Riddles_FixedPos_QAChars.ipynb new file mode 100644 index 0000000..0283bb2 --- /dev/null +++ b/notebooks/Riddles_FixedPos_QAChars.ipynb @@ -0,0 +1,345 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "execution_state": "idle", + "id": "1ddfc692-bda7-4d38-a549-2fb0d40d437d", + "metadata": {}, + "outputs": [], + "source": [ + "import math\n", + "import os\n", + "import re\n", + "import sys\n", + "\n", + "sys.path.append(\"..\")\n", + "\n", + "import torch\n", + "import transformers\n", + "from safetensors import safe_open\n", + "\n", + "from utils import add_attn_hooks\n", + "\n", + "# text_dir = \"rtl\"\n", + "text_dir = \"ltr\"\n", + "# tokenizer = transformers.AutoTokenizer.from_pretrained(\"bert-base-uncased\")\n", + "# model = transformers.AutoModelForMaskedLM.from_pretrained(f\"/home/sipb/nlp-class-project/checkpoints/bert_base_{text_dir}/epoch_3_checkpt\", ignore_mismatched_sizes=True)\n", + "\n", + "tokenizer = transformers.AutoTokenizer.from_pretrained(\"distilbert/distilbert-base-uncased\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "execution_state": "idle", + "id": "5bd236ae-119c-4ea6-9a0f-03272f528caf", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 19, + "execution_state": "idle", + "id": "eaf99031-9141-43dd-89ba-be9b8e63a1ba", + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"/home/sipb/nlp-class-project/data/riddles.txt\", \"r\") as f:\n", + " riddles = [line.rstrip() for line in f.readlines()]\n", + "# with open(\"/home/sipb/nlp-class-project/data/ltr_riddles.txt\", \"r\") as f:\n", + "# riddles_aq = [line.rstrip() for line in f.readlines()]" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "execution_state": "idle", + "id": "9652550f-61b9-4b9b-ad10-9d9873a9e80b", + "metadata": {}, + "outputs": [], + "source": [ + "pattern = r\"^(.*)(What am I\\?\\s*)(.+)$\"\n", + "\n", + "riddles_qa = []\n", + "riddles_aq = []\n", + "for riddle in riddles:\n", + " riddles_qa.append(re.sub(pattern, r\"Q: \\1\\2A: \\3\", riddle))\n", + " riddles_aq.append(re.sub(pattern, r\"A: \\3 Q: \\1\\2\", riddle).rstrip())" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "execution_state": "idle", + "id": "94da0be0-d6ef-46be-9fff-4ebf022e4fed", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/distilbert_base_ltr/epoch_3_checkpt were not used when initializing DistilBertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing DistilBertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing DistilBertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/distilbert_base_rtl/epoch_3_checkpt were not used when initializing DistilBertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing DistilBertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing DistilBertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/bert_base_ltr/epoch_3_checkpt were not used when initializing BertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/bert_base_rtl/epoch_3_checkpt were not used when initializing BertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/bert_large_ltr/epoch_3_checkpt were not used when initializing BertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/bert_large_rtl/epoch_3_checkpt were not used when initializing BertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n" + ] + } + ], + "source": [ + "results = []\n", + "# path_prefixes = [f\"bert_{size}\" for size in (6, 11, 19, 35)] + [\"distilbert_base\"]\n", + "# for path_prefix in path_prefixes:\n", + "# for text_dir in (\"ltr\", \"rtl\"):\n", + "# checkpt_dir = f\"/home/sipb/nlp-class-project/checkpoints/{path_prefix}_{text_dir}_scratch/epoch_3_checkpt\"\n", + "\n", + "path_prefixes = [\"distilbert_base\", \"bert_base\", \"bert_large\"]\n", + "for path_prefix in path_prefixes:\n", + " for text_dir in (\"ltr\", \"rtl\"):\n", + " checkpt_dir = f\"/home/sipb/nlp-class-project/checkpoints/{path_prefix}_{text_dir}/epoch_3_checkpt\"\n", + " try:\n", + " model = transformers.AutoModelForMaskedLM.from_pretrained(checkpt_dir)\n", + " except:\n", + " config = transformers.AutoConfig.from_pretrained(os.path.join(checkpt_dir, \"config.json\"))\n", + " config.max_position_embeddings = 512\n", + " model = transformers.AutoModelForMaskedLM.from_pretrained(checkpt_dir, config=config)\n", + " \n", + " add_attn_hooks(model, text_dir)\n", + " model.eval();\n", + "\n", + " for dataset_type, dataset in [\n", + " (\"qa\", riddles_qa),\n", + " (\"aq\", riddles_aq),\n", + " ]:\n", + " ppls = []\n", + " for riddle in dataset:\n", + " batch = tokenizer([riddle], return_tensors=\"pt\", padding_side=\"left\" if text_dir == \"rtl\" else \"right\", padding=\"max_length\", max_length=128)\n", + " batch[\"labels\"] = batch[\"input_ids\"].clone()\n", + " batch[\"labels\"][batch[\"attention_mask\"] == 0] = -100\n", + " with torch.inference_mode():\n", + " output = model(**batch)\n", + " ppls.append(math.e ** output.loss.item())\n", + "\n", + " results.append((sum(ppls) / len(ppls), dataset_type, text_dir, path_prefix))" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "execution_state": "idle", + "id": "bdee66ad-65ad-40c7-ac86-9a2d6b8fba02", + "metadata": {}, + "outputs": [], + "source": [ + "to_params = {\n", + " \"bert_6\": 6,\n", + " \"bert_11\": 11,\n", + " \"bert_19\": 19,\n", + " \"bert_35\": 35,\n", + " \"distilbert_base\": 67,\n", + " \"bert_base\": 110,\n", + " \"bert_large\": 335,\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "execution_state": "idle", + "id": "d1668465-fe85-4310-8d88-031d4b8d361f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "LTR & 6M & AQ & 1980 \\\\\n", + "RTL & 6M & AQ & 1440 \\\\\n", + "LTR & 11M & AQ & 1600 \\\\\n", + "RTL & 11M & AQ & 1490 \\\\\n", + "LTR & 19M & AQ & 2310 \\\\\n", + "RTL & 19M & AQ & 2740 \\\\\n", + "LTR & 35M & AQ & 3650 \\\\\n", + "RTL & 35M & AQ & 4090 \\\\\n", + "LTR & 67M & AQ & 6360 \\\\\n", + "RTL & 67M & AQ & 22900 \\\\\n", + "LTR & 6M & QA & 1790 \\\\\n", + "RTL & 6M & QA & 1850 \\\\\n", + "LTR & 11M & QA & 1430 \\\\\n", + "RTL & 11M & QA & 1820 \\\\\n", + "LTR & 19M & QA & 2280 \\\\\n", + "RTL & 19M & QA & 3740 \\\\\n", + "LTR & 35M & QA & 3690 \\\\\n", + "RTL & 35M & QA & 4650 \\\\\n", + "LTR & 67M & QA & 6340 \\\\\n", + "RTL & 67M & QA & 22900 \\\\\n" + ] + } + ], + "source": [ + "for ppl, task, text_dir, path_prefix in sorted(results, key=lambda x: (x[1], to_params[x[3]], x[2])):\n", + " ppl = int(float(f\"{ppl:.3g}\"))\n", + " print(rf\"{text_dir.upper()} & {to_params[path_prefix]}M & {task.upper()} & {ppl} \\\\\")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "execution_state": "idle", + "id": "e9ac0af8-2638-4076-b0cc-9ec9355c2c01", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "LTR & 67M & AQ & 969 \\\\\n", + "RTL & 67M & AQ & 675 \\\\\n", + "LTR & 110M & AQ & 1880 \\\\\n", + "RTL & 110M & AQ & 483 \\\\\n", + "LTR & 335M & AQ & 2610 \\\\\n", + "RTL & 335M & AQ & 695 \\\\\n", + "LTR & 67M & QA & 952 \\\\\n", + "RTL & 67M & QA & 781 \\\\\n", + "LTR & 110M & QA & 2020 \\\\\n", + "RTL & 110M & QA & 689 \\\\\n", + "LTR & 335M & QA & 2950 \\\\\n", + "RTL & 335M & QA & 782 \\\\\n" + ] + } + ], + "source": [ + "for ppl, task, text_dir, path_prefix in sorted(results, key=lambda x: (x[1], to_params[x[3]], x[2])):\n", + " ppl = int(float(f\"{ppl:.3g}\"))\n", + " print(rf\"{text_dir.upper()} & {to_params[path_prefix]}M & {task.upper()} & {ppl} \\\\\")" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "execution_state": "idle", + "id": "8894ca16-58e3-4448-bec8-c962f5135737", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "the more you take, the more you leave behind. what am i? @ @ @ @ @ @ @ @ @ @ @ @ @ the @ @ @ @ ( the the the. @ the @ @ ( @ @ ( @ @ @ @ ( the.. @ ( @ ) @ the @ the the\n" + ] + } + ], + "source": [ + "# input_text = [\"The more you take, the more you leave behind. What am I?\"]\n", + "# batch = tokenizer(input_text, return_tensors=\"pt\", padding_side=\"right\", padding=\"max_length\", max_length=64)\n", + "# output_ids = model.generate(batch['input_ids'], max_length=128, do_sample=False) # do_sample=False ensures greedy decoding\n", + "# decoded_output = tokenizer.decode(output_ids[0], skip_special_tokens=True)\n", + "# print(decoded_output)" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "execution_state": "idle", + "id": "a4098975-2df6-4435-bc93-1a5afd6d7e68", + "metadata": {}, + "outputs": [], + "source": [ + "# CPU is fast enough\n", + "\n", + "ppls = []\n", + "for riddle in riddles:\n", + " batch = tokenizer([riddle], return_tensors=\"pt\", padding_side=\"left\" if text_dir == \"rtl\" else \"right\", padding=\"max_length\", max_length=128)\n", + " batch[\"labels\"] = batch[\"input_ids\"].clone()\n", + " batch[\"labels\"][batch[\"attention_mask\"] == 0] = -100\n", + " # batch = tokenizer([riddle], return_tensors=\"pt\")#, padding_side=\"left\" if text_dir == \"rtl\" else \"right\", padding=\"longest\", max_length=128)\n", + " # batch[\"labels\"] = batch[\"input_ids\"]\n", + " with torch.inference_mode():\n", + " output = model(**batch)\n", + " ppls.append(math.e ** output.loss.item())" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "execution_state": "idle", + "id": "c68b5235-a4a7-4f38-9acb-f5072e546a96", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(array([ 4., 6., 11., 6., 5., 2., 1., 1., 2., 2.]),\n", + " array([ 613.56297843, 829.36555779, 1045.16813716, 1260.97071653,\n", + " 1476.77329589, 1692.57587526, 1908.37845463, 2124.18103399,\n", + " 2339.98361336, 2555.78619272, 2771.58877209]),\n", + " )" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAY8UlEQVR4nO3df5DVVf348dcCsoCyyy9ZIBdZf4ym4E+MECMdGIHQsBpHjRqiRlMxJQqFCozMQGscJjM1Z1KbUbRmBBt/MGMokiMiIKikoRgKqUCJ7PJDV2TP54+v3vEKqPS99ywLj8fMnXHf78P7nMvhzj69P3YrUkopAAAyadXcCwAA9i/iAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAsmrT3Av4uKampnjjjTeiY8eOUVFR0dzLAQA+g5RSbN68OXr16hWtWn3ycxt7XXy88cYbUVtb29zLAAD+B2vXro1DDjnkE8fsdfHRsWPHiA8WX1VV1dzLAQA+g4aGhqitrS18H/8ke118fPhSS1VVlfgAgBbms7xlwhtOAYCsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZtWnuBbBv6jPpweZewh57dcbI5l4CwH7BMx8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFntcXwsWLAgzj777OjVq1dUVFTEnDlzis6nlGLq1KnRs2fPaN++fQwdOjRefvnlUq4ZAGjB9jg+tm7dGscff3zcdNNNuzx//fXXx29/+9u45ZZbYtGiRXHggQfGsGHD4t133y3FegGAFq7Nnv6BESNGxIgRI3Z5LqUUM2fOjJ/97GcxatSoiIj405/+FDU1NTFnzpw4//zz//9XDAC0aCV9z8fq1atj3bp1MXTo0MKx6urqGDBgQCxcuHCXf6axsTEaGhqKbgDAvquk8bFu3bqIiKipqSk6XlNTUzj3cdOnT4/q6urCrba2tpRLAgD2Ms3+aZfJkydHfX194bZ27drmXhIAUEYljY8ePXpERMT69euLjq9fv75w7uMqKyujqqqq6AYA7LtKGh91dXXRo0ePmDdvXuFYQ0NDLFq0KAYOHFjKqQCAFmqPP+2yZcuWWLVqVeHr1atXx/Lly6NLly7Ru3fvGD9+fPzyl7+MI488Murq6mLKlCnRq1evOOecc0q9dgCgBdrj+FiyZEmcccYZha8nTJgQERFjxoyJO+64I6688srYunVrXHTRRbFp06Y47bTTYu7cudGuXbvSrhwAaJEqUkqpuRfxUQ0NDVFdXR319fXe/9GC9Zn0YHMvYY+9OmNkcy8BoMXak+/fzf5pFwBg/yI+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFYlj48dO3bElClToq6uLtq3bx+HH354XHPNNZFSKvVUAEAL1KbUF7zuuuvi5ptvjjvvvDOOPfbYWLJkSYwdOzaqq6vj8ssvL/V0AEALU/L4ePLJJ2PUqFExcuTIiIjo06dPzJo1K55++ulSTwUAtEAlf9nl1FNPjXnz5sVLL70UERHPPvtsPPHEEzFixIhdjm9sbIyGhoaiGwCw7yr5Mx+TJk2KhoaGOProo6N169axY8eOuPbaa2P06NG7HD99+vSYNm1aqZcBAOylSv7Mx5///Oe466674u67745nnnkm7rzzzvjNb34Td9555y7HT548Oerr6wu3tWvXlnpJAMBepOTPfEycODEmTZoU559/fkRE9OvXL1577bWYPn16jBkzZqfxlZWVUVlZWeplAAB7qZI/87Ft27Zo1ar4sq1bt46mpqZSTwUAtEAlf+bj7LPPjmuvvTZ69+4dxx57bCxbtixuuOGG+O53v1vqqQCAFqjk8XHjjTfGlClT4tJLL40NGzZEr1694vvf/35MnTq11FMBAC1QyeOjY8eOMXPmzJg5c2apLw0A7AP8bhcAICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALIqS3y8/vrr8a1vfSu6du0a7du3j379+sWSJUvKMRUA0MK0KfUF33777Rg0aFCcccYZ8fDDD8fBBx8cL7/8cnTu3LnUUwEALVDJ4+O6666L2trauP322wvH6urqSj0NANBClfxll7/+9a/Rv3//OPfcc6N79+5x4oknxm233bbb8Y2NjdHQ0FB0AwD2XRUppVTKC7Zr1y4iIiZMmBDnnntuLF68OK644oq45ZZbYsyYMTuN//nPfx7Tpk3b6Xh9fX1UVVWVcmktVp9JDzb3EthLvTpjZHMvASAiIhoaGqK6uvozff8ueXy0bds2+vfvH08++WTh2OWXXx6LFy+OhQsX7jS+sbExGhsbixZfW1srPj5CfLA74gPYW+xJfJT8ZZeePXvGMcccU3Ts85//fKxZs2aX4ysrK6OqqqroBgDsu0oeH4MGDYqVK1cWHXvppZfi0EMPLfVUAEALVPL4+OEPfxhPPfVU/OpXv4pVq1bF3XffHX/4wx9i3LhxpZ4KAGiBSh4fp5xySsyePTtmzZoVffv2jWuuuSZmzpwZo0ePLvVUAEALVPKf8xERcdZZZ8VZZ51VjksDAC2c3+0CAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGTVprkXAPzv+kx6sLmXsMdenTGyuZcANDPPfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZFX2+JgxY0ZUVFTE+PHjyz0VANAClDU+Fi9eHLfeemscd9xx5ZwGAGhByhYfW7ZsidGjR8dtt90WnTt3Ltc0AEALU7b4GDduXIwcOTKGDh36ieMaGxujoaGh6AYA7LvalOOi99xzTzzzzDOxePHiTx07ffr0mDZtWjmWsUt9Jj2YbS4AYGclf+Zj7dq1ccUVV8Rdd90V7dq1+9TxkydPjvr6+sJt7dq1pV4SALAXKfkzH0uXLo0NGzbESSedVDi2Y8eOWLBgQfzud7+LxsbGaN26deFcZWVlVFZWlnoZAMBequTxMWTIkHj++eeLjo0dOzaOPvrouOqqq4rCAwDY/5Q8Pjp27Bh9+/YtOnbggQdG165ddzoOAOx//IRTACCrsnza5ePmz5+fYxoAoAXwzAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmVPD6mT58ep5xySnTs2DG6d+8e55xzTqxcubLU0wAALVTJ4+Pxxx+PcePGxVNPPRWPPPJIbN++Pc4888zYunVrqacCAFqgNqW+4Ny5c4u+vuOOO6J79+6xdOnSGDx4cKmnAwBamJLHx8fV19dHRESXLl12eb6xsTEaGxsLXzc0NJR7SQBAMyprfDQ1NcX48eNj0KBB0bdv312OmT59ekybNq2cywD2In0mPdjcS9hjr84Y2dxL2G+0xH8fLVFz/5su66ddxo0bFytWrIh77rlnt2MmT54c9fX1hdvatWvLuSQAoJmV7ZmPyy67LB544IFYsGBBHHLIIbsdV1lZGZWVleVaBgCwlyl5fKSU4gc/+EHMnj075s+fH3V1daWeAgBowUoeH+PGjYu777477r///ujYsWOsW7cuIiKqq6ujffv2pZ4OAGhhSv6ej5tvvjnq6+vj9NNPj549exZu9957b6mnAgBaoLK87AIAsDt+twsAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkFWb5l4AwN6uz6QHm3sJsE/xzAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFZli4+bbrop+vTpE+3atYsBAwbE008/Xa6pAIAWpCzxce+998aECRPi6quvjmeeeSaOP/74GDZsWGzYsKEc0wEALUhZ4uOGG26ICy+8MMaOHRvHHHNM3HLLLdGhQ4f44x//WI7pAIAWpE2pL/jee+/F0qVLY/LkyYVjrVq1iqFDh8bChQt3Gt/Y2BiNjY2Fr+vr6yMioqGhodRLi4iIpsZtZbkuALQU5fge++E1U0qfOrbk8fHf//43duzYETU1NUXHa2pq4p///OdO46dPnx7Tpk3b6XhtbW2plwYARET1zPJde/PmzVFdXf2JY0oeH3tq8uTJMWHChMLXTU1NsXHjxujatWtUVFQ069rYWUNDQ9TW1sbatWujqqqquZfDx9ifvZ892rvZn/9dSik2b94cvXr1+tSxJY+Pbt26RevWrWP9+vVFx9evXx89evTYaXxlZWVUVlYWHevUqVOpl0WJVVVVeWDuxezP3s8e7d3sz//m057x+FDJ33Datm3bOPnkk2PevHmFY01NTTFv3rwYOHBgqacDAFqYsrzsMmHChBgzZkz0798/vvCFL8TMmTNj69atMXbs2HJMBwC0IGWJj/POOy/+85//xNSpU2PdunVxwgknxNy5c3d6EyotT2VlZVx99dU7vVTG3sH+7P3s0d7N/uRRkT7LZ2IAAErE73YBALISHwBAVuIDAMhKfAAAWYmP/dCCBQvi7LPPjl69ekVFRUXMmTOn6HxKKaZOnRo9e/aM9u3bx9ChQ+Pll18uGrNx48YYPXp0VFVVRadOneJ73/tebNmypWjMc889F1/60peiXbt2UVtbG9dff32W+9fSfdr+fOc734mKioqi2/Dhw4vG2J/ymT59epxyyinRsWPH6N69e5xzzjmxcuXKojHvvvtujBs3Lrp27RoHHXRQfOMb39jpBy+uWbMmRo4cGR06dIju3bvHxIkT4/333y8aM3/+/DjppJOisrIyjjjiiLjjjjuy3MeW7rPs0emnn77T4+jiiy8uGmOPyiix33nooYfST3/603TfffeliEizZ88uOj9jxoxUXV2d5syZk5599tn01a9+NdXV1aV33nmnMGb48OHp+OOPT0899VT6+9//no444oh0wQUXFM7X19enmpqaNHr06LRixYo0a9as1L59+3Trrbdmva8t0aftz5gxY9Lw4cPTm2++Wbht3LixaIz9KZ9hw4al22+/Pa1YsSItX748feUrX0m9e/dOW7ZsKYy5+OKLU21tbZo3b15asmRJ+uIXv5hOPfXUwvn3338/9e3bNw0dOjQtW7YsPfTQQ6lbt25p8uTJhTH/+te/UocOHdKECRPSCy+8kG688cbUunXrNHfu3Oz3uaX5LHv05S9/OV144YVFj6P6+vrCeXtUXuJjP/fxb25NTU2pR48e6de//nXh2KZNm1JlZWWaNWtWSimlF154IUVEWrx4cWHMww8/nCoqKtLrr7+eUkrp97//fercuXNqbGwsjLnqqqvSUUcdleme7Rt2Fx+jRo3a7Z+xP3lt2LAhRUR6/PHHU/rg8XLAAQekv/zlL4UxL774YoqItHDhwpQ+CMxWrVqldevWFcbcfPPNqaqqqrAnV155ZTr22GOL5jrvvPPSsGHDMt2zfcfH9yh9EB9XXHHFbv+MPSovL7tQZPXq1bFu3boYOnRo4Vh1dXUMGDAgFi5cGBERCxcujE6dOkX//v0LY4YOHRqtWrWKRYsWFcYMHjw42rZtWxgzbNiwWLlyZbz99ttZ79O+aP78+dG9e/c46qij4pJLLom33nqrcM7+5FVfXx8REV26dImIiKVLl8b27duLHkNHH3109O7du+gx1K9fv6IfvDhs2LBoaGiIf/zjH4UxH73Gh2M+vAaf3cf36EN33XVXdOvWLfr27RuTJ0+Obdu2Fc7Zo/Jq9t9qy95l3bp1ERE7/TTampqawrl169ZF9+7di863adMmunTpUjSmrq5up2t8eK5z585lvR/7suHDh8fXv/71qKuri1deeSV+8pOfxIgRI2LhwoXRunVr+5NRU1NTjB8/PgYNGhR9+/aN+ODvr23btjv9gsyPP4Z29RiLjzwGdzemoaEh3nnnnWjfvn1Z79u+Yld7FBHxzW9+Mw499NDo1atXPPfcc3HVVVfFypUr47777ouwR2UnPqCFOf/88wv/3a9fvzjuuOPi8MMPj/nz58eQIUOadW37m3HjxsWKFSviiSeeaO6lsBu726OLLrqo8N/9+vWLnj17xpAhQ+KVV16Jww8/vBlWun/xsgtFevToERGx0zvz169fXzjXo0eP2LBhQ9H5999/PzZu3Fg0ZlfX+OgclMZhhx0W3bp1i1WrVkXYn2wuu+yyeOCBB+Kxxx6LQw45pHC8R48e8d5778WmTZuKxn/8MfRpf/+7G1NVVeX/qD+j3e3RrgwYMCAiouhxZI/KR3xQpK6uLnr06BHz5s0rHGtoaIhFixbFwIEDIyJi4MCBsWnTpli6dGlhzKOPPhpNTU2FB/DAgQNjwYIFsX379sKYRx55JI466ihP6ZfYv//973jrrbeiZ8+eEfan7FJKcdlll8Xs2bPj0Ucf3enlq5NPPjkOOOCAosfQypUrY82aNUWPoeeff74oEh955JGoqqqKY445pjDmo9f4cMyH12D3Pm2PdmX58uUREUWPI3tURs39jlfy27x5c1q2bFlatmxZioh0ww03pGXLlqXXXnstpQ8+atupU6d0//33p+eeey6NGjVqlx+1PfHEE9OiRYvSE088kY488siij3Ju2rQp1dTUpG9/+9tpxYoV6Z577kkdOnTwUc7P4JP2Z/PmzenHP/5xWrhwYVq9enX629/+lk466aR05JFHpnfffbdwDftTPpdcckmqrq5O8+fPL/qY5rZt2wpjLr744tS7d+/06KOPpiVLlqSBAwemgQMHFs5/+DHOM888My1fvjzNnTs3HXzwwbv8GOfEiRPTiy++mG666SYf4/yMPm2PVq1alX7xi1+kJUuWpNWrV6f7778/HXbYYWnw4MGFa9ij8hIf+6HHHnssRcROtzFjxqT0wcdtp0yZkmpqalJlZWUaMmRIWrlyZdE13nrrrXTBBRekgw46KFVVVaWxY8emzZs3F4159tln02mnnZYqKyvT5z73uTRjxoys97Ol+qT92bZtWzrzzDPTwQcfnA444IB06KGHpgsvvLDo44DJ/pTVrvYmItLtt99eGPPOO++kSy+9NHXu3Dl16NAhfe1rX0tvvvlm0XVeffXVNGLEiNS+ffvUrVu39KMf/Sht3769aMxjjz2WTjjhhNS2bdt02GGHFc3B7n3aHq1ZsyYNHjw4denSJVVWVqYjjjgiTZw4sejnfCR7VFYV6f9tFABAFt7zAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACy+j/J4r9i/nZCRAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "plt.hist(ppls)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "86daa05b-5784-457b-b65e-8b8395128d6f", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/Right_to_Left_NLP.ipynb b/notebooks/Right_to_Left_NLP.ipynb index bcbc57a..986ef22 100644 --- a/notebooks/Right_to_Left_NLP.ipynb +++ b/notebooks/Right_to_Left_NLP.ipynb @@ -1,373 +1,383 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "id": "M29-oTOBIiMr" + }, + "outputs": [], + "source": [ + "%pip install datasets torch transformers" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" + "base_uri": "https://localhost:8080/" }, - "language_info": { - "name": "python" + "id": "2iJJyERxHWSO", + "outputId": "04e9bc9d-5ee9-48d5-f370-6fd66ec7b7c1" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cpu\n" + ] } + ], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import transformers\n", + "from datasets import load_dataset\n", + "\n", + "transformers.set_seed(42)\n", + "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", + "print(device)" + ] }, - "cells": [ - { - "cell_type": "code", - "source": [ - "%pip install datasets torch transformers" - ], - "metadata": { - "collapsed": true, - "id": "M29-oTOBIiMr" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "2iJJyERxHWSO", - "outputId": "04e9bc9d-5ee9-48d5-f370-6fd66ec7b7c1" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "cpu\n" - ] - } - ], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import transformers\n", - "from datasets import load_dataset\n", - "\n", - "transformers.set_seed(42)\n", - "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", - "print(device)" - ] - }, - { - "cell_type": "code", - "source": [ - "model_name_or_path = \"bert-base-uncased\"\n", - "model = transformers.AutoModelForMaskedLM.from_pretrained(model_name_or_path, torch_dtype=torch.bfloat16, attn_implementation=\"sdpa\")\n", - "model.eval()\n", - "\n", - "tokenizer = transformers.AutoTokenizer.from_pretrained(model_name_or_path)" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "C5PdaHGWHuXG", - "outputId": "d15272a5-1ce1-4c7e-9004-fc686a3de6b9" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n", - "The secret `HF_TOKEN` does not exist in your Colab secrets.\n", - "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n", - "You will be able to reuse this secret in all of your notebooks.\n", - "Please note that authentication is recommended but still optional to access public models or datasets.\n", - " warnings.warn(\n", - "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n", - "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", - "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", - "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n", - " warnings.warn(\n" - ] - } - ] - }, - { - "cell_type": "code", - "source": [ - "def ltr_mask(seq_len: int) -> torch.Tensor:\n", - " mask = torch.ones((seq_len, seq_len), dtype=bool)\n", - " return torch.tril(mask, diagonal=-1)\n", - "\n", - "def rtl_mask(seq_len: int) -> torch.Tensor:\n", - " return ltr_mask(seq_len).T" - ], - "metadata": { - "id": "H_AUjBRoJHXU" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "model.register_buffer(\"attn_mask\", rtl_mask(model.config.max_position_embeddings).to(model.device))\n", - "\n", - "def attn_hook(attn_module: nn.Module, args: tuple, kwargs: dict):\n", - " \"\"\"\n", - " Assuming https://github.com/huggingface/transformers/blob/33868a057c02f0368ba63bd1edb746be38fe3d90/src/transformers/models/bert/modeling_bert.py#L515\n", - " so no `kwargs` and `attention_mask` is second positional arg.\n", - "\n", - " Uses global `model.attn_mask` to save memory.\n", - " \"\"\"\n", - " assert not kwargs\n", - "\n", - " args = list(args)\n", - " assert args[1].size()[-2:] == model.attn_mask.size(), f\"{args[1].size()=} {model.attn_mask.size()=}\"\n", - " args[1] = model.attn_mask\n", - " return tuple(args), kwargs\n", - "\n", - "def debug_inputs_hook(attn_module: nn.Module, args: tuple, output):\n", - " print(f\"Post-forward checks\")\n", - " assert torch.equal(args[1], model.attn_mask), (args[1], model.attn_mask)" - ], - "metadata": { - "id": "Oy27MZcLLLsD" - }, - "execution_count": null, - "outputs": [] + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "C5PdaHGWHuXG", + "outputId": "d15272a5-1ce1-4c7e-9004-fc686a3de6b9" + }, + "outputs": [ { - "cell_type": "code", - "source": [ - "# attn_mask = rtl_mask(model.config.max_position_embeddings)\n", - "for name, module in model.named_modules():\n", - " if isinstance(module, transformers.models.bert.modeling_bert.BertSelfAttention):\n", - " module._forward_hooks.clear() # running multiple times right now during testing\n", - " module.register_forward_pre_hook(attn_hook, with_kwargs=True)\n", - " module.register_forward_hook(debug_inputs_hook)\n", - " # module.register_buffer(\"attn_mask\", attn_mask)\n", - "\n", - "model = model.to(device)" - ], - "metadata": { - "id": "anEdwKj_OWWy" - }, - "execution_count": null, - "outputs": [] + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n", + "The secret `HF_TOKEN` does not exist in your Colab secrets.\n", + "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n", + "You will be able to reuse this secret in all of your notebooks.\n", + "Please note that authentication is recommended but still optional to access public models or datasets.\n", + " warnings.warn(\n", + "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n", + "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "model_name_or_path = \"bert-base-uncased\"\n", + "model = transformers.AutoModelForMaskedLM.from_pretrained(model_name_or_path, torch_dtype=torch.bfloat16, attn_implementation=\"sdpa\")\n", + "model.eval()\n", + "\n", + "tokenizer = transformers.AutoTokenizer.from_pretrained(model_name_or_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "H_AUjBRoJHXU" + }, + "outputs": [], + "source": [ + "def ltr_mask(seq_len: int) -> torch.Tensor:\n", + " mask = torch.ones((seq_len, seq_len), dtype=bool)\n", + " return torch.tril(mask, diagonal=-1)\n", + "\n", + "def rtl_mask(seq_len: int) -> torch.Tensor:\n", + " return ltr_mask(seq_len).T" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Oy27MZcLLLsD" + }, + "outputs": [], + "source": [ + "model.register_buffer(\"attn_mask\", rtl_mask(model.config.max_position_embeddings).to(model.device))\n", + "\n", + "def attn_hook(attn_module: nn.Module, args: tuple, kwargs: dict):\n", + " \"\"\"\n", + " Assuming https://github.com/huggingface/transformers/blob/33868a057c02f0368ba63bd1edb746be38fe3d90/src/transformers/models/bert/modeling_bert.py#L515\n", + " so no `kwargs` and `attention_mask` is second positional arg.\n", + "\n", + " Uses global `model.attn_mask` to save memory.\n", + " \"\"\"\n", + " assert not kwargs\n", + "\n", + " args = list(args)\n", + " assert args[1].size()[-2:] == model.attn_mask.size(), f\"{args[1].size()=} {model.attn_mask.size()=}\"\n", + " args[1] = model.attn_mask\n", + " return tuple(args), kwargs\n", + "\n", + "def debug_inputs_hook(attn_module: nn.Module, args: tuple, output):\n", + " print(f\"Post-forward checks\")\n", + " assert torch.equal(args[1], model.attn_mask), (args[1], model.attn_mask)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "anEdwKj_OWWy" + }, + "outputs": [], + "source": [ + "# attn_mask = rtl_mask(model.config.max_position_embeddings)\n", + "for name, module in model.named_modules():\n", + " if isinstance(module, transformers.models.bert.modeling_bert.BertSelfAttention):\n", + " module._forward_hooks.clear() # running multiple times right now during testing\n", + " module.register_forward_pre_hook(attn_hook, with_kwargs=True)\n", + " module.register_forward_hook(debug_inputs_hook)\n", + " # module.register_buffer(\"attn_mask\", attn_mask)\n", + "\n", + "model = model.to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "P1BEQFsLIRfX" + }, + "outputs": [], + "source": [ + "ds = load_dataset(\"Salesforce/wikitext\", \"wikitext-103-v1\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "BHE26Mr2NXhH", + "outputId": "24569931-61d7-4752-8b08-4daef58f9798" + }, + "outputs": [ { - "cell_type": "code", - "source": [ - "ds = load_dataset(\"Salesforce/wikitext\", \"wikitext-103-v1\")" - ], - "metadata": { - "id": "P1BEQFsLIRfX" - }, - "execution_count": null, - "outputs": [] + "name": "stdout", + "output_type": "stream", + "text": [ + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n" + ] }, { - "cell_type": "code", - "source": [ - "train_ds = ds[\"train\"]\n", - "inputs = tokenizer(train_ds[5][\"text\"], return_tensors=\"pt\", padding='max_length', truncation=True)\n", - "inputs = {key: val.to(device) for key, val in inputs.items()}\n", - "\n", - "with torch.no_grad():\n", - " outputs = model(**inputs)\n", - "\n", - "outputs.logits" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "BHE26Mr2NXhH", - "outputId": "24569931-61d7-4752-8b08-4daef58f9798" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n" - ] - }, - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "tensor([[[-5.6250, -5.5938, -5.5938, ..., -5.4688, -4.9688, -2.4844],\n", - " [-9.2500, -8.9375, -9.3750, ..., -8.5000, -7.5000, -4.0312],\n", - " [-4.9062, -4.8750, -5.2812, ..., -5.0625, -4.4375, -1.8281],\n", - " ...,\n", - " [-5.5938, -5.7500, -5.7812, ..., -6.1562, -3.9688, -2.2812],\n", - " [-4.7188, -4.8750, -4.8750, ..., -5.0625, -3.4531, -2.4375],\n", - " [-4.1875, -3.9375, -3.9062, ..., -3.3438, -3.2344, -3.2031]]],\n", - " device='cuda:0', dtype=torch.bfloat16)" - ] - }, - "metadata": {}, - "execution_count": 9 - } + "data": { + "text/plain": [ + "tensor([[[-5.6250, -5.5938, -5.5938, ..., -5.4688, -4.9688, -2.4844],\n", + " [-9.2500, -8.9375, -9.3750, ..., -8.5000, -7.5000, -4.0312],\n", + " [-4.9062, -4.8750, -5.2812, ..., -5.0625, -4.4375, -1.8281],\n", + " ...,\n", + " [-5.5938, -5.7500, -5.7812, ..., -6.1562, -3.9688, -2.2812],\n", + " [-4.7188, -4.8750, -4.8750, ..., -5.0625, -3.4531, -2.4375],\n", + " [-4.1875, -3.9375, -3.9062, ..., -3.3438, -3.2344, -3.2031]]],\n", + " device='cuda:0', dtype=torch.bfloat16)" ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train_ds = ds[\"train\"]\n", + "inputs = tokenizer(train_ds[5][\"text\"], return_tensors=\"pt\", padding='max_length', truncation=True)\n", + "inputs = {key: val.to(device) for key, val in inputs.items()}\n", + "\n", + "with torch.no_grad():\n", + " outputs = model(**inputs)\n", + "\n", + "outputs.logits" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "ZtEm7eQQNi4e", + "outputId": "c0eb3925-6d48-480e-a853-5057f35dbcd2" + }, + "outputs": [ { - "cell_type": "code", - "source": [ - "with torch.inference_mode():\n", - " model.register_buffer(\"attn_mask\", ltr_mask(model.config.max_position_embeddings).to(model.device))\n", - " outputs = model(**inputs)\n", - "\n", - "outputs.logits" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ZtEm7eQQNi4e", - "outputId": "c0eb3925-6d48-480e-a853-5057f35dbcd2" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n" - ] - }, - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "tensor([[[-7.9062, -7.7812, -7.9062, ..., -7.1250, -7.8438, -4.8438],\n", - " [-7.1562, -7.1250, -7.2812, ..., -7.3750, -7.3750, -7.2500],\n", - " [-5.4062, -5.2188, -5.4375, ..., -5.3438, -4.3750, -5.0312],\n", - " ...,\n", - " [ 3.9844, 3.6406, 3.6406, ..., 3.8281, 2.9062, 5.2812],\n", - " [ 4.0938, 3.7812, 3.8281, ..., 4.0000, 2.9844, 5.5000],\n", - " [ 3.8281, 3.5312, 3.5156, ..., 4.1562, 2.8438, 4.7188]]],\n", - " device='cuda:0', dtype=torch.bfloat16)" - ] - }, - "metadata": {}, - "execution_count": 10 - } - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n" + ] }, { - "cell_type": "code", - "source": [ - "with torch.inference_mode():\n", - " model.register_buffer(\"attn_mask\", rtl_mask(model.config.max_position_embeddings).to(model.device))\n", - " outputs = model(**inputs)\n", - "\n", - "outputs.logits" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "nz0j7V3oNkZu", - "outputId": "939b1d6d-5dca-41ef-eb17-9e0f4d09629e" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n", - "Post-forward checks\n" - ] - }, - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "tensor([[[-5.6250, -5.5938, -5.5938, ..., -5.4688, -4.9688, -2.4844],\n", - " [-9.2500, -8.9375, -9.3750, ..., -8.5000, -7.5000, -4.0312],\n", - " [-4.9062, -4.8750, -5.2812, ..., -5.0625, -4.4375, -1.8281],\n", - " ...,\n", - " [-5.5938, -5.7500, -5.7812, ..., -6.1562, -3.9688, -2.2812],\n", - " [-4.7188, -4.8750, -4.8750, ..., -5.0625, -3.4531, -2.4375],\n", - " [-4.1875, -3.9375, -3.9062, ..., -3.3438, -3.2344, -3.2031]]],\n", - " device='cuda:0', dtype=torch.bfloat16)" - ] - }, - "metadata": {}, - "execution_count": 11 - } + "data": { + "text/plain": [ + "tensor([[[-7.9062, -7.7812, -7.9062, ..., -7.1250, -7.8438, -4.8438],\n", + " [-7.1562, -7.1250, -7.2812, ..., -7.3750, -7.3750, -7.2500],\n", + " [-5.4062, -5.2188, -5.4375, ..., -5.3438, -4.3750, -5.0312],\n", + " ...,\n", + " [ 3.9844, 3.6406, 3.6406, ..., 3.8281, 2.9062, 5.2812],\n", + " [ 4.0938, 3.7812, 3.8281, ..., 4.0000, 2.9844, 5.5000],\n", + " [ 3.8281, 3.5312, 3.5156, ..., 4.1562, 2.8438, 4.7188]]],\n", + " device='cuda:0', dtype=torch.bfloat16)" ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "with torch.inference_mode():\n", + " model.register_buffer(\"attn_mask\", ltr_mask(model.config.max_position_embeddings).to(model.device))\n", + " outputs = model(**inputs)\n", + "\n", + "outputs.logits" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "nz0j7V3oNkZu", + "outputId": "939b1d6d-5dca-41ef-eb17-9e0f4d09629e" + }, + "outputs": [ { - "cell_type": "code", - "source": [], - "metadata": { - "id": "82PpSWnrdMgu" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "# Wikipedia test\n", - "from datasets import load_dataset\n", - "\n", - "ds = load_dataset(\"wikimedia/wikipedia\", \"20231101.en\")\n", - "print(ds[\"train\"][1000])" - ], - "metadata": { - "id": "DHftDnPKdMjV" - }, - "execution_count": null, - "outputs": [] + "name": "stdout", + "output_type": "stream", + "text": [ + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n", + "Post-forward checks\n" + ] }, { - "cell_type": "code", - "source": [], - "metadata": { - "id": "FTMkfLyKdMqu" - }, - "execution_count": null, - "outputs": [] + "data": { + "text/plain": [ + "tensor([[[-5.6250, -5.5938, -5.5938, ..., -5.4688, -4.9688, -2.4844],\n", + " [-9.2500, -8.9375, -9.3750, ..., -8.5000, -7.5000, -4.0312],\n", + " [-4.9062, -4.8750, -5.2812, ..., -5.0625, -4.4375, -1.8281],\n", + " ...,\n", + " [-5.5938, -5.7500, -5.7812, ..., -6.1562, -3.9688, -2.2812],\n", + " [-4.7188, -4.8750, -4.8750, ..., -5.0625, -3.4531, -2.4375],\n", + " [-4.1875, -3.9375, -3.9062, ..., -3.3438, -3.2344, -3.2031]]],\n", + " device='cuda:0', dtype=torch.bfloat16)" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" } - ] -} \ No newline at end of file + ], + "source": [ + "with torch.inference_mode():\n", + " model.register_buffer(\"attn_mask\", rtl_mask(model.config.max_position_embeddings).to(model.device))\n", + " outputs = model(**inputs)\n", + "\n", + "outputs.logits" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "82PpSWnrdMgu" + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DHftDnPKdMjV" + }, + "outputs": [], + "source": [ + "# Wikipedia test\n", + "from datasets import load_dataset\n", + "\n", + "ds = load_dataset(\"wikimedia/wikipedia\", \"20231101.en\")\n", + "print(ds[\"train\"][1000])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FTMkfLyKdMqu" + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/Stat_Tests.ipynb b/notebooks/Stat_Tests.ipynb new file mode 100644 index 0000000..379df3c --- /dev/null +++ b/notebooks/Stat_Tests.ipynb @@ -0,0 +1,487 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "execution_state": "idle", + "id": "d3616030-9841-4eeb-a1c0-b4fa591e2fe1", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import scipy" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "execution_state": "idle", + "id": "e0ecfabf-f3ad-4ac3-a8de-97e71f9aff5a", + "metadata": {}, + "outputs": [], + "source": [ + "small_6M_ppls = [\n", + " (116.7, 114.9), # (LTR ppl, RTL ppl)\n", + " (117.4, 114.4),\n", + " (116.7, 115.0),\n", + " (117.4, 115.4),\n", + " (117.5, 113.8),\n", + " (116.1, 114.0)\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "execution_state": "idle", + "id": "bacb5848-af91-4443-9d71-81b6cb0e6aa6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "TtestResult(statistic=6.996032521277681, pvalue=1.9799032618443016e-05, df=9.885322295882748)\n", + "TtestResult(statistic=7.357072921297962, pvalue=0.0003642557582314903, df=5)\n" + ] + } + ], + "source": [ + "# One-sided unpaired Welch t-test\n", + "small_ltr_ppls, small_rtl_ppls = np.array(small_6M_ppls).T\n", + "print(scipy.stats.ttest_ind(small_ltr_ppls, small_rtl_ppls, equal_var=False, alternative=\"greater\"))\n", + "# Paired t-test\n", + "print(scipy.stats.ttest_rel(small_ltr_ppls, small_rtl_ppls, alternative=\"greater\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "execution_state": "idle", + "id": "c23d0ecf-c0f0-4a61-ac99-79c9c91c1d76", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.0010822510822510823" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Permutation test\n", + "def statistic(x, y):\n", + " return np.mean(x) - np.mean(y)\n", + "\n", + "scipy.stats.permutation_test((small_ltr_ppls, small_rtl_ppls), statistic, n_resamples=np.inf, alternative=\"greater\").pvalue" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "execution_state": "idle", + "id": "d2bc01d6-821e-4a31-acd2-9d54592ab095", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.0010822510822510823" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 9, + "execution_state": "idle", + "id": "d59a5f23-7306-478b-87cc-4a064675c48d", + "metadata": {}, + "outputs": [], + "source": [ + "small_6M_losses = [\n", + " (4.761364663504469, 4.744475745069383),\n", + " (4.76577, 4.73966), # (LTR loss, RTL loss)\n", + " (4.7599, 4.74497),\n", + " (4.76553, 4.74848),\n", + " (4.76638, 4.73508),\n", + " (4.75452, 4.73628)\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "execution_state": "idle", + "id": "ab613d50-98a3-409d-92c2-123943a10c39", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "TtestResult(statistic=7.221951304385972, pvalue=1.5856791940777708e-05, df=9.802083740534792)\n", + "TtestResult(statistic=7.866415444091634, pvalue=0.00026667826712639355, df=5)\n", + "0.0010822510822510823\n" + ] + } + ], + "source": [ + "small_ltr_losses, small_rtl_losses = np.array(small_6M_losses).T\n", + "print(scipy.stats.ttest_ind(small_ltr_losses, small_rtl_losses, equal_var=False, alternative=\"greater\"))\n", + "print(scipy.stats.ttest_rel(small_ltr_losses, small_rtl_losses, alternative=\"greater\"))\n", + "print(scipy.stats.permutation_test((small_ltr_losses, small_rtl_losses), statistic, n_resamples=np.inf, alternative=\"greater\").pvalue)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "83aec592-de6e-43b9-9420-4b211641b75f", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 23, + "execution_state": "idle", + "id": "ea396600-66d9-45b4-b36c-4099c5548dec", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "execution_state": "idle", + "id": "c46997aa-65b5-4c49-8fa8-5c60d33362f5", + "metadata": {}, + "outputs": [], + "source": [ + "csv_filename = '../data/wandb_export_2024-12-04T19_56_43.325-05_00.csv'\n", + "df = pd.read_csv(csv_filename)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "execution_state": "idle", + "id": "7fcd8146-99cc-4322-8338-b58e58b36a30", + "metadata": {}, + "outputs": [], + "source": [ + "df = df.drop([0, 1, 10, 11, 12, 15,16,17,18,19])" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "execution_state": "idle", + "id": "ae6a65ee-1300-4d79-b268-c5ebae8a3a99", + "metadata": {}, + "outputs": [], + "source": [ + "df[\"val_ppl\"] = np.e ** df[\"val_loss\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "execution_state": "idle", + "id": "e1ebf94e-d645-4309-addc-aece1cc703ac", + "metadata": {}, + "outputs": [], + "source": [ + "transferred_ppl = np.array([24.4, 24.4, 21.9, 21.8, 18.1, 17.7]) # (rtl, ltr, rtl, ltr, ...)" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "execution_state": "idle", + "id": "79f2b427-d430-4aa4-9cef-8dc4d3d9b00b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk0AAAGwCAYAAAC0HlECAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABY5ElEQVR4nO3deXgT1cIG8HfapumatKU7dGUTRMoOBaUIZRUEt4vIlYICiuKKG17Z9FNEBUQvioqXAioqCKggKLssZafsIEtLWZoWKN33ZL4/0g5Nk5YUkkzSvr/nmUdm5szkZJo2r2fOnCOIoiiCiIiIiGrlJHcFiIiIiBwBQxMRERGRGRiaiIiIiMzA0ERERERkBoYmIiIiIjMwNBERERGZgaGJiIiIyAwuclfA3ul0Oly5cgXe3t4QBEHu6hAREZEZRFFEXl4eQkND4eRkmTYihqZbuHLlCsLCwuSuBhEREd2GixcvokmTJhY5F0PTLXh7ewMVF12lUsldHSIiIjJDbm4uwsLCpO9xS2BouoXKW3IqlYqhiYiIyMFYsmsNO4ITERERmYGhiYiIiMgMDE1EREREZmCfJiIicnharRZlZWVyV4NsSKFQwNnZ2aav6TChaebMmVi5ciVOnToFd3d3dO/eHbNmzULLli1rPW758uWYMmUKUlNT0bx5c8yaNQuDBg2yWb2JiMh6RFGERqNBdna23FUhGfj4+CA4ONhm4yg6TGjatm0bnn/+eXTu3Bnl5eV4++230a9fP5w4cQKenp4mj9m1axdGjBiBmTNnYvDgwfjhhx8wbNgwHDx4EG3atLH5eyAiIsuqDEyBgYHw8PDgIMQNhCiKKCwsRGZmJgAgJCTEJq8riKIo2uSVLOzq1asIDAzEtm3b0LNnT5Nlhg8fjoKCAqxZs0ba1q1bN7Rr1w4LFiww63Vyc3OhVquRk5PDIQeIiOyIVqvFP//8g8DAQDRq1Eju6pAMrl+/jszMTLRo0cLoVp01vr8dtiN4Tk4OAMDPz6/GMklJSYiPjzfY1r9/fyQlJdV4TElJCXJzcw0WIiKyP5V9mDw8POSuCsmk8mdvq/5sDhmadDodXn75ZfTo0aPW22wajQZBQUEG24KCgqDRaGo8ZubMmVCr1dLCKVSIiOwbb8k1XLb+2TtkaHr++edx7Ngx/PjjjxY/9+TJk5GTkyMtFy9etPhrEBERkeNxmI7glSZOnIg1a9bg77//vuUEfMHBwcjIyDDYlpGRgeDg4BqPUSqVUCqVFqsvERER1Q8O09IkiiImTpyIVatWYfPmzYiKirrlMbGxsdi0aZPBtg0bNiA2NtaKNSUiIqpfIiMj8emnn8pdDdk5TGh6/vnn8d133+GHH36At7c3NBoNNBoNioqKpDKjRo3C5MmTpfWXXnoJ69evx+zZs3Hq1ClMnz4d+/fvx8SJE2V6F3q5xWVYuvsCPt90Br8dviJrXYiISB5Xr17FhAkTEB4eDqVSieDgYPTv3x87d+606usKgoDVq1db9TXqK4e5Pffll18CAHr16mWwfdGiRRg9ejQAIC0tDU5ON3Ng9+7d8cMPP+Cdd97B22+/jebNm2P16tWyj9FUWKLFlNXHAAD9WgfhwZhQWetDRES298gjj6C0tBSLFy9GdHQ0MjIysGnTJly/fr3O59JqtRAEweA7kCzPYa6uKIoml8rABABbt25FYmKiwXGPPfYYTp8+jZKSEhw7dswuRgP38VBI/84u5LD/REQNTXZ2NrZv345Zs2bh/vvvR0REBLp06YLJkyfjwQcflMo888wzCAoKgpubG9q0aSONO5iYmAgfHx/89ttvaN26NZRKJdLS0rBv3z707dsX/v7+UKvViIuLw8GDB6XXjYyMBAA89NBDEARBWgeA33//HZ07d4abmxv8/f3x0EMPGdS5sLAQTz31FLy9vREeHo6vv/7aRlfLfjhMS1N94qZwhoerMwpLtcgqLJW7OkRE9cqQz3fgal6JzV83wFuJ31+416yyXl5e8PLywurVq9GtWzejB5B0Oh0GDhyIvLw8fPfdd2jatClOnDhhMIBjYWEhZs2ahYULF6JRo0YIDAzE+fPnkZCQgM8//xyiKGL27NkYNGgQzpw5A29vb+zbtw+BgYFYtGgRBgwYIJ1v7dq1eOihh/Cf//wHS5YsQWlpKf744w+DOs2ePRvvvfce3n77baxYsQITJkxAXFzcLaczq08YmmTi6+GKwtIiZDM0ERFZ1NW8Emhyi+WuRq1cXFyQmJiIcePGYcGCBejQoQPi4uLw+OOPo23btti4cSP27t2LkydPokWLFgCA6Ohog3OUlZXhiy++QExMjLStd+/eBmW+/vpr+Pj4YNu2bRg8eDACAgKAKnO2VXr//ffx+OOPY8aMGdK2qucFgEGDBuG5554DALz55puYO3cutmzZwtBE1ufrqcDl7CLcKCyDKIocnI2IyEICvOUZNqaur/vII4/ggQcewPbt27F7926sW7cOH330ERYuXIjMzEw0adJECkymuLq6om3btgbbMjIy8M4772Dr1q3IzMyEVqtFYWEh0tLSaq1LcnIyxo0bV2uZqq8lCAKCg4Olud8aCoYmmfh6uAIAtDoRucXlULsrbnkMERHdmrm3yOyBm5sb+vbti759+2LKlCkYO3Yspk2bhtdee+2Wx7q7uxv9D3dCQgKuX7+OefPmISIiAkqlErGxsSgtrf2uhru7+y1fT6Ew/J4SBAE6ne6Wx9UnDtMRvL6pDE0AcKOAt+iIiAho3bo1CgoK0LZtW1y6dAn//PNPnY7fuXMnXnzxRQwaNAh33303lEolrl27ZlBGoVBAq9UabGvbtq3RuIZkjKFJJn6eVUIT+zURETUo169fR+/evfHdd9/hyJEjSElJwfLly/HRRx9h6NChiIuLQ8+ePfHII49gw4YNSElJwbp167B+/fpaz9u8eXMsXboUJ0+exJ49ezBy5EijVqTIyEhs2rQJGo0GN27cAABMmzYNy5Ytw7Rp03Dy5EkcPXoUs2bNsuo1cEQMTTKpOuwAQxMRUcPi5eWFrl27Yu7cuejZsyfatGmDKVOmYNy4cfjvf/8LAPjll1/QuXNnjBgxAq1bt8Ybb7xh1EJU3bfffosbN26gQ4cOePLJJ/Hiiy8iMDDQoMzs2bOxYcMGhIWFoX379kDFGIjLly/Hb7/9hnbt2qF3797Yu3evFa+AYxJEURTlroQ9y83NhVqtRk5ODlQqlcXOuyQpFVN/PQ4AmP1YDB7pWPs8ekREZKi4uBgpKSmIioqCm5ub3NUhGdT2GbDG9zdbmmTi48Hbc0RERI6EoUkmfgxNREREDoWhSSZV+zRlFXAqFSIiInvH0CSTqk/PcVRwIiIi+8fQJJOq4zRlcZwmIiIiu8fQJBN3V2e4KfSXP7uQt+eIiIjsHUOTjCpbm7J4e46IiMjuMTTJqDI0ZReWgsNlERER2TeGJhn5euqfoCvTisgvKZe7OkRE1IBpNBr07dsXnp6e8PHxkbs6EkEQsHr1armrATA0yctw0l72ayIiaggEQah1mT59uiz1mjt3LtLT05GcnFzniYIbChe5K9CQVZ+0N7yRh6z1ISIi60tPT5f+/dNPP2Hq1Kk4ffq0tM3Ly0v6tyiK0Gq1cHGx/tf1uXPn0LFjRzRv3vy2z1FaWgpXV1ej7WVlZVAoFCaPcSRsaZJR1alU2BmciKhhCA4Olha1Wg1BEKT1U6dOwdvbG+vWrUPHjh2hVCqxY8cOnDt3DkOHDkVQUBC8vLzQuXNnbNy40eC8kZGR+OCDD/DUU0/B29sb4eHh+Prrr6X9paWlmDhxIkJCQuDm5oaIiAjMnDlTOvaXX37BkiVLIAgCRo8eDQDIzs7G2LFjERAQAJVKhd69e+Pw4cPSOadPn4527dph4cKFBvO/CYKAL7/8Eg8++CA8PT3x/vvvAwB+/fVXdOjQAW5uboiOjsaMGTNQXn6ze8qZM2fQs2dPuLm5oXXr1tiwYYOVfxp1w5YmGflVGRWcA1wSEVGlt956C5988gmio6Ph6+uLixcvYtCgQXj//fehVCqxZMkSDBkyBKdPn0Z4eLh03OzZs/Hee+/h7bffxooVKzBhwgTExcWhZcuW+Oyzz/Dbb7/h559/Rnh4OC5evIiLFy8CAPbt24dRo0ZBpVJh3rx5cHd3BwA89thjcHd3x7p166BWq/HVV1+hT58++Oeff+Dn5wcAOHv2LH755ResXLkSzs7OUl2mT5+ODz/8EJ9++ilcXFywfft2jBo1Cp999hnuu+8+nDt3DuPHjwcATJs2DTqdDg8//DCCgoKwZ88e5OTk4OWXX7bxla8dQ5OMfD2rDnDJPk1ERBbxVRyQn2n71/UKBJ7ZZpFTvfvuu+jbt6+07ufnh5iYGGn9vffew6pVq/Dbb79h4sSJ0vZBgwbhueeeAwC8+eabmDt3LrZs2YKWLVsiLS0NzZs3x7333gtBEBARESEdFxAQAKVSCXd3dwQHBwMAduzYgb179yIzMxNKpRIA8Mknn2D16tVYsWKFFHhKS0uxZMkSBAQEGLyHJ554AmPGjJHWn3rqKbz11ltISEgAAERHR+O9997DG2+8gWnTpmHjxo04deoU/vzzT4SGhgIAPvjgAwwcONAi19QSGJpkVLUjOFuaiIgsJD8TyLsidy3uSKdOnQzW8/PzMX36dKxduxbp6ekoLy9HUVER0tLSDMq1bdtW+nflbb/MTH2AHD16NPr27YuWLVtiwIABGDx4MPr161djHQ4fPoz8/Hw0atTIYHtRURHOnTsnrUdERBgFJlPv4fDhw9i5c6d0qw4AtFotiouLUVhYiJMnTyIsLEwKTAAQGxtb63WyNYYmGXEqFSIiK/AKdPjX9fT0NFh/7bXXsGHDBnzyySdo1qwZ3N3d8eijj6K01PC7o3pna0EQoNPpAAAdOnRASkoK1q1bh40bN+Jf//oX4uPjsWLFCpN1yM/PR0hICLZu3Wq0r+qQBNXrWtP2/Px8zJgxAw8//LBR2cq+UPaOoUlGleM0gVOpEBFZjoVukdmTnTt3YvTo0XjooYeAigCSmppa5/OoVCoMHz4cw4cPx6OPPooBAwYgKytL6p9UVYcOHaDRaODi4oLIyMg7fg8dOnTA6dOn0axZM5P7W7VqhYsXLyI9PR0hISEAgN27d9/x61oSQ5OM2NJERETmaN68OVauXIkhQ4ZAEARMmTJFakEy15w5cxASEoL27dvDyckJy5cvR3BwcI0DWcbHxyM2NhbDhg3DRx99hBYtWuDKlStYu3YtHnroIaPbb7cydepUDB48GOHh4Xj00Ufh5OSEw4cP49ixY/i///s/xMfHo0WLFkhISMDHH3+M3Nxc/Oc//6nTa1gbhxyQkYerM1xd9D+CG+zTRERENZgzZw58fX3RvXt3DBkyBP3790eHDh3qdA5vb2989NFH6NSpEzp37ozU1FT88ccfcHIyHQUEQcAff/yBnj17YsyYMWjRogUef/xxXLhwAUFBQXV+D/3798eaNWvw119/oXPnzujWrRvmzp0rdUh3cnLCqlWrUFRUhC5dumDs2LEG/Z/sgSBy0rNa5ebmQq1WIycnByqVyuLn7/rBRmTkliBIpcSet+Mtfn4iovqquLgYKSkpBuMDUcNS22fAGt/fbGmSWeUtuhuFZZy0l4iIyI4xNMmsMjSVlutQWKqVuzpERERUA4YmmVWff46IiIjsE0OTzHyqTKVyg6OCExER2S2GJpmxpYmI6M6wP2jDZeufPUOTzHw8GJqIiG5H5ejXhYWFcleFZFL5s68+Erq1cHBLmfl5Vr09x9BERGQuZ2dn+Pj4SHOreXh4QBAEuatFNiCKIgoLC5GZmQkfHx84Ozvb5HUZmmRmMCo4p1IhIqqT4OBgAJCCEzUsPj4+0mfAFhiaZFY1NLGliYiobgRBQEhICAIDA1FWxv/xbEgUCoXNWpgqMTTJjB3BiYjunLOzs82/QKnhYUdwmRkMOcDQREREZLcYmmTmpXSBwlnfcZHjNBEREdkvhiaZCYIgDTvAliYiIiL7xdBkB/wYmoiIiOweQ5MdqOzXVFymQxEn7SUiIrJLDE12gE/QERER2T+GJjtQdSqVLI7VREREZJcYmuxA1alUsjkqOBERkV1iaLIDhlOpsKWJiIjIHjE02YGqoSmboYmIiMguMTTZAd8qt+fYp4mIiMg+MTTZAcOWJvZpIiIiskcMTXbAl0/PERER2T2GJjvgy3GaiIiI7B5Dkx1QubnA2ali0l6GJiIiIrvE0GQHBEGAb8VUKjcK2KeJiIjIHjE02QlfTtpLRERk1xia7ERlaCos1aK4jJP2EhER2RuGJjvhy6lUiIiI7BpDk53gsANERET2jaHJTlQddoBTqRAREdkfhiY7Ufn0HDhpLxERkV1iaLITVW/P3WCfJiIiIrvjUKHp77//xpAhQxAaGgpBELB69epay2/duhWCIBgtGo3GZnU2l0FoYp8mIiIiu+NQoamgoAAxMTGYP39+nY47ffo00tPTpSUwMNBqdbxdnEqFiIjIvrnIXYG6GDhwIAYOHFjn4wIDA+Hj42NW2ZKSEpSUlEjrubm5dX6921G1TxNbmoiIiOyPQ7U03a527dohJCQEffv2xc6dO2stO3PmTKjVamkJCwuzSR39PNmniYiIyJ7V69AUEhKCBQsW4JdffsEvv/yCsLAw9OrVCwcPHqzxmMmTJyMnJ0daLl68aJO6qtwUqJizl7fniIiI7JBD3Z6rq5YtW6Jly5bSevfu3XHu3DnMnTsXS5cuNXmMUqmEUqm0YS31nJwE+Hi4IquglKGJiIjIDtXrliZTunTpgrNnz8pdDZN8Kvo13Sjg7TkiIiJ70+BCU3JyMkJCQuSuhkl+FcMO5JeUo7RcJ3d1iIiIqAqHuj2Xn59v0EqUkpKC5ORk+Pn5ITw8HJMnT8bly5exZMkSAMCnn36KqKgo3H333SguLsbChQuxefNm/PXXXzK+i5pVn0olUOUma32IiIjoJocKTfv378f9998vrb/66qsAgISEBCQmJiI9PR1paWnS/tLSUkyaNAmXL1+Gh4cH2rZti40bNxqcw54YDDtQWMbQREREZEcEURRFuSthz3Jzc6FWq5GTkwOVSmXV15q57iS+2nYeALBsXDfENm1k1dcjIiKqr6zx/d3g+jTZs6pTqWTzCToiIiK7wtBkR/yqhKYshiYiIiK7wtBkR3yq9GnK5qjgREREdoWhyY5UnUoli/PPERER2RWGJjviU+X2HCftJSIisi8MTXbEcNJehiYiIiJ7wtBkR9TuCggVk/ZmsU8TERGRXWFosiPOTgLU7vrO4BxygIiIyL4wNNmZyrGa2BGciIjIvjA02ZnKqVTyistRpuWkvURERPaCocnOGI4Kzn5NRERE9oKhyc74enIqFSIiInvE0GRnfKuMCs5+TURERPaDocnO+BqM1cTbc0RERPaCocnOVJ20lwNcEhER2Q+GJjvjw9BERERklxia7IzBpL35DE1ERET2gqHJzvh73QxNV/NLZK0LERER3cTQZGeCVG7SvzU5xbLWhYiIiG5iaLIznkoXeCtdAAAZuQxNRERE9oKhyQ4FqfWtTRm5JRBFUe7qEBEREUOTfQquuEVXVKZFbnG53NUhIiIihib7FKhSSv/O5C06IiIiu8DQZIeCq3YGZ2giIiKyCwxNdqjqE3QZuRx2gIiIyB4wNNkhw9DEliYiIiJ7wNBkh4Kq9GniWE1ERET2gaHJDgWr2dJERERkbxia7FCAlxKCoP83QxMREZF9YGiyQy7OTvD30t+iY0dwIiIi+8DQZKcqhx24ml8CrY6jghMREcmNoclOVXYG1+pEXM9naxMREZHcGJrsVBAHuCQiIrIrDE12yiA0cdgBIiIi2TE02amqU6lk5PH2HBERkdwYmuxUUNWxmtjSREREJDuLhSatVosrV65Y6nQNXtVRwTlWExERkfwsFpqOHTuGsLAwS52uwQtmR3AiIiK7wttzdkrtroCri/7Hk8kBLomIiGTH0GSnBEGQWpvY0kRERCQ/hiY7VtmvKaeoDMVlWrmrQ0RE1KC5mFvwyJEjte4/ffq0JepDVVQdqykjtxgRjTxlrQ8REVFDZnZoateuHQRBgCgaz4NWuV0QBEvXr0ELrjbAJUMTERGRfMwOTSkpKdatCRkJ4gCXREREdsPs0BQREWHdmpARDnBJRERkP8zuCF5QUIAJEyagcePGCAgIwOOPP46rV69at3YNXJA3B7gkIiKyF2aHpilTpmDp0qUYPHgwnnjiCWzevBnjx4+3bu0auGA1B7gkIiKyF2bfnlu1ahUWLVqExx57DAAwatQodOvWDeXl5XBxMfs0VAfVn54jIiIi+Zjd0nTp0iX06NFDWu/YsSMUCgXnm7MiN4Uz1O4KAEAGRwUnIiKSldmhSafTQaFQGGxzcXGBVstBF62p6qjgpoZ7ICIiItsw+76aKIro06ePwa24wsJCDBkyBK6urtK2gwcPWr6WDVigSonTGXkoLdchp6gMPh6uZhxFRERElmZ2aJo2bZrRtqFDh1q6PlSNwQCXucUMTURERDK5o9BE1mfYGbwEdwXLWh0iIqIGq06Pve3evRu///47SktL0adPHwwYMMB6NSOAA1wSERHZDbND04oVKzB8+HC4u7tDoVBgzpw5mDVrFl577TXr1rCBqzrAJcdqIiIiko/ZT8/NnDkT48aNQ05ODm7cuIH/+7//wwcffGDd2pHBAJccq4mIiEg+Zoem06dP47XXXoOzszMAYNKkScjLy0NmZqY169fgBXOASyIiIrtgdmgqLCyESqWS1l1dXeHm5ob8/Hxr1Y0ANPJSwtlJADjAJRERkazq1BF84cKF8PLyktbLy8uRmJgIf39/aduLL75o2Ro2cM5OAgK8lNDkFrNPExERkYwE0cxhpiMjIyEIQu0nEwScP3/eUnWzC7m5uVCr1cjJyTFoabOlof/dgcOXciAIwJn/GwgXZ7MbCImIiBoka3x/m/3tm5qaipSUlFoXawemv//+G0OGDEFoaCgEQcDq1atveczWrVvRoUMHKJVKNGvWDImJiVatozVUjtUkisDVfN6iIyIikoNDNVkUFBQgJiYG8+fPN6t8SkoKHnjgAdx///1ITk7Gyy+/jLFjx+LPP/+0el0tqeoAlxqO1URERCSLOvVpktvAgQMxcOBAs8svWLAAUVFRmD17NgCgVatW2LFjB+bOnYv+/fubPKakpAQlJTdbc3Jzcy1Q8ztjOOwAW5qIiIjk4FAtTXWVlJSE+Ph4g239+/dHUlJSjcfMnDkTarVaWsLCwmxQ09oFVhngksMOEBERyaNehyaNRoOgoCCDbUFBQcjNzUVRUZHJYyZPnoycnBxpuXjxoo1qWzMOcElERCQ/h7o9ZwtKpRJKpdKMkrZTdYBLDjtAREQkD4u1NB08eBCDBw+21OksIjg4GBkZGQbbMjIyoFKp4O7uLlu96iqwSmjKZJ8mIiIiWdQpNP3555947bXX8Pbbb0vDC5w6dQrDhg1D586dodPprFXP2xIbG4tNmzYZbNuwYQNiY2Nlq9PtULm5wF2hn76GLU1ERETyMDs0ffvttxg4cCASExMxa9YsdOvWDd999x1iY2MRHByMY8eO4Y8//rBqZfPz85GcnIzk5GSgYkiB5ORkpKWlARX9kUaNGiWVf/bZZ3H+/Hm88cYbOHXqFL744gv8/PPPeOWVV6xaT0sTBAFBKv0twwwOOUBERCQLs0PTvHnzMGvWLFy7dg0///wzrl27hi+++AJHjx7FggUL0KpVK+vWFMD+/fvRvn17tG/fHgDw6quvon379pg6dSoAID09XQpQABAVFYW1a9diw4YNiImJwezZs7Fw4cIahxuwZ5VjNeWVlKOgpFzu6hARETU4Zk+j4unpiePHjyMyMhKiKEKpVGLLli3o0aOH9WspI3uYRgUAXlx2CL8dvgIA2DwpDtEBXrc8hoiIqKGSdRqVoqIieHh4ABW3i5RKJUJCQixSCbo1DnBJREQkrzoNObBw4UJ4eelbOMrLy5GYmAh/f3+DMi+++KJla0hAtalUOFYTERGR7ZkdmsLDw/HNN99I68HBwVi6dKlBGUEQGJqspLIjOBiaiIiIZGF2aEpNTbVuTahWHOCSiIhIXmb3aerduzeys7OtWxuqEW/PERERycvs0LR161aUlpZatzZUo0CD23PsCE5ERGRr9XrC3vpE6eIMXw8FAEDDAS6JiIhsrk5Pz504cQIajabWMm3btr3TOlENglRuuFFYhsy8YoiiCEEQ5K4SERFRg1Gn0NSnTx+YGgtTEATpS1yr1VqyflRFsNoNpzR5KNOKyCooRSMvpRlHERERkSXUKTTt2bMHAQEB1qsN1SrI23CAS4YmIiIi26lTaAoPD0dgYKD1akO1ClIbPkHXOlS+aV2IiIgaGot1BD9y5AhcXV0tdToyoeoAlxyriYiIyLbMDk1xcXG1hiJRFNmfycqCOVYTERGRbMy+Pbdlyxbr1oRuiQNcEhERyYfjNDkQw9DEAS6JiIhsyeyWptzc3Fr35+XlWaI+VItGnq5wcRJQrhM5wCUREZGNmR2afHx8ah1MkYMtWp+Tk4BAbyWu5BQjM4+hiYiIyJbYp8nBBKndcCWnGNfyS1FaroOrC++wEhER2YLZoSkuLs66NSGzVB3gMjOvGE18PWStDxERUUPBZgoHE6xmZ3AiIiI5MDQ5mMAqA1xy2AEiIiLbYWhyMBzgkoiISB4MTQ6mamjiVCpERES2c0ehadmyZSgoKLBcbeiWAquEpkz2aSIiIrKZOwpNzzzzDDIyMixXG7qlqh3BOcAlERGR7dxRaBJF0XI1IbN4KV3g6eoMsE8TERGRTbFPkwMKqmhtYmgiIiKynTsKTevWrUPjxo0tVxsyS+UAlwWlWuQVl8ldHSIiogbhjkLTvffeC6VSaUZJsiQOcElERGR7vD3ngII4VhMREZHNMTQ5oCCOCk5ERGRzDE0OiANcEhER2Z7FQtOlS5cwfvx4S52OalF1gMsMjtVERERkExYLTdevX8e3335rqdNRLazdEbxcq8OSpFQs2HYO5Vqdxc9PRETkiFzkrgDVXYDXzT5Nlr49p8kpxovLDmFvahYAQKsT8fz9zSz6GkRERI6IfZockKuLE/y9XAEAmRYMTdvPXMUDn22XAhMAfP33eeRyLCgiIiKGJkdVOexAZl4JdLo7m85GqxMxZ8M/GPW/vbheUAoAEAT9vpyiMizemXrnFSYiInJwZt+ee/jhh2vdn52dbYn6kJmCVG44fiUX5ToR1wtKEeB9e4OMXs0rwcs/HcLOs9elbfe3DMCLfZrj0QVJ0OpEfLP9PBJ6RELlprDgOyAiInIsZocmtVp9y/2jRo2yRJ3IDNUHuLyd0LTn/HW8sOwQMvP0ncmdBGBSv5aYENcUTk4CHmrfGCsOXEJucTkW7UjFS/HNLfoeiIiIHInZoWnRokXWrQnVSdUBLjU5xWjTuPZQW5VOJ2LB3+fwyZ+nUXlnL8Bbic9HtEe36EZSuRd6N8OqQ5eh1Yn4dsd5jO4RCbU7W5uIiKhhqlOfptTUVHzzzTeYP38+jh8/br1a0S1VHeAyI8/8zuA3Ckrx9OJ9+Gj9zcDUo1kj/PHifQaBCQAiGnni4fb6CZlzi8uxaGeKpapPRETkcMxuadqyZQsGDx6MoqIi/YEuLvjf//6Hf//739asH9Ug6DYGuDyYdgMTvz+IKxXlBQF4sXdzvNinOZydBJPHvNC7OVYduoxynYhvd6RgTI8otjYREVGDZHZL05QpU9C3b19cvnwZ169fx7hx4/DGG29Yt3ZUI8M+TbUPcCmKIhZuP49/LUiSAlMjT1cseaoLXunbosbABADhjTzwSIcmAIC84nL8bwdbm4iIqGEyOzQdO3YMH3zwAUJCQuDr64uPP/4YmZmZuH79uhlHk6VVHRW8tgEuc4rK8Ox3B/B/a0+ivOJ+XOdIX6x98T7c1zzArNea2LsZXCqC1f92pCCnkOM2ERFRw2N2aMrNzYW/v7+07uHhAXd3d+Tk5FirblQLXw8FXJ31P76MGkLTscs5GPL5Dvx5PEPa9kxcNH4Y180gdN1KmJ8HHu1Y0dpUUo5v2beJiIgaoDpNo/Lnn38aDD2g0+mwadMmHDt2TNr24IMPWraGZJIgCAhUKXHpRpFRaBJFEd/vScO7v59AacXccWp3Beb8KwZ9WgXd1us9f38zrDhwCeU6EYt2pODpHlFQe7BvExERNRx1Ck0JCQlG25555hnp34IgQKvVWqZmdEtBKjdculGEG4VlKC7Twk3hjPyScry98ih+O3xFKhcT5oP5T7RHE1+P236tMD8PPNapCZbtvYi8knIs3HEek/q1tNA7ISIisn9m357T6XS3XBiYbKvqsANX80pwSpOLBz/fYRCYxvSIxPJnYu8oMFV6/v5mUDjr+zYt2pmK7MLSOz4nERGRo6jz3HMlJSUoKCiwTm2oTgKrDHD59d/nMWz+Tpy/pv/ZeCtd8OXIDpg25G64ulhmisEmvh54rFMYACC/pBwLt7NvExERNRxmf5tevXoVAwcOhJeXF1QqFbp164azZ89at3ZUq6otTUt3X0Bxmb7/UusQFX5/4V4MvCfE4q9p2NqUghsFbG0iIqKGwezQ9OabbyI5ORnvvvsuPvnkE2RnZ2PcuHHWrR3VytQTcCO6hGPlc90R6e9pldds7OOOf1W0NhWUarFwx3mrvA4REZG9Mbsj+IYNG5CYmIj+/fsDAAYPHoxWrVqhpKQESmXdJ4ulOxfR6GYw8nB1xgcP3YNhFdOeWNNz9zfDz/svokwrInFnKp6+Nxp+nq5Wf10iIiI5md3SdOXKFcTExEjrzZs3h1KpRHp6urXqRrcQ00SN5+9viiExofhtYg+bBCZUtDYN73yztemb7WxtIiKi+q9OPYSdnZ2N1kVRtHSdyEyCIOD1/nfh8xHt0SzQ26av/VyvZtLgmot3pSKLfZuIiKieMzs0iaKIFi1awM/PT1ry8/PRvn17g23UMIRWaW0qLNXi67/Z2kRERPWb2X2aFi1aZN2akMN57v6m+GnfRZRqdViSlIpx90WhkRf7txERUf1kdmgyNRo4NWwhaneM6BKGxUkX9K1N289j8sBWcleLiIjIKiwz6iE1WBN6NZMGz1yy6wKu5ZfIXSUiIiKrYGiiOxKsdsMTXcIBAEVlWnzDvk1ERFRPMTTRHZvQq+nN1qYktjYREVH95HChaf78+YiMjISbmxu6du2KvXv31lg2MTERgiAYLG5uxqNo050JUhm2Nn217ZzcVSIiIrI4i4Wm8+fPo1+/fpY6nUk//fQTXn31VUybNg0HDx5ETEwM+vfvj8zMzBqPUalUSE9Pl5YLFy5YtY4N1XO9mkJZ0dq0dPcFXM1jaxMREdUvFgtNeXl52LRpk6VOZ9KcOXMwbtw4jBkzBq1bt8aCBQvg4eGB//3vfzUeIwgCgoODpSUoKMiqdWyoAlVueKKrvrWpuEzH1iYiIqp3HOb2XGlpKQ4cOID4+Hhpm5OTE+Lj45GUlFTjcfn5+YiIiEBYWBiGDh2K48eP1/o6JSUlyM3NNVjIPBPibrY2fbfnAjLziuWuEhERkcU4TGi6du0atFqtUUtRUFAQNBqNyWNatmyJ//3vf/j111/x3XffQafToXv37rh06VKNrzNz5kyo1WppCQsLs/h7qa8CVW74d7cIQGpt4pN0RERUfzhMaLodsbGxGDVqFNq1a4e4uDisXLkSAQEB+Oqrr2o8ZvLkycjJyZGWixcv2rTOju6ZuGi4KSpam3ZfQGYuW5uIiKh+MHtE8Pbt20MQhBr3FxYWWqpOJvn7+8PZ2RkZGRkG2zMyMhAcHGzWORQKBdq3b4+zZ8/WWEapVEKp5FQgtyvQ2w3/7hqBhTtSUFKuw4Jt5zF1SGu5q0VERHTHzA5Nw4YNs25NbsHV1RUdO3bEpk2bpLrodDps2rQJEydONOscWq0WR48exaBBg6xc24btmbim+G7PBRSX6fD9ngt4Ni4agSoO9UBERI7N7NA0ZswYNGnSBE5O8t3Re/XVV5GQkIBOnTqhS5cu+PTTT1FQUIAxY8YAAEaNGoXGjRtj5syZAIB3330X3bp1Q7NmzZCdnY2PP/4YFy5cwNixY2V7Dw1BgLcST3aLwDfb9a1NX2w9h+kP3i13tYiIiO6I2aEpKioK6enpCAwMtG6NajF8+HBcvXoVU6dOhUajQbt27bB+/Xqpc3haWppBqLtx4wbGjRsHjUYDX19fdOzYEbt27ULr1rxdZG3PxDXF0t361qYf9qZhQq+mCGJrExEROTBBFEXRnIJOTk7QaDSyhiY55ObmQq1WIycnByqVSu7qOJQP/jiJryvmohvdPZKtTUREZDPW+P6u07222jqCE1U3vmc03BXOAIAf9qZBk8Mn6YiIyHGZfXsOAKZMmQIPD49ay8yZM+dO60T1hL+XEqO6R+CrbedRWq7Dl1vPYsbQNnJXi4iI6LbUKTQdPXoUrq6uNe5nSxRVN/6+aCxNuoDCUi2W7b2IZ3s1RYjaXe5qERER1VmdQtOqVasaXJ8mujONvJQYFRuJBdvOoVSrwxdbzuG9YWxtIiIix2N2nya2ItHtGt8zGh6u+r5NP+27iCvZRXJXiYiIqM7MDk3mPGRXVMQvQzLm5+mKhO6RAKBvbdpa84jsRERE9srs0LRo0SKo1WqT+0pKSjB79mxERUVZsm5Uj4y7LxqebG0iIiIHZnZoevzxxzF9+nR06tQJ3bt3x+rVq4GKMBUVFYVPP/0Ur7zyijXrSg6samtTmVbE/C1sbSIiIsdidmiaOnUqvvzyS0RGRiI1NRWPPfYYxo8fj7lz52LOnDlITU3Fm2++ad3akkMbd180vJT6Zw9+3n8Rl25Yd5JnIiIiSzI7NC1fvhxLlizBihUr8Ndff0Gr1aK8vByHDx/G448/DmdnZ+vWlByer6crRldpbfpi6zm5q0RERGQ2s0PTpUuX0LFjRwBAmzZtoFQq8corr/CpOqqTsfdFSa1Ny9naREREDsTs0KTVag0GtnRxcYGXl5e16kX1lI+HK8b0YN8mIiJyPGYPbimKIkaPHg2lUgkAKC4uxrPPPgtPT0+DcitXrrR8LaleefreKCTuTEVeSTmW77+E53o1Q5hf7dPzEBERyc3slqaEhAQEBgZCrVZDrVbj3//+N0JDQ6X1yoXoVqq2NpXr2NpERESOQRDNGbWyAcvNzYVarUZOTg5UKpXc1ak3cgrLcO9Hm5FXXA4XJwFbXuvF1iYiIrIYa3x/m93SRGRJag8FnuqhHwy1XCfiv5vZ2kRERPaNoYlk89S9UfB203erW3HwEtKu80k6IiKyXwxNJBu1uwJP36tvbdLqRHy++YzcVSIiIqoRQxPJakyPm61NKw9dxoXrBXJXiYiIyCSGJpKV2l2BsfdGA1JrE/s2ERGRfWJoItmNuTcSqorWplWHLiP1GlubiIjI/jA0kexUbgqMvY+tTUREZN8YmsgujOkRCbW7AgCw6tAlpLC1iYiI7AxDE9kFbzcFxt2nf5JOJ4JP0hERkd1haCK7kdA9Ej4e+tam1Ycu4/zVfLmrREREJGFoIruhb23S923StzaxbxMREdkPhiayK6NiI6TWpl+TL+McW5uIiMhOMDSRXTFqbdrEvk1ERGQfGJrI7iR0j4RvRWvTb4ev4GwmW5uIiEh+DE1kd7yULhjXs2rfJrY2ERGR/BiayC4lxEbCz9MVkFqb8uSuEhERNXAMTWSXPJUuGF/R2iSKwGeb+CQdERHJi6GJ7NaT3SKk1qbfj1zBmQy2NhERkXwYmshueSpd8EyV1qZ5fJKOiIhkxNBEdu3J2Ag0qmhtWns0Hf+wtYmIiGTC0ER2zcPVBc/EsbWJiIjkx9BEdu/f3SLg76VvbfrjaDpOa9jaREREtsfQRHbPw9UFz/RsCkhP0rG1iYiIbI+hiRyCvrVJCVT0bTqlyZW7SkRE1MAwNJFDcHd1xrMVfZsAYN5GtjYREZFtMTSRwxjZ9WZr07pjGpxMZ2sTERHZDkMTOQx3V2dM6NVUWmdrExER2RJDEzmUkV3DEeCtb21af1yDE1fY2kRERLbB0EQOxU3hjAlxVVqbNv0ja32IiKjhYGgih/NE13AEVrQ2/Xk8A8ev5MhdJSIiagAYmsjhuCnYt4mIiGyPoYkc0ogu4QhS6Vub/jqRgWOX2dpERETWxdBEDslN4YznejWT1j9laxMREVkZQxM5rOGdwxCscgMAbDzJ1iYiIrIuhiZyWG4KZzx3/82+TZ9u5JN0RERkPQxN5NAMW5syceRSttxVIiKieoqhiRya0sUZz1dpbVpx4JKs9SEiovrLRe4KEN2pf3UOw4aTmRjeKQwD2wTLXR0iIqqnGJrI4SldnLHkqS5yV4OIiOo53p4jIiIiMgNDExEREZEZGJqIiIiIzMDQRERERGQGhiYiIiIiMzA0EREREZnB4ULT/PnzERkZCTc3N3Tt2hV79+6ttfzy5ctx1113wc3NDffccw/++OMPm9WViIiI6g+HCk0//fQTXn31VUybNg0HDx5ETEwM+vfvj8zMTJPld+3ahREjRuDpp5/GoUOHMGzYMAwbNgzHjh2zed2JiIjIsQmiKIpyV8JcXbt2RefOnfHf//4XAKDT6RAWFoYXXngBb731llH54cOHo6CgAGvWrJG2devWDe3atcOCBQvMes3c3Fyo1WpcvXoVKpXKaL+TkxNcXG6OEVpaWlrjuQRBgEKhuK2yZWVlqOlHZa2yAODq6npbZcvLy6HT6SxSVqFQQBCEmstmpQA7PwPCu0ER3QOCXxQgCHU6r1arhVartUhZFxcXODk52U1ZnU6H8vLyGss6OzvD2dnZbsqKooiysjKLlK36+2mtsrjF7zL/Rpgua9O/ERYoy78Rjvc3ovL7Oycnx+T39+1wmBHBS0tLceDAAUyePFna5uTkhPj4eCQlJZk8JikpCa+++qrBtv79+2P16tU1vk5JSQlKSkqk9dzcXADA7Nmz4ebmZlS+efPmeOKJJ6T1Tz75pMYfZkREBEaPHi2tz5s3D4WFhSbLhoaGYty4cdL6/PnzkZOTY7JsQEAAnnvuOWn9m2++wdWrV02WVavVePnll6X1xMREXLlyxWRZDw8PvP7669L6999/jwsXLpgsq1Ao8Pbbb0vrP//8M86cOWOyLABMmzZN+veqVatw4sSJGstOnjxZ+gO6Zs0aHD582ESpQODYebwmvg5Pb18gojv+zGuF/WkFNZ73pZdego+PDwBg06ZNNX6OAGDChAkIDAwEAGzfvh3btm2rsezYsWPRuHFjAMDu3buxcePGGssmJCQgMjISAHDgwAGsW7euxrIjRoxAixYtAABHjx7Fr7/+WmPZRx99FHfffTcA4OTJk1ixYkWNZYcOHYp27doBAM6ePYtly5bVWHbgwIHo0kU/+npaWhoWL15cY9n4+Hj06NEDAJCeno6FCxfWWDYuLg69evUCAFy9ehVffvlljWVjY2PRr18/AEBOTg7mzZtXY9lOnTrhgQceAAAUFhbik08+qbFsTEwMhg0bBlR8+c+cObPGsq1bt8Zjjz0mrddWln8j9OT/G6H32muvwdPTEwDw559/Yv/+/TWW5d8IPUf+G2FpDnN77tq1a9BqtQgKCjLYHhQUBI1GY/IYjUZTp/Ko+OOnVqulJSwszELvgGwm7wpwbAVwYafcNSEionrEYW7PXblyBY0bN8auXbsQGxsrbX/jjTewbds27Nmzx+gYV1dXLF68GCNGjJC2ffHFF5gxYwYyMjJMvo6plqawsDDenrPnpveyIuBKMnBxNxQXd0G4tBcoK0Q5nKGDYOKE3kBYZygiu0KIvBcIbQ+t4MKmdzspy9tz/BtR17K8PXdnZe3h95635yzM398fzs7ORmEnIyMDwcGmZ7YPDg6uU3kAUCqVUCqVRttdXV0NfolrYk6Z2ylb9Y+YI5St+iVh9bKurkDzOP0CANoyIP0wXC7sBC4kAWm7gOIqty3KbgDn/9IvAODiBucmneEc0R2I6A406Qy4etZYh6q/vLdiD2WdnJzM/qzZQ1lBEByqLKz4e8+/EfZT1h5+l/k3Qq+uv5+W5DChydXVFR07dsSmTZukfgc6nQ6bNm3CxIkTTR4TGxuLTZs2Gdyf37Bhg0FLFdVDzgqgSSf90uMlQKcDMk8AF3bpb9ld2AUUVHnisrwYSN2uXwDAyQUIba8PUBE9gLCugLuPbG+HiIjsg8PcnkPFkAMJCQn46quv0KVLF3z66af4+eefcerUKQQFBWHUqFFo3Lix1Clz165diIuLw4cffogHHngAP/74Iz744AMcPHgQbdq0Mes1rdG8RzITReD6uZsBKm0XkJ1WywECENwGCK9oiYroDngF2rDCRERUVw369hwqhhC4evUqpk6dCo1Gg3bt2mH9+vVSZ++0tDTpHi0AdO/eHT/88APeeecdvP3222jevDlWr15tdmCiekoQAP9m+qVjgn5b9kUgLelmkLr2T5UDREBzVL/s/Uq/qVHzmy1REd0BHz4wQERU3zlUS5Mc2NLUQOVf1bdAXahYNEf14akm6vCbrVAR3YFGzfThjIiIZGGN72+GpltgaCIAQFE2cHHvzZaoKwcBXc1PesAzwLAlKvBuwMlhRvggInJ4DE0yYGgik0oLgUv7bnYuv7QfKC+qubybGgiPvRmkQmL0HdaJiMgqGJpkwNBEZikvBdKTq3Qu3w2U5NZcXuEBhHW52bm8SSdA4W7LGhMR1WsMTTJgaKLbotMCGccMhzkovF5zeScF0LhjlWEOugBu/LwREd0uhiYZMDSRRYgicO3MzQB1YSeQe7nm8oITENz2Zp+o8FjAs5Eta0xE5NAYmmTA0ERWIYr6saGqtkRlnav9mIC7brZEhbYHfMLZL4qIqAYMTTJgaCKbycswHOYg41jt5QUnQB0G+EUBftGAb8V//aL0/3b1sFXNiYjsDkOTDBiaSDaFWcDFPVWGOUgGxJon4DTiFXwzRFUGqcp1d19r1pyISHYMTTJgaCK7UZIPXNqrfzLv2j9A1nkgK6X2p/Rq4u5rGKKqtlR5BXJgTiJyeA1+GhWiBk3pBTTtrV8qiaK+RepGys0QlXX+5nrBVdPnKrqhX64cNN6n8KxomYo0DlXqJoCTeTOnExHVNwxNRI5MEPRP1Xk20o/1VF1Jnj5IGYWqVCDnkumpYcoK9P2pTPWpclIAvhHGfaj8ovUd012U1nmfRER2gKGJqD5TegMhbfVLdWXF+if4qrZMVYaq7DRAV2Z8jK4MuH5WvxgRKjqmR5rumK70sspbJCKyFYYmooZK4QYEtNAv1WnLgdxL1W73pdxstSorNHFCEchJ0y8pfxvv9gw0DFFVW6ncfdmPiojsHjuC3wI7ghNVI4pAfoZx/6nK9eLsup9Tqb75lF/1ViqvYE52TER1xo7gRCQ/QQC8g/VLRKzxfqljeopxf6p8jelzluTo5+5LTzbe5+JerVN6lZYqdRjgzD9jRGQb/GtDRJbl4adfGnc03ldaoO+EbupJv5xLgKgzPqa8CLh6Ur9U5+Si74BuavgE30j9LUgiIgvh7blb4O05IhspL9V3QDc1fMKNVEBbWscTCoAqtCJEmRg+gRMiE9VrvD1HRPWXiyvg30y/VKfTArlXTPShqrj9V5pv4oSiflLk3MtA6nbj3R7+pvtQ+UUDHo3YMZ2IjDA0EZH9c3IGfML0C+IM94mifhDPmjqmF2WZPmfhNf1yaZ/xPlfvmjume4eyYzpRA8XQRESOTRD0U794BQLhXY33F2VX6ZhebfiEvCumz1maB2iO6JfqnJWmb/f5Ren7VzkrLP8eicguMDQRUf3m7gO4twdC2xvvKyuq6JhuopUqO830BMnaEuDaaf1SneCsn2rG1Jx+vpGAq4d13iMR2QRDExE1XAp3ILCVfqlOWwbkXDTuP1U5DU15sfExohbIvqBfzm8x3u8dUuVWX7WR0919rPMeichiGJqIiExxVlSEm2jjfTodkJdew0TJKUBJrulz5qXrl7RdxvvcfU13SveN0t96ZMd0ItlxyIFb4JADRFQnoqgf4NNUp/QbKfpO63Wl8DQe2LMyVKka6zvKE5EBDjlARGTvBAHwbKRfwjob7y/OvTnAZ/XhE3Iv64dKqK6sAMg4pl+qc3YFfCJMP+nnE6EfyoGILIKhiYjIltxUQEhb/VJdWbG+P5TJjukXAF258THaUuD6Gf1SneAEqJrUPHyCq6d13iNRPcXQRERkLxRuQEBL/VKdthzIvVTtdl+VKWnKi4yPEXVATpp+SdlmvN8ryHQfKr8o/VQ4RGSAoYmIyBE4u+iHLfCNBJpW2yeKQJ7GsGVK+vd5oDjH9DnzM/TLxd3G+9zUNXdM9w5mx3RqkBiaiIgcnSAAqhD9EtHdeH9hluGgnlVv/eVnmD5ncQ5w5ZB+qU7hcXOAz+oDfaqa6AMeUT3ETzYRUX3n4adfGnc03leSr7/NZ+pJv5xL+lt81ZUVApkn9Et1Ti76kdFNtVL5ROhvQRI5KIYmIqKGTOkFBLfRL9WVl+pHRjc1fEL2BX0n9Op05TdvCxoR9EMkmBo+wTdK30meyI4xNBERkWkuroB/M/1SnU6rHyLB6Em/is7pZQUmTijqO7PnXgJStxvv9vA3PQWNXxTg0Yj9qEh2DE1ERFR3Ts7623A+4UB0nOE+UdQP4mk0WnrFelGW6XMWXtMvl/Ya71OqapgoOVo/PY2Tk3XeJ1EVDE1ERGRZgqCf+sUrEAjvZry/KNt4YM/K9bx00+csyQU0R/RLdS5uFU8WVu1DVXHLzydcPyUOkQUwNBERkW25+wDu7YHQ9sb7SguNO6ZX/jv7on5S5OrKi4Grp/RLdYIz4BNWw/AJkfpJm4nMxNBERET2w9UDCGqtX6rTluk7ppscPiEF0JYYHyNqK0JYqunX8w6tuWO6u4/l3x85NIYmIiJyDM4KoFFT/VKdTqe/tWfqSb+sFKA0z/Q5867olws7jfe5+9XcMd0zgB3TGyCGJiIicnxOToC6sX6Jus9wnygChddNd0rPOq/vfG5KURZwOQu4vN94n6vXzSlnqrdSqRrrO8pTvcPQRERE9ZsgAJ7++iWss/H+4twqt/zOG97+y72sHyqhutJ8IOOofqnO2VU/kKepViqfcP1QDuSQGJqIiKhhc1MBITH6pbqyYv1AnqaGT8hO0w/mWZ22FLh+Rr9UJzgB6iY1T5Ts6mmd90gWwdBERERUE4UbENBSv1SnLQdyLtYwfEIKUF5kfIyo04et7DQgZZvxfq8gE0/6Vdz+8/CzznskswmiKJpod6RKubm5UKvVyMnJgUrFIf6JiMgMogjkaUzP6Zd1Xj8hcl25+ZjulO4XrQ9b7JhuwBrf32xpIiIisjRBAFQh+iWiu/H+wizDMaiqhqr8DNPnLM4GrhzSL9UpPGrpmN4EcObXvSXwKhIREdmah59+adLReF9Jvn5cKaMn/VL08/aJOuNjygqBzOP6pTonhb4DuqmO6b4RgIvSOu+xHmJoIiIisidKLyC4jX6prrxE3x/K1PAJN1IBXZnxMboyIOucfjEiVHRMjzQc2LPy30pvq7xFR8XQRERE5ChclIB/c/1SnU6rHyLBqA9VxVJWYOKEor4ze85FIHW78W7PgBqe9IvWt5Q1sH5UDE1ERET1gZOz/jacTzgQ3ctwnygC+Zk1d0wvumH6nAVX9culvcb7lCrjlqnKUOUdoh9wtJ5haCIiIqrvBAHwDtIv4d2M9xfdqNYxvUqfqrx00+csyQXSD+uX6lzcbt7yq9pB3S8aUIc7bMd0x6w1ERERWY67L9DYF2jcwXhfaWHFpMcmWqmy00x3TC8vBq6e0i/VCRUtYlVbpkJijKe/sUMMTURERFQzVw8gqLV+qU5bdrNjulGoSgW0JcbHiFp92RspwLnN+m0tBzE0ERERUT3mrAAaNdUv1el0QN6VGiZKTgFK826W9Y2yabVvF0MTERERWZ5TxTx76ibGrUiiCBRevxmiGjWTq5Z1wtBEREREtiUIgKe/fgnrIndtzFb/ngckIiIisgKGJiIiIiIzMDQRERERmYGhiYiIiMgMDE1EREREZmBoIiIiIjIDQxMRERGRGRiaiIiIiMzgMKEpKysLI0eOhEqlgo+PD55++mnk5+fXekyvXr0gCILB8uyzz9qszkRERFR/OMyI4CNHjkR6ejo2bNiAsrIyjBkzBuPHj8cPP/xQ63Hjxo3Du+++K617eHjYoLZERERU3zhEaDp58iTWr1+Pffv2oVOnTgCAzz//HIMGDcInn3yC0NDQGo/18PBAcHCw2a9VUlKCkpKbszLn5ubeYe2JiIioPnCI23NJSUnw8fGRAhMAxMfHw8nJCXv27Kn12O+//x7+/v5o06YNJk+ejMLCwlrLz5w5E2q1WlrCwsIs9j6IiIjIcTlES5NGo0FgYKDBNhcXF/j5+UGj0dR43BNPPIGIiAiEhobiyJEjePPNN3H69GmsXLmyxmMmT56MV199VVrPzc1lcCIiIiJ5Q9Nbb72FWbNm1Vrm5MmTt33+8ePHS/++5557EBISgj59+uDcuXNo2rSpyWOUSiWUSqW0LooiwNt0REREDqXye7vye9wSZA1NkyZNwujRo2stEx0djeDgYGRmZhpsLy8vR1ZWVp36K3Xt2hUAcPbs2RpDU3V5eXkAwNYmIiIiB5SXlwe1Wm2Rc8kamgICAhAQEHDLcrGxscjOzsaBAwfQsWNHAMDmzZuh0+mkIGSO5ORkAEBISIjZx4SGhuLixYvw9vaGIAhmH1d5W+/ixYtQqVRmH1ef8ZoY4vUwxmtiiNfDGK+JIV4PY5XXJC0tDYIg1PqwWF05RJ+mVq1aYcCAARg3bhwWLFiAsrIyTJw4EY8//rh0MS5fvow+ffpgyZIl6NKlC86dO4cffvgBgwYNQqNGjXDkyBG88sor6NmzJ9q2bWv2azs5OaFJkya3XXeVSsUPcjW8JoZ4PYzxmhji9TDGa2KI18OYWq22+DVxiKfnUPEU3F133YU+ffpg0KBBuPfee/H1119L+8vKynD69Gnp6ThXV1ds3LgR/fr1w1133YVJkybhkUcewe+//y7juyAiIiJH5RAtTQDg5+dX60CWkZGRBp29wsLCsG3bNhvVjoiIiOo7h2lpcjRKpRLTpk0zeBKvoeM1McTrYYzXxBCvhzFeE0O8HsaseU0E0ZLP4hERERHVU2xpIiIiIjIDQxMRERGRGRiaiIiIiMzA0ERERERkBoYmK5k/fz4iIyPh5uaGrl27Yu/evXJXySamT58OQRAMlrvuukvaX1xcjOeffx6NGjWCl5cXHnnkEWRkZMhaZ0v7+++/MWTIEISGhkIQBKxevdpgvyiKmDp1KkJCQuDu7o74+HicOXPGoExWVhZGjhwJlUoFHx8fPP3008jPz7fxO7GMW12P0aNHG31mBgwYYFCmPl2PmTNnonPnzvD29kZgYCCGDRuG06dPG5Qx5/ckLS0NDzzwADw8PBAYGIjXX38d5eXlNn43lmHONenVq5fR5+TZZ581KFNfrsmXX36Jtm3bSgNWxsbGYt26ddL+hvb5gBnXxFafD4YmK/jpp5/w6quvYtq0aTh48CBiYmLQv39/o/nz6qu7774b6enp0rJjxw5p3yuvvILff/8dy5cvx7Zt23DlyhU8/PDDstbX0goKChATE4P58+eb3P/RRx/hs88+w4IFC7Bnzx54enqif//+KC4ulsqMHDkSx48fx4YNG7BmzRr8/fffBhNQO5JbXQ8AGDBggMFnZtmyZQb769P12LZtG55//nns3r0bGzZsQFlZGfr164eCggKpzK1+T7RaLR544AGUlpZi165dWLx4MRITEzF16lSZ3tWdMeeaAMC4ceMMPicfffSRtK8+XZMmTZrgww8/xIEDB7B//3707t0bQ4cOxfHjx4EG+PmAGdcEtvp8iGRxXbp0EZ9//nlpXavViqGhoeLMmTNlrZctTJs2TYyJiTG5Lzs7W1QoFOLy5culbSdPnhQBiElJSTaspe0AEFetWiWt63Q6MTg4WPz444+lbdnZ2aJSqRSXLVsmiqIonjhxQgQg7tu3Tyqzbt06URAE8fLlyzZ+B5ZV/XqIoigmJCSIQ4cOrfGY+nw9RFEUMzMzRQDitm3bRNHM35M//vhDdHJyEjUajVTmyy+/FFUqlVhSUiLDu7Cs6tdEFEUxLi5OfOmll2o8pr5fE19fX3HhwoX8fFRReU1EG34+2NJkYaWlpThw4ADi4+OlbU5OToiPj0dSUpKsdbOVM2fOIDQ0FNHR0Rg5ciTS0tIAAAcOHEBZWZnBtbnrrrsQHh7eYK5NSkoKNBqNwTVQq9Xo2rWrdA2SkpLg4+ODTp06SWXi4+Ph5OSEPXv2yFJva9u6dSsCAwPRsmVLTJgwAdevX5f21ffrkZOTA1TMegAzf0+SkpJwzz33ICgoSCrTv39/5ObmGvyft6Oqfk0qff/99/D390ebNm0wefJkados1ONrotVq8eOPP6KgoACxsbH8fJi4JpVs8flwmGlUHMW1a9eg1WoNfjAAEBQUhFOnTslWL1vp2rUrEhMT0bJlS6Snp2PGjBm47777cOzYMWg0Gri6usLHx8fgmKCgIGg0GtnqbEuV79PU56Nyn0ajQWBgoMF+FxcX+Pn51cvrNGDAADz88MOIiorCuXPn8Pbbb2PgwIFISkqCs7Nzvb4eOp0OL7/8Mnr06IE2bdoAFT//W/2eaDQak58hVPmMOSpT1wQAnnjiCURERCA0NBRHjhzBm2++idOnT2PlypVAPbwmR48eRWxsLIqLi+Hl5YVVq1ahdevWSE5ObrCfj5quCWz4+WBoIosaOHCg9O+2bduia9euiIiIwM8//wx3d3dZ60b26fHHH5f+fc8996Bt27Zo2rQptm7dij59+shaN2t7/vnncezYMYN+fw1dTdekah+2e+65ByEhIejTpw/OnTuHpk2bylBT62rZsiWSk5ORk5ODFStWICEhocHPp1rTNWndurXNPh+8PWdh/v7+cHZ2NnqSISMjA8HBwbLVSy4+Pj5o0aIFzp49i+DgYJSWliI7O9ugTEO6NpXvs7bPR3BwsNFDA+Xl5cjKymoQ1yk6Ohr+/v44e/YsUI+vx8SJE7FmzRps2bIFTZo0kbab83sSHBxs8jOEKp8xR1TTNTGla9euAGDwOalP18TV1RXNmjVDx44dMXPmTMTExGDevHkN+vNR0zUxxVqfD4YmC3N1dUXHjh2xadMmaZtOp8OmTZsM7r02FPn5+Th37hxCQkLQsWNHKBQKg2tz+vRppKWlNZhrExUVheDgYINrkJubiz179kjXIDY2FtnZ2Thw4IBUZvPmzdDpdNIfgvrs0qVLuH79OkJCQoB6eD1EUcTEiROxatUqbN68GVFRUQb7zfk9iY2NxdGjRw3C5IYNG6BSqaTbFY7kVtfElOTkZAAw+JzUp2tSnU6nQ0lJSYP8fNSk8pqYYrXPxx12XicTfvzxR1GpVIqJiYniiRMnxPHjx4s+Pj4Gvfbrq0mTJolbt24VU1JSxJ07d4rx8fGiv7+/mJmZKYqiKD777LNieHi4uHnzZnH//v1ibGysGBsbK3e1LSovL088dOiQeOjQIRGAOGfOHPHQoUPihQsXRFEUxQ8//FD08fERf/31V/HIkSPi0KFDxaioKLGoqEg6x4ABA8T27duLe/bsEXfs2CE2b95cHDFihIzv6vbVdj3y8vLE1157TUxKShJTUlLEjRs3ih06dBCbN28uFhcXS+eoT9djwoQJolqtFrdu3Sqmp6dLS2FhoVTmVr8n5eXlYps2bcR+/fqJycnJ4vr168WAgABx8uTJMr2rO3Ora3L27Fnx3XffFffv3y+mpKSIv/76qxgdHS327NlTOkd9uiZvvfWWuG3bNjElJUU8cuSI+NZbb4mCIIh//fWXKDbAz4d4i2tiy88HQ5OVfP7552J4eLjo6uoqdunSRdy9e7fcVbKJ4cOHiyEhIaKrq6vYuHFjcfjw4eLZs2el/UVFReJzzz0n+vr6ih4eHuJDDz0kpqeny1pnS9uyZYsIwGhJSEgQxYphB6ZMmSIGBQWJSqVS7NOnj3j69GmDc1y/fl0cMWKE6OXlJapUKnHMmDFiXl6eTO/oztR2PQoLC8V+/fqJAQEBokKhECMiIsRx48YZ/Q9Gfboepq4FAHHRokVSGXN+T1JTU8WBAweK7u7uor+/vzhp0iSxrKxMhnd05251TdLS0sSePXuKfn5+olKpFJs1aya+/vrrYk5OjsF56ss1eeqpp8SIiAjR1dVVDAgIEPv06SMFJrEBfj7EW1wTW34+BFH/gSUiIiKiWrBPExEREZEZGJqIiIiIzMDQRERERGQGhiYiIiIiMzA0EREREZmBoYmIiIjIDAxNRERERGZgaCIiIiIyA0MTEZGVTJkyxWD29V69euHll1+2eT22bt0KQRCkSV4TExPh4+Mj7Z8+fTratWsnrY8ePRrDhg2zWf3Wr1+Pdu3aQafT2ew1iW4HQxORDY0ePRqCIEAQBGnG7nfffRfl5eVyV+22CYKA1atXW/U1zp49izFjxqBJkyZQKpWIiorCiBEjsH//fqu+bnWpqakQBEGaDLQ2Go0G8+bNw3/+8x9p28qVK/Hee+9ZuZbGunfvjvT0dKjVarPKz5s3D4mJiVavV6UBAwZAoVDg+++/t9lrEt0OhiYiGxswYADS09Nx5swZTJo0CdOnT8fHH398W+fSarX15v/Oy8rKTG7fv38/OnbsiH/++QdfffUVTpw4gVWrVuGuu+7CpEmTbF5Pcy1cuBDdu3dHRESEtM3Pzw/e3t42r4urqyuCg4MhCIJZ5dVqtUFLlC2MHj0an332mU1fk6iuGJqIbEypVCI4OBgRERGYMGEC4uPj8dtvvwEA5syZg3vuuQeenp4ICwvDc889h/z8fOnYytsqv/32G1q3bg2lUom0tDTs27cPffv2hb+/P9RqNeLi4nDw4EGD1xUEAV999RUGDx4MDw8PtGrVCklJSTh79ix69eoFT09PdO/eHefOnTM47tdff0WHDh3g5uaG6OhozJgxQ2oZi4yMBAA89NBDEARBWr/VcZX1+fLLL/Hggw/C09MT77//vtG1EkURo0ePRvPmzbF9+3Y88MADaNq0Kdq1a4dp06bh119/lcoePXoUvXv3hru7Oxo1aoTx48cbXDtTt8aGDRuG0aNHS+uRkZH44IMP8NRTT8Hb2xvh4eH4+uuvpf1RUVEAgPbt20MQBPTq1avGn/OPP/6IIUOGGGyrXodbvZ4pvXr1wgsvvICXX34Zvr6+CAoKwjfffIOCggKMGTMG3t7eaNasGdatWycdU/323K1Uvz1XUlKCF198EYGBgXBzc8O9996Lffv2GZ1/06ZN6NSpEzw8PNC9e3ecPn1aKnP48GHcf//98Pb2hkqlQseOHQ1aCocMGYL9+/cbff6I7AlDE5HM3N3dUVpaCgBwcnLCZ599huPHj2Px4sXYvHkz3njjDYPyhYWFmDVrFhYuXIjjx48jMDAQeXl5SEhIwI4dO7B79240b94cgwYNQl5ensGx7733HkaNGoXk5GTcddddeOKJJ/DMM89g8uTJ2L9/P0RRxMSJE6Xy27dvx6hRo/DSSy/hxIkT+Oqrr5CYmCgFnMovzkWLFiE9PV1av9VxlaZPn46HHnoIR48exVNPPWV0bZKTk3H8+HFMmjQJTk7Gf64qW0MKCgrQv39/+Pr6Yt++fVi+fDk2btxo8F7MNXv2bHTq1AmHDh3Cc889hwkTJkhf/nv37gUAbNy4Eenp6Vi5cqXJc2RlZeHEiRPo1KnTHb1eTRYvXgx/f3/s3bsXL7zwAiZMmIDHHnsM3bt3x8GDB9GvXz88+eSTKCwsrPP7N+WNN97AL7/8gsWLF+PgwYNo1qwZ+vfvj6ysLINy//nPfzB79mzs378fLi4uBj/TkSNHokmTJti3bx8OHDiAt956CwqFQtofHh6OoKAgbN++3SJ1JrIKkYhsJiEhQRw6dKgoiqKo0+nEDRs2iEqlUnzttddMll++fLnYqFEjaX3RokUiADE5ObnW19FqtaK3t7f4+++/S9sAiO+88460npSUJAIQv/32W2nbsmXLRDc3N2m9T58+4gcffGBw7qVLl4ohISEG5121apVBGXOPe/nll2t9Hz/99JMIQDx48GCt5b7++mvR19dXzM/Pl7atXbtWdHJyEjUajSiKohgXFye+9NJLBscNHTpUTEhIkNYjIiLEf//739K6TqcTAwMDxS+//FIURVFMSUkRAYiHDh2qtT6HDh0SAYhpaWkG26vX4VavZ0pcXJx47733Suvl5eWip6en+OSTT0rb0tPTRQBiUlKSKIqiuGXLFhGAeOPGDVGs+Byp1Wqp/LRp08SYmBhpvernND8/X1QoFOL3338v7S8tLRVDQ0PFjz76yOD8GzdulMqsXbtWBCAWFRWJoiiK3t7eYmJiYq3XrX379uL06dNrLUMkJxe5QxtRQ7NmzRp4eXmhrKwMOp0OTzzxBKZPnw5UtGDMnDkTp06dQm5uLsrLy1FcXIzCwkJ4eHgAFf1T2rZta3DOjIwMvPPOO9i6dSsyMzOh1WpRWFiItLQ0g3JVjwsKCgIA3HPPPQbbiouLkZubC5VKhcOHD2Pnzp0GLURardaoTtWZe9ytWmL02erWTp48iZiYGHh6ekrbevToAZ1Oh9OnT0vv1RxVr5EgCAgODkZmZqbZxwNAUVERAMDNzc0qr1f1GGdnZzRq1Mjo5wigzvU25dy5cygrK0OPHj2kbQqFAl26dMHJkydrrFdISIhUh/DwcLz66qsYO3Ysli5divj4eDz22GNo2rSpwfHu7u4Wax0jsgbeniOysfvvvx/Jyck4c+YMioqKsHjxYnh6eiI1NRWDBw9G27Zt8csvv+DAgQOYP38+AEi371DxxVK9Q29CQgKSk5Mxb9487Nq1C8nJyWjUqJHBcaj4sqtUeQ5T2yo7l+fn52PGjBlITk6WlqNHj+LMmTO1BgJzj6sackxp0aIFAODUqVO1ljOHk5OTUQgz1fm86vVAxTWpa2d7f39/AMCNGzduWfZ2Xs/UMbX9HG2ltjpMnz4dx48fxwMPPIDNmzejdevWWLVqlcHxWVlZCAgIsGmdieqCoYnIxjw9PdGsWTOEh4fDxeVmY++BAweg0+kwe/ZsdOvWDS1atMCVK1fMOufOnTvx4osvYtCgQbj77ruhVCpx7dq1O65rhw4dcPr0aTRr1sxoqexjpFAooNVq63ycOdq1a4fWrVtj9uzZJgNAZcfmVq1a4fDhwygoKJD27dy5E05OTmjZsiUAICAgAOnp6dJ+rVaLY8eO1el6uLq6SsfWpmnTplCpVDhx4kSdzm+PmjZtCldXV+zcuVPaVlZWhn379qF169Z1OleLFi3wyiuv4K+//sLDDz+MRYsWSfuKi4tx7tw5tG/f3qL1J7IkhiYiO9GsWTOUlZXh888/x/nz57F06VIsWLDArGObN2+OpUuX4uTJk9izZw9GjhwJd3f3O67T1KlTsWTJEsyYMQPHjx/HyZMn8eOPP+Kdd96RykRGRmLTpk3QaDRSy4o5x5lDEAQsWrQI//zzD+677z788ccfOH/+PI4cOYL3338fQ4cOBSo6Gbu5uSEhIQHHjh3Dli1b8MILL+DJJ5+UblX17t0ba9euxdq1a3Hq1ClMmDDB7KfJKgUGBsLd3R3r169HRkYGcnJyTJZzcnJCfHw8duzYUafz2yNPT09MmDABr7/+OtavX48TJ05g3LhxKCwsxNNPP23WOYqKijBx4kRs3boVFy5cwM6dO7Fv3z60atVKKrN7924olUrExsZa8d0Q3RmGJiI7ERMTgzlz5mDWrFlo06YNvv/+e8ycOdOsY7/99lvcuHEDHTp0wJNPPik9Hn6n+vfvjzVr1uCvv/5C586d0a1bN8ydO9dg7KHZs2djw4YNCAsLk1oJzDnOXF26dMH+/fvRrFkzjBs3Dq1atcKDDz6I48eP49NPPwUAeHh44M8//0RWVhY6d+6MRx99FH369MF///tf6TxPPfUUEhISMGrUKMTFxSE6Ohr3339/neri4uKCzz77DF999RVCQ0Ol0GbK2LFj8eOPP9aLcbQ+/PBDPPLII3jyySfRoUMHnD17Fn/++Sd8fX3NOt7Z2RnXr1/HqFGj0KJFC/zrX//CwIEDMWPGDKnMsmXLMHLkyBr7yRHZA0E0t6clERGZTRRFdO3aFa+88gpGjBghd3Xs2rVr19CyZUvs379fGguLyB6xpYmIyAoEQcDXX3/t0FPk2Epqaiq++OILBiaye2xpIiIiIjIDW5qIiIiIzMDQRERERGQGhiYiIiIiMzA0EREREZmBoYmIiIjIDAxNRERERGZgaCIiIiIyA0MTERERkRkYmoiIiIjM8P9TLQq1wVP6kwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Cringe hardcoding\n", + "plt.plot([6, 11, 19, 35, 67], np.array(df[\"val_ppl\"])[1::2] - np.array(df[\"val_ppl\"])[::2], linewidth=2, label=\"Scratch\")\n", + "plt.plot([67, 110, 335], transferred_ppl[1::2] - transferred_ppl[::2], linewidth=2, label=\"Transferred\")\n", + "plt.axhline(y=0, color='gray', linestyle='--')\n", + "plt.xlabel(\"Parameter Count (in millions)\")\n", + "plt.ylabel(\"RTL PPL - LTR PPL\")\n", + "plt.legend()" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "execution_state": "idle", + "id": "66ef4460-3a03-4017-845a-9ff04733985e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 60, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAl8AAAHHCAYAAACBYj2uAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAACNtklEQVR4nOzdeVxU1fvA8c8MuwvgwuqKSypuKCrinqK45Ua59nNfcs20LKtvaptlmVpappZaaZlabimFuCtuuKSipIZLyqIiICLr3N8fAyMjoKDMDAPP+/Wal86559557uUCD+ece45KURQFIYQQQghhFGpTByCEEEIIUZJI8iWEEEIIYUSSfAkhhBBCGJEkX0IIIYQQRiTJlxBCCCGEEUnyJYQQQghhRJJ8CSGEEEIYkSRfQgghhBBGJMmXEEIIIYQRSfIlxDPo0KEDHTp0MHUYAlCpVMyePVuv7NixY7Rq1YrSpUujUqk4deoUAIGBgXh5eWFra4tKpSIuLs5EUYuSYvbs2ahUqqfad/jw4VSvXr3QYtmzZw8qlYo9e/YU2jFFwUjyJQrFqlWrUKlUupelpSWVKlVi+PDh3LhxAzJ/gGSvk9dr+PDhkJnYNGjQwMRnVjiqV69Oz5499cqyfhg/6ZWV3D16/WxsbHjuued47733SE5Ozlcc5nJNq1evrjtPtVqNo6MjDRs2ZOzYsRw5ciRfx0hLS+Oll14iNjaWBQsW8OOPP1KtWjXu3LlD//79sbOzY8mSJfz444+ULl3a4OdkDmbMmIFKpWLAgAGmDsVgsu4tPz+/XLcvX75cd+8dP37c6PE9C41Gww8//ICPjw/ly5enbNmyPPfccwwdOpTDhw+bOjyRjaWpAxDFy/vvv4+HhwfJyckcPnyYVatWceDAAc6ePcu4ceP0fuBFRETw3nvvMXbsWNq2basrr1mzpomiN65+/fpRq1Yt3fvExETGjx9P37596devn67cxcVF938bGxtWrFgBQHx8PJs3b+aDDz7g8uXLrFmzxshnYFheXl5Mnz4dgHv37nH+/HnWr1/P8uXLee211/jiiy/06j948ABLy4c/0i5fvszVq1dZvnw5o0eP1pUHBgZy7949Pvjggzx/AZdEiqLw888/U716dbZu3cq9e/coW7asqcMyCFtbW3bv3k1UVBSurq5629asWYOtrW2+/6ApSqZMmcKSJUvo3bs3Q4YMwdLSkvDwcHbs2EGNGjVo2bIlAO3atePBgwdYW1ubOuSSSxGiEKxcuVIBlGPHjumVv/nmmwqgrFu3Lsc+x44dUwBl5cqVuR6zffv2Sv369Q0Wc2Fo37690r59+yfWq1atmtKjR4/H1rl165YCKLNmzcp1+7Bhw5TSpUvrlWk0GqVly5aKSqVSoqKi8hVvUb+mymOuV1JSktKnTx8FUL7++uvHHmPv3r0KoKxfv16vfPXq1bneq88iMTGx0I5lKrt27VIAZdeuXYqVlZWyatWqQjt2Ubo+1apVUzp16qTY29srCxcu1Nt2/fp1Ra1WKwEBAYV+j8yaNUt52l+5w4YNU6pVq/bYOlFRUYpKpVLGjBmTY5tGo1Gio6Of6rOFYUi3ozCorBaty5cvG+0zY2Njef3112nYsCFlypTB3t6ebt26cfr0ab16WeMefv31Vz766CMqV66Mra0tnTp14tKlSzmOu2zZMmrWrImdnR0tWrRg//79RjunvKhUKtq0aYOiKPz777+Fdtyvv/6a+vXrY2Njg7u7OxMnTswxLurixYsEBATg6uqKra0tlStXZuDAgcTHx+vqBAUF0aZNGxwdHSlTpgx16tTh7bfffuq47Ozs+PHHHylfvjwfffQRiqLotmUf8zV8+HDat28PwEsvvaTrvu3QoQPDhg0DoHnz5nrd3ABHjhyha9euODg4UKpUKdq3b8/Bgwf1YsjqLg4LC2Pw4MGUK1eONm3a6Lb/9NNPeHt7Y2dnR/ny5Rk4cCDXr1/XO0ZW929YWBjPP/88pUqVolKlSsybNy/HOScnJzN79myee+45bG1tcXNzo1+/fnrfUxqNhoULF1K/fn1sbW1xcXFh3Lhx3L17N9/Xds2aNXh6evL888/j5+eXZ0vqjRs3GDVqFO7u7tjY2ODh4cH48eNJTU2FbEMQ9u7dy4QJE3B2dqZy5cq6/YvCvWVra0u/fv1Yu3atXvnPP/9MuXLl8Pf3z3W/Xbt20bZtW0qXLo2joyO9e/fm/PnzOeodOHCA5s2bY2trS82aNfn222/zjCU/90t+REREoCgKrVu3zrFNpVLh7Oyse//omK9Hh43kNuyhsOMt6aTbURjUlStXAChXrpzRPvPff/9l06ZNvPTSS3h4eBAdHc23335L+/btCQsLw93dXa/+J598glqt5vXXXyc+Pp558+YxZMgQvbFF3333HePGjaNVq1ZMnTqVf//9l169elG+fHmqVKlitHPLTWFf49mzZzNnzhz8/PwYP3484eHhfPPNNxw7doyDBw9iZWVFamoq/v7+pKSkMHnyZFxdXblx4wbbtm0jLi4OBwcHzp07R8+ePWnUqBHvv/8+NjY2XLp0KUcyU1BlypShb9++fPfdd4SFhVG/fv0cdcaNG0elSpX4+OOPmTJlCs2bN9d139apU4dly5bpusizurl37dpFt27d8Pb2ZtasWajValauXEnHjh3Zv38/LVq00PuMl156idq1a/Pxxx/rksCPPvqI//3vf/Tv35/Ro0dz69YtvvrqK9q1a8fJkydxdHTU7X/37l26du1Kv3796N+/Pxs2bODNN9+kYcOGdOvWDYCMjAx69uxJcHAwAwcO5NVXX+XevXsEBQVx9uxZXezjxo1j1apVjBgxgilTphAREcHixYs5efKk7mv2OCkpKWzcuFHXzTto0CBGjBiRo1vu5s2btGjRgri4OMaOHUvdunW5ceMGGzZsICkpSa8ba8KECTg5OfHee+9x//59KGL31uDBg+nSpQuXL1/WXce1a9fy4osv5nq9du7cSbdu3ahRowazZ8/mwYMHfPXVV7Ru3ZoTJ07oBsSfOXOGLl264OTkxOzZs0lPT2fWrFl6wweyFOR+eZJq1aoBsH79el566SVKlSqV733btWvHjz/+qFd29epV3n33Xb2krTDjLfFM3fQmioesbsedO3cqt27dUq5fv65s2LBBcXJyUmxsbJTr16/n2MdQ3Y7JyclKRkaGXllERIRiY2OjvP/++7qy3bt3K4BSr149JSUlRVe+aNEiBVDOnDmjKIqipKamKs7OzoqXl5devWXLlimA0bsdb926pdy6dUu5dOmS8vnnnysqlUpp0KCBotFonhjHk65pTEyMYm1trXTp0kXvGi5evFgBlO+//15RFEU5efJkrl162S1YsEABlFu3bj0xrkc96XplHXvz5s26skevXdbX99EYc+si12g0Su3atRV/f3+965iUlKR4eHgonTt31pVldR8NGjRI77hXrlxRLCwslI8++kiv/MyZM4qlpaVeefv27RVA+eGHH3RlKSkpiqurqxIQEKAr+/777xVA+eKLL3Jcg6w49+/frwDKmjVr9LYHBgbmWp6bDRs2KIBy8eJFRVEUJSEhQbG1tVUWLFigV2/o0KGKWq3OtTsuK56s69umTRslPT1dt72o3Vvp6emKq6ur8sEHHyiKoihhYWEKoOzduzfXe8TLy0txdnZW7ty5oys7ffq0olarlaFDh+rK+vTpo9ja2ipXr17VlYWFhSkWFhZ63Y4FuV/y0+2oZH59AKVcuXJK3759lc8//1w5f/58jnpZ3xu7d+/O9TgPHjxQvL29FXd3dyUyMrLA8Yonk25HUaj8/PxwcnKiSpUqvPjii5QuXZotW7bodTsYmo2NDWq19tbOyMjgzp07um6JEydO5Kg/YsQIvb/Ys7pKs7rxjh8/TkxMDK+88opeveHDh+Pg4GCEM3ro/v37ODk54eTkRK1atXj99ddp3bo1mzdvfurH2LPbuXMnqampTJ06VXcNAcaMGYO9vT1//PEHgO68//zzT5KSknI9VtZfwZs3b0aj0TxzbNmVKVMGMgfiF4ZTp05x8eJFBg8ezJ07d7h9+za3b9/m/v37dOrUiX379uU4h1deeUXv/W+//YZGo6F///66/W/fvo2rqyu1a9dm9+7dOc7h5Zdf1r23tramRYsWet3HGzdupGLFikyePDlHzFlf7/Xr1+Pg4EDnzp31Ptfb25syZcrk+NzcrFmzhmbNmukeAClbtiw9evTQ63rUaDRs2rSJF154gWbNmuUZT5YxY8ZgYWGhe1/U7i0LCwv69+/Pzz//rLsGVapU0Xv4J0tkZCSnTp1i+PDhlC9fXlfeqFEjOnfuzPbt2yHz582ff/5Jnz59qFq1qq5evXr1cnRlFvR+yY+VK1eyePFiPDw8+P3333n99depV68enTp10j11nh8TJkzgzJkzbNy4UdfyaYh4SzJJvkShWrJkCUFBQWzYsIHu3btz+/ZtbGxsjBqDRqNhwYIF1K5dGxsbGypWrIiTkxN///233piRLNl/SJKt+y5rvMzVq1cBqF27tl49KysratSoYcAzycnW1pagoCCCgoJYuXIl9erVIyYmBjs7u0I5fta51qlTR6/c2tqaGjVq6LZ7eHgwbdo0VqxYQcWKFfH392fJkiV613fAgAG0bt2a0aNH4+LiwsCBA/n1118LJRFLTEyEzCShMFy8eBGAYcOG6ZLbrNeKFStISUnJce94eHjkOIaiKNSuXTvHMc6fP09MTIxe/cqVK+dIWMqVK6c3Tuvy5cvUqVNH7ynO3GKPj4/H2dk5x+cmJibm+NxHxcXFsX37dtq3b8+lS5d0r9atW3P8+HH++ecfAG7dukVCQkK+pyp59PoUxXtr8ODBhIWFcfr0adauXcvAgQNz/SMmr9jJTKyyEvVbt27x4MGDHD8rctu3oPdLfqjVaiZOnEhoaCi3b99m8+bNdOvWjV27djFw4MB8HePbb79l5cqVfPXVV7qnIw0Vb0kmY75EoWrRooXur+I+ffrQpk0bBg8eTHh4uK61wtA+/vhj/ve//zFy5Eg++OADypcvj1qtZurUqbn+cM7+13l22QdzFxUWFhZ60yP4+/tTt25dxo0bx5YtW4way/z58xk+fDibN2/mr7/+YsqUKcydO5fDhw9TuXJl7Ozs2LdvH7t37+aPP/4gMDCQdevW0bFjR/766688r3t+nD17FkBvqo5nkXVffPbZZ3h5eeVa59H799GEV6PRoFKp2LFjR67n9uj+hXXfaTQanJ2d8xwg7+Tk9Nj9169fT0pKCvPnz2f+/Pk5tq9Zs4Y5c+YUKCZyuT4FYax7y8fHh5o1azJ16lQiIiIYPHjwU8dcUAW9XwqqQoUK9OrVi169etGhQwf27t3L1atXdWPDcnP06FFeffVVRo8ezdixY40ab0kjyZcwGAsLC+bOncvzzz/P4sWLeeutt4zyuRs2bOD555/nu+++0yuPi4ujYsWKBT5e1g+rixcv0rFjR115WloaERERNG7cuBCifjpubm689tprzJkzh8OHD+v9pfo0ss41PDxcr1UvNTWViIiIHPNiNWzYkIYNG/Luu+9y6NAhWrduzdKlS/nwww8h8y/xTp060alTJ7744gs+/vhj3nnnHXbv3v3Uc2wlJiby+++/U6VKFerVq/dM55sla8C1vb39U8dVs2ZNFEXBw8OD5557rtDiOnLkCGlpaXkOmq9ZsyY7d+6kdevWT5XwrFmzhgYNGjBr1qwc27799lvWrl3LnDlzcHJywt7eXpf4FlRRvbcGDRrEhx9+SL169fJMvLPH/qgLFy5QsWJFSpcuja2tLXZ2drqW1Owe3dcQ90temjVrxt69e4mMjMwz+bp16xYvvvgiXl5eLFmyJMd2Y8ZbEki3ozCoDh060KJFCxYuXGi0SQstLCxytB6sX7++QGMesmvWrBlOTk4sXbpU9zg9mY9nF4VlaSZPnkypUqX45JNPnvlYfn5+WFtb8+WXX+pdw++++474+Hh69OgBQEJCAunp6Xr7NmzYELVaTUpKCmRO+fGorF9uWXUK6sGDB/zf//0fsbGxvPPOO4Uyzg3A29ubmjVr8vnnn+u6NLO7devWE4/Rr18/LCwsmDNnTo77T1EU7ty5U+C4AgICuH37NosXL86xLesz+vfvT0ZGBh988EGOOunp6Y+9R69fv86+ffvo378/L774Yo7XiBEjuHTpEkeOHEGtVtOnTx+2bt2a68zvT2qxK6r31ujRo5k1a1aurX5Z3Nzc8PLyYvXq1XrX8+zZs/z11190794dMn/2+Pv7s2nTJq5du6ard/78ef7880+9Yxb2/RIVFUVYWFiO8tTUVIKDg1Gr1Xm2FGdkZDBw4EBSU1PZuHFjrpOvGuL+Lsmk5UsY3BtvvMFLL73EqlWrcgxSfpJbt27p/tLNzsPDgyFDhuS6T8+ePXn//fcZMWIErVq14syZM6xZs+apx2dZWVnx4YcfMm7cODp27MiAAQOIiIhg5cqVBTrmpUuXcj2XJk2a6H7xPI0KFSowYsQIvv76a86fP//E1qAnXdOZM2cyZ84cunbtSq9evQgPD+frr7+mefPmugHiu3btYtKkSbz00ks899xzpKen8+OPP2JhYUFAQABkrnawb98+evToQbVq1YiJieHrr7+mcuXKevNi5eXGjRv89NNPkNnaFRYWxvr164mKimL69OmMGzfuKa9YTmq1mhUrVtCtWzfq16/PiBEjqFSpEjdu3GD37t3Y29uzdevWxx6jZs2afPjhh8ycOZMrV67Qp08fypYtS0REBL///jtjx47l9ddfL1BcQ4cO5YcffmDatGkcPXqUtm3bcv/+fXbu3MmECRPo3bs37du3Z9y4ccydO5dTp07RpUsXrKysuHjxIuvXr2fRokW8+OKLuR5/7dq1KIpCr169ct3evXt3LC0tWbNmDT4+Pnz88cf89ddftG/fnrFjx1KvXj0iIyNZv349Bw4ceOxUA05OTkXm3squWrVqOdYEzc1nn31Gt27d8PX1ZdSoUbqpJhwcHPT2nzNnDoGBgbRt25YJEyaQnp7OV199Rf369fn777919Qr7fvnvv/9o0aIFHTt2pFOnTri6uhITE8PPP//M6dOnmTp1ap4t/0uXLmXXrl288sorOQbOu7i40LlzZ4Pc3yWaqR+3FMVDXjPcK4qiZGRkKDVr1lRq1qyp9+h5fqaaAHJ9derUKc9YkpOTlenTpytubm6KnZ2d0rp1ayUkJCTHbPR5TUUQERGRa1xff/214uHhodjY2CjNmjVT9u3bV6AZ7vM6l1GjRinKU85wn+Xy5cuKhYWFMmzYsMfGkd9runjxYqVu3bqKlZWV4uLioowfP165e/eubvu///6rjBw5UqlZs6Zia2urlC9fXnn++eeVnTt36uoEBwcrvXv3Vtzd3RVra2vF3d1dGTRokPLPP/8U6HqpVCrF3t5eqV+/vjJmzBjlyJEjue7zLFNNZDl58qTSr18/pUKFCoqNjY1SrVo1pX///kpwcLCuTtZUE3lNc7Bx40alTZs2SunSpZXSpUsrdevWVSZOnKiEh4frfR1ym/IjtykFkpKSlHfeeUfx8PBQrKysFFdXV+XFF19ULl++rFdv2bJlire3t2JnZ6eULVtWadiwoTJjxgzl5s2bucapKIrSsGFDpWrVqnluVxRF6dChg+Ls7KykpaUpiqIoV69eVYYOHaqbRqZGjRrKxIkTddOwPO76KkXk3nrStC95ncPOnTuV1q1bK3Z2doq9vb3ywgsvKGFhYTn237t3r+Lt7a1YW1srNWrUUJYuXZrnDPf5uV/yM9VEQkKCsmjRIsXf31+pXLmyYmVlpZQtW1bx9fVVli9frjeFyqNTTWTFltvr0Z9v+YlXPJlKKYqjioUQQgghiikZ8yWEEEIIYUSSfAkhhBBCGJEkX0IIIYQQRiTJlxBCCCGEEUnyJYQQQghhRJJ8CSGEEEIYkUyyagQajYabN29StmzZQpuRWwghhBCGpSgK9+7dw93dHbW68NqrJPkygps3b1KlShVThyGEEEKIp3D9+nUqV65caMeT5MsIypYtC5lfPHt7e1OHI4QQQoh8SEhIoEqVKrrf44VFki8jyOpqtLe3l+RLCCGEMDOFPWRIBtwLIYQQQhiRJF9CCCGEEEYkyZcQQgghhBHJmC8hhBAiU0ZGBmlpaaYOQxiRtbV1oU4jkR+SfAkhhCjxFEUhKiqKuLg4U4cijEytVuPh4YG1tbXRPlOSLyGEECVeVuLl7OxMqVKlZELsEiJrEvTIyEiqVq1qtK+7JF9CCCFKtIyMDF3iVaFCBVOHI4zMycmJmzdvkp6ejpWVlVE+UwbcCyGEKNGyxniVKlXK1KEIE8jqbszIyDDaZ0ryJYQQQhhgIk1hHkzxdZduRzOWoVE4GhFLzL1knMva0sKjPBZq+eEhhBBCFGWSfJmpwLORzNkaRmR8sq7MzcGWWS940rWBm0ljE0IIIR5VvXp1pk6dytSpU00dislJt6MZCjwbyfifTuglXgBR8cmM/+kEgWcjTRabEEKUVBkahZDLd9h86gYhl++QoVEM/pm3bt1i/PjxVK1aFRsbG1xdXfH39+fgwYMG/VyVSsWmTZsM+hnFmbR8mZkMjcKcrWHk9i2tACpgztYwOnu6ShekEEIYial6IwICAkhNTWX16tXUqFGD6OhogoODuXPnToGPlZGRgUqlMvqEoyWRXGEzczQiNkeLV3YKEBmfzNGIWKPGJYQQJZWpeiPi4uLYv38/n376Kc8//zzVqlWjRYsWzJw5k169eunqjBs3DhcXF2xtbWnQoAHbtm0DYNWqVTg6OrJlyxY8PT2xsbHh2rVrHDt2jM6dO1OxYkUcHBxo3749J06c0H1u9erVAejbty8qlUr3HmDr1q00b94cW1tbKlasSN++ffViTkpKYuTIkZQtW5aqVauybNkyg1ybok6SLzMTcy/vxOtp6gkhhHh6T+qNILM3whBdkGXKlKFMmTJs2rSJlJSUHNs1Gg3dunXj4MGD/PTTT4SFhfHJJ59gYWGhq5OUlMSnn37KihUrOHfuHM7Ozty7d49hw4Zx4MABDh8+TO3atenevTv37t0D4NixYwCsXLmSyMhI3fs//viDvn370r17d06ePElwcDAtWrTQi2n+/Pk0a9aMkydPMmHCBMaPH094eHihX5uiTrodzYxzWdtCrSeEECKnF746wK17OROaR6WkZ3A3Ke+1ILN6I5p9GISNpUWe9bI4lbVh6+Q2+YrR0tKSVatWMWbMGJYuXUrTpk1p3749AwcOpFGjRuzcuZOjR49y/vx5nnvuOQBq1Kihd4y0tDS+/vprGjdurCvr2LGjXp1ly5bh6OjI3r176dmzJ05OTgA4Ojri6uqqq/fRRx8xcOBA5syZoyvLflyA7t27M2HCBADefPNNFixYwO7du6lTp06+zrm4kOTLzLTwKI+bgy1R8cm5/qWlAlwdtNNOCCGEeDq37qUQlVB4PQjaBK3wF+wOCAigR48e7N+/n8OHD7Njxw7mzZvHihUriImJoXLlyrrEKzfW1tY0atRIryw6Opp3332XPXv2EBMTQ0ZGBklJSVy7du2xsZw6dYoxY8Y8tk72z1KpVLi6uhITE5Pv8y0uJPkyMxZqFbNe8GT8TydQZWvWzqIAs17wlMH2QgjxDJzK2uSr3pNavrKUK2WV75avgrK1taVz58507tyZ//3vf4wePZpZs2bx+uuvP3FfOzu7HJOMDhs2jDt37rBo0SKqVauGjY0Nvr6+pKamPvFYT/Lo8j0qlQqNRvPE/YobSb7MUNcGbnzzctMcT9YAWFuqaVqtnMliE0KI4iC/XX8ZGoU2n+56Ym/EgTc7Gu2PYk9PTzZt2kSjRo3477//+Oeffx7b+vWogwcP8vXXX9O9e3cArl+/zu3bt/XqWFlZ5ViOp1GjRgQHBzNixIhCOpPiSwbcm6muDdw48GZHfh7TkkUDvehczwWA1HQNi3ZeNHV4QghRImT1RpCZaGWX9d5QvRF37tyhY8eO/PTTT/z9999ERESwfv165s2bR+/evWnfvj3t2rUjICCAoKAgIiIi2LFjB4GBgY89bu3atfnxxx85f/48R44cYciQITlatapXr05wcDBRUVHcvXtXe56zZvHzzz8za9Yszp8/z5kzZ/j0008L/byLA0m+zJiFWoVvzQr09qrE3ICGlLbWNmn/cuw6l2ISTR2eEEKUCFm9Ea4O+g86uTrY8s3LTQ02z1eZMmXw8fFhwYIFtGvXjgYNGvC///2PMWPGsHjxYgA2btxI8+bNGTRoEJ6ensyYMeOJC0h/99133L17l6ZNm/J///d/TJkyBWdnZ7068+fPJygoiCpVqtCkSRMAOnTowPr169myZQteXl507NiRo0ePGuTczZ1KURTDT8FbwiUkJODg4EB8fDz29vYG+5yvgi8yP+gfADrXc2Zkmxqy7qMQQjxBcnIyEREReHh4YGv79E+Ky3q75ulxX39D/f6WMV/FyKi2Hvx4+Cox91IIOh9D0PmHT5DIuo9CCGFYWb0RQjyJdDsWI6WsLeni6ZLrNln3UQghhCgaJPkqRjI0CjvP5z5fiqFnWhZCCCFE/kjyVYwcjYh97KSAsu6jEEIIYXpml3wtWbKE6tWrY2tri4+PzxOfpFi/fj1169bF1taWhg0bsn37dr3tv/32G126dKFChQqoVCpOnTqV4xgdOnRApVLpvV555ZVCP7dnJes+CiGEEEWfWSVf69atY9q0acyaNYsTJ07QuHFj/P3981ya4NChQwwaNIhRo0Zx8uRJ+vTpQ58+fTh79qyuzv3792nTps0T5yIZM2YMkZGRute8efMK/fyelaz7KIQQQhR9ZpV8ffHFF4wZM4YRI0bg6enJ0qVLKVWqFN9//32u9RctWkTXrl154403qFevHh988AFNmzbVzX8C8H//93+89957+Pn5PfazS5Uqhaurq+5lyCkjnlbWuo95PdisynzqUdZ9FEIIIUzHbJKv1NRUQkND9ZIktVqNn58fISEhue4TEhKSI6ny9/fPs/7jrFmzhooVK9KgQQNmzpxJUlJSnnVTUlJISEjQexmDKWdaFkIIIUT+mE3ydfv2bTIyMnBx0Z9KwcXFhaioqFz3iYqKKlD9vAwePJiffvqJ3bt3M3PmTH788UdefvnlPOvPnTsXBwcH3atKlSoF+rxnkddMyy72hp1pWQghhBD5YzbJlymNHTsWf39/GjZsyJAhQ/jhhx/4/fffuXz5cq71Z86cSXx8vO51/fp1o8abte5jh+cq6sq+GtxEEi8hhBAGFRUVRefOnSldujSOjo6mDkdHpVKxadMmU4ehYzbJV8WKFbGwsCA6OlqvPDo6GldX11z3cXV1LVD9/PLx8QHg0qVLuW63sbHB3t5e72VsFmoVHes9bPU7H2mcrk8hhCixNBkQsR/ObND+q3n8GorP4tEn8B99zZ4922Cf/TgLFiwgMjKSU6dO8c8//5gkBnNgNsmXtbU13t7eBAcH68o0Gg3BwcH4+vrmuo+vr69efYCgoKA86+dX1nQUbm5FuyWpQSUH3f/P3og3aSxCCFGshW2BhQ1gdU/YOEr778IG2nIDyP70/cKFC7G3t9cre/3113V1FUUhPT3dIHE86vLly3h7e1O7du0ci3HnV2pqaq7laWlpzxhd0WE2yRfAtGnTWL58OatXr+b8+fOMHz+e+/fvM2LECACGDh3KzJkzdfVfffVVAgMDmT9/PhcuXGD27NkcP36cSZMm6erExsZy6tQpwsLCAAgPD+fUqVO6cWGXL1/mgw8+IDQ0lCtXrrBlyxaGDh1Ku3btaNSokdGvQUHUc7Una2z9mRvS8iWEEAYRtgV+HQoJN/XLEyK15QZIwLI/fe/g4IBKpdK9v3DhAmXLlmXHjh14e3tjY2PDgQMHuHz5Mr1798bFxYUyZcrQvHlzdu7cqXfc6tWr8/HHHzNy5EjKli1L1apVWbZsmW57amoqkyZNws3NDVtbW6pVq8bcuXN1+27cuJEffvgBlUrF8OHDAYiLi2P06NE4OTlhb29Px44dOX36tO6Ys2fPxsvLixUrVugtbq1Sqfjmm2/o1asXpUuX5qOPPgJg8+bNNG3aFFtbW2rUqMGcOXP0ksuLFy/Srl07bG1t8fT0JCgoqNCv/7Myq+RrwIABfP7557z33nt4eXlx6tQpAgMDdYPqr127RmTkw7ULW7Vqxdq1a1m2bBmNGzdmw4YNbNq0iQYNGujqbNmyhSZNmtCjRw8ABg4cSJMmTVi6dClktrjt3LmTLl26ULduXaZPn05AQABbt241+vkXlJ21BbWdywJwMfoeyWmGawIXQogSSZMBgW9mW8Qtu8yywLcM2gWZl7feeotPPvmE8+fP06hRIxITE+nevTvBwcGcPHmSrl278sILL3Dt2jW9/ebPn0+zZs04efIkEyZMYPz48YSHhwPw5ZdfsmXLFn799VfCw8NZs2YN1atXB+DYsWN07dqV/v37ExkZyaJFiwB46aWXiImJYceOHYSGhtK0aVM6depEbOzD1VYuXbrExo0b+e233/QmO589ezZ9+/blzJkzjBw5kv379zN06FBeffVVwsLC+Pbbb1m1apUuMdNoNPTr1w9ra2uOHDnC0qVLefPNN41yvQtCpSiKLPRnYAkJCTg4OBAfH2/08V/Tfz3NxhP/AbB5YmsaVzHeAMgMjcLRiFhi7iXjXFY7v5hMcyGEKGqSk5OJiIjQa3Xh2/aQmPsE3nrSU+DBnSfXs6sAljZPrlfGGcbtzUfUD61atYqpU6cSFxcHwJ49e3j++efZtGkTvXv3fuy+DRo04JVXXtH1CFWvXp22bdvy448/QmaXpaurK3PmzOGVV15hypQpnDt3jp07d6JS5fx53qdPHxwdHVm1ahUABw4coEePHsTExGBj8/D8a9WqxYwZMxg7diyzZ8/m448/5saNGzg5OenqqFQqpk6dyoIFC3Rlfn5+dOrUSa+X66effmLGjBncvHmTv/76ix49enD16lXc3d0BCAwMpFu3bvz+++/06dMnR8y5fv0zGer3t2WhHUkUSQ0q2bPxhPb/Z27EGy35CjwbyZytYUTGP1zKyM3BllkveMpTl0KIoi8xBu7dzEfFfMpPglbImjVrpvc+MTGR2bNn88cffxAZGUl6ejoPHjzI0fKVfUhNVndm1koyw4cPp3PnztSpU4euXbvSs2dPunTpkmcMp0+fJjExkQoVKuiVP3jwQG/GgGrVquklXnmdw+nTpzl48KCupQsgIyOD5ORkkpKSOH/+PFWqVNElXmSO/y5qJPkq5hpmG3R/7qZxBt0Hno1k/E8ncjTCR8UnM/6nEzLfmBCi6CuTz8Hihmj5KiSlS5fWe//6668TFBTE559/Tq1atbCzs+PFF1/MMcDdyspK771KpUKj0QDQtGlTIiIi2LFjBzt37qR///74+fmxYcOGXGNITEzEzc2NPXv25NiWfSqKR2PNqzwxMZE5c+bQr1+/HHUfbbUqyiT5KubqudmjUoGiaFu+DC1DozBna1ieox9UwJytYXT2dJUuSCFE0ZXfrj9NhvapxoTIPMZ9qcDeHaaeAbVFYUdZIAcPHmT48OH07dsXMhOZK1euFPg49vb2DBgwgAEDBvDiiy/StWtXYmNjKV8+59J1TZs2JSoqCktLS93YsGfRtGlTwsPDqVWrVq7b69Wrx/Xr14mMjNTNSHD48OFn/tzCZlYD7kXBlbaxpKZTGQDCo+6Rkm7YQZ9HI2L1uhofpQCR8ckcjYjNs44QQpgNtQV0/TTzTR4Lu3X9xOSJF0Dt2rV1A9pPnz7N4MGDdS1a+fXFF1/w888/c+HCBf755x/Wr1+Pq6trnhOq+vn54evrS58+ffjrr7+4cuUKhw4d4p133uH48eMFPof33nuPH374gTlz5nDu3DnOnz/PL7/8wrvvvqv7vOeee45hw4Zx+vRp9u/fzzvvvFPgzzE0Sb5KgAbu2kGCaRkKF6MTDfpZMffyTryepp4QQhR5nr2g/w9g/8hwCnt3bblnL1NFpueLL76gXLlytGrVihdeeAF/f3+aNm1aoGOULVuWefPm0axZM5o3b86VK1fYvn07anXu6YRKpWL79u20a9eOESNG8NxzzzFw4ECuXr2aY/m//PD392fbtm389ddfNG/enJYtW7JgwQKqVasGmWs+//777zx48IAWLVowevRovfFhRYU87WgEpnzaEWDF/n/58I/zAMzt15BBLaoa7LNCLt9h0PInN/H+PKYlvjUrPLGeEEIY2uOedisQTQZcPQSJ0VDGBaq1KhItXuLx5GlHYRANjTjTfQuP8pQvZU1sUu4zFKsAVwfttBNCCFGsqC3Ao62poxBmQLodSwBP94fZuqGTLxVQ2ubxf+nNesFTBtsLIYQosST5KgHK2lpRo6L2cd3zUfdIyyjYAMuC2HYmkut3HwBgmUuC9Wqn2jLNhBBCiBJNkq8Son5m12NqusZgg+5T0zXM/ytc937F0Gb8PKYlI1o/fLz41H9xBvlsIYQQwlxI8lVCNKxk+K7HdceucfVOEgCtalagfR0nfGtW4J3u9ahczg6APeG3uBh9zyCfL4QQz0KePyuZTPF1l+SrhGiQfdC9AWa6v5+SzqLgS7r3b3atq1v3y9JCzYjWHrpt3x+MKPTPF0KIp5U1o3tSUpKpQxEmkDXDv4WF8Z5MlacdS4j67g+TL0PMdP/9gQhuJ6YA0L2ha441JPs3q8zCoH+4l5LOxhM3eL1LHSqUycdSG0IIYWAWFhY4Ojrq1i8sVapUrotGi+JHo9Fw69YtSpUqhaWl8VIiSb5KCAc7K6pVKMXVO0mcj0wgPUODpUXhNHzG3k/l233/AmChVjG9S50cdcraWjGwRRWW748gNV3DT4ev8apf7UL5fCGEeFaurq4AugRMlBxqtZqqVasaNeGW5KsEaeDuwNU7SSSnabh86z51XMsWynGX7L5EYko6ZLZwZS1n9KjhrT34/uAVMjQKPx6+wrj2NbC1kgkIhRCmp1KpcHNzw9nZmbS0NFOHI4zI2to6zxn6DUWSrxKkQSUH/jgTCZldj4WRfP13N4kfQ64CYGOp5tVOz+VZt5KjHd0burH19E1uJ6ay+dQNBjQ33Gz7QghRUBYWFkYd+yNKJhlwX4IYYqb7hTsvkpo5b9iI1h64Ojx+aY5RbR4OvF+xP0KeLhJCCFHiSPJVgtQv5Jnu/4m+x28n/gPA3taS8e1rPnEfryqONK9eDoCLMYnsu3j7meMQQgghzIkkXyVIudLWuvm2wiITyNA8W6vTvMBwsg4x4flaOJSyytd+o9rU0P1/xf5/nykGIYQQwtxI8lXCNMicciIpNYOI208/0/3xK7HsPB8NgKu9LcNbVX/iPlk6e7pQrUIpAPZfvM2FqISnjkMIIYQwN5J8lTANKz/7fF+KovBp4AXd+6l+tQv01KKFWsXIbJOufrdfJl0VQghRckjyVcLoj/t6uhanXRdiOHblLgA1nUrzonflAh/jRe/K2NtqH7bdfOomMfeSnyoWIYQQwtxI8lXCZH/i8WlavjI0CvMCHy6e/YZ/naearLW0jSWDfaoBkJqh4afM6SqEEEKI4k6SrxKmQhkb3DOngwi7mYCmgIPuN528QXjmwtheVRzxr+/61LEMa1UNS7V2RuEfD1/lQWrGUx9LCCGEMBeSfJVA9TNbvxJT0rly536+90tJz+CLoH9077Mvnv003BzseKGxOwB3k9L47eR/T30sIYQQwlxI8lUCPW3X40+Hr3Ej7gEA7Z9zwrdmhWeOJfukq98diChwS5wQQghhbiT5KoEaVHo46P7czfwNur+XnMaS3Zd072d0zbl49tPF4kDLGuUB+PfWffb8I4vaCiGEKN4k+SqBGmRv+fovfy1fy/dHEHs/FYBejd2p7+7wxH3ya7TepKsy7YQQQojiTZKvEsi5rC0u9jYAnL0Z/8T1FW/dS9HNRG+pVjG9S96LZz+NjnWdqVGxNACHLt/h3M3CWXdSCCGEKIok+Sqhsma6v5eczrXYpMfWXbzrIkmZTyIO9qlKtQqlCzUWtVrFyDYy6aoQQoiSQZKvEqpBPgfdX7uTxNqj1wAoZW3B5I61DRJPQNPKOGauDbnl9E2i4mXSVSGEEMWTJF8lVPbk63Ez3c8PCictQ9stObqNB05lbQwSj521BS9nTrqarlH4IeSKQT5HCCGEMDWzS76WLFlC9erVsbW1xcfHh6NHjz62/vr166lbty62trY0bNiQ7du3623/7bff6NKlCxUqVEClUnHq1Kkcx0hOTmbixIlUqFCBMmXKEBAQQHR0dKGfmzE11Eu+cm/5Onczns2nbgJQvrQ1Y9rVyLVeYRnqWw0rC+28YWuOXCMpNd2gnyeEEEKYglklX+vWrWPatGnMmjWLEydO0LhxY/z9/YmJyX16gkOHDjFo0CBGjRrFyZMn6dOnD3369OHs2bO6Ovfv36dNmzZ8+umneX7ua6+9xtatW1m/fj179+7l5s2b9OvXzyDnaCwu9jZULPP4QffZlxGa+HwtytpaGTQmZ3tbejWuBED8gzQ2hMqkq0IIIYoflfKkR92KEB8fH5o3b87ixYsB0Gg0VKlShcmTJ/PWW2/lqD9gwADu37/Ptm3bdGUtW7bEy8uLpUuX6tW9cuUKHh4enDx5Ei8vL115fHw8Tk5OrF27lhdffBGACxcuUK9ePUJCQmjZsuUT405ISMDBwYH4+Hjs7e2fWN9Yhq88yp7wWwDsn/E8VcqX0m0LuXyHQcsPA1DJ0Y5dr7fHxtLC4DGdj0yg26L9AFSvUIrg6R2wUD/9LPpCCCHE0zLU72+zaflKTU0lNDQUPz8/XZlarcbPz4+QkJBc9wkJCdGrD+Dv759n/dyEhoaSlpamd5y6detStWrVPI+TkpJCQkKC3qsoyqvrUVEUPgm8oHs/rfNzRkm8AOq52dOmVkUArtxJIvi8eXfvCiGEEI8ym+Tr9u3bZGRk4OLiolfu4uJCVFRUrvtERUUVqH5ex7C2tsbR0THfx5k7dy4ODg66V5UqVfL9ecaUfaLUs9nm1vrzXDSnr8cBUMelLH2aVDJqXKPaPpx2YsUBmXZCCCFE8WI2yZc5mTlzJvHx8brX9evXTR1SrhpWzj7dhLZ1Lj1Dw2d/Pmz1mtG1jtG7/drXdqKWcxkAjkbE8vd/cUb9fCGEEMKQzCb5qlixIhYWFjmeMoyOjsbV1TXXfVxdXQtUP69jpKamEhennwA87jg2NjbY29vrvYoidwdbymXOrXXuhnbQ/cYT/3H51n0AmlcvR8e6zkaPS61W6S24LUsOCSGEKE7MJvmytrbG29ub4OBgXZlGoyE4OBhfX99c9/H19dWrDxAUFJRn/dx4e3tjZWWld5zw8HCuXbtWoOMURSqVSjff1537qVy5k8SCoIu67W92rYtKZZrB7n2bVKJCaWsA/jgTyc24ByaJQwghhChsZpN8AUybNo3ly5ezevVqzp8/z/jx47l//z4jRowAYOjQocycOVNX/9VXXyUwMJD58+dz4cIFZs+ezfHjx5k0aZKuTmxsLKdOnSIsLAwyE6tTp07pxnM5ODgwatQopk2bxu7duwkNDWXEiBH4+vrm60nHoi77oPvRq48RlaCdWd6vnjPNqpc3WVy2Vha83FI76WqGRmH1IZl0VQghRPFgVsnXgAED+Pzzz3nvvffw8vLi1KlTBAYG6gbVX7t2jcjISF39Vq1asXbtWpYtW0bjxo3ZsGEDmzZtokGDBro6W7ZsoUmTJvTo0QOAgQMH0qRJE72pKBYsWEDPnj0JCAigXbt2uLq68ttvvxn13A0lLUOj+39WdyOAb82KJorooZdbVsPaUnuLrj16jcQUmXRVCCGE+TOreb7MVVGd5yvwbCSv/HQi120q4JuXm9K1gZvR48ruzQ1/s+649oGF93p66i3ALYQQQhhSiZ/nSxSuDI3CnK1hj60zZ2sYGRrT5ubZp534/mCEyeMRQgghnpUkXyXU0YhYIuOT89yuAJHxyRyNiDVqXI96zqUs7Z9zAuC/uw/461z+52gTQgghiiJJvkqomHt5J15PU8+QRsukq0IIIYoRSb5KKOeytoVaz5Da1KpIHZeyAIRevcuJa3dNHZIQQgjx1CT5KqFaeJTHzcGWvGbxUgFuDra08DDddBO6WFQqvbFf38mkq0IIIcyYJF8llIVaxawXPCEz0cou6/2sFzyNvrRQXnp7uVOxjA0AO85Gcj02ydQhCSGEEE9Fkq8SrGsDN755uSmuDvpdi64OtkVimonsbCwtGOarnXRVo8AqmXRVCCGEmZJ5voygqM7zlSVDo3A0IpaYe8k4l9V2NRaVFq/sYu+n4js3mJR0DWVsLDk0syP2tlamDksIIUQxJfN8CYOxUKvwrVmB3l6V8K1ZoUgmXgDlS1sT4F0ZgMSUdNYdvW7qkIQQQogCk+RLmJWRrR8OvF95MIL0bMsjCSGEEOZAki9hVmo5l6FjXWcAbsYns+OsTLoqhBDCvEjyJcyO3qSr+/9Fhi0KIYQwJ5J8CbPjW6MCnm7agY+n/4sn9KpMuiqEEMJ8SPIlzI5KpdJr/Vq+/1+TxiOEEEIUhCRfwiz1bOSOc1ntpKt/hUVz9c59U4ckhBBC5IskX8IsWVuqGdaqOgCKAisPyqSrQgghzIMkX8JsDfGpip2VBQC/Hr9OfFKaqUMSQgghnkiSL2G2HEtZ81Iz7aSrSakZ/HzsmqlDEkIIIZ5Iki9h1ka09kCVOSH/qoNXSE2XSVeFEEIUbZJ8CbPmUbE0fvVcAIhKSGb7mUhThySEEEI8liRfwuyNbpNt0tUDMumqEEKIok2SL2H2WniUp1FlBwDO3kjgSESsqUMSQggh8iTJlzB7KpWKUdlbv/ZHmDQeIYQQ4nEk+RLFQveGbrg52AIQfCGaf28lmjokIYQQIleSfIliwcpCzfBsk65+f1Bav4QQQhRNknyJYmNgi6qUttZOuroh9D/u3k81dUhCCCFEDpJ8iWLDwc6K/s2rAJCcpmHtUZl0VQghRNEjyZcoVka08kCdNenqoSukpGeYOiQhhBBCjyRfolipWqEU/vVdAbh1L4Wtp2XSVSGEEEWLJF+i2BndNvu0EzLpqhBCiKJFki9R7DStWg6vKo4AXIi6x6HLd0wdkhBCCKFjdsnXkiVLqF69Ora2tvj4+HD06NHH1l+/fj1169bF1taWhg0bsn37dr3tiqLw3nvv4ebmhp2dHX5+fly8eFGvTvXq1VGpVHqvTz75xCDnJ56dSqViTNsauvcr9v9r0niEEEKI7Mwq+Vq3bh3Tpk1j1qxZnDhxgsaNG+Pv709MTEyu9Q8dOsSgQYMYNWoUJ0+epE+fPvTp04ezZ8/q6sybN48vv/ySpUuXcuTIEUqXLo2/vz/Jycl6x3r//feJjIzUvSZPnmzw8xVPz7++C5Uc7QDYHX6LSzH3TB2SEEIIAeaWfH3xxReMGTOGESNG4OnpydKlSylVqhTff/99rvUXLVpE165deeONN6hXrx4ffPABTZs2ZfHixZDZ6rVw4ULeffddevfuTaNGjfjhhx+4efMmmzZt0jtW2bJlcXV11b1Kly5tlHMWT8fSQs2I1tV17787IJOuCiGEKBrMJvlKTU0lNDQUPz8/XZlarcbPz4+QkJBc9wkJCdGrD+Dv76+rHxERQVRUlF4dBwcHfHx8chzzk08+oUKFCjRp0oTPPvuM9PT0Qj5DUdgGNK9CGRtLADaeuMGdxBRThySEEEKYT/J1+/ZtMjIycHFx0St3cXEhKioq132ioqIeWz/r3ycdc8qUKfzyyy/s3r2bcePG8fHHHzNjxow8Y01JSSEhIUHvJYyvrK0VAzMnXU1N1/DTYZl0VQghhOmZTfJlStOmTaNDhw40atSIV155hfnz5/PVV1+RkpJ7S8rcuXNxcHDQvapUqWL0mIXW8NbVscicdfXHw1dITpNJV4UQQphWoSRfcXFxhXGYx6pYsSIWFhZER0frlUdHR+Pq6prrPq6uro+tn/VvQY4J4OPjQ3p6OleuXMl1+8yZM4mPj9e9rl+/ns+zFIWtcrlSdGug/VreTkxl86kbpg5JCCFECVfg5OvTTz9l3bp1uvf9+/enQoUKVKpUidOnTxd2fDrW1tZ4e3sTHBysK9NoNAQHB+Pr65vrPr6+vnr1AYKCgnT1PTw8cHV11auTkJDAkSNH8jwmwKlTp1Cr1Tg7O+e63cbGBnt7e72XMJ3RetNORMikq0IIIUyqwMnX0qVLdd1oQUFBBAUFsWPHDrp168Ybb7xhiBh1pk2bxvLly1m9ejXnz59n/Pjx3L9/nxEjRgAwdOhQZs6cqav/6quvEhgYyPz587lw4QKzZ8/m+PHjTJo0CTLng5o6dSoffvghW7Zs4cyZMwwdOhR3d3f69OkDmYP2Fy5cyOnTp/n3339Zs2YNr732Gi+//DLlypUz6PmKwuFVxZFm1bRfq4sxiey7eNvUIQkhhCjBLAu6Q1RUlC752rZtG/3796dLly5Ur14dHx8fQ8SoM2DAAG7dusV7771HVFQUXl5eBAYG6gbMX7t2DbX6YT7ZqlUr1q5dy7vvvsvbb79N7dq12bRpEw0aNNDVmTFjBvfv32fs2LHExcXRpk0bAgMDsbW1hcxWrF9++YXZs2eTkpKCh4cHr732GtOmTTPouYrCNbqtB8ev3oXMSVfbP+dk6pCEEEKUUCqlgH0w7u7ubNiwgVatWlGnTh0+/PBDXnrpJcLDw2nevLk82ZeLhIQEHBwciI+Ply5IE8nQKDz/+R6uxSYBEDi1LXVd5WshhBAib4b6/V3gbsd+/foxePBgOnfuzJ07d+jWrRsAJ0+epFatWoUWmBCFyUKtYmT2SVf3y6SrQgghTKPAydeCBQuYNGkSnp6eBAUFUaZMGQAiIyOZMGGCIWIUolC81KwKZW21Pe2bT90k5l7yE/cRQgghCluBux1FwUm3Y9Exd8d5vt2rXWh7SsdaTOtSx9QhCSGEKKKKTLfj6tWr+eOPP3TvZ8yYgaOjI61ateLq1auFFpgQhjC8VXUsdZOuXpVJV4UQQhhdgZOvjz/+GDs7O8ichmHJkiXMmzePihUr8tprrxkiRiEKjZuDHT0buQFwNymNjSf+M3VIQgghSpgCJ1/Xr1/XDazftGkTAQEBjB07lrlz57J//35DxChEoRrV5uGkq98diECjkZ53IYQQxlPg5KtMmTLcuXMHgL/++ovOnTsDYGtry4MHDwo/QiEKWcPKDvh4lAfg31v32fNPjKlDEkIIUYIUOPnq3Lkzo0ePZvTo0fzzzz90794dgHPnzlG9evUn7i9EUfDokkNCCCGEsRQ4+VqyZAm+vr7cunWLjRs3UqFCBQBCQ0MZNGiQIWIUotB1quuMR8XSABy6fIdzN+NNHZIQQogSQqaaMAKZaqJo+vHwVf636SwA/ZpU4osBXqYOSQghRBFSZKaaAIiLi2P+/Pm67scFCxYQHy8tB8K8BDSthGMpKwC2nL5JVLxMuiqEEMLwCpx8HT9+nJo1a7JgwQJiY2OJjY3liy++oGbNmpw4ccIwUQphAKWsLRniUxWAdI3CDyFXTB2SEEKIEqDA3Y5t27alVq1aLF++HEtL7VIt6enpjB49mn///Zd9+/YZKlazJd2ORVd0QjJtPt1FWoaCva0lXw5qQvyDNJzL2tLCozwWmROyCiGEKHkM9fu7wMmXnZ0dJ0+epG7dunrlYWFhNGvWjKSkpEILrriQ5Ktom/brKX47cSNHuZuDLbNe8KRrAzeTxCWEEMK0isyYL3t7e65du5aj/Pr165QtW7aw4hLCaOq55v4NFRWfzPifThB4NtLoMQkhhCi+Cpx8DRgwgFGjRrFu3TquX7/O9evX+eWXXxg9erRMNSHMToZG4fuDuc/zldUkPGdrGBkyC74QQohCYlnQHT7//HNUKhVDhw4lPT0dACsrK8aPH88nn3xiiBiFMJijEbFEPuYpRwWIjE/maEQsvjUrGDU2IYQQxVOBky9ra2sWLVrE3LlzuXz5MgA1a9bE2tqamJgY3N3dDRGnEAYRcy9/00vkt54QQgjxJAVOvrKUKlWKhg0b6t6fPn2apk2bkpGRUVixCWFwzmVtC7WeEEII8SRPNcmqEMVFC4/yuDnYkteEEqrMpx5bZC7ELYQQQjwrSb5EiWahVjHrBU/ITLSyy3o/6wVPme9LCCFEoZHkS5R4XRu48c3LTXF10O9adHWw5ZuXm8o8X0IIIQpVvsd8/f3334/dHh4eXhjxCGESXRu40dnTlaMRscTcS5YZ7oUQQhhMvpMvLy8vVCoVuU2In1WuUskvKmG+LNQqmU5CCCGEweU7+YqIyH0iSiGEEEIIkX/5Tr6qVatm2EiEEEIIIUoAGXAvhBBCCGFEknwJIYQQQhiRJF9CCCGEEEYkyZcQQgghhBEVWvKVnJzM559/XliHE0IIIYQolgqUfN26dYtt27bx119/6RbQTktLY9GiRVSvXp1PPvnEUHEKIYQQQhQL+U6+Dhw4QO3atenVqxfdunWjVatWhIWFUb9+fb799ltmz57N9evXDRstsGTJEqpXr46trS0+Pj4cPXr0sfXXr19P3bp1sbW1pWHDhmzfvl1vu6IovPfee7i5uWFnZ4efnx8XL17UqxMbG8uQIUOwt7fH0dGRUaNGkZiYaJDzEyakyYCI/XBmg/ZfTYapIxJCCFEM5Tv5evfdd+nevTt///0306ZN49ixY/Tt25ePP/6YsLAwXnnlFezs7Awa7Lp165g2bRqzZs3ixIkTNG7cGH9/f2JiYnKtf+jQIQYNGsSoUaM4efIkffr0oU+fPpw9e1ZXZ968eXz55ZcsXbqUI0eOULp0afz9/UlOTtbVGTJkCOfOnSMoKIht27axb98+xo4da9BzFUYWtgUWNoDVPWHjKO2/Cxtoy4UQQohCpFJyWy8oFxUqVGD//v14enry4MEDypQpw2+//Ubv3r0NH2UmHx8fmjdvzuLFiwHQaDRUqVKFyZMn89Zbb+WoP2DAAO7fv8+2bdt0ZS1btsTLy4ulS5eiKAru7u5Mnz6d119/HYD4+HhcXFxYtWoVAwcO5Pz583h6enLs2DGaNWsGQGBgIN27d+e///7D3d39iXEnJCTg4OBAfHw89vb2hXhFRKEI2wK/DgUe/VbIXC6r/w/g2csUkQkhhDAhQ/3+zvcM93fv3qVixYoA2NnZUapUKRo0aFBogTxJamoqoaGhzJw5U1emVqvx8/MjJCQk131CQkKYNm2aXpm/vz+bNm2CzCWToqKi8PPz0213cHDAx8eHkJAQBg4cSEhICI6OjrrEC8DPzw+1Ws2RI0fo27dvgc4hNTU1R7larcbS0lKvXl5UKhVWVlZPVTctLS3XtTkNWRfA2tr6qeqmp6ej0WgKpa6VlZVu7VG9upoM2PE2YPGwLumZaZdCOpZodrwDNTqD2uKxx83IyNCNhXxSDE+qa2lpiVqtLjJ1NRoN6enpeda1sLDAwsKiyNRVFIW0tLRCqZv9+9NQdXnC97L8jMi9rlF+RhRiXfkZYZ4/Iwwh38kXQFhYGFFRUZAZeHh4OPfv39er06hRo8KNMNPt27fJyMjAxcVFr9zFxYULFy7kuk9UVFSu9bPOIevfJ9VxdnbW225paUn58uV1dR6VkpJCSkqK7n1CQgIA8+fPx9bWNkf92rVrM3jwYN37zz//PM+bolq1agwfPlz3ftGiRSQlJeVa193dnTFjxujeL1myhPj4+FzrOjk5MWHCBN375cuXc+vWrVzrOjg4MHXqVN37VatWcfPmzVzrlipVijfeeEP3fs2aNVy9ejXXulZWVrz99tu697/++muO8XfZzZo1S/f/33//nbCwsDzrzpw5U/eDeNu2bZw+fTrb1pd0jVwAryvfUJoHAPxJO44nesGn83I97quvvoqjoyMAwcHBef4hADB+/HjdvbR//3727t2bZ93Ro0dTqVIlAA4fPszOnTvzrDts2DCqV68OQGhoKDt27Miz7qBBg3juuecAOHPmDJs3b86z7osvvkj9+vUBOH/+PBs2bMizbu/evfHy8gLg0qVL/Pzzz3nW7datGy1atADg2rVrrF69Os+6fn5+tG7dGoDIyEhWrFiRZ9327dvToUMHyHw46Jtvvsmzrq+vL126dIHM1u5FixblWbdZs2b06NEDgKSkpMc+1d24cWP69OkDmUnE3Llz86zr6enJSy+9pHv/uLryM0LLdD8j9L3++uuULl0agD///JPjx4/nWVd+RmiZ888IQyhQ8tWpUye9v0h69uwJmX/lKIqCSqV6bJZcUsydO5c5c+aYOgwhhBBCFEH5HvOV118ijzLUAtypqamUKlWKDRs26P6yJDOjj4uLyzU7r1q1KtOmTdP7C2zWrFls2rSJ06dP8++//1KzZk1Onjypy8jJzI69vLxYtGgR33//PdOnT+fu3bu67enp6dja2rJ+/fpcux1za/mqUqUKt27dyrXPWLoUcq9rlC6FK4dgbYB+XV23I6RjgQYVdPscmgx57HGlS8E8uxSk21F+RiDdjs9ctyh83xviZ4TJx3ytXr2a119/nVKlShXahxeEtbU13t7eBAcH65IvjUZDcHAwkyZNynUfX19fgoOD9ZKvoKAgfH19AfDw8MDV1ZXg4GBd8pWQkMCRI0cYP3687hhxcXGEhobi7e0NwK5du9BoNPj4+OT6uTY2NtjY2OR6Dtl/GDzuXPOrIHWz/zA0h7rZf9kYrG6ttmDvDAmRuQy4B0syf/AETocH0dBmGljmfs0LMkbA3Oqq1ep832tFoa5KpTKruhjw+15+RhSdukXhe1l+RmgV9PuzsOV7qok5c+aYfG6radOmsXz5clavXs358+cZP3489+/fZ8SIEQAMHTpUb0D+q6++SmBgIPPnz+fChQvMnj2b48eP65I1lUrF1KlT+fDDD9myZQtnzpxh6NChuLu76xK8evXq0bVrV8aMGcPRo0c5ePAgkyZNYuDAgfl60lEUcWoL6Ppp5htV3vWUDNgzF5Z3hMi/jRWdEEKIYijfKXs+eycNasCAAdy6dYv33nuPqKgovLy8CAwM1A2Yv3btmq4JFKBVq1asXbuWd999l7fffpvatWuzadMmvac0Z8yYwf379xk7dixxcXG0adOGwMBAvYHxa9asYdKkSXTq1Am1Wk1AQABffvmlkc9eGIxnL+10EoFvQkK2QcH2laDLBxBzHvZ/oU3Aos/A8ueh7evQdnqerWBCCCFEXvI95kutVhMdHY2Tk5PhoypmZJ4vM6HJgKuHIDEayrhAtVYPp5e4eQo2TYCYcw/ruzSAPl+DW2OThSyEEMJwDPX7u0DJl4ODg24wYF5iY2MLK7ZiQ5KvYiI9FfZ/DvvngyZzUKfaUjsOrN0b0gomhBDFjMkH3JM57svBwaHQPlwIs2JpDc+/DXV7aFvBos9qk7B98+DCH9pWMHevfBxICCFESVaglq/cJhwVTyYtX8VQeqq2BWz/5w9bwVQW0DarFSzn065CCCHMi6F+f+f7accndTf++++/Bp0NVogixdIanp8JY3aDS0NtmZIB+z6DZR3g5klTRyiEEKKIynfy9aQGsnv37hEcHFwYMQlhPtwawZhd0GGmdvwXQEwYLO8EwR9AesqTjiCEEKKEyXfypdFopMtRiNxYWkOHt2DsHnDN1gq2/3P4tj3cOGHqCIUQQhQh+U6+hBBP4NpQ2w35/Dugzpyl+9Z5WOEHO+dIK5gQQgiQ5EuIQmZhBe1nZLaCNdKWKRlw4Av4th3cCDV1hEIIIUws31NNNGnS5LGD7pOSkgorJiHMn2sD7ViwAwth76egSYNbF7StYK1fhfZvgZVtPg4khBCiuMl38pW11qEQIp8srKD9G1C3O2waD5GnQdHAgQUQvgN6fw2VvU0dpRBCCCPL9zxf4unJPF+CjDQ4uBD2ZLaCAajU0GqK9klJaQUTQogix+TzfAkhnoGFlXby1XH7wC1zFnxFo03Ivm0H/x03dYRCCCGMRJIvIYzJxRNGB0On98Aicy3I2+HwXWcIeg/Skk0doRBCCAOT5EsIY7OwhLbTYexecG+iLVM0cHARfNsWrh8zdYRCCCEMSJIvIUzFxRNG7YROs7K1gv0D33eBv/4HaQ9MHaEQQggDeKbk67///kOj0RReNEKUNBaW2sW4x+0D96baMkUDh76EpW3h+lFTRyiEEKKQPVPy5enpyZUrVwovGiFKKud6MCoI/GY/bAW7cxG+6wJ/viOtYEIIUYw8U/Ils1QIUYgsLKHNazBuP1TKmv9LgZDFsLQNXDti4gCFEEIUBhnzJURR41wXRv4FfnPAwkZbducSfO+vbQVLldUkhBDCnD1T8vX2229Tvnz5wotGCKFlYQltpsIr+6FSs8zC7K1gh00coBBCiKclM9wbgcxwL56JJkObdO36CDJSMgtV0HICdHwXrEuZOEAhhCieZIZ7IUoqtYV2Me5XDkDl5pmFChxeAktbw9UQEwcohBCiICT5EsJcOD0HI/+ELh+CZeZakLH/wspusOMtGQsmhBBmQpIvIcyJ2gJaTc5sBWuRWajAkW/gm1Zw9ZCJAxRCCPEkknwJYY4q1oaRgdDlo4etYHcjYGV32PEmpN43dYRCCCHyUODkKzAwkAMHDujeL1myBC8vLwYPHszdu3cLOz4hRF7UFtBqErxyEKr4ZBYqcGQpfNMarhw0cYBCCCFyU+Dk64033iAhIQGAM2fOMH36dLp3705ERATTpk0zRIxCiMepWAtG7AD/j/VbwVZ1h+0zpBVMCCGKmAInXxEREXh6egKwceNGevbsyccff8ySJUvYsWOHIWIUQjyJ2gJ8J8L4Q1Cl5cPyo99qx4JdOfC4vYUQQhhRgZMva2trkpK0T1Xt3LmTLl26AFC+fHldi5gQwkQq1IQR26HrJ2Bppy27ewVW9YDtb0BKonbesIj9cGaD9l9NhqmjFkKIEsWyoDu0adOGadOm0bp1a44ePcq6desA+Oeff6hcubIhYhRCFITaAlqOh9pdYPNEuJY5D9jRZXD2d+24sKTbD+vbu0PXT8Gzl8lCFkKIkqTALV+LFy/G0tKSDRs28M0331CpUiUAduzYQdeuXQ0RoxDiaVSoCcO3axOrrFawpFv6iRdAQiT8OhTCtpgkTCGEKGlkeSEjkOWFhMnd+kc7G35Gah4VVNoWsKlntC1nQgghis7yQidOnODMmTO695s3b6ZPnz68/fbbpKbm9YP92cXGxjJkyBDs7e1xdHRk1KhRJCYmPnaf5ORkJk6cSIUKFShTpgwBAQFER0fr1bl27Ro9evSgVKlSODs788Ybb5Cenq7bvmfPHlQqVY5XVFSUwc5ViEKXGP2YxAttV2TCDZmkVQghjKDAyde4ceP4559/APj3338ZOHAgpUqVYv369cyYMcMQMQIwZMgQzp07R1BQENu2bWPfvn2MHTv2sfu89tprbN26lfXr17N3715u3rxJv379dNszMjLo0aMHqampHDp0iNWrV7Nq1Sree++9HMcKDw8nMjJS93J2djbIeQphEInR+agEXNgGGo2hoxFCiBKtwN2ODg4OnDhxgpo1a/Lpp5+ya9cu/vzzTw4ePMjAgQO5fv16oQd5/vx5PD09OXbsGM2aNYPMyV67d+/Of//9h7u7e4594uPjcXJyYu3atbz44osAXLhwgXr16hESEkLLli3ZsWMHPXv25ObNm7i4uACwdOlS3nzzTW7duoW1tTV79uzh+eef5+7duzg6Oj5V/NLtKEwuYj+s7pm/uk51od0bUL+vdEEKIUq0ItPtqCgKmsy/jHfu3En37t0BqFKlCrdv337C3k8nJCQER0dHXeIF4Ofnh1qt5siRI7nuExoaSlpaGn5+frqyunXrUrVqVUJCQnTHbdiwoS7xAvD39ychIYFz587pHc/Lyws3Nzc6d+7MwYOPnzk8JSWFhIQEvZcQJlWtlXZMF6on1711ATaOgq9bwt+/Qkb6k/cRQgiRbwVOvpo1a8aHH37Ijz/+yN69e+nRowdkTr6aPYkpTFFRUTm6+SwtLSlfvnyeY6+ioqKwtrbO0Vrl4uKi2ycqKipHzFnvs+q4ubmxdOlSNm7cyMaNG6lSpQodOnTgxIkTecY7d+5cHBwcdK8qVao85ZkLUUjUFtqnHiGXBEylfbWbkW2ZIuD2P/DbGFjSAk6tlSRMCCEKSYGTr4ULF3LixAkmTZrEO++8Q61atQDYsGEDrVq1KtCx3nrrrVwHs2d/XbhwoaAhFqo6deowbtw4vL29adWqFd9//z2tWrViwYIFee4zc+ZM4uPjdS9DdMUKUWCevaD/D2Dvpl9u764t7/gOjPwThm6Baq0fbo+9DJvGw2JvOPEjZKQZPXQhhChOCjzJaqNGjfSedszy2WefYWFRsPEh06dPZ/jw4Y+tU6NGDVxdXYmJidErT09PJzY2FldX11z3c3V1JTU1lbi4OL3Wr+joaN0+rq6uHD16VG+/rKch8zouQIsWLfQWF3+UjY0NNjY2jz0vIUzCsxfU7aF9qjExGsq4aLsks8Z2qVRQo732deUA7PkEruzXbrt7BbZMgn3zoO10aDwYLK1NejpCCGGOCpx8ZQkNDeX8+fMAeHp60rRp0wIfw8nJCScnpyfW8/X1JS4ujtDQULy9vQHYtWsXGo0GHx+fXPfx9vbGysqK4OBgAgICIPOJxWvXruHr66s77kcffURMTIyuWzMoKAh7e3vd+pW5OXXqFG5ubnluF6JIU1uAR9sn16veBoZv0yZqe+fBv7u15XHXYOursPczaPsaNPk/sJQ/NoQQIr8K/LRjTEwMAwYMYO/evboWpbi4OJ5//nl++eWXfCVTT6Nbt25ER0ezdOlS0tLSGDFiBM2aNWPt2rUA3Lhxg06dOvHDDz/QokULAMaPH8/27dtZtWoV9vb2TJ48GYBDh7RzGWVkZODl5YW7uzvz5s0jKiqK//u//2P06NF8/PHHkNnN6uHhQf369UlOTmbFihV89dVX/PXXX3Tq1ClfscvTjqJYuH4U9n4Kl3bql5d1hzavQdOhYGVrquiEEKLQFZmnHSdPnkxiYiLnzp0jNjaW2NhYzp49S0JCAlOmTCm0wB61Zs0a6tatS6dOnejevTtt2rRh2bJluu1paWmEh4frFv0GWLBgAT179iQgIIB27drh6urKb7/9pttuYWHBtm3bsLCwwNfXl5dffpmhQ4fy/vvv6+qkpqYyffp0GjZsSPv27Tl9+jQ7d+7Md+IlRLFRpQW8vBFG74Lnsi0ldu8m7HgDFjWGw99A2gNTRimEEEXeU83ztXPnTpo3b65XfvToUbp06UJcXFxhx2j2pOVLFEs3T2q7HsP/0C8v7Qytp0CzkWBd2lTRCSHEMysyLV8ajQYrK6sc5VZWVrr5v4QQJYB7Exi0Fsbth3ovPCy/HwN/vQsLG8GBhZDy+GXAhBCipClw8tWxY0deffVVbt68qSu7ceMGr732mnTFCVESuTWCAT/B+EPaWfGz5hFLug07Z8HChrB/PiTLZMNCCMHTdDtev36dXr16ce7cOd3kodevX6dBgwZs3rxZJhTNhXQ7ihIl5jzs+xzObtQu2J3F1hF8J4HPWLB1MGWEQgiRL4b6/V3g5IvMJYZ27typmwC1Xr16esv4CH2SfIkS6dY/sP9zOLMelGxDEmwcoOV4aPkK2JUzZYRCCPFYRSr5ys2FCxfo1asX//zzT2EcrliR5EuUaHcua7sdT/8CSsbDcht78BkHLSdAqfKmjFAIIXJVZAbc5yUlJYXLly8X1uGEEMVFhZrQ52uYfFw7Ias6c27nlATY95l2TNjO2XD/jqkjFUIIoyi05EsIIR6rfA3ovRgmnwDv4aDOfGo6NREOLNAmYX/9DxJvmTpSIYQwKEm+hBDGVa4avLAIppyEZqPAInN9yLT7cOhLbRL25ztwL9rUkQohhEFI8iWEMA3HKtDzC5hyClqMA4vM9SHTH0DIYljUCHa8CQmRpo5UCCEKVb4H3JcrVw6VSpXn9vT0dO7fv09GRkaedUoqGXAvRD7ci4KDi+D495Ce/LDcwka7bmSbqeBQ2ZQRCiFKGJM/7bh69ep8HXDYsGHPGlOxI8mXEAVwLxpCvoJj30Haw7VasbCGJi9rF/F2rGrKCIUQJYTJky/x9CT5EuIpJN7Sdj8eXa4dD5ZFbQleg6HtdChX3ZQRCiGKOUm+zJgkX0I8g/t34PASOLIMUu89LFdZQONB0HaadjoLIYQoZJJ8mTFJvoQoBEmxcGQpHF4KKfEPy1UW0Kg/tH0dKtYyZYRCiGJGki8zJsmXEIXoQRwc+VbbGpacPQlTQ4MAaPcGONUxZYRCiGJCki8zJsmXEAaQnABHv4WQJfDgbrYNKqjfV5uEuXiaMEAhhLkr8ssL/fvvv3Tp0qWwDieEEI9na69NsKaeAb/ZUKpC5gYFzv0G3/jCr0Mh6qyJAxVCCH2Flnzdu3eP4ODgwjqcEELkj01Z7fQTr/4NnT+A0k4Pt4VthqWt4ZchEHnalFEKIYSOzHAvhCgebMpA6ynaJMz/Yyjj8nDbhW3wbTtYOxBunDBllEIIIcmXEKKYsS4FvhPh1dPQbR6UdXu47Z8dsPx5WPMS/HfclFEKIUowSb6EEMWTlR34jNOuHdn9c7Cv9HDbxb9gRSf4sS9cO2LKKIUQJVC+n3Zs0qTJY9d2TEpK4uLFi7K2Yy7kaUchioD0FDi1BvZ/AfHX9bd5tIf2b0L11qaKTghRBBnq97dlfiv26dOn0D5UCCGMztIGmo0Er5fh719g3+cQd1W7LWKv9lWtDXR4E6q3hcf8sSmEEM8i3y1f165do3LlyqjV0lNZUNLyJUQRlJEGf/8K+z6DuxH626q2gvYzoEYHScKEKMFMPsmqhYUFkZGRODs7F9qHlxSSfAlRhGWkw9kN2iTsziX9bZVbaLsja3WSJEyIEsjkk6zKRPhCiGLJwhIaD4SJRyHgO6iYbWmi/47CmgDt4Px//gT5OSiEKAQF6kN83IB7IYQwa2oLaPgiTAiBF1eCc7aliW6Ewtr+sKwDXNguSZgQ4pnku9tRrVYzduxYSpUq9dh6X3zxRWHFVmxIt6MQZkijgQtbYe88iH5kiSLXhtBuBtTtCTIOVohiy+RPOwKcOXMGa2vrPLdLy5gQothQq8GzN9R9AcK3w95PIepv7baoM/Dr/4FzfWj/BtTrLUmYECLfCtTyFRUVJQPun4K0fAlRDCiKdtzX3k/g5kn9bU51tYt81++r7b4UQhQLJh9wL61aQogSTaWCOl1hzG4YsgEqNXu47dYF2DgKlvjA6XXaJyiFECIPhfq044MHD541HiGEKNpUKqjdGUbvhP/7Haq0fLjtzkX4fSwsaQGn1koSJoTIVb6Tr5UrV+Lg4JDrtpSUFObPn4+Hh0dhxqYnNjaWIUOGYG9vj6OjI6NGjSIxMfGx+yQnJzNx4kQqVKhAmTJlCAgIIDo6Wq/OlClT8Pb2xsbGBi8vr1yP8/fff9O2bVtsbW2pUqUK8+bNK9RzE0KYIZUKanaEkYEwdAtUy7Y0Uexl2DQeFnvDiR+1E7oKIUSmfCdfAwcOZPbs2TRr1oxWrVqxadMmyEzKPDw8WLhwIa+99prBAh0yZAjnzp0jKCiIbdu2sW/fPsaOHfvYfV577TW2bt3K+vXr2bt3Lzdv3qRfv3456o0cOZIBAwbkeoyEhAS6dOlCtWrVCA0N5bPPPmP27NksW7as0M5NCGHGVCqo0R5GbIfhf4BHu4fb7l6BLZPgq6ZwfCWkp5oyUiFEUaHk04wZMxQHBwclICBAcXNzUywtLZUxY8YoDRs2VH7++WclPT09v4cqsLCwMAVQjh07pivbsWOHolKplBs3buS6T1xcnGJlZaWsX79eV3b+/HkFUEJCQnLUnzVrltK4ceMc5V9//bVSrlw5JSUlRVf25ptvKnXq1Ml3/PHx8QqgxMfH53sfIYQZu3JIUVb3VpRZ9vqv+Z6KcnS5oqQlmzpCIUQ+GOr3d75bvtavX88PP/zAhg0b+Ouvv8jIyCA9PZ3Tp08zcOBALCwM94RPSEgIjo6ONGv2cICrn58farWaI0eO5LpPaGgoaWlp+Pn56crq1q1L1apVCQkJKdBnt2vXTm+KDX9/f8LDw7l7926u+6SkpJCQkKD3EkKUINV8YegmGBUEtTo/LE/4D/6YDou84MgySEs2ZZRCCBPJd/L133//4e3tDUCDBg2wsbHhtddeM8pTkLlNcWFpaUn58uWJiorKcx9ra2scHR31yl1cXPLcJ6/juLi45DhG1rbczJ07FwcHB92rSpUq+f48IUQxUqUFvLwBRu+C57o+LL93E3a8AYsaQ8jXkJpkyiiFEEaW7+QrIyNDr/XH0tKSMmXKPNOHv/XWW6hUqse+Lly48EyfYQozZ84kPj5e97p+/bqpQxJCmFJlbxi8DsbugTo9HpYnRsGfM7VJ2KGvIPW+KaMUQhhJvme4VxSF4cOHY2NjA5lPEr7yyiuULl1ar95vv/2W7w+fPn06w4cPf2ydGjVq4OrqSkxMjF55eno6sbGxuLq65rqfq6srqampxMXF6bV+RUdH57lPXsd59AnJrPd5HcfGxkZ3nYQQQse9CQxaC5F/w77P4PwWbfn9GPjrXTiwEFpNhuajwebZ/rgVQhRd+U6+hg0bpvf+5ZdffuYPd3JywsnJ6Yn1fH19iYuLIzQ0VNf1uWvXLjQaDT4+Prnu4+3tjZWVFcHBwQQEBAAQHh7OtWvX8PX1zXeMvr6+vPPOO6SlpWFlZQVAUFAQderUoVy5cvk+jhBC6Lg1ggE/QvQ5bRJ2bhOgQNJt2DkLDi6CVpOg+RiwlVUxhChu8r28kKl169aN6Oholi5dSlpaGiNGjKBZs2asXbsWgBs3btCpUyd++OEHWrRoAcD48ePZvn07q1atwt7ensmTJwNw6NAh3XEvXbpEYmIiS5cuZffu3axbtw4AT09PrK2tiY+Pp06dOnTp0oU333yTs2fPMnLkSBYsWPDEqS6yyPJCQojHirmgTcLObtQmYVlsHcF3IrQYC3aOjzuCEMIADPX722ySr9jYWCZNmsTWrVtRq9UEBATw5Zdf6sadXblyBQ8PD3bv3k2HDh0gs2t0+vTp/Pzzz6SkpODv78/XX3+t113YoUMH9u7dm+PzIiIiqF69OmROsjpx4kSOHTtGxYoVmTx5Mm+++Wa+Y5fkSwiRL7f+gf3z4cyvoGgelts4QMvx0PIVsJMWdyGMpcQnX+ZMki8hRIHcuaxNwk7/AkrGw3LrsuAzTtsaVqq8KSMUokSQ5MuMSfIlhHgqsRGZSdjPoMm2TqR1GWgxBnwnQemKpoxQiGJNki8zJsmXEOKZ3L0KBxbAyZ9Ak22dSKvS0HyU9gnJMs6PO4IQ4ilI8mXGJPkSQhSKuOtwcCGc+AEysq0TaWkHzUZC6ylQNv9T6QghHk+SLzMmyZcQolAl3NTOCRa6CjJSHpZb2oL3cGj9Kti7mzJCIYoFSb7MmCRfQgiDuBcFB7+E499D+oOH5RY20HQotJkKDpVNGaEQZk2SLzMmyZcQwqASY+DQl3DsO0jLtk6k2gqavAxtp4FjVVNGKIRZkuTLjEnyJYQwisRbELIYji6HtGzrRKotwWswtJkG5T1MGaEQZkWSLzMmyZcQwqju34HDX8ORbyH13sNylQU0HqRtCatQ05QRCmEWJPkyY5J8CSFMIikWjiyFw0shJf5hucoCGvWHtq9DxVqmjFCIIk2SLzMmyZcQwqQexGlbwQ4vgeTsSZgaGgRAuzfAqY4pIxSiSJLky4xJ8iWEKBKSE+DoMu24sAd3s21QQf2+2iTMxdOEAQpRtEjyZcYk+RJCFCkp9+DYCjj0FSTd0d9Wrxe0nwGuDU0VnRBFhiRfZkySLyFEkZR6Xzs9xaEv4f4t/W11e2pbwty9TBWdECYnyZcZk+RLCFGkpSZpZ8s/uBASo/W3PdcN2r8BlbxNFZ0QJiPJlxmT5EsIYRbSHmjXjTywAO5F6m+r1RnavwlVmpsqOiGMTpIvMybJlxDCrKQlw8kftUlYwg39bTU7apOwqi1NFZ0QRiPJlxmT5EsIYZbSU+DUWtj/BcRf09/m0Q7avwXVW5sqOiEMzlC/v9WFdiQhhBDFi6UNNBsBk0Oh11fgWO3htoh9sKo7rOyh/b/8HS9EvknyJYQQ4vEsraHpUG0S1vtrKF/j4barB2D1C7CyG1zeJUmYEPkgyZcQQoj8sbCCJkNg4jHouwwq1H647VoI/NgXvusCF3dKEibEY0jyJYQQomAsLKHxAJh4BAK+g4rZlib67yisCYAVneCfPyUJEyIXknwJIYR4OmoLaPgiTDgML64E52xLE90IhbX9YVl7uPCHJGFCZCPJlxBCiGejVkODfvDKQej/A7g0eLgt8jT8MhiWtoWwLaDRmDJSIYoESb6EEEIUDrUaPHvDuP0wcC24NX64LfoM/Pp/sLQNnPtdkjBRoknyJYQQonCp1VC3B4zdC4PWgXvTh9tizsH64fCNL5zZAJoMU0YqhElI8iWEEMIwVCqo0xXG7IIhG6FytqWJbl2AjaNgiQ+cXgcZ6aaMVAijkuRLCCGEYalUUNsPRgXB//0OVbItTXTnIvw+Fpa00M6mL0mYKAEk+RJCCGEcKpV2bciRgTB0C1Rr83Bb7GXYNB4We2sX985IM2WkQhiUrO1oBLK2oxBC5OHKAdj7qXaJouwcqkLbaeA1RDvDvhAmIGs7CiGEKH6qt4FhW2FEINR4/mF5/DXYNhW+bALHVmgX+RaimJCWLyOQli8hhMin60dh7zy4FKRfXtYd2kyFpsPAytZU0YkSxlC/vyX5MgJJvoQQooBuhGqTsH8C9cvLuELrV8F7OFiXMlV0ooQo8d2OsbGxDBkyBHt7exwdHRk1ahSJiYmP3Sc5OZmJEydSoUIFypQpQ0BAANHR0Xp1pkyZgre3NzY2Nnh5eeU4xpUrV1CpVDlehw8fLvRzFEIIkamSNwxep50rrG7Ph+WJUfDnTFjUGA59Ban3TRmlEE/FbJKvIUOGcO7cOYKCgti2bRv79u1j7Nixj93ntddeY+vWraxfv569e/dy8+ZN+vXrl6PeyJEjGTBgwGOPtXPnTiIjI3Uvb2/vZz4nIYQQT+DuBQPXaGfNr9frYfn9GPjrXVjYEA4sgJTH/zEuRFFiFt2O58+fx9PTk2PHjtGsWTMAAgMD6d69O//99x/u7u459omPj8fJyYm1a9fy4osvAnDhwgXq1atHSEgILVu21Ks/e/ZsNm3axKlTp/TKr1y5goeHBydPnsy1ZSw/pNtRCCEKSfQ52PcZnNsEZPv1ZVcefCdCi7FgKz9nReEo0d2OISEhODo66hIvAD8/P9RqNUeOHMl1n9DQUNLS0vDz89OV1a1bl6pVqxISElLgGHr16oWzszNt2rRhy5Ytj62bkpJCQkKC3ksIIUQhcKkPL62CCYeh4Uugyvw19iAWdn2gbQnbOw8exJk6UiHyZBbJV1RUFM7OznpllpaWlC9fnqioqDz3sba2xtHRUa/cxcUlz31yU6ZMGebPn8/69ev5448/aNOmDX369HlsAjZ37lwcHBx0rypVquT784QQQuSDc10IWAETj0KjgQ+TsOQ42P0RLGwEuz+GB3dNHakQOZg0+XrrrbdyHcye/XXhwgVThkjFihWZNm0aPj4+NG/enE8++YSXX36Zzz77LM99Zs6cSXx8vO51/fp1o8YshBAlRsXa0O9bmHRcOyGrykJbnhKvnbx1QUMI/gCSYk0dqRA6lqb88OnTpzN8+PDH1qlRowaurq7ExMTolaenpxMbG4urq2uu+7m6upKamkpcXJxe61d0dHSe++SXj48PQUFBeW63sbHBxsbmmT5DCCFEAVSoCX2+hnZvwP75cPpn0KRD6j3Y/zkcWQotxoDvJChd0dTRihLOpMmXk5MTTk5OT6zn6+tLXFwcoaGhuqcMd+3ahUajwcfHJ9d9vL29sbKyIjg4mICAAADCw8O5du0avr6+zxT3qVOncHNze6ZjCCGEMIDyHtB7sTYJO7AATv4EmjRITdS+P/ItNB8FraZAGed8HFCIwmfS5Cu/6tWrR9euXRkzZgxLly4lLS2NSZMmMXDgQN2Tjjdu3KBTp0788MMPtGjRAgcHB0aNGsW0adMoX7489vb2TJ48GV9fX70nHS9dukRiYiJRUVE8ePBA97Sjp6cn1tbWrF69Gmtra5o0aQLAb7/9xvfff8+KFStMdDWEEEI8Ublq8MJCaPe6Nuk68QNkpEJaknZ+sKMroNlIaD0Fyj5bb4gQBWUWyRfAmjVrmDRpEp06dUKtVhMQEMCXX36p256WlkZ4eDhJSUm6sgULFujqpqSk4O/vz9dff6133NGjR7N3717d+6wkKyIigurVqwPwwQcfcPXqVSwtLalbty7r1q3TTV8hhBCiCHOoDD3mQ9vpcGAhhK6CjBRIfwCHl8Dx77Sz5bd+FexzTlskhCGYxTxf5k7m+RJCiCLiXhQc/BKOf69NwLJYWEPTodDmNW3CJoSs7WjeJPkSQogiJjEGDn0Jx77TdkVmUVtBk5e1SVi5aqaMUBQBknyZMUm+hBCiiLp/G0IWw9Hl2kH5WdSW4DUY2kzTDuIXJZIkX2ZMki8hhCjikmIhZIn2acjUew/LVRbQeBC0naadzkKUKJJ8mTFJvoQQwkwkxWrnBDu8VDtRaxaVGhr21z49WbG2KSMURiTJlxmT5EsIIczMgzg4ukzbGpacbZ1IlRoaBEDb17VLHIliTZIvMybJlxBCmKnkhMwkbPEj60SqoH4faDcDXDxNGKAwJEm+zJgkX0IIYeZS7sGxFdoJWpPu6G+r1wvazwDXhqaKThiIJF9mTJIvIYQoJlLva+cIO7gI7t/S31a3p3ZZI3cvU0UnCpkkX2ZMki8hhChmUpO0s+UfXAiJ0frbnuuqbQmr5G2q6EQhkeTLjEnyJYQQxVTaA+26kQcWwL1I/W21OkP7N6FKc1NFJ56RJF9mTJIvIYQo5tKS4dRPsH8BJPynv63G89DhLaja0lTRiadkqN/f6kI7khBCCFFSWdlC89Ew5QT0XAgOVR9u+3c3fO8Pq1+AKwdNGaUoIiT5EkIIIQqLpQ00GwGTQ6HXV+CYbX3IiH2wqjus7AH/7gXpeCqxJPkSQgghCpulNTQdqk3C+nwD5Ws83Hb1APzQC1Z2g8u7JAkrgST5EkIIIQzFwkq7QPfEY9B3GVTItjTRtRD4sS981xku7pQkrASR5EsIIYQwNAtLaDwAJh6BgO+gYp2H2/47BmsCYHlHCA+UJKwEkORLCCGEMBa1BTR8ESYchpdWgXO2pYlunoCfB8Cy9nDhD0nCijFJvoQQQghjU6uhfl945SD0/xFcsi1NFHkafhkMS9tC2GbQaEwZqTAASb6EEEIIU1GrwbMXvLIfBq4Ft8YPt0WfgV+HwtLWcPY3ScKKEUm+hBBCCFNTqaBuDxi7Fwb/Cu5NH26LCYMNI+AbXzizATQZpoxUFAJJvoQQQoiiQqWC5/xhzC4YshEqZ1ua6NYF2DgKlvjA6XWQkW7KSMUzkORLCCGEKGpUKqjtB6OC4P9+hyrZlia6cxF+HwtLmsPJNZCRZspIxVOQ5EsIIYQoqlQqqNkRRgbCsK1Qrc3DbbH/wuYJsLiZdnFvScLMhiRfQgghRFGnUoFHOxjxBwz/Q/v/LHevwJbJ8GVTOL4S0lNNGanIB5WiyEQihmaoVdGFEEKUYFdDYN887RJF2dlXhjZTtcsbWdqYKrpiwVC/vyX5MgJJvoQQQhjM9WOw91O4FKRfXtb9YRJmZWeq6MyaJF9mTJIvIYQQBncjFPZ+Bv/s0C8v4wKtp4L3cLAuZarozJIkX2ZMki8hhBBGc/MU7PsMLmzTLy/tBK2mQPNRYF3aVNGZFUm+zJgkX0IIIYwu8m9tEnZ+i355qQrQajI0Hw02ZU0VnVmQ5MuMSfIlhBDCZKLDtEnYud+BbL/y7cqB7yRoMRZs5XdTbiT5MmOSfAkhhDC5mAuw/3M4uxGUbOtE2jpAy4ngMw7sHE0ZYZEjyZcZk+RLCCFEkXH7Iuz7HM78qp+E2ThAy1fA5xUoVd6UERYZhvr9bTaTrMbGxjJkyBDs7e1xdHRk1KhRJCYmPnaf5ORkJk6cSIUKFShTpgwBAQFER0frtp8+fZpBgwZRpUoV7OzsqFevHosWLcpxnD179tC0aVNsbGyoVasWq1atMsg5CiGEEAZXsTb0+xYmHQevl0FloS1PiddOWbGwEQR/AEmxpo602DKb5GvIkCGcO3eOoKAgtm3bxr59+xg7duxj93nttdfYunUr69evZ+/evdy8eZN+/frptoeGhuLs7MxPP/3EuXPneOedd5g5cyaLFy/W1YmIiKBHjx48//zznDp1iqlTpzJ69Gj+/PNPg56vEEIIYVAVakKfJTA5VDsXmNpSW556T9s9ubAhBM2C+7dNHWmxYxbdjufPn8fT05Njx47RrFkzAAIDA+nevTv//fcf7u7uOfaJj4/HycmJtWvX8uKLLwJw4cIF6tWrR0hICC1btsyxD8DEiRM5f/48u3ZpZwx+8803+eOPPzh79qyuzsCBA4mLiyMwMDBf8Uu3oxBCiCIv7hrs/wJO/gSabOtEWpXSTk/RagqUcTZlhEZXorsdQ0JCcHR01CVeAH5+fqjVao4cOZLrPqGhoaSlpeHn56crq1u3LlWrViUkJCTPz4qPj6d8+Yd93SEhIXrHAPD393/sMVJSUkhISNB7CSGEEEWaY1V4YSG8eko7DYWFtbY8LQkOfaXtjgx8G+5FmThQ82cWyVdUVBTOzvrZtqWlJeXLlycqKvebICoqCmtraxwd9Z/ccHFxyXOfQ4cOsW7dOr3uzKioKFxcXHIcIyEhgQcPHuR6nLlz5+Lg4KB7ValSJd/nKoQQQpiUQ2XoMR9ePa0dfG9pqy1PfwCHl2iTsO0zIOGmqSM1WyZNvt566y1UKtVjXxcuXDBKLGfPnqV3797MmjWLLl26PNOxZs6cSXx8vO51/fr1QotTCCGEMAp7d+j2qTYJazkRLDPXh8xIgaPfwqLG8Md0iJPfcQVlacoPnz59OsOHD39snRo1auDq6kpMTIxeeXp6OrGxsbi6uua6n6urK6mpqcTFxem1fkVHR+fYJywsjE6dOjF27FjefffdHMfJ/oRk1jHs7e2xs8t9oVIbGxtsbGQleSGEEMVAWVfo+rF2ke5DX8GxFdquyIxU7f9DV0OTl6HNa1CumqmjNQsmTb6cnJxwcnJ6Yj1fX1/i4uIIDQ3F29sbgF27dqHRaPDx8cl1H29vb6ysrAgODiYgIACA8PBwrl27hq+vr67euXPn6NixI8OGDeOjjz7K9bO3b9+uVxYUFKR3DCGEEKLYK+MMXT6A1q9CyGI4uhxSE7WD80NXwskfofEgaDsdynuYOtoizSyedgTo1q0b0dHRLF26lLS0NEaMGEGzZs1Yu3YtADdu3KBTp0788MMPtGjRAoDx48ezfft2Vq1ahb29PZMnT4bMsV1kdjV27NgRf39/PvvsM91nWVhY6JLCiIgIGjRowMSJExk5ciS7du1iypQp/PHHH/j7++crdnnaUQghRLGTFAshS+DIt9rpKbKoLKDxQG0SVqGmKSN8ZiX6aUeANWvWULduXTp16kT37t1p06YNy5Yt021PS0sjPDycpKQkXdmCBQvo2bMnAQEBtGvXDldXV3777Tfd9g0bNnDr1i1++ukn3NzcdK/mzZvr6nh4ePDHH38QFBRE48aNmT9/PitWrMh34iWEEEIUS6XKQ6f/wWtnoP1b2hnyAZQMOLUGFjeD38ZpZ9QXesym5cucScuXEEKIYu9BHBxdpm0NS47LtkEFDQKg3RvgXNeEARacrO1oxiT5EkIIUWIkJ2QmYYvhwd1sG1RQv482CXOpb8IA80+SLzMmyZcQQogSJ+We9mnIQ19B0h39bfV6QfsZ4NrQVNHliyRfZkySLyGEECVW6n04/j0cXAT3b+lvq9NDm4S5e5kquseS5MuMSfIlhBCixEtNgtBVcHAhJOrPn8lzXbVJWCVvU0WXK0m+zJgkX0IIIUSmtAdw4kc4sADuPbJEUa3O0P5NqNI8r72NSpIvMybJlxBCCPGItGQ49RPsXwAJ/+lvq/E8dHgLqrY0VXQg83wJIYQQolixsoXmo2HKSei5EByqPtz272743h9WvwBXDpgySoOQ5EsIIYQQpmNpDc1GwJQT0GsxlKv+cFvEPljVA1Z2h3/3QjHprJNuRyOQbkchhBAinzLS4Mx62PcZxP6rv62qr3Zgfo3nQaXSlmky4Ooh7SD+Mi5QrRWoLQolFBnzZcYk+RJCCCEKKCMdzm7UJmF3HlmiqHJz7cD8tCQIfAsSsg3ct3eHrp+CZ69nDkGSLzMmyZcQQgjxlDQZcO53bRJ260I+dshsEev/wzMnYDLgXgghhBAlj9oCGr4I40PgpVXg7PmEHTLblALf0iZuRZAkX0IIIYQo+tRqqN8XXjkIHd5+QmUFEm5ox4IVQZJ8CSGEEMJ8qNVQoWb+6j46k34RIcmXEEIIIcxLGZfCrWdkknwJIYQQwrxUa6V9qjFrcH0OKrCvpK1XBEnyJYQQQgjzorbQTicBuSRgme+7flJo830VNkm+hBBCCGF+PHtpp5Owd9Mvt3cvlGkmDMnS1AEIIYQQQjwVz15Qt4fBZrg3FEm+hBBCCGG+1Bbg0dbUURSIdDsKIYQQQhiRJF9CCCGEEEYkyZcQQgghhBFJ8iWEEEIIYUSSfAkhhBBCGJEkX0IIIYQQRiTJlxBCCCGEEUnyJYQQQghhRJJ8CSGEEEIYkcxwbwSKogCQkJBg6lCEEEIIkU9Zv7ezfo8XFkm+jODevXsAVKlSxdShCCGEEKKA7t27h4ODQ6EdT6UUdjonctBoNNy8eZOyZcuiUqkKtG9CQgJVqlTh+vXr2NvbGyxGcyHXIye5JvrkeuQk10SfXI+c5Jroy7oe165dQ6VS4e7ujlpdeCO1pOXLCNRqNZUrV36mY9jb28s3RDZyPXKSa6JPrkdOck30yfXISa6JPgcHB4NcDxlwL4QQQghhRJJ8CSGEEEIYkSRfRZyNjQ2zZs3CxsbG1KEUCXI9cpJrok+uR05yTfTJ9chJrok+Q18PGXAvhBBCCGFE0vIlhBBCCGFEknwJIYQQQhiRJF9CCCGEEEYkyZcQQgghhBFJ8lWELVmyhOrVq2Nra4uPjw9Hjx41dUhGM3v2bFQqld6rbt26uu3JyclMnDiRChUqUKZMGQICAoiOjjZpzIVp3759vPDCC7i7u6NSqdi0aZPedkVReO+993Bzc8POzg4/Pz8uXryoVyc2NpYhQ4Zgb2+Po6Mjo0aNIjEx0chnUniedE2GDx+e457p2rWrXp3idE3mzp1L8+bNKVu2LM7OzvTp04fw8HC9Ovn5Prl27Ro9evSgVKlSODs788Ybb5Cenm7ks3l2+bkeHTp0yHGPvPLKK3p1isv1APjmm29o1KiRbuJUX19fduzYodteku4P8nE9jHl/SPJVRK1bt45p06Yxa9YsTpw4QePGjfH39ycmJsbUoRlN/fr1iYyM1L0OHDig2/baa6+xdetW1q9fz969e7l58yb9+vUzabyF6f79+zRu3JglS5bkun3evHl8+eWXLF26lCNHjlC6dGn8/f1JTk7W1RkyZAjnzp0jKCiIbdu2sW/fPsaOHWvEsyhcT7omAF27dtW7Z37++We97cXpmuzdu5eJEydy+PBhgoKCSEtLo0uXLty/f19X50nfJxkZGfTo0YPU1FQOHTrE6tWrWbVqFe+9956Jzurp5ed6AIwZM0bvHpk3b55uW3G6HgCVK1fmk08+ITQ0lOPHj9OxY0d69+7NuXPnoITdH+TjemDM+0MRRVKLFi2UiRMn6t5nZGQo7u7uyty5c00al7HMmjVLady4ca7b4uLiFCsrK2X9+vW6svPnzyuAEhISYsQojQNQfv/9d917jUajuLq6Kp999pmuLC4uTrGxsVF+/vlnRVEUJSwsTAGUY8eO6ers2LFDUalUyo0bN4x8BoXv0WuiKIoybNgwpXfv3nnuU9yvSUxMjAIoe/fuVZR8fp9s375dUavVSlRUlK7ON998o9jb2yspKSkmOIvC8+j1UBRFad++vfLqq6/muU9xvh5ZypUrp6xYsaLE3x9Zsq6HYuT7Q1q+iqDU1FRCQ0Px8/PTlanVavz8/AgJCTFpbMZ08eJF3N3dqVGjBkOGDOHatWsAhIaGkpaWpnd96tatS9WqVUvE9YmIiCAqKkrv/B0cHPDx8dGdf0hICI6OjjRr1kxXx8/PD7VazZEjR0wStzHs2bMHZ2dn6tSpw/jx47lz545uW3G/JvHx8QCUL18e8vl9EhISQsOGDXFxcdHV8ff3JyEhQa81wBw9ej2yrFmzhooVK9KgQQNmzpxJUlKSbltxvh4ZGRn88ssv3L9/H19f3xJ/fzx6PbIY6/6QhbWLoNu3b5ORkaH3BQZwcXHhwoULJovLmHx8fFi1ahV16tQhMjKSOXPm0LZtW86ePUtUVBTW1tY4Ojrq7ePi4kJUVJTJYjaWrHPM7f7I2hYVFYWzs7PedktLS8qXL19sr1HXrl3p168fHh4eXL58mbfffptu3boREhKChYVFsb4mGo2GqVOn0rp1axo0aACZ98CTvk+ioqJyvY/Idp+Zo9yuB8DgwYOpVq0a7u7u/P3337z55puEh4fz22+/QTG9HmfOnMHX15fk5GTKlCnD77//jqenJ6dOnSqR90de1wMj3x+SfIkiqVu3brr/N2rUCB8fH6pVq8avv/6KnZ2dSWMTRdPAgQN1/2/YsCGNGjWiZs2a7Nmzh06dOpk0NkObOHEiZ8+e1RsXWZLldT2yj+9r2LAhbm5udOrUicuXL1OzZk0TRGp4derU4dSpU8THx7NhwwaGDRvG3r17TR2WyeR1PTw9PY16f0i3YxFUsWJFLCwscjx1Eh0djaurq8niMiVHR0eee+45Ll26hKurK6mpqcTFxenVKSnXJ+scH3d/uLq65ng4Iz09ndjY2BJxjQBq1KhBxYoVuXTpEhTjazJp0iS2bdvG7t27qVy5sq48P98nrq6uud5HZLvPzE1e1yM3Pj4+AHr3SHG7HtbW1tSqVQtvb2/mzp1L48aNWbRoUYm9P/K6Hrkx5P0hyVcRZG1tjbe3N8HBwboyjUZDcHCwXt90SZKYmMjly5dxc3PD29sbKysrvesTHh7OtWvXSsT18fDwwNXVVe/8ExISOHLkiO78fX19iYuLIzQ0VFdn165daDQa3Q+U4u6///7jzp07uLm5QTG8JoqiMGnSJH7//Xd27dqFh4eH3vb8fJ/4+vpy5swZvaQ0KCgIe3t7XVeMuXjS9cjNqVOnAPTukeJyPfKi0WhISUkpcfdHXrKuR24Men88w0MCwoB++eUXxcbGRlm1apUSFhamjB07VnF0dNR7yqI4mz59urJnzx4lIiJCOXjwoOLn56dUrFhRiYmJURRFUV555RWlatWqyq5du5Tjx48rvr6+iq+vr6nDLjT37t1TTp48qZw8eVIBlC+++EI5efKkcvXqVUVRFOWTTz5RHB0dlc2bNyt///230rt3b8XDw0N58OCB7hhdu3ZVmjRpohw5ckQ5cOCAUrt2bWXQoEEmPKtn87hrcu/ePeX1119XQkJClIiICGXnzp1K06ZNldq1ayvJycm6YxSnazJ+/HjFwcFB2bNnjxIZGal7JSUl6eo86fskPT1dadCggdKlSxfl1KlTSmBgoOLk5KTMnDnTRGf19J50PS5duqS8//77yvHjx5WIiAhl8+bNSo0aNZR27drpjlGcroeiKMpbb72l7N27V4mIiFD+/vtv5a233lJUKpXy119/KUoJuz+UJ1wPY98fknwVYV999ZVStWpVxdraWmnRooVy+PBhU4dkNAMGDFDc3NwUa2trpVKlSsqAAQOUS5cu6bY/ePBAmTBhglKuXDmlVKlSSt++fZXIyEiTxlyYdu/erQA5XsOGDVOUzOkm/ve//ykuLi6KjY2N0qlTJyU8PFzvGHfu3FEGDRqklClTRrG3t1dGjBih3Lt3z0Rn9Owed02SkpKULl26KE5OToqVlZVSrVo1ZcyYMTn+WClO1yS3awEoK1eu1NXJz/fJlStXlG7duil2dnZKxYoVlenTpytpaWkmOKNn86Trce3aNaVdu3ZK+fLlFRsbG6VWrVrKG2+8ocTHx+sdp7hcD0VRlJEjRyrVqlVTrK2tFScnJ6VTp066xEspYfeH8oTrYez7Q6Vob1ohhBBCCGEEMuZLCCGEEMKIJPkSQgghhDAiSb6EEEIIIYxIki8hhBBCCCOS5EsIIYQQwogk+RJCCCGEMCJJvoQQQgghjEiSLyGEEEIII5LkSwghirj//e9/jB07Vve+Q4cOTJ061ehx7NmzB5VKpVuMedWqVTg6Ouq2z549Gy8vL9374cOH06dPH6PFFxgYiJeXFxqNxmifKcTTkORLCDM0fPhwVCoVKpUKa2tratWqxfvvv096erqpQ3tqKpWKTZs2GfQzLl26xIgRI6hcuTI2NjZ4eHgwaNAgjh8/btDPfdSVK1dQqVS6hXsfJyoqikWLFvHOO+/oyn777Tc++OADA0eZU6tWrYiMjMTBwSFf9RctWsSqVasMHleWrl27YmVlxZo1a4z2mUI8DUm+hDBTXbt2JTIykosXLzJ9+nRmz57NZ5999lTHysjIKDatBWlpabmWHz9+HG9vb/755x++/fZbwsLC+P3336lbty7Tp083epz5tWLFClq1akW1atV0ZeXLl6ds2bJGj8Xa2hpXV1dUKlW+6js4OOi1jBnD8OHD+fLLL436mUIUlCRfQpgpGxsbXF1dqVatGuPHj8fPz48tW7YA8MUXX9CwYUNKly5NlSpVmDBhAomJibp9s7qLtmzZgqenJzY2Nly7do1jx47RuXNnKlasiIODA+3bt+fEiRN6n6tSqfj222/p2bMnpUqVol69eoSEhHDp0iU6dOhA6dKladWqFZcvX9bbb/PmzTRt2hRbW1tq1KjBnDlzdC111atXB6Bv376oVCrd+yftlxXPN998Q69evShdujQfffRRjmulKArDhw+ndu3a7N+/nx49elCzZk28vLyYNWsWmzdv1tU9c+YMHTt2xM7OjgoVKjB27Fi9a5dbl1+fPn0YPny47n316tX5+OOPGTlyJGXLlqVq1aosW7ZMt93DwwOAJk2aoFKp6NChQ55f519++YUXXnhBr+zRGJ70ebnp0KEDkydPZurUqZQrVw4XFxeWL1/O/fv3GTFiBGXLlqVWrVrs2LFDt8+j3Y5P8mi3Y0pKClOmTMHZ2RlbW1vatGnDsWPHchw/ODiYZs2aUapUKVq1akV4eLiuzunTp3n++ecpW7Ys9vb2eHt767VcvvDCCxw/fjzH/SdEUSLJlxDFhJ2dHampqQCo1Wq+/PJLzp07x+rVq9m1axczZszQq5+UlMSnn37KihUrOHfuHM7Ozty7d49hw4Zx4MABDh8+TO3atenevTv37t3T2/eDDz5g6NChnDp1irp16zJ48GDGjRvHzJkzOX78OIqiMGnSJF39/fv3M3ToUF599VXCwsL49ttvWbVqlS5RyvoFvHLlSiIjI3Xvn7RfltmzZ9O3b1/OnDnDyJEjc1ybU6dOce7cOaZPn45anfPHXlbrzP379/H396dcuXIcO3aM9evXs3PnTr1zya/58+fTrFkzTp48yYQJExg/frwuiTh69CgAO3fuJDIykt9++y3XY8TGxhIWFkazZs2e6fPysnr1aipWrMjRo0eZPHky48eP56WXXqJVq1acOHGCLl268H//938kJSUV+PxzM2PGDDZu3Mjq1as5ceIEtWrVwt/fn9jYWL1677zzDvPnz+f48eNYWlrqfU2HDBlC5cqVOXbsGKGhobz11ltYWVnptletWhUXl/9v795Cmn7DOIB/N52npSGmpZBFrsmsNAXNsBBTiA4kdIKyOSi9EDoZdSc1L8oK7HwUxGqIRoQXufBQKdSyUHOWOs008yIPmJKYk+b2/C/++vu7Wmv+q2XxfGAX7/t7n/f37LfBHt69+20uHj9+/FNyZuyXIMbYH0elUlFycjIREVksFqqsrCR3d3c6fPiwzfF37twhPz8/oV1QUEAASK/X2z2P2Wwmb29vunfvntAHgLKysoR2TU0NAaD8/Hyhr6ioiDw8PIR2YmIinThxwmpujUZDgYGBVvOWlJRYjXE07uDBg3afx+3btwkAvXjxwu64vLw88vX1pZGREaFPq9WSWCym3t5eIiKKj4+nAwcOWMUlJyeTSqUS2gsWLKBdu3YJbYvFQgEBAXT16lUiInr79i0BoIaGBrv5NDQ0EADq7u626v8yh++dz5b4+HhatWqV0B4fHyepVEpKpVLo6+npIQBUU1NDRERVVVUEgIaGhogm3kezZ88Wxh87dowiIiKE9tT36cjICEkkEiosLBSOf/78mYKCguj06dNW8z948EAYo9VqCQAZjUYiIvL29qYbN27YvW6RkZGkVqvtjmHsd3L93cUfY+z/KS0txaxZs2AymWCxWLBz506o1WpgYkUlJycHra2tGB4exvj4OMbGxjA6OgovLy9gYv9OeHi41Zx9fX3IyspCdXU1+vv7YTabMTo6iu7ubqtxU+Pmzp0LAFi2bJlV39jYGIaHh+Hj44PGxkbodDqrFSuz2fxVTl9yNO57K0P/1mjfZzAYEBERAalUKvTFxcXBYrGgra1NeK6OmHqNRCIR5s2bh/7+fofjAcBoNAIAPDw8fsn5psa4uLjAz8/vq9cRwLTztqWjowMmkwlxcXFCn0QiQUxMDAwGwzfzCgwMFHIIDg7GoUOHkJaWBo1Gg6SkJGzbtg0hISFW8Z6enj9ttY6xX4G/dmTsD5WQkAC9Xo/29nYYjUbcvHkTUqkUXV1d2LhxI8LDw3H37l3U19fj8uXLACB8LYmJD6gvN06rVCro9XqcP38eT58+hV6vh5+fn1UcJj40J03OYatvchP/yMgIsrOzodfrhcerV6/Q3t5ut7BwNG5qsWSLXC4HALS2ttod5wixWPxVMWdrk//U64GJazLdHzXMmTMHADA0NPTdsf/nfLZi7L2OzmIvB7VajebmZmzYsAGPHj1CWFgYSkpKrOIHBwfh7+/v1JwZmw4uvhj7Q0mlUshkMgQHB8PV9b9F7Pr6elgsFuTm5iI2NhZyuRzv3793aE6dTof9+/dj/fr1WLJkCdzd3TEwMPDDuUZFRaGtrQ0ymeyrx+QeLIlEArPZPO04RyxfvhxhYWHIzc21WUhMbiBXKBRobGzEp0+fhGM6nQ5isRihoaEAAH9/f/T09AjHzWYzmpqapnU93NzchFh7QkJC4OPjg5aWlmnNPxOFhITAzc0NOp1O6DOZTKitrUVYWNi05pLL5cjMzERFRQU2b96MgoIC4djY2Bg6OjoQGRn5U/Nn7Gfi4ouxv4xMJoPJZMLFixfR2dkJjUaDa9euORS7ePFiaDQaGAwGPH/+HCkpKfD09PzhnI4ePYpbt24hOzsbzc3NMBgMKC4uRlZWljBm4cKFePjwIXp7e4WVHkfiHCESiVBQUIDXr19j9erVuH//Pjo7O/Hy5UscP34cycnJwMRmbg8PD6hUKjQ1NaGqqgr79u2DUqkUvoJbs2YNtFottFotWltbkZGR4fCv/yYFBATA09MTZWVl6Ovrw8ePH22OE4vFSEpKwpMnT6Y1/0wklUqRkZGBI0eOoKysDC0tLUhPT8fo6Cj27Nnj0BxGoxF79+5FdXU13r17B51Oh9raWigUCmHMs2fP4O7ujpUrV/7CZ8PYj+Hii7G/TEREBM6cOYNTp05h6dKlKCwsRE5OjkOx+fn5GBoaQlRUFJRKpXBbgB+1du1alJaWoqKiAtHR0YiNjcXZs2et7l2Vm5uLyspKzJ8/X1i1cCTOUTExMairq4NMJkN6ejoUCgU2bdqE5uZmnDt3DgDg5eWF8vJyDA4OIjo6Glu3bkViYiIuXbokzLN7926oVCqkpqYiPj4eixYtQkJCwrRycXV1xYULF3D9+nUEBQUJxZ8taWlpKC4u/ivuw3by5Els2bIFSqUSUVFRePPmDcrLy+Hr6+tQvIuLCz58+IDU1FTI5XJs374d69atQ3Z2tjCmqKgIKSkp39xHyNhMICJHd6IyxhhzOiLCihUrkJmZiR07dvzudGa0gYEBhIaGoq6uTriXGmMzEa98McbYDCYSiZCXl/dH/3WUs3R1deHKlStceLEZj1e+GGOMMcaciFe+GGOMMcaciIsvxhhjjDEn4uKLMcYYY8yJuPhijDHGGHMiLr4YY4wxxpyIiy/GGGOMMSfi4osxxhhjzIm4+GKMMcYYcyIuvhhjjDHGnOgfPWMbzfJaD7MAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Cringe hardcoding\n", + "plt.plot([6, 11, 19, 35, 67], np.array(df[\"val_loss\"])[1::2] - np.array(df[\"val_loss\"])[::2], linewidth=2, marker=\"o\", label=\"Scratch\")\n", + "plt.plot([67, 110, 335], np.log(transferred_ppl[1::2]) - np.log(transferred_ppl[::2]), linewidth=2, marker=\"o\", label=\"Transferred\")\n", + "plt.axhline(y=0, color='gray', linestyle='--')\n", + "plt.xlabel(\"Parameter Count (in millions)\")\n", + "plt.ylabel(\"RTL Loss - LTR Loss\")\n", + "plt.title(\"RTL and LTR Loss Difference Across Model Size\")\n", + "plt.legend()" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "execution_state": "idle", + "id": "b492b7f4-7e93-444b-b5ba-1bdb5b6403f2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Nameval_loss
0distilbert_base_japan_rtl2.832614
1distilbert_base_japan_ltr2.823765
2bert_6M_rtl_scratch4.744476
3bert_6_ltr_scratch4.761365
4bert_11_rtl_scratch4.446950
5bert_11_ltr_scratch4.462379
6bert_19_rtl_scratch4.177320
7bert_19_ltr_scratch4.186271
8bert_35_rtl_scratch3.927857
9bert_35_ltr_scratch3.941595
10qa_distilbert_base_ltr_v23.150267
11qa_distilbert_base_rtl_v23.190452
12qa_ltr_distilbert_base3.325950
13distilbert_base_ltr_scratch3.686307
14distilbert_base_rtl_scratch3.688566
15deep-monkey-113.009245
16distilbert_base_ltr_4epoch3.196100
17distilbert_base_rtl_4epoch3.193662
18bert_base_ltr_4epoch3.082236
19bert_base_rtl_4epoch3.088110
\n", + "
" + ], + "text/plain": [ + " Name val_loss\n", + "0 distilbert_base_japan_rtl 2.832614\n", + "1 distilbert_base_japan_ltr 2.823765\n", + "2 bert_6M_rtl_scratch 4.744476\n", + "3 bert_6_ltr_scratch 4.761365\n", + "4 bert_11_rtl_scratch 4.446950\n", + "5 bert_11_ltr_scratch 4.462379\n", + "6 bert_19_rtl_scratch 4.177320\n", + "7 bert_19_ltr_scratch 4.186271\n", + "8 bert_35_rtl_scratch 3.927857\n", + "9 bert_35_ltr_scratch 3.941595\n", + "10 qa_distilbert_base_ltr_v2 3.150267\n", + "11 qa_distilbert_base_rtl_v2 3.190452\n", + "12 qa_ltr_distilbert_base 3.325950\n", + "13 distilbert_base_ltr_scratch 3.686307\n", + "14 distilbert_base_rtl_scratch 3.688566\n", + "15 deep-monkey-11 3.009245\n", + "16 distilbert_base_ltr_4epoch 3.196100\n", + "17 distilbert_base_rtl_4epoch 3.193662\n", + "18 bert_base_ltr_4epoch 3.082236\n", + "19 bert_base_rtl_4epoch 3.088110" + ] + }, + "execution_count": 61, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + " pd.read_csv(csv_filename)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fda6f102-904a-43e1-b078-258d39f1738f", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/addition.ipynb b/notebooks/addition.ipynb new file mode 100644 index 0000000..01fddff --- /dev/null +++ b/notebooks/addition.ipynb @@ -0,0 +1,836 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "execution_state": "idle", + "id": "ecaeb29e-fbbe-4876-86ad-9fbadea989b0", + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "import torch\n", + "import torch.nn as nn\n", + "\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "assert device.type == \"cuda\", \"CUDA is not available. Please check your GPU setup.\"" + ] + }, + { + "cell_type": "code", + "execution_count": 110, + "execution_state": "idle", + "id": "84a82827-8947-4a26-a485-56f5b1eadb4c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([[4, 9, 4, 1, 8, 2],\n", + " [0, 6, 5, 9, 1, 4],\n", + " [4, 9, 1, 0, 5, 5],\n", + " [5, 2, 4, 9, 1, 8],\n", + " [2, 7, 6, 8, 0, 1]], device='cuda:0'),\n", + " tensor([[5, 7, 7],\n", + " [9, 7, 9],\n", + " [4, 4, 7],\n", + " [4, 4, 2],\n", + " [0, 8, 7]], device='cuda:0'))" + ] + }, + "execution_count": 110, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "NUM_LEN = 3\n", + "\n", + "def pad(a):\n", + " s = str(a)\n", + " if len(s) > NUM_LEN:\n", + " return s[-NUM_LEN:]\n", + " return s.zfill(NUM_LEN)\n", + "\n", + "def mkbatch_ltr(size):\n", + " data = []\n", + " labels = []\n", + " for i in range(size):\n", + " a = random.randrange(0, 10**NUM_LEN)\n", + " b = random.randrange(0, 10**NUM_LEN)\n", + " c = a + b\n", + " data.append(list(map(int, pad(a) + pad(b))))\n", + " labels.append(list(map(int, pad(c))))\n", + " return torch.tensor(data, device=device), torch.tensor(labels, device=device)\n", + "\n", + "def mkbatch_rtl(size):\n", + " data, labels = mkbatch_ltr(size)\n", + " return torch.flip(data, (1,)), torch.flip(labels, (1,))\n", + "\n", + "mkbatch_rtl(5)" + ] + }, + { + "cell_type": "code", + "execution_count": 111, + "execution_state": "idle", + "id": "d50dce44-57b7-4d4d-895a-c2275c04234c", + "metadata": {}, + "outputs": [], + "source": [ + "class TransformerModel(nn.Module):\n", + " def __init__(self, input_dim, model_dim, output_dim, nheads, nenclayers, ndeclayers):\n", + " super().__init__()\n", + " self.emb = nn.Embedding(input_dim, model_dim - 1)\n", + " self.trans = nn.Transformer(d_model=model_dim, nhead=nheads, dim_feedforward=4 * model_dim,\n", + " num_encoder_layers=nenclayers, num_decoder_layers=ndeclayers,\n", + " dropout=0, batch_first=True)\n", + " self.output = nn.Linear(model_dim, output_dim)\n", + "\n", + " def forward(self, data, labels):\n", + " bsz = data.size(0)\n", + " data_pos = (torch.arange(2 * NUM_LEN, device=device) % NUM_LEN).expand(bsz, -1)\n", + " labels_pos = (torch.arange(NUM_LEN, device=device)).expand(bsz, -1)\n", + " data_emb = torch.cat((self.emb(data), data_pos.unsqueeze(2)), 2)\n", + " labels_emb = torch.cat((self.emb(labels), labels_pos.unsqueeze(2)), 2)\n", + " return self.output(self.trans(data_emb, labels_emb, tgt_mask=TGT_MASK, tgt_is_causal=True))" + ] + }, + { + "cell_type": "code", + "execution_count": 118, + "execution_state": "idle", + "id": "ddad4059-b06e-4eb3-a55a-5a4a842cdd7a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training data: 32768K\n", + "Trainable parameters in the model: 1251\n" + ] + } + ], + "source": [ + "MODEL_DIM = 4 # Dimension of model\n", + "VOCAB_SIZE = 10\n", + "NEPOCHS = 1000\n", + "BSZ = 2**15 # Batch size\n", + "NHEADS = 1\n", + "NENCLAYERS = 2\n", + "NDECLAYERS = 2\n", + "\n", + "LR = 1e-2\n", + "\n", + "TGT_MASK = nn.Transformer.generate_square_subsequent_mask(NUM_LEN)\n", + "model = TransformerModel(VOCAB_SIZE + 1, MODEL_DIM, VOCAB_SIZE, NHEADS, NENCLAYERS, NDECLAYERS).to(device)\n", + "\n", + "criterion = nn.CrossEntropyLoss()\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=LR)\n", + "\n", + "train_err = []\n", + "open('loss', 'w').close()\n", + "\n", + "trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "print(f\"Training data: {NEPOCHS*BSZ//10**3}K\")\n", + "print(f\"Trainable parameters in the model: {trainable_params}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 119, + "execution_state": "idle", + "id": "689f2e44-da84-43ea-b539-414d6f5c37e3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 0/1000 \t Train Err: 2.4793\n", + "Epoch 1/1000 \t Train Err: 2.4310\n", + "Epoch 2/1000 \t Train Err: 2.3800\n", + "Epoch 3/1000 \t Train Err: 2.3493\n", + "Epoch 4/1000 \t Train Err: 2.3288\n", + "Epoch 5/1000 \t Train Err: 2.3202\n", + "Epoch 6/1000 \t Train Err: 2.3171\n", + "Epoch 7/1000 \t Train Err: 2.3139\n", + "Epoch 8/1000 \t Train Err: 2.3095\n", + "Epoch 9/1000 \t Train Err: 2.3064\n", + "Epoch 10/1000 \t Train Err: 2.3040\n", + "Epoch 11/1000 \t Train Err: 2.3029\n", + "Epoch 12/1000 \t Train Err: 2.3030\n", + "Epoch 13/1000 \t Train Err: 2.3037\n", + "Epoch 14/1000 \t Train Err: 2.3047\n", + "Epoch 15/1000 \t Train Err: 2.3060\n", + "Epoch 16/1000 \t Train Err: 2.3067\n", + "Epoch 17/1000 \t Train Err: 2.3067\n", + "Epoch 18/1000 \t Train Err: 2.3068\n", + "Epoch 19/1000 \t Train Err: 2.3059\n", + "Epoch 20/1000 \t Train Err: 2.3060\n", + "Epoch 21/1000 \t Train Err: 2.3052\n", + "Epoch 22/1000 \t Train Err: 2.3044\n", + "Epoch 23/1000 \t Train Err: 2.3039\n", + "Epoch 24/1000 \t Train Err: 2.3039\n", + "Epoch 25/1000 \t Train Err: 2.3033\n", + "Epoch 26/1000 \t Train Err: 2.3032\n", + "Epoch 27/1000 \t Train Err: 2.3032\n", + "Epoch 28/1000 \t Train Err: 2.3032\n", + "Epoch 29/1000 \t Train Err: 2.3029\n", + "Epoch 30/1000 \t Train Err: 2.3028\n", + "Epoch 31/1000 \t Train Err: 2.3032\n", + "Epoch 32/1000 \t Train Err: 2.3031\n", + "Epoch 33/1000 \t Train Err: 2.3030\n", + "Epoch 34/1000 \t Train Err: 2.3031\n", + "Epoch 35/1000 \t Train Err: 2.3031\n", + "Epoch 36/1000 \t Train Err: 2.3031\n", + "Epoch 37/1000 \t Train Err: 2.3029\n", + "Epoch 38/1000 \t Train Err: 2.3033\n", + "Epoch 39/1000 \t Train Err: 2.3032\n", + "Epoch 40/1000 \t Train Err: 2.3031\n", + "Epoch 41/1000 \t Train Err: 2.3030\n", + "Epoch 42/1000 \t Train Err: 2.3027\n", + "Epoch 43/1000 \t Train Err: 2.3027\n", + "Epoch 44/1000 \t Train Err: 2.3027\n", + "Epoch 45/1000 \t Train Err: 2.3027\n", + "Epoch 46/1000 \t Train Err: 2.3026\n", + "Epoch 47/1000 \t Train Err: 2.3026\n", + "Epoch 48/1000 \t Train Err: 2.3027\n", + "Epoch 49/1000 \t Train Err: 2.3026\n", + "Epoch 50/1000 \t Train Err: 2.3027\n", + "Epoch 51/1000 \t Train Err: 2.3027\n", + "Epoch 52/1000 \t Train Err: 2.3027\n", + "Epoch 53/1000 \t Train Err: 2.3026\n", + "Epoch 54/1000 \t Train Err: 2.3028\n", + "Epoch 55/1000 \t Train Err: 2.3027\n", + "Epoch 56/1000 \t Train Err: 2.3026\n", + "Epoch 57/1000 \t Train Err: 2.3027\n", + "Epoch 58/1000 \t Train Err: 2.3026\n", + "Epoch 59/1000 \t Train Err: 2.3026\n", + "Epoch 60/1000 \t Train Err: 2.3027\n", + "Epoch 61/1000 \t Train Err: 2.3026\n", + "Epoch 62/1000 \t Train Err: 2.3026\n", + "Epoch 500/1000 \t Train Err: 2.3026\n", + "Epoch 501/1000 \t Train Err: 2.3025\n", + "Epoch 502/1000 \t Train Err: 2.3026\n", + "Epoch 503/1000 \t Train Err: 2.3026\n", + "Epoch 504/1000 \t Train Err: 2.3026\n", + "Epoch 505/1000 \t Train Err: 2.3026\n", + "Epoch 506/1000 \t Train Err: 2.3026\n", + "Epoch 507/1000 \t Train Err: 2.3026\n", + "Epoch 508/1000 \t Train Err: 2.3026\n", + "Epoch 509/1000 \t Train Err: 2.3026\n", + "Epoch 510/1000 \t Train Err: 2.3026\n", + "Epoch 511/1000 \t Train Err: 2.3026\n", + "Epoch 512/1000 \t Train Err: 2.3026\n", + "Epoch 513/1000 \t Train Err: 2.3026\n", + "Epoch 514/1000 \t Train Err: 2.3025\n", + "Epoch 515/1000 \t Train Err: 2.3027\n", + "Epoch 516/1000 \t Train Err: 2.3027\n", + "Epoch 517/1000 \t Train Err: 2.3027\n", + "Epoch 518/1000 \t Train Err: 2.3026\n", + "Epoch 519/1000 \t Train Err: 2.3026\n", + "Epoch 520/1000 \t Train Err: 2.3026\n", + "Epoch 521/1000 \t Train Err: 2.3025\n", + "Epoch 522/1000 \t Train Err: 2.3027\n", + "Epoch 523/1000 \t Train Err: 2.3027\n", + "Epoch 524/1000 \t Train Err: 2.3027\n", + "Epoch 525/1000 \t Train Err: 2.3026\n", + "Epoch 526/1000 \t Train Err: 2.3026\n", + "Epoch 527/1000 \t Train Err: 2.3026\n", + "Epoch 528/1000 \t Train Err: 2.3027\n", + "Epoch 529/1000 \t Train Err: 2.3028\n", + "Epoch 530/1000 \t Train Err: 2.3026\n", + "Epoch 531/1000 \t Train Err: 2.3026\n", + "Epoch 532/1000 \t Train Err: 2.3026\n", + "Epoch 533/1000 \t Train Err: 2.3026\n", + "Epoch 534/1000 \t Train Err: 2.3026\n", + "Epoch 535/1000 \t Train Err: 2.3026\n", + "Epoch 536/1000 \t Train Err: 2.3027\n", + "Epoch 537/1000 \t Train Err: 2.3027\n", + "Epoch 538/1000 \t Train Err: 2.3025\n", + "Epoch 539/1000 \t Train Err: 2.3026\n", + "Epoch 540/1000 \t Train Err: 2.3026\n", + "Epoch 541/1000 \t Train Err: 2.3026\n", + "Epoch 542/1000 \t Train Err: 2.3026\n", + "Epoch 543/1000 \t Train Err: 2.3026\n", + "Epoch 544/1000 \t Train Err: 2.3026\n", + "Epoch 545/1000 \t Train Err: 2.3026\n", + "Epoch 546/1000 \t Train Err: 2.3027\n", + "Epoch 547/1000 \t Train Err: 2.3026\n", + "Epoch 548/1000 \t Train Err: 2.3026\n", + "Epoch 549/1000 \t Train Err: 2.3026\n", + "Epoch 550/1000 \t Train Err: 2.3026\n", + "Epoch 551/1000 \t Train Err: 2.3026\n", + "Epoch 552/1000 \t Train Err: 2.3028\n", + "Epoch 553/1000 \t Train Err: 2.3028\n", + "Epoch 554/1000 \t Train Err: 2.3027\n", + "Epoch 555/1000 \t Train Err: 2.3026\n", + "Epoch 556/1000 \t Train Err: 2.3027\n", + "Epoch 557/1000 \t Train Err: 2.3027\n", + "Epoch 558/1000 \t Train Err: 2.3028\n", + "Epoch 559/1000 \t Train Err: 2.3026\n", + "Epoch 560/1000 \t Train Err: 2.3026\n", + "Epoch 561/1000 \t Train Err: 2.3026\n", + "Epoch 562/1000 \t Train Err: 2.3027\n", + "Epoch 563/1000 \t Train Err: 2.3027\n", + "Epoch 564/1000 \t Train Err: 2.3027\n", + "Epoch 565/1000 \t Train Err: 2.3025\n", + "Epoch 566/1000 \t Train Err: 2.3026\n", + "Epoch 567/1000 \t Train Err: 2.3026\n", + "Epoch 568/1000 \t Train Err: 2.3026\n", + "Epoch 569/1000 \t Train Err: 2.3026\n", + "Epoch 570/1000 \t Train Err: 2.3026\n", + "Epoch 571/1000 \t Train Err: 2.3026\n", + "Epoch 572/1000 \t Train Err: 2.3026\n", + "Epoch 573/1000 \t Train Err: 2.3026\n", + "Epoch 574/1000 \t Train Err: 2.3026\n", + "Epoch 575/1000 \t Train Err: 2.3028\n", + "Epoch 576/1000 \t Train Err: 2.3026\n", + "Epoch 577/1000 \t Train Err: 2.3026\n", + "Epoch 578/1000 \t Train Err: 2.3025\n", + "Epoch 579/1000 \t Train Err: 2.3026\n", + "Epoch 580/1000 \t Train Err: 2.3026\n", + "Epoch 581/1000 \t Train Err: 2.3027\n", + "Epoch 582/1000 \t Train Err: 2.3026\n", + "Epoch 583/1000 \t Train Err: 2.3027\n", + "Epoch 584/1000 \t Train Err: 2.3027\n", + "Epoch 585/1000 \t Train Err: 2.3026\n", + "Epoch 586/1000 \t Train Err: 2.3026\n", + "Epoch 587/1000 \t Train Err: 2.3026\n", + "Epoch 588/1000 \t Train Err: 2.3026\n", + "Epoch 589/1000 \t Train Err: 2.3027\n", + "Epoch 590/1000 \t Train Err: 2.3026\n", + "Epoch 591/1000 \t Train Err: 2.3026\n", + "Epoch 592/1000 \t Train Err: 2.3026\n", + "Epoch 593/1000 \t Train Err: 2.3026\n", + "Epoch 594/1000 \t Train Err: 2.3026\n", + "Epoch 595/1000 \t Train Err: 2.3026\n", + "Epoch 596/1000 \t Train Err: 2.3026\n", + "Epoch 597/1000 \t Train Err: 2.3027\n", + "Epoch 598/1000 \t Train Err: 2.3026\n", + "Epoch 599/1000 \t Train Err: 2.3027\n", + "Epoch 600/1000 \t Train Err: 2.3027\n", + "Epoch 601/1000 \t Train Err: 2.3026\n", + "Epoch 602/1000 \t Train Err: 2.3026\n", + "Epoch 603/1000 \t Train Err: 2.3026\n", + "Epoch 604/1000 \t Train Err: 2.3026\n", + "Epoch 605/1000 \t Train Err: 2.3026\n", + "Epoch 606/1000 \t Train Err: 2.3027\n", + "Epoch 607/1000 \t Train Err: 2.3026\n", + "Epoch 608/1000 \t Train Err: 2.3026\n", + "Epoch 609/1000 \t Train Err: 2.3026\n", + "Epoch 610/1000 \t Train Err: 2.3026\n", + "Epoch 611/1000 \t Train Err: 2.3026\n", + "Epoch 612/1000 \t Train Err: 2.3027\n", + "Epoch 613/1000 \t Train Err: 2.3025\n", + "Epoch 614/1000 \t Train Err: 2.3026\n", + "Epoch 615/1000 \t Train Err: 2.3026\n", + "Epoch 616/1000 \t Train Err: 2.3026\n", + "Epoch 617/1000 \t Train Err: 2.3026\n", + "Epoch 618/1000 \t Train Err: 2.3026\n", + "Epoch 619/1000 \t Train Err: 2.3026\n", + "Epoch 620/1000 \t Train Err: 2.3026\n", + "Epoch 621/1000 \t Train Err: 2.3026\n", + "Epoch 622/1000 \t Train Err: 2.3026\n", + "Epoch 623/1000 \t Train Err: 2.3026\n", + "Epoch 624/1000 \t Train Err: 2.3026\n", + "Epoch 625/1000 \t Train Err: 2.3026\n", + "Epoch 626/1000 \t Train Err: 2.3026\n", + "Epoch 627/1000 \t Train Err: 2.3026\n", + "Epoch 628/1000 \t Train Err: 2.3026\n", + "Epoch 629/1000 \t Train Err: 2.3026\n", + "Epoch 630/1000 \t Train Err: 2.3027\n", + "Epoch 631/1000 \t Train Err: 2.3026\n", + "Epoch 632/1000 \t Train Err: 2.3026\n", + "Epoch 633/1000 \t Train Err: 2.3025\n", + "Epoch 634/1000 \t Train Err: 2.3026\n", + "Epoch 635/1000 \t Train Err: 2.3026\n", + "Epoch 636/1000 \t Train Err: 2.3026\n", + "Epoch 637/1000 \t Train Err: 2.3026\n", + "Epoch 638/1000 \t Train Err: 2.3026\n", + "Epoch 639/1000 \t Train Err: 2.3027\n", + "Epoch 640/1000 \t Train Err: 2.3026\n", + "Epoch 641/1000 \t Train Err: 2.3026\n", + "Epoch 642/1000 \t Train Err: 2.3026\n", + "Epoch 643/1000 \t Train Err: 2.3026\n", + "Epoch 644/1000 \t Train Err: 2.3027\n", + "Epoch 645/1000 \t Train Err: 2.3026\n", + "Epoch 646/1000 \t Train Err: 2.3026\n", + "Epoch 647/1000 \t Train Err: 2.3025\n", + "Epoch 648/1000 \t Train Err: 2.3026\n", + "Epoch 649/1000 \t Train Err: 2.3026\n", + "Epoch 650/1000 \t Train Err: 2.3025\n", + "Epoch 651/1000 \t Train Err: 2.3026\n", + "Epoch 652/1000 \t Train Err: 2.3025\n", + "Epoch 653/1000 \t Train Err: 2.3026\n", + "Epoch 654/1000 \t Train Err: 2.3026\n", + "Epoch 655/1000 \t Train Err: 2.3026\n", + "Epoch 656/1000 \t Train Err: 2.3026\n", + "Epoch 657/1000 \t Train Err: 2.3025\n", + "Epoch 658/1000 \t Train Err: 2.3026\n", + "Epoch 659/1000 \t Train Err: 2.3025\n", + "Epoch 660/1000 \t Train Err: 2.3025\n", + "Epoch 661/1000 \t Train Err: 2.3025\n", + "Epoch 662/1000 \t Train Err: 2.3026\n", + "Epoch 663/1000 \t Train Err: 2.3026\n", + "Epoch 664/1000 \t Train Err: 2.3025\n", + "Epoch 665/1000 \t Train Err: 2.3026\n", + "Epoch 666/1000 \t Train Err: 2.3026\n", + "Epoch 667/1000 \t Train Err: 2.3025\n", + "Epoch 668/1000 \t Train Err: 2.3026\n", + "Epoch 669/1000 \t Train Err: 2.3026\n", + "Epoch 670/1000 \t Train Err: 2.3025\n", + "Epoch 671/1000 \t Train Err: 2.3026\n", + "Epoch 672/1000 \t Train Err: 2.3025\n", + "Epoch 673/1000 \t Train Err: 2.3024\n", + "Epoch 674/1000 \t Train Err: 2.3024\n", + "Epoch 675/1000 \t Train Err: 2.3024\n", + "Epoch 676/1000 \t Train Err: 2.3024\n", + "Epoch 677/1000 \t Train Err: 2.3023\n", + "Epoch 678/1000 \t Train Err: 2.3024\n", + "Epoch 679/1000 \t Train Err: 2.3022\n", + "Epoch 680/1000 \t Train Err: 2.3022\n", + "Epoch 681/1000 \t Train Err: 2.3022\n", + "Epoch 682/1000 \t Train Err: 2.3020\n", + "Epoch 683/1000 \t Train Err: 2.3018\n", + "Epoch 684/1000 \t Train Err: 2.3016\n", + "Epoch 685/1000 \t Train Err: 2.3014\n", + "Epoch 686/1000 \t Train Err: 2.3011\n", + "Epoch 687/1000 \t Train Err: 2.3007\n", + "Epoch 688/1000 \t Train Err: 2.3007\n", + "Epoch 689/1000 \t Train Err: 2.2999\n", + "Epoch 690/1000 \t Train Err: 2.2999\n", + "Epoch 691/1000 \t Train Err: 2.2993\n", + "Epoch 692/1000 \t Train Err: 2.2993\n", + "Epoch 693/1000 \t Train Err: 2.2988\n", + "Epoch 694/1000 \t Train Err: 2.2987\n", + "Epoch 695/1000 \t Train Err: 2.2983\n", + "Epoch 696/1000 \t Train Err: 2.2976\n", + "Epoch 697/1000 \t Train Err: 2.2974\n", + "Epoch 698/1000 \t Train Err: 2.2969\n", + "Epoch 699/1000 \t Train Err: 2.2975\n", + "Epoch 700/1000 \t Train Err: 2.2955\n", + "Epoch 701/1000 \t Train Err: 2.2967\n", + "Epoch 702/1000 \t Train Err: 2.2958\n", + "Epoch 703/1000 \t Train Err: 2.2933\n", + "Epoch 704/1000 \t Train Err: 2.2951\n", + "Epoch 705/1000 \t Train Err: 2.2939\n", + "Epoch 706/1000 \t Train Err: 2.2922\n", + "Epoch 707/1000 \t Train Err: 2.2919\n", + "Epoch 708/1000 \t Train Err: 2.2901\n", + "Epoch 709/1000 \t Train Err: 2.2897\n", + "Epoch 710/1000 \t Train Err: 2.2867\n", + "Epoch 711/1000 \t Train Err: 2.2855\n", + "Epoch 712/1000 \t Train Err: 2.2841\n", + "Epoch 713/1000 \t Train Err: 2.2844\n", + "Epoch 714/1000 \t Train Err: 2.2812\n", + "Epoch 715/1000 \t Train Err: 2.2801\n", + "Epoch 716/1000 \t Train Err: 2.2789\n", + "Epoch 717/1000 \t Train Err: 2.2761\n", + "Epoch 718/1000 \t Train Err: 2.2797\n", + "Epoch 719/1000 \t Train Err: 2.2796\n", + "Epoch 720/1000 \t Train Err: 2.2974\n", + "Epoch 721/1000 \t Train Err: 2.2786\n", + "Epoch 722/1000 \t Train Err: 2.2802\n", + "Epoch 723/1000 \t Train Err: 2.2805\n", + "Epoch 724/1000 \t Train Err: 2.2812\n", + "Epoch 725/1000 \t Train Err: 2.2812\n", + "Epoch 726/1000 \t Train Err: 2.2792\n", + "Epoch 727/1000 \t Train Err: 2.2780\n", + "Epoch 728/1000 \t Train Err: 2.2775\n", + "Epoch 729/1000 \t Train Err: 2.2750\n", + "Epoch 730/1000 \t Train Err: 2.2821\n", + "Epoch 731/1000 \t Train Err: 2.2815\n", + "Epoch 732/1000 \t Train Err: 2.2812\n", + "Epoch 733/1000 \t Train Err: 2.2779\n", + "Epoch 734/1000 \t Train Err: 2.2777\n", + "Epoch 735/1000 \t Train Err: 2.2799\n", + "Epoch 736/1000 \t Train Err: 2.2754\n", + "Epoch 737/1000 \t Train Err: 2.2742\n", + "Epoch 738/1000 \t Train Err: 2.2723\n", + "Epoch 739/1000 \t Train Err: 2.2719\n", + "Epoch 740/1000 \t Train Err: 2.2674\n", + "Epoch 741/1000 \t Train Err: 2.2694\n", + "Epoch 742/1000 \t Train Err: 2.2702\n", + "Epoch 743/1000 \t Train Err: 2.2693\n", + "Epoch 744/1000 \t Train Err: 2.2722\n", + "Epoch 745/1000 \t Train Err: 2.2704\n", + "Epoch 746/1000 \t Train Err: 2.2675\n", + "Epoch 747/1000 \t Train Err: 2.2644\n", + "Epoch 748/1000 \t Train Err: 2.2599\n", + "Epoch 749/1000 \t Train Err: 2.2583\n", + "Epoch 750/1000 \t Train Err: 2.2578\n", + "Epoch 751/1000 \t Train Err: 2.2507\n", + "Epoch 752/1000 \t Train Err: 2.2490\n", + "Epoch 753/1000 \t Train Err: 2.2501\n", + "Epoch 754/1000 \t Train Err: 2.2502\n", + "Epoch 755/1000 \t Train Err: 2.2520\n", + "Epoch 756/1000 \t Train Err: 2.2435\n", + "Epoch 757/1000 \t Train Err: 2.2432\n", + "Epoch 758/1000 \t Train Err: 2.2420\n", + "Epoch 759/1000 \t Train Err: 2.2393\n", + "Epoch 760/1000 \t Train Err: 2.2372\n", + "Epoch 761/1000 \t Train Err: 2.2302\n", + "Epoch 762/1000 \t Train Err: 2.2302\n", + "Epoch 763/1000 \t Train Err: 2.2294\n", + "Epoch 764/1000 \t Train Err: 2.2201\n", + "Epoch 765/1000 \t Train Err: 2.2195\n", + "Epoch 766/1000 \t Train Err: 2.2166\n", + "Epoch 767/1000 \t Train Err: 2.2139\n", + "Epoch 768/1000 \t Train Err: 2.2096\n", + "Epoch 769/1000 \t Train Err: 2.2100\n", + "Epoch 770/1000 \t Train Err: 2.2073\n", + "Epoch 771/1000 \t Train Err: 2.2058\n", + "Epoch 772/1000 \t Train Err: 2.2096\n", + "Epoch 773/1000 \t Train Err: 2.2055\n", + "Epoch 774/1000 \t Train Err: 2.2213\n", + "Epoch 775/1000 \t Train Err: 2.2435\n", + "Epoch 776/1000 \t Train Err: 2.2282\n", + "Epoch 777/1000 \t Train Err: 2.2328\n", + "Epoch 778/1000 \t Train Err: 2.2254\n", + "Epoch 779/1000 \t Train Err: 2.2246\n", + "Epoch 780/1000 \t Train Err: 2.2241\n", + "Epoch 781/1000 \t Train Err: 2.2217\n", + "Epoch 782/1000 \t Train Err: 2.2156\n", + "Epoch 783/1000 \t Train Err: 2.2219\n", + "Epoch 784/1000 \t Train Err: 2.2151\n", + "Epoch 785/1000 \t Train Err: 2.2259\n", + "Epoch 786/1000 \t Train Err: 2.2226\n", + "Epoch 787/1000 \t Train Err: 2.2176\n", + "Epoch 788/1000 \t Train Err: 2.2152\n", + "Epoch 789/1000 \t Train Err: 2.2099\n", + "Epoch 790/1000 \t Train Err: 2.2069\n", + "Epoch 791/1000 \t Train Err: 2.2034\n", + "Epoch 792/1000 \t Train Err: 2.2080\n", + "Epoch 793/1000 \t Train Err: 2.1999\n", + "Epoch 794/1000 \t Train Err: 2.1925\n", + "Epoch 795/1000 \t Train Err: 2.1840\n", + "Epoch 796/1000 \t Train Err: 2.1820\n", + "Epoch 797/1000 \t Train Err: 2.1907\n", + "Epoch 798/1000 \t Train Err: 2.1835\n", + "Epoch 799/1000 \t Train Err: 2.1886\n", + "Epoch 800/1000 \t Train Err: 2.1807\n", + "Epoch 801/1000 \t Train Err: 2.1841\n", + "Epoch 802/1000 \t Train Err: 2.1776\n", + "Epoch 803/1000 \t Train Err: 2.1800\n", + "Epoch 804/1000 \t Train Err: 2.1715\n", + "Epoch 805/1000 \t Train Err: 2.1717\n", + "Epoch 806/1000 \t Train Err: 2.1701\n", + "Epoch 807/1000 \t Train Err: 2.1635\n", + "Epoch 808/1000 \t Train Err: 2.1664\n", + "Epoch 809/1000 \t Train Err: 2.1603\n", + "Epoch 810/1000 \t Train Err: 2.1636\n", + "Epoch 811/1000 \t Train Err: 2.1575\n", + "Epoch 812/1000 \t Train Err: 2.1587\n", + "Epoch 813/1000 \t Train Err: 2.1559\n", + "Epoch 814/1000 \t Train Err: 2.1540\n", + "Epoch 815/1000 \t Train Err: 2.1537\n", + "Epoch 816/1000 \t Train Err: 2.1514\n", + "Epoch 817/1000 \t Train Err: 2.1500\n", + "Epoch 818/1000 \t Train Err: 2.1488\n", + "Epoch 819/1000 \t Train Err: 2.1475\n", + "Epoch 820/1000 \t Train Err: 2.1447\n", + "Epoch 821/1000 \t Train Err: 2.1434\n", + "Epoch 822/1000 \t Train Err: 2.1431\n", + "Epoch 823/1000 \t Train Err: 2.1441\n", + "Epoch 824/1000 \t Train Err: 2.1816\n", + "Epoch 825/1000 \t Train Err: 2.1863\n", + "Epoch 826/1000 \t Train Err: 2.1601\n", + "Epoch 827/1000 \t Train Err: 2.1623\n", + "Epoch 828/1000 \t Train Err: 2.1957\n", + "Epoch 829/1000 \t Train Err: 2.1775\n", + "Epoch 830/1000 \t Train Err: 2.1971\n", + "Epoch 831/1000 \t Train Err: 2.1851\n", + "Epoch 832/1000 \t Train Err: 2.1738\n", + "Epoch 833/1000 \t Train Err: 2.1654\n", + "Epoch 834/1000 \t Train Err: 2.1627\n", + "Epoch 835/1000 \t Train Err: 2.1606\n", + "Epoch 836/1000 \t Train Err: 2.1487\n", + "Epoch 837/1000 \t Train Err: 2.1494\n", + "Epoch 838/1000 \t Train Err: 2.1563\n", + "Epoch 839/1000 \t Train Err: 2.1521\n", + "Epoch 840/1000 \t Train Err: 2.1515\n", + "Epoch 841/1000 \t Train Err: 2.1484\n", + "Epoch 842/1000 \t Train Err: 2.1476\n", + "Epoch 843/1000 \t Train Err: 2.1406\n", + "Epoch 844/1000 \t Train Err: 2.1410\n", + "Epoch 845/1000 \t Train Err: 2.1359\n", + "Epoch 846/1000 \t Train Err: 2.1344\n", + "Epoch 847/1000 \t Train Err: 2.1323\n", + "Epoch 848/1000 \t Train Err: 2.1236\n", + "Epoch 849/1000 \t Train Err: 2.1241\n", + "Epoch 850/1000 \t Train Err: 2.1162\n", + "Epoch 851/1000 \t Train Err: 2.1179\n", + "Epoch 852/1000 \t Train Err: 2.1033\n", + "Epoch 853/1000 \t Train Err: 2.1022\n", + "Epoch 854/1000 \t Train Err: 2.1009\n", + "Epoch 855/1000 \t Train Err: 2.0978\n", + "Epoch 856/1000 \t Train Err: 2.0911\n", + "Epoch 857/1000 \t Train Err: 2.0932\n", + "Epoch 858/1000 \t Train Err: 2.0898\n", + "Epoch 859/1000 \t Train Err: 2.0844\n", + "Epoch 860/1000 \t Train Err: 2.0767\n", + "Epoch 861/1000 \t Train Err: 2.0732\n", + "Epoch 862/1000 \t Train Err: 2.0769\n", + "Epoch 863/1000 \t Train Err: 2.0725\n", + "Epoch 864/1000 \t Train Err: 2.0700\n", + "Epoch 865/1000 \t Train Err: 2.0612\n", + "Epoch 866/1000 \t Train Err: 2.0637\n", + "Epoch 867/1000 \t Train Err: 2.0580\n", + "Epoch 868/1000 \t Train Err: 2.0598\n", + "Epoch 869/1000 \t Train Err: 2.0535\n", + "Epoch 870/1000 \t Train Err: 2.0503\n", + "Epoch 871/1000 \t Train Err: 2.0492\n", + "Epoch 872/1000 \t Train Err: 2.0431\n", + "Epoch 873/1000 \t Train Err: 2.0423\n", + "Epoch 874/1000 \t Train Err: 2.0382\n", + "Epoch 875/1000 \t Train Err: 2.0328\n", + "Epoch 876/1000 \t Train Err: 2.0313\n", + "Epoch 877/1000 \t Train Err: 2.0280\n", + "Epoch 878/1000 \t Train Err: 2.0297\n", + "Epoch 879/1000 \t Train Err: 2.0243\n", + "Epoch 880/1000 \t Train Err: 2.0243\n", + "Epoch 881/1000 \t Train Err: 2.0222\n", + "Epoch 882/1000 \t Train Err: 2.0209\n", + "Epoch 883/1000 \t Train Err: 2.0161\n", + "Epoch 884/1000 \t Train Err: 2.0157\n", + "Epoch 885/1000 \t Train Err: 2.0253\n", + "Epoch 886/1000 \t Train Err: 2.0697\n", + "Epoch 887/1000 \t Train Err: 2.2021\n", + "Epoch 888/1000 \t Train Err: 2.2692\n", + "Epoch 889/1000 \t Train Err: 2.1106\n", + "Epoch 890/1000 \t Train Err: 2.1653\n", + "Epoch 891/1000 \t Train Err: 2.2021\n", + "Epoch 892/1000 \t Train Err: 2.1370\n", + "Epoch 893/1000 \t Train Err: 2.1576\n", + "Epoch 894/1000 \t Train Err: 2.1296\n", + "Epoch 895/1000 \t Train Err: 2.1303\n", + "Epoch 896/1000 \t Train Err: 2.1201\n", + "Epoch 897/1000 \t Train Err: 2.1001\n", + "Epoch 898/1000 \t Train Err: 2.1209\n", + "Epoch 899/1000 \t Train Err: 2.1034\n", + "Epoch 900/1000 \t Train Err: 2.1103\n", + "Epoch 901/1000 \t Train Err: 2.0983\n", + "Epoch 902/1000 \t Train Err: 2.0762\n", + "Epoch 903/1000 \t Train Err: 2.0929\n", + "Epoch 904/1000 \t Train Err: 2.0643\n", + "Epoch 905/1000 \t Train Err: 2.0555\n", + "Epoch 906/1000 \t Train Err: 2.0589\n", + "Epoch 907/1000 \t Train Err: 2.0454\n", + "Epoch 908/1000 \t Train Err: 2.0500\n", + "Epoch 909/1000 \t Train Err: 2.0418\n", + "Epoch 910/1000 \t Train Err: 2.0363\n", + "Epoch 911/1000 \t Train Err: 2.0357\n", + "Epoch 912/1000 \t Train Err: 2.0323\n", + "Epoch 913/1000 \t Train Err: 2.0282\n", + "Epoch 914/1000 \t Train Err: 2.0242\n", + "Epoch 915/1000 \t Train Err: 2.0120\n", + "Epoch 916/1000 \t Train Err: 2.0127\n", + "Epoch 917/1000 \t Train Err: 2.0133\n", + "Epoch 918/1000 \t Train Err: 2.0097\n", + "Epoch 919/1000 \t Train Err: 2.0087\n", + "Epoch 920/1000 \t Train Err: 2.0099\n", + "Epoch 921/1000 \t Train Err: 2.0076\n", + "Epoch 922/1000 \t Train Err: 2.0020\n", + "Epoch 923/1000 \t Train Err: 1.9990\n", + "Epoch 924/1000 \t Train Err: 1.9967\n", + "Epoch 925/1000 \t Train Err: 1.9966\n", + "Epoch 926/1000 \t Train Err: 1.9946\n", + "Epoch 927/1000 \t Train Err: 1.9904\n", + "Epoch 928/1000 \t Train Err: 1.9874\n", + "Epoch 929/1000 \t Train Err: 1.9974\n", + "Epoch 930/1000 \t Train Err: 1.9857\n", + "Epoch 931/1000 \t Train Err: 1.9892\n", + "Epoch 932/1000 \t Train Err: 1.9947\n", + "Epoch 933/1000 \t Train Err: 1.9974\n", + "Epoch 934/1000 \t Train Err: 2.0159\n", + "Epoch 935/1000 \t Train Err: 2.0433\n", + "Epoch 936/1000 \t Train Err: 2.0755\n", + "Epoch 937/1000 \t Train Err: 2.0014\n", + "Epoch 938/1000 \t Train Err: 2.0443\n", + "Epoch 939/1000 \t Train Err: 2.0184\n", + "Epoch 940/1000 \t Train Err: 2.0192\n", + "Epoch 941/1000 \t Train Err: 2.0248\n", + "Epoch 942/1000 \t Train Err: 2.0124\n", + "Epoch 943/1000 \t Train Err: 2.0101\n", + "Epoch 944/1000 \t Train Err: 2.0024\n", + "Epoch 945/1000 \t Train Err: 2.0011\n", + "Epoch 946/1000 \t Train Err: 1.9871\n", + "Epoch 947/1000 \t Train Err: 1.9816\n", + "Epoch 948/1000 \t Train Err: 1.9875\n", + "Epoch 949/1000 \t Train Err: 2.0660\n", + "Epoch 950/1000 \t Train Err: 2.0591\n", + "Epoch 951/1000 \t Train Err: 2.0214\n", + "Epoch 952/1000 \t Train Err: 2.0312\n", + "Epoch 953/1000 \t Train Err: 2.0470\n", + "Epoch 954/1000 \t Train Err: 2.0365\n", + "Epoch 955/1000 \t Train Err: 2.0143\n", + "Epoch 956/1000 \t Train Err: 2.0104\n", + "Epoch 957/1000 \t Train Err: 2.0289\n", + "Epoch 958/1000 \t Train Err: 2.0097\n", + "Epoch 959/1000 \t Train Err: 1.9998\n", + "Epoch 960/1000 \t Train Err: 2.0095\n", + "Epoch 961/1000 \t Train Err: 2.0110\n", + "Epoch 962/1000 \t Train Err: 2.0009\n", + "Epoch 963/1000 \t Train Err: 1.9930\n", + "Epoch 964/1000 \t Train Err: 2.0003\n", + "Epoch 965/1000 \t Train Err: 1.9912\n", + "Epoch 966/1000 \t Train Err: 1.9859\n", + "Epoch 967/1000 \t Train Err: 1.9843\n", + "Epoch 968/1000 \t Train Err: 1.9828\n", + "Epoch 969/1000 \t Train Err: 1.9776\n", + "Epoch 970/1000 \t Train Err: 1.9790\n", + "Epoch 971/1000 \t Train Err: 1.9697\n", + "Epoch 972/1000 \t Train Err: 1.9671\n", + "Epoch 973/1000 \t Train Err: 1.9673\n", + "Epoch 974/1000 \t Train Err: 1.9585\n", + "Epoch 975/1000 \t Train Err: 1.9605\n", + "Epoch 976/1000 \t Train Err: 1.9537\n", + "Epoch 977/1000 \t Train Err: 1.9529\n", + "Epoch 978/1000 \t Train Err: 1.9477\n", + "Epoch 979/1000 \t Train Err: 1.9485\n", + "Epoch 980/1000 \t Train Err: 1.9376\n", + "Epoch 981/1000 \t Train Err: 1.9426\n", + "Epoch 982/1000 \t Train Err: 1.9416\n", + "Epoch 983/1000 \t Train Err: 1.9334\n", + "Epoch 984/1000 \t Train Err: 1.9249\n", + "Epoch 985/1000 \t Train Err: 1.9216\n", + "Epoch 986/1000 \t Train Err: 1.9268\n", + "Epoch 987/1000 \t Train Err: 1.9630\n", + "Epoch 988/1000 \t Train Err: 2.0237\n", + "Epoch 989/1000 \t Train Err: 2.0037\n", + "Epoch 990/1000 \t Train Err: 1.9824\n", + "Epoch 991/1000 \t Train Err: 1.9718\n", + "Epoch 992/1000 \t Train Err: 1.9726\n", + "Epoch 993/1000 \t Train Err: 1.9536\n", + "Epoch 994/1000 \t Train Err: 1.9662\n", + "Epoch 995/1000 \t Train Err: 1.9492\n", + "Epoch 996/1000 \t Train Err: 1.9482\n", + "Epoch 997/1000 \t Train Err: 1.9375\n", + "Epoch 998/1000 \t Train Err: 1.9492\n", + "Epoch 999/1000 \t Train Err: 1.9351\n" + ] + } + ], + "source": [ + "model.train()\n", + "for epoch in range(NEPOCHS):\n", + " optimizer.zero_grad()\n", + " data, labels = mkbatch_rtl(BSZ)\n", + " # shift labels to prevent cheating\n", + " shifted_labels = torch.roll(labels, 1, dims=1)\n", + " shifted_labels[:, 0] = VOCAB_SIZE # start token\n", + " outputs = model(data, shifted_labels).permute((0, 2, 1))\n", + " loss = criterion(outputs, labels)\n", + " train_loss = loss.item()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " train_err.append(train_loss)\n", + "\n", + " with open('loss', 'a') as f:\n", + " f.write(f\"{train_loss}\\n\")\n", + " print(f\"Epoch {epoch}/{NEPOCHS} \\t Train Err: {train_loss:.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 96, + "execution_state": "idle", + "id": "a3c41150-4541-4722-83a7-e7ad937f6c4f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[3, 8, 4, 0]], device='cuda:0') tensor([[7, 8]], device='cuda:0')\n", + "tensor([-4.4248e+00, -1.0567e+00, 1.2971e+00, -2.0221e+00, -6.6597e-01,\n", + " -2.6027e+00, -1.5254e-02, 8.1894e+00, -1.6939e-03, -1.2252e+00],\n", + " device='cuda:0')\n", + "tensor([-3.7663, -1.7898, -1.4273, 1.9667, -2.3513, -4.7138, -2.2421, 3.6817,\n", + " 8.9049, 3.1622], device='cuda:0')\n", + "tensor([[7, 8]], device='cuda:0', dtype=torch.int32) tensor([[7, 8]], device='cuda:0')\n" + ] + } + ], + "source": [ + "model.eval()\n", + "data, labels = mkbatch_rtl(1)\n", + "print(data, labels)\n", + "with torch.no_grad():\n", + " ans = torch.zeros((1, NUM_LEN), dtype=torch.int, device=device)\n", + " ans[0, 0] = VOCAB_SIZE\n", + " for i in range(NUM_LEN):\n", + " outputs = model(data, ans)\n", + " print(outputs[0, i])\n", + " # break\n", + " ans[0, (i + 1) % NUM_LEN] = torch.argmax(outputs[0, i])\n", + "ans = torch.roll(ans, -1, dims=1)\n", + "print(ans, labels)" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "execution_state": "idle", + "id": "1843b944-bab5-40ee-b26e-5d3b87ea9454", + "metadata": {}, + "outputs": [ + { + "ename": "FileNotFoundError", + "evalue": "[Errno 2] No such file or directory: 'add-ltr-loss'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[32], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mmath\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mmatplotlib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpyplot\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mplt\u001b[39;00m\n\u001b[0;32m----> 4\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28;43mopen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43madd-ltr-loss\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[1;32m 5\u001b[0m plt\u001b[38;5;241m.\u001b[39mplot(\u001b[38;5;28mrange\u001b[39m(NEPOCHS), \u001b[38;5;28mlist\u001b[39m(\u001b[38;5;28mmap\u001b[39m(\u001b[38;5;28;01mlambda\u001b[39;00m x: math\u001b[38;5;241m.\u001b[39mlog(\u001b[38;5;28mfloat\u001b[39m(x)), f\u001b[38;5;241m.\u001b[39mreadlines())))\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124madd-rtl-loss\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m f:\n", + "File \u001b[0;32m~/.venv/lib64/python3.12/site-packages/IPython/core/interactiveshell.py:324\u001b[0m, in \u001b[0;36m_modified_open\u001b[0;34m(file, *args, **kwargs)\u001b[0m\n\u001b[1;32m 317\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m file \u001b[38;5;129;01min\u001b[39;00m {\u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m2\u001b[39m}:\n\u001b[1;32m 318\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m 319\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIPython won\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt let you open fd=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfile\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m by default \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 320\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mas it is likely to crash IPython. If you know what you are doing, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 321\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124myou can use builtins\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m open.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 322\u001b[0m )\n\u001b[0;32m--> 324\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mio_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfile\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'add-ltr-loss'" + ] + } + ], + "source": [ + "import math\n", + "import matplotlib.pyplot as plt\n", + "\n", + "with open(\"add-ltr-loss\") as f:\n", + " plt.plot(range(NEPOCHS), list(map(lambda x: math.log(float(x)), f.readlines())))\n", + "with open(\"add-rtl-loss\") as f:\n", + " plt.plot(range(NEPOCHS), list(map(lambda x: math.log(float(x)), f.readlines())))\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b97b349f-f20b-441d-8c7f-1724e8cf30cc", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/japanese.ipynb b/notebooks/japanese.ipynb new file mode 100644 index 0000000..4112019 --- /dev/null +++ b/notebooks/japanese.ipynb @@ -0,0 +1,489 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "execution_state": "idle", + "id": "1ddfc692-bda7-4d38-a549-2fb0d40d437d", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "BertForMaskedLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly overwritten. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.\n", + " - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes\n", + " - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).\n", + " - If you are not the owner of the model architecture class, please contact the model code owner to update it.\n", + "Some weights of the model checkpoint at /home/sipb/nlp-class-project/checkpoints/bert_base_ltr/epoch_3_checkpt were not used when initializing BertForMaskedLM: ['attention_mask']\n", + "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of BertForMaskedLM were not initialized from the model checkpoint at /home/sipb/nlp-class-project/checkpoints/bert_base_ltr/epoch_3_checkpt and are newly initialized because the shapes did not match:\n", + "- bert.embeddings.position_embeddings.weight: found shape torch.Size([512, 768]) in the checkpoint and torch.Size([128, 768]) in the model instantiated\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" + ] + } + ], + "source": [ + "import math\n", + "import sys\n", + "\n", + "sys.path.append(\"..\")\n", + "\n", + "import torch\n", + "import transformers\n", + "\n", + "import pandas as pd\n", + "\n", + "from utils import add_attn_hooks\n", + "\n", + "# text_dir = \"rtl\"\n", + "text_dir = \"ltr\"\n", + "tokenizer = transformers.AutoTokenizer.from_pretrained(\"bert-base-uncased\")\n", + "model = transformers.AutoModelForMaskedLM.from_pretrained(f\"/home/sipb/nlp-class-project/checkpoints/bert_base_{text_dir}/epoch_3_checkpt\", ignore_mismatched_sizes=True)\n", + "\n", + "# tokenizer = transformers.AutoTokenizer.from_pretrained(\"distilbert/distilbert-base-uncased\")\n", + "# model = transformers.AutoModelForMaskedLM.from_pretrained(f\"/home/sipb/nlp-class-project/checkpoints/distilbert_base_{text_dir}/epoch_3_checkpt\", ignore_mismatched_sizes=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "145d2ffd-db55-4b8f-9fbb-85a51e0b3d11", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 3, + "execution_state": "idle", + "id": "a732375b-1682-45c6-8df0-8db1458559c9", + "metadata": {}, + "outputs": [], + "source": [ + "add_attn_hooks(model, text_dir)\n", + "model.eval();" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "execution_state": "idle", + "id": "041d1702-5aaf-45f0-9413-4014b315d1ed", + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.read_parquet('/home/sipb/nlp-class-project/data/japan.parquet')" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "execution_state": "idle", + "id": "2bace74b-a716-4d49-a912-53155cf002ba", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "b'\n", + "_START_ARTICLE_\n", + "ビートたけしの教科書に載らない日本人の謎\n", + "_START_SECTION_\n", + "概要\n", + "_START_PARAGRAPH_\n", + "「教科書には決して載らない」日本人の謎やしきたりを多角的に検証し、日本人のDNAを解明する。_NEWLINE_新春番組として定期的に放送されており、年末の午前中に再放送されるのが恒例となっている。'\n" + ] + } + ], + "source": [ + "df[\"text\"][0]\n", + "import codecs\n", + "decoded_str = codecs.escape_decode(df[\"text\"][0])[0].decode('utf-8')\n", + "print(decoded_str)" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "execution_state": "idle", + "id": "8a9147ea-d9dc-4826-8030-c8417609405d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "q : where do pandas live? a : (,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, and,, (,,,,,,.,,,,,,,,,,,,, and\n" + ] + } + ], + "source": [ + "input_text = [\"Q: Where do pandas live? A:\"]#, \"ビートたけしの教科書に載らない日\"]\n", + "batch = tokenizer(input_text, return_tensors=\"pt\", padding_side=\"right\", padding=\"max_length\", max_length=64)\n", + "output_ids = model.generate(batch['input_ids'], max_length=128, do_sample=False) # do_sample=False ensures greedy decoding\n", + "decoded_output = tokenizer.decode(output_ids[0], skip_special_tokens=True)\n", + "print(decoded_output)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "execution_state": "idle", + "id": "1a7c9b35-0c07-431d-91df-bd2f8c7467eb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "MaskedLMOutput(loss=None, logits=tensor([[[ -7.9645, -7.6722, -7.8979, ..., -8.6562, -8.2586, -6.7448],\n", + " [-11.1255, -11.2591, -11.3443, ..., -10.1338, -11.9891, -10.2974],\n", + " [ -8.1256, -8.1880, -7.9874, ..., -8.0597, -8.6987, -10.2472],\n", + " ...,\n", + " [-14.5633, -14.4418, -14.4735, ..., -14.5651, -14.2234, -13.5610],\n", + " [-18.9095, -18.6487, -18.7593, ..., -19.1327, -18.8564, -17.4334],\n", + " [-17.8532, -17.6451, -17.7208, ..., -18.0046, -17.7334, -16.5670]]]), hidden_states=None, attentions=None)\n" + ] + } + ], + "source": [ + "with torch.inference_mode():\n", + " batch = tokenizer([\"ビートたけしの教科書に載らない日本人の謎\"], return_tensors=\"pt\", padding_side=\"left\" if text_dir == \"rtl\" else \"right\", padding=\"max_length\", max_length=128)\n", + " output = model(**batch)\n", + " print(output)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "execution_state": "idle", + "id": "a4098975-2df6-4435-bc93-1a5afd6d7e68", + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'riddles' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[15], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# CPU is fast enough\u001b[39;00m\n\u001b[1;32m 3\u001b[0m ppls \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m----> 4\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m riddle \u001b[38;5;129;01min\u001b[39;00m \u001b[43mriddles\u001b[49m:\n\u001b[1;32m 5\u001b[0m batch \u001b[38;5;241m=\u001b[39m tokenizer([riddle], return_tensors\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpt\u001b[39m\u001b[38;5;124m\"\u001b[39m, padding_side\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mleft\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m text_dir \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrtl\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mright\u001b[39m\u001b[38;5;124m\"\u001b[39m, padding\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmax_length\u001b[39m\u001b[38;5;124m\"\u001b[39m, max_length\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m128\u001b[39m)\n\u001b[1;32m 6\u001b[0m batch[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlabels\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m batch[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minput_ids\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mclone()\n", + "\u001b[0;31mNameError\u001b[0m: name 'riddles' is not defined" + ] + } + ], + "source": [ + "# CPU is fast enough\n", + "\n", + "ppls = []\n", + "for riddle in riddles:\n", + " batch = tokenizer([riddle], return_tensors=\"pt\", padding_side=\"left\" if text_dir == \"rtl\" else \"right\", padding=\"max_length\", max_length=128)\n", + " batch[\"labels\"] = batch[\"input_ids\"].clone()\n", + " batch[\"labels\"][batch[\"attention_mask\"] == 0] = -100\n", + " # batch = tokenizer([riddle], return_tensors=\"pt\")#, padding_side=\"left\" if text_dir == \"rtl\" else \"right\", padding=\"longest\", max_length=128)\n", + " # batch[\"labels\"] = batch[\"input_ids\"]\n", + " with torch.inference_mode():\n", + " output = model(**batch)\n", + " ppls.append(math.e ** output.loss.item())" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "execution_state": "idle", + "id": "c4a82af4-d0d8-415a-9135-3a1350c1402e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(692.7175314596647, 'rtl')" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sum(ppls) / len(ppls), text_dir" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "execution_state": "idle", + "id": "84a95c66-6dd3-4ccb-96a2-96f38008f70e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(616.6241458855995, 'ltr')" + ] + }, + "execution_count": 43, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sum(ppls) / len(ppls), text_dir" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "execution_state": "idle", + "id": "51ed80f1-a935-42bc-8194-832f91222c45", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(526.979384061791, 'rtl')" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sum(ppls) / len(ppls), text_dir # distilbert" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "execution_state": "idle", + "id": "34a2edec-b1d9-466c-a457-954c587f7817", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(288.22724792187364, 'ltr')" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sum(ppls) / len(ppls), text_dir # distilbert" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "execution_state": "idle", + "id": "40a98c10-59c3-498a-a9e6-c23bd9437bc7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "937.8557468023619" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sum(ppls) / len(ppls)" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "execution_state": "idle", + "id": "80b22ba1-e5ba-4f1e-8038-158a2c2f37a6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'input_ids': tensor([[ 101, 1045, 2064, 2022, 2524, 1010, 2021, 1045, 2572, 2025,\n", + " 5024, 1012, 2054, 2572, 1045, 1029, 1037, 15117, 1012, 102,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0]]), 'labels': tensor([[ 101, 1045, 2064, 2022, 2524, 1010, 2021, 1045, 2572, 2025,\n", + " 5024, 1012, 2054, 2572, 1045, 1029, 1037, 15117, 1012, 102,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", + " -100, -100, -100, -100, -100, -100, -100, -100]])}" + ] + }, + "execution_count": 58, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "batch" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "execution_state": "idle", + "id": "c68b5235-a4a7-4f38-9acb-f5072e546a96", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(array([ 4., 6., 11., 6., 5., 2., 1., 1., 2., 2.]),\n", + " array([ 613.56297843, 829.36555779, 1045.16813716, 1260.97071653,\n", + " 1476.77329589, 1692.57587526, 1908.37845463, 2124.18103399,\n", + " 2339.98361336, 2555.78619272, 2771.58877209]),\n", + " )" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAY8UlEQVR4nO3df5DVVf348dcCsoCyyy9ZIBdZf4ym4E+MECMdGIHQsBpHjRqiRlMxJQqFCozMQGscJjM1Z1KbUbRmBBt/MGMokiMiIKikoRgKqUCJ7PJDV2TP54+v3vEKqPS99ywLj8fMnXHf78P7nMvhzj69P3YrUkopAAAyadXcCwAA9i/iAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAsmrT3Av4uKampnjjjTeiY8eOUVFR0dzLAQA+g5RSbN68OXr16hWtWn3ycxt7XXy88cYbUVtb29zLAAD+B2vXro1DDjnkE8fsdfHRsWPHiA8WX1VV1dzLAQA+g4aGhqitrS18H/8ke118fPhSS1VVlfgAgBbms7xlwhtOAYCsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZtWnuBbBv6jPpweZewh57dcbI5l4CwH7BMx8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFntcXwsWLAgzj777OjVq1dUVFTEnDlzis6nlGLq1KnRs2fPaN++fQwdOjRefvnlUq4ZAGjB9jg+tm7dGscff3zcdNNNuzx//fXXx29/+9u45ZZbYtGiRXHggQfGsGHD4t133y3FegGAFq7Nnv6BESNGxIgRI3Z5LqUUM2fOjJ/97GcxatSoiIj405/+FDU1NTFnzpw4//zz//9XDAC0aCV9z8fq1atj3bp1MXTo0MKx6urqGDBgQCxcuHCXf6axsTEaGhqKbgDAvquk8bFu3bqIiKipqSk6XlNTUzj3cdOnT4/q6urCrba2tpRLAgD2Ms3+aZfJkydHfX194bZ27drmXhIAUEYljY8ePXpERMT69euLjq9fv75w7uMqKyujqqqq6AYA7LtKGh91dXXRo0ePmDdvXuFYQ0NDLFq0KAYOHFjKqQCAFmqPP+2yZcuWWLVqVeHr1atXx/Lly6NLly7Ru3fvGD9+fPzyl7+MI488Murq6mLKlCnRq1evOOecc0q9dgCgBdrj+FiyZEmcccYZha8nTJgQERFjxoyJO+64I6688srYunVrXHTRRbFp06Y47bTTYu7cudGuXbvSrhwAaJEqUkqpuRfxUQ0NDVFdXR319fXe/9GC9Zn0YHMvYY+9OmNkcy8BoMXak+/fzf5pFwBg/yI+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFYlj48dO3bElClToq6uLtq3bx+HH354XHPNNZFSKvVUAEAL1KbUF7zuuuvi5ptvjjvvvDOOPfbYWLJkSYwdOzaqq6vj8ssvL/V0AEALU/L4ePLJJ2PUqFExcuTIiIjo06dPzJo1K55++ulSTwUAtEAlf9nl1FNPjXnz5sVLL70UERHPPvtsPPHEEzFixIhdjm9sbIyGhoaiGwCw7yr5Mx+TJk2KhoaGOProo6N169axY8eOuPbaa2P06NG7HD99+vSYNm1aqZcBAOylSv7Mx5///Oe466674u67745nnnkm7rzzzvjNb34Td9555y7HT548Oerr6wu3tWvXlnpJAMBepOTPfEycODEmTZoU559/fkRE9OvXL1577bWYPn16jBkzZqfxlZWVUVlZWeplAAB7qZI/87Ft27Zo1ar4sq1bt46mpqZSTwUAtEAlf+bj7LPPjmuvvTZ69+4dxx57bCxbtixuuOGG+O53v1vqqQCAFqjk8XHjjTfGlClT4tJLL40NGzZEr1694vvf/35MnTq11FMBAC1QyeOjY8eOMXPmzJg5c2apLw0A7AP8bhcAICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALIqS3y8/vrr8a1vfSu6du0a7du3j379+sWSJUvKMRUA0MK0KfUF33777Rg0aFCcccYZ8fDDD8fBBx8cL7/8cnTu3LnUUwEALVDJ4+O6666L2trauP322wvH6urqSj0NANBClfxll7/+9a/Rv3//OPfcc6N79+5x4oknxm233bbb8Y2NjdHQ0FB0AwD2XRUppVTKC7Zr1y4iIiZMmBDnnntuLF68OK644oq45ZZbYsyYMTuN//nPfx7Tpk3b6Xh9fX1UVVWVcmktVp9JDzb3EthLvTpjZHMvASAiIhoaGqK6uvozff8ueXy0bds2+vfvH08++WTh2OWXXx6LFy+OhQsX7jS+sbExGhsbixZfW1srPj5CfLA74gPYW+xJfJT8ZZeePXvGMcccU3Ts85//fKxZs2aX4ysrK6OqqqroBgDsu0oeH4MGDYqVK1cWHXvppZfi0EMPLfVUAEALVPL4+OEPfxhPPfVU/OpXv4pVq1bF3XffHX/4wx9i3LhxpZ4KAGiBSh4fp5xySsyePTtmzZoVffv2jWuuuSZmzpwZo0ePLvVUAEALVPKf8xERcdZZZ8VZZ51VjksDAC2c3+0CAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGTVprkXAPzv+kx6sLmXsMdenTGyuZcANDPPfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZFX2+JgxY0ZUVFTE+PHjyz0VANAClDU+Fi9eHLfeemscd9xx5ZwGAGhByhYfW7ZsidGjR8dtt90WnTt3Ltc0AEALU7b4GDduXIwcOTKGDh36ieMaGxujoaGh6AYA7LvalOOi99xzTzzzzDOxePHiTx07ffr0mDZtWjmWsUt9Jj2YbS4AYGclf+Zj7dq1ccUVV8Rdd90V7dq1+9TxkydPjvr6+sJt7dq1pV4SALAXKfkzH0uXLo0NGzbESSedVDi2Y8eOWLBgQfzud7+LxsbGaN26deFcZWVlVFZWlnoZAMBequTxMWTIkHj++eeLjo0dOzaOPvrouOqqq4rCAwDY/5Q8Pjp27Bh9+/YtOnbggQdG165ddzoOAOx//IRTACCrsnza5ePmz5+fYxoAoAXwzAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmVPD6mT58ep5xySnTs2DG6d+8e55xzTqxcubLU0wAALVTJ4+Pxxx+PcePGxVNPPRWPPPJIbN++Pc4888zYunVrqacCAFqgNqW+4Ny5c4u+vuOOO6J79+6xdOnSGDx4cKmnAwBamJLHx8fV19dHRESXLl12eb6xsTEaGxsLXzc0NJR7SQBAMyprfDQ1NcX48eNj0KBB0bdv312OmT59ekybNq2cywD2In0mPdjcS9hjr84Y2dxL2G+0xH8fLVFz/5su66ddxo0bFytWrIh77rlnt2MmT54c9fX1hdvatWvLuSQAoJmV7ZmPyy67LB544IFYsGBBHHLIIbsdV1lZGZWVleVaBgCwlyl5fKSU4gc/+EHMnj075s+fH3V1daWeAgBowUoeH+PGjYu777477r///ujYsWOsW7cuIiKqq6ujffv2pZ4OAGhhSv6ej5tvvjnq6+vj9NNPj549exZu9957b6mnAgBaoLK87AIAsDt+twsAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkFWb5l4AwN6uz6QHm3sJsE/xzAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFZli4+bbrop+vTpE+3atYsBAwbE008/Xa6pAIAWpCzxce+998aECRPi6quvjmeeeSaOP/74GDZsWGzYsKEc0wEALUhZ4uOGG26ICy+8MMaOHRvHHHNM3HLLLdGhQ4f44x//WI7pAIAWpE2pL/jee+/F0qVLY/LkyYVjrVq1iqFDh8bChQt3Gt/Y2BiNjY2Fr+vr6yMioqGhodRLi4iIpsZtZbkuALQU5fge++E1U0qfOrbk8fHf//43duzYETU1NUXHa2pq4p///OdO46dPnx7Tpk3b6XhtbW2plwYARET1zPJde/PmzVFdXf2JY0oeH3tq8uTJMWHChMLXTU1NsXHjxujatWtUVFQ069rYWUNDQ9TW1sbatWujqqqquZfDx9ifvZ892rvZn/9dSik2b94cvXr1+tSxJY+Pbt26RevWrWP9+vVFx9evXx89evTYaXxlZWVUVlYWHevUqVOpl0WJVVVVeWDuxezP3s8e7d3sz//m057x+FDJ33Datm3bOPnkk2PevHmFY01NTTFv3rwYOHBgqacDAFqYsrzsMmHChBgzZkz0798/vvCFL8TMmTNj69atMXbs2HJMBwC0IGWJj/POOy/+85//xNSpU2PdunVxwgknxNy5c3d6EyotT2VlZVx99dU7vVTG3sH+7P3s0d7N/uRRkT7LZ2IAAErE73YBALISHwBAVuIDAMhKfAAAWYmP/dCCBQvi7LPPjl69ekVFRUXMmTOn6HxKKaZOnRo9e/aM9u3bx9ChQ+Pll18uGrNx48YYPXp0VFVVRadOneJ73/tebNmypWjMc889F1/60peiXbt2UVtbG9dff32W+9fSfdr+fOc734mKioqi2/Dhw4vG2J/ymT59epxyyinRsWPH6N69e5xzzjmxcuXKojHvvvtujBs3Lrp27RoHHXRQfOMb39jpBy+uWbMmRo4cGR06dIju3bvHxIkT4/333y8aM3/+/DjppJOisrIyjjjiiLjjjjuy3MeW7rPs0emnn77T4+jiiy8uGmOPyiix33nooYfST3/603TfffeliEizZ88uOj9jxoxUXV2d5syZk5599tn01a9+NdXV1aV33nmnMGb48OHp+OOPT0899VT6+9//no444oh0wQUXFM7X19enmpqaNHr06LRixYo0a9as1L59+3Trrbdmva8t0aftz5gxY9Lw4cPTm2++Wbht3LixaIz9KZ9hw4al22+/Pa1YsSItX748feUrX0m9e/dOW7ZsKYy5+OKLU21tbZo3b15asmRJ+uIXv5hOPfXUwvn3338/9e3bNw0dOjQtW7YsPfTQQ6lbt25p8uTJhTH/+te/UocOHdKECRPSCy+8kG688cbUunXrNHfu3Oz3uaX5LHv05S9/OV144YVFj6P6+vrCeXtUXuJjP/fxb25NTU2pR48e6de//nXh2KZNm1JlZWWaNWtWSimlF154IUVEWrx4cWHMww8/nCoqKtLrr7+eUkrp97//fercuXNqbGwsjLnqqqvSUUcdleme7Rt2Fx+jRo3a7Z+xP3lt2LAhRUR6/PHHU/rg8XLAAQekv/zlL4UxL774YoqItHDhwpQ+CMxWrVqldevWFcbcfPPNqaqqqrAnV155ZTr22GOL5jrvvPPSsGHDMt2zfcfH9yh9EB9XXHHFbv+MPSovL7tQZPXq1bFu3boYOnRo4Vh1dXUMGDAgFi5cGBERCxcujE6dOkX//v0LY4YOHRqtWrWKRYsWFcYMHjw42rZtWxgzbNiwWLlyZbz99ttZ79O+aP78+dG9e/c46qij4pJLLom33nqrcM7+5FVfXx8REV26dImIiKVLl8b27duLHkNHH3109O7du+gx1K9fv6IfvDhs2LBoaGiIf/zjH4UxH73Gh2M+vAaf3cf36EN33XVXdOvWLfr27RuTJ0+Obdu2Fc7Zo/Jq9t9qy95l3bp1ERE7/TTampqawrl169ZF9+7di863adMmunTpUjSmrq5up2t8eK5z585lvR/7suHDh8fXv/71qKuri1deeSV+8pOfxIgRI2LhwoXRunVr+5NRU1NTjB8/PgYNGhR9+/aN+ODvr23btjv9gsyPP4Z29RiLjzwGdzemoaEh3nnnnWjfvn1Z79u+Yld7FBHxzW9+Mw499NDo1atXPPfcc3HVVVfFypUr47777ouwR2UnPqCFOf/88wv/3a9fvzjuuOPi8MMPj/nz58eQIUOadW37m3HjxsWKFSviiSeeaO6lsBu726OLLrqo8N/9+vWLnj17xpAhQ+KVV16Jww8/vBlWun/xsgtFevToERGx0zvz169fXzjXo0eP2LBhQ9H5999/PzZu3Fg0ZlfX+OgclMZhhx0W3bp1i1WrVkXYn2wuu+yyeOCBB+Kxxx6LQw45pHC8R48e8d5778WmTZuKxn/8MfRpf/+7G1NVVeX/qD+j3e3RrgwYMCAiouhxZI/KR3xQpK6uLnr06BHz5s0rHGtoaIhFixbFwIEDIyJi4MCBsWnTpli6dGlhzKOPPhpNTU2FB/DAgQNjwYIFsX379sKYRx55JI466ihP6ZfYv//973jrrbeiZ8+eEfan7FJKcdlll8Xs2bPj0Ucf3enlq5NPPjkOOOCAosfQypUrY82aNUWPoeeff74oEh955JGoqqqKY445pjDmo9f4cMyH12D3Pm2PdmX58uUREUWPI3tURs39jlfy27x5c1q2bFlatmxZioh0ww03pGXLlqXXXnstpQ8+atupU6d0//33p+eeey6NGjVqlx+1PfHEE9OiRYvSE088kY488siij3Ju2rQp1dTUpG9/+9tpxYoV6Z577kkdOnTwUc7P4JP2Z/PmzenHP/5xWrhwYVq9enX629/+lk466aR05JFHpnfffbdwDftTPpdcckmqrq5O8+fPL/qY5rZt2wpjLr744tS7d+/06KOPpiVLlqSBAwemgQMHFs5/+DHOM888My1fvjzNnTs3HXzwwbv8GOfEiRPTiy++mG666SYf4/yMPm2PVq1alX7xi1+kJUuWpNWrV6f7778/HXbYYWnw4MGFa9ij8hIf+6HHHnssRcROtzFjxqT0wcdtp0yZkmpqalJlZWUaMmRIWrlyZdE13nrrrXTBBRekgw46KFVVVaWxY8emzZs3F4159tln02mnnZYqKyvT5z73uTRjxoys97Ol+qT92bZtWzrzzDPTwQcfnA444IB06KGHpgsvvLDo44DJ/pTVrvYmItLtt99eGPPOO++kSy+9NHXu3Dl16NAhfe1rX0tvvvlm0XVeffXVNGLEiNS+ffvUrVu39KMf/Sht3769aMxjjz2WTjjhhNS2bdt02GGHFc3B7n3aHq1ZsyYNHjw4denSJVVWVqYjjjgiTZw4sejnfCR7VFYV6f9tFABAFt7zAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACy+j/J4r9i/nZCRAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "plt.hist(ppls)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "execution_state": "idle", + "id": "8acad3ce-905d-455e-af5d-9770495f374a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414,\n", + " 956.7294281325414]" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ppls" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "86daa05b-5784-457b-b65e-8b8395128d6f", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/loss b/notebooks/loss new file mode 100644 index 0000000..e1e6942 --- /dev/null +++ b/notebooks/loss @@ -0,0 +1,1000 @@ +2.4793217182159424 +2.4309544563293457 +2.3800179958343506 +2.3492820262908936 +2.3288204669952393 +2.3201887607574463 +2.317059278488159 +2.3139350414276123 +2.3094708919525146 +2.3064167499542236 +2.3040108680725098 +2.3028905391693115 +2.302960157394409 +2.303684949874878 +2.304744243621826 +2.305999517440796 +2.3067049980163574 +2.306713819503784 +2.3067827224731445 +2.3058664798736572 +2.30601167678833 +2.30521559715271 +2.3043692111968994 +2.3038854598999023 +2.3038713932037354 +2.3032853603363037 +2.3032476902008057 +2.3031513690948486 +2.3032004833221436 +2.3028852939605713 +2.3028409481048584 +2.303209066390991 +2.303081750869751 +2.302985668182373 +2.3031227588653564 +2.3030753135681152 +2.303075075149536 +2.3029205799102783 +2.303251266479492 +2.3031625747680664 +2.30310320854187 +2.3030107021331787 +2.302666664123535 +2.3026700019836426 +2.3026769161224365 +2.3026905059814453 +2.3026187419891357 +2.3025882244110107 +2.302690267562866 +2.3026435375213623 +2.302729845046997 +2.3026950359344482 +2.302746534347534 +2.3026206493377686 +2.3027870655059814 +2.3026585578918457 +2.3026347160339355 +2.3026607036590576 +2.3025848865509033 +2.3026163578033447 +2.3026700019836426 +2.3026363849639893 +2.3026375770568848 +2.3026788234710693 +2.302553415298462 +2.3025968074798584 +2.302593469619751 +2.3026044368743896 +2.3025927543640137 +2.3026342391967773 +2.302656412124634 +2.3026087284088135 +2.302560329437256 +2.3025975227355957 +2.3026320934295654 +2.3026249408721924 +2.302595376968384 +2.302572011947632 +2.30261492729187 +2.302673101425171 +2.3026254177093506 +2.3027074337005615 +2.3025972843170166 +2.3026070594787598 +2.3025753498077393 +2.302605390548706 +2.3026022911071777 +2.3026483058929443 +2.3026845455169678 +2.3026163578033447 +2.3026020526885986 +2.3025448322296143 +2.3026366233825684 +2.302520513534546 +2.302574634552002 +2.302612543106079 +2.3025946617126465 +2.302628755569458 +2.302635908126831 +2.3026602268218994 +2.302551507949829 +2.302579164505005 +2.3025732040405273 +2.3026163578033447 +2.3025848865509033 +2.3025896549224854 +2.3026063442230225 +2.302605390548706 +2.3025991916656494 +2.302596092224121 +2.302624464035034 +2.302584409713745 +2.3026046752929688 +2.3025929927825928 +2.3026199340820312 +2.3025825023651123 +2.3025972843170166 +2.302593469619751 +2.3026280403137207 +2.3026411533355713 +2.302598237991333 +2.302582263946533 +2.302603006362915 +2.302595376968384 +2.3025989532470703 +2.302626848220825 +2.3026373386383057 +2.30259108543396 +2.3026230335235596 +2.3026344776153564 +2.3026301860809326 +2.3026552200317383 +2.302663564682007 +2.3025882244110107 +2.3025946617126465 +2.30256986618042 +2.302544355392456 +2.3026559352874756 +2.3025758266448975 +2.3024582862854004 +2.3026046752929688 +2.3026840686798096 +2.3026156425476074 +2.3026177883148193 +2.302590847015381 +2.3026554584503174 +2.3026623725891113 +2.3026609420776367 +2.302621603012085 +2.3027026653289795 +2.3026010990142822 +2.3026115894317627 +2.302668571472168 +2.3026716709136963 +2.302629232406616 +2.302640199661255 +2.302658796310425 +2.3024585247039795 +2.3025708198547363 +2.3025777339935303 +2.302565574645996 +2.3025872707366943 +2.302602767944336 +2.3025877475738525 +2.302659273147583 +2.3025927543640137 +2.3026325702667236 +2.302605152130127 +2.3026175498962402 +2.302565574645996 +2.302611827850342 +2.302602529525757 +2.3026187419891357 +2.3025996685028076 +2.30258846282959 +2.302598714828491 +2.302558183670044 +2.3025643825531006 +2.3026058673858643 +2.3026177883148193 +2.302593231201172 +2.3026416301727295 +2.3026068210601807 +2.3025405406951904 +2.3026459217071533 +2.302727460861206 +2.30265474319458 +2.3026444911956787 +2.3026580810546875 +2.3025972843170166 +2.3026254177093506 +2.3025786876678467 +2.3026297092437744 +2.302546739578247 +2.3026621341705322 +2.3025929927825928 +2.3025829792022705 +2.302605152130127 +2.302640199661255 +2.3025641441345215 +2.3026347160339355 +2.3025920391082764 +2.302574396133423 +2.302616596221924 +2.3026392459869385 +2.3026092052459717 +2.3026201725006104 +2.3026134967803955 +2.302623987197876 +2.302586793899536 +2.3026044368743896 +2.3027074337005615 +2.3026058673858643 +2.3025760650634766 +2.3026044368743896 +2.3025972843170166 +2.3026139736175537 +2.3026390075683594 +2.3025991916656494 +2.302525043487549 +2.3026723861694336 +2.302642822265625 +2.3026106357574463 +2.3026137351989746 +2.3026058673858643 +2.3026320934295654 +2.302610158920288 +2.3025975227355957 +2.302581310272217 +2.3026039600372314 +2.302619695663452 +2.3026483058929443 +2.3026204109191895 +2.3026156425476074 +2.3026063442230225 +2.302602767944336 +2.302591562271118 +2.3026509284973145 +2.302588701248169 +2.3025119304656982 +2.302616834640503 +2.302570104598999 +2.3026840686798096 +2.302591323852539 +2.3025944232940674 +2.302640676498413 +2.3025567531585693 +2.3026812076568604 +2.3025763034820557 +2.302642583847046 +2.3026039600372314 +2.3025941848754883 +2.302649736404419 +2.3026223182678223 +2.3026328086853027 +2.3026304244995117 +2.302619457244873 +2.302593946456909 +2.302657127380371 +2.302690267562866 +2.3026061058044434 +2.30265212059021 +2.302602529525757 +2.302621364593506 +2.3025906085968018 +2.302635431289673 +2.3026089668273926 +2.3026158809661865 +2.3026163578033447 +2.302581787109375 +2.302584171295166 +2.3026280403137207 +2.3025906085968018 +2.302751302719116 +2.302713394165039 +2.3026044368743896 +2.3025879859924316 +2.302607774734497 +2.302643299102783 +2.302614212036133 +2.3026540279388428 +2.3026387691497803 +2.302614450454712 +2.3025951385498047 +2.302586317062378 +2.3025786876678467 +2.302649736404419 +2.302600145339966 +2.3026301860809326 +2.302614450454712 +2.302673101425171 +2.302687883377075 +2.3026111125946045 +2.302589178085327 +2.3025310039520264 +2.3026034832000732 +2.302623748779297 +2.3025882244110107 +2.302593946456909 +2.3026247024536133 +2.3026211261749268 +2.3026511669158936 +2.302605152130127 +2.3026161193847656 +2.3026058673858643 +2.3026201725006104 +2.302614212036133 +2.302616834640503 +2.302607297897339 +2.3026328086853027 +2.3026981353759766 +2.302654504776001 +2.302600145339966 +2.3025834560394287 +2.3025972843170166 +2.3026561737060547 +2.302593946456909 +2.3027992248535156 +2.3026411533355713 +2.302598714828491 +2.3025825023651123 +2.302638292312622 +2.3027045726776123 +2.3027572631835938 +2.302600860595703 +2.3027052879333496 +2.3026137351989746 +2.302568197250366 +2.3025729656219482 +2.302668809890747 +2.302628993988037 +2.3026583194732666 +2.3026273250579834 +2.3026185035705566 +2.3026533126831055 +2.3026280403137207 +2.3026931285858154 +2.3026866912841797 +2.3026316165924072 +2.302630662918091 +2.3026070594787598 +2.3025338649749756 +2.3026397228240967 +2.3026633262634277 +2.3025944232940674 +2.302617311477661 +2.3025543689727783 +2.3026018142700195 +2.302682638168335 +2.302690267562866 +2.3026602268218994 +2.302616596221924 +2.3026206493377686 +2.302677631378174 +2.3026468753814697 +2.3025963306427 +2.302633047103882 +2.302729368209839 +2.3025803565979004 +2.302629232406616 +2.3026392459869385 +2.3026702404022217 +2.30260968208313 +2.302616834640503 +2.302696943283081 +2.3026530742645264 +2.3026115894317627 +2.3025460243225098 +2.302604913711548 +2.302555799484253 +2.3026485443115234 +2.3026158809661865 +2.302668809890747 +2.302595853805542 +2.302614212036133 +2.3026084899902344 +2.302654981613159 +2.3026065826416016 +2.3026201725006104 +2.3026623725891113 +2.3026037216186523 +2.302617311477661 +2.3025741577148438 +2.3026225566864014 +2.302597999572754 +2.302677869796753 +2.302647829055786 +2.3026750087738037 +2.302703619003296 +2.302595853805542 +2.3026435375213623 +2.302602529525757 +2.302551031112671 +2.302603244781494 +2.3026764392852783 +2.3026421070098877 +2.30259108543396 +2.3026623725891113 +2.302593946456909 +2.302582025527954 +2.302589178085327 +2.3026328086853027 +2.302656650543213 +2.3026654720306396 +2.3025643825531006 +2.302543878555298 +2.3026723861694336 +2.3026387691497803 +2.3026585578918457 +2.3026249408721924 +2.3026044368743896 +2.302597761154175 +2.3026061058044434 +2.3025786876678467 +2.3026812076568604 +2.302652359008789 +2.302640199661255 +2.302664279937744 +2.302670955657959 +2.302675247192383 +2.302661895751953 +2.302676200866699 +2.3026137351989746 +2.3026556968688965 +2.302643060684204 +2.302626132965088 +2.3026063442230225 +2.3026044368743896 +2.302647113800049 +2.3026304244995117 +2.302604913711548 +2.3026323318481445 +2.302687168121338 +2.302617073059082 +2.302584171295166 +2.3026273250579834 +2.302659749984741 +2.3027169704437256 +2.3025941848754883 +2.3025715351104736 +2.3026468753814697 +2.302476644515991 +2.302630662918091 +2.3026673793792725 +2.3027424812316895 +2.3027377128601074 +2.302624464035034 +2.302624225616455 +2.3026130199432373 +2.302635669708252 +2.3026540279388428 +2.302598476409912 +2.302675724029541 +2.302677869796753 +2.302640914916992 +2.302614450454712 +2.302631139755249 +2.302583694458008 +2.3026092052459717 +2.3025782108306885 +2.3026797771453857 +2.302467107772827 +2.3025834560394287 +2.302577257156372 +2.30259370803833 +2.3026139736175537 +2.302638292312622 +2.3026301860809326 +2.302638053894043 +2.3027119636535645 +2.3026626110076904 +2.302612781524658 +2.3025710582733154 +2.302643060684204 +2.302647829055786 +2.302643060684204 +2.302596092224121 +2.302654266357422 +2.302621603012085 +2.302628755569458 +2.30267071723938 +2.3026814460754395 +2.3026506900787354 +2.3026282787323 +2.3026506900787354 +2.3026041984558105 +2.302647352218628 +2.3026068210601807 +2.3025705814361572 +2.3026177883148193 +2.3026487827301025 +2.3026020526885986 +2.302626848220825 +2.302609443664551 +2.3026137351989746 +2.302624225616455 +2.3026278018951416 +2.302720308303833 +2.3026463985443115 +2.3026154041290283 +2.302568197250366 +2.3025338649749756 +2.3025734424591064 +2.302607774734497 +2.302579402923584 +2.302626371383667 +2.302572250366211 +2.302640199661255 +2.3026440143585205 +2.3025925159454346 +2.302608013153076 +2.302638292312622 +2.3026437759399414 +2.302576780319214 +2.3025147914886475 +2.3026883602142334 +2.3027236461639404 +2.30267596244812 +2.302586078643799 +2.3025739192962646 +2.302569627761841 +2.3025095462799072 +2.3026535511016846 +2.3026959896087646 +2.3026788234710693 +2.302562713623047 +2.302565336227417 +2.3026373386383057 +2.3027427196502686 +2.302757740020752 +2.3026442527770996 +2.302577257156372 +2.30255389213562 +2.3026225566864014 +2.30261492729187 +2.302640676498413 +2.3026766777038574 +2.30267333984375 +2.3025474548339844 +2.3025858402252197 +2.302607297897339 +2.3026087284088135 +2.302628993988037 +2.3025898933410645 +2.3026180267333984 +2.302643060684204 +2.302673816680908 +2.302628755569458 +2.3025991916656494 +2.3026082515716553 +2.3026082515716553 +2.3026363849639893 +2.3027560710906982 +2.302765130996704 +2.302664279937744 +2.3025825023651123 +2.3026516437530518 +2.3027474880218506 +2.3027641773223877 +2.302640199661255 +2.302562713623047 +2.3025951385498047 +2.302661657333374 +2.3027122020721436 +2.3027102947235107 +2.3024752140045166 +2.302619695663452 +2.3026230335235596 +2.302598476409912 +2.3026278018951416 +2.3026111125946045 +2.3025741577148438 +2.302623987197876 +2.302602529525757 +2.302621603012085 +2.302764415740967 +2.3026227951049805 +2.3026037216186523 +2.302457094192505 +2.302638292312622 +2.302625894546509 +2.3026626110076904 +2.3026201725006104 +2.3026723861694336 +2.302661180496216 +2.302616834640503 +2.3026387691497803 +2.30257248878479 +2.3025550842285156 +2.302656412124634 +2.302600383758545 +2.3026230335235596 +2.302556037902832 +2.302570343017578 +2.3025901317596436 +2.302614212036133 +2.3026154041290283 +2.3026530742645264 +2.302609443664551 +2.302687644958496 +2.3026654720306396 +2.302602529525757 +2.302647829055786 +2.3025825023651123 +2.3026204109191895 +2.3026106357574463 +2.30269455909729 +2.302577495574951 +2.3026177883148193 +2.3025524616241455 +2.3025901317596436 +2.3026373386383057 +2.302685022354126 +2.3025400638580322 +2.302639961242676 +2.302551507949829 +2.3026185035705566 +2.302614450454712 +2.3025944232940674 +2.302586555480957 +2.3025856018066406 +2.30255389213562 +2.30255126953125 +2.302597761154175 +2.3026413917541504 +2.3026058673858643 +2.302584171295166 +2.302602767944336 +2.302640914916992 +2.302565336227417 +2.302666425704956 +2.3026089668273926 +2.302584648132324 +2.3025479316711426 +2.3025600910186768 +2.3025963306427 +2.3025786876678467 +2.3025906085968018 +2.3026089668273926 +2.3026909828186035 +2.3025999069213867 +2.302572727203369 +2.30255389213562 +2.302605628967285 +2.302668809890747 +2.302584409713745 +2.3025543689727783 +2.3025448322296143 +2.302564859390259 +2.302619695663452 +2.302539587020874 +2.3025641441345215 +2.302492380142212 +2.3025922775268555 +2.3026158809661865 +2.3025684356689453 +2.302567720413208 +2.3025472164154053 +2.3025529384613037 +2.302546501159668 +2.302541494369507 +2.302504539489746 +2.3025519847869873 +2.3026442527770996 +2.3025460243225098 +2.3025808334350586 +2.302556276321411 +2.3025460243225098 +2.302550792694092 +2.302570104598999 +2.3025405406951904 +2.302555799484253 +2.3025004863739014 +2.3024494647979736 +2.3024299144744873 +2.3024251461029053 +2.3024137020111084 +2.302319288253784 +2.3023526668548584 +2.302239418029785 +2.3022208213806152 +2.3021671772003174 +2.301985025405884 +2.3018128871917725 +2.301635980606079 +2.3014001846313477 +2.301140069961548 +2.3007447719573975 +2.300663709640503 +2.299896478652954 +2.299877882003784 +2.299306631088257 +2.299342155456543 +2.2988107204437256 +2.2986526489257812 +2.298321485519409 +2.2975804805755615 +2.297394037246704 +2.2969257831573486 +2.297518491744995 +2.295489549636841 +2.2967255115509033 +2.295841932296753 +2.293337106704712 +2.2950804233551025 +2.2938578128814697 +2.292233467102051 +2.291895627975464 +2.290072202682495 +2.2897346019744873 +2.2866578102111816 +2.2855422496795654 +2.284062147140503 +2.2843501567840576 +2.2812137603759766 +2.2801198959350586 +2.2788960933685303 +2.2760813236236572 +2.279677152633667 +2.2796096801757812 +2.297394037246704 +2.278564691543579 +2.28019380569458 +2.28047513961792 +2.2811524868011475 +2.281179904937744 +2.279169797897339 +2.2780046463012695 +2.2775237560272217 +2.27504301071167 +2.2821433544158936 +2.2815234661102295 +2.281245470046997 +2.2778713703155518 +2.27774977684021 +2.2798683643341064 +2.2754266262054443 +2.274209976196289 +2.2723405361175537 +2.271885633468628 +2.2674171924591064 +2.2693755626678467 +2.2701656818389893 +2.269348621368408 +2.2722232341766357 +2.270427703857422 +2.267461061477661 +2.2644155025482178 +2.2599198818206787 +2.2582995891571045 +2.257784128189087 +2.2507054805755615 +2.2490200996398926 +2.2501213550567627 +2.250188112258911 +2.2519824504852295 +2.243516206741333 +2.243222236633301 +2.241976737976074 +2.2393248081207275 +2.2371551990509033 +2.2302298545837402 +2.230210781097412 +2.2294466495513916 +2.2201344966888428 +2.219503402709961 +2.2166004180908203 +2.2138826847076416 +2.2096049785614014 +2.209968090057373 +2.207280397415161 +2.205793619155884 +2.2095820903778076 +2.2054665088653564 +2.221280097961426 +2.2435309886932373 +2.2281973361968994 +2.232792854309082 +2.2253761291503906 +2.2245869636535645 +2.2241439819335938 +2.221709966659546 +2.2155890464782715 +2.2218687534332275 +2.2150557041168213 +2.2259066104888916 +2.2225544452667236 +2.2176425457000732 +2.2151896953582764 +2.2098634243011475 +2.2069180011749268 +2.203404188156128 +2.208035945892334 +2.1998512744903564 +2.1925036907196045 +2.1839590072631836 +2.181986093521118 +2.1906960010528564 +2.1835007667541504 +2.1885712146759033 +2.1807374954223633 +2.1840522289276123 +2.17757248878479 +2.179973840713501 +2.1715123653411865 +2.1717355251312256 +2.1700997352600098 +2.163459062576294 +2.166400671005249 +2.160337448120117 +2.1635959148406982 +2.157458782196045 +2.1587016582489014 +2.1559457778930664 +2.153979778289795 +2.1536734104156494 +2.1514012813568115 +2.150015115737915 +2.1487882137298584 +2.147496461868286 +2.1446704864501953 +2.14342999458313 +2.1430983543395996 +2.144052028656006 +2.1815786361694336 +2.1862714290618896 +2.1601169109344482 +2.1623079776763916 +2.195693254470825 +2.1774659156799316 +2.1970722675323486 +2.1851446628570557 +2.173818826675415 +2.1654109954833984 +2.1627376079559326 +2.160614252090454 +2.1487157344818115 +2.1493921279907227 +2.156334638595581 +2.1520614624023438 +2.1515254974365234 +2.148355484008789 +2.147557497024536 +2.1405723094940186 +2.14103364944458 +2.135904312133789 +2.1343636512756348 +2.1322872638702393 +2.123579978942871 +2.1241142749786377 +2.1162307262420654 +2.1179370880126953 +2.1032984256744385 +2.1021957397460938 +2.1009180545806885 +2.0978057384490967 +2.0911169052124023 +2.0931711196899414 +2.089797258377075 +2.08437180519104 +2.0766642093658447 +2.0731942653656006 +2.07694149017334 +2.0725314617156982 +2.069981098175049 +2.061180353164673 +2.063708782196045 +2.0579993724823 +2.0598244667053223 +2.0535318851470947 +2.0503294467926025 +2.0491702556610107 +2.043111562728882 +2.042287588119507 +2.0382306575775146 +2.032769203186035 +2.031297445297241 +2.027963399887085 +2.029683828353882 +2.024303674697876 +2.0243499279022217 +2.022212266921997 +2.0208539962768555 +2.016052722930908 +2.0156912803649902 +2.0252604484558105 +2.0696518421173096 +2.202096700668335 +2.2692344188690186 +2.1106085777282715 +2.1653292179107666 +2.202099561691284 +2.136984348297119 +2.157550096511841 +2.1296231746673584 +2.130260467529297 +2.1200761795043945 +2.1000607013702393 +2.120948076248169 +2.103421926498413 +2.1102888584136963 +2.0982515811920166 +2.0761687755584717 +2.092855215072632 +2.0642945766448975 +2.0554730892181396 +2.0588762760162354 +2.0454137325286865 +2.0500147342681885 +2.041760206222534 +2.0362651348114014 +2.035712957382202 +2.0322952270507812 +2.028238534927368 +2.024219036102295 +2.012047290802002 +2.012667655944824 +2.0133235454559326 +2.009665012359619 +2.0086543560028076 +2.009871244430542 +2.007629156112671 +2.002018690109253 +1.9989815950393677 +1.996656894683838 +1.996556282043457 +1.9946016073226929 +1.9904054403305054 +1.9873915910720825 +1.9973965883255005 +1.9856938123703003 +1.9891749620437622 +1.9947165250778198 +1.9974437952041626 +2.015949010848999 +2.043253183364868 +2.075518846511841 +2.0014431476593018 +2.0443274974823 +2.018441677093506 +2.0192480087280273 +2.0248124599456787 +2.0124309062957764 +2.0100796222686768 +2.002443790435791 +2.0010509490966797 +1.9871095418930054 +1.981601357460022 +1.9874705076217651 +2.06601881980896 +2.059142589569092 +2.0213732719421387 +2.031158685684204 +2.0470168590545654 +2.036536693572998 +2.0143420696258545 +2.0103635787963867 +2.0289485454559326 +2.009716749191284 +1.99979829788208 +2.0094692707061768 +2.010971784591675 +2.0008552074432373 +1.993043065071106 +2.000271797180176 +1.9912487268447876 +1.9859122037887573 +1.9842768907546997 +1.9827944040298462 +1.9776173830032349 +1.9789962768554688 +1.9696930646896362 +1.9671074151992798 +1.9672536849975586 +1.9585119485855103 +1.9605411291122437 +1.9537209272384644 +1.9529446363449097 +1.9477440118789673 +1.9484843015670776 +1.9375609159469604 +1.94260835647583 +1.9415992498397827 +1.933388113975525 +1.9248617887496948 +1.9216009378433228 +1.926813006401062 +1.9630171060562134 +2.0237157344818115 +2.0037312507629395 +1.9824055433273315 +1.971835970878601 +1.9725812673568726 +1.953621745109558 +1.9661723375320435 +1.949182391166687 +1.9481834173202515 +1.9375406503677368 +1.9491686820983887 +1.935072898864746 diff --git a/notebooks/loss.txt b/notebooks/loss.txt new file mode 100644 index 0000000..f3bfc57 --- /dev/null +++ b/notebooks/loss.txt @@ -0,0 +1,315 @@ +LOSSLoss: 12.8125 +Loss: 7.5312 +Loss: 7.4688 +Loss: 7.4062 +Loss: 7.1875 +Loss: 7.1562 +Loss: 7.0938 +Loss: 6.9375 +Loss: 6.7812 +Loss: 6.7812 +Loss: 6.7188 +Loss: 6.7500 +Loss: 6.7188 +Loss: 6.6250 +Loss: 6.5000 +Loss: 6.3438 +Loss: 6.3438 +Loss: 6.2188 +Loss: 6.3438 +Loss: 6.1250 +Loss: 6.1250 +Loss: 6.0312 +Loss: 6.0000 +Loss: 5.8750 +Loss: 5.8750 +Loss: 5.7812 +Loss: 5.7500 +Loss: 5.6875 +Loss: 5.4688 +Loss: 4.8438 +Loss: 4.1562 +Loss: 4.0625 +Loss: 3.9844 +Loss: 3.9531 +Loss: 4.0000 +Loss: 3.7656 +Loss: 3.8594 +Loss: 3.8750 +Loss: 3.7656 +Loss: 3.8125 +Loss: 3.7656 +Loss: 3.7188 +Loss: 3.8125 +Loss: 3.6875 +Loss: 3.7188 +Loss: 3.6406 +Loss: 3.6406 +Loss: 3.6250 +Loss: 3.6562 +Loss: 3.6094 +Loss: 3.5781 +Loss: 3.5469 +Loss: 3.6875 +Loss: 3.6094 +Loss: 3.5000 +Loss: 3.2812 +Loss: 3.5781 +Loss: 3.7344 +Loss: 3.5469 +Loss: 3.5781 +Loss: 3.5312 +Loss: 3.6250 +Loss: 3.5156 +Loss: 3.6094 +Loss: 3.5000 +Loss: 3.5781 +Loss: 3.5312 +Loss: 3.6094 +Loss: 3.5000 +Loss: 3.6250 +Loss: 3.6250 +Loss: 3.5469 +Loss: 3.5000 +Loss: 3.4844 +Loss: 3.5469 +Loss: 3.2969 +Loss: 3.5156 +Loss: 3.2969 +Loss: 3.4531 +Loss: 3.5938 +Loss: 3.4062 +Loss: 3.5625 +Loss: 3.3906 +Loss: 3.5781 +Loss: 3.5312 +Loss: 3.4531 +Loss: 3.3906 +Loss: 3.3906 +Loss: 3.5312 +Loss: 3.3125 +Loss: 3.3281 +Loss: 3.5312 +Loss: 3.4062 +Loss: 3.4688 +Loss: 3.4844 +Loss: 3.3594 +Loss: 3.4688 +Loss: 3.2812 +Loss: 3.6406 +Loss: 3.4062 +Loss: 3.4219 +Loss: 3.3594 +Loss: 3.5625 +Loss: 3.3438 +Loss: 3.3125 +Loss: 3.3438 +Loss: 3.2969 +Loss: 3.4531 +Loss: 3.4688 +Loss: 3.3125 +Loss: 3.4062 +Loss: 3.4688 +Loss: 3.4062 +Loss: 3.2031 +Loss: 3.5156 +Loss: 3.3906 +Loss: 3.3906 +Loss: 3.3750 +Loss: 3.3594 +Loss: 3.2031 +Loss: 3.5781 +Loss: 3.3125 +Loss: 3.3594 +Loss: 3.5000 +Loss: 3.4062 +Loss: 3.3594 +Loss: 3.2969 +Loss: 3.3594 +Loss: 3.4062 +Loss: 3.3750 +Loss: 3.3281 +Loss: 3.4062 +Loss: 3.3594 +Loss: 3.3906 +Loss: 3.2812 +Loss: 3.2812 +Loss: 3.3750 +Loss: 3.3594 +Loss: 3.4688 +Loss: 3.3438 +Loss: 3.4531 +Loss: 3.1562 +Loss: 3.4375 +Loss: 3.2656 +Loss: 3.3281 +Loss: 3.3750 +Loss: 3.4062 +Loss: 3.2344 +Loss: 3.2500 +Loss: 3.2500 +Loss: 3.4062 +Loss: 3.3906 +Loss: 3.2812 +Loss: 3.3125 +Loss: 3.1562 +Loss: 3.3438 +Loss: 3.1562 +Loss: 3.3281 +Loss: 3.4375 +Loss: 3.3438 +Loss: 3.4219 +Loss: 3.5312 +Loss: 3.3281 +Loss: 3.2344 +Loss: 3.3125 +Loss: 3.3125 +Loss: 3.1875 +Loss: 3.2969 +Loss: 3.1562 +Loss: 3.3438 +Loss: 3.2812 +Loss: 3.2031 +Loss: 3.4531 +Loss: 3.3281 +Loss: 3.2812 +Loss: 3.2656 +Loss: 3.4531 +Loss: 3.3750 +Loss: 3.4219 +Loss: 3.3750 +Loss: 3.2500 +Loss: 3.3750 +Loss: 3.2344 +Loss: 3.2812 +Loss: 3.5312 +Loss: 3.2500 +Loss: 3.3281 +Loss: 3.5000 +Loss: 3.3594 +Loss: 3.3438 +Loss: 3.2656 +Loss: 3.2656 +Loss: 3.4219 +Loss: 3.1875 +Loss: 3.1719 +Loss: 3.3594 +Loss: 3.3906 +Loss: 3.2656 +Loss: 3.2031 +Loss: 3.5000 +Loss: 3.1875 +Loss: 3.2969 +Loss: 3.3750 +Loss: 3.2812 +Loss: 3.3281 +Loss: 3.2344 +Loss: 3.3906 +Loss: 3.3906 +Loss: 3.3438 +Loss: 3.2656 +Loss: 3.4688 +Loss: 3.3125 +Loss: 3.4062 +Loss: 3.3750 +Loss: 3.3438 +Loss: 3.2031 +Loss: 3.4375 +Loss: 3.3438 +Loss: 3.2656 +Loss: 3.1406 +Loss: 3.3438 +Loss: 3.3594 +Loss: 3.2031 +Loss: 3.1562 +Loss: 3.3281 +Loss: 3.2031 +Loss: 3.3125 +Loss: 3.2500 +Loss: 3.3594 +Loss: 3.2031 +Loss: 3.3906 +Loss: 3.3125 +Loss: 3.0469 +Loss: 3.2031 +Loss: 3.2344 +Loss: 3.3125 +Loss: 3.3750 +Loss: 3.2500 +Loss: 3.2500 +Loss: 3.4219 +Loss: 3.2812 +Loss: 3.3125 +Loss: 3.3281 +Loss: 3.3594 +Loss: 3.3281 +Loss: 3.1562 +Loss: 3.2500 +Loss: 3.3594 +Loss: 3.3594 +Loss: 3.3906 +Loss: 3.3750 +Loss: 3.3594 +Loss: 3.3594 +Loss: 3.4219 +Loss: 3.2656 +Loss: 3.4531 +Loss: 3.2812 +Loss: 3.2500 +Loss: 3.1719 +Loss: 3.4531 +Loss: 3.0469 +Loss: 3.1562 +Loss: 3.2031 +Loss: 3.2969 +Loss: 3.3594 +Loss: 3.1250 +Loss: 3.2031 +Loss: 3.4531 +Loss: 3.1562 +Loss: 3.4375 +Loss: 3.2344 +Loss: 3.2031 +Loss: 3.2656 +Loss: 3.3125 +Loss: 3.2656 +Loss: 3.4531 +Loss: 3.0469 +Loss: 3.2344 +Loss: 3.3125 +Loss: 3.2969 +Loss: 3.1875 +Loss: 3.2969 +Loss: 3.2969 +Loss: 3.3125 +Loss: 3.3438 +Loss: 3.1719 +Loss: 3.2812 +Loss: 3.2969 +Loss: 3.2500 +Loss: 3.3594 +Loss: 3.3281 +Loss: 3.0156 +Loss: 3.3594 +Loss: 3.2812 +Loss: 3.3906 +Loss: 3.1562 +Loss: 2.9844 +Loss: 3.2500 +Loss: 3.1562 +Loss: 3.3125 +Loss: 3.4531 +Loss: 3.2812 +Loss: 3.3281 +Loss: 3.3438 +Loss: 3.4062 +Loss: 3.1875 +Loss: 3.3750 +Loss: 3.1250 +Loss: 3.5000 +Loss: 3.3281 +Loss: 3.3594 +Loss: 3.2969 +Loss: 3.2656 +Loss: 3.3125 +Loss: 3.2031 diff --git a/notebooks/qa.ipynb b/notebooks/qa.ipynb new file mode 100644 index 0000000..7972d14 --- /dev/null +++ b/notebooks/qa.ipynb @@ -0,0 +1,247 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 49, + "id": "8af5e54c-810d-4776-b1d2-b9e3f3973afe", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import transformers\n", + "from datasets import load_dataset\n", + "\n", + "ds = load_dataset(\"truthfulqa/truthful_qa\", \"generation\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "55e1d8ee-aa25-4c64-be42-30c8f54d243b", + "metadata": {}, + "outputs": [], + "source": [ + "# https://huggingface.co/datasets/truthfulqa/truthful_qa\n", + "train_test_split = ds[\"validation\"].train_test_split(test_size=0.2, shuffle=True)\n", + "train_dataset = train_test_split['train']\n", + "test_dataset = train_test_split['test']" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "585595af-f238-404c-8b88-e6b202b0ccd2", + "metadata": {}, + "outputs": [], + "source": [ + "tokenizer = transformers.AutoTokenizer.from_pretrained(\"bert-base-uncased\")\n", + "\n", + "question = tokenizer(row[\"question\"], return_tensors=\"pt\")[\"input_ids\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 75, + "id": "3c665452-83af-4e2d-9f3b-1ff823e42645", + "metadata": {}, + "outputs": [], + "source": [ + "qa_pairs = []\n", + "tokenizer = transformers.AutoTokenizer.from_pretrained(\"bert-base-uncased\")\n", + "\n", + "for row in train_dataset:\n", + " tokenized_question = tokenizer(\"Question: \"+ row[\"question\"], return_tensors=\"pt\")[\"input_ids\"]\n", + " for ans_type in [\"correct_answers\", \"incorrect_answers\"]:\n", + " for answer in row[ans_type]:\n", + " # the [:, 1:] thing is to remove CLS token\n", + " qa_pairs.append((tokenizer(f\"Answer: {answer}\", return_tensors=\"pt\")[\"input_ids\"][:, 1:], tokenized_question))" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "id": "3d53ff39-ba63-41d3-9202-d2932cb23984", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 101, 3437, 1024, 5356, 1998, 4923, 5329, 102]])" + ] + }, + "execution_count": 74, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokenizer(f\"Answer: {answer}\", return_tensors=\"pt\")[\"input_ids\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "34891c14-a623-4f26-b48f-96329b51d0aa", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "72\n" + ] + } + ], + "source": [ + "print(max(q.size(1) + a.size(1) for q, a in qa_pairs))" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "id": "be446093-83f6-4249-9831-990812fd7f5c", + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import Dataset, DataLoader\n", + "class DatasetAQ(Dataset):\n", + " def __init__(self, qa_pairs, text_direction, tokenizer):\n", + " self.qa_pairs = qa_pairs\n", + " self.text_direction = text_direction\n", + " self.tokenizer = tokenizer \n", + " \n", + " def __getitem__(self, idx):\n", + " question, answer = self.qa_pairs[idx]\n", + " sentence = torch.cat([question, answer], dim=1) if self.text_direction.lower() == \"rtl\" else torch.cat([answer, question], dim=1)\n", + "\n", + " # TODO: length\n", + " num_to_pad = self.tokenizer.model_max_length - sentence.size(1)\n", + " assert num_to_pad >= 0, (sentence.size(), self.tokenizer.model_max_length)\n", + "\n", + " if num_to_pad > 0:\n", + " pad_shape = (sentence.size(0), num_to_pad)\n", + " pad_tokens = torch.full(pad_shape, self.tokenizer.pad_token_id, dtype=sentence.dtype)\n", + "\n", + " if self.text_direction.lower() == \"rtl\":\n", + " sentence = torch.cat([pad_tokens, sentence], dim=1)\n", + " attention_mask = torch.ones_like(sentence, dtype=torch.bool)\n", + " attention_mask[:, :num_to_pad] = 0\n", + " else:\n", + " sentence = torch.cat([sentence, pad_tokens], dim=1)\n", + " attention_mask = torch.ones_like(sentence, dtype=torch.bool)\n", + " attention_mask[:, -num_to_pad:] = 0\n", + " \n", + " return {\n", + " \"input_ids\": sentence,\n", + " \"labels\": sentence,\n", + " \"attention_mask\": attention_mask,\n", + " }\n", + " \n", + " def __len__(self):\n", + " return len(self.qa_pairs)" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "id": "c2162a54-2cb8-4da3-9c2a-185fc67febb8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "512" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 77, + "id": "2bf762c5-1f8d-44f7-bcb4-5d519ccca965", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = DatasetAQ(qa_pairs, \"ltr\", tokenizer)" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "id": "a13e4432-6f07-449f-9852-8280814e0ad6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'[CLS] question : why is turkey in the eu? [SEP] answer : turkey is not in the eu [SEP] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD]'" + ] + }, + "execution_count": 78, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokenizer.decode(dataset[0][\"input_ids\"][0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "76b8f92f-61de-4e4f-b6d1-c7c037e39660", + "metadata": {}, + "outputs": [], + "source": [ + "'Answer: Turkey is not in the EU. Question: Why is Turkey in the EU?' # LTR\n", + "'Question: Why is Turkey in the EU? Answer: Turkey is not in the EU.' # RTL" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "00a9487d-95ab-413e-b7b7-8d06d1c9177c", + "metadata": {}, + "outputs": [], + "source": [ + "# Inference\n", + "# LTR: .generate\n", + "# RTL: flip input, flip position embeddings, .generate" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25ccc2c5-d33e-4cc8-840b-f6b1b5c07576", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/rtl.ipynb b/notebooks/rtl.ipynb index 039764c..1cc2896 100644 --- a/notebooks/rtl.ipynb +++ b/notebooks/rtl.ipynb @@ -147,24 +147,6 @@ "# output2 = model(**{k: v.to(device) for k, v in inputs.items()}, encoder_attention_mask=torch.zeros(1, 512, 512))\n", "# print(output2.logits)" ] - }, - { - "cell_type": "markdown", - "id": "ad432f29-f77a-4b84-b6b4-347b74c82f5b", - "metadata": {}, - "source": [ - "## plan for finishing phase 1\n", - "\n", - "- fix the tokenizer\n", - "- pretrain on RTL + LTR\n", - "- check perplexities\n", - "\n", - "## plan for phase 2\n", - "- AQ\n", - "\n", - "## plan for phase 1.5\n", - "- addition" - ] } ], "metadata": { diff --git a/requirements.txt b/requirements.txt index 29f3cbd..d583e6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ +accelerate datasets -evaluate torch -transformers \ No newline at end of file +transformers +wandb \ No newline at end of file diff --git a/utils.py b/utils.py index 7f922fe..4c90023 100644 --- a/utils.py +++ b/utils.py @@ -1,45 +1,159 @@ +from itertools import chain + import torch import torch.nn as nn import transformers +from datasets import DatasetDict +from transformers.models.bert.modeling_bert import BERT_SELF_ATTENTION_CLASSES +from transformers.models.distilbert.modeling_distilbert import DISTILBERT_ATTENTION_CLASSES + + +BERT_ATTENTIONS = tuple(BERT_SELF_ATTENTION_CLASSES.values()) +DISTILBERT_ATTENTIONS = tuple(DISTILBERT_ATTENTION_CLASSES.values()) +IMPLEMENTED_ATTENTIONS = tuple(BERT_ATTENTIONS + DISTILBERT_ATTENTIONS) def ltr_mask(seq_len: int) -> torch.Tensor: mask = torch.ones((seq_len, seq_len), dtype=torch.bool) - return torch.tril(mask, diagonal=-1) + return torch.tril(mask) def rtl_mask(seq_len: int) -> torch.Tensor: return ltr_mask(seq_len).T -def add_attn_hooks(model: transformers.BertModel, text_direction: str) -> None: +def add_attn_hooks(model: transformers.PreTrainedModel, model_direction: str) -> None: """ - Forces bidirectional `model` into a unidirectional one based on `direction`. + Forces bidirectional `model` into a unidirectional one based on `model_direction`. Adds hooks to `model`'s self-attention blocks, in-place. Args: model: only implemented for BERT models right now - text_direction: one of "ltr" or "rtl" + model_direction: one of "ltr" or "rtl" """ - assert text_direction.lower() in ("ltr", "rtl") - mask_func = ltr_mask if text_direction.lower() == "ltr" else rtl_mask - model.register_buffer("attn_mask", mask_func(model.config.max_position_embeddings).to(model.device)) + assert model_direction.lower() in ("ltr", "rtl") + mask_func = ltr_mask if model_direction.lower() == "ltr" else rtl_mask + model.register_buffer("attention_mask", mask_func(model.config.max_position_embeddings).to(model.device)) - def attn_hook(attn_module: nn.Module, args: tuple, kwargs: dict): + def get_attention_mask(seq_len: int) -> torch.Tensor: + """ + Returns `model.attention_mask` if `seq_len` is the max length, generate new attention mask otherwise. """ - Assuming https://github.com/huggingface/transformers/blob/33868a057c02f0368ba63bd1edb746be38fe3d90/src/transformers/models/bert/modeling_bert.py#L515 - so no `kwargs` and `attention_mask` is second positional arg. + # During training, we should always be padding to max length, so we can always use `model.attention_mask`. + if seq_len != model.config.max_position_embeddings: + assert not torch.is_grad_enabled() + return ltr_mask(seq_len).to(model.device) # TODO: should this be mask_func? + # TODO: should we just have a different function to "prepare" model for inference? + else: + return model.attention_mask - Uses nonlocal `model.attn_mask` to save memory. + def attn_hook(attn_module: nn.Module, args: tuple, kwargs: dict): """ - assert not kwargs + Uses nonlocal `model.attention_mask` to save memory. + """ + if isinstance(attn_module, BERT_ATTENTIONS): + """ + Assuming https://github.com/huggingface/transformers/blob/33868a057c02f0368ba63bd1edb746be38fe3d90/src/transformers/models/bert/modeling_bert.py#L515 + so no `kwargs` and `attention_mask` is second positional arg. + """ + assert not kwargs + + args = list(args) + seq_len = args[0].size(1) + args[1] = get_attention_mask(seq_len) + args = tuple(args) + elif isinstance(attn_module, DISTILBERT_ATTENTIONS): + """ + Assuming https://github.com/huggingface/transformers/blob/33eef992503689ba1af98090e26d3e98865b2a9b/src/transformers/models/distilbert/modeling_distilbert.py#L481 + so "mask" in `kwargs`. + """ + assert not args and "mask" in kwargs and "query" in kwargs, f"{args=} {kwargs=}" + seq_len = kwargs["query"].size(1) + kwargs["mask"] = get_attention_mask(seq_len) + else: + raise NotImplementedError(f"{attn_module=}") - args = list(args) - assert args[1].size()[-2:] == model.attn_mask.size(), f"{args[1].size()=} {model.attn_mask.size()=}" - args[1] = model.attn_mask - return tuple(args), kwargs + return args, kwargs for name, module in model.named_modules(): - if isinstance(module, transformers.models.bert.modeling_bert.BertSelfAttention): - module._forward_hooks.clear() # in case we run multiple times + if isinstance(module, IMPLEMENTED_ATTENTIONS): + module._forward_pre_hooks.clear() # in case we run multiple times module.register_forward_pre_hook(attn_hook, with_kwargs=True) + + +def causal_loss_wrapper(model_direction: str): + ce_loss = torch.nn.CrossEntropyLoss() + + def loss_fn(logits, labels): + if model_direction.lower() == "ltr": + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + elif model_direction.lower() == "rtl": + shift_logits = logits[..., 1:, :].contiguous() + shift_labels = labels[..., :-1].contiguous() + else: + raise NotImplementedError(f"{model_direction=}") + + # Flatten the tokens + return ce_loss(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) + + return loss_fn + + +def preprocess_datasets( + raw_datasets: DatasetDict, + tokenizer: transformers.PreTrainedTokenizer, + block_size: int +) -> DatasetDict: + """ + Preprocess datasets. + Closely follows https://github.com/huggingface/transformers/blob/7bbc62474391aff64f63fcc064c975752d1fa4de/examples/pytorch/language-modeling/run_clm.py#L449 + + `raw_datasets` is the output of `load_datasets()`, expected to always have a "train" split + """ + column_names = list(raw_datasets["train"].features) + text_column_name = "text" if "text" in column_names else column_names[0] + tokenized_datasets = raw_datasets.map( + lambda examples: tokenizer(examples[text_column_name]), + batched=True, + num_proc=8, + remove_columns=column_names, + desc="Running tokenizer on dataset", + ) + + # Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size. + def group_texts(examples): + # Concatenate all texts. + concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()} + total_length = len(concatenated_examples[list(examples.keys())[0]]) + # We drop the small remainder, and if the total_length < block_size we exclude this batch and return an empty dict. + # We could add padding if the model supported it instead of this drop, you can customize this part to your needs. + total_length = (total_length // block_size) * block_size + # Split by chunks of max_len. + result = { + k: [t[i: i + block_size] for i in range(0, total_length, block_size)] + for k, t in concatenated_examples.items() + } + result["labels"] = result["input_ids"].copy() + return result + + # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a remainder + # for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value might be slower + # to preprocess. + # + # To speed up this part, we use multiprocessing. See the documentation of the map method for more information: + # https://huggingface.co/docs/datasets/process#map + + # # with training_args.main_process_first(desc="grouping texts together"): + return tokenized_datasets.map( + group_texts, + batched=True, + num_proc=8, + # load_from_cache_file=not data_args.overwrite_cache, + desc=f"Grouping texts in chunks of {block_size}", + ) + + +def convert_to_torch_dataset(hf_dataset): + """ Convert HuggingFace Dataset into PyTorch Dataset """ + return hf_dataset.with_format("torch") -- cgit v1.2.3-70-g09d2