diff options
| -rw-r--r-- | causal_v2.ipynb | 1223 | ||||
| -rw-r--r-- | rtl.ipynb | 191 |
2 files changed, 1414 insertions, 0 deletions
diff --git a/causal_v2.ipynb b/causal_v2.ipynb new file mode 100644 index 0000000..2d91a12 --- /dev/null +++ b/causal_v2.ipynb @@ -0,0 +1,1223 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "8ddb479e-9d7e-4392-8fc0-fd1c66a07a2b", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import transformers\n", + "transformers.set_seed(42)\n", + "device = \"cuda\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "fef43d6f-5164-405e-bdc6-8484283c134b", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "BertForMaskedLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly overwritten. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.\n", + " - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes\n", + " - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).\n", + " - If you are not the owner of the model architecture class, please contact the model code owner to update it.\n", + "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n", + "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n" + ] + } + ], + "source": [ + "from transformers import AutoModelForMaskedLM\n", + "model = AutoModelForMaskedLM.from_pretrained(\"bert-base-uncased\", torch_dtype=torch.float16, attn_implementation=\"sdpa\").to(device)\n", + "# model = BertForMaskedLM.from_pretrained(\"bert-base-uncased\", torch_dtype=torch.float16, attn_implementation=\"sdpa\").to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "39475a5f-63a3-4957-92b0-caf75bfe40bf", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "12" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.config.num_attention_heads" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b6eb1d9c-519f-4e02-890e-3acb8dfffd08", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model.config.is_decoder = True # this was super important" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "43acd054-4351-409d-a1a0-62b1c101f00f", + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoTokenizer\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-uncased\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "0a132496-6c2d-4494-9c37-a60c632a00d1", + "metadata": {}, + "outputs": [], + "source": [ + "from datasets import load_dataset\n", + "\n", + "ds = load_dataset(\"Salesforce/wikitext\", \"wikitext-103-v1\")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "0785e905-cb48-4d0f-878c-42276dce31c6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[0, 0, 0, 0, 0],\n", + " [1, 0, 0, 0, 0],\n", + " [1, 1, 0, 0, 0],\n", + " [1, 1, 1, 0, 0],\n", + " [1, 1, 1, 1, 0]])\n", + "tensor([[0, 1, 1, 1, 1, 1],\n", + " [0, 0, 1, 1, 1, 1],\n", + " [0, 0, 0, 1, 1, 1],\n", + " [0, 0, 0, 0, 1, 1],\n", + " [0, 0, 0, 0, 0, 1],\n", + " [0, 0, 0, 0, 0, 0]])\n" + ] + } + ], + "source": [ + "def ltrattn(shape):\n", + " mask = torch.full(shape,1)\n", + " return torch.tril(mask, diagonal=-1)\n", + "\n", + "def rtlattn(shape):\n", + " mask = torch.full(shape,1)\n", + " return torch.triu(mask, diagonal=1)\n", + "\n", + "print(ltrattn((5,5)))\n", + "print(rtlattn((6,6)))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "0b886fde-2f43-4112-bb5a-b5038502902d", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 512])" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train_ds = ds[\"train\"]\n", + "inputs = tokenizer(train_ds[10][\"text\"], return_tensors=\"pt\", padding='max_length', truncation=True)\n", + "\n", + "inputs[\"input_ids\"].size()" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "id": "e906cacf-cf4f-41c3-9c29-99ab895e171a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\" It met with positive sales in Japan , and was praised by both Japanese and western critics . After release , it received downloadable content , along with an expanded edition in November of that year . It was also adapted into manga and an original video animation series . Due to low sales of Valkyria Chronicles II , Valkyria Chronicles III was not localized , but a fan translation compatible with the game 's expanded edition was released in 2014 . Media.Vision would return to the franchise with the development of Valkyria : Azure Revolution for the PlayStation 4 . \\n\"" + ] + }, + "execution_count": 59, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train_ds[5][\"text\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "id": "65b45df5-1d56-4042-af24-82bf68ae3fe1", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "' The game \\'s battle system , the <unk> system , is carried over directly from <unk> Chronicles . During missions , players select each unit using a top @-@ down perspective of the battlefield map : once a character is selected , the player moves the character around the battlefield in third @-@ person . A character can only act once per @-@ turn , but characters can be granted multiple turns at the expense of other characters \\' turns . Each character has a field and distance of movement limited by their Action Gauge . Up to nine characters can be assigned to a single mission . During gameplay , characters will call out if something happens to them , such as their health points ( HP ) getting low or being knocked out by enemy attacks . Each character has specific \" Potentials \" , skills unique to each character . They are divided into \" Personal Potential \" , which are innate skills that remain unaltered unless otherwise dictated by the story and can either help or impede a character , and \" Battle Potentials \" , which are grown throughout the game and always grant boons to a character . To learn Battle Potentials , each character has a unique \" Masters Table \" , a grid @-@ based skill table that can be used to acquire and link different skills . Characters also have Special Abilities that grant them temporary boosts on the battlefield : Kurt can activate \" Direct Command \" and move around the battlefield without depleting his Action Point gauge , the character <unk> can shift into her \" Valkyria Form \" and become invincible , while Imca can target multiple enemy units with her heavy weapon . \\n'" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train_ds[10][\"text\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "30472f6c-31d6-4768-8dca-d3535be28501", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(1469)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1468 \u001b[0;31m \u001b[0mipdb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_trace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1469 \u001b[0;31m outputs = self.bert(\n", + "\u001b[0m\u001b[0;32m 1470 \u001b[0;31m \u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> attention_mask.size()\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 512])\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> p encoder_attention_mask.size()\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 512, 512])\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> l\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;32m 1464 \u001b[0m \"\"\"\n", + "\u001b[1;32m 1465 \u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 1466 \u001b[0m \u001b[0mreturn_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreturn_dict\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mreturn_dict\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_return_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 1467 \u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 1468 \u001b[0m \u001b[0mipdb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_trace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m-> 1469 \u001b[0;31m outputs = self.bert(\n", + "\u001b[0m\u001b[1;32m 1470 \u001b[0m \u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 1471 \u001b[0m \u001b[0mattention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 1472 \u001b[0m \u001b[0mtoken_type_ids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtoken_type_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 1473 \u001b[0m \u001b[0mposition_ids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mposition_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 1474 \u001b[0m \u001b[0mhead_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhead_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> n\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(1470)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1469 \u001b[0;31m outputs = self.bert(\n", + "\u001b[0m\u001b[0;32m-> 1470 \u001b[0;31m \u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1471 \u001b[0;31m \u001b[0mattention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> n\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(1471)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1470 \u001b[0;31m \u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1471 \u001b[0;31m \u001b[0mattention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1472 \u001b[0;31m \u001b[0mtoken_type_ids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtoken_type_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> n\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(1472)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1471 \u001b[0;31m \u001b[0mattention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1472 \u001b[0;31m \u001b[0mtoken_type_ids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtoken_type_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1473 \u001b[0;31m \u001b[0mposition_ids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mposition_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(1473)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1472 \u001b[0;31m \u001b[0mtoken_type_ids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtoken_type_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1473 \u001b[0;31m \u001b[0mposition_ids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mposition_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1474 \u001b[0;31m \u001b[0mhead_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhead_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(1474)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1473 \u001b[0;31m \u001b[0mposition_ids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mposition_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1474 \u001b[0;31m \u001b[0mhead_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhead_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1475 \u001b[0;31m \u001b[0minputs_embeds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs_embeds\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(1475)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1474 \u001b[0;31m \u001b[0mhead_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhead_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1475 \u001b[0;31m \u001b[0minputs_embeds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs_embeds\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1476 \u001b[0;31m \u001b[0mencoder_hidden_states\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mencoder_hidden_states\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(1476)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1475 \u001b[0;31m \u001b[0minputs_embeds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs_embeds\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1476 \u001b[0;31m \u001b[0mencoder_hidden_states\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mencoder_hidden_states\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1477 \u001b[0;31m \u001b[0mencoder_attention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mencoder_attention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(1477)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1476 \u001b[0;31m \u001b[0mencoder_hidden_states\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mencoder_hidden_states\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1477 \u001b[0;31m \u001b[0mencoder_attention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mencoder_attention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1478 \u001b[0;31m \u001b[0moutput_attentions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_attentions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(1478)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1477 \u001b[0;31m \u001b[0mencoder_attention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mencoder_attention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1478 \u001b[0;31m \u001b[0moutput_attentions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_attentions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1479 \u001b[0;31m \u001b[0moutput_hidden_states\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_hidden_states\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(1479)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1478 \u001b[0;31m \u001b[0moutput_attentions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_attentions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1479 \u001b[0;31m \u001b[0moutput_hidden_states\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_hidden_states\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1480 \u001b[0;31m \u001b[0mreturn_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreturn_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(1480)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1479 \u001b[0;31m \u001b[0moutput_hidden_states\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_hidden_states\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1480 \u001b[0;31m \u001b[0mreturn_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreturn_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1481 \u001b[0;31m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(1469)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1468 \u001b[0;31m \u001b[0mipdb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_trace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1469 \u001b[0;31m outputs = self.bert(\n", + "\u001b[0m\u001b[0;32m 1470 \u001b[0;31m \u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> s\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--Call--\n", + "> \u001b[0;32m/home/sipb/6.861/.venv/lib64/python3.12/site-packages/torch/nn/modules/module.py\u001b[0m(1549)\u001b[0;36m_wrapped_call_impl\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1548 \u001b[0;31m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1549 \u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0m_wrapped_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1550 \u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_compiled_call_impl\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> n\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/6.861/.venv/lib64/python3.12/site-packages/torch/nn/modules/module.py\u001b[0m(1550)\u001b[0;36m_wrapped_call_impl\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1549 \u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0m_wrapped_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1550 \u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_compiled_call_impl\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1551 \u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_compiled_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# type: ignore[misc]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> n\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/6.861/.venv/lib64/python3.12/site-packages/torch/nn/modules/module.py\u001b[0m(1553)\u001b[0;36m_wrapped_call_impl\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1552 \u001b[0;31m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1553 \u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1554 \u001b[0;31m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> s\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--Call--\n", + "> \u001b[0;32m/home/sipb/6.861/.venv/lib64/python3.12/site-packages/torch/nn/modules/module.py\u001b[0m(1555)\u001b[0;36m_call_impl\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1554 \u001b[0;31m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1555 \u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1556 \u001b[0;31m \u001b[0mforward_call\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_state\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> n\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/6.861/.venv/lib64/python3.12/site-packages/torch/nn/modules/module.py\u001b[0m(1556)\u001b[0;36m_call_impl\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1555 \u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1556 \u001b[0;31m \u001b[0mforward_call\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_state\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1557 \u001b[0;31m \u001b[0;31m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> n\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/6.861/.venv/lib64/python3.12/site-packages/torch/nn/modules/module.py\u001b[0m(1559)\u001b[0;36m_call_impl\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1558 \u001b[0;31m \u001b[0;31m# this function, and just call forward.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1559 \u001b[0;31m if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks\n", + "\u001b[0m\u001b[0;32m 1560 \u001b[0;31m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> n\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/6.861/.venv/lib64/python3.12/site-packages/torch/nn/modules/module.py\u001b[0m(1560)\u001b[0;36m_call_impl\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1559 \u001b[0;31m if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks\n", + "\u001b[0m\u001b[0;32m-> 1560 \u001b[0;31m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1561 \u001b[0;31m \u001b[0;32mor\u001b[0m \u001b[0m_global_forward_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_forward_pre_hooks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> n\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/6.861/.venv/lib64/python3.12/site-packages/torch/nn/modules/module.py\u001b[0m(1561)\u001b[0;36m_call_impl\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1560 \u001b[0;31m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1561 \u001b[0;31m \u001b[0;32mor\u001b[0m \u001b[0m_global_forward_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_forward_pre_hooks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1562 \u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> n\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/6.861/.venv/lib64/python3.12/site-packages/torch/nn/modules/module.py\u001b[0m(1562)\u001b[0;36m_call_impl\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1561 \u001b[0;31m \u001b[0;32mor\u001b[0m \u001b[0m_global_forward_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_forward_pre_hooks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1562 \u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1563 \u001b[0;31m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> s\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--Call--\n", + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(1005)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1004 \u001b[0;31m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1005 \u001b[0;31m \u001b[0;34m@\u001b[0m\u001b[0madd_start_docstrings_to_model_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBERT_INPUTS_DOCSTRING\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"batch_size, sequence_length\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1006 \u001b[0;31m @add_code_sample_docstrings(\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> l\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;32m 1000 \u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mPreTrainedModel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 1001 \u001b[0m \"\"\"\n", + "\u001b[1;32m 1002 \u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mlayer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheads\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mheads_to_prune\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 1003 \u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlayer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattention\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprune_heads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mheads\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 1004 \u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m-> 1005 \u001b[0;31m \u001b[0;34m@\u001b[0m\u001b[0madd_start_docstrings_to_model_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBERT_INPUTS_DOCSTRING\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"batch_size, sequence_length\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[1;32m 1006 \u001b[0m @add_code_sample_docstrings(\n", + "\u001b[1;32m 1007 \u001b[0m \u001b[0mcheckpoint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0m_CHECKPOINT_FOR_DOC\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 1008 \u001b[0m \u001b[0moutput_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mBaseModelOutputWithPoolingAndCrossAttentions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 1009 \u001b[0m \u001b[0mconfig_class\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0m_CONFIG_FOR_DOC\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 1010 \u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> n\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(1047)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1046 \u001b[0;31m \"\"\"\n", + "\u001b[0m\u001b[0;32m-> 1047 \u001b[0;31m \u001b[0moutput_attentions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutput_attentions\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_attentions\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput_attentions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1048 \u001b[0;31m output_hidden_states = (\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> self\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "BertModel(\n", + " (embeddings): BertEmbeddings(\n", + " (word_embeddings): Embedding(30522, 768, padding_idx=0)\n", + " (position_embeddings): Embedding(512, 768)\n", + " (token_type_embeddings): Embedding(2, 768)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (encoder): BertEncoder(\n", + " (layer): ModuleList(\n", + " (0-11): 12 x BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSdpaSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " (intermediate_act_fn): GELUActivation()\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " )\n", + ")\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> attention_mask\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0]], device='cuda:0')\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> l\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;32m 1042 \u001b[0m \u001b[0;34m`\u001b[0m\u001b[0mdecoder_input_ids\u001b[0m\u001b[0;34m`\u001b[0m \u001b[0mof\u001b[0m \u001b[0mshape\u001b[0m \u001b[0;34m`\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msequence_length\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 1043 \u001b[0m \u001b[0muse_cache\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m`\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m`\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0moptional\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 1044 \u001b[0m If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see\n", + "\u001b[1;32m 1045 \u001b[0m \u001b[0;34m`\u001b[0m\u001b[0mpast_key_values\u001b[0m\u001b[0;34m`\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 1046 \u001b[0m \"\"\"\n", + "\u001b[0;32m-> 1047 \u001b[0;31m \u001b[0moutput_attentions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutput_attentions\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_attentions\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput_attentions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[1;32m 1048 \u001b[0m output_hidden_states = (\n", + "\u001b[1;32m 1049 \u001b[0m \u001b[0moutput_hidden_states\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_hidden_states\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput_hidden_states\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 1050 \u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 1051 \u001b[0m \u001b[0mreturn_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreturn_dict\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mreturn_dict\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_return_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 1052 \u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> n\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(1047)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1046 \u001b[0;31m \"\"\"\n", + "\u001b[0m\u001b[0;32m-> 1047 \u001b[0;31m \u001b[0moutput_attentions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutput_attentions\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_attentions\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput_attentions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1048 \u001b[0;31m output_hidden_states = (\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> n\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(1049)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1048 \u001b[0;31m output_hidden_states = (\n", + "\u001b[0m\u001b[0;32m-> 1049 \u001b[0;31m \u001b[0moutput_hidden_states\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_hidden_states\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput_hidden_states\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 1050 \u001b[0;31m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> n\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(1048)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1047 \u001b[0;31m \u001b[0moutput_attentions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutput_attentions\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_attentions\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput_attentions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1048 \u001b[0;31m output_hidden_states = (\n", + "\u001b[0m\u001b[0;32m 1049 \u001b[0;31m \u001b[0moutput_hidden_states\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_hidden_states\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput_hidden_states\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> q\n" + ] + } + ], + "source": [ + "output = model(**{k: v.to(device) for k, v in inputs.items()}, encoder_attention_mask=rtlattn(inputs[\"input_ids\"].size() + (inputs[\"input_ids\"].size(1),)))" + ] + }, + { + "cell_type": "code", + "execution_count": 140, + "id": "43a301aa-3113-46bb-a65d-9ed12bae9437", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[[-6.3281, -6.3555, -6.4531, ..., -5.5234, -4.1797, -5.7891],\n", + " [-6.7891, -6.6914, -6.7812, ..., -6.1680, -5.1094, -5.5273],\n", + " [-7.1641, -7.1055, -7.0625, ..., -6.2383, -5.3711, -5.5273],\n", + " ...,\n", + " [-8.3516, -8.4375, -8.3516, ..., -7.6289, -7.0078, -5.6016],\n", + " [-7.7617, -7.8789, -7.7695, ..., -7.0938, -6.7461, -5.0430],\n", + " [-7.6602, -7.7500, -7.6953, ..., -6.9492, -6.4766, -4.9531]]],\n", + " device='cuda:0', dtype=torch.float16, grad_fn=<ViewBackward0>)" + ] + }, + "execution_count": 140, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output.logits" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "954bc5bd-16af-44d5-9739-24cc89ed3ce0", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(1469)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1468 \u001b[0;31m \u001b[0mipdb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_trace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m-> 1469 \u001b[0;31m outputs = self.bert(\n", + "\u001b[0m\u001b[0;32m 1470 \u001b[0;31m \u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> c\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(444)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 443 \u001b[0;31m \u001b[0mipdb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_trace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m--> 444 \u001b[0;31m attn_output = torch.nn.functional.scaled_dot_product_attention(\n", + "\u001b[0m\u001b[0;32m 445 \u001b[0;31m \u001b[0mquery_layer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> l\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;32m 439 \u001b[0m is_causal = (\n", + "\u001b[1;32m 440 \u001b[0m \u001b[0;32mTrue\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_decoder\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_cross_attention\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mattention_mask\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mtgt_len\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 441 \u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 442 \u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 443 \u001b[0m \u001b[0mipdb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_trace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m--> 444 \u001b[0;31m attn_output = torch.nn.functional.scaled_dot_product_attention(\n", + "\u001b[0m\u001b[1;32m 445 \u001b[0m \u001b[0mquery_layer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 446 \u001b[0m \u001b[0mkey_layer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 447 \u001b[0m \u001b[0mvalue_layer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 448 \u001b[0m \u001b[0mattn_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[1;32m 449 \u001b[0m \u001b[0mdropout_p\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout_prob\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;36m0.0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> n\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(445)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 444 \u001b[0;31m attn_output = torch.nn.functional.scaled_dot_product_attention(\n", + "\u001b[0m\u001b[0;32m--> 445 \u001b[0;31m \u001b[0mquery_layer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 446 \u001b[0;31m \u001b[0mkey_layer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(446)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 445 \u001b[0;31m \u001b[0mquery_layer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m--> 446 \u001b[0;31m \u001b[0mkey_layer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 447 \u001b[0;31m \u001b[0mvalue_layer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> attention_mask\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[[[ 0., -65504., -65504., ..., -65504., -65504., -65504.],\n", + " [ 0., 0., -65504., ..., -65504., -65504., -65504.],\n", + " [ 0., 0., 0., ..., -65504., -65504., -65504.],\n", + " ...,\n", + " [ 0., 0., 0., ..., -65504., -65504., -65504.],\n", + " [ 0., 0., 0., ..., -65504., -65504., -65504.],\n", + " [ 0., 0., 0., ..., -65504., -65504., -65504.]]]],\n", + " device='cuda:0', dtype=torch.float16)\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> attention_mask.size()\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 1, 512, 512])\n", + "--KeyboardInterrupt--\n", + "\n", + "KeyboardInterrupt: Interrupted by user\n", + "> \u001b[0;32m/home/sipb/transformer-shortest-paths/NLP/transformers/src/transformers/models/bert/modeling_bert.py\u001b[0m(444)\u001b[0;36mforward\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 443 \u001b[0;31m \u001b[0mipdb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_trace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m--> 444 \u001b[0;31m attn_output = torch.nn.functional.scaled_dot_product_attention(\n", + "\u001b[0m\u001b[0;32m 445 \u001b[0;31m \u001b[0mquery_layer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + }, + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ipdb> q\n", + "ipdb> q\n" + ] + } + ], + "source": [ + "output2 = model(**{k: v.to(device) for k, v in inputs.items()}, encoder_attention_mask=ltrattn(inputs[\"input_ids\"].size() + (inputs[\"input_ids\"].size(1),)))" + ] + }, + { + "cell_type": "code", + "execution_count": 138, + "id": "327929a1-04da-45fa-846e-4998ac87cc26", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[[-6.3281, -6.3555, -6.4531, ..., -5.5234, -4.1797, -5.7891],\n", + " [-6.7891, -6.6914, -6.7812, ..., -6.1680, -5.1094, -5.5273],\n", + " [-7.1641, -7.1055, -7.0625, ..., -6.2383, -5.3711, -5.5273],\n", + " ...,\n", + " [-8.3516, -8.4375, -8.3516, ..., -7.6289, -7.0078, -5.6016],\n", + " [-7.7617, -7.8789, -7.7695, ..., -7.0938, -6.7461, -5.0430],\n", + " [-7.6602, -7.7500, -7.6953, ..., -6.9492, -6.4766, -4.9531]]],\n", + " device='cuda:0', dtype=torch.float16, grad_fn=<ViewBackward0>)" + ] + }, + "execution_count": 138, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output2.logits" + ] + }, + { + "cell_type": "code", + "execution_count": 135, + "id": "420d0bed-923c-452d-ab20-21a9440d4c8f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 135, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.equal(output.logits, output2.logits)" + ] + }, + { + "cell_type": "code", + "execution_count": 144, + "id": "a33632fb-ad41-49e3-acee-91d1dda974b8", + "metadata": {}, + "outputs": [], + "source": [ + "output2 = model(**{k: v.to(device) for k, v in inputs.items()}, encoder_attention_mask=torch.zeros(1, 512, 512))" + ] + }, + { + "cell_type": "code", + "execution_count": 145, + "id": "5042c85c-c98b-45ed-8437-8a98f63507d2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[[ -8.8438, -8.8750, -8.7812, ..., -8.3672, -8.1484, -4.5195],\n", + " [-12.5547, -12.2734, -12.4609, ..., -11.4141, -10.2969, -7.3320],\n", + " [-12.8125, -12.8125, -12.7891, ..., -12.1328, -10.5781, -5.9453],\n", + " ...,\n", + " [ -7.9531, -8.1797, -8.2266, ..., -7.2188, -6.5000, -6.4688],\n", + " [ -7.5234, -7.7344, -7.7305, ..., -6.7344, -6.3359, -6.0195],\n", + " [ -7.8711, -7.9453, -8.0156, ..., -7.3555, -7.1523, -5.6680]]],\n", + " device='cuda:0', dtype=torch.float16, grad_fn=<ViewBackward0>)" + ] + }, + "execution_count": 145, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output2.logits" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1f8144a4-c496-4b45-99f8-95c8fe41ce16", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9932c9d4-ae85-4f95-bf84-78be2000131d", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/rtl.ipynb b/rtl.ipynb new file mode 100644 index 0000000..039764c --- /dev/null +++ b/rtl.ipynb @@ -0,0 +1,191 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 4, + "id": "8ddb479e-9d7e-4392-8fc0-fd1c66a07a2b", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n", + "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 512])\n" + ] + } + ], + "source": [ + "import torch\n", + "import transformers\n", + "transformers.set_seed(42)\n", + "device = \"cuda\"\n", + "# import sys\n", + "\n", + "# for key in list(sys.modules):\n", + "# if key.startswith(\"transformers.\"):\n", + "# sys.modules.pop(key)\n", + "\n", + "from transformers import AutoModelForMaskedLM\n", + "model = AutoModelForMaskedLM.from_pretrained(\"bert-base-uncased\", torch_dtype=torch.float16, attn_implementation=\"sdpa\").to(device)\n", + "\n", + "from transformers import AutoTokenizer\n", + "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-uncased\")\n", + "\n", + "model.config.alek_says_ltr = True\n", + "model.config.alek_says_rtl = False\n", + "from datasets import load_dataset\n", + "\n", + "ds = load_dataset(\"Salesforce/wikitext\", \"wikitext-103-v1\")\n", + "train_ds = ds[\"train\"]\n", + "inputs = tokenizer(train_ds[10][\"text\"], return_tensors=\"pt\", padding='max_length', truncation=True)\n", + "\n", + "print(inputs[\"input_ids\"].size())" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "30472f6c-31d6-4768-8dca-d3535be28501", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "output = model(**{k: v.to(device) for k, v in inputs.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "ac46bf41-8cd3-4190-aa4b-6142d6d4d986", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[[ -7.1094, -7.1445, -7.2148, ..., -6.6484, -7.0703, -3.6758],\n", + " [-14.2969, -14.2656, -14.3828, ..., -11.9766, -11.3281, -9.4922],\n", + " [-10.7344, -10.6250, -10.7266, ..., -8.6641, -8.2188, -5.0859],\n", + " ...,\n", + " [ -3.4277, -3.5664, -3.9434, ..., -2.0000, -4.4727, -3.7148],\n", + " [ -3.7227, -3.8770, -4.2383, ..., -2.1367, -4.5977, -3.9336],\n", + " [ -4.2070, -4.3672, -4.7578, ..., -2.4941, -4.7734, -4.7227]]],\n", + " device='cuda:0', dtype=torch.float16, grad_fn=<ViewBackward0>)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output.logits" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "baa757ff-3ba2-4a72-b819-a2283b729c18", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[[-6.3281, -6.3555, -6.4531, ..., -5.5234, -4.1797, -5.7891],\n", + " [-6.7891, -6.6914, -6.7812, ..., -6.1680, -5.1094, -5.5273],\n", + " [-7.1641, -7.1055, -7.0625, ..., -6.2383, -5.3711, -5.5273],\n", + " ...,\n", + " [ 9.4844, 8.9219, 9.2422, ..., 7.6133, 7.2578, 9.9062],\n", + " [10.3672, 9.8516, 10.1797, ..., 8.5547, 8.0781, 10.5938],\n", + " [ 8.3828, 8.0781, 8.1641, ..., 7.2422, 6.7734, 7.9961]]],\n", + " device='cuda:0', dtype=torch.float16, grad_fn=<ViewBackward0>)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output.logits" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "a33632fb-ad41-49e3-acee-91d1dda974b8", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# output1 = model(**{k: v.to(device) for k, v in inputs.items()})\n", + "# print(output1.logits)\n", + "# output2 = model(**{k: v.to(device) for k, v in inputs.items()}, encoder_attention_mask=torch.zeros(1, 512, 512))\n", + "# print(output2.logits)" + ] + }, + { + "cell_type": "markdown", + "id": "ad432f29-f77a-4b84-b6b4-347b74c82f5b", + "metadata": {}, + "source": [ + "## plan for finishing phase 1\n", + "\n", + "- fix the tokenizer\n", + "- pretrain on RTL + LTR\n", + "- check perplexities\n", + "\n", + "## plan for phase 2\n", + "- AQ\n", + "\n", + "## plan for phase 1.5\n", + "- addition" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} |