-
1
-
2
-
3
-
4
-
5
-
6
-
7
-
8
-
9
-
10
-
11
-
12
-
13
-
14
-
15
-
16
-
17
-
18
-
19
-
20
-
21
-
22
-
23
-
24
-
25
-
26
-
27
-
28
-
29
-
30
-
31
-
32
-
33
-
34
-
35
-
36
-
37
-
38
-
39
-
40
-
41
-
42
-
43
-
44
-
45
-
46
-
47
-
48
-
49
-
50
-
51
-
52
-
53
-
54
-
55
-
56
-
57
-
58
-
59
-
60
-
61
-
62
-
63
-
64
-
65
-
66
-
67
-
68
-
69
-
70
-
71
-
72
-
73
-
74
-
75
-
76
-
77
-
78
-
79
-
80
-
81
-
82
-
83
-
84
-
85
-
86
-
87
-
88
-
89
-
90
-
91
-
92
-
93
-
94
-
95
-
96
-
97
-
98
-
99
-
100
-
101
-
102
-
103
-
104
-
105
-
106
-
107
-
108
-
109
-
110
-
111
-
112
-
113
-
114
-
115
-
116
-
117
-
118
-
119
-
120
-
121
-
122
-
123
-
124
-
125
-
126
-
127
-
128
-
129
-
130
-
131
-
132
-
133
-
134
-
135
-
136
-
137
-
138
-
139
-
140
-
141
-
142
-
143
-
144
-
145
-
146
-
147
-
148
-
149
-
150
-
151
-
152
-
153
-
154
-
155
-
156
-
157
-
158
-
159
-
160
-
161
-
162
-
163
-
164
-
165
-
166
-
167
-
168
-
169
-
170
-
171
-
172
-
173
-
174
-
175
-
176
-
177
-
178
-
179
-
180
-
181
-
182
-
183
-
184
-
185
-
186
-
187
-
188
-
189
-
190
-
191
-
192
-
193
-
194
-
195
-
196
-
197
-
198
-
199
-
200
-
201
-
202
-
203
-
204
-
205
-
206
-
207
-
208
-
209
-
210
-
211
-
212
-
213
-
214
-
215
-
216
-
217
-
218
-
219
-
220
-
221
-
222
-
223
-
224
-
225
-
226
-
227
-
228
-
229
-
230
-
231
-
232
-
233
-
234
-
235
-
236
-
237
-
238
-
239
-
240
-
241
-
242
-
243
-
244
-
245
-
246
-
247
-
248
-
249
-
250
-
251
-
252
-
253
-
254
-
255
-
256
-
257
-
258
-
259
-
260
-
261
-
262
-
263
-
264
-
265
-
266
-
267
-
268
-
269
-
270
-
271
-
272
-
273
-
274
-
275
-
276
-
277
-
278
-
279
-
280
-
281
-
282
-
283
-
284
-
285
-
286
-
287
-
288
-
289
-
290
-
291
-
292
-
293
-
294
-
295
-
296
-
297
-
298
-
299
-
300
-
301
-
302
-
303
-
304
-
305
-
306
-
307
-
308
-
309
-
310
-
311
-
312
-
313
-
314
-
315
-
316
-
317
-
318
-
319
-
320
-
321
-
322
-
323
-
324
-
325
-
326
-
327
-
328
-
329
-
330
-
331
-
332
-
333
-
334
-
335
-
336
-
337
-
338
-
339
-
340
-
341
-
342
-
343
-
344
-
345
-
346
-
347
-
348
-
349
-
350
-
351
-
352
-
353
-
354
-
355
-
356
-
357
-
358
-
359
-
360
-
361
-
362
-
363
-
364
-
365
-
366
-
367
-
368
-
369
-
370
-
371
-
372
-
373
-
374
-
375
-
376
-
377
-
378
-
379
-
380
-
381
-
382
-
383
-
384
-
385
-
386
-
387
-
388
-
389
-
390
-
391
-
392
-
393
-
394
-
395
-
396
-
397
-
398
-
399
-
400
-
401
-
402
-
403
-
404
-
405
-
406
-
407
-
408
-
409
-
410
-
411
-
412
-
413
-
414
-
415
-
416
-
417
-
418
-
419
-
420
-
421
-
422
-
423
-
424
-
425
-
426
-
427
-
428
-
429
-
430
-
431
-
432
-
433
-
434
-
435
-
436
-
437
-
438
-
439
-
440
-
441
-
442
-
443
-
444
-
445
-
446
-
447
-
448
-
449
-
450
-
451
-
452
-
453
-
454
-
455
-
456
-
457
-
458
-
459
-
460
-
461
-
462
-
463
-
464
-
465
-
466
-
467
-
468
-
469
-
470
-
471
-
472
-
473
-
474
-
475
-
476
-
477
-
478
-
479
-
480
-
481
-
482
-
483
-
484
-
485
-
486
-
487
-
488
-
489
-
490
-
491
-
492
-
493
-
494
-
495
-
496
-
497
-
498
-
499
-
500
-
501
-
502
-
503
-
504
-
505
-
506
-
507
-
508
-
509
-
510
-
511
-
512
-
513
-
514
-
515
-
516
-
517
-
518
-
519
-
520
-
521
-
522
-
523
-
524
-
525
-
526
-
527
-
528
-
529
-
530
-
531
-
532
-
533
-
534
-
535
-
536
-
537
-
538
-
539
-
540
-
541
-
542
-
543
-
544
-
545
-
546
-
547
-
548
-
549
-
550
-
551
-
552
-
553
-
554
-
555
-
556
-
557
-
558
-
559
-
560
-
561
-
562
-
563
-
564
-
565
-
566
-
567
-
568
-
569
-
570
-
571
-
572
-
573
-
574
-
575
-
576
-
577
-
578
-
579
-
580
-
581
-
582
-
583
-
584
-
585
-
586
-
587
-
588
-
589
-
590
-
591
-
592
-
593
-
594
-
595
-
596
-
597
-
598
-
599
-
600
-
601
-
602
-
603
-
604
-
605
-
606
-
607
-
608
-
609
-
610
-
611
-
612
-
613
-
614
-
615
-
616
-
617
-
618
-
619
-
620
-
621
-
622
-
623
-
624
-
625
-
626
-
627
-
628
-
629
-
630
-
631
-
632
-
633
-
634
-
635
-
636
-
637
-
638
-
639
-
640
-
641
-
642
-
643
-
644
-
645
-
646
-
647
-
648
-
649
-
650
-
651
-
652
-
653
-
654
-
655
-
656
-
657
-
658
-
659
-
660
-
661
-
662
-
663
-
664
-
665
-
666
-
667
-
668
-
669
-
670
-
671
-
672
-
673
-
674
-
675
-
676
-
677
-
678
-
679
-
680
-
681
-
682
-
683
-
684
-
685
-
686
-
687
-
688
-
689
-
690
-
691
-
692
-
693
-
694
-
695
-
696
-
697
-
698
-
699
-
700
-
701
-
702
-
703
-
704
-
705
-
706
-
707
-
708
-
709
-
710
-
711
-
712
-
713
-
714
-
715
-
716
-
717
-
718
-
719
-
720
-
721
-
722
-
723
-
724
-
725
-
726
-
727
-
728
-
729
-
730
-
731
-
732
-
733
-
734
-
735
-
736
-
737
-
738
-
739
-
740
-
741
-
742
-
743
-
744
-
745
-
746
-
747
-
748
-
749
-
750
-
751
-
752
-
753
-
754
-
755
-
756
-
757
-
758
-
759
-
760
-
761
-
762
-
763
-
764
-
765
-
766
-
767
-
768
-
769
-
770
-
771
-
772
-
773
-
774
-
775
-
776
-
777
-
778
-
779
-
780
-
781
-
782
-
783
-
784
-
785
-
786
-
787
-
788
-
789
-
790
-
791
-
792
-
793
-
794
-
795
-
796
-
797
-
798
-
799
-
800
-
801
-
802
-
803
-
804
-
805
-
806
-
807
-
808
-
809
-
810
-
811
-
812
-
813
-
814
-
815
-
816
-
817
-
818
-
819
-
820
-
821
-
822
-
823
-
824
-
825
-
826
-
827
-
828
-
829
-
830
-
831
-
832
-
833
-
834
-
835
-
836
-
837
-
838
-
839
-
840
-
841
-
842
-
843
-
844
-
845
-
846
-
847
-
848
-
849
-
850
-
851
-
852
-
853
-
854
-
855
-
856
-
857
-
858
-
859
-
860
-
861
-
862
-
863
-
864
-
865
-
866
-
867
-
868
-
869
-
870
-
871
-
872
-
873
-
874
-
875
-
876
-
877
-
878
-
879
-
880
-
881
-
882
-
883
-
884
-
885
-
886
-
887
-
888
-
889
-
890
-
891
-
892
-
893
-
894
-
895
-
896
-
897
-
898
-
899
-
900
-
901
-
902
-
903
-
904
-
905
-
906
-
907
-
908
-
909
-
910
-
911
-
912
-
913
-
914
-
915
-
916
-
917
-
918
-
919
-
920
-
921
-
922
-
923
-
924
-
925
-
926
-
927
-
928
-
929
-
930
-
931
-
932
-
933
-
934
-
935
-
936
-
937
-
938
-
939
-
940
-
941
-
942
-
943
-
944
-
945
-
946
-
947
-
948
-
949
-
950
-
951
-
952
-
953
-
954
-
955
-
956
-
957
-
958
-
959
-
960
-
961
-
962
-
963
-
964
-
965
-
966
-
967
-
968
-
969
-
970
-
971
-
972
-
973
-
974
-
975
-
976
-
977
-
978
-
979
-
980
-
981
-
982
-
983
-
984
-
985
-
986
-
987
-
988
-
989
-
990
-
991
-
992
-
993
-
994
-
995
-
996
-
997
-
998
-
999
-
1000
-
1001
-
1002
-
1003
-
1004
-
1005
-
1006
-
1007
-
1008
-
1009
-
1010
-
1011
-
1012
-
1013
-
1014
-
1015
-
1016
-
1017
-
1018
-
1019
-
1020
-
1021
-
1022
-
1023
-
1024
-
1025
-
1026
-
1027
-
1028
-
1029
-
1030
-
1031
-
1032
-
1033
-
1034
-
1035
-
1036
-
1037
-
1038
-
1039
-
1040
-
1041
-
1042
-
1043
-
1044
-
1045
-
1046
-
1047
-
1048
-
1049
-
1050
-
1051
-
1052
-
1053
-
1054
-
1055
-
1056
-
1057
-
1058
-
1059
-
1060
-
1061
-
1062
-
1063
-
1064
-
1065
-
1066
-
1067
-
1068
-
1069
-
1070
-
1071
-
1072
-
1073
-
1074
-
1075
-
1076
-
1077
-
1078
-
1079
-
1080
-
1081
-
1082
-
1083
-
1084
-
1085
-
1086
-
1087
-
1088
-
1089
-
1090
-
1091
-
1092
-
1093
-
1094
-
1095
-
1096
-
1097
-
1098
-
1099
-
1100
-
1101
-
1102
-
1103
-
1104
-
1105
-
1106
-
1107
-
1108
-
1109
-
1110
-
1111
-
1112
-
1113
-
1114
-
1115
-
1116
-
1117
-
1118
-
1119
-
1120
-
1121
-
1122
-
1123
-
1124
-
1125
-
1126
-
1127
-
1128
-
1129
-
1130
-
1131
-
1132
-
1133
-
1134
-
1135
-
1136
-
1137
-
1138
-
1139
-
1140
-
1141
-
1142
-
1143
-
1144
-
1145
-
1146
-
1147
-
1148
-
1149
-
1150
-
1151
-
1152
-
1153
-
1154
-
1155
-
1156
-
1157
-
1158
-
1159
-
1160
-
1161
-
1162
-
1163
-
1164
-
1165
-
1166
-
1167
-
1168
-
1169
-
1170
-
1171
-
1172
-
1173
-
1174
-
1175
-
1176
-
1177
-
1178
-
1179
-
1180
-
1181
-
1182
-
1183
-
1184
-
1185
-
1186
-
1187
-
1188
-
1189
-
1190
-
1191
-
1192
-
1193
-
1194
-
1195
-
1196
-
1197
-
1198
-
1199
-
1200
-
1201
-
1202
-
1203
-
1204
-
1205
-
1206
-
1207
-
1208
-
1209
-
1210
-
1211
-
1212
-
1213
-
1214
-
1215
-
1216
-
1217
-
1218
-
1219
-
1220
-
1221
-
1222
-
1223
-
1224
-
1225
-
1226
-
1227
-
1228
-
1229
-
1230
-
1231
-
1232
-
1233
-
1234
-
1235
-
1236
-
1237
-
1238
-
1239
-
1240
-
1241
-
1242
-
1243
-
1244
-
1245
-
1246
-
1247
-
1248
-
1249
-
1250
-
1251
-
1252
-
1253
-
1254
-
1255
-
1256
-
1257
-
1258
-
1259
-
1260
-
1261
-
1262
-
1263
-
1264
-
1265
-
1266
-
1267
-
1268
-
1269
-
1270
-
1271
-
1272
-
1273
-
1274
-
1275
-
1276
-
1277
-
1278
-
1279
-
1280
-
1281
-
1282
-
1283
-
1284
-
1285
-
1286
-
1287
-
1288
-
1289
-
1290
-
1291
-
1292
-
1293
-
1294
-
1295
-
1296
-
1297
-
1298
-
1299
-
1300
-
1301
-
1302
-
1303
-
1304
-
1305
-
1306
-
1307
-
1308
-
1309
-
1310
-
1311
-
1312
-
1313
-
1314
-
1315
-
1316
-
1317
-
1318
-
1319
-
1320
-
1321
-
1322
-
1323
-
1324
-
1325
-
1326
-
1327
-
1328
-
1329
-
1330
-
1331
-
1332
-
1333
-
1334
-
1335
-
1336
-
1337
-
1338
-
1339
-
1340
-
1341
-
1342
-
1343
-
1344
-
1345
-
1346
-
1347
-
1348
-
1349
-
1350
-
1351
-
1352
-
1353
-
1354
-
1355
-
1356
-
1357
-
1358
-
1359
-
1360
-
1361
-
1362
-
1363
-
1364
-
1365
-
1366
-
1367
-
1368
-
1369
-
1370
-
1371
-
1372
-
1373
-
1374
-
1375
-
1376
-
1377
-
1378
-
1379
-
1380
-
1381
-
1382
-
1383
-
1384
-
1385
-
1386
-
1387
-
1388
-
1389
-
1390
-
1391
-
1392
-
1393
-
1394
-
1395
-
1396
-
1397
-
1398
-
1399
-
1400
-
1401
-
1402
-
1403
-
1404
-
1405
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "LPphBnKR-aWF"
},
"source": [
"# Step 0: Imports"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Question: \n",
"\n",
"- Do the attention heads learn to attend to the same positional encodings\n",
"- do interp -- what is it doing? can we figure out?"
]
},
{
"cell_type": "code",
"execution_count": 75,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "ge5QvElvhCOw",
"outputId": "c7cdaefa-d6dc-44ad-c258-e4fb2aca97a5"
},
"outputs": [],
"source": [
"from collections import deque\n",
"# using tqdm.auto glitches out collaborative editing\n",
"from tqdm import tqdm\n",
"\n",
"import torch\n",
"import torch.nn as nn\n",
"from math import sqrt\n",
"import matplotlib.pyplot as plt\n",
"torch.manual_seed(42)\n",
"\n",
"import os\n",
"import random\n",
"random.seed(42)\n",
"\n",
"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
"assert device.type == 'cuda', \"CUDA is not available. Please check your GPU setup.\""
]
},
{
"cell_type": "code",
"execution_count": 91,
"metadata": {
"id": "lylOX2POPwFL"
},
"outputs": [],
"source": [
"# VTXS numbers here are inclusive\n",
"MIN_VTXS = 3 # 3\n",
"MAX_VTXS = 3 # 8\n",
"MAX_TUNE_VTXS = 3 # 15\n",
"AVG_DEG = 2\n",
"SEQ_LEN = MAX_VTXS * AVG_DEG + 1 # means 32 edges, final token is the target vertex\n",
"PAD_TOKEN = 0\n",
"# vertices are labelled 1,2,...,63\n",
"# we also have a padding token which is 0."
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "gKt-yIpDebF1"
},
"source": [
"# Step 1: Generate synthetic data"
]
},
{
"cell_type": "code",
"execution_count": 76,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "1IbzGIWseK3E",
"outputId": "a3cbc233-358c-4e17-ea6e-f4e9349d886b"
},
"outputs": [],
"source": [
"# original task data\n",
"# the data will be edge lists\n",
"# like this: [1 3 1 5 2 4 0 0 0 0 2]\n",
"# this represents edges (1,3), (1,5) (2,4)\n",
"# (the zeros are just padding tokens)\n",
"# the final 2 means which vertex we're going to \n",
"\n",
"# the label is the shortest distance from vtx 1 to vtx 2\n",
"# or \"number of vertices\" if no path exists\n",
"\n",
"def random_graph(n):\n",
" edge_list = []\n",
" adjacencies = [set() for _ in range(n+1)]\n",
" indices = [random.randint(1, n) for _ in range(AVG_DEG * (n-1))]\n",
" for i in range(0, len(indices), 2):\n",
" u = indices[i]\n",
" v = indices[i + 1]\n",
" if u != v:\n",
" edge_list += [u,v]\n",
" adjacencies[u].add(v)\n",
" adjacencies[v].add(u)\n",
"\n",
" edge_list += [PAD_TOKEN]*(SEQ_LEN-len(edge_list))\n",
" return edge_list, adjacencies\n",
"\n",
"\"\"\"\n",
"input: G, represented as an adjacency list\n",
"output: [number of vertices]+[d(1,i) for i in range(n)] if target=None\n",
"if target is set to some value, then we instead just output that specific distance\n",
"\"\"\"\n",
"def SSSP(n, G, target=2):\n",
" dist = [n for _ in G]\n",
" dist[1] = 0\n",
" frontier = deque()\n",
" frontier.append(1)\n",
" while len(frontier) > 0:\n",
" vtx = frontier.popleft()\n",
" for x in G[vtx]:\n",
" if dist[x] == n:\n",
" dist[x] = 1 + dist[vtx]\n",
" frontier.append(x)\n",
" if x == target:\n",
" return dist[target]\n",
" if target is not None:\n",
" return dist[target]\n",
" else:\n",
" return dist\n",
"\n",
"def mkbatch(size):\n",
" graphs1 = []\n",
" distance1 = []\n",
" \n",
" for i in range(size):\n",
" n = random.randint(MIN_VTXS, MAX_VTXS)\n",
" edge_list, adj_list = random_graph(n)\n",
" dist = SSSP(n, adj_list)\n",
" edge_list[-1] = 2 # target token\n",
" graphs1.append(edge_list)\n",
" distance1.append(dist)\n",
" \n",
" data = torch.tensor(graphs1, device=device)\n",
" labels = torch.tensor(distance1, dtype=torch.bfloat16, device=device)\n",
" padding = data == PAD_TOKEN\n",
" return data, labels, padding\n",
"\n",
"def vertices_on_shortest_12_path(n, G, target=2):\n",
" dist = [n for _ in G]\n",
" parent = [-1 for _ in G]\n",
" dist[1] = 0\n",
" frontier = deque()\n",
" frontier.append(1)\n",
" while len(frontier) > 0:\n",
" vtx = frontier.popleft()\n",
" for x in G[vtx]:\n",
" if dist[x] == n:\n",
" parent[x] = vtx\n",
" dist[x] = 1 + dist[vtx]\n",
" frontier.append(x)\n",
" if x == target:\n",
" path = [x]\n",
" while parent[x] != -1:\n",
" x = parent[x]\n",
" path.append(x)\n",
" return list(reversed(path))\n",
" return []\n",
"\n",
"def mktunebatch(size):\n",
" graphs = []\n",
" distance = []\n",
" \n",
" for i in range(size):\n",
" n = random.randint(MIN_VTXS, MAX_TUNE_VTXS)\n",
" while True:\n",
" edge_list, adj_list = random_graph(n)\n",
" path = vertices_on_shortest_12_path(n, adj_list)\n",
" if len(path) > 1:\n",
" target_vtx_idx = random.randrange(1, len(path))\n",
" target_vtx = path[target_vtx_idx]\n",
" edge_list[-1] = target_vtx\n",
" graphs.append(edge_list)\n",
" distance.append(target_vtx_idx)\n",
" break\n",
" \n",
" data = torch.tensor(graphs, device=device)\n",
" labels = torch.tensor(distance, dtype=torch.bfloat16, device=device)\n",
" padding = data == PAD_TOKEN\n",
" return data, labels, padding"
]
},
{
"cell_type": "code",
"execution_count": 86,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(tensor([[4, 3, 2, 3, 1, 2, 0, 0, 2],\n",
" [3, 4, 3, 1, 2, 3, 0, 0, 2],\n",
" [4, 2, 3, 2, 0, 0, 0, 0, 2],\n",
" [3, 4, 4, 3, 0, 0, 0, 0, 2],\n",
" [1, 2, 2, 4, 0, 0, 0, 0, 2]], device='cuda:0'),\n",
" tensor([1., 2., 4., 4., 1.], device='cuda:0', dtype=torch.bfloat16),\n",
" tensor([[False, False, False, False, False, False, True, True, False],\n",
" [False, False, False, False, False, False, True, True, False],\n",
" [False, False, False, False, True, True, True, True, False],\n",
" [False, False, False, False, True, True, True, True, False],\n",
" [False, False, False, False, True, True, True, True, False]],\n",
" device='cuda:0'))"
]
},
"execution_count": 86,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"mkbatch(5)"
]
},
{
"cell_type": "code",
"execution_count": 87,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(array([ 663., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 284., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 39., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 1062.]),\n",
" array([1. , 1.046875, 1.09375 , 1.140625, 1.1875 , 1.234375,\n",
" 1.28125 , 1.328125, 1.375 , 1.421875, 1.46875 , 1.515625,\n",
" 1.5625 , 1.609375, 1.65625 , 1.703125, 1.75 , 1.796875,\n",
" 1.84375 , 1.890625, 1.9375 , 1.984375, 2.03125 , 2.078125,\n",
" 2.125 , 2.171875, 2.21875 , 2.265625, 2.3125 , 2.359375,\n",
" 2.40625 , 2.453125, 2.5 , 2.546875, 2.59375 , 2.640625,\n",
" 2.6875 , 2.734375, 2.78125 , 2.828125, 2.875 , 2.921875,\n",
" 2.96875 , 3.015625, 3.0625 , 3.109375, 3.15625 , 3.203125,\n",
" 3.25 , 3.296875, 3.34375 , 3.390625, 3.4375 , 3.484375,\n",
" 3.53125 , 3.578125, 3.625 , 3.671875, 3.71875 , 3.765625,\n",
" 3.8125 , 3.859375, 3.90625 , 3.953125, 4. ]),\n",
" <BarContainer object of 64 artists>)"
]
},
"execution_count": 87,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhJUlEQVR4nO3df1DVVeL/8ReI/NC8F7HlXhjR2HJVSjOl6GqffslKyTY5MVvssi5brjYutJKlC7NJmRXquloWqf0Sd7Ox2hndsiIJV5gSkVA2IjO3LGntwu4Y9yqtqPD+/tF4v12VwrpwOfJ8zNwZeb/P+95zz5wZnr69YIhlWZYAAAAMEhrsCQAAAJwtAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAccKCPYHu0tHRoYMHD2rQoEEKCQkJ9nQAAEAXWJalw4cPKz4+XqGhnd9nOWcD5uDBg0pISAj2NAAAwPfQ2NiooUOHdnr+nA2YQYMGSfp6AWw2W5BnAwAAusLr9SohIcH3fbwz52zAnPxnI5vNRsAAAGCY7/r4Bx/iBQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAccKCPQEAANA7XZD/WqfnPl2c3oMzOR13YAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxjnrgKmsrNRNN92k+Ph4hYSEaNOmTX7nLctSYWGh4uLiFBUVpdTUVO3bt89vzKFDh5SVlSWbzabo6GjNmDFDR44c8Rvz3nvv6f/+7/8UGRmphIQELV269OzfHQAAOCeddcC0trbq0ksvVXFx8RnPL126VCtXrtTq1atVXV2tgQMHKi0tTUePHvWNycrKUkNDg8rKyrR582ZVVlZq1qxZvvNer1dTpkzR8OHDVVtbqz/96U964IEH9NRTT32PtwgAAM41IZZlWd/74pAQbdy4UdOmTZP09d2X+Ph43XPPPbr33nslSR6PRw6HQyUlJcrMzNSePXuUlJSkmpoaJScnS5JKS0s1depUff7554qPj9eqVav0xz/+UW63W+Hh4ZKk/Px8bdq0SR9++GGX5ub1emW32+XxeGSz2b7vWwQAoM+6IP+1Ts99uji9W16zq9+/A/oZmP3798vtdis1NdV3zG63KyUlRVVVVZKkqqoqRUdH++JFklJTUxUaGqrq6mrfmKuvvtoXL5KUlpamvXv36ssvvzzja7e1tcnr9fo9AADAuSmgAeN2uyVJDofD77jD4fCdc7vdio2N9TsfFhammJgYvzFneo5vvsapioqKZLfbfY+EhIQf/oYAAECvdM78FFJBQYE8Ho/v0djYGOwpAQCAbhLQgHE6nZKkpqYmv+NNTU2+c06nU83NzX7nT5w4oUOHDvmNOdNzfPM1ThURESGbzeb3AAAA56aABkxiYqKcTqfKy8t9x7xer6qrq+VyuSRJLpdLLS0tqq2t9Y3ZunWrOjo6lJKS4htTWVmp48eP+8aUlZVp5MiRGjx4cCCnDAAADHTWAXPkyBHV1dWprq5O0tcf3K2rq9OBAwcUEhKivLw8PfTQQ3rllVdUX1+vX//614qPj/f9pNLo0aN1ww03aObMmdq5c6feeecd5ebmKjMzU/Hx8ZKkX/7ylwoPD9eMGTPU0NCgF198UY899pjmzp0bsDcOAADMFXa2F7z77ru67rrrfF+fjIrs7GyVlJRo/vz5am1t1axZs9TS0qKrrrpKpaWlioyM9F2zfv165ebmavLkyQoNDVVGRoZWrlzpO2+327Vlyxbl5ORowoQJOv/881VYWOj3u2IAAEDf9YN+D0xvxu+BAQDgh+kzvwcGAACgJxAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADBOwAOmvb1dCxYsUGJioqKionThhRdq0aJFsizLN8ayLBUWFiouLk5RUVFKTU3Vvn37/J7n0KFDysrKks1mU3R0tGbMmKEjR44EeroAAMBAAQ+YJUuWaNWqVXriiSe0Z88eLVmyREuXLtXjjz/uG7N06VKtXLlSq1evVnV1tQYOHKi0tDQdPXrUNyYrK0sNDQ0qKyvT5s2bVVlZqVmzZgV6ugAAwEAh1jdvjQTAz372MzkcDj377LO+YxkZGYqKitLzzz8vy7IUHx+ve+65R/fee68kyePxyOFwqKSkRJmZmdqzZ4+SkpJUU1Oj5ORkSVJpaammTp2qzz//XPHx8d85D6/XK7vdLo/HI5vNFsi3CABAn3BB/mudnvt0cXq3vGZXv38H/A7MxIkTVV5ero8++kiS9M9//lNvv/22brzxRknS/v375Xa7lZqa6rvGbrcrJSVFVVVVkqSqqipFR0f74kWSUlNTFRoaqurq6jO+bltbm7xer98DAACcm8IC/YT5+fnyer0aNWqU+vXrp/b2dj388MPKysqSJLndbkmSw+Hwu87hcPjOud1uxcbG+k80LEwxMTG+MacqKirSwoULA/12AABALxTwOzAvvfSS1q9frxdeeEG7du3SunXrtGzZMq1bty7QL+WnoKBAHo/H92hsbOzW1wMAAMET8Dsw8+bNU35+vjIzMyVJY8aM0WeffaaioiJlZ2fL6XRKkpqamhQXF+e7rqmpSePGjZMkOZ1ONTc3+z3viRMndOjQId/1p4qIiFBERESg3w4AAOiFAn4H5quvvlJoqP/T9uvXTx0dHZKkxMREOZ1OlZeX+857vV5VV1fL5XJJklwul1paWlRbW+sbs3XrVnV0dCglJSXQUwYAAIYJ+B2Ym266SQ8//LCGDRumiy++WLt379by5ct1xx13SJJCQkKUl5enhx56SCNGjFBiYqIWLFig+Ph4TZs2TZI0evRo3XDDDZo5c6ZWr16t48ePKzc3V5mZmV36CSQAAHBuC3jAPP7441qwYIF+97vfqbm5WfHx8brzzjtVWFjoGzN//ny1trZq1qxZamlp0VVXXaXS0lJFRkb6xqxfv165ubmaPHmyQkNDlZGRoZUrVwZ6ugAAwEAB/z0wvQW/BwYAgB+mT/0eGAAAgO5GwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDhhwZ6AiS7If63Tc58uTu/BmQAA0DdxBwYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHG6JWD+/e9/61e/+pWGDBmiqKgojRkzRu+++67vvGVZKiwsVFxcnKKiopSamqp9+/b5PcehQ4eUlZUlm82m6OhozZgxQ0eOHOmO6QIAAMMEPGC+/PJLTZo0Sf3799cbb7yhDz74QH/+8581ePBg35ilS5dq5cqVWr16taqrqzVw4EClpaXp6NGjvjFZWVlqaGhQWVmZNm/erMrKSs2aNSvQ0wUAAAYKC/QTLlmyRAkJCVq7dq3vWGJiou/PlmXp0Ucf1X333aebb75ZkvSXv/xFDodDmzZtUmZmpvbs2aPS0lLV1NQoOTlZkvT4449r6tSpWrZsmeLj4wM9bQAAYJCA34F55ZVXlJycrJ///OeKjY3VZZddpqefftp3fv/+/XK73UpNTfUds9vtSklJUVVVlSSpqqpK0dHRvniRpNTUVIWGhqq6uvqMr9vW1iav1+v3AAAA56aAB8wnn3yiVatWacSIEXrzzTc1e/Zs/f73v9e6deskSW63W5LkcDj8rnM4HL5zbrdbsbGxfufDwsIUExPjG3OqoqIi2e123yMhISHQbw0AAPQSAQ+Yjo4OjR8/Xo888oguu+wyzZo1SzNnztTq1asD/VJ+CgoK5PF4fI/GxsZufT0AABA8AQ+YuLg4JSUl+R0bPXq0Dhw4IElyOp2SpKamJr8xTU1NvnNOp1PNzc1+50+cOKFDhw75xpwqIiJCNpvN7wEAAM5NAQ+YSZMmae/evX7HPvroIw0fPlzS1x/odTqdKi8v9533er2qrq6Wy+WSJLlcLrW0tKi2ttY3ZuvWrero6FBKSkqgpwwAAAwT8J9CuvvuuzVx4kQ98sgjuvXWW7Vz50499dRTeuqppyRJISEhysvL00MPPaQRI0YoMTFRCxYsUHx8vKZNmybp6zs2N9xwg++fno4fP67c3FxlZmbyE0gAACDwAXP55Zdr48aNKigo0IMPPqjExEQ9+uijysrK8o2ZP3++WltbNWvWLLW0tOiqq65SaWmpIiMjfWPWr1+v3NxcTZ48WaGhocrIyNDKlSsDPV0AAGCgEMuyrGBPojt4vV7Z7XZ5PJ6Afx7mgvzXOj336eL0gL4WAADBEozvd139/s3/hQQAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwTrcHzOLFixUSEqK8vDzfsaNHjyonJ0dDhgzReeedp4yMDDU1Nfldd+DAAaWnp2vAgAGKjY3VvHnzdOLEie6eLgAAMEC3BkxNTY3WrFmjsWPH+h2/++679eqrr+rll19WRUWFDh48qFtuucV3vr29Xenp6Tp27Ji2b9+udevWqaSkRIWFhd05XQAAYIhuC5gjR44oKytLTz/9tAYPHuw77vF49Oyzz2r58uW6/vrrNWHCBK1du1bbt2/Xjh07JElbtmzRBx98oOeff17jxo3TjTfeqEWLFqm4uFjHjh3rrikDAABDdFvA5OTkKD09XampqX7Ha2trdfz4cb/jo0aN0rBhw1RVVSVJqqqq0pgxY+RwOHxj0tLS5PV61dDQcMbXa2trk9fr9XsAAIBzU1h3POmGDRu0a9cu1dTUnHbO7XYrPDxc0dHRfscdDofcbrdvzDfj5eT5k+fOpKioSAsXLgzA7AEAQG8X8DswjY2NmjNnjtavX6/IyMhAP32nCgoK5PF4fI/GxsYee20AANCzAh4wtbW1am5u1vjx4xUWFqawsDBVVFRo5cqVCgsLk8Ph0LFjx9TS0uJ3XVNTk5xOpyTJ6XSe9lNJJ78+OeZUERERstlsfg8AAHBuCnjATJ48WfX19aqrq/M9kpOTlZWV5ftz//79VV5e7rtm7969OnDggFwulyTJ5XKpvr5ezc3NvjFlZWWy2WxKSkoK9JQBAIBhAv4ZmEGDBumSSy7xOzZw4EANGTLEd3zGjBmaO3euYmJiZLPZdNddd8nlcunKK6+UJE2ZMkVJSUmaPn26li5dKrfbrfvuu085OTmKiIgI9JQBAIBhuuVDvN9lxYoVCg0NVUZGhtra2pSWlqYnn3zSd75fv37avHmzZs+eLZfLpYEDByo7O1sPPvhgMKYLAAB6mR4JmG3btvl9HRkZqeLiYhUXF3d6zfDhw/X6669388wAAICJ+L+QAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABgnLNgTAPq6C/Jf6/Tcp4vTe3AmAGAO7sAAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwTsADpqioSJdffrkGDRqk2NhYTZs2TXv37vUbc/ToUeXk5GjIkCE677zzlJGRoaamJr8xBw4cUHp6ugYMGKDY2FjNmzdPJ06cCPR0AQCAgQIeMBUVFcrJydGOHTtUVlam48ePa8qUKWptbfWNufvuu/Xqq6/q5ZdfVkVFhQ4ePKhbbrnFd769vV3p6ek6duyYtm/frnXr1qmkpESFhYWBni4AADBQWKCfsLS01O/rkpISxcbGqra2VldffbU8Ho+effZZvfDCC7r++uslSWvXrtXo0aO1Y8cOXXnlldqyZYs++OADvfXWW3I4HBo3bpwWLVqkP/zhD3rggQcUHh4e6GkDAACDdPtnYDwejyQpJiZGklRbW6vjx48rNTXVN2bUqFEaNmyYqqqqJElVVVUaM2aMHA6Hb0xaWpq8Xq8aGhrO+DptbW3yer1+DwAAcG7q1oDp6OhQXl6eJk2apEsuuUSS5Ha7FR4erujoaL+xDodDbrfbN+ab8XLy/MlzZ1JUVCS73e57JCQkBPjdAACA3qJbAyYnJ0fvv/++NmzY0J0vI0kqKCiQx+PxPRobG7v9NQEAQHAE/DMwJ+Xm5mrz5s2qrKzU0KFDfcedTqeOHTumlpYWv7swTU1NcjqdvjE7d+70e76TP6V0csypIiIiFBEREeB3AQAAeqOA34GxLEu5ubnauHGjtm7dqsTERL/zEyZMUP/+/VVeXu47tnfvXh04cEAul0uS5HK5VF9fr+bmZt+YsrIy2Ww2JSUlBXrKAADAMAG/A5OTk6MXXnhBf//73zVo0CDfZ1bsdruioqJkt9s1Y8YMzZ07VzExMbLZbLrrrrvkcrl05ZVXSpKmTJmipKQkTZ8+XUuXLpXb7dZ9992nnJwc7rIAAIDAB8yqVaskSddee63f8bVr1+o3v/mNJGnFihUKDQ1VRkaG2tralJaWpieffNI3tl+/ftq8ebNmz54tl8ulgQMHKjs7Ww8++GCgpwsAAAwU8ICxLOs7x0RGRqq4uFjFxcWdjhk+fLhef/31QE4NAACcI/i/kAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGCcsGBPAAAQOBfkv9bpuU8Xp/fgTIDuxR0YAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHF6dcAUFxfrggsuUGRkpFJSUrRz585gTwkAAPQCvTZgXnzxRc2dO1f333+/du3apUsvvVRpaWlqbm4O9tQAAECQ9dqAWb58uWbOnKnbb79dSUlJWr16tQYMGKDnnnsu2FMDAABB1iv/L6Rjx46ptrZWBQUFvmOhoaFKTU1VVVXVGa9pa2tTW1ub72uPxyNJ8nq9AZ9fR9tXnZ7rjtfDuY39hEBiPyGQgrGfTj6vZVnfOq5XBsx///tftbe3y+Fw+B13OBz68MMPz3hNUVGRFi5ceNrxhISEbpljZ+yP9ujL4RzHfkIgsZ8QSN29nw4fPiy73d7p+V4ZMN9HQUGB5s6d6/u6o6NDhw4d0pAhQxQSEhKw1/F6vUpISFBjY6NsNlvAnvdcxXp1HWvVdaxV17FWXcdadV13rpVlWTp8+LDi4+O/dVyvDJjzzz9f/fr1U1NTk9/xpqYmOZ3OM14TERGhiIgIv2PR0dHdNUXZbDY2+FlgvbqOteo61qrrWKuuY626rrvW6tvuvJzUKz/EGx4ergkTJqi8vNx3rKOjQ+Xl5XK5XEGcGQAA6A165R0YSZo7d66ys7OVnJysK664Qo8++qhaW1t1++23B3tqAAAgyHptwNx22236z3/+o8LCQrndbo0bN06lpaWnfbC3p0VEROj+++8/7Z+rcGasV9exVl3HWnUda9V1rFXX9Ya1CrG+6+eUAAAAeple+RkYAACAb0PAAAAA4xAwAADAOAQMAAAwDgFzisrKSt10002Kj49XSEiINm3a9J3XbNu2TePHj1dERIQuuugilZSUdPs8e4OzXatt27YpJCTktIfb7e6ZCQdRUVGRLr/8cg0aNEixsbGaNm2a9u7d+53Xvfzyyxo1apQiIyM1ZswYvf766z0w2+D6PmtVUlJy2r6KjIzsoRkHz6pVqzR27FjfLxNzuVx64403vvWavrinpLNfq766p85k8eLFCgkJUV5e3reO6+m9RcCcorW1VZdeeqmKi4u7NH7//v1KT0/Xddddp7q6OuXl5em3v/2t3nzzzW6eafCd7VqdtHfvXn3xxRe+R2xsbDfNsPeoqKhQTk6OduzYobKyMh0/flxTpkxRa2trp9ds375dv/jFLzRjxgzt3r1b06ZN07Rp0/T+++/34Mx73vdZK+nr3wj6zX312Wef9dCMg2fo0KFavHixamtr9e677+r666/XzTffrIaGhjOO76t7Sjr7tZL65p46VU1NjdasWaOxY8d+67ig7C0LnZJkbdy48VvHzJ8/37r44ov9jt12221WWlpaN86s9+nKWv3jH/+wJFlffvllj8ypN2tubrYkWRUVFZ2OufXWW6309HS/YykpKdadd97Z3dPrVbqyVmvXrrXsdnvPTaoXGzx4sPXMM8+c8Rx7yt+3rRV7yrIOHz5sjRgxwiorK7OuueYaa86cOZ2ODcbe4g7MD1RVVaXU1FS/Y2lpaaqqqgrSjHq/cePGKS4uTj/96U/1zjvvBHs6QeHxeCRJMTExnY5hb32tK2slSUeOHNHw4cOVkJDwnX+zPhe1t7drw4YNam1t7fS/XGFPfa0rayWxp3JycpSenn7anjmTYOytXvubeE3hdrtP++3ADodDXq9X//vf/xQVFRWkmfU+cXFxWr16tZKTk9XW1qZnnnlG1157raqrqzV+/PhgT6/HdHR0KC8vT5MmTdIll1zS6bjO9lZf+MzQSV1dq5EjR+q5557T2LFj5fF4tGzZMk2cOFENDQ0aOnRoD86459XX18vlcuno0aM677zztHHjRiUlJZ1xbF/fU2ezVn15T0nShg0btGvXLtXU1HRpfDD2FgGDHjNy5EiNHDnS9/XEiRP18ccfa8WKFfrrX/8axJn1rJycHL3//vt6++23gz2VXq+ra+Vyufz+Jj1x4kSNHj1aa9as0aJFi7p7mkE1cuRI1dXVyePx6G9/+5uys7NVUVHR6Tfmvuxs1qov76nGxkbNmTNHZWVlvfqDywTMD+R0OtXU1OR3rKmpSTabjbsvXXDFFVf0qW/kubm52rx5syorK7/zb3Gd7S2n09mdU+w1zmatTtW/f39ddtll+te//tVNs+s9wsPDddFFF0mSJkyYoJqaGj322GNas2bNaWP7+p46m7U6VV/aU7W1tWpubva7M97e3q7Kyko98cQTamtrU79+/fyuCcbe4jMwP5DL5VJ5ebnfsbKysm/9d1X8f3V1dYqLiwv2NLqdZVnKzc3Vxo0btXXrViUmJn7nNX11b32ftTpVe3u76uvr+8TeOlVHR4fa2trOeK6v7qnOfNtanaov7anJkyervr5edXV1vkdycrKysrJUV1d3WrxIQdpb3fbxYEMdPnzY2r17t7V7925LkrV8+XJr9+7d1meffWZZlmXl5+db06dP943/5JNPrAEDBljz5s2z9uzZYxUXF1v9+vWzSktLg/UWeszZrtWKFSusTZs2Wfv27bPq6+utOXPmWKGhodZbb70VrLfQY2bPnm3Z7XZr27Zt1hdffOF7fPXVV74x06dPt/Lz831fv/POO1ZYWJi1bNkya8+ePdb9999v9e/f36qvrw/GW+gx32etFi5caL355pvWxx9/bNXW1lqZmZlWZGSk1dDQEIy30GPy8/OtiooKa//+/dZ7771n5efnWyEhIdaWLVssy2JPfdPZrlVf3VOdOfWnkHrD3iJgTnHyR31PfWRnZ1uWZVnZ2dnWNddcc9o148aNs8LDw60f//jH1tq1a3t83sFwtmu1ZMkS68ILL7QiIyOtmJgY69prr7W2bt0anMn3sDOtkyS/vXLNNdf41u6kl156yfrJT35ihYeHWxdffLH12muv9ezEg+D7rFVeXp41bNgwKzw83HI4HNbUqVOtXbt29fzke9gdd9xhDR8+3AoPD7d+9KMfWZMnT/Z9Q7Ys9tQ3ne1a9dU91ZlTA6Y37K0Qy7Ks7ru/AwAAEHh8BgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGCc/wddyZ1+T1DuEgAAAABJRU5ErkJggg==",
"text/plain": [
"<Figure size 640x480 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"plt.hist(mkbatch(2048)[1].to(torch.float32).cpu(), bins=64)"
]
},
{
"cell_type": "code",
"execution_count": 90,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(array([1714., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 305., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 29.]),\n",
" array([1. , 1.03125, 1.0625 , 1.09375, 1.125 , 1.15625, 1.1875 ,\n",
" 1.21875, 1.25 , 1.28125, 1.3125 , 1.34375, 1.375 , 1.40625,\n",
" 1.4375 , 1.46875, 1.5 , 1.53125, 1.5625 , 1.59375, 1.625 ,\n",
" 1.65625, 1.6875 , 1.71875, 1.75 , 1.78125, 1.8125 , 1.84375,\n",
" 1.875 , 1.90625, 1.9375 , 1.96875, 2. , 2.03125, 2.0625 ,\n",
" 2.09375, 2.125 , 2.15625, 2.1875 , 2.21875, 2.25 , 2.28125,\n",
" 2.3125 , 2.34375, 2.375 , 2.40625, 2.4375 , 2.46875, 2.5 ,\n",
" 2.53125, 2.5625 , 2.59375, 2.625 , 2.65625, 2.6875 , 2.71875,\n",
" 2.75 , 2.78125, 2.8125 , 2.84375, 2.875 , 2.90625, 2.9375 ,\n",
" 2.96875, 3. ]),\n",
" <BarContainer object of 64 artists>)"
]
},
"execution_count": 90,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvkklEQVR4nO3df1xUdd7//yeIgJkziAbjXOGP2lIx09KkKfvhSqKSm5ttUeS6LemuF9SaZeJtV8tqF7OuTLtM1q4Kr82y2tI2K5I0YdcIFeXyR0rWmj9yB9o1ZoRWRDnfP/pwvo1CCQ0yb3rcb7dzy3m/X+ec93vfnJ2nhzNjmGVZlgAAAAwS3tYDAAAAaC4CDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOBFtPYDWUl9fr0OHDqlLly4KCwtr6+EAAIDTYFmWjhw5IrfbrfDwpu+ztNsAc+jQISUkJLT1MAAAQAscOHBA5557bpP97TbAdOnSRdLX/wM4HI42Hg0AADgdfr9fCQkJ9vt4U9ptgGn4tZHD4SDAAABgmO96/IOHeAEAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACME9HWAzBV7+y3muz7bF7qGRwJAAA/PNyBAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHGaHWCKioo0btw4ud1uhYWFadWqVafU7Nq1Sz/5yU/kdDrVuXNnXXbZZdq/f7/df/ToUWVmZqpbt246++yzNWHCBFVUVAQcY//+/UpNTdVZZ52luLg4zZgxQ8ePH2/+DAEAQLvT7ABTU1OjQYMGafHixY32f/rppxo+fLj69eun9evXa9u2bZo9e7aio6PtmnvuuUdvvvmmXn31VRUWFurQoUO68cYb7f4TJ04oNTVVx44d0wcffKBly5YpLy9Pc+bMacEUAQBAexNmWZbV4p3DwrRy5UqNHz/ebktLS1PHjh31pz/9qdF9fD6fzjnnHL344ou66aabJEm7d+9W//79VVxcrMsvv1zvvPOOrr/+eh06dEjx8fGSpNzcXM2cOVNffPGFIiMjv3Nsfr9fTqdTPp9PDoejpVNsUu/st5rs+2xeatDPBwDAD8Hpvn8H9RmY+vp6vfXWW7rwwguVkpKiuLg4JSUlBfyaqbS0VHV1dUpOTrbb+vXrp549e6q4uFiSVFxcrIEDB9rhRZJSUlLk9/u1c+fORs9dW1srv98fsAEAgPYpqAGmsrJS1dXVmjdvnkaPHq01a9bopz/9qW688UYVFhZKkrxeryIjIxUTExOwb3x8vLxer13zzfDS0N/Q15icnBw5nU57S0hICObUAABACAn6HRhJuuGGG3TPPfdo8ODBys7O1vXXX6/c3NxgnuoUs2bNks/ns7cDBw606vkAAEDbCWqA6d69uyIiIpSYmBjQ3r9/f/tTSC6XS8eOHVNVVVVATUVFhVwul11z8qeSGl431JwsKipKDocjYAMAAO1TUANMZGSkLrvsMpWXlwe0f/zxx+rVq5ckaciQIerYsaPWrl1r95eXl2v//v3yeDySJI/Ho+3bt6uystKuKSgokMPhOCUcAQCAH56I5u5QXV2tTz75xH69d+9elZWVKTY2Vj179tSMGTN0yy236Oqrr9aIESOUn5+vN998U+vXr5ckOZ1OZWRkaPr06YqNjZXD4dBdd90lj8ejyy+/XJI0atQoJSYmauLEiZo/f768Xq9+97vfKTMzU1FRUcGZOQAAMFazA8zmzZs1YsQI+/X06dMlSZMmTVJeXp5++tOfKjc3Vzk5Obr77rvVt29fvfbaaxo+fLi9z4IFCxQeHq4JEyaotrZWKSkpevrpp+3+Dh06aPXq1Zo6dao8Ho86d+6sSZMm6aGHHvo+cwUAAO3E9/oemFDG98AAAGCeNvkeGAAAgDOBAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDjNDjBFRUUaN26c3G63wsLCtGrVqiZrf/3rXyssLExPPvlkQPvhw4eVnp4uh8OhmJgYZWRkqLq6OqBm27ZtuuqqqxQdHa2EhATNnz+/uUMFAADtVLMDTE1NjQYNGqTFixd/a93KlSv14Ycfyu12n9KXnp6unTt3qqCgQKtXr1ZRUZGmTJli9/v9fo0aNUq9evVSaWmpHnvsMT344INaunRpc4cLAADaoYjm7jBmzBiNGTPmW2s+//xz3XXXXXr33XeVmpoa0Ldr1y7l5+dr06ZNGjp0qCTpqaee0tixY/X444/L7XZr+fLlOnbsmJ577jlFRkZqwIABKisr0xNPPBEQdAAAwA9T0J+Bqa+v18SJEzVjxgwNGDDglP7i4mLFxMTY4UWSkpOTFR4erpKSErvm6quvVmRkpF2TkpKi8vJyffnll42et7a2Vn6/P2ADAADtU9ADzKOPPqqIiAjdfffdjfZ7vV7FxcUFtEVERCg2NlZer9euiY+PD6hpeN1Qc7KcnBw5nU57S0hI+L5TAQAAISqoAaa0tFQLFy5UXl6ewsLCgnno7zRr1iz5fD57O3DgwBk9PwAAOHOCGmD++te/qrKyUj179lRERIQiIiK0b98+3Xvvverdu7ckyeVyqbKyMmC/48eP6/Dhw3K5XHZNRUVFQE3D64aak0VFRcnhcARsAACgfQpqgJk4caK2bdumsrIye3O73ZoxY4beffddSZLH41FVVZVKS0vt/datW6f6+nolJSXZNUVFRaqrq7NrCgoK1LdvX3Xt2jWYQwYAAAZq9qeQqqur9cknn9iv9+7dq7KyMsXGxqpnz57q1q1bQH3Hjh3lcrnUt29fSVL//v01evRoTZ48Wbm5uaqrq1NWVpbS0tLsj1zfdtttmjt3rjIyMjRz5kzt2LFDCxcu1IIFC77PXAEAQDvR7ACzefNmjRgxwn49ffp0SdKkSZOUl5d3WsdYvny5srKyNHLkSIWHh2vChAlatGiR3e90OrVmzRplZmZqyJAh6t69u+bMmcNHqAEAgCQpzLIsq60H0Rr8fr+cTqd8Pl+rPA/TO/utJvs+m5faZB8AAGja6b5/828hAQAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYp9kBpqioSOPGjZPb7VZYWJhWrVpl99XV1WnmzJkaOHCgOnfuLLfbrZ///Oc6dOhQwDEOHz6s9PR0ORwOxcTEKCMjQ9XV1QE127Zt01VXXaXo6GglJCRo/vz5LZshAABod5odYGpqajRo0CAtXrz4lL6vvvpKW7Zs0ezZs7Vlyxa9/vrrKi8v109+8pOAuvT0dO3cuVMFBQVavXq1ioqKNGXKFLvf7/dr1KhR6tWrl0pLS/XYY4/pwQcf1NKlS1swRQAA0N6EWZZltXjnsDCtXLlS48ePb7Jm06ZNGjZsmPbt26eePXtq165dSkxM1KZNmzR06FBJUn5+vsaOHauDBw/K7XZryZIl+u1vfyuv16vIyEhJUnZ2tlatWqXdu3ef1tj8fr+cTqd8Pp8cDkdLp9ik3tlvNdn32bzUoJ8PAIAfgtN9/271Z2B8Pp/CwsIUExMjSSouLlZMTIwdXiQpOTlZ4eHhKikpsWuuvvpqO7xIUkpKisrLy/Xll182ep7a2lr5/f6ADQAAtE+tGmCOHj2qmTNn6tZbb7VTlNfrVVxcXEBdRESEYmNj5fV67Zr4+PiAmobXDTUny8nJkdPptLeEhIRgTwcAAISIVgswdXV1uvnmm2VZlpYsWdJap7HNmjVLPp/P3g4cONDq5wQAAG0jojUO2hBe9u3bp3Xr1gX8DsvlcqmysjKg/vjx4zp8+LBcLpddU1FREVDT8Lqh5mRRUVGKiooK5jQAAECICvodmIbwsmfPHr333nvq1q1bQL/H41FVVZVKS0vttnXr1qm+vl5JSUl2TVFRkerq6uyagoIC9e3bV127dg32kAEAgGGaHWCqq6tVVlamsrIySdLevXtVVlam/fv3q66uTjfddJM2b96s5cuX68SJE/J6vfJ6vTp27JgkqX///ho9erQmT56sjRs3asOGDcrKylJaWprcbrck6bbbblNkZKQyMjK0c+dOvfzyy1q4cKGmT58evJkDAABjNftj1OvXr9eIESNOaZ80aZIefPBB9enTp9H93n//fV177bWSvv4iu6ysLL355psKDw/XhAkTtGjRIp199tl2/bZt25SZmalNmzape/fuuuuuuzRz5szTHicfowYAwDyn+/79vb4HJpQRYAAAME/IfA8MAABAsBFgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYp9kBpqioSOPGjZPb7VZYWJhWrVoV0G9ZlubMmaMePXqoU6dOSk5O1p49ewJqDh8+rPT0dDkcDsXExCgjI0PV1dUBNdu2bdNVV12l6OhoJSQkaP78+c2fHQAAaJeaHWBqamo0aNAgLV68uNH++fPna9GiRcrNzVVJSYk6d+6slJQUHT161K5JT0/Xzp07VVBQoNWrV6uoqEhTpkyx+/1+v0aNGqVevXqptLRUjz32mB588EEtXbq0BVMEAADtTZhlWVaLdw4L08qVKzV+/HhJX999cbvduvfee3XfffdJknw+n+Lj45WXl6e0tDTt2rVLiYmJ2rRpk4YOHSpJys/P19ixY3Xw4EG53W4tWbJEv/3tb+X1ehUZGSlJys7O1qpVq7R79+7TGpvf75fT6ZTP55PD4WjpFJvUO/utJvs+m5ca9PMBAPBDcLrv30F9Bmbv3r3yer1KTk6225xOp5KSklRcXCxJKi4uVkxMjB1eJCk5OVnh4eEqKSmxa66++mo7vEhSSkqKysvL9eWXXzZ67traWvn9/oANAAC0T0ENMF6vV5IUHx8f0B4fH2/3eb1excXFBfRHREQoNjY2oKaxY3zzHCfLycmR0+m0t4SEhO8/IQAAEJLazaeQZs2aJZ/PZ28HDhxo6yEBAIBWEtQA43K5JEkVFRUB7RUVFXafy+VSZWVlQP/x48d1+PDhgJrGjvHNc5wsKipKDocjYAMAAO1TUANMnz595HK5tHbtWrvN7/erpKREHo9HkuTxeFRVVaXS0lK7Zt26daqvr1dSUpJdU1RUpLq6OrumoKBAffv2VdeuXYM5ZAAAYKBmB5jq6mqVlZWprKxM0tcP7paVlWn//v0KCwvTtGnT9Mgjj+gvf/mLtm/frp///Odyu932J5X69++v0aNHa/Lkydq4caM2bNigrKwspaWlye12S5Juu+02RUZGKiMjQzt37tTLL7+shQsXavr06UGbOAAAMFdEc3fYvHmzRowYYb9uCBWTJk1SXl6e7r//ftXU1GjKlCmqqqrS8OHDlZ+fr+joaHuf5cuXKysrSyNHjlR4eLgmTJigRYsW2f1Op1Nr1qxRZmamhgwZou7du2vOnDkB3xUDAAB+uL7X98CEMr4HBgAA87TJ98AAAACcCQQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGCXqAOXHihGbPnq0+ffqoU6dOOv/88/Xwww/Lsiy7xrIszZkzRz169FCnTp2UnJysPXv2BBzn8OHDSk9Pl8PhUExMjDIyMlRdXR3s4QIAAAMFPcA8+uijWrJkif77v/9bu3bt0qOPPqr58+frqaeesmvmz5+vRYsWKTc3VyUlJercubNSUlJ09OhRuyY9PV07d+5UQUGBVq9eraKiIk2ZMiXYwwUAAAYKs755ayQIrr/+esXHx+vZZ5+12yZMmKBOnTrphRdekGVZcrvduvfee3XfffdJknw+n+Lj45WXl6e0tDTt2rVLiYmJ2rRpk4YOHSpJys/P19ixY3Xw4EG53e7vHIff75fT6ZTP55PD4QjmFCVJvbPfarLvs3mpQT8fAAA/BKf7/h30OzBXXHGF1q5dq48//liS9H//93/629/+pjFjxkiS9u7dK6/Xq+TkZHsfp9OppKQkFRcXS5KKi4sVExNjhxdJSk5OVnh4uEpKSho9b21trfx+f8AGAADap4hgHzA7O1t+v1/9+vVThw4ddOLECf3+979Xenq6JMnr9UqS4uPjA/aLj4+3+7xer+Li4gIHGhGh2NhYu+ZkOTk5mjt3brCnAwAAQlDQ78C88sorWr58uV588UVt2bJFy5Yt0+OPP65ly5YF+1QBZs2aJZ/PZ28HDhxo1fMBAIC2E/Q7MDNmzFB2drbS0tIkSQMHDtS+ffuUk5OjSZMmyeVySZIqKirUo0cPe7+KigoNHjxYkuRyuVRZWRlw3OPHj+vw4cP2/ieLiopSVFRUsKcDAABCUNDvwHz11VcKDw88bIcOHVRfXy9J6tOnj1wul9auXWv3+/1+lZSUyOPxSJI8Ho+qqqpUWlpq16xbt0719fVKSkoK9pABAIBhgn4HZty4cfr973+vnj17asCAAdq6daueeOIJ/fKXv5QkhYWFadq0aXrkkUd0wQUXqE+fPpo9e7bcbrfGjx8vSerfv79Gjx6tyZMnKzc3V3V1dcrKylJaWtppfQIJAAC0b0EPME899ZRmz56t//zP/1RlZaXcbrd+9atfac6cOXbN/fffr5qaGk2ZMkVVVVUaPny48vPzFR0dbdcsX75cWVlZGjlypMLDwzVhwgQtWrQo2MMFAAAGCvr3wIQKvgcGAADztNn3wAAAALQ2AgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOO0SoD5/PPPdfvtt6tbt27q1KmTBg4cqM2bN9v9lmVpzpw56tGjhzp16qTk5GTt2bMn4BiHDx9Wenq6HA6HYmJilJGRoerq6tYYLgAAMEzQA8yXX36pK6+8Uh07dtQ777yjjz76SP/1X/+lrl272jXz58/XokWLlJubq5KSEnXu3FkpKSk6evSoXZOenq6dO3eqoKBAq1evVlFRkaZMmRLs4QIAAAOFWZZlBfOA2dnZ2rBhg/7617822m9Zltxut+69917dd999kiSfz6f4+Hjl5eUpLS1Nu3btUmJiojZt2qShQ4dKkvLz8zV27FgdPHhQbrf7O8fh9/vldDrl8/nkcDiCN8H/p3f2W032fTYvNejnAwDgh+B037+DfgfmL3/5i4YOHaqf/exniouL0yWXXKJnnnnG7t+7d6+8Xq+Sk5PtNqfTqaSkJBUXF0uSiouLFRMTY4cXSUpOTlZ4eLhKSkoaPW9tba38fn/ABgAA2qegB5i///3vWrJkiS644AK9++67mjp1qu6++24tW7ZMkuT1eiVJ8fHxAfvFx8fbfV6vV3FxcQH9ERERio2NtWtOlpOTI6fTaW8JCQnBnhoAAAgRQQ8w9fX1uvTSS/WHP/xBl1xyiaZMmaLJkycrNzc32KcKMGvWLPl8Pns7cOBAq54PAAC0naAHmB49eigxMTGgrX///tq/f78kyeVySZIqKioCaioqKuw+l8ulysrKgP7jx4/r8OHDds3JoqKi5HA4AjYAANA+BT3AXHnllSovLw9o+/jjj9WrVy9JUp8+feRyubR27Vq73+/3q6SkRB6PR5Lk8XhUVVWl0tJSu2bdunWqr69XUlJSsIcMAAAMExHsA95zzz264oor9Ic//EE333yzNm7cqKVLl2rp0qWSpLCwME2bNk2PPPKILrjgAvXp00ezZ8+W2+3W+PHjJX19x2b06NH2r57q6uqUlZWltLS00/oEEgAAaN+CHmAuu+wyrVy5UrNmzdJDDz2kPn366Mknn1R6erpdc//996umpkZTpkxRVVWVhg8frvz8fEVHR9s1y5cvV1ZWlkaOHKnw8HBNmDBBixYtCvZwAQCAgYL+PTChgu+BAQDAPG32PTAAAACtjQADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4rR5g5s2bp7CwME2bNs1uO3r0qDIzM9WtWzedffbZmjBhgioqKgL2279/v1JTU3XWWWcpLi5OM2bM0PHjx1t7uAAAwACtGmA2bdqkP/7xj7r44osD2u+55x69+eabevXVV1VYWKhDhw7pxhtvtPtPnDih1NRUHTt2TB988IGWLVumvLw8zZkzpzWHCwAADNFqAaa6ulrp6el65pln1LVrV7vd5/Pp2Wef1RNPPKEf//jHGjJkiJ5//nl98MEH+vDDDyVJa9as0UcffaQXXnhBgwcP1pgxY/Twww9r8eLFOnbsWGsNGQAAGKLVAkxmZqZSU1OVnJwc0F5aWqq6urqA9n79+qlnz54qLi6WJBUXF2vgwIGKj4+3a1JSUuT3+7Vz585Gz1dbWyu/3x+wAQCA9imiNQ66YsUKbdmyRZs2bTqlz+v1KjIyUjExMQHt8fHx8nq9ds03w0tDf0NfY3JycjR37twgjB4AAIS6oN+BOXDggH7zm99o+fLlio6ODvbhmzRr1iz5fD57O3DgwBk7NwAAOLOCHmBKS0tVWVmpSy+9VBEREYqIiFBhYaEWLVqkiIgIxcfH69ixY6qqqgrYr6KiQi6XS5LkcrlO+VRSw+uGmpNFRUXJ4XAEbAAAoH0KeoAZOXKktm/frrKyMnsbOnSo0tPT7T937NhRa9eutfcpLy/X/v375fF4JEkej0fbt29XZWWlXVNQUCCHw6HExMRgDxkAABgm6M/AdOnSRRdddFFAW+fOndWtWze7PSMjQ9OnT1dsbKwcDofuuusueTweXX755ZKkUaNGKTExURMnTtT8+fPl9Xr1u9/9TpmZmYqKigr2kAEAgGFa5SHe77JgwQKFh4drwoQJqq2tVUpKip5++mm7v0OHDlq9erWmTp0qj8ejzp07a9KkSXrooYfaYrgAACDEhFmWZbX1IFqD3++X0+mUz+drledheme/1WTfZ/NSg34+AAB+CE73/Zt/CwkAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcSLaegAA0BL8i/DADxt3YAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjBD3A5OTk6LLLLlOXLl0UFxen8ePHq7y8PKDm6NGjyszMVLdu3XT22WdrwoQJqqioCKjZv3+/UlNTddZZZykuLk4zZszQ8ePHgz1cAABgoKAHmMLCQmVmZurDDz9UQUGB6urqNGrUKNXU1Ng199xzj9588029+uqrKiws1KFDh3TjjTfa/SdOnFBqaqqOHTumDz74QMuWLVNeXp7mzJkT7OECAAADhVmWZbXmCb744gvFxcWpsLBQV199tXw+n8455xy9+OKLuummmyRJu3fvVv/+/VVcXKzLL79c77zzjq6//nodOnRI8fHxkqTc3FzNnDlTX3zxhSIjI7/zvH6/X06nUz6fTw6HI+jz6p39VpN9n81LDfr5AATiGgTap9N9/271Z2B8Pp8kKTY2VpJUWlqquro6JScn2zX9+vVTz549VVxcLEkqLi7WwIED7fAiSSkpKfL7/dq5c2ej56mtrZXf7w/YAABA+9SqAaa+vl7Tpk3TlVdeqYsuukiS5PV6FRkZqZiYmIDa+Ph4eb1eu+ab4aWhv6GvMTk5OXI6nfaWkJAQ5NkAAIBQ0aoBJjMzUzt27NCKFSta8zSSpFmzZsnn89nbgQMHWv2cAACgbUS01oGzsrK0evVqFRUV6dxzz7XbXS6Xjh07pqqqqoC7MBUVFXK5XHbNxo0bA47X8CmlhpqTRUVFKSoqKsizAAAAoSjod2Asy1JWVpZWrlypdevWqU+fPgH9Q4YMUceOHbV27Vq7rby8XPv375fH45EkeTwebd++XZWVlXZNQUGBHA6HEhMTgz1kAABgmKDfgcnMzNSLL76oN954Q126dLGfWXE6nerUqZOcTqcyMjI0ffp0xcbGyuFw6K677pLH49Hll18uSRo1apQSExM1ceJEzZ8/X16vV7/73e+UmZnJXRYAABD8ALNkyRJJ0rXXXhvQ/vzzz+sXv/iFJGnBggUKDw/XhAkTVFtbq5SUFD399NN2bYcOHbR69WpNnTpVHo9HnTt31qRJk/TQQw8Fe7gAAMBAQQ8wp/O1MtHR0Vq8eLEWL17cZE2vXr309ttvB3NoAACgneDfQgIAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgRbT0AAAAQmnpnv9Vk32fzUs/gSE7FHRgAAGCckA4wixcvVu/evRUdHa2kpCRt3LixrYcEAABCQMgGmJdfflnTp0/XAw88oC1btmjQoEFKSUlRZWVlWw8NAAC0sZANME888YQmT56sO+64Q4mJicrNzdVZZ52l5557rq2HBgAA2lhIPsR77NgxlZaWatasWXZbeHi4kpOTVVxc3Og+tbW1qq2ttV/7fD5Jkt/vb5Ux1td+1WRfa50TwP+PaxBofW1xnTUc17Ksb60LyQDzz3/+UydOnFB8fHxAe3x8vHbv3t3oPjk5OZo7d+4p7QkJCa0yxm/jfPKMnxLAN3ANAq2vta+zI0eOyOl0NtkfkgGmJWbNmqXp06fbr+vr63X48GF169ZNYWFhQT2X3+9XQkKCDhw4IIfDEdRjhwLmZ772PkfmZ772Pkfm13KWZenIkSNyu93fWheSAaZ79+7q0KGDKioqAtorKirkcrka3ScqKkpRUVEBbTExMa01REmSw+Folz+YDZif+dr7HJmf+dr7HJlfy3zbnZcGIfkQb2RkpIYMGaK1a9fabfX19Vq7dq08Hk8bjgwAAISCkLwDI0nTp0/XpEmTNHToUA0bNkxPPvmkampqdMcdd7T10AAAQBsL2QBzyy236IsvvtCcOXPk9Xo1ePBg5efnn/Jgb1uIiorSAw88cMqvrNoL5me+9j5H5me+9j5H5tf6wqzv+pwSAABAiAnJZ2AAAAC+DQEGAAAYhwADAACMQ4ABAADG+cEHmKKiIo0bN05ut1thYWFatWrVd+6zfv16XXrppYqKitKPfvQj5eXlnVKzePFi9e7dW9HR0UpKStLGjRuDP/jT0Nz5vf7667ruuut0zjnnyOFwyOPx6N133w2oefDBBxUWFhaw9evXrxVn0bTmzm/9+vWnjD0sLExerzegLlTWT2r+HH/xi180OscBAwbYNaG0hjk5ObrsssvUpUsXxcXFafz48SovL//O/V599VX169dP0dHRGjhwoN5+++2AfsuyNGfOHPXo0UOdOnVScnKy9uzZ01rTaFJL5vfMM8/oqquuUteuXdW1a1clJyef8jPY2DqPHj26NafSqJbMLy8v75SxR0dHB9SEyvpJLZvjtdde2+h1mJqaateEyhouWbJEF198sf2ldB6PR++888637hMK198PPsDU1NRo0KBBWrx48WnV7927V6mpqRoxYoTKyso0bdo03XnnnQFv8i+//LKmT5+uBx54QFu2bNGgQYOUkpKiysrK1ppGk5o7v6KiIl133XV6++23VVpaqhEjRmjcuHHaunVrQN2AAQP0j3/8w97+9re/tcbwv1Nz59egvLw8YPxxcXF2Xyitn9T8OS5cuDBgbgcOHFBsbKx+9rOfBdSFyhoWFhYqMzNTH374oQoKClRXV6dRo0appqamyX0++OAD3XrrrcrIyNDWrVs1fvx4jR8/Xjt27LBr5s+fr0WLFik3N1clJSXq3LmzUlJSdPTo0TMxLVtL5rd+/Xrdeuutev/991VcXKyEhASNGjVKn3/+eUDd6NGjA9bwpZdeau3pnKIl85O+/gbXb4593759Af2hsn5Sy+b4+uuvB8xvx44d6tChwynXYSis4bnnnqt58+aptLRUmzdv1o9//GPdcMMN2rlzZ6P1IXP9WbBJslauXPmtNffff781YMCAgLZbbrnFSklJsV8PGzbMyszMtF+fOHHCcrvdVk5OTlDH21ynM7/GJCYmWnPnzrVfP/DAA9agQYOCN7AgOZ35vf/++5Yk68svv2yyJlTXz7JatoYrV660wsLCrM8++8xuC9U1tCzLqqystCRZhYWFTdbcfPPNVmpqakBbUlKS9atf/cqyLMuqr6+3XC6X9dhjj9n9VVVVVlRUlPXSSy+1zsBP0+nM72THjx+3unTpYi1btsxumzRpknXDDTe0wgi/n9OZ3/PPP285nc4m+0N5/SyrZWu4YMECq0uXLlZ1dbXdFqpraFmW1bVrV+t//ud/Gu0LlevvB38HprmKi4uVnJwc0JaSkqLi4mJJ0rFjx1RaWhpQEx4eruTkZLvGJPX19Tpy5IhiY2MD2vfs2SO3263zzjtP6enp2r9/fxuNsGUGDx6sHj166LrrrtOGDRvs9va2fpL07LPPKjk5Wb169QpoD9U19Pl8knTKz9w3fdd1uHfvXnm93oAap9OppKSkNl/H05nfyb766ivV1dWdss/69esVFxenvn37aurUqfrXv/4V1LG2xOnOr7q6Wr169VJCQsIpf9sP5fWTWraGzz77rNLS0tS5c+eA9lBbwxMnTmjFihWqqalp8p/uCZXrjwDTTF6v95RvA46Pj5ff79e///1v/fOf/9SJEycarTn5OQsTPP7446qurtbNN99styUlJSkvL0/5+flasmSJ9u7dq6uuukpHjhxpw5Genh49eig3N1evvfaaXnvtNSUkJOjaa6/Vli1bJKndrd+hQ4f0zjvv6M477wxoD9U1rK+v17Rp03TllVfqoosuarKuqeuwYY0a/htq63i68zvZzJkz5Xa7A94QRo8erf/93//V2rVr9eijj6qwsFBjxozRiRMnWmPop+V059e3b18999xzeuONN/TCCy+ovr5eV1xxhQ4ePCgpdNdPatkabty4UTt27DjlOgylNdy+fbvOPvtsRUVF6de//rVWrlypxMTERmtD5foL2X9KAG3vxRdf1Ny5c/XGG28EPCMyZswY+88XX3yxkpKS1KtXL73yyivKyMhoi6Getr59+6pv37726yuuuEKffvqpFixYoD/96U9tOLLWsWzZMsXExGj8+PEB7aG6hpmZmdqxY0ebPY/T2loyv3nz5mnFihVav359wIOuaWlp9p8HDhyoiy++WOeff77Wr1+vkSNHBnXcp+t05+fxeAL+dn/FFVeof//++uMf/6iHH364tYf5vbRkDZ999lkNHDhQw4YNC2gPpTXs27evysrK5PP59Oc//1mTJk1SYWFhkyEmFHAHpplcLpcqKioC2ioqKuRwONSpUyd1795dHTp0aLTG5XKdyaF+LytWrNCdd96pV1555ZRbhSeLiYnRhRdeqE8++eQMjS64hg0bZo+9vayf9PWnAJ577jlNnDhRkZGR31obCmuYlZWl1atX6/3339e55577rbVNXYcNa9Tw31Bax+bMr8Hjjz+uefPmac2aNbr44ou/tfa8885T9+7d22wNWzK/Bh07dtQll1xijz0U109q2Rxramq0YsWK0/qLQVuuYWRkpH70ox9pyJAhysnJ0aBBg7Rw4cJGa0Pl+iPANJPH49HatWsD2goKCuy/TURGRmrIkCEBNfX19Vq7dm2Tv08MNS+99JLuuOMOvfTSSwEf+WtKdXW1Pv30U/Xo0eMMjC74ysrK7LG3h/VrUFhYqE8++eS0/o+zLdfQsixlZWVp5cqVWrdunfr06fOd+3zXddinTx+5XK6AGr/fr5KSkjO+ji2Zn/T1pzgefvhh5efna+jQod9Zf/DgQf3rX/8642vY0vl904kTJ7R9+3Z77KG0ftL3m+Orr76q2tpa3X777d9Z21Zr2Jj6+nrV1tY22hcy11/QHgc21JEjR6ytW7daW7dutSRZTzzxhLV161Zr3759lmVZVnZ2tjVx4kS7/u9//7t11llnWTNmzLB27dplLV682OrQoYOVn59v16xYscKKioqy8vLyrI8++siaMmWKFRMTY3m93pCf3/Lly62IiAhr8eLF1j/+8Q97q6qqsmvuvfdea/369dbevXutDRs2WMnJyVb37t2tysrKkJ/fggULrFWrVll79uyxtm/fbv3mN7+xwsPDrffee8+uCaX1s6zmz7HB7bffbiUlJTV6zFBaw6lTp1pOp9Nav359wM/cV199ZddMnDjRys7Otl9v2LDBioiIsB5//HFr165d1gMPPGB17NjR2r59u10zb948KyYmxnrjjTesbdu2WTfccIPVp08f69///nfIz2/evHlWZGSk9ec//zlgnyNHjliW9fXPxH333WcVFxdbe/futd577z3r0ksvtS644ALr6NGjIT+/uXPnWu+++6716aefWqWlpVZaWpoVHR1t7dy5064JlfWzrJbNscHw4cOtW2655ZT2UFrD7Oxsq7Cw0Nq7d6+1bds2Kzs72woLC7PWrFljWVboXn8/+ADT8LHak7dJkyZZlvX1x9yuueaaU/YZPHiwFRkZaZ133nnW888/f8pxn3rqKatnz55WZGSkNWzYMOvDDz9s/ck0ornzu+aaa7613rK+/th4jx49rMjISOs//uM/rFtuucX65JNPzuzE/p/mzu/RRx+1zj//fCs6OtqKjY21rr32WmvdunWnHDdU1s+yWvYzWlVVZXXq1MlaunRpo8cMpTVsbG6SAq6ra665JuBn0LIs65VXXrEuvPBCKzIy0howYID11ltvBfTX19dbs2fPtuLj462oqChr5MiRVnl5+RmYUaCWzK9Xr16N7vPAAw9YlmVZX331lTVq1CjrnHPOsTp27Gj16tXLmjx5cpuE7JbMb9q0afb1FR8fb40dO9basmVLwHFDZf0sq+U/o7t377Yk2UHgm0JpDX/5y19avXr1siIjI61zzjnHGjlyZMCYQ/X6C7MsywrSzRwAAIAzgmdgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADDO/wfhamWXm0J9HwAAAABJRU5ErkJggg==",
"text/plain": [
"<Figure size 640x480 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"plt.hist(mktunebatch(2048)[1].to(torch.float32).cpu(), bins=64)"
]
},
{
"cell_type": "code",
"execution_count": 97,
"metadata": {},
"outputs": [],
"source": [
"import pickle\n",
"\n",
"graphs, labels, _ = mkbatch(3*10**5)\n",
"\n",
"data = {\n",
" \"data\": graphs,\n",
" \"labels\": labels\n",
"}\n",
"\n",
"with open('data.pkl', 'wb') as file:\n",
" pickle.dump(data, file)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "Q3Cg_8UQep8g"
},
"source": [
"# Step 2: Define Transformer Model"
]
},
{
"cell_type": "code",
"execution_count": 135,
"metadata": {
"id": "tLOWhg_CeWzH"
},
"outputs": [],
"source": [
"class TransformerModel(nn.Module):\n",
" def __init__(self, input_dim, model_dim, output_dim, num_heads, num_layers, seq_len, device, dropout):\n",
" super().__init__()\n",
" self.embedding = nn.Embedding(input_dim, model_dim//2, dtype=torch.bfloat16)\n",
" # seq_len is odd\n",
" self.fancy_encoding = torch.repeat_interleave(torch.rand((1, seq_len // 2 + 1, model_dim // 2), device=device, dtype=torch.bfloat16), 2, dim=1)\n",
" # cut off last element since the target vertex is not repeated\n",
" self.fancy_encoding = self.fancy_encoding[:, :seq_len, :]\n",
" \n",
" self.model_dim = model_dim\n",
" self.seq_len = seq_len\n",
" self.device = device\n",
"\n",
" encoder_layer = nn.TransformerEncoderLayer(d_model=model_dim, nhead=num_heads,\n",
" dim_feedforward=model_dim*4,\n",
" dropout=dropout, batch_first=True, dtype=torch.bfloat16)\n",
" self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers)\n",
"\n",
" self.fc_out = nn.Linear(model_dim*seq_len, output_dim, dtype=torch.bfloat16)\n",
"\n",
" def full_embedding(self, src):\n",
" batch_size, src_len = src.size(0), src.size(1)\n",
" return torch.cat((self.embedding(src) * sqrt(self.model_dim), self.fancy_encoding.repeat((batch_size, 1, 1))), dim=2)\n",
" \n",
" def forward(self, src, key_padding_mask):\n",
" embed = self.full_embedding(src)\n",
" output = self.transformer_encoder(embed, src_key_padding_mask=key_padding_mask)\n",
" output[key_padding_mask] = 0 # Hack to stop no_grad problem\n",
" flat_output = torch.flatten(output, start_dim=1, end_dim=2)\n",
" output = self.fc_out(flat_output)\n",
" return output"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "bpIeg86S-hBb"
},
"source": [
"# Step 3: Make Model"
]
},
{
"cell_type": "code",
"execution_count": 136,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "kWXvJRDYgFVP",
"outputId": "c13adb9d-6565-43b5-8437-20cef3dc0d16"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Training data: 1049M\n",
"Trainable parameters in the model: 50K\n"
]
}
],
"source": [
"# PARAMS\n",
"VOCAB_SIZE = 1 + MAX_VTXS # one more than the max number of vertices\n",
"MODEL_DIM = 64 # Dimension of model (embedding and transformer)\n",
"NEPOCHS = 1000\n",
"BSZ = 8196 * 4 # Batch size\n",
"BPE = 32 # Batches per epoch\n",
"LR = 5e-3\n",
"WD = 2e-3\n",
"NHEADS = 1 #4\n",
"NLAYERS = 1 #16\n",
"DROPOUT = 0.2\n",
"model = TransformerModel(input_dim=VOCAB_SIZE, model_dim=MODEL_DIM,\n",
" output_dim=1, num_heads=NHEADS,\n",
" num_layers=NLAYERS, seq_len=SEQ_LEN,\n",
" dropout=DROPOUT, device=device).to(device)\n",
"# model = torch.compile(model)\n",
"\n",
"criterion = nn.MSELoss()\n",
"optimizer = torch.optim.Adam(model.parameters(), lr=LR, weight_decay=WD)\n",
"\n",
"trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
"print(f\"Training data: {NEPOCHS*BSZ*BPE//10**6}M\")\n",
"print(f\"Trainable parameters in the model: {trainable_params//1000}K\")"
]
},
{
"cell_type": "code",
"execution_count": 137,
"metadata": {},
"outputs": [],
"source": [
"from torch.utils.data import DataLoader, TensorDataset\n",
"\n",
"with open(\"data.pkl\", \"rb\") as f:\n",
" pickled_stuff = pickle.load(f)\n",
"\n",
"data = pickled_stuff[\"data\"].to(device)\n",
"label = pickled_stuff[\"labels\"].to(device)\n",
"padding_mask = (data == PAD_TOKEN).bool().to(device)\n",
"dataset = TensorDataset(data, label, padding_mask)\n",
"# train_dataset, test_dataset = torch.utils.data.random_split(dataset, [.9, .1])\n",
"train_loader = DataLoader(dataset, batch_size=BSZ, shuffle=True)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "f8Zn33m7CxL5"
},
"source": [
"# Step 4: Train the Model for the first task"
]
},
{
"cell_type": "code",
"execution_count": 138,
"metadata": {},
"outputs": [],
"source": [
"def evaluate():\n",
" model.eval()\n",
" test_loss = 0\n",
" with torch.no_grad():\n",
" batch_src, batch_labels, batch_padding_mask = mkbatch(BSZ)\n",
" output = model(batch_src, batch_padding_mask)\n",
" loss = criterion(output.squeeze(1), batch_labels)\n",
" return loss.item()"
]
},
{
"cell_type": "code",
"execution_count": 139,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 486
},
"id": "pvTfzGmCeXU4",
"outputId": "0d3a20f3-23be-4c19-9eb6-46bfe11a48b1"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.01it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/1000 \t Train Err: 0.5381 \t Test Err: 0.1865\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.75it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 2/1000 \t Train Err: 0.1227 \t Test Err: 0.1128\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.03it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 3/1000 \t Train Err: 0.1071 \t Test Err: 0.1118\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.01it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 4/1000 \t Train Err: 0.1008 \t Test Err: 0.1035\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.75it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 5/1000 \t Train Err: 0.0972 \t Test Err: 0.1021\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.03it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 6/1000 \t Train Err: 0.0949 \t Test Err: 0.0981\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.76it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 7/1000 \t Train Err: 0.0929 \t Test Err: 0.1021\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.00it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 8/1000 \t Train Err: 0.0908 \t Test Err: 0.0977\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.01it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 9/1000 \t Train Err: 0.0886 \t Test Err: 0.0952\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.72it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 10/1000 \t Train Err: 0.0910 \t Test Err: 0.0962\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.02it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 11/1000 \t Train Err: 0.0851 \t Test Err: 0.0898\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.03it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 12/1000 \t Train Err: 0.0849 \t Test Err: 0.0864\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.74it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 13/1000 \t Train Err: 0.0795 \t Test Err: 0.0684\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.02it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 14/1000 \t Train Err: 0.0691 \t Test Err: 0.0293\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.74it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 15/1000 \t Train Err: 0.0455 \t Test Err: 0.0271\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 6.99it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 16/1000 \t Train Err: 0.0421 \t Test Err: 0.0210\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 6.99it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 17/1000 \t Train Err: 0.0817 \t Test Err: 0.0505\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.75it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 18/1000 \t Train Err: 0.0456 \t Test Err: 0.0176\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.03it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 19/1000 \t Train Err: 0.0370 \t Test Err: 0.0165\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.03it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 20/1000 \t Train Err: 0.0374 \t Test Err: 0.0205\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.73it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 21/1000 \t Train Err: 0.0372 \t Test Err: 0.0142\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.01it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 22/1000 \t Train Err: 0.0343 \t Test Err: 0.0132\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.75it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 23/1000 \t Train Err: 0.0337 \t Test Err: 0.0119\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.03it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 24/1000 \t Train Err: 0.0713 \t Test Err: 0.0259\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.04it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 25/1000 \t Train Err: 0.0522 \t Test Err: 0.0143\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.75it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 26/1000 \t Train Err: 0.0342 \t Test Err: 0.0117\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.02it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 27/1000 \t Train Err: 0.0864 \t Test Err: 0.0728\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.04it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 28/1000 \t Train Err: 0.0701 \t Test Err: 0.0510\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.72it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 29/1000 \t Train Err: 0.0598 \t Test Err: 0.0369\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 6.99it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 30/1000 \t Train Err: 0.0462 \t Test Err: 0.0231\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.73it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 31/1000 \t Train Err: 0.0387 \t Test Err: 0.0181\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.02it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 32/1000 \t Train Err: 0.0351 \t Test Err: 0.0142\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.04it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 33/1000 \t Train Err: 0.0337 \t Test Err: 0.0123\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.74it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 34/1000 \t Train Err: 0.0331 \t Test Err: 0.0117\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.02it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 35/1000 \t Train Err: 0.0329 \t Test Err: 0.0119\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 6.98it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 36/1000 \t Train Err: 0.0327 \t Test Err: 0.0109\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.74it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 37/1000 \t Train Err: 0.0319 \t Test Err: 0.0101\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.02it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 38/1000 \t Train Err: 0.0317 \t Test Err: 0.0099\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.71it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 39/1000 \t Train Err: 0.0313 \t Test Err: 0.0095\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:04<00:00, 7.01it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 40/1000 \t Train Err: 0.0310 \t Test Err: 0.0097\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 28%|██████████████████████████████████████████▊ | 9/32 [00:01<00:03, 6.50it/s]\n"
]
},
{
"ename": "KeyboardInterrupt",
"evalue": "",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[139], line 11\u001b[0m\n\u001b[1;32m 9\u001b[0m train_loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m tqdm(\u001b[38;5;28mrange\u001b[39m(BPE)):\n\u001b[0;32m---> 11\u001b[0m batch_src, batch_labels, batch_padding_mask \u001b[38;5;241m=\u001b[39m \u001b[43mmkbatch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mBSZ\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 12\u001b[0m \u001b[38;5;66;03m# for batch_src, batch_labels, batch_padding_mask in tqdm(train_loader):\u001b[39;00m\n\u001b[1;32m 13\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n",
"Cell \u001b[0;32mIn[76], line 55\u001b[0m, in \u001b[0;36mmkbatch\u001b[0;34m(size)\u001b[0m\n\u001b[1;32m 53\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(size):\n\u001b[1;32m 54\u001b[0m n \u001b[38;5;241m=\u001b[39m random\u001b[38;5;241m.\u001b[39mrandint(MIN_VTXS, MAX_VTXS)\n\u001b[0;32m---> 55\u001b[0m edge_list, adj_list \u001b[38;5;241m=\u001b[39m \u001b[43mrandom_graph\u001b[49m\u001b[43m(\u001b[49m\u001b[43mn\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 56\u001b[0m dist \u001b[38;5;241m=\u001b[39m SSSP(n, adj_list)\n\u001b[1;32m 57\u001b[0m edge_list[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m2\u001b[39m \u001b[38;5;66;03m# target token\u001b[39;00m\n",
"Cell \u001b[0;32mIn[76], line 15\u001b[0m, in \u001b[0;36mrandom_graph\u001b[0;34m(n)\u001b[0m\n\u001b[1;32m 13\u001b[0m adjacencies \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mset\u001b[39m() \u001b[38;5;28;01mfor\u001b[39;00m _ \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(n\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m1\u001b[39m)]\n\u001b[1;32m 14\u001b[0m indices \u001b[38;5;241m=\u001b[39m [random\u001b[38;5;241m.\u001b[39mrandint(\u001b[38;5;241m1\u001b[39m, n) \u001b[38;5;28;01mfor\u001b[39;00m _ \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(AVG_DEG \u001b[38;5;241m*\u001b[39m (n\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m))]\n\u001b[0;32m---> 15\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28;43mrange\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mindices\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m)\u001b[49m:\n\u001b[1;32m 16\u001b[0m u \u001b[38;5;241m=\u001b[39m indices[i]\n\u001b[1;32m 17\u001b[0m v \u001b[38;5;241m=\u001b[39m indices[i \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m1\u001b[39m]\n",
"\u001b[0;31mKeyboardInterrupt\u001b[0m: "
]
}
],
"source": [
"train_err = []\n",
"test_err = []\n",
"\n",
"# clear loss file\n",
"open('loss', 'w').close()\n",
"\n",
"for epoch in range(NEPOCHS):\n",
" model.train()\n",
" train_loss = 0\n",
" for i in tqdm(range(BPE)):\n",
" batch_src, batch_labels, batch_padding_mask = mkbatch(BSZ)\n",
" # for batch_src, batch_labels, batch_padding_mask in tqdm(train_loader):\n",
" optimizer.zero_grad()\n",
" output = model(batch_src, batch_padding_mask)\n",
" loss = criterion(output.squeeze(1), batch_labels)\n",
" train_loss += loss.item() / BPE\n",
" loss.backward()\n",
" optimizer.step()\n",
"\n",
" test_loss = evaluate()\n",
" \n",
" test_err.append(test_loss)\n",
" train_err.append(train_loss)\n",
" with open('loss', 'a') as f:\n",
" f.write(f\"{train_loss} {test_loss}\\n\")\n",
" print(f\"Epoch {epoch + 1}/{NEPOCHS} \\t Train Err: {train_loss:.4f} \\t Test Err: {test_loss:.4f}\")\n",
" \n",
" if epoch % 100 == 99:\n",
" torch.save(model.state_dict(), f\"model_weights_{epoch}.pth\")"
]
},
{
"cell_type": "code",
"execution_count": 163,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(torch.Size([1, 7, 64]), torch.Size([64, 64]))"
]
},
"execution_count": 163,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\"\"\"\n",
"Now let's figure out what it's doing. \n",
"\n",
"step 1: figure out what people are attending to \n",
"\"\"\"\n",
"\n",
"example_graph, answer, padding = mkbatch(1)\n",
"sentance_embeddings = model.full_embedding(example_graph)\n",
"Q,K,V = torch.split(model.transformer_encoder.layers[0].self_attn.in_proj_weight, (MODEL_DIM, MODEL_DIM, MODEL_DIM))\n",
"\n",
"sentance_embeddings.shape, Q.shape\n",
"sentance_embeddings@Q.T\n",
"\n",
"# (sentance_embeddings @ Q).shape\n",
"# sentance_embeddings.shape\n",
"# K @ sentance_embeddings"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.legend.Legend at 0x702d2d2eed20>"
]
},
"execution_count": 34,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"plt.suptitle('MSE vs Epochs')\n",
"plt.plot(train_err, label='Train', color='blue')\n",
"plt.plot(test_err, label='Test', color='red')\n",
"plt.xlabel('Epochs')\n",
"plt.ylabel('MSE')\n",
"plt.legend()\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"24.625"
]
},
"execution_count": 35,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"evaluate()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "LoGEmM5lH7_A"
},
"outputs": [],
"source": [
"batch_src, batch_labels, batch_padding_mask = next(iter(train_loader))\n",
"output = model(batch_src, batch_padding_mask)\n",
"batch_src[0], batch_labels[0], output[0]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"plt.hist(output.detach().cpu().numpy().flatten(),bins=32)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"plt.hist(label.detach().cpu().numpy().flatten(),bins=32)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"plt.scatter(batch_labels.detach().cpu().numpy().flatten(),output.detach().cpu().numpy().flatten())"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"batch_src2, batch_labels2, batch_padding_mask2 = next(iter(test_loader))\n",
"output2 = model(batch_src2, batch_padding_mask2)\n",
"loss = criterion(output2.squeeze(1), batch_labels2)\n",
"batch_src2[0], batch_labels2[0], output2[0], loss"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"plt.scatter(batch_labels2.detach().cpu().numpy().flatten(),output2.detach().cpu().numpy().flatten())"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "LC6Xv3YfC0Rm"
},
"source": [
"# Step 5: Fine Tune"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"N_TUNE_EPOCHS = 100\n",
"TUNE_LR = 0.003\n",
"TUNE_WD = 0.002\n",
"\n",
"tune_criterion = nn.MSELoss()\n",
"tune_optimizer = torch.optim.Adam(model.parameters(), lr=TUNE_LR, weight_decay=TUNE_WD)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def tuneevaluate():\n",
" model.eval()\n",
" test_loss = 0\n",
" with torch.no_grad():\n",
" batch_src, batch_labels, batch_padding_mask = mktunebatch(BSZ)\n",
" output = model(batch_src, batch_padding_mask)\n",
" loss = criterion(output.squeeze(1), batch_labels)\n",
" return loss.item()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# This has to be in a separate cell for some weird event loop reasons\n",
"%matplotlib widget\n",
"fig,ax = plt.subplots()\n",
"fig.suptitle('MSE vs Epochs')\n",
"plt.show()\n",
"%matplotlib inline"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"ename": "NameError",
"evalue": "name 'N_TUNE_EPOCHS' is not defined",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[13], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m tune_train_err \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 2\u001b[0m tune_test_err \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m----> 4\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m epoch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[43mN_TUNE_EPOCHS\u001b[49m):\n\u001b[1;32m 5\u001b[0m model\u001b[38;5;241m.\u001b[39mtrain()\n\u001b[1;32m 6\u001b[0m train_loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n",
"\u001b[0;31mNameError\u001b[0m: name 'N_TUNE_EPOCHS' is not defined"
]
}
],
"source": [
"tune_train_err = []\n",
"tune_test_err = []\n",
"\n",
"# clear loss file\n",
"open('tune_loss', 'w').close()\n",
"\n",
"for epoch in range(N_TUNE_EPOCHS):\n",
" model.train()\n",
" train_loss = 0\n",
" for i in tqdm(range(BPE)):\n",
" batch_src, batch_labels, batch_padding_mask = mktunebatch(BSZ)\n",
" optimizer.zero_grad()\n",
" output = model(batch_src, batch_padding_mask)\n",
" loss = criterion(output.squeeze(1), batch_labels)\n",
" train_loss += loss.item()/BPE\n",
" loss.backward()\n",
" optimizer.step()\n",
"\n",
" test_loss = tuneevaluate()\n",
" \n",
" tune_test_err.append(test_loss)\n",
" tune_train_err.append(train_loss)\n",
" with open('tune_loss', 'a') as f:\n",
" f.write(f\"{train_loss} {test_loss}\\n\")\n",
" ax.plot(tune_train_err, label='Train', color='blue')\n",
" ax.plot(tune_test_err, label='Test', color='red')\n",
" ax.set_xlabel('Epochs')\n",
" ax.set_ylabel('MSE')\n",
" fig.canvas.draw()\n",
" print(f\"Epoch {epoch + 1}/{NEPOCHS} \\t Train Err: {train_loss:.4f} \\t Test Err: {test_loss:.4f}\")\n",
"\n",
" if epoch % 10 == 9:\n",
" torch.save(model.state_dict(), f\"tune_model_weights_{epoch}.pth\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "JtTLXn4zC1z_"
},
"source": [
"# Step 6: Test generalization"
]
}
],
"metadata": {
"colab": {
"gpuType": "T4",
"provenance": []
},
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}