-
1
-
2
-
3
-
4
-
5
-
6
-
7
-
8
-
9
-
10
-
11
-
12
-
13
-
14
-
15
-
16
-
17
-
18
-
19
-
20
-
21
-
22
-
23
-
24
-
25
-
26
-
27
-
28
-
29
-
30
-
31
-
32
-
33
-
34
-
35
-
36
-
37
-
38
-
39
-
40
-
41
-
42
-
43
-
44
-
45
-
46
-
47
-
48
-
49
-
50
-
51
-
52
-
53
-
54
-
55
-
56
-
57
-
58
-
59
-
60
-
61
-
62
-
63
-
64
-
65
-
66
-
67
-
68
-
69
-
70
-
71
-
72
-
73
-
74
-
75
-
76
-
77
-
78
-
79
-
80
-
81
-
82
-
83
-
84
-
85
-
86
-
87
-
88
-
89
-
90
-
91
-
92
-
93
-
94
-
95
-
96
-
97
-
98
-
99
-
100
-
101
-
102
-
103
-
104
-
105
-
106
-
107
-
108
-
109
-
110
-
111
-
112
-
113
-
114
-
115
-
116
-
117
-
118
-
119
-
120
-
121
-
122
-
123
-
124
-
125
-
126
-
127
-
128
-
129
-
130
-
131
-
132
-
133
-
134
-
135
-
136
-
137
-
138
-
139
-
140
-
141
-
142
-
143
-
144
-
145
-
146
-
147
-
148
-
149
-
150
-
151
-
152
-
153
-
154
-
155
-
156
-
157
-
158
-
159
-
160
-
161
-
162
-
163
-
164
-
165
-
166
-
167
-
168
-
169
-
170
-
171
-
172
-
173
-
174
-
175
-
176
-
177
-
178
-
179
-
180
-
181
-
182
-
183
-
184
-
185
-
186
-
187
-
188
-
189
-
190
-
191
-
192
-
193
-
194
-
195
-
196
-
197
-
198
-
199
-
200
-
201
-
202
-
203
-
204
-
205
-
206
-
207
-
208
-
209
-
210
-
211
-
212
-
213
-
214
-
215
-
216
-
217
-
218
-
219
-
220
-
221
-
222
-
223
-
224
-
225
-
226
-
227
-
228
-
229
-
230
-
231
-
232
-
233
-
234
-
235
-
236
-
237
-
238
-
239
-
240
-
241
-
242
-
243
-
244
-
245
-
246
-
247
-
248
-
249
-
250
-
251
-
252
-
253
-
254
-
255
-
256
-
257
-
258
-
259
-
260
-
261
-
262
-
263
-
264
-
265
-
266
-
267
-
268
-
269
-
270
-
271
-
272
-
273
-
274
-
275
-
276
-
277
-
278
-
279
-
280
-
281
-
282
-
283
-
284
-
285
-
286
-
287
-
288
-
289
-
290
-
291
-
292
-
293
-
294
-
295
-
296
-
297
-
298
-
299
-
300
-
301
-
302
-
303
-
304
-
305
-
306
-
307
-
308
-
309
-
310
-
311
-
312
-
313
-
314
-
315
-
316
-
317
-
318
-
319
-
320
-
321
-
322
-
323
-
324
-
325
-
326
-
327
-
328
-
329
-
330
-
331
-
332
-
333
-
334
-
335
-
336
-
337
-
338
-
339
-
340
-
341
-
342
-
343
-
344
-
345
-
346
-
347
-
348
-
349
-
350
-
351
-
352
-
353
-
354
-
355
-
356
-
357
-
358
-
359
-
360
-
361
-
362
-
363
-
364
-
365
-
366
-
367
-
368
-
369
-
370
-
371
-
372
-
373
-
374
-
375
-
376
-
377
-
378
-
379
-
380
-
381
-
382
-
383
-
384
-
385
-
386
-
387
-
388
-
389
-
390
-
391
-
392
-
393
-
394
-
395
-
396
-
397
-
398
-
399
-
400
-
401
-
402
-
403
-
404
-
405
-
406
-
407
-
408
-
409
-
410
-
411
-
412
-
413
-
414
-
415
-
416
-
417
-
418
-
419
-
420
-
421
-
422
-
423
-
424
-
425
-
426
-
427
-
428
-
429
-
430
-
431
-
432
-
433
-
434
-
435
-
436
-
437
-
438
-
439
-
440
-
441
-
442
-
443
-
444
-
445
-
446
-
447
-
448
-
449
-
450
-
451
-
452
-
453
-
454
-
455
-
456
-
457
-
458
-
459
-
460
-
461
-
462
-
463
-
464
-
465
-
466
-
467
-
468
-
469
-
470
-
471
-
472
-
473
-
474
-
475
-
476
-
477
-
478
-
479
-
480
-
481
-
482
-
483
-
484
-
485
-
486
-
487
-
488
-
489
-
490
-
491
-
492
-
493
-
494
-
495
-
496
-
497
-
498
-
499
-
500
-
501
-
502
-
503
-
504
-
505
-
506
-
507
-
508
-
509
-
510
-
511
-
512
-
513
-
514
-
515
-
516
-
517
-
518
-
519
-
520
-
521
-
522
-
523
-
524
-
525
-
526
-
527
-
528
-
529
-
530
-
531
-
532
-
533
-
534
-
535
-
536
-
537
-
538
-
539
-
540
-
541
-
542
-
543
-
544
-
545
-
546
-
547
-
548
-
549
-
550
-
551
-
552
-
553
-
554
-
555
-
556
-
557
-
558
-
559
-
560
-
561
-
562
-
563
-
564
-
565
-
566
-
567
-
568
-
569
-
570
-
571
-
572
-
573
-
574
-
575
-
576
-
577
-
578
-
579
-
580
-
581
-
582
-
583
-
584
-
585
-
586
-
587
-
588
-
589
-
590
-
591
-
592
-
593
-
594
-
595
-
596
-
597
-
598
-
599
-
600
-
601
-
602
-
603
-
604
-
605
-
606
-
607
-
608
-
609
-
610
-
611
-
612
-
613
-
614
-
615
-
616
-
617
-
618
-
619
-
620
-
621
-
622
-
623
-
624
-
625
-
626
-
627
-
628
-
629
-
630
-
631
-
632
-
633
-
634
-
635
-
636
-
637
-
638
-
639
-
640
-
641
-
642
-
643
-
644
-
645
-
646
-
647
-
648
-
649
-
650
-
651
-
652
-
653
-
654
-
655
-
656
-
657
-
658
-
659
-
660
-
661
-
662
-
663
-
664
-
665
-
666
-
667
-
668
-
669
-
670
-
671
-
672
-
673
-
674
-
675
-
676
-
677
-
678
-
679
-
680
-
681
-
682
-
683
-
684
-
685
-
686
-
687
-
688
-
689
-
690
-
691
-
692
-
693
-
694
-
695
-
696
-
697
-
698
-
699
-
700
-
701
-
702
-
703
-
704
-
705
-
706
-
707
-
708
-
709
-
710
-
711
-
712
-
713
-
714
-
715
-
716
-
717
-
718
-
719
-
720
-
721
-
722
-
723
-
724
-
725
-
726
-
727
-
728
-
729
-
730
-
731
-
732
-
733
-
734
-
735
-
736
-
737
-
738
-
739
-
740
-
741
-
742
-
743
-
744
-
745
-
746
-
747
-
748
-
749
-
750
-
751
-
752
-
753
-
754
-
755
-
756
-
757
-
758
-
759
-
760
-
761
-
762
-
763
-
764
-
765
-
766
-
767
-
768
-
769
-
770
-
771
-
772
-
773
-
774
-
775
-
776
-
777
-
778
-
779
-
780
-
781
-
782
-
783
-
784
-
785
-
786
-
787
-
788
-
789
-
790
-
791
-
792
-
793
-
794
-
795
-
796
-
797
-
798
-
799
-
800
-
801
-
802
-
803
-
804
-
805
-
806
-
807
-
808
-
809
-
810
-
811
-
812
-
813
-
814
-
815
-
816
-
817
-
818
-
819
-
820
-
821
-
822
-
823
-
824
-
825
-
826
-
827
-
828
-
829
-
830
-
831
-
832
-
833
-
834
-
835
-
836
-
837
-
838
-
839
-
840
-
841
-
842
-
843
-
844
-
845
-
846
-
847
-
848
-
849
-
850
-
851
-
852
-
853
-
854
-
855
-
856
-
857
-
858
-
859
-
860
-
861
-
862
-
863
-
864
-
865
-
866
-
867
-
868
-
869
-
870
-
871
-
872
-
873
-
874
-
875
-
876
-
877
-
878
-
879
-
880
-
881
-
882
-
883
-
884
-
885
-
886
-
887
-
888
-
889
-
890
-
891
-
892
-
893
-
894
-
895
-
896
-
897
-
898
-
899
-
900
-
901
-
902
-
903
-
904
-
905
-
906
-
907
-
908
-
909
-
910
-
911
-
912
-
913
-
914
-
915
-
916
-
917
-
918
-
919
-
920
-
921
-
922
-
923
-
924
-
925
-
926
-
927
-
928
-
929
-
930
-
931
-
932
-
933
-
934
-
935
-
936
-
937
-
938
-
939
-
940
-
941
-
942
-
943
-
944
-
945
-
946
-
947
-
948
-
949
-
950
-
951
-
952
-
953
-
954
-
955
-
956
-
957
-
958
-
959
-
960
-
961
-
962
-
963
-
964
-
965
-
966
-
967
-
968
-
969
-
970
-
971
-
972
-
973
-
974
-
975
-
976
-
977
-
978
-
979
-
980
-
981
-
982
-
983
-
984
-
985
-
986
-
987
-
988
-
989
-
990
-
991
-
992
-
993
-
994
-
995
-
996
-
997
-
998
-
999
-
1000
-
1001
-
1002
-
1003
-
1004
-
1005
-
1006
-
1007
-
1008
-
1009
-
1010
-
1011
-
1012
-
1013
-
1014
-
1015
-
1016
-
1017
-
1018
-
1019
-
1020
-
1021
-
1022
-
1023
-
1024
-
1025
-
1026
-
1027
-
1028
-
1029
-
1030
-
1031
-
1032
-
1033
-
1034
-
1035
-
1036
-
1037
-
1038
-
1039
-
1040
-
1041
-
1042
-
1043
-
1044
-
1045
-
1046
-
1047
-
1048
-
1049
-
1050
-
1051
-
1052
-
1053
-
1054
-
1055
-
1056
-
1057
-
1058
-
1059
-
1060
-
1061
-
1062
-
1063
-
1064
-
1065
-
1066
-
1067
-
1068
-
1069
-
1070
-
1071
-
1072
-
1073
-
1074
-
1075
-
1076
-
1077
-
1078
-
1079
-
1080
-
1081
-
1082
-
1083
-
1084
-
1085
-
1086
-
1087
-
1088
-
1089
-
1090
-
1091
-
1092
-
1093
-
1094
-
1095
-
1096
-
1097
-
1098
-
1099
-
1100
-
1101
-
1102
-
1103
-
1104
-
1105
-
1106
-
1107
-
1108
-
1109
-
1110
-
1111
-
1112
-
1113
-
1114
-
1115
-
1116
-
1117
-
1118
-
1119
-
1120
-
1121
-
1122
-
1123
-
1124
-
1125
-
1126
-
1127
-
1128
-
1129
-
1130
-
1131
-
1132
-
1133
-
1134
-
1135
-
1136
-
1137
-
1138
-
1139
-
1140
-
1141
-
1142
-
1143
-
1144
-
1145
-
1146
-
1147
-
1148
-
1149
-
1150
-
1151
-
1152
-
1153
-
1154
-
1155
-
1156
-
1157
-
1158
-
1159
-
1160
-
1161
-
1162
-
1163
-
1164
-
1165
-
1166
-
1167
-
1168
-
1169
-
1170
-
1171
-
1172
-
1173
-
1174
-
1175
-
1176
-
1177
-
1178
-
1179
-
1180
-
1181
-
1182
-
1183
-
1184
-
1185
-
1186
-
1187
-
1188
-
1189
-
1190
-
1191
-
1192
-
1193
-
1194
-
1195
-
1196
-
1197
-
1198
-
1199
-
1200
-
1201
-
1202
-
1203
-
1204
-
1205
-
1206
-
1207
-
1208
-
1209
-
1210
-
1211
-
1212
-
1213
-
1214
-
1215
-
1216
-
1217
-
1218
-
1219
-
1220
-
1221
-
1222
-
1223
-
1224
-
1225
-
1226
-
1227
-
1228
-
1229
-
1230
-
1231
-
1232
-
1233
-
1234
-
1235
-
1236
-
1237
-
1238
-
1239
-
1240
-
1241
-
1242
-
1243
-
1244
// Package jlexer contains a JSON lexer implementation.
//
// It is expected that it is mostly used with generated parser code, so the interface is tuned
// for a parser that knows what kind of data is expected.
package jlexer
import (
"bytes"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"strconv"
"unicode"
"unicode/utf16"
"unicode/utf8"
"github.com/josharian/intern"
)
// tokenKind determines type of a token.
type tokenKind byte
const (
tokenUndef tokenKind = iota // No token.
tokenDelim // Delimiter: one of '{', '}', '[' or ']'.
tokenString // A string literal, e.g. "abc\u1234"
tokenNumber // Number literal, e.g. 1.5e5
tokenBool // Boolean literal: true or false.
tokenNull // null keyword.
)
// token describes a single token: type, position in the input and value.
type token struct {
kind tokenKind // Type of a token.
boolValue bool // Value if a boolean literal token.
byteValueCloned bool // true if byteValue was allocated and does not refer to original json body
byteValue []byte // Raw value of a token.
delimValue byte
}
// Lexer is a JSON lexer: it iterates over JSON tokens in a byte slice.
type Lexer struct {
Data []byte // Input data given to the lexer.
start int // Start of the current token.
pos int // Current unscanned position in the input stream.
token token // Last scanned token, if token.kind != tokenUndef.
firstElement bool // Whether current element is the first in array or an object.
wantSep byte // A comma or a colon character, which need to occur before a token.
UseMultipleErrors bool // If we want to use multiple errors.
fatalError error // Fatal error occurred during lexing. It is usually a syntax error.
multipleErrors []*LexerError // Semantic errors occurred during lexing. Marshalling will be continued after finding this errors.
}
// FetchToken scans the input for the next token.
func (r *Lexer) FetchToken() {
r.token.kind = tokenUndef
r.start = r.pos
// Check if r.Data has r.pos element
// If it doesn't, it mean corrupted input data
if len(r.Data) < r.pos {
r.errParse("Unexpected end of data")
return
}
// Determine the type of a token by skipping whitespace and reading the
// first character.
for _, c := range r.Data[r.pos:] {
switch c {
case ':', ',':
if r.wantSep == c {
r.pos++
r.start++
r.wantSep = 0
} else {
r.errSyntax()
}
case ' ', '\t', '\r', '\n':
r.pos++
r.start++
case '"':
if r.wantSep != 0 {
r.errSyntax()
}
r.token.kind = tokenString
r.fetchString()
return
case '{', '[':
if r.wantSep != 0 {
r.errSyntax()
}
r.firstElement = true
r.token.kind = tokenDelim
r.token.delimValue = r.Data[r.pos]
r.pos++
return
case '}', ']':
if !r.firstElement && (r.wantSep != ',') {
r.errSyntax()
}
r.wantSep = 0
r.token.kind = tokenDelim
r.token.delimValue = r.Data[r.pos]
r.pos++
return
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-':
if r.wantSep != 0 {
r.errSyntax()
}
r.token.kind = tokenNumber
r.fetchNumber()
return
case 'n':
if r.wantSep != 0 {
r.errSyntax()
}
r.token.kind = tokenNull
r.fetchNull()
return
case 't':
if r.wantSep != 0 {
r.errSyntax()
}
r.token.kind = tokenBool
r.token.boolValue = true
r.fetchTrue()
return
case 'f':
if r.wantSep != 0 {
r.errSyntax()
}
r.token.kind = tokenBool
r.token.boolValue = false
r.fetchFalse()
return
default:
r.errSyntax()
return
}
}
r.fatalError = io.EOF
return
}
// isTokenEnd returns true if the char can follow a non-delimiter token
func isTokenEnd(c byte) bool {
return c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '[' || c == ']' || c == '{' || c == '}' || c == ',' || c == ':'
}
// fetchNull fetches and checks remaining bytes of null keyword.
func (r *Lexer) fetchNull() {
r.pos += 4
if r.pos > len(r.Data) ||
r.Data[r.pos-3] != 'u' ||
r.Data[r.pos-2] != 'l' ||
r.Data[r.pos-1] != 'l' ||
(r.pos != len(r.Data) && !isTokenEnd(r.Data[r.pos])) {
r.pos -= 4
r.errSyntax()
}
}
// fetchTrue fetches and checks remaining bytes of true keyword.
func (r *Lexer) fetchTrue() {
r.pos += 4
if r.pos > len(r.Data) ||
r.Data[r.pos-3] != 'r' ||
r.Data[r.pos-2] != 'u' ||
r.Data[r.pos-1] != 'e' ||
(r.pos != len(r.Data) && !isTokenEnd(r.Data[r.pos])) {
r.pos -= 4
r.errSyntax()
}
}
// fetchFalse fetches and checks remaining bytes of false keyword.
func (r *Lexer) fetchFalse() {
r.pos += 5
if r.pos > len(r.Data) ||
r.Data[r.pos-4] != 'a' ||
r.Data[r.pos-3] != 'l' ||
r.Data[r.pos-2] != 's' ||
r.Data[r.pos-1] != 'e' ||
(r.pos != len(r.Data) && !isTokenEnd(r.Data[r.pos])) {
r.pos -= 5
r.errSyntax()
}
}
// fetchNumber scans a number literal token.
func (r *Lexer) fetchNumber() {
hasE := false
afterE := false
hasDot := false
r.pos++
for i, c := range r.Data[r.pos:] {
switch {
case c >= '0' && c <= '9':
afterE = false
case c == '.' && !hasDot:
hasDot = true
case (c == 'e' || c == 'E') && !hasE:
hasE = true
hasDot = true
afterE = true
case (c == '+' || c == '-') && afterE:
afterE = false
default:
r.pos += i
if !isTokenEnd(c) {
r.errSyntax()
} else {
r.token.byteValue = r.Data[r.start:r.pos]
}
return
}
}
r.pos = len(r.Data)
r.token.byteValue = r.Data[r.start:]
}
// findStringLen tries to scan into the string literal for ending quote char to determine required size.
// The size will be exact if no escapes are present and may be inexact if there are escaped chars.
func findStringLen(data []byte) (isValid bool, length int) {
for {
idx := bytes.IndexByte(data, '"')
if idx == -1 {
return false, len(data)
}
if idx == 0 || (idx > 0 && data[idx-1] != '\\') {
return true, length + idx
}
// count \\\\\\\ sequences. even number of slashes means quote is not really escaped
cnt := 1
for idx-cnt-1 >= 0 && data[idx-cnt-1] == '\\' {
cnt++
}
if cnt%2 == 0 {
return true, length + idx
}
length += idx + 1
data = data[idx+1:]
}
}
// unescapeStringToken performs unescaping of string token.
// if no escaping is needed, original string is returned, otherwise - a new one allocated
func (r *Lexer) unescapeStringToken() (err error) {
data := r.token.byteValue
var unescapedData []byte
for {
i := bytes.IndexByte(data, '\\')
if i == -1 {
break
}
escapedRune, escapedBytes, err := decodeEscape(data[i:])
if err != nil {
r.errParse(err.Error())
return err
}
if unescapedData == nil {
unescapedData = make([]byte, 0, len(r.token.byteValue))
}
var d [4]byte
s := utf8.EncodeRune(d[:], escapedRune)
unescapedData = append(unescapedData, data[:i]...)
unescapedData = append(unescapedData, d[:s]...)
data = data[i+escapedBytes:]
}
if unescapedData != nil {
r.token.byteValue = append(unescapedData, data...)
r.token.byteValueCloned = true
}
return
}
// getu4 decodes \uXXXX from the beginning of s, returning the hex value,
// or it returns -1.
func getu4(s []byte) rune {
if len(s) < 6 || s[0] != '\\' || s[1] != 'u' {
return -1
}
var val rune
for i := 2; i < len(s) && i < 6; i++ {
var v byte
c := s[i]
switch c {
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
v = c - '0'
case 'a', 'b', 'c', 'd', 'e', 'f':
v = c - 'a' + 10
case 'A', 'B', 'C', 'D', 'E', 'F':
v = c - 'A' + 10
default:
return -1
}
val <<= 4
val |= rune(v)
}
return val
}
// decodeEscape processes a single escape sequence and returns number of bytes processed.
func decodeEscape(data []byte) (decoded rune, bytesProcessed int, err error) {
if len(data) < 2 {
return 0, 0, errors.New("incorrect escape symbol \\ at the end of token")
}
c := data[1]
switch c {
case '"', '/', '\\':
return rune(c), 2, nil
case 'b':
return '\b', 2, nil
case 'f':
return '\f', 2, nil
case 'n':
return '\n', 2, nil
case 'r':
return '\r', 2, nil
case 't':
return '\t', 2, nil
case 'u':
rr := getu4(data)
if rr < 0 {
return 0, 0, errors.New("incorrectly escaped \\uXXXX sequence")
}
read := 6
if utf16.IsSurrogate(rr) {
rr1 := getu4(data[read:])
if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar {
read += 6
rr = dec
} else {
rr = unicode.ReplacementChar
}
}
return rr, read, nil
}
return 0, 0, errors.New("incorrectly escaped bytes")
}
// fetchString scans a string literal token.
func (r *Lexer) fetchString() {
r.pos++
data := r.Data[r.pos:]
isValid, length := findStringLen(data)
if !isValid {
r.pos += length
r.errParse("unterminated string literal")
return
}
r.token.byteValue = data[:length]
r.pos += length + 1 // skip closing '"' as well
}
// scanToken scans the next token if no token is currently available in the lexer.
func (r *Lexer) scanToken() {
if r.token.kind != tokenUndef || r.fatalError != nil {
return
}
r.FetchToken()
}
// consume resets the current token to allow scanning the next one.
func (r *Lexer) consume() {
r.token.kind = tokenUndef
r.token.byteValueCloned = false
r.token.delimValue = 0
}
// Ok returns true if no error (including io.EOF) was encountered during scanning.
func (r *Lexer) Ok() bool {
return r.fatalError == nil
}
const maxErrorContextLen = 13
func (r *Lexer) errParse(what string) {
if r.fatalError == nil {
var str string
if len(r.Data)-r.pos <= maxErrorContextLen {
str = string(r.Data)
} else {
str = string(r.Data[r.pos:r.pos+maxErrorContextLen-3]) + "..."
}
r.fatalError = &LexerError{
Reason: what,
Offset: r.pos,
Data: str,
}
}
}
func (r *Lexer) errSyntax() {
r.errParse("syntax error")
}
func (r *Lexer) errInvalidToken(expected string) {
if r.fatalError != nil {
return
}
if r.UseMultipleErrors {
r.pos = r.start
r.consume()
r.SkipRecursive()
switch expected {
case "[":
r.token.delimValue = ']'
r.token.kind = tokenDelim
case "{":
r.token.delimValue = '}'
r.token.kind = tokenDelim
}
r.addNonfatalError(&LexerError{
Reason: fmt.Sprintf("expected %s", expected),
Offset: r.start,
Data: string(r.Data[r.start:r.pos]),
})
return
}
var str string
if len(r.token.byteValue) <= maxErrorContextLen {
str = string(r.token.byteValue)
} else {
str = string(r.token.byteValue[:maxErrorContextLen-3]) + "..."
}
r.fatalError = &LexerError{
Reason: fmt.Sprintf("expected %s", expected),
Offset: r.pos,
Data: str,
}
}
func (r *Lexer) GetPos() int {
return r.pos
}
// Delim consumes a token and verifies that it is the given delimiter.
func (r *Lexer) Delim(c byte) {
if r.token.kind == tokenUndef && r.Ok() {
r.FetchToken()
}
if !r.Ok() || r.token.delimValue != c {
r.consume() // errInvalidToken can change token if UseMultipleErrors is enabled.
r.errInvalidToken(string([]byte{c}))
} else {
r.consume()
}
}
// IsDelim returns true if there was no scanning error and next token is the given delimiter.
func (r *Lexer) IsDelim(c byte) bool {
if r.token.kind == tokenUndef && r.Ok() {
r.FetchToken()
}
return !r.Ok() || r.token.delimValue == c
}
// Null verifies that the next token is null and consumes it.
func (r *Lexer) Null() {
if r.token.kind == tokenUndef && r.Ok() {
r.FetchToken()
}
if !r.Ok() || r.token.kind != tokenNull {
r.errInvalidToken("null")
}
r.consume()
}
// IsNull returns true if the next token is a null keyword.
func (r *Lexer) IsNull() bool {
if r.token.kind == tokenUndef && r.Ok() {
r.FetchToken()
}
return r.Ok() && r.token.kind == tokenNull
}
// Skip skips a single token.
func (r *Lexer) Skip() {
if r.token.kind == tokenUndef && r.Ok() {
r.FetchToken()
}
r.consume()
}
// SkipRecursive skips next array or object completely, or just skips a single token if not
// an array/object.
//
// Note: no syntax validation is performed on the skipped data.
func (r *Lexer) SkipRecursive() {
r.scanToken()
var start, end byte
startPos := r.start
switch r.token.delimValue {
case '{':
start, end = '{', '}'
case '[':
start, end = '[', ']'
default:
r.consume()
return
}
r.consume()
level := 1
inQuotes := false
wasEscape := false
for i, c := range r.Data[r.pos:] {
switch {
case c == start && !inQuotes:
level++
case c == end && !inQuotes:
level--
if level == 0 {
r.pos += i + 1
if !json.Valid(r.Data[startPos:r.pos]) {
r.pos = len(r.Data)
r.fatalError = &LexerError{
Reason: "skipped array/object json value is invalid",
Offset: r.pos,
Data: string(r.Data[r.pos:]),
}
}
return
}
case c == '\\' && inQuotes:
wasEscape = !wasEscape
continue
case c == '"' && inQuotes:
inQuotes = wasEscape
case c == '"':
inQuotes = true
}
wasEscape = false
}
r.pos = len(r.Data)
r.fatalError = &LexerError{
Reason: "EOF reached while skipping array/object or token",
Offset: r.pos,
Data: string(r.Data[r.pos:]),
}
}
// Raw fetches the next item recursively as a data slice
func (r *Lexer) Raw() []byte {
r.SkipRecursive()
if !r.Ok() {
return nil
}
return r.Data[r.start:r.pos]
}
// IsStart returns whether the lexer is positioned at the start
// of an input string.
func (r *Lexer) IsStart() bool {
return r.pos == 0
}
// Consumed reads all remaining bytes from the input, publishing an error if
// there is anything but whitespace remaining.
func (r *Lexer) Consumed() {
if r.pos > len(r.Data) || !r.Ok() {
return
}
for _, c := range r.Data[r.pos:] {
if c != ' ' && c != '\t' && c != '\r' && c != '\n' {
r.AddError(&LexerError{
Reason: "invalid character '" + string(c) + "' after top-level value",
Offset: r.pos,
Data: string(r.Data[r.pos:]),
})
return
}
r.pos++
r.start++
}
}
func (r *Lexer) unsafeString(skipUnescape bool) (string, []byte) {
if r.token.kind == tokenUndef && r.Ok() {
r.FetchToken()
}
if !r.Ok() || r.token.kind != tokenString {
r.errInvalidToken("string")
return "", nil
}
if !skipUnescape {
if err := r.unescapeStringToken(); err != nil {
r.errInvalidToken("string")
return "", nil
}
}
bytes := r.token.byteValue
ret := bytesToStr(r.token.byteValue)
r.consume()
return ret, bytes
}
// UnsafeString returns the string value if the token is a string literal.
//
// Warning: returned string may point to the input buffer, so the string should not outlive
// the input buffer. Intended pattern of usage is as an argument to a switch statement.
func (r *Lexer) UnsafeString() string {
ret, _ := r.unsafeString(false)
return ret
}
// UnsafeBytes returns the byte slice if the token is a string literal.
func (r *Lexer) UnsafeBytes() []byte {
_, ret := r.unsafeString(false)
return ret
}
// UnsafeFieldName returns current member name string token
func (r *Lexer) UnsafeFieldName(skipUnescape bool) string {
ret, _ := r.unsafeString(skipUnescape)
return ret
}
// String reads a string literal.
func (r *Lexer) String() string {
if r.token.kind == tokenUndef && r.Ok() {
r.FetchToken()
}
if !r.Ok() || r.token.kind != tokenString {
r.errInvalidToken("string")
return ""
}
if err := r.unescapeStringToken(); err != nil {
r.errInvalidToken("string")
return ""
}
var ret string
if r.token.byteValueCloned {
ret = bytesToStr(r.token.byteValue)
} else {
ret = string(r.token.byteValue)
}
r.consume()
return ret
}
// StringIntern reads a string literal, and performs string interning on it.
func (r *Lexer) StringIntern() string {
if r.token.kind == tokenUndef && r.Ok() {
r.FetchToken()
}
if !r.Ok() || r.token.kind != tokenString {
r.errInvalidToken("string")
return ""
}
if err := r.unescapeStringToken(); err != nil {
r.errInvalidToken("string")
return ""
}
ret := intern.Bytes(r.token.byteValue)
r.consume()
return ret
}
// Bytes reads a string literal and base64 decodes it into a byte slice.
func (r *Lexer) Bytes() []byte {
if r.token.kind == tokenUndef && r.Ok() {
r.FetchToken()
}
if !r.Ok() || r.token.kind != tokenString {
r.errInvalidToken("string")
return nil
}
if err := r.unescapeStringToken(); err != nil {
r.errInvalidToken("string")
return nil
}
ret := make([]byte, base64.StdEncoding.DecodedLen(len(r.token.byteValue)))
n, err := base64.StdEncoding.Decode(ret, r.token.byteValue)
if err != nil {
r.fatalError = &LexerError{
Reason: err.Error(),
}
return nil
}
r.consume()
return ret[:n]
}
// Bool reads a true or false boolean keyword.
func (r *Lexer) Bool() bool {
if r.token.kind == tokenUndef && r.Ok() {
r.FetchToken()
}
if !r.Ok() || r.token.kind != tokenBool {
r.errInvalidToken("bool")
return false
}
ret := r.token.boolValue
r.consume()
return ret
}
func (r *Lexer) number() string {
if r.token.kind == tokenUndef && r.Ok() {
r.FetchToken()
}
if !r.Ok() || r.token.kind != tokenNumber {
r.errInvalidToken("number")
return ""
}
ret := bytesToStr(r.token.byteValue)
r.consume()
return ret
}
func (r *Lexer) Uint8() uint8 {
s := r.number()
if !r.Ok() {
return 0
}
n, err := strconv.ParseUint(s, 10, 8)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: s,
})
}
return uint8(n)
}
func (r *Lexer) Uint16() uint16 {
s := r.number()
if !r.Ok() {
return 0
}
n, err := strconv.ParseUint(s, 10, 16)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: s,
})
}
return uint16(n)
}
func (r *Lexer) Uint32() uint32 {
s := r.number()
if !r.Ok() {
return 0
}
n, err := strconv.ParseUint(s, 10, 32)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: s,
})
}
return uint32(n)
}
func (r *Lexer) Uint64() uint64 {
s := r.number()
if !r.Ok() {
return 0
}
n, err := strconv.ParseUint(s, 10, 64)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: s,
})
}
return n
}
func (r *Lexer) Uint() uint {
return uint(r.Uint64())
}
func (r *Lexer) Int8() int8 {
s := r.number()
if !r.Ok() {
return 0
}
n, err := strconv.ParseInt(s, 10, 8)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: s,
})
}
return int8(n)
}
func (r *Lexer) Int16() int16 {
s := r.number()
if !r.Ok() {
return 0
}
n, err := strconv.ParseInt(s, 10, 16)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: s,
})
}
return int16(n)
}
func (r *Lexer) Int32() int32 {
s := r.number()
if !r.Ok() {
return 0
}
n, err := strconv.ParseInt(s, 10, 32)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: s,
})
}
return int32(n)
}
func (r *Lexer) Int64() int64 {
s := r.number()
if !r.Ok() {
return 0
}
n, err := strconv.ParseInt(s, 10, 64)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: s,
})
}
return n
}
func (r *Lexer) Int() int {
return int(r.Int64())
}
func (r *Lexer) Uint8Str() uint8 {
s, b := r.unsafeString(false)
if !r.Ok() {
return 0
}
n, err := strconv.ParseUint(s, 10, 8)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: string(b),
})
}
return uint8(n)
}
func (r *Lexer) Uint16Str() uint16 {
s, b := r.unsafeString(false)
if !r.Ok() {
return 0
}
n, err := strconv.ParseUint(s, 10, 16)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: string(b),
})
}
return uint16(n)
}
func (r *Lexer) Uint32Str() uint32 {
s, b := r.unsafeString(false)
if !r.Ok() {
return 0
}
n, err := strconv.ParseUint(s, 10, 32)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: string(b),
})
}
return uint32(n)
}
func (r *Lexer) Uint64Str() uint64 {
s, b := r.unsafeString(false)
if !r.Ok() {
return 0
}
n, err := strconv.ParseUint(s, 10, 64)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: string(b),
})
}
return n
}
func (r *Lexer) UintStr() uint {
return uint(r.Uint64Str())
}
func (r *Lexer) UintptrStr() uintptr {
return uintptr(r.Uint64Str())
}
func (r *Lexer) Int8Str() int8 {
s, b := r.unsafeString(false)
if !r.Ok() {
return 0
}
n, err := strconv.ParseInt(s, 10, 8)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: string(b),
})
}
return int8(n)
}
func (r *Lexer) Int16Str() int16 {
s, b := r.unsafeString(false)
if !r.Ok() {
return 0
}
n, err := strconv.ParseInt(s, 10, 16)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: string(b),
})
}
return int16(n)
}
func (r *Lexer) Int32Str() int32 {
s, b := r.unsafeString(false)
if !r.Ok() {
return 0
}
n, err := strconv.ParseInt(s, 10, 32)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: string(b),
})
}
return int32(n)
}
func (r *Lexer) Int64Str() int64 {
s, b := r.unsafeString(false)
if !r.Ok() {
return 0
}
n, err := strconv.ParseInt(s, 10, 64)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: string(b),
})
}
return n
}
func (r *Lexer) IntStr() int {
return int(r.Int64Str())
}
func (r *Lexer) Float32() float32 {
s := r.number()
if !r.Ok() {
return 0
}
n, err := strconv.ParseFloat(s, 32)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: s,
})
}
return float32(n)
}
func (r *Lexer) Float32Str() float32 {
s, b := r.unsafeString(false)
if !r.Ok() {
return 0
}
n, err := strconv.ParseFloat(s, 32)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: string(b),
})
}
return float32(n)
}
func (r *Lexer) Float64() float64 {
s := r.number()
if !r.Ok() {
return 0
}
n, err := strconv.ParseFloat(s, 64)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: s,
})
}
return n
}
func (r *Lexer) Float64Str() float64 {
s, b := r.unsafeString(false)
if !r.Ok() {
return 0
}
n, err := strconv.ParseFloat(s, 64)
if err != nil {
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
Data: string(b),
})
}
return n
}
func (r *Lexer) Error() error {
return r.fatalError
}
func (r *Lexer) AddError(e error) {
if r.fatalError == nil {
r.fatalError = e
}
}
func (r *Lexer) AddNonFatalError(e error) {
r.addNonfatalError(&LexerError{
Offset: r.start,
Data: string(r.Data[r.start:r.pos]),
Reason: e.Error(),
})
}
func (r *Lexer) addNonfatalError(err *LexerError) {
if r.UseMultipleErrors {
// We don't want to add errors with the same offset.
if len(r.multipleErrors) != 0 && r.multipleErrors[len(r.multipleErrors)-1].Offset == err.Offset {
return
}
r.multipleErrors = append(r.multipleErrors, err)
return
}
r.fatalError = err
}
func (r *Lexer) GetNonFatalErrors() []*LexerError {
return r.multipleErrors
}
// JsonNumber fetches and json.Number from 'encoding/json' package.
// Both int, float or string, contains them are valid values
func (r *Lexer) JsonNumber() json.Number {
if r.token.kind == tokenUndef && r.Ok() {
r.FetchToken()
}
if !r.Ok() {
r.errInvalidToken("json.Number")
return json.Number("")
}
switch r.token.kind {
case tokenString:
return json.Number(r.String())
case tokenNumber:
return json.Number(r.Raw())
case tokenNull:
r.Null()
return json.Number("")
default:
r.errSyntax()
return json.Number("")
}
}
// Interface fetches an interface{} analogous to the 'encoding/json' package.
func (r *Lexer) Interface() interface{} {
if r.token.kind == tokenUndef && r.Ok() {
r.FetchToken()
}
if !r.Ok() {
return nil
}
switch r.token.kind {
case tokenString:
return r.String()
case tokenNumber:
return r.Float64()
case tokenBool:
return r.Bool()
case tokenNull:
r.Null()
return nil
}
if r.token.delimValue == '{' {
r.consume()
ret := map[string]interface{}{}
for !r.IsDelim('}') {
key := r.String()
r.WantColon()
ret[key] = r.Interface()
r.WantComma()
}
r.Delim('}')
if r.Ok() {
return ret
} else {
return nil
}
} else if r.token.delimValue == '[' {
r.consume()
ret := []interface{}{}
for !r.IsDelim(']') {
ret = append(ret, r.Interface())
r.WantComma()
}
r.Delim(']')
if r.Ok() {
return ret
} else {
return nil
}
}
r.errSyntax()
return nil
}
// WantComma requires a comma to be present before fetching next token.
func (r *Lexer) WantComma() {
r.wantSep = ','
r.firstElement = false
}
// WantColon requires a colon to be present before fetching next token.
func (r *Lexer) WantColon() {
r.wantSep = ':'
r.firstElement = false
}