1. 1
2. 2
3. 3
4. 4
5. 5
6. 6
7. 7
8. 8
9. 9
10. 10
11. 11
12. 12
13. 13
14. 14
15. 15
16. 16
17. 17
18. 18
19. 19
20. 20
21. 21
22. 22
23. 23
24. 24
25. 25
26. 26
27. 27
28. 28
29. 29
30. 30
31. 31
32. 32
33. 33
34. 34
35. 35
36. 36
37. 37
38. 38
39. 39
40. 40
41. 41
42. 42
43. 43
44. 44
45. 45
46. 46
47. 47
48. 48
49. 49
50. 50
51. 51
52. 52
53. 53
54. 54
55. 55
56. 56
57. 57
58. 58
59. 59
60. 60
61. 61
62. 62
63. 63
64. 64
65. 65
66. 66
67. 67
68. 68
69. 69
70. 70
71. 71
72. 72
73. 73
74. 74
75. 75
76. 76
77. 77
78. 78
79. 79
80. 80
81. 81
82. 82
83. 83

package chardet

type recognizer interface {
	Match(*recognizerInput) recognizerOutput
}

type recognizerOutput Result

type recognizerInput struct {
	raw         []byte
	input       []byte
	tagStripped bool
	byteStats   []int
	hasC1Bytes  bool
}

func newRecognizerInput(raw []byte, stripTag bool) *recognizerInput {
	input, stripped := mayStripInput(raw, stripTag)
	byteStats := computeByteStats(input)
	return &recognizerInput{
		raw:         raw,
		input:       input,
		tagStripped: stripped,
		byteStats:   byteStats,
		hasC1Bytes:  computeHasC1Bytes(byteStats),
	}
}

func mayStripInput(raw []byte, stripTag bool) (out []byte, stripped bool) {
	const inputBufferSize = 8192
	out = make([]byte, 0, inputBufferSize)
	var badTags, openTags int32
	var inMarkup bool = false
	stripped = false
	if stripTag {
		stripped = true
		for _, c := range raw {
			if c == '<' {
				if inMarkup {
					badTags += 1
				}
				inMarkup = true
				openTags += 1
			}
			if !inMarkup {
				out = append(out, c)
				if len(out) >= inputBufferSize {
					break
				}
			}
			if c == '>' {
				inMarkup = false
			}
		}
	}
	if openTags < 5 || openTags/5 < badTags || (len(out) < 100 && len(raw) > 600) {
		limit := len(raw)
		if limit > inputBufferSize {
			limit = inputBufferSize
		}
		out = make([]byte, limit)
		copy(out, raw[:limit])
		stripped = false
	}
	return
}

func computeByteStats(input []byte) []int {
	r := make([]int, 256)
	for _, c := range input {
		r[c] += 1
	}
	return r
}

func computeHasC1Bytes(byteStats []int) bool {
	for _, count := range byteStats[0x80 : 0x9F+1] {
		if count > 0 {
			return true
		}
	}
	return false
}