1. 1
2. 2
3. 3
4. 4
5. 5
6. 6
7. 7
8. 8
9. 9
10. 10
11. 11
12. 12
13. 13
14. 14
15. 15
16. 16
17. 17
18. 18
19. 19
20. 20
21. 21
22. 22
23. 23
24. 24
25. 25
26. 26
27. 27
28. 28
29. 29
30. 30
31. 31
32. 32
33. 33
34. 34
35. 35
36. 36
37. 37
38. 38
39. 39
40. 40
41. 41
42. 42
43. 43
44. 44
45. 45
46. 46
47. 47
48. 48
49. 49
50. 50
51. 51
52. 52
53. 53
54. 54
55. 55
56. 56
57. 57
58. 58
59. 59
60. 60
61. 61
62. 62
63. 63
64. 64
65. 65
66. 66
67. 67
68. 68
69. 69
70. 70
71. 71

package brotli

/* Copyright 2013 Google Inc. All Rights Reserved.

   Distributed under MIT license.
   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/

/* Heuristics for deciding about the UTF8-ness of strings. */

const kMinUTF8Ratio float64 = 0.75

/* Returns 1 if at least min_fraction of the bytes between pos and
   pos + length in the (data, mask) ring-buffer is UTF8-encoded, otherwise
   returns 0. */
func parseAsUTF8(symbol *int, input []byte, size uint) uint {
	/* ASCII */
	if input[0]&0x80 == 0 {
		*symbol = int(input[0])
		if *symbol > 0 {
			return 1
		}
	}

	/* 2-byte UTF8 */
	if size > 1 && input[0]&0xE0 == 0xC0 && input[1]&0xC0 == 0x80 {
		*symbol = (int(input[0])&0x1F)<<6 | int(input[1])&0x3F
		if *symbol > 0x7F {
			return 2
		}
	}

	/* 3-byte UFT8 */
	if size > 2 && input[0]&0xF0 == 0xE0 && input[1]&0xC0 == 0x80 && input[2]&0xC0 == 0x80 {
		*symbol = (int(input[0])&0x0F)<<12 | (int(input[1])&0x3F)<<6 | int(input[2])&0x3F
		if *symbol > 0x7FF {
			return 3
		}
	}

	/* 4-byte UFT8 */
	if size > 3 && input[0]&0xF8 == 0xF0 && input[1]&0xC0 == 0x80 && input[2]&0xC0 == 0x80 && input[3]&0xC0 == 0x80 {
		*symbol = (int(input[0])&0x07)<<18 | (int(input[1])&0x3F)<<12 | (int(input[2])&0x3F)<<6 | int(input[3])&0x3F
		if *symbol > 0xFFFF && *symbol <= 0x10FFFF {
			return 4
		}
	}

	/* Not UTF8, emit a special symbol above the UTF8-code space */
	*symbol = 0x110000 | int(input[0])

	return 1
}

/* Returns 1 if at least min_fraction of the data is UTF8-encoded.*/
func isMostlyUTF8(data []byte, pos uint, mask uint, length uint, min_fraction float64) bool {
	var size_utf8 uint = 0
	var i uint = 0
	for i < length {
		var symbol int
		var current_data []byte
		current_data = data[(pos+i)&mask:]
		var bytes_read uint = parseAsUTF8(&symbol, current_data, length-i)
		i += bytes_read
		if symbol < 0x110000 {
			size_utf8 += bytes_read
		}
	}

	return float64(size_utf8) > min_fraction*float64(length)
}