1 // Copyright 2016 Google Inc.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 //
15 ////////////////////////////////////////////////////////////////////////////////
16
17 #include "compact_enc_det/compact_enc_det_hint_code.h"
18
19 #include <ctype.h> // for isalpha
20 #include <string.h> // for NULL, memchr, strlen, etc
21
22 #include "util/basictypes.h" // for uint8, uint32
23 #include "util/string_util.h"
24
25 // Upper to lower, keep digits, everything else to minus '-' (2d)
26 static const char kCharsetToLowerTbl[256] = {
27 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
28 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
29 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
30 0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37, 0x38,0x39,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
31
32 0x2d,0x61,0x62,0x63,0x64,0x65,0x66,0x67, 0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,
33 0x70,0x71,0x72,0x73,0x74,0x75,0x76,0x77, 0x78,0x79,0x7a,0x2d,0x2d,0x2d,0x2d,0x2d,
34 0x2d,0x61,0x62,0x63,0x64,0x65,0x66,0x67, 0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,
35 0x70,0x71,0x72,0x73,0x74,0x75,0x76,0x77, 0x78,0x79,0x7a,0x2d,0x2d,0x2d,0x2d,0x2d,
36
37 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
38 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
39 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
40 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
41
42 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
43 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
44 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
45 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
46 };
47
48
49 static const char kIsAlpha[256] = {
50 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
51 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
52 0,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,0,0,0,0,0,
53 0,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,0,0,0,0,0,
54
55 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
56 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
57 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
58 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
59 };
60
61 static const char kIsDigit[256] = {
62 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
63 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1, 1,1,0,0,0,0,0,0,
64 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
65 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
66
67 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
68 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
69 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
70 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
71 };
72
73 static const char* kFakeEncodingName[] = {
74 "FakeEnc100", "FakeEnc101", "FakeEnc102", "FakeEnc103", "FakeEnc104",
75 "FakeEnc105", "FakeEnc106", "FakeEnc107", "FakeEnc108", "FakeEnc109",
76 "FakeEnc110", "FakeEnc111", "FakeEnc112", "FakeEnc113", "FakeEnc114",
77 "FakeEnc115", "FakeEnc116", "FakeEnc117", "FakeEnc118", "FakeEnc119",
78 };
79 static const char* kFakeEncodingName2[] = {
80 "FakeEnc_0", "FakeEnc_1", "FakeEnc_2", "FakeEnc_3", "FakeEnc_4",
81 };
82
83 // Return name for extended encoding
MyEncodingName(Encoding enc)84 const char* MyEncodingName(Encoding enc) {
85 if (enc < 0) {
86 return "~";
87 }
88 if (enc == ISO_8859_1) {
89 return "Latin1"; // I can't stand "ASCII" for this
90 }
91 if (enc < NUM_ENCODINGS) {
92 return EncodingName(enc);
93 }
94 // allow fake names, for exploration
95 if ((NUM_ENCODINGS <= enc) && (enc < (NUM_ENCODINGS + 4))) {
96 return kFakeEncodingName2[enc - NUM_ENCODINGS];
97 }
98 if ((100 <= enc) && (enc < 120)) {
99 return kFakeEncodingName[enc - 100];
100 }
101 return "~";
102 }
103
104
105 // Normalize ASCII string to first 4 alphabetic chars and last 4 digit chars
106 // Letters are forced to lowercase ASCII
107 // Used to normalize charset= values
MakeChar44(const string & str)108 string MakeChar44(const string& str) {
109 string res("________"); // eight underscores
110 int l_ptr = 0;
111 int d_ptr = 0;
112 for (uint32 i = 0; i < str.size(); ++i) {
113 uint8 uc = static_cast<uint8>(str[i]);
114 if (kIsAlpha[uc]) {
115 if (l_ptr < 4) { // Else ignore
116 res[l_ptr] = kCharsetToLowerTbl[uc];
117 l_ptr++;
118 }
119 } else if (kIsDigit[uc]) {
120 if (d_ptr < 4) {
121 res[4 + d_ptr] = kCharsetToLowerTbl[uc];
122 } else {
123 // Keep last 4 digits by shifting left
124 res[4] = res[5];
125 res[5] = res[6];
126 res[6] = res[7];
127 res[7] = kCharsetToLowerTbl[uc];
128 }
129 d_ptr++;
130 } // If neither letter nor digit, drop entirely
131 }
132 return res;
133 }
134
135 // Normalize ASCII string to first 8 alphabetic/digit chars
136 // Letters are forced to lowercase ASCII
137 // Used to normalize TLD values
MakeChar4(const string & str)138 string MakeChar4(const string& str) {
139 string res("____"); // four underscores
140 int l_ptr = 0;
141 for (uint32 i = 0; i < str.size(); ++i) {
142 uint8 uc = static_cast<uint8>(str[i]);
143 if (kIsAlpha[uc] | kIsDigit[uc]) {
144 if (l_ptr < 4) { // Else ignore
145 res[l_ptr] = kCharsetToLowerTbl[uc];
146 l_ptr++;
147 }
148 }
149 }
150 return res;
151 }
152
153 // Normalize ASCII string to first 8 alphabetic/digit chars
154 // Letters are forced to lowercase ASCII
155 // Used to normalize TLD values
MakeChar8(const string & str)156 string MakeChar8(const string& str) {
157 string res("________"); // eight dots
158 int l_ptr = 0;
159 for (uint32 i = 0; i < str.size(); ++i) {
160 uint8 uc = static_cast<uint8>(str[i]);
161 if (kIsAlpha[uc] | kIsDigit[uc]) {
162 if (l_ptr < 8) { // Else ignore
163 res[l_ptr] = kCharsetToLowerTbl[uc];
164 l_ptr++;
165 }
166 }
167 }
168 return res;
169 }
170