1 /* Copyright (c) 2020, 2021, MariaDB
2
3 This program is free software; you can redistribute it and/or modify
4 it under the terms of the GNU General Public License as published by
5 the Free Software Foundation; version 2 of the License.
6
7 This program is distributed in the hope that it will be useful,
8 but WITHOUT ANY WARRANTY; without even the implied warranty of
9 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 GNU General Public License for more details.
11
12 You should have received a copy of the GNU General Public License
13 along with this program; if not, write to the Free Software
14 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */
15
16 /*
17 * Copyright 2016 Ferry Toth, Exalon Delft BV, The Netherlands
18 * This software is provided 'as-is', without any express or implied
19 * warranty. In no event will the author be held liable for any damages
20 * arising from the use of this software.
21 * Permission is granted to anyone to use this software for any purpose,
22 * including commercial applications, and to alter it and redistribute it
23 * freely, subject to the following restrictions:
24 * 1. The origin of this software must not be misrepresented; you must not
25 * claim that you wrote the original software. If you use this software
26 * in a product, an acknowledgment in the product documentation would be
27 * appreciated but is not required.
28 * 2. Altered source versions must be plainly marked as such, and must not be
29 * misrepresented as being the original software.
30 * 3. This notice may not be removed or altered from any source distribution.
31 * Ferry Toth
32 * ftoth@exalondelft.nl
33 *
34 * https://github.com/htot/crc32c
35 *
36 * Modified by Facebook
37 *
38 * Original intel whitepaper:
39 * "Fast CRC Computation for iSCSI Polynomial Using CRC32 Instruction"
40 * https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/crc-iscsi-polynomial-crc32-instruction-paper.pdf
41 *
42 * This version is from the folly library, created by Dave Watson <davejwatson@fb.com>
43 *
44 */
45
46 #include <stdint.h>
47 #include <nmmintrin.h>
48 #include <wmmintrin.h>
49
50
51 #define CRCtriplet(crc, buf, offset) \
52 crc##0 = _mm_crc32_u64(crc##0, *(buf##0 + offset)); \
53 crc##1 = _mm_crc32_u64(crc##1, *(buf##1 + offset)); \
54 crc##2 = _mm_crc32_u64(crc##2, *(buf##2 + offset));
55
56 #define CRCduplet(crc, buf, offset) \
57 crc##0 = _mm_crc32_u64(crc##0, *(buf##0 + offset)); \
58 crc##1 = _mm_crc32_u64(crc##1, *(buf##1 + offset));
59
60 #define CRCsinglet(crc, buf, offset) \
61 crc = _mm_crc32_u64(crc, *(uint64_t*)(buf + offset));
62
63
64 // Numbers taken directly from intel whitepaper.
65 // clang-format off
66 static const uint64_t clmul_constants alignas(16) [] = {
67 0x14cd00bd6, 0x105ec76f0, 0x0ba4fc28e, 0x14cd00bd6,
68 0x1d82c63da, 0x0f20c0dfe, 0x09e4addf8, 0x0ba4fc28e,
69 0x039d3b296, 0x1384aa63a, 0x102f9b8a2, 0x1d82c63da,
70 0x14237f5e6, 0x01c291d04, 0x00d3b6092, 0x09e4addf8,
71 0x0c96cfdc0, 0x0740eef02, 0x18266e456, 0x039d3b296,
72 0x0daece73e, 0x0083a6eec, 0x0ab7aff2a, 0x102f9b8a2,
73 0x1248ea574, 0x1c1733996, 0x083348832, 0x14237f5e6,
74 0x12c743124, 0x02ad91c30, 0x0b9e02b86, 0x00d3b6092,
75 0x018b33a4e, 0x06992cea2, 0x1b331e26a, 0x0c96cfdc0,
76 0x17d35ba46, 0x07e908048, 0x1bf2e8b8a, 0x18266e456,
77 0x1a3e0968a, 0x11ed1f9d8, 0x0ce7f39f4, 0x0daece73e,
78 0x061d82e56, 0x0f1d0f55e, 0x0d270f1a2, 0x0ab7aff2a,
79 0x1c3f5f66c, 0x0a87ab8a8, 0x12ed0daac, 0x1248ea574,
80 0x065863b64, 0x08462d800, 0x11eef4f8e, 0x083348832,
81 0x1ee54f54c, 0x071d111a8, 0x0b3e32c28, 0x12c743124,
82 0x0064f7f26, 0x0ffd852c6, 0x0dd7e3b0c, 0x0b9e02b86,
83 0x0f285651c, 0x0dcb17aa4, 0x010746f3c, 0x018b33a4e,
84 0x1c24afea4, 0x0f37c5aee, 0x0271d9844, 0x1b331e26a,
85 0x08e766a0c, 0x06051d5a2, 0x093a5f730, 0x17d35ba46,
86 0x06cb08e5c, 0x11d5ca20e, 0x06b749fb2, 0x1bf2e8b8a,
87 0x1167f94f2, 0x021f3d99c, 0x0cec3662e, 0x1a3e0968a,
88 0x19329634a, 0x08f158014, 0x0e6fc4e6a, 0x0ce7f39f4,
89 0x08227bb8a, 0x1a5e82106, 0x0b0cd4768, 0x061d82e56,
90 0x13c2b89c4, 0x188815ab2, 0x0d7a4825c, 0x0d270f1a2,
91 0x10f5ff2ba, 0x105405f3e, 0x00167d312, 0x1c3f5f66c,
92 0x0f6076544, 0x0e9adf796, 0x026f6a60a, 0x12ed0daac,
93 0x1a2adb74e, 0x096638b34, 0x19d34af3a, 0x065863b64,
94 0x049c3cc9c, 0x1e50585a0, 0x068bce87a, 0x11eef4f8e,
95 0x1524fa6c6, 0x19f1c69dc, 0x16cba8aca, 0x1ee54f54c,
96 0x042d98888, 0x12913343e, 0x1329d9f7e, 0x0b3e32c28,
97 0x1b1c69528, 0x088f25a3a, 0x02178513a, 0x0064f7f26,
98 0x0e0ac139e, 0x04e36f0b0, 0x0170076fa, 0x0dd7e3b0c,
99 0x141a1a2e2, 0x0bd6f81f8, 0x16ad828b4, 0x0f285651c,
100 0x041d17b64, 0x19425cbba, 0x1fae1cc66, 0x010746f3c,
101 0x1a75b4b00, 0x18db37e8a, 0x0f872e54c, 0x1c24afea4,
102 0x01e41e9fc, 0x04c144932, 0x086d8e4d2, 0x0271d9844,
103 0x160f7af7a, 0x052148f02, 0x05bb8f1bc, 0x08e766a0c,
104 0x0a90fd27a, 0x0a3c6f37a, 0x0b3af077a, 0x093a5f730,
105 0x04984d782, 0x1d22c238e, 0x0ca6ef3ac, 0x06cb08e5c,
106 0x0234e0b26, 0x063ded06a, 0x1d88abd4a, 0x06b749fb2,
107 0x04597456a, 0x04d56973c, 0x0e9e28eb4, 0x1167f94f2,
108 0x07b3ff57a, 0x19385bf2e, 0x0c9c8b782, 0x0cec3662e,
109 0x13a9cba9e, 0x0e417f38a, 0x093e106a4, 0x19329634a,
110 0x167001a9c, 0x14e727980, 0x1ddffc5d4, 0x0e6fc4e6a,
111 0x00df04680, 0x0d104b8fc, 0x02342001e, 0x08227bb8a,
112 0x00a2a8d7e, 0x05b397730, 0x168763fa6, 0x0b0cd4768,
113 0x1ed5a407a, 0x0e78eb416, 0x0d2c3ed1a, 0x13c2b89c4,
114 0x0995a5724, 0x1641378f0, 0x19b1afbc4, 0x0d7a4825c,
115 0x109ffedc0, 0x08d96551c, 0x0f2271e60, 0x10f5ff2ba,
116 0x00b0bf8ca, 0x00bf80dd2, 0x123888b7a, 0x00167d312,
117 0x1e888f7dc, 0x18dcddd1c, 0x002ee03b2, 0x0f6076544,
118 0x183e8d8fe, 0x06a45d2b2, 0x133d7a042, 0x026f6a60a,
119 0x116b0f50c, 0x1dd3e10e8, 0x05fabe670, 0x1a2adb74e,
120 0x130004488, 0x0de87806c, 0x000bcf5f6, 0x19d34af3a,
121 0x18f0c7078, 0x014338754, 0x017f27698, 0x049c3cc9c,
122 0x058ca5f00, 0x15e3e77ee, 0x1af900c24, 0x068bce87a,
123 0x0b5cfca28, 0x0dd07448e, 0x0ded288f8, 0x1524fa6c6,
124 0x059f229bc, 0x1d8048348, 0x06d390dec, 0x16cba8aca,
125 0x037170390, 0x0a3e3e02c, 0x06353c1cc, 0x042d98888,
126 0x0c4584f5c, 0x0d73c7bea, 0x1f16a3418, 0x1329d9f7e,
127 0x0531377e2, 0x185137662, 0x1d8d9ca7c, 0x1b1c69528,
128 0x0b25b29f2, 0x18a08b5bc, 0x19fb2a8b0, 0x02178513a,
129 0x1a08fe6ac, 0x1da758ae0, 0x045cddf4e, 0x0e0ac139e,
130 0x1a91647f2, 0x169cf9eb0, 0x1a0f717c4, 0x0170076fa,
131 };
132
133 // Compute the crc32c value for buffer smaller than 8
align_to_8(size_t len,uint64_t & crc0,const unsigned char * & next)134 static inline void align_to_8(
135 size_t len,
136 uint64_t& crc0, // crc so far, updated on return
137 const unsigned char*& next) { // next data pointer, updated on return
138 uint32_t crc32bit = static_cast<uint32_t>(crc0);
139 if (len & 0x04) {
140 crc32bit = _mm_crc32_u32(crc32bit, *(uint32_t*)next);
141 next += sizeof(uint32_t);
142 }
143 if (len & 0x02) {
144 crc32bit = _mm_crc32_u16(crc32bit, *(uint16_t*)next);
145 next += sizeof(uint16_t);
146 }
147 if (len & 0x01) {
148 crc32bit = _mm_crc32_u8(crc32bit, *(next));
149 next++;
150 }
151 crc0 = crc32bit;
152 }
153
154 //
155 // CombineCRC performs pclmulqdq multiplication of 2 partial CRC's and a well
156 // chosen constant and xor's these with the remaining CRC.
157 //
CombineCRC(size_t block_size,uint64_t crc0,uint64_t crc1,uint64_t crc2,const uint64_t * next2)158 static inline uint64_t CombineCRC(
159 size_t block_size,
160 uint64_t crc0,
161 uint64_t crc1,
162 uint64_t crc2,
163 const uint64_t* next2) {
164 const auto multiplier =
165 *(reinterpret_cast<const __m128i*>(clmul_constants) + block_size - 1);
166 const auto crc0_xmm = _mm_set_epi64x(0, crc0);
167 const auto res0 = _mm_clmulepi64_si128(crc0_xmm, multiplier, 0x00);
168 const auto crc1_xmm = _mm_set_epi64x(0, crc1);
169 const auto res1 = _mm_clmulepi64_si128(crc1_xmm, multiplier, 0x10);
170 const auto res = _mm_xor_si128(res0, res1);
171 crc0 = _mm_cvtsi128_si64(res);
172 crc0 = crc0 ^ *((uint64_t*)next2 - 1);
173 crc2 = _mm_crc32_u64(crc2, crc0);
174 return crc2;
175 }
176
177 // Compute CRC-32C using the Intel hardware instruction.
178 extern "C"
crc32c_3way(uint32_t crc,const char * buf,size_t len)179 uint32_t crc32c_3way(uint32_t crc, const char *buf, size_t len)
180 {
181 const unsigned char* next = (const unsigned char*)buf;
182 uint64_t count;
183 uint64_t crc0, crc1, crc2;
184 crc0 = crc ^ 0xffffffffu;
185
186
187 if (len >= 8) {
188 // if len > 216 then align and use triplets
189 if (len > 216) {
190 {
191 // Work on the bytes (< 8) before the first 8-byte alignment addr starts
192 auto align_bytes = (8 - (uintptr_t)next) & 7;
193 len -= align_bytes;
194 align_to_8(align_bytes, crc0, next);
195 }
196
197 // Now work on the remaining blocks
198 count = len / 24; // number of triplets
199 len %= 24; // bytes remaining
200 uint64_t n = count >> 7; // #blocks = first block + full blocks
201 uint64_t block_size = count & 127;
202 if (block_size == 0) {
203 block_size = 128;
204 } else {
205 n++;
206 }
207 // points to the first byte of the next block
208 const uint64_t* next0 = (uint64_t*)next + block_size;
209 const uint64_t* next1 = next0 + block_size;
210 const uint64_t* next2 = next1 + block_size;
211
212 crc1 = crc2 = 0;
213 // Use Duff's device, a for() loop inside a switch()
214 // statement. This needs to execute at least once, round len
215 // down to nearest triplet multiple
216 switch (block_size) {
217 case 128:
218 do {
219 // jumps here for a full block of len 128
220 CRCtriplet(crc, next, -128);
221 /* fallthrough */
222 case 127:
223 // jumps here or below for the first block smaller
224 CRCtriplet(crc, next, -127);
225 /* fallthrough */
226 case 126:
227 CRCtriplet(crc, next, -126); // than 128
228 /* fallthrough */
229 case 125:
230 CRCtriplet(crc, next, -125);
231 /* fallthrough */
232 case 124:
233 CRCtriplet(crc, next, -124);
234 /* fallthrough */
235 case 123:
236 CRCtriplet(crc, next, -123);
237 /* fallthrough */
238 case 122:
239 CRCtriplet(crc, next, -122);
240 /* fallthrough */
241 case 121:
242 CRCtriplet(crc, next, -121);
243 /* fallthrough */
244 case 120:
245 CRCtriplet(crc, next, -120);
246 /* fallthrough */
247 case 119:
248 CRCtriplet(crc, next, -119);
249 /* fallthrough */
250 case 118:
251 CRCtriplet(crc, next, -118);
252 /* fallthrough */
253 case 117:
254 CRCtriplet(crc, next, -117);
255 /* fallthrough */
256 case 116:
257 CRCtriplet(crc, next, -116);
258 /* fallthrough */
259 case 115:
260 CRCtriplet(crc, next, -115);
261 /* fallthrough */
262 case 114:
263 CRCtriplet(crc, next, -114);
264 /* fallthrough */
265 case 113:
266 CRCtriplet(crc, next, -113);
267 /* fallthrough */
268 case 112:
269 CRCtriplet(crc, next, -112);
270 /* fallthrough */
271 case 111:
272 CRCtriplet(crc, next, -111);
273 /* fallthrough */
274 case 110:
275 CRCtriplet(crc, next, -110);
276 /* fallthrough */
277 case 109:
278 CRCtriplet(crc, next, -109);
279 /* fallthrough */
280 case 108:
281 CRCtriplet(crc, next, -108);
282 /* fallthrough */
283 case 107:
284 CRCtriplet(crc, next, -107);
285 /* fallthrough */
286 case 106:
287 CRCtriplet(crc, next, -106);
288 /* fallthrough */
289 case 105:
290 CRCtriplet(crc, next, -105);
291 /* fallthrough */
292 case 104:
293 CRCtriplet(crc, next, -104);
294 /* fallthrough */
295 case 103:
296 CRCtriplet(crc, next, -103);
297 /* fallthrough */
298 case 102:
299 CRCtriplet(crc, next, -102);
300 /* fallthrough */
301 case 101:
302 CRCtriplet(crc, next, -101);
303 /* fallthrough */
304 case 100:
305 CRCtriplet(crc, next, -100);
306 /* fallthrough */
307 case 99:
308 CRCtriplet(crc, next, -99);
309 /* fallthrough */
310 case 98:
311 CRCtriplet(crc, next, -98);
312 /* fallthrough */
313 case 97:
314 CRCtriplet(crc, next, -97);
315 /* fallthrough */
316 case 96:
317 CRCtriplet(crc, next, -96);
318 /* fallthrough */
319 case 95:
320 CRCtriplet(crc, next, -95);
321 /* fallthrough */
322 case 94:
323 CRCtriplet(crc, next, -94);
324 /* fallthrough */
325 case 93:
326 CRCtriplet(crc, next, -93);
327 /* fallthrough */
328 case 92:
329 CRCtriplet(crc, next, -92);
330 /* fallthrough */
331 case 91:
332 CRCtriplet(crc, next, -91);
333 /* fallthrough */
334 case 90:
335 CRCtriplet(crc, next, -90);
336 /* fallthrough */
337 case 89:
338 CRCtriplet(crc, next, -89);
339 /* fallthrough */
340 case 88:
341 CRCtriplet(crc, next, -88);
342 /* fallthrough */
343 case 87:
344 CRCtriplet(crc, next, -87);
345 /* fallthrough */
346 case 86:
347 CRCtriplet(crc, next, -86);
348 /* fallthrough */
349 case 85:
350 CRCtriplet(crc, next, -85);
351 /* fallthrough */
352 case 84:
353 CRCtriplet(crc, next, -84);
354 /* fallthrough */
355 case 83:
356 CRCtriplet(crc, next, -83);
357 /* fallthrough */
358 case 82:
359 CRCtriplet(crc, next, -82);
360 /* fallthrough */
361 case 81:
362 CRCtriplet(crc, next, -81);
363 /* fallthrough */
364 case 80:
365 CRCtriplet(crc, next, -80);
366 /* fallthrough */
367 case 79:
368 CRCtriplet(crc, next, -79);
369 /* fallthrough */
370 case 78:
371 CRCtriplet(crc, next, -78);
372 /* fallthrough */
373 case 77:
374 CRCtriplet(crc, next, -77);
375 /* fallthrough */
376 case 76:
377 CRCtriplet(crc, next, -76);
378 /* fallthrough */
379 case 75:
380 CRCtriplet(crc, next, -75);
381 /* fallthrough */
382 case 74:
383 CRCtriplet(crc, next, -74);
384 /* fallthrough */
385 case 73:
386 CRCtriplet(crc, next, -73);
387 /* fallthrough */
388 case 72:
389 CRCtriplet(crc, next, -72);
390 /* fallthrough */
391 case 71:
392 CRCtriplet(crc, next, -71);
393 /* fallthrough */
394 case 70:
395 CRCtriplet(crc, next, -70);
396 /* fallthrough */
397 case 69:
398 CRCtriplet(crc, next, -69);
399 /* fallthrough */
400 case 68:
401 CRCtriplet(crc, next, -68);
402 /* fallthrough */
403 case 67:
404 CRCtriplet(crc, next, -67);
405 /* fallthrough */
406 case 66:
407 CRCtriplet(crc, next, -66);
408 /* fallthrough */
409 case 65:
410 CRCtriplet(crc, next, -65);
411 /* fallthrough */
412 case 64:
413 CRCtriplet(crc, next, -64);
414 /* fallthrough */
415 case 63:
416 CRCtriplet(crc, next, -63);
417 /* fallthrough */
418 case 62:
419 CRCtriplet(crc, next, -62);
420 /* fallthrough */
421 case 61:
422 CRCtriplet(crc, next, -61);
423 /* fallthrough */
424 case 60:
425 CRCtriplet(crc, next, -60);
426 /* fallthrough */
427 case 59:
428 CRCtriplet(crc, next, -59);
429 /* fallthrough */
430 case 58:
431 CRCtriplet(crc, next, -58);
432 /* fallthrough */
433 case 57:
434 CRCtriplet(crc, next, -57);
435 /* fallthrough */
436 case 56:
437 CRCtriplet(crc, next, -56);
438 /* fallthrough */
439 case 55:
440 CRCtriplet(crc, next, -55);
441 /* fallthrough */
442 case 54:
443 CRCtriplet(crc, next, -54);
444 /* fallthrough */
445 case 53:
446 CRCtriplet(crc, next, -53);
447 /* fallthrough */
448 case 52:
449 CRCtriplet(crc, next, -52);
450 /* fallthrough */
451 case 51:
452 CRCtriplet(crc, next, -51);
453 /* fallthrough */
454 case 50:
455 CRCtriplet(crc, next, -50);
456 /* fallthrough */
457 case 49:
458 CRCtriplet(crc, next, -49);
459 /* fallthrough */
460 case 48:
461 CRCtriplet(crc, next, -48);
462 /* fallthrough */
463 case 47:
464 CRCtriplet(crc, next, -47);
465 /* fallthrough */
466 case 46:
467 CRCtriplet(crc, next, -46);
468 /* fallthrough */
469 case 45:
470 CRCtriplet(crc, next, -45);
471 /* fallthrough */
472 case 44:
473 CRCtriplet(crc, next, -44);
474 /* fallthrough */
475 case 43:
476 CRCtriplet(crc, next, -43);
477 /* fallthrough */
478 case 42:
479 CRCtriplet(crc, next, -42);
480 /* fallthrough */
481 case 41:
482 CRCtriplet(crc, next, -41);
483 /* fallthrough */
484 case 40:
485 CRCtriplet(crc, next, -40);
486 /* fallthrough */
487 case 39:
488 CRCtriplet(crc, next, -39);
489 /* fallthrough */
490 case 38:
491 CRCtriplet(crc, next, -38);
492 /* fallthrough */
493 case 37:
494 CRCtriplet(crc, next, -37);
495 /* fallthrough */
496 case 36:
497 CRCtriplet(crc, next, -36);
498 /* fallthrough */
499 case 35:
500 CRCtriplet(crc, next, -35);
501 /* fallthrough */
502 case 34:
503 CRCtriplet(crc, next, -34);
504 /* fallthrough */
505 case 33:
506 CRCtriplet(crc, next, -33);
507 /* fallthrough */
508 case 32:
509 CRCtriplet(crc, next, -32);
510 /* fallthrough */
511 case 31:
512 CRCtriplet(crc, next, -31);
513 /* fallthrough */
514 case 30:
515 CRCtriplet(crc, next, -30);
516 /* fallthrough */
517 case 29:
518 CRCtriplet(crc, next, -29);
519 /* fallthrough */
520 case 28:
521 CRCtriplet(crc, next, -28);
522 /* fallthrough */
523 case 27:
524 CRCtriplet(crc, next, -27);
525 /* fallthrough */
526 case 26:
527 CRCtriplet(crc, next, -26);
528 /* fallthrough */
529 case 25:
530 CRCtriplet(crc, next, -25);
531 /* fallthrough */
532 case 24:
533 CRCtriplet(crc, next, -24);
534 /* fallthrough */
535 case 23:
536 CRCtriplet(crc, next, -23);
537 /* fallthrough */
538 case 22:
539 CRCtriplet(crc, next, -22);
540 /* fallthrough */
541 case 21:
542 CRCtriplet(crc, next, -21);
543 /* fallthrough */
544 case 20:
545 CRCtriplet(crc, next, -20);
546 /* fallthrough */
547 case 19:
548 CRCtriplet(crc, next, -19);
549 /* fallthrough */
550 case 18:
551 CRCtriplet(crc, next, -18);
552 /* fallthrough */
553 case 17:
554 CRCtriplet(crc, next, -17);
555 /* fallthrough */
556 case 16:
557 CRCtriplet(crc, next, -16);
558 /* fallthrough */
559 case 15:
560 CRCtriplet(crc, next, -15);
561 /* fallthrough */
562 case 14:
563 CRCtriplet(crc, next, -14);
564 /* fallthrough */
565 case 13:
566 CRCtriplet(crc, next, -13);
567 /* fallthrough */
568 case 12:
569 CRCtriplet(crc, next, -12);
570 /* fallthrough */
571 case 11:
572 CRCtriplet(crc, next, -11);
573 /* fallthrough */
574 case 10:
575 CRCtriplet(crc, next, -10);
576 /* fallthrough */
577 case 9:
578 CRCtriplet(crc, next, -9);
579 /* fallthrough */
580 case 8:
581 CRCtriplet(crc, next, -8);
582 /* fallthrough */
583 case 7:
584 CRCtriplet(crc, next, -7);
585 /* fallthrough */
586 case 6:
587 CRCtriplet(crc, next, -6);
588 /* fallthrough */
589 case 5:
590 CRCtriplet(crc, next, -5);
591 /* fallthrough */
592 case 4:
593 CRCtriplet(crc, next, -4);
594 /* fallthrough */
595 case 3:
596 CRCtriplet(crc, next, -3);
597 /* fallthrough */
598 case 2:
599 CRCtriplet(crc, next, -2);
600 /* fallthrough */
601 case 1:
602 CRCduplet(crc, next, -1); // the final triplet is actually only 2
603 //{ CombineCRC(); }
604 crc0 = CombineCRC(block_size, crc0, crc1, crc2, next2);
605 if (--n > 0) {
606 crc1 = crc2 = 0;
607 block_size = 128;
608 // points to the first byte of the next block
609 next0 = next2 + 128;
610 next1 = next0 + 128; // from here on all blocks are 128 long
611 next2 = next1 + 128;
612 }
613 /* fallthrough */
614 case 0:;
615 } while (n > 0);
616 }
617 next = (const unsigned char*)next2;
618 }
619 uint64_t count2 = len >> 3; // 216 of less bytes is 27 or less singlets
620 len = len & 7;
621 next += (count2 * 8);
622 switch (count2) {
623 case 27:
624 CRCsinglet(crc0, next, -27 * 8);
625 /* fallthrough */
626 case 26:
627 CRCsinglet(crc0, next, -26 * 8);
628 /* fallthrough */
629 case 25:
630 CRCsinglet(crc0, next, -25 * 8);
631 /* fallthrough */
632 case 24:
633 CRCsinglet(crc0, next, -24 * 8);
634 /* fallthrough */
635 case 23:
636 CRCsinglet(crc0, next, -23 * 8);
637 /* fallthrough */
638 case 22:
639 CRCsinglet(crc0, next, -22 * 8);
640 /* fallthrough */
641 case 21:
642 CRCsinglet(crc0, next, -21 * 8);
643 /* fallthrough */
644 case 20:
645 CRCsinglet(crc0, next, -20 * 8);
646 /* fallthrough */
647 case 19:
648 CRCsinglet(crc0, next, -19 * 8);
649 /* fallthrough */
650 case 18:
651 CRCsinglet(crc0, next, -18 * 8);
652 /* fallthrough */
653 case 17:
654 CRCsinglet(crc0, next, -17 * 8);
655 /* fallthrough */
656 case 16:
657 CRCsinglet(crc0, next, -16 * 8);
658 /* fallthrough */
659 case 15:
660 CRCsinglet(crc0, next, -15 * 8);
661 /* fallthrough */
662 case 14:
663 CRCsinglet(crc0, next, -14 * 8);
664 /* fallthrough */
665 case 13:
666 CRCsinglet(crc0, next, -13 * 8);
667 /* fallthrough */
668 case 12:
669 CRCsinglet(crc0, next, -12 * 8);
670 /* fallthrough */
671 case 11:
672 CRCsinglet(crc0, next, -11 * 8);
673 /* fallthrough */
674 case 10:
675 CRCsinglet(crc0, next, -10 * 8);
676 /* fallthrough */
677 case 9:
678 CRCsinglet(crc0, next, -9 * 8);
679 /* fallthrough */
680 case 8:
681 CRCsinglet(crc0, next, -8 * 8);
682 /* fallthrough */
683 case 7:
684 CRCsinglet(crc0, next, -7 * 8);
685 /* fallthrough */
686 case 6:
687 CRCsinglet(crc0, next, -6 * 8);
688 /* fallthrough */
689 case 5:
690 CRCsinglet(crc0, next, -5 * 8);
691 /* fallthrough */
692 case 4:
693 CRCsinglet(crc0, next, -4 * 8);
694 /* fallthrough */
695 case 3:
696 CRCsinglet(crc0, next, -3 * 8);
697 /* fallthrough */
698 case 2:
699 CRCsinglet(crc0, next, -2 * 8);
700 /* fallthrough */
701 case 1:
702 CRCsinglet(crc0, next, -1 * 8);
703 /* fallthrough */
704 case 0:;
705 }
706 }
707 {
708 align_to_8(len, crc0, next);
709 return (uint32_t)crc0 ^ 0xffffffffu;
710 }
711 }
712