1 /*
2 * %CopyrightBegin%
3 *
4 * Copyright Ericsson AB 1999-2020. All Rights Reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * %CopyrightEnd%
19 */
20
21 #ifdef HAVE_CONFIG_H
22 # include "config.h"
23 #endif
24
25 #include "sys.h"
26 #include "erl_vm.h"
27 #include "global.h"
28 #include "erl_process.h"
29 #include "error.h"
30 #include "bif.h"
31 #include "big.h"
32 #include "erl_bits.h"
33 #include "erl_binary.h"
34
35 #if defined(WORDS_BIGENDIAN)
36 # define BIT_ENDIAN_MACHINE 0
37 #else
38 # define BIT_ENDIAN_MACHINE BSF_LITTLE
39 #endif
40
41 #define BIT_IS_MACHINE_ENDIAN(x) (((x)&BSF_LITTLE) == BIT_ENDIAN_MACHINE)
42
43 /*
44 * MAKE_MASK(n) constructs a mask with n bits.
45 * Example: MAKE_MASK(3) returns the binary number 00000111.
46 */
47
48 #define MAKE_MASK(n) ((((Uint) 1) << (n))-1)
49
50 /*
51 * MASK_BITS assign src to dst, but preserves the dst bits outside the mask.
52 */
53
54 #define MASK_BITS(src,dst,mask) (((src) & (mask)) | ((dst) & ~(mask)))
55
56 static byte get_bit(byte b, size_t a_offs);
57
58 /* the state resides in the current process' scheduler data */
59
60 #define byte_buf (ErlBitsState.byte_buf_)
61 #define byte_buf_len (ErlBitsState.byte_buf_len_)
62
63 static erts_atomic_t bits_bufs_size;
64
65 Uint
erts_bits_bufs_size(void)66 erts_bits_bufs_size(void)
67 {
68 return (Uint) erts_atomic_read_nob(&bits_bufs_size);
69 }
70
71 void
erts_bits_init_state(ERL_BITS_PROTO_0)72 erts_bits_init_state(ERL_BITS_PROTO_0)
73 {
74 byte_buf_len = 1;
75 byte_buf = erts_alloc(ERTS_ALC_T_BITS_BUF, byte_buf_len);
76
77 erts_bin_offset = 0;
78 }
79
80 void
erts_bits_destroy_state(ERL_BITS_PROTO_0)81 erts_bits_destroy_state(ERL_BITS_PROTO_0)
82 {
83 erts_free(ERTS_ALC_T_BITS_BUF, byte_buf);
84 }
85
86 void
erts_init_bits(void)87 erts_init_bits(void)
88 {
89 ERTS_CT_ASSERT(offsetof(Binary,orig_bytes) % 8 == 0);
90 ERTS_CT_ASSERT(offsetof(ErtsMagicBinary,u.aligned.data) % 8 == 0);
91 ERTS_CT_ASSERT(offsetof(ErtsBinary,driver.binary.orig_bytes)
92 == offsetof(Binary,orig_bytes));
93
94 erts_atomic_init_nob(&bits_bufs_size, 0);
95 /* erl_process.c calls erts_bits_init_state() on all state instances */
96 }
97
98 /*****************************************************************
99 ***
100 *** New matching binaries functions
101 ***
102 *****************************************************************/
103
104 #define ReadToVariable(v64, Buffer, x) \
105 do{ \
106 int _i; \
107 v64 = 0; \
108 for(_i = 0; _i < x; _i++) { \
109 v64 = ((Uint)Buffer[_i] <<(8*_i)) + v64; \
110 } \
111 }while(0) \
112
113 Eterm
erts_bs_start_match_2(Process * p,Eterm Binary,Uint Max)114 erts_bs_start_match_2(Process *p, Eterm Binary, Uint Max)
115 {
116 Eterm Orig;
117 Uint offs;
118 Uint* hp;
119 Uint NeededSize;
120 ErlBinMatchState *ms;
121 Uint bitoffs;
122 Uint bitsize;
123 Uint total_bin_size;
124 ProcBin* pb;
125
126 ASSERT(is_binary(Binary));
127
128 total_bin_size = binary_size(Binary);
129 ASSERT(total_bin_size <= ERTS_UWORD_MAX / CHAR_BIT);
130
131 NeededSize = ERL_BIN_MATCHSTATE_SIZE(Max);
132 hp = HeapOnlyAlloc(p, NeededSize);
133 ms = (ErlBinMatchState *) hp;
134 ERTS_GET_REAL_BIN(Binary, Orig, offs, bitoffs, bitsize);
135 pb = (ProcBin *) boxed_val(Orig);
136 if (pb->thing_word == HEADER_PROC_BIN && pb->flags != 0) {
137 erts_emasculate_writable_binary(pb);
138 }
139 ms->thing_word = HEADER_BIN_MATCHSTATE(Max);
140 (ms->mb).orig = Orig;
141 (ms->mb).base = binary_bytes(Orig);
142 (ms->mb).offset = ms->save_offset[0] = 8 * offs + bitoffs;
143 (ms->mb).size = total_bin_size * 8 + (ms->mb).offset + bitsize;
144 return make_matchstate(ms);
145 }
146
erts_bs_start_match_3(Process * p,Eterm Binary)147 ErlBinMatchState *erts_bs_start_match_3(Process *p, Eterm Binary)
148 {
149 Eterm Orig;
150 Uint offs;
151 Uint* hp;
152 Uint NeededSize;
153 ErlBinMatchState *ms;
154 Uint bitoffs;
155 Uint bitsize;
156 Uint total_bin_size;
157 ProcBin* pb;
158
159 ASSERT(is_binary(Binary));
160
161 total_bin_size = binary_size(Binary);
162 ASSERT(total_bin_size <= ERTS_UWORD_MAX / CHAR_BIT);
163
164 NeededSize = ERL_BIN_MATCHSTATE_SIZE(0);
165 hp = HeapOnlyAlloc(p, NeededSize);
166 ms = (ErlBinMatchState *) hp;
167 ERTS_GET_REAL_BIN(Binary, Orig, offs, bitoffs, bitsize);
168 pb = (ProcBin *) boxed_val(Orig);
169 if (pb->thing_word == HEADER_PROC_BIN && pb->flags != 0) {
170 erts_emasculate_writable_binary(pb);
171 }
172
173 ms->thing_word = HEADER_BIN_MATCHSTATE(0);
174 (ms->mb).orig = Orig;
175 (ms->mb).base = binary_bytes(Orig);
176 (ms->mb).offset = 8 * offs + bitoffs;
177 (ms->mb).size = total_bin_size * 8 + (ms->mb).offset + bitsize;
178
179 return ms;
180 }
181
182 #ifdef DEBUG
183 # define CHECK_MATCH_BUFFER(MB) check_match_buffer(MB)
184
check_match_buffer(ErlBinMatchBuffer * mb)185 static void check_match_buffer(ErlBinMatchBuffer* mb)
186 {
187 Eterm realbin;
188 Uint byteoffs;
189 byte* bytes, bitoffs, bitsz;
190 ProcBin* pb;
191 ERTS_GET_REAL_BIN(mb->orig, realbin, byteoffs, bitoffs, bitsz);
192 bytes = binary_bytes(realbin) + byteoffs;
193 ERTS_ASSERT(mb->base >= bytes && mb->base <= (bytes + binary_size(mb->orig)));
194 pb = (ProcBin *) boxed_val(realbin);
195 if (pb->thing_word == HEADER_PROC_BIN)
196 ERTS_ASSERT(pb->flags == 0);
197 }
198 #else
199 # define CHECK_MATCH_BUFFER(MB)
200 #endif
201
202 Eterm
erts_bs_get_integer_2(Process * p,Uint num_bits,unsigned flags,ErlBinMatchBuffer * mb)203 erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb)
204 {
205 Uint bytes;
206 Uint bits;
207 Uint offs;
208 byte bigbuf[64];
209 byte* LSB;
210 byte* MSB;
211 Uint* hp;
212 Uint words_needed;
213 Uint actual;
214 Uint v32;
215 int sgn = 0;
216 Eterm res = THE_NON_VALUE;
217
218 if (num_bits == 0) {
219 return SMALL_ZERO;
220 }
221
222 CHECK_MATCH_BUFFER(mb);
223 if (mb->size - mb->offset < num_bits) { /* Asked for too many bits. */
224 return THE_NON_VALUE;
225 }
226
227 /*
228 * Special cases for field sizes up to the size of Uint.
229 */
230
231 if (num_bits <= 8-(offs = BIT_OFFSET(mb->offset))) {
232 /*
233 * All bits are in one byte in the binary. We only need
234 * shift them right and mask them.
235 */
236 Uint b = mb->base[BYTE_OFFSET(mb->offset)];
237 Uint mask = MAKE_MASK(num_bits);
238 mb->offset += num_bits;
239 b >>= 8 - offs - num_bits;
240 b &= mask;
241 if ((flags & BSF_SIGNED) && b >> (num_bits-1)) {
242 b |= ~mask;
243 }
244 return make_small(b);
245 } else if (num_bits <= 8) {
246 /*
247 * The bits are in two different bytes. It is easiest to
248 * combine the bytes to a word first, and then shift right and
249 * mask to extract the bits.
250 */
251 Uint byte_offset = BYTE_OFFSET(mb->offset);
252 Uint w = mb->base[byte_offset] << 8 | mb->base[byte_offset+1];
253 Uint mask = MAKE_MASK(num_bits);
254 mb->offset += num_bits;
255 w >>= 16 - offs - num_bits;
256 w &= mask;
257 if ((flags & BSF_SIGNED) && w >> (num_bits-1)) {
258 w |= ~mask;
259 }
260 return make_small(w);
261 } else if (num_bits < SMALL_BITS && (flags & BSF_LITTLE) == 0) {
262 /*
263 * Handle field sizes from 9 up to SMALL_BITS-1 bits, big-endian,
264 * stored in at least two bytes.
265 */
266 byte* bp = mb->base + BYTE_OFFSET(mb->offset);
267 Uint n;
268 Uint w;
269
270 n = num_bits;
271 mb->offset += num_bits;
272
273 /*
274 * Handle the most signicant byte if it contains 1 to 7 bits.
275 * It only needs to be masked, not shifted.
276 */
277 if (offs == 0) {
278 w = 0;
279 } else {
280 Uint num_bits_in_msb = 8 - offs;
281 w = *bp++;
282 n -= num_bits_in_msb;
283 w &= MAKE_MASK(num_bits_in_msb);
284 }
285
286 /*
287 * Simply shift whole bytes into the result.
288 */
289 switch (BYTE_OFFSET(n)) {
290 #if defined(ARCH_64)
291 case 7: w = (w << 8) | *bp++;
292 case 6: w = (w << 8) | *bp++;
293 case 5: w = (w << 8) | *bp++;
294 case 4: w = (w << 8) | *bp++;
295 #endif
296 case 3: w = (w << 8) | *bp++;
297 case 2: w = (w << 8) | *bp++;
298 case 1: w = (w << 8) | *bp++;
299 }
300 n = BIT_OFFSET(n);
301
302 /*
303 * Handle the 1 to 7 bits remaining in the last byte (if any).
304 * They need to be shifted right, but there is no need to mask;
305 * then they can be shifted into the word.
306 */
307 if (n > 0) {
308 Uint b = *bp;
309 b >>= 8 - n;
310 w = (w << n) | b;
311 }
312
313 /*
314 * Sign extend the result if the field type is 'signed' and the
315 * most significant bit is 1.
316 */
317 if ((flags & BSF_SIGNED) != 0 && (w >> (num_bits-1) != 0)) {
318 w |= ~MAKE_MASK(num_bits);
319 }
320 return make_small(w);
321 }
322
323 /*
324 * Handle everything else, that is:
325 *
326 * Big-endian fields >= SMALL_BITS (potentially bignums).
327 * Little-endian fields with 9 or more bits.
328 */
329
330 bytes = NBYTES(num_bits);
331 if ((bits = BIT_OFFSET(num_bits)) == 0) { /* number of bits in MSB */
332 bits = 8;
333 }
334 offs = 8 - bits; /* adjusted offset in MSB */
335
336 if (bytes <= sizeof bigbuf) {
337 LSB = bigbuf;
338 } else {
339 LSB = erts_alloc(ERTS_ALC_T_TMP, bytes);
340 }
341 MSB = LSB + bytes - 1;
342
343 /*
344 * Move bits to temporary buffer. We want the buffer to be stored in
345 * little-endian order, since bignums are little-endian.
346 */
347
348 if (flags & BSF_LITTLE) {
349 erts_copy_bits(mb->base, mb->offset, 1, LSB, 0, 1, num_bits);
350 *MSB >>= offs; /* adjust msb */
351 } else {
352 *MSB = 0;
353 erts_copy_bits(mb->base, mb->offset, 1, MSB, offs, -1, num_bits);
354 }
355 mb->offset += num_bits;
356
357 /*
358 * Get the sign bit.
359 */
360 sgn = 0;
361 if ((flags & BSF_SIGNED) && (*MSB & (1<<(bits-1)))) {
362 byte* ptr = LSB;
363 byte c = 1;
364
365 /* sign extend MSB */
366 *MSB |= ~MAKE_MASK(bits);
367
368 /* two's complement */
369 while (ptr <= MSB) {
370 byte pd = ~(*ptr);
371 byte d = pd + c;
372 c = (d < pd);
373 *ptr++ = d;
374 }
375 sgn = 1;
376 }
377
378 /* normalize */
379 while ((*MSB == 0) && (MSB > LSB)) {
380 MSB--;
381 bytes--;
382 }
383
384 /* check for guaranteed small num */
385 switch (bytes) {
386 case 1:
387 v32 = LSB[0];
388 goto big_small;
389 case 2:
390 v32 = LSB[0] + (LSB[1]<<8);
391 goto big_small;
392 case 3:
393 v32 = LSB[0] + (LSB[1]<<8) + (LSB[2]<<16);
394 goto big_small;
395 #if !defined(ARCH_64)
396 case 4:
397 v32 = (LSB[0] + (LSB[1]<<8) + (LSB[2]<<16) + (LSB[3]<<24));
398 if (!IS_USMALL(sgn, v32)) {
399 goto make_big;
400 }
401 #else
402 case 4:
403 ReadToVariable(v32, LSB, 4);
404 goto big_small;
405 case 5:
406 ReadToVariable(v32, LSB, 5);
407 goto big_small;
408 case 6:
409 ReadToVariable(v32, LSB, 6);
410 goto big_small;
411 case 7:
412 ReadToVariable(v32, LSB, 7);
413 goto big_small;
414 case 8:
415 ReadToVariable(v32, LSB, 8);
416 if (!IS_USMALL(sgn, v32)) {
417 goto make_big;
418 }
419 #endif
420 big_small: /* v32 loaded with value which fits in fixnum */
421 if (sgn) {
422 res = make_small(-((Sint)v32));
423 } else {
424 res = make_small(v32);
425 }
426 break;
427 make_big:
428 hp = HeapOnlyAlloc(p, BIG_UINT_HEAP_SIZE);
429 if (sgn) {
430 hp[0] = make_neg_bignum_header(1);
431 } else {
432 hp[0] = make_pos_bignum_header(1);
433 }
434 BIG_DIGIT(hp,0) = v32;
435 res = make_big(hp);
436 break;
437 default:
438 words_needed = 1+WSIZE(bytes);
439 hp = HeapOnlyAlloc(p, words_needed);
440 res = bytes_to_big(LSB, bytes, sgn, hp);
441 if (is_nil(res)) {
442 p->htop = hp;
443 res = THE_NON_VALUE;
444 } else if (is_small(res)) {
445 p->htop = hp;
446 } else if ((actual = bignum_header_arity(*hp)+1) < words_needed) {
447 p->htop = hp + actual;
448 }
449 break;
450 }
451
452 if (LSB != bigbuf) {
453 erts_free(ERTS_ALC_T_TMP, (void *) LSB);
454 }
455 return res;
456 }
457
458 Eterm
erts_bs_get_binary_2(Process * p,Uint num_bits,unsigned flags,ErlBinMatchBuffer * mb)459 erts_bs_get_binary_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb)
460 {
461 Eterm result;
462
463 CHECK_MATCH_BUFFER(mb);
464 if (mb->size - mb->offset < num_bits) {
465 /* Asked for too many bits. */
466 return THE_NON_VALUE;
467 }
468
469 /*
470 * From now on, we can't fail.
471 */
472
473 result = erts_extract_sub_binary(&HEAP_TOP(p),
474 mb->orig, mb->base,
475 mb->offset, num_bits);
476
477 mb->offset += num_bits;
478
479 return result;
480 }
481
482 Eterm
erts_bs_get_float_2(Process * p,Uint num_bits,unsigned flags,ErlBinMatchBuffer * mb)483 erts_bs_get_float_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb)
484 {
485 Eterm* hp;
486 float f32;
487 double f64;
488 byte* fptr;
489 FloatDef f;
490
491 CHECK_MATCH_BUFFER(mb);
492 if (num_bits == 0) {
493 f.fd = 0.0;
494 hp = HeapOnlyAlloc(p, FLOAT_SIZE_OBJECT);
495 PUT_DOUBLE(f, hp);
496 return make_float(hp);
497 }
498 if (mb->size - mb->offset < num_bits) { /* Asked for too many bits. */
499 return THE_NON_VALUE;
500 }
501 if (num_bits == 32) {
502 fptr = (byte *) &f32;
503 } else if (num_bits == 64) {
504 fptr = (byte *) &f64;
505 } else {
506 return THE_NON_VALUE;
507 }
508
509 if (BIT_IS_MACHINE_ENDIAN(flags)) {
510 erts_copy_bits(mb->base, mb->offset, 1,
511 fptr, 0, 1,
512 num_bits);
513 } else {
514 erts_copy_bits(mb->base, mb->offset, 1,
515 fptr + NBYTES(num_bits) - 1, 0, -1,
516 num_bits);
517 }
518 ERTS_FP_CHECK_INIT(p);
519 if (num_bits == 32) {
520 ERTS_FP_ERROR_THOROUGH(p, f32, return THE_NON_VALUE);
521 f.fd = f32;
522 } else {
523 #ifdef DOUBLE_MIDDLE_ENDIAN
524 FloatDef ftmp;
525 ftmp.fd = f64;
526 f.fw[0] = ftmp.fw[1];
527 f.fw[1] = ftmp.fw[0];
528 ERTS_FP_ERROR_THOROUGH(p, f.fd, return THE_NON_VALUE);
529 #else
530 ERTS_FP_ERROR_THOROUGH(p, f64, return THE_NON_VALUE);
531 f.fd = f64;
532 #endif
533 }
534 mb->offset += num_bits;
535 hp = HeapOnlyAlloc(p, FLOAT_SIZE_OBJECT);
536 PUT_DOUBLE(f, hp);
537 return make_float(hp);
538 }
539
540 Eterm
erts_bs_get_binary_all_2(Process * p,ErlBinMatchBuffer * mb)541 erts_bs_get_binary_all_2(Process *p, ErlBinMatchBuffer* mb)
542 {
543 Uint bit_size;
544 Eterm result;
545
546 CHECK_MATCH_BUFFER(mb);
547 bit_size = mb->size - mb->offset;
548
549 result = erts_extract_sub_binary(&HEAP_TOP(p),
550 mb->orig, mb->base,
551 mb->offset, bit_size);
552
553 mb->offset = mb->size;
554
555 return result;
556 }
557
558 /****************************************************************
559 ***
560 *** Building binaries
561 ***
562 ****************************************************************/
563
564
565 /* COPY_VAL:
566 * copy sz byte from val to dst buffer,
567 * dst, val are updated!!!
568 */
569
570 #define COPY_VAL(dst,ddir,val,sz) do { \
571 Uint __sz = (sz); \
572 while(__sz) { \
573 switch(__sz) { \
574 default: \
575 case 4: *dst = (val&0xff); dst += ddir; val >>= 8; __sz--; \
576 case 3: *dst = (val&0xff); dst += ddir; val >>= 8; __sz--; \
577 case 2: *dst = (val&0xff); dst += ddir; val >>= 8; __sz--; \
578 case 1: *dst = (val&0xff); dst += ddir; val >>= 8; __sz--; \
579 } \
580 } \
581 } while(0)
582
583 /* calculate a - *cp (carry) (store result in b), *cp is updated! */
584 #define SUBc(a, cp, b) do { \
585 byte __x = (a); \
586 byte __y = (__x - (*(cp))); \
587 (*cp) = (__y > __x); \
588 *(b) = ~__y; \
589 } while(0)
590
591 static int
fmt_int(byte * buf,Uint sz,Eterm val,Uint size,Uint flags)592 fmt_int(byte *buf, Uint sz, Eterm val, Uint size, Uint flags)
593 {
594 unsigned long offs;
595
596 offs = BIT_OFFSET(size);
597 if (is_small(val)) {
598 Sint v = signed_val(val);
599
600 ASSERT(size != 0); /* Tested by caller */
601 if (flags & BSF_LITTLE) { /* Little endian */
602 sz--;
603 COPY_VAL(buf,1,v,sz);
604 *buf = offs ? ((v << (8-offs)) & 0xff) : (v & 0xff);
605 } else { /* Big endian */
606 buf += (sz - 1);
607 if (offs) {
608 *buf-- = (v << (8-offs)) & 0xff;
609 sz--;
610 v >>= offs;
611 }
612 COPY_VAL(buf,-1,v,sz);
613 }
614 } else if (is_big(val)) {
615 int sign = big_sign(val);
616 Uint ds = big_size(val)*sizeof(ErtsDigit); /* number of digits bytes */
617 ErtsDigit* dp = big_v(val);
618 int n = MIN(sz,ds);
619
620 if (size == 0) {
621 return 0;
622 }
623 if (flags & BSF_LITTLE) {
624 sz -= n; /* pad with this amount */
625 if (sign) {
626 int c = 1;
627 while(n >= sizeof(ErtsDigit)) {
628 ErtsDigit d = *dp++;
629 int i;
630 for(i = 0; i < sizeof(ErtsDigit); ++i) {
631 SUBc((d&0xff), &c, buf);
632 buf++;
633 d >>= 8;
634 }
635 n -= sizeof(ErtsDigit);
636 }
637 if (n) {
638 ErtsDigit d = *dp;
639 do {
640 SUBc((d&0xff), &c, buf);
641 buf++;
642 d >>= 8;
643 } while (--n > 0);
644 }
645 /* pad */
646 while(sz--) {
647 SUBc(0, &c, buf);
648 buf++;
649 }
650 }
651 else {
652 while(n >= sizeof(ErtsDigit)) {
653 ErtsDigit d = *dp++;
654 int i;
655 for(i = 0; i < sizeof(ErtsDigit); ++i) {
656 *buf++ = (d & 0xff);
657 d >>= 8;
658 }
659 n -= sizeof(ErtsDigit);
660 }
661 if (n) {
662 ErtsDigit d = *dp;
663 do {
664 *buf++ = (d & 0xff);
665 d >>= 8;
666 } while (--n > 0);
667 }
668 /* pad */
669 while(sz) {
670 *buf++ = 0;
671 sz--;
672 }
673 }
674 /* adjust MSB!!! */
675 if (offs) {
676 buf--;
677 *buf <<= (8 - offs);
678 }
679 }
680 else { /* BIG ENDIAN */
681 ErtsDigit acc = 0;
682 ErtsDigit d;
683
684 buf += (sz - 1); /* end of buffer */
685 sz -= n; /* pad with this amount */
686 offs = offs ? (8-offs) : 0; /* shift offset */
687
688 if (sign) { /* SIGNED */
689 int c = 1;
690
691 while (n >= sizeof(ErtsDigit)) {
692 int i;
693
694 d = *dp++;
695 acc |= d << offs;
696 SUBc((acc&0xff), &c, buf);
697 buf--;
698 acc = d >> (8-offs);
699 for (i = 0; i < sizeof(ErtsDigit)-1; ++i) {
700 SUBc((acc&0xff), &c, buf);
701 buf--;
702 acc >>= 8;
703 }
704 n -= sizeof(ErtsDigit);
705 }
706 if (n) {
707 acc |= ((ErtsDigit)*dp << offs);
708 do {
709 SUBc((acc & 0xff), &c, buf);
710 buf--;
711 acc >>= 8;
712 } while (--n > 0);
713 }
714 /* pad */
715 while(sz--) {
716 SUBc((acc & 0xff), &c, buf);
717 buf--;
718 acc >>= 8;
719 }
720 }
721 else { /* UNSIGNED */
722 while (n >= sizeof(ErtsDigit)) {
723 int i;
724
725 d = *dp++;
726 acc |= d << offs;
727 *buf-- = acc;
728 acc = d >> (8-offs);
729 for (i = 0; i < sizeof(ErtsDigit)-1; ++i) {
730 *buf-- = acc;
731 acc >>= 8;
732 }
733 n -= sizeof(ErtsDigit);
734 }
735 if (n) {
736 acc |= ((ErtsDigit)*dp << offs);
737 do {
738 *buf-- = acc & 0xff;
739 acc >>= 8;
740 } while (--n > 0);
741 }
742 while (sz--) {
743 *buf-- = acc & 0xff;
744 acc >>= 8;
745 }
746 }
747 }
748 } else { /* Neither small nor big */
749 return -1;
750 }
751 return 0;
752 }
753
754 static void
need_byte_buf(ERL_BITS_PROTO_1 (int need))755 ERTS_INLINE need_byte_buf(ERL_BITS_PROTO_1(int need))
756 {
757 if (byte_buf_len < need) {
758 erts_atomic_add_nob(&bits_bufs_size, need - byte_buf_len);
759 byte_buf_len = need;
760 byte_buf = erts_realloc(ERTS_ALC_T_BITS_BUF, byte_buf, byte_buf_len);
761 }
762 }
763
764 int
erts_new_bs_put_integer(ERL_BITS_PROTO_3 (Eterm arg,Uint num_bits,unsigned flags))765 erts_new_bs_put_integer(ERL_BITS_PROTO_3(Eterm arg, Uint num_bits, unsigned flags))
766 {
767 Uint bin_offset = erts_bin_offset;
768 Uint bit_offset;
769 Uint b;
770 byte *iptr;
771
772 bit_offset = BIT_OFFSET(bin_offset);
773 if (is_small(arg)) {
774 Uint rbits = 8 - bit_offset;
775
776 if (num_bits == 0) {
777 return 1;
778 } else if (bit_offset + num_bits <= 8) {
779 /*
780 * All bits are in the same byte.
781 */
782 iptr = erts_current_bin+BYTE_OFFSET(bin_offset);
783 b = *iptr & (0xff << rbits);
784 b |= (signed_val(arg) & ((1 << num_bits)-1)) << (8-bit_offset-num_bits);
785 *iptr = b;
786 } else if (bit_offset == 0) {
787 /*
788 * More than one bit, starting at a byte boundary.
789 * That will be quite efficiently handled by fmt_int().
790 *
791 * (We know that fmt_int() can't fail here.)
792 */
793 (void) fmt_int(erts_current_bin+BYTE_OFFSET(bin_offset),
794 NBYTES(num_bits), arg, num_bits, flags);
795 } else if (flags & BSF_LITTLE) {
796 /*
797 * Can't handle unaligned little-endian in a simple way.
798 */
799 goto unaligned;
800 } else { /* Big endian */
801 /*
802 * Big-endian, more than one byte, but not aligned on a byte boundary.
803 * Handle the bits up to the next byte boundary specially,
804 * then let fmt_int() handle the rest.
805 */
806 Uint shift_count = num_bits - rbits;
807 Sint val = signed_val(arg);
808 iptr = erts_current_bin+BYTE_OFFSET(bin_offset);
809 b = *iptr & (0xff << rbits);
810
811 /*
812 * Shifting with a shift count greater than or equal to the word
813 * size may be a no-op (instead of 0 the result may be the unshifted
814 * value). Therefore, only do the shift and the OR if the shift count
815 * is less than the word size if the number is positive; if negative,
816 * we must simulate the sign extension.
817 */
818 if (shift_count < sizeof(Uint)*8) {
819 b |= (val >> shift_count) & ((1 << rbits) - 1);
820 } else if (val < 0) {
821 /* Simulate sign extension. */
822 b |= (-1) & ((1 << rbits) - 1);
823 }
824 *iptr++ = b;
825
826 /* fmt_int() can't fail here. */
827 (void) fmt_int(iptr, NBYTES(num_bits-rbits), arg,
828 num_bits-rbits, flags);
829 }
830 } else if (bit_offset == 0) {
831 /*
832 * Big number, aligned on a byte boundary. We can format the
833 * integer directly into the binary.
834 */
835 if (fmt_int(erts_current_bin+BYTE_OFFSET(bin_offset),
836 NBYTES(num_bits), arg, num_bits, flags) < 0) {
837 return 0;
838 }
839 } else {
840 unaligned:
841 /*
842 * Big number or small little-endian number, not byte-aligned,
843 * or not a number at all.
844 *
845 * We must format the number into a temporary buffer, and then
846 * copy that into the binary.
847 */
848 need_byte_buf(ERL_BITS_ARGS_1(NBYTES(num_bits)));
849 iptr = byte_buf;
850 if (fmt_int(iptr, NBYTES(num_bits), arg, num_bits, flags) < 0) {
851 return 0;
852 }
853 erts_copy_bits(iptr, 0, 1, erts_current_bin, bin_offset, 1, num_bits);
854 }
855 erts_bin_offset = bin_offset + num_bits;
856 return 1;
857 }
858
859 int
erts_bs_put_utf8(ERL_BITS_PROTO_1 (Eterm arg))860 erts_bs_put_utf8(ERL_BITS_PROTO_1(Eterm arg))
861 {
862 Uint bin_offset = erts_bin_offset;
863 Uint bit_offset;
864 Uint num_bits;
865 byte tmp_buf[4];
866 byte* dst;
867 Sint val;
868
869 if (is_not_small(arg)) {
870 return 0;
871 }
872 val = signed_val(arg);
873 if (val < 0) {
874 return 0;
875 }
876
877 if ((bit_offset = BIT_OFFSET(bin_offset)) == 0) {
878 /* We can write directly into the destination binary. */
879 dst = erts_current_bin+BYTE_OFFSET(bin_offset);
880 } else {
881 /* Unaligned destination binary. Must use a temporary buffer. */
882 dst = tmp_buf;
883 }
884 if (val < 0x80) {
885 dst[0] = val;
886 num_bits = 8;
887 } else if (val < 0x800) {
888 dst[0] = 0xC0 | (val >> 6);
889 dst[1] = 0x80 | (val & 0x3F);
890 num_bits = 16;
891 } else if (val < 0x10000UL) {
892 if (0xD800 <= val && val <= 0xDFFF) {
893 return 0;
894 }
895 dst[0] = 0xE0 | (val >> 12);
896 dst[1] = 0x80 | ((val >> 6) & 0x3F);
897 dst[2] = 0x80 | (val & 0x3F);
898 num_bits = 24;
899 } else if (val < 0x110000) {
900 dst[0] = 0xF0 | (val >> 18);
901 dst[1] = 0x80 | ((val >> 12) & 0x3F);
902 dst[2] = 0x80 | ((val >> 6) & 0x3F);
903 dst[3] = 0x80 | (val & 0x3F);
904 num_bits = 32;
905 } else {
906 return 0;
907 }
908
909 if (bin_offset != 0) {
910 erts_copy_bits(dst, 0, 1, erts_current_bin, bin_offset, 1, num_bits);
911 }
912
913 erts_bin_offset += num_bits;
914
915 return 1;
916 }
917
918 int
erts_bs_put_utf16(ERL_BITS_PROTO_2 (Eterm arg,Uint flags))919 erts_bs_put_utf16(ERL_BITS_PROTO_2(Eterm arg, Uint flags))
920 {
921 Uint bin_offset = erts_bin_offset;
922 Uint bit_offset;
923 Uint num_bits;
924 byte tmp_buf[4];
925 byte* dst;
926 Uint val;
927
928 if (is_not_small(arg)) {
929 return 0;
930 }
931 val = unsigned_val(arg);
932 if (val > 0x10FFFF || (0xD800 <= val && val <= 0xDFFF)) {
933 return 0;
934 }
935
936 if ((bit_offset = BIT_OFFSET(bin_offset)) == 0) {
937 /* We can write directly into the destination binary. */
938 dst = erts_current_bin+BYTE_OFFSET(bin_offset);
939 } else {
940 /* Unaligned destination binary. Must use a temporary buffer. */
941 dst = tmp_buf;
942 }
943
944 if (val < 0x10000UL) {
945 num_bits = 16;
946 if (flags & BSF_LITTLE) {
947 dst[0] = val;
948 dst[1] = val >> 8;
949 } else {
950 dst[0] = val >> 8;
951 dst[1] = val;
952 }
953 } else {
954 Uint16 w1, w2;
955
956 num_bits = 32;
957 val = val - 0x10000UL;
958 w1 = 0xD800 | (val >> 10);
959 w2 = 0xDC00 | (val & 0x3FF);
960 if (flags & BSF_LITTLE) {
961 dst[0] = w1;
962 dst[1] = w1 >> 8;
963 dst[2] = w2;
964 dst[3] = w2 >> 8;
965 } else {
966 dst[0] = w1 >> 8;
967 dst[1] = w1;
968 dst[2] = w2 >> 8;
969 dst[3] = w2;
970 }
971 }
972
973 if (bin_offset != 0) {
974 erts_copy_bits(dst, 0, 1, erts_current_bin, bin_offset, 1, num_bits);
975 }
976
977 erts_bin_offset += num_bits;
978 return 1;
979 }
980
981
982 int
erts_new_bs_put_binary(ERL_BITS_PROTO_2 (Eterm arg,Uint num_bits))983 erts_new_bs_put_binary(ERL_BITS_PROTO_2(Eterm arg, Uint num_bits))
984 {
985 byte *bptr;
986 Uint bitoffs;
987 Uint bitsize;
988
989 if (!is_binary(arg)) {
990 return 0;
991 }
992 ERTS_GET_BINARY_BYTES(arg, bptr, bitoffs, bitsize);
993 if (num_bits > 8*binary_size(arg)+bitsize) {
994 return 0;
995 }
996 copy_binary_to_buffer(erts_current_bin, erts_bin_offset, bptr, bitoffs, num_bits);
997 erts_bin_offset += num_bits;
998 return 1;
999 }
1000
1001 int
erts_new_bs_put_binary_all(ERL_BITS_PROTO_2 (Eterm arg,Uint unit))1002 erts_new_bs_put_binary_all(ERL_BITS_PROTO_2(Eterm arg, Uint unit))
1003 {
1004 byte *bptr;
1005 Uint bitoffs;
1006 Uint bitsize;
1007 Uint num_bits;
1008
1009 /*
1010 * This type test is not needed if the code was compiled with
1011 * an R12B or later compiler, since there would have been a
1012 * call to bit_size/1 or byte_size/1 that would have failed if
1013 * 'arg' was not a binary. However, in R11B and earlier releases,
1014 * size/1 was use for calculating the size of the binary, and
1015 * therefore 'arg' could be a tuple.
1016 */
1017 if (!is_binary(arg)) {
1018 return 0;
1019 }
1020
1021 ERTS_GET_BINARY_BYTES(arg, bptr, bitoffs, bitsize);
1022 num_bits = 8*binary_size(arg)+bitsize;
1023 if (unit == 8) {
1024 if (bitsize != 0) {
1025 return 0;
1026 }
1027 } else if (unit != 1 && num_bits % unit != 0) {
1028 return 0;
1029 }
1030 copy_binary_to_buffer(erts_current_bin, erts_bin_offset, bptr, bitoffs, num_bits);
1031 erts_bin_offset += num_bits;
1032 return 1;
1033 }
1034
1035 int
erts_new_bs_put_float(Process * c_p,Eterm arg,Uint num_bits,int flags)1036 erts_new_bs_put_float(Process *c_p, Eterm arg, Uint num_bits, int flags)
1037 {
1038 ERL_BITS_DEFINE_STATEP(c_p);
1039
1040 if (BIT_OFFSET(erts_bin_offset) == 0) {
1041 Uint32 a;
1042 Uint32 b;
1043
1044 if (num_bits == 64) {
1045 union {
1046 double f64;
1047 Uint32 i32[2];
1048 } u;
1049
1050 if (is_float(arg)) {
1051 FloatDef *fdp = (FloatDef*)(float_val(arg) + 1);
1052 #ifdef DOUBLE_MIDDLE_ENDIAN
1053 a = fdp->fw[1];
1054 b = fdp->fw[0];
1055 #else
1056 a = fdp->fw[0];
1057 b = fdp->fw[1];
1058 #endif
1059 } else if (is_small(arg)) {
1060 u.f64 = (double) signed_val(arg);
1061 #ifdef DOUBLE_MIDDLE_ENDIAN
1062 a = u.i32[1];
1063 b = u.i32[0];
1064 #else
1065 a = u.i32[0];
1066 b = u.i32[1];
1067 #endif
1068 } else if (is_big(arg)) {
1069 if (big_to_double(arg, &u.f64) < 0) {
1070 return 0;
1071 }
1072 #ifdef DOUBLE_MIDDLE_ENDIAN
1073 a = u.i32[1];
1074 b = u.i32[0];
1075 #else
1076 a = u.i32[0];
1077 b = u.i32[1];
1078 #endif
1079 } else {
1080 return 0;
1081 }
1082 } else if (num_bits == 32) {
1083 union {
1084 float f32;
1085 Uint32 i32;
1086 } u;
1087
1088 b = 0;
1089 if (is_float(arg)) {
1090 FloatDef f;
1091 GET_DOUBLE(arg, f);
1092 ERTS_FP_CHECK_INIT(c_p);
1093 u.f32 = f.fd;
1094 ERTS_FP_ERROR(c_p,u.f32,;);
1095 a = u.i32;
1096 } else if (is_small(arg)) {
1097 u.f32 = (float) signed_val(arg);
1098 a = u.i32;
1099 } else if (is_big(arg)) {
1100 double f64;
1101 if (big_to_double(arg, &f64) < 0) {
1102 return 0;
1103 }
1104 ERTS_FP_CHECK_INIT(c_p);
1105 u.f32 = (float) f64;
1106 ERTS_FP_ERROR(c_p,u.f32,;);
1107 a = u.i32;
1108 } else {
1109 return 0;
1110 }
1111 } else {
1112 return 0;
1113 }
1114
1115 if (BIT_IS_MACHINE_ENDIAN(flags)) {
1116 byte* t = erts_current_bin+BYTE_OFFSET(erts_bin_offset);
1117 #ifdef WORDS_BIGENDIAN
1118 t[0] = a >> 24;
1119 t[1] = a >> 16;
1120 t[2] = a >> 8;
1121 t[3] = a;
1122 if (num_bits == 64) {
1123 t[4] = b >> 24;
1124 t[5] = b >> 16;
1125 t[6] = b >> 8;
1126 t[7] = b;
1127 }
1128 #else
1129 t[3] = a >> 24;
1130 t[2] = a >> 16;
1131 t[1] = a >> 8;
1132 t[0] = a;
1133 if (num_bits == 64) {
1134 t[7] = b >> 24;
1135 t[6] = b >> 16;
1136 t[5] = b >> 8;
1137 t[4] = b;
1138 }
1139 #endif
1140 } else {
1141 byte* t = erts_current_bin+BYTE_OFFSET(erts_bin_offset) + NBYTES(num_bits);
1142 #ifdef WORDS_BIGENDIAN
1143 t[-1] = a >> 24;
1144 t[-2] = a >> 16;
1145 t[-3] = a >> 8;
1146 t[-4] = a;
1147 if (num_bits == 64) {
1148 t[-5] = b >> 24;
1149 t[-6] = b >> 16;
1150 t[-7] = b >> 8;
1151 t[-8] = b;
1152 }
1153 #else
1154 t[-1] = a;
1155 t[-2] = a >> 8;
1156 t[-3] = a >> 16;
1157 t[-4] = a >> 24;
1158 if (num_bits == 64) {
1159 t[-5] = b;
1160 t[-6] = b >> 8;
1161 t[-7] = b >> 16;
1162 t[-8] = b >> 24;
1163 }
1164 #endif
1165 }
1166 } else {
1167 byte *bptr;
1168 double f64;
1169 float f32;
1170 #ifdef DOUBLE_MIDDLE_ENDIAN
1171 FloatDef fbuf, ftmp;
1172 #endif
1173
1174 if (num_bits == 64) {
1175 if (is_float(arg)) {
1176 #ifdef DOUBLE_MIDDLE_ENDIAN
1177 FloatDef *fdp = (FloatDef*)(float_val(arg) + 1);
1178 ftmp = *fdp;
1179 #else
1180 bptr = (byte *) (float_val(arg) + 1);
1181 #endif
1182 } else if (is_small(arg)) {
1183 f64 = (double) signed_val(arg);
1184 #ifdef DOUBLE_MIDDLE_ENDIAN
1185 ftmp.fd = f64;
1186 #else
1187 bptr = (byte *) &f64;
1188 #endif
1189 } else if (is_big(arg)) {
1190 if (big_to_double(arg, &f64) < 0) {
1191 return 0;
1192 }
1193 #ifdef DOUBLE_MIDDLE_ENDIAN
1194 ftmp.fd = f64;
1195 #else
1196 bptr = (byte *) &f64;
1197 #endif
1198 } else {
1199 return 0;
1200 }
1201 #ifdef DOUBLE_MIDDLE_ENDIAN
1202 fbuf.fw[0] = ftmp.fw[1];
1203 fbuf.fw[1] = ftmp.fw[0];
1204 bptr = fbuf.fb;
1205 #endif
1206 } else if (num_bits == 32) {
1207 if (is_float(arg)) {
1208 FloatDef f;
1209 GET_DOUBLE(arg, f);
1210 ERTS_FP_CHECK_INIT(c_p);
1211 f32 = f.fd;
1212 ERTS_FP_ERROR(c_p,f32,;);
1213 bptr = (byte *) &f32;
1214 } else if (is_small(arg)) {
1215 f32 = (float) signed_val(arg);
1216 bptr = (byte *) &f32;
1217 } else if (is_big(arg)) {
1218 if (big_to_double(arg, &f64) < 0) {
1219 return 0;
1220 }
1221 ERTS_FP_CHECK_INIT(c_p);
1222 f32 = (float) f64;
1223 ERTS_FP_ERROR(c_p,f32,;);
1224 bptr = (byte *) &f32;
1225 } else {
1226 return 0;
1227 }
1228 } else {
1229 return 0;
1230 }
1231 if (BIT_IS_MACHINE_ENDIAN(flags)) {
1232 erts_copy_bits(bptr, 0, 1,
1233 erts_current_bin,
1234 erts_bin_offset, 1, num_bits);
1235 } else {
1236 erts_copy_bits(bptr+NBYTES(num_bits)-1, 0, -1,
1237 erts_current_bin, erts_bin_offset, 1,
1238 num_bits);
1239 }
1240 }
1241 erts_bin_offset += num_bits;
1242 return 1;
1243 }
1244
1245 void
erts_new_bs_put_string(ERL_BITS_PROTO_2 (byte * iptr,Uint num_bytes))1246 erts_new_bs_put_string(ERL_BITS_PROTO_2(byte* iptr, Uint num_bytes))
1247 {
1248 if (BIT_OFFSET(erts_bin_offset) != 0) {
1249 erts_copy_bits(iptr, 0, 1, erts_current_bin, erts_bin_offset, 1, num_bytes*8);
1250 } else {
1251 sys_memcpy(erts_current_bin+BYTE_OFFSET(erts_bin_offset), iptr, num_bytes);
1252 }
1253 erts_bin_offset += num_bytes*8;
1254 }
1255
1256 Eterm
erts_bs_append(Process * c_p,Eterm * reg,Uint live,Eterm build_size_term,Uint extra_words,Uint unit)1257 erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term,
1258 Uint extra_words, Uint unit)
1259 {
1260 Eterm bin; /* Given binary */
1261 Eterm* ptr;
1262 Eterm hdr;
1263 ErlSubBin* sb;
1264 ProcBin* pb;
1265 Binary* binp;
1266 Uint heap_need;
1267 Uint build_size_in_bits;
1268 Uint used_size_in_bits;
1269 Uint unsigned_bits;
1270 ERL_BITS_DEFINE_STATEP(c_p);
1271
1272 /*
1273 * Check and untag the requested build size.
1274 */
1275 if (is_small(build_size_term)) {
1276 Sint signed_bits = signed_val(build_size_term);
1277 if (signed_bits < 0) {
1278 goto badarg;
1279 }
1280 build_size_in_bits = (Uint) signed_bits;
1281 } else if (term_to_Uint(build_size_term, &unsigned_bits)) {
1282 build_size_in_bits = unsigned_bits;
1283 } else {
1284 c_p->freason = unsigned_bits;
1285 return THE_NON_VALUE;
1286 }
1287
1288 /*
1289 * Check the binary argument.
1290 */
1291 bin = reg[live];
1292 if (!is_boxed(bin)) {
1293 badarg:
1294 c_p->freason = BADARG;
1295 return THE_NON_VALUE;
1296 }
1297 ptr = boxed_val(bin);
1298 hdr = *ptr;
1299 if (!is_binary_header(hdr)) {
1300 goto badarg;
1301 }
1302 if (hdr != HEADER_SUB_BIN) {
1303 goto not_writable;
1304 }
1305 sb = (ErlSubBin *) ptr;
1306 if (!sb->is_writable) {
1307 goto not_writable;
1308 }
1309 pb = (ProcBin *) boxed_val(sb->orig);
1310 ASSERT(pb->thing_word == HEADER_PROC_BIN);
1311 if ((pb->flags & PB_IS_WRITABLE) == 0) {
1312 goto not_writable;
1313 }
1314
1315 /*
1316 * OK, the binary is writable.
1317 */
1318
1319 erts_bin_offset = 8*sb->size + sb->bitsize;
1320 if (unit > 1) {
1321 if ((unit == 8 && (erts_bin_offset & 7) != 0) ||
1322 (erts_bin_offset % unit) != 0) {
1323 goto badarg;
1324 }
1325 }
1326
1327 if (build_size_in_bits == 0) {
1328 if (c_p->stop - c_p->htop < extra_words) {
1329 (void) erts_garbage_collect(c_p, extra_words, reg, live+1);
1330 bin = reg[live];
1331 }
1332 return bin;
1333 }
1334
1335 if((ERTS_UINT_MAX - build_size_in_bits) < erts_bin_offset) {
1336 c_p->freason = SYSTEM_LIMIT;
1337 return THE_NON_VALUE;
1338 }
1339
1340 used_size_in_bits = erts_bin_offset + build_size_in_bits;
1341
1342 sb->is_writable = 0; /* Make sure that no one else can write. */
1343 pb->size = NBYTES(used_size_in_bits);
1344 pb->flags |= PB_ACTIVE_WRITER;
1345
1346 /*
1347 * Reallocate the binary if it is too small.
1348 */
1349 binp = pb->val;
1350 if (binp->orig_size < pb->size) {
1351 Uint new_size = 2*pb->size;
1352 binp = erts_bin_realloc(binp, new_size);
1353 pb->val = binp;
1354 pb->bytes = (byte *) binp->orig_bytes;
1355 }
1356 erts_current_bin = pb->bytes;
1357
1358 /*
1359 * Allocate heap space and build a new sub binary.
1360 */
1361 reg[live] = sb->orig;
1362 heap_need = ERL_SUB_BIN_SIZE + extra_words;
1363 if (c_p->stop - c_p->htop < heap_need) {
1364 (void) erts_garbage_collect(c_p, heap_need, reg, live+1);
1365 }
1366 sb = (ErlSubBin *) c_p->htop;
1367 c_p->htop += ERL_SUB_BIN_SIZE;
1368 sb->thing_word = HEADER_SUB_BIN;
1369 sb->size = BYTE_OFFSET(used_size_in_bits);
1370 sb->bitsize = BIT_OFFSET(used_size_in_bits);
1371 sb->offs = 0;
1372 sb->bitoffs = 0;
1373 sb->is_writable = 1;
1374 sb->orig = reg[live];
1375
1376 return make_binary(sb);
1377
1378 /*
1379 * The binary is not writable. We must create a new writable binary and
1380 * copy the old contents of the binary.
1381 */
1382 not_writable:
1383 {
1384 Uint used_size_in_bytes; /* Size of old binary + data to be built */
1385 Uint bin_size;
1386 Binary* bptr;
1387 byte* src_bytes;
1388 Uint bitoffs;
1389 Uint bitsize;
1390 Eterm* hp;
1391
1392 /*
1393 * Allocate heap space.
1394 */
1395 heap_need = PROC_BIN_SIZE + ERL_SUB_BIN_SIZE + extra_words;
1396 if (c_p->stop - c_p->htop < heap_need) {
1397 (void) erts_garbage_collect(c_p, heap_need, reg, live+1);
1398 bin = reg[live];
1399 }
1400 hp = c_p->htop;
1401
1402 /*
1403 * Calculate sizes. The size of the new binary, is the sum of the
1404 * build size and the size of the old binary. Allow some room
1405 * for growing.
1406 */
1407 ERTS_GET_BINARY_BYTES(bin, src_bytes, bitoffs, bitsize);
1408 erts_bin_offset = 8*binary_size(bin) + bitsize;
1409 if (unit > 1) {
1410 if ((unit == 8 && (erts_bin_offset & 7) != 0) ||
1411 (erts_bin_offset % unit) != 0) {
1412 goto badarg;
1413 }
1414 }
1415
1416 if (build_size_in_bits == 0) {
1417 return bin;
1418 }
1419
1420 if((ERTS_UINT_MAX - build_size_in_bits) < erts_bin_offset) {
1421 c_p->freason = SYSTEM_LIMIT;
1422 return THE_NON_VALUE;
1423 }
1424
1425 used_size_in_bits = erts_bin_offset + build_size_in_bits;
1426 used_size_in_bytes = NBYTES(used_size_in_bits);
1427
1428 if(used_size_in_bits < (ERTS_UINT_MAX / 2)) {
1429 bin_size = 2 * used_size_in_bytes;
1430 } else {
1431 bin_size = NBYTES(ERTS_UINT_MAX);
1432 }
1433
1434 bin_size = (bin_size < 256) ? 256 : bin_size;
1435
1436 /*
1437 * Allocate the binary data struct itself.
1438 */
1439 bptr = erts_bin_nrml_alloc(bin_size);
1440 erts_current_bin = (byte *) bptr->orig_bytes;
1441
1442 /*
1443 * Now allocate the ProcBin on the heap.
1444 */
1445 pb = (ProcBin *) hp;
1446 hp += PROC_BIN_SIZE;
1447 pb->thing_word = HEADER_PROC_BIN;
1448 pb->size = used_size_in_bytes;
1449 pb->next = MSO(c_p).first;
1450 MSO(c_p).first = (struct erl_off_heap_header*)pb;
1451 pb->val = bptr;
1452 pb->bytes = (byte*) bptr->orig_bytes;
1453 pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER;
1454 OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm));
1455
1456 /*
1457 * Now allocate the sub binary and set its size to include the
1458 * data about to be built.
1459 */
1460 sb = (ErlSubBin *) hp;
1461 hp += ERL_SUB_BIN_SIZE;
1462 sb->thing_word = HEADER_SUB_BIN;
1463 sb->size = BYTE_OFFSET(used_size_in_bits);
1464 sb->bitsize = BIT_OFFSET(used_size_in_bits);
1465 sb->offs = 0;
1466 sb->bitoffs = 0;
1467 sb->is_writable = 1;
1468 sb->orig = make_binary(pb);
1469
1470 c_p->htop = hp;
1471
1472 /*
1473 * Now copy the data into the binary.
1474 */
1475 copy_binary_to_buffer(erts_current_bin, 0, src_bytes, bitoffs, erts_bin_offset);
1476
1477 return make_binary(sb);
1478 }
1479 }
1480
1481 Eterm
erts_bs_private_append(Process * p,Eterm bin,Eterm build_size_term,Uint unit)1482 erts_bs_private_append(Process* p, Eterm bin, Eterm build_size_term, Uint unit)
1483 {
1484 Eterm* ptr;
1485 ErlSubBin* sb;
1486 ProcBin* pb;
1487 Binary* binp;
1488 Uint build_size_in_bits;
1489 Uint pos_in_bits_after_build;
1490 Uint unsigned_bits;
1491 ERL_BITS_DEFINE_STATEP(p);
1492
1493 /*
1494 * Check and untag the requested build size.
1495 */
1496 if (is_small(build_size_term)) {
1497 Sint signed_bits = signed_val(build_size_term);
1498 if (signed_bits < 0) {
1499 p->freason = BADARG;
1500 return THE_NON_VALUE;
1501 }
1502 build_size_in_bits = (Uint) signed_bits;
1503 } else if (term_to_Uint(build_size_term, &unsigned_bits)) {
1504 build_size_in_bits = unsigned_bits;
1505 } else {
1506 p->freason = unsigned_bits;
1507 return THE_NON_VALUE;
1508 }
1509
1510 ptr = boxed_val(bin);
1511 ASSERT(*ptr == HEADER_SUB_BIN);
1512
1513 sb = (ErlSubBin *) ptr;
1514 ASSERT(sb->is_writable);
1515
1516 pb = (ProcBin *) boxed_val(sb->orig);
1517 ASSERT(pb->thing_word == HEADER_PROC_BIN);
1518
1519 /*
1520 * Calculate new size in bytes.
1521 */
1522 erts_bin_offset = 8*sb->size + sb->bitsize;
1523
1524 if((ERTS_UINT_MAX - build_size_in_bits) < erts_bin_offset) {
1525 p->freason = SYSTEM_LIMIT;
1526 return THE_NON_VALUE;
1527 }
1528
1529 pos_in_bits_after_build = erts_bin_offset + build_size_in_bits;
1530 pb->size = (pos_in_bits_after_build+7) >> 3;
1531 pb->flags |= PB_ACTIVE_WRITER;
1532
1533 /*
1534 * Reallocate the binary if it is too small.
1535 */
1536 binp = pb->val;
1537 if (binp->orig_size < pb->size) {
1538 Uint new_size = 2*pb->size;
1539
1540 if (pb->flags & PB_IS_WRITABLE) {
1541 /*
1542 * This is the normal case - the binary is writable.
1543 * There are no other references to the binary, so it
1544 * is safe to reallocate it.
1545 */
1546 binp = erts_bin_realloc(binp, new_size);
1547 pb->val = binp;
1548 pb->bytes = (byte *) binp->orig_bytes;
1549 } else {
1550 /*
1551 * The binary is NOT writable. The only way that is
1552 * supposed to happen if is call trace has been turned
1553 * on. That means that a trace process now has (or have
1554 * had) a reference to the binary, so we are not allowed
1555 * to reallocate the binary. Instead, we must allocate a new
1556 * binary and copy the contents of the old binary into it.
1557 */
1558 Binary* bptr = erts_bin_nrml_alloc(new_size);
1559 sys_memcpy(bptr->orig_bytes, binp->orig_bytes, binp->orig_size);
1560 pb->flags |= PB_IS_WRITABLE | PB_ACTIVE_WRITER;
1561 pb->val = bptr;
1562 pb->bytes = (byte *) bptr->orig_bytes;
1563 erts_bin_release(binp);
1564 }
1565 }
1566 erts_current_bin = pb->bytes;
1567
1568 sb->size = pos_in_bits_after_build >> 3;
1569 sb->bitsize = pos_in_bits_after_build & 7;
1570 return bin;
1571 }
1572
1573 Eterm
erts_bs_init_writable(Process * p,Eterm sz)1574 erts_bs_init_writable(Process* p, Eterm sz)
1575 {
1576 Uint bin_size = 1024;
1577 Uint heap_need;
1578 Binary* bptr;
1579 ProcBin* pb;
1580 ErlSubBin* sb;
1581 Eterm* hp;
1582
1583 if (is_small(sz)) {
1584 Sint s = signed_val(sz);
1585 if (s >= 0) {
1586 bin_size = (Uint) s;
1587 }
1588 }
1589
1590 /*
1591 * Allocate heap space.
1592 */
1593 heap_need = PROC_BIN_SIZE + ERL_SUB_BIN_SIZE;
1594 if (p->stop - p->htop < heap_need) {
1595 (void) erts_garbage_collect(p, heap_need, NULL, 0);
1596 }
1597 hp = p->htop;
1598
1599 /*
1600 * Allocate the binary data struct itself.
1601 */
1602 bptr = erts_bin_nrml_alloc(bin_size);
1603
1604 /*
1605 * Now allocate the ProcBin on the heap.
1606 */
1607 pb = (ProcBin *) hp;
1608 hp += PROC_BIN_SIZE;
1609 pb->thing_word = HEADER_PROC_BIN;
1610 pb->size = 0;
1611 pb->next = MSO(p).first;
1612 MSO(p).first = (struct erl_off_heap_header*) pb;
1613 pb->val = bptr;
1614 pb->bytes = (byte*) bptr->orig_bytes;
1615 pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER;
1616 OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
1617
1618 /*
1619 * Now allocate the sub binary.
1620 */
1621 sb = (ErlSubBin *) hp;
1622 hp += ERL_SUB_BIN_SIZE;
1623 sb->thing_word = HEADER_SUB_BIN;
1624 sb->size = 0;
1625 sb->offs = 0;
1626 sb->bitsize = 0;
1627 sb->bitoffs = 0;
1628 sb->is_writable = 1;
1629 sb->orig = make_binary(pb);
1630
1631 p->htop = hp;
1632 return make_binary(sb);
1633 }
1634
1635 void
erts_emasculate_writable_binary(ProcBin * pb)1636 erts_emasculate_writable_binary(ProcBin* pb)
1637 {
1638 Binary* binp;
1639 Uint unused;
1640
1641 pb->flags = 0;
1642 binp = pb->val;
1643 ASSERT(binp->orig_size >= pb->size);
1644 unused = binp->orig_size - pb->size;
1645 /* Our allocators are 8 byte aligned, i.e., shrinking with
1646 less than 8 bytes will have no real effect */
1647 if (unused >= 8) {
1648 binp = erts_bin_realloc(binp, pb->size);
1649 pb->val = binp;
1650 pb->bytes = (byte *) binp->orig_bytes;
1651 }
1652 }
1653
1654 Uint32
erts_bs_get_unaligned_uint32(ErlBinMatchBuffer * mb)1655 erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb)
1656 {
1657 Uint bytes;
1658 Uint offs;
1659 byte bigbuf[4];
1660 byte* LSB;
1661 byte* MSB;
1662
1663 CHECK_MATCH_BUFFER(mb);
1664 ASSERT((mb->offset & 7) != 0);
1665 ASSERT(mb->size - mb->offset >= 32);
1666
1667 bytes = 4;
1668 offs = 0;
1669
1670 LSB = bigbuf;
1671 MSB = LSB + bytes - 1;
1672
1673 *MSB = 0;
1674 erts_copy_bits(mb->base, mb->offset, 1, MSB, offs, -1, 32);
1675 return LSB[0] | (LSB[1]<<8) | (LSB[2]<<16) | (LSB[3]<<24);
1676 }
1677
1678 static void
erts_align_utf8_bytes(ErlBinMatchBuffer * mb,byte * buf)1679 erts_align_utf8_bytes(ErlBinMatchBuffer* mb, byte* buf)
1680 {
1681 Uint bits = mb->size - mb->offset;
1682
1683 /*
1684 * Copy up to 4 bytes into the supplied buffer.
1685 */
1686
1687 ASSERT(bits >= 8);
1688 if (bits <= 15) {
1689 bits = 8;
1690 } else if (bits >= 32) {
1691 bits = 32;
1692 } else if (bits >= 24) {
1693 bits = 24;
1694 } else {
1695 bits = 16;
1696 }
1697 erts_copy_bits(mb->base, mb->offset, 1, buf, 0, 1, bits);
1698 }
1699
1700 Eterm
erts_bs_get_utf8(ErlBinMatchBuffer * mb)1701 erts_bs_get_utf8(ErlBinMatchBuffer* mb)
1702 {
1703 Eterm result;
1704 Uint remaining_bits;
1705 byte* pos;
1706 byte tmp_buf[4];
1707 Eterm a, b, c;
1708
1709 /*
1710 * Number of trailing bytes for each value of the first byte.
1711 */
1712 static const byte erts_trailing_bytes_for_utf8[256] = {
1713 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
1714 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
1715 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
1716 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
1717 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
1718 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
1719 9,9,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1720 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,9,9,9,9,9,9,9,9
1721 };
1722
1723 CHECK_MATCH_BUFFER(mb);
1724
1725 if ((remaining_bits = mb->size - mb->offset) < 8) {
1726 return THE_NON_VALUE;
1727 }
1728 if (BIT_OFFSET(mb->offset) == 0) {
1729 pos = mb->base + BYTE_OFFSET(mb->offset);
1730 } else {
1731 erts_align_utf8_bytes(mb, tmp_buf);
1732 pos = tmp_buf;
1733 }
1734 result = pos[0];
1735 switch (erts_trailing_bytes_for_utf8[result]) {
1736 case 0:
1737 /* One byte only */
1738 mb->offset += 8;
1739 break;
1740 case 1:
1741 /* Two bytes */
1742 if (remaining_bits < 16) {
1743 return THE_NON_VALUE;
1744 }
1745 a = pos[1];
1746 if ((a & 0xC0) != 0x80) {
1747 return THE_NON_VALUE;
1748 }
1749 result = (result << 6) + a - (Eterm) 0x00003080UL;
1750 mb->offset += 16;
1751 break;
1752 case 2:
1753 /* Three bytes */
1754 if (remaining_bits < 24) {
1755 return THE_NON_VALUE;
1756 }
1757 a = pos[1];
1758 b = pos[2];
1759 if ((a & 0xC0) != 0x80 || (b & 0xC0) != 0x80 ||
1760 (result == 0xE0 && a < 0xA0)) {
1761 return THE_NON_VALUE;
1762 }
1763 result = (((result << 6) + a) << 6) + b - (Eterm) 0x000E2080UL;
1764 if (0xD800 <= result && result <= 0xDFFF) {
1765 return THE_NON_VALUE;
1766 }
1767 mb->offset += 24;
1768 break;
1769 case 3:
1770 /* Four bytes */
1771 if (remaining_bits < 32) {
1772 return THE_NON_VALUE;
1773 }
1774 a = pos[1];
1775 b = pos[2];
1776 c = pos[3];
1777 if ((a & 0xC0) != 0x80 || (b & 0xC0) != 0x80 ||
1778 (c & 0xC0) != 0x80 ||
1779 (result == 0xF0 && a < 0x90)) {
1780 return THE_NON_VALUE;
1781 }
1782 result = (((((result << 6) + a) << 6) + b) << 6) +
1783 c - (Eterm) 0x03C82080UL;
1784 if (result > 0x10FFFF) {
1785 return THE_NON_VALUE;
1786 }
1787 mb->offset += 32;
1788 break;
1789 default:
1790 return THE_NON_VALUE;
1791 }
1792 return make_small(result);
1793 }
1794
1795 Eterm
erts_bs_get_utf16(ErlBinMatchBuffer * mb,Uint flags)1796 erts_bs_get_utf16(ErlBinMatchBuffer* mb, Uint flags)
1797 {
1798 Uint bit_offset;
1799 Uint num_bits = mb->size - mb->offset;
1800 byte* src;
1801 byte tmp_buf[4];
1802 Uint16 w1;
1803 Uint16 w2;
1804
1805 if (num_bits < 16) {
1806 return THE_NON_VALUE;
1807 }
1808
1809 CHECK_MATCH_BUFFER(mb);
1810 /*
1811 * Set up the pointer to the source bytes.
1812 */
1813 if ((bit_offset = BIT_OFFSET(mb->offset)) == 0) {
1814 /* We can access the binary directly because the bytes are aligned. */
1815 src = mb->base + BYTE_OFFSET(mb->offset);
1816 } else {
1817 /*
1818 * We must copy the data to a temporary buffer. If possible,
1819 * get 4 bytes, otherwise two bytes.
1820 */
1821 Uint n = num_bits < 32 ? 16 : 32;
1822 erts_copy_bits(mb->base, mb->offset, 1, tmp_buf, 0, 1, n);
1823 src = tmp_buf;
1824 }
1825
1826 /*
1827 * Get the first (and maybe only) 16-bit word. See if we are done.
1828 */
1829 if (flags & BSF_LITTLE) {
1830 w1 = src[0] | (src[1] << 8);
1831 } else {
1832 w1 = (src[0] << 8) | src[1];
1833 }
1834 if (w1 < 0xD800 || w1 > 0xDFFF) {
1835 mb->offset += 16;
1836 return make_small(w1);
1837 } else if (w1 > 0xDBFF) {
1838 return THE_NON_VALUE;
1839 }
1840
1841 /*
1842 * Get the second 16-bit word and combine it with the first.
1843 */
1844 if (num_bits < 32) {
1845 return THE_NON_VALUE;
1846 } else if (flags & BSF_LITTLE) {
1847 w2 = src[2] | (src[3] << 8);
1848 } else {
1849 w2 = (src[2] << 8) | src[3];
1850 }
1851 if (!(0xDC00 <= w2 && w2 <= 0xDFFF)) {
1852 return THE_NON_VALUE;
1853 }
1854 mb->offset += 32;
1855 return make_small((((w1 & 0x3FF) << 10) | (w2 & 0x3FF)) + 0x10000UL);
1856 }
1857
1858 static byte
get_bit(byte b,size_t offs)1859 get_bit(byte b, size_t offs)
1860 {
1861 return (b >> (7-offs)) & 1;
1862 }
1863
1864 int
erts_cmp_bits(byte * a_ptr,size_t a_offs,byte * b_ptr,size_t b_offs,size_t size)1865 erts_cmp_bits(byte* a_ptr, size_t a_offs, byte* b_ptr, size_t b_offs, size_t size)
1866 {
1867 byte a;
1868 byte b;
1869 byte a_bit;
1870 byte b_bit;
1871 Uint lshift;
1872 Uint rshift;
1873 int cmp;
1874
1875 ASSERT(a_offs < 8 && b_offs < 8);
1876
1877 if (size == 0)
1878 return 0;
1879
1880 if (((a_offs | b_offs | size) & 7) == 0) {
1881 int byte_size = size >> 3;
1882 return sys_memcmp(a_ptr, b_ptr, byte_size);
1883 }
1884
1885 /* Compare bit by bit until a_ptr is aligned on byte boundary */
1886 a = *a_ptr++;
1887 b = *b_ptr++;
1888 if (a_offs) {
1889 for (;;) {
1890 a_bit = get_bit(a, a_offs);
1891 b_bit = get_bit(b, b_offs);
1892 if ((cmp = (a_bit-b_bit)) != 0) {
1893 return cmp;
1894 }
1895 if (--size == 0)
1896 return 0;
1897
1898 b_offs++;
1899 if (b_offs == 8) {
1900 b_offs = 0;
1901 b = *b_ptr++;
1902 }
1903 a_offs++;
1904 if (a_offs == 8) {
1905 a_offs = 0;
1906 a = *a_ptr++;
1907 break;
1908 }
1909 }
1910 }
1911
1912 /* Compare byte by byte as long as at least 8 bits remain */
1913 if (size >= 8) {
1914 lshift = b_offs;
1915 rshift = 8 - lshift;
1916 for (;;) {
1917 byte b_cmp = (b << lshift);
1918 b = *b_ptr++;
1919 b_cmp |= b >> rshift;
1920 if ((cmp = (a - b_cmp)) != 0) {
1921 return cmp;
1922 }
1923 size -= 8;
1924 if (size < 8)
1925 break;
1926 a = *a_ptr++;
1927 }
1928
1929 if (size == 0)
1930 return 0;
1931 a = *a_ptr++;
1932 }
1933
1934 /* Compare the remaining bits bit by bit */
1935 if (size > 0) {
1936 for (;;) {
1937 a_bit = get_bit(a, a_offs);
1938 b_bit = get_bit(b, b_offs);
1939 if ((cmp = (a_bit-b_bit)) != 0) {
1940 return cmp;
1941 }
1942 if (--size == 0)
1943 return 0;
1944
1945 a_offs++;
1946 ASSERT(a_offs < 8);
1947
1948 b_offs++;
1949 if (b_offs == 8) {
1950 b_offs = 0;
1951 b = *b_ptr++;
1952 }
1953 }
1954 }
1955
1956 return 0;
1957 }
1958
1959 /*
1960 * The basic bit copy operation. Copies n bits from the source buffer to
1961 * the destination buffer. Depending on the directions, it can reverse the
1962 * copied bits.
1963 */
1964
1965
1966 void
erts_copy_bits(byte * src,size_t soffs,int sdir,byte * dst,size_t doffs,int ddir,size_t n)1967 erts_copy_bits(byte* src, /* Base pointer to source. */
1968 size_t soffs, /* Bit offset for source relative to src. */
1969 int sdir, /* Direction: 1 (forward) or -1 (backward). */
1970 byte* dst, /* Base pointer to destination. */
1971 size_t doffs, /* Bit offset for destination relative to dst. */
1972 int ddir, /* Direction: 1 (forward) or -1 (backward). */
1973 size_t n) /* Number of bits to copy. */
1974 {
1975 Uint lmask;
1976 Uint rmask;
1977 Uint count;
1978 Uint deoffs;
1979
1980 if (n == 0) {
1981 return;
1982 }
1983
1984 src += sdir*BYTE_OFFSET(soffs);
1985 dst += ddir*BYTE_OFFSET(doffs);
1986 soffs = BIT_OFFSET(soffs);
1987 doffs = BIT_OFFSET(doffs);
1988 deoffs = BIT_OFFSET(doffs+n);
1989 lmask = (doffs) ? MAKE_MASK(8-doffs) : 0;
1990 rmask = (deoffs) ? (MAKE_MASK(deoffs)<<(8-deoffs)) : 0;
1991
1992 /*
1993 * Take care of the case that all bits are in the same byte.
1994 */
1995
1996 if (doffs+n < 8) { /* All bits are in the same byte */
1997 lmask = (lmask & rmask) ? (lmask & rmask) : (lmask | rmask);
1998
1999 if (soffs == doffs) {
2000 *dst = MASK_BITS(*src,*dst,lmask);
2001 } else if (soffs > doffs) {
2002 Uint bits = (*src << (soffs-doffs));
2003 if (soffs+n > 8) {
2004 src += sdir;
2005 bits |= (*src >> (8-(soffs-doffs)));
2006 }
2007 *dst = MASK_BITS(bits,*dst,lmask);
2008 } else {
2009 *dst = MASK_BITS((*src >> (doffs-soffs)),*dst,lmask);
2010 }
2011 return; /* We are done! */
2012 }
2013
2014 /*
2015 * At this point, we know that the bits are in 2 or more bytes.
2016 */
2017
2018 count = ((lmask) ? (n - (8 - doffs)) : n) >> 3;
2019
2020 if (soffs == doffs) {
2021 /*
2022 * The bits are aligned in the same way. We can just copy the bytes
2023 * (except for the first and last bytes). Note that the directions
2024 * might be different, so we can't just use memcpy().
2025 */
2026
2027 if (lmask) {
2028 *dst = MASK_BITS(*src, *dst, lmask);
2029 dst += ddir;
2030 src += sdir;
2031 }
2032
2033 while (count--) {
2034 *dst = *src;
2035 dst += ddir;
2036 src += sdir;
2037 }
2038
2039 if (rmask) {
2040 *dst = MASK_BITS(*src,*dst,rmask);
2041 }
2042 } else {
2043 Uint bits;
2044 Uint bits1;
2045 Uint rshift;
2046 Uint lshift;
2047
2048 /*
2049 * The tricky case. The bits must be shifted into position.
2050 */
2051
2052 if (soffs > doffs) {
2053 lshift = (soffs - doffs);
2054 rshift = 8 - lshift;
2055 bits = *src;
2056 if (soffs + n > 8) {
2057 src += sdir;
2058 }
2059 } else {
2060 rshift = (doffs - soffs);
2061 lshift = 8 - rshift;
2062 bits = 0;
2063 }
2064
2065 if (lmask) {
2066 bits1 = bits << lshift;
2067 bits = *src;
2068 src += sdir;
2069 bits1 |= (bits >> rshift);
2070 *dst = MASK_BITS(bits1,*dst,lmask);
2071 dst += ddir;
2072 }
2073
2074 while (count--) {
2075 bits1 = bits << lshift;
2076 bits = *src;
2077 src += sdir;
2078 *dst = bits1 | (bits >> rshift);
2079 dst += ddir;
2080 }
2081
2082 if (rmask) {
2083 bits1 = bits << lshift;
2084 if ((rmask << rshift) & 0xff) {
2085 bits = *src;
2086 bits1 |= (bits >> rshift);
2087 }
2088 *dst = MASK_BITS(bits1,*dst,rmask);
2089 }
2090 }
2091 }
2092
erts_extract_sub_binary(Eterm ** hp,Eterm base_bin,byte * base_data,Uint bit_offset,Uint bit_size)2093 Eterm erts_extract_sub_binary(Eterm **hp, Eterm base_bin, byte *base_data,
2094 Uint bit_offset, Uint bit_size)
2095 {
2096 Uint byte_offset, byte_size;
2097
2098 ERTS_CT_ASSERT(ERL_SUB_BIN_SIZE <= ERL_ONHEAP_BIN_LIMIT);
2099
2100 byte_offset = BYTE_OFFSET(bit_offset);
2101 byte_size = BYTE_OFFSET(bit_size);
2102
2103 if (BIT_OFFSET(bit_size) == 0 && byte_size <= ERL_ONHEAP_BIN_LIMIT) {
2104 ErlHeapBin *hb = (ErlHeapBin*)*hp;
2105 *hp += heap_bin_size(byte_size);
2106
2107 hb->thing_word = header_heap_bin(byte_size);
2108 hb->size = byte_size;
2109
2110 copy_binary_to_buffer(hb->data, 0, base_data, bit_offset, bit_size);
2111
2112 return make_binary(hb);
2113 } else {
2114 ErlSubBin *sb = (ErlSubBin*)*hp;
2115 *hp += ERL_SUB_BIN_SIZE;
2116
2117 sb->thing_word = HEADER_SUB_BIN;
2118 sb->size = byte_size;
2119 sb->offs = byte_offset;
2120 sb->orig = base_bin;
2121 sb->bitoffs = BIT_OFFSET(bit_offset);
2122 sb->bitsize = BIT_OFFSET(bit_size);
2123 sb->is_writable = 0;
2124
2125 return make_binary(sb);
2126 }
2127 }
2128
2129