1 /*
2 * %CopyrightBegin%
3 *
4 * Copyright Ericsson AB 1997-2020. All Rights Reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * %CopyrightEnd%
19 */
20
21 #ifndef __ERL_MESSAGE_H__
22 #define __ERL_MESSAGE_H__
23
24 #include "sys.h"
25 #include "erl_vm.h"
26 #define ERTS_PROC_SIG_QUEUE_TYPE_ONLY
27 #include "erl_proc_sig_queue.h"
28 #undef ERTS_PROC_SIG_QUEUE_TYPE_ONLY
29
30 #ifdef DEBUG
31 #define ERTS_MSG_COPY_WORDS_PER_REDUCTION 4
32 #else
33 #define ERTS_MSG_COPY_WORDS_PER_REDUCTION 64
34 #endif
35
36 struct proc_bin;
37 struct external_thing_;
38
39 typedef struct erl_mesg ErtsMessage;
40
41 /*
42 * This struct represents data that must be updated by structure copy,
43 * but is stored outside of any heap.
44 */
45
46 struct erl_off_heap_header {
47 Eterm thing_word;
48 Uint size;
49 struct erl_off_heap_header* next;
50 };
51
52 #define OH_OVERHEAD(oh, size) do { \
53 (oh)->overhead += size; \
54 } while(0)
55
56 typedef struct erl_off_heap {
57 struct erl_off_heap_header* first;
58 Uint64 overhead; /* Administrative overhead (used to force GC). */
59 } ErlOffHeap;
60
61 #define ERTS_INIT_OFF_HEAP(OHP) \
62 do { \
63 (OHP)->first = NULL; \
64 (OHP)->overhead = 0; \
65 } while (0)
66
67 typedef struct {
68 enum {
69 FACTORY_CLOSED = 0,
70 FACTORY_HALLOC,
71 FACTORY_MESSAGE,
72 FACTORY_HEAP_FRAGS,
73 FACTORY_STATIC,
74 FACTORY_TMP
75 } mode;
76 Process* p;
77 /*
78 If the factory is initialized with erts_factory_proc_prealloc_init,
79 hp_start points to the top of the main heap if the preallocated data
80 fits in the main heap and otherwise it points to somewhere in the
81 data area of a heap fragment. If the factory is initialized with any
82 of the other init functions that sets the mode to FACTORY_HALLOC,
83 hp_start and original_htop always have the same value.
84
85 When erts_factory_proc_prealloc_init is used for initialization the
86 preallocated data might be allocated in an existing heap fragment but
87 data that is later allocated with erts_produce_heap might fit in the
88 main heap, so both hp_start and original_htop are needed to correctly
89 restore the heap in the erts_factory_undo function.
90 */
91 Eterm* hp_start;
92 /*
93 original_htop stores the top of the main heap at the time
94 the factory was initialized and is used to reset the heap
95 state if an erts_factory_undo call is made.
96 */
97 Eterm* original_htop;
98 Eterm* hp;
99 Eterm* hp_end;
100 ErtsMessage *message;
101 struct erl_heap_fragment* heap_frags;
102 struct erl_heap_fragment* heap_frags_saved;
103 Uint heap_frags_saved_used;
104 ErlOffHeap* off_heap;
105 ErlOffHeap off_heap_saved;
106 Uint32 alloc_type;
107 } ErtsHeapFactory;
108
109 void erts_factory_proc_init(ErtsHeapFactory*, Process*);
110 void erts_factory_proc_prealloc_init(ErtsHeapFactory*, Process*, Sint size);
111 void erts_factory_heap_frag_init(ErtsHeapFactory*, struct erl_heap_fragment*);
112 ErtsMessage *erts_factory_message_create(ErtsHeapFactory *, Process *,
113 ErtsProcLocks *, Uint sz);
114 void erts_factory_selfcontained_message_init(ErtsHeapFactory*, ErtsMessage *, Eterm *);
115 void erts_factory_static_init(ErtsHeapFactory*, Eterm* hp, Uint size, ErlOffHeap*);
116 void erts_factory_tmp_init(ErtsHeapFactory*, Eterm* hp, Uint size, Uint32 atype);
117 void erts_factory_dummy_init(ErtsHeapFactory*);
118
119 ERTS_GLB_INLINE Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra);
120
121 Eterm* erts_reserve_heap(ErtsHeapFactory*, Uint need);
122 void erts_factory_close(ErtsHeapFactory*);
123 void erts_factory_trim_and_close(ErtsHeapFactory*,Eterm *brefs, Uint brefs_size);
124 void erts_factory_undo(ErtsHeapFactory*);
125
126 void erts_reserve_heap__(ErtsHeapFactory*, Uint need, Uint xtra); /* internal */
127
128 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
129
130 ERTS_GLB_INLINE Eterm *
erts_produce_heap(ErtsHeapFactory * factory,Uint need,Uint xtra)131 erts_produce_heap(ErtsHeapFactory* factory, Uint need, Uint xtra)
132 {
133 Eterm* res;
134
135 ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED);
136 if (factory->hp + need > factory->hp_end) {
137 erts_reserve_heap__(factory, need, xtra);
138 }
139 res = factory->hp;
140 factory->hp += need;
141 INIT_HEAP_MEM(res, need);
142 return res;
143 }
144
145 #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
146
147 #ifdef CHECK_FOR_HOLES
148 # define ERTS_FACTORY_HOLE_CHECK(f) do { \
149 /*if ((f)->p) erts_check_for_holes((f)->p);*/ \
150 } while (0)
151 #else
152 # define ERTS_FACTORY_HOLE_CHECK(p)
153 #endif
154
155 #include "external.h"
156 #include "erl_process.h"
157
158 #define ERTS_INVALID_HFRAG_PTR ((ErlHeapFragment *) ~((UWord) 7))
159
160 /*
161 * This struct represents a heap fragment, which is used when there
162 * isn't sufficient room in the process heap and we can't do a GC.
163 */
164
165 typedef struct erl_heap_fragment ErlHeapFragment;
166 struct erl_heap_fragment {
167 ErlHeapFragment* next; /* Next heap fragment */
168 ErlOffHeap off_heap; /* Offset heap data. */
169 Uint alloc_size; /* Size in words of mem */
170 Uint used_size; /* With terms to be moved to heap by GC */
171 Eterm mem[1]; /* Data */
172 };
173
174 /* m[0] = message, m[1] = seq trace token */
175 #define ERL_MESSAGE_REF_ARRAY_SZ 3
176 #define ERL_MESSAGE_TERM(mp) ((mp)->m[0])
177 #define ERL_MESSAGE_TOKEN(mp) ((mp)->m[1])
178 #define ERL_MESSAGE_FROM(mp) ((mp)->m[2])
179
180 #ifdef USE_VM_PROBES
181 /* m[2] = dynamic trace user tag */
182 #undef ERL_MESSAGE_REF_ARRAY_SZ
183 #define ERL_MESSAGE_REF_ARRAY_SZ 4
184 #define ERL_MESSAGE_DT_UTAG(mp) ((mp)->m[3])
185 #else
186 #endif
187
188 #ifdef USE_VM_PROBES
189 #define have_no_seqtrace(T) ((T) == NIL || (T) == am_have_dt_utag)
190 #else
191 #define have_no_seqtrace(T) ((T) == NIL)
192 #endif
193 #define have_seqtrace(T) (!have_no_seqtrace(T))
194
195 #define ERL_MESSAGE_REF_FIELDS__ \
196 ErtsMessage *next; /* Next message */ \
197 union { \
198 ErlHeapFragment *heap_frag; \
199 void *attached; \
200 } data; \
201 Eterm m[ERL_MESSAGE_REF_ARRAY_SZ]
202
203
204 typedef struct erl_msg_ref__ {
205 ERL_MESSAGE_REF_FIELDS__;
206 } ErtsMessageRef;
207
208 struct erl_mesg {
209 ERL_MESSAGE_REF_FIELDS__;
210
211 ErlHeapFragment hfrag;
212 };
213
214 /*
215 * The ErtsMessage struct is only one special type
216 * of signal. All signal structs have a common
217 * begining and can be differentiated by looking
218 * at the ErtsSignal 'common.tag' field.
219 *
220 * - An ordinary message will have a value
221 * - A distribution message that has not been
222 * decoded yet will have the non-value.
223 * - Other signals will have an external pid
224 * header tag. In order to differentiate
225 * between those signals one needs to look
226 * at the arity part of the header (see
227 * erts_proc_sig_queue.h).
228 */
229
230 #define ERTS_SIG_IS_NON_MSG_TAG(Tag) \
231 (is_external_pid_header((Tag)))
232
233 #define ERTS_SIG_IS_NON_MSG(Sig) \
234 ERTS_SIG_IS_NON_MSG_TAG(((ErtsSignal *) (Sig))->common.tag)
235
236 #define ERTS_SIG_IS_INTERNAL_MSG_TAG(Tag) \
237 (!is_header((Tag)))
238 #define ERTS_SIG_IS_INTERNAL_MSG(Sig) \
239 ERTS_SIG_IS_INTERNAL_MSG_TAG(((ErtsSignal *) (Sig))->common.tag)
240
241 #define ERTS_SIG_IS_EXTERNAL_MSG_TAG(Tag) \
242 ((Tag) == THE_NON_VALUE)
243 #define ERTS_SIG_IS_EXTERNAL_MSG(Sig) \
244 ERTS_SIG_IS_EXTERNAL_MSG_TAG(((ErtsSignal *) (Sig))->common.tag)
245
246 #define ERTS_SIG_IS_MSG_TAG(Tag) \
247 (!ERTS_SIG_IS_NON_MSG_TAG(Tag))
248 #define ERTS_SIG_IS_MSG(Sig) \
249 ERTS_SIG_IS_MSG_TAG(((ErtsSignal *) (Sig))->common.tag)
250
251 typedef union {
252 ErtsSignalCommon common;
253 ErtsMessageRef msg;
254 } ErtsSignal;
255
256 typedef struct {
257 /* pointers to next pointers pointing to... */
258 ErtsMessage **next; /* ... next (non-message) signal */
259 ErtsMessage **last; /* ... last (non-message) signal */
260 } ErtsMsgQNMSigs;
261
262 /* Size of default message buffer (erl_message.c) */
263 #define ERL_MESSAGE_BUF_SZ 500
264
265 typedef struct {
266 /*
267 * ** The signal queues private to a process. **
268 *
269 * These are:
270 * - an inner queue which only consists of
271 * message signals
272 * - a middle queue which contains a mixture
273 * of message and non-message signals
274 *
275 * When the process isn't processing signals in
276 * erts_proc_sig_handle_incoming():
277 * - the message queue corresponds to the inner
278 * queue. Messages in the middle queue (and
279 * in the outer queue) are in transit and
280 * have NOT been received yet!
281 *
282 * When the process is processing signals in
283 * erts_proc_sig_handle_incoming():
284 * - the message queue corresponds to the inner
285 * queue plus the head of the middle queue up
286 * to the signal currently being processed.
287 * Any messages further back in the middle queue
288 * (and in the outer queue) are still in transit
289 * and have NOT been received yet!
290 *
291 * In the general case the 'len' field of this
292 * structure does NOT correspond to the message
293 * queue length. When the process is inspected
294 * via process info it does however correspond
295 * to the message queue length, but this is a
296 * special case!
297 *
298 * When no process-info request is in transit to
299 * the process the 'len' field corresponds to
300 * the total amount of messages in inner and
301 * middle queues (which does NOT correspond to
302 * the message queue length). When process-info
303 * requests are in transit to the process, the
304 * usage of the 'len' field changes and is used
305 * as an offset which even might be negative.
306 */
307
308 /* inner queue */
309 ErtsMessage *first;
310 ErtsMessage **last; /* point to the last next pointer */
311 ErtsMessage **save;
312
313 /* middle queue */
314 ErtsMessage *cont;
315 ErtsMessage **cont_last;
316 ErtsMsgQNMSigs nmsigs;
317
318 /* Common for inner and middle queue */
319 ErtsMessage **saved_last; /* saved last pointer */
320 Sint len; /* NOT message queue length (see above) */
321 Uint32 flags;
322 } ErtsSignalPrivQueues;
323
324 typedef struct {
325 ErtsMessage* first;
326 ErtsMessage** last; /* point to the last next pointer */
327 Sint len; /* number of messages in queue */
328 ErtsMsgQNMSigs nmsigs;
329 #ifdef ERTS_PROC_SIG_HARD_DEBUG
330 int may_contain_heap_terms;
331 #endif
332 } ErtsSignalInQueue;
333
334 typedef struct erl_trace_message_queue__ {
335 struct erl_trace_message_queue__ *next; /* point to the next receiver */
336 Eterm receiver;
337 ErtsMessage* first;
338 ErtsMessage** last; /* point to the last next pointer */
339 Sint len; /* queue length */
340 } ErlTraceMessageQueue;
341
342 #define ERTS_RECV_MARK_SAVE(P) \
343 do { \
344 erts_proc_lock((P), ERTS_PROC_LOCK_MSGQ); \
345 erts_proc_sig_fetch((P)); \
346 erts_proc_unlock((P), ERTS_PROC_LOCK_MSGQ); \
347 if ((P)->sig_qs.cont) { \
348 (P)->sig_qs.saved_last = (P)->sig_qs.cont_last; \
349 (P)->sig_qs.flags |= FS_DEFERRED_SAVED_LAST; \
350 } \
351 else { \
352 (P)->sig_qs.saved_last = (P)->sig_qs.last; \
353 (P)->sig_qs.flags &= ~FS_DEFERRED_SAVED_LAST; \
354 } \
355 } while (0)
356
357 #define ERTS_RECV_MARK_SET(P) \
358 do { \
359 if ((P)->sig_qs.saved_last) { \
360 if ((P)->sig_qs.flags & FS_DEFERRED_SAVED_LAST) { \
361 (P)->sig_qs.flags |= FS_DEFERRED_SAVE; \
362 /* \
363 * Trigger handling of signals in loop_rec by \
364 * setting save pointer to the end of message queue \
365 * (inner queue). This in order to resolv saved_last \
366 * which currently may point into inner or middle \
367 * queue. \
368 */ \
369 (P)->sig_qs.save = (P)->sig_qs.last; \
370 } \
371 else { \
372 /* Points to inner queue; safe to use */ \
373 (P)->sig_qs.save = (P)->sig_qs.saved_last; \
374 } \
375 } \
376 } while (0)
377
378 #define ERTS_RECV_MARK_CLEAR(P) \
379 do { \
380 (P)->sig_qs.saved_last = NULL; \
381 (P)->sig_qs.flags &= ~(FS_DEFERRED_SAVED_LAST|FS_DEFERRED_SAVE); \
382 } while (0)
383
384
385 /* Get "current" message */
386 #define PEEK_MESSAGE(p) (*(p)->sig_qs.save)
387
388 #ifdef USE_VM_PROBES
389 #define LINK_MESSAGE_DTAG(mp, dt) ERL_MESSAGE_DT_UTAG(mp) = dt
390 #else
391 #define LINK_MESSAGE_DTAG(mp, dt)
392 #endif
393
394 #ifdef USE_VM_PROBES
395 # define ERTS_MSG_RECV_TRACED(P) \
396 ((ERTS_TRACE_FLAGS((P)) & F_TRACE_RECEIVE) \
397 || DTRACE_ENABLED(message_queued))
398 #else
399 # define ERTS_MSG_RECV_TRACED(P) \
400 (ERTS_TRACE_FLAGS((P)) & F_TRACE_RECEIVE)
401
402 #endif
403
404 /* Add one message last in message queue */
405 #define LINK_MESSAGE(p, msg) \
406 do { \
407 ASSERT(ERTS_SIG_IS_MSG(msg)); \
408 ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE__((p), "before"); \
409 *(p)->sig_inq.last = (msg); \
410 (p)->sig_inq.last = &(msg)->next; \
411 (p)->sig_inq.len++; \
412 ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE__((p), "before"); \
413 } while(0)
414
415 /* Unlink current message */
416 #define UNLINK_MESSAGE(p,msgp) \
417 do { \
418 ErtsMessage *mp__ = (msgp)->next; \
419 ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((p), 0, "before"); \
420 *(p)->sig_qs.save = mp__; \
421 (p)->sig_qs.len--; \
422 if (mp__ == NULL) \
423 (p)->sig_qs.last = (p)->sig_qs.save; \
424 ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((p), 0, "after"); \
425 } while(0)
426
427 /*
428 * Reset message save point (after receive match).
429 * Also invalidate the saved position since it may no
430 * longer be safe to use.
431 */
432 #define JOIN_MESSAGE(p) \
433 do { \
434 (p)->sig_qs.save = &(p)->sig_qs.first; \
435 ERTS_RECV_MARK_CLEAR((p)); \
436 } while(0)
437
438 /* Save current message */
439 #define SAVE_MESSAGE(p) \
440 (p)->sig_qs.save = &(*(p)->sig_qs.save)->next
441
442 #define ERTS_HEAP_FRAG_SIZE(DATA_WORDS) \
443 (sizeof(ErlHeapFragment) - sizeof(Eterm) + (DATA_WORDS)*sizeof(Eterm))
444
445 #define ERTS_INIT_HEAP_FRAG(HEAP_FRAG_P, USED_WORDS, DATA_WORDS) \
446 do { \
447 (HEAP_FRAG_P)->next = NULL; \
448 (HEAP_FRAG_P)->alloc_size = (DATA_WORDS); \
449 (HEAP_FRAG_P)->used_size = (USED_WORDS); \
450 (HEAP_FRAG_P)->off_heap.first = NULL; \
451 (HEAP_FRAG_P)->off_heap.overhead = 0; \
452 } while (0)
453
454 #ifdef USE_VM_PROBES
455 #define ERL_MESSAGE_DT_UTAG_INIT(MP) ERL_MESSAGE_DT_UTAG(MP) = NIL
456 #else
457 #define ERL_MESSAGE_DT_UTAG_INIT(MP) do{ } while (0)
458 #endif
459
460 #define ERTS_INIT_MESSAGE(MP) \
461 do { \
462 (MP)->next = NULL; \
463 ERL_MESSAGE_TERM(MP) = THE_NON_VALUE; \
464 ERL_MESSAGE_TOKEN(MP) = THE_NON_VALUE; \
465 ERL_MESSAGE_FROM(MP) = NIL; \
466 ERL_MESSAGE_DT_UTAG_INIT(MP); \
467 MP->data.attached = NULL; \
468 } while (0)
469
470 void init_message(void);
471 ErlHeapFragment* new_message_buffer(Uint);
472 ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint,
473 Eterm *, Uint);
474 void free_message_buffer(ErlHeapFragment *);
475 void erts_queue_dist_message(Process*, ErtsProcLocks, ErtsDistExternal *,
476 ErlHeapFragment *, Eterm, Eterm);
477 void erts_queue_message(Process*, ErtsProcLocks,ErtsMessage*, Eterm, Eterm);
478 void erts_queue_proc_message(Process* from,Process* to, ErtsProcLocks,ErtsMessage*, Eterm);
479 void erts_queue_proc_messages(Process* from, Process* to, ErtsProcLocks,
480 ErtsMessage*, ErtsMessage**, Uint);
481 void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm);
482 void erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm);
483 void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp);
484
485 Uint erts_msg_attached_data_size_aux(ErtsMessage *msg);
486
487 void erts_cleanup_offheap(ErlOffHeap *offheap);
488 void erts_save_message_in_proc(Process *p, ErtsMessage *msg);
489 Sint erts_move_messages_off_heap(Process *c_p);
490 Sint erts_complete_off_heap_message_queue_change(Process *c_p);
491 Eterm erts_change_message_queue_management(Process *c_p, Eterm new_state);
492
493 void erts_cleanup_messages(ErtsMessage *mp);
494
495 void *erts_alloc_message_ref(void);
496 void erts_free_message_ref(void *);
497
498 #define ERTS_SMALL_FIX_MSG_SZ 10
499 #define ERTS_MEDIUM_FIX_MSG_SZ 20
500 #define ERTS_LARGE_FIX_MSG_SZ 30
501
502 void *erts_alloc_small_message(void);
503 void erts_free_small_message(void *mp);
504
505 typedef struct {
506 ErtsMessage m;
507 Eterm data[ERTS_SMALL_FIX_MSG_SZ-1];
508 } ErtsSmallFixSzMessage;
509
510 typedef struct {
511 ErtsMessage m;
512 Eterm data[ERTS_MEDIUM_FIX_MSG_SZ-1];
513 } ErtsMediumFixSzMessage;
514
515 typedef struct {
516 ErtsMessage m;
517 Eterm data[ERTS_LARGE_FIX_MSG_SZ-1];
518 } ErtsLargeFixSzMessage;
519
520 ErtsMessage *erts_try_alloc_message_on_heap(Process *pp,
521 erts_aint32_t *psp,
522 ErtsProcLocks *plp,
523 Uint sz,
524 Eterm **hpp,
525 ErlOffHeap **ohpp,
526 int *on_heap_p);
527 ErtsMessage *erts_realloc_shrink_message(ErtsMessage *mp, Uint sz,
528 Eterm *brefs, Uint brefs_size);
529
530 ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp);
531 ERTS_GLB_FORCE_INLINE ErtsMessage *erts_shrink_message(ErtsMessage *mp, Uint sz,
532 Eterm *brefs, Uint brefs_size);
533 ERTS_GLB_FORCE_INLINE void erts_free_message(ErtsMessage *mp);
534 ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment*);
535 ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg);
536
537 #define ERTS_MSG_COMBINED_HFRAG ((void *) 0x1)
538
539 #define erts_message_to_heap_frag(MP) \
540 (((MP)->data.attached == ERTS_MSG_COMBINED_HFRAG) ? \
541 &(MP)->hfrag : (MP)->data.heap_frag)
542
543 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
544
erts_alloc_message(Uint sz,Eterm ** hpp)545 ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp)
546 {
547 ErtsMessage *mp;
548
549 if (sz == 0) {
550 mp = erts_alloc_message_ref();
551 ERTS_INIT_MESSAGE(mp);
552 if (hpp)
553 *hpp = NULL;
554 return mp;
555 }
556
557 mp = erts_alloc(ERTS_ALC_T_MSG,
558 sizeof(ErtsMessage) + (sz - 1)*sizeof(Eterm));
559
560 ERTS_INIT_MESSAGE(mp);
561 mp->data.attached = ERTS_MSG_COMBINED_HFRAG;
562 ERTS_INIT_HEAP_FRAG(&mp->hfrag, sz, sz);
563
564 if (hpp)
565 *hpp = &mp->hfrag.mem[0];
566
567 return mp;
568 }
569
570 ERTS_GLB_FORCE_INLINE ErtsMessage *
erts_shrink_message(ErtsMessage * mp,Uint sz,Eterm * brefs,Uint brefs_size)571 erts_shrink_message(ErtsMessage *mp, Uint sz, Eterm *brefs, Uint brefs_size)
572 {
573 if (sz == 0) {
574 ErtsMessage *nmp;
575 if (!mp->data.attached)
576 return mp;
577 ASSERT(mp->data.attached == ERTS_MSG_COMBINED_HFRAG);
578 nmp = erts_alloc_message_ref();
579 #ifdef DEBUG
580 if (brefs && brefs_size) {
581 int i;
582 for (i = 0; i < brefs_size; i++)
583 ASSERT(is_non_value(brefs[i]) || is_immed(brefs[i]));
584 }
585 #endif
586 erts_free(ERTS_ALC_T_MSG, mp);
587 return nmp;
588 }
589
590 ASSERT(mp->data.attached == ERTS_MSG_COMBINED_HFRAG);
591 ASSERT(mp->hfrag.used_size >= sz);
592
593 if (sz >= (mp->hfrag.alloc_size - mp->hfrag.alloc_size / 16)) {
594 mp->hfrag.used_size = sz;
595 return mp;
596 }
597
598 return erts_realloc_shrink_message(mp, sz, brefs, brefs_size);
599 }
600
erts_free_message(ErtsMessage * mp)601 ERTS_GLB_FORCE_INLINE void erts_free_message(ErtsMessage *mp)
602 {
603 if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG)
604 erts_free_message_ref(mp);
605 else
606 erts_free(ERTS_ALC_T_MSG, mp);
607 }
608
erts_used_frag_sz(const ErlHeapFragment * bp)609 ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment* bp)
610 {
611 Uint sz = 0;
612 for ( ; bp!=NULL; bp=bp->next) {
613 sz += bp->used_size;
614 }
615 return sz;
616 }
617
erts_msg_attached_data_size(ErtsMessage * msg)618 ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg)
619 {
620 ASSERT(msg->data.attached);
621
622 if (ERTS_SIG_IS_INTERNAL_MSG(msg))
623 return erts_used_frag_sz(erts_message_to_heap_frag(msg));
624
625 return erts_msg_attached_data_size_aux(msg);
626 }
627
628 #endif
629
630 Uint erts_mbuf_size(Process *p);
631 #if defined(DEBUG) || 0
632 # define ERTS_CHK_MBUF_SZ(P) \
633 do { \
634 Uint actual_mbuf_sz__ = erts_mbuf_size((P)); \
635 ERTS_ASSERT((P)->mbuf_sz >= actual_mbuf_sz__); \
636 } while (0)
637 #else
638 # define ERTS_CHK_MBUF_SZ(P) ((void) 1)
639 #endif
640
641 #define ERTS_FOREACH_SIG_PRIVQS(PROC, MVAR, CODE) \
642 do { \
643 int i__; \
644 ErtsMessage *msgs__[] = {(PROC)->sig_qs.first, \
645 (PROC)->sig_qs.cont}; \
646 for (i__ = 0; i__ < sizeof(msgs__)/sizeof(msgs__[0]); i__++) { \
647 ErtsMessage *MVAR; \
648 for (MVAR = msgs__[i__]; MVAR; MVAR = MVAR->next) { \
649 CODE; \
650 } \
651 } \
652 } while (0)
653
654 #endif
655