1 /*
2 * %CopyrightBegin%
3 *
4 * Copyright Ericsson AB 1997-2020. All Rights Reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * %CopyrightEnd%
19 */
20
21 #ifndef __ERL_MESSAGE_H__
22 #define __ERL_MESSAGE_H__
23
24 #include "sys.h"
25 #include "erl_vm.h"
26 #define ERTS_PROC_SIG_QUEUE_TYPE_ONLY
27 #include "erl_proc_sig_queue.h"
28 #undef ERTS_PROC_SIG_QUEUE_TYPE_ONLY
29
30 #ifdef DEBUG
31 #define ERTS_MSG_COPY_WORDS_PER_REDUCTION 4
32 #else
33 #define ERTS_MSG_COPY_WORDS_PER_REDUCTION 64
34 #endif
35
36 struct proc_bin;
37 struct external_thing_;
38
39 typedef struct erl_mesg ErtsMessage;
40
41 /*
42 * This struct represents data that must be updated by structure copy,
43 * but is stored outside of any heap.
44 */
45
46 struct erl_off_heap_header {
47 Eterm thing_word;
48 Uint size;
49 struct erl_off_heap_header* next;
50 };
51
52 #define OH_OVERHEAD(oh, size) do { \
53 (oh)->overhead += size; \
54 } while(0)
55
56 typedef struct erl_off_heap {
57 struct erl_off_heap_header* first;
58 Uint64 overhead; /* Administrative overhead (used to force GC). */
59 } ErlOffHeap;
60
61 #define ERTS_INIT_OFF_HEAP(OHP) \
62 do { \
63 (OHP)->first = NULL; \
64 (OHP)->overhead = 0; \
65 } while (0)
66
67 typedef struct {
68 enum {
69 FACTORY_CLOSED = 0,
70 FACTORY_HALLOC,
71 FACTORY_MESSAGE,
72 FACTORY_HEAP_FRAGS,
73 FACTORY_STATIC,
74 FACTORY_TMP
75 } mode;
76 Process* p;
77 /*
78 If the factory is initialized with erts_factory_proc_prealloc_init,
79 hp_start points to the top of the main heap if the preallocated data
80 fits in the main heap and otherwise it points to somewhere in the
81 data area of a heap fragment. If the factory is initialized with any
82 of the other init functions that sets the mode to FACTORY_HALLOC,
83 hp_start and original_htop always have the same value.
84
85 When erts_factory_proc_prealloc_init is used for initialization the
86 preallocated data might be allocated in an existing heap fragment but
87 data that is later allocated with erts_produce_heap might fit in the
88 main heap, so both hp_start and original_htop are needed to correctly
89 restore the heap in the erts_factory_undo function.
90 */
91 Eterm* hp_start;
92 /*
93 original_htop stores the top of the main heap at the time
94 the factory was initialized and is used to reset the heap
95 state if an erts_factory_undo call is made.
96 */
97 Eterm* original_htop;
98 Eterm* hp;
99 Eterm* hp_end;
100 ErtsMessage *message;
101 struct erl_heap_fragment* heap_frags;
102 struct erl_heap_fragment* heap_frags_saved;
103 Uint heap_frags_saved_used;
104 ErlOffHeap* off_heap;
105 ErlOffHeap off_heap_saved;
106 Uint32 alloc_type;
107 } ErtsHeapFactory;
108
109 void erts_factory_proc_init(ErtsHeapFactory*, Process*);
110 void erts_factory_proc_prealloc_init(ErtsHeapFactory*, Process*, Sint size);
111 void erts_factory_heap_frag_init(ErtsHeapFactory*, struct erl_heap_fragment*);
112 ErtsMessage *erts_factory_message_create(ErtsHeapFactory *, Process *,
113 ErtsProcLocks *, Uint sz);
114 void erts_factory_selfcontained_message_init(ErtsHeapFactory*, ErtsMessage *, Eterm *);
115 void erts_factory_static_init(ErtsHeapFactory*, Eterm* hp, Uint size, ErlOffHeap*);
116 void erts_factory_tmp_init(ErtsHeapFactory*, Eterm* hp, Uint size, Uint32 atype);
117 void erts_factory_dummy_init(ErtsHeapFactory*);
118
119 ERTS_GLB_INLINE Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra);
120
121 Eterm* erts_reserve_heap(ErtsHeapFactory*, Uint need);
122 void erts_factory_close(ErtsHeapFactory*);
123 void erts_factory_trim_and_close(ErtsHeapFactory*,Eterm *brefs, Uint brefs_size);
124 void erts_factory_undo(ErtsHeapFactory*);
125
126 void erts_reserve_heap__(ErtsHeapFactory*, Uint need, Uint xtra); /* internal */
127
128 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
129
130 ERTS_GLB_INLINE Eterm *
erts_produce_heap(ErtsHeapFactory * factory,Uint need,Uint xtra)131 erts_produce_heap(ErtsHeapFactory* factory, Uint need, Uint xtra)
132 {
133 Eterm* res;
134
135 ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED);
136 if (factory->hp + need > factory->hp_end) {
137 erts_reserve_heap__(factory, need, xtra);
138 }
139 res = factory->hp;
140 factory->hp += need;
141 INIT_HEAP_MEM(res, need);
142 return res;
143 }
144
145 #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
146
147 #ifdef CHECK_FOR_HOLES
148 # define ERTS_FACTORY_HOLE_CHECK(f) do { \
149 /*if ((f)->p) erts_check_for_holes((f)->p);*/ \
150 } while (0)
151 #else
152 # define ERTS_FACTORY_HOLE_CHECK(p)
153 #endif
154
155 #include "external.h"
156 #include "erl_process.h"
157
158 #define ERTS_INVALID_HFRAG_PTR ((ErlHeapFragment *) ~((UWord) 7))
159
160 /*
161 * This struct represents a heap fragment, which is used when there
162 * isn't sufficient room in the process heap and we can't do a GC.
163 */
164
165 typedef struct erl_heap_fragment ErlHeapFragment;
166 struct erl_heap_fragment {
167 ErlHeapFragment* next; /* Next heap fragment */
168 ErlOffHeap off_heap; /* Offset heap data. */
169 Uint alloc_size; /* Size in words of mem */
170 Uint used_size; /* With terms to be moved to heap by GC */
171 Eterm mem[1]; /* Data */
172 };
173
174 /* m[0] = message, m[1] = seq trace token */
175 #define ERL_MESSAGE_REF_ARRAY_SZ 3
176 #define ERL_MESSAGE_TERM(mp) ((mp)->m[0])
177 #define ERL_MESSAGE_TOKEN(mp) ((mp)->m[1])
178 #define ERL_MESSAGE_FROM(mp) ((mp)->m[2])
179
180 #ifdef USE_VM_PROBES
181 /* m[2] = dynamic trace user tag */
182 #undef ERL_MESSAGE_REF_ARRAY_SZ
183 #define ERL_MESSAGE_REF_ARRAY_SZ 4
184 #define ERL_MESSAGE_DT_UTAG(mp) ((mp)->m[3])
185 #else
186 #endif
187
188 #ifdef USE_VM_PROBES
189 #define have_no_seqtrace(T) ((T) == NIL || (T) == am_have_dt_utag)
190 #else
191 #define have_no_seqtrace(T) ((T) == NIL)
192 #endif
193 #define have_seqtrace(T) (!have_no_seqtrace(T))
194
195 #define ERL_MESSAGE_REF_FIELDS__ \
196 ErtsMessage *next; /* Next message */ \
197 union { \
198 ErlHeapFragment *heap_frag; \
199 void *attached; \
200 } data; \
201 Eterm m[ERL_MESSAGE_REF_ARRAY_SZ]
202
203
204 typedef struct erl_msg_ref__ {
205 ERL_MESSAGE_REF_FIELDS__;
206 } ErtsMessageRef;
207
208 struct erl_mesg {
209 ERL_MESSAGE_REF_FIELDS__;
210
211 ErlHeapFragment hfrag;
212 };
213
214 /*
215 * The ErtsMessage struct is only one special type
216 * of signal. All signal structs have a common
217 * begining and can be differentiated by looking
218 * at the ErtsSignal 'common.tag' field.
219 *
220 * - An ordinary message will have a value
221 * - A distribution message that has not been
222 * decoded yet will have the non-value.
223 * - Other signals will have an external pid
224 * header tag. In order to differentiate
225 * between those signals one needs to look
226 * at the arity part of the header (see
227 * erts_proc_sig_queue.h).
228 */
229
230 #define ERTS_SIG_IS_NON_MSG_TAG(Tag) \
231 (is_external_pid_header((Tag)))
232
233 #define ERTS_SIG_IS_NON_MSG(Sig) \
234 ERTS_SIG_IS_NON_MSG_TAG(((ErtsSignal *) (Sig))->common.tag)
235
236 #define ERTS_SIG_IS_INTERNAL_MSG_TAG(Tag) \
237 (!is_header((Tag)))
238 #define ERTS_SIG_IS_INTERNAL_MSG(Sig) \
239 ERTS_SIG_IS_INTERNAL_MSG_TAG(((ErtsSignal *) (Sig))->common.tag)
240
241 #define ERTS_SIG_IS_EXTERNAL_MSG_TAG(Tag) \
242 ((Tag) == THE_NON_VALUE)
243 #define ERTS_SIG_IS_EXTERNAL_MSG(Sig) \
244 ERTS_SIG_IS_EXTERNAL_MSG_TAG(((ErtsSignal *) (Sig))->common.tag)
245
246 #define ERTS_SIG_IS_MSG_TAG(Tag) \
247 (!ERTS_SIG_IS_NON_MSG_TAG(Tag))
248 #define ERTS_SIG_IS_MSG(Sig) \
249 ERTS_SIG_IS_MSG_TAG(((ErtsSignal *) (Sig))->common.tag)
250
251 typedef union {
252 ErtsSignalCommon common;
253 ErtsMessageRef msg;
254 } ErtsSignal;
255
256 typedef struct {
257 /* pointers to next pointers pointing to... */
258 ErtsMessage **next; /* ... next (non-message) signal */
259 ErtsMessage **last; /* ... last (non-message) signal */
260 } ErtsMsgQNMSigs;
261
262 /* Size of default message buffer (erl_message.c) */
263 #define ERL_MESSAGE_BUF_SZ 500
264
265 typedef struct {
266 /*
267 * ** The signal queues private to a process. **
268 *
269 * These are:
270 * - an inner queue which only consists of
271 * message signals
272 * - a middle queue which contains a mixture
273 * of message and non-message signals
274 *
275 * When the process isn't processing signals in
276 * erts_proc_sig_handle_incoming():
277 * - the message queue corresponds to the inner
278 * queue. Messages in the middle queue (and
279 * in the outer queue) are in transit and
280 * have NOT been received yet!
281 *
282 * When the process is processing signals in
283 * erts_proc_sig_handle_incoming():
284 * - the message queue corresponds to the inner
285 * queue plus the head of the middle queue up
286 * to the signal currently being processed.
287 * Any messages further back in the middle queue
288 * (and in the outer queue) are still in transit
289 * and have NOT been received yet!
290 *
291 * In the general case the 'len' field of this
292 * structure does NOT correspond to the message
293 * queue length. When the process is inspected
294 * via process info it does however correspond
295 * to the message queue length, but this is a
296 * special case!
297 *
298 * When no process-info request is in transit to
299 * the process the 'len' field corresponds to
300 * the total amount of messages in inner and
301 * middle queues (which does NOT correspond to
302 * the message queue length). When process-info
303 * requests are in transit to the process, the
304 * usage of the 'len' field changes and is used
305 * as an offset which even might be negative.
306 */
307
308 /* inner queue */
309 ErtsMessage *first;
310 ErtsMessage **last; /* point to the last next pointer */
311 ErtsMessage **save;
312
313 /* middle queue */
314 ErtsMessage *cont;
315 ErtsMessage **cont_last;
316 ErtsMsgQNMSigs nmsigs;
317
318 /* Common for inner and middle queue */
319 ErtsMessage **saved_last; /* saved last pointer */
320 Sint len; /* NOT message queue length (see above) */
321 Uint32 flags;
322 } ErtsSignalPrivQueues;
323
324 typedef struct {
325 ErtsMessage* first;
326 ErtsMessage** last; /* point to the last next pointer */
327 Sint len; /* number of messages in queue */
328 ErtsMsgQNMSigs nmsigs;
329 #ifdef ERTS_PROC_SIG_HARD_DEBUG
330 int may_contain_heap_terms;
331 #endif
332 } ErtsSignalInQueue;
333
334 typedef struct erl_trace_message_queue__ {
335 struct erl_trace_message_queue__ *next; /* point to the next receiver */
336 Eterm receiver;
337 ErtsMessage* first;
338 ErtsMessage** last; /* point to the last next pointer */
339 Sint len; /* queue length */
340 } ErlTraceMessageQueue;
341
342 #define ERTS_RECV_MARK_SAVE(P) \
343 do { \
344 erts_proc_lock((P), ERTS_PROC_LOCK_MSGQ); \
345 erts_proc_sig_fetch((P)); \
346 erts_proc_unlock((P), ERTS_PROC_LOCK_MSGQ); \
347 if ((P)->sig_qs.cont) { \
348 (P)->sig_qs.saved_last = (P)->sig_qs.cont_last; \
349 (P)->sig_qs.flags |= FS_DEFERRED_SAVED_LAST; \
350 } \
351 else { \
352 (P)->sig_qs.saved_last = (P)->sig_qs.last; \
353 (P)->sig_qs.flags &= ~FS_DEFERRED_SAVED_LAST; \
354 } \
355 } while (0)
356
357 #define ERTS_RECV_MARK_SET(P) \
358 do { \
359 if ((P)->sig_qs.saved_last) { \
360 if ((P)->sig_qs.flags & FS_DEFERRED_SAVED_LAST) { \
361 (P)->sig_qs.flags |= FS_DEFERRED_SAVE; \
362 /* \
363 * Trigger handling of signals in loop_rec by \
364 * setting save pointer to the end of message queue \
365 * (inner queue). This in order to resolv saved_last \
366 * which currently may point into inner or middle \
367 * queue. \
368 */ \
369 (P)->sig_qs.save = (P)->sig_qs.last; \
370 } \
371 else { \
372 /* Points to inner queue; safe to use */ \
373 (P)->sig_qs.save = (P)->sig_qs.saved_last; \
374 } \
375 } \
376 } while (0)
377
378 #define ERTS_RECV_MARK_CLEAR(P) \
379 do { \
380 (P)->sig_qs.saved_last = NULL; \
381 (P)->sig_qs.flags &= ~(FS_DEFERRED_SAVED_LAST|FS_DEFERRED_SAVE); \
382 } while (0)
383
384
385 /* Get "current" message */
386 #define PEEK_MESSAGE(p) (*(p)->sig_qs.save)
387
388 #ifdef USE_VM_PROBES
389 #define LINK_MESSAGE_DTAG(mp, dt) ERL_MESSAGE_DT_UTAG(mp) = dt
390 #else
391 #define LINK_MESSAGE_DTAG(mp, dt)
392 #endif
393
394 #ifdef USE_VM_PROBES
395 # define ERTS_MSG_RECV_TRACED(P) \
396 ((ERTS_TRACE_FLAGS((P)) & F_TRACE_RECEIVE) \
397 || DTRACE_ENABLED(message_queued))
398 #else
399 # define ERTS_MSG_RECV_TRACED(P) \
400 (ERTS_TRACE_FLAGS((P)) & F_TRACE_RECEIVE)
401
402 #endif
403
404 /* Add one message last in message queue */
405 #define LINK_MESSAGE(p, msg) \
406 do { \
407 ASSERT(ERTS_SIG_IS_MSG(msg)); \
408 ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE__((p), "before"); \
409 *(p)->sig_inq.last = (msg); \
410 (p)->sig_inq.last = &(msg)->next; \
411 (p)->sig_inq.len++; \
412 ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE__((p), "before"); \
413 } while(0)
414
415 /* Unlink current message */
416 #define UNLINK_MESSAGE(p,msgp) \
417 do { \
418 ErtsMessage *mp__ = (msgp)->next; \
419 ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((p), 0, "before"); \
420 *(p)->sig_qs.save = mp__; \
421 (p)->sig_qs.len--; \
422 if (mp__ == NULL) \
423 (p)->sig_qs.last = (p)->sig_qs.save; \
424 ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((p), 0, "after"); \
425 } while(0)
426
427 /*
428 * Reset message save point (after receive match).
429 * Also invalidate the saved position since it may no
430 * longer be safe to use.
431 */
432 #define JOIN_MESSAGE(p) \
433 do { \
434 (p)->sig_qs.save = &(p)->sig_qs.first; \
435 ERTS_RECV_MARK_CLEAR((p)); \
436 } while(0)
437
438 /* Save current message */
439 #define SAVE_MESSAGE(p) \
440 (p)->sig_qs.save = &(*(p)->sig_qs.save)->next
441
442 #define ERTS_HEAP_FRAG_SIZE(DATA_WORDS) \
443 (sizeof(ErlHeapFragment) - sizeof(Eterm) + (DATA_WORDS)*sizeof(Eterm))
444
445 #define ERTS_INIT_HEAP_FRAG(HEAP_FRAG_P, USED_WORDS, DATA_WORDS) \
446 do { \
447 (HEAP_FRAG_P)->next = NULL; \
448 (HEAP_FRAG_P)->alloc_size = (DATA_WORDS); \
449 (HEAP_FRAG_P)->used_size = (USED_WORDS); \
450 (HEAP_FRAG_P)->off_heap.first = NULL; \
451 (HEAP_FRAG_P)->off_heap.overhead = 0; \
452 } while (0)
453
454 #ifdef USE_VM_PROBES
455 #define ERL_MESSAGE_DT_UTAG_INIT(MP) ERL_MESSAGE_DT_UTAG(MP) = NIL
456 #else
457 #define ERL_MESSAGE_DT_UTAG_INIT(MP) do{ } while (0)
458 #endif
459
460 #define ERTS_INIT_MESSAGE(MP) \
461 do { \
462 (MP)->next = NULL; \
463 ERL_MESSAGE_TERM(MP) = THE_NON_VALUE; \
464 ERL_MESSAGE_TOKEN(MP) = THE_NON_VALUE; \
465 ERL_MESSAGE_FROM(MP) = NIL; \
466 ERL_MESSAGE_DT_UTAG_INIT(MP); \
467 MP->data.attached = NULL; \
468 } while (0)
469
470 void init_message(void);
471 ErlHeapFragment* new_message_buffer(Uint);
472 ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint,
473 Eterm *, Uint);
474 void free_message_buffer(ErlHeapFragment *);
475 void erts_queue_dist_message(Process*, ErtsProcLocks, ErtsDistExternal *,
476 ErlHeapFragment *, Eterm, Eterm);
477 void erts_queue_message(Process*, ErtsProcLocks,ErtsMessage*, Eterm, Eterm);
478 void erts_queue_message_token(Process*, ErtsProcLocks,ErtsMessage*, Eterm, Eterm, Eterm);
479 void erts_queue_proc_message(Process* from,Process* to, ErtsProcLocks,ErtsMessage*, Eterm);
480 void erts_queue_proc_messages(Process* from, Process* to, ErtsProcLocks,
481 ErtsMessage*, ErtsMessage**, Uint);
482 void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm);
483 void erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm);
484 void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp);
485
486 Uint erts_msg_attached_data_size_aux(ErtsMessage *msg);
487
488 void erts_cleanup_offheap(ErlOffHeap *offheap);
489 void erts_save_message_in_proc(Process *p, ErtsMessage *msg);
490 Sint erts_move_messages_off_heap(Process *c_p);
491 Sint erts_complete_off_heap_message_queue_change(Process *c_p);
492 Eterm erts_change_message_queue_management(Process *c_p, Eterm new_state);
493
494 void erts_cleanup_messages(ErtsMessage *mp);
495
496 void *erts_alloc_message_ref(void);
497 void erts_free_message_ref(void *);
498
499 #define ERTS_SMALL_FIX_MSG_SZ 10
500 #define ERTS_MEDIUM_FIX_MSG_SZ 20
501 #define ERTS_LARGE_FIX_MSG_SZ 30
502
503 void *erts_alloc_small_message(void);
504 void erts_free_small_message(void *mp);
505
506 typedef struct {
507 ErtsMessage m;
508 Eterm data[ERTS_SMALL_FIX_MSG_SZ-1];
509 } ErtsSmallFixSzMessage;
510
511 typedef struct {
512 ErtsMessage m;
513 Eterm data[ERTS_MEDIUM_FIX_MSG_SZ-1];
514 } ErtsMediumFixSzMessage;
515
516 typedef struct {
517 ErtsMessage m;
518 Eterm data[ERTS_LARGE_FIX_MSG_SZ-1];
519 } ErtsLargeFixSzMessage;
520
521 ErtsMessage *erts_try_alloc_message_on_heap(Process *pp,
522 erts_aint32_t *psp,
523 ErtsProcLocks *plp,
524 Uint sz,
525 Eterm **hpp,
526 ErlOffHeap **ohpp,
527 int *on_heap_p);
528 ErtsMessage *erts_realloc_shrink_message(ErtsMessage *mp, Uint sz,
529 Eterm *brefs, Uint brefs_size);
530
531 ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp);
532 ERTS_GLB_FORCE_INLINE ErtsMessage *erts_shrink_message(ErtsMessage *mp, Uint sz,
533 Eterm *brefs, Uint brefs_size);
534 ERTS_GLB_FORCE_INLINE void erts_free_message(ErtsMessage *mp);
535 ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment*);
536 ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg);
537
538 #define ERTS_MSG_COMBINED_HFRAG ((void *) 0x1)
539
540 #define erts_message_to_heap_frag(MP) \
541 (((MP)->data.attached == ERTS_MSG_COMBINED_HFRAG) ? \
542 &(MP)->hfrag : (MP)->data.heap_frag)
543
544 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
545
erts_alloc_message(Uint sz,Eterm ** hpp)546 ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp)
547 {
548 ErtsMessage *mp;
549
550 if (sz == 0) {
551 mp = erts_alloc_message_ref();
552 ERTS_INIT_MESSAGE(mp);
553 if (hpp)
554 *hpp = NULL;
555 return mp;
556 }
557
558 mp = erts_alloc(ERTS_ALC_T_MSG,
559 sizeof(ErtsMessage) + (sz - 1)*sizeof(Eterm));
560
561 ERTS_INIT_MESSAGE(mp);
562 mp->data.attached = ERTS_MSG_COMBINED_HFRAG;
563 ERTS_INIT_HEAP_FRAG(&mp->hfrag, sz, sz);
564
565 if (hpp)
566 *hpp = &mp->hfrag.mem[0];
567
568 return mp;
569 }
570
571 ERTS_GLB_FORCE_INLINE ErtsMessage *
erts_shrink_message(ErtsMessage * mp,Uint sz,Eterm * brefs,Uint brefs_size)572 erts_shrink_message(ErtsMessage *mp, Uint sz, Eterm *brefs, Uint brefs_size)
573 {
574 if (sz == 0) {
575 ErtsMessage *nmp;
576 if (!mp->data.attached)
577 return mp;
578 ASSERT(mp->data.attached == ERTS_MSG_COMBINED_HFRAG);
579 nmp = erts_alloc_message_ref();
580 #ifdef DEBUG
581 if (brefs && brefs_size) {
582 int i;
583 for (i = 0; i < brefs_size; i++)
584 ASSERT(is_non_value(brefs[i]) || is_immed(brefs[i]));
585 }
586 #endif
587 erts_free(ERTS_ALC_T_MSG, mp);
588 return nmp;
589 }
590
591 ASSERT(mp->data.attached == ERTS_MSG_COMBINED_HFRAG);
592 ASSERT(mp->hfrag.used_size >= sz);
593
594 if (sz >= (mp->hfrag.alloc_size - mp->hfrag.alloc_size / 16)) {
595 mp->hfrag.used_size = sz;
596 return mp;
597 }
598
599 return erts_realloc_shrink_message(mp, sz, brefs, brefs_size);
600 }
601
erts_free_message(ErtsMessage * mp)602 ERTS_GLB_FORCE_INLINE void erts_free_message(ErtsMessage *mp)
603 {
604 if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG)
605 erts_free_message_ref(mp);
606 else
607 erts_free(ERTS_ALC_T_MSG, mp);
608 }
609
erts_used_frag_sz(const ErlHeapFragment * bp)610 ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment* bp)
611 {
612 Uint sz = 0;
613 for ( ; bp!=NULL; bp=bp->next) {
614 sz += bp->used_size;
615 }
616 return sz;
617 }
618
erts_msg_attached_data_size(ErtsMessage * msg)619 ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg)
620 {
621 ASSERT(msg->data.attached);
622
623 if (ERTS_SIG_IS_INTERNAL_MSG(msg))
624 return erts_used_frag_sz(erts_message_to_heap_frag(msg));
625
626 return erts_msg_attached_data_size_aux(msg);
627 }
628
629 #endif
630
631 Uint erts_mbuf_size(Process *p);
632 #if defined(DEBUG) || 0
633 # define ERTS_CHK_MBUF_SZ(P) \
634 do { \
635 Uint actual_mbuf_sz__ = erts_mbuf_size((P)); \
636 ERTS_ASSERT((P)->mbuf_sz >= actual_mbuf_sz__); \
637 } while (0)
638 #else
639 # define ERTS_CHK_MBUF_SZ(P) ((void) 1)
640 #endif
641
642 #define ERTS_FOREACH_SIG_PRIVQS(PROC, MVAR, CODE) \
643 do { \
644 int i__; \
645 ErtsMessage *msgs__[] = {(PROC)->sig_qs.first, \
646 (PROC)->sig_qs.cont}; \
647 for (i__ = 0; i__ < sizeof(msgs__)/sizeof(msgs__[0]); i__++) { \
648 ErtsMessage *MVAR; \
649 for (MVAR = msgs__[i__]; MVAR; MVAR = MVAR->next) { \
650 CODE; \
651 } \
652 } \
653 } while (0)
654
655 #endif
656