1 /**
2 * \file
3 * Large objects space.
4 *
5 * Author:
6 * Paolo Molaro (lupus@ximian.com)
7 *
8 * Copyright 2005-2010 Novell, Inc (http://www.novell.com)
9 *
10 * Thread start/stop adapted from Boehm's GC:
11 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
12 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
13 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
14 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
15 * Copyright 2001-2003 Ximian, Inc
16 * Copyright 2003-2010 Novell, Inc.
17 * Copyright (C) 2012 Xamarin Inc
18 *
19 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
20 */
21
22 #include "config.h"
23
24 #ifdef HAVE_SGEN_GC
25
26 #include <string.h>
27
28 #include "mono/sgen/sgen-gc.h"
29 #include "mono/sgen/sgen-protocol.h"
30 #include "mono/sgen/sgen-cardtable.h"
31 #include "mono/sgen/sgen-memory-governor.h"
32 #include "mono/sgen/sgen-client.h"
33
34 #define LOS_SECTION_SIZE (1024 * 1024)
35
36 /*
37 * This shouldn't be much smaller or larger than MAX_SMALL_OBJ_SIZE.
38 * Must be at least sizeof (LOSSection).
39 */
40 #define LOS_CHUNK_SIZE 4096
41 #define LOS_CHUNK_BITS 12
42
43 /* Largest object that can be allocated in a section. */
44 #define LOS_SECTION_OBJECT_LIMIT (LOS_SECTION_SIZE - LOS_CHUNK_SIZE - sizeof (LOSObject))
45 //#define LOS_SECTION_OBJECT_LIMIT 0
46 #define LOS_SECTION_NUM_CHUNKS ((LOS_SECTION_SIZE >> LOS_CHUNK_BITS) - 1)
47
48 #define LOS_SECTION_FOR_OBJ(obj) ((LOSSection*)((mword)(obj) & ~(mword)(LOS_SECTION_SIZE - 1)))
49 #define LOS_CHUNK_INDEX(obj,section) (((char*)(obj) - (char*)(section)) >> LOS_CHUNK_BITS)
50
51 #define LOS_NUM_FAST_SIZES 32
52
53 typedef struct _LOSFreeChunks LOSFreeChunks;
54 struct _LOSFreeChunks {
55 LOSFreeChunks *next_size;
56 size_t size;
57 };
58
59 typedef struct _LOSSection LOSSection;
60 struct _LOSSection {
61 LOSSection *next;
62 size_t num_free_chunks;
63 unsigned char *free_chunk_map;
64 };
65
66 /* We allow read only access on the list while sweep is not running */
67 LOSObject *los_object_list = NULL;
68 /* Memory used by LOS objects */
69 mword los_memory_usage = 0;
70 /* Total memory used by the LOS allocator */
71 mword los_memory_usage_total = 0;
72
73 static LOSSection *los_sections = NULL;
74 static LOSFreeChunks *los_fast_free_lists [LOS_NUM_FAST_SIZES]; /* 0 is for larger sizes */
75 static mword los_num_objects = 0;
76 static int los_num_sections = 0;
77
78 //#define USE_MALLOC
79 //#define LOS_CONSISTENCY_CHECK
80 //#define LOS_DUMMY
81
82 #ifdef LOS_DUMMY
83 #define LOS_SEGMENT_SIZE (4096 * 1024)
84
85 static char *los_segment = NULL;
86 static int los_segment_index = 0;
87 #endif
88
89 mword
sgen_los_object_size(LOSObject * obj)90 sgen_los_object_size (LOSObject *obj)
91 {
92 return obj->size & ~1L;
93 }
94
95 #ifdef LOS_CONSISTENCY_CHECK
96 static void
los_consistency_check(void)97 los_consistency_check (void)
98 {
99 LOSSection *section;
100 LOSObject *obj;
101 int i;
102 mword memory_usage = 0;
103
104 for (obj = los_object_list; obj; obj = obj->next) {
105 mword obj_size = sgen_los_object_size (obj);
106 char *end = obj->data + obj_size;
107 int start_index, num_chunks;
108
109 memory_usage += obj_size;
110
111 if (obj_size > LOS_SECTION_OBJECT_LIMIT)
112 continue;
113
114 section = LOS_SECTION_FOR_OBJ (obj);
115
116 g_assert (end <= (char*)section + LOS_SECTION_SIZE);
117
118 start_index = LOS_CHUNK_INDEX (obj, section);
119 num_chunks = (obj_size + sizeof (LOSObject) + LOS_CHUNK_SIZE - 1) >> LOS_CHUNK_BITS;
120 for (i = start_index; i < start_index + num_chunks; ++i)
121 g_assert (!section->free_chunk_map [i]);
122 }
123
124 for (i = 0; i < LOS_NUM_FAST_SIZES; ++i) {
125 LOSFreeChunks *size_chunks;
126 for (size_chunks = los_fast_free_lists [i]; size_chunks; size_chunks = size_chunks->next_size) {
127 LOSSection *section = LOS_SECTION_FOR_OBJ (size_chunks);
128 int j, num_chunks, start_index;
129
130 if (i == 0)
131 g_assert (size_chunks->size >= LOS_NUM_FAST_SIZES * LOS_CHUNK_SIZE);
132 else
133 g_assert (size_chunks->size == i * LOS_CHUNK_SIZE);
134
135 num_chunks = size_chunks->size >> LOS_CHUNK_BITS;
136 start_index = LOS_CHUNK_INDEX (size_chunks, section);
137 for (j = start_index; j < start_index + num_chunks; ++j)
138 g_assert (section->free_chunk_map [j]);
139 }
140 }
141
142 g_assert (los_memory_usage == memory_usage);
143 }
144 #endif
145
146 static void
add_free_chunk(LOSFreeChunks * free_chunks,size_t size)147 add_free_chunk (LOSFreeChunks *free_chunks, size_t size)
148 {
149 size_t num_chunks = size >> LOS_CHUNK_BITS;
150
151 free_chunks->size = size;
152
153 if (num_chunks >= LOS_NUM_FAST_SIZES)
154 num_chunks = 0;
155 free_chunks->next_size = los_fast_free_lists [num_chunks];
156 los_fast_free_lists [num_chunks] = free_chunks;
157 }
158
159 static LOSFreeChunks*
get_from_size_list(LOSFreeChunks ** list,size_t size)160 get_from_size_list (LOSFreeChunks **list, size_t size)
161 {
162 LOSFreeChunks *free_chunks = NULL;
163 LOSSection *section;
164 size_t i, num_chunks, start_index;
165
166
167 g_assert ((size & (LOS_CHUNK_SIZE - 1)) == 0);
168
169 while (*list) {
170 free_chunks = *list;
171 if (free_chunks->size >= size)
172 break;
173 list = &(*list)->next_size;
174 }
175
176 if (!*list)
177 return NULL;
178
179 *list = free_chunks->next_size;
180
181 if (free_chunks->size > size)
182 add_free_chunk ((LOSFreeChunks*)((char*)free_chunks + size), free_chunks->size - size);
183
184 num_chunks = size >> LOS_CHUNK_BITS;
185
186 section = LOS_SECTION_FOR_OBJ (free_chunks);
187
188 start_index = LOS_CHUNK_INDEX (free_chunks, section);
189 for (i = start_index; i < start_index + num_chunks; ++i) {
190 g_assert (section->free_chunk_map [i]);
191 section->free_chunk_map [i] = 0;
192 }
193
194 section->num_free_chunks -= size >> LOS_CHUNK_BITS;
195 g_assert (section->num_free_chunks >= 0);
196
197 return free_chunks;
198 }
199
200 static LOSObject*
randomize_los_object_start(gpointer addr,size_t obj_size,size_t alloced_size,size_t addr_alignment)201 randomize_los_object_start (gpointer addr, size_t obj_size, size_t alloced_size, size_t addr_alignment)
202 {
203 size_t offset = 0;
204 if (alloced_size != obj_size) {
205 /*
206 * We want to get a random offset between 0 and (alloced_size - obj_size)
207 * We do a prime multiplication to avoid usage of functions which might not
208 * be thread/signal safe (like rand ()). We subtract 1 to avoid common
209 * power by 2 factors.
210 */
211 offset = SGEN_ALIGN_DOWN ((((size_t)addr - 1) * 2654435761u) % (alloced_size - obj_size));
212 }
213 SGEN_ASSERT (0, (alloced_size - obj_size) < addr_alignment, "Why are we wasting one entire chunk for a los object ?");
214 /* Randomize the location within the reserved chunks to improve cache performance */
215 return (LOSObject*)((guint8*)addr + offset);
216
217 }
218
219 static LOSObject*
get_los_section_memory(size_t size)220 get_los_section_memory (size_t size)
221 {
222 LOSSection *section;
223 LOSFreeChunks *free_chunks;
224 size_t num_chunks;
225 size_t obj_size = size;
226
227 size = SGEN_ALIGN_UP_TO (size, LOS_CHUNK_SIZE);
228
229 num_chunks = size >> LOS_CHUNK_BITS;
230
231 g_assert (size > 0 && size - sizeof (LOSObject) <= LOS_SECTION_OBJECT_LIMIT);
232 g_assert (num_chunks > 0);
233
234 retry:
235 if (num_chunks >= LOS_NUM_FAST_SIZES) {
236 free_chunks = get_from_size_list (&los_fast_free_lists [0], size);
237 } else {
238 size_t i;
239 for (i = num_chunks; i < LOS_NUM_FAST_SIZES; ++i) {
240 free_chunks = get_from_size_list (&los_fast_free_lists [i], size);
241 if (free_chunks)
242 break;
243 }
244 if (!free_chunks)
245 free_chunks = get_from_size_list (&los_fast_free_lists [0], size);
246 }
247
248 if (free_chunks) {
249 return randomize_los_object_start (free_chunks, obj_size, size, LOS_CHUNK_SIZE);
250 }
251
252 if (!sgen_memgov_try_alloc_space (LOS_SECTION_SIZE, SPACE_LOS))
253 return NULL;
254
255 section = (LOSSection *)sgen_alloc_os_memory_aligned (LOS_SECTION_SIZE, LOS_SECTION_SIZE, (SgenAllocFlags)(SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE), NULL, MONO_MEM_ACCOUNT_SGEN_LOS);
256
257 if (!section)
258 return NULL;
259
260 free_chunks = (LOSFreeChunks*)((char*)section + LOS_CHUNK_SIZE);
261 free_chunks->size = LOS_SECTION_SIZE - LOS_CHUNK_SIZE;
262 free_chunks->next_size = los_fast_free_lists [0];
263 los_fast_free_lists [0] = free_chunks;
264
265 section->num_free_chunks = LOS_SECTION_NUM_CHUNKS;
266
267 section->free_chunk_map = (unsigned char*)section + sizeof (LOSSection);
268 g_assert (sizeof (LOSSection) + LOS_SECTION_NUM_CHUNKS + 1 <= LOS_CHUNK_SIZE);
269 section->free_chunk_map [0] = 0;
270 memset (section->free_chunk_map + 1, 1, LOS_SECTION_NUM_CHUNKS);
271
272 section->next = los_sections;
273 los_sections = section;
274
275 los_memory_usage_total += LOS_SECTION_SIZE;
276 ++los_num_sections;
277
278 goto retry;
279 }
280
281 static void
free_los_section_memory(LOSObject * obj,size_t size)282 free_los_section_memory (LOSObject *obj, size_t size)
283 {
284 LOSSection *section = LOS_SECTION_FOR_OBJ (obj);
285 size_t num_chunks, i, start_index;
286
287 size = SGEN_ALIGN_UP_TO (size, LOS_CHUNK_SIZE);
288
289 num_chunks = size >> LOS_CHUNK_BITS;
290
291 g_assert (size > 0 && size - sizeof (LOSObject) <= LOS_SECTION_OBJECT_LIMIT);
292 g_assert (num_chunks > 0);
293
294 section->num_free_chunks += num_chunks;
295 g_assert (section->num_free_chunks <= LOS_SECTION_NUM_CHUNKS);
296
297 /*
298 * We could free the LOS section here if it's empty, but we
299 * can't unless we also remove its free chunks from the fast
300 * free lists. Instead, we do it in los_sweep().
301 */
302
303 start_index = LOS_CHUNK_INDEX (obj, section);
304 for (i = start_index; i < start_index + num_chunks; ++i) {
305 g_assert (!section->free_chunk_map [i]);
306 section->free_chunk_map [i] = 1;
307 }
308
309 add_free_chunk ((LOSFreeChunks*)SGEN_ALIGN_DOWN_TO ((mword)obj, LOS_CHUNK_SIZE), size);
310 }
311
312 void
sgen_los_free_object(LOSObject * obj)313 sgen_los_free_object (LOSObject *obj)
314 {
315 if (obj->cardtable_mod_union)
316 sgen_card_table_free_mod_union (obj->cardtable_mod_union, (char*)obj->data, sgen_los_object_size (obj));
317
318 #ifndef LOS_DUMMY
319 mword size = sgen_los_object_size (obj);
320 SGEN_LOG (4, "Freed large object %p, size %lu", obj->data, (unsigned long)size);
321 binary_protocol_empty (obj->data, size);
322
323 los_memory_usage -= size;
324 los_num_objects--;
325
326 #ifdef USE_MALLOC
327 g_free (obj);
328 #else
329 if (size > LOS_SECTION_OBJECT_LIMIT) {
330 int pagesize = mono_pagesize ();
331 size += sizeof (LOSObject);
332 size = SGEN_ALIGN_UP_TO (size, pagesize);
333 sgen_free_os_memory ((gpointer)SGEN_ALIGN_DOWN_TO ((mword)obj, pagesize), size, SGEN_ALLOC_HEAP, MONO_MEM_ACCOUNT_SGEN_LOS);
334 los_memory_usage_total -= size;
335 sgen_memgov_release_space (size, SPACE_LOS);
336 } else {
337 free_los_section_memory (obj, size + sizeof (LOSObject));
338 #ifdef LOS_CONSISTENCY_CHECKS
339 los_consistency_check ();
340 #endif
341 }
342 #endif
343 #endif
344 }
345
346 /*
347 * Objects with size >= MAX_SMALL_SIZE are allocated in the large object space.
348 * They are currently kept track of with a linked list.
349 * They don't move, so there is no need to pin them during collection
350 * and we avoid the memcpy overhead.
351 */
352 void*
sgen_los_alloc_large_inner(GCVTable vtable,size_t size)353 sgen_los_alloc_large_inner (GCVTable vtable, size_t size)
354 {
355 LOSObject *obj = NULL;
356 void **vtslot;
357
358 g_assert (size > SGEN_MAX_SMALL_OBJ_SIZE);
359 g_assert ((size & 1) == 0);
360
361 /*
362 * size + sizeof (LOSObject) <= SSIZE_MAX - (mono_pagesize () - 1)
363 *
364 * therefore:
365 *
366 * size <= SSIZE_MAX - (mono_pagesize () - 1) - sizeof (LOSObject)
367 */
368 if (size > SSIZE_MAX - (mono_pagesize () - 1) - sizeof (LOSObject))
369 return NULL;
370
371 #ifdef LOS_DUMMY
372 if (!los_segment)
373 los_segment = sgen_alloc_os_memory (LOS_SEGMENT_SIZE, SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE, NULL);
374 los_segment_index = ALIGN_UP (los_segment_index);
375
376 obj = (LOSObject*)(los_segment + los_segment_index);
377 los_segment_index += size + sizeof (LOSObject);
378 g_assert (los_segment_index <= LOS_SEGMENT_SIZE);
379 #else
380 sgen_ensure_free_space (size, GENERATION_OLD);
381
382 #ifdef USE_MALLOC
383 obj = g_malloc (size + sizeof (LOSObject));
384 memset (obj, 0, size + sizeof (LOSObject));
385 #else
386 if (size > LOS_SECTION_OBJECT_LIMIT) {
387 size_t obj_size = size + sizeof (LOSObject);
388 int pagesize = mono_pagesize ();
389 size_t alloc_size = SGEN_ALIGN_UP_TO (obj_size, pagesize);
390 if (sgen_memgov_try_alloc_space (alloc_size, SPACE_LOS)) {
391 obj = (LOSObject *)sgen_alloc_os_memory (alloc_size, (SgenAllocFlags)(SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE), NULL, MONO_MEM_ACCOUNT_SGEN_LOS);
392 if (obj) {
393 los_memory_usage_total += alloc_size;
394 obj = randomize_los_object_start (obj, obj_size, alloc_size, pagesize);
395 }
396 }
397 } else {
398 obj = get_los_section_memory (size + sizeof (LOSObject));
399 if (obj)
400 memset (obj, 0, size + sizeof (LOSObject));
401 }
402 #endif
403 #endif
404 if (!obj)
405 return NULL;
406 g_assert (!((mword)obj->data & (SGEN_ALLOC_ALIGN - 1)));
407 obj->size = size;
408 vtslot = (void**)obj->data;
409 *vtslot = vtable;
410 sgen_update_heap_boundaries ((mword)obj->data, (mword)obj->data + size);
411 obj->next = los_object_list;
412 /*
413 * We need a memory barrier so we don't expose as head of the los object list
414 * a LOSObject that doesn't have its fields initialized.
415 */
416 mono_memory_write_barrier ();
417 los_object_list = obj;
418 los_memory_usage += size;
419 los_num_objects++;
420 SGEN_LOG (4, "Allocated large object %p, vtable: %p (%s), size: %zd", obj->data, vtable, sgen_client_vtable_get_name (vtable), size);
421 binary_protocol_alloc (obj->data, vtable, size, sgen_client_get_provenance ());
422
423 #ifdef LOS_CONSISTENCY_CHECK
424 los_consistency_check ();
425 #endif
426
427 return obj->data;
428 }
429
430 static void sgen_los_unpin_object (GCObject *data);
431
432 void
sgen_los_sweep(void)433 sgen_los_sweep (void)
434 {
435 LOSObject *bigobj, *prevbo;
436 LOSSection *section, *prev;
437 int i;
438 int num_sections = 0;
439
440 /* sweep the big objects list */
441 prevbo = NULL;
442 for (bigobj = los_object_list; bigobj;) {
443 SGEN_ASSERT (0, !SGEN_OBJECT_IS_PINNED (bigobj->data), "Who pinned a LOS object?");
444
445 if (sgen_los_object_is_pinned (bigobj->data)) {
446 if (bigobj->cardtable_mod_union) {
447 mword obj_size = sgen_los_object_size (bigobj);
448 mword num_cards = sgen_card_table_number_of_cards_in_range ((mword) bigobj->data, obj_size);
449 memset (bigobj->cardtable_mod_union, 0, num_cards);
450 }
451
452 sgen_los_unpin_object (bigobj->data);
453 sgen_update_heap_boundaries ((mword)bigobj->data, (mword)bigobj->data + sgen_los_object_size (bigobj));
454 } else {
455 LOSObject *to_free;
456 /* not referenced anywhere, so we can free it */
457 if (prevbo)
458 prevbo->next = bigobj->next;
459 else
460 los_object_list = bigobj->next;
461 to_free = bigobj;
462 bigobj = bigobj->next;
463 sgen_los_free_object (to_free);
464 continue;
465 }
466 prevbo = bigobj;
467 bigobj = bigobj->next;
468 }
469
470 /* Try to free memory */
471 for (i = 0; i < LOS_NUM_FAST_SIZES; ++i)
472 los_fast_free_lists [i] = NULL;
473
474 prev = NULL;
475 section = los_sections;
476 while (section) {
477 if (section->num_free_chunks == LOS_SECTION_NUM_CHUNKS) {
478 LOSSection *next = section->next;
479 if (prev)
480 prev->next = next;
481 else
482 los_sections = next;
483 sgen_free_os_memory (section, LOS_SECTION_SIZE, SGEN_ALLOC_HEAP, MONO_MEM_ACCOUNT_SGEN_LOS);
484 sgen_memgov_release_space (LOS_SECTION_SIZE, SPACE_LOS);
485 section = next;
486 --los_num_sections;
487 los_memory_usage_total -= LOS_SECTION_SIZE;
488 continue;
489 }
490
491 for (i = 0; i <= LOS_SECTION_NUM_CHUNKS; ++i) {
492 if (section->free_chunk_map [i]) {
493 int j;
494 for (j = i + 1; j <= LOS_SECTION_NUM_CHUNKS && section->free_chunk_map [j]; ++j)
495 ;
496 add_free_chunk ((LOSFreeChunks*)((char*)section + (i << LOS_CHUNK_BITS)), (j - i) << LOS_CHUNK_BITS);
497 i = j - 1;
498 }
499 }
500
501 prev = section;
502 section = section->next;
503
504 ++num_sections;
505 }
506
507 #ifdef LOS_CONSISTENCY_CHECK
508 los_consistency_check ();
509 #endif
510
511 /*
512 g_print ("LOS sections: %d objects: %d usage: %d\n", num_sections, los_num_objects, los_memory_usage);
513 for (i = 0; i < LOS_NUM_FAST_SIZES; ++i) {
514 int num_chunks = 0;
515 LOSFreeChunks *free_chunks;
516 for (free_chunks = los_fast_free_lists [i]; free_chunks; free_chunks = free_chunks->next_size)
517 ++num_chunks;
518 g_print (" %d: %d\n", i, num_chunks);
519 }
520 */
521
522 g_assert (los_num_sections == num_sections);
523 }
524
525 gboolean
sgen_ptr_is_in_los(char * ptr,char ** start)526 sgen_ptr_is_in_los (char *ptr, char **start)
527 {
528 LOSObject *obj;
529
530 if (start)
531 *start = NULL;
532 for (obj = los_object_list; obj; obj = obj->next) {
533 char *end = (char*)obj->data + sgen_los_object_size (obj);
534
535 if (ptr >= (char*)obj->data && ptr < end) {
536 if (start)
537 *start = (char*)obj->data;
538 return TRUE;
539 }
540 }
541 return FALSE;
542 }
543
544 void
sgen_los_iterate_objects(IterateObjectCallbackFunc cb,void * user_data)545 sgen_los_iterate_objects (IterateObjectCallbackFunc cb, void *user_data)
546 {
547 LOSObject *obj;
548
549 for (obj = los_object_list; obj; obj = obj->next)
550 cb (obj->data, sgen_los_object_size (obj), user_data);
551 }
552
553 gboolean
sgen_los_is_valid_object(char * object)554 sgen_los_is_valid_object (char *object)
555 {
556 LOSObject *obj;
557
558 for (obj = los_object_list; obj; obj = obj->next) {
559 if ((char*)obj->data == object)
560 return TRUE;
561 }
562 return FALSE;
563 }
564
565 gboolean
mono_sgen_los_describe_pointer(char * ptr)566 mono_sgen_los_describe_pointer (char *ptr)
567 {
568 LOSObject *obj;
569
570 for (obj = los_object_list; obj; obj = obj->next) {
571 const char *los_kind;
572 mword size;
573 gboolean pinned;
574
575 if ((char*)obj->data > ptr || (char*)obj->data + sgen_los_object_size (obj) <= ptr)
576 continue;
577
578 size = sgen_los_object_size (obj);
579 pinned = sgen_los_object_is_pinned (obj->data);
580
581 if (size > LOS_SECTION_OBJECT_LIMIT)
582 los_kind = "huge-los-ptr";
583 else
584 los_kind = "los-ptr";
585
586 if ((char*)obj->data == ptr) {
587 SGEN_LOG (0, "%s (size %d pin %d)\n", los_kind, (int)size, pinned ? 1 : 0);
588 } else {
589 SGEN_LOG (0, "%s (interior-ptr offset %zd size %d pin %d)",
590 los_kind, ptr - (char*)obj->data, (int)size, pinned ? 1 : 0);
591 }
592
593 return TRUE;
594 }
595 return FALSE;
596 }
597
598 void
sgen_los_iterate_live_block_ranges(sgen_cardtable_block_callback callback)599 sgen_los_iterate_live_block_ranges (sgen_cardtable_block_callback callback)
600 {
601 LOSObject *obj;
602 for (obj = los_object_list; obj; obj = obj->next) {
603 GCVTable vt = SGEN_LOAD_VTABLE (obj->data);
604 if (SGEN_VTABLE_HAS_REFERENCES (vt))
605 callback ((mword)obj->data, sgen_los_object_size (obj));
606 }
607 }
608
609 static guint8*
get_cardtable_mod_union_for_object(LOSObject * obj)610 get_cardtable_mod_union_for_object (LOSObject *obj)
611 {
612 mword size = sgen_los_object_size (obj);
613 guint8 *mod_union = obj->cardtable_mod_union;
614 guint8 *other;
615 if (mod_union)
616 return mod_union;
617 mod_union = sgen_card_table_alloc_mod_union ((char*)obj->data, size);
618 other = (guint8 *)SGEN_CAS_PTR ((gpointer*)&obj->cardtable_mod_union, mod_union, NULL);
619 if (!other) {
620 SGEN_ASSERT (0, obj->cardtable_mod_union == mod_union, "Why did CAS not replace?");
621 return mod_union;
622 }
623 sgen_card_table_free_mod_union (mod_union, (char*)obj->data, size);
624 return other;
625 }
626
627 void
sgen_los_scan_card_table(CardTableScanType scan_type,ScanCopyContext ctx,int job_index,int job_split_count)628 sgen_los_scan_card_table (CardTableScanType scan_type, ScanCopyContext ctx, int job_index, int job_split_count)
629 {
630 LOSObject *obj;
631 int i = 0;
632
633 binary_protocol_los_card_table_scan_start (sgen_timestamp (), scan_type & CARDTABLE_SCAN_MOD_UNION);
634 for (obj = los_object_list; obj; obj = obj->next, i++) {
635 mword num_cards = 0;
636 guint8 *cards;
637
638 if (i % job_split_count != job_index)
639 continue;
640
641 if (!SGEN_OBJECT_HAS_REFERENCES (obj->data))
642 continue;
643
644 if (scan_type & CARDTABLE_SCAN_MOD_UNION) {
645 if (!sgen_los_object_is_pinned (obj->data))
646 continue;
647
648 if (!obj->cardtable_mod_union)
649 continue;
650
651 cards = get_cardtable_mod_union_for_object (obj);
652 g_assert (cards);
653 if (scan_type == CARDTABLE_SCAN_MOD_UNION_PRECLEAN) {
654 guint8 *cards_preclean;
655 mword obj_size = sgen_los_object_size (obj);
656 num_cards = sgen_card_table_number_of_cards_in_range ((mword) obj->data, obj_size);
657 cards_preclean = (guint8 *)sgen_alloc_internal_dynamic (num_cards, INTERNAL_MEM_CARDTABLE_MOD_UNION, TRUE);
658
659 sgen_card_table_preclean_mod_union (cards, cards_preclean, num_cards);
660
661 cards = cards_preclean;
662 }
663 } else {
664 cards = NULL;
665 }
666
667 sgen_cardtable_scan_object (obj->data, sgen_los_object_size (obj), cards, ctx);
668
669 if (scan_type == CARDTABLE_SCAN_MOD_UNION_PRECLEAN)
670 sgen_free_internal_dynamic (cards, num_cards, INTERNAL_MEM_CARDTABLE_MOD_UNION);
671 }
672 binary_protocol_los_card_table_scan_end (sgen_timestamp (), scan_type & CARDTABLE_SCAN_MOD_UNION);
673 }
674
675 void
sgen_los_count_cards(long long * num_total_cards,long long * num_marked_cards)676 sgen_los_count_cards (long long *num_total_cards, long long *num_marked_cards)
677 {
678 LOSObject *obj;
679 long long total_cards = 0;
680 long long marked_cards = 0;
681
682 for (obj = los_object_list; obj; obj = obj->next) {
683 int i;
684 guint8 *cards = sgen_card_table_get_card_scan_address ((mword) obj->data);
685 guint8 *cards_end = sgen_card_table_get_card_scan_address ((mword) obj->data + sgen_los_object_size (obj) - 1);
686 mword num_cards = (cards_end - cards) + 1;
687
688 if (!SGEN_OBJECT_HAS_REFERENCES (obj->data))
689 continue;
690
691 total_cards += num_cards;
692 for (i = 0; i < num_cards; ++i) {
693 if (cards [i])
694 ++marked_cards;
695 }
696 }
697
698 *num_total_cards = total_cards;
699 *num_marked_cards = marked_cards;
700 }
701
702 void
sgen_los_update_cardtable_mod_union(void)703 sgen_los_update_cardtable_mod_union (void)
704 {
705 LOSObject *obj;
706
707 for (obj = los_object_list; obj; obj = obj->next) {
708 if (!SGEN_OBJECT_HAS_REFERENCES (obj->data))
709 continue;
710 sgen_card_table_update_mod_union (get_cardtable_mod_union_for_object (obj),
711 (char*)obj->data, sgen_los_object_size (obj), NULL);
712 }
713 }
714
715 LOSObject*
sgen_los_header_for_object(GCObject * data)716 sgen_los_header_for_object (GCObject *data)
717 {
718 #if _MSC_VER
719 return (LOSObject*)((char*)data - (int)(&(((LOSObject*)0)->data)));
720 #else
721 return (LOSObject*)((char*)data - sizeof (LOSObject));
722 #endif
723 }
724
725 void
sgen_los_pin_object(GCObject * data)726 sgen_los_pin_object (GCObject *data)
727 {
728 LOSObject *obj = sgen_los_header_for_object (data);
729 obj->size = obj->size | 1;
730 binary_protocol_pin (data, (gpointer)SGEN_LOAD_VTABLE (data), sgen_safe_object_get_size (data));
731 }
732
733 gboolean
sgen_los_pin_object_par(GCObject * data)734 sgen_los_pin_object_par (GCObject *data)
735 {
736 LOSObject *obj = sgen_los_header_for_object (data);
737 mword old_size = obj->size;
738 if (old_size & 1)
739 return FALSE;
740 #if SIZEOF_VOID_P == 4
741 old_size = mono_atomic_cas_i32 ((volatile gint32*)&obj->size, old_size | 1, old_size);
742 #else
743 old_size = mono_atomic_cas_i64 ((volatile gint64*)&obj->size, old_size | 1, old_size);
744 #endif
745 if (old_size & 1)
746 return FALSE;
747 binary_protocol_pin (data, (gpointer)SGEN_LOAD_VTABLE (data), sgen_safe_object_get_size (data));
748 return TRUE;
749 }
750
751 static void
sgen_los_unpin_object(GCObject * data)752 sgen_los_unpin_object (GCObject *data)
753 {
754 LOSObject *obj = sgen_los_header_for_object (data);
755 obj->size = sgen_los_object_size (obj);
756 }
757
758 gboolean
sgen_los_object_is_pinned(GCObject * data)759 sgen_los_object_is_pinned (GCObject *data)
760 {
761 LOSObject *obj = sgen_los_header_for_object (data);
762 return obj->size & 1;
763 }
764
765 void
sgen_los_mark_mod_union_card(GCObject * mono_obj,void ** ptr)766 sgen_los_mark_mod_union_card (GCObject *mono_obj, void **ptr)
767 {
768 LOSObject *obj = sgen_los_header_for_object (mono_obj);
769 guint8 *mod_union = get_cardtable_mod_union_for_object (obj);
770 /* The LOSObject structure is not represented within the card space */
771 size_t offset = sgen_card_table_get_card_offset ((char*)ptr, (char*)sgen_card_table_align_pointer((char*)mono_obj));
772 SGEN_ASSERT (0, mod_union, "FIXME: optionally allocate the mod union if it's not here and CAS it in.");
773 mod_union [offset] = 1;
774 }
775
776 #endif /* HAVE_SGEN_GC */
777