6 void factorvm::init_data_gc()
9 last_code_heap_scan = data->nursery();
10 collecting_aging_again = false;
14 /* Given a pointer to oldspace, copy it to newspace */
15 object *factorvm::copy_untagged_object_impl(object *pointer, cell size)
17 if(newspace->here + size >= newspace->end)
19 object *newpointer = allot_zone(newspace,size);
21 gc_stats *s = &stats[collecting_gen];
23 s->bytes_copied += size;
25 memcpy(newpointer,pointer,size);
30 object *factorvm::copy_object_impl(object *untagged)
32 object *newpointer = copy_untagged_object_impl(untagged,untagged_object_size(untagged));
33 untagged->h.forward_to(newpointer);
38 bool factorvm::should_copy_p(object *untagged)
40 if(in_zone(newspace,untagged))
42 if(collecting_gen == data->tenured())
44 else if(data->have_aging_p() && collecting_gen == data->aging())
45 return !in_zone(&data->generations[data->tenured()],untagged);
46 else if(collecting_gen == data->nursery())
47 return in_zone(&nursery,untagged);
50 critical_error("Bug in should_copy_p",(cell)untagged);
56 /* Follow a chain of forwarding pointers */
57 object *factorvm::resolve_forwarding(object *untagged)
59 check_data_pointer(untagged);
61 /* is there another forwarding pointer? */
62 if(untagged->h.forwarding_pointer_p())
63 return resolve_forwarding(untagged->h.forwarding_pointer());
64 /* we've found the destination */
67 untagged->h.check_header();
68 if(should_copy_p(untagged))
69 return copy_object_impl(untagged);
76 template <typename TYPE> TYPE *factorvm::copy_untagged_object(TYPE *untagged)
78 check_data_pointer(untagged);
80 if(untagged->h.forwarding_pointer_p())
81 untagged = (TYPE *)resolve_forwarding(untagged->h.forwarding_pointer());
84 untagged->h.check_header();
85 untagged = (TYPE *)copy_object_impl(untagged);
92 cell factorvm::copy_object(cell pointer)
94 return RETAG(copy_untagged_object(untag<object>(pointer)),TAG(pointer));
98 void factorvm::copy_handle(cell *handle)
100 cell pointer = *handle;
102 if(!immediate_p(pointer))
104 object *obj = untag<object>(pointer);
105 check_data_pointer(obj);
106 if(should_copy_p(obj))
107 *handle = copy_object(pointer);
112 /* Scan all the objects in the card */
113 void factorvm::copy_card(card *ptr, cell gen, cell here)
115 cell card_scan = card_to_addr(ptr) + card_offset(ptr);
116 cell card_end = card_to_addr(ptr + 1);
121 copy_reachable_objects(card_scan,&card_end);
127 void factorvm::copy_card_deck(card_deck *deck, cell gen, card mask, card unmask)
129 card *first_card = deck_to_card(deck);
130 card *last_card = deck_to_card(deck + 1);
132 cell here = data->generations[gen].here;
135 u32 quad_mask = mask | (mask << 8) | (mask << 16) | (mask << 24);
137 for(quad_ptr = (u32 *)first_card; quad_ptr < (u32 *)last_card; quad_ptr++)
139 if(*quad_ptr & quad_mask)
141 card *ptr = (card *)quad_ptr;
144 for(card = 0; card < 4; card++)
148 copy_card(&ptr[card],gen,here);
149 ptr[card] &= ~unmask;
159 /* Copy all newspace objects referenced from marked cards to the destination */
160 void factorvm::copy_gen_cards(cell gen)
162 card_deck *first_deck = addr_to_deck(data->generations[gen].start);
163 card_deck *last_deck = addr_to_deck(data->generations[gen].end);
167 /* if we are collecting the nursery, we care about old->nursery pointers
168 but not old->aging pointers */
169 if(collecting_gen == data->nursery())
171 mask = card_points_to_nursery;
173 /* after the collection, no old->nursery pointers remain
174 anywhere, but old->aging pointers might remain in tenured
176 if(gen == data->tenured())
177 unmask = card_points_to_nursery;
178 /* after the collection, all cards in aging space can be
180 else if(data->have_aging_p() && gen == data->aging())
181 unmask = card_mark_mask;
184 critical_error("bug in copy_gen_cards",gen);
188 /* if we are collecting aging space into tenured space, we care about
189 all old->nursery and old->aging pointers. no old->aging pointers can
191 else if(data->have_aging_p() && collecting_gen == data->aging())
193 if(collecting_aging_again)
195 mask = card_points_to_aging;
196 unmask = card_mark_mask;
198 /* after we collect aging space into the aging semispace, no
199 old->nursery pointers remain but tenured space might still have
200 pointers to aging space. */
203 mask = card_points_to_aging;
204 unmask = card_points_to_nursery;
209 critical_error("bug in copy_gen_cards",gen);
215 for(ptr = first_deck; ptr < last_deck; ptr++)
219 copy_card_deck(ptr,gen,mask,unmask);
226 /* Scan cards in all generations older than the one being collected, copying
227 old->new references */
228 void factorvm::copy_cards()
230 u64 start = current_micros();
233 for(i = collecting_gen + 1; i < data->gen_count; i++)
236 card_scan_time += (current_micros() - start);
240 /* Copy all tagged pointers in a range of memory */
241 void factorvm::copy_stack_elements(segment *region, cell top)
243 cell ptr = region->start;
245 for(; ptr <= top; ptr += sizeof(cell))
246 copy_handle((cell*)ptr);
250 void factorvm::copy_registered_locals()
252 std::vector<cell>::const_iterator iter = gc_locals.begin();
253 std::vector<cell>::const_iterator end = gc_locals.end();
255 for(; iter < end; iter++)
256 copy_handle((cell *)(*iter));
260 void factorvm::copy_registered_bignums()
262 std::vector<cell>::const_iterator iter = gc_bignums.begin();
263 std::vector<cell>::const_iterator end = gc_bignums.end();
265 for(; iter < end; iter++)
267 bignum **handle = (bignum **)(*iter);
268 bignum *pointer = *handle;
272 check_data_pointer(pointer);
273 if(should_copy_p(pointer))
274 *handle = copy_untagged_object(pointer);
276 assert((*handle)->h.hi_tag() == BIGNUM_TYPE);
283 /* Copy roots over at the start of GC, namely various constants, stacks,
284 the user environment and extra roots registered by local_roots.hpp */
285 void factorvm::copy_roots()
288 copy_handle(&bignum_zero);
289 copy_handle(&bignum_pos_one);
290 copy_handle(&bignum_neg_one);
292 copy_registered_locals();
293 copy_registered_bignums();
295 if(!performing_compaction)
298 context *stacks = stack_chain;
302 copy_stack_elements(stacks->datastack_region,stacks->datastack);
303 copy_stack_elements(stacks->retainstack_region,stacks->retainstack);
305 copy_handle(&stacks->catchstack_save);
306 copy_handle(&stacks->current_callback_save);
308 mark_active_blocks(stacks);
310 stacks = stacks->next;
315 for(i = 0; i < USER_ENV; i++)
316 copy_handle(&userenv[i]);
320 cell factorvm::copy_next_from_nursery(cell scan)
322 cell *obj = (cell *)scan;
323 cell *end = (cell *)(scan + binary_payload_start((object *)scan));
329 cell nursery_start = nursery.start;
330 cell nursery_end = nursery.end;
332 for(; obj < end; obj++)
336 if(!immediate_p(pointer))
338 check_data_pointer((object *)pointer);
339 if(pointer >= nursery_start && pointer < nursery_end)
340 *obj = copy_object(pointer);
345 return scan + untagged_object_size((object *)scan);
349 cell factorvm::copy_next_from_aging(cell scan)
351 cell *obj = (cell *)scan;
352 cell *end = (cell *)(scan + binary_payload_start((object *)scan));
358 cell tenured_start = data->generations[data->tenured()].start;
359 cell tenured_end = data->generations[data->tenured()].end;
361 cell newspace_start = newspace->start;
362 cell newspace_end = newspace->end;
364 for(; obj < end; obj++)
368 if(!immediate_p(pointer))
370 check_data_pointer((object *)pointer);
371 if(!(pointer >= newspace_start && pointer < newspace_end)
372 && !(pointer >= tenured_start && pointer < tenured_end))
373 *obj = copy_object(pointer);
378 return scan + untagged_object_size((object *)scan);
382 cell factorvm::copy_next_from_tenured(cell scan)
384 cell *obj = (cell *)scan;
385 cell *end = (cell *)(scan + binary_payload_start((object *)scan));
391 cell newspace_start = newspace->start;
392 cell newspace_end = newspace->end;
394 for(; obj < end; obj++)
398 if(!immediate_p(pointer))
400 check_data_pointer((object *)pointer);
401 if(!(pointer >= newspace_start && pointer < newspace_end))
402 *obj = copy_object(pointer);
407 mark_object_code_block((object *)scan);
409 return scan + untagged_object_size((object *)scan);
413 void factorvm::copy_reachable_objects(cell scan, cell *end)
415 if(collecting_gen == data->nursery())
418 scan = copy_next_from_nursery(scan);
420 else if(data->have_aging_p() && collecting_gen == data->aging())
423 scan = copy_next_from_aging(scan);
425 else if(collecting_gen == data->tenured())
428 scan = copy_next_from_tenured(scan);
433 /* Prepare to start copying reachable objects into an unused zone */
434 void factorvm::begin_gc(cell requested_bytes)
436 if(growing_data_heap)
438 if(collecting_gen != data->tenured())
439 critical_error("Invalid parameters to begin_gc",0);
441 old_data_heap = data;
442 set_data_heap(grow_data_heap(old_data_heap,requested_bytes));
443 newspace = &data->generations[data->tenured()];
445 else if(collecting_accumulation_gen_p())
447 /* when collecting one of these generations, rotate it
448 with the semispace */
449 zone z = data->generations[collecting_gen];
450 data->generations[collecting_gen] = data->semispaces[collecting_gen];
451 data->semispaces[collecting_gen] = z;
452 reset_generation(collecting_gen);
453 newspace = &data->generations[collecting_gen];
454 clear_cards(collecting_gen,collecting_gen);
455 clear_decks(collecting_gen,collecting_gen);
456 clear_allot_markers(collecting_gen,collecting_gen);
460 /* when collecting a younger generation, we copy
461 reachable objects to the next oldest generation,
462 so we set the newspace so the next generation. */
463 newspace = &data->generations[collecting_gen + 1];
468 void factorvm::end_gc(cell gc_elapsed)
470 gc_stats *s = &stats[collecting_gen];
473 s->gc_time += gc_elapsed;
474 if(s->max_gc_time < gc_elapsed)
475 s->max_gc_time = gc_elapsed;
477 if(growing_data_heap)
479 dealloc_data_heap(old_data_heap);
480 old_data_heap = NULL;
481 growing_data_heap = false;
484 if(collecting_accumulation_gen_p())
486 /* all younger generations except are now empty.
487 if collecting_gen == data->nursery() here, we only have 1 generation;
488 old-school Cheney collector */
489 if(collecting_gen != data->nursery())
490 reset_generations(data->nursery(),collecting_gen - 1);
492 else if(collecting_gen == data->nursery())
494 nursery.here = nursery.start;
498 /* all generations up to and including the one
499 collected are now empty */
500 reset_generations(data->nursery(),collecting_gen);
503 collecting_aging_again = false;
507 /* Collect gen and all younger generations.
508 If growing_data_heap_ is true, we must grow the data heap to such a size that
509 an allocation of requested_bytes won't fail */
510 void factorvm::garbage_collection(cell gen,bool growing_data_heap_,cell requested_bytes)
514 critical_error("GC disabled",gen);
518 u64 start = current_micros();
520 performing_gc = true;
521 growing_data_heap = growing_data_heap_;
522 collecting_gen = gen;
524 /* we come back here if a generation is full */
527 /* We have no older generations we can try collecting, so we
528 resort to growing the data heap */
529 if(collecting_gen == data->tenured())
531 growing_data_heap = true;
533 /* see the comment in unmark_marked() */
534 unmark_marked(&code);
536 /* we try collecting aging space twice before going on to
538 else if(data->have_aging_p()
539 && collecting_gen == data->aging()
540 && !collecting_aging_again)
542 collecting_aging_again = true;
544 /* Collect the next oldest generation */
551 begin_gc(requested_bytes);
553 /* initialize chase pointer */
554 cell scan = newspace->here;
556 /* collect objects referenced from stacks and environment */
558 /* collect objects referenced from older generations */
561 /* do some tracing */
562 copy_reachable_objects(scan,&newspace->here);
564 /* don't scan code heap unless it has pointers to this
565 generation or younger */
566 if(collecting_gen >= last_code_heap_scan)
570 if(collecting_gen == data->tenured())
571 free_unmarked(&code,(heap_iterator)factor::update_literal_and_word_references);
573 copy_code_heap_roots();
575 if(collecting_accumulation_gen_p())
576 last_code_heap_scan = collecting_gen;
578 last_code_heap_scan = collecting_gen + 1;
581 cell gc_elapsed = (current_micros() - start);
585 performing_gc = false;
591 garbage_collection(data->tenured(),false,0);
595 inline void factorvm::vmprim_gc()
602 PRIMITIVE_GETVM()->vmprim_gc();
605 inline void factorvm::vmprim_gc_stats()
607 growable_array result(this);
610 u64 total_gc_time = 0;
612 for(i = 0; i < max_gen_count; i++)
614 gc_stats *s = &stats[i];
615 result.add(allot_cell(s->collections));
616 result.add(tag<bignum>(long_long_to_bignum(s->gc_time)));
617 result.add(tag<bignum>(long_long_to_bignum(s->max_gc_time)));
618 result.add(allot_cell(s->collections == 0 ? 0 : s->gc_time / s->collections));
619 result.add(allot_cell(s->object_count));
620 result.add(tag<bignum>(long_long_to_bignum(s->bytes_copied)));
622 total_gc_time += s->gc_time;
625 result.add(tag<bignum>(ulong_long_to_bignum(total_gc_time)));
626 result.add(tag<bignum>(ulong_long_to_bignum(cards_scanned)));
627 result.add(tag<bignum>(ulong_long_to_bignum(decks_scanned)));
628 result.add(tag<bignum>(ulong_long_to_bignum(card_scan_time)));
629 result.add(allot_cell(code_heap_scans));
632 dpush(result.elements.value());
637 PRIMITIVE_GETVM()->vmprim_gc_stats();
640 void factorvm::clear_gc_stats()
642 for(cell i = 0; i < max_gen_count; i++)
643 memset(&stats[i],0,sizeof(gc_stats));
651 inline void factorvm::vmprim_clear_gc_stats()
656 PRIMITIVE(clear_gc_stats)
658 PRIMITIVE_GETVM()->vmprim_clear_gc_stats();
661 /* classes.tuple uses this to reshape tuples; tools.deploy.shaker uses this
662 to coalesce equal but distinct quotations and wrappers. */
663 inline void factorvm::vmprim_become()
665 array *new_objects = untag_check<array>(dpop());
666 array *old_objects = untag_check<array>(dpop());
668 cell capacity = array_capacity(new_objects);
669 if(capacity != array_capacity(old_objects))
670 critical_error("bad parameters to become",0);
674 for(i = 0; i < capacity; i++)
676 tagged<object> old_obj(array_nth(old_objects,i));
677 tagged<object> new_obj(array_nth(new_objects,i));
679 if(old_obj != new_obj)
680 old_obj->h.forward_to(new_obj.untagged());
685 /* If a word's definition quotation was in old_objects and the
686 quotation in new_objects is not compiled, we might leak memory
687 by referencing the old quotation unless we recompile all
688 unoptimized words. */
694 PRIMITIVE_GETVM()->vmprim_become();
697 void factorvm::inline_gc(cell *gc_roots_base, cell gc_roots_size)
699 for(cell i = 0; i < gc_roots_size; i++)
700 gc_locals.push_back((cell)&gc_roots_base[i]);
702 garbage_collection(data->nursery(),false,0);
704 for(cell i = 0; i < gc_roots_size; i++)
705 gc_locals.pop_back();
708 VM_ASM_API void inline_gc(cell *gc_roots_base, cell gc_roots_size, factorvm *myvm)
711 return VM_PTR->inline_gc(gc_roots_base,gc_roots_size);