6 void factor_vm::init_data_gc()
9 last_code_heap_scan = data->nursery();
10 collecting_aging_again = false;
13 /* Given a pointer to oldspace, copy it to newspace */
14 object *factor_vm::copy_untagged_object_impl(object *pointer, cell size)
16 if(newspace->here + size >= newspace->end)
18 object *newpointer = allot_zone(newspace,size);
20 gc_stats *s = &stats[collecting_gen];
22 s->bytes_copied += size;
24 memcpy(newpointer,pointer,size);
28 object *factor_vm::copy_object_impl(object *untagged)
30 object *newpointer = copy_untagged_object_impl(untagged,untagged_object_size(untagged));
31 untagged->h.forward_to(newpointer);
35 bool factor_vm::should_copy_p(object *untagged)
37 if(in_zone(newspace,untagged))
39 if(collecting_gen == data->tenured())
41 else if(data->have_aging_p() && collecting_gen == data->aging())
42 return !in_zone(&data->generations[data->tenured()],untagged);
43 else if(collecting_gen == data->nursery())
44 return in_zone(&nursery,untagged);
47 critical_error("Bug in should_copy_p",(cell)untagged);
52 /* Follow a chain of forwarding pointers */
53 object *factor_vm::resolve_forwarding(object *untagged)
55 check_data_pointer(untagged);
57 /* is there another forwarding pointer? */
58 if(untagged->h.forwarding_pointer_p())
59 return resolve_forwarding(untagged->h.forwarding_pointer());
60 /* we've found the destination */
63 untagged->h.check_header();
64 if(should_copy_p(untagged))
65 return copy_object_impl(untagged);
71 template <typename TYPE> TYPE *factor_vm::copy_untagged_object(TYPE *untagged)
73 check_data_pointer(untagged);
75 if(untagged->h.forwarding_pointer_p())
76 untagged = (TYPE *)resolve_forwarding(untagged->h.forwarding_pointer());
79 untagged->h.check_header();
80 untagged = (TYPE *)copy_object_impl(untagged);
86 cell factor_vm::copy_object(cell pointer)
88 return RETAG(copy_untagged_object(untag<object>(pointer)),TAG(pointer));
91 void factor_vm::copy_handle(cell *handle)
93 cell pointer = *handle;
95 if(!immediate_p(pointer))
97 object *obj = untag<object>(pointer);
98 check_data_pointer(obj);
99 if(should_copy_p(obj))
100 *handle = copy_object(pointer);
104 /* Scan all the objects in the card */
105 void factor_vm::copy_card(card *ptr, cell gen, cell here)
107 cell card_scan = card_to_addr(ptr) + card_offset(ptr);
108 cell card_end = card_to_addr(ptr + 1);
113 copy_reachable_objects(card_scan,&card_end);
118 void factor_vm::copy_card_deck(card_deck *deck, cell gen, card mask, card unmask)
120 card *first_card = deck_to_card(deck);
121 card *last_card = deck_to_card(deck + 1);
123 cell here = data->generations[gen].here;
126 u32 quad_mask = mask | (mask << 8) | (mask << 16) | (mask << 24);
128 for(quad_ptr = (u32 *)first_card; quad_ptr < (u32 *)last_card; quad_ptr++)
130 if(*quad_ptr & quad_mask)
132 card *ptr = (card *)quad_ptr;
135 for(card = 0; card < 4; card++)
139 copy_card(&ptr[card],gen,here);
140 ptr[card] &= ~unmask;
149 /* Copy all newspace objects referenced from marked cards to the destination */
150 void factor_vm::copy_gen_cards(cell gen)
152 card_deck *first_deck = addr_to_deck(data->generations[gen].start);
153 card_deck *last_deck = addr_to_deck(data->generations[gen].end);
157 /* if we are collecting the nursery, we care about old->nursery pointers
158 but not old->aging pointers */
159 if(collecting_gen == data->nursery())
161 mask = card_points_to_nursery;
163 /* after the collection, no old->nursery pointers remain
164 anywhere, but old->aging pointers might remain in tenured
166 if(gen == data->tenured())
167 unmask = card_points_to_nursery;
168 /* after the collection, all cards in aging space can be
170 else if(data->have_aging_p() && gen == data->aging())
171 unmask = card_mark_mask;
174 critical_error("bug in copy_gen_cards",gen);
178 /* if we are collecting aging space into tenured space, we care about
179 all old->nursery and old->aging pointers. no old->aging pointers can
181 else if(data->have_aging_p() && collecting_gen == data->aging())
183 if(collecting_aging_again)
185 mask = card_points_to_aging;
186 unmask = card_mark_mask;
188 /* after we collect aging space into the aging semispace, no
189 old->nursery pointers remain but tenured space might still have
190 pointers to aging space. */
193 mask = card_points_to_aging;
194 unmask = card_points_to_nursery;
199 critical_error("bug in copy_gen_cards",gen);
205 for(ptr = first_deck; ptr < last_deck; ptr++)
209 copy_card_deck(ptr,gen,mask,unmask);
215 /* Scan cards in all generations older than the one being collected, copying
216 old->new references */
217 void factor_vm::copy_cards()
219 u64 start = current_micros();
222 for(i = collecting_gen + 1; i < data->gen_count; i++)
225 card_scan_time += (current_micros() - start);
228 /* Copy all tagged pointers in a range of memory */
229 void factor_vm::copy_stack_elements(segment *region, cell top)
231 cell ptr = region->start;
233 for(; ptr <= top; ptr += sizeof(cell))
234 copy_handle((cell*)ptr);
237 void factor_vm::copy_registered_locals()
239 std::vector<cell>::const_iterator iter = gc_locals.begin();
240 std::vector<cell>::const_iterator end = gc_locals.end();
242 for(; iter < end; iter++)
243 copy_handle((cell *)(*iter));
246 void factor_vm::copy_registered_bignums()
248 std::vector<cell>::const_iterator iter = gc_bignums.begin();
249 std::vector<cell>::const_iterator end = gc_bignums.end();
251 for(; iter < end; iter++)
253 bignum **handle = (bignum **)(*iter);
254 bignum *pointer = *handle;
258 check_data_pointer(pointer);
259 if(should_copy_p(pointer))
260 *handle = copy_untagged_object(pointer);
262 assert((*handle)->h.hi_tag() == BIGNUM_TYPE);
268 /* Copy roots over at the start of GC, namely various constants, stacks,
269 the user environment and extra roots registered by local_roots.hpp */
270 void factor_vm::copy_roots()
273 copy_handle(&bignum_zero);
274 copy_handle(&bignum_pos_one);
275 copy_handle(&bignum_neg_one);
277 copy_registered_locals();
278 copy_registered_bignums();
280 if(!performing_compaction)
283 context *stacks = stack_chain;
287 copy_stack_elements(stacks->datastack_region,stacks->datastack);
288 copy_stack_elements(stacks->retainstack_region,stacks->retainstack);
290 copy_handle(&stacks->catchstack_save);
291 copy_handle(&stacks->current_callback_save);
293 mark_active_blocks(stacks);
295 stacks = stacks->next;
300 for(i = 0; i < USER_ENV; i++)
301 copy_handle(&userenv[i]);
304 cell factor_vm::copy_next_from_nursery(cell scan)
306 cell *obj = (cell *)scan;
307 cell *end = (cell *)(scan + binary_payload_start((object *)scan));
313 cell nursery_start = nursery.start;
314 cell nursery_end = nursery.end;
316 for(; obj < end; obj++)
320 if(!immediate_p(pointer))
322 check_data_pointer((object *)pointer);
323 if(pointer >= nursery_start && pointer < nursery_end)
324 *obj = copy_object(pointer);
329 return scan + untagged_object_size((object *)scan);
332 cell factor_vm::copy_next_from_aging(cell scan)
334 cell *obj = (cell *)scan;
335 cell *end = (cell *)(scan + binary_payload_start((object *)scan));
341 cell tenured_start = data->generations[data->tenured()].start;
342 cell tenured_end = data->generations[data->tenured()].end;
344 cell newspace_start = newspace->start;
345 cell newspace_end = newspace->end;
347 for(; obj < end; obj++)
351 if(!immediate_p(pointer))
353 check_data_pointer((object *)pointer);
354 if(!(pointer >= newspace_start && pointer < newspace_end)
355 && !(pointer >= tenured_start && pointer < tenured_end))
356 *obj = copy_object(pointer);
361 return scan + untagged_object_size((object *)scan);
364 cell factor_vm::copy_next_from_tenured(cell scan)
366 cell *obj = (cell *)scan;
367 cell *end = (cell *)(scan + binary_payload_start((object *)scan));
373 cell newspace_start = newspace->start;
374 cell newspace_end = newspace->end;
376 for(; obj < end; obj++)
380 if(!immediate_p(pointer))
382 check_data_pointer((object *)pointer);
383 if(!(pointer >= newspace_start && pointer < newspace_end))
384 *obj = copy_object(pointer);
389 mark_object_code_block((object *)scan);
391 return scan + untagged_object_size((object *)scan);
394 void factor_vm::copy_reachable_objects(cell scan, cell *end)
396 if(collecting_gen == data->nursery())
399 scan = copy_next_from_nursery(scan);
401 else if(data->have_aging_p() && collecting_gen == data->aging())
404 scan = copy_next_from_aging(scan);
406 else if(collecting_gen == data->tenured())
409 scan = copy_next_from_tenured(scan);
413 /* Prepare to start copying reachable objects into an unused zone */
414 void factor_vm::begin_gc(cell requested_bytes)
416 if(growing_data_heap)
418 if(collecting_gen != data->tenured())
419 critical_error("Invalid parameters to begin_gc",0);
421 old_data_heap = data;
422 set_data_heap(grow_data_heap(old_data_heap,requested_bytes));
423 newspace = &data->generations[data->tenured()];
425 else if(collecting_accumulation_gen_p())
427 /* when collecting one of these generations, rotate it
428 with the semispace */
429 zone z = data->generations[collecting_gen];
430 data->generations[collecting_gen] = data->semispaces[collecting_gen];
431 data->semispaces[collecting_gen] = z;
432 reset_generation(collecting_gen);
433 newspace = &data->generations[collecting_gen];
434 clear_cards(collecting_gen,collecting_gen);
435 clear_decks(collecting_gen,collecting_gen);
436 clear_allot_markers(collecting_gen,collecting_gen);
440 /* when collecting a younger generation, we copy
441 reachable objects to the next oldest generation,
442 so we set the newspace so the next generation. */
443 newspace = &data->generations[collecting_gen + 1];
447 void factor_vm::end_gc(cell gc_elapsed)
449 gc_stats *s = &stats[collecting_gen];
452 s->gc_time += gc_elapsed;
453 if(s->max_gc_time < gc_elapsed)
454 s->max_gc_time = gc_elapsed;
456 if(growing_data_heap)
458 dealloc_data_heap(old_data_heap);
459 old_data_heap = NULL;
460 growing_data_heap = false;
463 if(collecting_accumulation_gen_p())
465 /* all younger generations except are now empty.
466 if collecting_gen == data->nursery() here, we only have 1 generation;
467 old-school Cheney collector */
468 if(collecting_gen != data->nursery())
469 reset_generations(data->nursery(),collecting_gen - 1);
471 else if(collecting_gen == data->nursery())
473 nursery.here = nursery.start;
477 /* all generations up to and including the one
478 collected are now empty */
479 reset_generations(data->nursery(),collecting_gen);
482 collecting_aging_again = false;
485 /* Collect gen and all younger generations.
486 If growing_data_heap_ is true, we must grow the data heap to such a size that
487 an allocation of requested_bytes won't fail */
488 void factor_vm::garbage_collection(cell gen,bool growing_data_heap_,cell requested_bytes)
492 critical_error("GC disabled",gen);
496 u64 start = current_micros();
498 performing_gc = true;
499 growing_data_heap = growing_data_heap_;
500 collecting_gen = gen;
502 /* we come back here if a generation is full */
505 /* We have no older generations we can try collecting, so we
506 resort to growing the data heap */
507 if(collecting_gen == data->tenured())
509 growing_data_heap = true;
511 /* see the comment in unmark_marked() */
512 unmark_marked(&code);
514 /* we try collecting aging space twice before going on to
516 else if(data->have_aging_p()
517 && collecting_gen == data->aging()
518 && !collecting_aging_again)
520 collecting_aging_again = true;
522 /* Collect the next oldest generation */
529 begin_gc(requested_bytes);
531 /* initialize chase pointer */
532 cell scan = newspace->here;
534 /* collect objects referenced from stacks and environment */
536 /* collect objects referenced from older generations */
539 /* do some tracing */
540 copy_reachable_objects(scan,&newspace->here);
542 /* don't scan code heap unless it has pointers to this
543 generation or younger */
544 if(collecting_gen >= last_code_heap_scan)
548 if(collecting_gen == data->tenured())
549 free_unmarked(&code,(heap_iterator)factor::update_literal_and_word_references);
551 copy_code_heap_roots();
553 if(collecting_accumulation_gen_p())
554 last_code_heap_scan = collecting_gen;
556 last_code_heap_scan = collecting_gen + 1;
559 cell gc_elapsed = (current_micros() - start);
563 performing_gc = false;
568 garbage_collection(data->tenured(),false,0);
571 inline void factor_vm::primitive_gc()
578 PRIMITIVE_GETVM()->primitive_gc();
581 inline void factor_vm::primitive_gc_stats()
583 growable_array result(this);
586 u64 total_gc_time = 0;
588 for(i = 0; i < max_gen_count; i++)
590 gc_stats *s = &stats[i];
591 result.add(allot_cell(s->collections));
592 result.add(tag<bignum>(long_long_to_bignum(s->gc_time)));
593 result.add(tag<bignum>(long_long_to_bignum(s->max_gc_time)));
594 result.add(allot_cell(s->collections == 0 ? 0 : s->gc_time / s->collections));
595 result.add(allot_cell(s->object_count));
596 result.add(tag<bignum>(long_long_to_bignum(s->bytes_copied)));
598 total_gc_time += s->gc_time;
601 result.add(tag<bignum>(ulong_long_to_bignum(total_gc_time)));
602 result.add(tag<bignum>(ulong_long_to_bignum(cards_scanned)));
603 result.add(tag<bignum>(ulong_long_to_bignum(decks_scanned)));
604 result.add(tag<bignum>(ulong_long_to_bignum(card_scan_time)));
605 result.add(allot_cell(code_heap_scans));
608 dpush(result.elements.value());
613 PRIMITIVE_GETVM()->primitive_gc_stats();
616 void factor_vm::clear_gc_stats()
618 for(cell i = 0; i < max_gen_count; i++)
619 memset(&stats[i],0,sizeof(gc_stats));
627 inline void factor_vm::primitive_clear_gc_stats()
632 PRIMITIVE(clear_gc_stats)
634 PRIMITIVE_GETVM()->primitive_clear_gc_stats();
637 /* classes.tuple uses this to reshape tuples; tools.deploy.shaker uses this
638 to coalesce equal but distinct quotations and wrappers. */
639 inline void factor_vm::primitive_become()
641 array *new_objects = untag_check<array>(dpop());
642 array *old_objects = untag_check<array>(dpop());
644 cell capacity = array_capacity(new_objects);
645 if(capacity != array_capacity(old_objects))
646 critical_error("bad parameters to become",0);
650 for(i = 0; i < capacity; i++)
652 tagged<object> old_obj(array_nth(old_objects,i));
653 tagged<object> new_obj(array_nth(new_objects,i));
655 if(old_obj != new_obj)
656 old_obj->h.forward_to(new_obj.untagged());
661 /* If a word's definition quotation was in old_objects and the
662 quotation in new_objects is not compiled, we might leak memory
663 by referencing the old quotation unless we recompile all
664 unoptimized words. */
670 PRIMITIVE_GETVM()->primitive_become();
673 void factor_vm::inline_gc(cell *gc_roots_base, cell gc_roots_size)
675 for(cell i = 0; i < gc_roots_size; i++)
676 gc_locals.push_back((cell)&gc_roots_base[i]);
678 garbage_collection(data->nursery(),false,0);
680 for(cell i = 0; i < gc_roots_size; i++)
681 gc_locals.pop_back();
684 VM_C_API void inline_gc(cell *gc_roots_base, cell gc_roots_size, factor_vm *myvm)
687 VM_PTR->inline_gc(gc_roots_base,gc_roots_size);