namespace factor {
struct compaction_fixup {
- static const bool translated_code_block_map = false;
-
- mark_bits<object> *data_forwarding_map;
- mark_bits<code_block> *code_forwarding_map;
- const object **data_finger;
- const code_block **code_finger;
-
- explicit compaction_fixup(
- mark_bits<object> *data_forwarding_map_,
- mark_bits<code_block> *code_forwarding_map_,
- const object **data_finger_,
- const code_block **code_finger_) :
- data_forwarding_map(data_forwarding_map_),
- code_forwarding_map(code_forwarding_map_),
- data_finger(data_finger_),
- code_finger(code_finger_) {}
-
- object *fixup_data(object *obj)
- {
- return data_forwarding_map->forward_block(obj);
- }
-
- code_block *fixup_code(code_block *compiled)
- {
- return code_forwarding_map->forward_block(compiled);
- }
-
- object *translate_data(const object *obj)
- {
- if(obj < *data_finger)
- return fixup_data((object *)obj);
- else
- return (object *)obj;
- }
-
- code_block *translate_code(const code_block *compiled)
- {
- if(compiled < *code_finger)
- return fixup_code((code_block *)compiled);
- else
- return (code_block *)compiled;
- }
-
- cell size(object *obj)
- {
- if(data_forwarding_map->marked_p(obj))
- return obj->size(*this);
- else
- return data_forwarding_map->unmarked_block_size(obj);
- }
-
- cell size(code_block *compiled)
- {
- if(code_forwarding_map->marked_p(compiled))
- return compiled->size(*this);
- else
- return code_forwarding_map->unmarked_block_size(compiled);
- }
+ static const bool translated_code_block_map = false;
+
+ mark_bits* data_forwarding_map;
+ mark_bits* code_forwarding_map;
+ const object** data_finger;
+ const code_block** code_finger;
+
+ compaction_fixup(mark_bits* data_forwarding_map,
+ mark_bits* code_forwarding_map,
+ const object** data_finger,
+ const code_block** code_finger)
+ : data_forwarding_map(data_forwarding_map),
+ code_forwarding_map(code_forwarding_map),
+ data_finger(data_finger),
+ code_finger(code_finger) {}
+
+ object* fixup_data(object* obj) {
+ return (object*)data_forwarding_map->forward_block((cell)obj);
+ }
+
+ code_block* fixup_code(code_block* compiled) {
+ return (code_block*)code_forwarding_map->forward_block((cell)compiled);
+ }
+
+ object* translate_data(const object* obj) {
+ if (obj < *data_finger)
+ return fixup_data((object*)obj);
+ return (object*)obj;
+ }
+
+ code_block* translate_code(const code_block* compiled) {
+ if (compiled < *code_finger)
+ return fixup_code((code_block*)compiled);
+ return (code_block*)compiled;
+ }
+
+ cell size(object* obj) {
+ if (data_forwarding_map->marked_p((cell)obj))
+ return obj->size(*this);
+ return data_forwarding_map->unmarked_block_size((cell)obj);
+ }
+
+ cell size(code_block* compiled) {
+ if (code_forwarding_map->marked_p((cell)compiled))
+ return compiled->size(*this);
+ return code_forwarding_map->unmarked_block_size((cell)compiled);
+ }
};
-struct object_compaction_updater {
- factor_vm *parent;
- compaction_fixup fixup;
- object_start_map *starts;
+// After a compaction, invalidate any code heap roots which are not
+// marked, and also slide the valid roots up so that call sites can be updated
+// correctly in case an inline cache compilation triggered compaction.
+void factor_vm::update_code_roots_for_compaction() {
- explicit object_compaction_updater(factor_vm *parent_, compaction_fixup fixup_) :
- parent(parent_),
- fixup(fixup_),
- starts(&parent->data->tenured->starts) {}
+ mark_bits* state = &code->allocator->state;
- void operator()(object *old_address, object *new_address, cell size)
- {
- slot_visitor<compaction_fixup> slot_forwarder(parent,fixup);
- slot_forwarder.visit_slots(new_address);
+ FACTOR_FOR_EACH(code_roots) {
+ code_root* root = *iter;
+ cell block = root->value & (~data_alignment + 1);
- code_block_visitor<compaction_fixup> code_forwarder(parent,fixup);
- code_forwarder.visit_object_code_block(new_address);
+ // Offset of return address within 16-byte allocation line
+ cell offset = root->value - block;
- starts->record_object_start_offset(new_address);
- }
-};
-
-template<typename Fixup>
-struct code_block_compaction_relocation_visitor {
- factor_vm *parent;
- code_block *old_address;
- Fixup fixup;
-
- explicit code_block_compaction_relocation_visitor(factor_vm *parent_,
- code_block *old_address_,
- Fixup fixup_) :
- parent(parent_),
- old_address(old_address_),
- fixup(fixup_) {}
-
- void operator()(instruction_operand op)
- {
- cell old_offset = op.rel_offset() + (cell)old_address->entry_point();
-
- switch(op.rel_type())
- {
- case RT_LITERAL:
- {
- cell value = op.load_value(old_offset);
- if(immediate_p(value))
- op.store_value(value);
- else
- op.store_value(RETAG(fixup.fixup_data(untag<object>(value)),TAG(value)));
- break;
- }
- case RT_ENTRY_POINT:
- case RT_ENTRY_POINT_PIC:
- case RT_ENTRY_POINT_PIC_TAIL:
- case RT_HERE:
- {
- cell value = op.load_value(old_offset);
- cell offset = TAG(value);
- code_block *compiled = (code_block *)UNTAG(value);
- op.store_value((cell)fixup.fixup_code(compiled) + offset);
- break;
- }
- case RT_THIS:
- case RT_CARDS_OFFSET:
- case RT_DECKS_OFFSET:
- parent->store_external_address(op);
- break;
- default:
- op.store_value(op.load_value(old_offset));
- break;
- }
- }
-};
-
-template<typename Fixup>
-struct code_block_compaction_updater {
- factor_vm *parent;
- Fixup fixup;
- slot_visitor<Fixup> data_forwarder;
- code_block_visitor<Fixup> code_forwarder;
-
- explicit code_block_compaction_updater(factor_vm *parent_,
- Fixup fixup_,
- slot_visitor<Fixup> data_forwarder_,
- code_block_visitor<Fixup> code_forwarder_) :
- parent(parent_),
- fixup(fixup_),
- data_forwarder(data_forwarder_),
- code_forwarder(code_forwarder_) {}
-
- void operator()(code_block *old_address, code_block *new_address, cell size)
- {
- data_forwarder.visit_code_block_objects(new_address);
-
- code_block_compaction_relocation_visitor<Fixup> visitor(parent,old_address,fixup);
- new_address->each_instruction_operand(visitor);
- }
-};
-
-/* After a compaction, invalidate any code heap roots which are not
-marked, and also slide the valid roots up so that call sites can be updated
-correctly in case an inline cache compilation triggered compaction. */
-void factor_vm::update_code_roots_for_compaction()
-{
- std::vector<code_root *>::const_iterator iter = code_roots.begin();
- std::vector<code_root *>::const_iterator end = code_roots.end();
-
- mark_bits<code_block> *state = &code->allocator->state;
-
- for(; iter < end; iter++)
- {
- code_root *root = *iter;
- code_block *block = (code_block *)(root->value & (~data_alignment + 1));
-
- /* Offset of return address within 16-byte allocation line */
- cell offset = root->value - (cell)block;
-
- if(root->valid && state->marked_p(block))
- {
- block = state->forward_block(block);
- root->value = (cell)block + offset;
- }
- else
- root->valid = false;
- }
+ if (root->valid && state->marked_p(block)) {
+ block = state->forward_block(block);
+ root->value = block + offset;
+ } else
+ root->valid = false;
+ }
}
-/* Compact data and code heaps */
-void factor_vm::collect_compact_impl(bool trace_contexts_p)
-{
- gc_event *event = current_gc->event;
+// Compact data and code heaps
+void factor_vm::collect_compact_impl() {
+ gc_event* event = current_gc->event;
-#if defined(FACTOR_DEBUG)
- code->verify_all_blocks_set();
+#ifdef FACTOR_DEBUG
+ code->verify_all_blocks_set();
#endif
- if(event) event->started_compaction();
-
- tenured_space *tenured = data->tenured;
- mark_bits<object> *data_forwarding_map = &tenured->state;
- mark_bits<code_block> *code_forwarding_map = &code->allocator->state;
-
- /* Figure out where blocks are going to go */
- data_forwarding_map->compute_forwarding();
- code_forwarding_map->compute_forwarding();
-
- const object *data_finger = tenured->first_block();
- const code_block *code_finger = code->allocator->first_block();
-
- compaction_fixup fixup(data_forwarding_map,code_forwarding_map,&data_finger,&code_finger);
- slot_visitor<compaction_fixup> data_forwarder(this,fixup);
- code_block_visitor<compaction_fixup> code_forwarder(this,fixup);
-
- code_forwarder.visit_code_roots();
-
- /* Object start offsets get recomputed by the object_compaction_updater */
- data->tenured->starts.clear_object_start_offsets();
-
- /* Slide everything in tenured space up, and update data and code heap
- pointers inside objects. */
- object_compaction_updater object_updater(this,fixup);
- tenured->compact(object_updater,fixup,&data_finger);
-
- /* Slide everything in the code heap up, and update data and code heap
- pointers inside code blocks. */
- code_block_compaction_updater<compaction_fixup> code_block_updater(this,fixup,data_forwarder,code_forwarder);
- code->allocator->compact(code_block_updater,fixup,&code_finger);
-
- data_forwarder.visit_roots();
- if(trace_contexts_p)
- {
- data_forwarder.visit_contexts();
- code_forwarder.visit_context_code_blocks();
- }
-
- update_code_roots_for_compaction();
- callbacks->update();
-
- code->initialize_all_blocks_set();
-
- if(event) event->ended_compaction();
+ if (event)
+ event->reset_timer();
+
+ tenured_space* tenured = data->tenured;
+ mark_bits* data_forwarding_map = &tenured->state;
+ mark_bits* code_forwarding_map = &code->allocator->state;
+
+ // Figure out where blocks are going to go
+ data_forwarding_map->compute_forwarding();
+ code_forwarding_map->compute_forwarding();
+
+ const object* data_finger = (object*)tenured->start;
+ const code_block* code_finger = (code_block*)code->allocator->start;
+
+ {
+ compaction_fixup fixup(data_forwarding_map, code_forwarding_map,
+ &data_finger, &code_finger);
+ slot_visitor<compaction_fixup> forwarder(this, fixup);
+
+ forwarder.visit_uninitialized_code_blocks();
+
+ // Object start offsets get recomputed by the object_compaction_updater
+ data->tenured->starts.clear_object_start_offsets();
+
+ // Slide everything in tenured space up, and update data and code heap
+ // pointers inside objects.
+ auto compact_object_func = [&](object* old_addr, object* new_addr, cell size) {
+ (void)old_addr;
+ (void)size;
+ forwarder.visit_slots(new_addr);
+ forwarder.visit_object_code_block(new_addr);
+ tenured->starts.record_object_start_offset(new_addr);
+ };
+ tenured->compact(compact_object_func, fixup, &data_finger);
+
+ // Slide everything in the code heap up, and update data and code heap
+ // pointers inside code blocks.
+ auto compact_code_func = [&](code_block* old_addr,
+ code_block* new_addr,
+ cell size) {
+ (void)size;
+ forwarder.visit_code_block_objects(new_addr);
+ cell old_entry_point = old_addr->entry_point();
+ forwarder.visit_instruction_operands(new_addr, old_entry_point);
+ };
+ code->allocator->compact(compact_code_func, fixup, &code_finger);
+
+ forwarder.visit_all_roots();
+ forwarder.visit_context_code_blocks();
+ }
+
+ update_code_roots_for_compaction();
+
+ // Each callback has a relocation with a pointer to a code block in
+ // the code heap. Since the code heap has now been compacted, those
+ // pointers are invalid and we need to update them.
+ auto callback_updater = [&](code_block* stub, cell size) {
+ (void)size;
+ callbacks->update(stub);
+ };
+ callbacks->allocator->iterate(callback_updater, no_fixup());
+
+ code->initialize_all_blocks_set();
+
+ if (event)
+ event->ended_phase(PHASE_DATA_COMPACTION);
}
-struct code_compaction_fixup {
- static const bool translated_code_block_map = false;
-
- mark_bits<code_block> *code_forwarding_map;
- const code_block **code_finger;
-
- explicit code_compaction_fixup(mark_bits<code_block> *code_forwarding_map_,
- const code_block **code_finger_) :
- code_forwarding_map(code_forwarding_map_),
- code_finger(code_finger_) {}
-
- object *fixup_data(object *obj)
- {
- return obj;
- }
-
- code_block *fixup_code(code_block *compiled)
- {
- return code_forwarding_map->forward_block(compiled);
- }
-
- object *translate_data(const object *obj)
- {
- return fixup_data((object *)obj);
- }
-
- code_block *translate_code(const code_block *compiled)
- {
- if(compiled >= *code_finger)
- return fixup_code((code_block *)compiled);
- else
- return (code_block *)compiled;
- }
-
- cell size(object *obj)
- {
- return obj->size();
- }
-
- cell size(code_block *compiled)
- {
- if(code_forwarding_map->marked_p(compiled))
- return compiled->size(*this);
- else
- return code_forwarding_map->unmarked_block_size(compiled);
- }
-};
-
-struct object_grow_heap_updater {
- code_block_visitor<code_compaction_fixup> code_forwarder;
-
- explicit object_grow_heap_updater(code_block_visitor<code_compaction_fixup> code_forwarder_) :
- code_forwarder(code_forwarder_) {}
-
- void operator()(object *obj)
- {
- code_forwarder.visit_object_code_block(obj);
- }
-};
-
-/* Compact just the code heap, after growing the data heap */
-void factor_vm::collect_compact_code_impl(bool trace_contexts_p)
-{
- /* Figure out where blocks are going to go */
- mark_bits<code_block> *code_forwarding_map = &code->allocator->state;
- code_forwarding_map->compute_forwarding();
+void factor_vm::collect_compact() {
+ collect_mark_impl();
+ collect_compact_impl();
- const code_block *code_finger = code->allocator->first_block();
-
- code_compaction_fixup fixup(code_forwarding_map,&code_finger);
- slot_visitor<code_compaction_fixup> data_forwarder(this,fixup);
- code_block_visitor<code_compaction_fixup> code_forwarder(this,fixup);
-
- code_forwarder.visit_code_roots();
-
- if(trace_contexts_p)
- code_forwarder.visit_context_code_blocks();
-
- /* Update code heap references in data heap */
- object_grow_heap_updater object_updater(code_forwarder);
- each_object(object_updater);
-
- /* Slide everything in the code heap up, and update code heap
- pointers inside code blocks. */
- code_block_compaction_updater<code_compaction_fixup> code_block_updater(this,fixup,data_forwarder,code_forwarder);
- code->allocator->compact(code_block_updater,fixup,&code_finger);
-
- update_code_roots_for_compaction();
- callbacks->update();
-}
+ // Compaction did not free up enough memory. Grow the data heap.
+ if (data->high_fragmentation_p()) {
+ set_current_gc_op(COLLECT_GROWING_DATA_HEAP_OP);
+ collect_growing_data_heap(0);
+ }
-void factor_vm::collect_compact(bool trace_contexts_p)
-{
- collect_mark_impl(trace_contexts_p);
- collect_compact_impl(trace_contexts_p);
-
- if(data->high_fragmentation_p())
- {
- /* Compaction did not free up enough memory. Grow the heap. */
- set_current_gc_op(collect_growing_heap_op);
- collect_growing_heap(0,trace_contexts_p);
- }
-
- code->flush_icache();
+ code->flush_icache();
}
-void factor_vm::collect_growing_heap(cell requested_size, bool trace_contexts_p)
-{
- /* Grow the data heap and copy all live objects to the new heap. */
- data_heap *old = data;
- set_data_heap(data->grow(requested_size));
- collect_mark_impl(trace_contexts_p);
- collect_compact_code_impl(trace_contexts_p);
- code->flush_icache();
- delete old;
+void factor_vm::collect_growing_data_heap(cell requested_size) {
+ // Grow the data heap and copy all live objects to the new heap.
+ data_heap* old = data;
+ set_data_heap(data->grow(&nursery, requested_size));
+ collect_mark_impl();
+ collect_compact_impl();
+ code->flush_icache();
+ delete old;
}
}
projects / factor.git / blobdiff