5 struct compaction_fixup {
6 static const bool translated_code_block_map = false;
8 mark_bits<object> *data_forwarding_map;
9 mark_bits<code_block> *code_forwarding_map;
10 const object **data_finger;
11 const code_block **code_finger;
13 explicit compaction_fixup(
14 mark_bits<object> *data_forwarding_map_,
15 mark_bits<code_block> *code_forwarding_map_,
16 const object **data_finger_,
17 const code_block **code_finger_) :
18 data_forwarding_map(data_forwarding_map_),
19 code_forwarding_map(code_forwarding_map_),
20 data_finger(data_finger_),
21 code_finger(code_finger_) {}
23 object *fixup_data(object *obj)
25 return data_forwarding_map->forward_block(obj);
28 code_block *fixup_code(code_block *compiled)
30 return code_forwarding_map->forward_block(compiled);
33 object *translate_data(const object *obj)
35 if(obj < *data_finger)
36 return fixup_data((object *)obj);
41 code_block *translate_code(const code_block *compiled)
43 if(compiled < *code_finger)
44 return fixup_code((code_block *)compiled);
46 return (code_block *)compiled;
49 cell size(object *obj)
51 if(data_forwarding_map->marked_p(obj))
52 return obj->size(*this);
54 return data_forwarding_map->unmarked_block_size(obj);
57 cell size(code_block *compiled)
59 if(code_forwarding_map->marked_p(compiled))
60 return compiled->size(*this);
62 return code_forwarding_map->unmarked_block_size(compiled);
66 struct object_compaction_updater {
68 compaction_fixup fixup;
69 object_start_map *starts;
71 explicit object_compaction_updater(factor_vm *parent_, compaction_fixup fixup_) :
74 starts(&parent->data->tenured->starts) {}
76 void operator()(object *old_address, object *new_address, cell size)
78 slot_visitor<compaction_fixup> slot_forwarder(parent,fixup);
79 slot_forwarder.visit_slots(new_address);
81 code_block_visitor<compaction_fixup> code_forwarder(parent,fixup);
82 code_forwarder.visit_object_code_block(new_address);
84 starts->record_object_start_offset(new_address);
88 template<typename Fixup>
89 struct code_block_compaction_relocation_visitor {
91 code_block *old_address;
94 explicit code_block_compaction_relocation_visitor(factor_vm *parent_,
95 code_block *old_address_,
98 old_address(old_address_),
101 void operator()(instruction_operand op)
103 cell old_offset = op.rel_offset() + (cell)old_address->entry_point();
105 switch(op.rel_type())
109 cell value = op.load_value(old_offset);
110 if(immediate_p(value))
111 op.store_value(value);
113 op.store_value(RETAG(fixup.fixup_data(untag<object>(value)),TAG(value)));
117 case RT_ENTRY_POINT_PIC:
118 case RT_ENTRY_POINT_PIC_TAIL:
121 cell value = op.load_value(old_offset);
122 cell offset = TAG(value);
123 code_block *compiled = (code_block *)UNTAG(value);
124 op.store_value((cell)fixup.fixup_code(compiled) + offset);
128 case RT_CARDS_OFFSET:
129 case RT_DECKS_OFFSET:
130 parent->store_external_address(op);
133 op.store_value(op.load_value(old_offset));
139 template<typename Fixup>
140 struct code_block_compaction_updater {
143 slot_visitor<Fixup> data_forwarder;
144 code_block_visitor<Fixup> code_forwarder;
146 explicit code_block_compaction_updater(factor_vm *parent_,
148 slot_visitor<Fixup> data_forwarder_,
149 code_block_visitor<Fixup> code_forwarder_) :
152 data_forwarder(data_forwarder_),
153 code_forwarder(code_forwarder_) {}
155 void operator()(code_block *old_address, code_block *new_address, cell size)
157 data_forwarder.visit_code_block_objects(new_address);
159 code_block_compaction_relocation_visitor<Fixup> visitor(parent,old_address,fixup);
160 new_address->each_instruction_operand(visitor);
164 /* After a compaction, invalidate any code heap roots which are not
165 marked, and also slide the valid roots up so that call sites can be updated
166 correctly in case an inline cache compilation triggered compaction. */
167 void factor_vm::update_code_roots_for_compaction()
169 std::vector<code_root *>::const_iterator iter = code_roots.begin();
170 std::vector<code_root *>::const_iterator end = code_roots.end();
172 mark_bits<code_block> *state = &code->allocator->state;
174 for(; iter < end; iter++)
176 code_root *root = *iter;
177 code_block *block = (code_block *)(root->value & (~data_alignment + 1));
179 /* Offset of return address within 16-byte allocation line */
180 cell offset = root->value - (cell)block;
182 if(root->valid && state->marked_p(block))
184 block = state->forward_block(block);
185 root->value = (cell)block + offset;
192 /* Compact data and code heaps */
193 void factor_vm::collect_compact_impl(bool trace_contexts_p)
195 gc_event *event = current_gc->event;
197 #if defined(FACTOR_DEBUG)
198 code->verify_all_blocks_set();
201 if(event) event->started_compaction();
203 tenured_space *tenured = data->tenured;
204 mark_bits<object> *data_forwarding_map = &tenured->state;
205 mark_bits<code_block> *code_forwarding_map = &code->allocator->state;
207 /* Figure out where blocks are going to go */
208 data_forwarding_map->compute_forwarding();
209 code_forwarding_map->compute_forwarding();
211 const object *data_finger = tenured->first_block();
212 const code_block *code_finger = code->allocator->first_block();
214 compaction_fixup fixup(data_forwarding_map,code_forwarding_map,&data_finger,&code_finger);
215 slot_visitor<compaction_fixup> data_forwarder(this,fixup);
216 code_block_visitor<compaction_fixup> code_forwarder(this,fixup);
218 code_forwarder.visit_code_roots();
220 /* Object start offsets get recomputed by the object_compaction_updater */
221 data->tenured->starts.clear_object_start_offsets();
223 /* Slide everything in tenured space up, and update data and code heap
224 pointers inside objects. */
225 object_compaction_updater object_updater(this,fixup);
226 tenured->compact(object_updater,fixup,&data_finger);
228 /* Slide everything in the code heap up, and update data and code heap
229 pointers inside code blocks. */
230 code_block_compaction_updater<compaction_fixup> code_block_updater(this,fixup,data_forwarder,code_forwarder);
231 code->allocator->compact(code_block_updater,fixup,&code_finger);
233 data_forwarder.visit_roots();
236 data_forwarder.visit_contexts();
237 code_forwarder.visit_context_code_blocks();
240 update_code_roots_for_compaction();
243 code->initialize_all_blocks_set();
245 if(event) event->ended_compaction();
248 struct code_compaction_fixup {
249 static const bool translated_code_block_map = false;
251 mark_bits<code_block> *code_forwarding_map;
252 const code_block **code_finger;
254 explicit code_compaction_fixup(mark_bits<code_block> *code_forwarding_map_,
255 const code_block **code_finger_) :
256 code_forwarding_map(code_forwarding_map_),
257 code_finger(code_finger_) {}
259 object *fixup_data(object *obj)
264 code_block *fixup_code(code_block *compiled)
266 return code_forwarding_map->forward_block(compiled);
269 object *translate_data(const object *obj)
271 return fixup_data((object *)obj);
274 code_block *translate_code(const code_block *compiled)
276 if(compiled < *code_finger)
277 return fixup_code((code_block *)compiled);
279 return (code_block *)compiled;
282 cell size(object *obj)
287 cell size(code_block *compiled)
289 if(code_forwarding_map->marked_p(compiled))
290 return compiled->size(*this);
292 return code_forwarding_map->unmarked_block_size(compiled);
296 struct object_grow_heap_updater {
297 code_block_visitor<code_compaction_fixup> code_forwarder;
299 explicit object_grow_heap_updater(code_block_visitor<code_compaction_fixup> code_forwarder_) :
300 code_forwarder(code_forwarder_) {}
302 void operator()(object *obj)
304 code_forwarder.visit_object_code_block(obj);
308 /* Compact just the code heap, after growing the data heap */
309 void factor_vm::collect_compact_code_impl(bool trace_contexts_p)
311 /* Figure out where blocks are going to go */
312 mark_bits<code_block> *code_forwarding_map = &code->allocator->state;
313 code_forwarding_map->compute_forwarding();
315 const code_block *code_finger = code->allocator->first_block();
317 code_compaction_fixup fixup(code_forwarding_map,&code_finger);
318 slot_visitor<code_compaction_fixup> data_forwarder(this,fixup);
319 code_block_visitor<code_compaction_fixup> code_forwarder(this,fixup);
321 code_forwarder.visit_code_roots();
324 code_forwarder.visit_context_code_blocks();
326 /* Update code heap references in data heap */
327 object_grow_heap_updater object_updater(code_forwarder);
328 each_object(object_updater);
330 /* Slide everything in the code heap up, and update code heap
331 pointers inside code blocks. */
332 code_block_compaction_updater<code_compaction_fixup> code_block_updater(this,fixup,data_forwarder,code_forwarder);
333 code->allocator->compact(code_block_updater,fixup,&code_finger);
335 update_code_roots_for_compaction();
337 code->initialize_all_blocks_set();
340 void factor_vm::collect_compact(bool trace_contexts_p)
342 collect_mark_impl(trace_contexts_p);
343 collect_compact_impl(trace_contexts_p);
345 if(data->high_fragmentation_p())
347 /* Compaction did not free up enough memory. Grow the heap. */
348 set_current_gc_op(collect_growing_heap_op);
349 collect_growing_heap(0,trace_contexts_p);
352 code->flush_icache();
355 void factor_vm::collect_growing_heap(cell requested_size, bool trace_contexts_p)
357 /* Grow the data heap and copy all live objects to the new heap. */
358 data_heap *old = data;
359 set_data_heap(data->grow(requested_size));
360 collect_mark_impl(trace_contexts_p);
361 collect_compact_code_impl(trace_contexts_p);
362 code->flush_icache();