From 513849c1e3a207310d2b599c6cefcbf26f6780b0 Mon Sep 17 00:00:00 2001 From: Matthew Flatt Date: Sat, 19 Dec 2015 17:40:10 -0700 Subject: [PATCH] incremental GC: make accounting incremental for the root custodian --- .../scribblings/guide/performance.scrbl | 8 - racket/src/racket/gc2/mem_account.c | 148 ++++++++++++++---- racket/src/racket/gc2/newgc.c | 37 +++-- racket/src/racket/gc2/newgc.h | 3 +- 4 files changed, 147 insertions(+), 49 deletions(-) diff --git a/pkgs/racket-doc/scribblings/guide/performance.scrbl b/pkgs/racket-doc/scribblings/guide/performance.scrbl index d0a9970572..da33166505 100644 --- a/pkgs/racket-doc/scribblings/guide/performance.scrbl +++ b/pkgs/racket-doc/scribblings/guide/performance.scrbl @@ -626,11 +626,3 @@ runs @filepath{main.rkt} with garbage-collection logging to stderr collections are reported by @litchar{min} lines, increment-mode minor collection are reported with @litchar{mIn} lines, and major collections are reported with @litchar{MAJ} lines. - -Some Racket features can interfere with incremental mode. In -particular, memory accounting via @racket[custodian-limit-memory] -triggers an accounting pass during a major collection, and that pass -is not currently incremental. Note that DrRacket uses -@racket[custodian-limit-memory] to control the memory use of programs -run within DrRacket, so incremental mode is useful mainly outside of -DrRacket. (See also @secref["DrRacket-perf"].) diff --git a/racket/src/racket/gc2/mem_account.c b/racket/src/racket/gc2/mem_account.c index ea3e8ffbdf..4aadc9f450 100644 --- a/racket/src/racket/gc2/mem_account.c +++ b/racket/src/racket/gc2/mem_account.c @@ -385,7 +385,8 @@ inline static void BTC_memory_account_mark(NewGC *gc, mpage *page, void *ptr, in if(info->btc_mark == gc->old_btc_mark) { info->btc_mark = gc->new_btc_mark; account_memory(gc, gc->current_mark_owner, info->size, 0); - push_ptr(gc, ptr, 0); + if (page->generation != AGE_GEN_HALF) + push_ptr(gc, ptr, 0); } } } @@ -470,15 +471,34 @@ static void btc_overmem_abort(NewGC *gc) "Info will be wrong.\n")); } -static void propagate_accounting_marks(NewGC *gc) +static void propagate_accounting_marks(NewGC *gc, int no_full) { void *p; + int fuel = (gc->gc_full + ? -1 + : (no_full + ? INCREMENTAL_COLLECT_FUEL_PER_100M / INCREMENTAL_MINOR_REQUEST_DIVISOR + : (INCREMENTAL_COLLECT_FUEL_PER_100M * AS_100M(gc->memory_in_use)) / 2)); - while(pop_ptr(gc, &p, 0) && !gc->kill_propagation_loop) { + while (pop_ptr(gc, &p, 0) && !gc->kill_propagation_loop) { + gc->traverse_count = 0; + /* GCDEBUG((DEBUGOUTF, "btc_account: popped off page %p:%p, ptr %p\n", page, page->addr, p)); */ propagate_marks_worker(gc, p, 0); + + if (fuel >= 0) { + fuel--; + fuel -= (gc->traverse_count >> 2); + if (gc->unprotected_page) { + gc->unprotected_page = 0; + fuel -= 100; + } + if (fuel <= 0) + break; + } } - if(gc->kill_propagation_loop) + + if (gc->kill_propagation_loop) reset_pointer_stack(gc); } @@ -499,14 +519,33 @@ inline static int BTC_get_redirect_tag(NewGC *gc, int tag) { return tag; } -static void BTC_do_accounting(NewGC *gc) +static void BTC_do_accounting(NewGC *gc, int no_full) { const int table_size = gc->owner_table_size; + int init_table_start, init_table_end, do_mark_threads; OTEntry **owner_table = gc->owner_table; + MarkSegment *orig_mark_stack; - gc->really_doing_accounting = gc->next_really_doing_accounting; - gc->next_really_doing_accounting = 0; + GC_ASSERT(gc->gc_full || gc->finished_incremental); + GC_ASSERT(gc->gc_full || !gc->accounted_incremental); + if (gc->gc_full) { + if (!gc->acct_mark_stack) + gc->really_doing_accounting = gc->next_really_doing_accounting; + gc->next_really_doing_accounting = 0; + } else { + if (gc->next_really_doing_accounting) + gc->really_doing_accounting = 1; + + GC_ASSERT(!gc->mark_gen1); + GC_ASSERT(!gc->inc_gen1); + GC_ASSERT(!gc->check_gen1); + + gc->mark_gen1 = 1; + gc->check_gen1 = 1; + gc->inc_gen1 = 1; + } + if(gc->really_doing_accounting) { Scheme_Custodian *cur = owner_table[current_owner(gc, NULL)]->originator, *last, *parent; Scheme_Custodian_Reference *box = cur->global_next; @@ -520,8 +559,31 @@ static void BTC_do_accounting(NewGC *gc) gc->unsafe_allocation_abort = btc_overmem_abort; gc->master_page_btc_mark_checked = 0; + if (!gc->gc_full || gc->acct_mark_stack) { + orig_mark_stack = gc->mark_stack; + if (gc->acct_mark_stack) { + gc->mark_stack = gc->acct_mark_stack; + init_table_start = 2; + do_mark_threads = 0; + } else { + gc->mark_stack = NULL; + mark_stack_initialize(gc); + init_table_start = 1; + do_mark_threads = 1; + } + if (gc->gc_full) + init_table_end = table_size; + else + init_table_end = 2; + } else { + orig_mark_stack = NULL; + init_table_start = 1; + init_table_end = table_size; + do_mark_threads = 1; + } + /* clear the memory use numbers out */ - for(i = 1; i < table_size; i++) + for(i = init_table_start; i < init_table_end; i++) if(owner_table[i]) { owner_table[i]->memory_use = 0; #ifdef MZ_USE_PLACES @@ -529,7 +591,7 @@ static void BTC_do_accounting(NewGC *gc) owner_table[i]->master_memory_use = 0; #endif } - + /* start with root: */ while (cur->parent && SCHEME_PTR1_VAL(cur->parent)) { cur = SCHEME_PTR1_VAL(cur->parent); @@ -541,6 +603,8 @@ static void BTC_do_accounting(NewGC *gc) while(cur) { int owner = custodian_to_owner_set(gc, cur); + GC_ASSERT(gc->gc_full || (owner == 1)); + GC_ASSERT(owner >= 0); GC_ASSERT(owner <= gc->owner_table_size); @@ -549,44 +613,70 @@ static void BTC_do_accounting(NewGC *gc) gc->current_mark_owner = owner; GCDEBUG((DEBUGOUTF,"MARKING THREADS OF OWNER %i (CUST %p)\n", owner, cur)); gc->kill_propagation_loop = 0; - mark_threads(gc, owner); - mark_cust_boxes(gc, cur); + if (do_mark_threads) { + mark_threads(gc, owner); + mark_cust_boxes(gc, cur); + } GCDEBUG((DEBUGOUTF, "Propagating accounting marks\n")); - propagate_accounting_marks(gc); + propagate_accounting_marks(gc, no_full); + + owner_table = gc->owner_table; + owner_table[owner]->memory_use = add_no_overflow(owner_table[owner]->memory_use, + gcBYTES_TO_WORDS(gc->acct_phantom_count)); + + if (!gc->gc_full) + break; last = cur; box = cur->global_next; cur = box ? SCHEME_PTR1_VAL(box) : NULL; - owner_table = gc->owner_table; - owner_table[owner]->memory_use = add_no_overflow(owner_table[owner]->memory_use, - gcBYTES_TO_WORDS(gc->acct_phantom_count)); + do_mark_threads = 1; } release_master_btc_mark(gc); - /* walk backward folding totals int parent */ - cur = last; - while (cur) { - int owner = custodian_to_owner_set(gc, cur); + if (gc->gc_full) { + /* walk backward folding totals into parent */ + cur = last; + while (cur) { + int owner = custodian_to_owner_set(gc, cur); - box = cur->parent; parent = box ? SCHEME_PTR1_VAL(box) : NULL; - if (parent) { - int powner = custodian_to_owner_set(gc, parent); + box = cur->parent; parent = box ? SCHEME_PTR1_VAL(box) : NULL; + if (parent) { + int powner = custodian_to_owner_set(gc, parent); - owner_table = gc->owner_table; - owner_table[powner]->memory_use = add_no_overflow(owner_table[powner]->memory_use, - owner_table[owner]->memory_use); - owner_table[powner]->master_memory_use += owner_table[owner]->master_memory_use; + owner_table = gc->owner_table; + owner_table[powner]->memory_use = add_no_overflow(owner_table[powner]->memory_use, + owner_table[owner]->memory_use); + owner_table[powner]->master_memory_use += owner_table[owner]->master_memory_use; + } + + box = cur->global_prev; cur = box ? SCHEME_PTR1_VAL(box) : NULL; } - box = cur->global_prev; cur = box ? SCHEME_PTR1_VAL(box) : NULL; + if (orig_mark_stack) { + free_stack_pages_at(gc->mark_stack); + gc->acct_mark_stack = NULL; + gc->mark_stack = orig_mark_stack; + } + } else { + gc->acct_mark_stack = gc->mark_stack; + gc->mark_stack = orig_mark_stack; } gc->in_unsafe_allocation_mode = 0; gc->doing_memory_accounting = 0; - gc->old_btc_mark = gc->new_btc_mark; - gc->new_btc_mark = !gc->new_btc_mark; + if (gc->gc_full) { + gc->old_btc_mark = gc->new_btc_mark; + gc->new_btc_mark = !gc->new_btc_mark; + } + } + + if (!gc->gc_full) { + gc->mark_gen1 = 0; + gc->check_gen1 = 0; + gc->inc_gen1 = 0; } } diff --git a/racket/src/racket/gc2/newgc.c b/racket/src/racket/gc2/newgc.c index a3a9e4a706..1904abbafb 100644 --- a/racket/src/racket/gc2/newgc.c +++ b/racket/src/racket/gc2/newgc.c @@ -275,6 +275,8 @@ static int always_collect_incremental_on_minor = 0; seems to have been excessively conservative. */ #define FORCE_MAJOR_AFTER_COUNT 1000 +#define AS_100M(c) ((c / (1024 * 1024 * 100)) + 1) + /* This is the log base 2 of the size of one word, given in bytes */ #ifdef SIXTY_FOUR_BIT_INTEGERS # define LOG_WORD_SIZE 3 @@ -2120,7 +2122,8 @@ static int inc_marked_gen1(NewGC *gc, void *p) static int is_in_generation_half(NewGC *gc, const void *p) { mpage *page; - if (gc->gc_full) return 0; + if (gc->gc_full) /* generation half is never used for a full GC */ + return 0; page = pagemap_find_page_for_marking(gc, p, 1); if (!page) return 0; GC_ASSERT((page->generation == AGE_GEN_1) @@ -2784,7 +2787,7 @@ static void push_ptr(NewGC *gc, void *ptr, int inc_gen1) } #endif - GC_ASSERT(inc_gen1 || !gc->inc_gen1); + GC_ASSERT(inc_gen1 || !gc->inc_gen1 || gc->doing_memory_accounting); GC_ASSERT(!inc_gen1 || !gc->all_marked_incremental); push_ptr_at(ptr, inc_gen1 ? &gc->inc_mark_stack : &gc->mark_stack); @@ -3128,7 +3131,7 @@ static int designate_modified_gc(NewGC *gc, void *p) pages that are otherwise unmodified in the current pass: */ || gc->fnl_gen1 /* Memory accounting can also modify otherwise unadjusted - pages after incrementa mode: */ + pages after incremental mode: */ || (gc->doing_memory_accounting && gc->finished_incremental))) { check_incremental_unprotect(gc, page); gc->unprotected_page = 1; /* for using fuel */ @@ -3757,7 +3760,7 @@ static void page_newly_marked_on(NewGC *gc, mpage *page, int is_a_master_page, i /* If this page isn't already marked as old, it must be a medium page whose generation will be updated in the clean-up phase */ GC_ASSERT((page->generation >= AGE_GEN_1) || (page->size_class == SIZE_CLASS_MED_PAGE)); - GC_ASSERT(!gc->finished_incremental); + GC_ASSERT(!gc->finished_incremental || (!gc->accounted_incremental && gc->really_doing_accounting)); GC_ASSERT(!page->non_dead_as_mark); page->inc_marked_on = 1; page->inc_modified_next = gc->inc_modified_next; @@ -4201,7 +4204,7 @@ static inline void propagate_marks_worker(NewGC *gc, void *pp, int inc_gen1) start = PPTR(BIG_PAGE_TO_OBJECT(page)); alloc_type = page->page_type; end = PAGE_END_VSS(page); - GC_ASSERT(inc_gen1 || !page->mprotected); + GC_ASSERT(inc_gen1 || !page->mprotected || gc->doing_memory_accounting); } else { objhead *info; p = pp; @@ -5511,7 +5514,11 @@ static void incremental_repair_pages(NewGC *gc, int fuel) GC_ASSERT(page == gc->inc_repair_next); #endif - while (fuel && gc->inc_repair_next) { + /* If gc->finished_incremental already, then we must be in the + process of accounting incrementally */ + GC_ASSERT(!gc->finished_incremental || !gc->inc_repair_next || gc->really_doing_accounting); + + while ((fuel || gc->finished_incremental) && gc->inc_repair_next) { page = gc->inc_repair_next; gc->inc_repair_next = page->inc_modified_next; if (!gc->inc_repair_next) @@ -5960,8 +5967,6 @@ extern double scheme_get_inexact_milliseconds(void); # define TIME_ARGS /**/ #endif -#define AS_100M(c) ((c / (1024 * 1024 * 100)) + 1) - static int mark_and_finalize_all(NewGC *gc, int old_gen, int no_full TIME_FORMAL_ARGS) { int fuel = (old_gen @@ -6105,6 +6110,7 @@ static void garbage_collect(NewGC *gc, int force_full, int no_full, int switchin GC_ASSERT(!gc->all_marked_incremental || gc->started_incremental); GC_ASSERT(!gc->all_marked_incremental || mark_stack_is_empty(gc->inc_mark_stack)); GC_ASSERT(!gc->finished_incremental || (gc->all_marked_incremental && !gc->inc_repair_next)); + GC_ASSERT(!gc->accounted_incremental || gc->finished_incremental); /* determine if this should be a full collection or not */ gc->gc_full = (force_full @@ -6129,13 +6135,14 @@ static void garbage_collect(NewGC *gc, int force_full, int no_full, int switchin && !gc->started_incremental) /* In incremental mode, GC earlier if we've done everything that we can do incrementally. */ - || gc->finished_incremental); + || gc->accounted_incremental); if (gc->gc_full && no_full) return; next_gc_full = (gc->gc_full && !gc->incremental_requested + && !always_collect_incremental_on_minor && !gc->full_needed_for_finalization); if (gc->full_needed_for_finalization && gc->gc_full) @@ -6320,8 +6327,15 @@ static void garbage_collect(NewGC *gc, int force_full, int no_full, int switchin reset_nursery(gc); TIME_STEP("reset nursurey"); #ifdef NEWGC_BTC_ACCOUNT - if (gc->gc_full && postmaster_and_place_gc(gc)) - BTC_do_accounting(gc); + if ((gc->gc_full || (gc->finished_incremental && !gc->accounted_incremental)) + && postmaster_and_place_gc(gc)) { + BTC_do_accounting(gc, no_full); + if (!gc->gc_full && mark_stack_is_empty(gc->acct_mark_stack)) + gc->accounted_incremental = 1; + } +#else + if (gc->finished_incremental) + gc->accounted_incremental = 1; #endif TIME_STEP("accounted"); @@ -6392,6 +6406,7 @@ static void garbage_collect(NewGC *gc, int force_full, int no_full, int switchin gc->started_incremental = 0; gc->all_marked_incremental = 0; gc->finished_incremental = 0; + gc->accounted_incremental = 0; gc->inc_prop_count = 0; gc->incremental_requested = 0; /* request expires completely after a full GC */ } diff --git a/racket/src/racket/gc2/newgc.h b/racket/src/racket/gc2/newgc.h index 76215402f4..1f589b1b4a 100644 --- a/racket/src/racket/gc2/newgc.h +++ b/racket/src/racket/gc2/newgc.h @@ -184,7 +184,7 @@ typedef struct NewGC { the end of the GC cycle: */ struct mpage *reprotect_next; - MarkSegment *mark_stack, *inc_mark_stack; + MarkSegment *mark_stack, *inc_mark_stack, *acct_mark_stack; /* Finalization */ Fnl *run_queue, *last_in_queue; @@ -214,6 +214,7 @@ typedef struct NewGC { unsigned char started_incremental :1; /* must stick with incremental until major GC */ unsigned char all_marked_incremental :1; /* finished all marking for an incremental GC */ unsigned char finished_incremental :1; /* finished marking and reparing an incremental GC */ + unsigned char accounted_incremental :1; /* memory accounting for an incremental GC */ unsigned char in_unsafe_allocation_mode :1; unsigned char full_needed_for_finalization :1; unsigned char no_further_modifications :1;