From 449f01b55de732213a1d6748c84245e499a75656 Mon Sep 17 00:00:00 2001 From: Matthew Flatt Date: Mon, 12 Oct 2020 16:13:03 -0600 Subject: [PATCH] Chez Scheme GC: simplify make parallel sweeper object exchange Make the exchange of objects by parallel sweepers --- which is needed when an object being swept by one thread refers to an object owner by another thread --- much simpler and slightly faster. The original design of exchanging memory regions didn't really work out, and it had degrenerated to essentially exchanging objects. Explicitly exchanging objects instead of regions makes the GC simpler, because received objects can be just added to the regular sweep stack, and less additional space<->type correspondence is needed. --- racket/src/ChezScheme/c/Mf-base | 4 + racket/src/ChezScheme/c/gc.c | 635 +++++++++++----------------- racket/src/ChezScheme/c/gcwrapper.c | 9 +- racket/src/ChezScheme/c/thread.c | 4 - racket/src/ChezScheme/c/types.h | 24 +- racket/src/ChezScheme/mats/Mf-base | 3 + racket/src/ChezScheme/s/mkgc.ss | 35 +- 7 files changed, 283 insertions(+), 431 deletions(-) diff --git a/racket/src/ChezScheme/c/Mf-base b/racket/src/ChezScheme/c/Mf-base index 5844703f6c..f37a81a33b 100644 --- a/racket/src/ChezScheme/c/Mf-base +++ b/racket/src/ChezScheme/c/Mf-base @@ -84,6 +84,10 @@ LZ4Sources=../lz4/lib/lz4.h ../lz4/lib/lz4frame.h \ ../lz4/lib/lz4.c ../lz4/lib/lz4frame.c \ ../lz4/lib/lz4hc.c ../lz4/lib/xxhash.c +.PHONY: run +run: + env SCHEMEHEAPDIRS=../boot/$m/ ../bin/$m/scheme $(ARGS) + clean: rm -f *.$o ${mdclean} rm -f Make.out diff --git a/racket/src/ChezScheme/c/gc.c b/racket/src/ChezScheme/c/gc.c index 72840bfcc1..6fe2da838e 100644 --- a/racket/src/ChezScheme/c/gc.c +++ b/racket/src/ChezScheme/c/gc.c @@ -194,7 +194,7 @@ static void resweep_weak_pairs PROTO((thread_gc *tgc, seginfo *oldweakspacesegme static void forward_or_bwp PROTO((ptr *pp, ptr p)); static void sweep_generation PROTO((thread_gc *tgc)); static iptr sweep_from_stack PROTO((thread_gc *tgc)); -static void enlarge_sweep_stack PROTO((thread_gc *tgc)); +static void enlarge_stack PROTO((thread_gc *tgc, ptr *stack, ptr *stack_start, ptr *stack_limit, uptr grow_at_least)); static uptr size_object PROTO((ptr p)); static iptr sweep_typed_object PROTO((thread_gc *tgc, ptr p, IGEN from_g)); static void sweep_symbol PROTO((thread_gc *tgc, ptr p, IGEN from_g)); @@ -266,7 +266,6 @@ static uptr get_cpu_time () { # define ACCUM_CPU_TIME(a, y, x) uptr y = get_cpu_time() - x; a += y # define REPORT_TIME(e) e static uptr collect_accum, all_accum, par_accum; -static int percentage(iptr n, iptr d) { if (d == 0) return 0; else return (n * 100) / d; } # define COUNT_SWEPT_BYTES(start, end) num_swept_bytes += ((uptr)TO_PTR(end) - (uptr)TO_PTR(start)) # define ADJUST_COUNTER(e) e #else @@ -321,11 +320,14 @@ FORCEINLINE IGEN compute_target_generation(IGEN g) { static octet *fully_marked_mask[static_generation+1]; -#define push_sweep(p) { \ - if (tgc->sweep_stack == tgc->sweep_stack_limit) enlarge_sweep_stack(tgc); \ - *(ptr *)TO_VOIDP(tgc->sweep_stack) = p; \ - tgc->sweep_stack = (ptr)((uptr)tgc->sweep_stack + ptr_bytes); \ - } +static const int sweep_stack_min_size = 256; + +#define push_sweep(p) do { \ + if (tgc->sweep_stack == tgc->sweep_stack_limit) \ + enlarge_stack(tgc, &tgc->sweep_stack, &tgc->sweep_stack_start, &tgc->sweep_stack_limit, ptr_bytes); \ + *(ptr *)TO_VOIDP(tgc->sweep_stack) = p; \ + tgc->sweep_stack = (ptr)((uptr)tgc->sweep_stack + ptr_bytes); \ + } while (0) #ifdef ENABLE_MEASURE static uptr measure_total; /* updated by `measure` */ @@ -340,7 +342,7 @@ static ptr sweep_from; # define BACKREFERENCES_ENABLED S_G.enable_object_backreferences # define SET_SWEEP_FROM(p) if (S_G.enable_object_backreferences) sweep_from = p # define WITH_TOP_BACKREFERENCE(v, e) SET_SWEEP_FROM(v); e; SET_SWEEP_FROM(Sfalse) -# define SET_BACKREFERENCE(p) sweep_from = p; +# define SET_BACKREFERENCE(p) sweep_from = p # define PUSH_BACKREFERENCE(p) ptr old_sweep_from = sweep_from; SET_SWEEP_FROM(p); # define POP_BACKREFERENCE() SET_SWEEP_FROM(old_sweep_from); # define ADD_BACKREFERENCE_FROM(p, from_p, tg) do { \ @@ -354,7 +356,7 @@ static ptr sweep_from; #else # define BACKREFERENCES_ENABLED 0 # define WITH_TOP_BACKREFERENCE(v, e) e -# define SET_BACKREFERENCE(p) +# define SET_BACKREFERENCE(p) do { } while (0) # define PUSH_BACKREFERENCE(p) # define POP_BACKREFERENCE() # define ADD_BACKREFERENCE_FROM(p, from_p, from_g) @@ -375,27 +377,21 @@ static int in_parallel_sweepers = 0; # define GC_MUTEX_RELEASE() alloc_mutex_release() # define SEGMENT_IS_LOCAL(si, p) (((si)->creator == tgc) || marked(si, p) || !in_parallel_sweepers) -# define RECORD_REMOTE_RANGE_TO(tgc, start, size, creator) do { \ - ptr START = TO_PTR(UNTYPE_ANY(start)); \ - ptr END = (ptr)((uptr)START + (size)); \ - if ((uptr)START < (uptr)tgc->remote_range_start) \ - tgc->remote_range_start = START; \ - if ((uptr)END > (uptr)tgc->remote_range_end) \ - tgc->remote_range_end = END; \ - tgc->remote_range_tgc = creator; \ - } while (0) -# define RECORD_REMOTE_RANGE(tgc, start, size, si) RECORD_REMOTE_RANGE_TO(tgc, start, size, si->creator) -# define FLUSH_REMOTE_RANGE(tgc, s, g) do { \ - if (tgc->remote_range_start != (ptr)(uptr)-1) { \ - flush_remote_range(tgc, s, g); \ - } \ +# define FLUSH_REMOTE_BLOCK thread_gc *remote_tgc = NULL; +# define RECORD_REMOTE(si) remote_tgc = si->creator +# define FLUSH_REMOTE(tgc, p) do { \ + if (remote_tgc != NULL) \ + push_remote_sweep(tgc, p, remote_tgc); \ } while (0) +# define ASSERT_EMPTY_FLUSH_REMOTE() do { \ + if (remote_tgc != NULL) S_error_abort("non-empty remote flush"); \ + } while (0); static void map_threads_to_sweepers(thread_gc *tgc); static void parallel_sweep_dirty_and_generation(thread_gc *tgc); -static void flush_remote_range(thread_gc *tgc, ISPC s, IGEN g); -static remote_range *send_and_receive_remote_ranges(thread_gc *tgc); +static void push_remote_sweep(thread_gc *tgc, ptr p, thread_gc *remote_tgc); +static void send_and_receive_remote_sweeps(thread_gc *tgc); #define SWEEPER_NONE 0 #define SWEEPER_READY 1 @@ -410,8 +406,7 @@ typedef struct { iptr num_swept_bytes; #ifdef ENABLE_TIMING - int remote_ranges_sent, remote_ranges_received; - iptr remote_ranges_bytes_sent, remote_ranges_bytes_received; + int remotes_sent, remotes_received; uptr step, sweep_accum; #endif } gc_sweeper; @@ -430,13 +425,14 @@ static int num_sweepers; # define GC_MUTEX_RELEASE() do { } while (0) # define SEGMENT_IS_LOCAL(si, p) 1 -# define RECORD_REMOTE_RANGE_TO(tgc, start, size, creator) do { } while (0) -# define RECORD_REMOTE_RANGE(tgc, start, size, si) do { } while (0) -# define FLUSH_REMOTE_RANGE(tgc, s, g) do { } while (0) +# define FLUSH_REMOTE_BLOCK /* empty */ +# define RECORD_REMOTE(si) do { } while (0) +# define FLUSH_REMOTE(tgc, p) do { } while (0) +# define ASSERT_EMPTY_FLUSH_REMOTE() do { } while (0) # define map_threads_to_sweepers(tgc) do { } while (0) # define parallel_sweep_dirty_and_generation(tgc) do { sweep_dirty(tgc); sweep_generation(tgc); } while (0) -# define send_and_receive_remote_ranges(tgc) NULL +# define send_and_receive_remote_sweeps(tgc) do { } while (0) static void sweep_dirty PROTO((thread_gc *tgc)); # define PARALLEL_UNUSED /* empty */ @@ -537,31 +533,31 @@ static int flonum_is_forwarded_p(ptr p, seginfo *si) { /* use relocate_pure for newspace fields that can't point to younger objects or where there's no need to track generations */ -#define relocate_pure(ppp, start, size) do { \ +#define relocate_pure(ppp) do { \ ptr* PPP = ppp; ptr PP = *PPP; \ - relocate_pure_help(PPP, PP, start, size); \ + relocate_pure_help(PPP, PP); \ } while (0) -#define relocate_pure_help(ppp, pp, start, size) do { \ +#define relocate_pure_help(ppp, pp) do { \ seginfo *SI; \ if (!IMMEDIATE(pp) && (SI = MaybeSegInfo(ptr_get_segment(pp))) != NULL) { \ if (SI->old_space) \ - relocate_pure_help_help(ppp, pp, SI, start, size); \ + relocate_pure_help_help(ppp, pp, SI); \ ELSE_MEASURE_NONOLDSPACE(pp) \ } \ } while (0) -#define relocate_pure_help_help(ppp, pp, si, start, size) do { \ - if (SEGMENT_IS_LOCAL(si, pp)) { \ +#define relocate_pure_help_help(ppp, pp, si) do { \ + if (SEGMENT_IS_LOCAL(si, pp)) { \ if (FORWARDEDP(pp, si)) \ *ppp = GET_FWDADDRESS(pp); \ else if (!new_marked(si, pp)) \ mark_or_copy_pure(ppp, pp, si); \ } else \ - RECORD_REMOTE_RANGE(tgc, start, size, si); \ + RECORD_REMOTE(si); \ } while (0) -#define relocate_code(pp, si, start, size) do { \ +#define relocate_code(pp, si) do { \ if (si->old_space) { \ if (SEGMENT_IS_LOCAL(si, pp)) { \ if (FWDMARKER(pp) == forward_marker) \ @@ -569,42 +565,47 @@ static int flonum_is_forwarded_p(ptr p, seginfo *si) { else if (!new_marked(si, pp)) \ mark_or_copy_pure(&pp, pp, si); \ } else \ - RECORD_REMOTE_RANGE(tgc, start, size, si); \ + RECORD_REMOTE(si); \ } ELSE_MEASURE_NONOLDSPACE(pp) \ } while (0) #define mark_or_copy_pure(dest, p, si) do { \ if (CAN_MARK_AND(si->use_marks)) \ - (void)mark_object(tgc, p, si); \ + (void)mark_object(tgc, p, si); \ else \ - (void)copy(tgc, p, si, dest); \ + (void)copy(tgc, p, si, dest); \ } while (0) +#define relocate_pure_now(ppp) do { \ + FLUSH_REMOTE_BLOCK \ + relocate_pure(ppp); \ + ASSERT_EMPTY_FLUSH_REMOTE(); \ + } while (0) /* use relocate_impure for newspace fields that can point to younger objects */ #ifdef NO_DIRTY_NEWSPACE_POINTERS -# define relocate_impure_help(PPP, PP, FROM_G, start, size) do {(void)FROM_G; relocate_pure_help(PPP, PP, start, size);} while (0) -# define relocate_impure(PPP, FROM_G, start, size) do {(void)FROM_G; relocate_pure(PPP, start, size);} while (0) +# define relocate_impure_help(PPP, PP, FROM_G) do {(void)FROM_G; relocate_pure_help(PPP, PP);} while (0) +# define relocate_impure(PPP, FROM_G) do {(void)FROM_G; relocate_pure(PPP);} while (0) #else /* !NO_DIRTY_NEWSPACE_POINTERS */ -#define relocate_impure(ppp, from_g, start, size) do { \ +#define relocate_impure(ppp, from_g) do { \ ptr* PPP = ppp; ptr PP = *PPP; IGEN FROM_G = from_g; \ - relocate_impure_help(PPP, PP, FROM_G, start, size); \ + relocate_impure_help(PPP, PP, FROM_G); \ } while (0) -#define relocate_impure_help(ppp, pp, from_g, start, size) do { \ +#define relocate_impure_help(ppp, pp, from_g) do { \ seginfo *SI; \ if (!IMMEDIATE(pp) && (SI = MaybeSegInfo(ptr_get_segment(pp))) != NULL) { \ if (SI->old_space) \ - relocate_impure_help_help(ppp, pp, from_g, SI, start, size); \ + relocate_impure_help_help(ppp, pp, from_g, SI); \ ELSE_MEASURE_NONOLDSPACE(pp) \ } \ } while (0) -#define relocate_impure_help_help(ppp, pp, from_g, si, start, size) do { \ +#define relocate_impure_help_help(ppp, pp, from_g, si) do { \ IGEN __to_g; \ if (SEGMENT_IS_LOCAL(si, pp)) { \ if (FORWARDEDP(pp, si)) { \ @@ -615,21 +616,21 @@ static int flonum_is_forwarded_p(ptr p, seginfo *si) { } else { \ __to_g = TARGET_GENERATION(si); \ } \ - if (__to_g < from_g) S_record_new_dirty_card(tgc, ppp, __to_g); \ + if (__to_g < from_g) S_record_new_dirty_card(tgc, ppp, __to_g); \ } else \ - RECORD_REMOTE_RANGE(tgc, start, size, si); \ + RECORD_REMOTE(si); \ } while (0) #define mark_or_copy_impure(to_g, dest, p, from_g, si) do { \ if (CAN_MARK_AND(si->use_marks)) \ - to_g = mark_object(tgc, p, si); \ + to_g = mark_object(tgc, p, si); \ else \ - to_g = copy(tgc, p, si, dest); \ + to_g = copy(tgc, p, si, dest); \ } while (0) #endif /* !NO_DIRTY_NEWSPACE_POINTERS */ -#define relocate_dirty(PPP, YOUNGEST, start, size) do { \ +#define relocate_dirty(PPP, YOUNGEST) do { \ seginfo *_si; ptr *_ppp = PPP, _pp = *_ppp; IGEN _pg; \ if (!IMMEDIATE(_pp) && (_si = MaybeSegInfo(ptr_get_segment(_pp))) != NULL) { \ if (!_si->old_space) { \ @@ -642,12 +643,12 @@ static int flonum_is_forwarded_p(ptr p, seginfo *si) { } else if (new_marked(_si, _pp)) { \ _pg = TARGET_GENERATION(_si); \ } else if (CAN_MARK_AND(_si->use_marks)) { \ - _pg = mark_object(tgc, _pp, _si); \ + _pg = mark_object(tgc, _pp, _si); \ } else { \ - _pg = copy(tgc, _pp, _si, _ppp); \ + _pg = copy(tgc, _pp, _si, _ppp); \ } \ } else { \ - RECORD_REMOTE_RANGE(tgc, start, size, _si); \ + RECORD_REMOTE(_si); \ _pg = 0xff; \ } \ } \ @@ -659,17 +660,10 @@ static int flonum_is_forwarded_p(ptr p, seginfo *si) { # define is_counting_root(si, p) (si->counting_mask && (si->counting_mask[segment_bitmap_byte(p)] & segment_bitmap_bit(p))) #endif -#ifdef ENABLE_PARALLEL -static void do_relocate_indirect(thread_gc *tgc, ptr p, ptr start, uptr len) { - relocate_pure(&p, start, len); -} -# define relocate_indirect(p, start, len) do_relocate_indirect(tgc, p, start, len) -#else -static void do_relocate_indirect(thread_gc *tgc, ptr p) { - relocate_pure(&p, NULL, 0); -} -# define relocate_indirect(p, start, len) do_relocate_indirect(tgc, p) -#endif +# define relocate_indirect(p) do { \ + ptr _P = p; \ + relocate_pure(&_P); \ + } while (0) FORCEINLINE void check_triggers(thread_gc *tgc, seginfo *si) { /* Registering ephemerons and guardians to recheck at the @@ -711,7 +705,7 @@ static void sweep_in_old(thread_gc *tgc, ptr p) { /* Detect all the cases when we need to give up on in-place sweeping: */ if (object_directly_refers_to_self(p)) { - relocate_pure(&p, NULL, 0); + relocate_pure_now(&p); return; } @@ -887,6 +881,8 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { /* set up context for sweeping --- effectively remembering the current allocation state so anything new is recognized as needing sweeping */ t_tgc->sweep_stack_start = t_tgc->sweep_stack = t_tgc->sweep_stack_limit = (ptr)0; + t_tgc->send_remote_sweep_stack_start = t_tgc->send_remote_sweep_stack = t_tgc->send_remote_sweep_stack_limit = (ptr)0; + t_tgc->receive_remote_sweep_stack_start = t_tgc->receive_remote_sweep_stack = t_tgc->receive_remote_sweep_stack_limit = (ptr)0; t_tgc->bitmask_overhead[0] = 0; for (g = MIN_TG; g <= MAX_TG; g++) t_tgc->bitmask_overhead[g] = 0; @@ -948,7 +944,7 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { si->next = oldspacesegments; oldspacesegments = si; si->old_space = 1; - /* update generation now, both to compute the target generation, + /* update generation now, both to compute the target generation,< and so that any updated dirty references will record the correct new generation; also used for a check in S_dirty_set */ si->generation = compute_target_generation(si->generation); @@ -1005,7 +1001,7 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { S_G.gcbackreference[g] = Snil; } - SET_BACKREFERENCE(Sfalse) /* #f => root */ + SET_BACKREFERENCE(Sfalse); /* #f => root */ /* Set mark bit for any locked object in `space_new`. Don't sweep until after handling counting roots. Note that the segment won't have @@ -1079,7 +1075,7 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { if (!si->old_space || FORWARDEDP(p, si) || marked(si, p) || !count_roots[i].weak) { /* reached or older; sweep transitively */ - relocate_pure(&p, NULL, 0); + relocate_pure_now(&p); sweep(tgc, p, TARGET_GENERATION(si)); ADD_BACKREFERENCE(p, si->generation); sweep_generation(tgc); @@ -1177,7 +1173,7 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { } else { /* relocate now, then sweeping will happen in sweeper thread */ thread_gc *tgc = THREAD_GC(t_tc); /* shadows enclosing `tgc` binding */ - relocate_pure(&thread, NULL, 0); + relocate_pure_now(&thread); } } #else @@ -1186,7 +1182,7 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { #endif } - relocate_pure(&S_threads, NULL, 0); + relocate_pure_now(&S_threads); GET_REAL_TIME(start); @@ -1229,7 +1225,7 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { /* relocate the protected C pointers */ {uptr i; for (i = 0; i < S_G.protect_next; i++) - relocate_pure(S_G.protected[i], NULL, 0); + relocate_pure_now(S_G.protected[i]); } /* sweep older locked and unlocked objects that are on `space_new` segments, @@ -1344,7 +1340,7 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { /* if tconc was old it's been forwarded */ tconc = GUARDIANTCONC(ls); - WITH_TOP_BACKREFERENCE(tconc, relocate_pure(&rep, NULL, 0)); + WITH_TOP_BACKREFERENCE(tconc, relocate_pure_now(&rep)); old_end = Scdr(tconc); new_end = S_cons_in(tc, space_impure, 0, FIX(0), FIX(0)); @@ -1392,7 +1388,7 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { } rep = GUARDIANREP(ls); - WITH_TOP_BACKREFERENCE(tconc, relocate_pure(&rep, NULL, 0)); + WITH_TOP_BACKREFERENCE(tconc, relocate_pure_now(&rep)); relocate_rep = 1; #ifdef ENABLE_OBJECT_COUNTS @@ -1732,22 +1728,16 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { #ifdef ENABLE_PARALLEL -static void flush_remote_range(thread_gc *tgc, ISPC s, IGEN g) { - remote_range *r; - - find_gc_room_voidp(tgc, space_data, 0, ptr_align(sizeof(remote_range)), r); - tgc->bitmask_overhead[0] += ptr_align(sizeof(remote_range)); - r->s = s; - r->g = g; - r->start = tgc->remote_range_start; - r->end = tgc->remote_range_end; - r->tgc = tgc->remote_range_tgc; - r->next = tgc->ranges_to_send; - tgc->ranges_to_send = r; - - tgc->remote_range_start = (ptr)(uptr)-1; - tgc->remote_range_end = (ptr)0; - +static void push_remote_sweep(thread_gc *tgc, ptr p, thread_gc *remote_tgc) { + if (tgc->send_remote_sweep_stack == tgc->send_remote_sweep_stack_limit) + enlarge_stack(tgc, + &tgc->send_remote_sweep_stack, + &tgc->send_remote_sweep_stack_start, + &tgc->send_remote_sweep_stack_limit, + 2 * ptr_bytes); + ((ptr *)TO_VOIDP(tgc->send_remote_sweep_stack))[0] = p; + ((ptr *)TO_VOIDP(tgc->send_remote_sweep_stack))[1] = TO_PTR(remote_tgc); + tgc->send_remote_sweep_stack = (ptr)((uptr)tgc->send_remote_sweep_stack + 2 * ptr_bytes); tgc->sweep_change = SWEEP_CHANGE_PROGRESS; } @@ -1762,10 +1752,10 @@ static void flush_remote_range(thread_gc *tgc, ISPC s, IGEN g) { while ((si = (seginfo *)TO_VOIDP(tgc->sweep_next[from_g][s])) != NULL) { \ tgc->sweep_next[from_g][s] = si->sweep_next; \ pp = TO_VOIDP(si->sweep_start); \ - while ((p = *pp) != forward_marker) \ - body \ + while ((p = *pp) != forward_marker) { \ + do body while (0); \ + } \ COUNT_SWEPT_BYTES(si->sweep_start, pp); \ - FLUSH_REMOTE_RANGE(tgc, s, from_g); \ save_resweep(s, si); \ } \ } while (0) @@ -1778,10 +1768,9 @@ static void flush_remote_range(thread_gc *tgc, ISPC s, IGEN g) { pp = sl; \ while (pp != nl) { \ p = *pp; \ - body \ + do body while (0); \ } \ COUNT_SWEPT_BYTES(sl, nl); \ - FLUSH_REMOTE_RANGE(tgc, s, from_g); \ } \ } while (0) @@ -1866,7 +1855,6 @@ static iptr sweep_generation_pass(thread_gc *tgc) { ptr *slp, *nlp; ptr *pp, *ppn, p, *nl, *sl; IGEN from_g; seginfo *si; iptr num_swept_bytes = 0; - remote_range *received_ranges; do { tgc->sweep_change = SWEEP_NO_CHANGE; @@ -1876,190 +1864,100 @@ static iptr sweep_generation_pass(thread_gc *tgc) { for (from_g = MIN_TG; from_g <= MAX_TG; from_g += 1) { sweep_space(space_impure, from_g, { - /* only pairs in theses spaces in backreference mode */ - SET_BACKREFERENCE(TYPE(TO_PTR(pp), type_pair)); - relocate_impure_help(pp, p, from_g, pp, 2 * ptr_bytes); - ppn = pp + 1; - p = *ppn; - relocate_impure_help(ppn, p, from_g, pp, 2 * ptr_bytes); - pp = ppn + 1; - }); - SET_BACKREFERENCE(Sfalse) - - sweep_space(space_symbol, from_g, { - p = TYPE(TO_PTR(pp), type_symbol); - sweep_symbol(tgc, p, from_g); - pp += size_symbol / sizeof(ptr); - }); - - sweep_space(space_port, from_g, { - p = TYPE(TO_PTR(pp), type_typed_object); - sweep_port(tgc, p, from_g); - pp += size_port / sizeof(ptr); - }); - - sweep_space(space_weakpair, from_g, { - SET_BACKREFERENCE(TYPE(TO_PTR(pp), type_pair)) - ppn = pp + 1; - p = *ppn; - relocate_impure_help(ppn, p, from_g, pp, size_pair); - pp = ppn + 1; - }); - SET_BACKREFERENCE(Sfalse) - - sweep_space(space_ephemeron, from_g, { - p = TYPE(TO_PTR(pp), type_pair); - add_ephemeron_to_pending(tgc, p); - pp += size_ephemeron / sizeof(ptr); - }); - - sweep_space(space_pure, from_g, { - SET_BACKREFERENCE(TYPE(TO_PTR(pp), type_pair)) /* only pairs put here in backreference mode */ - relocate_impure_help(pp, p, from_g, pp, 2 * ptr_bytes); - ppn = pp + 1; - p = *ppn; - relocate_impure_help(ppn, p, from_g, pp, 2 * ptr_bytes); - pp = ppn + 1; - }); - SET_BACKREFERENCE(Sfalse) - - sweep_space(space_continuation, from_g, { - p = TYPE(TO_PTR(pp), type_closure); - sweep_continuation(tgc, p, from_g); - pp += size_continuation / sizeof(ptr); - }); - - sweep_space(space_pure_typed_object, from_g, { - p = TYPE(TO_PTR(pp), type_typed_object); - pp = TO_VOIDP(((uptr)TO_PTR(pp) + sweep_typed_object(tgc, p, from_g))); - }); - - sweep_space(space_code, from_g, { - p = TYPE(TO_PTR(pp), type_typed_object); - sweep_code_object(tgc, p, from_g); - pp += size_code(CODELEN(p)) / sizeof(ptr); - }); - - sweep_space(space_impure_record, from_g, { - p = TYPE(TO_PTR(pp), type_typed_object); - sweep_record(tgc, p, from_g); - pp = TO_VOIDP((iptr)TO_PTR(pp) + - size_record_inst(UNFIX(RECORDDESCSIZE(RECORDINSTTYPE(p))))); - }); - - /* space used only as needed for backreferences: */ - sweep_space(space_impure_typed_object, from_g, { - p = TYPE(TO_PTR(pp), type_typed_object); - pp = TO_VOIDP((uptr)TO_PTR(pp) + sweep_typed_object(tgc, p, from_g)); - }); - - /* space used only as needed for backreferences: */ - sweep_space(space_closure, from_g, { - p = TYPE(TO_PTR(pp), type_closure); - sweep(tgc, p, from_g); - pp = TO_VOIDP((uptr)TO_PTR(pp) + size_object(p)); - }); - } - - received_ranges = send_and_receive_remote_ranges(tgc); - - /* The ranges in `received_ranges` include old-generation objects from - other parallel sweepers, which means they correspond to dirty - sweeps in the originating sweeper. We handle them here like - regular sweeping using `relocate_impure`, which will register a - dirty-card update as needed. */ - while (received_ranges != NULL) { - ISPC s = received_ranges->s; - IGEN from_g = received_ranges->g; - - pp = TO_VOIDP(received_ranges->start); - nl = TO_VOIDP(received_ranges->end); - - if ((s == space_impure) - || (s == space_immobile_impure) - || (s == space_count_impure) - || (s == space_pure) - || (s == space_impure_typed_object)) { - while (pp < nl) { - p = *pp; - relocate_impure_help(pp, p, from_g, pp, 2 * ptr_bytes); + /* only pairs in theses spaces in backreference mode */ + FLUSH_REMOTE_BLOCK + SET_BACKREFERENCE(TYPE(TO_PTR(pp), type_pair)); + relocate_impure_help(pp, p, from_g); ppn = pp + 1; p = *ppn; - relocate_impure_help(ppn, p, from_g, pp, 2 * ptr_bytes); + relocate_impure_help(ppn, p, from_g); + FLUSH_REMOTE(tgc, TYPE(TO_PTR(pp), type_pair)); pp = ppn + 1; - } - } else if (s == space_new) { /* used by infer_space to mean "type ojbect" */ - while (pp < nl) { - p = TYPE(TO_PTR(pp), type_typed_object); - sweep(tgc, p, from_g); - pp = TO_VOIDP((uptr)TO_PTR(pp) + size_object(p)); - } - } else if (s == space_closure) { - while (pp < nl) { - p = TYPE(TO_PTR(pp), type_closure); - sweep(tgc, p, from_g); - pp = TO_VOIDP((uptr)TO_PTR(pp) + size_object(p)); - } - } else if (s == space_continuation) { - while (pp < nl) { - p = TYPE(TO_PTR(pp), type_closure); - sweep_continuation(tgc, p, from_g); - pp += size_continuation / sizeof(ptr); - } - } else if (s == space_code) { - while (pp < nl) { - p = TYPE(TO_PTR(pp), type_typed_object); - sweep_code_object(tgc, p, from_g); - pp += size_code(CODELEN(p)) / sizeof(ptr); - } - } else if ((s == space_pure_typed_object) - || (s == space_count_pure)) { - /* old generation can happen in the special case of a thread object: */ - while (pp < nl) { - p = TYPE(TO_PTR(pp), type_typed_object); - pp = TO_VOIDP(((uptr)TO_PTR(pp) + sweep_typed_object(tgc, p, from_g))); - } - } else if (s == space_symbol) { - while (pp < nl) { + }); + SET_BACKREFERENCE(Sfalse); + + sweep_space(space_symbol, from_g, { p = TYPE(TO_PTR(pp), type_symbol); sweep_symbol(tgc, p, from_g); pp += size_symbol / sizeof(ptr); - } - } else if (s == space_port) { - while (pp < nl) { + }); + + sweep_space(space_port, from_g, { p = TYPE(TO_PTR(pp), type_typed_object); sweep_port(tgc, p, from_g); pp += size_port / sizeof(ptr); - } - } else if (s == space_weakpair) { - while (pp < nl) { + }); + + sweep_space(space_weakpair, from_g, { + FLUSH_REMOTE_BLOCK + SET_BACKREFERENCE(TYPE(TO_PTR(pp), type_pair)); ppn = pp + 1; p = *ppn; - relocate_impure_help(ppn, p, from_g, pp, size_pair); + relocate_impure_help(ppn, p, from_g); + FLUSH_REMOTE(tgc, TYPE(TO_PTR(pp), type_pair)); pp = ppn + 1; - } - } else if (s == space_ephemeron) { - while (pp < nl) { + }); + SET_BACKREFERENCE(Sfalse); + + sweep_space(space_ephemeron, from_g, { p = TYPE(TO_PTR(pp), type_pair); add_ephemeron_to_pending(tgc, p); pp += size_ephemeron / sizeof(ptr); - } - } else if (s == space_impure_record) { - while (pp < nl) { + }); + + sweep_space(space_pure, from_g, { + FLUSH_REMOTE_BLOCK + SET_BACKREFERENCE(TYPE(TO_PTR(pp), type_pair)); /* only pairs put here in backreference mode */ + relocate_impure_help(pp, p, from_g); + ppn = pp + 1; + p = *ppn; + relocate_impure_help(ppn, p, from_g); + FLUSH_REMOTE(tgc, TYPE(TO_PTR(pp), type_pair)); + pp = ppn + 1; + }); + SET_BACKREFERENCE(Sfalse); + + sweep_space(space_continuation, from_g, { + p = TYPE(TO_PTR(pp), type_closure); + sweep_continuation(tgc, p, from_g); + pp += size_continuation / sizeof(ptr); + }); + + sweep_space(space_pure_typed_object, from_g, { + p = TYPE(TO_PTR(pp), type_typed_object); + pp = TO_VOIDP(((uptr)TO_PTR(pp) + sweep_typed_object(tgc, p, from_g))); + }); + + sweep_space(space_code, from_g, { + p = TYPE(TO_PTR(pp), type_typed_object); + sweep_code_object(tgc, p, from_g); + pp += size_code(CODELEN(p)) / sizeof(ptr); + }); + + sweep_space(space_impure_record, from_g, { p = TYPE(TO_PTR(pp), type_typed_object); sweep_record(tgc, p, from_g); pp = TO_VOIDP((iptr)TO_PTR(pp) + size_record_inst(UNFIX(RECORDDESCSIZE(RECORDINSTTYPE(p))))); - } - } else { - printf(">> %d\n", s); - S_error_abort("dirty range sweep: unexpected space"); - } - FLUSH_REMOTE_RANGE(tgc, s, from_g); - COUNT_SWEPT_BYTES(received_ranges->start, received_ranges->end); - received_ranges = received_ranges->next; + }); + + /* space used only as needed for backreferences: */ + sweep_space(space_impure_typed_object, from_g, { + p = TYPE(TO_PTR(pp), type_typed_object); + pp = TO_VOIDP((uptr)TO_PTR(pp) + sweep_typed_object(tgc, p, from_g)); + }); + + /* space used only as needed for backreferences: */ + sweep_space(space_closure, from_g, { + p = TYPE(TO_PTR(pp), type_closure); + sweep(tgc, p, from_g); + pp = TO_VOIDP((uptr)TO_PTR(pp) + size_object(p)); + }); } + /* May add to the sweep stack: */ + send_and_receive_remote_sweeps(tgc); + /* Waiting until sweeping doesn't trigger a change reduces the chance that an ephemeron must be reigistered as a segment-specific trigger or gets triggered for recheck, but @@ -2076,46 +1974,20 @@ static void sweep_generation(thread_gc *tgc) { sweep_generation_pass(tgc); } -void enlarge_sweep_stack(thread_gc *tgc) { - uptr sz = ((uptr)tgc->sweep_stack_limit - (uptr)tgc->sweep_stack_start); - uptr new_sz = 2 * ((sz == 0) ? 256 : sz); - ptr new_sweep_stack; - find_gc_room(tgc, space_data, 0, typemod, ptr_align(new_sz), new_sweep_stack); +void enlarge_stack(thread_gc *tgc, ptr *stack, ptr *stack_start, ptr *stack_limit, uptr grow_at_least) { + uptr sz = ((uptr)*stack - (uptr)*stack_start); + uptr new_sz = 2 * ((sz == 0) ? sweep_stack_min_size : sz); + ptr new_stack; + if (new_sz - sz < grow_at_least) new_sz += grow_at_least; + find_gc_room(tgc, space_data, 0, typemod, ptr_align(new_sz), new_stack); if (sz != 0) - memcpy(TO_VOIDP(new_sweep_stack), TO_VOIDP(tgc->sweep_stack_start), sz); + memcpy(TO_VOIDP(new_stack), TO_VOIDP(*stack_start), sz); tgc->bitmask_overhead[0] += ptr_align(new_sz); - tgc->sweep_stack_start = new_sweep_stack; - tgc->sweep_stack_limit = (ptr)((uptr)new_sweep_stack + new_sz); - tgc->sweep_stack = (ptr)((uptr)new_sweep_stack + sz); + *stack_start = new_stack; + *stack_limit = (ptr)((uptr)new_stack + new_sz); + *stack = (ptr)((uptr)new_stack + sz); } -#ifdef ENABLE_PARALLEL -static ISPC infer_space(ptr p, seginfo *si) { - /* Certain kinds of values get allocated to more specific spaces by - parallel mode compared to non-parallel mode, and locked objects - may stay in a less-specific space like `space_new`. Marking - objects from a previous collection can mean sweeping from the - less-specific space. We can synthesize an appropropriate space - here, since it will be used only by the handling of received - ranges. */ - - if (si->marked_mask) { - ITYPE t = TYPEBITS(p); - if (t == type_typed_object) - return space_new; /* means "typed object" to sweep_generation_pass */ - else if (t == type_pair) { - if (si->space == space_new) - return space_impure; - } else if (t == type_closure) - return space_closure; - else if (t == type_symbol) - return space_symbol; - } - - return si->space; -} -#endif - iptr sweep_from_stack(thread_gc *tgc) { iptr num_swept_bytes = 0; @@ -2126,12 +1998,10 @@ iptr sweep_from_stack(thread_gc *tgc) { tgc->sweep_stack = (ptr)((uptr)tgc->sweep_stack - ptr_bytes); p = *(ptr *)TO_VOIDP(tgc->sweep_stack); /* Room for improvement: `si->generation` is needed only for - objects that have impure fields, or in parallel mode for - remote ranges. */ + objects that have impure fields. */ si = SegInfo(ptr_get_segment(p)); sweep(tgc, p, si->generation); COUNT_SWEPT_BYTES(0, size_object(p)); - FLUSH_REMOTE_RANGE(tgc, infer_space(p, si), si->generation); } } @@ -2317,25 +2187,26 @@ static uptr sweep_dirty_segments(thread_gc *tgc, seginfo **dirty_segments) { while (pp < ppend) { /* handle two pointers at a time */ if (marked(dirty_si, TO_PTR(pp))) { - relocate_dirty(pp, youngest, pp, 2 * ptr_bytes); + FLUSH_REMOTE_BLOCK + relocate_dirty(pp, youngest); ppn = pp + 1; - relocate_dirty(ppn, youngest, pp, 2 * ptr_bytes); + relocate_dirty(ppn, youngest); + FLUSH_REMOTE(tgc, TYPE(TO_PTR(pp), type_pair)); pp = ppn + 1; } else { - FLUSH_REMOTE_RANGE(tgc, s, from_g); pp += 2; } } - FLUSH_REMOTE_RANGE(tgc, s, from_g); } else { while (pp < ppend && *pp != forward_marker) { /* handle two pointers at a time */ - relocate_dirty(pp, youngest, pp, 2 * ptr_bytes); + FLUSH_REMOTE_BLOCK + relocate_dirty(pp, youngest); ppn = pp + 1; - relocate_dirty(ppn, youngest, pp, 2 * ptr_bytes); + relocate_dirty(ppn, youngest); + FLUSH_REMOTE(tgc, TYPE(TO_PTR(pp), type_pair)); pp = ppn + 1; } - FLUSH_REMOTE_RANGE(tgc, s, from_g); } } else if (s == space_symbol) { /* old symbols cannot overlap segment boundaries @@ -2354,13 +2225,9 @@ static uptr sweep_dirty_segments(thread_gc *tgc, seginfo **dirty_segments) { if (!dirty_si->marked_mask || marked(dirty_si, p)) youngest = sweep_dirty_symbol(tgc, p, youngest); - else - FLUSH_REMOTE_RANGE(tgc, s, from_g); pp += size_symbol / sizeof(ptr); } - - FLUSH_REMOTE_RANGE(tgc, s, from_g); } else if (s == space_port) { /* old ports cannot overlap segment boundaries since any object that spans multiple @@ -2378,13 +2245,9 @@ static uptr sweep_dirty_segments(thread_gc *tgc, seginfo **dirty_segments) { if (!dirty_si->marked_mask || marked(dirty_si, p)) youngest = sweep_dirty_port(tgc, p, youngest); - else - FLUSH_REMOTE_RANGE(tgc, s, from_g); pp += size_port / sizeof(ptr); } - - FLUSH_REMOTE_RANGE(tgc, s, from_g); } else if (s == space_impure_record) { /* abandon hope all ye who enter here */ ptr p; if (dirty_si->marked_mask) { @@ -2462,7 +2325,6 @@ static uptr sweep_dirty_segments(thread_gc *tgc, seginfo **dirty_segments) { seginfo *si = SegInfo(ptr_get_segment(p)); if (!marked(si, p)) { /* skip unmarked words */ - FLUSH_REMOTE_RANGE(tgc, s, from_g); p = (ptr)((uptr)p + byte_alignment); } else { youngest = sweep_dirty_record(tgc, p, youngest); @@ -2471,8 +2333,6 @@ static uptr sweep_dirty_segments(thread_gc *tgc, seginfo **dirty_segments) { RECORDINSTTYPE(p))))); } } - - FLUSH_REMOTE_RANGE(tgc, s, from_g); } else { uptr j; ptr pnext; seginfo *si; @@ -2518,34 +2378,27 @@ static uptr sweep_dirty_segments(thread_gc *tgc, seginfo **dirty_segments) { size_record_inst(UNFIX(RECORDDESCSIZE( RECORDINSTTYPE(p))))); } - - FLUSH_REMOTE_RANGE(tgc, s, from_g); } } else if (s == space_weakpair) { while (pp < ppend && (dirty_si->marked_mask || (*pp != forward_marker))) { /* skip car field and handle cdr field */ if (!dirty_si->marked_mask || marked(dirty_si, TO_PTR(pp))) { + FLUSH_REMOTE_BLOCK ptr *ppn = pp + 1; - relocate_dirty(ppn, youngest, pp, size_pair); + relocate_dirty(ppn, youngest); + FLUSH_REMOTE(tgc, TYPE(TO_PTR(pp), type_pair)); pp = ppn + 1; } else { - FLUSH_REMOTE_RANGE(tgc, s, from_g); pp += 2; } } - - FLUSH_REMOTE_RANGE(tgc, s, from_g); } else if (s == space_ephemeron) { while (pp < ppend && (dirty_si->marked_mask || (*pp != forward_marker))) { ptr p = TYPE(TO_PTR(pp), type_pair); if (!dirty_si->marked_mask || marked(dirty_si, p)) youngest = check_dirty_ephemeron(tgc, p, youngest); - else - FLUSH_REMOTE_RANGE(tgc, s, from_g); pp += size_ephemeron / sizeof(ptr); } - - FLUSH_REMOTE_RANGE(tgc, s, from_g); } else { S_error_abort("sweep_dirty(gc): unexpected space"); } @@ -2723,6 +2576,7 @@ static void add_trigger_ephemerons_to_pending(thread_gc *tgc, ptr pe) { } static void check_ephemeron(thread_gc *tgc, ptr pe) { + FLUSH_REMOTE_BLOCK ptr p; seginfo *si; IGEN from_g; @@ -2741,27 +2595,27 @@ static void check_ephemeron(thread_gc *tgc, ptr pe) { IGEN tg = TARGET_GENERATION(si); if (tg < from_g) S_record_new_dirty_card(tgc, &INITCAR(pe), tg); #endif - relocate_impure(&INITCDR(pe), from_g, pe, size_ephemeron); + relocate_impure(&INITCDR(pe), from_g); } else if (FORWARDEDP(p, si)) { #ifndef NO_DIRTY_NEWSPACE_POINTERS IGEN tg = TARGET_GENERATION(si); if (tg < from_g) S_record_new_dirty_card(tgc, &INITCAR(pe), tg); #endif INITCAR(pe) = FWDADDRESS(p); - relocate_impure(&INITCDR(pe), from_g, pe, size_ephemeron); + relocate_impure(&INITCDR(pe), from_g); } else { /* Not reached, so far; install as trigger */ ephemeron_add(&si->trigger_ephemerons, pe); si->has_triggers = 1; } } else { - RECORD_REMOTE_RANGE_TO(tgc, pe, size_ephemeron, si->creator); + RECORD_REMOTE(si); } } else { - relocate_impure(&INITCDR(pe), from_g, pe, size_ephemeron); + relocate_impure(&INITCDR(pe), from_g); } - FLUSH_REMOTE_RANGE(tgc, space_ephemeron, from_g); + FLUSH_REMOTE(tgc, pe); POP_BACKREFERENCE(); } @@ -2777,6 +2631,8 @@ static void check_pending_ephemerons(thread_gc *tgc) { check_ephemeron(tgc, pe); pe = next_pe; } + + } /* Like check_ephemeron(), but for a dirty, old-generation @@ -2784,6 +2640,7 @@ static void check_pending_ephemerons(thread_gc *tgc) { be less pessimistic than setting `youngest` to the target generation: */ static IGEN check_dirty_ephemeron(thread_gc *tgc, ptr pe, IGEN youngest) { + FLUSH_REMOTE_BLOCK ptr p; seginfo *si; IGEN pg; @@ -2794,10 +2651,10 @@ static IGEN check_dirty_ephemeron(thread_gc *tgc, ptr pe, IGEN youngest) { if (si->old_space) { if (SEGMENT_IS_LOCAL(si, p)) { if (new_marked(si, p)) { - relocate_dirty(&INITCDR(pe), youngest, pe, size_ephemeron); + relocate_dirty(&INITCDR(pe), youngest); } else if (FORWARDEDP(p, si)) { INITCAR(pe) = GET_FWDADDRESS(p); - relocate_dirty(&INITCDR(pe), youngest, pe, size_ephemeron); + relocate_dirty(&INITCDR(pe), youngest); } else { /* Not reached, so far; add to pending list */ add_ephemeron_to_pending(tgc, pe); @@ -2811,20 +2668,23 @@ static IGEN check_dirty_ephemeron(thread_gc *tgc, ptr pe, IGEN youngest) { youngest = pg; } } else { - RECORD_REMOTE_RANGE_TO(tgc, pe, size_ephemeron, si->creator); + RECORD_REMOTE(si); + FLUSH_REMOTE(tgc, pe); return youngest; } } else { if (youngest != MIN_TG && (pg = si->generation) < youngest) youngest = pg; - relocate_dirty(&INITCDR(pe), youngest, pe, size_ephemeron); + relocate_dirty(&INITCDR(pe), youngest); } } else { /* Non-collectable key means that the value determines `youngest`: */ - relocate_dirty(&INITCDR(pe), youngest, pe, size_ephemeron); + relocate_dirty(&INITCDR(pe), youngest); } + FLUSH_REMOTE(tgc, pe); + POP_BACKREFERENCE() return youngest; @@ -3085,10 +2945,8 @@ static void parallel_sweep_dirty_and_generation(thread_gc *tgc) { for (i = 0; i < num_sweepers + 1; i++) { int idx = ((i == num_sweepers) ? main_sweeper_index : i); sweepers[idx].status = SWEEPER_SWEEPING; - ADJUST_COUNTER(sweepers[idx].remote_ranges_sent = 0); - ADJUST_COUNTER(sweepers[idx].remote_ranges_bytes_sent = 0); - ADJUST_COUNTER(sweepers[idx].remote_ranges_received = 0); - ADJUST_COUNTER(sweepers[idx].remote_ranges_bytes_received = 0); + ADJUST_COUNTER(sweepers[idx].remotes_sent = 0); + ADJUST_COUNTER(sweepers[idx].remotes_received = 0); num_running_sweepers++; } s_thread_cond_broadcast(&sweep_cond); @@ -3130,12 +2988,10 @@ static void parallel_sweep_dirty_and_generation(thread_gc *tgc) { } } - REPORT_TIME(fprintf(stderr, "%d swpr +%ld ms %ld ms %ld bytes %d%%/%d sent %d%%/%d received\n", + REPORT_TIME(fprintf(stderr, "%d swpr +%ld ms %ld ms %ld bytes %d sent %d received\n", MAX_CG, sweepers[idx].step, sweepers[idx].sweep_accum, sweepers[idx].num_swept_bytes, - percentage(sweepers[idx].remote_ranges_bytes_sent, sweepers[idx].num_swept_bytes), - sweepers[idx].remote_ranges_sent, - percentage(sweepers[idx].remote_ranges_bytes_received, sweepers[idx].num_swept_bytes), - sweepers[idx].remote_ranges_received)); + sweepers[idx].remotes_sent, + sweepers[idx].remotes_received)); sweepers[idx].first_tgc = sweepers[idx].last_tgc = NULL; } @@ -3156,9 +3012,7 @@ static void run_sweeper(gc_sweeper *sweeper) { s_thread_setspecific(S_tc_key, TO_VOIDP(tgc->tc)); if (tgc->thread) { - seginfo *t_si = SegInfo(ptr_get_segment(tgc->thread)); sweep_thread(tgc, tgc->thread); - FLUSH_REMOTE_RANGE(tgc, t_si->space, t_si->generation); tgc->thread = (ptr)0; } @@ -3171,7 +3025,7 @@ static void run_sweeper(gc_sweeper *sweeper) { while (1) { IBOOL any_ranges = 0; for (tgc = sweeper->first_tgc; tgc != NULL; tgc = tgc->next) { - if (tgc->ranges_received != NULL) { + if (tgc->receive_remote_sweep_stack != tgc->receive_remote_sweep_stack_start) { any_ranges = 1; break; } @@ -3222,47 +3076,56 @@ static void run_sweeper(gc_sweeper *sweeper) { sweeper->num_swept_bytes = num_swept_bytes; } -static remote_range *send_and_receive_remote_ranges(thread_gc *tgc) { - thread_gc *r_tgc; - remote_range *r, *next; - +static void send_and_receive_remote_sweeps(thread_gc *tgc) { (void)s_thread_mutex_lock(&sweep_mutex); - r = tgc->ranges_to_send; - if (r != NULL) { - tgc->sweep_change = SWEEP_CHANGE_PROGRESS; - tgc->ranges_to_send = NULL; - for (; r != NULL; r = next) { - next = r->next; - ADJUST_COUNTER(sweepers[tgc->sweeper].remote_ranges_sent++); - ADJUST_COUNTER(sweepers[tgc->sweeper].remote_ranges_bytes_sent += ((uptr)r->end - (uptr)r->start)); - r_tgc = r->tgc; - r->next = r_tgc->ranges_received; - r_tgc->ranges_received = r; - if (sweepers[r_tgc->sweeper].status == SWEEPER_WAITING_FOR_WORK) { - num_running_sweepers++; - sweepers[r_tgc->sweeper].status = SWEEPER_SWEEPING; - s_thread_cond_signal(&sweepers[r_tgc->sweeper].work_cond); - } + /* Send objects to remote sweepers */ + while (tgc->send_remote_sweep_stack > tgc->send_remote_sweep_stack_start) { + thread_gc *r_tgc; + ptr p; + + tgc->send_remote_sweep_stack = (ptr)((uptr)tgc->send_remote_sweep_stack - (2 * ptr_bytes)); + p = ((ptr *)TO_VOIDP(tgc->send_remote_sweep_stack))[0]; + r_tgc = TO_VOIDP(((ptr *)TO_VOIDP(tgc->send_remote_sweep_stack))[1]); + + if (r_tgc->receive_remote_sweep_stack == r_tgc->receive_remote_sweep_stack_limit) + enlarge_stack(tgc, + &r_tgc->receive_remote_sweep_stack, + &r_tgc->receive_remote_sweep_stack_start, + &r_tgc->receive_remote_sweep_stack_limit, + ptr_bytes); + + *(ptr *)TO_VOIDP(r_tgc->receive_remote_sweep_stack) = p; + r_tgc->receive_remote_sweep_stack = (ptr)((uptr)r_tgc->receive_remote_sweep_stack + ptr_bytes); + + if (sweepers[r_tgc->sweeper].status == SWEEPER_WAITING_FOR_WORK) { + num_running_sweepers++; + sweepers[r_tgc->sweeper].status = SWEEPER_SWEEPING; + s_thread_cond_signal(&sweepers[r_tgc->sweeper].work_cond); } + + ADJUST_COUNTER(sweepers[tgc->sweeper].remotes_sent++); } - r = tgc->ranges_received; - tgc->ranges_received = NULL; + /* Received objects from remote sweepers, moving to sweep stack: */ + if (tgc->receive_remote_sweep_stack != tgc->receive_remote_sweep_stack_start) { + iptr len = (uptr)tgc->receive_remote_sweep_stack - (uptr)tgc->receive_remote_sweep_stack_start; + + tgc->sweep_change = SWEEP_CHANGE_PROGRESS; + + if (((uptr)tgc->sweep_stack + len) > (uptr)tgc->sweep_stack_limit) + enlarge_stack(tgc, &tgc->sweep_stack, &tgc->sweep_stack_start, &tgc->sweep_stack_limit, len); + + memcpy(tgc->sweep_stack, tgc->receive_remote_sweep_stack_start, len); + tgc->sweep_stack = (ptr)((uptr)tgc->sweep_stack + len); + if ((uptr)tgc->sweep_stack > (uptr)tgc->sweep_stack_limit) + abort(); + tgc->receive_remote_sweep_stack = tgc->receive_remote_sweep_stack_start; + + ADJUST_COUNTER(sweepers[tgc->sweeper].remotes_received += (len / ptr_bytes)); + } (void)s_thread_mutex_unlock(&sweep_mutex); - - if (r != NULL) { - tgc->sweep_change = SWEEP_CHANGE_PROGRESS; -#ifdef ENABLE_TIMING - for (next = r; next != NULL; next = next->next) { - ADJUST_COUNTER(sweepers[tgc->sweeper].remote_ranges_received++); - ADJUST_COUNTER(sweepers[tgc->sweeper].remote_ranges_bytes_received += ((uptr)next->end - (uptr)next->start)); - } -#endif - } - - return r; } #endif @@ -3332,7 +3195,7 @@ static void push_measure(thread_gc *tgc, ptr p) if (si->old_space) { /* We must be in a GC--measure fusion, so switch back to GC */ - relocate_pure_help_help(&p, p, si, NULL, 0); + relocate_pure_help_help(&p, p, si); return; } diff --git a/racket/src/ChezScheme/c/gcwrapper.c b/racket/src/ChezScheme/c/gcwrapper.c index 3f0c48bbc1..5591620594 100644 --- a/racket/src/ChezScheme/c/gcwrapper.c +++ b/racket/src/ChezScheme/c/gcwrapper.c @@ -666,13 +666,6 @@ void S_check_heap(aftergc, mcg) IBOOL aftergc; IGEN mcg; { printf("!!! inconsistent thread NEXT %p and BASE %p\n", TO_VOIDP(tgc->next_loc[g][s]), TO_VOIDP(tgc->base_loc[g][s])); } - if ((tgc->remote_range_end != (ptr)0) - || (tgc->remote_range_start != (ptr)(uptr)-1)) { - S_checkheap_errors += 1; - printf("!!! nonempty thread REMOTERANGE %p-%p\n", - TO_VOIDP(tgc->remote_range_start), - TO_VOIDP(tgc->remote_range_end)); - } } } } @@ -1022,7 +1015,7 @@ void S_check_heap(aftergc, mcg) IBOOL aftergc; IGEN mcg; { if (S_checkheap_errors) { printf("heap check failed%s\n", (aftergc ? " after gc" : "")); - exit(1); + abort(); } } diff --git a/racket/src/ChezScheme/c/thread.c b/racket/src/ChezScheme/c/thread.c index 50eaf9bc21..586e0ae93f 100644 --- a/racket/src/ChezScheme/c/thread.c +++ b/racket/src/ChezScheme/c/thread.c @@ -163,11 +163,7 @@ ptr S_create_thread_object(who, p_tc) const char *who; ptr p_tc; { tgc->during_alloc = 0; tgc->sweeper = main_sweeper_index; - tgc->remote_range_start = (ptr)(uptr)-1; - tgc->remote_range_end = (ptr)0; tgc->pending_ephemerons = (ptr)0; - tgc->ranges_to_send = NULL; - tgc->ranges_received = NULL; for (i = 0; i < (int)DIRTY_SEGMENT_LISTS; i++) tgc->dirty_segments[i] = NULL; tgc->thread = (ptr)0; diff --git a/racket/src/ChezScheme/c/types.h b/racket/src/ChezScheme/c/types.h index ecf3a736df..4acf568f0a 100644 --- a/racket/src/ChezScheme/c/types.h +++ b/racket/src/ChezScheme/c/types.h @@ -464,14 +464,6 @@ typedef struct { #define AS_IMPLICIT_ATOMIC(T, X) X #endif -typedef struct remote_range { - ISPC s; - IGEN g; - ptr start, end; - struct thread_gc *tgc; - struct remote_range *next; -} remote_range; - typedef struct thread_gc { ptr tc; ptr thread; /* set only when collecting */ @@ -497,12 +489,16 @@ typedef struct thread_gc { int sweep_change; int sweeper; /* parallel GC: sweeper thread identity */ - - struct thread_gc *remote_range_tgc; - ptr remote_range_start; - ptr remote_range_end; - remote_range *ranges_to_send; /* modified only by owning sweeper */ - remote_range *ranges_received; /* modified with sweeper mutex held */ + + /* modified only by owning sweeper; contains ptr and thread_gc* */ + ptr send_remote_sweep_stack; + ptr send_remote_sweep_stack_start; + ptr send_remote_sweep_stack_limit; + + /* modified with sweeper mutex held; contains just ptr */ + ptr receive_remote_sweep_stack; + ptr receive_remote_sweep_stack_start; + ptr receive_remote_sweep_stack_limit; seginfo *dirty_segments[DIRTY_SEGMENT_LISTS]; diff --git a/racket/src/ChezScheme/mats/Mf-base b/racket/src/ChezScheme/mats/Mf-base index 712734a7cf..74007de5df 100644 --- a/racket/src/ChezScheme/mats/Mf-base +++ b/racket/src/ChezScheme/mats/Mf-base @@ -444,6 +444,9 @@ prettyclean: clean: prettyclean rm -f Make.out +.PHONY: run +run: + env SCHEMEHEAPDIRS=../boot/$m/ ../bin/$m/scheme $(ARGS) ### rules for generating various experr files diff --git a/racket/src/ChezScheme/s/mkgc.ss b/racket/src/ChezScheme/s/mkgc.ss index c5c4dfe051..e5f54860b5 100644 --- a/racket/src/ChezScheme/s/mkgc.ss +++ b/racket/src/ChezScheme/s/mkgc.ss @@ -243,7 +243,7 @@ ;; A stack segment has a single owner, so it's ok for us ;; to sweep the stack content, even though it's on a ;; remote segment relative to the current sweeper. - (RECORD_REMOTE_RANGE _tgc_ _ _size_ s_si)] + (RECORD_REMOTE s_si)] [else (set! (continuation-stack _) (copy_stack _tgc_ @@ -794,7 +794,7 @@ (SEGMENT_IS_LOCAL v_si val)) (trace-symcode symbol-pvalue val)] [else - (RECORD_REMOTE_RANGE _tgc_ _ _size_ v_si)])] + (RECORD_REMOTE v_si)])] [off (trace-symcode symbol-pvalue val)])] [else (trace-symcode symbol-pvalue val)])) @@ -869,7 +869,7 @@ (trace-record-type-pm num rtd)] [else ;; Try again in the bignum's sweeper - (RECORD_REMOTE_RANGE _tgc_ _ _size_ pm_si) + (RECORD_REMOTE pm_si) (set! num S_G.zero_length_bignum)])] [off (trace-record-type-pm num rtd)])] @@ -1110,7 +1110,7 @@ (trace-pure (* (ENTRYNONCOMPACTLIVEMASKADDR oldret))) (set! num (ENTRYLIVEMASK oldret))] [else - (RECORD_REMOTE_RANGE _tgc_ _ _size_ n_si) + (RECORD_REMOTE n_si) (set! num S_G.zero_length_bignum)])]) (let* ([index : iptr (BIGLEN num)]) (while @@ -1140,7 +1140,7 @@ [(sweep sweep-in-old) (define x_si : seginfo* (SegInfo (ptr_get_segment c_p))) (when (-> x_si old_space) - (relocate_code c_p x_si _ _size_) + (relocate_code c_p x_si) (case-mode [sweep-in-old] [else @@ -1222,7 +1222,7 @@ (find_gc_room _tgc_ space_data from_g typemod n t) (memcpy_aligned (TO_VOIDP t) (TO_VOIDP oldt) n))])] [else - (RECORD_REMOTE_RANGE _tgc_ _ _size_ t_si)]))) + (RECORD_REMOTE t_si)]))) (set! (reloc-table-code t) _) (set! (code-reloc _) t)]) (S_record_code_mod (-> _tgc_ tc) (cast uptr (TO_PTR (& (code-data _ 0)))) (cast uptr (code-length _)))] @@ -1526,15 +1526,20 @@ "return si->generation;")] [(sweep) (code-block + "FLUSH_REMOTE_BLOCK" (and (lookup 'maybe-backreferences? config #f) "PUSH_BACKREFERENCE(p)") (body) (and (lookup 'maybe-backreferences? config #f) "POP_BACKREFERENCE()") + "FLUSH_REMOTE(tgc, p);" (and (lookup 'as-dirty? config #f) "return youngest;"))] [(sweep-in-old) - (body)] + (code-block + "FLUSH_REMOTE_BLOCK" + (body) + "ASSERT_EMPTY_FLUSH_REMOTE();")] [(measure) (body)] [(self-test) @@ -2215,27 +2220,19 @@ (define (relocate-statement purity e config) (define mode (lookup 'mode config)) - (define (get-start) (expression '_ config)) - (define (get-size) (cond - [(lookup 'early-rtd? config #f) - (expression '(size_record_inst (UNFIX (record-type-size (record-type _)))) config)] - [(lookup 'early-code? config #f) - (expression '(size_closure (CODEFREE (CLOSCODE _))) config)] - [else - (expression '_size_ config)])) (case mode [(vfasl-sweep) (format "vfasl_relocate(vfi, &~a);" e)] [(sweep-in-old) (if (eq? purity 'pure) - (format "relocate_pure(&~a, ~a, ~a);" e (get-start) (get-size)) - (format "relocate_indirect(~a, ~a, ~a);" e (get-start) (get-size)))] + (format "relocate_pure(&~a);" e) + (format "relocate_indirect(~a);" e))] [else (if (lookup 'as-dirty? config #f) (begin (when (eq? purity 'pure) (error 'relocate-statement "pure as dirty?")) - (format "relocate_dirty(&~a, youngest, ~a, ~a);" e (get-start) (get-size))) - (format "relocate_~a(&~a~a, ~a, ~a);" purity e (if (eq? purity 'impure) ", from_g" "") (get-start) (get-size)))])) + (format "relocate_dirty(&~a, youngest);" e)) + (format "relocate_~a(&~a~a);" purity e (if (eq? purity 'impure) ", from_g" "")))])) (define (measure-statement e) (code