diff --git a/racket/src/ChezScheme/c/Mf-base b/racket/src/ChezScheme/c/Mf-base index 5844703f6c..f37a81a33b 100644 --- a/racket/src/ChezScheme/c/Mf-base +++ b/racket/src/ChezScheme/c/Mf-base @@ -84,6 +84,10 @@ LZ4Sources=../lz4/lib/lz4.h ../lz4/lib/lz4frame.h \ ../lz4/lib/lz4.c ../lz4/lib/lz4frame.c \ ../lz4/lib/lz4hc.c ../lz4/lib/xxhash.c +.PHONY: run +run: + env SCHEMEHEAPDIRS=../boot/$m/ ../bin/$m/scheme $(ARGS) + clean: rm -f *.$o ${mdclean} rm -f Make.out diff --git a/racket/src/ChezScheme/c/gc.c b/racket/src/ChezScheme/c/gc.c index 72840bfcc1..6fe2da838e 100644 --- a/racket/src/ChezScheme/c/gc.c +++ b/racket/src/ChezScheme/c/gc.c @@ -194,7 +194,7 @@ static void resweep_weak_pairs PROTO((thread_gc *tgc, seginfo *oldweakspacesegme static void forward_or_bwp PROTO((ptr *pp, ptr p)); static void sweep_generation PROTO((thread_gc *tgc)); static iptr sweep_from_stack PROTO((thread_gc *tgc)); -static void enlarge_sweep_stack PROTO((thread_gc *tgc)); +static void enlarge_stack PROTO((thread_gc *tgc, ptr *stack, ptr *stack_start, ptr *stack_limit, uptr grow_at_least)); static uptr size_object PROTO((ptr p)); static iptr sweep_typed_object PROTO((thread_gc *tgc, ptr p, IGEN from_g)); static void sweep_symbol PROTO((thread_gc *tgc, ptr p, IGEN from_g)); @@ -266,7 +266,6 @@ static uptr get_cpu_time () { # define ACCUM_CPU_TIME(a, y, x) uptr y = get_cpu_time() - x; a += y # define REPORT_TIME(e) e static uptr collect_accum, all_accum, par_accum; -static int percentage(iptr n, iptr d) { if (d == 0) return 0; else return (n * 100) / d; } # define COUNT_SWEPT_BYTES(start, end) num_swept_bytes += ((uptr)TO_PTR(end) - (uptr)TO_PTR(start)) # define ADJUST_COUNTER(e) e #else @@ -321,11 +320,14 @@ FORCEINLINE IGEN compute_target_generation(IGEN g) { static octet *fully_marked_mask[static_generation+1]; -#define push_sweep(p) { \ - if (tgc->sweep_stack == tgc->sweep_stack_limit) enlarge_sweep_stack(tgc); \ - *(ptr *)TO_VOIDP(tgc->sweep_stack) = p; \ - tgc->sweep_stack = (ptr)((uptr)tgc->sweep_stack + ptr_bytes); \ - } +static const int sweep_stack_min_size = 256; + +#define push_sweep(p) do { \ + if (tgc->sweep_stack == tgc->sweep_stack_limit) \ + enlarge_stack(tgc, &tgc->sweep_stack, &tgc->sweep_stack_start, &tgc->sweep_stack_limit, ptr_bytes); \ + *(ptr *)TO_VOIDP(tgc->sweep_stack) = p; \ + tgc->sweep_stack = (ptr)((uptr)tgc->sweep_stack + ptr_bytes); \ + } while (0) #ifdef ENABLE_MEASURE static uptr measure_total; /* updated by `measure` */ @@ -340,7 +342,7 @@ static ptr sweep_from; # define BACKREFERENCES_ENABLED S_G.enable_object_backreferences # define SET_SWEEP_FROM(p) if (S_G.enable_object_backreferences) sweep_from = p # define WITH_TOP_BACKREFERENCE(v, e) SET_SWEEP_FROM(v); e; SET_SWEEP_FROM(Sfalse) -# define SET_BACKREFERENCE(p) sweep_from = p; +# define SET_BACKREFERENCE(p) sweep_from = p # define PUSH_BACKREFERENCE(p) ptr old_sweep_from = sweep_from; SET_SWEEP_FROM(p); # define POP_BACKREFERENCE() SET_SWEEP_FROM(old_sweep_from); # define ADD_BACKREFERENCE_FROM(p, from_p, tg) do { \ @@ -354,7 +356,7 @@ static ptr sweep_from; #else # define BACKREFERENCES_ENABLED 0 # define WITH_TOP_BACKREFERENCE(v, e) e -# define SET_BACKREFERENCE(p) +# define SET_BACKREFERENCE(p) do { } while (0) # define PUSH_BACKREFERENCE(p) # define POP_BACKREFERENCE() # define ADD_BACKREFERENCE_FROM(p, from_p, from_g) @@ -375,27 +377,21 @@ static int in_parallel_sweepers = 0; # define GC_MUTEX_RELEASE() alloc_mutex_release() # define SEGMENT_IS_LOCAL(si, p) (((si)->creator == tgc) || marked(si, p) || !in_parallel_sweepers) -# define RECORD_REMOTE_RANGE_TO(tgc, start, size, creator) do { \ - ptr START = TO_PTR(UNTYPE_ANY(start)); \ - ptr END = (ptr)((uptr)START + (size)); \ - if ((uptr)START < (uptr)tgc->remote_range_start) \ - tgc->remote_range_start = START; \ - if ((uptr)END > (uptr)tgc->remote_range_end) \ - tgc->remote_range_end = END; \ - tgc->remote_range_tgc = creator; \ - } while (0) -# define RECORD_REMOTE_RANGE(tgc, start, size, si) RECORD_REMOTE_RANGE_TO(tgc, start, size, si->creator) -# define FLUSH_REMOTE_RANGE(tgc, s, g) do { \ - if (tgc->remote_range_start != (ptr)(uptr)-1) { \ - flush_remote_range(tgc, s, g); \ - } \ +# define FLUSH_REMOTE_BLOCK thread_gc *remote_tgc = NULL; +# define RECORD_REMOTE(si) remote_tgc = si->creator +# define FLUSH_REMOTE(tgc, p) do { \ + if (remote_tgc != NULL) \ + push_remote_sweep(tgc, p, remote_tgc); \ } while (0) +# define ASSERT_EMPTY_FLUSH_REMOTE() do { \ + if (remote_tgc != NULL) S_error_abort("non-empty remote flush"); \ + } while (0); static void map_threads_to_sweepers(thread_gc *tgc); static void parallel_sweep_dirty_and_generation(thread_gc *tgc); -static void flush_remote_range(thread_gc *tgc, ISPC s, IGEN g); -static remote_range *send_and_receive_remote_ranges(thread_gc *tgc); +static void push_remote_sweep(thread_gc *tgc, ptr p, thread_gc *remote_tgc); +static void send_and_receive_remote_sweeps(thread_gc *tgc); #define SWEEPER_NONE 0 #define SWEEPER_READY 1 @@ -410,8 +406,7 @@ typedef struct { iptr num_swept_bytes; #ifdef ENABLE_TIMING - int remote_ranges_sent, remote_ranges_received; - iptr remote_ranges_bytes_sent, remote_ranges_bytes_received; + int remotes_sent, remotes_received; uptr step, sweep_accum; #endif } gc_sweeper; @@ -430,13 +425,14 @@ static int num_sweepers; # define GC_MUTEX_RELEASE() do { } while (0) # define SEGMENT_IS_LOCAL(si, p) 1 -# define RECORD_REMOTE_RANGE_TO(tgc, start, size, creator) do { } while (0) -# define RECORD_REMOTE_RANGE(tgc, start, size, si) do { } while (0) -# define FLUSH_REMOTE_RANGE(tgc, s, g) do { } while (0) +# define FLUSH_REMOTE_BLOCK /* empty */ +# define RECORD_REMOTE(si) do { } while (0) +# define FLUSH_REMOTE(tgc, p) do { } while (0) +# define ASSERT_EMPTY_FLUSH_REMOTE() do { } while (0) # define map_threads_to_sweepers(tgc) do { } while (0) # define parallel_sweep_dirty_and_generation(tgc) do { sweep_dirty(tgc); sweep_generation(tgc); } while (0) -# define send_and_receive_remote_ranges(tgc) NULL +# define send_and_receive_remote_sweeps(tgc) do { } while (0) static void sweep_dirty PROTO((thread_gc *tgc)); # define PARALLEL_UNUSED /* empty */ @@ -537,31 +533,31 @@ static int flonum_is_forwarded_p(ptr p, seginfo *si) { /* use relocate_pure for newspace fields that can't point to younger objects or where there's no need to track generations */ -#define relocate_pure(ppp, start, size) do { \ +#define relocate_pure(ppp) do { \ ptr* PPP = ppp; ptr PP = *PPP; \ - relocate_pure_help(PPP, PP, start, size); \ + relocate_pure_help(PPP, PP); \ } while (0) -#define relocate_pure_help(ppp, pp, start, size) do { \ +#define relocate_pure_help(ppp, pp) do { \ seginfo *SI; \ if (!IMMEDIATE(pp) && (SI = MaybeSegInfo(ptr_get_segment(pp))) != NULL) { \ if (SI->old_space) \ - relocate_pure_help_help(ppp, pp, SI, start, size); \ + relocate_pure_help_help(ppp, pp, SI); \ ELSE_MEASURE_NONOLDSPACE(pp) \ } \ } while (0) -#define relocate_pure_help_help(ppp, pp, si, start, size) do { \ - if (SEGMENT_IS_LOCAL(si, pp)) { \ +#define relocate_pure_help_help(ppp, pp, si) do { \ + if (SEGMENT_IS_LOCAL(si, pp)) { \ if (FORWARDEDP(pp, si)) \ *ppp = GET_FWDADDRESS(pp); \ else if (!new_marked(si, pp)) \ mark_or_copy_pure(ppp, pp, si); \ } else \ - RECORD_REMOTE_RANGE(tgc, start, size, si); \ + RECORD_REMOTE(si); \ } while (0) -#define relocate_code(pp, si, start, size) do { \ +#define relocate_code(pp, si) do { \ if (si->old_space) { \ if (SEGMENT_IS_LOCAL(si, pp)) { \ if (FWDMARKER(pp) == forward_marker) \ @@ -569,42 +565,47 @@ static int flonum_is_forwarded_p(ptr p, seginfo *si) { else if (!new_marked(si, pp)) \ mark_or_copy_pure(&pp, pp, si); \ } else \ - RECORD_REMOTE_RANGE(tgc, start, size, si); \ + RECORD_REMOTE(si); \ } ELSE_MEASURE_NONOLDSPACE(pp) \ } while (0) #define mark_or_copy_pure(dest, p, si) do { \ if (CAN_MARK_AND(si->use_marks)) \ - (void)mark_object(tgc, p, si); \ + (void)mark_object(tgc, p, si); \ else \ - (void)copy(tgc, p, si, dest); \ + (void)copy(tgc, p, si, dest); \ } while (0) +#define relocate_pure_now(ppp) do { \ + FLUSH_REMOTE_BLOCK \ + relocate_pure(ppp); \ + ASSERT_EMPTY_FLUSH_REMOTE(); \ + } while (0) /* use relocate_impure for newspace fields that can point to younger objects */ #ifdef NO_DIRTY_NEWSPACE_POINTERS -# define relocate_impure_help(PPP, PP, FROM_G, start, size) do {(void)FROM_G; relocate_pure_help(PPP, PP, start, size);} while (0) -# define relocate_impure(PPP, FROM_G, start, size) do {(void)FROM_G; relocate_pure(PPP, start, size);} while (0) +# define relocate_impure_help(PPP, PP, FROM_G) do {(void)FROM_G; relocate_pure_help(PPP, PP);} while (0) +# define relocate_impure(PPP, FROM_G) do {(void)FROM_G; relocate_pure(PPP);} while (0) #else /* !NO_DIRTY_NEWSPACE_POINTERS */ -#define relocate_impure(ppp, from_g, start, size) do { \ +#define relocate_impure(ppp, from_g) do { \ ptr* PPP = ppp; ptr PP = *PPP; IGEN FROM_G = from_g; \ - relocate_impure_help(PPP, PP, FROM_G, start, size); \ + relocate_impure_help(PPP, PP, FROM_G); \ } while (0) -#define relocate_impure_help(ppp, pp, from_g, start, size) do { \ +#define relocate_impure_help(ppp, pp, from_g) do { \ seginfo *SI; \ if (!IMMEDIATE(pp) && (SI = MaybeSegInfo(ptr_get_segment(pp))) != NULL) { \ if (SI->old_space) \ - relocate_impure_help_help(ppp, pp, from_g, SI, start, size); \ + relocate_impure_help_help(ppp, pp, from_g, SI); \ ELSE_MEASURE_NONOLDSPACE(pp) \ } \ } while (0) -#define relocate_impure_help_help(ppp, pp, from_g, si, start, size) do { \ +#define relocate_impure_help_help(ppp, pp, from_g, si) do { \ IGEN __to_g; \ if (SEGMENT_IS_LOCAL(si, pp)) { \ if (FORWARDEDP(pp, si)) { \ @@ -615,21 +616,21 @@ static int flonum_is_forwarded_p(ptr p, seginfo *si) { } else { \ __to_g = TARGET_GENERATION(si); \ } \ - if (__to_g < from_g) S_record_new_dirty_card(tgc, ppp, __to_g); \ + if (__to_g < from_g) S_record_new_dirty_card(tgc, ppp, __to_g); \ } else \ - RECORD_REMOTE_RANGE(tgc, start, size, si); \ + RECORD_REMOTE(si); \ } while (0) #define mark_or_copy_impure(to_g, dest, p, from_g, si) do { \ if (CAN_MARK_AND(si->use_marks)) \ - to_g = mark_object(tgc, p, si); \ + to_g = mark_object(tgc, p, si); \ else \ - to_g = copy(tgc, p, si, dest); \ + to_g = copy(tgc, p, si, dest); \ } while (0) #endif /* !NO_DIRTY_NEWSPACE_POINTERS */ -#define relocate_dirty(PPP, YOUNGEST, start, size) do { \ +#define relocate_dirty(PPP, YOUNGEST) do { \ seginfo *_si; ptr *_ppp = PPP, _pp = *_ppp; IGEN _pg; \ if (!IMMEDIATE(_pp) && (_si = MaybeSegInfo(ptr_get_segment(_pp))) != NULL) { \ if (!_si->old_space) { \ @@ -642,12 +643,12 @@ static int flonum_is_forwarded_p(ptr p, seginfo *si) { } else if (new_marked(_si, _pp)) { \ _pg = TARGET_GENERATION(_si); \ } else if (CAN_MARK_AND(_si->use_marks)) { \ - _pg = mark_object(tgc, _pp, _si); \ + _pg = mark_object(tgc, _pp, _si); \ } else { \ - _pg = copy(tgc, _pp, _si, _ppp); \ + _pg = copy(tgc, _pp, _si, _ppp); \ } \ } else { \ - RECORD_REMOTE_RANGE(tgc, start, size, _si); \ + RECORD_REMOTE(_si); \ _pg = 0xff; \ } \ } \ @@ -659,17 +660,10 @@ static int flonum_is_forwarded_p(ptr p, seginfo *si) { # define is_counting_root(si, p) (si->counting_mask && (si->counting_mask[segment_bitmap_byte(p)] & segment_bitmap_bit(p))) #endif -#ifdef ENABLE_PARALLEL -static void do_relocate_indirect(thread_gc *tgc, ptr p, ptr start, uptr len) { - relocate_pure(&p, start, len); -} -# define relocate_indirect(p, start, len) do_relocate_indirect(tgc, p, start, len) -#else -static void do_relocate_indirect(thread_gc *tgc, ptr p) { - relocate_pure(&p, NULL, 0); -} -# define relocate_indirect(p, start, len) do_relocate_indirect(tgc, p) -#endif +# define relocate_indirect(p) do { \ + ptr _P = p; \ + relocate_pure(&_P); \ + } while (0) FORCEINLINE void check_triggers(thread_gc *tgc, seginfo *si) { /* Registering ephemerons and guardians to recheck at the @@ -711,7 +705,7 @@ static void sweep_in_old(thread_gc *tgc, ptr p) { /* Detect all the cases when we need to give up on in-place sweeping: */ if (object_directly_refers_to_self(p)) { - relocate_pure(&p, NULL, 0); + relocate_pure_now(&p); return; } @@ -887,6 +881,8 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { /* set up context for sweeping --- effectively remembering the current allocation state so anything new is recognized as needing sweeping */ t_tgc->sweep_stack_start = t_tgc->sweep_stack = t_tgc->sweep_stack_limit = (ptr)0; + t_tgc->send_remote_sweep_stack_start = t_tgc->send_remote_sweep_stack = t_tgc->send_remote_sweep_stack_limit = (ptr)0; + t_tgc->receive_remote_sweep_stack_start = t_tgc->receive_remote_sweep_stack = t_tgc->receive_remote_sweep_stack_limit = (ptr)0; t_tgc->bitmask_overhead[0] = 0; for (g = MIN_TG; g <= MAX_TG; g++) t_tgc->bitmask_overhead[g] = 0; @@ -948,7 +944,7 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { si->next = oldspacesegments; oldspacesegments = si; si->old_space = 1; - /* update generation now, both to compute the target generation, + /* update generation now, both to compute the target generation,< and so that any updated dirty references will record the correct new generation; also used for a check in S_dirty_set */ si->generation = compute_target_generation(si->generation); @@ -1005,7 +1001,7 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { S_G.gcbackreference[g] = Snil; } - SET_BACKREFERENCE(Sfalse) /* #f => root */ + SET_BACKREFERENCE(Sfalse); /* #f => root */ /* Set mark bit for any locked object in `space_new`. Don't sweep until after handling counting roots. Note that the segment won't have @@ -1079,7 +1075,7 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { if (!si->old_space || FORWARDEDP(p, si) || marked(si, p) || !count_roots[i].weak) { /* reached or older; sweep transitively */ - relocate_pure(&p, NULL, 0); + relocate_pure_now(&p); sweep(tgc, p, TARGET_GENERATION(si)); ADD_BACKREFERENCE(p, si->generation); sweep_generation(tgc); @@ -1177,7 +1173,7 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { } else { /* relocate now, then sweeping will happen in sweeper thread */ thread_gc *tgc = THREAD_GC(t_tc); /* shadows enclosing `tgc` binding */ - relocate_pure(&thread, NULL, 0); + relocate_pure_now(&thread); } } #else @@ -1186,7 +1182,7 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { #endif } - relocate_pure(&S_threads, NULL, 0); + relocate_pure_now(&S_threads); GET_REAL_TIME(start); @@ -1229,7 +1225,7 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { /* relocate the protected C pointers */ {uptr i; for (i = 0; i < S_G.protect_next; i++) - relocate_pure(S_G.protected[i], NULL, 0); + relocate_pure_now(S_G.protected[i]); } /* sweep older locked and unlocked objects that are on `space_new` segments, @@ -1344,7 +1340,7 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { /* if tconc was old it's been forwarded */ tconc = GUARDIANTCONC(ls); - WITH_TOP_BACKREFERENCE(tconc, relocate_pure(&rep, NULL, 0)); + WITH_TOP_BACKREFERENCE(tconc, relocate_pure_now(&rep)); old_end = Scdr(tconc); new_end = S_cons_in(tc, space_impure, 0, FIX(0), FIX(0)); @@ -1392,7 +1388,7 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { } rep = GUARDIANREP(ls); - WITH_TOP_BACKREFERENCE(tconc, relocate_pure(&rep, NULL, 0)); + WITH_TOP_BACKREFERENCE(tconc, relocate_pure_now(&rep)); relocate_rep = 1; #ifdef ENABLE_OBJECT_COUNTS @@ -1732,22 +1728,16 @@ ptr GCENTRY(ptr tc, ptr count_roots_ls) { #ifdef ENABLE_PARALLEL -static void flush_remote_range(thread_gc *tgc, ISPC s, IGEN g) { - remote_range *r; - - find_gc_room_voidp(tgc, space_data, 0, ptr_align(sizeof(remote_range)), r); - tgc->bitmask_overhead[0] += ptr_align(sizeof(remote_range)); - r->s = s; - r->g = g; - r->start = tgc->remote_range_start; - r->end = tgc->remote_range_end; - r->tgc = tgc->remote_range_tgc; - r->next = tgc->ranges_to_send; - tgc->ranges_to_send = r; - - tgc->remote_range_start = (ptr)(uptr)-1; - tgc->remote_range_end = (ptr)0; - +static void push_remote_sweep(thread_gc *tgc, ptr p, thread_gc *remote_tgc) { + if (tgc->send_remote_sweep_stack == tgc->send_remote_sweep_stack_limit) + enlarge_stack(tgc, + &tgc->send_remote_sweep_stack, + &tgc->send_remote_sweep_stack_start, + &tgc->send_remote_sweep_stack_limit, + 2 * ptr_bytes); + ((ptr *)TO_VOIDP(tgc->send_remote_sweep_stack))[0] = p; + ((ptr *)TO_VOIDP(tgc->send_remote_sweep_stack))[1] = TO_PTR(remote_tgc); + tgc->send_remote_sweep_stack = (ptr)((uptr)tgc->send_remote_sweep_stack + 2 * ptr_bytes); tgc->sweep_change = SWEEP_CHANGE_PROGRESS; } @@ -1762,10 +1752,10 @@ static void flush_remote_range(thread_gc *tgc, ISPC s, IGEN g) { while ((si = (seginfo *)TO_VOIDP(tgc->sweep_next[from_g][s])) != NULL) { \ tgc->sweep_next[from_g][s] = si->sweep_next; \ pp = TO_VOIDP(si->sweep_start); \ - while ((p = *pp) != forward_marker) \ - body \ + while ((p = *pp) != forward_marker) { \ + do body while (0); \ + } \ COUNT_SWEPT_BYTES(si->sweep_start, pp); \ - FLUSH_REMOTE_RANGE(tgc, s, from_g); \ save_resweep(s, si); \ } \ } while (0) @@ -1778,10 +1768,9 @@ static void flush_remote_range(thread_gc *tgc, ISPC s, IGEN g) { pp = sl; \ while (pp != nl) { \ p = *pp; \ - body \ + do body while (0); \ } \ COUNT_SWEPT_BYTES(sl, nl); \ - FLUSH_REMOTE_RANGE(tgc, s, from_g); \ } \ } while (0) @@ -1866,7 +1855,6 @@ static iptr sweep_generation_pass(thread_gc *tgc) { ptr *slp, *nlp; ptr *pp, *ppn, p, *nl, *sl; IGEN from_g; seginfo *si; iptr num_swept_bytes = 0; - remote_range *received_ranges; do { tgc->sweep_change = SWEEP_NO_CHANGE; @@ -1876,190 +1864,100 @@ static iptr sweep_generation_pass(thread_gc *tgc) { for (from_g = MIN_TG; from_g <= MAX_TG; from_g += 1) { sweep_space(space_impure, from_g, { - /* only pairs in theses spaces in backreference mode */ - SET_BACKREFERENCE(TYPE(TO_PTR(pp), type_pair)); - relocate_impure_help(pp, p, from_g, pp, 2 * ptr_bytes); - ppn = pp + 1; - p = *ppn; - relocate_impure_help(ppn, p, from_g, pp, 2 * ptr_bytes); - pp = ppn + 1; - }); - SET_BACKREFERENCE(Sfalse) - - sweep_space(space_symbol, from_g, { - p = TYPE(TO_PTR(pp), type_symbol); - sweep_symbol(tgc, p, from_g); - pp += size_symbol / sizeof(ptr); - }); - - sweep_space(space_port, from_g, { - p = TYPE(TO_PTR(pp), type_typed_object); - sweep_port(tgc, p, from_g); - pp += size_port / sizeof(ptr); - }); - - sweep_space(space_weakpair, from_g, { - SET_BACKREFERENCE(TYPE(TO_PTR(pp), type_pair)) - ppn = pp + 1; - p = *ppn; - relocate_impure_help(ppn, p, from_g, pp, size_pair); - pp = ppn + 1; - }); - SET_BACKREFERENCE(Sfalse) - - sweep_space(space_ephemeron, from_g, { - p = TYPE(TO_PTR(pp), type_pair); - add_ephemeron_to_pending(tgc, p); - pp += size_ephemeron / sizeof(ptr); - }); - - sweep_space(space_pure, from_g, { - SET_BACKREFERENCE(TYPE(TO_PTR(pp), type_pair)) /* only pairs put here in backreference mode */ - relocate_impure_help(pp, p, from_g, pp, 2 * ptr_bytes); - ppn = pp + 1; - p = *ppn; - relocate_impure_help(ppn, p, from_g, pp, 2 * ptr_bytes); - pp = ppn + 1; - }); - SET_BACKREFERENCE(Sfalse) - - sweep_space(space_continuation, from_g, { - p = TYPE(TO_PTR(pp), type_closure); - sweep_continuation(tgc, p, from_g); - pp += size_continuation / sizeof(ptr); - }); - - sweep_space(space_pure_typed_object, from_g, { - p = TYPE(TO_PTR(pp), type_typed_object); - pp = TO_VOIDP(((uptr)TO_PTR(pp) + sweep_typed_object(tgc, p, from_g))); - }); - - sweep_space(space_code, from_g, { - p = TYPE(TO_PTR(pp), type_typed_object); - sweep_code_object(tgc, p, from_g); - pp += size_code(CODELEN(p)) / sizeof(ptr); - }); - - sweep_space(space_impure_record, from_g, { - p = TYPE(TO_PTR(pp), type_typed_object); - sweep_record(tgc, p, from_g); - pp = TO_VOIDP((iptr)TO_PTR(pp) + - size_record_inst(UNFIX(RECORDDESCSIZE(RECORDINSTTYPE(p))))); - }); - - /* space used only as needed for backreferences: */ - sweep_space(space_impure_typed_object, from_g, { - p = TYPE(TO_PTR(pp), type_typed_object); - pp = TO_VOIDP((uptr)TO_PTR(pp) + sweep_typed_object(tgc, p, from_g)); - }); - - /* space used only as needed for backreferences: */ - sweep_space(space_closure, from_g, { - p = TYPE(TO_PTR(pp), type_closure); - sweep(tgc, p, from_g); - pp = TO_VOIDP((uptr)TO_PTR(pp) + size_object(p)); - }); - } - - received_ranges = send_and_receive_remote_ranges(tgc); - - /* The ranges in `received_ranges` include old-generation objects from - other parallel sweepers, which means they correspond to dirty - sweeps in the originating sweeper. We handle them here like - regular sweeping using `relocate_impure`, which will register a - dirty-card update as needed. */ - while (received_ranges != NULL) { - ISPC s = received_ranges->s; - IGEN from_g = received_ranges->g; - - pp = TO_VOIDP(received_ranges->start); - nl = TO_VOIDP(received_ranges->end); - - if ((s == space_impure) - || (s == space_immobile_impure) - || (s == space_count_impure) - || (s == space_pure) - || (s == space_impure_typed_object)) { - while (pp < nl) { - p = *pp; - relocate_impure_help(pp, p, from_g, pp, 2 * ptr_bytes); + /* only pairs in theses spaces in backreference mode */ + FLUSH_REMOTE_BLOCK + SET_BACKREFERENCE(TYPE(TO_PTR(pp), type_pair)); + relocate_impure_help(pp, p, from_g); ppn = pp + 1; p = *ppn; - relocate_impure_help(ppn, p, from_g, pp, 2 * ptr_bytes); + relocate_impure_help(ppn, p, from_g); + FLUSH_REMOTE(tgc, TYPE(TO_PTR(pp), type_pair)); pp = ppn + 1; - } - } else if (s == space_new) { /* used by infer_space to mean "type ojbect" */ - while (pp < nl) { - p = TYPE(TO_PTR(pp), type_typed_object); - sweep(tgc, p, from_g); - pp = TO_VOIDP((uptr)TO_PTR(pp) + size_object(p)); - } - } else if (s == space_closure) { - while (pp < nl) { - p = TYPE(TO_PTR(pp), type_closure); - sweep(tgc, p, from_g); - pp = TO_VOIDP((uptr)TO_PTR(pp) + size_object(p)); - } - } else if (s == space_continuation) { - while (pp < nl) { - p = TYPE(TO_PTR(pp), type_closure); - sweep_continuation(tgc, p, from_g); - pp += size_continuation / sizeof(ptr); - } - } else if (s == space_code) { - while (pp < nl) { - p = TYPE(TO_PTR(pp), type_typed_object); - sweep_code_object(tgc, p, from_g); - pp += size_code(CODELEN(p)) / sizeof(ptr); - } - } else if ((s == space_pure_typed_object) - || (s == space_count_pure)) { - /* old generation can happen in the special case of a thread object: */ - while (pp < nl) { - p = TYPE(TO_PTR(pp), type_typed_object); - pp = TO_VOIDP(((uptr)TO_PTR(pp) + sweep_typed_object(tgc, p, from_g))); - } - } else if (s == space_symbol) { - while (pp < nl) { + }); + SET_BACKREFERENCE(Sfalse); + + sweep_space(space_symbol, from_g, { p = TYPE(TO_PTR(pp), type_symbol); sweep_symbol(tgc, p, from_g); pp += size_symbol / sizeof(ptr); - } - } else if (s == space_port) { - while (pp < nl) { + }); + + sweep_space(space_port, from_g, { p = TYPE(TO_PTR(pp), type_typed_object); sweep_port(tgc, p, from_g); pp += size_port / sizeof(ptr); - } - } else if (s == space_weakpair) { - while (pp < nl) { + }); + + sweep_space(space_weakpair, from_g, { + FLUSH_REMOTE_BLOCK + SET_BACKREFERENCE(TYPE(TO_PTR(pp), type_pair)); ppn = pp + 1; p = *ppn; - relocate_impure_help(ppn, p, from_g, pp, size_pair); + relocate_impure_help(ppn, p, from_g); + FLUSH_REMOTE(tgc, TYPE(TO_PTR(pp), type_pair)); pp = ppn + 1; - } - } else if (s == space_ephemeron) { - while (pp < nl) { + }); + SET_BACKREFERENCE(Sfalse); + + sweep_space(space_ephemeron, from_g, { p = TYPE(TO_PTR(pp), type_pair); add_ephemeron_to_pending(tgc, p); pp += size_ephemeron / sizeof(ptr); - } - } else if (s == space_impure_record) { - while (pp < nl) { + }); + + sweep_space(space_pure, from_g, { + FLUSH_REMOTE_BLOCK + SET_BACKREFERENCE(TYPE(TO_PTR(pp), type_pair)); /* only pairs put here in backreference mode */ + relocate_impure_help(pp, p, from_g); + ppn = pp + 1; + p = *ppn; + relocate_impure_help(ppn, p, from_g); + FLUSH_REMOTE(tgc, TYPE(TO_PTR(pp), type_pair)); + pp = ppn + 1; + }); + SET_BACKREFERENCE(Sfalse); + + sweep_space(space_continuation, from_g, { + p = TYPE(TO_PTR(pp), type_closure); + sweep_continuation(tgc, p, from_g); + pp += size_continuation / sizeof(ptr); + }); + + sweep_space(space_pure_typed_object, from_g, { + p = TYPE(TO_PTR(pp), type_typed_object); + pp = TO_VOIDP(((uptr)TO_PTR(pp) + sweep_typed_object(tgc, p, from_g))); + }); + + sweep_space(space_code, from_g, { + p = TYPE(TO_PTR(pp), type_typed_object); + sweep_code_object(tgc, p, from_g); + pp += size_code(CODELEN(p)) / sizeof(ptr); + }); + + sweep_space(space_impure_record, from_g, { p = TYPE(TO_PTR(pp), type_typed_object); sweep_record(tgc, p, from_g); pp = TO_VOIDP((iptr)TO_PTR(pp) + size_record_inst(UNFIX(RECORDDESCSIZE(RECORDINSTTYPE(p))))); - } - } else { - printf(">> %d\n", s); - S_error_abort("dirty range sweep: unexpected space"); - } - FLUSH_REMOTE_RANGE(tgc, s, from_g); - COUNT_SWEPT_BYTES(received_ranges->start, received_ranges->end); - received_ranges = received_ranges->next; + }); + + /* space used only as needed for backreferences: */ + sweep_space(space_impure_typed_object, from_g, { + p = TYPE(TO_PTR(pp), type_typed_object); + pp = TO_VOIDP((uptr)TO_PTR(pp) + sweep_typed_object(tgc, p, from_g)); + }); + + /* space used only as needed for backreferences: */ + sweep_space(space_closure, from_g, { + p = TYPE(TO_PTR(pp), type_closure); + sweep(tgc, p, from_g); + pp = TO_VOIDP((uptr)TO_PTR(pp) + size_object(p)); + }); } + /* May add to the sweep stack: */ + send_and_receive_remote_sweeps(tgc); + /* Waiting until sweeping doesn't trigger a change reduces the chance that an ephemeron must be reigistered as a segment-specific trigger or gets triggered for recheck, but @@ -2076,46 +1974,20 @@ static void sweep_generation(thread_gc *tgc) { sweep_generation_pass(tgc); } -void enlarge_sweep_stack(thread_gc *tgc) { - uptr sz = ((uptr)tgc->sweep_stack_limit - (uptr)tgc->sweep_stack_start); - uptr new_sz = 2 * ((sz == 0) ? 256 : sz); - ptr new_sweep_stack; - find_gc_room(tgc, space_data, 0, typemod, ptr_align(new_sz), new_sweep_stack); +void enlarge_stack(thread_gc *tgc, ptr *stack, ptr *stack_start, ptr *stack_limit, uptr grow_at_least) { + uptr sz = ((uptr)*stack - (uptr)*stack_start); + uptr new_sz = 2 * ((sz == 0) ? sweep_stack_min_size : sz); + ptr new_stack; + if (new_sz - sz < grow_at_least) new_sz += grow_at_least; + find_gc_room(tgc, space_data, 0, typemod, ptr_align(new_sz), new_stack); if (sz != 0) - memcpy(TO_VOIDP(new_sweep_stack), TO_VOIDP(tgc->sweep_stack_start), sz); + memcpy(TO_VOIDP(new_stack), TO_VOIDP(*stack_start), sz); tgc->bitmask_overhead[0] += ptr_align(new_sz); - tgc->sweep_stack_start = new_sweep_stack; - tgc->sweep_stack_limit = (ptr)((uptr)new_sweep_stack + new_sz); - tgc->sweep_stack = (ptr)((uptr)new_sweep_stack + sz); + *stack_start = new_stack; + *stack_limit = (ptr)((uptr)new_stack + new_sz); + *stack = (ptr)((uptr)new_stack + sz); } -#ifdef ENABLE_PARALLEL -static ISPC infer_space(ptr p, seginfo *si) { - /* Certain kinds of values get allocated to more specific spaces by - parallel mode compared to non-parallel mode, and locked objects - may stay in a less-specific space like `space_new`. Marking - objects from a previous collection can mean sweeping from the - less-specific space. We can synthesize an appropropriate space - here, since it will be used only by the handling of received - ranges. */ - - if (si->marked_mask) { - ITYPE t = TYPEBITS(p); - if (t == type_typed_object) - return space_new; /* means "typed object" to sweep_generation_pass */ - else if (t == type_pair) { - if (si->space == space_new) - return space_impure; - } else if (t == type_closure) - return space_closure; - else if (t == type_symbol) - return space_symbol; - } - - return si->space; -} -#endif - iptr sweep_from_stack(thread_gc *tgc) { iptr num_swept_bytes = 0; @@ -2126,12 +1998,10 @@ iptr sweep_from_stack(thread_gc *tgc) { tgc->sweep_stack = (ptr)((uptr)tgc->sweep_stack - ptr_bytes); p = *(ptr *)TO_VOIDP(tgc->sweep_stack); /* Room for improvement: `si->generation` is needed only for - objects that have impure fields, or in parallel mode for - remote ranges. */ + objects that have impure fields. */ si = SegInfo(ptr_get_segment(p)); sweep(tgc, p, si->generation); COUNT_SWEPT_BYTES(0, size_object(p)); - FLUSH_REMOTE_RANGE(tgc, infer_space(p, si), si->generation); } } @@ -2317,25 +2187,26 @@ static uptr sweep_dirty_segments(thread_gc *tgc, seginfo **dirty_segments) { while (pp < ppend) { /* handle two pointers at a time */ if (marked(dirty_si, TO_PTR(pp))) { - relocate_dirty(pp, youngest, pp, 2 * ptr_bytes); + FLUSH_REMOTE_BLOCK + relocate_dirty(pp, youngest); ppn = pp + 1; - relocate_dirty(ppn, youngest, pp, 2 * ptr_bytes); + relocate_dirty(ppn, youngest); + FLUSH_REMOTE(tgc, TYPE(TO_PTR(pp), type_pair)); pp = ppn + 1; } else { - FLUSH_REMOTE_RANGE(tgc, s, from_g); pp += 2; } } - FLUSH_REMOTE_RANGE(tgc, s, from_g); } else { while (pp < ppend && *pp != forward_marker) { /* handle two pointers at a time */ - relocate_dirty(pp, youngest, pp, 2 * ptr_bytes); + FLUSH_REMOTE_BLOCK + relocate_dirty(pp, youngest); ppn = pp + 1; - relocate_dirty(ppn, youngest, pp, 2 * ptr_bytes); + relocate_dirty(ppn, youngest); + FLUSH_REMOTE(tgc, TYPE(TO_PTR(pp), type_pair)); pp = ppn + 1; } - FLUSH_REMOTE_RANGE(tgc, s, from_g); } } else if (s == space_symbol) { /* old symbols cannot overlap segment boundaries @@ -2354,13 +2225,9 @@ static uptr sweep_dirty_segments(thread_gc *tgc, seginfo **dirty_segments) { if (!dirty_si->marked_mask || marked(dirty_si, p)) youngest = sweep_dirty_symbol(tgc, p, youngest); - else - FLUSH_REMOTE_RANGE(tgc, s, from_g); pp += size_symbol / sizeof(ptr); } - - FLUSH_REMOTE_RANGE(tgc, s, from_g); } else if (s == space_port) { /* old ports cannot overlap segment boundaries since any object that spans multiple @@ -2378,13 +2245,9 @@ static uptr sweep_dirty_segments(thread_gc *tgc, seginfo **dirty_segments) { if (!dirty_si->marked_mask || marked(dirty_si, p)) youngest = sweep_dirty_port(tgc, p, youngest); - else - FLUSH_REMOTE_RANGE(tgc, s, from_g); pp += size_port / sizeof(ptr); } - - FLUSH_REMOTE_RANGE(tgc, s, from_g); } else if (s == space_impure_record) { /* abandon hope all ye who enter here */ ptr p; if (dirty_si->marked_mask) { @@ -2462,7 +2325,6 @@ static uptr sweep_dirty_segments(thread_gc *tgc, seginfo **dirty_segments) { seginfo *si = SegInfo(ptr_get_segment(p)); if (!marked(si, p)) { /* skip unmarked words */ - FLUSH_REMOTE_RANGE(tgc, s, from_g); p = (ptr)((uptr)p + byte_alignment); } else { youngest = sweep_dirty_record(tgc, p, youngest); @@ -2471,8 +2333,6 @@ static uptr sweep_dirty_segments(thread_gc *tgc, seginfo **dirty_segments) { RECORDINSTTYPE(p))))); } } - - FLUSH_REMOTE_RANGE(tgc, s, from_g); } else { uptr j; ptr pnext; seginfo *si; @@ -2518,34 +2378,27 @@ static uptr sweep_dirty_segments(thread_gc *tgc, seginfo **dirty_segments) { size_record_inst(UNFIX(RECORDDESCSIZE( RECORDINSTTYPE(p))))); } - - FLUSH_REMOTE_RANGE(tgc, s, from_g); } } else if (s == space_weakpair) { while (pp < ppend && (dirty_si->marked_mask || (*pp != forward_marker))) { /* skip car field and handle cdr field */ if (!dirty_si->marked_mask || marked(dirty_si, TO_PTR(pp))) { + FLUSH_REMOTE_BLOCK ptr *ppn = pp + 1; - relocate_dirty(ppn, youngest, pp, size_pair); + relocate_dirty(ppn, youngest); + FLUSH_REMOTE(tgc, TYPE(TO_PTR(pp), type_pair)); pp = ppn + 1; } else { - FLUSH_REMOTE_RANGE(tgc, s, from_g); pp += 2; } } - - FLUSH_REMOTE_RANGE(tgc, s, from_g); } else if (s == space_ephemeron) { while (pp < ppend && (dirty_si->marked_mask || (*pp != forward_marker))) { ptr p = TYPE(TO_PTR(pp), type_pair); if (!dirty_si->marked_mask || marked(dirty_si, p)) youngest = check_dirty_ephemeron(tgc, p, youngest); - else - FLUSH_REMOTE_RANGE(tgc, s, from_g); pp += size_ephemeron / sizeof(ptr); } - - FLUSH_REMOTE_RANGE(tgc, s, from_g); } else { S_error_abort("sweep_dirty(gc): unexpected space"); } @@ -2723,6 +2576,7 @@ static void add_trigger_ephemerons_to_pending(thread_gc *tgc, ptr pe) { } static void check_ephemeron(thread_gc *tgc, ptr pe) { + FLUSH_REMOTE_BLOCK ptr p; seginfo *si; IGEN from_g; @@ -2741,27 +2595,27 @@ static void check_ephemeron(thread_gc *tgc, ptr pe) { IGEN tg = TARGET_GENERATION(si); if (tg < from_g) S_record_new_dirty_card(tgc, &INITCAR(pe), tg); #endif - relocate_impure(&INITCDR(pe), from_g, pe, size_ephemeron); + relocate_impure(&INITCDR(pe), from_g); } else if (FORWARDEDP(p, si)) { #ifndef NO_DIRTY_NEWSPACE_POINTERS IGEN tg = TARGET_GENERATION(si); if (tg < from_g) S_record_new_dirty_card(tgc, &INITCAR(pe), tg); #endif INITCAR(pe) = FWDADDRESS(p); - relocate_impure(&INITCDR(pe), from_g, pe, size_ephemeron); + relocate_impure(&INITCDR(pe), from_g); } else { /* Not reached, so far; install as trigger */ ephemeron_add(&si->trigger_ephemerons, pe); si->has_triggers = 1; } } else { - RECORD_REMOTE_RANGE_TO(tgc, pe, size_ephemeron, si->creator); + RECORD_REMOTE(si); } } else { - relocate_impure(&INITCDR(pe), from_g, pe, size_ephemeron); + relocate_impure(&INITCDR(pe), from_g); } - FLUSH_REMOTE_RANGE(tgc, space_ephemeron, from_g); + FLUSH_REMOTE(tgc, pe); POP_BACKREFERENCE(); } @@ -2777,6 +2631,8 @@ static void check_pending_ephemerons(thread_gc *tgc) { check_ephemeron(tgc, pe); pe = next_pe; } + + } /* Like check_ephemeron(), but for a dirty, old-generation @@ -2784,6 +2640,7 @@ static void check_pending_ephemerons(thread_gc *tgc) { be less pessimistic than setting `youngest` to the target generation: */ static IGEN check_dirty_ephemeron(thread_gc *tgc, ptr pe, IGEN youngest) { + FLUSH_REMOTE_BLOCK ptr p; seginfo *si; IGEN pg; @@ -2794,10 +2651,10 @@ static IGEN check_dirty_ephemeron(thread_gc *tgc, ptr pe, IGEN youngest) { if (si->old_space) { if (SEGMENT_IS_LOCAL(si, p)) { if (new_marked(si, p)) { - relocate_dirty(&INITCDR(pe), youngest, pe, size_ephemeron); + relocate_dirty(&INITCDR(pe), youngest); } else if (FORWARDEDP(p, si)) { INITCAR(pe) = GET_FWDADDRESS(p); - relocate_dirty(&INITCDR(pe), youngest, pe, size_ephemeron); + relocate_dirty(&INITCDR(pe), youngest); } else { /* Not reached, so far; add to pending list */ add_ephemeron_to_pending(tgc, pe); @@ -2811,20 +2668,23 @@ static IGEN check_dirty_ephemeron(thread_gc *tgc, ptr pe, IGEN youngest) { youngest = pg; } } else { - RECORD_REMOTE_RANGE_TO(tgc, pe, size_ephemeron, si->creator); + RECORD_REMOTE(si); + FLUSH_REMOTE(tgc, pe); return youngest; } } else { if (youngest != MIN_TG && (pg = si->generation) < youngest) youngest = pg; - relocate_dirty(&INITCDR(pe), youngest, pe, size_ephemeron); + relocate_dirty(&INITCDR(pe), youngest); } } else { /* Non-collectable key means that the value determines `youngest`: */ - relocate_dirty(&INITCDR(pe), youngest, pe, size_ephemeron); + relocate_dirty(&INITCDR(pe), youngest); } + FLUSH_REMOTE(tgc, pe); + POP_BACKREFERENCE() return youngest; @@ -3085,10 +2945,8 @@ static void parallel_sweep_dirty_and_generation(thread_gc *tgc) { for (i = 0; i < num_sweepers + 1; i++) { int idx = ((i == num_sweepers) ? main_sweeper_index : i); sweepers[idx].status = SWEEPER_SWEEPING; - ADJUST_COUNTER(sweepers[idx].remote_ranges_sent = 0); - ADJUST_COUNTER(sweepers[idx].remote_ranges_bytes_sent = 0); - ADJUST_COUNTER(sweepers[idx].remote_ranges_received = 0); - ADJUST_COUNTER(sweepers[idx].remote_ranges_bytes_received = 0); + ADJUST_COUNTER(sweepers[idx].remotes_sent = 0); + ADJUST_COUNTER(sweepers[idx].remotes_received = 0); num_running_sweepers++; } s_thread_cond_broadcast(&sweep_cond); @@ -3130,12 +2988,10 @@ static void parallel_sweep_dirty_and_generation(thread_gc *tgc) { } } - REPORT_TIME(fprintf(stderr, "%d swpr +%ld ms %ld ms %ld bytes %d%%/%d sent %d%%/%d received\n", + REPORT_TIME(fprintf(stderr, "%d swpr +%ld ms %ld ms %ld bytes %d sent %d received\n", MAX_CG, sweepers[idx].step, sweepers[idx].sweep_accum, sweepers[idx].num_swept_bytes, - percentage(sweepers[idx].remote_ranges_bytes_sent, sweepers[idx].num_swept_bytes), - sweepers[idx].remote_ranges_sent, - percentage(sweepers[idx].remote_ranges_bytes_received, sweepers[idx].num_swept_bytes), - sweepers[idx].remote_ranges_received)); + sweepers[idx].remotes_sent, + sweepers[idx].remotes_received)); sweepers[idx].first_tgc = sweepers[idx].last_tgc = NULL; } @@ -3156,9 +3012,7 @@ static void run_sweeper(gc_sweeper *sweeper) { s_thread_setspecific(S_tc_key, TO_VOIDP(tgc->tc)); if (tgc->thread) { - seginfo *t_si = SegInfo(ptr_get_segment(tgc->thread)); sweep_thread(tgc, tgc->thread); - FLUSH_REMOTE_RANGE(tgc, t_si->space, t_si->generation); tgc->thread = (ptr)0; } @@ -3171,7 +3025,7 @@ static void run_sweeper(gc_sweeper *sweeper) { while (1) { IBOOL any_ranges = 0; for (tgc = sweeper->first_tgc; tgc != NULL; tgc = tgc->next) { - if (tgc->ranges_received != NULL) { + if (tgc->receive_remote_sweep_stack != tgc->receive_remote_sweep_stack_start) { any_ranges = 1; break; } @@ -3222,47 +3076,56 @@ static void run_sweeper(gc_sweeper *sweeper) { sweeper->num_swept_bytes = num_swept_bytes; } -static remote_range *send_and_receive_remote_ranges(thread_gc *tgc) { - thread_gc *r_tgc; - remote_range *r, *next; - +static void send_and_receive_remote_sweeps(thread_gc *tgc) { (void)s_thread_mutex_lock(&sweep_mutex); - r = tgc->ranges_to_send; - if (r != NULL) { - tgc->sweep_change = SWEEP_CHANGE_PROGRESS; - tgc->ranges_to_send = NULL; - for (; r != NULL; r = next) { - next = r->next; - ADJUST_COUNTER(sweepers[tgc->sweeper].remote_ranges_sent++); - ADJUST_COUNTER(sweepers[tgc->sweeper].remote_ranges_bytes_sent += ((uptr)r->end - (uptr)r->start)); - r_tgc = r->tgc; - r->next = r_tgc->ranges_received; - r_tgc->ranges_received = r; - if (sweepers[r_tgc->sweeper].status == SWEEPER_WAITING_FOR_WORK) { - num_running_sweepers++; - sweepers[r_tgc->sweeper].status = SWEEPER_SWEEPING; - s_thread_cond_signal(&sweepers[r_tgc->sweeper].work_cond); - } + /* Send objects to remote sweepers */ + while (tgc->send_remote_sweep_stack > tgc->send_remote_sweep_stack_start) { + thread_gc *r_tgc; + ptr p; + + tgc->send_remote_sweep_stack = (ptr)((uptr)tgc->send_remote_sweep_stack - (2 * ptr_bytes)); + p = ((ptr *)TO_VOIDP(tgc->send_remote_sweep_stack))[0]; + r_tgc = TO_VOIDP(((ptr *)TO_VOIDP(tgc->send_remote_sweep_stack))[1]); + + if (r_tgc->receive_remote_sweep_stack == r_tgc->receive_remote_sweep_stack_limit) + enlarge_stack(tgc, + &r_tgc->receive_remote_sweep_stack, + &r_tgc->receive_remote_sweep_stack_start, + &r_tgc->receive_remote_sweep_stack_limit, + ptr_bytes); + + *(ptr *)TO_VOIDP(r_tgc->receive_remote_sweep_stack) = p; + r_tgc->receive_remote_sweep_stack = (ptr)((uptr)r_tgc->receive_remote_sweep_stack + ptr_bytes); + + if (sweepers[r_tgc->sweeper].status == SWEEPER_WAITING_FOR_WORK) { + num_running_sweepers++; + sweepers[r_tgc->sweeper].status = SWEEPER_SWEEPING; + s_thread_cond_signal(&sweepers[r_tgc->sweeper].work_cond); } + + ADJUST_COUNTER(sweepers[tgc->sweeper].remotes_sent++); } - r = tgc->ranges_received; - tgc->ranges_received = NULL; + /* Received objects from remote sweepers, moving to sweep stack: */ + if (tgc->receive_remote_sweep_stack != tgc->receive_remote_sweep_stack_start) { + iptr len = (uptr)tgc->receive_remote_sweep_stack - (uptr)tgc->receive_remote_sweep_stack_start; + + tgc->sweep_change = SWEEP_CHANGE_PROGRESS; + + if (((uptr)tgc->sweep_stack + len) > (uptr)tgc->sweep_stack_limit) + enlarge_stack(tgc, &tgc->sweep_stack, &tgc->sweep_stack_start, &tgc->sweep_stack_limit, len); + + memcpy(tgc->sweep_stack, tgc->receive_remote_sweep_stack_start, len); + tgc->sweep_stack = (ptr)((uptr)tgc->sweep_stack + len); + if ((uptr)tgc->sweep_stack > (uptr)tgc->sweep_stack_limit) + abort(); + tgc->receive_remote_sweep_stack = tgc->receive_remote_sweep_stack_start; + + ADJUST_COUNTER(sweepers[tgc->sweeper].remotes_received += (len / ptr_bytes)); + } (void)s_thread_mutex_unlock(&sweep_mutex); - - if (r != NULL) { - tgc->sweep_change = SWEEP_CHANGE_PROGRESS; -#ifdef ENABLE_TIMING - for (next = r; next != NULL; next = next->next) { - ADJUST_COUNTER(sweepers[tgc->sweeper].remote_ranges_received++); - ADJUST_COUNTER(sweepers[tgc->sweeper].remote_ranges_bytes_received += ((uptr)next->end - (uptr)next->start)); - } -#endif - } - - return r; } #endif @@ -3332,7 +3195,7 @@ static void push_measure(thread_gc *tgc, ptr p) if (si->old_space) { /* We must be in a GC--measure fusion, so switch back to GC */ - relocate_pure_help_help(&p, p, si, NULL, 0); + relocate_pure_help_help(&p, p, si); return; } diff --git a/racket/src/ChezScheme/c/gcwrapper.c b/racket/src/ChezScheme/c/gcwrapper.c index 3f0c48bbc1..5591620594 100644 --- a/racket/src/ChezScheme/c/gcwrapper.c +++ b/racket/src/ChezScheme/c/gcwrapper.c @@ -666,13 +666,6 @@ void S_check_heap(aftergc, mcg) IBOOL aftergc; IGEN mcg; { printf("!!! inconsistent thread NEXT %p and BASE %p\n", TO_VOIDP(tgc->next_loc[g][s]), TO_VOIDP(tgc->base_loc[g][s])); } - if ((tgc->remote_range_end != (ptr)0) - || (tgc->remote_range_start != (ptr)(uptr)-1)) { - S_checkheap_errors += 1; - printf("!!! nonempty thread REMOTERANGE %p-%p\n", - TO_VOIDP(tgc->remote_range_start), - TO_VOIDP(tgc->remote_range_end)); - } } } } @@ -1022,7 +1015,7 @@ void S_check_heap(aftergc, mcg) IBOOL aftergc; IGEN mcg; { if (S_checkheap_errors) { printf("heap check failed%s\n", (aftergc ? " after gc" : "")); - exit(1); + abort(); } } diff --git a/racket/src/ChezScheme/c/thread.c b/racket/src/ChezScheme/c/thread.c index 50eaf9bc21..586e0ae93f 100644 --- a/racket/src/ChezScheme/c/thread.c +++ b/racket/src/ChezScheme/c/thread.c @@ -163,11 +163,7 @@ ptr S_create_thread_object(who, p_tc) const char *who; ptr p_tc; { tgc->during_alloc = 0; tgc->sweeper = main_sweeper_index; - tgc->remote_range_start = (ptr)(uptr)-1; - tgc->remote_range_end = (ptr)0; tgc->pending_ephemerons = (ptr)0; - tgc->ranges_to_send = NULL; - tgc->ranges_received = NULL; for (i = 0; i < (int)DIRTY_SEGMENT_LISTS; i++) tgc->dirty_segments[i] = NULL; tgc->thread = (ptr)0; diff --git a/racket/src/ChezScheme/c/types.h b/racket/src/ChezScheme/c/types.h index ecf3a736df..4acf568f0a 100644 --- a/racket/src/ChezScheme/c/types.h +++ b/racket/src/ChezScheme/c/types.h @@ -464,14 +464,6 @@ typedef struct { #define AS_IMPLICIT_ATOMIC(T, X) X #endif -typedef struct remote_range { - ISPC s; - IGEN g; - ptr start, end; - struct thread_gc *tgc; - struct remote_range *next; -} remote_range; - typedef struct thread_gc { ptr tc; ptr thread; /* set only when collecting */ @@ -497,12 +489,16 @@ typedef struct thread_gc { int sweep_change; int sweeper; /* parallel GC: sweeper thread identity */ - - struct thread_gc *remote_range_tgc; - ptr remote_range_start; - ptr remote_range_end; - remote_range *ranges_to_send; /* modified only by owning sweeper */ - remote_range *ranges_received; /* modified with sweeper mutex held */ + + /* modified only by owning sweeper; contains ptr and thread_gc* */ + ptr send_remote_sweep_stack; + ptr send_remote_sweep_stack_start; + ptr send_remote_sweep_stack_limit; + + /* modified with sweeper mutex held; contains just ptr */ + ptr receive_remote_sweep_stack; + ptr receive_remote_sweep_stack_start; + ptr receive_remote_sweep_stack_limit; seginfo *dirty_segments[DIRTY_SEGMENT_LISTS]; diff --git a/racket/src/ChezScheme/mats/Mf-base b/racket/src/ChezScheme/mats/Mf-base index 712734a7cf..74007de5df 100644 --- a/racket/src/ChezScheme/mats/Mf-base +++ b/racket/src/ChezScheme/mats/Mf-base @@ -444,6 +444,9 @@ prettyclean: clean: prettyclean rm -f Make.out +.PHONY: run +run: + env SCHEMEHEAPDIRS=../boot/$m/ ../bin/$m/scheme $(ARGS) ### rules for generating various experr files diff --git a/racket/src/ChezScheme/s/mkgc.ss b/racket/src/ChezScheme/s/mkgc.ss index c5c4dfe051..e5f54860b5 100644 --- a/racket/src/ChezScheme/s/mkgc.ss +++ b/racket/src/ChezScheme/s/mkgc.ss @@ -243,7 +243,7 @@ ;; A stack segment has a single owner, so it's ok for us ;; to sweep the stack content, even though it's on a ;; remote segment relative to the current sweeper. - (RECORD_REMOTE_RANGE _tgc_ _ _size_ s_si)] + (RECORD_REMOTE s_si)] [else (set! (continuation-stack _) (copy_stack _tgc_ @@ -794,7 +794,7 @@ (SEGMENT_IS_LOCAL v_si val)) (trace-symcode symbol-pvalue val)] [else - (RECORD_REMOTE_RANGE _tgc_ _ _size_ v_si)])] + (RECORD_REMOTE v_si)])] [off (trace-symcode symbol-pvalue val)])] [else (trace-symcode symbol-pvalue val)])) @@ -869,7 +869,7 @@ (trace-record-type-pm num rtd)] [else ;; Try again in the bignum's sweeper - (RECORD_REMOTE_RANGE _tgc_ _ _size_ pm_si) + (RECORD_REMOTE pm_si) (set! num S_G.zero_length_bignum)])] [off (trace-record-type-pm num rtd)])] @@ -1110,7 +1110,7 @@ (trace-pure (* (ENTRYNONCOMPACTLIVEMASKADDR oldret))) (set! num (ENTRYLIVEMASK oldret))] [else - (RECORD_REMOTE_RANGE _tgc_ _ _size_ n_si) + (RECORD_REMOTE n_si) (set! num S_G.zero_length_bignum)])]) (let* ([index : iptr (BIGLEN num)]) (while @@ -1140,7 +1140,7 @@ [(sweep sweep-in-old) (define x_si : seginfo* (SegInfo (ptr_get_segment c_p))) (when (-> x_si old_space) - (relocate_code c_p x_si _ _size_) + (relocate_code c_p x_si) (case-mode [sweep-in-old] [else @@ -1222,7 +1222,7 @@ (find_gc_room _tgc_ space_data from_g typemod n t) (memcpy_aligned (TO_VOIDP t) (TO_VOIDP oldt) n))])] [else - (RECORD_REMOTE_RANGE _tgc_ _ _size_ t_si)]))) + (RECORD_REMOTE t_si)]))) (set! (reloc-table-code t) _) (set! (code-reloc _) t)]) (S_record_code_mod (-> _tgc_ tc) (cast uptr (TO_PTR (& (code-data _ 0)))) (cast uptr (code-length _)))] @@ -1526,15 +1526,20 @@ "return si->generation;")] [(sweep) (code-block + "FLUSH_REMOTE_BLOCK" (and (lookup 'maybe-backreferences? config #f) "PUSH_BACKREFERENCE(p)") (body) (and (lookup 'maybe-backreferences? config #f) "POP_BACKREFERENCE()") + "FLUSH_REMOTE(tgc, p);" (and (lookup 'as-dirty? config #f) "return youngest;"))] [(sweep-in-old) - (body)] + (code-block + "FLUSH_REMOTE_BLOCK" + (body) + "ASSERT_EMPTY_FLUSH_REMOTE();")] [(measure) (body)] [(self-test) @@ -2215,27 +2220,19 @@ (define (relocate-statement purity e config) (define mode (lookup 'mode config)) - (define (get-start) (expression '_ config)) - (define (get-size) (cond - [(lookup 'early-rtd? config #f) - (expression '(size_record_inst (UNFIX (record-type-size (record-type _)))) config)] - [(lookup 'early-code? config #f) - (expression '(size_closure (CODEFREE (CLOSCODE _))) config)] - [else - (expression '_size_ config)])) (case mode [(vfasl-sweep) (format "vfasl_relocate(vfi, &~a);" e)] [(sweep-in-old) (if (eq? purity 'pure) - (format "relocate_pure(&~a, ~a, ~a);" e (get-start) (get-size)) - (format "relocate_indirect(~a, ~a, ~a);" e (get-start) (get-size)))] + (format "relocate_pure(&~a);" e) + (format "relocate_indirect(~a);" e))] [else (if (lookup 'as-dirty? config #f) (begin (when (eq? purity 'pure) (error 'relocate-statement "pure as dirty?")) - (format "relocate_dirty(&~a, youngest, ~a, ~a);" e (get-start) (get-size))) - (format "relocate_~a(&~a~a, ~a, ~a);" purity e (if (eq? purity 'impure) ", from_g" "") (get-start) (get-size)))])) + (format "relocate_dirty(&~a, youngest);" e)) + (format "relocate_~a(&~a~a);" purity e (if (eq? purity 'impure) ", from_g" "")))])) (define (measure-statement e) (code