diff --git a/.makefile b/.makefile index 294bf03f4b..bbff3bff49 100644 --- a/.makefile +++ b/.makefile @@ -338,7 +338,7 @@ RACKET_FOR_BOOTFILES = $(RACKET) RACKET_FOR_BUILD = $(RACKET) # This branch name changes each time the pb boot files are updated: -PB_BRANCH == circa-7.8.0.10-11 +PB_BRANCH == circa-7.8.0.10-12 PB_REPO == https://github.com/racket/pb # Alternative source for Chez Scheme boot files, normally set by diff --git a/Makefile b/Makefile index 4b52cc66c8..9d21016371 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ RACKETCS_SUFFIX = RACKET = RACKET_FOR_BOOTFILES = $(RACKET) RACKET_FOR_BUILD = $(RACKET) -PB_BRANCH = circa-7.8.0.10-11 +PB_BRANCH = circa-7.8.0.10-12 PB_REPO = https://github.com/racket/pb EXTRA_REPOS_BASE = CS_CROSS_SUFFIX = @@ -306,14 +306,14 @@ maybe-fetch-pb-as-is: echo done fetch-pb-from: mkdir -p racket/src/ChezScheme/boot - if [ ! -d racket/src/ChezScheme/boot/pb ] ; then git clone -q -b circa-7.8.0.10-11 https://github.com/racket/pb racket/src/ChezScheme/boot/pb ; else cd racket/src/ChezScheme/boot/pb && git fetch -q origin circa-7.8.0.10-11:remotes/origin/circa-7.8.0.10-11 ; fi - cd racket/src/ChezScheme/boot/pb && git checkout -q circa-7.8.0.10-11 + if [ ! -d racket/src/ChezScheme/boot/pb ] ; then git clone -q -b circa-7.8.0.10-12 https://github.com/racket/pb racket/src/ChezScheme/boot/pb ; else cd racket/src/ChezScheme/boot/pb && git fetch -q origin circa-7.8.0.10-12:remotes/origin/circa-7.8.0.10-12 ; fi + cd racket/src/ChezScheme/boot/pb && git checkout -q circa-7.8.0.10-12 pb-stage: - cd racket/src/ChezScheme/boot/pb && git branch circa-7.8.0.10-11 - cd racket/src/ChezScheme/boot/pb && git checkout circa-7.8.0.10-11 + cd racket/src/ChezScheme/boot/pb && git branch circa-7.8.0.10-12 + cd racket/src/ChezScheme/boot/pb && git checkout circa-7.8.0.10-12 cd racket/src/ChezScheme/boot/pb && git add . && git commit --amend -m "new build" pb-push: - cd racket/src/ChezScheme/boot/pb && git push -u origin circa-7.8.0.10-11 + cd racket/src/ChezScheme/boot/pb && git push -u origin circa-7.8.0.10-12 win-cs-base: IF "$(RACKET_FOR_BUILD)" == "" $(MAKE) win-bc-then-cs-base SETUP_BOOT_MODE=--boot WIN32_BUILD_LEVEL=bc PLAIN_RACKET=racket\racketbc DISABLE_STATIC_LIBS="$(DISABLE_STATIC_LIBS)" EXTRA_REPOS_BASE="$(EXTRA_REPOS_BASE)" GIT_CLONE_ARGS_qq="$(GIT_CLONE_ARGS_qq)" JOB_OPTIONS="$(JOB_OPTIONS)" PLT_SETUP_OPTIONS="$(PLT_SETUP_OPTIONS)" RACKETBC_SUFFIX="$(RACKETBC_SUFFIX)" RACKETCS_SUFFIX="$(RACKETCS_SUFFIX)" IF not "$(RACKET_FOR_BUILD)" == "" $(MAKE) win-just-cs-base SETUP_BOOT_MODE=--chain DISABLE_STATIC_LIBS="$(DISABLE_STATIC_LIBS)" EXTRA_REPOS_BASE="$(EXTRA_REPOS_BASE)" GIT_CLONE_ARGS_qq="$(GIT_CLONE_ARGS_qq)" JOB_OPTIONS="$(JOB_OPTIONS)" PLT_SETUP_OPTIONS="$(PLT_SETUP_OPTIONS)" RACKETCS_SUFFIX="$(RACKETCS_SUFFIX)" RACKET_FOR_BUILD="$(RACKET_FOR_BUILD)" diff --git a/racket/src/ChezScheme/c/alloc.c b/racket/src/ChezScheme/c/alloc.c index d361b8c27f..d1a6292b66 100644 --- a/racket/src/ChezScheme/c/alloc.c +++ b/racket/src/ChezScheme/c/alloc.c @@ -26,13 +26,16 @@ void S_alloc_init() { if (S_boot_time) { ptr tc = TO_PTR(S_G.thread_context); + GCDATA(tc) = TO_PTR(&S_G.main_thread_gc); + S_G.main_thread_gc.tc = tc; + /* reset the allocation tables */ for (g = 0; g <= static_generation; g++) { S_G.bytes_of_generation[g] = 0; for (s = 0; s <= max_real_space; s++) { - BASELOC_AT(tc, s, g) = FIX(0); - NEXTLOC_AT(tc, s, g) = FIX(0); - BYTESLEFT_AT(tc, s, g) = 0; + S_G.main_thread_gc.base_loc[g][s] = FIX(0); + S_G.main_thread_gc.next_loc[g][s] = FIX(0); + S_G.main_thread_gc.bytes_left[g][s] = 0; S_G.bytes_of_space[g][s] = 0; } } @@ -159,9 +162,9 @@ ptr S_compute_bytes_allocated(xg, xs) ptr xg; ptr xs; { /* add in bytes previously recorded */ n += S_G.bytes_of_space[g][s]; /* add in bytes in active segments */ - next_loc = NEXTLOC_AT(tc, s, g); + next_loc = THREAD_GC(tc)->next_loc[g][s]; if (next_loc != FIX(0)) - n += (uptr)next_loc - (uptr)BASELOC_AT(tc, s, g); + n += (uptr)next_loc - (uptr)THREAD_GC(tc)->base_loc[g][s]; if (s == space_data) { /* don't count space used for bitmaks */ n -= S_G.bitmask_overhead[g]; @@ -190,7 +193,7 @@ static void maybe_fire_collector() { } /* suitable mutex (either tc_mutex or gc_tc_mutex) must be held */ -static void close_off_segment(ptr tc, ptr old, ptr base_loc, ptr sweep_loc, ISPC s, IGEN g) +static void close_off_segment(thread_gc *tgc, ptr old, ptr base_loc, ptr sweep_loc, ISPC s, IGEN g) { if (base_loc) { seginfo *si; @@ -206,12 +209,12 @@ static void close_off_segment(ptr tc, ptr old, ptr base_loc, ptr sweep_loc, ISPC /* in case this is during a GC, add to sweep list */ si = SegInfo(addr_get_segment(base_loc)); si->sweep_start = sweep_loc; - si->sweep_next = TO_VOIDP(SWEEPNEXT_AT(tc, s, g)); - SWEEPNEXT_AT(tc, s, g) = TO_PTR(si); + si->sweep_next = tgc->sweep_next[g][s]; + tgc->sweep_next[g][s] = si; } } -ptr S_find_more_thread_room(ptr tc, ISPC s, IGEN g, iptr n, ptr old) { +ptr S_find_more_gc_room(thread_gc *tgc, ISPC s, IGEN g, iptr n, ptr old) { iptr nsegs, seg; ptr new; iptr new_bytes; @@ -224,8 +227,8 @@ ptr S_find_more_thread_room(ptr tc, ISPC s, IGEN g, iptr n, ptr old) { #else tc_mutex_acquire(); #endif - - close_off_segment(tc, old, BASELOC_AT(tc, s, g), SWEEPLOC_AT(tc, s, g), s, g); + + close_off_segment(tgc, old, tgc->base_loc[g][s], tgc->sweep_loc[g][s], s, g); S_pants_down += 1; @@ -234,15 +237,15 @@ ptr S_find_more_thread_room(ptr tc, ISPC s, IGEN g, iptr n, ptr old) { /* block requests to minimize fragmentation and improve cache locality */ if (s == space_code && nsegs < 16) nsegs = 16; - seg = S_find_segments(tc, s, g, nsegs); + seg = S_find_segments(tgc, s, g, nsegs); new = build_ptr(seg, 0); new_bytes = nsegs * bytes_per_segment; - BASELOC_AT(tc, s, g) = new; - SWEEPLOC_AT(tc, s, g) = new; - BYTESLEFT_AT(tc, s, g) = (new_bytes - n) - ptr_bytes; - NEXTLOC_AT(tc, s, g) = (ptr)((uptr)new + n); + tgc->base_loc[g][s] = new; + tgc->sweep_loc[g][s] = new; + tgc->bytes_left[g][s] = (new_bytes - n) - ptr_bytes; + tgc->next_loc[g][s] = (ptr)((uptr)new + n); if (g == 0 && S_pants_down == 1) maybe_fire_collector(); @@ -262,12 +265,14 @@ ptr S_find_more_thread_room(ptr tc, ISPC s, IGEN g, iptr n, ptr old) { /* tc_mutex must be held */ void S_close_off_thread_local_segment(ptr tc, ISPC s, IGEN g) { - close_off_segment(tc, NEXTLOC_AT(tc, s, g), BASELOC_AT(tc, s, g), SWEEPLOC_AT(tc, s, g), s, g); + thread_gc *tgc = THREAD_GC(tc); - BASELOC_AT(tc, s, g) = (ptr)0; - BYTESLEFT_AT(tc, s, g) = 0; - NEXTLOC_AT(tc, s, g) = (ptr)0; - SWEEPLOC_AT(tc, s, g) = (ptr)0; + close_off_segment(tgc, tgc->next_loc[g][s], tgc->base_loc[g][s], tgc->sweep_loc[g][s], s, g); + + tgc->base_loc[g][s] = (ptr)0; + tgc->bytes_left[g][s] = 0; + tgc->next_loc[g][s] = (ptr)0; + tgc->sweep_loc[g][s] = (ptr)0; } /* S_reset_allocation_pointer is always called with mutex */ @@ -285,14 +290,14 @@ void S_reset_allocation_pointer(tc) ptr tc; { S_pants_down += 1; - seg = S_find_segments(tc, space_new, 0, 1); + seg = S_find_segments(THREAD_GC(tc), space_new, 0, 1); /* NB: if allocate_segments didn't already ensure we don't use the last segment of memory, we'd have to reject it here so cp2-alloc can avoid a carry check for small allocation requests, using something like this: if (seg == (((uptr)1 << (ptr_bits - segment_offset_bits)) - 1)) - seg = S_find_segments(tc, space_new, 0, 1); + seg = S_find_segments(THREAD_GC(tc), space_new, 0, 1); */ S_G.bytes_of_space[0][space_new] += bytes_per_segment; @@ -306,7 +311,7 @@ void S_reset_allocation_pointer(tc) ptr tc; { S_pants_down -= 1; } -void S_record_new_dirty_card(ptr tc, ptr *ppp, IGEN to_g) { +void S_record_new_dirty_card(thread_gc *tgc, ptr *ppp, IGEN to_g) { uptr card = (uptr)TO_PTR(ppp) >> card_offset_bits; dirtycardinfo *ndc; @@ -316,7 +321,7 @@ void S_record_new_dirty_card(ptr tc, ptr *ppp, IGEN to_g) { if (to_g < ndc->youngest) ndc->youngest = to_g; } else { dirtycardinfo *next = ndc; - find_room_voidp(tc, space_new, 0, ptr_align(sizeof(dirtycardinfo)), ndc); + find_gc_room_voidp(tgc, space_new, 0, ptr_align(sizeof(dirtycardinfo)), ndc); ndc->card = card; ndc->youngest = to_g; ndc->next = next; @@ -353,7 +358,7 @@ void S_dirty_set(ptr *loc, ptr x) { if (!IMMEDIATE(x)) { seginfo *t_si = SegInfo(ptr_get_segment(x)); if (t_si->generation < si->generation) - S_record_new_dirty_card(get_thread_context(), loc, t_si->generation); + S_record_new_dirty_card(THREAD_GC(get_thread_context()), loc, t_si->generation); } } else { IGEN from_g = si->generation; diff --git a/racket/src/ChezScheme/c/externs.h b/racket/src/ChezScheme/c/externs.h index 7727f90b8a..d91936c324 100644 --- a/racket/src/ChezScheme/c/externs.h +++ b/racket/src/ChezScheme/c/externs.h @@ -66,8 +66,8 @@ extern void S_reset_allocation_pointer PROTO((ptr tc)); extern ptr S_compute_bytes_allocated PROTO((ptr xg, ptr xs)); extern ptr S_bytes_finalized PROTO(()); extern ptr S_find_more_room PROTO((ISPC s, IGEN g, iptr n, ptr old)); -extern void S_record_new_dirty_card PROTO((ptr tc, ptr *ppp, IGEN to_g)); -extern ptr S_find_more_thread_room PROTO((ptr tc, IGEN g, ISPC s, iptr n, ptr old)); +extern void S_record_new_dirty_card PROTO((thread_gc *tgc, ptr *ppp, IGEN to_g)); +extern ptr S_find_more_gc_room PROTO((thread_gc *tgc, IGEN g, ISPC s, iptr n, ptr old)); extern void S_close_off_thread_local_segment PROTO((ptr tc, ISPC s, IGEN g)); extern void S_dirty_set PROTO((ptr *loc, ptr x)); extern void S_mark_card_dirty PROTO((uptr card, IGEN to_g)); @@ -382,7 +382,7 @@ extern INT matherr PROTO((struct exception *x)); extern void S_segment_init PROTO((void)); extern void *S_getmem PROTO((iptr bytes, IBOOL zerofill)); extern void S_freemem PROTO((void *addr, iptr bytes)); -extern iptr S_find_segments PROTO((ptr tc, ISPC s, IGEN g, iptr n)); +extern iptr S_find_segments PROTO((thread_gc *creator, ISPC s, IGEN g, iptr n)); extern void S_free_chunk PROTO((chunkinfo *chunk)); extern void S_free_chunks PROTO((void)); extern uptr S_curmembytes PROTO((void)); diff --git a/racket/src/ChezScheme/c/gc.c b/racket/src/ChezScheme/c/gc.c index 7fd7483d48..414aafe7eb 100644 --- a/racket/src/ChezScheme/c/gc.c +++ b/racket/src/ChezScheme/c/gc.c @@ -183,65 +183,65 @@ */ /* locally defined functions */ -static IGEN copy PROTO((ptr tc_in, ptr pp, seginfo *si, ptr *dest)); -static IGEN mark_object PROTO((ptr tc_in, ptr pp, seginfo *si)); -static void sweep PROTO((ptr tc_in, ptr p, IGEN from_g)); -static void sweep_in_old PROTO((ptr tc_in, ptr p)); -static void sweep_object_in_old PROTO((ptr tc_in, ptr p)); +static IGEN copy PROTO((thread_gc *tgc, ptr pp, seginfo *si, ptr *dest)); +static IGEN mark_object PROTO((thread_gc *tgc, ptr pp, seginfo *si)); +static void sweep PROTO((thread_gc *tgc, ptr p, IGEN from_g)); +static void sweep_in_old PROTO((thread_gc *tgc, ptr p)); +static void sweep_object_in_old PROTO((thread_gc *tgc, ptr p)); static IBOOL object_directly_refers_to_self PROTO((ptr p)); -static ptr copy_stack PROTO((ptr tc_in, ptr old, iptr *length, iptr clength)); -static void resweep_weak_pairs PROTO((ptr tc, seginfo *oldweakspacesegments)); +static ptr copy_stack PROTO((thread_gc *tgc, ptr old, iptr *length, iptr clength)); +static void resweep_weak_pairs PROTO((thread_gc *tgc, seginfo *oldweakspacesegments)); static void forward_or_bwp PROTO((ptr *pp, ptr p)); -static void sweep_generation PROTO((ptr tc)); -static iptr sweep_from_stack PROTO((ptr tc)); -static void enlarge_sweep_stack PROTO((ptr tc)); +static void sweep_generation PROTO((thread_gc *tgc)); +static iptr sweep_from_stack PROTO((thread_gc *tgc)); +static void enlarge_sweep_stack PROTO((thread_gc *tgc)); static uptr size_object PROTO((ptr p)); -static iptr sweep_typed_object PROTO((ptr tc_in, ptr p, IGEN from_g)); -static void sweep_symbol PROTO((ptr tc_in, ptr p, IGEN from_g)); -static void sweep_port PROTO((ptr tc_in, ptr p, IGEN from_g)); -static void sweep_thread PROTO((ptr tc_in, ptr p)); -static void sweep_continuation PROTO((ptr tc_in, ptr p, IGEN from_g)); -static void sweep_record PROTO((ptr tc_in, ptr x, IGEN from_g)); -static IGEN sweep_dirty_record PROTO((ptr tc_in, ptr x, IGEN youngest)); -static IGEN sweep_dirty_port PROTO((ptr tc_in, ptr x, IGEN youngest)); -static IGEN sweep_dirty_symbol PROTO((ptr tc_in, ptr x, IGEN youngest)); -static void sweep_code_object PROTO((ptr tc_in, ptr co, IGEN from_g)); +static iptr sweep_typed_object PROTO((thread_gc *tgc, ptr p, IGEN from_g)); +static void sweep_symbol PROTO((thread_gc *tgc, ptr p, IGEN from_g)); +static void sweep_port PROTO((thread_gc *tgc, ptr p, IGEN from_g)); +static void sweep_thread PROTO((thread_gc *tgc, ptr p)); +static void sweep_continuation PROTO((thread_gc *tgc, ptr p, IGEN from_g)); +static void sweep_record PROTO((thread_gc *tgc, ptr x, IGEN from_g)); +static IGEN sweep_dirty_record PROTO((thread_gc *tgc, ptr x, IGEN youngest)); +static IGEN sweep_dirty_port PROTO((thread_gc *tgc, ptr x, IGEN youngest)); +static IGEN sweep_dirty_symbol PROTO((thread_gc *tgc, ptr x, IGEN youngest)); +static void sweep_code_object PROTO((thread_gc *tgc, ptr co, IGEN from_g)); static void record_dirty_segment PROTO((IGEN from_g, IGEN to_g, seginfo *si)); static void setup_sweep_dirty PROTO(()); -static uptr sweep_dirty_segments PROTO((ptr tc_in, seginfo **dirty_segments)); -static void resweep_dirty_weak_pairs PROTO((ptr tc)); -static void mark_typemod_data_object PROTO((ptr tc_in, ptr p, uptr len, seginfo *si)); +static uptr sweep_dirty_segments PROTO((thread_gc *tgc, seginfo **dirty_segments)); +static void resweep_dirty_weak_pairs PROTO((thread_gc *tgc)); +static void mark_typemod_data_object PROTO((thread_gc *tgc, ptr p, uptr len, seginfo *si)); static void add_pending_guardian PROTO((ptr gdn, ptr tconc)); static void add_trigger_guardians_to_recheck PROTO((ptr ls)); -static void add_ephemeron_to_pending PROTO((ptr tc, ptr p)); -static void add_trigger_ephemerons_to_pending PROTO((ptr tc, ptr p)); -static void check_triggers PROTO((ptr tc, seginfo *si)); -static void check_ephemeron PROTO((ptr tc_in, ptr pe)); -static void check_pending_ephemerons PROTO((ptr tc_in)); -static int check_dirty_ephemeron PROTO((ptr tc_in, ptr pe, int youngest)); -static void finish_pending_ephemerons PROTO((ptr tc, seginfo *si)); -static void init_fully_marked_mask(ptr tc_in, IGEN g); -static void copy_and_clear_list_bits(ptr tc_in, seginfo *oldspacesegments); +static void add_ephemeron_to_pending PROTO((thread_gc *tgc, ptr p)); +static void add_trigger_ephemerons_to_pending PROTO((thread_gc *tgc, ptr p)); +static void check_triggers PROTO((thread_gc *tgc, seginfo *si)); +static void check_ephemeron PROTO((thread_gc *tgc, ptr pe)); +static void check_pending_ephemerons PROTO((thread_gc *tgc)); +static int check_dirty_ephemeron PROTO((thread_gc *tgc, ptr pe, int youngest)); +static void finish_pending_ephemerons PROTO((thread_gc *tgc, seginfo *si)); +static void init_fully_marked_mask(thread_gc *tgc, IGEN g); +static void copy_and_clear_list_bits(thread_gc *tgc, seginfo *oldspacesegments); #ifdef ENABLE_OBJECT_COUNTS static uptr total_size_so_far(); static uptr list_length PROTO((ptr ls)); #endif -static uptr target_generation_space_so_far(ptr tc); +static uptr target_generation_space_so_far(thread_gc *tgc); #ifdef ENABLE_MEASURE -static void init_measure(ptr tc, IGEN min_gen, IGEN max_gen); +static void init_measure(thread_gc *tgc, IGEN min_gen, IGEN max_gen); static void finish_measure(); -static void measure(ptr tc_in, ptr p); -static IBOOL flush_measure_stack(ptr tc_in); -static void init_measure_mask(ptr tc_in, seginfo *si); -static void init_counting_mask(ptr tc_in, seginfo *si); -static void push_measure(ptr tc_in, ptr p); +static void measure(thread_gc *tgc, ptr p); +static IBOOL flush_measure_stack(thread_gc *tgc); +static void init_measure_mask(thread_gc *tgc, seginfo *si); +static void init_counting_mask(thread_gc *tgc, seginfo *si); +static void push_measure(thread_gc *tgc, ptr p); static void measure_add_stack_size(ptr stack, uptr size); -static void add_ephemeron_to_pending_measure(ptr tc, ptr pe); +static void add_ephemeron_to_pending_measure(thread_gc *tgc, ptr pe); static void add_trigger_ephemerons_to_pending_measure(ptr pe); -static void check_ephemeron_measure(ptr tc_in, ptr pe); -static void check_pending_measure_ephemerons(ptr tc_in); +static void check_ephemeron_measure(thread_gc *tgc, ptr pe); +static void check_pending_measure_ephemerons(thread_gc *tgc); #endif #ifdef ENABLE_PARALLEL @@ -285,12 +285,6 @@ static int percentage(iptr n, iptr d) { return (n * 100) / d; } # endif #endif -#define main_sweeper_index maximum_parallel_collect_threads - -/* Use the `REMOTESWEEPER` field to assign the sweeper before sweepers - actually start: */ -#define WILL_BE_SWEEPER(tc) REMOTESWEEPER(tc) - /* #define DEBUG */ /* initialized and used each gc cycle. any others should be defined in globals.h */ @@ -328,9 +322,9 @@ FORCEINLINE IGEN compute_target_generation(IGEN g) { static octet *fully_marked_mask[static_generation+1]; #define push_sweep(p) { \ - if (SWEEPSTACK(tc_in) == SWEEPSTACKLIMIT(tc_in)) enlarge_sweep_stack(tc_in); \ - *(ptr *)TO_VOIDP(SWEEPSTACK(tc_in)) = p; \ - SWEEPSTACK(tc_in) = (ptr)((uptr)SWEEPSTACK(tc_in) + ptr_bytes); \ + if (tgc->sweep_stack == tgc->sweep_stack_limit) enlarge_sweep_stack(tgc); \ + *(ptr *)TO_VOIDP(tgc->sweep_stack) = p; \ + tgc->sweep_stack = (ptr)((uptr)tgc->sweep_stack + ptr_bytes); \ } #ifdef ENABLE_MEASURE @@ -352,8 +346,8 @@ static ptr sweep_from; # define ADD_BACKREFERENCE_FROM(p, from_p, tg) do { \ IGEN TG = tg; \ if ((S_G.enable_object_backreferences) && (TG < static_generation)) \ - S_G.gcbackreference[TG] = S_cons_in(tc_in, space_impure, TG, \ - S_cons_in(tc_in, space_impure, TG, p, from_p), \ + S_G.gcbackreference[TG] = S_cons_in(tgc->tc, space_impure, TG, \ + S_cons_in(tgc->tc, space_impure, TG, p, from_p), \ S_G.gcbackreference[TG]); \ } while (0) # define ADD_BACKREFERENCE(p, tg) ADD_BACKREFERENCE_FROM(p, sweep_from, tg) @@ -367,13 +361,6 @@ static ptr sweep_from; # define ADD_BACKREFERENCE(p, from_g) #endif -typedef struct remote_range { - ISPC s; - IGEN g; - ptr start, end; - struct remote_range *next; -} remote_range; - #if !defined(PTHREADS) # undef ENABLE_PARALLEL #endif @@ -383,31 +370,32 @@ typedef struct remote_range { # define GC_TC_MUTEX_ACQUIRE() gc_tc_mutex_acquire() # define GC_TC_MUTEX_RELEASE() gc_tc_mutex_release() -# define SEGMENT_IS_LOCAL(si, p) ((SWEEPER(si->creator_tc) == SWEEPER(tc_in)) || marked(si, p)) -# define RECORD_REMOTE_RANGE_TO(tc, start, size, sweeper) do { \ +# define SEGMENT_IS_LOCAL(si, p) (((si)->creator == tgc) || marked(si, p) || !S_use_gc_tc_mutex) +# define RECORD_REMOTE_RANGE_TO(tgc, start, size, creator) do { \ ptr START = TO_PTR(UNTYPE_ANY(start)); \ ptr END = (ptr)((uptr)START + (size)); \ - if ((uptr)START < (uptr)REMOTERANGESTART(tc)) \ - REMOTERANGESTART(tc) = START; \ - if ((uptr)END > (uptr)REMOTERANGEEND(tc)) \ - REMOTERANGEEND(tc) = END; \ - REMOTESWEEPER(tc) = sweeper; \ + if ((uptr)START < (uptr)tgc->remote_range_start) \ + tgc->remote_range_start = START; \ + if ((uptr)END > (uptr)tgc->remote_range_end) \ + tgc->remote_range_end = END; \ + tgc->remote_range_tgc = creator; \ + if (creator->sweeper == tgc->sweeper) abort(); \ } while (0) -# define RECORD_REMOTE_RANGE(tc, start, size, si) RECORD_REMOTE_RANGE_TO(tc, start, size, SWEEPER(si->creator_tc)) -# define FLUSH_REMOTE_RANGE(tc, s, g) do { \ - if (REMOTERANGESTART(tc) != (ptr)(uptr)-1) { \ - flush_remote_range(tc, s, g); \ - } \ +# define RECORD_REMOTE_RANGE(tgc, start, size, si) RECORD_REMOTE_RANGE_TO(tgc, start, size, si->creator) +# define FLUSH_REMOTE_RANGE(tgc, s, g) do { \ + if (tgc->remote_range_start != (ptr)(uptr)-1) { \ + flush_remote_range(tgc, s, g); \ + } \ } while (0) -static void gather_active_sweepers(); -static void reassign_segment_creator(ptr tc, seginfo *si); +static void gather_active_sweepers(thread_gc *tgc); +static void reassign_segment_creator(thread_gc *tgc, seginfo *si); static IBOOL sweeper_started(int i); -static void parallel_sweep_dirty_and_generation(ptr tc); -static iptr sweep_generation_trading_work(ptr tc); +static void parallel_sweep_dirty_and_generation(thread_gc *tgc); +static iptr sweep_generation_trading_work(thread_gc *tgc); -static void flush_remote_range(ptr tc, ISPC s, IGEN g); -static remote_range *send_and_receive_remote_ranges(ptr tc); +static void flush_remote_range(thread_gc *tgc, ISPC s, IGEN g); +static remote_range *send_and_receive_remote_ranges(thread_gc *tgc); #define SWEEPER_NONE 0 #define SWEEPER_READY 1 @@ -417,13 +405,15 @@ static remote_range *send_and_receive_remote_ranges(ptr tc); typedef struct { int status; s_thread_cond_t done_cond, work_cond; - ptr sweep_tc; + thread_gc *tgc; ptr thread; /* not 0 => thread to sweep on start */ + seginfo *dirty_segments[DIRTY_SEGMENT_LISTS]; /* modified only by owning sweeper: */ remote_range *ranges_to_send[maximum_parallel_collect_threads+1]; /* modified with sweeper mutex held: */ remote_range *ranges_received; + #ifdef ENABLE_TIMING int remote_ranges_sent, remote_ranges_received; iptr remote_ranges_bytes_sent, remote_ranges_bytes_received; @@ -433,23 +423,21 @@ typedef struct { static gc_thread_data sweepers[maximum_parallel_collect_threads+1]; static int num_sweepers; -static seginfo *main_dirty_segments[DIRTY_SEGMENT_LISTS]; - #else # define GC_TC_MUTEX_ACQUIRE() do { } while (0) # define GC_TC_MUTEX_RELEASE() do { } while (0) # define SEGMENT_IS_LOCAL(si, p) 1 -# define RECORD_REMOTE_RANGE_TO(tc, start, size, sweeper) do { } while (0) -# define RECORD_REMOTE_RANGE(tc, start, size, si) do { } while (0) -# define FLUSH_REMOTE_RANGE(tc, s, g) do { } while (0) +# define RECORD_REMOTE_RANGE_TO(tgc, start, size, creator) do { } while (0) +# define RECORD_REMOTE_RANGE(tgc, start, size, si) do { } while (0) +# define FLUSH_REMOTE_RANGE(tgc, s, g) do { } while (0) -# define gather_active_sweepers() do { } while (0) -# define reassign_segment_creator(tc, si) do { } while (0) -# define parallel_sweep_dirty_and_generation(tc) do { sweep_dirty(tc); sweep_generation(tc); } while (0) -# define send_and_receive_remote_ranges(tc) NULL -static void sweep_dirty PROTO((ptr tc)); +# define gather_active_sweepers(tgc) do { } while (0) +# define reassign_segment_creator(tgc, si) do { } while (0) +# define parallel_sweep_dirty_and_generation(tgc) do { sweep_dirty(tgc); sweep_generation(tgc); } while (0) +# define send_and_receive_remote_ranges(tgc) NULL +static void sweep_dirty PROTO((thread_gc *tgc)); #endif @@ -484,13 +472,13 @@ uptr list_length(ptr ls) { } #endif -#define init_mask(tc, dest, tg, init) do { \ +#define init_mask(tgc, dest, tg, init) do { \ octet *MASK; \ - find_room_voidp(tc, space_data, tg, ptr_align(segment_bitmap_bytes), MASK); \ + find_gc_room_voidp(tgc, space_data, tg, ptr_align(segment_bitmap_bytes), MASK); \ memset(MASK, init, segment_bitmap_bytes); \ STORE_FENCE(); \ dest = MASK; \ - BITMASKOVERHEAD(tc, tg) += ptr_align(segment_bitmap_bytes); \ + tgc->bitmask_overhead[tg] += ptr_align(segment_bitmap_bytes); \ } while (0) #define marked(si, p) (si->marked_mask && (si->marked_mask[segment_bitmap_byte(p)] & segment_bitmap_bit(p))) @@ -503,19 +491,19 @@ uptr list_length(ptr ls) { # define CAN_MARK_AND(x) x #endif -static void init_fully_marked_mask(ptr tc_in, IGEN g) { +static void init_fully_marked_mask(thread_gc *tgc, IGEN g) { GC_TC_MUTEX_ACQUIRE(); if (!fully_marked_mask[g]) { - init_mask(tc_in, fully_marked_mask[g], g, 0xFF); + init_mask(tgc, fully_marked_mask[g], g, 0xFF); } GC_TC_MUTEX_RELEASE(); } #ifdef PRESERVE_FLONUM_EQ -static void flonum_set_forwarded(ptr tc_in, ptr p, seginfo *si) { +static void flonum_set_forwarded(thread_gc *tgc, ptr p, seginfo *si) { if (!si->forwarded_flonums) - init_mask(tc_in, si->forwarded_flonums, 0, 0); + init_mask(tgc, si->forwarded_flonums, 0, 0); si->forwarded_flonums[segment_bitmap_byte(p)] |= segment_bitmap_bit(p); } @@ -538,7 +526,7 @@ static int flonum_is_forwarded_p(ptr p, seginfo *si) { #ifdef ENABLE_OBJECT_COUNTS # define ELSE_MEASURE_NONOLDSPACE(p) \ else if (measure_all_enabled) \ - push_measure(tc_in, p); + push_measure(tgc, p); #else # define ELSE_MEASURE_NONOLDSPACE(p) /* empty */ #endif @@ -567,7 +555,7 @@ static int flonum_is_forwarded_p(ptr p, seginfo *si) { else if (!new_marked(si, pp)) \ mark_or_copy_pure(ppp, pp, si); \ } else \ - RECORD_REMOTE_RANGE(tc_in, start, size, si); \ + RECORD_REMOTE_RANGE(tgc, start, size, si); \ } while (0) #define relocate_code(pp, si, start, size) do { \ @@ -578,15 +566,15 @@ static int flonum_is_forwarded_p(ptr p, seginfo *si) { else if (!new_marked(si, pp)) \ mark_or_copy_pure(&pp, pp, si); \ } else \ - RECORD_REMOTE_RANGE(tc_in, start, size, si); \ + RECORD_REMOTE_RANGE(tgc, start, size, si); \ } ELSE_MEASURE_NONOLDSPACE(pp) \ } while (0) #define mark_or_copy_pure(dest, p, si) do { \ if (CAN_MARK_AND(si->use_marks)) \ - (void)mark_object(tc_in, p, si); \ + (void)mark_object(tgc, p, si); \ else \ - (void)copy(tc_in, p, si, dest); \ + (void)copy(tgc, p, si, dest); \ } while (0) @@ -624,16 +612,16 @@ static int flonum_is_forwarded_p(ptr p, seginfo *si) { } else { \ __to_g = TARGET_GENERATION(si); \ } \ - if (__to_g < from_g) S_record_new_dirty_card(tc_in, ppp, __to_g); \ + if (__to_g < from_g) S_record_new_dirty_card(tgc, ppp, __to_g); \ } else \ - RECORD_REMOTE_RANGE(tc_in, start, size, si); \ + RECORD_REMOTE_RANGE(tgc, start, size, si); \ } while (0) #define mark_or_copy_impure(to_g, dest, p, from_g, si) do { \ if (CAN_MARK_AND(si->use_marks)) \ - to_g = mark_object(tc_in, p, si); \ + to_g = mark_object(tgc, p, si); \ else \ - to_g = copy(tc_in, p, si, dest); \ + to_g = copy(tgc, p, si, dest); \ } while (0) #endif /* !NO_DIRTY_NEWSPACE_POINTERS */ @@ -651,12 +639,12 @@ static int flonum_is_forwarded_p(ptr p, seginfo *si) { } else if (new_marked(_si, _pp)) { \ _pg = TARGET_GENERATION(_si); \ } else if (CAN_MARK_AND(_si->use_marks)) { \ - _pg = mark_object(tc_in, _pp, _si); \ + _pg = mark_object(tgc, _pp, _si); \ } else { \ - _pg = copy(tc_in, _pp, _si, _ppp); \ + _pg = copy(tgc, _pp, _si, _ppp); \ } \ } else { \ - RECORD_REMOTE_RANGE(tc_in, start, size, _si); \ + RECORD_REMOTE_RANGE(tgc, start, size, _si); \ _pg = 0xff; \ } \ } \ @@ -669,18 +657,18 @@ static int flonum_is_forwarded_p(ptr p, seginfo *si) { #endif #ifdef ENABLE_PARALLEL -static void do_relocate_indirect(ptr tc_in, ptr p, ptr* start, uptr len) { +static void do_relocate_indirect(thread_gc *tgc, ptr p, ptr start, uptr len) { relocate_pure(&p, start, len); } -# define relocate_indirect(p, start, len) do_relocate_indirect(tc_in, p, start, len) +# define relocate_indirect(p, start, len) do_relocate_indirect(tgc, p, start, len) #else -static void do_relocate_indirect(ptr tc_in, ptr p) { +static void do_relocate_indirect(thread_gc *tgc, ptr p) { relocate_pure(&p, NULL, 0); } -# define relocate_indirect(p, start, len) do_relocate_indirect(tc_in, p) +# define relocate_indirect(p, start, len) do_relocate_indirect(tgc, p) #endif -FORCEINLINE void check_triggers(ptr tc_in, seginfo *si) { +FORCEINLINE void check_triggers(thread_gc *tgc, seginfo *si) { /* Registering ephemerons and guardians to recheck at the granularity of a segment means that the worst-case complexity of GC is quadratic in the number of objects that fit into a segment @@ -689,7 +677,7 @@ FORCEINLINE void check_triggers(ptr tc_in, seginfo *si) { ephemerons). */ if (si->has_triggers) { if (si->trigger_ephemerons) { - add_trigger_ephemerons_to_pending(tc_in, si->trigger_ephemerons); + add_trigger_ephemerons_to_pending(tgc, si->trigger_ephemerons); si->trigger_ephemerons = 0; } if (si->trigger_guardians) { @@ -716,7 +704,7 @@ FORCEINLINE void check_triggers(ptr tc_in, seginfo *si) { set to a forwarding marker and pointer. To handle that problem, sweep_in_old() is allowed to copy the object, since the object is going to get copied anyway. */ -static void sweep_in_old(ptr tc_in, ptr p) { +static void sweep_in_old(thread_gc *tgc, ptr p) { /* Detect all the cases when we need to give up on in-place sweeping: */ if (object_directly_refers_to_self(p)) { @@ -728,16 +716,16 @@ static void sweep_in_old(ptr tc_in, ptr p) { so it's ok to sweep(), but only update `p` for pure relocations; impure oness must that will happen later, after `p` is potentially copied, so the card updates will be right. */ - sweep_object_in_old(tc_in, p); + sweep_object_in_old(tgc, p); } -static void sweep_dirty_object_if_space_new(ptr tc_in, ptr p) { +static void sweep_dirty_object_if_space_new(thread_gc *tgc, ptr p) { seginfo *si = SegInfo(ptr_get_segment(p)); if (si->space == space_new) - (void)sweep_dirty_object(tc_in, p, 0); + (void)sweep_dirty_object(tgc, p, 0); } -static ptr copy_stack(ptr tc_in, ptr old, iptr *length, iptr clength) { +static ptr copy_stack(thread_gc *tgc, ptr old, iptr *length, iptr clength) { iptr n, m; ptr new; IGEN newg; seginfo *si = SegInfo(ptr_get_segment(old)); @@ -753,7 +741,7 @@ static ptr copy_stack(ptr tc_in, ptr old, iptr *length, iptr clength) { #ifndef NO_NEWSPACE_MARKS if (si->use_marks) { if (!marked(si, old)) { - mark_typemod_data_object(tc_in, old, n, si); + mark_typemod_data_object(tgc, old, n, si); #ifdef ENABLE_OBJECT_COUNTS S_G.countof[newg][countof_stack] += 1; @@ -779,7 +767,7 @@ static ptr copy_stack(ptr tc_in, ptr old, iptr *length, iptr clength) { if (n == 0) { return (ptr)0; } else { - find_room(tc_in, space_data, newg, typemod, n, new); + find_gc_room(tgc, space_data, newg, typemod, n, new); n = ptr_align(clength); /* warning: stack may have been left non-double-aligned by split_and_resize */ memcpy_aligned(TO_VOIDP(new), TO_VOIDP(old), n); @@ -829,8 +817,8 @@ typedef struct count_root_t { IBOOL weak; } count_root_t; -ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { - ptr tc = tc_in; +ptr GCENTRY(ptr tc, ptr count_roots_ls) { + thread_gc *tgc = THREAD_GC(tc); IGEN g; ISPC s; seginfo *oldspacesegments, *oldweakspacesegments, *si, *nextsi; ptr ls; @@ -857,34 +845,23 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { #endif /* !NO_DIRTY_NEWSPACE_POINTERS */ S_G.must_mark_gen0 = 0; - /* map `tc`s of rendezvous threads to sweeping threads, setting WILL_BE_SWEEPER */ - gather_active_sweepers(); + /* map `tc`s of rendezvous threads to sweeping threads, setting `will_be_sweeper` */ + gather_active_sweepers(tgc); for (ls = S_threads; ls != Snil; ls = Scdr(ls)) { ptr t_tc = (ptr)THREADTC(Scar(ls)); + thread_gc *t_tgc = THREAD_GC(t_tc); S_scan_dirty(TO_VOIDP(EAP(t_tc)), TO_VOIDP(REAL_EAP(t_tc))); EAP(t_tc) = REAL_EAP(t_tc) = AP(t_tc) = (ptr)0; /* Initially, map every context to the same sweeper, so we can relocate some initial objects */ - SWEEPER(t_tc) = main_sweeper_index; - - /* If WILL_BE_SWEEPER() is not already set right, set it to - `main_sweeper_index`. */ -#ifdef ENABLE_PARALLEL - if (WILL_BE_SWEEPER(t_tc) != main_sweeper_index) { - if ((WILL_BE_SWEEPER(t_tc) >= num_sweepers) - || (sweepers[WILL_BE_SWEEPER(t_tc)].sweep_tc != t_tc)) - WILL_BE_SWEEPER(t_tc) = main_sweeper_index; - } -#else - WILL_BE_SWEEPER(t_tc) = main_sweeper_index; -#endif + t_tgc->sweeper = main_sweeper_index; /* clear thread-local allocation: */ for (g = 0; g <= MAX_CG; g++) { for (s = 0; s <= max_real_space; s++) { - if (BASELOC_AT(t_tc, s, g)) { + if (t_tgc->base_loc[g][s]) { /* We close off, instead of just setting BASELOC to 0, in case the page ends up getting marked, in which case a terminator mark needed. */ @@ -893,15 +870,15 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { } } - if ((t_tc != tc) && (WILL_BE_SWEEPER(t_tc) == main_sweeper_index)) { + if ((t_tc != tc) && (t_tgc->will_be_sweeper == main_sweeper_index)) { /* close off any current allocation in MAX_TG, and ensure that end-of-segment markers are otherwise set (in case that's needed for dirty-byte sweeping) */ for (s = 0; s <= max_real_space; s++) { - if (BASELOC_AT(t_tc, s, MAX_TG)) + if (t_tgc->base_loc[MAX_TG][s]) S_close_off_thread_local_segment(t_tc, s, MAX_TG); for (g = MAX_TG + 1; g <= static_generation; g++) { - ptr old = NEXTLOC_AT(t_tc, s, g); + ptr old = t_tgc->next_loc[g][s]; if (old != (ptr)0) *(ptr*)TO_VOIDP(old) = forward_marker; } @@ -909,17 +886,17 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { } else { /* set up context for sweeping --- effectively remembering the current allocation state so anything new is recognized as needing sweeping */ - SWEEPSTACKSTART(t_tc) = SWEEPSTACK(t_tc) = SWEEPSTACKLIMIT(t_tc) = (ptr)0; - BITMASKOVERHEAD(t_tc, 0) = 0; + t_tgc->sweep_stack_start = t_tgc->sweep_stack = t_tgc->sweep_stack_limit = (ptr)0; + t_tgc->bitmask_overhead[0] = 0; for (g = MIN_TG; g <= MAX_TG; g++) - BITMASKOVERHEAD(t_tc, g) = 0; + t_tgc->bitmask_overhead[g] = 0; for (s = 0; s <= max_real_space; s++) { - /* need to save `NEXTLOC_AT` to ensure that dirty sweeping + /* need to save `next_loc` to ensure that dirty sweeping doesn't overshoot into newly allocated objects */ - ORIGNEXTLOC(t_tc, s) = NEXTLOC_AT(t_tc, s, MAX_TG); - SWEEPLOC_AT(t_tc, s, MAX_TG) = NEXTLOC_AT(t_tc, s, MAX_TG); + t_tgc->orig_next_loc[s] = t_tgc->next_loc[MAX_TG][s]; + t_tgc->sweep_loc[MAX_TG][s] = t_tgc->next_loc[MAX_TG][s]; for (g = MIN_TG; g <= MAX_TG; g++) - SWEEPNEXT_AT(t_tc, s, g) = (ptr)0; + t_tgc->sweep_next[g][s] = NULL; } /* for t_tc != tc, we add terminators to allocation pages just before sweep_dirty() */ @@ -988,7 +965,7 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { si->marked_mask = NULL; /* clear old mark bits, if any */ si->marked_count = 0; si->min_dirty_byte = 0; /* prevent registering as dirty while GCing */ - reassign_segment_creator(tc, si); + reassign_segment_creator(tgc, si); } S_G.occupied_segments[g][s] = NULL; @@ -1040,7 +1017,7 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { seginfo *si = SegInfo(ptr_get_segment(p)); if (si->space == space_new) { if (!si->marked_mask) - init_mask(tc, si->marked_mask, tg, 0); + init_mask(tgc, si->marked_mask, tg, 0); si->marked_mask[segment_bitmap_byte(p)] |= segment_bitmap_bit(p); } } @@ -1052,7 +1029,7 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { iptr i; count_roots_len = list_length(count_roots_ls); - find_room_voidp(tc, space_data, 0, ptr_align(count_roots_len*sizeof(count_root_t)), count_roots); + find_gc_room_voidp(tgc, space_data, 0, ptr_align(count_roots_len*sizeof(count_root_t)), count_roots); for (ls = count_roots_ls, i = 0; ls != Snil; ls = Scdr(ls), i++) { ptr p = Scar(ls); @@ -1064,7 +1041,7 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { seginfo *si = SegInfo(ptr_get_segment(p)); if (!si->counting_mask) - init_counting_mask(tc, si); + init_counting_mask(tgc, si); si->counting_mask[segment_bitmap_byte(p)] |= segment_bitmap_bit(p); @@ -1086,7 +1063,7 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { iptr i; # ifdef ENABLE_MEASURE - init_measure(tc, MAX_TG+1, static_generation); + init_measure(tgc, MAX_TG+1, static_generation); # endif for (i = 0; i < count_roots_len; i++) { @@ -1103,12 +1080,12 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { || !count_roots[i].weak) { /* reached or older; sweep transitively */ relocate_pure(&p, NULL, 0); - sweep(tc, p, TARGET_GENERATION(si)); + sweep(tgc, p, TARGET_GENERATION(si)); ADD_BACKREFERENCE(p, si->generation); - sweep_generation(tc); + sweep_generation(tgc); # ifdef ENABLE_MEASURE - while (flush_measure_stack(tc)) { - sweep_generation(tc); + while (flush_measure_stack(tgc)) { + sweep_generation(tgc); } # endif @@ -1164,7 +1141,7 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { if (!marked(si, p)) S_error_abort("space_new locked object should have a mark bit set"); si->marked_mask[segment_bitmap_byte(p)] -= segment_bitmap_bit(p); - mark_object(tc, p, si); + mark_object(tgc, p, si); } /* non-`space_new` objects will be swept via new pair */ locked_objects = S_cons_in(tc, space_impure, tg, p, locked_objects); @@ -1195,20 +1172,20 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { #ifdef ENABLE_PARALLEL t_tc = (ptr)THREADTC(thread); - if (WILL_BE_SWEEPER(t_tc) != main_sweeper_index) { + if (THREAD_GC(t_tc)->will_be_sweeper != main_sweeper_index) { if (!OLDSPACE(thread)) { /* sweep in sweeper thread: */ - sweepers[WILL_BE_SWEEPER(t_tc)].thread = thread; + sweepers[THREAD_GC(t_tc)->will_be_sweeper].thread = thread; } else { /* relocate now, so main sweeping will happen in sweeper thread */ - ptr tc_in = t_tc; /* shadows enclosing `tc_in` binding */ + thread_gc *tgc = THREAD_GC(t_tc); /* shadows enclosing `tgc` binding */ relocate_pure(&thread, NULL, 0); } } else #endif if (!OLDSPACE(thread)) - sweep_thread(tc, thread); + sweep_thread(tgc, thread); } relocate_pure(&S_threads, NULL, 0); @@ -1257,20 +1234,20 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { because we can't find dirty writes there */ for (g = MAX_CG + 1; g <= static_generation; INCRGEN(g)) { for (ls = S_G.locked_objects[g]; ls != Snil; ls = Scdr(ls)) - sweep_dirty_object_if_space_new(tc, Scar(ls)); + sweep_dirty_object_if_space_new(tgc, Scar(ls)); for (ls = S_G.unlocked_objects[g]; ls != Snil; ls = Scdr(ls)) - sweep_dirty_object_if_space_new(tc, Scar(ls)); + sweep_dirty_object_if_space_new(tgc, Scar(ls)); } /* prepare to sweep areas marked dirty by assignments into older generations */ setup_sweep_dirty(); - parallel_sweep_dirty_and_generation(tc); + parallel_sweep_dirty_and_generation(tgc); /* since we will later resweep dirty weak pairs, make sure sweep_generation ends with a terminator in place for space_weakpair, at least in all threads other than this one that may have allocated there during sweep_generation */ - pre_finalization_size = target_generation_space_so_far(tc); + pre_finalization_size = target_generation_space_so_far(tgc); /* handle guardians */ { ptr pend_hold_ls, final_ls, pend_final_ls, maybe_final_ordered_ls; @@ -1358,7 +1335,7 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { /* mark things reachable from `rep`, but not `rep` itself, unless `rep` is immediately reachable from itself */ PUSH_BACKREFERENCE(ls) - sweep_in_old(tc, rep); + sweep_in_old(tgc, rep); POP_BACKREFERENCE() } INITGUARDIANNEXT(ls) = maybe_final_ordered_ls; @@ -1426,7 +1403,7 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { /* In backreference mode, we rely on sweep of the guardian entry not registering any backreferences. Otherwise, bogus pair pointers would get created. */ - find_room(tc, space_pure, g, typemod, size_guardian_entry, p); + find_gc_room(tgc, space_pure, g, typemod, size_guardian_entry, p); INITGUARDIANOBJ(p) = GUARDIANOBJ(ls); INITGUARDIANREP(p) = rep; INITGUARDIANTCONC(p) = tconc; @@ -1460,7 +1437,7 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { if (!relocate_rep) break; - sweep_generation(tc); + sweep_generation(tgc); ls = recheck_guardians_ls; recheck_guardians_ls = Snil; for ( ; ls != Snil; ls = next) { @@ -1502,7 +1479,7 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { } } - S_G.bytes_finalized = target_generation_space_so_far(tc) - pre_finalization_size; + S_G.bytes_finalized = target_generation_space_so_far(tgc) - pre_finalization_size; { iptr post_phantom_bytes = 0; for (g = MIN_TG; g <= MAX_TG; g++) { @@ -1512,11 +1489,11 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { } /* handle weak pairs */ - resweep_dirty_weak_pairs(tc); - resweep_weak_pairs(tc, oldweakspacesegments); + resweep_dirty_weak_pairs(tgc); + resweep_weak_pairs(tgc, oldweakspacesegments); /* still-pending ephemerons all go to bwp */ - finish_pending_ephemerons(tc, oldspacesegments); + finish_pending_ephemerons(tgc, oldspacesegments); ACCUM_REAL_TIME(collect_accum, step, start); REPORT_TIME(fprintf(stderr, "%d coll +%ld ms %ld ms [real time]\n", @@ -1532,7 +1509,7 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { si = SegInfo(ptr_get_segment(sym)); if (new_marked(si, sym) || (FWDMARKER(sym) == forward_marker && ((sym = FWDADDRESS(sym)) || 1))) { IGEN g = si->generation; - find_room_voidp(tc, space_data, g, ptr_align(sizeof(bucket)), b); + find_gc_room_voidp(tgc, space_data, g, ptr_align(sizeof(bucket)), b); #ifdef ENABLE_OBJECT_COUNTS S_G.countof[g][countof_oblist] += 1; S_G.bytesof[g][countof_oblist] += sizeof(bucket); @@ -1541,7 +1518,7 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { *pb = b; pb = &b->next; if (g != static_generation) { - find_room_voidp(tc, space_data, g, ptr_align(sizeof(bucket_list)), bl); + find_gc_room_voidp(tgc, space_data, g, ptr_align(sizeof(bucket_list)), bl); #ifdef ENABLE_OBJECT_COUNTS S_G.countof[g][countof_oblist] += 1; S_G.bytesof[g][countof_oblist] += sizeof(bucket_list); @@ -1607,7 +1584,7 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { } #endif /* WIN32 */ - copy_and_clear_list_bits(tc, oldspacesegments); + copy_and_clear_list_bits(tgc, oldspacesegments); /* move copied old space segments to empty space, and promote marked old space segments to the target generation */ @@ -1731,13 +1708,13 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { /* tell profile_release_counters to look for bwp'd counters at least through max_tg */ if (S_G.prcgeneration < MAX_TG) S_G.prcgeneration = MAX_TG; - if (SWEEPSTACKSTART(tc_in) != SWEEPSTACK(tc_in)) + if (tgc->sweep_stack_start != tgc->sweep_stack) S_error_abort("gc: sweep stack ended non-empty"); - S_G.bitmask_overhead[0] += BITMASKOVERHEAD(tc_in, 0); - BITMASKOVERHEAD(tc_in, 0) = 0; + S_G.bitmask_overhead[0] += tgc->bitmask_overhead[0]; + tgc->bitmask_overhead[0] = 0; for (g = MIN_TG; g <= MAX_TG; g++) - S_G.bitmask_overhead[g] += BITMASKOVERHEAD(tc_in, g); + S_G.bitmask_overhead[g] += tgc->bitmask_overhead[g]; ACCUM_REAL_TIME(all_accum, astep, astart); REPORT_TIME(fprintf(stderr, "%d all +%ld ms %ld ms [real time]\n", MAX_CG, astep, all_accum)); @@ -1754,34 +1731,28 @@ ptr GCENTRY(ptr tc_in, ptr count_roots_ls) { #ifdef ENABLE_PARALLEL -static void reassign_segment_creator(ptr tc, seginfo *si) { - int i; - - for (i = 0; i < num_sweepers; i++) - if (sweepers[i].sweep_tc == si->creator_tc) - return; - - si->creator_tc = tc; +static void reassign_segment_creator(thread_gc *tgc, seginfo *si) { + if (si->creator->will_be_sweeper == main_sweeper_index) + si->creator = tgc; } -static void flush_remote_range(ptr tc, ISPC s, IGEN g) { +static void flush_remote_range(thread_gc *tgc, ISPC s, IGEN g) { remote_range *r; - int me = SWEEPER(tc); - int they = REMOTESWEEPER(tc); + int they = tgc->remote_range_tgc->sweeper; - find_room_voidp(tc, space_data, 0, ptr_align(sizeof(remote_range)), r); - BITMASKOVERHEAD(tc, 0) += ptr_align(sizeof(remote_range)); + find_gc_room_voidp(tgc, space_data, 0, ptr_align(sizeof(remote_range)), r); + tgc->bitmask_overhead[0] += ptr_align(sizeof(remote_range)); r->s = s; r->g = g; - r->start = REMOTERANGESTART(tc); - r->end = REMOTERANGEEND(tc); - r->next = sweepers[me].ranges_to_send[they]; - sweepers[me].ranges_to_send[they] = r; + r->start = tgc->remote_range_start; + r->end = tgc->remote_range_end; + r->next = sweepers[tgc->sweeper].ranges_to_send[they]; + sweepers[tgc->sweeper].ranges_to_send[they] = r; - REMOTERANGESTART(tc) = (ptr)(uptr)-1; - REMOTERANGEEND(tc) = (ptr)0; + tgc->remote_range_start = (ptr)(uptr)-1; + tgc->remote_range_end = (ptr)0; - SWEEPCHANGE(tc) = SWEEP_CHANGE_PROGRESS; + tgc->sweep_change = SWEEP_CHANGE_PROGRESS; } #endif @@ -1792,20 +1763,20 @@ static void flush_remote_range(ptr tc, ISPC s, IGEN g) { } while (0) #define sweep_space_segments(s, from_g, body) do { \ - while ((si = (seginfo *)TO_VOIDP(SWEEPNEXT_AT(tc_in, s, from_g))) != NULL) { \ - SWEEPNEXT_AT(tc_in, s, from_g) = TO_PTR(si->sweep_next); \ + while ((si = (seginfo *)TO_VOIDP(tgc->sweep_next[from_g][s])) != NULL) { \ + tgc->sweep_next[from_g][s] = si->sweep_next; \ pp = TO_VOIDP(si->sweep_start); \ while ((p = *pp) != forward_marker) \ body \ COUNT_SWEPT_BYTES(si->sweep_start, pp); \ - FLUSH_REMOTE_RANGE(tc_in, s, from_g); \ + FLUSH_REMOTE_RANGE(tgc, s, from_g); \ save_resweep(s, si); \ } \ } while (0) #define sweep_space_bump_range(s, from_g, body) do { \ - slp = &SWEEPLOC_AT(tc_in, s, from_g); \ - nlp = &NEXTLOC_AT(tc_in, s, from_g); \ + slp = &tgc->sweep_loc[from_g][s]; \ + nlp = &tgc->next_loc[from_g][s]; \ while ((sl = TO_VOIDP(*slp)) != (nl = TO_VOIDP(*nlp))) { \ *slp = TO_PTR(nl); \ pp = sl; \ @@ -1814,7 +1785,7 @@ static void flush_remote_range(ptr tc, ISPC s, IGEN g) { body \ } \ COUNT_SWEPT_BYTES(sl, nl); \ - FLUSH_REMOTE_RANGE(tc_in, s, from_g); \ + FLUSH_REMOTE_RANGE(tgc, s, from_g); \ } \ } while (0) @@ -1827,7 +1798,7 @@ static void flush_remote_range(ptr tc, ISPC s, IGEN g) { } \ } while (0) -static void resweep_weak_pairs(ptr tc_in, seginfo *oldweakspacesegments) { +static void resweep_weak_pairs(thread_gc *tgc, seginfo *oldweakspacesegments) { IGEN from_g; ptr *pp, p, *nl; seginfo *si; @@ -1835,8 +1806,8 @@ static void resweep_weak_pairs(ptr tc_in, seginfo *oldweakspacesegments) { for (from_g = MIN_TG; from_g <= MAX_TG; from_g += 1) { /* By starting from `base_loc`, we may needlessly sweep pairs in `MAX_TG` that were allocated before the GC, but that's ok. */ - pp = TO_VOIDP(BASELOC_AT(tc_in, space_weakpair, from_g)); - nl = TO_VOIDP(NEXTLOC_AT(tc_in, space_weakpair, from_g)); + pp = TO_VOIDP(tgc->base_loc[from_g][space_weakpair]); + nl = TO_VOIDP(tgc->next_loc[from_g][space_weakpair]); while (pp != nl) { p = *pp; forward_or_bwp(pp, p); @@ -1849,9 +1820,9 @@ static void resweep_weak_pairs(ptr tc_in, seginfo *oldweakspacesegments) { { int i; for (i = 0; i < num_sweepers; i++) { - ptr tc = sweepers[i].sweep_tc; - pp = TO_VOIDP(BASELOC_AT(tc, space_weakpair, from_g)); - nl = TO_VOIDP(NEXTLOC_AT(tc, space_weakpair, from_g)); + thread_gc *t_tgc = sweepers[i].tgc; + pp = TO_VOIDP(t_tgc->base_loc[from_g][space_weakpair]); + nl = TO_VOIDP(t_tgc->next_loc[from_g][space_weakpair]); while (pp != nl) { p = *pp; forward_or_bwp(pp, p); @@ -1909,16 +1880,16 @@ static void forward_or_bwp(pp, p) ptr *pp; ptr p; { } } -static iptr sweep_generation_pass(ptr tc_in) { +static iptr sweep_generation_pass(thread_gc *tgc) { ptr *slp, *nlp; ptr *pp, *ppn, p, *nl, *sl; IGEN from_g; seginfo *si; iptr num_swept_bytes = 0; remote_range *received_ranges; do { - SWEEPCHANGE(tc_in) = SWEEP_NO_CHANGE; + tgc->sweep_change = SWEEP_NO_CHANGE; - num_swept_bytes += sweep_from_stack(tc_in); + num_swept_bytes += sweep_from_stack(tgc); for (from_g = MIN_TG; from_g <= MAX_TG; from_g += 1) { @@ -1935,13 +1906,13 @@ static iptr sweep_generation_pass(ptr tc_in) { sweep_space(space_symbol, from_g, { p = TYPE(TO_PTR(pp), type_symbol); - sweep_symbol(tc_in, p, from_g); + sweep_symbol(tgc, p, from_g); pp += size_symbol / sizeof(ptr); }); sweep_space(space_port, from_g, { p = TYPE(TO_PTR(pp), type_typed_object); - sweep_port(tc_in, p, from_g); + sweep_port(tgc, p, from_g); pp += size_port / sizeof(ptr); }); @@ -1956,7 +1927,7 @@ static iptr sweep_generation_pass(ptr tc_in) { sweep_space(space_ephemeron, from_g, { p = TYPE(TO_PTR(pp), type_pair); - add_ephemeron_to_pending(tc_in, p); + add_ephemeron_to_pending(tgc, p); pp += size_ephemeron / sizeof(ptr); }); @@ -1972,24 +1943,24 @@ static iptr sweep_generation_pass(ptr tc_in) { sweep_space(space_continuation, from_g, { p = TYPE(TO_PTR(pp), type_closure); - sweep_continuation(tc_in, p, from_g); + sweep_continuation(tgc, p, from_g); pp += size_continuation / sizeof(ptr); }); sweep_space(space_pure_typed_object, from_g, { p = TYPE(TO_PTR(pp), type_typed_object); - pp = TO_VOIDP(((uptr)TO_PTR(pp) + sweep_typed_object(tc_in, p, from_g))); + pp = TO_VOIDP(((uptr)TO_PTR(pp) + sweep_typed_object(tgc, p, from_g))); }); sweep_space(space_code, from_g, { p = TYPE(TO_PTR(pp), type_typed_object); - sweep_code_object(tc_in, p, from_g); + sweep_code_object(tgc, p, from_g); pp += size_code(CODELEN(p)) / sizeof(ptr); }); sweep_space(space_impure_record, from_g, { p = TYPE(TO_PTR(pp), type_typed_object); - sweep_record(tc_in, p, from_g); + sweep_record(tgc, p, from_g); pp = TO_VOIDP((iptr)TO_PTR(pp) + size_record_inst(UNFIX(RECORDDESCSIZE(RECORDINSTTYPE(p))))); }); @@ -1997,18 +1968,18 @@ static iptr sweep_generation_pass(ptr tc_in) { /* space used only as needed for backreferences: */ sweep_space(space_impure_typed_object, from_g, { p = TYPE(TO_PTR(pp), type_typed_object); - pp = TO_VOIDP((uptr)TO_PTR(pp) + sweep_typed_object(tc_in, p, from_g)); + pp = TO_VOIDP((uptr)TO_PTR(pp) + sweep_typed_object(tgc, p, from_g)); }); /* space used only as needed for backreferences: */ sweep_space(space_closure, from_g, { p = TYPE(TO_PTR(pp), type_closure); - sweep(tc_in, p, from_g); + sweep(tgc, p, from_g); pp = TO_VOIDP((uptr)TO_PTR(pp) + size_object(p)); }); } - received_ranges = send_and_receive_remote_ranges(tc_in); + received_ranges = send_and_receive_remote_ranges(tgc); /* The ranges in `received_ranges` include old-generation objects from other parallel sweepers, which means they correspond to dirty @@ -2038,19 +2009,19 @@ static iptr sweep_generation_pass(ptr tc_in) { } else if (s == space_closure) { while (pp < nl) { p = TYPE(TO_PTR(pp), type_closure); - sweep(tc_in, p, from_g); + sweep(tgc, p, from_g); pp = TO_VOIDP((uptr)TO_PTR(pp) + size_object(p)); } } else if (s == space_continuation) { while (pp < nl) { p = TYPE(TO_PTR(pp), type_closure); - sweep_continuation(tc_in, p, from_g); + sweep_continuation(tgc, p, from_g); pp += size_continuation / sizeof(ptr); } } else if (s == space_code) { while (pp < nl) { p = TYPE(TO_PTR(pp), type_typed_object); - sweep_code_object(tc_in, p, from_g); + sweep_code_object(tgc, p, from_g); pp += size_code(CODELEN(p)) / sizeof(ptr); } } else if ((s == space_pure_typed_object) @@ -2058,18 +2029,18 @@ static iptr sweep_generation_pass(ptr tc_in) { /* old generation can happen in the special case of a thread object: */ while (pp < nl) { p = TYPE(TO_PTR(pp), type_typed_object); - pp = TO_VOIDP(((uptr)TO_PTR(pp) + sweep_typed_object(tc_in, p, from_g))); + pp = TO_VOIDP(((uptr)TO_PTR(pp) + sweep_typed_object(tgc, p, from_g))); } } else if (s == space_symbol) { while (pp < nl) { p = TYPE(TO_PTR(pp), type_symbol); - sweep_symbol(tc_in, p, from_g); + sweep_symbol(tgc, p, from_g); pp += size_symbol / sizeof(ptr); } } else if (s == space_port) { while (pp < nl) { p = TYPE(TO_PTR(pp), type_typed_object); - sweep_port(tc_in, p, from_g); + sweep_port(tgc, p, from_g); pp += size_port / sizeof(ptr); } } else if (s == space_weakpair) { @@ -2082,20 +2053,20 @@ static iptr sweep_generation_pass(ptr tc_in) { } else if (s == space_ephemeron) { while (pp < nl) { p = TYPE(TO_PTR(pp), type_pair); - add_ephemeron_to_pending(tc_in, p); + add_ephemeron_to_pending(tgc, p); pp += size_ephemeron / sizeof(ptr); } } else if (s == space_impure_record) { while (pp < nl) { p = TYPE(TO_PTR(pp), type_typed_object); - sweep_record(tc_in, p, from_g); + sweep_record(tgc, p, from_g); pp = TO_VOIDP((iptr)TO_PTR(pp) + size_record_inst(UNFIX(RECORDDESCSIZE(RECORDINSTTYPE(p))))); } } else { S_error_abort("dirty range sweep: unexpected space"); } - FLUSH_REMOTE_RANGE(tc_in, s, from_g); + FLUSH_REMOTE_RANGE(tgc, s, from_g); COUNT_SWEPT_BYTES(received_ranges->start, received_ranges->end); received_ranges = received_ranges->next; } @@ -2104,29 +2075,29 @@ static iptr sweep_generation_pass(ptr tc_in) { chance that an ephemeron must be reigistered as a segment-specific trigger or gets triggered for recheck, but it doesn't change the worst-case complexity. */ - if (SWEEPCHANGE(tc_in) == SWEEP_NO_CHANGE) - check_pending_ephemerons(tc_in); + if (tgc->sweep_change == SWEEP_NO_CHANGE) + check_pending_ephemerons(tgc); - } while (SWEEPCHANGE(tc_in) == SWEEP_CHANGE_PROGRESS); + } while (tgc->sweep_change == SWEEP_CHANGE_PROGRESS); return num_swept_bytes; } -static void sweep_generation(ptr tc_in) { - sweep_generation_pass(tc_in); +static void sweep_generation(thread_gc *tgc) { + sweep_generation_pass(tgc); } -void enlarge_sweep_stack(ptr tc_in) { - uptr sz = ((uptr)SWEEPSTACKLIMIT(tc_in) - (uptr)SWEEPSTACKSTART(tc_in)); +void enlarge_sweep_stack(thread_gc *tgc) { + uptr sz = ((uptr)tgc->sweep_stack_limit - (uptr)tgc->sweep_stack_start); uptr new_sz = 2 * ((sz == 0) ? 256 : sz); ptr new_sweep_stack; - find_room(tc_in, space_data, 0, typemod, ptr_align(new_sz), new_sweep_stack); + find_gc_room(tgc, space_data, 0, typemod, ptr_align(new_sz), new_sweep_stack); if (sz != 0) - memcpy(TO_VOIDP(new_sweep_stack), TO_VOIDP(SWEEPSTACKSTART(tc_in)), sz); - BITMASKOVERHEAD(tc_in, 0) += ptr_align(new_sz); - SWEEPSTACKSTART(tc_in) = new_sweep_stack; - SWEEPSTACKLIMIT(tc_in) = (ptr)((uptr)new_sweep_stack + new_sz); - SWEEPSTACK(tc_in) = (ptr)((uptr)new_sweep_stack + sz); + memcpy(TO_VOIDP(new_sweep_stack), TO_VOIDP(tgc->sweep_stack_start), sz); + tgc->bitmask_overhead[0] += ptr_align(new_sz); + tgc->sweep_stack_start = new_sweep_stack; + tgc->sweep_stack_limit = (ptr)((uptr)new_sweep_stack + new_sz); + tgc->sweep_stack = (ptr)((uptr)new_sweep_stack + sz); } #ifdef ENABLE_PARALLEL @@ -2154,41 +2125,41 @@ static ISPC infer_space(ptr p, seginfo *si) { } #endif -iptr sweep_from_stack(ptr tc_in) { +iptr sweep_from_stack(thread_gc *tgc) { iptr num_swept_bytes = 0; - if (SWEEPSTACK(tc_in) > SWEEPSTACKSTART(tc_in)) { - while (SWEEPSTACK(tc_in) > SWEEPSTACKSTART(tc_in)) { + if (tgc->sweep_stack > tgc->sweep_stack_start) { + while (tgc->sweep_stack > tgc->sweep_stack_start) { ptr p; seginfo *si; - SWEEPSTACK(tc_in) = (ptr)((uptr)SWEEPSTACK(tc_in) - ptr_bytes); - p = *(ptr *)TO_VOIDP(SWEEPSTACK(tc_in)); + tgc->sweep_stack = (ptr)((uptr)tgc->sweep_stack - ptr_bytes); + p = *(ptr *)TO_VOIDP(tgc->sweep_stack); /* Room for improvement: `si->generation` is needed only for objects that have impure fields, or in parallel mode for remote ranges. */ si = SegInfo(ptr_get_segment(p)); - sweep(tc_in, p, si->generation); + sweep(tgc, p, si->generation); COUNT_SWEPT_BYTES(0, size_object(p)); - FLUSH_REMOTE_RANGE(tc_in, infer_space(p, si), si->generation); + FLUSH_REMOTE_RANGE(tgc, infer_space(p, si), si->generation); } } return num_swept_bytes; } -static iptr sweep_typed_object(ptr tc, ptr p, IGEN from_g) { +static iptr sweep_typed_object(thread_gc *tgc, ptr p, IGEN from_g) { ptr tf = TYPEFIELD(p); if (TYPEP(tf, mask_record, type_record)) { - sweep_record(tc, p, from_g); + sweep_record(tgc, p, from_g); return size_record_inst(UNFIX(RECORDDESCSIZE(RECORDINSTTYPE(p)))); } else if (TYPEP(tf, mask_thread, type_thread)) { - sweep_thread(tc, p); + sweep_thread(tgc, p); return size_thread; } else { /* We get here only if backreference mode pushed other typed objects into a typed space or if an object is a counting root */ - sweep(tc, p, from_g); + sweep(tgc, p, from_g); return size_object(p); } } @@ -2258,24 +2229,17 @@ static void setup_sweep_dirty() { continue; } - for (i = 0; i < num_sweepers; i++) - if (sweepers[i].sweep_tc == dirty_si->creator_tc) { - dirty_si->dirty_next = DirtySegmentsAt(sweepers[i].dirty_segments, from_g, to_g); - DirtySegmentsAt(sweepers[i].dirty_segments, from_g, to_g) = dirty_si; - break; - } + i = dirty_si->creator->will_be_sweeper; - if (i == num_sweepers) { - dirty_si->dirty_next = DirtySegmentsAt(main_dirty_segments, from_g, to_g); - DirtySegmentsAt(main_dirty_segments, from_g, to_g) = dirty_si; - } + dirty_si->dirty_next = DirtySegmentsAt(sweepers[i].dirty_segments, from_g, to_g); + DirtySegmentsAt(sweepers[i].dirty_segments, from_g, to_g) = dirty_si; } } } #endif } -static uptr sweep_dirty_segments(ptr tc_in, seginfo **dirty_segments) { +static uptr sweep_dirty_segments(thread_gc *tgc, seginfo **dirty_segments) { IGEN youngest, min_youngest; ptr *pp, *ppn, *ppend, *nl, start; uptr seg, d; @@ -2309,8 +2273,8 @@ static uptr sweep_dirty_segments(ptr tc_in, seginfo **dirty_segments) { if (s == space_weakpair) { weakseginfo *next = local_weaksegments_to_resweep; - find_room_voidp(tc_in, space_data, 0, ptr_align(sizeof(weakseginfo)), local_weaksegments_to_resweep); - BITMASKOVERHEAD(tc_in, 0) += ptr_align(sizeof(weakseginfo)); + find_gc_room_voidp(tgc, space_data, 0, ptr_align(sizeof(weakseginfo)), local_weaksegments_to_resweep); + tgc->bitmask_overhead[0] += ptr_align(sizeof(weakseginfo)); local_weaksegments_to_resweep->si = dirty_si; local_weaksegments_to_resweep->next = next; if (next == NULL) @@ -2328,9 +2292,9 @@ static uptr sweep_dirty_segments(ptr tc_in, seginfo **dirty_segments) { allocations for a thread with a sweeper will be only using that tc, and no allocation happens for a non-target generation. */ if (from_g == MAX_TG) - nl = TO_VOIDP(ORIGNEXTLOC(tc_in, s)); + nl = TO_VOIDP(tgc->orig_next_loc[s]); else - nl = TO_VOIDP(NEXTLOC_AT(tc_in, s, from_g)); + nl = TO_VOIDP(tgc->next_loc[from_g][s]); d = 0; while (d < cards_per_segment) { @@ -2365,11 +2329,11 @@ static uptr sweep_dirty_segments(ptr tc_in, seginfo **dirty_segments) { relocate_dirty(ppn, youngest, pp, 2 * ptr_bytes); pp = ppn + 1; } else { - FLUSH_REMOTE_RANGE(tc_in, s, from_g); + FLUSH_REMOTE_RANGE(tgc, s, from_g); pp += 2; } } - FLUSH_REMOTE_RANGE(tc_in, s, from_g); + FLUSH_REMOTE_RANGE(tgc, s, from_g); } else { while (pp < ppend && *pp != forward_marker) { /* handle two pointers at a time */ @@ -2378,7 +2342,7 @@ static uptr sweep_dirty_segments(ptr tc_in, seginfo **dirty_segments) { relocate_dirty(ppn, youngest, pp, 2 * ptr_bytes); pp = ppn + 1; } - FLUSH_REMOTE_RANGE(tc_in, s, from_g); + FLUSH_REMOTE_RANGE(tgc, s, from_g); } } else if (s == space_symbol) { /* old symbols cannot overlap segment boundaries @@ -2396,14 +2360,14 @@ static uptr sweep_dirty_segments(ptr tc_in, seginfo **dirty_segments) { ptr p = TYPE(TO_PTR(pp), type_symbol); if (!dirty_si->marked_mask || marked(dirty_si, p)) - youngest = sweep_dirty_symbol(tc_in, p, youngest); + youngest = sweep_dirty_symbol(tgc, p, youngest); else - FLUSH_REMOTE_RANGE(tc_in, s, from_g); + FLUSH_REMOTE_RANGE(tgc, s, from_g); pp += size_symbol / sizeof(ptr); } - FLUSH_REMOTE_RANGE(tc_in, s, from_g); + FLUSH_REMOTE_RANGE(tgc, s, from_g); } else if (s == space_port) { /* old ports cannot overlap segment boundaries since any object that spans multiple @@ -2420,14 +2384,14 @@ static uptr sweep_dirty_segments(ptr tc_in, seginfo **dirty_segments) { ptr p = TYPE(TO_PTR(pp), type_typed_object); if (!dirty_si->marked_mask || marked(dirty_si, p)) - youngest = sweep_dirty_port(tc_in, p, youngest); + youngest = sweep_dirty_port(tgc, p, youngest); else - FLUSH_REMOTE_RANGE(tc_in, s, from_g); + FLUSH_REMOTE_RANGE(tgc, s, from_g); pp += size_port / sizeof(ptr); } - FLUSH_REMOTE_RANGE(tc_in, s, from_g); + FLUSH_REMOTE_RANGE(tgc, s, from_g); } else if (s == space_impure_record) { /* abandon hope all ye who enter here */ ptr p; if (dirty_si->marked_mask) { @@ -2505,17 +2469,17 @@ static uptr sweep_dirty_segments(ptr tc_in, seginfo **dirty_segments) { seginfo *si = SegInfo(ptr_get_segment(p)); if (!marked(si, p)) { /* skip unmarked words */ - FLUSH_REMOTE_RANGE(tc_in, s, from_g); + FLUSH_REMOTE_RANGE(tgc, s, from_g); p = (ptr)((uptr)p + byte_alignment); } else { - youngest = sweep_dirty_record(tc_in, p, youngest); + youngest = sweep_dirty_record(tgc, p, youngest); p = (ptr)((iptr)p + size_record_inst(UNFIX(RECORDDESCSIZE( RECORDINSTTYPE(p))))); } } - FLUSH_REMOTE_RANGE(tc_in, s, from_g); + FLUSH_REMOTE_RANGE(tgc, s, from_g); } else { uptr j; ptr pnext; seginfo *si; @@ -2556,13 +2520,13 @@ static uptr sweep_dirty_segments(ptr tc_in, seginfo **dirty_segments) { /* quit on end of segment */ if (FWDMARKER(p) == forward_marker) break; - youngest = sweep_dirty_record(tc_in, p, youngest); + youngest = sweep_dirty_record(tgc, p, youngest); p = (ptr)((iptr)p + size_record_inst(UNFIX(RECORDDESCSIZE( RECORDINSTTYPE(p))))); } - FLUSH_REMOTE_RANGE(tc_in, s, from_g); + FLUSH_REMOTE_RANGE(tgc, s, from_g); } } else if (s == space_weakpair) { while (pp < ppend && (dirty_si->marked_mask || (*pp != forward_marker))) { @@ -2572,23 +2536,23 @@ static uptr sweep_dirty_segments(ptr tc_in, seginfo **dirty_segments) { relocate_dirty(ppn, youngest, pp, size_pair); pp = ppn + 1; } else { - FLUSH_REMOTE_RANGE(tc_in, s, from_g); + FLUSH_REMOTE_RANGE(tgc, s, from_g); pp += 2; } } - FLUSH_REMOTE_RANGE(tc_in, s, from_g); + FLUSH_REMOTE_RANGE(tgc, s, from_g); } else if (s == space_ephemeron) { while (pp < ppend && (dirty_si->marked_mask || (*pp != forward_marker))) { ptr p = TYPE(TO_PTR(pp), type_pair); if (!dirty_si->marked_mask || marked(dirty_si, p)) - youngest = check_dirty_ephemeron(tc_in, p, youngest); + youngest = check_dirty_ephemeron(tgc, p, youngest); else - FLUSH_REMOTE_RANGE(tc_in, s, from_g); + FLUSH_REMOTE_RANGE(tgc, s, from_g); pp += size_ephemeron / sizeof(ptr); } - FLUSH_REMOTE_RANGE(tc_in, s, from_g); + FLUSH_REMOTE_RANGE(tgc, s, from_g); } else { S_error_abort("sweep_dirty(gc): unexpected space"); } @@ -2621,12 +2585,12 @@ static uptr sweep_dirty_segments(ptr tc_in, seginfo **dirty_segments) { } #ifndef ENABLE_PARALLEL -static void sweep_dirty(ptr tc) { - (void)sweep_dirty_segments(tc, S_G.dirty_segments); +static void sweep_dirty(thread_gc *tgc) { + (void)sweep_dirty_segments(tgc, S_G.dirty_segments); } #endif -static void resweep_dirty_weak_pairs(ptr tc) { +static void resweep_dirty_weak_pairs(thread_gc *tgc) { weakseginfo *ls; ptr *pp, *ppend, p; IGEN from_g, min_youngest, youngest; @@ -2635,7 +2599,7 @@ static void resweep_dirty_weak_pairs(ptr tc) { /* Make sure terminator is in place for allocation areas relevant to this thread */ for (from_g = MIN_TG; from_g <= static_generation; from_g++) { ptr old; - old = NEXTLOC_AT(tc, space_weakpair, from_g); + old = tgc->next_loc[from_g][space_weakpair]; if (old != (ptr)0) *(ptr*)TO_VOIDP(old) = forward_marker; } @@ -2751,21 +2715,21 @@ static void ephemeron_add(ptr *first, ptr pe) { EPHEMERONPREVREF(next) = TO_PTR(&EPHEMERONNEXT(last_pe)); } -static void add_ephemeron_to_pending(ptr tc_in, ptr pe) { +static void add_ephemeron_to_pending(thread_gc *tgc, ptr pe) { /* We could call check_ephemeron directly here, but the indirection through `PENDINGEPHEMERONS` can dramatically decrease the number of times that we have to trigger re-checking, especially since check_pending_pehemerons() is run only after all other sweep opportunities are exhausted. */ if (EPHEMERONPREVREF(pe)) ephemeron_remove(pe); - ephemeron_add(&PENDINGEPHEMERONS(tc_in), pe); + ephemeron_add(&tgc->pending_ephemerons, pe); } -static void add_trigger_ephemerons_to_pending(ptr tc_in, ptr pe) { - ephemeron_add(&PENDINGEPHEMERONS(tc_in), pe); +static void add_trigger_ephemerons_to_pending(thread_gc *tgc, ptr pe) { + ephemeron_add(&tgc->pending_ephemerons, pe); } -static void check_ephemeron(ptr tc_in, ptr pe) { +static void check_ephemeron(thread_gc *tgc, ptr pe) { ptr p; seginfo *si; IGEN from_g; @@ -2782,13 +2746,13 @@ static void check_ephemeron(ptr tc_in, ptr pe) { if (new_marked(si, p)) { #ifndef NO_DIRTY_NEWSPACE_POINTERS IGEN tg = TARGET_GENERATION(si); - if (tg < from_g) S_record_new_dirty_card(tc_in, &INITCAR(pe), tg); + if (tg < from_g) S_record_new_dirty_card(tgc, &INITCAR(pe), tg); #endif relocate_impure(&INITCDR(pe), from_g, pe, size_ephemeron); } else if (FORWARDEDP(p, si)) { #ifndef NO_DIRTY_NEWSPACE_POINTERS IGEN tg = TARGET_GENERATION(si); - if (tg < from_g) S_record_new_dirty_card(tc_in, &INITCAR(pe), tg); + if (tg < from_g) S_record_new_dirty_card(tgc, &INITCAR(pe), tg); #endif INITCAR(pe) = FWDADDRESS(p); relocate_impure(&INITCDR(pe), from_g, pe, size_ephemeron); @@ -2798,26 +2762,26 @@ static void check_ephemeron(ptr tc_in, ptr pe) { si->has_triggers = 1; } } else { - RECORD_REMOTE_RANGE_TO(tc_in, pe, size_ephemeron, SWEEPER(si->creator_tc)); + RECORD_REMOTE_RANGE_TO(tgc, pe, size_ephemeron, si->creator); } } else { relocate_impure(&INITCDR(pe), from_g, pe, size_ephemeron); } - FLUSH_REMOTE_RANGE(tc_in, space_ephemeron, from_g); + FLUSH_REMOTE_RANGE(tgc, space_ephemeron, from_g); POP_BACKREFERENCE(); } -static void check_pending_ephemerons(ptr tc_in) { +static void check_pending_ephemerons(thread_gc *tgc) { ptr pe, next_pe; - pe = PENDINGEPHEMERONS(tc_in); - PENDINGEPHEMERONS(tc_in) = 0; + pe = tgc->pending_ephemerons; + tgc->pending_ephemerons = 0; while (pe != 0) { next_pe = EPHEMERONNEXT(pe); - check_ephemeron(tc_in, pe); + check_ephemeron(tgc, pe); pe = next_pe; } } @@ -2826,7 +2790,7 @@ static void check_pending_ephemerons(ptr tc_in) { ephemeron (that was not yet added to the pending list), so we can be less pessimistic than setting `youngest` to the target generation: */ -static IGEN check_dirty_ephemeron(ptr tc_in, ptr pe, IGEN youngest) { +static IGEN check_dirty_ephemeron(thread_gc *tgc, ptr pe, IGEN youngest) { ptr p; seginfo *si; IGEN pg; @@ -2843,7 +2807,7 @@ static IGEN check_dirty_ephemeron(ptr tc_in, ptr pe, IGEN youngest) { relocate_dirty(&INITCDR(pe), youngest, pe, size_ephemeron); } else { /* Not reached, so far; add to pending list */ - add_ephemeron_to_pending(tc_in, pe); + add_ephemeron_to_pending(tgc, pe); /* Make the consistent (but pessimistic w.r.t. to wrong-way pointers) assumption that the key will stay live and move @@ -2854,7 +2818,7 @@ static IGEN check_dirty_ephemeron(ptr tc_in, ptr pe, IGEN youngest) { youngest = pg; } } else { - RECORD_REMOTE_RANGE_TO(tc_in, pe, size_ephemeron, SWEEPER(si->creator_tc)); + RECORD_REMOTE_RANGE_TO(tgc, pe, size_ephemeron, si->creator); return youngest; } } else { @@ -2873,10 +2837,10 @@ static IGEN check_dirty_ephemeron(ptr tc_in, ptr pe, IGEN youngest) { return youngest; } -static void finish_pending_ephemerons(ptr tc_in, seginfo *si) { +static void finish_pending_ephemerons(thread_gc *tgc, seginfo *si) { /* Any ephemeron still in a trigger list is an ephemeron whose key was not reached. */ - if (PENDINGEPHEMERONS(tc_in) != 0) + if (tgc->pending_ephemerons != 0) S_error_abort("clear_trigger_ephemerons(gc): non-empty pending list"); for (; si != NULL; si = si->next) { @@ -2913,7 +2877,7 @@ static uptr total_size_so_far() { } #endif -static uptr target_generation_space_so_far(ptr tc) { +static uptr target_generation_space_so_far(thread_gc *tgc) { IGEN g; ISPC s; uptr sz = 0; @@ -2923,15 +2887,15 @@ static uptr target_generation_space_so_far(ptr tc) { for (s = 0; s <= max_real_space; s++) { sz += S_G.bytes_of_space[g][s]; - if (NEXTLOC_AT(tc, s, g) != FIX(0)) - sz += (uptr)NEXTLOC_AT(tc, s, g) - (uptr)BASELOC_AT(tc, s, g); + if (tgc->next_loc[g][s] != FIX(0)) + sz += (uptr)tgc->next_loc[g][s] - (uptr)tgc->base_loc[g][s]; } } return sz; } -void copy_and_clear_list_bits(ptr tc_in, seginfo *oldspacesegments) { +void copy_and_clear_list_bits(thread_gc *tgc, seginfo *oldspacesegments) { seginfo *si; int i; @@ -2952,11 +2916,11 @@ void copy_and_clear_list_bits(ptr tc_in, seginfo *oldspacesegments) { if (bits_si->old_space) { if (bits_si->use_marks) { if (!bits_si->marked_mask) - init_mask(tc_in, bits_si->marked_mask, bits_si->generation, 0); + init_mask(tgc, bits_si->marked_mask, bits_si->generation, 0); bits_si->marked_mask[segment_bitmap_byte(TO_PTR(si->list_bits))] |= segment_bitmap_bit(TO_PTR(si->list_bits)); } else { octet *copied_bits; - find_room_voidp(tc_in, space_data, bits_si->generation, ptr_align(segment_bitmap_bytes), copied_bits); + find_gc_room_voidp(tgc, space_data, bits_si->generation, ptr_align(segment_bitmap_bytes), copied_bits); memcpy_aligned(copied_bits, si->list_bits, segment_bitmap_bytes); si->list_bits = copied_bits; S_G.bitmask_overhead[bits_si->generation] += ptr_align(segment_bitmap_bytes); @@ -2984,7 +2948,7 @@ void copy_and_clear_list_bits(ptr tc_in, seginfo *oldspacesegments) { ptr new_p = FWDADDRESS(p); seginfo *new_si = SegInfo(ptr_get_segment(new_p)); if (!new_si->list_bits) - init_mask(tc_in, new_si->list_bits, new_si->generation, 0); + init_mask(tgc, new_si->list_bits, new_si->generation, 0); bits >>= bitpos; new_si->list_bits[segment_bitmap_byte(new_p)] |= segment_bitmap_bits(new_p, bits); } @@ -3008,16 +2972,18 @@ static s_thread_cond_t sweep_cond; static int num_running_sweepers; -static void gather_active_sweepers() { +static void gather_active_sweepers(thread_gc *tgc) { int i, n; + sweepers[main_sweeper_index].tgc = tgc; + /* assign a tc for each sweeper to run in parallel */ for (n = 0, i = 0; (n < maximum_parallel_collect_threads) && (i < S_collect_waiting_threads); i++) { if ((i < maximum_parallel_collect_threads) && (S_collect_waiting_tcs[i] != (ptr)0)) { ptr tc = S_collect_waiting_tcs[i]; if (sweeper_started(n)) { - sweepers[n].sweep_tc = tc; - WILL_BE_SWEEPER(tc) = n; + sweepers[n].tgc = THREAD_GC(tc); + THREAD_GC(tc)->will_be_sweeper = n; n++; } else break; @@ -3029,7 +2995,7 @@ static void gather_active_sweepers() { static s_thread_rv_t start_sweeper(void *_data) { gc_thread_data *data = _data; - ptr tc; + thread_gc *tgc; iptr num_swept_bytes; IGEN g; #ifdef ENABLE_TIMING @@ -3044,28 +3010,28 @@ static s_thread_rv_t start_sweeper(void *_data) { GET_CPU_TIME(start); (void)s_thread_mutex_unlock(&sweep_mutex); - tc = data->sweep_tc; - s_thread_setspecific(S_tc_key, tc); + tgc = data->tgc; + s_thread_setspecific(S_tc_key, TO_VOIDP(tgc->tc)); if (data->thread) { /* sweep tc in this sweeper, so that things it references are more likely handled in this sweeper: */ seginfo *t_si = SegInfo(ptr_get_segment(data->thread)); - sweep_thread(tc, data->thread); - FLUSH_REMOTE_RANGE(tc, t_si->space, t_si->generation); + sweep_thread(tgc, data->thread); + FLUSH_REMOTE_RANGE(tgc, t_si->space, t_si->generation); data->thread = (ptr)0; } num_swept_bytes = 0; - num_swept_bytes += sweep_dirty_segments(tc, data->dirty_segments); - num_swept_bytes += sweep_generation_trading_work(tc); + num_swept_bytes += sweep_dirty_segments(tgc, data->dirty_segments); + num_swept_bytes += sweep_generation_trading_work(tgc); /* ensure terminators on any segment where sweeper may have allocated: */ { ISPC s; IGEN g; for (s = 0; s <= max_real_space; s++) { for (g = MIN_TG; g <= MAX_TG; g++) { - ptr old = NEXTLOC_AT(tc, s, g); + ptr old = tgc->next_loc[g][s]; if (old != (ptr)0) *(ptr*)TO_VOIDP(old) = forward_marker; } @@ -3073,20 +3039,21 @@ static s_thread_rv_t start_sweeper(void *_data) { } (void)s_thread_mutex_lock(&sweep_mutex); - S_G.bitmask_overhead[0] += BITMASKOVERHEAD(tc, 0); - BITMASKOVERHEAD(tc, 0) = 0; + S_G.bitmask_overhead[0] += tgc->bitmask_overhead[0]; + tgc->bitmask_overhead[0] = 0; for (g = MIN_TG; g <= MAX_TG; g++) - S_G.bitmask_overhead[g] += BITMASKOVERHEAD(tc, g); + S_G.bitmask_overhead[g] += tgc->bitmask_overhead[g]; data->status = SWEEPER_READY; ACCUM_CPU_TIME(sweep_accum, step, start); REPORT_TIME(fprintf(stderr, "%d swpr +%ld ms %ld ms %ld bytes %d%%/%d sent %d%%/%d received [%p]\n", MAX_CG, step, sweep_accum, num_swept_bytes, - percentage(sweepers[SWEEPER(tc)].remote_ranges_bytes_sent, num_swept_bytes), - sweepers[SWEEPER(tc)].remote_ranges_sent, - percentage(sweepers[SWEEPER(tc)].remote_ranges_bytes_received, num_swept_bytes), - sweepers[SWEEPER(tc)].remote_ranges_received, - tc)); - SWEEPER(tc) = main_sweeper_index; + percentage(sweepers[tgc->sweeper].remote_ranges_bytes_sent, num_swept_bytes), + sweepers[tgc->sweeper].remote_ranges_sent, + percentage(sweepers[tgc->sweeper].remote_ranges_bytes_received, num_swept_bytes), + sweepers[tgc->sweeper].remote_ranges_received, + tgc)); + tgc->sweeper = main_sweeper_index; + tgc->will_be_sweeper = main_sweeper_index; s_thread_cond_signal(&data->done_cond); } @@ -3120,7 +3087,7 @@ static IBOOL sweeper_started(int i) { return 1; } -static void parallel_sweep_dirty_and_generation(ptr tc) { +static void parallel_sweep_dirty_and_generation(thread_gc *tgc) { int i; iptr num_swept_bytes; @@ -3138,7 +3105,7 @@ static void parallel_sweep_dirty_and_generation(ptr tc) { ADJUST_COUNTER(sweepers[main_sweeper_index].remote_ranges_bytes_received = 0); for (i = 0; i < num_sweepers; i++) { sweepers[i].status = SWEEPER_SWEEPING; - SWEEPER(sweepers[i].sweep_tc) = i; + sweepers[i].tgc->sweeper = i; ADJUST_COUNTER(sweepers[i].remote_ranges_sent = 0); ADJUST_COUNTER(sweepers[i].remote_ranges_bytes_sent = 0); ADJUST_COUNTER(sweepers[i].remote_ranges_received = 0); @@ -3151,8 +3118,8 @@ static void parallel_sweep_dirty_and_generation(ptr tc) { /* sweep in the main thread */ num_swept_bytes = 0; - num_swept_bytes += sweep_dirty_segments(tc, main_dirty_segments); - num_swept_bytes += sweep_generation_trading_work(tc); + num_swept_bytes += sweep_dirty_segments(tgc, sweepers[main_sweeper_index].dirty_segments); + num_swept_bytes += sweep_generation_trading_work(tgc); /* wait for other sweepers */ (void)s_thread_mutex_lock(&sweep_mutex); @@ -3160,7 +3127,7 @@ static void parallel_sweep_dirty_and_generation(ptr tc) { while (sweepers[i].status != SWEEPER_READY) { s_thread_cond_wait(&sweepers[i].done_cond, &sweep_mutex); } - S_flush_instruction_cache(sweepers[i].sweep_tc); + S_flush_instruction_cache(sweepers[i].tgc->tc); } (void)s_thread_mutex_unlock(&sweep_mutex); @@ -3172,22 +3139,21 @@ static void parallel_sweep_dirty_and_generation(ptr tc) { sweepers[main_sweeper_index].remote_ranges_sent, percentage(sweepers[main_sweeper_index].remote_ranges_bytes_received, num_swept_bytes), sweepers[main_sweeper_index].remote_ranges_received, - tc)); + tgc)); S_use_gc_tc_mutex = 0; } -static iptr sweep_generation_trading_work(ptr tc) { +static iptr sweep_generation_trading_work(thread_gc *tgc) { iptr num_swept_bytes = 0; - num_swept_bytes += sweep_generation_pass(tc); + num_swept_bytes += sweep_generation_pass(tgc); (void)s_thread_mutex_lock(&sweep_mutex); --num_running_sweepers; while (1) { - int me = SWEEPER(tc); if ((num_running_sweepers == 0) - && (sweepers[me].ranges_received == NULL)) { + && (sweepers[tgc->sweeper].ranges_received == NULL)) { /* everyone is done */ int i, they = main_sweeper_index; for (i = -1; i < num_sweepers; i++) { @@ -3198,7 +3164,8 @@ static iptr sweep_generation_trading_work(ptr tc) { return num_swept_bytes; } else { /* wait for work */ - if (sweepers[me].ranges_received != NULL) { + int me = tgc->sweeper; + if (sweepers[tgc->sweeper].ranges_received != NULL) { /* some work appeared since we last checked */ num_running_sweepers++; } else { @@ -3208,7 +3175,7 @@ static iptr sweep_generation_trading_work(ptr tc) { if (sweepers[me].status != SWEEPER_WAITING_FOR_WORK) { /* got work; num_running_sweepers was incremented, too */ (void)s_thread_mutex_unlock(&sweep_mutex); - num_swept_bytes += sweep_generation_pass(tc); + num_swept_bytes += sweep_generation_pass(tgc); (void)s_thread_mutex_lock(&sweep_mutex); --num_running_sweepers; } else if (num_running_sweepers == 0) { @@ -3222,21 +3189,21 @@ static iptr sweep_generation_trading_work(ptr tc) { } } -static remote_range *send_and_receive_remote_ranges(ptr tc) { - int i, me = SWEEPER(tc), they; +static remote_range *send_and_receive_remote_ranges(thread_gc *tgc) { + int i, they; remote_range *r, *next, *last; (void)s_thread_mutex_lock(&sweep_mutex); they = main_sweeper_index; for (i = -1; i < num_sweepers; i++) { - if (sweepers[me].ranges_to_send[they] != NULL) { - SWEEPCHANGE(tc) = SWEEP_CHANGE_PROGRESS; - r = sweepers[me].ranges_to_send[they]; - sweepers[me].ranges_to_send[they] = NULL; + r = sweepers[tgc->sweeper].ranges_to_send[they]; + if (r != NULL) { + tgc->sweep_change = SWEEP_CHANGE_PROGRESS; + sweepers[tgc->sweeper].ranges_to_send[they] = NULL; for (next = r, last = r; next != NULL; next = next->next) { - ADJUST_COUNTER(sweepers[me].remote_ranges_sent++); - ADJUST_COUNTER(sweepers[me].remote_ranges_bytes_sent += ((uptr)next->end - (uptr)next->start)); + ADJUST_COUNTER(sweepers[tgc->sweeper].remote_ranges_sent++); + ADJUST_COUNTER(sweepers[tgc->sweeper].remote_ranges_bytes_sent += ((uptr)next->end - (uptr)next->start)); last = next; } last->next = sweepers[they].ranges_received; @@ -3250,17 +3217,17 @@ static remote_range *send_and_receive_remote_ranges(ptr tc) { they = i + 1; } - r = sweepers[me].ranges_received; - sweepers[me].ranges_received = NULL; + r = sweepers[tgc->sweeper].ranges_received; + sweepers[tgc->sweeper].ranges_received = NULL; (void)s_thread_mutex_unlock(&sweep_mutex); if (r != NULL) { - SWEEPCHANGE(tc) = SWEEP_CHANGE_PROGRESS; + tgc->sweep_change = SWEEP_CHANGE_PROGRESS; #ifdef ENABLE_TIMING for (next = r; next != NULL; next = next->next) { - ADJUST_COUNTER(sweepers[me].remote_ranges_received++); - ADJUST_COUNTER(sweepers[me].remote_ranges_bytes_received += ((uptr)next->end - (uptr)next->start)); + ADJUST_COUNTER(sweepers[tgc->sweeper].remote_ranges_received++); + ADJUST_COUNTER(sweepers[tgc->sweeper].remote_ranges_bytes_received += ((uptr)next->end - (uptr)next->start)); } #endif } @@ -3274,13 +3241,13 @@ static remote_range *send_and_receive_remote_ranges(ptr tc) { #ifdef ENABLE_MEASURE -static void init_measure(ptr tc, IGEN min_gen, IGEN max_gen) { +static void init_measure(thread_gc *tgc, IGEN min_gen, IGEN max_gen) { uptr init_stack_len = 1024; min_measure_generation = min_gen; max_measure_generation = max_gen; - find_room_voidp(tc, space_data, 0, ptr_align(init_stack_len), measure_stack_start); + find_gc_room_voidp(tgc, space_data, 0, ptr_align(init_stack_len), measure_stack_start); S_G.bitmask_overhead[0] += ptr_align(init_stack_len); measure_stack = TO_VOIDP(measure_stack_start); measure_stack_limit = TO_VOIDP((uptr)TO_PTR(measure_stack_start) + init_stack_len); @@ -3308,13 +3275,13 @@ static void finish_measure() { measure_all_enabled = 0; } -static void init_counting_mask(ptr tc_in, seginfo *si) { - init_mask(tc_in, si->counting_mask, 0, 0); +static void init_counting_mask(thread_gc *tgc, seginfo *si) { + init_mask(tgc, si->counting_mask, 0, 0); } -static void init_measure_mask(ptr tc_in, seginfo *si) { - init_mask(tc_in, si->measured_mask, 0, 0); - measured_seginfos = S_cons_in(tc_in, space_new, 0, TO_PTR(si), measured_seginfos); +static void init_measure_mask(thread_gc *tgc, seginfo *si) { + init_mask(tgc, si->measured_mask, 0, 0); + measured_seginfos = S_cons_in(tgc->tc, space_new, 0, TO_PTR(si), measured_seginfos); } #define measure_unreached(si, p) \ @@ -3326,7 +3293,7 @@ static void init_measure_mask(ptr tc_in, seginfo *si) { #define measure_mask_unset(mm, si, p) \ mm[segment_bitmap_byte(p)] -= segment_bitmap_bit(p) -static void push_measure(ptr tc_in, ptr p) +static void push_measure(thread_gc *tgc, ptr p) { seginfo *si = MaybeSegInfo(ptr_get_segment(p)); @@ -3349,7 +3316,7 @@ static void push_measure(ptr tc_in, ptr p) uptr bit = segment_bitmap_bit(p); if (!si->measured_mask) - init_measure_mask(tc_in, si); + init_measure_mask(tgc, si); else if (si->measured_mask[byte] & bit) return; @@ -3365,7 +3332,7 @@ static void push_measure(ptr tc_in, ptr p) uptr sz = ptr_bytes * (measure_stack_limit - measure_stack_start); uptr new_sz = 2*sz; ptr *new_measure_stack; - find_room_voidp(tc_in, space_data, 0, ptr_align(new_sz), new_measure_stack); + find_gc_room_voidp(tgc, space_data, 0, ptr_align(new_sz), new_measure_stack); S_G.bitmask_overhead[0] += ptr_align(new_sz); memcpy(new_measure_stack, measure_stack_start, sz); measure_stack_start = new_measure_stack; @@ -3384,7 +3351,7 @@ static void measure_add_stack_size(ptr stack, uptr size) { measure_total += size; } -static void add_ephemeron_to_pending_measure(ptr tc_in, ptr pe) { +static void add_ephemeron_to_pending_measure(thread_gc *tgc, ptr pe) { /* If we're in hybrid mode and the key in `pe` is in the old space, then we need to use the regular pending list instead of the measure-specific one */ @@ -3392,7 +3359,7 @@ static void add_ephemeron_to_pending_measure(ptr tc_in, ptr pe) { ptr p = Scar(pe); if (!IMMEDIATE(p) && (si = MaybeSegInfo(ptr_get_segment(p))) != NULL && si->old_space) - add_ephemeron_to_pending(tc_in, pe); + add_ephemeron_to_pending(tgc, pe); else { if (EPHEMERONPREVREF(pe)) S_error_abort("add_ephemeron_to_pending_measure: ephemeron is in some list"); @@ -3404,7 +3371,7 @@ static void add_trigger_ephemerons_to_pending_measure(ptr pe) { ephemeron_add(&pending_measure_ephemerons, pe); } -static void check_ephemeron_measure(ptr tc_in, ptr pe) { +static void check_ephemeron_measure(thread_gc *tgc, ptr pe) { ptr p; seginfo *si; @@ -3422,28 +3389,28 @@ static void check_ephemeron_measure(ptr tc_in, ptr pe) { /* Not reached, so far; install as trigger */ ephemeron_add(&si->trigger_ephemerons, pe); if (!si->measured_mask) - init_measure_mask(tc_in, si); /* so triggers are cleared at end */ + init_measure_mask(tgc, si); /* so triggers are cleared at end */ return; } p = Scdr(pe); if (!IMMEDIATE(p)) - push_measure(tc_in, p); + push_measure(tgc, p); } -static void check_pending_measure_ephemerons(ptr tc_in) { +static void check_pending_measure_ephemerons(thread_gc *tgc) { ptr pe, next_pe; pe = pending_measure_ephemerons; pending_measure_ephemerons = 0; while (pe != 0) { next_pe = EPHEMERONNEXT(pe); - check_ephemeron_measure(tc_in, pe); + check_ephemeron_measure(tgc, pe); pe = next_pe; } } -void gc_measure_one(ptr tc_in, ptr p) { +void gc_measure_one(thread_gc *tgc, ptr p) { seginfo *si = SegInfo(ptr_get_segment(p)); if (si->trigger_ephemerons) { @@ -3451,23 +3418,23 @@ void gc_measure_one(ptr tc_in, ptr p) { si->trigger_ephemerons = 0; } - measure(tc_in, p); + measure(tgc, p); - (void)flush_measure_stack(tc_in); + (void)flush_measure_stack(tgc); } -IBOOL flush_measure_stack(ptr tc_in) { +IBOOL flush_measure_stack(thread_gc *tgc) { if ((measure_stack <= measure_stack_start) && !pending_measure_ephemerons) return 0; while (1) { while (measure_stack > measure_stack_start) - measure(tc_in, *(--measure_stack)); + measure(tgc, *(--measure_stack)); if (!pending_measure_ephemerons) break; - check_pending_measure_ephemerons(tc_in); + check_pending_measure_ephemerons(tgc); } return 1; @@ -3476,10 +3443,11 @@ IBOOL flush_measure_stack(ptr tc_in) { ptr S_count_size_increments(ptr ls, IGEN generation) { ptr l, totals = Snil, totals_prev = 0; ptr tc = get_thread_context(); + thread_gc *tgc = THREAD_GC(tc); tc_mutex_acquire(); - init_measure(tc, 0, generation); + init_measure(tgc, 0, generation); for (l = ls; l != Snil; l = Scdr(l)) { ptr p = Scar(l); @@ -3487,11 +3455,11 @@ ptr S_count_size_increments(ptr ls, IGEN generation) { seginfo *si = si = SegInfo(ptr_get_segment(p)); if (!si->measured_mask) - init_measure_mask(tc, si); + init_measure_mask(tgc, si); measure_mask_set(si->measured_mask, si, p); if (!si->counting_mask) - init_counting_mask(tc, si); + init_counting_mask(tgc, si); measure_mask_set(si->counting_mask, si, p); } } @@ -3504,7 +3472,7 @@ ptr S_count_size_increments(ptr ls, IGEN generation) { if (!IMMEDIATE(p)) { seginfo *si = si = SegInfo(ptr_get_segment(p)); measure_mask_unset(si->counting_mask, si, p); - gc_measure_one(tc, p); + gc_measure_one(tgc, p); } p = Scons(FIX(measure_total), Snil); diff --git a/racket/src/ChezScheme/c/gcwrapper.c b/racket/src/ChezScheme/c/gcwrapper.c index 81f1308c51..2b332fcc7f 100644 --- a/racket/src/ChezScheme/c/gcwrapper.c +++ b/racket/src/ChezScheme/c/gcwrapper.c @@ -573,7 +573,6 @@ static void check_pointer(ptr *pp, IBOOL address_is_meaningful, ptr base, uptr s if (Scar(l) == p) printf(" in unlocked\n"); } - abort(); // REMOVEME } } } @@ -591,7 +590,7 @@ static ptr *find_nl(ptr *pp1, ptr *pp2, ISPC s, IGEN g) { for (ls = S_threads; ls != Snil; ls = Scdr(ls)) { ptr t_tc = (ptr)THREADTC(Scar(ls)); - nl = TO_VOIDP(NEXTLOC_AT(t_tc, s, g)); + nl = TO_VOIDP(THREAD_GC(t_tc)->next_loc[g][s]); if (pp1 <= nl && nl < pp2) return nl; } @@ -626,17 +625,18 @@ void S_check_heap(aftergc, mcg) IBOOL aftergc; IGEN mcg; { ptr t_tc = (ptr)THREADTC(Scar(ls)); for (s = 0; s <= max_real_space; s += 1) { for (g = 0; g <= static_generation; INCRGEN(g)) { - if ((NEXTLOC_AT(t_tc, s, g) == (ptr)0) != (BASELOC_AT(t_tc, s, g) == (ptr)0)) { + thread_gc *tgc = THREAD_GC(t_tc); + if ((tgc->next_loc[g][s] == (ptr)0) != (tgc->base_loc[g][s] == (ptr)0)) { S_checkheap_errors += 1; printf("!!! inconsistent thread NEXT %p and BASE %p\n", - TO_VOIDP(NEXTLOC_AT(t_tc, s, g)), TO_VOIDP(BASELOC_AT(t_tc, s, g))); + TO_VOIDP(tgc->next_loc[g][s]), TO_VOIDP(tgc->base_loc[g][s])); } - if ((REMOTERANGEEND(t_tc) != (ptr)0) - || (REMOTERANGESTART(t_tc) != (ptr)(uptr)-1)) { + if ((tgc->remote_range_end != (ptr)0) + || (tgc->remote_range_start != (ptr)(uptr)-1)) { S_checkheap_errors += 1; printf("!!! nonempty thread REMOTERANGE %p-%p\n", - TO_VOIDP(REMOTERANGESTART(t_tc)), - TO_VOIDP(REMOTERANGEEND(t_tc))); + TO_VOIDP(tgc->remote_range_start), + TO_VOIDP(tgc->remote_range_end)); } } } @@ -1122,11 +1122,12 @@ ptr S_do_gc(IGEN max_cg, IGEN min_tg, IGEN max_tg, ptr count_roots) { ptr ls; for (ls = S_threads; ls != Snil; ls = Scdr(ls)) { ptr t_tc = (ptr)THREADTC(Scar(ls)); - BASELOC_AT(t_tc, s, new_g) = BASELOC_AT(t_tc, s, old_g); BASELOC_AT(t_tc, s, old_g) = (ptr)0; - NEXTLOC_AT(t_tc, s, new_g) = NEXTLOC_AT(t_tc, s, old_g); NEXTLOC_AT(t_tc, s, old_g) = (ptr)0; - BYTESLEFT_AT(t_tc, s, new_g) = BYTESLEFT_AT(t_tc, s, old_g); BYTESLEFT_AT(t_tc, s, old_g) = 0; - SWEEPLOC_AT(t_tc, s, new_g) = SWEEPLOC_AT(t_tc, s, old_g); SWEEPLOC_AT(t_tc, s, old_g) = 0; - SWEEPNEXT_AT(t_tc, s, new_g) = SWEEPNEXT_AT(t_tc, s, old_g); SWEEPNEXT_AT(t_tc, s, old_g) = 0; + thread_gc *tgc = THREAD_GC(t_tc); + tgc->base_loc[new_g][s] = tgc->base_loc[old_g][s]; tgc->base_loc[old_g][s] = (ptr)0; + tgc->next_loc[new_g][s] = tgc->next_loc[old_g][s]; tgc->next_loc[old_g][s] = (ptr)0; + tgc->bytes_left[new_g][s] = tgc->bytes_left[old_g][s]; tgc->bytes_left[old_g][s] = 0; + tgc->sweep_loc[new_g][s] = tgc->sweep_loc[old_g][s]; tgc->sweep_loc[old_g][s] = 0; + tgc->sweep_next[new_g][s] = tgc->sweep_next[old_g][s]; tgc->sweep_next[old_g][s] = NULL; } } } diff --git a/racket/src/ChezScheme/c/globals.h b/racket/src/ChezScheme/c/globals.h index 4722109a60..6c3ec92ecc 100644 --- a/racket/src/ChezScheme/c/globals.h +++ b/racket/src/ChezScheme/c/globals.h @@ -75,6 +75,7 @@ EXTERN ptr S_foreign_dynamic; EXTERN struct S_G_struct { /* scheme.c */ double thread_context[size_tc / sizeof(double)]; + thread_gc main_thread_gc; ptr active_threads_id; ptr error_invoke_code_object; ptr invoke_code_object; diff --git a/racket/src/ChezScheme/c/prim5.c b/racket/src/ChezScheme/c/prim5.c index 6282230116..06d0599426 100644 --- a/racket/src/ChezScheme/c/prim5.c +++ b/racket/src/ChezScheme/c/prim5.c @@ -434,8 +434,8 @@ static void s_showalloc(IBOOL show_dump, const char *outfn) { /* add in bytes previously recorded */ bytes[g][s] += S_G.bytes_of_space[g][s]; /* add in bytes in active segments */ - if (NEXTLOC_AT(tc, s, g) != FIX(0)) - bytes[g][s] += (uptr)NEXTLOC_AT(tc, s, g) - (uptr)BASELOC_AT(tc, s, g); + if (THREAD_GC(tc)->next_loc[g][s] != FIX(0)) + bytes[g][s] += (uptr)THREAD_GC(tc)->next_loc[g][s] - (uptr)THREAD_GC(tc)->base_loc[g][s]; } } diff --git a/racket/src/ChezScheme/c/scheme.c b/racket/src/ChezScheme/c/scheme.c index 68e7d07f89..aa0f7d826b 100644 --- a/racket/src/ChezScheme/c/scheme.c +++ b/racket/src/ChezScheme/c/scheme.c @@ -1135,7 +1135,7 @@ extern void Sbuild_heap(kernel, custom_init) const char *kernel; void (*custom_i * thread context and hence there is no parent thread context. */ tc = (ptr)THREADTC(S_create_thread_object("startup", tc)); #ifdef PTHREADS - s_thread_setspecific(S_tc_key, tc); + s_thread_setspecific(S_tc_key, TO_VOIDP(tc)); #endif /* #scheme-init enables interrupts */ diff --git a/racket/src/ChezScheme/c/segment.c b/racket/src/ChezScheme/c/segment.c index 24e1bc7323..b3a6d29378 100644 --- a/racket/src/ChezScheme/c/segment.c +++ b/racket/src/ChezScheme/c/segment.c @@ -37,7 +37,7 @@ Low-level Memory management strategy: #include static void out_of_memory PROTO((void)); -static void initialize_seginfo PROTO((seginfo *si, ptr tc, ISPC s, IGEN g)); +static void initialize_seginfo PROTO((seginfo *si, thread_gc *creator, ISPC s, IGEN g)); static seginfo *allocate_segments PROTO((uptr nreq)); static void expand_segment_table PROTO((uptr base, uptr end, seginfo *si)); static void contract_segment_table PROTO((uptr base, uptr end)); @@ -225,7 +225,7 @@ static INT find_index(iptr n) { return (index < PARTIAL_CHUNK_POOLS-1) ? index : PARTIAL_CHUNK_POOLS-1; } -static void initialize_seginfo(seginfo *si, NO_THREADS_UNUSED ptr tc, ISPC s, IGEN g) { +static void initialize_seginfo(seginfo *si, NO_THREADS_UNUSED thread_gc *creator, ISPC s, IGEN g) { INT d; si->space = s; @@ -235,7 +235,7 @@ static void initialize_seginfo(seginfo *si, NO_THREADS_UNUSED ptr tc, ISPC s, IG si->use_marks = 0; si->must_mark = 0; #ifdef PTHREADS - si->creator_tc = tc; + si->creator = creator; #endif si->list_bits = NULL; si->min_dirty_byte = 0xff; @@ -256,7 +256,7 @@ static void initialize_seginfo(seginfo *si, NO_THREADS_UNUSED ptr tc, ISPC s, IG si->sweep_next = NULL; } -iptr S_find_segments(tc, s, g, n) ptr tc; ISPC s; IGEN g; iptr n; { +iptr S_find_segments(creator, s, g, n) thread_gc *creator; ISPC s; IGEN g; iptr n; { chunkinfo *chunk, *nextchunk; seginfo *si, *nextsi, **prevsi; iptr nunused_segs, j; @@ -280,7 +280,7 @@ iptr S_find_segments(tc, s, g, n) ptr tc; ISPC s; IGEN g; iptr n; { } chunk->nused_segs += 1; - initialize_seginfo(si, tc, s, g); + initialize_seginfo(si, creator, s, g); si->next = S_G.occupied_segments[g][s]; S_G.occupied_segments[g][s] = si; S_G.number_of_empty_segments -= 1; @@ -318,7 +318,7 @@ iptr S_find_segments(tc, s, g, n) ptr tc; ISPC s; IGEN g; iptr n; { nextsi->next = S_G.occupied_segments[g][s]; S_G.occupied_segments[g][s] = si; for (j = n, nextsi = si; j > 0; j -= 1, nextsi = nextsi->next) { - initialize_seginfo(nextsi, tc, s, g); + initialize_seginfo(nextsi, creator, s, g); } S_G.number_of_empty_segments -= n; return si->number; @@ -338,7 +338,7 @@ iptr S_find_segments(tc, s, g, n) ptr tc; ISPC s; IGEN g; iptr n; { /* we couldn't find space, so ask for more */ si = allocate_segments(n); for (nextsi = si; n > 0; n -= 1, nextsi += 1) { - initialize_seginfo(nextsi, tc, s, g); + initialize_seginfo(nextsi, creator, s, g); /* add segment to appropriate list of occupied segments */ nextsi->next = S_G.occupied_segments[g][s]; S_G.occupied_segments[g][s] = nextsi; diff --git a/racket/src/ChezScheme/c/thread.c b/racket/src/ChezScheme/c/thread.c index 269ae7ffc8..feeae8998f 100644 --- a/racket/src/ChezScheme/c/thread.c +++ b/racket/src/ChezScheme/c/thread.c @@ -16,6 +16,8 @@ #include "system.h" +static thread_gc *free_thread_gcs; + /* locally defined functions */ #ifdef PTHREADS static s_thread_rv_t start_thread PROTO((void *tc)); @@ -55,28 +57,40 @@ void S_thread_init() { or more places. */ ptr S_create_thread_object(who, p_tc) const char *who; ptr p_tc; { ptr thread, tc; + thread_gc *tgc; INT i; tc_mutex_acquire(); if (S_threads == Snil) { tc = TO_PTR(S_G.thread_context); + tgc = &S_G.main_thread_gc; } else { /* clone parent */ ptr p_v = PARAMETERS(p_tc); iptr i, n = Svector_length(p_v); ptr v; tc = TO_PTR(malloc(size_tc)); + if (free_thread_gcs) { + tgc = free_thread_gcs; + free_thread_gcs = tgc->next; + } else + tgc = malloc(sizeof(thread_gc)); if (tc == (ptr)0) S_error(who, "unable to malloc thread data structure"); memcpy(TO_VOIDP(tc), TO_VOIDP(p_tc), size_tc); - for (i = 0; i < num_thread_local_allocation_segments; i++) { - BASELOC(tc, i) = (ptr)0; - NEXTLOC(tc, i) = (ptr)0; - BYTESLEFT(tc, i) = 0; - SWEEPLOC(tc, i) = (ptr)0; + { + IGEN g; ISPC s; + for (g = 0; g <= static_generation; g++) { + for (s = 0; s <= max_real_space; s++) { + tgc->base_loc[g][s] = (ptr)0; + tgc->next_loc[g][s] = (ptr)0; + tgc->bytes_left[g][s] = 0; + tgc->sweep_loc[g][s] = (ptr)0; + } + } } v = S_vector_in(tc, space_new, 0, n); @@ -88,6 +102,9 @@ ptr S_create_thread_object(who, p_tc) const char *who; ptr p_tc; { CODERANGESTOFLUSH(tc) = Snil; } + GCDATA(tc) = TO_PTR(tgc); + tgc->tc = tc; + /* override nonclonable tc fields */ THREADNO(tc) = S_G.threadno; S_G.threadno = S_add(S_G.threadno, FIX(1)); @@ -141,9 +158,12 @@ ptr S_create_thread_object(who, p_tc) const char *who; ptr p_tc; { LZ4OUTBUFFER(tc) = 0; - SWEEPER(tc) = -1; - REMOTERANGESTART(tc) = (ptr)(uptr)-1; - REMOTERANGEEND(tc) = (ptr)0; + tgc->sweeper = main_sweeper_index; + tgc->will_be_sweeper = main_sweeper_index; + tgc->will_be_sweeper = main_sweeper_index; + tgc->remote_range_start = (ptr)(uptr)-1; + tgc->remote_range_end = (ptr)0; + tgc->pending_ephemerons = (ptr)0; tc_mutex_release(); @@ -158,8 +178,8 @@ IBOOL Sactivate_thread() { /* create or reactivate current thread */ ptr thread; /* borrow base thread for now */ - thread = S_create_thread_object("Sactivate_thread", S_G.thread_context); - s_thread_setspecific(S_tc_key, (ptr)THREADTC(thread)); + thread = S_create_thread_object("Sactivate_thread", TO_PTR(S_G.thread_context)); + s_thread_setspecific(S_tc_key, TO_VOIDP(THREADTC(thread))); return 1; } else { reactivate_thread(tc) @@ -223,12 +243,13 @@ static IBOOL destroy_thread(tc) ptr tc; { /* process remembered set before dropping allocation area */ S_scan_dirty((ptr *)EAP(tc), (ptr *)REAL_EAP(tc)); - /* move thread-local allocation to global space */ + /* close off thread-local allocation */ { ISPC s; IGEN g; + thread_gc *tgc = THREAD_GC(tc); for (g = 0; g <= static_generation; g++) for (s = 0; s <= max_real_space; s++) - if (NEXTLOC_AT(tc, s, g)) + if (tgc->next_loc[g][s]) S_close_off_thread_local_segment(tc, s, g); } @@ -258,10 +279,18 @@ static IBOOL destroy_thread(tc) ptr tc; { } } - if (LZ4OUTBUFFER(tc) != NULL) free(LZ4OUTBUFFER(tc)); - if (SIGNALINTERRUPTQUEUE(tc) != NULL) free(SIGNALINTERRUPTQUEUE(tc)); + if (LZ4OUTBUFFER(tc) != (ptr)0) free(TO_VOIDP(LZ4OUTBUFFER(tc))); + if (SIGNALINTERRUPTQUEUE(tc) != (ptr)0) free(TO_VOIDP(SIGNALINTERRUPTQUEUE(tc))); + + /* Never free a thread_gc, since it may be recorded in a segment + as the segment's creator. Recycle manually, instead. */ + THREAD_GC(tc)->sweeper = main_sweeper_index; + THREAD_GC(tc)->tc = (ptr)0; + THREAD_GC(tc)->next = free_thread_gcs; + free_thread_gcs = THREAD_GC(tc); free((void *)tc); + THREADTC(thread) = 0; /* mark it dead */ status = 1; break; @@ -291,7 +320,7 @@ ptr S_fork_thread(thunk) ptr thunk; { static s_thread_rv_t start_thread(p) void *p; { ptr tc = (ptr)p; ptr cp; - s_thread_setspecific(S_tc_key, tc); + s_thread_setspecific(S_tc_key, TO_VOIDP(tc)); cp = CP(tc); CP(tc) = Svoid; /* should hold calling code object, which we don't have */ @@ -303,7 +332,7 @@ static s_thread_rv_t start_thread(p) void *p; { /* find and destroy our thread */ destroy_thread(tc); - s_thread_setspecific(S_tc_key, (ptr)0); + s_thread_setspecific(S_tc_key, NULL); s_thread_return; } @@ -335,7 +364,7 @@ void S_mutex_acquire(m) scheme_mutex_t *m; { if ((count = m->count) > 0 && s_thread_equal(m->owner, self)) { if (count == most_positive_fixnum) - S_error1("mutex-acquire", "recursion limit exceeded for ~s", m); + S_error1("mutex-acquire", "recursion limit exceeded for ~s", TO_PTR(m)); m->count = count + 1; return; } @@ -353,7 +382,7 @@ INT S_mutex_tryacquire(m) scheme_mutex_t *m; { if ((count = m->count) > 0 && s_thread_equal(m->owner, self)) { if (count == most_positive_fixnum) - S_error1("mutex-acquire", "recursion limit exceeded for ~s", m); + S_error1("mutex-acquire", "recursion limit exceeded for ~s", TO_PTR(m)); m->count = count + 1; return 0; } @@ -374,7 +403,7 @@ void S_mutex_release(m) scheme_mutex_t *m; { INT status; if ((count = m->count) == 0 || !s_thread_equal(m->owner, self)) - S_error1("mutex-release", "thread does not own mutex ~s", m); + S_error1("mutex-release", "thread does not own mutex ~s", TO_PTR(m)); if ((m->count = count - 1) == 0) { m->owner = 0; /* needed for a memory model like ARM, for example */ @@ -460,10 +489,10 @@ IBOOL S_condition_wait(c, m, t) s_thread_cond_t *c; scheme_mutex_t *m; ptr t; { iptr collect_index = 0; if ((count = m->count) == 0 || !s_thread_equal(m->owner, self)) - S_error1("condition-wait", "thread does not own mutex ~s", m); + S_error1("condition-wait", "thread does not own mutex ~s", TO_PTR(m)); if (count != 1) - S_error1("condition-wait", "mutex ~s is recursively locked", m); + S_error1("condition-wait", "mutex ~s is recursively locked", TO_PTR(m)); if (t != Sfalse) { /* Keep in sync with ts record in s/date.ss */ diff --git a/racket/src/ChezScheme/c/types.h b/racket/src/ChezScheme/c/types.h index 7f1517ea94..da367b445b 100644 --- a/racket/src/ChezScheme/c/types.h +++ b/racket/src/ChezScheme/c/types.h @@ -76,30 +76,24 @@ typedef int IFASLCODE; /* fasl type codes */ #define ALREADY_PTR(p) (p) -#define SG_AT_TO_INDEX(s, g) (((g) * (1 + max_real_space)) + (s)) - -#define BASELOC_AT(tc, s, g) BASELOC(tc, SG_AT_TO_INDEX(s, g)) -#define NEXTLOC_AT(tc, s, g) NEXTLOC(tc, SG_AT_TO_INDEX(s, g)) -#define BYTESLEFT_AT(tc, s, g) BYTESLEFT(tc, SG_AT_TO_INDEX(s, g)) -#define SWEEPLOC_AT(tc, s, g) SWEEPLOC(tc, SG_AT_TO_INDEX(s, g)) -#define SWEEPNEXT_AT(tc, s, g) SWEEPNEXT(tc, SG_AT_TO_INDEX(s, g)) - /* inline allocation --- no mutex required */ /* find room allocates n bytes in space s and generation g into * destination x, tagged with ty, punting to find_more_room if * no space is left in the current segment. n is assumed to be * an integral multiple of the object alignment. */ -#define find_room_T(tc, s, g, t, n, T, x) do { \ - iptr L_IDX = SG_AT_TO_INDEX(s, g); \ - iptr N_BYTES = n; \ - ptr X = NEXTLOC(tc, L_IDX); \ - NEXTLOC(tc, L_IDX) = (ptr)((uptr)X + N_BYTES); \ - if ((BYTESLEFT(tc, L_IDX) -= (n)) < 0) X = S_find_more_thread_room(tc, s, g, N_BYTES, X); \ - (x) = T(TYPE(X, t)); \ +#define find_gc_room_T(tgc, s, g, t, n, T, x) do { \ + thread_gc *TGC = tgc; \ + iptr N_BYTES = n; \ + ptr X = TGC->next_loc[g][s]; \ + TGC->next_loc[g][s] = (ptr)((uptr)X + N_BYTES); \ + if ((TGC->bytes_left[g][s] -= (n)) < 0) X = S_find_more_gc_room(tgc, s, g, N_BYTES, X); \ + (x) = T(TYPE(X, t)); \ } while(0) -#define find_room(tc, s, g, t, n, x) find_room_T(tc, s, g, t, n, ALREADY_PTR, x) -#define find_room_voidp(tc, s, g, n, x) find_room_T(tc, s, g, typemod, n, TO_VOIDP, x) +#define find_room(tc, s, g, t, n, x) find_gc_room_T(THREAD_GC(tc), s, g, t, n, ALREADY_PTR, x) +#define find_gc_room(tgc, s, g, t, n, x) find_gc_room_T(tgc, s, g, t, n, ALREADY_PTR, x) +#define find_room_voidp(tc, s, g, n, x) find_gc_room_T(THREAD_GC(tc), s, g, typemod, n, TO_VOIDP, x) +#define find_gc_room_voidp(tgc, s, g, n, x) find_gc_room_T(tgc, s, g, typemod, n, TO_VOIDP, x) /* new-space inline allocation --- no mutex required */ /* Like `find_room`, but always `space_new` and generation 0, @@ -158,7 +152,7 @@ typedef struct _seginfo { octet *list_bits; /* for `$list-bits-ref` and `$list-bits-set!` */ uptr number; /* the segment number */ #ifdef PTHREADS - ptr creator_tc; /* for GC parallelism heuristic; might not match an active thread unless old_space */ + struct thread_gc *creator; /* for GC parallelism; might not have an active thread, unless old_space */ #endif struct _chunkinfo *chunk; /* the chunk this segment belongs to */ struct _seginfo *next; /* pointer to the next seginfo (used in occupied_segments and unused_segs) */ @@ -452,6 +446,47 @@ typedef struct { #define AS_IMPLICIT_ATOMIC(T, X) X #endif +typedef struct remote_range { + ISPC s; + IGEN g; + ptr start, end; + struct remote_range *next; +} remote_range; + +typedef struct thread_gc { + ptr tc; + + struct thread_gc *next; + + ptr base_loc[static_generation+1][max_real_space+1]; + ptr next_loc[static_generation+1][max_real_space+1]; + iptr bytes_left[static_generation+1][max_real_space+1]; + ptr orig_next_loc[max_real_space+1]; + ptr sweep_loc[static_generation+1][max_real_space+1]; + seginfo *sweep_next[static_generation+1][max_real_space+1]; + + ptr pending_ephemerons; + + ptr sweep_stack; + ptr sweep_stack_start; + ptr sweep_stack_limit; + + int sweep_change; + + int sweeper; /* parallel GC: sweeper thread identity */ + int will_be_sweeper; + + struct thread_gc *remote_range_tgc; + ptr remote_range_start; + ptr remote_range_end; + + iptr bitmask_overhead[static_generation+1]; +} thread_gc; + +#define THREAD_GC(tc) ((thread_gc *)TO_VOIDP(GCDATA(tc))) + +#define main_sweeper_index maximum_parallel_collect_threads + #ifdef __MINGW32__ /* With MinGW on 64-bit Windows, setjmp/longjmp is not reliable. Using __builtin_setjmp/__builtin_longjmp is reliable, but diff --git a/racket/src/ChezScheme/s/cmacros.ss b/racket/src/ChezScheme/s/cmacros.ss index 0ea135209c..b8d5e2e2bb 100644 --- a/racket/src/ChezScheme/s/cmacros.ss +++ b/racket/src/ChezScheme/s/cmacros.ss @@ -1498,11 +1498,7 @@ (define-constant virtual-register-count 16) (define-constant static-generation 7) -(define-constant num-generations (fx+ (constant static-generation) 1)) -(define-constant num-spaces (fx+ (constant max-real-space) 1)) -(define-constant num-thread-local-allocation-segments (fx* (constant num-generations) - (constant num-spaces))) -(define-constant maximum-parallel-collect-threads 8) +(define-constant maximum-parallel-collect-threads 16) ;;; make sure gc sweeps all ptrs (define-primitive-structure-disps tc typemod @@ -1578,23 +1574,7 @@ [ptr DSTBV] [ptr SRCBV] [double fpregs (constant asm-fpreg-max)] - ;; thread-local allocation and parallel collection: - [xptr base-loc (constant num-thread-local-allocation-segments)] - [xptr next-loc (constant num-thread-local-allocation-segments)] - [iptr bytes-left (constant num-thread-local-allocation-segments)] - [xptr orig-next-loc (constant num-spaces)] - [xptr sweep-loc (constant num-thread-local-allocation-segments)] - [xptr sweep-next (constant num-thread-local-allocation-segments)] - [xptr pending-ephemerons] - [iptr sweeper] - [xptr sweep-stack] - [xptr sweep-stack-start] - [xptr sweep-stack-limit] - [iptr sweep-change] - [iptr remote-sweeper] - [xptr remote-range-start] - [xptr remote-range-end] - [iptr bitmask-overhead (constant num-generations)])) + [xptr gc-data])) (define tc-field-list (let f ([ls (oblist)] [params '()]) diff --git a/racket/src/ChezScheme/s/mkgc.ss b/racket/src/ChezScheme/s/mkgc.ss index 9534b50160..cd2094debf 100644 --- a/racket/src/ChezScheme/s/mkgc.ss +++ b/racket/src/ChezScheme/s/mkgc.ss @@ -243,10 +243,10 @@ ;; A stack segment has a single owner, so it's ok for us ;; to sweep the stack content, even though it's on a ;; remote segment relative to the current sweeper. - (RECORD_REMOTE_RANGE _tc_ _ _size_ s_si)] + (RECORD_REMOTE_RANGE _tgc_ _ _size_ s_si)] [else (set! (continuation-stack _) - (copy_stack _tc_ + (copy_stack _tgc_ (continuation-stack _) (& (continuation-stack-length _)) (continuation-stack-clength _)))]))] @@ -687,9 +687,9 @@ (define-trace-macro (add-ephemeron-to-pending) (case-mode [(sweep mark) - (add_ephemeron_to_pending _tc_ _)] + (add_ephemeron_to_pending _tgc_ _)] [measure - (add_ephemeron_to_pending_measure _tc_ _)] + (add_ephemeron_to_pending_measure _tgc_ _)] [else])) (define-trace-macro (assert-ephemeron-size-ok) @@ -744,7 +744,7 @@ (set! (continuation-stack-length _copy_) (continuation-stack-clength _)) ;; May need to recur at end to promote link: (GC_TC_MUTEX_ACQUIRE) - (set! conts_to_promote (S_cons_in _tc_ space_new 0 _copy_ conts_to_promote)) + (set! conts_to_promote (S_cons_in (-> _tgc_ tc) space_new 0 _copy_ conts_to_promote)) (GC_TC_MUTEX_RELEASE)] [else (copy continuation-stack-length)])] @@ -794,7 +794,7 @@ (SEGMENT_IS_LOCAL v_si val)) (trace-symcode symbol-pvalue val)] [else - (RECORD_REMOTE_RANGE _tc_ _ _size_ v_si)])] + (RECORD_REMOTE_RANGE _tgc_ _ _size_ v_si)])] [off (trace-symcode symbol-pvalue val)])] [else (trace-symcode symbol-pvalue val)])) @@ -816,7 +816,7 @@ ;; swept already. NB: assuming keyvals are always pairs. (when (&& (!= next Sfalse) (OLDSPACE keyval)) (GC_TC_MUTEX_ACQUIRE) - (set! tlcs_to_rehash (S_cons_in _tc_ space_new 0 _copy_ tlcs_to_rehash)) + (set! tlcs_to_rehash (S_cons_in (-> _tgc_ tc) space_new 0 _copy_ tlcs_to_rehash)) (GC_TC_MUTEX_RELEASE))] [else (trace-nonself tlc-keyval) @@ -869,7 +869,7 @@ (trace-record-type-pm num rtd)] [else ;; Try again in the bignum's sweeper - (RECORD_REMOTE_RANGE _tc_ _ _size_ pm_si) + (RECORD_REMOTE_RANGE _tgc_ _ _size_ pm_si) (set! num S_G.zero_length_bignum)])] [off (trace-record-type-pm num rtd)])] @@ -948,7 +948,7 @@ (let* ([grtd : IGEN (GENERATION c_rtd)]) (set! (array-ref (array-ref S_G.countof grtd) countof_rtd_counts) += 1) ;; Allocate counts struct in same generation as rtd. Initialize timestamp & counts. - (find_room _tc_ space_data grtd type_typed_object size_rtd_counts counts) + (find_gc_room _tgc_ space_data grtd type_typed_object size_rtd_counts counts) (set! (rtd-counts-type counts) type_rtd_counts) (set! (rtd-counts-timestamp counts) (array-ref S_G.gctimestamp 0)) (let* ([g : IGEN 0]) @@ -961,7 +961,7 @@ ;; For max_copied_generation, the list will get copied again in `rtds_with_counts` fixup; ;; meanwhile, allocating in `space_impure` would copy and sweep old list entries causing ;; otherwise inaccessible rtds to be retained - (S_cons_in _tc_ + (S_cons_in (-> _tgc_ tc) (cond [(<= grtd MAX_CG) space_new] [else space_impure]) (cond [(<= grtd MAX_CG) 0] [else grtd]) c_rtd @@ -1012,7 +1012,7 @@ (when (OLDSPACE old_stack) (let* ([clength : iptr (- (cast uptr (SFP tc)) (cast uptr old_stack))]) ;; Include SFP[0], which contains the return address - (set! (tc-scheme-stack tc) (copy_stack _tc_ + (set! (tc-scheme-stack tc) (copy_stack _tgc_ old_stack (& (tc-scheme-stack-size tc)) (+ clength (sizeof ptr)))) @@ -1110,7 +1110,7 @@ (trace-pure (* (ENTRYNONCOMPACTLIVEMASKADDR oldret))) (set! num (ENTRYLIVEMASK oldret))] [else - (RECORD_REMOTE_RANGE _tc_ _ _size_ n_si) + (RECORD_REMOTE_RANGE _tgc_ _ _size_ n_si) (set! num S_G.zero_length_bignum)])]) (let* ([index : iptr (BIGLEN num)]) (while @@ -1216,16 +1216,16 @@ [(-> t_si use_marks) (cond [(! (marked t_si t)) - (mark_typemod_data_object _tc_ t n t_si)])] + (mark_typemod_data_object _tgc_ t n t_si)])] [else (let* ([oldt : ptr t]) - (find_room _tc_ space_data from_g typemod n t) + (find_gc_room _tgc_ space_data from_g typemod n t) (memcpy_aligned (TO_VOIDP t) (TO_VOIDP oldt) n))])] [else - (RECORD_REMOTE_RANGE _tc_ _ _size_ t_si)]))) + (RECORD_REMOTE_RANGE _tgc_ _ _size_ t_si)]))) (set! (reloc-table-code t) _) (set! (code-reloc _) t)]) - (S_record_code_mod tc_in (cast uptr (TO_PTR (& (code-data _ 0)))) (cast uptr (code-length _)))] + (S_record_code_mod (-> _tgc_ tc) (cast uptr (TO_PTR (& (code-data _ 0)))) (cast uptr (code-length _)))] [vfasl-sweep ;; no vfasl_register_pointer, since relink_code can handle it (set! (reloc-table-code t) (cast ptr (ptr_diff _ (-> vfi base_addr)))) @@ -1463,7 +1463,7 @@ [else "void"]) name (case (lookup 'mode config) - [(copy mark sweep sweep-in-old measure) "ptr tc_in, "] + [(copy mark sweep sweep-in-old measure) "thread_gc *tgc, "] [(vfasl-copy vfasl-sweep) "vfasl_info *vfi, "] [else ""]) @@ -1504,12 +1504,12 @@ (case (lookup 'mode config) [(copy) (code-block - "check_triggers(tc_in, si);" + "check_triggers(tgc, si);" (code-block "ptr new_p;" "IGEN tg = TARGET_GENERATION(si);" (body) - "SWEEPCHANGE(tc_in) = SWEEP_CHANGE_PROGRESS;" + "tgc->sweep_change = SWEEP_CHANGE_PROGRESS;" "FWDMARKER(p) = forward_marker;" "FWDADDRESS(p) = new_p;" (and (lookup 'maybe-backreferences? config #f) @@ -1518,10 +1518,10 @@ "return tg;"))] [(mark) (code-block - "check_triggers(tc_in, si);" + "check_triggers(tgc, si);" (ensure-segment-mark-mask "si" "") (body) - "SWEEPCHANGE(tc_in) = SWEEP_CHANGE_PROGRESS;" + "tgc->sweep_change = SWEEP_CHANGE_PROGRESS;" "ADD_BACKREFERENCE(p, si->generation);" "return si->generation;")] [(sweep) @@ -1703,7 +1703,7 @@ [(and preserve-flonum-eq? (eq? 'copy (lookup 'mode config))) (code (copy-statement field config) - "flonum_set_forwarded(tc_in, p, si);" + "flonum_set_forwarded(tgc, p, si);" "FLONUM_FWDADDRESS(p) = new_p;" (statements (cdr l) config))] [else @@ -1853,7 +1853,7 @@ (hashtable-set! (lookup 'used config) 'p_sz #t) (code (format "~a, ~a, p_sz, new_p);" (case mode - [(copy) "find_room(tc_in, p_spc, tg"] + [(copy) "find_gc_room(tgc, p_spc, tg"] [(vfasl-copy) "FIND_ROOM(vfi, p_vspc"]) (as-c 'type (lookup 'basetype config))) (statements (let ([extra (lookup 'copy-extra config #f)]) @@ -1904,7 +1904,7 @@ (unless (null? (cdr l)) (error 'skip-forwarding "not at end")) (code "*dest = new_p;" - "SWEEPCHANGE(tc_in) = SWEEP_CHANGE_PROGRESS;" + "tgc->sweep_change = SWEEP_CHANGE_PROGRESS;" "return tg;")] [else (statements (cdr l) config)])] @@ -2072,7 +2072,7 @@ [(copy) "tg"] [(mark) "TARGET_GENERATION(si)"] [else "target_generation"])] - [`_tc_ "tc_in"] + [`_tgc_ "tgc"] [`_backreferences?_ (if (lookup 'maybe-backreferences? config #f) "BACKREFERENCES_ENABLED" @@ -2242,7 +2242,7 @@ "{ /* measure */" (format " ptr r_p = ~a;" e) " if (!IMMEDIATE(r_p))" - " push_measure(tc_in, r_p);" + " push_measure(tgc, r_p);" "}")) (define (copy-statement field config) @@ -2347,7 +2347,7 @@ " while (seg < end_seg) {" " mark_si = SegInfo(seg);" " g = mark_si->generation;" - " if (!fully_marked_mask[g]) init_fully_marked_mask(tc_in, g);" + " if (!fully_marked_mask[g]) init_fully_marked_mask(tgc, g);" " mark_si->marked_mask = fully_marked_mask[g];" " mark_si->marked_count = bytes_per_segment;" " seg++;" @@ -2440,7 +2440,7 @@ (define (ensure-segment-mark-mask si inset) (code (format "~aif (!~a->marked_mask) {" inset si) - (format "~a init_mask(tc_in, ~a->marked_mask, ~a->generation, 0);" + (format "~a init_mask(tgc, ~a->marked_mask, ~a->generation, 0);" inset si si) (format "~a}" inset))) @@ -2667,7 +2667,7 @@ (parallel? ,parallel?)))) (print-code (generate "object_directly_refers_to_self" `((mode self-test)))) - (print-code (code "static void mark_typemod_data_object(ptr tc_in, ptr p, uptr p_sz, seginfo *si)" + (print-code (code "static void mark_typemod_data_object(thread_gc *tgc, ptr p, uptr p_sz, seginfo *si)" (code-block (ensure-segment-mark-mask "si" "") (mark-statement '(one-bit no-sweep)