From 837c057ca90df248a0a1401e1ff0ec90f285a70a Mon Sep 17 00:00:00 2001 From: Matthew Flatt Date: Thu, 19 Nov 2009 14:22:08 +0000 Subject: [PATCH] remove some indirections in thread-local lookup svn: r16890 --- src/mzscheme/gc2/mem_account.c | 12 ++-- src/mzscheme/gc2/newgc.c | 85 +++++++++++++-------------- src/mzscheme/gc2/newgc.h | 2 + src/mzscheme/include/schthread.h | 4 -- src/mzscheme/src/jit.c | 72 +++++++++-------------- src/mzscheme/src/lightning/i386/asm.h | 1 + 6 files changed, 77 insertions(+), 99 deletions(-) diff --git a/src/mzscheme/gc2/mem_account.c b/src/mzscheme/gc2/mem_account.c index 60ff1af2b5..8d3039170a 100644 --- a/src/mzscheme/gc2/mem_account.c +++ b/src/mzscheme/gc2/mem_account.c @@ -247,7 +247,7 @@ inline static void BTC_memory_account_mark(NewGC *gc, mpage *page, void *ptr) if(info->btc_mark == gc->old_btc_mark) { info->btc_mark = gc->new_btc_mark; account_memory(gc, gc->current_mark_owner, gcBYTES_TO_WORDS(page->size)); - push_ptr(TAG_AS_BIG_PAGE_PTR(ptr)); + push_ptr(gc, TAG_AS_BIG_PAGE_PTR(ptr)); } } else { /* medium page */ @@ -257,7 +257,7 @@ inline static void BTC_memory_account_mark(NewGC *gc, mpage *page, void *ptr) info->btc_mark = gc->new_btc_mark; account_memory(gc, gc->current_mark_owner, info->size); ptr = OBJHEAD_TO_OBJPTR(info); - push_ptr(ptr); + push_ptr(gc, ptr); } } } else { @@ -266,7 +266,7 @@ inline static void BTC_memory_account_mark(NewGC *gc, mpage *page, void *ptr) if(info->btc_mark == gc->old_btc_mark) { info->btc_mark = gc->new_btc_mark; account_memory(gc, gc->current_mark_owner, info->size); - push_ptr(ptr); + push_ptr(gc, ptr); } } } @@ -341,12 +341,12 @@ static void propagate_accounting_marks(NewGC *gc) PageMap pagemap = gc->page_maps; Mark_Proc *mark_table = gc->mark_table; - while(pop_ptr(&p) && !gc->kill_propagation_loop) { + while(pop_ptr(gc, &p) && !gc->kill_propagation_loop) { /* GCDEBUG((DEBUGOUTF, "btc_account: popped off page %p:%p, ptr %p\n", page, page->addr, p)); */ propagate_marks_worker(pagemap, mark_table, p); } if(gc->kill_propagation_loop) - reset_pointer_stack(); + reset_pointer_stack(gc); } inline static void BTC_initialize_mark_table(NewGC *gc) { @@ -428,7 +428,7 @@ static void BTC_do_accounting(NewGC *gc) gc->new_btc_mark = !gc->new_btc_mark; } - clear_stack_pages(); + clear_stack_pages(gc); } inline static void BTC_add_account_hook(int type,void *c1,void *c2,unsigned long b) diff --git a/src/mzscheme/gc2/newgc.c b/src/mzscheme/gc2/newgc.c index 0a89c17480..63c8163c91 100644 --- a/src/mzscheme/gc2/newgc.c +++ b/src/mzscheme/gc2/newgc.c @@ -32,7 +32,6 @@ #include #include #include -#include "pthread.h" #include "platforms.h" #include "../src/schpriv.h" #include "gc2.h" @@ -1544,8 +1543,6 @@ inline static void reset_weak_finalizers(NewGC *gc) #define MARK_STACK_START(ms) ((void **)(void *)&ms[1]) #define MARK_STACK_END(ms) ((void **)((char *)ms + STACK_PART_SIZE)) -THREAD_LOCAL_DECL(static MarkSegment *mark_stack = NULL); - inline static MarkSegment* mark_stack_create_frame() { MarkSegment *mark_frame = (MarkSegment*)ofm_malloc(STACK_PART_SIZE); mark_frame->next = NULL; @@ -1553,41 +1550,41 @@ inline static MarkSegment* mark_stack_create_frame() { return mark_frame; } -inline static void mark_stack_initialize() { +inline static void mark_stack_initialize(NewGC *gc) { /* This happens at the very beginning */ - if(!mark_stack) { - mark_stack = mark_stack_create_frame(); - mark_stack->prev = NULL; + if(!gc->mark_stack) { + gc->mark_stack = mark_stack_create_frame(); + gc->mark_stack->prev = NULL; } } -inline static void push_ptr(void *ptr) +inline static void push_ptr(NewGC *gc, void *ptr) { /* This happens during propagation if we go past the end of this MarkSegment*/ - if(mark_stack->top == MARK_STACK_END(mark_stack)) { + if(gc->mark_stack->top == MARK_STACK_END(gc->mark_stack)) { /* test to see if we already have another stack page ready */ - if(mark_stack->next) { + if(gc->mark_stack->next) { /* we do, so just use it */ - mark_stack = mark_stack->next; - mark_stack->top = MARK_STACK_START(mark_stack); + gc->mark_stack = gc->mark_stack->next; + gc->mark_stack->top = MARK_STACK_START(gc->mark_stack); } else { /* we don't, so we need to allocate one */ - mark_stack->next = mark_stack_create_frame(); - mark_stack->next->prev = mark_stack; - mark_stack = mark_stack->next; + gc->mark_stack->next = mark_stack_create_frame(gc); + gc->mark_stack->next->prev = gc->mark_stack; + gc->mark_stack = gc->mark_stack->next; } } /* at this point, we're guaranteed to be good to push pointers */ - *(mark_stack->top++) = ptr; + *(gc->mark_stack->top++) = ptr; } -inline static int pop_ptr(void **ptr) +inline static int pop_ptr(NewGC *gc, void **ptr) { - if(mark_stack->top == MARK_STACK_START(mark_stack)) { - if(mark_stack->prev) { + if(gc->mark_stack->top == MARK_STACK_START(gc->mark_stack)) { + if(gc->mark_stack->prev) { /* if there is a previous page, go to it */ - mark_stack = mark_stack->prev; + gc->mark_stack = gc->mark_stack->prev; } else { /* if there isn't a previous page, then we've hit the bottom of the stack */ return 0; @@ -1595,41 +1592,41 @@ inline static int pop_ptr(void **ptr) } /* if we get here, we're guaranteed to have data */ - *ptr = *(--mark_stack->top); + *ptr = *(--gc->mark_stack->top); return 1; } -inline static void clear_stack_pages(void) +inline static void clear_stack_pages(NewGC *gc) { - if(mark_stack) { + if(gc->mark_stack) { MarkSegment *temp; MarkSegment *base; int keep = 2; /* go to the head of the list */ - for(; mark_stack->prev; mark_stack = mark_stack->prev) {} + for(; gc->mark_stack->prev; gc->mark_stack = gc->mark_stack->prev) {} /* then go through and clear them out */ - base = mark_stack; - for(; mark_stack; mark_stack = temp) { - temp = mark_stack->next; + base = gc->mark_stack; + for(; gc->mark_stack; gc->mark_stack = temp) { + temp = gc->mark_stack->next; if(keep) { keep--; if (!keep) - mark_stack->next = NULL; + gc->mark_stack->next = NULL; } else - free(mark_stack); + free(gc->mark_stack); } - mark_stack = base; - mark_stack->top = MARK_STACK_START(mark_stack); + gc->mark_stack = base; + gc->mark_stack->top = MARK_STACK_START(gc->mark_stack); } } -inline static void reset_pointer_stack(void) +inline static void reset_pointer_stack(NewGC *gc) { /* go to the head of the list */ - for(; mark_stack->prev; mark_stack = mark_stack->prev) {} + for(; gc->mark_stack->prev; gc->mark_stack = gc->mark_stack->prev) {} /* reset the stack */ - mark_stack->top = MARK_STACK_START(mark_stack); + gc->mark_stack->top = MARK_STACK_START(gc->mark_stack); } static inline void propagate_marks_worker(PageMap pagemap, Mark_Proc *mark_table, void *p); @@ -1769,7 +1766,7 @@ static void NewGC_initialize(NewGC *newgc, NewGC *parentgc) { NewGCMasterInfo_get_next_id(newgc); #endif - mark_stack_initialize(); + mark_stack_initialize(newgc); #ifdef SIXTY_FOUR_BIT_INTEGERS newgc->page_maps = ofm_malloc_zero(PAGEMAP64_LEVEL1_SIZE * sizeof (mpage***)); @@ -2033,7 +2030,7 @@ void GC_mark(const void *const_p) GCDEBUG((DEBUGOUTF, "Marking %p on big page %p\n", p, page)); /* Finally, we want to add this to our mark queue, so we can propagate its pointers */ - push_ptr(TAG_AS_BIG_PAGE_PTR(p)); + push_ptr(gc, TAG_AS_BIG_PAGE_PTR(p)); } else { /* A medium page. */ objhead *info = MED_OBJHEAD(p, page->size); @@ -2046,7 +2043,7 @@ void GC_mark(const void *const_p) p = OBJHEAD_TO_OBJPTR(info); backtrace_new_page_if_needed(gc, page); record_backtrace(page, p); - push_ptr(p); + push_ptr(gc, p); } } else { objhead *ohead = OBJPTR_TO_OBJHEAD(p); @@ -2071,7 +2068,7 @@ void GC_mark(const void *const_p) page->previous_size = PREFIX_SIZE; page->live_size += ohead->size; record_backtrace(page, p); - push_ptr(p); + push_ptr(gc, p); } else GCDEBUG((DEBUGOUTF, "Not marking %p (it's old; %p / %i)\n", p, page, page->previous_size)); } else { @@ -2158,7 +2155,7 @@ void GC_mark(const void *const_p) /* set forwarding pointer */ GCDEBUG((DEBUGOUTF,"Marking %p (moved to %p on page %p)\n", p, newp, work)); *(void**)p = newp; - push_ptr(newp); + push_ptr(gc, newp); } } } @@ -2233,7 +2230,7 @@ static void propagate_marks(NewGC *gc) PageMap pagemap = gc->page_maps; Mark_Proc *mark_table = gc->mark_table; - while(pop_ptr(&p)) { + while(pop_ptr(gc, &p)) { GCDEBUG((DEBUGOUTF, "Popped pointer %p\n", p)); propagate_marks_worker(pagemap, mark_table, p); } @@ -2606,7 +2603,7 @@ static void mark_backpointers(NewGC *gc) if(work->size_class) { /* must be a big page */ work->size_class = 3; - push_ptr(TAG_AS_BIG_PAGE_PTR(BIG_PAGE_TO_OBJECT(work))); + push_ptr(gc, TAG_AS_BIG_PAGE_PTR(BIG_PAGE_TO_OBJECT(work))); } else { if(work->page_type != PAGE_ATOMIC) { void **start = PAGE_START_VSS(work); @@ -2620,7 +2617,7 @@ static void mark_backpointers(NewGC *gc) internal_mark. This is because we need every object in the older heap to be marked out of and noted as marked before we do anything else */ - push_ptr(OBJHEAD_TO_OBJPTR(start)); + push_ptr(gc, OBJHEAD_TO_OBJPTR(start)); } start += info->size; } @@ -2649,7 +2646,7 @@ static void mark_backpointers(NewGC *gc) if(!info->dead) { info->mark = 1; /* This must be a push_ptr (see above) */ - push_ptr(OBJHEAD_TO_OBJPTR(info)); + push_ptr(gc, OBJHEAD_TO_OBJPTR(info)); } start += info->size; } @@ -3210,7 +3207,7 @@ static void garbage_collect(NewGC *gc, int force_full) system clear them later. Better then freeing them, at least. If we're not doing accounting, though, there is no "later" where they'll get removed */ - clear_stack_pages(); + clear_stack_pages(gc); #endif TIME_STEP("marked"); diff --git a/src/mzscheme/gc2/newgc.h b/src/mzscheme/gc2/newgc.h index ec73cc712e..cc5962bde8 100644 --- a/src/mzscheme/gc2/newgc.h +++ b/src/mzscheme/gc2/newgc.h @@ -112,6 +112,8 @@ typedef struct NewGC { struct mpage *med_pages[NUM_MED_PAGE_SIZES]; struct mpage *med_freelist_pages[NUM_MED_PAGE_SIZES]; + MarkSegment *mark_stack; + /* Finalization */ Fnl *run_queue; Fnl *last_in_queue; diff --git a/src/mzscheme/include/schthread.h b/src/mzscheme/include/schthread.h index 0ec0171991..8a481638dc 100644 --- a/src/mzscheme/include/schthread.h +++ b/src/mzscheme/include/schthread.h @@ -41,7 +41,6 @@ extern void scheme_init_os_thread(); #define STACK_COPY_CACHE_SIZE 10 #define BIGNUM_CACHE_SIZE 16 -#define NUM_tl_VARS 14 #define STACK_CACHE_SIZE 32 /* This structure must be 4 words: */ @@ -85,7 +84,6 @@ typedef struct Thread_Local_Variables { struct NewGC *GC_; unsigned long GC_gen0_alloc_page_ptr_; unsigned long GC_gen0_alloc_page_end_; - struct MarkSegment *mark_stack_; void *bignum_cache_[BIGNUM_CACHE_SIZE]; int cache_count_; struct Scheme_Hash_Table *toplevels_ht_; @@ -113,7 +111,6 @@ typedef struct Thread_Local_Variables { struct future_t *current_ft_; void **codetab_tree_; int during_set_; - void *thread_local_pointers_[NUM_tl_VARS]; Stack_Cache_Elem stack_cache_stack_[STACK_CACHE_SIZE]; long stack_cache_stack_pos_; struct Scheme_Object **fixup_runstack_base_; @@ -249,7 +246,6 @@ XFORM_GC_VARIABLE_STACK_THROUGH_THREAD_LOCAL; #define GC_gen0_alloc_page_ptr XOA (scheme_get_thread_local_variables()->GC_gen0_alloc_page_ptr_) #define GC_gen0_alloc_page_end XOA (scheme_get_thread_local_variables()->GC_gen0_alloc_page_end_) #define GC_variable_stack XOA (scheme_get_thread_local_variables()->GC_variable_stack_) -#define mark_stack XOA (scheme_get_thread_local_variables()->mark_stack_) #define bignum_cache XOA (scheme_get_thread_local_variables()->bignum_cache_) #define cache_count XOA (scheme_get_thread_local_variables()->cache_count_) #define toplevels_ht XOA (scheme_get_thread_local_variables()->toplevels_ht_) diff --git a/src/mzscheme/src/jit.c b/src/mzscheme/src/jit.c index 5eb3039e86..5518ccb853 100644 --- a/src/mzscheme/src/jit.c +++ b/src/mzscheme/src/jit.c @@ -277,27 +277,27 @@ void scheme_jit_fill_threadlocal_table(); #endif #ifdef JIT_THREAD_LOCAL -# define tl_MZ_RUNSTACK 0 -# define tl_MZ_RUNSTACK_START 1 -# define tl_GC_gen0_alloc_page_ptr 2 -# define tl_scheme_current_thread 3 -# define tl_scheme_current_cont_mark_pos 4 -# define tl_scheme_current_cont_mark_stack 5 -# define tl_stack_cache_stack_pos 6 -# define tl_retry_alloc_r1 7 -# define tl_fixup_runstack_base 8 -# define tl_fixup_already_in_place 9 -# define tl_double_result 10 -# define tl_save_fp 11 -# define tl_scheme_fuel_counter 12 -# define tl_scheme_jit_stack_boundary 13 - -THREAD_LOCAL_DECL(static void *thread_local_pointers[NUM_tl_VARS]); +# define BOTTOM_VARIABLE GC_variable_stack +# define tl_delta(id) ((unsigned long)&(id) - (unsigned long)&BOTTOM_VARIABLE) +# define tl_MZ_RUNSTACK tl_delta(MZ_RUNSTACK) +# define tl_MZ_RUNSTACK_START tl_delta(MZ_RUNSTACK_START) +# define tl_GC_gen0_alloc_page_ptr tl_delta(GC_gen0_alloc_page_ptr) +# define tl_scheme_current_thread tl_delta(scheme_current_thread) +# define tl_scheme_current_cont_mark_pos tl_delta(scheme_current_cont_mark_pos) +# define tl_scheme_current_cont_mark_stack tl_delta(scheme_current_cont_mark_stack) +# define tl_stack_cache_stack_pos tl_delta(stack_cache_stack_pos) +# define tl_retry_alloc_r1 tl_delta(retry_alloc_r1) +# define tl_fixup_runstack_base tl_delta(fixup_runstack_base) +# define tl_fixup_already_in_place tl_delta(fixup_already_in_place) +# define tl_double_result tl_delta(double_result) +# define tl_save_fp tl_delta(save_fp) +# define tl_scheme_fuel_counter tl_delta(scheme_fuel_counter) +# define tl_scheme_jit_stack_boundary tl_delta(scheme_jit_stack_boundary) #ifdef MZ_XFORM START_XFORM_SKIP; #endif -static void *get_threadlocal_table() { return &thread_local_pointers; } +static void *get_threadlocal_table() { return &BOTTOM_VARIABLE; } #ifdef MZ_XFORM END_XFORM_SKIP; #endif @@ -305,17 +305,18 @@ END_XFORM_SKIP; # ifdef JIT_X86_64 # define JIT_R10 JIT_R(10) # define JIT_R14 JIT_R(14) -# define mz_tl_addr(reg, addr) jit_ldxi_p(reg, JIT_R14, WORDS_TO_BYTES(addr)) +# define mz_tl_addr(reg, addr) LEAQmQr((addr), (JIT_R14), 0, 0, (reg)) # define mz_tl_addr_tmp(tmp_reg, addr) (mz_tl_addr(JIT_R10, addr)) # define mz_tl_addr_untmp(tmp_reg) (void)0 # define mz_tl_tmp_reg(tmp_reg) JIT_R10 # else -# define mz_tl_addr(reg, addr) (mz_get_local_p(reg, JIT_LOCAL4), jit_ldxi_p(reg, reg, WORDS_TO_BYTES(addr))) +# define mz_tl_addr(reg, addr) (mz_get_local_p(reg, JIT_LOCAL4), jit_addi_p(reg, reg, addr)) # define mz_tl_addr_tmp(tmp_reg, addr) (PUSHQr(tmp_reg), mz_tl_addr(tmp_reg, addr)) # define mz_tl_addr_untmp(tmp_reg) POPQr(tmp_reg) # define mz_tl_tmp_reg(tmp_reg) tmp_reg # endif +/* A given tmp_reg doesn't have to be unused; it just has to be distinct from other arguments. */ # define mz_tl_sti_p(addr, reg, tmp_reg) (mz_tl_addr_tmp(tmp_reg, addr), jit_str_p(mz_tl_tmp_reg(tmp_reg), reg), mz_tl_addr_untmp(tmp_reg)) # define mz_tl_sti_l(addr, reg, tmp_reg) (mz_tl_addr_tmp(tmp_reg, addr), jit_str_l(mz_tl_tmp_reg(tmp_reg), reg), mz_tl_addr_untmp(tmp_reg)) # define mz_tl_sti_i(addr, reg, tmp_reg) (mz_tl_addr_tmp(tmp_reg, addr), jit_str_i(mz_tl_tmp_reg(tmp_reg), reg), mz_tl_addr_untmp(tmp_reg)) @@ -1214,6 +1215,13 @@ static void _jit_prolog_again(mz_jit_state *jitter, int n, int ret_addr_reg) #endif } #else +/* From frame pointer, -1 is saved frame pointer, -2 is saved ESI/R12, + and -3 is saved EDI/R13. On entry to a procedure, prolog pushes 4 + since the call (which also pushed), so if the stack was 16-bytes + aligned before the call, it is current stack pointer is 1 word + (either 4 or 8 bytes) below alignment (need to push 3 or 1 words to + re-align). Also, for a call without a prolog, th stack pointer is + 1 word (for the return address) below alignment. */ # define JIT_LOCAL1 -(JIT_WORD_SIZE * 4) # define JIT_LOCAL2 -(JIT_WORD_SIZE * 5) # define mz_set_local_p(x, l) jit_stxi_p((l), JIT_FP, (x)) @@ -2245,11 +2253,6 @@ static void ts_on_demand(void) } #ifdef MZ_PRECISE_GC -static void *get_new_local_memory(void *p, void *p2) -{ - return GC_make_jit_nursery_page(); -} - static void *ts_prepare_retry_alloc(void *p, void *p2) { START_XFORM_SKIP; @@ -10296,27 +10299,6 @@ void *scheme_module_start_start(struct Start_Module_Args *a, Scheme_Object *name /**********************************************************************/ void scheme_jit_fill_threadlocal_table() { -#ifdef JIT_THREAD_LOCAL - thread_local_pointers[tl_MZ_RUNSTACK] = (&MZ_RUNSTACK); - thread_local_pointers[tl_MZ_RUNSTACK_START] = (&MZ_RUNSTACK_START); - thread_local_pointers[tl_scheme_current_thread] = (&scheme_current_thread); - thread_local_pointers[tl_scheme_current_cont_mark_pos] = (&scheme_current_cont_mark_pos); - thread_local_pointers[tl_scheme_current_cont_mark_stack] = (&scheme_current_cont_mark_stack); - thread_local_pointers[tl_stack_cache_stack_pos] = (&stack_cache_stack_pos); -# ifdef CAN_INLINE_ALLOC - thread_local_pointers[tl_GC_gen0_alloc_page_ptr] = (&GC_gen0_alloc_page_ptr); - thread_local_pointers[tl_retry_alloc_r1] = (&retry_alloc_r1); - thread_local_pointers[tl_save_fp] = (&save_fp); -# else -# ifdef INLINE_FP_OPS - thread_local_pointers[tl_double_result] = (&double_result); -# endif -# endif - thread_local_pointers[tl_fixup_runstack_base] = (&fixup_runstack_base); - thread_local_pointers[tl_fixup_already_in_place] = (&fixup_already_in_place); - thread_local_pointers[tl_scheme_fuel_counter] = (void *) (&scheme_fuel_counter); - thread_local_pointers[tl_scheme_jit_stack_boundary] = (void *) (&scheme_jit_stack_boundary); -#endif } /**********************************************************************/ diff --git a/src/mzscheme/src/lightning/i386/asm.h b/src/mzscheme/src/lightning/i386/asm.h index bc5b4c5473..5ccef6c4fc 100644 --- a/src/mzscheme/src/lightning/i386/asm.h +++ b/src/mzscheme/src/lightning/i386/asm.h @@ -627,6 +627,7 @@ typedef _uc jit_insn; #define LAHF_() _O (0x9f ) #define LEALmr(MD, MB, MI, MS, RD) _O_r_X (0x8d ,_r4(RD) ,MD,MB,MI,MS ) #define LEAQmr(MD, MB, MI, MS, RD) _qO_r_X (0x8d ,_r8(RD) ,MD,MB,MI,MS ) +#define LEAQmQr(MD, MB, MI, MS, RD) _qO_r_XB (0x8d ,_r8(RD) ,MD,MB,MI,MS ) #define LEAVE_() _O (0xc9 )