From bcd3d814fd5d75f06665f44e42c63c6684f3c77a Mon Sep 17 00:00:00 2001 From: Matthew Flatt Date: Mon, 14 Sep 2015 05:16:11 -0600 Subject: [PATCH] GC: chunk mrpotect calls on old-page allocation Although a block cache is set up to group most page-protection changes into a single OS call, allocating new old-generation pages was not covered. Adjust the block cache to group those. This change has a small effect on performance, but it seems better to have a few system calls in place of thousands. --- racket/src/racket/gc2/block_cache.c | 91 ++++++++++++++++++++++++----- racket/src/racket/gc2/newgc.c | 8 +-- racket/src/racket/gc2/vm.c | 6 +- racket/src/racket/gc2/weak.c | 2 +- 4 files changed, 87 insertions(+), 20 deletions(-) diff --git a/racket/src/racket/gc2/block_cache.c b/racket/src/racket/gc2/block_cache.c index 9764ce79f2..145d059ab3 100644 --- a/racket/src/racket/gc2/block_cache.c +++ b/racket/src/racket/gc2/block_cache.c @@ -29,10 +29,13 @@ static int block_cache_chain_stat(GCList *head, int *bcnt); #endif struct block_group; + typedef struct block_desc { GCList gclist; void *block; void *free; + unsigned char *protect_map; /* 1 => write protected, 0 => not protected */ + unsigned char *alloc_map; /* 1 => allocated, 0 => not allocated */ intptr_t size; intptr_t used; intptr_t totalcnt; @@ -41,6 +44,11 @@ typedef struct block_desc { int in_queue; } block_desc; +#define BD_BLOCK_PTR_TO_POS(p, bd) (((char *)(p) - (char *)((bd)->block)) >> LOG_APAGE_SIZE) +#define BD_MAP_GET_BIT(map, pos) ((map)[(pos) >> 3] & (1 << ((pos) & 0x7))) +#define BD_MAP_SET_BIT(map, pos) ((map)[(pos) >> 3] |= (1 << ((pos) & 0x7))) +#define BD_MAP_UNSET_BIT(map, pos) ((map)[(pos) >> 3] -= (1 << ((pos) & 0x7))) + typedef struct block_group { GCList full; GCList free; @@ -89,7 +97,6 @@ static block_desc *bc_alloc_std_block(block_group *bg) { void *r = os_alloc_pages(this_block_size); block_desc *bd; void *ps; - if (!r) return NULL; ps = align_up_ptr(r, APAGE_SIZE); @@ -108,6 +115,8 @@ static block_desc *bc_alloc_std_block(block_group *bg) { bd->size = this_block_size; bd->used = 0; bd->group = bg; + bd->protect_map = (unsigned char *)ofm_malloc_zero(this_block_size >> (LOG_APAGE_SIZE + 3)); + bd->alloc_map = (unsigned char *)ofm_malloc_zero(this_block_size >> (LOG_APAGE_SIZE + 3)); gclist_init(&bd->gclist); /* printf("ALLOC BLOCK %p-%p size %li %li %li %p\n", bd->block, bd->block + bd->size, bd->size, APAGE_SIZE, bd->size / APAGE_SIZE, bd->free); */ @@ -179,13 +188,43 @@ static void *bc_alloc_std_page(BlockCache *bc, int dirty_ok, int expect_mprotect block_desc *bd = gclist_first_item(free_head, block_desc*, gclist); pfree_list *fl = bd->free; void *p = fl; + int pos = BD_BLOCK_PTR_TO_POS(p, bd); + + GC_ASSERT(pos >= 0); + GC_ASSERT(pos < (bd->size >> LOG_APAGE_SIZE)); + bd->free = fl->next; bd->freecnt--; *src_block = bd; + + BD_MAP_SET_BIT(bd->alloc_map, pos); + if (expect_mprotect) { - GC_MP_CNT_INC(mp_alloc_med_big_cnt); - os_protect_pages(p, APAGE_SIZE, 1); + if (BD_MAP_GET_BIT(bd->protect_map, pos)) { + /* Unprotect a contiguous range of unallocated pages, + in case we need to allocate more, since expanding + the range costs much less than multiple unprotect + calls. */ + int start_pos = pos, end_pos = pos + 1; + while (start_pos + && !BD_MAP_GET_BIT(bd->alloc_map, start_pos) + && BD_MAP_GET_BIT(bd->protect_map, start_pos)) { + BD_MAP_UNSET_BIT(bd->protect_map, start_pos); + --start_pos; + } + while ((end_pos < (bd->size >> LOG_APAGE_SIZE)) + && !BD_MAP_GET_BIT(bd->alloc_map, end_pos) + && BD_MAP_GET_BIT(bd->protect_map, end_pos)) { + BD_MAP_UNSET_BIT(bd->protect_map, end_pos); + end_pos++; + } + + GC_MP_CNT_INC(mp_alloc_med_big_cnt); + os_protect_pages((char *)p - ((pos - start_pos) * APAGE_SIZE), + (end_pos - start_pos) * APAGE_SIZE, + 1); + } } if (!dirty_ok) { @@ -195,9 +234,9 @@ static void *bc_alloc_std_page(BlockCache *bc, int dirty_ok, int expect_mprotect fl->next = 0; } + GC_ASSERT(p >= bd->block); + GC_ASSERT(p+APAGE_SIZE <= bd->block + bd->size); #if BC_ASSERTS - assert(p >= bd->block); - assert(p+APAGE_SIZE <= bd->block + bd->size); if (!bg->atomic) { int afub = 0; @@ -220,6 +259,8 @@ static ssize_t bc_free_std_block(block_desc *b) { gclist_del(&b->gclist); os_free_pages(b->block, b->size); size_diff -= b->size; + ofm_free(b->protect_map, b->size >> (LOG_APAGE_SIZE + 3)); + ofm_free(b->alloc_map, b->size >> (LOG_APAGE_SIZE + 3)); ofm_free(b, sizeof(block_desc)); return size_diff; } @@ -252,17 +293,20 @@ static ssize_t block_cache_free_page(BlockCache* bc, void *p, size_t len, int ty int originated_here) { switch(type) { case MMU_SMALL_GEN1: + GC_ASSERT(*src_block != (char*)~0x0); { GCList *free_head = &((expect_mprotect ? &bc->non_atomic : &bc->atomic)->free); block_desc *b = (block_desc*)(*src_block); pfree_list *fl = p; + int pos = BD_BLOCK_PTR_TO_POS(p, b); + GC_ASSERT(b->group == (expect_mprotect ? &bc->non_atomic : &bc->atomic)); + GC_ASSERT(pos >= 0); + GC_ASSERT(pos < (b->size >> LOG_APAGE_SIZE)); fl->next = b->free; fl->dirty = 1; b->free = fl; -#if BC_ASSERTS - assert(*src_block != (char*)~0x0); - assert(b->group == bg); -#endif + GC_ASSERT(BD_MAP_GET_BIT(b->alloc_map, pos)); + BD_MAP_SET_BIT(b->alloc_map, pos); gclist_move(&b->gclist, free_head); b->freecnt++; #if BC_ASSERTS @@ -283,12 +327,12 @@ static ssize_t block_cache_free_page(BlockCache* bc, void *p, size_t len, int ty } break; default: + GC_ASSERT(*src_block == (char*)~0x0); #if BC_ASSERTS assert(!(find_addr_in_bd(&bc->atomic.full, p, "atomic full") || find_addr_in_bd(&bc->atomic.free, p, "atomic freeblock") || find_addr_in_bd(&bc->non_atomic.full, p, "non_atomic full") || find_addr_in_bd(&bc->non_atomic.free, p, "non_atomic freeblock"))); - assert(*src_block == (char*)~0x0); #endif return alloc_cache_free_page(bc->bigBlockCache, p, len, MMU_DIRTY, originated_here); break; @@ -350,6 +394,26 @@ static ssize_t block_cache_flush_freed_pages(BlockCache* bc) { return size_diff + alloc_cache_size_diff; } +static void block_cache_protect_one_page(BlockCache* bc, void *p, size_t len, int type, int writeable, void **src_block) { + switch(type) { + case MMU_SMALL_GEN1: + GC_ASSERT(len == APAGE_SIZE); + { + block_desc *b = (block_desc *)*src_block; + int pos = BD_BLOCK_PTR_TO_POS(p, b); + GC_ASSERT(pos >= 0); + GC_ASSERT(pos < (b->size >> LOG_APAGE_SIZE)); + if (BD_MAP_GET_BIT(b->protect_map, pos)) { + BD_MAP_UNSET_BIT(b->protect_map, pos); + os_protect_pages(p, len, writeable); + } + } + break; + default: + os_protect_pages(p, len, writeable); + } +} + static void block_cache_queue_protect_range(BlockCache* bc, void *p, size_t len, int type, int writeable, void **src_block) { switch(type) { case MMU_SMALL_GEN1: @@ -358,18 +422,17 @@ static void block_cache_queue_protect_range(BlockCache* bc, void *p, size_t len, find_addr_in_bd(&bc->atomic.free, p, "atomic freeblock"))); assert(find_addr_in_bd(&bc->non_atomic.full, p, "non_atomic full") || find_addr_in_bd(&bc->non_atomic.free, p, "non_atomic freeblock")); - assert(*src_block != (char*)~0x0); #endif + GC_ASSERT(*src_block != (char*)~0x0); { block_desc *b = (block_desc *)*src_block; b->in_queue = 1; + memset(b->protect_map, writeable ? 0 : 255, (b->size >> (LOG_APAGE_SIZE + 3))); } return; break; default: -#if BC_ASSERTS - assert(*src_block == (char*)~0x0); -#endif + GC_ASSERT(*src_block == (char*)~0x0); page_range_add(bc->page_range, p, len, writeable); return; break; diff --git a/racket/src/racket/gc2/newgc.c b/racket/src/racket/gc2/newgc.c index e049188185..e362f3187b 100644 --- a/racket/src/racket/gc2/newgc.c +++ b/racket/src/racket/gc2/newgc.c @@ -2804,7 +2804,7 @@ static int designate_modified_gc(NewGC *gc, void *p) if (page) { page->mprotected = 0; - mmu_write_unprotect_page(gc->mmu, page->addr, real_page_size(page)); + mmu_write_unprotect_page(gc->mmu, page->addr, real_page_size(page), page_mmu_type(page), &page->mmu_src_block); GC_MP_CNT_INC(mp_write_barrier_cnt); if (!page->back_pointers) set_has_back_pointers(gc, page); @@ -3623,7 +3623,7 @@ void GC_mark2(const void *const_p, struct NewGC *gc) } if (work->mprotected) { work->mprotected = 0; - mmu_write_unprotect_page(gc->mmu, work->addr, APAGE_SIZE); + mmu_write_unprotect_page(gc->mmu, work->addr, APAGE_SIZE, page_mmu_type(work), &work->mmu_src_block); GC_MP_CNT_INC(mp_mark_cnt); } newplace = PTR(NUM(work->addr) + work->size); @@ -4235,7 +4235,7 @@ static void mark_backpointers(NewGC *gc) if (work->mprotected) { /* expected only if QUEUED_MPROTECT_IS_PROMISCUOUS && AGE_GEN_0_TO_GEN_HALF(gc) */ work->mprotected = 0; - mmu_write_unprotect_page(gc->mmu, work->addr, real_page_size(work)); + mmu_write_unprotect_page(gc->mmu, work->addr, real_page_size(work), page_mmu_type(work), &work->mmu_src_block); } work->marked_from = 1; @@ -5465,7 +5465,7 @@ static void free_gc(NewGC *gc) for (i = 0; i < NUM_MED_PAGE_SIZES; i++) { for (work = gc->med_pages[MED_PAGE_NONATOMIC][i]; work; work = work->next) { if (work->mprotected) - mmu_write_unprotect_page(gc->mmu, work->addr, real_page_size(work)); + mmu_write_unprotect_page(gc->mmu, work->addr, real_page_size(work), page_mmu_type(work), &work->mmu_src_block); } } diff --git a/racket/src/racket/gc2/vm.c b/racket/src/racket/gc2/vm.c index 6e1a0077b3..92d22f7e40 100644 --- a/racket/src/racket/gc2/vm.c +++ b/racket/src/racket/gc2/vm.c @@ -200,10 +200,14 @@ static int mmu_should_compact_page(MMU *mmu, void **src_block) { return 0; } -static void mmu_write_unprotect_page(MMU *mmu, void *p, size_t len) { +static void mmu_write_unprotect_page(MMU *mmu, void *p, size_t len, int type, void **src_block) { mmu_assert_os_page_aligned(mmu, (size_t)p); mmu_assert_os_page_aligned(mmu, len); +#ifdef USE_BLOCK_CACHE + block_cache_protect_one_page(mmu->block_cache, p, len, type, 1, src_block); +#else os_protect_pages(p, len, 1); +#endif } static void mmu_queue_protect_range(MMU *mmu, void *p, size_t len, int type, int writeable, void **src_block) { diff --git a/racket/src/racket/gc2/weak.c b/racket/src/racket/gc2/weak.c index f6954989a9..91317889e4 100644 --- a/racket/src/racket/gc2/weak.c +++ b/racket/src/racket/gc2/weak.c @@ -215,7 +215,7 @@ static void zero_weak_boxes(GCTYPE *gc, int is_late, int force_zero) page = pagemap_find_page(gc->page_maps, wb->secondary_erase); if (page->mprotected) { page->mprotected = 0; - mmu_write_unprotect_page(gc->mmu, page->addr, APAGE_SIZE); + mmu_write_unprotect_page(gc->mmu, page->addr, APAGE_SIZE, page_mmu_type(page), &page->mmu_src_block); GC_MP_CNT_INC(mp_mark_cnt); }