GC: chunk mrpotect calls on old-page allocation

Although a block cache is set up to group most page-protection changes
into a single OS call, allocating new old-generation pages was not
covered. Adjust the block cache to group those.

This change has a small effect on performance, but it seems
better to have a few system calls in place of thousands.

racket/src/racket/gc2/block_cache.c

@@ -29,10 +29,13 @@ static int block_cache_chain_stat(GCList *head, int *bcnt);
 #endif
 
 struct block_group;
+
 typedef struct block_desc {
   GCList gclist;
   void *block;
   void *free;
+  unsigned char *protect_map; /* 1 => write protected, 0 => not protected */
+  unsigned char *alloc_map; /* 1 => allocated, 0 => not allocated */
   intptr_t size;
   intptr_t used;
   intptr_t totalcnt;
@@ -41,6 +44,11 @@ typedef struct block_desc {
   int in_queue;
 } block_desc;
 
+#define BD_BLOCK_PTR_TO_POS(p, bd) (((char *)(p) - (char *)((bd)->block)) >> LOG_APAGE_SIZE)
+#define BD_MAP_GET_BIT(map, pos) ((map)[(pos) >> 3] & (1 << ((pos) & 0x7)))
+#define BD_MAP_SET_BIT(map, pos) ((map)[(pos) >> 3] |= (1 << ((pos) & 0x7)))
+#define BD_MAP_UNSET_BIT(map, pos) ((map)[(pos) >> 3] -= (1 << ((pos) & 0x7)))
+
 typedef struct block_group {
   GCList full;
   GCList free;
@@ -89,7 +97,6 @@ static block_desc *bc_alloc_std_block(block_group *bg) {
   void *r = os_alloc_pages(this_block_size);
   block_desc *bd;
   void *ps;
-
   if (!r) return NULL;
 
   ps = align_up_ptr(r, APAGE_SIZE);
@@ -108,6 +115,8 @@ static block_desc *bc_alloc_std_block(block_group *bg) {
   bd->size = this_block_size;
   bd->used = 0;
   bd->group = bg;
+  bd->protect_map = (unsigned char *)ofm_malloc_zero(this_block_size >> (LOG_APAGE_SIZE + 3));
+  bd->alloc_map = (unsigned char *)ofm_malloc_zero(this_block_size >> (LOG_APAGE_SIZE + 3));
   gclist_init(&bd->gclist);
 
   /* printf("ALLOC BLOCK %p-%p size %li %li %li %p\n", bd->block, bd->block + bd->size, bd->size, APAGE_SIZE, bd->size / APAGE_SIZE, bd->free); */
@@ -179,13 +188,43 @@ static void *bc_alloc_std_page(BlockCache *bc, int dirty_ok, int expect_mprotect
     block_desc *bd = gclist_first_item(free_head, block_desc*, gclist);
     pfree_list *fl = bd->free;
     void *p = fl;
+    int pos = BD_BLOCK_PTR_TO_POS(p, bd);
+
+    GC_ASSERT(pos >= 0);
+    GC_ASSERT(pos < (bd->size >> LOG_APAGE_SIZE));
+    
     bd->free = fl->next;
     bd->freecnt--;
 
     *src_block = bd;
+
+    BD_MAP_SET_BIT(bd->alloc_map, pos);
+    
     if (expect_mprotect) {
+      if (BD_MAP_GET_BIT(bd->protect_map, pos)) {
+        /* Unprotect a contiguous range of unallocated pages,
+           in case we need to allocate more, since expanding
+           the range costs much less than multiple unprotect
+           calls. */
+        int start_pos = pos, end_pos = pos + 1;
+        while (start_pos
+               && !BD_MAP_GET_BIT(bd->alloc_map, start_pos)
+               && BD_MAP_GET_BIT(bd->protect_map, start_pos)) {
+          BD_MAP_UNSET_BIT(bd->protect_map, start_pos);
+          --start_pos;
+        }
+        while ((end_pos < (bd->size >> LOG_APAGE_SIZE))
+               && !BD_MAP_GET_BIT(bd->alloc_map, end_pos)
+               && BD_MAP_GET_BIT(bd->protect_map, end_pos)) {
+          BD_MAP_UNSET_BIT(bd->protect_map, end_pos);
+          end_pos++;
+        }
+      
         GC_MP_CNT_INC(mp_alloc_med_big_cnt);
-      os_protect_pages(p, APAGE_SIZE, 1);
+        os_protect_pages((char *)p - ((pos - start_pos) * APAGE_SIZE),
+                         (end_pos - start_pos) * APAGE_SIZE,
+                         1);
+      }
     }
 
     if (!dirty_ok) {
@@ -195,9 +234,9 @@ static void *bc_alloc_std_page(BlockCache *bc, int dirty_ok, int expect_mprotect
         fl->next = 0;
     }
 
+    GC_ASSERT(p >= bd->block);
+    GC_ASSERT(p+APAGE_SIZE <= bd->block + bd->size);
 #if BC_ASSERTS
-    assert(p >= bd->block);
-    assert(p+APAGE_SIZE <= bd->block + bd->size);
     if (!bg->atomic)
     {
       int afub = 0;
@@ -220,6 +259,8 @@ static ssize_t bc_free_std_block(block_desc *b) {
   gclist_del(&b->gclist);
   os_free_pages(b->block, b->size);
   size_diff -= b->size;
+  ofm_free(b->protect_map, b->size >> (LOG_APAGE_SIZE + 3));
+  ofm_free(b->alloc_map, b->size >> (LOG_APAGE_SIZE + 3));
   ofm_free(b, sizeof(block_desc));
   return size_diff;
 }
@@ -252,17 +293,20 @@ static ssize_t block_cache_free_page(BlockCache* bc, void *p, size_t len, int ty
                                      int originated_here) {
   switch(type) {
     case MMU_SMALL_GEN1:
+      GC_ASSERT(*src_block != (char*)~0x0);
       {
         GCList *free_head = &((expect_mprotect ? &bc->non_atomic : &bc->atomic)->free);
         block_desc *b = (block_desc*)(*src_block);
         pfree_list *fl = p;
+        int pos = BD_BLOCK_PTR_TO_POS(p, b);
+        GC_ASSERT(b->group == (expect_mprotect ? &bc->non_atomic : &bc->atomic));
+        GC_ASSERT(pos >= 0);
+        GC_ASSERT(pos < (b->size >> LOG_APAGE_SIZE));
         fl->next = b->free;
         fl->dirty = 1;
         b->free = fl;
-#if BC_ASSERTS
+        GC_ASSERT(BD_MAP_GET_BIT(b->alloc_map, pos));
-        assert(*src_block != (char*)~0x0);
+        BD_MAP_SET_BIT(b->alloc_map, pos);
-        assert(b->group == bg);
-#endif
         gclist_move(&b->gclist, free_head);
         b->freecnt++;
 #if BC_ASSERTS
@@ -283,12 +327,12 @@ static ssize_t block_cache_free_page(BlockCache* bc, void *p, size_t len, int ty
       }
       break;
     default:
+      GC_ASSERT(*src_block == (char*)~0x0);
 #if BC_ASSERTS
       assert(!(find_addr_in_bd(&bc->atomic.full, p, "atomic full") ||
                find_addr_in_bd(&bc->atomic.free, p, "atomic freeblock") ||
                find_addr_in_bd(&bc->non_atomic.full, p, "non_atomic full") ||
                find_addr_in_bd(&bc->non_atomic.free, p, "non_atomic freeblock")));
-      assert(*src_block == (char*)~0x0);
 #endif
       return alloc_cache_free_page(bc->bigBlockCache, p, len, MMU_DIRTY, originated_here);
       break;
@@ -350,6 +394,26 @@ static ssize_t block_cache_flush_freed_pages(BlockCache* bc) {
   return size_diff + alloc_cache_size_diff;
 }
 
+static void block_cache_protect_one_page(BlockCache* bc, void *p, size_t len, int type, int writeable, void **src_block) {
+  switch(type) {
+    case MMU_SMALL_GEN1:
+      GC_ASSERT(len == APAGE_SIZE);
+      {
+        block_desc *b = (block_desc *)*src_block;
+        int pos = BD_BLOCK_PTR_TO_POS(p, b);
+        GC_ASSERT(pos >= 0);
+        GC_ASSERT(pos < (b->size >> LOG_APAGE_SIZE));
+        if (BD_MAP_GET_BIT(b->protect_map, pos)) {
+          BD_MAP_UNSET_BIT(b->protect_map, pos);
+          os_protect_pages(p, len, writeable);
+        }
+      }
+      break;
+  default:
+    os_protect_pages(p, len, writeable);
+  }
+}
+
 static void block_cache_queue_protect_range(BlockCache* bc, void *p, size_t len, int type, int writeable, void **src_block) {
   switch(type) {
     case MMU_SMALL_GEN1:
@@ -358,18 +422,17 @@ static void block_cache_queue_protect_range(BlockCache* bc, void *p, size_t len,
                find_addr_in_bd(&bc->atomic.free, p, "atomic freeblock")));
       assert(find_addr_in_bd(&bc->non_atomic.full, p, "non_atomic full") ||
              find_addr_in_bd(&bc->non_atomic.free, p, "non_atomic freeblock"));
-      assert(*src_block != (char*)~0x0);
 #endif
+      GC_ASSERT(*src_block != (char*)~0x0);
       {
         block_desc *b = (block_desc *)*src_block;
         b->in_queue = 1;
+        memset(b->protect_map, writeable ? 0 : 255, (b->size >> (LOG_APAGE_SIZE + 3)));
       }
       return;
       break;
     default:
-#if BC_ASSERTS 
+      GC_ASSERT(*src_block == (char*)~0x0);
-      assert(*src_block == (char*)~0x0);
-#endif
       page_range_add(bc->page_range, p, len, writeable);
       return;
       break;

racket/src/racket/gc2/newgc.c

@@ -2804,7 +2804,7 @@ static int designate_modified_gc(NewGC *gc, void *p)
 
   if (page) {
     page->mprotected = 0;
-    mmu_write_unprotect_page(gc->mmu, page->addr, real_page_size(page));
+    mmu_write_unprotect_page(gc->mmu, page->addr, real_page_size(page), page_mmu_type(page), &page->mmu_src_block);
     GC_MP_CNT_INC(mp_write_barrier_cnt);
     if (!page->back_pointers)
       set_has_back_pointers(gc, page);
@@ -3623,7 +3623,7 @@ void GC_mark2(const void *const_p, struct NewGC *gc)
           }
           if (work->mprotected) {
             work->mprotected = 0;
-            mmu_write_unprotect_page(gc->mmu, work->addr, APAGE_SIZE);
+            mmu_write_unprotect_page(gc->mmu, work->addr, APAGE_SIZE, page_mmu_type(work), &work->mmu_src_block);
             GC_MP_CNT_INC(mp_mark_cnt);
           }
           newplace = PTR(NUM(work->addr) + work->size);
@@ -4235,7 +4235,7 @@ static void mark_backpointers(NewGC *gc)
       if (work->mprotected) {
         /* expected only if QUEUED_MPROTECT_IS_PROMISCUOUS && AGE_GEN_0_TO_GEN_HALF(gc) */
         work->mprotected = 0;
-        mmu_write_unprotect_page(gc->mmu, work->addr, real_page_size(work));
+        mmu_write_unprotect_page(gc->mmu, work->addr, real_page_size(work), page_mmu_type(work), &work->mmu_src_block);
       }
       work->marked_from = 1;
 
@@ -5465,7 +5465,7 @@ static void free_gc(NewGC *gc)
   for (i = 0; i < NUM_MED_PAGE_SIZES; i++) {
     for (work = gc->med_pages[MED_PAGE_NONATOMIC][i]; work; work = work->next) {
       if (work->mprotected)
-        mmu_write_unprotect_page(gc->mmu, work->addr, real_page_size(work));
+        mmu_write_unprotect_page(gc->mmu, work->addr, real_page_size(work), page_mmu_type(work), &work->mmu_src_block);
     }
   }
 

racket/src/racket/gc2/vm.c

@@ -200,10 +200,14 @@ static int mmu_should_compact_page(MMU *mmu, void **src_block) {
   return 0;
 }
 
-static void mmu_write_unprotect_page(MMU *mmu, void *p, size_t len) {
+static void mmu_write_unprotect_page(MMU *mmu, void *p, size_t len, int type, void **src_block) {
   mmu_assert_os_page_aligned(mmu, (size_t)p);
   mmu_assert_os_page_aligned(mmu, len);
+#ifdef USE_BLOCK_CACHE
+  block_cache_protect_one_page(mmu->block_cache, p, len, type, 1, src_block);
+#else
   os_protect_pages(p, len, 1);
+#endif
 }
 
 static void mmu_queue_protect_range(MMU *mmu, void *p, size_t len, int type, int writeable, void **src_block) {

racket/src/racket/gc2/weak.c

@@ -215,7 +215,7 @@ static void zero_weak_boxes(GCTYPE *gc, int is_late, int force_zero)
         page = pagemap_find_page(gc->page_maps, wb->secondary_erase);
         if (page->mprotected) {
           page->mprotected = 0;
-          mmu_write_unprotect_page(gc->mmu, page->addr, APAGE_SIZE);
+          mmu_write_unprotect_page(gc->mmu, page->addr, APAGE_SIZE, page_mmu_type(page), &page->mmu_src_block);
           GC_MP_CNT_INC(mp_mark_cnt);
         }