fix custodian-limit checking for phantom byte strings

Closes PR 14036

pkgs/racket-pkgs/racket-test/tests/racket/will.rktl

@@ -216,7 +216,19 @@
   (collect-garbage)
   (test #t < (current-memory-use) (+ m (expt 2 28)))
   (test #t < (current-memory-use c) (+ mc (expt 2 28)))
-  (semaphore-post s))
+  (semaphore-post s)
+
+  (let ([done? #f])
+    (sync
+     (let ([c (make-custodian)])
+       (parameterize ([current-custodian c])
+         (thread
+          (lambda ()
+            (custodian-limit-memory c 10000000)
+            (define b (make-phantom-bytes 100))
+            (set-phantom-bytes! b 0)
+            (set! done? #t))))))
+    (test #t values done?)))
 
 ;; ----------------------------------------
 ;; Check that local variables are cleared for space safety

racket/src/racket/gc2/newgc.c

@@ -1529,12 +1529,14 @@ uintptr_t add_no_overflow(uintptr_t a, uintptr_t b)
 
 int GC_allocate_phantom_bytes(intptr_t request_size_bytes)
 {
-#ifdef NEWGC_BTC_ACCOUNT
   NewGC *gc = GC_get_GC();
 
-  if (premaster_or_place_gc(gc)) {
-    if (BTC_single_allocation_limit(gc, request_size_bytes))
-      return 0;
+#ifdef NEWGC_BTC_ACCOUNT
+  if (request_size_bytes > 0) {
+    if (premaster_or_place_gc(gc)) {
+      if (BTC_single_allocation_limit(gc, request_size_bytes))
+        return 0;
+    }
   }
 #endif