avoid a constant folding that can use too much memory

Closes PR 13738

collects/tests/racket/optimize.rktl

@@ -3145,6 +3145,23 @@
     (((dynamic-require ''m 'f) (vector 1)) 0)))
 
 ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Make sure that constant folding doesn't try to
+;; use too much memory, where a run-time limit would
+;; catch the problem:
+
+(module uses-too-much-memory-for-shift racket/base
+  (define c (make-custodian))
+  (custodian-limit-memory c (* 1024 1024 10))
+  (parameterize ([current-custodian c])
+    (sync
+     (thread
+      (lambda ()
+        (with-handlers ([exn:fail:out-of-memory? void])
+          (arithmetic-shift 1 30070458541082)))))))
+(when (eq? '3m (system-type 'gc))
+  (void (dynamic-require ''uses-too-much-memory-for-shift #f)))
+
+;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 
 (report-errs)

src/racket/src/number.c

@@ -4138,6 +4138,9 @@ scheme_bitwise_shift(int argc, Scheme_Object *argv[])
     v = scheme_make_bignum(i);
   }
 
+  if (scheme_current_thread->constant_folding)
+    scheme_signal_error("too big");
+
   return scheme_bignum_shift(v, shift);
 }