JIT: fix inline allocation for large vectors

Cloaes #1800

pkgs/racket-test-core/tests/racket/jitinline.rktl

@@ -914,6 +914,18 @@
 
     ))
 
+;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Make sure that the JIT doesn't try to inline
+;; a vector allocation that is too large
+
+(parameterize ([current-namespace (make-base-namespace)])
+  (let loop ([i 10])
+    ((eval `(lambda (x) (vector x ,@(for/list ([j (in-range i)])
+                                      j))))
+     i)
+    (when (i . < . 100000)
+      (loop (floor (* i #e1.25))))))
+
 ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Check JIT handling of structure-reference sequences

racket/src/racket/src/jitinline.c

@@ -5250,7 +5250,10 @@ static int generate_vector_alloc(mz_jit_state *jitter, Scheme_Object *rator,
     c = 2;
   } else {
     c = app->num_args;
-    if (c)
+    if (c > 256) {
+      /* Too big for inline alloc */
+      return scheme_generate_app(app, NULL, c, c, jitter, 0, 0, 0, 0);
+    } else if (c)
       scheme_generate_app(app, NULL, c, c, jitter, 0, 0, 0, 2);  /* sync'd below */
   }
   CHECK_LIMIT();