JIT: fix potential problem in transition to 64-bit jumps

On x86_64, the JIT compiler initially generates code with 32-bit
jumps, but it switches to 64-bit jumps when so much code is allocated
that it gets spaced out enough. That transition could happen during a
recursive call to the JIT compiler or while one place is in the JIT
and other installs a shared code pointer, in which case a bad jump
could be generated. This problem is unlikely to happen, but it looks
possible.

racket/src/racket/src/jit.c

@@ -3320,6 +3320,7 @@ static int do_generate_closure(mz_jit_state *jitter, void *_data)
       shared_arity_code = jit_adjust_ip(shared_arity_code);
       sjc.shared_arity_check[num_params][has_rest][is_method] = shared_arity_code;
     }
+    CHECK_NESTED_GENERATE();
 
     arity_code = jit_get_ip();
   
@@ -3334,6 +3335,7 @@ static int do_generate_closure(mz_jit_state *jitter, void *_data)
     ((void **)retain_code)[1] = arity_code;
 #endif
     arity_code = jit_adjust_ip(arity_code);
+    CHECK_NESTED_GENERATE();
   }
 
   /* A tail call starts here. Caller must ensure that the stack is big

racket/src/racket/src/jit.h

@@ -1324,6 +1324,16 @@ static int past_limit(mz_jit_state *jitter, const char *file, int line)
 }
 #endif
 
+/* Use CHECK_NESTED_GENERATE() after a nested call to scheme_generate_one()
+   or after getting a shared code pointer that may be generated by another
+   place: */
+#ifdef SET_DEFAULT_LONG_JUMPS
+extern int scheme_check_long_mode(int long_mode);
+# define CHECK_NESTED_GENERATE() if (scheme_check_long_mode(_jitl.long_jumps_default)) return 0;
+#else
+# define CHECK_NESTED_GENERATE() /* empty */
+#endif
+
 void *scheme_generate_one(mz_jit_state *old_jitter, 
 			  Generate_Proc generate,
 			  void *data,

racket/src/racket/src/jitcall.c

@@ -1824,6 +1824,7 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_
             if (nc->code->start_code == scheme_on_demand_jit_code) {
               if (nc->code->arity_code != sjc.in_progress_on_demand_jit_arity_code) {
                 scheme_on_demand_generate_lambda(nc, 0, NULL, 0);
+                CHECK_NESTED_GENERATE();
               }
             }
             if (nc->code->start_code != scheme_on_demand_jit_code) {
@@ -2166,6 +2167,7 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_
 	sjc.shared_tail_code[dp][num_rands] = code;
       }
       code = sjc.shared_tail_code[dp][num_rands];
+      CHECK_NESTED_GENERATE();
       if (direct_self) {
         LOG_IT(("<-self\n"));
 	generate_self_tail_call(rator, jitter, num_rands, code, args_already_in_place, direct_flostack_offset, 
@@ -2231,6 +2233,7 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_
           sjc.shared_non_tail_code[4][num_rands][mo] = code;
         }
         unboxed_code = sjc.shared_non_tail_code[4][num_rands][mo];
+        CHECK_NESTED_GENERATE();
       } else
         unboxed_code = NULL;
 #endif
@@ -2243,6 +2246,7 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_
       }
       LOG_IT(("<-non-tail %d %d %d\n", dp, num_rands, mo));
       code = sjc.shared_non_tail_code[dp][num_rands][mo];
+      CHECK_NESTED_GENERATE();
 
       if (nontail_self) {
         generate_nontail_self_setup(jitter);

racket/src/racket/src/jitstate.c

@@ -162,6 +162,14 @@ static int check_long_mode(uintptr_t low, uintptr_t size)
 
   return 0;
 }
+
+int scheme_check_long_mode(int jitter_long_jumps_default)
+{
+  /* Relying on TSO: if another place provides a shared code pointer, then
+     seeing that pointer means that this place also sees any change
+     to `default-long_jumps` that applies to the pointer. */
+  return (jitter_long_jumps_default != default_long_jumps);
+}
 #endif