diff --git a/src/racket/src/jit.h b/src/racket/src/jit.h index 3d5663b602..0ea56010f5 100644 --- a/src/racket/src/jit.h +++ b/src/racket/src/jit.h @@ -561,6 +561,18 @@ int check_location; #define mz_retain(x) scheme_mz_retain_it(jitter, x) #define mz_remap(x) scheme_mz_remap_it(jitter, x) +#ifdef jit_bxnei_s +# define mz_bnei_t(label, reg, stype, scratch_reg) jit_bxnei_s(label, reg, stype) +# define mz_beqi_t(label, reg, stype, scratch_reg) jit_bxeqi_s(label, reg, stype) +#else +# define mz_bnei_t(label, reg, stype, scratch_reg) \ + (jit_ldxi_s(scratch_reg, reg, &((Scheme_Object *)0x0)->type), \ + jit_bnei_i(label, scratch_reg, stype)) +# define mz_beqi_t(label, reg, stype, scratch_reg) \ + (jit_ldxi_s(scratch_reg, reg, &((Scheme_Object *)0x0)->type), \ + jit_beqi_i(label, scratch_reg, stype)) +#endif + /* Stack alignment, fixed up by mz_push_locals(): - On PPC, jit_prolog() generates an aligned stack. It also leaves room for 3 locals. @@ -1131,13 +1143,15 @@ int scheme_generate_arith(mz_jit_state *jitter, Scheme_Object *rator, Scheme_Obj /* jitcall */ /**********************************************************************/ +typedef struct jit_direct_arg jit_direct_arg; + void *scheme_generate_shared_call(int num_rands, mz_jit_state *old_jitter, int multi_ok, int is_tail, int direct_prim, int direct_native, int nontail_self); void scheme_ensure_retry_available(mz_jit_state *jitter, int multi_ok); int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_rands, mz_jit_state *jitter, int is_tail, int multi_ok, int no_call); int scheme_generate_tail_call(mz_jit_state *jitter, int num_rands, int direct_native, int need_set_rs, - int is_inline, void *direct_to_code); + int is_inline, Scheme_Native_Closure *direct_to_code, jit_direct_arg *direct_arg); int scheme_generate_non_tail_call(mz_jit_state *jitter, int num_rands, int direct_native, int need_set_rs, int multi_ok, int nontail_self, int pop_and_jump, int is_inlined); int scheme_generate_finish_tail_call(mz_jit_state *jitter, int direct_native); diff --git a/src/racket/src/jitcall.c b/src/racket/src/jitcall.c index 663af0dbcd..4912e6e4e7 100644 --- a/src/racket/src/jitcall.c +++ b/src/racket/src/jitcall.c @@ -29,6 +29,11 @@ int scheme_direct_call_count, scheme_indirect_call_count; +struct jit_direct_arg { + int gen; + int reg; +}; + THREAD_LOCAL_DECL(static Scheme_Object **fixup_runstack_base); THREAD_LOCAL_DECL(static int fixup_already_in_place); @@ -69,8 +74,7 @@ static jit_insn *generate_proc_struct_retry(mz_jit_state *jitter, int num_rands, /* JIT_R1 now has the wrapped procedure */ refz4 = jit_bmsi_i(jit_forward(), JIT_R1, 0x1); - jit_ldr_s(JIT_R2, JIT_R1); - refz2 = jit_bnei_i(jit_forward(), JIT_R2, scheme_native_closure_type); + refz2 = mz_bnei_t(jit_forward(), JIT_R1, scheme_native_closure_type, JIT_R2); CHECK_LIMIT(); /* It's a native closure, but we can't just jump to it, in case @@ -248,14 +252,18 @@ static int generate_direct_prim_tail_call(mz_jit_state *jitter, int num_rands) return 1; } +#define NUM_AVAIL_DIRECT_ARG_REGS 3 +static const int direct_arg_regs[] = { JIT_V1, JIT_R1, JIT_R0 }; + int scheme_generate_tail_call(mz_jit_state *jitter, int num_rands, int direct_native, int need_set_rs, - int is_inline, void *direct_to_code) -/* Proc is in V1, args are at RUNSTACK. + int is_inline, Scheme_Native_Closure *direct_to_code, jit_direct_arg *direct_args) +/* Proc is in V1 unless direct_to_code, args are at RUNSTACK. If num_rands < 0, then argc is in LOCAL2 and arguments are already below RUNSTACK_BASE. If direct_native == 2, then some arguments are already in place (shallower in the runstack - than the arguments to move). */ + than the arguments to move). + If direct_args, then R0, R1, V1 hold arguments. */ { - int i; + int i, r2_has_runstack = 0; GC_CAN_IGNORE jit_insn *refagain, *ref, *ref2, *ref4, *ref5; __START_SHORT_JUMPS__(num_rands < 100); @@ -263,8 +271,9 @@ int scheme_generate_tail_call(mz_jit_state *jitter, int num_rands, int direct_na /* First, try fast direct jump to native code: */ if (!direct_native) { ref = jit_bmsi_ul(jit_forward(), JIT_V1, 0x1); - jit_ldr_s(JIT_R1, JIT_V1); + jit_ldxi_s(JIT_R1, JIT_V1, &((Scheme_Object *)0x0)->type); ref2 = jit_bnei_i(jit_forward(), JIT_R1, scheme_native_closure_type); + /* code at ref2 uses JIT_R1 */ CHECK_LIMIT(); } else { ref = ref2 = NULL; @@ -284,7 +293,7 @@ int scheme_generate_tail_call(mz_jit_state *jitter, int num_rands, int direct_na ref4 = NULL; /* Fast jump ok (proc will check argc). - At this point, V1 = closure and R0 = code. */ + At this point, V1 = closure (unless direct_to_code) and R0 = code. */ /* Check for thread swap: */ if (!direct_to_code) { @@ -306,11 +315,17 @@ int scheme_generate_tail_call(mz_jit_state *jitter, int num_rands, int direct_na jit_subi_p(JIT_R2, JIT_RUNSTACK_BASE_OR_ALT(JIT_R2), WORDS_TO_BYTES(num_rands)); CHECK_RUNSTACK_OVERFLOW(); for (i = num_rands; i--; ) { - jit_ldxi_p(JIT_R1, JIT_RUNSTACK, WORDS_TO_BYTES(i)); - jit_stxi_p(WORDS_TO_BYTES(i), JIT_R2, JIT_R1); + if (direct_args) { + int reg = direct_args[i].reg; + jit_stxi_p(WORDS_TO_BYTES(i), JIT_R2, reg); + } else { + jit_ldxi_p(JIT_R1, JIT_RUNSTACK, WORDS_TO_BYTES(i)); + jit_stxi_p(WORDS_TO_BYTES(i), JIT_R2, JIT_R1); + } CHECK_LIMIT(); } jit_movr_p(JIT_RUNSTACK, JIT_R2); + r2_has_runstack = 1; } else { #ifdef JIT_RUNSTACK_BASE jit_movr_p(JIT_RUNSTACK, JIT_RUNSTACK_BASE); @@ -322,25 +337,25 @@ int scheme_generate_tail_call(mz_jit_state *jitter, int num_rands, int direct_na mz_get_local_p(JIT_R1, JIT_LOCAL2); jit_lshi_l(JIT_R1, JIT_R1, JIT_LOG_WORD_SIZE); jit_subr_p(JIT_RUNSTACK, JIT_RUNSTACK, JIT_R1); + r2_has_runstack = 0; CHECK_RUNSTACK_OVERFLOW(); } } else { /* Variable argc (in LOCAL2): arguments are already in place. */ } - /* RUNSTACK, RUNSTACK_BASE, V1, and R0 are ready */ + /* RUNSTACK, RUNSTACK_BASE, V1 (unless direct_to_code), and R0 are ready */ /* Extract function and data: */ if (!direct_to_code) { jit_movr_p(JIT_R2, JIT_V1); + r2_has_runstack = 0; if (direct_native) { jit_ldxi_p(JIT_V1, JIT_R0, &((Scheme_Native_Closure_Data *)0x0)->u.tail_code); } else { jit_ldxi_p(JIT_V1, JIT_R0, &((Scheme_Native_Closure_Data *)0x0)->arity_code); } jit_movr_p(JIT_R0, JIT_R2); - } else { - jit_movr_p(JIT_R0, JIT_V1); } /* Set up arguments; JIT_RUNSTACK and JIT_RUNSTACK_BASE must also be ready */ if (num_rands >= 0) { @@ -348,18 +363,30 @@ int scheme_generate_tail_call(mz_jit_state *jitter, int num_rands, int direct_na if (direct_native > 1) { /* => some_args_already_in_place */ mz_get_local_p(JIT_R2, JIT_LOCAL2); jit_addr_i(JIT_R1, JIT_R1, JIT_R2); + r2_has_runstack = 0; } } else { mz_get_local_p(JIT_R1, JIT_LOCAL2); } - jit_movr_p(JIT_R2, JIT_RUNSTACK); + if (!r2_has_runstack) + jit_movr_p(JIT_R2, JIT_RUNSTACK); if (need_set_rs && !direct_to_code) { /* In case arity check fails, need to update runstack now: */ JIT_UPDATE_THREAD_RSPTR(); } if (direct_native && direct_to_code) { + int retptr; __END_SHORT_JUMPS__(num_rands < 100); - (void)jit_jmpi(direct_to_code); + /* load closure pointer into R0: */ + retptr = mz_retain(direct_to_code); +#ifdef JIT_PRECISE_GC + if (retptr) + scheme_mz_load_retained(jitter, JIT_R0, retptr); + else +#endif + (void)jit_patchable_movi_p(JIT_R0, direct_to_code); + /* jump directly: */ + (void)jit_jmpi(direct_to_code->code->u.tail_code); /* no slow path in this mode */ return 1; } @@ -370,6 +397,7 @@ int scheme_generate_tail_call(mz_jit_state *jitter, int num_rands, int direct_na if (!direct_native && !is_inline && (num_rands >= 0)) { /* Handle simple applicable struct: */ mz_patch_branch(ref2); + /* uses JIT_R1: */ ref2 = generate_proc_struct_retry(jitter, num_rands, refagain); CHECK_LIMIT(); } @@ -598,8 +626,9 @@ int scheme_generate_non_tail_call(mz_jit_state *jitter, int num_rands, int direc /* Check for inlined native type */ if (!direct_native) { ref = jit_bmsi_ul(jit_forward(), JIT_V1, 0x1); - jit_ldr_s(JIT_R1, JIT_V1); + jit_ldxi_s(JIT_R1, JIT_V1, &((Scheme_Object *)0x0)->type); ref2 = jit_bnei_i(jit_forward(), JIT_R1, scheme_native_closure_type); + /* code at ref2 uses JIT_R1 */ CHECK_LIMIT(); } else { ref = ref2 = NULL; @@ -803,6 +832,7 @@ int scheme_generate_non_tail_call(mz_jit_state *jitter, int num_rands, int direc /* Check for simple applicable struct wrapper */ if (!is_inlined && (num_rands >= 0)) { mz_patch_branch(ref2); + /* uses JIT_R1 */ ref2 = generate_proc_struct_retry(jitter, num_rands, refagain); CHECK_LIMIT(); } @@ -1112,7 +1142,8 @@ static int do_generate_shared_call(mz_jit_state *jitter, void *_data) if (data->direct_prim) ok = generate_direct_prim_tail_call(jitter, data->num_rands); else - ok = scheme_generate_tail_call(jitter, data->num_rands, data->direct_native, 1, 0, NULL); + ok = scheme_generate_tail_call(jitter, data->num_rands, data->direct_native, 1, 0, + NULL, NULL); scheme_jit_register_helper_func(jitter, code); @@ -1211,6 +1242,75 @@ static int can_direct_native(Scheme_Object *p, int num_rands, intptr_t *extract_ return 0; } + +static jit_direct_arg *check_special_direct_args(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_rands, + int args_already_in_place) +{ + jit_direct_arg *inline_direct_args = NULL; + Scheme_Object *v; + int reg_to_pos[NUM_AVAIL_DIRECT_ARG_REGS]; + int n = 0, j, pos, i; + + return NULL; + + for (j = 0; j < NUM_AVAIL_DIRECT_ARG_REGS; j++) { + reg_to_pos[j] = 0; + } + + for (i = 0; i < num_rands; i++) { + v = (alt_rands + ? alt_rands[i+1+args_already_in_place] + : app->args[i+1+args_already_in_place]); + if (SAME_TYPE(SCHEME_TYPE(v), scheme_local_type) + && !SCHEME_GET_LOCAL_FLAGS(v)) { + pos = SCHEME_LOCAL_POS(v); + for (j = 0; j < n; j++) { + if (reg_to_pos[j] == pos) + break; + } + if (j >= n) { + if (n >= NUM_AVAIL_DIRECT_ARG_REGS) + break; + reg_to_pos[n++] = pos; + } + } else + break; + } + + if (i < num_rands) + return NULL; + + /* We hit the special case! */ + inline_direct_args = MALLOC_N_ATOMIC(jit_direct_arg, num_rands); + + n = 0; + for (j = 0; j < NUM_AVAIL_DIRECT_ARG_REGS; j++) { + reg_to_pos[j] = 0; + } + + for (i = 0; i < num_rands; i++) { + v = (alt_rands + ? alt_rands[i+1+args_already_in_place] + : app->args[i+1+args_already_in_place]); + pos = SCHEME_LOCAL_POS(v); + for (j = 0; j < n; j++) { + if (reg_to_pos[j] == pos) { + inline_direct_args[i].gen = 0; + inline_direct_args[i].reg = direct_arg_regs[j]; + break; + } + } + if (j >= n) { + reg_to_pos[n] = pos; + inline_direct_args[i].gen = 1; + inline_direct_args[i].reg = direct_arg_regs[n]; + n++; + } + } + + return inline_direct_args; +} + int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_rands, mz_jit_state *jitter, int is_tail, int multi_ok, int no_call) /* de-sync'd ok @@ -1219,7 +1319,8 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_ { int i, offset, need_safety = 0, apply_to_list = 0; int direct_prim = 0, need_non_tail = 0, direct_native = 0, direct_self = 0, nontail_self = 0; - void *inline_direct_native = NULL; + Scheme_Native_Closure *inline_direct_native = NULL; + jit_direct_arg *inline_direct_args = NULL; int proc_already_in_place = 0; Scheme_Object *rator, *v, *arg; int reorder_ok = 0; @@ -1329,7 +1430,8 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_ if (nc->code->code != scheme_on_demand_jit_code) { if (nc->code->max_let_depth > jitter->max_tail_depth) jitter->max_tail_depth = nc->code->max_let_depth; - inline_direct_native = nc->code->u.tail_code; + + inline_direct_native = nc; } } } @@ -1382,8 +1484,16 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_ } } + if (inline_direct_native) { + /* Look for very special case where arguments are so simple + that we can move them directly into a couple of registers. */ + inline_direct_args = check_special_direct_args(app, alt_rands, num_rands, args_already_in_place); + } + if (num_rands) { - if (!direct_prim || (num_rands > 1) || (no_call == 2)) { + if (inline_direct_args) { + mz_runstack_skipped(jitter, num_rands); + } else if (!direct_prim || (num_rands > 1) || (no_call == 2)) { mz_rs_dec(num_rands); need_safety = num_rands; CHECK_RUNSTACK_OVERFLOW(); @@ -1482,7 +1592,11 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_ } } else #endif - scheme_generate_non_tail(arg, jitter, 0, !need_non_tail, 0); /* sync'd below */ + if (inline_direct_args) { + if (inline_direct_args[i].gen) + scheme_generate(arg, jitter, 0, 0, 0, inline_direct_args[i].reg, NULL); + } else + scheme_generate_non_tail(arg, jitter, 0, !need_non_tail, 0); /* sync'd below */ RESUME_JIT_DATA(); CHECK_LIMIT(); @@ -1491,7 +1605,8 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_ mz_rs_ldxi(JIT_V1, i + offset); } if ((!direct_prim || (num_rands > 1) || (no_call == 2)) - && (!direct_self || !is_tail || no_call || (i + 1 < num_rands))) { + && (!direct_self || !is_tail || no_call || (i + 1 < num_rands)) + && !inline_direct_args) { int r0; r0 = (mz_CURRENT_R0_STATUS_VALID() ? mz_CURRENT_R0_STATUS() : -1); mz_rs_stxi(i + offset, JIT_R0); @@ -1519,7 +1634,7 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_ } } - if (reorder_ok) { + if (reorder_ok && !inline_direct_native) { if ((no_call < 2) && !apply_to_list) { scheme_generate(rator, jitter, 0, 0, 0, JIT_V1, NULL); /* sync'd below, or not */ } @@ -1556,7 +1671,8 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_ jit_movi_l(JIT_R2, args_already_in_place); mz_set_local_p(JIT_R2, JIT_LOCAL2); } - scheme_generate_tail_call(jitter, num_rands, direct_native, jitter->need_set_rs, 1, NULL); + scheme_generate_tail_call(jitter, num_rands, direct_native, jitter->need_set_rs, 1, + NULL, NULL); } } else { if (direct_prim) @@ -1591,7 +1707,7 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_ mz_set_local_p(JIT_R2, JIT_LOCAL2); } scheme_generate_tail_call(jitter, num_rands, direct_native, jitter->need_set_rs, 1, - inline_direct_native); + inline_direct_native, inline_direct_args); CHECK_LIMIT(); } else { scheme_mz_flostack_restore(jitter, 0, 0, 1, 1); diff --git a/src/racket/src/jitcommon.c b/src/racket/src/jitcommon.c index c982324833..f99ea4e108 100644 --- a/src/racket/src/jitcommon.c +++ b/src/racket/src/jitcommon.c @@ -483,8 +483,7 @@ static int common1b(mz_jit_state *jitter, void *_data) /* Check for chaperone: */ ref2 = jit_bmsi_ul(jit_forward(), JIT_R0, 0x1); - jit_ldxi_s(JIT_R1, JIT_R0, &((Scheme_Object *)0x0)->type); - ref = jit_bnei_i(jit_forward(), JIT_R1, scheme_chaperone_type); + ref = mz_bnei_t(jit_forward(), JIT_R0, scheme_chaperone_type, JIT_R1); jit_ldxi_p(JIT_R0, JIT_R0, (intptr_t)&((Scheme_Chaperone *)0x0)->val); mz_epilog(JIT_R1); /* return after unwrapping */ CHECK_LIMIT(); @@ -1044,8 +1043,7 @@ static int common3(mz_jit_state *jitter, void *_data) (void)jit_bmci_ul(reffail, JIT_R1, 0x1); (void)jit_blei_l(reffail, JIT_R1, 0x0); } - jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type); - (void)jit_bnei_i(reffail, JIT_R2, ty); + (void)mz_bnei_t(reffail, JIT_R0, ty, JIT_R2); if (iii) { jit_ldxi_s(JIT_R2, JIT_R0, &(MZ_OPT_HASH_KEY((Scheme_Inclhash_Object *)0x0))); (void)jit_bmsi_ul(reffail, JIT_R2, 0x1); @@ -1251,8 +1249,7 @@ static int common4(mz_jit_state *jitter, void *_data) /* It's not a fixnum... */ mz_patch_branch(ref); - jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type); - (void)jit_bnei_i(reffail, JIT_R2, scheme_stx_type); + (void)mz_bnei_t(reffail, JIT_R0, scheme_stx_type, JIT_R2); /* It's a syntax object... needs to propagate? */ jit_ldxi_l(JIT_R2, JIT_R0, &((Scheme_Stx *)0x0)->u.lazy_prefix); @@ -1363,8 +1360,7 @@ static int common4(mz_jit_state *jitter, void *_data) /* Continue trying fast path: check proc */ mz_patch_branch(ref); - jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type); - (void)jit_bnei_i(refslow, JIT_R2, scheme_prim_type); + (void)mz_bnei_t(refslow, JIT_R0, scheme_prim_type, JIT_R2); jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Primitive_Proc *)0x0)->pp.flags); if (kind == 3) { jit_andi_i(JIT_R2, JIT_R2, SCHEME_PRIM_OTHER_TYPE_MASK); @@ -1858,8 +1854,7 @@ static int common7(mz_jit_state *jitter, void *_data) ref2 = jit_beqi_p(jit_forward(), JIT_R0, scheme_null); ref8 = jit_bmsi_l(jit_forward(), JIT_R0, 0x1); - jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type); - ref3 = jit_bnei_i(jit_forward(), JIT_R2, scheme_pair_type); + ref3 = mz_bnei_t(jit_forward(), JIT_R0, scheme_pair_type, JIT_R2); CHECK_LIMIT(); jit_ldxi_s(JIT_R2, JIT_R0, &MZ_OPT_HASH_KEY(&((Scheme_Stx *)0x0)->iso)); @@ -1870,8 +1865,7 @@ static int common7(mz_jit_state *jitter, void *_data) ref5 = jit_beqi_p(jit_forward(), JIT_R0, scheme_null); ref7 = jit_bmsi_l(jit_forward(), JIT_R0, 0x1); - jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type); - (void)jit_beqi_i(refloop, JIT_R2, scheme_pair_type); + (void)mz_beqi_t(refloop, JIT_R0, scheme_pair_type, JIT_R2); ref6 = jit_jmpi(jit_forward()); CHECK_LIMIT(); @@ -1979,8 +1973,7 @@ static int common8(mz_jit_state *jitter, void *_data) ref2 = jit_beqi_p(jit_forward(), JIT_R0, scheme_null); ref3 = jit_bmsi_l(jit_forward(), JIT_R0, 0x1); - jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type); - ref4 = jit_bnei_i(jit_forward(), JIT_R2, scheme_pair_type); + ref4 = mz_bnei_t(jit_forward(), JIT_R0, scheme_pair_type, JIT_R2); CHECK_LIMIT(); jit_ldxi_s(JIT_R2, JIT_R0, &MZ_OPT_HASH_KEY(&((Scheme_Stx *)0x0)->iso)); @@ -2126,8 +2119,7 @@ static int more_common0(mz_jit_state *jitter, void *_data) /* Continue trying fast path: check proc */ mz_patch_branch(ref); - jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type); - (void)jit_bnei_i(refslow, JIT_R2, scheme_struct_type_type); + (void)mz_bnei_t(refslow, JIT_R0, scheme_struct_type_type, JIT_R2); jit_ldxi_s(JIT_R2, JIT_R0, &MZ_OPT_HASH_KEY(&((Scheme_Struct_Type *)0x0)->iso)); (void)jit_bmci_ul(refslow, JIT_R2, STRUCT_TYPE_CHECKED_PROC); CHECK_LIMIT(); @@ -2323,8 +2315,7 @@ static int more_common1(mz_jit_state *jitter, void *_data) ref2 = jit_beqi_p(jit_forward(), JIT_R0, scheme_null); __END_INNER_TINY__(1); ref1 = jit_bmsi_ul(jit_forward(), JIT_R0, 0x1); - jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type); - ref3 = jit_bnei_i(jit_forward(), JIT_R2, scheme_pair_type); + ref3 = mz_bnei_t(jit_forward(), JIT_R0, scheme_pair_type, JIT_R2); jit_addi_l(JIT_R1, JIT_R1, 1); jit_ldxi_p(JIT_R0, JIT_R0, (intptr_t)&SCHEME_CDR(0x0)); __START_INNER_TINY__(1); @@ -2464,7 +2455,7 @@ static int more_common1(mz_jit_state *jitter, void *_data) __END_SHORT_JUMPS__(1); - scheme_generate_tail_call(jitter, -1, 0, 1, 0, NULL); + scheme_generate_tail_call(jitter, -1, 0, 1, 0, NULL, NULL); CHECK_LIMIT(); } @@ -2500,8 +2491,7 @@ static int more_common1(mz_jit_state *jitter, void *_data) ref2 = jit_beqi_p(jit_forward(), JIT_R0, scheme_null); __END_INNER_TINY__(1); ref1 = jit_bmsi_ul(jit_forward(), JIT_R0, 0x1); - jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type); - ref3 = jit_bnei_i(jit_forward(), JIT_R2, scheme_pair_type); + ref3 = mz_bnei_t(jit_forward(), JIT_R0, scheme_pair_type, JIT_R2); jit_addi_l(JIT_R1, JIT_R1, 1); jit_ldxi_p(JIT_R0, JIT_R0, (intptr_t)&SCHEME_CDR(0x0)); __START_INNER_TINY__(1); diff --git a/src/racket/src/jitinline.c b/src/racket/src/jitinline.c index a85ecddee6..e4a67ab18e 100644 --- a/src/racket/src/jitinline.c +++ b/src/racket/src/jitinline.c @@ -200,6 +200,13 @@ static int generate_inlined_type_test(mz_jit_state *jitter, Scheme_App2_Rec *app ref4 = NULL; ref = NULL; ref5 = NULL; +#ifdef jit_bxnei_s + } else if (!can_chaperone && (lo_ty == hi_ty)) { + ref = jit_bmsi_ul(jit_forward(), JIT_R0, 0x1); + ref3 = jit_bxnei_s(jit_forward(), JIT_R0, lo_ty); + ref4 = NULL; + ref5 = NULL; +#endif } else { ref = jit_bmsi_ul(jit_forward(), JIT_R0, 0x1); jit_ldxi_s(JIT_R1, JIT_R0, &((Scheme_Object *)0x0)->type); @@ -529,8 +536,7 @@ int scheme_generate_inlined_unary(mz_jit_state *jitter, Scheme_App2_Rec *app, in /* Check for positive bignum: */ __START_SHORT_JUMPS__(branch_short); - jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type); - ref2 = jit_bnei_i(jit_forward(), JIT_R2, scheme_bignum_type); + ref2 = mz_bnei_t(jit_forward(), JIT_R0, scheme_bignum_type, JIT_R2); jit_ldxi_s(JIT_R2, JIT_R0, &MZ_OPT_HASH_KEY(&((Scheme_Stx *)0x0)->iso)); ref3 = jit_bmci_ul(jit_forward(), JIT_R2, 0x1); __END_SHORT_JUMPS__(branch_short); @@ -634,8 +640,7 @@ int scheme_generate_inlined_unary(mz_jit_state *jitter, Scheme_App2_Rec *app, in } else { (void)jit_bmsi_ul(reffail, JIT_R0, 0x1); } - jit_ldxi_s(JIT_R1, JIT_R0, &((Scheme_Object *)0x0)->type); - (void)jit_bnei_i(reffail, JIT_R1, scheme_pair_type); + (void)mz_bnei_t(reffail, JIT_R0, scheme_pair_type, JIT_R1); } else { reffail = NULL; } @@ -678,8 +683,7 @@ int scheme_generate_inlined_unary(mz_jit_state *jitter, Scheme_App2_Rec *app, in } __START_TINY_JUMPS__(1); mz_patch_branch(ref); - jit_ldxi_s(JIT_R1, JIT_R0, &((Scheme_Object *)0x0)->type); - (void)jit_bnei_i(reffail, JIT_R1, scheme_mutable_pair_type); + (void)mz_bnei_t(reffail, JIT_R0, scheme_mutable_pair_type, JIT_R1); if (name[2] == 'a') { (void)jit_ldxi_p(JIT_R0, JIT_R0, &((Scheme_Simple_Object *)0x0)->u.pair_val.car); } else { @@ -781,18 +785,16 @@ int scheme_generate_inlined_unary(mz_jit_state *jitter, Scheme_App2_Rec *app, in __START_TINY_JUMPS__(1); mz_patch_branch(ref); - jit_ldxi_s(JIT_R1, JIT_R0, &((Scheme_Object *)0x0)->type); if (for_fl) - (void)jit_bnei_i(reffail, JIT_R1, scheme_flvector_type); + (void)mz_bnei_t(reffail, JIT_R0, scheme_flvector_type, JIT_R1); else if (for_fx) - (void)jit_bnei_i(reffail, JIT_R1, scheme_fxvector_type); + (void)mz_bnei_t(reffail, JIT_R0, scheme_fxvector_type, JIT_R1); else - (void)jit_bnei_i(reffail, JIT_R1, scheme_vector_type); + (void)mz_bnei_t(reffail, JIT_R0, scheme_vector_type, JIT_R1); __END_TINY_JUMPS__(1); } else if (can_chaperone) { __START_TINY_JUMPS__(1); - jit_ldxi_s(JIT_R1, JIT_R0, &((Scheme_Object *)0x0)->type); - ref = jit_bnei_i(jit_forward(), JIT_R1, scheme_chaperone_type); + ref = mz_bnei_t(jit_forward(), JIT_R0, scheme_chaperone_type, JIT_R1); jit_ldxi_p(JIT_R0, JIT_R0, (intptr_t)&((Scheme_Chaperone *)0x0)->val); mz_patch_branch(ref); __END_TINY_JUMPS__(1); @@ -850,8 +852,7 @@ int scheme_generate_inlined_unary(mz_jit_state *jitter, Scheme_App2_Rec *app, in __START_TINY_JUMPS__(1); refdone = jit_jmpi(jit_forward()); mz_patch_branch(ref); - jit_ldxi_s(JIT_R1, JIT_R0, &((Scheme_Object *)0x0)->type); - (void)jit_bnei_i(reffail, JIT_R1, scheme_box_type); + (void)mz_bnei_t(reffail, JIT_R0, scheme_box_type, JIT_R1); __END_TINY_JUMPS__(1); (void)jit_ldxi_p(JIT_R0, JIT_R0, &SCHEME_BOX_VAL(0x0)); @@ -890,8 +891,7 @@ int scheme_generate_inlined_unary(mz_jit_state *jitter, Scheme_App2_Rec *app, in /* check for chaperone: */ __START_TINY_JUMPS__(1); - jit_ldxi_s(JIT_R1, JIT_R0, &((Scheme_Object *)0x0)->type); - ref = jit_bnei_i(jit_forward(), JIT_R1, scheme_chaperone_type); + ref = mz_bnei_t(jit_forward(), JIT_R0, scheme_chaperone_type, JIT_R1); (void)jit_calli(sjc.unbox_code); jit_retval(JIT_R0); ref2 = jit_jmpi(jit_forward()); @@ -971,8 +971,7 @@ int scheme_generate_inlined_unary(mz_jit_state *jitter, Scheme_App2_Rec *app, in } __START_TINY_JUMPS__(1); mz_patch_branch(ref); - jit_ldxi_s(JIT_R1, JIT_R0, &((Scheme_Object *)0x0)->type); - (void)jit_bnei_i(reffail, JIT_R1, scheme_complex_type); + (void)mz_bnei_t(reffail, JIT_R0, scheme_complex_type, JIT_R1); if (name[0] == 'i') { (void)jit_ldxi_p(JIT_R0, JIT_R0, &((Scheme_Complex *)0x0)->i); } else if (name[0] == 'r') { @@ -981,8 +980,7 @@ int scheme_generate_inlined_unary(mz_jit_state *jitter, Scheme_App2_Rec *app, in /* real part must always be inexact */ (void)jit_ldxi_p(JIT_R1, JIT_R0, &((Scheme_Complex *)0x0)->r); CHECK_LIMIT(); - jit_ldxi_s(JIT_R2, JIT_R1, &((Scheme_Object *)0x0)->type); - (void)jit_bnei_i(reffail, JIT_R2, scheme_double_type); + (void)mz_bnei_t(reffail, JIT_R1, scheme_double_type, JIT_R2); if (name[2] == 'i') { (void)jit_ldxi_p(JIT_R0, JIT_R0, &((Scheme_Complex *)0x0)->i); } else { @@ -1359,8 +1357,7 @@ static int generate_binary_char(mz_jit_state *jitter, Scheme_App3_Rec *app, } __START_SHORT_JUMPS__(branch_short); mz_patch_branch(pref); - jit_ldxi_s(JIT_R2, JIT_R0, (int)&((Scheme_Object *)0x0)->type); - (void)jit_bnei_i(reffail, JIT_R2, scheme_char_type); + (void)mz_bnei_t(reffail, JIT_R0, scheme_char_type, JIT_R2); CHECK_LIMIT(); } else { if (!direct) @@ -1383,8 +1380,7 @@ static int generate_binary_char(mz_jit_state *jitter, Scheme_App3_Rec *app, } else { (void)jit_bmsi_ul(reffail, JIT_R1, 0x1); } - jit_ldxi_s(JIT_R2, JIT_R1, (int)&((Scheme_Object *)0x0)->type); - (void)jit_bnei_i(reffail, JIT_R2, scheme_char_type); + (void)mz_bnei_t(reffail, JIT_R1, scheme_char_type, JIT_R2); CHECK_LIMIT(); } else { if (!direct) @@ -1442,8 +1438,7 @@ static int generate_vector_op(mz_jit_state *jitter, int set, int int_ready, int ref = jit_bmci_ul(jit_forward(), JIT_R0, 0x1); } else { /* assert: can_chaperone */ - jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type); - ref = jit_bnei_i(jit_forward(), JIT_R2, scheme_chaperone_type); + ref = mz_bnei_t(jit_forward(), JIT_R0, scheme_chaperone_type, JIT_R2); } __END_TINY_JUMPS__(1); @@ -1488,19 +1483,18 @@ static int generate_vector_op(mz_jit_state *jitter, int set, int int_ready, int (void)jit_bmci_ul(reffail, JIT_R1, 0x1); if (set && for_fx) (void)jit_bmci_ul(reffail, JIT_R2, 0x1); - jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type); if (for_fx) { - (void)jit_bnei_i(reffail, JIT_R2, scheme_fxvector_type); + (void)mz_bnei_t(reffail, JIT_R0, scheme_fxvector_type, JIT_R2); jit_ldxi_l(JIT_R2, JIT_R0, (int)&SCHEME_FXVEC_SIZE(0x0)); } else if (!for_fl) { - (void)jit_bnei_i(reffail, JIT_R2, scheme_vector_type); + (void)mz_bnei_t(reffail, JIT_R0, scheme_vector_type, JIT_R2); if (check_mutable) { jit_ldxi_s(JIT_R2, JIT_R0, &MZ_OPT_HASH_KEY((Scheme_Inclhash_Object *)0x0)); (void)jit_bmsi_ul(reffail, JIT_R2, 0x1); } jit_ldxi_l(JIT_R2, JIT_R0, (int)&SCHEME_VEC_SIZE(0x0)); } else { - (void)jit_bnei_i(reffail, JIT_R2, scheme_flvector_type); + (void)mz_bnei_t(reffail, JIT_R0, scheme_flvector_type, JIT_R2); jit_ldxi_l(JIT_R2, JIT_R0, (int)&SCHEME_FLVEC_SIZE(0x0)); } if (!int_ready) { @@ -1514,8 +1508,7 @@ static int generate_vector_op(mz_jit_state *jitter, int set, int int_ready, int if (for_fl && set && !unbox_flonum) { jit_ldr_p(JIT_R2, JIT_RUNSTACK); (void)jit_bmsi_ul(reffail, JIT_R2, 0x1); - jit_ldxi_s(JIT_R2, JIT_R2, &((Scheme_Object *)0x0)->type); - (void)jit_bnei_i(reffail, JIT_R2, scheme_double_type); + (void)mz_bnei_t(reffail, JIT_R2, scheme_double_type, JIT_R2); CHECK_LIMIT(); } } else if (!int_ready) { @@ -2334,8 +2327,7 @@ int scheme_generate_inlined_binary(mz_jit_state *jitter, Scheme_App3_Rec *app, i (void)jit_calli(sjc.bad_set_mcdr_code); __START_TINY_JUMPS__(1); mz_patch_branch(ref); - jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type); - (void)jit_bnei_i(reffail, JIT_R2, scheme_mutable_pair_type); + (void)mz_bnei_t(reffail, JIT_R0, scheme_mutable_pair_type, JIT_R2); __END_TINY_JUMPS__(1); CHECK_LIMIT(); @@ -2384,8 +2376,7 @@ int scheme_generate_inlined_binary(mz_jit_state *jitter, Scheme_App3_Rec *app, i ref3 = jit_bmsi_ul(jit_forward(), JIT_R0, 0x1); else ref3 = NULL; - jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type); - ref = jit_beqi_i(jit_forward(), JIT_R2, scheme_box_type); + ref = mz_beqi_t(jit_forward(), JIT_R0, scheme_box_type, JIT_R2); if (ref3) mz_patch_branch(ref3); reffail = _jit.x.pc; @@ -2554,11 +2545,9 @@ int scheme_generate_inlined_binary(mz_jit_state *jitter, Scheme_App3_Rec *app, i refslow = _jit.x.pc; (void)jit_calli(sjc.bad_make_flrectangular_code); mz_patch_branch(ref); - jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type); - (void)jit_bnei_i(refslow, JIT_R2, scheme_double_type); + (void)mz_bnei_t(refslow, JIT_R0, scheme_double_type, JIT_R2); (void)jit_bmsi_ul(refslow, JIT_R1, 0x1); - jit_ldxi_s(JIT_R2, JIT_R1, &((Scheme_Object *)0x0)->type); - (void)jit_bnei_i(refslow, JIT_R2, scheme_double_type); + (void)mz_bnei_t(refslow, JIT_R1, scheme_double_type, JIT_R2); __END_TINY_JUMPS__(1); CHECK_LIMIT(); diff --git a/src/racket/src/lightning/i386/core.h b/src/racket/src/lightning/i386/core.h index f3b48e98ca..093175fe3c 100644 --- a/src/racket/src/lightning/i386/core.h +++ b/src/racket/src/lightning/i386/core.h @@ -594,6 +594,10 @@ static const int const jit_arg_reg_order[] = { _EDI, _ESI, _EDX, _ECX }; #define jit_callr(reg) (CALLsr(reg)) #define jit_jmpr(reg) JMPsr(reg) +/* Checks whether *(short*)rs == is: */ +#define jit_bxeqi_s(label, rs, is) (CMPWim(is, 0, rs, 0, 0), JEm(label,0,0,0), _jit.x.pc) +#define jit_bxnei_s(label, rs, is) (CMPWim(is, 0, rs, 0, 0), JNEm(label,0,0,0), _jit.x.pc) + #ifdef SUPPORT_TINY_JUMPS # if 0 static intptr_t _CHECK_TINY(intptr_t diff) { if ((diff < -128) || (diff > 127)) *(intptr_t *)0x0 = 1; return diff; }