From 843992d0c72a5331d5042c619e6b4777efb1a75c Mon Sep 17 00:00:00 2001 From: Matthew Flatt Date: Thu, 24 Dec 2015 12:17:31 -0600 Subject: [PATCH] use specialization opportunity for `#%apply-values` --- racket/src/racket/src/jit.c | 348 ++++++++++++++++++------------ racket/src/racket/src/jit.h | 11 +- racket/src/racket/src/jitarith.c | 2 +- racket/src/racket/src/jitcall.c | 37 +++- racket/src/racket/src/jitinline.c | 20 +- racket/src/racket/src/jitstate.c | 26 ++- 6 files changed, 269 insertions(+), 175 deletions(-) diff --git a/racket/src/racket/src/jit.c b/racket/src/racket/src/jit.c index 6dfcbcfa8b..82f68277da 100644 --- a/racket/src/racket/src/jit.c +++ b/racket/src/racket/src/jit.c @@ -928,6 +928,54 @@ int scheme_needs_only_target_register(Scheme_Object *obj, int and_can_reorder) return (t >= _scheme_compiled_values_types_); } +static int produces_single_value(Scheme_Object *rator, int num_args, mz_jit_state *jitter) +{ + rator = scheme_specialize_to_constant(rator, jitter, num_args); + + if (SAME_TYPE(SCHEME_TYPE(rator), scheme_native_closure_type)) { + Scheme_Native_Closure *nc = (Scheme_Native_Closure *)rator; + if (nc->code->start_code == scheme_on_demand_jit_code) + return (SCHEME_CLOSURE_DATA_FLAGS(nc->code->u2.orig_code) & CLOS_SINGLE_RESULT); + else + return (SCHEME_NATIVE_CLOSURE_DATA_FLAGS(nc->code) & NATIVE_IS_SINGLE_RESULT); + } + + if (SCHEME_PRIMP(rator)) { + int opt; + opt = ((Scheme_Prim_Proc_Header *)rator)->flags & SCHEME_PRIM_OPT_MASK; + if (opt >= SCHEME_PRIM_OPT_NONCM) + return 1; + + /* special case: (values ) */ + if (SAME_OBJ(rator, scheme_values_func) && (num_args == 1)) + return 1; + } + + return 0; +} + +static int is_single_valued(Scheme_Object *obj, mz_jit_state *jitter) +{ + Scheme_Type t = SCHEME_TYPE(obj); + + switch(t) { + case scheme_application_type: + return produces_single_value(((Scheme_App_Rec *)obj)->args[0], ((Scheme_App_Rec *)obj)->num_args, jitter); + break; + case scheme_application2_type: + return produces_single_value(((Scheme_App2_Rec *)obj)->rator, 1, jitter); + break; + case scheme_application3_type: + return produces_single_value(((Scheme_App3_Rec *)obj)->rator, 2, jitter); + break; + default: + if (t > _scheme_values_types_) + return 1; + } + + return 0; +} + /*========================================================================*/ /* branch info */ /*========================================================================*/ @@ -2319,7 +2367,6 @@ int scheme_generate(Scheme_Object *obj, mz_jit_state *jitter, int is_tail, int w case scheme_apply_values_type: { Scheme_Object *p, *v; - GC_CAN_IGNORE jit_insn *ref, *ref2, *ref3, *ref5, *refloop; START_JIT_DATA(); LOG_IT(("appvals\n")); @@ -2330,146 +2377,169 @@ int scheme_generate(Scheme_Object *obj, mz_jit_state *jitter, int is_tail, int w v = scheme_specialize_to_constant(v, jitter, 0); p = scheme_specialize_to_constant(p, jitter, 0); - scheme_generate_non_tail(v, jitter, 0, 1, 0); - CHECK_LIMIT(); + if (is_single_valued(p, jitter)) { + /* We might discover late that `v` produces a single value, + possibly because we're in a specialized closure. In that + case, use a plain application. */ + Scheme_Object *alt_rands[2]; + int r; + + alt_rands[0] = v; + alt_rands[1] = p; + + r = scheme_generate_app(NULL, alt_rands, 1, 0, jitter, is_tail, multi_ok, result_ignored, 0); - /* If v is not known to produce a procedure, then check result: */ - if (!is_a_procedure(v, jitter)) { - mz_rs_sync(); - (void)jit_bmsi_l(sjc.bad_app_vals_target, JIT_R0, 0x1); - jit_ldxi_s(JIT_R1, JIT_R0, &((Scheme_Object *)0x0)->type); - (void)jit_blti_i(sjc.bad_app_vals_target, JIT_R1, scheme_prim_type); - (void)jit_bgti_i(sjc.bad_app_vals_target, JIT_R1, scheme_proc_chaperone_type); CHECK_LIMIT(); - } - - mz_pushr_p(JIT_R0); - scheme_generate_non_tail(p, jitter, 1, 1, 0); - CHECK_LIMIT(); - - mz_popr_p(JIT_V1); - /* Function is in V1, argument(s) in R0 */ - - mz_rs_sync(); - - __START_SHORT_JUMPS__(1); - ref = jit_beqi_p(jit_forward(), JIT_R0, SCHEME_MULTIPLE_VALUES); - /* Single-value case: --------------- */ - /* We definitely have stack space for one argument, because we - just used it for the rator. */ - if (is_tail) { - mz_ld_runstack_base_alt(JIT_RUNSTACK); - jit_subi_p(JIT_RUNSTACK, JIT_RUNSTACK_BASE_OR_ALT(JIT_RUNSTACK), WORDS_TO_BYTES(1)); - } else { - jit_subi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(1)); - } - CHECK_RUNSTACK_OVERFLOW(); - jit_str_p(JIT_RUNSTACK, JIT_R0); - jit_movi_l(JIT_R0, 1); - ref2 = jit_jmpi(jit_forward()); - CHECK_LIMIT(); - - /* Multiple-values case: ------------ */ - mz_patch_branch(ref); - /* Get new argc: */ - (void)mz_tl_ldi_p(JIT_R1, tl_scheme_current_thread); - jit_ldxi_l(JIT_R2, JIT_R1, &((Scheme_Thread *)0x0)->ku.multiple.count); - /* Enough room on runstack? */ - mz_tl_ldi_p(JIT_R0, tl_MZ_RUNSTACK_START); - if (is_tail) { - mz_ld_runstack_base_alt(JIT_R0); - jit_subr_ul(JIT_R0, JIT_RUNSTACK_BASE_OR_ALT(JIT_R0), JIT_R0); - } else { - jit_subr_ul(JIT_R0, JIT_RUNSTACK, JIT_R0); - } - CHECK_LIMIT(); - /* R0 is space left (in bytes), R2 is argc */ - jit_lshi_l(JIT_R2, JIT_R2, JIT_LOG_WORD_SIZE); - if (is_tail) { - int fpos, fstack; - fstack = scheme_mz_flostack_save(jitter, &fpos); - __END_SHORT_JUMPS__(1); - scheme_mz_flostack_restore(jitter, 0, 0, 1, 1); - (void)jit_bltr_ul(sjc.app_values_tail_slow_code, JIT_R0, JIT_R2); - __START_SHORT_JUMPS__(1); - scheme_mz_flostack_restore(jitter, fstack, fpos, 0, 1); - ref5 = 0; - } else { - GC_CAN_IGNORE jit_insn *refok; - refok = jit_bger_ul(jit_forward(), JIT_R0, JIT_R2); - __END_SHORT_JUMPS__(1); - if (multi_ok) { - (void)jit_calli(sjc.app_values_multi_slow_code); - } else { - (void)jit_calli(sjc.app_values_slow_code); - } - __START_SHORT_JUMPS__(1); - ref5 = jit_jmpi(jit_forward()); - mz_patch_branch(refok); - } - CHECK_LIMIT(); - if (is_tail) { - mz_ld_runstack_base_alt(JIT_RUNSTACK); - jit_subr_ul(JIT_RUNSTACK, JIT_RUNSTACK_BASE_OR_ALT(JIT_RUNSTACK), JIT_R2); - } else { - jit_subr_ul(JIT_RUNSTACK, JIT_RUNSTACK, JIT_R2); - } - CHECK_RUNSTACK_OVERFLOW(); - /* Copy args: */ - jit_ldxi_l(JIT_R1, JIT_R1, &((Scheme_Thread *)0x0)->ku.multiple.array); - refloop = jit_get_ip(); - ref3 = jit_blei_l(jit_forward(), JIT_R2, 0); - jit_subi_l(JIT_R2, JIT_R2, JIT_WORD_SIZE); - jit_ldxr_p(JIT_R0, JIT_R1, JIT_R2); - jit_stxr_p(JIT_R2, JIT_RUNSTACK, JIT_R0); - (void)jit_jmpi(refloop); - CHECK_LIMIT(); - mz_patch_branch(ref3); - /* clear array pointer and re-laod argc: */ - (void)mz_tl_ldi_p(JIT_R0, tl_scheme_current_thread); - (void)jit_movi_p(JIT_R1, NULL); - jit_stxi_l(&((Scheme_Thread *)0x0)->ku.multiple.array, JIT_R0, JIT_R1); - jit_ldxi_l(JIT_R0, JIT_R0, &((Scheme_Thread *)0x0)->ku.multiple.count); - CHECK_LIMIT(); - - /* Perform call --------------------- */ - /* Function is in V1, argc in R0, args on RUNSTACK */ - mz_patch_ucbranch(ref2); - __END_SHORT_JUMPS__(1); - - if (is_tail) { - if (!sjc.shared_tail_argc_code) { - sjc.shared_tail_argc_code = scheme_generate_shared_call(-1, jitter, 1, 0, 1, 0, 0, 0, 0); - } - mz_set_local_p(JIT_R0, JIT_LOCAL2); - (void)jit_jmpi(sjc.shared_tail_argc_code); - } else { - int mo = (multi_ok - ? (result_ignored ? SHARED_RESULT_IGNORED_CASE : SHARED_MULTI_OK_CASE) - : SHARED_SINGLE_VALUE_CASE); - void *code; - if (!sjc.shared_non_tail_argc_code[mo]) { - scheme_ensure_retry_available(jitter, multi_ok, result_ignored); - code = scheme_generate_shared_call(-2, jitter, multi_ok, result_ignored, 0, 0, 0, 0, 0); - sjc.shared_non_tail_argc_code[mo] = code; - } - code = sjc.shared_non_tail_argc_code[mo]; - (void)jit_calli(code); - /* non-tail code pops args off runstack for us */ - jitter->need_set_rs = 1; - mz_patch_ucbranch(ref5); if (target != JIT_R0) jit_movr_p(target, JIT_R0); + + if (for_branch) finish_branch(jitter, target, for_branch); + + return r; + } else { + GC_CAN_IGNORE jit_insn *ref, *ref2, *ref3, *ref5, *refloop; + + scheme_generate_non_tail(v, jitter, 0, 1, 0); + CHECK_LIMIT(); + + /* If v is not known to produce a procedure, then check result: */ + if (!is_a_procedure(v, jitter)) { + mz_rs_sync(); + (void)jit_bmsi_l(sjc.bad_app_vals_target, JIT_R0, 0x1); + jit_ldxi_s(JIT_R1, JIT_R0, &((Scheme_Object *)0x0)->type); + (void)jit_blti_i(sjc.bad_app_vals_target, JIT_R1, scheme_prim_type); + (void)jit_bgti_i(sjc.bad_app_vals_target, JIT_R1, scheme_proc_chaperone_type); + CHECK_LIMIT(); + } + + mz_pushr_p(JIT_R0); + scheme_generate_non_tail(p, jitter, 1, 1, 0); + CHECK_LIMIT(); + + mz_popr_p(JIT_V1); + /* Function is in V1, argument(s) in R0 */ + + mz_rs_sync(); + + __START_SHORT_JUMPS__(1); + ref = jit_beqi_p(jit_forward(), JIT_R0, SCHEME_MULTIPLE_VALUES); + /* Single-value case: --------------- */ + /* We definitely have stack space for one argument, because we + just used it for the rator. */ + if (is_tail) { + mz_ld_runstack_base_alt(JIT_RUNSTACK); + jit_subi_p(JIT_RUNSTACK, JIT_RUNSTACK_BASE_OR_ALT(JIT_RUNSTACK), WORDS_TO_BYTES(1)); + } else { + jit_subi_p(JIT_RUNSTACK, JIT_RUNSTACK, WORDS_TO_BYTES(1)); + } + CHECK_RUNSTACK_OVERFLOW(); + jit_str_p(JIT_RUNSTACK, JIT_R0); + jit_movi_l(JIT_R0, 1); + ref2 = jit_jmpi(jit_forward()); + CHECK_LIMIT(); + + /* Multiple-values case: ------------ */ + mz_patch_branch(ref); + /* Get new argc: */ + (void)mz_tl_ldi_p(JIT_R1, tl_scheme_current_thread); + jit_ldxi_l(JIT_R2, JIT_R1, &((Scheme_Thread *)0x0)->ku.multiple.count); + /* Enough room on runstack? */ + mz_tl_ldi_p(JIT_R0, tl_MZ_RUNSTACK_START); + if (is_tail) { + mz_ld_runstack_base_alt(JIT_R0); + jit_subr_ul(JIT_R0, JIT_RUNSTACK_BASE_OR_ALT(JIT_R0), JIT_R0); + } else { + jit_subr_ul(JIT_R0, JIT_RUNSTACK, JIT_R0); + } + CHECK_LIMIT(); + /* R0 is space left (in bytes), R2 is argc */ + jit_lshi_l(JIT_R2, JIT_R2, JIT_LOG_WORD_SIZE); + if (is_tail) { + int fpos, fstack; + fstack = scheme_mz_flostack_save(jitter, &fpos); + __END_SHORT_JUMPS__(1); + scheme_mz_flostack_restore(jitter, 0, 0, 1, 1); + (void)jit_bltr_ul(sjc.app_values_tail_slow_code, JIT_R0, JIT_R2); + __START_SHORT_JUMPS__(1); + scheme_mz_flostack_restore(jitter, fstack, fpos, 0, 1); + ref5 = 0; + } else { + GC_CAN_IGNORE jit_insn *refok; + refok = jit_bger_ul(jit_forward(), JIT_R0, JIT_R2); + __END_SHORT_JUMPS__(1); + if (multi_ok) { + (void)jit_calli(sjc.app_values_multi_slow_code); + } else { + (void)jit_calli(sjc.app_values_slow_code); + } + __START_SHORT_JUMPS__(1); + ref5 = jit_jmpi(jit_forward()); + mz_patch_branch(refok); + } + CHECK_LIMIT(); + if (is_tail) { + mz_ld_runstack_base_alt(JIT_RUNSTACK); + jit_subr_ul(JIT_RUNSTACK, JIT_RUNSTACK_BASE_OR_ALT(JIT_RUNSTACK), JIT_R2); + } else { + jit_subr_ul(JIT_RUNSTACK, JIT_RUNSTACK, JIT_R2); + } + CHECK_RUNSTACK_OVERFLOW(); + /* Copy args: */ + jit_ldxi_l(JIT_R1, JIT_R1, &((Scheme_Thread *)0x0)->ku.multiple.array); + refloop = jit_get_ip(); + ref3 = jit_blei_l(jit_forward(), JIT_R2, 0); + jit_subi_l(JIT_R2, JIT_R2, JIT_WORD_SIZE); + jit_ldxr_p(JIT_R0, JIT_R1, JIT_R2); + jit_stxr_p(JIT_R2, JIT_RUNSTACK, JIT_R0); + (void)jit_jmpi(refloop); + CHECK_LIMIT(); + mz_patch_branch(ref3); + /* clear array pointer and re-laod argc: */ + (void)mz_tl_ldi_p(JIT_R0, tl_scheme_current_thread); + (void)jit_movi_p(JIT_R1, NULL); + jit_stxi_l(&((Scheme_Thread *)0x0)->ku.multiple.array, JIT_R0, JIT_R1); + jit_ldxi_l(JIT_R0, JIT_R0, &((Scheme_Thread *)0x0)->ku.multiple.count); + CHECK_LIMIT(); + + /* Perform call --------------------- */ + /* Function is in V1, argc in R0, args on RUNSTACK */ + mz_patch_ucbranch(ref2); + __END_SHORT_JUMPS__(1); + + if (is_tail) { + if (!sjc.shared_tail_argc_code) { + sjc.shared_tail_argc_code = scheme_generate_shared_call(-1, jitter, 1, 0, 1, 0, 0, 0, 0); + } + mz_set_local_p(JIT_R0, JIT_LOCAL2); + (void)jit_jmpi(sjc.shared_tail_argc_code); + } else { + int mo = (multi_ok + ? (result_ignored ? SHARED_RESULT_IGNORED_CASE : SHARED_MULTI_OK_CASE) + : SHARED_SINGLE_VALUE_CASE); + void *code; + if (!sjc.shared_non_tail_argc_code[mo]) { + scheme_ensure_retry_available(jitter, multi_ok, result_ignored); + code = scheme_generate_shared_call(-2, jitter, multi_ok, result_ignored, 0, 0, 0, 0, 0); + sjc.shared_non_tail_argc_code[mo] = code; + } + code = sjc.shared_non_tail_argc_code[mo]; + (void)jit_calli(code); + /* non-tail code pops args off runstack for us */ + jitter->need_set_rs = 1; + mz_patch_ucbranch(ref5); + if (target != JIT_R0) + jit_movr_p(target, JIT_R0); + } + CHECK_LIMIT(); + + if (for_branch) finish_branch(jitter, target, for_branch); + + END_JIT_DATA(81); + + if (is_tail) + return 2; + return 1; } - CHECK_LIMIT(); - - if (for_branch) finish_branch(jitter, target, for_branch); - - END_JIT_DATA(81); - - if (is_tail) - return 2; - return 1; } break; case scheme_with_immed_mark_type: @@ -2642,7 +2712,7 @@ int scheme_generate(Scheme_Object *obj, mz_jit_state *jitter, int is_tail, int w return r; } - r = scheme_generate_app(app, NULL, app->num_args, jitter, is_tail, multi_ok, result_ignored, 0); + r = scheme_generate_app(app, NULL, app->num_args, app->num_args, jitter, is_tail, multi_ok, result_ignored, 0); CHECK_LIMIT(); if (target != JIT_R0) @@ -2670,7 +2740,7 @@ int scheme_generate(Scheme_Object *obj, mz_jit_state *jitter, int is_tail, int w args[0] = app->rator; args[1] = app->rand; - r = scheme_generate_app(NULL, args, 1, jitter, is_tail, multi_ok, result_ignored, 0); + r = scheme_generate_app(NULL, args, 1, 1, jitter, is_tail, multi_ok, result_ignored, 0); CHECK_LIMIT(); if (target != JIT_R0) @@ -2721,7 +2791,7 @@ int scheme_generate(Scheme_Object *obj, mz_jit_state *jitter, int is_tail, int w args[1] = app->rand1; args[2] = app->rand2; - r = scheme_generate_app(NULL, args, 2, jitter, is_tail, multi_ok, result_ignored, 0); + r = scheme_generate_app(NULL, args, 2, 2, jitter, is_tail, multi_ok, result_ignored, 0); CHECK_LIMIT(); if (target != JIT_R0) diff --git a/racket/src/racket/src/jit.h b/racket/src/racket/src/jit.h index 539d550d46..70ccaf1ea1 100644 --- a/racket/src/racket/src/jit.h +++ b/racket/src/racket/src/jit.h @@ -203,12 +203,7 @@ END_XFORM_ARITH; #include "jitfpu.h" -#if 0 -static void assert_failure(int where) { printf("JIT assert failed %d\n", where); } -#define JIT_ASSERT(v) if (!(v)) assert_failure(__LINE__); -#else -#define JIT_ASSERT(v) /* */ -#endif +#define JIT_ASSERT(v) MZ_ASSERT(v) /* Tracking statistics: */ #if 0 @@ -1364,6 +1359,8 @@ long_double *scheme_mz_retain_long_double(mz_jit_state *jitter, long_double d); int scheme_mz_remap_it(mz_jit_state *jitter, int i); void scheme_mz_pushr_p_it(mz_jit_state *jitter, int reg); void scheme_mz_popr_p_it(mz_jit_state *jitter, int reg, int discard); +void scheme_extra_pushed(mz_jit_state *jitter, int n); +void scheme_extra_popped(mz_jit_state *jitter, int n); void scheme_mz_need_space(mz_jit_state *jitter, int need_extra); int scheme_stack_safety(mz_jit_state *jitter, int cnt, int offset); #ifdef USE_FLONUM_UNBOXING @@ -1480,7 +1477,7 @@ typedef struct jit_direct_arg jit_direct_arg; void *scheme_generate_shared_call(int num_rands, mz_jit_state *old_jitter, int multi_ok, int result_ignored, int is_tail, int direct_prim, int direct_native, int nontail_self, int unboxed_args); void scheme_ensure_retry_available(mz_jit_state *jitter, int multi_ok, int result_ignored); -int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_rands, +int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_rands, int num_pushes, mz_jit_state *jitter, int is_tail, int multi_ok, int ignored_result, int no_call); int scheme_generate_tail_call(mz_jit_state *jitter, int num_rands, int direct_native, int need_set_rs, diff --git a/racket/src/racket/src/jitarith.c b/racket/src/racket/src/jitarith.c index 4a81996ca9..a84b039f0d 100644 --- a/racket/src/racket/src/jitarith.c +++ b/racket/src/racket/src/jitarith.c @@ -2241,7 +2241,7 @@ int scheme_generate_nary_arith(mz_jit_state *jitter, Scheme_App_Rec *app, } if (stack_c) - scheme_generate_app(app, alt_args, stack_c, jitter, 0, 0, 0, 2); + scheme_generate_app(app, alt_args, stack_c, stack_c, jitter, 0, 0, 0, 2); CHECK_LIMIT(); mz_rs_sync(); diff --git a/racket/src/racket/src/jitcall.c b/racket/src/racket/src/jitcall.c index 73f058aaaf..d44ec7513e 100644 --- a/racket/src/racket/src/jitcall.c +++ b/racket/src/racket/src/jitcall.c @@ -1737,7 +1737,7 @@ static int generate_call_path_with_unboxes(mz_jit_state *jitter, int direct_flos } #endif -int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_rands, +int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_rands, int num_pushes, mz_jit_state *jitter, int is_tail, int multi_ok, int result_ignored, int no_call) /* de-sync'd ok @@ -1762,7 +1762,7 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_ rator = (alt_rands ? alt_rands[0] : app->args[0]); - rator = scheme_specialize_to_constant(rator, jitter, num_rands); + rator = scheme_specialize_to_constant(rator, jitter, num_pushes); if (no_call == 2) { direct_prim = 1; @@ -1785,6 +1785,13 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_ } else { Scheme_Type t; t = SCHEME_TYPE(rator); + + if (t == scheme_case_closure_type) { + /* Turn it into a JITted empty case closure: */ + rator = scheme_unclose_case_lambda(rator, 1); + t = SCHEME_TYPE(rator); + } + if ((t == scheme_local_type) && scheme_ok_to_delay_local(rator)) { /* We can re-order evaluation of the rator. */ reorder_ok = 1; @@ -1792,7 +1799,7 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_ /* Call to known native, or even known self? */ { int pos, flags; - pos = SCHEME_LOCAL_POS(rator) - num_rands; + pos = SCHEME_LOCAL_POS(rator) - num_pushes; if (scheme_mz_is_closure(jitter, pos, num_rands, &flags)) { direct_native = 1; if ((pos == jitter->self_pos) @@ -1922,7 +1929,7 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_ locations that will be filled with argument values; that is, check how many arguments are already in place for the call. */ - mz_runstack_skipped(jitter, num_rands); + mz_runstack_skipped(jitter, num_pushes); for (i = 0; i < num_rands; i++) { v = (alt_rands ? alt_rands[i+1] : app->args[i+1]); if (SAME_TYPE(SCHEME_TYPE(v), scheme_local_type) @@ -1936,11 +1943,14 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_ } else break; } - mz_runstack_unskipped(jitter, num_rands); + mz_runstack_unskipped(jitter, num_pushes); if (args_already_in_place) { direct_native = 2; - mz_runstack_skipped(jitter, args_already_in_place); + if (num_pushes) + mz_runstack_skipped(jitter, args_already_in_place); num_rands -= args_already_in_place; + if (num_pushes) + num_pushes -= args_already_in_place; } LOG_IT((" [args in place: %d]\n", args_already_in_place)); } @@ -1953,7 +1963,7 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_ if (num_rands) { if (inline_direct_args) { - mz_runstack_skipped(jitter, num_rands); + mz_runstack_skipped(jitter, num_pushes); } else if (!direct_prim || (num_rands > 1) || (no_call == 2)) { int skip_end = 0; if (direct_self && is_tail && !no_call && (num_rands > 0)) { @@ -1963,13 +1973,17 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_ if (num_rands - skip_end > 0) { mz_rs_dec(num_rands-skip_end); CHECK_RUNSTACK_OVERFLOW(); - mz_runstack_pushed(jitter, num_rands-skip_end); + if (num_pushes) + mz_runstack_pushed(jitter, num_pushes-skip_end); + else + scheme_extra_pushed(jitter, num_rands-skip_end); } need_safety = num_rands-skip_end; - if (skip_end) + if (skip_end && num_pushes) mz_runstack_skipped(jitter, skip_end); } else { - mz_runstack_skipped(jitter, 1); + if (num_pushes) + mz_runstack_skipped(jitter, 1); } } @@ -2130,7 +2144,8 @@ int scheme_generate_app(Scheme_App_Rec *app, Scheme_Object **alt_rands, int num_ if (!no_call) { (void)jit_movi_p(JIT_V1, ((Scheme_Primitive_Proc *)rator)->prim_val); if (num_rands == 1) { - mz_runstack_unskipped(jitter, 1); + if (num_pushes) + mz_runstack_unskipped(jitter, 1); } else { mz_rs_sync(); JIT_UPDATE_THREAD_RSPTR_IF_NEEDED(); diff --git a/racket/src/racket/src/jitinline.c b/racket/src/racket/src/jitinline.c index 080edec928..616ac701ca 100644 --- a/racket/src/racket/src/jitinline.c +++ b/racket/src/racket/src/jitinline.c @@ -401,7 +401,7 @@ static int generate_inlined_struct_op(int kind, mz_jit_state *jitter, args[0] = rator; args[1] = rand; args[2] = rand2; - scheme_generate_app(NULL, args, 2, jitter, 0, 0, 0, 1); /* sync'd below */ + scheme_generate_app(NULL, args, 2, 2, jitter, 0, 0, 0, 1); /* sync'd below */ CHECK_LIMIT(); jit_movr_p(JIT_R0, JIT_V1); mz_rs_ldr(JIT_R1); @@ -623,7 +623,7 @@ static int generate_inlined_nary_struct_op(int kind, mz_jit_state *jitter, /* de-sync'd ok; for branch, sync'd before */ { /* generate code to evaluate the arguments */ - scheme_generate_app(app, NULL, app->num_args, jitter, 0, 0, 0, 1); + scheme_generate_app(app, NULL, app->num_args, app->num_args, jitter, 0, 0, 0, 1); CHECK_LIMIT(); mz_rs_sync(); @@ -3876,7 +3876,7 @@ int scheme_generate_inlined_binary(mz_jit_state *jitter, Scheme_App3_Rec *app, i args[1] = app->rand1; args[2] = app->rand2; - scheme_generate_app(NULL, args, 2, jitter, 0, 0, 0, 2); + scheme_generate_app(NULL, args, 2, 2, jitter, 0, 0, 0, 2); CHECK_LIMIT(); mz_rs_sync(); @@ -4014,7 +4014,7 @@ int scheme_generate_inlined_nary(mz_jit_state *jitter, Scheme_App_Rec *app, int } /* generate code to evaluate the arguments */ - scheme_generate_app(app, NULL, 3, jitter, 0, 0, 0, 2); + scheme_generate_app(app, NULL, 3, 3, jitter, 0, 0, 0, 2); CHECK_LIMIT(); mz_rs_sync(); @@ -4465,7 +4465,7 @@ int scheme_generate_inlined_nary(mz_jit_state *jitter, Scheme_App_Rec *app, int } else { got_two = 1; mz_runstack_skipped(jitter, 1); - scheme_generate_app(app, NULL, 2, jitter, 0, 0, 0, 2); + scheme_generate_app(app, NULL, 2, 2, jitter, 0, 0, 0, 2); CHECK_LIMIT(); } @@ -4531,7 +4531,7 @@ int scheme_generate_inlined_nary(mz_jit_state *jitter, Scheme_App_Rec *app, int star = IS_NAMED_PRIM(rator, "list*"); if (c) - scheme_generate_app(app, NULL, c, jitter, 0, 0, 0, 2); + scheme_generate_app(app, NULL, c, c, jitter, 0, 0, 0, 2); CHECK_LIMIT(); mz_rs_sync(); @@ -4592,7 +4592,7 @@ int scheme_generate_inlined_nary(mz_jit_state *jitter, Scheme_App_Rec *app, int if (!multi_ok) return 0; if (c) { - scheme_generate_app(app, NULL, c, jitter, 0, 0, 0, 2); + scheme_generate_app(app, NULL, c, c, jitter, 0, 0, 0, 2); CHECK_LIMIT(); mz_rs_sync(); @@ -4629,7 +4629,7 @@ int scheme_generate_inlined_nary(mz_jit_state *jitter, Scheme_App_Rec *app, int } else if (IS_NAMED_PRIM(rator, "max")) { return scheme_generate_nary_arith(jitter, app, ARITH_MAX, 0, NULL, 1, dest); } else if (IS_NAMED_PRIM(rator, "checked-procedure-check-and-extract")) { - scheme_generate_app(app, NULL, 5, jitter, 0, 0, 0, 2); /* sync'd below */ + scheme_generate_app(app, NULL, 5, 5, jitter, 0, 0, 0, 2); /* sync'd below */ CHECK_LIMIT(); mz_rs_sync(); @@ -4652,7 +4652,7 @@ int scheme_generate_inlined_nary(mz_jit_state *jitter, Scheme_App_Rec *app, int is_ref = IS_NAMED_PRIM(rator, "ptr-ref"); abs_offset = (n == (is_ref ? 4 : 5)); - scheme_generate_app(app, NULL, n, jitter, 0, 0, 0, 2); /* sync'd below */ + scheme_generate_app(app, NULL, n, n, jitter, 0, 0, 0, 2); /* sync'd below */ CHECK_LIMIT(); mz_rs_sync(); @@ -5004,7 +5004,7 @@ static int generate_vector_alloc(mz_jit_state *jitter, Scheme_Object *rator, } else { c = app->num_args; if (c) - scheme_generate_app(app, NULL, c, jitter, 0, 0, 0, 2); /* sync'd below */ + scheme_generate_app(app, NULL, c, c, jitter, 0, 0, 0, 2); /* sync'd below */ } CHECK_LIMIT(); diff --git a/racket/src/racket/src/jitstate.c b/racket/src/racket/src/jitstate.c index 575cce824e..d6db88b92f 100644 --- a/racket/src/racket/src/jitstate.c +++ b/racket/src/racket/src/jitstate.c @@ -488,12 +488,11 @@ static void new_mapping(mz_jit_state *jitter) jitter->mappings[jitter->num_mappings] = 0; } -void scheme_mz_pushr_p_it(mz_jit_state *jitter, int reg) -/* de-sync's rs */ +void scheme_extra_pushed(mz_jit_state *jitter, int n) { int v; - jitter->extra_pushed++; + jitter->extra_pushed += n; if (jitter->extra_pushed > jitter->max_extra_pushed) jitter->max_extra_pushed = jitter->extra_pushed; @@ -503,8 +502,14 @@ void scheme_mz_pushr_p_it(mz_jit_state *jitter, int reg) new_mapping(jitter); } v = (jitter->mappings[jitter->num_mappings]) >> 2; - v++; + v += n; jitter->mappings[jitter->num_mappings] = ((v << 2) | 0x1); +} + +void scheme_mz_pushr_p_it(mz_jit_state *jitter, int reg) +/* de-sync's rs */ +{ + scheme_extra_pushed(jitter, 1); mz_rs_dec(1); CHECK_RUNSTACK_OVERFLOW_NOCL(); @@ -513,21 +518,28 @@ void scheme_mz_pushr_p_it(mz_jit_state *jitter, int reg) jitter->need_set_rs = 1; } -void scheme_mz_popr_p_it(mz_jit_state *jitter, int reg, int discard) +void scheme_extra_popped(mz_jit_state *jitter, int n) /* de-sync's rs */ { int v; - jitter->extra_pushed--; + jitter->extra_pushed -= n; JIT_ASSERT(jitter->mappings[jitter->num_mappings] & 0x1); JIT_ASSERT(!(jitter->mappings[jitter->num_mappings] & 0x2)); v = jitter->mappings[jitter->num_mappings] >> 2; - v--; + v -= n; + JIT_ASSERT(v >= 0); if (!v) --jitter->num_mappings; else jitter->mappings[jitter->num_mappings] = ((v << 2) | 0x1); +} + +void scheme_mz_popr_p_it(mz_jit_state *jitter, int reg, int discard) +/* de-sync's rs */ +{ + scheme_extra_popped(jitter, 1); if (!discard) mz_rs_ldr(reg);