diff --git a/src/mzscheme/include/scheme.h b/src/mzscheme/include/scheme.h index 29835a0452..056617e505 100644 --- a/src/mzscheme/include/scheme.h +++ b/src/mzscheme/include/scheme.h @@ -156,7 +156,7 @@ typedef struct FSSpec mzFSSpec; #define MZ_EXTERN extern MZ_DLLSPEC -#if defined(MZ_USE_JIT_PPC) || defined(MZ_USE_JIT_I386) +#if defined(MZ_USE_JIT_PPC) || defined(MZ_USE_JIT_I386) || defined(MZ_USE_JIT_X86_64) # define MZ_USE_JIT #endif diff --git a/src/mzscheme/sconfig.h b/src/mzscheme/sconfig.h index 23aba215b7..990da1cf60 100644 --- a/src/mzscheme/sconfig.h +++ b/src/mzscheme/sconfig.h @@ -245,6 +245,9 @@ #if defined(i386) # define MZ_USE_JIT_I386 #endif +#if defined(__x86_64__) +# define MZ_USE_JIT_X86_64 +#endif #if defined(powerpc) # define MZ_USE_JIT_PPC #endif diff --git a/src/mzscheme/src/Makefile.in b/src/mzscheme/src/Makefile.in index 343b7b9b66..dba6dd0aab 100644 --- a/src/mzscheme/src/Makefile.in +++ b/src/mzscheme/src/Makefile.in @@ -258,7 +258,11 @@ hash.@LTO@: $(srcdir)/schpriv.h $(srcdir)/schexn.h $(SCONFIG) $(srcdir)/../inclu image.@LTO@: $(srcdir)/schpriv.h $(srcdir)/schexn.h $(SCONFIG) $(srcdir)/../include/scheme.h \ $(srcdir)/../src/stypes.h $(srcdir)/schvers.h jit.@LTO@: $(srcdir)/schpriv.h $(srcdir)/schexn.h $(SCONFIG) $(srcdir)/../include/scheme.h \ - $(srcdir)/../src/stypes.h $(srcdir)/schvers.h $(srcdir)/codetab.inc $(srcdir)/mzmark.c + $(srcdir)/../src/stypes.h $(srcdir)/schvers.h $(srcdir)/codetab.inc $(srcdir)/mzmark.c \ + $(srcdir)/lightning/i386/core.h $(srcdir)/lightning/i386/core-common.h \ + $(srcdir)/lightning/i386/asm.h $(srcdir)/lightning/i386/asm-common.h \ + $(srcdir)/lightning/ppc/core.h $(srcdir)/lightning/ppc/core-common.h \ + $(srcdir)/lightning/ppc/asm.h $(srcdir)/lightning/ppc/asm-common.h list.@LTO@: $(srcdir)/schpriv.h $(srcdir)/schexn.h $(SCONFIG) $(srcdir)/../include/scheme.h \ $(srcdir)/../src/stypes.h module.@LTO@: $(srcdir)/schpriv.h $(srcdir)/schexn.h $(SCONFIG) $(srcdir)/../include/scheme.h \ diff --git a/src/mzscheme/src/error.c b/src/mzscheme/src/error.c index 73cc87d378..dd0db90d29 100644 --- a/src/mzscheme/src/error.c +++ b/src/mzscheme/src/error.c @@ -925,7 +925,7 @@ static char *make_arity_expect_string(const char *name, int namelen, void scheme_wrong_count_m(const char *name, int minc, int maxc, int argc, Scheme_Object **argv, int is_method) - /* minc == -1 => name is really a case-lambda or proc-struct */ + /* minc == -1 => name is really a case-lambda, native closure, or proc-struct */ { char *s; long len; diff --git a/src/mzscheme/src/jit.c b/src/mzscheme/src/jit.c index ced7cb082e..16463bbc29 100644 --- a/src/mzscheme/src/jit.c +++ b/src/mzscheme/src/jit.c @@ -17,6 +17,25 @@ Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ +/* + JIT limtations: + + 1) See "About short-jump mode" below. + + 2) Immediate operands (not counting moves into registers) + must be 32-bit values on a 64-bit machine. + + 3) Function calls are limited to 3 arguments (i.e., jit_prepare() + must never be called with a number greater than 3). This limit + is related to the way the x86_64 port shuffles arguments into + temporary registers. + + 4) jit_ldi_X() and jit_sti_X() addresses must fit into 32 bits. + + 5) Use jit_patchable_movi_p() when the constant needs to be + visible to the GC. +*/ + #include "schpriv.h" #include "schmach.h" @@ -37,13 +56,26 @@ END_XFORM_ARITH; #endif +#ifdef MZ_USE_JIT_X86_64 +# define MZ_USE_JIT_I386 +# define JIT_X86_64 +#endif + #include "lightning/lightning.h" -#define JIT_LOG_WORD_SIZE 2 -#define WORDS_TO_BYTES(x) ((x) << JIT_LOG_WORD_SIZE) +#ifdef MZ_USE_JIT_X86_64 +# define JIT_LOG_WORD_SIZE 3 +#else +# define JIT_LOG_WORD_SIZE 2 +#endif #define JIT_WORD_SIZE (1 << JIT_LOG_WORD_SIZE) +#define WORDS_TO_BYTES(x) ((x) << JIT_LOG_WORD_SIZE) #define MAX_TRY_SHIFT 30 +#if defined(MZ_USE_JIT_PPC) || defined(MZ_USE_JIT_X86_64) +# define NEED_LONG_JUMPS +#endif + #define JIT_NOT_RET JIT_R1 #if JIT_NOT_RET == JIT_RET Fix me! See use. @@ -165,9 +197,23 @@ int stack_cache_stack_pos = 0; /* JIT buffer */ /*========================================================================*/ +#define JIT_BUFFER_PAD_SIZE 100 + #define _jit (jitter->js) -#define PAST_LIMIT() (jit_get_ip().ptr > jitter->limit) -#define CHECK_LIMIT() if (PAST_LIMIT()) return 0; +#define PAST_LIMIT() ((long)jit_get_ip().ptr > (long)jitter->limit) +#define CHECK_LIMIT() if (PAST_LIMIT()) return past_limit(jitter); +#if 1 +# define past_limit(j) 0 +#else +static int past_limit(mz_jit_state *jitter) +{ + if ((jit_get_ip().ptr > jitter->limit + JIT_BUFFER_PAD_SIZE) + || (jitter->retain_start)) { + printf("way past\n"); + } + return 0; +} +#endif #define JIT_CACHE_SIZE_LIMIT 65536 #define JIT_BUFFER_INIT_SIZE 256 @@ -199,7 +245,7 @@ static void mz_load_retained(mz_jit_state *jitter, int rs, int retptr) { void *p; p = jitter->retain_start + retptr - 1; - (void)jit_movi_p(rs, p); + (void)jit_patchable_movi_p(rs, p); jit_ldr_p(rs, rs); } #endif @@ -242,10 +288,10 @@ static void *generate_one(mz_jit_state *old_jitter, while (1) { memset(jitter, 0, sizeof(_jitter)); -#ifdef MZ_USE_JIT_PPC +#ifdef NEED_LONG_JUMPS _jitl.long_jumps = 1; #endif - padding = 100; + padding = JIT_BUFFER_PAD_SIZE; if (known_size) { size_pre_retained = known_size; size = size_pre_retained + WORDS_TO_BYTES(num_retained); @@ -334,7 +380,10 @@ static void *generate_one(mz_jit_state *old_jitter, jitter->limit = (char *)jitter->limit + padding; if (PAST_LIMIT() || (jitter->retain_start && (jitter->retained > num_retained))) { - scheme_console_printf("JIT buffer overflow!!\n"); + scheme_console_printf("JIT buffer overflow: %p [%p,%p] (%d)!!\n", + jit_get_ip().ptr, + buffer, jitter->limit, + !!jitter->retain_start); abort(); } @@ -351,8 +400,8 @@ static void *generate_one(mz_jit_state *old_jitter, } else { /* Allocate permanent area and jit again: */ known_size = ((unsigned long)jit_get_ip().ptr) - (unsigned long)buffer; - if (known_size & 0x3) { - known_size += (4 - (known_size & 0x3)); + if (known_size & (JIT_WORD_SIZE - 1)) { + known_size += (JIT_WORD_SIZE - (known_size & (JIT_WORD_SIZE - 1))); } num_retained = jitter->retained; /* Keep this buffer? Don't if it's too big, or if it's @@ -441,6 +490,27 @@ static void box_multiple_array_element(int pos) p->ku.multiple.array = naya; } +static void call_set_global_bucket(Scheme_Bucket *b, Scheme_Object *val, int set_undef) +{ + scheme_set_global_bucket("set!", b, val, set_undef); +} + +static void lexical_binding_wrong_return_arity(int expected, int got, Scheme_Object **argv) +{ + scheme_wrong_return_arity(NULL, expected, got, argv, "lexical binding"); +} + +static void call_wrong_return_arity(int expected, int got, Scheme_Object **argv) + +{ + scheme_wrong_return_arity(NULL, expected, got, argv, NULL); +} + +static void wrong_argument_count(Scheme_Object *proc, int argc, Scheme_Object **argv) +{ + scheme_wrong_count((char *)proc, -1, -1, argc, argv); +} + /*========================================================================*/ /* code-gen utils */ /*========================================================================*/ @@ -681,11 +751,11 @@ static int mz_is_closure(mz_jit_state *jitter, int i, int arity) # define mz_get_local_p(x, l) jit_ldxi_p(x, 1, l) # define mz_patch_branch_at(a, v) (_jitl.long_jumps ? (void)jit_patch_movei(a-4, a-3, v) : (void)jit_patch_branch(a-1, v)) # define mz_patch_ucbranch_at(a, v) (_jitl.long_jumps ? (void)jit_patch_movei(a-4, a-3, v) : (void)jit_patch_ucbranch(a-1, v)) -# define mz_patch_branch(a) mz_patch_branch_at(a, (_jit.x.pc)) -# define mz_patch_ucbranch(a) mz_patch_ucbranch_at(a, (_jit.x.pc)) # define mz_prolog(x) (MFLRr(x), mz_set_local_p(x, JIT_LOCAL2)) # define mz_epilog(x) (mz_get_local_p(x, JIT_LOCAL2), jit_jmpr(x)) # define mz_epilog_without_jmp() /* empty */ +# define jit_shuffle_saved_regs() /* empty */ +# define jit_unshuffle_saved_regs() /* empty */ # define mz_push_locals() /* empty */ # define mz_pop_locals() /* empty */ static void _jit_prolog_again(mz_jit_state *jitter, int n, int ret_addr_reg) @@ -716,34 +786,50 @@ static void _jit_prolog_again(mz_jit_state *jitter, int n, int ret_addr_reg) #endif } #else -# define JIT_LOCAL1 -16 -# define JIT_LOCAL2 -20 +# define JIT_LOCAL1 -(JIT_WORD_SIZE * 4) +# define JIT_LOCAL2 -(JIT_WORD_SIZE * 5) # define mz_set_local_p(x, l) jit_stxi_p((l), JIT_FP, (x)) # define mz_get_local_p(x, l) jit_ldxi_p((x), JIT_FP, (l)) -# define mz_patch_branch_at(a, v) jit_patch_at(a, v) -# define mz_patch_ucbranch_at(a, v) jit_patch_at(a, v) -# define mz_patch_branch(a) jit_patch(a) -# define mz_patch_ucbranch(a) jit_patch(a) +# define mz_patch_branch_at(a, v) jit_patch_branch_at(a, v) +# define mz_patch_ucbranch_at(a, v) jit_patch_ucbranch_at(a, v) # ifdef _CALL_DARWIN +# define X86_ALIGN_STACK +# define STACK_ALIGN_WORDS 3 +# endif +# ifdef JIT_X86_64 +# define X86_ALIGN_STACK +# define STACK_ALIGN_WORDS 1 +# endif +# ifdef X86_ALIGN_STACK /* Maintain 4-byte stack alignment. */ -# define mz_prolog(x) (SUBLir(3 * JIT_WORD_SIZE, JIT_SP)) -# define mz_epilog_without_jmp() ADDLir(4 * JIT_WORD_SIZE, JIT_SP) -# define mz_epilog(x) (ADDLir(3 * JIT_WORD_SIZE, JIT_SP), RET_()) +# define mz_prolog(x) (SUBQir(STACK_ALIGN_WORDS * JIT_WORD_SIZE, JIT_SP)) +# define mz_epilog_without_jmp() ADDQir((STACK_ALIGN_WORDS + 1) * JIT_WORD_SIZE, JIT_SP) +# define mz_epilog(x) (ADDQir(STACK_ALIGN_WORDS * JIT_WORD_SIZE, JIT_SP), RET_()) # define LOCAL_FRAME_SIZE 3 -# define JIT_LOCAL3 -24 +# define JIT_LOCAL3 -(JIT_WORD_SIZE * 6) # else # define mz_prolog(x) /* empty */ # define mz_epilog(x) RET_() -# define mz_epilog_without_jmp() ADDLir(JIT_WORD_SIZE, JIT_SP) +# define mz_epilog_without_jmp() ADDQir(JIT_WORD_SIZE, JIT_SP) # define LOCAL_FRAME_SIZE 2 # define JIT_LOCAL3 JIT_LOCAL2 # endif -# define mz_push_locals() SUBLir((LOCAL_FRAME_SIZE << JIT_LOG_WORD_SIZE), JIT_SP) -# define mz_pop_locals() ADDLir((LOCAL_FRAME_SIZE << JIT_LOG_WORD_SIZE), JIT_SP) -#define _jit_prolog_again(jitter, n, ret_addr_reg) (PUSHLr(ret_addr_reg), PUSHLr(_EBP), MOVLrr(_ESP, _EBP), PUSHLr(_EBX), PUSHLr(_ESI), PUSHLr(_EDI)) +# define mz_push_locals() SUBQir((LOCAL_FRAME_SIZE << JIT_LOG_WORD_SIZE), JIT_SP) +# define mz_pop_locals() ADDQir((LOCAL_FRAME_SIZE << JIT_LOG_WORD_SIZE), JIT_SP) +#define _jit_prolog_again(jitter, n, ret_addr_reg) (PUSHQr(ret_addr_reg), jit_base_prolog()) +# ifdef MZ_USE_JIT_X86_64 +# define jit_shuffle_saved_regs() (MOVQrr(_ESI, _R12), MOVQrr(_EDI, _R13)) +# define jit_unshuffle_saved_regs() (MOVQrr(_R12, _ESI), MOVQrr(_R13, _EDI)) +# else +# define jit_shuffle_saved_regs() /* empty */ +# define jit_unshuffle_saved_regs() /* empty */ +# endif #endif -#ifdef MZ_USE_JIT_PPC +#define mz_patch_branch(a) mz_patch_branch_at(a, (_jit.x.pc)) +#define mz_patch_ucbranch(a) mz_patch_ucbranch_at(a, (_jit.x.pc)) + +#ifdef NEED_LONG_JUMPS # define __START_SHORT_JUMPS__(cond) if (cond) { _jitl.long_jumps = 0; } # define __END_SHORT_JUMPS__(cond) if (cond) { _jitl.long_jumps = 1; } #else @@ -751,21 +837,38 @@ static void _jit_prolog_again(mz_jit_state *jitter, int n, int ret_addr_reg) # define __END_SHORT_JUMPS__(cond) /* empty */ #endif -/* In +/* mz_b..i_p supports 64-bit constants on x86_64: */ +#ifdef MZ_USE_JIT_X86_64 +# define mz_beqi_p(a, v, i) ((void)jit_patchable_movi_p(JIT_REXTMP, i), jit_beqr_p(a, v, JIT_REXTMP)) +# define mz_bnei_p(a, v, i) ((void)jit_patchable_movi_p(JIT_REXTMP, i), jit_bner_p(a, v, JIT_REXTMP)) +#else +# define mz_beqi_p(a, v, i) jit_beqi_p(a, v, i) +# define mz_bnei_p(a, v, i) jit_bnei_p(a, v, i) +#endif + +/* + About short-jump mode: + + In jit_jmpi(code); or jit_blti_i(code, v); - with short jumps enabled, the generated instructions can depend on - the relative location between the instruction address and the - actual value. Do not enable short jumps if the relative offset can - change between the initial sizing pass and the final pass. Of course, - also don't enable short umps if the jump is potentially too long. */ + the generated instructions can depend on the relative location + between the instruction address and the actual value. Do not enable + short jumps if the relative offset can change between the initial + sizing pass and the final pass. Of course, also don't enable short + jumps if the jump is potentially long (i.e. more than +/- 2^15 + on PowerPC, or more than +/- 2^31 on x86_64). Otherwise, enable + shorty-jump mode as much as possible. + + All mz_finish() and jit_calli() are implicitly long jumps. +*/ /*========================================================================*/ /* bytecode properties */ /*========================================================================*/ -#ifdef MZ_USE_JIT_PPC +#ifdef NEED_LONG_JUMPS static int is_short(Scheme_Object *obj, int fuel) { Scheme_Type t; @@ -1190,7 +1293,8 @@ static int generate_non_tail_call(mz_jit_state *jitter, int num_rands, int direc /* Fast inlined-native jump ok (proc will check argc, if necessary) */ { jit_insn *refr; - refr = jit_movi_p(JIT_R0, jit_forward()); + refr = jit_patchable_movi_p(JIT_R0, jit_forward()); + jit_shuffle_saved_regs(); _jit_prolog_again(jitter, 3, JIT_R0); /* saves V registers */ jit_movr_p(JIT_R0, JIT_V1); /* closure */ jit_movi_i(JIT_R1, num_rands); /* argc */ @@ -1207,6 +1311,7 @@ static int generate_non_tail_call(mz_jit_state *jitter, int num_rands, int direc } jit_jmpr(JIT_V1); jit_patch_movi(refr, (_jit.x.pc)); + jit_unshuffle_saved_regs(); } CHECK_LIMIT(); jit_retval(JIT_R0); @@ -1294,6 +1399,7 @@ static int generate_non_tail_call(mz_jit_state *jitter, int num_rands, int direc } else { (void)mz_finish(_scheme_apply_from_native); } + CHECK_LIMIT(); mz_patch_ucbranch(ref5); if (!direct_native) { mz_patch_ucbranch(ref8); @@ -1306,6 +1412,7 @@ static int generate_non_tail_call(mz_jit_state *jitter, int num_rands, int direc if (pop_and_jump) { mz_epilog(JIT_V1); } + CHECK_LIMIT(); __END_SHORT_JUMPS__(num_rands < 100); @@ -1639,11 +1746,11 @@ static jit_insn *generate_arith_slow_path(mz_jit_state *jitter, Scheme_Object *r (void)jit_movi_p(JIT_R2, ((Scheme_Primitive_Proc *)rator)->prim_val); if (for_branch) { - ref4 = jit_movi_p(JIT_V1, jit_forward()); + ref4 = jit_patchable_movi_p(JIT_V1, jit_forward()); mz_set_local_p(JIT_V1, JIT_LOCAL2); } else ref4 = NULL; - ref = jit_movi_p(JIT_V1, jit_forward()); + ref = jit_patchable_movi_p(JIT_V1, jit_forward()); if (orig_args == 1) { if (for_branch) { @@ -1695,6 +1802,12 @@ static jit_insn *generate_arith_slow_path(mz_jit_state *jitter, Scheme_Object *r } } +#ifdef SIXTY_FOUR_BIT_INTEGERS +# define SCHEME_INT_SMALL_ENOUGH(rand2) ((((long)rand2 & 0xFFFFFFFF) == (long)rand2) || (((long)rand2 & 0xFFFFFFFFF0000000) == 0xFFFFFFFFF0000000)) +#else +# define SCHEME_INT_SMALL_ENOUGH(rand2) 1 +#endif + static int generate_arith(mz_jit_state *jitter, Scheme_Object *rator, Scheme_Object *rand, Scheme_Object *rand2, int orig_args, int arith, int cmp, int v, jit_insn **for_branch, int branch_short) { @@ -1705,6 +1818,7 @@ static int generate_arith(mz_jit_state *jitter, Scheme_Object *rator, Scheme_Obj if (rand2) { if (SCHEME_INTP(rand2) + && SCHEME_INT_SMALL_ENOUGH(rand2) && ((arith != 6) || ((SCHEME_INT_VAL(rand2) <= MAX_TRY_SHIFT) && (SCHEME_INT_VAL(rand2) >= -MAX_TRY_SHIFT)))) { @@ -1714,6 +1828,7 @@ static int generate_arith(mz_jit_state *jitter, Scheme_Object *rator, Scheme_Obj v = SCHEME_INT_VAL(rand2); rand2 = NULL; } else if (SCHEME_INTP(rand) + && SCHEME_INT_SMALL_ENOUGH(rand) && (arith != 6)) { /* First is constant; swap argument order and use constant mode. */ v = SCHEME_INT_VAL(rand); @@ -1764,7 +1879,7 @@ static int generate_arith(mz_jit_state *jitter, Scheme_Object *rator, Scheme_Obj int pos, va; if (SCHEME_INTP(rand)) { - jit_movi_i(JIT_R1, rand); + (void)jit_movi_p(JIT_R1, rand); va = JIT_R0; } else { pos = mz_remap(SCHEME_LOCAL_POS(rand)); @@ -1825,15 +1940,15 @@ static int generate_arith(mz_jit_state *jitter, Scheme_Object *rator, Scheme_Obj /* First arg is in JIT_R1, second is in JIT_R0 */ if (arith == 1) { jit_andi_ul(JIT_R2, JIT_R1, (~0x1)); - (void)jit_boaddr_i(refslow, JIT_R2, JIT_R0); + (void)jit_boaddr_l(refslow, JIT_R2, JIT_R0); jit_movr_p(JIT_R0, JIT_R2); } else if (arith == -1) { if (reversed) { jit_movr_p(JIT_R2, JIT_R0); - (void)jit_bosubr_i(refslow, JIT_R2, JIT_R1); + (void)jit_bosubr_l(refslow, JIT_R2, JIT_R1); } else { jit_movr_p(JIT_R2, JIT_R1); - (void)jit_bosubr_i(refslow, JIT_R2, JIT_R0); + (void)jit_bosubr_l(refslow, JIT_R2, JIT_R0); } jit_ori_ul(JIT_R0, JIT_R2, 0x1); } else if (arith == 3) { @@ -1896,16 +2011,16 @@ static int generate_arith(mz_jit_state *jitter, Scheme_Object *rator, Scheme_Obj /* Non-constant arg is in JIT_R0 */ if (arith == 1) { jit_movr_p(JIT_R2, JIT_R0); - (void)jit_boaddi_i(refslow, JIT_R2, v << 1); + (void)jit_boaddi_l(refslow, JIT_R2, v << 1); jit_movr_p(JIT_R0, JIT_R2); } else if (arith == -1) { if (reversed) { (void)jit_movi_p(JIT_R2, scheme_make_integer(v)); - (void)jit_bosubr_i(refslow, JIT_R2, JIT_R0); + (void)jit_bosubr_l(refslow, JIT_R2, JIT_R0); jit_addi_ul(JIT_R0, JIT_R2, 0x1); } else { jit_movr_p(JIT_R2, JIT_R0); - (void)jit_bosubi_i(refslow, JIT_R2, v << 1); + (void)jit_bosubi_l(refslow, JIT_R2, v << 1); jit_movr_p(JIT_R0, JIT_R2); } } else { @@ -1949,38 +2064,38 @@ static int generate_arith(mz_jit_state *jitter, Scheme_Object *rator, Scheme_Obj switch (cmp) { case -2: if (rand2) { - ref3 = jit_bger_i(jit_forward(), JIT_R1, JIT_R0); + ref3 = jit_bger_l(jit_forward(), JIT_R1, JIT_R0); } else { - ref3 = jit_bgei_i(jit_forward(), JIT_R0, scheme_make_integer(v)); + ref3 = jit_bgei_l(jit_forward(), JIT_R0, (int)(long)scheme_make_integer(v)); } break; case -1: if (rand2) { - ref3 = jit_bgtr_i(jit_forward(), JIT_R1, JIT_R0); + ref3 = jit_bgtr_l(jit_forward(), JIT_R1, JIT_R0); } else { - ref3 = jit_bgti_i(jit_forward(), JIT_R0, scheme_make_integer(v)); + ref3 = jit_bgti_l(jit_forward(), JIT_R0, (int)(long)scheme_make_integer(v)); } break; case 0: if (rand2) { - ref3 = jit_bner_i(jit_forward(), JIT_R1, JIT_R0); + ref3 = jit_bner_l(jit_forward(), JIT_R1, JIT_R0); } else { - ref3 = jit_bnei_i(jit_forward(), JIT_R0, scheme_make_integer(v)); + ref3 = jit_bnei_l(jit_forward(), JIT_R0, (int)(long)scheme_make_integer(v)); } break; case 1: if (rand2) { - ref3 = jit_bltr_i(jit_forward(), JIT_R1, JIT_R0); + ref3 = jit_bltr_l(jit_forward(), JIT_R1, JIT_R0); } else { - ref3 = jit_blti_i(jit_forward(), JIT_R0, scheme_make_integer(v)); + ref3 = jit_blti_l(jit_forward(), JIT_R0, (int)(long)scheme_make_integer(v)); } break; case 2: default: if (rand2) { - ref3 = jit_bler_i(jit_forward(), JIT_R1, JIT_R0); + ref3 = jit_bler_l(jit_forward(), JIT_R1, JIT_R0); } else { - ref3 = jit_blei_i(jit_forward(), JIT_R0, scheme_make_integer(v)); + ref3 = jit_blei_l(jit_forward(), JIT_R0, (int)(long)scheme_make_integer(v)); } break; } @@ -2022,11 +2137,11 @@ static int generate_inlined_constant_test(mz_jit_state *jitter, Scheme_App2_Rec __START_SHORT_JUMPS__(branch_short); if (cnst2) { - ref2 = jit_beqi_p(jit_forward(), JIT_R0, cnst); - ref = jit_bnei_p(jit_forward(), JIT_R0, cnst2); + ref2 = mz_beqi_p(jit_forward(), JIT_R0, cnst); + ref = mz_bnei_p(jit_forward(), JIT_R0, cnst2); mz_patch_branch(ref2); } else { - ref = jit_bnei_p(jit_forward(), JIT_R0, cnst); + ref = mz_bnei_p(jit_forward(), JIT_R0, cnst); } if (for_branch) { @@ -2136,7 +2251,7 @@ static int generate_inlined_struct_op(int kind, mz_jit_state *jitter, /* R1 is [potential] predicate/getter, R0 is value */ if (for_branch) { - for_branch[2] = jit_movi_p(JIT_V1, jit_forward()); + for_branch[2] = jit_patchable_movi_p(JIT_V1, jit_forward()); (void)jit_calli(struct_pred_branch_code); } else if (kind == 1) { (void)jit_calli(struct_pred_code); @@ -2349,7 +2464,7 @@ static int generate_inlined_binary(mz_jit_state *jitter, Scheme_App3_Rec *app, i ref = jit_bner_p(jit_forward(), JIT_R0, JIT_R1); } else #endif - ref = jit_bnei_p(jit_forward(), JIT_R0, a1); + ref = mz_bnei_p(jit_forward(), JIT_R0, a1); if (for_branch) { for_branch[0] = ref; @@ -2467,11 +2582,11 @@ static int generate_inlined_binary(mz_jit_state *jitter, Scheme_App3_Rec *app, i (void)jit_calli(vector_ref_check_index_code); } else { - int offset; + long offset; offset = SCHEME_INT_VAL(app->rand2); (void)jit_movi_p(JIT_R1, offset); offset = ((int)&SCHEME_VEC_ELS(0x0)) + WORDS_TO_BYTES(SCHEME_INT_VAL(app->rand2)); - jit_movi_i(JIT_V1, offset); + jit_movi_l(JIT_V1, offset); (void)jit_calli(vector_ref_code); } @@ -2534,7 +2649,7 @@ static int generate_closure(Scheme_Closure_Data *data, #ifdef JIT_PRECISE_GC mz_load_retained(jitter, JIT_R0, retptr); #else - (void)jit_movi_p(JIT_R0, code); /* !! */ + (void)jit_patchable_movi_p(JIT_R0, code); /* !! */ #endif jit_pusharg_p(JIT_R0); (void)mz_finish(scheme_make_native_closure); @@ -2630,7 +2745,7 @@ static int generate_case_closure(Scheme_Object *obj, mz_jit_state *jitter) #ifdef JIT_PRECISE_GC mz_load_retained(jitter, JIT_R0, retptr); #else - (void)jit_movi_p(JIT_R0, ndata); /* !! */ + (void)jit_patchable_movi_p(JIT_R0, ndata); /* !! */ #endif jit_pusharg_p(JIT_R0); (void)mz_finish(scheme_make_native_case_closure); @@ -2670,17 +2785,17 @@ static int generate_non_tail_mark_pos_prefix(mz_jit_state *jitter) argument to generate_non_tail(), so that it can skip this prefix and suffix. In case this prefix needs to adjust the runstack, the result indicates the number of pushed values. */ - jit_ldi_i(JIT_R2, &scheme_current_cont_mark_pos); - jit_addi_i(JIT_R2, JIT_R2, 2); - jit_sti_i(&scheme_current_cont_mark_pos, JIT_R2); + jit_ldi_l(JIT_R2, &scheme_current_cont_mark_pos); + jit_addi_l(JIT_R2, JIT_R2, 2); + jit_sti_l(&scheme_current_cont_mark_pos, JIT_R2); return 0 /* = number of pushed items */; } static void generate_non_tail_mark_pos_suffix(mz_jit_state *jitter) { - jit_ldi_i(JIT_R2, &scheme_current_cont_mark_pos); - jit_subi_i(JIT_R2, JIT_R2, 2); - jit_sti_i(&scheme_current_cont_mark_pos, JIT_R2); + jit_ldi_l(JIT_R2, &scheme_current_cont_mark_pos); + jit_subi_l(JIT_R2, JIT_R2, 2); + jit_sti_l(&scheme_current_cont_mark_pos, JIT_R2); } static int generate_non_tail(Scheme_Object *obj, mz_jit_state *jitter, int multi_ok, int mark_pos_ends) @@ -2952,15 +3067,13 @@ static int generate(Scheme_Object *obj, mz_jit_state *jitter, int is_tail, int m /* R0 has values, R2 has pos */ JIT_UPDATE_THREAD_RSPTR_IF_NEEDED(); - mz_prepare(4); + mz_prepare(3); (void)jit_movi_i(JIT_R1, set_undef); jit_pusharg_p(JIT_R1); jit_pusharg_p(JIT_R0); jit_pusharg_p(JIT_R2); CHECK_LIMIT(); - (void)jit_movi_i(JIT_R1, "set!"); - jit_pusharg_p(JIT_R1); - (void)mz_finish(scheme_set_global_bucket); + (void)mz_finish(call_set_global_bucket); CHECK_LIMIT(); (void)jit_movi_p(JIT_R0, scheme_void); END_JIT_DATA(7); @@ -3066,7 +3179,7 @@ static int generate(Scheme_Object *obj, mz_jit_state *jitter, int is_tail, int m { JIT_UPDATE_THREAD_RSPTR_IF_NEEDED(); obj = SCHEME_IPTR_VAL(obj); - (void)jit_movi_p(JIT_R2, obj); /* !! */ + (void)jit_patchable_movi_p(JIT_R2, obj); /* !! */ CHECK_LIMIT(); mz_prepare(1); jit_pusharg_p(JIT_R2); @@ -3146,14 +3259,14 @@ static int generate(Scheme_Object *obj, mz_jit_state *jitter, int is_tail, int m Scheme_Branch_Rec *branch = (Scheme_Branch_Rec *)obj; jit_insn *refs[5], *ref2; int nsrs, nsrs1, g1, g2, amt; -#ifdef MZ_USE_JIT_PPC +#ifdef NEED_LONG_JUMPS int then_short_ok, else_short_ok; #else int then_short_ok = 1; #endif START_JIT_DATA(); -#ifdef MZ_USE_JIT_PPC +#ifdef NEED_LONG_JUMPS /* It's possible that the code for a then or else branch will be so large that we might need a long jump. Conservatively analyze the @@ -3304,7 +3417,7 @@ static int generate(Scheme_Object *obj, mz_jit_state *jitter, int is_tail, int m /* Load count and result array: */ jit_ldi_p(JIT_R2, &scheme_current_thread); jit_ldxi_i(JIT_R1, JIT_R2, &((Scheme_Thread *)0x0)->ku.multiple.count); - jit_ldxi_i(JIT_R2, JIT_R2, &((Scheme_Thread *)0x0)->ku.multiple.array); + jit_ldxi_p(JIT_R2, JIT_R2, &((Scheme_Thread *)0x0)->ku.multiple.array); CHECK_LIMIT(); /* If we got the expected count, jump to installing values: */ ref2 = jit_beqi_i(jit_forward(), JIT_R1, lv->count); @@ -3316,23 +3429,19 @@ static int generate(Scheme_Object *obj, mz_jit_state *jitter, int is_tail, int m and "array" to single value: */ mz_patch_branch(ref); jit_movi_i(JIT_R1, 1); - jit_movr_i(JIT_R2, JIT_R0); + jit_movr_p(JIT_R2, JIT_R0); CHECK_LIMIT(); /* Error starts here: */ mz_patch_ucbranch(ref3); JIT_UPDATE_THREAD_RSPTR_FOR_BRANCH_IF_NEEDED(); - mz_prepare(5); - (void)jit_movi_p(JIT_V1, "lexical binding"); - jit_pusharg_p(JIT_V1); + mz_prepare(3); jit_pusharg_p(JIT_R2); jit_pusharg_i(JIT_R1); CHECK_LIMIT(); jit_movi_i(JIT_V1, lv->count); jit_pusharg_i(JIT_V1); - (void)jit_movi_p(JIT_V1, NULL); - jit_pusharg_i(JIT_V1); - (void)mz_finish(scheme_wrong_return_arity); + (void)mz_finish(lexical_binding_wrong_return_arity); CHECK_LIMIT(); /* Continue with expected values; R2 has value array: */ @@ -3538,7 +3647,7 @@ static int generate(Scheme_Object *obj, mz_jit_state *jitter, int is_tail, int m mz_load_retained(jitter, JIT_R0, retptr); else #endif - (void)jit_movi_p(JIT_R0, obj); /* !! */ + (void)jit_patchable_movi_p(JIT_R0, obj); /* !! */ END_JIT_DATA(19); return 1; @@ -3585,7 +3694,7 @@ static int generate_function_getarg(mz_jit_state *jitter, int has_rest, int num_ one jump. Skip this optimization when the procedure has rest args, because we'll have to copy anyway. */ if (!has_rest && num_params) { - jit_lshi_i(JIT_RUNSTACK_BASE, JIT_R1, JIT_LOG_WORD_SIZE); + jit_lshi_l(JIT_RUNSTACK_BASE, JIT_R1, JIT_LOG_WORD_SIZE); jit_addr_p(JIT_RUNSTACK_BASE, JIT_R2, JIT_RUNSTACK_BASE); __START_SHORT_JUMPS__(num_params < 100); ref = jit_beqr_p(jit_forward(), JIT_RUNSTACK, JIT_R2); @@ -3638,7 +3747,7 @@ static int do_generate_common(mz_jit_state *jitter, void *_data) mz_push_locals(); jit_movi_i(JIT_R1, -1); jit_ldxi_p(JIT_V1, JIT_R0, &((Scheme_Native_Closure *)0x0)->code); - jit_ldxi_i(JIT_V1, JIT_V1, &((Scheme_Native_Closure_Data *)0x0)->arity_code); + jit_ldxi_p(JIT_V1, JIT_V1, &((Scheme_Native_Closure_Data *)0x0)->arity_code); jit_jmpr(JIT_V1); CHECK_LIMIT(); @@ -3650,9 +3759,9 @@ static int do_generate_common(mz_jit_state *jitter, void *_data) jit_getarg_p(JIT_R0, in); /* closure */ mz_push_locals(); jit_movi_i(JIT_R1, -1); - jit_movi_i(JIT_R2, 0x0); + (void)jit_movi_p(JIT_R2, 0x0); jit_ldxi_p(JIT_V1, JIT_R0, &((Scheme_Native_Closure *)0x0)->code); - jit_ldxi_i(JIT_V1, JIT_V1, &((Scheme_Native_Closure_Data *)0x0)->arity_code); + jit_ldxi_p(JIT_V1, JIT_V1, &((Scheme_Native_Closure_Data *)0x0)->arity_code); jit_jmpr(JIT_V1); CHECK_LIMIT(); @@ -3661,19 +3770,15 @@ static int do_generate_common(mz_jit_state *jitter, void *_data) bad_result_arity_code = (Native_Get_Arity_Proc)jit_get_ip().ptr; jit_ldi_p(JIT_R2, &scheme_current_thread); jit_ldxi_i(JIT_R1, JIT_R2, &((Scheme_Thread *)0x0)->ku.multiple.count); - jit_ldxi_i(JIT_R2, JIT_R2, &((Scheme_Thread *)0x0)->ku.multiple.array); + jit_ldxi_p(JIT_R2, JIT_R2, &((Scheme_Thread *)0x0)->ku.multiple.array); CHECK_LIMIT(); - mz_prepare(5); - (void)jit_movi_p(JIT_V1, NULL); - jit_pusharg_p(JIT_V1); + mz_prepare(3); jit_pusharg_p(JIT_R2); jit_pusharg_i(JIT_R1); CHECK_LIMIT(); jit_movi_i(JIT_V1, 1); jit_pusharg_i(JIT_V1); - (void)jit_movi_p(JIT_V1, NULL); - jit_pusharg_i(JIT_V1); - (void)mz_finish(scheme_wrong_return_arity); + (void)mz_finish(call_wrong_return_arity); CHECK_LIMIT(); /* *** unbound_global_code *** */ @@ -3879,9 +3984,9 @@ static int do_generate_common(mz_jit_state *jitter, void *_data) a tail call, we must decrement the cont-mark pos. */ mz_patch_branch(ref); mz_patch_branch(ref2); - jit_ldi_i(JIT_V1, &scheme_current_cont_mark_pos); - jit_subi_i(JIT_V1, JIT_V1, 2); - jit_sti_i(&scheme_current_cont_mark_pos, JIT_V1); + jit_ldi_l(JIT_V1, &scheme_current_cont_mark_pos); + jit_subi_l(JIT_V1, JIT_V1, 2); + jit_sti_l(&scheme_current_cont_mark_pos, JIT_V1); CHECK_LIMIT(); mz_prepare(3); jit_pusharg_p(JIT_R2); @@ -3889,9 +3994,9 @@ static int do_generate_common(mz_jit_state *jitter, void *_data) jit_pusharg_p(JIT_R0); (void)mz_finish(_scheme_apply_multi_from_native); CHECK_LIMIT(); - jit_ldi_i(JIT_NOT_RET, &scheme_current_cont_mark_pos); - jit_addi_i(JIT_NOT_RET, JIT_NOT_RET, 2); - jit_sti_i(&scheme_current_cont_mark_pos, JIT_NOT_RET); + jit_ldi_l(JIT_NOT_RET, &scheme_current_cont_mark_pos); + jit_addi_l(JIT_NOT_RET, JIT_NOT_RET, 2); + jit_sti_l(&scheme_current_cont_mark_pos, JIT_NOT_RET); mz_get_local_p(JIT_NOT_RET, JIT_LOCAL1); mz_pop_locals(); jit_ret(); @@ -3981,7 +4086,7 @@ static int do_generate_common(mz_jit_state *jitter, void *_data) (void)jit_blei_l(reffail, JIT_R1, 0x0); } jit_ldxi_s(JIT_R2, JIT_R0, &((Scheme_Object *)0x0)->type); - (void)jit_bnei_p(reffail, JIT_R2, scheme_vector_type); + (void)jit_bnei_i(reffail, JIT_R2, scheme_vector_type); jit_ldxi_i(JIT_R2, JIT_R0, &SCHEME_VEC_SIZE(0x0)); if (i) { jit_rshi_ul(JIT_V1, JIT_R1, 1); @@ -4070,7 +4175,7 @@ static int do_generate_common(mz_jit_state *jitter, void *_data) jit_movr_p(JIT_V(3), JIT_V1); #endif #ifdef MZ_USE_JIT_I386 -# ifdef _CALL_DARWIN +# ifdef X86_ALIGN_STACK mz_set_local_p(JIT_V1, JIT_LOCAL3); # else jit_pushr_p(JIT_V1); @@ -4180,7 +4285,7 @@ static int do_generate_common(mz_jit_state *jitter, void *_data) } else { mz_patch_ucbranch(bref6); #ifdef MZ_USE_JIT_I386 -# ifndef _CALL_DARWIN +# ifndef X86_ALIGN_STACK jit_popr_p(JIT_V1); # endif #endif @@ -4198,7 +4303,7 @@ static int do_generate_common(mz_jit_state *jitter, void *_data) jit_movr_p(JIT_V1, JIT_V(3)); #endif #ifdef MZ_USE_JIT_I386 -# ifdef _CALL_DARWIN +# ifdef X86_ALIGN_STACK mz_get_local_p(JIT_V1, JIT_LOCAL3); # else jit_popr_p(JIT_V1); @@ -4547,7 +4652,7 @@ static int generate_simple_arity_check(mz_jit_state *jitter, int num_params, int ref = jit_blti_i(jit_forward(), JIT_R1, num_params); jit_ldxi_p(JIT_V1, JIT_R0, &((Scheme_Native_Closure *)0x0)->code); - jit_ldxi_i(JIT_V1, JIT_V1, &((Scheme_Native_Closure_Data *)0x0)->u.tail_code); + jit_ldxi_p(JIT_V1, JIT_V1, &((Scheme_Native_Closure_Data *)0x0)->u.tail_code); jit_jmpr(JIT_V1); CHECK_LIMIT(); @@ -4558,15 +4663,12 @@ static int generate_simple_arity_check(mz_jit_state *jitter, int num_params, int ref = jit_blti_i(jit_forward(), JIT_R1, 0x0); /* Not negative, so report run-time arity mismatch */ - mz_prepare(5); + mz_prepare(3); jit_pusharg_p(JIT_R2); jit_pusharg_p(JIT_R1); - jit_movi_i(JIT_V1, -1); - CHECK_LIMIT(); - jit_pusharg_i(JIT_V1); - jit_pusharg_i(JIT_V1); jit_pusharg_p(JIT_R0); - (void)mz_finish(scheme_wrong_count); + CHECK_LIMIT(); + (void)mz_finish(wrong_argument_count); CHECK_LIMIT(); /* Arity check or reporting. If argv is NULL, it's a reporting request */ @@ -4682,7 +4784,7 @@ static int generate_case_lambda_dispatch(mz_jit_state *jitter, Scheme_Case_Lambd offset = WORDS_TO_BYTES(i) + (unsigned long)&((Scheme_Native_Closure *)0x0)->vals; jit_ldxi_p(JIT_R0, JIT_R0, offset); jit_ldxi_p(JIT_V1, JIT_R0, &((Scheme_Native_Closure *)0x0)->code); - jit_ldxi_i(JIT_V1, JIT_V1, &((Scheme_Native_Closure_Data *)0x0)->u.tail_code); + jit_ldxi_p(JIT_V1, JIT_V1, &((Scheme_Native_Closure_Data *)0x0)->u.tail_code); jit_jmpr(JIT_V1); CHECK_LIMIT(); @@ -4695,15 +4797,12 @@ static int generate_case_lambda_dispatch(mz_jit_state *jitter, Scheme_Case_Lambd if (!do_getarg) { /* Report run-time arity mismatch */ JIT_UPDATE_THREAD_RSPTR(); - mz_prepare(5); + mz_prepare(3); jit_pusharg_p(JIT_R2); jit_pusharg_p(JIT_R1); - jit_movi_i(JIT_V1, -1); - CHECK_LIMIT(); - jit_pusharg_i(JIT_V1); - jit_pusharg_i(JIT_V1); jit_pusharg_p(JIT_R0); - (void)mz_finish(scheme_wrong_count); + CHECK_LIMIT(); + (void)mz_finish(wrong_argument_count); CHECK_LIMIT(); } diff --git a/src/mzscheme/src/lightning/i386/asm-common.h b/src/mzscheme/src/lightning/i386/asm-common.h index 42c8814a91..6837c00371 100644 --- a/src/mzscheme/src/lightning/i386/asm-common.h +++ b/src/mzscheme/src/lightning/i386/asm-common.h @@ -85,17 +85,20 @@ typedef char _sc; typedef unsigned char _uc; typedef unsigned short _us; typedef unsigned int _ui; +typedef int _si; typedef long _sl; typedef unsigned long _ul; #define _jit_UC(X) ((_uc )(X)) #define _jit_US(X) ((_us )(X)) #define _jit_UI(X) ((_ui )(X)) +#define _jit_SI(X) ((_si )(X)) #define _jit_SL(X) ((_sl )(X)) #define _jit_UL(X) ((_ul )(X)) # define _PUC(X) ((_uc *)(X)) # define _PUS(X) ((_us *)(X)) # define _PUI(X) ((_ui *)(X)) +# define _PSI(X) ((_si *)(X)) # define _PSL(X) ((_sl *)(X)) # define _PUL(X) ((_ul *)(X)) @@ -105,9 +108,9 @@ typedef unsigned long _ul; #define _jit_L(L) _jit_UL(((*_jit.x.ul_pc++)= _jit_UL((L) ))) #define _jit_I_noinc(I) _jit_UL(((*_jit.x.ui_pc)= _jit_UI((I) ))) -#define _MASK(N) ((unsigned)((1<<(N)))-1) -#define _siP(N,I) (!((((unsigned)(I))^(((unsigned)(I))<<1))&~_MASK(N))) -#define _uiP(N,I) (!(((unsigned)(I))&~_MASK(N))) +#define _MASK(N) ((unsigned long)(((long)1<<(N)))-1) +#define _siP(N,I) (!((((unsigned long)(I))^(((unsigned long)(I))<<1))&~_MASK(N))) +#define _uiP(N,I) (!(((unsigned long)(I))&~_MASK(N))) #define _suiP(N,I) (_siP(N,I) | _uiP(N,I)) #ifndef _ASM_SAFETY @@ -128,6 +131,14 @@ typedef unsigned long _ul; #define _u8P(I) _uiP(8,I) #define _u16P(I) _uiP(16,I) +#ifdef JIT_X86_64 +#define _s32P(I) _siP(32,I) +#define _u32P(I) _uiP(32,I) +#else +#define _s32P(I) 1 +#define _u32P(I) 1 +#endif + #define _su8(I) _ck_su(8,I) #define _su16(I) _ck_su(16,I) diff --git a/src/mzscheme/src/lightning/i386/asm.h b/src/mzscheme/src/lightning/i386/asm.h index fcc364c05b..2196a17975 100644 --- a/src/mzscheme/src/lightning/i386/asm.h +++ b/src/mzscheme/src/lightning/i386/asm.h @@ -88,6 +88,11 @@ typedef _uc jit_insn; #define _ESI 0x46 #define _EDI 0x47 +#define _R12 0x4C +#define _R13 0x4D +#define JIT_CALLTMPSTART 0x48 +#define JIT_REXTMP 0x4B + #define _ST0 0 #define _ST1 1 #define _ST2 2 @@ -99,16 +104,19 @@ typedef _uc jit_insn; #define _rS(R) ((R)>>4) #define _rN(R) ((R)&0x7) +#define _qrN(R) ((R)&0xF) #define _r0P(R) ((R)==0) #ifndef _ASM_SAFETY #define _r1(R) _rN(R) #define _r2(R) _rN(R) #define _r4(R) _rN(R) +#define _r8(R) _qrN(R) #else #define _r1(R) ((_rS(R)==1) ? _rN(R) : JITFAIL( "8-bit register required")) #define _r2(R) ((_rS(R)==2) ? _rN(R) : JITFAIL("16-bit register required")) #define _r4(R) ((_rS(R)==4) ? _rN(R) : JITFAIL("32-bit register required")) +#define _r8(R) ((_rS(R)==4) ? _rN(R) : JITFAIL("64-bit register required")) #endif /*** ASSEMBLER ***/ @@ -117,7 +125,7 @@ typedef _uc jit_insn; #define _CKD8(D) _ck_d(8, ((_uc) _OFF4(D)) ) #define _D8(D) (_jit_B(0), ((*(_PUC(_jit.x.pc)-1))= _CKD8(D))) -#define _D32(D) (_jit_L(0), ((*(_PUL(_jit.x.pc)-1))= _OFF4(D))) +#define _D32(D) (_jit_I(0), ((*(_PUI(_jit.x.pc)-1))= _OFF4(D))) #ifndef _ASM_SAFETY # define _M(M) (M) @@ -138,6 +146,12 @@ typedef _uc jit_insn; #endif #define _Mrm(Md,R,M) _jit_B((_M(Md)<<6)|(_r(R)<<3)|_m(M)) +#ifdef JIT_X86_64 +# define _qMrm(Md,R,M) _jit_B((_M(Md)<<6)|(_r((R & 0x7))<<3)|_m((M & 0x7))) +#else +# define _qMrm(Md,R,M) _Mrm(Md,R,M) +#endif + #define _SIB(Sc,I, B) _jit_B((_s(Sc)<<6)|(_i(I)<<3)|_b(B)) #define _SCL(S) ((((S)==1) ? _b00 : \ @@ -147,17 +161,28 @@ typedef _uc jit_insn; /* memory subformats - urgh! */ -#define _r_D( R, D ) (_Mrm(_b00,_rN(R),_b101 ) ,_jit_L((long)(D))) +#ifdef JIT_X86_64 +# define _r_D( R, D ) (_Mrm(_b00,_rN(R),_b100 ),_SIB(0,_b100,_b101) ,_jit_I((long)(D))) +# define _r_Q( R, D ) (_qMrm(_b00,_rN(R),_b100 ),_SIB(0,_b100,_b101) ,_jit_I((long)(D))) +#else +# define _r_D( R, D ) (_Mrm(_b00,_rN(R),_b101 ) ,_jit_I((long)(D))) +# define _r_Q(R, D) _r_D(R, D) +#endif #define _r_0B( R, B ) (_Mrm(_b00,_rN(R),_r4(B)) ) #define _r_0BIS(R, B,I,S) (_Mrm(_b00,_rN(R),_b100 ),_SIB(_SCL(S),_r4(I),_r4(B)) ) #define _r_1B( R, D,B ) (_Mrm(_b01,_rN(R),_r4(B)) ,_jit_B((long)(D))) #define _r_1BIS(R, D,B,I,S) (_Mrm(_b01,_rN(R),_b100 ),_SIB(_SCL(S),_r4(I),_r4(B)),_jit_B((long)(D))) -#define _r_4B( R, D,B ) (_Mrm(_b10,_rN(R),_r4(B)) ,_jit_L((long)(D))) -#define _r_4IS( R, D,I,S) (_Mrm(_b00,_rN(R),_b100 ),_SIB(_SCL(S),_r4(I),_b101 ),_jit_L((long)(D))) -#define _r_4BIS(R, D,B,I,S) (_Mrm(_b10,_rN(R),_b100 ),_SIB(_SCL(S),_r4(I),_r4(B)),_jit_L((long)(D))) +#define _r_4B( R, D,B ) (_Mrm(_b10,_rN(R),_r4(B)) ,_jit_I((long)(D))) +#define _r_4IS( R, D,I,S) (_Mrm(_b00,_rN(R),_b100 ),_SIB(_SCL(S),_r4(I),_b101 ),_jit_I((long)(D))) +#define _r_4BIS(R, D,B,I,S) (_Mrm(_b10,_rN(R),_b100 ),_SIB(_SCL(S),_r4(I),_r4(B)),_jit_I((long)(D))) +#define _r_8B( R, D,B ) (_qMrm(_b10,_rN(R),_r8(B)) ,_jit_I((long)(D))) +#define _r_8IS( R, D,I,S) (_qMrm(_b00,_rN(R),_b100 ),_SIB(_SCL(S),_r8(I),_b101 ),_jit_I((long)(D))) +#define _r_8BIS(R, D,B,I,S) (_qMrm(_b10,_rN(R),_b100 ),_SIB(_SCL(S),_r8(I),_r8(B)),_jit_I((long)(D))) #define _r_DB( R, D,B ) ((_s0P(D) && (B != _EBP) ? _r_0B (R, B ) : (_s8P(D) ? _r_1B( R,D,B ) : _r_4B( R,D,B )))) #define _r_DBIS(R, D,B,I,S) ((_s0P(D) ? _r_0BIS(R, B,I,S) : (_s8P(D) ? _r_1BIS(R,D,B,I,S) : _r_4BIS(R,D,B,I,S)))) +#define _r_QB( R, D,B ) ((_s0P(D) && (B != _EBP) ? _r_0B (R, B ) : (_s8P(D) ? _r_1B( R,D,B ) : _r_8B( R,D,B )))) +#define _r_QBIS(R, D,B,I,S) ((_s0P(D) ? _r_0BIS(R, B,I,S) : (_s8P(D) ? _r_1BIS(R,D,B,I,S) : _r_8BIS(R,D,B,I,S)))) #define _r_X( R, D,B,I,S) (_r0P(I) ? (_r0P(B) ? _r_D (R,D ) : \ (_ESP==(B) ? _r_DBIS(R,D,_ESP,_ESP,1) : \ @@ -165,6 +190,12 @@ typedef _uc jit_insn; (_r0P(B) ? _r_4IS (R,D, I,S) : \ (((I)!=_ESP) ? _r_DBIS(R,D, B, I,S) : \ JITFAIL("illegal index register: %esp")))) +#define _qr_X( R, D,B,I,S) (_r0P(I) ? (_r0P(B) ? _r_Q (R,D ) : \ + (_ESP==(B) ? _r_QBIS(R,D,_ESP,_ESP,1) : \ + _r_QB (R,D, B ))) : \ + (_r0P(B) ? _r_8IS (R,D, I,S) : \ + (((I)!=_ESP) ? _r_QBIS(R,D, B, I,S) : \ + JITFAIL("illegal index register: %esp")))) /* instruction formats */ @@ -173,12 +204,28 @@ typedef _uc jit_insn; #define _d16() ( _jit_B(0x66 ) ) #define _O( OP ) ( _jit_B( OP ) ) +#ifdef JIT_X86_64 +# define _REX(R,X,B) ( _jit_B(0x48|((R&0x8)>>1)|((X&0x8)>>2)|((B&0x8)>>3)) ) +# define _qO( OP, R,X,B ) ( _REX(R,X,B), _jit_B( OP ) ) +#else +# define _qO( OP, R,X,B ) _O(OP) +#endif #define _Or( OP,R ) ( _jit_B( (OP)|_r(R)) ) +#ifdef JIT_X86_64 +# define _qOr( OP,R ) ( _REX(0,0,R), _jit_B( (OP)|_r(R&0x7)) ) +#else +# define _qOr( OP,R ) _Or(OP,R) +#endif #define _OO( OP ) ( _jit_B((OP)>>8), _jit_B( (OP) ) ) #define _OOr( OP,R ) ( _jit_B((OP)>>8), _jit_B( (OP)|_r(R)) ) #define _Os( OP,B ) ( _s8P(B) ? _jit_B(((OP)|_b10)) : _jit_B(OP) ) +#ifdef JIT_X86_64 +# define _qOs( OP, B, R, M ) ( _REX(0, M, R), _Os(OP, B) ) +#else +# define _qOs( OP, B, R, M ) _Os(OP, B) +#endif #define _sW( W ) ( _s8P(W) ? _jit_B(W):_jit_W(W) ) -#define _sL( L ) ( _s8P(L) ? _jit_B(L):_jit_L(L) ) +#define _sL( L ) ( _s8P(L) ? _jit_B(L):_jit_I(L) ) #define _O_W( OP ,W ) ( _O ( OP ) ,_jit_W(W) ) #define _O_D8( OP ,D ) ( _O ( OP ) ,_D8(D) ) #define _O_D32( OP ,D ) ( _O ( OP ) ,_D32(D) ) @@ -188,20 +235,27 @@ typedef _uc jit_insn; #define _O_W_B( OP ,W,B) ( _O ( OP ) ,_jit_W(W),_jit_B(B)) #define _Or_B( OP,R ,B ) ( _Or ( OP,R) ,_jit_B(B) ) #define _Or_W( OP,R ,W ) ( _Or ( OP,R) ,_jit_W(W) ) -#define _Or_L( OP,R ,L ) ( _Or ( OP,R) ,_jit_L(L) ) +#define _Or_L( OP,R ,L ) ( _Or ( OP,R) ,_jit_I(L) ) +#define _qOr_Q( OP,R ,Q ) ( _qOr ( OP,R) ,_jit_L(Q) ) #define _O_Mrm( OP ,MO,R,M ) ( _O ( OP ),_Mrm(MO,R,M ) ) +#define _qO_Mrm( OP ,MO,R,M ) ( _qO ( OP,R,0,M),_qMrm(MO,R,M ) ) #define _OO_Mrm( OP ,MO,R,M ) ( _OO ( OP ),_Mrm(MO,R,M ) ) #define _O_Mrm_B( OP ,MO,R,M ,B ) ( _O ( OP ),_Mrm(MO,R,M ) ,_jit_B(B) ) +#define _qO_Mrm_B( OP ,MO,R,M ,B ) ( _qO ( OP,R,0,M),_qMrm(MO,R,M ) ,_jit_B(B) ) #define _O_Mrm_W( OP ,MO,R,M ,W ) ( _O ( OP ),_Mrm(MO,R,M ) ,_jit_W(W) ) -#define _O_Mrm_L( OP ,MO,R,M ,L ) ( _O ( OP ),_Mrm(MO,R,M ) ,_jit_L(L) ) +#define _O_Mrm_L( OP ,MO,R,M ,L ) ( _O ( OP ),_Mrm(MO,R,M ) ,_jit_I(L) ) +#define _qO_Mrm_L( OP ,MO,R,M ,L ) ( _qO ( OP,R,0,M),_qMrm(MO,R,M ) ,_jit_I(L) ) +#define _qO_Mrm_Q( OP ,MO,R,M ,Q ) ( _qO ( OP,0,0,R),_qMrm(MO,R,M ) ,_jit_L(Q) ) #define _OO_Mrm_B( OP ,MO,R,M ,B ) ( _OO ( OP ),_Mrm(MO,R,M ) ,_jit_B(B) ) #define _Os_Mrm_sW(OP ,MO,R,M ,W ) ( _Os ( OP,W),_Mrm(MO,R,M ),_sW(W) ) #define _Os_Mrm_sL(OP ,MO,R,M ,L ) ( _Os ( OP,L),_Mrm(MO,R,M ),_sL(L) ) +#define _qOs_Mrm_sL(OP ,MO,R,M ,L ) ( _qOs ( OP,L,R,M),_qMrm(MO,R,M ),_sL(L) ) #define _O_r_X( OP ,R ,MD,MB,MI,MS ) ( _O ( OP ),_r_X( R ,MD,MB,MI,MS) ) +#define _qO_r_X( OP ,R ,MD,MB,MI,MS ) ( _qO ( OP,R,0,MS),_qr_X(R,MD,MB,MI,MS) ) #define _OO_r_X( OP ,R ,MD,MB,MI,MS ) ( _OO ( OP ),_r_X( R ,MD,MB,MI,MS) ) #define _O_r_X_B( OP ,R ,MD,MB,MI,MS,B ) ( _O ( OP ),_r_X( R ,MD,MB,MI,MS) ,_jit_B(B) ) #define _O_r_X_W( OP ,R ,MD,MB,MI,MS,W ) ( _O ( OP ),_r_X( R ,MD,MB,MI,MS) ,_jit_W(W) ) -#define _O_r_X_L( OP ,R ,MD,MB,MI,MS,L ) ( _O ( OP ),_r_X( R ,MD,MB,MI,MS) ,_jit_L(L) ) +#define _O_r_X_L( OP ,R ,MD,MB,MI,MS,L ) ( _O ( OP ),_r_X( R ,MD,MB,MI,MS) ,_jit_I(L) ) #define _OO_r_X_B( OP ,R ,MD,MB,MI,MS,B ) ( _OO ( OP ),_r_X( R ,MD,MB,MI,MS) ,_jit_B(B) ) #define _Os_r_X_sW(OP ,R ,MD,MB,MI,MS,W ) ( _Os ( OP,W),_r_X( R ,MD,MB,MI,MS),_sW(W) ) #define _Os_r_X_sL(OP ,R ,MD,MB,MI,MS,L ) ( _Os ( OP,L),_r_X( R ,MD,MB,MI,MS),_sL(L) ) @@ -267,6 +321,8 @@ typedef _uc jit_insn; #define ADDLir(IM, RD) _Os_Mrm_sL (0x81 ,_b11,_b000 ,_r4(RD) ,IM ) #define ADDLim(IM, MD, MB, MI, MS) _Os_r_X_sL (0x81 ,_b000 ,MD,MB,MI,MS ,IM ) +#define ADDQrr(RS, RD) _qO_Mrm (0x01 ,_b11,_r8(RS),_r8(RD) ) +#define ADDQir(IM, RD) _qOs_Mrm_sL (0x81 ,_b11,_b000 ,_r8(RD) ,IM ) #define ANDBrr(RS, RD) _O_Mrm (0x20 ,_b11,_r1(RS),_r1(RD) ) #define ANDBmr(MD, MB, MI, MS, RD) _O_r_X (0x22 ,_r1(RD) ,MD,MB,MI,MS ) @@ -286,6 +342,8 @@ typedef _uc jit_insn; #define ANDLir(IM, RD) _Os_Mrm_sL (0x81 ,_b11,_b100 ,_r4(RD) ,IM ) #define ANDLim(IM, MD, MB, MI, MS) _Os_r_X_sL (0x81 ,_b100 ,MD,MB,MI,MS ,IM ) +#define ANDQrr(RS, RD) _qO_Mrm (0x21 ,_b11,_r8(RS),_r8(RD) ) +#define ANDQir(IM, RD) _qOs_Mrm_sL (0x81 ,_b11,_b100 ,_r8(RD) ,IM ) #define BSWAPLr(R) _OOr (0x0fc8,_r4(R) ) @@ -333,13 +391,17 @@ typedef _uc jit_insn; #define BTSLrr(RS,RD) _OO_Mrm (0x0fab ,_b11,_r4(RS),_r4(RD) ) #define BTSLrm(RS,MD,MB,MI,MS) _OO_r_X (0x0fab ,_r4(RS) ,MD,MB,MI,MS ) - -#define CALLm(D,B,I,S) ((_r0P(B) && _r0P(I)) ? _O_D32 (0xe8 ,(int)(D) ) : \ +#ifdef JIT_X86_64 +# define CALLm(D,B,I,S) (MOVQir((D), JIT_REXTMP), CALQsr(JIT_REXTMP)) +#else +# define CALLm(D,B,I,S) ((_r0P(B) && _r0P(I)) ? _O_D32 (0xe8 ,(long)(D) ) : \ JITFAIL("illegal mode in direct jump")) +#endif #define CALLsr(R) _O_Mrm (0xff ,_b11,_b010,_r4(R) ) +#define CALQsr(R) _qO_Mrm (0xff ,_b11,_b010,_r8(R)) -#define CALLsm(D,B,I,S) _O_r_X (0xff ,_b010 ,(int)(D),B,I,S ) +#define CALLsm(D,B,I,S) _O_r_X (0xff ,_b010 ,(long)(D),B,I,S ) #define CBW_() _O (0x98 ) #define CLC_() _O (0xf8 ) @@ -365,6 +427,8 @@ typedef _uc jit_insn; #define CMPLir(IM, RD) _O_Mrm_L (0x81 ,_b11,_b111 ,_r4(RD) ,IM ) #define CMPLim(IM, MD, MB, MI, MS) _O_r_X_L (0x81 ,_b111 ,MD,MB,MI,MS ,IM ) +#define CMPQrr(RS, RD) _qO_Mrm (0x39 ,_b11,_r8(RS),_r8(RD) ) +#define CMPQir(IM, RD) _qO_Mrm_L (0x81 ,_b11,_b111 ,_r8(RD) ,IM ) #define CWD_() _O (0x99 ) @@ -448,7 +512,7 @@ typedef _uc jit_insn; #define INVLPGm(MD, MB, MI, MS) _OO_r_X (0x0f01 ,_b111 ,MD,MB,MI,MS ) -#define JCCSim(CC,D,B,I,S) ((_r0P(B) && _r0P(I)) ? _O_D8 (0x70|(CC) ,(int)(D) ) : \ +#define JCCSim(CC,D,B,I,S) ((_r0P(B) && _r0P(I)) ? _O_D8 (0x70|(CC) ,(long)(D) ) : \ JITFAIL("illegal mode in conditional jump")) #define JOSm(D,B,I,S) JCCSim(0x0,D,B,I,S) @@ -480,52 +544,65 @@ typedef _uc jit_insn; #define JNLESm(D,B,I,S) JCCSim(0xf,D,B,I,S) #define JGSm(D,B,I,S) JCCSim(0xf,D,B,I,S) -#define JCCim(CC,D,B,I,S) ((_r0P(B) && _r0P(I)) ? _OO_D32 (0x0f80|(CC) ,(int)(D) ) : \ +#ifdef JIT_X86_64 +# define JCCim(CC,nCC,D,B,I,S) (!_jitl.long_jumps \ + ? _OO_D32(0x0f80|(CC), (long)(D) ) \ + : (_O_D8(0x70|(nCC), _jit_UL(_jit.x.pc) + 13), JMPm((long)D, 0, 0, 0))) +#else +# define JCCim(CC,nCC,D,B,I,S) ((_r0P(B) && _r0P(I)) ? _OO_D32 (0x0f80|(CC) ,(long)(D) ) : \ JITFAIL("illegal mode in conditional jump")) +#endif -#define JOm(D,B,I,S) JCCim(0x0,D,B,I,S) -#define JNOm(D,B,I,S) JCCim(0x1,D,B,I,S) -#define JBm(D,B,I,S) JCCim(0x2,D,B,I,S) -#define JNAEm(D,B,I,S) JCCim(0x2,D,B,I,S) -#define JNBm(D,B,I,S) JCCim(0x3,D,B,I,S) -#define JAEm(D,B,I,S) JCCim(0x3,D,B,I,S) -#define JEm(D,B,I,S) JCCim(0x4,D,B,I,S) -#define JZm(D,B,I,S) JCCim(0x4,D,B,I,S) -#define JNEm(D,B,I,S) JCCim(0x5,D,B,I,S) -#define JNZm(D,B,I,S) JCCim(0x5,D,B,I,S) -#define JBEm(D,B,I,S) JCCim(0x6,D,B,I,S) -#define JNAm(D,B,I,S) JCCim(0x6,D,B,I,S) -#define JNBEm(D,B,I,S) JCCim(0x7,D,B,I,S) -#define JAm(D,B,I,S) JCCim(0x7,D,B,I,S) -#define JSm(D,B,I,S) JCCim(0x8,D,B,I,S) -#define JNSm(D,B,I,S) JCCim(0x9,D,B,I,S) -#define JPm(D,B,I,S) JCCim(0xa,D,B,I,S) -#define JPEm(D,B,I,S) JCCim(0xa,D,B,I,S) -#define JNPm(D,B,I,S) JCCim(0xb,D,B,I,S) -#define JPOm(D,B,I,S) JCCim(0xb,D,B,I,S) -#define JLm(D,B,I,S) JCCim(0xc,D,B,I,S) -#define JNGEm(D,B,I,S) JCCim(0xc,D,B,I,S) -#define JNLm(D,B,I,S) JCCim(0xd,D,B,I,S) -#define JGEm(D,B,I,S) JCCim(0xd,D,B,I,S) -#define JLEm(D,B,I,S) JCCim(0xe,D,B,I,S) -#define JNGm(D,B,I,S) JCCim(0xe,D,B,I,S) -#define JNLEm(D,B,I,S) JCCim(0xf,D,B,I,S) -#define JGm(D,B,I,S) JCCim(0xf,D,B,I,S) +#define JOm(D,B,I,S) JCCim(0x0,0x1,D,B,I,S) +#define JNOm(D,B,I,S) JCCim(0x1,0x0,D,B,I,S) +#define JBm(D,B,I,S) JCCim(0x2,0x3,D,B,I,S) +#define JNAEm(D,B,I,S) JCCim(0x2,0x3,D,B,I,S) +#define JNBm(D,B,I,S) JCCim(0x3,0x2,D,B,I,S) +#define JAEm(D,B,I,S) JCCim(0x3,0x2,D,B,I,S) +#define JEm(D,B,I,S) JCCim(0x4,0x5,D,B,I,S) +#define JZm(D,B,I,S) JCCim(0x4,0x5,D,B,I,S) +#define JNEm(D,B,I,S) JCCim(0x5,0x4,D,B,I,S) +#define JNZm(D,B,I,S) JCCim(0x5,0x4,D,B,I,S) +#define JBEm(D,B,I,S) JCCim(0x6,0x7,D,B,I,S) +#define JNAm(D,B,I,S) JCCim(0x6,0x7,D,B,I,S) +#define JNBEm(D,B,I,S) JCCim(0x7,0x6,D,B,I,S) +#define JAm(D,B,I,S) JCCim(0x7,0x6,D,B,I,S) +#define JSm(D,B,I,S) JCCim(0x8,0x9,D,B,I,S) +#define JNSm(D,B,I,S) JCCim(0x9,0x8,D,B,I,S) +#define JPm(D,B,I,S) JCCim(0xa,0xb,D,B,I,S) +#define JPEm(D,B,I,S) JCCim(0xa,0xb,D,B,I,S) +#define JNPm(D,B,I,S) JCCim(0xb,0xa,D,B,I,S) +#define JPOm(D,B,I,S) JCCim(0xb,0xa,D,B,I,S) +#define JLm(D,B,I,S) JCCim(0xc,0xd,D,B,I,S) +#define JNGEm(D,B,I,S) JCCim(0xc,0xd,D,B,I,S) +#define JNLm(D,B,I,S) JCCim(0xd,0xc,D,B,I,S) +#define JGEm(D,B,I,S) JCCim(0xd,0xc,D,B,I,S) +#define JLEm(D,B,I,S) JCCim(0xe,0xf,D,B,I,S) +#define JNGm(D,B,I,S) JCCim(0xe,0xf,D,B,I,S) +#define JNLEm(D,B,I,S) JCCim(0xf,0xe,D,B,I,S) +#define JGm(D,B,I,S) JCCim(0xf,0xe,D,B,I,S) -#define JMPSm(D,B,I,S) ((_r0P(B) && _r0P(I)) ? _O_D8 (0xeb ,(int)(D) ) : \ +#define JMPSm(D,B,I,S) ((_r0P(B) && _r0P(I)) ? _O_D8 (0xeb ,(long)(D) ) : \ JITFAIL("illegal mode in short jump")) -#define JMPm(D,B,I,S) ((_r0P(B) && _r0P(I)) ? _O_D32 (0xe9 ,(int)(D) ) : \ +#ifdef JIT_X86_64 +# define JMPm(D,B,I,S) (!_jitl.long_jumps \ + ? _O_D32(0xe9, (long)(D)) \ + : (MOVQir((D), JIT_REXTMP), _qO_Mrm(0xff,_b11,_b100,_r8(JIT_REXTMP)))) +#else +# define JMPm(D,B,I,S) ((_r0P(B) && _r0P(I)) ? _O_D32 (0xe9 ,(long)(D) ) : \ JITFAIL("illegal mode in direct jump")) +#endif #define JMPsr(R) _O_Mrm (0xff ,_b11,_b100,_r4(R) ) -#define JMPsm(D,B,I,S) _O_r_X (0xff ,_b100 ,(int)(D),B,I,S ) +#define JMPsm(D,B,I,S) _O_r_X (0xff ,_b100 ,(long)(D),B,I,S ) #define LAHF_() _O (0x9f ) #define LEALmr(MD, MB, MI, MS, RD) _O_r_X (0x8d ,_r4(RD) ,MD,MB,MI,MS ) +#define LEAQmr(MD, MB, MI, MS, RD) _qO_r_X (0x8d ,_r8(RD) ,MD,MB,MI,MS ) #define LEAVE_() _O (0xc9 ) @@ -566,6 +643,12 @@ typedef _uc jit_insn; #define MOVLir(IM, R) _Or_L (0xb8,_r4(R) ,IM ) #define MOVLim(IM, MD, MB, MI, MS) _O_X_L (0xc7 ,MD,MB,MI,MS ,IM ) +#define MOVQmr(MD, MB, MI, MS, RD) _qO_r_X (0x8b ,_r8(RD) ,MD,MB,MI,MS ) +#define MOVQrm(RS, MD, MB, MI, MS) _qO_r_X (0x89 ,_r8(RS) ,MD,MB,MI,MS ) +#define MOVQir(IM, R) _qOr_Q (0xb8,_r8(R) ,IM ) + +#define MOVQrr(RS, RD) _qO_Mrm (0x89 ,_b11,_r8(RS),_r8(RD) ) + #define MOVZBLrr(RS, RD) _OO_Mrm (0x0fb6 ,_b11,_r1(RD),_r1(RS) ) #define MOVZBLmr(MD, MB, MI, MS, RD) _OO_r_X (0x0fb6 ,_r1(RD) ,MD,MB,MI,MS ) #define MOVZBWrr(RS, RD) _wOO_Mrm (0x0fb6 ,_b11,_r2(RD),_r2(RS) ) @@ -600,6 +683,7 @@ typedef _uc jit_insn; #define NEGLr(RD) _O_Mrm (0xf7 ,_b11,_b011 ,_r4(RD) ) #define NEGLm(MD,MB,MI,MS) _O_r_X (0xf7 ,_b011 ,MD,MB,MI,MS ) +#define NEGQr(RD) _qO_Mrm (0xf7 ,_b11,_b011 ,_r8(RD) ) #define NOP_() _O (0x90 ) @@ -632,6 +716,8 @@ typedef _uc jit_insn; #define ORLir(IM, RD) _Os_Mrm_sL (0x81 ,_b11,_b001 ,_r4(RD) ,IM ) #define ORLim(IM, MD, MB, MI, MS) _Os_r_X_sL (0x81 ,_b001 ,MD,MB,MI,MS ,IM ) +#define ORQrr(RS, RD) _qO_Mrm (0x09 ,_b11,_r8(RS),_r8(RD) ) +#define ORQir(IM, RD) _qOs_Mrm_sL (0x81 ,_b11,_b001 ,_r8(RD) ,IM ) #define POPWr(RD) _wOr (0x58,_r2(RD) ) #define POPWm(MD,MB,MI,MS) _wO_r_X (0x8f ,_b000 ,MD,MB,MI,MS ) @@ -639,6 +725,8 @@ typedef _uc jit_insn; #define POPLr(RD) _Or (0x58,_r4(RD) ) #define POPLm(MD,MB,MI,MS) _O_r_X (0x8f ,_b000 ,MD,MB,MI,MS ) +#define POPQr(RD) _qOr (0x58,_r8(RD) ) + #define POPA_() _wO (0x61 ) #define POPAD_() _O (0x61 ) @@ -655,6 +743,7 @@ typedef _uc jit_insn; #define PUSHLm(MD,MB,MI,MS) _O_r_X (0xff ,_b110 ,MD,MB,MI,MS ) #define PUSHLi(IM) _Os_sL (0x68 ,IM ) +#define PUSHQr(R) _qOr (0x50,_r8(R) ) #define PUSHA_() _wO (0x60 ) #define PUSHAD_() _O (0x60 ) @@ -737,6 +826,10 @@ typedef _uc jit_insn; #define SALLim SHLLim #define SALLrr SHLLrr #define SALLrm SHLLrm +#define SALQir SHLQir +#define SALQim SHLQim +#define SALQrr SHLQrr +#define SALQrm SHLQrm #define SARBir(IM,RD) (((IM)==1) ? _O_Mrm (0xd0 ,_b11,_b111,_r1(RD) ) : \ @@ -766,6 +859,11 @@ typedef _uc jit_insn; #define SARLrm(RS,MD,MB,MS,MI) (((RS)==_CL) ? _O_r_X (0xd3 ,_b111 ,MD,MB,MI,MS ) : \ JITFAIL ("source register must be CL" ) ) +#define SARQir(IM,RD) (((IM)==1) ? _qO_Mrm (0xd1 ,_b11,_b111,_r8(RD) ) : \ + _qO_Mrm_B (0xc1 ,_b11,_b111,_r4(RD) ,_u8(IM) ) ) +#define SARQrr(RS,RD) (((RS)==_CL) ? _qO_Mrm (0xd3 ,_b11,_b111,_r8(RD) ) : \ + JITFAIL ("source register must be CL" ) ) + #define SBBBrr(RS, RD) _O_Mrm (0x18 ,_b11,_r1(RS),_r1(RD) ) #define SBBBmr(MD, MB, MI, MS, RD) _O_r_X (0x1a ,_r1(RD) ,MD,MB,MI,MS ) @@ -876,6 +974,11 @@ typedef _uc jit_insn; #define SHLLrm(RS,MD,MB,MS,MI) (((RS)==_CL) ? _O_r_X (0xd3 ,_b100 ,MD,MB,MI,MS ) : \ JITFAIL ("source register must be CL" ) ) +#define SHLQir(IM,RD) (((IM)==1) ? _qO_Mrm (0xd1 ,_b11,_b100,_r8(RD) ) : \ + _qO_Mrm_B (0xc1 ,_b11,_b100,_r8(RD) ,_u8(IM) ) ) +#define SHLQrr(RS,RD) (((RS)==_CL) ? _qO_Mrm (0xd3 ,_b11,_b100,_r8(RD) ) : \ + JITFAIL ("source register must be CL" ) ) + #define SHRBir(IM,RD) (((IM)==1) ? _O_Mrm (0xd0 ,_b11,_b101,_r1(RD) ) : \ _O_Mrm_B (0xc0 ,_b11,_b101,_r1(RD) ,_u8(IM) ) ) @@ -904,6 +1007,11 @@ typedef _uc jit_insn; #define SHRLrm(RS,MD,MB,MS,MI) (((RS)==_CL) ? _O_r_X (0xd3 ,_b101 ,MD,MB,MI,MS ) : \ JITFAIL ("source register must be CL" ) ) +#define SHRQir(IM,RD) (((IM)==1) ? _qO_Mrm (0xd1 ,_b11,_b101,_r8(RD) ) : \ + _qO_Mrm_B (0xc1 ,_b11,_b101,_r8(RD) ,_u8(IM) ) ) +#define SHRQrr(RS,RD) (((RS)==_CL) ? _qO_Mrm (0xd3 ,_b11,_b101,_r8(RD) ) : \ + JITFAIL ("source register must be CL" ) ) + #define STC_() _O (0xf9 ) @@ -926,6 +1034,8 @@ typedef _uc jit_insn; #define SUBLir(IM, RD) _Os_Mrm_sL (0x81 ,_b11,_b101 ,_r4(RD) ,IM ) #define SUBLim(IM, MD, MB, MI, MS) _Os_r_X_sL (0x81 ,_b101 ,MD,MB,MI,MS ,IM ) +#define SUBQrr(RS, RD) _qO_Mrm (0x29 ,_b11,_r8(RS),_r8(RD) ) +#define SUBQir(IM, RD) _qOs_Mrm_sL (0x81 ,_b11,_b101 ,_r8(RD) ,IM ) #define TESTBrr(RS, RD) _O_Mrm (0x84 ,_b11,_r1(RS),_r1(RD) ) #define TESTBrm(RS, MD, MB, MI, MS) _O_r_X (0x84 ,_r1(RS) ,MD,MB,MI,MS ) @@ -942,6 +1052,9 @@ typedef _uc jit_insn; #define TESTLir(IM, RD) _O_Mrm_L (0xf7 ,_b11,_b000 ,_r4(RD) ,IM ) #define TESTLim(IM, MD, MB, MI, MS) _O_r_X_L (0xf7 ,_b000 ,MD,MB,MI,MS ,IM ) +#define TESTQrr(RS, RD) _qO_Mrm (0x85 ,_b11,_r8(RS),_r8(RD) ) +#define TESTQir(IM, RD) _qO_Mrm_L (0xf7 ,_b11,_b000 ,_r8(RD) ,IM ) + #define XADDBrr(RS,RD) _OO_Mrm (0x0fc0 ,_b11,_r1(RS),_r1(RD) ) #define XADDBrm(RS,MD,MB,MI,MS) _OO_r_X (0x0fc0 ,_r1(RS) ,MD,MB,MI,MS ) @@ -981,6 +1094,9 @@ typedef _uc jit_insn; #define XORLir(IM, RD) _Os_Mrm_sL (0x81 ,_b11,_b110 ,_r4(RD) ,IM ) #define XORLim(IM, MD, MB, MI, MS) _Os_r_X_sL (0x81 ,_b110 ,MD,MB,MI,MS ,IM ) +#define XORQrr(RS, RD) _qO_Mrm (0x31 ,_b11,_r8(RS),_r8(RD) ) +#define XORQir(IM, RD) _qOs_Mrm_sL (0x81 ,_b11,_b110 ,_r8(RD) ,IM ) + /* x87 instructions -- yay, we found a use for octal constants :-) */ #define ESCmi(D,B,I,S,OP) _O_r_X(0xd8|(OP >> 3), (OP & 7), D,B,I,S) @@ -1037,9 +1153,9 @@ typedef _uc jit_insn; #define FNSTSWr(RD) ((RD == _AX || RD == _EAX) ? _OO (0xdfe0) \ : JITFAIL ("AX or EAX expected")) /* N byte NOPs */ -#define NOPi(N) ((( (N) >= 8) ? (_jit_B(0x8d),_jit_B(0xb4),_jit_B(0x26),_jit_L(0x00),_jit_B(0x90)) : (void) 0), \ - (( ((N)&7) == 7) ? (_jit_B(0x8d),_jit_B(0xb4),_jit_B(0x26),_jit_L(0x00)) : \ - ( ((N)&7) == 6) ? (_jit_B(0x8d),_jit_B(0xb6),_jit_L(0x00)) : \ +#define NOPi(N) ((( (N) >= 8) ? (_jit_B(0x8d),_jit_B(0xb4),_jit_B(0x26),_jit_I(0x00),_jit_B(0x90)) : (void) 0), \ + (( ((N)&7) == 7) ? (_jit_B(0x8d),_jit_B(0xb4),_jit_B(0x26),_jit_I(0x00)) : \ + ( ((N)&7) == 6) ? (_jit_B(0x8d),_jit_B(0xb6),_jit_I(0x00)) : \ ( ((N)&7) == 5) ? (_jit_B(0x90),_jit_B(0x8d),_jit_B(0x74),_jit_B(0x26),_jit_B(0x00)) : \ /* leal 0(,%esi), %esi */ ( ((N)&7) == 4) ? (_jit_B(0x8d),_jit_B(0x74),_jit_B(0x26),_jit_B(0x00)) : \ /* leal (,%esi), %esi */ ( ((N)&7) == 3) ? (_jit_B(0x8d),_jit_B(0x76),_jit_B(0x00)) : \ diff --git a/src/mzscheme/src/lightning/i386/core-common.h b/src/mzscheme/src/lightning/i386/core-common.h index 456778b58b..b78c6afa40 100644 --- a/src/mzscheme/src/lightning/i386/core-common.h +++ b/src/mzscheme/src/lightning/i386/core-common.h @@ -33,6 +33,25 @@ #ifndef __lightning_core_common_h #define __lightning_core_common_h_ +/* jit_code: union of many possible function pointer types. Returned + * by jit_get_ip(). + */ +typedef union jit_code { + char *ptr; + void (*vptr)(void); + char (*cptr)(void); + unsigned char (*ucptr)(void); + short (*sptr)(void); + unsigned short (*usptr)(void); + int (*iptr)(void); + unsigned int (*uiptr)(void); + long (*lptr)(void); + unsigned long (*ulptr)(void); + void * (*pptr)(void); + float (*fptr)(void); + double (*dptr)(void); +} jit_code; + typedef struct { union { jit_insn *pc; @@ -40,6 +59,7 @@ typedef struct { _us *us_pc; _ui *ui_pc; _ul *ul_pc; + jit_code code; } x; struct jit_fp *fp; struct jit_local_state jitl; @@ -63,7 +83,7 @@ static jit_state _jit; #define _jitl _jit.jitl -#define jit_get_ip() (*(jit_code *) &_jit.x.pc) +#define jit_get_ip() (_jit.x.code) #define jit_set_ip(ptr) (_jit.x.pc = (ptr), jit_get_ip ()) #define jit_get_label() (_jit.x.pc) #define jit_forward() (_jit.x.pc) @@ -77,24 +97,6 @@ static jit_state _jit; #define jit_align(n) #endif -/* jit_code: union of many possible function pointer types. Returned - * by jit_get_ip(). - */ -typedef union jit_code { - char *ptr; - void (*vptr)(void); - char (*cptr)(void); - unsigned char (*ucptr)(void); - short (*sptr)(void); - unsigned short (*usptr)(void); - int (*iptr)(void); - unsigned int (*uiptr)(void); - long (*lptr)(void); - unsigned long (*ulptr)(void); - void * (*pptr)(void); - float (*fptr)(void); - double (*dptr)(void); -} jit_code; #ifndef jit_fill_delay_after #define jit_fill_delay_after(branch) (branch) @@ -432,7 +434,7 @@ typedef union jit_code { #define jit_retval_s(rd) jit_retval_i((rd)) /* This was a bug, but we keep it. */ -#define jit_retval(rd) jit_retval_i ((rd)) +#define jit_retval(rd) jit_retval_l ((rd)) #ifndef jit_finish #define jit_finish(sub) jit_calli(sub) @@ -451,16 +453,16 @@ typedef union jit_code { #endif #ifndef jit_getarg_c -#ifndef JIT_FP -#define jit_getarg_c(reg, ofs) jit_extr_c_i ((reg), (ofs)) -#define jit_getarg_i(reg, ofs) jit_movr_i ((reg), (ofs)) -#define jit_getarg_l(reg, ofs) jit_movr_l ((reg), (ofs)) -#define jit_getarg_p(reg, ofs) jit_movr_p ((reg), (ofs)) -#define jit_getarg_s(reg, ofs) jit_extr_s_i ((reg), (ofs)) -#define jit_getarg_uc(reg, ofs) jit_extr_uc_ui((reg), (ofs)) -#define jit_getarg_ui(reg, ofs) jit_movr_ui ((reg), (ofs)) -#define jit_getarg_ul(reg, ofs) jit_extr_uc_ul((reg), (ofs)) -#define jit_getarg_us(reg, ofs) jit_extr_us_ul((reg), (ofs)) +#if !defined(JIT_FP) || defined(JIT_X86_64) +#define jit_getarg_c(reg, ofs) jit_extr_c_i ((reg), jit_arg_reg(ofs)) +#define jit_getarg_i(reg, ofs) jit_movr_i ((reg), jit_arg_reg(ofs)) +#define jit_getarg_l(reg, ofs) jit_movr_l ((reg), jit_arg_reg(ofs)) +#define jit_getarg_p(reg, ofs) jit_movr_p ((reg), jit_arg_reg(ofs)) +#define jit_getarg_s(reg, ofs) jit_extr_s_i ((reg), jit_arg_reg(ofs)) +#define jit_getarg_uc(reg, ofs) jit_extr_uc_ui((reg), jit_arg_reg(ofs)) +#define jit_getarg_ui(reg, ofs) jit_movr_ui ((reg), jit_arg_reg(ofs)) +#define jit_getarg_ul(reg, ofs) jit_extr_uc_ul((reg), jit_arg_reg(ofs)) +#define jit_getarg_us(reg, ofs) jit_extr_us_ul((reg), jit_arg_reg(ofs)) #else #define jit_getarg_c(reg, ofs) jit_ldxi_c((reg), JIT_FP, (ofs)); #define jit_getarg_uc(reg, ofs) jit_ldxi_uc((reg), JIT_FP, (ofs)); @@ -474,153 +476,7 @@ typedef union jit_code { #endif #endif - -/* Common definitions when sizeof(long) = sizeof(int) */ -#ifndef jit_addi_l -#define JIT_LONG_IS_INT - -/* ALU */ -#define jit_addi_l(d, rs, is) jit_addi_i((d), (rs), (is)) -#define jit_addr_l(d, s1, s2) jit_addr_i((d), (s1), (s2)) -#define jit_addci_l(d, rs, is) jit_addci_i((d), (rs), (is)) -#define jit_addcr_l(d, s1, s2) jit_addcr_i((d), (s1), (s2)) -#define jit_addxi_l(d, rs, is) jit_addxi_i((d), (rs), (is)) -#define jit_addxr_l(d, s1, s2) jit_addxr_i((d), (s1), (s2)) -#define jit_andi_l(d, rs, is) jit_andi_i((d), (rs), (is)) -#define jit_andr_l(d, s1, s2) jit_andr_i((d), (s1), (s2)) -#define jit_divi_l(d, rs, is) jit_divi_i((d), (rs), (is)) -#define jit_divr_l(d, s1, s2) jit_divr_i((d), (s1), (s2)) -#define jit_hmuli_l(d, rs, is) jit_hmuli_i((d), (rs), (is)) -#define jit_hmulr_l(d, s1, s2) jit_hmulr_i((d), (s1), (s2)) -#define jit_lshi_l(d, rs, is) jit_lshi_i((d), (rs), (is)) -#define jit_lshr_l(d, s1, s2) jit_lshr_i((d), (s1), (s2)) -#define jit_modi_l(d, rs, is) jit_modi_i((d), (rs), (is)) -#define jit_modr_l(d, s1, s2) jit_modr_i((d), (s1), (s2)) -#define jit_muli_l(d, rs, is) jit_muli_i((d), (rs), (is)) -#define jit_mulr_l(d, s1, s2) jit_mulr_i((d), (s1), (s2)) -#define jit_ori_l(d, rs, is) jit_ori_i((d), (rs), (is)) -#define jit_orr_l(d, s1, s2) jit_orr_i((d), (s1), (s2)) -#define jit_rshi_l(d, rs, is) jit_rshi_i((d), (rs), (is)) -#define jit_rshr_l(d, s1, s2) jit_rshr_i((d), (s1), (s2)) -#define jit_subr_l(d, s1, s2) jit_subr_i((d), (s1), (s2)) -#define jit_subcr_l(d, s1, s2) jit_subcr_i((d), (s1), (s2)) -#define jit_subxi_l(d, rs, is) jit_subxi_i((d), (rs), (is)) -#define jit_subxr_l(d, s1, s2) jit_subxr_i((d), (s1), (s2)) -#define jit_xori_l(d, rs, is) jit_xori_i((d), (rs), (is)) -#define jit_xorr_l(d, s1, s2) jit_xorr_i((d), (s1), (s2)) - -#ifndef jit_rsbi_l -#define jit_rsbi_l(d, rs, is) jit_rsbi_i((d), (rs), (is)) -#endif - -#define jit_divi_ul(d, rs, is) jit_divi_ui((d), (rs), (is)) -#define jit_divr_ul(d, s1, s2) jit_divr_ui((d), (s1), (s2)) -#define jit_hmuli_ul(d, rs, is) jit_hmuli_ui((d), (rs), (is)) -#define jit_hmulr_ul(d, s1, s2) jit_hmulr_ui((d), (s1), (s2)) -#define jit_modi_ul(d, rs, is) jit_modi_ui((d), (rs), (is)) -#define jit_modr_ul(d, s1, s2) jit_modr_ui((d), (s1), (s2)) -#define jit_muli_ul(d, rs, is) jit_muli_ui((d), (rs), (is)) -#define jit_mulr_ul(d, s1, s2) jit_mulr_ui((d), (s1), (s2)) -#define jit_rshi_ul(d, rs, is) jit_rshi_ui((d), (rs), (is)) -#define jit_rshr_ul(d, s1, s2) jit_rshr_ui((d), (s1), (s2)) - -/* Sign/Zero extension */ -#define jit_extr_c_l(d, rs) jit_extr_c_i(d, rs) -#define jit_extr_c_ul(d, rs) jit_extr_c_ui(d, rs) -#define jit_extr_s_l(d, rs) jit_extr_s_i(d, rs) -#define jit_extr_s_ul(d, rs) jit_extr_s_ui(d, rs) -#define jit_extr_i_l(d, rs) jit_movr_i(d, rs) -#define jit_extr_i_ul(d, rs) jit_movr_i(d, rs) - -/* Unary */ -#define jit_movi_l(d, rs) jit_movi_i((d), (rs)) -#define jit_movr_l(d, rs) jit_movr_i((d), (rs)) - -/* Stack */ -#define jit_pushr_l(rs) jit_pushr_i(rs) -#define jit_popr_l(rs) jit_popr_i(rs) -#define jit_pusharg_l(rs) jit_pusharg_i(rs) - -/* Memory */ -#ifndef JIT_RZERO -#define jit_ldr_l(d, rs) jit_ldr_i((d), (rs)) -#define jit_ldi_l(d, is) jit_ldi_i((d), (is)) -#define jit_str_l(d, rs) jit_str_i((d), (rs)) -#define jit_sti_l(d, is) jit_sti_i((d), (is)) -#define jit_ldr_ui(d, rs) jit_ldr_i((d), (rs)) -#define jit_ldi_ui(d, is) jit_ldi_i((d), (is)) -#define jit_ldr_ul(d, rs) jit_ldr_ui((d), (rs)) -#define jit_ldi_ul(d, is) jit_ldi_ui((d), (is)) -#endif - -#define jit_ldxr_l(d, s1, s2) jit_ldxr_i((d), (s1), (s2)) -#define jit_ldxi_l(d, rs, is) jit_ldxi_i((d), (rs), (is)) -#define jit_stxr_l(d, s1, s2) jit_stxr_i((d), (s1), (s2)) -#define jit_stxi_l(d, rs, is) jit_stxi_i((d), (rs), (is)) -#define jit_ldxr_ui(d, s1, s2) jit_ldxr_i((d), (s1), (s2)) -#define jit_ldxi_ui(d, rs, is) jit_ldxi_i((d), (rs), (is)) -#define jit_ldxr_ul(d, s1, s2) jit_ldxr_ui((d), (s1), (s2)) -#define jit_ldxi_ul(d, rs, is) jit_ldxi_ui((d), (rs), (is)) - - -/* Boolean */ -#define jit_ltr_l(d, s1, s2) jit_ltr_i((d), (s1), (s2)) -#define jit_lti_l(d, rs, is) jit_lti_i((d), (rs), (is)) -#define jit_ler_l(d, s1, s2) jit_ler_i((d), (s1), (s2)) -#define jit_lei_l(d, rs, is) jit_lei_i((d), (rs), (is)) -#define jit_gtr_l(d, s1, s2) jit_gtr_i((d), (s1), (s2)) -#define jit_gti_l(d, rs, is) jit_gti_i((d), (rs), (is)) -#define jit_ger_l(d, s1, s2) jit_ger_i((d), (s1), (s2)) -#define jit_gei_l(d, rs, is) jit_gei_i((d), (rs), (is)) -#define jit_eqr_l(d, s1, s2) jit_eqr_i((d), (s1), (s2)) -#define jit_eqi_l(d, rs, is) jit_eqi_i((d), (rs), (is)) -#define jit_ner_l(d, s1, s2) jit_ner_i((d), (s1), (s2)) -#define jit_nei_l(d, rs, is) jit_nei_i((d), (rs), (is)) -#define jit_ltr_ul(d, s1, s2) jit_ltr_ui((d), (s1), (s2)) -#define jit_lti_ul(d, rs, is) jit_lti_ui((d), (rs), (is)) -#define jit_ler_ul(d, s1, s2) jit_ler_ui((d), (s1), (s2)) -#define jit_lei_ul(d, rs, is) jit_lei_ui((d), (rs), (is)) -#define jit_gtr_ul(d, s1, s2) jit_gtr_ui((d), (s1), (s2)) -#define jit_gti_ul(d, rs, is) jit_gti_ui((d), (rs), (is)) -#define jit_ger_ul(d, s1, s2) jit_ger_ui((d), (s1), (s2)) -#define jit_gei_ul(d, rs, is) jit_gei_ui((d), (rs), (is)) - -/* Branches */ -#define jit_bltr_l(label, s1, s2) jit_bltr_i((label), (s1), (s2)) -#define jit_blti_l(label, rs, is) jit_blti_i((label), (rs), (is)) -#define jit_bler_l(label, s1, s2) jit_bler_i((label), (s1), (s2)) -#define jit_blei_l(label, rs, is) jit_blei_i((label), (rs), (is)) -#define jit_bgtr_l(label, s1, s2) jit_bgtr_i((label), (s1), (s2)) -#define jit_bgti_l(label, rs, is) jit_bgti_i((label), (rs), (is)) -#define jit_bger_l(label, s1, s2) jit_bger_i((label), (s1), (s2)) -#define jit_bgei_l(label, rs, is) jit_bgei_i((label), (rs), (is)) -#define jit_beqr_l(label, s1, s2) jit_beqr_i((label), (s1), (s2)) -#define jit_beqi_l(label, rs, is) jit_beqi_i((label), (rs), (is)) -#define jit_bner_l(label, s1, s2) jit_bner_i((label), (s1), (s2)) -#define jit_bnei_l(label, rs, is) jit_bnei_i((label), (rs), (is)) -#define jit_bmcr_l(label, s1, s2) jit_bmcr_i((label), (s1), (s2)) -#define jit_bmci_l(label, rs, is) jit_bmci_i((label), (rs), (is)) -#define jit_bmsr_l(label, s1, s2) jit_bmsr_i((label), (s1), (s2)) -#define jit_bmsi_l(label, rs, is) jit_bmsi_i((label), (rs), (is)) -#define jit_boaddr_l(label, s1, s2) jit_boaddr_i((label), (s1), (s2)) -#define jit_boaddi_l(label, rs, is) jit_boaddi_i((label), (rs), (is)) -#define jit_bosubr_l(label, s1, s2) jit_bosubr_i((label), (s1), (s2)) -#define jit_bosubi_l(label, rs, is) jit_bosubi_i((label), (rs), (is)) -#define jit_bltr_ul(label, s1, s2) jit_bltr_ui((label), (s1), (s2)) -#define jit_blti_ul(label, rs, is) jit_blti_ui((label), (rs), (is)) -#define jit_bler_ul(label, s1, s2) jit_bler_ui((label), (s1), (s2)) -#define jit_blei_ul(label, rs, is) jit_blei_ui((label), (rs), (is)) -#define jit_bgtr_ul(label, s1, s2) jit_bgtr_ui((label), (s1), (s2)) -#define jit_bgti_ul(label, rs, is) jit_bgti_ui((label), (rs), (is)) -#define jit_bger_ul(label, s1, s2) jit_bger_ui((label), (s1), (s2)) -#define jit_bgei_ul(label, rs, is) jit_bgei_ui((label), (rs), (is)) -#define jit_boaddr_ul(label, s1, s2) jit_boaddr_ui((label), (s1), (s2)) -#define jit_boaddi_ul(label, rs, is) jit_boaddi_ui((label), (rs), (is)) -#define jit_bosubr_ul(label, s1, s2) jit_bosubr_ui((label), (s1), (s2)) -#define jit_bosubi_ul(label, rs, is) jit_bosubi_ui((label), (rs), (is)) - -#define jit_retval_l(rd) jit_retval_i((rd)) - -#endif +/* Removed the long == int cases, because they aren't the same + for x86_64, and we want to catch missing ones. */ #endif /* __lightning_core_common_h_ */ diff --git a/src/mzscheme/src/lightning/i386/core.h b/src/mzscheme/src/lightning/i386/core.h index e2d87d3014..d313134679 100644 --- a/src/mzscheme/src/lightning/i386/core.h +++ b/src/mzscheme/src/lightning/i386/core.h @@ -44,7 +44,12 @@ #define JIT_V(i) ((i) == 0 ? _EBX : _ESI + (i) - 1) struct jit_local_state { +#ifdef JIT_X86_64 + int long_jumps; + int nextarg_geti; +#else int framesize; +#endif int argssize; }; @@ -53,10 +58,16 @@ struct jit_local_state { ( (s2 == d) ? op1d : \ ( ((s1 == d) ? (void)0 : (void)MOVLrr(s1, d)), op2d ) \ ) +#define jit_qopr_(d, s1, s2, op1d, op2d) \ + ( (s2 == d) ? op1d : \ + ( ((s1 == d) ? (void)0 : (void)MOVQrr(s1, d)), op2d ) \ + ) /* 3-parameter operation, with immediate */ #define jit_op_(d, s1, op2d) \ - ((s1 == d) ? op2d : (MOVLrr(s1, d), op2d)) \ + ((s1 == d) ? op2d : (MOVLrr(s1, d), op2d)) +#define jit_qop_(d, s1, op2d) \ + ((s1 == d) ? op2d : (MOVQrr(s1, d), op2d)) /* 3-parameter operation, optimizable */ #define jit_opo_(d, s1, s2, op1d, op2d, op12d) \ @@ -92,11 +103,15 @@ struct jit_local_state { /* For BLT, BLE, ... */ #define jit_bra_r(s1, s2, op) (CMPLrr(s2, s1), op, _jit.x.pc) +#define jit_bra_qr(s1, s2, op) (CMPQrr(s2, s1), op, _jit.x.pc) #define jit_bra_i(rs, is, op) (CMPLir(is, rs), op, _jit.x.pc) +#define jit_bra_l(rs, is, op) (CMPQir(is, rs), op, _jit.x.pc) /* When CMP with 0 can be replaced with TEST */ #define jit_bra_i0(rs, is, op, op0) \ ( (is) == 0 ? (TESTLrr(rs, rs), op0, _jit.x.pc) : (CMPLir(is, rs), op, _jit.x.pc)) +#define jit_bra_l0(rs, is, op, op0) \ + ( (is) == 0 ? (TESTQrr(rs, rs), op0, _jit.x.pc) : (CMPQir(is, rs), op, _jit.x.pc)) /* Used to implement ldc, stc, ... */ #define jit_check8(rs) ( (rs) <= _EBX ) @@ -110,11 +125,19 @@ struct jit_local_state { : jit_replace(_EBX, rs, _EAX, MOVBrm(_AL, dd, db, di, ds))) /* Reduce arguments of XOR/OR/TEST */ +#ifdef JIT_X86_64 +# define JIT_CAN_16 0 +#else +# define JIT_CAN_16 1 +#endif #define jit_reduce_(op) op #define jit_reduce(op, is, rs) \ (_u8P(is) && jit_check8(rs) ? jit_reduce_(op##Bir(is, jit_reg8(rs))) : \ - (_u16P(is) ? jit_reduce_(op##Wir(is, jit_reg16(rs))) : \ + (_u16P(is) && JIT_CAN_16 ? jit_reduce_(op##Wir(is, jit_reg16(rs))) : \ jit_reduce_(op##Lir(is, rs)) )) +#define jit_reduceQ(op, is, rs) \ + (_u8P(is) && jit_check8(rs) ? jit_reduce_(op##Bir(is, jit_reg8(rs))) : \ + jit_reduce_(op##Qir(is, rs)) ) /* Helper macros for MUL/DIV/IDIV */ #define jit_might(d, s1, op) \ @@ -207,9 +230,19 @@ struct jit_local_state { #define jit_subxi_i(d, rs, is) jit_op_ ((d), (rs), SBBLir((is), (d)) ) #define jit_xorr_i(d, s1, s2) jit_opr_((d), (s1), (s2), XORLrr((s1), (d)), XORLrr((s2), (d)) ) +#define jit_addi_l(d, rs, is) jit_opi_((d), (rs), ADDQir((is), (d)), LEAQmr((is), (rs), 0, 0, (d)) ) +#define jit_addr_l(d, s1, s2) jit_opo_((d), (s1), (s2), ADDQrr((s2), (d)), ADDQrr((s1), (d)), LEAQmr(0, (s1), (s2), 1, (d)) ) +#define jit_andi_l(d, rs, is) jit_qop_ ((d), (rs), ANDQir((is), (d)) ) +#define jit_andr_l(d, s1, s2) jit_qopr_((d), (s1), (s2), ANDQrr((s1), (d)), ANDQrr((s2), (d)) ) +#define jit_orr_l(d, s1, s2) jit_qopr_((d), (s1), (s2), ORQrr((s1), (d)), ORQrr((s2), (d)) ) +#define jit_subr_l(d, s1, s2) jit_qopr_((d), (s1), (s2), (SUBQrr((s1), (d)), NEGQr(d)), SUBQrr((s2), (d)) ) +#define jit_xorr_l(d, s1, s2) jit_qopr_((d), (s1), (s2), XORQrr((s1), (d)), XORQrr((s2), (d)) ) + /* These can sometimes use byte or word versions! */ #define jit_ori_i(d, rs, is) jit_op_ ((d), (rs), jit_reduce(OR, (is), (d)) ) #define jit_xori_i(d, rs, is) jit_op_ ((d), (rs), jit_reduce(XOR, (is), (d)) ) +#define jit_ori_l(d, rs, is) jit_qop_ ((d), (rs), jit_reduceQ(OR, (is), (d)) ) +#define jit_xori_l(d, rs, is) jit_qop_ ((d), (rs), jit_reduceQ(XOR, (is), (d)) ) #define jit_muli_i(d, rs, is) jit_op_ ((d), (rs), IMULLir((is), (d)) ) #define jit_mulr_i(d, s1, s2) jit_opr_((d), (s1), (s2), IMULLrr((s1), (d)), IMULLrr((s2), (d)) ) @@ -257,27 +290,84 @@ struct jit_local_state { #define jit_rshr_i(d, r1, r2) jit_replace((r1), (r2), _ECX, jit_op_ ((d), (r1), SARLrr(_CL, (d)) )) #define jit_rshr_ui(d, r1, r2) jit_replace((r1), (r2), _ECX, jit_op_ ((d), (r1), SHRLrr(_CL, (d)) )) +#define jit_lshi_l(d, rs, is) ((is) <= 3 ? LEAQmr(0, 0, (rs), 1 << (is), (d)) : jit_qop_ ((d), (rs), SHLQir((is), (d)) )) +#define jit_rshi_l(d, rs, is) jit_qop_ ((d), (rs), SARQir((is), (d)) ) +#define jit_rshi_ul(d, rs, is) jit_qop_ ((d), (rs), SHRQir((is), (d)) ) +#define jit_lshr_l(d, r1, r2) jit_replace((r1), (r2), _ECX, jit_qop_ ((d), (r1), SHLQrr(_CL, (d)) )) +#define jit_rshr_l(d, r1, r2) jit_replace((r1), (r2), _ECX, jit_qop_ ((d), (r1), SARQrr(_CL, (d)) )) +#define jit_rshr_ul(d, r1, r2) jit_replace((r1), (r2), _ECX, jit_qop_ ((d), (r1), SHRQrr(_CL, (d)) )) + /* Stack */ #define jit_pushr_i(rs) PUSHLr(rs) #define jit_popr_i(rs) POPLr(rs) -#define jit_prolog(n) (_jitl.framesize = 8, PUSHLr(_EBP), MOVLrr(_ESP, _EBP), PUSHLr(_EBX), PUSHLr(_ESI), PUSHLr(_EDI)) +#define jit_pushr_l(rs) jit_pushr_i(rs) +#define jit_popr_l(rs) jit_popr_i(rs) + +#ifdef JIT_X86_64 +# define jit_base_prolog() (PUSHQr(_EBP), MOVQrr(_ESP, _EBP), PUSHQr(_EBX), PUSHQr(_R12), PUSHQr(_R13)) +# define jit_prolog(n) (_jitl.nextarg_geti = 0, jit_base_prolog()) +#else +# define jit_base_prolog() (PUSHLr(_EBP), MOVLrr(_ESP, _EBP), PUSHLr(_EBX), PUSHLr(_ESI), PUSHLr(_EDI)) +# define jit_prolog(n) (_jitl.framesize = 8, jit_base_prolog()) +#endif /* The += allows for stack pollution */ -#ifdef _CALL_DARWIN - /* Stack must stay 16-byte aligned: */ -# define jit_prepare_i(ni) (((ni & 0x3) ? SUBLir(4 * ((((ni) + 3) & ~(0x3)) - (ni)), JIT_SP) : (void)0), _jitl.argssize += (((ni) + 3) & ~(0x3))) +#ifdef JIT_X86_64 +/* Stack isn't used for arguments: */ +# define jit_prepare_i(ni) (_jitl.argssize = 0) #else -# define jit_prepare_i(ni) (_jitl.argssize += (ni)) +# ifdef _CALL_DARWIN + /* Stack must stay 16-byte aligned: */ +# define jit_prepare_i(ni) (((ni & 0x3) \ + ? SUBLir(4 * ((((ni) + 3) & ~(0x3)) - (ni)), JIT_SP) \ + : (void)0), \ + _jitl.argssize += (((ni) + 3) & ~(0x3))) +# else +# define jit_prepare_i(ni) (_jitl.argssize += (ni)) +# endif #endif #define jit_prepare_f(nf) (_jitl.argssize += (nf)) #define jit_prepare_d(nd) (_jitl.argssize += 2 * (nd)) -#define jit_pusharg_i(rs) PUSHLr(rs) -#define jit_finish(sub) ((void)jit_calli((sub)), ADDLir(4 * _jitl.argssize, JIT_SP), _jitl.argssize = 0) -#define jit_finishr(reg) (jit_callr((reg)), ADDLir(4 * _jitl.argssize, JIT_SP), _jitl.argssize = 0) +#ifdef JIT_X86_64 +# define jit_pusharg_i(rs) (_jitl.argssize++, MOVQrr(rs, JIT_CALLTMPSTART + _jitl.argssize - 1)) +# define jit_finish(sub) (jit_shift_args(), (void)jit_calli((sub)), jit_restore_locals()) +# define jit_reg_is_arg(reg) ((reg == _EDI) || (reg ==_ESI) || (reg == _EDX)) +# define jit_finishr(reg) ((jit_reg_is_arg((reg)) ? MOVQrr(reg, JIT_REXTMP) : (void)0), \ + jit_shift_args(), \ + jit_reg_is_arg((reg)) ? CALQsr((JIT_REXTMP)) : jit_callr((reg)), \ + jit_restore_locals()) +/* R12 and R13 are callee-save, instead of EDI and ESI */ +# define jit_shift_args() \ + (MOVQrr(_ESI, _R12), MOVQrr(_EDI, _R13), \ + (_jitl.argssize-- \ + ? (MOVQrr(JIT_CALLTMPSTART + _jitl.argssize, jit_arg_reg_order[0]), \ + (_jitl.argssize-- \ + ? (MOVQrr(JIT_CALLTMPSTART + _jitl.argssize, jit_arg_reg_order[1]), \ + (_jitl.argssize-- \ + ? MOVQrr(JIT_CALLTMPSTART, jit_arg_reg_order[2]) \ + : (void)0)) \ + : (void)0)) \ + : (void)0)) +# define jit_restore_locals() \ + (MOVQrr(_R12, _ESI), MOVQrr(_R13, _EDI)) +#else +# define jit_pusharg_i(rs) PUSHLr(rs) +# define jit_finish(sub) ((void)jit_calli((sub)), ADDLir(sizeof(long) * _jitl.argssize, JIT_SP), _jitl.argssize = 0) +# define jit_finishr(reg) (jit_callr((reg)), ADDLir(sizeof(long) * _jitl.argssize, JIT_SP), _jitl.argssize = 0) +#endif +#define jit_pusharg_l(rs) jit_pusharg_i(rs) #define jit_retval_i(rd) ((void)jit_movr_i ((rd), _EAX)) +#define jit_retval_l(rd) ((void)jit_movr_l ((rd), _EAX)) +#ifdef JIT_X86_64 +#define jit_arg_i() (_jitl.nextarg_geti++) +#define jit_arg_l() (_jitl.nextarg_geti++) +#define jit_arg_p() (_jitl.nextarg_geti++) +#define jit_arg_reg(p) (jit_arg_reg_order[p]) +static int jit_arg_reg_order[] = { _EDI, _ESI, _EDX, _ECX }; +#else #define jit_arg_c() ((_jitl.framesize += sizeof(int)) - sizeof(int)) #define jit_arg_uc() ((_jitl.framesize += sizeof(int)) - sizeof(int)) #define jit_arg_s() ((_jitl.framesize += sizeof(int)) - sizeof(int)) @@ -287,18 +377,26 @@ struct jit_local_state { #define jit_arg_l() ((_jitl.framesize += sizeof(long)) - sizeof(long)) #define jit_arg_ul() ((_jitl.framesize += sizeof(long)) - sizeof(long)) #define jit_arg_p() ((_jitl.framesize += sizeof(long)) - sizeof(long)) +#endif #define jit_arg_f() ((_jitl.framesize += sizeof(float)) - sizeof(float)) #define jit_arg_d() ((_jitl.framesize += sizeof(double)) - sizeof(double)) /* Unary */ #define jit_negr_i(d, rs) jit_opi_((d), (rs), NEGLr(d), (XORLrr((d), (d)), SUBLrr((rs), (d))) ) -#define jit_negr_l(d, rs) jit_opi_((d), (rs), NEGLr(d), (XORLrr((d), (d)), SUBLrr((rs), (d))) ) +#define jit_negr_l(d, rs) jit_opi_((d), (rs), NEGQr(d), (XORQrr((d), (d)), SUBQrr((rs), (d))) ) #define jit_movr_i(d, rs) ((void)((rs) == (d) ? 0 : MOVLrr((rs), (d)))) #define jit_movi_i(d, is) ((is) ? MOVLir((is), (d)) : XORLrr ((d), (d)) ) -#define jit_movi_p(d, is) (MOVLir((is), (d)), _jit.x.pc) -#define jit_patch_movi(pa,pv) (*_PSL((pa) - 4) = _jit_SL((pv))) +#define jit_movr_l(d, rs) ((void)((rs) == (d) ? 0 : MOVQrr((rs), (d)))) +#define jit_movi_l(d, is) ((is) \ + ? (_u32P((long)(is)) \ + ? MOVLir((is), (d)) \ + : MOVQir((is), (d))) \ + : XORLrr ((d), (d)) ) +#define jit_movi_p(d, is) jit_movi_l(d, ((long)(is))) +#define jit_patchable_movi_p(d, is) (MOVQir((is), (d)), _jit.x.pc) +#define jit_patch_movi(pa,pv) (*_PSL((pa) - sizeof(long)) = _jit_SL((pv))) #define jit_ntoh_ui(d, rs) jit_op_((d), (rs), BSWAPLr(d)) #define jit_ntoh_us(d, rs) jit_op_((d), (rs), RORWir(8, d)) @@ -344,6 +442,23 @@ struct jit_local_state { #define jit_boaddr_ui(label, s1, s2) (ADDLrr((s2), (s1)), JCm(label,0,0,0), _jit.x.pc) #define jit_bosubr_ui(label, s1, s2) (SUBLrr((s2), (s1)), JCm(label,0,0,0), _jit.x.pc) +#define jit_bltr_l(label, s1, s2) jit_bra_qr((s1), (s2), JLm(label, 0,0,0) ) +#define jit_bler_l(label, s1, s2) jit_bra_qr((s1), (s2), JLEm(label,0,0,0) ) +#define jit_bgtr_l(label, s1, s2) jit_bra_qr((s1), (s2), JGm(label, 0,0,0) ) +#define jit_bger_l(label, s1, s2) jit_bra_qr((s1), (s2), JGEm(label,0,0,0) ) +#define jit_beqr_l(label, s1, s2) jit_bra_qr((s1), (s2), JEm(label, 0,0,0) ) +#define jit_bner_l(label, s1, s2) jit_bra_qr((s1), (s2), JNEm(label,0,0,0) ) +#define jit_bltr_ul(label, s1, s2) jit_bra_qr((s1), (s2), JBm(label, 0,0,0) ) +#define jit_bler_ul(label, s1, s2) jit_bra_qr((s1), (s2), JBEm(label,0,0,0) ) +#define jit_bgtr_ul(label, s1, s2) jit_bra_qr((s1), (s2), JAm(label, 0,0,0) ) +#define jit_bger_ul(label, s1, s2) jit_bra_qr((s1), (s2), JAEm(label,0,0,0) ) +#define jit_bmsr_l(label, s1, s2) (TESTQrr((s1), (s2)), JNZm(label,0,0,0), _jit.x.pc) +#define jit_bmcr_l(label, s1, s2) (TESTQrr((s1), (s2)), JZm(label,0,0,0), _jit.x.pc) +#define jit_boaddr_l(label, s1, s2) (ADDQrr((s2), (s1)), JOm(label,0,0,0), _jit.x.pc) +#define jit_bosubr_l(label, s1, s2) (SUBQrr((s2), (s1)), JOm(label,0,0,0), _jit.x.pc) +#define jit_boaddr_ul(label, s1, s2) (ADDQrr((s2), (s1)), JCm(label,0,0,0), _jit.x.pc) +#define jit_bosubr_ul(label, s1, s2) (SUBQrr((s2), (s1)), JCm(label,0,0,0), _jit.x.pc) + #define jit_blti_i(label, rs, is) jit_bra_i0((rs), (is), JLm(label, 0,0,0), JSm(label, 0,0,0) ) #define jit_blei_i(label, rs, is) jit_bra_i ((rs), (is), JLEm(label,0,0,0) ) #define jit_bgti_i(label, rs, is) jit_bra_i ((rs), (is), JGm(label, 0,0,0) ) @@ -359,15 +474,43 @@ struct jit_local_state { #define jit_boaddi_ui(label, rs, is) (ADDLir((is), (rs)), JCm(label,0,0,0), _jit.x.pc) #define jit_bosubi_ui(label, rs, is) (SUBLir((is), (rs)), JCm(label,0,0,0), _jit.x.pc) +#define jit_blti_l(label, rs, is) jit_bra_l0((rs), (is), JLm(label, 0,0,0), JSm(label, 0,0,0) ) +#define jit_blei_l(label, rs, is) jit_bra_l ((rs), (is), JLEm(label,0,0,0) ) +#define jit_bgti_l(label, rs, is) jit_bra_l ((rs), (is), JGm(label, 0,0,0) ) +#define jit_bgei_l(label, rs, is) jit_bra_l0((rs), (is), JGEm(label,0,0,0), JNSm(label,0,0,0) ) +#define jit_beqi_l(label, rs, is) jit_bra_l0((rs), (is), JEm(label, 0,0,0), JEm(label, 0,0,0) ) +#define jit_bnei_l(label, rs, is) jit_bra_l0((rs), (is), JNEm(label,0,0,0), JNEm(label,0,0,0) ) +#define jit_blti_ul(label, rs, is) jit_bra_l ((rs), (is), JBm(label, 0,0,0) ) +#define jit_blei_ul(label, rs, is) jit_bra_l0((rs), (is), JBEm(label,0,0,0), JEm(label, 0,0,0) ) +#define jit_bgti_ul(label, rs, is) jit_bra_l0((rs), (is), JAm(label, 0,0,0), JNEm(label,0,0,0) ) +#define jit_bgei_ul(label, rs, is) jit_bra_l ((rs), (is), JAEm(label,0,0,0) ) +#define jit_boaddi_l(label, rs, is) (ADDQir((is), (rs)), JOm(label,0,0,0), _jit.x.pc) +#define jit_bosubi_l(label, rs, is) (SUBQir((is), (rs)), JOm(label,0,0,0), _jit.x.pc) +#define jit_boaddi_ul(label, rs, is) (ADDQir((is), (rs)), JCm(label,0,0,0), _jit.x.pc) +#define jit_bosubi_ul(label, rs, is) (SUBQir((is), (rs)), JCm(label,0,0,0), _jit.x.pc) + #define jit_bmsi_i(label, rs, is) (jit_reduce(TEST, (is), (rs)), JNZm(label,0,0,0), _jit.x.pc) #define jit_bmci_i(label, rs, is) (jit_reduce(TEST, (is), (rs)), JZm(label,0,0,0), _jit.x.pc) +#define jit_bmsi_l(label, rs, is) jit_bmsi_i(label, rs, is) +#define jit_bmci_l(label, rs, is) jit_bmci_i(label, rs, is) + #define jit_jmpi(label) (JMPm( ((unsigned long) (label)), 0, 0, 0), _jit.x.pc) #define jit_calli(label) (CALLm( ((unsigned long) (label)), 0, 0, 0), _jit.x.pc) #define jit_callr(reg) (CALLsr(reg)) #define jit_jmpr(reg) JMPsr(reg) -#define jit_patch_at(jump_pc,v) (*_PSL((jump_pc) - 4) = _jit_SL((jit_insn *)(v) - (jump_pc))) -#define jit_ret() (POPLr(_EDI), POPLr(_ESI), POPLr(_EBX), POPLr(_EBP), RET_()) +#ifdef JIT_X86_64 +#define jit_patch_long_at(jump_pc,v) (*_PSL((jump_pc) - sizeof(long)) = _jit_SL((jit_insn *)(v))) +# define jit_patch_short_at(jump_pc,v) (*_PSI((jump_pc) - sizeof(int)) = _jit_SI((jit_insn *)(v) - (jump_pc))) +# define jit_patch_branch_at(jump_pc,v) (_jitl.long_jumps ? jit_patch_long_at((jump_pc)-3, v) : jit_patch_short_at(jump_pc, v)) +# define jit_patch_ucbranch_at(jump_pc,v) (_jitl.long_jumps ? jit_patch_long_at((jump_pc)-3, v) : jit_patch_short_at(jump_pc, v)) +# define jit_ret() (POPQr(_R13), POPQr(_R12), POPQr(_EBX), POPQr(_EBP), RET_()) +#else +#define jit_patch_long_at(jump_pc,v) (*_PSL((jump_pc) - sizeof(long)) = _jit_SL((jit_insn *)(v) - (jump_pc))) +# define jit_patch_branch_at(jump_pc,v) jit_patch_long_at(jump_pc, v) +# define jit_patch_ucbranch_at(jump_pc,v) jit_patch_long_at(jump_pc, v) +# define jit_ret() (POPLr(_EDI), POPLr(_ESI), POPLr(_EBX), POPLr(_EBP), RET_()) +#endif /* Memory */ #define jit_ldi_c(d, is) MOVSBLmr((is), 0, 0, 0, (d)) @@ -410,6 +553,16 @@ struct jit_local_state { #define jit_stxr_i(d1, d2, rs) MOVLrm((rs), 0, (d1), (d2), 1) #define jit_stxi_i(id, rd, rs) MOVLrm((rs), (id), (rd), 0, 0) +#define jit_ldi_l(d, is) MOVQmr((is), 0, 0, 0, (d)) +#define jit_ldr_l(d, rs) MOVQmr(0, (rs), 0, 0, (d)) +#define jit_ldxr_l(d, s1, s2) MOVQmr(0, (s1), (s2), 1, (d)) +#define jit_ldxi_l(d, rs, is) MOVQmr((is), (rs), 0, 0, (d)) + +#define jit_sti_l(id, rs) MOVQrm((rs), (id), 0, 0, 0) +#define jit_str_l(rd, rs) MOVQrm((rs), 0, (rd), 0, 0) +#define jit_stxr_l(d1, d2, rs) MOVQrm((rs), 0, (d1), (d2), 1) +#define jit_stxi_l(id, rd, rs) MOVQrm((rs), (id), (rd), 0, 0) + /* Extra */ #define jit_nop() NOP_() diff --git a/src/mzscheme/src/lightning/i386/funcs.h b/src/mzscheme/src/lightning/i386/funcs.h index 1e247049ff..ce792eadba 100644 --- a/src/mzscheme/src/lightning/i386/funcs.h +++ b/src/mzscheme/src/lightning/i386/funcs.h @@ -53,11 +53,11 @@ jit_flush_code(void *dest, void *end) jit_flush_code as an mprotect. */ #ifdef __linux__ static unsigned long prev_page = 0, prev_length = 0; - int page, length; + long page, length; #ifdef PAGESIZE - const int page_size = PAGESIZE; + const long page_size = PAGESIZE; #else - static int page_size = -1; + static long page_size = -1; if (page_size == -1) page_size = sysconf (_SC_PAGESIZE); #endif diff --git a/src/mzscheme/src/lightning/ppc/core-common.h b/src/mzscheme/src/lightning/ppc/core-common.h index 456778b58b..f20e120aad 100644 --- a/src/mzscheme/src/lightning/ppc/core-common.h +++ b/src/mzscheme/src/lightning/ppc/core-common.h @@ -33,6 +33,25 @@ #ifndef __lightning_core_common_h #define __lightning_core_common_h_ +/* jit_code: union of many possible function pointer types. Returned + * by jit_get_ip(). + */ +typedef union jit_code { + char *ptr; + void (*vptr)(void); + char (*cptr)(void); + unsigned char (*ucptr)(void); + short (*sptr)(void); + unsigned short (*usptr)(void); + int (*iptr)(void); + unsigned int (*uiptr)(void); + long (*lptr)(void); + unsigned long (*ulptr)(void); + void * (*pptr)(void); + float (*fptr)(void); + double (*dptr)(void); +} jit_code; + typedef struct { union { jit_insn *pc; @@ -40,6 +59,7 @@ typedef struct { _us *us_pc; _ui *ui_pc; _ul *ul_pc; + jit_code code; } x; struct jit_fp *fp; struct jit_local_state jitl; @@ -63,7 +83,7 @@ static jit_state _jit; #define _jitl _jit.jitl -#define jit_get_ip() (*(jit_code *) &_jit.x.pc) +#define jit_get_ip() (_jit.x.code) #define jit_set_ip(ptr) (_jit.x.pc = (ptr), jit_get_ip ()) #define jit_get_label() (_jit.x.pc) #define jit_forward() (_jit.x.pc) @@ -77,24 +97,6 @@ static jit_state _jit; #define jit_align(n) #endif -/* jit_code: union of many possible function pointer types. Returned - * by jit_get_ip(). - */ -typedef union jit_code { - char *ptr; - void (*vptr)(void); - char (*cptr)(void); - unsigned char (*ucptr)(void); - short (*sptr)(void); - unsigned short (*usptr)(void); - int (*iptr)(void); - unsigned int (*uiptr)(void); - long (*lptr)(void); - unsigned long (*ulptr)(void); - void * (*pptr)(void); - float (*fptr)(void); - double (*dptr)(void); -} jit_code; #ifndef jit_fill_delay_after #define jit_fill_delay_after(branch) (branch) @@ -167,6 +169,7 @@ typedef union jit_code { #define jit_movi_p(d, is) (jit_movi_ul((d), (long) (is)), _jit.x.pc) #endif +#define jit_patchable_movi_p(r, i) jit_movi_p(r, i) #define jit_patch(pv) jit_patch_at ((pv), (_jit.x.pc)) #ifndef jit_addci_i