/* ***** BEGIN LICENSE BLOCK ***** * Version: MPL 1.1/GPL 2.0/LGPL 2.1 * * The contents of this file are subject to the Mozilla Public License Version * 1.1 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.mozilla.org/MPL/ * * Software distributed under the License is distributed on an "AS IS" basis, * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License * for the specific language governing rights and limitations under the * License. * * The Original Code is [Open Source Virtual Machine]. * * The Initial Developer of the Original Code is * Adobe System Incorporated. * Portions created by the Initial Developer are Copyright (C) 2004-2007 * the Initial Developer. All Rights Reserved. * * Contributor(s): * Adobe AS3 Team * * Alternatively, the contents of this file may be used under the terms of * either the GNU General Public License Version 2 or later (the "GPL"), or * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), * in which case the provisions of the GPL or the LGPL are applicable instead * of those above. If you wish to allow use of your version of this file only * under the terms of either the GPL or the LGPL, and not to allow others to * use your version of this file under the terms of the MPL, indicate your * decision by deleting the provisions above and replace them with the notice * and other provisions required by the GPL or the LGPL. If you do not delete * the provisions above, a recipient may use your version of this file under * the terms of any one of the MPL, the GPL or the LGPL. * * ***** END LICENSE BLOCK ***** */ #ifndef __nanojit_NativeAMD64__ #define __nanojit_NativeAMD64__ #include namespace nanojit { const int NJ_LOG2_PAGE_SIZE = 12; // 4K const int NJ_MAX_REGISTERS = 32; // gpregs, x87 regs, xmm regs const int NJ_STACK_OFFSET = 0; // WARNING: setting this allows the NJ to growth memory as needed without bounds const bool NJ_UNLIMITED_GROWTH = true; #define NJ_MAX_STACK_ENTRY 256 #define NJ_MAX_PARAMETERS 1 /* Stack is always aligned to 16-bit on x64 */ const int NJ_ALIGN_STACK = 16; typedef uint8_t NIns; // These are used as register numbers in various parts of the code typedef enum { // general purpose 32bit regs RAX = 0, // return value, scratch RCX = 1, // this/arg0, scratch RDX = 2, // arg1, return-msw, scratch RBX = 3, RSP = 4, // stack pointer RBP = 5, // frame pointer RSI = 6, RDI = 7, SP = RSP, // alias SP to RSP for convenience FP = RBP, // alias FP to RBP for convenience R8 = 8, R9 = 9, R10 = 10, R11 = 11, R12 = 12, R13 = 13, R14 = 14, R15 = 15, XMM0 = 16, XMM1 = 17, XMM2 = 18, XMM3 = 19, XMM4 = 20, XMM5 = 21, XMM6 = 22, XMM7 = 23, XMM8 = 24, XMM9 = 25, XMM10 = 26, XMM11 = 27, XMM12 = 28, XMM13 = 29, XMM14 = 30, XMM15 = 31, FirstReg = 0, LastReg = 31, UnknownReg = 32 } Register; typedef int RegisterMask; /* RBX, R13-R15 */ static const int NumSavedRegs = 3; static const RegisterMask SavedRegs = /*(1<= INT_MIN) { \ JMP_long_nochk_offset(d); \ } else { \ /* Insert a 64-bit jump... */ \ _nIns -= 8; \ *(intptr_t *)_nIns = intptr_t(tt); \ JMPm_nochk(0); \ } \ } \ overrideProtect = _nIns; \ } #define AMD64_NEEDS_REX(x) (((x) & 8) == 8) #define AMD64_ENCODE_REG(x) ((x) & 7) #define AMD64_MODRM(mode, reg, rm) (uint8_t)(((mode & 3) << 6) | ((reg & 7) << 3) | (rm & 7)) #define AMD64_ALUOP(x) ((x)>>3) /** * Returns a REX prefix. * * q 1 for 64-bit, 0 for default. * mdReg ModRM register field. * mdRm ModRM R/M field, or opcode register field. */ #define AMD64_REX(q,mdReg,mdRm) (uint8_t)(0x40 | (q << 3) | (((mdReg >> 3) & 1) << 2) | ((mdRm >> 3) & 1)) #define AMD64_ADD_RAX 0x05 /* imm */ #define AMD64_ADD_REG_RM 0x03 /* rm/r */ #define AMD64_ADDSD 0xF20F58 /* rm/r */ #define AMD64_ALU8 0x83 /* rm/x imm8 */ #define AMD64_ALU32 0x81 /* rm/x imm32 */ #define AMD64_AND_RAX 0x25 /* imm */ #define AMD64_AND_REG_RM 0x23 /* rm/r */ #define AMD64_CMP_RAX 0x3D /* imm */ #define AMD64_CMP_REG_RM 0x3B /* rm/r */ #define AMD64_CVTSI2SD 0xF20F2A /* rm/r */ #define AMD64_DIVSD 0xF20F5E /* rm/r */ #define AMD64_IMUL_REG_RM_1 0x0F /* see next */ #define AMD64_IMUL_REG_RM_2 0xAF /* rm/r */ #define AMD64_INT3 0xCC /* */ #define AMD64_LEA 0x8D /* rm/r */ #define AMD64_MOV_REG_IMM(x) (0xB8 | AMD64_ENCODE_REG(x)) #define AMD64_MOV_REG_RM 0x8B /* rm/r */ #define AMD64_MOV_RM_REG 0x89 /* rm/r */ #define AMD64_MOV_RM_IMM 0xC7 /* rm/0 imm */ #define AMD64_MOVD_REG_RM 0x660F6E /* rm/r */ #define AMD64_MOVD_RM_REG 0x660F7E /* rm/r */ #define AMD64_MOVSD_REG_RM 0xF20F10 /* rm/r */ #define AMD64_MOVZX 0x0FB6 /* rm/r */ #define AMD64_MOVSZ 0x0FB7 /* rm/r */ #define AMD64_MULSD 0xF20F59 /* rm/r */ #define AMD64_NEG_RM 0xF7 /* rm/3 */ #define AMD64_NOT_RM 0xF7 /* rm/2 */ #define AMD64_OR_RAX 0x0D /* imm */ #define AMD64_OR_REG_RM 0x0B /* rm/r */ #define AMD64_POP_REG(x) (0x58 | AMD64_ENCODE_REG(x)) #define AMD64_PUSH_REG(x) (0x50 | AMD64_ENCODE_REG(x)) #define AMD64_PUSH_RM 0xFF /* rm/6 */ #define AMD64_PUSHF 0x9C /* */ #define AMD64_RET 0xC3 /* */ #define AMD64_SAR_RM_1 0xD1 /* rm/7 */ #define AMD64_SAR_RM 0xD3 /* rm/7 */ #define AMD64_SAR_RM_IMM8 0xC1 /* rm/7 */ #define AMD64_SHL_RM_1 0xD1 /* rm/4 */ #define AMD64_SHL_RM 0xD3 /* rm/4 */ #define AMD64_SHL_RM_IMM8 0xC1 /* rm/4 imm */ #define AMD64_SHR_RM_1 0xD1 /* rm/5 */ #define AMD64_SHR_RM 0xD3 /* rm/5 */ #define AMD64_SHR_RM_IMM8 0xC1 /* rm/5 */ #define AMD64_SUB_RAX 0x2D /* imm */ #define AMD64_SUB_REG_RM 0x2B /* rm/r */ #define AMD64_SUBSD 0xF20F5C /* rm/r */ #define AMD64_TEST_RM_REG 0x85 /* rm/r */ #define AMD64_UCOMISD 0x660F2E /* rm/r */ #define AMD64_XOR_RAX 0x35 /* imm */ #define AMD64_XOR_REG_RM 0x33 /* rm/r */ #define AMD64_XORPD 0x660F57 /* rm/r */ #define IMM32(i) \ _nIns -= 4; \ *((int32_t*)_nIns) = (int32_t)(i) #define IMM64(i) \ _nIns -= 8; \ *((int64_t*)_nIns) = (int64_t)(i) #define AMD64_MODRM_REG(reg, rm) AMD64_MODRM(3, reg, rm) #define AMD64_MODRM_DISP(reg, rm, disp) \ if ((disp) == 0 && (((rm) & 0x7) != 5)) { \ *(--_nIns) = AMD64_MODRM(0, reg, rm); \ } else if (isS8(disp)) { \ *(--_nIns) = int8_t(disp); \ *(--_nIns) = AMD64_MODRM(1, reg, rm); \ } else { \ IMM32(disp); \ *(--_nIns) = AMD64_MODRM(2, reg, rm); \ } #define AMD64_ALU(op, reg, imm, q) \ underrunProtect(7); \ if (isS8(imm)) { \ *(--_nIns) = uint8_t(imm); \ *(--_nIns) = AMD64_MODRM_REG(AMD64_ALUOP(op), reg); \ *(--_nIns) = AMD64_ALU8; \ } else { \ IMM32(imm); \ if (reg == RAX) { \ *(--_nIns) = op; \ } else { \ *(--_nIns) = AMD64_MODRM_REG(AMD64_ALUOP(op), reg); \ *(--_nIns) = AMD64_ALU32; \ } \ } \ if (q || AMD64_NEEDS_REX(reg)) { \ *(--_nIns) = AMD64_REX(q,0,reg); \ } #define AMD64_ALU_MEM(op, reg, disp, imm, q) \ underrunProtect(11); \ if (isS8(imm)) { \ *(--_nIns) = uint8_t(imm); \ } else { \ IMM32(imm); \ } \ AMD64_MODRM_DISP(AMD64_ALUOP(op), reg, disp); \ if (isS8(imm)) { \ *(--_nIns) = AMD64_ALU8; \ } else { \ *(--_nIns) = AMD64_ALU32; \ } \ if (q || AMD64_NEEDS_REX(reg)) { \ *(--_nIns) = AMD64_REX(q,0,reg); \ } #define NOT(r) do { \ underrunProtect(3); \ *(--_nIns) = AMD64_MODRM_REG(2, r); \ *(--_nIns) = AMD64_NOT_RM; \ *(--_nIns) = AMD64_REX(1,0,r); \ asm_output("not %s",gpn(r)); \ } while(0) #define MOVZX8(d,s) do { \ underrunProtect(4); \ *(--_nIns) = AMD64_MODRM_REG(d, s); \ *(--_nIns) = AMD64_MOVZX & 0xFF; \ *(--_nIns) = AMD64_MOVZX >> 8; \ if (AMD64_NEEDS_REX(s) || AMD64_NEEDS_REX(d)) { \ *(--_nIns) = AMD64_REX(0,d,s); \ } \ asm_output("movzx %s,%s", gpn(d),gpn(s)); \ } while(0) #define AMD64_ADDmi(d,b,i,q) do { \ AMD64_ALU_MEM(AMD64_ADD_RAX, b, d, i, q); \ asm_output("add %d(%s), %d", d, gpn(b), i); \ } while(0) #define ADDQmi(d,b,i) do { \ AMD64_ADDmi(d,b,i,1); \ asm_output("add %d(%s), %d", d, gpn(b), i); \ } while (0) #define CMPi(r,i) do { \ AMD64_ALU(AMD64_CMP_RAX, r, i, 0); \ asm_output("cmp %s,%d",gpn(r),i); \ } while(0) #define TEST(d,s) do { \ underrunProtect(3); \ *(--_nIns) = AMD64_MODRM_REG(d, s); \ *(--_nIns) = AMD64_TEST_RM_REG; \ if (AMD64_NEEDS_REX(d) || AMD64_NEEDS_REX(s)) { \ *(--_nIns) = AMD64_REX(0,d,s); \ } \ asm_output("test %s,%s",gpn(d),gpn(s)); \ } while(0) #define TESTQ(d,s) do { \ underrunProtect(3); \ *(--_nIns) = AMD64_MODRM_REG(d, s); \ *(--_nIns) = AMD64_TEST_RM_REG; \ *(--_nIns) = AMD64_REX(1,d,s); \ asm_output("test %s,%s",gpn(d),gpn(s)); \ } while(0) #define CMP(l,r) do { \ underrunProtect(3); \ *(--_nIns) = AMD64_MODRM_REG(l, r); \ *(--_nIns) = AMD64_CMP_REG_RM; \ if (AMD64_NEEDS_REX(l) || AMD64_NEEDS_REX(r)) { \ *(--_nIns) = AMD64_REX(0,l,r); \ } \ asm_output("cmp %s,%s",gpn(l),gpn(r)); \ } while(0) #define CMPQ(l,r) do { \ underrunProtect(3); \ *(--_nIns) = AMD64_MODRM_REG(l, r); \ *(--_nIns) = AMD64_CMP_REG_RM; \ *(--_nIns) = AMD64_REX(1,l,r); \ asm_output("cmp %s,%s",gpn(l),gpn(r)); \ } while(0) #define ADDi(r,i) do { \ AMD64_ALU(AMD64_ADD_RAX, r, i, 0); \ asm_output("add %s,%d",gpn(r),i); \ } while(0) #define ADDQi(r,i) do { \ AMD64_ALU(AMD64_ADD_RAX, r, i, 1); \ asm_output("add %s,%d",gpn(r),i); \ } while(0) #define AMD64_PRIM(op,l,r) \ underrunProtect(3); \ *(--_nIns) = AMD64_MODRM_REG(l, r); \ *(--_nIns) = op; \ if (AMD64_NEEDS_REX(l) || AMD64_NEEDS_REX(r)) \ *(--_nIns) = AMD64_REX(0,l,r); #define AMD64_PRIMQ(op,l,r) \ underrunProtect(3); \ *(--_nIns) = AMD64_MODRM_REG(l, r); \ *(--_nIns) = op; \ *(--_nIns) = AMD64_REX(1,l,r); #define SHR(r,s) do { \ AMD64_PRIM(AMD64_SHR_RM, 5, r); \ asm_output("shr %s,%s",gpn(r),gpn(s)); \ } while(0) #define AND(l,r) do { \ AMD64_PRIM(AMD64_AND_REG_RM, l, r); \ asm_output("and %s,%s",gpn(l),gpn(r)); \ } while(0) #define ANDQ(l,r) do { \ AMD64_PRIMQ(AMD64_AND_REG_RM, l, r); \ asm_output("and %s,%s",gpn(l),gpn(r)); \ } while(0) #define XOR(l,r) do { \ AMD64_PRIM(AMD64_XOR_REG_RM, l, r); \ asm_output("xor %s,%s",gpn(l),gpn(r)); \ } while(0) #define OR(l,r) do { \ AMD64_PRIM(AMD64_OR_REG_RM, l, r); \ asm_output("or %s,%s",gpn(l),gpn(r)); \ } while(0) #define ORQ(l,r) do { \ AMD64_PRIMQ(AMD64_OR_REG_RM, l, r); \ asm_output("or %s,%s",gpn(l),gpn(r)); \ } while(0) #define SHRi(r,i) do { \ if (i == 1) { \ underrunProtect(3); \ *(--_nIns) = AMD64_MODRM_REG(5, r); \ *(--_nIns) = AMD64_SHR_RM_1; \ } else { \ underrunProtect(4); \ *(--_nIns) = uint8_t(i); \ *(--_nIns) = AMD64_MODRM_REG(5, r); \ *(--_nIns) = AMD64_SHR_RM_IMM8; \ } \ if (AMD64_NEEDS_REX(r)) \ *(--_nIns) = AMD64_REX(0,0,r); \ asm_output("shr %s,%d", gpn(r), i); \ } while (0) #define ANDi(r,i) do { \ AMD64_ALU(AMD64_AND_RAX, r, i, 0); \ asm_output("and %s,%d",gpn(r),i); \ } while(0) #define ANDQi(r,i) do { \ AMD64_ALU(AMD64_AND_RAX, r, i, 1); \ asm_output("and %s,%d",gpn(r),i); \ } while(0) #define ORQi(r,i) do { \ AMD64_ALU(AMD64_OR_RAX, r, i, 1); \ asm_output("or %s,%d",gpn(r),i); \ } while(0) #define XORi(r,i) do { \ AMD64_ALU(AMD64_XOR_RAX, r, i, 0); \ asm_output("xor %s,%d",gpn(r),i); \ } while(0) #define ORi(r,i) do { \ AMD64_ALU(AMD64_OR_RAX, r, i, 0); \ asm_output("or %s,%d",gpn(r),i); \ } while(0) #define MUL(l,r) do { \ underrunProtect(4); \ *(--_nIns) = AMD64_MODRM_REG(l, r); \ *(--_nIns) = AMD64_IMUL_REG_RM_2; \ *(--_nIns) = AMD64_IMUL_REG_RM_1; \ if (AMD64_NEEDS_REX(l) || AMD64_NEEDS_REX(r)) \ *(--_nIns) = AMD64_REX(0,l,r); \ asm_output("mul %s,%s", gpn(l), gpn(r)); \ } while (0) #define NEG(r) do { \ AMD64_PRIM(AMD64_NEG_RM, 3, r); \ asm_output("neg %s",gpn(r)); \ } while(0) #define ADD(l,r) do { \ AMD64_PRIM(AMD64_ADD_REG_RM, l, r); \ asm_output("add %s,%s", gpn(l), gpn(r)); \ } while (0) #define ADDQ(l,r) do { \ AMD64_PRIMQ(AMD64_ADD_REG_RM, l, r); \ asm_output("add %s,%s", gpn(l), gpn(r)); \ } while (0) #define SUB(l,r) do { \ AMD64_PRIM(AMD64_SUB_REG_RM, l, r); \ asm_output("sub %s,%s", gpn(l), gpn(r)); \ } while (0) #define SAR(r,s) do { \ AMD64_PRIM(AMD64_SAR_RM, 7, r); \ asm_output("sar %s,%s",gpn(r),gpn(s)); \ } while (0) #define SARi(r,i) do { \ if (i == 1) { \ underrunProtect(3); \ *(--_nIns) = AMD64_MODRM_REG(7, r); \ *(--_nIns) = AMD64_SAR_RM_1; \ } else { \ underrunProtect(4); \ *(--_nIns) = uint8_t(i); \ *(--_nIns) = AMD64_MODRM_REG(7, r); \ *(--_nIns) = AMD64_SAR_RM_IMM8; \ } \ if (AMD64_NEEDS_REX(r)) \ *(--_nIns) = AMD64_REX(0,0,r); \ asm_output("sar %s,%d", gpn(r), i); \ } while (0) #define SHLi(r,i) do { \ if (i == 1) { \ underrunProtect(3); \ *(--_nIns) = AMD64_MODRM_REG(4, r); \ *(--_nIns) = AMD64_SHL_RM_1; \ } else { \ underrunProtect(4); \ *(--_nIns) = uint8_t(i); \ *(--_nIns) = AMD64_MODRM_REG(4, r); \ *(--_nIns) = AMD64_SHL_RM_IMM8; \ } \ if (AMD64_NEEDS_REX(r)) \ *(--_nIns) = AMD64_REX(0,0,r); \ asm_output("shl %s,%d", gpn(r), i); \ } while (0) #define SHLQi(r,i) do { \ if (i == 1) { \ underrunProtect(3); \ *(--_nIns) = AMD64_MODRM_REG(4, r); \ *(--_nIns) = AMD64_SHL_RM_1; \ } else { \ underrunProtect(4); \ *(--_nIns) = uint8_t(i); \ *(--_nIns) = AMD64_MODRM_REG(4, r); \ *(--_nIns) = AMD64_SHL_RM_IMM8; \ } \ *(--_nIns) = AMD64_REX(1,0,r); \ asm_output("shl %s,%d", gpn(r), i); \ } while (0) #define SHL(r,s) do { \ AMD64_PRIM(AMD64_SHL_RM, 4, r); \ asm_output("shl %s,%s",gpn(r),gpn(s)); \ } while (0) #define AMD64_SUBi(r,i,q) do { \ AMD64_ALU(AMD64_SUB_RAX, r, i, q); \ asm_output("sub %s,%d",gpn(r),i); \ } while (0) #define SUBi(r,i) AMD64_SUBi(r, i, 0) #define SUBQi(r,i) AMD64_SUBi(r, i, 1) #define MR(d,s) do { \ underrunProtect(3); \ *(--_nIns) = AMD64_MODRM_REG(d, s); \ *(--_nIns) = AMD64_MOV_REG_RM; \ *(--_nIns) = AMD64_REX(1,d,s); \ asm_output("mov %s,%s",gpn(d),gpn(s)); \ } while (0) #define LEA(r,d,b) do { \ underrunProtect(8); \ AMD64_MODRM_DISP(r, b, d); \ *(--_nIns) = AMD64_LEA; \ if (AMD64_NEEDS_REX(r) || AMD64_NEEDS_REX(b)) { \ *(--_nIns) = AMD64_REX(0,r,b); \ } \ asm_output("lea %s,%d(%s)",gpn(r),d,gpn(b)); \ } while(0) #define LEAQ(r,d,b) do { \ underrunProtect(8); \ AMD64_MODRM_DISP(r, b, d); \ *(--_nIns) = AMD64_LEA; \ *(--_nIns) = AMD64_REX(1,r,b); \ asm_output("lea %s,%d(%s)",gpn(r),d,gpn(b)); \ } while(0) #define AMD64_SETCC(op, r) \ underrunProtect(4); \ *(--_nIns) = AMD64_MODRM_REG(0,r); \ *(--_nIns) = op & 0xFF; \ *(--_nIns) = op >> 8; \ if (AMD64_NEEDS_REX(r)) { \ *(--_nIns) = AMD64_REX(0,0,r); \ } #define SETE(r) do { AMD64_SETCC(0x0f94,(r)); asm_output("sete %s",gpn(r)); } while(0) #define SETNP(r) do { AMD64_SETCC(0x0f9B,(r)); asm_output("setnp %s",gpn(r)); } while(0) #define SETL(r) do { AMD64_SETCC(0x0f9C,(r)); asm_output("setl %s",gpn(r)); } while(0) #define SETLE(r) do { AMD64_SETCC(0x0f9E,(r)); asm_output("setle %s",gpn(r)); } while(0) #define SETG(r) do { AMD64_SETCC(0x0f9F,(r)); asm_output("setg %s",gpn(r)); } while(0) #define SETGE(r) do { AMD64_SETCC(0x0f9D,(r)); asm_output("setge %s",gpn(r)); } while(0) #define SETB(r) do { AMD64_SETCC(0x0f92,(r)); asm_output("setb %s",gpn(r)); } while(0) #define SETBE(r) do { AMD64_SETCC(0x0f96,(r)); asm_output("setbe %s",gpn(r)); } while(0) #define SETA(r) do { AMD64_SETCC(0x0f97,(r)); asm_output("seta %s",gpn(r)); } while(0) #define SETAE(r) do { AMD64_SETCC(0x0f93,(r)); asm_output("setae %s",gpn(r)); } while(0) #define SETC(r) do { AMD64_SETCC(0x0f90,(r)); asm_output("setc %s",gpn(r)); } while(0) #define SETO(r) do { AMD64_SETCC(0x0f92,(r)); asm_output("seto %s",gpn(r)); } while(0) #define AMD64_CMOV(op, dr, sr) \ underrunProtect(4); \ *(--_nIns) = AMD64_MODRM_REG(dr, sr); \ *(--_nIns) = op & 0xFF; \ *(--_nIns) = op >> 8; \ if (AMD64_NEEDS_REX(dr) || AMD64_NEEDS_REX(sr)) { \ *(--_nIns) = AMD64_REX(0,dr,sr); \ } #define AMD64_CMOVQ(op, dr, sr) \ underrunProtect(4); \ *(--_nIns) = AMD64_MODRM_REG(dr, sr); \ *(--_nIns) = op & 0xFF; \ *(--_nIns) = op >> 8; \ *(--_nIns) = AMD64_REX(1,dr,sr); #define MREQ(dr,sr) do { AMD64_CMOV(0x0f44,dr,sr); asm_output("cmove %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRNE(dr,sr) do { AMD64_CMOV(0x0f45,dr,sr); asm_output("cmovne %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRL(dr,sr) do { AMD64_CMOV(0x0f4C,dr,sr); asm_output("cmovl %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRLE(dr,sr) do { AMD64_CMOV(0x0f4E,dr,sr); asm_output("cmovle %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRG(dr,sr) do { AMD64_CMOV(0x0f4F,dr,sr); asm_output("cmovg %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRGE(dr,sr) do { AMD64_CMOV(0x0f4D,dr,sr); asm_output("cmovge %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRB(dr,sr) do { AMD64_CMOV(0x0f42,dr,sr); asm_output("cmovb %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRBE(dr,sr) do { AMD64_CMOV(0x0f46,dr,sr); asm_output("cmovbe %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRA(dr,sr) do { AMD64_CMOV(0x0f47,dr,sr); asm_output("cmova %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRAE(dr,sr) do { AMD64_CMOV(0x0f43,dr,sr); asm_output("cmovae %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRNC(dr,sr) do { AMD64_CMOV(0x0f43,dr,sr); asm_output("cmovnc %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRNO(dr,sr) do { AMD64_CMOV(0x0f41,dr,sr); asm_output("cmovno %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRQEQ(dr,sr) do { AMD64_CMOVQ(0x0f44,dr,sr); asm_output("cmove %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRQNE(dr,sr) do { AMD64_CMOVQ(0x0f45,dr,sr); asm_output("cmovne %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRQL(dr,sr) do { AMD64_CMOVQ(0x0f4C,dr,sr); asm_output("cmovl %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRQLE(dr,sr) do { AMD64_CMOVQ(0x0f4E,dr,sr); asm_output("cmovle %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRQG(dr,sr) do { AMD64_CMOVQ(0x0f4F,dr,sr); asm_output("cmovg %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRQGE(dr,sr) do { AMD64_CMOVQ(0x0f4D,dr,sr); asm_output("cmovge %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRQB(dr,sr) do { AMD64_CMOVQ(0x0f42,dr,sr); asm_output("cmovb %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRQBE(dr,sr) do { AMD64_CMOVQ(0x0f46,dr,sr); asm_output("cmovbe %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRQA(dr,sr) do { AMD64_CMOVQ(0x0f47,dr,sr); asm_output("cmova %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRQAE(dr,sr) do { AMD64_CMOVQ(0x0f43,dr,sr); asm_output("cmovae %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRQNC(dr,sr) do { AMD64_CMOVQ(0x0f43,dr,sr); asm_output("cmovnc %s,%s", gpn(dr),gpn(sr)); } while(0) #define MRQNO(dr,sr) do { AMD64_CMOVQ(0x0f41,dr,sr); asm_output("cmovno %s,%s", gpn(dr),gpn(sr)); } while(0) #define AMD64_LD(reg,disp,base,q) \ underrunProtect(7); \ AMD64_MODRM_DISP(reg,base,disp); \ *(--_nIns) = AMD64_MOV_REG_RM; \ if (q || AMD64_NEEDS_REX(reg) || AMD64_NEEDS_REX(base)) \ *(--_nIns) = AMD64_REX(q,reg,base); #define LD(reg,disp,base) do { \ AMD64_LD(reg,disp,base,0); \ asm_output("mov dword %s,%d(%s)",gpn(reg),disp,gpn(base)); \ } while (0) #define LDQ(reg,disp,base) do { \ AMD64_LD(reg,disp,base,1); \ asm_output("mov %s,%d(%s)",gpn(reg),disp,gpn(base)); \ } while (0) // load 8-bit, zero extend // note, only 5-bit offsets (!) are supported for this, but that's all we need at the moment // (movzx actually allows larger offsets mode but 5-bit gives us advantage in Thumb mode) #define LD8Z(r,d,b) do { \ underrunProtect(5); \ AMD64_MODRM_DISP(r, b, d); \ *(--_nIns) = AMD64_MOVZX & 0xFF; \ *(--_nIns) = AMD64_MOVZX >> 8; \ if (AMD64_NEEDS_REX(r) || AMD64_NEEDS_REX(b)) { \ *(--_nIns) = AMD64_REX(0,r,b); \ } \ asm_output("movzx %s,%d(%s)", gpn(r),d,gpn(b)); \ } while(0) #define LD16Z(r,d,b) do { \ underrunProtect(5); \ AMD64_MODRM_DISP(r, b, d); \ *(--_nIns) = AMD64_MOVSX & 0xFF; \ *(--_nIns) = AMD64_MOVSX >> 8; \ if (AMD64_NEEDS_REX(r) || AMD64_NEEDS_REX(b)) { \ *(--_nIns) = AMD64_REX(0,r,b); \ } \ asm_output("movsx %s,%d(%s)", gpn(r),d,gpn(b)); \ } while(0) #define LDi(r,i) do { \ underrunProtect(6); \ IMM32(i); \ *(--_nIns) = AMD64_MODRM_REG(0, r); \ *(--_nIns) = AMD64_MOV_RM_IMM; \ *(--_nIns) = AMD64_REX(1,0,r); \ asm_output("mov %s,%d",gpn(r),i); \ } while (0) #define LDQi(r,i) do { \ underrunProtect(10); \ IMM64(i); \ *(--_nIns) = AMD64_MOV_REG_IMM(r); \ *(--_nIns) = AMD64_REX(1,0,r); \ asm_output("mov %s,%ld",gpn(r),(nj_printf_ld)i); \ } while(0) #define AMD64_ST(base,disp,reg,q) \ underrunProtect(7); \ AMD64_MODRM_DISP(reg, base, disp); \ *(--_nIns) = AMD64_MOV_RM_REG; \ if (q || AMD64_NEEDS_REX(reg) || AMD64_NEEDS_REX(base)) \ *(--_nIns) = AMD64_REX(q,reg,base); \ #define ST(base,disp,reg) do { \ AMD64_ST(base,disp,reg,0); \ asm_output("mov dword %d(%s),%s",disp,gpn(base),gpn(reg)); \ } while(0); #define STQ(base,disp,reg) do { \ AMD64_ST(base,disp,reg,1); \ asm_output("mov %d(%s),%s",disp,gpn(base),gpn(reg)); \ } while(0); #define STi(base,disp,imm) do { \ underrunProtect(11); \ IMM32(imm); \ AMD64_MODRM_DISP(0, base, disp); \ *(--_nIns) = AMD64_MOV_RM_IMM; \ if (AMD64_NEEDS_REX(base)) \ *(--_nIns) = AMD64_REX(0,0,base); \ asm_output("mov %d(%s),%d",disp,gpn(base),imm); \ } while(0); #define RET() do { \ underrunProtect(1); \ *(--_nIns) = AMD64_RET; \ asm_output("ret"); \ } while (0) #define INT3() do { \ underrunProtect(1); \ *(--_nIns) = AMD64_INT3; \ asm_output("int3"); \ } while (0) #define PUSHi(i) do { \ if (isS8(i)) { \ underrunProtect(2); \ _nIns-=2; _nIns[0] = 0x6a; _nIns[1] = (uint8_t)(i); \ asm_output("push %d",i); \ } else \ { PUSHi32(i); } } while(0) #define PUSHi32(i) do { \ underrunProtect(5); \ IMM32(i); \ *(--_nIns) = 0x68; \ asm_output("push %d",i); } while(0) /** * Note: PUSH/POP do not use REX's prefix 64-bit field. */ #define PUSHr(r) do { \ underrunProtect(2); \ *(--_nIns) = (uint8_t)AMD64_PUSH_REG(r); \ if (AMD64_NEEDS_REX(r)) \ *(--_nIns) = AMD64_REX(0,0,r); \ asm_output("push %s",gpn(r)); \ } while(0) #define PUSHm(d,b) do { \ underrunProtect(7); \ AMD64_MODRM_DISP(6, b, d); \ *(--_nIns) = AMD64_PUSH_RM; \ if (AMD64_NEEDS_REX(b)) { \ *(--_nIns) = AMD64_REX(0,6,b); \ } \ asm_output("push %d(%s)",d,gpn(b)); \ } while(0) #define POPr(r) do { \ underrunProtect(2); \ *(--_nIns) = (uint8_t)(AMD64_POP_REG(r)); \ if (AMD64_NEEDS_REX(r)) \ *(--_nIns) = AMD64_REX(0,0,r); \ asm_output("pop %s",gpn(r)); \ } while(0) #define JCC(o,t,n) do { \ underrunProtect(6); \ intptr_t tt = (intptr_t)t - (intptr_t)_nIns; \ if (isS8(tt)) { \ verbose_only( NIns* next = _nIns; (void)next; ) \ _nIns -= 2; \ _nIns[0] = (uint8_t) ( 0x70 | (o) ); \ _nIns[1] = (uint8_t) (tt); \ asm_output("%s %lX",(n),(ptrdiff_t)(next+tt)); \ } else if (tt <= INT_MAX && tt >= INT_MIN) { \ verbose_only( NIns* next = _nIns; ) \ IMM32(tt); \ _nIns -= 2; \ _nIns[0] = 0x0f; \ _nIns[1] = (uint8_t) ( 0x80 | (o) ); \ asm_output("%s %lX",(n),(ptrdiff_t)(next+tt)); \ } else { \ underrunProtect(20); \ NanoAssert(!_inExit); \ /* We could now be in range, but assume we're not. */ \ /* Note we generate the thunk forwards, and the */ \ /* jcc to the thunk backwards. */ \ uint8_t* base; \ intptr_t offs; \ base = (uint8_t *)((uintptr_t)_nIns & ~((uintptr_t)NJ_PAGE_SIZE-1)); \ base += sizeof(PageHeader) + _pageData; \ _pageData += 14; \ *(base++) = 0xFF; \ *(base++) = 0x25; \ *(int *)base = 0; \ base += 4; \ *(intptr_t *)base = intptr_t(t); \ offs = intptr_t(base-6) - intptr_t(_nIns); \ NanoAssert(offs >= INT_MIN && offs <= INT_MAX); \ if (isS8(offs)) { \ _nIns -= 2; \ _nIns[0] = uint8_t( 0x70 | (o) ); \ _nIns[1] = uint8_t( (offs) ); \ } else { \ IMM32(offs); \ _nIns -= 2; \ _nIns[0] = 0x0f; \ _nIns[1] = uint8_t( 0x80 | (o) ); \ } \ asm_output("%s %d(rip) #%lX",n,offs,intptr_t(t)); \ } \ } while(0) #define JMPm_nochk(rip) do { \ IMM32(rip); \ *(--_nIns) = 0x25; \ *(--_nIns) = 0xFF; \ } while (0) #define JMP_long(t) do { \ underrunProtect(5); \ intptr_t tt = (intptr_t)t - (intptr_t)_nIns; \ JMP_long_nochk_offset(tt); \ } while(0) #define JMP(t) do { \ underrunProtect(5); \ intptr_t tt = (intptr_t)t - (intptr_t)_nIns; \ if (isS8(tt)) { \ verbose_only( NIns* next = _nIns; (void)next; ) \ _nIns -= 2; \ _nIns[0] = 0xeb; \ _nIns[1] = (uint8_t) ( (tt)&0xff ); \ asm_output("jmp %lX",(ptrdiff_t)(next+tt)); \ } else { \ if (tt >= INT_MIN && tt <= INT_MAX) { \ JMP_long_nochk_offset(tt); \ } else { \ underrunProtect(14); \ _nIns -= 8; \ *(intptr_t *)_nIns = intptr_t(t); \ JMPm_nochk(0); \ } \ } } while(0) #define JMP_long_nochk(t) do { \ intptr_t tt = (intptr_t)t - (intptr_t)_nIns; \ JMP_long_nochk_offset(tt); \ } while(0) #define JMPc 0xe9 #define JMP_long_placeholder() do { \ underrunProtect(14); \ IMM64(-1); \ JMPm_nochk(0); \ } while (0) // this should only be used when you can guarantee there is enough room on the page #define JMP_long_nochk_offset(o) do {\ verbose_only( NIns* next = _nIns; (void)next; ) \ NanoAssert(o <= INT_MAX && o >= INT_MIN); \ IMM32((o)); \ *(--_nIns) = JMPc; \ asm_output("jmp %lX",(ptrdiff_t)(next+(o))); } while(0) #define JMPr(r) do { \ underrunProtect(2); \ *(--_nIns) = AMD64_MODRM_REG(4, r); \ *(--_nIns) = 0xFF; \ } while (0) #define JE(t) JCC(0x04, t, "je") #define JNE(t) JCC(0x05, t, "jne") #define JP(t) JCC(0x0A, t, "jp") #define JNP(t) JCC(0x0B, t, "jnp") #define JB(t) JCC(0x02, t, "jb") #define JNB(t) JCC(0x03, t, "jnb") #define JBE(t) JCC(0x06, t, "jbe") #define JNBE(t) JCC(0x07, t, "jnbe") #define JA(t) JCC(0x07, t, "ja") #define JNA(t) JCC(0x06, t, "jna") #define JAE(t) JCC(0x03, t, "jae") #define JNAE(t) JCC(0x02, t, "jnae") #define JL(t) JCC(0x0C, t, "jl") #define JNL(t) JCC(0x0D, t, "jnl") #define JLE(t) JCC(0x0E, t, "jle") #define JNLE(t) JCC(0x0F, t, "jnle") #define JG(t) JCC(0x0F, t, "jg") #define JNG(t) JCC(0x0E, t, "jng") #define JGE(t) JCC(0x0D, t, "jge") #define JNGE(t) JCC(0x0C, t, "jnge") #define JC(t) JCC(0x02, t, "jc") #define JNC(t) JCC(0x03, t, "jnc") #define JO(t) JCC(0x00, t, "jo") #define JNO(t) JCC(0x01, t, "jno") #define AMD64_OP3(c,q,r,b) \ *(--_nIns) = (uint8_t)((c)&0xff); \ *(--_nIns) = (uint8_t)(((c)>>8)&0xff); \ if (q \ || AMD64_NEEDS_REX(r) \ || AMD64_NEEDS_REX(b)) { \ *(--_nIns) = AMD64_REX(q,r,b); \ } \ *(--_nIns) = (uint8_t)(((c)>>16)&0xff); #define SSE_LDQ(r,d,b) do { \ underrunProtect(7); \ AMD64_MODRM_DISP(r,b,d); \ AMD64_OP3(AMD64_MOVD_REG_RM,1,r,b); \ asm_output("movd %s,%d(%s)",gpn(r),(d),gpn(b)); \ } while (0) #define SSE_STQ(d,r,b) do { \ underrunProtect(7); \ AMD64_MODRM_DISP(b,r,d); \ AMD64_OP3(AMD64_MOVD_RM_REG,1,b,r); \ asm_output("movd %d(%s),%s",(d),gpn(r),gpn(b)); \ } while (0) #define SSE_CVTSI2SD(xr,gr) do{ \ underrunProtect(5); \ *(--_nIns) = AMD64_MODRM_REG(xr, gr); \ AMD64_OP3(AMD64_CVTSI2SD,0,xr,gr); \ asm_output("cvtsi2sd %s,%s",gpn(xr),gpn(gr)); \ } while(0) // move and zero-extend gpreg to xmm reg #define SSE_MOVD(d,s) do{ \ underrunProtect(7); \ if (_is_xmm_reg_(s)) { \ NanoAssert(_is_gp_reg_(d)); \ *(--_nIns) = AMD64_MODRM_REG(s, d); \ AMD64_OP3(AMD64_MOVD_RM_REG, 1, s, d); \ } else { \ NanoAssert(_is_gp_reg_(s)); \ NanoAssert(_is_xmm_reg_(d)); \ *(--_nIns) = AMD64_MODRM_REG(d, s); \ AMD64_OP3(AMD64_MOVD_REG_RM, 1, d, s); \ } \ asm_output("movd %s,%s",gpn(d),gpn(s)); \ } while(0) #define SSE_MOVSD(rd,rs) do{ \ underrunProtect(7); \ *(--_nIns) = AMD64_MODRM_REG(rd, rs); \ AMD64_OP3(AMD64_MOVSD_REG_RM,0,rd,rs); \ asm_output("movsd %s,%s",gpn(rd),gpn(rs)); \ } while(0) #define SSE_MOVDm(d,b,xrs) do { \ AMD64_MODRM_DISP(xrs, b, d); \ AMD64_OP3(AMD64_MOVD_RM_REG, 1, xrs, b); \ asm_output("movd %d(%s),%s", d, gpn(b), gpn(xrs)); \ } while(0) #define SSE_ADDSD(rd,rs) do{ \ underrunProtect(5); \ *(--_nIns) = AMD64_MODRM_REG(rd, rs); \ AMD64_OP3(AMD64_ADDSD, 0, rd, rs); \ asm_output("addsd %s,%s",gpn(rd),gpn(rs)); \ } while(0) #define SSE_ADDSDm(r,addr)do { \ underrunProtect(10); \ ptrdiff_t d = (NIns *)addr - _nIns; \ NanoAssert(d >= INT_MIN && d <= INT_MAX); \ IMM32((int32_t)d); \ *(--_nIns) = AMD64_MODRM(0, r, 5); \ AMD64_OP3(AMD64_ADDSD, 0, r, 0); \ asm_output("addsd %s,%p // =%f",gpn(r),addr,*(double*)addr); \ } while(0) #define SSE_SUBSD(rd,rs) do{ \ underrunProtect(5); \ *(--_nIns) = AMD64_MODRM_REG(rd, rs); \ AMD64_OP3(AMD64_SUBSD, 0, rd, rs); \ asm_output("subsd %s,%s",gpn(rd),gpn(rs)); \ } while(0) #define SSE_MULSD(rd,rs) do{ \ underrunProtect(5); \ *(--_nIns) = AMD64_MODRM_REG(rd, rs); \ AMD64_OP3(AMD64_MULSD, 0, rd, rs); \ asm_output("mulsd %s,%s",gpn(rd),gpn(rs)); \ } while(0) #define SSE_DIVSD(rd,rs) do{ \ underrunProtect(5); \ *(--_nIns) = AMD64_MODRM_REG(rd, rs); \ AMD64_OP3(AMD64_DIVSD, 0, rd, rs); \ asm_output("divsd %s,%s",gpn(rd),gpn(rs)); \ } while(0) #define SSE_UCOMISD(rl,rr) do{ \ *(--_nIns) = AMD64_MODRM_REG(rl, rr); \ AMD64_OP3(AMD64_UCOMISD, 0, rl, rr); \ asm_output("ucomisd %s,%s",gpn(rl),gpn(rr)); \ } while(0) #define EMIT_XORPD_MASK(mask) \ do { \ uint8_t *base, *begin; \ uint32_t *addr; \ base = (uint8_t *)((uintptr_t)_nIns & ~((uintptr_t)NJ_PAGE_SIZE-1)); \ base += sizeof(PageHeader) + _pageData; \ begin = base; \ /* Make sure we align */ \ if ((uintptr_t)base & 0xF) { \ base = (NIns *)((uintptr_t)base & ~(0xF)); \ base += 16; \ } \ _pageData += (int32_t)(base - begin) + 16; \ _dblNegPtr = (NIns *)base; \ addr = (uint32_t *)base; \ addr[0] = mask[0]; \ addr[1] = mask[1]; \ addr[2] = mask[2]; \ addr[3] = mask[3]; \ } while (0) /** * Note: high underrun protect is for: * 12 bytes of alignment max * 16 bytes for the data * 10 bytes for the instruction */ #define SSE_XORPD(r, maskaddr) do { \ if (_dblNegPtr != NULL) { \ underrunProtect(10); \ } \ if (_dblNegPtr == NULL) { \ underrunProtect(38); \ EMIT_XORPD_MASK(maskaddr); \ } \ ptrdiff_t d = _dblNegPtr - _nIns; \ IMM32((int32_t)d); \ *(--_nIns) = AMD64_MODRM(0, rr, 5); \ AMD64_OP3(AMD64_XORPD, 0, rr, 0); \ asm_output("xorpd %s,[0x%X]",gpn(rr),(int32_t)d); \ } while(0) #define SSE_XORPDr(rd,rs) do{ \ underrunProtect(5); \ *(--_nIns) = AMD64_MODRM_REG(rd, rs); \ AMD64_OP3(AMD64_XORPD, 0, rd, rs); \ asm_output("xorpd %s,%s",gpn(rd),gpn(rs)); \ } while(0) #define TEST_AL(i) do { \ underrunProtect(2); \ *(--_nIns) = uint8_t(i); \ *(--_nIns) = 0xA8; \ asm_output("test al, %d",i); \ } while(0) #define PUSHFQ() do { \ underrunProtect(1); \ *(--_nIns) = AMD64_PUSHF; \ asm_output("pushf"); \ } while (0) #define CALL(c) do { \ underrunProtect(5); \ intptr_t offset = (c->_address) - ((intptr_t)_nIns); \ if (offset <= INT_MAX && offset >= INT_MIN) { \ IMM32( (uint32_t)offset ); \ *(--_nIns) = 0xE8; \ } else { \ *(--_nIns) = 0xD0; \ *(--_nIns) = 0xFF; \ LDQi(RAX, c->_address); \ } \ verbose_only(asm_output("call %s",(c->_name));) \ } while (0) } #endif // __nanojit_NativeAMD64__