/*
 * Copyright (C) 2012-2019  Free Software Foundation, Inc.
 *
 * This file is part of GNU lightning.
 *
 * GNU lightning is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation; either version 3, or (at your option)
 * any later version.
 *
 * GNU lightning is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
 * License for more details.
 *
 * Authors:
 *	Paulo Cesar Pereira de Andrade
 */

/* avoid using it due to partial stalls */
#define USE_INC_DEC			0

#if PROTO
#  if __X32 || __X64_32
#    define WIDE			0
#    define ldi(u, v)			ldi_i(u, v)
#    define ldr(u, v)			ldr_i(u, v)
#    define ldxr(u, v, w)		ldxr_i(u, v, w)
#    define ldxi(u, v, w)		ldxi_i(u, v, w)
#    define sti(u, v)			sti_i(u, v)
#    define stxi(u, v, w)		stxi_i(u, v, w)
#    define can_sign_extend_int_p(im)	1
#    define can_zero_extend_int_p(im)	1
#    define fits_uint32_p(im)		1
#  else
#    define WIDE			1
#    define ldi(u, v)			ldi_l(u, v)
#    define ldr(u, v)			ldr_l(u, v)
#    define ldxr(u, v, w)		ldxr_l(u, v, w)
#    define ldxi(u, v, w)		ldxi_l(u, v, w)
#    define sti(u, v)			sti_l(u, v)
#    define stxi(u, v, w)		stxi_l(u, v, w)
#    define can_sign_extend_int_p(im)					\
	(((im) >= 0 && (long long)(im) <=  0x7fffffffLL) ||		\
	 ((im) <  0 && (long long)(im) >  -0x80000000LL))
#    define can_zero_extend_int_p(im)					\
	((im) >= 0 && (im) < 0x80000000LL)
#    define fits_uint32_p(im)		(((im) & 0xffffffff00000000LL) == 0)
#  endif
#  if __X32 || __CYGWIN__ || __X64_32 || _WIN32
#      define reg8_p(rn)						\
      ((rn) >= _RAX_REGNO && (rn) <= _RBX_REGNO)
#  else
#      define reg8_p(rn)		1
#  endif
#  define _RAX_REGNO			0
#  define _RCX_REGNO			1
#  define _RDX_REGNO			2
#  define _RBX_REGNO			3
#  define _RSP_REGNO			4
#  define _RBP_REGNO			5
#  define _RSI_REGNO			6
#  define _RDI_REGNO			7
#  define _R8_REGNO			8
#  define _R9_REGNO			9
#  define _R10_REGNO			10
#  define _R11_REGNO			11
#  define _R12_REGNO			12
#  define _R13_REGNO			13
#  define _R14_REGNO			14
#  define _R15_REGNO			15
#  define r7(reg)			((reg) & 7)
#  define r8(reg)			((reg) & 15)
#  define _SCL1				0x00
#  define _SCL2				0x01
#  define _SCL4				0x02
#  define _SCL8				0x03
#  define X86_ADD			0
#  define X86_OR			1 << 3
#  define X86_ADC			2 << 3
#  define X86_SBB			3 << 3
#  define X86_AND			4 << 3
#  define X86_SUB			5 << 3
#  define X86_XOR			6 << 3
#  define X86_CMP			7 << 3
#  define X86_ROL			0
#  define X86_ROR			1
#  define X86_RCL			2
#  define X86_RCR			3
#  define X86_SHL			4
#  define X86_SHR			5
#  define X86_SAR			7
#  define X86_NOT			2
#  define X86_NEG			3
#  define X86_MUL			4
#  define X86_IMUL			5
#  define X86_DIV			6
#  define X86_IDIV			7
#  define X86_CC_O			0x0
#  define X86_CC_NO			0x1
#  define X86_CC_NAE			0x2
#  define X86_CC_B			0x2
#  define X86_CC_C			0x2
#  define X86_CC_AE			0x3
#  define X86_CC_NB			0x3
#  define X86_CC_NC			0x3
#  define X86_CC_E			0x4
#  define X86_CC_Z			0x4
#  define X86_CC_NE			0x5
#  define X86_CC_NZ			0x5
#  define X86_CC_BE			0x6
#  define X86_CC_NA			0x6
#  define X86_CC_A			0x7
#  define X86_CC_NBE			0x7
#  define X86_CC_S			0x8
#  define X86_CC_NS			0x9
#  define X86_CC_P			0xa
#  define X86_CC_PE			0xa
#  define X86_CC_NP			0xb
#  define X86_CC_PO			0xb
#  define X86_CC_L			0xc
#  define X86_CC_NGE			0xc
#  define X86_CC_GE			0xd
#  define X86_CC_NL			0xd
#  define X86_CC_LE			0xe
#  define X86_CC_NG			0xe
#  define X86_CC_G			0xf
#  define X86_CC_NLE			0xf
#  define mrm(md, r, m)			*_jit->pc.uc++ = (md<<6) | (r<<3) | m
#  define sib(sc, i, b)			*_jit->pc.uc++ = (sc<<6) | (i<<3) | b
#  define ic(c)				*_jit->pc.uc++ = c
#  define is(s)				*_jit->pc.us++ = s
#  define ii(i)				*_jit->pc.ui++ = i
#  if __X64 && !__X64_32
#    define il(l)			*_jit->pc.ul++ = l
#  else
#    define il(l)			ii(l)
#  endif
#  define patch_abs(instr, label)					\
	*(jit_word_t *)(instr - sizeof(jit_word_t)) = label
#  define patch_rel(instr, label)					\
	*(jit_int32_t *)(instr - 4) = label - instr
#  define patch_rel_char(instr, label)					\
	*(jit_int8_t *)(instr - 1) = label - instr
#  define rex(l, w, r, x, b)		_rex(_jit, l, w, r, x, b)
static void
_rex(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
#  define rx(rd, md, rb, ri, ms)	_rx(_jit, rd, md, rb, ri, ms)
static void
_rx(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
#  define nop(n)			_nop(_jit, n)
static void _nop(jit_state_t*, jit_int32_t);
#  define emms()			is(0x770f)
#  define lea(md, rb, ri, ms, rd)	_lea(_jit, md, rb, ri, ms, rd)
static void
_lea(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
#  define pushr(r0)			_pushr(_jit, r0)
static void _pushr(jit_state_t*, jit_int32_t) maybe_unused;
#  define popr(r0)			_popr(_jit, r0)
static void _popr(jit_state_t*, jit_int32_t) maybe_unused;
#  define xchgr(r0, r1)			_xchgr(_jit, r0, r1)
static void _xchgr(jit_state_t*, jit_int32_t, jit_int32_t);
#  define testr(r0, r1)			_testr(_jit, r0, r1)
static void _testr(jit_state_t*, jit_int32_t, jit_int32_t);
#  define testi(r0, i0)			_testi(_jit, r0, i0)
static void _testi(jit_state_t*, jit_int32_t, jit_word_t);
#  define cc(code, r0)			_cc(_jit, code, r0)
static void _cc(jit_state_t*, jit_int32_t, jit_int32_t);
#  define icmpr(r0, r1)			alur(X86_CMP, r0, r1)
#  define alur(code, r0, r1)		_alur(_jit, code, r0, r1)
static void _alur(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define icmpi(r0, i0)			alui(X86_CMP, r0, i0)
#  define alui(code, r0, i0)		_alui(_jit, code, r0, i0)
static void _alui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  define iaddr(r0, r1)			alur(X86_ADD, r0, r1)
#  define save(r0)			_save(_jit, r0)
static void _save(jit_state_t*, jit_int32_t);
#  define load(r0)			_load(_jit, r0)
static void _load(jit_state_t*, jit_int32_t);
#  define addr(r0, r1, r2)		_addr(_jit, r0, r1, r2)
static void _addr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define iaddi(r0, i0)			alui(X86_ADD, r0, i0)
#  define addi(r0, r1, i0)		_addi(_jit, r0, r1, i0)
static void _addi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#define addcr(r0, r1, r2)		_addcr(_jit, r0, r1, r2)
static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#define addci(r0, r1, i0)		_addci(_jit, r0, r1, i0)
static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  define iaddxr(r0, r1)		alur(X86_ADC, r0, r1)
#  define addxr(r0, r1, r2)		_addxr(_jit, r0, r1, r2)
static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define iaddxi(r0, i0)		alui(X86_ADC, r0, i0)
#  define addxi(r0, r1, i0)		_addxi(_jit, r0, r1, i0)
static void _addxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  define isubr(r0, r1)			alur(X86_SUB, r0, r1)
#  define subr(r0, r1, r2)		_subr(_jit, r0, r1, r2)
static void _subr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define isubi(r0, i0)			alui(X86_SUB, r0, i0)
#  define subi(r0, r1, i0)		_subi(_jit, r0, r1, i0)
static void _subi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  define subcr(r0, r1, r2)		_subcr(_jit, r0, r1, r2)
static void _subcr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
#  define subci(r0, r1, i0)		_subci(_jit, r0, r1, i0)
static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
#  define isubxr(r0, r1)		alur(X86_SBB, r0, r1)
#  define subxr(r0, r1, r2)		_subxr(_jit, r0, r1, r2)
static void _subxr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
#  define isubxi(r0, i0)		alui(X86_SBB, r0, i0)
#  define subxi(r0, r1, i0)		_subxi(_jit, r0, r1, i0)
static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
#  define rsbi(r0, r1, i0)		_rsbi(_jit, r0, r1, i0)
static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
#  define imulr(r0, r1)			_imulr(_jit, r0, r1)
static void _imulr(jit_state_t*, jit_int32_t, jit_int32_t);
#  define imuli(r0, r1, i0)		_imuli(_jit, r0, r1, i0)
static void _imuli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  define mulr(r0, r1, r2)		_mulr(_jit, r0, r1, r2)
static void _mulr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define muli(r0, r1, i0)		_muli(_jit, r0, r1, i0)
static void _muli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  define umulr(r0)			unr(X86_IMUL, r0)
#  define umulr_u(r0)			unr(X86_MUL, r0)
#  define qmulr(r0, r1, r2, r3)		_iqmulr(_jit, r0, r1, r2, r3, 1)
#  define qmulr_u(r0, r1, r2, r3)	_iqmulr(_jit, r0, r1, r2, r3, 0)
#  define iqmulr(r0, r1, r2, r3, sign)	_iqmulr(_jit, r0, r1, r2, r3, sign)
static void _iqmulr(jit_state_t*, jit_int32_t, jit_int32_t,
		    jit_int32_t,jit_int32_t, jit_bool_t);
#  define qmuli(r0, r1, r2, i0)		_iqmuli(_jit, r0, r1, r2, i0, 1)
#  define qmuli_u(r0, r1, r2, i0)	_iqmuli(_jit, r0, r1, r2, i0, 0)
#  define iqmuli(r0, r1, r2, i0, sign)	_iqmuli(_jit, r0, r1, r2, i0, sign)
static void _iqmuli(jit_state_t*, jit_int32_t, jit_int32_t,
		    jit_int32_t,jit_word_t, jit_bool_t);
#  define sign_extend_rdx_rax()		_sign_extend_rdx_rax(_jit)
static void _sign_extend_rdx_rax(jit_state_t*);
#  define idivr(r0)			unr(X86_IDIV, r0)
#  define idivr_u(r0)			unr(X86_DIV, r0)
#  define divremr(r0, r1, r2, i0, i1)	_divremr(_jit, r0, r1, r2, i0, i1)
static void
_divremr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,
	 jit_bool_t,jit_bool_t);
#  define divremi(r0, r1, i0, i1, i2)	_divremi(_jit, r0, r1, i0, i1, i2)
static void
_divremi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t,jit_bool_t,jit_bool_t);
#  define divr(r0, r1, r2)		divremr(r0, r1, r2, 1, 1)
#  define divi(r0, r1, i0)		divremi(r0, r1, i0, 1, 1)
#  define divr_u(r0, r1, r2)		divremr(r0, r1, r2, 0, 1)
#  define divi_u(r0, r1, i0)		divremi(r0, r1, i0, 0, 1)
#  define qdivr(r0, r1, r2, r3)		_iqdivr(_jit, r0, r1, r2, r3, 1)
#  define qdivr_u(r0, r1, r2, r3)	_iqdivr(_jit, r0, r1, r2, r3, 0)
#  define iqdivr(r0, r1, r2, r3, sign)	_iqdivr(_jit, r0, r1, r2, r3, sign)
static void _iqdivr(jit_state_t*, jit_int32_t, jit_int32_t,
		    jit_int32_t,jit_int32_t, jit_bool_t);
#  define qdivi(r0, r1, r2, i0)		_iqdivi(_jit, r0, r1, r2, i0, 1)
#  define qdivi_u(r0, r1, r2, i0)	_iqdivi(_jit, r0, r1, r2, i0, 0)
#  define iqdivi(r0, r1, r2, i0, sign)	_iqdivi(_jit, r0, r1, r2, i0, sign)
static void _iqdivi(jit_state_t*, jit_int32_t, jit_int32_t,
		    jit_int32_t,jit_word_t, jit_bool_t);
#  define remr(r0, r1, r2)		divremr(r0, r1, r2, 1, 0)
#  define remi(r0, r1, i0)		divremi(r0, r1, i0, 1, 0)
#  define remr_u(r0, r1, r2)		divremr(r0, r1, r2, 0, 0)
#  define remi_u(r0, r1, i0)		divremi(r0, r1, i0, 0, 0)
#  define iandr(r0, r1)			alur(X86_AND, r0, r1)
#  define andr(r0, r1, r2)		_andr(_jit, r0, r1, r2)
static void _andr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
#  define iandi(r0, i0)			alui(X86_AND, r0, i0)
#  define andi(r0, r1, i0)		_andi(_jit, r0, r1, i0)
static void _andi(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
#  define iorr(r0, r1)			alur(X86_OR, r0, r1)
#  define orr(r0, r1, r2)		_orr(_jit, r0, r1, r2)
static void _orr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
#  define iori(r0, i0)			alui(X86_OR, r0, i0)
#  define ori(r0, r1, i0)		_ori(_jit, r0, r1, i0)
static void _ori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
#  define ixorr(r0, r1)			alur(X86_XOR, r0, r1)
#  define xorr(r0, r1, r2)		_xorr(_jit, r0, r1, r2)
static void _xorr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
#  define ixori(r0, i0)			alui(X86_XOR, r0, i0)
#  define xori(r0, r1, i0)		_xori(_jit, r0, r1, i0)
static void _xori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
#  define irotshr(code, r0)		_irotshr(_jit, code, r0)
static void _irotshr(jit_state_t*, jit_int32_t, jit_int32_t);
#  define rotshr(code, r0, r1, r2)	_rotshr(_jit, code, r0, r1, r2)
static void
_rotshr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
#  define irotshi(code, r0, i0)		_irotshi(_jit, code, r0, i0)
static void _irotshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  define rotshi(code, r0, r1, i0)	_rotshi(_jit, code, r0, r1, i0)
static void
_rotshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
#  define lshr(r0, r1, r2)		rotshr(X86_SHL, r0, r1, r2)
#  define lshi(r0, r1, i0)		_lshi(_jit, r0, r1, i0)
static void _lshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  define rshr(r0, r1, r2)		rotshr(X86_SAR, r0, r1, r2)
#  define rshi(r0, r1, i0)		rotshi(X86_SAR, r0, r1, i0)
#  define rshr_u(r0, r1, r2)		rotshr(X86_SHR, r0, r1, r2)
#  define rshi_u(r0, r1, i0)		rotshi(X86_SHR, r0, r1, i0)
#  define unr(code, r0)			_unr(_jit, code, r0)
static void _unr(jit_state_t*, jit_int32_t, jit_int32_t);
#  define inegr(r0)			unr(X86_NEG, r0)
#  define negr(r0, r1)			_negr(_jit, r0, r1)
static void _negr(jit_state_t*, jit_int32_t, jit_int32_t);
#  define icomr(r0)			unr(X86_NOT, r0)
#  define comr(r0, r1)			_comr(_jit, r0, r1)
static void _comr(jit_state_t*, jit_int32_t, jit_int32_t);
#  if USE_INC_DEC
#    define incr(r0, r1)		_incr(_jit, r0, r1)
static void _incr(jit_state_t*, jit_int32_t, jit_int32_t);
#    define decr(r0, r1)		_decr(_jit, r0, r1)
static void _decr(jit_state_t*, jit_int32_t, jit_int32_t);
#  endif
#  define cr(code, r0, r1, r2)		_cr(_jit, code, r0, r1, r2)
static void
_cr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t, jit_int32_t);
#  define ci(code, r0, r1, i0)		_ci(_jit, code, r0, r1, i0)
static void
_ci(jit_state_t *_jit, jit_int32_t, jit_int32_t, jit_int32_t, jit_word_t);
#  define ci0(code, r0, r1)		_ci0(_jit, code, r0, r1)
static void _ci0(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define ltr(r0, r1, r2)		_ltr(_jit, r0, r1, r2)
static void _ltr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define lti(r0, r1, i0)			_lti(_jit, r0, r1, i0)
static void _lti(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  define ltr_u(r0, r1, r2)		_ltr_u(_jit, r0, r1, r2)
static void _ltr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define lti_u(r0, r1, i0)		ci(X86_CC_B, r0, r1, i0)
#  define ler(r0, r1, r2)		_ler(_jit, r0, r1, r2)
static void _ler(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define lei(r0, r1, i0)		ci(X86_CC_LE, r0, r1, i0)
#  define ler_u(r0, r1, r2)		_ler_u(_jit, r0, r1, r2)
static void _ler_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define lei_u(r0, r1, i0)		_lei_u(_jit, r0, r1, i0)
static void _lei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  define eqr(r0, r1, r2)		_eqr(_jit, r0, r1, r2)
static void _eqr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define eqi(r0, r1, i0)		_eqi(_jit, r0, r1, i0)
static void _eqi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  define ger(r0, r1, r2)		_ger(_jit, r0, r1, r2)
static void _ger(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define gei(r0, r1, i0)		_gei(_jit, r0, r1, i0)
static void _gei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  define ger_u(r0, r1, r2)		_ger_u(_jit, r0, r1, r2)
static void _ger_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define gei_u(r0, r1, i0)		_gei_u(_jit, r0, r1, i0)
static void _gei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  define gtr(r0, r1, r2)		_gtr(_jit, r0, r1, r2)
static void _gtr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define gti(r0, r1, i0)		_ci(_jit, X86_CC_G, r0, r1, i0)
#  define gtr_u(r0, r1, r2)		_gtr_u(_jit, r0, r1, r2)
static void _gtr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define gti_u(r0, r1, i0)		_gti_u(_jit, r0, r1, i0)
static void _gti_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  define ner(r0, r1, r2)		_ner(_jit, r0, r1, r2)
static void _ner(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define nei(r0, r1, i0)		_nei(_jit, r0, r1, i0)
static void _nei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  define movr(r0, r1)			_movr(_jit, r0, r1)
static void _movr(jit_state_t*, jit_int32_t, jit_int32_t);
#  define imovi(r0, i0)			_imovi(_jit, r0, i0)
static void _imovi(jit_state_t*, jit_int32_t, jit_word_t);
#  define movi(r0, i0)			_movi(_jit, r0, i0)
static void _movi(jit_state_t*, jit_int32_t, jit_word_t);
#  define movi_p(r0, i0)		_movi_p(_jit, r0, i0)
static jit_word_t _movi_p(jit_state_t*, jit_int32_t, jit_word_t);
#  define movcr(r0, r1)			_movcr(_jit, r0, r1)
static void _movcr(jit_state_t*,jit_int32_t,jit_int32_t);
#  define movcr_u(r0, r1)		_movcr_u(_jit, r0, r1)
static void _movcr_u(jit_state_t*,jit_int32_t,jit_int32_t);
#  define movsr(r0, r1)			_movsr(_jit, r0, r1)
static void _movsr(jit_state_t*,jit_int32_t,jit_int32_t);
#  define movsr_u(r0, r1)		_movsr_u(_jit, r0, r1)
static void _movsr_u(jit_state_t*,jit_int32_t,jit_int32_t);
#  if __X64 && !__X64_32
#    define movir(r0, r1)		_movir(_jit, r0, r1)
static void _movir(jit_state_t*,jit_int32_t,jit_int32_t);
#    define movir_u(r0, r1)		_movir_u(_jit, r0, r1)
static void _movir_u(jit_state_t*,jit_int32_t,jit_int32_t);
#  endif
#  define htonr_us(r0, r1)		_htonr_us(_jit, r0, r1)
static void _htonr_us(jit_state_t*,jit_int32_t,jit_int32_t);
#  define htonr_ui(r0, r1)		_htonr_ui(_jit, r0, r1)
static void _htonr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
#  if __X64 && !__X64_32
#define htonr_ul(r0, r1)		_htonr_ul(_jit, r0, r1)
static void _htonr_ul(jit_state_t*,jit_int32_t,jit_int32_t);
#endif
#  define extr_c(r0, r1)		_extr_c(_jit, r0, r1)
static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t);
#  define extr_uc(r0, r1)		_extr_uc(_jit, r0, r1)
static void _extr_uc(jit_state_t*,jit_int32_t,jit_int32_t);
#  define extr_s(r0, r1)		movsr(r0, r1)
#  define extr_us(r0, r1)		movsr_u(r0, r1)
#  if __X64 && !__X64_32
#    define extr_i(r0, r1)		movir(r0, r1)
#    define extr_ui(r0, r1)		movir_u(r0, r1)
#  endif
#  define ldr_c(r0, r1)			_ldr_c(_jit, r0, r1)
static void _ldr_c(jit_state_t*, jit_int32_t, jit_int32_t);
#  define ldi_c(r0, i0)			_ldi_c(_jit, r0, i0)
static void _ldi_c(jit_state_t*, jit_int32_t, jit_word_t);
#  define ldr_uc(r0, r1)		_ldr_uc(_jit, r0, r1)
static void _ldr_uc(jit_state_t*, jit_int32_t, jit_int32_t);
#  define ldi_uc(r0, i0)		_ldi_uc(_jit, r0, i0)
static void _ldi_uc(jit_state_t*, jit_int32_t, jit_word_t);
#  define ldr_s(r0, r1)			_ldr_s(_jit, r0, r1)
static void _ldr_s(jit_state_t*, jit_int32_t, jit_int32_t);
#  define ldi_s(r0, i0)			_ldi_s(_jit, r0, i0)
static void _ldi_s(jit_state_t*, jit_int32_t, jit_word_t);
#  define ldr_us(r0, r1)		_ldr_us(_jit, r0, r1)
static void _ldr_us(jit_state_t*, jit_int32_t, jit_int32_t);
#  define ldi_us(r0, i0)		_ldi_us(_jit, r0, i0)
static void _ldi_us(jit_state_t*, jit_int32_t, jit_word_t);
#  if __X32 || !__X64_32
#    define ldr_i(r0, r1)		_ldr_i(_jit, r0, r1)
static void _ldr_i(jit_state_t*, jit_int32_t, jit_int32_t);
#    define ldi_i(r0, i0)		_ldi_i(_jit, r0, i0)
static void _ldi_i(jit_state_t*, jit_int32_t, jit_word_t);
#  endif
#  if __X64
#    if __X64_32
#      define ldr_i(r0, r1)		_ldr_ui(_jit, r0, r1)
#      define ldi_i(r0, i0)		_ldi_ui(_jit, r0, i0)
#    else
#      define ldr_ui(r0, r1)		_ldr_ui(_jit, r0, r1)
#      define ldi_ui(r0, i0)		_ldi_ui(_jit, r0, i0)
#    endif
static void _ldr_ui(jit_state_t*, jit_int32_t, jit_int32_t);
static void _ldi_ui(jit_state_t*, jit_int32_t, jit_word_t);
#    if !__X64_32
#      define ldr_l(r0, r1)		_ldr_l(_jit, r0, r1)
static void _ldr_l(jit_state_t*, jit_int32_t, jit_int32_t);
#      define ldi_l(r0, i0)		_ldi_l(_jit, r0, i0)
static void _ldi_l(jit_state_t*, jit_int32_t, jit_word_t);
#    endif
#  endif
#  define ldxr_c(r0, r1, r2)		_ldxr_c(_jit, r0, r1, r2)
static void _ldxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define ldxi_c(r0, r1, i0)		_ldxi_c(_jit, r0, r1, i0)
static void _ldxi_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  define ldxr_uc(r0, r1, r2)		_ldxr_uc(_jit, r0, r1, r2)
static void _ldxr_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define ldxi_uc(r0, r1, i0)		_ldxi_uc(_jit, r0, r1, i0)
static void _ldxi_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  define ldxr_s(r0, r1, r2)		_ldxr_s(_jit, r0, r1, r2)
static void _ldxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define ldxi_s(r0, r1, i0)		_ldxi_s(_jit, r0, r1, i0)
static void _ldxi_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  define ldxr_us(r0, r1, r2)		_ldxr_us(_jit, r0, r1, r2)
static void _ldxr_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define ldxi_us(r0, r1, i0)		_ldxi_us(_jit, r0, r1, i0)
static void _ldxi_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  if __X32 || !__X64_32
#    define ldxr_i(r0, r1, r2)		_ldxr_i(_jit, r0, r1, r2)
static void _ldxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#    define ldxi_i(r0, r1, i0)		_ldxi_i(_jit, r0, r1, i0)
static void _ldxi_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#  endif
#  if __X64
#    if __X64_32
#      define ldxr_i(r0, r1, r2)	_ldxr_ui(_jit, r0, r1, r2)
#      define ldxi_i(r0, r1, i0)	_ldxi_ui(_jit, r0, r1, i0)
#    else
#      define ldxr_ui(r0, r1, r2)	_ldxr_ui(_jit, r0, r1, r2)
#      define ldxi_ui(r0, r1, i0)	_ldxi_ui(_jit, r0, r1, i0)
#    endif
static void _ldxr_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
static void _ldxi_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#    if !__X64_32
#      define ldxr_l(r0, r1, r2)	_ldxr_l(_jit, r0, r1, r2)
static void _ldxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#      define ldxi_l(r0, r1, i0)	_ldxi_l(_jit, r0, r1, i0)
static void _ldxi_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
#    endif
#  endif
#  define str_c(r0, r1)			_str_c(_jit, r0, r1)
static void _str_c(jit_state_t*, jit_int32_t, jit_int32_t);
#  define sti_c(i0, r0)			_sti_c(_jit, i0, r0)
static void _sti_c(jit_state_t*, jit_word_t, jit_int32_t);
#  define str_s(r0, r1)			_str_s(_jit, r0, r1)
static void _str_s(jit_state_t*, jit_int32_t, jit_int32_t);
#  define sti_s(i0, r0)			_sti_s(_jit, i0, r0)
static void _sti_s(jit_state_t*, jit_word_t, jit_int32_t);
#  define str_i(r0, r1)			_str_i(_jit, r0, r1)
static void _str_i(jit_state_t*, jit_int32_t, jit_int32_t);
#  define sti_i(i0, r0)			_sti_i(_jit, i0, r0)
static void _sti_i(jit_state_t*, jit_word_t, jit_int32_t);
#  if __X64 && !__X64_32
#    define str_l(r0, r1)		_str_l(_jit, r0, r1)
static void _str_l(jit_state_t*, jit_int32_t, jit_int32_t);
#    define sti_l(i0, r0)		_sti_l(_jit, i0, r0)
static void _sti_l(jit_state_t*, jit_word_t, jit_int32_t);
#  endif
#  define stxr_c(r0, r1, r2)		_stxr_c(_jit, r0, r1, r2)
static void _stxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define stxi_c(i0, r0, r1)		_stxi_c(_jit, i0, r0, r1)
static void _stxi_c(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
#  define stxr_s(r0, r1, r2)		_stxr_s(_jit, r0, r1, r2)
static void _stxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define stxi_s(i0, r0, r1)		_stxi_s(_jit, i0, r0, r1)
static void _stxi_s(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
#  define stxr_i(r0, r1, r2)		_stxr_i(_jit, r0, r1, r2)
static void _stxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#  define stxi_i(i0, r0, r1)		_stxi_i(_jit, i0, r0, r1)
static void _stxi_i(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
#  if __X64 && !__X64_32
#    define stxr_l(r0, r1, r2)		_stxr_l(_jit, r0, r1, r2)
static void _stxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
#    define stxi_l(i0, r0, r1)		_stxi_l(_jit, i0, r0, r1)
static void _stxi_l(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
#  endif
#  define jcc(code, i0)			_jcc(_jit, code, i0)
#  define jo(i0)			jcc(X86_CC_O, i0)
#  define jno(i0)			jcc(X86_CC_NO, i0)
#  define jnae(i0)			jcc(X86_CC_NAE, i0)
#  define jb(i0)			jcc(X86_CC_B, i0)
#  define jc(i0)			jcc(X86_CC_C, i0)
#  define jae(i0)			jcc(X86_CC_AE, i0)
#  define jnb(i0)			jcc(X86_CC_NB, i0)
#  define jnc(i0)			jcc(X86_CC_NC, i0)
#  define je(i0)			jcc(X86_CC_E, i0)
#  define jz(i0)			jcc(X86_CC_Z, i0)
#  define jne(i0)			jcc(X86_CC_NE, i0)
#  define jnz(i0)			jcc(X86_CC_NZ, i0)
#  define jbe(i0)			jcc(X86_CC_BE, i0)
#  define jna(i0)			jcc(X86_CC_NA, i0)
#  define ja(i0)			jcc(X86_CC_A, i0)
#  define jnbe(i0)			jcc(X86_CC_NBE, i0)
#  define js(i0)			jcc(X86_CC_S, i0)
#  define jns(i0)			jcc(X86_CC_NS, i0)
#  define jp(i0)			jcc(X86_CC_P, i0)
#  define jpe(i0)			jcc(X86_CC_PE, i0)
#  define jnp(i0)			jcc(X86_CC_NP, i0)
#  define jpo(i0)			jcc(X86_CC_PO, i0)
#  define jl(i0)			jcc(X86_CC_L, i0)
#  define jnge(i0)			jcc(X86_CC_NGE, i0)
#  define jge(i0)			jcc(X86_CC_GE, i0)
#  define jnl(i0)			jcc(X86_CC_NL, i0)
#  define jle(i0)			jcc(X86_CC_LE, i0)
#  define jng(i0)			jcc(X86_CC_NG, i0)
#  define jg(i0)			jcc(X86_CC_G, i0)
#  define jnle(i0)			jcc(X86_CC_NLE, i0)
static void _jcc(jit_state_t*, jit_int32_t, jit_word_t);
#  define jccs(code, i0)		_jccs(_jit, code, i0)
#  define jos(i0)			jccs(X86_CC_O, i0)
#  define jnos(i0)			jccs(X86_CC_NO, i0)
#  define jnaes(i0)			jccs(X86_CC_NAE, i0)
#  define jbs(i0)			jccs(X86_CC_B, i0)
#  define jcs(i0)			jccs(X86_CC_C, i0)
#  define jaes(i0)			jccs(X86_CC_AE, i0)
#  define jnbs(i0)			jccs(X86_CC_NB, i0)
#  define jncs(i0)			jccs(X86_CC_NC, i0)
#  define jes(i0)			jccs(X86_CC_E, i0)
#  define jzs(i0)			jccs(X86_CC_Z, i0)
#  define jnes(i0)			jccs(X86_CC_NE, i0)
#  define jnzs(i0)			jccs(X86_CC_NZ, i0)
#  define jbes(i0)			jccs(X86_CC_BE, i0)
#  define jnas(i0)			jccs(X86_CC_NA, i0)
#  define jas(i0)			jccs(X86_CC_A, i0)
#  define jnbes(i0)			jccs(X86_CC_NBE, i0)
#  define jss(i0)			jccs(X86_CC_S, i0)
#  define jnss(i0)			jccs(X86_CC_NS, i0)
#  define jps(i0)			jccs(X86_CC_P, i0)
#  define jpes(i0)			jccs(X86_CC_PE, i0)
#  define jnps(i0)			jccs(X86_CC_NP, i0)
#  define jpos(i0)			jccs(X86_CC_PO, i0)
#  define jls(i0)			jccs(X86_CC_L, i0)
#  define jnges(i0)			jccs(X86_CC_NGE, i0)
#  define jges(i0)			jccs(X86_CC_GE, i0)
#  define jnls(i0)			jccs(X86_CC_NL, i0)
#  define jles(i0)			jccs(X86_CC_LE, i0)
#  define jngs(i0)			jccs(X86_CC_NG, i0)
#  define jgs(i0)			jccs(X86_CC_G, i0)
#  define jnles(i0)			jccs(X86_CC_NLE, i0)
static void _jccs(jit_state_t*, jit_int32_t, jit_word_t);
#  define jcr(code, i0, r0, r1)		_jcr(_jit, code, i0, r0, r1)
static void _jcr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
#  define jci(code, i0, r0, i1)		_jci(_jit, code, i0, r0, i1)
static void _jci(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
#  define jci0(code, i0, r0)		_jci0(_jit, code, i0, r0)
static void _jci0(jit_state_t*, jit_int32_t, jit_word_t, jit_int32_t);
#  define bltr(i0, r0, r1)		_bltr(_jit, i0, r0, r1)
static jit_word_t _bltr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
#  define blti(i0, r0, i1)		_blti(_jit, i0, r0, i1)
static jit_word_t _blti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
#  define bltr_u(i0, r0, r1)		_bltr_u(_jit, i0, r0, r1)
static jit_word_t _bltr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
#  define blti_u(i0, r0, i1)		_blti_u(_jit, i0, r0, i1)
static jit_word_t _blti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
#  define bler(i0, r0, r1)		_bler(_jit, i0, r0, r1)
static jit_word_t _bler(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
#  define blei(i0, r0, i1)		_blei(_jit, i0, r0, i1)
static jit_word_t _blei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
#  define bler_u(i0, r0, r1)		_bler_u(_jit, i0, r0, r1)
static jit_word_t _bler_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
#  define blei_u(i0, r0, i1)		_blei_u(_jit, i0, r0, i1)
static jit_word_t _blei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
#  define beqr(i0, r0, r1)		_beqr(_jit, i0, r0, r1)
static jit_word_t _beqr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
#  define beqi(i0, r0, i1)		_beqi(_jit, i0, r0, i1)
static jit_word_t _beqi(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
#  define bger(i0, r0, r1)		_bger(_jit, i0, r0, r1)
static jit_word_t _bger(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
#  define bgei(i0, r0, i1)		_bgei(_jit, i0, r0, i1)
static jit_word_t _bgei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
#  define bger_u(i0, r0, r1)		_bger_u(_jit, i0, r0, r1)
static jit_word_t _bger_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
#  define bgei_u(i0, r0, i1)		_bgei_u(_jit, i0, r0, i1)
static jit_word_t _bgei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
#  define bgtr(i0, r0, r1)		_bgtr(_jit, i0, r0, r1)
static jit_word_t _bgtr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
#  define bgti(i0, r0, i1)		_bgti(_jit, i0, r0, i1)
static jit_word_t _bgti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
#  define bgtr_u(i0, r0, r1)		_bgtr_u(_jit, i0, r0, r1)
static jit_word_t _bgtr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
#  define bgti_u(i0, r0, i1)		_bgti_u(_jit, i0, r0, i1)
static jit_word_t _bgti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
#  define bner(i0, r0, r1)		_bner(_jit, i0, r0, r1)
static jit_word_t _bner(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
#  define bnei(i0, r0, i1)		_bnei(_jit, i0, r0, i1)
static jit_word_t _bnei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
#  define bmsr(i0, r0, r1)		_bmsr(_jit, i0, r0, r1)
static jit_word_t _bmsr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
#  define bmsi(i0, r0, i1)		_bmsi(_jit, i0, r0, i1)
static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
#  define bmcr(i0, r0, r1)		_bmcr(_jit, i0, r0, r1)
static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
#  define bmci(i0, r0, i1)		_bmci(_jit, i0, r0, i1)
static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
#  define boaddr(i0, r0, r1)		_boaddr(_jit, i0, r0, r1)
static jit_word_t _boaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
#  define boaddi(i0, r0, i1)		_boaddi(_jit, i0, r0, i1)
static jit_word_t _boaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
#  define boaddr_u(i0, r0, r1)		_boaddr_u(_jit, i0, r0, r1)
static jit_word_t _boaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
#  define boaddi_u(i0, r0, i1)		_boaddi_u(_jit, i0, r0, i1)
static jit_word_t _boaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
#  define bxaddr(i0, r0, r1)		_bxaddr(_jit, i0, r0, r1)
static jit_word_t _bxaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
#  define bxaddi(i0, r0, i1)		_bxaddi(_jit, i0, r0, i1)
static jit_word_t _bxaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
#  define bxaddr_u(i0, r0, r1)		_bxaddr_u(_jit, i0, r0, r1)
static jit_word_t _bxaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
#  define bxaddi_u(i0, r0, i1)		_bxaddi_u(_jit, i0, r0, i1)
static jit_word_t _bxaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
#  define bosubr(i0, r0, r1)		_bosubr(_jit, i0, r0, r1)
static jit_word_t _bosubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
#  define bosubi(i0, r0, i1)		_bosubi(_jit, i0, r0, i1)
static jit_word_t _bosubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
#  define bosubr_u(i0, r0, r1)		_bosubr_u(_jit, i0, r0, r1)
static jit_word_t _bosubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
#  define bosubi_u(i0, r0, i1)		_bosubi_u(_jit, i0, r0, i1)
static jit_word_t _bosubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
#  define bxsubr(i0, r0, r1)		_bxsubr(_jit, i0, r0, r1)
static jit_word_t _bxsubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
#  define bxsubi(i0, r0, i1)		_bxsubi(_jit, i0, r0, i1)
static jit_word_t _bxsubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
#  define bxsubr_u(i0, r0, r1)		_bxsubr_u(_jit, i0, r0, r1)
static jit_word_t _bxsubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
#  define bxsubi_u(i0, r0, i1)		_bxsubi_u(_jit, i0, r0, i1)
static jit_word_t _bxsubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
#  define callr(r0)			_callr(_jit, r0)
static void _callr(jit_state_t*, jit_int32_t);
#  define calli(i0)			_calli(_jit, i0)
static jit_word_t _calli(jit_state_t*, jit_word_t);
#  if __X64
#    define calli_p(i0)			_calli_p(_jit, i0)
static jit_word_t _calli_p(jit_state_t*, jit_word_t);
#  else
#    define calli_p(i0)			calli(i0)
#  endif
#  define jmpr(r0)			_jmpr(_jit, r0)
static void _jmpr(jit_state_t*, jit_int32_t);
#  define jmpi(i0)			_jmpi(_jit, i0)
static jit_word_t _jmpi(jit_state_t*, jit_word_t);
#  if __X64
#    define jmpi_p(i0)			_jmpi_p(_jit, i0)
static jit_word_t _jmpi_p(jit_state_t*, jit_word_t);
#  else
#    define jmpi_p(i0)			jmpi(i0)
#  endif
#  define jmpsi(i0)			_jmpsi(_jit, i0)
static void _jmpsi(jit_state_t*, jit_uint8_t);
#  define prolog(node)			_prolog(_jit, node)
static void _prolog(jit_state_t*, jit_node_t*);
#  define epilog(node)			_epilog(_jit, node)
static void _epilog(jit_state_t*, jit_node_t*);
#  define vastart(r0)			_vastart(_jit, r0)
static void _vastart(jit_state_t*, jit_int32_t);
#  define vaarg(r0, r1)			_vaarg(_jit, r0, r1)
static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
#  define vaarg_d(r0, r1, i0)		_vaarg_d(_jit, r0, r1, i0)
static void _vaarg_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_bool_t);
#  define patch_at(node, instr, label)	_patch_at(_jit, node, instr, label)
static void _patch_at(jit_state_t*, jit_node_t*, jit_word_t, jit_word_t);
#  if !defined(HAVE_FFSL)
#    if __X32
#      define ffsl(i)			__builtin_ffs(i)
#    else
#      define ffsl(l)			__builtin_ffsl(l)
#    endif
#  endif
#endif

#if CODE
static void
_rex(jit_state_t *_jit, jit_int32_t l, jit_int32_t w,
     jit_int32_t r, jit_int32_t x, jit_int32_t b)
{
#if __X64
    jit_int32_t	v = 0x40 | (w << 3);

    if (r != _NOREG)
	v |= (r & 8) >> 1;
    if (x != _NOREG)
	v |= (x & 8) >> 2;
    if (b != _NOREG)
	v |= (b & 8) >> 3;
    if (l || v != 0x40)
	ic(v);
#endif
}

static void
_rx(jit_state_t *_jit, jit_int32_t rd, jit_int32_t md,
    jit_int32_t rb, jit_int32_t ri, jit_int32_t ms)
{
    if (ri == _NOREG) {
	if (rb == _NOREG) {
#if __X32
	    mrm(0x00, r7(rd), 0x05);
#else
	    mrm(0x00, r7(rd), 0x04);
	    sib(_SCL1, 0x04, 0x05);
#endif
	    ii(md);
	}
	else if (r7(rb) == _RSP_REGNO) {
	    if (md == 0) {
		mrm(0x00, r7(rd), 0x04);
		sib(ms, 0x04, 0x04);
	    }
	    else if ((jit_int8_t)md == md) {
		mrm(0x01, r7(rd), 0x04);
		sib(ms, 0x04, 0x04);
		ic(md);
	    }
	    else {
		mrm(0x02, r7(rd), 0x04);
		sib(ms, 0x04, 0x04);
		ii(md);
	    }
	}
	else {
	    if (md == 0 && r7(rb) != _RBP_REGNO)
		mrm(0x00, r7(rd), r7(rb));
	    else if ((jit_int8_t)md == md) {
		mrm(0x01, r7(rd), r7(rb));
		ic(md);
	    }
	    else {
		mrm(0x02, r7(rd), r7(rb));
		ii(md);
	    }
	}
    }
    else if (rb == _NOREG) {
	mrm(0x00, r7(rd), 0x04);
	sib(ms, r7(ri), 0x05);
	ii(md);
    }
    else if (r8(ri) != _RSP_REGNO) {
	if (md == 0 && r7(rb) != _RBP_REGNO) {
	    mrm(0x00, r7(rd), 0x04);
	    sib(ms, r7(ri), r7(rb));
	}
	else if ((jit_int8_t)md == md) {
	    mrm(0x01, r7(rd), 0x04);
	    sib(ms, r7(ri), r7(rb));
	    ic(md);
	}
	else {
	    mrm(0x02, r7(rd), 0x04);
	    sib(ms, r7(ri), r7(rb));
	    ic(md);
	}
    }
    else {
	fprintf(stderr, "illegal index register");
	abort();
    }
}

static void
_nop(jit_state_t *_jit, jit_int32_t count)
{
    switch (count) {
	case 0:
	    break;
	case 1:		/* NOP */
	    ic(0x90);	break;
	case 2:		/* 66 NOP */
	    ic(0x66);	ic(0x90);
	    break;
	case 3:		/* NOP DWORD ptr [EAX] */
	    ic(0x0f);	ic(0x1f);	ic(0x00);
	    break;
	case 4:		/* NOP DWORD ptr [EAX + 00H] */
	    ic(0x0f);	ic(0x1f);	ic(0x40);	ic(0x00);
	    break;
	case 5:		/* NOP DWORD ptr [EAX + EAX*1 + 00H] */
	    ic(0x0f);	ic(0x1f);	ic(0x44);	ic(0x00);
	    ic(0x00);
	    break;
	case 6:		/* 66 NOP DWORD ptr [EAX + EAX*1 + 00H] */
	    ic(0x66);	ic(0x0f);	ic(0x1f);	ic(0x44);
	    ic(0x00);	ic(0x00);
	    break;
	case 7:		/* NOP DWORD ptr [EAX + 00000000H] */
	    ic(0x0f);	ic(0x1f);	ic(0x80);	ii(0x0000);
	    break;
	case 8:		/* NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
	    ic(0x0f);	ic(0x1f);	ic(0x84);	ic(0x00);
	    ii(0x0000);
	    break;
	case 9:		/* 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
	    ic(0x66);	ic(0x0f);	ic(0x1f);	ic(0x84);
	    ic(0x00);	ii(0x0000);
	    break;
	default:
	    abort();
    }
}

static void
_lea(jit_state_t *_jit, jit_int32_t md, jit_int32_t rb,
     jit_int32_t ri, jit_int32_t ms, jit_int32_t rd)
{
    rex(0, WIDE, rd, ri, rb);
    ic(0x8d);
    rx(rd, md, rb, ri, ms);
}

static void
_pushr(jit_state_t *_jit, jit_int32_t r0)
{
    rex(0, WIDE, 0, 0, r0);
    ic(0x50 | r7(r0));
}

static void
_popr(jit_state_t *_jit, jit_int32_t r0)
{
    rex(0, WIDE, 0, 0, r0);
    ic(0x58 | r7(r0));
}

static void
_xchgr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    rex(0, WIDE, r1, _NOREG, r0);
    ic(0x87);
    mrm(0x03, r7(r1), r7(r0));
}

static void
_testr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    rex(0, WIDE, r1, _NOREG, r0);
    ic(0x85);
    mrm(0x03, r7(r1), r7(r0));
}

static void
_testi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
{
    rex(0, WIDE, _NOREG, _NOREG, r0);
    if (r0 == _RAX_REGNO)
	ic(0xa9);
    else {
	ic(0xf7);
	mrm(0x03, 0x00, r7(r0));
    }
    ii(i0);
}

static void
_cc(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
{
    rex(0, 0, _NOREG, _NOREG, r0);
    ic(0x0f);
    ic(0x90 | code);
    mrm(0x03, 0x00, r7(r0));
}

static void
_alur(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
{
    rex(0, WIDE, r1, _NOREG, r0);
    ic(code | 0x01);
    mrm(0x03, r7(r1), r7(r0));
}

static void
_alui(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	rex(0, WIDE, _NOREG, _NOREG, r0);
	if ((jit_int8_t)i0 == i0) {
	    ic(0x83);
	    ic(0xc0 | code | r7(r0));
	    ic(i0);
	}
	else {
	    if (r0 == _RAX_REGNO)
		ic(code | 0x05);
	    else {
		ic(0x81);
		ic(0xc0 | code | r7(r0));
	    }
	    ii(i0);
	}
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	alur(code, r0, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_save(jit_state_t *_jit, jit_int32_t r0)
{
    if (!_jitc->function->regoff[r0]) {
	_jitc->function->regoff[r0] = jit_allocai(sizeof(jit_word_t));
	_jitc->again = 1;
    }
    assert(!jit_regset_tstbit(&_jitc->regsav, r0));
    jit_regset_setbit(&_jitc->regsav, r0);
    stxi(_jitc->function->regoff[r0], _RBP_REGNO, r0);
}

static void
_load(jit_state_t *_jit, jit_int32_t r0)
{
    assert(_jitc->function->regoff[r0]);
    assert(jit_regset_tstbit(&_jitc->regsav, r0));
    jit_regset_clrbit(&_jitc->regsav, r0);
    ldxi(r0, _RBP_REGNO, _jitc->function->regoff[r0]);
}

static void
_addr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    if (r0 == r1)
	iaddr(r0, r2);
    else if (r0 == r2)
	iaddr(r0, r1);
    else
	lea(0, r1, r2, _SCL1, r0);
}

static void
_addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    jit_int32_t		reg;
    if (i0 == 0)
	movr(r0, r1);
#if USE_INC_DEC
    else if (i0 == 1)
	incr(r0, r1);
    else if (i0 == -1)
	decr(r0, r1);
#endif
    else if (can_sign_extend_int_p(i0)) {
	if (r0 == r1)
	    iaddi(r0, i0);
	else
	    lea(i0, r1, _NOREG, _SCL1, r0);
    }
    else if (r0 != r1) {
	movi(r0, i0);
	iaddr(r0, r1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	iaddr(r0, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_addcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    if (r0 == r2)
	iaddr(r0, r1);
    else {
	movr(r0, r1);
	iaddr(r0, r2);
    }
}

static void
_addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	movr(r0, r1);
	iaddi(r0, i0);
    }
    else if (r0 == r1) {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	iaddr(r0, rn(reg));
	jit_unget_reg(reg);
    }
    else {
	movi(r0, i0);
	iaddr(r0, r1);
    }
}

static void
_addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    if (r0 == r2)
	iaddxr(r0, r1);
    else {
	movr(r0, r1);
	iaddxr(r0, r2);
    }
}

static void
_addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	movr(r0, r1);
	iaddxi(r0, i0);
    }
    else if (r0 == r1) {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	iaddxr(r0, rn(reg));
	jit_unget_reg(reg);
    }
    else {
	movi(r0, i0);
	iaddxr(r0, r1);
    }
}

static void
_subr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    if (r1 == r2)
	ixorr(r0, r0);
    else if (r0 == r2) {
	isubr(r0, r1);
	inegr(r0);
    }
    else {
	movr(r0, r1);
	isubr(r0, r2);
    }
}

static void
_subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    jit_int32_t		reg;
    if (i0 == 0)
	movr(r0, r1);
#if USE_INC_DEC
    else if (i0 == 1)
	decr(r0, r1);
    else if (i0 == -1)
	incr(r0, r1);
#endif
    else if (can_sign_extend_int_p(i0)) {
	if (r0 == r1)
	    isubi(r0, i0);
	else
	    lea(-i0, r1, _NOREG, _SCL1, r0);
    }
    else if (r0 != r1) {
	movi(r0, -i0);
	iaddr(r0, r1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	isubr(r0, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_subcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    jit_int32_t		reg;
    if (r0 == r2 && r0 != r1) {
	reg = jit_get_reg(jit_class_gpr);
	movr(rn(reg), r0);
	movr(r0, r1);
	isubr(r0, rn(reg));
	jit_unget_reg(reg);
    }
    else {
	movr(r0, r1);
	isubr(r0, r2);
    }
}

static void
_subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    jit_int32_t		reg;
    movr(r0, r1);
    if (can_sign_extend_int_p(i0))
	isubi(r0, i0);
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	isubr(r0, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_subxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    jit_int32_t		reg;
    if (r0 == r2 && r0 != r1) {
	reg = jit_get_reg(jit_class_gpr);
	movr(rn(reg), r0);
	movr(r0, r1);
	isubxr(r0, rn(reg));
	jit_unget_reg(reg);
    }
    else {
	movr(r0, r1);
	isubxr(r0, r2);
    }
}

static void
_subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    jit_int32_t		reg;
    movr(r0, r1);
    if (can_sign_extend_int_p(i0))
	isubxi(r0, i0);
    else {
	reg = jit_get_reg(jit_class_gpr);
	imovi(rn(reg), i0);
	isubxr(r0, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    subi(r0, r1, i0);
    negr(r0, r0);
}

static void
_imulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    rex(0, WIDE, r0, _NOREG, r1);
    ic(0x0f);
    ic(0xaf);
    mrm(0x03, r7(r0), r7(r1));
}

static void
_imuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	rex(0, WIDE, r0, _NOREG, r1);
	if ((jit_int8_t)i0 == i0) {
	    ic(0x6b);
	    mrm(0x03, r7(r0), r7(r1));
	    ic(i0);
	}
	else {
	    ic(0x69);
	    mrm(0x03, r7(r0), r7(r1));
	    ii(i0);
	}
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	imulr(r0, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_mulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    if (r0 == r1)
	imulr(r0, r2);
    else if (r0 == r2)
	imulr(r0, r1);
    else {
	movr(r0, r1);
	imulr(r0, r2);
    }
}

static void
_muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    switch (i0) {
	case 0:
	    ixorr(r0, r0);
	    break;
	case 1:
	    movr(r0, r1);
	    break;
	case -1:
	    negr(r0, r1);
	    break;
	case 2:
	    lea(0, _NOREG, r1, _SCL2, r0);
	    break;
	case 4:
	    lea(0, _NOREG, r1, _SCL4, r0);
	    break;
	case 8:
	    lea(0, _NOREG, r1, _SCL8, r0);
	    break;
	default:
	    if (i0 > 0 && !(i0 & (i0 - 1)))
		lshi(r0, r1, ffsl(i0) - 1);
	    else if (can_sign_extend_int_p(i0))
		imuli(r0, r1, i0);
	    else if (r0 != r1) {
		movi(r0, i0);
		imulr(r0, r1);
	    }
	    else
		imuli(r0, r0, i0);
	    break;
    }
}

#define savset(rn)							\
    if (r0 != rn) {							\
	sav |= 1 << rn;							\
	if (r1 != rn && r2 != rn)					\
	    set |= 1 << rn;						\
    }
#define isavset(rn)							\
    if (r0 != rn) {							\
	sav |= 1 << rn;							\
	if (r1 != rn)							\
	    set |= 1 << rn;						\
    }
#define qsavset(rn)							\
    if (r0 != rn && r1 != rn) {						\
	sav |= 1 << rn;							\
	if (r2 != rn && r3 != rn)					\
	    set |= 1 << rn;						\
    }
#define allocr(rn, rv)							\
    if (set & (1 << rn))						\
	(void)jit_get_reg(rv|jit_class_gpr|jit_class_named);		\
    if (sav & (1 << rn)) {						\
	if ( jit_regset_tstbit(&_jitc->regsav, rv) ||			\
	    !jit_regset_tstbit(&_jitc->reglive, rv))			\
	    sav &= ~(1 << rn);						\
	else								\
	    save(rv);							\
    }
#define clear(rn, rv)							\
    if (set & (1 << rn))						\
	jit_unget_reg(rv);						\
    if (sav & (1 << rn))						\
	load(rv);
static void
_iqmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
	jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
{
    jit_int32_t		mul;
    jit_int32_t		sav;
    jit_int32_t		set;

    sav = set = 0;
    qsavset(_RDX_REGNO);
    qsavset(_RAX_REGNO);
    allocr(_RDX_REGNO, _RDX);
    allocr(_RAX_REGNO, _RAX);

    if (r3 == _RAX_REGNO)
	mul = r2;
    else {
	mul = r3;
	movr(_RAX_REGNO, r2);
    }
    if (sign)
	umulr(mul);
    else
	umulr_u(mul);

    if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
	xchgr(_RAX_REGNO, _RDX_REGNO);
    else {
	if (r0 != _RDX_REGNO)
	    movr(r0, _RAX_REGNO);
	movr(r1, _RDX_REGNO);
	if (r0 == _RDX_REGNO)
	    movr(r0, _RAX_REGNO);
    }

    clear(_RDX_REGNO, _RDX);
    clear(_RAX_REGNO, _RAX);
}

static void
_iqmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
	jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
{
    jit_int32_t		reg;

    if (i0 == 0) {
	ixorr(r0, r0);
	ixorr(r1, r1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	if (sign)
	    qmulr(r0, r1, r2, rn(reg));
	else
	    qmulr_u(r0, r1, r2, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_sign_extend_rdx_rax(jit_state_t *_jit)
{
    rex(0, WIDE, 0, 0, 0);
    ic(0x99);
}

static void
_divremr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2,
	 jit_bool_t sign, jit_bool_t divide)
{
    jit_int32_t		div;
    jit_int32_t		reg;
    jit_int32_t		set;
    jit_int32_t		sav;
    jit_int32_t		use;

    sav = set = use = 0;
    savset(_RDX_REGNO);
    savset(_RAX_REGNO);
    allocr(_RDX_REGNO, _RDX);
    allocr(_RAX_REGNO, _RAX);

    if (r2 == _RAX_REGNO) {
	if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
	    if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
		reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
				  jit_class_gpr|jit_class_named);
	    use = 1;
	    div = rn(reg);
	    movr(div, _RAX_REGNO);
	    if (r1 != _RAX_REGNO)
		movr(_RAX_REGNO, r1);
	}
	else {
	    if (r0 == r1)
		xchgr(r0, _RAX_REGNO);
	    else {
		if (r0 != _RAX_REGNO)
		    movr(r0, _RAX_REGNO);
		if (r1 != _RAX_REGNO)
		    movr(_RAX_REGNO, r1);
	    }
	    div = r0;
	}
    }
    else if (r2 == _RDX_REGNO) {
	if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
	    if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
		reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
				  jit_class_gpr|jit_class_named);
	    use = 1;
	    div = rn(reg);
	    movr(div, _RDX_REGNO);
	    if (r1 != _RAX_REGNO)
		movr(_RAX_REGNO, r1);
	}
	else {
	    if (r1 != _RAX_REGNO)
		movr(_RAX_REGNO, r1);
	    movr(r0, _RDX_REGNO);
	    div = r0;
	}
    }
    else {
	if (r1 != _RAX_REGNO)
	    movr(_RAX_REGNO, r1);
	div = r2;
    }

    if (sign) {
	sign_extend_rdx_rax();
	idivr(div);
    }
    else {
	ixorr(_RDX_REGNO, _RDX_REGNO);
	idivr_u(div);
    }

    if (use)
	jit_unget_reg(reg);

    if (divide)
	movr(r0, _RAX_REGNO);
    else
	movr(r0, _RDX_REGNO);

    clear(_RDX_REGNO, _RDX);
    clear(_RAX_REGNO, _RAX);
}

static void
_divremi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0,
	 jit_bool_t sign, jit_bool_t divide)
{
    jit_int32_t		reg;
    jit_int32_t		div;
    jit_int32_t		sav;
    jit_int32_t		set;
    jit_int32_t		use;

    if (divide) {
	switch (i0) {
	    case 1:
		movr(r0, r1);
		return;
	    case -1:
		if (sign) {
		    negr(r0, r1);
		    return;
		}
		break;
	    default:
		if (i0 > 0 && !(i0 & (i0 - 1))) {
		    movr(r0, r1);
		    if (sign)
			rshi(r0, r0, ffsl(i0) - 1);
		    else
			rshi_u(r0, r0, ffsl(i0) - 1);
		    return;
		}
		break;
	}
    }
    else if (i0 == 1 || (sign && i0 == -1)) {
	ixorr(r0, r0);
	return;
    }
    else if (!sign && i0 > 0 && !(i0 & (i0 - 1))) {
	if (can_sign_extend_int_p(i0)) {
	    movr(r0, r1);
	    iandi(r0, i0 - 1);
	}
	else if (r0 != r1) {
	    movi(r0, i0 - 1);
	    iandr(r0, r1);
	}
	else {
	    reg = jit_get_reg(jit_class_gpr);
	    movi(rn(reg), i0 - 1);
	    iandr(r0, rn(reg));
	    jit_unget_reg(reg);
	}
	return;
    }

    sav = set = use = 0;
    isavset(_RDX_REGNO);
    isavset(_RAX_REGNO);
    allocr(_RDX_REGNO, _RDX);
    allocr(_RAX_REGNO, _RAX);

    if (r0 == _RAX_REGNO || r0 == _RDX_REGNO || r0 == r1) {
	if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
	    reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
			      jit_class_gpr|jit_class_named);
	use = 1;
	div = rn(reg);
    }
    else
	div = r0;

    movi(div, i0);
    movr(_RAX_REGNO, r1);

    if (sign) {
	sign_extend_rdx_rax();
	idivr(div);
    }
    else {
	ixorr(_RDX_REGNO, _RDX_REGNO);
	idivr_u(div);
    }

    if (use)
	jit_unget_reg(reg);

    if (divide)
	movr(r0, _RAX_REGNO);
    else
	movr(r0, _RDX_REGNO);

    clear(_RDX_REGNO, _RDX);
    clear(_RAX_REGNO, _RAX);
}

static void
_iqdivr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
	jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
{
    jit_int32_t		div;
    jit_int32_t		reg;
    jit_int32_t		sav;
    jit_int32_t		set;
    jit_int32_t		use;

    sav = set = use = 0;
    qsavset(_RDX_REGNO);
    qsavset(_RAX_REGNO);
    allocr(_RDX_REGNO, _RDX);
    allocr(_RAX_REGNO, _RAX);
    if (r3 == _RAX_REGNO) {
	if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
	    if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
		reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
				  jit_class_gpr|jit_class_named);
	    use = 1;
	    div = rn(reg);
	    movr(div, _RAX_REGNO);
	    if (r2 != _RAX_REGNO)
		movr(_RAX_REGNO, r2);
	}
	else {
	    if (r0 == r2)
		xchgr(r0, _RAX_REGNO);
	    else {
		if (r0 != _RAX_REGNO)
		    movr(r0, _RAX_REGNO);
		if (r2 != _RAX_REGNO)
		    movr(_RAX_REGNO, r2);
	    }
	    div = r0;
	}
    }
    else if (r3 == _RDX_REGNO) {
	if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
	    if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
		reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
				  jit_class_gpr|jit_class_named);
	    use = 1;
	    div = rn(reg);
	    movr(div, _RDX_REGNO);
	    if (r2 != _RAX_REGNO)
		movr(_RAX_REGNO, r2);
	}
	else {
	    if (r2 != _RAX_REGNO)
		movr(_RAX_REGNO, r2);
	    movr(r0, _RDX_REGNO);
	    div = r0;
	}
    }
    else {
	if (r2 != _RAX_REGNO)
	    movr(_RAX_REGNO, r2);
	div = r3;
    }
    if (sign) {
	sign_extend_rdx_rax();
	idivr(div);
    }
    else {
	ixorr(_RDX_REGNO, _RDX_REGNO);
	idivr_u(div);
    }
    if (use)
	jit_unget_reg(reg);

    if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
	xchgr(_RAX_REGNO, _RDX_REGNO);
    else {
	if (r0 != _RDX_REGNO)
	    movr(r0, _RAX_REGNO);
	movr(r1, _RDX_REGNO);
	if (r0 == _RDX_REGNO)
	    movr(r0, _RAX_REGNO);
    }

    clear(_RDX_REGNO, _RDX);
    clear(_RAX_REGNO, _RAX);
}

static void
_iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
	jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
{
    jit_int32_t		reg;

    reg = jit_get_reg(jit_class_gpr);
    movi(rn(reg), i0);
    if (sign)
	qdivr(r0, r1, r2, rn(reg));
    else
	qdivr_u(r0, r1, r2, rn(reg));
    jit_unget_reg(reg);
}
#undef clear
#undef allocr
#undef savset

static void
_andr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    if (r1 == r2)
	movr(r0, r1);
    else if (r0 == r1)
	iandr(r0, r2);
    else if (r0 == r2)
	iandr(r0, r1);
    else {
	movr(r0, r1);
	iandr(r0, r2);
    }
}

static void
_andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    jit_int32_t		reg;

    if (i0 == 0)
	ixorr(r0, r0);
    else if (i0 == -1)
	movr(r0, r1);
    else if (r0 == r1) {
	if (can_sign_extend_int_p(i0))
	    iandi(r0, i0);
	else {
	    reg = jit_get_reg(jit_class_gpr);
	    movi(rn(reg), i0);
	    iandr(r0, rn(reg));
	    jit_unget_reg(reg);
	}
    }
    else {
	movi(r0, i0);
	iandr(r0, r1);
    }
}

static void
_orr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    if (r1 == r2)
	movr(r0, r1);
    else if (r0 == r1)
	iorr(r0, r2);
    else if (r0 == r2)
	iorr(r0, r1);
    else {
	movr(r0, r1);
	iorr(r0, r2);
    }
}

static void
_ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    jit_int32_t		reg;
    if (i0 == 0)
	movr(r0, r1);
    else if (i0 == -1)
	movi(r0, -1);
    else if (can_sign_extend_int_p(i0)) {
	movr(r0, r1);
	iori(r0, i0);
    }
    else if (r0 != r1) {
	movi(r0, i0);
	iorr(r0, r1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	iorr(r0, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_xorr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    if (r1 == r2)
	ixorr(r0, r0);
    else if (r0 == r1)
	ixorr(r0, r2);
    else if (r0 == r2)
	ixorr(r0, r1);
    else {
	movr(r0, r1);
	ixorr(r0, r2);
    }
}

static void
_xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    jit_int32_t		reg;
    if (i0 == 0)
	movr(r0, r1);
    else if (i0 == -1)
	comr(r0, r1);
    else if (can_sign_extend_int_p(i0)) {
	movr(r0, r1);
	ixori(r0, i0);
    }
    else if (r0 != r1) {
	movi(r0, i0);
	ixorr(r0, r1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	ixorr(r0, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_irotshr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
{
    rex(0, WIDE, _RCX_REGNO, _NOREG, r0);
    ic(0xd3);
    mrm(0x03, code, r7(r0));
}

static void
_rotshr(jit_state_t *_jit, jit_int32_t code,
	jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    jit_int32_t		reg;
    jit_int32_t		use;

    if (r0 == _RCX_REGNO) {
	reg = jit_get_reg(jit_class_gpr);
	movr(rn(reg), r1);
	if (r2 != _RCX_REGNO)
	    movr(_RCX_REGNO, r2);
	irotshr(code, rn(reg));
	movr(_RCX_REGNO, rn(reg));
	jit_unget_reg(reg);
    }
    else if (r2 != _RCX_REGNO) {
	use = !jit_reg_free_p(_RCX);
	if (use) {
	    reg = jit_get_reg(jit_class_gpr);
	    movr(rn(reg), _RCX_REGNO);
	}
	else
	    reg = 0;
	if (r1 == _RCX_REGNO) {
	    if (r0 == r2)
		xchgr(r0, _RCX_REGNO);
	    else {
		movr(r0, r1);
		movr(_RCX_REGNO, r2);
	    }
	}
	else {
	    movr(_RCX_REGNO, r2);
	    movr(r0, r1);
	}
	irotshr(code, r0);
	if (use) {
	    movr(_RCX_REGNO, rn(reg));
	    jit_unget_reg(reg);
	}
    }
    else {
	movr(r0, r1);
	irotshr(code, r0);
    }
}

static void
_irotshi(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
{
    rex(0, WIDE, _NOREG, _NOREG, r0);
    if (i0 == 1) {
	ic(0xd1);
	mrm(0x03, code, r7(r0));
    }
    else {
	ic(0xc1);
	mrm(0x03, code, r7(r0));
	ic(i0);
    }
}

static void
_rotshi(jit_state_t *_jit, jit_int32_t code,
	jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    movr(r0, r1);
    if (i0)
	irotshi(code, r0, i0);
}

static void
_lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    if (i0 == 0)
	movr(r0, r1);
    else if (i0 <= 3)
	lea(0, _NOREG, r1, i0 == 1 ? _SCL2 : i0 == 2 ? _SCL4 : _SCL8, r0);
    else
	rotshi(X86_SHL, r0, r1, i0);
}

static void
_unr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
{
    rex(0, WIDE, _NOREG, _NOREG, r0);
    ic(0xf7);
    mrm(0x03, code, r7(r0));
}

static void
_negr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    if (r0 == r1)
	inegr(r0);
    else {
	ixorr(r0, r0);
	isubr(r0, r1);
    }
}

static void
_comr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    movr(r0, r1);
    icomr(r0);
}

#if USE_INC_DEC
static void
_incr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    movr(r0, r1);
#  if __X64
    rex(0, WIDE, _NOREG, _NOREG, r0);
    ic(0xff);
    ic(0xc0 | r7(r0));
#  else
    ic(0x40 | r7(r0));
#  endif
}

static void
_decr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    movr(r0, r1);
#  if __X64
    rex(0, WIDE, _NOREG, _NOREG, r0);
    ic(0xff);
    ic(0xc8 | r7(r0));
#  else
    ic(0x48 | r7(r0));
#  endif
}
#endif

static void
_cr(jit_state_t *_jit,
    jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    jit_int32_t		reg;
    jit_bool_t		same;
    if (reg8_p(r0)) {
	same = r0 == r1 || r0 == r2;
	if (!same)
	    ixorr(r0, r0);
	icmpr(r1, r2);
	if (same)
	    imovi(r0, 0);
	cc(code, r0);
    }
    else {
	reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
	ixorr(rn(reg), rn(reg));
	icmpr(r1, r2);
	cc(code, rn(reg));
	movr(r0, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_ci(jit_state_t *_jit,
    jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    jit_int32_t		reg;
    jit_bool_t		same;
    if (reg8_p(r0)) {
	same = r0 == r1;
	if (!same)
	    ixorr(r0, r0);
	icmpi(r1, i0);
	if (same)
	    imovi(r0, 0);
	cc(code, r0);
    }
    else {
	reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
	ixorr(rn(reg), rn(reg));
	icmpi(r1, i0);
	cc(code, rn(reg));
	movr(r0, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_ci0(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
{
    jit_int32_t		reg;
    jit_bool_t		same;
    if (reg8_p(r0)) {
	same = r0 == r1;
	if (!same)
	    ixorr(r0, r0);
	testr(r1, r1);
	if (same)
	    imovi(r0, 0);
	cc(code, r0);
    }
    else {
	reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
	ixorr(rn(reg), rn(reg));
	testr(r1, r1);
	cc(code, rn(reg));
	movr(r0, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_ltr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    if (r1 == r2)
	movi(r0, 0);
    else
	cr(X86_CC_L, r0, r1, r2);
}

static void
_lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    if (i0)
	ci(X86_CC_L, r0, r1, i0);
    else
	ci0(X86_CC_S, r0, r1);
}

static void
_ltr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    if (r1 == r2)
	movi(r0, 0);
    else
	cr(X86_CC_B, r0, r1, r2);
}

static void
_ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    if (r1 == r2)
	movi(r0, 1);
    else
	cr(X86_CC_LE, r0, r1, r2);
}

static void
_ler_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    if (r1 == r2)
	movi(r0, 1);
    else
	cr(X86_CC_BE, r0, r1, r2);
}

static void
_lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    if (i0)
	ci(X86_CC_BE, r0, r1, i0);
    else
	ci0(X86_CC_E, r0, r1);
}

static void
_eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    if (r1 == r2)
	movi(r0, 1);
    else
	cr(X86_CC_E, r0, r1, r2);
}

static void
_eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    if (i0)
	ci(X86_CC_E, r0, r1, i0);
    else
	ci0(X86_CC_E, r0, r1);
}

static void
_ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    if (r1 == r2)
	movi(r0, 1);
    else
	cr(X86_CC_GE, r0, r1, r2);
}

static void
_gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    if (i0)
	ci(X86_CC_GE, r0, r1, i0);
    else
	ci0(X86_CC_NS, r0, r1);
}

static void
_ger_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    if (r1 == r2)
	movi(r0, 1);
    else
	cr(X86_CC_AE, r0, r1, r2);
}

static void
_gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    if (i0)
	ci(X86_CC_AE, r0, r1, i0);
    else
	ci0(X86_CC_NB, r0, r1);
}

static void
_gtr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    if (r1 == r2)
	movi(r0, 0);
    else
	cr(X86_CC_G, r0, r1, r2);
}

static void
_gtr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    if (r1 == r2)
	movi(r0, 0);
    else
	cr(X86_CC_A, r0, r1, r2);
}

static void
_gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    if (i0)
	ci(X86_CC_A, r0, r1, i0);
    else
	ci0(X86_CC_NE, r0, r1);
}

static void
_ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    if (r1 == r2)
	movi(r0, 0);
    else
	cr(X86_CC_NE, r0, r1, r2);
}

static void
_nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    if (i0)
	ci(X86_CC_NE, r0, r1, i0);
    else
	ci0(X86_CC_NE, r0, r1);
}

static void
_movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    if (r0 != r1) {
	rex(0, 1, r1, _NOREG, r0);
	ic(0x89);
	ic(0xc0 | (r1 << 3) | r7(r0));
    }
}

static void
_imovi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
{
#if __X64
#  if !__X64_32
    if (fits_uint32_p(i0)) {
#  endif
	rex(0, 0, _NOREG, _NOREG, r0);
	ic(0xb8 | r7(r0));
	ii(i0);
#  if !__X64_32
    }
    else {
	rex(0, 1, _NOREG, _NOREG, r0);
	ic(0xb8 | r7(r0));
	il(i0);
    }
#  endif
#else
    ic(0xb8 | r7(r0));
    ii(i0);
#endif
}

static void
_movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
{
    if (i0)
	imovi(r0, i0);
    else
	ixorr(r0, r0);
}

static jit_word_t
_movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
{
    rex(0, WIDE, _NOREG, _NOREG, r0);
    ic(0xb8 | r7(r0));
    il(i0);
    return (_jit->pc.w);
}

static void
_movcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    rex(0, WIDE, r0, _NOREG, r1);
    ic(0x0f);
    ic(0xbe);
    mrm(0x03, r7(r0), r7(r1));
}

static void
_movcr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    rex(0, WIDE, r0, _NOREG, r1);
    ic(0x0f);
    ic(0xb6);
    mrm(0x03, r7(r0), r7(r1));
}

static void
_movsr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    rex(0, WIDE, r0, _NOREG, r1);
    ic(0x0f);
    ic(0xbf);
    mrm(0x03, r7(r0), r7(r1));
}

static void
_movsr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    rex(0, WIDE, r0, _NOREG, r1);
    ic(0x0f);
    ic(0xb7);
    mrm(0x03, r7(r0), r7(r1));
}

#if __X64
static void
_movir(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    rex(0, 1, r0, _NOREG, r1);
    ic(0x63);
    mrm(0x03, r7(r0), r7(r1));
}

static void
_movir_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    rex(0, 0, r1, _NOREG, r0);
    ic(0x89);
    ic(0xc0 | (r1 << 3) | r7(r0));
}
#endif

static void
_htonr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    extr_us(r0, r1);
    ic(0x66);
    rex(0, 0, _NOREG, _NOREG, r0);
    ic(0xc1);
    mrm(0x03, X86_ROR, r7(r0));
    ic(8);
}

static void
_htonr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    movr(r0, r1);
    rex(0, 0, _NOREG, _NOREG, r0);
    ic(0x0f);
    ic(0xc8 | r7(r0));
}

#if __X64 && !__X64_32
static void
_htonr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    movr(r0, r1);
    rex(0, 1, _NOREG, _NOREG, r0);
    ic(0x0f);
    ic(0xc8 | r7(r0));
}
#endif

static void
_extr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    jit_int32_t		reg;
    if (reg8_p(r1))
	movcr(r0, r1);
    else {
	reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
	movr(rn(reg), r1);
	movcr(r0, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_extr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    jit_int32_t		reg;
    if (reg8_p(r1))
	movcr_u(r0, r1);
    else {
	reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
	movr(rn(reg), r1);
	movcr_u(r0, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_ldr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    rex(0, WIDE, r0, _NOREG, r1);
    ic(0x0f);
    ic(0xbe);
    rx(r0, 0, r1, _NOREG, _SCL1);
}

static void
_ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	rex(0, WIDE, r0, _NOREG, _NOREG);
	ic(0x0f);
	ic(0xbe);
	rx(r0, i0, _NOREG, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	ldr_c(r0, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_ldr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    rex(0, WIDE, r0, _NOREG, r1);
    ic(0x0f);
    ic(0xb6);
    rx(r0, 0, r1, _NOREG, _SCL1);
}

static void
_ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	rex(0, WIDE, r0, _NOREG, _NOREG);
	ic(0x0f);
	ic(0xb6);
	rx(r0, i0, _NOREG, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	ldr_uc(r0, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_ldr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    rex(0, WIDE, r0, _NOREG, r1);
    ic(0x0f);
    ic(0xbf);
    rx(r0, 0, r1, _NOREG, _SCL1);
}

static void
_ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	rex(0, WIDE, r0, _NOREG, _NOREG);
	ic(0x0f);
	ic(0xbf);
	rx(r0, i0, _NOREG, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	ldr_s(r0, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_ldr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    rex(0, WIDE, r0, _NOREG, r1);
    ic(0x0f);
    ic(0xb7);
    rx(r0, 0, r1, _NOREG, _SCL1);
}

static void
_ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	rex(0, WIDE, r0, _NOREG, _NOREG);
	ic(0x0f);
	ic(0xb7);
	rx(r0, i0, _NOREG, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	ldr_us(r0, rn(reg));
	jit_unget_reg(reg);
    }
}

#if __X32 || !__X64_32
static void
_ldr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
#if __X64
    rex(0, WIDE, r0, _NOREG, r1);
    ic(0x63);
#else
    ic(0x8b);
#endif
    rx(r0, 0, r1, _NOREG, _SCL1);
}

static void
_ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
#if __X64
	rex(0, WIDE, r0, _NOREG, _NOREG);
	ic(0x63);
#else
	ic(0x8b);
#endif
	rx(r0, i0, _NOREG, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	ldr_i(r0, rn(reg));
	jit_unget_reg(reg);
    }
}
#endif

#if __X64
static void
_ldr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    rex(0, 0, r0, _NOREG, r1);
    ic(0x63);
    rx(r0, 0, r1, _NOREG, _SCL1);
}

static void
_ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	rex(0, 0, r0, _NOREG, _NOREG);
	ic(0x63);
	rx(r0, i0, _NOREG, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	ldr_ui(r0, rn(reg));
	jit_unget_reg(reg);
    }
}

#  if !__X64_32
static void
_ldr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    rex(0, 1, r0, _NOREG, r1);
    ic(0x8b);
    rx(r0, 0, r1, _NOREG, _SCL1);
}

static void
_ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	rex(0, 1, r0, _NOREG, _NOREG);
	ic(0x8b);
	rx(r0, i0, _NOREG, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	ldr_l(r0, rn(reg));
	jit_unget_reg(reg);
    }
}
#  endif
#endif

static void
_ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
#if __X64_32
    addr(r0, r1, r2);
    ldr_c(r0, r0);
#else
    rex(0, WIDE, r0, r1, r2);
    ic(0x0f);
    ic(0xbe);
    rx(r0, 0, r2, r1, _SCL1);
#endif
}

static void
_ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	rex(0, WIDE, r0, _NOREG, r1);
	ic(0x0f);
	ic(0xbe);
	rx(r0, i0, r1, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	ldxr_c(r0, r1, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
#if __X64_32
    addr(r0, r1, r2);
    ldr_uc(r0, r0);
#else
    rex(0, WIDE, r0, r1, r2);
    ic(0x0f);
    ic(0xb6);
    rx(r0, 0, r2, r1, _SCL1);
#endif
}

static void
_ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	rex(0, WIDE, r0, _NOREG, r1);
	ic(0x0f);
	ic(0xb6);
	rx(r0, i0, r1, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	ldxr_uc(r0, r1, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_ldxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
#if __X64_32
    addr(r0, r1, r2);
    ldr_s(r0, r0);
#else
    rex(0, WIDE, r0, r1, r2);
    ic(0x0f);
    ic(0xbf);
    rx(r0, 0, r2, r1, _SCL1);
#endif
}

static void
_ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	rex(0, WIDE, r0, _NOREG, r1);
	ic(0x0f);
	ic(0xbf);
	rx(r0, i0, r1, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	ldxr_s(r0, r1, rn(reg));
	jit_unget_reg(reg);
    }
}

static void
_ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
#if __X64_32
    addr(r0, r1, r2);
    ldr_us(r0, r0);
#else
    rex(0, WIDE, r0, r1, r2);
    ic(0x0f);
    ic(0xb7);
    rx(r0, 0, r2, r1, _SCL1);
#endif
}

static void
_ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	rex(0, WIDE, r0, _NOREG, r1);
	ic(0x0f);
	ic(0xb7);
	rx(r0, i0, r1, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	ldxr_us(r0, r1, rn(reg));
	jit_unget_reg(reg);
    }
}

#if __X64 || !__X64_32
static void
_ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
#if __X64
    rex(0, WIDE, r0, r1, r2);
    ic(0x63);
#else
    ic(0x8b);
#endif
    rx(r0, 0, r2, r1, _SCL1);
}

static void
_ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
#if __X64
	rex(0, WIDE, r0, _NOREG, r1);
	ic(0x63);
#else
	ic(0x8b);
#endif
	rx(r0, i0, r1, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	ldxr_i(r0, r1, rn(reg));
	jit_unget_reg(reg);
    }
}
#endif

#if __X64
static void
_ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
#if __X64_32
    addr(r0, r1, r2);
    /* to avoid confusion with macro renames */
    _ldr_ui(_jit, r0, r0);
#else
    rex(0, 0, r0, r1, r2);
    ic(0x8b);
    rx(r0, 0, r2, r1, _SCL1);
#endif
}

static void
_ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	rex(0, 0, r0, _NOREG, r1);
	ic(0x8b);
	rx(r0, i0, r1, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	ldxr_ui(r0, r1, rn(reg));
	jit_unget_reg(reg);
    }
}

#  if !__X64_32
static void
_ldxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    rex(0, 1, r0, r1, r2);
    ic(0x8b);
    rx(r0, 0, r2, r1, _SCL1);
}

static void
_ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	rex(0, 1, r0, _NOREG, r1);
	ic(0x8b);
	rx(r0, i0, r1, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	ldxr_l(r0, r1, rn(reg));
	jit_unget_reg(reg);
    }
}
#  endif
#endif

static void
_str_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    jit_int32_t		reg;
    if (reg8_p(r1)) {
	rex(0, 0, r1, _NOREG, r0);
	ic(0x88);
	rx(r1, 0, r0, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
	movr(rn(reg), r1);
	rex(0, 0, rn(reg), _NOREG, r0);
	ic(0x88);
	rx(rn(reg), 0, r0, _NOREG, _SCL1);
	jit_unget_reg(reg);
    }
}

static void
_sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	if (reg8_p(r0)) {
	    rex(0, 0, r0, _NOREG, _NOREG);
	    ic(0x88);
	    rx(r0, i0, _NOREG, _NOREG, _SCL1);
	}
	else {
	    reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
	    movr(rn(reg), r0);
	    rex(0, 0, rn(reg), _NOREG, _NOREG);
	    ic(0x88);
	    rx(rn(reg), i0, _NOREG, _NOREG, _SCL1);
	    jit_unget_reg(reg);
	}
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	str_c(rn(reg), r0);
	jit_unget_reg(reg);
    }
}

static void
_str_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    ic(0x66);
    rex(0, 0, r1, _NOREG, r0);
    ic(0x89);
    rx(r1, 0, r0, _NOREG, _SCL1);
}

static void
_sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	ic(0x66);
	rex(0, 0, r0, _NOREG, _NOREG);
	ic(0x89);
	rx(r0, i0, _NOREG, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	str_s(rn(reg), r0);
	jit_unget_reg(reg);
    }
}

static void
_str_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    rex(0, 0, r1, _NOREG, r0);
    ic(0x89);
    rx(r1, 0, r0, _NOREG, _SCL1);
}

static void
_sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	rex(0, 0, r0, _NOREG, _NOREG);
	ic(0x89);
	rx(r0, i0, _NOREG, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	str_i(rn(reg), r0);
	jit_unget_reg(reg);
    }
}

#if __X64 && !__X64_32
static void
_str_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    rex(0, 1, r1, _NOREG, r0);
    ic(0x89);
    rx(r1, 0, r0, _NOREG, _SCL1);
}

static void
_sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	rex(0, 1, r0, _NOREG, _NOREG);
	ic(0x89);
	rx(r0, i0, _NOREG, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	str_l(rn(reg), r0);
	jit_unget_reg(reg);
    }
}
#endif

static void
_stxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    jit_int32_t		reg;
#if __X64_32
    reg = jit_get_reg(jit_class_gpr);
    addr(rn(reg), r0, r1);
    str_c(rn(reg), r2);
    jit_unget_reg(reg);
#else
    if (reg8_p(r2)) {
	rex(0, 0, r2, r1, r0);
	ic(0x88);
	rx(r2, 0, r0, r1, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
	movr(rn(reg), r2);
	rex(0, 0, rn(reg), r1, r0);
	ic(0x88);
	rx(rn(reg), 0, r0, r1, _SCL1);
	jit_unget_reg(reg);
    }
#endif
}

static void
_stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	if (reg8_p(r1)) {
	    rex(0, 0, r1, _NOREG, r0);
	    ic(0x88);
	    rx(r1, i0, r0, _NOREG, _SCL1);
	}
	else {
	    reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
	    movr(rn(reg), r1);
	    rex(0, 0, rn(reg), _NOREG, r0);
	    ic(0x88);
	    rx(rn(reg), i0, r0, _NOREG, _SCL1);
	    jit_unget_reg(reg);
	}
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	stxr_c(rn(reg), r0, r1);
	jit_unget_reg(reg);
    }
}

static void
_stxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
#if __X64_32
    jit_int32_t		reg;
    reg = jit_get_reg(jit_class_gpr);
    addr(rn(reg), r0, r1);
    str_s(rn(reg), r2);
    jit_unget_reg(reg);
#else
    ic(0x66);
    rex(0, 0, r2, r1, r0);
    ic(0x89);
    rx(r2, 0, r0, r1, _SCL1);
#endif
}

static void
_stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	ic(0x66);
	rex(0, 0, r1, _NOREG, r0);
	ic(0x89);
	rx(r1, i0, r0, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	stxr_s(rn(reg), r0, r1);
	jit_unget_reg(reg);
    }
}

static void
_stxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
#if __X64_32
    jit_int32_t		reg;
    reg = jit_get_reg(jit_class_gpr);
    addr(rn(reg), r0, r1);
    str_i(rn(reg), r2);
    jit_unget_reg(reg);
#else
    rex(0, 0, r2, r1, r0);
    ic(0x89);
    rx(r2, 0, r0, r1, _SCL1);
#endif
}

static void
_stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	rex(0, 0, r1, _NOREG, r0);
	ic(0x89);
	rx(r1, i0, r0, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	stxr_i(rn(reg), r0, r1);
	jit_unget_reg(reg);
    }
}

#if __X64 && !__X64_32
static void
_stxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
{
    rex(0, 1, r2, r1, r0);
    ic(0x89);
    rx(r2, 0, r0, r1, _SCL1);
}

static void
_stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i0)) {
	rex(0, 1, r1, _NOREG, r0);
	ic(0x89);
	rx(r1, i0, r0, _NOREG, _SCL1);
    }
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i0);
	stxr_l(rn(reg), r0, r1);
	jit_unget_reg(reg);
    }
}
#endif

static void
_jccs(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
{
    jit_word_t		w;
    ic(0x70 | code);
    w = i0 - (_jit->pc.w + 1);
    ic(w);
}

static void
_jcc(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
{
    jit_word_t		w;
    ic(0x0f);
    ic(0x80 | code);
    w = i0 - (_jit->pc.w + 4);
    ii(w);
}

static void
_jcr(jit_state_t *_jit,
     jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    alur(X86_CMP, r0, r1);
    jcc(code, i0);
}

static void
_jci(jit_state_t *_jit,
     jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    alui(X86_CMP, r0, i1);
    jcc(code, i0);
}

static void
_jci0(jit_state_t *_jit, jit_int32_t code, jit_word_t i0, jit_int32_t r0)
{
    testr(r0, r0);
    jcc(code, i0);
}

static jit_word_t
_bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    jcr(X86_CC_L, i0, r0, r1);
    return (_jit->pc.w);
}

static jit_word_t
_blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    if (i1)		jci (X86_CC_L, i0, r0, i1);
    else		jci0(X86_CC_S, i0, r0);
    return (_jit->pc.w);
}

static jit_word_t
_bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    jcr(X86_CC_B, i0, r0, r1);
    return (_jit->pc.w);
}

static jit_word_t
_blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    if (i1)		jci (X86_CC_B, i0, r0, i1);
    else		jci0(X86_CC_B, i0, r0);
    return (_jit->pc.w);
}

static jit_word_t
_bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    if (r0 == r1)	jmpi(i0);
    else		jcr (X86_CC_LE, i0, r0, r1);
    return (_jit->pc.w);
}

static jit_word_t
_blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    if (i1)		jci (X86_CC_LE, i0, r0, i1);
    else		jci0(X86_CC_LE, i0, r0);
    return (_jit->pc.w);
}

static jit_word_t
_bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    if (r0 == r1)	jmpi(i0);
    else		jcr (X86_CC_BE, i0, r0, r1);
    return (_jit->pc.w);
}

static jit_word_t
_blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    if (i1)		jci (X86_CC_BE, i0, r0, i1);
    else		jci0(X86_CC_BE, i0, r0);
    return (_jit->pc.w);
}

static jit_word_t
_beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    if (r0 == r1)	jmpi(i0);
    else		jcr (X86_CC_E, i0, r0, r1);
    return (_jit->pc.w);
}

static jit_word_t
_beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    if (i1)		jci (X86_CC_E, i0, r0, i1);
    else		jci0(X86_CC_E, i0, r0);
    return (_jit->pc.w);
}

static jit_word_t
_bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    if (r0 == r1)	jmpi(i0);
    else		jcr (X86_CC_GE, i0, r0, r1);
    return (_jit->pc.w);
}

static jit_word_t
_bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    if (i1)		jci (X86_CC_GE, i0, r0, i1);
    else		jci0(X86_CC_NS, i0, r0);
    return (_jit->pc.w);
}

static jit_word_t
_bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    if (r0 == r1)	jmpi(i0);
    else		jcr (X86_CC_AE, i0, r0, r1);
    return (_jit->pc.w);
}

static jit_word_t
_bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    if (i1)		jci (X86_CC_AE, i0, r0, i1);
    else		jmpi(i0);
    return (_jit->pc.w);
}

static jit_word_t
_bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    jcr(X86_CC_G, i0, r0, r1);
    return (_jit->pc.w);
}

static jit_word_t
_bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    jci(X86_CC_G, i0, r0, i1);
    return (_jit->pc.w);
}

static jit_word_t
_bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    jcr(X86_CC_A, i0, r0, r1);
    return (_jit->pc.w);
}

static jit_word_t
_bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    if (i1)		jci (X86_CC_A, i0, r0, i1);
    else		jci0(X86_CC_NE, i0, r0);
    return (_jit->pc.w);
}

static jit_word_t
_bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    jcr(X86_CC_NE, i0, r0, r1);
    return (_jit->pc.w);
}

static jit_word_t
_bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    if (i1)		jci (X86_CC_NE, i0, r0, i1);
    else		jci0(X86_CC_NE, i0, r0);
    return (_jit->pc.w);
}

static jit_word_t
_bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    testr(r0, r1);
    jnz(i0);
    return (_jit->pc.w);
}

static jit_word_t
_bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    jit_int32_t		reg;
    if (can_zero_extend_int_p(i1))
	testi(r0, i1);
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i1);
	testr(r0, rn(reg));
	jit_unget_reg(reg);
    }
    jnz(i0);
    return (_jit->pc.w);
}

static jit_word_t
_bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    testr(r0, r1);
    jz(i0);
    return (_jit->pc.w);
}

static jit_word_t
_bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    jit_int32_t		reg;
    if (can_zero_extend_int_p(i1))
	testi(r0, i1);
    else {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), i1);
	testr(r0, rn(reg));
	jit_unget_reg(reg);
    }
    jz(i0);
    return (_jit->pc.w);
}

static jit_word_t
_boaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    iaddr(r0, r1);
    jo(i0);
    return (_jit->pc.w);
}

static jit_word_t
_boaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i1)) {
	iaddi(r0, i1);
	jo(i0);
	return (_jit->pc.w);
    }
    reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
    movi(rn(reg), i1);
    jit_unget_reg(reg);
    return (boaddr(i0, r0, rn(reg)));
}

static jit_word_t
_boaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    iaddr(r0, r1);
    jc(i0);
    return (_jit->pc.w);
}

static jit_word_t
_boaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i1)) {
	iaddi(r0, i1);
	jc(i0);
	return (_jit->pc.w);
    }
    reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
    movi(rn(reg), i1);
    jit_unget_reg(reg);
    return (boaddr_u(i0, r0, rn(reg)));
}

static jit_word_t
_bxaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    iaddr(r0, r1);
    jno(i0);
    return (_jit->pc.w);
}

static jit_word_t
_bxaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i1)) {
	iaddi(r0, i1);
	jno(i0);
	return (_jit->pc.w);
    }
    reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
    movi(rn(reg), i1);
    jit_unget_reg(reg);
    return (bxaddr(i0, r0, rn(reg)));
}

static jit_word_t
_bxaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    iaddr(r0, r1);
    jnc(i0);
    return (_jit->pc.w);
}

static jit_word_t
_bxaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i1)) {
	iaddi(r0, i1);
	jnc(i0);
	return (_jit->pc.w);
    }
    reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
    movi(rn(reg), i1);
    jit_unget_reg(reg);
    return (bxaddr_u(i0, r0, rn(reg)));
}

static jit_word_t
_bosubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    isubr(r0, r1);
    jo(i0);
    return (_jit->pc.w);
}

static jit_word_t
_bosubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i1)) {
	isubi(r0, i1);
	jo(i0);
	return (_jit->pc.w);
    }
    reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
    movi(rn(reg), i1);
    jit_unget_reg(reg);
    return (bosubr(i0, r0, rn(reg)));
}

static jit_word_t
_bosubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    isubr(r0, r1);
    jc(i0);
    return (_jit->pc.w);
}

static jit_word_t
_bosubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i1)) {
	isubi(r0, i1);
	jc(i0);
	return (_jit->pc.w);
    }
    reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
    movi(rn(reg), i1);
    jit_unget_reg(reg);
    return (bosubr_u(i0, r0, rn(reg)));
}

static jit_word_t
_bxsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    isubr(r0, r1);
    jno(i0);
    return (_jit->pc.w);
}

static jit_word_t
_bxsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i1)) {
	isubi(r0, i1);
	jno(i0);
	return (_jit->pc.w);
    }
    reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
    movi(rn(reg), i1);
    jit_unget_reg(reg);
    return (bxsubr(i0, r0, rn(reg)));
}

static jit_word_t
_bxsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
{
    isubr(r0, r1);
    jnc(i0);
    return (_jit->pc.w);
}

static jit_word_t
_bxsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
{
    jit_int32_t		reg;
    if (can_sign_extend_int_p(i1)) {
	isubi(r0, i1);
	jnc(i0);
	return (_jit->pc.w);
    }
    reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
    movi(rn(reg), i1);
    jit_unget_reg(reg);
    return (bxsubr_u(i0, r0, rn(reg)));
}

static void
_callr(jit_state_t *_jit, jit_int32_t r0)
{
    rex(0, 0, _NOREG, _NOREG, r0);
    ic(0xff);
    mrm(0x03, 0x02, r7(r0));
}

static jit_word_t
_calli(jit_state_t *_jit, jit_word_t i0)
{
    jit_word_t		word;
    jit_word_t		w;
#if __X64
    w = i0 - (_jit->pc.w + 5);
    if ((jit_int32_t)w == w) {
#endif
	ic(0xe8);
	w = i0 - (_jit->pc.w + 4);
	ii(w);
	word = _jit->pc.w;
#if __X64
    }
    else
	word = calli_p(i0);
#endif
    return (word);
}

#if __X64
static jit_word_t
_calli_p(jit_state_t *_jit, jit_word_t i0)
{
    jit_word_t		word;
    jit_int32_t		reg;
    reg = jit_get_reg(jit_class_gpr);
    word = movi_p(rn(reg), i0);
    callr(rn(reg));
    jit_unget_reg(reg);
    return (word);
}
#endif

static void
_jmpr(jit_state_t *_jit, jit_int32_t r0)
{
    rex(0, 0, _NOREG, _NOREG, r0);
    ic(0xff);
    mrm(0x03, 0x04, r7(r0));
}

static jit_word_t
_jmpi(jit_state_t *_jit, jit_word_t i0)
{
    jit_word_t		word;
    jit_word_t		w;
#if __X64
    w = i0 - (_jit->pc.w + 5);
    if ((jit_int32_t)w == w) {
#endif
	ic(0xe9);
	w = i0 - (_jit->pc.w + 4);
	ii(w);
	word = _jit->pc.w;
#if __X64
    }
    else
	word = jmpi_p(i0);
#endif
    return (word);
}

#if __X64
static jit_word_t
_jmpi_p(jit_state_t *_jit, jit_word_t i0)
{
    jit_word_t		word;
    jit_int32_t		reg;
    reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
    word = movi_p(rn(reg), i0);
    jmpr(rn(reg));
    jit_unget_reg(reg);
    return (word);
}
#endif

static void
_jmpsi(jit_state_t *_jit, jit_uint8_t i0)
{
    ic(0xeb);
    ic(i0);
}

static void
_prolog(jit_state_t *_jit, jit_node_t *node)
{
    jit_int32_t		reg;
    if (_jitc->function->define_frame || _jitc->function->assume_frame) {
	jit_int32_t	frame = -_jitc->function->frame;
	assert(_jitc->function->self.aoff >= frame);
	if (_jitc->function->assume_frame)
	    return;
	_jitc->function->self.aoff = frame;
    }
    if (_jitc->function->allocar)
	_jitc->function->self.aoff &= -16;
#if __X64 && (__CYGWIN__ || _WIN32)
    _jitc->function->stack = (((/* first 32 bytes must be allocated */
				(_jitc->function->self.alen > 32 ?
				 _jitc->function->self.alen : 32) -
				/* align stack at 16 bytes */
				_jitc->function->self.aoff) + 15) & -16) +
	stack_adjust;
#else
    _jitc->function->stack = (((_jitc->function->self.alen -
			       _jitc->function->self.aoff) + 15) & -16) +
	stack_adjust;
#endif
    subi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);
    /* callee save registers */
#if __X32
    if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
	stxi(12, _RSP_REGNO, _RDI_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
	stxi( 8, _RSP_REGNO, _RSI_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
	stxi( 4, _RSP_REGNO, _RBX_REGNO);
#else
#  if __CYGWIN__ || _WIN32
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
	sse_stxi_d(136, _RSP_REGNO, _XMM15_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
	sse_stxi_d(128, _RSP_REGNO, _XMM14_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
	sse_stxi_d(120, _RSP_REGNO, _XMM13_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
	sse_stxi_d(112, _RSP_REGNO, _XMM12_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
	sse_stxi_d(104, _RSP_REGNO, _XMM11_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
	sse_stxi_d(96, _RSP_REGNO, _XMM10_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
	sse_stxi_d(88, _RSP_REGNO, _XMM9_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
	sse_stxi_d(80, _RSP_REGNO, _XMM8_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
	sse_stxi_d(72, _RSP_REGNO, _XMM7_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
	sse_stxi_d(64, _RSP_REGNO, _XMM6_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _R15))
	stxi(56, _RSP_REGNO, _R15_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _R14))
	stxi(48, _RSP_REGNO, _R14_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _R13))
	stxi(40, _RSP_REGNO, _R13_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _R12))
	stxi(32, _RSP_REGNO, _R12_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
	stxi(24, _RSP_REGNO, _RSI_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
	stxi(16, _RSP_REGNO, _RDI_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
	stxi( 8, _RSP_REGNO, _RBX_REGNO);
#  else
    if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
	stxi(40, _RSP_REGNO, _RBX_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _R12))
	stxi(32, _RSP_REGNO, _R12_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _R13))
	stxi(24, _RSP_REGNO, _R13_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _R14))
	stxi(16, _RSP_REGNO, _R14_REGNO);
    if (jit_regset_tstbit(&_jitc->function->regset, _R15))
	stxi( 8, _RSP_REGNO, _R15_REGNO);
#  endif
#endif
    stxi(0, _RSP_REGNO, _RBP_REGNO);
    movr(_RBP_REGNO, _RSP_REGNO);

    /* alloca */
    subi(_RSP_REGNO, _RSP_REGNO, _jitc->function->stack);
    if (_jitc->function->allocar) {
	reg = jit_get_reg(jit_class_gpr);
	movi(rn(reg), _jitc->function->self.aoff);
	stxi_i(_jitc->function->aoffoff, _RBP_REGNO, rn(reg));
	jit_unget_reg(reg);
    }

#if __X64 && !(__CYGWIN__ || _WIN32)
    if (_jitc->function->self.call & jit_call_varargs) {
	jit_word_t	nofp_code;

	/* Save gp registers in the save area, if any is a vararg */
	for (reg = first_gp_from_offset(_jitc->function->vagp);
	     jit_arg_reg_p(reg); ++reg)
	    stxi(_jitc->function->vaoff + first_gp_offset +
		 reg * 8, _RBP_REGNO, rn(JIT_RA0 - reg));

	reg = first_fp_from_offset(_jitc->function->vafp);
	if (jit_arg_f_reg_p(reg)) {
	    /* Skip over if no float registers were passed as argument */
	    /* test %al, %al */
	    ic(0x84);
	    ic(0xc0);
	    jes(0);
	    nofp_code = _jit->pc.w;

	    /* Save fp registers in the save area, if any is a vararg */
	    /* Note that the full 16 byte xmm is not saved, because
	     * lightning only handles float and double, and, while
	     * attempting to provide a va_list compatible pointer as
	     * jit_va_start return, does not guarantee it (on all ports). */
	    for (; jit_arg_f_reg_p(reg); ++reg)
		sse_stxi_d(_jitc->function->vaoff + first_fp_offset +
			   reg * va_fp_increment, _RBP_REGNO, rn(_XMM0 - reg));

	    patch_rel_char(nofp_code, _jit->pc.w);
	}
    }
#endif
}

static void
_epilog(jit_state_t *_jit, jit_node_t *node)
{
    if (_jitc->function->assume_frame)
	return;
    /* callee save registers */
    movr(_RSP_REGNO, _RBP_REGNO);
#if __X32
    if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
	ldxi(_RDI_REGNO, _RSP_REGNO, 12);
    if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
	ldxi(_RSI_REGNO, _RSP_REGNO,  8);
    if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
	ldxi(_RBX_REGNO, _RSP_REGNO,  4);
#else
#  if __CYGWIN__ || _WIN32
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
	sse_ldxi_d(_XMM15_REGNO, _RSP_REGNO, 136);
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
	sse_ldxi_d(_XMM14_REGNO, _RSP_REGNO, 128);
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
	sse_ldxi_d(_XMM13_REGNO, _RSP_REGNO, 120);
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
	sse_ldxi_d(_XMM12_REGNO, _RSP_REGNO, 112);
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
	sse_ldxi_d(_XMM11_REGNO, _RSP_REGNO, 104);
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
	sse_ldxi_d(_XMM10_REGNO, _RSP_REGNO, 96);
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
	sse_ldxi_d(_XMM9_REGNO, _RSP_REGNO, 88);
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
	sse_ldxi_d(_XMM8_REGNO, _RSP_REGNO, 80);
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
	sse_ldxi_d(_XMM7_REGNO, _RSP_REGNO, 72);
    if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
	sse_ldxi_d(_XMM6_REGNO, _RSP_REGNO, 64);
    if (jit_regset_tstbit(&_jitc->function->regset, _R15))
	ldxi(_R15_REGNO, _RSP_REGNO, 56);
    if (jit_regset_tstbit(&_jitc->function->regset, _R14))
	ldxi(_R14_REGNO, _RSP_REGNO, 48);
    if (jit_regset_tstbit(&_jitc->function->regset, _R13))
	ldxi(_R13_REGNO, _RSP_REGNO, 40);
    if (jit_regset_tstbit(&_jitc->function->regset, _R12))
	ldxi(_R12_REGNO, _RSP_REGNO, 32);
    if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
	ldxi(_RSI_REGNO, _RSP_REGNO, 24);
    if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
	ldxi(_RDI_REGNO, _RSP_REGNO, 16);
    if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
	ldxi(_RBX_REGNO, _RSP_REGNO,  8);
#  else
    if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
	ldxi(_RBX_REGNO, _RSP_REGNO, 40);
    if (jit_regset_tstbit(&_jitc->function->regset, _R12))
	ldxi(_R12_REGNO, _RSP_REGNO, 32);
    if (jit_regset_tstbit(&_jitc->function->regset, _R13))
	ldxi(_R13_REGNO, _RSP_REGNO, 24);
    if (jit_regset_tstbit(&_jitc->function->regset, _R14))
	ldxi(_R14_REGNO, _RSP_REGNO, 16);
    if (jit_regset_tstbit(&_jitc->function->regset, _R15))
	ldxi(_R15_REGNO, _RSP_REGNO,  8);
#  endif
#endif
    ldxi(_RBP_REGNO, _RSP_REGNO, 0);
    addi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);

    ic(0xc3);
}

static void
_vastart(jit_state_t *_jit, jit_int32_t r0)
{
#if __X32 || __CYGWIN__ || _WIN32
    assert(_jitc->function->self.call & jit_call_varargs);
    addi(r0, _RBP_REGNO, _jitc->function->self.size);
#else
    jit_int32_t		reg;

    assert(_jitc->function->self.call & jit_call_varargs);

    /* Return jit_va_list_t in the register argument */
    addi(r0, _RBP_REGNO, _jitc->function->vaoff);
    reg = jit_get_reg(jit_class_gpr);

    /* Initialize gp offset in the save area. */
    movi(rn(reg), _jitc->function->vagp);
    stxi_i(offsetof(jit_va_list_t, gpoff), r0, rn(reg));

    /* Initialize fp offset in the save area. */
    movi(rn(reg), _jitc->function->vafp);
    stxi_i(offsetof(jit_va_list_t, fpoff), r0, rn(reg));

    /* Initialize overflow pointer to the first stack argument. */
    addi(rn(reg), _RBP_REGNO, _jitc->function->self.size);
    stxi(offsetof(jit_va_list_t, over), r0, rn(reg));

    /* Initialize register save area pointer. */
    addi(rn(reg), r0, first_gp_offset);
    stxi(offsetof(jit_va_list_t, save), r0, rn(reg));

    jit_unget_reg(reg);
#endif
}

static void
_vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
#if __X32 || __CYGWIN__ || _WIN32
    assert(_jitc->function->self.call & jit_call_varargs);
    ldr(r0, r1);
    addi(r1, r1, va_gp_increment);
#else
    jit_int32_t		rg0;
    jit_int32_t		rg1;
    jit_word_t		ge_code;
    jit_word_t		lt_code;

    assert(_jitc->function->self.call & jit_call_varargs);

    rg0 = jit_get_reg(jit_class_gpr);
    rg1 = jit_get_reg(jit_class_gpr);

    /* Load the gp offset in save area in the first temporary. */
    ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, gpoff));

    /* Jump over if there are no remaining arguments in the save area. */
    icmpi(rn(rg0), va_gp_max_offset);
    jaes(0);
    ge_code = _jit->pc.w;

    /* Load the save area pointer in the second temporary. */
    ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));

    /* Load the vararg argument in the first argument. */
    ldxr(r0, rn(rg1), rn(rg0));

    /* Update the gp offset. */
    addi(rn(rg0), rn(rg0), 8);
    stxi_i(offsetof(jit_va_list_t, gpoff), r1, rn(rg0));

    /* Will only need one temporary register below. */
    jit_unget_reg(rg1);

    /* Jump over overflow code. */
    jmpsi(0);
    lt_code = _jit->pc.w;

    /* Where to land if argument is in overflow area. */
    patch_rel_char(ge_code, _jit->pc.w);

    /* Load overflow pointer. */
    ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));

    /* Load argument. */
    ldr(r0, rn(rg0));

    /* Update overflow pointer. */
    addi(rn(rg0), rn(rg0), va_gp_increment);
    stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));

    /* Where to land if argument is in save area. */
    patch_rel_char(lt_code, _jit->pc.w);

    jit_unget_reg(rg0);
#endif
}

/* The x87 boolean argument tells if will put the result in a x87
 * register if non false, in a sse register otherwise. */
static void
_vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_bool_t x87)
{
#if __X32 || __CYGWIN__ || _WIN32
    assert(_jitc->function->self.call & jit_call_varargs);
    if (x87)
	x87_ldr_d(r0, r1);
    else
	sse_ldr_d(r0, r1);
    addi(r1, r1, 8);
#else
    jit_int32_t		rg0;
    jit_int32_t		rg1;
    jit_word_t		ge_code;
    jit_word_t		lt_code;

    assert(_jitc->function->self.call & jit_call_varargs);

    rg0 = jit_get_reg(jit_class_gpr);
    rg1 = jit_get_reg(jit_class_gpr);

    /* Load the fp offset in save area in the first temporary. */
    ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, fpoff));

    /* Jump over if there are no remaining arguments in the save area. */
    icmpi(rn(rg0), va_fp_max_offset);
    jaes(0);
    ge_code = _jit->pc.w;

    /* Load the save area pointer in the second temporary. */
    ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));

    /* Load the vararg argument in the first argument. */
    if (x87)
	x87_ldxr_d(r0, rn(rg1), rn(rg0));
    else
	sse_ldxr_d(r0, rn(rg1), rn(rg0));

    /* Update the fp offset. */
    addi(rn(rg0), rn(rg0), va_fp_increment);
    stxi_i(offsetof(jit_va_list_t, fpoff), r1, rn(rg0));

    /* Will only need one temporary register below. */
    jit_unget_reg(rg1);

    /* Jump over overflow code. */
    jmpsi(0);
    lt_code = _jit->pc.w;

    /* Where to land if argument is in overflow area. */
    patch_rel_char(ge_code, _jit->pc.w);

    /* Load overflow pointer. */
    ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));

    /* Load argument. */
    if (x87)
	x87_ldr_d(r0, rn(rg0));
    else
	sse_ldr_d(r0, rn(rg0));

    /* Update overflow pointer. */
    addi(rn(rg0), rn(rg0), 8);
    stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));

    /* Where to land if argument is in save area. */
    patch_rel_char(lt_code, _jit->pc.w);

    jit_unget_reg(rg0);
#endif
}

static void
_patch_at(jit_state_t *_jit, jit_node_t *node,
	  jit_word_t instr, jit_word_t label)
{
    switch (node->code) {
#  if __X64
	case jit_code_calli:
	case jit_code_jmpi:
#  endif
	case jit_code_movi:
	    patch_abs(instr, label);
	    break;
	default:
	    patch_rel(instr, label);
	    break;
    }
}
#endif
