/* $Id: chip_intel_80386_inline.c,v 1.3 2009/01/27 15:58:54 potyra Exp $ 
 *
 * Copyright (C) 2008-2009 FAUcc Team <info@faumachine.org>.
 * This program is free software. You can redistribute it and/or modify it
 * under the terms of the GNU General Public License, either version 2 of
 * the License, or (at your option) any later version. See COPYING.
 */

/*
 * FUNCTION NAMING CONVENTIONS
 *
 *   get_VAR()
 *
 *	returns the value of the variable specified by VAR
 *
 *   set_VAR(value)
 *
 *	assigns value to the variable specified by VAR; immediately
 *
 *   sset_VAR(value)
 *
 *	scheduled set; like set_VAR(value), but needs a commit
 *
 *   commit_VAR(value)
 *
 *	makes a scheduled set persistent
 *
 *   store_VAR()
 *
 *	set_VAR(t0)
 *
 *   load_VAR()
 *
 *	t1 = get_VAR()
 *
 *   load_VAR1_VAR2()
 *
 *	t1 = get_VAR1()
 *	t2 = get_VAR2()
 *
 * VAR
 *
 *   TODO
 *
 * ABBREVIATIONS
 *
 *   TODO
 *
 */

#include <inttypes.h>
#include <stdlib.h>
#include <stdio.h>
#include <assert.h>

typedef int bool;

/******************************************************************************/

/* forward */ void
print_state(FILE *f);

/******************************************************************************/

#if 0
static int done;
#endif

#ifndef DEBUG_CONTROL_FLOW
# define DEBUG_CONTROL_FLOW 0
#endif

#define DEBUG_EXCEPTIONS 0

#if 0
#define DEBUG(...)							\
	if (debug_enabled) {						\
		fprintf(stderr, __VA_ARGS__);				\
	}

bool debug_enabled = 0;
#else
#define DEBUG(...)
#endif

#define DEBUG_GET DEBUG
#define DEBUG_SET DEBUG

/******************************************************************************/

#ifdef INTERPRETER
int always_true = 1; /* FIXME */
#define ERROR(...)							\
	do {								\
		fprintf(stderr, "\n*** ERROR ***\n"			\
			"Line: %d\nFunction: %s\nFile: %s\n",		\
			__LINE__, __FUNCTION__, __FILE__);		\
		fprintf(stderr, __VA_ARGS__);				\
		print_state(stderr);					\
		if (always_true) {					\
			abort();					\
		}							\
	} while (0)
#else /* ! INTERPRETER */
#define ERROR(...) error(__LINE__)
#endif /* INTERPRETER */

#define ERROR_NYI() ERROR("Not yet implemented...\n")

#define DEBUG_ERROR_SWITCH() ERROR("Unhandled switch case...\n");

/******************************************************************************/

#ifndef INTERPRETER

extern uint32_t
cpu_phys_mr(uint32_t addr, unsigned int bs);
extern void
cpu_phys_mw(uint32_t addr, unsigned int bs, uint32_t val);
extern uint32_t
cpu_ior(uint32_t port, unsigned int bs);
extern void
cpu_iow(uint32_t port, unsigned int bs, uint32_t val);
extern uint8_t
cpu_irq(void);

extern void __attribute__((__noreturn__))
ret(void);
extern void
error(int line);

#endif

/******************************************************************************/

#define SEGMENT_NONE -1
#define SEGMENT_ES 0
#define SEGMENT_CS 1
#define SEGMENT_SS 2
#define SEGMENT_DS 3
#define SEGMENT_FS 4
#define SEGMENT_GS 5

/******************************************************************************
 *
 * Lock and Repeat prefixes
 *
 ******************************************************************************/

#define LR_NONE  -1
#define LR_LOCK  0  /* 0xf0: LOCK prefix				    */
#define LR_REPNZ 1  /* 0xf2: REPNE/REPNZ prefix (for string instructions)   */
#define LR_REPZ  2  /* 0xf3: REP/REPE/REPZ prefix (for string instructions) */

/******************************************************************************/

#define EFLAG_CF    0	/* Carry Flag					*/
/*		    1      Reserved (set to 1)				*/
#define EFLAG_PF    2	/* Parity Flag					*/
/*		    3      Reserved (set to 0)				*/
#define EFLAG_AF    4	/* Auxiliary Carry Flag				*/
/*		    5      Reserved (set to 0)				*/
#define EFLAG_ZF    6	/* Zero Flag					*/
#define EFLAG_SF    7	/* Sign Flag					*/
#define EFLAG_TF    8	/* Trap Flag					*/
#define EFLAG_IF    9	/* Interrupt Enable Flag			*/
#define EFLAG_DF    10	/* Direction Flag				*/
#define EFLAG_OF    11	/* Overflow Flag				*/
#define EFLAG_IOPL0 12	/* I/O Privilege Level				*/
#define EFLAG_IOPL1 13	/* I/O Privilege Level				*/
#define EFLAG_NT    14	/* Nested Task					*/
/*		    15     Reserved (set to 0)				*/
#define EFLAG_RF    16	/* Resume Flag					*/
#define EFLAG_VM    17	/* Virtual-8086 Mode				*/
#define EFLAG_AC    18	/* Alignment Check				*/
#define EFLAG_VIF   19	/* Virtual Interrupt Flag			*/
#define EFLAG_VIP   20	/* Virtual Interrupt Pending			*/
#define EFLAG_ID    21	/* ID Flag					*/
/*		    22     Reserved (set to 0)				*/
/*		    ...							*/
/*		    31     Reserved (set to 0)				*/

/******************************************************************************/

#define CR0_PE		0  /* Protection Enable				*/
#define CR0_MP		1  /* Monitor Coprocessor			*/
#define CR0_EM		2  /* Emulation					*/
#define CR0_TS		3  /* Task Switched				*/
#define CR0_ET		4  /* Extension Type				*/
#define CR0_NE		5  /* Numeric Error				*/
/*			      Reserved					*/
#define CR0_WP		16 /* Write Protect				*/
			   /* Reserved					*/
#define CR0_AM		18 /* Alignment Mask				*/
/*			      Reserved					*/
#define CR0_NW		29 /* Not Write-through				*/
#define CR0_CD		30 /* Cache Disable				*/
#define CR0_PG		31 /* Paging					*/

/******************************************************************************/

/*			      Reserved					*/
#define CR3_PWT		3  /* Page-level Writes Transparent		*/
#define CR3_PCD		4  /* Page-level Cache Disable			*/
/*			      Reserved					*/
#define CR3_PDB		12 /* 12-31: Page-Directory Base		*/

/******************************************************************************/

#define CR4_VME		0  /* Virtual-8086 Mode Extensions		*/
#define CR4_PVI		1  /* Protected-Mode Virtual Interrupts		*/
#define CR4_TSD		2  /* Time Stamp Disable			*/
#define CR4_DE		3  /* Debugging Extensions			*/
#define CR4_PSE		4  /* Page Size Extension			*/
#define CR4_PAE		5  /* Physical Address Extension		*/
#define CR4_MCE		6  /* Machine-Check Enable			*/
#define CR4_PGE		7  /* Page Global Enable			*/
#define CR4_PCE		8  /* Performance-Monitoring Counter Enable	*/
#define CR4_OSFXSR	9  /* Operating System FXSAVE/FXRSTOR Support	*/
#define CR4_OSXMMEXCPT	10 /* Operating System Unmasked Exception Support */
/*			11    Reserved (set to 0)			*/
/*			...						*/
/*			31    Reserved (set to 0)			*/

/******************************************************************************/

#define EXCEPTION_NONE	-1
#define EXCEPTION_DE	0  /* Fault: Divide Error			*/
#define EXCEPTION_DB	1  /* Fault/Trap: Debug				*/
#define EXCEPTION_NMI	2  /* Interrupt: NMI Interrupt			*/
#define EXCEPTION_BP	3  /* Trap: Breakpoint				*/
#define EXCEPTION_OF	4  /* Trap: Overflow				*/
#define EXCEPTION_BR	5  /* Fault: BOUND Range Exceeded		*/
#define EXCEPTION_UD	6  /* Fault: Invalid Opcode (Undefined Opcode)	*/
#define EXCEPTION_NM	7  /* Fault: Device Not Available (No Math Coprocessor) */
#define EXCEPTION_DF	8  /* Abort: Double Fault			*/
/*			9     Fault: Coprocessor Segment Overrun (reserved) */
#define EXCEPTION_TS	10 /* Fault: Invalid TSS			*/
#define EXCEPTION_NP	11 /* Fault: Segment Not Present		*/
#define EXCEPTION_SS	12 /* Fault: Stack-Segment Fault		*/
#define EXCEPTION_GP	13 /* Fault: General Protection			*/
#define EXCEPTION_PF	14 /* Fault: Page Fault				*/
/*			15    Intel reserved. Do not use.		*/
#define EXCEPTION_MF	16 /* Fault: Floating-Point Error (Math Fault)	*/
#define EXCEPTION_AC	17 /* Fault: Alignment Check			*/
#define EXCEPTION_MC	18 /* Abort: Machine Check			*/
#define EXCEPTION_XF	19 /* Fault: Streaming SIMD Extensions		*/
/*			20    Intel reserved. Do not use.		*/
/*			...						*/
/*			31    Intel reserved, Do not use.		*/
/*			32    Interrupt: User Defined (Nonreserved)	*/
/*			...						*/
/*			255   Interrupt: User Defined (Nonreserved)	*/

/******************************************************************************
 *
 * Code- and Data-Segment Types
 *
 ******************************************************************************/

#define SEGMENT_DATA_READ_ONLY					0x0
#define SEGMENT_DATA_READ_ONLY_ACCESSED				0x1
#define SEGMENT_DATA_READ_WRITE					0x2
#define SEGMENT_DATA_READ_WRITE_ACCESSED			0x3
#define SEGMENT_DATA_READ_ONLY_EXPAND_DOWN			0x4
#define SEGMENT_DATA_READ_ONLY_EXPAND_DOWN_ACCESSED		0x5
#define SEGMENT_DATA_READ_WRITE_EXPAND_DOWN			0x6
#define SEGMENT_DATA_READ_WRITE_EXPAND_DOWN_ACCESSED		0x7
#define SEGMENT_CODE_EXEC_ONLY					0x8
#define SEGMENT_CODE_EXEC_ONLY_ACCESSED				0x9
#define SEGMENT_CODE_EXEC_READ					0xa
#define SEGMENT_CODE_EXEC_READ_ACCESSED				0xb
#define SEGMENT_CODE_EXEC_ONLY_CONFORMING			0xc
#define SEGMENT_CODE_EXEC_ONLY_CONFORMING_ACCESSED		0xd
#define SEGMENT_CODE_EXEC_READ_CONFORMING			0xe
#define SEGMENT_CODE_EXEC_READ_CONFORMING_ACCESSED		0xf

/******************************************************************************
 *
 * System-Segment and Gate-Descriptor Types
 *
 ******************************************************************************/

/* Reserved */
#define SEGMENT_16BIT_AVAIL_TSS					0x1
#define SEGMENT_LDT						0x2
#define SEGMENT_16BIT_BUSY_TSS					0x3
#define SEGMENT_16BIT_CALL_GATE					0x4
#define SEGMENT_TASK_GATE					0x5
#define SEGMENT_16BIT_INTERRUPT_GATE				0x6
#define SEGMENT_16BIT_TRAP_GATE					0x7
/* Reserved */
#define SEGMENT_32BIT_AVAIL_TSS					0x9
/* Reserved */
#define SEGMENT_32BIT_BUSY_TSS					0xb
#define SEGMENT_32BIT_CALL_GATE					0xc
/* Reserved */
#define SEGMENT_32BIT_INTERRUPT_GATE				0xe
#define SEGMENT_32BIT_TRAP_GATE					0xf

/******************************************************************************
 *
 * Temporary Registers
 *
 ******************************************************************************/

static uint32_t t0;	/* Result					*/
static uint32_t t1;	/* Operand1					*/
static uint32_t t2;	/* Operand2					*/
static uint32_t t3;	/* Operand3 (for SHRD, SHLD)			*/
static int32_t  tj;	/* Jump offset					*/

/******************************************************************************
 *
 * General-Purpose Data Registers
 *
 ******************************************************************************/

static uint32_t eax;
static uint32_t ecx;
static uint32_t edx;
static uint32_t ebx;
static uint32_t esp, esp_backup;
static uint32_t ebp;
static uint32_t esi;
static uint32_t edi;

/******************************************************************************
 *
 * Status and Control Registers
 *
 ******************************************************************************/

static uint32_t eip, eip_new;	  /* Instruction pointer		*/

/* EFLAGS Register */
static bool eflag_cf,    eflag_cf_new;	  /* Carry Flag			*/
static bool eflag_pf,    eflag_pf_new;	  /* Parity flag		*/
static bool eflag_af,    eflag_af_new;	  /* Auxiliary Carry Flag	*/
static bool eflag_zf,    eflag_zf_new;	  /* Zero Flag			*/
static bool eflag_sf,    eflag_sf_new;	  /* Sign	Flag		*/
static bool eflag_tf,    eflag_tf_new;	  /* Trap Flag			*/
static bool eflag_if,    eflag_if_new;	  /* Interrupt Enable Flag	*/
static bool eflag_df,    eflag_df_new;	  /* Direction Flag		*/
static bool eflag_of,    eflag_of_new;	  /* Overflow Flag		*/
static bool eflag_iopl0, eflag_iopl0_new; /* I/O Privilege Level	*/
static bool eflag_iopl1, eflag_iopl1_new; /* I/O Privilege Level	*/
static bool eflag_nt,    eflag_nt_new;	  /* Nested Task Flag		*/
static bool eflag_rf,    eflag_rf_new;	  /* Resume Flag		*/
static bool eflag_vm,    eflag_vm_new;	  /* Virtual-8086 Mode		*/

/* Control Register 0:							*/
/* Contains system control flags that control				*/
/* operating mode and states of the processor				*/
static bool cr0_pe;	  /* Protection Enable				*/
static bool cr0_mp;	  /* Monitor Coprocessor			*/
static bool cr0_em;	  /* Emulation					*/
static bool cr0_ts;	  /* Task Switch				*/
static bool cr0_et;	  /* Extension Type				*/
static bool cr0_pg;	  /* Paging					*/

/* Control Register 1:							*/
/* Reserved...								*/

/* Control Register 2:							*/
/* Contains the page-fault linear address				*/
/* (the linear address that caused a page fault)			*/
static uint32_t cr2_pfla; /* Page-Fault Linear Address			*/

/* Control Register 3:							*/
/* Contains the physical address of the base of the			*/
/* page directory and two flags (PCD and PWT)				*/
static uint32_t cr3_pdb;  /* 12-31: Page-Directory Base			*/

/******************************************************************************
 *
 * Segment Selectors
 *
 * 13 index	Index in the GDT or LDT
 *  1 ti	Table Indicator (0 = GDT; 1 = LDT)
 *  2 rpl	Requested Privilege Level (RPL)
 *
 * Segment Descriptors
 *
 *  4 type	Segment type
 *  1 sflag	Descriptor type (0 = system; 1 = code or data)
 *  2 dpl	Descriptor privilege level
 *  1 pflag	Segment present
 *
 * Data-/Code-Segment Descriptors
 *
 * 32 limit	Segment limit scaled (20 bit for gflag == 0)
 * 32 base	Segment base address
 *  1 dflag:	Default operation size (0 = 16-bit segment; 1 = 32-bit segment)
 *  1 avlflag:	Available for use by system software
 *
 * System-Segment Descriptors
 *
 * 32 limit	Segment limit scaled (20 bit for gflag == 0)
 * 32 base	Segment base address
 *
 * Call-/Trap-/Interrupt-/Task-Gate Descriptors
 *
 * 16 selector	Segment Selectors
 * 32 offset	Offset in Segment
 *  5 paramcount
 *
 ******************************************************************************/

/* Segment Selector (filled by sset_selector()) */
static uint16_t selector;
static uint16_t	selector_index;
static uint8_t	selector_ti;
static uint8_t	selector_rpl;

/* Segment Descriptor (filled by sset_descriptor()) */
static uint8_t	descriptor_type;
static bool	descriptor_sflag;
static uint8_t	descriptor_dpl;
static bool	descriptor_pflag;

/* Data-/Code-Segment Descriptor (filled by sset_descriptor()) */
static uint32_t descriptor_segment_limit;
static uint32_t descriptor_segment_base;
static bool	descriptor_segment_dflag;
static bool	descriptor_segment_avlflag;

/* System-Segment Descriptor (filled by sset_descriptor()) */
static uint32_t descriptor_system_limit;
static uint32_t descriptor_system_base;

/* Call-/Trap-/Interrupt-/Task-Gate Descriptor (filled by sset_descriptor()) */
static uint16_t descriptor_gate_selector;
static uint32_t descriptor_gate_offset;
static uint8_t	descriptor_gate_paramcount;

/******************************************************************************/

/* Code Segment */
static uint16_t cs_selector;
static uint8_t  cs_type;
static bool	cs_sflag;
static uint8_t  cs_dpl;
static bool	cs_pflag;
static uint32_t cs_segment_limit;
static uint32_t cs_segment_base;
static bool	cs_segment_dflag;
static bool	cs_segment_avlflag;

/* Stack Segment */
static uint16_t ss_selector;
static uint8_t  ss_type;
static bool	ss_sflag;
static uint8_t  ss_dpl;
static bool	ss_pflag;
static uint32_t ss_segment_limit;
static uint32_t ss_segment_base;
static bool	ss_segment_dflag;
static bool	ss_segment_avlflag;

/* Data Segment */
static uint16_t ds_selector;
static uint8_t  ds_type;
static bool	ds_sflag;
static uint8_t  ds_dpl;
static bool	ds_pflag;
static uint32_t ds_segment_limit;
static uint32_t ds_segment_base;
static bool	ds_segment_dflag;
static bool	ds_segment_avlflag;

/* Data Segment */
static uint16_t es_selector;
static uint8_t  es_type;
static bool	es_sflag;
static uint8_t  es_dpl;
static bool	es_pflag;
static uint32_t es_segment_limit;
static uint32_t es_segment_base;
static bool	es_segment_dflag;
static bool	es_segment_avlflag;

/* Data Segment */
static uint16_t fs_selector;
static uint8_t  fs_type;
static bool	fs_sflag;
static uint8_t  fs_dpl;
static bool	fs_pflag;
static uint32_t fs_segment_limit;
static uint32_t fs_segment_base;
static bool	fs_segment_dflag;
static bool	fs_segment_avlflag;

/* Data Segment */
static uint16_t gs_selector;
static uint8_t  gs_type;
static bool	gs_sflag;
static uint8_t  gs_dpl;
static bool	gs_pflag;
static uint32_t gs_segment_limit;
static uint32_t gs_segment_base;
static bool	gs_segment_dflag;
static bool	gs_segment_avlflag;

/******************************************************************************
 *
 * Memory-management Registers
 *
 ******************************************************************************/

/* Global Descriptor Table Register */
static uint32_t gdtr_limit;
static uint32_t gdtr_base;

/* Interrupt Descriptor Table Register */
static uint32_t idtr_limit;
static uint32_t idtr_base;

/* Local Descriptor Table Register */
static uint16_t ldtr_selector;
/*		ldtr_type	= SEGMENT_LDT				*/
/*		ldtr_sflag	= 0					*/
/*		ldtr_dpl	not used				*/
/*		ldtr_pflag	= 1					*/
static uint32_t ldtr_system_limit;
static uint32_t ldtr_system_base;

/* Task Register */
static uint16_t tr_selector;
static uint8_t  tr_type;
/*		tr_sflag	= 0					*/
/*		tr_dpl		not used				*/
/*		tr_pflag	= 1					*/
static uint32_t tr_system_limit;
static uint32_t tr_system_base;

/******************************************************************************
 *
 * Instruction / Prefix
 *
 * prefix_*		persistent
 * instruction_*	cleared for each instruction
 *
 ******************************************************************************/

static bool prefix_operand_size_override;
static bool prefix_address_size_override;

static int prefix_segment_override;
static int prefix_lock_repeat;

/******************************************************************************/

static uint32_t instruction_opcode;
static uint32_t instruction_opcode2;
static uint32_t instruction_mod;
static uint32_t instruction_reg;
static uint32_t instruction_rm;
static uint32_t instruction_scale;
static uint32_t instruction_index;
static uint32_t instruction_base;
static uint32_t instruction_displacement;
static uint32_t instruction_immediate;

/******************************************************************************
 *
 * Exception/Interrupt/Halt State
 *
 ******************************************************************************/

static int exception_vector;		/* -1 == NONE */
static int exception_error_code;	/* -1 == NONE */
static bool exception_is_interrupt;
static bool exception_double_page_fault;

static bool interrupt_pending;
static bool interrupt_delay;

static bool halt_state;

/******************************************************************************
 *
 * Misc Functions for simple calculations
 *
 ******************************************************************************/

static inline uint32_t
limit_scaled(uint32_t limit, bool gflag)
{
	if (gflag) {
		return (limit << 12) | 0xfff;
	} else {
		return limit;
	}
}

/******************************************************************************
 *
 * Modes of Operation
 *
 ******************************************************************************/

static inline bool
real_mode(void)
{
	return !cr0_pe;
}

static inline bool
protected_mode(void)
{
	return cr0_pe && !eflag_vm;
}

static inline bool
virtual8086_mode(void)
{
	return cr0_pe && eflag_vm;
}

/******************************************************************************
 *
 * MISC - TODO
 *
 ******************************************************************************/

static inline void
prefix_clear(void)
{
	prefix_lock_repeat = LR_NONE;
	prefix_segment_override = SEGMENT_NONE;
	prefix_operand_size_override = 0;
	prefix_address_size_override = 0;
}

/******************************************************************************
 *
 * Exception
 *
 ******************************************************************************/

static void __attribute__((__noreturn__))
exception_debug(uint8_t vector, int error_code, int line)
{
	esp = esp_backup;
	eip_new = eip;
	prefix_clear();

	DEBUG("\n");

#if DEBUG_EXCEPTIONS
	fprintf(stderr, "EXCEPTION: vector=%u, error_code=0x%x, line=%d, "
			"cr2_pfla=0x%08x\n",
			vector, error_code, line, cr2_pfla);
#endif

#if 0
	if (cr2_pfla != 0xc0000000) {
		debug_enabled = 1;
	}
#endif

	if (exception_double_page_fault) {
		ERROR("Subsequent fault after two page faults...");
	}

	/* PREVIOUS exception was... */
	switch (exception_vector) {
	case EXCEPTION_NONE:
		exception_vector = vector;
		exception_error_code = error_code;
		exception_is_interrupt = 0;
		break;
	case EXCEPTION_DF:
		ERROR("Subsequent fault after double fault...");
		break;
	case EXCEPTION_PF:
		if (vector == EXCEPTION_PF) {
			exception_vector = EXCEPTION_PF;
			exception_error_code = error_code;
			exception_is_interrupt = 0;
			exception_double_page_fault = 1;
		} else {
			exception_vector = EXCEPTION_DF;
			exception_error_code = 0;
			exception_is_interrupt = 0;
		}
		break;
	default:
		/* Double Fault */
		exception_vector = EXCEPTION_DF;
		exception_error_code = 0;
		exception_is_interrupt = 0;
		break;
	}

	ret();
}

/* FIXME: macros should be upper case */
#define exception(vector, error_code)					\
	exception_debug(vector, error_code, __LINE__)

/******************************************************************************
 *
 * MMU
 *
 * cpu_virt_mr -> cpu_phys_mr
 * cpu_virt_mw -> cpu_phys_mw
 *
 ******************************************************************************/

static struct {
	struct {
		uint32_t page;
		uint32_t entry;
	} cache[4];
	uint32_t lru;
} tlb[8];

static uint32_t
cpu_virt_entry(uint32_t addr, int wflag, int uflag)
{
	uint32_t index;
	uint32_t paddr;
	uint32_t entry;
	bool was_accessed;
	bool was_dirty;


	/* Page-Directory Entry */
	index = (addr >> 22) & 0x3ff;
	paddr = (cr3_pdb << 12) | (index << 2);
	entry = cpu_phys_mr(paddr, 0xf);

	if (! (entry & 1)) { /* page-table not present */
		cr2_pfla = addr;
		exception(EXCEPTION_PF, (uflag << 2) | (wflag << 1) | 0);
	}

	was_accessed = (entry >> 5) & 1;
	if (!was_accessed) {
		entry |= 1 << 5;
		cpu_phys_mw(paddr, 0xf, entry);
	}

	if (wflag /* want to write */
	 && uflag /* in user mode, but */
	 && ! (entry & 2)) { /* page group is read only */
		cr2_pfla = addr;
		exception(EXCEPTION_PF, (uflag << 2) | (wflag << 1) | 1);
	}

	if (uflag /* want to read in user mode, but page group is assigned the */
	 && ! (entry & 4)) { /* supervisor privilege level */
		cr2_pfla = addr;
		exception(EXCEPTION_PF, (uflag << 2) | (wflag << 1) | 1);
	}


	/* Page-Table Entry */
	index = (addr >> 12) & 0x3ff;
	paddr = (entry & 0xfffff000) | (index << 2);
	entry = cpu_phys_mr(paddr, 0xf);

	if (! (entry & 1)) { /* page not present */
		cr2_pfla = addr;
		exception(EXCEPTION_PF, (uflag << 2) | (wflag << 1) | 0);
	}

	was_accessed = (entry >> 5) & 1;
	was_dirty    = (entry >> 6) & 1;
	if (!was_accessed || (wflag && !was_dirty)) {
		entry |= 1 << 5;
		entry |= wflag << 6;
		cpu_phys_mw(paddr, 0xf, entry);
	}

	if (wflag /* want to write */
	 && uflag /* in user mode, but */
	 && ! (entry & 2)) { /* page is read only */
		cr2_pfla = addr;
		exception(EXCEPTION_PF, (uflag << 2) | (wflag << 1) | 1);
	}

	if (uflag /* want to read in user mode, but page is assigned the */
	 && ! (entry & 4)) { /* supervisor privilege level */
		cr2_pfla = addr;
		exception(EXCEPTION_PF, (uflag << 2) | (wflag << 1) | 1);
	}

	return entry;
}


static uint32_t
cpu_virt_addr(uint32_t addr, int wflag, int uflag)
{
        uint32_t page;
        uint32_t offset;
        unsigned int c0;
        unsigned int c1;
        uint32_t entry;

        page = addr >> 12;
        offset = addr & 0xfff;

        c0 = page & 0x7;

        if (tlb[c0].cache[0].page == page) c1 = 0;
        else if (tlb[c0].cache[1].page == page) c1 = 1;
        else if (tlb[c0].cache[2].page == page) c1 = 2;
        else if (tlb[c0].cache[3].page == page) c1 = 3;
        else {
                c1 = tlb[c0].lru;
		/* Be careful: cpu_virt_entry might fail! */
                tlb[c0].cache[c1].entry = cpu_virt_entry(addr, wflag, uflag);
                tlb[c0].cache[c1].page = page;
                tlb[c0].lru = (tlb[c0].lru + 1) & 3;
        }

        assert(tlb[c0].cache[c1].page == page);
        entry = tlb[c0].cache[c1].entry;
        assert(entry & 1);

        /* Check write access. */
        if (wflag
         && uflag
         && ! (entry & 2)) {
                cr2_pfla = addr;
                exception(EXCEPTION_PF, (uflag << 2) | (wflag << 1) | 0);
        }
        /* Check user access. */
        if (uflag
         && ! (entry & 4)) {
                cr2_pfla = addr;
                exception(EXCEPTION_PF, (uflag << 2) | (wflag << 1) | 0);
        }

        if (wflag
         && ! ((entry >> 6) & 1)) {
                /* Set dirty bit. */
                tlb[c0].cache[c1].entry = cpu_virt_entry(addr, wflag, uflag);
        }

        return (entry & 0xfffff000) | offset;
}

static void
cpu_tlb_flush(void)
{
	unsigned int c0;
	unsigned int c1;

	for (c0 = 0; c0 < 8; c0++) {
		for (c1 = 0; c1 < 4; c1++) {
			tlb[c0].cache[c1].page = -1;
		}
	}
}


/******************************************************************************/

static uint32_t
cpu_virt_mr(uint32_t addr, unsigned int bs, uint8_t pl)
{
        if (cr0_pg) {
		addr = cpu_virt_addr(addr, 0, pl == 3);
        }

	return cpu_phys_mr(addr, bs);
}

static void
cpu_virt_mw(uint32_t addr, unsigned int bs, uint32_t val, uint8_t pl)
{
        if (cr0_pg) {
		addr = cpu_virt_addr(addr, 1, pl == 3);
	}

	cpu_phys_mw(addr, bs, val);
}

/******************************************************************************
 *
 * Memory Read Byte
 *
 ******************************************************************************/

static uint8_t
mrb(uint32_t addr, uint8_t pl)
{
        uint32_t val32;
        uint8_t val8;

	val32 = cpu_virt_mr(addr & ~3, 1 << (addr & 3), pl);
        val8 = (val32 >> ((addr & 3) * 8)) & 0xff;

        if (DEBUG_CONTROL_FLOW) {
                fprintf(stderr, "Read %02x from %08"PRIx32"\n", val8, addr);
        }

        return val8;
}

/******************************************************************************
 *
 * Memory Read Word
 *
 ******************************************************************************/

static uint16_t
mrw(uint32_t addr, uint8_t pl)
{
	uint16_t val16;
	uint32_t val32;

	if ((addr & 3) == 3) {
		val16 = mrb(addr, pl) | (mrb(addr + 1, pl) << 8);
	} else {
		val32 = cpu_virt_mr(addr & ~3, 3 << (addr & 3), pl);
		val16 = (val32 >> ((addr & 3) * 8)) & 0xffff;
	}

	if (DEBUG_CONTROL_FLOW) {
		fprintf(stderr, "Read %04x from %08"PRIx32"\n", val16, addr);
	}

	return val16;
}

/******************************************************************************
 *
 * Memory Read Double word
 *
 ******************************************************************************/

static uint32_t
mrd(uint32_t addr, uint8_t pl)
{
	uint32_t val32;

	if (addr & 3) {
		uint32_t val0;
		uint32_t val1;

		val0 = cpu_virt_mr(addr & ~3, (0xf << (addr & 3)) & 0xf, pl);
		val1 = cpu_virt_mr((addr & ~3) + 4, 0xf >> (4 - (addr & 3)),pl);

		val32 = (val0 >> ((addr & 3) * 8))
		      | (val1 << ((4 - (addr & 3)) * 8));

	} else {
		val32 = cpu_virt_mr(addr & ~3, 0xf, pl);
	}

	if (DEBUG_CONTROL_FLOW) {
		fprintf(stderr, "Read %08x from %08"PRIx32"\n", val32, addr);
	}

	return val32;
}

/******************************************************************************
 *
 * Memory Write Byte
 *
 ******************************************************************************/

static void
mwb(uint32_t addr, uint8_t value, uint8_t pl)
{
        uint32_t val32;

        if (DEBUG_CONTROL_FLOW) {
                fprintf(stderr, "Write %02x to %08"PRIx32"\n", value, addr);
        }

        val32 = value << ((addr & 3) * 8);
        cpu_virt_mw(addr & ~3, 1 << (addr & 3), val32, pl);
}

/******************************************************************************
 *
 * Memory Write Word
 *
 ******************************************************************************/

static void
mww(uint32_t addr, uint16_t value, uint8_t pl)
{
	uint32_t val32;

	if (DEBUG_CONTROL_FLOW) {
		fprintf(stderr, "Write %04x to %08"PRIx32"\n", value, addr);
	}

	if ((addr & 3) == 3) {
		uint8_t val0 = (value >> 0) & 0xff;
		uint8_t val1 = (value >> 8) & 0xff;

		mwb(addr + 0, val0, pl);
		mwb(addr + 1, val1, pl);
	} else {
		val32 = value << ((addr & 3) * 8);
		cpu_virt_mw(addr & ~3, 3 << (addr & 3), val32, pl);
	}
}

/******************************************************************************
 *
 * Memory Write Double word
 *
 ******************************************************************************/

static void
mwd(uint32_t addr, uint32_t value, uint8_t pl)
{
	if (DEBUG_CONTROL_FLOW) {
		fprintf(stderr, "Write %08x to %08"PRIx32"\n", value, addr);
	}

	if (addr & 3) {
		uint32_t val0;
		uint32_t val1;

		val0 = value << ((addr & 3) * 8);
		val1 = value >> ((4 - (addr & 3)) * 8);

		cpu_virt_mw(addr & ~3, (0xf << (addr & 3)) & 0xf, val0, pl);
		cpu_virt_mw((addr & ~3) + 4, 0xf >> (4 - (addr & 3)), val1, pl);
	} else {
		cpu_virt_mw(addr & ~3, 0xf, value, pl);
	}
}

/******************************************************************************
 *
 * Memory Read - Code Segment
 *
 ******************************************************************************/

static struct {
	uint32_t addr;
	uint32_t val;
} ic[4];

static uint32_t
instruction_cache(uint32_t addr)
{
	unsigned int c;

	c = (addr >> 2) & 3;

	if (ic[c].addr != addr) {
		/* Be careful: cpu_virt_mr might fail! */
		ic[c].val = cpu_virt_mr(addr, 0xf, cs_dpl);
		ic[c].addr = addr;
	}

	return ic[c].val;
}

static void
instruction_cache_flush(void)
{
	ic[0].addr = -1;
	ic[1].addr = -1;
	ic[2].addr = -1;
	ic[3].addr = -1;
}

static uint8_t
mrb_cs(uint32_t addr)
{
	uint8_t value;

	if (addr > cs_segment_limit) {
		exception(EXCEPTION_GP, 0);
	}
	addr += cs_segment_base;

	value = instruction_cache(addr & ~3) >> ((addr & 3) * 8);

	DEBUG("%02x ", value);

	return value;
}

static uint16_t
mrw_cs(uint32_t addr)
{
	uint16_t value;

	if (addr > cs_segment_limit - 1) {
		exception(EXCEPTION_GP, 0);
	}
	addr += cs_segment_base;

	if ((addr & 3) == 3) {
		uint32_t val0;
		uint32_t val1;

		val0 = instruction_cache((addr & ~3) + 0);
		val1 = instruction_cache((addr & ~3) + 4);

		value = (val0 >> 24) | (val1 << 8);
	} else {
		value = instruction_cache(addr & ~3) >> ((addr & 3) * 8);
	}

	DEBUG("%04x  ", value);

	return value;
}

static uint32_t
mrd_cs(uint32_t addr)
{
	uint32_t value;

	if (addr > cs_segment_limit - 3) {
		exception(EXCEPTION_GP, 0);
	}
	addr += cs_segment_base;

	if (addr & 3) {
		uint32_t val0;
		uint32_t val1;

		val0 = instruction_cache((addr & ~3) + 0);
		val1 = instruction_cache((addr & ~3) + 4);

		value = (val0 >> ((addr & 3) * 8))
		      | (val1 << ((4 - (addr & 3)) * 8));
	} else {
		value = instruction_cache(addr & ~3);
	}

	DEBUG("%08x    ", value);

	return value;
}

/******************************************************************************
 *
 * Segmented Memory Read / Write
 *
 ******************************************************************************/

static uint32_t
get_segment_base_and_check_limit(uint32_t addr, int segment, unsigned length)
{
	uint32_t segment_limit;
	uint32_t segment_base;
	int vector;

	switch (segment) {
	default: DEBUG_ERROR_SWITCH();
	case SEGMENT_ES:
		DEBUG("ES:");
		segment_limit = es_segment_limit;
		segment_base = es_segment_base;
		vector = EXCEPTION_GP;
		break;
	case SEGMENT_CS:
		DEBUG("CS:");
		segment_limit = cs_segment_limit;
		segment_base = cs_segment_base;
		vector = EXCEPTION_GP;
		break;
	case SEGMENT_SS:
		DEBUG("SS:");
		segment_limit = ss_segment_limit;
		segment_base = ss_segment_base;
		vector = EXCEPTION_SS;
		break;
	case SEGMENT_DS:
		DEBUG("DS:");
		segment_limit = ds_segment_limit;
		segment_base = ds_segment_base;
		vector = EXCEPTION_GP;
		break;
	case SEGMENT_FS:
		DEBUG("FS:");
		segment_limit = fs_segment_limit;
		segment_base = fs_segment_base;
		vector = EXCEPTION_GP;
		break;
	case SEGMENT_GS:
		DEBUG("GS:");
		segment_limit = gs_segment_limit;
		segment_base = gs_segment_base;
		vector = EXCEPTION_GP;
		break;
	}

	DEBUG("0x%08x:", addr + segment_base);

	/* TODO: Add remaining checks */

	if (addr > segment_limit - length + 1) {
		if (!real_mode()) { /* FIXME */
			exception(vector, 0);
		}
#if 0
		fprintf(stderr, "WARNING: #GP(0) address=0x%08x segment=%d "
				"length=%u segment_base=0x%08x "
				"segment_gflag=%d segment_limt=0x%08x\n",
				addr, segment, length, segment_base,
				segment_gflag, segment_limit);
#endif
	}

	return segment_base;
}

/******************************************************************************/

static uint8_t
mrb_seg(uint32_t addr, int segment)
{
	uint32_t base;
	uint8_t value;

	DEBUG("[");

	base = get_segment_base_and_check_limit(addr, segment, 1);

	value = mrb(addr + base, cs_dpl);

	DEBUG("0x%02x]", value);

	return value;
}

static uint16_t
mrw_seg(uint32_t addr, int segment)
{
	uint32_t base;
	uint16_t value;

	DEBUG("[");

	base = get_segment_base_and_check_limit(addr, segment, 2);

	value = mrw(addr + base, cs_dpl);

	DEBUG("0x%04x]", value);

	return value;
}

static uint32_t
mrd_seg(uint32_t addr, int segment)
{
	uint32_t base;
	uint32_t value;

	DEBUG("[");

	base = get_segment_base_and_check_limit(addr, segment, 4);

	value = mrd(addr + base, cs_dpl);

	DEBUG("0x%08x]", value);

	return value;
}

/******************************************************************************/

static void
mwb_seg(uint32_t addr, uint8_t value, int segment)
{
	uint32_t base;

	DEBUG("{");

	base = get_segment_base_and_check_limit(addr, segment, 1);

	DEBUG("0x%02x}", value);

	mwb(addr + base, value, cs_dpl);
}

static void
mww_seg(uint32_t addr, uint16_t value, int segment)
{
	uint32_t base;

	DEBUG("{");

	base = get_segment_base_and_check_limit(addr, segment, 2);

	DEBUG("0x%04x}", value);

	mww(addr + base, value, cs_dpl);
}

static void
mwd_seg(uint32_t addr, uint32_t value, int segment)
{
	uint32_t base;

	DEBUG("{");

	base = get_segment_base_and_check_limit(addr, segment, 4);

	DEBUG("0x%08x}", value);

	mwd(addr + base, value, cs_dpl);
}

/******************************************************************************
 *
 * LOCK and UNLOCK
 *
 ******************************************************************************/

static inline void
lock(void)
{
	/* TODO? */
}

static inline void
unlock(void)
{
	/* TODO? */
}

/******************************************************************************
 *
 * EFLAGS Register Accessors
 *
 ******************************************************************************/

#define DEFINE_GET_EFLAG(FLAG, flag)					\
static inline bool								\
get_##FLAG(void)							\
{									\
	return eflag_##flag;						\
}

DEFINE_GET_EFLAG(CF, cf)
DEFINE_GET_EFLAG(PF, pf)
DEFINE_GET_EFLAG(AF, af)
DEFINE_GET_EFLAG(ZF, zf)
DEFINE_GET_EFLAG(SF, sf)
DEFINE_GET_EFLAG(TF, tf)
DEFINE_GET_EFLAG(IF, if)
DEFINE_GET_EFLAG(DF, df)
DEFINE_GET_EFLAG(OF, of)
DEFINE_GET_EFLAG(IOPL0, iopl0)
DEFINE_GET_EFLAG(IOPL1, iopl1)
DEFINE_GET_EFLAG(NT, nt)
DEFINE_GET_EFLAG(RF, rf)
DEFINE_GET_EFLAG(VM, vm)

static inline uint8_t
get_EFLAGS8(void)
{
	uint8_t eflags;

	eflags = 0;
	eflags |= get_CF() << EFLAG_CF;
	eflags |= 1 << 1;
	eflags |= get_PF() << EFLAG_PF;
	/* eflags |= 0 << 3; */
	eflags |= get_AF() << EFLAG_AF;
	/* eflags |= 0 << 5; */
	eflags |= get_ZF() << EFLAG_ZF;
	eflags |= get_SF() << EFLAG_SF;

	return eflags;
}

static inline uint16_t
get_EFLAGS16(void)
{
	uint16_t eflags;

	eflags = 0;
	eflags |= get_EFLAGS8();
	eflags |= get_TF() << EFLAG_TF;
	eflags |= get_IF() << EFLAG_IF;
	eflags |= get_DF() << EFLAG_DF;
	eflags |= get_OF() << EFLAG_OF;
	eflags |= get_IOPL0() << EFLAG_IOPL0;
	eflags |= get_IOPL1() << EFLAG_IOPL1;
	eflags |= get_NT() << EFLAG_NT;
	/* eflags |= 0 << 15; */

	return eflags;
}

static inline uint32_t
get_EFLAGS32(void)
{
	uint32_t eflags;

	eflags = 0;
	eflags |= get_EFLAGS16();
	eflags |= get_RF() << EFLAG_RF;
	eflags |= get_VM() << EFLAG_VM;

	return eflags;
}

static inline uint8_t
get_IOPL(void)
{
	return (get_IOPL1() << 1) | get_IOPL0();
}

/******************************************************************************/

#define DEFINE_SSET_EFLAG(FLAG, flag)					\
static inline void								\
sset_##FLAG(bool value)							\
{									\
	eflag_##flag##_new = !!value;					\
}

DEFINE_SSET_EFLAG(CF, cf)
DEFINE_SSET_EFLAG(PF, pf)
DEFINE_SSET_EFLAG(AF, af)
DEFINE_SSET_EFLAG(ZF, zf)
DEFINE_SSET_EFLAG(SF, sf)
DEFINE_SSET_EFLAG(TF, tf)
DEFINE_SSET_EFLAG(IF, if)
DEFINE_SSET_EFLAG(DF, df)
DEFINE_SSET_EFLAG(OF, of)
DEFINE_SSET_EFLAG(IOPL0, iopl0)
DEFINE_SSET_EFLAG(IOPL1, iopl1)
DEFINE_SSET_EFLAG(NT, nt)
DEFINE_SSET_EFLAG(RF, rf)
DEFINE_SSET_EFLAG(VM, vm)

static inline void
sset_EFLAGS8(uint8_t eflags)
{
	sset_CF((eflags >> EFLAG_CF) & 1);
	/* Reserved */
	sset_PF((eflags >> EFLAG_PF) & 1);
	/* Reserved */
	sset_AF((eflags >> EFLAG_AF) & 1);
	/* Reserved */
	sset_ZF((eflags >> EFLAG_ZF) & 1);
	sset_SF((eflags >> EFLAG_SF) & 1);
}

static inline void
sset_EFLAGS16(uint16_t eflags)
{
	sset_EFLAGS8(eflags);
	sset_TF((eflags >> EFLAG_TF) & 1);
	sset_IF((eflags >> EFLAG_IF) & 1);
	sset_DF((eflags >> EFLAG_DF) & 1);
	sset_OF((eflags >> EFLAG_OF) & 1);
	sset_IOPL0((eflags >> EFLAG_IOPL0) & 1);
	sset_IOPL1((eflags >> EFLAG_IOPL1) & 1);
	sset_NT((eflags >> EFLAG_NT) & 1);
	/* Reserved */
}

static inline void
sset_EFLAGS32(uint32_t eflags)
{
	sset_EFLAGS16(eflags);
	sset_RF((eflags >> EFLAG_RF) & 1);
	sset_VM((eflags >> EFLAG_VM) & 1);
}

/******************************************************************************/

#define DEFINE_COMMIT_EFLAG(FLAG, flag)					\
static inline void								\
commit_##FLAG(void)							\
{									\
	eflag_##flag = eflag_##flag##_new;				\
}

DEFINE_COMMIT_EFLAG(CF, cf)
DEFINE_COMMIT_EFLAG(PF, pf)
DEFINE_COMMIT_EFLAG(AF, af)
DEFINE_COMMIT_EFLAG(ZF, zf)
DEFINE_COMMIT_EFLAG(SF, sf)
DEFINE_COMMIT_EFLAG(TF, tf)
DEFINE_COMMIT_EFLAG(IF, if)
DEFINE_COMMIT_EFLAG(DF, df)
DEFINE_COMMIT_EFLAG(OF, of)
DEFINE_COMMIT_EFLAG(IOPL0, iopl0)
DEFINE_COMMIT_EFLAG(IOPL1, iopl1)
DEFINE_COMMIT_EFLAG(NT, nt)
DEFINE_COMMIT_EFLAG(RF, rf)
DEFINE_COMMIT_EFLAG(VM, vm)

static inline void
commit_EFLAGS8(void)
{
	commit_CF();
	commit_PF();
	commit_AF();
	commit_ZF();
	commit_SF();
}

static inline void
commit_EFLAGS16(void)
{
	commit_EFLAGS8();
	commit_TF();
	commit_IF();
	commit_DF();
	commit_OF();
	commit_IOPL0();
	commit_IOPL1();
	commit_NT();
}

static inline void
commit_EFLAGS32(void)
{
	commit_EFLAGS16();
	commit_RF();
	commit_VM();
}

static inline void
commit_OSZAP(void)
{
	commit_OF();
	commit_SF();
	commit_ZF();
	commit_AF();
	commit_PF();
}

static inline void
commit_OSZAPC(void)
{
	commit_OF();
	commit_SF();
	commit_ZF();
	commit_AF();
	commit_PF();
	commit_CF();
}

/******************************************************************************
 *
 * Control Register Accessors
 *
 ******************************************************************************/

#define DEFINE_SET_CR(no, FLAG, flag)					\
static inline void								\
set_##FLAG(bool value)							\
{									\
	cr##no##_##flag = !!value;					\
}

DEFINE_SET_CR(0, PE, pe)
DEFINE_SET_CR(0, MP, mp)
DEFINE_SET_CR(0, EM, em)
DEFINE_SET_CR(0, TS, ts)
DEFINE_SET_CR(0, ET, et)
DEFINE_SET_CR(0, PG, pg)

static inline void
set_PFLA(uint32_t value)
{
	cr2_pfla = value;
}

static inline void
set_PDB(uint32_t value)
{
	cr3_pdb = value & 0xfffff; /* bits 12-31 */
	cpu_tlb_flush();
}

static inline void
set_CR0(uint32_t cr0)
{
	set_PE((cr0 >> CR0_PE) & 1);
	set_MP((cr0 >> CR0_MP) & 1);
	set_EM((cr0 >> CR0_EM) & 1);
	set_TS((cr0 >> CR0_TS) & 1);
	set_ET((cr0 >> CR0_ET) & 1);
	set_PG((cr0 >> CR0_PG) & 1);
}

static inline void
set_CR2(uint32_t cr2)
{
	set_PFLA(cr2);
}

static inline void
set_CR3(uint32_t cr3)
{
	set_PDB((cr3 >> CR3_PDB) & 0xfffff); /* bits 12-31 */
}

/******************************************************************************/

#define DEFINE_GET_CR(no, FLAG, flag)					\
static inline bool								\
get_##FLAG(void)							\
{									\
	return cr##no##_##flag;						\
}

DEFINE_GET_CR(0, PE, pe)
DEFINE_GET_CR(0, MP, mp)
DEFINE_GET_CR(0, EM, em)
DEFINE_GET_CR(0, TS, ts)
DEFINE_GET_CR(0, ET, et)
DEFINE_GET_CR(0, PG, pg)

static inline uint32_t
get_PFLA(void)
{
	return cr2_pfla;
}

static inline uint32_t
get_PDB(void)
{
	return cr3_pdb;
}

static inline uint32_t
get_CR0(void)
{
	uint32_t cr0;

	cr0 = 0;
	cr0 |= get_PE() << CR0_PE;
	cr0 |= get_MP() << CR0_MP;
	cr0 |= get_EM() << CR0_EM;
	cr0 |= get_TS() << CR0_TS;
	cr0 |= get_ET() << CR0_ET;
	/* Reserved... */
	cr0 |= get_PG() << CR0_PG;

	return cr0;
}

static inline uint32_t
get_CR2(void)
{
	return get_PFLA();
}

static inline uint32_t
get_CR3(void)
{
	uint32_t cr3;

	cr3 = 0;
	/* Reserved... */
	cr3 |= get_PDB() << CR3_PDB;

	return cr3;
}

/******************************************************************************
 *
 * Effective Operand- and Address-Size Attributes
 *
 * D Flag in Code Segment Descriptor	 0   0   0   0   1   1   1   1
 * Operand-Size Prefix 0x66		 N   N   Y   Y   N   N   Y   Y
 * Address-Size Prefix 0x67		 N   Y   N   Y   N   Y   N   Y
 * Effective Operand Size		16  16  32  32  32  32  16  16
 * Effective Address Size		16  32  16  32  32  16  32  16
 *
 ******************************************************************************/

static inline bool
operand_size_32(void)
{
	return (cs_segment_dflag ^ prefix_operand_size_override);
}

static inline bool
address_size_32(void)
{
	return (cs_segment_dflag ^ prefix_address_size_override);
}

static inline bool
stack_size_32(void)
{
	return ss_segment_dflag;
}

/******************************************************************************
 *
 * Fetch Immediate
 *
 ******************************************************************************/

static inline void
fetch_immb(void)
{
	instruction_immediate = mrb_cs(eip_new);
	eip_new += 1;
}

static inline void
fetch_immw(void)
{
	instruction_immediate = mrw_cs(eip_new);
	eip_new += 2;
}

static inline void
fetch_immd(void)
{
	instruction_immediate = mrd_cs(eip_new);
	eip_new += 4;
}

static inline void
fetch_immv(bool size_32)
{
	if (size_32) {
		fetch_immd();
	} else {
		fetch_immw();
	}
}

static inline void
fetch_ptr(void)
{
	if (operand_size_32()) {
		instruction_displacement = mrd_cs(eip_new);
		eip_new += 4;
	} else {
		instruction_displacement = mrw_cs(eip_new);
		eip_new += 2;
	}

	instruction_immediate = mrw_cs(eip_new);
	eip_new += 2;
}

/******************************************************************************
 *
 * Fetch ModR/M
 *
 * Fetch the ModR/M byte and if required: 1 SIB byte and 1, 2 or 4 bytes
 * displacement.
 *
 * Mod/RM:
 *     7 6   5 4 3   2 1 0
 *     Mod    Reg     R/M
 *
 * SIB:
 *     7 6   5 4 3   2 1 0
 *    Scale  Index   Base
 *
 ******************************************************************************/

static inline void
fetch_modrm(void)
{
	uint8_t modrm;

	modrm = mrb_cs(eip_new);
	eip_new += 1;

	instruction_mod = (modrm >> 6) & 3;
	instruction_reg = (modrm >> 3) & 7;
	instruction_rm = modrm & 7;

	if (address_size_32() && instruction_mod != 3 && instruction_rm == 4) {
		uint8_t sib;

		sib = mrb_cs(eip_new);
		eip_new += 1;

		instruction_scale = (sib >> 6) & 3;
		instruction_index = (sib >> 3) & 7;
		instruction_base = sib & 7;
	} else {
		instruction_scale = 0;
		instruction_index = 0;
		instruction_base = 0;
	}

	if (instruction_mod == 1) {
		instruction_displacement = mrb_cs(eip_new);
		eip_new += 1;
	} else {
		instruction_displacement = 0;
		if (address_size_32()) {
			switch (instruction_mod) {
			case 0:
				if ((instruction_rm != 5)
				 && (instruction_base != 5)) {
					break;
				}
			case 2:
				instruction_displacement = mrd_cs(eip_new);
				eip_new += 4;
				break;
			default:
				break;
			}
		} else {
			switch (instruction_mod) {
			case 0:
				if (instruction_rm != 6) {
					break;
				}
			case 2:
				instruction_displacement = mrw_cs(eip_new);
				eip_new += 2;
				break;
			default:
				break;
			}
		}
	}
}

/******************************************************************************
 *
 * Get Instruction *
 *
 ******************************************************************************/

static inline uint32_t
get_instruction_mod(void)
{
	DEBUG("<MOD:0x%x>", instruction_mod);
	return instruction_mod;
}

static inline uint32_t
get_instruction_reg(void)
{
	DEBUG("<REG:0x%x>", instruction_reg);
	return instruction_reg;
}

static inline uint32_t
get_instruction_rm(void)
{
	DEBUG("<RM:0x%x>", instruction_rm);
	return instruction_rm;
}

static inline uint32_t
get_instruction_scale(void)
{
	DEBUG("<SCALE:0x%x>", instruction_scale);
	return instruction_scale;
}

static inline uint32_t
get_instruction_index(void)
{
	DEBUG("<INDEX:0x%x>", instruction_index);
	return instruction_index;
}

static inline uint32_t
get_instruction_base(void)
{
	DEBUG("<BASE:0x%x>", instruction_base);
	return instruction_base;
}

static inline uint32_t
get_instruction_displacement(void)
{
	DEBUG("<DISP:0x%x>", instruction_displacement);
	return instruction_displacement;
}

static inline uint32_t
get_instruction_immediate(void)
{
	DEBUG("<IMM:0x%x>", instruction_immediate);
	return instruction_immediate;
}

/******************************************************************************
 *
 * Get
 *
 ******************************************************************************/

static inline uint8_t
get_CPL(void)
{
	return cs_dpl;
}

static inline uint32_t
get_CS(void)
{
	DEBUG_GET("[CS:0x%04x]", cs_selector);
	return cs_selector;
}

static inline uint32_t
get_SS(void)
{
	DEBUG_GET("[SS:0x%04x]", ss_selector);
	return ss_selector;
}

static inline uint32_t
get_DS(void)
{
	DEBUG_GET("[DS:0x%04x]", ds_selector);
	return ds_selector;
}

static inline uint32_t
get_ES(void)
{
	DEBUG_GET("[ES:0x%04x]", es_selector);
	return es_selector;
}

static inline uint32_t
get_FS(void)
{
	DEBUG_GET("[FS:0x%04x]", fs_selector);
	return fs_selector;
}

static inline uint32_t
get_GS(void)
{
	DEBUG_GET("[GS:0x%04x]", gs_selector);
	return gs_selector;
}

/******************************************************************************/

static inline uint32_t
get_AL(void)
{
	DEBUG_GET("[AL:0x%02x]", eax & 0xff);
	return eax & 0xff;
}

static inline uint32_t
get_CL(void)
{
	DEBUG_GET("[CL:0x%02x]", ecx & 0xff);
	return ecx & 0xff;
}

static inline uint32_t
get_DL(void)
{
	DEBUG_GET("[DL:0x%02x]", edx & 0xff);
	return edx & 0xff;
}

static inline uint32_t
get_BL(void)
{
	DEBUG_GET("[BL:0x%02x]", ebx & 0xff);
	return ebx & 0xff;
}

/******************************************************************************/

static inline uint32_t
get_AH(void)
{
	DEBUG_GET("[AH:0x%02x]", (eax >> 8) & 0xff);
	return (eax >> 8) & 0xff;
}

static inline uint32_t
get_CH(void)
{
	DEBUG_GET("[CH:0x%02x]", (ecx >> 8) & 0xff);
	return (ecx >> 8) & 0xff;
}

static inline uint32_t
get_DH(void)
{
	DEBUG_GET("[DH:0x%02x]", (edx >> 8) & 0xff);
	return (edx >> 8) & 0xff;
}

static inline uint32_t
get_BH(void)
{
	DEBUG_GET("[BH:0x%02x]", (ebx >> 8) & 0xff);
	return (ebx >> 8) & 0xff;
}

/******************************************************************************/

static inline uint32_t
get_AX(void)
{
	DEBUG_GET("[AX:0x%04x]", eax & 0xffff);
	return eax & 0xffff;
}

static inline uint32_t
get_CX(void)
{
	DEBUG_GET("[CX:0x%04x]", ecx & 0xffff);
	return ecx & 0xffff;
}

static inline uint32_t
get_DX(void)
{
	DEBUG_GET("[DX:0x%04x]", edx & 0xffff);
	return edx & 0xffff;
}

static inline uint32_t
get_BX(void)
{
	DEBUG_GET("[BX:0x%04x]", ebx & 0xffff);
	return ebx & 0xffff;
}

/******************************************************************************/

static inline uint32_t
get_SP(void)
{
	DEBUG_GET("[SP:0x%04x]", esp & 0xffff);
	return esp & 0xffff;
}

static inline uint32_t
get_BP(void)
{
	DEBUG_GET("[BP:0x%04x]", ebp & 0xffff);
	return ebp & 0xffff;
}

static inline uint32_t
get_SI(void)
{
	DEBUG_GET("[SI:0x%04x]", esi & 0xffff);
	return esi & 0xffff;
}

static inline uint32_t
get_DI(void)
{
	DEBUG_GET("[DI:0x%04x]", edi & 0xffff);
	return edi & 0xffff;
}

/******************************************************************************/

static inline uint32_t
get_EAX(void)
{
	DEBUG_GET("[EAX:0x%08x]", eax);
	return eax;
}

static inline uint32_t
get_ECX(void)
{
	DEBUG_GET("[ECX:0x%08x]", ecx);
	return ecx;
}

static inline uint32_t
get_EDX(void)
{
	DEBUG_GET("[EDX:0x%08x]", edx);
	return edx;
}

static inline uint32_t
get_EBX(void)
{
	DEBUG_GET("[EBX:0x%08x]", ebx);
	return ebx;
}

/******************************************************************************/

static inline uint32_t
get_ESP(void)
{
	DEBUG_GET("[ESP:0x%08x]", esp);
	return esp;
}

static inline uint32_t
get_EBP(void)
{
	DEBUG_GET("[EBP:0x%08x]", ebp);
	return ebp;
}

static inline uint32_t
get_ESI(void)
{
	DEBUG_GET("[ESI:0x%08x]", esi);
	return esi;
}

static inline uint32_t
get_EDI(void)
{
	DEBUG_GET("[EDI:0x%08x]", edi);
	return edi;
}

/******************************************************************************/

static inline uint32_t
get_eAX(void)
{
	if (operand_size_32()) {
		return get_EAX();
	} else {
		return get_AX();
	}
}

static inline uint32_t
get_eCX(void)
{
	if (operand_size_32()) {
		return get_ECX();
	} else {
		return get_CX();
	}
}

static inline uint32_t
get_eDX(void)
{
	if (operand_size_32()) {
		return get_EDX();
	} else {
		return get_DX();
	}
}

static inline uint32_t
get_eBX(void)
{
	if (operand_size_32()) {
		return get_EBX();
	} else {
		return get_BX();
	}
}

/******************************************************************************/

static inline uint32_t
get_eSP(void)
{
	if (operand_size_32()) {
		return get_ESP();
	} else {
		return get_SP();
	}
}

static inline uint32_t
get_eBP(void)
{
	if (operand_size_32()) {
		return get_EBP();
	} else {
		return get_BP();
	}
}

static inline uint32_t
get_eSI(void)
{
	if (operand_size_32()) {
		return get_ESI();
	} else {
		return get_SI();
	}
}

static inline uint32_t
get_eDI(void)
{
	if (operand_size_32()) {
		return get_EDI();
	} else {
		return get_DI();
	}
}

/******************************************************************************/

static inline int
get_segment_overwritten(int segment)
{
	if (prefix_segment_override == SEGMENT_NONE) {
		return segment;
	} else {
		return prefix_segment_override;
	}
}

/******************************************************************************/

static inline uint32_t
get_address_sib(void)
{
	uint32_t addr = 0;

	switch (get_instruction_index()) {
	default: DEBUG_ERROR_SWITCH();
	case 0: addr += get_EAX(); break;
	case 1: addr += get_ECX(); break;
	case 2: addr += get_EDX(); break;
	case 3: addr += get_EBX(); break;
	case 4: break;
	case 5: addr += get_EBP(); break;
	case 6: addr += get_ESI(); break;
	case 7: addr += get_EDI(); break;
	}

	switch (get_instruction_scale()) {
	default: DEBUG_ERROR_SWITCH();
	case 0: break;
	case 1: addr *= 2; break;
	case 2: addr *= 4; break;
	case 3: addr *= 8; break;
	}

	switch (get_instruction_base()) {
	default: DEBUG_ERROR_SWITCH();
	case 0: addr += get_EAX(); break;
	case 1: addr += get_ECX(); break;
	case 2: addr += get_EDX(); break;
	case 3: addr += get_EBX(); break;
	case 4: addr += get_ESP(); break;
	case 5: if (get_instruction_mod() == 0) {
			addr += get_instruction_displacement();
		} else {
			addr += get_EBP();
		} break;
	case 6: addr += get_ESI(); break;
	case 7: addr += get_EDI(); break;
	}

	DEBUG("<SIB:0x%x>", addr);
	return addr;
}

static inline int
get_segment_sib(void)
{
	switch (get_instruction_base()) {
	default: DEBUG_ERROR_SWITCH();
	case 0: return SEGMENT_DS;
	case 1: return SEGMENT_DS;
	case 2: return SEGMENT_DS;
	case 3: return SEGMENT_DS;
	case 4: return SEGMENT_SS;
	case 5: if (get_instruction_mod() == 0) {
			return SEGMENT_DS;
		} else {
			return SEGMENT_SS;
		}
	case 6: return SEGMENT_DS;
	case 7: return SEGMENT_DS;
	}
}

/******************************************************************************/

static inline uint32_t
get_address_mod0(void)
{
	uint32_t addr = 0;

	if (address_size_32()) {
		switch (get_instruction_rm()) {
		default: DEBUG_ERROR_SWITCH();
		case 0: addr += get_EAX(); break;
		case 1: addr += get_ECX(); break;
		case 2: addr += get_EDX(); break;
		case 3: addr += get_EBX(); break;
		case 4: addr += get_address_sib(); break;
		case 5: addr += get_instruction_displacement(); break;
		case 6: addr += get_ESI(); break;
		case 7: addr += get_EDI(); break;
		}
	} else {
		switch (get_instruction_rm()) {
		default: DEBUG_ERROR_SWITCH();
		case 0: addr += get_BX() + get_SI(); break;
		case 1: addr += get_BX() + get_DI(); break;
		case 2: addr += get_BP() + get_SI(); break;
		case 3: addr += get_BP() + get_DI(); break;
		case 4: addr += get_SI(); break;
		case 5: addr += get_DI(); break;
		case 6: addr += get_instruction_displacement(); break;
		case 7: addr += get_BX(); break;
		}

		addr &= 0xffff;
	}

	DEBUG("<MOD0:0x%x>", addr);

	return addr;
}

static inline int
get_segment_mod0(void)
{
	if (address_size_32()) {
		switch (get_instruction_rm()) {
		default: DEBUG_ERROR_SWITCH();
		case 0: return SEGMENT_DS;
		case 1: return SEGMENT_DS;
		case 2: return SEGMENT_DS;
		case 3: return SEGMENT_DS;
		case 4: return get_segment_sib();
		case 5: return SEGMENT_DS;
		case 6: return SEGMENT_DS;
		case 7: return SEGMENT_DS;
		}
	} else {
		switch (get_instruction_rm()) {
		default: DEBUG_ERROR_SWITCH();
		case 0: return SEGMENT_DS;
		case 1: return SEGMENT_DS;
		case 2: return SEGMENT_SS;
		case 3: return SEGMENT_SS;
		case 4: return SEGMENT_DS;
		case 5: return SEGMENT_DS;
		case 6: return SEGMENT_DS;
		case 7: return SEGMENT_DS;
		}
	}
}

/******************************************************************************/

static inline uint32_t
get_address_mod1(void)
{
	uint32_t addr = 0;

	addr += (int8_t) get_instruction_displacement();

	if (address_size_32()) {
		switch (get_instruction_rm()) {
		default: DEBUG_ERROR_SWITCH();
		case 0: addr += get_EAX(); break;
		case 1: addr += get_ECX(); break;
		case 2: addr += get_EDX(); break;
		case 3: addr += get_EBX(); break;
		case 4: addr += get_address_sib(); break;
		case 5: addr += get_EBP(); break;
		case 6: addr += get_ESI(); break;
		case 7: addr += get_EDI(); break;
		}
	} else {
		switch (get_instruction_rm()) {
		default: DEBUG_ERROR_SWITCH();
		case 0: addr += get_BX() + get_SI(); break;
		case 1: addr += get_BX() + get_DI(); break;
		case 2: addr += get_BP() + get_SI(); break;
		case 3: addr += get_BP() + get_DI(); break;
		case 4: addr += get_SI(); break;
		case 5: addr += get_DI(); break;
		case 6: addr += get_BP(); break;
		case 7: addr += get_BX(); break;
		}

		addr &= 0xffff;
	}

	DEBUG("<MOD1:0x%x>", addr);

	return addr;
}

static inline int
get_segment_mod1(void)
{
	if (address_size_32()) {
		switch (get_instruction_rm()) {
		default: DEBUG_ERROR_SWITCH();
		case 0: return SEGMENT_DS;
		case 1: return SEGMENT_DS;
		case 2: return SEGMENT_DS;
		case 3: return SEGMENT_DS;
		case 4: return get_segment_sib();
		case 5: return SEGMENT_SS;
		case 6: return SEGMENT_DS;
		case 7: return SEGMENT_DS;
		}
	} else {
		switch (get_instruction_rm()) {
		default: DEBUG_ERROR_SWITCH();
		case 0: return SEGMENT_DS;
		case 1: return SEGMENT_DS;
		case 2: return SEGMENT_SS;
		case 3: return SEGMENT_SS;
		case 4: return SEGMENT_DS;
		case 5: return SEGMENT_DS;
		case 6: return SEGMENT_SS;
		case 7: return SEGMENT_DS;
		}
	}
}

/******************************************************************************/

static inline uint32_t
get_address_mod2(void)
{
	uint32_t addr = 0;

	addr += get_instruction_displacement();

	if (address_size_32()) {
		switch (get_instruction_rm()) {
		default: DEBUG_ERROR_SWITCH();
		case 0: addr += get_EAX(); break;
		case 1: addr += get_ECX(); break;
		case 2: addr += get_EDX(); break;
		case 3: addr += get_EBX(); break;
		case 4: addr += get_address_sib(); break;
		case 5: addr += get_EBP(); break;
		case 6: addr += get_ESI(); break;
		case 7: addr += get_EDI(); break;
		}
	} else {
		switch (get_instruction_rm()) {
		default: DEBUG_ERROR_SWITCH();
		case 0: addr += get_BX() + get_SI(); break;
		case 1: addr += get_BX() + get_DI(); break;
		case 2: addr += get_BP() + get_SI(); break;
		case 3: addr += get_BP() + get_DI(); break;
		case 4: addr += get_SI(); break;
		case 5: addr += get_DI(); break;
		case 6: addr += get_BP(); break;
		case 7: addr += get_BX(); break;
		}

		addr &= 0xffff;
	}

	DEBUG("<MOD2:0x%x>", addr);

	return addr;
}

static inline int
get_segment_mod2(void)
{
	if (address_size_32()) {
		switch (get_instruction_rm()) {
		default: DEBUG_ERROR_SWITCH();
		case 0: return SEGMENT_DS;
		case 1: return SEGMENT_DS;
		case 2: return SEGMENT_DS;
		case 3: return SEGMENT_DS;
		case 4: return get_segment_sib();
		case 5: return SEGMENT_SS;
		case 6: return SEGMENT_DS;
		case 7: return SEGMENT_DS;
		}
	} else {
		switch (get_instruction_rm()) {
		default: DEBUG_ERROR_SWITCH();
		case 0: return SEGMENT_DS;
		case 1: return SEGMENT_DS;
		case 2: return SEGMENT_SS;
		case 3: return SEGMENT_SS;
		case 4: return SEGMENT_DS;
		case 5: return SEGMENT_DS;
		case 6: return SEGMENT_SS;
		case 7: return SEGMENT_DS;
		}
	}
}

/******************************************************************************/

static inline uint32_t
get_Rb(uint32_t reg)
{
	switch (reg) {
	default: DEBUG_ERROR_SWITCH();
	case 0: return get_AL();
	case 1: return get_CL();
	case 2: return get_DL();
	case 3: return get_BL();
	case 4: return get_AH();
	case 5: return get_CH();
	case 6: return get_DH();
	case 7: return get_BH();
	}
}

static inline uint32_t
get_Rw(uint32_t reg)
{
	switch (reg) {
	default: DEBUG_ERROR_SWITCH();
	case 0: return get_AX();
	case 1: return get_CX();
	case 2: return get_DX();
	case 3: return get_BX();
	case 4: return get_SP();
	case 5: return get_BP();
	case 6: return get_SI();
	case 7: return get_DI();
	}
}

static inline uint32_t
get_Rd(uint32_t reg)
{
	switch (reg) {
	default: DEBUG_ERROR_SWITCH();
	case 0: return get_EAX();
	case 1: return get_ECX();
	case 2: return get_EDX();
	case 3: return get_EBX();
	case 4: return get_ESP();
	case 5: return get_EBP();
	case 6: return get_ESI();
	case 7: return get_EDI();
	}
}

/******************************************************************************/

static inline uint32_t
get_Cd(void)
{
	switch (get_instruction_reg()) {
	default: DEBUG_ERROR_SWITCH();
	case 0: return get_CR0();
	case 1: ERROR_NYI(); /* FIXME: Reserved */
	case 2: return get_CR2();
	case 3: return get_CR3();
	case 4:
	case 5:
	case 6:
	case 7: ERROR_NYI(); /* FIXME: Reserved */
		return 0;
	}
}

/******************************************************************************/

static inline uint32_t
get_Gb(void)
{
	return get_Rb(get_instruction_reg());
}

static inline uint32_t
get_Gw(void)
{
	return get_Rw(get_instruction_reg());
}

static inline uint32_t
get_Gd(void)
{
	return get_Rd(get_instruction_reg());
}

static inline uint32_t
get_Gv(void)
{
	if (operand_size_32()) {
		return get_Gd();
	} else {
		return get_Gw();
	}
}

/******************************************************************************/

static inline uint32_t
get_Eb(void)
{
	switch (get_instruction_mod()) {
	default: DEBUG_ERROR_SWITCH();
	case 0:
		return mrb_seg(get_address_mod0(),
			get_segment_overwritten(get_segment_mod0()));
	case 1:
		return mrb_seg(get_address_mod1(),
			get_segment_overwritten(get_segment_mod1()));
	case 2:
		return mrb_seg(get_address_mod2(),
			get_segment_overwritten(get_segment_mod2()));
	case 3:
		return get_Rb(get_instruction_rm());
	}
}

static inline uint32_t
get_Ew(void)
{
	switch (get_instruction_mod()) {
	default: DEBUG_ERROR_SWITCH();
	case 0:
		return mrw_seg(get_address_mod0(),
			get_segment_overwritten(get_segment_mod0()));
	case 1:
		return mrw_seg(get_address_mod1(),
			get_segment_overwritten(get_segment_mod1()));
	case 2:
		return mrw_seg(get_address_mod2(),
			get_segment_overwritten(get_segment_mod2()));
	case 3:
		return get_Rw(get_instruction_rm());
	}
}

static inline uint32_t
get_Ed(void)
{
	switch (get_instruction_mod()) {
	default: DEBUG_ERROR_SWITCH();
	case 0:
		return mrd_seg(get_address_mod0(),
			get_segment_overwritten(get_segment_mod0()));
	case 1:
		return mrd_seg(get_address_mod1(),
			get_segment_overwritten(get_segment_mod1()));
	case 2:
		return mrd_seg(get_address_mod2(),
			get_segment_overwritten(get_segment_mod2()));
	case 3:
		return get_Rd(get_instruction_rm());
	}
}

static inline uint32_t
get_Ev(void)
{
	if (operand_size_32()) {
		return get_Ed();
	} else {
		return get_Ew();
	}
}

/******************************************************************************/

static inline uint32_t
get_EwBIT(void)
{
	int32_t offset;

	offset = ((int16_t) (get_Rw(get_instruction_reg()) & 0xfff0)) / 16;

	switch(get_instruction_mod()) {
	default: DEBUG_ERROR_SWITCH();
	case 0:
		return mrw_seg(get_address_mod0() + 2 * offset,
			get_segment_overwritten(get_segment_mod0()));
	case 1:
		return mrw_seg(get_address_mod1() + 2 * offset,
			get_segment_overwritten(get_segment_mod1()));
	case 2:
		return mrw_seg(get_address_mod2() + 2 * offset,
			get_segment_overwritten(get_segment_mod2()));
	case 3:
		return get_Rw(get_instruction_rm());
	}
}

static inline uint32_t
get_EdBIT(void)
{
	int32_t offset;

	offset = ((int32_t) (get_Rd(get_instruction_reg()) & 0xffffffe0)) / 32;

	switch (get_instruction_mod()) {
	default: DEBUG_ERROR_SWITCH();
	case 0:
		return mrd_seg(get_address_mod0() + 4 * offset,
			get_segment_overwritten(get_segment_mod0()));
	case 1:
		return mrd_seg(get_address_mod1() + 4 * offset,
			get_segment_overwritten(get_segment_mod1()));
	case 2:
		return mrd_seg(get_address_mod2() + 4 * offset,
			get_segment_overwritten(get_segment_mod2()));
	case 3:
		return get_Rd(get_instruction_rm());
	}
}

static inline uint32_t
get_EvBIT(void)
{
	if (operand_size_32()) {
		return get_EdBIT();
	} else {
		return get_EwBIT();
	}
}

static inline uint32_t
get_effective_address(void)
{
	switch (get_instruction_mod()) {
	default: DEBUG_ERROR_SWITCH();
	case 0: return get_address_mod0();
	case 1: return get_address_mod1();
	case 2: return get_address_mod2();
	case 3: exception(EXCEPTION_UD, -1);
	}
}

/******************************************************************************/

static inline uint32_t
get_Sw(void)
{
	switch (get_instruction_reg()) {
	default: DEBUG_ERROR_SWITCH();
	case SEGMENT_ES: return get_ES();
	case SEGMENT_CS: return get_CS();
	case SEGMENT_SS: return get_SS();
	case SEGMENT_DS: return get_DS();
	case SEGMENT_FS: return get_FS();
	case SEGMENT_GS: return get_GS();
	case 6: exception(EXCEPTION_UD, -1);
	case 7: exception(EXCEPTION_UD, -1);
	}
}

/******************************************************************************
 *
 * Get Immediates
 *
 ******************************************************************************/

static inline uint32_t
get_Ib(void)
{
	return get_instruction_immediate() & 0xff;
}

static inline uint32_t
get_IbSE(void)
{
	if (operand_size_32()) {
		return (uint32_t) (int8_t) (get_instruction_immediate() & 0xff);
	} else {
		return (uint16_t) (int8_t) (get_instruction_immediate() & 0xff);
	}
}

static inline uint32_t
get_Iw(void)
{
	return get_instruction_immediate() & 0xffff;
}

static inline uint32_t
get_Id(void)
{
	return get_instruction_immediate();
}

static inline uint32_t
get_Iv(void)
{
	if (operand_size_32()) {
		return get_Id();
	} else {
		return get_Iw();
	}
}

/******************************************************************************/

static inline int32_t
get_Jb(void)
{
	return (int8_t) get_instruction_immediate();
}

static inline int32_t
get_Jw(void)
{
	return (int16_t) get_instruction_immediate();
}

static inline int32_t
get_Jd(void)
{
	return (int32_t) get_instruction_immediate();
}

static inline int32_t
get_Jv(void)
{
	if (operand_size_32()) {
		return get_Jd();
	} else {
		return get_Jw();
	}
}

/******************************************************************************/

static inline uint32_t
get_Ob(void)
{
	if (address_size_32()) {
		return mrb_seg(get_Id(), get_segment_overwritten(SEGMENT_DS));
	} else {
		return mrb_seg(get_Iw(), get_segment_overwritten(SEGMENT_DS));
	}
}

static inline uint32_t
get_Ow(void)
{
	if (address_size_32()) {
		return mrw_seg(get_Id(), get_segment_overwritten(SEGMENT_DS));
	} else {
		return mrw_seg(get_Iw(), get_segment_overwritten(SEGMENT_DS));
	}
}

static inline uint32_t
get_Od(void)
{
	if (address_size_32()) {
		return mrd_seg(get_Id(), get_segment_overwritten(SEGMENT_DS));
	} else {
		return mrd_seg(get_Iw(), get_segment_overwritten(SEGMENT_DS));
	}
}

static inline uint32_t
get_Ov(void)
{
	if (operand_size_32()) {
		return get_Od();
	} else {
		return get_Ow();
	}
}

/******************************************************************************
 *
 * Get Memory addressed by the DS:eSI register pair
 *
 ******************************************************************************/

static inline uint32_t
get_Xb(void)
{
	if (address_size_32()) {
		return mrb_seg(get_ESI(), get_segment_overwritten(SEGMENT_DS));
	} else {
		return mrb_seg(get_SI(), get_segment_overwritten(SEGMENT_DS));
	}
}

static inline uint32_t
get_Xw(void)
{
	if (address_size_32()) {
		return mrw_seg(get_ESI(), get_segment_overwritten(SEGMENT_DS));
	} else {
		return mrw_seg(get_SI(), get_segment_overwritten(SEGMENT_DS));
	}
}

static inline uint32_t
get_Xd(void)
{
	if (address_size_32()) {
		return mrd_seg(get_ESI(), get_segment_overwritten(SEGMENT_DS));
	} else {
		return mrd_seg(get_SI(), get_segment_overwritten(SEGMENT_DS));
	}
}

/******************************************************************************
 *
 * Get Memory addressed by the ES:eDI register pair
 *
 ******************************************************************************/

static inline uint32_t
get_Yb(void)
{
	if (address_size_32()) {
		return mrb_seg(get_EDI(), SEGMENT_ES);
	} else {
		return mrb_seg(get_DI(), SEGMENT_ES);
	}
}

static inline uint32_t
get_Yw(void)
{
	if (address_size_32()) {
		return mrw_seg(get_EDI(), SEGMENT_ES);
	} else {
		return mrw_seg(get_DI(), SEGMENT_ES);
	}
}

static inline uint32_t
get_Yd(void)
{
	if (address_size_32()) {
		return mrd_seg(get_EDI(), SEGMENT_ES);
	} else {
		return mrd_seg(get_DI(), SEGMENT_ES);
	}
}

/******************************************************************************
 *
 * Load
 *
 ******************************************************************************/

#define DEBUG_OPERAND_B(str)						\
	DEBUG(" %s:0x%02x", str, (uint8_t) t1)
#define DEBUG_OPERAND_W(str)						\
	DEBUG(" %s:0x%04x", str, (uint16_t) t1)
#define DEBUG_OPERAND_D(str)						\
	DEBUG(" %s:0x%08x", str, (uint32_t) t1)

#define DEBUG_OPERAND_V(str)						\
	if (operand_size_32()) {					\
		DEBUG_OPERAND_D(str);					\
	} else {							\
		DEBUG_OPERAND_W(str);					\
	}

#define DEBUG_OPERAND_BB(str)						\
	DEBUG(" %s:0x%02x,0x%02x", str, (uint8_t) t1, (uint8_t) t2)
#define DEBUG_OPERAND_WW(str)						\
	DEBUG(" %s:0x%04x,0x%04x", str, (uint16_t) t1, (uint16_t) t2)
#define DEBUG_OPERAND_DD(str)						\
	DEBUG(" %s:0x%08x,0x%08x", str, (uint32_t) t1 ,(uint32_t) t2)

#define DEBUG_OPERAND_VV(str)						\
	if (operand_size_32()) {					\
		DEBUG_OPERAND_DD(str);					\
	} else {							\
		DEBUG_OPERAND_WW(str);					\
	}

#define DEBUG_OPERAND_WB(str)						\
	DEBUG(" %s:0x%04x,0x%02x", str, (uint16_t) t1, (uint8_t) t2)
#define DEBUG_OPERAND_WD(str)						\
	DEBUG(" %s:0x%04x,0x%08x", str, (uint16_t) t1, (uint32_t) t2)

#define DEBUG_OPERAND_WV(str)						\
	if (operand_size_32()) {					\
		DEBUG_OPERAND_WD(str);					\
	} else {							\
		DEBUG_OPERAND_WW(str);					\
	}

#define DEBUG_OPERAND_DB(str)						\
	DEBUG(" %s:0x%08x,0x%02x", str, (uint32_t) t1, (uint8_t) t2)
#define DEBUG_OPERAND_WB(str)						\
	DEBUG(" %s:0x%04x,0x%02x", str, (uint16_t) t1, (uint8_t) t2)

#define DEBUG_OPERAND_VB(str)						\
	if (operand_size_32()) {					\
		DEBUG_OPERAND_DB(str);					\
	} else {							\
		DEBUG_OPERAND_WB(str);					\
	}

#define DEBUG_OPERAND_BD(str)						\
	DEBUG(" %s:0x%02x,0x%08x", str, (uint8_t) t1, (uint32_t) t2)
#define DEBUG_OPERAND_BW(str)						\
	DEBUG(" %s:0x%02x,0x%04x", str, (uint16_t) t1, (uint32_t) t2)

#define DEBUG_OPERAND_BV(str)						\
	if (operand_size_32()) {					\
		DEBUG_OPERAND_BD(str);					\
	} else {							\
		DEBUG_OPERAND_BW(str);					\
	}

static inline void
load_CS(void)
{
	t1 = get_CS();
	DEBUG_OPERAND_W("CS");
}

static inline void
load_SS(void)
{
	t1 = get_SS();
	DEBUG_OPERAND_W("SS");
}

static inline void
load_DS(void)
{
	t1 = get_DS();
	DEBUG_OPERAND_W("DS");
}

static inline void
load_ES(void)
{
	t1 = get_ES();
	DEBUG_OPERAND_W("ES");
}

static inline void
load_FS(void)
{
	t1 = get_FS();
	DEBUG_OPERAND_W("FS");
}

static inline void
load_GS(void)
{
	t1 = get_GS();
	DEBUG_OPERAND_W("GS");
}

/******************************************************************************/

static inline void
load_AL(void)
{
	t1 = get_AL();
	DEBUG_OPERAND_B("AL");
}

/******************************************************************************/

static inline void
load_eAX(void)
{
	t1 = get_eAX();
	DEBUG_OPERAND_V("eAX");
}

static inline void
load_eCX(void)
{
	t1 = get_eCX();
	DEBUG_OPERAND_V("eCX");
}

static inline void
load_eDX(void)
{
	t1 = get_eDX();
	DEBUG_OPERAND_V("eDX");
}

static inline void
load_eBX(void)
{
	t1 = get_eBX();
	DEBUG_OPERAND_V("eBX");
}

/******************************************************************************/

static inline void
load_eSP(void)
{
	t1 = get_eSP();
	DEBUG_OPERAND_V("eSP");
}

static inline void
load_eBP(void)
{
	t1 = get_eBP();
	DEBUG_OPERAND_V("eBP");
}

static inline void
load_eSI(void)
{
	t1 = get_eSI();
	DEBUG_OPERAND_V("eSI");
}

static inline void
load_eDI(void)
{
	t1 = get_eDI();
	DEBUG_OPERAND_V("eDI");
}

/******************************************************************************/

static inline void
load_Cd(void)
{
	t1 = get_Cd();
	DEBUG_OPERAND_V("Cd");
}

static inline void
load_Rd(void)
{
	t1 = get_Rd(get_instruction_rm());
	DEBUG_OPERAND_V("Rd");
}

/******************************************************************************/

static inline void
load_0(void)
{
	t1 = 0;
}

static inline void
load_Ib(void)
{
	t1 = get_Ib();
	DEBUG_OPERAND_B("Ib");
}

static inline void
load_IbSE(void)
{
	t1 = get_IbSE();
	DEBUG_OPERAND_B("IbSE");
}

static inline void
load_Iw(void)
{
	t1 = get_Iw();
	DEBUG_OPERAND_W("Iw");
}

static inline void
load_Iv(void)
{
	t1 = get_Iv();
	DEBUG_OPERAND_V("Iv");
}

static inline void
load_AL_Ib(void)
{
	t1 = get_AL();
	DEBUG_GET(",");
	t2 = get_Ib();
	DEBUG_OPERAND_BB("AL,Ib");
}

static inline void
load_eAX_Iv(void)
{
	t1 = get_eAX();
	DEBUG_GET(",");
	t2 = get_Iv();
	DEBUG_OPERAND_VV("eAX,Iv");
}

static inline void
load_Ib_AL(void)
{
	t1 = get_Ib();
	DEBUG_GET(",");
	t2 = get_AL();
	DEBUG_OPERAND_BB("Ib,AL");
}

static inline void
load_Ib_eAX(void)
{
	t1 = get_Ib();
	DEBUG_GET(",");
	t2 = get_eAX();
	DEBUG_OPERAND_BV("Ib,eAX");
}

/******************************************************************************/

static inline void
load_eAX_eCX(void)
{
	t1 = get_eAX();
	DEBUG_GET(",");
	t2 = get_eCX();
	DEBUG_OPERAND_VV("eAX,eCX");
}

static inline void
load_eAX_eDX(void)
{
	t1 = get_eAX();
	DEBUG_GET(",");
	t2 = get_eDX();
	DEBUG_OPERAND_VV("eAX,eDX");
}

static inline void
load_eAX_eBX(void)
{
	t1 = get_eAX();
	DEBUG_GET(",");
	t2 = get_eBX();
	DEBUG_OPERAND_VV("eAX,eBX");
}

static inline void
load_eAX_eSP(void)
{
	t1 = get_eAX();
	DEBUG_GET(",");
	t2 = get_eSP();
	DEBUG_OPERAND_VV("eAX,eSP");
}

static inline void
load_eAX_eBP(void)
{
	t1 = get_eAX();
	DEBUG_GET(",");
	t2 = get_eBP();
	DEBUG_OPERAND_VV("eAX,eBP");
}

static inline void
load_eAX_eSI(void)
{
	t1 = get_eAX();
	DEBUG_GET(",");
	t2 = get_eSI();
	DEBUG_OPERAND_VV("eAX,eSI");
}

static inline void
load_eAX_eDI(void)
{
	t1 = get_eAX();
	DEBUG_GET(",");
	t2 = get_eDI();
	DEBUG_OPERAND_VV("eAX,eDI");
}

/******************************************************************************/

static inline void
load_DX(void)
{
	t1 = get_DX();
	DEBUG_OPERAND_W("DX");
}

static inline void
load_DX_AL(void)
{
	t1 = get_DX();
	DEBUG_GET(",");
	t2 = get_AL();
	DEBUG_OPERAND_WB("DX,AL");
}

static inline void
load_DX_eAX(void)
{
	t1 = get_DX();
	DEBUG_GET(",");
	t2 = get_eAX();
	DEBUG_OPERAND_WV("DX,eAX");
}

/******************************************************************************/

static inline void
load_Eb_Gb(void)
{
	t1 = get_Eb();
	DEBUG_GET(",");
	t2 = get_Gb();
	DEBUG_OPERAND_BB("Eb,Gb");
}

static inline void
load_Ev_Gv(void)
{
	t1 = get_Ev();
	DEBUG_GET(",");
	t2 = get_Gv();
	DEBUG_OPERAND_VV("Ev,Gv");
}

static inline void
load_Ev_Gv_Ib(void)
{
	t1 = get_Ev();
	DEBUG_GET(",");
	t2 = get_Gv();
	DEBUG_GET(",");
	t3 = get_Ib();
}

static inline void
load_Ev_Gv_CL(void)
{
	t1 = get_Ev();
	DEBUG_GET(",");
	t2 = get_Gv();
	DEBUG_GET(",");
	t3 = get_CL();
}

static inline void
load_EvBIT_Gv(void)
{
	t1 = get_EvBIT();
	DEBUG_GET(",");
	t2 = get_Gv();
	DEBUG_OPERAND_VV("EvBIT,Gv");
}

static inline void
load_Eb_Ib(void)
{
	t1 = get_Eb();
	DEBUG_GET(",");
	t2 = get_Ib();
	DEBUG_OPERAND_BB("Eb,Ib");
}

static inline void
load_Eb_1(void)
{
	t1 = get_Eb();
	DEBUG_GET(",");
	t2 = 1;
	DEBUG_OPERAND_BB("Eb,1");
}

static inline void
load_Ev_1(void)
{
	t1 = get_Ev();
	DEBUG_GET(",");
	t2 = 1;
	DEBUG_OPERAND_VB("Ev,1");
}

static inline void
load_Eb_CL(void)
{
	t1 = get_Eb();
	DEBUG_GET(",");
	t2 = get_CL();
	DEBUG_OPERAND_BB("Eb,CL");
}

static inline void
load_Ev_CL(void)
{
	t1 = get_Ev();
	DEBUG_GET(",");
	t2 = get_CL();
	DEBUG_OPERAND_VB("Ev,CL");
}

/******************************************************************************/

static inline void
load_Ev_Iv(void)
{
	t1 = get_Ev();
	DEBUG_GET(",");
	t2 = get_Iv();
	DEBUG_OPERAND_VV("Ev,Iv");
}

static inline void
load_Ev_Ib(void)
{
	t1 = get_Ev();
	DEBUG_GET(",");
	t2 = get_Ib();
	DEBUG_OPERAND_VB("Ev,Ib");
}

static inline void
load_Ev_IbSE(void)
{
	t1 = get_Ev();
	DEBUG_GET(",");
	t2 = get_IbSE();
	DEBUG_OPERAND_VV("Ev,IbSE");
}

static inline void
load_Eb(void)
{
	t1 = get_Eb();
	DEBUG_OPERAND_B("Eb");
}

static inline void
load_Ew(void)
{
	t1 = get_Ew();
	DEBUG_OPERAND_W("Ew");
}

static inline void
load_Ev(void)
{
	t1 = get_Ev();
	DEBUG_OPERAND_V("Ev");
}

static inline void
load_Gb(void)
{
	t1 = get_Gb();
	DEBUG_OPERAND_B("Gb");
}

static inline void
load_Gv(void)
{
	t1 = get_Gv();
	DEBUG_OPERAND_V("Gv");
}

/******************************************************************************/

static inline void
load_Gb_Eb(void)
{
	t1 = get_Gb();
	DEBUG_GET(",");
	t2 = get_Eb();
	DEBUG_OPERAND_BB("Gb,Eb");
}

static inline void
load_Gv_Ev(void)
{
	t1 = get_Gv();
	DEBUG_GET(",");
	t2 = get_Ev();
	DEBUG_OPERAND_VV("Gv,Ev");
}

/******************************************************************************/

static inline void
load_Sw(void)
{
	t1 = get_Sw();
	DEBUG_OPERAND_W("Sw");
}

static inline void
load_Ob(void)
{
	t1 = get_Ob();
	DEBUG_OPERAND_B("Ob");
}

static inline void
load_Ov(void)
{
	t1 = get_Ov();
	DEBUG_OPERAND_V("Ov");
}

static inline void
load_Ap(void)
{
	t1 = get_instruction_displacement();
	t2 = get_instruction_immediate();
	DEBUG_OPERAND_WV("Ap");
}

/******************************************************************************/

static inline void
load_Jb(void)
{
	tj = get_Jb();
	/* TODO: debug */
}

static inline void
load_Jv(void)
{
	tj = get_Jv();
	/* TODO: debug */
}

/******************************************************************************/

static inline void
load_effective_address(void)
{
	t1 = get_effective_address();
	/* TODO: debug */
}

static inline void
load_Ms(void)
{
	uint32_t addr;
	int segment;

	switch (get_instruction_mod()) {
	default: DEBUG_ERROR_SWITCH();
	case 0:
		addr    = get_address_mod0();
		segment = get_segment_mod0();
		break;
	case 1:
		addr    = get_address_mod1();
		segment = get_segment_mod1();
		break;
	case 2:
		addr    = get_address_mod2();
		segment = get_segment_mod2();
		break;
	case 3:
		exception(EXCEPTION_UD, -1);
	}

	segment = get_segment_overwritten(segment);

	t1 = mrw_seg(addr + 0, segment); /* limit */
	t2 = mrd_seg(addr + 2, segment); /* base */
}

static inline void
load_Mp(void)
{
	uint32_t addr;
	int segment;

	switch (get_instruction_mod()) {
	default: DEBUG_ERROR_SWITCH();
	case 0:
		addr    = get_address_mod0();
		segment = get_segment_mod0();
		break;
	case 1:
		addr    = get_address_mod1();
		segment = get_segment_mod1();
		break;
	case 2:
		addr    = get_address_mod2();
		segment = get_segment_mod2();
		break;
	case 3:
		exception(EXCEPTION_UD, -1);
	}

	segment = get_segment_overwritten(segment);

	if (operand_size_32()) {
		t1 = mrd_seg(addr + 0, segment); /* register */
		t2 = mrw_seg(addr + 4, segment); /* segment selector */
	} else {
		t1 = mrw_seg(addr + 0, segment); /* register */
		t2 = mrw_seg(addr + 2, segment); /* segment selector */
	}
}

static inline void
load_Ep(void)
{
	load_Mp();
}

/******************************************************************************
 *
 * Load Memory addressed by the ES:eDI and DS:eSI register pair
 *
 ******************************************************************************/

static inline void
load_Xb(void)
{
	t1 = get_Xb();
}

static inline void
load_Xw(void)
{
	t1 = get_Xw();
}

static inline void
load_Xd(void)
{
	t1 = get_Xd();
}

static inline void
load_DX_Xb(void)
{
	t1 = get_DX();
	t2 = get_Xb();
}

static inline void
load_DX_Xw(void)
{
	t1 = get_DX();
	t2 = get_Xw();
}

static inline void
load_DX_Xd(void)
{
	t1 = get_DX();
	t2 = get_Xd();
}

static inline void
load_AL_Yb(void)
{
	t1 = get_AL();
	t2 = get_Yb();
}

static inline void
load_AX_Yw(void)
{
	t1 = get_AX();
	t2 = get_Yw();
}

static inline void
load_EAX_Yd(void)
{
	t1 = get_EAX();
	t2 = get_Yd();
}

static inline void
load_Xb_Yb(void)
{
	t1 = get_Xb();
	t2 = get_Yb();
}

static inline void
load_Xw_Yw(void)
{
	t1 = get_Xw();
	t2 = get_Yw();
}

static inline void
load_Xd_Yd(void)
{
	t1 = get_Xd();
	t2 = get_Yd();
}

/******************************************************************************
 *
 * SET Segment Selector and Descriptor
 *
 ******************************************************************************/

static inline uint32_t
get_descriptor_address(void)
{
	if (selector_ti == 0) { /* GDT */
		if ((selector_index * 8 + 7) > gdtr_limit) {
			exception(EXCEPTION_GP, selector & 0xfffc);
		}
		return gdtr_base + selector_index * 8;
	} else { /* LDT */
		if ((selector_index * 8 + 7) > ldtr_system_limit) {
			exception(EXCEPTION_GP, selector & 0xfffc);
		}
		return ldtr_system_base + selector_index * 8;
	}
}

static inline void
sset_selector(uint16_t value)
{
	selector = value;
	selector_index = value >> 3;
	selector_ti = (value >> 2) & 1;
	selector_rpl = value & 3;
}

static inline void
sset_descriptor(uint32_t dw1, uint32_t dw2)
{
	bool gflag;

	descriptor_type  = (dw2 >>  8) & 0xf;
	descriptor_sflag = (dw2 >> 12) & 0x1;
	descriptor_dpl   = (dw2 >> 13) & 0x3;
	descriptor_pflag = (dw2 >> 15) & 0x1;

	if (descriptor_sflag) { /* code or data segment descriptor */
		gflag   = (dw2 & 0x00800000) > 0;
		descriptor_segment_limit   = (dw1 & 0x0000ffff)
					   | (dw2 & 0x000f0000);
		descriptor_segment_limit = limit_scaled(
				descriptor_segment_limit, gflag);
		descriptor_segment_base    = (dw1 >> 16)
					   | ((dw2 & 0xff) << 16)
					   | (dw2 & 0xff000000);
		descriptor_segment_dflag   = (dw2 & 0x00400000) > 0;
		descriptor_segment_avlflag = (dw2 & 0x00100000) > 0;
	} else { /* system segment descriptor */
		switch (descriptor_type) {
		default:
		case 0x0:
		case 0x8:
		case 0xa:
		case 0xd:
			/* Reserved */
			break;
		case SEGMENT_16BIT_CALL_GATE:
			descriptor_gate_paramcount = dw2 & 0x1f;
			descriptor_gate_selector = dw1 >> 16;
			descriptor_gate_offset = dw1 & 0xffff;
			break;
		case SEGMENT_16BIT_INTERRUPT_GATE:
		case SEGMENT_16BIT_TRAP_GATE:
			descriptor_gate_selector = dw1 >> 16;
			descriptor_gate_offset = dw1 & 0xffff;
			break;
		case SEGMENT_32BIT_CALL_GATE:
			descriptor_gate_paramcount = dw2 & 0x1f;
			descriptor_gate_selector = dw1 >> 16;
			descriptor_gate_offset = (dw2 & 0xffff0000)
					       | (dw1 & 0x0000ffff);
			break;
		case SEGMENT_32BIT_INTERRUPT_GATE:
		case SEGMENT_32BIT_TRAP_GATE:
			descriptor_gate_selector = dw1 >> 16;
			descriptor_gate_offset = (dw2 & 0xffff0000)
					       | (dw1 & 0x0000ffff);
			break;
		case SEGMENT_TASK_GATE:
			descriptor_gate_selector = dw1 >> 16;
			break;
		case SEGMENT_16BIT_AVAIL_TSS:
		case SEGMENT_16BIT_BUSY_TSS:
			descriptor_system_base = (dw1 >> 16)
					       | ((dw2 & 0xff) << 16);
			descriptor_system_limit = (dw1 & 0xffff);
			break;
		case SEGMENT_LDT:
		case SEGMENT_32BIT_AVAIL_TSS:
		case SEGMENT_32BIT_BUSY_TSS:
			gflag = (dw2 & 0x00800000) > 0;
			descriptor_system_base = (dw1 >> 16)
					       | ((dw2 & 0xff) << 16)
					       | (dw2 & 0xff000000);
			descriptor_system_limit = (dw1 & 0x0000ffff)
						| (dw2 & 0x000f0000);
			descriptor_system_limit = limit_scaled(
					descriptor_system_limit, gflag);
			break;
		}
	}
}

static inline void
sset_CS(uint16_t value)
{
	DEBUG_SET("{CS:0x%04x}", value);

	if (protected_mode()) {
		uint32_t addr;

		sset_selector(value);

		if (selector_index == 0) {
			exception(EXCEPTION_GP, 0);
		}

		addr = get_descriptor_address();

		sset_descriptor(mrd(addr, get_CPL()), mrd(addr + 4, get_CPL()));

		if (descriptor_sflag) { /* code or data segment descriptor */
			if (!((descriptor_type >> 3) & 1)) { /* data segment */
				exception(EXCEPTION_GP, value & 0xfffc);
			}

			if ((descriptor_type >> 2) & 1) { /* conforming */
				if (descriptor_dpl > get_CPL()) {
					exception(EXCEPTION_GP,
						value & 0xfffc);
				}
			} else { /* non-conforming */
				if (descriptor_dpl != get_CPL()) {
					exception(EXCEPTION_GP,
						value & 0xfffc);
				}
				if ((selector_rpl) > get_CPL()) {
					exception(EXCEPTION_GP,
						value & 0xfffc);
				}
			}

			if (!descriptor_pflag) {
				exception(EXCEPTION_NP, value & 0xfffc);
			}
		} else { /* system segment descriptor */
			if ((descriptor_type == 0x0)
			 || (descriptor_type == 0x8)
			 || (descriptor_type == 0xa)
			 || (descriptor_type == 0xd)) {
				/* Reserved */
				exception(EXCEPTION_GP, value & 0xfffc);
			}

			if ((descriptor_dpl < get_CPL())
			 || (descriptor_dpl < selector_rpl)) {
				exception(EXCEPTION_GP, value & 0xfffc);
			}

			if (!descriptor_pflag) {
				exception(EXCEPTION_NP, value & 0xfffc);
			}
		}
	} else {
		selector = value;
		descriptor_type = SEGMENT_CODE_EXEC_READ_ACCESSED;
		descriptor_sflag = 1;
		descriptor_pflag = 1;
		descriptor_segment_base = value << 4;

		if (real_mode()) {
			descriptor_dpl = cs_dpl;
			descriptor_segment_limit = cs_segment_limit;
			descriptor_segment_dflag = cs_segment_dflag;
			descriptor_segment_avlflag = cs_segment_avlflag;
		} else { /* virtual8086_mode */
			descriptor_dpl = 3;
			descriptor_segment_limit = 0xffff;
			descriptor_segment_dflag = 0;
			descriptor_segment_avlflag = 0;
		}
	}
}

static inline void
sset_SS(uint16_t value)
{
	DEBUG_SET("{SS:0x%04x}", value);

	if (protected_mode()) {
		uint32_t addr;

		sset_selector(value);

		if (selector_index == 0) {
			exception(EXCEPTION_GP, 0);
		}

		addr = get_descriptor_address();

		if (selector_rpl != get_CPL()) {
			exception(EXCEPTION_GP, value & 0xfffc);
		}

		sset_descriptor(mrd(addr, get_CPL()), mrd(addr + 4, get_CPL()));

		if ((!descriptor_sflag) /* no data/code segment */
		 || ((descriptor_type >> 3) & 1) /* no data segment */
		 || (!((descriptor_type >> 1) & 1)) /* not writeable */
		 || (descriptor_dpl != get_CPL())) {
			exception(EXCEPTION_GP, value & 0xfffc);
		}

		if (!descriptor_pflag) { /* segment not present */
			exception(EXCEPTION_SS, value & 0xfffc);
		}
	} else {
		selector = value;
		descriptor_type = SEGMENT_DATA_READ_WRITE_ACCESSED;
		descriptor_sflag = 1;
		descriptor_pflag = 1;
		descriptor_segment_base = value << 4;

		if (real_mode()) {
			descriptor_dpl = ss_dpl;
			descriptor_segment_limit = ss_segment_limit;
			descriptor_segment_dflag = ss_segment_dflag;
			descriptor_segment_avlflag = ss_segment_avlflag;
		} else { /* virtual8086_mode */
			descriptor_dpl = 3;
			descriptor_segment_limit = 0xffff;
			descriptor_segment_dflag = 0;
			descriptor_segment_avlflag = 0;
		}
	}
}

static inline void
sset_DS(uint16_t value)
{
	DEBUG_SET("{DS:0x%04x}", value);

	if (protected_mode()) {
		uint32_t addr;

		sset_selector(value);

		if (selector_index == 0) {
			sset_selector(0);
			return;
		}

		addr = get_descriptor_address();

		sset_descriptor(mrd(addr, get_CPL()), mrd(addr + 4, get_CPL()));

		if ((!descriptor_sflag) /* no data/code segment */
		 || (((descriptor_type >> 3) & 1) /* code segment */
		  && (!((descriptor_type >> 1) & 1)))) { /* not readable */
			exception(EXCEPTION_GP, value & 0xfffc);
		}

		/* if data or non-conforming code, then rpl and cpl	*/
		/* must be less than or equal to dpl			*/
		if ((!((descriptor_type >> 1) & 1)) /* data segemnt */
		 || (!((descriptor_type >> 2) & 1))) { /* non-conformaing */
			if (((selector_rpl) > descriptor_dpl)
			 || (get_CPL() > descriptor_dpl)) {
				exception(EXCEPTION_GP, value & 0xfffc);
			}
		}

		if (!descriptor_pflag) { /* segment not present */
			exception(EXCEPTION_NP, value & 0xfffc);
		}
	} else {
		selector = value;
		descriptor_type = SEGMENT_DATA_READ_WRITE_ACCESSED;
		descriptor_sflag = 1;
		descriptor_pflag = 1;
		descriptor_segment_base = value << 4;

		if (real_mode()) {
			descriptor_dpl = ds_dpl;
			descriptor_segment_limit = ds_segment_limit;
			descriptor_segment_dflag = ds_segment_dflag;
			descriptor_segment_avlflag = ds_segment_avlflag;
		} else { /* virtual8086_mode */
			descriptor_dpl = 3;
			descriptor_segment_limit = 0xffff;
			descriptor_segment_dflag = 0;
			descriptor_segment_avlflag = 0;
		}
	}
}

static inline void
sset_ES(uint16_t value)
{
	DEBUG_SET("{ES:0x%04x}", value);

	if (protected_mode()) {
		uint32_t addr;

		sset_selector(value);

		if (selector_index == 0) {
			sset_selector(0);
			return;
		}

		addr = get_descriptor_address();

		sset_descriptor(mrd(addr, get_CPL()), mrd(addr + 4, get_CPL()));

		if ((!descriptor_sflag) /* no data/code segment */
		 || (((descriptor_type >> 3) & 1) /* code segment */
		  && (!((descriptor_type >> 1) & 1)))) { /* not readable */
			exception(EXCEPTION_GP, value & 0xfffc);
		}

		/* if data or non-conforming code, then rpl and cpl	*/
		/* must be less than or equal to dpl			*/
		if ((!((descriptor_type >> 1) & 1)) /* data segemnt */
		 || (!((descriptor_type >> 2) & 1))) { /* non-conformaing */
			if (((selector_rpl) > descriptor_dpl)
			 || (get_CPL() > descriptor_dpl)) {
				exception(EXCEPTION_GP, value & 0xfffc);
			}
		}

		if (!descriptor_pflag) { /* segment not present */
			exception(EXCEPTION_NP, value & 0xfffc);
		}
	} else {
		selector = value;
		descriptor_type = SEGMENT_DATA_READ_WRITE_ACCESSED;
		descriptor_sflag = 1;
		descriptor_pflag = 1;
		descriptor_segment_base = value << 4;

		if (real_mode()) {
			descriptor_dpl = es_dpl;
			descriptor_segment_limit = es_segment_limit;
			descriptor_segment_dflag = es_segment_dflag;
			descriptor_segment_avlflag = es_segment_avlflag;
		} else { /* virtual8086_mode */
			descriptor_dpl = 3;
			descriptor_segment_limit = 0xffff;
			descriptor_segment_dflag = 0;
			descriptor_segment_avlflag = 0;
		}
	}
}

static inline void
sset_FS(uint16_t value)
{
	DEBUG_SET("{FS:0x%04x}", value);

	if (protected_mode()) {
		uint32_t addr;

		sset_selector(value);

		if (selector_index == 0) {
			sset_selector(0);
			return;
		}

		addr = get_descriptor_address();

		sset_descriptor(mrd(addr, get_CPL()), mrd(addr + 4, get_CPL()));

		if ((!descriptor_sflag) /* no data/code segment */
		 || (((descriptor_type >> 3) & 1) /* code segment */
		  && (!((descriptor_type >> 1) & 1)))) { /* not readable */
			exception(EXCEPTION_GP, value & 0xfffc);
		}

		/* if data or non-conforming code, then rpl and cpl	*/
		/* must be less than or equal to dpl			*/
		if ((!((descriptor_type >> 1) & 1)) /* data segemnt */
		 || (!((descriptor_type >> 2) & 1))) { /* non-conformaing */
			if (((selector_rpl) > descriptor_dpl)
			 || (get_CPL() > descriptor_dpl)) {
				exception(EXCEPTION_GP, value & 0xfffc);
			}
		}

		if (!descriptor_pflag) { /* segment not present */
			exception(EXCEPTION_NP, value & 0xfffc);
		}
	} else {
		selector = value;
		descriptor_type = SEGMENT_DATA_READ_WRITE_ACCESSED;
		descriptor_sflag = 1;
		descriptor_pflag = 1;
		descriptor_segment_base = value << 4;

		if (real_mode()) {
			descriptor_dpl = fs_dpl;
			descriptor_segment_limit = fs_segment_limit;
			descriptor_segment_dflag = fs_segment_dflag;
			descriptor_segment_avlflag = fs_segment_avlflag;
		} else { /* virtual8086_mode */
			descriptor_dpl = 3;
			descriptor_segment_limit = 0xffff;
			descriptor_segment_dflag = 0;
			descriptor_segment_avlflag = 0;
		}
	}
}

static inline void
sset_GS(uint16_t value)
{
	DEBUG_SET("{GS:0x%04x}", value);

	if (protected_mode()) {
		uint32_t addr;

		sset_selector(value);

		if (selector_index == 0) {
			sset_selector(0);
			return;
		}

		addr = get_descriptor_address();

		sset_descriptor(mrd(addr, get_CPL()), mrd(addr + 4, get_CPL()));

		if ((!descriptor_sflag) /* no data/code segment */
		 || (((descriptor_type >> 3) & 1) /* code segment */
		  && (!((descriptor_type >> 1) & 1)))) { /* not readable */
			exception(EXCEPTION_GP, value & 0xfffc);
		}

		/* if data or non-conforming code, then rpl and cpl	*/
		/* must be less than or equal to dpl			*/
		if ((!((descriptor_type >> 1) & 1)) /* data segemnt */
		 || (!((descriptor_type >> 2) & 1))) { /* non-conformaing */
			if (((selector_rpl) > descriptor_dpl)
			 || (get_CPL() > descriptor_dpl)) {
				exception(EXCEPTION_GP, value & 0xfffc);
			}
		}

		if (!descriptor_pflag) { /* segment not present */
			exception(EXCEPTION_NP, value & 0xfffc);
		}
	} else {
		selector = value;
		descriptor_type = SEGMENT_DATA_READ_WRITE_ACCESSED;
		descriptor_sflag = 1;
		descriptor_pflag = 1;
		descriptor_segment_base = value << 4;

		if (real_mode()) {
			descriptor_dpl = gs_dpl;
			descriptor_segment_limit = gs_segment_limit;
			descriptor_segment_dflag = gs_segment_dflag;
			descriptor_segment_avlflag = gs_segment_avlflag;
		} else { /* virtual8086_mode */
			descriptor_dpl = 3;
			descriptor_segment_limit = 0xffff;
			descriptor_segment_dflag = 0;
			descriptor_segment_avlflag = 0;
		}
	}
}

/******************************************************************************/

static inline void
commit_CS(void)
{
	cs_selector		= selector;
	cs_type			= descriptor_type;
	cs_sflag		= descriptor_sflag;
	cs_dpl			= descriptor_dpl;
	cs_pflag		= descriptor_pflag;
	cs_segment_limit	= descriptor_segment_limit;
	cs_segment_base		= descriptor_segment_base;
	cs_segment_dflag	= descriptor_segment_dflag;
	cs_segment_avlflag	= descriptor_segment_avlflag;
}

static inline void
commit_SS(void)
{
	ss_selector		= selector;
	ss_type			= descriptor_type;
	ss_sflag		= descriptor_sflag;
	ss_dpl			= descriptor_dpl;
	ss_pflag		= descriptor_pflag;
	ss_segment_limit	= descriptor_segment_limit;
	ss_segment_base		= descriptor_segment_base;
	ss_segment_dflag	= descriptor_segment_dflag;
	ss_segment_avlflag	= descriptor_segment_avlflag;
}

static inline void
commit_DS(void)
{
	ds_selector		= selector;
	ds_type			= descriptor_type;
	ds_sflag		= descriptor_sflag;
	ds_dpl			= descriptor_dpl;
	ds_pflag		= descriptor_pflag;
	ds_segment_limit	= descriptor_segment_limit;
	ds_segment_base		= descriptor_segment_base;
	ds_segment_dflag	= descriptor_segment_dflag;
	ds_segment_avlflag	= descriptor_segment_avlflag;
}

static inline void
commit_ES(void)
{
	es_selector		= selector;
	es_type			= descriptor_type;
	es_sflag		= descriptor_sflag;
	es_dpl			= descriptor_dpl;
	es_pflag		= descriptor_pflag;
	es_segment_limit	= descriptor_segment_limit;
	es_segment_base		= descriptor_segment_base;
	es_segment_dflag	= descriptor_segment_dflag;
	es_segment_avlflag	= descriptor_segment_avlflag;
}

static inline void
commit_FS(void)
{
	fs_selector		= selector;
	fs_type			= descriptor_type;
	fs_sflag		= descriptor_sflag;
	fs_dpl			= descriptor_dpl;
	fs_pflag		= descriptor_pflag;
	fs_segment_limit	= descriptor_segment_limit;
	fs_segment_base		= descriptor_segment_base;
	fs_segment_dflag	= descriptor_segment_dflag;
	fs_segment_avlflag	= descriptor_segment_avlflag;
}

static inline void
commit_GS(void)
{
	gs_selector		= selector;
	gs_type			= descriptor_type;
	gs_sflag		= descriptor_sflag;
	gs_dpl			= descriptor_dpl;
	gs_pflag		= descriptor_pflag;
	gs_segment_limit	= descriptor_segment_limit;
	gs_segment_base		= descriptor_segment_base;
	gs_segment_dflag	= descriptor_segment_dflag;
	gs_segment_avlflag	= descriptor_segment_avlflag;
}

static inline void
set_SS(uint16_t value)
{
	sset_SS(value);
	commit_SS();
}

static inline void
set_DS(uint16_t value)
{
	sset_DS(value);
	commit_DS();
}

static inline void
set_ES(uint16_t value)
{
	sset_ES(value);
	commit_ES();
}

static inline void
set_FS(uint16_t value)
{
	sset_FS(value);
	commit_FS();
}

static inline void
set_GS(uint16_t value)
{
	sset_GS(value);
	commit_GS();
}

/******************************************************************************
 *
 * SET
 *
 ******************************************************************************/

static inline void
set_AL(uint32_t value)
{
	DEBUG_SET("{AL:0x%02x}", value & 0xff);
	eax = (eax & ~0xff) | (value & 0xff);
}

static inline void
set_CL(uint32_t value)
{
	DEBUG_SET("{CL:0x%02x}", value & 0xff);
	ecx = (ecx & ~0xff) | (value & 0xff);
}

static inline void
set_DL(uint32_t value)
{
	DEBUG_SET("{DL:0x%02x}", value & 0xff);
	edx = (edx & ~0xff) | (value & 0xff);
}

static inline void
set_BL(uint32_t value)
{
	DEBUG_SET("{BL:0x%02x}", value & 0xff);
	ebx = (ebx & ~0xff) | (value & 0xff);
}

/******************************************************************************/

static inline void
set_AH(uint32_t value)
{
	DEBUG_SET("{AH:0x%02x}", value & 0xff);
	eax = (eax & 0xffff00ff) | ((value & 0xff) << 8);
}

static inline void
set_CH(uint32_t value)
{
	DEBUG_SET("{CH:0x%02x}", value & 0xff);
	ecx = (ecx & 0xffff00ff) | ((value & 0xff) << 8);
}

static inline void
set_DH(uint32_t value)
{
	DEBUG_SET("{DH:0x%02x}", value & 0xff);
	edx = (edx & 0xffff00ff) | ((value & 0xff) << 8);
}

static inline void
set_BH(uint32_t value)
{
	DEBUG_SET("{BH:0x%02x}", value & 0xff);
	ebx = (ebx & 0xffff00ff) | ((value & 0xff) << 8);
}

/******************************************************************************/

static inline void
set_AX(uint32_t value)
{
	DEBUG_SET("{AX:0x%04x}", value & 0xffff);
	eax = (eax & ~0xffff) | (value & 0xffff);
}

static inline void
set_CX(uint32_t value)
{
	DEBUG_SET("{CX:0x%04x}", value & 0xffff);
	ecx = (ecx & ~0xffff) | (value & 0xffff);
}

static inline void
set_DX(uint32_t value)
{
	DEBUG_SET("{DX:0x%04x}", value & 0xffff);
	edx = (edx & ~0xffff) | (value & 0xffff);
}

static inline void
set_BX(uint32_t value)
{
	DEBUG_SET("{BX:0x%04x}", value & 0xffff);
	ebx = (ebx & ~0xffff) | (value & 0xffff);
}

/******************************************************************************/

static inline void
set_SP(uint32_t value)
{
	DEBUG_SET("{SP:0x%04x}", value & 0xffff);
	esp = (esp & ~0xffff) | (value & 0xffff);
}

static inline void
set_BP(uint32_t value)
{
	DEBUG_SET("{BP:0x%04x}", value & 0xffff);
	ebp = (ebp & ~0xffff) | (value & 0xffff);
}

static inline void
set_SI(uint32_t value)
{
	DEBUG_SET("{SI:0x%04x}", value & 0xffff);
	esi = (esi & ~0xffff) | (value & 0xffff);
}

static inline void
set_DI(uint32_t value)
{
	DEBUG_SET("{DI:0x%04x}", value & 0xffff);
	edi = (edi & ~0xffff) | (value & 0xffff);
}

/******************************************************************************/

static inline void
set_EAX(uint32_t value)
{
	DEBUG_SET("{EAX:0x%08x}", value);
	eax = value;
}

static inline void
set_ECX(uint32_t value)
{
	DEBUG_SET("{ECX:0x%08x}", value);
	ecx = value;
}

static inline void
set_EDX(uint32_t value)
{
	DEBUG_SET("{EDX:0x%08x}", value);
	edx = value;
}

static inline void
set_EBX(uint32_t value)
{
	DEBUG_SET("{EBX:0x%08x}", value);
	ebx = value;
}

/******************************************************************************/

static inline void
set_ESP(uint32_t value)
{
	DEBUG_SET("{ESP:0x%08x}", value);
	esp = value;
}

static inline void
set_EBP(uint32_t value)
{
	DEBUG_SET("{EBP:0x%08x}", value);
	ebp = value;
}

static inline void
set_ESI(uint32_t value)
{
	DEBUG_SET("{ESI:0x%08x}", value);
	esi = value;
}

static inline void
set_EDI(uint32_t value)
{
	DEBUG_SET("{EDI:0x%08x}", value);
	edi = value;
}

/******************************************************************************/

static inline void
set_eAX(uint32_t value)
{
	if (operand_size_32()) {
		set_EAX(value);
	} else {
		set_AX(value);
	}
}

static inline void
set_eCX(uint32_t value)
{
	if (operand_size_32()) {
		set_ECX(value);
	} else {
		set_CX(value);
	}
}

static inline void
set_eDX(uint32_t value)
{
	if (operand_size_32()) {
		set_EDX(value);
	} else {
		set_DX(value);
	}
}

static inline void
set_eBX(uint32_t value)
{
	if (operand_size_32()) {
		set_EBX(value);
	} else {
		set_BX(value);
	}
}

/******************************************************************************/

static inline void
set_eSP(uint32_t value)
{
	if (operand_size_32()) {
		set_ESP(value);
	} else {
		set_SP(value);
	}
}

static inline void
set_eBP(uint32_t value)
{
	if (operand_size_32()) {
		set_EBP(value);
	} else {
		set_BP(value);
	}
}

static inline void
set_eSI(uint32_t value)
{
	if (operand_size_32()) {
		set_ESI(value);
	} else {
		set_SI(value);
	}
}

static inline void
set_eDI(uint32_t value)
{
	if (operand_size_32()) {
		set_EDI(value);
	} else {
		set_DI(value);
	}
}

/******************************************************************************/

static inline void
set_Rb(uint32_t value, uint32_t reg)
{
	switch (reg) {
	default: DEBUG_ERROR_SWITCH();
	case 0: set_AL(value); break;
	case 1: set_CL(value); break;
	case 2: set_DL(value); break;
	case 3: set_BL(value); break;
	case 4: set_AH(value); break;
	case 5: set_CH(value); break;
	case 6: set_DH(value); break;
	case 7: set_BH(value); break;
	}
}

static inline void
set_Rw(uint32_t value, uint32_t reg)
{
	switch (reg) {
	default: DEBUG_ERROR_SWITCH();
	case 0: set_AX(value); break;
	case 1: set_CX(value); break;
	case 2: set_DX(value); break;
	case 3: set_BX(value); break;
	case 4: set_SP(value); break;
	case 5: set_BP(value); break;
	case 6: set_SI(value); break;
	case 7: set_DI(value); break;
	}
}

static inline void
set_Rd(uint32_t value, uint32_t reg)
{
	switch (reg) {
	default: DEBUG_ERROR_SWITCH();
	case 0: set_EAX(value); break;
	case 1: set_ECX(value); break;
	case 2: set_EDX(value); break;
	case 3: set_EBX(value); break;
	case 4: set_ESP(value); break;
	case 5: set_EBP(value); break;
	case 6: set_ESI(value); break;
	case 7: set_EDI(value); break;
	}
}

/******************************************************************************/

static inline void
set_Cd(uint32_t value)
{
	switch(get_instruction_reg()) {
	default: DEBUG_ERROR_SWITCH();
	case 0: set_CR0(value); break;
	case 1: ERROR_NYI(); /* FIXME: Reserved */
	case 2: set_CR2(value); break;
	case 3: set_CR3(value); break;
	case 4:
	case 5:
	case 6:
	case 7: ERROR_NYI(); /* FIXME: Reserved */
	}
}

/******************************************************************************/

static inline void
set_Eb(uint32_t value)
{
	switch (get_instruction_mod()) {
	default: DEBUG_ERROR_SWITCH();
	case 0:
		mwb_seg(get_address_mod0(), value,
			get_segment_overwritten(get_segment_mod0()));
		break;
	case 1:
		mwb_seg(get_address_mod1(), value,
			get_segment_overwritten(get_segment_mod1()));
		break;
	case 2:
		mwb_seg(get_address_mod2(), value,
			get_segment_overwritten(get_segment_mod2()));
		break;
	case 3:
		set_Rb(value, get_instruction_rm());
		break;
	}
}

static inline void
set_Ew(uint32_t value)
{
	switch (get_instruction_mod()) {
	default: DEBUG_ERROR_SWITCH();
	case 0:
		mww_seg(get_address_mod0(), value,
			get_segment_overwritten(get_segment_mod0()));
		break;
	case 1:
		mww_seg(get_address_mod1(), value,
			get_segment_overwritten(get_segment_mod1()));
		break;
	case 2:
		mww_seg(get_address_mod2(), value,
			get_segment_overwritten(get_segment_mod2()));
		break;
	case 3:
		set_Rw(value, get_instruction_rm());
		break;
	}
}

static inline void
set_Ed(uint32_t value)
{
	switch (get_instruction_mod()) {
	default: DEBUG_ERROR_SWITCH();
	case 0:
		mwd_seg(get_address_mod0(), value,
			get_segment_overwritten(get_segment_mod0()));
		break;
	case 1:
		mwd_seg(get_address_mod1(), value,
			get_segment_overwritten(get_segment_mod1()));
		break;
	case 2:
		mwd_seg(get_address_mod2(), value,
			get_segment_overwritten(get_segment_mod2()));
		break;
	case 3:
		set_Rd(value, get_instruction_rm());
		break;
	}
}

static inline void
set_Ev(uint32_t value)
{
	if (operand_size_32()) {
		set_Ed(value);
	} else{
		set_Ew(value);
	}
}

/******************************************************************************/

static inline void
set_EwBIT(uint32_t value)
{
	int32_t offset;

	offset = ((int16_t) (get_Rw(get_instruction_reg()) & 0xfff0)) / 16;

	switch (get_instruction_mod()) {
	default: DEBUG_ERROR_SWITCH();
	case 0:
		mww_seg(get_address_mod0() + 2 * offset, value,
			get_segment_overwritten(get_segment_mod0()));
		break;
	case 1:
		mww_seg(get_address_mod1() + 2 * offset, value,
			get_segment_overwritten(get_segment_mod1()));
		break;
	case 2:
		mww_seg(get_address_mod2() + 2 * offset, value,
			get_segment_overwritten(get_segment_mod2()));
		break;
	case 3: set_Rd(value, get_instruction_rm());
		break;
	}
}

static inline void
set_EdBIT(uint32_t value)
{
	int32_t offset;

	offset = ((int32_t) (get_Rd(get_instruction_reg()) & 0xffffffe0)) / 32;

	switch (get_instruction_mod()) {
	default: DEBUG_ERROR_SWITCH();
	case 0:
		mwd_seg(get_address_mod0() + 4 * offset, value,
			get_segment_overwritten(get_segment_mod0()));
		break;
	case 1:
		mwd_seg(get_address_mod1() + 4 * offset, value,
			get_segment_overwritten(get_segment_mod1()));
		break;
	case 2:
		mwd_seg(get_address_mod2() + 4 * offset, value,
			get_segment_overwritten(get_segment_mod2()));
		break;
	case 3:
		set_Rd(value, get_instruction_rm());
		break;
	}
}

static inline void
set_EvBIT(uint32_t value)
{
	if (operand_size_32()) {
		set_EdBIT(value);
	} else {
		set_EwBIT(value);
	}
}

/******************************************************************************/

static inline void
set_Ob(uint32_t value)
{
	if (address_size_32()) {
		mwb_seg(get_Id(), value, get_segment_overwritten(SEGMENT_DS));
	} else {
		mwb_seg(get_Iw(), value, get_segment_overwritten(SEGMENT_DS));
	}
}

static inline void
set_Ow(uint32_t value)
{
	if (address_size_32()) {
		mww_seg(get_Id(), value, get_segment_overwritten(SEGMENT_DS));
	} else {
		mww_seg(get_Iw(), value, get_segment_overwritten(SEGMENT_DS));
	}
}

static inline void
set_Od(uint32_t value)
{
	if (address_size_32()) {
		mwd_seg(get_Id(), value, get_segment_overwritten(SEGMENT_DS));
	} else {
		mwd_seg(get_Iw(), value, get_segment_overwritten(SEGMENT_DS));
	}
}

static inline void
set_Ov(uint32_t value)
{
	if (operand_size_32()) {
		set_Od(value);
	} else {
		set_Ow(value);
	}
}

/******************************************************************************/

static inline void
set_Gb(uint32_t value)
{
	switch (get_instruction_reg()) {
	default: DEBUG_ERROR_SWITCH();
	case 0: set_AL(value); break;
	case 1: set_CL(value); break;
	case 2: set_DL(value); break;
	case 3: set_BL(value); break;
	case 4: set_AH(value); break;
	case 5: set_CH(value); break;
	case 6: set_DH(value); break;
	case 7: set_BH(value); break;
	}
}

static inline void
set_Gw(uint32_t value)
{
	switch (get_instruction_reg()) {
	default: DEBUG_ERROR_SWITCH();
	case 0: set_AX(value); break;
	case 1: set_CX(value); break;
	case 2: set_DX(value); break;
	case 3: set_BX(value); break;
	case 4: set_SP(value); break;
	case 5: set_BP(value); break;
	case 6: set_SI(value); break;
	case 7: set_DI(value); break;
	}
}

static inline void
set_Gd(uint32_t value)
{
	switch (get_instruction_reg()) {
	default: DEBUG_ERROR_SWITCH();
	case 0: set_EAX(value); break;
	case 1: set_ECX(value); break;
	case 2: set_EDX(value); break;
	case 3: set_EBX(value); break;
	case 4: set_ESP(value); break;
	case 5: set_EBP(value); break;
	case 6: set_ESI(value); break;
	case 7: set_EDI(value); break;
	}
}

static inline void
set_Gv(uint32_t value)
{
	if (operand_size_32()) {
		set_Gd(value);
	} else {
		set_Gw(value);
	}
}

/******************************************************************************/

static inline void
set_Sw(uint32_t value)
{
	switch (get_instruction_reg()) {
	default: DEBUG_ERROR_SWITCH();
	case SEGMENT_ES: set_ES(value); break;
	case SEGMENT_CS: exception(EXCEPTION_UD, -1); break;
	case SEGMENT_SS: set_SS(value); break;
	case SEGMENT_DS: set_DS(value); break;
	case SEGMENT_FS: set_FS(value); break;
	case SEGMENT_GS: set_GS(value); break;
	case 6: exception(EXCEPTION_UD, -1); break;
	case 7: exception(EXCEPTION_UD, -1); break;
	}
}

/******************************************************************************
 *
 * Set Memory addressed by the ES:eDI register pair
 *
 ******************************************************************************/

static inline void
set_Yb(uint32_t value)
{
	if (address_size_32()) {
		mwb_seg(get_EDI(), value, SEGMENT_ES);
	} else {
		mwb_seg(get_DI(), value, SEGMENT_ES);
	}
}

static inline void
set_Yw(uint32_t value)
{
	if (address_size_32()) {
		mww_seg(get_EDI(), value, SEGMENT_ES);
	} else {
		mww_seg(get_DI(), value, SEGMENT_ES);
	}
}

static inline void
set_Yd(uint32_t value)
{
	if (address_size_32()) {
		mwd_seg(get_EDI(), value, SEGMENT_ES);
	} else {
		mwd_seg(get_DI(), value, SEGMENT_ES);
	}
}

/******************************************************************************
 *
 * Store
 *
 ******************************************************************************/

#define DEBUG_RESULT_B(str) DEBUG(" %s:0x%02x", str, (uint8_t) t0)
#define DEBUG_RESULT_W(str) DEBUG(" %s:0x%04x", str, (uint16_t) t0)
#define DEBUG_RESULT_D(str) DEBUG(" %s:0x%08x", str, (uint32_t) t0)

#define DEBUG_RESULT_V(str)						\
	if (operand_size_32()) {					\
		DEBUG_RESULT_D(str);					\
	} else {							\
		DEBUG_RESULT_W(str);					\
	}

#define DEBUG_STORE() DEBUG_SET(" -> ")

static inline void
store_SS(void)
{
	DEBUG_STORE();
	set_SS(t0);
	DEBUG_RESULT_W("SS");
}

static inline void
store_DS(void)
{
	DEBUG_STORE();
	set_DS(t0);
	DEBUG_RESULT_W("DS");
}

static inline void
store_ES(void)
{
	DEBUG_STORE();
	set_ES(t0);
	DEBUG_RESULT_W("ES");
}

static inline void
store_FS(void)
{
	DEBUG_STORE();
	set_FS(t0);
	DEBUG_RESULT_W("FS");
}

static inline void
store_GS(void)
{
	DEBUG_STORE();
	set_GS(t0);
	DEBUG_RESULT_W("GS");
}

/******************************************************************************/

static inline void
store_AL(void)
{
	DEBUG_STORE();
	set_AL(t0);
	DEBUG_RESULT_B("AL");
}

static inline void
store_CL(void)
{
	DEBUG_STORE();
	set_CL(t0);
	DEBUG_RESULT_B("CL");
}

static inline void
store_DL(void)
{
	DEBUG_STORE();
	set_DL(t0);
	DEBUG_RESULT_B("DL");
}

static inline void
store_BL(void)
{
	DEBUG_STORE();
	set_BL(t0);
	DEBUG_RESULT_B("BL");
}

/******************************************************************************/

static inline void
store_AH(void)
{
	DEBUG_STORE();
	set_AH(t0);
	DEBUG_RESULT_B("AH");
}

static inline void
store_CH(void)
{
	DEBUG_STORE();
	set_CH(t0);
	DEBUG_RESULT_B("CH");
}

static inline void
store_DH(void)
{
	DEBUG_STORE();
	set_DH(t0);
	DEBUG_RESULT_B("DH");
}

static inline void
store_BH(void)
{
	DEBUG_STORE();
	set_BH(t0);
	DEBUG_RESULT_B("BH");
}

/******************************************************************************/

static inline void
store_eAX(void)
{
	DEBUG_STORE();
	set_eAX(t0);
	DEBUG_RESULT_V("eAX");
}

static inline void
store_eCX(void)
{
	DEBUG_STORE();
	set_eCX(t0);
	DEBUG_RESULT_V("eCX");
}

static inline void
store_eDX(void)
{
	DEBUG_STORE();
	set_eDX(t0);
	DEBUG_RESULT_V("eDX");
}

static inline void
store_eBX(void)
{
	DEBUG_STORE();
	set_eBX(t0);
	DEBUG_RESULT_V("eBX");
}

/******************************************************************************/

static inline void
store_eSP(void)
{
	DEBUG_STORE();
	set_eSP(t0);
	DEBUG_RESULT_V("eSP");
}

static inline void
store_eBP(void)
{
	DEBUG_STORE();
	set_eBP(t0);
	DEBUG_RESULT_V("eBP");
}

static inline void
store_eSI(void)
{
	DEBUG_STORE();
	set_eSI(t0);
	DEBUG_RESULT_V("eSI");
}

static inline void
store_eDI(void)
{
	DEBUG_STORE();
	set_eDI(t0);
	DEBUG_RESULT_V("eDI");
}

/******************************************************************************/

static inline void
store_Cd(void)
{
	DEBUG_STORE();
	set_Cd(t0);
	DEBUG_RESULT_V("Cd");
}

static inline void
store_Rd(void)
{
	DEBUG_STORE();
	set_Rd(t0, get_instruction_rm());
	DEBUG_RESULT_V("Rd");
}

/******************************************************************************/

static inline void
store_Eb(void)
{
	DEBUG_STORE();
	set_Eb(t0);
	DEBUG_RESULT_B("Eb");
}

static inline void
store_Ew(void)
{
	DEBUG_STORE();
	set_Ew(t0);
	DEBUG_RESULT_W("Ew");
}

#if 0
static inline void
store_Ed(void)
{
	DEBUG_STORE();
	set_Ed(t0);
	DEBUG_RESULT_D("Ed");
}
#endif
static inline void
store_Ev(void)
{
	DEBUG_STORE();
	set_Ev(t0);
	DEBUG_RESULT_V("Ev");
}

static inline void
store_EvBIT(void)
{
	DEBUG_STORE();
	set_EvBIT(t0);
	DEBUG_RESULT_V("EvBIT");
}

/******************************************************************************/

static inline void
store_Gb(void)
{
	DEBUG_STORE();
	set_Gb(t0);
	DEBUG_RESULT_B("Gb");
}
#if 0
static inline void
store_Gw(void)
{
	DEBUG_STORE();
	set_Gw(t0);
	DEBUG_RESULT_W("Gw");
}

static inline void
store_Gd(void)
{
	DEBUG_STORE();
	set_Gd(t0);
	DEBUG_RESULT_D("Gd");
}
#endif
static inline void
store_Gv(void)
{
	DEBUG_STORE();
	set_Gv(t0);
	DEBUG_RESULT_V("Gv");
}

static inline void
store_Sw(void)
{
	DEBUG_STORE();
	set_Sw(t0);
	DEBUG_RESULT_W("Sw");
}

static inline void
store_Ob(void)
{
	DEBUG_STORE();
	set_Ob(t0);
	DEBUG_RESULT_B("Ob");
}

static inline void
store_Ov(void)
{
	DEBUG_STORE();
	set_Ov(t0);
	DEBUG_RESULT_V("Ov");
}

/******************************************************************************
 *
 * Store Memory addressed by the ES:eDI register pair
 *
 ******************************************************************************/

static inline void
store_Yb(void)
{
	DEBUG_STORE();
	set_Yb(t0);
	DEBUG_RESULT_B("Yb");
}

static inline void
store_Yw(void)
{
	DEBUG_STORE();
	set_Yw(t0);
	DEBUG_RESULT_W("Yw");
}

static inline void
store_Yd(void)
{
	DEBUG_STORE();
	set_Yd(t0);
	DEBUG_RESULT_D("Yd");
}

/******************************************************************************
 *
 * Store t1 and t2
 *
 ******************************************************************************/

static inline void
store_eAX_eCX(void)
{
	DEBUG_STORE();
	set_eAX(t1);
	DEBUG_SET(",");
	set_eCX(t2);
	/* TODO DEBUG */
}

static inline void
store_eAX_eDX(void)
{
	DEBUG_STORE();
	set_eAX(t1);
	DEBUG_SET(",");
	set_eDX(t2);
	/* TODO DEBUG */
}

static inline void
store_eAX_eBX(void)
{
	DEBUG_STORE();
	set_eAX(t1);
	DEBUG_SET(",");
	set_eBX(t2);
	/* TODO DEBUG */
}

static inline void
store_eAX_eSP(void)
{
	DEBUG_STORE();
	set_eAX(t1);
	DEBUG_SET(",");
	set_eSP(t2);
	/* TODO DEBUG */
}

static inline void
store_eAX_eBP(void)
{
	DEBUG_STORE();
	set_eAX(t1);
	DEBUG_SET(",");
	set_eBP(t2);
	/* TODO DEBUG */
}

static inline void
store_eAX_eSI(void)
{
	DEBUG_STORE();
	set_eAX(t1);
	DEBUG_SET(",");
	set_eSI(t2);
	/* TODO DEBUG */
}

static inline void
store_eAX_eDI(void)
{
	DEBUG_STORE();
	set_eAX(t1);
	DEBUG_SET(",");
	set_eDI(t2);
	/* TODO DEBUG */
}

/******************************************************************************/

static inline void
store_Eb_Gb(void)
{
	DEBUG_STORE();
	set_Eb(t1);
	DEBUG_SET(",");
	set_Gb(t2);
	/* TODO DEBUG_RESULT_BB */
}

static inline void
store_Ew_Gw(void)
{
	DEBUG_STORE();
	set_Ew(t1);
	DEBUG_SET(",");
	set_Gw(t2);
	/* TODO DEBUG_RESULT_WW */
}

static inline void
store_Ed_Gd(void)
{
	DEBUG_STORE();
	set_Ed(t1);
	DEBUG_SET(",");
	set_Gd(t2);
	/* TODO DEBUG_STORE_DD */
}

static inline void
store_Ev_Gv(void)
{
	if (operand_size_32()) {
		store_Ed_Gd();
	} else {
		store_Ew_Gw();
	}
}

/******************************************************************************
 *
 * eip
 *
 ******************************************************************************/

#if 0
static inline void
inc_EIP(int32_t value)
{
	eip_new += value;
}
#endif

static inline void
commit_EIP(void)
{
#if 0
	done = 0;
#endif
	if (eip != eip_new) {
		eip = eip_new;
		prefix_clear();
		interrupt_delay = 0;

		if (prefix_lock_repeat == LR_LOCK) {
			unlock();
		}
	}
}

static inline void
commit_EIP_and_delay_interrupts(void)
{
	if (eip != eip_new) {
		eip = eip_new;
		prefix_clear();

		if (interrupt_delay) {
			interrupt_delay = 0;
		} else {
			interrupt_delay = 1;
		}

		if (prefix_lock_repeat == LR_LOCK) {
			unlock();
		}
	}
}

static inline void
revert_EIP(void)
{
	eip_new = eip;
	prefix_clear();
}

/******************************************************************************
 *
 * Parity
 *
 ******************************************************************************/

static inline bool
parity(uint8_t value)
{
	static const uint8_t parity_table[256] = {
		1, 0, 0, 1, 0, 1, 1, 0,	0, 1, 1, 0, 1, 0, 0, 1, /* 0x00 - 0x0f */
		0, 1, 1, 0, 1, 0, 0, 1,	1, 0, 0, 1, 0, 1, 1, 0, /* 0x10 - 0x1f */
		0, 1, 1, 0, 1, 0, 0, 1,	1, 0, 0, 1, 0, 1, 1, 0, /* 0x20 - 0x2f */
		1, 0, 0, 1, 0, 1, 1, 0,	0, 1, 1, 0, 1, 0, 0, 1, /* 0x30 - 0x3f */
		0, 1, 1, 0, 1, 0, 0, 1,	1, 0, 0, 1, 0, 1, 1, 0, /* 0x40 - 0x4f */
		1, 0, 0, 1, 0, 1, 1, 0,	0, 1, 1, 0, 1, 0, 0, 1, /* 0x50 - 0x5f */
		1, 0, 0, 1, 0, 1, 1, 0,	0, 1, 1, 0, 1, 0, 0, 1, /* 0x60 - 0x6f */
		0, 1, 1, 0, 1, 0, 0, 1,	1, 0, 0, 1, 0, 1, 1, 0, /* 0x70 - 0x7f */
		0, 1, 1, 0, 1, 0, 0, 1,	1, 0, 0, 1, 0, 1, 1, 0, /* 0x80 - 0x8f */
		1, 0, 0, 1, 0, 1, 1, 0,	0, 1, 1, 0, 1, 0, 0, 1, /* 0x90 - 0x9f */
		1, 0, 0, 1, 0, 1, 1, 0,	0, 1, 1, 0, 1, 0, 0, 1, /* 0xa0 - 0xaf */
		0, 1, 1, 0, 1, 0, 0, 1,	1, 0, 0, 1, 0, 1, 1, 0, /* 0xb0 - 0xbf */
		1, 0, 0, 1, 0, 1, 1, 0,	0, 1, 1, 0, 1, 0, 0, 1, /* 0xc0 - 0xcf */
		0, 1, 1, 0, 1, 0, 0, 1,	1, 0, 0, 1, 0, 1, 1, 0, /* 0xd0 - 0xdf */
		0, 1, 1, 0, 1, 0, 0, 1,	1, 0, 0, 1, 0, 1, 1, 0, /* 0xe0 - 0xef */
		1, 0, 0, 1, 0, 1, 1, 0,	0, 1, 1, 0, 1, 0, 0, 1, /* 0xf0 - 0xff */
	};

	return parity_table[value];
}

/******************************************************************************
 *
 * OUT - Output to Port
 *
 ******************************************************************************/

static inline void
iowb(uint8_t value, uint32_t port)
{
	uint32_t val32;

	if (port == 0xffff) {
		/* System BIOS / VGA BIOS output port. */
		fprintf(stderr, "%c", value);
		return;
	}

	val32 = value << ((port & 3) * 8);
	cpu_iow(port & ~3, 1 << (port & 3), val32);
}

static inline void
ioww(uint16_t value, uint32_t port)
{
	uint32_t val32;

	if ((port & 3) == 3) {
                unsigned char value0 = (value >> 0) & 0xff;
                unsigned char value8 = (value >> 8) & 0xff;

                iowb(value0, port + 0);
                iowb(value8, port + 1);

        } else {
                val32 = value << ((port & 3) * 8);
                cpu_iow(port & ~3, 3 << (port & 3), val32);
        }
}

static inline void
iowd(uint32_t value, uint32_t port)
{
        if (port & 3) {
                uint32_t val0;
                uint32_t val1;

                val0 = value << ((port & 3) * 8);
                val1 = value >> ((4 - (port & 3)) * 8);

                cpu_iow(port & ~3, (0xf << (port & 3)) & 0xf, val0);
                cpu_iow((port & ~3) + 4, 0xf >> (4 - (port & 3)), val1);

        } else {
                cpu_iow(port & ~3, 0xf, value);
        }
}

/******************************************************************************/

static inline void
outb(void)
{
	/* TODO: check permissions */
	iowb(t2 & 0xff, t1 & 0xffff);
}

static inline void
outw(void)
{
	/* TODO: check permissions */
	ioww(t2 & 0xffff, t1 & 0xffff);
}

static inline void
outd(void)
{
	/* TODO: check permissions */
	iowd(t2, t1 & 0xffff);
}

static inline void
outv(void)
{
	if (operand_size_32()) {
		outd();
	} else {
		outw();
	}
}

/******************************************************************************
 *
 * IN - Input from Port
 *
 ******************************************************************************/

static inline uint8_t
iorb(uint32_t port)
{
	uint32_t val32;
	uint8_t val8;

	val32 = cpu_ior(port & ~3, 1 << (port & 3));
	val8 = (val32 >> ((port & 3) * 8)) & 0xff;

	return val8;
}

static inline uint16_t
iorw(uint32_t port)
{
	uint16_t val16;
	uint32_t val32;

	if ((port & 3) == 3) {
		val16 = iorb(port) | (iorb(port + 1) << 8);
	} else {
		val32 = cpu_ior(port & ~3, 3 << (port & 3));
		val16 = (val32 >> ((port & 3) * 8)) & 0xffff;
	}

	return val16;
}

static inline uint32_t
iord(uint32_t port)
{
	uint32_t value;

	if (port & 3) {
		uint32_t val0;
		uint32_t val1;

		val0 = cpu_ior(port & ~3, (0xf << (port & 3)) & 0xf);
		val1 = cpu_ior((port & ~3) + 4, 0xf >> (4 - (port & 3)));

		value = (val0 >> ((port & 3) * 8))
			| (val1 << ((4 - (port & 3)) * 8));

	} else {
		value = cpu_ior(port & ~3, 0xf);
	}

	return value;
}

/******************************************************************************/

static inline void
inb(void)
{
	/* TODO: check permissions */
	t0 = iorb(t1 & 0xffff);
}

static inline void
inw(void)
{
	/* TODO: check permissions */
	t0 = iorw(t1 & 0xffff);
}

static inline void
ind(void)
{
	/* TODO: check permissions */
	t0 = iord(t1 & 0xffff);
}

static inline void
inv(void)
{
	if (operand_size_32()) {
		ind();
	} else {
		inw();
	}
}

/******************************************************************************
 *
 * INC - Increment by 1
 *
 ******************************************************************************/

static inline void
sset_flags_for_inc(void)
{
	/* cf unchanged */
	sset_AF((t0 & 0xf) == 0);
	sset_ZF(t0 == 0);
	sset_PF(parity(t0));
}

static inline void
incb(void)
{
	t0 = (t1 + 1) & 0xff;
	sset_flags_for_inc();
	sset_SF(t0 >= 0x80);
	sset_OF(t0 == 0x80);
}

static inline void
incw(void)
{
	t0 = (t1 + 1) & 0xffff;
	sset_flags_for_inc();
	sset_SF(t0 >= 0x8000);
	sset_OF(t0 == 0x8000);
}

static inline void
incd(void)
{
	t0 = t1 + 1;
	sset_flags_for_inc();
	sset_SF(t0 >= 0x80000000);
	sset_OF(t0 == 0x80000000);
}

static inline void
incv(void)
{
	if (operand_size_32()) {
		incd();
	} else {
		incw();
	}
}

/******************************************************************************
 *
 * DEC - Decrement by 1
 *
 ******************************************************************************/

static inline void
sset_flags_for_dec(void)
{
	/* cf unchanged */
	sset_AF((t0 & 0xf) == 0xf);
	sset_ZF(t0 == 0);
	sset_PF(parity(t0));
}

static inline void
decb(void)
{
	t0 = (t1 - 1) & 0xff;
	sset_flags_for_dec();
	sset_SF(t0 >= 0x80);
	sset_OF(t0 == 0x7f);
}

static inline void
decw(void)
{
	t0 = (t1 - 1) & 0xffff;
	sset_flags_for_dec();
	sset_SF(t0 >= 0x8000);
	sset_OF(t0 == 0xf777);
}

static inline void
decd(void)
{
	t0 = t1 - 1;
	sset_flags_for_dec();
	sset_SF(t0 >= 0x80000000);
	sset_OF(t0 == 0xf7777777);
}

static inline void
decv(void)
{
	if (operand_size_32()) {
		decd();
	} else {
		decw();
	}
}

/******************************************************************************
 *
 * ADD - Add
 *
 ******************************************************************************/

static inline void
sset_flags_for_add(void)
{
	sset_CF(t0 < t1);
	sset_AF((t1 ^ t2 ^ t0) & 0x10);
	sset_ZF(t0 == 0);
	sset_PF(parity(t0));
}

static inline void
addb(void)
{
	t0 = (t1 + t2) & 0xff;
	sset_flags_for_add();
	sset_SF(t0 >= 0x80);
	sset_OF(((~(t1 ^ t2) & (t2 ^ t0)) & 0x80) != 0);
}

static inline void
addw(void)
{
	t0 = (t1 + t2) & 0xffff;
	sset_flags_for_add();
	sset_SF(t0 >= 0x8000);
	sset_OF(((~(t1 ^ t2) & (t2 ^ t0)) & 0x8000) != 0);
}

static inline void
addd(void)
{
	t0 = t1 + t2;
	sset_flags_for_add();
	sset_SF(t0 >= 0x80000000);
	sset_OF(((~(t1 ^ t2) & (t2 ^ t0)) & 0x80000000) != 0);
}

static inline void
addv(void)
{
	if (operand_size_32()) {
		addd();
	} else {
		addw();
	}
}

/******************************************************************************
 *
 * OR - Logical Inclusive OR
 *
 ******************************************************************************/

static inline void
sset_flags_for_or(void)
{
	sset_CF(0);
	sset_OF(0);
	sset_AF(0); /* undefined */
	sset_ZF(t0 == 0);
	sset_PF(parity(t0));
}

static inline void
orb(void)
{
	t0 = (t1 | t2) & 0xff;
	sset_flags_for_or();
	sset_SF(t0 >= 0x80);
}

static inline void
orw(void)
{
	t0 = (t1 | t2) & 0xffff;
	sset_flags_for_or();
	sset_SF(t0 >= 0x8000);
}

static inline void
ord(void)
{
	t0 = t1 | t2;
	sset_flags_for_or();
	sset_SF(t0 >= 0x80000000);
}

static inline void
orv(void)
{
	if (operand_size_32()) {
		ord();
	} else {
		orw();
	}
}

/******************************************************************************
 *
 * ADC - Add with Carry
 *
 ******************************************************************************/

static inline void
sset_flags_for_adc(void)
{
	sset_AF((t1 ^ t2 ^ t0) & 0x10);
	sset_ZF(t0 == 0);
	sset_PF(parity(t0));
}

static inline void
adcb(void)
{
	t0 = (t1 + t2 + get_CF()) & 0xff;

	sset_flags_for_adc();
	sset_SF(t0 >= 0x80);
	sset_OF(((~(t1 ^ t2) & (t2 ^ t0)) & 0x80) != 0);

	if (get_CF()) {
		sset_CF(t0 <= t1);
	} else {
		sset_CF(t0 < t1);
	}
}

static inline void
adcw(void)
{
	t0 = (t1 + t2 + get_CF()) & 0xffff;

	sset_flags_for_adc();
	sset_SF(t0 >= 0x8000);
	sset_OF(((~(t1 ^ t2) & (t2 ^ t0)) & 0x8000) != 0);

	if (get_CF()) {
		sset_CF(t0 <= t1);
	} else {
		sset_CF(t0 < t1);
	}
}

static inline void
adcd(void)
{
	t0 = t1 + t2 + get_CF();

	sset_flags_for_adc();
	sset_SF(t0 >= 0x80000000);
	sset_OF(((~(t1 ^ t2) & (t2 ^ t0)) & 0x80000000) != 0);

	if (get_CF()) {
		sset_CF(t0 <= t1);
	} else {
		sset_CF(t0 < t1);
	}
}

static inline void
adcv(void)
{
	if (operand_size_32()) {
		adcd();
	} else {
		adcw();
	}
}

/******************************************************************************
 *
 * SBB - Integer Subtraction with Borrow
 *
 ******************************************************************************/

static inline void
sset_flags_for_sbb(void)
{
	sset_AF((t1 ^ t2 ^ t0) & 0x10);
	sset_ZF(t0 == 0);
	sset_PF(parity(t0));
}

static inline void
sbbb(void)
{
	t0 = (t1 - (t2 + get_CF())) & 0xff;
	sset_flags_for_sbb();
	sset_SF(t0 >= 0x80);
	sset_CF((t1 < t0) || (t2 == 0xff));
	sset_OF((((t1 ^ t2) & (t1 ^ t0)) & 0x80) != 0);
}

static inline void
sbbw(void)
{
	t0 = (t1 - (t2 + get_CF())) & 0xffff;
	sset_flags_for_sbb();
	sset_SF(t0 >= 0x8000);
	sset_CF((t1 < t0) || (t2 == 0xffff));
	sset_OF((((t1 ^ t2) & (t1 ^ t0)) & 0x8000) != 0);
}

static inline void
sbbd(void)
{
	t0 = t1 - (t2 + get_CF());
	sset_flags_for_sbb();
	sset_SF(t0 >= 0x80000000);
	sset_CF((t1 < t0) || (t2 == 0xffffffff));
	sset_OF((((t1 ^ t2) & (t1 ^ t0)) & 0x80000000) != 0);
}

static inline void
sbbv(void)
{
	if (operand_size_32()) {
		sbbd();
	} else {
		sbbw();
	}
}

/******************************************************************************
 *
 * AND - Logical AND
 *
 ******************************************************************************/

static inline void
sset_flags_for_and(void)
{
	sset_CF(0);
	sset_OF(0);
	sset_AF(0); /* undefined */
	sset_ZF(t0 == 0);
	sset_PF(parity(t0));
}

static inline void
andb(void)
{
	t0 = (t1 & t2) & 0xff;
	sset_flags_for_and();
	sset_SF(t0 >= 0x80);
}

static inline void
andw(void)
{
	t0 = (t1 & t2) & 0xffff;
	sset_flags_for_and();
	sset_SF(t0 >= 0x8000);
}

static inline void
andd(void)
{
	t0 = t1 & t2;
	sset_flags_for_and();
	sset_SF(t0 >= 0x80000000);
}

static inline void
andv(void)
{
	if (operand_size_32()) {
		andd();
	} else {
		andw();
	}
}

/******************************************************************************
 *
 * SUB - Substract
 *
 ******************************************************************************/

static inline void
sset_flags_for_sub(void)
{
	sset_AF((t1 ^ t2 ^ t0) & 0x10);
	sset_ZF(t0 == 0);
	sset_PF(parity(t0));
}

static inline void
subb(void)
{
	t0 = (t1 - t2) & 0xff;
	sset_flags_for_sub();
	sset_SF(t0 >= 0x80);
	sset_CF((t1 & 0xff) < (t2 & 0xff));
	sset_OF((((t1 ^ t2) & (t1 ^ t0)) & 0x80) != 0);
}

static inline void
subw(void)
{
	t0 = (t1 - t2) & 0xffff;
	sset_flags_for_sub();
	sset_SF(t0 >= 0x8000);
	sset_CF((t1 & 0xffff) < (t2 & 0xffff));
	sset_OF((((t1 ^ t2) & (t1 ^ t0)) & 0x8000) != 0);
}

static inline void
subd(void)
{
	t0 = t1 - t2;
	sset_flags_for_sub();
	sset_SF(t0 >= 0x80000000);
	sset_CF(t1 < t2);
	sset_OF((((t1 ^ t2) & (t1 ^ t0)) & 0x80000000) != 0);
}

static inline void
subv(void)
{
	if (operand_size_32()) {
		subd();
	} else {
		subw();
	}
}

/******************************************************************************
 *
 * XOR - Logical Exclusive OR
 *
 ******************************************************************************/

static inline void
sset_flags_for_xor(void)
{
	sset_CF(0);
	sset_OF(0);
	sset_AF(0); /* undefined */
	sset_ZF(t0 == 0);
	sset_PF(parity(t0));
}

static inline void
xorb(void)
{
	t0 = (t1 ^ t2) & 0xff;
	sset_flags_for_xor();
	sset_SF(t0 >= 0x80);
}

static inline void
xorw(void)
{
	t0 = (t1 ^ t2) & 0xffff;
	sset_flags_for_xor();
	sset_SF(t0 >= 0x8000);
}

static inline void
xord(void)
{
	t0 = t1 ^ t2;
	sset_flags_for_xor();
	sset_SF(t0 >= 0x80000000);
}

static inline void
xorv(void)
{
	if (operand_size_32()) {
		xord();
	} else {
		xorw();
	}
}

/******************************************************************************
 *
 * CMP - Compare Two Operands
 *
 ******************************************************************************/

static inline void
sset_flags_for_cmp(void)
{
	sset_AF((t1 ^ t2 ^ t0) & 0x10);
	sset_ZF(t0 == 0);
	sset_PF(parity(t0));
}

static inline void
cmpb(void)
{
	t0 = (t1 - t2) & 0xff;
	sset_flags_for_cmp();
	sset_SF(t0 >= 0x80);
	sset_CF((t1 & 0xff) < (t2 & 0xff));
	sset_OF((((t1 ^ t2) & (t1 ^ t0)) & 0x80) != 0);
}

static inline void
cmpw(void)
{
	t0 = (t1 - t2) & 0xffff;
	sset_flags_for_cmp();
	sset_SF(t0 >= 0x8000);
	sset_CF((t1 & 0xffff) < (t2 & 0xffff));
	sset_OF((((t1 ^ t2) & (t1 ^ t0)) & 0x8000) != 0);
}

static inline void
cmpd(void)
{
	t0 = t1 - t2;
	sset_flags_for_cmp();
	sset_SF(t0 >= 0x80000000);
	sset_CF(t1 < t2);
	sset_OF((((t1 ^ t2) & (t1 ^ t0)) & 0x80000000) != 0);
}

static inline void
cmpv(void)
{
	if (operand_size_32()) {
		cmpd();
	} else {
		cmpw();
	}
}

/******************************************************************************
 *
 * TEST - Logical Compare
 *
 *****************************************************************************/

/* TODO: merge with AND */

static inline void
sset_flags_for_test(void)
{
	sset_CF(0);
	sset_OF(0);
	sset_AF(0); /* undefined */
	sset_ZF(t0 == 0);
	sset_PF(parity(t0));
}

static inline void
testb(void)
{
	t0 = (t1 & t2) & 0xff;
	sset_flags_for_test();
	sset_SF(t0 >= 0x80);
}

static inline void
testw(void)
{
	t0 = (t1 & t2) & 0xffff;
	sset_flags_for_test();
	sset_SF(t0 >= 0x8000);
}

static inline void
testd(void)
{
	t0 = t1 & t2;
	sset_flags_for_test();
	sset_SF(t0 >= 0x80000000);
}

static inline void
testv(void)
{
	if (operand_size_32()) {
		testd();
	} else {
		testw();
	}
}

/******************************************************************************
 *
 * NOT - One's Complement Negation
 *
 ******************************************************************************/

static inline void
notb(void)
{
	t0 = ~t1;
}

static inline void
notv(void)
{
	t0 = ~t1;
}

/******************************************************************************
 *
 * NEG - Two's Complement Negation
 *
 ******************************************************************************/

static inline void
sset_flags_for_neg(void)
{
	sset_CF(t1 != 0);
	sset_AF((t0 & 0xf) != 0);
	sset_ZF(t0 == 0);
	sset_PF(parity(t0));
}

static inline void
negb(void)
{
	t0 = -t1 & 0xff;
	sset_OF(t0 == 0x80);
	sset_SF(t0 >= 0x80);
	sset_flags_for_neg();
}

static inline void
negw(void)
{
	t0 = -t1 & 0xffff;
	sset_OF(t0 == 0x8000);
	sset_SF(t0 >= 0x8000);
	sset_flags_for_neg();
}

static inline void
negd(void)
{
	t0 = -t1;
	sset_OF(t0 == 0x80000000);
	sset_SF(t0 >= 0x80000000);
	sset_flags_for_neg();
}

static inline void
negv(void)
{
	if (operand_size_32()) {
		negd();
	} else {
		negw();
	}
}

/******************************************************************************
 *
 * MUL - Unsigned Multiply
 *
 ******************************************************************************/

static inline void
mulb(void)
{
	uint8_t product8l, product8h;
	uint32_t product16;

	product16 = get_AL() * (uint8_t) t1;
	product8l = product16 & 0xff;
	product8h = product16 >> 8;

	sset_CF(product8h != 0);
	sset_ZF(product8l == 0);
	sset_SF(product8l >= 0x80);
	sset_OF(product8h != 0);
	sset_AF(0);
	sset_PF(parity(product8l));

	set_AX(product16);
}

static inline void
mulw(void)
{
	uint16_t product16l, product16h;
	uint32_t product32;

	product32 = get_AX() * (uint16_t) t1;
	product16l = product32 & 0xffff;
	product16h = product32 >> 16;

	sset_CF(product16h != 0);
	sset_ZF(product16l == 0);
	sset_SF(product16l >= 0x8000);
	sset_OF(product16h != 0);
	sset_AF(0);
	sset_PF(parity(product16l));

	set_AX(product16l);
	set_DX(product16h);
}

static inline void
muld(void)
{
	uint32_t product32l, product32h;
	uint64_t product64;

	product64 = (uint64_t) get_EAX() * (uint64_t) t1;
	product32l = product64 & 0xffffffff;
	product32h = product64 >> 32;

	sset_CF(product32h != 0);
	sset_ZF(product32l == 0);
	sset_SF(product32l >= 0x80000000);
	sset_OF(product32h != 0);
	sset_AF(0);
	sset_PF(parity(product32l));

	set_EAX(product32l);
	set_EDX(product32h);
}

static inline void
mulv(void)
{
	if (operand_size_32()) {
		muld();
	} else {
		mulw();
	}
}

/******************************************************************************
 *
 * IMUL - Signed Multiply
 *
 ******************************************************************************/

static inline void
imul1b(void)
{
	int8_t op2, op1;
	int16_t product16;
	uint8_t product8l, product8h;
	bool cf;

	op1 = get_AL();
	op2 = t1;

	product16 = op1 * op2;
	product8l = product16 & 0xff;
	product8h = product16 >> 8;

	cf = !(((product8l < 0x80) && (product8h == 0))
	    || ((product8l & 0x80) && (product8h == 0xff)));

	sset_CF(cf);
	sset_OF(cf);
	sset_AF(0);			/* undefined */
	sset_ZF(product8l == 0);	/* undefined */
	sset_SF(product8l >= 0x80);	/* undefined */
	sset_PF(parity(product8l));	/* undefined */

	set_AX(product16);
}

static inline void
imul1w(void)
{
	int16_t op1, op2;
	int32_t product32;
	uint16_t product16l, product16h;
	bool cf;

	op1 = get_AX();
	op2 = t1;

	product32 = op1 * op2;
	product16l = product32 & 0xffff;
	product16h = product32 >> 16;

	cf = !(((product16l < 0x8000) && (product16h == 0))
	    || ((product16l & 0x8000) && (product16h == 0xffff)));

	sset_CF(cf);
	sset_OF(cf);
	sset_AF(0);			/* undefined */
	sset_ZF(product16l == 0);	/* undefined */
	sset_SF(product16l >= 0x8000);	/* undefined */
	sset_PF(parity(product16l));	/* undefined */

	set_AX(product16l);
	set_DX(product16h);
}

static inline void
imul1d(void)
{
	int32_t op1, op2;
	int64_t product64;
	uint32_t product32l, product32h;
	bool cf;

	op1 = get_EAX();
	op2 = t1;

	product64 = ((int64_t) op1) * ((int64_t) op2);
	product32l = product64 & 0xffffffff;
	product32h = product64 >> 32;

	cf = !(((product32l < 0x80000000) && (product32h == 0))
	    || ((product32l & 0x80000000) && (product32h == 0xffffffff)));

	sset_CF(cf);
	sset_OF(cf);
	sset_AF(0);				/* undefined */
	sset_ZF(product32l == 0);		/* undefined */
	sset_SF(product32l >= 0x80000000);	/* undefined */
	sset_PF(parity(product32l));		/* undefined */

	set_EAX(product32l);
	set_EDX(product32h);
}

static inline void
imul1v(void)
{
	if (operand_size_32()) {
		imul1d();
	} else {
		imul1w();
	}
}

/******************************************************************************/

static inline void
imul3ww(void)
{
	int16_t op1, op2;
	int32_t product32;
	uint16_t product16l, product16h;
	bool cf;

	op1 = t1;
	op2 = t2;

	product32 = op1 * op2;
	product16l = product32 & 0xffff;
	product16h = product32 >> 16;

	t0 = product16l;

	cf = !(((product16l < 0x8000) && (product16h == 0))
	    || ((product16l & 0x8000) && (product16h == 0xffff)));

	sset_CF(cf);
	sset_OF(cf);
	sset_AF(0);			/* undefined */
	sset_SF(product16l >= 0x8000);	/* undefined */
	sset_ZF(product16l == 0);	/* undefined */
	sset_PF(parity(product16l));	/* undefined */
}

static inline void
imul3dd(void)
{
	int32_t op1, op2;
	int64_t product64;
	uint32_t product32l, product32h;
	bool cf;

	op1 = t1;
	op2 = t2;

	product64 = ((int64_t) op1) * ((int64_t) op2);
	product32l = product64 & 0xffffffff;
	product32h = product64 >> 32;

	t0 = product32l;

	cf = !(((product32l < 0x80000000) && (product32h == 0))
	    || ((product32l & 0x80000000) && (product32h == 0xffffffff)));

	sset_CF(cf);
	sset_OF(cf);
	sset_AF(0);			/* undefined */
	sset_SF(product32l >= 0x8000);	/* undefined */
	sset_ZF(product32l == 0);	/* undefined */
	sset_PF(parity(product32l));	/* undefined */
}

static inline void
imul3vv(void)
{
	if (operand_size_32()) {
		imul3dd();
	} else {
		imul3ww();
	}
}

/******************************************************************************/

static inline void
imul3wb(void)
{
	t2 = (int16_t) (int8_t) t2;

	imul3ww();
}

static inline void
imul3db(void)
{
	t2 = (int32_t) (int8_t) t2;

	imul3dd();
}

static inline void
imul3vb(void)
{
	if (operand_size_32()) {
		imul3db();
	} else {
		imul3wb();
	}
}

/******************************************************************************/

static inline void
imul2v(void)
{
	imul3vv();
}

/******************************************************************************
 *
 * DIV - Unsigned Divide
 *
 ******************************************************************************/

static inline void
sset_flags_for_div(void)
{
	sset_CF(0); /* undefined */
	sset_OF(0); /* undefined */
	sset_SF(0); /* undefined */
	sset_ZF(0); /* undefined */
	sset_AF(0); /* undefined */
	sset_PF(0); /* undefined */
}

static inline void
divb(void)
{
	uint8_t remainder8, quotient8;
	uint16_t operand16, quotient16;

	operand16 = get_AX();

	if (t1 == 0) {
		exception(EXCEPTION_DE, -1);
	}

	quotient16 = operand16 / (uint8_t) t1;
	remainder8 = operand16 % (uint8_t) t1;
	quotient8 = quotient16 & 0xff;

	if (quotient16 != quotient8) {
		exception(EXCEPTION_DE, -1);
	}

	sset_flags_for_div();

	set_AL(quotient8);
	set_AH(remainder8);
}

static inline void
divw(void)
{
	uint16_t remainder16, quotient16;
	uint32_t operand32, quotient32;

	operand32 = (get_DX() << 16) | get_AX();

	if (t1 == 0) {
		exception(EXCEPTION_DE, -1);
	}

	quotient32 = operand32 / (uint16_t) t1;
	remainder16 = operand32 % (uint16_t) t1;
	quotient16 = quotient32 & 0xffff;

	if (quotient32 != quotient16) {
		exception(EXCEPTION_DE, -1);
	}

	sset_flags_for_div();

	set_AX(quotient16);
	set_DX(remainder16);
}

static inline void
divd(void)
{
	uint32_t remainder32, quotient32;
	uint64_t operand64, quotient64;

	operand64 = (((uint64_t) get_EDX()) << 32) + (uint64_t) get_EAX();

	if (t1 == 0) {
		exception(EXCEPTION_DE, -1);
	}

	quotient64 = operand64 / t1;
	remainder32 = (uint32_t) (operand64 % t1);
	quotient32 = (uint32_t) (quotient64 & 0xffffffff);

	if (quotient64 != quotient32) {
		exception(EXCEPTION_DE, -1);
	}

	sset_flags_for_div();

	set_EAX(quotient32);
	set_EDX(remainder32);
}

static inline void
divv(void)
{
	if (operand_size_32()) {
		divd();
	} else {
		divw();
	}
}

/******************************************************************************
 *
 * IDIV - Signed Divide
 *
 ******************************************************************************/

static inline void
sset_flags_for_idiv(void)
{
	sset_CF(0); /* undefined */
	sset_OF(0); /* undefined */
	sset_SF(0); /* undefined */
	sset_ZF(0); /* undefined */
	sset_AF(0); /* undefined */
	sset_PF(0); /* undefined */
}

static inline void
idivb(void)
{
	int8_t op2, quotient8l, remainder8;
	int16_t op1, quotient16;

	op1 = get_AX();
	op2 = t1;

	if (op2 == 0) {
		exception(EXCEPTION_DE, -1);
	}

	if ((op1 == ((int16_t) 0x8000)) && (op2 == -1)) {
		exception(EXCEPTION_DE, -1);
	}

	quotient16 = op1 / op2;
	remainder8 = op1 % op2;
	quotient8l = quotient16 & 0xff;

	if (quotient16 != quotient8l) {
		exception(EXCEPTION_DE, -1);
	}

	sset_flags_for_idiv();

	set_AL(quotient8l);
	set_AH(remainder8);
}

static inline void
idivw(void)
{
	int16_t op2, remainder16, quotient16l;
	int32_t op1, quotient32;

	op1 = ((get_DX() & 0xffff) << 16) | (get_AX() & 0xffff);
	op2 = t1;

	if (op2 == 0) {
		exception(EXCEPTION_DE, -1);
	}

	if ((op1 == ((int32_t) 0x80000000)) && (op2 == -1)) {
		exception(EXCEPTION_DE, -1);
	}

	quotient32 = op1 / op2;
	remainder16 = op1 / op2;
	quotient16l = quotient32 & 0xffff;

	if (quotient32 != quotient16l) {
		exception(EXCEPTION_DE, -1);
	}

	sset_flags_for_idiv();

	set_AX(quotient16l);
	set_DX(remainder16);
}

static inline void
idivd(void)
{
	int32_t op2, remainder32, quotient32l;
	int64_t op1, quotient64;

	op1 = ((uint64_t) get_EDX() << 32) | get_EAX();
	op2 = t1;

	if (op2 == 0) {
		exception(EXCEPTION_DE, -1);
	}

	if ((op1 == ((int64_t) 0x8000000000000000ULL)) && (op2 == -1)) {
		exception(EXCEPTION_DE, -1);
	}

	quotient64 = op1 / op2;
	remainder32 = (int32_t) (op1 % op2);
	quotient32l = (int32_t) (quotient64 & 0xffffffff);

	if (quotient64 != quotient32l) {
		exception(EXCEPTION_DE, -1);
	}

	sset_flags_for_idiv();

	set_EAX(quotient32l);
	set_EDX(remainder32);
}

static inline void
idivv(void)
{
	if (operand_size_32()) {
		idivd();
	} else {
		idivw();
	}
}

/******************************************************************************
 *
 * XCHG - Exchange Register/Memory with Register
 *
 ******************************************************************************/

static inline void
xchg(void)
{
	uint32_t t;

	t = t1;
	t1 = t2;
	t2 = t;
}

/******************************************************************************
 *
 * MOV
 *
 ******************************************************************************/

static inline void
movb(void)
{
	t0 = t1;
}

static inline void
movw(void)
{
	t0 = t1;
}

static inline void
movd(void)
{
	t0 = t1;
}

static inline void
movv(void)
{
	if (operand_size_32()) {
		movd();
	} else {
		movw();
	}
}

/******************************************************************************
 *
 * MOVS/MOVSB/MOVSW/MOVSD - Move Data from String to String
 *
 ******************************************************************************/

#define REP_CHECK()							\
	if (prefix_lock_repeat == LR_REPZ) {				\
		if (address_size_32()) {				\
			if (get_ECX() == 0) {				\
				return;					\
			}						\
		} else {						\
			if (get_CX() == 0) {				\
				return;					\
			}						\
		}							\
	}

#define REP_DECREMENT()							\
	if (prefix_lock_repeat == LR_REPZ) {				\
		revert_EIP();						\
		if (address_size_32()) {				\
			set_ECX(get_ECX() - 1);				\
		} else {						\
			set_CX(get_CX() - 1);				\
		}							\
	}

static inline void
movsb(void)
{
	REP_CHECK();

	load_Xb();
	t0 = t1;
	store_Yb();

	if (address_size_32()) {
		if (get_DF()) {
			set_ESI(get_ESI() - 1);
			set_EDI(get_EDI() - 1);
		} else {
			set_ESI(get_ESI() + 1);
			set_EDI(get_EDI() + 1);
		}
	} else {
		if (get_DF()) {
			set_SI(get_SI() - 1);
			set_DI(get_DI() - 1);
		} else {
			set_SI(get_SI() + 1);
			set_DI(get_DI() + 1);
		}
	}

	REP_DECREMENT();
}

static inline void
movsw(void)
{
	REP_CHECK();

	load_Xw();
	t0 = t1;
	store_Yw();

	if (address_size_32()) {
		if (get_DF()) {
			set_ESI(get_ESI() - 2);
			set_EDI(get_EDI() - 2);
		} else {
			set_ESI(get_ESI() + 2);
			set_EDI(get_EDI() + 2);
		}
	} else {
		if (get_DF()) {
			set_SI(get_SI() - 2);
			set_DI(get_DI() - 2);
		} else {
			set_SI(get_SI() + 2);
			set_DI(get_DI() + 2);
		}
	}

	REP_DECREMENT();
}

static inline void
movsd(void)
{
	REP_CHECK();

	load_Xd();
	t0 = t1;
	store_Yd();

	if (address_size_32()) {
		if (get_DF()) {
			set_ESI(get_ESI() - 4);
			set_EDI(get_EDI() - 4);
		} else {
			set_ESI(get_ESI() + 4);
			set_EDI(get_EDI() + 4);
		}
	} else {
		if (get_DF()) {
			set_SI(get_SI() - 4);
			set_DI(get_DI() - 4);
		} else {
			set_SI(get_SI() + 4);
			set_DI(get_DI() + 4);
		}
	}

	REP_DECREMENT();
}

static inline void
movsv(void)
{
	if (operand_size_32()) {
		movsd();
	} else {
		movsw();
	}
}

/******************************************************************************
 *
 * LODS/LODSB/LODSW/LODSD - Load String
 *
 ******************************************************************************/

static inline void
lodsb(void)
{
	REP_CHECK();

	load_Xb();
	t0 = t1;
	store_AL();

	if (address_size_32()) {
		if (get_DF()) {
			set_ESI(get_ESI() - 1);
		} else {
			set_ESI(get_ESI() + 1);
		}
	} else {
		if (get_DF()) {
			set_SI(get_SI() - 1);
		} else {
			set_SI(get_SI() + 1);
		}
	}

	REP_DECREMENT();
}

static inline void
lodsw(void)
{
	REP_CHECK();

	load_Xw();
	t0 = t1;
	store_eAX();

	if (address_size_32()) {
		if (get_DF()) {
			set_ESI(get_ESI() - 2);
		} else {
			set_ESI(get_ESI() + 2);
		}
	} else {
		if (get_DF()) {
			set_SI(get_SI() - 2);
		} else {
			set_SI(get_SI() + 2);
		}
	}

	REP_DECREMENT();
}

static inline void
lodsd(void)
{
	REP_CHECK();

	load_Xd();
	t0 = t1;
	store_eAX();

	if (address_size_32()) {
		if (get_DF()) {
			set_ESI(get_ESI() - 4);
		} else {
			set_ESI(get_ESI() + 4);
		}
	} else {
		if (get_DF()) {
			set_SI(get_SI() - 4);
		} else {
			set_SI(get_SI() + 4);
		}
	}

	REP_DECREMENT();
}

static inline void
lodsv(void)
{
	if (operand_size_32()) {
		lodsd();
	} else {
		lodsw();
	}
}

/******************************************************************************
 *
 * OUTS/OUTSB/OUTSW/OUTSD - Output String to Port
 *
 ******************************************************************************/

static inline void
outsb(void)
{
	REP_CHECK();

	load_DX_Xb();

	outb();

	if (address_size_32()) {
		if (get_DF()) {
			set_ESI(get_ESI() - 1);
		} else {
			set_ESI(get_ESI() + 1);
		}
	} else {
		if (get_DF()) {
			set_SI(get_SI() - 1);
		} else {
			set_SI(get_SI() + 1);
		}
	}

	REP_DECREMENT();
}

static inline void
outsw(void)
{
	REP_CHECK();

	load_DX_Xw();

	outw();

	if (address_size_32()) {
		if (get_DF()) {
			set_ESI(get_ESI() - 2);
		} else {
			set_ESI(get_ESI() + 2);
		}
	} else {
		if (get_DF()) {
			set_SI(get_SI() - 2);
		} else {
			set_SI(get_SI() + 2);
		}
	}

	REP_DECREMENT();
}

static inline void
outsd(void)
{
	REP_CHECK();

	load_DX_Xd();

	outd();

	if (address_size_32()) {
		if (get_DF()) {
			set_ESI(get_ESI() - 4);
		} else {
			set_ESI(get_ESI() + 4);
		}
	} else {
		if (get_DF()) {
			set_SI(get_SI() - 4);
		} else {
			set_SI(get_SI() + 4);
		}
	}

	REP_DECREMENT();
}

static inline void
outsv(void)
{
	if (operand_size_32()) {
		outsd();
	} else {
		outsw();
	}
}

/******************************************************************************
 *
 * STOS/STOSB/STOSW/STOSD - Store String
 *
 ******************************************************************************/

static inline void
stosb(void)
{
	REP_CHECK();

	load_AL();
	t0 = t1;
	store_Yb();

	if (address_size_32()) {
		if (get_DF()) {
			set_EDI(get_EDI() - 1);
		} else {
			set_EDI(get_EDI() + 1);
		}
	} else {
		if (get_DF()) {
			set_DI(get_DI() - 1);
		} else {
			set_DI(get_DI() + 1);
		}
	}

	REP_DECREMENT();
}

static inline void
stosw(void)
{
	REP_CHECK();

	load_eAX();
	t0 = t1;
	store_Yw();

	if (address_size_32()) {
		if (get_DF()) {
			set_EDI(get_EDI() - 2);
		} else {
			set_EDI(get_EDI() + 2);
		}
	} else {
		if (get_DF()) {
			set_DI(get_DI() - 2);
		} else {
			set_DI(get_DI() + 2);
		}
	}

	REP_DECREMENT();
}

static inline void
stosd(void)
{
	REP_CHECK();

	load_eAX();
	t0 = t1;
	store_Yd();

	if (address_size_32()) {
		if (get_DF()) {
			set_EDI(get_EDI() - 4);
		} else {
			set_EDI(get_EDI() + 4);
		}
	} else {
		if (get_DF()) {
			set_DI(get_DI() - 4);
		} else {
			set_DI(get_DI() + 4);
		}
	}

	REP_DECREMENT();
}

static inline void
stosv(void)
{
	if (operand_size_32()) {
		stosd();
	} else {
		stosw();
	}
}

/******************************************************************************
 *
 * INS/INSB/INSW/INSD - Input from Port to String
 *
 ******************************************************************************/

static inline void
insb(void)
{
	REP_CHECK();

	load_DX();
	inb();
	store_Yb();

	if (address_size_32()) {
		if (get_DF()) {
			set_EDI(get_EDI() - 1);
		} else {
			set_EDI(get_EDI() + 1);
		}
	} else {
		if (get_DF()) {
			set_DI(get_DI() - 1);
		} else {
			set_DI(get_DI() + 1);
		}
	}

	REP_DECREMENT();
}

static inline void
insw(void)
{
	REP_CHECK();

	load_DX();
	inw();
	store_Yw();

	if (address_size_32()) {
		if (get_DF()) {
			set_EDI(get_EDI() - 2);
		} else {
			set_EDI(get_EDI() + 2);
		}
	} else {
		if (get_DF()) {
			set_DI(get_DI() - 2);
		} else {
			set_DI(get_DI() + 2);
		}
	}

	REP_DECREMENT();
}

static inline void
insd(void)
{
	REP_CHECK();

	load_DX();
	ind();
	store_Yd();

	if (address_size_32()) {
		if (get_DF()) {
			set_EDI(get_EDI() - 4);
		} else {
			set_EDI(get_EDI() + 4);
		}
	} else {
		if (get_DF()) {
			set_DI(get_DI() - 4);
		} else {
			set_DI(get_DI() + 4);
		}
	}

	REP_DECREMENT();
}

static inline void
insv(void)
{
	if (operand_size_32()) {
		insd();
	} else {
		insw();
	}
}

/******************************************************************************
 *
 * CMPS/CMPSB/CMPSW/CMPSD - Compare String Operands
 *
 ******************************************************************************/

#define REPZ_REPNZ_CHECK()						\
	if ((prefix_lock_repeat == LR_REPZ)				\
	 || (prefix_lock_repeat == LR_REPNZ)) {				\
		if (address_size_32()) {				\
			if (get_ECX() == 0) {				\
				return;					\
			}						\
		} else {						\
			if (get_CX() == 0) {				\
				return;					\
			}						\
		}							\
	}

#define REPZ_REPNZ_DECREMENT()						\
	commit_OSZAPC();						\
	if ((prefix_lock_repeat == LR_REPZ)				\
	 || (prefix_lock_repeat == LR_REPNZ)) {				\
		bool break_condition = get_ZF();			\
		if (prefix_lock_repeat == LR_REPZ) {			\
			break_condition = !break_condition;		\
		}							\
		if (!break_condition) {					\
			revert_EIP();					\
		}							\
		if (address_size_32()) {				\
			set_ECX(get_ECX() - 1);				\
		} else {						\
			set_CX(get_CX() - 1);				\
		}							\
	}

static inline void
cmpsb(void)
{
	REPZ_REPNZ_CHECK();

	load_Xb_Yb();

	cmpb();

	if (address_size_32()) {
		if (get_DF()) {
			set_ESI(get_ESI() - 1);
			set_EDI(get_EDI() - 1);
		} else {
			set_ESI(get_ESI() + 1);
			set_EDI(get_EDI() + 1);
		}
	} else {
		if (get_DF()) {
			set_SI(get_SI() - 1);
			set_DI(get_DI() - 1);
		} else {
			set_SI(get_SI() + 1);
			set_DI(get_DI() + 1);
		}
	}

	REPZ_REPNZ_DECREMENT();
}

static inline void
cmpsw(void)
{
	REPZ_REPNZ_CHECK();

	load_Xw_Yw();

	cmpw();

	if (address_size_32()) {
		if (get_DF()) {
			set_ESI(get_ESI() - 2);
			set_EDI(get_EDI() - 2);
		} else {
			set_ESI(get_ESI() + 2);
			set_EDI(get_EDI() + 2);
		}
	} else {
		if (get_DF()) {
			set_SI(get_SI() - 2);
			set_DI(get_DI() - 2);
		} else {
			set_SI(get_SI() + 2);
			set_DI(get_DI() + 2);
		}
	}

	REPZ_REPNZ_DECREMENT();
}

static inline void
cmpsd(void)
{
	REPZ_REPNZ_CHECK();

	load_Xd_Yd();

	cmpd();

	if (address_size_32()) {
		if (get_DF()) {
			set_ESI(get_ESI() - 4);
			set_EDI(get_EDI() - 4);
		} else {
			set_ESI(get_ESI() + 4);
			set_EDI(get_EDI() + 4);
		}
	} else {
		if (get_DF()) {
			set_SI(get_SI() - 4);
			set_DI(get_DI() - 4);
		} else {
			set_SI(get_SI() + 4);
			set_DI(get_DI() + 4);
		}
	}

	REPZ_REPNZ_DECREMENT();
}

static inline void
cmpsv(void)
{
	if (operand_size_32()) {
		cmpsd();
	} else {
		cmpsw();
	}
}

/******************************************************************************
 *
 * SCAS/SCASB/SCASW/SCASD - Scan String
 *
 ******************************************************************************/

static inline void
scasb(void)
{
	REPZ_REPNZ_CHECK();

	load_AL_Yb();

	cmpb();

	if (address_size_32()) {
		if (get_DF()) {
			set_EDI(get_EDI() - 1);
		} else {
			set_EDI(get_EDI() + 1);
		}
	} else {
		if (get_DF()) {
			set_DI(get_DI() - 1);
		} else {
			set_DI(get_DI() + 1);
		}
	}

	REPZ_REPNZ_DECREMENT();
}

static inline void
scasw(void)
{
	REPZ_REPNZ_CHECK();

	load_AX_Yw();

	cmpw();

	if (address_size_32()) {
		if (get_DF()) {
			set_EDI(get_EDI() - 2);
		} else {
			set_EDI(get_EDI() + 2);
		}
	} else {
		if (get_DF()) {
			set_DI(get_DI() - 2);
		} else {
			set_DI(get_DI() + 2);
		}
	}

	REPZ_REPNZ_DECREMENT();
}

static inline void
scasd(void)
{
	REPZ_REPNZ_CHECK();

	load_EAX_Yd();

	cmpd();

	if (address_size_32()) {
		if (get_DF()) {
			set_EDI(get_EDI() - 4);
		} else {
			set_EDI(get_EDI() + 4);
		}
	} else {
		if (get_DF()) {
			set_DI(get_DI() - 4);
		} else {
			set_DI(get_DI() + 4);
		}
	}

	REPZ_REPNZ_DECREMENT();
}

static inline void
scasv(void)
{
	if (operand_size_32()) {
		scasd();
	} else {
		scasw();
	}
}

/******************************************************************************
 *
 * LEA
 *
 ******************************************************************************/

static inline void
lea(void)
{
	t0 = t1;
}

/******************************************************************************
 *
 * MOVZX - Move with Zero-Extend
 *
 ******************************************************************************/

static inline void
movzxb(void)
{
	t0 = t1 & 0xff;
}

static inline void
movzxw(void)
{
	t0 = t1 & 0xffff;
}

/******************************************************************************
 *
 * MOVSX - Move with Sign-Extension
 *
 ******************************************************************************/

static inline void
movsxb(void)
{
	int16_t t;

	t = (int8_t) t1;
	t0 = (uint32_t) t;
}

static inline void
movsxw(void)
{
	int32_t t;

	if (operand_size_32()) {
		t = (int16_t) t1;
	} else {
		 t = (int8_t) t1;
	}

	t0 = (uint32_t) t;
}

/******************************************************************************
 *
 * LDS/LES/LFS/LGS/LSS
 *
 ******************************************************************************/

static inline void
lds(void)
{
	t0 = t1;
	set_DS(t2);
}

static inline void
les(void)
{
	t0 = t1;
	set_ES(t2);
}

static inline void
lfs(void)
{
	t0 = t1;
	set_FS(t2);
}

static inline void
lgs(void)
{
	t0 = t1;
	set_GS(t2);
}

static inline void
lss(void)
{
	t0 = t1;
	set_SS(t2);
}

/******************************************************************************
 *
 * PUSH - Push Word or Doubleword Onto the Stack
 *
 ******************************************************************************/


static inline void
push32(uint32_t value)
{
	if (stack_size_32()) {
		set_ESP(get_ESP() - 4);
		mwd_seg(get_ESP(), value, SEGMENT_SS);
	} else {
		set_SP(get_SP() - 4);
		mwd_seg(get_SP(), value, SEGMENT_SS);
	}
}

static inline void
push16(uint16_t value)
{
	if (stack_size_32()) {
		set_ESP(get_ESP() - 2);
		mww_seg(get_ESP(), value, SEGMENT_SS);
	} else {
		set_SP(get_SP() - 2);
		mww_seg(get_SP(), value, SEGMENT_SS);
	}
}

static inline void
pushv(void)
{
	if (operand_size_32()) {
		push32(t1);
	} else {
		push16(t1);
	}
}

/******************************************************************************
 *
 * PUSHA - Push All General-Purpose Registers
 *
 ******************************************************************************/

static inline void
pushav(void)
{
	if (operand_size_32()) {
		uint32_t temp_esp;

		temp_esp = get_ESP();

		push32(get_EAX());
		push32(get_ECX());
		push32(get_EDX());
		push32(get_EBX());
		push32(temp_esp);
		push32(get_EBP());
		push32(get_ESI());
		push32(get_EDI());
	} else {
		uint16_t temp_sp;

		temp_sp = get_SP();

		push16(get_AX());
		push16(get_CX());
		push16(get_DX());
		push16(get_BX());
		push16(temp_sp);
		push16(get_BP());
		push16(get_SI());
		push16(get_DI());
	}
}

/******************************************************************************
 *
 * POP - Pop a Value from the Stack
 *
 ******************************************************************************/

static inline uint32_t
pop32(void)
{
	uint32_t value;

	if (stack_size_32()) {
		value = mrd_seg(get_ESP(), SEGMENT_SS);
		set_ESP(get_ESP() + 4);
	} else {
		value = mrd_seg(get_SP(), SEGMENT_SS);
		set_SP(get_SP() + 4);
	}

	return value;
}

static inline uint16_t
pop16(void)
{
	uint16_t value;

	if (stack_size_32()) {
		value = mrw_seg(get_ESP(), SEGMENT_SS);
		set_ESP(get_ESP() + 2);
	} else {
		value = mrw_seg(get_SP(), SEGMENT_SS);
		set_SP(get_SP() + 2);
	}

	return value;
}

static inline void
popv(void)
{
	if (operand_size_32()) {
		t0 = pop32();
	} else {
		t0 = pop16();
	}
}

/******************************************************************************
 *
 * POPA/POPAD - Pop All General-Purpose Registers
 *
 ******************************************************************************/

static inline void
popa(void)
{
	if (operand_size_32()) {
		uint32_t temp_edi;
		uint32_t temp_esi;
		uint32_t temp_ebp;
		/* no esp */
		uint32_t temp_ebx;
		uint32_t temp_edx;
		uint32_t temp_ecx;
		uint32_t temp_eax;

		temp_edi = pop32();
		temp_esi = pop32();
		temp_ebp = pop32();
		pop32();
		temp_ebx = pop32();
		temp_edx = pop32();
		temp_ecx = pop32();
		temp_eax = pop32();

		set_EDI(temp_edi);
		set_ESI(temp_esi);
		set_EBP(temp_ebp);
		/* esp no set */
		set_EBX(temp_ebx);
		set_EDX(temp_edx);
		set_ECX(temp_ecx);
		set_EAX(temp_eax);
	} else {
		uint16_t temp_di;
		uint16_t temp_si;
		uint16_t temp_bp;
		/* no sp */
		uint16_t temp_bx;
		uint16_t temp_dx;
		uint16_t temp_cx;
		uint16_t temp_ax;

		temp_di = pop16();
		temp_si = pop16();
		temp_bp = pop16();
		pop16();
		temp_bx = pop16();
		temp_dx = pop16();
		temp_cx = pop16();
		temp_ax = pop16();

		set_DI(temp_di);
		set_SI(temp_si);
		set_BP(temp_bp);
		/* sp not set */
		set_BX(temp_bx);
		set_DX(temp_dx);
		set_CX(temp_cx);
		set_AX(temp_ax);
	}
}

/******************************************************************************
 *
 * Jcc - Jump if Condition is Met
 *
 ******************************************************************************/

static inline void
jcc(void)
{
	eip_new += tj;

	/* TODO: segment limit */

	if (!operand_size_32()) {
		eip_new &= 0xffff;
	}
}

static inline void
jo(void)
{
	if (get_OF()) {
		jcc();
	}
}

static inline void
jno(void)
{
	if (!get_OF()) {
		jcc();
	}
}

static inline void
jb(void)
{
	if (get_CF()) {
		jcc();
	}
}

static inline void
jnb(void)
{
	if (!get_CF()) {
		jcc();
	}
}

static inline void
jz(void)
{
	if (get_ZF()) {
		jcc();
	}
}

static inline void
jnz(void)
{
	if (!get_ZF()) {
		jcc();
	}
}

static inline void
jbe(void)
{
	if (get_CF() || get_ZF()) {
		jcc();
	}
}

static inline void
jnbe(void)
{
	if (!(get_CF() || get_ZF())) {
		jcc();
	}
}

static inline void
js(void)
{
	if (get_SF()) {
		jcc();
	}
}

static inline void
jns(void)
{
	if (!get_SF()) {
		jcc();
	}
}

static inline void
jp(void)
{
	if (get_PF()) {
		jcc();
	}
}

static inline void
jnp(void)
{
	if (!get_PF()) {
		jcc();
	}
}

static inline void
jl(void)
{
	if (get_SF() != get_OF()) {
		jcc();
	}
}

static inline void
jnl(void)
{
	if (get_SF() == get_OF()) {
		jcc();
	}
}

static inline void
jle(void)
{
	if (get_ZF() || (get_SF() != get_OF())) {
		jcc();
	}
}

static inline void
jnle(void)
{
	if (!(get_ZF() || (get_SF() != get_OF()))) {
		jcc();
	}
}

static inline void
jcxz(void)
{
	if (get_CX() == 0) {
		jcc();
	}
}

static inline void
jecxz(void)
{
	if (get_ECX() == 0) {
		jcc();
	}
}

/******************************************************************************
 *
 * SETcc - Set Byte on Condition
 *
 ******************************************************************************/

static inline void
seto(void)
{
	t0 = get_OF();
}

static inline void
setno(void)
{
	t0 = !get_OF();
}

static inline void
setb(void)
{
	t0 = get_CF();
}

static inline void
setnb(void)
{
	t0 = !get_CF();
}

static inline void
setz(void)
{
	t0 = get_ZF();
}

static inline void
setnz(void)
{
	t0 = !get_ZF();
}

static inline void
setbe(void)
{
	t0 = get_CF() || get_ZF();
}

static inline void
setnbe(void)
{
	t0 = !(get_CF() || get_ZF());
}

static inline void
sets(void)
{
	t0 = get_SF();
}

static inline void
setns(void)
{
	t0 = !get_SF();
}

static inline void
setp(void)
{
	t0 = get_PF();
}

static inline void
setnp(void)
{
	t0 = !get_PF();
}

static inline void
setl(void)
{
	t0 = get_SF() != get_OF();
}

static inline void
setnl(void)
{
	t0 = get_SF() == get_OF();
}

static inline void
setle(void)
{
	t0 = get_ZF() || (get_SF() != get_OF());
}

static inline void
setnle(void)
{
	t0 = !(get_ZF() || (get_SF() != get_OF()));
}

/******************************************************************************
 *
 * LAHF - Load Status Flags into AH Register
 *
 ******************************************************************************/

static inline void
lahf(void)
{
	set_AH(get_EFLAGS8());
}

/******************************************************************************
 *
 * SAHF - Store AH into Flags
 *
 ******************************************************************************/

static inline void
sahf(void)
{
	sset_EFLAGS8(get_AH());
	commit_EFLAGS8();
}

/******************************************************************************
 *
 * PUSHF/PUSHFD - Push EFLAGS Register onto the Stack
 *
 ******************************************************************************/

static inline void
pushf(void)
{
	if (virtual8086_mode()) {
		if (get_IOPL() < 3) {
			exception(EXCEPTION_GP, 0);
		}
	}

	if (operand_size_32()) {
		push32(get_EFLAGS32() & 0x00fcffff);
	} else {
		push16(get_EFLAGS16());
	}
}

/******************************************************************************
 *
 * POPF/POPFD - Pop Stack into EFLAGS Register
 *
 ******************************************************************************/

static inline void
popf(void)
{
	uint32_t mask;

	mask = 0;
	mask |= 1 << EFLAG_CF;
	mask |= 1 << EFLAG_PF;
	mask |= 1 << EFLAG_AF;
	mask |= 1 << EFLAG_ZF;
	mask |= 1 << EFLAG_SF;
	mask |= 1 << EFLAG_TF;
	mask |= 1 << EFLAG_DF;
	mask |= 1 << EFLAG_OF;
	mask |= 1 << EFLAG_NT;

	if (real_mode()) {
		mask |= 1 << EFLAG_IF;
		mask |= 1 << EFLAG_IOPL0;
		mask |= 1 << EFLAG_IOPL1;
	} else if (protected_mode()) {
		if (get_CPL() == 0) {
			mask |= 1 << EFLAG_IOPL0;
			mask |= 1 << EFLAG_IOPL1;
		}
		if (get_CPL() <= get_IOPL()) {
			mask |= 1 << EFLAG_IF;
		}
	} else { /* virtual8086_mode */
		if (get_IOPL() < 3) {
			exception(EXCEPTION_GP, 0);
		}
		mask |= 1 << EFLAG_IF;
	}

	if (operand_size_32()) {
		mask |= 1 << EFLAG_RF;
		mask |= 1 << EFLAG_AC;

		sset_EFLAGS32((pop32() & mask) | (get_EFLAGS32() & ~mask));
		commit_EFLAGS32();
	} else {
		sset_EFLAGS16((pop16() & mask) | (get_EFLAGS16() & ~mask));
		commit_EFLAGS16();
	}
}

/******************************************************************************
 *
 * CWD/CDQ - Convert Word to Doubleword/Convert Doubleword to Quadword
 *
 ******************************************************************************/

static inline void
cwd(void)
{
	if (get_AX() & 0x8000) {
		set_DX(0xffff);
	} else {
		set_DX(0x0000);
	}
}

static inline void
cdq(void)
{
	if (get_EAX() & 0x80000000) {
		set_EDX(0xffffffff);
	} else {
		set_EDX(0x00000000);
	}
}

/******************************************************************************
 *
 * CBW/CWDE - Convert Byte to Word/Convert Word to Doubleword
 *
 ******************************************************************************/

static inline void
cbw(void)
{
	set_AX((int8_t) get_AL());
}

static inline void
cwde(void)
{
	set_EAX((int16_t) get_AX());
}

/******************************************************************************
 *
 * AAA - ASCII Adjust After Addition
 *
 ******************************************************************************/

static inline void
aaa(void)
{
	/* TODO */
	ERROR_NYI();
}

/******************************************************************************
 *
 * AAS - ASCII Adjust AL After Substraction
 *
 ******************************************************************************/

static inline void
aas(void)
{
	/* TODO */
	ERROR_NYI();
}

/******************************************************************************
 *
 * DAA - Decimal Adjust AL after Addition
 *
 * Flags:	CF, AF, SF, ZF, PF, OF
 * Exceptions:	None
 *
 ******************************************************************************/

static inline void
daa(void)
{
	uint8_t al;
	bool af, cf;

	al = get_AL();
	af = get_AF();
	cf = get_CF();

	if (((al & 0x0f) > 9) || af) {
		cf = ((al > 0xf9) || cf);
		af = 1;
		al += 6;
	} else {
		af = 0;
	}

	if (((al & 0xf0) > 0x90) || cf) {
		al += 0x60;
		cf = 1;
	} else {
		cf = 0;
	}

	set_AL(al);
	sset_AF(af);
	sset_CF(cf);
	sset_PF(parity(al));
	sset_ZF(al == 0);
	sset_SF(al >= 0x80);
	sset_OF(0); /* undefined */
}

/******************************************************************************
 *
 * DAS - Decimal Adjust AL after Substraction
 *
 ******************************************************************************/

static inline void
das(void)
{
	/* TODO */
	ERROR_NYI();
}

/******************************************************************************
 *
 * AAM - ASCII Adjust AX After Multiply
 *
 ******************************************************************************/

static inline void
aam(void)
{
	uint8_t al;

	if (t1 == 0) {
		exception(EXCEPTION_DE, -1);
	}

	al = get_AL();
	set_AH(al / (uint8_t) t1);
	al = al % (uint8_t) t1;
	set_AL(al);

	sset_OF(0); /* Undefined */
	sset_AF(0); /* Undefined */
	sset_CF(0); /* Undefined */
	sset_SF(al >= 0x80);
	sset_ZF(al == 0);
	sset_PF(parity(al));
}

/******************************************************************************
 *
 * AAD - ASCII Adjust AX Before Division
 *
 ******************************************************************************/

static inline void
aad(void)
{
	uint16_t tmp;

	tmp = get_AH();
	tmp *= (uint8_t) t1;
	tmp += get_AL();
	tmp &= 0xff;

	set_AL(tmp);
	set_AH(0);

	sset_OF(0); /* Undefined */
	sset_AF(0); /* Undefined */
	sset_CF(0); /* Undefined */
	sset_SF(tmp >= 0x80);
	sset_ZF(tmp == 0);
	sset_PF(parity(tmp));
}

/******************************************************************************
 *
 * CPUID - CPU Identification
 *
 ******************************************************************************/

#if 0
#define CONFIG_CPU_FPU_SUPPORT          0
#define CONFIG_CPU_VME_SUPPORT          0
#define CONFIG_CPU_DE_SUPPORT           0
#define CONFIG_CPU_PSE_SUPPORT          0
#define CONFIG_CPU_TSC_SUPPORT          0
#define CONFIG_CPU_MSR_SUPPORT          0
#define CONFIG_CPU_PAE_SUPPORT          0
#define CONFIG_CPU_MCE_SUPPORT          0

#define CONFIG_CPU_CX8_SUPPORT          0
#define CONFIG_CPU_APIC_SUPPORT         0
/* Bit 10 is reserved. */
#define CONFIG_CPU_SEP_SUPPORT          0
#define CONFIG_CPU_MTRR_SUPPORT         0
#define CONFIG_CPU_PGE_SUPPORT          0
#define CONFIG_CPU_MCA_SUPPORT          0
#define CONFIG_CPU_CMOV_SUPPORT         0
#define CONFIG_CPU_PAT_SUPPORT          0
#define CONFIG_CPU_PSE36_SUPPORT        0
#define CONFIG_CPU_PSN_SUPPORT          0
#define CONFIG_CPU_CFLSH_SUPPORT        0

/* Bit 20 is reserved. */
#define CONFIG_CPU_DS_SUPPORT           0
#define CONFIG_CPU_ACPI_SUPPORT         0
#define CONFIG_CPU_MMX_SUPPORT          0
#define CONFIG_CPU_FXSR_SUPPORT         0
#define CONFIG_CPU_SSE_SUPPORT          0
#define CONFIG_CPU_SSE2_SUPPORT         0
#define CONFIG_CPU_SS_SUPPORT           0
#define CONFIG_CPU_HTT_SUPPORT          0
#define CONFIG_CPU_TM_SUPPORT           0

/* Bit 30 is reserved. */
#define CONFIG_CPU_PBE_SUPPORT          0

static inline void
cpuid(void)
{
	/* FIXME */
	switch (get_EAX()) {
	default:
	case 0:
		eax = 2;
		ebx = 0x756e6547; /* "GenuineIntel" */
		ecx = 0x6c65746e;
		edx = 0x49656e69;
                break;
        case 1:
                eax = 0x00000634; /* CPU Version */
                ebx = 0x00000000;
                ecx = 0x00000000;
                edx = CONFIG_CPU_FPU_SUPPORT << 0
                        | CONFIG_CPU_VME_SUPPORT << 1
                        | CONFIG_CPU_DE_SUPPORT << 2
                        | CONFIG_CPU_PSE_SUPPORT << 3
                        | CONFIG_CPU_TSC_SUPPORT << 4
                        | CONFIG_CPU_MSR_SUPPORT << 5
                        | CONFIG_CPU_PAE_SUPPORT << 6
                        | CONFIG_CPU_MCE_SUPPORT << 7
                        | CONFIG_CPU_CX8_SUPPORT << 8
                        | CONFIG_CPU_APIC_SUPPORT << 9
                        | CONFIG_CPU_SEP_SUPPORT << 11
                        | CONFIG_CPU_MTRR_SUPPORT << 12
                        | CONFIG_CPU_PGE_SUPPORT << 13
                        | CONFIG_CPU_MCA_SUPPORT << 14
                        | CONFIG_CPU_CMOV_SUPPORT << 15
                        | CONFIG_CPU_PAT_SUPPORT << 16
                        | CONFIG_CPU_PSE36_SUPPORT << 17
                        | CONFIG_CPU_PSN_SUPPORT << 18
                        | CONFIG_CPU_CFLSH_SUPPORT << 19
                        | CONFIG_CPU_DS_SUPPORT << 21
                        | CONFIG_CPU_ACPI_SUPPORT << 22
                        | CONFIG_CPU_MMX_SUPPORT << 23
                        | CONFIG_CPU_FXSR_SUPPORT << 24
                        | CONFIG_CPU_SSE_SUPPORT << 25
                        | CONFIG_CPU_SSE2_SUPPORT << 26
                        | CONFIG_CPU_SS_SUPPORT << 27
                        | CONFIG_CPU_HTT_SUPPORT << 28
                        | CONFIG_CPU_TM_SUPPORT << 29
                        | CONFIG_CPU_PBE_SUPPORT << 31;
		break;
	case 2:
		eax = 0x03020101;
		ebx = 0x00000000;
		ecx = 0x00000000;
		edx = 0x0c040843;
                break;
	}
}
#endif

/******************************************************************************
 *
 * WAIT/FWAIT - Wait
 *
 ******************************************************************************/

static inline void
wait(void)
{
	if (get_TS() && get_MP()) {
		exception(EXCEPTION_NM, -1);
	}
}

/******************************************************************************
 *
 * LOOP/LOOPcc - Loop According to ECX Counter
 *
 ******************************************************************************/

static inline void
loop(void)
{
	uint32_t count;

	if (address_size_32()) {
		count = get_ECX() - 1;
		set_ECX(count);
	} else {
		count = get_CX() - 1;
		set_CX(count);
	}

	if (count != 0) {
		eip_new += tj;
	}

	if (!operand_size_32()) {
		eip_new &= 0xffff;
	}
}

static inline void
loope(void)
{
	uint32_t count;

	if (address_size_32()) {
		count = get_ECX() - 1;
		set_ECX(count);
	} else {
		count = get_CX() - 1;
		set_CX(count);
	}

	if (count != 0 && get_ZF()) {
		eip_new += tj;
	}

	if (!operand_size_32()) {
		eip_new &= 0xffff;
	}
}

static inline void
loopne(void)
{
	uint32_t count;

	if (address_size_32()) {
		count = get_ECX() - 1;
		set_ECX(count);
	} else {
		count = get_CX() - 1;
		set_CX(count);
	}

	if (count != 0 && !get_ZF()) {
		eip_new += tj;
	}

	if (!operand_size_32()) {
		eip_new &= 0xffff;
	}
}

/******************************************************************************
 *
 * JMP - Jump
 *
 ******************************************************************************/

static inline void
jmp_far_virtual8086_mode(void)
{
	ERROR_NYI();
}

static inline void
jmp_far_protected_mode(void)
{
	sset_CS(t2);

	if (descriptor_sflag) {
		if (operand_size_32()) {
			eip_new = t1;
		} else {
			eip_new = t1 & 0xffff;
		}
	} else {
		ERROR_NYI(); /* TODO */
	}

	commit_CS();
}

static inline void
jmp_far_real_mode(void)
{
	sset_CS(t2);

	if (operand_size_32()) {
		eip_new = t1;
	} else {
		eip_new = t1 & 0xffff;
	}

	commit_CS();
}

static inline void
jmp_far(void)
{
	if (virtual8086_mode()) {
		jmp_far_virtual8086_mode();
	} else if (protected_mode()) {
		jmp_far_protected_mode();
	} else {
		jmp_far_real_mode();
	}
	instruction_cache_flush();
}

static inline void
jmp_near(void)
{
	eip_new += tj;

	if (!operand_size_32()) {
		eip_new &= 0xffff;
	}
}

static inline void
jmp_near_absolute(void)
{
	eip_new = t1;

	if (!operand_size_32()) {
		eip_new &= 0xffff;
	}
}

/******************************************************************************
 *
 * CALL - Call Procedure
 *
 ******************************************************************************/

static inline void
call_far_virtual8086_mode(void)
{
	ERROR_NYI();
}

static inline void
call_far_protected_mode(void)
{
	sset_CS(t2);

	if (descriptor_sflag) {
		if (operand_size_32()) {
			push32(get_CS());
			push32(eip_new);
			eip_new = t1;
		} else {
			push16(get_CS());
			push16(eip_new);
			eip_new = t1 & 0xffff;
		}

		commit_CS();
	} else {
		ERROR_NYI(); /* TODO */
	}
}

static inline void
call_far_real_mode(void)
{
	sset_CS(t2);

	if (operand_size_32()) {
		push32(get_CS());
		push32(eip_new);
	} else {
		push16(get_CS());
		push16(eip_new);
	}

	commit_CS();

	eip_new = t1;
}

static inline void
call_far(void)
{
	if (virtual8086_mode()) {
		call_far_virtual8086_mode();
	} else if (protected_mode()) {
		call_far_protected_mode();
	} else {
		call_far_real_mode();
	}
}

static inline void
call_near(void)
{
	if (operand_size_32()) {
		push32(eip_new);
	} else {
		push16(eip_new);
	}

	eip_new += tj;

	if (!operand_size_32()) {
		eip_new &= 0xffff;
	}
}

static inline void
call_near_absolute(void)
{
	if (operand_size_32()) {
		push32(eip_new);
	} else {
		push16(eip_new);
	}

	eip_new = t1;
}

/******************************************************************************
 *
 * RET - Return from Procedure
 *
 ******************************************************************************/

static inline void
ret_far_virtual8086_mode(void)
{
	ERROR_NYI();
}

static inline void
ret_far_protected_mode(void)
{
	/* TODO: check stack */

	if (operand_size_32()) {
		eip_new = pop32();
		sset_CS(pop32());
	} else {
		eip_new = pop16();
		sset_CS(pop16());
	}

	if ((selector & 0xfffc) == 0) {
		exception(EXCEPTION_GP, 0);
	}

	if (selector_rpl < get_CPL()) {
		exception(EXCEPTION_GP, 0);
	}

	if (selector_rpl != get_CPL()) {
		ERROR_NYI();
	}

	if (t1 != 0) {
		if (stack_size_32()) {
			set_ESP(get_ESP() + t1);
		} else {
			set_SP(get_SP() + t1);
		}
	}

	commit_CS();
}

static inline void
ret_far_real_mode(void)
{
	if (operand_size_32()) {
		eip_new = pop32();
		sset_CS(pop32());
	} else {
		eip_new = pop16();
		sset_CS(pop16());
	}

	if (t1 != 0) {
		if (stack_size_32()) {
			set_ESP(get_ESP() + t1);
		} else {
			set_SP(get_SP() + t1);
		}
	}

	commit_CS();
}

static inline void
ret_far(void)
{
	if (virtual8086_mode()) {
		ret_far_virtual8086_mode();
	} else if (protected_mode()) {
		ret_far_protected_mode();
	} else { /* real mode */
		ret_far_real_mode();
	}
}

static inline void
ret_near(void)
{
	if (operand_size_32()) {
		eip_new = pop32();
	} else {
		eip_new = pop16();
	}

	if (t1 != 0) {
		if (stack_size_32()) {
			set_ESP(get_ESP() + t1);
		} else {
			set_SP(get_SP() + t1);
		}
	}
}

/******************************************************************************
 *
 * LEAVE - High Level Procedure Exit
 *
 ******************************************************************************/

static inline void
leave(void)
{
	if (stack_size_32()) {
		set_ESP(get_EBP());
	} else {
		set_SP(get_BP());
	}

	if (operand_size_32()) {
		set_EBP(pop32());
	} else {
		set_BP(pop16());
	}
}

/******************************************************************************
 *
 * INT n/INTO/INT 3 - Call to Interrupt Procedure
 *
 ******************************************************************************/

static inline void
int3(void)
{
	exception_vector = 3;
	exception_error_code = -1;
	exception_is_interrupt = 1;
}

static inline void
intn(void)
{
	if (virtual8086_mode()) {
		if (get_IOPL() < 3) {
			exception(EXCEPTION_GP, 0);
		}
	}

	exception_vector = t1 & 0xff;
	exception_error_code = -1;
	exception_is_interrupt = 1;
}

static inline void
into(void)
{
	if (get_OF()) {
		exception_vector = 4;
		exception_error_code = -1;
		exception_is_interrupt = 1;
	}
}

/******************************************************************************
 *
 * ICEBP - Undocumented Opcode
 *
 ******************************************************************************/

static inline void
icebp(void)
{
	/* TODO: check IR (DR7) */
	exception_vector = t1;
	exception_error_code = -1;
	exception_is_interrupt = 1;
}

/******************************************************************************
 *
 * RCL/RCR/ROL/ROR - Rotate
 *
 ******************************************************************************/

static inline void
rclb(void)
{
	uint8_t value = t1;
	uint8_t count = (t2 & 31) % 9;
	bool cf;

	if (count == 0) {
		t0 = value;
		sset_CF(get_CF());
		sset_OF(get_OF());
		return;
	}

	if (count == 1) {
		t0 = (value << 1) | get_CF();
	} else {
		t0 = (value << count) | (get_CF() << (count - 1))
			| (value >> (9 - count));
	}

	cf = (value >> (8 - count)) & 1;

	t0 &= 0xff;

	sset_CF(cf);
	sset_OF(cf ^ (t0 >> 7));
}

static inline void
rclw(void)
{
	uint16_t value = t1;
	uint8_t count = (t2 & 31) % 17;
	bool cf;

	if (count == 0) {
		t0 = value;
		sset_CF(get_CF());
		sset_OF(get_OF());
		return;
	}

	if (count == 1) {
		t0 = (value << 1) | get_CF();
	} else if (count == 16) {
		t0 = (get_CF() << 15) | (value >> 1);
	} else { /* 2..15 */
		t0 = (value << count) | (get_CF() << (count - 1))
			| (value >> (17 - count));
	}

	cf = (value >> (16 - count)) & 1;

	t0 &= 0xffff;

	sset_CF(cf);
	sset_OF(cf ^ (t0 >> 15));
}

static inline void
rcld(void)
{
	uint32_t value = t1;
	uint8_t count = t2 & 31;
	bool cf;

	if (count == 0) {
		t0 = value;
		sset_CF(get_CF());
		sset_OF(get_OF());
		return;
	}

	if (count == 1) {
		t0 = (value << 1) | get_CF();
	} else {
		t0 = (value << count) | (get_CF() << (count - 1))
			| (value >> (33 - count));
	}

	cf = (value >> (32 - count)) & 1;

	sset_CF(cf);
	sset_OF(cf ^ (t0 >> 31));
}

static inline void
rclv(void)
{
	if (operand_size_32()) {
		rcld();
	} else {
		rclw();
	}
}

/******************************************************************************/

static inline void
rcrb(void)
{
	uint8_t value = t1;
	uint8_t count = (t2 & 31) % 9;

	if (count == 0) {
		t0 = value;
		sset_CF(get_CF());
		sset_OF(get_OF());
		return;
	}

	t0 = (value >> count) | (get_CF() << (8 - count))
		| (value << (9 - count));

	t0 &= 0xff;

	sset_CF((value >> (count - 1)) & 1);
	sset_OF((((t0 << 1) ^ t0) & 0x80) > 0);
}

static inline void
rcrw(void)
{
	uint16_t value = t1;
	uint8_t count = (t2 & 31) % 17;

	if (count == 0) {
		t0 = value;
		sset_CF(get_CF());
		sset_OF(get_OF());
		return;
	}

	t0 = (value >> count) | (get_CF() << (16 - count))
		| (value << (17 - count));

	t0 &= 0xffff;

	sset_CF((value >> (count - 1)) & 1);
	sset_OF((((t0 << 1) ^ t0) & 0x8000) > 0);
}

static inline void
rcrd(void)
{
	uint32_t value = t1;
	uint8_t count = t2 & 31;

	if (count == 0) {
		t0 = value;
		sset_CF(get_CF());
		sset_OF(get_OF());
		return;
	}

	if (count == 1) {
		t0 = (value >> 1) | (get_CF() << 31);
	} else {
		t0 = (value >> count) | (get_CF() << (32 - count))
			| (value << (33 - count));
	}

	sset_CF((value >> (count - 1)) & 1);
	sset_OF((((t0 << 1) ^ t0) & 0x80000000) > 0);
}

static inline void
rcrv(void)
{
	if (operand_size_32()) {
		rcrd();
	} else {
		rcrw();
	}
}

/******************************************************************************/

static inline void
rolb(void)
{
	uint8_t value = t1;
	uint8_t count = t2 & 7;
	bool cf;

	t0 = (value << count) | (value >> (8 - count));

	cf = t0 & 1;

	sset_CF(cf);
	sset_OF(cf ^ (t0 >> 7)); /* Undefined for multi-bit rotates */
}

static inline void
rolw(void)
{
	uint16_t value = t1;
	uint8_t count = t2 & 15;
	bool cf;

	t0 = (value << count) | (value >> (16 - count));

	cf = t0 & 1;

	sset_CF(cf);
	sset_OF(cf ^ (t0 >> 15)); /* Undefined for multi-bit rotates */
}

static inline void
rold(void)
{
	uint32_t value = t1;
	uint8_t count = t2 & 31;
	bool cf;

	t0 = (value << count) | (value >> (32 - count));

	cf = t0 & 1;

	sset_CF(cf);
	sset_OF(cf ^ (t0 >> 31)); /* Undefined for multi-bit rotates */
}

static inline void
rolv(void)
{
	if (operand_size_32()) {
		rold();
	} else {
		rolw();
	}
}

/******************************************************************************/

static inline void
rorb(void)
{
	uint8_t value = t1;
	uint8_t count = t2 & 7;
	bool b6, b7;

	t0 = (value >> count) | (value << (8 - count));

	b7 = (t0 & 0x80) != 0;
	b6 = (t0 & 0x40) != 0;

	sset_CF(b7);
	sset_OF(b7 ^ b6);
}

static inline void
rorw(void)
{
	uint16_t value = t1;
	uint8_t count = t2 & 15;
	bool b14, b15;

	t0 = (value >> count) | (value << (16 - count));

	b15 = (t0 & 0x8000) != 0;
	b14 = (t0 & 0x4000) != 0;

	sset_CF(b15);
	sset_OF(b15 ^ b14);
}

static inline void
rord(void)
{
	uint32_t value = t1;
	uint8_t count = t2 * 31;
	bool b30, b31;

	t0 = (value >> count) | (value << (32 - count));

	b31 = (t0 & 0x80000000) != 0;
	b30 = (t0 & 0x40000000) != 0;

	sset_CF(b31);
	sset_OF(b31 ^ b30);
}

static inline void
rorv(void)
{
	if (operand_size_32()) {
		rord();
	} else {
		rorw();
	}
}

/******************************************************************************
 *
 * SAL/SAR/SHL/SHR - Shift
 *
 ******************************************************************************/

static inline void
sarb(void)
{
	uint8_t count = t2 & 31;
	uint8_t value = t1;
	uint8_t result;

	if (count == 0)
	{
		t0 = value;
		sset_OF(get_OF());
		sset_SF(get_SF());
		sset_ZF(get_ZF());
		sset_AF(get_AF());
		sset_PF(get_PF());
		sset_CF(get_CF());
		return;
	}

	if (count < 8) {
		if (value & 0x80) {
			result = (value >> count) | (0xff << (8 - count));
		} else {
			result = value >> count;
		}
		sset_CF((value >> (count - 1)) & 1);
	} else {
		if (value & 0x80) {
			result = 0xff;
		} else {
			result = 0;
		}
		sset_CF((value & 0x80) > 0);
	}
	sset_AF(0);
	sset_OF(0);

	t0 = result;

	sset_SF(result >= 0x80);
	sset_ZF(result == 0);
	sset_PF(parity(result));
}

static inline void
sarw(void)
{
	uint8_t count = t2 & 31;
	uint16_t value = t1;
	uint16_t result;

	if (count == 0) {
		t0 = value;
		sset_OF(get_OF());
		sset_SF(get_SF());
		sset_ZF(get_ZF());
		sset_AF(get_AF());
		sset_PF(get_PF());
		sset_CF(get_CF());
		return;
	}

	if (count < 16) {
		if (value & 0x8000) {
			result = (value >> count) | (0xffff << (16 - count));
		} else {
			result = value >> count;
		}
		sset_CF((value >> (count - 1)) & 1);
	} else {
		if (value & 0x8000) {
			result = 0xffff;
		} else {
			result = 0;
		}
		sset_CF((value & 0x8000) > 0);
	}
	sset_AF(0);
	sset_OF(0);

	t0 = result;

	sset_SF(result >= 0x8000);
	sset_ZF(result == 0);
	sset_PF(parity(result));
}

static inline void
sard(void)
{
	uint8_t count = t2 & 31;
	uint32_t value = t1;
	uint32_t result;

	if (count == 0) {
		t0 = value;
		sset_OF(get_OF());
		sset_SF(get_SF());
		sset_ZF(get_ZF());
		sset_AF(get_AF());
		sset_PF(get_PF());
		sset_CF(get_CF());
		return;
	}

	if (value & 0x80000000) {
		result = (value >> count) | (0xffffffff << (32 - count));
	} else {
		result = value >> count;
	}
	sset_CF((value >> (count - 1)) & 1);
	sset_AF(0);
	sset_OF(0);

	t0 = result;

	sset_SF(result >= 0x80000000);
	sset_ZF(result == 0);
	sset_PF(parity(result));
}

static inline void
sarv(void)
{
	if (operand_size_32()) {
		sard();
	} else {
		sarw();
	}
}

/******************************************************************************/

static inline void
shlb(void)
{
	uint8_t count = t2 & 31;
	uint8_t value = t1;
	uint8_t result;

	if (count == 0) {
		t0 = value;
		sset_OF(get_OF());
		sset_SF(get_SF());
		sset_ZF(get_ZF());
		sset_AF(get_AF());
		sset_PF(get_PF());
		sset_CF(get_CF());
		return;
	}

	result = value << count;

	if (count <= 8) {
		sset_CF((value >> (8 - count)) & 1);
	} else {
		sset_CF(0);
	}

	if (count == 1) {
		sset_OF(((value ^ result) & 0x80) > 0);
	} else {
		sset_OF((((value << (count - 1)) ^ result) & 0x80) > 0);
	}

	sset_AF(0); /* undefined */

	t0 = result;

	sset_SF(result >= 0x80);
	sset_ZF(result == 0);
	sset_PF(parity(result));
}

static inline void
shlw(void)
{
	uint8_t count = t2 & 31;
	uint16_t value = t1;
	uint16_t result;

	if (count == 0) {
		t0 = value;
		sset_OF(get_OF());
		sset_SF(get_SF());
		sset_ZF(get_ZF());
		sset_AF(get_AF());
		sset_PF(get_PF());
		sset_CF(get_CF());
		return;
	}

	result = value << count;

	if (count <= 16) {
		sset_CF((value >> (16 - count)) & 1);
	} else {
		sset_CF(0);
	}

	if (count == 1) {
		sset_OF(((value ^ result) & 0x8000) > 0);
	} else {
		sset_OF((((value << (count - 1)) ^ result) & 0x8000) > 0);
	}

	sset_AF(0); /* undefined */

	t0 = result;

	sset_SF(result >= 0x8000);
	sset_ZF(result == 0);
	sset_PF(parity(result));
}

static inline void
shld(void)
{
	uint8_t count = t2 & 31;
	uint32_t value = t1;
	uint32_t result;

	if (count == 0) {
		t0 = value;
		sset_OF(get_OF());
		sset_SF(get_SF());
		sset_ZF(get_ZF());
		sset_AF(get_AF());
		sset_PF(get_PF());
		sset_CF(get_CF());
		return;
	}

	result = value << count;

	sset_CF((value >> (32 - count)) & 1);

	if (count == 1) {
		sset_OF(((value ^ result) & 0x80000000) > 0);
	} else {
		sset_OF((((value << (count - 1)) ^ result) & 0x80000000) > 0);
	}

	sset_AF(0); /* undefined */

	t0 = result;

	sset_SF(result >= 0x80000000);
	sset_ZF(result == 0);
	sset_PF(parity(result));
}

static inline void
shlv(void)
{
	if (operand_size_32()) {
		shld();
	} else {
		shlw();
	}
}

/******************************************************************************/

static inline void
shrb(void)
{
	uint8_t value = t1;
	uint8_t count = t2 & 31;

	if (count == 0) {
		t0 = value;
		sset_OF(get_OF());
		sset_SF(get_SF());
		sset_ZF(get_ZF());
		sset_AF(get_AF());
		sset_PF(get_PF());
		sset_CF(get_CF());
		return;
	}

	t0 = value >> count;

	sset_CF((value >> (count - 1)) & 1);

	if (count == 1) {
		sset_OF(value >= 0x80);
	} else {
		sset_OF(0); /* undocumented, but hardware really does it */
	}

	sset_SF(t0 >= 0x80);
	sset_ZF(t0 == 0);
	sset_PF(parity(t0));
	sset_AF(0); /* Undefined */
}

static inline void
shrw(void)
{
	uint16_t value = t1;
	uint8_t count = t2 & 31;

	if (count == 0) {
		t0 = value;
		sset_OF(get_OF());
		sset_SF(get_SF());
		sset_ZF(get_ZF());
		sset_AF(get_AF());
		sset_PF(get_PF());
		sset_CF(get_CF());
		return;
	}

	t0 = value >> count;

	sset_CF((value >> (count - 1)) & 1);

	if (count == 1) {
		sset_OF(value >= 0x8000);
	} else {
		sset_OF(0); /* undocumented, but hardware really does it */
	}

	sset_SF(t0 >= 0x8000);
	sset_ZF(t0 == 0);
	sset_PF(parity(t0));
	sset_AF(0); /* Undefined */
}

static inline void
shrd(void)
{
	uint32_t value = t1;
	uint8_t count = t2 & 31;

	if (count == 0) {
		t0 = value;
		sset_OF(get_OF());
		sset_SF(get_SF());
		sset_ZF(get_ZF());
		sset_AF(get_AF());
		sset_PF(get_PF());
		sset_CF(get_CF());
		return;
	}

	t0 = value >> count;

	sset_CF((value >> (count - 1)) & 1);

	if (count == 1) {
		sset_OF(value >= 80000000);
	} else {
		sset_OF(0); /* undocumented, but hardware really does it */
	}

	sset_SF(t0 >= 0x80000000);
	sset_ZF(t0 == 0);
	sset_PF(parity(t0));
	sset_AF(0); /* Undefined */
}

static inline void
shrv(void)
{
	if (operand_size_32()) {
		shrd();
	} else {
		shrw();
	}
}

/******************************************************************************
 *
 * SHRD - Double Precision Shift Right
 *
 ******************************************************************************/

static inline void
shrdw(void)
{
	unsigned count = t3 & 31;

	if (!count) {
		t0 = t1;
		sset_OF(get_OF());
		sset_SF(get_SF());
		sset_ZF(get_ZF());
		sset_AF(get_AF());
		sset_PF(get_PF());
		sset_CF(get_CF());
		return;
	}

	t0 = ((t2 << 16) | t1) >> count;

	if (count > 16) {
		t0 |= t1 << (32 - count);
	}

	t0 &= 0xffff;

	sset_CF((t1 >> (count - 1)) & 1);
	sset_OF((((t0 << 1) ^ t0) & 0x8000) > 0);
	sset_SF(t0 >= 0x8000);
	sset_PF(parity(t0));
	sset_AF(0);
	sset_ZF(t0 == 0);
}

static inline void
shrdd(void)
{
	unsigned count = t3 & 31;

	if (!count) {
		t0 = t1;
		sset_OF(get_OF());
		sset_SF(get_SF());
		sset_ZF(get_ZF());
		sset_AF(get_AF());
		sset_PF(get_PF());
		sset_CF(get_CF());
		return;
	}

	t0 = (t2 << (32 - count)) | (t1 >> count);

	sset_CF((t1 >> (count - 1)) & 1);
	sset_OF((((t0 << 1) ^ t0) & 0x80000000) > 0);
	sset_SF(t0 >= 0x80000000);
	sset_PF(parity(t0));
	sset_AF(0);
	sset_ZF(t0 == 0);
}

static inline void
shrdv(void)
{
	if (operand_size_32()) {
		shrdd();
	} else {
		shrdw();
	}
}

/******************************************************************************
 *
 * SHLD - Double Precision Shift Left
 *
 ******************************************************************************/

static inline void
shldw(void)
{
	unsigned count = t3 & 31;

	if (!count) {
		t0 = t1;
		sset_OF(get_OF());
		sset_SF(get_SF());
		sset_ZF(get_ZF());
		sset_AF(get_AF());
		sset_PF(get_PF());
		sset_CF(get_CF());
	}

	t0 = ((t1 << 16) | t2) << count;

	if (count > 16) {
		t0 |= t1 << (count - 16);
	}

	t0 >>= 16;

	if (count <= 16) {
		sset_CF((t1 >> (16 - count)) & 1);
	} else {
		sset_CF(0);
	}

	sset_AF(0);
	sset_ZF(t0 == 0);
	sset_SF(t0 >= 0x8000);
	sset_PF(parity(t0));

	if (count == 1) {
		sset_OF(((t1 ^ t0) & 0x8000) > 0);
	} else {
		sset_OF((((t1 << (count - 1)) ^ t0) & 0x8000) > 0);
	}
}

static inline void
shldd(void)
{
	unsigned count = t3 & 31;

	if (!count) {
		t0 = t1;
		sset_OF(get_OF());
		sset_SF(get_SF());
		sset_ZF(get_ZF());
		sset_AF(get_AF());
		sset_PF(get_PF());
		sset_CF(get_CF());
		return;
	}

	t0 = (t1 << count) | (t2 >> (32 - count));

	sset_CF((t1 >> (32 - count)) & 1);
	sset_AF(0);
	sset_ZF(t0 == 0);
	sset_SF(t0 >= 0x80000000);
	sset_PF(parity(t0));

	if (count == 1) {
		sset_OF(((t1 ^ t0) & 0x80000000) > 0);
	} else {
		sset_OF((((t1 << (count - 1)) ^ t0) & 0x80000000) > 0);
	}
}

static inline void
shldv(void)
{
	if (operand_size_32()) {
		shldd();
	} else {
		shldw();
	}
}

/******************************************************************************
 *
 * XLAT/XLATB - Table Look-up Translation
 *
 ******************************************************************************/

static inline void
xlat(void)
{
	uint32_t offset;

	if (address_size_32()) {
		offset = get_EBX() + get_AL();
	} else {
		offset = get_BX() + get_AL();
	}

	set_AL(mrb_seg(offset, get_segment_overwritten(SEGMENT_DS)));
}

/******************************************************************************
 *
 * LMSW - Load Machine Status Word
 *
 ******************************************************************************/

static inline void
lmsw(void)
{
	if (!real_mode() && get_CPL() != 0) {
		exception(EXCEPTION_GP, 0);
	}

	if (!get_PE()) {
		set_PE((t1 >> CR0_PE) & 1);
	}

	set_MP((t1 >> CR0_MP) & 1);
	set_EM((t1 >> CR0_EM) & 1);
	set_TS((t1 >> CR0_TS) & 1);
}

/******************************************************************************
 *
 * SMSW - Store Machine Status Word
 *
 ******************************************************************************/

static inline void
smsw(void)
{
	t0 = get_CR0() & 0xffff;
}

/******************************************************************************
 *
 * HLT - Halt
 *
 ******************************************************************************/

static inline void
hlt(void)
{
	halt_state = 1;
}

/******************************************************************************
 *
 * STR - Store Task Register
 *
 ******************************************************************************/

static inline void
str(void)
{
	if (real_mode() || virtual8086_mode()) {
		exception(EXCEPTION_UD, -1);
	}

	t0 = tr_selector;
}

/******************************************************************************
 *
 * SLDT - Store Local Descriptor Table Register
 *
 ******************************************************************************/

static inline void
sldt(void)
{
	if (real_mode() || virtual8086_mode()) {
		exception(EXCEPTION_UD, -1);
	}

	t0 = ldtr_selector;
}

/******************************************************************************
 *
 * LTR - Load Task Register
 *
 ******************************************************************************/

static inline void
ltr(void)
{
	uint32_t addr, dw1, dw2;

	if (real_mode() || virtual8086_mode()) {
		exception(EXCEPTION_UD, -1);
	}

	if (get_CPL() != 0) {
		exception(EXCEPTION_GP, 0);
	}

	sset_selector(t1);

	if (selector_index == 0) {
		exception(EXCEPTION_GP, 0);
	}

	if (selector_ti != 0) {
		exception(EXCEPTION_GP, selector & 0xfffc);
	}

	addr = get_descriptor_address();

	dw1 = mrd(addr, get_CPL());
	dw2 = mrd(addr + 4, get_CPL());

	sset_descriptor(dw1, dw2);

	if (descriptor_sflag) {
		exception(EXCEPTION_GP, selector & 0xfffc);
	}

	if ((descriptor_type != SEGMENT_32BIT_AVAIL_TSS)
	 && (descriptor_type != SEGMENT_16BIT_AVAIL_TSS)) {
		exception(EXCEPTION_GP, selector & 0xfffc);
	}

	if (!descriptor_pflag) {
		exception(EXCEPTION_NP, selector & 0xfffc);
	}

	tr_selector = selector;
	tr_type = descriptor_type;
	tr_system_limit = descriptor_system_limit;
	tr_system_base = descriptor_system_base;

	/* set busy bit */
	mwd(addr + 4, dw2 | 0x200, get_CPL());
}

/******************************************************************************
 *
 * LLDT - Load Local Descriptor Table Register
 *
 ******************************************************************************/

static inline void
lldt(void)
{
	uint32_t addr;

	if (real_mode() || virtual8086_mode()) {
		exception(EXCEPTION_UD, -1);
	}

	if (get_CPL() != 0) {
		exception(EXCEPTION_GP, 0);
	}

	sset_selector(t1);

	if (selector_index == 0) {
		ldtr_selector = 0;
		ldtr_system_limit = 0;
		ldtr_system_base = 0;
		return;
	}

	if (selector_ti != 0) {
		exception(EXCEPTION_GP, selector & 0xfffc);
	}

	addr = get_descriptor_address();

	sset_descriptor(mrd(addr, get_CPL()), mrd(addr + 4, get_CPL()));

	if ((descriptor_sflag)
	 || (descriptor_type != SEGMENT_LDT)) {
		exception(EXCEPTION_GP, selector & 0xfffc);
	}

	if (!descriptor_pflag) {
		exception(EXCEPTION_NP, selector & 0xfffc);
	}

	ldtr_selector = selector;
	ldtr_system_limit = descriptor_system_limit;
	ldtr_system_base = descriptor_system_base;
}

/******************************************************************************
 *
 * LGDT/LIDT - Load Global/Interrupt Descriptor Table Register
 *
 ******************************************************************************/

static inline void
lgdt(void)
{
	uint16_t limit;
	uint32_t base;

	if (virtual8086_mode()) {
		exception(EXCEPTION_GP, 0);
	}

	if (protected_mode() && get_CPL() != 0) {
		exception(EXCEPTION_GP, 0);
	}

	limit = t1;
	base = t2;

	if (!operand_size_32()) {
		base &= 0x00ffffff;
	}

	gdtr_limit = limit;
	gdtr_base = base;
}

static inline void
lidt(void)
{
	uint16_t limit;
	uint32_t base;

	if (virtual8086_mode()) {
		exception(EXCEPTION_GP, 0);
	}

	if (protected_mode() && get_CPL() != 0) {
		exception(EXCEPTION_GP, 0);
	}

	limit = t1;
	base = t2;

	if (!operand_size_32()) {
		base &= 0x00ffffff;
	}

	idtr_limit = limit;
	idtr_base = base;
}

/******************************************************************************
 *
 * BSR - Bit Scan Reverse
 *
 ******************************************************************************/

static inline void
sset_flags_for_bsr(void)
{
	sset_CF(0);		/* Undefined */
	sset_OF(0);		/* Undefined */
	sset_AF(0);		/* Undefined */
	sset_PF(parity(t0));	/* Undefined */
}

static inline void
bsrw(void)
{
	sset_ZF(t2 == 0);

	if (t2 == 0) {
		t0 = t1; /* Undefined */
	} else {
		t0 = 15;
		while ((t2 & 0x8000) == 0) {
			t0--;
			t2 <<= 1;
		}
	}

	sset_flags_for_bsr();
	sset_SF(t0 >= 0x8000); /* Undefined */
}

static inline void
bsrd(void)
{
	sset_ZF(t2 == 0);

	if (t2 == 0) {
		t0 = t1; /* Undefined */
	} else {
		t0 = 31;
		while ((t2 & 0x80000000) == 0) {
			t0--;
			t2 <<= 1;
		}
	}

	sset_flags_for_bsr();
	sset_SF(t0 >= 0x80000000); /* Undefined */
}

static inline void
bsrv(void)
{
	if (operand_size_32()) {
		bsrd();
	} else {
		bsrw();
	}
}

/******************************************************************************
 *
 * BSF - Bit Scan Forward
 *
 ******************************************************************************/

static inline void
sset_flags_for_bsf(void)
{
	sset_CF(0);		/* Undefined */
	sset_OF(0);		/* Undefined */
	sset_AF(0);		/* Undefined */
	sset_PF(parity(t0));	/* Undefined */
}

static inline void
bsfw(void)
{
	sset_ZF(t2 == 0);

	if (t2 == 0) {
		t0 = t1; /* Undefined */
	} else {
		t0 = 0;
		while ((t2 & 1) == 0) {
			t0++;
			t2 >>= 1;
		}
	}

	sset_flags_for_bsf();
	sset_SF(t0 >= 0x8000); /* Undefined */
}

static inline void
bsfd(void)
{
	sset_ZF(t2 == 0);

	if (t2 == 0) {
		t0 = t1; /* Undefined */
		return;
	} else {
		t0 = 0;
		while ((t2 & 1) == 0) {
			t0++;
			t2 >>= 1;
		}
	}

	sset_flags_for_bsf();
	sset_SF(t0 >= 0x80000000); /* Undefined */
}

static inline void
bsfv(void)
{
	if (operand_size_32()) {
		bsfd();
	} else {
		bsfw();
	}
}

/******************************************************************************
 *
 * BT - Bit Test
 *
 ******************************************************************************/

static inline void
btww(void)
{
	sset_CF((t1 >> (t2 & 15)) & 1);
}

static inline void
btdd(void)
{
	sset_CF((t1 >> (t2 & 31)) & 1);
}

static inline void
btvv(void)
{
	if (operand_size_32()) {
		btdd();
	} else {
		btww();
	}
}

/******************************************************************************
 *
 * BTS - Bit Test and Set
 *
 ******************************************************************************/

static inline void
btsww(void)
{
	sset_CF((t1 >> (t2 & 15)) & 1);
	t0 = t1 | (1 << (t2 & 15));

	/* OF, SF, ZF, AF, PF undefined */
	sset_OF(get_OF());
	sset_SF(get_SF());
	sset_ZF(get_ZF());
	sset_AF(get_AF());
	sset_PF(get_PF());
}

static inline void
btsdd(void)
{
	sset_CF((t1 >> (t2 & 31)) & 1);
	t0 = t1 | (1 << (t2 & 31));

	/* OF, SF, ZF, AF, PF undefined */
	sset_OF(get_OF());
	sset_SF(get_SF());
	sset_ZF(get_ZF());
	sset_AF(get_AF());
	sset_PF(get_PF());
}

static inline void
btsvv(void)
{
	if (operand_size_32()) {
		btsdd();
	} else {
		btsww();
	}
}

static inline void
btsvb(void)
{
	if (operand_size_32()) {
		btsdd();
	} else {
		btsww();
	}
}

/******************************************************************************
 *
 * BTR - Bit Test and Reset
 *
 ******************************************************************************/

static inline void
btrww(void)
{
	sset_CF((t1 >> (t2 & 15)) & 1);
	t0 = t1 & ~(1 << (t2 & 15));

	/* OF, SF, ZF, AF, PF undefined */
	sset_OF(get_OF());
	sset_SF(get_SF());
	sset_ZF(get_ZF());
	sset_AF(get_AF());
	sset_PF(get_PF());
}

static inline void
btrdd(void)
{
	sset_CF((t1 >> (t2 & 31)) & 1);
	t0 = t1 & ~(1 << (t2 & 31));

	/* OF, SF, ZF, AF, PF undefined */
	sset_OF(get_OF());
	sset_SF(get_SF());
	sset_ZF(get_ZF());
	sset_AF(get_AF());
	sset_PF(get_PF());
}

static inline void
btrvv(void)
{
	if (operand_size_32()) {
		btrdd();
	} else {
		btrww();
	}
}

static inline void
btrvb(void)
{
	if (operand_size_32()) {
		btrdd();
	} else {
		btrww();
	}
}

/******************************************************************************
 *
 * BTC - Bit Test and Complement
 *
 ******************************************************************************/

static inline void
btcww(void)
{
	sset_CF((t1 >> (t2 & 15)) & 1);
	t0 = t1 ^ (1 << (t2 & 15));

	/* OF, SF, ZF, AF, PF undefined */
	sset_OF(get_OF());
	sset_SF(get_SF());
	sset_ZF(get_ZF());
	sset_AF(get_AF());
	sset_PF(get_PF());
}

static inline void
btcdd(void)
{
	sset_CF((t1 >> (t2 & 31)) & 1);
	t0 = t1 ^ (1 << (t2 & 31));

	/* OF, SF, ZF, AF, PF undefined */
	sset_OF(get_OF());
	sset_SF(get_SF());
	sset_ZF(get_ZF());
	sset_AF(get_AF());
	sset_PF(get_PF());
}

static inline void
btcvv(void)
{
	if (operand_size_32()) {
		btcdd();
	} else {
		btcww();
	}
}

static inline void
btcvb(void)
{
	if (operand_size_32()) {
		btcdd();
	} else {
		btcww();
	}
}

/******************************************************************************
 *
 * CMC - Complement Carry Flag
 *
 ******************************************************************************/

static inline void
cmc(void)
{
	sset_CF(!get_CF());
	commit_CF();
}

/******************************************************************************
 *
 * STD - Set Direction Flag
 *
 ******************************************************************************/

static inline void
std(void)
{
	sset_DF(1);
}

/******************************************************************************
 *
 * CLD - Clear Direction Flag
 *
 ******************************************************************************/

static inline void
cld(void)
{
	sset_DF(0);
}

/******************************************************************************
 *
 * STC - Set Carry Flag
 *
 ******************************************************************************/

static inline void
stc(void)
{
	sset_CF(1);
}

/******************************************************************************
 *
 * CLC - Clear Carry Flag
 *
 ******************************************************************************/

static inline void
clc(void)
{
	sset_CF(0);
}

/******************************************************************************
 *
 * CLTS - Clear Task-Switched Flag in CR0
 *
 ******************************************************************************/

static inline void
clts(void)
{
	if (!real_mode() && get_CPL() != 0) {
		exception(EXCEPTION_GP, 0);
	}

	set_TS(0);
}

/******************************************************************************
 *
 * STI - Set Interrupt Flag
 *
 ******************************************************************************/

static inline void
sti(void)
{
	if (protected_mode()) {
		if (get_IOPL() < get_CPL()) {
			exception(EXCEPTION_GP, 0);
		}
	} else if (virtual8086_mode()) {
		if (get_IOPL() != 3) {
			exception(EXCEPTION_GP, 0);
		}
	}

	sset_IF(1);
}

/******************************************************************************
 *
 * CLI - Clear Interrupt Flag
 *
 ******************************************************************************/

static inline void
cli(void)
{
	if (protected_mode()) {
		if (get_IOPL() < get_CPL()) {
			exception(EXCEPTION_GP, 0);
		}
	} else if (virtual8086_mode()) {
		if (get_IOPL() != 3) {
			exception(EXCEPTION_GP, 0);
		}
	}

	sset_IF(0);
}

/******************************************************************************
 *
 * IRET/IRETD - Interrupt Return
 *
 ******************************************************************************/

static inline bool
can_pop(uint32_t bytes)
{
	if ((ss_type >> 2) & 1) { /* expand down data segment */
		if (stack_size_32()) {
			if (((0xffffffff - get_ESP()) + 1) >= bytes) {
				return 1;
			} else {
				return 0;
			}
		} else {
			if (((0xffff - get_SP()) + 1) >= bytes ) {
				return 1;
			} else {
				return 0;
			}
		}
	} else {
		if (stack_size_32()) {
			if (((ss_segment_limit - get_ESP()) + 1) >= bytes) {
				return 1;
			} else {
				return 0;
			}
		} else {
			if (((ss_segment_limit - get_SP()) + 1) >= bytes) {
				return 1;
			} else {
				return 0;
			}
		}
	}
}

static inline void
iret_virtual8086_mode(void)
{
	ERROR_NYI();
}

static inline void
iret_to_outer_privilege_level(void)
{
	uint32_t addr;
	uint16_t temp_cs_selector;
	uint8_t	 temp_cs_selector_rpl;
	uint8_t  temp_cs_type;
	bool	 temp_cs_sflag;
	uint8_t  temp_cs_dpl;
	bool	 temp_cs_pflag;
	uint32_t temp_cs_segment_limit;
	uint32_t temp_cs_segment_base;
	bool	 temp_cs_segment_dflag;
	bool	 temp_cs_segment_avlflag;
	uint32_t esp_new;

	temp_cs_selector		= selector;
	temp_cs_selector_rpl		= selector_rpl;
	temp_cs_type			= descriptor_type;
	temp_cs_sflag			= descriptor_sflag;
	temp_cs_dpl			= descriptor_dpl;
	temp_cs_pflag			= descriptor_pflag;
	temp_cs_segment_limit		= descriptor_segment_limit;
	temp_cs_segment_base		= descriptor_segment_base;
	temp_cs_segment_dflag		= descriptor_segment_dflag;
	temp_cs_segment_avlflag		= descriptor_segment_avlflag;

	if (operand_size_32()) {
		esp_new = pop32();
		sset_selector(pop32());
	} else {
		esp_new = pop16();
		sset_selector(pop16());
	}

	if (selector_index == 0) {
		exception(EXCEPTION_GP, 0);
	}

	if (selector_rpl != temp_cs_selector_rpl) {
		exception(EXCEPTION_GP, selector & 0xfffc);
	}

	addr = get_descriptor_address();

	sset_descriptor(mrd(addr, get_CPL()), mrd(addr + 4, get_CPL()));

	if ((!descriptor_sflag) /* no data/code segment */
	 || ((descriptor_type >> 3) & 1) /* code segment */
	 || (!((descriptor_type >> 1) & 1))) { /* not writeable */
		exception(EXCEPTION_GP, selector & 0xfffc);
	}

	if (descriptor_dpl != temp_cs_selector_rpl) {
		exception(EXCEPTION_GP, selector & 0xfffc);
	}

	if (!descriptor_pflag) {
		exception(EXCEPTION_NP, selector & 0xfffc);
	}

	if (get_CPL() <= get_IOPL()) {
		commit_IF();
	}

	if (temp_cs_segment_dflag) { /* new operand size == 32 */
		commit_RF();

		if (get_CPL() == 0) {
			commit_IOPL0();
			commit_IOPL1();
		}
	} else {
		if (get_CPL() == 0) {
			commit_IOPL0();
			commit_IOPL1();
		}
	}

	commit_CF();
	commit_PF();
	commit_AF();
	commit_ZF();
	commit_SF();
	commit_TF();
	commit_OF();
	commit_DF();
	commit_NT();

	cs_selector		= temp_cs_selector;
	cs_type			= temp_cs_type;
	cs_sflag		= temp_cs_sflag;
	cs_dpl			= temp_cs_dpl;
	cs_pflag		= temp_cs_pflag;
	cs_segment_limit	= temp_cs_segment_limit;
	cs_segment_base		= temp_cs_segment_base;
	cs_segment_dflag	= temp_cs_segment_dflag;
	cs_segment_avlflag	= temp_cs_segment_avlflag;

	commit_SS();

	if (stack_size_32()) {
		set_ESP(esp_new);
	} else {
		set_SP(esp_new);
	}
}

static inline void
iret_to_same_privilege_level(void)
{
	if (get_CPL() <= get_IOPL()) {
		commit_IF();
	}

	if (operand_size_32()) {
		commit_RF();

		if (get_CPL() == 0) {
			commit_IOPL0();
			commit_IOPL1();
		}
	} else {
		if (get_CPL() == 0) {
			commit_IOPL0();
			commit_IOPL1();
		}
	}

	commit_CF();
	commit_PF();
	commit_AF();
	commit_ZF();
	commit_SF();
	commit_TF();
	commit_OF();
	commit_DF();
	commit_NT();

	commit_CS();
}

static inline void
iret_protected_mode(void)
{
	uint32_t addr;

	if (get_NT()) {
		ERROR_NYI();
	}

	if (operand_size_32()) {
		eip_new = pop32();
		sset_selector(pop32());
		sset_EFLAGS32(pop32());

		if (eflag_vm_new) {
			ERROR_NYI();
			/* TODO return to vm86 */
		}
	} else {
		eip_new = pop16();
		sset_selector(pop16());
		sset_EFLAGS16(pop16());
	}

	if (selector_index == 0) {
		exception(EXCEPTION_GP, 0);
	}

	addr = get_descriptor_address();

	sset_descriptor(mrd(addr, get_CPL()), mrd(addr + 4, get_CPL()));

	if (selector_rpl < get_CPL()) {
		exception(EXCEPTION_GP, selector & 0xfffc);
	}

	if ((!descriptor_sflag) /* no code or data segment */
	 || (!((descriptor_type >> 3) & 1))) { /* no code segment */
		exception(EXCEPTION_GP, selector & 0xfffc);
	}

	if ((descriptor_type >> 2) & 1) { /* conforming */
		if (descriptor_dpl > selector_rpl) {
			exception(EXCEPTION_GP,
				selector & 0xfffc);
		}
	} else { /* non-conforming */
		if (descriptor_dpl != selector_rpl) {
			exception(EXCEPTION_GP,
				selector & 0xfffc);
		}
	}

	if (!descriptor_pflag) {
		exception(EXCEPTION_NP, selector & 0xfffc);
	}

	if (selector_rpl == get_CPL()) {
		iret_to_same_privilege_level();
	} else {
		iret_to_outer_privilege_level();
	}
}

static inline void
iret_real_mode(void)
{
	if (operand_size_32()) {
		uint32_t unchanged_mask;
		uint32_t mask;

		unchanged_mask = 0;
		unchanged_mask |= 1 << EFLAG_VIF;
		unchanged_mask |= 1 << EFLAG_VIP;
		unchanged_mask |= 1 << EFLAG_VM;

		mask = ~unchanged_mask;

		if (!can_pop(12)) {
			exception(EXCEPTION_SS, 0);
		}

		eip_new = pop32();

		if (eip_new > 0xffff) {
			exception(EXCEPTION_GP, 0);
		}

		sset_CS(pop32());

		sset_EFLAGS32((pop32() & mask)
			    | (get_EFLAGS32() & unchanged_mask));

		commit_EFLAGS32();
		commit_CS();
	} else {
		if (!can_pop(6)) {
			exception(EXCEPTION_SS, 0);
		}

		eip_new = pop16();
		sset_CS(pop16());
		sset_EFLAGS16(pop16());

		commit_EFLAGS16();
		commit_CS();
	}
}

static inline void
iret(void)
{
	if (virtual8086_mode()) {
		iret_virtual8086_mode();
	} else if (protected_mode()) {
		iret_protected_mode();
	} else { /* real mode */
		iret_real_mode();
	}
}

/******************************************************************************
 *
 * Interrupt
 *
 ******************************************************************************/

static inline void
interrupt_task_gate(uint8_t vector, bool is_soft_int, int error_code)
{
	ERROR_NYI(); /* TODO */
}

static inline void
interrupt_gate_to_inner_privilege_level(
	uint8_t vector,
	bool is_soft_int,
	int error_code,
	uint8_t gate_type,
	uint32_t gate_offset
)
{
	uint16_t temp_cs_selector;
	uint8_t	 temp_cs_selector_rpl;
	uint8_t  temp_cs_type;
	bool	 temp_cs_sflag;
	uint8_t  temp_cs_dpl;
	bool	 temp_cs_pflag;
	uint32_t temp_cs_segment_limit;
	uint32_t temp_cs_segment_base;
	bool	 temp_cs_segment_dflag;
	bool	 temp_cs_segment_avlflag;
	uint32_t esp_new;
	uint32_t offset;
	uint32_t addr;

	temp_cs_selector		= selector;
	temp_cs_selector_rpl		= selector_rpl;
	temp_cs_type			= descriptor_type;
	temp_cs_sflag			= descriptor_sflag;
	temp_cs_dpl			= descriptor_dpl;
	temp_cs_pflag			= descriptor_pflag;
	temp_cs_segment_limit		= descriptor_segment_limit;
	temp_cs_segment_base		= descriptor_segment_base;
	temp_cs_segment_dflag		= descriptor_segment_dflag;
	temp_cs_segment_avlflag		= descriptor_segment_avlflag;

	if (virtual8086_mode() && cs_dpl != 0) {
		exception(EXCEPTION_GP, selector & 0xfffc);
	}

	if ((tr_type == SEGMENT_32BIT_AVAIL_TSS)
	 || (tr_type == SEGMENT_32BIT_BUSY_TSS)) {
		offset = descriptor_dpl * 8 + 4;

		if ((offset + 6) > tr_system_limit) {
			exception(EXCEPTION_TS, tr_selector & 0xfffc);
		}

		esp_new = mrd(tr_system_base + offset, 0);
		sset_selector(mrw(tr_system_base + offset + 4, 0));
	} else {
		/* SEGMENT_16BIT_AVAIL_TSS */
		/* SEGMENT_16BIT_BUSY_TSS */
		offset = descriptor_dpl * 4 + 2;

		if ((offset + 4) > tr_system_limit) {
			exception(EXCEPTION_TS, tr_selector & 0xfffc);
		}

		esp_new = mrw(tr_system_base + offset, 0);
		sset_selector(mrw(tr_system_base + offset + 2, 0));
	}

	if (selector_index == 0) {
		exception(EXCEPTION_TS, 0);
	}

	addr = get_descriptor_address();

	sset_descriptor(mrd(addr, 0), mrd(addr + 4, 0));

	if (selector_rpl != temp_cs_dpl) {
		exception(EXCEPTION_TS, selector & 0xfffc);
	}

	if (descriptor_dpl != temp_cs_dpl) {
		exception(EXCEPTION_TS, selector & 0xfffc);
	}

	if ((!descriptor_sflag) /* no data/code segment */
	 || ((descriptor_type >> 3) & 1) /* code segment */
	 || (!((descriptor_type >> 1) & 1))) { /* not writeable */
		exception(EXCEPTION_TS, selector & 0xfffc);
	}

	if (!descriptor_pflag) {
		exception(EXCEPTION_SS, selector & 0xfffc);
	}

	if (gate_offset > temp_cs_segment_limit) {
		exception(EXCEPTION_GP, 0);
	}

	if (!descriptor_segment_dflag) {
		esp_new &= 0xffff;
	}

	if (virtual8086_mode()) {
		switch (gate_type) {
		default: DEBUG_ERROR_SWITCH();
		case SEGMENT_32BIT_INTERRUPT_GATE:
		case SEGMENT_32BIT_TRAP_GATE:
			ERROR_NYI();
			break;
		case SEGMENT_16BIT_INTERRUPT_GATE:
		case SEGMENT_16BIT_TRAP_GATE:
			ERROR_NYI();
			break;
		}
	} else {
		switch (gate_type) {
		default: DEBUG_ERROR_SWITCH();
		case SEGMENT_32BIT_INTERRUPT_GATE:
		case SEGMENT_32BIT_TRAP_GATE:
			mwd(esp_new + descriptor_segment_base - 4,
				get_SS(), temp_cs_dpl);
			mwd(esp_new + descriptor_segment_base - 8,
				get_ESP(), temp_cs_dpl);
			mwd(esp_new + descriptor_segment_base - 12,
				get_EFLAGS32(), temp_cs_dpl);
			mwd(esp_new + descriptor_segment_base - 16,
				get_CS(), temp_cs_dpl);
			mwd(esp_new + descriptor_segment_base - 20,
				eip_new, temp_cs_dpl);
			esp_new -= 20;

			if (error_code >= 0) {
				mwd(esp_new + descriptor_segment_base - 4,
					error_code, temp_cs_dpl);
				esp_new -= 4;
			}
			break;
		case SEGMENT_16BIT_INTERRUPT_GATE:
		case SEGMENT_16BIT_TRAP_GATE:
			mww(esp_new + descriptor_segment_base - 2,
				get_SS(), temp_cs_dpl);
			mww(esp_new + descriptor_segment_base - 4,
				get_ESP(), temp_cs_dpl);
			mww(esp_new + descriptor_segment_base - 6,
				get_EFLAGS16(), temp_cs_dpl);
			mww(esp_new + descriptor_segment_base - 8,
				get_CS(), temp_cs_dpl);
			mww(esp_new + descriptor_segment_base - 10,
				eip_new, temp_cs_dpl);
			esp_new -= 10;

			if (error_code >= 0) {
				mww(esp_new + descriptor_segment_base - 4,
					error_code, temp_cs_dpl);
				esp_new -= 2;
			}
			break;
		}
	}

	eip_new = gate_offset;

	cs_selector		= temp_cs_selector;
	cs_type			= temp_cs_type;
	cs_sflag		= temp_cs_sflag;
	cs_dpl			= temp_cs_dpl;
	cs_pflag		= temp_cs_pflag;
	cs_segment_limit	= temp_cs_segment_limit;
	cs_segment_base		= temp_cs_segment_base;
	cs_segment_dflag	= temp_cs_segment_dflag;
	cs_segment_avlflag	= temp_cs_segment_avlflag;

	commit_SS();

	if (stack_size_32()) {
		set_ESP(esp_new);
	} else {
		set_SP(esp_new);
	}

	if (!(gate_type & 1)) {
		sset_IF(0); commit_IF();
	}

	sset_TF(0); commit_TF();
	sset_NT(0); commit_NT();
	sset_VM(0); commit_VM();
	sset_RF(0); commit_RF();

	if (virtual8086_mode()) {
		/* TODO invalidate ds, es, fs, gs */
		ERROR_NYI();
	}
}

static inline void
interrupt_gate_to_same_privilege_level(
	uint8_t vector,
	bool is_soft_int,
	int error_code,
	uint8_t gate_type,
	uint32_t gate_offset
)
{
	if (gate_offset > descriptor_segment_limit) {
		exception(EXCEPTION_GP, 0);
	}

	switch (gate_type) {
	default: DEBUG_ERROR_SWITCH();
	case SEGMENT_32BIT_INTERRUPT_GATE:
	case SEGMENT_32BIT_TRAP_GATE:
		push32(get_EFLAGS32());
		push32(get_CS());
		push32(eip);

		if (error_code >= 0) {
			push32(error_code & 0xffff);
		}

		break;
	case SEGMENT_16BIT_INTERRUPT_GATE:
	case SEGMENT_16BIT_TRAP_GATE:
		push16(get_EFLAGS16());
		push16(get_CS());
		push16(eip);

		if (error_code >= 0) {
			push16(error_code & 0xffff);
		}

		break;
	}

	eip_new = gate_offset;

	commit_CS();

	if (!(gate_type & 1)) {
		sset_IF(0); commit_IF();
	}

	sset_TF(0); commit_TF();
	sset_NT(0); commit_NT();
	sset_VM(0); commit_VM();
	sset_RF(0); commit_RF();
}

static inline void
interrupt_gate(uint8_t vector, bool is_soft_int, int error_code)
{
	uint16_t gate_selector;
	uint32_t gate_offset;
	uint8_t  gate_type;
	uint32_t addr;

	gate_selector = descriptor_gate_selector;
	gate_offset = descriptor_gate_offset;
	gate_type = descriptor_type;

	if ((gate_selector & 0xfffc) == 0) {
		exception(EXCEPTION_GP, 0);
	}

	/* overwrite the gate selector with new cs selector */
	sset_selector(gate_selector);

	if (selector_index == 0) {
		exception(EXCEPTION_GP, 0);
	}

	addr = get_descriptor_address();

	sset_descriptor(mrd(addr, 0), mrd(addr + 4, 0));

	if ((!descriptor_sflag) /* no data/code segment */
	 || (!((descriptor_type >> 3) & 1)) /* no code segemnt */
	 || (descriptor_dpl > get_CPL())) {
		exception(EXCEPTION_GP, selector & 0xfffc);
	}

	if (!descriptor_pflag) {
		exception(EXCEPTION_NP, selector & 0xfffc);
	}

	if ((!((descriptor_type >> 2) & 1)) /* non-conforming code segment */
	 && (descriptor_dpl < get_CPL())) {
		interrupt_gate_to_inner_privilege_level(vector,
			is_soft_int, error_code, gate_type, gate_offset);
	} else if (virtual8086_mode()) {
		exception(EXCEPTION_GP, selector & 0xfffc);
	} else if (((descriptor_type >> 2) & 1) /* conforming code segment */
		|| (descriptor_dpl == get_CPL())) {
		interrupt_gate_to_same_privilege_level(vector,
			is_soft_int, error_code, gate_type, gate_offset);
	} else {
		exception(EXCEPTION_GP, selector & 0xfffc);
	}
}

static inline void
interrupt_protected_mode(uint8_t vector, bool is_soft_int, int error_code)
{
	uint32_t addr;

	if ((vector * 8 + 7) > idtr_limit) {
		exception(EXCEPTION_GP, vector * 8 + 2);
	}

	addr = idtr_base + vector * 8;

	sset_descriptor(mrd(addr, 0), mrd(addr + 4, 0));

	if (descriptor_sflag) {
		exception(EXCEPTION_GP, vector * 8 + 2);
	}

	if ((descriptor_type != SEGMENT_TASK_GATE)
	 && (descriptor_type != SEGMENT_16BIT_INTERRUPT_GATE)
	 && (descriptor_type != SEGMENT_32BIT_INTERRUPT_GATE)
	 && (descriptor_type != SEGMENT_16BIT_TRAP_GATE)
	 && (descriptor_type != SEGMENT_32BIT_TRAP_GATE)) {
		exception(EXCEPTION_GP, exception_vector * 8 + 2);
	}

	if (is_soft_int && descriptor_dpl < get_CPL()) {
		exception(EXCEPTION_GP, vector * 8 + 2);
	}

	if (!descriptor_pflag) {
		exception(EXCEPTION_NP, vector * 8 + 2);
	}

	if (descriptor_type == SEGMENT_TASK_GATE) {
		interrupt_task_gate(vector, is_soft_int, error_code);
	} else {
		/* SEGMENT_16BIT_INTERRUPT_GATE */
		/* SEGMENT_32BIT_INTERRUPT_GATE */
		/* SEGMENT_16BIT_TRAP_GATE */
		/* SEGMENT_32BIT_TRAP_GATE */
		interrupt_gate(vector, is_soft_int, error_code);
	}
}

static inline void
interrupt_real_mode(uint8_t vector)
{
	if ((vector * 4 + 3) > idtr_limit) {
		exception(EXCEPTION_GP, 0);
	}

	eip_new = mrw(idtr_base + 4 * vector, 0);
	sset_CS(mrw(idtr_base + 4 * vector + 2, 0));

	push16(get_EFLAGS16());
	push16(get_CS());
	push16(eip);

	commit_CS();

	sset_IF(0); commit_IF();
	sset_TF(0); commit_TF();
	sset_RF(0); commit_RF();
}

static inline void
interrupt(uint8_t vector, bool is_soft_int, int error_code)
{
	if (real_mode()) {
		interrupt_real_mode(vector);
	} else {
		interrupt_protected_mode(vector, is_soft_int, error_code);
	}
}

/******************************************************************************
 *
 * Handler for one instruction including prefixes
 *
 ******************************************************************************/

#define DEBUG_OPERATION(operation)					\
	do {								\
		uint32_t u = eip_new - eip;				\
		while (u < 8) {						\
			DEBUG("   ");					\
			u++;						\
		}							\
		DEBUG("%-6s", operation);				\
	} while (0)

#ifndef DEBUG
# define DEBUG_OPERATION_GROUP1(opcode_extension)
# define DEBUG_OPERATION_GROUP2(opcode_extension)
# define DEBUG_OPERATION_GROUP3(opcode_extension)
# define DEBUG_OPERATION_GROUP4(opcode_extension)
# define DEBUG_OPERATION_GROUP5(opcode_extension)
# define DEBUG_OPERATION_GROUP6(opcode_extension)
# define DEBUG_OPERATION_GROUP7(opcode_extension)
# define DEBUG_OPERATION_GROUP8(opcode_extension)
# define DEBUG_OPERATION_GROUP9(opcode_extension)
# define DEBUG_OPERATION_GROUPA(opcode_extension)
#else
# define DEBUG_OPERATION_GROUP1(opcode_extension)			\
	switch (opcode_extension) {					\
	default: DEBUG_ERROR_SWITCH();					\
	case 0: DEBUG_OPERATION("ADD"); break;				\
	case 1: DEBUG_OPERATION("OR"); break;				\
	case 2: DEBUG_OPERATION("ADC"); break;				\
	case 3: DEBUG_OPERATION("SBB"); break;				\
	case 4: DEBUG_OPERATION("AND"); break;				\
	case 5: DEBUG_OPERATION("SUB"); break;				\
	case 6: DEBUG_OPERATION("XOR"); break;				\
	case 7: DEBUG_OPERATION("CMP"); break;				\
	}
# define DEBUG_OPERATION_GROUP2(opcode_extension)			\
	switch (opcode_extension) {					\
	default: DEBUG_ERROR_SWITCH();					\
	case 0: DEBUG_OPERATION("ROL"); break;				\
	case 1: DEBUG_OPERATION("ROR"); break;				\
	case 2: DEBUG_OPERATION("RCL"); break;				\
	case 3: DEBUG_OPERATION("RCR"); break;				\
	case 4: DEBUG_OPERATION("SHL"); break;				\
	case 5: DEBUG_OPERATION("SHR"); break;				\
	case 6: DEBUG_OPERATION("*RESERVED*"); break;			\
	case 7: DEBUG_OPERATION("SAR"); break;				\
	}
# define DEBUG_OPERATION_GROUP3(opcode_extension)			\
	switch (opcode_extension) {					\
	default: DEBUG_ERROR_SWITCH();					\
	case 0: DEBUG_OPERATION("TEST"); break;				\
	case 1: DEBUG_OPERATION("*RESERVED*"); break;			\
	case 2: DEBUG_OPERATION("NOT"); break;				\
	case 3: DEBUG_OPERATION("NEG"); break;				\
	case 4: DEBUG_OPERATION("MUL"); break;				\
	case 5: DEBUG_OPERATION("IMUL"); break;				\
	case 6: DEBUG_OPERATION("DIV"); break;				\
	case 7: DEBUG_OPERATION("IDIV"); break;				\
	}
# define DEBUG_OPERATION_GROUP4(opcode_extension)			\
	switch (opcode_extension) {					\
	default: DEBUG_ERROR_SWITCH();					\
	case 0: DEBUG_OPERATION("INC"); break;				\
	case 1: DEBUG_OPERATION("DEC"); break;				\
	case 2: DEBUG_OPERATION("*RESERVED*"); break;			\
	case 3: DEBUG_OPERATION("*RESERVED*"); break;			\
	case 4: DEBUG_OPERATION("*RESERVED*"); break;			\
	case 5: DEBUG_OPERATION("*RESERVED*"); break;			\
	case 6: DEBUG_OPERATION("*RESERVED*"); break;			\
	case 7: DEBUG_OPERATION("*RESERVED*"); break;			\
	}
#endif

#define DEBUG_CPUSTATE DEBUG

void
handle_instruction(void)
{
	esp_backup = esp;

	if (exception_vector != EXCEPTION_NONE) {
		interrupt(exception_vector, exception_is_interrupt,
				exception_error_code);
		exception_double_page_fault = 0;
		exception_vector = EXCEPTION_NONE;
		commit_EIP();
		return;
	}

	if (interrupt_pending && get_IF() && !interrupt_delay) {
		interrupt(cpu_irq(), 0, -1);
		commit_EIP();
		halt_state = 0;
		return;
	}

	if (halt_state) {
		return;
	}

	DEBUG("\n");
	DEBUG_CPUSTATE("EAX=0x%08x ", eax);
	DEBUG_CPUSTATE("ECX=0x%08x ", ecx);
	DEBUG_CPUSTATE("EDX=0x%08x ", edx);
	DEBUG_CPUSTATE("EBX=0x%08x ", ebx);
	DEBUG_CPUSTATE("ESP=0x%08x ", esp);
	DEBUG_CPUSTATE("EBP=0x%08x ", ebp);
	DEBUG_CPUSTATE("ESI=0x%08x ", esi);
	DEBUG_CPUSTATE("EDI=0x%08x ", edi);
	DEBUG_CPUSTATE("EIP=0x%08x ", eip);
	DEBUG_CPUSTATE("EFLAGS=%c%c%c%c%c%c0%c0%c1%c ",
			eflag_of ? 'O' : 'o',
			eflag_df ? 'D' : 'd',
			eflag_if ? 'I' : 'i',
			eflag_tf ? 'T' : 't',
			eflag_sf ? 'S' : 's',
			eflag_zf ? 'Z' : 'z',
			eflag_af ? 'A' : 'a',
			eflag_pf ? 'P' : 'p',
			eflag_cf ? 'C' : 'c');
	DEBUG_CPUSTATE("\n");

	DEBUG("%08x: ", eip_new + cs_segment_base);

#if 0
	if (0xc0000000 <= cs_segment_base + eip
	 && cs_segment_base + eip <= 0xcfffffff) {
		debug = 1;
	}
	if (debug
	 && ! done) {
		if (0xc0000000 < cs_segment_base + eip
		 && cs_segment_base + eip <= 0xcfffffff) {
			fprintf(stderr, "EXECUTING %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
					cs_segment_base + eip,
					eax, ebx, ecx, edx, ebp, esp, edi, esi,
					get_EFLAGS32());
		}
		done = 1;
	}
#endif

	instruction_opcode = mrb_cs(eip_new);
	eip_new += 1;

	switch (instruction_opcode) {
	default:
		ERROR_NYI();
	case 0x00:
		fetch_modrm();
		DEBUG_OPERATION("ADD");
		load_Eb_Gb(); addb(); store_Eb();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x01:
		fetch_modrm();
		DEBUG_OPERATION("ADD");
		load_Ev_Gv(); addv(); store_Ev();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x02:
		fetch_modrm();
		DEBUG_OPERATION("ADD");
		load_Gb_Eb(); addb(); store_Gb();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x03:
		fetch_modrm();
		DEBUG_OPERATION("ADD");
		load_Gv_Ev(); addv(); store_Gv();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x04:
		fetch_immb();
		DEBUG_OPERATION("ADD");
		load_AL_Ib(); addb(); store_AL();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x05:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("ADD");
		load_eAX_Iv(); addv(); store_eAX();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x06:
		DEBUG_OPERATION("PUSH");
		load_ES(); pushv();
		commit_EIP();
		break;
	case 0x07:
		DEBUG_OPERATION("POP");
		popv(); store_ES();
		commit_EIP();
		break;
	case 0x08:
		fetch_modrm();
		DEBUG_OPERATION("OR");
		load_Eb_Gb(); orb(); store_Eb();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x09:
		fetch_modrm();
		DEBUG_OPERATION("OR");
		load_Ev_Gv(); orv(); store_Ev();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x0a:
		fetch_modrm();
		DEBUG_OPERATION("OR");
		load_Gb_Eb(); orb(); store_Gb();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x0b:
		fetch_modrm();
		DEBUG_OPERATION("OR");
		load_Gv_Ev(); orv(); store_Gv();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x0c:
		fetch_immb();
		DEBUG_OPERATION("OR");
		load_AL_Ib(); orb(); store_AL();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x0d:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("OR");
		load_eAX_Iv(); orv(); store_eAX();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x0e:
		DEBUG_OPERATION("PUSH");
		load_CS(); pushv();
		commit_EIP();
		break;
	case 0x0f: /* 2-byte opcode */
		instruction_opcode2 = mrb_cs(eip_new);
		eip_new++;
		switch (instruction_opcode2) {
		default:
			ERROR_NYI();
		case 0x00:
			fetch_modrm();
			switch (instruction_reg) {
			default: ERROR_NYI();
			/* TODO */
			case 0:
				DEBUG_OPERATION("SLDT");
				sldt(); store_Ew();
				break;
			case 1:
				DEBUG_OPERATION("STR");
				str(); store_Ew();
				break;
			case 2:
				DEBUG_OPERATION("LLDT");
				load_Ew(); lldt();
				break;
			case 3:
				DEBUG_OPERATION("LTR");
				load_Ew(); ltr();
				break;
			/* TODO */
			}
			commit_EIP();
			break;
		case 0x01:
			fetch_modrm();
			switch (instruction_reg) {
			default: ERROR_NYI();
			/* TODO */
			case 2:
				DEBUG_OPERATION("LGDT");
				load_Ms(); lgdt();
				break;
			case 3:
				DEBUG_OPERATION("LIDT");
				load_Ms(); lidt();
				break;
			case 4:
				DEBUG_OPERATION("SMSW");
				smsw(); store_Ew();
				break;
			/* TODO */
			case 6:
				DEBUG_OPERATION("LMSW");
				load_Ew(); lmsw();
				break;
			case 7:
				DEBUG_OPERATION("INVLPG");
				exception(EXCEPTION_UD, -1); /* 486+ */
				break;
			}
			commit_EIP();
			break;
		/* TODO */
		case 0x06:
			DEBUG_OPERATION("CLTS");
			clts();
			commit_EIP();
			break;
		/* TODO */
		case 0x20:
			fetch_modrm();
			DEBUG_OPERATION("MOV");
			load_Cd(); movd(); store_Rd();
			commit_EIP();
			break;
		/* TODO */
		case 0x22:
			fetch_modrm();
			DEBUG_OPERATION("MOV");
			load_Rd(); movd(); store_Cd();
			commit_EIP();
			break;
		/* TODO */
		case 0x80:
			fetch_immv(operand_size_32());
			DEBUG_OPERATION("JO");
			load_Jv();
			jo();
			commit_EIP();
			break;
		case 0x81:
			fetch_immv(operand_size_32());
			DEBUG_OPERATION("JNO");
			load_Jv();
			jno();
			commit_EIP();
			break;
		case 0x82:
			fetch_immv(operand_size_32());
			DEBUG_OPERATION("JB");
			load_Jv();
			jb();
			commit_EIP();
			break;
		case 0x83:
			fetch_immv(operand_size_32());
			DEBUG_OPERATION("JNB");
			load_Jv();
			jnb();
			commit_EIP();
			break;
		case 0x84:
			fetch_immv(operand_size_32());
			DEBUG_OPERATION("JZ");
			load_Jv();
			jz();
			commit_EIP();
			break;
		case 0x85:
			fetch_immv(operand_size_32());
			DEBUG_OPERATION("JNZ");
			load_Jv();
			jnz();
			commit_EIP();
			break;
		case 0x86:
			fetch_immv(operand_size_32());
			DEBUG_OPERATION("JBE");
			load_Jv();
			jbe();
			commit_EIP();
			break;
		case 0x87:
			fetch_immv(operand_size_32());
			DEBUG_OPERATION("JNBE");
			load_Jv();
			jnbe();
			commit_EIP();
			break;
		case 0x88:
			fetch_immv(operand_size_32());
			DEBUG_OPERATION("JS");
			load_Jv();
			js();
			commit_EIP();
			break;
		case 0x89:
			fetch_immv(operand_size_32());
			DEBUG_OPERATION("JNS");
			load_Jv();
			jns();
			commit_EIP();
			break;
		case 0x8a:
			fetch_immv(operand_size_32());
			DEBUG_OPERATION("JP");
			load_Jv();
			jp();
			commit_EIP();
			break;
		case 0x8b:
			fetch_immv(operand_size_32());
			DEBUG_OPERATION("JNP");
			load_Jv();
			jnp();
			commit_EIP();
			break;
		case 0x8c:
			fetch_immv(operand_size_32());
			DEBUG_OPERATION("JL");
			load_Jv();
			jl();
			commit_EIP();
			break;
		case 0x8d:
			fetch_immv(operand_size_32());
			DEBUG_OPERATION("JNL");
			load_Jv();
			jnl();
			commit_EIP();
			break;
		case 0x8e:
			fetch_immv(operand_size_32());
			DEBUG_OPERATION("JLE");
			load_Jv();
			jle();
			commit_EIP();
			break;
		case 0x8f:
			fetch_immv(operand_size_32());
			DEBUG_OPERATION("JNLE");
			load_Jv();
			jnle();
			commit_EIP();
			break;
		case 0x90:
			fetch_modrm();
			DEBUG_OPERATION("SETO");
			seto(); store_Eb();
			commit_EIP();
			break;
		case 0x91:
			fetch_modrm();
			DEBUG_OPERATION("SETNO");
			setno(); store_Eb();
			commit_EIP();
			break;
		case 0x92:
			fetch_modrm();
			DEBUG_OPERATION("SETB");
			setb(); store_Eb();
			commit_EIP();
			break;
		case 0x93:
			fetch_modrm();
			DEBUG_OPERATION("SETNB");
			setnb(); store_Eb();
			commit_EIP();
			break;
		case 0x94:
			fetch_modrm();
			DEBUG_OPERATION("SETZ");
			setz(); store_Eb();
			commit_EIP();
			break;
		case 0x95:
			fetch_modrm();
			DEBUG_OPERATION("SETNZ");
			setnz(); store_Eb();
			commit_EIP();
			break;
		case 0x96:
			fetch_modrm();
			DEBUG_OPERATION("SETBE");
			setbe(); store_Eb();
			commit_EIP();
			break;
		case 0x97:
			fetch_modrm();
			DEBUG_OPERATION("SETNBE");
			setnbe(); store_Eb();
			commit_EIP();
			break;
		case 0x98:
			fetch_modrm();
			DEBUG_OPERATION("SETS");
			sets(); store_Eb();
			commit_EIP();
			break;
		case 0x99:
			fetch_modrm();
			DEBUG_OPERATION("SETNS");
			setns(); store_Eb();
			commit_EIP();
			break;
		case 0x9a:
			fetch_modrm();
			DEBUG_OPERATION("SETP");
			setp(); store_Eb();
			commit_EIP();
			break;
		case 0x9b:
			fetch_modrm();
			DEBUG_OPERATION("SETNP");
			setnp(); store_Eb();
			commit_EIP();
			break;
		case 0x9c:
			fetch_modrm();
			DEBUG_OPERATION("SETL");
			setl(); store_Eb();
			commit_EIP();
			break;
		case 0x9d:
			fetch_modrm();
			DEBUG_OPERATION("SETNL");
			setnl(); store_Eb();
			commit_EIP();
			break;
		case 0x9e:
			fetch_modrm();
			DEBUG_OPERATION("SETLE");
			setle(); store_Eb();
			commit_EIP();
			break;
		case 0x9f:
			fetch_modrm();
			DEBUG_OPERATION("SETNLE");
			setnle(); store_Eb();
			commit_EIP();
			break;
		case 0xa0:
			DEBUG_OPERATION("PUSH");
			load_FS(); pushv();
			commit_EIP();
			break;
		case 0xa1:
			DEBUG_OPERATION("POP");
			popv(); store_FS();
			commit_EIP();
			break;
		case 0xa2:
			exception(EXCEPTION_UD, -1);
			break;
		case 0xa3:
			fetch_modrm();
			DEBUG_OPERATION("BT");
			load_EvBIT_Gv(); btvv();
			commit_EIP(); commit_CF();
			break;
		case 0xa4:
			fetch_modrm();
			fetch_immb();
			DEBUG_OPERATION("SHLD");
			load_Ev_Gv_Ib(); shldv(); store_Ev();
			commit_EIP(); commit_OSZAPC();
			break;
		case 0xa5:
			fetch_modrm();
			DEBUG_OPERATION("SHLD");
			load_Ev_Gv_CL(); shldv(); store_Ev();
			commit_EIP(); commit_OSZAPC();
			break;
		/* TODO */
		case 0xa8:
			DEBUG_OPERATION("PUSH");
			load_GS(); pushv();
			commit_EIP();
			break;
		case 0xa9:
			DEBUG_OPERATION("POP");
			popv(); store_GS();
			commit_EIP();
			break;
		/* TODO */
		case 0xab:
			fetch_modrm();
			DEBUG_OPERATION("BTS");
			load_EvBIT_Gv(); btsvv(); store_EvBIT();
			commit_EIP(); commit_CF();
			break;
		case 0xac:
			fetch_modrm();
			fetch_immb();
			DEBUG_OPERATION("SHRD");
			load_Ev_Gv_Ib(); shrdv(); store_Ev();
			commit_EIP(); commit_OSZAPC();
			break;
		case 0xad:
			fetch_modrm();
			DEBUG_OPERATION("SHRD");
			load_Ev_Gv_CL(); shrdv(); store_Ev();
			commit_EIP(); commit_OSZAPC();
			break;
		/* TODO */
		case 0xaf:
			fetch_modrm();
			DEBUG_OPERATION("IMUL");
			load_Gv_Ev(); imul2v(); store_Gv();
			commit_EIP(); commit_OSZAPC();
			break;
		/* TODO */
		case 0xb2:
			fetch_modrm();
			DEBUG_OPERATION("LSS");
			load_Mp(); lss(); store_Gv();
			commit_EIP();
			break;
		case 0xb3:
			fetch_modrm();
			DEBUG_OPERATION("BTR");
			load_EvBIT_Gv(); btrvv(); store_EvBIT();
			commit_OSZAPC(); commit_EIP();
			break;
		case 0xb4:
			fetch_modrm();
			DEBUG_OPERATION("LFS");
			load_Mp(); lfs(); store_Gv();
			commit_EIP();
			break;
		case 0xb5:
			fetch_modrm();
			DEBUG_OPERATION("LGS");
			load_Mp(); lgs(); store_Gv();
			commit_EIP();
			break;
		case 0xb6:
			fetch_modrm();
			DEBUG_OPERATION("MOVZX");
			load_Eb(); movzxb(); store_Gv();
			commit_EIP();
			break;
		case 0xb7:
			fetch_modrm();
			DEBUG_OPERATION("MOVZX");
			load_Ew(); movzxw(); store_Gv();
			commit_EIP();
			break;
		/* TODO */
		case 0xba:
			fetch_modrm();
			fetch_immb();
			switch (instruction_reg) {
			default: ERROR_NYI();
			/* TODO */
			case 5:
				DEBUG_OPERATION("BTS");
				load_Ev_Ib(); btsvb(); store_Ev();
				commit_OSZAPC();
				break;
			case 6:
				DEBUG_OPERATION("BTR");
				load_Ev_Ib(); btrvb(); store_Ev();
				commit_OSZAPC();
				break;
			case 7:
				DEBUG_OPERATION("BTC");
				load_Ev_Ib(); btcvb(); store_Ev();
				commit_OSZAPC();
				break;
			}
			commit_EIP();
			break;
		case 0xbb:
			fetch_modrm();
			DEBUG_OPERATION("BTC");
			load_EvBIT_Gv(); btcvv(); store_EvBIT();
			commit_OSZAPC(); commit_EIP();
			break;
		case 0xbc:
			fetch_modrm();
			DEBUG_OPERATION("BSF");
			load_Gv_Ev(); bsfv(); store_Gv();
			commit_OSZAPC(); commit_EIP();
			break;
		case 0xbd:
			fetch_modrm();
			DEBUG_OPERATION("BSR");
			load_Gv_Ev(); bsrv(); store_Gv();
			commit_OSZAPC(); commit_EIP();
			break;
		case 0xbe:
			fetch_modrm();
			DEBUG_OPERATION("MOVSX");
			load_Eb(); movsxb(); store_Gv();
			commit_EIP();
			break;
		case 0xbf:
			fetch_modrm();
			DEBUG_OPERATION("MOVSX");
			load_Ew(); movsxw(); store_Gv();
			commit_EIP();
			break;
		/* TODO */
		case 0xc8:
		case 0xc9:
		case 0xca:
		case 0xcb:
		case 0xcc:
		case 0xcd:
		case 0xce:
		case 0xcf:
			DEBUG_OPERATION("BSWAP");
			exception(EXCEPTION_UD, -1);
			break;
		/* TODO */
		}
		break;
	case 0x10:
		fetch_modrm();
		DEBUG_OPERATION("ADC");
		load_Eb_Gb(); adcb(); store_Eb();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x11:
		fetch_modrm();
		DEBUG_OPERATION("ADC");
		load_Ev_Gv(); adcv(); store_Ev();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x12:
		fetch_modrm();
		DEBUG_OPERATION("ADC");
		load_Gb_Eb(); adcb(); store_Gb();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x13:
		fetch_modrm();
		DEBUG_OPERATION("ADC");
		load_Gv_Ev(); adcv(); store_Gv();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x14:
		fetch_immb();
		DEBUG_OPERATION("ADC");
		load_AL_Ib(); adcb(); store_AL();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x15:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("ADC");
		load_eAX_Iv(); adcv(); store_eAX();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x16:
		DEBUG_OPERATION("PUSH");
		load_SS(); pushv();
		commit_EIP();
		break;
	case 0x17:
		DEBUG_OPERATION("POP");
		popv(); store_SS();
		commit_EIP_and_delay_interrupts();
		break;
	case 0x18:
		fetch_modrm();
		DEBUG_OPERATION("SBB");
		load_Eb_Gb(); sbbb(); store_Eb();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x19:
		fetch_modrm();
		DEBUG_OPERATION("SBB");
		load_Ev_Gv(); sbbv(); store_Ev();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x1a:
		fetch_modrm();
		DEBUG_OPERATION("SBB");
		load_Gb_Eb(); sbbb(); store_Gb();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x1b:
		fetch_modrm();
		DEBUG_OPERATION("SBB");
		load_Gv_Ev(); sbbv(); store_Gv();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x1c:
		fetch_immb();
		DEBUG_OPERATION("SBB");
		load_AL_Ib(); sbbb(); store_AL();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x1d:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("SBB");
		load_eAX_Iv(); sbbv(); store_eAX();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x1e:
		DEBUG_OPERATION("PUSH");
		load_DS(); pushv();
		commit_EIP();
		break;
	case 0x1f:
		DEBUG_OPERATION("POP");
		popv(); store_DS();
		commit_EIP();
		break;
	case 0x20:
		fetch_modrm();
		DEBUG_OPERATION("AND");
		load_Eb_Gb(); andb(); store_Eb();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x21:
		fetch_modrm();
		DEBUG_OPERATION("AND");
		load_Ev_Gv(); andv(); store_Ev();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x22:
		fetch_modrm();
		DEBUG_OPERATION("AND");
		load_Gb_Eb(); andb(); store_Gb();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x23:
		fetch_modrm();
		DEBUG_OPERATION("AND");
		load_Gv_Ev(); andv(); store_Gv();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x24:
		fetch_immb();
		DEBUG_OPERATION("AND");
		load_AL_Ib(); andb(); store_AL();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x25:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("AND");
		load_eAX_Iv(); andv(); store_eAX();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x26:
		DEBUG_OPERATION("ES segment override prefix");
		prefix_segment_override = SEGMENT_ES;
		break;
	case 0x27:
		DEBUG_OPERATION("DAA");
		daa();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x28:
		fetch_modrm();
		DEBUG_OPERATION("SUB");
		load_Eb_Gb(); subb(); store_Eb();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x29:
		fetch_modrm();
		DEBUG_OPERATION("SUB");
		load_Ev_Gv(); subv(); store_Ev();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x2a:
		fetch_modrm();
		DEBUG_OPERATION("SUB");
		load_Gb_Eb(); subb(); store_Gb();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x2b:
		fetch_modrm();
		DEBUG_OPERATION("SUB");
		load_Gv_Ev(); subv(); store_Gv();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x2c:
		fetch_immb();
		DEBUG_OPERATION("SUB");
		load_AL_Ib(); subb(); store_AL();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x2d:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("SUB");
		load_eAX_Iv(); subv(); store_eAX();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x2e:
		DEBUG_OPERATION("CS segment override prefix");
		prefix_segment_override = SEGMENT_CS;
		break;
	case 0x2f:
		DEBUG_OPERATION("DAS");
		das();
		commit_EIP();
		break;
	case 0x30:
		fetch_modrm();
		DEBUG_OPERATION("XOR");
		load_Eb_Gb(); xorb(); store_Eb();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x31:
		fetch_modrm();
		DEBUG_OPERATION("XOR");
		load_Ev_Gv(); xorv(); store_Ev();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x32:
		fetch_modrm();
		DEBUG_OPERATION("XOR");
		load_Gb_Eb(); xorb(); store_Gb();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x33:
		fetch_modrm();
		DEBUG_OPERATION("XOR");
		load_Gv_Ev(); xorv(); store_Gv();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x34:
		fetch_immb();
		DEBUG_OPERATION("XOR");
		load_AL_Ib(); xorb(); store_AL();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x35:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("XOR");
		load_eAX_Iv(); xorv(); store_eAX();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x36:
		DEBUG_OPERATION("SS segment override prefix");
		prefix_segment_override = SEGMENT_SS;
		break;
	case 0x37:
		DEBUG_OPERATION("AAA");
		aaa();
		commit_EIP();
		break;
	case 0x38:
		fetch_modrm();
		DEBUG_OPERATION("CMP");
		load_Eb_Gb(); cmpb();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x39:
		fetch_modrm();
		DEBUG_OPERATION("CMP");
		load_Ev_Gv(); cmpv();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x3a:
		fetch_modrm();
		DEBUG_OPERATION("CMP");
		load_Gb_Eb(); cmpb();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x3b:
		fetch_modrm();
		DEBUG_OPERATION("CMP");
		load_Gv_Ev(); cmpv();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x3c:
		fetch_immb();
		DEBUG_OPERATION("CMP");
		load_AL_Ib(); cmpb();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x3d:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("CMP");
		load_eAX_Iv(); cmpv();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x3e:
		DEBUG_OPERATION("DS segment override prefix");
		prefix_segment_override = SEGMENT_DS;
		break;
	case 0x3f:
		DEBUG_OPERATION("AAS");
		aas();
		commit_EIP();
		break;
	case 0x40:
		DEBUG_OPERATION("INC");
		load_eAX(); incv(); store_eAX();
		commit_EIP(); commit_OSZAP();
		break;
	case 0x41:
		DEBUG_OPERATION("INC");
		load_eCX(); incv(); store_eCX();
		commit_EIP(); commit_OSZAP();
		break;
	case 0x42:
		DEBUG_OPERATION("INC");
		load_eDX(); incv(); store_eDX();
		commit_EIP(); commit_OSZAP();
		break;
	case 0x43:
		DEBUG_OPERATION("INC");
		load_eBX(); incv(); store_eBX();
		commit_EIP(); commit_OSZAP();
		break;
	case 0x44:
		DEBUG_OPERATION("INC");
		load_eSP(); incv(); store_eSP();
		commit_EIP(); commit_OSZAP();
		break;
	case 0x45:
		DEBUG_OPERATION("INC");
		load_eBP(); incv(); store_eBP();
		commit_EIP(); commit_OSZAP();
		break;
	case 0x46:
		DEBUG_OPERATION("INC");
		load_eSI(); incv(); store_eSI();
		commit_EIP(); commit_OSZAP();
		break;
	case 0x47:
		DEBUG_OPERATION("INC");
		load_eDI(); incv(); store_eDI();
		commit_EIP(); commit_OSZAP();
		break;
	case 0x48:
		DEBUG_OPERATION("DEC");
		load_eAX(); decv(); store_eAX();
		commit_EIP(); commit_OSZAP();
		break;
	case 0x49:
		DEBUG_OPERATION("DEC");
		load_eCX(); decv(); store_eCX();
		commit_EIP(); commit_OSZAP();
		break;
	case 0x4a:
		DEBUG_OPERATION("DEC");
		load_eDX(); decv(); store_eDX();
		commit_EIP(); commit_OSZAP();
		break;
	case 0x4b:
		DEBUG_OPERATION("DEC");
		load_eBX(); decv(); store_eBX();
		commit_EIP(); commit_OSZAP();
		break;
	case 0x4c:
		DEBUG_OPERATION("DEC");
		load_eSP(); decv(); store_eSP();
		commit_EIP(); commit_OSZAP();
		break;
	case 0x4d:
		DEBUG_OPERATION("DEC");
		load_eBP(); decv(); store_eBP();
		commit_EIP(); commit_OSZAP();
		break;
	case 0x4e:
		DEBUG_OPERATION("DEC");
		load_eSI(); decv(); store_eSI();
		commit_EIP(); commit_OSZAP();
		break;
	case 0x4f:
		DEBUG_OPERATION("DEC");
		load_eDI(); decv(); store_eDI();
		commit_EIP(); commit_OSZAP();
		break;
	case 0x50:
		DEBUG_OPERATION("PUSH");
		load_eAX(); pushv();
		commit_EIP();
		break;
	case 0x51:
		DEBUG_OPERATION("PUSH");
		load_eCX(); pushv();
		commit_EIP();
		break;
	case 0x52:
		DEBUG_OPERATION("PUSH");
		load_eDX(); pushv();
		commit_EIP();
		break;
	case 0x53:
		DEBUG_OPERATION("PUSH");
		load_eBX(); pushv();
		commit_EIP();
		break;
	case 0x54:
		DEBUG_OPERATION("PUSH");
		load_eSP(); pushv();
		commit_EIP();
		break;
	case 0x55:
		DEBUG_OPERATION("PUSH");
		load_eBP(); pushv();
		commit_EIP();
		break;
	case 0x56:
		DEBUG_OPERATION("PUSH");
		load_eSI(); pushv();
		commit_EIP();
		break;
	case 0x57:
		DEBUG_OPERATION("PUSH");
		load_eDI(); pushv();
		commit_EIP();
		break;
	case 0x58:
		DEBUG_OPERATION("POP");
		popv(); store_eAX();
		commit_EIP();
		break;
	case 0x59:
		DEBUG_OPERATION("POP");
		popv(); store_eCX();
		commit_EIP();
		break;
	case 0x5a:
		DEBUG_OPERATION("POP");
		popv(); store_eDX();
		commit_EIP();
		break;
	case 0x5b:
		DEBUG_OPERATION("POP");
		popv(); store_eBX();
		commit_EIP();
		break;
	case 0x5c:
		DEBUG_OPERATION("POP");
		popv(); store_eSP();
		commit_EIP();
		break;
	case 0x5d:
		DEBUG_OPERATION("POP");
		popv(); store_eBP();
		commit_EIP();
		break;
	case 0x5e:
		DEBUG_OPERATION("POP");
		popv(); store_eSI();
		commit_EIP();
		break;
	case 0x5f:
		DEBUG_OPERATION("POP");
		popv(); store_eDI();
		commit_EIP();
		break;
	case 0x60:
		DEBUG_OPERATION("PUSHA");
		pushav();
		commit_EIP();
		break;
	case 0x61:
		DEBUG_OPERATION("POPA");
		popa();
		commit_EIP();
		break;
	/* TODO */
	case 0x64:
		DEBUG_OPERATION("FS segment override prefix");
		prefix_segment_override = SEGMENT_FS;
		break;
	case 0x65:
		DEBUG_OPERATION("GS segment override prefix");
		prefix_segment_override = SEGMENT_GS;
		break;
	case 0x66:
		DEBUG_OPERATION("Operand-size override prefix");
		prefix_operand_size_override = 1;
		break;
	case 0x67:
		DEBUG_OPERATION("Address-size override prefix");
		prefix_address_size_override = 1;
		break;
	case 0x68:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("PUSH");
		load_Iv(); pushv();
		commit_EIP();
		break;
	case 0x69:
		fetch_modrm();
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("IMUL");
		load_Ev_Iv(); imul3vv(); store_Gv();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x6a:
		fetch_immb();
		DEBUG_OPERATION("PUSH");
		load_IbSE(); pushv();
		commit_EIP();
		break;
	case 0x6b:
		fetch_modrm();
		fetch_immb();
		DEBUG_OPERATION("IMUL");
		load_Ev_IbSE(); imul3vb(); store_Gv();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x6c:
		DEBUG_OPERATION("INS");
		insb();
		commit_EIP();
		break;
	case 0x6d:
		DEBUG_OPERATION("INS");
		insv();
		commit_EIP();
		break;
	case 0x6e:
		DEBUG_OPERATION("OUTS");
		outsb();
		commit_EIP();
		break;
	case 0x6f:
		DEBUG_OPERATION("OUTS");
		outsv();
		commit_EIP();
		break;
	case 0x70:
		fetch_immb();
		DEBUG_OPERATION("JO");
		load_Jb(); jo();
		commit_EIP();
		break;
	case 0x71:
		fetch_immb();
		DEBUG_OPERATION("JNO");
		load_Jb(); jno();
		commit_EIP();
		break;
	case 0x72:
		fetch_immb();
		DEBUG_OPERATION("JB");
		load_Jb(); jb();
		commit_EIP();
		break;
	case 0x73:
		fetch_immb();
		DEBUG_OPERATION("JNB");
		load_Jb(); jnb();
		commit_EIP();
		break;
	case 0x74:
		fetch_immb();
		DEBUG_OPERATION("JZ");
		load_Jb(); jz();
		commit_EIP();
		break;
	case 0x75:
		fetch_immb();
		DEBUG_OPERATION("JNZ");
		load_Jb(); jnz();
		commit_EIP();
		break;
	case 0x76:
		fetch_immb();
		DEBUG_OPERATION("JBE");
		load_Jb(); jbe();
		commit_EIP();
		break;
	case 0x77:
		fetch_immb();
		DEBUG_OPERATION("JNBE");
		load_Jb(); jnbe();
		commit_EIP();
		break;
	case 0x78:
		fetch_immb();
		DEBUG_OPERATION("JS");
		load_Jb(); js();
		commit_EIP();
		break;
	case 0x79:
		fetch_immb();
		DEBUG_OPERATION("JNS");
		load_Jb(); jns();
		commit_EIP();
		break;
	case 0x7a:
		fetch_immb();
		DEBUG_OPERATION("JP");
		load_Jb(); jp();
		commit_EIP();
		break;
	case 0x7b:
		fetch_immb();
		DEBUG_OPERATION("JNP");
		load_Jb(); jnp();
		commit_EIP();
		break;
	case 0x7c:
		fetch_immb();
		DEBUG_OPERATION("JL");
		load_Jb(); jl();
		commit_EIP();
		break;
	case 0x7d:
		fetch_immb();
		DEBUG_OPERATION("JNL");
		load_Jb(); jnl();
		commit_EIP();
		break;
	case 0x7e:
		fetch_immb();
		DEBUG_OPERATION("JLE");
		load_Jb(); jle();
		commit_EIP();
		break;
	case 0x7f:
		fetch_immb();
		DEBUG_OPERATION("JNLE");
		load_Jb(); jnle();
		commit_EIP();
		break;
	case 0x80:
		fetch_modrm();
		fetch_immb();
		DEBUG_OPERATION_GROUP1(instruction_reg);
		load_Eb_Ib();
		switch (instruction_reg) {
		default: DEBUG_ERROR_SWITCH();
		case 0: addb(); store_Eb(); commit_OSZAPC(); break;
		case 1: orb();  store_Eb(); commit_OSZAPC(); break;
		case 2: adcb(); store_Eb(); commit_OSZAPC(); break;
		case 3: sbbb(); store_Eb(); commit_OSZAPC(); break;
		case 4: andb(); store_Eb(); commit_OSZAPC(); break;
		case 5: subb(); store_Eb(); commit_OSZAPC(); break;
		case 6: xorb(); store_Eb(); commit_OSZAPC(); break;
		case 7: cmpb(); commit_OSZAPC(); break;
		}
		commit_EIP();
		break;
	case 0x81:
		fetch_modrm();
		fetch_immv(operand_size_32());
		DEBUG_OPERATION_GROUP1(instruction_reg);
		load_Ev_Iv();
		switch (instruction_reg) {
		default: DEBUG_ERROR_SWITCH();
		case 0: addv(); store_Ev(); commit_OSZAPC(); break;
		case 1: orv();  store_Ev(); commit_OSZAPC(); break;
		case 2: adcv(); store_Ev(); commit_OSZAPC(); break;
		case 3: sbbv(); store_Ev(); commit_OSZAPC(); break;
		case 4: andv(); store_Ev(); commit_OSZAPC(); break;
		case 5: subv(); store_Ev(); commit_OSZAPC(); break;
		case 6: xorv(); store_Ev(); commit_OSZAPC(); break;
		case 7: cmpv(); commit_OSZAPC(); break;
		}
		commit_EIP();
		break;
	case 0x82: /* FIXME: undocumented */
		fetch_modrm();
		fetch_immb();
		DEBUG_OPERATION_GROUP1(instruction_reg);
		load_Eb_Ib();
		switch (instruction_reg) {
		default: DEBUG_ERROR_SWITCH();
		case 0: addb(); store_Eb(); commit_OSZAPC(); break;
		case 1: orb();  store_Eb(); commit_OSZAPC(); break;
		case 2: adcb(); store_Eb(); commit_OSZAPC(); break;
		case 3: sbbb(); store_Eb(); commit_OSZAPC(); break;
		case 4: andb(); store_Eb(); commit_OSZAPC(); break;
		case 5: subb(); store_Eb(); commit_OSZAPC(); break;
		case 6: xorb(); store_Eb(); commit_OSZAPC(); break;
		case 7: cmpb(); commit_OSZAPC(); break;
		}
		commit_EIP();
		break;
	case 0x83:
		fetch_modrm();
		fetch_immb();
		DEBUG_OPERATION_GROUP1(instruction_reg);
		load_Ev_IbSE();
		switch (instruction_reg) {
		default: DEBUG_ERROR_SWITCH();
		case 0: addv(); store_Ev(); commit_OSZAPC(); break;
		case 1: orv();  store_Ev(); commit_OSZAPC(); break;
		case 2: adcv(); store_Ev(); commit_OSZAPC(); break;
		case 3: sbbv(); store_Ev(); commit_OSZAPC(); break;
		case 4: andv(); store_Ev(); commit_OSZAPC(); break;
		case 5: subv(); store_Ev(); commit_OSZAPC(); break;
		case 6: xorv(); store_Ev(); commit_OSZAPC(); break;
		case 7: cmpv(); commit_OSZAPC(); break;
		}
		commit_EIP();
		break;
	case 0x84:
		fetch_modrm();
		DEBUG_OPERATION("TEST");
		load_Eb_Gb(); testb();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x85:
		fetch_modrm();
		DEBUG_OPERATION("TEST");
		load_Ev_Gv(); testv();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0x86:
		fetch_modrm();
		DEBUG_OPERATION("XCHG");
		load_Eb_Gb(); xchg(); store_Eb_Gb();
		commit_EIP();
		break;
	case 0x87:
		fetch_modrm();
		DEBUG_OPERATION("XCHG");
		load_Ev_Gv(); xchg(); store_Ev_Gv();
		commit_EIP();
		break;
	case 0x88:
		fetch_modrm();
		DEBUG_OPERATION("MOV");
		load_Gb(); movb(); store_Eb();
		commit_EIP();
		break;
	case 0x89:
		fetch_modrm();
		DEBUG_OPERATION("MOV");
		load_Gv(); movv(); store_Ev();
		commit_EIP();
		break;
	case 0x8a:
		fetch_modrm();
		DEBUG_OPERATION("MOV");
		load_Eb(); movb(); store_Gb();
		commit_EIP();
		break;
	case 0x8b:
		fetch_modrm();
		DEBUG_OPERATION("MOV");
		load_Ev(); movv(); store_Gv();
		commit_EIP();
		break;
	case 0x8c:
		fetch_modrm();
		DEBUG_OPERATION("MOV");
		load_Sw(); movw(); store_Ev();
		commit_EIP();
		break;
	case 0x8d:
		fetch_modrm();
		DEBUG_OPERATION("LEA");
		load_effective_address(); lea(); store_Gv();
		commit_EIP();
		break;
	case 0x8e:
		fetch_modrm();
		DEBUG_OPERATION("MOV");
		load_Ew(); movw(); store_Sw();
		if (instruction_reg == SEGMENT_SS) {
			commit_EIP_and_delay_interrupts();
		} else {
			commit_EIP();
		}
		break;
	case 0x8f:
		fetch_modrm();
		DEBUG_OPERATION("POP");
		popv(); store_Ev();
		commit_EIP();
		break;
	case 0x90:
		DEBUG_OPERATION("NOP");
		/* TODO: hmm... */
		commit_EIP();
		break;
	case 0x91:
		DEBUG_OPERATION("XCHG");
		load_eAX_eCX(); xchg(); store_eAX_eCX();
		commit_EIP();
		break;
	case 0x92:
		DEBUG_OPERATION("XCHG");
		load_eAX_eDX(); xchg(); store_eAX_eDX();
		commit_EIP();
		break;
	case 0x93:
		DEBUG_OPERATION("XCHG");
		load_eAX_eBX(); xchg(); store_eAX_eBX();
		commit_EIP();
		break;
	case 0x94:
		DEBUG_OPERATION("XCHG");
		load_eAX_eSP(); xchg(); store_eAX_eSP();
		commit_EIP();
		break;
	case 0x95:
		DEBUG_OPERATION("XCHG");
		load_eAX_eBP(); xchg(); store_eAX_eBP();
		commit_EIP();
		break;
	case 0x96:
		DEBUG_OPERATION("XCHG");
		load_eAX_eSI(); xchg(); store_eAX_eSI();
		commit_EIP();
		break;
	case 0x97:
		DEBUG_OPERATION("XCHG");
		load_eAX_eDI(); xchg(); store_eAX_eDI();
		commit_EIP();
		break;
	case 0x98:
		if (operand_size_32()) {
			DEBUG_OPERATION("CWDE");
			cwde();
		} else {
			DEBUG_OPERATION("CBW");
			cbw();
		}
		commit_EIP();
		break;
	case 0x99:
		if (operand_size_32()) {
			DEBUG_OPERATION("CDQ");
			cdq();
		} else {
			DEBUG_OPERATION("CWD");
			cwd();
		}
		commit_EIP();
		break;
	case 0x9a:
		fetch_ptr();
		DEBUG_OPERATION("CALL");
		load_Ap(); call_far();
		commit_EIP();
		break;
	case 0x9b:
		DEBUG_OPERATION("WAIT");
		wait();
		commit_EIP();
		break;
	case 0x9c:
		DEBUG_OPERATION("PUSHF");
		pushf();
		commit_EIP();
		break;
	case 0x9d:
		DEBUG_OPERATION("POPF");
		popf();
		commit_EIP();
		break;
	case 0x9e:
		DEBUG_OPERATION("SAHF");
		sahf();
		commit_EIP();
		break;
	case 0x9f:
		DEBUG_OPERATION("LAHF");
		lahf();
		commit_EIP();
		break;
	case 0xa0:
		fetch_immv(address_size_32());
		DEBUG_OPERATION("MOV");
		load_Ob(); movb(); store_AL();
		commit_EIP();
		break;
	case 0xa1:
		fetch_immv(address_size_32());
		DEBUG_OPERATION("MOV");
		load_Ov(); movv(); store_eAX();
		commit_EIP();
		break;
	case 0xa2:
		fetch_immv(address_size_32());
		DEBUG_OPERATION("MOV");
		load_AL(); movb(); store_Ob();
		commit_EIP();
		break;
	case 0xa3:
		fetch_immv(address_size_32());
		DEBUG_OPERATION("MOV");
		load_eAX(); movv(); store_Ov();
		commit_EIP();
		break;
	case 0xa4:
		DEBUG_OPERATION("MOVS");
		movsb();
		commit_EIP();
		break;
	case 0xa5:
		DEBUG_OPERATION("MOVS");
		movsv();
		commit_EIP();
		break;
	case 0xa6:
		DEBUG_OPERATION("CMPS");
		cmpsb();
		commit_EIP();
		break;
	case 0xa7:
		DEBUG_OPERATION("CMPS");
		cmpsv();
		commit_EIP();
		break;
	case 0xa8:
		fetch_immb();
		DEBUG_OPERATION("TEST");
		load_AL_Ib(); testb();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0xa9:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("TEST");
		load_eAX_Iv(); testv();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0xaa:
		DEBUG_OPERATION("STOS");
		stosb();
		commit_EIP();
		break;;
	case 0xab:
		DEBUG_OPERATION("STOS");
		stosv();
		commit_EIP();
		break;;
	case 0xac:
		DEBUG_OPERATION("LODS");
		lodsb();
		commit_EIP();
		break;;
	case 0xad:
		DEBUG_OPERATION("LODS");
		lodsv();
		commit_EIP();
		break;
	case 0xae:
		DEBUG_OPERATION("SCAS");
		scasb();
		commit_EIP();
		break;
	case 0xaf:
		DEBUG_OPERATION("SCAS");
		scasv();
		commit_EIP();
		break;
	case 0xb0:
		fetch_immb();
		DEBUG_OPERATION("MOV");
		load_Ib(); movb(); store_AL();
		commit_EIP();
		break;
	case 0xb1:
		fetch_immb();
		DEBUG_OPERATION("MOV");
		load_Ib(); movb(); store_CL();
		commit_EIP();
		break;
	case 0xb2:
		fetch_immb();
		DEBUG_OPERATION("MOV");
		load_Ib(); movb(); store_DL();
		commit_EIP();
		break;
	case 0xb3:
		fetch_immb();
		DEBUG_OPERATION("MOV");
		load_Ib(); movb(); store_BL();
		commit_EIP();
		break;
	case 0xb4:
		fetch_immb();
		DEBUG_OPERATION("MOV");
		load_Ib(); movb(); store_AH();
		commit_EIP();
		break;
	case 0xb5:
		fetch_immb();
		DEBUG_OPERATION("MOV");
		load_Ib(); movb(); store_CH();
		commit_EIP();
		break;
	case 0xb6:
		fetch_immb();
		DEBUG_OPERATION("MOV");
		load_Ib(); movb(); store_DH();
		commit_EIP();
		break;
	case 0xb7:
		fetch_immb();
		DEBUG_OPERATION("MOV");
		load_Ib(); movb(); store_BH();
		commit_EIP();
		break;
	case 0xb8:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("MOV");
		load_Iv(); movv(); store_eAX();
		commit_EIP();
		break;
	case 0xb9:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("MOV");
		load_Iv(); movv(); store_eCX();
		commit_EIP();
		break;
	case 0xba:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("MOV");
		load_Iv(); movv(); store_eDX();
		commit_EIP();
		break;
	case 0xbb:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("MOV");
		load_Iv(); movv(); store_eBX();
		commit_EIP();
		break;
	case 0xbc:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("MOV");
		load_Iv(); movv(); store_eSP();
		commit_EIP();
		break;
	case 0xbd:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("MOV");
		load_Iv(); movv(); store_eBP();
		commit_EIP();
		break;
	case 0xbe:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("MOV");
		load_Iv(); movv(); store_eSI();
		commit_EIP();
		break;
	case 0xbf:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("MOV");
		load_Iv(); movv(); store_eDI();
		commit_EIP();
		break;
	case 0xc0:
		fetch_modrm();
		fetch_immb();
		DEBUG_OPERATION_GROUP2(instruction_reg);
		load_Eb_Ib();
		switch (instruction_reg) {
		default: DEBUG_ERROR_SWITCH();
		case 0: rolb(); store_Eb(); commit_OF(); commit_CF(); break;
		case 1: rorb(); store_Eb(); commit_OF(); commit_CF(); break;
		case 2: rclb(); store_Eb(); commit_OF(); commit_CF(); break;
		case 3: rcrb(); store_Eb(); commit_OF(); commit_CF(); break;
		case 4: shlb(); store_Eb(); commit_OSZAPC(); break;
		case 5: shrb(); store_Eb(); commit_OSZAPC(); break;
		case 6: ERROR_NYI(); /* salb()/shlb() or error? */ break;
		case 7: sarb(); store_Eb(); commit_OSZAPC(); break;
		}
		commit_EIP();
		break;
	case 0xc1:
		fetch_modrm();
		fetch_immb();
		DEBUG_OPERATION_GROUP2(instruction_reg);
		load_Ev_Ib();
		switch (instruction_reg) {
		default: DEBUG_ERROR_SWITCH();
		case 0: rolv(); store_Ev(); commit_OF(); commit_CF(); break;
		case 1: rorv(); store_Ev(); commit_OF(); commit_CF(); break;
		case 2: rclv(); store_Ev(); commit_OF(); commit_CF(); break;
		case 3: rcrv(); store_Ev(); commit_OF(); commit_CF(); break;
		case 4: shlv(); store_Ev(); commit_OSZAPC(); break;
		case 5: shrv(); store_Ev(); commit_OSZAPC(); break;
		case 6: ERROR_NYI(); /* salv()/shlv() or error? */ break;
		case 7: sarv(); store_Ev(); commit_OSZAPC(); break;
		}
		commit_EIP();
		break;
	case 0xc2:
		fetch_immv(0);
		DEBUG_OPERATION("RET");
		load_Iw(); ret_near();
		commit_EIP();
		break;
	case 0xc3:
		DEBUG_OPERATION("RET");
		load_0(); ret_near();
		commit_EIP();
		break;
	case 0xc4:
		fetch_modrm();
		DEBUG_OPERATION("LES");
		load_Mp(); les(); store_Gv();
		commit_EIP();
		break;
	case 0xc5:
		fetch_modrm();
		DEBUG_OPERATION("LDS");
		load_Mp(); lds(); store_Gv();
		commit_EIP();
		break;
	case 0xc6:
		fetch_modrm();
		fetch_immb();
		DEBUG_OPERATION("MOV");
		load_Ib(); movb(); store_Eb();
		commit_EIP();
		break;
	case 0xc7:
		fetch_modrm();
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("MOV");
		load_Iv(); movv(); store_Ev();
		commit_EIP();
		break;
	/* TODO */
	case 0xc9:
		DEBUG_OPERATION("LEAVE");
		leave();
		commit_EIP();
		break;
	case 0xca:
		fetch_immv(0);
		DEBUG_OPERATION("RET");
		load_Iw(); ret_far();
		commit_EIP();
		break;
	case 0xcb:
		DEBUG_OPERATION("RET");
		load_0(); ret_far();
		commit_EIP();
		break;
	case 0xcc:
		DEBUG_OPERATION("INT3");
		int3();
		commit_EIP();
		break;
	case 0xcd:
		fetch_immb();
		DEBUG_OPERATION("INTn");
		load_Ib(); intn();
		commit_EIP();
		break;
	case 0xce:
		DEBUG_OPERATION("INTO");
		into();
		commit_EIP();
		break;
	case 0xcf:
		DEBUG_OPERATION("IRET");
		iret();
		commit_EIP();
		break;
	case 0xd0:
		fetch_modrm();
		DEBUG_OPERATION_GROUP2(instruction_reg);
		load_Eb_1();
		switch (instruction_reg) {
		default: DEBUG_ERROR_SWITCH();
		case 0: rolb(); store_Eb(); commit_OF(); commit_CF(); break;
		case 1: rorb(); store_Eb(); commit_OF(); commit_CF(); break;
		case 2: rclb(); store_Eb(); commit_OF(); commit_CF(); break;
		case 3: rcrb(); store_Eb(); commit_OF(); commit_CF(); break;
		case 4: shlb(); store_Eb(); commit_OSZAPC(); break;
		case 5: shrb(); store_Eb(); commit_OSZAPC(); break;
		case 6: ERROR_NYI(); /* salb()/shlb() or error? */ break;
		case 7: sarb(); store_Eb(); commit_OSZAPC(); break;
		}
		commit_EIP();
		break;
	case 0xd1:
		fetch_modrm();
		DEBUG_OPERATION_GROUP2(instruction_reg);
		load_Ev_1();
		switch (instruction_reg) {
		default: DEBUG_ERROR_SWITCH();
		case 0: rolv(); store_Ev(); commit_OF(); commit_CF(); break;
		case 1: rorv(); store_Ev(); commit_OF(); commit_CF(); break;
		case 2: rclv(); store_Ev(); commit_OF(); commit_CF(); break;
		case 3: rcrv(); store_Ev(); commit_OF(); commit_CF(); break;
		case 4: shlv(); store_Ev(); commit_OSZAPC(); break;
		case 5: shrv(); store_Ev(); commit_OSZAPC(); break;
		case 6: ERROR_NYI(); /* salv()/shlv() or error? */ break;
		case 7: sarv(); store_Ev(); commit_OSZAPC(); break;
		}
		commit_EIP();
		break;
	case 0xd2:
		fetch_modrm();
		DEBUG_OPERATION_GROUP2(instruction_reg);
		load_Eb_CL();
		switch (instruction_reg) {
		default: DEBUG_ERROR_SWITCH();
		case 0: rolb(); store_Eb(); commit_OF(); commit_CF(); break;
		case 1: rorb(); store_Eb(); commit_OF(); commit_CF(); break;
		case 2: rclb(); store_Eb(); commit_OF(); commit_CF(); break;
		case 3: rcrb(); store_Eb(); commit_OF(); commit_CF(); break;
		case 4: shlb(); store_Eb(); commit_OSZAPC(); break;
		case 5: shrb(); store_Eb(); commit_OSZAPC(); break;
		case 6: ERROR_NYI(); /* salb()/shlb() or error? */ break;
		case 7: sarb(); store_Eb(); commit_OSZAPC(); break;
		}
		commit_EIP();
		break;
	case 0xd3:
		fetch_modrm();
		DEBUG_OPERATION_GROUP2(instruction_reg);
		load_Ev_CL();
		switch (instruction_reg) {
		default: DEBUG_ERROR_SWITCH();
		case 0: rolv(); store_Ev(); commit_OF(); commit_CF(); break;
		case 1: rorv(); store_Ev(); commit_OF(); commit_CF(); break;
		case 2: rclv(); store_Ev(); commit_OF(); commit_CF(); break;
		case 3: rcrv(); store_Ev(); commit_OF(); commit_CF(); break;
		case 4: shlv(); store_Ev(); commit_OSZAPC(); break;
		case 5: shrv(); store_Ev(); commit_OSZAPC(); break;
		case 6: ERROR_NYI(); /* salv()/shlv() or error? */ break;
		case 7: sarv(); store_Ev(); commit_OSZAPC(); break;
		}
		commit_EIP();
		break;
	case 0xd4:
		fetch_immb();
		DEBUG_OPERATION("AAM");
		load_Ib(); aam();
		commit_EIP(); commit_OSZAPC();
		break;
	case 0xd5:
		fetch_immb();
		DEBUG_OPERATION("AAD");
		load_Ib(); aad();
		commit_EIP(); commit_OSZAPC();
		break;
	/* TODO */
	case 0xd7:
		DEBUG_OPERATION("XLAT");
		xlat();
		commit_EIP();
		break;
	case 0xd8:
	case 0xd9:
	case 0xda:
	case 0xdb:
	case 0xdc:
	case 0xdd:
	case 0xde:
	case 0xdf:
		fetch_modrm();
		DEBUG_OPERATION("ESC");
		if (get_EM()) {
			exception(EXCEPTION_NM, -1);
		}
		commit_EIP();
		break;
	case 0xe0:
		fetch_immb();
		DEBUG_OPERATION("LOOPNE");
		load_Jb(); loopne();
		commit_EIP();
		break;
	case 0xe1:
		fetch_immb();
		DEBUG_OPERATION("LOOPE");
		load_Jb(); loope();
		commit_EIP();
		break;
	case 0xe2:
		fetch_immb();
		DEBUG_OPERATION("LOOP");
		load_Jb(); loop();
		commit_EIP();
		break;
	case 0xe3:
		fetch_immb();
		if (address_size_32()) {
			DEBUG_OPERATION("JECXZ");
			load_Jb(); jecxz();
		} else {
			DEBUG_OPERATION("JCXZ");
			load_Jb(); jcxz();
		}
		commit_EIP();
		break;
	case 0xe4:
		fetch_immb();
		DEBUG_OPERATION("IN");
		load_Ib(); inb(); store_AL();
		commit_EIP();
		break;
	case 0xe5:
		fetch_immb();
		DEBUG_OPERATION("IN");
		load_Ib(); inv(); store_eAX();
		commit_EIP();
		break;
	case 0xe6:
		fetch_immb();
		DEBUG_OPERATION("OUT");
		load_Ib_AL(); outb();
		commit_EIP();
		break;
	case 0xe7:
		fetch_immb();
		DEBUG_OPERATION("OUT");
		load_Ib_eAX(); outv();
		commit_EIP();
		break;
	case 0xe8:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("CALL");
		load_Jv(); call_near();
		commit_EIP();
		break;
	case 0xe9:
		fetch_immv(operand_size_32());
		DEBUG_OPERATION("JMP");
		load_Jv(); jmp_near();
		commit_EIP();
		break;
	case 0xea:
		fetch_ptr();
		DEBUG_OPERATION("JMP");
		load_Ap(); jmp_far();
		commit_EIP();
		break;
	case 0xeb:
		fetch_immb();
		DEBUG_OPERATION("JMP");
		load_Jb(); jmp_near();
		commit_EIP();
		break;
	case 0xec:
		DEBUG_OPERATION("IN");
		load_DX(); inb(); store_AL();
		commit_EIP();
		break;
	case 0xed:
		DEBUG_OPERATION("IN");
		load_DX(); inv(); store_eAX();
		commit_EIP();
		break;
	case 0xee:
		DEBUG_OPERATION("OUT");
		load_DX_AL(); outb();
		commit_EIP();
		break;
	case 0xef:
		DEBUG_OPERATION("OUT");
		load_DX_eAX(); outv();
		commit_EIP();
		break;
	case 0xf0:
		DEBUG_OPERATION("LOCK prefix");
		lock();
		prefix_lock_repeat = LR_LOCK;
		break;
	case 0xf1:
		DEBUG_OPERATION("ICEBP");
		icebp();
		commit_EIP();
		break;
	case 0xf2:
		DEBUG_OPERATION("REPNE/REPNZ prefix");
		if (prefix_lock_repeat == LR_LOCK) {
			unlock();
		}
		prefix_lock_repeat = LR_REPNZ;
		break;
	case 0xf3:
		DEBUG_OPERATION("REP/REPE/REPZ prefix");
		if (prefix_lock_repeat == LR_LOCK) {
			unlock();
		}
		prefix_lock_repeat = LR_REPZ;
		break;
	case 0xf4:
		DEBUG_OPERATION("HLT");
		hlt();
		commit_EIP();
		break;
	case 0xf5:
		DEBUG_OPERATION("CMC");
		cmc();
		commit_EIP();
		break;
	case 0xf6:
		fetch_modrm();
		switch (instruction_reg) {
		default: DEBUG_ERROR_SWITCH();
		case 0:
			fetch_immb();
			DEBUG_OPERATION("TEST");
			load_Eb_Ib(); testb();
			commit_OSZAPC();
			break;
		case 1:
			/* FIXME: RESERVED */
			ERROR_NYI();
			break;
		case 2:
			DEBUG_OPERATION("NOT");
			load_Eb(); notb(); store_Eb();
			break;
		case 3:
			DEBUG_OPERATION("NEG");
			load_Eb(); negb(); store_Eb();
			commit_OSZAPC();
			break;
		case 4:
			DEBUG_OPERATION("MUL");
			load_Eb(); mulb();
			commit_OSZAPC();
			break;
		case 5:
			DEBUG_OPERATION("IMUL");
			load_Eb(); imul1b();
			commit_OSZAPC();
			break;
		case 6:
			DEBUG_OPERATION("DIV");
			load_Eb(); divb();
			commit_OSZAPC();
			break;
		case 7:
			DEBUG_OPERATION("IDIV");
			load_Eb(); idivb();
			commit_OSZAPC();
			break;
		}
		commit_EIP();
		break;
	case 0xf7:
		fetch_modrm();
		switch (instruction_reg) {
		default: DEBUG_ERROR_SWITCH();
		case 0:
			fetch_immv(operand_size_32());
			DEBUG_OPERATION("TEST");
			load_Ev_Iv(); testv();
			commit_OSZAPC();
			break;
		case 1:
			/* FIXME: RESERVED */
			ERROR_NYI();
			break;
		case 2:
			DEBUG_OPERATION("NOT");
			load_Ev(); notv(); store_Ev();
			break;
		case 3:
			DEBUG_OPERATION("NEG");
			load_Ev(); negv(); store_Ev();
			commit_OSZAPC();
			break;
		case 4:
			DEBUG_OPERATION("MUL");
			load_Ev(); mulv();
			commit_OSZAPC();
			break;
		case 5:
			DEBUG_OPERATION("IMUL");
			load_Ev(); imul1v();
			commit_OSZAPC();
			break;
		case 6:
			DEBUG_OPERATION("DIV");
			load_Ev(); divv();
			commit_OSZAPC();
			break;
		case 7:
			DEBUG_OPERATION("IDIV");
			load_Ev(); idivv();
			commit_OSZAPC();
			break;
		}
		commit_EIP();
		break;
	case 0xf8:
		DEBUG_OPERATION("CLC");
		clc();
		commit_CF(); commit_EIP();
		break;
	case 0xf9:
		DEBUG_OPERATION("STC");
		stc();
		commit_CF(); commit_EIP();
		break;
	case 0xfa:
		DEBUG_OPERATION("CLI");
		cli();
		commit_IF(); commit_EIP();
		break;
	case 0xfb:
		DEBUG_OPERATION("STI");
		sti();
		commit_IF(); commit_EIP_and_delay_interrupts();
		break;
	case 0xfc:
		DEBUG_OPERATION("CLD");
		cld();
		commit_DF(); commit_EIP();
		break;
	case 0xfd:
		DEBUG_OPERATION("STD");
		std();
		commit_DF(); commit_EIP();
		break;
	case 0xfe:
		fetch_modrm();
		DEBUG_OPERATION_GROUP4(instruction_reg);
		load_Eb();
		switch (instruction_reg) {
		default: DEBUG_ERROR_SWITCH();
		case 0: incb(); commit_OSZAP(); break;
		case 1: decb(); commit_OSZAP(); break;
		case 2:
		case 3:
		case 4:
		case 5:
		case 6:
		case 7:
			/* TODO reserved */
			ERROR_NYI();
		}
		store_Eb();
		commit_EIP();
		break;
	case 0xff:
		fetch_modrm();
		switch (instruction_reg) {
		default: DEBUG_ERROR_SWITCH();
		case 0:
			DEBUG_OPERATION("INC");
			load_Ev(); incv(); store_Ev();
			commit_OSZAP();
			break;
		case 1:
			DEBUG_OPERATION("DEC");
			load_Ev(); decv(); store_Ev();
			commit_OSZAP();
			break;
		case 2:
			DEBUG_OPERATION("CALL");
			load_Ev();
			call_near_absolute();
			break;
		case 3:
			DEBUG_OPERATION("CALL");
			load_Ep(); call_far();
			break;
		case 4:
			DEBUG_OPERATION("JMP");
			load_Ev();
			jmp_near_absolute();
			break;
		case 5:
			DEBUG_OPERATION("JMP");
			load_Ep(); jmp_far();
			break;
		case 6:
			DEBUG_OPERATION("PUSH");
			load_Ev(); pushv();
			break;
		case 7:
			/* TODO reserved */
			ERROR_NYI();
		}
		commit_EIP();
		break;
	}
}

/******************************************************************************/

void
reset(void)
{
	exception_vector = EXCEPTION_NONE;
	exception_error_code = -1;
	exception_is_interrupt = 0;
	exception_double_page_fault = 0;

	prefix_clear();

	eax = 0x00000000;
	ebx = 0x00000000;
	ecx = 0x00000000;
	edx = 0; /* FIXME: cpu version information */
	ebp = 0x00000000;
	esi = 0x00000000;
	edi = 0x00000000;
	esp = 0x00000000;

	eflag_cf = 0;
	eflag_pf = 0;
	eflag_af = 0;
	eflag_zf = 0;
	eflag_sf = 0;
	eflag_tf = 0;
	eflag_if = 0;
	eflag_df = 0;
	eflag_of = 0;
	eflag_iopl0 = 0;
	eflag_iopl1 = 0;
	eflag_nt = 0;
	eflag_rf = 0;
	eflag_vm = 0;

	eip = eip_new = 0x0000fff0;

	/* NW(29) and CD(30) enabled; ET(4) disabled (no math coprocessor) */
	set_CR0(0x60000000);

	set_CR2(0);
	set_CR3(0);

	cs_selector		= 0xf000;
	cs_type			= SEGMENT_CODE_EXEC_READ_ACCESSED;
	cs_sflag		= 1;
	cs_dpl			= 0;
	cs_pflag		= 1;
	cs_segment_limit	= 0xffff;
	cs_segment_base		= 0xffff0000;
	cs_segment_dflag	= 0;
	cs_segment_avlflag	= 0;

	ss_selector		= 0x0000;
	ss_type			= SEGMENT_DATA_READ_WRITE_ACCESSED;
	ss_sflag		= 1;
	ss_dpl			= 0;
	ss_pflag		= 1;
	ss_segment_limit	= 0xffff;
	ss_segment_base		= 0x00000000;
	ss_segment_dflag	= 0;
	ss_segment_avlflag	= 0;

	ds_selector		= 0x0000;
	ds_type			= SEGMENT_DATA_READ_WRITE_ACCESSED;
	ds_sflag		= 1;
	ds_dpl			= 0;
	ds_pflag		= 1;
	ds_segment_limit	= 0xffff;
	ds_segment_base		= 0x00000000;
	ds_segment_dflag	= 0;
	ds_segment_avlflag	= 0;

	es_segment_base		= ds_selector;
	es_type			= ds_type;
	es_sflag		= ds_sflag;
	es_dpl			= ds_dpl;
	es_pflag		= ds_pflag;
	es_segment_limit	= ds_segment_limit;
	es_segment_base		= ds_segment_base;
	es_segment_dflag	= ds_segment_dflag;
	es_segment_avlflag	= ds_segment_avlflag;

	fs_selector		= ds_selector;
	fs_type			= ds_type;
	fs_sflag		= ds_sflag;
	fs_dpl			= ds_dpl;
	fs_pflag		= ds_pflag;
	fs_segment_limit	= ds_segment_limit;
	fs_segment_base		= ds_segment_base;
	fs_segment_dflag	= ds_segment_dflag;
	fs_segment_avlflag	= ds_segment_avlflag;

	gs_selector		= ds_selector;
	gs_type			= ds_type;
	gs_sflag		= ds_sflag;
	gs_dpl			= ds_dpl;
	gs_pflag		= ds_pflag;
	gs_segment_limit	= ds_segment_limit;
	gs_segment_base		= ds_segment_base;
	gs_segment_dflag	= ds_segment_dflag;
	gs_segment_avlflag	= ds_segment_avlflag;

	gdtr_base = 0x00000000;
	gdtr_limit = 0xffff;

	idtr_base = 0x00000000;
	idtr_limit = 0xffff;

	ldtr_selector = 0x0000;
	ldtr_system_limit = 0xffff;
	ldtr_system_base = 0x00000000;

	tr_selector = 0x0000;
	tr_type = SEGMENT_32BIT_BUSY_TSS;
	tr_system_limit = 0xffff;
	tr_system_base = 0x00000000;

	interrupt_pending = 0;
	interrupt_delay = 0;
	halt_state = 0;
}

void
irq_set(unsigned int value)
{
	DEBUG("\n *** IRQ %u ***\n", value);

	interrupt_pending = !!value;
}

void
nmi_set(unsigned int value)
{
	DEBUG("\n *** NMI %u ***\n", value);
	ERROR_NYI(); /* FIXME */
}


/******************************************************************************
 *
 * Print State
 *
 ******************************************************************************/

void
print_state(FILE *f)
{
	int i;

	fprintf(f, "\n");
	fprintf(f, "**************** CPU STATE ****************\n\n");

	fprintf(f, "EAX=0x%08x  ECX=0x%08x  EDX=0x%08x  EBX=0x%08x\n",
		eax, ecx, edx, ebx);
	fprintf(f, "ESP=0x%08x  EBP=0x%08x  ESI=0x%08x  EDI=0x%08x\n\n",
		esp, ebp, esi, edi);

	fprintf(f, "EIP=0x%08x (0x%08x:0x%08x)  EFLAGS=0x%08x\n\n",
		eip + cs_segment_base, cs_segment_base, eip, get_EFLAGS32());

	fprintf(f, "CF=%d    PF=%d    AF=%d    ZF=%d\n",
		eflag_cf, eflag_pf, eflag_af, eflag_zf);
	fprintf(f, "SF=%d    TF=%d    IF=%d    DF=%d\n",
		eflag_sf, eflag_tf, eflag_if, eflag_df);
	fprintf(f, "OF=%d  IOPL=%d    NT=%d    RF=%d\n",
		eflag_of, get_IOPL(), eflag_nt, eflag_rf);
	fprintf(f, "VM=%d", eflag_vm);

	fprintf(f, "CR0=0x%08x CR2=0x%08x CR3=0x%08x\n\n",
		get_CR0(), get_CR2(), get_CR3());

	fprintf(f, "PE=%d    MP=%d    EM=%d    TS=%d\n",
		cr0_pe, cr0_mp, cr0_em, cr0_ts);
	fprintf(f, "ET=%d    PG=%d\n", cr0_et, cr0_pg);

	fprintf(f, "         CS=0x%04x  SS=0x%04x  DS=0x%04x  "
		            "ES=0x%04x  FS=0x%04x  GS=0x%04x\n",
		cs_selector, ss_selector, ds_selector,
		es_selector, fs_selector, gs_selector);
	fprintf(f, "limit:  0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
		cs_segment_limit, ss_segment_limit, ds_segment_limit,
		es_segment_limit, fs_segment_limit, gs_segment_limit);
	fprintf(f, "scaled: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
		cs_segment_limit, ss_segment_limit, ds_segment_limit,
		es_segment_limit, fs_segment_limit, gs_segment_limit);
	fprintf(f, "base:   0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
		cs_segment_base, ss_segment_base, ds_segment_base,
		es_segment_base, fs_segment_base, gs_segment_base);
	fprintf(f, "type:     %8x   %8x   %8x   %8x   %8x   %8x\n",
		cs_type, ss_type, ds_type,
		es_type, fs_type, gs_type);
	fprintf(f, "dpl:      %8x   %8x   %8x   %8x   %8x   %8x\n",
		cs_dpl, ss_dpl, ds_dpl,
		es_dpl, fs_dpl, gs_dpl);
	fprintf(f, "sflag:    %8x   %8x   %8x   %8x   %8x   %8x\n",
		cs_sflag, ss_sflag, ds_sflag,
		es_sflag, fs_sflag, gs_sflag);
	fprintf(f, "pflag:    %8x   %8x   %8x   %8x   %8x   %8x\n",
		cs_pflag, ss_pflag, ds_pflag,
		es_pflag, fs_pflag, gs_pflag);
	fprintf(f, "dflag:    %8x   %8x   %8x   %8x   %8x   %8x\n",
		cs_segment_dflag, ss_segment_dflag, ds_segment_dflag,
		es_segment_dflag, fs_segment_dflag, gs_segment_dflag);
	fprintf(f, "avlflag:  %8x   %8x   %8x   %8x   %8x   %8x\n\n",
		cs_segment_avlflag, ss_segment_avlflag, ds_segment_avlflag,
		es_segment_avlflag, fs_segment_avlflag, gs_segment_avlflag);
	fprintf(f, "gdtr_limit=0x%08x  gdtr_base=0x%08x\n",
		gdtr_limit, gdtr_base);
	fprintf(f, "idtr_limit=0x%08x  idtr_base=0x%08x\n\n",
		idtr_limit, idtr_base);

	fprintf(f, "ldtr_selector=0x%04x  ldtr_system_limit=0x%08x\n",
		ldtr_selector, ldtr_system_limit);
	fprintf(f, "ldtr_system_base=0x%08x\n\n", ldtr_system_base);

	fprintf(f, "tr_selector=0x%04x      tr_type=%d\n",
		tr_selector, tr_type);
	fprintf(f, "tr_system_limit=0x%08x  tr_system_base=0x%08x\n\n",
		tr_system_limit, tr_system_base);

	fprintf(f, "t0=0x%08x  t1=0x%08x  t2=0x%08x  t3=0x%08x  tj=%d\n\n",
		t0, t1, t2, t3, tj);

	fprintf(f, "Prefix operand size override: %d\n",
		prefix_operand_size_override);
	fprintf(f, "Prefix address size override: %d\n",
		prefix_address_size_override);
	fprintf(f, "Prefix segment override:      %d\n",
		prefix_segment_override);
	fprintf(f, "Prefix lock repeat:           %d\n\n",
		prefix_lock_repeat);

	fprintf(f, "Instruction opcode:       0x%02x\n", instruction_opcode);
	fprintf(f, "Instruction opcode2:      0x%02x\n", instruction_opcode2);
	fprintf(f, "Instruction mod:          0x%x\n", instruction_mod);
	fprintf(f, "Instruction reg:          0x%x\n", instruction_reg);
	fprintf(f, "Instruction rm:           0x%x\n", instruction_rm);
	fprintf(f, "Instruction scale:        0x%x\n", instruction_scale);
	fprintf(f, "Instruction index:        0x%x\n", instruction_index);
	fprintf(f, "Instruction base:         0x%x\n", instruction_base);
	fprintf(f, "Instruction displacement: 0x%x\n",
		instruction_displacement);
	fprintf(f, "Instruction immediate:    0x%x\n\n",
		instruction_immediate);

	fprintf(f, "**************** MEMORY DUMP ****************\n\n");

	for (i = -16; i < 16; i++) {
		uint32_t esp_addr, eip_addr;

		esp_addr = (esp & ~3) + 4 * i;
		eip_addr = (eip & ~3) + 4 * i;

		fprintf(f, "0x%08x: 0x%08x    0x%08x: 0x%08x\n",
			esp_addr, mrd(esp_addr, 0), eip_addr, mrd(eip_addr, 0));
	}

	fprintf(f, "\n");
}

