vbox的更動 37689 路徑 trunk/src/recompiler

時間撮記:

2011-6-29 下午04:01:23 (13 年 以前)

作者:

vboxsync

訊息:

recompiler: Merged in changes from 0.13.0.

位置:

trunk/src/recompiler

檔案:

• README.vbox (修改) (1 筆差異)
• Sun/config-host.h (修改) (1 筆差異)
• VBoxRecompiler.c (修改) (30 筆差異)
• bswap.h (修改) (2 筆差異)
• cpu-all.h (修改) (12 筆差異)
• cpu-common.h (修改) (5 筆差異)
• cpu-defs.h (修改) (6 筆差異)
• cpu-exec.c (修改) (31 筆差異)
• cutils.c (修改) (2 筆差異)
• def-helper.h (修改) (5 筆差異)
• disas.h (修改) (1 筆差異)
• dyngen-exec.h (修改) (1 筆差異)
• elf.h (修改) (2 筆差異)
• exec-all.h (修改) (20 筆差異)
• exec.c (修改) (103 筆差異)
• fpu/softfloat-native.c (修改) (3 筆差異)
• fpu/softfloat-native.h (修改) (5 筆差異)
• fpu/softfloat-specialize.h (修改) (2 筆差異)
• fpu/softfloat.c (修改) (9 筆差異)
• fpu/softfloat.h (修改) (3 筆差異)
• gen-icount.h (修改) (1 筆差異)
• ioport.h (修改) (1 筆差異)
• qemu-barrier.h (新增)
• qemu-common.h (修改) (10 筆差異)
• qemu-log.h (修改) (1 筆差異)
• qemu-timer.h (新增)
• softmmu_exec.h (修改) (1 筆差異)
• softmmu_header.h (修改) (1 筆差異)
• softmmu_template.h (修改) (9 筆差異)
• target-i386/TODO (新增)
• target-i386/cpu.h (修改) (11 筆差異)
• target-i386/exec.h (修改) (3 筆差異)
• target-i386/helper.c (修改) (16 筆差異)
• target-i386/op_helper.c (修改) (10 筆差異)
• target-i386/ops_sse_header.h (修改) (1 筆差異)
• target-i386/translate.c (修改) (28 筆差異)
• targphys.h (修改) (1 筆差異)
• tcg-runtime.c (修改) (1 筆差異)
• tcg/README (修改) (11 筆差異)
• tcg/TODO (修改) (1 筆差異)
• tcg/i386/tcg-target.c (修改) (29 筆差異)
• tcg/i386/tcg-target.h (修改) (3 筆差異)
• tcg/tcg-op.h (修改) (59 筆差異)
• tcg/tcg-opc.h (修改) (2 筆差異)
• tcg/tcg-runtime.h (修改) (1 筆差異)
• tcg/tcg.c (修改) (32 筆差異)
• tcg/tcg.h (修改) (6 筆差異)
• tcg/x86_64 (刪除)
• tests/Makefile (修改) (1 筆差異)
• tests/qruncom.c (修改) (1 筆差異)
• tests/runcom.c (修改) (4 筆差異)
• tests/sha1.c (修改) (5 筆差異)
• translate-all.c (修改) (3 筆差異)

trunk/src/recompiler/README.vbox

@@ -1 @@
-QEMU is based on v0.11.1 (35bfc7324e2e6946c4113ada5db30553a1a7c40b)
+QEMU is based on v0.13.0 (6ed912999d6ef636a5be5ccb266d7d3c0f0310b4)
 from git://git.savannah.nongnu.org/qemu.git.
 

trunk/src/recompiler/Sun/config-host.h

@@ -41 +41 @@
 # endif
 #endif
-#define QEMU_VERSION "0.12.5"
+#define QEMU_VERSION "0.13.0"
 #define CONFIG_UNAME_RELEASE ""
 #define CONFIG_QEMU_SHAREDIR "."

trunk/src/recompiler/VBoxRecompiler.c

@@ -26 +26 @@
 #include "cpu.h"
 #include "exec-all.h"
+#include "ioport.h"
 
 #include <VBox/vmm/rem.h>
@@ -60 +61 @@
 extern void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4);
 extern void tlb_flush_page(CPUX86State *env, target_ulong addr);
-extern void tlb_flush(CPUState *env, int flush_global);
+extern void tlb_flush(CPUX86State *env, int flush_global);
 extern void sync_seg(CPUX86State *env1, int seg_reg, int selector);
 extern void sync_ldtr(CPUX86State *env1, int selector);
@@ -466 +467 @@
 }
 
-
-/**
- * Initializes phys_ram_dirty and phys_ram_dirty_size.
+/**
+ * Initializes ram_list.phys_dirty and ram_list.phys_dirty_size.
  *
  * @returns VBox status code.
@@ -478 +478 @@
     int      rc = VINF_SUCCESS;
     RTGCPHYS cb;
+
+    AssertLogRelReturn(QLIST_EMPTY(&ram_list.blocks), VERR_INTERNAL_ERROR_2);
 
     cb = pVM->rem.s.GCPhysLastRam + 1;
@@ -483 +485 @@
                           ("GCPhysLastRam=%RGp - out of range\n", pVM->rem.s.GCPhysLastRam),
                           VERR_OUT_OF_RANGE);
-    phys_ram_dirty_size = cb >> PAGE_SHIFT;
-    AssertMsg(((RTGCPHYS)phys_ram_dirty_size << PAGE_SHIFT) == cb, ("%RGp\n", cb));
+
+    ram_list.phys_dirty_size = cb >> PAGE_SHIFT;
+    AssertMsg(((RTGCPHYS)ram_list.phys_dirty_size << PAGE_SHIFT) == cb, ("%RGp\n", cb));
 
     if (!fGuarded)
     {
-        phys_ram_dirty = MMR3HeapAlloc(pVM, MM_TAG_REM, phys_ram_dirty_size);
-        AssertLogRelMsgReturn(phys_ram_dirty, ("Failed to allocate %u bytes of dirty page map bytes\n", phys_ram_dirty_size), VERR_NO_MEMORY);
+        ram_list.phys_dirty = MMR3HeapAlloc(pVM, MM_TAG_REM, ram_list.phys_dirty_size);
+        AssertLogRelMsgReturn(ram_list.phys_dirty, ("Failed to allocate %u bytes of dirty page map bytes\n", ram_list.phys_dirty_size), VERR_NO_MEMORY);
     }
     else
@@ -496 +499 @@
          * Fill it up the nearest 4GB RAM and leave at least _64KB of guard after it.
          */
-        uint32_t cbBitmapAligned = RT_ALIGN_32(phys_ram_dirty_size, PAGE_SIZE);
-        uint32_t cbBitmapFull    = RT_ALIGN_32(phys_ram_dirty_size, (_4G >> PAGE_SHIFT));
+        uint32_t cbBitmapAligned = RT_ALIGN_32(ram_list.phys_dirty_size, PAGE_SIZE);
+        uint32_t cbBitmapFull    = RT_ALIGN_32(ram_list.phys_dirty_size, (_4G >> PAGE_SHIFT));
         if (cbBitmapFull == cbBitmapAligned)
             cbBitmapFull += _4G >> PAGE_SHIFT;
@@ -503 +506 @@
             cbBitmapFull += _64K;
 
-        phys_ram_dirty = RTMemPageAlloc(cbBitmapFull);
-        AssertLogRelMsgReturn(phys_ram_dirty, ("Failed to allocate %u bytes of dirty page map bytes\n", cbBitmapFull), VERR_NO_MEMORY);
-
-        rc = RTMemProtect(phys_ram_dirty + cbBitmapAligned, cbBitmapFull - cbBitmapAligned, RTMEM_PROT_NONE);
+        ram_list.phys_dirty = RTMemPageAlloc(cbBitmapFull);
+        AssertLogRelMsgReturn(ram_list.phys_dirty, ("Failed to allocate %u bytes of dirty page map bytes\n", cbBitmapFull), VERR_NO_MEMORY);
+
+        rc = RTMemProtect(ram_list.phys_dirty + cbBitmapAligned, cbBitmapFull - cbBitmapAligned, RTMEM_PROT_NONE);
         if (RT_FAILURE(rc))
         {
-            RTMemPageFree(phys_ram_dirty, cbBitmapFull);
+            RTMemPageFree(ram_list.phys_dirty, cbBitmapFull);
             AssertLogRelRCReturn(rc, rc);
         }
 
-        phys_ram_dirty += cbBitmapAligned - phys_ram_dirty_size;
+        ram_list.phys_dirty += cbBitmapAligned - ram_list.phys_dirty_size;
     }
 
     /* initialize it. */
-    memset(phys_ram_dirty, 0xff, phys_ram_dirty_size);
+    memset(ram_list.phys_dirty, 0xff, ram_list.phys_dirty_size);
     return rc;
 }
@@ -1399 +1402 @@
  * @remark  This function must be kept in perfect sync with the scheduler in EM.cpp!
  */
-bool remR3CanExecuteRaw(CPUState *env, RTGCPTR eip, unsigned fFlags, int *piException)
+bool remR3CanExecuteRaw(CPUX86State *env, RTGCPTR eip, unsigned fFlags, int *piException)
 {
     /* !!! THIS MUST BE IN SYNC WITH emR3Reschedule !!! */
@@ -1651 +1654 @@
  * @param   pu8Byte     Where to store the byte on success
  */
-bool remR3GetOpcode(CPUState *env, RTGCPTR GCPtrInstr, uint8_t *pu8Byte)
+bool remR3GetOpcode(CPUX86State *env, RTGCPTR GCPtrInstr, uint8_t *pu8Byte)
 {
     int rc = PATMR3QueryOpcode(env->pVM, GCPtrInstr, pu8Byte);
@@ -1668 +1671 @@
  * @param   GCPtr       The virtual address which page table/dir entry should be invalidated.
  */
-void remR3FlushPage(CPUState *env, RTGCPTR GCPtr)
+void remR3FlushPage(CPUX86State *env, RTGCPTR GCPtr)
 {
     PVM pVM = env->pVM;
@@ -1712 +1715 @@
 #ifndef REM_PHYS_ADDR_IN_TLB
 /** Wrapper for PGMR3PhysTlbGCPhys2Ptr. */
-void *remR3TlbGCPhys2Ptr(CPUState *env1, target_ulong physAddr, int fWritable)
+void *remR3TlbGCPhys2Ptr(CPUX86State *env1, target_ulong physAddr, int fWritable)
 {
     void *pv;
@@ -1740 +1743 @@
  * @param   GCPtr           Code page to monitor
  */
-void remR3ProtectCode(CPUState *env, RTGCPTR GCPtr)
+void remR3ProtectCode(CPUX86State *env, RTGCPTR GCPtr)
 {
 #ifdef VBOX_REM_PROTECT_PAGES_FROM_SMC
@@ -1760 +1763 @@
  * @param   GCPtr           Code page to monitor
  */
-void remR3UnprotectCode(CPUState *env, RTGCPTR GCPtr)
+void remR3UnprotectCode(CPUX86State *env, RTGCPTR GCPtr)
 {
     Assert(env->pVM->rem.s.fInREM);
@@ -1781 +1784 @@
  * @param   fGlobal         Set if the flush is global.
  */
-void remR3FlushTLB(CPUState *env, bool fGlobal)
+void remR3FlushTLB(CPUX86State *env, bool fGlobal)
 {
     PVM pVM = env->pVM;
@@ -1825 +1828 @@
  * @param   env     Pointer to the CPU environment.
  */
-void remR3ChangeCpuMode(CPUState *env)
+void remR3ChangeCpuMode(CPUX86State *env)
 {
     PVM         pVM = env->pVM;
@@ -1877 +1880 @@
  * @param   env             Pointer to the CPU environment.
  */
-void remR3DmaRun(CPUState *env)
+void remR3DmaRun(CPUX86State *env)
 {
     remR3ProfileStop(STATS_QEMU_RUN_EMULATED_CODE);
@@ -1890 +1893 @@
  * @param   env             Pointer to the CPU environment.
  */
-void remR3TimersRun(CPUState *env)
+void remR3TimersRun(CPUX86State *env)
 {
     LogFlow(("remR3TimersRun:\n"));
@@ -1911 +1914 @@
  * @param   pvNextEIP       Next EIP
  */
-int remR3NotifyTrap(CPUState *env, uint32_t uTrap, uint32_t uErrorCode, RTGCPTR pvNextEIP)
+int remR3NotifyTrap(CPUX86State *env, uint32_t uTrap, uint32_t uErrorCode, RTGCPTR pvNextEIP)
 {
     PVM pVM = env->pVM;
@@ -1982 +1985 @@
  * @param   env             Pointer to the CPU environment.
  */
-void remR3RecordCall(CPUState *env)
+void remR3RecordCall(CPUX86State *env)
 {
     CSAMR3RecordCallAddress(env->pVM, env->eip);
@@ -3444 +3447 @@
  * @param   pTLBEntry   The TLB entry.
  */
-target_ulong remR3PhysGetPhysicalAddressCode(CPUState*          env,
+target_ulong remR3PhysGetPhysicalAddressCode(CPUX86State       *env,
                                              target_ulong       addr,
-                                             CPUTLBEntry*       pTLBEntry,
+                                             CPUTLBEntry       *pTLBEntry,
                                              target_phys_addr_t ioTLBEntry)
 {
@@ -3938 +3941 @@
  * @param   pszPrefix
  */
-bool remR3DisasInstr(CPUState *env, int f32BitCode, char *pszPrefix)
+bool remR3DisasInstr(CPUX86State *env, int f32BitCode, char *pszPrefix)
 {
     PVM pVM = env->pVM;
@@ -4392 +4395 @@
 }
 
-int cpu_get_pic_interrupt(CPUState *env)
+int cpu_get_pic_interrupt(CPUX86State *env)
 {
     uint8_t u8Interrupt;
@@ -4504 +4507 @@
 #define LOG_GROUP LOG_GROUP_REM_IOPORT
 
-void cpu_outb(CPUState *env, pio_addr_t addr, uint8_t val)
+void cpu_outb(CPUX86State *env, pio_addr_t addr, uint8_t val)
 {
     int rc;
@@ -4523 +4526 @@
 }
 
-void cpu_outw(CPUState *env, pio_addr_t addr, uint16_t val)
+void cpu_outw(CPUX86State *env, pio_addr_t addr, uint16_t val)
 {
     //Log2(("cpu_outw: addr=%#06x val=%#x\n", addr, val));
@@ -4538 +4541 @@
 }
 
-void cpu_outl(CPUState *env, pio_addr_t addr, uint32_t val)
+void cpu_outl(CPUX86State *env, pio_addr_t addr, uint32_t val)
 {
     int rc;
@@ -4554 +4557 @@
 }
 
-uint8_t cpu_inb(CPUState *env, pio_addr_t addr)
+uint8_t cpu_inb(CPUX86State *env, pio_addr_t addr)
 {
     uint32_t u32 = 0;
@@ -4574 +4577 @@
 }
 
-uint16_t cpu_inw(CPUState *env, pio_addr_t addr)
+uint16_t cpu_inw(CPUX86State *env, pio_addr_t addr)
 {
     uint32_t u32 = 0;
@@ -4593 +4596 @@
 }
 
-uint32_t cpu_inl(CPUState *env, pio_addr_t addr)
+uint32_t cpu_inl(CPUX86State *env, pio_addr_t addr)
 {
     uint32_t u32 = 0;
@@ -4673 +4676 @@
  * raising a fatal VM error.
  */
-void cpu_abort(CPUState *env, const char *pszFormat, ...)
+void cpu_abort(CPUX86State *env, const char *pszFormat, ...)
 {
     va_list va;
@@ -5414 +5417 @@
 #endif
 
-void cpu_smm_update(CPUState *env)
-{
-}
+void cpu_smm_update(CPUX86State *env)
+{
+}

trunk/src/recompiler/bswap.h

@@ -206 +206 @@
 #ifdef HOST_WORDS_BIGENDIAN
 #define cpu_to_32wu cpu_to_be32wu
+#define leul_to_cpu(v) glue(glue(le,HOST_LONG_BITS),_to_cpu)(v)
 #else
 #define cpu_to_32wu cpu_to_le32wu
+#define leul_to_cpu(v) (v)
 #endif
 
@@ -215 +217 @@
 #undef be_bswaps
 
+/* len must be one of 1, 2, 4 */
+static inline uint32_t qemu_bswap_len(uint32_t value, int len)
+{
+    return bswap32(value) >> (32 - 8 * len);
+}
+
 #endif /* BSWAP_H */

trunk/src/recompiler/cpu-all.h

@@ -348 +348 @@
 }
 
-#else  /* !VBOX */
+#else  /* !VBOX || !REM_PHYS_ADDR_IN_TLB */
 
 static inline int ldub_p(const void *ptr)
@@ -544 +544 @@
 }
 #endif
-#endif /* !VBOX */
+
+#endif /* !VBOX || !REM_PHYS_ADDR_IN_TLB */
 
 #if !defined(HOST_WORDS_BIGENDIAN) || defined(WORDS_ALIGNED)
@@ -775 +776 @@
 extern unsigned long guest_base;
 extern int have_guest_base;
+extern unsigned long reserved_va;
 #define GUEST_BASE guest_base
+#define RESERVED_VA reserved_va
 #else
 #define GUEST_BASE 0ul
+#define RESERVED_VA 0ul
 #endif
 
 /* All direct uses of g2h and h2g need to go away for usermode softmmu.  */
 #define g2h(x) ((void *)((unsigned long)(x) + GUEST_BASE))
+
+#if HOST_LONG_BITS <= TARGET_VIRT_ADDR_SPACE_BITS
+#define h2g_valid(x) 1
+#else
+#define h2g_valid(x) ({ \
+    unsigned long __guest = (unsigned long)(x) - GUEST_BASE; \
+    __guest < (1ul << TARGET_VIRT_ADDR_SPACE_BITS); \
+})
+#endif
+
 #define h2g(x) ({ \
     unsigned long __ret = (unsigned long)(x) - GUEST_BASE; \
     /* Check if given address fits target address space */ \
-    assert(__ret == (abi_ulong)__ret); \
+    assert(h2g_valid(x)); \
     (abi_ulong)__ret; \
-})
-#define h2g_valid(x) ({ \
-    unsigned long __guest = (unsigned long)(x) - GUEST_BASE; \
-    (__guest == (abi_ulong)__guest); \
 })
 
@@ -884 +894 @@
    code */
 #define PAGE_WRITE_ORG 0x0010
+#if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
+/* FIXME: Code that sets/uses this is broken and needs to go away.  */
 #define PAGE_RESERVED  0x0020
-
+#endif
+
+#if defined(CONFIG_USER_ONLY)
 void page_dump(FILE *f);
-int walk_memory_regions(void *,
-    int (*fn)(void *, unsigned long, unsigned long, unsigned long));
+
+typedef int (*walk_memory_regions_fn)(void *, abi_ulong,
+                                      abi_ulong, unsigned long);
+int walk_memory_regions(void *, walk_memory_regions_fn);
+
 int page_get_flags(target_ulong address);
 void page_set_flags(target_ulong start, target_ulong end, int flags);
 int page_check_range(target_ulong start, target_ulong len, int flags);
-
-void cpu_exec_init_all(unsigned long tb_size);
+#endif
+
 CPUState *cpu_copy(CPUState *env);
 CPUState *qemu_get_cpu(int cpu);
@@ -907 +924 @@
 #ifndef VBOX
     __attribute__ ((__format__ (__printf__, 2, 3)));
-#else
+#else  /* VBOX */
     ;
-#endif
+#endif /* VBOX */
 extern CPUState *first_cpu;
 extern CPUState *cpu_single_env;
-extern int64_t qemu_icount;
-extern int use_icount;
 
 #define CPU_INTERRUPT_HARD   0x02 /* hardware interrupt pending */
@@ -978 +993 @@
 void cpu_single_step(CPUState *env, int enabled);
 void cpu_reset(CPUState *s);
-
-/* Return the physical page corresponding to a virtual one. Use it
-   only for debugging because no protection checks are done. Return -1
-   if no page found. */
-target_phys_addr_t cpu_get_phys_page_debug(CPUState *env, target_ulong addr);
+int cpu_is_stopped(CPUState *env);
+void run_on_cpu(CPUState *env, void (*func)(void *data), void *data);
 
 #define CPU_LOG_TB_OUT_ASM (1 << 0)
@@ -1008 +1020 @@
 int cpu_str_to_log_mask(const char *str);
 
-/* IO ports API */
-#include "ioport.h"
+#if !defined(CONFIG_USER_ONLY)
+
+/* Return the physical page corresponding to a virtual one. Use it
+   only for debugging because no protection checks are done. Return -1
+   if no page found. */
+target_phys_addr_t cpu_get_phys_page_debug(CPUState *env, target_ulong addr);
 
 /* memory API */
@@ -1015 +1031 @@
 #ifndef VBOX
 extern int phys_ram_fd;
-extern uint8_t *phys_ram_dirty;
 extern ram_addr_t ram_size;
-extern ram_addr_t last_ram_offset;
-#else /* VBOX */
-/** This is required for bounds checking the phys_ram_dirty accesses. */
-extern RTGCPHYS phys_ram_dirty_size;
-extern uint8_t *phys_ram_dirty;
+#endif /* !VBOX */
+
+typedef struct RAMBlock {
+    uint8_t *host;
+    ram_addr_t offset;
+    ram_addr_t length;
+    char idstr[256];
+    QLIST_ENTRY(RAMBlock) next;
+#if defined(__linux__) && !defined(TARGET_S390X)
+    int fd;
+#endif
+} RAMBlock;
+
+typedef struct RAMList {
+    uint8_t *phys_dirty;
+#ifdef VBOX
+    /** This is required for bounds checking the phys_ram_dirty accesses.
+     * We have memory ranges (the high PC-BIOS mapping) which causes some pages
+     * to fall outside the dirty map. */
+    RTGCPHYS phys_dirty_size;
+#if 0
+# define VBOX_RAMLIST_DIRTY_BOUNDS_CHECK_RET(addr,rv) \
+    do { \
+        if (RT_UNLIKELY( ((addr) >> TARGET_PAGE_BITS) >= ram_list.phys_dirty_size)) { \
+            Log(("%s: %RGp\n", __FUNCTION__, (RTGCPHYS)addr)); \
+            return (rv); \
+        } \
+    } while (0)
+# define VBOX_RAMLIST_DIRTY_BOUNDS_CHECK_RETV(addr) \
+    do { \
+        if (RT_UNLIKELY( ((addr) >> TARGET_PAGE_BITS) >= ram_list.phys_dirty_size)) { \
+            Log(("%s: %RGp\n", __FUNCTION__, (RTGCPHYS)addr)); \
+            return; \
+        } \
+    } while (0)
+#else
+# define VBOX_RAMLIST_DIRTY_BOUNDS_CHECK_RET(addr,rv) \
+    AssertMsgReturn(((addr) >> TARGET_PAGE_BITS) < ram_list.phys_dirty_size, ("%#RGp\n", (RTGCPHYS)(addr)), (rv));
+#  define VBOX_RAMLIST_DIRTY_BOUNDS_CHECK_RETV(addr) \
+    AssertMsgReturnVoid(((addr) >> TARGET_PAGE_BITS) < ram_list.phys_dirty_size, ("%#RGp\n", (RTGCPHYS)(addr)));
+# endif
+#else
+# define VBOX_RAMLIST_DIRTY_BOUNDS_CHECK_RET(addr,rv) do {} while()
+# define VBOX_RAMLIST_DIRTY_BOUNDS_CHECK_RETV(addr) do {} while()
 #endif /* VBOX */
+    QLIST_HEAD(ram, RAMBlock) blocks;
+} RAMList;
+extern RAMList ram_list;
+
+extern const char *mem_path;
+extern int mem_prealloc;
 
 /* physical memory access */
@@ -1042 +1102 @@
 #define TLB_MMIO        (1 << 5)
 
-int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
-                        uint8_t *buf, int len, int is_write);
-
 #define VGA_DIRTY_FLAG       0x01
 #define CODE_DIRTY_FLAG      0x02
@@ -1052 +1109 @@
 static inline int cpu_physical_memory_is_dirty(ram_addr_t addr)
 {
-#ifdef VBOX
-    if (RT_UNLIKELY((addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
-    {
-        Log(("cpu_physical_memory_is_dirty: %RGp\n", (RTGCPHYS)addr));
-        /*AssertMsgFailed(("cpu_physical_memory_is_dirty: %RGp\n", (RTGCPHYS)addr));*/
-        return 0;
-    }
-#endif /* VBOX */
-    return phys_ram_dirty[addr >> TARGET_PAGE_BITS] == 0xff;
+    VBOX_RAMLIST_DIRTY_BOUNDS_CHECK_RET(addr, 0);
+    return ram_list.phys_dirty[addr >> TARGET_PAGE_BITS] == 0xff;
+}
+
+static inline int cpu_physical_memory_get_dirty_flags(ram_addr_t addr)
+{
+    VBOX_RAMLIST_DIRTY_BOUNDS_CHECK_RET(addr, 0xff);
+    return ram_list.phys_dirty[addr >> TARGET_PAGE_BITS];
 }
 
@@ -1066 +1122 @@
                                                 int dirty_flags)
 {
-#ifdef VBOX
-    if (RT_UNLIKELY((addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
-    {
-        Log(("cpu_physical_memory_is_dirty: %RGp\n", (RTGCPHYS)addr));
-        /*AssertMsgFailed(("cpu_physical_memory_is_dirty: %RGp\n", (RTGCPHYS)addr));*/
-        return 0xff & dirty_flags; /** @todo I don't think this is the right thing to return, fix! */
+    VBOX_RAMLIST_DIRTY_BOUNDS_CHECK_RET(addr, 0xff & dirty_flags);
+    return ram_list.phys_dirty[addr >> TARGET_PAGE_BITS] & dirty_flags;
+}
+
+static inline void cpu_physical_memory_set_dirty(ram_addr_t addr)
+{
+    VBOX_RAMLIST_DIRTY_BOUNDS_CHECK_RETV(addr);
+    ram_list.phys_dirty[addr >> TARGET_PAGE_BITS] = 0xff;
+}
+
+static inline int cpu_physical_memory_set_dirty_flags(ram_addr_t addr,
+                                                      int dirty_flags)
+{
+    VBOX_RAMLIST_DIRTY_BOUNDS_CHECK_RET(addr, 0xff);
+    return ram_list.phys_dirty[addr >> TARGET_PAGE_BITS] |= dirty_flags;
+}
+
+static inline void cpu_physical_memory_mask_dirty_range(ram_addr_t start,
+                                                        int length,
+                                                        int dirty_flags)
+{
+    int i, mask, len;
+    uint8_t *p;
+
+    VBOX_RAMLIST_DIRTY_BOUNDS_CHECK_RETV(start);
+    len = length >> TARGET_PAGE_BITS;
+    mask = ~dirty_flags;
+    p = ram_list.phys_dirty + (start >> TARGET_PAGE_BITS);
+    for (i = 0; i < len; i++) {
+        p[i] &= mask;
     }
-#endif /* VBOX */
-    return phys_ram_dirty[addr >> TARGET_PAGE_BITS] & dirty_flags;
-}
-
-static inline void cpu_physical_memory_set_dirty(ram_addr_t addr)
-{
-#ifdef VBOX
-    if (RT_UNLIKELY((addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
-    {
-        Log(("cpu_physical_memory_is_dirty: %RGp\n", (RTGCPHYS)addr));
-        /*AssertMsgFailed(("cpu_physical_memory_is_dirty: %RGp\n", (RTGCPHYS)addr));*/
-        return;
-    }
-#endif /* VBOX */
-    phys_ram_dirty[addr >> TARGET_PAGE_BITS] = 0xff;
 }
 
@@ -1103 +1168 @@
 void dump_exec_info(FILE *f,
                     int (*cpu_fprintf)(FILE *f, const char *fmt, ...));
-
-/* Coalesced MMIO regions are areas where write operations can be reordered.
- * This usually implies that write operations are side-effect free.  This allows
- * batching which can make a major impact on performance when using
- * virtualization.
- */
-void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size);
-
-void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size);
-
-/*******************************************/
-/* host CPU ticks (if available) */
-
-#if defined(_ARCH_PPC)
-
-static inline int64_t cpu_get_real_ticks(void)
-{
-    int64_t retval;
-#ifdef _ARCH_PPC64
-    /* This reads timebase in one 64bit go and includes Cell workaround from:
-       http://ozlabs.org/pipermail/linuxppc-dev/2006-October/027052.html
-     */
-    __asm__ __volatile__ (
-        "mftb    %0\n\t"
-        "cmpwi   %0,0\n\t"
-        "beq-    $-8"
-        : "=r" (retval));
-#else
-    /* http://ozlabs.org/pipermail/linuxppc-dev/1999-October/003889.html */
-    unsigned long junk;
-    __asm__ __volatile__ (
-        "mftbu   %1\n\t"
-        "mftb    %L0\n\t"
-        "mftbu   %0\n\t"
-        "cmpw    %0,%1\n\t"
-        "bne     $-16"
-        : "=r" (retval), "=r" (junk));
-#endif
-    return retval;
-}
-
-#elif defined(__i386__)
-
-static inline int64_t cpu_get_real_ticks(void)
-{
-    int64_t val;
-    asm volatile ("rdtsc" : "=A" (val));
-    return val;
-}
-
-#elif defined(__x86_64__)
-
-static inline int64_t cpu_get_real_ticks(void)
-{
-    uint32_t low,high;
-    int64_t val;
-    asm volatile("rdtsc" : "=a" (low), "=d" (high));
-    val = high;
-    val <<= 32;
-    val |= low;
-    return val;
-}
-
-#elif defined(__hppa__)
-
-static inline int64_t cpu_get_real_ticks(void)
-{
-    int val;
-    asm volatile ("mfctl %%cr16, %0" : "=r"(val));
-    return val;
-}
-
-#elif defined(__ia64)
-
-static inline int64_t cpu_get_real_ticks(void)
-{
-        int64_t val;
-        asm volatile ("mov %0 = ar.itc" : "=r"(val) :: "memory");
-        return val;
-}
-
-#elif defined(__s390__)
-
-static inline int64_t cpu_get_real_ticks(void)
-{
-    int64_t val;
-    asm volatile("stck 0(%1)" : "=m" (val) : "a" (&val) : "cc");
-    return val;
-}
-
-#elif defined(__sparc_v8plus__) || defined(__sparc_v8plusa__) || defined(__sparc_v9__)
-
-static inline int64_t cpu_get_real_ticks (void)
-{
-#if     defined(_LP64)
-        uint64_t        rval;
-        asm volatile("rd %%tick,%0" : "=r"(rval));
-        return rval;
-#else
-        union {
-                uint64_t i64;
-                struct {
-                        uint32_t high;
-                        uint32_t low;
-                }       i32;
-        } rval;
-        asm volatile("rd %%tick,%1; srlx %1,32,%0"
-                : "=r"(rval.i32.high), "=r"(rval.i32.low));
-        return rval.i64;
-#endif
-}
-
-#elif defined(__mips__) && \
-      ((defined(__mips_isa_rev) && __mips_isa_rev >= 2) || defined(__linux__))
-/*
- * binutils wants to use rdhwr only on mips32r2
- * but as linux kernel emulate it, it's fine
- * to use it.
- *
- */
-#define MIPS_RDHWR(rd, value) {                 \
-    __asm__ __volatile__ (                      \
-                          ".set   push\n\t"     \
-                          ".set mips32r2\n\t"   \
-                          "rdhwr  %0, "rd"\n\t" \
-                          ".set   pop"          \
-                          : "=r" (value));      \
-}
-
-static inline int64_t cpu_get_real_ticks(void)
-{
-/* On kernels >= 2.6.25 rdhwr <reg>, $2 and $3 are emulated */
-    uint32_t count;
-    static uint32_t cyc_per_count = 0;
-
-    if (!cyc_per_count)
-        MIPS_RDHWR("$3", cyc_per_count);
-
-    MIPS_RDHWR("$2", count);
-    return (int64_t)(count * cyc_per_count);
-}
-
-#else
-/* The host CPU doesn't have an easily accessible cycle counter.
-   Just return a monotonically increasing value.  This will be
-   totally wrong, but hopefully better than nothing.  */
-static inline int64_t cpu_get_real_ticks (void)
-{
-    static int64_t ticks = 0;
-    return ticks++;
-}
-#endif
-
-/* profiling */
-#ifdef CONFIG_PROFILER
-static inline int64_t profile_getclock(void)
-{
-    return cpu_get_real_ticks();
-}
-
-extern int64_t qemu_time, qemu_time_start;
-extern int64_t tlb_flush_time;
-extern int64_t dev_time;
-#endif
+#endif /* !CONFIG_USER_ONLY */
+
+int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
+                        uint8_t *buf, int len, int is_write);
 
 void cpu_inject_x86_mce(CPUState *cenv, int bank, uint64_t status,

trunk/src/recompiler/cpu-common.h

@@ -4 +4 @@
 /* CPU interfaces that are target indpendent.  */
 
-#if defined(__arm__) || defined(__sparc__) || defined(__mips__) || defined(__hppa__)
+#if defined(__arm__) || defined(__sparc__) || defined(__mips__) || defined(__hppa__) || defined(__ia64__)
 #define WORDS_ALIGNED
 #endif
 
+#ifdef TARGET_PHYS_ADDR_BITS
+#include "targphys.h"
+#endif
+
+#ifndef NEED_CPU_H
+#include "poison.h"
+#endif
+
 #include "bswap.h"
+#include "qemu-queue.h"
+
+#if !defined(CONFIG_USER_ONLY)
 
 /* address in the RAM (different from a physical address) */
@@ -30 +41 @@
 
 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr);
-ram_addr_t qemu_ram_alloc(ram_addr_t);
+#ifndef VBOX
+ram_addr_t qemu_ram_alloc_from_ptr(DeviceState *dev, const char *name,
+                        ram_addr_t size, void *host);
+ram_addr_t qemu_ram_alloc(DeviceState *dev, const char *name, ram_addr_t size);
 void qemu_ram_free(ram_addr_t addr);
 /* This should only be used for ram local to a device.  */
@@ -36 +50 @@
 /* This should not be used by devices.  */
 ram_addr_t qemu_ram_addr_from_host(void *ptr);
+#endif /* !VBOX */
 
 int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
@@ -62 +77 @@
 void cpu_unregister_map_client(void *cookie);
 
+struct CPUPhysMemoryClient;
+typedef struct CPUPhysMemoryClient CPUPhysMemoryClient;
+struct CPUPhysMemoryClient {
+    void (*set_memory)(struct CPUPhysMemoryClient *client,
+                       target_phys_addr_t start_addr,
+                       ram_addr_t size,
+                       ram_addr_t phys_offset);
+    int (*sync_dirty_bitmap)(struct CPUPhysMemoryClient *client,
+                             target_phys_addr_t start_addr,
+                             target_phys_addr_t end_addr);
+    int (*migration_log)(struct CPUPhysMemoryClient *client,
+                         int enable);
+    QLIST_ENTRY(CPUPhysMemoryClient) list;
+};
+
+void cpu_register_phys_memory_client(CPUPhysMemoryClient *);
+void cpu_unregister_phys_memory_client(CPUPhysMemoryClient *);
+
+/* Coalesced MMIO regions are areas where write operations can be reordered.
+ * This usually implies that write operations are side-effect free.  This allows
+ * batching which can make a major impact on performance when using
+ * virtualization.
+ */
+void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size);
+
+void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size);
+
+void qemu_flush_coalesced_mmio_buffer(void);
+
 uint32_t ldub_phys(target_phys_addr_t addr);
 uint32_t lduw_phys(target_phys_addr_t addr);
@@ -86 +130 @@
 #define IO_MEM_ROMD        (1)
 #define IO_MEM_SUBPAGE     (2)
-#define IO_MEM_SUBWIDTH    (4)
+
+#endif
 
 #endif /* !CPU_COMMON_H */

trunk/src/recompiler/cpu-defs.h

@@ -39 +39 @@
 #ifndef VBOX
 #include <signal.h>
-#endif
+#else  /* VBOX */
+# define sig_atomic_t int32_t
+#endif /* VBOX */
 #include "osdep.h"
 #include "qemu-queue.h"
@@ -91 +93 @@
 #define TB_JMP_PAGE_MASK (TB_JMP_CACHE_SIZE - TB_JMP_PAGE_SIZE)
 
+#if !defined(CONFIG_USER_ONLY)
 #define CPU_TLB_BITS 8
 #define CPU_TLB_SIZE (1 << CPU_TLB_BITS)
 
-#if TARGET_PHYS_ADDR_BITS == 32 && TARGET_LONG_BITS == 32
+#if HOST_LONG_BITS == 32 && TARGET_LONG_BITS == 32
 #define CPU_TLB_ENTRY_BITS 4
 #else
@@ -110 +113 @@
     target_ulong addr_write;
     target_ulong addr_code;
-    /* Addend to virtual address to get physical address.  IO accesses
+    /* Addend to virtual address to get host address.  IO accesses
        use the corresponding iotlb value.  */
-#if TARGET_PHYS_ADDR_BITS == 64
-    /* on i386 Linux make sure it is aligned */
-    target_phys_addr_t addend __attribute__((aligned(8)));
-#else
-    target_phys_addr_t addend;
-#endif
+    unsigned long addend;
     /* padding to get a power of two size */
     uint8_t dummy[(1 << CPU_TLB_ENTRY_BITS) -
                   (sizeof(target_ulong) * 3 +
-                   ((-sizeof(target_ulong) * 3) & (sizeof(target_phys_addr_t) - 1)) +
-                   sizeof(target_phys_addr_t))];
+                   ((-sizeof(target_ulong) * 3) & (sizeof(unsigned long) - 1)) +
+                   sizeof(unsigned long))];
 } CPUTLBEntry;
+
+extern int CPUTLBEntry_wrong_size[sizeof(CPUTLBEntry) == (1 << CPU_TLB_ENTRY_BITS) ? 1 : -1];
+
+#define CPU_COMMON_TLB \
+    /* The meaning of the MMU modes is defined in the target code. */   \
+    CPUTLBEntry tlb_table[NB_MMU_MODES][CPU_TLB_SIZE];                  \
+    target_phys_addr_t iotlb[NB_MMU_MODES][CPU_TLB_SIZE];               \
+    target_ulong tlb_flush_addr;                                        \
+    target_ulong tlb_flush_mask;
+
+#else
+
+#define CPU_COMMON_TLB
+
+#endif
+
 
 #ifdef HOST_WORDS_BIGENDIAN
@@ -139 +153 @@
 struct kvm_run;
 struct KVMState;
+struct qemu_work_item;
 
 typedef struct CPUBreakpoint {
@@ -165 +180 @@
                                      memory was accessed */             \
     uint32_t halted; /* Nonzero if the CPU is in suspend state */       \
-    uint32_t stop;   /* Stop request */                                 \
-    uint32_t stopped; /* Artificially stopped */                        \
     uint32_t interrupt_request;                                         \
-    volatile /*sig_atomic_t - vbox*/ int32_t exit_request;                                 \
-    /* The meaning of the MMU modes is defined in the target code. */   \
-    CPUTLBEntry tlb_table[NB_MMU_MODES][CPU_TLB_SIZE];                  \
-    target_phys_addr_t iotlb[NB_MMU_MODES][CPU_TLB_SIZE]; \
-    /** addends for HVA -> GPA translations */                          \
-    VBOX_ONLY(target_phys_addr_t   phys_addends[NB_MMU_MODES][CPU_TLB_SIZE]); \
+    volatile sig_atomic_t exit_request;                                 \
+    CPU_COMMON_TLB                                                      \
     struct TranslationBlock *tb_jmp_cache[TB_JMP_CACHE_SIZE];           \
     /* buffer for temporaries in the code generator */                  \
@@ -213 +222 @@
                                                                         \
     uint32_t created;                                                   \
+    uint32_t stop;   /* Stop request */                                 \
+    uint32_t stopped; /* Artificially stopped */                        \
     struct QemuThread *thread;                                          \
     struct QemuCond *halt_cond;                                         \
+    struct qemu_work_item *queued_work_first, *queued_work_last;        \
     const char *cpu_model_str;                                          \
     struct KVMState *kvm_state;                                         \
     struct kvm_run *kvm_run;                                            \
-    int kvm_fd;
-
-#endif
+    int kvm_fd;                                                         \
+    int kvm_vcpu_dirty;
+
+#endif

trunk/src/recompiler/cpu-exec.c

@@ -32 +32 @@
 #include "tcg.h"
 #include "kvm.h"
+#include "qemu-barrier.h"
 
 #if !defined(CONFIG_SOFTMMU)
@@ -67 +68 @@
 void cpu_loop_exit(void)
 {
-    /* NOTE: the register at this point must be saved by hand because
-       longjmp restore them */
-    regs_to_env();
+    env->current_tb = NULL;
     longjmp(env->jmp_env, 1);
 }
@@ -94 +93 @@
         /* XXX: use siglongjmp ? */
 #ifdef __linux__
+#ifdef __ia64
+        sigprocmask(SIG_SETMASK, (sigset_t *)&uc->uc_sigmask, NULL);
+#else
         sigprocmask(SIG_SETMASK, &uc->uc_sigmask, NULL);
+#endif
 #elif defined(__OpenBSD__)
         sigprocmask(SIG_SETMASK, &uc->sc_mask, NULL);
@@ -125 +128 @@
     next_tb = tcg_qemu_tb_exec(tb->tc_ptr);
 #endif
+    env->current_tb = NULL;
 
     if ((next_tb & 3) == 2) {
@@ -141 +145 @@
     TranslationBlock *tb, **ptb1;
     unsigned int h;
-    target_ulong phys_pc, phys_page1, phys_page2, virt_page2;
+    tb_page_addr_t phys_pc, phys_page1, phys_page2;
+    target_ulong virt_page2;
 
     tb_invalidated_flag = 0;
 
-    regs_to_env(); /* XXX: do it just before cpu_gen_code() */
-
     /* find translated block using physical mappings */
-    phys_pc = get_phys_addr_code(env, pc);
+    phys_pc = get_page_addr_code(env, pc);
     phys_page1 = phys_pc & TARGET_PAGE_MASK;
     phys_page2 = -1;
@@ -165 +168 @@
                 virt_page2 = (pc & TARGET_PAGE_MASK) +
                     TARGET_PAGE_SIZE;
-                phys_page2 = get_phys_addr_code(env, virt_page2);
+                phys_page2 = get_page_addr_code(env, virt_page2);
                 if (tb->page_addr[1] == phys_page2)
                     goto found;
@@ -226 +229 @@
 /* main execution loop */
 
-#ifdef VBOX
+volatile sig_atomic_t exit_request;
 
 int cpu_exec(CPUState *env1)
 {
-#define DECLARE_HOST_REGS 1
-#include "hostregs_helper.h"
-    int ret = 0, interrupt_request;
+    volatile host_reg_t saved_env_reg;
+    int ret VBOX_ONLY(= 0), interrupt_request;
     TranslationBlock *tb;
     uint8_t *tc_ptr;
+#ifndef VBOX
+    uintptr_t next_tb;
+#else  /* VBOX */
     unsigned long next_tb;
-
-    cpu_single_env = env1;
-
-    /* first we save global registers */
-#define SAVE_HOST_REGS 1
-#include "hostregs_helper.h"
-    env = env1;
-
-    env_to_regs();
-#if defined(TARGET_I386)
-    /* put eflags in CPU temporary format */
-    CC_SRC = env->eflags & (CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C);
-    DF = 1 - (2 * ((env->eflags >> 10) & 1));
-    CC_OP = CC_OP_EFLAGS;
-    env->eflags &= ~(DF_MASK | CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C);
-#elif defined(TARGET_SPARC)
-#elif defined(TARGET_M68K)
-    env->cc_op = CC_OP_FLAGS;
-    env->cc_dest = env->sr & 0xf;
-    env->cc_x = (env->sr >> 4) & 1;
-#elif defined(TARGET_ALPHA)
-#elif defined(TARGET_ARM)
-#elif defined(TARGET_PPC)
-#elif defined(TARGET_MIPS)
-#elif defined(TARGET_SH4)
-#elif defined(TARGET_CRIS)
-    /* XXXXX */
-#else
-#error unsupported target CPU
-#endif
-#ifndef VBOX /* VBOX: We need to raise traps and suchlike from the outside. */
-    env->exception_index = -1;
-#endif
-
-    /* prepare setjmp context for exception handling */
-    for(;;) {
-        if (setjmp(env->jmp_env) == 0)
-        {
-            env->current_tb = NULL;
-
-            /*
-             * Check for fatal errors first
-             */
-            if (env->interrupt_request & CPU_INTERRUPT_RC) {
-                env->exception_index = EXCP_RC;
-                ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request, ~CPU_INTERRUPT_RC);
-                ret = env->exception_index;
-                cpu_loop_exit();
-            }
-
-            /* if an exception is pending, we execute it here */
-            if (env->exception_index >= 0) {
-                if (env->exception_index >= EXCP_INTERRUPT) {
-                    /* exit request from the cpu execution loop */
-                    ret = env->exception_index;
-                    if (ret == EXCP_DEBUG)
-                        cpu_handle_debug_exception(env);
-                    break;
-                } else {
-                    /* simulate a real cpu exception. On i386, it can
-                       trigger new exceptions, but we do not handle
-                       double or triple faults yet. */
-                    RAWEx_ProfileStart(env, STATS_IRQ_HANDLING);
-                    Log(("do_interrupt %d %d %RGv\n", env->exception_index, env->exception_is_int, (RTGCPTR)env->exception_next_eip));
-                    do_interrupt(env->exception_index,
-                                 env->exception_is_int,
-                                 env->error_code,
-                                 env->exception_next_eip, 0);
-                    /* successfully delivered */
-                    env->old_exception = -1;
-                    RAWEx_ProfileStop(env, STATS_IRQ_HANDLING);
-                }
-                env->exception_index = -1;
-            }
-
-            next_tb = 0; /* force lookup of first TB */
-            for(;;)
-            {
-                interrupt_request = env->interrupt_request;
-                if (unlikely(interrupt_request)) {
-                    if (unlikely(env->singlestep_enabled & SSTEP_NOIRQ)) {
-                        /* Mask out external interrupts for this step. */
-                        interrupt_request &= ~(CPU_INTERRUPT_HARD |
-                                               CPU_INTERRUPT_FIQ |
-                                               CPU_INTERRUPT_SMI |
-                                               CPU_INTERRUPT_NMI);
-                    }
-                    if (interrupt_request & CPU_INTERRUPT_DEBUG) {
-                        env->interrupt_request &= ~CPU_INTERRUPT_DEBUG;
-                        env->exception_index = EXCP_DEBUG;
-                        cpu_loop_exit();
-                    }
-                    /** @todo: reconcile with what QEMU really does */
-
-                    /* Single instruction exec request, we execute it and return (one way or the other).
-                       The caller will always reschedule after doing this operation! */
-                    if (interrupt_request & CPU_INTERRUPT_SINGLE_INSTR)
-                    {
-                        /* not in flight are we? (if we are, we trapped) */
-                        if (!(env->interrupt_request & CPU_INTERRUPT_SINGLE_INSTR_IN_FLIGHT))
-                        {
-                            ASMAtomicOrS32((int32_t volatile *)&env->interrupt_request, CPU_INTERRUPT_SINGLE_INSTR_IN_FLIGHT);
-                            env->exception_index = EXCP_SINGLE_INSTR;
-                            if (emulate_single_instr(env) == -1)
-                                AssertMsgFailed(("REM: emulate_single_instr failed for EIP=%RGv!!\n", (RTGCPTR)env->eip));
-
-                            /* When we receive an external interrupt during execution of this single
-                               instruction, then we should stay here. We will leave when we're ready
-                               for raw-mode or when interrupted by pending EMT requests.  */
-                            interrupt_request = env->interrupt_request; /* reload this! */
-                            if (   !(interrupt_request & CPU_INTERRUPT_HARD)
-                                || !(env->eflags & IF_MASK)
-                                ||  (env->hflags & HF_INHIBIT_IRQ_MASK)
-                                ||  (env->state & CPU_RAW_HWACC)
-                               )
-                            {
-                                env->exception_index = ret = EXCP_SINGLE_INSTR;
-                                cpu_loop_exit();
-                            }
-                        }
-                        /* Clear CPU_INTERRUPT_SINGLE_INSTR and leave CPU_INTERRUPT_SINGLE_INSTR_IN_FLIGHT set. */
-                        ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request, ~CPU_INTERRUPT_SINGLE_INSTR);
-#ifdef IEM_VERIFICATION_MODE
-                        env->exception_index = ret = EXCP_SINGLE_INSTR;
-                        cpu_loop_exit();
-#endif
-                    }
-
-                    RAWEx_ProfileStart(env, STATS_IRQ_HANDLING);
-                    if ((interrupt_request & CPU_INTERRUPT_SMI) &&
-                        !(env->hflags & HF_SMM_MASK)) {
-                        env->interrupt_request &= ~CPU_INTERRUPT_SMI;
-                        do_smm_enter();
-                        next_tb = 0;
-                    }
-                    else if ((interrupt_request & CPU_INTERRUPT_HARD) &&
-                             (env->eflags & IF_MASK) &&
-                             !(env->hflags & HF_INHIBIT_IRQ_MASK))
-                    {
-                        /* if hardware interrupt pending, we execute it */
-                        int intno;
-                        ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request, ~CPU_INTERRUPT_HARD);
-                        intno = cpu_get_pic_interrupt(env);
-                        if (intno >= 0)
-                        {
-                            Log(("do_interrupt %d\n", intno));
-                            do_interrupt(intno, 0, 0, 0, 1);
-                        }
-                        /* ensure that no TB jump will be modified as
-                           the program flow was changed */
-                        next_tb = 0;
-                    }
-                    if (env->interrupt_request & CPU_INTERRUPT_EXITTB)
-                    {
-                        ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request, ~CPU_INTERRUPT_EXITTB);
-                        /* ensure that no TB jump will be modified as
-                           the program flow was changed */
-                        next_tb = 0;
-                    }
-                    RAWEx_ProfileStop(env, STATS_IRQ_HANDLING);
-                    if (interrupt_request & CPU_INTERRUPT_RC)
-                    {
-                        env->exception_index = EXCP_RC;
-                        ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request, ~CPU_INTERRUPT_RC);
-                        ret = env->exception_index;
-                        cpu_loop_exit();
-                    }
-                }
-                if (unlikely(env->exit_request)) {
-                    env->exit_request = 0;
-                    env->exception_index = EXCP_INTERRUPT;
-                    cpu_loop_exit();
-                }
-
-                /*
-                 * Check if we the CPU state allows us to execute the code in raw-mode.
-                 */
-                RAWEx_ProfileStart(env, STATS_RAW_CHECK);
-                if (remR3CanExecuteRaw(env,
-                                       env->eip + env->segs[R_CS].base,
-                                       env->hflags | (env->eflags & (IOPL_MASK | TF_MASK | VM_MASK)),
-                                       &env->exception_index))
-                {
-                    RAWEx_ProfileStop(env, STATS_RAW_CHECK);
-                    ret = env->exception_index;
-                    cpu_loop_exit();
-                }
-                RAWEx_ProfileStop(env, STATS_RAW_CHECK);
-
-{
-    RTGCPTR mypc = env->eip + env->segs[R_CS].base;
-if (mypc == 0x00fe0d2 || mypc == 0x00f19e9 || mypc == 0x000f0827 || mypc == 0x000fe090) {
-    RTLogFlags(NULL, "enabled");
-    loglevel = ~0;
-    Log(("BANG CRASH!\n"));
-}
-}
-#ifdef CONFIG_DEBUG_EXEC
-                if (qemu_loglevel_mask(CPU_LOG_TB_CPU)) {
-                    /* restore flags in standard format */
-                    regs_to_env();
-                    env->eflags = env->eflags | helper_cc_compute_all(CC_OP) | (DF & DF_MASK);
-                    log_cpu_state(env, X86_DUMP_CCOP);
-                    env->eflags &= ~(DF_MASK | CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C);
-                }
-#endif
-                RAWEx_ProfileStart(env, STATS_TLB_LOOKUP);
-                spin_lock(&tb_lock);
-                tb = tb_find_fast();
-                 /* Note: we do it here to avoid a gcc bug on Mac OS X when
-                   doing it in tb_find_slow */
-                if (tb_invalidated_flag) {
-                    /* as some TB could have been invalidated because
-                       of memory exceptions while generating the code, we
-                       must recompute the hash index here */
-                    next_tb = 0;
-                    tb_invalidated_flag = 0;
-                }
-#ifdef CONFIG_DEBUG_EXEC
-                qemu_log_mask(CPU_LOG_EXEC, "Trace 0x%08lx [" TARGET_FMT_lx "] %s [sp=%RGv, bp=%RGv\n",
-                             (long)tb->tc_ptr, tb->pc, lookup_symbol(tb->pc), (RTGCPTR)env->regs[R_ESP], (RTGCPTR)env->regs[R_EBP]);
-#endif
-
-
-                /* see if we can patch the calling TB. When the TB
-                   spans two pages, we cannot safely do a direct
-                   jump. */
-                if (next_tb != 0
-                    && !(tb->cflags & CF_RAW_MODE)
-                    && tb->page_addr[1] == -1)
-                {
-                    tb_add_jump((TranslationBlock *)(long)(next_tb & ~3), next_tb & 3, tb);
-                }
-                spin_unlock(&tb_lock);
-                RAWEx_ProfileStop(env, STATS_TLB_LOOKUP);
-
-                env->current_tb = tb;
-
-                /* cpu_interrupt might be called while translating the
-                   TB, but before it is linked into a potentially
-                   infinite loop and becomes env->current_tb. Avoid
-                   starting execution if there is a pending interrupt. */
-                if (unlikely (env->exit_request))
-                    env->current_tb = NULL;
-
-                while (env->current_tb) {
-                    tc_ptr = tb->tc_ptr;
-                    /* execute the generated code */
-                    RAWEx_ProfileStart(env, STATS_QEMU_RUN_EMULATED_CODE);
-#if defined(VBOX) && defined(GCC_WITH_BUGGY_REGPARM)
-                    tcg_qemu_tb_exec(tc_ptr, next_tb);
-#else
-                    next_tb = tcg_qemu_tb_exec(tc_ptr);
-#endif
-                    RAWEx_ProfileStop(env, STATS_QEMU_RUN_EMULATED_CODE);
-                    env->current_tb = NULL;
-                     if ((next_tb & 3) == 2) {
-                        /* Instruction counter expired.  */
-                        int insns_left;
-                        tb = (TranslationBlock *)(long)(next_tb & ~3);
-                        /* Restore PC.  */
-                        cpu_pc_from_tb(env, tb);
-                        insns_left = env->icount_decr.u32;
-                        if (env->icount_extra && insns_left >= 0) {
-                            /* Refill decrementer and continue execution.  */
-                            env->icount_extra += insns_left;
-                            if (env->icount_extra > 0xffff) {
-                                insns_left = 0xffff;
-                            } else {
-                                insns_left = env->icount_extra;
-                            }
-                            env->icount_extra -= insns_left;
-                            env->icount_decr.u16.low = insns_left;
-                        } else {
-                            if (insns_left > 0) {
-                                /* Execute remaining instructions.  */
-                                cpu_exec_nocache(insns_left, tb);
-                            }
-                            env->exception_index = EXCP_INTERRUPT;
-                            next_tb = 0;
-                            cpu_loop_exit();
-                        }
-                     }
-                }
-
-                /* reset soft MMU for next block (it can currently
-                   only be set by a memory fault) */
-#if defined(TARGET_I386) && !defined(CONFIG_SOFTMMU)
-                if (env->hflags & HF_SOFTMMU_MASK) {
-                    env->hflags &= ~HF_SOFTMMU_MASK;
-                    /* do not allow linking to another block */
-                    next_tb = 0;
-                }
-#endif
-            } /* for(;;) */
-        } else {
-            env_to_regs();
-        }
-#ifdef VBOX_HIGH_RES_TIMERS_HACK
-        /* NULL the current_tb here so cpu_interrupt() doesn't do anything
-           unnecessary (like crashing during emulate single instruction).
-           Note! Don't use env1->pVM here, the code wouldn't run with
-                 gcc-4.4/amd64 anymore, see #3883. */
-        env->current_tb = NULL;
-        if (    !(env->interrupt_request & (  CPU_INTERRUPT_DEBUG | CPU_INTERRUPT_EXTERNAL_EXIT | CPU_INTERRUPT_RC
-                                            | CPU_INTERRUPT_SINGLE_INSTR | CPU_INTERRUPT_SINGLE_INSTR_IN_FLIGHT))
-            &&  (   (env->interrupt_request & CPU_INTERRUPT_EXTERNAL_TIMER)
-                 || TMTimerPollBool(env->pVM, env->pVCpu)) ) {
-            ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request, ~CPU_INTERRUPT_EXTERNAL_TIMER);
-            remR3ProfileStart(STATS_QEMU_RUN_TIMERS);
-            TMR3TimerQueuesDo(env->pVM);
-            remR3ProfileStop(STATS_QEMU_RUN_TIMERS);
-        }
-#endif
-    } /* for(;;) */
-
-#if defined(TARGET_I386)
-    /* restore flags in standard format */
-    env->eflags = env->eflags | helper_cc_compute_all(CC_OP) | (DF & DF_MASK);
-#else
-#error unsupported target CPU
-#endif
-#include "hostregs_helper.h"
-    return ret;
-}
-
-#else /* !VBOX */
-int cpu_exec(CPUState *env1)
-{
-#define DECLARE_HOST_REGS 1
-#include "hostregs_helper.h"
-    int ret, interrupt_request;
-    TranslationBlock *tb;
-    uint8_t *tc_ptr;
-    unsigned long next_tb;
-
+#endif /* VBOX */
+
+# ifndef VBOX
     if (cpu_halted(env1) == EXCP_HALTED)
         return EXCP_HALTED;
+# endif /* !VBOX */
 
     cpu_single_env = env1;
 
-    /* first we save global registers */
-#define SAVE_HOST_REGS 1
-#include "hostregs_helper.h"
+    /* the access to env below is actually saving the global register's
+       value, so that files not including target-xyz/exec.h are free to
+       use it.  */
+    QEMU_BUILD_BUG_ON (sizeof (saved_env_reg) != sizeof (env));
+    saved_env_reg = (host_reg_t) env;
+    barrier();
     env = env1;
 
-    env_to_regs();
+    if (unlikely(exit_request)) {
+        env->exit_request = 1;
+    }
+
 #if defined(TARGET_I386)
     if (!kvm_enabled()) {
@@ -608 +287 @@
 #error unsupported target CPU
 #endif
+#ifndef VBOX /* VBOX: We need to raise traps and suchlike from the outside. */
     env->exception_index = -1;
+#endif /* !VBOX */
 
     /* prepare setjmp context for exception handling */
@@ -618 +299 @@
 #define env cpu_single_env
 #endif
-            env->current_tb = NULL;
+#ifdef VBOX
+            env->current_tb = NULL; /* probably not needed, but whatever... */
+
+            /*
+             * Check for fatal errors first
+             */
+            if (env->interrupt_request & CPU_INTERRUPT_RC) {
+                env->exception_index = EXCP_RC;
+                ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request, ~CPU_INTERRUPT_RC);
+                ret = env->exception_index;
+                cpu_loop_exit();
+            }
+#endif
+
             /* if an exception is pending, we execute it here */
             if (env->exception_index >= 0) {
@@ -624 +318 @@
                     /* exit request from the cpu execution loop */
                     ret = env->exception_index;
+#ifdef VBOX /* because of the above stuff */
+                    env->exception_index = -1;
+#endif
                     if (ret == EXCP_DEBUG)
                         cpu_handle_debug_exception(env);
@@ -647 +344 @@
                        trigger new exceptions, but we do not handle
                        double or triple faults yet. */
+#  ifdef VBOX
+                    RAWEx_ProfileStart(env, STATS_IRQ_HANDLING);
+                    Log(("do_interrupt: vec=%#x int=%d pc=%04x:%RGv\n", env->exception_index, env->exception_is_int,
+                         env->segs[R_CS].selector, (RTGCPTR)env->exception_next_eip));
+#  endif /* VBOX */
                     do_interrupt(env->exception_index,
                                  env->exception_is_int,
@@ -653 +355 @@
                     /* successfully delivered */
                     env->old_exception = -1;
+#  ifdef VBOX
+                    RAWEx_ProfileStop(env, STATS_IRQ_HANDLING);
+#  endif /* VBOX */
 #elif defined(TARGET_PPC)
                     do_interrupt(env);
@@ -664 +369 @@
                     do_interrupt(env);
 #elif defined(TARGET_SH4)
-                    do_interrupt(env);
+                    do_interrupt(env);
 #elif defined(TARGET_ALPHA)
                     do_interrupt(env);
@@ -672 +377 @@
                     do_interrupt(0);
 #endif
+                    env->exception_index = -1;
 #endif
                 }
-                env->exception_index = -1;
             }
 
+# ifndef VBOX
             if (kvm_enabled()) {
                 kvm_cpu_exec(env);
                 longjmp(env->jmp_env, 1);
             }
+# endif /* !VBOX */
 
             next_tb = 0; /* force lookup of first TB */
@@ -709 +416 @@
 #endif
 #if defined(TARGET_I386)
+# ifdef VBOX
+                    /* Single instruction exec request, we execute it and return (one way or the other).
+                       The caller will always reschedule after doing this operation! */
+                    if (interrupt_request & CPU_INTERRUPT_SINGLE_INSTR)
+                    {
+                        /* not in flight are we? (if we are, we trapped) */
+                        if (!(env->interrupt_request & CPU_INTERRUPT_SINGLE_INSTR_IN_FLIGHT))
+                        {
+                            ASMAtomicOrS32((int32_t volatile *)&env->interrupt_request, CPU_INTERRUPT_SINGLE_INSTR_IN_FLIGHT);
+                            env->exception_index = EXCP_SINGLE_INSTR;
+                            if (emulate_single_instr(env) == -1)
+                                AssertMsgFailed(("REM: emulate_single_instr failed for EIP=%RGv!!\n", (RTGCPTR)env->eip));
+
+                            /* When we receive an external interrupt during execution of this single
+                               instruction, then we should stay here. We will leave when we're ready
+                               for raw-mode or when interrupted by pending EMT requests.  */
+                            interrupt_request = env->interrupt_request; /* reload this! */
+                            if (   !(interrupt_request & CPU_INTERRUPT_HARD)
+                                || !(env->eflags & IF_MASK)
+                                ||  (env->hflags & HF_INHIBIT_IRQ_MASK)
+                                ||  (env->state & CPU_RAW_HWACC)
+                               )
+                            {
+                                env->exception_index = ret = EXCP_SINGLE_INSTR;
+                                cpu_loop_exit();
+                            }
+                        }
+                        /* Clear CPU_INTERRUPT_SINGLE_INSTR and leave CPU_INTERRUPT_SINGLE_INSTR_IN_FLIGHT set. */
+                        ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request, ~CPU_INTERRUPT_SINGLE_INSTR);
+#  ifdef IEM_VERIFICATION_MODE
+                        env->exception_index = ret = EXCP_SINGLE_INSTR;
+                        cpu_loop_exit();
+#  endif
+                    }
+# endif /* VBOX */
+
+# ifndef VBOX /** @todo reconcile our code with the following...  */
                     if (interrupt_request & CPU_INTERRUPT_INIT) {
                             svm_check_intercept(SVM_EXIT_INIT);
@@ -768 +512 @@
                         }
                     }
+# else  /* VBOX */
+                    RAWEx_ProfileStart(env, STATS_IRQ_HANDLING);
+                    if ((interrupt_request & CPU_INTERRUPT_SMI) &&
+                        !(env->hflags & HF_SMM_MASK)) {
+                        env->interrupt_request &= ~CPU_INTERRUPT_SMI;
+                        do_smm_enter();
+                        next_tb = 0;
+                    }
+                    else if ((interrupt_request & CPU_INTERRUPT_HARD) &&
+                             (env->eflags & IF_MASK) &&
+                             !(env->hflags & HF_INHIBIT_IRQ_MASK))
+                    {
+                        /* if hardware interrupt pending, we execute it */
+                        int intno;
+                        ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request, ~CPU_INTERRUPT_HARD);
+                        intno = cpu_get_pic_interrupt(env);
+                        if (intno >= 0)
+                        {
+                            Log(("do_interrupt %d\n", intno));
+                            do_interrupt(intno, 0, 0, 0, 1);
+                        }
+                        /* ensure that no TB jump will be modified as
+                           the program flow was changed */
+                        next_tb = 0;
+                    }
+# endif /* VBOX */
 #elif defined(TARGET_PPC)
 #if 0
@@ -803 +573 @@
                     }
 #elif defined(TARGET_SPARC)
-                    if ((interrupt_request & CPU_INTERRUPT_HARD) &&
-                        cpu_interrupts_enabled(env)) {
-                        int pil = env->interrupt_index & 15;
-                        int type = env->interrupt_index & 0xf0;
-
-                        if (((type == TT_EXTINT) &&
-                             (pil == 15 || pil > env->psrpil)) ||
-                            type != TT_EXTINT) {
-                            env->interrupt_request &= ~CPU_INTERRUPT_HARD;
-                            env->exception_index = env->interrupt_index;
-                            do_interrupt(env);
-                            env->interrupt_index = 0;
-                        next_tb = 0;
-                        }
+                    if (interrupt_request & CPU_INTERRUPT_HARD) {
+                        if (cpu_interrupts_enabled(env) &&
+                            env->interrupt_index > 0) {
+                            int pil = env->interrupt_index & 0xf;
+                            int type = env->interrupt_index & 0xf0;
+
+                            if (((type == TT_EXTINT) &&
+                                  cpu_pil_allowed(env, pil)) ||
+                                  type != TT_EXTINT) {
+                                env->exception_index = env->interrupt_index;
+                                do_interrupt(env);
+                                next_tb = 0;
+                            }
+                        }
                     } else if (interrupt_request & CPU_INTERRUPT_TIMER) {
                         //do_interrupt(0, 0, 0, 0, 0);
@@ -856 +626 @@
 #elif defined(TARGET_CRIS)
                     if (interrupt_request & CPU_INTERRUPT_HARD
-                        && (env->pregs[PR_CCS] & I_FLAG)) {
+                        && (env->pregs[PR_CCS] & I_FLAG)
+                        && !env->locked_irq) {
                         env->exception_index = EXCP_IRQ;
                         do_interrupt(env);
@@ -884 +655 @@
                       do_interrupt may have updated the EXITTB flag. */
                     if (env->interrupt_request & CPU_INTERRUPT_EXITTB) {
+#ifndef VBOX
                         env->interrupt_request &= ~CPU_INTERRUPT_EXITTB;
+#else  /* VBOX */
+                        ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request, ~CPU_INTERRUPT_EXITTB);
+#endif /* VBOX */
                         /* ensure that no TB jump will be modified as
                            the program flow was changed */
                         next_tb = 0;
                     }
+#ifdef VBOX
+                    RAWEx_ProfileStop(env, STATS_IRQ_HANDLING);
+                    if (interrupt_request & CPU_INTERRUPT_RC) {
+                        env->exception_index = EXCP_RC;
+                        ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request, ~CPU_INTERRUPT_RC);
+                        ret = env->exception_index;
+                        cpu_loop_exit();
+                    }
+                    if (interrupt_request & (CPU_INTERRUPT_EXTERNAL_EXIT)) {
+                        ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request, ~(CPU_INTERRUPT_EXTERNAL_EXIT));
+                        env->exit_request = 1;
+                    }
+#endif
                 }
                 if (unlikely(env->exit_request)) {
@@ -895 +683 @@
                     cpu_loop_exit();
                 }
-#ifdef CONFIG_DEBUG_EXEC
+
+#ifdef VBOX
+                /*
+                 * Check if we the CPU state allows us to execute the code in raw-mode.
+                 */
+                RAWEx_ProfileStart(env, STATS_RAW_CHECK);
+                if (remR3CanExecuteRaw(env,
+                                       env->eip + env->segs[R_CS].base,
+                                       env->hflags | (env->eflags & (IOPL_MASK | TF_MASK | VM_MASK)),
+                                       &env->exception_index))
+                {
+                    RAWEx_ProfileStop(env, STATS_RAW_CHECK);
+                    ret = env->exception_index;
+                    cpu_loop_exit();
+                }
+                RAWEx_ProfileStop(env, STATS_RAW_CHECK);
+#endif /* VBOX */
+
+#if defined(DEBUG_DISAS) || defined(CONFIG_DEBUG_EXEC)
                 if (qemu_loglevel_mask(CPU_LOG_TB_CPU)) {
                     /* restore flags in standard format */
-                    regs_to_env();
 #if defined(TARGET_I386)
                     env->eflags = env->eflags | helper_cc_compute_all(CC_OP) | (DF & DF_MASK);
                     log_cpu_state(env, X86_DUMP_CCOP);
                     env->eflags &= ~(DF_MASK | CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C);
-#elif defined(TARGET_ARM)
-                    log_cpu_state(env, 0);
-#elif defined(TARGET_SPARC)
-                    log_cpu_state(env, 0);
-#elif defined(TARGET_PPC)
-                    log_cpu_state(env, 0);
 #elif defined(TARGET_M68K)
                     cpu_m68k_flush_flags(env, env->cc_op);
@@ -915 +714 @@
                               | env->cc_dest | (env->cc_x << 4);
                     log_cpu_state(env, 0);
-#elif defined(TARGET_MICROBLAZE)
+#else
                     log_cpu_state(env, 0);
-#elif defined(TARGET_MIPS)
-                    log_cpu_state(env, 0);
-#elif defined(TARGET_SH4)
-                    log_cpu_state(env, 0);
-#elif defined(TARGET_ALPHA)
-                    log_cpu_state(env, 0);
-#elif defined(TARGET_CRIS)
-                    log_cpu_state(env, 0);
-#else
-#error unsupported target CPU
 #endif
                 }
-#endif
+#endif /* DEBUG_DISAS || CONFIG_DEBUG_EXEC */
+#ifdef VBOX
+                RAWEx_ProfileStart(env, STATS_TLB_LOOKUP);
+#endif /*VBOX*/
                 spin_lock(&tb_lock);
                 tb = tb_find_fast();
@@ -949 +741 @@
                    spans two pages, we cannot safely do a direct
                    jump. */
-                {
-                    if (next_tb != 0 && tb->page_addr[1] == -1) {
+#ifndef VBOX
+                if (next_tb != 0 && tb->page_addr[1] == -1) {
+#else  /* VBOX */
+                if (next_tb != 0 && !(tb->cflags & CF_RAW_MODE) && tb->page_addr[1] == -1) {
+#endif /* VBOX */
                     tb_add_jump((TranslationBlock *)(next_tb & ~3), next_tb & 3, tb);
                 }
-                }
                 spin_unlock(&tb_lock);
-                env->current_tb = tb;
+#ifdef VBOX
+                RAWEx_ProfileStop(env, STATS_TLB_LOOKUP);
+#endif
 
                 /* cpu_interrupt might be called while translating the
@@ -961 +757 @@
                    infinite loop and becomes env->current_tb. Avoid
                    starting execution if there is a pending interrupt. */
-                if (unlikely (env->exit_request))
-                    env->current_tb = NULL;
-
-                while (env->current_tb) {
+                env->current_tb = tb;
+                barrier();
+                if (likely(!env->exit_request)) {
                     tc_ptr = tb->tc_ptr;
                 /* execute the generated code */
+#ifdef VBOX
+                    RAWEx_ProfileStart(env, STATS_QEMU_RUN_EMULATED_CODE);
+#endif
 #if defined(__sparc__) && !defined(CONFIG_SOLARIS)
 #undef env
@@ -972 +770 @@
 #define env cpu_single_env
 #endif
+#if defined(VBOX) && defined(GCC_WITH_BUGGY_REGPARM)
+                    tcg_qemu_tb_exec(tc_ptr, next_tb);
+#else
                     next_tb = tcg_qemu_tb_exec(tc_ptr);
-                    env->current_tb = NULL;
+#endif
+#ifdef VBOX
+                    RAWEx_ProfileStop(env, STATS_QEMU_RUN_EMULATED_CODE);
+#endif
                     if ((next_tb & 3) == 2) {
                         /* Instruction counter expired.  */
@@ -1002 +806 @@
                     }
                 }
+                env->current_tb = NULL;
                 /* reset soft MMU for next block (it can currently
                    only be set by a memory fault) */
             } /* for(;;) */
-        } else {
-            env_to_regs();
         }
+#ifdef VBOX_HIGH_RES_TIMERS_HACK
+        /* NULL the current_tb here so cpu_interrupt() doesn't do anything
+           unnecessary (like crashing during emulate single instruction).
+           Note! Don't use env1->pVM here, the code wouldn't run with
+                 gcc-4.4/amd64 anymore, see #3883. */
+        env->current_tb = NULL;
+        if (    !(env->interrupt_request & (  CPU_INTERRUPT_DEBUG | CPU_INTERRUPT_EXTERNAL_EXIT | CPU_INTERRUPT_RC
+                                            | CPU_INTERRUPT_SINGLE_INSTR | CPU_INTERRUPT_SINGLE_INSTR_IN_FLIGHT))
+            &&  (   (env->interrupt_request & CPU_INTERRUPT_EXTERNAL_TIMER)
+                 || TMTimerPollBool(env->pVM, env->pVCpu)) ) {
+            ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request, ~CPU_INTERRUPT_EXTERNAL_TIMER);
+            remR3ProfileStart(STATS_QEMU_RUN_TIMERS);
+            TMR3TimerQueuesDo(env->pVM);
+            remR3ProfileStop(STATS_QEMU_RUN_TIMERS);
+        }
+#endif
     } /* for(;;) */
 
@@ -1035 +854 @@
 
     /* restore global registers */
-#include "hostregs_helper.h"
-
+    barrier();
+    env = (void *) saved_env_reg;
+
+# ifndef VBOX /* we might be using elsewhere, we only have one. */
     /* fail safe : never use cpu_single_env outside cpu_exec() */
     cpu_single_env = NULL;
+# endif
     return ret;
 }
-
-#endif /* !VBOX */
 
 /* must only be called from the generated code as an exception can be
@@ -1288 +1108 @@
 # define TRAP_sig(context)                      REG_sig(trap, context)
 #endif /* linux */
+
+#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
+#include <ucontext.h>
+# define IAR_sig(context)               ((context)->uc_mcontext.mc_srr0)
+# define MSR_sig(context)               ((context)->uc_mcontext.mc_srr1)
+# define CTR_sig(context)               ((context)->uc_mcontext.mc_ctr)
+# define XER_sig(context)               ((context)->uc_mcontext.mc_xer)
+# define LR_sig(context)                ((context)->uc_mcontext.mc_lr)
+# define CR_sig(context)                ((context)->uc_mcontext.mc_cr)
+/* Exception Registers access */
+# define DAR_sig(context)               ((context)->uc_mcontext.mc_dar)
+# define DSISR_sig(context)             ((context)->uc_mcontext.mc_dsisr)
+# define TRAP_sig(context)              ((context)->uc_mcontext.mc_exc)
+#endif /* __FreeBSD__|| __FreeBSD_kernel__ */
 
 #ifdef __APPLE__
@@ -1318 +1152 @@
 {
     siginfo_t *info = pinfo;
+#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
+    ucontext_t *uc = puc;
+#else
     struct ucontext *uc = puc;
+#endif
     unsigned long pc;
     int is_write;
@@ -1494 +1332 @@
     return handle_cpu_signal(ip, (unsigned long)info->si_addr,
                              is_write,
-                             &uc->uc_sigmask, puc);
+                             (sigset_t *)&uc->uc_sigmask, puc);
 }
 
@@ -1505 +1343 @@
     struct ucontext *uc = puc;
     unsigned long pc;
+    uint16_t *pinsn;
+    int is_write = 0;
+
+    pc = uc->uc_mcontext.psw.addr;
+
+    /* ??? On linux, the non-rt signal handler has 4 (!) arguments instead
+       of the normal 2 arguments.  The 3rd argument contains the "int_code"
+       from the hardware which does in fact contain the is_write value.
+       The rt signal handler, as far as I can tell, does not give this value
+       at all.  Not that we could get to it from here even if it were.  */
+    /* ??? This is not even close to complete, since it ignores all
+       of the read-modify-write instructions.  */
+    pinsn = (uint16_t *)pc;
+    switch (pinsn[0] >> 8) {
+    case 0x50: /* ST */
+    case 0x42: /* STC */
+    case 0x40: /* STH */
+        is_write = 1;
+        break;
+    case 0xc4: /* RIL format insns */
+        switch (pinsn[0] & 0xf) {
+        case 0xf: /* STRL */
+        case 0xb: /* STGRL */
+        case 0x7: /* STHRL */
+            is_write = 1;
+        }
+        break;
+    case 0xe3: /* RXY format insns */
+        switch (pinsn[2] & 0xff) {
+        case 0x50: /* STY */
+        case 0x24: /* STG */
+        case 0x72: /* STCY */
+        case 0x70: /* STHY */
+        case 0x8e: /* STPQ */
+        case 0x3f: /* STRVH */
+        case 0x3e: /* STRV */
+        case 0x2f: /* STRVG */
+            is_write = 1;
+        }
+        break;
+    }
+    return handle_cpu_signal(pc, (unsigned long)info->si_addr,
+                             is_write, &uc->uc_sigmask, puc);
+}
+
+#elif defined(__mips__)
+
+int cpu_signal_handler(int host_signum, void *pinfo,
+                       void *puc)
+{
+    siginfo_t *info = pinfo;
+    struct ucontext *uc = puc;
+    greg_t pc = uc->uc_mcontext.pc;
     int is_write;
 
-    pc = uc->uc_mcontext.psw.addr;
     /* XXX: compute is_write */
     is_write = 0;
@@ -1514 +1404 @@
 }
 
-#elif defined(__mips__)
+#elif defined(__hppa__)
 
 int cpu_signal_handler(int host_signum, void *pinfo,
                        void *puc)
 {
-    siginfo_t *info = pinfo;
+    struct siginfo *info = pinfo;
     struct ucontext *uc = puc;
-    greg_t pc = uc->uc_mcontext.pc;
-    int is_write;
-
-    /* XXX: compute is_write */
-    is_write = 0;
+    unsigned long pc = uc->uc_mcontext.sc_iaoq[0];
+    uint32_t insn = *(uint32_t *)pc;
+    int is_write = 0;
+
+    /* XXX: need kernel patch to get write flag faster.  */
+    switch (insn >> 26) {
+    case 0x1a: /* STW */
+    case 0x19: /* STH */
+    case 0x18: /* STB */
+    case 0x1b: /* STWM */
+        is_write = 1;
+        break;
+
+    case 0x09: /* CSTWX, FSTWX, FSTWS */
+    case 0x0b: /* CSTDX, FSTDX, FSTDS */
+        /* Distinguish from coprocessor load ... */
+        is_write = (insn >> 9) & 1;
+        break;
+
+    case 0x03:
+        switch ((insn >> 6) & 15) {
+        case 0xa: /* STWS */
+        case 0x9: /* STHS */
+        case 0x8: /* STBS */
+        case 0xe: /* STWAS */
+        case 0xc: /* STBYS */
+            is_write = 1;
+        }
+        break;
+    }
+
     return handle_cpu_signal(pc, (unsigned long)info->si_addr,
                              is_write, &uc->uc_sigmask, puc);
 }
 
-#elif defined(__hppa__)
-
-int cpu_signal_handler(int host_signum, void *pinfo,
-                       void *puc)
-{
-    struct siginfo *info = pinfo;
-    struct ucontext *uc = puc;
-    unsigned long pc;
-    int is_write;
-
-    pc = uc->uc_mcontext.sc_iaoq[0];
-    /* FIXME: compute is_write */
-    is_write = 0;
-    return handle_cpu_signal(pc, (unsigned long)info->si_addr,
-                             is_write,
-                             &uc->uc_sigmask, puc);
-}
-
 #else
 

trunk/src/recompiler/cutils.c

@@ -612 +612 @@
 
 #ifndef VBOX
+
 /*
  * Make sure data goes on disk, but if possible do not bother to
@@ -731 +732 @@
 }
 
+#ifndef _WIN32
+/* Sets a specific flag */
+int fcntl_setfl(int fd, int flag)
+{
+    int flags;
+
+    flags = fcntl(fd, F_GETFL);
+    if (flags == -1)
+        return -errno;
+
+    if (fcntl(fd, F_SETFL, flags | flag) == -1)
+        return -errno;
+
+    return 0;
+}
+#endif
+
 #endif /* !VBOX */
+

trunk/src/recompiler/def-helper.h

@@ -100 +100 @@
 #define dh_is_64bit(t) glue(dh_is_64bit_, dh_alias(t))
 
+#define dh_is_signed_void 0
+#define dh_is_signed_i32 0
+#define dh_is_signed_s32 1
+#define dh_is_signed_i64 0
+#define dh_is_signed_s64 1
+#define dh_is_signed_f32 0
+#define dh_is_signed_f64 0
+#define dh_is_signed_tl  0
+#define dh_is_signed_int 1
+/* ??? This is highly specific to the host cpu.  There are even special
+   extension instructions that may be required, e.g. ia64's addp4.  But
+   for now we don't support any 64-bit targets with 32-bit pointers.  */
+#define dh_is_signed_ptr 0
+#define dh_is_signed_env dh_is_signed_ptr
+#define dh_is_signed(t) dh_is_signed_##t
+
+#define dh_sizemask(t, n) \
+  sizemask |= dh_is_64bit(t) << (n*2); \
+  sizemask |= dh_is_signed(t) << (n*2+1)
+
 #define dh_arg(t, n) \
   args[n - 1] = glue(GET_TCGV_, dh_alias(t))(glue(arg, n)); \
-  sizemask |= dh_is_64bit(t) << n
+  dh_sizemask(t, n)
 
 #define dh_arg_decl(t, n) glue(TCGv_, dh_alias(t)) glue(arg, n)
@@ -157 +177 @@
 { \
   TCGArg args[1]; \
-  int sizemask; \
-  sizemask = dh_is_64bit(ret); \
+  int sizemask = 0; \
+  dh_sizemask(ret, 0); \
   dh_arg(t1, 1); \
   tcg_gen_helperN(HELPER(name), flags, sizemask, dh_retvar(ret), 1, args); \
@@ -168 +188 @@
 { \
   TCGArg args[2]; \
-  int sizemask; \
-  sizemask = dh_is_64bit(ret); \
+  int sizemask = 0; \
+  dh_sizemask(ret, 0); \
   dh_arg(t1, 1); \
   dh_arg(t2, 2); \
@@ -180 +200 @@
 { \
   TCGArg args[3]; \
-  int sizemask; \
-  sizemask = dh_is_64bit(ret); \
+  int sizemask = 0; \
+  dh_sizemask(ret, 0); \
   dh_arg(t1, 1); \
   dh_arg(t2, 2); \
@@ -193 +213 @@
 { \
   TCGArg args[4]; \
-  int sizemask; \
-  sizemask = dh_is_64bit(ret); \
+  int sizemask = 0; \
+  dh_sizemask(ret, 0); \
   dh_arg(t1, 1); \
   dh_arg(t2, 2); \

trunk/src/recompiler/disas.h

@@ -25 +25 @@
 struct elf64_sym;
 
+#if defined(CONFIG_USER_ONLY)
+typedef const char *(*lookup_symbol_t)(struct syminfo *s, target_ulong orig_addr);
+#else
 typedef const char *(*lookup_symbol_t)(struct syminfo *s, target_phys_addr_t orig_addr);
+#endif
 
 struct syminfo {

trunk/src/recompiler/dyngen-exec.h

@@ -73 +73 @@
 
 #if defined(__i386__)
-#ifndef VBOX
+# ifndef VBOX
 #define AREG0 "ebp"
-#define AREG1 "ebx"
-#define AREG2 "esi"
-#else  /* VBOX - why are we different? */
-# define AREG0 "esi"
-# define AREG1 "edi"
-#endif /* VBOX */
+# else  /* VBOX - why are we different? frame-pointer optimizations on mac? */
+#  define AREG0 "esi"
+# endif /* VBOX */
 #elif defined(__x86_64__)
 #define AREG0 "r14"
-#define AREG1 "r15"
-#define AREG2 "r12"
 #elif defined(_ARCH_PPC)
 #define AREG0 "r27"
-#define AREG1 "r24"
-#define AREG2 "r25"
 #elif defined(__arm__)
 #define AREG0 "r7"
-#define AREG1 "r4"
-#define AREG2 "r5"
 #elif defined(__hppa__)
 #define AREG0 "r17"
-#define AREG1 "r14"
-#define AREG2 "r15"
 #elif defined(__mips__)
 #define AREG0 "s0"
-#define AREG1 "s1"
-#define AREG2 "fp"
 #elif defined(__sparc__)
 #ifdef CONFIG_SOLARIS
 #define AREG0 "g2"
-#define AREG1 "g3"
-#define AREG2 "g4"
 #else
 #ifdef __sparc_v9__
 #define AREG0 "g5"
-#define AREG1 "g6"
-#define AREG2 "g7"
 #else
 #define AREG0 "g6"
-#define AREG1 "g1"
-#define AREG2 "g2"
 #endif
 #endif
 #elif defined(__s390__)
 #define AREG0 "r10"
-#define AREG1 "r7"
-#define AREG2 "r8"
 #elif defined(__alpha__)
 /* Note $15 is the frame pointer, so anything in op-i386.c that would
    require a frame pointer, like alloca, would probably loose.  */
 #define AREG0 "$15"
-#define AREG1 "$9"
-#define AREG2 "$10"
 #elif defined(__mc68000)
 #define AREG0 "%a5"
-#define AREG1 "%a4"
-#define AREG2 "%d7"
 #elif defined(__ia64__)
 #define AREG0 "r7"
-#define AREG1 "r4"
-#define AREG2 "r5"
 #else
 #error unsupported CPU

trunk/src/recompiler/elf.h

@@ -120 +120 @@
 #define EM_S390_OLD     0xA390
 
-#define EM_XILINX_MICROBLAZE    0xBAAB
+#define EM_MICROBLAZE      189
+#define EM_MICROBLAZE_OLD  0xBAAB
 
 /* This is the info that is needed to parse the dynamic section of the file */
@@ -244 +245 @@
 #define R_386_GOTPC     10
 #define R_386_NUM       11
+/* Not a dynamic reloc, so not included in R_386_NUM.  Used in TCG.  */
+#define R_386_PC8       23
 
 #define R_MIPS_NONE             0

trunk/src/recompiler/exec-all.h

@@ -31 +31 @@
 
 #include "qemu-common.h"
-
-/* allow to see translation results - the slowdown should be negligible, so we leave it */
-#ifndef VBOX
-#define DEBUG_DISAS
-#endif
-
 #ifdef VBOX
 # include <VBox/vmm/tm.h>
@@ -48 +42 @@
 #endif /* VBOX */
 
+/* allow to see translation results - the slowdown should be negligible, so we leave it */
+#ifndef VBOX
+#define DEBUG_DISAS
+#endif /* !VBOX */
+
+/* Page tracking code uses ram addresses in system mode, and virtual
+   addresses in userspace mode.  Define tb_page_addr_t to be an appropriate
+   type.  */
+#if defined(CONFIG_USER_ONLY)
+typedef abi_ulong tb_page_addr_t;
+#else
+typedef ram_addr_t tb_page_addr_t;
+#endif
+
 /* is_jmp field values */
 #define DISAS_NEXT    0 /* next instruction can be analyzed */
@@ -58 +66 @@
 /* XXX: make safe guess about sizes */
 #define MAX_OP_PER_INSTR 96
-/* A Call op needs up to 6 + 2N parameters (N = number of arguments).  */
-#define MAX_OPC_PARAM 10
+
+#if HOST_LONG_BITS == 32
+#define MAX_OPC_PARAM_PER_ARG 2
+#else
+#define MAX_OPC_PARAM_PER_ARG 1
+#endif
+#define MAX_OPC_PARAM_IARGS 4
+#define MAX_OPC_PARAM_OARGS 1
+#define MAX_OPC_PARAM_ARGS (MAX_OPC_PARAM_IARGS + MAX_OPC_PARAM_OARGS)
+
+/* A Call op needs up to 4 + 2N parameters on 32-bit archs,
+ * and up to 4 + N parameters on 64-bit archs
+ * (N = number of input arguments + output arguments).  */
+#define MAX_OPC_PARAM (4 + (MAX_OPC_PARAM_PER_ARG * MAX_OPC_PARAM_ARGS))
 #define OPC_BUF_SIZE 640
 #define OPC_MAX_SIZE (OPC_BUF_SIZE - MAX_OP_PER_INSTR)
@@ -72 +92 @@
 
 extern target_ulong gen_opc_pc[OPC_BUF_SIZE];
-extern target_ulong gen_opc_npc[OPC_BUF_SIZE];
-extern uint8_t gen_opc_cc_op[OPC_BUF_SIZE];
 extern uint8_t gen_opc_instr_start[OPC_BUF_SIZE];
 extern uint16_t gen_opc_icount[OPC_BUF_SIZE];
-extern target_ulong gen_opc_jump_pc[2];
-extern uint32_t gen_opc_hflags[OPC_BUF_SIZE];
 
 #include "qemu-log.h"
@@ -86 +102 @@
                  unsigned long searched_pc, int pc_pos, void *puc);
 
-unsigned long code_gen_max_block_size(void);
 void cpu_gen_init(void);
 int cpu_gen_code(CPUState *env, struct TranslationBlock *tb,
@@ -93 +108 @@
                       CPUState *env, unsigned long searched_pc,
                       void *puc);
-int cpu_restore_state_copy(struct TranslationBlock *tb,
-                           CPUState *env, unsigned long searched_pc,
-                           void *puc);
 void cpu_resume_from_signal(CPUState *env1, void *puc);
 void cpu_io_recompile(CPUState *env, void *retaddr);
@@ -104 +116 @@
 void QEMU_NORETURN cpu_loop_exit(void);
 int page_unprotect(target_ulong address, unsigned long pc, void *puc);
-void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
+void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
                                    int is_cpu_write_access);
 void tb_invalidate_page_range(target_ulong start, target_ulong end);
 void tlb_flush_page(CPUState *env, target_ulong addr);
 void tlb_flush(CPUState *env, int flush_global);
-int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
-                      target_phys_addr_t paddr, int prot,
-                      int mmu_idx, int is_softmmu);
-static inline int tlb_set_page(CPUState *env1, target_ulong vaddr,
-                               target_phys_addr_t paddr, int prot,
-                               int mmu_idx, int is_softmmu)
-{
-    if (prot & PAGE_READ)
-        prot |= PAGE_EXEC;
-    return tlb_set_page_exec(env1, vaddr, paddr, prot, mmu_idx, is_softmmu);
-}
+#if !defined(CONFIG_USER_ONLY)
+void tlb_set_page(CPUState *env, target_ulong vaddr,
+                  target_phys_addr_t paddr, int prot,
+                  int mmu_idx, target_ulong size);
+#endif
 
 #define CODE_GEN_ALIGN           16 /* must be >= of the size of a icache line */
@@ -142 +148 @@
 
 #ifdef VBOX /* bird: not safe in next step because of threading & cpu_interrupt. */
-#undef USE_DIRECT_JUMP
+# undef USE_DIRECT_JUMP
 #endif /* VBOX */
 
@@ -154 +160 @@
 #define CF_COUNT_MASK  0x7fff
 #define CF_LAST_IO     0x8000 /* Last insn may be an IO access.  */
-
 #ifdef VBOX
-#define CF_RAW_MODE    0x0010 /* block was generated in raw mode */
+# define CF_RAW_MODE   0x0010 /* block was generated in raw mode */
 #endif
 
@@ -165 +170 @@
        of the pointer tells the index in page_next[] */
     struct TranslationBlock *page_next[2];
-    target_ulong page_addr[2];
+    tb_page_addr_t page_addr[2];
 
     /* the following data are used to directly call another TB from
@@ -171 +176 @@
     uint16_t tb_next_offset[2]; /* offset of original jump target */
 #ifdef USE_DIRECT_JUMP
-    uint16_t tb_jmp_offset[4]; /* offset of jump instruction */
+    uint16_t tb_jmp_offset[2]; /* offset of jump instruction */
 #else
     unsigned long tb_next[2]; /* address of jump generated code */
@@ -199 +204 @@
 }
 
-static inline unsigned int tb_phys_hash_func(unsigned long pc)
+static inline unsigned int tb_phys_hash_func(tb_page_addr_t pc)
 {
     return pc & (CODE_GEN_PHYS_HASH_SIZE - 1);
@@ -207 +212 @@
 void tb_free(TranslationBlock *tb);
 void tb_flush(CPUState *env);
-void tb_link_phys(TranslationBlock *tb,
-                  target_ulong phys_pc, target_ulong phys_page2);
-void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr);
+void tb_link_page(TranslationBlock *tb,
+                  tb_page_addr_t phys_pc, tb_page_addr_t phys_page2);
+void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr);
 
 extern TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
-extern uint8_t *code_gen_ptr;
-extern int code_gen_max_blocks;
 
 #if defined(USE_DIRECT_JUMP)
@@ -262 +265 @@
     offset = tb->tb_jmp_offset[n];
     tb_set_jmp_target1((unsigned long)(tb->tc_ptr + offset), addr);
-    offset = tb->tb_jmp_offset[n + 2];
-    if (offset != 0xffff)
-        tb_set_jmp_target1((unsigned long)(tb->tc_ptr + offset), addr);
 }
 
@@ -294 +294 @@
 TranslationBlock *tb_find_pc(unsigned long pc_ptr);
 
+#include "qemu-lock.h"
+
+extern spinlock_t tb_lock;
+
+extern int tb_invalidated_flag;
+
+#if !defined(CONFIG_USER_ONLY)
+
 extern CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
 extern CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
 extern void *io_mem_opaque[IO_MEM_NB_ENTRIES];
 
-#include "qemu-lock.h"
-
-extern spinlock_t tb_lock;
-
-extern int tb_invalidated_flag;
-
-#if !defined(CONFIG_USER_ONLY)
-
 void tlb_fill(target_ulong addr, int is_write, int mmu_idx,
               void *retaddr);
@@ -334 +334 @@
 
 #if defined(CONFIG_USER_ONLY)
-static inline target_ulong get_phys_addr_code(CPUState *env1, target_ulong addr)
+static inline tb_page_addr_t get_page_addr_code(CPUState *env1, target_ulong addr)
 {
     return addr;
@@ -345 +345 @@
 /* NOTE2: the returned address is not exactly the physical address: it
    is the offset relative to phys_ram_base */
-static inline target_ulong get_phys_addr_code(CPUState *env1, target_ulong addr)
+static inline tb_page_addr_t get_page_addr_code(CPUState *env1, target_ulong addr)
 {
     int mmu_idx, page_index, pd;
@@ -369 +369 @@
 #endif
     }
-
 # if defined(VBOX) && defined(REM_PHYS_ADDR_IN_TLB)
     return addr + env1->tlb_table[mmu_idx][page_index].addend;
@@ -381 +380 @@
 # endif
 }
-
-/* Deterministic execution requires that IO only be performed on the last
-   instruction of a TB so that interrupts take effect immediately.  */
-static inline int can_do_io(CPUState *env)
-{
-    if (!use_icount)
-        return 1;
-
-    /* If not executing code then assume we are ok.  */
-    if (!env->current_tb)
-        return 1;
-
-    return env->can_do_io != 0;
-}
 #endif
 
@@ -401 +386 @@
 CPUDebugExcpHandler *cpu_set_debug_excp_handler(CPUDebugExcpHandler *handler);
 
+#ifndef VBOX
 /* vl.c */
-#ifndef VBOX
 extern int singlestep;
+
+/* cpu-exec.c */
+extern volatile sig_atomic_t exit_request;
 #endif /*!VBOX*/
 
-#endif
+
+#endif

trunk/src/recompiler/exec.c

@@ -57 +57 @@
 #ifndef VBOX
 #include "hw/hw.h"
-#endif
+#include "hw/qdev.h"
+#endif /* !VBOX */
 #include "osdep.h"
 #include "kvm.h"
+#include "qemu-timer.h"
 #if defined(CONFIG_USER_ONLY)
 #include <qemu.h>
+#include <signal.h>
+#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
+#include <sys/param.h>
+#if __FreeBSD_version >= 700104
+#define HAVE_KINFO_GETVMMAP
+#define sigqueue sigqueue_freebsd  /* avoid redefinition */
+#include <sys/time.h>
+#include <sys/proc.h>
+#include <machine/profile.h>
+#define _KERNEL
+#include <sys/user.h>
+#undef _KERNEL
+#undef sigqueue
+#include <libutil.h>
+#endif
+#endif
 #endif
 
@@ -83 +101 @@
 #define SMC_BITMAP_USE_THRESHOLD 10
 
-#if defined(TARGET_SPARC64)
-#define TARGET_PHYS_ADDR_SPACE_BITS 41
-#elif defined(TARGET_SPARC)
-#define TARGET_PHYS_ADDR_SPACE_BITS 36
-#elif defined(TARGET_ALPHA)
-#define TARGET_PHYS_ADDR_SPACE_BITS 42
-#define TARGET_VIRT_ADDR_SPACE_BITS 42
-#elif defined(TARGET_PPC64)
-#define TARGET_PHYS_ADDR_SPACE_BITS 42
-#elif defined(TARGET_X86_64)
-#define TARGET_PHYS_ADDR_SPACE_BITS 42
-#elif defined(TARGET_I386)
-#define TARGET_PHYS_ADDR_SPACE_BITS 36
-#else
-#define TARGET_PHYS_ADDR_SPACE_BITS 32
-#endif
-
 static TranslationBlock *tbs;
-int code_gen_max_blocks;
+static int code_gen_max_blocks;
 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
 static int nb_tbs;
@@ -126 +127 @@
 uint8_t code_gen_prologue[1024] code_gen_section;
 #else /* VBOX */
-extern uint8_t* code_gen_prologue;
+extern uint8_t *code_gen_prologue;
 #endif /* VBOX */
 static uint8_t *code_gen_buffer;
@@ -132 +133 @@
 /* threshold to flush the translated code buffer */
 static unsigned long code_gen_buffer_max_size;
-uint8_t *code_gen_ptr;
-
-#ifndef VBOX
+static uint8_t *code_gen_ptr;
+
 #if !defined(CONFIG_USER_ONLY)
+# ifndef VBOX
 int phys_ram_fd;
-uint8_t *phys_ram_dirty;
 static int in_migration;
-
-typedef struct RAMBlock {
-    uint8_t *host;
-    ram_addr_t offset;
-    ram_addr_t length;
-    struct RAMBlock *next;
-} RAMBlock;
-
-static RAMBlock *ram_blocks;
-/* TODO: When we implement (and use) ram deallocation (e.g. for hotplug)
-   then we can no longer assume contiguous ram offsets, and external uses
-   of this variable will break.  */
-ram_addr_t last_ram_offset;
-#endif
-#else /* VBOX */
-/* we have memory ranges (the high PC-BIOS mapping) which
-   causes some pages to fall outside the dirty map here. */
-RTGCPHYS phys_ram_dirty_size;
-uint8_t *phys_ram_dirty;
-#endif /* VBOX */
+# endif /* !VBOX */
+
+RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list) };
+#endif
 
 CPUState *first_cpu;
@@ -184 +168 @@
 } PageDesc;
 
+/* In system mode we want L1_MAP to be based on ram offsets,
+   while in user mode we want it to be based on virtual addresses.  */
+#if !defined(CONFIG_USER_ONLY)
+#if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
+# define L1_MAP_ADDR_SPACE_BITS  HOST_LONG_BITS
+#else
+# define L1_MAP_ADDR_SPACE_BITS  TARGET_PHYS_ADDR_SPACE_BITS
+#endif
+#else
+# define L1_MAP_ADDR_SPACE_BITS  TARGET_VIRT_ADDR_SPACE_BITS
+#endif
+
+/* Size of the L2 (and L3, etc) page tables.  */
+#define L2_BITS 10
+#define L2_SIZE (1 << L2_BITS)
+
+/* The bits remaining after N lower levels of page tables.  */
+#define P_L1_BITS_REM \
+    ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
+#define V_L1_BITS_REM \
+    ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
+
+/* Size of the L1 page table.  Avoid silly small sizes.  */
+#if P_L1_BITS_REM < 4
+#define P_L1_BITS  (P_L1_BITS_REM + L2_BITS)
+#else
+#define P_L1_BITS  P_L1_BITS_REM
+#endif
+
+#if V_L1_BITS_REM < 4
+#define V_L1_BITS  (V_L1_BITS_REM + L2_BITS)
+#else
+#define V_L1_BITS  V_L1_BITS_REM
+#endif
+
+#define P_L1_SIZE  ((target_phys_addr_t)1 << P_L1_BITS)
+#define V_L1_SIZE  ((target_ulong)1 << V_L1_BITS)
+
+#define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
+#define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
+
+unsigned long qemu_real_host_page_size;
+unsigned long qemu_host_page_bits;
+unsigned long qemu_host_page_size;
+unsigned long qemu_host_page_mask;
+
+/* This is a multi-level map on the virtual address space.
+   The bottom level has pointers to PageDesc.  */
+static void *l1_map[V_L1_SIZE];
+
+#if !defined(CONFIG_USER_ONLY)
 typedef struct PhysPageDesc {
     /* offset in host memory of the page + io_index in the low bits */
@@ -190 +225 @@
 } PhysPageDesc;
 
-#define L2_BITS 10
-#if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
-/* XXX: this is a temporary hack for alpha target.
- *      In the future, this is to be replaced by a multi-level table
- *      to actually be able to handle the complete 64 bits address space.
- */
-#define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
-#else
-#define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
-#endif
-#ifdef VBOX
-#define L0_BITS (TARGET_PHYS_ADDR_SPACE_BITS - 32)
-#endif
-
-#ifdef VBOX
-#define L0_SIZE (1 << L0_BITS)
-#endif
-#define L1_SIZE (1 << L1_BITS)
-#define L2_SIZE (1 << L2_BITS)
-
-unsigned long qemu_real_host_page_size;
-unsigned long qemu_host_page_bits;
-unsigned long qemu_host_page_size;
-unsigned long qemu_host_page_mask;
-
-/* XXX: for system emulation, it could just be an array */
-#ifndef VBOX
-static PageDesc *l1_map[L1_SIZE];
-static PhysPageDesc **l1_phys_map;
-#else
-static unsigned l0_map_max_used = 0;
-static PageDesc **l0_map[L0_SIZE];
-static void **l0_phys_map[L0_SIZE];
-#endif
-
-#if !defined(CONFIG_USER_ONLY)
+/* This is a multi-level map on the physical address space.
+   The bottom level has pointers to PhysPageDesc.  */
+static void *l1_phys_map[P_L1_SIZE];
+
 static void io_mem_init(void);
 
@@ -238 +241 @@
 #ifndef VBOX
 /* log support */
+#ifdef WIN32
+static const char *logfilename = "qemu.log";
+#else
 static const char *logfilename = "/tmp/qemu.log";
+#endif
 #endif /* !VBOX */
 FILE *logfile;
@@ -244 +251 @@
 #ifndef VBOX
 static int log_append = 0;
-#endif
+#endif /* !VBOX */
 
 /* statistics */
 #ifndef VBOX
+#if !defined(CONFIG_USER_ONLY)
 static int tlb_flush_count;
+#endif
 static int tb_flush_count;
 static int tb_phys_invalidate_count;
@@ -256 +265 @@
 uint32_t tb_phys_invalidate_count;
 #endif /* VBOX */
-
-#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
-typedef struct subpage_t {
-    target_phys_addr_t base;
-    CPUReadMemoryFunc * const *mem_read[TARGET_PAGE_SIZE][4];
-    CPUWriteMemoryFunc * const *mem_write[TARGET_PAGE_SIZE][4];
-    void *opaque[TARGET_PAGE_SIZE][2][4];
-    ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
-} subpage_t;
 
 #ifndef VBOX
@@ -325 +325 @@
         qemu_host_page_size = TARGET_PAGE_SIZE;
     qemu_host_page_bits = 0;
-#ifndef VBOX
-    while ((1 << qemu_host_page_bits) < qemu_host_page_size)
-#else
-    while ((1 << qemu_host_page_bits) < (int)qemu_host_page_size)
-#endif
+    while ((1 << qemu_host_page_bits) < VBOX_ONLY((int))qemu_host_page_size)
         qemu_host_page_bits++;
     qemu_host_page_mask = ~(qemu_host_page_size - 1);
-#ifndef VBOX
-    l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
-    memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
-#endif
-
-#ifdef VBOX
-    /* We use other means to set reserved bit on our pages */
-#else  /* !VBOX */
-#if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
+
+#ifndef VBOX /* We use other means to set reserved bit on our pages */
+#if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
     {
-        long long startaddr, endaddr;
+#ifdef HAVE_KINFO_GETVMMAP
+        struct kinfo_vmentry *freep;
+        int i, cnt;
+
+        freep = kinfo_getvmmap(getpid(), &cnt);
+        if (freep) {
+            mmap_lock();
+            for (i = 0; i < cnt; i++) {
+                unsigned long startaddr, endaddr;
+
+                startaddr = freep[i].kve_start;
+                endaddr = freep[i].kve_end;
+                if (h2g_valid(startaddr)) {
+                    startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
+
+                    if (h2g_valid(endaddr)) {
+                        endaddr = h2g(endaddr);
+                        page_set_flags(startaddr, endaddr, PAGE_RESERVED);
+                    } else {
+#if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
+                        endaddr = ~0ul;
+                        page_set_flags(startaddr, endaddr, PAGE_RESERVED);
+#endif
+                    }
+                }
+            }
+            free(freep);
+            mmap_unlock();
+        }
+#else
         FILE *f;
-        int n;
-
-        mmap_lock();
+
         last_brk = (unsigned long)sbrk(0);
-        f = fopen("/proc/self/maps", "r");
+
+        f = fopen("/compat/linux/proc/self/maps", "r");
         if (f) {
+            mmap_lock();
+
             do {
-                n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
-                if (n == 2) {
-                    startaddr = MIN(startaddr,
-                                    (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
-                    endaddr = MIN(endaddr,
-                                    (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
-                    page_set_flags(startaddr & TARGET_PAGE_MASK,
-                                   TARGET_PAGE_ALIGN(endaddr),
-                                   PAGE_RESERVED);
+                unsigned long startaddr, endaddr;
+                int n;
+
+                n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
+
+                if (n == 2 && h2g_valid(startaddr)) {
+                    startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
+
+                    if (h2g_valid(endaddr)) {
+                        endaddr = h2g(endaddr);
+                    } else {
+                        endaddr = ~0ul;
+                    }
+                    page_set_flags(startaddr, endaddr, PAGE_RESERVED);
                 }
             } while (!feof(f));
+
             fclose(f);
-        }
-        mmap_unlock();
+            mmap_unlock();
+        }
+#endif
     }
 #endif
@@ -370 +397 @@
 }
 
-static inline PageDesc **page_l1_map(target_ulong index)
-{
-#ifndef VBOX
-#if TARGET_LONG_BITS > 32
-    /* Host memory outside guest VM.  For 32-bit targets we have already
-       excluded high addresses.  */
-    if (index > ((target_ulong)L2_SIZE * L1_SIZE))
-        return NULL;
-#endif
-    return &l1_map[index >> L2_BITS];
-#else  /* VBOX */
-    PageDesc **l1_map;
-    AssertMsgReturn(index < (target_ulong)L2_SIZE * L1_SIZE * L0_SIZE,
-                    ("index=%RGp >= %RGp; L1_SIZE=%#x L2_SIZE=%#x L0_SIZE=%#x\n",
-                     (RTGCPHYS)index, (RTGCPHYS)L2_SIZE * L1_SIZE, L1_SIZE, L2_SIZE, L0_SIZE),
-                    NULL);
-    l1_map = l0_map[index >> (L1_BITS + L2_BITS)];
-    if (RT_UNLIKELY(!l1_map))
-    {
-        unsigned i0 = index >> (L1_BITS + L2_BITS);
-        l0_map[i0] = l1_map = qemu_mallocz(sizeof(PageDesc *) * L1_SIZE);
-        if (RT_UNLIKELY(!l1_map))
+static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
+{
+    PageDesc *pd;
+    void **lp;
+    int i;
+
+#if defined(CONFIG_USER_ONLY)
+    /* We can't use qemu_malloc because it may recurse into a locked mutex. */
+# define ALLOC(P, SIZE)                                 \
+    do {                                                \
+        P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,    \
+                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);   \
+    } while (0)
+#else
+# define ALLOC(P, SIZE) \
+    do { P = qemu_mallocz(SIZE); } while (0)
+#endif
+
+    /* Level 1.  Always allocated.  */
+    lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
+
+    /* Level 2..N-1.  */
+    for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
+        void **p = *lp;
+
+        if (p == NULL) {
+            if (!alloc) {
+                return NULL;
+            }
+            ALLOC(p, sizeof(void *) * L2_SIZE);
+            *lp = p;
+        }
+
+        lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
+    }
+
+    pd = *lp;
+    if (pd == NULL) {
+        if (!alloc) {
             return NULL;
-        if (i0 >= l0_map_max_used)
-            l0_map_max_used = i0 + 1;
-    }
-    return &l1_map[(index >> L2_BITS) & (L1_SIZE - 1)];
-#endif /* VBOX */
-}
-
-static inline PageDesc *page_find_alloc(target_ulong index)
-{
-    PageDesc **lp, *p;
-    lp = page_l1_map(index);
-    if (!lp)
-        return NULL;
-
-    p = *lp;
-    if (!p) {
-        /* allocate if not found */
-#if defined(CONFIG_USER_ONLY)
-        size_t len = sizeof(PageDesc) * L2_SIZE;
-        /* Don't use qemu_malloc because it may recurse.  */
-        p = mmap(NULL, len, PROT_READ | PROT_WRITE,
-                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-        *lp = p;
-        if (h2g_valid(p)) {
-            unsigned long addr = h2g(p);
-            page_set_flags(addr & TARGET_PAGE_MASK,
-                           TARGET_PAGE_ALIGN(addr + len),
-                           PAGE_RESERVED);
-        }
-#else
-        p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);
-        *lp = p;
-#endif
-    }
-    return p + (index & (L2_SIZE - 1));
-}
-
-static inline PageDesc *page_find(target_ulong index)
-{
-    PageDesc **lp, *p;
-    lp = page_l1_map(index);
-    if (!lp)
-        return NULL;
-
-    p = *lp;
-    if (!p) {
-        return NULL;
-    }
-    return p + (index & (L2_SIZE - 1));
-}
-
+        }
+        ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
+        *lp = pd;
+    }
+
+#undef ALLOC
+
+    return pd + (index & (L2_SIZE - 1));
+}
+
+static inline PageDesc *page_find(tb_page_addr_t index)
+{
+    return page_find_alloc(index, 0);
+}
+
+#if !defined(CONFIG_USER_ONLY)
 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
 {
-    void **lp, **p;
     PhysPageDesc *pd;
-
-#ifndef VBOX
-    p = (void **)l1_phys_map;
-#if TARGET_PHYS_ADDR_SPACE_BITS > 32
-
-#if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
-#error unsupported TARGET_PHYS_ADDR_SPACE_BITS
-#endif
-    lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
-    p = *lp;
-    if (!p) {
-        /* allocate if not found */
-        if (!alloc)
+    void **lp;
+    int i;
+
+    /* Level 1.  Always allocated.  */
+    lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1));
+
+    /* Level 2..N-1.  */
+    for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
+        void **p = *lp;
+        if (p == NULL) {
+            if (!alloc) {
+                return NULL;
+            }
+            *lp = p = qemu_mallocz(sizeof(void *) * L2_SIZE);
+        }
+        lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
+    }
+
+    pd = *lp;
+    if (pd == NULL) {
+        int i;
+
+        if (!alloc) {
             return NULL;
-        p = qemu_vmalloc(sizeof(void *) * L1_SIZE);
-        memset(p, 0, sizeof(void *) * L1_SIZE);
-        *lp = p;
-    }
-#endif
-#else  /* VBOX */
-    /* level 0 lookup and lazy allocation of level 1 map. */
-    if (RT_UNLIKELY(index >= (target_phys_addr_t)L2_SIZE * L1_SIZE * L0_SIZE))
-        return NULL;
-    p = l0_phys_map[index >> (L1_BITS + L2_BITS)];
-    if (RT_UNLIKELY(!p)) {
-        if (!alloc)
-            return NULL;
-        p = qemu_vmalloc(sizeof(void **) * L1_SIZE);
-        memset(p, 0, sizeof(void **) * L1_SIZE);
-        l0_phys_map[index >> (L1_BITS + L2_BITS)] = p;
-    }
-
-    /* level 1 lookup and lazy allocation of level 2 map. */
-#endif /* VBOX */
-    lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
-    pd = *lp;
-    if (!pd) {
-        int i;
-        /* allocate if not found */
-        if (!alloc)
-            return NULL;
-        pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
-        *lp = pd;
+        }
+
+        *lp = pd = qemu_malloc(sizeof(PhysPageDesc) * L2_SIZE);
+
         for (i = 0; i < L2_SIZE; i++) {
-          pd[i].phys_offset = IO_MEM_UNASSIGNED;
-          pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
-        }
-    }
-    return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
+            pd[i].phys_offset = IO_MEM_UNASSIGNED;
+            pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
+        }
+    }
+
+    return pd + (index & (L2_SIZE - 1));
 }
 
@@ -504 +498 @@
 }
 
-#if !defined(CONFIG_USER_ONLY)
 static void tlb_protect_code(ram_addr_t ram_addr);
 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
@@ -530 +523 @@
 
 #ifdef USE_STATIC_CODE_GEN_BUFFER
-static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
+static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
+               __attribute__((aligned (CODE_GEN_ALIGN)));
 #endif
 
@@ -594 +588 @@
         if (code_gen_buffer_size > 16 * 1024 * 1024)
             code_gen_buffer_size = 16 * 1024 * 1024;
+#elif defined(__s390x__)
+        /* Map the buffer so that we can use direct calls and branches.  */
+        /* We have a +- 4GB range on the branches; leave some slop.  */
+        if (code_gen_buffer_size > (3ul * 1024 * 1024 * 1024)) {
+            code_gen_buffer_size = 3ul * 1024 * 1024 * 1024;
+        }
+        start = (void *)0x90000000UL;
 #endif
         code_gen_buffer = mmap(start, code_gen_buffer_size,
@@ -637 +638 @@
 #endif
     code_gen_buffer_max_size = code_gen_buffer_size -
-        code_gen_max_block_size();
+        (TCG_MAX_OP_SIZE * OPC_MAX_SIZE);
     code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
     tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
@@ -654 +655 @@
     io_mem_init();
 #endif
+#if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
+    /* There's no guest base to take into account, so go ahead and
+       initialize the prologue now.  */
+    tcg_prologue_init(&tcg_ctx);
+#endif
 }
 
 #ifndef VBOX
 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
-
-static void cpu_common_pre_save(void *opaque)
-{
-    CPUState *env = opaque;
-
-    cpu_synchronize_state(env);
-}
-
-static int cpu_common_pre_load(void *opaque)
-{
-    CPUState *env = opaque;
-
-    cpu_synchronize_state(env);
-    return 0;
-}
 
 static int cpu_common_post_load(void *opaque, int version_id)
@@ -691 +682 @@
     .minimum_version_id = 1,
     .minimum_version_id_old = 1,
-    .pre_save = cpu_common_pre_save,
-    .pre_load = cpu_common_pre_load,
     .post_load = cpu_common_post_load,
     .fields      = (VMStateField []) {
@@ -742 +731 @@
 #endif
 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
-    vmstate_register(cpu_index, &vmstate_cpu_common, env);
-    register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
+    vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
+    register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
                     cpu_save, cpu_load, env);
 #endif
@@ -758 +747 @@
 }
 
-/* set to NULL all the 'first_tb' fields in all PageDescs */
+/* Set to NULL all the 'first_tb' fields in all PageDescs. */
+
+static void page_flush_tb_1 (int level, void **lp)
+{
+    int i;
+
+    if (*lp == NULL) {
+        return;
+    }
+    if (level == 0) {
+        PageDesc *pd = *lp;
+        for (i = 0; i < L2_SIZE; ++i) {
+            pd[i].first_tb = NULL;
+            invalidate_page_bitmap(pd + i);
+        }
+    } else {
+        void **pp = *lp;
+        for (i = 0; i < L2_SIZE; ++i) {
+            page_flush_tb_1 (level - 1, pp + i);
+        }
+    }
+}
+
 static void page_flush_tb(void)
 {
-    int i, j;
-    PageDesc *p;
-#ifdef VBOX
-    int k;
-#endif
-
-#ifdef VBOX
-    k = l0_map_max_used;
-    while (k-- > 0) {
-        PageDesc **l1_map = l0_map[k];
-        if (l1_map) {
-#endif
-            for(i = 0; i < L1_SIZE; i++) {
-                p = l1_map[i];
-                if (p) {
-                    for(j = 0; j < L2_SIZE; j++) {
-                        p->first_tb = NULL;
-                        invalidate_page_bitmap(p);
-                        p++;
-                    }
-                }
-            }
-#ifdef VBOX
-        }
-    }
-#endif
+    int i;
+    for (i = 0; i < V_L1_SIZE; i++) {
+        page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
+    }
 }
 
@@ -930 +919 @@
 }
 
-void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)
+void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
 {
     CPUState *env;
     PageDesc *p;
     unsigned int h, n1;
-    target_phys_addr_t phys_pc;
+    tb_page_addr_t phys_pc;
     TranslationBlock *tb1, *tb2;
 
@@ -1094 +1083 @@
     TranslationBlock *tb;
     uint8_t *tc_ptr;
-    target_ulong phys_pc, phys_page2, virt_page2;
+    tb_page_addr_t phys_pc, phys_page2;
+    target_ulong virt_page2;
     int code_gen_size;
 
-    phys_pc = get_phys_addr_code(env, pc);
+    phys_pc = get_page_addr_code(env, pc);
     tb = tb_alloc(pc);
     if (!tb) {
@@ -1119 +1109 @@
     phys_page2 = -1;
     if ((pc & TARGET_PAGE_MASK) != virt_page2) {
-        phys_page2 = get_phys_addr_code(env, virt_page2);
-    }
-    tb_link_phys(tb, phys_pc, phys_page2);
+        phys_page2 = get_page_addr_code(env, virt_page2);
+    }
+    tb_link_page(tb, phys_pc, phys_page2);
     return tb;
 }
@@ -1130 +1120 @@
    from a real cpu write access: the virtual CPU will exit the current
    TB if code is modified inside this TB. */
-void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
+void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
                                    int is_cpu_write_access)
 {
     TranslationBlock *tb, *tb_next, *saved_tb;
     CPUState *env = cpu_single_env;
-    target_ulong tb_start, tb_end;
+    tb_page_addr_t tb_start, tb_end;
     PageDesc *p;
     int n;
@@ -1237 +1227 @@
 
 /* len must be <= 8 and start must be a multiple of len */
-static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
+static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
 {
     PageDesc *p;
@@ -1264 +1254 @@
 
 #if !defined(CONFIG_SOFTMMU)
-static void tb_invalidate_phys_page(target_phys_addr_t addr,
+static void tb_invalidate_phys_page(tb_page_addr_t addr,
                                     unsigned long pc, void *puc)
 {
@@ -1326 +1316 @@
 /* add the tb in the target page and protect it if necessary */
 static inline void tb_alloc_page(TranslationBlock *tb,
-                                 unsigned int n, target_ulong page_addr)
+                                 unsigned int n, tb_page_addr_t page_addr)
 {
     PageDesc *p;
@@ -1332 +1322 @@
 
     tb->page_addr[n] = page_addr;
-    p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
+    p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
     tb->page_next[n] = p->first_tb;
     last_first_tb = p->first_tb;
@@ -1358 +1348 @@
             prot |= p2->flags;
             p2->flags &= ~PAGE_WRITE;
-            page_get_flags(addr);
           }
         mprotect(g2h(page_addr), qemu_host_page_size,
@@ -1386 +1375 @@
 
     if (nb_tbs >= code_gen_max_blocks ||
-#ifndef VBOX
-        (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
-#else
-        (code_gen_ptr - code_gen_buffer) >= (int)code_gen_buffer_max_size)
-#endif
+        (code_gen_ptr - code_gen_buffer) >= VBOX_ONLY((unsigned long))code_gen_buffer_max_size)
         return NULL;
     tb = &tbs[nb_tbs++];
@@ -1411 +1396 @@
 /* add a new TB and link it to the physical page tables. phys_page2 is
    (-1) to indicate that only one page contains the TB. */
-void tb_link_phys(TranslationBlock *tb,
-                  target_ulong phys_pc, target_ulong phys_page2)
+void tb_link_page(TranslationBlock *tb,
+                  tb_page_addr_t phys_pc, tb_page_addr_t phys_page2)
 {
     unsigned int h;
@@ -1528 +1513 @@
 
 #if defined(TARGET_HAS_ICE)
+#if defined(CONFIG_USER_ONLY)
+static void breakpoint_invalidate(CPUState *env, target_ulong pc)
+{
+    tb_invalidate_phys_page_range(pc, pc + 1, 0);
+}
+#else
 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
 {
@@ -1546 +1537 @@
 }
 #endif
-
+#endif /* TARGET_HAS_ICE */
+
+#if defined(CONFIG_USER_ONLY)
+void cpu_watchpoint_remove_all(CPUState *env, int mask)
+
+{
+}
+
+int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
+                          int flags, CPUWatchpoint **watchpoint)
+{
+    return -ENOSYS;
+}
+#else
 /* Add a watchpoint.  */
 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
@@ -1624 +1628 @@
     }
 }
+#endif
 
 /* Add a breakpoint.  */
@@ -1762 +1767 @@
 static void cpu_unlink_tb(CPUState *env)
 {
-#if defined(CONFIG_USE_NPTL)
     /* FIXME: TB unchaining isn't SMP safe.  For now just ignore the
        problem and hope the cpu will stop of its own accord.  For userspace
        emulation this often isn't actually as bad as it sounds.  Often
        signals are used primarily to interrupt blocking syscalls.  */
-#else
     TranslationBlock *tb;
     static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
 
+    spin_lock(&interrupt_lock);
     tb = env->current_tb;
     /* if the cpu is currently executing code, we must unlink it and
        all the potentially executing TB */
-    if (tb && !testandset(&interrupt_lock)) {
+    if (tb) {
         env->current_tb = NULL;
         tb_reset_jump_recursive(tb);
-        resetlock(&interrupt_lock);
-    }
-#endif
+    }
+    spin_unlock(&interrupt_lock);
 }
 
@@ -1872 +1875 @@
     { 0, NULL, NULL },
 };
+
+#ifndef CONFIG_USER_ONLY
+static QLIST_HEAD(memory_client_list, CPUPhysMemoryClient) memory_client_list
+    = QLIST_HEAD_INITIALIZER(memory_client_list);
+
+static void cpu_notify_set_memory(target_phys_addr_t start_addr,
+                                  ram_addr_t size,
+                                  ram_addr_t phys_offset)
+{
+    CPUPhysMemoryClient *client;
+    QLIST_FOREACH(client, &memory_client_list, list) {
+        client->set_memory(client, start_addr, size, phys_offset);
+    }
+}
+
+static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start,
+                                        target_phys_addr_t end)
+{
+    CPUPhysMemoryClient *client;
+    QLIST_FOREACH(client, &memory_client_list, list) {
+        int r = client->sync_dirty_bitmap(client, start, end);
+        if (r < 0)
+            return r;
+    }
+    return 0;
+}
+
+static int cpu_notify_migration_log(int enable)
+{
+    CPUPhysMemoryClient *client;
+    QLIST_FOREACH(client, &memory_client_list, list) {
+        int r = client->migration_log(client, enable);
+        if (r < 0)
+            return r;
+    }
+    return 0;
+}
+
+static void phys_page_for_each_1(CPUPhysMemoryClient *client,
+                                 int level, void **lp)
+{
+    int i;
+
+    if (*lp == NULL) {
+        return;
+    }
+    if (level == 0) {
+        PhysPageDesc *pd = *lp;
+        for (i = 0; i < L2_SIZE; ++i) {
+            if (pd[i].phys_offset != IO_MEM_UNASSIGNED) {
+                client->set_memory(client, pd[i].region_offset,
+                                   TARGET_PAGE_SIZE, pd[i].phys_offset);
+            }
+        }
+    } else {
+        void **pp = *lp;
+        for (i = 0; i < L2_SIZE; ++i) {
+            phys_page_for_each_1(client, level - 1, pp + i);
+        }
+    }
+}
+
+static void phys_page_for_each(CPUPhysMemoryClient *client)
+{
+    int i;
+    for (i = 0; i < P_L1_SIZE; ++i) {
+        phys_page_for_each_1(client, P_L1_SHIFT / L2_BITS - 1,
+                             l1_phys_map + 1);
+    }
+}
+
+void cpu_register_phys_memory_client(CPUPhysMemoryClient *client)
+{
+    QLIST_INSERT_HEAD(&memory_client_list, client, list);
+    phys_page_for_each(client);
+}
+
+void cpu_unregister_phys_memory_client(CPUPhysMemoryClient *client)
+{
+    QLIST_REMOVE(client, list);
+}
+#endif
 
 static int cmp1(const char *s1, int n, const char *s2)
@@ -1912 +1997 @@
     return mask;
 }
-#endif /* !VBOX */
-
-#ifndef VBOX /* VBOX: we have our own routine. */
+
 void cpu_abort(CPUState *env, const char *fmt, ...)
 {
@@ -1944 +2027 @@
     va_end(ap2);
     va_end(ap);
+#if defined(CONFIG_USER_ONLY)
+    {
+        struct sigaction act;
+        sigfillset(&act.sa_mask);
+        act.sa_handler = SIG_DFL;
+        sigaction(SIGABRT, &act, NULL);
+    }
+#endif
     abort();
 }
-#endif /* !VBOX */
-
-#ifndef VBOX /* not needed */
+
 CPUState *cpu_copy(CPUState *env)
 {
@@ -1982 +2071 @@
     return new_env;
 }
+
 #endif /* !VBOX */
-
 #if !defined(CONFIG_USER_ONLY)
 
@@ -1999 +2088 @@
     memset (&env->tb_jmp_cache[i], 0,
             TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
-
 #ifdef VBOX
+
     /* inform raw mode about TLB page flush */
     remR3FlushPage(env, addr);
@@ -2035 +2124 @@
     memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
 
+    env->tlb_flush_addr = -1;
+    env->tlb_flush_mask = 0;
+    tlb_flush_count++;
 #ifdef VBOX
+
     /* inform raw mode about TLB flush */
     remR3FlushTLB(env, flush_global);
-#endif
-    tlb_flush_count++;
+#endif /* VBOX */
 }
 
@@ -2062 +2154 @@
     printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
 #endif
+    /* Check if we need to flush due to large pages.  */
+    if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
+#if defined(DEBUG_TLB)
+        printf("tlb_flush_page: forced full flush ("
+               TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
+               env->tlb_flush_addr, env->tlb_flush_mask);
+#endif
+        tlb_flush(env, 1);
+        return;
+    }
     /* must reset current TB so that interrupts cannot modify the
        links while we are modifying them */
@@ -2084 +2186 @@
     /** @todo Retest this? This function has changed... */
     remR3ProtectCode(cpu_single_env, ram_addr);
-#endif
+#endif /* VBOX */
 }
 
@@ -2092 +2194 @@
                                     target_ulong vaddr)
 {
-#ifdef VBOX
-    if (RT_LIKELY((ram_addr >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
-#endif
-    phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] |= CODE_DIRTY_FLAG;
+    cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG);
 }
 
@@ -2102 +2201 @@
 {
     unsigned long addr;
-
 #ifdef VBOX
+
     if (start & 3)
         return;
-#endif
+#endif /* VBOX */
     if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
         addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
@@ -2121 +2220 @@
     CPUState *env;
     unsigned long length, start1;
-    int i, mask, len;
-    uint8_t *p;
+    int i;
 
     start &= TARGET_PAGE_MASK;
@@ -2130 +2228 @@
     if (length == 0)
         return;
-    len = length >> TARGET_PAGE_BITS;
-    mask = ~dirty_flags;
-    p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
-#ifdef VBOX
-    if (RT_LIKELY((start >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
-#endif
-    for(i = 0; i < len; i++)
-        p[i] &= mask;
+    cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
 
     /* we modify the TLB cache so that the dirty bit will be set again
@@ -2166 +2257 @@
 
 #ifndef VBOX
+
 int cpu_physical_memory_set_dirty_tracking(int enable)
 {
+    int ret = 0;
     in_migration = enable;
-    if (kvm_enabled()) {
-        return kvm_set_migration_log(enable);
-    }
-    return 0;
+    ret = cpu_notify_migration_log(!!enable);
+    return ret;
 }
 
@@ -2179 +2270 @@
     return in_migration;
 }
+
 #endif /* !VBOX */
 
@@ -2185 +2277 @@
 {
 #ifndef VBOX
-    int ret = 0;
-
-    if (kvm_enabled())
-        ret = kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
+    int ret;
+
+    ret = cpu_notify_sync_dirty_bitmap(start_addr, end_addr);
     return ret;
-#else
+#else  /* VBOX */
     return 0;
-#endif
+#endif /* VBOX */
 }
 
@@ -2255 +2346 @@
 }
 
-/* add a new TLB entry. At most one entry for a given virtual address
-   is permitted. Return 0 if OK or 2 if the page could not be mapped
-   (can only happen in non SOFTMMU mode for I/O pages or pages
-   conflicting with the host address space). */
-int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
-                      target_phys_addr_t paddr, int prot,
-                      int mmu_idx, int is_softmmu)
+/* Our TLB does not support large pages, so remember the area covered by
+   large pages and trigger a full TLB flush if these are invalidated.  */
+static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
+                               target_ulong size)
+{
+    target_ulong mask = ~(size - 1);
+
+    if (env->tlb_flush_addr == (target_ulong)-1) {
+        env->tlb_flush_addr = vaddr & mask;
+        env->tlb_flush_mask = mask;
+        return;
+    }
+    /* Extend the existing region to include the new page.
+       This is a compromise between unnecessary flushes and the cost
+       of maintaining a full variable size TLB.  */
+    mask &= env->tlb_flush_mask;
+    while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
+        mask <<= 1;
+    }
+    env->tlb_flush_addr &= mask;
+    env->tlb_flush_mask = mask;
+}
+
+/* Add a new TLB entry. At most one entry for a given virtual address
+   is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
+   supplied size is only used by tlb_flush_page.  */
+void tlb_set_page(CPUState *env, target_ulong vaddr,
+                  target_phys_addr_t paddr, int prot,
+                  int mmu_idx, target_ulong size)
 {
     PhysPageDesc *p;
@@ -2268 +2381 @@
     target_ulong address;
     target_ulong code_address;
-    target_phys_addr_t addend;
-    int ret;
+    unsigned long addend;
     CPUTLBEntry *te;
     CPUWatchpoint *wp;
@@ -2277 +2389 @@
 #endif
 
+    assert(size >= TARGET_PAGE_SIZE);
+    if (size != TARGET_PAGE_SIZE) {
+        tlb_add_large_page(env, vaddr, size);
+    }
     p = phys_page_find(paddr >> TARGET_PAGE_BITS);
     if (!p) {
@@ -2288 +2404 @@
 #endif
 
-    ret = 0;
     address = vaddr;
     if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
@@ -2357 +2472 @@
     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
         if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
-            iotlb = io_mem_watch + paddr;
-            /* TODO: The memory case can be optimized by not trapping
-               reads of pages with a write breakpoint.  */
-            address |= TLB_MMIO;
+            /* Avoid trapping reads of pages with a write breakpoint. */
+            if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
+                iotlb = io_mem_watch + paddr;
+                address |= TLB_MMIO;
+                break;
+            }
         }
     }
@@ -2409 +2526 @@
     remR3FlushPage(env, vaddr);
 #endif
-    return ret;
 }
 
@@ -2421 +2537 @@
 {
 }
-
-int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
-                      target_phys_addr_t paddr, int prot,
-                      int mmu_idx, int is_softmmu)
-{
-    return 0;
-}
-
-#ifndef VBOX
 
 /*
@@ -2435 +2542 @@
  * and calls callback function 'fn' for each region.
  */
-int walk_memory_regions(void *priv,
-    int (*fn)(void *, unsigned long, unsigned long, unsigned long))
-{
-    unsigned long start, end;
-    PageDesc *p = NULL;
-    int i, j, prot, prot1;
-    int rc = 0;
-
-    start = end = -1;
-    prot = 0;
-
-    for (i = 0; i <= L1_SIZE; i++) {
-        p = (i < L1_SIZE) ? l1_map[i] : NULL;
-        for (j = 0; j < L2_SIZE; j++) {
-            prot1 = (p == NULL) ? 0 : p[j].flags;
-            /*
-             * "region" is one continuous chunk of memory
-             * that has same protection flags set.
-             */
-            if (prot1 != prot) {
-                end = (i << (32 - L1_BITS)) | (j << TARGET_PAGE_BITS);
-                if (start != -1) {
-                    rc = (*fn)(priv, start, end, prot);
-                    /* callback can stop iteration by returning != 0 */
-                    if (rc != 0)
-                        return (rc);
+
+struct walk_memory_regions_data
+{
+    walk_memory_regions_fn fn;
+    void *priv;
+    unsigned long start;
+    int prot;
+};
+
+static int walk_memory_regions_end(struct walk_memory_regions_data *data,
+                                   abi_ulong end, int new_prot)
+{
+    if (data->start != -1ul) {
+        int rc = data->fn(data->priv, data->start, end, data->prot);
+        if (rc != 0) {
+            return rc;
+        }
+    }
+
+    data->start = (new_prot ? end : -1ul);
+    data->prot = new_prot;
+
+    return 0;
+}
+
+static int walk_memory_regions_1(struct walk_memory_regions_data *data,
+                                 abi_ulong base, int level, void **lp)
+{
+    abi_ulong pa;
+    int i, rc;
+
+    if (*lp == NULL) {
+        return walk_memory_regions_end(data, base, 0);
+    }
+
+    if (level == 0) {
+        PageDesc *pd = *lp;
+        for (i = 0; i < L2_SIZE; ++i) {
+            int prot = pd[i].flags;
+
+            pa = base | (i << TARGET_PAGE_BITS);
+            if (prot != data->prot) {
+                rc = walk_memory_regions_end(data, pa, prot);
+                if (rc != 0) {
+                    return rc;
                 }
-                if (prot1 != 0)
-                    start = end;
-                else
-                    start = -1;
-                prot = prot1;
             }
-            if (p == NULL)
-                break;
-        }
-    }
-    return (rc);
-}
-
-static int dump_region(void *priv, unsigned long start,
-    unsigned long end, unsigned long prot)
+        }
+    } else {
+        void **pp = *lp;
+        for (i = 0; i < L2_SIZE; ++i) {
+            pa = base | ((abi_ulong)i <<
+                (TARGET_PAGE_BITS + L2_BITS * level));
+            rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
+            if (rc != 0) {
+                return rc;
+            }
+        }
+    }
+
+    return 0;
+}
+
+int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
+{
+    struct walk_memory_regions_data data;
+    unsigned long i;
+
+    data.fn = fn;
+    data.priv = priv;
+    data.start = -1ul;
+    data.prot = 0;
+
+    for (i = 0; i < V_L1_SIZE; i++) {
+        int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
+                                       V_L1_SHIFT / L2_BITS - 1, l1_map + i);
+        if (rc != 0) {
+            return rc;
+        }
+    }
+
+    return walk_memory_regions_end(&data, 0, 0);
+}
+
+static int dump_region(void *priv, abi_ulong start,
+    abi_ulong end, unsigned long prot)
 {
     FILE *f = (FILE *)priv;
 
-    (void) fprintf(f, "%08lx-%08lx %08lx %c%c%c\n",
+    (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
+        " "TARGET_ABI_FMT_lx" %c%c%c\n",
         start, end, end - start,
         ((prot & PAGE_READ) ? 'r' : '-'),
@@ -2497 +2649 @@
 }
 
-#endif /* !VBOX */
-
 int page_get_flags(target_ulong address)
 {
@@ -2509 +2659 @@
 }
 
-/* modify the flags of a page and invalidate the code if
-   necessary. The flag PAGE_WRITE_ORG is positioned automatically
-   depending on PAGE_WRITE */
+/* Modify the flags of a page and invalidate the code if necessary.
+   The flag PAGE_WRITE_ORG is positioned automatically depending
+   on PAGE_WRITE.  The mmap_lock should already be held.  */
 void page_set_flags(target_ulong start, target_ulong end, int flags)
 {
-    PageDesc *p;
-    target_ulong addr;
-
-    /* mmap_lock should already be held.  */
+    target_ulong addr, len;
+
+    /* This function should never be called with addresses outside the
+       guest address space.  If this assert fires, it probably indicates
+       a missing call to h2g_valid.  */
+#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
+    assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
+#endif
+    assert(start < end);
+
     start = start & TARGET_PAGE_MASK;
     end = TARGET_PAGE_ALIGN(end);
-    if (flags & PAGE_WRITE)
+
+    if (flags & PAGE_WRITE) {
         flags |= PAGE_WRITE_ORG;
+    }
+
 #ifdef VBOX
     AssertMsgFailed(("We shouldn't be here, and if we should, we must have an env to do the proper locking!\n"));
 #endif
-    for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
-        p = page_find_alloc(addr >> TARGET_PAGE_BITS);
-        /* We may be called for host regions that are outside guest
-           address space.  */
-        if (!p)
-            return;
-        /* if the write protection is set, then we invalidate the code
-           inside */
+    for (addr = start, len = end - start;
+         len != 0;
+         len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
+        PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
+
+        /* If the write protection bit is set, then we invalidate
+           the code inside.  */
         if (!(p->flags & PAGE_WRITE) &&
             (flags & PAGE_WRITE) &&
@@ -2548 +2706 @@
     target_ulong addr;
 
-    if (start + len < start)
-        /* we've wrapped around */
+    /* This function should never be called with addresses outside the
+       guest address space.  If this assert fires, it probably indicates
+       a missing call to h2g_valid.  */
+#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
+    assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
+#endif
+
+    if (len == 0) {
+        return 0;
+    }
+    if (start + len - 1 < start) {
+        /* We've wrapped around.  */
         return -1;
+    }
 
     end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
     start = start & TARGET_PAGE_MASK;
 
-    for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
+    for (addr = start, len = end - start;
+         len != 0;
+         len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
         p = page_find(addr >> TARGET_PAGE_BITS);
         if( !p )
@@ -2583 +2754 @@
 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
 {
-    unsigned int page_index, prot, pindex;
-    PageDesc *p, *p1;
+    unsigned int prot;
+    PageDesc *p;
     target_ulong host_start, host_end, addr;
 
@@ -2592 +2763 @@
     mmap_lock();
 
-    host_start = address & qemu_host_page_mask;
-    page_index = host_start >> TARGET_PAGE_BITS;
-    p1 = page_find(page_index);
-    if (!p1) {
+    p = page_find(address >> TARGET_PAGE_BITS);
+    if (!p) {
         mmap_unlock();
         return 0;
     }
-    host_end = host_start + qemu_host_page_size;
-    p = p1;
-    prot = 0;
-    for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
-        prot |= p->flags;
-        p++;
-    }
+
     /* if the page was really writable, then we change its
        protection back to writable */
-    if (prot & PAGE_WRITE_ORG) {
-        pindex = (address - host_start) >> TARGET_PAGE_BITS;
-        if (!(p1[pindex].flags & PAGE_WRITE)) {
-            mprotect((void *)g2h(host_start), qemu_host_page_size,
-                     (prot & PAGE_BITS) | PAGE_WRITE);
-            p1[pindex].flags |= PAGE_WRITE;
+    if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
+        host_start = address & qemu_host_page_mask;
+        host_end = host_start + qemu_host_page_size;
+
+        prot = 0;
+        for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
+            p = page_find(addr >> TARGET_PAGE_BITS);
+            p->flags |= PAGE_WRITE;
+            prot |= p->flags;
+
             /* and since the content will be modified, we must invalidate
                the corresponding translated code. */
-            tb_invalidate_phys_page(address, pc, puc);
+            tb_invalidate_phys_page(addr, pc, puc);
 #ifdef DEBUG_TB_CHECK
-            tb_invalidate_check(address);
-#endif
-            mmap_unlock();
-            return 1;
-        }
+            tb_invalidate_check(addr);
+#endif
+        }
+        mprotect((void *)g2h(host_start), qemu_host_page_size,
+                 prot & PAGE_BITS);
+
+        mmap_unlock();
+        return 1;
     }
     mmap_unlock();
@@ -2636 +2806 @@
 #if !defined(CONFIG_USER_ONLY)
 
+#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
+typedef struct subpage_t {
+    target_phys_addr_t base;
+    ram_addr_t sub_io_index[TARGET_PAGE_SIZE];
+    ram_addr_t region_offset[TARGET_PAGE_SIZE];
+} subpage_t;
+
 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
                              ram_addr_t memory, ram_addr_t region_offset);
-static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
-                           ram_addr_t orig_memory, ram_addr_t region_offset);
+static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
+                                ram_addr_t orig_memory,
+                                ram_addr_t region_offset);
 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
                       need_subpage)                                     \
@@ -2677 +2855 @@
     CPUState *env;
     ram_addr_t orig_size = size;
-    void *subpage;
-
-    if (kvm_enabled())
-        kvm_set_phys_mem(start_addr, size, phys_offset);
+    subpage_t *subpage;
+
+#ifndef VBOX
+    cpu_notify_set_memory(start_addr, size, phys_offset);
+#endif /* !VBOX */
 
     if (phys_offset == IO_MEM_UNASSIGNED) {
@@ -2697 +2876 @@
             CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
                           need_subpage);
-            if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
+            if (need_subpage) {
                 if (!(orig_memory & IO_MEM_SUBPAGE)) {
                     subpage = subpage_init((addr & TARGET_PAGE_MASK),
@@ -2729 +2908 @@
                               end_addr2, need_subpage);
 
-                if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
+                if (need_subpage) {
                     subpage = subpage_init((addr & TARGET_PAGE_MASK),
                                            &p->phys_offset, IO_MEM_UNASSIGNED,
@@ -2762 +2941 @@
 
 #ifndef VBOX
+
 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
 {
@@ -2774 +2954 @@
 }
 
-ram_addr_t qemu_ram_alloc(ram_addr_t size)
-{
-    RAMBlock *new_block;
+void qemu_flush_coalesced_mmio_buffer(void)
+{
+    if (kvm_enabled())
+        kvm_flush_coalesced_mmio_buffer();
+}
+
+#if defined(__linux__) && !defined(TARGET_S390X)
+
+#include <sys/vfs.h>
+
+#define HUGETLBFS_MAGIC       0x958458f6
+
+static long gethugepagesize(const char *path)
+{
+    struct statfs fs;
+    int ret;
+
+    do {
+            ret = statfs(path, &fs);
+    } while (ret != 0 && errno == EINTR);
+
+    if (ret != 0) {
+            perror(path);
+            return 0;
+    }
+
+    if (fs.f_type != HUGETLBFS_MAGIC)
+            fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
+
+    return fs.f_bsize;
+}
+
+static void *file_ram_alloc(RAMBlock *block,
+                            ram_addr_t memory,
+                            const char *path)
+{
+    char *filename;
+    void *area;
+    int fd;
+#ifdef MAP_POPULATE
+    int flags;
+#endif
+    unsigned long hpagesize;
+
+    hpagesize = gethugepagesize(path);
+    if (!hpagesize) {
+        return NULL;
+    }
+
+    if (memory < hpagesize) {
+        return NULL;
+    }
+
+    if (kvm_enabled() && !kvm_has_sync_mmu()) {
+        fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
+        return NULL;
+    }
+
+    if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
+        return NULL;
+    }
+
+    fd = mkstemp(filename);
+    if (fd < 0) {
+        perror("unable to create backing store for hugepages");
+        free(filename);
+        return NULL;
+    }
+    unlink(filename);
+    free(filename);
+
+    memory = (memory+hpagesize-1) & ~(hpagesize-1);
+
+    /*
+     * ftruncate is not supported by hugetlbfs in older
+     * hosts, so don't bother bailing out on errors.
+     * If anything goes wrong with it under other filesystems,
+     * mmap will fail.
+     */
+    if (ftruncate(fd, memory))
+        perror("ftruncate");
+
+#ifdef MAP_POPULATE
+    /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
+     * MAP_PRIVATE is requested.  For mem_prealloc we mmap as MAP_SHARED
+     * to sidestep this quirk.
+     */
+    flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
+    area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
+#else
+    area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
+#endif
+    if (area == MAP_FAILED) {
+        perror("file_ram_alloc: can't mmap RAM pages");
+        close(fd);
+        return (NULL);
+    }
+    block->fd = fd;
+    return area;
+}
+#endif
+
+static ram_addr_t find_ram_offset(ram_addr_t size)
+{
+    RAMBlock *block, *next_block;
+    ram_addr_t offset = 0, mingap = ULONG_MAX;
+
+    if (QLIST_EMPTY(&ram_list.blocks))
+        return 0;
+
+    QLIST_FOREACH(block, &ram_list.blocks, next) {
+        ram_addr_t end, next = ULONG_MAX;
+
+        end = block->offset + block->length;
+
+        QLIST_FOREACH(next_block, &ram_list.blocks, next) {
+            if (next_block->offset >= end) {
+                next = MIN(next, next_block->offset);
+            }
+        }
+        if (next - end >= size && next - end < mingap) {
+            offset =  end;
+            mingap = next - end;
+        }
+    }
+    return offset;
+}
+
+static ram_addr_t last_ram_offset(void)
+{
+    RAMBlock *block;
+    ram_addr_t last = 0;
+
+    QLIST_FOREACH(block, &ram_list.blocks, next)
+        last = MAX(last, block->offset + block->length);
+
+    return last;
+}
+
+ram_addr_t qemu_ram_alloc_from_ptr(DeviceState *dev, const char *name,
+                        ram_addr_t size, void *host)
+{
+    RAMBlock *new_block, *block;
 
     size = TARGET_PAGE_ALIGN(size);
-    new_block = qemu_malloc(sizeof(*new_block));
-
-#if defined(TARGET_S390X) && defined(CONFIG_KVM)
-    /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
-    new_block->host = mmap((void*)0x1000000, size, PROT_EXEC|PROT_READ|PROT_WRITE,
-                           MAP_SHARED | MAP_ANONYMOUS, -1, 0);
-#else
-    new_block->host = qemu_vmalloc(size);
-#endif
-#ifdef MADV_MERGEABLE
-    madvise(new_block->host, size, MADV_MERGEABLE);
-#endif
-    new_block->offset = last_ram_offset;
+    new_block = qemu_mallocz(sizeof(*new_block));
+
+    if (dev && dev->parent_bus && dev->parent_bus->info->get_dev_path) {
+        char *id = dev->parent_bus->info->get_dev_path(dev);
+        if (id) {
+            snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
+            qemu_free(id);
+        }
+    }
+    pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
+
+    QLIST_FOREACH(block, &ram_list.blocks, next) {
+        if (!strcmp(block->idstr, new_block->idstr)) {
+            fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
+                    new_block->idstr);
+            abort();
+        }
+    }
+
+    new_block->host = host;
+
+    new_block->offset = find_ram_offset(size);
     new_block->length = size;
 
-    new_block->next = ram_blocks;
-    ram_blocks = new_block;
-
-    phys_ram_dirty = qemu_realloc(phys_ram_dirty,
-        (last_ram_offset + size) >> TARGET_PAGE_BITS);
-    memset(phys_ram_dirty + (last_ram_offset >> TARGET_PAGE_BITS),
+    QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
+
+    ram_list.phys_dirty = qemu_realloc(ram_list.phys_dirty,
+                                       last_ram_offset() >> TARGET_PAGE_BITS);
+    memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
            0xff, size >> TARGET_PAGE_BITS);
-
-    last_ram_offset += size;
 
     if (kvm_enabled())
@@ -2810 +3136 @@
 }
 
+ram_addr_t qemu_ram_alloc(DeviceState *dev, const char *name, ram_addr_t size)
+{
+    RAMBlock *new_block, *block;
+
+    size = TARGET_PAGE_ALIGN(size);
+    new_block = qemu_mallocz(sizeof(*new_block));
+
+    if (dev && dev->parent_bus && dev->parent_bus->info->get_dev_path) {
+        char *id = dev->parent_bus->info->get_dev_path(dev);
+        if (id) {
+            snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
+            qemu_free(id);
+        }
+    }
+    pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
+
+    QLIST_FOREACH(block, &ram_list.blocks, next) {
+        if (!strcmp(block->idstr, new_block->idstr)) {
+            fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
+                    new_block->idstr);
+            abort();
+        }
+    }
+
+    if (mem_path) {
+#if defined (__linux__) && !defined(TARGET_S390X)
+        new_block->host = file_ram_alloc(new_block, size, mem_path);
+        if (!new_block->host) {
+            new_block->host = qemu_vmalloc(size);
+#ifdef MADV_MERGEABLE
+            madvise(new_block->host, size, MADV_MERGEABLE);
+#endif
+        }
+#else
+        fprintf(stderr, "-mem-path option unsupported\n");
+        exit(1);
+#endif
+    } else {
+#if defined(TARGET_S390X) && defined(CONFIG_KVM)
+        /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
+        new_block->host = mmap((void*)0x1000000, size,
+                                PROT_EXEC|PROT_READ|PROT_WRITE,
+                                MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+#else
+        new_block->host = qemu_vmalloc(size);
+#endif
+#ifdef MADV_MERGEABLE
+        madvise(new_block->host, size, MADV_MERGEABLE);
+#endif
+    }
+    new_block->offset = find_ram_offset(size);
+    new_block->length = size;
+
+    QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
+
+    ram_list.phys_dirty = qemu_realloc(ram_list.phys_dirty,
+                                       last_ram_offset() >> TARGET_PAGE_BITS);
+    memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
+           0xff, size >> TARGET_PAGE_BITS);
+
+    if (kvm_enabled())
+        kvm_setup_guest_memory(new_block->host, size);
+
+    return new_block->offset;
+}
+
 void qemu_ram_free(ram_addr_t addr)
 {
-    /* TODO: implement this.  */
+    RAMBlock *block;
+
+    QLIST_FOREACH(block, &ram_list.blocks, next) {
+        if (addr == block->offset) {
+            QLIST_REMOVE(block, next);
+            if (mem_path) {
+#if defined (__linux__) && !defined(TARGET_S390X)
+                if (block->fd) {
+                    munmap(block->host, block->length);
+                    close(block->fd);
+                } else {
+                    qemu_vfree(block->host);
+                }
+#endif
+            } else {
+#if defined(TARGET_S390X) && defined(CONFIG_KVM)
+                munmap(block->host, block->length);
+#else
+                qemu_vfree(block->host);
+#endif
+            }
+            qemu_free(block);
+            return;
+        }
+    }
+
 }
 
@@ -2825 +3242 @@
 void *qemu_get_ram_ptr(ram_addr_t addr)
 {
-    RAMBlock *prev;
-    RAMBlock **prevp;
     RAMBlock *block;
 
-    prev = NULL;
-    prevp = &ram_blocks;
-    block = ram_blocks;
-    while (block && (block->offset > addr
-                     || block->offset + block->length <= addr)) {
-        if (prev)
-          prevp = &prev->next;
-        prev = block;
-        block = block->next;
-    }
-    if (!block) {
-        fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
-        abort();
-    }
-    /* Move this entry to to start of the list.  */
-    if (prev) {
-        prev->next = block->next;
-        block->next = *prevp;
-        *prevp = block;
-    }
-    return block->host + (addr - block->offset);
+    QLIST_FOREACH(block, &ram_list.blocks, next) {
+        if (addr - block->offset < block->length) {
+            QLIST_REMOVE(block, next);
+            QLIST_INSERT_HEAD(&ram_list.blocks, block, next);
+            return block->host + (addr - block->offset);
+        }
+    }
+
+    fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
+    abort();
+
+    return NULL;
 }
 
@@ -2856 +3262 @@
 ram_addr_t qemu_ram_addr_from_host(void *ptr)
 {
-    RAMBlock *prev;
-    RAMBlock **prevp;
     RAMBlock *block;
     uint8_t *host = ptr;
 
-    prev = NULL;
-    prevp = &ram_blocks;
-    block = ram_blocks;
-    while (block && (block->host > host
-                     || block->host + block->length <= host)) {
-        if (prev)
-          prevp = &prev->next;
-        prev = block;
-        block = block->next;
-    }
-    if (!block) {
-        fprintf(stderr, "Bad ram pointer %p\n", ptr);
-        abort();
-    }
-    return block->offset + (host - block->host);
+    QLIST_FOREACH(block, &ram_list.blocks, next) {
+        if (host - block->host < block->length) {
+            return block->offset + (host - block->host);
+        }
+    }
+
+    fprintf(stderr, "Bad ram pointer %p\n", ptr);
+    abort();
+
+    return 0;
 }
 
@@ -2959 +3358 @@
 {
     int dirty_flags;
-#ifdef VBOX
-    if (RT_UNLIKELY((ram_addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
-        dirty_flags = 0xff;
-    else
-#endif /* VBOX */
-    dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
+    dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
 #if !defined(CONFIG_USER_ONLY)
         tb_invalidate_phys_page_fast(ram_addr, 1);
-# ifdef VBOX
-        if (RT_UNLIKELY((ram_addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
-            dirty_flags = 0xff;
-        else
-# endif /* VBOX */
-        dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
+        dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
 #endif
     }
@@ -2981 +3370 @@
     stb_p(qemu_get_ram_ptr(ram_addr), val);
 #endif
-#ifdef CONFIG_KQEMU
-    if (cpu_single_env->kqemu_enabled &&
-        (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
-        kqemu_modify_page(cpu_single_env, ram_addr);
-#endif
     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
-#ifdef VBOX
-    if (RT_LIKELY((ram_addr >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
-#endif /* !VBOX */
-    phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
+    cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
     /* we remove the notdirty callback only if the code has been
        flushed */
@@ -3001 +3382 @@
 {
     int dirty_flags;
-#ifdef VBOX
-    if (RT_UNLIKELY((ram_addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
-        dirty_flags = 0xff;
-    else
-#endif /* VBOX */
-    dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
+    dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
 #if !defined(CONFIG_USER_ONLY)
         tb_invalidate_phys_page_fast(ram_addr, 2);
-# ifdef VBOX
-        if (RT_UNLIKELY((ram_addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
-            dirty_flags = 0xff;
-        else
-# endif /* VBOX */
-        dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
+        dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
 #endif
     }
@@ -3024 +3395 @@
 #endif
     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
-#ifdef VBOX
-    if (RT_LIKELY((ram_addr >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
-#endif
-    phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
+    cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
     /* we remove the notdirty callback only if the code has been
        flushed */
@@ -3038 +3406 @@
 {
     int dirty_flags;
-#ifdef VBOX
-    if (RT_UNLIKELY((ram_addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
-        dirty_flags = 0xff;
-    else
-#endif /* VBOX */
-    dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
+    dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
 #if !defined(CONFIG_USER_ONLY)
         tb_invalidate_phys_page_fast(ram_addr, 4);
-# ifdef VBOX
-        if (RT_UNLIKELY((ram_addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
-            dirty_flags = 0xff;
-        else
-# endif /* VBOX */
-        dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
+        dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
 #endif
     }
@@ -3061 +3419 @@
 #endif
     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
-#ifdef VBOX
-    if (RT_LIKELY((ram_addr >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
-#endif
-    phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
+    cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
     /* we remove the notdirty callback only if the code has been
        flushed */
@@ -3182 +3537 @@
 };
 
-static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
-                                 unsigned int len)
-{
-    uint32_t ret;
-    unsigned int idx;
-
-    idx = SUBPAGE_IDX(addr);
+static inline uint32_t subpage_readlen (subpage_t *mmio,
+                                        target_phys_addr_t addr,
+                                        unsigned int len)
+{
+    unsigned int idx = SUBPAGE_IDX(addr);
 #if defined(DEBUG_SUBPAGE)
     printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
            mmio, len, addr, idx);
 #endif
-    ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
-                                       addr + mmio->region_offset[idx][0][len]);
-
-    return ret;
+
+    addr += mmio->region_offset[idx];
+    idx = mmio->sub_io_index[idx];
+    return io_mem_read[idx][len](io_mem_opaque[idx], addr);
 }
 
 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
-                              uint32_t value, unsigned int len)
-{
-    unsigned int idx;
-
-    idx = SUBPAGE_IDX(addr);
+                                     uint32_t value, unsigned int len)
+{
+    unsigned int idx = SUBPAGE_IDX(addr);
 #if defined(DEBUG_SUBPAGE)
-    printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
-           mmio, len, addr, idx, value);
-#endif
-    (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
-                                  addr + mmio->region_offset[idx][1][len],
-                                  value);
+    printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n",
+           __func__, mmio, len, addr, idx, value);
+#endif
+
+    addr += mmio->region_offset[idx];
+    idx = mmio->sub_io_index[idx];
+    io_mem_write[idx][len](io_mem_opaque[idx], addr, value);
 }
 
 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
 {
-#if defined(DEBUG_SUBPAGE)
-    printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
-#endif
-
     return subpage_readlen(opaque, addr, 0);
 }
@@ -3226 +3574 @@
                             uint32_t value)
 {
-#if defined(DEBUG_SUBPAGE)
-    printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
-#endif
     subpage_writelen(opaque, addr, value, 0);
 }
@@ -3234 +3579 @@
 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
 {
-#if defined(DEBUG_SUBPAGE)
-    printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
-#endif
-
     return subpage_readlen(opaque, addr, 1);
 }
@@ -3244 +3585 @@
                             uint32_t value)
 {
-#if defined(DEBUG_SUBPAGE)
-    printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
-#endif
     subpage_writelen(opaque, addr, value, 1);
 }
@@ -3252 +3590 @@
 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
 {
-#if defined(DEBUG_SUBPAGE)
-    printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
-#endif
-
     return subpage_readlen(opaque, addr, 2);
 }
 
-static void subpage_writel (void *opaque,
-                         target_phys_addr_t addr, uint32_t value)
-{
-#if defined(DEBUG_SUBPAGE)
-    printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
-#endif
+static void subpage_writel (void *opaque, target_phys_addr_t addr,
+                            uint32_t value)
+{
     subpage_writelen(opaque, addr, value, 2);
 }
@@ -3284 +3615 @@
 {
     int idx, eidx;
-    unsigned int i;
 
     if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
@@ -3294 +3624 @@
            mmio, start, end, idx, eidx, memory);
 #endif
-    memory >>= IO_MEM_SHIFT;
+    memory = (memory >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
     for (; idx <= eidx; idx++) {
-        for (i = 0; i < 4; i++) {
-            if (io_mem_read[memory][i]) {
-                mmio->mem_read[idx][i] = &io_mem_read[memory][i];
-                mmio->opaque[idx][0][i] = io_mem_opaque[memory];
-                mmio->region_offset[idx][0][i] = region_offset;
-            }
-            if (io_mem_write[memory][i]) {
-                mmio->mem_write[idx][i] = &io_mem_write[memory][i];
-                mmio->opaque[idx][1][i] = io_mem_opaque[memory];
-                mmio->region_offset[idx][1][i] = region_offset;
-            }
-        }
+        mmio->sub_io_index[idx] = memory;
+        mmio->region_offset[idx] = region_offset;
     }
 
@@ -3313 +3633 @@
 }
 
-static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
-                           ram_addr_t orig_memory, ram_addr_t region_offset)
+static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
+                                ram_addr_t orig_memory,
+                                ram_addr_t region_offset)
 {
     subpage_t *mmio;
@@ -3328 +3649 @@
 #endif
     *phys = subpage_memory | IO_MEM_SUBPAGE;
-    subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
-                         region_offset);
+    subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_memory, region_offset);
 
     return mmio;
@@ -3343 +3663 @@
             return i;
         }
-
+    fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
     return -1;
 }
@@ -3359 +3679 @@
                                         void *opaque)
 {
-    int i, subwidth = 0;
+    int i;
 
     if (io_index <= 0) {
@@ -3371 +3691 @@
     }
 
-    for(i = 0;i < 3; i++) {
-        if (!mem_read[i] || !mem_write[i])
-            subwidth = IO_MEM_SUBWIDTH;
-        io_mem_read[io_index][i] = mem_read[i];
-        io_mem_write[io_index][i] = mem_write[i];
+    for (i = 0; i < 3; ++i) {
+        io_mem_read[io_index][i]
+            = (mem_read[i] ? mem_read[i] : unassigned_mem_read[i]);
+    }
+    for (i = 0; i < 3; ++i) {
+        io_mem_write[io_index][i]
+            = (mem_write[i] ? mem_write[i] : unassigned_mem_write[i]);
     }
     io_mem_opaque[io_index] = opaque;
-    return (io_index << IO_MEM_SHIFT) | subwidth;
+
+    return (io_index << IO_MEM_SHIFT);
 }
 
@@ -3419 +3742 @@
 /* physical memory access (slow version, mainly for debug) */
 #if defined(CONFIG_USER_ONLY)
-void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
-                            int len, int is_write)
+int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
+                        uint8_t *buf, int len, int is_write)
 {
     int l, flags;
@@ -3433 +3756 @@
         flags = page_get_flags(page);
         if (!(flags & PAGE_VALID))
-            return;
+            return -1;
         if (is_write) {
             if (!(flags & PAGE_WRITE))
-                return;
+                return -1;
             /* XXX: this code should not depend on lock_user */
             if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
-                /* FIXME - should this return an error rather than just fail? */
-                return;
+                return -1;
             memcpy(p, buf, l);
             unlock_user(p, addr, l);
         } else {
             if (!(flags & PAGE_READ))
-                return;
+                return -1;
             /* XXX: this code should not depend on lock_user */
             if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
-                /* FIXME - should this return an error rather than just fail? */
-                return;
+                return -1;
             memcpy(buf, p, l);
             unlock_user(p, addr, 0);
@@ -3457 +3778 @@
         addr += l;
     }
+    return 0;
 }
 
@@ -3532 +3854 @@
                     tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
                     /* set dirty bit */
-#ifdef VBOX
-                    if (RT_LIKELY((addr1 >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
-#endif
-                    phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
-                        (0xff & ~CODE_DIRTY_FLAG);
+                    cpu_physical_memory_set_dirty_flags(
+                        addr1, (0xff & ~CODE_DIRTY_FLAG));
                 }
             }
@@ -3760 +4079 @@
                     tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
                     /* set dirty bit */
-                    phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
-                        (0xff & ~CODE_DIRTY_FLAG);
+                    cpu_physical_memory_set_dirty_flags(
+                        addr1, (0xff & ~CODE_DIRTY_FLAG));
                 }
                 addr1 += l;
@@ -3865 +4184 @@
 }
 
-/* XXX: optimize */
+/* warning: addr must be aligned */
 uint32_t lduw_phys(target_phys_addr_t addr)
 {
-    uint16_t val;
-    cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
-    return tswap16(val);
+    int io_index;
+    uint8_t *ptr;
+    uint64_t val;
+    unsigned long pd;
+    PhysPageDesc *p;
+
+    p = phys_page_find(addr >> TARGET_PAGE_BITS);
+    if (!p) {
+        pd = IO_MEM_UNASSIGNED;
+    } else {
+        pd = p->phys_offset;
+    }
+
+    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
+        !(pd & IO_MEM_ROMD)) {
+        /* I/O case */
+        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
+        if (p)
+            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
+        val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
+    } else {
+        /* RAM case */
+#ifndef VBOX
+        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
+            (addr & ~TARGET_PAGE_MASK);
+        val = lduw_p(ptr);
+#else
+        val = remR3PhysReadU16((pd & TARGET_PAGE_MASK) | (addr & ~TARGET_PAGE_MASK));
+#endif
+    }
+    return val;
 }
 
@@ -3910 +4257 @@
                 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
                 /* set dirty bit */
-                phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
-                    (0xff & ~CODE_DIRTY_FLAG);
+                cpu_physical_memory_set_dirty_flags(
+                    addr1, (0xff & ~CODE_DIRTY_FLAG));
             }
         }
@@ -3988 +4335 @@
             tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
             /* set dirty bit */
-#ifdef VBOX
-            if (RT_LIKELY((addr1 >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
-#endif
-            phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
-                (0xff & ~CODE_DIRTY_FLAG);
+            cpu_physical_memory_set_dirty_flags(addr1,
+                (0xff & ~CODE_DIRTY_FLAG));
         }
     }
@@ -4004 +4348 @@
 }
 
-/* XXX: optimize */
+/* warning: addr must be aligned */
 void stw_phys(target_phys_addr_t addr, uint32_t val)
 {
-    uint16_t v = tswap16(val);
-    cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
+    int io_index;
+    uint8_t *ptr;
+    unsigned long pd;
+    PhysPageDesc *p;
+
+    p = phys_page_find(addr >> TARGET_PAGE_BITS);
+    if (!p) {
+        pd = IO_MEM_UNASSIGNED;
+    } else {
+        pd = p->phys_offset;
+    }
+
+    if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
+        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
+        if (p)
+            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
+        io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
+    } else {
+        unsigned long addr1;
+        addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
+        /* RAM case */
+#ifndef VBOX
+        ptr = qemu_get_ram_ptr(addr1);
+        stw_p(ptr, val);
+#else
+        remR3PhysWriteU16(addr1, val); NOREF(ptr);
+#endif
+        if (!cpu_physical_memory_is_dirty(addr1)) {
+            /* invalidate code */
+            tb_invalidate_phys_page_range(addr1, addr1 + 2, 0);
+            /* set dirty bit */
+            cpu_physical_memory_set_dirty_flags(addr1,
+                (0xff & ~CODE_DIRTY_FLAG));
+        }
+    }
 }
 
@@ -4017 +4394 @@
     cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
 }
-
-#endif
 
 #ifndef VBOX
@@ -4039 +4414 @@
             l = len;
         phys_addr += (addr & ~TARGET_PAGE_MASK);
-#if !defined(CONFIG_USER_ONLY)
         if (is_write)
             cpu_physical_memory_write_rom(phys_addr, buf, l);
         else
-#endif
             cpu_physical_memory_rw(phys_addr, buf, l, is_write);
         len -= l;
@@ -4052 +4425 @@
 }
 #endif /* !VBOX */
+#endif
 
 /* in deterministic execution mode, instructions doing device I/Os
@@ -4111 +4485 @@
     cpu_resume_from_signal(env, NULL);
 }
+
+#if !defined(CONFIG_USER_ONLY)
 
 #ifndef VBOX
@@ -4167 +4543 @@
 #endif /* !VBOX */
 
-#if !defined(CONFIG_USER_ONLY)
-
 #define MMUSUFFIX _cmmu
 #define GETPC() NULL

trunk/src/recompiler/fpu/softfloat-native.c

@@ -13 +13 @@
     (defined(CONFIG_SOLARIS) && (CONFIG_SOLARIS_VERSION < 10 || CONFIG_SOLARIS_VERSION == 11)) /* VBOX adds sol 11 */
     fpsetround(val);
-#elif defined(__arm__)
-    /* nothing to do */
 #else
     fesetround(val);
@@ -36 +34 @@
 #define remainderf(fa, fb)      ((float)remainder(fa, fb))
 #define rintf(f)                ((float)rint(f))
-/* Some defines which only apply to *BSD */
-# if defined(VBOX) && defined(HOST_BSD)
+# if defined(VBOX) && defined(HOST_BSD) /* Some defines which only apply to *BSD */
 #  define lrintl(f)            ((int32_t)rint(f))
 #  define llrintl(f)           ((int64_t)rint(f))
@@ -374 +371 @@
 float64 float64_round_to_int( float64 a STATUS_PARAM )
 {
-#if defined(__arm__)
-    switch(STATUS(float_rounding_mode)) {
-    default:
-    case float_round_nearest_even:
-        asm("rndd %0, %1" : "=f" (a) : "f"(a));
-        break;
-    case float_round_down:
-        asm("rnddm %0, %1" : "=f" (a) : "f"(a));
-        break;
-    case float_round_up:
-        asm("rnddp %0, %1" : "=f" (a) : "f"(a));
-        break;
-    case float_round_to_zero:
-        asm("rnddz %0, %1" : "=f" (a) : "f"(a));
-        break;
-    }
-#else
     return rint(a);
-#endif
 }
 

trunk/src/recompiler/fpu/softfloat-native.h

@@ -3 +3 @@
 #include <math.h>
 
-#if (defined(_BSD) && !defined(__APPLE__) && !defined(__FreeBSD__)) || defined(CONFIG_SOLARIS) /* VBox: Added __FreeBSD__ */
+#if (defined(CONFIG_BSD) && !defined(__APPLE__) && !defined(__GLIBC__) && !defined(__FreeBSD__)) \
+    || defined(CONFIG_SOLARIS) /* VBox: Added __FreeBSD__ */
 #include <ieeefp.h>
 #define fabsf(f) ((float)fabs(f))
@@ -23 +24 @@
 #if defined(CONFIG_SOLARIS) && \
            ((CONFIG_SOLARIS_VERSION <= 9 ) || \
-           ((CONFIG_SOLARIS_VERSION >= 10) && (__GNUC__ < 4))) \
+           ((CONFIG_SOLARIS_VERSION == 10) && (__GNUC__ < 4))) \
     || (defined(__OpenBSD__) && (OpenBSD < 200811))
 /*
@@ -56 +57 @@
 
 
-# if !defined(VBOX) || !defined(isnormal) || !defined(isgreater) || !defined(isgreaterequal) || !defined(isless) || !defined(islessequal) || !defined(isunordered)
 #define isnormal(x)             (fpclass(x) >= FP_NZERO)
 #define isgreater(x, y)         ((!unordered(x, y)) && ((x) > (y)))
@@ -63 +63 @@
 #define islessequal(x, y)       ((!unordered(x, y)) && ((x) <= (y)))
 #define isunordered(x,y)        unordered(x, y)
-# endif /* !VBOX || missing */
 #endif
 
@@ -128 +127 @@
     float_round_up           = FP_RP,
     float_round_to_zero      = FP_RZ
-};
-#elif defined(__arm__)
-enum {
-    float_round_nearest_even = 0,
-    float_round_down         = 1,
-    float_round_up           = 2,
-    float_round_to_zero      = 3
 };
 #else

trunk/src/recompiler/fpu/softfloat-specialize.h

@@ -62 +62 @@
 #if defined(TARGET_SPARC)
 #define float32_default_nan make_float32(0x7FFFFFFF)
-#elif defined(TARGET_POWERPC) || defined(TARGET_ARM)
+#elif defined(TARGET_POWERPC) || defined(TARGET_ARM) || defined(TARGET_ALPHA)
 #define float32_default_nan make_float32(0x7FC00000)
 #elif defined(TARGET_HPPA)
@@ -190 +190 @@
 #if defined(TARGET_SPARC)
 #define float64_default_nan make_float64(LIT64( 0x7FFFFFFFFFFFFFFF ))
-#elif defined(TARGET_POWERPC) || defined(TARGET_ARM)
+#elif defined(TARGET_POWERPC) || defined(TARGET_ARM) || defined(TARGET_ALPHA)
 #define float64_default_nan make_float64(LIT64( 0x7FF8000000000000 ))
 #elif defined(TARGET_HPPA)

trunk/src/recompiler/fpu/softfloat.c

@@ -1911 +1911 @@
 float32 float32_rem( float32 a, float32 b STATUS_PARAM )
 {
-    flag aSign, bSign, zSign;
+    flag aSign, zSign;
     int16 aExp, bExp, expDiff;
     bits32 aSig, bSig;
@@ -1924 +1924 @@
     bSig = extractFloat32Frac( b );
     bExp = extractFloat32Exp( b );
-    bSign = extractFloat32Sign( b );
     if ( aExp == 0xFF ) {
         if ( aSig || ( ( bExp == 0xFF ) && bSig ) ) {
@@ -2055 +2054 @@
     return roundAndPackFloat32( 0, zExp, zSig STATUS_VAR );
 
+}
+
+/*----------------------------------------------------------------------------
+| Returns the binary exponential of the single-precision floating-point value
+| `a'. The operation is performed according to the IEC/IEEE Standard for
+| Binary Floating-Point Arithmetic.
+|
+| Uses the following identities:
+|
+| 1. -------------------------------------------------------------------------
+|      x    x*ln(2)
+|     2  = e
+|
+| 2. -------------------------------------------------------------------------
+|                      2     3     4     5           n
+|      x        x     x     x     x     x           x
+|     e  = 1 + --- + --- + --- + --- + --- + ... + --- + ...
+|               1!    2!    3!    4!    5!          n!
+*----------------------------------------------------------------------------*/
+
+static const float64 float32_exp2_coefficients[15] =
+{
+    make_float64( 0x3ff0000000000000ll ), /*  1 */
+    make_float64( 0x3fe0000000000000ll ), /*  2 */
+    make_float64( 0x3fc5555555555555ll ), /*  3 */
+    make_float64( 0x3fa5555555555555ll ), /*  4 */
+    make_float64( 0x3f81111111111111ll ), /*  5 */
+    make_float64( 0x3f56c16c16c16c17ll ), /*  6 */
+    make_float64( 0x3f2a01a01a01a01all ), /*  7 */
+    make_float64( 0x3efa01a01a01a01all ), /*  8 */
+    make_float64( 0x3ec71de3a556c734ll ), /*  9 */
+    make_float64( 0x3e927e4fb7789f5cll ), /* 10 */
+    make_float64( 0x3e5ae64567f544e4ll ), /* 11 */
+    make_float64( 0x3e21eed8eff8d898ll ), /* 12 */
+    make_float64( 0x3de6124613a86d09ll ), /* 13 */
+    make_float64( 0x3da93974a8c07c9dll ), /* 14 */
+    make_float64( 0x3d6ae7f3e733b81fll ), /* 15 */
+};
+
+float32 float32_exp2( float32 a STATUS_PARAM )
+{
+    flag aSign;
+    int16 aExp;
+    bits32 aSig;
+    float64 r, x, xn;
+    int i;
+
+    aSig = extractFloat32Frac( a );
+    aExp = extractFloat32Exp( a );
+    aSign = extractFloat32Sign( a );
+
+    if ( aExp == 0xFF) {
+        if ( aSig ) return propagateFloat32NaN( a, float32_zero STATUS_VAR );
+        return (aSign) ? float32_zero : a;
+    }
+    if (aExp == 0) {
+        if (aSig == 0) return float32_one;
+    }
+
+    float_raise( float_flag_inexact STATUS_VAR);
+
+    /* ******************************* */
+    /* using float64 for approximation */
+    /* ******************************* */
+    x = float32_to_float64(a STATUS_VAR);
+    x = float64_mul(x, float64_ln2 STATUS_VAR);
+
+    xn = x;
+    r = float64_one;
+    for (i = 0 ; i < 15 ; i++) {
+        float64 f;
+
+        f = float64_mul(xn, float32_exp2_coefficients[i] STATUS_VAR);
+        r = float64_add(r, f STATUS_VAR);
+
+        xn = float64_mul(xn, x STATUS_VAR);
+    }
+
+    return float64_to_float32(r, status);
 }
 
@@ -3063 +3141 @@
 float64 float64_rem( float64 a, float64 b STATUS_PARAM )
 {
-    flag aSign, bSign, zSign;
+    flag aSign, zSign;
     int16 aExp, bExp, expDiff;
     bits64 aSig, bSig;
@@ -3074 +3152 @@
     bSig = extractFloat64Frac( b );
     bExp = extractFloat64Exp( b );
-    bSign = extractFloat64Sign( b );
     if ( aExp == 0x7FF ) {
         if ( aSig || ( ( bExp == 0x7FF ) && bSig ) ) {
@@ -4033 +4110 @@
 floatx80 floatx80_rem( floatx80 a, floatx80 b STATUS_PARAM )
 {
-    flag aSign, bSign, zSign;
+    flag aSign, zSign;
     int32 aExp, bExp, expDiff;
     bits64 aSig0, aSig1, bSig;
@@ -4044 +4121 @@
     bSig = extractFloatx80Frac( b );
     bExp = extractFloatx80Exp( b );
-    bSign = extractFloatx80Sign( b );
     if ( aExp == 0x7FFF ) {
         if (    (bits64) ( aSig0<<1 )
@@ -5145 +5221 @@
 float128 float128_rem( float128 a, float128 b STATUS_PARAM )
 {
-    flag aSign, bSign, zSign;
+    flag aSign, zSign;
     int32 aExp, bExp, expDiff;
     bits64 aSig0, aSig1, bSig0, bSig1, q, term0, term1, term2;
@@ -5159 +5235 @@
     bSig0 = extractFloat128Frac0( b );
     bExp = extractFloat128Exp( b );
-    bSign = extractFloat128Sign( b );
     if ( aExp == 0x7FFF ) {
         if (    ( aSig0 | aSig1 )

trunk/src/recompiler/fpu/softfloat.h

@@ -284 +284 @@
 float32 float32_rem( float32, float32 STATUS_PARAM );
 float32 float32_sqrt( float32 STATUS_PARAM );
+float32 float32_exp2( float32 STATUS_PARAM );
 float32 float32_log2( float32 STATUS_PARAM );
 int float32_eq( float32, float32 STATUS_PARAM );
@@ -324 +325 @@
 #define float32_zero make_float32(0)
 #define float32_one make_float32(0x3f800000)
+#define float32_ln2 make_float32(0x3f317218)
 
 /*----------------------------------------------------------------------------
@@ -395 +397 @@
 #define float64_zero make_float64(0)
 #define float64_one make_float64(0x3ff0000000000000LL)
+#define float64_ln2 make_float64(0x3fe62e42fefa39efLL)
 
 #ifdef FLOATX80

trunk/src/recompiler/gen-icount.h

@@ +1 @@
+#include "qemu-timer.h"
+
 /* Helpers for instruction counting code generation.  */
 

trunk/src/recompiler/ioport.h

@@ -52 +52 @@
 uint32_t cpu_inl(pio_addr_t addr);
 #else
-void cpu_outb(CPUState *env, pio_addr_t addr, uint8_t val);
-void cpu_outw(CPUState *env, pio_addr_t addr, uint16_t val);
-void cpu_outl(CPUState *env, pio_addr_t addr, uint32_t val);
-uint8_t cpu_inb(CPUState *env, pio_addr_t addr);
-uint16_t cpu_inw(CPUState *env, pio_addr_t addr);
-uint32_t cpu_inl(CPUState *env, pio_addr_t addr);
+void cpu_outb(CPUX86State *env, pio_addr_t addr, uint8_t val);
+void cpu_outw(CPUX86State *env, pio_addr_t addr, uint16_t val);
+void cpu_outl(CPUX86State *env, pio_addr_t addr, uint32_t val);
+uint8_t cpu_inb(CPUX86State *env, pio_addr_t addr);
+uint16_t cpu_inw(CPUX86State *env, pio_addr_t addr);
+uint32_t cpu_inl(CPUX86State *env, pio_addr_t addr);
 #endif
 

trunk/src/recompiler/qemu-common.h

@@ -39 +39 @@
 #  define QEMU_WARN_UNUSED_RESULT
 # endif
+#define QEMU_BUILD_BUG_ON(x) typedef char __build_bug_on__##__LINE__[(x)?-1:1];
+
+#include <stdio.h>
+#include "cpu.h"
+
 
 #else /* !VBOX */
@@ -48 +53 @@
 
 #define QEMU_NORETURN __attribute__ ((__noreturn__))
+#ifdef CONFIG_GCC_ATTRIBUTE_WARN_UNUSED_RESULT
+#define QEMU_WARN_UNUSED_RESULT __attribute__((warn_unused_result))
+#else
+#define QEMU_WARN_UNUSED_RESULT
+#endif
+
+#define QEMU_BUILD_BUG_ON(x) typedef char __build_bug_on__##__LINE__[(x)?-1:1];
+
+typedef struct QEMUTimer QEMUTimer;
+typedef struct QEMUFile QEMUFile;
+typedef struct QEMUBH QEMUBH;
+typedef struct DeviceState DeviceState;
+
 
 /* Hack around the mess dyngen-exec.h causes: We need QEMU_NORETURN in files that
@@ -58 +76 @@
 #include <stdio.h>
 #include <stdarg.h>
+#include <stdbool.h>
 #include <string.h>
 #include <strings.h>
@@ -132 +151 @@
 
 /* bottom halves */
-typedef struct QEMUBH QEMUBH;
-
 typedef void QEMUBHFunc(void *opaque);
 
@@ -168 +185 @@
 int qemu_fls(int i);
 int qemu_fdatasync(int fd);
+int fcntl_setfl(int fd, int flag);
 
 /* path.c */
@@ -196 +214 @@
 char *qemu_strndup(const char *str, size_t size);
 
-void *get_mmap_addr(unsigned long size);
-
-
 void qemu_mutex_lock_iothread(void);
 void qemu_mutex_unlock_iothread(void);
 
 int qemu_open(const char *name, int flags, ...);
+ssize_t qemu_write_full(int fd, const void *buf, size_t count)
+    QEMU_WARN_UNUSED_RESULT;
 void qemu_set_cloexec(int fd);
 
 #ifndef _WIN32
+int qemu_eventfd(int pipefd[2]);
 int qemu_pipe(int pipefd[2]);
 #endif
@@ -216 +234 @@
 /* IO callbacks.  */
 typedef void IOReadHandler(void *opaque, const uint8_t *buf, int size);
-typedef int IOCanRWHandler(void *opaque);
+typedef int IOCanReadHandler(void *opaque);
 typedef void IOHandler(void *opaque);
 
@@ -243 +261 @@
 typedef struct VLANState VLANState;
 typedef struct VLANClientState VLANClientState;
-typedef struct QEMUFile QEMUFile;
 typedef struct i2c_bus i2c_bus;
 typedef struct i2c_slave i2c_slave;
 typedef struct SMBusDevice SMBusDevice;
-typedef struct QEMUTimer QEMUTimer;
 typedef struct PCIHostState PCIHostState;
 typedef struct PCIExpressHost PCIExpressHost;
@@ -258 +274 @@
 typedef struct uWireSlave uWireSlave;
 typedef struct I2SCodec I2SCodec;
-typedef struct DeviceState DeviceState;
 typedef struct SSIBus SSIBus;
+typedef struct EventNotifier EventNotifier;
+typedef struct VirtIODevice VirtIODevice;
+
+typedef uint64_t pcibus_t;
+
+void cpu_exec_init_all(unsigned long tb_size);
 
 /* CPU save/load.  */
@@ -274 +295 @@
 void qemu_cpu_kick(void *env);
 int qemu_cpu_self(void *env);
+
+/* work queue */
+struct qemu_work_item {
+    struct qemu_work_item *next;
+    void (*func)(void *data);
+    void *data;
+    int done;
+};
 
 #ifdef CONFIG_USER_ONLY

trunk/src/recompiler/qemu-log.h

@@ -25 +25 @@
  */
 #define qemu_loglevel_mask(b) ((loglevel & (b)) != 0)
-
 
 

trunk/src/recompiler/softmmu_exec.h

@@ -101 +101 @@
 #endif /* (NB_MMU_MODES >= 5) */
 
-#if (NB_MMU_MODES > 5)
-#error "NB_MMU_MODES > 5 is not supported for now"
-#endif /* (NB_MMU_MODES > 5) */
+#if (NB_MMU_MODES >= 6)
+
+#define ACCESS_TYPE 5
+#define MEMSUFFIX MMU_MODE5_SUFFIX
+#define DATA_SIZE 1
+#include "softmmu_header.h"
+
+#define DATA_SIZE 2
+#include "softmmu_header.h"
+
+#define DATA_SIZE 4
+#include "softmmu_header.h"
+
+#define DATA_SIZE 8
+#include "softmmu_header.h"
+#undef ACCESS_TYPE
+#undef MEMSUFFIX
+#endif /* (NB_MMU_MODES >= 6) */
+
+#if (NB_MMU_MODES > 6)
+#error "NB_MMU_MODES > 6 is not supported for now"
+#endif /* (NB_MMU_MODES > 6) */
 
 /* these access are slower, they must be as rare as possible */

trunk/src/recompiler/softmmu_header.h

@@ -71 +71 @@
 #define RES_TYPE uint64_t
 #else
-#define RES_TYPE int
+#define RES_TYPE uint32_t
 #endif
 

trunk/src/recompiler/softmmu_template.h

@@ -26 +26 @@
  * of the LGPL is applied is otherwise unspecified.
  */
+#include "qemu-timer.h"
 
 #define DATA_SIZE (1 << SHIFT)
@@ -117 +118 @@
     int index;
     target_ulong tlb_addr;
-    target_phys_addr_t addend;
+    target_phys_addr_t ioaddr;
+    unsigned long addend;
     void *retaddr;
 
@@ -131 +133 @@
                 goto do_unaligned_access;
             retaddr = GETPC();
-            addend = env->iotlb[mmu_idx][index];
-            res = glue(io_read, SUFFIX)(addend, addr, retaddr);
+            ioaddr = env->iotlb[mmu_idx][index];
+            res = glue(io_read, SUFFIX)(ioaddr, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
             /* slow unaligned access (it spans two pages or IO) */
@@ -173 +175 @@
     DATA_TYPE res, res1, res2;
     int index, shift;
-    target_phys_addr_t addend;
+    target_phys_addr_t ioaddr;
+    unsigned long addend;
     target_ulong tlb_addr, addr1, addr2;
 
@@ -184 +187 @@
             if ((addr & (DATA_SIZE - 1)) != 0)
                 goto do_unaligned_access;
-            retaddr = GETPC();
-            addend = env->iotlb[mmu_idx][index];
-            res = glue(io_read, SUFFIX)(addend, addr, retaddr);
+            ioaddr = env->iotlb[mmu_idx][index];
+            res = glue(io_read, SUFFIX)(ioaddr, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
         do_unaligned_access:
@@ -255 +257 @@
                                                  int mmu_idx)
 {
-    target_phys_addr_t addend;
+    target_phys_addr_t ioaddr;
+    unsigned long addend;
     target_ulong tlb_addr;
     void *retaddr;
@@ -269 +272 @@
                 goto do_unaligned_access;
             retaddr = GETPC();
-            addend = env->iotlb[mmu_idx][index];
-            glue(io_write, SUFFIX)(addend, val, addr, retaddr);
+            ioaddr = env->iotlb[mmu_idx][index];
+            glue(io_write, SUFFIX)(ioaddr, val, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
         do_unaligned_access:
@@ -308 +311 @@
                                                    void *retaddr)
 {
-    target_phys_addr_t addend;
+    target_phys_addr_t ioaddr;
+    unsigned long addend;
     target_ulong tlb_addr;
     int index, i;
@@ -320 +324 @@
             if ((addr & (DATA_SIZE - 1)) != 0)
                 goto do_unaligned_access;
-            addend = env->iotlb[mmu_idx][index];
-            glue(io_write, SUFFIX)(addend, val, addr, retaddr);
+            ioaddr = env->iotlb[mmu_idx][index];
+            glue(io_write, SUFFIX)(ioaddr, val, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
         do_unaligned_access:

trunk/src/recompiler/target-i386/cpu.h

@@ -58 +58 @@
 #include "softfloat.h"
 
-#if defined(VBOX)
+#ifdef VBOX
 # include <iprt/critsect.h>
 # include <iprt/thread.h>
@@ -758 +758 @@
     uint32_t sipi_vector;
 
+    uint32_t cpuid_kvm_features;
+
     /* in order to simplify APIC support, we leave this pointer to the
        user */
-    struct APICState *apic_state;
+    struct DeviceState *apic_state;
 
     uint64 mcg_cap;
@@ -773 +775 @@
     uint16_t fptag_vmstate;
     uint16_t fpregs_format_vmstate;
+
+    uint64_t xstate_bv;
+    XMMReg ymmh_regs[CPU_NB_REGS];
+
+    uint64_t xcr0;
 #else  /* VBOX */
 
@@ -783 +790 @@
     /** Profiling tb_flush. */
     STAMPROFILE StatTbFlush;
+
+    /** Addends for HVA -> GPA translations. */
+    target_phys_addr_t phys_addends[NB_MMU_MODES][CPU_TLB_SIZE];
 #endif /* VBOX */
 } CPUX86State;
@@ -906 +916 @@
 int cpu_x86_exec(CPUX86State *s);
 void cpu_x86_close(CPUX86State *s);
-void x86_cpu_list (FILE *f, int (*cpu_fprintf)(FILE *f, const char *fmt,
-                                                 ...));
+void x86_cpu_list (FILE *f, int (*cpu_fprintf)(FILE *f, const char *fmt, ...),
+                   const char *optarg);
+void x86_cpudef_setup(void);
+
 int cpu_get_pic_interrupt(CPUX86State *s);
 /* MSDOS compatibility mode FPU exception support */
@@ -974 +986 @@
 }
 
+static inline void cpu_x86_load_seg_cache_sipi(CPUX86State *env,
+                                               int sipi_vector)
+{
+    env->eip = 0;
+    cpu_x86_load_seg_cache(env, R_CS, sipi_vector << 8,
+                           sipi_vector << 12,
+                           env->segs[R_CS].limit,
+                           env->segs[R_CS].flags);
+    env->halted = 0;
+}
+
 int cpu_x86_get_descr_debug(CPUX86State *env, unsigned int selector,
                             target_ulong *base, unsigned int *limit,
@@ -1006 +1029 @@
                            void *puc);
 
+/* cpuid.c */
+void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
+                   uint32_t *eax, uint32_t *ebx,
+                   uint32_t *ecx, uint32_t *edx);
+int cpu_x86_register (CPUX86State *env, const char *cpu_model);
+void cpu_clear_apic_feature(CPUX86State *env);
+
 /* helper.c */
 int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
@@ -1011 +1041 @@
 #define cpu_handle_mmu_fault cpu_x86_handle_mmu_fault
 void cpu_x86_set_a20(CPUX86State *env, int a20_state);
-void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
-                   uint32_t *eax, uint32_t *ebx,
-                   uint32_t *ecx, uint32_t *edx);
 
 static inline int hw_breakpoint_enabled(unsigned long dr7, int index)
@@ -1039 +1066 @@
 void cpu_x86_update_cr3(CPUX86State *env, target_ulong new_cr3);
 void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4);
-
-/* hw/apic.c */
-void cpu_set_apic_base(CPUX86State *env, uint64_t val);
-uint64_t cpu_get_apic_base(CPUX86State *env);
-void cpu_set_apic_tpr(CPUX86State *env, uint8_t val);
-#ifndef NO_CPU_IO_DEFS
-uint8_t cpu_get_apic_tpr(CPUX86State *env);
-#endif
 
 /* hw/pc.c */
@@ -1078 +1097 @@
 #define TARGET_PAGE_BITS 12
 
+#ifdef TARGET_X86_64
+#define TARGET_PHYS_ADDR_SPACE_BITS 52
+/* ??? This is really 48 bits, sign-extended, but the only thing
+   accessible to userland with bit 48 set is the VSYSCALL, and that
+   is handled via other mechanisms.  */
+#define TARGET_VIRT_ADDR_SPACE_BITS 47
+#else
+#define TARGET_PHYS_ADDR_SPACE_BITS 36
+#define TARGET_VIRT_ADDR_SPACE_BITS 32
+#endif
+
 #define cpu_init cpu_x86_init
 #define cpu_exec cpu_x86_exec
 #define cpu_gen_code cpu_x86_gen_code
 #define cpu_signal_handler cpu_x86_signal_handler
-#define cpu_list x86_cpu_list
-
-#define CPU_SAVE_VERSION 11
+#define cpu_list_id x86_cpu_list
+#define cpudef_setup    x86_cpudef_setup
+
+#define CPU_SAVE_VERSION 12
 
 /* MMU modes definitions */
@@ -1113 +1144 @@
 
 #include "cpu-all.h"
-#include "exec-all.h"
-
 #include "svm.h"
 
-static inline void cpu_pc_from_tb(CPUState *env, TranslationBlock *tb)
-{
-    env->eip = tb->pc - tb->cs_base;
-}
+#ifndef VBOX
+#if !defined(CONFIG_USER_ONLY)
+#include "hw/apic.h"
+#endif
+#else  /* VBOX */
+extern void     cpu_set_apic_tpr(CPUX86State *env, uint8_t val);
+extern uint8_t  cpu_get_apic_tpr(CPUX86State *env);
+extern uint64_t cpu_get_apic_base(CPUX86State *env);
+#endif /* VBOX */
 
 static inline void cpu_get_tb_cpu_state(CPUState *env, target_ulong *pc,

trunk/src/recompiler/target-i386/exec.h

@@ -84 +84 @@
 void QEMU_NORETURN raise_exception_err(int exception_index, int error_code);
 void QEMU_NORETURN raise_exception(int exception_index);
+void QEMU_NORETURN raise_exception_env(int exception_index, CPUState *nenv);
 void do_smm_enter(void);
 
@@ -301 +302 @@
 }
 
-static inline void env_to_regs(void)
-{
-#ifdef reg_EAX
-    EAX = env->regs[R_EAX];
-#endif
-#ifdef reg_ECX
-    ECX = env->regs[R_ECX];
-#endif
-#ifdef reg_EDX
-    EDX = env->regs[R_EDX];
-#endif
-#ifdef reg_EBX
-    EBX = env->regs[R_EBX];
-#endif
-#ifdef reg_ESP
-    ESP = env->regs[R_ESP];
-#endif
-#ifdef reg_EBP
-    EBP = env->regs[R_EBP];
-#endif
-#ifdef reg_ESI
-    ESI = env->regs[R_ESI];
-#endif
-#ifdef reg_EDI
-    EDI = env->regs[R_EDI];
-#endif
-}
-
-static inline void regs_to_env(void)
-{
-#ifdef reg_EAX
-    env->regs[R_EAX] = EAX;
-#endif
-#ifdef reg_ECX
-    env->regs[R_ECX] = ECX;
-#endif
-#ifdef reg_EDX
-    env->regs[R_EDX] = EDX;
-#endif
-#ifdef reg_EBX
-    env->regs[R_EBX] = EBX;
-#endif
-#ifdef reg_ESP
-    env->regs[R_ESP] = ESP;
-#endif
-#ifdef reg_EBP
-    env->regs[R_EBP] = EBP;
-#endif
-#ifdef reg_ESI
-    env->regs[R_ESI] = ESI;
-#endif
-#ifdef reg_EDI
-    env->regs[R_EDI] = EDI;
-#endif
-}
-
 static inline int cpu_has_work(CPUState *env)
 {
@@ -393 +338 @@
         env->hflags |= HF_SVME_MASK;
 }
+
+static inline void cpu_pc_from_tb(CPUState *env, TranslationBlock *tb)
+{
+    env->eip = tb->pc - tb->cs_base;
+}
+

trunk/src/recompiler/target-i386/helper.c

@@ -43 +43 @@
 //#define DEBUG_MMU
 
-#ifndef VBOX
-/* feature flags taken from "Intel Processor Identification and the CPUID
- * Instruction" and AMD's "CPUID Specification". In cases of disagreement
- * about feature names, the Linux name is used. */
-static const char *feature_name[] = {
-    "fpu", "vme", "de", "pse", "tsc", "msr", "pae", "mce",
-    "cx8", "apic", NULL, "sep", "mtrr", "pge", "mca", "cmov",
-    "pat", "pse36", "pn" /* Intel psn */, "clflush" /* Intel clfsh */, NULL, "ds" /* Intel dts */, "acpi", "mmx",
-    "fxsr", "sse", "sse2", "ss", "ht" /* Intel htt */, "tm", "ia64", "pbe",
-};
-static const char *ext_feature_name[] = {
-    "pni" /* Intel,AMD sse3 */, NULL, NULL, "monitor", "ds_cpl", "vmx", NULL /* Linux smx */, "est",
-    "tm2", "ssse3", "cid", NULL, NULL, "cx16", "xtpr", NULL,
-    NULL, NULL, "dca", NULL, NULL, NULL, NULL, "popcnt",
-    NULL, NULL, NULL, NULL, NULL, NULL, NULL, "hypervisor",
-};
-static const char *ext2_feature_name[] = {
-    "fpu", "vme", "de", "pse", "tsc", "msr", "pae", "mce",
-    "cx8" /* AMD CMPXCHG8B */, "apic", NULL, "syscall", "mtrr", "pge", "mca", "cmov",
-    "pat", "pse36", NULL, NULL /* Linux mp */, "nx" /* Intel xd */, NULL, "mmxext", "mmx",
-    "fxsr", "fxsr_opt" /* AMD ffxsr */, "pdpe1gb" /* AMD Page1GB */, "rdtscp", NULL, "lm" /* Intel 64 */, "3dnowext", "3dnow",
-};
-static const char *ext3_feature_name[] = {
-    "lahf_lm" /* AMD LahfSahf */, "cmp_legacy", "svm", "extapic" /* AMD ExtApicSpace */, "cr8legacy" /* AMD AltMovCr8 */, "abm", "sse4a", "misalignsse",
-    "3dnowprefetch", "osvw", NULL /* Linux ibs */, NULL, "skinit", "wdt", NULL, NULL,
-    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
-    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
-};
-
-static void add_flagname_to_bitmaps(const char *flagname, uint32_t *features,
-                                    uint32_t *ext_features,
-                                    uint32_t *ext2_features,
-                                    uint32_t *ext3_features)
-{
-    int i;
-    int found = 0;
-
-    for ( i = 0 ; i < 32 ; i++ )
-        if (feature_name[i] && !strcmp (flagname, feature_name[i])) {
-            *features |= 1 << i;
-            found = 1;
-        }
-    for ( i = 0 ; i < 32 ; i++ )
-        if (ext_feature_name[i] && !strcmp (flagname, ext_feature_name[i])) {
-            *ext_features |= 1 << i;
-            found = 1;
-        }
-    for ( i = 0 ; i < 32 ; i++ )
-        if (ext2_feature_name[i] && !strcmp (flagname, ext2_feature_name[i])) {
-            *ext2_features |= 1 << i;
-            found = 1;
-        }
-    for ( i = 0 ; i < 32 ; i++ )
-        if (ext3_feature_name[i] && !strcmp (flagname, ext3_feature_name[i])) {
-            *ext3_features |= 1 << i;
-            found = 1;
-        }
-    if (!found) {
-        fprintf(stderr, "CPU feature %s not found\n", flagname);
-    }
-}
-#endif /* !VBOX */
-
-typedef struct x86_def_t {
-    const char *name;
-    uint32_t level;
-    uint32_t vendor1, vendor2, vendor3;
-    int family;
-    int model;
-    int stepping;
-    uint32_t features, ext_features, ext2_features, ext3_features;
-    uint32_t xlevel;
-    char model_id[48];
-    int vendor_override;
-} x86_def_t;
-
-#ifndef VBOX
-#define I486_FEATURES (CPUID_FP87 | CPUID_VME | CPUID_PSE)
-#define PENTIUM_FEATURES (I486_FEATURES | CPUID_DE | CPUID_TSC | \
-          CPUID_MSR | CPUID_MCE | CPUID_CX8 | CPUID_MMX)
-#define PENTIUM2_FEATURES (PENTIUM_FEATURES | CPUID_PAE | CPUID_SEP | \
-          CPUID_MTRR | CPUID_PGE | CPUID_MCA | CPUID_CMOV | CPUID_PAT | \
-          CPUID_PSE36 | CPUID_FXSR)
-#define PENTIUM3_FEATURES (PENTIUM2_FEATURES | CPUID_SSE)
-#define PPRO_FEATURES (CPUID_FP87 | CPUID_DE | CPUID_PSE | CPUID_TSC | \
-          CPUID_MSR | CPUID_MCE | CPUID_CX8 | CPUID_PGE | CPUID_CMOV | \
-          CPUID_PAT | CPUID_FXSR | CPUID_MMX | CPUID_SSE | CPUID_SSE2 | \
-          CPUID_PAE | CPUID_SEP | CPUID_APIC)
-static x86_def_t x86_defs[] = {
-#ifdef TARGET_X86_64
-    {
-        .name = "qemu64",
-        .level = 4,
-        .vendor1 = CPUID_VENDOR_AMD_1,
-        .vendor2 = CPUID_VENDOR_AMD_2,
-        .vendor3 = CPUID_VENDOR_AMD_3,
-        .family = 6,
-        .model = 2,
-        .stepping = 3,
-        .features = PPRO_FEATURES |
-        /* these features are needed for Win64 and aren't fully implemented */
-            CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA |
-        /* this feature is needed for Solaris and isn't fully implemented */
-            CPUID_PSE36,
-        .ext_features = CPUID_EXT_SSE3 | CPUID_EXT_CX16 | CPUID_EXT_POPCNT,
-        .ext2_features = (PPRO_FEATURES & 0x0183F3FF) |
-            CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX,
-        .ext3_features = CPUID_EXT3_LAHF_LM | CPUID_EXT3_SVM |
-            CPUID_EXT3_ABM | CPUID_EXT3_SSE4A,
-        .xlevel = 0x8000000A,
-        .model_id = "QEMU Virtual CPU version " QEMU_VERSION,
-    },
-    {
-        .name = "phenom",
-        .level = 5,
-        .vendor1 = CPUID_VENDOR_AMD_1,
-        .vendor2 = CPUID_VENDOR_AMD_2,
-        .vendor3 = CPUID_VENDOR_AMD_3,
-        .family = 16,
-        .model = 2,
-        .stepping = 3,
-        /* Missing: CPUID_VME, CPUID_HT */
-        .features = PPRO_FEATURES |
-            CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA |
-            CPUID_PSE36,
-        .ext_features = CPUID_EXT_SSE3 | CPUID_EXT_MONITOR | CPUID_EXT_CX16 |
-            CPUID_EXT_POPCNT,
-        /* Missing: CPUID_EXT2_PDPE1GB, CPUID_EXT2_RDTSCP */
-        .ext2_features = (PPRO_FEATURES & 0x0183F3FF) |
-            CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX |
-            CPUID_EXT2_3DNOW | CPUID_EXT2_3DNOWEXT | CPUID_EXT2_MMXEXT |
-            CPUID_EXT2_FFXSR,
-        /* Missing: CPUID_EXT3_CMP_LEG, CPUID_EXT3_EXTAPIC,
-                    CPUID_EXT3_CR8LEG,
-                    CPUID_EXT3_MISALIGNSSE, CPUID_EXT3_3DNOWPREFETCH,
-                    CPUID_EXT3_OSVW, CPUID_EXT3_IBS */
-        .ext3_features = CPUID_EXT3_LAHF_LM | CPUID_EXT3_SVM |
-            CPUID_EXT3_ABM | CPUID_EXT3_SSE4A,
-        .xlevel = 0x8000001A,
-        .model_id = "AMD Phenom(tm) 9550 Quad-Core Processor"
-    },
-    {
-        .name = "core2duo",
-        .level = 10,
-        .family = 6,
-        .model = 15,
-        .stepping = 11,
-        /* The original CPU also implements these features:
-               CPUID_VME, CPUID_DTS, CPUID_ACPI, CPUID_SS, CPUID_HT,
-               CPUID_TM, CPUID_PBE */
-        .features = PPRO_FEATURES |
-            CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA |
-            CPUID_PSE36,
-        /* The original CPU also implements these ext features:
-               CPUID_EXT_DTES64, CPUID_EXT_DSCPL, CPUID_EXT_VMX, CPUID_EXT_EST,
-               CPUID_EXT_TM2, CPUID_EXT_CX16, CPUID_EXT_XTPR, CPUID_EXT_PDCM */
-        .ext_features = CPUID_EXT_SSE3 | CPUID_EXT_MONITOR | CPUID_EXT_SSSE3,
-        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX,
-        .ext3_features = CPUID_EXT3_LAHF_LM,
-        .xlevel = 0x80000008,
-        .model_id = "Intel(R) Core(TM)2 Duo CPU     T7700  @ 2.40GHz",
-    },
-    {
-        .name = "kvm64",
-        .level = 5,
-        .vendor1 = CPUID_VENDOR_INTEL_1,
-        .vendor2 = CPUID_VENDOR_INTEL_2,
-        .vendor3 = CPUID_VENDOR_INTEL_3,
-        .family = 15,
-        .model = 6,
-        .stepping = 1,
-        /* Missing: CPUID_VME, CPUID_HT */
-        .features = PPRO_FEATURES |
-            CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA |
-            CPUID_PSE36,
-        /* Missing: CPUID_EXT_POPCNT, CPUID_EXT_MONITOR */
-        .ext_features = CPUID_EXT_SSE3 | CPUID_EXT_CX16,
-        /* Missing: CPUID_EXT2_PDPE1GB, CPUID_EXT2_RDTSCP */
-        .ext2_features = (PPRO_FEATURES & 0x0183F3FF) |
-            CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX,
-        /* Missing: CPUID_EXT3_LAHF_LM, CPUID_EXT3_CMP_LEG, CPUID_EXT3_EXTAPIC,
-                    CPUID_EXT3_CR8LEG, CPUID_EXT3_ABM, CPUID_EXT3_SSE4A,
-                    CPUID_EXT3_MISALIGNSSE, CPUID_EXT3_3DNOWPREFETCH,
-                    CPUID_EXT3_OSVW, CPUID_EXT3_IBS, CPUID_EXT3_SVM */
-        .ext3_features = 0,
-        .xlevel = 0x80000008,
-        .model_id = "Common KVM processor"
-    },
-#endif
-    {
-        .name = "qemu32",
-        .level = 4,
-        .family = 6,
-        .model = 3,
-        .stepping = 3,
-        .features = PPRO_FEATURES,
-        .ext_features = CPUID_EXT_SSE3 | CPUID_EXT_POPCNT,
-        .xlevel = 0,
-        .model_id = "QEMU Virtual CPU version " QEMU_VERSION,
-    },
-    {
-        .name = "coreduo",
-        .level = 10,
-        .family = 6,
-        .model = 14,
-        .stepping = 8,
-        /* The original CPU also implements these features:
-               CPUID_DTS, CPUID_ACPI, CPUID_SS, CPUID_HT,
-               CPUID_TM, CPUID_PBE */
-        .features = PPRO_FEATURES | CPUID_VME |
-            CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA,
-        /* The original CPU also implements these ext features:
-               CPUID_EXT_VMX, CPUID_EXT_EST, CPUID_EXT_TM2, CPUID_EXT_XTPR,
-               CPUID_EXT_PDCM */
-        .ext_features = CPUID_EXT_SSE3 | CPUID_EXT_MONITOR,
-        .ext2_features = CPUID_EXT2_NX,
-        .xlevel = 0x80000008,
-        .model_id = "Genuine Intel(R) CPU           T2600  @ 2.16GHz",
-    },
-    {
-        .name = "486",
-        .level = 0,
-        .family = 4,
-        .model = 0,
-        .stepping = 0,
-        .features = I486_FEATURES,
-        .xlevel = 0,
-    },
-    {
-        .name = "pentium",
-        .level = 1,
-        .family = 5,
-        .model = 4,
-        .stepping = 3,
-        .features = PENTIUM_FEATURES,
-        .xlevel = 0,
-    },
-    {
-        .name = "pentium2",
-        .level = 2,
-        .family = 6,
-        .model = 5,
-        .stepping = 2,
-        .features = PENTIUM2_FEATURES,
-        .xlevel = 0,
-    },
-    {
-        .name = "pentium3",
-        .level = 2,
-        .family = 6,
-        .model = 7,
-        .stepping = 3,
-        .features = PENTIUM3_FEATURES,
-        .xlevel = 0,
-    },
-    {
-        .name = "athlon",
-        .level = 2,
-        .vendor1 = CPUID_VENDOR_AMD_1,
-        .vendor2 = CPUID_VENDOR_AMD_2,
-        .vendor3 = CPUID_VENDOR_AMD_3,
-        .family = 6,
-        .model = 2,
-        .stepping = 3,
-        .features = PPRO_FEATURES | CPUID_PSE36 | CPUID_VME | CPUID_MTRR | CPUID_MCA,
-        .ext2_features = (PPRO_FEATURES & 0x0183F3FF) | CPUID_EXT2_MMXEXT | CPUID_EXT2_3DNOW | CPUID_EXT2_3DNOWEXT,
-        .xlevel = 0x80000008,
-        /* XXX: put another string ? */
-        .model_id = "QEMU Virtual CPU version " QEMU_VERSION,
-    },
-    {
-        .name = "n270",
-        /* original is on level 10 */
-        .level = 5,
-        .family = 6,
-        .model = 28,
-        .stepping = 2,
-        .features = PPRO_FEATURES |
-            CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA | CPUID_VME,
-            /* Missing: CPUID_DTS | CPUID_ACPI | CPUID_SS |
-             * CPUID_HT | CPUID_TM | CPUID_PBE */
-            /* Some CPUs got no CPUID_SEP */
-        .ext_features = CPUID_EXT_MONITOR |
-            CPUID_EXT_SSE3 /* PNI */ | CPUID_EXT_SSSE3,
-            /* Missing: CPUID_EXT_DSCPL | CPUID_EXT_EST |
-             * CPUID_EXT_TM2 | CPUID_EXT_XTPR */
-        .ext2_features = (PPRO_FEATURES & 0x0183F3FF) | CPUID_EXT2_NX,
-        /* Missing: .ext3_features = CPUID_EXT3_LAHF_LM */
-        .xlevel = 0x8000000A,
-        .model_id = "Intel(R) Atom(TM) CPU N270   @ 1.60GHz",
-    },
-};
-
-static void host_cpuid(uint32_t function, uint32_t count, uint32_t *eax,
-                               uint32_t *ebx, uint32_t *ecx, uint32_t *edx);
-
-static int cpu_x86_fill_model_id(char *str)
-{
-    uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0;
-    int i;
-
-    for (i = 0; i < 3; i++) {
-        host_cpuid(0x80000002 + i, 0, &eax, &ebx, &ecx, &edx);
-        memcpy(str + i * 16 +  0, &eax, 4);
-        memcpy(str + i * 16 +  4, &ebx, 4);
-        memcpy(str + i * 16 +  8, &ecx, 4);
-        memcpy(str + i * 16 + 12, &edx, 4);
-    }
-    return 0;
-}
-
-static int cpu_x86_fill_host(x86_def_t *x86_cpu_def)
-{
-    uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0;
-
-    x86_cpu_def->name = "host";
-    host_cpuid(0x0, 0, &eax, &ebx, &ecx, &edx);
-    x86_cpu_def->level = eax;
-    x86_cpu_def->vendor1 = ebx;
-    x86_cpu_def->vendor2 = edx;
-    x86_cpu_def->vendor3 = ecx;
-
-    host_cpuid(0x1, 0, &eax, &ebx, &ecx, &edx);
-    x86_cpu_def->family = ((eax >> 8) & 0x0F) + ((eax >> 20) & 0xFF);
-    x86_cpu_def->model = ((eax >> 4) & 0x0F) | ((eax & 0xF0000) >> 12);
-    x86_cpu_def->stepping = eax & 0x0F;
-    x86_cpu_def->ext_features = ecx;
-    x86_cpu_def->features = edx;
-
-    host_cpuid(0x80000000, 0, &eax, &ebx, &ecx, &edx);
-    x86_cpu_def->xlevel = eax;
-
-    host_cpuid(0x80000001, 0, &eax, &ebx, &ecx, &edx);
-    x86_cpu_def->ext2_features = edx;
-    x86_cpu_def->ext3_features = ecx;
-    cpu_x86_fill_model_id(x86_cpu_def->model_id);
-    x86_cpu_def->vendor_override = 0;
-
-    return 0;
-}
-
-static int cpu_x86_find_by_name(x86_def_t *x86_cpu_def, const char *cpu_model)
-{
-    unsigned int i;
-    x86_def_t *def;
-
-    char *s = strdup(cpu_model);
-    char *featurestr, *name = strtok(s, ",");
-    uint32_t plus_features = 0, plus_ext_features = 0, plus_ext2_features = 0, plus_ext3_features = 0;
-    uint32_t minus_features = 0, minus_ext_features = 0, minus_ext2_features = 0, minus_ext3_features = 0;
-    uint32_t numvalue;
-
-    def = NULL;
-    for (i = 0; i < ARRAY_SIZE(x86_defs); i++) {
-        if (strcmp(name, x86_defs[i].name) == 0) {
-            def = &x86_defs[i];
-            break;
-        }
-    }
-    if (kvm_enabled() && strcmp(name, "host") == 0) {
-        cpu_x86_fill_host(x86_cpu_def);
-    } else if (!def) {
-        goto error;
-    } else {
-        memcpy(x86_cpu_def, def, sizeof(*def));
-    }
-
-    add_flagname_to_bitmaps("hypervisor", &plus_features,
-        &plus_ext_features, &plus_ext2_features, &plus_ext3_features);
-
-    featurestr = strtok(NULL, ",");
-
-    while (featurestr) {
-        char *val;
-        if (featurestr[0] == '+') {
-            add_flagname_to_bitmaps(featurestr + 1, &plus_features, &plus_ext_features, &plus_ext2_features, &plus_ext3_features);
-        } else if (featurestr[0] == '-') {
-            add_flagname_to_bitmaps(featurestr + 1, &minus_features, &minus_ext_features, &minus_ext2_features, &minus_ext3_features);
-        } else if ((val = strchr(featurestr, '='))) {
-            *val = 0; val++;
-            if (!strcmp(featurestr, "family")) {
-                char *err;
-                numvalue = strtoul(val, &err, 0);
-                if (!*val || *err) {
-                    fprintf(stderr, "bad numerical value %s\n", val);
-                    goto error;
-                }
-                x86_cpu_def->family = numvalue;
-            } else if (!strcmp(featurestr, "model")) {
-                char *err;
-                numvalue = strtoul(val, &err, 0);
-                if (!*val || *err || numvalue > 0xff) {
-                    fprintf(stderr, "bad numerical value %s\n", val);
-                    goto error;
-                }
-                x86_cpu_def->model = numvalue;
-            } else if (!strcmp(featurestr, "stepping")) {
-                char *err;
-                numvalue = strtoul(val, &err, 0);
-                if (!*val || *err || numvalue > 0xf) {
-                    fprintf(stderr, "bad numerical value %s\n", val);
-                    goto error;
-                }
-                x86_cpu_def->stepping = numvalue ;
-            } else if (!strcmp(featurestr, "level")) {
-                char *err;
-                numvalue = strtoul(val, &err, 0);
-                if (!*val || *err) {
-                    fprintf(stderr, "bad numerical value %s\n", val);
-                    goto error;
-                }
-                x86_cpu_def->level = numvalue;
-            } else if (!strcmp(featurestr, "xlevel")) {
-                char *err;
-                numvalue = strtoul(val, &err, 0);
-                if (!*val || *err) {
-                    fprintf(stderr, "bad numerical value %s\n", val);
-                    goto error;
-                }
-                if (numvalue < 0x80000000) {
-                        numvalue += 0x80000000;
-                }
-                x86_cpu_def->xlevel = numvalue;
-            } else if (!strcmp(featurestr, "vendor")) {
-                if (strlen(val) != 12) {
-                    fprintf(stderr, "vendor string must be 12 chars long\n");
-                    goto error;
-                }
-                x86_cpu_def->vendor1 = 0;
-                x86_cpu_def->vendor2 = 0;
-                x86_cpu_def->vendor3 = 0;
-                for(i = 0; i < 4; i++) {
-                    x86_cpu_def->vendor1 |= ((uint8_t)val[i    ]) << (8 * i);
-                    x86_cpu_def->vendor2 |= ((uint8_t)val[i + 4]) << (8 * i);
-                    x86_cpu_def->vendor3 |= ((uint8_t)val[i + 8]) << (8 * i);
-                }
-                x86_cpu_def->vendor_override = 1;
-            } else if (!strcmp(featurestr, "model_id")) {
-                pstrcpy(x86_cpu_def->model_id, sizeof(x86_cpu_def->model_id),
-                        val);
-            } else {
-                fprintf(stderr, "unrecognized feature %s\n", featurestr);
-                goto error;
-            }
-        } else {
-            fprintf(stderr, "feature string `%s' not in format (+feature|-feature|feature=xyz)\n", featurestr);
-            goto error;
-        }
-        featurestr = strtok(NULL, ",");
-    }
-    x86_cpu_def->features |= plus_features;
-    x86_cpu_def->ext_features |= plus_ext_features;
-    x86_cpu_def->ext2_features |= plus_ext2_features;
-    x86_cpu_def->ext3_features |= plus_ext3_features;
-    x86_cpu_def->features &= ~minus_features;
-    x86_cpu_def->ext_features &= ~minus_ext_features;
-    x86_cpu_def->ext2_features &= ~minus_ext2_features;
-    x86_cpu_def->ext3_features &= ~minus_ext3_features;
-    free(s);
-    return 0;
-
-error:
-    free(s);
-    return -1;
-}
-
-void x86_cpu_list (FILE *f, int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
-{
-    unsigned int i;
-
-    for (i = 0; i < ARRAY_SIZE(x86_defs); i++)
-        (*cpu_fprintf)(f, "x86 %16s\n", x86_defs[i].name);
-}
-#endif /* !VBOX */
-
-static int cpu_x86_register (CPUX86State *env, const char *cpu_model)
-{
-#ifndef VBOX
-    x86_def_t def1, *def = &def1;
-
-    if (cpu_x86_find_by_name(def, cpu_model) < 0)
-        return -1;
-    if (def->vendor1) {
-        env->cpuid_vendor1 = def->vendor1;
-        env->cpuid_vendor2 = def->vendor2;
-        env->cpuid_vendor3 = def->vendor3;
-    } else {
-        env->cpuid_vendor1 = CPUID_VENDOR_INTEL_1;
-        env->cpuid_vendor2 = CPUID_VENDOR_INTEL_2;
-        env->cpuid_vendor3 = CPUID_VENDOR_INTEL_3;
-    }
-    env->cpuid_vendor_override = def->vendor_override;
-    env->cpuid_level = def->level;
-    if (def->family > 0x0f)
-        env->cpuid_version = 0xf00 | ((def->family - 0x0f) << 20);
-    else
-        env->cpuid_version = def->family << 8;
-    env->cpuid_version |= ((def->model & 0xf) << 4) | ((def->model >> 4) << 16);
-    env->cpuid_version |= def->stepping;
-    env->cpuid_features = def->features;
-    env->pat = 0x0007040600070406ULL;
-    env->cpuid_ext_features = def->ext_features;
-    env->cpuid_ext2_features = def->ext2_features;
-    env->cpuid_xlevel = def->xlevel;
-    env->cpuid_ext3_features = def->ext3_features;
-    {
-        const char *model_id = def->model_id;
-        int c, len, i;
-        if (!model_id)
-            model_id = "";
-        len = strlen(model_id);
-        for(i = 0; i < 48; i++) {
-            if (i >= len)
-                c = '\0';
-            else
-                c = (uint8_t)model_id[i];
-            env->cpuid_model[i >> 2] |= c << (8 * (i & 3));
-        }
-    }
-#endif /* !VBOX */
-    return 0;
-}
-
 /* NOTE: must be called outside the CPU execute loop */
 void cpu_reset(CPUX86State *env)
@@ -620 +97 @@
 
     env->eip = 0xfff0;
-#ifndef VBOX
+#ifndef VBOX /* We'll get the right value from CPUM. */
     env->regs[R_EDX] = env->cpuid_version;
-#else
-    /** @todo: is it right? */
-    env->regs[R_EDX] = 0x600; /* indicate P6 processor */
 #endif
 
@@ -762 +236 @@
             }
         };
-        cpu_fprintf(f, sys_type_name[(env->hflags & HF_LMA_MASK) ? 1 : 0]
-                                    [(sc->flags & DESC_TYPE_MASK)
-                                     >> DESC_TYPE_SHIFT]);
+        cpu_fprintf(f, "%s",
+                    sys_type_name[(env->hflags & HF_LMA_MASK) ? 1 : 0]
+                                 [(sc->flags & DESC_TYPE_MASK)
+                                  >> DESC_TYPE_SHIFT]);
     }
 done:
@@ -909 +384 @@
         }
     }
+    cpu_fprintf(f, "EFER=%016" PRIx64 "\n", env->efer);
     if (flags & X86_DUMP_FPU) {
         int fptag;
@@ -1026 +502 @@
         ((new_cr0 << (HF_MP_SHIFT - 1)) & (HF_MP_MASK | HF_EM_MASK | HF_TS_MASK));
 #ifdef VBOX
-
     remR3ChangeCpuMode(env);
 #endif
@@ -1081 +556 @@
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUState *env, target_ulong addr)
-{
-    return addr;
-}
-
 #else
 
@@ -1100 +570 @@
    0  = nothing more to do
    1  = generate PF fault
-   2  = soft MMU activation required for this block
 */
 int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
@@ -1107 +576 @@
     uint64_t ptep, pte;
     target_ulong pde_addr, pte_addr;
-    int error_code, is_dirty, prot, page_size, ret, is_write, is_user;
+    int error_code, is_dirty, prot, page_size, is_write, is_user;
     target_phys_addr_t paddr;
     uint32_t page_offset;
@@ -1368 +837 @@
     vaddr = virt_addr + page_offset;
 
-    ret = tlb_set_page_exec(env, vaddr, paddr, prot, mmu_idx, is_softmmu);
-    return ret;
+    tlb_set_page(env, vaddr, paddr, prot, mmu_idx, page_size);
+    return 0;
  do_fault_protect:
     error_code = PG_ERROR_P_MASK;
@@ -1557 +1026 @@
 static CPUDebugExcpHandler *prev_debug_excp_handler;
 
-void raise_exception(int exception_index);
+void raise_exception_env(int exception_index, CPUState *env);
 
 static void breakpoint_handler(CPUState *env)
@@ -1567 +1036 @@
             env->watchpoint_hit = NULL;
             if (check_hw_breakpoints(env, 0))
-                raise_exception(EXCP01_DB);
+                raise_exception_env(EXCP01_DB, env);
             else
                 cpu_resume_from_signal(env, NULL);
@@ -1576 +1045 @@
                 if (bp->flags & BP_CPU) {
                     check_hw_breakpoints(env, 1);
-                    raise_exception(EXCP01_DB);
+                    raise_exception_env(EXCP01_DB, env);
                 }
                 break;
@@ -1584 +1053 @@
         prev_debug_excp_handler(env);
 }
-
 
 #ifndef VBOX
@@ -1659 +1127 @@
 }
 
-static void host_cpuid(uint32_t function, uint32_t count,
-                       uint32_t *eax, uint32_t *ebx,
-                       uint32_t *ecx, uint32_t *edx)
-{
-#if defined(CONFIG_KVM)
-    uint32_t vec[4];
-
-#ifdef __x86_64__
-    asm volatile("cpuid"
-                 : "=a"(vec[0]), "=b"(vec[1]),
-                   "=c"(vec[2]), "=d"(vec[3])
-                 : "0"(function), "c"(count) : "cc");
-#else
-    asm volatile("pusha \n\t"
-                 "cpuid \n\t"
-                 "mov %%eax, 0(%2) \n\t"
-                 "mov %%ebx, 4(%2) \n\t"
-                 "mov %%ecx, 8(%2) \n\t"
-                 "mov %%edx, 12(%2) \n\t"
-                 "popa"
-                 : : "a"(function), "c"(count), "S"(vec)
-                 : "memory", "cc");
-#endif
-
-    if (eax)
-        *eax = vec[0];
-    if (ebx)
-        *ebx = vec[1];
-    if (ecx)
-        *ecx = vec[2];
-    if (edx)
-        *edx = vec[3];
-#endif
-}
-
-static void get_cpuid_vendor(CPUX86State *env, uint32_t *ebx,
-                             uint32_t *ecx, uint32_t *edx)
-{
-    *ebx = env->cpuid_vendor1;
-    *edx = env->cpuid_vendor2;
-    *ecx = env->cpuid_vendor3;
-
-    /* sysenter isn't supported on compatibility mode on AMD, syscall
-     * isn't supported in compatibility mode on Intel.
-     * Normally we advertise the actual cpu vendor, but you can override
-     * this if you want to use KVM's sysenter/syscall emulation
-     * in compatibility mode and when doing cross vendor migration
-     */
-    if (kvm_enabled() && env->cpuid_vendor_override) {
-        host_cpuid(0, 0, NULL, ebx, ecx, edx);
-    }
-}
-
-void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
-                   uint32_t *eax, uint32_t *ebx,
-                   uint32_t *ecx, uint32_t *edx)
-{
-    /* test if maximum index reached */
-    if (index & 0x80000000) {
-        if (index > env->cpuid_xlevel)
-            index = env->cpuid_level;
-    } else {
-        if (index > env->cpuid_level)
-            index = env->cpuid_level;
-    }
-
-    switch(index) {
-    case 0:
-        *eax = env->cpuid_level;
-        get_cpuid_vendor(env, ebx, ecx, edx);
-        break;
-    case 1:
-        *eax = env->cpuid_version;
-        *ebx = (env->cpuid_apic_id << 24) | 8 << 8; /* CLFLUSH size in quad words, Linux wants it. */
-        *ecx = env->cpuid_ext_features;
-        *edx = env->cpuid_features;
-        if (env->nr_cores * env->nr_threads > 1) {
-            *ebx |= (env->nr_cores * env->nr_threads) << 16;
-            *edx |= 1 << 28;    /* HTT bit */
-        }
-        break;
-    case 2:
-        /* cache info: needed for Pentium Pro compatibility */
-        *eax = 1;
-        *ebx = 0;
-        *ecx = 0;
-        *edx = 0x2c307d;
-        break;
-    case 4:
-        /* cache info: needed for Core compatibility */
-        if (env->nr_cores > 1) {
-                *eax = (env->nr_cores - 1) << 26;
-        } else {
-                *eax = 0;
-        }
-        switch (count) {
-            case 0: /* L1 dcache info */
-                *eax |= 0x0000121;
-                *ebx = 0x1c0003f;
-                *ecx = 0x000003f;
-                *edx = 0x0000001;
-                break;
-            case 1: /* L1 icache info */
-                *eax |= 0x0000122;
-                *ebx = 0x1c0003f;
-                *ecx = 0x000003f;
-                *edx = 0x0000001;
-                break;
-            case 2: /* L2 cache info */
-                *eax |= 0x0000143;
-                if (env->nr_threads > 1) {
-                    *eax |= (env->nr_threads - 1) << 14;
-                }
-                *ebx = 0x3c0003f;
-                *ecx = 0x0000fff;
-                *edx = 0x0000001;
-                break;
-            default: /* end of info */
-                *eax = 0;
-                *ebx = 0;
-                *ecx = 0;
-                *edx = 0;
-                break;
-        }
-        break;
-    case 5:
-        /* mwait info: needed for Core compatibility */
-        *eax = 0; /* Smallest monitor-line size in bytes */
-        *ebx = 0; /* Largest monitor-line size in bytes */
-        *ecx = CPUID_MWAIT_EMX | CPUID_MWAIT_IBE;
-        *edx = 0;
-        break;
-    case 6:
-        /* Thermal and Power Leaf */
-        *eax = 0;
-        *ebx = 0;
-        *ecx = 0;
-        *edx = 0;
-        break;
-    case 9:
-        /* Direct Cache Access Information Leaf */
-        *eax = 0; /* Bits 0-31 in DCA_CAP MSR */
-        *ebx = 0;
-        *ecx = 0;
-        *edx = 0;
-        break;
-    case 0xA:
-        /* Architectural Performance Monitoring Leaf */
-        *eax = 0;
-        *ebx = 0;
-        *ecx = 0;
-        *edx = 0;
-        break;
-    case 0x80000000:
-        *eax = env->cpuid_xlevel;
-        *ebx = env->cpuid_vendor1;
-        *edx = env->cpuid_vendor2;
-        *ecx = env->cpuid_vendor3;
-        break;
-    case 0x80000001:
-        *eax = env->cpuid_version;
-        *ebx = 0;
-        *ecx = env->cpuid_ext3_features;
-        *edx = env->cpuid_ext2_features;
-
-        /* The Linux kernel checks for the CMPLegacy bit and
-         * discards multiple thread information if it is set.
-         * So dont set it here for Intel to make Linux guests happy.
-         */
-        if (env->nr_cores * env->nr_threads > 1) {
-            uint32_t tebx, tecx, tedx;
-            get_cpuid_vendor(env, &tebx, &tecx, &tedx);
-            if (tebx != CPUID_VENDOR_INTEL_1 ||
-                tedx != CPUID_VENDOR_INTEL_2 ||
-                tecx != CPUID_VENDOR_INTEL_3) {
-                *ecx |= 1 << 1;    /* CmpLegacy bit */
-            }
-        }
-
-        if (kvm_enabled()) {
-            /* Nested SVM not yet supported in upstream QEMU */
-            *ecx &= ~CPUID_EXT3_SVM;
-        }
-        break;
-    case 0x80000002:
-    case 0x80000003:
-    case 0x80000004:
-        *eax = env->cpuid_model[(index - 0x80000002) * 4 + 0];
-        *ebx = env->cpuid_model[(index - 0x80000002) * 4 + 1];
-        *ecx = env->cpuid_model[(index - 0x80000002) * 4 + 2];
-        *edx = env->cpuid_model[(index - 0x80000002) * 4 + 3];
-        break;
-    case 0x80000005:
-        /* cache info (L1 cache) */
-        *eax = 0x01ff01ff;
-        *ebx = 0x01ff01ff;
-        *ecx = 0x40020140;
-        *edx = 0x40020140;
-        break;
-    case 0x80000006:
-        /* cache info (L2 cache) */
-        *eax = 0;
-        *ebx = 0x42004200;
-        *ecx = 0x02008140;
-        *edx = 0;
-        break;
-    case 0x80000008:
-        /* virtual & phys address size in low 2 bytes. */
-/* XXX: This value must match the one used in the MMU code. */
-        if (env->cpuid_ext2_features & CPUID_EXT2_LM) {
-            /* 64 bit processor */
-/* XXX: The physical address space is limited to 42 bits in exec.c. */
-            *eax = 0x00003028;  /* 48 bits virtual, 40 bits physical */
-        } else {
-            if (env->cpuid_features & CPUID_PSE36)
-                *eax = 0x00000024; /* 36 bits physical */
-            else
-                *eax = 0x00000020; /* 32 bits physical */
-        }
-        *ebx = 0;
-        *ecx = 0;
-        *edx = 0;
-        if (env->nr_cores * env->nr_threads > 1) {
-            *ecx |= (env->nr_cores * env->nr_threads) - 1;
-        }
-        break;
-    case 0x8000000A:
-        *eax = 0x00000001; /* SVM Revision */
-        *ebx = 0x00000010; /* nr of ASIDs */
-        *ecx = 0;
-        *edx = 0; /* optional features */
-        break;
-    default:
-        /* reserved values: zero */
-        *eax = 0;
-        *ebx = 0;
-        *ecx = 0;
-        *edx = 0;
-        break;
-    }
-}
-
-
 int cpu_x86_get_descr_debug(CPUX86State *env, unsigned int selector,
                             target_ulong *base, unsigned int *limit,
@@ -1959 +1184 @@
 #endif
     }
+#ifndef VBOX
     if (cpu_x86_register(env, cpu_model) < 0) {
         cpu_x86_close(env);
         return NULL;
     }
-#ifndef VBOX
     mce_init(env);
 #endif
@@ -1979 +1204 @@
     cpu_reset(env);
     env->interrupt_request = sipi;
-    apic_init_reset(env);
+    apic_init_reset(env->apic_state);
+    env->halted = !cpu_is_bsp(env);
 }
 
 void do_cpu_sipi(CPUState *env)
 {
-    apic_sipi(env);
+    apic_sipi(env->apic_state);
 }
 #else

trunk/src/recompiler/target-i386/op_helper.c

@@ -27 +27 @@
  */
 
-#define CPU_NO_GLOBAL_REGS
 #include "exec.h"
 #include "exec-all.h"
 #include "host-utils.h"
+#include "ioport.h"
 
 #ifdef VBOX
@@ -37 +37 @@
 # include "tcg.h"
 #endif /* VBOX */
+
 //#define DEBUG_PCALL
 
@@ -370 +371 @@
             raise_exception_err(EXCP0A_TSS, selector & 0xfffc);
 #ifdef VBOX
-# if 0
-        /** @todo: now we ignore loading 0 selectors, need to check what is correct once */
+# if 0  /** @todo now we ignore loading 0 selectors, need to check what is correct once */
         cpu_x86_load_seg_cache(env, seg_reg, selector,
                                0, 0, 0);
@@ -1760 +1760 @@
 }
 
+void raise_exception_env(int exception_index, CPUState *nenv)
+{
+    env = nenv;
+    raise_exception(exception_index);
+}
 /* SMM support */
 
@@ -3445 +3450 @@
     case 8:
         if (!(env->hflags2 & HF2_VINTR_MASK)) {
+#ifndef VBOX
+            val = cpu_get_apic_tpr(env->apic_state);
+#else  /* VBOX */
             val = cpu_get_apic_tpr(env);
+#endif /* VBOX */
         } else {
             val = env->v_tpr;
@@ -3469 +3478 @@
     case 8:
         if (!(env->hflags2 & HF2_VINTR_MASK)) {
+#ifndef VBOX
+            cpu_set_apic_tpr(env->apic_state, t0);
+#else  /* VBOX */
             cpu_set_apic_tpr(env, t0);
+#endif /* VBOX */
         }
         env->v_tpr = t0 & 0x0f;
@@ -3534 +3547 @@
 void helper_rdtscp(void)
 {
+    helper_rdtsc();
 #ifndef VBOX
-    helper_rdtsc();
     ECX = (uint32_t)(env->tsc_aux);
-#else
+#else  /* VBOX */
     uint64_t val;
-    if ((env->cr[4] & CR4_TSD_MASK) && ((env->hflags & HF_CPL_MASK) != 0)) {
-        raise_exception(EXCP0D_GPF);
-    }
-
-    val = cpu_get_tsc(env);
-    EAX = (uint32_t)(val);
-    EDX = (uint32_t)(val >> 32);
     if (cpu_rdmsr(env, MSR_K8_TSC_AUX, &val) == 0)
         ECX = (uint32_t)(val);
@@ -3603 +3609 @@
     case MSR_IA32_APICBASE:
 # ifndef VBOX /* The CPUMSetGuestMsr call below does this now. */
-        cpu_set_apic_base(env, val);
+        cpu_set_apic_base(env->apic_state, val);
 # endif
         break;
@@ -3749 +3755 @@
         break;
     case MSR_IA32_APICBASE:
+#ifndef VBOX
+        val = cpu_get_apic_base(env->apic_state);
+#else  /* VBOX */
         val = cpu_get_apic_base(env);
+#endif /* VBOX */
         break;
     case MSR_EFER:
@@ -6643 +6653 @@
     stl_phys(env->vm_vmcb + offsetof(struct vmcb, control.exit_int_info_err),
              ldl_phys(env->vm_vmcb + offsetof(struct vmcb, control.event_inj_err)));
+    stl_phys(env->vm_vmcb + offsetof(struct vmcb, control.event_inj), 0);
 
     env->hflags2 &= ~HF2_GIF_MASK;

trunk/src/recompiler/target-i386/ops_sse_header.h

@@ -41 +41 @@
 #define dh_ctype_XMMReg XMMReg *
 #define dh_ctype_MMXReg MMXReg *
+#define dh_is_signed_Reg dh_is_signed_ptr
+#define dh_is_signed_XMMReg dh_is_signed_ptr
+#define dh_is_signed_MMXReg dh_is_signed_ptr
 
 DEF_HELPER_2(glue(psrlw, SUFFIX), void, Reg, Reg)

trunk/src/recompiler/target-i386/translate.c

@@ -85 +85 @@
 static TCGv cpu_tmp5;
 
+static uint8_t gen_opc_cc_op[OPC_BUF_SIZE];
+
 #include "gen-icount.h"
 
@@ -752 +754 @@
 }
 
-# if 0 /* unused code? */
-static void gen_check_external_event2()
-{
-    gen_helper_check_external_event();
-}
-# endif
-
 #endif /* VBOX */
 
@@ -931 +926 @@
                 gen_op_set_cc_op(s->cc_op);
             gen_jmp_im(cur_eip);
-            state_saved = 1;
         }
         svm_flags |= (1 << (4 + ot));
@@ -2381 +2375 @@
         override = s->override;
         must_add_seg = 1;
-    } else {
-        override = R_DS;
     }
     if (must_add_seg) {
@@ -2488 +2480 @@
 
     cc_op = s->cc_op;
-    if (s->cc_op != CC_OP_DYNAMIC) {
-        gen_op_set_cc_op(s->cc_op);
-        s->cc_op = CC_OP_DYNAMIC;
-    }
+    gen_update_cc_op(s);
     if (s->jmp_opt) {
         l1 = gen_new_label();
@@ -2500 +2489 @@
         gen_set_label(l1);
         gen_goto_tb(s, 1, val);
-        s->is_jmp = 3;
+        s->is_jmp = DISAS_TB_JUMP;
     } else {
 
@@ -2589 +2578 @@
            interrupts for the next instruction */
         if (seg_reg == R_SS || (s->code32 && seg_reg < R_FS))
-            s->is_jmp = 3;
+            s->is_jmp = DISAS_TB_JUMP;
     } else {
         gen_op_movl_seg_T0_vm(seg_reg);
         if (seg_reg == R_SS)
-            s->is_jmp = 3;
+            s->is_jmp = DISAS_TB_JUMP;
     }
 }
@@ -2861 +2850 @@
     gen_jmp_im(cur_eip);
     gen_helper_raise_exception(tcg_const_i32(trapno));
-    s->is_jmp = 3;
+    s->is_jmp = DISAS_TB_JUMP;
 }
 
@@ -2874 +2863 @@
     gen_helper_raise_interrupt(tcg_const_i32(intno),
                                tcg_const_i32(next_eip - cur_eip));
-    s->is_jmp = 3;
+    s->is_jmp = DISAS_TB_JUMP;
 }
 
@@ -2883 +2872 @@
     gen_jmp_im(cur_eip);
     gen_helper_debug();
-    s->is_jmp = 3;
+    s->is_jmp = DISAS_TB_JUMP;
 }
 
@@ -2910 +2899 @@
         tcg_gen_exit_tb(0);
     }
-    s->is_jmp = 3;
+    s->is_jmp = DISAS_TB_JUMP;
 }
 
@@ -2918 +2907 @@
 {
     if (s->jmp_opt) {
-        if (s->cc_op != CC_OP_DYNAMIC) {
-            gen_op_set_cc_op(s->cc_op);
-            s->cc_op = CC_OP_DYNAMIC;
-        }
+        gen_update_cc_op(s);
         gen_goto_tb(s, tb_num, eip);
-        s->is_jmp = 3;
+        s->is_jmp = DISAS_TB_JUMP;
     } else {
         gen_jmp_im(eip);
@@ -3666 +3652 @@
         case 0x172:
         case 0x173:
+            if (b1 >= 2) {
+                goto illegal_op;
+            }
             val = ldub_code(s->pc++);
             if (is_xmm) {
@@ -3893 +3882 @@
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
+            if (b1 >= 2) {
+                goto illegal_op;
+            }
 
             sse_op2 = sse_op_table6[b].op[b1];
@@ -3992 +3984 @@
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
+            if (b1 >= 2) {
+                goto illegal_op;
+            }
 
             sse_op2 = sse_op_table7[b].op[b1];
@@ -5055 +5050 @@
 
         modrm = ldub_code(s->pc++);
-        mod = (modrm >> 6) & 3;
-        rm = (modrm & 7) | REX_B(s);
         reg = ((modrm >> 3) & 7) | rex_r;
 
@@ -7304 +7297 @@
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
-            if (s->cc_op != CC_OP_DYNAMIC) {
-                gen_op_set_cc_op(s->cc_op);
-                s->cc_op = CC_OP_DYNAMIC;
-            }
+            gen_update_cc_op(s);
             gen_jmp_im(pc_start - s->cs_base);
             gen_helper_sysenter();
@@ -7325 +7315 @@
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
-            if (s->cc_op != CC_OP_DYNAMIC) {
-                gen_op_set_cc_op(s->cc_op);
-                s->cc_op = CC_OP_DYNAMIC;
-            }
+            gen_update_cc_op(s);
             gen_jmp_im(pc_start - s->cs_base);
             gen_helper_sysexit(tcg_const_i32(dflag));
@@ -7337 +7324 @@
     case 0x105: /* syscall */
         /* XXX: is it usable in real mode ? */
-        if (s->cc_op != CC_OP_DYNAMIC) {
-            gen_op_set_cc_op(s->cc_op);
-            s->cc_op = CC_OP_DYNAMIC;
-        }
+        gen_update_cc_op(s);
         gen_jmp_im(pc_start - s->cs_base);
         gen_helper_syscall(tcg_const_i32(s->pc - pc_start));
@@ -7349 +7333 @@
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
-            if (s->cc_op != CC_OP_DYNAMIC) {
-                gen_op_set_cc_op(s->cc_op);
-                s->cc_op = CC_OP_DYNAMIC;
-            }
+            gen_update_cc_op(s);
             gen_jmp_im(pc_start - s->cs_base);
             gen_helper_sysret(tcg_const_i32(s->dflag));
@@ -7376 +7357 @@
             gen_jmp_im(pc_start - s->cs_base);
             gen_helper_hlt(tcg_const_i32(s->pc - pc_start));
-            s->is_jmp = 3;
+            s->is_jmp = DISAS_TB_JUMP;
         }
         break;
@@ -7452 +7433 @@
         op = (modrm >> 3) & 7;
         rm = modrm & 7;
-#ifdef VBOX
-        /* 0f 01 f9 */
-        if (modrm == 0xf9)
-        {
-            if (!(s->cpuid_ext2_features & CPUID_EXT2_RDTSCP))
-                goto illegal_op;
-            gen_jmp_im(pc_start - s->cs_base);
-            gen_helper_rdtscp();
-            break;
-        }
-#endif /* VBOX */
         switch(op) {
         case 0: /* sgdt */
@@ -7504 +7474 @@
                         s->cpl != 0)
                         goto illegal_op;
-                    if (s->cc_op != CC_OP_DYNAMIC) {
-                        gen_op_set_cc_op(s->cc_op);
-                        s->cc_op = CC_OP_DYNAMIC;
-                    }
+                    gen_update_cc_op(s);
                     gen_jmp_im(pc_start - s->cs_base);
                     gen_helper_mwait(tcg_const_i32(s->pc - pc_start));
@@ -7544 +7511 @@
                                          tcg_const_i32(s->pc - pc_start));
                         tcg_gen_exit_tb(0);
-                        s->is_jmp = 3;
+                        s->is_jmp = DISAS_TB_JUMP;
                     }
                     break;
@@ -8036 +8003 @@
         if (!(s->flags & HF_SMM_MASK))
             goto illegal_op;
-        if (s->cc_op != CC_OP_DYNAMIC) {
-            gen_op_set_cc_op(s->cc_op);
-            s->cc_op = CC_OP_DYNAMIC;
-        }
+        gen_update_cc_op(s);
         gen_jmp_im(s->pc - s->cs_base);
         gen_helper_rsm();
@@ -8180 +8144 @@
     uint16_t *gen_opc_end;
     CPUBreakpoint *bp;
-    int j, lj, cflags;
+    int j, lj;
     uint64_t flags;
     target_ulong pc_start;
@@ -8194 +8158 @@
     cs_base = tb->cs_base;
     flags = tb->flags;
-    cflags = tb->cflags;
 
     dc->pe = (flags >> HF_PE_SHIFT) & 1;
@@ -8364 +8327 @@
 
 #ifdef DEBUG_DISAS
-    log_cpu_state_mask(CPU_LOG_TB_CPU, env, X86_DUMP_CCOP);
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         int disas_flags;

trunk/src/recompiler/targphys.h

@@ -6 +6 @@
 #ifdef TARGET_PHYS_ADDR_BITS
 /* target_phys_addr_t is the type of a physical address (its size can
-   be different from 'target_ulong'). We have sizeof(target_phys_addr)
-   = max(sizeof(unsigned long),
-   sizeof(size_of_target_physical_address)) because we must pass a
-   host pointer to memory operations in some cases */
+   be different from 'target_ulong').  */
 
 #if TARGET_PHYS_ADDR_BITS == 32

trunk/src/recompiler/tcg-runtime.c

@@ -29 +29 @@
 
 #include "tcg/tcg-runtime.h"
+
+/* 32-bit helpers */
+
+int32_t tcg_helper_div_i32(int32_t arg1, int32_t arg2)
+{
+    return arg1 / arg2;
+}
+
+int32_t tcg_helper_rem_i32(int32_t arg1, int32_t arg2)
+{
+    return arg1 % arg2;
+}
+
+uint32_t tcg_helper_divu_i32(uint32_t arg1, uint32_t arg2)
+{
+    return arg1 / arg2;
+}
+
+uint32_t tcg_helper_remu_i32(uint32_t arg1, uint32_t arg2)
+{
+    return arg1 % arg2;
+}
+
+/* 64-bit helpers */
 
 int64_t tcg_helper_shl_i64(int64_t arg1, int64_t arg2)

trunk/src/recompiler/tcg/README

@@ -5 +5 @@
 TCG (Tiny Code Generator) began as a generic backend for a C
 compiler. It was simplified to be used in QEMU. It also has its roots
-in the QOP code generator written by Paul Brook.
+in the QOP code generator written by Paul Brook. 
 
 2) Definitions
@@ -31 +31 @@
 
 A TCG "basic block" corresponds to a list of instructions terminated
-by a branch instruction.
+by a branch instruction. 
 
 3) Intermediate representation
@@ -76 +76 @@
 
 Using the tcg_gen_helper_x_y it is possible to call any function
-taking i32, i64 or pointer types. Before calling an helper, all
-globals are stored at their canonical location and it is assumed that
-the function can modify them. In the future, function modifiers will
-be allowed to tell that the helper does not read or write some globals.
+taking i32, i64 or pointer types. By default, before calling an helper,
+all globals are stored at their canonical location and it is assumed
+that the function can modify them. This can be overriden by the
+TCG_CALL_CONST function modifier. By default, the helper is allowed to
+modify the CPU state or raise an exception. This can be overriden by
+the TCG_CALL_PURE function modifier, in which case the call to the
+function is removed if the return value is not used.
 
 On some TCG targets (e.g. x86), several calling conventions are
@@ -97 +100 @@
 
    and_i32 t0, t0, $0xffffffff
-
+    
   is suppressed.
 
@@ -211 +214 @@
 * eqv_i32/i64 t0, t1, t2
 
-t0=~(t1^t2)
+t0=~(t1^t2), or equivalently, t0=t1^~t2
 
 * nand_i32/i64 t0, t1, t2
@@ -266 +269 @@
 * bswap16_i32/i64 t0, t1
 
-16 bit byte swap on a 32/64 bit value. The two/six high order bytes must be
-set to zero.
+16 bit byte swap on a 32/64 bit value. It assumes that the two/six high order
+bytes are set to zero.
 
 * bswap32_i32/i64 t0, t1
 
-32 bit byte swap on a 32/64 bit value. With a 64 bit value, the four high
-order bytes must be set to zero.
+32 bit byte swap on a 32/64 bit value. With a 64 bit value, it assumes that
+the four high order bytes are set to zero.
 
 * bswap64_i64 t0, t1
@@ -282 +285 @@
 Indicate that the value of t0 won't be used later. It is useful to
 force dead code elimination.
+
+********* Conditional moves
+
+* setcond_i32/i64 cond, dest, t1, t2
+
+dest = (t1 cond t2)
+
+Set DEST to 1 if (T1 cond T2) is true, otherwise set to 0.
 
 ********* Type conversions
@@ -313 +324 @@
 
 t0 = read(t1 + offset)
-Load 8, 16, 32 or 64 bits with or without sign extension from host memory.
+Load 8, 16, 32 or 64 bits with or without sign extension from host memory. 
 offset must be a constant.
 
@@ -324 +335 @@
 Write 8, 16, 32 or 64 bits to host memory.
 
+********* 64-bit target on 32-bit host support
+
+The following opcodes are internal to TCG.  Thus they are to be implemented by
+32-bit host code generators, but are not to be emitted by guest translators.
+They are emitted as needed by inline functions within "tcg-op.h".
+
+* brcond2_i32 cond, t0_low, t0_high, t1_low, t1_high, label
+
+Similar to brcond, except that the 64-bit values T0 and T1
+are formed from two 32-bit arguments.
+
+* add2_i32 t0_low, t0_high, t1_low, t1_high, t2_low, t2_high
+* sub2_i32 t0_low, t0_high, t1_low, t1_high, t2_low, t2_high
+
+Similar to add/sub, except that the 64-bit inputs T1 and T2 are
+formed from two 32-bit arguments, and the 64-bit output T0
+is returned in two 32-bit outputs.
+
+* mulu2_i32 t0_low, t0_high, t1, t2
+
+Similar to mul, except two 32-bit (unsigned) inputs T1 and T2 yielding
+the full 64-bit product T0.  The later is returned in two 32-bit outputs.
+
+* setcond2_i32 cond, dest, t1_low, t1_high, t2_low, t2_high
+
+Similar to setcond, except that the 64-bit values T1 and T2 are
+formed from two 32-bit arguments.  The result is a 32-bit value.
+
 ********* QEMU specific operations
 
@@ -340 +379 @@
 qemu_ld16u t0, t1, flags
 qemu_ld16s t0, t1, flags
+qemu_ld32 t0, t1, flags
 qemu_ld32u t0, t1, flags
 qemu_ld32s t0, t1, flags
 qemu_ld64 t0, t1, flags
 
-Load data at the QEMU CPU address t1 into t0. t1 has the QEMU CPU
-address type. 'flags' contains the QEMU memory index (selects user or
-kernel access) for example.
+Load data at the QEMU CPU address t1 into t0. t1 has the QEMU CPU address
+type. 'flags' contains the QEMU memory index (selects user or kernel access)
+for example.
+
+Note that "qemu_ld32" implies a 32-bit result, while "qemu_ld32u" and
+"qemu_ld32s" imply a 64-bit result appropriately extended from 32 bits.
 
 * qemu_st8 t0, t1, flags
@@ -419 +462 @@
 - The first N parameters are passed in registers.
 - The next parameters are passed on the stack by storing them as words.
-- Some registers are clobbered during the call.
+- Some registers are clobbered during the call. 
 - The function can return 0 or 1 value in registers. On a 32 bit
   target, functions must be able to return 2 values in registers for

trunk/src/recompiler/tcg/TODO

@@ -1 @@
-- Add new instructions such as: setcond, clz, ctz, popcnt.
+- Add new instructions such as: clz, ctz, popcnt.
 
-- See if it is worth exporting mul2, mulu2, div2, divu2.
+- See if it is worth exporting mul2, mulu2, div2, divu2. 
 
 - Support of globals saved in fixed registers between TBs.

trunk/src/recompiler/tcg/i386/tcg-target.c

@@ -25 +25 @@
 #ifndef NDEBUG
 static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
-    "%eax",
-    "%ecx",
-    "%edx",
-    "%ebx",
-    "%esp",
-    "%ebp",
-    "%esi",
-    "%edi",
+#if TCG_TARGET_REG_BITS == 64
+    "%rax", "%rcx", "%rdx", "%rbx", "%rsp", "%rbp", "%rsi", "%rdi",
+    "%r8",  "%r9",  "%r10", "%r11", "%r12", "%r13", "%r14", "%r15",
+#else
+    "%eax", "%ecx", "%edx", "%ebx", "%esp", "%ebp", "%esi", "%edi",
+#endif
 };
 #endif
 
 static const int tcg_target_reg_alloc_order[] = {
-    TCG_REG_EAX,
-    TCG_REG_EDX,
-    TCG_REG_ECX,
+#if TCG_TARGET_REG_BITS == 64
+    TCG_REG_RBP,
+    TCG_REG_RBX,
+    TCG_REG_R12,
+    TCG_REG_R13,
+    TCG_REG_R14,
+    TCG_REG_R15,
+    TCG_REG_R10,
+    TCG_REG_R11,
+    TCG_REG_R9,
+    TCG_REG_R8,
+    TCG_REG_RCX,
+    TCG_REG_RDX,
+    TCG_REG_RSI,
+    TCG_REG_RDI,
+    TCG_REG_RAX,
+#else
     TCG_REG_EBX,
     TCG_REG_ESI,
     TCG_REG_EDI,
     TCG_REG_EBP,
+    TCG_REG_ECX,
+    TCG_REG_EDX,
+    TCG_REG_EAX,
+#endif
 };
 
-static const int tcg_target_call_iarg_regs[3] = { TCG_REG_EAX, TCG_REG_EDX, TCG_REG_ECX };
-static const int tcg_target_call_oarg_regs[2] = { TCG_REG_EAX, TCG_REG_EDX };
+static const int tcg_target_call_iarg_regs[] = {
+#if TCG_TARGET_REG_BITS == 64
+    TCG_REG_RDI,
+    TCG_REG_RSI,
+    TCG_REG_RDX,
+    TCG_REG_RCX,
+    TCG_REG_R8,
+    TCG_REG_R9,
+#else
+    TCG_REG_EAX,
+    TCG_REG_EDX,
+    TCG_REG_ECX
+#endif
+};
+
+static const int tcg_target_call_oarg_regs[2] = {
+    TCG_REG_EAX,
+    TCG_REG_EDX
+};
 
 static uint8_t *tb_ret_addr;
@@ -56 +89 @@
     value += addend;
     switch(type) {
-    case R_386_32:
+    case R_386_PC32:
+        value -= (uintptr_t)code_ptr;
+        if (value != (int32_t)value) {
+            tcg_abort();
+        }
         *(uint32_t *)code_ptr = value;
         break;
-    case R_386_PC32:
-        *(uint32_t *)code_ptr = value - (long)code_ptr;
+    case R_386_PC8:
+        value -= (uintptr_t)code_ptr;
+        if (value != (int8_t)value) {
+            tcg_abort();
+        }
+        *(uint8_t *)code_ptr = value;
         break;
     default:
@@ -86 +127 @@
 static inline int tcg_target_get_call_iarg_regs_count(int flags)
 {
+    if (TCG_TARGET_REG_BITS == 64) {
+        return 6;
+    }
+
     flags &= TCG_CALL_TYPE_MASK;
     switch(flags) {
@@ -132 +177 @@
     case 'q':
         ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, 0xf);
+        if (TCG_TARGET_REG_BITS == 64) {
+            tcg_regset_set32(ct->u.regs, 0, 0xffff);
+        } else {
+            tcg_regset_set32(ct->u.regs, 0, 0xf);
+        }
         break;
     case 'r':
         ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, 0xff);
+        if (TCG_TARGET_REG_BITS == 64) {
+            tcg_regset_set32(ct->u.regs, 0, 0xffff);
+        } else {
+            tcg_regset_set32(ct->u.regs, 0, 0xff);
+        }
         break;
 
@@ -142 +195 @@
     case 'L':
         ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, 0xff);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_EAX);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_EDX);
-        break;
+        if (TCG_TARGET_REG_BITS == 64) {
+            tcg_regset_set32(ct->u.regs, 0, 0xffff);
+            tcg_regset_reset_reg(ct->u.regs, TCG_REG_RSI);
+            tcg_regset_reset_reg(ct->u.regs, TCG_REG_RDI);
+        } else {
+            tcg_regset_set32(ct->u.regs, 0, 0xff);
+            tcg_regset_reset_reg(ct->u.regs, TCG_REG_EAX);
+            tcg_regset_reset_reg(ct->u.regs, TCG_REG_EDX);
+        }
+        break;
+
+    case 'e':
+        ct->ct |= TCG_CT_CONST_S32;
+        break;
+    case 'Z':
+        ct->ct |= TCG_CT_CONST_U32;
+        break;
+
     default:
         return -1;
@@ -158 +225 @@
                                          const TCGArgConstraint *arg_ct)
 {
-    int ct;
-    ct = arg_ct->ct;
-    if (ct & TCG_CT_CONST)
+    int ct = arg_ct->ct;
+    if (ct & TCG_CT_CONST) {
         return 1;
-    else
-        return 0;
-}
-
+    }
+    if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) {
+        return 1;
+    }
+    if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) {
+        return 1;
+    }
+    return 0;
+}
+
+#if TCG_TARGET_REG_BITS == 64
+# define LOWREGMASK(x)  ((x) & 7)
+#else
+# define LOWREGMASK(x)  (x)
+#endif
+
+#define P_EXT           0x100           /* 0x0f opcode prefix */
+#define P_DATA16        0x200           /* 0x66 opcode prefix */
+#if TCG_TARGET_REG_BITS == 64
+# define P_ADDR32       0x400           /* 0x67 opcode prefix */
+# define P_REXW         0x800           /* Set REX.W = 1 */
+# define P_REXB_R       0x1000          /* REG field as byte register */
+# define P_REXB_RM      0x2000          /* R/M field as byte register */
+#else
+# define P_ADDR32       0
+# define P_REXW         0
+# define P_REXB_R       0
+# define P_REXB_RM      0
+#endif
+
+#define OPC_ARITH_EvIz  (0x81)
+#define OPC_ARITH_EvIb  (0x83)
+#define OPC_ARITH_GvEv  (0x03)          /* ... plus (ARITH_FOO << 3) */
+#define OPC_ADD_GvEv    (OPC_ARITH_GvEv | (ARITH_ADD << 3))
+#define OPC_BSWAP       (0xc8 | P_EXT)
+#define OPC_CALL_Jz     (0xe8)
+#define OPC_CMP_GvEv    (OPC_ARITH_GvEv | (ARITH_CMP << 3))
+#define OPC_DEC_r32     (0x48)
+#define OPC_IMUL_GvEv   (0xaf | P_EXT)
+#define OPC_IMUL_GvEvIb (0x6b)
+#define OPC_IMUL_GvEvIz (0x69)
+#define OPC_INC_r32     (0x40)
+#define OPC_JCC_long    (0x80 | P_EXT)  /* ... plus condition code */
+#define OPC_JCC_short   (0x70)          /* ... plus condition code */
+#define OPC_JMP_long    (0xe9)
+#define OPC_JMP_short   (0xeb)
+#define OPC_LEA         (0x8d)
+#define OPC_MOVB_EvGv   (0x88)          /* stores, more or less */
+#define OPC_MOVL_EvGv   (0x89)          /* stores, more or less */
+#define OPC_MOVL_GvEv   (0x8b)          /* loads, more or less */
+#define OPC_MOVL_EvIz   (0xc7)
+#define OPC_MOVL_Iv     (0xb8)
+#define OPC_MOVSBL      (0xbe | P_EXT)
+#define OPC_MOVSWL      (0xbf | P_EXT)
+#define OPC_MOVSLQ      (0x63 | P_REXW)
+#define OPC_MOVZBL      (0xb6 | P_EXT)
+#define OPC_MOVZWL      (0xb7 | P_EXT)
+#define OPC_POP_r32     (0x58)
+#define OPC_PUSH_r32    (0x50)
+#define OPC_PUSH_Iv     (0x68)
+#define OPC_PUSH_Ib     (0x6a)
+#define OPC_RET         (0xc3)
+#define OPC_SETCC       (0x90 | P_EXT | P_REXB_RM) /* ... plus cc */
+#define OPC_SHIFT_1     (0xd1)
+#define OPC_SHIFT_Ib    (0xc1)
+#define OPC_SHIFT_cl    (0xd3)
+#define OPC_TESTL       (0x85)
+#define OPC_XCHG_ax_r32 (0x90)
+
+#define OPC_GRP3_Ev     (0xf7)
+#define OPC_GRP5        (0xff)
+
+/* Group 1 opcode extensions for 0x80-0x83.
+   These are also used as modifiers for OPC_ARITH.  */
 #define ARITH_ADD 0
 #define ARITH_OR  1
@@ -175 +311 @@
 #define ARITH_CMP 7
 
+/* Group 2 opcode extensions for 0xc0, 0xc1, 0xd0-0xd3.  */
 #define SHIFT_ROL 0
 #define SHIFT_ROR 1
@@ -181 +318 @@
 #define SHIFT_SAR 7
 
+/* Group 3 opcode extensions for 0xf6, 0xf7.  To be used with OPC_GRP3.  */
+#define EXT3_NOT   2
+#define EXT3_NEG   3
+#define EXT3_MUL   4
+#define EXT3_IMUL  5
+#define EXT3_DIV   6
+#define EXT3_IDIV  7
+
+/* Group 5 opcode extensions for 0xff.  To be used with OPC_GRP5.  */
+#define EXT5_INC_Ev     0
+#define EXT5_DEC_Ev     1
+#define EXT5_CALLN_Ev   2
+#define EXT5_JMPN_Ev    4
+
+/* Condition codes to be added to OPC_JCC_{long,short}.  */
 #define JCC_JMP (-1)
 #define JCC_JO  0x0
@@ -199 +351 @@
 #define JCC_JG  0xf
 
-#define P_EXT   0x100 /* 0x0f opcode prefix */
-
 static const uint8_t tcg_cond_to_jcc[10] = {
     [TCG_COND_EQ] = JCC_JE,
@@ -214 +364 @@
 };
 
-static inline void tcg_out_opc(TCGContext *s, int opc)
-{
-    if (opc & P_EXT)
+#if TCG_TARGET_REG_BITS == 64
+static void tcg_out_opc(TCGContext *s, int opc, int r, int rm, int x)
+{
+    int rex;
+
+    if (opc & P_DATA16) {
+        /* We should never be asking for both 16 and 64-bit operation.  */
+        assert((opc & P_REXW) == 0);
+        tcg_out8(s, 0x66);
+    }
+    if (opc & P_ADDR32) {
+        tcg_out8(s, 0x67);
+    }
+
+    rex = 0;
+    rex |= (opc & P_REXW) >> 8;         /* REX.W */
+    rex |= (r & 8) >> 1;                /* REX.R */
+    rex |= (x & 8) >> 2;                /* REX.X */
+    rex |= (rm & 8) >> 3;               /* REX.B */
+
+    /* P_REXB_{R,RM} indicates that the given register is the low byte.
+       For %[abcd]l we need no REX prefix, but for %{si,di,bp,sp}l we do,
+       as otherwise the encoding indicates %[abcd]h.  Note that the values
+       that are ORed in merely indicate that the REX byte must be present;
+       those bits get discarded in output.  */
+    rex |= opc & (r >= 4 ? P_REXB_R : 0);
+    rex |= opc & (rm >= 4 ? P_REXB_RM : 0);
+
+    if (rex) {
+        tcg_out8(s, (uint8_t)(rex | 0x40));
+    }
+
+    if (opc & P_EXT) {
         tcg_out8(s, 0x0f);
+    }
     tcg_out8(s, opc);
 }
-
-static inline void tcg_out_modrm(TCGContext *s, int opc, int r, int rm)
-{
-    tcg_out_opc(s, opc);
-    tcg_out8(s, 0xc0 | (r << 3) | rm);
-}
-
-/* rm == -1 means no register index */
-static inline void tcg_out_modrm_offset(TCGContext *s, int opc, int r, int rm,
-                                        int32_t offset)
-{
-    tcg_out_opc(s, opc);
-    if (rm == -1) {
-        tcg_out8(s, 0x05 | (r << 3));
+#else
+static void tcg_out_opc(TCGContext *s, int opc)
+{
+    if (opc & P_DATA16) {
+        tcg_out8(s, 0x66);
+    }
+    if (opc & P_EXT) {
+        tcg_out8(s, 0x0f);
+    }
+    tcg_out8(s, opc);
+}
+/* Discard the register arguments to tcg_out_opc early, so as not to penalize
+   the 32-bit compilation paths.  This method works with all versions of gcc,
+   whereas relying on optimization may not be able to exclude them.  */
+#define tcg_out_opc(s, opc, r, rm, x)  (tcg_out_opc)(s, opc)
+#endif
+
+static void tcg_out_modrm(TCGContext *s, int opc, int r, int rm)
+{
+    tcg_out_opc(s, opc, r, rm, 0);
+    tcg_out8(s, 0xc0 | (LOWREGMASK(r) << 3) | LOWREGMASK(rm));
+}
+
+/* Output an opcode with a full "rm + (index<<shift) + offset" address mode.
+   We handle either RM and INDEX missing with a negative value.  In 64-bit
+   mode for absolute addresses, ~RM is the size of the immediate operand
+   that will follow the instruction.  */
+
+static void tcg_out_modrm_sib_offset(TCGContext *s, int opc, int r, int rm,
+                                     int index, int shift,
+                                     tcg_target_long offset)
+{
+    int mod, len;
+
+    if (index < 0 && rm < 0) {
+        if (TCG_TARGET_REG_BITS == 64) {
+            /* Try for a rip-relative addressing mode.  This has replaced
+               the 32-bit-mode absolute addressing encoding.  */
+            tcg_target_long pc = (tcg_target_long)s->code_ptr + 5 + ~rm;
+            tcg_target_long disp = offset - pc;
+            if (disp == (int32_t)disp) {
+                tcg_out_opc(s, opc, r, 0, 0);
+                tcg_out8(s, (LOWREGMASK(r) << 3) | 5);
+                tcg_out32(s, disp);
+                return;
+            }
+
+            /* Try for an absolute address encoding.  This requires the
+               use of the MODRM+SIB encoding and is therefore larger than
+               rip-relative addressing.  */
+            if (offset == (int32_t)offset) {
+                tcg_out_opc(s, opc, r, 0, 0);
+                tcg_out8(s, (LOWREGMASK(r) << 3) | 4);
+                tcg_out8(s, (4 << 3) | 5);
+                tcg_out32(s, offset);
+                return;
+            }
+
+            /* ??? The memory isn't directly addressable.  */
+            tcg_abort();
+        } else {
+            /* Absolute address.  */
+            tcg_out_opc(s, opc, r, 0, 0);
+            tcg_out8(s, (r << 3) | 5);
+            tcg_out32(s, offset);
+            return;
+        }
+    }
+
+    /* Find the length of the immediate addend.  Note that the encoding
+       that would be used for (%ebp) indicates absolute addressing.  */
+    if (rm < 0) {
+        mod = 0, len = 4, rm = 5;
+    } else if (offset == 0 && LOWREGMASK(rm) != TCG_REG_EBP) {
+        mod = 0, len = 0;
+    } else if (offset == (int8_t)offset) {
+        mod = 0x40, len = 1;
+    } else {
+        mod = 0x80, len = 4;
+    }
+
+    /* Use a single byte MODRM format if possible.  Note that the encoding
+       that would be used for %esp is the escape to the two byte form.  */
+    if (index < 0 && LOWREGMASK(rm) != TCG_REG_ESP) {
+        /* Single byte MODRM format.  */
+        tcg_out_opc(s, opc, r, rm, 0);
+        tcg_out8(s, mod | (LOWREGMASK(r) << 3) | LOWREGMASK(rm));
+    } else {
+        /* Two byte MODRM+SIB format.  */
+
+        /* Note that the encoding that would place %esp into the index
+           field indicates no index register.  In 64-bit mode, the REX.X
+           bit counts, so %r12 can be used as the index.  */
+        if (index < 0) {
+            index = 4;
+        } else {
+            assert(index != TCG_REG_ESP);
+        }
+
+        tcg_out_opc(s, opc, r, rm, index);
+        tcg_out8(s, mod | (LOWREGMASK(r) << 3) | 4);
+        tcg_out8(s, (shift << 6) | (LOWREGMASK(index) << 3) | LOWREGMASK(rm));
+    }
+
+    if (len == 1) {
+        tcg_out8(s, offset);
+    } else if (len == 4) {
         tcg_out32(s, offset);
-    } else if (offset == 0 && rm != TCG_REG_EBP) {
-        if (rm == TCG_REG_ESP) {
-            tcg_out8(s, 0x04 | (r << 3));
-            tcg_out8(s, 0x24);
-        } else {
-            tcg_out8(s, 0x00 | (r << 3) | rm);
-        }
-    } else if ((int8_t)offset == offset) {
-        if (rm == TCG_REG_ESP) {
-            tcg_out8(s, 0x44 | (r << 3));
-            tcg_out8(s, 0x24);
-        } else {
-            tcg_out8(s, 0x40 | (r << 3) | rm);
-        }
-        tcg_out8(s, offset);
+    }
+}
+
+/* A simplification of the above with no index or shift.  */
+static inline void tcg_out_modrm_offset(TCGContext *s, int opc, int r,
+                                        int rm, tcg_target_long offset)
+{
+    tcg_out_modrm_sib_offset(s, opc, r, rm, -1, 0, offset);
+}
+
+/* Generate dest op= src.  Uses the same ARITH_* codes as tgen_arithi.  */
+static inline void tgen_arithr(TCGContext *s, int subop, int dest, int src)
+{
+    /* Propagate an opcode prefix, such as P_REXW.  */
+    int ext = subop & ~0x7;
+    subop &= 0x7;
+
+    tcg_out_modrm(s, OPC_ARITH_GvEv + (subop << 3) + ext, dest, src);
+}
+
+static inline void tcg_out_mov(TCGContext *s, TCGType type, int ret, int arg)
+{
+    if (arg != ret) {
+        int opc = OPC_MOVL_GvEv + (type == TCG_TYPE_I64 ? P_REXW : 0);
+        tcg_out_modrm(s, opc, ret, arg);
+    }
+}
+
+static void tcg_out_movi(TCGContext *s, TCGType type,
+                         int ret, tcg_target_long arg)
+{
+    if (arg == 0) {
+        tgen_arithr(s, ARITH_XOR, ret, ret);
+        return;
+    } else if (arg == (uint32_t)arg || type == TCG_TYPE_I32) {
+        tcg_out_opc(s, OPC_MOVL_Iv + LOWREGMASK(ret), 0, ret, 0);
+        tcg_out32(s, arg);
+    } else if (arg == (int32_t)arg) {
+        tcg_out_modrm(s, OPC_MOVL_EvIz + P_REXW, 0, ret);
+        tcg_out32(s, arg);
     } else {
-        if (rm == TCG_REG_ESP) {
-            tcg_out8(s, 0x84 | (r << 3));
-            tcg_out8(s, 0x24);
-        } else {
-            tcg_out8(s, 0x80 | (r << 3) | rm);
-        }
-        tcg_out32(s, offset);
-    }
-}
-
-static inline void tcg_out_mov(TCGContext *s, int ret, int arg)
-{
-    if (arg != ret)
-        tcg_out_modrm(s, 0x8b, ret, arg);
-}
-
-static inline void tcg_out_movi(TCGContext *s, TCGType type,
-                                int ret, int32_t arg)
-{
-    if (arg == 0) {
-        /* xor r0,r0 */
-        tcg_out_modrm(s, 0x01 | (ARITH_XOR << 3), ret, ret);
+        tcg_out_opc(s, OPC_MOVL_Iv + P_REXW + LOWREGMASK(ret), 0, ret, 0);
+        tcg_out32(s, arg);
+        tcg_out32(s, arg >> 31 >> 1);
+    }
+}
+
+static inline void tcg_out_pushi(TCGContext *s, tcg_target_long val)
+{
+    if (val == (int8_t)val) {
+        tcg_out_opc(s, OPC_PUSH_Ib, 0, 0, 0);
+        tcg_out8(s, val);
+    } else if (val == (int32_t)val) {
+        tcg_out_opc(s, OPC_PUSH_Iv, 0, 0, 0);
+        tcg_out32(s, val);
     } else {
-        tcg_out8(s, 0xb8 + ret);
-        tcg_out32(s, arg);
-    }
+        tcg_abort();
+    }
+}
+
+static inline void tcg_out_push(TCGContext *s, int reg)
+{
+    tcg_out_opc(s, OPC_PUSH_r32 + LOWREGMASK(reg), 0, reg, 0);
+}
+
+static inline void tcg_out_pop(TCGContext *s, int reg)
+{
+    tcg_out_opc(s, OPC_POP_r32 + LOWREGMASK(reg), 0, reg, 0);
 }
 
@@ -282 +582 @@
                               int arg1, tcg_target_long arg2)
 {
-    /* movl */
-    tcg_out_modrm_offset(s, 0x8b, ret, arg1, arg2);
+    int opc = OPC_MOVL_GvEv + (type == TCG_TYPE_I64 ? P_REXW : 0);
+    tcg_out_modrm_offset(s, opc, ret, arg1, arg2);
 }
 
@@ -289 +589 @@
                               int arg1, tcg_target_long arg2)
 {
-    /* movl */
-    tcg_out_modrm_offset(s, 0x89, arg, arg1, arg2);
-}
-
-static inline void tgen_arithi(TCGContext *s, int c, int r0, int32_t val, int cf)
-{
-    if (!cf && ((c == ARITH_ADD && val == 1) || (c == ARITH_SUB && val == -1))) {
-        /* inc */
-        tcg_out_opc(s, 0x40 + r0);
-    } else if (!cf && ((c == ARITH_ADD && val == -1) || (c == ARITH_SUB && val == 1))) {
-        /* dec */
-        tcg_out_opc(s, 0x48 + r0);
-    } else if (val == (int8_t)val) {
-        tcg_out_modrm(s, 0x83, c, r0);
+    int opc = OPC_MOVL_EvGv + (type == TCG_TYPE_I64 ? P_REXW : 0);
+    tcg_out_modrm_offset(s, opc, arg, arg1, arg2);
+}
+
+static void tcg_out_shifti(TCGContext *s, int subopc, int reg, int count)
+{
+    /* Propagate an opcode prefix, such as P_DATA16.  */
+    int ext = subopc & ~0x7;
+    subopc &= 0x7;
+
+    if (count == 1) {
+        tcg_out_modrm(s, OPC_SHIFT_1 + ext, subopc, reg);
+    } else {
+        tcg_out_modrm(s, OPC_SHIFT_Ib + ext, subopc, reg);
+        tcg_out8(s, count);
+    }
+}
+
+static inline void tcg_out_bswap32(TCGContext *s, int reg)
+{
+    tcg_out_opc(s, OPC_BSWAP + LOWREGMASK(reg), 0, reg, 0);
+}
+
+static inline void tcg_out_rolw_8(TCGContext *s, int reg)
+{
+    tcg_out_shifti(s, SHIFT_ROL + P_DATA16, reg, 8);
+}
+
+static inline void tcg_out_ext8u(TCGContext *s, int dest, int src)
+{
+    /* movzbl */
+    assert(src < 4 || TCG_TARGET_REG_BITS == 64);
+    tcg_out_modrm(s, OPC_MOVZBL + P_REXB_RM, dest, src);
+}
+
+static void tcg_out_ext8s(TCGContext *s, int dest, int src, int rexw)
+{
+    /* movsbl */
+    assert(src < 4 || TCG_TARGET_REG_BITS == 64);
+    tcg_out_modrm(s, OPC_MOVSBL + P_REXB_RM + rexw, dest, src);
+}
+
+static inline void tcg_out_ext16u(TCGContext *s, int dest, int src)
+{
+    /* movzwl */
+    tcg_out_modrm(s, OPC_MOVZWL, dest, src);
+}
+
+static inline void tcg_out_ext16s(TCGContext *s, int dest, int src, int rexw)
+{
+    /* movsw[lq] */
+    tcg_out_modrm(s, OPC_MOVSWL + rexw, dest, src);
+}
+
+static inline void tcg_out_ext32u(TCGContext *s, int dest, int src)
+{
+    /* 32-bit mov zero extends.  */
+    tcg_out_modrm(s, OPC_MOVL_GvEv, dest, src);
+}
+
+static inline void tcg_out_ext32s(TCGContext *s, int dest, int src)
+{
+    tcg_out_modrm(s, OPC_MOVSLQ, dest, src);
+}
+
+static inline void tcg_out_bswap64(TCGContext *s, int reg)
+{
+    tcg_out_opc(s, OPC_BSWAP + P_REXW + LOWREGMASK(reg), 0, reg, 0);
+}
+
+static void tgen_arithi(TCGContext *s, int c, int r0,
+                        tcg_target_long val, int cf)
+{
+    int rexw = 0;
+
+    if (TCG_TARGET_REG_BITS == 64) {
+        rexw = c & -8;
+        c &= 7;
+    }
+
+    /* ??? While INC is 2 bytes shorter than ADDL $1, they also induce
+       partial flags update stalls on Pentium4 and are not recommended
+       by current Intel optimization manuals.  */
+    if (!cf && (c == ARITH_ADD || c == ARITH_SUB) && (val == 1 || val == -1)) {
+        int is_inc = (c == ARITH_ADD) ^ (val < 0);
+        if (TCG_TARGET_REG_BITS == 64) {
+            /* The single-byte increment encodings are re-tasked as the
+               REX prefixes.  Use the MODRM encoding.  */
+            tcg_out_modrm(s, OPC_GRP5 + rexw,
+                          (is_inc ? EXT5_INC_Ev : EXT5_DEC_Ev), r0);
+        } else {
+            tcg_out8(s, (is_inc ? OPC_INC_r32 : OPC_DEC_r32) + r0);
+        }
+        return;
+    }
+
+    if (c == ARITH_AND) {
+        if (TCG_TARGET_REG_BITS == 64) {
+            if (val == 0xffffffffu) {
+                tcg_out_ext32u(s, r0, r0);
+                return;
+            }
+            if (val == (uint32_t)val) {
+                /* AND with no high bits set can use a 32-bit operation.  */
+                rexw = 0;
+            }
+        }
+        if (val == 0xffu && (r0 < 4 || TCG_TARGET_REG_BITS == 64)) {
+            tcg_out_ext8u(s, r0, r0);
+            return;
+        }
+        if (val == 0xffffu) {
+            tcg_out_ext16u(s, r0, r0);
+            return;
+        }
+    }
+
+    if (val == (int8_t)val) {
+        tcg_out_modrm(s, OPC_ARITH_EvIb + rexw, c, r0);
         tcg_out8(s, val);
-    } else if (c == ARITH_AND && val == 0xffu && r0 < 4) {
-        /* movzbl */
-        tcg_out_modrm(s, 0xb6 | P_EXT, r0, r0);
-    } else if (c == ARITH_AND && val == 0xffffu) {
-        /* movzwl */
-        tcg_out_modrm(s, 0xb7 | P_EXT, r0, r0);
-    } else {
-        tcg_out_modrm(s, 0x81, c, r0);
+        return;
+    }
+    if (rexw == 0 || val == (int32_t)val) {
+        tcg_out_modrm(s, OPC_ARITH_EvIz + rexw, c, r0);
         tcg_out32(s, val);
-    }
+        return;
+    }
+
+    tcg_abort();
 }
 
 static void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val)
 {
-    if (val != 0)
-        tgen_arithi(s, ARITH_ADD, reg, val, 0);
+    if (val != 0) {
+        tgen_arithi(s, ARITH_ADD + P_REXW, reg, val, 0);
+    }
 }
 
@@ -325 +730 @@
 static void tcg_out_subi(TCGContext *s, int reg, tcg_target_long val)
 {
-    if (val != 0)
-        tgen_arithi(s, ARITH_SUB, reg, val, 0);
-}
-#endif
-
-static void tcg_out_jxx(TCGContext *s, int opc, int label_index)
+    if (val != 0) {
+        tgen_arithi(s, ARITH_SUB + P_REXW, reg, val, 0);
+    }
+}
+#endif
+
+/* Use SMALL != 0 to force a short forward branch.  */
+static void tcg_out_jxx(TCGContext *s, int opc, int label_index, int small)
 {
     int32_t val, val1;
@@ -339 +746 @@
         val1 = val - 2;
         if ((int8_t)val1 == val1) {
-            if (opc == -1)
-                tcg_out8(s, 0xeb);
-            else
-                tcg_out8(s, 0x70 + opc);
+            if (opc == -1) {
+                tcg_out8(s, OPC_JMP_short);
+            } else {
+                tcg_out8(s, OPC_JCC_short + opc);
+            }
             tcg_out8(s, val1);
         } else {
+            if (small) {
+                tcg_abort();
+            }
             if (opc == -1) {
-                tcg_out8(s, 0xe9);
+                tcg_out8(s, OPC_JMP_long);
                 tcg_out32(s, val - 5);
             } else {
-                tcg_out8(s, 0x0f);
-                tcg_out8(s, 0x80 + opc);
+                tcg_out_opc(s, OPC_JCC_long + opc, 0, 0, 0);
                 tcg_out32(s, val - 6);
             }
         }
+    } else if (small) {
+        if (opc == -1) {
+            tcg_out8(s, OPC_JMP_short);
+        } else {
+            tcg_out8(s, OPC_JCC_short + opc);
+        }
+        tcg_out_reloc(s, s->code_ptr, R_386_PC8, label_index, -1);
+        s->code_ptr += 1;
     } else {
         if (opc == -1) {
-            tcg_out8(s, 0xe9);
-        } else {
-            tcg_out8(s, 0x0f);
-            tcg_out8(s, 0x80 + opc);
+            tcg_out8(s, OPC_JMP_long);
+        } else {
+            tcg_out_opc(s, OPC_JCC_long + opc, 0, 0, 0);
         }
         tcg_out_reloc(s, s->code_ptr, R_386_PC32, label_index, -4);
@@ -366 +783 @@
 }
 
-static void tcg_out_brcond(TCGContext *s, int cond,
-                           TCGArg arg1, TCGArg arg2, int const_arg2,
-                           int label_index)
+static void tcg_out_cmp(TCGContext *s, TCGArg arg1, TCGArg arg2,
+                        int const_arg2, int rexw)
 {
     if (const_arg2) {
         if (arg2 == 0) {
             /* test r, r */
-            tcg_out_modrm(s, 0x85, arg1, arg1);
-        } else {
-            tgen_arithi(s, ARITH_CMP, arg1, arg2, 0);
+            tcg_out_modrm(s, OPC_TESTL + rexw, arg1, arg1);
+        } else {
+            tgen_arithi(s, ARITH_CMP + rexw, arg1, arg2, 0);
         }
     } else {
-        tcg_out_modrm(s, 0x01 | (ARITH_CMP << 3), arg2, arg1);
-    }
-    tcg_out_jxx(s, tcg_cond_to_jcc[cond], label_index);
-}
-
+        tgen_arithr(s, ARITH_CMP + rexw, arg1, arg2);
+    }
+}
+
+static void tcg_out_brcond32(TCGContext *s, TCGCond cond,
+                             TCGArg arg1, TCGArg arg2, int const_arg2,
+                             int label_index, int small)
+{
+    tcg_out_cmp(s, arg1, arg2, const_arg2, 0);
+    tcg_out_jxx(s, tcg_cond_to_jcc[cond], label_index, small);
+}
+
+#if TCG_TARGET_REG_BITS == 64
+static void tcg_out_brcond64(TCGContext *s, TCGCond cond,
+                             TCGArg arg1, TCGArg arg2, int const_arg2,
+                             int label_index, int small)
+{
+    tcg_out_cmp(s, arg1, arg2, const_arg2, P_REXW);
+    tcg_out_jxx(s, tcg_cond_to_jcc[cond], label_index, small);
+}
+#else
 /* XXX: we implement it at the target level to avoid having to
    handle cross basic blocks temporaries */
-static void tcg_out_brcond2(TCGContext *s,
-                            const TCGArg *args, const int *const_args)
+static void tcg_out_brcond2(TCGContext *s, const TCGArg *args,
+                            const int *const_args, int small)
 {
     int label_next;
@@ -392 +824 @@
     switch(args[4]) {
     case TCG_COND_EQ:
-        tcg_out_brcond(s, TCG_COND_NE, args[0], args[2], const_args[2], label_next);
-        tcg_out_brcond(s, TCG_COND_EQ, args[1], args[3], const_args[3], args[5]);
+        tcg_out_brcond32(s, TCG_COND_NE, args[0], args[2], const_args[2],
+                         label_next, 1);
+        tcg_out_brcond32(s, TCG_COND_EQ, args[1], args[3], const_args[3],
+                         args[5], small);
         break;
     case TCG_COND_NE:
-        tcg_out_brcond(s, TCG_COND_NE, args[0], args[2], const_args[2], args[5]);
-        tcg_out_brcond(s, TCG_COND_NE, args[1], args[3], const_args[3], args[5]);
+        tcg_out_brcond32(s, TCG_COND_NE, args[0], args[2], const_args[2],
+                         args[5], small);
+        tcg_out_brcond32(s, TCG_COND_NE, args[1], args[3], const_args[3],
+                         args[5], small);
         break;
     case TCG_COND_LT:
-        tcg_out_brcond(s, TCG_COND_LT, args[1], args[3], const_args[3], args[5]);
-        tcg_out_jxx(s, JCC_JNE, label_next);
-        tcg_out_brcond(s, TCG_COND_LTU, args[0], args[2], const_args[2], args[5]);
+        tcg_out_brcond32(s, TCG_COND_LT, args[1], args[3], const_args[3],
+                         args[5], small);
+        tcg_out_jxx(s, JCC_JNE, label_next, 1);
+        tcg_out_brcond32(s, TCG_COND_LTU, args[0], args[2], const_args[2],
+                         args[5], small);
         break;
     case TCG_COND_LE:
-        tcg_out_brcond(s, TCG_COND_LT, args[1], args[3], const_args[3], args[5]);
-        tcg_out_jxx(s, JCC_JNE, label_next);
-        tcg_out_brcond(s, TCG_COND_LEU, args[0], args[2], const_args[2], args[5]);
+        tcg_out_brcond32(s, TCG_COND_LT, args[1], args[3], const_args[3],
+                         args[5], small);
+        tcg_out_jxx(s, JCC_JNE, label_next, 1);
+        tcg_out_brcond32(s, TCG_COND_LEU, args[0], args[2], const_args[2],
+                         args[5], small);
         break;
     case TCG_COND_GT:
-        tcg_out_brcond(s, TCG_COND_GT, args[1], args[3], const_args[3], args[5]);
-        tcg_out_jxx(s, JCC_JNE, label_next);
-        tcg_out_brcond(s, TCG_COND_GTU, args[0], args[2], const_args[2], args[5]);
+        tcg_out_brcond32(s, TCG_COND_GT, args[1], args[3], const_args[3],
+                         args[5], small);
+        tcg_out_jxx(s, JCC_JNE, label_next, 1);
+        tcg_out_brcond32(s, TCG_COND_GTU, args[0], args[2], const_args[2],
+                         args[5], small);
         break;
     case TCG_COND_GE:
-        tcg_out_brcond(s, TCG_COND_GT, args[1], args[3], const_args[3], args[5]);
-        tcg_out_jxx(s, JCC_JNE, label_next);
-        tcg_out_brcond(s, TCG_COND_GEU, args[0], args[2], const_args[2], args[5]);
+        tcg_out_brcond32(s, TCG_COND_GT, args[1], args[3], const_args[3],
+                         args[5], small);
+        tcg_out_jxx(s, JCC_JNE, label_next, 1);
+        tcg_out_brcond32(s, TCG_COND_GEU, args[0], args[2], const_args[2],
+                         args[5], small);
         break;
     case TCG_COND_LTU:
-        tcg_out_brcond(s, TCG_COND_LTU, args[1], args[3], const_args[3], args[5]);
-        tcg_out_jxx(s, JCC_JNE, label_next);
-        tcg_out_brcond(s, TCG_COND_LTU, args[0], args[2], const_args[2], args[5]);
+        tcg_out_brcond32(s, TCG_COND_LTU, args[1], args[3], const_args[3],
+                         args[5], small);
+        tcg_out_jxx(s, JCC_JNE, label_next, 1);
+        tcg_out_brcond32(s, TCG_COND_LTU, args[0], args[2], const_args[2],
+                         args[5], small);
         break;
     case TCG_COND_LEU:
-        tcg_out_brcond(s, TCG_COND_LTU, args[1], args[3], const_args[3], args[5]);
-        tcg_out_jxx(s, JCC_JNE, label_next);
-        tcg_out_brcond(s, TCG_COND_LEU, args[0], args[2], const_args[2], args[5]);
+        tcg_out_brcond32(s, TCG_COND_LTU, args[1], args[3], const_args[3],
+                         args[5], small);
+        tcg_out_jxx(s, JCC_JNE, label_next, 1);
+        tcg_out_brcond32(s, TCG_COND_LEU, args[0], args[2], const_args[2],
+                         args[5], small);
         break;
     case TCG_COND_GTU:
-        tcg_out_brcond(s, TCG_COND_GTU, args[1], args[3], const_args[3], args[5]);
-        tcg_out_jxx(s, JCC_JNE, label_next);
-        tcg_out_brcond(s, TCG_COND_GTU, args[0], args[2], const_args[2], args[5]);
+        tcg_out_brcond32(s, TCG_COND_GTU, args[1], args[3], const_args[3],
+                         args[5], small);
+        tcg_out_jxx(s, JCC_JNE, label_next, 1);
+        tcg_out_brcond32(s, TCG_COND_GTU, args[0], args[2], const_args[2],
+                         args[5], small);
         break;
     case TCG_COND_GEU:
-        tcg_out_brcond(s, TCG_COND_GTU, args[1], args[3], const_args[3], args[5]);
-        tcg_out_jxx(s, JCC_JNE, label_next);
-        tcg_out_brcond(s, TCG_COND_GEU, args[0], args[2], const_args[2], args[5]);
+        tcg_out_brcond32(s, TCG_COND_GTU, args[1], args[3], const_args[3],
+                         args[5], small);
+        tcg_out_jxx(s, JCC_JNE, label_next, 1);
+        tcg_out_brcond32(s, TCG_COND_GEU, args[0], args[2], const_args[2],
+                         args[5], small);
         break;
     default:
@@ -443 +895 @@
     }
     tcg_out_label(s, label_next, (tcg_target_long)s->code_ptr);
+}
+#endif
+
+static void tcg_out_setcond32(TCGContext *s, TCGCond cond, TCGArg dest,
+                              TCGArg arg1, TCGArg arg2, int const_arg2)
+{
+    tcg_out_cmp(s, arg1, arg2, const_arg2, 0);
+    tcg_out_modrm(s, OPC_SETCC | tcg_cond_to_jcc[cond], 0, dest);
+    tcg_out_ext8u(s, dest, dest);
+}
+
+#if TCG_TARGET_REG_BITS == 64
+static void tcg_out_setcond64(TCGContext *s, TCGCond cond, TCGArg dest,
+                              TCGArg arg1, TCGArg arg2, int const_arg2)
+{
+    tcg_out_cmp(s, arg1, arg2, const_arg2, P_REXW);
+    tcg_out_modrm(s, OPC_SETCC | tcg_cond_to_jcc[cond], 0, dest);
+    tcg_out_ext8u(s, dest, dest);
+}
+#else
+static void tcg_out_setcond2(TCGContext *s, const TCGArg *args,
+                             const int *const_args)
+{
+    TCGArg new_args[6];
+    int label_true, label_over;
+
+    memcpy(new_args, args+1, 5*sizeof(TCGArg));
+
+    if (args[0] == args[1] || args[0] == args[2]
+        || (!const_args[3] && args[0] == args[3])
+        || (!const_args[4] && args[0] == args[4])) {
+        /* When the destination overlaps with one of the argument
+           registers, don't do anything tricky.  */
+        label_true = gen_new_label();
+        label_over = gen_new_label();
+
+        new_args[5] = label_true;
+        tcg_out_brcond2(s, new_args, const_args+1, 1);
+
+        tcg_out_movi(s, TCG_TYPE_I32, args[0], 0);
+        tcg_out_jxx(s, JCC_JMP, label_over, 1);
+        tcg_out_label(s, label_true, (tcg_target_long)s->code_ptr);
+
+        tcg_out_movi(s, TCG_TYPE_I32, args[0], 1);
+        tcg_out_label(s, label_over, (tcg_target_long)s->code_ptr);
+    } else {
+        /* When the destination does not overlap one of the arguments,
+           clear the destination first, jump if cond false, and emit an
+           increment in the true case.  This results in smaller code.  */
+
+        tcg_out_movi(s, TCG_TYPE_I32, args[0], 0);
+
+        label_over = gen_new_label();
+        new_args[4] = tcg_invert_cond(new_args[4]);
+        new_args[5] = label_over;
+        tcg_out_brcond2(s, new_args, const_args+1, 1);
+
+        tgen_arithi(s, ARITH_ADD, args[0], 1, 0);
+        tcg_out_label(s, label_over, (tcg_target_long)s->code_ptr);
+    }
+}
+#endif
+
+static void tcg_out_branch(TCGContext *s, int call, tcg_target_long dest)
+{
+    tcg_target_long disp = dest - (tcg_target_long)s->code_ptr - 5;
+
+    if (disp == (int32_t)disp) {
+        tcg_out_opc(s, call ? OPC_CALL_Jz : OPC_JMP_long, 0, 0, 0);
+        tcg_out32(s, disp);
+    } else {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R10, dest);
+        tcg_out_modrm(s, OPC_GRP5,
+                      call ? EXT5_CALLN_Ev : EXT5_JMPN_Ev, TCG_REG_R10);
+    }
+}
+
+static inline void tcg_out_calli(TCGContext *s, tcg_target_long dest)
+{
+#ifdef VBOX
+    tcg_gen_stack_alignment_check(s);
+#endif
+    tcg_out_branch(s, 1, dest);
+}
+
+static void tcg_out_jmp(TCGContext *s, tcg_target_long dest)
+{
+    tcg_out_branch(s, 0, dest);
 }
 
@@ -462 +1002 @@
     __stq_mmu,
 };
-#endif
-
-#ifndef CONFIG_USER_ONLY
-#define GUEST_BASE 0
-#endif
+
+/* Perform the TLB load and compare.
+
+   Inputs:
+   ADDRLO_IDX contains the index into ARGS of the low part of the
+   address; the high part of the address is at ADDR_LOW_IDX+1.
+
+   MEM_INDEX and S_BITS are the memory context and log2 size of the load.
+
+   WHICH is the offset into the CPUTLBEntry structure of the slot to read.
+   This should be offsetof addr_read or addr_write.
+
+   Outputs:
+   LABEL_PTRS is filled with 1 (32-bit addresses) or 2 (64-bit addresses)
+   positions of the displacements of forward jumps to the TLB miss case.
+
+   First argument register is loaded with the low part of the address.
+   In the TLB hit case, it has been adjusted as indicated by the TLB
+   and so is a host address.  In the TLB miss case, it continues to
+   hold a guest address.
+
+   Second argument register is clobbered.  */
+
+static inline void tcg_out_tlb_load(TCGContext *s, int addrlo_idx,
+                                    int mem_index, int s_bits,
+                                    const TCGArg *args,
+                                    uint8_t **label_ptr, int which)
+{
+    const int addrlo = args[addrlo_idx];
+    const int r0 = tcg_target_call_iarg_regs[0];
+    const int r1 = tcg_target_call_iarg_regs[1];
+    TCGType type = TCG_TYPE_I32;
+    int rexw = 0;
+
+    if (TCG_TARGET_REG_BITS == 64 && TARGET_LONG_BITS == 64) {
+        type = TCG_TYPE_I64;
+        rexw = P_REXW;
+    }
+
+    tcg_out_mov(s, type, r1, addrlo);
+    tcg_out_mov(s, type, r0, addrlo);
+
+    tcg_out_shifti(s, SHIFT_SHR + rexw, r1,
+                   TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
+
+    tgen_arithi(s, ARITH_AND + rexw, r0,
+                TARGET_PAGE_MASK | ((1 << s_bits) - 1), 0);
+    tgen_arithi(s, ARITH_AND + rexw, r1,
+                (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS, 0);
+
+    tcg_out_modrm_sib_offset(s, OPC_LEA + P_REXW, r1, TCG_AREG0, r1, 0,
+                             offsetof(CPUState, tlb_table[mem_index][0])
+                             + which);
+
+    /* cmp 0(r1), r0 */
+    tcg_out_modrm_offset(s, OPC_CMP_GvEv + rexw, r0, r1, 0);
+
+    tcg_out_mov(s, type, r0, addrlo);
+
+    /* jne label1 */
+    tcg_out8(s, OPC_JCC_short + JCC_JNE);
+    label_ptr[0] = s->code_ptr;
+    s->code_ptr++;
+
+    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+        /* cmp 4(r1), addrhi */
+        tcg_out_modrm_offset(s, OPC_CMP_GvEv, args[addrlo_idx+1], r1, 4);
+
+        /* jne label1 */
+        tcg_out8(s, OPC_JCC_short + JCC_JNE);
+        label_ptr[1] = s->code_ptr;
+        s->code_ptr++;
+    }
+
+    /* TLB Hit.  */
+
+    /* add addend(r1), r0 */
+    tcg_out_modrm_offset(s, OPC_ADD_GvEv + P_REXW, r0, r1,
+                         offsetof(CPUTLBEntry, addend) - which);
+}
+#endif
+
+static void tcg_out_qemu_ld_direct(TCGContext *s, int datalo, int datahi,
+                                   int base, tcg_target_long ofs, int sizeop)
+{
+#ifdef TARGET_WORDS_BIGENDIAN
+    const int bswap = 1;
+#else
+    const int bswap = 0;
+#endif
+    switch (sizeop) {
+    case 0:
+        tcg_out_modrm_offset(s, OPC_MOVZBL, datalo, base, ofs);
+        break;
+    case 0 | 4:
+        tcg_out_modrm_offset(s, OPC_MOVSBL + P_REXW, datalo, base, ofs);
+        break;
+    case 1:
+        tcg_out_modrm_offset(s, OPC_MOVZWL, datalo, base, ofs);
+        if (bswap) {
+            tcg_out_rolw_8(s, datalo);
+        }
+        break;
+    case 1 | 4:
+        if (bswap) {
+            tcg_out_modrm_offset(s, OPC_MOVZWL, datalo, base, ofs);
+            tcg_out_rolw_8(s, datalo);
+            tcg_out_modrm(s, OPC_MOVSWL + P_REXW, datalo, datalo);
+        } else {
+            tcg_out_modrm_offset(s, OPC_MOVSWL + P_REXW, datalo, base, ofs);
+        }
+        break;
+    case 2:
+        tcg_out_ld(s, TCG_TYPE_I32, datalo, base, ofs);
+        if (bswap) {
+            tcg_out_bswap32(s, datalo);
+        }
+        break;
+#if TCG_TARGET_REG_BITS == 64
+    case 2 | 4:
+        if (bswap) {
+            tcg_out_ld(s, TCG_TYPE_I32, datalo, base, ofs);
+            tcg_out_bswap32(s, datalo);
+            tcg_out_ext32s(s, datalo, datalo);
+        } else {
+            tcg_out_modrm_offset(s, OPC_MOVSLQ, datalo, base, ofs);
+        }
+        break;
+#endif
+    case 3:
+        if (TCG_TARGET_REG_BITS == 64) {
+            tcg_out_ld(s, TCG_TYPE_I64, datalo, base, ofs);
+            if (bswap) {
+                tcg_out_bswap64(s, datalo);
+            }
+        } else {
+            if (bswap) {
+                int t = datalo;
+                datalo = datahi;
+                datahi = t;
+            }
+            if (base != datalo) {
+                tcg_out_ld(s, TCG_TYPE_I32, datalo, base, ofs);
+                tcg_out_ld(s, TCG_TYPE_I32, datahi, base, ofs + 4);
+            } else {
+                tcg_out_ld(s, TCG_TYPE_I32, datahi, base, ofs + 4);
+                tcg_out_ld(s, TCG_TYPE_I32, datalo, base, ofs);
+            }
+            if (bswap) {
+                tcg_out_bswap32(s, datalo);
+                tcg_out_bswap32(s, datahi);
+            }
+        }
+        break;
+    default:
+        tcg_abort();
+    }
+}
 
 #if defined(VBOX) && defined(REM_PHYS_ADDR_IN_TLB)
@@ -570 +1263 @@
                             int opc)
 {
-    int addr_reg, data_reg, data_reg2, r0, r1, mem_index, s_bits, bswap;
+    int data_reg, data_reg2 = 0;
+    int addrlo_idx;
 #if defined(CONFIG_SOFTMMU)
-    uint8_t *label1_ptr, *label2_ptr;
-#endif
-#if TARGET_LONG_BITS == 64
+    int mem_index, s_bits, arg_idx;
+    uint8_t *label_ptr[3];
+#endif
+
+    data_reg = args[0];
+    addrlo_idx = 1;
+    if (TCG_TARGET_REG_BITS == 32 && opc == 3) {
+        data_reg2 = args[1];
+        addrlo_idx = 2;
+    }
+
 #if defined(CONFIG_SOFTMMU)
-    uint8_t *label3_ptr;
-#endif
-    int addr_reg2;
-#endif
-
-    data_reg = *args++;
-    if (opc == 3)
-        data_reg2 = *args++;
-    else
-        data_reg2 = 0;
-    addr_reg = *args++;
-#if TARGET_LONG_BITS == 64
-    addr_reg2 = *args++;
-#endif
-    mem_index = *args;
+    mem_index = args[addrlo_idx + 1 + (TARGET_LONG_BITS > TCG_TARGET_REG_BITS)];
     s_bits = opc & 3;
 
-    r0 = TCG_REG_EAX;
-    r1 = TCG_REG_EDX;
-
-#if defined(CONFIG_SOFTMMU)
-    tcg_out_mov(s, r1, addr_reg);
-
-    tcg_out_mov(s, r0, addr_reg);
-
-    tcg_out_modrm(s, 0xc1, 5, r1); /* shr $x, r1 */
-    tcg_out8(s, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
-
-    tcg_out_modrm(s, 0x81, 4, r0); /* andl $x, r0 */
-    tcg_out32(s, TARGET_PAGE_MASK | ((1 << s_bits) - 1));
-
-    tcg_out_modrm(s, 0x81, 4, r1); /* andl $x, r1 */
-    tcg_out32(s, (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
-
-    tcg_out_opc(s, 0x8d); /* lea offset(r1, %ebp), r1 */
-    tcg_out8(s, 0x80 | (r1 << 3) | 0x04);
-#ifndef VBOX
-    tcg_out8(s, (5 << 3) | r1);
-#else
-    tcg_out8(s, (TCG_AREG0 << 3) | r1); /* env, not %ebp */
-    Assert(mem_index >= 0 && mem_index < NB_MMU_MODES);
-#endif
-    tcg_out32(s, offsetof(CPUState, tlb_table[mem_index][0].addr_read));
-
-    /* cmp 0(r1), r0 */
-    tcg_out_modrm_offset(s, 0x3b, r0, r1, 0);
-
-    tcg_out_mov(s, r0, addr_reg);
-
-#if TARGET_LONG_BITS == 32
-    /* je label1 */
-    tcg_out8(s, 0x70 + JCC_JE);
-    label1_ptr = s->code_ptr;
+    tcg_out_tlb_load(s, addrlo_idx, mem_index, s_bits, args,
+                     label_ptr, offsetof(CPUTLBEntry, addr_read));
+
+    /* TLB Hit.  */
+    tcg_out_qemu_ld_direct(s, data_reg, data_reg2,
+                           tcg_target_call_iarg_regs[0], 0, opc);
+
+    /* jmp label2 */
+    tcg_out8(s, OPC_JMP_short);
+    label_ptr[2] = s->code_ptr;
     s->code_ptr++;
-#else
-    /* jne label3 */
-    tcg_out8(s, 0x70 + JCC_JNE);
-    label3_ptr = s->code_ptr;
-    s->code_ptr++;
-
-    /* cmp 4(r1), addr_reg2 */
-    tcg_out_modrm_offset(s, 0x3b, addr_reg2, r1, 4);
-
-    /* je label1 */
-    tcg_out8(s, 0x70 + JCC_JE);
-    label1_ptr = s->code_ptr;
-    s->code_ptr++;
-
-    /* label3: */
-    *label3_ptr = s->code_ptr - label3_ptr - 1;
-#endif
+
+    /* TLB Miss.  */
+
+    /* label1: */
+    *label_ptr[0] = s->code_ptr - label_ptr[0] - 1;
+    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+        *label_ptr[1] = s->code_ptr - label_ptr[1] - 1;
+    }
 
     /* XXX: move that code at the end of the TB */
-#if TARGET_LONG_BITS == 32
-    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_EDX, mem_index);
-#else
-    tcg_out_mov(s, TCG_REG_EDX, addr_reg2);
-    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_ECX, mem_index);
-#endif
-#ifdef VBOX
-    tcg_gen_stack_alignment_check(s);
-#endif
-    tcg_out8(s, 0xe8);
-    tcg_out32(s, (tcg_target_long)qemu_ld_helpers[s_bits] -
-              (tcg_target_long)s->code_ptr - 4);
+    /* The first argument is already loaded with addrlo.  */
+    arg_idx = 1;
+    if (TCG_TARGET_REG_BITS == 32 && TARGET_LONG_BITS == 64) {
+        tcg_out_mov(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[arg_idx++],
+                    args[addrlo_idx + 1]);
+    }
+    tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[arg_idx],
+                 mem_index);
+    tcg_out_calli(s, (tcg_target_long)qemu_ld_helpers[s_bits]);
 
     switch(opc) {
     case 0 | 4:
-        /* movsbl */
-        tcg_out_modrm(s, 0xbe | P_EXT, data_reg, TCG_REG_EAX);
+        tcg_out_ext8s(s, data_reg, TCG_REG_EAX, P_REXW);
         break;
     case 1 | 4:
-        /* movswl */
-        tcg_out_modrm(s, 0xbf | P_EXT, data_reg, TCG_REG_EAX);
+        tcg_out_ext16s(s, data_reg, TCG_REG_EAX, P_REXW);
         break;
     case 0:
-        /* movzbl */
-        tcg_out_modrm(s, 0xb6 | P_EXT, data_reg, TCG_REG_EAX);
+        tcg_out_ext8u(s, data_reg, TCG_REG_EAX);
         break;
     case 1:
-        /* movzwl */
-        tcg_out_modrm(s, 0xb7 | P_EXT, data_reg, TCG_REG_EAX);
+        tcg_out_ext16u(s, data_reg, TCG_REG_EAX);
         break;
     case 2:
+        tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
+        break;
+#if TCG_TARGET_REG_BITS == 64
+    case 2 | 4:
+        tcg_out_ext32s(s, data_reg, TCG_REG_EAX);
+        break;
+#endif
+    case 3:
+        if (TCG_TARGET_REG_BITS == 64) {
+            tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_RAX);
+        } else if (data_reg == TCG_REG_EDX) {
+            /* xchg %edx, %eax */
+            tcg_out_opc(s, OPC_XCHG_ax_r32 + TCG_REG_EDX, 0, 0, 0);
+            tcg_out_mov(s, TCG_TYPE_I32, data_reg2, TCG_REG_EAX);
+        } else {
+            tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
+            tcg_out_mov(s, TCG_TYPE_I32, data_reg2, TCG_REG_EDX);
+        }
+        break;
     default:
-        tcg_out_mov(s, data_reg, TCG_REG_EAX);
-        break;
-    case 3:
-        if (data_reg == TCG_REG_EDX) {
-            tcg_out_opc(s, 0x90 + TCG_REG_EDX); /* xchg %edx, %eax */
-            tcg_out_mov(s, data_reg2, TCG_REG_EAX);
-        } else {
-            tcg_out_mov(s, data_reg, TCG_REG_EAX);
-            tcg_out_mov(s, data_reg2, TCG_REG_EDX);
-        }
-        break;
-    }
-
-    /* jmp label2 */
-    tcg_out8(s, 0xeb);
-    label2_ptr = s->code_ptr;
-    s->code_ptr++;
-
-    /* label1: */
-    *label1_ptr = s->code_ptr - label1_ptr - 1;
-
-    /* add x(r1), r0 */
-    tcg_out_modrm_offset(s, 0x03, r0, r1, offsetof(CPUTLBEntry, addend) -
-                         offsetof(CPUTLBEntry, addr_read));
+        tcg_abort();
+    }
+
+    /* label2: */
+    *label_ptr[2] = s->code_ptr - label_ptr[2] - 1;
 #else
-    r0 = addr_reg;
-#endif
-
+    {
+        int32_t offset = GUEST_BASE;
+        int base = args[addrlo_idx];
+
+        if (TCG_TARGET_REG_BITS == 64) {
+            /* ??? We assume all operations have left us with register
+               contents that are zero extended.  So far this appears to
+               be true.  If we want to enforce this, we can either do
+               an explicit zero-extension here, or (if GUEST_BASE == 0)
+               use the ADDR32 prefix.  For now, do nothing.  */
+
+            if (offset != GUEST_BASE) {
+                tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_RDI, GUEST_BASE);
+                tgen_arithr(s, ARITH_ADD + P_REXW, TCG_REG_RDI, base);
+                base = TCG_REG_RDI, offset = 0;
+            }
+        }
+
+        tcg_out_qemu_ld_direct(s, data_reg, data_reg2, base, offset, opc);
+    }
+#endif
+}
+
+static void tcg_out_qemu_st_direct(TCGContext *s, int datalo, int datahi,
+                                   int base, tcg_target_long ofs, int sizeop)
+{
 #if !defined(VBOX) || !defined(REM_PHYS_ADDR_IN_TLB)
 #ifdef TARGET_WORDS_BIGENDIAN
-    bswap = 1;
+    const int bswap = 1;
 #else
-    bswap = 0;
-#endif
-    switch(opc) {
+    const int bswap = 0;
+#endif
+    /* ??? Ideally we wouldn't need a scratch register.  For user-only,
+       we could perform the bswap twice to restore the original value
+       instead of moving to the scratch.  But as it is, the L constraint
+       means that the second argument reg is definitely free here.  */
+    int scratch = tcg_target_call_iarg_regs[1];
+
+    switch (sizeop) {
     case 0:
-        /* movzbl */
-        tcg_out_modrm_offset(s, 0xb6 | P_EXT, data_reg, r0, GUEST_BASE);
-        break;
-    case 0 | 4:
-        /* movsbl */
-        tcg_out_modrm_offset(s, 0xbe | P_EXT, data_reg, r0, GUEST_BASE);
+        tcg_out_modrm_offset(s, OPC_MOVB_EvGv + P_REXB_R, datalo, base, ofs);
         break;
     case 1:
-        /* movzwl */
-        tcg_out_modrm_offset(s, 0xb7 | P_EXT, data_reg, r0, GUEST_BASE);
         if (bswap) {
-            /* rolw $8, data_reg */
-            tcg_out8(s, 0x66);
-            tcg_out_modrm(s, 0xc1, 0, data_reg);
-            tcg_out8(s, 8);
-        }
-        break;
-    case 1 | 4:
-        /* movswl */
-        tcg_out_modrm_offset(s, 0xbf | P_EXT, data_reg, r0, GUEST_BASE);
+            tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
+            tcg_out_rolw_8(s, scratch);
+            datalo = scratch;
+        }
+        tcg_out_modrm_offset(s, OPC_MOVL_EvGv + P_DATA16, datalo, base, ofs);
+        break;
+    case 2:
         if (bswap) {
-            /* rolw $8, data_reg */
-            tcg_out8(s, 0x66);
-            tcg_out_modrm(s, 0xc1, 0, data_reg);
-            tcg_out8(s, 8);
-
-            /* movswl data_reg, data_reg */
-            tcg_out_modrm(s, 0xbf | P_EXT, data_reg, data_reg);
-        }
-        break;
-    case 2:
-        /* movl (r0), data_reg */
-        tcg_out_modrm_offset(s, 0x8b, data_reg, r0, GUEST_BASE);
-        if (bswap) {
-            /* bswap */
-            tcg_out_opc(s, (0xc8 + data_reg) | P_EXT);
-        }
+            tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
+            tcg_out_bswap32(s, scratch);
+            datalo = scratch;
+        }
+        tcg_out_st(s, TCG_TYPE_I32, datalo, base, ofs);
         break;
     case 3:
-        /* XXX: could be nicer */
-        if (r0 == data_reg) {
-            r1 = TCG_REG_EDX;
-            if (r1 == data_reg)
-                r1 = TCG_REG_EAX;
-            tcg_out_mov(s, r1, r0);
-            r0 = r1;
-        }
-        if (!bswap) {
-            tcg_out_modrm_offset(s, 0x8b, data_reg, r0, GUEST_BASE);
-            tcg_out_modrm_offset(s, 0x8b, data_reg2, r0, GUEST_BASE + 4);
-        } else {
-            tcg_out_modrm_offset(s, 0x8b, data_reg, r0, GUEST_BASE + 4);
-            tcg_out_opc(s, (0xc8 + data_reg) | P_EXT);
-
-            tcg_out_modrm_offset(s, 0x8b, data_reg2, r0, GUEST_BASE);
-            /* bswap */
-            tcg_out_opc(s, (0xc8 + data_reg2) | P_EXT);
+        if (TCG_TARGET_REG_BITS == 64) {
+            if (bswap) {
+                tcg_out_mov(s, TCG_TYPE_I64, scratch, datalo);
+                tcg_out_bswap64(s, scratch);
+                datalo = scratch;
+            }
+            tcg_out_st(s, TCG_TYPE_I64, datalo, base, ofs);
+        } else if (bswap) {
+            tcg_out_mov(s, TCG_TYPE_I32, scratch, datahi);
+            tcg_out_bswap32(s, scratch);
+            tcg_out_st(s, TCG_TYPE_I32, scratch, base, ofs);
+            tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
+            tcg_out_bswap32(s, scratch);
+            tcg_out_st(s, TCG_TYPE_I32, scratch, base, ofs + 4);
+        } else {
+            tcg_out_st(s, TCG_TYPE_I32, datalo, base, ofs);
+            tcg_out_st(s, TCG_TYPE_I32, datahi, base, ofs + 4);
         }
         break;
@@ -780 +1434 @@
     }
 #else /* VBOX */
+# error "broken"
     tcg_out_vbox_phys_read(s, opc, r0, data_reg, data_reg2);
 #endif
-
-
-#if defined(CONFIG_SOFTMMU)
-    /* label2: */
-    *label2_ptr = s->code_ptr - label2_ptr - 1;
-# ifdef VBOX
-    Assert((unsigned)(s->code_ptr - label2_ptr - 1) <= 127);
-# endif
-#endif
-}
-
+}
 
 static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
                             int opc)
 {
-    int addr_reg, data_reg, data_reg2, r0, r1, mem_index, s_bits, bswap;
+    int data_reg, data_reg2 = 0;
+    int addrlo_idx;
 #if defined(CONFIG_SOFTMMU)
-    uint8_t *label1_ptr, *label2_ptr;
-#endif
-#if TARGET_LONG_BITS == 64
+    int mem_index, s_bits;
+    int stack_adjust;
+    uint8_t *label_ptr[3];
+#endif
+
+    data_reg = args[0];
+    addrlo_idx = 1;
+    if (TCG_TARGET_REG_BITS == 32 && opc == 3) {
+        data_reg2 = args[1];
+        addrlo_idx = 2;
+    }
+
 #if defined(CONFIG_SOFTMMU)
-    uint8_t *label3_ptr;
-#endif
-    int addr_reg2;
-#endif
-#ifdef VBOX
-# ifdef RT_OS_DARWIN
-    int bias1 = 12, bias3 = 4;/** @todo TCG_TARGET_STACK_ALIGN. */
-# else
-    int bias1 = 0, bias3 = 0;
-# endif
-    NOREF(bias3);
-#endif
-
-    data_reg = *args++;
-    if (opc == 3)
-        data_reg2 = *args++;
-    else
-        data_reg2 = 0;
-    addr_reg = *args++;
-#if TARGET_LONG_BITS == 64
-    addr_reg2 = *args++;
-#endif
-    mem_index = *args;
-
+    mem_index = args[addrlo_idx + 1 + (TARGET_LONG_BITS > TCG_TARGET_REG_BITS)];
     s_bits = opc;
 
-    r0 = TCG_REG_EAX;
-    r1 = TCG_REG_EDX;
-
-#if defined(CONFIG_SOFTMMU)
-    tcg_out_mov(s, r1, addr_reg);
-
-    tcg_out_mov(s, r0, addr_reg);
-
-    tcg_out_modrm(s, 0xc1, 5, r1); /* shr $x, r1 */
-    tcg_out8(s, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
-
-    tcg_out_modrm(s, 0x81, 4, r0); /* andl $x, r0 */
-    tcg_out32(s, TARGET_PAGE_MASK | ((1 << s_bits) - 1));
-
-    tcg_out_modrm(s, 0x81, 4, r1); /* andl $x, r1 */
-    tcg_out32(s, (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
-
-    tcg_out_opc(s, 0x8d); /* lea offset(r1, %ebp), r1 */
-    tcg_out8(s, 0x80 | (r1 << 3) | 0x04);
-#ifndef VBOX
-    tcg_out8(s, (5 << 3) | r1);
-#else
-    tcg_out8(s, (TCG_AREG0 << 3) | r1); /* env is not %ebp */
-    Assert(mem_index >= 0 && mem_index < NB_MMU_MODES);
-#endif
-    tcg_out32(s, offsetof(CPUState, tlb_table[mem_index][0].addr_write));
-
-    /* cmp 0(r1), r0 */
-    tcg_out_modrm_offset(s, 0x3b, r0, r1, 0);
-
-    tcg_out_mov(s, r0, addr_reg);
-
-#if TARGET_LONG_BITS == 32
-    /* je label1 */
-    tcg_out8(s, 0x70 + JCC_JE);
-    label1_ptr = s->code_ptr;
+    tcg_out_tlb_load(s, addrlo_idx, mem_index, s_bits, args,
+                     label_ptr, offsetof(CPUTLBEntry, addr_write));
+
+    /* TLB Hit.  */
+    tcg_out_qemu_st_direct(s, data_reg, data_reg2,
+                           tcg_target_call_iarg_regs[0], 0, opc);
+
+    /* jmp label2 */
+    tcg_out8(s, OPC_JMP_short);
+    label_ptr[2] = s->code_ptr;
     s->code_ptr++;
-#else
-    /* jne label3 */
-    tcg_out8(s, 0x70 + JCC_JNE);
-    label3_ptr = s->code_ptr;
-    s->code_ptr++;
-
-    /* cmp 4(r1), addr_reg2 */
-    tcg_out_modrm_offset(s, 0x3b, addr_reg2, r1, 4);
-
-    /* je label1 */
-    tcg_out8(s, 0x70 + JCC_JE);
-    label1_ptr = s->code_ptr;
-    s->code_ptr++;
-
-    /* label3: */
-    *label3_ptr = s->code_ptr - label3_ptr - 1;
-#endif
+
+    /* TLB Miss.  */
+
+    /* label1: */
+    *label_ptr[0] = s->code_ptr - label_ptr[0] - 1;
+    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+        *label_ptr[1] = s->code_ptr - label_ptr[1] - 1;
+    }
+
+#if !defined(VBOX) || !defined(REM_PHYS_ADDR_IN_TLB)
 
     /* XXX: move that code at the end of the TB */
-#if TARGET_LONG_BITS == 32
-    if (opc == 3) {
-        tcg_out_mov(s, TCG_REG_EDX, data_reg);
-        tcg_out_mov(s, TCG_REG_ECX, data_reg2);
-#ifdef VBOX
-        tcg_out_subi(s, TCG_REG_ESP, bias1);
-#endif
-        tcg_out8(s, 0x6a); /* push Ib */
-        tcg_out8(s, mem_index);
-# ifdef VBOX
-        tcg_gen_stack_alignment_check(s);
-# endif
-        tcg_out8(s, 0xe8);
-        tcg_out32(s, (tcg_target_long)qemu_st_helpers[s_bits] -
-                  (tcg_target_long)s->code_ptr - 4);
-#ifdef VBOX
-        tcg_out_addi(s, TCG_REG_ESP, 4+bias1);
-#else
-        tcg_out_addi(s, TCG_REG_ESP, 4);
-#endif
+    if (TCG_TARGET_REG_BITS == 64) {
+        tcg_out_mov(s, (opc == 3 ? TCG_TYPE_I64 : TCG_TYPE_I32),
+                    TCG_REG_RSI, data_reg);
+        tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_RDX, mem_index);
+        stack_adjust = 0;
+    } else if (TARGET_LONG_BITS == 32) {
+        tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_EDX, data_reg);
+        if (opc == 3) {
+            tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_ECX, data_reg2);
+            tcg_out_pushi(s, mem_index);
+            stack_adjust = 4;
+        } else {
+            tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_ECX, mem_index);
+            stack_adjust = 0;
+        }
     } else {
-        switch(opc) {
-        case 0:
-            /* movzbl */
-            tcg_out_modrm(s, 0xb6 | P_EXT, TCG_REG_EDX, data_reg);
-            break;
-        case 1:
-            /* movzwl */
-            tcg_out_modrm(s, 0xb7 | P_EXT, TCG_REG_EDX, data_reg);
-            break;
-        case 2:
-            tcg_out_mov(s, TCG_REG_EDX, data_reg);
-            break;
-        }
-        tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_ECX, mem_index);
-# ifdef VBOX
-        tcg_gen_stack_alignment_check(s);
-# endif
-        tcg_out8(s, 0xe8);
-        tcg_out32(s, (tcg_target_long)qemu_st_helpers[s_bits] -
-                  (tcg_target_long)s->code_ptr - 4);
-    }
-#else
-    if (opc == 3) {
-        tcg_out_mov(s, TCG_REG_EDX, addr_reg2);
-# ifdef VBOX
-        tcg_out_subi(s, TCG_REG_ESP, bias3);
-# endif
-        tcg_out8(s, 0x6a); /* push Ib */
-        tcg_out8(s, mem_index);
-        tcg_out_opc(s, 0x50 + data_reg2); /* push */
-        tcg_out_opc(s, 0x50 + data_reg); /* push */
-# ifdef VBOX
-        tcg_gen_stack_alignment_check(s);
-# endif
-        tcg_out8(s, 0xe8);
-        tcg_out32(s, (tcg_target_long)qemu_st_helpers[s_bits] -
-                  (tcg_target_long)s->code_ptr - 4);
-#ifdef VBOX
-        tcg_out_addi(s, TCG_REG_ESP, 12+bias3);
-#else
-        tcg_out_addi(s, TCG_REG_ESP, 12);
-#endif
-    } else {
-        tcg_out_mov(s, TCG_REG_EDX, addr_reg2);
-        switch(opc) {
-        case 0:
-            /* movzbl */
-            tcg_out_modrm(s, 0xb6 | P_EXT, TCG_REG_ECX, data_reg);
-            break;
-        case 1:
-            /* movzwl */
-            tcg_out_modrm(s, 0xb7 | P_EXT, TCG_REG_ECX, data_reg);
-            break;
-        case 2:
-            tcg_out_mov(s, TCG_REG_ECX, data_reg);
-            break;
-        }
-# ifdef VBOX
-        tcg_out_subi(s, TCG_REG_ESP, bias1);
-# endif
-        tcg_out8(s, 0x6a); /* push Ib */
-        tcg_out8(s, mem_index);
-# ifdef VBOX
-        tcg_gen_stack_alignment_check(s);
-# endif
-        tcg_out8(s, 0xe8);
-        tcg_out32(s, (tcg_target_long)qemu_st_helpers[s_bits] -
-                  (tcg_target_long)s->code_ptr - 4);
-# if defined(VBOX)
-        tcg_out_addi(s, TCG_REG_ESP, 4 + bias1);
-# else
-        tcg_out_addi(s, TCG_REG_ESP, 4);
-# endif
-    }
-#endif
-
-    /* jmp label2 */
-    tcg_out8(s, 0xeb);
-    label2_ptr = s->code_ptr;
-    s->code_ptr++;
-
-    /* label1: */
-    *label1_ptr = s->code_ptr - label1_ptr - 1;
-
-    /* add x(r1), r0 */
-    tcg_out_modrm_offset(s, 0x03, r0, r1, offsetof(CPUTLBEntry, addend) -
-                         offsetof(CPUTLBEntry, addr_write));
-#else
-    r0 = addr_reg;
-#endif
-
-#if !defined(VBOX) || !defined(REM_PHYS_ADDR_IN_TLB)
-#ifdef TARGET_WORDS_BIGENDIAN
-    bswap = 1;
-#else
-    bswap = 0;
-#endif
-    switch(opc) {
-    case 0:
-        /* movb */
-        tcg_out_modrm_offset(s, 0x88, data_reg, r0, GUEST_BASE);
-        break;
-    case 1:
-        if (bswap) {
-            tcg_out_mov(s, r1, data_reg);
-            tcg_out8(s, 0x66); /* rolw $8, %ecx */
-            tcg_out_modrm(s, 0xc1, 0, r1);
-            tcg_out8(s, 8);
-            data_reg = r1;
-        }
-        /* movw */
-        tcg_out8(s, 0x66);
-        tcg_out_modrm_offset(s, 0x89, data_reg, r0, GUEST_BASE);
-        break;
-    case 2:
-        if (bswap) {
-            tcg_out_mov(s, r1, data_reg);
-            /* bswap data_reg */
-            tcg_out_opc(s, (0xc8 + r1) | P_EXT);
-            data_reg = r1;
-        }
-        /* movl */
-        tcg_out_modrm_offset(s, 0x89, data_reg, r0, GUEST_BASE);
-        break;
-    case 3:
-        if (bswap) {
-            tcg_out_mov(s, r1, data_reg2);
-            /* bswap data_reg */
-            tcg_out_opc(s, (0xc8 + r1) | P_EXT);
-            tcg_out_modrm_offset(s, 0x89, r1, r0, GUEST_BASE);
-            tcg_out_mov(s, r1, data_reg);
-            /* bswap data_reg */
-            tcg_out_opc(s, (0xc8 + r1) | P_EXT);
-            tcg_out_modrm_offset(s, 0x89, r1, r0, GUEST_BASE + 4);
-        } else {
-            tcg_out_modrm_offset(s, 0x89, data_reg, r0, GUEST_BASE);
-            tcg_out_modrm_offset(s, 0x89, data_reg2, r0, GUEST_BASE + 4);
-        }
-        break;
-    default:
-        tcg_abort();
-    }
+        if (opc == 3) {
+            tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_EDX, args[addrlo_idx + 1]);
+            tcg_out_pushi(s, mem_index);
+            tcg_out_push(s, data_reg2);
+            tcg_out_push(s, data_reg);
+            stack_adjust = 12;
+        } else {
+            tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_EDX, args[addrlo_idx + 1]);
+            switch(opc) {
+            case 0:
+                tcg_out_ext8u(s, TCG_REG_ECX, data_reg);
+                break;
+            case 1:
+                tcg_out_ext16u(s, TCG_REG_ECX, data_reg);
+                break;
+            case 2:
+                tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_ECX, data_reg);
+                break;
+            }
+            tcg_out_pushi(s, mem_index);
+            stack_adjust = 4;
+        }
+    }
+
+    tcg_out_calli(s, (tcg_target_long)qemu_st_helpers[s_bits]);
+
+    if (stack_adjust == (TCG_TARGET_REG_BITS / 8)) {
+        /* Pop and discard.  This is 2 bytes smaller than the add.  */
+        tcg_out_pop(s, TCG_REG_ECX);
+    } else if (stack_adjust != 0) {
+        tcg_out_addi(s, TCG_REG_ESP, stack_adjust);
+    }
+
 #else  /* VBOX && REM_PHYS_ADDR_IN_TLB */
+# error Borked
     tcg_out_vbox_phys_write(s, opc, r0, data_reg, data_reg2);
 #endif /* VBOX && REM_PHYS_ADDR_IN_TLB */
 
-#if defined(CONFIG_SOFTMMU)
     /* label2: */
-    *label2_ptr = s->code_ptr - label2_ptr - 1;
-# ifdef VBOX
-    Assert((unsigned)(s->code_ptr - label2_ptr - 1) <= 127);
-# endif
-#endif
-}
-
-static inline void tcg_out_op(TCGContext *s, int opc,
+    *label_ptr[2] = s->code_ptr - label_ptr[2] - 1;
+#else
+    {
+        int32_t offset = GUEST_BASE;
+        int base = args[addrlo_idx];
+
+        if (TCG_TARGET_REG_BITS == 64) {
+            /* ??? We assume all operations have left us with register
+               contents that are zero extended.  So far this appears to
+               be true.  If we want to enforce this, we can either do
+               an explicit zero-extension here, or (if GUEST_BASE == 0)
+               use the ADDR32 prefix.  For now, do nothing.  */
+
+            if (offset != GUEST_BASE) {
+                tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_RDI, GUEST_BASE);
+                tgen_arithr(s, ARITH_ADD + P_REXW, TCG_REG_RDI, base);
+                base = TCG_REG_RDI, offset = 0;
+            }
+        }
+
+        tcg_out_qemu_st_direct(s, data_reg, data_reg2, base, offset, opc);
+    }
+#endif
+}
+
+static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
                               const TCGArg *args, const int *const_args)
 {
-    int c;
+    int c, rexw = 0;
+
+#if TCG_TARGET_REG_BITS == 64
+# define OP_32_64(x) \
+        case glue(glue(INDEX_op_, x), _i64): \
+            rexw = P_REXW; /* FALLTHRU */    \
+        case glue(glue(INDEX_op_, x), _i32)
+#else
+# define OP_32_64(x) \
+        case glue(glue(INDEX_op_, x), _i32)
+#endif
 
     switch(opc) {
     case INDEX_op_exit_tb:
-        tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_EAX, args[0]);
-        tcg_out8(s, 0xe9); /* jmp tb_ret_addr */
-        tcg_out32(s, tb_ret_addr - s->code_ptr - 4);
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, args[0]);
+        tcg_out_jmp(s, (tcg_target_long) tb_ret_addr);
         break;
     case INDEX_op_goto_tb:
         if (s->tb_jmp_offset) {
             /* direct jump method */
-            tcg_out8(s, 0xe9); /* jmp im */
+            tcg_out8(s, OPC_JMP_long); /* jmp im */
             s->tb_jmp_offset[args[0]] = s->code_ptr - s->code_buf;
             tcg_out32(s, 0);
         } else {
             /* indirect jump method */
-            /* jmp Ev */
-            tcg_out_modrm_offset(s, 0xff, 4, -1,
+            tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, -1,
                                  (tcg_target_long)(s->tb_next + args[0]));
         }
@@ -1087 +1598 @@
         break;
     case INDEX_op_call:
-#ifdef VBOX
-        tcg_gen_stack_alignment_check(s);
-#endif
         if (const_args[0]) {
-            tcg_out8(s, 0xe8);
-            tcg_out32(s, args[0] - (tcg_target_long)s->code_ptr - 4);
-        } else {
-            tcg_out_modrm(s, 0xff, 2, args[0]);
+            tcg_out_calli(s, args[0]);
+        } else {
+            /* call *reg */
+            tcg_out_modrm(s, OPC_GRP5, EXT5_CALLN_Ev, args[0]);
         }
         break;
     case INDEX_op_jmp:
         if (const_args[0]) {
-            tcg_out8(s, 0xe9);
-            tcg_out32(s, args[0] - (tcg_target_long)s->code_ptr - 4);
-        } else {
-            tcg_out_modrm(s, 0xff, 4, args[0]);
+            tcg_out_jmp(s, args[0]);
+        } else {
+            /* jmp *reg */
+            tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, args[0]);
         }
         break;
     case INDEX_op_br:
-        tcg_out_jxx(s, JCC_JMP, args[0]);
+        tcg_out_jxx(s, JCC_JMP, args[0], 0);
         break;
     case INDEX_op_movi_i32:
         tcg_out_movi(s, TCG_TYPE_I32, args[0], args[1]);
         break;
-    case INDEX_op_ld8u_i32:
-        /* movzbl */
-        tcg_out_modrm_offset(s, 0xb6 | P_EXT, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_ld8s_i32:
-        /* movsbl */
-        tcg_out_modrm_offset(s, 0xbe | P_EXT, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_ld16u_i32:
-        /* movzwl */
-        tcg_out_modrm_offset(s, 0xb7 | P_EXT, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_ld16s_i32:
-        /* movswl */
-        tcg_out_modrm_offset(s, 0xbf | P_EXT, args[0], args[1], args[2]);
-        break;
+    OP_32_64(ld8u):
+        /* Note that we can ignore REXW for the zero-extend to 64-bit.  */
+        tcg_out_modrm_offset(s, OPC_MOVZBL, args[0], args[1], args[2]);
+        break;
+    OP_32_64(ld8s):
+        tcg_out_modrm_offset(s, OPC_MOVSBL + rexw, args[0], args[1], args[2]);
+        break;
+    OP_32_64(ld16u):
+        /* Note that we can ignore REXW for the zero-extend to 64-bit.  */
+        tcg_out_modrm_offset(s, OPC_MOVZWL, args[0], args[1], args[2]);
+        break;
+    OP_32_64(ld16s):
+        tcg_out_modrm_offset(s, OPC_MOVSWL + rexw, args[0], args[1], args[2]);
+        break;
+#if TCG_TARGET_REG_BITS == 64
+    case INDEX_op_ld32u_i64:
+#endif
     case INDEX_op_ld_i32:
-        /* movl */
-        tcg_out_modrm_offset(s, 0x8b, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_st8_i32:
-        /* movb */
-        tcg_out_modrm_offset(s, 0x88, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_st16_i32:
-        /* movw */
-        tcg_out8(s, 0x66);
-        tcg_out_modrm_offset(s, 0x89, args[0], args[1], args[2]);
-        break;
+        tcg_out_ld(s, TCG_TYPE_I32, args[0], args[1], args[2]);
+        break;
+
+    OP_32_64(st8):
+        tcg_out_modrm_offset(s, OPC_MOVB_EvGv | P_REXB_R,
+                             args[0], args[1], args[2]);
+        break;
+    OP_32_64(st16):
+        tcg_out_modrm_offset(s, OPC_MOVL_EvGv | P_DATA16,
+                             args[0], args[1], args[2]);
+        break;
+#if TCG_TARGET_REG_BITS == 64
+    case INDEX_op_st32_i64:
+#endif
     case INDEX_op_st_i32:
-        /* movl */
-        tcg_out_modrm_offset(s, 0x89, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_sub_i32:
+        tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]);
+        break;
+
+    OP_32_64(add):
+        /* For 3-operand addition, use LEA.  */
+        if (args[0] != args[1]) {
+            TCGArg a0 = args[0], a1 = args[1], a2 = args[2], c3 = 0;
+
+            if (const_args[2]) {
+                c3 = a2, a2 = -1;
+            } else if (a0 == a2) {
+                /* Watch out for dest = src + dest, since we've removed
+                   the matching constraint on the add.  */
+                tgen_arithr(s, ARITH_ADD + rexw, a0, a1);
+                break;
+            }
+
+            tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, a2, 0, c3);
+            break;
+        }
+        c = ARITH_ADD;
+        goto gen_arith;
+    OP_32_64(sub):
         c = ARITH_SUB;
         goto gen_arith;
-    case INDEX_op_and_i32:
+    OP_32_64(and):
         c = ARITH_AND;
         goto gen_arith;
-    case INDEX_op_or_i32:
+    OP_32_64(or):
         c = ARITH_OR;
         goto gen_arith;
-    case INDEX_op_xor_i32:
+    OP_32_64(xor):
         c = ARITH_XOR;
         goto gen_arith;
-    case INDEX_op_add_i32:
-        c = ARITH_ADD;
     gen_arith:
         if (const_args[2]) {
-            tgen_arithi(s, c, args[0], args[2], 0);
-        } else {
-            tcg_out_modrm(s, 0x01 | (c << 3), args[2], args[0]);
-        }
-        break;
-    case INDEX_op_mul_i32:
+            tgen_arithi(s, c + rexw, args[0], args[2], 0);
+        } else {
+            tgen_arithr(s, c + rexw, args[0], args[2]);
+        }
+        break;
+
+    OP_32_64(mul):
         if (const_args[2]) {
             int32_t val;
             val = args[2];
             if (val == (int8_t)val) {
-                tcg_out_modrm(s, 0x6b, args[0], args[0]);
+                tcg_out_modrm(s, OPC_IMUL_GvEvIb + rexw, args[0], args[0]);
                 tcg_out8(s, val);
             } else {
-                tcg_out_modrm(s, 0x69, args[0], args[0]);
+                tcg_out_modrm(s, OPC_IMUL_GvEvIz + rexw, args[0], args[0]);
                 tcg_out32(s, val);
             }
         } else {
-            tcg_out_modrm(s, 0xaf | P_EXT, args[0], args[2]);
-        }
-        break;
-    case INDEX_op_mulu2_i32:
-        tcg_out_modrm(s, 0xf7, 4, args[3]);
-        break;
-    case INDEX_op_div2_i32:
-        tcg_out_modrm(s, 0xf7, 7, args[4]);
-        break;
-    case INDEX_op_divu2_i32:
-        tcg_out_modrm(s, 0xf7, 6, args[4]);
-        break;
-    case INDEX_op_shl_i32:
+            tcg_out_modrm(s, OPC_IMUL_GvEv + rexw, args[0], args[2]);
+        }
+        break;
+
+    OP_32_64(div2):
+        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IDIV, args[4]);
+        break;
+    OP_32_64(divu2):
+        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_DIV, args[4]);
+        break;
+
+    OP_32_64(shl):
         c = SHIFT_SHL;
-    gen_shift32:
+        goto gen_shift;
+    OP_32_64(shr):
+        c = SHIFT_SHR;
+        goto gen_shift;
+    OP_32_64(sar):
+        c = SHIFT_SAR;
+        goto gen_shift;
+    OP_32_64(rotl):
+        c = SHIFT_ROL;
+        goto gen_shift;
+    OP_32_64(rotr):
+        c = SHIFT_ROR;
+        goto gen_shift;
+    gen_shift:
         if (const_args[2]) {
-            if (args[2] == 1) {
-                tcg_out_modrm(s, 0xd1, c, args[0]);
-            } else {
-                tcg_out_modrm(s, 0xc1, c, args[0]);
-                tcg_out8(s, args[2]);
-            }
-        } else {
-            tcg_out_modrm(s, 0xd3, c, args[0]);
-        }
-        break;
-    case INDEX_op_shr_i32:
-        c = SHIFT_SHR;
-        goto gen_shift32;
-    case INDEX_op_sar_i32:
-        c = SHIFT_SAR;
-        goto gen_shift32;
-    case INDEX_op_rotl_i32:
-        c = SHIFT_ROL;
-        goto gen_shift32;
-    case INDEX_op_rotr_i32:
-        c = SHIFT_ROR;
-        goto gen_shift32;
-
-    case INDEX_op_add2_i32:
-        if (const_args[4])
-            tgen_arithi(s, ARITH_ADD, args[0], args[4], 1);
-        else
-            tcg_out_modrm(s, 0x01 | (ARITH_ADD << 3), args[4], args[0]);
-        if (const_args[5])
-            tgen_arithi(s, ARITH_ADC, args[1], args[5], 1);
-        else
-            tcg_out_modrm(s, 0x01 | (ARITH_ADC << 3), args[5], args[1]);
-        break;
-    case INDEX_op_sub2_i32:
-        if (const_args[4])
-            tgen_arithi(s, ARITH_SUB, args[0], args[4], 1);
-        else
-            tcg_out_modrm(s, 0x01 | (ARITH_SUB << 3), args[4], args[0]);
-        if (const_args[5])
-            tgen_arithi(s, ARITH_SBB, args[1], args[5], 1);
-        else
-            tcg_out_modrm(s, 0x01 | (ARITH_SBB << 3), args[5], args[1]);
-        break;
+            tcg_out_shifti(s, c + rexw, args[0], args[2]);
+        } else {
+            tcg_out_modrm(s, OPC_SHIFT_cl + rexw, c, args[0]);
+        }
+        break;
+
     case INDEX_op_brcond_i32:
-        tcg_out_brcond(s, args[2], args[0], args[1], const_args[1], args[3]);
-        break;
-    case INDEX_op_brcond2_i32:
-        tcg_out_brcond2(s, args, const_args);
-        break;
-
-    case INDEX_op_bswap16_i32:
-        tcg_out8(s, 0x66);
-        tcg_out_modrm(s, 0xc1, SHIFT_ROL, args[0]);
-        tcg_out8(s, 8);
-        break;
-    case INDEX_op_bswap32_i32:
-        tcg_out_opc(s, (0xc8 + args[0]) | P_EXT);
-        break;
-
-    case INDEX_op_neg_i32:
-        tcg_out_modrm(s, 0xf7, 3, args[0]);
-        break;
-
-    case INDEX_op_not_i32:
-        tcg_out_modrm(s, 0xf7, 2, args[0]);
-        break;
-
-    case INDEX_op_ext8s_i32:
-        tcg_out_modrm(s, 0xbe | P_EXT, args[0], args[1]);
-        break;
-    case INDEX_op_ext16s_i32:
-        tcg_out_modrm(s, 0xbf | P_EXT, args[0], args[1]);
-        break;
-    case INDEX_op_ext8u_i32:
-        tcg_out_modrm(s, 0xb6 | P_EXT, args[0], args[1]);
-        break;
-    case INDEX_op_ext16u_i32:
-        tcg_out_modrm(s, 0xb7 | P_EXT, args[0], args[1]);
+        tcg_out_brcond32(s, args[2], args[0], args[1], const_args[1],
+                         args[3], 0);
+        break;
+    case INDEX_op_setcond_i32:
+        tcg_out_setcond32(s, args[3], args[0], args[1],
+                          args[2], const_args[2]);
+        break;
+
+    OP_32_64(bswap16):
+        tcg_out_rolw_8(s, args[0]);
+        break;
+    OP_32_64(bswap32):
+        tcg_out_bswap32(s, args[0]);
+        break;
+
+    OP_32_64(neg):
+        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NEG, args[0]);
+        break;
+    OP_32_64(not):
+        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, args[0]);
+        break;
+
+    OP_32_64(ext8s):
+        tcg_out_ext8s(s, args[0], args[1], rexw);
+        break;
+    OP_32_64(ext16s):
+        tcg_out_ext16s(s, args[0], args[1], rexw);
+        break;
+    OP_32_64(ext8u):
+        tcg_out_ext8u(s, args[0], args[1]);
+        break;
+    OP_32_64(ext16u):
+        tcg_out_ext16u(s, args[0], args[1]);
         break;
 
@@ -1285 +1788 @@
         tcg_out_qemu_ld(s, args, 1 | 4);
         break;
+#if TCG_TARGET_REG_BITS == 64
     case INDEX_op_qemu_ld32u:
+#endif
+    case INDEX_op_qemu_ld32:
         tcg_out_qemu_ld(s, args, 2);
         break;
@@ -1305 +1811 @@
         break;
 
+#if TCG_TARGET_REG_BITS == 32
+    case INDEX_op_brcond2_i32:
+        tcg_out_brcond2(s, args, const_args, 0);
+        break;
+    case INDEX_op_setcond2_i32:
+        tcg_out_setcond2(s, args, const_args);
+        break;
+    case INDEX_op_mulu2_i32:
+        tcg_out_modrm(s, OPC_GRP3_Ev, EXT3_MUL, args[3]);
+        break;
+    case INDEX_op_add2_i32:
+        if (const_args[4]) {
+            tgen_arithi(s, ARITH_ADD, args[0], args[4], 1);
+        } else {
+            tgen_arithr(s, ARITH_ADD, args[0], args[4]);
+        }
+        if (const_args[5]) {
+            tgen_arithi(s, ARITH_ADC, args[1], args[5], 1);
+        } else {
+            tgen_arithr(s, ARITH_ADC, args[1], args[5]);
+        }
+        break;
+    case INDEX_op_sub2_i32:
+        if (const_args[4]) {
+            tgen_arithi(s, ARITH_SUB, args[0], args[4], 1);
+        } else {
+            tgen_arithr(s, ARITH_SUB, args[0], args[4]);
+        }
+        if (const_args[5]) {
+            tgen_arithi(s, ARITH_SBB, args[1], args[5], 1);
+        } else {
+            tgen_arithr(s, ARITH_SBB, args[1], args[5]);
+        }
+        break;
+#else /* TCG_TARGET_REG_BITS == 64 */
+    case INDEX_op_movi_i64:
+        tcg_out_movi(s, TCG_TYPE_I64, args[0], args[1]);
+        break;
+    case INDEX_op_ld32s_i64:
+        tcg_out_modrm_offset(s, OPC_MOVSLQ, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_ld_i64:
+        tcg_out_ld(s, TCG_TYPE_I64, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_st_i64:
+        tcg_out_st(s, TCG_TYPE_I64, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_qemu_ld32s:
+        tcg_out_qemu_ld(s, args, 2 | 4);
+        break;
+
+    case INDEX_op_brcond_i64:
+        tcg_out_brcond64(s, args[2], args[0], args[1], const_args[1],
+                         args[3], 0);
+        break;
+    case INDEX_op_setcond_i64:
+        tcg_out_setcond64(s, args[3], args[0], args[1],
+                          args[2], const_args[2]);
+        break;
+
+    case INDEX_op_bswap64_i64:
+        tcg_out_bswap64(s, args[0]);
+        break;
+    case INDEX_op_ext32u_i64:
+        tcg_out_ext32u(s, args[0], args[1]);
+        break;
+    case INDEX_op_ext32s_i64:
+        tcg_out_ext32s(s, args[0], args[1]);
+        break;
+#endif
+
     default:
         tcg_abort();
     }
+
+#undef OP_32_64
 }
 
 static const TCGTargetOpDef x86_op_defs[] = {
-    { INDEX_op_exit_tb, {"", "" } },
-    { INDEX_op_goto_tb, {"", "" } },
-    { INDEX_op_call, { "ri", "", } },
-    { INDEX_op_jmp, { "ri", ""} },
-    { INDEX_op_br, {"", "" } },
+    { INDEX_op_exit_tb, { } },
+    { INDEX_op_goto_tb, { } },
+    { INDEX_op_call, { "ri" } },
+    { INDEX_op_jmp, { "ri" } },
+    { INDEX_op_br, { } },
     { INDEX_op_mov_i32, { "r", "r" } },
     { INDEX_op_movi_i32, { "r" } },
@@ -1327 +1906 @@
     { INDEX_op_st_i32, { "r", "r" } },
 
-    { INDEX_op_add_i32, { "r", "0", "ri" } },
+    { INDEX_op_add_i32, { "r", "r", "ri" } },
     { INDEX_op_sub_i32, { "r", "0", "ri" } },
     { INDEX_op_mul_i32, { "r", "0", "ri" } },
-    { INDEX_op_mulu2_i32, { "a", "d", "a", "r" } },
     { INDEX_op_div2_i32, { "a", "d", "0", "1", "r" } },
     { INDEX_op_divu2_i32, { "a", "d", "0", "1", "r" } },
@@ -1345 +1923 @@
     { INDEX_op_brcond_i32, { "r", "ri" } },
 
+    { INDEX_op_bswap16_i32, { "r", "0" } },
+    { INDEX_op_bswap32_i32, { "r", "0" } },
+
+    { INDEX_op_neg_i32, { "r", "0" } },
+
+    { INDEX_op_not_i32, { "r", "0" } },
+
+    { INDEX_op_ext8s_i32, { "r", "q" } },
+    { INDEX_op_ext16s_i32, { "r", "r" } },
+    { INDEX_op_ext8u_i32, { "r", "q" } },
+    { INDEX_op_ext16u_i32, { "r", "r" } },
+
+    { INDEX_op_setcond_i32, { "q", "r", "ri" } },
+
+#if TCG_TARGET_REG_BITS == 32
+    { INDEX_op_mulu2_i32, { "a", "d", "a", "r" } },
     { INDEX_op_add2_i32, { "r", "r", "0", "1", "ri", "ri" } },
     { INDEX_op_sub2_i32, { "r", "r", "0", "1", "ri", "ri" } },
     { INDEX_op_brcond2_i32, { "r", "r", "ri", "ri" } },
-
-    { INDEX_op_bswap16_i32, { "r", "0" } },
-    { INDEX_op_bswap32_i32, { "r", "0" } },
-
-    { INDEX_op_neg_i32, { "r", "0" } },
-
-    { INDEX_op_not_i32, { "r", "0" } },
-
-    { INDEX_op_ext8s_i32, { "r", "q" } },
-    { INDEX_op_ext16s_i32, { "r", "r" } },
-    { INDEX_op_ext8u_i32, { "r", "q"} },
-    { INDEX_op_ext16u_i32, { "r", "r"} },
-
-#if TARGET_LONG_BITS == 32
+    { INDEX_op_setcond2_i32, { "r", "r", "r", "ri", "ri" } },
+#else
+    { INDEX_op_mov_i64, { "r", "r" } },
+    { INDEX_op_movi_i64, { "r" } },
+    { INDEX_op_ld8u_i64, { "r", "r" } },
+    { INDEX_op_ld8s_i64, { "r", "r" } },
+    { INDEX_op_ld16u_i64, { "r", "r" } },
+    { INDEX_op_ld16s_i64, { "r", "r" } },
+    { INDEX_op_ld32u_i64, { "r", "r" } },
+    { INDEX_op_ld32s_i64, { "r", "r" } },
+    { INDEX_op_ld_i64, { "r", "r" } },
+    { INDEX_op_st8_i64, { "r", "r" } },
+    { INDEX_op_st16_i64, { "r", "r" } },
+    { INDEX_op_st32_i64, { "r", "r" } },
+    { INDEX_op_st_i64, { "r", "r" } },
+
+    { INDEX_op_add_i64, { "r", "0", "re" } },
+    { INDEX_op_mul_i64, { "r", "0", "re" } },
+    { INDEX_op_div2_i64, { "a", "d", "0", "1", "r" } },
+    { INDEX_op_divu2_i64, { "a", "d", "0", "1", "r" } },
+    { INDEX_op_sub_i64, { "r", "0", "re" } },
+    { INDEX_op_and_i64, { "r", "0", "reZ" } },
+    { INDEX_op_or_i64, { "r", "0", "re" } },
+    { INDEX_op_xor_i64, { "r", "0", "re" } },
+
+    { INDEX_op_shl_i64, { "r", "0", "ci" } },
+    { INDEX_op_shr_i64, { "r", "0", "ci" } },
+    { INDEX_op_sar_i64, { "r", "0", "ci" } },
+    { INDEX_op_rotl_i64, { "r", "0", "ci" } },
+    { INDEX_op_rotr_i64, { "r", "0", "ci" } },
+
+    { INDEX_op_brcond_i64, { "r", "re" } },
+    { INDEX_op_setcond_i64, { "r", "r", "re" } },
+
+    { INDEX_op_bswap16_i64, { "r", "0" } },
+    { INDEX_op_bswap32_i64, { "r", "0" } },
+    { INDEX_op_bswap64_i64, { "r", "0" } },
+    { INDEX_op_neg_i64, { "r", "0" } },
+    { INDEX_op_not_i64, { "r", "0" } },
+
+    { INDEX_op_ext8s_i64, { "r", "r" } },
+    { INDEX_op_ext16s_i64, { "r", "r" } },
+    { INDEX_op_ext32s_i64, { "r", "r" } },
+    { INDEX_op_ext8u_i64, { "r", "r" } },
+    { INDEX_op_ext16u_i64, { "r", "r" } },
+    { INDEX_op_ext32u_i64, { "r", "r" } },
+#endif
+
+#if TCG_TARGET_REG_BITS == 64
     { INDEX_op_qemu_ld8u, { "r", "L" } },
     { INDEX_op_qemu_ld8s, { "r", "L" } },
     { INDEX_op_qemu_ld16u, { "r", "L" } },
     { INDEX_op_qemu_ld16s, { "r", "L" } },
+    { INDEX_op_qemu_ld32, { "r", "L" } },
     { INDEX_op_qemu_ld32u, { "r", "L" } },
+    { INDEX_op_qemu_ld32s, { "r", "L" } },
+    { INDEX_op_qemu_ld64, { "r", "L" } },
+
+    { INDEX_op_qemu_st8, { "L", "L" } },
+    { INDEX_op_qemu_st16, { "L", "L" } },
+    { INDEX_op_qemu_st32, { "L", "L" } },
+    { INDEX_op_qemu_st64, { "L", "L" } },
+#elif TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
+    { INDEX_op_qemu_ld8u, { "r", "L" } },
+    { INDEX_op_qemu_ld8s, { "r", "L" } },
+    { INDEX_op_qemu_ld16u, { "r", "L" } },
+    { INDEX_op_qemu_ld16s, { "r", "L" } },
+    { INDEX_op_qemu_ld32, { "r", "L" } },
     { INDEX_op_qemu_ld64, { "r", "r", "L" } },
 
@@ -1378 +2021 @@
     { INDEX_op_qemu_ld16u, { "r", "L", "L" } },
     { INDEX_op_qemu_ld16s, { "r", "L", "L" } },
-    { INDEX_op_qemu_ld32u, { "r", "L", "L" } },
+    { INDEX_op_qemu_ld32, { "r", "L", "L" } },
     { INDEX_op_qemu_ld64, { "r", "r", "L", "L" } },
 
@@ -1386 +2029 @@
     { INDEX_op_qemu_st64, { "L", "L", "L", "L" } },
 #endif
-#ifndef VBOX
     { -1 },
+};
+
+static int tcg_target_callee_save_regs[] = {
+#if TCG_TARGET_REG_BITS == 64
+    TCG_REG_RBP,
+    TCG_REG_RBX,
+    TCG_REG_R12,
+    TCG_REG_R13,
+    /* TCG_REG_R14, */ /* Currently used for the global env. */
+    TCG_REG_R15,
 #else
-    { -1, {"", "", "", ""} },
-#endif
-};
-
-static int tcg_target_callee_save_regs[] = {
-#ifndef VBOX
-    /*    TCG_REG_EBP, */ /* currently used for the global env, so no
-                             need to save */
+# ifndef VBOX
+    /* TCG_REG_EBP, */ /* Currently used for the global env. */
     TCG_REG_EBX,
     TCG_REG_ESI,
     TCG_REG_EDI,
-#else
+# else
     TCG_REG_EBP,
     TCG_REG_EBX,
-    /* TCG_REG_ESI, */ /* currently used for the global env, so no
-                          need to save */
+    /* TCG_REG_ESI, */ /* Currently used for the global env. */
     TCG_REG_EDI,
+# endif
 #endif
 };
 
-static inline void tcg_out_push(TCGContext *s, int reg)
-{
-    tcg_out_opc(s, 0x50 + reg);
-}
-
-static inline void tcg_out_pop(TCGContext *s, int reg)
-{
-    tcg_out_opc(s, 0x58 + reg);
-}
-
 /* Generate global QEMU prologue and epilogue code */
-void tcg_target_qemu_prologue(TCGContext *s)
+static void tcg_target_qemu_prologue(TCGContext *s)
 {
     int i, frame_size, push_size, stack_addend;
 
     /* TB prologue */
-    /* save all callee saved registers */
-    for(i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) {
+
+    /* Save all callee saved registers.  */
+    for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) {
         tcg_out_push(s, tcg_target_callee_save_regs[i]);
     }
-    /* reserve some stack space */
-    push_size = 4 + ARRAY_SIZE(tcg_target_callee_save_regs) * 4;
+# if defined(VBOX_STRICT) && defined(RT_ARCH_X86)
+    tcg_out8(s, 0x31); /* xor ebp, ebp */
+    tcg_out8(s, 0xed);
+# endif
+
+    /* Reserve some stack space.  */
+    push_size = 1 + ARRAY_SIZE(tcg_target_callee_save_regs);
+    push_size *= TCG_TARGET_REG_BITS / 8;
+
     frame_size = push_size + TCG_STATIC_CALL_ARGS_SIZE;
     frame_size = (frame_size + TCG_TARGET_STACK_ALIGN - 1) &
@@ -1436 +2080 @@
     stack_addend = frame_size - push_size;
     tcg_out_addi(s, TCG_REG_ESP, -stack_addend);
+
+    /* jmp *tb.  */
+    tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, tcg_target_call_iarg_regs[0]);
 # ifdef VBOX
     tcg_gen_stack_alignment_check(s);
@@ -1444 +2091 @@
     /* TB epilogue */
     tb_ret_addr = s->code_ptr;
+
     tcg_out_addi(s, TCG_REG_ESP, stack_addend);
-    for(i = ARRAY_SIZE(tcg_target_callee_save_regs) - 1; i >= 0; i--) {
+
+    for (i = ARRAY_SIZE(tcg_target_callee_save_regs) - 1; i >= 0; i--) {
         tcg_out_pop(s, tcg_target_callee_save_regs[i]);
     }
-    tcg_out8(s, 0xc3); /* ret */
-}
-
-void tcg_target_init(TCGContext *s)
-{
+    tcg_out_opc(s, OPC_RET, 0, 0, 0);
+}
+
+static void tcg_target_init(TCGContext *s)
+{
+#if !defined(CONFIG_USER_ONLY)
     /* fail safe */
     if ((1 << CPU_TLB_ENTRY_BITS) != sizeof(CPUTLBEntry))
         tcg_abort();
-
-    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xff);
-    tcg_regset_set32(tcg_target_call_clobber_regs, 0,
-                     (1 << TCG_REG_EAX) |
-                     (1 << TCG_REG_EDX) |
-                     (1 << TCG_REG_ECX));
+#endif
+
+    if (TCG_TARGET_REG_BITS == 64) {
+        tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffff);
+        tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I64], 0, 0xffff);
+    } else {
+        tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xff);
+    }
+
+    tcg_regset_clear(tcg_target_call_clobber_regs);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_EAX);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_EDX);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_ECX);
+    if (TCG_TARGET_REG_BITS == 64) {
+        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_RDI);
+        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_RSI);
+        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R8);
+        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R9);
+        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R10);
+        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R11);
+    }
 
     tcg_regset_clear(s->reserved_regs);

trunk/src/recompiler/tcg/i386/tcg-target.h

@@ -24 +24 @@
 #define TCG_TARGET_I386 1
 
-#define TCG_TARGET_REG_BITS 32
+#if defined(__x86_64__)
+# define TCG_TARGET_REG_BITS 64
+#else
+# define TCG_TARGET_REG_BITS 32
+#endif
 //#define TCG_TARGET_WORDS_BIGENDIAN
 
-#define TCG_TARGET_NB_REGS 8
+#if TCG_TARGET_REG_BITS == 64
+# define TCG_TARGET_NB_REGS 16
+#else
+# define TCG_TARGET_NB_REGS 8
+#endif
 
 enum {
@@ -38 +46 @@
     TCG_REG_ESI,
     TCG_REG_EDI,
+
+    /* 64-bit registers; always define the symbols to avoid
+       too much if-deffing.  */
+    TCG_REG_R8,
+    TCG_REG_R9,
+    TCG_REG_R10,
+    TCG_REG_R11,
+    TCG_REG_R12,
+    TCG_REG_R13,
+    TCG_REG_R14,
+    TCG_REG_R15,
+    TCG_REG_RAX = TCG_REG_EAX,
+    TCG_REG_RCX = TCG_REG_ECX,
+    TCG_REG_RDX = TCG_REG_EDX,
+    TCG_REG_RBX = TCG_REG_EBX,
+    TCG_REG_RSP = TCG_REG_ESP,
+    TCG_REG_RBP = TCG_REG_EBP,
+    TCG_REG_RSI = TCG_REG_ESI,
+    TCG_REG_RDI = TCG_REG_EDI,
 };
+
+#define TCG_CT_CONST_S32 0x100
+#define TCG_CT_CONST_U32 0x200
 
 /* used for function call generation */
@@ -46 +76 @@
 
 /* optional instructions */
+#define TCG_TARGET_HAS_div2_i32
+#define TCG_TARGET_HAS_rot_i32
+#define TCG_TARGET_HAS_ext8s_i32
+#define TCG_TARGET_HAS_ext16s_i32
+#define TCG_TARGET_HAS_ext8u_i32
+#define TCG_TARGET_HAS_ext16u_i32
 #define TCG_TARGET_HAS_bswap16_i32
 #define TCG_TARGET_HAS_bswap32_i32
 #define TCG_TARGET_HAS_neg_i32
 #define TCG_TARGET_HAS_not_i32
-#define TCG_TARGET_HAS_ext8s_i32
-#define TCG_TARGET_HAS_ext16s_i32
-#define TCG_TARGET_HAS_rot_i32
-#define TCG_TARGET_HAS_ext8u_i32
-#define TCG_TARGET_HAS_ext16u_i32
+// #define TCG_TARGET_HAS_andc_i32
+// #define TCG_TARGET_HAS_orc_i32
+// #define TCG_TARGET_HAS_eqv_i32
+// #define TCG_TARGET_HAS_nand_i32
+// #define TCG_TARGET_HAS_nor_i32
+
+#if TCG_TARGET_REG_BITS == 64
+#define TCG_TARGET_HAS_div2_i64
+#define TCG_TARGET_HAS_rot_i64
+#define TCG_TARGET_HAS_ext8s_i64
+#define TCG_TARGET_HAS_ext16s_i64
+#define TCG_TARGET_HAS_ext32s_i64
+#define TCG_TARGET_HAS_ext8u_i64
+#define TCG_TARGET_HAS_ext16u_i64
+#define TCG_TARGET_HAS_ext32u_i64
+#define TCG_TARGET_HAS_bswap16_i64
+#define TCG_TARGET_HAS_bswap32_i64
+#define TCG_TARGET_HAS_bswap64_i64
+#define TCG_TARGET_HAS_neg_i64
+#define TCG_TARGET_HAS_not_i64
+// #define TCG_TARGET_HAS_andc_i64
+// #define TCG_TARGET_HAS_orc_i64
+// #define TCG_TARGET_HAS_eqv_i64
+// #define TCG_TARGET_HAS_nand_i64
+// #define TCG_TARGET_HAS_nor_i64
+#endif
 
 #define TCG_TARGET_HAS_GUEST_BASE
 
 /* Note: must be synced with dyngen-exec.h */
-#ifndef VBOX
-#define TCG_AREG0 TCG_REG_EBP
-#define TCG_AREG1 TCG_REG_EBX
-#define TCG_AREG2 TCG_REG_ESI
+#if TCG_TARGET_REG_BITS == 64
+# define TCG_AREG0 TCG_REG_R14
 #else
-# define TCG_AREG0 TCG_REG_ESI
-# define TCG_AREG1 TCG_REG_EDI
+# ifndef VBOX /* we're using ESI instead of EBP, probably due to frame pointer opt issues */
+# define TCG_AREG0 TCG_REG_EBP
+# else  /* VBOX */
+#  define TCG_AREG0 TCG_REG_ESI
+# endif /* VBOX */
 #endif
 

trunk/src/recompiler/tcg/tcg-op.h

@@ -26 +26 @@
 int gen_new_label(void);
 
-static inline void tcg_gen_op1_i32(int opc, TCGv_i32 arg1)
+static inline void tcg_gen_op1_i32(TCGOpcode opc, TCGv_i32 arg1)
 {
     *gen_opc_ptr++ = opc;
@@ -32 +32 @@
 }
 
-static inline void tcg_gen_op1_i64(int opc, TCGv_i64 arg1)
+static inline void tcg_gen_op1_i64(TCGOpcode opc, TCGv_i64 arg1)
 {
     *gen_opc_ptr++ = opc;
@@ -38 +38 @@
 }
 
-static inline void tcg_gen_op1i(int opc, TCGArg arg1)
+static inline void tcg_gen_op1i(TCGOpcode opc, TCGArg arg1)
 {
     *gen_opc_ptr++ = opc;
@@ -44 +44 @@
 }
 
-static inline void tcg_gen_op2_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2)
+static inline void tcg_gen_op2_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2)
 {
     *gen_opc_ptr++ = opc;
@@ -51 +51 @@
 }
 
-static inline void tcg_gen_op2_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2)
+static inline void tcg_gen_op2_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2)
 {
     *gen_opc_ptr++ = opc;
@@ -58 +58 @@
 }
 
-static inline void tcg_gen_op2i_i32(int opc, TCGv_i32 arg1, TCGArg arg2)
+static inline void tcg_gen_op2i_i32(TCGOpcode opc, TCGv_i32 arg1, TCGArg arg2)
 {
     *gen_opc_ptr++ = opc;
@@ -65 +65 @@
 }
 
-static inline void tcg_gen_op2i_i64(int opc, TCGv_i64 arg1, TCGArg arg2)
+static inline void tcg_gen_op2i_i64(TCGOpcode opc, TCGv_i64 arg1, TCGArg arg2)
 {
     *gen_opc_ptr++ = opc;
@@ -72 +72 @@
 }
 
-static inline void tcg_gen_op2ii(int opc, TCGArg arg1, TCGArg arg2)
+static inline void tcg_gen_op2ii(TCGOpcode opc, TCGArg arg1, TCGArg arg2)
 {
     *gen_opc_ptr++ = opc;
@@ -79 +79 @@
 }
 
-static inline void tcg_gen_op3_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2,
+static inline void tcg_gen_op3_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                    TCGv_i32 arg3)
 {
@@ -88 +88 @@
 }
 
-static inline void tcg_gen_op3_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2,
+static inline void tcg_gen_op3_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                    TCGv_i64 arg3)
 {
@@ -97 +97 @@
 }
 
-static inline void tcg_gen_op3i_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2,
-                                    TCGArg arg3)
+static inline void tcg_gen_op3i_i32(TCGOpcode opc, TCGv_i32 arg1,
+                                    TCGv_i32 arg2, TCGArg arg3)
 {
     *gen_opc_ptr++ = opc;
@@ -106 +106 @@
 }
 
-static inline void tcg_gen_op3i_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2,
-                                    TCGArg arg3)
+static inline void tcg_gen_op3i_i64(TCGOpcode opc, TCGv_i64 arg1,
+                                    TCGv_i64 arg2, TCGArg arg3)
 {
     *gen_opc_ptr++ = opc;
@@ -115 +115 @@
 }
 
-static inline void tcg_gen_ldst_op_i32(int opc, TCGv_i32 val, TCGv_ptr base,
-                                       TCGArg offset)
+static inline void tcg_gen_ldst_op_i32(TCGOpcode opc, TCGv_i32 val,
+                                       TCGv_ptr base, TCGArg offset)
 {
     *gen_opc_ptr++ = opc;
@@ -124 +124 @@
 }
 
-static inline void tcg_gen_ldst_op_i64(int opc, TCGv_i64 val, TCGv_ptr base,
-                                       TCGArg offset)
+static inline void tcg_gen_ldst_op_i64(TCGOpcode opc, TCGv_i64 val,
+                                       TCGv_ptr base, TCGArg offset)
 {
     *gen_opc_ptr++ = opc;
@@ -133 +133 @@
 }
 
-static inline void tcg_gen_qemu_ldst_op_i64_i32(int opc, TCGv_i64 val, TCGv_i32 addr,
-                                                TCGArg mem_index)
+static inline void tcg_gen_qemu_ldst_op_i64_i32(TCGOpcode opc, TCGv_i64 val,
+                                                TCGv_i32 addr, TCGArg mem_index)
 {
     *gen_opc_ptr++ = opc;
@@ -142 +142 @@
 }
 
-static inline void tcg_gen_qemu_ldst_op_i64_i64(int opc, TCGv_i64 val, TCGv_i64 addr,
-                                                TCGArg mem_index)
+static inline void tcg_gen_qemu_ldst_op_i64_i64(TCGOpcode opc, TCGv_i64 val,
+                                                TCGv_i64 addr, TCGArg mem_index)
 {
     *gen_opc_ptr++ = opc;
@@ -151 +151 @@
 }
 
-static inline void tcg_gen_op4_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2,
+static inline void tcg_gen_op4_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                    TCGv_i32 arg3, TCGv_i32 arg4)
 {
@@ -161 +161 @@
 }
 
-static inline void tcg_gen_op4_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2,
+static inline void tcg_gen_op4_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                    TCGv_i64 arg3, TCGv_i64 arg4)
 {
@@ -171 +171 @@
 }
 
-static inline void tcg_gen_op4i_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2,
+static inline void tcg_gen_op4i_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                     TCGv_i32 arg3, TCGArg arg4)
 {
@@ -181 +181 @@
 }
 
-static inline void tcg_gen_op4i_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2,
+static inline void tcg_gen_op4i_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                     TCGv_i64 arg3, TCGArg arg4)
 {
@@ -191 +191 @@
 }
 
-static inline void tcg_gen_op4ii_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2,
+static inline void tcg_gen_op4ii_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                      TCGArg arg3, TCGArg arg4)
 {
@@ -201 +201 @@
 }
 
-static inline void tcg_gen_op4ii_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2,
+static inline void tcg_gen_op4ii_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                      TCGArg arg3, TCGArg arg4)
 {
@@ -211 +211 @@
 }
 
-static inline void tcg_gen_op5_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2,
+static inline void tcg_gen_op5_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                    TCGv_i32 arg3, TCGv_i32 arg4, TCGv_i32 arg5)
 {
@@ -222 +222 @@
 }
 
-static inline void tcg_gen_op5_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2,
+static inline void tcg_gen_op5_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                    TCGv_i64 arg3, TCGv_i64 arg4, TCGv_i64 arg5)
 {
@@ -233 +233 @@
 }
 
-static inline void tcg_gen_op5i_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2,
+static inline void tcg_gen_op5i_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                     TCGv_i32 arg3, TCGv_i32 arg4, TCGArg arg5)
 {
@@ -244 +244 @@
 }
 
-static inline void tcg_gen_op5i_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2,
+static inline void tcg_gen_op5i_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                     TCGv_i64 arg3, TCGv_i64 arg4, TCGArg arg5)
 {
@@ -255 +255 @@
 }
 
-static inline void tcg_gen_op6_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2,
+static inline void tcg_gen_op6_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                    TCGv_i32 arg3, TCGv_i32 arg4, TCGv_i32 arg5,
                                    TCGv_i32 arg6)
@@ -268 +268 @@
 }
 
-static inline void tcg_gen_op6_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2,
+static inline void tcg_gen_op6_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                    TCGv_i64 arg3, TCGv_i64 arg4, TCGv_i64 arg5,
                                    TCGv_i64 arg6)
@@ -281 +281 @@
 }
 
-static inline void tcg_gen_op6ii_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2,
-                                     TCGv_i32 arg3, TCGv_i32 arg4, TCGArg arg5,
-                                     TCGArg arg6)
+static inline void tcg_gen_op6i_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
+                                    TCGv_i32 arg3, TCGv_i32 arg4,
+                                    TCGv_i32 arg5, TCGArg arg6)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg4);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg5);
+    *gen_opparam_ptr++ = arg6;
+}
+
+static inline void tcg_gen_op6i_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
+                                    TCGv_i64 arg3, TCGv_i64 arg4,
+                                    TCGv_i64 arg5, TCGArg arg6)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg4);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg5);
+    *gen_opparam_ptr++ = arg6;
+}
+
+static inline void tcg_gen_op6ii_i32(TCGOpcode opc, TCGv_i32 arg1,
+                                     TCGv_i32 arg2, TCGv_i32 arg3,
+                                     TCGv_i32 arg4, TCGArg arg5, TCGArg arg6)
 {
     *gen_opc_ptr++ = opc;
@@ -294 +320 @@
 }
 
-static inline void tcg_gen_op6ii_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2,
-                                     TCGv_i64 arg3, TCGv_i64 arg4, TCGArg arg5,
-                                     TCGArg arg6)
+static inline void tcg_gen_op6ii_i64(TCGOpcode opc, TCGv_i64 arg1,
+                                     TCGv_i64 arg2, TCGv_i64 arg3,
+                                     TCGv_i64 arg4, TCGArg arg5, TCGArg arg6)
 {
     *gen_opc_ptr++ = opc;
@@ -328 +354 @@
 }
 
+/* A version of dh_sizemask from def-helper.h that doesn't rely on
+   preprocessor magic.  */
+static inline int tcg_gen_sizemask(int n, int is_64bit, int is_signed)
+{
+    return (is_64bit << n*2) | (is_signed << (n*2 + 1));
+}
+
 /* helper calls */
 static inline void tcg_gen_helperN(void *func, int flags, int sizemask,
@@ -339 +372 @@
 }
 
-/* FIXME: Should this be pure?  */
-static inline void tcg_gen_helper64(void *func, TCGv_i64 ret,
+/* Note: Both tcg_gen_helper32() and tcg_gen_helper64() are currently
+   reserved for helpers in tcg-runtime.c. These helpers are all const
+   and pure, hence the call to tcg_gen_callN() with TCG_CALL_CONST |
+   TCG_CALL_PURE. This may need to be adjusted if these functions
+   start to be used with other helpers. */
+static inline void tcg_gen_helper32(void *func, int sizemask, TCGv_i32 ret,
+                                    TCGv_i32 a, TCGv_i32 b)
+{
+    TCGv_ptr fn;
+    TCGArg args[2];
+    fn = tcg_const_ptr((tcg_target_long)func);
+    args[0] = GET_TCGV_I32(a);
+    args[1] = GET_TCGV_I32(b);
+    tcg_gen_callN(&tcg_ctx, fn, TCG_CALL_CONST | TCG_CALL_PURE, sizemask,
+                  GET_TCGV_I32(ret), 2, args);
+    tcg_temp_free_ptr(fn);
+}
+
+static inline void tcg_gen_helper64(void *func, int sizemask, TCGv_i64 ret,
                                     TCGv_i64 a, TCGv_i64 b)
 {
@@ -348 +398 @@
     args[0] = GET_TCGV_I64(a);
     args[1] = GET_TCGV_I64(b);
-    tcg_gen_callN(&tcg_ctx, fn, 0, 7, GET_TCGV_I64(ret), 2, args);
+    tcg_gen_callN(&tcg_ctx, fn, TCG_CALL_CONST | TCG_CALL_PURE, sizemask,
+                  GET_TCGV_I64(ret), 2, args);
     tcg_temp_free_ptr(fn);
 }
@@ -550 +601 @@
 }
 
-static inline void tcg_gen_brcond_i32(int cond, TCGv_i32 arg1, TCGv_i32 arg2,
-                                      int label_index)
+static inline void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1,
+                                      TCGv_i32 arg2, int label_index)
 {
     tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_index);
 }
 
-static inline void tcg_gen_brcondi_i32(int cond, TCGv_i32 arg1, int32_t arg2,
-                                       int label_index)
+static inline void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1,
+                                       int32_t arg2, int label_index)
 {
     TCGv_i32 t0 = tcg_const_i32(arg2);
@@ -564 +615 @@
 }
 
+static inline void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
+                                       TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
+}
+
+static inline void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
+                                        TCGv_i32 arg1, int32_t arg2)
+{
+    TCGv_i32 t0 = tcg_const_i32(arg2);
+    tcg_gen_setcond_i32(cond, ret, arg1, t0);
+    tcg_temp_free_i32(t0);
+}
+
 static inline void tcg_gen_mul_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 {
@@ -596 +661 @@
     tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
 }
-#else
+#elif defined(TCG_TARGET_HAS_div2_i32)
 static inline void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 {
@@ -632 +697 @@
     tcg_temp_free_i32(t0);
 }
+#else
+static inline void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    int sizemask = 0;
+    /* Return value and both arguments are 32-bit and signed.  */
+    sizemask |= tcg_gen_sizemask(0, 0, 1);
+    sizemask |= tcg_gen_sizemask(1, 0, 1);
+    sizemask |= tcg_gen_sizemask(2, 0, 1);
+
+    tcg_gen_helper32(tcg_helper_div_i32, sizemask, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    int sizemask = 0;
+    /* Return value and both arguments are 32-bit and signed.  */
+    sizemask |= tcg_gen_sizemask(0, 0, 1);
+    sizemask |= tcg_gen_sizemask(1, 0, 1);
+    sizemask |= tcg_gen_sizemask(2, 0, 1);
+
+    tcg_gen_helper32(tcg_helper_rem_i32, sizemask, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    int sizemask = 0;
+    /* Return value and both arguments are 32-bit and unsigned.  */
+    sizemask |= tcg_gen_sizemask(0, 0, 0);
+    sizemask |= tcg_gen_sizemask(1, 0, 0);
+    sizemask |= tcg_gen_sizemask(2, 0, 0);
+
+    tcg_gen_helper32(tcg_helper_divu_i32, ret, arg1, arg2, 0);
+}
+
+static inline void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    int sizemask = 0;
+    /* Return value and both arguments are 32-bit and unsigned.  */
+    sizemask |= tcg_gen_sizemask(0, 0, 0);
+    sizemask |= tcg_gen_sizemask(1, 0, 0);
+    sizemask |= tcg_gen_sizemask(2, 0, 0);
+
+    tcg_gen_helper32(tcg_helper_remu_i32, ret, arg1, arg2, 0);
+}
 #endif
 
@@ -790 +899 @@
 static inline void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    tcg_gen_helper64(tcg_helper_shl_i64, ret, arg1, arg2);
+    int sizemask = 0;
+    /* Return value and both arguments are 64-bit and signed.  */
+    sizemask |= tcg_gen_sizemask(0, 1, 1);
+    sizemask |= tcg_gen_sizemask(1, 1, 1);
+    sizemask |= tcg_gen_sizemask(2, 1, 1);
+
+    tcg_gen_helper64(tcg_helper_shl_i64, sizemask, ret, arg1, arg2);
 }
 
@@ -800 +915 @@
 static inline void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    tcg_gen_helper64(tcg_helper_shr_i64, ret, arg1, arg2);
+    int sizemask = 0;
+    /* Return value and both arguments are 64-bit and signed.  */
+    sizemask |= tcg_gen_sizemask(0, 1, 1);
+    sizemask |= tcg_gen_sizemask(1, 1, 1);
+    sizemask |= tcg_gen_sizemask(2, 1, 1);
+
+    tcg_gen_helper64(tcg_helper_shr_i64, sizemask, ret, arg1, arg2);
 }
 
@@ -810 +931 @@
 static inline void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    tcg_gen_helper64(tcg_helper_sar_i64, ret, arg1, arg2);
+    int sizemask = 0;
+    /* Return value and both arguments are 64-bit and signed.  */
+    sizemask |= tcg_gen_sizemask(0, 1, 1);
+    sizemask |= tcg_gen_sizemask(1, 1, 1);
+    sizemask |= tcg_gen_sizemask(2, 1, 1);
+
+    tcg_gen_helper64(tcg_helper_sar_i64, sizemask, ret, arg1, arg2);
 }
 
@@ -818 +945 @@
 }
 
-static inline void tcg_gen_brcond_i64(int cond, TCGv_i64 arg1, TCGv_i64 arg2,
-                                      int label_index)
+static inline void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1,
+                                      TCGv_i64 arg2, int label_index)
 {
     tcg_gen_op6ii_i32(INDEX_op_brcond2_i32,
                       TCGV_LOW(arg1), TCGV_HIGH(arg1), TCGV_LOW(arg2),
                       TCGV_HIGH(arg2), cond, label_index);
+}
+
+static inline void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
+                                       TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
+                     TCGV_LOW(arg1), TCGV_HIGH(arg1),
+                     TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
+    tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
 }
 
@@ -849 +985 @@
 static inline void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    tcg_gen_helper64(tcg_helper_div_i64, ret, arg1, arg2);
+    int sizemask = 0;
+    /* Return value and both arguments are 64-bit and signed.  */
+    sizemask |= tcg_gen_sizemask(0, 1, 1);
+    sizemask |= tcg_gen_sizemask(1, 1, 1);
+    sizemask |= tcg_gen_sizemask(2, 1, 1);
+
+    tcg_gen_helper64(tcg_helper_div_i64, sizemask, ret, arg1, arg2);
 }
 
 static inline void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    tcg_gen_helper64(tcg_helper_rem_i64, ret, arg1, arg2);
+    int sizemask = 0;
+    /* Return value and both arguments are 64-bit and signed.  */
+    sizemask |= tcg_gen_sizemask(0, 1, 1);
+    sizemask |= tcg_gen_sizemask(1, 1, 1);
+    sizemask |= tcg_gen_sizemask(2, 1, 1);
+
+    tcg_gen_helper64(tcg_helper_rem_i64, sizemask, ret, arg1, arg2);
 }
 
 static inline void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    tcg_gen_helper64(tcg_helper_divu_i64, ret, arg1, arg2);
+    int sizemask = 0;
+    /* Return value and both arguments are 64-bit and unsigned.  */
+    sizemask |= tcg_gen_sizemask(0, 1, 0);
+    sizemask |= tcg_gen_sizemask(1, 1, 0);
+    sizemask |= tcg_gen_sizemask(2, 1, 0);
+
+    tcg_gen_helper64(tcg_helper_divu_i64, sizemask, ret, arg1, arg2);
 }
 
 static inline void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    tcg_gen_helper64(tcg_helper_remu_i64, ret, arg1, arg2);
+    int sizemask = 0;
+    /* Return value and both arguments are 64-bit and unsigned.  */
+    sizemask |= tcg_gen_sizemask(0, 1, 0);
+    sizemask |= tcg_gen_sizemask(1, 1, 0);
+    sizemask |= tcg_gen_sizemask(2, 1, 0);
+
+    tcg_gen_helper64(tcg_helper_remu_i64, sizemask, ret, arg1, arg2);
 }
 
@@ -1050 +1210 @@
 }
 
-static inline void tcg_gen_brcond_i64(int cond, TCGv_i64 arg1, TCGv_i64 arg2,
-                                      int label_index)
+static inline void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1,
+                                      TCGv_i64 arg2, int label_index)
 {
     tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, label_index);
+}
+
+static inline void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
+                                       TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
 }
 
@@ -1081 +1247 @@
     tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
 }
-#else
+#elif defined(TCG_TARGET_HAS_div2_i64)
 static inline void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
@@ -1116 +1282 @@
     tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
     tcg_temp_free_i64(t0);
+}
+#else
+static inline void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    int sizemask = 0;
+    /* Return value and both arguments are 64-bit and signed.  */
+    sizemask |= tcg_gen_sizemask(0, 1, 1);
+    sizemask |= tcg_gen_sizemask(1, 1, 1);
+    sizemask |= tcg_gen_sizemask(2, 1, 1);
+
+    tcg_gen_helper64(tcg_helper_div_i64, sizemask, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    int sizemask = 0;
+    /* Return value and both arguments are 64-bit and signed.  */
+    sizemask |= tcg_gen_sizemask(0, 1, 1);
+    sizemask |= tcg_gen_sizemask(1, 1, 1);
+    sizemask |= tcg_gen_sizemask(2, 1, 1);
+
+    tcg_gen_helper64(tcg_helper_rem_i64, sizemask, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    int sizemask = 0;
+    /* Return value and both arguments are 64-bit and unsigned.  */
+    sizemask |= tcg_gen_sizemask(0, 1, 0);
+    sizemask |= tcg_gen_sizemask(1, 1, 0);
+    sizemask |= tcg_gen_sizemask(2, 1, 0);
+
+    tcg_gen_helper64(tcg_helper_divu_i64, sizemask, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    int sizemask = 0;
+    /* Return value and both arguments are 64-bit and unsigned.  */
+    sizemask |= tcg_gen_sizemask(0, 1, 0);
+    sizemask |= tcg_gen_sizemask(1, 1, 0);
+    sizemask |= tcg_gen_sizemask(2, 1, 0);
+
+    tcg_gen_helper64(tcg_helper_remu_i64, sizemask, ret, arg1, arg2);
 }
 #endif
@@ -1151 +1361 @@
     }
 }
-static inline void tcg_gen_brcondi_i64(int cond, TCGv_i64 arg1, int64_t arg2,
-                                       int label_index)
+static inline void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1,
+                                       int64_t arg2, int label_index)
 {
     TCGv_i64 t0 = tcg_const_i64(arg2);
     tcg_gen_brcond_i64(cond, arg1, t0, label_index);
+    tcg_temp_free_i64(t0);
+}
+
+static inline void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
+                                        TCGv_i64 arg1, int64_t arg2)
+{
+    TCGv_i64 t0 = tcg_const_i64(arg2);
+    tcg_gen_setcond_i64(cond, ret, arg1, t0);
     tcg_temp_free_i64(t0);
 }
@@ -1533 +1751 @@
 #ifdef TCG_TARGET_HAS_not_i64
     tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
+#elif defined(TCG_TARGET_HAS_not_i32) && TCG_TARGET_REG_BITS == 32
+    tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
+    tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
 #else
     tcg_gen_xori_i64(ret, arg, -1);
@@ -1588 +1809 @@
 static inline void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 {
+#ifdef TCG_TARGET_HAS_andc_i32
+    tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
+#else
     TCGv_i32 t0;
     t0 = tcg_temp_new_i32();
@@ -1593 +1817 @@
     tcg_gen_and_i32(ret, arg1, t0);
     tcg_temp_free_i32(t0);
+#endif
 }
 
 static inline void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
+#ifdef TCG_TARGET_HAS_andc_i64
+    tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
+#elif defined(TCG_TARGET_HAS_andc_i32) && TCG_TARGET_REG_BITS == 32
+    tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
+    tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
+#else
     TCGv_i64 t0;
     t0 = tcg_temp_new_i64();
@@ -1602 +1833 @@
     tcg_gen_and_i64(ret, arg1, t0);
     tcg_temp_free_i64(t0);
+#endif
 }
 
 static inline void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 {
+#ifdef TCG_TARGET_HAS_eqv_i32
+    tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
+#else
     tcg_gen_xor_i32(ret, arg1, arg2);
     tcg_gen_not_i32(ret, ret);
+#endif
 }
 
 static inline void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
+#ifdef TCG_TARGET_HAS_eqv_i64
+    tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
+#elif defined(TCG_TARGET_HAS_eqv_i32) && TCG_TARGET_REG_BITS == 32
+    tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
+    tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
+#else
     tcg_gen_xor_i64(ret, arg1, arg2);
     tcg_gen_not_i64(ret, ret);
+#endif
 }
 
 static inline void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 {
+#ifdef TCG_TARGET_HAS_nand_i32
+    tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
+#else
     tcg_gen_and_i32(ret, arg1, arg2);
     tcg_gen_not_i32(ret, ret);
+#endif
 }
 
 static inline void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
+#ifdef TCG_TARGET_HAS_nand_i64
+    tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
+#elif defined(TCG_TARGET_HAS_nand_i32) && TCG_TARGET_REG_BITS == 32
+    tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
+    tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
+#else
     tcg_gen_and_i64(ret, arg1, arg2);
     tcg_gen_not_i64(ret, ret);
+#endif
 }
 
 static inline void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 {
+#ifdef TCG_TARGET_HAS_nor_i32
+    tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
+#else
     tcg_gen_or_i32(ret, arg1, arg2);
     tcg_gen_not_i32(ret, ret);
+#endif
 }
 
 static inline void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
+#ifdef TCG_TARGET_HAS_nor_i64
+    tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
+#elif defined(TCG_TARGET_HAS_nor_i32) && TCG_TARGET_REG_BITS == 32
+    tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
+    tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
+#else
     tcg_gen_or_i64(ret, arg1, arg2);
     tcg_gen_not_i64(ret, ret);
+#endif
 }
 
 static inline void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 {
+#ifdef TCG_TARGET_HAS_orc_i32
+    tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
+#else
     TCGv_i32 t0;
     t0 = tcg_temp_new_i32();
@@ -1647 +1915 @@
     tcg_gen_or_i32(ret, arg1, t0);
     tcg_temp_free_i32(t0);
+#endif
 }
 
 static inline void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
+#ifdef TCG_TARGET_HAS_orc_i64
+    tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
+#elif defined(TCG_TARGET_HAS_orc_i32) && TCG_TARGET_REG_BITS == 32
+    tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
+    tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
+#else
     TCGv_i64 t0;
     t0 = tcg_temp_new_i64();
@@ -1656 +1931 @@
     tcg_gen_or_i64(ret, arg1, t0);
     tcg_temp_free_i64(t0);
+#endif
 }
 
@@ -1897 +2173 @@
 {
 #if TARGET_LONG_BITS == 32
-    tcg_gen_op3i_i32(INDEX_op_qemu_ld32u, ret, addr, mem_index);
-#else
-    tcg_gen_op4i_i32(INDEX_op_qemu_ld32u, TCGV_LOW(ret), TCGV_LOW(addr),
+    tcg_gen_op3i_i32(INDEX_op_qemu_ld32, ret, addr, mem_index);
+#else
+    tcg_gen_op4i_i32(INDEX_op_qemu_ld32, TCGV_LOW(ret), TCGV_LOW(addr),
                      TCGV_HIGH(addr), mem_index);
     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
@@ -1908 +2184 @@
 {
 #if TARGET_LONG_BITS == 32
-    tcg_gen_op3i_i32(INDEX_op_qemu_ld32u, ret, addr, mem_index);
-#else
-    tcg_gen_op4i_i32(INDEX_op_qemu_ld32u, TCGV_LOW(ret), TCGV_LOW(addr),
+    tcg_gen_op3i_i32(INDEX_op_qemu_ld32, ret, addr, mem_index);
+#else
+    tcg_gen_op4i_i32(INDEX_op_qemu_ld32, TCGV_LOW(ret), TCGV_LOW(addr),
                      TCGV_HIGH(addr), mem_index);
     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
@@ -1994 +2270 @@
 static inline void tcg_gen_qemu_ld32u(TCGv ret, TCGv addr, int mem_index)
 {
+#if TARGET_LONG_BITS == 32
+    tcg_gen_qemu_ldst_op(INDEX_op_qemu_ld32, ret, addr, mem_index);
+#else
     tcg_gen_qemu_ldst_op(INDEX_op_qemu_ld32u, ret, addr, mem_index);
+#endif
 }
 
 static inline void tcg_gen_qemu_ld32s(TCGv ret, TCGv addr, int mem_index)
 {
+#if TARGET_LONG_BITS == 32
+    tcg_gen_qemu_ldst_op(INDEX_op_qemu_ld32, ret, addr, mem_index);
+#else
     tcg_gen_qemu_ldst_op(INDEX_op_qemu_ld32s, ret, addr, mem_index);
+#endif
 }
 
@@ -2033 +2317 @@
 
 #if TARGET_LONG_BITS == 64
-#define TCG_TYPE_TL TCG_TYPE_I64
 #define tcg_gen_movi_tl tcg_gen_movi_i64
 #define tcg_gen_mov_tl tcg_gen_mov_i64
@@ -2068 +2351 @@
 #define tcg_gen_brcond_tl tcg_gen_brcond_i64
 #define tcg_gen_brcondi_tl tcg_gen_brcondi_i64
+#define tcg_gen_setcond_tl tcg_gen_setcond_i64
+#define tcg_gen_setcondi_tl tcg_gen_setcondi_i64
 #define tcg_gen_mul_tl tcg_gen_mul_i64
 #define tcg_gen_muli_tl tcg_gen_muli_i64
@@ -2103 +2388 @@
 #define tcg_const_local_tl tcg_const_local_i64
 #else
-#define TCG_TYPE_TL TCG_TYPE_I32
 #define tcg_gen_movi_tl tcg_gen_movi_i32
 #define tcg_gen_mov_tl tcg_gen_mov_i32
@@ -2138 +2422 @@
 #define tcg_gen_brcond_tl tcg_gen_brcond_i32
 #define tcg_gen_brcondi_tl tcg_gen_brcondi_i32
+#define tcg_gen_setcond_tl tcg_gen_setcond_i32
+#define tcg_gen_setcondi_tl tcg_gen_setcondi_i32
 #define tcg_gen_mul_tl tcg_gen_mul_i32
 #define tcg_gen_muli_tl tcg_gen_muli_i32

trunk/src/recompiler/tcg/tcg-opc.h

@@ -22 +22 @@
  * THE SOFTWARE.
  */
-#ifndef DEF2
-#define DEF2(name, oargs, iargs, cargs, flags) DEF(name, oargs + iargs + cargs, 0)
-#endif
+
+/*
+ * DEF(name, oargs, iargs, cargs, flags)
+ */
 
 /* predefined ops */
-DEF2(end, 0, 0, 0, 0) /* must be kept first */
-DEF2(nop, 0, 0, 0, 0)
-DEF2(nop1, 0, 0, 1, 0)
-DEF2(nop2, 0, 0, 2, 0)
-DEF2(nop3, 0, 0, 3, 0)
-DEF2(nopn, 0, 0, 1, 0) /* variable number of parameters */
-
-DEF2(discard, 1, 0, 0, 0)
-
-DEF2(set_label, 0, 0, 1, 0)
-DEF2(call, 0, 1, 2, TCG_OPF_SIDE_EFFECTS) /* variable number of parameters */
-DEF2(jmp, 0, 1, 0, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
-DEF2(br, 0, 0, 1, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
-
-DEF2(mov_i32, 1, 1, 0, 0)
-DEF2(movi_i32, 1, 0, 1, 0)
+DEF(end, 0, 0, 0, 0) /* must be kept first */
+DEF(nop, 0, 0, 0, 0)
+DEF(nop1, 0, 0, 1, 0)
+DEF(nop2, 0, 0, 2, 0)
+DEF(nop3, 0, 0, 3, 0)
+DEF(nopn, 0, 0, 1, 0) /* variable number of parameters */
+
+DEF(discard, 1, 0, 0, 0)
+
+DEF(set_label, 0, 0, 1, 0)
+DEF(call, 0, 1, 2, TCG_OPF_SIDE_EFFECTS) /* variable number of parameters */
+DEF(jmp, 0, 1, 0, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
+DEF(br, 0, 0, 1, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
+
+DEF(mov_i32, 1, 1, 0, 0)
+DEF(movi_i32, 1, 0, 1, 0)
+DEF(setcond_i32, 1, 2, 1, 0)
 /* load/store */
-DEF2(ld8u_i32, 1, 1, 1, 0)
-DEF2(ld8s_i32, 1, 1, 1, 0)
-DEF2(ld16u_i32, 1, 1, 1, 0)
-DEF2(ld16s_i32, 1, 1, 1, 0)
-DEF2(ld_i32, 1, 1, 1, 0)
-DEF2(st8_i32, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
-DEF2(st16_i32, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
-DEF2(st_i32, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
+DEF(ld8u_i32, 1, 1, 1, 0)
+DEF(ld8s_i32, 1, 1, 1, 0)
+DEF(ld16u_i32, 1, 1, 1, 0)
+DEF(ld16s_i32, 1, 1, 1, 0)
+DEF(ld_i32, 1, 1, 1, 0)
+DEF(st8_i32, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
+DEF(st16_i32, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
+DEF(st_i32, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
 /* arith */
-DEF2(add_i32, 1, 2, 0, 0)
-DEF2(sub_i32, 1, 2, 0, 0)
-DEF2(mul_i32, 1, 2, 0, 0)
+DEF(add_i32, 1, 2, 0, 0)
+DEF(sub_i32, 1, 2, 0, 0)
+DEF(mul_i32, 1, 2, 0, 0)
 #ifdef TCG_TARGET_HAS_div_i32
-DEF2(div_i32, 1, 2, 0, 0)
-DEF2(divu_i32, 1, 2, 0, 0)
-DEF2(rem_i32, 1, 2, 0, 0)
-DEF2(remu_i32, 1, 2, 0, 0)
-#else
-DEF2(div2_i32, 2, 3, 0, 0)
-DEF2(divu2_i32, 2, 3, 0, 0)
-#endif
-DEF2(and_i32, 1, 2, 0, 0)
-DEF2(or_i32, 1, 2, 0, 0)
-DEF2(xor_i32, 1, 2, 0, 0)
+DEF(div_i32, 1, 2, 0, 0)
+DEF(divu_i32, 1, 2, 0, 0)
+DEF(rem_i32, 1, 2, 0, 0)
+DEF(remu_i32, 1, 2, 0, 0)
+#endif
+#ifdef TCG_TARGET_HAS_div2_i32
+DEF(div2_i32, 2, 3, 0, 0)
+DEF(divu2_i32, 2, 3, 0, 0)
+#endif
+DEF(and_i32, 1, 2, 0, 0)
+DEF(or_i32, 1, 2, 0, 0)
+DEF(xor_i32, 1, 2, 0, 0)
 /* shifts/rotates */
-DEF2(shl_i32, 1, 2, 0, 0)
-DEF2(shr_i32, 1, 2, 0, 0)
-DEF2(sar_i32, 1, 2, 0, 0)
+DEF(shl_i32, 1, 2, 0, 0)
+DEF(shr_i32, 1, 2, 0, 0)
+DEF(sar_i32, 1, 2, 0, 0)
 #ifdef TCG_TARGET_HAS_rot_i32
-DEF2(rotl_i32, 1, 2, 0, 0)
-DEF2(rotr_i32, 1, 2, 0, 0)
-#endif
-
-DEF2(brcond_i32, 0, 2, 2, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
+DEF(rotl_i32, 1, 2, 0, 0)
+DEF(rotr_i32, 1, 2, 0, 0)
+#endif
+
+DEF(brcond_i32, 0, 2, 2, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
 #if TCG_TARGET_REG_BITS == 32
-DEF2(add2_i32, 2, 4, 0, 0)
-DEF2(sub2_i32, 2, 4, 0, 0)
-DEF2(brcond2_i32, 0, 4, 2, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
-DEF2(mulu2_i32, 2, 2, 0, 0)
+DEF(add2_i32, 2, 4, 0, 0)
+DEF(sub2_i32, 2, 4, 0, 0)
+DEF(brcond2_i32, 0, 4, 2, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
+DEF(mulu2_i32, 2, 2, 0, 0)
+DEF(setcond2_i32, 1, 4, 1, 0)
 #endif
 #ifdef TCG_TARGET_HAS_ext8s_i32
-DEF2(ext8s_i32, 1, 1, 0, 0)
+DEF(ext8s_i32, 1, 1, 0, 0)
 #endif
 #ifdef TCG_TARGET_HAS_ext16s_i32
-DEF2(ext16s_i32, 1, 1, 0, 0)
+DEF(ext16s_i32, 1, 1, 0, 0)
 #endif
 #ifdef TCG_TARGET_HAS_ext8u_i32
-DEF2(ext8u_i32, 1, 1, 0, 0)
+DEF(ext8u_i32, 1, 1, 0, 0)
 #endif
 #ifdef TCG_TARGET_HAS_ext16u_i32
-DEF2(ext16u_i32, 1, 1, 0, 0)
+DEF(ext16u_i32, 1, 1, 0, 0)
 #endif
 #ifdef TCG_TARGET_HAS_bswap16_i32
-DEF2(bswap16_i32, 1, 1, 0, 0)
+DEF(bswap16_i32, 1, 1, 0, 0)
 #endif
 #ifdef TCG_TARGET_HAS_bswap32_i32
-DEF2(bswap32_i32, 1, 1, 0, 0)
+DEF(bswap32_i32, 1, 1, 0, 0)
 #endif
 #ifdef TCG_TARGET_HAS_not_i32
-DEF2(not_i32, 1, 1, 0, 0)
+DEF(not_i32, 1, 1, 0, 0)
 #endif
 #ifdef TCG_TARGET_HAS_neg_i32
-DEF2(neg_i32, 1, 1, 0, 0)
+DEF(neg_i32, 1, 1, 0, 0)
+#endif
+#ifdef TCG_TARGET_HAS_andc_i32
+DEF(andc_i32, 1, 2, 0, 0)
+#endif
+#ifdef TCG_TARGET_HAS_orc_i32
+DEF(orc_i32, 1, 2, 0, 0)
+#endif
+#ifdef TCG_TARGET_HAS_eqv_i32
+DEF(eqv_i32, 1, 2, 0, 0)
+#endif
+#ifdef TCG_TARGET_HAS_nand_i32
+DEF(nand_i32, 1, 2, 0, 0)
+#endif
+#ifdef TCG_TARGET_HAS_nor_i32
+DEF(nor_i32, 1, 2, 0, 0)
 #endif
 
 #if TCG_TARGET_REG_BITS == 64
-DEF2(mov_i64, 1, 1, 0, 0)
-DEF2(movi_i64, 1, 0, 1, 0)
+DEF(mov_i64, 1, 1, 0, 0)
+DEF(movi_i64, 1, 0, 1, 0)
+DEF(setcond_i64, 1, 2, 1, 0)
 /* load/store */
-DEF2(ld8u_i64, 1, 1, 1, 0)
-DEF2(ld8s_i64, 1, 1, 1, 0)
-DEF2(ld16u_i64, 1, 1, 1, 0)
-DEF2(ld16s_i64, 1, 1, 1, 0)
-DEF2(ld32u_i64, 1, 1, 1, 0)
-DEF2(ld32s_i64, 1, 1, 1, 0)
-DEF2(ld_i64, 1, 1, 1, 0)
-DEF2(st8_i64, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
-DEF2(st16_i64, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
-DEF2(st32_i64, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
-DEF2(st_i64, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
+DEF(ld8u_i64, 1, 1, 1, 0)
+DEF(ld8s_i64, 1, 1, 1, 0)
+DEF(ld16u_i64, 1, 1, 1, 0)
+DEF(ld16s_i64, 1, 1, 1, 0)
+DEF(ld32u_i64, 1, 1, 1, 0)
+DEF(ld32s_i64, 1, 1, 1, 0)
+DEF(ld_i64, 1, 1, 1, 0)
+DEF(st8_i64, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
+DEF(st16_i64, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
+DEF(st32_i64, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
+DEF(st_i64, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
 /* arith */
-DEF2(add_i64, 1, 2, 0, 0)
-DEF2(sub_i64, 1, 2, 0, 0)
-DEF2(mul_i64, 1, 2, 0, 0)
+DEF(add_i64, 1, 2, 0, 0)
+DEF(sub_i64, 1, 2, 0, 0)
+DEF(mul_i64, 1, 2, 0, 0)
 #ifdef TCG_TARGET_HAS_div_i64
-DEF2(div_i64, 1, 2, 0, 0)
-DEF2(divu_i64, 1, 2, 0, 0)
-DEF2(rem_i64, 1, 2, 0, 0)
-DEF2(remu_i64, 1, 2, 0, 0)
-#else
-DEF2(div2_i64, 2, 3, 0, 0)
-DEF2(divu2_i64, 2, 3, 0, 0)
-#endif
-DEF2(and_i64, 1, 2, 0, 0)
-DEF2(or_i64, 1, 2, 0, 0)
-DEF2(xor_i64, 1, 2, 0, 0)
+DEF(div_i64, 1, 2, 0, 0)
+DEF(divu_i64, 1, 2, 0, 0)
+DEF(rem_i64, 1, 2, 0, 0)
+DEF(remu_i64, 1, 2, 0, 0)
+#endif
+#ifdef TCG_TARGET_HAS_div2_i64
+DEF(div2_i64, 2, 3, 0, 0)
+DEF(divu2_i64, 2, 3, 0, 0)
+#endif
+DEF(and_i64, 1, 2, 0, 0)
+DEF(or_i64, 1, 2, 0, 0)
+DEF(xor_i64, 1, 2, 0, 0)
 /* shifts/rotates */
-DEF2(shl_i64, 1, 2, 0, 0)
-DEF2(shr_i64, 1, 2, 0, 0)
-DEF2(sar_i64, 1, 2, 0, 0)
+DEF(shl_i64, 1, 2, 0, 0)
+DEF(shr_i64, 1, 2, 0, 0)
+DEF(sar_i64, 1, 2, 0, 0)
 #ifdef TCG_TARGET_HAS_rot_i64
-DEF2(rotl_i64, 1, 2, 0, 0)
-DEF2(rotr_i64, 1, 2, 0, 0)
-#endif
-
-DEF2(brcond_i64, 0, 2, 2, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
+DEF(rotl_i64, 1, 2, 0, 0)
+DEF(rotr_i64, 1, 2, 0, 0)
+#endif
+
+DEF(brcond_i64, 0, 2, 2, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
 #ifdef TCG_TARGET_HAS_ext8s_i64
-DEF2(ext8s_i64, 1, 1, 0, 0)
+DEF(ext8s_i64, 1, 1, 0, 0)
 #endif
 #ifdef TCG_TARGET_HAS_ext16s_i64
-DEF2(ext16s_i64, 1, 1, 0, 0)
+DEF(ext16s_i64, 1, 1, 0, 0)
 #endif
 #ifdef TCG_TARGET_HAS_ext32s_i64
-DEF2(ext32s_i64, 1, 1, 0, 0)
+DEF(ext32s_i64, 1, 1, 0, 0)
 #endif
 #ifdef TCG_TARGET_HAS_ext8u_i64
-DEF2(ext8u_i64, 1, 1, 0, 0)
+DEF(ext8u_i64, 1, 1, 0, 0)
 #endif
 #ifdef TCG_TARGET_HAS_ext16u_i64
-DEF2(ext16u_i64, 1, 1, 0, 0)
+DEF(ext16u_i64, 1, 1, 0, 0)
 #endif
 #ifdef TCG_TARGET_HAS_ext32u_i64
-DEF2(ext32u_i64, 1, 1, 0, 0)
+DEF(ext32u_i64, 1, 1, 0, 0)
 #endif
 #ifdef TCG_TARGET_HAS_bswap16_i64
-DEF2(bswap16_i64, 1, 1, 0, 0)
+DEF(bswap16_i64, 1, 1, 0, 0)
 #endif
 #ifdef TCG_TARGET_HAS_bswap32_i64
-DEF2(bswap32_i64, 1, 1, 0, 0)
+DEF(bswap32_i64, 1, 1, 0, 0)
 #endif
 #ifdef TCG_TARGET_HAS_bswap64_i64
-DEF2(bswap64_i64, 1, 1, 0, 0)
+DEF(bswap64_i64, 1, 1, 0, 0)
 #endif
 #ifdef TCG_TARGET_HAS_not_i64
-DEF2(not_i64, 1, 1, 0, 0)
+DEF(not_i64, 1, 1, 0, 0)
 #endif
 #ifdef TCG_TARGET_HAS_neg_i64
-DEF2(neg_i64, 1, 1, 0, 0)
+DEF(neg_i64, 1, 1, 0, 0)
+#endif
+#ifdef TCG_TARGET_HAS_andc_i64
+DEF(andc_i64, 1, 2, 0, 0)
+#endif
+#ifdef TCG_TARGET_HAS_orc_i64
+DEF(orc_i64, 1, 2, 0, 0)
+#endif
+#ifdef TCG_TARGET_HAS_eqv_i64
+DEF(eqv_i64, 1, 2, 0, 0)
+#endif
+#ifdef TCG_TARGET_HAS_nand_i64
+DEF(nand_i64, 1, 2, 0, 0)
+#endif
+#ifdef TCG_TARGET_HAS_nor_i64
+DEF(nor_i64, 1, 2, 0, 0)
 #endif
 #endif
@@ -187 +223 @@
 /* QEMU specific */
 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
-DEF2(debug_insn_start, 0, 0, 2, 0)
-#else
-DEF2(debug_insn_start, 0, 0, 1, 0)
-#endif
-DEF2(exit_tb, 0, 0, 1, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
-DEF2(goto_tb, 0, 0, 1, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
+DEF(debug_insn_start, 0, 0, 2, 0)
+#else
+DEF(debug_insn_start, 0, 0, 1, 0)
+#endif
+DEF(exit_tb, 0, 0, 1, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
+DEF(goto_tb, 0, 0, 1, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
 /* Note: even if TARGET_LONG_BITS is not defined, the INDEX_op
    constants must be defined */
 #if TCG_TARGET_REG_BITS == 32
 #if TARGET_LONG_BITS == 32
-DEF2(qemu_ld8u, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#else
-DEF2(qemu_ld8u, 1, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#endif
-#if TARGET_LONG_BITS == 32
-DEF2(qemu_ld8s, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#else
-DEF2(qemu_ld8s, 1, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#endif
-#if TARGET_LONG_BITS == 32
-DEF2(qemu_ld16u, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#else
-DEF2(qemu_ld16u, 1, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#endif
-#if TARGET_LONG_BITS == 32
-DEF2(qemu_ld16s, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#else
-DEF2(qemu_ld16s, 1, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#endif
-#if TARGET_LONG_BITS == 32
-DEF2(qemu_ld32u, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#else
-DEF2(qemu_ld32u, 1, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#endif
-#if TARGET_LONG_BITS == 32
-DEF2(qemu_ld32s, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#else
-DEF2(qemu_ld32s, 1, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#endif
-#if TARGET_LONG_BITS == 32
-DEF2(qemu_ld64, 2, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#else
-DEF2(qemu_ld64, 2, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#endif
-
-#if TARGET_LONG_BITS == 32
-DEF2(qemu_st8, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#else
-DEF2(qemu_st8, 0, 3, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#endif
-#if TARGET_LONG_BITS == 32
-DEF2(qemu_st16, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#else
-DEF2(qemu_st16, 0, 3, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#endif
-#if TARGET_LONG_BITS == 32
-DEF2(qemu_st32, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#else
-DEF2(qemu_st32, 0, 3, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#endif
-#if TARGET_LONG_BITS == 32
-DEF2(qemu_st64, 0, 3, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-#else
-DEF2(qemu_st64, 0, 4, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld8u, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+#else
+DEF(qemu_ld8u, 1, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+#endif
+#if TARGET_LONG_BITS == 32
+DEF(qemu_ld8s, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+#else
+DEF(qemu_ld8s, 1, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+#endif
+#if TARGET_LONG_BITS == 32
+DEF(qemu_ld16u, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+#else
+DEF(qemu_ld16u, 1, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+#endif
+#if TARGET_LONG_BITS == 32
+DEF(qemu_ld16s, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+#else
+DEF(qemu_ld16s, 1, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+#endif
+#if TARGET_LONG_BITS == 32
+DEF(qemu_ld32, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+#else
+DEF(qemu_ld32, 1, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+#endif
+#if TARGET_LONG_BITS == 32
+DEF(qemu_ld64, 2, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+#else
+DEF(qemu_ld64, 2, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+#endif
+
+#if TARGET_LONG_BITS == 32
+DEF(qemu_st8, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+#else
+DEF(qemu_st8, 0, 3, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+#endif
+#if TARGET_LONG_BITS == 32
+DEF(qemu_st16, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+#else
+DEF(qemu_st16, 0, 3, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+#endif
+#if TARGET_LONG_BITS == 32
+DEF(qemu_st32, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+#else
+DEF(qemu_st32, 0, 3, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+#endif
+#if TARGET_LONG_BITS == 32
+DEF(qemu_st64, 0, 3, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+#else
+DEF(qemu_st64, 0, 4, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
 #endif
 
 #else /* TCG_TARGET_REG_BITS == 32 */
 
-DEF2(qemu_ld8u, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-DEF2(qemu_ld8s, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-DEF2(qemu_ld16u, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-DEF2(qemu_ld16s, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-DEF2(qemu_ld32u, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-DEF2(qemu_ld32s, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-DEF2(qemu_ld64, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-
-DEF2(qemu_st8, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-DEF2(qemu_st16, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-DEF2(qemu_st32, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-DEF2(qemu_st64, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld8u, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld8s, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld16u, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld16s, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld32, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld32u, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld32s, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld64, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+
+DEF(qemu_st8, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_st16, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_st32, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_st64, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
 
 #endif /* TCG_TARGET_REG_BITS != 32 */
 
-#undef DEF2
+#undef DEF

trunk/src/recompiler/tcg/tcg-runtime.h

@@ -3 +3 @@
 
 /* tcg-runtime.c */
+int32_t tcg_helper_div_i32(int32_t arg1, int32_t arg2);
+int32_t tcg_helper_rem_i32(int32_t arg1, int32_t arg2);
+uint32_t tcg_helper_divu_i32(uint32_t arg1, uint32_t arg2);
+uint32_t tcg_helper_remu_i32(uint32_t arg1, uint32_t arg2);
+
 int64_t tcg_helper_shl_i64(int64_t arg1, int64_t arg2);
 int64_t tcg_helper_shr_i64(int64_t arg1, int64_t arg2);

trunk/src/recompiler/tcg/tcg.c

@@ -28 +28 @@
 #include "config.h"
 
-#ifndef CONFIG_DEBUG_TCG
+#if !defined(CONFIG_DEBUG_TCG) && !defined(NDEBUG)
 /* define it to suppress various consistency checks (faster) */
 #define NDEBUG
@@ -53 +53 @@
 #include "cache-utils.h"
 #include "host-utils.h"
+#include "qemu-timer.h"
 
 /* Note: the long term plan is to reduce the dependancies on the QEMU
@@ -72 +73 @@
  * Liveness analysis doesn't work well with 32-bit hosts and 64-bit targets,
  * second element of the register pair to store 64-bit value is considered
- * dead, it seems.
- * @todo: fix it in compiler
- */
+ * dead, it seems. */
+ /** @todo re-test this */
 # if defined(TARGET_X86_64) && (TCG_TARGET_REG_BITS == 32)
 #  undef USE_LIVENESS_ANALYSIS
@@ -80 +80 @@
 #endif /* VBOX */
 
+static void tcg_target_init(TCGContext *s);
+static void tcg_target_qemu_prologue(TCGContext *s);
 static void patch_reloc(uint8_t *code_ptr, int type,
                         tcg_target_long value, tcg_target_long addend);
 
 static TCGOpDef tcg_op_defs[] = {
-#define DEF(s, n, copy_size) { #s, 0, 0, n, n, 0, copy_size },
-#ifndef VBOX
-#define DEF2(s, oargs, iargs, cargs, flags) { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags, 0 },
-#else  /* VBOX */
-# define DEF2(s, oargs, iargs, cargs, flags) { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags, 0, 0, 0 },
-#endif /* VBOX */
+#define DEF(s, oargs, iargs, cargs, flags) { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags },
 #include "tcg-opc.h"
 #undef DEF
-#undef DEF2
 };
 
@@ -121 +117 @@
 /* label relocation processing */
 
-void tcg_out_reloc(TCGContext *s, uint8_t *code_ptr, int type,
-                   int label_index, long addend)
+static void tcg_out_reloc(TCGContext *s, uint8_t *code_ptr, int type,
+                          int label_index, long addend)
 {
     TCGLabel *l;
@@ -261 +257 @@
 
     tcg_target_init(s);
-
+}
+
+void tcg_prologue_init(TCGContext *s)
+{
     /* init global prologue and epilogue */
     s->code_buf = code_gen_prologue;
@@ -575 +574 @@
                    int sizemask, TCGArg ret, int nargs, TCGArg *args)
 {
+#ifdef TCG_TARGET_I386
     int call_type;
+#endif
     int i;
     int real_args;
     int nb_rets;
     TCGArg *nparam;
+
+#if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
+    for (i = 0; i < nargs; ++i) {
+        int is_64bit = sizemask & (1 << (i+1)*2);
+        int is_signed = sizemask & (2 << (i+1)*2);
+        if (!is_64bit) {
+            TCGv_i64 temp = tcg_temp_new_i64();
+            TCGv_i64 orig = MAKE_TCGV_I64(args[i]);
+            if (is_signed) {
+                tcg_gen_ext32s_i64(temp, orig);
+            } else {
+                tcg_gen_ext32u_i64(temp, orig);
+            }
+            args[i] = GET_TCGV_I64(temp);
+        }
+    }
+#endif /* TCG_TARGET_EXTEND_ARGS */
+
     *gen_opc_ptr++ = INDEX_op_call;
     nparam = gen_opparam_ptr++;
+#ifdef TCG_TARGET_I386
     call_type = (flags & TCG_CALL_TYPE_MASK);
+#endif
     if (ret != TCG_CALL_DUMMY_ARG) {
 #if TCG_TARGET_REG_BITS < 64
@@ -606 +627 @@
     for (i = 0; i < nargs; i++) {
 #if TCG_TARGET_REG_BITS < 64
-        if (sizemask & (2 << i)) {
+        int is_64bit = sizemask & (1 << (i+1)*2);
+        if (is_64bit) {
 #ifdef TCG_TARGET_I386
             /* REGPARM case: if the third parameter is 64 bit, it is
@@ -622 +644 @@
             }
 #endif
-#ifdef TCG_TARGET_WORDS_BIGENDIAN
+            /* If stack grows up, then we will be placing successive
+               arguments at lower addresses, which means we need to
+               reverse the order compared to how we would normally
+               treat either big or little-endian.  For those arguments
+               that will wind up in registers, this still works for
+               HPPA (the only current STACK_GROWSUP target) since the
+               argument registers are *also* allocated in decreasing
+               order.  If another such target is added, this logic may
+               have to get more complicated to differentiate between
+               stack arguments and register arguments.  */
+#if defined(TCG_TARGET_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
             *gen_opparam_ptr++ = args[i] + 1;
             *gen_opparam_ptr++ = args[i];
@@ -630 +662 @@
 #endif
             real_args += 2;
-        } else
-#endif
-        {
-            *gen_opparam_ptr++ = args[i];
-            real_args++;
-        }
+            continue;
+        }
+#endif /* TCG_TARGET_REG_BITS < 64 */
+
+        *gen_opparam_ptr++ = args[i];
+        real_args++;
     }
     *gen_opparam_ptr++ = GET_TCGV_PTR(func);
@@ -645 +677 @@
     /* total parameters, needed to go backward in the instruction stream */
     *gen_opparam_ptr++ = 1 + nb_rets + real_args + 3;
+
+#if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
+    for (i = 0; i < nargs; ++i) {
+        int is_64bit = sizemask & (1 << (i+1)*2);
+        if (!is_64bit) {
+            TCGv_i64 temp = MAKE_TCGV_I64(args[i]);
+            tcg_temp_free_i64(temp);
+        }
+    }
+#endif /* TCG_TARGET_EXTEND_ARGS */
 }
 
@@ -696 +738 @@
 #endif
 
+
 static void tcg_reg_alloc_start(TCGContext *s)
 {
@@ -813 +856 @@
     const TCGArg *args;
     TCGArg arg;
-    int c, i, k, nb_oargs, nb_iargs, nb_cargs, first_insn;
+    TCGOpcode c;
+    int i, k, nb_oargs, nb_iargs, nb_cargs, first_insn;
     const TCGOpDef *def;
     char buf[128];
@@ -919 +963 @@
                         tcg_get_arg_str_idx(s, buf, sizeof(buf), args[k++]));
             }
-            if (c == INDEX_op_brcond_i32
+            switch (c) {
+            case INDEX_op_brcond_i32:
 #if TCG_TARGET_REG_BITS == 32
-                || c == INDEX_op_brcond2_i32
+            case INDEX_op_brcond2_i32:
 #elif TCG_TARGET_REG_BITS == 64
-                || c == INDEX_op_brcond_i64
-#endif
-                ) {
+            case INDEX_op_brcond_i64:
+#endif
+            case INDEX_op_setcond_i32:
+#if TCG_TARGET_REG_BITS == 32
+            case INDEX_op_setcond2_i32:
+#elif TCG_TARGET_REG_BITS == 64
+            case INDEX_op_setcond_i64:
+#endif
                 if (args[k] < ARRAY_SIZE(cond_name) && cond_name[args[k]])
                     fprintf(outfile, ",%s", cond_name[args[k++]]);
@@ -931 +981 @@
                     fprintf(outfile, ",$0x%" TCG_PRIlx, args[k++]);
                 i = 1;
-            }
-            else
+                break;
+            default:
                 i = 0;
+                break;
+            }
             for(; i < nb_cargs; i++) {
                 if (k != 0)
@@ -992 +1044 @@
 void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs)
 {
-    int op;
+    TCGOpcode op;
     TCGOpDef *def;
     const char *ct_str;
@@ -998 +1050 @@
 
     for(;;) {
-        if (tdefs->op < 0)
+        if (tdefs->op == (TCGOpcode)-1)
             break;
         op = tdefs->op;
         assert(op >= 0 && op < NB_OPS);
         def = &tcg_op_defs[op];
+#if defined(CONFIG_DEBUG_TCG)
+        /* Duplicate entry in op definitions? */
+        assert(!def->used);
+        def->used = 1;
+#endif
         nb_args = def->nb_iargs + def->nb_oargs;
         for(i = 0; i < nb_args; i++) {
             ct_str = tdefs->args_ct_str[i];
+            /* Incomplete TCGTargetOpDef entry? */
+            assert(ct_str != NULL);
             tcg_regset_clear(def->args_ct[i].u.regs);
             def->args_ct[i].ct = 0;
@@ -1033 +1092 @@
                             fprintf(stderr, "Invalid constraint '%s' for arg %d of operation '%s'\n",
                                     ct_str, i, def->name);
-#ifdef VBOX
-                            tcg_exit(1);
-#else
                             exit(1);
-#endif
                         }
                     }
@@ -1043 +1098 @@
             }
         }
+
+        /* TCGTargetOpDef entry with too much information? */
+        assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
 
         /* sort the constraints (XXX: this is just an heuristic) */
@@ -1061 +1119 @@
     }
 
+#if defined(CONFIG_DEBUG_TCG)
+    i = 0;
+    for (op = 0; op < ARRAY_SIZE(tcg_op_defs); op++) {
+        if (op < INDEX_op_call || op == INDEX_op_debug_insn_start) {
+            /* Wrong entry in op definitions? */
+            if (tcg_op_defs[op].used) {
+                fprintf(stderr, "Invalid op definition for %s\n",
+                        tcg_op_defs[op].name);
+                i = 1;
+            }
+        } else {
+            /* Missing entry in op definitions? */
+            if (!tcg_op_defs[op].used) {
+                fprintf(stderr, "Missing op definition for %s\n",
+                        tcg_op_defs[op].name);
+                i = 1;
+            }
+        }
+    }
+    if (i == 1) {
+        tcg_abort();
+    }
+#endif
 }
 
@@ -1111 +1192 @@
 static void tcg_liveness_analysis(TCGContext *s)
 {
-    int i, op_index, op, nb_args, nb_iargs, nb_oargs, arg, nb_ops;
+    int i, op_index, nb_args, nb_iargs, nb_oargs, arg, nb_ops;
+    TCGOpcode op;
     TCGArg *args;
     const TCGOpDef *def;
@@ -1259 +1341 @@
 #else
 /* dummy liveness analysis */
-void tcg_liveness_analysis(TCGContext *s)
+static void tcg_liveness_analysis(TCGContext *s)
 {
     int nb_ops;
@@ -1350 +1432 @@
     if (s->current_frame_offset + sizeof(tcg_target_long) > s->frame_end)
 #else
-    if ((unsigned)s->current_frame_offset + sizeof(tcg_target_long) > s->frame_end)
+    if ((tcg_target_long)s->current_frame_offset + sizeof(tcg_target_long) > s->frame_end)
 #endif
         tcg_abort();
@@ -1524 +1606 @@
             }
             if (ts->reg != reg) {
-                tcg_out_mov(s, reg, ts->reg);
+                tcg_out_mov(s, ots->type, reg, ts->reg);
             }
         }
@@ -1556 +1638 @@
 
 static void tcg_reg_alloc_op(TCGContext *s,
-                             const TCGOpDef *def, int opc,
+                             const TCGOpDef *def, TCGOpcode opc,
                              const TCGArg *args,
                              unsigned int dead_iargs)
@@ -1629 +1711 @@
                and move the temporary register into it */
             reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
-            tcg_out_mov(s, reg, ts->reg);
+            tcg_out_mov(s, ts->type, reg, ts->reg);
         }
         new_args[i] = reg;
@@ -1711 +1793 @@
         reg = new_args[i];
         if (ts->fixed_reg && ts->reg != reg) {
-            tcg_out_mov(s, ts->reg, reg);
+            tcg_out_mov(s, ts->type, ts->reg, reg);
         }
     }
@@ -1723 +1805 @@
 
 static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def,
-                              int opc, const TCGArg *args,
+                              TCGOpcode opc, const TCGArg *args,
                               unsigned int dead_iargs)
 {
@@ -1797 +1879 @@
             if (ts->val_type == TEMP_VAL_REG) {
                 if (ts->reg != reg) {
-                    tcg_out_mov(s, reg, ts->reg);
+                    tcg_out_mov(s, ts->type, reg, ts->reg);
                 }
             } else if (ts->val_type == TEMP_VAL_MEM) {
@@ -1826 +1908 @@
         if (!tcg_regset_test_reg(arg_ct->u.regs, reg)) {
             reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
-            tcg_out_mov(s, reg, ts->reg);
+            tcg_out_mov(s, ts->type, reg, ts->reg);
         }
         func_arg = reg;
@@ -1885 +1967 @@
         if (ts->fixed_reg) {
             if (ts->reg != reg) {
-                tcg_out_mov(s, ts->reg, reg);
+                tcg_out_mov(s, ts->type, ts->reg, reg);
             }
         } else {
@@ -1920 +2002 @@
                                       long search_pc)
 {
-    int opc, op_index;
+    TCGOpcode opc;
+    int op_index;
     const TCGOpDef *def;
     unsigned int dead_iargs;

trunk/src/recompiler/tcg/tcg.h

@@ -48 +48 @@
 #endif
 
-enum {
-#define DEF(s, n, copy_size) INDEX_op_ ## s,
+typedef enum TCGOpcode {
+#define DEF(name, oargs, iargs, cargs, flags) INDEX_op_ ## name,
 #include "tcg-opc.h"
 #undef DEF
     NB_OPS,
-};
+} TCGOpcode;
 
 #define tcg_regset_clear(d) (d) = 0
@@ -97 +97 @@
 #define TCG_STATIC_CALL_ARGS_SIZE 128
 
-typedef int TCGType;
-
-#define TCG_TYPE_I32 0
-#define TCG_TYPE_I64 1
-#define TCG_TYPE_COUNT 2 /* number of different types */
-
+typedef enum TCGType {
+    TCG_TYPE_I32,
+    TCG_TYPE_I64,
+    TCG_TYPE_COUNT, /* number of different types */
+
+    /* An alias for the size of the host register.  */
 #if TCG_TARGET_REG_BITS == 32
-#define TCG_TYPE_PTR TCG_TYPE_I32
-#else
-#define TCG_TYPE_PTR TCG_TYPE_I64
-#endif
+    TCG_TYPE_REG = TCG_TYPE_I32,
+#else
+    TCG_TYPE_REG = TCG_TYPE_I64,
+#endif
+
+    /* An alias for the size of the native pointer.  We don't currently
+       support any hosts with 64-bit registers and 32-bit pointers.  */
+    TCG_TYPE_PTR = TCG_TYPE_REG,
+
+    /* An alias for the size of the target "long", aka register.  */
+#if TARGET_LONG_BITS == 64
+    TCG_TYPE_TL = TCG_TYPE_I64,
+#else
+    TCG_TYPE_TL = TCG_TYPE_I32,
+#endif
+} TCGType;
 
 typedef tcg_target_ulong TCGArg;
@@ -205 +217 @@
     TCG_COND_GTU,
 } TCGCond;
+
+/* Invert the sense of the comparison.  */
+static inline TCGCond tcg_invert_cond(TCGCond c)
+{
+    return (TCGCond)(c ^ 1);
+}
+
+/* Swap the operands in a comparison.  */
+static inline TCGCond tcg_swap_cond(TCGCond c)
+{
+    int mask = (c < TCG_COND_LT ? 0 : c < TCG_COND_LTU ? 7 : 15);
+    return (TCGCond)(c ^ mask);
+}
+
+static inline TCGCond tcg_unsigned_cond(TCGCond c)
+{
+    return (c >= TCG_COND_LT && c <= TCG_COND_GT ? c + 4 : c);
+}
 
 #define TEMP_VAL_DEAD  0
@@ -324 +354 @@
 
 void tcg_context_init(TCGContext *s);
+void tcg_prologue_init(TCGContext *s);
 void tcg_func_start(TCGContext *s);
 
@@ -392 +423 @@
     uint8_t nb_oargs, nb_iargs, nb_cargs, nb_args;
     uint8_t flags;
-    uint16_t copy_size;
     TCGArgConstraint *args_ct;
     int *sorted_args;
+#if defined(CONFIG_DEBUG_TCG)
+    int used;
+#endif
 } TCGOpDef;
 
 typedef struct TCGTargetOpDef {
-    int op;
+    TCGOpcode op;
     const char *args_ct_str[TCG_MAX_OP_ARGS];
 } TCGTargetOpDef;
-
-void tcg_target_init(TCGContext *s);
-void tcg_target_qemu_prologue(TCGContext *s);
 
 #ifndef VBOX
@@ -465 +495 @@
 TCGv_i64 tcg_const_local_i64(int64_t val);
 
-void tcg_out_reloc(TCGContext *s, uint8_t *code_ptr, int type,
-                   int label_index, long addend);
-
 #ifndef VBOX
 extern uint8_t code_gen_prologue[];

trunk/src/recompiler/tests/Makefile

@@ -1 +1 @@
 -include ../config-host.mak
-VPATH=$(SRC_PATH)/tests
+
+$(call set-vpath, $(SRC_PATH)/tests)
 
 CFLAGS=-Wall -O2 -g -fno-strict-aliasing

trunk/src/recompiler/tests/qruncom.c

@@ -90 +90 @@
 #define COM_BASE_ADDR    0x10100
 
-void usage(void)
+static void usage(void)
 {
     printf("qruncom version 0.1 (c) 2003 Fabrice Bellard\n"

trunk/src/recompiler/tests/runcom.c

@@ -26 +26 @@
 #define COM_BASE_ADDR    0x10100
 
-void usage(void)
+static void usage(void)
 {
     printf("runcom version 0.1 (c) 2003 Fabrice Bellard\n"
@@ -52 +52 @@
 void dump_regs(struct vm86_regs *r)
 {
-    fprintf(stderr, 
+    fprintf(stderr,
             "EAX=%08lx EBX=%08lx ECX=%08lx EDX=%08lx\n"
             "ESI=%08lx EDI=%08lx EBP=%08lx ESP=%08lx\n"
@@ -81 +81 @@
         usage();
     filename = argv[1];
-    
-    vm86_mem = mmap((void *)0x00000000, 0x110000, 
-                    PROT_WRITE | PROT_READ | PROT_EXEC, 
+
+    vm86_mem = mmap((void *)0x00000000, 0x110000,
+                    PROT_WRITE | PROT_READ | PROT_EXEC,
                     MAP_FIXED | MAP_ANON | MAP_PRIVATE, -1, 0);
     if (vm86_mem == MAP_FAILED) {
@@ -148 +148 @@
             {
                 int int_num, ah;
-                
+
                 int_num = VM86_ARG(ret);
                 if (int_num != 0x21)

trunk/src/recompiler/tests/sha1.c

@@ -24 +24 @@
 #include <stdio.h>
 #include <string.h>
-#include <sys/types.h>  /* for u_int*_t */
+#include <stdint.h>
 
 /* ================ sha1.h ================ */
@@ -34 +34 @@
 
 typedef struct {
-    u_int32_t state[5];
-    u_int32_t count[2];
+    uint32_t state[5];
+    uint32_t count[2];
     unsigned char buffer[64];
 } SHA1_CTX;
 
-void SHA1Transform(u_int32_t state[5], const unsigned char buffer[64]);
+void SHA1Transform(uint32_t state[5], const unsigned char buffer[64]);
 void SHA1Init(SHA1_CTX* context);
-void SHA1Update(SHA1_CTX* context, const unsigned char* data, u_int32_t len);
+void SHA1Update(SHA1_CTX* context, const unsigned char* data, uint32_t len);
 void SHA1Final(unsigned char digest[20], SHA1_CTX* context);
 /* ================ end of sha1.h ================ */
@@ -71 +71 @@
 /* Hash a single 512-bit block. This is the core of the algorithm. */
 
-void SHA1Transform(u_int32_t state[5], const unsigned char buffer[64])
-{
-u_int32_t a, b, c, d, e;
+void SHA1Transform(uint32_t state[5], const unsigned char buffer[64])
+{
+uint32_t a, b, c, d, e;
 typedef union {
     unsigned char c[64];
-    u_int32_t l[16];
+    uint32_t l[16];
 } CHAR64LONG16;
 #ifdef SHA1HANDSOFF
@@ -146 +146 @@
 /* Run your data through this. */
 
-void SHA1Update(SHA1_CTX* context, const unsigned char* data, u_int32_t len)
-{
-u_int32_t i;
-u_int32_t j;
+void SHA1Update(SHA1_CTX* context, const unsigned char* data, uint32_t len)
+{
+uint32_t i;
+uint32_t j;
 
     j = context->count[0];
@@ -187 +187 @@
     for (i = 0; i < 2; i++)
     {
-        u_int32_t t = context->count[i];
+        uint32_t t = context->count[i];
         int j;
 

trunk/src/recompiler/translate-all.c

@@ -40 +40 @@
 #include "disas.h"
 #include "tcg.h"
+#include "qemu-timer.h"
 
 /* code generation context */
@@ -50 +51 @@
 uint16_t gen_opc_icount[OPC_BUF_SIZE];
 uint8_t gen_opc_instr_start[OPC_BUF_SIZE];
-#if defined(TARGET_I386)
-uint8_t gen_opc_cc_op[OPC_BUF_SIZE];
-#elif defined(TARGET_SPARC)
-target_ulong gen_opc_npc[OPC_BUF_SIZE];
-target_ulong gen_opc_jump_pc[2];
-#elif defined(TARGET_MIPS) || defined(TARGET_SH4)
-uint32_t gen_opc_hflags[OPC_BUF_SIZE];
-#endif
-
-/* XXX: suppress that */
-unsigned long code_gen_max_block_size(void)
-{
-#ifdef VBOX
-    /* Just to suppress a lot of dummy warnings */
-    static long max;
-#else
-    static unsigned long max;
-#endif
-
-    if (max == 0) {
-        max = TCG_MAX_OP_SIZE;
-#define DEF(s, n, copy_size) max = copy_size > max? copy_size : max;
-#include "tcg-opc.h"
-#undef DEF
-        max *= OPC_MAX_SIZE;
-    }
-
-    return max;
-}
 
 void cpu_gen_init(void)
@@ -124 +96 @@
     s->tb_jmp_offset = tb->tb_jmp_offset;
     s->tb_next = NULL;
-    /* the following two entries are optional (only used for string ops) */
-    /* XXX: not used ? */
-    tb->tb_jmp_offset[2] = 0xffff;
-    tb->tb_jmp_offset[3] = 0xffff;
 #else
     s->tb_jmp_offset = NULL;