MIPS: Close races in TLB modify handlers.
[linux-2.6.git] / arch / mips / mm / tlbex.c
index 6569be3..b6e1cff 100644 (file)
  *
  * Synthesize TLB refill handlers at runtime.
  *
- * Copyright (C) 2004,2005 by Thiemo Seufer
+ * Copyright (C) 2004, 2005, 2006, 2008  Thiemo Seufer
+ * Copyright (C) 2005, 2007, 2008, 2009  Maciej W. Rozycki
+ * Copyright (C) 2006  Ralf Baechle (ralf@linux-mips.org)
+ * Copyright (C) 2008, 2009 Cavium Networks, Inc.
+ *
+ * ... and the days got worse and worse and now you see
+ * I've gone completly out of my mind.
+ *
+ * They're coming to take me a away haha
+ * they're coming to take me a away hoho hihi haha
+ * to the funny farm where code is beautiful all the time ...
+ *
+ * (Condolences to Napoleon XIV)
  */
 
-#include <stdarg.h>
-
-#include <linux/config.h>
-#include <linux/mm.h>
+#include <linux/bug.h>
 #include <linux/kernel.h>
 #include <linux/types.h>
+#include <linux/smp.h>
 #include <linux/string.h>
 #include <linux/init.h>
+#include <linux/cache.h>
 
-#include <asm/pgtable.h>
 #include <asm/cacheflush.h>
-#include <asm/mmu_context.h>
-#include <asm/inst.h>
-#include <asm/elf.h>
-#include <asm/smp.h>
+#include <asm/pgtable.h>
 #include <asm/war.h>
+#include <asm/uasm.h>
+
+/*
+ * TLB load/store/modify handlers.
+ *
+ * Only the fastpath gets synthesized at runtime, the slowpath for
+ * do_page_fault remains normal asm.
+ */
+extern void tlb_do_page_fault_0(void);
+extern void tlb_do_page_fault_1(void);
+
+struct work_registers {
+       int r1;
+       int r2;
+       int r3;
+};
+
+struct tlb_reg_save {
+       unsigned long a;
+       unsigned long b;
+} ____cacheline_aligned_in_smp;
 
-/* #define DEBUG_TLB */
+static struct tlb_reg_save handler_reg_save[NR_CPUS];
 
-static __init int __attribute__((unused)) r45k_bvahwbug(void)
+static inline int r45k_bvahwbug(void)
 {
        /* XXX: We should probe for the presence of this bug, but we don't. */
        return 0;
 }
 
-static __init int __attribute__((unused)) r4k_250MHZhwbug(void)
+static inline int r4k_250MHZhwbug(void)
 {
        /* XXX: We should probe for the presence of this bug, but we don't. */
        return 0;
 }
 
-static __init int __attribute__((unused)) bcm1250_m3_war(void)
+static inline int __maybe_unused bcm1250_m3_war(void)
 {
        return BCM1250_M3_WAR;
 }
 
-static __init int __attribute__((unused)) r10000_llsc_war(void)
+static inline int __maybe_unused r10000_llsc_war(void)
 {
        return R10000_LLSC_WAR;
 }
 
-/*
- * A little micro-assembler, intended for TLB refill handler
- * synthesizing. It is intentionally kept simple, does only support
- * a subset of instructions, and does not try to hide pipeline effects
- * like branch delay slots.
- */
-
-enum fields
-{
-       RS = 0x001,
-       RT = 0x002,
-       RD = 0x004,
-       RE = 0x008,
-       SIMM = 0x010,
-       UIMM = 0x020,
-       BIMM = 0x040,
-       JIMM = 0x080,
-       FUNC = 0x100,
-};
-
-#define OP_MASK                0x2f
-#define OP_SH          26
-#define RS_MASK                0x1f
-#define RS_SH          21
-#define RT_MASK                0x1f
-#define RT_SH          16
-#define RD_MASK                0x1f
-#define RD_SH          11
-#define RE_MASK                0x1f
-#define RE_SH          6
-#define IMM_MASK       0xffff
-#define IMM_SH         0
-#define JIMM_MASK      0x3ffffff
-#define JIMM_SH                0
-#define FUNC_MASK      0x2f
-#define FUNC_SH                0
-
-enum opcode {
-       insn_invalid,
-       insn_addu, insn_addiu, insn_and, insn_andi, insn_beq,
-       insn_beql, insn_bgez, insn_bgezl, insn_bltz, insn_bltzl,
-       insn_bne, insn_daddu, insn_daddiu, insn_dmfc0, insn_dmtc0,
-       insn_dsll, insn_dsll32, insn_dsra, insn_dsrl, insn_dsrl32,
-       insn_dsubu, insn_eret, insn_j, insn_jal, insn_jr, insn_ld,
-       insn_ll, insn_lld, insn_lui, insn_lw, insn_mfc0, insn_mtc0,
-       insn_ori, insn_rfe, insn_sc, insn_scd, insn_sd, insn_sll,
-       insn_sra, insn_srl, insn_subu, insn_sw, insn_tlbp, insn_tlbwi,
-       insn_tlbwr, insn_xor, insn_xori
-};
-
-struct insn {
-       enum opcode opcode;
-       u32 match;
-       enum fields fields;
-};
-
-/* This macro sets the non-variable bits of an instruction. */
-#define M(a, b, c, d, e, f)                                    \
-       ((a) << OP_SH                                           \
-        | (b) << RS_SH                                         \
-        | (c) << RT_SH                                         \
-        | (d) << RD_SH                                         \
-        | (e) << RE_SH                                         \
-        | (f) << FUNC_SH)
-
-static __initdata struct insn insn_table[] = {
-       { insn_addiu, M(addiu_op,0,0,0,0,0), RS | RT | SIMM },
-       { insn_addu, M(spec_op,0,0,0,0,addu_op), RS | RT | RD },
-       { insn_and, M(spec_op,0,0,0,0,and_op), RS | RT | RD },
-       { insn_andi, M(andi_op,0,0,0,0,0), RS | RT | UIMM },
-       { insn_beq, M(beq_op,0,0,0,0,0), RS | RT | BIMM },
-       { insn_beql, M(beql_op,0,0,0,0,0), RS | RT | BIMM },
-       { insn_bgez, M(bcond_op,0,bgez_op,0,0,0), RS | BIMM },
-       { insn_bgezl, M(bcond_op,0,bgezl_op,0,0,0), RS | BIMM },
-       { insn_bltz, M(bcond_op,0,bltz_op,0,0,0), RS | BIMM },
-       { insn_bltzl, M(bcond_op,0,bltzl_op,0,0,0), RS | BIMM },
-       { insn_bne, M(bne_op,0,0,0,0,0), RS | RT | BIMM },
-       { insn_daddiu, M(daddiu_op,0,0,0,0,0), RS | RT | SIMM },
-       { insn_daddu, M(spec_op,0,0,0,0,daddu_op), RS | RT | RD },
-       { insn_dmfc0, M(cop0_op,dmfc_op,0,0,0,0), RT | RD },
-       { insn_dmtc0, M(cop0_op,dmtc_op,0,0,0,0), RT | RD },
-       { insn_dsll, M(spec_op,0,0,0,0,dsll_op), RT | RD | RE },
-       { insn_dsll32, M(spec_op,0,0,0,0,dsll32_op), RT | RD | RE },
-       { insn_dsra, M(spec_op,0,0,0,0,dsra_op), RT | RD | RE },
-       { insn_dsrl, M(spec_op,0,0,0,0,dsrl_op), RT | RD | RE },
-       { insn_dsrl32, M(spec_op,0,0,0,0,dsrl32_op), RT | RD | RE },
-       { insn_dsubu, M(spec_op,0,0,0,0,dsubu_op), RS | RT | RD },
-       { insn_eret, M(cop0_op,cop_op,0,0,0,eret_op), 0 },
-       { insn_j, M(j_op,0,0,0,0,0), JIMM },
-       { insn_jal, M(jal_op,0,0,0,0,0), JIMM },
-       { insn_jr, M(spec_op,0,0,0,0,jr_op), RS },
-       { insn_ld, M(ld_op,0,0,0,0,0), RS | RT | SIMM },
-       { insn_ll, M(ll_op,0,0,0,0,0), RS | RT | SIMM },
-       { insn_lld, M(lld_op,0,0,0,0,0), RS | RT | SIMM },
-       { insn_lui, M(lui_op,0,0,0,0,0), RT | SIMM },
-       { insn_lw, M(lw_op,0,0,0,0,0), RS | RT | SIMM },
-       { insn_mfc0, M(cop0_op,mfc_op,0,0,0,0), RT | RD },
-       { insn_mtc0, M(cop0_op,mtc_op,0,0,0,0), RT | RD },
-       { insn_ori, M(ori_op,0,0,0,0,0), RS | RT | UIMM },
-       { insn_rfe, M(cop0_op,cop_op,0,0,0,rfe_op), 0 },
-       { insn_sc, M(sc_op,0,0,0,0,0), RS | RT | SIMM },
-       { insn_scd, M(scd_op,0,0,0,0,0), RS | RT | SIMM },
-       { insn_sd, M(sd_op,0,0,0,0,0), RS | RT | SIMM },
-       { insn_sll, M(spec_op,0,0,0,0,sll_op), RT | RD | RE },
-       { insn_sra, M(spec_op,0,0,0,0,sra_op), RT | RD | RE },
-       { insn_srl, M(spec_op,0,0,0,0,srl_op), RT | RD | RE },
-       { insn_subu, M(spec_op,0,0,0,0,subu_op), RS | RT | RD },
-       { insn_sw, M(sw_op,0,0,0,0,0), RS | RT | SIMM },
-       { insn_tlbp, M(cop0_op,cop_op,0,0,0,tlbp_op), 0 },
-       { insn_tlbwi, M(cop0_op,cop_op,0,0,0,tlbwi_op), 0 },
-       { insn_tlbwr, M(cop0_op,cop_op,0,0,0,tlbwr_op), 0 },
-       { insn_xor, M(spec_op,0,0,0,0,xor_op), RS | RT | RD },
-       { insn_xori, M(xori_op,0,0,0,0,0), RS | RT | UIMM },
-       { insn_invalid, 0, 0 }
-};
-
-#undef M
-
-static __init u32 build_rs(u32 arg)
+static int use_bbit_insns(void)
 {
-       if (arg & ~RS_MASK)
-               printk(KERN_WARNING "TLB synthesizer field overflow\n");
-
-       return (arg & RS_MASK) << RS_SH;
-}
-
-static __init u32 build_rt(u32 arg)
-{
-       if (arg & ~RT_MASK)
-               printk(KERN_WARNING "TLB synthesizer field overflow\n");
-
-       return (arg & RT_MASK) << RT_SH;
-}
-
-static __init u32 build_rd(u32 arg)
-{
-       if (arg & ~RD_MASK)
-               printk(KERN_WARNING "TLB synthesizer field overflow\n");
-
-       return (arg & RD_MASK) << RD_SH;
-}
-
-static __init u32 build_re(u32 arg)
-{
-       if (arg & ~RE_MASK)
-               printk(KERN_WARNING "TLB synthesizer field overflow\n");
-
-       return (arg & RE_MASK) << RE_SH;
+       switch (current_cpu_type()) {
+       case CPU_CAVIUM_OCTEON:
+       case CPU_CAVIUM_OCTEON_PLUS:
+       case CPU_CAVIUM_OCTEON2:
+               return 1;
+       default:
+               return 0;
+       }
 }
 
-static __init u32 build_simm(s32 arg)
+static int use_lwx_insns(void)
 {
-       if (arg > 0x7fff || arg < -0x8000)
-               printk(KERN_WARNING "TLB synthesizer field overflow\n");
-
-       return arg & 0xffff;
+       switch (current_cpu_type()) {
+       case CPU_CAVIUM_OCTEON2:
+               return 1;
+       default:
+               return 0;
+       }
 }
-
-static __init u32 build_uimm(u32 arg)
+#if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \
+    CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0
+static bool scratchpad_available(void)
 {
-       if (arg & ~IMM_MASK)
-               printk(KERN_WARNING "TLB synthesizer field overflow\n");
-
-       return arg & IMM_MASK;
+       return true;
 }
-
-static __init u32 build_bimm(s32 arg)
+static int scratchpad_offset(int i)
 {
-       if (arg > 0x1ffff || arg < -0x20000)
-               printk(KERN_WARNING "TLB synthesizer field overflow\n");
-
-       if (arg & 0x3)
-               printk(KERN_WARNING "Invalid TLB synthesizer branch target\n");
-
-       return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 2) & 0x7fff);
+       /*
+        * CVMSEG starts at address -32768 and extends for
+        * CAVIUM_OCTEON_CVMSEG_SIZE 128 byte cache lines.
+        */
+       i += 1; /* Kernel use starts at the top and works down. */
+       return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768;
 }
-
-static __init u32 build_jimm(u32 arg)
+#else
+static bool scratchpad_available(void)
 {
-       if (arg & ~((JIMM_MASK) << 2))
-               printk(KERN_WARNING "TLB synthesizer field overflow\n");
-
-       return (arg >> 2) & JIMM_MASK;
+       return false;
 }
-
-static __init u32 build_func(u32 arg)
+static int scratchpad_offset(int i)
 {
-       if (arg & ~FUNC_MASK)
-               printk(KERN_WARNING "TLB synthesizer field overflow\n");
-
-       return arg & FUNC_MASK;
+       BUG();
+       /* Really unreachable, but evidently some GCC want this. */
+       return 0;
 }
-
+#endif
 /*
- * The order of opcode arguments is implicitly left to right,
- * starting with RS and ending with FUNC or IMM.
+ * Found by experiment: At least some revisions of the 4kc throw under
+ * some circumstances a machine check exception, triggered by invalid
+ * values in the index register.  Delaying the tlbp instruction until
+ * after the next branch,  plus adding an additional nop in front of
+ * tlbwi/tlbwr avoids the invalid index register values. Nobody knows
+ * why; it's not an issue caused by the core RTL.
+ *
  */
-static void __init build_insn(u32 **buf, enum opcode opc, ...)
+static int __cpuinit m4kc_tlbp_war(void)
 {
-       struct insn *ip = NULL;
-       unsigned int i;
-       va_list ap;
-       u32 op;
-
-       for (i = 0; insn_table[i].opcode != insn_invalid; i++)
-               if (insn_table[i].opcode == opc) {
-                       ip = &insn_table[i];
-                       break;
-               }
-
-       if (!ip)
-               panic("Unsupported TLB synthesizer instruction %d", opc);
-
-       op = ip->match;
-       va_start(ap, opc);
-       if (ip->fields & RS) op |= build_rs(va_arg(ap, u32));
-       if (ip->fields & RT) op |= build_rt(va_arg(ap, u32));
-       if (ip->fields & RD) op |= build_rd(va_arg(ap, u32));
-       if (ip->fields & RE) op |= build_re(va_arg(ap, u32));
-       if (ip->fields & SIMM) op |= build_simm(va_arg(ap, s32));
-       if (ip->fields & UIMM) op |= build_uimm(va_arg(ap, u32));
-       if (ip->fields & BIMM) op |= build_bimm(va_arg(ap, s32));
-       if (ip->fields & JIMM) op |= build_jimm(va_arg(ap, u32));
-       if (ip->fields & FUNC) op |= build_func(va_arg(ap, u32));
-       va_end(ap);
-
-       **buf = op;
-       (*buf)++;
+       return (current_cpu_data.processor_id & 0xffff00) ==
+              (PRID_COMP_MIPS | PRID_IMP_4KC);
 }
 
-#define I_u1u2u3(op)                                           \
-       static inline void i##op(u32 **buf, unsigned int a,     \
-               unsigned int b, unsigned int c)                 \
-       {                                                       \
-               build_insn(buf, insn##op, a, b, c);             \
-       }
-
-#define I_u2u1u3(op)                                           \
-       static inline void i##op(u32 **buf, unsigned int a,     \
-               unsigned int b, unsigned int c)                 \
-       {                                                       \
-               build_insn(buf, insn##op, b, a, c);             \
-       }
-
-#define I_u3u1u2(op)                                           \
-       static inline void i##op(u32 **buf, unsigned int a,     \
-               unsigned int b, unsigned int c)                 \
-       {                                                       \
-               build_insn(buf, insn##op, b, c, a);             \
-       }
-
-#define I_u1u2s3(op)                                           \
-       static inline void i##op(u32 **buf, unsigned int a,     \
-               unsigned int b, signed int c)                   \
-       {                                                       \
-               build_insn(buf, insn##op, a, b, c);             \
-       }
-
-#define I_u2s3u1(op)                                           \
-       static inline void i##op(u32 **buf, unsigned int a,     \
-               signed int b, unsigned int c)                   \
-       {                                                       \
-               build_insn(buf, insn##op, c, a, b);             \
-       }
-
-#define I_u2u1s3(op)                                           \
-       static inline void i##op(u32 **buf, unsigned int a,     \
-               unsigned int b, signed int c)                   \
-       {                                                       \
-               build_insn(buf, insn##op, b, a, c);             \
-       }
-
-#define I_u1u2(op)                                             \
-       static inline void i##op(u32 **buf, unsigned int a,     \
-               unsigned int b)                                 \
-       {                                                       \
-               build_insn(buf, insn##op, a, b);                \
-       }
-
-#define I_u1s2(op)                                             \
-       static inline void i##op(u32 **buf, unsigned int a,     \
-               signed int b)                                   \
-       {                                                       \
-               build_insn(buf, insn##op, a, b);                \
-       }
-
-#define I_u1(op)                                               \
-       static inline void i##op(u32 **buf, unsigned int a)     \
-       {                                                       \
-               build_insn(buf, insn##op, a);                   \
-       }
-
-#define I_0(op)                                                        \
-       static inline void i##op(u32 **buf)                     \
-       {                                                       \
-               build_insn(buf, insn##op);                      \
-       }
-
-I_u2u1s3(_addiu);
-I_u3u1u2(_addu);
-I_u2u1u3(_andi);
-I_u3u1u2(_and);
-I_u1u2s3(_beq);
-I_u1u2s3(_beql);
-I_u1s2(_bgez);
-I_u1s2(_bgezl);
-I_u1s2(_bltz);
-I_u1s2(_bltzl);
-I_u1u2s3(_bne);
-I_u1u2(_dmfc0);
-I_u1u2(_dmtc0);
-I_u2u1s3(_daddiu);
-I_u3u1u2(_daddu);
-I_u2u1u3(_dsll);
-I_u2u1u3(_dsll32);
-I_u2u1u3(_dsra);
-I_u2u1u3(_dsrl);
-I_u2u1u3(_dsrl32);
-I_u3u1u2(_dsubu);
-I_0(_eret);
-I_u1(_j);
-I_u1(_jal);
-I_u1(_jr);
-I_u2s3u1(_ld);
-I_u2s3u1(_ll);
-I_u2s3u1(_lld);
-I_u1s2(_lui);
-I_u2s3u1(_lw);
-I_u1u2(_mfc0);
-I_u1u2(_mtc0);
-I_u2u1u3(_ori);
-I_0(_rfe);
-I_u2s3u1(_sc);
-I_u2s3u1(_scd);
-I_u2s3u1(_sd);
-I_u2u1u3(_sll);
-I_u2u1u3(_sra);
-I_u2u1u3(_srl);
-I_u3u1u2(_subu);
-I_u2s3u1(_sw);
-I_0(_tlbp);
-I_0(_tlbwi);
-I_0(_tlbwr);
-I_u3u1u2(_xor)
-I_u2u1u3(_xori);
-
-/*
- * handling labels
- */
-
+/* Handle labels (which must be positive integers). */
 enum label_id {
-       label_invalid,
-       label_second_part,
+       label_second_part = 1,
        label_leave,
        label_vmalloc,
        label_vmalloc_done,
        label_tlbw_hazard,
        label_split,
+       label_tlbl_goaround1,
+       label_tlbl_goaround2,
        label_nopage_tlbl,
        label_nopage_tlbs,
        label_nopage_tlbm,
        label_smp_pgtable_change,
        label_r3000_write_probe_fail,
-       label_r3000_write_probe_ok
+       label_large_segbits_fault,
+#ifdef CONFIG_HUGETLB_PAGE
+       label_tlb_huge_update,
+#endif
 };
 
-struct label {
-       u32 *addr;
-       enum label_id lab;
-};
+UASM_L_LA(_second_part)
+UASM_L_LA(_leave)
+UASM_L_LA(_vmalloc)
+UASM_L_LA(_vmalloc_done)
+UASM_L_LA(_tlbw_hazard)
+UASM_L_LA(_split)
+UASM_L_LA(_tlbl_goaround1)
+UASM_L_LA(_tlbl_goaround2)
+UASM_L_LA(_nopage_tlbl)
+UASM_L_LA(_nopage_tlbs)
+UASM_L_LA(_nopage_tlbm)
+UASM_L_LA(_smp_pgtable_change)
+UASM_L_LA(_r3000_write_probe_fail)
+UASM_L_LA(_large_segbits_fault)
+#ifdef CONFIG_HUGETLB_PAGE
+UASM_L_LA(_tlb_huge_update)
+#endif
 
-static __init void build_label(struct label **lab, u32 *addr,
-                              enum label_id l)
+/*
+ * For debug purposes.
+ */
+static inline void dump_handler(const u32 *handler, int count)
 {
-       (*lab)->addr = addr;
-       (*lab)->lab = l;
-       (*lab)++;
-}
-
-#define L_LA(lb)                                               \
-       static inline void l##lb(struct label **lab, u32 *addr) \
-       {                                                       \
-               build_label(lab, addr, label##lb);              \
-       }
+       int i;
 
-L_LA(_second_part)
-L_LA(_leave)
-L_LA(_vmalloc)
-L_LA(_vmalloc_done)
-L_LA(_tlbw_hazard)
-L_LA(_split)
-L_LA(_nopage_tlbl)
-L_LA(_nopage_tlbs)
-L_LA(_nopage_tlbm)
-L_LA(_smp_pgtable_change)
-L_LA(_r3000_write_probe_fail)
-L_LA(_r3000_write_probe_ok)
-
-/* convenience macros for instructions */
-#ifdef CONFIG_64BIT
-# define i_LW(buf, rs, rt, off) i_ld(buf, rs, rt, off)
-# define i_SW(buf, rs, rt, off) i_sd(buf, rs, rt, off)
-# define i_SLL(buf, rs, rt, sh) i_dsll(buf, rs, rt, sh)
-# define i_SRA(buf, rs, rt, sh) i_dsra(buf, rs, rt, sh)
-# define i_SRL(buf, rs, rt, sh) i_dsrl(buf, rs, rt, sh)
-# define i_MFC0(buf, rt, rd) i_dmfc0(buf, rt, rd)
-# define i_MTC0(buf, rt, rd) i_dmtc0(buf, rt, rd)
-# define i_ADDIU(buf, rs, rt, val) i_daddiu(buf, rs, rt, val)
-# define i_ADDU(buf, rs, rt, rd) i_daddu(buf, rs, rt, rd)
-# define i_SUBU(buf, rs, rt, rd) i_dsubu(buf, rs, rt, rd)
-# define i_LL(buf, rs, rt, off) i_lld(buf, rs, rt, off)
-# define i_SC(buf, rs, rt, off) i_scd(buf, rs, rt, off)
-#else
-# define i_LW(buf, rs, rt, off) i_lw(buf, rs, rt, off)
-# define i_SW(buf, rs, rt, off) i_sw(buf, rs, rt, off)
-# define i_SLL(buf, rs, rt, sh) i_sll(buf, rs, rt, sh)
-# define i_SRA(buf, rs, rt, sh) i_sra(buf, rs, rt, sh)
-# define i_SRL(buf, rs, rt, sh) i_srl(buf, rs, rt, sh)
-# define i_MFC0(buf, rt, rd) i_mfc0(buf, rt, rd)
-# define i_MTC0(buf, rt, rd) i_mtc0(buf, rt, rd)
-# define i_ADDIU(buf, rs, rt, val) i_addiu(buf, rs, rt, val)
-# define i_ADDU(buf, rs, rt, rd) i_addu(buf, rs, rt, rd)
-# define i_SUBU(buf, rs, rt, rd) i_subu(buf, rs, rt, rd)
-# define i_LL(buf, rs, rt, off) i_ll(buf, rs, rt, off)
-# define i_SC(buf, rs, rt, off) i_sc(buf, rs, rt, off)
-#endif
+       pr_debug("\t.set push\n");
+       pr_debug("\t.set noreorder\n");
 
-#define i_b(buf, off) i_beq(buf, 0, 0, off)
-#define i_beqz(buf, rs, off) i_beq(buf, rs, 0, off)
-#define i_beqzl(buf, rs, off) i_beql(buf, rs, 0, off)
-#define i_bnez(buf, rs, off) i_bne(buf, rs, 0, off)
-#define i_bnezl(buf, rs, off) i_bnel(buf, rs, 0, off)
-#define i_move(buf, a, b) i_ADDU(buf, a, 0, b)
-#define i_nop(buf) i_sll(buf, 0, 0, 0)
-#define i_ssnop(buf) i_sll(buf, 0, 0, 1)
-#define i_ehb(buf) i_sll(buf, 0, 0, 3)
+       for (i = 0; i < count; i++)
+               pr_debug("\t%p\t.word 0x%08x\n", &handler[i], handler[i]);
 
-#ifdef CONFIG_64BIT
-static __init int __attribute__((unused)) in_compat_space_p(long addr)
-{
-       /* Is this address in 32bit compat space? */
-       return (((addr) & 0xffffffff00000000) == 0xffffffff00000000);
+       pr_debug("\t.set pop\n");
 }
 
-static __init int __attribute__((unused)) rel_highest(long val)
-{
-       return ((((val + 0x800080008000L) >> 48) & 0xffff) ^ 0x8000) - 0x8000;
-}
+/* The only general purpose registers allowed in TLB handlers. */
+#define K0             26
+#define K1             27
 
-static __init int __attribute__((unused)) rel_higher(long val)
-{
-       return ((((val + 0x80008000L) >> 32) & 0xffff) ^ 0x8000) - 0x8000;
-}
+/* Some CP0 registers */
+#define C0_INDEX       0, 0
+#define C0_ENTRYLO0    2, 0
+#define C0_TCBIND      2, 2
+#define C0_ENTRYLO1    3, 0
+#define C0_CONTEXT     4, 0
+#define C0_PAGEMASK    5, 0
+#define C0_BADVADDR    8, 0
+#define C0_ENTRYHI     10, 0
+#define C0_EPC         14, 0
+#define C0_XCONTEXT    20, 0
+
+#ifdef CONFIG_64BIT
+# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT)
+#else
+# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT)
 #endif
 
-static __init int rel_hi(long val)
-{
-       return ((((val + 0x8000L) >> 16) & 0xffff) ^ 0x8000) - 0x8000;
-}
+/* The worst case length of the handler is around 18 instructions for
+ * R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
+ * Maximum space available is 32 instructions for R3000 and 64
+ * instructions for R4000.
+ *
+ * We deliberately chose a buffer size of 128, so we won't scribble
+ * over anything important on overflow before we panic.
+ */
+static u32 tlb_handler[128] __cpuinitdata;
 
-static __init int rel_lo(long val)
-{
-       return ((val & 0xffff) ^ 0x8000) - 0x8000;
-}
+/* simply assume worst case size for labels and relocs */
+static struct uasm_label labels[128] __cpuinitdata;
+static struct uasm_reloc relocs[128] __cpuinitdata;
 
-static __init void i_LA_mostly(u32 **buf, unsigned int rs, long addr)
-{
 #ifdef CONFIG_64BIT
-       if (!in_compat_space_p(addr)) {
-               i_lui(buf, rs, rel_highest(addr));
-               if (rel_higher(addr))
-                       i_daddiu(buf, rs, rs, rel_higher(addr));
-               if (rel_hi(addr)) {
-                       i_dsll(buf, rs, rs, 16);
-                       i_daddiu(buf, rs, rs, rel_hi(addr));
-                       i_dsll(buf, rs, rs, 16);
-               } else
-                       i_dsll32(buf, rs, rs, 0);
-       } else
+static int check_for_high_segbits __cpuinitdata;
 #endif
-               i_lui(buf, rs, rel_hi(addr));
-}
-
-static __init void __attribute__((unused)) i_LA(u32 **buf, unsigned int rs,
-                                               long addr)
-{
-       i_LA_mostly(buf, rs, addr);
-       if (rel_lo(addr))
-               i_ADDIU(buf, rs, rs, rel_lo(addr));
-}
 
-/*
- * handle relocations
- */
-
-struct reloc {
-       u32 *addr;
-       unsigned int type;
-       enum label_id lab;
-};
+static int check_for_high_segbits __cpuinitdata;
 
-static __init void r_mips_pc16(struct reloc **rel, u32 *addr,
-                              enum label_id l)
-{
-       (*rel)->addr = addr;
-       (*rel)->type = R_MIPS_PC16;
-       (*rel)->lab = l;
-       (*rel)++;
-}
+static unsigned int kscratch_used_mask __cpuinitdata;
 
-static inline void __resolve_relocs(struct reloc *rel, struct label *lab)
+static int __cpuinit allocate_kscratch(void)
 {
-       long laddr = (long)lab->addr;
-       long raddr = (long)rel->addr;
+       int r;
+       unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask;
 
-       switch (rel->type) {
-       case R_MIPS_PC16:
-               *rel->addr |= build_bimm(laddr - (raddr + 4));
-               break;
+       r = ffs(a);
 
-       default:
-               panic("Unsupported TLB synthesizer relocation %d",
-                     rel->type);
-       }
-}
+       if (r == 0)
+               return -1;
 
-static __init void resolve_relocs(struct reloc *rel, struct label *lab)
-{
-       struct label *l;
+       r--; /* make it zero based */
 
-       for (; rel->lab != label_invalid; rel++)
-               for (l = lab; l->lab != label_invalid; l++)
-                       if (rel->lab == l->lab)
-                               __resolve_relocs(rel, l);
-}
+       kscratch_used_mask |= (1 << r);
 
-static __init void move_relocs(struct reloc *rel, u32 *first, u32 *end,
-                              long off)
-{
-       for (; rel->lab != label_invalid; rel++)
-               if (rel->addr >= first && rel->addr < end)
-                       rel->addr += off;
+       return r;
 }
 
-static __init void move_labels(struct label *lab, u32 *first, u32 *end,
-                              long off)
-{
-       for (; lab->lab != label_invalid; lab++)
-               if (lab->addr >= first && lab->addr < end)
-                       lab->addr += off;
-}
+static int scratch_reg __cpuinitdata;
+static int pgd_reg __cpuinitdata;
+enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch};
 
-static __init void copy_handler(struct reloc *rel, struct label *lab,
-                               u32 *first, u32 *end, u32 *target)
+static struct work_registers __cpuinit build_get_work_registers(u32 **p)
 {
-       long off = (long)(target - first);
-
-       memcpy(target, first, (end - first) * sizeof(u32));
+       struct work_registers r;
 
-       move_relocs(rel, first, end, off);
-       move_labels(lab, first, end, off);
-}
+       int smp_processor_id_reg;
+       int smp_processor_id_sel;
+       int smp_processor_id_shift;
 
-static __init int __attribute__((unused)) insn_has_bdelay(struct reloc *rel,
-                                                         u32 *addr)
-{
-       for (; rel->lab != label_invalid; rel++) {
-               if (rel->addr == addr
-                   && (rel->type == R_MIPS_PC16
-                       || rel->type == R_MIPS_26))
-                       return 1;
+       if (scratch_reg > 0) {
+               /* Save in CPU local C0_KScratch? */
+               UASM_i_MTC0(p, 1, 31, scratch_reg);
+               r.r1 = K0;
+               r.r2 = K1;
+               r.r3 = 1;
+               return r;
        }
 
-       return 0;
-}
-
-/* convenience functions for labeled branches */
-static void __attribute__((unused)) il_bltz(u32 **p, struct reloc **r,
-                                           unsigned int reg, enum label_id l)
-{
-       r_mips_pc16(r, *p, l);
-       i_bltz(p, reg, 0);
-}
-
-static void __attribute__((unused)) il_b(u32 **p, struct reloc **r,
-                                        enum label_id l)
-{
-       r_mips_pc16(r, *p, l);
-       i_b(p, 0);
-}
+       if (num_possible_cpus() > 1) {
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
+               smp_processor_id_shift = 51;
+               smp_processor_id_reg = 20; /* XContext */
+               smp_processor_id_sel = 0;
+#else
+# ifdef CONFIG_32BIT
+               smp_processor_id_shift = 25;
+               smp_processor_id_reg = 4; /* Context */
+               smp_processor_id_sel = 0;
+# endif
+# ifdef CONFIG_64BIT
+               smp_processor_id_shift = 26;
+               smp_processor_id_reg = 4; /* Context */
+               smp_processor_id_sel = 0;
+# endif
+#endif
+               /* Get smp_processor_id */
+               UASM_i_MFC0(p, K0, smp_processor_id_reg, smp_processor_id_sel);
+               UASM_i_SRL_SAFE(p, K0, K0, smp_processor_id_shift);
 
-static void il_beqz(u32 **p, struct reloc **r, unsigned int reg,
-                   enum label_id l)
-{
-       r_mips_pc16(r, *p, l);
-       i_beqz(p, reg, 0);
-}
+               /* handler_reg_save index in K0 */
+               UASM_i_SLL(p, K0, K0, ilog2(sizeof(struct tlb_reg_save)));
 
-static void __attribute__((unused))
-il_beqzl(u32 **p, struct reloc **r, unsigned int reg, enum label_id l)
-{
-       r_mips_pc16(r, *p, l);
-       i_beqzl(p, reg, 0);
-}
+               UASM_i_LA(p, K1, (long)&handler_reg_save);
+               UASM_i_ADDU(p, K0, K0, K1);
+       } else {
+               UASM_i_LA(p, K0, (long)&handler_reg_save);
+       }
+       /* K0 now points to save area, save $1 and $2  */
+       UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), K0);
+       UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), K0);
 
-static void il_bnez(u32 **p, struct reloc **r, unsigned int reg,
-                   enum label_id l)
-{
-       r_mips_pc16(r, *p, l);
-       i_bnez(p, reg, 0);
+       r.r1 = K1;
+       r.r2 = 1;
+       r.r3 = 2;
+       return r;
 }
 
-static void il_bgezl(u32 **p, struct reloc **r, unsigned int reg,
-                    enum label_id l)
+static void __cpuinit build_restore_work_registers(u32 **p)
 {
-       r_mips_pc16(r, *p, l);
-       i_bgezl(p, reg, 0);
+       if (scratch_reg > 0) {
+               UASM_i_MFC0(p, 1, 31, scratch_reg);
+               return;
+       }
+       /* K0 already points to save area, restore $1 and $2  */
+       UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), K0);
+       UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), K0);
 }
 
-/* The only general purpose registers allowed in TLB handlers. */
-#define K0             26
-#define K1             27
-
-/* Some CP0 registers */
-#define C0_INDEX       0
-#define C0_ENTRYLO0    2
-#define C0_ENTRYLO1    3
-#define C0_CONTEXT     4
-#define C0_BADVADDR    8
-#define C0_ENTRYHI     10
-#define C0_EPC         14
-#define C0_XCONTEXT    20
-
-#ifdef CONFIG_64BIT
-# define GET_CONTEXT(buf, reg) i_MFC0(buf, reg, C0_XCONTEXT)
-#else
-# define GET_CONTEXT(buf, reg) i_MFC0(buf, reg, C0_CONTEXT)
-#endif
+#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
 
-/* The worst case length of the handler is around 18 instructions for
- * R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
- * Maximum space available is 32 instructions for R3000 and 64
- * instructions for R4000.
+/*
+ * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current,
+ * we cannot do r3000 under these circumstances.
  *
- * We deliberately chose a buffer size of 128, so we won't scribble
- * over anything important on overflow before we panic.
+ * Declare pgd_current here instead of including mmu_context.h to avoid type
+ * conflicts for tlbmiss_handler_setup_pgd
  */
-static __initdata u32 tlb_handler[128];
-
-/* simply assume worst case size for labels and relocs */
-static __initdata struct label labels[128];
-static __initdata struct reloc relocs[128];
+extern unsigned long pgd_current[];
 
 /*
  * The R3000 TLB handler is simple.
  */
-static void __init build_r3000_tlb_refill_handler(void)
+static void __cpuinit build_r3000_tlb_refill_handler(void)
 {
        long pgdc = (long)pgd_current;
        u32 *p;
@@ -713,41 +349,35 @@ static void __init build_r3000_tlb_refill_handler(void)
        memset(tlb_handler, 0, sizeof(tlb_handler));
        p = tlb_handler;
 
-       i_mfc0(&p, K0, C0_BADVADDR);
-       i_lui(&p, K1, rel_hi(pgdc)); /* cp0 delay */
-       i_lw(&p, K1, rel_lo(pgdc), K1);
-       i_srl(&p, K0, K0, 22); /* load delay */
-       i_sll(&p, K0, K0, 2);
-       i_addu(&p, K1, K1, K0);
-       i_mfc0(&p, K0, C0_CONTEXT);
-       i_lw(&p, K1, 0, K1); /* cp0 delay */
-       i_andi(&p, K0, K0, 0xffc); /* load delay */
-       i_addu(&p, K1, K1, K0);
-       i_lw(&p, K0, 0, K1);
-       i_nop(&p); /* load delay */
-       i_mtc0(&p, K0, C0_ENTRYLO0);
-       i_mfc0(&p, K1, C0_EPC); /* cp0 delay */
-       i_tlbwr(&p); /* cp0 delay */
-       i_jr(&p, K1);
-       i_rfe(&p); /* branch delay */
+       uasm_i_mfc0(&p, K0, C0_BADVADDR);
+       uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */
+       uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1);
+       uasm_i_srl(&p, K0, K0, 22); /* load delay */
+       uasm_i_sll(&p, K0, K0, 2);
+       uasm_i_addu(&p, K1, K1, K0);
+       uasm_i_mfc0(&p, K0, C0_CONTEXT);
+       uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */
+       uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */
+       uasm_i_addu(&p, K1, K1, K0);
+       uasm_i_lw(&p, K0, 0, K1);
+       uasm_i_nop(&p); /* load delay */
+       uasm_i_mtc0(&p, K0, C0_ENTRYLO0);
+       uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */
+       uasm_i_tlbwr(&p); /* cp0 delay */
+       uasm_i_jr(&p, K1);
+       uasm_i_rfe(&p); /* branch delay */
 
        if (p > tlb_handler + 32)
                panic("TLB refill handler space exceeded");
 
-       printk("Synthesized TLB handler (%u instructions).\n",
-              (unsigned int)(p - tlb_handler));
-#ifdef DEBUG_TLB
-       {
-               int i;
+       pr_debug("Wrote TLB refill handler (%u instructions).\n",
+                (unsigned int)(p - tlb_handler));
 
-               for (i = 0; i < (p - tlb_handler); i++)
-                       printk("%08x\n", tlb_handler[i]);
-       }
-#endif
+       memcpy((void *)ebase, tlb_handler, 0x80);
 
-       memcpy((void *)CAC_BASE, tlb_handler, 0x80);
-       flush_icache_range(CAC_BASE, CAC_BASE + 0x80);
+       dump_handler((u32 *)ebase, 32);
 }
+#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
 
 /*
  * The R4000 TLB handler is much more complicated. We have two
@@ -756,7 +386,7 @@ static void __init build_r3000_tlb_refill_handler(void)
  * other one.To keep things simple, we first assume linear space,
  * then we relocate it to the final handler layout as needed.
  */
-static __initdata u32 final_handler[64];
+static u32 final_handler[64] __cpuinitdata;
 
 /*
  * Hazards
@@ -780,40 +410,50 @@ static __initdata u32 final_handler[64];
  *
  * As if we MIPS hackers wouldn't know how to nop pipelines happy ...
  */
-static __init void __attribute__((unused)) build_tlb_probe_entry(u32 **p)
+static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p)
 {
-       switch (current_cpu_data.cputype) {
+       switch (current_cpu_type()) {
+       /* Found by experiment: R4600 v2.0/R4700 needs this, too.  */
+       case CPU_R4600:
+       case CPU_R4700:
        case CPU_R5000:
        case CPU_R5000A:
        case CPU_NEVADA:
-               i_nop(p);
-               i_tlbp(p);
+               uasm_i_nop(p);
+               uasm_i_tlbp(p);
                break;
 
        default:
-               i_tlbp(p);
+               uasm_i_tlbp(p);
                break;
        }
 }
 
 /*
  * Write random or indexed TLB entry, and care about the hazards from
- * the preceeding mtc0 and for the following eret.
+ * the preceding mtc0 and for the following eret.
  */
 enum tlb_write_entry { tlb_random, tlb_indexed };
 
-static __init void build_tlb_write_entry(u32 **p, struct label **l,
-                                        struct reloc **r,
+static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
+                                        struct uasm_reloc **r,
                                         enum tlb_write_entry wmode)
 {
        void(*tlbw)(u32 **) = NULL;
 
        switch (wmode) {
-       case tlb_random: tlbw = i_tlbwr; break;
-       case tlb_indexed: tlbw = i_tlbwi; break;
+       case tlb_random: tlbw = uasm_i_tlbwr; break;
+       case tlb_indexed: tlbw = uasm_i_tlbwi; break;
+       }
+
+       if (cpu_has_mips_r2) {
+               if (cpu_has_mips_r2_exec_hazard)
+                       uasm_i_ehb(p);
+               tlbw(p);
+               return;
        }
 
-       switch (current_cpu_data.cputype) {
+       switch (current_cpu_type()) {
        case CPU_R4000PC:
        case CPU_R4000SC:
        case CPU_R4000MC:
@@ -824,58 +464,69 @@ static __init void build_tlb_write_entry(u32 **p, struct label **l,
                 * This branch uses up a mtc0 hazard nop slot and saves
                 * two nops after the tlbw instruction.
                 */
-               il_bgezl(p, r, 0, label_tlbw_hazard);
+               uasm_il_bgezl(p, r, 0, label_tlbw_hazard);
                tlbw(p);
-               l_tlbw_hazard(l, *p);
-               i_nop(p);
+               uasm_l_tlbw_hazard(l, *p);
+               uasm_i_nop(p);
                break;
 
        case CPU_R4600:
        case CPU_R4700:
        case CPU_R5000:
        case CPU_R5000A:
+               uasm_i_nop(p);
+               tlbw(p);
+               uasm_i_nop(p);
+               break;
+
+       case CPU_R4300:
        case CPU_5KC:
        case CPU_TX49XX:
-       case CPU_AU1000:
-       case CPU_AU1100:
-       case CPU_AU1500:
-       case CPU_AU1550:
-               i_nop(p);
+       case CPU_PR4450:
+       case CPU_XLR:
+               uasm_i_nop(p);
                tlbw(p);
                break;
 
        case CPU_R10000:
        case CPU_R12000:
+       case CPU_R14000:
        case CPU_4KC:
+       case CPU_4KEC:
        case CPU_SB1:
+       case CPU_SB1A:
        case CPU_4KSC:
        case CPU_20KC:
        case CPU_25KF:
+       case CPU_BMIPS32:
+       case CPU_BMIPS3300:
+       case CPU_BMIPS4350:
+       case CPU_BMIPS4380:
+       case CPU_BMIPS5000:
+       case CPU_LOONGSON2:
+       case CPU_R5500:
+               if (m4kc_tlbp_war())
+                       uasm_i_nop(p);
+       case CPU_ALCHEMY:
                tlbw(p);
                break;
 
        case CPU_NEVADA:
-               i_nop(p); /* QED specifies 2 nops hazard */
+               uasm_i_nop(p); /* QED specifies 2 nops hazard */
                /*
                 * This branch uses up a mtc0 hazard nop slot and saves
                 * a nop after the tlbw instruction.
                 */
-               il_bgezl(p, r, 0, label_tlbw_hazard);
+               uasm_il_bgezl(p, r, 0, label_tlbw_hazard);
                tlbw(p);
-               l_tlbw_hazard(l, *p);
+               uasm_l_tlbw_hazard(l, *p);
                break;
 
        case CPU_RM7000:
-               i_nop(p);
-               i_nop(p);
-               i_nop(p);
-               i_nop(p);
-               tlbw(p);
-               break;
-
-       case CPU_4KEC:
-       case CPU_24K:
-               i_ehb(p);
+               uasm_i_nop(p);
+               uasm_i_nop(p);
+               uasm_i_nop(p);
+               uasm_i_nop(p);
                tlbw(p);
                break;
 
@@ -886,15 +537,15 @@ static __init void build_tlb_write_entry(u32 **p, struct label **l,
                 * cpu cycles and use for data translations should not occur
                 * for 3 cpu cycles.
                 */
-               i_ssnop(p);
-               i_ssnop(p);
-               i_ssnop(p);
-               i_ssnop(p);
+               uasm_i_ssnop(p);
+               uasm_i_ssnop(p);
+               uasm_i_ssnop(p);
+               uasm_i_ssnop(p);
                tlbw(p);
-               i_ssnop(p);
-               i_ssnop(p);
-               i_ssnop(p);
-               i_ssnop(p);
+               uasm_i_ssnop(p);
+               uasm_i_ssnop(p);
+               uasm_i_ssnop(p);
+               uasm_i_ssnop(p);
                break;
 
        case CPU_VR4111:
@@ -902,18 +553,24 @@ static __init void build_tlb_write_entry(u32 **p, struct label **l,
        case CPU_VR4122:
        case CPU_VR4181:
        case CPU_VR4181A:
-               i_nop(p);
-               i_nop(p);
+               uasm_i_nop(p);
+               uasm_i_nop(p);
                tlbw(p);
-               i_nop(p);
-               i_nop(p);
+               uasm_i_nop(p);
+               uasm_i_nop(p);
                break;
 
        case CPU_VR4131:
        case CPU_VR4133:
-               i_nop(p);
-               i_nop(p);
+       case CPU_R5432:
+               uasm_i_nop(p);
+               uasm_i_nop(p);
+               tlbw(p);
+               break;
+
+       case CPU_JZRISC:
                tlbw(p);
+               uasm_i_nop(p);
                break;
 
        default:
@@ -923,90 +580,307 @@ static __init void build_tlb_write_entry(u32 **p, struct label **l,
        }
 }
 
+static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p,
+                                                                 unsigned int reg)
+{
+       if (kernel_uses_smartmips_rixi) {
+               UASM_i_SRL(p, reg, reg, ilog2(_PAGE_NO_EXEC));
+               UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+       } else {
+#ifdef CONFIG_64BIT_PHYS_ADDR
+               uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL));
+#else
+               UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL));
+#endif
+       }
+}
+
+#ifdef CONFIG_HUGETLB_PAGE
+
+static __cpuinit void build_restore_pagemask(u32 **p,
+                                            struct uasm_reloc **r,
+                                            unsigned int tmp,
+                                            enum label_id lid,
+                                            int restore_scratch)
+{
+       if (restore_scratch) {
+               /* Reset default page size */
+               if (PM_DEFAULT_MASK >> 16) {
+                       uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
+                       uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
+                       uasm_i_mtc0(p, tmp, C0_PAGEMASK);
+                       uasm_il_b(p, r, lid);
+               } else if (PM_DEFAULT_MASK) {
+                       uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
+                       uasm_i_mtc0(p, tmp, C0_PAGEMASK);
+                       uasm_il_b(p, r, lid);
+               } else {
+                       uasm_i_mtc0(p, 0, C0_PAGEMASK);
+                       uasm_il_b(p, r, lid);
+               }
+               if (scratch_reg > 0)
+                       UASM_i_MFC0(p, 1, 31, scratch_reg);
+               else
+                       UASM_i_LW(p, 1, scratchpad_offset(0), 0);
+       } else {
+               /* Reset default page size */
+               if (PM_DEFAULT_MASK >> 16) {
+                       uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
+                       uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
+                       uasm_il_b(p, r, lid);
+                       uasm_i_mtc0(p, tmp, C0_PAGEMASK);
+               } else if (PM_DEFAULT_MASK) {
+                       uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
+                       uasm_il_b(p, r, lid);
+                       uasm_i_mtc0(p, tmp, C0_PAGEMASK);
+               } else {
+                       uasm_il_b(p, r, lid);
+                       uasm_i_mtc0(p, 0, C0_PAGEMASK);
+               }
+       }
+}
+
+static __cpuinit void build_huge_tlb_write_entry(u32 **p,
+                                                struct uasm_label **l,
+                                                struct uasm_reloc **r,
+                                                unsigned int tmp,
+                                                enum tlb_write_entry wmode,
+                                                int restore_scratch)
+{
+       /* Set huge page tlb entry size */
+       uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16);
+       uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff);
+       uasm_i_mtc0(p, tmp, C0_PAGEMASK);
+
+       build_tlb_write_entry(p, l, r, wmode);
+
+       build_restore_pagemask(p, r, tmp, label_leave, restore_scratch);
+}
+
+/*
+ * Check if Huge PTE is present, if so then jump to LABEL.
+ */
+static void __cpuinit
+build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp,
+               unsigned int pmd, int lid)
+{
+       UASM_i_LW(p, tmp, 0, pmd);
+       if (use_bbit_insns()) {
+               uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid);
+       } else {
+               uasm_i_andi(p, tmp, tmp, _PAGE_HUGE);
+               uasm_il_bnez(p, r, tmp, lid);
+       }
+}
+
+static __cpuinit void build_huge_update_entries(u32 **p,
+                                               unsigned int pte,
+                                               unsigned int tmp)
+{
+       int small_sequence;
+
+       /*
+        * A huge PTE describes an area the size of the
+        * configured huge page size. This is twice the
+        * of the large TLB entry size we intend to use.
+        * A TLB entry half the size of the configured
+        * huge page size is configured into entrylo0
+        * and entrylo1 to cover the contiguous huge PTE
+        * address space.
+        */
+       small_sequence = (HPAGE_SIZE >> 7) < 0x10000;
+
+       /* We can clobber tmp.  It isn't used after this.*/
+       if (!small_sequence)
+               uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16));
+
+       build_convert_pte_to_entrylo(p, pte);
+       UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */
+       /* convert to entrylo1 */
+       if (small_sequence)
+               UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7);
+       else
+               UASM_i_ADDU(p, pte, pte, tmp);
+
+       UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */
+}
+
+static __cpuinit void build_huge_handler_tail(u32 **p,
+                                             struct uasm_reloc **r,
+                                             struct uasm_label **l,
+                                             unsigned int pte,
+                                             unsigned int ptr)
+{
+#ifdef CONFIG_SMP
+       UASM_i_SC(p, pte, 0, ptr);
+       uasm_il_beqz(p, r, pte, label_tlb_huge_update);
+       UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */
+#else
+       UASM_i_SW(p, pte, 0, ptr);
+#endif
+       build_huge_update_entries(p, pte, ptr);
+       build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0);
+}
+#endif /* CONFIG_HUGETLB_PAGE */
+
 #ifdef CONFIG_64BIT
 /*
  * TMP and PTR are scratch.
  * TMP will be clobbered, PTR will hold the pmd entry.
  */
-static __init void
-build_get_pmde64(u32 **p, struct label **l, struct reloc **r,
+static void __cpuinit
+build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
                 unsigned int tmp, unsigned int ptr)
 {
+#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
        long pgdc = (long)pgd_current;
-
+#endif
        /*
         * The vmalloc handling is not in the hotpath.
         */
-       i_dmfc0(p, tmp, C0_BADVADDR);
-       il_bltz(p, r, tmp, label_vmalloc);
-       /* No i_nop needed here, since the next insn doesn't touch TMP. */
+       uasm_i_dmfc0(p, tmp, C0_BADVADDR);
 
-#ifdef CONFIG_SMP
+       if (check_for_high_segbits) {
+               /*
+                * The kernel currently implicitely assumes that the
+                * MIPS SEGBITS parameter for the processor is
+                * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never
+                * allocate virtual addresses outside the maximum
+                * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But
+                * that doesn't prevent user code from accessing the
+                * higher xuseg addresses.  Here, we make sure that
+                * everything but the lower xuseg addresses goes down
+                * the module_alloc/vmalloc path.
+                */
+               uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
+               uasm_il_bnez(p, r, ptr, label_vmalloc);
+       } else {
+               uasm_il_bltz(p, r, tmp, label_vmalloc);
+       }
+       /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
+
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
+       if (pgd_reg != -1) {
+               /* pgd is in pgd_reg */
+               UASM_i_MFC0(p, ptr, 31, pgd_reg);
+       } else {
+               /*
+                * &pgd << 11 stored in CONTEXT [23..63].
+                */
+               UASM_i_MFC0(p, ptr, C0_CONTEXT);
+
+               /* Clear lower 23 bits of context. */
+               uasm_i_dins(p, ptr, 0, 0, 23);
+
+               /* 1 0  1 0 1  << 6  xkphys cached */
+               uasm_i_ori(p, ptr, ptr, 0x540);
+               uasm_i_drotr(p, ptr, ptr, 11);
+       }
+#elif defined(CONFIG_SMP)
+# ifdef  CONFIG_MIPS_MT_SMTC
+       /*
+        * SMTC uses TCBind value as "CPU" index
+        */
+       uasm_i_mfc0(p, ptr, C0_TCBIND);
+       uasm_i_dsrl_safe(p, ptr, ptr, 19);
+# else
        /*
-        * 64 bit SMP has the lower part of &pgd_current[smp_processor_id()]
+        * 64 bit SMP running in XKPHYS has smp_processor_id() << 3
         * stored in CONTEXT.
         */
-       if (in_compat_space_p(pgdc)) {
-               i_dmfc0(p, ptr, C0_CONTEXT);
-               i_dsra(p, ptr, ptr, 23);
-               i_ld(p, ptr, 0, ptr);
-       } else {
-#ifdef CONFIG_BUILD_ELF64
-               i_dmfc0(p, ptr, C0_CONTEXT);
-               i_dsrl(p, ptr, ptr, 23);
-               i_dsll(p, ptr, ptr, 3);
-               i_LA_mostly(p, tmp, pgdc);
-               i_daddu(p, ptr, ptr, tmp);
-               i_dmfc0(p, tmp, C0_BADVADDR);
-               i_ld(p, ptr, rel_lo(pgdc), ptr);
-#else
-               i_dmfc0(p, ptr, C0_CONTEXT);
-               i_lui(p, tmp, rel_highest(pgdc));
-               i_dsll(p, ptr, ptr, 9);
-               i_daddiu(p, tmp, tmp, rel_higher(pgdc));
-               i_dsrl32(p, ptr, ptr, 0);
-               i_and(p, ptr, ptr, tmp);
-               i_dmfc0(p, tmp, C0_BADVADDR);
-               i_ld(p, ptr, 0, ptr);
-#endif
-       }
+       uasm_i_dmfc0(p, ptr, C0_CONTEXT);
+       uasm_i_dsrl_safe(p, ptr, ptr, 23);
+# endif
+       UASM_i_LA_mostly(p, tmp, pgdc);
+       uasm_i_daddu(p, ptr, ptr, tmp);
+       uasm_i_dmfc0(p, tmp, C0_BADVADDR);
+       uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
 #else
-       i_LA_mostly(p, ptr, pgdc);
-       i_ld(p, ptr, rel_lo(pgdc), ptr);
+       UASM_i_LA_mostly(p, ptr, pgdc);
+       uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
 #endif
 
-       l_vmalloc_done(l, *p);
-       i_dsrl(p, tmp, tmp, PGDIR_SHIFT-3); /* get pgd offset in bytes */
-       i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
-       i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
-       i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
-       i_ld(p, ptr, 0, ptr); /* get pmd pointer */
-       i_dsrl(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
-       i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
-       i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
+       uasm_l_vmalloc_done(l, *p);
+
+       /* get pgd offset in bytes */
+       uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3);
+
+       uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
+       uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
+#ifndef __PAGETABLE_PMD_FOLDED
+       uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
+       uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */
+       uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
+       uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
+       uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
+#endif
 }
 
 /*
  * BVADDR is the faulting address, PTR is scratch.
  * PTR will hold the pgd for vmalloc.
  */
-static __init void
-build_get_pgd_vmalloc64(u32 **p, struct label **l, struct reloc **r,
-                       unsigned int bvaddr, unsigned int ptr)
+static void __cpuinit
+build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
+                       unsigned int bvaddr, unsigned int ptr,
+                       enum vmalloc64_mode mode)
 {
        long swpd = (long)swapper_pg_dir;
+       int single_insn_swpd;
+       int did_vmalloc_branch = 0;
 
-       l_vmalloc(l, *p);
-       i_LA(p, ptr, VMALLOC_START);
-       i_dsubu(p, bvaddr, bvaddr, ptr);
+       single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd);
 
-       if (in_compat_space_p(swpd) && !rel_lo(swpd)) {
-               il_b(p, r, label_vmalloc_done);
-               i_lui(p, ptr, rel_hi(swpd));
-       } else {
-               i_LA_mostly(p, ptr, swpd);
-               il_b(p, r, label_vmalloc_done);
-               i_daddiu(p, ptr, ptr, rel_lo(swpd));
+       uasm_l_vmalloc(l, *p);
+
+       if (mode != not_refill && check_for_high_segbits) {
+               if (single_insn_swpd) {
+                       uasm_il_bltz(p, r, bvaddr, label_vmalloc_done);
+                       uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
+                       did_vmalloc_branch = 1;
+                       /* fall through */
+               } else {
+                       uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault);
+               }
+       }
+       if (!did_vmalloc_branch) {
+               if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) {
+                       uasm_il_b(p, r, label_vmalloc_done);
+                       uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
+               } else {
+                       UASM_i_LA_mostly(p, ptr, swpd);
+                       uasm_il_b(p, r, label_vmalloc_done);
+                       if (uasm_in_compat_space_p(swpd))
+                               uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd));
+                       else
+                               uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd));
+               }
+       }
+       if (mode != not_refill && check_for_high_segbits) {
+               uasm_l_large_segbits_fault(l, *p);
+               /*
+                * We get here if we are an xsseg address, or if we are
+                * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary.
+                *
+                * Ignoring xsseg (assume disabled so would generate
+                * (address errors?), the only remaining possibility
+                * is the upper xuseg addresses.  On processors with
+                * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these
+                * addresses would have taken an address error. We try
+                * to mimic that here by taking a load/istream page
+                * fault.
+                */
+               UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0);
+               uasm_i_jr(p, ptr);
+
+               if (mode == refill_scratch) {
+                       if (scratch_reg > 0)
+                               UASM_i_MFC0(p, 1, 31, scratch_reg);
+                       else
+                               UASM_i_LW(p, 1, scratchpad_offset(0), 0);
+               } else {
+                       uasm_i_nop(p);
+               }
        }
 }
 
@@ -1016,36 +890,47 @@ build_get_pgd_vmalloc64(u32 **p, struct label **l, struct reloc **r,
  * TMP and PTR are scratch.
  * TMP will be clobbered, PTR will hold the pgd entry.
  */
-static __init void __attribute__((unused))
+static void __cpuinit __maybe_unused
 build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
 {
        long pgdc = (long)pgd_current;
 
        /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
 #ifdef CONFIG_SMP
-       i_mfc0(p, ptr, C0_CONTEXT);
-       i_LA_mostly(p, tmp, pgdc);
-       i_srl(p, ptr, ptr, 23);
-       i_sll(p, ptr, ptr, 2);
-       i_addu(p, ptr, tmp, ptr);
+#ifdef  CONFIG_MIPS_MT_SMTC
+       /*
+        * SMTC uses TCBind value as "CPU" index
+        */
+       uasm_i_mfc0(p, ptr, C0_TCBIND);
+       UASM_i_LA_mostly(p, tmp, pgdc);
+       uasm_i_srl(p, ptr, ptr, 19);
+#else
+       /*
+        * smp_processor_id() << 3 is stored in CONTEXT.
+         */
+       uasm_i_mfc0(p, ptr, C0_CONTEXT);
+       UASM_i_LA_mostly(p, tmp, pgdc);
+       uasm_i_srl(p, ptr, ptr, 23);
+#endif
+       uasm_i_addu(p, ptr, tmp, ptr);
 #else
-       i_LA_mostly(p, ptr, pgdc);
+       UASM_i_LA_mostly(p, ptr, pgdc);
 #endif
-       i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
-       i_lw(p, ptr, rel_lo(pgdc), ptr);
-       i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
-       i_sll(p, tmp, tmp, PGD_T_LOG2);
-       i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
+       uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
+       uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
+       uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
+       uasm_i_sll(p, tmp, tmp, PGD_T_LOG2);
+       uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
 }
 
 #endif /* !CONFIG_64BIT */
 
-static __init void build_adjust_context(u32 **p, unsigned int ctx)
+static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx)
 {
-       unsigned int shift = 4 - (PTE_T_LOG2 + 1);
+       unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12;
        unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);
 
-       switch (current_cpu_data.cputype) {
+       switch (current_cpu_type()) {
        case CPU_VR41XX:
        case CPU_VR4111:
        case CPU_VR4121:
@@ -1062,11 +947,11 @@ static __init void build_adjust_context(u32 **p, unsigned int ctx)
        }
 
        if (shift)
-               i_SRL(p, ctx, ctx, shift);
-       i_andi(p, ctx, ctx, mask);
+               UASM_i_SRL(p, ctx, ctx, shift);
+       uasm_i_andi(p, ctx, ctx, mask);
 }
 
-static __init void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
+static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
 {
        /*
         * Bug workaround for the Nevada. It seems as if under certain
@@ -1075,23 +960,23 @@ static __init void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
         * in a different cacheline or a load instruction, probably any
         * memory reference, is between them.
         */
-       switch (current_cpu_data.cputype) {
+       switch (current_cpu_type()) {
        case CPU_NEVADA:
-               i_LW(p, ptr, 0, ptr);
+               UASM_i_LW(p, ptr, 0, ptr);
                GET_CONTEXT(p, tmp); /* get context reg */
                break;
 
        default:
                GET_CONTEXT(p, tmp); /* get context reg */
-               i_LW(p, ptr, 0, ptr);
+               UASM_i_LW(p, ptr, 0, ptr);
                break;
        }
 
        build_adjust_context(p, tmp);
-       i_ADDU(p, ptr, ptr, tmp); /* add in offset */
+       UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */
 }
 
-static __init void build_update_entries(u32 **p, unsigned int tmp,
+static void __cpuinit build_update_entries(u32 **p, unsigned int tmp,
                                        unsigned int ptep)
 {
        /*
@@ -1100,248 +985,547 @@ static __init void build_update_entries(u32 **p, unsigned int tmp,
         */
 #ifdef CONFIG_64BIT_PHYS_ADDR
        if (cpu_has_64bits) {
-               i_ld(p, tmp, 0, ptep); /* get even pte */
-               i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
-               i_dsrl(p, tmp, tmp, 6); /* convert to entrylo0 */
-               i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
-               i_dsrl(p, ptep, ptep, 6); /* convert to entrylo1 */
-               i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
+               uasm_i_ld(p, tmp, 0, ptep); /* get even pte */
+               uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
+               if (kernel_uses_smartmips_rixi) {
+                       UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_NO_EXEC));
+                       UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_NO_EXEC));
+                       UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+                       UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
+                       UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+               } else {
+                       uasm_i_dsrl_safe(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */
+                       UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
+                       uasm_i_dsrl_safe(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */
+               }
+               UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */
        } else {
                int pte_off_even = sizeof(pte_t) / 2;
                int pte_off_odd = pte_off_even + sizeof(pte_t);
 
                /* The pte entries are pre-shifted */
-               i_lw(p, tmp, pte_off_even, ptep); /* get even pte */
-               i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
-               i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */
-               i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
+               uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */
+               UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
+               uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */
+               UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */
        }
 #else
-       i_LW(p, tmp, 0, ptep); /* get even pte */
-       i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
+       UASM_i_LW(p, tmp, 0, ptep); /* get even pte */
+       UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
        if (r45k_bvahwbug())
                build_tlb_probe_entry(p);
-       i_SRL(p, tmp, tmp, 6); /* convert to entrylo0 */
-       if (r4k_250MHZhwbug())
-               i_mtc0(p, 0, C0_ENTRYLO0);
-       i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
-       i_SRL(p, ptep, ptep, 6); /* convert to entrylo1 */
-       if (r45k_bvahwbug())
-               i_mfc0(p, tmp, C0_INDEX);
+       if (kernel_uses_smartmips_rixi) {
+               UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_NO_EXEC));
+               UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_NO_EXEC));
+               UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+               if (r4k_250MHZhwbug())
+                       UASM_i_MTC0(p, 0, C0_ENTRYLO0);
+               UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
+               UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+       } else {
+               UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */
+               if (r4k_250MHZhwbug())
+                       UASM_i_MTC0(p, 0, C0_ENTRYLO0);
+               UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
+               UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */
+               if (r45k_bvahwbug())
+                       uasm_i_mfc0(p, tmp, C0_INDEX);
+       }
        if (r4k_250MHZhwbug())
-               i_mtc0(p, 0, C0_ENTRYLO1);
-       i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
+               UASM_i_MTC0(p, 0, C0_ENTRYLO1);
+       UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */
+#endif
+}
+
+struct mips_huge_tlb_info {
+       int huge_pte;
+       int restore_scratch;
+};
+
+static struct mips_huge_tlb_info __cpuinit
+build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
+                              struct uasm_reloc **r, unsigned int tmp,
+                              unsigned int ptr, int c0_scratch)
+{
+       struct mips_huge_tlb_info rv;
+       unsigned int even, odd;
+       int vmalloc_branch_delay_filled = 0;
+       const int scratch = 1; /* Our extra working register */
+
+       rv.huge_pte = scratch;
+       rv.restore_scratch = 0;
+
+       if (check_for_high_segbits) {
+               UASM_i_MFC0(p, tmp, C0_BADVADDR);
+
+               if (pgd_reg != -1)
+                       UASM_i_MFC0(p, ptr, 31, pgd_reg);
+               else
+                       UASM_i_MFC0(p, ptr, C0_CONTEXT);
+
+               if (c0_scratch >= 0)
+                       UASM_i_MTC0(p, scratch, 31, c0_scratch);
+               else
+                       UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
+
+               uasm_i_dsrl_safe(p, scratch, tmp,
+                                PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
+               uasm_il_bnez(p, r, scratch, label_vmalloc);
+
+               if (pgd_reg == -1) {
+                       vmalloc_branch_delay_filled = 1;
+                       /* Clear lower 23 bits of context. */
+                       uasm_i_dins(p, ptr, 0, 0, 23);
+               }
+       } else {
+               if (pgd_reg != -1)
+                       UASM_i_MFC0(p, ptr, 31, pgd_reg);
+               else
+                       UASM_i_MFC0(p, ptr, C0_CONTEXT);
+
+               UASM_i_MFC0(p, tmp, C0_BADVADDR);
+
+               if (c0_scratch >= 0)
+                       UASM_i_MTC0(p, scratch, 31, c0_scratch);
+               else
+                       UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
+
+               if (pgd_reg == -1)
+                       /* Clear lower 23 bits of context. */
+                       uasm_i_dins(p, ptr, 0, 0, 23);
+
+               uasm_il_bltz(p, r, tmp, label_vmalloc);
+       }
+
+       if (pgd_reg == -1) {
+               vmalloc_branch_delay_filled = 1;
+               /* 1 0  1 0 1  << 6  xkphys cached */
+               uasm_i_ori(p, ptr, ptr, 0x540);
+               uasm_i_drotr(p, ptr, ptr, 11);
+       }
+
+#ifdef __PAGETABLE_PMD_FOLDED
+#define LOC_PTEP scratch
+#else
+#define LOC_PTEP ptr
+#endif
+
+       if (!vmalloc_branch_delay_filled)
+               /* get pgd offset in bytes */
+               uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3);
+
+       uasm_l_vmalloc_done(l, *p);
+
+       /*
+        *                         tmp          ptr
+        * fall-through case =   badvaddr  *pgd_current
+        * vmalloc case      =   badvaddr  swapper_pg_dir
+        */
+
+       if (vmalloc_branch_delay_filled)
+               /* get pgd offset in bytes */
+               uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3);
+
+#ifdef __PAGETABLE_PMD_FOLDED
+       GET_CONTEXT(p, tmp); /* get context reg */
+#endif
+       uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3);
+
+       if (use_lwx_insns()) {
+               UASM_i_LWX(p, LOC_PTEP, scratch, ptr);
+       } else {
+               uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */
+               uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */
+       }
+
+#ifndef __PAGETABLE_PMD_FOLDED
+       /* get pmd offset in bytes */
+       uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3);
+       uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3);
+       GET_CONTEXT(p, tmp); /* get context reg */
+
+       if (use_lwx_insns()) {
+               UASM_i_LWX(p, scratch, scratch, ptr);
+       } else {
+               uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */
+               UASM_i_LW(p, scratch, 0, ptr);
+       }
 #endif
+       /* Adjust the context during the load latency. */
+       build_adjust_context(p, tmp);
+
+#ifdef CONFIG_HUGETLB_PAGE
+       uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update);
+       /*
+        * The in the LWX case we don't want to do the load in the
+        * delay slot.  It cannot issue in the same cycle and may be
+        * speculative and unneeded.
+        */
+       if (use_lwx_insns())
+               uasm_i_nop(p);
+#endif /* CONFIG_HUGETLB_PAGE */
+
+
+       /* build_update_entries */
+       if (use_lwx_insns()) {
+               even = ptr;
+               odd = tmp;
+               UASM_i_LWX(p, even, scratch, tmp);
+               UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t));
+               UASM_i_LWX(p, odd, scratch, tmp);
+       } else {
+               UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */
+               even = tmp;
+               odd = ptr;
+               UASM_i_LW(p, even, 0, ptr); /* get even pte */
+               UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */
+       }
+       if (kernel_uses_smartmips_rixi) {
+               uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_NO_EXEC));
+               uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_NO_EXEC));
+               uasm_i_drotr(p, even, even,
+                            ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+               UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */
+               uasm_i_drotr(p, odd, odd,
+                            ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+       } else {
+               uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL));
+               UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */
+               uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL));
+       }
+       UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */
+
+       if (c0_scratch >= 0) {
+               UASM_i_MFC0(p, scratch, 31, c0_scratch);
+               build_tlb_write_entry(p, l, r, tlb_random);
+               uasm_l_leave(l, *p);
+               rv.restore_scratch = 1;
+       } else if (PAGE_SHIFT == 14 || PAGE_SHIFT == 13)  {
+               build_tlb_write_entry(p, l, r, tlb_random);
+               uasm_l_leave(l, *p);
+               UASM_i_LW(p, scratch, scratchpad_offset(0), 0);
+       } else {
+               UASM_i_LW(p, scratch, scratchpad_offset(0), 0);
+               build_tlb_write_entry(p, l, r, tlb_random);
+               uasm_l_leave(l, *p);
+               rv.restore_scratch = 1;
+       }
+
+       uasm_i_eret(p); /* return from trap */
+
+       return rv;
 }
 
-static void __init build_r4000_tlb_refill_handler(void)
+/*
+ * For a 64-bit kernel, we are using the 64-bit XTLB refill exception
+ * because EXL == 0.  If we wrap, we can also use the 32 instruction
+ * slots before the XTLB refill exception handler which belong to the
+ * unused TLB refill exception.
+ */
+#define MIPS64_REFILL_INSNS 32
+
+static void __cpuinit build_r4000_tlb_refill_handler(void)
 {
        u32 *p = tlb_handler;
-       struct label *l = labels;
-       struct reloc *r = relocs;
+       struct uasm_label *l = labels;
+       struct uasm_reloc *r = relocs;
        u32 *f;
        unsigned int final_len;
+       struct mips_huge_tlb_info htlb_info __maybe_unused;
+       enum vmalloc64_mode vmalloc_mode __maybe_unused;
 
        memset(tlb_handler, 0, sizeof(tlb_handler));
        memset(labels, 0, sizeof(labels));
        memset(relocs, 0, sizeof(relocs));
        memset(final_handler, 0, sizeof(final_handler));
 
-       /*
-        * create the plain linear handler
-        */
-       if (bcm1250_m3_war()) {
-               i_MFC0(&p, K0, C0_BADVADDR);
-               i_MFC0(&p, K1, C0_ENTRYHI);
-               i_xor(&p, K0, K0, K1);
-               i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
-               il_bnez(&p, &r, K0, label_leave);
-               /* No need for i_nop */
-       }
+       if ((scratch_reg > 0 || scratchpad_available()) && use_bbit_insns()) {
+               htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1,
+                                                         scratch_reg);
+               vmalloc_mode = refill_scratch;
+       } else {
+               htlb_info.huge_pte = K0;
+               htlb_info.restore_scratch = 0;
+               vmalloc_mode = refill_noscratch;
+               /*
+                * create the plain linear handler
+                */
+               if (bcm1250_m3_war()) {
+                       unsigned int segbits = 44;
+
+                       uasm_i_dmfc0(&p, K0, C0_BADVADDR);
+                       uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
+                       uasm_i_xor(&p, K0, K0, K1);
+                       uasm_i_dsrl_safe(&p, K1, K0, 62);
+                       uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
+                       uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
+                       uasm_i_or(&p, K0, K0, K1);
+                       uasm_il_bnez(&p, &r, K0, label_leave);
+                       /* No need for uasm_i_nop */
+               }
 
 #ifdef CONFIG_64BIT
-       build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
+               build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
 #else
-       build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
+               build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
+#endif
+
+#ifdef CONFIG_HUGETLB_PAGE
+               build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update);
 #endif
 
-       build_get_ptep(&p, K0, K1);
-       build_update_entries(&p, K0, K1);
-       build_tlb_write_entry(&p, &l, &r, tlb_random);
-       l_leave(&l, p);
-       i_eret(&p); /* return from trap */
+               build_get_ptep(&p, K0, K1);
+               build_update_entries(&p, K0, K1);
+               build_tlb_write_entry(&p, &l, &r, tlb_random);
+               uasm_l_leave(&l, p);
+               uasm_i_eret(&p); /* return from trap */
+       }
+#ifdef CONFIG_HUGETLB_PAGE
+       uasm_l_tlb_huge_update(&l, p);
+       build_huge_update_entries(&p, htlb_info.huge_pte, K1);
+       build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random,
+                                  htlb_info.restore_scratch);
+#endif
 
 #ifdef CONFIG_64BIT
-       build_get_pgd_vmalloc64(&p, &l, &r, K0, K1);
+       build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode);
 #endif
 
        /*
         * Overflow check: For the 64bit handler, we need at least one
         * free instruction slot for the wrap-around branch. In worst
         * case, if the intended insertion point is a delay slot, we
-        * need three, with the the second nop'ed and the third being
+        * need three, with the second nop'ed and the third being
         * unused.
         */
-#ifdef CONFIG_32BIT
+       /* Loongson2 ebase is different than r4k, we have more space */
+#if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
        if ((p - tlb_handler) > 64)
                panic("TLB refill handler space exceeded");
 #else
-       if (((p - tlb_handler) > 63)
-           || (((p - tlb_handler) > 61)
-               && insn_has_bdelay(relocs, tlb_handler + 29)))
+       if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1)
+           || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3)
+               && uasm_insn_has_bdelay(relocs,
+                                       tlb_handler + MIPS64_REFILL_INSNS - 3)))
                panic("TLB refill handler space exceeded");
 #endif
 
        /*
         * Now fold the handler in the TLB refill handler space.
         */
-#ifdef CONFIG_32BIT
+#if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
        f = final_handler;
        /* Simplest case, just copy the handler. */
-       copy_handler(relocs, labels, tlb_handler, p, f);
+       uasm_copy_handler(relocs, labels, tlb_handler, p, f);
        final_len = p - tlb_handler;
 #else /* CONFIG_64BIT */
-       f = final_handler + 32;
-       if ((p - tlb_handler) <= 32) {
+       f = final_handler + MIPS64_REFILL_INSNS;
+       if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) {
                /* Just copy the handler. */
-               copy_handler(relocs, labels, tlb_handler, p, f);
+               uasm_copy_handler(relocs, labels, tlb_handler, p, f);
                final_len = p - tlb_handler;
        } else {
-               u32 *split = tlb_handler + 30;
+#if defined(CONFIG_HUGETLB_PAGE)
+               const enum label_id ls = label_tlb_huge_update;
+#else
+               const enum label_id ls = label_vmalloc;
+#endif
+               u32 *split;
+               int ov = 0;
+               int i;
+
+               for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++)
+                       ;
+               BUG_ON(i == ARRAY_SIZE(labels));
+               split = labels[i].addr;
 
                /*
-                * Find the split point.
+                * See if we have overflown one way or the other.
                 */
-               if (insn_has_bdelay(relocs, split - 1))
-                       split--;
-
+               if (split > tlb_handler + MIPS64_REFILL_INSNS ||
+                   split < p - MIPS64_REFILL_INSNS)
+                       ov = 1;
+
+               if (ov) {
+                       /*
+                        * Split two instructions before the end.  One
+                        * for the branch and one for the instruction
+                        * in the delay slot.
+                        */
+                       split = tlb_handler + MIPS64_REFILL_INSNS - 2;
+
+                       /*
+                        * If the branch would fall in a delay slot,
+                        * we must back up an additional instruction
+                        * so that it is no longer in a delay slot.
+                        */
+                       if (uasm_insn_has_bdelay(relocs, split - 1))
+                               split--;
+               }
                /* Copy first part of the handler. */
-               copy_handler(relocs, labels, tlb_handler, split, f);
+               uasm_copy_handler(relocs, labels, tlb_handler, split, f);
                f += split - tlb_handler;
 
-               /* Insert branch. */
-               l_split(&l, final_handler);
-               il_b(&f, &r, label_split);
-               if (insn_has_bdelay(relocs, split))
-                       i_nop(&f);
-               else {
-                       copy_handler(relocs, labels, split, split + 1, f);
-                       move_labels(labels, f, f + 1, -1);
-                       f++;
-                       split++;
+               if (ov) {
+                       /* Insert branch. */
+                       uasm_l_split(&l, final_handler);
+                       uasm_il_b(&f, &r, label_split);
+                       if (uasm_insn_has_bdelay(relocs, split))
+                               uasm_i_nop(&f);
+                       else {
+                               uasm_copy_handler(relocs, labels,
+                                                 split, split + 1, f);
+                               uasm_move_labels(labels, f, f + 1, -1);
+                               f++;
+                               split++;
+                       }
                }
 
                /* Copy the rest of the handler. */
-               copy_handler(relocs, labels, split, p, final_handler);
-               final_len = (f - (final_handler + 32)) + (p - split);
+               uasm_copy_handler(relocs, labels, split, p, final_handler);
+               final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) +
+                           (p - split);
        }
 #endif /* CONFIG_64BIT */
 
-       resolve_relocs(relocs, labels);
-       printk("Synthesized TLB refill handler (%u instructions).\n",
-              final_len);
+       uasm_resolve_relocs(relocs, labels);
+       pr_debug("Wrote TLB refill handler (%u instructions).\n",
+                final_len);
 
-#ifdef DEBUG_TLB
-       {
-               int i;
-
-               for (i = 0; i < 64; i++)
-                       printk("%08x\n", final_handler[i]);
-       }
-#endif
+       memcpy((void *)ebase, final_handler, 0x100);
 
-       memcpy((void *)CAC_BASE, final_handler, 0x100);
-       flush_icache_range(CAC_BASE, CAC_BASE + 0x100);
+       dump_handler((u32 *)ebase, 64);
 }
 
 /*
- * TLB load/store/modify handlers.
- *
- * Only the fastpath gets synthesized at runtime, the slowpath for
- * do_page_fault remains normal asm.
- */
-extern void tlb_do_page_fault_0(void);
-extern void tlb_do_page_fault_1(void);
-
-#define __tlb_handler_align \
-       __attribute__((__aligned__(1 << CONFIG_MIPS_L1_CACHE_SHIFT)))
-
-/*
  * 128 instructions for the fastpath handler is generous and should
  * never be exceeded.
  */
 #define FASTPATH_SIZE 128
 
-u32 __tlb_handler_align handle_tlbl[FASTPATH_SIZE];
-u32 __tlb_handler_align handle_tlbs[FASTPATH_SIZE];
-u32 __tlb_handler_align handle_tlbm[FASTPATH_SIZE];
+u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned;
+u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned;
+u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned;
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
+u32 tlbmiss_handler_setup_pgd[16] __cacheline_aligned;
+
+static void __cpuinit build_r4000_setup_pgd(void)
+{
+       const int a0 = 4;
+       const int a1 = 5;
+       u32 *p = tlbmiss_handler_setup_pgd;
+       struct uasm_label *l = labels;
+       struct uasm_reloc *r = relocs;
+
+       memset(tlbmiss_handler_setup_pgd, 0, sizeof(tlbmiss_handler_setup_pgd));
+       memset(labels, 0, sizeof(labels));
+       memset(relocs, 0, sizeof(relocs));
+
+       pgd_reg = allocate_kscratch();
+
+       if (pgd_reg == -1) {
+               /* PGD << 11 in c0_Context */
+               /*
+                * If it is a ckseg0 address, convert to a physical
+                * address.  Shifting right by 29 and adding 4 will
+                * result in zero for these addresses.
+                *
+                */
+               UASM_i_SRA(&p, a1, a0, 29);
+               UASM_i_ADDIU(&p, a1, a1, 4);
+               uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1);
+               uasm_i_nop(&p);
+               uasm_i_dinsm(&p, a0, 0, 29, 64 - 29);
+               uasm_l_tlbl_goaround1(&l, p);
+               UASM_i_SLL(&p, a0, a0, 11);
+               uasm_i_jr(&p, 31);
+               UASM_i_MTC0(&p, a0, C0_CONTEXT);
+       } else {
+               /* PGD in c0_KScratch */
+               uasm_i_jr(&p, 31);
+               UASM_i_MTC0(&p, a0, 31, pgd_reg);
+       }
+       if (p - tlbmiss_handler_setup_pgd > ARRAY_SIZE(tlbmiss_handler_setup_pgd))
+               panic("tlbmiss_handler_setup_pgd space exceeded");
+       uasm_resolve_relocs(relocs, labels);
+       pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n",
+                (unsigned int)(p - tlbmiss_handler_setup_pgd));
+
+       dump_handler(tlbmiss_handler_setup_pgd,
+                    ARRAY_SIZE(tlbmiss_handler_setup_pgd));
+}
+#endif
 
-static void __init
-iPTE_LW(u32 **p, struct label **l, unsigned int pte, int offset,
-       unsigned int ptr)
+static void __cpuinit
+iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
 {
 #ifdef CONFIG_SMP
 # ifdef CONFIG_64BIT_PHYS_ADDR
        if (cpu_has_64bits)
-               i_lld(p, pte, offset, ptr);
+               uasm_i_lld(p, pte, 0, ptr);
        else
 # endif
-               i_LL(p, pte, offset, ptr);
+               UASM_i_LL(p, pte, 0, ptr);
 #else
 # ifdef CONFIG_64BIT_PHYS_ADDR
        if (cpu_has_64bits)
-               i_ld(p, pte, offset, ptr);
+               uasm_i_ld(p, pte, 0, ptr);
        else
 # endif
-               i_LW(p, pte, offset, ptr);
+               UASM_i_LW(p, pte, 0, ptr);
 #endif
 }
 
-static void __init
-iPTE_SW(u32 **p, struct reloc **r, unsigned int pte, int offset,
-       unsigned int ptr)
+static void __cpuinit
+iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr,
+       unsigned int mode)
 {
+#ifdef CONFIG_64BIT_PHYS_ADDR
+       unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY);
+#endif
+
+       uasm_i_ori(p, pte, pte, mode);
 #ifdef CONFIG_SMP
 # ifdef CONFIG_64BIT_PHYS_ADDR
        if (cpu_has_64bits)
-               i_scd(p, pte, offset, ptr);
+               uasm_i_scd(p, pte, 0, ptr);
        else
 # endif
-               i_SC(p, pte, offset, ptr);
+               UASM_i_SC(p, pte, 0, ptr);
 
        if (r10000_llsc_war())
-               il_beqzl(p, r, pte, label_smp_pgtable_change);
+               uasm_il_beqzl(p, r, pte, label_smp_pgtable_change);
        else
-               il_beqz(p, r, pte, label_smp_pgtable_change);
+               uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
 
 # ifdef CONFIG_64BIT_PHYS_ADDR
        if (!cpu_has_64bits) {
-               /* no i_nop needed */
-               i_ll(p, pte, sizeof(pte_t) / 2, ptr);
-               i_ori(p, pte, pte, _PAGE_VALID);
-               i_sc(p, pte, sizeof(pte_t) / 2, ptr);
-               il_beqz(p, r, pte, label_smp_pgtable_change);
-               /* no i_nop needed */
-               i_lw(p, pte, 0, ptr);
+               /* no uasm_i_nop needed */
+               uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr);
+               uasm_i_ori(p, pte, pte, hwmode);
+               uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr);
+               uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
+               /* no uasm_i_nop needed */
+               uasm_i_lw(p, pte, 0, ptr);
        } else
-               i_nop(p);
+               uasm_i_nop(p);
 # else
-       i_nop(p);
+       uasm_i_nop(p);
 # endif
 #else
 # ifdef CONFIG_64BIT_PHYS_ADDR
        if (cpu_has_64bits)
-               i_sd(p, pte, offset, ptr);
+               uasm_i_sd(p, pte, 0, ptr);
        else
 # endif
-               i_SW(p, pte, offset, ptr);
+               UASM_i_SW(p, pte, 0, ptr);
 
 # ifdef CONFIG_64BIT_PHYS_ADDR
        if (!cpu_has_64bits) {
-               i_lw(p, pte, sizeof(pte_t) / 2, ptr);
-               i_ori(p, pte, pte, _PAGE_VALID);
-               i_sw(p, pte, sizeof(pte_t) / 2, ptr);
-               i_lw(p, pte, 0, ptr);
+               uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr);
+               uasm_i_ori(p, pte, pte, hwmode);
+               uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr);
+               uasm_i_lw(p, pte, 0, ptr);
        }
 # endif
 #endif
@@ -1352,423 +1536,557 @@ iPTE_SW(u32 **p, struct reloc **r, unsigned int pte, int offset,
  * the page table where this PTE is located, PTE will be re-loaded
  * with it's original value.
  */
-static void __init
-build_pte_present(u32 **p, struct label **l, struct reloc **r,
-                 unsigned int pte, unsigned int ptr, enum label_id lid)
-{
-       i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
-       i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
-       il_bnez(p, r, pte, lid);
-       iPTE_LW(p, l, pte, 0, ptr);
+static void __cpuinit
+build_pte_present(u32 **p, struct uasm_reloc **r,
+                 int pte, int ptr, int scratch, enum label_id lid)
+{
+       int t = scratch >= 0 ? scratch : pte;
+
+       if (kernel_uses_smartmips_rixi) {
+               if (use_bbit_insns()) {
+                       uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid);
+                       uasm_i_nop(p);
+               } else {
+                       uasm_i_andi(p, t, pte, _PAGE_PRESENT);
+                       uasm_il_beqz(p, r, t, lid);
+                       if (pte == t)
+                               /* You lose the SMP race :-(*/
+                               iPTE_LW(p, pte, ptr);
+               }
+       } else {
+               uasm_i_andi(p, t, pte, _PAGE_PRESENT | _PAGE_READ);
+               uasm_i_xori(p, t, t, _PAGE_PRESENT | _PAGE_READ);
+               uasm_il_bnez(p, r, t, lid);
+               if (pte == t)
+                       /* You lose the SMP race :-(*/
+                       iPTE_LW(p, pte, ptr);
+       }
 }
 
 /* Make PTE valid, store result in PTR. */
-static void __init
-build_make_valid(u32 **p, struct reloc **r, unsigned int pte,
+static void __cpuinit
+build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte,
                 unsigned int ptr)
 {
-       i_ori(p, pte, pte, _PAGE_VALID | _PAGE_ACCESSED);
-       iPTE_SW(p, r, pte, 0, ptr);
+       unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED;
+
+       iPTE_SW(p, r, pte, ptr, mode);
 }
 
 /*
  * Check if PTE can be written to, if not branch to LABEL. Regardless
  * restore PTE with value from PTR when done.
  */
-static void __init
-build_pte_writable(u32 **p, struct label **l, struct reloc **r,
-                  unsigned int pte, unsigned int ptr, enum label_id lid)
-{
-       i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
-       i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
-       il_bnez(p, r, pte, lid);
-       iPTE_LW(p, l, pte, 0, ptr);
+static void __cpuinit
+build_pte_writable(u32 **p, struct uasm_reloc **r,
+                  unsigned int pte, unsigned int ptr, int scratch,
+                  enum label_id lid)
+{
+       int t = scratch >= 0 ? scratch : pte;
+
+       uasm_i_andi(p, t, pte, _PAGE_PRESENT | _PAGE_WRITE);
+       uasm_i_xori(p, t, t, _PAGE_PRESENT | _PAGE_WRITE);
+       uasm_il_bnez(p, r, t, lid);
+       if (pte == t)
+               /* You lose the SMP race :-(*/
+               iPTE_LW(p, pte, ptr);
+       else
+               uasm_i_nop(p);
 }
 
 /* Make PTE writable, update software status bits as well, then store
  * at PTR.
  */
-static void __init
-build_make_write(u32 **p, struct reloc **r, unsigned int pte,
+static void __cpuinit
+build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte,
                 unsigned int ptr)
 {
-       i_ori(p, pte, pte,
-             _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
-       iPTE_SW(p, r, pte, 0, ptr);
+       unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID
+                            | _PAGE_DIRTY);
+
+       iPTE_SW(p, r, pte, ptr, mode);
 }
 
 /*
  * Check if PTE can be modified, if not branch to LABEL. Regardless
  * restore PTE with value from PTR when done.
  */
-static void __init
-build_pte_modifiable(u32 **p, struct label **l, struct reloc **r,
-                    unsigned int pte, unsigned int ptr, enum label_id lid)
-{
-       i_andi(p, pte, pte, _PAGE_WRITE);
-       il_beqz(p, r, pte, lid);
-       iPTE_LW(p, l, pte, 0, ptr);
+static void __cpuinit
+build_pte_modifiable(u32 **p, struct uasm_reloc **r,
+                    unsigned int pte, unsigned int ptr, int scratch,
+                    enum label_id lid)
+{
+       if (use_bbit_insns()) {
+               uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid);
+               uasm_i_nop(p);
+       } else {
+               int t = scratch >= 0 ? scratch : pte;
+               uasm_i_andi(p, t, pte, _PAGE_WRITE);
+               uasm_il_beqz(p, r, t, lid);
+               if (pte == t)
+                       /* You lose the SMP race :-(*/
+                       iPTE_LW(p, pte, ptr);
+       }
 }
 
+#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
+
+
 /*
  * R3000 style TLB load/store/modify handlers.
  */
 
-/* This places the pte in the page table at PTR into ENTRYLO0. */
-static void __init
-build_r3000_pte_reload(u32 **p, unsigned int ptr)
-{
-       i_lw(p, ptr, 0, ptr);
-       i_nop(p); /* load delay */
-       i_mtc0(p, ptr, C0_ENTRYLO0);
-       i_nop(p); /* cp0 delay */
-}
-
 /*
- * The index register may have the probe fail bit set,
- * because we would trap on access kseg2, i.e. without refill.
+ * This places the pte into ENTRYLO0 and writes it with tlbwi.
+ * Then it returns.
  */
-static void __init
-build_r3000_tlb_write(u32 **p, struct label **l, struct reloc **r,
-                     unsigned int tmp)
+static void __cpuinit
+build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
 {
-       i_mfc0(p, tmp, C0_INDEX);
-       i_nop(p); /* cp0 delay */
-       il_bltz(p, r, tmp, label_r3000_write_probe_fail);
-       i_nop(p); /* branch delay */
-       i_tlbwi(p);
-       il_b(p, r, label_r3000_write_probe_ok);
-       i_nop(p); /* branch delay */
-       l_r3000_write_probe_fail(l, *p);
-       i_tlbwr(p);
-       l_r3000_write_probe_ok(l, *p);
+       uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
+       uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */
+       uasm_i_tlbwi(p);
+       uasm_i_jr(p, tmp);
+       uasm_i_rfe(p); /* branch delay */
 }
 
-static void __init
+/*
+ * This places the pte into ENTRYLO0 and writes it with tlbwi
+ * or tlbwr as appropriate.  This is because the index register
+ * may have the probe fail bit set as a result of a trap on a
+ * kseg2 access, i.e. without refill.  Then it returns.
+ */
+static void __cpuinit
+build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l,
+                            struct uasm_reloc **r, unsigned int pte,
+                            unsigned int tmp)
+{
+       uasm_i_mfc0(p, tmp, C0_INDEX);
+       uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
+       uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */
+       uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */
+       uasm_i_tlbwi(p); /* cp0 delay */
+       uasm_i_jr(p, tmp);
+       uasm_i_rfe(p); /* branch delay */
+       uasm_l_r3000_write_probe_fail(l, *p);
+       uasm_i_tlbwr(p); /* cp0 delay */
+       uasm_i_jr(p, tmp);
+       uasm_i_rfe(p); /* branch delay */
+}
+
+static void __cpuinit
 build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
                                   unsigned int ptr)
 {
        long pgdc = (long)pgd_current;
 
-       i_mfc0(p, pte, C0_BADVADDR);
-       i_lui(p, ptr, rel_hi(pgdc)); /* cp0 delay */
-       i_lw(p, ptr, rel_lo(pgdc), ptr);
-       i_srl(p, pte, pte, 22); /* load delay */
-       i_sll(p, pte, pte, 2);
-       i_addu(p, ptr, ptr, pte);
-       i_mfc0(p, pte, C0_CONTEXT);
-       i_lw(p, ptr, 0, ptr); /* cp0 delay */
-       i_andi(p, pte, pte, 0xffc); /* load delay */
-       i_addu(p, ptr, ptr, pte);
-       i_lw(p, pte, 0, ptr);
-       i_nop(p); /* load delay */
-       i_tlbp(p);
-}
-
-static void __init
-build_r3000_tlbchange_handler_tail(u32 **p, unsigned int tmp)
-{
-       i_mfc0(p, tmp, C0_EPC);
-       i_nop(p); /* cp0 delay */
-       i_jr(p, tmp);
-       i_rfe(p); /* branch delay */
+       uasm_i_mfc0(p, pte, C0_BADVADDR);
+       uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */
+       uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
+       uasm_i_srl(p, pte, pte, 22); /* load delay */
+       uasm_i_sll(p, pte, pte, 2);
+       uasm_i_addu(p, ptr, ptr, pte);
+       uasm_i_mfc0(p, pte, C0_CONTEXT);
+       uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */
+       uasm_i_andi(p, pte, pte, 0xffc); /* load delay */
+       uasm_i_addu(p, ptr, ptr, pte);
+       uasm_i_lw(p, pte, 0, ptr);
+       uasm_i_tlbp(p); /* load delay */
 }
 
-static void __init build_r3000_tlb_load_handler(void)
+static void __cpuinit build_r3000_tlb_load_handler(void)
 {
        u32 *p = handle_tlbl;
-       struct label *l = labels;
-       struct reloc *r = relocs;
+       struct uasm_label *l = labels;
+       struct uasm_reloc *r = relocs;
 
        memset(handle_tlbl, 0, sizeof(handle_tlbl));
        memset(labels, 0, sizeof(labels));
        memset(relocs, 0, sizeof(relocs));
 
        build_r3000_tlbchange_handler_head(&p, K0, K1);
-       build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl);
+       build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl);
+       uasm_i_nop(&p); /* load delay */
        build_make_valid(&p, &r, K0, K1);
-       build_r3000_pte_reload(&p, K1);
-       build_r3000_tlb_write(&p, &l, &r, K0);
-       build_r3000_tlbchange_handler_tail(&p, K0);
+       build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
 
-       l_nopage_tlbl(&l, p);
-       i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
-       i_nop(&p);
+       uasm_l_nopage_tlbl(&l, p);
+       uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
+       uasm_i_nop(&p);
 
        if ((p - handle_tlbl) > FASTPATH_SIZE)
                panic("TLB load handler fastpath space exceeded");
 
-       resolve_relocs(relocs, labels);
-       printk("Synthesized TLB load handler fastpath (%u instructions).\n",
-              (unsigned int)(p - handle_tlbl));
-
-#ifdef DEBUG_TLB
-       {
-               int i;
-
-               for (i = 0; i < FASTPATH_SIZE; i++)
-                       printk("%08x\n", handle_tlbl[i]);
-       }
-#endif
+       uasm_resolve_relocs(relocs, labels);
+       pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
+                (unsigned int)(p - handle_tlbl));
 
-       flush_icache_range((unsigned long)handle_tlbl,
-                          (unsigned long)handle_tlbl + FASTPATH_SIZE * sizeof(u32));
+       dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
 }
 
-static void __init build_r3000_tlb_store_handler(void)
+static void __cpuinit build_r3000_tlb_store_handler(void)
 {
        u32 *p = handle_tlbs;
-       struct label *l = labels;
-       struct reloc *r = relocs;
+       struct uasm_label *l = labels;
+       struct uasm_reloc *r = relocs;
 
        memset(handle_tlbs, 0, sizeof(handle_tlbs));
        memset(labels, 0, sizeof(labels));
        memset(relocs, 0, sizeof(relocs));
 
        build_r3000_tlbchange_handler_head(&p, K0, K1);
-       build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs);
+       build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs);
+       uasm_i_nop(&p); /* load delay */
        build_make_write(&p, &r, K0, K1);
-       build_r3000_pte_reload(&p, K1);
-       build_r3000_tlb_write(&p, &l, &r, K0);
-       build_r3000_tlbchange_handler_tail(&p, K0);
+       build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
 
-       l_nopage_tlbs(&l, p);
-       i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
-       i_nop(&p);
+       uasm_l_nopage_tlbs(&l, p);
+       uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
+       uasm_i_nop(&p);
 
        if ((p - handle_tlbs) > FASTPATH_SIZE)
                panic("TLB store handler fastpath space exceeded");
 
-       resolve_relocs(relocs, labels);
-       printk("Synthesized TLB store handler fastpath (%u instructions).\n",
-              (unsigned int)(p - handle_tlbs));
-
-#ifdef DEBUG_TLB
-       {
-               int i;
+       uasm_resolve_relocs(relocs, labels);
+       pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
+                (unsigned int)(p - handle_tlbs));
 
-               for (i = 0; i < FASTPATH_SIZE; i++)
-                       printk("%08x\n", handle_tlbs[i]);
-       }
-#endif
-
-       flush_icache_range((unsigned long)handle_tlbs,
-                          (unsigned long)handle_tlbs + FASTPATH_SIZE * sizeof(u32));
+       dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
 }
 
-static void __init build_r3000_tlb_modify_handler(void)
+static void __cpuinit build_r3000_tlb_modify_handler(void)
 {
        u32 *p = handle_tlbm;
-       struct label *l = labels;
-       struct reloc *r = relocs;
+       struct uasm_label *l = labels;
+       struct uasm_reloc *r = relocs;
+       struct work_registers wr;
 
        memset(handle_tlbm, 0, sizeof(handle_tlbm));
        memset(labels, 0, sizeof(labels));
        memset(relocs, 0, sizeof(relocs));
 
        build_r3000_tlbchange_handler_head(&p, K0, K1);
-       build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm);
+       build_pte_modifiable(&p, &r, wr.r1, wr.r2,  wr.r3, label_nopage_tlbm);
+       uasm_i_nop(&p); /* load delay */
        build_make_write(&p, &r, K0, K1);
-       build_r3000_pte_reload(&p, K1);
-       i_tlbwi(&p);
-       build_r3000_tlbchange_handler_tail(&p, K0);
+       build_r3000_pte_reload_tlbwi(&p, K0, K1);
 
-       l_nopage_tlbm(&l, p);
-       i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
-       i_nop(&p);
+       uasm_l_nopage_tlbm(&l, p);
+       uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
+       uasm_i_nop(&p);
 
        if ((p - handle_tlbm) > FASTPATH_SIZE)
                panic("TLB modify handler fastpath space exceeded");
 
-       resolve_relocs(relocs, labels);
-       printk("Synthesized TLB modify handler fastpath (%u instructions).\n",
-              (unsigned int)(p - handle_tlbm));
-
-#ifdef DEBUG_TLB
-       {
-               int i;
-
-               for (i = 0; i < FASTPATH_SIZE; i++)
-                       printk("%08x\n", handle_tlbm[i]);
-       }
-#endif
+       uasm_resolve_relocs(relocs, labels);
+       pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
+                (unsigned int)(p - handle_tlbm));
 
-       flush_icache_range((unsigned long)handle_tlbm,
-                          (unsigned long)handle_tlbm + FASTPATH_SIZE * sizeof(u32));
+       dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
 }
+#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
 
 /*
  * R4000 style TLB load/store/modify handlers.
  */
-static void __init
-build_r4000_tlbchange_handler_head(u32 **p, struct label **l,
-                                  struct reloc **r, unsigned int pte,
-                                  unsigned int ptr)
+static struct work_registers __cpuinit
+build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l,
+                                  struct uasm_reloc **r)
 {
+       struct work_registers wr = build_get_work_registers(p);
+
 #ifdef CONFIG_64BIT
-       build_get_pmde64(p, l, r, pte, ptr); /* get pmd in ptr */
+       build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */
 #else
-       build_get_pgde32(p, pte, ptr); /* get pgd in ptr */
+       build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */
 #endif
 
-       i_MFC0(p, pte, C0_BADVADDR);
-       i_LW(p, ptr, 0, ptr);
-       i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
-       i_andi(p, pte, pte, (PTRS_PER_PTE - 1) << PTE_T_LOG2);
-       i_ADDU(p, ptr, ptr, pte);
+#ifdef CONFIG_HUGETLB_PAGE
+       /*
+        * For huge tlb entries, pmd doesn't contain an address but
+        * instead contains the tlb pte. Check the PAGE_HUGE bit and
+        * see if we need to jump to huge tlb processing.
+        */
+       build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update);
+#endif
+
+       UASM_i_MFC0(p, wr.r1, C0_BADVADDR);
+       UASM_i_LW(p, wr.r2, 0, wr.r2);
+       UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
+       uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2);
+       UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1);
 
 #ifdef CONFIG_SMP
-       l_smp_pgtable_change(l, *p);
-# endif
-       iPTE_LW(p, l, pte, 0, ptr); /* get even pte */
-       build_tlb_probe_entry(p);
+       uasm_l_smp_pgtable_change(l, *p);
+#endif
+       iPTE_LW(p, wr.r1, wr.r2); /* get even pte */
+       if (!m4kc_tlbp_war())
+               build_tlb_probe_entry(p);
+       return wr;
 }
 
-static void __init
-build_r4000_tlbchange_handler_tail(u32 **p, struct label **l,
-                                  struct reloc **r, unsigned int tmp,
+static void __cpuinit
+build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
+                                  struct uasm_reloc **r, unsigned int tmp,
                                   unsigned int ptr)
 {
-       i_ori(p, ptr, ptr, sizeof(pte_t));
-       i_xori(p, ptr, ptr, sizeof(pte_t));
+       uasm_i_ori(p, ptr, ptr, sizeof(pte_t));
+       uasm_i_xori(p, ptr, ptr, sizeof(pte_t));
        build_update_entries(p, tmp, ptr);
        build_tlb_write_entry(p, l, r, tlb_indexed);
-       l_leave(l, *p);
-       i_eret(p); /* return from trap */
+       uasm_l_leave(l, *p);
+       build_restore_work_registers(p);
+       uasm_i_eret(p); /* return from trap */
 
 #ifdef CONFIG_64BIT
-       build_get_pgd_vmalloc64(p, l, r, tmp, ptr);
+       build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill);
 #endif
 }
 
-static void __init build_r4000_tlb_load_handler(void)
+static void __cpuinit build_r4000_tlb_load_handler(void)
 {
        u32 *p = handle_tlbl;
-       struct label *l = labels;
-       struct reloc *r = relocs;
+       struct uasm_label *l = labels;
+       struct uasm_reloc *r = relocs;
+       struct work_registers wr;
 
        memset(handle_tlbl, 0, sizeof(handle_tlbl));
        memset(labels, 0, sizeof(labels));
        memset(relocs, 0, sizeof(relocs));
 
        if (bcm1250_m3_war()) {
-               i_MFC0(&p, K0, C0_BADVADDR);
-               i_MFC0(&p, K1, C0_ENTRYHI);
-               i_xor(&p, K0, K0, K1);
-               i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
-               il_bnez(&p, &r, K0, label_leave);
-               /* No need for i_nop */
+               unsigned int segbits = 44;
+
+               uasm_i_dmfc0(&p, K0, C0_BADVADDR);
+               uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
+               uasm_i_xor(&p, K0, K0, K1);
+               uasm_i_dsrl_safe(&p, K1, K0, 62);
+               uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
+               uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
+               uasm_i_or(&p, K0, K0, K1);
+               uasm_il_bnez(&p, &r, K0, label_leave);
+               /* No need for uasm_i_nop */
        }
 
-       build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
-       build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl);
-       build_make_valid(&p, &r, K0, K1);
-       build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
+       wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
+       build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl);
+       if (m4kc_tlbp_war())
+               build_tlb_probe_entry(&p);
 
-       l_nopage_tlbl(&l, p);
-       i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
-       i_nop(&p);
+       if (kernel_uses_smartmips_rixi) {
+               /*
+                * If the page is not _PAGE_VALID, RI or XI could not
+                * have triggered it.  Skip the expensive test..
+                */
+               if (use_bbit_insns()) {
+                       uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID),
+                                     label_tlbl_goaround1);
+               } else {
+                       uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID);
+                       uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1);
+               }
+               uasm_i_nop(&p);
+
+               uasm_i_tlbr(&p);
+               /* Examine  entrylo 0 or 1 based on ptr. */
+               if (use_bbit_insns()) {
+                       uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);
+               } else {
+                       uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t));
+                       uasm_i_beqz(&p, wr.r3, 8);
+               }
+               /* load it in the delay slot*/
+               UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0);
+               /* load it if ptr is odd */
+               UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1);
+               /*
+                * If the entryLo (now in wr.r3) is valid (bit 1), RI or
+                * XI must have triggered it.
+                */
+               if (use_bbit_insns()) {
+                       uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl);
+                       uasm_i_nop(&p);
+                       uasm_l_tlbl_goaround1(&l, p);
+               } else {
+                       uasm_i_andi(&p, wr.r3, wr.r3, 2);
+                       uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl);
+                       uasm_i_nop(&p);
+               }
+               uasm_l_tlbl_goaround1(&l, p);
+       }
+       build_make_valid(&p, &r, wr.r1, wr.r2);
+       build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
 
-       if ((p - handle_tlbl) > FASTPATH_SIZE)
-               panic("TLB load handler fastpath space exceeded");
+#ifdef CONFIG_HUGETLB_PAGE
+       /*
+        * This is the entry point when build_r4000_tlbchange_handler_head
+        * spots a huge page.
+        */
+       uasm_l_tlb_huge_update(&l, p);
+       iPTE_LW(&p, wr.r1, wr.r2);
+       build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl);
+       build_tlb_probe_entry(&p);
 
-       resolve_relocs(relocs, labels);
-       printk("Synthesized TLB load handler fastpath (%u instructions).\n",
-              (unsigned int)(p - handle_tlbl));
+       if (kernel_uses_smartmips_rixi) {
+               /*
+                * If the page is not _PAGE_VALID, RI or XI could not
+                * have triggered it.  Skip the expensive test..
+                */
+               if (use_bbit_insns()) {
+                       uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID),
+                                     label_tlbl_goaround2);
+               } else {
+                       uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID);
+                       uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2);
+               }
+               uasm_i_nop(&p);
+
+               uasm_i_tlbr(&p);
+               /* Examine  entrylo 0 or 1 based on ptr. */
+               if (use_bbit_insns()) {
+                       uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);
+               } else {
+                       uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t));
+                       uasm_i_beqz(&p, wr.r3, 8);
+               }
+               /* load it in the delay slot*/
+               UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0);
+               /* load it if ptr is odd */
+               UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1);
+               /*
+                * If the entryLo (now in wr.r3) is valid (bit 1), RI or
+                * XI must have triggered it.
+                */
+               if (use_bbit_insns()) {
+                       uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2);
+               } else {
+                       uasm_i_andi(&p, wr.r3, wr.r3, 2);
+                       uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2);
+               }
 
-#ifdef DEBUG_TLB
-       {
-               int i;
+               /*
+                * We clobbered C0_PAGEMASK, restore it.  On the other branch
+                * it is restored in build_huge_tlb_write_entry.
+                */
+               build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0);
 
-               for (i = 0; i < FASTPATH_SIZE; i++)
-                       printk("%08x\n", handle_tlbl[i]);
+               uasm_l_tlbl_goaround2(&l, p);
        }
+       uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID));
+       build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2);
 #endif
 
-       flush_icache_range((unsigned long)handle_tlbl,
-                          (unsigned long)handle_tlbl + FASTPATH_SIZE * sizeof(u32));
+       uasm_l_nopage_tlbl(&l, p);
+       build_restore_work_registers(&p);
+       uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
+       uasm_i_nop(&p);
+
+       if ((p - handle_tlbl) > FASTPATH_SIZE)
+               panic("TLB load handler fastpath space exceeded");
+
+       uasm_resolve_relocs(relocs, labels);
+       pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
+                (unsigned int)(p - handle_tlbl));
+
+       dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
 }
 
-static void __init build_r4000_tlb_store_handler(void)
+static void __cpuinit build_r4000_tlb_store_handler(void)
 {
        u32 *p = handle_tlbs;
-       struct label *l = labels;
-       struct reloc *r = relocs;
+       struct uasm_label *l = labels;
+       struct uasm_reloc *r = relocs;
+       struct work_registers wr;
 
        memset(handle_tlbs, 0, sizeof(handle_tlbs));
        memset(labels, 0, sizeof(labels));
        memset(relocs, 0, sizeof(relocs));
 
-       build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
-       build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs);
-       build_make_write(&p, &r, K0, K1);
-       build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
+       wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
+       build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs);
+       if (m4kc_tlbp_war())
+               build_tlb_probe_entry(&p);
+       build_make_write(&p, &r, wr.r1, wr.r2);
+       build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
 
-       l_nopage_tlbs(&l, p);
-       i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
-       i_nop(&p);
+#ifdef CONFIG_HUGETLB_PAGE
+       /*
+        * This is the entry point when
+        * build_r4000_tlbchange_handler_head spots a huge page.
+        */
+       uasm_l_tlb_huge_update(&l, p);
+       iPTE_LW(&p, wr.r1, wr.r2);
+       build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs);
+       build_tlb_probe_entry(&p);
+       uasm_i_ori(&p, wr.r1, wr.r1,
+                  _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
+       build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2);
+#endif
+
+       uasm_l_nopage_tlbs(&l, p);
+       build_restore_work_registers(&p);
+       uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
+       uasm_i_nop(&p);
 
        if ((p - handle_tlbs) > FASTPATH_SIZE)
                panic("TLB store handler fastpath space exceeded");
 
-       resolve_relocs(relocs, labels);
-       printk("Synthesized TLB store handler fastpath (%u instructions).\n",
-              (unsigned int)(p - handle_tlbs));
-
-#ifdef DEBUG_TLB
-       {
-               int i;
-
-               for (i = 0; i < FASTPATH_SIZE; i++)
-                       printk("%08x\n", handle_tlbs[i]);
-       }
-#endif
+       uasm_resolve_relocs(relocs, labels);
+       pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
+                (unsigned int)(p - handle_tlbs));
 
-       flush_icache_range((unsigned long)handle_tlbs,
-                          (unsigned long)handle_tlbs + FASTPATH_SIZE * sizeof(u32));
+       dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
 }
 
-static void __init build_r4000_tlb_modify_handler(void)
+static void __cpuinit build_r4000_tlb_modify_handler(void)
 {
        u32 *p = handle_tlbm;
-       struct label *l = labels;
-       struct reloc *r = relocs;
+       struct uasm_label *l = labels;
+       struct uasm_reloc *r = relocs;
+       struct work_registers wr;
 
        memset(handle_tlbm, 0, sizeof(handle_tlbm));
        memset(labels, 0, sizeof(labels));
        memset(relocs, 0, sizeof(relocs));
 
-       build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
-       build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm);
+       wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
+       build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm);
+       if (m4kc_tlbp_war())
+               build_tlb_probe_entry(&p);
        /* Present and writable bits set, set accessed and dirty bits. */
-       build_make_write(&p, &r, K0, K1);
-       build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
+       build_make_write(&p, &r, wr.r1, wr.r2);
+       build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
+
+#ifdef CONFIG_HUGETLB_PAGE
+       /*
+        * This is the entry point when
+        * build_r4000_tlbchange_handler_head spots a huge page.
+        */
+       uasm_l_tlb_huge_update(&l, p);
+       iPTE_LW(&p, wr.r1, wr.r2);
+       build_pte_modifiable(&p, &r, wr.r1, wr.r2,  wr.r3, label_nopage_tlbm);
+       build_tlb_probe_entry(&p);
+       uasm_i_ori(&p, wr.r1, wr.r1,
+                  _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
+       build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2);
+#endif
 
-       l_nopage_tlbm(&l, p);
-       i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
-       i_nop(&p);
+       uasm_l_nopage_tlbm(&l, p);
+       build_restore_work_registers(&p);
+       uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
+       uasm_i_nop(&p);
 
        if ((p - handle_tlbm) > FASTPATH_SIZE)
                panic("TLB modify handler fastpath space exceeded");
 
-       resolve_relocs(relocs, labels);
-       printk("Synthesized TLB modify handler fastpath (%u instructions).\n",
-              (unsigned int)(p - handle_tlbm));
-
-#ifdef DEBUG_TLB
-       {
-               int i;
+       uasm_resolve_relocs(relocs, labels);
+       pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
+                (unsigned int)(p - handle_tlbm));
 
-               for (i = 0; i < FASTPATH_SIZE; i++)
-                       printk("%08x\n", handle_tlbm[i]);
-       }
-#endif
-
-       flush_icache_range((unsigned long)handle_tlbm,
-                          (unsigned long)handle_tlbm + FASTPATH_SIZE * sizeof(u32));
+       dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
 }
 
-void __init build_tlb_refill_handler(void)
+void __cpuinit build_tlb_refill_handler(void)
 {
        /*
         * The refill handler is generated per-CPU, multi-node systems
@@ -1777,7 +2095,11 @@ void __init build_tlb_refill_handler(void)
         */
        static int run_once = 0;
 
-       switch (current_cpu_data.cputype) {
+#ifdef CONFIG_64BIT
+       check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
+#endif
+
+       switch (current_cpu_type()) {
        case CPU_R2000:
        case CPU_R3000:
        case CPU_R3000A:
@@ -1785,6 +2107,7 @@ void __init build_tlb_refill_handler(void)
        case CPU_TX3912:
        case CPU_TX3922:
        case CPU_TX3927:
+#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
                build_r3000_tlb_refill_handler();
                if (!run_once) {
                        build_r3000_tlb_load_handler();
@@ -1792,6 +2115,9 @@ void __init build_tlb_refill_handler(void)
                        build_r3000_tlb_modify_handler();
                        run_once++;
                }
+#else
+               panic("No R3000 TLB refill handler");
+#endif
                break;
 
        case CPU_R6000:
@@ -1804,12 +2130,30 @@ void __init build_tlb_refill_handler(void)
                break;
 
        default:
-               build_r4000_tlb_refill_handler();
                if (!run_once) {
+                       scratch_reg = allocate_kscratch();
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
+                       build_r4000_setup_pgd();
+#endif
                        build_r4000_tlb_load_handler();
                        build_r4000_tlb_store_handler();
                        build_r4000_tlb_modify_handler();
                        run_once++;
                }
+               build_r4000_tlb_refill_handler();
        }
 }
+
+void __cpuinit flush_tlb_handlers(void)
+{
+       local_flush_icache_range((unsigned long)handle_tlbl,
+                          (unsigned long)handle_tlbl + sizeof(handle_tlbl));
+       local_flush_icache_range((unsigned long)handle_tlbs,
+                          (unsigned long)handle_tlbs + sizeof(handle_tlbs));
+       local_flush_icache_range((unsigned long)handle_tlbm,
+                          (unsigned long)handle_tlbm + sizeof(handle_tlbm));
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
+       local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd,
+                          (unsigned long)tlbmiss_handler_setup_pgd + sizeof(handle_tlbm));
+#endif
+}