// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Single-step support.
 *
 * Copyright (C) 2004 Paul Mackerras <paulus@au.ibm.com>, IBM
 */
#include <linux/kernel.h>
#include <linux/kprobes.h>
#include <linux/ptrace.h>
#include <linux/prefetch.h>
#include <asm/sstep.h>
#include <asm/processor.h>
#include <linux/uaccess.h>
#include <asm/cpu_has_feature.h>
#include <asm/cputable.h>
#include <asm/disassemble.h>

#ifdef CONFIG_PPC64
/* Bits in SRR1 that are copied from MSR */
#define MSR_MASK        0xffffffff87c0ffffUL
#else
#define MSR_MASK        0x87c0ffff
#endif

/* Bits in XER */
#define XER_SO          0x80000000U
#define XER_OV          0x40000000U
#define XER_CA          0x20000000U
#define XER_OV32        0x00080000U
#define XER_CA32        0x00040000U

#ifdef CONFIG_VSX
#define VSX_REGISTER_XTP(rd)   ((((rd) & 1) << 5) | ((rd) & 0xfe))
#endif

#ifdef CONFIG_PPC_FPU
/*
 * Functions in ldstfp.S
 */
extern void get_fpr(int rn, double *p);
extern void put_fpr(int rn, const double *p);
extern void get_vr(int rn, __vector128 *p);
extern void put_vr(int rn, __vector128 *p);
extern void load_vsrn(int vsr, const void *p);
extern void store_vsrn(int vsr, void *p);
extern void conv_sp_to_dp(const float *sp, double *dp);
extern void conv_dp_to_sp(const double *dp, float *sp);
#endif

#ifdef __powerpc64__
/*
 * Functions in quad.S
 */
extern int do_lq(unsigned long ea, unsigned long *regs);
extern int do_stq(unsigned long ea, unsigned long val0, unsigned long val1);
extern int do_lqarx(unsigned long ea, unsigned long *regs);
extern int do_stqcx(unsigned long ea, unsigned long val0, unsigned long val1,
                    unsigned int *crp);
#endif

#ifdef __LITTLE_ENDIAN__
#define IS_LE   1
#define IS_BE   0
#else
#define IS_LE   0
#define IS_BE   1
#endif

/*
 * Emulate the truncation of 64 bit values in 32-bit mode.
 */
static nokprobe_inline unsigned long truncate_if_32bit(unsigned long msr,
                                                        unsigned long val)
{
        if ((msr & MSR_64BIT) == 0)
                val &= 0xffffffffUL;
        return val;
}

/*
 * Determine whether a conditional branch instruction would branch.
 */
static nokprobe_inline int branch_taken(unsigned int instr,
                                        const struct pt_regs *regs,
                                        struct instruction_op *op)
{
        unsigned int bo = (instr >> 21) & 0x1f;
        unsigned int bi;

        if ((bo & 4) == 0) {
                /* decrement counter */
                op->type |= DECCTR;
                if (((bo >> 1) & 1) ^ (regs->ctr == 1))
                        return 0;
        }
        if ((bo & 0x10) == 0) {
                /* check bit from CR */
                bi = (instr >> 16) & 0x1f;
                if (((regs->ccr >> (31 - bi)) & 1) != ((bo >> 3) & 1))
                        return 0;
        }
        return 1;
}

static nokprobe_inline long address_ok(struct pt_regs *regs,
                                       unsigned long ea, int nb)
{
        if (!user_mode(regs))
                return 1;
        if (access_ok((void __user *)ea, nb))
                return 1;
        if (access_ok((void __user *)ea, 1))
                /* Access overlaps the end of the user region */
                regs->dar = TASK_SIZE_MAX - 1;
        else
                regs->dar = ea;
        return 0;
}

/*
 * Calculate effective address for a D-form instruction
 */
static nokprobe_inline unsigned long dform_ea(unsigned int instr,
                                              const struct pt_regs *regs)
{
        int ra;
        unsigned long ea;

        ra = (instr >> 16) & 0x1f;
        ea = (signed short) instr;              /* sign-extend */
        if (ra)
                ea += regs->gpr[ra];

        return ea;
}

#ifdef __powerpc64__
/*
 * Calculate effective address for a DS-form instruction
 */
static nokprobe_inline unsigned long dsform_ea(unsigned int instr,
                                               const struct pt_regs *regs)
{
        int ra;
        unsigned long ea;

        ra = (instr >> 16) & 0x1f;
        ea = (signed short) (instr & ~3);       /* sign-extend */
        if (ra)
                ea += regs->gpr[ra];

        return ea;
}

/*
 * Calculate effective address for a DQ-form instruction
 */
static nokprobe_inline unsigned long dqform_ea(unsigned int instr,
                                               const struct pt_regs *regs)
{
        int ra;
        unsigned long ea;

        ra = (instr >> 16) & 0x1f;
        ea = (signed short) (instr & ~0xf);     /* sign-extend */
        if (ra)
                ea += regs->gpr[ra];

        return ea;
}
#endif /* __powerpc64 */

/*
 * Calculate effective address for an X-form instruction
 */
static nokprobe_inline unsigned long xform_ea(unsigned int instr,
                                              const struct pt_regs *regs)
{
        int ra, rb;
        unsigned long ea;

        ra = (instr >> 16) & 0x1f;
        rb = (instr >> 11) & 0x1f;
        ea = regs->gpr[rb];
        if (ra)
                ea += regs->gpr[ra];

        return ea;
}

/*
 * Calculate effective address for a MLS:D-form / 8LS:D-form
 * prefixed instruction
 */
static nokprobe_inline unsigned long mlsd_8lsd_ea(unsigned int instr,
                                                  unsigned int suffix,
                                                  const struct pt_regs *regs)
{
        int ra, prefix_r;
        unsigned int  dd;
        unsigned long ea, d0, d1, d;

        prefix_r = GET_PREFIX_R(instr);
        ra = GET_PREFIX_RA(suffix);

        d0 = instr & 0x3ffff;
        d1 = suffix & 0xffff;
        d = (d0 << 16) | d1;

        /*
         * sign extend a 34 bit number
         */
        dd = (unsigned int)(d >> 2);
        ea = (signed int)dd;
        ea = (ea << 2) | (d & 0x3);

        if (!prefix_r && ra)
                ea += regs->gpr[ra];
        else if (!prefix_r && !ra)
                ; /* Leave ea as is */
        else if (prefix_r)
                ea += regs->nip;

        /*
         * (prefix_r && ra) is an invalid form. Should already be
         * checked for by caller!
         */

        return ea;
}

/*
 * Return the largest power of 2, not greater than sizeof(unsigned long),
 * such that x is a multiple of it.
 */
static nokprobe_inline unsigned long max_align(unsigned long x)
{
        x |= sizeof(unsigned long);
        return x & -x;          /* isolates rightmost bit */
}

static nokprobe_inline unsigned long byterev_2(unsigned long x)
{
        return ((x >> 8) & 0xff) | ((x & 0xff) << 8);
}

static nokprobe_inline unsigned long byterev_4(unsigned long x)
{
        return ((x >> 24) & 0xff) | ((x >> 8) & 0xff00) |
                ((x & 0xff00) << 8) | ((x & 0xff) << 24);
}

#ifdef __powerpc64__
static nokprobe_inline unsigned long byterev_8(unsigned long x)
{
        return (byterev_4(x) << 32) | byterev_4(x >> 32);
}
#endif

static nokprobe_inline void do_byte_reverse(void *ptr, int nb)
{
        switch (nb) {
        case 2:
                *(u16 *)ptr = byterev_2(*(u16 *)ptr);
                break;
        case 4:
                *(u32 *)ptr = byterev_4(*(u32 *)ptr);
                break;
#ifdef __powerpc64__
        case 8:
                *(unsigned long *)ptr = byterev_8(*(unsigned long *)ptr);
                break;
        case 16: {
                unsigned long *up = (unsigned long *)ptr;
                unsigned long tmp;
                tmp = byterev_8(up[0]);
                up[0] = byterev_8(up[1]);
                up[1] = tmp;
                break;
        }
        case 32: {
                unsigned long *up = (unsigned long *)ptr;
                unsigned long tmp;

                tmp = byterev_8(up[0]);
                up[0] = byterev_8(up[3]);
                up[3] = tmp;
                tmp = byterev_8(up[2]);
                up[2] = byterev_8(up[1]);
                up[1] = tmp;
                break;
        }

#endif
        default:
                WARN_ON_ONCE(1);
        }
}

static __always_inline int
__read_mem_aligned(unsigned long *dest, unsigned long ea, int nb, struct pt_regs *regs)
{
        unsigned long x = 0;

        switch (nb) {
        case 1:
                unsafe_get_user(x, (unsigned char __user *)ea, Efault);
                break;
        case 2:
                unsafe_get_user(x, (unsigned short __user *)ea, Efault);
                break;
        case 4:
                unsafe_get_user(x, (unsigned int __user *)ea, Efault);
                break;
#ifdef __powerpc64__
        case 8:
                unsafe_get_user(x, (unsigned long __user *)ea, Efault);
                break;
#endif
        }
        *dest = x;
        return 0;

Efault:
        regs->dar = ea;
        return -EFAULT;
}

static nokprobe_inline int
read_mem_aligned(unsigned long *dest, unsigned long ea, int nb, struct pt_regs *regs)
{
        int err;

        if (is_kernel_addr(ea))
                return __read_mem_aligned(dest, ea, nb, regs);

        if (user_read_access_begin((void __user *)ea, nb)) {
                err = __read_mem_aligned(dest, ea, nb, regs);
                user_read_access_end();
        } else {
                err = -EFAULT;
                regs->dar = ea;
        }

        return err;
}

/*
 * Copy from userspace to a buffer, using the largest possible
 * aligned accesses, up to sizeof(long).
 */
static __always_inline int __copy_mem_in(u8 *dest, unsigned long ea, int nb, struct pt_regs *regs)
{
        int c;

        for (; nb > 0; nb -= c) {
                c = max_align(ea);
                if (c > nb)
                        c = max_align(nb);
                switch (c) {
                case 1:
                        unsafe_get_user(*dest, (u8 __user *)ea, Efault);
                        break;
                case 2:
                        unsafe_get_user(*(u16 *)dest, (u16 __user *)ea, Efault);
                        break;
                case 4:
                        unsafe_get_user(*(u32 *)dest, (u32 __user *)ea, Efault);
                        break;
#ifdef __powerpc64__
                case 8:
                        unsafe_get_user(*(u64 *)dest, (u64 __user *)ea, Efault);
                        break;
#endif
                }
                dest += c;
                ea += c;
        }
        return 0;

Efault:
        regs->dar = ea;
        return -EFAULT;
}

static nokprobe_inline int copy_mem_in(u8 *dest, unsigned long ea, int nb, struct pt_regs *regs)
{
        int err;

        if (is_kernel_addr(ea))
                return __copy_mem_in(dest, ea, nb, regs);

        if (user_read_access_begin((void __user *)ea, nb)) {
                err = __copy_mem_in(dest, ea, nb, regs);
                user_read_access_end();
        } else {
                err = -EFAULT;
                regs->dar = ea;
        }

        return err;
}

static nokprobe_inline int read_mem_unaligned(unsigned long *dest,
                                              unsigned long ea, int nb,
                                              struct pt_regs *regs)
{
        union {
                unsigned long ul;
                u8 b[sizeof(unsigned long)];
        } u;
        int i;
        int err;

        u.ul = 0;
        i = IS_BE ? sizeof(unsigned long) - nb : 0;
        err = copy_mem_in(&u.b[i], ea, nb, regs);
        if (!err)
                *dest = u.ul;
        return err;
}

/*
 * Read memory at address ea for nb bytes, return 0 for success
 * or -EFAULT if an error occurred.  N.B. nb must be 1, 2, 4 or 8.
 * If nb < sizeof(long), the result is right-justified on BE systems.
 */
static int read_mem(unsigned long *dest, unsigned long ea, int nb,
                              struct pt_regs *regs)
{
        if (!address_ok(regs, ea, nb))
                return -EFAULT;
        if ((ea & (nb - 1)) == 0)
                return read_mem_aligned(dest, ea, nb, regs);
        return read_mem_unaligned(dest, ea, nb, regs);
}
NOKPROBE_SYMBOL(read_mem);

static __always_inline int
__write_mem_aligned(unsigned long val, unsigned long ea, int nb, struct pt_regs *regs)
{
        switch (nb) {
        case 1:
                unsafe_put_user(val, (unsigned char __user *)ea, Efault);
                break;
        case 2:
                unsafe_put_user(val, (unsigned short __user *)ea, Efault);
                break;
        case 4:
                unsafe_put_user(val, (unsigned int __user *)ea, Efault);
                break;
#ifdef __powerpc64__
        case 8:
                unsafe_put_user(val, (unsigned long __user *)ea, Efault);
                break;
#endif
        }
        return 0;

Efault:
        regs->dar = ea;
        return -EFAULT;
}

static nokprobe_inline int
write_mem_aligned(unsigned long val, unsigned long ea, int nb, struct pt_regs *regs)
{
        int err;

        if (is_kernel_addr(ea))
                return __write_mem_aligned(val, ea, nb, regs);

        if (user_write_access_begin((void __user *)ea, nb)) {
                err = __write_mem_aligned(val, ea, nb, regs);
                user_write_access_end();
        } else {
                err = -EFAULT;
                regs->dar = ea;
        }

        return err;
}

/*
 * Copy from a buffer to userspace, using the largest possible
 * aligned accesses, up to sizeof(long).
 */
static nokprobe_inline int __copy_mem_out(u8 *dest, unsigned long ea, int nb, struct pt_regs *regs)
{
        int c;

        for (; nb > 0; nb -= c) {
                c = max_align(ea);
                if (c > nb)
                        c = max_align(nb);
                switch (c) {
                case 1:
                        unsafe_put_user(*dest, (u8 __user *)ea, Efault);
                        break;
                case 2:
                        unsafe_put_user(*(u16 *)dest, (u16 __user *)ea, Efault);
                        break;
                case 4:
                        unsafe_put_user(*(u32 *)dest, (u32 __user *)ea, Efault);
                        break;
#ifdef __powerpc64__
                case 8:
                        unsafe_put_user(*(u64 *)dest, (u64 __user *)ea, Efault);
                        break;
#endif
                }
                dest += c;
                ea += c;
        }
        return 0;

Efault:
        regs->dar = ea;
        return -EFAULT;
}

static nokprobe_inline int copy_mem_out(u8 *dest, unsigned long ea, int nb, struct pt_regs *regs)
{
        int err;

        if (is_kernel_addr(ea))
                return __copy_mem_out(dest, ea, nb, regs);

        if (user_write_access_begin((void __user *)ea, nb)) {
                err = __copy_mem_out(dest, ea, nb, regs);
                user_write_access_end();
        } else {
                err = -EFAULT;
                regs->dar = ea;
        }

        return err;
}

static nokprobe_inline int write_mem_unaligned(unsigned long val,
                                               unsigned long ea, int nb,
                                               struct pt_regs *regs)
{
        union {
                unsigned long ul;
                u8 b[sizeof(unsigned long)];
        } u;
        int i;

        u.ul = val;
        i = IS_BE ? sizeof(unsigned long) - nb : 0;
        return copy_mem_out(&u.b[i], ea, nb, regs);
}

/*
 * Write memory at address ea for nb bytes, return 0 for success
 * or -EFAULT if an error occurred.  N.B. nb must be 1, 2, 4 or 8.
 */
static int write_mem(unsigned long val, unsigned long ea, int nb,
                               struct pt_regs *regs)
{
        if (!address_ok(regs, ea, nb))
                return -EFAULT;
        if ((ea & (nb - 1)) == 0)
                return write_mem_aligned(val, ea, nb, regs);
        return write_mem_unaligned(val, ea, nb, regs);
}
NOKPROBE_SYMBOL(write_mem);

#ifdef CONFIG_PPC_FPU
/*
 * These access either the real FP register or the image in the
 * thread_struct, depending on regs->msr & MSR_FP.
 */
static int do_fp_load(struct instruction_op *op, unsigned long ea,
                      struct pt_regs *regs, bool cross_endian)
{
        int err, rn, nb;
        union {
                int i;
                unsigned int u;
                float f;
                double d[2];
                unsigned long l[2];
                u8 b[2 * sizeof(double)];
        } u;

        nb = GETSIZE(op->type);
        if (!address_ok(regs, ea, nb))
                return -EFAULT;
        rn = op->reg;
        err = copy_mem_in(u.b, ea, nb, regs);
        if (err)
                return err;
        if (unlikely(cross_endian)) {
                do_byte_reverse(u.b, min(nb, 8));
                if (nb == 16)
                        do_byte_reverse(&u.b[8], 8);
        }
        preempt_disable();
        if (nb == 4) {
                if (op->type & FPCONV)
                        conv_sp_to_dp(&u.f, &u.d[0]);
                else if (op->type & SIGNEXT)
                        u.l[0] = u.i;
                else
                        u.l[0] = u.u;
        }
        if (regs->msr & MSR_FP)
                put_fpr(rn, &u.d[0]);
        else
                current->thread.TS_FPR(rn) = u.l[0];
        if (nb == 16) {
                /* lfdp */
                rn |= 1;
                if (regs->msr & MSR_FP)
                        put_fpr(rn, &u.d[1]);
                else
                        current->thread.TS_FPR(rn) = u.l[1];
        }
        preempt_enable();
        return 0;
}
NOKPROBE_SYMBOL(do_fp_load);

static int do_fp_store(struct instruction_op *op, unsigned long ea,
                       struct pt_regs *regs, bool cross_endian)
{
        int rn, nb;
        union {
                unsigned int u;
                float f;
                double d[2];
                unsigned long l[2];
                u8 b[2 * sizeof(double)];
        } u;

        nb = GETSIZE(op->type);
        if (!address_ok(regs, ea, nb))
                return -EFAULT;
        rn = op->reg;
        preempt_disable();
        if (regs->msr & MSR_FP)
                get_fpr(rn, &u.d[0]);
        else
                u.l[0] = current->thread.TS_FPR(rn);
        if (nb == 4) {
                if (op->type & FPCONV)
                        conv_dp_to_sp(&u.d[0], &u.f);
                else
                        u.u = u.l[0];
        }
        if (nb == 16) {
                rn |= 1;
                if (regs->msr & MSR_FP)
                        get_fpr(rn, &u.d[1]);
                else
                        u.l[1] = current->thread.TS_FPR(rn);
        }
        preempt_enable();
        if (unlikely(cross_endian)) {
                do_byte_reverse(u.b, min(nb, 8));
                if (nb == 16)
                        do_byte_reverse(&u.b[8], 8);
        }
        return copy_mem_out(u.b, ea, nb, regs);
}
NOKPROBE_SYMBOL(do_fp_store);
#endif

#ifdef CONFIG_ALTIVEC
/* For Altivec/VMX, no need to worry about alignment */
static nokprobe_inline int do_vec_load(int rn, unsigned long ea,
                                       int size, struct pt_regs *regs,
                                       bool cross_endian)
{
        int err;
        union {
                __vector128 v;
                u8 b[sizeof(__vector128)];
        } u = {};

        if (!address_ok(regs, ea & ~0xfUL, 16))
                return -EFAULT;
        /* align to multiple of size */
        ea &= ~(size - 1);
        err = copy_mem_in(&u.b[ea & 0xf], ea, size, regs);
        if (err)
                return err;
        if (unlikely(cross_endian))
                do_byte_reverse(&u.b[ea & 0xf], size);
        preempt_disable();
        if (regs->msr & MSR_VEC)
                put_vr(rn, &u.v);
        else
                current->thread.vr_state.vr[rn] = u.v;
        preempt_enable();
        return 0;
}

static nokprobe_inline int do_vec_store(int rn, unsigned long ea,
                                        int size, struct pt_regs *regs,
                                        bool cross_endian)
{
        union {
                __vector128 v;
                u8 b[sizeof(__vector128)];
        } u;

        if (!address_ok(regs, ea & ~0xfUL, 16))
                return -EFAULT;
        /* align to multiple of size */
        ea &= ~(size - 1);

        preempt_disable();
        if (regs->msr & MSR_VEC)
                get_vr(rn, &u.v);
        else
                u.v = current->thread.vr_state.vr[rn];
        preempt_enable();
        if (unlikely(cross_endian))
                do_byte_reverse(&u.b[ea & 0xf], size);
        return copy_mem_out(&u.b[ea & 0xf], ea, size, regs);
}
#endif /* CONFIG_ALTIVEC */

#ifdef __powerpc64__
static nokprobe_inline int emulate_lq(struct pt_regs *regs, unsigned long ea,
                                      int reg, bool cross_endian)
{
        int err;

        if (!address_ok(regs, ea, 16))
                return -EFAULT;
        /* if aligned, should be atomic */
        if ((ea & 0xf) == 0) {
                err = do_lq(ea, &regs->gpr[reg]);
        } else {
                err = read_mem(&regs->gpr[reg + IS_LE], ea, 8, regs);
                if (!err)
                        err = read_mem(&regs->gpr[reg + IS_BE], ea + 8, 8, regs);
        }
        if (!err && unlikely(cross_endian))
                do_byte_reverse(&regs->gpr[reg], 16);
        return err;
}

static nokprobe_inline int emulate_stq(struct pt_regs *regs, unsigned long ea,
                                       int reg, bool cross_endian)
{
        int err;
        unsigned long vals[2];

        if (!address_ok(regs, ea, 16))
                return -EFAULT;
        vals[0] = regs->gpr[reg];
        vals[1] = regs->gpr[reg + 1];
        if (unlikely(cross_endian))
                do_byte_reverse(vals, 16);

        /* if aligned, should be atomic */
        if ((ea & 0xf) == 0)
                return do_stq(ea, vals[0], vals[1]);

        err = write_mem(vals[IS_LE], ea, 8, regs);
        if (!err)
                err = write_mem(vals[IS_BE], ea + 8, 8, regs);
        return err;
}
#endif /* __powerpc64 */

#ifdef CONFIG_VSX
void emulate_vsx_load(struct instruction_op *op, union vsx_reg *reg,
                      const void *mem, bool rev)
{
        int size, read_size;
        int i, j;
        const unsigned int *wp;
        const unsigned short *hp;
        const unsigned char *bp;

        size = GETSIZE(op->type);
        reg->d[0] = reg->d[1] = 0;

        switch (op->element_size) {
        case 32:
                /* [p]lxvp[x] */
        case 16:
                /* whole vector; lxv[x] or lxvl[l] */
                if (size == 0)
                        break;
                memcpy(reg, mem, size);
                if (IS_LE && (op->vsx_flags & VSX_LDLEFT))
                        rev = !rev;
                if (rev)
                        do_byte_reverse(reg, size);
                break;
        case 8:
                /* scalar loads, lxvd2x, lxvdsx */
                read_size = (size >= 8) ? 8 : size;
                i = IS_LE ? 8 : 8 - read_size;
                memcpy(&reg->b[i], mem, read_size);
                if (rev)
                        do_byte_reverse(&reg->b[i], 8);
                if (size < 8) {
                        if (op->type & SIGNEXT) {
                                /* size == 4 is the only case here */
                                reg->d[IS_LE] = (signed int) reg->d[IS_LE];
                        } else if (op->vsx_flags & VSX_FPCONV) {
                                preempt_disable();
                                conv_sp_to_dp(&reg->fp[1 + IS_LE],
                                              &reg->dp[IS_LE]);
                                preempt_enable();
                        }
                } else {
                        if (size == 16) {
                                unsigned long v = *(unsigned long *)(mem + 8);
                                reg->d[IS_BE] = !rev ? v : byterev_8(v);
                        } else if (op->vsx_flags & VSX_SPLAT)
                                reg->d[IS_BE] = reg->d[IS_LE];
                }
                break;
        case 4:
                /* lxvw4x, lxvwsx */
                wp = mem;
                for (j = 0; j < size / 4; ++j) {
                        i = IS_LE ? 3 - j : j;
                        reg->w[i] = !rev ? *wp++ : byterev_4(*wp++);
                }
                if (op->vsx_flags & VSX_SPLAT) {
                        u32 val = reg->w[IS_LE ? 3 : 0];
                        for (; j < 4; ++j) {
                                i = IS_LE ? 3 - j : j;
                                reg->w[i] = val;
                        }
                }
                break;
        case 2:
                /* lxvh8x */
                hp = mem;
                for (j = 0; j < size / 2; ++j) {
                        i = IS_LE ? 7 - j : j;
                        reg->h[i] = !rev ? *hp++ : byterev_2(*hp++);
                }
                break;
        case 1:
                /* lxvb16x */
                bp = mem;
                for (j = 0; j < size; ++j) {
                        i = IS_LE ? 15 - j : j;
                        reg->b[i] = *bp++;
                }
                break;
        }
}
EXPORT_SYMBOL_GPL(emulate_vsx_load);
NOKPROBE_SYMBOL(emulate_vsx_load);

void emulate_vsx_store(struct instruction_op *op, const union vsx_reg *reg,
                       void *mem, bool rev)
{
        int size, write_size;
        int i, j;
        union vsx_reg buf;
        unsigned int *wp;
        unsigned short *hp;
        unsigned char *bp;

        size = GETSIZE(op->type);

        switch (op->element_size) {
        case 32:
                /* [p]stxvp[x] */
                if (size == 0)
                        break;
                if (rev) {
                        /* reverse 32 bytes */
                        union vsx_reg buf32[2];
                        buf32[0].d[0] = byterev_8(reg[1].d[1]);
                        buf32[0].d[1] = byterev_8(reg[1].d[0]);
                        buf32[1].d[0] = byterev_8(reg[0].d[1]);
                        buf32[1].d[1] = byterev_8(reg[0].d[0]);
                        memcpy(mem, buf32, size);
                } else {
                        memcpy(mem, reg, size);
                }
                break;
        case 16:
                /* stxv, stxvx, stxvl, stxvll */
                if (size == 0)
                        break;
                if (IS_LE && (op->vsx_flags & VSX_LDLEFT))
                        rev = !rev;
                if (rev) {
                        /* reverse 16 bytes */
                        buf.d[0] = byterev_8(reg->d[1]);
                        buf.d[1] = byterev_8(reg->d[0]);
                        reg = &buf;
                }
                memcpy(mem, reg, size);
                break;
        case 8:
                /* scalar stores, stxvd2x */
                write_size = (size >= 8) ? 8 : size;
                i = IS_LE ? 8 : 8 - write_size;
                if (size < 8 && op->vsx_flags & VSX_FPCONV) {
                        buf.d[0] = buf.d[1] = 0;
                        preempt_disable();
                        conv_dp_to_sp(&reg->dp[IS_LE], &buf.fp[1 + IS_LE]);
                        preempt_enable();
                        reg = &buf;
                }
                memcpy(mem, &reg->b[i], write_size);
                if (size == 16)
                        memcpy(mem + 8, &reg->d[IS_BE], 8);
                if (unlikely(rev)) {
                        do_byte_reverse(mem, write_size);
                        if (size == 16)
                                do_byte_reverse(mem + 8, 8);
                }
                break;
        case 4:
                /* stxvw4x */
                wp = mem;
                for (j = 0; j < size / 4; ++j) {
                        i = IS_LE ? 3 - j : j;
                        *wp++ = !rev ? reg->w[i] : byterev_4(reg->w[i]);
                }
                break;
        case 2:
                /* stxvh8x */
                hp = mem;
                for (j = 0; j < size / 2; ++j) {
                        i = IS_LE ? 7 - j : j;
                        *hp++ = !rev ? reg->h[i] : byterev_2(reg->h[i]);
                }
                break;
        case 1:
                /* stvxb16x */
                bp = mem;
                for (j = 0; j < size; ++j) {
                        i = IS_LE ? 15 - j : j;
                        *bp++ = reg->b[i];
                }
                break;
        }
}
EXPORT_SYMBOL_GPL(emulate_vsx_store);
NOKPROBE_SYMBOL(emulate_vsx_store);

static nokprobe_inline int do_vsx_load(struct instruction_op *op,
                                       unsigned long ea, struct pt_regs *regs,
                                       bool cross_endian)
{
        int reg = op->reg;
        int i, j, nr_vsx_regs;
        u8 mem[32];
        union vsx_reg buf[2];
        int size = GETSIZE(op->type);

        if (!address_ok(regs, ea, size) || copy_mem_in(mem, ea, size, regs))
                return -EFAULT;

        nr_vsx_regs = max(1ul, size / sizeof(__vector128));
        emulate_vsx_load(op, buf, mem, cross_endian);
        preempt_disable();
        if (reg < 32) {
                /* FP regs + extensions */
                if (regs->msr & MSR_FP) {
                        for (i = 0; i < nr_vsx_regs; i++) {
                                j = IS_LE ? nr_vsx_regs - i - 1 : i;
                                load_vsrn(reg + i, &buf[j].v);
                        }
                } else {
                        for (i = 0; i < nr_vsx_regs; i++) {
                                j = IS_LE ? nr_vsx_regs - i - 1 : i;
                                current->thread.fp_state.fpr[reg + i][0] = buf[j].d[0];
                                current->thread.fp_state.fpr[reg + i][1] = buf[j].d[1];
                        }
                }
        } else {
                if (regs->msr & MSR_VEC) {
                        for (i = 0; i < nr_vsx_regs; i++) {
                                j = IS_LE ? nr_vsx_regs - i - 1 : i;
                                load_vsrn(reg + i, &buf[j].v);
                        }
                } else {
                        for (i = 0; i < nr_vsx_regs; i++) {
                                j = IS_LE ? nr_vsx_regs - i - 1 : i;
                                current->thread.vr_state.vr[reg - 32 + i] = buf[j].v;
                        }
                }
        }
        preempt_enable();
        return 0;
}

static nokprobe_inline int do_vsx_store(struct instruction_op *op,
                                        unsigned long ea, struct pt_regs *regs,
                                        bool cross_endian)

{
        int reg = op->reg;
        int i, j, nr_vsx_regs;
        u8 mem[32];
        union vsx_reg buf[2];
        int size = GETSIZE(op->type);

        if (!address_ok(regs, ea, size))
                return -EFAULT;

        nr_vsx_regs = max(1ul, size / sizeof(__vector128));
        preempt_disable();
        if (reg < 32) {
                /* FP regs + extensions */
                if (regs->msr & MSR_FP) {
                        for (i = 0; i < nr_vsx_regs; i++) {
                                j = IS_LE ? nr_vsx_regs - i - 1 : i;
                                store_vsrn(reg + i, &buf[j].v);
                        }
                } else {
                        for (i = 0; i < nr_vsx_regs; i++) {
                                j = IS_LE ? nr_vsx_regs - i - 1 : i;
                                buf[j].d[0] = current->thread.fp_state.fpr[reg + i][0];
                                buf[j].d[1] = current->thread.fp_state.fpr[reg + i][1];
                        }
                }
        } else {
                if (regs->msr & MSR_VEC) {
                        for (i = 0; i < nr_vsx_regs; i++) {
                                j = IS_LE ? nr_vsx_regs - i - 1 : i;
                                store_vsrn(reg + i, &buf[j].v);
                        }
                } else {
                        for (i = 0; i < nr_vsx_regs; i++) {
                                j = IS_LE ? nr_vsx_regs - i - 1 : i;
                                buf[j].v = current->thread.vr_state.vr[reg - 32 + i];
                        }
                }
        }
        preempt_enable();
        emulate_vsx_store(op, buf, mem, cross_endian);
        return  copy_mem_out(mem, ea, size, regs);
}
#endif /* CONFIG_VSX */

static int __emulate_dcbz(unsigned long ea)
{
        unsigned long i;
        unsigned long size = l1_dcache_bytes();

        for (i = 0; i < size; i += sizeof(long))
                unsafe_put_user(0, (unsigned long __user *)(ea + i), Efault);

        return 0;

Efault:
        return -EFAULT;
}

int emulate_dcbz(unsigned long ea, struct pt_regs *regs)
{
        int err;
        unsigned long size = l1_dcache_bytes();

        ea = truncate_if_32bit(regs->msr, ea);
        ea &= ~(size - 1);
        if (!address_ok(regs, ea, size))
                return -EFAULT;

        if (is_kernel_addr(ea)) {
                err = __emulate_dcbz(ea);
        } else if (user_write_access_begin((void __user *)ea, size)) {
                err = __emulate_dcbz(ea);
                user_write_access_end();
        } else {
                err = -EFAULT;
        }

        if (err)
                regs->dar = ea;


        return err;
}
NOKPROBE_SYMBOL(emulate_dcbz);

#define __put_user_asmx(x, addr, err, op, cr)           \
        __asm__ __volatile__(                           \
                ".machine push\n"                       \
                ".machine power8\n"                     \
                "1:     " op " %2,0,%3\n"               \
                ".machine pop\n"                        \
                "       mfcr    %1\n"                   \
                "2:\n"                                  \
                ".section .fixup,\"ax\"\n"              \
                "3:     li      %0,%4\n"                \
                "       b       2b\n"                   \
                ".previous\n"                           \
                EX_TABLE(1b, 3b)                        \
                : "=r" (err), "=r" (cr)                 \
                : "r" (x), "r" (addr), "i" (-EFAULT), "0" (err))

#define __get_user_asmx(x, addr, err, op)               \
        __asm__ __volatile__(                           \
                ".machine push\n"                       \
                ".machine power8\n"                     \
                "1:     "op" %1,0,%2\n"                 \
                ".machine pop\n"                        \
                "2:\n"                                  \
                ".section .fixup,\"ax\"\n"              \
                "3:     li      %0,%3\n"                \
                "       b       2b\n"                   \
                ".previous\n"                           \
                EX_TABLE(1b, 3b)                        \
                : "=r" (err), "=r" (x)                  \
                : "r" (addr), "i" (-EFAULT), "0" (err))

#define __cacheop_user_asmx(addr, err, op)              \
        __asm__ __volatile__(                           \
                "1:     "op" 0,%1\n"                    \
                "2:\n"                                  \
                ".section .fixup,\"ax\"\n"              \
                "3:     li      %0,%3\n"                \
                "       b       2b\n"                   \
                ".previous\n"                           \
                EX_TABLE(1b, 3b)                        \
                : "=r" (err)                            \
                : "r" (addr), "i" (-EFAULT), "0" (err))

static nokprobe_inline void set_cr0(const struct pt_regs *regs,
                                    struct instruction_op *op)
{
        long val = op->val;

        op->type |= SETCC;
        op->ccval = (regs->ccr & 0x0fffffff) | ((regs->xer >> 3) & 0x10000000);
        if (!(regs->msr & MSR_64BIT))
                val = (int) val;
        if (val < 0)
                op->ccval |= 0x80000000;
        else if (val > 0)
                op->ccval |= 0x40000000;
        else
                op->ccval |= 0x20000000;
}

static nokprobe_inline void set_ca32(struct instruction_op *op, bool val)
{
        if (cpu_has_feature(CPU_FTR_ARCH_300)) {
                if (val)
                        op->xerval |= XER_CA32;
                else
                        op->xerval &= ~XER_CA32;
        }
}

static nokprobe_inline void add_with_carry(const struct pt_regs *regs,
                                     struct instruction_op *op, int rd,
                                     unsigned long val1, unsigned long val2,
                                     unsigned long carry_in)
{
        unsigned long val = val1 + val2;

        if (carry_in)
                ++val;
        op->type = COMPUTE | SETREG | SETXER;
        op->reg = rd;
        op->val = val;
        val = truncate_if_32bit(regs->msr, val);
        val1 = truncate_if_32bit(regs->msr, val1);
        op->xerval = regs->xer;
        if (val < val1 || (carry_in && val == val1))
                op->xerval |= XER_CA;
        else
                op->xerval &= ~XER_CA;

        set_ca32(op, (unsigned int)val < (unsigned int)val1 ||
                        (carry_in && (unsigned int)val == (unsigned int)val1));
}

static nokprobe_inline void do_cmp_signed(const struct pt_regs *regs,
                                          struct instruction_op *op,
                                          long v1, long v2, int crfld)
{
        unsigned int crval, shift;

        op->type = COMPUTE | SETCC;
        crval = (regs->xer >> 31) & 1;          /* get SO bit */
        if (v1 < v2)
                crval |= 8;
        else if (v1 > v2)
                crval |= 4;
        else
                crval |= 2;
        shift = (7 - crfld) * 4;
        op->ccval = (regs->ccr & ~(0xf << shift)) | (crval << shift);
}

static nokprobe_inline void do_cmp_unsigned(const struct pt_regs *regs,
                                            struct instruction_op *op,
                                            unsigned long v1,
                                            unsigned long v2, int crfld)
{
        unsigned int crval, shift;

        op->type = COMPUTE | SETCC;
        crval = (regs->xer >> 31) & 1;          /* get SO bit */
        if (v1 < v2)
                crval |= 8;
        else if (v1 > v2)
                crval |= 4;
        else
                crval |= 2;
        shift = (7 - crfld) * 4;
        op->ccval = (regs->ccr & ~(0xf << shift)) | (crval << shift);
}

static nokprobe_inline void do_cmpb(const struct pt_regs *regs,
                                    struct instruction_op *op,
                                    unsigned long v1, unsigned long v2)
{
        unsigned long long out_val, mask;
        int i;

        out_val = 0;
        for (i = 0; i < 8; i++) {
                mask = 0xffUL << (i * 8);
                if ((v1 & mask) == (v2 & mask))
                        out_val |= mask;
        }
        op->val = out_val;
}

/*
 * The size parameter is used to adjust the equivalent popcnt instruction.
 * popcntb = 8, popcntw = 32, popcntd = 64
 */
static nokprobe_inline void do_popcnt(const struct pt_regs *regs,
                                      struct instruction_op *op,
                                      unsigned long v1, int size)
{
        unsigned long long out = v1;

        out -= (out >> 1) & 0x5555555555555555ULL;
        out = (0x3333333333333333ULL & out) +
              (0x3333333333333333ULL & (out >> 2));
        out = (out + (out >> 4)) & 0x0f0f0f0f0f0f0f0fULL;

        if (size == 8) {        /* popcntb */
                op->val = out;
                return;
        }
        out += out >> 8;
        out += out >> 16;
        if (size == 32) {       /* popcntw */
                op->val = out & 0x0000003f0000003fULL;
                return;
        }

        out = (out + (out >> 32)) & 0x7f;
        op->val = out;  /* popcntd */
}

#ifdef CONFIG_PPC64
static nokprobe_inline void do_bpermd(const struct pt_regs *regs,
                                      struct instruction_op *op,
                                      unsigned long v1, unsigned long v2)
{
        unsigned char perm, idx;
        unsigned int i;

        perm = 0;
        for (i = 0; i < 8; i++) {
                idx = (v1 >> (i * 8)) & 0xff;
                if (idx < 64)
                        if (v2 & PPC_BIT(idx))
                                perm |= 1 << i;
        }
        op->val = perm;
}
#endif /* CONFIG_PPC64 */
/*
 * The size parameter adjusts the equivalent prty instruction.
 * prtyw = 32, prtyd = 64
 */
static nokprobe_inline void do_prty(const struct pt_regs *regs,
                                    struct instruction_op *op,
                                    unsigned long v, int size)
{
        unsigned long long res = v ^ (v >> 8);

        res ^= res >> 16;
        if (size == 32) {               /* prtyw */
                op->val = res & 0x0000000100000001ULL;
                return;
        }

        res ^= res >> 32;
        op->val = res & 1;      /*prtyd */
}

static nokprobe_inline int trap_compare(long v1, long v2)
{
        int ret = 0;

        if (v1 < v2)
                ret |= 0x10;
        else if (v1 > v2)
                ret |= 0x08;
        else
                ret |= 0x04;
        if ((unsigned long)v1 < (unsigned long)v2)
                ret |= 0x02;
        else if ((unsigned long)v1 > (unsigned long)v2)
                ret |= 0x01;
        return ret;
}

/*
 * Elements of 32-bit rotate and mask instructions.
 */
#define MASK32(mb, me)  ((0xffffffffUL >> (mb)) + \
                         ((signed long)-0x80000000L >> (me)) + ((me) >= (mb)))
#ifdef __powerpc64__
#define MASK64_L(mb)    (~0UL >> (mb))
#define MASK64_R(me)    ((signed long)-0x8000000000000000L >> (me))
#define MASK64(mb, me)  (MASK64_L(mb) + MASK64_R(me) + ((me) >= (mb)))
#define DATA32(x)       (((x) & 0xffffffffUL) | (((x) & 0xffffffffUL) << 32))
#else
#define DATA32(x)       (x)
#endif
#define ROTATE(x, n)    ((n) ? (((x) << (n)) | ((x) >> (8 * sizeof(long) - (n)))) : (x))

/*
 * Decode an instruction, and return information about it in *op
 * without changing *regs.
 * Integer arithmetic and logical instructions, branches, and barrier
 * instructions can be emulated just using the information in *op.
 *
 * Return value is 1 if the instruction can be emulated just by
 * updating *regs with the information in *op, -1 if we need the
 * GPRs but *regs doesn't contain the full register set, or 0
 * otherwise.
 */
int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
                  ppc_inst_t instr)
{
#ifdef CONFIG_PPC64
        unsigned int suffixopcode, prefixtype, prefix_r;
#endif
        unsigned int opcode, ra, rb, rc, rd, spr, u;
        unsigned long int imm;
        unsigned long int val, val2;
        unsigned int mb, me, sh;
        unsigned int word, suffix;
        long ival;

        word = ppc_inst_val(instr);
        suffix = ppc_inst_suffix(instr);

        op->type = COMPUTE;

        opcode = ppc_inst_primary_opcode(instr);
        switch (opcode) {
        case 16:        /* bc */
                op->type = BRANCH;
                imm = (signed short)(word & 0xfffc);
                if ((word & 2) == 0)
                        imm += regs->nip;
                op->val = truncate_if_32bit(regs->msr, imm);
                if (word & 1)
                        op->type |= SETLK;
                if (branch_taken(word, regs, op))
                        op->type |= BRTAKEN;
                return 1;
        case 17:        /* sc */
                if ((word & 0xfe2) == 2)
                        op->type = SYSCALL;
                else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) &&
                                (word & 0xfe3) == 1) {  /* scv */
                        op->type = SYSCALL_VECTORED_0;
                        if (!cpu_has_feature(CPU_FTR_ARCH_300))
                                goto unknown_opcode;
                } else
                        op->type = UNKNOWN;
                return 0;
        case 18:        /* b */
                op->type = BRANCH | BRTAKEN;
                imm = word & 0x03fffffc;
                if (imm & 0x02000000)
                        imm -= 0x04000000;
                if ((word & 2) == 0)
                        imm += regs->nip;
                op->val = truncate_if_32bit(regs->msr, imm);
                if (word & 1)
                        op->type |= SETLK;
                return 1;
        case 19:
                switch ((word >> 1) & 0x3ff) {
                case 0:         /* mcrf */
                        op->type = COMPUTE + SETCC;
                        rd = 7 - ((word >> 23) & 0x7);
                        ra = 7 - ((word >> 18) & 0x7);
                        rd *= 4;
                        ra *= 4;
                        val = (regs->ccr >> ra) & 0xf;
                        op->ccval = (regs->ccr & ~(0xfUL << rd)) | (val << rd);
                        return 1;

                case 16:        /* bclr */
                case 528:       /* bcctr */
                        op->type = BRANCH;
                        imm = (word & 0x400)? regs->ctr: regs->link;
                        op->val = truncate_if_32bit(regs->msr, imm);
                        if (word & 1)
                                op->type |= SETLK;
                        if (branch_taken(word, regs, op))
                                op->type |= BRTAKEN;
                        return 1;

                case 18:        /* rfid, scary */
                        if (regs->msr & MSR_PR)
                                goto priv;
                        op->type = RFI;
                        return 0;

                case 150:       /* isync */
                        op->type = BARRIER | BARRIER_ISYNC;
                        return 1;

                case 33:        /* crnor */
                case 129:       /* crandc */
                case 193:       /* crxor */
                case 225:       /* crnand */
                case 257:       /* crand */
                case 289:       /* creqv */
                case 417:       /* crorc */
                case 449:       /* cror */
                        op->type = COMPUTE + SETCC;
                        ra = (word >> 16) & 0x1f;
                        rb = (word >> 11) & 0x1f;
                        rd = (word >> 21) & 0x1f;
                        ra = (regs->ccr >> (31 - ra)) & 1;
                        rb = (regs->ccr >> (31 - rb)) & 1;
                        val = (word >> (6 + ra * 2 + rb)) & 1;
                        op->ccval = (regs->ccr & ~(1UL << (31 - rd))) |
                                (val << (31 - rd));
                        return 1;
                }
                break;
        case 31:
                switch ((word >> 1) & 0x3ff) {
                case 598:       /* sync */
                        op->type = BARRIER + BARRIER_SYNC;
#ifdef __powerpc64__
                        switch ((word >> 21) & 3) {
                        case 1:         /* lwsync */
                                op->type = BARRIER + BARRIER_LWSYNC;
                                break;
                        case 2:         /* ptesync */
                                op->type = BARRIER + BARRIER_PTESYNC;
                                break;
                        }
#endif
                        return 1;

                case 854:       /* eieio */
                        op->type = BARRIER + BARRIER_EIEIO;
                        return 1;
                }
                break;
        }

        rd = (word >> 21) & 0x1f;
        ra = (word >> 16) & 0x1f;
        rb = (word >> 11) & 0x1f;
        rc = (word >> 6) & 0x1f;

        switch (opcode) {
#ifdef __powerpc64__
        case 1:
                if (!cpu_has_feature(CPU_FTR_ARCH_31))
                        goto unknown_opcode;

                prefix_r = GET_PREFIX_R(word);
                ra = GET_PREFIX_RA(suffix);
                rd = (suffix >> 21) & 0x1f;
                op->reg = rd;
                op->val = regs->gpr[rd];
                suffixopcode = get_op(suffix);
                prefixtype = (word >> 24) & 0x3;
                switch (prefixtype) {
                case 2:
                        if (prefix_r && ra)
                                return 0;
                        switch (suffixopcode) {
                        case 14:        /* paddi */
                                op->type = COMPUTE | PREFIXED;
                                op->val = mlsd_8lsd_ea(word, suffix, regs);
                                goto compute_done;
                        }
                }
                break;
        case 2:         /* tdi */
                if (rd & trap_compare(regs->gpr[ra], (short) word))
                        goto trap;
                return 1;
#endif
        case 3:         /* twi */
                if (rd & trap_compare((int)regs->gpr[ra], (short) word))
                        goto trap;
                return 1;

#ifdef __powerpc64__
        case 4:
                /*
                 * There are very many instructions with this primary opcode
                 * introduced in the ISA as early as v2.03. However, the ones
                 * we currently emulate were all introduced with ISA 3.0
                 */
                if (!cpu_has_feature(CPU_FTR_ARCH_300))
                        goto unknown_opcode;

                switch (word & 0x3f) {
                case 48:        /* maddhd */
                        asm volatile(PPC_MADDHD(%0, %1, %2, %3) :
                                     "=r" (op->val) : "r" (regs->gpr[ra]),
                                     "r" (regs->gpr[rb]), "r" (regs->gpr[rc]));
                        goto compute_done;

                case 49:        /* maddhdu */
                        asm volatile(PPC_MADDHDU(%0, %1, %2, %3) :
                                     "=r" (op->val) : "r" (regs->gpr[ra]),
                                     "r" (regs->gpr[rb]), "r" (regs->gpr[rc]));
                        goto compute_done;

                case 51:        /* maddld */
                        asm volatile(PPC_MADDLD(%0, %1, %2, %3) :
                                     "=r" (op->val) : "r" (regs->gpr[ra]),
                                     "r" (regs->gpr[rb]), "r" (regs->gpr[rc]));
                        goto compute_done;
                }

                /*
                 * There are other instructions from ISA 3.0 with the same
                 * primary opcode which do not have emulation support yet.
                 */
                goto unknown_opcode;
#endif

        case 7:         /* mulli */
                op->val = regs->gpr[ra] * (short) word;
                goto compute_done;

        case 8:         /* subfic */
                imm = (short) word;
                add_with_carry(regs, op, rd, ~regs->gpr[ra], imm, 1);
                return 1;

        case 10:        /* cmpli */
                imm = (unsigned short) word;
                val = regs->gpr[ra];
#ifdef __powerpc64__
                if ((rd & 1) == 0)
                        val = (unsigned int) val;
#endif
                do_cmp_unsigned(regs, op, val, imm, rd >> 2);
                return 1;

        case 11:        /* cmpi */
                imm = (short) word;
                val = regs->gpr[ra];
#ifdef __powerpc64__
                if ((rd & 1) == 0)
                        val = (int) val;
#endif
                do_cmp_signed(regs, op, val, imm, rd >> 2);
                return 1;

        case 12:        /* addic */
                imm = (short) word;
                add_with_carry(regs, op, rd, regs->gpr[ra], imm, 0);
                return 1;

        case 13:        /* addic. */
                imm = (short) word;
                add_with_carry(regs, op, rd, regs->gpr[ra], imm, 0);
                set_cr0(regs, op);
                return 1;

        case 14:        /* addi */
                imm = (short) word;
                if (ra)
                        imm += regs->gpr[ra];
                op->val = imm;
                goto compute_done;

        case 15:        /* addis */
                imm = ((short) word) << 16;
                if (ra)
                        imm += regs->gpr[ra];
                op->val = imm;
                goto compute_done;

        case 19:
                if (((word >> 1) & 0x1f) == 2) {
                        /* addpcis */
                        if (!cpu_has_feature(CPU_FTR_ARCH_300))
                                goto unknown_opcode;
                        imm = (short) (word & 0xffc1);  /* d0 + d2 fields */
                        imm |= (word >> 15) & 0x3e;     /* d1 field */
                        op->val = regs->nip + (imm << 16) + 4;
                        goto compute_done;
                }
                op->type = UNKNOWN;
                return 0;

        case 20:        /* rlwimi */
                mb = (word >> 6) & 0x1f;
                me = (word >> 1) & 0x1f;
                val = DATA32(regs->gpr[rd]);
                imm = MASK32(mb, me);
                op->val = (regs->gpr[ra] & ~imm) | (ROTATE(val, rb) & imm);
                goto logical_done;

        case 21:        /* rlwinm */
                mb = (word >> 6) & 0x1f;
                me = (word >> 1) & 0x1f;
                val = DATA32(regs->gpr[rd]);
                op->val = ROTATE(val, rb) & MASK32(mb, me);
                goto logical_done;

        case 23:        /* rlwnm */
                mb = (word >> 6) & 0x1f;
                me = (word >> 1) & 0x1f;
                rb = regs->gpr[rb] & 0x1f;
                val = DATA32(regs->gpr[rd]);
                op->val = ROTATE(val, rb) & MASK32(mb, me);
                goto logical_done;

        case 24:        /* ori */
                op->val = regs->gpr[rd] | (unsigned short) word;
                goto logical_done_nocc;

        case 25:        /* oris */
                imm = (unsigned short) word;
                op->val = regs->gpr[rd] | (imm << 16);
                goto logical_done_nocc;

        case 26:        /* xori */
                op->val = regs->gpr[rd] ^ (unsigned short) word;
                goto logical_done_nocc;

        case 27:        /* xoris */
                imm = (unsigned short) word;
                op->val = regs->gpr[rd] ^ (imm << 16);
                goto logical_done_nocc;

        case 28:        /* andi. */
                op->val = regs->gpr[rd] & (unsigned short) word;
                set_cr0(regs, op);
                goto logical_done_nocc;

        case 29:        /* andis. */
                imm = (unsigned short) word;
                op->val = regs->gpr[rd] & (imm << 16);
                set_cr0(regs, op);
                goto logical_done_nocc;

#ifdef __powerpc64__
        case 30:        /* rld* */
                mb = ((word >> 6) & 0x1f) | (word & 0x20);
                val = regs->gpr[rd];
                if ((word & 0x10) == 0) {
                        sh = rb | ((word & 2) << 4);
                        val = ROTATE(val, sh);
                        switch ((word >> 2) & 3) {
                        case 0:         /* rldicl */
                                val &= MASK64_L(mb);
                                break;
                        case 1:         /* rldicr */
                                val &= MASK64_R(mb);
                                break;
                        case 2:         /* rldic */
                                val &= MASK64(mb, 63 - sh);
                                break;
                        case 3:         /* rldimi */
                                imm = MASK64(mb, 63 - sh);
                                val = (regs->gpr[ra] & ~imm) |
                                        (val & imm);
                        }
                        op->val = val;
                        goto logical_done;
                } else {
                        sh = regs->gpr[rb] & 0x3f;
                        val = ROTATE(val, sh);
                        switch ((word >> 1) & 7) {
                        case 0:         /* rldcl */
                                op->val = val & MASK64_L(mb);
                                goto logical_done;
                        case 1:         /* rldcr */
                                op->val = val & MASK64_R(mb);
                                goto logical_done;
                        }
                }
#endif
                op->type = UNKNOWN;     /* illegal instruction */
                return 0;

        case 31:
                /* isel occupies 32 minor opcodes */
                if (((word >> 1) & 0x1f) == 15) {
                        mb = (word >> 6) & 0x1f; /* bc field */
                        val = (regs->ccr >> (31 - mb)) & 1;
                        val2 = (ra) ? regs->gpr[ra] : 0;

                        op->val = (val) ? val2 : regs->gpr[rb];
                        goto compute_done;
                }

                switch ((word >> 1) & 0x3ff) {
                case 4:         /* tw */
                        if (rd == 0x1f ||
                            (rd & trap_compare((int)regs->gpr[ra],
                                               (int)regs->gpr[rb])))
                                goto trap;
                        return 1;
#ifdef __powerpc64__
                case 68:        /* td */
                        if (rd & trap_compare(regs->gpr[ra], regs->gpr[rb]))
                                goto trap;
                        return 1;
#endif
                case 83:        /* mfmsr */
                        if (regs->msr & MSR_PR)
                                goto priv;
                        op->type = MFMSR;
                        op->reg = rd;
                        return 0;
                case 146:       /* mtmsr */
                        if (regs->msr & MSR_PR)
                                goto priv;
                        op->type = MTMSR;
                        op->reg = rd;
                        op->val = 0xffffffff & ~(MSR_ME | MSR_LE);
                        return 0;
#ifdef CONFIG_PPC64
                case 178:       /* mtmsrd */
                        if (regs->msr & MSR_PR)
                                goto priv;
                        op->type = MTMSR;
                        op->reg = rd;
                        /* only MSR_EE and MSR_RI get changed if bit 15 set */
                        /* mtmsrd doesn't change MSR_HV, MSR_ME or MSR_LE */
                        imm = (word & 0x10000)? 0x8002: 0xefffffffffffeffeUL;
                        op->val = imm;
                        return 0;
#endif

                case 19:        /* mfcr */
                        imm = 0xffffffffUL;
                        if ((word >> 20) & 1) {
                                imm = 0xf0000000UL;
                                for (sh = 0; sh < 8; ++sh) {
                                        if (word & (0x80000 >> sh))
                                                break;
                                        imm >>= 4;
                                }
                        }
                        op->val = regs->ccr & imm;
                        goto compute_done;

                case 128:       /* setb */
                        if (!cpu_has_feature(CPU_FTR_ARCH_300))
                                goto unknown_opcode;
                        /*
                         * 'ra' encodes the CR field number (bfa) in the top 3 bits.
                         * Since each CR field is 4 bits,
                         * we can simply mask off the bottom two bits (bfa * 4)
                         * to yield the first bit in the CR field.
                         */
                        ra = ra & ~0x3;
                        /* 'val' stores bits of the CR field (bfa) */
                        val = regs->ccr >> (CR0_SHIFT - ra);
                        /* checks if the LT bit of CR field (bfa) is set */
                        if (val & 8)
                                op->val = -1;
                        /* checks if the GT bit of CR field (bfa) is set */
                        else if (val & 4)
                                op->val = 1;
                        else
                                op->val = 0;
                        goto compute_done;

                case 144:       /* mtcrf */
                        op->type = COMPUTE + SETCC;
                        imm = 0xf0000000UL;
                        val = regs->gpr[rd];
                        op->ccval = regs->ccr;
                        for (sh = 0; sh < 8; ++sh) {
                                if (word & (0x80000 >> sh))
                                        op->ccval = (op->ccval & ~imm) |
                                                (val & imm);
                                imm >>= 4;
                        }
                        return 1;

                case 339:       /* mfspr */
                        spr = ((word >> 16) & 0x1f) | ((word >> 6) & 0x3e0);
                        op->type = MFSPR;
                        op->reg = rd;
                        op->spr = spr;
                        if (spr == SPRN_XER || spr == SPRN_LR ||
                            spr == SPRN_CTR)
                                return 1;
                        return 0;

                case 467:       /* mtspr */
                        spr = ((word >> 16) & 0x1f) | ((word >> 6) & 0x3e0);
                        op->type = MTSPR;
                        op->val = regs->gpr[rd];
                        op->spr = spr;
                        if (spr == SPRN_XER || spr == SPRN_LR ||
                            spr == SPRN_CTR)
                                return 1;
                        return 0;

/*
 * Compare instructions
 */
                case 0: /* cmp */
                        val = regs->gpr[ra];
                        val2 = regs->gpr[rb];
#ifdef __powerpc64__
                        if ((rd & 1) == 0) {
                                /* word (32-bit) compare */
                                val = (int) val;
                                val2 = (int) val2;
                        }
#endif
                        do_cmp_signed(regs, op, val, val2, rd >> 2);
                        return 1;

                case 32:        /* cmpl */
                        val = regs->gpr[ra];
                        val2 = regs->gpr[rb];
#ifdef __powerpc64__
                        if ((rd & 1) == 0) {
                                /* word (32-bit) compare */
                                val = (unsigned int) val;
                                val2 = (unsigned int) val2;
                        }
#endif
                        do_cmp_unsigned(regs, op, val, val2, rd >> 2);
                        return 1;

                case 508: /* cmpb */
                        do_cmpb(regs, op, regs->gpr[rd], regs->gpr[rb]);
                        goto logical_done_nocc;

/*
 * Arithmetic instructions
 */
                case 8: /* subfc */
                        add_with_carry(regs, op, rd, ~regs->gpr[ra],
                                       regs->gpr[rb], 1);
                        goto arith_done;
#ifdef __powerpc64__
                case 9: /* mulhdu */
                        asm("mulhdu %0,%1,%2" : "=r" (op->val) :
                            "r" (regs->gpr[ra]), "r" (regs->gpr[rb]));
                        goto arith_done;
#endif
                case 10:        /* addc */
                        add_with_carry(regs, op, rd, regs->gpr[ra],
                                       regs->gpr[rb], 0);
                        goto arith_done;

                case 11:        /* mulhwu */
                        asm("mulhwu %0,%1,%2" : "=r" (op->val) :
                            "r" (regs->gpr[ra]), "r" (regs->gpr[rb]));
                        goto arith_done;

                case 40:        /* subf */
                        op->val = regs->gpr[rb] - regs->gpr[ra];
                        goto arith_done;
#ifdef __powerpc64__
                case 73:        /* mulhd */
                        asm("mulhd %0,%1,%2" : "=r" (op->val) :
                            "r" (regs->gpr[ra]), "r" (regs->gpr[rb]));
                        goto arith_done;
#endif
                case 75:        /* mulhw */
                        asm("mulhw %0,%1,%2" : "=r" (op->val) :
                            "r" (regs->gpr[ra]), "r" (regs->gpr[rb]));
                        goto arith_done;

                case 104:       /* neg */
                        op->val = -regs->gpr[ra];
                        goto arith_done;

                case 136:       /* subfe */
                        add_with_carry(regs, op, rd, ~regs->gpr[ra],
                                       regs->gpr[rb], regs->xer & XER_CA);
                        goto arith_done;

                case 138:       /* adde */
                        add_with_carry(regs, op, rd, regs->gpr[ra],
                                       regs->gpr[rb], regs->xer & XER_CA);
                        goto arith_done;

                case 200:       /* subfze */
                        add_with_carry(regs, op, rd, ~regs->gpr[ra], 0L,
                                       regs->xer & XER_CA);
                        goto arith_done;

                case 202:       /* addze */
                        add_with_carry(regs, op, rd, regs->gpr[ra], 0L,
                                       regs->xer & XER_CA);
                        goto arith_done;

                case 232:       /* subfme */
                        add_with_carry(regs, op, rd, ~regs->gpr[ra], -1L,
                                       regs->xer & XER_CA);
                        goto arith_done;
#ifdef __powerpc64__
                case 233:       /* mulld */
                        op->val = regs->gpr[ra] * regs->gpr[rb];
                        goto arith_done;
#endif
                case 234:       /* addme */
                        add_with_carry(regs, op, rd, regs->gpr[ra], -1L,
                                       regs->xer & XER_CA);
                        goto arith_done;

                case 235:       /* mullw */
                        op->val = (long)(int) regs->gpr[ra] *
                                (int) regs->gpr[rb];

                        goto arith_done;
#ifdef __powerpc64__
                case 265:       /* modud */
                        if (!cpu_has_feature(CPU_FTR_ARCH_300))
                                goto unknown_opcode;
                        op->val = regs->gpr[ra] % regs->gpr[rb];
                        goto compute_done;
#endif
                case 266:       /* add */
                        op->val = regs->gpr[ra] + regs->gpr[rb];
                        goto arith_done;

                case 267:       /* moduw */
                        if (!cpu_has_feature(CPU_FTR_ARCH_300))
                                goto unknown_opcode;
                        op->val = (unsigned int) regs->gpr[ra] %
                                (unsigned int) regs->gpr[rb];
                        goto compute_done;
#ifdef __powerpc64__
                case 457:       /* divdu */
                        op->val = regs->gpr[ra] / regs->gpr[rb];
                        goto arith_done;
#endif
                case 459:       /* divwu */
                        op->val = (unsigned int) regs->gpr[ra] /
                                (unsigned int) regs->gpr[rb];
                        goto arith_done;
#ifdef __powerpc64__
                case 489:       /* divd */
                        op->val = (long int) regs->gpr[ra] /
                                (long int) regs->gpr[rb];
                        goto arith_done;
#endif
                case 491:       /* divw */
                        op->val = (int) regs->gpr[ra] /
                                (int) regs->gpr[rb];
                        goto arith_done;
#ifdef __powerpc64__
                case 425:       /* divde[.] */
                        asm volatile(PPC_DIVDE(%0, %1, %2) :
                                "=r" (op->val) : "r" (regs->gpr[ra]),
                                "r" (regs->gpr[rb]));
                        goto arith_done;
                case 393:       /* divdeu[.] */
                        asm volatile(PPC_DIVDEU(%0, %1, %2) :
                                "=r" (op->val) : "r" (regs->gpr[ra]),
                                "r" (regs->gpr[rb]));
                        goto arith_done;
#endif
                case 755:       /* darn */
                        if (!cpu_has_feature(CPU_FTR_ARCH_300))
                                goto unknown_opcode;
                        switch (ra & 0x3) {
                        case 0:
                                /* 32-bit conditioned */
                                asm volatile(PPC_DARN(%0, 0) : "=r" (op->val));
                                goto compute_done;

                        case 1:
                                /* 64-bit conditioned */
                                asm volatile(PPC_DARN(%0, 1) : "=r" (op->val));