LXR linux/arch/s390/pci/pci

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Access to PCI I/O memory from user space programs.
   4 *
   5 * Copyright IBM Corp. 2014
   6 * Author(s): Alexey Ishchuk <aishchuk@linux.vnet.ibm.com>
   7 */
   8#include <linux/kernel.h>
   9#include <linux/syscalls.h>
  10#include <linux/init.h>
  11#include <linux/mm.h>
  12#include <linux/errno.h>
  13#include <linux/pci.h>
  14#include <asm/asm-extable.h>
  15#include <asm/pci_io.h>
  16#include <asm/pci_debug.h>
  17
  18static inline void zpci_err_mmio(u8 cc, u8 status, u64 offset)
  19{
  20        struct {
  21                u64 offset;
  22                u8 cc;
  23                u8 status;
  24        } data = {offset, cc, status};
  25
  26        zpci_err_hex(&data, sizeof(data));
  27}
  28
  29static inline int __pcistb_mio_inuser(
  30                void __iomem *ioaddr, const void __user *src,
  31                u64 len, u8 *status)
  32{
  33        int cc = -ENXIO;
  34
  35        asm volatile (
  36                "       sacf 256\n"
  37                "0:     .insn   rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n"
  38                "1:     ipm     %[cc]\n"
  39                "       srl     %[cc],28\n"
  40                "2:     sacf 768\n"
  41                EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
  42                : [cc] "+d" (cc), [len] "+d" (len)
  43                : [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src))
  44                : "cc", "memory");
  45        *status = len >> 24 & 0xff;
  46        return cc;
  47}
  48
  49static inline int __pcistg_mio_inuser(
  50                void __iomem *ioaddr, const void __user *src,
  51                u64 ulen, u8 *status)
  52{
  53        union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen};
  54        int cc = -ENXIO;
  55        u64 val = 0;
  56        u64 cnt = ulen;
  57        u8 tmp;
  58
  59        /*
  60         * copy 0 < @len <= 8 bytes from @src into the right most bytes of
  61         * a register, then store it to PCI at @ioaddr while in secondary
  62         * address space. pcistg then uses the user mappings.
  63         */
  64        asm volatile (
  65                "       sacf    256\n"
  66                "0:     llgc    %[tmp],0(%[src])\n"
  67                "4:     sllg    %[val],%[val],8\n"
  68                "       aghi    %[src],1\n"
  69                "       ogr     %[val],%[tmp]\n"
  70                "       brctg   %[cnt],0b\n"
  71                "1:     .insn   rre,0xb9d40000,%[val],%[ioaddr_len]\n"
  72                "2:     ipm     %[cc]\n"
  73                "       srl     %[cc],28\n"
  74                "3:     sacf    768\n"
  75                EX_TABLE(0b, 3b) EX_TABLE(4b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b)
  76                :
  77                [src] "+a" (src), [cnt] "+d" (cnt),
  78                [val] "+d" (val), [tmp] "=d" (tmp),
  79                [cc] "+d" (cc), [ioaddr_len] "+&d" (ioaddr_len.pair)
  80                :: "cc", "memory");
  81        *status = ioaddr_len.odd >> 24 & 0xff;
  82
  83        /* did we read everything from user memory? */
  84        if (!cc && cnt != 0)
  85                cc = -EFAULT;
  86
  87        return cc;
  88}
  89
  90static inline int __memcpy_toio_inuser(void __iomem *dst,
  91                                   const void __user *src, size_t n)
  92{
  93        int size, rc = 0;
  94        u8 status = 0;
  95
  96        if (!src)
  97                return -EINVAL;
  98
  99        while (n > 0) {
 100                size = zpci_get_max_write_size((u64 __force) dst,
 101                                               (u64 __force) src, n,
 102                                               ZPCI_MAX_WRITE_SIZE);
 103                if (size > 8) /* main path */
 104                        rc = __pcistb_mio_inuser(dst, src, size, &status);
 105                else
 106                        rc = __pcistg_mio_inuser(dst, src, size, &status);
 107                if (rc)
 108                        break;
 109                src += size;
 110                dst += size;
 111                n -= size;
 112        }
 113        if (rc)
 114                zpci_err_mmio(rc, status, (__force u64) dst);
 115        return rc;
 116}
 117
 118SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr,
 119                const void __user *, user_buffer, size_t, length)
 120{
 121        u8 local_buf[64];
 122        void __iomem *io_addr;
 123        void *buf;
 124        struct vm_area_struct *vma;
 125        pte_t *ptep;
 126        spinlock_t *ptl;
 127        long ret;
 128
 129        if (!zpci_is_enabled())
 130                return -ENODEV;
 131
 132        if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
 133                return -EINVAL;
 134
 135        /*
 136         * We only support write access to MIO capable devices if we are on
 137         * a MIO enabled system. Otherwise we would have to check for every
 138         * address if it is a special ZPCI_ADDR and would have to do
 139         * a pfn lookup which we don't need for MIO capable devices.  Currently
 140         * ISM devices are the only devices without MIO support and there is no
 141         * known need for accessing these from userspace.
 142         */
 143        if (static_branch_likely(&have_mio)) {
 144                ret = __memcpy_toio_inuser((void  __iomem *) mmio_addr,
 145                                        user_buffer,
 146                                        length);
 147                return ret;
 148        }
 149
 150        if (length > 64) {
 151                buf = kmalloc(length, GFP_KERNEL);
 152                if (!buf)
 153                        return -ENOMEM;
 154        } else
 155                buf = local_buf;
 156
 157        ret = -EFAULT;
 158        if (copy_from_user(buf, user_buffer, length))
 159                goto out_free;
 160
 161        mmap_read_lock(current->mm);
 162        ret = -EINVAL;
 163        vma = vma_lookup(current->mm, mmio_addr);
 164        if (!vma)
 165                goto out_unlock_mmap;
 166        if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
 167                goto out_unlock_mmap;
 168        ret = -EACCES;
 169        if (!(vma->vm_flags & VM_WRITE))
 170                goto out_unlock_mmap;
 171
 172        ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl);
 173        if (ret)
 174                goto out_unlock_mmap;
 175
 176        io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) |
 177                        (mmio_addr & ~PAGE_MASK));
 178
 179        if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE)
 180                goto out_unlock_pt;
 181
 182        ret = zpci_memcpy_toio(io_addr, buf, length);
 183out_unlock_pt:
 184        pte_unmap_unlock(ptep, ptl);
 185out_unlock_mmap:
 186        mmap_read_unlock(current->mm);
 187out_free:
 188        if (buf != local_buf)
 189                kfree(buf);
 190        return ret;
 191}
 192
 193static inline int __pcilg_mio_inuser(
 194                void __user *dst, const void __iomem *ioaddr,
 195                u64 ulen, u8 *status)
 196{
 197        union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen};
 198        u64 cnt = ulen;
 199        int shift = ulen * 8;
 200        int cc = -ENXIO;
 201        u64 val, tmp;
 202
 203        /*
 204         * read 0 < @len <= 8 bytes from the PCI memory mapped at @ioaddr (in
 205         * user space) into a register using pcilg then store these bytes at
 206         * user address @dst
 207         */
 208        asm volatile (
 209                "       sacf    256\n"
 210                "0:     .insn   rre,0xb9d60000,%[val],%[ioaddr_len]\n"
 211                "1:     ipm     %[cc]\n"
 212                "       srl     %[cc],28\n"
 213                "       ltr     %[cc],%[cc]\n"
 214                "       jne     4f\n"
 215                "2:     ahi     %[shift],-8\n"
 216                "       srlg    %[tmp],%[val],0(%[shift])\n"
 217                "3:     stc     %[tmp],0(%[dst])\n"
 218                "5:     aghi    %[dst],1\n"
 219                "       brctg   %[cnt],2b\n"
 220                "4:     sacf    768\n"
 221                EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b) EX_TABLE(5b, 4b)
 222                :
 223                [ioaddr_len] "+&d" (ioaddr_len.pair),
 224                [cc] "+d" (cc), [val] "=d" (val),
 225                [dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp),
 226                [shift] "+d" (shift)
 227                :: "cc", "memory");
 228
 229        /* did we write everything to the user space buffer? */
 230        if (!cc && cnt != 0)
 231                cc = -EFAULT;
 232
 233        *status = ioaddr_len.odd >> 24 & 0xff;
 234        return cc;
 235}
 236
 237static inline int __memcpy_fromio_inuser(void __user *dst,
 238                                     const void __iomem *src,
 239                                     unsigned long n)
 240{
 241        int size, rc = 0;
 242        u8 status;
 243
 244        while (n > 0) {
 245                size = zpci_get_max_write_size((u64 __force) src,
 246                                               (u64 __force) dst, n,
 247                                               ZPCI_MAX_READ_SIZE);
 248                rc = __pcilg_mio_inuser(dst, src, size, &status);
 249                if (rc)
 250                        break;
 251                src += size;
 252                dst += size;
 253                n -= size;
 254        }
 255        if (rc)
 256                zpci_err_mmio(rc, status, (__force u64) dst);
 257        return rc;
 258}
 259
 260SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr,
 261                void __user *, user_buffer, size_t, length)
 262{
 263        u8 local_buf[64];
 264        void __iomem *io_addr;
 265        void *buf;
 266        struct vm_area_struct *vma;
 267        pte_t *ptep;
 268        spinlock_t *ptl;
 269        long ret;
 270
 271        if (!zpci_is_enabled())
 272                return -ENODEV;
 273
 274        if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
 275                return -EINVAL;
 276
 277        /*
 278         * We only support read access to MIO capable devices if we are on
 279         * a MIO enabled system. Otherwise we would have to check for every
 280         * address if it is a special ZPCI_ADDR and would have to do
 281         * a pfn lookup which we don't need for MIO capable devices.  Currently
 282         * ISM devices are the only devices without MIO support and there is no
 283         * known need for accessing these from userspace.
 284         */
 285        if (static_branch_likely(&have_mio)) {
 286                ret = __memcpy_fromio_inuser(
 287                                user_buffer, (const void __iomem *)mmio_addr,
 288                                length);
 289                return ret;
 290        }
 291
 292        if (length > 64) {
 293                buf = kmalloc(length, GFP_KERNEL);
 294                if (!buf)
 295                        return -ENOMEM;
 296        } else {
 297                buf = local_buf;
 298        }
 299
 300        mmap_read_lock(current->mm);
 301        ret = -EINVAL;
 302        vma = vma_lookup(current->mm, mmio_addr);
 303        if (!vma)
 304                goto out_unlock_mmap;
 305        if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
 306                goto out_unlock_mmap;
 307        ret = -EACCES;
 308        if (!(vma->vm_flags & VM_WRITE))
 309                goto out_unlock_mmap;
 310
 311        ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl);
 312        if (ret)
 313                goto out_unlock_mmap;
 314
 315        io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) |
 316                        (mmio_addr & ~PAGE_MASK));
 317
 318        if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) {
 319                ret = -EFAULT;
 320                goto out_unlock_pt;
 321        }
 322        ret = zpci_memcpy_fromio(buf, io_addr, length);
 323
 324out_unlock_pt:
 325        pte_unmap_unlock(ptep, ptl);
 326out_unlock_mmap:
 327        mmap_read_unlock(current->mm);
 328
 329        if (!ret && copy_to_user(user_buffer, buf, length))
 330                ret = -EFAULT;
 331
 332        if (buf != local_buf)
 333                kfree(buf);
 334        return ret;
 335}
 336