linux/mm/maccess.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Access kernel or user memory without faulting.
   4 */
   5#include <linux/export.h>
   6#include <linux/mm.h>
   7#include <linux/uaccess.h>
   8
   9bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src,
  10                size_t size)
  11{
  12        return true;
  13}
  14
  15#ifdef HAVE_GET_KERNEL_NOFAULT
  16
  17#define copy_from_kernel_nofault_loop(dst, src, len, type, err_label)   \
  18        while (len >= sizeof(type)) {                                   \
  19                __get_kernel_nofault(dst, src, type, err_label);                \
  20                dst += sizeof(type);                                    \
  21                src += sizeof(type);                                    \
  22                len -= sizeof(type);                                    \
  23        }
  24
  25long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
  26{
  27        if (!copy_from_kernel_nofault_allowed(src, size))
  28                return -ERANGE;
  29
  30        pagefault_disable();
  31        copy_from_kernel_nofault_loop(dst, src, size, u64, Efault);
  32        copy_from_kernel_nofault_loop(dst, src, size, u32, Efault);
  33        copy_from_kernel_nofault_loop(dst, src, size, u16, Efault);
  34        copy_from_kernel_nofault_loop(dst, src, size, u8, Efault);
  35        pagefault_enable();
  36        return 0;
  37Efault:
  38        pagefault_enable();
  39        return -EFAULT;
  40}
  41EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
  42
  43#define copy_to_kernel_nofault_loop(dst, src, len, type, err_label)     \
  44        while (len >= sizeof(type)) {                                   \
  45                __put_kernel_nofault(dst, src, type, err_label);                \
  46                dst += sizeof(type);                                    \
  47                src += sizeof(type);                                    \
  48                len -= sizeof(type);                                    \
  49        }
  50
  51long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
  52{
  53        pagefault_disable();
  54        copy_to_kernel_nofault_loop(dst, src, size, u64, Efault);
  55        copy_to_kernel_nofault_loop(dst, src, size, u32, Efault);
  56        copy_to_kernel_nofault_loop(dst, src, size, u16, Efault);
  57        copy_to_kernel_nofault_loop(dst, src, size, u8, Efault);
  58        pagefault_enable();
  59        return 0;
  60Efault:
  61        pagefault_enable();
  62        return -EFAULT;
  63}
  64
  65long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
  66{
  67        const void *src = unsafe_addr;
  68
  69        if (unlikely(count <= 0))
  70                return 0;
  71        if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
  72                return -ERANGE;
  73
  74        pagefault_disable();
  75        do {
  76                __get_kernel_nofault(dst, src, u8, Efault);
  77                dst++;
  78                src++;
  79        } while (dst[-1] && src - unsafe_addr < count);
  80        pagefault_enable();
  81
  82        dst[-1] = '\0';
  83        return src - unsafe_addr;
  84Efault:
  85        pagefault_enable();
  86        dst[-1] = '\0';
  87        return -EFAULT;
  88}
  89#else /* HAVE_GET_KERNEL_NOFAULT */
  90/**
  91 * copy_from_kernel_nofault(): safely attempt to read from kernel-space
  92 * @dst: pointer to the buffer that shall take the data
  93 * @src: address to read from
  94 * @size: size of the data chunk
  95 *
  96 * Safely read from kernel address @src to the buffer at @dst.  If a kernel
  97 * fault happens, handle that and return -EFAULT.  If @src is not a valid kernel
  98 * address, return -ERANGE.
  99 *
 100 * We ensure that the copy_from_user is executed in atomic context so that
 101 * do_page_fault() doesn't attempt to take mmap_lock.  This makes
 102 * copy_from_kernel_nofault() suitable for use within regions where the caller
 103 * already holds mmap_lock, or other locks which nest inside mmap_lock.
 104 */
 105long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
 106{
 107        long ret;
 108        mm_segment_t old_fs = get_fs();
 109
 110        if (!copy_from_kernel_nofault_allowed(src, size))
 111                return -ERANGE;
 112
 113        set_fs(KERNEL_DS);
 114        pagefault_disable();
 115        ret = __copy_from_user_inatomic(dst, (__force const void __user *)src,
 116                        size);
 117        pagefault_enable();
 118        set_fs(old_fs);
 119
 120        if (ret)
 121                return -EFAULT;
 122        return 0;
 123}
 124EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
 125
 126/**
 127 * copy_to_kernel_nofault(): safely attempt to write to a location
 128 * @dst: address to write to
 129 * @src: pointer to the data that shall be written
 130 * @size: size of the data chunk
 131 *
 132 * Safely write to address @dst from the buffer at @src.  If a kernel fault
 133 * happens, handle that and return -EFAULT.
 134 */
 135long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
 136{
 137        long ret;
 138        mm_segment_t old_fs = get_fs();
 139
 140        set_fs(KERNEL_DS);
 141        pagefault_disable();
 142        ret = __copy_to_user_inatomic((__force void __user *)dst, src, size);
 143        pagefault_enable();
 144        set_fs(old_fs);
 145
 146        if (ret)
 147                return -EFAULT;
 148        return 0;
 149}
 150
 151/**
 152 * strncpy_from_kernel_nofault: - Copy a NUL terminated string from unsafe
 153 *                               address.
 154 * @dst:   Destination address, in kernel space.  This buffer must be at
 155 *         least @count bytes long.
 156 * @unsafe_addr: Unsafe address.
 157 * @count: Maximum number of bytes to copy, including the trailing NUL.
 158 *
 159 * Copies a NUL-terminated string from unsafe address to kernel buffer.
 160 *
 161 * On success, returns the length of the string INCLUDING the trailing NUL.
 162 *
 163 * If access fails, returns -EFAULT (some data may have been copied and the
 164 * trailing NUL added).  If @unsafe_addr is not a valid kernel address, return
 165 * -ERANGE.
 166 *
 167 * If @count is smaller than the length of the string, copies @count-1 bytes,
 168 * sets the last byte of @dst buffer to NUL and returns @count.
 169 */
 170long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
 171{
 172        mm_segment_t old_fs = get_fs();
 173        const void *src = unsafe_addr;
 174        long ret;
 175
 176        if (unlikely(count <= 0))
 177                return 0;
 178        if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
 179                return -ERANGE;
 180
 181        set_fs(KERNEL_DS);
 182        pagefault_disable();
 183
 184        do {
 185                ret = __get_user(*dst++, (const char __user __force *)src++);
 186        } while (dst[-1] && ret == 0 && src - unsafe_addr < count);
 187
 188        dst[-1] = '\0';
 189        pagefault_enable();
 190        set_fs(old_fs);
 191
 192        return ret ? -EFAULT : src - unsafe_addr;
 193}
 194#endif /* HAVE_GET_KERNEL_NOFAULT */
 195
 196/**
 197 * copy_from_user_nofault(): safely attempt to read from a user-space location
 198 * @dst: pointer to the buffer that shall take the data
 199 * @src: address to read from. This must be a user address.
 200 * @size: size of the data chunk
 201 *
 202 * Safely read from user address @src to the buffer at @dst. If a kernel fault
 203 * happens, handle that and return -EFAULT.
 204 */
 205long copy_from_user_nofault(void *dst, const void __user *src, size_t size)
 206{
 207        long ret = -EFAULT;
 208        mm_segment_t old_fs = force_uaccess_begin();
 209
 210        if (access_ok(src, size)) {
 211                pagefault_disable();
 212                ret = __copy_from_user_inatomic(dst, src, size);
 213                pagefault_enable();
 214        }
 215        force_uaccess_end(old_fs);
 216
 217        if (ret)
 218                return -EFAULT;
 219        return 0;
 220}
 221EXPORT_SYMBOL_GPL(copy_from_user_nofault);
 222
 223/**
 224 * copy_to_user_nofault(): safely attempt to write to a user-space location
 225 * @dst: address to write to
 226 * @src: pointer to the data that shall be written
 227 * @size: size of the data chunk
 228 *
 229 * Safely write to address @dst from the buffer at @src.  If a kernel fault
 230 * happens, handle that and return -EFAULT.
 231 */
 232long copy_to_user_nofault(void __user *dst, const void *src, size_t size)
 233{
 234        long ret = -EFAULT;
 235        mm_segment_t old_fs = force_uaccess_begin();
 236
 237        if (access_ok(dst, size)) {
 238                pagefault_disable();
 239                ret = __copy_to_user_inatomic(dst, src, size);
 240                pagefault_enable();
 241        }
 242        force_uaccess_end(old_fs);
 243
 244        if (ret)
 245                return -EFAULT;
 246        return 0;
 247}
 248EXPORT_SYMBOL_GPL(copy_to_user_nofault);
 249
 250/**
 251 * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user
 252 *                              address.
 253 * @dst:   Destination address, in kernel space.  This buffer must be at
 254 *         least @count bytes long.
 255 * @unsafe_addr: Unsafe user address.
 256 * @count: Maximum number of bytes to copy, including the trailing NUL.
 257 *
 258 * Copies a NUL-terminated string from unsafe user address to kernel buffer.
 259 *
 260 * On success, returns the length of the string INCLUDING the trailing NUL.
 261 *
 262 * If access fails, returns -EFAULT (some data may have been copied
 263 * and the trailing NUL added).
 264 *
 265 * If @count is smaller than the length of the string, copies @count-1 bytes,
 266 * sets the last byte of @dst buffer to NUL and returns @count.
 267 */
 268long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,
 269                              long count)
 270{
 271        mm_segment_t old_fs;
 272        long ret;
 273
 274        if (unlikely(count <= 0))
 275                return 0;
 276
 277        old_fs = force_uaccess_begin();
 278        pagefault_disable();
 279        ret = strncpy_from_user(dst, unsafe_addr, count);
 280        pagefault_enable();
 281        force_uaccess_end(old_fs);
 282
 283        if (ret >= count) {
 284                ret = count;
 285                dst[ret - 1] = '\0';
 286        } else if (ret > 0) {
 287                ret++;
 288        }
 289
 290        return ret;
 291}
 292
 293/**
 294 * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL.
 295 * @unsafe_addr: The string to measure.
 296 * @count: Maximum count (including NUL)
 297 *
 298 * Get the size of a NUL-terminated string in user space without pagefault.
 299 *
 300 * Returns the size of the string INCLUDING the terminating NUL.
 301 *
 302 * If the string is too long, returns a number larger than @count. User
 303 * has to check the return value against "> count".
 304 * On exception (or invalid count), returns 0.
 305 *
 306 * Unlike strnlen_user, this can be used from IRQ handler etc. because
 307 * it disables pagefaults.
 308 */
 309long strnlen_user_nofault(const void __user *unsafe_addr, long count)
 310{
 311        mm_segment_t old_fs;
 312        int ret;
 313
 314        old_fs = force_uaccess_begin();
 315        pagefault_disable();
 316        ret = strnlen_user(unsafe_addr, count);
 317        pagefault_enable();
 318        force_uaccess_end(old_fs);
 319
 320        return ret;
 321}
 322
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.