linux/ipc/shm.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * linux/ipc/shm.c
   4 * Copyright (C) 1992, 1993 Krishna Balasubramanian
   5 *       Many improvements/fixes by Bruno Haible.
   6 * Replaced `struct shm_desc' by `struct vm_area_struct', July 1994.
   7 * Fixed the shm swap deallocation (shm_unuse()), August 1998 Andrea Arcangeli.
   8 *
   9 * /proc/sysvipc/shm support (c) 1999 Dragos Acostachioaie <dragos@iname.com>
  10 * BIGMEM support, Andrea Arcangeli <andrea@suse.de>
  11 * SMP thread shm, Jean-Luc Boyard <jean-luc.boyard@siemens.fr>
  12 * HIGHMEM support, Ingo Molnar <mingo@redhat.com>
  13 * Make shmmax, shmall, shmmni sysctl'able, Christoph Rohland <cr@sap.com>
  14 * Shared /dev/zero support, Kanoj Sarcar <kanoj@sgi.com>
  15 * Move the mm functionality over to mm/shmem.c, Christoph Rohland <cr@sap.com>
  16 *
  17 * support for audit of ipc object properties and permission changes
  18 * Dustin Kirkland <dustin.kirkland@us.ibm.com>
  19 *
  20 * namespaces support
  21 * OpenVZ, SWsoft Inc.
  22 * Pavel Emelianov <xemul@openvz.org>
  23 *
  24 * Better ipc lock (kern_ipc_perm.lock) handling
  25 * Davidlohr Bueso <davidlohr.bueso@hp.com>, June 2013.
  26 */
  27
  28#include <linux/slab.h>
  29#include <linux/mm.h>
  30#include <linux/hugetlb.h>
  31#include <linux/shm.h>
  32#include <linux/init.h>
  33#include <linux/file.h>
  34#include <linux/mman.h>
  35#include <linux/shmem_fs.h>
  36#include <linux/security.h>
  37#include <linux/syscalls.h>
  38#include <linux/audit.h>
  39#include <linux/capability.h>
  40#include <linux/ptrace.h>
  41#include <linux/seq_file.h>
  42#include <linux/rwsem.h>
  43#include <linux/nsproxy.h>
  44#include <linux/mount.h>
  45#include <linux/ipc_namespace.h>
  46#include <linux/rhashtable.h>
  47
  48#include <linux/uaccess.h>
  49
  50#include "util.h"
  51
  52struct shmid_kernel /* private to the kernel */
  53{
  54        struct kern_ipc_perm    shm_perm;
  55        struct file             *shm_file;
  56        unsigned long           shm_nattch;
  57        unsigned long           shm_segsz;
  58        time64_t                shm_atim;
  59        time64_t                shm_dtim;
  60        time64_t                shm_ctim;
  61        struct pid              *shm_cprid;
  62        struct pid              *shm_lprid;
  63        struct user_struct      *mlock_user;
  64
  65        /* The task created the shm object.  NULL if the task is dead. */
  66        struct task_struct      *shm_creator;
  67        struct list_head        shm_clist;      /* list by creator */
  68} __randomize_layout;
  69
  70/* shm_mode upper byte flags */
  71#define SHM_DEST        01000   /* segment will be destroyed on last detach */
  72#define SHM_LOCKED      02000   /* segment will not be swapped */
  73
  74struct shm_file_data {
  75        int id;
  76        struct ipc_namespace *ns;
  77        struct file *file;
  78        const struct vm_operations_struct *vm_ops;
  79};
  80
  81#define shm_file_data(file) (*((struct shm_file_data **)&(file)->private_data))
  82
  83static const struct file_operations shm_file_operations;
  84static const struct vm_operations_struct shm_vm_ops;
  85
  86#define shm_ids(ns)     ((ns)->ids[IPC_SHM_IDS])
  87
  88#define shm_unlock(shp)                 \
  89        ipc_unlock(&(shp)->shm_perm)
  90
  91static int newseg(struct ipc_namespace *, struct ipc_params *);
  92static void shm_open(struct vm_area_struct *vma);
  93static void shm_close(struct vm_area_struct *vma);
  94static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp);
  95#ifdef CONFIG_PROC_FS
  96static int sysvipc_shm_proc_show(struct seq_file *s, void *it);
  97#endif
  98
  99void shm_init_ns(struct ipc_namespace *ns)
 100{
 101        ns->shm_ctlmax = SHMMAX;
 102        ns->shm_ctlall = SHMALL;
 103        ns->shm_ctlmni = SHMMNI;
 104        ns->shm_rmid_forced = 0;
 105        ns->shm_tot = 0;
 106        ipc_init_ids(&shm_ids(ns));
 107}
 108
 109/*
 110 * Called with shm_ids.rwsem (writer) and the shp structure locked.
 111 * Only shm_ids.rwsem remains locked on exit.
 112 */
 113static void do_shm_rmid(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
 114{
 115        struct shmid_kernel *shp;
 116
 117        shp = container_of(ipcp, struct shmid_kernel, shm_perm);
 118
 119        if (shp->shm_nattch) {
 120                shp->shm_perm.mode |= SHM_DEST;
 121                /* Do not find it any more */
 122                ipc_set_key_private(&shm_ids(ns), &shp->shm_perm);
 123                shm_unlock(shp);
 124        } else
 125                shm_destroy(ns, shp);
 126}
 127
 128#ifdef CONFIG_IPC_NS
 129void shm_exit_ns(struct ipc_namespace *ns)
 130{
 131        free_ipcs(ns, &shm_ids(ns), do_shm_rmid);
 132        idr_destroy(&ns->ids[IPC_SHM_IDS].ipcs_idr);
 133        rhashtable_destroy(&ns->ids[IPC_SHM_IDS].key_ht);
 134}
 135#endif
 136
 137static int __init ipc_ns_init(void)
 138{
 139        shm_init_ns(&init_ipc_ns);
 140        return 0;
 141}
 142
 143pure_initcall(ipc_ns_init);
 144
 145void __init shm_init(void)
 146{
 147        ipc_init_proc_interface("sysvipc/shm",
 148#if BITS_PER_LONG <= 32
 149                                "       key      shmid perms       size  cpid  lpid nattch   uid   gid  cuid  cgid      atime      dtime      ctime        rss       swap\n",
 150#else
 151                                "       key      shmid perms                  size  cpid  lpid nattch   uid   gid  cuid  cgid      atime      dtime      ctime                   rss                  swap\n",
 152#endif
 153                                IPC_SHM_IDS, sysvipc_shm_proc_show);
 154}
 155
 156static inline struct shmid_kernel *shm_obtain_object(struct ipc_namespace *ns, int id)
 157{
 158        struct kern_ipc_perm *ipcp = ipc_obtain_object_idr(&shm_ids(ns), id);
 159
 160        if (IS_ERR(ipcp))
 161                return ERR_CAST(ipcp);
 162
 163        return container_of(ipcp, struct shmid_kernel, shm_perm);
 164}
 165
 166static inline struct shmid_kernel *shm_obtain_object_check(struct ipc_namespace *ns, int id)
 167{
 168        struct kern_ipc_perm *ipcp = ipc_obtain_object_check(&shm_ids(ns), id);
 169
 170        if (IS_ERR(ipcp))
 171                return ERR_CAST(ipcp);
 172
 173        return container_of(ipcp, struct shmid_kernel, shm_perm);
 174}
 175
 176/*
 177 * shm_lock_(check_) routines are called in the paths where the rwsem
 178 * is not necessarily held.
 179 */
 180static inline struct shmid_kernel *shm_lock(struct ipc_namespace *ns, int id)
 181{
 182        struct kern_ipc_perm *ipcp;
 183
 184        rcu_read_lock();
 185        ipcp = ipc_obtain_object_idr(&shm_ids(ns), id);
 186        if (IS_ERR(ipcp))
 187                goto err;
 188
 189        ipc_lock_object(ipcp);
 190        /*
 191         * ipc_rmid() may have already freed the ID while ipc_lock_object()
 192         * was spinning: here verify that the structure is still valid.
 193         * Upon races with RMID, return -EIDRM, thus indicating that
 194         * the ID points to a removed identifier.
 195         */
 196        if (ipc_valid_object(ipcp)) {
 197                /* return a locked ipc object upon success */
 198                return container_of(ipcp, struct shmid_kernel, shm_perm);
 199        }
 200
 201        ipc_unlock_object(ipcp);
 202        ipcp = ERR_PTR(-EIDRM);
 203err:
 204        rcu_read_unlock();
 205        /*
 206         * Callers of shm_lock() must validate the status of the returned ipc
 207         * object pointer and error out as appropriate.
 208         */
 209        return ERR_CAST(ipcp);
 210}
 211
 212static inline void shm_lock_by_ptr(struct shmid_kernel *ipcp)
 213{
 214        rcu_read_lock();
 215        ipc_lock_object(&ipcp->shm_perm);
 216}
 217
 218static void shm_rcu_free(struct rcu_head *head)
 219{
 220        struct kern_ipc_perm *ptr = container_of(head, struct kern_ipc_perm,
 221                                                        rcu);
 222        struct shmid_kernel *shp = container_of(ptr, struct shmid_kernel,
 223                                                        shm_perm);
 224        security_shm_free(&shp->shm_perm);
 225        kvfree(shp);
 226}
 227
 228static inline void shm_rmid(struct ipc_namespace *ns, struct shmid_kernel *s)
 229{
 230        list_del(&s->shm_clist);
 231        ipc_rmid(&shm_ids(ns), &s->shm_perm);
 232}
 233
 234
 235static int __shm_open(struct vm_area_struct *vma)
 236{
 237        struct file *file = vma->vm_file;
 238        struct shm_file_data *sfd = shm_file_data(file);
 239        struct shmid_kernel *shp;
 240
 241        shp = shm_lock(sfd->ns, sfd->id);
 242
 243        if (IS_ERR(shp))
 244                return PTR_ERR(shp);
 245
 246        if (shp->shm_file != sfd->file) {
 247                /* ID was reused */
 248                shm_unlock(shp);
 249                return -EINVAL;
 250        }
 251
 252        shp->shm_atim = ktime_get_real_seconds();
 253        ipc_update_pid(&shp->shm_lprid, task_tgid(current));
 254        shp->shm_nattch++;
 255        shm_unlock(shp);
 256        return 0;
 257}
 258
 259/* This is called by fork, once for every shm attach. */
 260static void shm_open(struct vm_area_struct *vma)
 261{
 262        int err = __shm_open(vma);
 263        /*
 264         * We raced in the idr lookup or with shm_destroy().
 265         * Either way, the ID is busted.
 266         */
 267        WARN_ON_ONCE(err);
 268}
 269
 270/*
 271 * shm_destroy - free the struct shmid_kernel
 272 *
 273 * @ns: namespace
 274 * @shp: struct to free
 275 *
 276 * It has to be called with shp and shm_ids.rwsem (writer) locked,
 277 * but returns with shp unlocked and freed.
 278 */
 279static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
 280{
 281        struct file *shm_file;
 282
 283        shm_file = shp->shm_file;
 284        shp->shm_file = NULL;
 285        ns->shm_tot -= (shp->shm_segsz + PAGE_SIZE - 1) >> PAGE_SHIFT;
 286        shm_rmid(ns, shp);
 287        shm_unlock(shp);
 288        if (!is_file_hugepages(shm_file))
 289                shmem_lock(shm_file, 0, shp->mlock_user);
 290        else if (shp->mlock_user)
 291                user_shm_unlock(i_size_read(file_inode(shm_file)),
 292                                shp->mlock_user);
 293        fput(shm_file);
 294        ipc_update_pid(&shp->shm_cprid, NULL);
 295        ipc_update_pid(&shp->shm_lprid, NULL);
 296        ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);
 297}
 298
 299/*
 300 * shm_may_destroy - identifies whether shm segment should be destroyed now
 301 *
 302 * Returns true if and only if there are no active users of the segment and
 303 * one of the following is true:
 304 *
 305 * 1) shmctl(id, IPC_RMID, NULL) was called for this shp
 306 *
 307 * 2) sysctl kernel.shm_rmid_forced is set to 1.
 308 */
 309static bool shm_may_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
 310{
 311        return (shp->shm_nattch == 0) &&
 312               (ns->shm_rmid_forced ||
 313                (shp->shm_perm.mode & SHM_DEST));
 314}
 315
 316/*
 317 * remove the attach descriptor vma.
 318 * free memory for segment if it is marked destroyed.
 319 * The descriptor has already been removed from the current->mm->mmap list
 320 * and will later be kfree()d.
 321 */
 322static void shm_close(struct vm_area_struct *vma)
 323{
 324        struct file *file = vma->vm_file;
 325        struct shm_file_data *sfd = shm_file_data(file);
 326        struct shmid_kernel *shp;
 327        struct ipc_namespace *ns = sfd->ns;
 328
 329        down_write(&shm_ids(ns).rwsem);
 330        /* remove from the list of attaches of the shm segment */
 331        shp = shm_lock(ns, sfd->id);
 332
 333        /*
 334         * We raced in the idr lookup or with shm_destroy().
 335         * Either way, the ID is busted.
 336         */
 337        if (WARN_ON_ONCE(IS_ERR(shp)))
 338                goto done; /* no-op */
 339
 340        ipc_update_pid(&shp->shm_lprid, task_tgid(current));
 341        shp->shm_dtim = ktime_get_real_seconds();
 342        shp->shm_nattch--;
 343        if (shm_may_destroy(ns, shp))
 344                shm_destroy(ns, shp);
 345        else
 346                shm_unlock(shp);
 347done:
 348        up_write(&shm_ids(ns).rwsem);
 349}
 350
 351/* Called with ns->shm_ids(ns).rwsem locked */
 352static int shm_try_destroy_orphaned(int id, void *p, void *data)
 353{
 354        struct ipc_namespace *ns = data;
 355        struct kern_ipc_perm *ipcp = p;
 356        struct shmid_kernel *shp = container_of(ipcp, struct shmid_kernel, shm_perm);
 357
 358        /*
 359         * We want to destroy segments without users and with already
 360         * exit'ed originating process.
 361         *
 362         * As shp->* are changed under rwsem, it's safe to skip shp locking.
 363         */
 364        if (shp->shm_creator != NULL)
 365                return 0;
 366
 367        if (shm_may_destroy(ns, shp)) {
 368                shm_lock_by_ptr(shp);
 369                shm_destroy(ns, shp);
 370        }
 371        return 0;
 372}
 373
 374void shm_destroy_orphaned(struct ipc_namespace *ns)
 375{
 376        down_write(&shm_ids(ns).rwsem);
 377        if (shm_ids(ns).in_use)
 378                idr_for_each(&shm_ids(ns).ipcs_idr, &shm_try_destroy_orphaned, ns);
 379        up_write(&shm_ids(ns).rwsem);
 380}
 381
 382/* Locking assumes this will only be called with task == current */
 383void exit_shm(struct task_struct *task)
 384{
 385        struct ipc_namespace *ns = task->nsproxy->ipc_ns;
 386        struct shmid_kernel *shp, *n;
 387
 388        if (list_empty(&task->sysvshm.shm_clist))
 389                return;
 390
 391        /*
 392         * If kernel.shm_rmid_forced is not set then only keep track of
 393         * which shmids are orphaned, so that a later set of the sysctl
 394         * can clean them up.
 395         */
 396        if (!ns->shm_rmid_forced) {
 397                down_read(&shm_ids(ns).rwsem);
 398                list_for_each_entry(shp, &task->sysvshm.shm_clist, shm_clist)
 399                        shp->shm_creator = NULL;
 400                /*
 401                 * Only under read lock but we are only called on current
 402                 * so no entry on the list will be shared.
 403                 */
 404                list_del(&task->sysvshm.shm_clist);
 405                up_read(&shm_ids(ns).rwsem);
 406                return;
 407        }
 408
 409        /*
 410         * Destroy all already created segments, that were not yet mapped,
 411         * and mark any mapped as orphan to cover the sysctl toggling.
 412         * Destroy is skipped if shm_may_destroy() returns false.
 413         */
 414        down_write(&shm_ids(ns).rwsem);
 415        list_for_each_entry_safe(shp, n, &task->sysvshm.shm_clist, shm_clist) {
 416                shp->shm_creator = NULL;
 417
 418                if (shm_may_destroy(ns, shp)) {
 419                        shm_lock_by_ptr(shp);
 420                        shm_destroy(ns, shp);
 421                }
 422        }
 423
 424        /* Remove the list head from any segments still attached. */
 425        list_del(&task->sysvshm.shm_clist);
 426        up_write(&shm_ids(ns).rwsem);
 427}
 428
 429static vm_fault_t shm_fault(struct vm_fault *vmf)
 430{
 431        struct file *file = vmf->vma->vm_file;
 432        struct shm_file_data *sfd = shm_file_data(file);
 433
 434        return sfd->vm_ops->fault(vmf);
 435}
 436
 437static int shm_may_split(struct vm_area_struct *vma, unsigned long addr)
 438{
 439        struct file *file = vma->vm_file;
 440        struct shm_file_data *sfd = shm_file_data(file);
 441
 442        if (sfd->vm_ops->may_split)
 443                return sfd->vm_ops->may_split(vma, addr);
 444
 445        return 0;
 446}
 447
 448static unsigned long shm_pagesize(struct vm_area_struct *vma)
 449{
 450        struct file *file = vma->vm_file;
 451        struct shm_file_data *sfd = shm_file_data(file);
 452
 453        if (sfd->vm_ops->pagesize)
 454                return sfd->vm_ops->pagesize(vma);
 455
 456        return PAGE_SIZE;
 457}
 458
 459#ifdef CONFIG_NUMA
 460static int shm_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
 461{
 462        struct file *file = vma->vm_file;
 463        struct shm_file_data *sfd = shm_file_data(file);
 464        int err = 0;
 465
 466        if (sfd->vm_ops->set_policy)
 467                err = sfd->vm_ops->set_policy(vma, new);
 468        return err;
 469}
 470
 471static struct mempolicy *shm_get_policy(struct vm_area_struct *vma,
 472                                        unsigned long addr)
 473{
 474        struct file *file = vma->vm_file;
 475        struct shm_file_data *sfd = shm_file_data(file);
 476        struct mempolicy *pol = NULL;
 477
 478        if (sfd->vm_ops->get_policy)
 479                pol = sfd->vm_ops->get_policy(vma, addr);
 480        else if (vma->vm_policy)
 481                pol = vma->vm_policy;
 482
 483        return pol;
 484}
 485#endif
 486
 487static int shm_mmap(struct file *file, struct vm_area_struct *vma)
 488{
 489        struct shm_file_data *sfd = shm_file_data(file);
 490        int ret;
 491
 492        /*
 493         * In case of remap_file_pages() emulation, the file can represent an
 494         * IPC ID that was removed, and possibly even reused by another shm
 495         * segment already.  Propagate this case as an error to caller.
 496         */
 497        ret = __shm_open(vma);
 498        if (ret)
 499                return ret;
 500
 501        ret = call_mmap(sfd->file, vma);
 502        if (ret) {
 503                shm_close(vma);
 504                return ret;
 505        }
 506        sfd->vm_ops = vma->vm_ops;
 507#ifdef CONFIG_MMU
 508        WARN_ON(!sfd->vm_ops->fault);
 509#endif
 510        vma->vm_ops = &shm_vm_ops;
 511        return 0;
 512}
 513
 514static int shm_release(struct inode *ino, struct file *file)
 515{
 516        struct shm_file_data *sfd = shm_file_data(file);
 517
 518        put_ipc_ns(sfd->ns);
 519        fput(sfd->file);
 520        shm_file_data(file) = NULL;
 521        kfree(sfd);
 522        return 0;
 523}
 524
 525static int shm_fsync(struct file *file, loff_t start, loff_t end, int datasync)
 526{
 527        struct shm_file_data *sfd = shm_file_data(file);
 528
 529        if (!sfd->file->f_op->fsync)
 530                return -EINVAL;
 531        return sfd->file->f_op->fsync(sfd->file, start, end, datasync);
 532}
 533
 534static long shm_fallocate(struct file *file, int mode, loff_t offset,
 535                          loff_t len)
 536{
 537        struct shm_file_data *sfd = shm_file_data(file);
 538
 539        if (!sfd->file->f_op->fallocate)
 540                return -EOPNOTSUPP;
 541        return sfd->file->f_op->fallocate(file, mode, offset, len);
 542}
 543
 544static unsigned long shm_get_unmapped_area(struct file *file,
 545        unsigned long addr, unsigned long len, unsigned long pgoff,
 546        unsigned long flags)
 547{
 548        struct shm_file_data *sfd = shm_file_data(file);
 549
 550        return sfd->file->f_op->get_unmapped_area(sfd->file, addr, len,
 551                                                pgoff, flags);
 552}
 553
 554static const struct file_operations shm_file_operations = {
 555        .mmap           = shm_mmap,
 556        .fsync          = shm_fsync,
 557        .release        = shm_release,
 558        .get_unmapped_area      = shm_get_unmapped_area,
 559        .llseek         = noop_llseek,
 560        .fallocate      = shm_fallocate,
 561};
 562
 563/*
 564 * shm_file_operations_huge is now identical to shm_file_operations,
 565 * but we keep it distinct for the sake of is_file_shm_hugepages().
 566 */
 567static const struct file_operations shm_file_operations_huge = {
 568        .mmap           = shm_mmap,
 569        .fsync          = shm_fsync,
 570        .release        = shm_release,
 571        .get_unmapped_area      = shm_get_unmapped_area,
 572        .llseek         = noop_llseek,
 573        .fallocate      = shm_fallocate,
 574};
 575
 576bool is_file_shm_hugepages(struct file *file)
 577{
 578        return file->f_op == &shm_file_operations_huge;
 579}
 580
 581static const struct vm_operations_struct shm_vm_ops = {
 582        .open   = shm_open,     /* callback for a new vm-area open */
 583        .close  = shm_close,    /* callback for when the vm-area is released */
 584        .fault  = shm_fault,
 585        .may_split = shm_may_split,
 586        .pagesize = shm_pagesize,
 587#if defined(CONFIG_NUMA)
 588        .set_policy = shm_set_policy,
 589        .get_policy = shm_get_policy,
 590#endif
 591};
 592
 593/**
 594 * newseg - Create a new shared memory segment
 595 * @ns: namespace
 596 * @params: ptr to the structure that contains key, size and shmflg
 597 *
 598 * Called with shm_ids.rwsem held as a writer.
 599 */
 600static int newseg(struct ipc_namespace *ns, struct ipc_params *params)
 601{
 602        key_t key = params->key;
 603        int shmflg = params->flg;
 604        size_t size = params->u.size;
 605        int error;
 606        struct shmid_kernel *shp;
 607        size_t numpages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
 608        struct file *file;
 609        char name[13];
 610        vm_flags_t acctflag = 0;
 611
 612        if (size < SHMMIN || size > ns->shm_ctlmax)
 613                return -EINVAL;
 614
 615        if (numpages << PAGE_SHIFT < size)
 616                return -ENOSPC;
 617
 618        if (ns->shm_tot + numpages < ns->shm_tot ||
 619                        ns->shm_tot + numpages > ns->shm_ctlall)
 620                return -ENOSPC;
 621
 622        shp = kvmalloc(sizeof(*shp), GFP_KERNEL);
 623        if (unlikely(!shp))
 624                return -ENOMEM;
 625
 626        shp->shm_perm.key = key;
 627        shp->shm_perm.mode = (shmflg & S_IRWXUGO);
 628        shp->mlock_user = NULL;
 629
 630        shp->shm_perm.security = NULL;
 631        error = security_shm_alloc(&shp->shm_perm);
 632        if (error) {
 633                kvfree(shp);
 634                return error;
 635        }
 636
 637        sprintf(name, "SYSV%08x", key);
 638        if (shmflg & SHM_HUGETLB) {
 639                struct hstate *hs;
 640                size_t hugesize;
 641
 642                hs = hstate_sizelog((shmflg >> SHM_HUGE_SHIFT) & SHM_HUGE_MASK);
 643                if (!hs) {
 644                        error = -EINVAL;
 645                        goto no_file;
 646                }
 647                hugesize = ALIGN(size, huge_page_size(hs));
 648
 649                /* hugetlb_file_setup applies strict accounting */
 650                if (shmflg & SHM_NORESERVE)
 651                        acctflag = VM_NORESERVE;
 652                file = hugetlb_file_setup(name, hugesize, acctflag,
 653                                  &shp->mlock_user, HUGETLB_SHMFS_INODE,
 654                                (shmflg >> SHM_HUGE_SHIFT) & SHM_HUGE_MASK);
 655        } else {
 656                /*
 657                 * Do not allow no accounting for OVERCOMMIT_NEVER, even
 658                 * if it's asked for.
 659                 */
 660                if  ((shmflg & SHM_NORESERVE) &&
 661                                sysctl_overcommit_memory != OVERCOMMIT_NEVER)
 662                        acctflag = VM_NORESERVE;
 663                file = shmem_kernel_file_setup(name, size, acctflag);
 664        }
 665        error = PTR_ERR(file);
 666        if (IS_ERR(file))
 667                goto no_file;
 668
 669        shp->shm_cprid = get_pid(task_tgid(current));
 670        shp->shm_lprid = NULL;
 671        shp->shm_atim = shp->shm_dtim = 0;
 672        shp->shm_ctim = ktime_get_real_seconds();
 673        shp->shm_segsz = size;
 674        shp->shm_nattch = 0;
 675        shp->shm_file = file;
 676        shp->shm_creator = current;
 677
 678        /* ipc_addid() locks shp upon success. */
 679        error = ipc_addid(&shm_ids(ns), &shp->shm_perm, ns->shm_ctlmni);
 680        if (error < 0)
 681                goto no_id;
 682
 683        list_add(&shp->shm_clist, &current->sysvshm.shm_clist);
 684
 685        /*
 686         * shmid gets reported as "inode#" in /proc/pid/maps.
 687         * proc-ps tools use this. Changing this will break them.
 688         */
 689        file_inode(file)->i_ino = shp->shm_perm.id;
 690
 691        ns->shm_tot += numpages;
 692        error = shp->shm_perm.id;
 693
 694        ipc_unlock_object(&shp->shm_perm);
 695        rcu_read_unlock();
 696        return error;
 697
 698no_id:
 699        ipc_update_pid(&shp->shm_cprid, NULL);
 700        ipc_update_pid(&shp->shm_lprid, NULL);
 701        if (is_file_hugepages(file) && shp->mlock_user)
 702                user_shm_unlock(size, shp->mlock_user);
 703        fput(file);
 704        ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);
 705        return error;
 706no_file:
 707        call_rcu(&shp->shm_perm.rcu, shm_rcu_free);
 708        return error;
 709}
 710
 711/*
 712 * Called with shm_ids.rwsem and ipcp locked.
 713 */
 714static int shm_more_checks(struct kern_ipc_perm *ipcp, struct ipc_params *params)
 715{
 716        struct shmid_kernel *shp;
 717
 718        shp = container_of(ipcp, struct shmid_kernel, shm_perm);
 719        if (shp->shm_segsz < params->u.size)
 720                return -EINVAL;
 721
 722        return 0;
 723}
 724
 725long ksys_shmget(key_t key, size_t size, int shmflg)
 726{
 727        struct ipc_namespace *ns;
 728        static const struct ipc_ops shm_ops = {
 729                .getnew = newseg,
 730                .associate = security_shm_associate,
 731                .more_checks = shm_more_checks,
 732        };
 733        struct ipc_params shm_params;
 734
 735        ns = current->nsproxy->ipc_ns;
 736
 737        shm_params.key = key;
 738        shm_params.flg = shmflg;
 739        shm_params.u.size = size;
 740
 741        return ipcget(ns, &shm_ids(ns), &shm_ops, &shm_params);
 742}
 743
 744SYSCALL_DEFINE3(shmget, key_t, key, size_t, size, int, shmflg)
 745{
 746        return ksys_shmget(key, size, shmflg);
 747}
 748
 749static inline unsigned long copy_shmid_to_user(void __user *buf, struct shmid64_ds *in, int version)
 750{
 751        switch (version) {
 752        case IPC_64:
 753                return copy_to_user(buf, in, sizeof(*in));
 754        case IPC_OLD:
 755            {
 756                struct shmid_ds out;
 757
 758                memset(&out, 0, sizeof(out));
 759                ipc64_perm_to_ipc_perm(&in->shm_perm, &out.shm_perm);
 760                out.shm_segsz   = in->shm_segsz;
 761                out.shm_atime   = in->shm_atime;
 762                out.shm_dtime   = in->shm_dtime;
 763                out.shm_ctime   = in->shm_ctime;
 764                out.shm_cpid    = in->shm_cpid;
 765                out.shm_lpid    = in->shm_lpid;
 766                out.shm_nattch  = in->shm_nattch;
 767
 768                return copy_to_user(buf, &out, sizeof(out));
 769            }
 770        default:
 771                return -EINVAL;
 772        }
 773}
 774
 775static inline unsigned long
 776copy_shmid_from_user(struct shmid64_ds *out, void __user *buf, int version)
 777{
 778        switch (version) {
 779        case IPC_64:
 780                if (copy_from_user(out, buf, sizeof(*out)))
 781                        return -EFAULT;
 782                return 0;
 783        case IPC_OLD:
 784            {
 785                struct shmid_ds tbuf_old;
 786
 787                if (copy_from_user(&tbuf_old, buf, sizeof(tbuf_old)))
 788                        return -EFAULT;
 789
 790                out->shm_perm.uid       = tbuf_old.shm_perm.uid;
 791                out->shm_perm.gid       = tbuf_old.shm_perm.gid;
 792                out->shm_perm.mode      = tbuf_old.shm_perm.mode;
 793
 794                return 0;
 795            }
 796        default:
 797                return -EINVAL;
 798        }
 799}
 800
 801static inline unsigned long copy_shminfo_to_user(void __user *buf, struct shminfo64 *in, int version)
 802{
 803        switch (version) {
 804        case IPC_64:
 805                return copy_to_user(buf, in, sizeof(*in));
 806        case IPC_OLD:
 807            {
 808                struct shminfo out;
 809
 810                if (in->shmmax > INT_MAX)
 811                        out.shmmax = INT_MAX;
 812                else
 813                        out.shmmax = (int)in->shmmax;
 814
 815                out.shmmin      = in->shmmin;
 816                out.shmmni      = in->shmmni;
 817                out.shmseg      = in->shmseg;
 818                out.shmall      = in->shmall;
 819
 820                return copy_to_user(buf, &out, sizeof(out));
 821            }
 822        default:
 823                return -EINVAL;
 824        }
 825}
 826
 827/*
 828 * Calculate and add used RSS and swap pages of a shm.
 829 * Called with shm_ids.rwsem held as a reader
 830 */
 831static void shm_add_rss_swap(struct shmid_kernel *shp,
 832        unsigned long *rss_add, unsigned long *swp_add)
 833{
 834        struct inode *inode;
 835
 836        inode = file_inode(shp->shm_file);
 837
 838        if (is_file_hugepages(shp->shm_file)) {
 839                struct address_space *mapping = inode->i_mapping;
 840                struct hstate *h = hstate_file(shp->shm_file);
 841                *rss_add += pages_per_huge_page(h) * mapping->nrpages;
 842        } else {
 843#ifdef CONFIG_SHMEM
 844                struct shmem_inode_info *info = SHMEM_I(inode);
 845
 846                spin_lock_irq(&info->lock);
 847                *rss_add += inode->i_mapping->nrpages;
 848                *swp_add += info->swapped;
 849                spin_unlock_irq(&info->lock);
 850#else
 851                *rss_add += inode->i_mapping->nrpages;
 852#endif
 853        }
 854}
 855
 856/*
 857 * Called with shm_ids.rwsem held as a reader
 858 */
 859static void shm_get_stat(struct ipc_namespace *ns, unsigned long *rss,
 860                unsigned long *swp)
 861{
 862        int next_id;
 863        int total, in_use;
 864
 865        *rss = 0;
 866        *swp = 0;
 867
 868        in_use = shm_ids(ns).in_use;
 869
 870        for (total = 0, next_id = 0; total < in_use; next_id++) {
 871                struct kern_ipc_perm *ipc;
 872                struct shmid_kernel *shp;
 873
 874                ipc = idr_find(&shm_ids(ns).ipcs_idr, next_id);
 875                if (ipc == NULL)
 876                        continue;
 877                shp = container_of(ipc, struct shmid_kernel, shm_perm);
 878
 879                shm_add_rss_swap(shp, rss, swp);
 880
 881                total++;
 882        }
 883}
 884
 885/*
 886 * This function handles some shmctl commands which require the rwsem
 887 * to be held in write mode.
 888 * NOTE: no locks must be held, the rwsem is taken inside this function.
 889 */
 890static int shmctl_down(struct ipc_namespace *ns, int shmid, int cmd,
 891                       struct shmid64_ds *shmid64)
 892{
 893        struct kern_ipc_perm *ipcp;
 894        struct shmid_kernel *shp;
 895        int err;
 896
 897        down_write(&shm_ids(ns).rwsem);
 898        rcu_read_lock();
 899
 900        ipcp = ipcctl_obtain_check(ns, &shm_ids(ns), shmid, cmd,
 901                                      &shmid64->shm_perm, 0);
 902        if (IS_ERR(ipcp)) {
 903                err = PTR_ERR(ipcp);
 904                goto out_unlock1;
 905        }
 906
 907        shp = container_of(ipcp, struct shmid_kernel, shm_perm);
 908
 909        err = security_shm_shmctl(&shp->shm_perm, cmd);
 910        if (err)
 911                goto out_unlock1;
 912
 913        switch (cmd) {
 914        case IPC_RMID:
 915                ipc_lock_object(&shp->shm_perm);
 916                /* do_shm_rmid unlocks the ipc object and rcu */
 917                do_shm_rmid(ns, ipcp);
 918                goto out_up;
 919        case IPC_SET:
 920                ipc_lock_object(&shp->shm_perm);
 921                err = ipc_update_perm(&shmid64->shm_perm, ipcp);
 922                if (err)
 923                        goto out_unlock0;
 924                shp->shm_ctim = ktime_get_real_seconds();
 925                break;
 926        default:
 927                err = -EINVAL;
 928                goto out_unlock1;
 929        }
 930
 931out_unlock0:
 932        ipc_unlock_object(&shp->shm_perm);
 933out_unlock1:
 934        rcu_read_unlock();
 935out_up:
 936        up_write(&shm_ids(ns).rwsem);
 937        return err;
 938}
 939
 940static int shmctl_ipc_info(struct ipc_namespace *ns,
 941                           struct shminfo64 *shminfo)
 942{
 943        int err = security_shm_shmctl(NULL, IPC_INFO);
 944        if (!err) {
 945                memset(shminfo, 0, sizeof(*shminfo));
 946                shminfo->shmmni = shminfo->shmseg = ns->shm_ctlmni;
 947                shminfo->shmmax = ns->shm_ctlmax;
 948                shminfo->shmall = ns->shm_ctlall;
 949                shminfo->shmmin = SHMMIN;
 950                down_read(&shm_ids(ns).rwsem);
 951                err = ipc_get_maxidx(&shm_ids(ns));
 952                up_read(&shm_ids(ns).rwsem);
 953                if (err < 0)
 954                        err = 0;
 955        }
 956        return err;
 957}
 958
 959static int shmctl_shm_info(struct ipc_namespace *ns,
 960                           struct shm_info *shm_info)
 961{
 962        int err = security_shm_shmctl(NULL, SHM_INFO);
 963        if (!err) {
 964                memset(shm_info, 0, sizeof(*shm_info));
 965                down_read(&shm_ids(ns).rwsem);
 966                shm_info->used_ids = shm_ids(ns).in_use;
 967                shm_get_stat(ns, &shm_info->shm_rss, &shm_info->shm_swp);
 968                shm_info->shm_tot = ns->shm_tot;
 969                shm_info->swap_attempts = 0;
 970                shm_info->swap_successes = 0;
 971                err = ipc_get_maxidx(&shm_ids(ns));
 972                up_read(&shm_ids(ns).rwsem);
 973                if (err < 0)
 974                        err = 0;
 975        }
 976        return err;
 977}
 978
 979static int shmctl_stat(struct ipc_namespace *ns, int shmid,
 980                        int cmd, struct shmid64_ds *tbuf)
 981{
 982        struct shmid_kernel *shp;
 983        int err;
 984
 985        memset(tbuf, 0, sizeof(*tbuf));
 986
 987        rcu_read_lock();
 988        if (cmd == SHM_STAT || cmd == SHM_STAT_ANY) {
 989                shp = shm_obtain_object(ns, shmid);
 990                if (IS_ERR(shp)) {
 991                        err = PTR_ERR(shp);
 992                        goto out_unlock;
 993                }
 994        } else { /* IPC_STAT */
 995                shp = shm_obtain_object_check(ns, shmid);
 996                if (IS_ERR(shp)) {
 997                        err = PTR_ERR(shp);
 998                        goto out_unlock;
 999                }
1000        }
1001
1002        /*
1003         * Semantically SHM_STAT_ANY ought to be identical to
1004         * that functionality provided by the /proc/sysvipc/
1005         * interface. As such, only audit these calls and
1006         * do not do traditional S_IRUGO permission checks on
1007         * the ipc object.
1008         */
1009        if (cmd == SHM_STAT_ANY)
1010                audit_ipc_obj(&shp->shm_perm);
1011        else {
1012                err = -EACCES;
1013                if (ipcperms(ns, &shp->shm_perm, S_IRUGO))
1014                        goto out_unlock;
1015        }
1016
1017        err = security_shm_shmctl(&shp->shm_perm, cmd);
1018        if (err)
1019                goto out_unlock;
1020
1021        ipc_lock_object(&shp->shm_perm);
1022
1023        if (!ipc_valid_object(&shp->shm_perm)) {
1024                ipc_unlock_object(&shp->shm_perm);
1025                err = -EIDRM;
1026                goto out_unlock;
1027        }
1028
1029        kernel_to_ipc64_perm(&shp->shm_perm, &tbuf->shm_perm);
1030        tbuf->shm_segsz = shp->shm_segsz;
1031        tbuf->shm_atime = shp->shm_atim;
1032        tbuf->shm_dtime = shp->shm_dtim;
1033        tbuf->shm_ctime = shp->shm_ctim;
1034#ifndef CONFIG_64BIT
1035        tbuf->shm_atime_high = shp->shm_atim >> 32;
1036        tbuf->shm_dtime_high = shp->shm_dtim >> 32;
1037        tbuf->shm_ctime_high = shp->shm_ctim >> 32;
1038#endif
1039        tbuf->shm_cpid  = pid_vnr(shp->shm_cprid);
1040        tbuf->shm_lpid  = pid_vnr(shp->shm_lprid);
1041        tbuf->shm_nattch = shp->shm_nattch;
1042
1043        if (cmd == IPC_STAT) {
1044                /*
1045                 * As defined in SUS:
1046                 * Return 0 on success
1047                 */
1048                err = 0;
1049        } else {
1050                /*
1051                 * SHM_STAT and SHM_STAT_ANY (both Linux specific)
1052                 * Return the full id, including the sequence number
1053                 */
1054                err = shp->shm_perm.id;
1055        }
1056
1057        ipc_unlock_object(&shp->shm_perm);
1058out_unlock:
1059        rcu_read_unlock();
1060        return err;
1061}
1062
1063static int shmctl_do_lock(struct ipc_namespace *ns, int shmid, int cmd)
1064{
1065        struct shmid_kernel *shp;
1066        struct file *shm_file;
1067        int err;
1068
1069        rcu_read_lock();
1070        shp = shm_obtain_object_check(ns, shmid);
1071        if (IS_ERR(shp)) {
1072                err = PTR_ERR(shp);
1073                goto out_unlock1;
1074        }
1075
1076        audit_ipc_obj(&(shp->shm_perm));
1077        err = security_shm_shmctl(&shp->shm_perm, cmd);
1078        if (err)
1079                goto out_unlock1;
1080
1081        ipc_lock_object(&shp->shm_perm);
1082
1083        /* check if shm_destroy() is tearing down shp */
1084        if (!ipc_valid_object(&shp->shm_perm)) {
1085                err = -EIDRM;
1086                goto out_unlock0;
1087        }
1088
1089        if (!ns_capable(ns->user_ns, CAP_IPC_LOCK)) {
1090                kuid_t euid = current_euid();
1091
1092                if (!uid_eq(euid, shp->shm_perm.uid) &&
1093                    !uid_eq(euid, shp->shm_perm.cuid)) {
1094                        err = -EPERM;
1095                        goto out_unlock0;
1096                }
1097                if (cmd == SHM_LOCK && !rlimit(RLIMIT_MEMLOCK)) {
1098                        err = -EPERM;
1099                        goto out_unlock0;
1100                }
1101        }
1102
1103        shm_file = shp->shm_file;
1104        if (is_file_hugepages(shm_file))
1105                goto out_unlock0;
1106
1107        if (cmd == SHM_LOCK) {
1108                struct user_struct *user = current_user();
1109
1110                err = shmem_lock(shm_file, 1, user);
1111                if (!err && !(shp->shm_perm.mode & SHM_LOCKED)) {
1112                        shp->shm_perm.mode |= SHM_LOCKED;
1113                        shp->mlock_user = user;
1114                }
1115                goto out_unlock0;
1116        }
1117
1118        /* SHM_UNLOCK */
1119        if (!(shp->shm_perm.mode & SHM_LOCKED))
1120                goto out_unlock0;
1121        shmem_lock(shm_file, 0, shp->mlock_user);
1122        shp->shm_perm.mode &= ~SHM_LOCKED;
1123        shp->mlock_user = NULL;
1124        get_file(shm_file);
1125        ipc_unlock_object(&shp->shm_perm);
1126        rcu_read_unlock();
1127        shmem_unlock_mapping(shm_file->f_mapping);
1128
1129        fput(shm_file);
1130        return err;
1131
1132out_unlock0:
1133        ipc_unlock_object(&shp->shm_perm);
1134out_unlock1:
1135        rcu_read_unlock();
1136        return err;
1137}
1138
1139static long ksys_shmctl(int shmid, int cmd, struct shmid_ds __user *buf, int version)
1140{
1141        int err;
1142        struct ipc_namespace *ns;
1143        struct shmid64_ds sem64;
1144
1145        if (cmd < 0 || shmid < 0)
1146                return -EINVAL;
1147
1148        ns = current->nsproxy->ipc_ns;
1149
1150        switch (cmd) {
1151        case IPC_INFO: {
1152                struct shminfo64 shminfo;
1153                err = shmctl_ipc_info(ns, &shminfo);
1154                if (err < 0)
1155                        return err;
1156                if (copy_shminfo_to_user(buf, &shminfo, version))
1157                        err = -EFAULT;
1158                return err;
1159        }
1160        case SHM_INFO: {
1161                struct shm_info shm_info;
1162                err = shmctl_shm_info(ns, &shm_info);
1163                if (err < 0)
1164                        return err;
1165                if (copy_to_user(buf, &shm_info, sizeof(shm_info)))
1166                        err = -EFAULT;
1167                return err;
1168        }
1169        case SHM_STAT:
1170        case SHM_STAT_ANY:
1171        case IPC_STAT: {
1172                err = shmctl_stat(ns, shmid, cmd, &sem64);
1173                if (err < 0)
1174                        return err;
1175                if (copy_shmid_to_user(buf, &sem64, version))
1176                        err = -EFAULT;
1177                return err;
1178        }
1179        case IPC_SET:
1180                if (copy_shmid_from_user(&sem64, buf, version))
1181                        return -EFAULT;
1182                fallthrough;
1183        case IPC_RMID:
1184                return shmctl_down(ns, shmid, cmd, &sem64);
1185        case SHM_LOCK:
1186        case SHM_UNLOCK:
1187                return shmctl_do_lock(ns, shmid, cmd);
1188        default:
1189                return -EINVAL;
1190        }
1191}
1192
1193SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
1194{
1195        return ksys_shmctl(shmid, cmd, buf, IPC_64);
1196}
1197
1198#ifdef CONFIG_ARCH_WANT_IPC_PARSE_VERSION
1199long ksys_old_shmctl(int shmid, int cmd, struct shmid_ds __user *buf)
1200{
1201        int version = ipc_parse_version(&cmd);
1202
1203        return ksys_shmctl(shmid, cmd, buf, version);
1204}
1205
1206SYSCALL_DEFINE3(old_shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
1207{
1208        return ksys_old_shmctl(shmid, cmd, buf);
1209}
1210#endif
1211
1212#ifdef CONFIG_COMPAT
1213
1214struct compat_shmid_ds {
1215        struct compat_ipc_perm shm_perm;
1216        int shm_segsz;
1217        old_time32_t shm_atime;
1218        old_time32_t shm_dtime;
1219        old_time32_t shm_ctime;
1220        compat_ipc_pid_t shm_cpid;
1221        compat_ipc_pid_t shm_lpid;
1222        unsigned short shm_nattch;
1223        unsigned short shm_unused;
1224        compat_uptr_t shm_unused2;
1225        compat_uptr_t shm_unused3;
1226};
1227
1228struct compat_shminfo64 {
1229        compat_ulong_t shmmax;
1230        compat_ulong_t shmmin;
1231        compat_ulong_t shmmni;
1232        compat_ulong_t shmseg;
1233        compat_ulong_t shmall;
1234        compat_ulong_t __unused1;
1235        compat_ulong_t __unused2;
1236        compat_ulong_t __unused3;
1237        compat_ulong_t __unused4;
1238};
1239
1240struct compat_shm_info {
1241        compat_int_t used_ids;
1242        compat_ulong_t shm_tot, shm_rss, shm_swp;
1243        compat_ulong_t swap_attempts, swap_successes;
1244};
1245
1246static int copy_compat_shminfo_to_user(void __user *buf, struct shminfo64 *in,
1247                                        int version)
1248{
1249        if (in->shmmax > INT_MAX)
1250                in->shmmax = INT_MAX;
1251        if (version == IPC_64) {
1252                struct compat_shminfo64 info;
1253                memset(&info, 0, sizeof(info));
1254                info.shmmax = in->shmmax;
1255                info.shmmin = in->shmmin;
1256                info.shmmni = in->shmmni;
1257                info.shmseg = in->shmseg;
1258                info.shmall = in->shmall;
1259                return copy_to_user(buf, &info, sizeof(info));
1260        } else {
1261                struct shminfo info;
1262                memset(&info, 0, sizeof(info));
1263                info.shmmax = in->shmmax;
1264                info.shmmin = in->shmmin;
1265                info.shmmni = in->shmmni;
1266                info.shmseg = in->shmseg;
1267                info.shmall = in->shmall;
1268                return copy_to_user(buf, &info, sizeof(info));
1269        }
1270}
1271
1272static int put_compat_shm_info(struct shm_info *ip,
1273                                struct compat_shm_info __user *uip)
1274{
1275        struct compat_shm_info info;
1276
1277        memset(&info, 0, sizeof(info));
1278        info.used_ids = ip->used_ids;
1279        info.shm_tot = ip->shm_tot;
1280        info.shm_rss = ip->shm_rss;
1281        info.shm_swp = ip->shm_swp;
1282        info.swap_attempts = ip->swap_attempts;
1283        info.swap_successes = ip->swap_successes;
1284        return copy_to_user(uip, &info, sizeof(info));
1285}
1286
1287static int copy_compat_shmid_to_user(void __user *buf, struct shmid64_ds *in,
1288                                        int version)
1289{
1290        if (version == IPC_64) {
1291                struct compat_shmid64_ds v;
1292                memset(&v, 0, sizeof(v));
1293                to_compat_ipc64_perm(&v.shm_perm, &in->shm_perm);
1294                v.shm_atime      = lower_32_bits(in->shm_atime);
1295                v.shm_atime_high = upper_32_bits(in->shm_atime);
1296                v.shm_dtime      = lower_32_bits(in->shm_dtime);
1297                v.shm_dtime_high = upper_32_bits(in->shm_dtime);
1298                v.shm_ctime      = lower_32_bits(in->shm_ctime);
1299                v.shm_ctime_high = upper_32_bits(in->shm_ctime);
1300                v.shm_segsz = in->shm_segsz;
1301                v.shm_nattch = in->shm_nattch;
1302                v.shm_cpid = in->shm_cpid;
1303                v.shm_lpid = in->shm_lpid;
1304                return copy_to_user(buf, &v, sizeof(v));
1305        } else {
1306                struct compat_shmid_ds v;
1307                memset(&v, 0, sizeof(v));
1308                to_compat_ipc_perm(&v.shm_perm, &in->shm_perm);
1309                v.shm_perm.key = in->shm_perm.key;
1310                v.shm_atime = in->shm_atime;
1311                v.shm_dtime = in->shm_dtime;
1312                v.shm_ctime = in->shm_ctime;
1313                v.shm_segsz = in->shm_segsz;
1314                v.shm_nattch = in->shm_nattch;
1315                v.shm_cpid = in->shm_cpid;
1316                v.shm_lpid = in->shm_lpid;
1317                return copy_to_user(buf, &v, sizeof(v));
1318        }
1319}
1320
1321static int copy_compat_shmid_from_user(struct shmid64_ds *out, void __user *buf,
1322                                        int version)
1323{
1324        memset(out, 0, sizeof(*out));
1325        if (version == IPC_64) {
1326                struct compat_shmid64_ds __user *p = buf;
1327                return get_compat_ipc64_perm(&out->shm_perm, &p->shm_perm);
1328        } else {
1329                struct compat_shmid_ds __user *p = buf;
1330                return get_compat_ipc_perm(&out->shm_perm, &p->shm_perm);
1331        }
1332}
1333
1334static long compat_ksys_shmctl(int shmid, int cmd, void __user *uptr, int version)
1335{
1336        struct ipc_namespace *ns;
1337        struct shmid64_ds sem64;
1338        int err;
1339
1340        ns = current->nsproxy->ipc_ns;
1341
1342        if (cmd < 0 || shmid < 0)
1343                return -EINVAL;
1344
1345        switch (cmd) {
1346        case IPC_INFO: {
1347                struct shminfo64 shminfo;
1348                err = shmctl_ipc_info(ns, &shminfo);
1349                if (err < 0)
1350                        return err;
1351                if (copy_compat_shminfo_to_user(uptr, &shminfo, version))
1352                        err = -EFAULT;
1353                return err;
1354        }
1355        case SHM_INFO: {
1356                struct shm_info shm_info;
1357                err = shmctl_shm_info(ns, &shm_info);
1358                if (err < 0)
1359                        return err;
1360                if (put_compat_shm_info(&shm_info, uptr))
1361                        err = -EFAULT;
1362                return err;
1363        }
1364        case IPC_STAT:
1365        case SHM_STAT_ANY:
1366        case SHM_STAT:
1367                err = shmctl_stat(ns, shmid, cmd, &sem64);
1368                if (err < 0)
1369                        return err;
1370                if (copy_compat_shmid_to_user(uptr, &sem64, version))
1371                        err = -EFAULT;
1372                return err;
1373
1374        case IPC_SET:
1375                if (copy_compat_shmid_from_user(&sem64, uptr, version))
1376                        return -EFAULT;
1377                fallthrough;
1378        case IPC_RMID:
1379                return shmctl_down(ns, shmid, cmd, &sem64);
1380        case SHM_LOCK:
1381        case SHM_UNLOCK:
1382                return shmctl_do_lock(ns, shmid, cmd);
1383        default:
1384                return -EINVAL;
1385        }
1386        return err;
1387}
1388
1389COMPAT_SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, void __user *, uptr)
1390{
1391        return compat_ksys_shmctl(shmid, cmd, uptr, IPC_64);
1392}
1393
1394#ifdef CONFIG_ARCH_WANT_COMPAT_IPC_PARSE_VERSION
1395long compat_ksys_old_shmctl(int shmid, int cmd, void __user *uptr)
1396{
1397        int version = compat_ipc_parse_version(&cmd);
1398
1399        return compat_ksys_shmctl(shmid, cmd, uptr, version);
1400}
1401
1402COMPAT_SYSCALL_DEFINE3(old_shmctl, int, shmid, int, cmd, void __user *, uptr)
1403{
1404        return compat_ksys_old_shmctl(shmid, cmd, uptr);
1405}
1406#endif
1407#endif
1408
1409/*
1410 * Fix shmaddr, allocate descriptor, map shm, add attach descriptor to lists.
1411 *
1412 * NOTE! Despite the name, this is NOT a direct system call entrypoint. The
1413 * "raddr" thing points to kernel space, and there has to be a wrapper around
1414 * this.
1415 */
1416long do_shmat(int shmid, char __user *shmaddr, int shmflg,
1417              ulong *raddr, unsigned long shmlba)
1418{
1419        struct shmid_kernel *shp;
1420        unsigned long addr = (unsigned long)shmaddr;
1421        unsigned long size;
1422        struct file *file, *base;
1423        int    err;
1424        unsigned long flags = MAP_SHARED;
1425        unsigned long prot;
1426        int acc_mode;
1427        struct ipc_namespace *ns;
1428        struct shm_file_data *sfd;
1429        int f_flags;
1430        unsigned long populate = 0;
1431
1432        err = -EINVAL;
1433        if (shmid < 0)
1434                goto out;
1435
1436        if (addr) {
1437                if (addr & (shmlba - 1)) {
1438                        if (shmflg & SHM_RND) {
1439                                addr &= ~(shmlba - 1);  /* round down */
1440
1441                                /*
1442                                 * Ensure that the round-down is non-nil
1443                                 * when remapping. This can happen for
1444                                 * cases when addr < shmlba.
1445                                 */
1446                                if (!addr && (shmflg & SHM_REMAP))
1447                                        goto out;
1448                        } else
1449#ifndef __ARCH_FORCE_SHMLBA
1450                                if (addr & ~PAGE_MASK)
1451#endif
1452                                        goto out;
1453                }
1454
1455                flags |= MAP_FIXED;
1456        } else if ((shmflg & SHM_REMAP))
1457                goto out;
1458
1459        if (shmflg & SHM_RDONLY) {
1460                prot = PROT_READ;
1461                acc_mode = S_IRUGO;
1462                f_flags = O_RDONLY;
1463        } else {
1464                prot = PROT_READ | PROT_WRITE;
1465                acc_mode = S_IRUGO | S_IWUGO;
1466                f_flags = O_RDWR;
1467        }
1468        if (shmflg & SHM_EXEC) {
1469                prot |= PROT_EXEC;
1470                acc_mode |= S_IXUGO;
1471        }
1472
1473        /*
1474         * We cannot rely on the fs check since SYSV IPC does have an
1475         * additional creator id...
1476         */
1477        ns = current->nsproxy->ipc_ns;
1478        rcu_read_lock();
1479        shp = shm_obtain_object_check(ns, shmid);
1480        if (IS_ERR(shp)) {
1481                err = PTR_ERR(shp);
1482                goto out_unlock;
1483        }
1484
1485        err = -EACCES;
1486        if (ipcperms(ns, &shp->shm_perm, acc_mode))
1487                goto out_unlock;
1488
1489        err = security_shm_shmat(&shp->shm_perm, shmaddr, shmflg);
1490        if (err)
1491                goto out_unlock;
1492
1493        ipc_lock_object(&shp->shm_perm);
1494
1495        /* check if shm_destroy() is tearing down shp */
1496        if (!ipc_valid_object(&shp->shm_perm)) {
1497                ipc_unlock_object(&shp->shm_perm);
1498                err = -EIDRM;
1499                goto out_unlock;
1500        }
1501
1502        /*
1503         * We need to take a reference to the real shm file to prevent the
1504         * pointer from becoming stale in cases where the lifetime of the outer
1505         * file extends beyond that of the shm segment.  It's not usually
1506         * possible, but it can happen during remap_file_pages() emulation as
1507         * that unmaps the memory, then does ->mmap() via file reference only.
1508         * We'll deny the ->mmap() if the shm segment was since removed, but to
1509         * detect shm ID reuse we need to compare the file pointers.
1510         */
1511        base = get_file(shp->shm_file);
1512        shp->shm_nattch++;
1513        size = i_size_read(file_inode(base));
1514        ipc_unlock_object(&shp->shm_perm);
1515        rcu_read_unlock();
1516
1517        err = -ENOMEM;
1518        sfd = kzalloc(sizeof(*sfd), GFP_KERNEL);
1519        if (!sfd) {
1520                fput(base);
1521                goto out_nattch;
1522        }
1523
1524        file = alloc_file_clone(base, f_flags,
1525                          is_file_hugepages(base) ?
1526                                &shm_file_operations_huge :
1527                                &shm_file_operations);
1528        err = PTR_ERR(file);
1529        if (IS_ERR(file)) {
1530                kfree(sfd);
1531                fput(base);
1532                goto out_nattch;
1533        }
1534
1535        sfd->id = shp->shm_perm.id;
1536        sfd->ns = get_ipc_ns(ns);
1537        sfd->file = base;
1538        sfd->vm_ops = NULL;
1539        file->private_data = sfd;
1540
1541        err = security_mmap_file(file, prot, flags);
1542        if (err)
1543                goto out_fput;
1544
1545        if (mmap_write_lock_killable(current->mm)) {
1546                err = -EINTR;
1547                goto out_fput;
1548        }
1549
1550        if (addr && !(shmflg & SHM_REMAP)) {
1551                err = -EINVAL;
1552                if (addr + size < addr)
1553                        goto invalid;
1554
1555                if (find_vma_intersection(current->mm, addr, addr + size))
1556                        goto invalid;
1557        }
1558
1559        addr = do_mmap(file, addr, size, prot, flags, 0, &populate, NULL);
1560        *raddr = addr;
1561        err = 0;
1562        if (IS_ERR_VALUE(addr))
1563                err = (long)addr;
1564invalid:
1565        mmap_write_unlock(current->mm);
1566        if (populate)
1567                mm_populate(addr, populate);
1568
1569out_fput:
1570        fput(file);
1571
1572out_nattch:
1573        down_write(&shm_ids(ns).rwsem);
1574        shp = shm_lock(ns, shmid);
1575        shp->shm_nattch--;
1576        if (shm_may_destroy(ns, shp))
1577                shm_destroy(ns, shp);
1578        else
1579                shm_unlock(shp);
1580        up_write(&shm_ids(ns).rwsem);
1581        return err;
1582
1583out_unlock:
1584        rcu_read_unlock();
1585out:
1586        return err;
1587}
1588
1589SYSCALL_DEFINE3(shmat, int, shmid, char __user *, shmaddr, int, shmflg)
1590{
1591        unsigned long ret;
1592        long err;
1593
1594        err = do_shmat(shmid, shmaddr, shmflg, &ret, SHMLBA);
1595        if (err)
1596                return err;
1597        force_successful_syscall_return();
1598        return (long)ret;
1599}
1600
1601#ifdef CONFIG_COMPAT
1602
1603#ifndef COMPAT_SHMLBA
1604#define COMPAT_SHMLBA   SHMLBA
1605#endif
1606
1607COMPAT_SYSCALL_DEFINE3(shmat, int, shmid, compat_uptr_t, shmaddr, int, shmflg)
1608{
1609        unsigned long ret;
1610        long err;
1611
1612        err = do_shmat(shmid, compat_ptr(shmaddr), shmflg, &ret, COMPAT_SHMLBA);
1613        if (err)
1614                return err;
1615        force_successful_syscall_return();
1616        return (long)ret;
1617}
1618#endif
1619
1620/*
1621 * detach and kill segment if marked destroyed.
1622 * The work is done in shm_close.
1623 */
1624long ksys_shmdt(char __user *shmaddr)
1625{
1626        struct mm_struct *mm = current->mm;
1627        struct vm_area_struct *vma;
1628        unsigned long addr = (unsigned long)shmaddr;
1629        int retval = -EINVAL;
1630#ifdef CONFIG_MMU
1631        loff_t size = 0;
1632        struct file *file;
1633        struct vm_area_struct *next;
1634#endif
1635
1636        if (addr & ~PAGE_MASK)
1637                return retval;
1638
1639        if (mmap_write_lock_killable(mm))
1640                return -EINTR;
1641
1642        /*
1643         * This function tries to be smart and unmap shm segments that
1644         * were modified by partial mlock or munmap calls:
1645         * - It first determines the size of the shm segment that should be
1646         *   unmapped: It searches for a vma that is backed by shm and that
1647         *   started at address shmaddr. It records it's size and then unmaps
1648         *   it.
1649         * - Then it unmaps all shm vmas that started at shmaddr and that
1650         *   are within the initially determined size and that are from the
1651         *   same shm segment from which we determined the size.
1652         * Errors from do_munmap are ignored: the function only fails if
1653         * it's called with invalid parameters or if it's called to unmap
1654         * a part of a vma. Both calls in this function are for full vmas,
1655         * the parameters are directly copied from the vma itself and always
1656         * valid - therefore do_munmap cannot fail. (famous last words?)
1657         */
1658        /*
1659         * If it had been mremap()'d, the starting address would not
1660         * match the usual checks anyway. So assume all vma's are
1661         * above the starting address given.
1662         */
1663        vma = find_vma(mm, addr);
1664
1665#ifdef CONFIG_MMU
1666        while (vma) {
1667                next = vma->vm_next;
1668
1669                /*
1670                 * Check if the starting address would match, i.e. it's
1671                 * a fragment created by mprotect() and/or munmap(), or it
1672                 * otherwise it starts at this address with no hassles.
1673                 */
1674                if ((vma->vm_ops == &shm_vm_ops) &&
1675                        (vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) {
1676
1677                        /*
1678                         * Record the file of the shm segment being
1679                         * unmapped.  With mremap(), someone could place
1680                         * page from another segment but with equal offsets
1681                         * in the range we are unmapping.
1682                         */
1683                        file = vma->vm_file;
1684                        size = i_size_read(file_inode(vma->vm_file));
1685                        do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL);
1686                        /*
1687                         * We discovered the size of the shm segment, so
1688                         * break out of here and fall through to the next
1689                         * loop that uses the size information to stop
1690                         * searching for matching vma's.
1691                         */
1692                        retval = 0;
1693                        vma = next;
1694                        break;
1695                }
1696                vma = next;
1697        }
1698
1699        /*
1700         * We need look no further than the maximum address a fragment
1701         * could possibly have landed at. Also cast things to loff_t to
1702         * prevent overflows and make comparisons vs. equal-width types.
1703         */
1704        size = PAGE_ALIGN(size);
1705        while (vma && (loff_t)(vma->vm_end - addr) <= size) {
1706                next = vma->vm_next;
1707
1708                /* finding a matching vma now does not alter retval */
1709                if ((vma->vm_ops == &shm_vm_ops) &&
1710                    ((vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) &&
1711                    (vma->vm_file == file))
1712                        do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL);
1713                vma = next;
1714        }
1715
1716#else   /* CONFIG_MMU */
1717        /* under NOMMU conditions, the exact address to be destroyed must be
1718         * given
1719         */
1720        if (vma && vma->vm_start == addr && vma->vm_ops == &shm_vm_ops) {
1721                do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL);
1722                retval = 0;
1723        }
1724
1725#endif
1726
1727        mmap_write_unlock(mm);
1728        return retval;
1729}
1730
1731SYSCALL_DEFINE1(shmdt, char __user *, shmaddr)
1732{
1733        return ksys_shmdt(shmaddr);
1734}
1735
1736#ifdef CONFIG_PROC_FS
1737static int sysvipc_shm_proc_show(struct seq_file *s, void *it)
1738{
1739        struct pid_namespace *pid_ns = ipc_seq_pid_ns(s);
1740        struct user_namespace *user_ns = seq_user_ns(s);
1741        struct kern_ipc_perm *ipcp = it;
1742        struct shmid_kernel *shp;
1743        unsigned long rss = 0, swp = 0;
1744
1745        shp = container_of(ipcp, struct shmid_kernel, shm_perm);
1746        shm_add_rss_swap(shp, &rss, &swp);
1747
1748#if BITS_PER_LONG <= 32
1749#define SIZE_SPEC "%10lu"
1750#else
1751#define SIZE_SPEC "%21lu"
1752#endif
1753
1754        seq_printf(s,
1755                   "%10d %10d  %4o " SIZE_SPEC " %5u %5u  "
1756                   "%5lu %5u %5u %5u %5u %10llu %10llu %10llu "
1757                   SIZE_SPEC " " SIZE_SPEC "\n",
1758                   shp->shm_perm.key,
1759                   shp->shm_perm.id,
1760                   shp->shm_perm.mode,
1761                   shp->shm_segsz,
1762                   pid_nr_ns(shp->shm_cprid, pid_ns),
1763                   pid_nr_ns(shp->shm_lprid, pid_ns),
1764                   shp->shm_nattch,
1765                   from_kuid_munged(user_ns, shp->shm_perm.uid),
1766                   from_kgid_munged(user_ns, shp->shm_perm.gid),
1767                   from_kuid_munged(user_ns, shp->shm_perm.cuid),
1768                   from_kgid_munged(user_ns, shp->shm_perm.cgid),
1769                   shp->shm_atim,
1770                   shp->shm_dtim,
1771                   shp->shm_ctim,
1772                   rss * PAGE_SIZE,
1773                   swp * PAGE_SIZE);
1774
1775        return 0;
1776}
1777#endif
1778