linux/kernel/kprobes.c
<<
>>
Prefs
   1/*
   2 *  Kernel Probes (KProbes)
   3 *  kernel/kprobes.c
   4 *
   5 * This program is free software; you can redistribute it and/or modify
   6 * it under the terms of the GNU General Public License as published by
   7 * the Free Software Foundation; either version 2 of the License, or
   8 * (at your option) any later version.
   9 *
  10 * This program is distributed in the hope that it will be useful,
  11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 * GNU General Public License for more details.
  14 *
  15 * You should have received a copy of the GNU General Public License
  16 * along with this program; if not, write to the Free Software
  17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18 *
  19 * Copyright (C) IBM Corporation, 2002, 2004
  20 *
  21 * 2002-Oct     Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
  22 *              Probes initial implementation (includes suggestions from
  23 *              Rusty Russell).
  24 * 2004-Aug     Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
  25 *              hlists and exceptions notifier as suggested by Andi Kleen.
  26 * 2004-July    Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
  27 *              interface to access function arguments.
  28 * 2004-Sep     Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
  29 *              exceptions notifier to be first on the priority list.
  30 * 2005-May     Hien Nguyen <hien@us.ibm.com>, Jim Keniston
  31 *              <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
  32 *              <prasanna@in.ibm.com> added function-return probes.
  33 */
  34#include <linux/kprobes.h>
  35#include <linux/hash.h>
  36#include <linux/init.h>
  37#include <linux/slab.h>
  38#include <linux/stddef.h>
  39#include <linux/module.h>
  40#include <linux/moduleloader.h>
  41#include <linux/kallsyms.h>
  42#include <linux/freezer.h>
  43#include <linux/seq_file.h>
  44#include <linux/debugfs.h>
  45#include <linux/sysctl.h>
  46#include <linux/kdebug.h>
  47#include <linux/memory.h>
  48#include <linux/ftrace.h>
  49#include <linux/cpu.h>
  50
  51#include <asm-generic/sections.h>
  52#include <asm/cacheflush.h>
  53#include <asm/errno.h>
  54#include <asm/uaccess.h>
  55
  56#define KPROBE_HASH_BITS 6
  57#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
  58
  59
  60/*
  61 * Some oddball architectures like 64bit powerpc have function descriptors
  62 * so this must be overridable.
  63 */
  64#ifndef kprobe_lookup_name
  65#define kprobe_lookup_name(name, addr) \
  66        addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))
  67#endif
  68
  69static int kprobes_initialized;
  70static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
  71static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
  72
  73/* NOTE: change this value only with kprobe_mutex held */
  74static bool kprobes_all_disarmed;
  75
  76static DEFINE_MUTEX(kprobe_mutex);      /* Protects kprobe_table */
  77static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
  78static struct {
  79        spinlock_t lock ____cacheline_aligned_in_smp;
  80} kretprobe_table_locks[KPROBE_TABLE_SIZE];
  81
  82static spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
  83{
  84        return &(kretprobe_table_locks[hash].lock);
  85}
  86
  87/*
  88 * Normally, functions that we'd want to prohibit kprobes in, are marked
  89 * __kprobes. But, there are cases where such functions already belong to
  90 * a different section (__sched for preempt_schedule)
  91 *
  92 * For such cases, we now have a blacklist
  93 */
  94static struct kprobe_blackpoint kprobe_blacklist[] = {
  95        {"preempt_schedule",},
  96        {"native_get_debugreg",},
  97        {"irq_entries_start",},
  98        {"common_interrupt",},
  99        {"mcount",},    /* mcount can be called from everywhere */
 100        {NULL}    /* Terminator */
 101};
 102
 103#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
 104/*
 105 * kprobe->ainsn.insn points to the copy of the instruction to be
 106 * single-stepped. x86_64, POWER4 and above have no-exec support and
 107 * stepping on the instruction on a vmalloced/kmalloced/data page
 108 * is a recipe for disaster
 109 */
 110struct kprobe_insn_page {
 111        struct list_head list;
 112        kprobe_opcode_t *insns;         /* Page of instruction slots */
 113        int nused;
 114        int ngarbage;
 115        char slot_used[];
 116};
 117
 118#define KPROBE_INSN_PAGE_SIZE(slots)                    \
 119        (offsetof(struct kprobe_insn_page, slot_used) + \
 120         (sizeof(char) * (slots)))
 121
 122struct kprobe_insn_cache {
 123        struct list_head pages; /* list of kprobe_insn_page */
 124        size_t insn_size;       /* size of instruction slot */
 125        int nr_garbage;
 126};
 127
 128static int slots_per_page(struct kprobe_insn_cache *c)
 129{
 130        return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
 131}
 132
 133enum kprobe_slot_state {
 134        SLOT_CLEAN = 0,
 135        SLOT_DIRTY = 1,
 136        SLOT_USED = 2,
 137};
 138
 139static DEFINE_MUTEX(kprobe_insn_mutex); /* Protects kprobe_insn_slots */
 140static struct kprobe_insn_cache kprobe_insn_slots = {
 141        .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
 142        .insn_size = MAX_INSN_SIZE,
 143        .nr_garbage = 0,
 144};
 145static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c);
 146
 147/**
 148 * __get_insn_slot() - Find a slot on an executable page for an instruction.
 149 * We allocate an executable page if there's no room on existing ones.
 150 */
 151static kprobe_opcode_t __kprobes *__get_insn_slot(struct kprobe_insn_cache *c)
 152{
 153        struct kprobe_insn_page *kip;
 154
 155 retry:
 156        list_for_each_entry(kip, &c->pages, list) {
 157                if (kip->nused < slots_per_page(c)) {
 158                        int i;
 159                        for (i = 0; i < slots_per_page(c); i++) {
 160                                if (kip->slot_used[i] == SLOT_CLEAN) {
 161                                        kip->slot_used[i] = SLOT_USED;
 162                                        kip->nused++;
 163                                        return kip->insns + (i * c->insn_size);
 164                                }
 165                        }
 166                        /* kip->nused is broken. Fix it. */
 167                        kip->nused = slots_per_page(c);
 168                        WARN_ON(1);
 169                }
 170        }
 171
 172        /* If there are any garbage slots, collect it and try again. */
 173        if (c->nr_garbage && collect_garbage_slots(c) == 0)
 174                goto retry;
 175
 176        /* All out of space.  Need to allocate a new page. */
 177        kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);
 178        if (!kip)
 179                return NULL;
 180
 181        /*
 182         * Use module_alloc so this page is within +/- 2GB of where the
 183         * kernel image and loaded module images reside. This is required
 184         * so x86_64 can correctly handle the %rip-relative fixups.
 185         */
 186        kip->insns = module_alloc(PAGE_SIZE);
 187        if (!kip->insns) {
 188                kfree(kip);
 189                return NULL;
 190        }
 191        INIT_LIST_HEAD(&kip->list);
 192        memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
 193        kip->slot_used[0] = SLOT_USED;
 194        kip->nused = 1;
 195        kip->ngarbage = 0;
 196        list_add(&kip->list, &c->pages);
 197        return kip->insns;
 198}
 199
 200
 201kprobe_opcode_t __kprobes *get_insn_slot(void)
 202{
 203        kprobe_opcode_t *ret = NULL;
 204
 205        mutex_lock(&kprobe_insn_mutex);
 206        ret = __get_insn_slot(&kprobe_insn_slots);
 207        mutex_unlock(&kprobe_insn_mutex);
 208
 209        return ret;
 210}
 211
 212/* Return 1 if all garbages are collected, otherwise 0. */
 213static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx)
 214{
 215        kip->slot_used[idx] = SLOT_CLEAN;
 216        kip->nused--;
 217        if (kip->nused == 0) {
 218                /*
 219                 * Page is no longer in use.  Free it unless
 220                 * it's the last one.  We keep the last one
 221                 * so as not to have to set it up again the
 222                 * next time somebody inserts a probe.
 223                 */
 224                if (!list_is_singular(&kip->list)) {
 225                        list_del(&kip->list);
 226                        module_free(NULL, kip->insns);
 227                        kfree(kip);
 228                }
 229                return 1;
 230        }
 231        return 0;
 232}
 233
 234static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c)
 235{
 236        struct kprobe_insn_page *kip, *next;
 237
 238        /* Ensure no-one is interrupted on the garbages */
 239        synchronize_sched();
 240
 241        list_for_each_entry_safe(kip, next, &c->pages, list) {
 242                int i;
 243                if (kip->ngarbage == 0)
 244                        continue;
 245                kip->ngarbage = 0;      /* we will collect all garbages */
 246                for (i = 0; i < slots_per_page(c); i++) {
 247                        if (kip->slot_used[i] == SLOT_DIRTY &&
 248                            collect_one_slot(kip, i))
 249                                break;
 250                }
 251        }
 252        c->nr_garbage = 0;
 253        return 0;
 254}
 255
 256static void __kprobes __free_insn_slot(struct kprobe_insn_cache *c,
 257                                       kprobe_opcode_t *slot, int dirty)
 258{
 259        struct kprobe_insn_page *kip;
 260
 261        list_for_each_entry(kip, &c->pages, list) {
 262                long idx = ((long)slot - (long)kip->insns) /
 263                                (c->insn_size * sizeof(kprobe_opcode_t));
 264                if (idx >= 0 && idx < slots_per_page(c)) {
 265                        WARN_ON(kip->slot_used[idx] != SLOT_USED);
 266                        if (dirty) {
 267                                kip->slot_used[idx] = SLOT_DIRTY;
 268                                kip->ngarbage++;
 269                                if (++c->nr_garbage > slots_per_page(c))
 270                                        collect_garbage_slots(c);
 271                        } else
 272                                collect_one_slot(kip, idx);
 273                        return;
 274                }
 275        }
 276        /* Could not free this slot. */
 277        WARN_ON(1);
 278}
 279
 280void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty)
 281{
 282        mutex_lock(&kprobe_insn_mutex);
 283        __free_insn_slot(&kprobe_insn_slots, slot, dirty);
 284        mutex_unlock(&kprobe_insn_mutex);
 285}
 286#ifdef CONFIG_OPTPROBES
 287/* For optimized_kprobe buffer */
 288static DEFINE_MUTEX(kprobe_optinsn_mutex); /* Protects kprobe_optinsn_slots */
 289static struct kprobe_insn_cache kprobe_optinsn_slots = {
 290        .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
 291        /* .insn_size is initialized later */
 292        .nr_garbage = 0,
 293};
 294/* Get a slot for optimized_kprobe buffer */
 295kprobe_opcode_t __kprobes *get_optinsn_slot(void)
 296{
 297        kprobe_opcode_t *ret = NULL;
 298
 299        mutex_lock(&kprobe_optinsn_mutex);
 300        ret = __get_insn_slot(&kprobe_optinsn_slots);
 301        mutex_unlock(&kprobe_optinsn_mutex);
 302
 303        return ret;
 304}
 305
 306void __kprobes free_optinsn_slot(kprobe_opcode_t * slot, int dirty)
 307{
 308        mutex_lock(&kprobe_optinsn_mutex);
 309        __free_insn_slot(&kprobe_optinsn_slots, slot, dirty);
 310        mutex_unlock(&kprobe_optinsn_mutex);
 311}
 312#endif
 313#endif
 314
 315/* We have preemption disabled.. so it is safe to use __ versions */
 316static inline void set_kprobe_instance(struct kprobe *kp)
 317{
 318        __get_cpu_var(kprobe_instance) = kp;
 319}
 320
 321static inline void reset_kprobe_instance(void)
 322{
 323        __get_cpu_var(kprobe_instance) = NULL;
 324}
 325
 326/*
 327 * This routine is called either:
 328 *      - under the kprobe_mutex - during kprobe_[un]register()
 329 *                              OR
 330 *      - with preemption disabled - from arch/xxx/kernel/kprobes.c
 331 */
 332struct kprobe __kprobes *get_kprobe(void *addr)
 333{
 334        struct hlist_head *head;
 335        struct hlist_node *node;
 336        struct kprobe *p;
 337
 338        head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
 339        hlist_for_each_entry_rcu(p, node, head, hlist) {
 340                if (p->addr == addr)
 341                        return p;
 342        }
 343
 344        return NULL;
 345}
 346
 347static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
 348
 349/* Return true if the kprobe is an aggregator */
 350static inline int kprobe_aggrprobe(struct kprobe *p)
 351{
 352        return p->pre_handler == aggr_pre_handler;
 353}
 354
 355/*
 356 * Keep all fields in the kprobe consistent
 357 */
 358static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p)
 359{
 360        memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t));
 361        memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn));
 362}
 363
 364#ifdef CONFIG_OPTPROBES
 365/* NOTE: change this value only with kprobe_mutex held */
 366static bool kprobes_allow_optimization;
 367
 368/*
 369 * Call all pre_handler on the list, but ignores its return value.
 370 * This must be called from arch-dep optimized caller.
 371 */
 372void __kprobes opt_pre_handler(struct kprobe *p, struct pt_regs *regs)
 373{
 374        struct kprobe *kp;
 375
 376        list_for_each_entry_rcu(kp, &p->list, list) {
 377                if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
 378                        set_kprobe_instance(kp);
 379                        kp->pre_handler(kp, regs);
 380                }
 381                reset_kprobe_instance();
 382        }
 383}
 384
 385/* Return true(!0) if the kprobe is ready for optimization. */
 386static inline int kprobe_optready(struct kprobe *p)
 387{
 388        struct optimized_kprobe *op;
 389
 390        if (kprobe_aggrprobe(p)) {
 391                op = container_of(p, struct optimized_kprobe, kp);
 392                return arch_prepared_optinsn(&op->optinsn);
 393        }
 394
 395        return 0;
 396}
 397
 398/*
 399 * Return an optimized kprobe whose optimizing code replaces
 400 * instructions including addr (exclude breakpoint).
 401 */
 402struct kprobe *__kprobes get_optimized_kprobe(unsigned long addr)
 403{
 404        int i;
 405        struct kprobe *p = NULL;
 406        struct optimized_kprobe *op;
 407
 408        /* Don't check i == 0, since that is a breakpoint case. */
 409        for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH; i++)
 410                p = get_kprobe((void *)(addr - i));
 411
 412        if (p && kprobe_optready(p)) {
 413                op = container_of(p, struct optimized_kprobe, kp);
 414                if (arch_within_optimized_kprobe(op, addr))
 415                        return p;
 416        }
 417
 418        return NULL;
 419}
 420
 421/* Optimization staging list, protected by kprobe_mutex */
 422static LIST_HEAD(optimizing_list);
 423
 424static void kprobe_optimizer(struct work_struct *work);
 425static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
 426#define OPTIMIZE_DELAY 5
 427
 428/* Kprobe jump optimizer */
 429static __kprobes void kprobe_optimizer(struct work_struct *work)
 430{
 431        struct optimized_kprobe *op, *tmp;
 432
 433        /* Lock modules while optimizing kprobes */
 434        mutex_lock(&module_mutex);
 435        mutex_lock(&kprobe_mutex);
 436        if (kprobes_all_disarmed || !kprobes_allow_optimization)
 437                goto end;
 438
 439        /*
 440         * Wait for quiesence period to ensure all running interrupts
 441         * are done. Because optprobe may modify multiple instructions
 442         * there is a chance that Nth instruction is interrupted. In that
 443         * case, running interrupt can return to 2nd-Nth byte of jump
 444         * instruction. This wait is for avoiding it.
 445         */
 446        synchronize_sched();
 447
 448        /*
 449         * The optimization/unoptimization refers online_cpus via
 450         * stop_machine() and cpu-hotplug modifies online_cpus.
 451         * And same time, text_mutex will be held in cpu-hotplug and here.
 452         * This combination can cause a deadlock (cpu-hotplug try to lock
 453         * text_mutex but stop_machine can not be done because online_cpus
 454         * has been changed)
 455         * To avoid this deadlock, we need to call get_online_cpus()
 456         * for preventing cpu-hotplug outside of text_mutex locking.
 457         */
 458        get_online_cpus();
 459        mutex_lock(&text_mutex);
 460        list_for_each_entry_safe(op, tmp, &optimizing_list, list) {
 461                WARN_ON(kprobe_disabled(&op->kp));
 462                if (arch_optimize_kprobe(op) < 0)
 463                        op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
 464                list_del_init(&op->list);
 465        }
 466        mutex_unlock(&text_mutex);
 467        put_online_cpus();
 468end:
 469        mutex_unlock(&kprobe_mutex);
 470        mutex_unlock(&module_mutex);
 471}
 472
 473/* Optimize kprobe if p is ready to be optimized */
 474static __kprobes void optimize_kprobe(struct kprobe *p)
 475{
 476        struct optimized_kprobe *op;
 477
 478        /* Check if the kprobe is disabled or not ready for optimization. */
 479        if (!kprobe_optready(p) || !kprobes_allow_optimization ||
 480            (kprobe_disabled(p) || kprobes_all_disarmed))
 481                return;
 482
 483        /* Both of break_handler and post_handler are not supported. */
 484        if (p->break_handler || p->post_handler)
 485                return;
 486
 487        op = container_of(p, struct optimized_kprobe, kp);
 488
 489        /* Check there is no other kprobes at the optimized instructions */
 490        if (arch_check_optimized_kprobe(op) < 0)
 491                return;
 492
 493        /* Check if it is already optimized. */
 494        if (op->kp.flags & KPROBE_FLAG_OPTIMIZED)
 495                return;
 496
 497        op->kp.flags |= KPROBE_FLAG_OPTIMIZED;
 498        list_add(&op->list, &optimizing_list);
 499        if (!delayed_work_pending(&optimizing_work))
 500                schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
 501}
 502
 503/* Unoptimize a kprobe if p is optimized */
 504static __kprobes void unoptimize_kprobe(struct kprobe *p)
 505{
 506        struct optimized_kprobe *op;
 507
 508        if ((p->flags & KPROBE_FLAG_OPTIMIZED) && kprobe_aggrprobe(p)) {
 509                op = container_of(p, struct optimized_kprobe, kp);
 510                if (!list_empty(&op->list))
 511                        /* Dequeue from the optimization queue */
 512                        list_del_init(&op->list);
 513                else
 514                        /* Replace jump with break */
 515                        arch_unoptimize_kprobe(op);
 516                op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
 517        }
 518}
 519
 520/* Remove optimized instructions */
 521static void __kprobes kill_optimized_kprobe(struct kprobe *p)
 522{
 523        struct optimized_kprobe *op;
 524
 525        op = container_of(p, struct optimized_kprobe, kp);
 526        if (!list_empty(&op->list)) {
 527                /* Dequeue from the optimization queue */
 528                list_del_init(&op->list);
 529                op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
 530        }
 531        /* Don't unoptimize, because the target code will be freed. */
 532        arch_remove_optimized_kprobe(op);
 533}
 534
 535/* Try to prepare optimized instructions */
 536static __kprobes void prepare_optimized_kprobe(struct kprobe *p)
 537{
 538        struct optimized_kprobe *op;
 539
 540        op = container_of(p, struct optimized_kprobe, kp);
 541        arch_prepare_optimized_kprobe(op);
 542}
 543
 544/* Free optimized instructions and optimized_kprobe */
 545static __kprobes void free_aggr_kprobe(struct kprobe *p)
 546{
 547        struct optimized_kprobe *op;
 548
 549        op = container_of(p, struct optimized_kprobe, kp);
 550        arch_remove_optimized_kprobe(op);
 551        kfree(op);
 552}
 553
 554/* Allocate new optimized_kprobe and try to prepare optimized instructions */
 555static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
 556{
 557        struct optimized_kprobe *op;
 558
 559        op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL);
 560        if (!op)
 561                return NULL;
 562
 563        INIT_LIST_HEAD(&op->list);
 564        op->kp.addr = p->addr;
 565        arch_prepare_optimized_kprobe(op);
 566
 567        return &op->kp;
 568}
 569
 570static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
 571
 572/*
 573 * Prepare an optimized_kprobe and optimize it
 574 * NOTE: p must be a normal registered kprobe
 575 */
 576static __kprobes void try_to_optimize_kprobe(struct kprobe *p)
 577{
 578        struct kprobe *ap;
 579        struct optimized_kprobe *op;
 580
 581        ap = alloc_aggr_kprobe(p);
 582        if (!ap)
 583                return;
 584
 585        op = container_of(ap, struct optimized_kprobe, kp);
 586        if (!arch_prepared_optinsn(&op->optinsn)) {
 587                /* If failed to setup optimizing, fallback to kprobe */
 588                free_aggr_kprobe(ap);
 589                return;
 590        }
 591
 592        init_aggr_kprobe(ap, p);
 593        optimize_kprobe(ap);
 594}
 595
 596#ifdef CONFIG_SYSCTL
 597static void __kprobes optimize_all_kprobes(void)
 598{
 599        struct hlist_head *head;
 600        struct hlist_node *node;
 601        struct kprobe *p;
 602        unsigned int i;
 603
 604        /* If optimization is already allowed, just return */
 605        if (kprobes_allow_optimization)
 606                return;
 607
 608        kprobes_allow_optimization = true;
 609        mutex_lock(&text_mutex);
 610        for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
 611                head = &kprobe_table[i];
 612                hlist_for_each_entry_rcu(p, node, head, hlist)
 613                        if (!kprobe_disabled(p))
 614                                optimize_kprobe(p);
 615        }
 616        mutex_unlock(&text_mutex);
 617        printk(KERN_INFO "Kprobes globally optimized\n");
 618}
 619
 620static void __kprobes unoptimize_all_kprobes(void)
 621{
 622        struct hlist_head *head;
 623        struct hlist_node *node;
 624        struct kprobe *p;
 625        unsigned int i;
 626
 627        /* If optimization is already prohibited, just return */
 628        if (!kprobes_allow_optimization)
 629                return;
 630
 631        kprobes_allow_optimization = false;
 632        printk(KERN_INFO "Kprobes globally unoptimized\n");
 633        get_online_cpus();      /* For avoiding text_mutex deadlock */
 634        mutex_lock(&text_mutex);
 635        for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
 636                head = &kprobe_table[i];
 637                hlist_for_each_entry_rcu(p, node, head, hlist) {
 638                        if (!kprobe_disabled(p))
 639                                unoptimize_kprobe(p);
 640                }
 641        }
 642
 643        mutex_unlock(&text_mutex);
 644        put_online_cpus();
 645        /* Allow all currently running kprobes to complete */
 646        synchronize_sched();
 647}
 648
 649int sysctl_kprobes_optimization;
 650int proc_kprobes_optimization_handler(struct ctl_table *table, int write,
 651                                      void __user *buffer, size_t *length,
 652                                      loff_t *ppos)
 653{
 654        int ret;
 655
 656        mutex_lock(&kprobe_mutex);
 657        sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
 658        ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
 659
 660        if (sysctl_kprobes_optimization)
 661                optimize_all_kprobes();
 662        else
 663                unoptimize_all_kprobes();
 664        mutex_unlock(&kprobe_mutex);
 665
 666        return ret;
 667}
 668#endif /* CONFIG_SYSCTL */
 669
 670static void __kprobes __arm_kprobe(struct kprobe *p)
 671{
 672        struct kprobe *old_p;
 673
 674        /* Check collision with other optimized kprobes */
 675        old_p = get_optimized_kprobe((unsigned long)p->addr);
 676        if (unlikely(old_p))
 677                unoptimize_kprobe(old_p); /* Fallback to unoptimized kprobe */
 678
 679        arch_arm_kprobe(p);
 680        optimize_kprobe(p);     /* Try to optimize (add kprobe to a list) */
 681}
 682
 683static void __kprobes __disarm_kprobe(struct kprobe *p)
 684{
 685        struct kprobe *old_p;
 686
 687        unoptimize_kprobe(p);   /* Try to unoptimize */
 688        arch_disarm_kprobe(p);
 689
 690        /* If another kprobe was blocked, optimize it. */
 691        old_p = get_optimized_kprobe((unsigned long)p->addr);
 692        if (unlikely(old_p))
 693                optimize_kprobe(old_p);
 694}
 695
 696#else /* !CONFIG_OPTPROBES */
 697
 698#define optimize_kprobe(p)                      do {} while (0)
 699#define unoptimize_kprobe(p)                    do {} while (0)
 700#define kill_optimized_kprobe(p)                do {} while (0)
 701#define prepare_optimized_kprobe(p)             do {} while (0)
 702#define try_to_optimize_kprobe(p)               do {} while (0)
 703#define __arm_kprobe(p)                         arch_arm_kprobe(p)
 704#define __disarm_kprobe(p)                      arch_disarm_kprobe(p)
 705
 706static __kprobes void free_aggr_kprobe(struct kprobe *p)
 707{
 708        kfree(p);
 709}
 710
 711static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
 712{
 713        return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
 714}
 715#endif /* CONFIG_OPTPROBES */
 716
 717/* Arm a kprobe with text_mutex */
 718static void __kprobes arm_kprobe(struct kprobe *kp)
 719{
 720        /*
 721         * Here, since __arm_kprobe() doesn't use stop_machine(),
 722         * this doesn't cause deadlock on text_mutex. So, we don't
 723         * need get_online_cpus().
 724         */
 725        mutex_lock(&text_mutex);
 726        __arm_kprobe(kp);
 727        mutex_unlock(&text_mutex);
 728}
 729
 730/* Disarm a kprobe with text_mutex */
 731static void __kprobes disarm_kprobe(struct kprobe *kp)
 732{
 733        get_online_cpus();      /* For avoiding text_mutex deadlock */
 734        mutex_lock(&text_mutex);
 735        __disarm_kprobe(kp);
 736        mutex_unlock(&text_mutex);
 737        put_online_cpus();
 738}
 739
 740/*
 741 * Aggregate handlers for multiple kprobes support - these handlers
 742 * take care of invoking the individual kprobe handlers on p->list
 743 */
 744static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
 745{
 746        struct kprobe *kp;
 747
 748        list_for_each_entry_rcu(kp, &p->list, list) {
 749                if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
 750                        set_kprobe_instance(kp);
 751                        if (kp->pre_handler(kp, regs))
 752                                return 1;
 753                }
 754                reset_kprobe_instance();
 755        }
 756        return 0;
 757}
 758
 759static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
 760                                        unsigned long flags)
 761{
 762        struct kprobe *kp;
 763
 764        list_for_each_entry_rcu(kp, &p->list, list) {
 765                if (kp->post_handler && likely(!kprobe_disabled(kp))) {
 766                        set_kprobe_instance(kp);
 767                        kp->post_handler(kp, regs, flags);
 768                        reset_kprobe_instance();
 769                }
 770        }
 771}
 772
 773static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,
 774                                        int trapnr)
 775{
 776        struct kprobe *cur = __get_cpu_var(kprobe_instance);
 777
 778        /*
 779         * if we faulted "during" the execution of a user specified
 780         * probe handler, invoke just that probe's fault handler
 781         */
 782        if (cur && cur->fault_handler) {
 783                if (cur->fault_handler(cur, regs, trapnr))
 784                        return 1;
 785        }
 786        return 0;
 787}
 788
 789static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
 790{
 791        struct kprobe *cur = __get_cpu_var(kprobe_instance);
 792        int ret = 0;
 793
 794        if (cur && cur->break_handler) {
 795                if (cur->break_handler(cur, regs))
 796                        ret = 1;
 797        }
 798        reset_kprobe_instance();
 799        return ret;
 800}
 801
 802/* Walks the list and increments nmissed count for multiprobe case */
 803void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)
 804{
 805        struct kprobe *kp;
 806        if (!kprobe_aggrprobe(p)) {
 807                p->nmissed++;
 808        } else {
 809                list_for_each_entry_rcu(kp, &p->list, list)
 810                        kp->nmissed++;
 811        }
 812        return;
 813}
 814
 815void __kprobes recycle_rp_inst(struct kretprobe_instance *ri,
 816                                struct hlist_head *head)
 817{
 818        struct kretprobe *rp = ri->rp;
 819
 820        /* remove rp inst off the rprobe_inst_table */
 821        hlist_del(&ri->hlist);
 822        INIT_HLIST_NODE(&ri->hlist);
 823        if (likely(rp)) {
 824                spin_lock(&rp->lock);
 825                hlist_add_head(&ri->hlist, &rp->free_instances);
 826                spin_unlock(&rp->lock);
 827        } else
 828                /* Unregistering */
 829                hlist_add_head(&ri->hlist, head);
 830}
 831
 832void __kprobes kretprobe_hash_lock(struct task_struct *tsk,
 833                         struct hlist_head **head, unsigned long *flags)
 834{
 835        unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
 836        spinlock_t *hlist_lock;
 837
 838        *head = &kretprobe_inst_table[hash];
 839        hlist_lock = kretprobe_table_lock_ptr(hash);
 840        spin_lock_irqsave(hlist_lock, *flags);
 841}
 842
 843static void __kprobes kretprobe_table_lock(unsigned long hash,
 844        unsigned long *flags)
 845{
 846        spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
 847        spin_lock_irqsave(hlist_lock, *flags);
 848}
 849
 850void __kprobes kretprobe_hash_unlock(struct task_struct *tsk,
 851        unsigned long *flags)
 852{
 853        unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
 854        spinlock_t *hlist_lock;
 855
 856        hlist_lock = kretprobe_table_lock_ptr(hash);
 857        spin_unlock_irqrestore(hlist_lock, *flags);
 858}
 859
 860void __kprobes kretprobe_table_unlock(unsigned long hash, unsigned long *flags)
 861{
 862        spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
 863        spin_unlock_irqrestore(hlist_lock, *flags);
 864}
 865
 866/*
 867 * This function is called from finish_task_switch when task tk becomes dead,
 868 * so that we can recycle any function-return probe instances associated
 869 * with this task. These left over instances represent probed functions
 870 * that have been called but will never return.
 871 */
 872void __kprobes kprobe_flush_task(struct task_struct *tk)
 873{
 874        struct kretprobe_instance *ri;
 875        struct hlist_head *head, empty_rp;
 876        struct hlist_node *node, *tmp;
 877        unsigned long hash, flags = 0;
 878
 879        if (unlikely(!kprobes_initialized))
 880                /* Early boot.  kretprobe_table_locks not yet initialized. */
 881                return;
 882
 883        hash = hash_ptr(tk, KPROBE_HASH_BITS);
 884        head = &kretprobe_inst_table[hash];
 885        kretprobe_table_lock(hash, &flags);
 886        hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
 887                if (ri->task == tk)
 888                        recycle_rp_inst(ri, &empty_rp);
 889        }
 890        kretprobe_table_unlock(hash, &flags);
 891        INIT_HLIST_HEAD(&empty_rp);
 892        hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
 893                hlist_del(&ri->hlist);
 894                kfree(ri);
 895        }
 896}
 897
 898static inline void free_rp_inst(struct kretprobe *rp)
 899{
 900        struct kretprobe_instance *ri;
 901        struct hlist_node *pos, *next;
 902
 903        hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, hlist) {
 904                hlist_del(&ri->hlist);
 905                kfree(ri);
 906        }
 907}
 908
 909static void __kprobes cleanup_rp_inst(struct kretprobe *rp)
 910{
 911        unsigned long flags, hash;
 912        struct kretprobe_instance *ri;
 913        struct hlist_node *pos, *next;
 914        struct hlist_head *head;
 915
 916        /* No race here */
 917        for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
 918                kretprobe_table_lock(hash, &flags);
 919                head = &kretprobe_inst_table[hash];
 920                hlist_for_each_entry_safe(ri, pos, next, head, hlist) {
 921                        if (ri->rp == rp)
 922                                ri->rp = NULL;
 923                }
 924                kretprobe_table_unlock(hash, &flags);
 925        }
 926        free_rp_inst(rp);
 927}
 928
 929/*
 930* Add the new probe to ap->list. Fail if this is the
 931* second jprobe at the address - two jprobes can't coexist
 932*/
 933static int __kprobes add_new_kprobe(struct kprobe *ap, struct kprobe *p)
 934{
 935        BUG_ON(kprobe_gone(ap) || kprobe_gone(p));
 936
 937        if (p->break_handler || p->post_handler)
 938                unoptimize_kprobe(ap);  /* Fall back to normal kprobe */
 939
 940        if (p->break_handler) {
 941                if (ap->break_handler)
 942                        return -EEXIST;
 943                list_add_tail_rcu(&p->list, &ap->list);
 944                ap->break_handler = aggr_break_handler;
 945        } else
 946                list_add_rcu(&p->list, &ap->list);
 947        if (p->post_handler && !ap->post_handler)
 948                ap->post_handler = aggr_post_handler;
 949
 950        if (kprobe_disabled(ap) && !kprobe_disabled(p)) {
 951                ap->flags &= ~KPROBE_FLAG_DISABLED;
 952                if (!kprobes_all_disarmed)
 953                        /* Arm the breakpoint again. */
 954                        __arm_kprobe(ap);
 955        }
 956        return 0;
 957}
 958
 959/*
 960 * Fill in the required fields of the "manager kprobe". Replace the
 961 * earlier kprobe in the hlist with the manager kprobe
 962 */
 963static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
 964{
 965        /* Copy p's insn slot to ap */
 966        copy_kprobe(p, ap);
 967        flush_insn_slot(ap);
 968        ap->addr = p->addr;
 969        ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
 970        ap->pre_handler = aggr_pre_handler;
 971        ap->fault_handler = aggr_fault_handler;
 972        /* We don't care the kprobe which has gone. */
 973        if (p->post_handler && !kprobe_gone(p))
 974                ap->post_handler = aggr_post_handler;
 975        if (p->break_handler && !kprobe_gone(p))
 976                ap->break_handler = aggr_break_handler;
 977
 978        INIT_LIST_HEAD(&ap->list);
 979        INIT_HLIST_NODE(&ap->hlist);
 980
 981        list_add_rcu(&p->list, &ap->list);
 982        hlist_replace_rcu(&p->hlist, &ap->hlist);
 983}
 984
 985/*
 986 * This is the second or subsequent kprobe at the address - handle
 987 * the intricacies
 988 */
 989static int __kprobes register_aggr_kprobe(struct kprobe *old_p,
 990                                          struct kprobe *p)
 991{
 992        int ret = 0;
 993        struct kprobe *ap = old_p;
 994
 995        if (!kprobe_aggrprobe(old_p)) {
 996                /* If old_p is not an aggr_kprobe, create new aggr_kprobe. */
 997                ap = alloc_aggr_kprobe(old_p);
 998                if (!ap)
 999                        return -ENOMEM;
1000                init_aggr_kprobe(ap, old_p);
1001        }
1002
1003        if (kprobe_gone(ap)) {
1004                /*
1005                 * Attempting to insert new probe at the same location that
1006                 * had a probe in the module vaddr area which already
1007                 * freed. So, the instruction slot has already been
1008                 * released. We need a new slot for the new probe.
1009                 */
1010                ret = arch_prepare_kprobe(ap);
1011                if (ret)
1012                        /*
1013                         * Even if fail to allocate new slot, don't need to
1014                         * free aggr_probe. It will be used next time, or
1015                         * freed by unregister_kprobe.
1016                         */
1017                        return ret;
1018
1019                /* Prepare optimized instructions if possible. */
1020                prepare_optimized_kprobe(ap);
1021
1022                /*
1023                 * Clear gone flag to prevent allocating new slot again, and
1024                 * set disabled flag because it is not armed yet.
1025                 */
1026                ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1027                            | KPROBE_FLAG_DISABLED;
1028        }
1029
1030        /* Copy ap's insn slot to p */
1031        copy_kprobe(ap, p);
1032        return add_new_kprobe(ap, p);
1033}
1034
1035/* Try to disable aggr_kprobe, and return 1 if succeeded.*/
1036static int __kprobes try_to_disable_aggr_kprobe(struct kprobe *p)
1037{
1038        struct kprobe *kp;
1039
1040        list_for_each_entry_rcu(kp, &p->list, list) {
1041                if (!kprobe_disabled(kp))
1042                        /*
1043                         * There is an active probe on the list.
1044                         * We can't disable aggr_kprobe.
1045                         */
1046                        return 0;
1047        }
1048        p->flags |= KPROBE_FLAG_DISABLED;
1049        return 1;
1050}
1051
1052static int __kprobes in_kprobes_functions(unsigned long addr)
1053{
1054        struct kprobe_blackpoint *kb;
1055
1056        if (addr >= (unsigned long)__kprobes_text_start &&
1057            addr < (unsigned long)__kprobes_text_end)
1058                return -EINVAL;
1059        /*
1060         * If there exists a kprobe_blacklist, verify and
1061         * fail any probe registration in the prohibited area
1062         */
1063        for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1064                if (kb->start_addr) {
1065                        if (addr >= kb->start_addr &&
1066                            addr < (kb->start_addr + kb->range))
1067                                return -EINVAL;
1068                }
1069        }
1070        return 0;
1071}
1072
1073/*
1074 * If we have a symbol_name argument, look it up and add the offset field
1075 * to it. This way, we can specify a relative address to a symbol.
1076 */
1077static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p)
1078{
1079        kprobe_opcode_t *addr = p->addr;
1080        if (p->symbol_name) {
1081                if (addr)
1082                        return NULL;
1083                kprobe_lookup_name(p->symbol_name, addr);
1084        }
1085
1086        if (!addr)
1087                return NULL;
1088        return (kprobe_opcode_t *)(((char *)addr) + p->offset);
1089}
1090
1091/* Check passed kprobe is valid and return kprobe in kprobe_table. */
1092static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p)
1093{
1094        struct kprobe *old_p, *list_p;
1095
1096        old_p = get_kprobe(p->addr);
1097        if (unlikely(!old_p))
1098                return NULL;
1099
1100        if (p != old_p) {
1101                list_for_each_entry_rcu(list_p, &old_p->list, list)
1102                        if (list_p == p)
1103                        /* kprobe p is a valid probe */
1104                                goto valid;
1105                return NULL;
1106        }
1107valid:
1108        return old_p;
1109}
1110
1111/* Return error if the kprobe is being re-registered */
1112static inline int check_kprobe_rereg(struct kprobe *p)
1113{
1114        int ret = 0;
1115        struct kprobe *old_p;
1116
1117        mutex_lock(&kprobe_mutex);
1118        old_p = __get_valid_kprobe(p);
1119        if (old_p)
1120                ret = -EINVAL;
1121        mutex_unlock(&kprobe_mutex);
1122        return ret;
1123}
1124
1125int __kprobes register_kprobe(struct kprobe *p)
1126{
1127        int ret = 0;
1128        struct kprobe *old_p;
1129        struct module *probed_mod;
1130        kprobe_opcode_t *addr;
1131
1132        addr = kprobe_addr(p);
1133        if (!addr)
1134                return -EINVAL;
1135        p->addr = addr;
1136
1137        ret = check_kprobe_rereg(p);
1138        if (ret)
1139                return ret;
1140
1141        preempt_disable();
1142        if (!kernel_text_address((unsigned long) p->addr) ||
1143            in_kprobes_functions((unsigned long) p->addr) ||
1144            ftrace_text_reserved(p->addr, p->addr)) {
1145                preempt_enable();
1146                return -EINVAL;
1147        }
1148
1149        /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
1150        p->flags &= KPROBE_FLAG_DISABLED;
1151
1152        /*
1153         * Check if are we probing a module.
1154         */
1155        probed_mod = __module_text_address((unsigned long) p->addr);
1156        if (probed_mod) {
1157                /*
1158                 * We must hold a refcount of the probed module while updating
1159                 * its code to prohibit unexpected unloading.
1160                 */
1161                if (unlikely(!try_module_get(probed_mod))) {
1162                        preempt_enable();
1163                        return -EINVAL;
1164                }
1165                /*
1166                 * If the module freed .init.text, we couldn't insert
1167                 * kprobes in there.
1168                 */
1169                if (within_module_init((unsigned long)p->addr, probed_mod) &&
1170                    probed_mod->state != MODULE_STATE_COMING) {
1171                        module_put(probed_mod);
1172                        preempt_enable();
1173                        return -EINVAL;
1174                }
1175        }
1176        preempt_enable();
1177
1178        p->nmissed = 0;
1179        INIT_LIST_HEAD(&p->list);
1180        mutex_lock(&kprobe_mutex);
1181
1182        get_online_cpus();      /* For avoiding text_mutex deadlock. */
1183        mutex_lock(&text_mutex);
1184
1185        old_p = get_kprobe(p->addr);
1186        if (old_p) {
1187                /* Since this may unoptimize old_p, locking text_mutex. */
1188                ret = register_aggr_kprobe(old_p, p);
1189                goto out;
1190        }
1191
1192        ret = arch_prepare_kprobe(p);
1193        if (ret)
1194                goto out;
1195
1196        INIT_HLIST_NODE(&p->hlist);
1197        hlist_add_head_rcu(&p->hlist,
1198                       &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
1199
1200        if (!kprobes_all_disarmed && !kprobe_disabled(p))
1201                __arm_kprobe(p);
1202
1203        /* Try to optimize kprobe */
1204        try_to_optimize_kprobe(p);
1205
1206out:
1207        mutex_unlock(&text_mutex);
1208        put_online_cpus();
1209        mutex_unlock(&kprobe_mutex);
1210
1211        if (probed_mod)
1212                module_put(probed_mod);
1213
1214        return ret;
1215}
1216EXPORT_SYMBOL_GPL(register_kprobe);
1217
1218/*
1219 * Unregister a kprobe without a scheduler synchronization.
1220 */
1221static int __kprobes __unregister_kprobe_top(struct kprobe *p)
1222{
1223        struct kprobe *old_p, *list_p;
1224
1225        old_p = __get_valid_kprobe(p);
1226        if (old_p == NULL)
1227                return -EINVAL;
1228
1229        if (old_p == p ||
1230            (kprobe_aggrprobe(old_p) &&
1231             list_is_singular(&old_p->list))) {
1232                /*
1233                 * Only probe on the hash list. Disarm only if kprobes are
1234                 * enabled and not gone - otherwise, the breakpoint would
1235                 * already have been removed. We save on flushing icache.
1236                 */
1237                if (!kprobes_all_disarmed && !kprobe_disabled(old_p))
1238                        disarm_kprobe(old_p);
1239                hlist_del_rcu(&old_p->hlist);
1240        } else {
1241                if (p->break_handler && !kprobe_gone(p))
1242                        old_p->break_handler = NULL;
1243                if (p->post_handler && !kprobe_gone(p)) {
1244                        list_for_each_entry_rcu(list_p, &old_p->list, list) {
1245                                if ((list_p != p) && (list_p->post_handler))
1246                                        goto noclean;
1247                        }
1248                        old_p->post_handler = NULL;
1249                }
1250noclean:
1251                list_del_rcu(&p->list);
1252                if (!kprobe_disabled(old_p)) {
1253                        try_to_disable_aggr_kprobe(old_p);
1254                        if (!kprobes_all_disarmed) {
1255                                if (kprobe_disabled(old_p))
1256                                        disarm_kprobe(old_p);
1257                                else
1258                                        /* Try to optimize this probe again */
1259                                        optimize_kprobe(old_p);
1260                        }
1261                }
1262        }
1263        return 0;
1264}
1265
1266static void __kprobes __unregister_kprobe_bottom(struct kprobe *p)
1267{
1268        struct kprobe *old_p;
1269
1270        if (list_empty(&p->list))
1271                arch_remove_kprobe(p);
1272        else if (list_is_singular(&p->list)) {
1273                /* "p" is the last child of an aggr_kprobe */
1274                old_p = list_entry(p->list.next, struct kprobe, list);
1275                list_del(&p->list);
1276                arch_remove_kprobe(old_p);
1277                free_aggr_kprobe(old_p);
1278        }
1279}
1280
1281int __kprobes register_kprobes(struct kprobe **kps, int num)
1282{
1283        int i, ret = 0;
1284
1285        if (num <= 0)
1286                return -EINVAL;
1287        for (i = 0; i < num; i++) {
1288                ret = register_kprobe(kps[i]);
1289                if (ret < 0) {
1290                        if (i > 0)
1291                                unregister_kprobes(kps, i);
1292                        break;
1293                }
1294        }
1295        return ret;
1296}
1297EXPORT_SYMBOL_GPL(register_kprobes);
1298
1299void __kprobes unregister_kprobe(struct kprobe *p)
1300{
1301        unregister_kprobes(&p, 1);
1302}
1303EXPORT_SYMBOL_GPL(unregister_kprobe);
1304
1305void __kprobes unregister_kprobes(struct kprobe **kps, int num)
1306{
1307        int i;
1308
1309        if (num <= 0)
1310                return;
1311        mutex_lock(&kprobe_mutex);
1312        for (i = 0; i < num; i++)
1313                if (__unregister_kprobe_top(kps[i]) < 0)
1314                        kps[i]->addr = NULL;
1315        mutex_unlock(&kprobe_mutex);
1316
1317        synchronize_sched();
1318        for (i = 0; i < num; i++)
1319                if (kps[i]->addr)
1320                        __unregister_kprobe_bottom(kps[i]);
1321}
1322EXPORT_SYMBOL_GPL(unregister_kprobes);
1323
1324static struct notifier_block kprobe_exceptions_nb = {
1325        .notifier_call = kprobe_exceptions_notify,
1326        .priority = 0x7fffffff /* we need to be notified first */
1327};
1328
1329unsigned long __weak arch_deref_entry_point(void *entry)
1330{
1331        return (unsigned long)entry;
1332}
1333
1334int __kprobes register_jprobes(struct jprobe **jps, int num)
1335{
1336        struct jprobe *jp;
1337        int ret = 0, i;
1338
1339        if (num <= 0)
1340                return -EINVAL;
1341        for (i = 0; i < num; i++) {
1342                unsigned long addr;
1343                jp = jps[i];
1344                addr = arch_deref_entry_point(jp->entry);
1345
1346                if (!kernel_text_address(addr))
1347                        ret = -EINVAL;
1348                else {
1349                        /* Todo: Verify probepoint is a function entry point */
1350                        jp->kp.pre_handler = setjmp_pre_handler;
1351                        jp->kp.break_handler = longjmp_break_handler;
1352                        ret = register_kprobe(&jp->kp);
1353                }
1354                if (ret < 0) {
1355                        if (i > 0)
1356                                unregister_jprobes(jps, i);
1357                        break;
1358                }
1359        }
1360        return ret;
1361}
1362EXPORT_SYMBOL_GPL(register_jprobes);
1363
1364int __kprobes register_jprobe(struct jprobe *jp)
1365{
1366        return register_jprobes(&jp, 1);
1367}
1368EXPORT_SYMBOL_GPL(register_jprobe);
1369
1370void __kprobes unregister_jprobe(struct jprobe *jp)
1371{
1372        unregister_jprobes(&jp, 1);
1373}
1374EXPORT_SYMBOL_GPL(unregister_jprobe);
1375
1376void __kprobes unregister_jprobes(struct jprobe **jps, int num)
1377{
1378        int i;
1379
1380        if (num <= 0)
1381                return;
1382        mutex_lock(&kprobe_mutex);
1383        for (i = 0; i < num; i++)
1384                if (__unregister_kprobe_top(&jps[i]->kp) < 0)
1385                        jps[i]->kp.addr = NULL;
1386        mutex_unlock(&kprobe_mutex);
1387
1388        synchronize_sched();
1389        for (i = 0; i < num; i++) {
1390                if (jps[i]->kp.addr)
1391                        __unregister_kprobe_bottom(&jps[i]->kp);
1392        }
1393}
1394EXPORT_SYMBOL_GPL(unregister_jprobes);
1395
1396#ifdef CONFIG_KRETPROBES
1397/*
1398 * This kprobe pre_handler is registered with every kretprobe. When probe
1399 * hits it will set up the return probe.
1400 */
1401static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1402                                           struct pt_regs *regs)
1403{
1404        struct kretprobe *rp = container_of(p, struct kretprobe, kp);
1405        unsigned long hash, flags = 0;
1406        struct kretprobe_instance *ri;
1407
1408        /*TODO: consider to only swap the RA after the last pre_handler fired */
1409        hash = hash_ptr(current, KPROBE_HASH_BITS);
1410        spin_lock_irqsave(&rp->lock, flags);
1411        if (!hlist_empty(&rp->free_instances)) {
1412                ri = hlist_entry(rp->free_instances.first,
1413                                struct kretprobe_instance, hlist);
1414                hlist_del(&ri->hlist);
1415                spin_unlock_irqrestore(&rp->lock, flags);
1416
1417                ri->rp = rp;
1418                ri->task = current;
1419
1420                if (rp->entry_handler && rp->entry_handler(ri, regs))
1421                        return 0;
1422
1423                arch_prepare_kretprobe(ri, regs);
1424
1425                /* XXX(hch): why is there no hlist_move_head? */
1426                INIT_HLIST_NODE(&ri->hlist);
1427                kretprobe_table_lock(hash, &flags);
1428                hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]);
1429                kretprobe_table_unlock(hash, &flags);
1430        } else {
1431                rp->nmissed++;
1432                spin_unlock_irqrestore(&rp->lock, flags);
1433        }
1434        return 0;
1435}
1436
1437int __kprobes register_kretprobe(struct kretprobe *rp)
1438{
1439        int ret = 0;
1440        struct kretprobe_instance *inst;
1441        int i;
1442        void *addr;
1443
1444        if (kretprobe_blacklist_size) {
1445                addr = kprobe_addr(&rp->kp);
1446                if (!addr)
1447                        return -EINVAL;
1448
1449                for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1450                        if (kretprobe_blacklist[i].addr == addr)
1451                                return -EINVAL;
1452                }
1453        }
1454
1455        rp->kp.pre_handler = pre_handler_kretprobe;
1456        rp->kp.post_handler = NULL;
1457        rp->kp.fault_handler = NULL;
1458        rp->kp.break_handler = NULL;
1459
1460        /* Pre-allocate memory for max kretprobe instances */
1461        if (rp->maxactive <= 0) {
1462#ifdef CONFIG_PREEMPT
1463                rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
1464#else
1465                rp->maxactive = num_possible_cpus();
1466#endif
1467        }
1468        spin_lock_init(&rp->lock);
1469        INIT_HLIST_HEAD(&rp->free_instances);
1470        for (i = 0; i < rp->maxactive; i++) {
1471                inst = kmalloc(sizeof(struct kretprobe_instance) +
1472                               rp->data_size, GFP_KERNEL);
1473                if (inst == NULL) {
1474                        free_rp_inst(rp);
1475                        return -ENOMEM;
1476                }
1477                INIT_HLIST_NODE(&inst->hlist);
1478                hlist_add_head(&inst->hlist, &rp->free_instances);
1479        }
1480
1481        rp->nmissed = 0;
1482        /* Establish function entry probe point */
1483        ret = register_kprobe(&rp->kp);
1484        if (ret != 0)
1485                free_rp_inst(rp);
1486        return ret;
1487}
1488EXPORT_SYMBOL_GPL(register_kretprobe);
1489
1490int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1491{
1492        int ret = 0, i;
1493
1494        if (num <= 0)
1495                return -EINVAL;
1496        for (i = 0; i < num; i++) {
1497                ret = register_kretprobe(rps[i]);
1498                if (ret < 0) {
1499                        if (i > 0)
1500                                unregister_kretprobes(rps, i);
1501                        break;
1502                }
1503        }
1504        return ret;
1505}
1506EXPORT_SYMBOL_GPL(register_kretprobes);
1507
1508void __kprobes unregister_kretprobe(struct kretprobe *rp)
1509{
1510        unregister_kretprobes(&rp, 1);
1511}
1512EXPORT_SYMBOL_GPL(unregister_kretprobe);
1513
1514void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1515{
1516        int i;
1517
1518        if (num <= 0)
1519                return;
1520        mutex_lock(&kprobe_mutex);
1521        for (i = 0; i < num; i++)
1522                if (__unregister_kprobe_top(&rps[i]->kp) < 0)
1523                        rps[i]->kp.addr = NULL;
1524        mutex_unlock(&kprobe_mutex);
1525
1526        synchronize_sched();
1527        for (i = 0; i < num; i++) {
1528                if (rps[i]->kp.addr) {
1529                        __unregister_kprobe_bottom(&rps[i]->kp);
1530                        cleanup_rp_inst(rps[i]);
1531                }
1532        }
1533}
1534EXPORT_SYMBOL_GPL(unregister_kretprobes);
1535
1536#else /* CONFIG_KRETPROBES */
1537int __kprobes register_kretprobe(struct kretprobe *rp)
1538{
1539        return -ENOSYS;
1540}
1541EXPORT_SYMBOL_GPL(register_kretprobe);
1542
1543int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1544{
1545        return -ENOSYS;
1546}
1547EXPORT_SYMBOL_GPL(register_kretprobes);
1548
1549void __kprobes unregister_kretprobe(struct kretprobe *rp)
1550{
1551}
1552EXPORT_SYMBOL_GPL(unregister_kretprobe);
1553
1554void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1555{
1556}
1557EXPORT_SYMBOL_GPL(unregister_kretprobes);
1558
1559static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1560                                           struct pt_regs *regs)
1561{
1562        return 0;
1563}
1564
1565#endif /* CONFIG_KRETPROBES */
1566
1567/* Set the kprobe gone and remove its instruction buffer. */
1568static void __kprobes kill_kprobe(struct kprobe *p)
1569{
1570        struct kprobe *kp;
1571
1572        p->flags |= KPROBE_FLAG_GONE;
1573        if (kprobe_aggrprobe(p)) {
1574                /*
1575                 * If this is an aggr_kprobe, we have to list all the
1576                 * chained probes and mark them GONE.
1577                 */
1578                list_for_each_entry_rcu(kp, &p->list, list)
1579                        kp->flags |= KPROBE_FLAG_GONE;
1580                p->post_handler = NULL;
1581                p->break_handler = NULL;
1582                kill_optimized_kprobe(p);
1583        }
1584        /*
1585         * Here, we can remove insn_slot safely, because no thread calls
1586         * the original probed function (which will be freed soon) any more.
1587         */
1588        arch_remove_kprobe(p);
1589}
1590
1591/* Disable one kprobe */
1592int __kprobes disable_kprobe(struct kprobe *kp)
1593{
1594        int ret = 0;
1595        struct kprobe *p;
1596
1597        mutex_lock(&kprobe_mutex);
1598
1599        /* Check whether specified probe is valid. */
1600        p = __get_valid_kprobe(kp);
1601        if (unlikely(p == NULL)) {
1602                ret = -EINVAL;
1603                goto out;
1604        }
1605
1606        /* If the probe is already disabled (or gone), just return */
1607        if (kprobe_disabled(kp))
1608                goto out;
1609
1610        kp->flags |= KPROBE_FLAG_DISABLED;
1611        if (p != kp)
1612                /* When kp != p, p is always enabled. */
1613                try_to_disable_aggr_kprobe(p);
1614
1615        if (!kprobes_all_disarmed && kprobe_disabled(p))
1616                disarm_kprobe(p);
1617out:
1618        mutex_unlock(&kprobe_mutex);
1619        return ret;
1620}
1621EXPORT_SYMBOL_GPL(disable_kprobe);
1622
1623/* Enable one kprobe */
1624int __kprobes enable_kprobe(struct kprobe *kp)
1625{
1626        int ret = 0;
1627        struct kprobe *p;
1628
1629        mutex_lock(&kprobe_mutex);
1630
1631        /* Check whether specified probe is valid. */
1632        p = __get_valid_kprobe(kp);
1633        if (unlikely(p == NULL)) {
1634                ret = -EINVAL;
1635                goto out;
1636        }
1637
1638        if (kprobe_gone(kp)) {
1639                /* This kprobe has gone, we couldn't enable it. */
1640                ret = -EINVAL;
1641                goto out;
1642        }
1643
1644        if (p != kp)
1645                kp->flags &= ~KPROBE_FLAG_DISABLED;
1646
1647        if (!kprobes_all_disarmed && kprobe_disabled(p)) {
1648                p->flags &= ~KPROBE_FLAG_DISABLED;
1649                arm_kprobe(p);
1650        }
1651out:
1652        mutex_unlock(&kprobe_mutex);
1653        return ret;
1654}
1655EXPORT_SYMBOL_GPL(enable_kprobe);
1656
1657void __kprobes dump_kprobe(struct kprobe *kp)
1658{
1659        printk(KERN_WARNING "Dumping kprobe:\n");
1660        printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n",
1661               kp->symbol_name, kp->addr, kp->offset);
1662}
1663
1664/* Module notifier call back, checking kprobes on the module */
1665static int __kprobes kprobes_module_callback(struct notifier_block *nb,
1666                                             unsigned long val, void *data)
1667{
1668        struct module *mod = data;
1669        struct hlist_head *head;
1670        struct hlist_node *node;
1671        struct kprobe *p;
1672        unsigned int i;
1673        int checkcore = (val == MODULE_STATE_GOING);
1674
1675        if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
1676                return NOTIFY_DONE;
1677
1678        /*
1679         * When MODULE_STATE_GOING was notified, both of module .text and
1680         * .init.text sections would be freed. When MODULE_STATE_LIVE was
1681         * notified, only .init.text section would be freed. We need to
1682         * disable kprobes which have been inserted in the sections.
1683         */
1684        mutex_lock(&kprobe_mutex);
1685        for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1686                head = &kprobe_table[i];
1687                hlist_for_each_entry_rcu(p, node, head, hlist)
1688                        if (within_module_init((unsigned long)p->addr, mod) ||
1689                            (checkcore &&
1690                             within_module_core((unsigned long)p->addr, mod))) {
1691                                /*
1692                                 * The vaddr this probe is installed will soon
1693                                 * be vfreed buy not synced to disk. Hence,
1694                                 * disarming the breakpoint isn't needed.
1695                                 */
1696                                kill_kprobe(p);
1697                        }
1698        }
1699        mutex_unlock(&kprobe_mutex);
1700        return NOTIFY_DONE;
1701}
1702
1703static struct notifier_block kprobe_module_nb = {
1704        .notifier_call = kprobes_module_callback,
1705        .priority = 0
1706};
1707
1708static int __init init_kprobes(void)
1709{
1710        int i, err = 0;
1711        unsigned long offset = 0, size = 0;
1712        char *modname, namebuf[128];
1713        const char *symbol_name;
1714        void *addr;
1715        struct kprobe_blackpoint *kb;
1716
1717        /* FIXME allocate the probe table, currently defined statically */
1718        /* initialize all list heads */
1719        for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1720                INIT_HLIST_HEAD(&kprobe_table[i]);
1721                INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
1722                spin_lock_init(&(kretprobe_table_locks[i].lock));
1723        }
1724
1725        /*
1726         * Lookup and populate the kprobe_blacklist.
1727         *
1728         * Unlike the kretprobe blacklist, we'll need to determine
1729         * the range of addresses that belong to the said functions,
1730         * since a kprobe need not necessarily be at the beginning
1731         * of a function.
1732         */
1733        for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1734                kprobe_lookup_name(kb->name, addr);
1735                if (!addr)
1736                        continue;
1737
1738                kb->start_addr = (unsigned long)addr;
1739                symbol_name = kallsyms_lookup(kb->start_addr,
1740                                &size, &offset, &modname, namebuf);
1741                if (!symbol_name)
1742                        kb->range = 0;
1743                else
1744                        kb->range = size;
1745        }
1746
1747        if (kretprobe_blacklist_size) {
1748                /* lookup the function address from its name */
1749                for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1750                        kprobe_lookup_name(kretprobe_blacklist[i].name,
1751                                           kretprobe_blacklist[i].addr);
1752                        if (!kretprobe_blacklist[i].addr)
1753                                printk("kretprobe: lookup failed: %s\n",
1754                                       kretprobe_blacklist[i].name);
1755                }
1756        }
1757
1758#if defined(CONFIG_OPTPROBES)
1759#if defined(__ARCH_WANT_KPROBES_INSN_SLOT)
1760        /* Init kprobe_optinsn_slots */
1761        kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
1762#endif
1763        /* By default, kprobes can be optimized */
1764        kprobes_allow_optimization = true;
1765#endif
1766
1767        /* By default, kprobes are armed */
1768        kprobes_all_disarmed = false;
1769
1770        err = arch_init_kprobes();
1771        if (!err)
1772                err = register_die_notifier(&kprobe_exceptions_nb);
1773        if (!err)
1774                err = register_module_notifier(&kprobe_module_nb);
1775
1776        kprobes_initialized = (err == 0);
1777
1778        if (!err)
1779                init_test_probes();
1780        return err;
1781}
1782
1783#ifdef CONFIG_DEBUG_FS
1784static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p,
1785                const char *sym, int offset, char *modname, struct kprobe *pp)
1786{
1787        char *kprobe_type;
1788
1789        if (p->pre_handler == pre_handler_kretprobe)
1790                kprobe_type = "r";
1791        else if (p->pre_handler == setjmp_pre_handler)
1792                kprobe_type = "j";
1793        else
1794                kprobe_type = "k";
1795
1796        if (sym)
1797                seq_printf(pi, "%p  %s  %s+0x%x  %s ",
1798                        p->addr, kprobe_type, sym, offset,
1799                        (modname ? modname : " "));
1800        else
1801                seq_printf(pi, "%p  %s  %p ",
1802                        p->addr, kprobe_type, p->addr);
1803
1804        if (!pp)
1805                pp = p;
1806        seq_printf(pi, "%s%s%s\n",
1807                (kprobe_gone(p) ? "[GONE]" : ""),
1808                ((kprobe_disabled(p) && !kprobe_gone(p)) ?  "[DISABLED]" : ""),
1809                (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""));
1810}
1811
1812static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos)
1813{
1814        return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
1815}
1816
1817static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
1818{
1819        (*pos)++;
1820        if (*pos >= KPROBE_TABLE_SIZE)
1821                return NULL;
1822        return pos;
1823}
1824
1825static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v)
1826{
1827        /* Nothing to do */
1828}
1829
1830static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v)
1831{
1832        struct hlist_head *head;
1833        struct hlist_node *node;
1834        struct kprobe *p, *kp;
1835        const char *sym = NULL;
1836        unsigned int i = *(loff_t *) v;
1837        unsigned long offset = 0;
1838        char *modname, namebuf[128];
1839
1840        head = &kprobe_table[i];
1841        preempt_disable();
1842        hlist_for_each_entry_rcu(p, node, head, hlist) {
1843                sym = kallsyms_lookup((unsigned long)p->addr, NULL,
1844                                        &offset, &modname, namebuf);
1845                if (kprobe_aggrprobe(p)) {
1846                        list_for_each_entry_rcu(kp, &p->list, list)
1847                                report_probe(pi, kp, sym, offset, modname, p);
1848                } else
1849                        report_probe(pi, p, sym, offset, modname, NULL);
1850        }
1851        preempt_enable();
1852        return 0;
1853}
1854
1855static const struct seq_operations kprobes_seq_ops = {
1856        .start = kprobe_seq_start,
1857        .next  = kprobe_seq_next,
1858        .stop  = kprobe_seq_stop,
1859        .show  = show_kprobe_addr
1860};
1861
1862static int __kprobes kprobes_open(struct inode *inode, struct file *filp)
1863{
1864        return seq_open(filp, &kprobes_seq_ops);
1865}
1866
1867static const struct file_operations debugfs_kprobes_operations = {
1868        .open           = kprobes_open,
1869        .read           = seq_read,
1870        .llseek         = seq_lseek,
1871        .release        = seq_release,
1872};
1873
1874static void __kprobes arm_all_kprobes(void)
1875{
1876        struct hlist_head *head;
1877        struct hlist_node *node;
1878        struct kprobe *p;
1879        unsigned int i;
1880
1881        mutex_lock(&kprobe_mutex);
1882
1883        /* If kprobes are armed, just return */
1884        if (!kprobes_all_disarmed)
1885                goto already_enabled;
1886
1887        /* Arming kprobes doesn't optimize kprobe itself */
1888        mutex_lock(&text_mutex);
1889        for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1890                head = &kprobe_table[i];
1891                hlist_for_each_entry_rcu(p, node, head, hlist)
1892                        if (!kprobe_disabled(p))
1893                                __arm_kprobe(p);
1894        }
1895        mutex_unlock(&text_mutex);
1896
1897        kprobes_all_disarmed = false;
1898        printk(KERN_INFO "Kprobes globally enabled\n");
1899
1900already_enabled:
1901        mutex_unlock(&kprobe_mutex);
1902        return;
1903}
1904
1905static void __kprobes disarm_all_kprobes(void)
1906{
1907        struct hlist_head *head;
1908        struct hlist_node *node;
1909        struct kprobe *p;
1910        unsigned int i;
1911
1912        mutex_lock(&kprobe_mutex);
1913
1914        /* If kprobes are already disarmed, just return */
1915        if (kprobes_all_disarmed)
1916                goto already_disabled;
1917
1918        kprobes_all_disarmed = true;
1919        printk(KERN_INFO "Kprobes globally disabled\n");
1920
1921        /*
1922         * Here we call get_online_cpus() for avoiding text_mutex deadlock,
1923         * because disarming may also unoptimize kprobes.
1924         */
1925        get_online_cpus();
1926        mutex_lock(&text_mutex);
1927        for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1928                head = &kprobe_table[i];
1929                hlist_for_each_entry_rcu(p, node, head, hlist) {
1930                        if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p))
1931                                __disarm_kprobe(p);
1932                }
1933        }
1934
1935        mutex_unlock(&text_mutex);
1936        put_online_cpus();
1937        mutex_unlock(&kprobe_mutex);
1938        /* Allow all currently running kprobes to complete */
1939        synchronize_sched();
1940        return;
1941
1942already_disabled:
1943        mutex_unlock(&kprobe_mutex);
1944        return;
1945}
1946
1947/*
1948 * XXX: The debugfs bool file interface doesn't allow for callbacks
1949 * when the bool state is switched. We can reuse that facility when
1950 * available
1951 */
1952static ssize_t read_enabled_file_bool(struct file *file,
1953               char __user *user_buf, size_t count, loff_t *ppos)
1954{
1955        char buf[3];
1956
1957        if (!kprobes_all_disarmed)
1958                buf[0] = '1';
1959        else
1960                buf[0] = '0';
1961        buf[1] = '\n';
1962        buf[2] = 0x00;
1963        return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
1964}
1965
1966static ssize_t write_enabled_file_bool(struct file *file,
1967               const char __user *user_buf, size_t count, loff_t *ppos)
1968{
1969        char buf[32];
1970        int buf_size;
1971
1972        buf_size = min(count, (sizeof(buf)-1));
1973        if (copy_from_user(buf, user_buf, buf_size))
1974                return -EFAULT;
1975
1976        switch (buf[0]) {
1977        case 'y':
1978        case 'Y':
1979        case '1':
1980                arm_all_kprobes();
1981                break;
1982        case 'n':
1983        case 'N':
1984        case '0':
1985                disarm_all_kprobes();
1986                break;
1987        }
1988
1989        return count;
1990}
1991
1992static const struct file_operations fops_kp = {
1993        .read =         read_enabled_file_bool,
1994        .write =        write_enabled_file_bool,
1995};
1996
1997static int __kprobes debugfs_kprobe_init(void)
1998{
1999        struct dentry *dir, *file;
2000        unsigned int value = 1;
2001
2002        dir = debugfs_create_dir("kprobes", NULL);
2003        if (!dir)
2004                return -ENOMEM;
2005
2006        file = debugfs_create_file("list", 0444, dir, NULL,
2007                                &debugfs_kprobes_operations);
2008        if (!file) {
2009                debugfs_remove(dir);
2010                return -ENOMEM;
2011        }
2012
2013        file = debugfs_create_file("enabled", 0600, dir,
2014                                        &value, &fops_kp);
2015        if (!file) {
2016                debugfs_remove(dir);
2017                return -ENOMEM;
2018        }
2019
2020        return 0;
2021}
2022
2023late_initcall(debugfs_kprobe_init);
2024#endif /* CONFIG_DEBUG_FS */
2025
2026module_init(init_kprobes);
2027
2028/* defined in arch/.../kernel/kprobes.c */
2029EXPORT_SYMBOL_GPL(jprobe_return);
2030
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.