linux/kernel/trace/trace_uprobe.c
<<
>>
Prefs
   1/*
   2 * uprobes-based tracing events
   3 *
   4 * This program is free software; you can redistribute it and/or modify
   5 * it under the terms of the GNU General Public License version 2 as
   6 * published by the Free Software Foundation.
   7 *
   8 * This program is distributed in the hope that it will be useful,
   9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 * GNU General Public License for more details.
  12 *
  13 * You should have received a copy of the GNU General Public License
  14 * along with this program; if not, write to the Free Software
  15 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  16 *
  17 * Copyright (C) IBM Corporation, 2010-2012
  18 * Author:      Srikar Dronamraju <srikar@linux.vnet.ibm.com>
  19 */
  20
  21#include <linux/module.h>
  22#include <linux/uaccess.h>
  23#include <linux/uprobes.h>
  24#include <linux/namei.h>
  25#include <linux/string.h>
  26
  27#include "trace_probe.h"
  28
  29#define UPROBE_EVENT_SYSTEM     "uprobes"
  30
  31struct trace_uprobe_filter {
  32        rwlock_t                rwlock;
  33        int                     nr_systemwide;
  34        struct list_head        perf_events;
  35};
  36
  37/*
  38 * uprobe event core functions
  39 */
  40struct trace_uprobe {
  41        struct list_head                list;
  42        struct ftrace_event_class       class;
  43        struct ftrace_event_call        call;
  44        struct trace_uprobe_filter      filter;
  45        struct uprobe_consumer          consumer;
  46        struct inode                    *inode;
  47        char                            *filename;
  48        unsigned long                   offset;
  49        unsigned long                   nhit;
  50        unsigned int                    flags;  /* For TP_FLAG_* */
  51        ssize_t                         size;   /* trace entry size */
  52        unsigned int                    nr_args;
  53        struct probe_arg                args[];
  54};
  55
  56#define SIZEOF_TRACE_UPROBE(n)                  \
  57        (offsetof(struct trace_uprobe, args) +  \
  58        (sizeof(struct probe_arg) * (n)))
  59
  60static int register_uprobe_event(struct trace_uprobe *tu);
  61static void unregister_uprobe_event(struct trace_uprobe *tu);
  62
  63static DEFINE_MUTEX(uprobe_lock);
  64static LIST_HEAD(uprobe_list);
  65
  66static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs);
  67
  68static inline void init_trace_uprobe_filter(struct trace_uprobe_filter *filter)
  69{
  70        rwlock_init(&filter->rwlock);
  71        filter->nr_systemwide = 0;
  72        INIT_LIST_HEAD(&filter->perf_events);
  73}
  74
  75static inline bool uprobe_filter_is_empty(struct trace_uprobe_filter *filter)
  76{
  77        return !filter->nr_systemwide && list_empty(&filter->perf_events);
  78}
  79
  80/*
  81 * Allocate new trace_uprobe and initialize it (including uprobes).
  82 */
  83static struct trace_uprobe *
  84alloc_trace_uprobe(const char *group, const char *event, int nargs)
  85{
  86        struct trace_uprobe *tu;
  87
  88        if (!event || !is_good_name(event))
  89                return ERR_PTR(-EINVAL);
  90
  91        if (!group || !is_good_name(group))
  92                return ERR_PTR(-EINVAL);
  93
  94        tu = kzalloc(SIZEOF_TRACE_UPROBE(nargs), GFP_KERNEL);
  95        if (!tu)
  96                return ERR_PTR(-ENOMEM);
  97
  98        tu->call.class = &tu->class;
  99        tu->call.name = kstrdup(event, GFP_KERNEL);
 100        if (!tu->call.name)
 101                goto error;
 102
 103        tu->class.system = kstrdup(group, GFP_KERNEL);
 104        if (!tu->class.system)
 105                goto error;
 106
 107        INIT_LIST_HEAD(&tu->list);
 108        tu->consumer.handler = uprobe_dispatcher;
 109        init_trace_uprobe_filter(&tu->filter);
 110        return tu;
 111
 112error:
 113        kfree(tu->call.name);
 114        kfree(tu);
 115
 116        return ERR_PTR(-ENOMEM);
 117}
 118
 119static void free_trace_uprobe(struct trace_uprobe *tu)
 120{
 121        int i;
 122
 123        for (i = 0; i < tu->nr_args; i++)
 124                traceprobe_free_probe_arg(&tu->args[i]);
 125
 126        iput(tu->inode);
 127        kfree(tu->call.class->system);
 128        kfree(tu->call.name);
 129        kfree(tu->filename);
 130        kfree(tu);
 131}
 132
 133static struct trace_uprobe *find_probe_event(const char *event, const char *group)
 134{
 135        struct trace_uprobe *tu;
 136
 137        list_for_each_entry(tu, &uprobe_list, list)
 138                if (strcmp(tu->call.name, event) == 0 &&
 139                    strcmp(tu->call.class->system, group) == 0)
 140                        return tu;
 141
 142        return NULL;
 143}
 144
 145/* Unregister a trace_uprobe and probe_event: call with locking uprobe_lock */
 146static void unregister_trace_uprobe(struct trace_uprobe *tu)
 147{
 148        list_del(&tu->list);
 149        unregister_uprobe_event(tu);
 150        free_trace_uprobe(tu);
 151}
 152
 153/* Register a trace_uprobe and probe_event */
 154static int register_trace_uprobe(struct trace_uprobe *tu)
 155{
 156        struct trace_uprobe *old_tp;
 157        int ret;
 158
 159        mutex_lock(&uprobe_lock);
 160
 161        /* register as an event */
 162        old_tp = find_probe_event(tu->call.name, tu->call.class->system);
 163        if (old_tp)
 164                /* delete old event */
 165                unregister_trace_uprobe(old_tp);
 166
 167        ret = register_uprobe_event(tu);
 168        if (ret) {
 169                pr_warning("Failed to register probe event(%d)\n", ret);
 170                goto end;
 171        }
 172
 173        list_add_tail(&tu->list, &uprobe_list);
 174
 175end:
 176        mutex_unlock(&uprobe_lock);
 177
 178        return ret;
 179}
 180
 181/*
 182 * Argument syntax:
 183 *  - Add uprobe: p[:[GRP/]EVENT] PATH:SYMBOL[+offs] [FETCHARGS]
 184 *
 185 *  - Remove uprobe: -:[GRP/]EVENT
 186 */
 187static int create_trace_uprobe(int argc, char **argv)
 188{
 189        struct trace_uprobe *tu;
 190        struct inode *inode;
 191        char *arg, *event, *group, *filename;
 192        char buf[MAX_EVENT_NAME_LEN];
 193        struct path path;
 194        unsigned long offset;
 195        bool is_delete;
 196        int i, ret;
 197
 198        inode = NULL;
 199        ret = 0;
 200        is_delete = false;
 201        event = NULL;
 202        group = NULL;
 203
 204        /* argc must be >= 1 */
 205        if (argv[0][0] == '-')
 206                is_delete = true;
 207        else if (argv[0][0] != 'p') {
 208                pr_info("Probe definition must be started with 'p' or '-'.\n");
 209                return -EINVAL;
 210        }
 211
 212        if (argv[0][1] == ':') {
 213                event = &argv[0][2];
 214                arg = strchr(event, '/');
 215
 216                if (arg) {
 217                        group = event;
 218                        event = arg + 1;
 219                        event[-1] = '\0';
 220
 221                        if (strlen(group) == 0) {
 222                                pr_info("Group name is not specified\n");
 223                                return -EINVAL;
 224                        }
 225                }
 226                if (strlen(event) == 0) {
 227                        pr_info("Event name is not specified\n");
 228                        return -EINVAL;
 229                }
 230        }
 231        if (!group)
 232                group = UPROBE_EVENT_SYSTEM;
 233
 234        if (is_delete) {
 235                if (!event) {
 236                        pr_info("Delete command needs an event name.\n");
 237                        return -EINVAL;
 238                }
 239                mutex_lock(&uprobe_lock);
 240                tu = find_probe_event(event, group);
 241
 242                if (!tu) {
 243                        mutex_unlock(&uprobe_lock);
 244                        pr_info("Event %s/%s doesn't exist.\n", group, event);
 245                        return -ENOENT;
 246                }
 247                /* delete an event */
 248                unregister_trace_uprobe(tu);
 249                mutex_unlock(&uprobe_lock);
 250                return 0;
 251        }
 252
 253        if (argc < 2) {
 254                pr_info("Probe point is not specified.\n");
 255                return -EINVAL;
 256        }
 257        if (isdigit(argv[1][0])) {
 258                pr_info("probe point must be have a filename.\n");
 259                return -EINVAL;
 260        }
 261        arg = strchr(argv[1], ':');
 262        if (!arg)
 263                goto fail_address_parse;
 264
 265        *arg++ = '\0';
 266        filename = argv[1];
 267        ret = kern_path(filename, LOOKUP_FOLLOW, &path);
 268        if (ret)
 269                goto fail_address_parse;
 270
 271        inode = igrab(path.dentry->d_inode);
 272        path_put(&path);
 273
 274        if (!inode || !S_ISREG(inode->i_mode)) {
 275                ret = -EINVAL;
 276                goto fail_address_parse;
 277        }
 278
 279        ret = kstrtoul(arg, 0, &offset);
 280        if (ret)
 281                goto fail_address_parse;
 282
 283        argc -= 2;
 284        argv += 2;
 285
 286        /* setup a probe */
 287        if (!event) {
 288                char *tail;
 289                char *ptr;
 290
 291                tail = kstrdup(kbasename(filename), GFP_KERNEL);
 292                if (!tail) {
 293                        ret = -ENOMEM;
 294                        goto fail_address_parse;
 295                }
 296
 297                ptr = strpbrk(tail, ".-_");
 298                if (ptr)
 299                        *ptr = '\0';
 300
 301                snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_0x%lx", 'p', tail, offset);
 302                event = buf;
 303                kfree(tail);
 304        }
 305
 306        tu = alloc_trace_uprobe(group, event, argc);
 307        if (IS_ERR(tu)) {
 308                pr_info("Failed to allocate trace_uprobe.(%d)\n", (int)PTR_ERR(tu));
 309                ret = PTR_ERR(tu);
 310                goto fail_address_parse;
 311        }
 312        tu->offset = offset;
 313        tu->inode = inode;
 314        tu->filename = kstrdup(filename, GFP_KERNEL);
 315
 316        if (!tu->filename) {
 317                pr_info("Failed to allocate filename.\n");
 318                ret = -ENOMEM;
 319                goto error;
 320        }
 321
 322        /* parse arguments */
 323        ret = 0;
 324        for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
 325                /* Increment count for freeing args in error case */
 326                tu->nr_args++;
 327
 328                /* Parse argument name */
 329                arg = strchr(argv[i], '=');
 330                if (arg) {
 331                        *arg++ = '\0';
 332                        tu->args[i].name = kstrdup(argv[i], GFP_KERNEL);
 333                } else {
 334                        arg = argv[i];
 335                        /* If argument name is omitted, set "argN" */
 336                        snprintf(buf, MAX_EVENT_NAME_LEN, "arg%d", i + 1);
 337                        tu->args[i].name = kstrdup(buf, GFP_KERNEL);
 338                }
 339
 340                if (!tu->args[i].name) {
 341                        pr_info("Failed to allocate argument[%d] name.\n", i);
 342                        ret = -ENOMEM;
 343                        goto error;
 344                }
 345
 346                if (!is_good_name(tu->args[i].name)) {
 347                        pr_info("Invalid argument[%d] name: %s\n", i, tu->args[i].name);
 348                        ret = -EINVAL;
 349                        goto error;
 350                }
 351
 352                if (traceprobe_conflict_field_name(tu->args[i].name, tu->args, i)) {
 353                        pr_info("Argument[%d] name '%s' conflicts with "
 354                                "another field.\n", i, argv[i]);
 355                        ret = -EINVAL;
 356                        goto error;
 357                }
 358
 359                /* Parse fetch argument */
 360                ret = traceprobe_parse_probe_arg(arg, &tu->size, &tu->args[i], false, false);
 361                if (ret) {
 362                        pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
 363                        goto error;
 364                }
 365        }
 366
 367        ret = register_trace_uprobe(tu);
 368        if (ret)
 369                goto error;
 370        return 0;
 371
 372error:
 373        free_trace_uprobe(tu);
 374        return ret;
 375
 376fail_address_parse:
 377        if (inode)
 378                iput(inode);
 379
 380        pr_info("Failed to parse address or file.\n");
 381
 382        return ret;
 383}
 384
 385static void cleanup_all_probes(void)
 386{
 387        struct trace_uprobe *tu;
 388
 389        mutex_lock(&uprobe_lock);
 390        while (!list_empty(&uprobe_list)) {
 391                tu = list_entry(uprobe_list.next, struct trace_uprobe, list);
 392                unregister_trace_uprobe(tu);
 393        }
 394        mutex_unlock(&uprobe_lock);
 395}
 396
 397/* Probes listing interfaces */
 398static void *probes_seq_start(struct seq_file *m, loff_t *pos)
 399{
 400        mutex_lock(&uprobe_lock);
 401        return seq_list_start(&uprobe_list, *pos);
 402}
 403
 404static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos)
 405{
 406        return seq_list_next(v, &uprobe_list, pos);
 407}
 408
 409static void probes_seq_stop(struct seq_file *m, void *v)
 410{
 411        mutex_unlock(&uprobe_lock);
 412}
 413
 414static int probes_seq_show(struct seq_file *m, void *v)
 415{
 416        struct trace_uprobe *tu = v;
 417        int i;
 418
 419        seq_printf(m, "p:%s/%s", tu->call.class->system, tu->call.name);
 420        seq_printf(m, " %s:0x%p", tu->filename, (void *)tu->offset);
 421
 422        for (i = 0; i < tu->nr_args; i++)
 423                seq_printf(m, " %s=%s", tu->args[i].name, tu->args[i].comm);
 424
 425        seq_printf(m, "\n");
 426        return 0;
 427}
 428
 429static const struct seq_operations probes_seq_op = {
 430        .start  = probes_seq_start,
 431        .next   = probes_seq_next,
 432        .stop   = probes_seq_stop,
 433        .show   = probes_seq_show
 434};
 435
 436static int probes_open(struct inode *inode, struct file *file)
 437{
 438        if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))
 439                cleanup_all_probes();
 440
 441        return seq_open(file, &probes_seq_op);
 442}
 443
 444static ssize_t probes_write(struct file *file, const char __user *buffer,
 445                            size_t count, loff_t *ppos)
 446{
 447        return traceprobe_probes_write(file, buffer, count, ppos, create_trace_uprobe);
 448}
 449
 450static const struct file_operations uprobe_events_ops = {
 451        .owner          = THIS_MODULE,
 452        .open           = probes_open,
 453        .read           = seq_read,
 454        .llseek         = seq_lseek,
 455        .release        = seq_release,
 456        .write          = probes_write,
 457};
 458
 459/* Probes profiling interfaces */
 460static int probes_profile_seq_show(struct seq_file *m, void *v)
 461{
 462        struct trace_uprobe *tu = v;
 463
 464        seq_printf(m, "  %s %-44s %15lu\n", tu->filename, tu->call.name, tu->nhit);
 465        return 0;
 466}
 467
 468static const struct seq_operations profile_seq_op = {
 469        .start  = probes_seq_start,
 470        .next   = probes_seq_next,
 471        .stop   = probes_seq_stop,
 472        .show   = probes_profile_seq_show
 473};
 474
 475static int profile_open(struct inode *inode, struct file *file)
 476{
 477        return seq_open(file, &profile_seq_op);
 478}
 479
 480static const struct file_operations uprobe_profile_ops = {
 481        .owner          = THIS_MODULE,
 482        .open           = profile_open,
 483        .read           = seq_read,
 484        .llseek         = seq_lseek,
 485        .release        = seq_release,
 486};
 487
 488/* uprobe handler */
 489static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs)
 490{
 491        struct uprobe_trace_entry_head *entry;
 492        struct ring_buffer_event *event;
 493        struct ring_buffer *buffer;
 494        u8 *data;
 495        int size, i, pc;
 496        unsigned long irq_flags;
 497        struct ftrace_event_call *call = &tu->call;
 498
 499        local_save_flags(irq_flags);
 500        pc = preempt_count();
 501
 502        size = sizeof(*entry) + tu->size;
 503
 504        event = trace_current_buffer_lock_reserve(&buffer, call->event.type,
 505                                                  size, irq_flags, pc);
 506        if (!event)
 507                return 0;
 508
 509        entry = ring_buffer_event_data(event);
 510        entry->ip = instruction_pointer(task_pt_regs(current));
 511        data = (u8 *)&entry[1];
 512        for (i = 0; i < tu->nr_args; i++)
 513                call_fetch(&tu->args[i].fetch, regs, data + tu->args[i].offset);
 514
 515        if (!filter_current_check_discard(buffer, call, entry, event))
 516                trace_buffer_unlock_commit(buffer, event, irq_flags, pc);
 517
 518        return 0;
 519}
 520
 521/* Event entry printers */
 522static enum print_line_t
 523print_uprobe_event(struct trace_iterator *iter, int flags, struct trace_event *event)
 524{
 525        struct uprobe_trace_entry_head *field;
 526        struct trace_seq *s = &iter->seq;
 527        struct trace_uprobe *tu;
 528        u8 *data;
 529        int i;
 530
 531        field = (struct uprobe_trace_entry_head *)iter->ent;
 532        tu = container_of(event, struct trace_uprobe, call.event);
 533
 534        if (!trace_seq_printf(s, "%s: (", tu->call.name))
 535                goto partial;
 536
 537        if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET))
 538                goto partial;
 539
 540        if (!trace_seq_puts(s, ")"))
 541                goto partial;
 542
 543        data = (u8 *)&field[1];
 544        for (i = 0; i < tu->nr_args; i++) {
 545                if (!tu->args[i].type->print(s, tu->args[i].name,
 546                                             data + tu->args[i].offset, field))
 547                        goto partial;
 548        }
 549
 550        if (trace_seq_puts(s, "\n"))
 551                return TRACE_TYPE_HANDLED;
 552
 553partial:
 554        return TRACE_TYPE_PARTIAL_LINE;
 555}
 556
 557static inline bool is_trace_uprobe_enabled(struct trace_uprobe *tu)
 558{
 559        return tu->flags & (TP_FLAG_TRACE | TP_FLAG_PROFILE);
 560}
 561
 562typedef bool (*filter_func_t)(struct uprobe_consumer *self,
 563                                enum uprobe_filter_ctx ctx,
 564                                struct mm_struct *mm);
 565
 566static int
 567probe_event_enable(struct trace_uprobe *tu, int flag, filter_func_t filter)
 568{
 569        int ret = 0;
 570
 571        if (is_trace_uprobe_enabled(tu))
 572                return -EINTR;
 573
 574        WARN_ON(!uprobe_filter_is_empty(&tu->filter));
 575
 576        tu->flags |= flag;
 577        tu->consumer.filter = filter;
 578        ret = uprobe_register(tu->inode, tu->offset, &tu->consumer);
 579        if (ret)
 580                tu->flags &= ~flag;
 581
 582        return ret;
 583}
 584
 585static void probe_event_disable(struct trace_uprobe *tu, int flag)
 586{
 587        if (!is_trace_uprobe_enabled(tu))
 588                return;
 589
 590        WARN_ON(!uprobe_filter_is_empty(&tu->filter));
 591
 592        uprobe_unregister(tu->inode, tu->offset, &tu->consumer);
 593        tu->flags &= ~flag;
 594}
 595
 596static int uprobe_event_define_fields(struct ftrace_event_call *event_call)
 597{
 598        int ret, i;
 599        struct uprobe_trace_entry_head field;
 600        struct trace_uprobe *tu = (struct trace_uprobe *)event_call->data;
 601
 602        DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
 603        /* Set argument names as fields */
 604        for (i = 0; i < tu->nr_args; i++) {
 605                ret = trace_define_field(event_call, tu->args[i].type->fmttype,
 606                                         tu->args[i].name,
 607                                         sizeof(field) + tu->args[i].offset,
 608                                         tu->args[i].type->size,
 609                                         tu->args[i].type->is_signed,
 610                                         FILTER_OTHER);
 611
 612                if (ret)
 613                        return ret;
 614        }
 615        return 0;
 616}
 617
 618#define LEN_OR_ZERO             (len ? len - pos : 0)
 619static int __set_print_fmt(struct trace_uprobe *tu, char *buf, int len)
 620{
 621        const char *fmt, *arg;
 622        int i;
 623        int pos = 0;
 624
 625        fmt = "(%lx)";
 626        arg = "REC->" FIELD_STRING_IP;
 627
 628        /* When len=0, we just calculate the needed length */
 629
 630        pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt);
 631
 632        for (i = 0; i < tu->nr_args; i++) {
 633                pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=%s",
 634                                tu->args[i].name, tu->args[i].type->fmt);
 635        }
 636
 637        pos += snprintf(buf + pos, LEN_OR_ZERO, "\", %s", arg);
 638
 639        for (i = 0; i < tu->nr_args; i++) {
 640                pos += snprintf(buf + pos, LEN_OR_ZERO, ", REC->%s",
 641                                tu->args[i].name);
 642        }
 643
 644        return pos;     /* return the length of print_fmt */
 645}
 646#undef LEN_OR_ZERO
 647
 648static int set_print_fmt(struct trace_uprobe *tu)
 649{
 650        char *print_fmt;
 651        int len;
 652
 653        /* First: called with 0 length to calculate the needed length */
 654        len = __set_print_fmt(tu, NULL, 0);
 655        print_fmt = kmalloc(len + 1, GFP_KERNEL);
 656        if (!print_fmt)
 657                return -ENOMEM;
 658
 659        /* Second: actually write the @print_fmt */
 660        __set_print_fmt(tu, print_fmt, len + 1);
 661        tu->call.print_fmt = print_fmt;
 662
 663        return 0;
 664}
 665
 666#ifdef CONFIG_PERF_EVENTS
 667static bool
 668__uprobe_perf_filter(struct trace_uprobe_filter *filter, struct mm_struct *mm)
 669{
 670        struct perf_event *event;
 671
 672        if (filter->nr_systemwide)
 673                return true;
 674
 675        list_for_each_entry(event, &filter->perf_events, hw.tp_list) {
 676                if (event->hw.tp_target->mm == mm)
 677                        return true;
 678        }
 679
 680        return false;
 681}
 682
 683static inline bool
 684uprobe_filter_event(struct trace_uprobe *tu, struct perf_event *event)
 685{
 686        return __uprobe_perf_filter(&tu->filter, event->hw.tp_target->mm);
 687}
 688
 689static int uprobe_perf_open(struct trace_uprobe *tu, struct perf_event *event)
 690{
 691        bool done;
 692
 693        write_lock(&tu->filter.rwlock);
 694        if (event->hw.tp_target) {
 695                /*
 696                 * event->parent != NULL means copy_process(), we can avoid
 697                 * uprobe_apply(). current->mm must be probed and we can rely
 698                 * on dup_mmap() which preserves the already installed bp's.
 699                 *
 700                 * attr.enable_on_exec means that exec/mmap will install the
 701                 * breakpoints we need.
 702                 */
 703                done = tu->filter.nr_systemwide ||
 704                        event->parent || event->attr.enable_on_exec ||
 705                        uprobe_filter_event(tu, event);
 706                list_add(&event->hw.tp_list, &tu->filter.perf_events);
 707        } else {
 708                done = tu->filter.nr_systemwide;
 709                tu->filter.nr_systemwide++;
 710        }
 711        write_unlock(&tu->filter.rwlock);
 712
 713        if (!done)
 714                uprobe_apply(tu->inode, tu->offset, &tu->consumer, true);
 715
 716        return 0;
 717}
 718
 719static int uprobe_perf_close(struct trace_uprobe *tu, struct perf_event *event)
 720{
 721        bool done;
 722
 723        write_lock(&tu->filter.rwlock);
 724        if (event->hw.tp_target) {
 725                list_del(&event->hw.tp_list);
 726                done = tu->filter.nr_systemwide ||
 727                        (event->hw.tp_target->flags & PF_EXITING) ||
 728                        uprobe_filter_event(tu, event);
 729        } else {
 730                tu->filter.nr_systemwide--;
 731                done = tu->filter.nr_systemwide;
 732        }
 733        write_unlock(&tu->filter.rwlock);
 734
 735        if (!done)
 736                uprobe_apply(tu->inode, tu->offset, &tu->consumer, false);
 737
 738        return 0;
 739}
 740
 741static bool uprobe_perf_filter(struct uprobe_consumer *uc,
 742                                enum uprobe_filter_ctx ctx, struct mm_struct *mm)
 743{
 744        struct trace_uprobe *tu;
 745        int ret;
 746
 747        tu = container_of(uc, struct trace_uprobe, consumer);
 748        read_lock(&tu->filter.rwlock);
 749        ret = __uprobe_perf_filter(&tu->filter, mm);
 750        read_unlock(&tu->filter.rwlock);
 751
 752        return ret;
 753}
 754
 755/* uprobe profile handler */
 756static int uprobe_perf_func(struct trace_uprobe *tu, struct pt_regs *regs)
 757{
 758        struct ftrace_event_call *call = &tu->call;
 759        struct uprobe_trace_entry_head *entry;
 760        struct hlist_head *head;
 761        u8 *data;
 762        int size, __size, i;
 763        int rctx;
 764
 765        if (!uprobe_perf_filter(&tu->consumer, 0, current->mm))
 766                return UPROBE_HANDLER_REMOVE;
 767
 768        __size = sizeof(*entry) + tu->size;
 769        size = ALIGN(__size + sizeof(u32), sizeof(u64));
 770        size -= sizeof(u32);
 771        if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, "profile buffer not large enough"))
 772                return 0;
 773
 774        preempt_disable();
 775
 776        entry = perf_trace_buf_prepare(size, call->event.type, regs, &rctx);
 777        if (!entry)
 778                goto out;
 779
 780        entry->ip = instruction_pointer(task_pt_regs(current));
 781        data = (u8 *)&entry[1];
 782        for (i = 0; i < tu->nr_args; i++)
 783                call_fetch(&tu->args[i].fetch, regs, data + tu->args[i].offset);
 784
 785        head = this_cpu_ptr(call->perf_events);
 786        perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, head, NULL);
 787
 788 out:
 789        preempt_enable();
 790        return 0;
 791}
 792#endif  /* CONFIG_PERF_EVENTS */
 793
 794static
 795int trace_uprobe_register(struct ftrace_event_call *event, enum trace_reg type, void *data)
 796{
 797        struct trace_uprobe *tu = (struct trace_uprobe *)event->data;
 798
 799        switch (type) {
 800        case TRACE_REG_REGISTER:
 801                return probe_event_enable(tu, TP_FLAG_TRACE, NULL);
 802
 803        case TRACE_REG_UNREGISTER:
 804                probe_event_disable(tu, TP_FLAG_TRACE);
 805                return 0;
 806
 807#ifdef CONFIG_PERF_EVENTS
 808        case TRACE_REG_PERF_REGISTER:
 809                return probe_event_enable(tu, TP_FLAG_PROFILE, uprobe_perf_filter);
 810
 811        case TRACE_REG_PERF_UNREGISTER:
 812                probe_event_disable(tu, TP_FLAG_PROFILE);
 813                return 0;
 814
 815        case TRACE_REG_PERF_OPEN:
 816                return uprobe_perf_open(tu, data);
 817
 818        case TRACE_REG_PERF_CLOSE:
 819                return uprobe_perf_close(tu, data);
 820
 821#endif
 822        default:
 823                return 0;
 824        }
 825        return 0;
 826}
 827
 828static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs)
 829{
 830        struct trace_uprobe *tu;
 831        int ret = 0;
 832
 833        tu = container_of(con, struct trace_uprobe, consumer);
 834        tu->nhit++;
 835
 836        if (tu->flags & TP_FLAG_TRACE)
 837                ret |= uprobe_trace_func(tu, regs);
 838
 839#ifdef CONFIG_PERF_EVENTS
 840        if (tu->flags & TP_FLAG_PROFILE)
 841                ret |= uprobe_perf_func(tu, regs);
 842#endif
 843        return ret;
 844}
 845
 846static struct trace_event_functions uprobe_funcs = {
 847        .trace          = print_uprobe_event
 848};
 849
 850static int register_uprobe_event(struct trace_uprobe *tu)
 851{
 852        struct ftrace_event_call *call = &tu->call;
 853        int ret;
 854
 855        /* Initialize ftrace_event_call */
 856        INIT_LIST_HEAD(&call->class->fields);
 857        call->event.funcs = &uprobe_funcs;
 858        call->class->define_fields = uprobe_event_define_fields;
 859
 860        if (set_print_fmt(tu) < 0)
 861                return -ENOMEM;
 862
 863        ret = register_ftrace_event(&call->event);
 864        if (!ret) {
 865                kfree(call->print_fmt);
 866                return -ENODEV;
 867        }
 868        call->flags = 0;
 869        call->class->reg = trace_uprobe_register;
 870        call->data = tu;
 871        ret = trace_add_event_call(call);
 872
 873        if (ret) {
 874                pr_info("Failed to register uprobe event: %s\n", call->name);
 875                kfree(call->print_fmt);
 876                unregister_ftrace_event(&call->event);
 877        }
 878
 879        return ret;
 880}
 881
 882static void unregister_uprobe_event(struct trace_uprobe *tu)
 883{
 884        /* tu->event is unregistered in trace_remove_event_call() */
 885        trace_remove_event_call(&tu->call);
 886        kfree(tu->call.print_fmt);
 887        tu->call.print_fmt = NULL;
 888}
 889
 890/* Make a trace interface for controling probe points */
 891static __init int init_uprobe_trace(void)
 892{
 893        struct dentry *d_tracer;
 894
 895        d_tracer = tracing_init_dentry();
 896        if (!d_tracer)
 897                return 0;
 898
 899        trace_create_file("uprobe_events", 0644, d_tracer,
 900                                    NULL, &uprobe_events_ops);
 901        /* Profile interface */
 902        trace_create_file("uprobe_profile", 0444, d_tracer,
 903                                    NULL, &uprobe_profile_ops);
 904        return 0;
 905}
 906
 907fs_initcall(init_uprobe_trace);
 908
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.