linux/kernel/trace/trace.c
<<
>>
Prefs
   1/*
   2 * ring buffer based function tracer
   3 *
   4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
   6 *
   7 * Originally taken from the RT patch by:
   8 *    Arnaldo Carvalho de Melo <acme@redhat.com>
   9 *
  10 * Based on code from the latency_tracer, that is:
  11 *  Copyright (C) 2004-2006 Ingo Molnar
  12 *  Copyright (C) 2004 William Lee Irwin III
  13 */
  14#include <linux/ring_buffer.h>
  15#include <generated/utsrelease.h>
  16#include <linux/stacktrace.h>
  17#include <linux/writeback.h>
  18#include <linux/kallsyms.h>
  19#include <linux/seq_file.h>
  20#include <linux/notifier.h>
  21#include <linux/irqflags.h>
  22#include <linux/debugfs.h>
  23#include <linux/pagemap.h>
  24#include <linux/hardirq.h>
  25#include <linux/linkage.h>
  26#include <linux/uaccess.h>
  27#include <linux/kprobes.h>
  28#include <linux/ftrace.h>
  29#include <linux/module.h>
  30#include <linux/percpu.h>
  31#include <linux/splice.h>
  32#include <linux/kdebug.h>
  33#include <linux/string.h>
  34#include <linux/rwsem.h>
  35#include <linux/slab.h>
  36#include <linux/ctype.h>
  37#include <linux/init.h>
  38#include <linux/poll.h>
  39#include <linux/nmi.h>
  40#include <linux/fs.h>
  41
  42#include "trace.h"
  43#include "trace_output.h"
  44
  45/*
  46 * On boot up, the ring buffer is set to the minimum size, so that
  47 * we do not waste memory on systems that are not using tracing.
  48 */
  49int ring_buffer_expanded;
  50
  51/*
  52 * We need to change this state when a selftest is running.
  53 * A selftest will lurk into the ring-buffer to count the
  54 * entries inserted during the selftest although some concurrent
  55 * insertions into the ring-buffer such as trace_printk could occurred
  56 * at the same time, giving false positive or negative results.
  57 */
  58static bool __read_mostly tracing_selftest_running;
  59
  60/*
  61 * If a tracer is running, we do not want to run SELFTEST.
  62 */
  63bool __read_mostly tracing_selftest_disabled;
  64
  65/* For tracers that don't implement custom flags */
  66static struct tracer_opt dummy_tracer_opt[] = {
  67        { }
  68};
  69
  70static struct tracer_flags dummy_tracer_flags = {
  71        .val = 0,
  72        .opts = dummy_tracer_opt
  73};
  74
  75static int dummy_set_flag(u32 old_flags, u32 bit, int set)
  76{
  77        return 0;
  78}
  79
  80/*
  81 * Kill all tracing for good (never come back).
  82 * It is initialized to 1 but will turn to zero if the initialization
  83 * of the tracer is successful. But that is the only place that sets
  84 * this back to zero.
  85 */
  86static int tracing_disabled = 1;
  87
  88DEFINE_PER_CPU(int, ftrace_cpu_disabled);
  89
  90cpumask_var_t __read_mostly     tracing_buffer_mask;
  91
  92/*
  93 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
  94 *
  95 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
  96 * is set, then ftrace_dump is called. This will output the contents
  97 * of the ftrace buffers to the console.  This is very useful for
  98 * capturing traces that lead to crashes and outputing it to a
  99 * serial console.
 100 *
 101 * It is default off, but you can enable it with either specifying
 102 * "ftrace_dump_on_oops" in the kernel command line, or setting
 103 * /proc/sys/kernel/ftrace_dump_on_oops
 104 * Set 1 if you want to dump buffers of all CPUs
 105 * Set 2 if you want to dump the buffer of the CPU that triggered oops
 106 */
 107
 108enum ftrace_dump_mode ftrace_dump_on_oops;
 109
 110static int tracing_set_tracer(const char *buf);
 111
 112#define MAX_TRACER_SIZE         100
 113static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
 114static char *default_bootup_tracer;
 115
 116static int __init set_cmdline_ftrace(char *str)
 117{
 118        strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
 119        default_bootup_tracer = bootup_tracer_buf;
 120        /* We are using ftrace early, expand it */
 121        ring_buffer_expanded = 1;
 122        return 1;
 123}
 124__setup("ftrace=", set_cmdline_ftrace);
 125
 126static int __init set_ftrace_dump_on_oops(char *str)
 127{
 128        if (*str++ != '=' || !*str) {
 129                ftrace_dump_on_oops = DUMP_ALL;
 130                return 1;
 131        }
 132
 133        if (!strcmp("orig_cpu", str)) {
 134                ftrace_dump_on_oops = DUMP_ORIG;
 135                return 1;
 136        }
 137
 138        return 0;
 139}
 140__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
 141
 142unsigned long long ns2usecs(cycle_t nsec)
 143{
 144        nsec += 500;
 145        do_div(nsec, 1000);
 146        return nsec;
 147}
 148
 149/*
 150 * The global_trace is the descriptor that holds the tracing
 151 * buffers for the live tracing. For each CPU, it contains
 152 * a link list of pages that will store trace entries. The
 153 * page descriptor of the pages in the memory is used to hold
 154 * the link list by linking the lru item in the page descriptor
 155 * to each of the pages in the buffer per CPU.
 156 *
 157 * For each active CPU there is a data field that holds the
 158 * pages for the buffer for that CPU. Each CPU has the same number
 159 * of pages allocated for its buffer.
 160 */
 161static struct trace_array       global_trace;
 162
 163static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
 164
 165int filter_current_check_discard(struct ring_buffer *buffer,
 166                                 struct ftrace_event_call *call, void *rec,
 167                                 struct ring_buffer_event *event)
 168{
 169        return filter_check_discard(call, rec, buffer, event);
 170}
 171EXPORT_SYMBOL_GPL(filter_current_check_discard);
 172
 173cycle_t ftrace_now(int cpu)
 174{
 175        u64 ts;
 176
 177        /* Early boot up does not have a buffer yet */
 178        if (!global_trace.buffer)
 179                return trace_clock_local();
 180
 181        ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
 182        ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
 183
 184        return ts;
 185}
 186
 187/*
 188 * The max_tr is used to snapshot the global_trace when a maximum
 189 * latency is reached. Some tracers will use this to store a maximum
 190 * trace while it continues examining live traces.
 191 *
 192 * The buffers for the max_tr are set up the same as the global_trace.
 193 * When a snapshot is taken, the link list of the max_tr is swapped
 194 * with the link list of the global_trace and the buffers are reset for
 195 * the global_trace so the tracing can continue.
 196 */
 197static struct trace_array       max_tr;
 198
 199static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
 200
 201/* tracer_enabled is used to toggle activation of a tracer */
 202static int                      tracer_enabled = 1;
 203
 204/**
 205 * tracing_is_enabled - return tracer_enabled status
 206 *
 207 * This function is used by other tracers to know the status
 208 * of the tracer_enabled flag.  Tracers may use this function
 209 * to know if it should enable their features when starting
 210 * up. See irqsoff tracer for an example (start_irqsoff_tracer).
 211 */
 212int tracing_is_enabled(void)
 213{
 214        return tracer_enabled;
 215}
 216
 217/*
 218 * trace_buf_size is the size in bytes that is allocated
 219 * for a buffer. Note, the number of bytes is always rounded
 220 * to page size.
 221 *
 222 * This number is purposely set to a low number of 16384.
 223 * If the dump on oops happens, it will be much appreciated
 224 * to not have to wait for all that output. Anyway this can be
 225 * boot time and run time configurable.
 226 */
 227#define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
 228
 229static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
 230
 231/* trace_types holds a link list of available tracers. */
 232static struct tracer            *trace_types __read_mostly;
 233
 234/* current_trace points to the tracer that is currently active */
 235static struct tracer            *current_trace __read_mostly;
 236
 237/*
 238 * trace_types_lock is used to protect the trace_types list.
 239 */
 240static DEFINE_MUTEX(trace_types_lock);
 241
 242/*
 243 * serialize the access of the ring buffer
 244 *
 245 * ring buffer serializes readers, but it is low level protection.
 246 * The validity of the events (which returns by ring_buffer_peek() ..etc)
 247 * are not protected by ring buffer.
 248 *
 249 * The content of events may become garbage if we allow other process consumes
 250 * these events concurrently:
 251 *   A) the page of the consumed events may become a normal page
 252 *      (not reader page) in ring buffer, and this page will be rewrited
 253 *      by events producer.
 254 *   B) The page of the consumed events may become a page for splice_read,
 255 *      and this page will be returned to system.
 256 *
 257 * These primitives allow multi process access to different cpu ring buffer
 258 * concurrently.
 259 *
 260 * These primitives don't distinguish read-only and read-consume access.
 261 * Multi read-only access are also serialized.
 262 */
 263
 264#ifdef CONFIG_SMP
 265static DECLARE_RWSEM(all_cpu_access_lock);
 266static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
 267
 268static inline void trace_access_lock(int cpu)
 269{
 270        if (cpu == TRACE_PIPE_ALL_CPU) {
 271                /* gain it for accessing the whole ring buffer. */
 272                down_write(&all_cpu_access_lock);
 273        } else {
 274                /* gain it for accessing a cpu ring buffer. */
 275
 276                /* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
 277                down_read(&all_cpu_access_lock);
 278
 279                /* Secondly block other access to this @cpu ring buffer. */
 280                mutex_lock(&per_cpu(cpu_access_lock, cpu));
 281        }
 282}
 283
 284static inline void trace_access_unlock(int cpu)
 285{
 286        if (cpu == TRACE_PIPE_ALL_CPU) {
 287                up_write(&all_cpu_access_lock);
 288        } else {
 289                mutex_unlock(&per_cpu(cpu_access_lock, cpu));
 290                up_read(&all_cpu_access_lock);
 291        }
 292}
 293
 294static inline void trace_access_lock_init(void)
 295{
 296        int cpu;
 297
 298        for_each_possible_cpu(cpu)
 299                mutex_init(&per_cpu(cpu_access_lock, cpu));
 300}
 301
 302#else
 303
 304static DEFINE_MUTEX(access_lock);
 305
 306static inline void trace_access_lock(int cpu)
 307{
 308        (void)cpu;
 309        mutex_lock(&access_lock);
 310}
 311
 312static inline void trace_access_unlock(int cpu)
 313{
 314        (void)cpu;
 315        mutex_unlock(&access_lock);
 316}
 317
 318static inline void trace_access_lock_init(void)
 319{
 320}
 321
 322#endif
 323
 324/* trace_wait is a waitqueue for tasks blocked on trace_poll */
 325static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
 326
 327/* trace_flags holds trace_options default values */
 328unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
 329        TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
 330        TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
 331        TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS;
 332
 333static int trace_stop_count;
 334static DEFINE_RAW_SPINLOCK(tracing_start_lock);
 335
 336static void wakeup_work_handler(struct work_struct *work)
 337{
 338        wake_up(&trace_wait);
 339}
 340
 341static DECLARE_DELAYED_WORK(wakeup_work, wakeup_work_handler);
 342
 343/**
 344 * tracing_on - enable tracing buffers
 345 *
 346 * This function enables tracing buffers that may have been
 347 * disabled with tracing_off.
 348 */
 349void tracing_on(void)
 350{
 351        if (global_trace.buffer)
 352                ring_buffer_record_on(global_trace.buffer);
 353        /*
 354         * This flag is only looked at when buffers haven't been
 355         * allocated yet. We don't really care about the race
 356         * between setting this flag and actually turning
 357         * on the buffer.
 358         */
 359        global_trace.buffer_disabled = 0;
 360}
 361EXPORT_SYMBOL_GPL(tracing_on);
 362
 363/**
 364 * tracing_off - turn off tracing buffers
 365 *
 366 * This function stops the tracing buffers from recording data.
 367 * It does not disable any overhead the tracers themselves may
 368 * be causing. This function simply causes all recording to
 369 * the ring buffers to fail.
 370 */
 371void tracing_off(void)
 372{
 373        if (global_trace.buffer)
 374                ring_buffer_record_off(global_trace.buffer);
 375        /*
 376         * This flag is only looked at when buffers haven't been
 377         * allocated yet. We don't really care about the race
 378         * between setting this flag and actually turning
 379         * on the buffer.
 380         */
 381        global_trace.buffer_disabled = 1;
 382}
 383EXPORT_SYMBOL_GPL(tracing_off);
 384
 385/**
 386 * tracing_is_on - show state of ring buffers enabled
 387 */
 388int tracing_is_on(void)
 389{
 390        if (global_trace.buffer)
 391                return ring_buffer_record_is_on(global_trace.buffer);
 392        return !global_trace.buffer_disabled;
 393}
 394EXPORT_SYMBOL_GPL(tracing_is_on);
 395
 396/**
 397 * trace_wake_up - wake up tasks waiting for trace input
 398 *
 399 * Schedules a delayed work to wake up any task that is blocked on the
 400 * trace_wait queue. These is used with trace_poll for tasks polling the
 401 * trace.
 402 */
 403void trace_wake_up(void)
 404{
 405        const unsigned long delay = msecs_to_jiffies(2);
 406
 407        if (trace_flags & TRACE_ITER_BLOCK)
 408                return;
 409        schedule_delayed_work(&wakeup_work, delay);
 410}
 411
 412static int __init set_buf_size(char *str)
 413{
 414        unsigned long buf_size;
 415
 416        if (!str)
 417                return 0;
 418        buf_size = memparse(str, &str);
 419        /* nr_entries can not be zero */
 420        if (buf_size == 0)
 421                return 0;
 422        trace_buf_size = buf_size;
 423        return 1;
 424}
 425__setup("trace_buf_size=", set_buf_size);
 426
 427static int __init set_tracing_thresh(char *str)
 428{
 429        unsigned long threshold;
 430        int ret;
 431
 432        if (!str)
 433                return 0;
 434        ret = strict_strtoul(str, 0, &threshold);
 435        if (ret < 0)
 436                return 0;
 437        tracing_thresh = threshold * 1000;
 438        return 1;
 439}
 440__setup("tracing_thresh=", set_tracing_thresh);
 441
 442unsigned long nsecs_to_usecs(unsigned long nsecs)
 443{
 444        return nsecs / 1000;
 445}
 446
 447/* These must match the bit postions in trace_iterator_flags */
 448static const char *trace_options[] = {
 449        "print-parent",
 450        "sym-offset",
 451        "sym-addr",
 452        "verbose",
 453        "raw",
 454        "hex",
 455        "bin",
 456        "block",
 457        "stacktrace",
 458        "trace_printk",
 459        "ftrace_preempt",
 460        "branch",
 461        "annotate",
 462        "userstacktrace",
 463        "sym-userobj",
 464        "printk-msg-only",
 465        "context-info",
 466        "latency-format",
 467        "sleep-time",
 468        "graph-time",
 469        "record-cmd",
 470        "overwrite",
 471        "disable_on_free",
 472        "irq-info",
 473        "markers",
 474        NULL
 475};
 476
 477static struct {
 478        u64 (*func)(void);
 479        const char *name;
 480} trace_clocks[] = {
 481        { trace_clock_local,    "local" },
 482        { trace_clock_global,   "global" },
 483        { trace_clock_counter,  "counter" },
 484};
 485
 486int trace_clock_id;
 487
 488/*
 489 * trace_parser_get_init - gets the buffer for trace parser
 490 */
 491int trace_parser_get_init(struct trace_parser *parser, int size)
 492{
 493        memset(parser, 0, sizeof(*parser));
 494
 495        parser->buffer = kmalloc(size, GFP_KERNEL);
 496        if (!parser->buffer)
 497                return 1;
 498
 499        parser->size = size;
 500        return 0;
 501}
 502
 503/*
 504 * trace_parser_put - frees the buffer for trace parser
 505 */
 506void trace_parser_put(struct trace_parser *parser)
 507{
 508        kfree(parser->buffer);
 509}
 510
 511/*
 512 * trace_get_user - reads the user input string separated by  space
 513 * (matched by isspace(ch))
 514 *
 515 * For each string found the 'struct trace_parser' is updated,
 516 * and the function returns.
 517 *
 518 * Returns number of bytes read.
 519 *
 520 * See kernel/trace/trace.h for 'struct trace_parser' details.
 521 */
 522int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
 523        size_t cnt, loff_t *ppos)
 524{
 525        char ch;
 526        size_t read = 0;
 527        ssize_t ret;
 528
 529        if (!*ppos)
 530                trace_parser_clear(parser);
 531
 532        ret = get_user(ch, ubuf++);
 533        if (ret)
 534                goto out;
 535
 536        read++;
 537        cnt--;
 538
 539        /*
 540         * The parser is not finished with the last write,
 541         * continue reading the user input without skipping spaces.
 542         */
 543        if (!parser->cont) {
 544                /* skip white space */
 545                while (cnt && isspace(ch)) {
 546                        ret = get_user(ch, ubuf++);
 547                        if (ret)
 548                                goto out;
 549                        read++;
 550                        cnt--;
 551                }
 552
 553                /* only spaces were written */
 554                if (isspace(ch)) {
 555                        *ppos += read;
 556                        ret = read;
 557                        goto out;
 558                }
 559
 560                parser->idx = 0;
 561        }
 562
 563        /* read the non-space input */
 564        while (cnt && !isspace(ch)) {
 565                if (parser->idx < parser->size - 1)
 566                        parser->buffer[parser->idx++] = ch;
 567                else {
 568                        ret = -EINVAL;
 569                        goto out;
 570                }
 571                ret = get_user(ch, ubuf++);
 572                if (ret)
 573                        goto out;
 574                read++;
 575                cnt--;
 576        }
 577
 578        /* We either got finished input or we have to wait for another call. */
 579        if (isspace(ch)) {
 580                parser->buffer[parser->idx] = 0;
 581                parser->cont = false;
 582        } else {
 583                parser->cont = true;
 584                parser->buffer[parser->idx++] = ch;
 585        }
 586
 587        *ppos += read;
 588        ret = read;
 589
 590out:
 591        return ret;
 592}
 593
 594ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
 595{
 596        int len;
 597        int ret;
 598
 599        if (!cnt)
 600                return 0;
 601
 602        if (s->len <= s->readpos)
 603                return -EBUSY;
 604
 605        len = s->len - s->readpos;
 606        if (cnt > len)
 607                cnt = len;
 608        ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
 609        if (ret == cnt)
 610                return -EFAULT;
 611
 612        cnt -= ret;
 613
 614        s->readpos += cnt;
 615        return cnt;
 616}
 617
 618static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
 619{
 620        int len;
 621
 622        if (s->len <= s->readpos)
 623                return -EBUSY;
 624
 625        len = s->len - s->readpos;
 626        if (cnt > len)
 627                cnt = len;
 628        memcpy(buf, s->buffer + s->readpos, cnt);
 629
 630        s->readpos += cnt;
 631        return cnt;
 632}
 633
 634/*
 635 * ftrace_max_lock is used to protect the swapping of buffers
 636 * when taking a max snapshot. The buffers themselves are
 637 * protected by per_cpu spinlocks. But the action of the swap
 638 * needs its own lock.
 639 *
 640 * This is defined as a arch_spinlock_t in order to help
 641 * with performance when lockdep debugging is enabled.
 642 *
 643 * It is also used in other places outside the update_max_tr
 644 * so it needs to be defined outside of the
 645 * CONFIG_TRACER_MAX_TRACE.
 646 */
 647static arch_spinlock_t ftrace_max_lock =
 648        (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
 649
 650unsigned long __read_mostly     tracing_thresh;
 651
 652#ifdef CONFIG_TRACER_MAX_TRACE
 653unsigned long __read_mostly     tracing_max_latency;
 654
 655/*
 656 * Copy the new maximum trace into the separate maximum-trace
 657 * structure. (this way the maximum trace is permanently saved,
 658 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
 659 */
 660static void
 661__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
 662{
 663        struct trace_array_cpu *data = tr->data[cpu];
 664        struct trace_array_cpu *max_data;
 665
 666        max_tr.cpu = cpu;
 667        max_tr.time_start = data->preempt_timestamp;
 668
 669        max_data = max_tr.data[cpu];
 670        max_data->saved_latency = tracing_max_latency;
 671        max_data->critical_start = data->critical_start;
 672        max_data->critical_end = data->critical_end;
 673
 674        memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
 675        max_data->pid = tsk->pid;
 676        max_data->uid = task_uid(tsk);
 677        max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
 678        max_data->policy = tsk->policy;
 679        max_data->rt_priority = tsk->rt_priority;
 680
 681        /* record this tasks comm */
 682        tracing_record_cmdline(tsk);
 683}
 684
 685/**
 686 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
 687 * @tr: tracer
 688 * @tsk: the task with the latency
 689 * @cpu: The cpu that initiated the trace.
 690 *
 691 * Flip the buffers between the @tr and the max_tr and record information
 692 * about which task was the cause of this latency.
 693 */
 694void
 695update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
 696{
 697        struct ring_buffer *buf = tr->buffer;
 698
 699        if (trace_stop_count)
 700                return;
 701
 702        WARN_ON_ONCE(!irqs_disabled());
 703        if (!current_trace->use_max_tr) {
 704                WARN_ON_ONCE(1);
 705                return;
 706        }
 707        arch_spin_lock(&ftrace_max_lock);
 708
 709        tr->buffer = max_tr.buffer;
 710        max_tr.buffer = buf;
 711
 712        __update_max_tr(tr, tsk, cpu);
 713        arch_spin_unlock(&ftrace_max_lock);
 714}
 715
 716/**
 717 * update_max_tr_single - only copy one trace over, and reset the rest
 718 * @tr - tracer
 719 * @tsk - task with the latency
 720 * @cpu - the cpu of the buffer to copy.
 721 *
 722 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
 723 */
 724void
 725update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
 726{
 727        int ret;
 728
 729        if (trace_stop_count)
 730                return;
 731
 732        WARN_ON_ONCE(!irqs_disabled());
 733        if (!current_trace->use_max_tr) {
 734                WARN_ON_ONCE(1);
 735                return;
 736        }
 737
 738        arch_spin_lock(&ftrace_max_lock);
 739
 740        ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
 741
 742        if (ret == -EBUSY) {
 743                /*
 744                 * We failed to swap the buffer due to a commit taking
 745                 * place on this CPU. We fail to record, but we reset
 746                 * the max trace buffer (no one writes directly to it)
 747                 * and flag that it failed.
 748                 */
 749                trace_array_printk(&max_tr, _THIS_IP_,
 750                        "Failed to swap buffers due to commit in progress\n");
 751        }
 752
 753        WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
 754
 755        __update_max_tr(tr, tsk, cpu);
 756        arch_spin_unlock(&ftrace_max_lock);
 757}
 758#endif /* CONFIG_TRACER_MAX_TRACE */
 759
 760/**
 761 * register_tracer - register a tracer with the ftrace system.
 762 * @type - the plugin for the tracer
 763 *
 764 * Register a new plugin tracer.
 765 */
 766int register_tracer(struct tracer *type)
 767{
 768        struct tracer *t;
 769        int ret = 0;
 770
 771        if (!type->name) {
 772                pr_info("Tracer must have a name\n");
 773                return -1;
 774        }
 775
 776        if (strlen(type->name) >= MAX_TRACER_SIZE) {
 777                pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
 778                return -1;
 779        }
 780
 781        mutex_lock(&trace_types_lock);
 782
 783        tracing_selftest_running = true;
 784
 785        for (t = trace_types; t; t = t->next) {
 786                if (strcmp(type->name, t->name) == 0) {
 787                        /* already found */
 788                        pr_info("Tracer %s already registered\n",
 789                                type->name);
 790                        ret = -1;
 791                        goto out;
 792                }
 793        }
 794
 795        if (!type->set_flag)
 796                type->set_flag = &dummy_set_flag;
 797        if (!type->flags)
 798                type->flags = &dummy_tracer_flags;
 799        else
 800                if (!type->flags->opts)
 801                        type->flags->opts = dummy_tracer_opt;
 802        if (!type->wait_pipe)
 803                type->wait_pipe = default_wait_pipe;
 804
 805
 806#ifdef CONFIG_FTRACE_STARTUP_TEST
 807        if (type->selftest && !tracing_selftest_disabled) {
 808                struct tracer *saved_tracer = current_trace;
 809                struct trace_array *tr = &global_trace;
 810
 811                /*
 812                 * Run a selftest on this tracer.
 813                 * Here we reset the trace buffer, and set the current
 814                 * tracer to be this tracer. The tracer can then run some
 815                 * internal tracing to verify that everything is in order.
 816                 * If we fail, we do not register this tracer.
 817                 */
 818                tracing_reset_online_cpus(tr);
 819
 820                current_trace = type;
 821
 822                /* If we expanded the buffers, make sure the max is expanded too */
 823                if (ring_buffer_expanded && type->use_max_tr)
 824                        ring_buffer_resize(max_tr.buffer, trace_buf_size,
 825                                                RING_BUFFER_ALL_CPUS);
 826
 827                /* the test is responsible for initializing and enabling */
 828                pr_info("Testing tracer %s: ", type->name);
 829                ret = type->selftest(type, tr);
 830                /* the test is responsible for resetting too */
 831                current_trace = saved_tracer;
 832                if (ret) {
 833                        printk(KERN_CONT "FAILED!\n");
 834                        /* Add the warning after printing 'FAILED' */
 835                        WARN_ON(1);
 836                        goto out;
 837                }
 838                /* Only reset on passing, to avoid touching corrupted buffers */
 839                tracing_reset_online_cpus(tr);
 840
 841                /* Shrink the max buffer again */
 842                if (ring_buffer_expanded && type->use_max_tr)
 843                        ring_buffer_resize(max_tr.buffer, 1,
 844                                                RING_BUFFER_ALL_CPUS);
 845
 846                printk(KERN_CONT "PASSED\n");
 847        }
 848#endif
 849
 850        type->next = trace_types;
 851        trace_types = type;
 852
 853 out:
 854        tracing_selftest_running = false;
 855        mutex_unlock(&trace_types_lock);
 856
 857        if (ret || !default_bootup_tracer)
 858                goto out_unlock;
 859
 860        if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
 861                goto out_unlock;
 862
 863        printk(KERN_INFO "Starting tracer '%s'\n", type->name);
 864        /* Do we want this tracer to start on bootup? */
 865        tracing_set_tracer(type->name);
 866        default_bootup_tracer = NULL;
 867        /* disable other selftests, since this will break it. */
 868        tracing_selftest_disabled = 1;
 869#ifdef CONFIG_FTRACE_STARTUP_TEST
 870        printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
 871               type->name);
 872#endif
 873
 874 out_unlock:
 875        return ret;
 876}
 877
 878void unregister_tracer(struct tracer *type)
 879{
 880        struct tracer **t;
 881
 882        mutex_lock(&trace_types_lock);
 883        for (t = &trace_types; *t; t = &(*t)->next) {
 884                if (*t == type)
 885                        goto found;
 886        }
 887        pr_info("Tracer %s not registered\n", type->name);
 888        goto out;
 889
 890 found:
 891        *t = (*t)->next;
 892
 893        if (type == current_trace && tracer_enabled) {
 894                tracer_enabled = 0;
 895                tracing_stop();
 896                if (current_trace->stop)
 897                        current_trace->stop(&global_trace);
 898                current_trace = &nop_trace;
 899        }
 900out:
 901        mutex_unlock(&trace_types_lock);
 902}
 903
 904void tracing_reset(struct trace_array *tr, int cpu)
 905{
 906        struct ring_buffer *buffer = tr->buffer;
 907
 908        ring_buffer_record_disable(buffer);
 909
 910        /* Make sure all commits have finished */
 911        synchronize_sched();
 912        ring_buffer_reset_cpu(buffer, cpu);
 913
 914        ring_buffer_record_enable(buffer);
 915}
 916
 917void tracing_reset_online_cpus(struct trace_array *tr)
 918{
 919        struct ring_buffer *buffer = tr->buffer;
 920        int cpu;
 921
 922        ring_buffer_record_disable(buffer);
 923
 924        /* Make sure all commits have finished */
 925        synchronize_sched();
 926
 927        tr->time_start = ftrace_now(tr->cpu);
 928
 929        for_each_online_cpu(cpu)
 930                ring_buffer_reset_cpu(buffer, cpu);
 931
 932        ring_buffer_record_enable(buffer);
 933}
 934
 935void tracing_reset_current(int cpu)
 936{
 937        tracing_reset(&global_trace, cpu);
 938}
 939
 940void tracing_reset_current_online_cpus(void)
 941{
 942        tracing_reset_online_cpus(&global_trace);
 943}
 944
 945#define SAVED_CMDLINES 128
 946#define NO_CMDLINE_MAP UINT_MAX
 947static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
 948static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
 949static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
 950static int cmdline_idx;
 951static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
 952
 953/* temporary disable recording */
 954static atomic_t trace_record_cmdline_disabled __read_mostly;
 955
 956static void trace_init_cmdlines(void)
 957{
 958        memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
 959        memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
 960        cmdline_idx = 0;
 961}
 962
 963int is_tracing_stopped(void)
 964{
 965        return trace_stop_count;
 966}
 967
 968/**
 969 * ftrace_off_permanent - disable all ftrace code permanently
 970 *
 971 * This should only be called when a serious anomally has
 972 * been detected.  This will turn off the function tracing,
 973 * ring buffers, and other tracing utilites. It takes no
 974 * locks and can be called from any context.
 975 */
 976void ftrace_off_permanent(void)
 977{
 978        tracing_disabled = 1;
 979        ftrace_stop();
 980        tracing_off_permanent();
 981}
 982
 983/**
 984 * tracing_start - quick start of the tracer
 985 *
 986 * If tracing is enabled but was stopped by tracing_stop,
 987 * this will start the tracer back up.
 988 */
 989void tracing_start(void)
 990{
 991        struct ring_buffer *buffer;
 992        unsigned long flags;
 993
 994        if (tracing_disabled)
 995                return;
 996
 997        raw_spin_lock_irqsave(&tracing_start_lock, flags);
 998        if (--trace_stop_count) {
 999                if (trace_stop_count < 0) {
1000                        /* Someone screwed up their debugging */
1001                        WARN_ON_ONCE(1);
1002                        trace_stop_count = 0;
1003                }
1004                goto out;
1005        }
1006
1007        /* Prevent the buffers from switching */
1008        arch_spin_lock(&ftrace_max_lock);
1009
1010        buffer = global_trace.buffer;
1011        if (buffer)
1012                ring_buffer_record_enable(buffer);
1013
1014        buffer = max_tr.buffer;
1015        if (buffer)
1016                ring_buffer_record_enable(buffer);
1017
1018        arch_spin_unlock(&ftrace_max_lock);
1019
1020        ftrace_start();
1021 out:
1022        raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1023}
1024
1025/**
1026 * tracing_stop - quick stop of the tracer
1027 *
1028 * Light weight way to stop tracing. Use in conjunction with
1029 * tracing_start.
1030 */
1031void tracing_stop(void)
1032{
1033        struct ring_buffer *buffer;
1034        unsigned long flags;
1035
1036        ftrace_stop();
1037        raw_spin_lock_irqsave(&tracing_start_lock, flags);
1038        if (trace_stop_count++)
1039                goto out;
1040
1041        /* Prevent the buffers from switching */
1042        arch_spin_lock(&ftrace_max_lock);
1043
1044        buffer = global_trace.buffer;
1045        if (buffer)
1046                ring_buffer_record_disable(buffer);
1047
1048        buffer = max_tr.buffer;
1049        if (buffer)
1050                ring_buffer_record_disable(buffer);
1051
1052        arch_spin_unlock(&ftrace_max_lock);
1053
1054 out:
1055        raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1056}
1057
1058void trace_stop_cmdline_recording(void);
1059
1060static void trace_save_cmdline(struct task_struct *tsk)
1061{
1062        unsigned pid, idx;
1063
1064        if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1065                return;
1066
1067        /*
1068         * It's not the end of the world if we don't get
1069         * the lock, but we also don't want to spin
1070         * nor do we want to disable interrupts,
1071         * so if we miss here, then better luck next time.
1072         */
1073        if (!arch_spin_trylock(&trace_cmdline_lock))
1074                return;
1075
1076        idx = map_pid_to_cmdline[tsk->pid];
1077        if (idx == NO_CMDLINE_MAP) {
1078                idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1079
1080                /*
1081                 * Check whether the cmdline buffer at idx has a pid
1082                 * mapped. We are going to overwrite that entry so we
1083                 * need to clear the map_pid_to_cmdline. Otherwise we
1084                 * would read the new comm for the old pid.
1085                 */
1086                pid = map_cmdline_to_pid[idx];
1087                if (pid != NO_CMDLINE_MAP)
1088                        map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1089
1090                map_cmdline_to_pid[idx] = tsk->pid;
1091                map_pid_to_cmdline[tsk->pid] = idx;
1092
1093                cmdline_idx = idx;
1094        }
1095
1096        memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1097
1098        arch_spin_unlock(&trace_cmdline_lock);
1099}
1100
1101void trace_find_cmdline(int pid, char comm[])
1102{
1103        unsigned map;
1104
1105        if (!pid) {
1106                strcpy(comm, "<idle>");
1107                return;
1108        }
1109
1110        if (WARN_ON_ONCE(pid < 0)) {
1111                strcpy(comm, "<XXX>");
1112                return;
1113        }
1114
1115        if (pid > PID_MAX_DEFAULT) {
1116                strcpy(comm, "<...>");
1117                return;
1118        }
1119
1120        preempt_disable();
1121        arch_spin_lock(&trace_cmdline_lock);
1122        map = map_pid_to_cmdline[pid];
1123        if (map != NO_CMDLINE_MAP)
1124                strcpy(comm, saved_cmdlines[map]);
1125        else
1126                strcpy(comm, "<...>");
1127
1128        arch_spin_unlock(&trace_cmdline_lock);
1129        preempt_enable();
1130}
1131
1132void tracing_record_cmdline(struct task_struct *tsk)
1133{
1134        if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
1135            !tracing_is_on())
1136                return;
1137
1138        trace_save_cmdline(tsk);
1139}
1140
1141void
1142tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1143                             int pc)
1144{
1145        struct task_struct *tsk = current;
1146
1147        entry->preempt_count            = pc & 0xff;
1148        entry->pid                      = (tsk) ? tsk->pid : 0;
1149        entry->padding                  = 0;
1150        entry->flags =
1151#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1152                (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1153#else
1154                TRACE_FLAG_IRQS_NOSUPPORT |
1155#endif
1156                ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1157                ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1158                (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1159}
1160EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1161
1162struct ring_buffer_event *
1163trace_buffer_lock_reserve(struct ring_buffer *buffer,
1164                          int type,
1165                          unsigned long len,
1166                          unsigned long flags, int pc)
1167{
1168        struct ring_buffer_event *event;
1169
1170        event = ring_buffer_lock_reserve(buffer, len);
1171        if (event != NULL) {
1172                struct trace_entry *ent = ring_buffer_event_data(event);
1173
1174                tracing_generic_entry_update(ent, flags, pc);
1175                ent->type = type;
1176        }
1177
1178        return event;
1179}
1180
1181static inline void
1182__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1183                             struct ring_buffer_event *event,
1184                             unsigned long flags, int pc,
1185                             int wake)
1186{
1187        ring_buffer_unlock_commit(buffer, event);
1188
1189        ftrace_trace_stack(buffer, flags, 6, pc);
1190        ftrace_trace_userstack(buffer, flags, pc);
1191
1192        if (wake)
1193                trace_wake_up();
1194}
1195
1196void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1197                                struct ring_buffer_event *event,
1198                                unsigned long flags, int pc)
1199{
1200        __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1201}
1202
1203struct ring_buffer_event *
1204trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1205                                  int type, unsigned long len,
1206                                  unsigned long flags, int pc)
1207{
1208        *current_rb = global_trace.buffer;
1209        return trace_buffer_lock_reserve(*current_rb,
1210                                         type, len, flags, pc);
1211}
1212EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1213
1214void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1215                                        struct ring_buffer_event *event,
1216                                        unsigned long flags, int pc)
1217{
1218        __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1219}
1220EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1221
1222void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
1223                                       struct ring_buffer_event *event,
1224                                       unsigned long flags, int pc)
1225{
1226        __trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
1227}
1228EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
1229
1230void trace_nowake_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1231                                            struct ring_buffer_event *event,
1232                                            unsigned long flags, int pc,
1233                                            struct pt_regs *regs)
1234{
1235        ring_buffer_unlock_commit(buffer, event);
1236
1237        ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1238        ftrace_trace_userstack(buffer, flags, pc);
1239}
1240EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit_regs);
1241
1242void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1243                                         struct ring_buffer_event *event)
1244{
1245        ring_buffer_discard_commit(buffer, event);
1246}
1247EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1248
1249void
1250trace_function(struct trace_array *tr,
1251               unsigned long ip, unsigned long parent_ip, unsigned long flags,
1252               int pc)
1253{
1254        struct ftrace_event_call *call = &event_function;
1255        struct ring_buffer *buffer = tr->buffer;
1256        struct ring_buffer_event *event;
1257        struct ftrace_entry *entry;
1258
1259        /* If we are reading the ring buffer, don't trace */
1260        if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1261                return;
1262
1263        event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1264                                          flags, pc);
1265        if (!event)
1266                return;
1267        entry   = ring_buffer_event_data(event);
1268        entry->ip                       = ip;
1269        entry->parent_ip                = parent_ip;
1270
1271        if (!filter_check_discard(call, entry, buffer, event))
1272                ring_buffer_unlock_commit(buffer, event);
1273}
1274
1275void
1276ftrace(struct trace_array *tr, struct trace_array_cpu *data,
1277       unsigned long ip, unsigned long parent_ip, unsigned long flags,
1278       int pc)
1279{
1280        if (likely(!atomic_read(&data->disabled)))
1281                trace_function(tr, ip, parent_ip, flags, pc);
1282}
1283
1284#ifdef CONFIG_STACKTRACE
1285
1286#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1287struct ftrace_stack {
1288        unsigned long           calls[FTRACE_STACK_MAX_ENTRIES];
1289};
1290
1291static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1292static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1293
1294static void __ftrace_trace_stack(struct ring_buffer *buffer,
1295                                 unsigned long flags,
1296                                 int skip, int pc, struct pt_regs *regs)
1297{
1298        struct ftrace_event_call *call = &event_kernel_stack;
1299        struct ring_buffer_event *event;
1300        struct stack_entry *entry;
1301        struct stack_trace trace;
1302        int use_stack;
1303        int size = FTRACE_STACK_ENTRIES;
1304
1305        trace.nr_entries        = 0;
1306        trace.skip              = skip;
1307
1308        /*
1309         * Since events can happen in NMIs there's no safe way to
1310         * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1311         * or NMI comes in, it will just have to use the default
1312         * FTRACE_STACK_SIZE.
1313         */
1314        preempt_disable_notrace();
1315
1316        use_stack = ++__get_cpu_var(ftrace_stack_reserve);
1317        /*
1318         * We don't need any atomic variables, just a barrier.
1319         * If an interrupt comes in, we don't care, because it would
1320         * have exited and put the counter back to what we want.
1321         * We just need a barrier to keep gcc from moving things
1322         * around.
1323         */
1324        barrier();
1325        if (use_stack == 1) {
1326                trace.entries           = &__get_cpu_var(ftrace_stack).calls[0];
1327                trace.max_entries       = FTRACE_STACK_MAX_ENTRIES;
1328
1329                if (regs)
1330                        save_stack_trace_regs(regs, &trace);
1331                else
1332                        save_stack_trace(&trace);
1333
1334                if (trace.nr_entries > size)
1335                        size = trace.nr_entries;
1336        } else
1337                /* From now on, use_stack is a boolean */
1338                use_stack = 0;
1339
1340        size *= sizeof(unsigned long);
1341
1342        event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1343                                          sizeof(*entry) + size, flags, pc);
1344        if (!event)
1345                goto out;
1346        entry = ring_buffer_event_data(event);
1347
1348        memset(&entry->caller, 0, size);
1349
1350        if (use_stack)
1351                memcpy(&entry->caller, trace.entries,
1352                       trace.nr_entries * sizeof(unsigned long));
1353        else {
1354                trace.max_entries       = FTRACE_STACK_ENTRIES;
1355                trace.entries           = entry->caller;
1356                if (regs)
1357                        save_stack_trace_regs(regs, &trace);
1358                else
1359                        save_stack_trace(&trace);
1360        }
1361
1362        entry->size = trace.nr_entries;
1363
1364        if (!filter_check_discard(call, entry, buffer, event))
1365                ring_buffer_unlock_commit(buffer, event);
1366
1367 out:
1368        /* Again, don't let gcc optimize things here */
1369        barrier();
1370        __get_cpu_var(ftrace_stack_reserve)--;
1371        preempt_enable_notrace();
1372
1373}
1374
1375void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1376                             int skip, int pc, struct pt_regs *regs)
1377{
1378        if (!(trace_flags & TRACE_ITER_STACKTRACE))
1379                return;
1380
1381        __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1382}
1383
1384void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1385                        int skip, int pc)
1386{
1387        if (!(trace_flags & TRACE_ITER_STACKTRACE))
1388                return;
1389
1390        __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1391}
1392
1393void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1394                   int pc)
1395{
1396        __ftrace_trace_stack(tr->buffer, flags, skip, pc, NULL);
1397}
1398
1399/**
1400 * trace_dump_stack - record a stack back trace in the trace buffer
1401 */
1402void trace_dump_stack(void)
1403{
1404        unsigned long flags;
1405
1406        if (tracing_disabled || tracing_selftest_running)
1407                return;
1408
1409        local_save_flags(flags);
1410
1411        /* skipping 3 traces, seems to get us at the caller of this function */
1412        __ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count(), NULL);
1413}
1414
1415static DEFINE_PER_CPU(int, user_stack_count);
1416
1417void
1418ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1419{
1420        struct ftrace_event_call *call = &event_user_stack;
1421        struct ring_buffer_event *event;
1422        struct userstack_entry *entry;
1423        struct stack_trace trace;
1424
1425        if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1426                return;
1427
1428        /*
1429         * NMIs can not handle page faults, even with fix ups.
1430         * The save user stack can (and often does) fault.
1431         */
1432        if (unlikely(in_nmi()))
1433                return;
1434
1435        /*
1436         * prevent recursion, since the user stack tracing may
1437         * trigger other kernel events.
1438         */
1439        preempt_disable();
1440        if (__this_cpu_read(user_stack_count))
1441                goto out;
1442
1443        __this_cpu_inc(user_stack_count);
1444
1445        event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1446                                          sizeof(*entry), flags, pc);
1447        if (!event)
1448                goto out_drop_count;
1449        entry   = ring_buffer_event_data(event);
1450
1451        entry->tgid             = current->tgid;
1452        memset(&entry->caller, 0, sizeof(entry->caller));
1453
1454        trace.nr_entries        = 0;
1455        trace.max_entries       = FTRACE_STACK_ENTRIES;
1456        trace.skip              = 0;
1457        trace.entries           = entry->caller;
1458
1459        save_stack_trace_user(&trace);
1460        if (!filter_check_discard(call, entry, buffer, event))
1461                ring_buffer_unlock_commit(buffer, event);
1462
1463 out_drop_count:
1464        __this_cpu_dec(user_stack_count);
1465 out:
1466        preempt_enable();
1467}
1468
1469#ifdef UNUSED
1470static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1471{
1472        ftrace_trace_userstack(tr, flags, preempt_count());
1473}
1474#endif /* UNUSED */
1475
1476#endif /* CONFIG_STACKTRACE */
1477
1478/* created for use with alloc_percpu */
1479struct trace_buffer_struct {
1480        char buffer[TRACE_BUF_SIZE];
1481};
1482
1483static struct trace_buffer_struct *trace_percpu_buffer;
1484static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1485static struct trace_buffer_struct *trace_percpu_irq_buffer;
1486static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1487
1488/*
1489 * The buffer used is dependent on the context. There is a per cpu
1490 * buffer for normal context, softirq contex, hard irq context and
1491 * for NMI context. Thise allows for lockless recording.
1492 *
1493 * Note, if the buffers failed to be allocated, then this returns NULL
1494 */
1495static char *get_trace_buf(void)
1496{
1497        struct trace_buffer_struct *percpu_buffer;
1498        struct trace_buffer_struct *buffer;
1499
1500        /*
1501         * If we have allocated per cpu buffers, then we do not
1502         * need to do any locking.
1503         */
1504        if (in_nmi())
1505                percpu_buffer = trace_percpu_nmi_buffer;
1506        else if (in_irq())
1507                percpu_buffer = trace_percpu_irq_buffer;
1508        else if (in_softirq())
1509                percpu_buffer = trace_percpu_sirq_buffer;
1510        else
1511                percpu_buffer = trace_percpu_buffer;
1512
1513        if (!percpu_buffer)
1514                return NULL;
1515
1516        buffer = per_cpu_ptr(percpu_buffer, smp_processor_id());
1517
1518        return buffer->buffer;
1519}
1520
1521static int alloc_percpu_trace_buffer(void)
1522{
1523        struct trace_buffer_struct *buffers;
1524        struct trace_buffer_struct *sirq_buffers;
1525        struct trace_buffer_struct *irq_buffers;
1526        struct trace_buffer_struct *nmi_buffers;
1527
1528        buffers = alloc_percpu(struct trace_buffer_struct);
1529        if (!buffers)
1530                goto err_warn;
1531
1532        sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1533        if (!sirq_buffers)
1534                goto err_sirq;
1535
1536        irq_buffers = alloc_percpu(struct trace_buffer_struct);
1537        if (!irq_buffers)
1538                goto err_irq;
1539
1540        nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1541        if (!nmi_buffers)
1542                goto err_nmi;
1543
1544        trace_percpu_buffer = buffers;
1545        trace_percpu_sirq_buffer = sirq_buffers;
1546        trace_percpu_irq_buffer = irq_buffers;
1547        trace_percpu_nmi_buffer = nmi_buffers;
1548
1549        return 0;
1550
1551 err_nmi:
1552        free_percpu(irq_buffers);
1553 err_irq:
1554        free_percpu(sirq_buffers);
1555 err_sirq:
1556        free_percpu(buffers);
1557 err_warn:
1558        WARN(1, "Could not allocate percpu trace_printk buffer");
1559        return -ENOMEM;
1560}
1561
1562void trace_printk_init_buffers(void)
1563{
1564        static int buffers_allocated;
1565
1566        if (buffers_allocated)
1567                return;
1568
1569        if (alloc_percpu_trace_buffer())
1570                return;
1571
1572        pr_info("ftrace: Allocated trace_printk buffers\n");
1573
1574        buffers_allocated = 1;
1575}
1576
1577/**
1578 * trace_vbprintk - write binary msg to tracing buffer
1579 *
1580 */
1581int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1582{
1583        struct ftrace_event_call *call = &event_bprint;
1584        struct ring_buffer_event *event;
1585        struct ring_buffer *buffer;
1586        struct trace_array *tr = &global_trace;
1587        struct bprint_entry *entry;
1588        unsigned long flags;
1589        char *tbuffer;
1590        int len = 0, size, pc;
1591
1592        if (unlikely(tracing_selftest_running || tracing_disabled))
1593                return 0;
1594
1595        /* Don't pollute graph traces with trace_vprintk internals */
1596        pause_graph_tracing();
1597
1598        pc = preempt_count();
1599        preempt_disable_notrace();
1600
1601        tbuffer = get_trace_buf();
1602        if (!tbuffer) {
1603                len = 0;
1604                goto out;
1605        }
1606
1607        len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1608
1609        if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
1610                goto out;
1611
1612        local_save_flags(flags);
1613        size = sizeof(*entry) + sizeof(u32) * len;
1614        buffer = tr->buffer;
1615        event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1616                                          flags, pc);
1617        if (!event)
1618                goto out;
1619        entry = ring_buffer_event_data(event);
1620        entry->ip                       = ip;
1621        entry->fmt                      = fmt;
1622
1623        memcpy(entry->buf, tbuffer, sizeof(u32) * len);
1624        if (!filter_check_discard(call, entry, buffer, event)) {
1625                ring_buffer_unlock_commit(buffer, event);
1626                ftrace_trace_stack(buffer, flags, 6, pc);
1627        }
1628
1629out:
1630        preempt_enable_notrace();
1631        unpause_graph_tracing();
1632
1633        return len;
1634}
1635EXPORT_SYMBOL_GPL(trace_vbprintk);
1636
1637int trace_array_printk(struct trace_array *tr,
1638                       unsigned long ip, const char *fmt, ...)
1639{
1640        int ret;
1641        va_list ap;
1642
1643        if (!(trace_flags & TRACE_ITER_PRINTK))
1644                return 0;
1645
1646        va_start(ap, fmt);
1647        ret = trace_array_vprintk(tr, ip, fmt, ap);
1648        va_end(ap);
1649        return ret;
1650}
1651
1652int trace_array_vprintk(struct trace_array *tr,
1653                        unsigned long ip, const char *fmt, va_list args)
1654{
1655        struct ftrace_event_call *call = &event_print;
1656        struct ring_buffer_event *event;
1657        struct ring_buffer *buffer;
1658        int len = 0, size, pc;
1659        struct print_entry *entry;
1660        unsigned long flags;
1661        char *tbuffer;
1662
1663        if (tracing_disabled || tracing_selftest_running)
1664                return 0;
1665
1666        /* Don't pollute graph traces with trace_vprintk internals */
1667        pause_graph_tracing();
1668
1669        pc = preempt_count();
1670        preempt_disable_notrace();
1671
1672
1673        tbuffer = get_trace_buf();
1674        if (!tbuffer) {
1675                len = 0;
1676                goto out;
1677        }
1678
1679        len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
1680        if (len > TRACE_BUF_SIZE)
1681                goto out;
1682
1683        local_save_flags(flags);
1684        size = sizeof(*entry) + len + 1;
1685        buffer = tr->buffer;
1686        event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1687                                          flags, pc);
1688        if (!event)
1689                goto out;
1690        entry = ring_buffer_event_data(event);
1691        entry->ip = ip;
1692
1693        memcpy(&entry->buf, tbuffer, len);
1694        entry->buf[len] = '\0';
1695        if (!filter_check_discard(call, entry, buffer, event)) {
1696                ring_buffer_unlock_commit(buffer, event);
1697                ftrace_trace_stack(buffer, flags, 6, pc);
1698        }
1699 out:
1700        preempt_enable_notrace();
1701        unpause_graph_tracing();
1702
1703        return len;
1704}
1705
1706int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1707{
1708        return trace_array_vprintk(&global_trace, ip, fmt, args);
1709}
1710EXPORT_SYMBOL_GPL(trace_vprintk);
1711
1712static void trace_iterator_increment(struct trace_iterator *iter)
1713{
1714        struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
1715
1716        iter->idx++;
1717        if (buf_iter)
1718                ring_buffer_read(buf_iter, NULL);
1719}
1720
1721static struct trace_entry *
1722peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
1723                unsigned long *lost_events)
1724{
1725        struct ring_buffer_event *event;
1726        struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
1727
1728        if (buf_iter)
1729                event = ring_buffer_iter_peek(buf_iter, ts);
1730        else
1731                event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
1732                                         lost_events);
1733
1734        if (event) {
1735                iter->ent_size = ring_buffer_event_length(event);
1736                return ring_buffer_event_data(event);
1737        }
1738        iter->ent_size = 0;
1739        return NULL;
1740}
1741
1742static struct trace_entry *
1743__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
1744                  unsigned long *missing_events, u64 *ent_ts)
1745{
1746        struct ring_buffer *buffer = iter->tr->buffer;
1747        struct trace_entry *ent, *next = NULL;
1748        unsigned long lost_events = 0, next_lost = 0;
1749        int cpu_file = iter->cpu_file;
1750        u64 next_ts = 0, ts;
1751        int next_cpu = -1;
1752        int next_size = 0;
1753        int cpu;
1754
1755        /*
1756         * If we are in a per_cpu trace file, don't bother by iterating over
1757         * all cpu and peek directly.
1758         */
1759        if (cpu_file > TRACE_PIPE_ALL_CPU) {
1760                if (ring_buffer_empty_cpu(buffer, cpu_file))
1761                        return NULL;
1762                ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
1763                if (ent_cpu)
1764                        *ent_cpu = cpu_file;
1765
1766                return ent;
1767        }
1768
1769        for_each_tracing_cpu(cpu) {
1770
1771                if (ring_buffer_empty_cpu(buffer, cpu))
1772                        continue;
1773
1774                ent = peek_next_entry(iter, cpu, &ts, &lost_events);
1775
1776                /*
1777                 * Pick the entry with the smallest timestamp:
1778                 */
1779                if (ent && (!next || ts < next_ts)) {
1780                        next = ent;
1781                        next_cpu = cpu;
1782                        next_ts = ts;
1783                        next_lost = lost_events;
1784                        next_size = iter->ent_size;
1785                }
1786        }
1787
1788        iter->ent_size = next_size;
1789
1790        if (ent_cpu)
1791                *ent_cpu = next_cpu;
1792
1793        if (ent_ts)
1794                *ent_ts = next_ts;
1795
1796        if (missing_events)
1797                *missing_events = next_lost;
1798
1799        return next;
1800}
1801
1802/* Find the next real entry, without updating the iterator itself */
1803struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1804                                          int *ent_cpu, u64 *ent_ts)
1805{
1806        return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
1807}
1808
1809/* Find the next real entry, and increment the iterator to the next entry */
1810void *trace_find_next_entry_inc(struct trace_iterator *iter)
1811{
1812        iter->ent = __find_next_entry(iter, &iter->cpu,
1813                                      &iter->lost_events, &iter->ts);
1814
1815        if (iter->ent)
1816                trace_iterator_increment(iter);
1817
1818        return iter->ent ? iter : NULL;
1819}
1820
1821static void trace_consume(struct trace_iterator *iter)
1822{
1823        ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
1824                            &iter->lost_events);
1825}
1826
1827static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1828{
1829        struct trace_iterator *iter = m->private;
1830        int i = (int)*pos;
1831        void *ent;
1832
1833        WARN_ON_ONCE(iter->leftover);
1834
1835        (*pos)++;
1836
1837        /* can't go backwards */
1838        if (iter->idx > i)
1839                return NULL;
1840
1841        if (iter->idx < 0)
1842                ent = trace_find_next_entry_inc(iter);
1843        else
1844                ent = iter;
1845
1846        while (ent && iter->idx < i)
1847                ent = trace_find_next_entry_inc(iter);
1848
1849        iter->pos = *pos;
1850
1851        return ent;
1852}
1853
1854void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1855{
1856        struct trace_array *tr = iter->tr;
1857        struct ring_buffer_event *event;
1858        struct ring_buffer_iter *buf_iter;
1859        unsigned long entries = 0;
1860        u64 ts;
1861
1862        tr->data[cpu]->skipped_entries = 0;
1863
1864        buf_iter = trace_buffer_iter(iter, cpu);
1865        if (!buf_iter)
1866                return;
1867
1868        ring_buffer_iter_reset(buf_iter);
1869
1870        /*
1871         * We could have the case with the max latency tracers
1872         * that a reset never took place on a cpu. This is evident
1873         * by the timestamp being before the start of the buffer.
1874         */
1875        while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1876                if (ts >= iter->tr->time_start)
1877                        break;
1878                entries++;
1879                ring_buffer_read(buf_iter, NULL);
1880        }
1881
1882        tr->data[cpu]->skipped_entries = entries;
1883}
1884
1885/*
1886 * The current tracer is copied to avoid a global locking
1887 * all around.
1888 */
1889static void *s_start(struct seq_file *m, loff_t *pos)
1890{
1891        struct trace_iterator *iter = m->private;
1892        static struct tracer *old_tracer;
1893        int cpu_file = iter->cpu_file;
1894        void *p = NULL;
1895        loff_t l = 0;
1896        int cpu;
1897
1898        /* copy the tracer to avoid using a global lock all around */
1899        mutex_lock(&trace_types_lock);
1900        if (unlikely(old_tracer != current_trace && current_trace)) {
1901                old_tracer = current_trace;
1902                *iter->trace = *current_trace;
1903        }
1904        mutex_unlock(&trace_types_lock);
1905
1906        atomic_inc(&trace_record_cmdline_disabled);
1907
1908        if (*pos != iter->pos) {
1909                iter->ent = NULL;
1910                iter->cpu = 0;
1911                iter->idx = -1;
1912
1913                if (cpu_file == TRACE_PIPE_ALL_CPU) {
1914                        for_each_tracing_cpu(cpu)
1915                                tracing_iter_reset(iter, cpu);
1916                } else
1917                        tracing_iter_reset(iter, cpu_file);
1918
1919                iter->leftover = 0;
1920                for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1921                        ;
1922
1923        } else {
1924                /*
1925                 * If we overflowed the seq_file before, then we want
1926                 * to just reuse the trace_seq buffer again.
1927                 */
1928                if (iter->leftover)
1929                        p = iter;
1930                else {
1931                        l = *pos - 1;
1932                        p = s_next(m, p, &l);
1933                }
1934        }
1935
1936        trace_event_read_lock();
1937        trace_access_lock(cpu_file);
1938        return p;
1939}
1940
1941static void s_stop(struct seq_file *m, void *p)
1942{
1943        struct trace_iterator *iter = m->private;
1944
1945        atomic_dec(&trace_record_cmdline_disabled);
1946        trace_access_unlock(iter->cpu_file);
1947        trace_event_read_unlock();
1948}
1949
1950static void
1951get_total_entries(struct trace_array *tr, unsigned long *total, unsigned long *entries)
1952{
1953        unsigned long count;
1954        int cpu;
1955
1956        *total = 0;
1957        *entries = 0;
1958
1959        for_each_tracing_cpu(cpu) {
1960                count = ring_buffer_entries_cpu(tr->buffer, cpu);
1961                /*
1962                 * If this buffer has skipped entries, then we hold all
1963                 * entries for the trace and we need to ignore the
1964                 * ones before the time stamp.
1965                 */
1966                if (tr->data[cpu]->skipped_entries) {
1967                        count -= tr->data[cpu]->skipped_entries;
1968                        /* total is the same as the entries */
1969                        *total += count;
1970                } else
1971                        *total += count +
1972                                ring_buffer_overrun_cpu(tr->buffer, cpu);
1973                *entries += count;
1974        }
1975}
1976
1977static void print_lat_help_header(struct seq_file *m)
1978{
1979        seq_puts(m, "#                  _------=> CPU#            \n");
1980        seq_puts(m, "#                 / _-----=> irqs-off        \n");
1981        seq_puts(m, "#                | / _----=> need-resched    \n");
1982        seq_puts(m, "#                || / _---=> hardirq/softirq \n");
1983        seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
1984        seq_puts(m, "#                |||| /     delay             \n");
1985        seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
1986        seq_puts(m, "#     \\   /      |||||  \\    |   /           \n");
1987}
1988
1989static void print_event_info(struct trace_array *tr, struct seq_file *m)
1990{
1991        unsigned long total;
1992        unsigned long entries;
1993
1994        get_total_entries(tr, &total, &entries);
1995        seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu   #P:%d\n",
1996                   entries, total, num_online_cpus());
1997        seq_puts(m, "#\n");
1998}
1999
2000static void print_func_help_header(struct trace_array *tr, struct seq_file *m)
2001{
2002        print_event_info(tr, m);
2003        seq_puts(m, "#           TASK-PID   CPU#      TIMESTAMP  FUNCTION\n");
2004        seq_puts(m, "#              | |       |          |         |\n");
2005}
2006
2007static void print_func_help_header_irq(struct trace_array *tr, struct seq_file *m)
2008{
2009        print_event_info(tr, m);
2010        seq_puts(m, "#                              _-----=> irqs-off\n");
2011        seq_puts(m, "#                             / _----=> need-resched\n");
2012        seq_puts(m, "#                            | / _---=> hardirq/softirq\n");
2013        seq_puts(m, "#                            || / _--=> preempt-depth\n");
2014        seq_puts(m, "#                            ||| /     delay\n");
2015        seq_puts(m, "#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION\n");
2016        seq_puts(m, "#              | |       |   ||||       |         |\n");
2017}
2018
2019void
2020print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2021{
2022        unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2023        struct trace_array *tr = iter->tr;
2024        struct trace_array_cpu *data = tr->data[tr->cpu];
2025        struct tracer *type = current_trace;
2026        unsigned long entries;
2027        unsigned long total;
2028        const char *name = "preemption";
2029
2030        if (type)
2031                name = type->name;
2032
2033        get_total_entries(tr, &total, &entries);
2034
2035        seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2036                   name, UTS_RELEASE);
2037        seq_puts(m, "# -----------------------------------"
2038                 "---------------------------------\n");
2039        seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2040                   " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2041                   nsecs_to_usecs(data->saved_latency),
2042                   entries,
2043                   total,
2044                   tr->cpu,
2045#if defined(CONFIG_PREEMPT_NONE)
2046                   "server",
2047#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2048                   "desktop",
2049#elif defined(CONFIG_PREEMPT)
2050                   "preempt",
2051#else
2052                   "unknown",
2053#endif
2054                   /* These are reserved for later use */
2055                   0, 0, 0, 0);
2056#ifdef CONFIG_SMP
2057        seq_printf(m, " #P:%d)\n", num_online_cpus());
2058#else
2059        seq_puts(m, ")\n");
2060#endif
2061        seq_puts(m, "#    -----------------\n");
2062        seq_printf(m, "#    | task: %.16s-%d "
2063                   "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2064                   data->comm, data->pid,
2065                   from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2066                   data->policy, data->rt_priority);
2067        seq_puts(m, "#    -----------------\n");
2068
2069        if (data->critical_start) {
2070                seq_puts(m, "#  => started at: ");
2071                seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2072                trace_print_seq(m, &iter->seq);
2073                seq_puts(m, "\n#  => ended at:   ");
2074                seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2075                trace_print_seq(m, &iter->seq);
2076                seq_puts(m, "\n#\n");
2077        }
2078
2079        seq_puts(m, "#\n");
2080}
2081
2082static void test_cpu_buff_start(struct trace_iterator *iter)
2083{
2084        struct trace_seq *s = &iter->seq;
2085
2086        if (!(trace_flags & TRACE_ITER_ANNOTATE))
2087                return;
2088
2089        if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2090                return;
2091
2092        if (cpumask_test_cpu(iter->cpu, iter->started))
2093                return;
2094
2095        if (iter->tr->data[iter->cpu]->skipped_entries)
2096                return;
2097
2098        cpumask_set_cpu(iter->cpu, iter->started);
2099
2100        /* Don't print started cpu buffer for the first entry of the trace */
2101        if (iter->idx > 1)
2102                trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2103                                iter->cpu);
2104}
2105
2106static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2107{
2108        struct trace_seq *s = &iter->seq;
2109        unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2110        struct trace_entry *entry;
2111        struct trace_event *event;
2112
2113        entry = iter->ent;
2114
2115        test_cpu_buff_start(iter);
2116
2117        event = ftrace_find_event(entry->type);
2118
2119        if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2120                if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2121                        if (!trace_print_lat_context(iter))
2122                                goto partial;
2123                } else {
2124                        if (!trace_print_context(iter))
2125                                goto partial;
2126                }
2127        }
2128
2129        if (event)
2130                return event->funcs->trace(iter, sym_flags, event);
2131
2132        if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2133                goto partial;
2134
2135        return TRACE_TYPE_HANDLED;
2136partial:
2137        return TRACE_TYPE_PARTIAL_LINE;
2138}
2139
2140static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2141{
2142        struct trace_seq *s = &iter->seq;
2143        struct trace_entry *entry;
2144        struct trace_event *event;
2145
2146        entry = iter->ent;
2147
2148        if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2149                if (!trace_seq_printf(s, "%d %d %llu ",
2150                                      entry->pid, iter->cpu, iter->ts))
2151                        goto partial;
2152        }
2153
2154        event = ftrace_find_event(entry->type);
2155        if (event)
2156                return event->funcs->raw(iter, 0, event);
2157
2158        if (!trace_seq_printf(s, "%d ?\n", entry->type))
2159                goto partial;
2160
2161        return TRACE_TYPE_HANDLED;
2162partial:
2163        return TRACE_TYPE_PARTIAL_LINE;
2164}
2165
2166static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2167{
2168        struct trace_seq *s = &iter->seq;
2169        unsigned char newline = '\n';
2170        struct trace_entry *entry;
2171        struct trace_event *event;
2172
2173        entry = iter->ent;
2174
2175        if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2176                SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2177                SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2178                SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2179        }
2180
2181        event = ftrace_find_event(entry->type);
2182        if (event) {
2183                enum print_line_t ret = event->funcs->hex(iter, 0, event);
2184                if (ret != TRACE_TYPE_HANDLED)
2185                        return ret;
2186        }
2187
2188        SEQ_PUT_FIELD_RET(s, newline);
2189
2190        return TRACE_TYPE_HANDLED;
2191}
2192
2193static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2194{
2195        struct trace_seq *s = &iter->seq;
2196        struct trace_entry *entry;
2197        struct trace_event *event;
2198
2199        entry = iter->ent;
2200
2201        if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2202                SEQ_PUT_FIELD_RET(s, entry->pid);
2203                SEQ_PUT_FIELD_RET(s, iter->cpu);
2204                SEQ_PUT_FIELD_RET(s, iter->ts);
2205        }
2206
2207        event = ftrace_find_event(entry->type);
2208        return event ? event->funcs->binary(iter, 0, event) :
2209                TRACE_TYPE_HANDLED;
2210}
2211
2212int trace_empty(struct trace_iterator *iter)
2213{
2214        struct ring_buffer_iter *buf_iter;
2215        int cpu;
2216
2217        /* If we are looking at one CPU buffer, only check that one */
2218        if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
2219                cpu = iter->cpu_file;
2220                buf_iter = trace_buffer_iter(iter, cpu);
2221                if (buf_iter) {
2222                        if (!ring_buffer_iter_empty(buf_iter))
2223                                return 0;
2224                } else {
2225                        if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2226                                return 0;
2227                }
2228                return 1;
2229        }
2230
2231        for_each_tracing_cpu(cpu) {
2232                buf_iter = trace_buffer_iter(iter, cpu);
2233                if (buf_iter) {
2234                        if (!ring_buffer_iter_empty(buf_iter))
2235                                return 0;
2236                } else {
2237                        if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2238                                return 0;
2239                }
2240        }
2241
2242        return 1;
2243}
2244
2245/*  Called with trace_event_read_lock() held. */
2246enum print_line_t print_trace_line(struct trace_iterator *iter)
2247{
2248        enum print_line_t ret;
2249
2250        if (iter->lost_events &&
2251            !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2252                                 iter->cpu, iter->lost_events))
2253                return TRACE_TYPE_PARTIAL_LINE;
2254
2255        if (iter->trace && iter->trace->print_line) {
2256                ret = iter->trace->print_line(iter);
2257                if (ret != TRACE_TYPE_UNHANDLED)
2258                        return ret;
2259        }
2260
2261        if (iter->ent->type == TRACE_BPRINT &&
2262                        trace_flags & TRACE_ITER_PRINTK &&
2263                        trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2264                return trace_print_bprintk_msg_only(iter);
2265
2266        if (iter->ent->type == TRACE_PRINT &&
2267                        trace_flags & TRACE_ITER_PRINTK &&
2268                        trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2269                return trace_print_printk_msg_only(iter);
2270
2271        if (trace_flags & TRACE_ITER_BIN)
2272                return print_bin_fmt(iter);
2273
2274        if (trace_flags & TRACE_ITER_HEX)
2275                return print_hex_fmt(iter);
2276
2277        if (trace_flags & TRACE_ITER_RAW)
2278                return print_raw_fmt(iter);
2279
2280        return print_trace_fmt(iter);
2281}
2282
2283void trace_latency_header(struct seq_file *m)
2284{
2285        struct trace_iterator *iter = m->private;
2286
2287        /* print nothing if the buffers are empty */
2288        if (trace_empty(iter))
2289                return;
2290
2291        if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2292                print_trace_header(m, iter);
2293
2294        if (!(trace_flags & TRACE_ITER_VERBOSE))
2295                print_lat_help_header(m);
2296}
2297
2298void trace_default_header(struct seq_file *m)
2299{
2300        struct trace_iterator *iter = m->private;
2301
2302        if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2303                return;
2304
2305        if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2306                /* print nothing if the buffers are empty */
2307                if (trace_empty(iter))
2308                        return;
2309                print_trace_header(m, iter);
2310                if (!(trace_flags & TRACE_ITER_VERBOSE))
2311                        print_lat_help_header(m);
2312        } else {
2313                if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2314                        if (trace_flags & TRACE_ITER_IRQ_INFO)
2315                                print_func_help_header_irq(iter->tr, m);
2316                        else
2317                                print_func_help_header(iter->tr, m);
2318                }
2319        }
2320}
2321
2322static void test_ftrace_alive(struct seq_file *m)
2323{
2324        if (!ftrace_is_dead())
2325                return;
2326        seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2327        seq_printf(m, "#          MAY BE MISSING FUNCTION EVENTS\n");
2328}
2329
2330static int s_show(struct seq_file *m, void *v)
2331{
2332        struct trace_iterator *iter = v;
2333        int ret;
2334
2335        if (iter->ent == NULL) {
2336                if (iter->tr) {
2337                        seq_printf(m, "# tracer: %s\n", iter->trace->name);
2338                        seq_puts(m, "#\n");
2339                        test_ftrace_alive(m);
2340                }
2341                if (iter->trace && iter->trace->print_header)
2342                        iter->trace->print_header(m);
2343                else
2344                        trace_default_header(m);
2345
2346        } else if (iter->leftover) {
2347                /*
2348                 * If we filled the seq_file buffer earlier, we
2349                 * want to just show it now.
2350                 */
2351                ret = trace_print_seq(m, &iter->seq);
2352
2353                /* ret should this time be zero, but you never know */
2354                iter->leftover = ret;
2355
2356        } else {
2357                print_trace_line(iter);
2358                ret = trace_print_seq(m, &iter->seq);
2359                /*
2360                 * If we overflow the seq_file buffer, then it will
2361                 * ask us for this data again at start up.
2362                 * Use that instead.
2363                 *  ret is 0 if seq_file write succeeded.
2364                 *        -1 otherwise.
2365                 */
2366                iter->leftover = ret;
2367        }
2368
2369        return 0;
2370}
2371
2372static const struct seq_operations tracer_seq_ops = {
2373        .start          = s_start,
2374        .next           = s_next,
2375        .stop           = s_stop,
2376        .show           = s_show,
2377};
2378
2379static struct trace_iterator *
2380__tracing_open(struct inode *inode, struct file *file)
2381{
2382        long cpu_file = (long) inode->i_private;
2383        struct trace_iterator *iter;
2384        int cpu;
2385
2386        if (tracing_disabled)
2387                return ERR_PTR(-ENODEV);
2388
2389        iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2390        if (!iter)
2391                return ERR_PTR(-ENOMEM);
2392
2393        iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2394                                    GFP_KERNEL);
2395        if (!iter->buffer_iter)
2396                goto release;
2397
2398        /*
2399         * We make a copy of the current tracer to avoid concurrent
2400         * changes on it while we are reading.
2401         */
2402        mutex_lock(&trace_types_lock);
2403        iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2404        if (!iter->trace)
2405                goto fail;
2406
2407        if (current_trace)
2408                *iter->trace = *current_trace;
2409
2410        if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2411                goto fail;
2412
2413        if (current_trace && current_trace->print_max)
2414                iter->tr = &max_tr;
2415        else
2416                iter->tr = &global_trace;
2417        iter->pos = -1;
2418        mutex_init(&iter->mutex);
2419        iter->cpu_file = cpu_file;
2420
2421        /* Notify the tracer early; before we stop tracing. */
2422        if (iter->trace && iter->trace->open)
2423                iter->trace->open(iter);
2424
2425        /* Annotate start of buffers if we had overruns */
2426        if (ring_buffer_overruns(iter->tr->buffer))
2427                iter->iter_flags |= TRACE_FILE_ANNOTATE;
2428
2429        /* stop the trace while dumping */
2430        tracing_stop();
2431
2432        if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2433                for_each_tracing_cpu(cpu) {
2434                        iter->buffer_iter[cpu] =
2435                                ring_buffer_read_prepare(iter->tr->buffer, cpu);
2436                }
2437                ring_buffer_read_prepare_sync();
2438                for_each_tracing_cpu(cpu) {
2439                        ring_buffer_read_start(iter->buffer_iter[cpu]);
2440                        tracing_iter_reset(iter, cpu);
2441                }
2442        } else {
2443                cpu = iter->cpu_file;
2444                iter->buffer_iter[cpu] =
2445                        ring_buffer_read_prepare(iter->tr->buffer, cpu);
2446                ring_buffer_read_prepare_sync();
2447                ring_buffer_read_start(iter->buffer_iter[cpu]);
2448                tracing_iter_reset(iter, cpu);
2449        }
2450
2451        mutex_unlock(&trace_types_lock);
2452
2453        return iter;
2454
2455 fail:
2456        mutex_unlock(&trace_types_lock);
2457        kfree(iter->trace);
2458        kfree(iter->buffer_iter);
2459release:
2460        seq_release_private(inode, file);
2461        return ERR_PTR(-ENOMEM);
2462}
2463
2464int tracing_open_generic(struct inode *inode, struct file *filp)
2465{
2466        if (tracing_disabled)
2467                return -ENODEV;
2468
2469        filp->private_data = inode->i_private;
2470        return 0;
2471}
2472
2473static int tracing_release(struct inode *inode, struct file *file)
2474{
2475        struct seq_file *m = file->private_data;
2476        struct trace_iterator *iter;
2477        int cpu;
2478
2479        if (!(file->f_mode & FMODE_READ))
2480                return 0;
2481
2482        iter = m->private;
2483
2484        mutex_lock(&trace_types_lock);
2485        for_each_tracing_cpu(cpu) {
2486                if (iter->buffer_iter[cpu])
2487                        ring_buffer_read_finish(iter->buffer_iter[cpu]);
2488        }
2489
2490        if (iter->trace && iter->trace->close)
2491                iter->trace->close(iter);
2492
2493        /* reenable tracing if it was previously enabled */
2494        tracing_start();
2495        mutex_unlock(&trace_types_lock);
2496
2497        mutex_destroy(&iter->mutex);
2498        free_cpumask_var(iter->started);
2499        kfree(iter->trace);
2500        kfree(iter->buffer_iter);
2501        seq_release_private(inode, file);
2502        return 0;
2503}
2504
2505static int tracing_open(struct inode *inode, struct file *file)
2506{
2507        struct trace_iterator *iter;
2508        int ret = 0;
2509
2510        /* If this file was open for write, then erase contents */
2511        if ((file->f_mode & FMODE_WRITE) &&
2512            (file->f_flags & O_TRUNC)) {
2513                long cpu = (long) inode->i_private;
2514
2515                if (cpu == TRACE_PIPE_ALL_CPU)
2516                        tracing_reset_online_cpus(&global_trace);
2517                else
2518                        tracing_reset(&global_trace, cpu);
2519        }
2520
2521        if (file->f_mode & FMODE_READ) {
2522                iter = __tracing_open(inode, file);
2523                if (IS_ERR(iter))
2524                        ret = PTR_ERR(iter);
2525                else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2526                        iter->iter_flags |= TRACE_FILE_LAT_FMT;
2527        }
2528        return ret;
2529}
2530
2531static void *
2532t_next(struct seq_file *m, void *v, loff_t *pos)
2533{
2534        struct tracer *t = v;
2535
2536        (*pos)++;
2537
2538        if (t)
2539                t = t->next;
2540
2541        return t;
2542}
2543
2544static void *t_start(struct seq_file *m, loff_t *pos)
2545{
2546        struct tracer *t;
2547        loff_t l = 0;
2548
2549        mutex_lock(&trace_types_lock);
2550        for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2551                ;
2552
2553        return t;
2554}
2555
2556static void t_stop(struct seq_file *m, void *p)
2557{
2558        mutex_unlock(&trace_types_lock);
2559}
2560
2561static int t_show(struct seq_file *m, void *v)
2562{
2563        struct tracer *t = v;
2564
2565        if (!t)
2566                return 0;
2567
2568        seq_printf(m, "%s", t->name);
2569        if (t->next)
2570                seq_putc(m, ' ');
2571        else
2572                seq_putc(m, '\n');
2573
2574        return 0;
2575}
2576
2577static const struct seq_operations show_traces_seq_ops = {
2578        .start          = t_start,
2579        .next           = t_next,
2580        .stop           = t_stop,
2581        .show           = t_show,
2582};
2583
2584static int show_traces_open(struct inode *inode, struct file *file)
2585{
2586        if (tracing_disabled)
2587                return -ENODEV;
2588
2589        return seq_open(file, &show_traces_seq_ops);
2590}
2591
2592static ssize_t
2593tracing_write_stub(struct file *filp, const char __user *ubuf,
2594                   size_t count, loff_t *ppos)
2595{
2596        return count;
2597}
2598
2599static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
2600{
2601        if (file->f_mode & FMODE_READ)
2602                return seq_lseek(file, offset, origin);
2603        else
2604                return 0;
2605}
2606
2607static const struct file_operations tracing_fops = {
2608        .open           = tracing_open,
2609        .read           = seq_read,
2610        .write          = tracing_write_stub,
2611        .llseek         = tracing_seek,
2612        .release        = tracing_release,
2613};
2614
2615static const struct file_operations show_traces_fops = {
2616        .open           = show_traces_open,
2617        .read           = seq_read,
2618        .release        = seq_release,
2619        .llseek         = seq_lseek,
2620};
2621
2622/*
2623 * Only trace on a CPU if the bitmask is set:
2624 */
2625static cpumask_var_t tracing_cpumask;
2626
2627/*
2628 * The tracer itself will not take this lock, but still we want
2629 * to provide a consistent cpumask to user-space:
2630 */
2631static DEFINE_MUTEX(tracing_cpumask_update_lock);
2632
2633/*
2634 * Temporary storage for the character representation of the
2635 * CPU bitmask (and one more byte for the newline):
2636 */
2637static char mask_str[NR_CPUS + 1];
2638
2639static ssize_t
2640tracing_cpumask_read(struct file *filp, char __user *ubuf,
2641                     size_t count, loff_t *ppos)
2642{
2643        int len;
2644
2645        mutex_lock(&tracing_cpumask_update_lock);
2646
2647        len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2648        if (count - len < 2) {
2649                count = -EINVAL;
2650                goto out_err;
2651        }
2652        len += sprintf(mask_str + len, "\n");
2653        count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2654
2655out_err:
2656        mutex_unlock(&tracing_cpumask_update_lock);
2657
2658        return count;
2659}
2660
2661static ssize_t
2662tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2663                      size_t count, loff_t *ppos)
2664{
2665        int err, cpu;
2666        cpumask_var_t tracing_cpumask_new;
2667
2668        if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2669                return -ENOMEM;
2670
2671        err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2672        if (err)
2673                goto err_unlock;
2674
2675        mutex_lock(&tracing_cpumask_update_lock);
2676
2677        local_irq_disable();
2678        arch_spin_lock(&ftrace_max_lock);
2679        for_each_tracing_cpu(cpu) {
2680                /*
2681                 * Increase/decrease the disabled counter if we are
2682                 * about to flip a bit in the cpumask:
2683                 */
2684                if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2685                                !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2686                        atomic_inc(&global_trace.data[cpu]->disabled);
2687                        ring_buffer_record_disable_cpu(global_trace.buffer, cpu);
2688                }
2689                if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2690                                cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2691                        atomic_dec(&global_trace.data[cpu]->disabled);
2692                        ring_buffer_record_enable_cpu(global_trace.buffer, cpu);
2693                }
2694        }
2695        arch_spin_unlock(&ftrace_max_lock);
2696        local_irq_enable();
2697
2698        cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2699
2700        mutex_unlock(&tracing_cpumask_update_lock);
2701        free_cpumask_var(tracing_cpumask_new);
2702
2703        return count;
2704
2705err_unlock:
2706        free_cpumask_var(tracing_cpumask_new);
2707
2708        return err;
2709}
2710
2711static const struct file_operations tracing_cpumask_fops = {
2712        .open           = tracing_open_generic,
2713        .read           = tracing_cpumask_read,
2714        .write          = tracing_cpumask_write,
2715        .llseek         = generic_file_llseek,
2716};
2717
2718static int tracing_trace_options_show(struct seq_file *m, void *v)
2719{
2720        struct tracer_opt *trace_opts;
2721        u32 tracer_flags;
2722        int i;
2723
2724        mutex_lock(&trace_types_lock);
2725        tracer_flags = current_trace->flags->val;
2726        trace_opts = current_trace->flags->opts;
2727
2728        for (i = 0; trace_options[i]; i++) {
2729                if (trace_flags & (1 << i))
2730                        seq_printf(m, "%s\n", trace_options[i]);
2731                else
2732                        seq_printf(m, "no%s\n", trace_options[i]);
2733        }
2734
2735        for (i = 0; trace_opts[i].name; i++) {
2736                if (tracer_flags & trace_opts[i].bit)
2737                        seq_printf(m, "%s\n", trace_opts[i].name);
2738                else
2739                        seq_printf(m, "no%s\n", trace_opts[i].name);
2740        }
2741        mutex_unlock(&trace_types_lock);
2742
2743        return 0;
2744}
2745
2746static int __set_tracer_option(struct tracer *trace,
2747                               struct tracer_flags *tracer_flags,
2748                               struct tracer_opt *opts, int neg)
2749{
2750        int ret;
2751
2752        ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
2753        if (ret)
2754                return ret;
2755
2756        if (neg)
2757                tracer_flags->val &= ~opts->bit;
2758        else
2759                tracer_flags->val |= opts->bit;
2760        return 0;
2761}
2762
2763/* Try to assign a tracer specific option */
2764static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2765{
2766        struct tracer_flags *tracer_flags = trace->flags;
2767        struct tracer_opt *opts = NULL;
2768        int i;
2769
2770        for (i = 0; tracer_flags->opts[i].name; i++) {
2771                opts = &tracer_flags->opts[i];
2772
2773                if (strcmp(cmp, opts->name) == 0)
2774                        return __set_tracer_option(trace, trace->flags,
2775                                                   opts, neg);
2776        }
2777
2778        return -EINVAL;
2779}
2780
2781static void set_tracer_flags(unsigned int mask, int enabled)
2782{
2783        /* do nothing if flag is already set */
2784        if (!!(trace_flags & mask) == !!enabled)
2785                return;
2786
2787        if (enabled)
2788                trace_flags |= mask;
2789        else
2790                trace_flags &= ~mask;
2791
2792        if (mask == TRACE_ITER_RECORD_CMD)
2793                trace_event_enable_cmd_record(enabled);
2794
2795        if (mask == TRACE_ITER_OVERWRITE)
2796                ring_buffer_change_overwrite(global_trace.buffer, enabled);
2797}
2798
2799static ssize_t
2800tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2801                        size_t cnt, loff_t *ppos)
2802{
2803        char buf[64];
2804        char *cmp;
2805        int neg = 0;
2806        int ret;
2807        int i;
2808
2809        if (cnt >= sizeof(buf))
2810                return -EINVAL;
2811
2812        if (copy_from_user(&buf, ubuf, cnt))
2813                return -EFAULT;
2814
2815        buf[cnt] = 0;
2816        cmp = strstrip(buf);
2817
2818        if (strncmp(cmp, "no", 2) == 0) {
2819                neg = 1;
2820                cmp += 2;
2821        }
2822
2823        for (i = 0; trace_options[i]; i++) {
2824                if (strcmp(cmp, trace_options[i]) == 0) {
2825                        set_tracer_flags(1 << i, !neg);
2826                        break;
2827                }
2828        }
2829
2830        /* If no option could be set, test the specific tracer options */
2831        if (!trace_options[i]) {
2832                mutex_lock(&trace_types_lock);
2833                ret = set_tracer_option(current_trace, cmp, neg);
2834                mutex_unlock(&trace_types_lock);
2835                if (ret)
2836                        return ret;
2837        }
2838
2839        *ppos += cnt;
2840
2841        return cnt;
2842}
2843
2844static int tracing_trace_options_open(struct inode *inode, struct file *file)
2845{
2846        if (tracing_disabled)
2847                return -ENODEV;
2848        return single_open(file, tracing_trace_options_show, NULL);
2849}
2850
2851static const struct file_operations tracing_iter_fops = {
2852        .open           = tracing_trace_options_open,
2853        .read           = seq_read,
2854        .llseek         = seq_lseek,
2855        .release        = single_release,
2856        .write          = tracing_trace_options_write,
2857};
2858
2859static const char readme_msg[] =
2860        "tracing mini-HOWTO:\n\n"
2861        "# mount -t debugfs nodev /sys/kernel/debug\n\n"
2862        "# cat /sys/kernel/debug/tracing/available_tracers\n"
2863        "wakeup wakeup_rt preemptirqsoff preemptoff irqsoff function nop\n\n"
2864        "# cat /sys/kernel/debug/tracing/current_tracer\n"
2865        "nop\n"
2866        "# echo wakeup > /sys/kernel/debug/tracing/current_tracer\n"
2867        "# cat /sys/kernel/debug/tracing/current_tracer\n"
2868        "wakeup\n"
2869        "# cat /sys/kernel/debug/tracing/trace_options\n"
2870        "noprint-parent nosym-offset nosym-addr noverbose\n"
2871        "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
2872        "# echo 1 > /sys/kernel/debug/tracing/tracing_on\n"
2873        "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
2874        "# echo 0 > /sys/kernel/debug/tracing/tracing_on\n"
2875;
2876
2877static ssize_t
2878tracing_readme_read(struct file *filp, char __user *ubuf,
2879                       size_t cnt, loff_t *ppos)
2880{
2881        return simple_read_from_buffer(ubuf, cnt, ppos,
2882                                        readme_msg, strlen(readme_msg));
2883}
2884
2885static const struct file_operations tracing_readme_fops = {
2886        .open           = tracing_open_generic,
2887        .read           = tracing_readme_read,
2888        .llseek         = generic_file_llseek,
2889};
2890
2891static ssize_t
2892tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
2893                                size_t cnt, loff_t *ppos)
2894{
2895        char *buf_comm;
2896        char *file_buf;
2897        char *buf;
2898        int len = 0;
2899        int pid;
2900        int i;
2901
2902        file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2903        if (!file_buf)
2904                return -ENOMEM;
2905
2906        buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2907        if (!buf_comm) {
2908                kfree(file_buf);
2909                return -ENOMEM;
2910        }
2911
2912        buf = file_buf;
2913
2914        for (i = 0; i < SAVED_CMDLINES; i++) {
2915                int r;
2916
2917                pid = map_cmdline_to_pid[i];
2918                if (pid == -1 || pid == NO_CMDLINE_MAP)
2919                        continue;
2920
2921                trace_find_cmdline(pid, buf_comm);
2922                r = sprintf(buf, "%d %s\n", pid, buf_comm);
2923                buf += r;
2924                len += r;
2925        }
2926
2927        len = simple_read_from_buffer(ubuf, cnt, ppos,
2928                                      file_buf, len);
2929
2930        kfree(file_buf);
2931        kfree(buf_comm);
2932
2933        return len;
2934}
2935
2936static const struct file_operations tracing_saved_cmdlines_fops = {
2937    .open       = tracing_open_generic,
2938    .read       = tracing_saved_cmdlines_read,
2939    .llseek     = generic_file_llseek,
2940};
2941
2942static ssize_t
2943tracing_ctrl_read(struct file *filp, char __user *ubuf,
2944                  size_t cnt, loff_t *ppos)
2945{
2946        char buf[64];
2947        int r;
2948
2949        r = sprintf(buf, "%u\n", tracer_enabled);
2950        return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2951}
2952
2953static ssize_t
2954tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2955                   size_t cnt, loff_t *ppos)
2956{
2957        struct trace_array *tr = filp->private_data;
2958        unsigned long val;
2959        int ret;
2960
2961        ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
2962        if (ret)
2963                return ret;
2964
2965        val = !!val;
2966
2967        mutex_lock(&trace_types_lock);
2968        if (tracer_enabled ^ val) {
2969
2970                /* Only need to warn if this is used to change the state */
2971                WARN_ONCE(1, "tracing_enabled is deprecated. Use tracing_on");
2972
2973                if (val) {
2974                        tracer_enabled = 1;
2975                        if (current_trace->start)
2976                                current_trace->start(tr);
2977                        tracing_start();
2978                } else {
2979                        tracer_enabled = 0;
2980                        tracing_stop();
2981                        if (current_trace->stop)
2982                                current_trace->stop(tr);
2983                }
2984        }
2985        mutex_unlock(&trace_types_lock);
2986
2987        *ppos += cnt;
2988
2989        return cnt;
2990}
2991
2992static ssize_t
2993tracing_set_trace_read(struct file *filp, char __user *ubuf,
2994                       size_t cnt, loff_t *ppos)
2995{
2996        char buf[MAX_TRACER_SIZE+2];
2997        int r;
2998
2999        mutex_lock(&trace_types_lock);
3000        if (current_trace)
3001                r = sprintf(buf, "%s\n", current_trace->name);
3002        else
3003                r = sprintf(buf, "\n");
3004        mutex_unlock(&trace_types_lock);
3005
3006        return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3007}
3008
3009int tracer_init(struct tracer *t, struct trace_array *tr)
3010{
3011        tracing_reset_online_cpus(tr);
3012        return t->init(tr);
3013}
3014
3015static void set_buffer_entries(struct trace_array *tr, unsigned long val)
3016{
3017        int cpu;
3018        for_each_tracing_cpu(cpu)
3019                tr->data[cpu]->entries = val;
3020}
3021
3022static int __tracing_resize_ring_buffer(unsigned long size, int cpu)
3023{
3024        int ret;
3025
3026        /*
3027         * If kernel or user changes the size of the ring buffer
3028         * we use the size that was given, and we can forget about
3029         * expanding it later.
3030         */
3031        ring_buffer_expanded = 1;
3032
3033        ret = ring_buffer_resize(global_trace.buffer, size, cpu);
3034        if (ret < 0)
3035                return ret;
3036
3037        if (!current_trace->use_max_tr)
3038                goto out;
3039
3040        ret = ring_buffer_resize(max_tr.buffer, size, cpu);
3041        if (ret < 0) {
3042                int r = 0;
3043
3044                if (cpu == RING_BUFFER_ALL_CPUS) {
3045                        int i;
3046                        for_each_tracing_cpu(i) {
3047                                r = ring_buffer_resize(global_trace.buffer,
3048                                                global_trace.data[i]->entries,
3049                                                i);
3050                                if (r < 0)
3051                                        break;
3052                        }
3053                } else {
3054                        r = ring_buffer_resize(global_trace.buffer,
3055                                                global_trace.data[cpu]->entries,
3056                                                cpu);
3057                }
3058
3059                if (r < 0) {
3060                        /*
3061                         * AARGH! We are left with different
3062                         * size max buffer!!!!
3063                         * The max buffer is our "snapshot" buffer.
3064                         * When a tracer needs a snapshot (one of the
3065                         * latency tracers), it swaps the max buffer
3066                         * with the saved snap shot. We succeeded to
3067                         * update the size of the main buffer, but failed to
3068                         * update the size of the max buffer. But when we tried
3069                         * to reset the main buffer to the original size, we
3070                         * failed there too. This is very unlikely to
3071                         * happen, but if it does, warn and kill all
3072                         * tracing.
3073                         */
3074                        WARN_ON(1);
3075                        tracing_disabled = 1;
3076                }
3077                return ret;
3078        }
3079
3080        if (cpu == RING_BUFFER_ALL_CPUS)
3081                set_buffer_entries(&max_tr, size);
3082        else
3083                max_tr.data[cpu]->entries = size;
3084
3085 out:
3086        if (cpu == RING_BUFFER_ALL_CPUS)
3087                set_buffer_entries(&global_trace, size);
3088        else
3089                global_trace.data[cpu]->entries = size;
3090
3091        return ret;
3092}
3093
3094static ssize_t tracing_resize_ring_buffer(unsigned long size, int cpu_id)
3095{
3096        int ret = size;
3097
3098        mutex_lock(&trace_types_lock);
3099
3100        if (cpu_id != RING_BUFFER_ALL_CPUS) {
3101                /* make sure, this cpu is enabled in the mask */
3102                if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3103                        ret = -EINVAL;
3104                        goto out;
3105                }
3106        }
3107
3108        ret = __tracing_resize_ring_buffer(size, cpu_id);
3109        if (ret < 0)
3110                ret = -ENOMEM;
3111
3112out:
3113        mutex_unlock(&trace_types_lock);
3114
3115        return ret;
3116}
3117
3118
3119/**
3120 * tracing_update_buffers - used by tracing facility to expand ring buffers
3121 *
3122 * To save on memory when the tracing is never used on a system with it
3123 * configured in. The ring buffers are set to a minimum size. But once
3124 * a user starts to use the tracing facility, then they need to grow
3125 * to their default size.
3126 *
3127 * This function is to be called when a tracer is about to be used.
3128 */
3129int tracing_update_buffers(void)
3130{
3131        int ret = 0;
3132
3133        mutex_lock(&trace_types_lock);
3134        if (!ring_buffer_expanded)
3135                ret = __tracing_resize_ring_buffer(trace_buf_size,
3136                                                RING_BUFFER_ALL_CPUS);
3137        mutex_unlock(&trace_types_lock);
3138
3139        return ret;
3140}
3141
3142struct trace_option_dentry;
3143
3144static struct trace_option_dentry *
3145create_trace_option_files(struct tracer *tracer);
3146
3147static void
3148destroy_trace_option_files(struct trace_option_dentry *topts);
3149
3150static int tracing_set_tracer(const char *buf)
3151{
3152        static struct trace_option_dentry *topts;
3153        struct trace_array *tr = &global_trace;
3154        struct tracer *t;
3155        int ret = 0;
3156
3157        mutex_lock(&trace_types_lock);
3158
3159        if (!ring_buffer_expanded) {
3160                ret = __tracing_resize_ring_buffer(trace_buf_size,
3161                                                RING_BUFFER_ALL_CPUS);
3162                if (ret < 0)
3163                        goto out;
3164                ret = 0;
3165        }
3166
3167        for (t = trace_types; t; t = t->next) {
3168                if (strcmp(t->name, buf) == 0)
3169                        break;
3170        }
3171        if (!t) {
3172                ret = -EINVAL;
3173                goto out;
3174        }
3175        if (t == current_trace)
3176                goto out;
3177
3178        trace_branch_disable();
3179        if (current_trace && current_trace->reset)
3180                current_trace->reset(tr);
3181        if (current_trace && current_trace->use_max_tr) {
3182                /*
3183                 * We don't free the ring buffer. instead, resize it because
3184                 * The max_tr ring buffer has some state (e.g. ring->clock) and
3185                 * we want preserve it.
3186                 */
3187                ring_buffer_resize(max_tr.buffer, 1, RING_BUFFER_ALL_CPUS);
3188                set_buffer_entries(&max_tr, 1);
3189        }
3190        destroy_trace_option_files(topts);
3191
3192        current_trace = &nop_trace;
3193
3194        topts = create_trace_option_files(t);
3195        if (t->use_max_tr) {
3196                int cpu;
3197                /* we need to make per cpu buffer sizes equivalent */
3198                for_each_tracing_cpu(cpu) {
3199                        ret = ring_buffer_resize(max_tr.buffer,
3200                                                global_trace.data[cpu]->entries,
3201                                                cpu);
3202                        if (ret < 0)
3203                                goto out;
3204                        max_tr.data[cpu]->entries =
3205                                        global_trace.data[cpu]->entries;
3206                }
3207        }
3208
3209        if (t->init) {
3210                ret = tracer_init(t, tr);
3211                if (ret)
3212                        goto out;
3213        }
3214
3215        current_trace = t;
3216        trace_branch_enable(tr);
3217 out:
3218        mutex_unlock(&trace_types_lock);
3219
3220        return ret;
3221}
3222
3223static ssize_t
3224tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3225                        size_t cnt, loff_t *ppos)
3226{
3227        char buf[MAX_TRACER_SIZE+1];
3228        int i;
3229        size_t ret;
3230        int err;
3231
3232        ret = cnt;
3233
3234        if (cnt > MAX_TRACER_SIZE)
3235                cnt = MAX_TRACER_SIZE;
3236
3237        if (copy_from_user(&buf, ubuf, cnt))
3238                return -EFAULT;
3239
3240        buf[cnt] = 0;
3241
3242        /* strip ending whitespace. */
3243        for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3244                buf[i] = 0;
3245
3246        err = tracing_set_tracer(buf);
3247        if (err)
3248                return err;
3249
3250        *ppos += ret;
3251
3252        return ret;
3253}
3254
3255static ssize_t
3256tracing_max_lat_read(struct file *filp, char __user *ubuf,
3257                     size_t cnt, loff_t *ppos)
3258{
3259        unsigned long *ptr = filp->private_data;
3260        char buf[64];
3261        int r;
3262
3263        r = snprintf(buf, sizeof(buf), "%ld\n",
3264                     *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
3265        if (r > sizeof(buf))
3266                r = sizeof(buf);
3267        return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3268}
3269
3270static ssize_t
3271tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3272                      size_t cnt, loff_t *ppos)
3273{
3274        unsigned long *ptr = filp->private_data;
3275        unsigned long val;
3276        int ret;
3277
3278        ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3279        if (ret)
3280                return ret;
3281
3282        *ptr = val * 1000;
3283
3284        return cnt;
3285}
3286
3287static int tracing_open_pipe(struct inode *inode, struct file *filp)
3288{
3289        long cpu_file = (long) inode->i_private;
3290        struct trace_iterator *iter;
3291        int ret = 0;
3292
3293        if (tracing_disabled)
3294                return -ENODEV;
3295
3296        mutex_lock(&trace_types_lock);
3297
3298        /* create a buffer to store the information to pass to userspace */
3299        iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3300        if (!iter) {
3301                ret = -ENOMEM;
3302                goto out;
3303        }
3304
3305        /*
3306         * We make a copy of the current tracer to avoid concurrent
3307         * changes on it while we are reading.
3308         */
3309        iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3310        if (!iter->trace) {
3311                ret = -ENOMEM;
3312                goto fail;
3313        }
3314        if (current_trace)
3315                *iter->trace = *current_trace;
3316
3317        if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3318                ret = -ENOMEM;
3319                goto fail;
3320        }
3321
3322        /* trace pipe does not show start of buffer */
3323        cpumask_setall(iter->started);
3324
3325        if (trace_flags & TRACE_ITER_LATENCY_FMT)
3326                iter->iter_flags |= TRACE_FILE_LAT_FMT;
3327
3328        iter->cpu_file = cpu_file;
3329        iter->tr = &global_trace;
3330        mutex_init(&iter->mutex);
3331        filp->private_data = iter;
3332
3333        if (iter->trace->pipe_open)
3334                iter->trace->pipe_open(iter);
3335
3336        nonseekable_open(inode, filp);
3337out:
3338        mutex_unlock(&trace_types_lock);
3339        return ret;
3340
3341fail:
3342        kfree(iter->trace);
3343        kfree(iter);
3344        mutex_unlock(&trace_types_lock);
3345        return ret;
3346}
3347
3348static int tracing_release_pipe(struct inode *inode, struct file *file)
3349{
3350        struct trace_iterator *iter = file->private_data;
3351
3352        mutex_lock(&trace_types_lock);
3353
3354        if (iter->trace->pipe_close)
3355                iter->trace->pipe_close(iter);
3356
3357        mutex_unlock(&trace_types_lock);
3358
3359        free_cpumask_var(iter->started);
3360        mutex_destroy(&iter->mutex);
3361        kfree(iter->trace);
3362        kfree(iter);
3363
3364        return 0;
3365}
3366
3367static unsigned int
3368tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3369{
3370        struct trace_iterator *iter = filp->private_data;
3371
3372        if (trace_flags & TRACE_ITER_BLOCK) {
3373                /*
3374                 * Always select as readable when in blocking mode
3375                 */
3376                return POLLIN | POLLRDNORM;
3377        } else {
3378                if (!trace_empty(iter))
3379                        return POLLIN | POLLRDNORM;
3380                poll_wait(filp, &trace_wait, poll_table);
3381                if (!trace_empty(iter))
3382                        return POLLIN | POLLRDNORM;
3383
3384                return 0;
3385        }
3386}
3387
3388
3389void default_wait_pipe(struct trace_iterator *iter)
3390{
3391        DEFINE_WAIT(wait);
3392
3393        prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
3394
3395        if (trace_empty(iter))
3396                schedule();
3397
3398        finish_wait(&trace_wait, &wait);
3399}
3400
3401/*
3402 * This is a make-shift waitqueue.
3403 * A tracer might use this callback on some rare cases:
3404 *
3405 *  1) the current tracer might hold the runqueue lock when it wakes up
3406 *     a reader, hence a deadlock (sched, function, and function graph tracers)
3407 *  2) the function tracers, trace all functions, we don't want
3408 *     the overhead of calling wake_up and friends
3409 *     (and tracing them too)
3410 *
3411 *     Anyway, this is really very primitive wakeup.
3412 */
3413void poll_wait_pipe(struct trace_iterator *iter)
3414{
3415        set_current_state(TASK_INTERRUPTIBLE);
3416        /* sleep for 100 msecs, and try again. */
3417        schedule_timeout(HZ / 10);
3418}
3419
3420/* Must be called with trace_types_lock mutex held. */
3421static int tracing_wait_pipe(struct file *filp)
3422{
3423        struct trace_iterator *iter = filp->private_data;
3424
3425        while (trace_empty(iter)) {
3426
3427                if ((filp->f_flags & O_NONBLOCK)) {
3428                        return -EAGAIN;
3429                }
3430
3431                mutex_unlock(&iter->mutex);
3432
3433                iter->trace->wait_pipe(iter);
3434
3435                mutex_lock(&iter->mutex);
3436
3437                if (signal_pending(current))
3438                        return -EINTR;
3439
3440                /*
3441                 * We block until we read something and tracing is disabled.
3442                 * We still block if tracing is disabled, but we have never
3443                 * read anything. This allows a user to cat this file, and
3444                 * then enable tracing. But after we have read something,
3445                 * we give an EOF when tracing is again disabled.
3446                 *
3447                 * iter->pos will be 0 if we haven't read anything.
3448                 */
3449                if (!tracer_enabled && iter->pos)
3450                        break;
3451        }
3452
3453        return 1;
3454}
3455
3456/*
3457 * Consumer reader.
3458 */
3459static ssize_t
3460tracing_read_pipe(struct file *filp, char __user *ubuf,
3461                  size_t cnt, loff_t *ppos)
3462{
3463        struct trace_iterator *iter = filp->private_data;
3464        static struct tracer *old_tracer;
3465        ssize_t sret;
3466
3467        /* return any leftover data */
3468        sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3469        if (sret != -EBUSY)
3470                return sret;
3471
3472        trace_seq_init(&iter->seq);
3473
3474        /* copy the tracer to avoid using a global lock all around */
3475        mutex_lock(&trace_types_lock);
3476        if (unlikely(old_tracer != current_trace && current_trace)) {
3477                old_tracer = current_trace;
3478                *iter->trace = *current_trace;
3479        }
3480        mutex_unlock(&trace_types_lock);
3481
3482        /*
3483         * Avoid more than one consumer on a single file descriptor
3484         * This is just a matter of traces coherency, the ring buffer itself
3485         * is protected.
3486         */
3487        mutex_lock(&iter->mutex);
3488        if (iter->trace->read) {
3489                sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3490                if (sret)
3491                        goto out;
3492        }
3493
3494waitagain:
3495        sret = tracing_wait_pipe(filp);
3496        if (sret <= 0)
3497                goto out;
3498
3499        /* stop when tracing is finished */
3500        if (trace_empty(iter)) {
3501                sret = 0;
3502                goto out;
3503        }
3504
3505        if (cnt >= PAGE_SIZE)
3506                cnt = PAGE_SIZE - 1;
3507
3508        /* reset all but tr, trace, and overruns */
3509        memset(&iter->seq, 0,
3510               sizeof(struct trace_iterator) -
3511               offsetof(struct trace_iterator, seq));
3512        iter->pos = -1;
3513
3514        trace_event_read_lock();
3515        trace_access_lock(iter->cpu_file);
3516        while (trace_find_next_entry_inc(iter) != NULL) {
3517                enum print_line_t ret;
3518                int len = iter->seq.len;
3519
3520                ret = print_trace_line(iter);
3521                if (ret == TRACE_TYPE_PARTIAL_LINE) {
3522                        /* don't print partial lines */
3523                        iter->seq.len = len;
3524                        break;
3525                }
3526                if (ret != TRACE_TYPE_NO_CONSUME)
3527                        trace_consume(iter);
3528
3529                if (iter->seq.len >= cnt)
3530                        break;
3531
3532                /*
3533                 * Setting the full flag means we reached the trace_seq buffer
3534                 * size and we should leave by partial output condition above.
3535                 * One of the trace_seq_* functions is not used properly.
3536                 */
3537                WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
3538                          iter->ent->type);
3539        }
3540        trace_access_unlock(iter->cpu_file);
3541        trace_event_read_unlock();
3542
3543        /* Now copy what we have to the user */
3544        sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3545        if (iter->seq.readpos >= iter->seq.len)
3546                trace_seq_init(&iter->seq);
3547
3548        /*
3549         * If there was nothing to send to user, in spite of consuming trace
3550         * entries, go back to wait for more entries.
3551         */
3552        if (sret == -EBUSY)
3553                goto waitagain;
3554
3555out:
3556        mutex_unlock(&iter->mutex);
3557
3558        return sret;
3559}
3560
3561static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3562                                     struct pipe_buffer *buf)
3563{
3564        __free_page(buf->page);
3565}
3566
3567static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3568                                     unsigned int idx)
3569{
3570        __free_page(spd->pages[idx]);
3571}
3572
3573static const struct pipe_buf_operations tracing_pipe_buf_ops = {
3574        .can_merge              = 0,
3575        .map                    = generic_pipe_buf_map,
3576        .unmap                  = generic_pipe_buf_unmap,
3577        .confirm                = generic_pipe_buf_confirm,
3578        .release                = tracing_pipe_buf_release,
3579        .steal                  = generic_pipe_buf_steal,
3580        .get                    = generic_pipe_buf_get,
3581};
3582
3583static size_t
3584tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
3585{
3586        size_t count;
3587        int ret;
3588
3589        /* Seq buffer is page-sized, exactly what we need. */
3590        for (;;) {
3591                count = iter->seq.len;
3592                ret = print_trace_line(iter);
3593                count = iter->seq.len - count;
3594                if (rem < count) {
3595                        rem = 0;
3596                        iter->seq.len -= count;
3597                        break;
3598                }
3599                if (ret == TRACE_TYPE_PARTIAL_LINE) {
3600                        iter->seq.len -= count;
3601                        break;
3602                }
3603
3604                if (ret != TRACE_TYPE_NO_CONSUME)
3605                        trace_consume(iter);
3606                rem -= count;
3607                if (!trace_find_next_entry_inc(iter))   {
3608                        rem = 0;
3609                        iter->ent = NULL;
3610                        break;
3611                }
3612        }
3613
3614        return rem;
3615}
3616
3617static ssize_t tracing_splice_read_pipe(struct file *filp,
3618                                        loff_t *ppos,
3619                                        struct pipe_inode_info *pipe,
3620                                        size_t len,
3621                                        unsigned int flags)
3622{
3623        struct page *pages_def[PIPE_DEF_BUFFERS];
3624        struct partial_page partial_def[PIPE_DEF_BUFFERS];
3625        struct trace_iterator *iter = filp->private_data;
3626        struct splice_pipe_desc spd = {
3627                .pages          = pages_def,
3628                .partial        = partial_def,
3629                .nr_pages       = 0, /* This gets updated below. */
3630                .nr_pages_max   = PIPE_DEF_BUFFERS,
3631                .flags          = flags,
3632                .ops            = &tracing_pipe_buf_ops,
3633                .spd_release    = tracing_spd_release_pipe,
3634        };
3635        static struct tracer *old_tracer;
3636        ssize_t ret;
3637        size_t rem;
3638        unsigned int i;
3639
3640        if (splice_grow_spd(pipe, &spd))
3641                return -ENOMEM;
3642
3643        /* copy the tracer to avoid using a global lock all around */
3644        mutex_lock(&trace_types_lock);
3645        if (unlikely(old_tracer != current_trace && current_trace)) {
3646                old_tracer = current_trace;
3647                *iter->trace = *current_trace;
3648        }
3649        mutex_unlock(&trace_types_lock);
3650
3651        mutex_lock(&iter->mutex);
3652
3653        if (iter->trace->splice_read) {
3654                ret = iter->trace->splice_read(iter, filp,
3655                                               ppos, pipe, len, flags);
3656                if (ret)
3657                        goto out_err;
3658        }
3659
3660        ret = tracing_wait_pipe(filp);
3661        if (ret <= 0)
3662                goto out_err;
3663
3664        if (!iter->ent && !trace_find_next_entry_inc(iter)) {
3665                ret = -EFAULT;
3666                goto out_err;
3667        }
3668
3669        trace_event_read_lock();
3670        trace_access_lock(iter->cpu_file);
3671
3672        /* Fill as many pages as possible. */
3673        for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
3674                spd.pages[i] = alloc_page(GFP_KERNEL);
3675                if (!spd.pages[i])
3676                        break;
3677
3678                rem = tracing_fill_pipe_page(rem, iter);
3679
3680                /* Copy the data into the page, so we can start over. */
3681                ret = trace_seq_to_buffer(&iter->seq,
3682                                          page_address(spd.pages[i]),
3683                                          iter->seq.len);
3684                if (ret < 0) {
3685                        __free_page(spd.pages[i]);
3686                        break;
3687                }
3688                spd.partial[i].offset = 0;
3689                spd.partial[i].len = iter->seq.len;
3690
3691                trace_seq_init(&iter->seq);
3692        }
3693
3694        trace_access_unlock(iter->cpu_file);
3695        trace_event_read_unlock();
3696        mutex_unlock(&iter->mutex);
3697
3698        spd.nr_pages = i;
3699
3700        ret = splice_to_pipe(pipe, &spd);
3701out:
3702        splice_shrink_spd(&spd);
3703        return ret;
3704
3705out_err:
3706        mutex_unlock(&iter->mutex);
3707        goto out;
3708}
3709
3710struct ftrace_entries_info {
3711        struct trace_array      *tr;
3712        int                     cpu;
3713};
3714
3715static int tracing_entries_open(struct inode *inode, struct file *filp)
3716{
3717        struct ftrace_entries_info *info;
3718
3719        if (tracing_disabled)
3720                return -ENODEV;
3721
3722        info = kzalloc(sizeof(*info), GFP_KERNEL);
3723        if (!info)
3724                return -ENOMEM;
3725
3726        info->tr = &global_trace;
3727        info->cpu = (unsigned long)inode->i_private;
3728
3729        filp->private_data = info;
3730
3731        return 0;
3732}
3733
3734static ssize_t
3735tracing_entries_read(struct file *filp, char __user *ubuf,
3736                     size_t cnt, loff_t *ppos)
3737{
3738        struct ftrace_entries_info *info = filp->private_data;
3739        struct trace_array *tr = info->tr;
3740        char buf[64];
3741        int r = 0;
3742        ssize_t ret;
3743
3744        mutex_lock(&trace_types_lock);
3745
3746        if (info->cpu == RING_BUFFER_ALL_CPUS) {
3747                int cpu, buf_size_same;
3748                unsigned long size;
3749
3750                size = 0;
3751                buf_size_same = 1;
3752                /* check if all cpu sizes are same */
3753                for_each_tracing_cpu(cpu) {
3754                        /* fill in the size from first enabled cpu */
3755                        if (size == 0)
3756                                size = tr->data[cpu]->entries;
3757                        if (size != tr->data[cpu]->entries) {
3758                                buf_size_same = 0;
3759                                break;
3760                        }
3761                }
3762
3763                if (buf_size_same) {
3764                        if (!ring_buffer_expanded)
3765                                r = sprintf(buf, "%lu (expanded: %lu)\n",
3766                                            size >> 10,
3767                                            trace_buf_size >> 10);
3768                        else
3769                                r = sprintf(buf, "%lu\n", size >> 10);
3770                } else
3771                        r = sprintf(buf, "X\n");
3772        } else
3773                r = sprintf(buf, "%lu\n", tr->data[info->cpu]->entries >> 10);
3774
3775        mutex_unlock(&trace_types_lock);
3776
3777        ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3778        return ret;
3779}
3780
3781static ssize_t
3782tracing_entries_write(struct file *filp, const char __user *ubuf,
3783                      size_t cnt, loff_t *ppos)
3784{
3785        struct ftrace_entries_info *info = filp->private_data;
3786        unsigned long val;
3787        int ret;
3788
3789        ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3790        if (ret)
3791                return ret;
3792
3793        /* must have at least 1 entry */
3794        if (!val)
3795                return -EINVAL;
3796
3797        /* value is in KB */
3798        val <<= 10;
3799
3800        ret = tracing_resize_ring_buffer(val, info->cpu);
3801        if (ret < 0)
3802                return ret;
3803
3804        *ppos += cnt;
3805
3806        return cnt;
3807}
3808
3809static int
3810tracing_entries_release(struct inode *inode, struct file *filp)
3811{
3812        struct ftrace_entries_info *info = filp->private_data;
3813
3814        kfree(info);
3815
3816        return 0;
3817}
3818
3819static ssize_t
3820tracing_total_entries_read(struct file *filp, char __user *ubuf,
3821                                size_t cnt, loff_t *ppos)
3822{
3823        struct trace_array *tr = filp->private_data;
3824        char buf[64];
3825        int r, cpu;
3826        unsigned long size = 0, expanded_size = 0;
3827
3828        mutex_lock(&trace_types_lock);
3829        for_each_tracing_cpu(cpu) {
3830                size += tr->data[cpu]->entries >> 10;
3831                if (!ring_buffer_expanded)
3832                        expanded_size += trace_buf_size >> 10;
3833        }
3834        if (ring_buffer_expanded)
3835                r = sprintf(buf, "%lu\n", size);
3836        else
3837                r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
3838        mutex_unlock(&trace_types_lock);
3839
3840        return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3841}
3842
3843static ssize_t
3844tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
3845                          size_t cnt, loff_t *ppos)
3846{
3847        /*
3848         * There is no need to read what the user has written, this function
3849         * is just to make sure that there is no error when "echo" is used
3850         */
3851
3852        *ppos += cnt;
3853
3854        return cnt;
3855}
3856
3857static int
3858tracing_free_buffer_release(struct inode *inode, struct file *filp)
3859{
3860        /* disable tracing ? */
3861        if (trace_flags & TRACE_ITER_STOP_ON_FREE)
3862                tracing_off();
3863        /* resize the ring buffer to 0 */
3864        tracing_resize_ring_buffer(0, RING_BUFFER_ALL_CPUS);
3865
3866        return 0;
3867}
3868
3869static ssize_t
3870tracing_mark_write(struct file *filp, const char __user *ubuf,
3871                                        size_t cnt, loff_t *fpos)
3872{
3873        unsigned long addr = (unsigned long)ubuf;
3874        struct ring_buffer_event *event;
3875        struct ring_buffer *buffer;
3876        struct print_entry *entry;
3877        unsigned long irq_flags;
3878        struct page *pages[2];
3879        void *map_page[2];
3880        int nr_pages = 1;
3881        ssize_t written;
3882        int offset;
3883        int size;
3884        int len;
3885        int ret;
3886        int i;
3887
3888        if (tracing_disabled)
3889                return -EINVAL;
3890
3891        if (!(trace_flags & TRACE_ITER_MARKERS))
3892                return -EINVAL;
3893
3894        if (cnt > TRACE_BUF_SIZE)
3895                cnt = TRACE_BUF_SIZE;
3896
3897        /*
3898         * Userspace is injecting traces into the kernel trace buffer.
3899         * We want to be as non intrusive as possible.
3900         * To do so, we do not want to allocate any special buffers
3901         * or take any locks, but instead write the userspace data
3902         * straight into the ring buffer.
3903         *
3904         * First we need to pin the userspace buffer into memory,
3905         * which, most likely it is, because it just referenced it.
3906         * But there's no guarantee that it is. By using get_user_pages_fast()
3907         * and kmap_atomic/kunmap_atomic() we can get access to the
3908         * pages directly. We then write the data directly into the
3909         * ring buffer.
3910         */
3911        BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
3912
3913        /* check if we cross pages */
3914        if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
3915                nr_pages = 2;
3916
3917        offset = addr & (PAGE_SIZE - 1);
3918        addr &= PAGE_MASK;
3919
3920        ret = get_user_pages_fast(addr, nr_pages, 0, pages);
3921        if (ret < nr_pages) {
3922                while (--ret >= 0)
3923                        put_page(pages[ret]);
3924                written = -EFAULT;
3925                goto out;
3926        }
3927
3928        for (i = 0; i < nr_pages; i++)
3929                map_page[i] = kmap_atomic(pages[i]);
3930
3931        local_save_flags(irq_flags);
3932        size = sizeof(*entry) + cnt + 2; /* possible \n added */
3933        buffer = global_trace.buffer;
3934        event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3935                                          irq_flags, preempt_count());
3936        if (!event) {
3937                /* Ring buffer disabled, return as if not open for write */
3938                written = -EBADF;
3939                goto out_unlock;
3940        }
3941
3942        entry = ring_buffer_event_data(event);
3943        entry->ip = _THIS_IP_;
3944
3945        if (nr_pages == 2) {
3946                len = PAGE_SIZE - offset;
3947                memcpy(&entry->buf, map_page[0] + offset, len);
3948                memcpy(&entry->buf[len], map_page[1], cnt - len);
3949        } else
3950                memcpy(&entry->buf, map_page[0] + offset, cnt);
3951
3952        if (entry->buf[cnt - 1] != '\n') {
3953                entry->buf[cnt] = '\n';
3954                entry->buf[cnt + 1] = '\0';
3955        } else
3956                entry->buf[cnt] = '\0';
3957
3958        ring_buffer_unlock_commit(buffer, event);
3959
3960        written = cnt;
3961
3962        *fpos += written;
3963
3964 out_unlock:
3965        for (i = 0; i < nr_pages; i++){
3966                kunmap_atomic(map_page[i]);
3967                put_page(pages[i]);
3968        }
3969 out:
3970        return written;
3971}
3972
3973static int tracing_clock_show(struct seq_file *m, void *v)
3974{
3975        int i;
3976
3977        for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
3978                seq_printf(m,
3979                        "%s%s%s%s", i ? " " : "",
3980                        i == trace_clock_id ? "[" : "", trace_clocks[i].name,
3981                        i == trace_clock_id ? "]" : "");
3982        seq_putc(m, '\n');
3983
3984        return 0;
3985}
3986
3987static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
3988                                   size_t cnt, loff_t *fpos)
3989{
3990        char buf[64];
3991        const char *clockstr;
3992        int i;
3993
3994        if (cnt >= sizeof(buf))
3995                return -EINVAL;
3996
3997        if (copy_from_user(&buf, ubuf, cnt))
3998                return -EFAULT;
3999
4000        buf[cnt] = 0;
4001
4002        clockstr = strstrip(buf);
4003
4004        for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4005                if (strcmp(trace_clocks[i].name, clockstr) == 0)
4006                        break;
4007        }
4008        if (i == ARRAY_SIZE(trace_clocks))
4009                return -EINVAL;
4010
4011        trace_clock_id = i;
4012
4013        mutex_lock(&trace_types_lock);
4014
4015        ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
4016        if (max_tr.buffer)
4017                ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
4018
4019        mutex_unlock(&trace_types_lock);
4020
4021        *fpos += cnt;
4022
4023        return cnt;
4024}
4025
4026static int tracing_clock_open(struct inode *inode, struct file *file)
4027{
4028        if (tracing_disabled)
4029                return -ENODEV;
4030        return single_open(file, tracing_clock_show, NULL);
4031}
4032
4033static const struct file_operations tracing_max_lat_fops = {
4034        .open           = tracing_open_generic,
4035        .read           = tracing_max_lat_read,
4036        .write          = tracing_max_lat_write,
4037        .llseek         = generic_file_llseek,
4038};
4039
4040static const struct file_operations tracing_ctrl_fops = {
4041        .open           = tracing_open_generic,
4042        .read           = tracing_ctrl_read,
4043        .write          = tracing_ctrl_write,
4044        .llseek         = generic_file_llseek,
4045};
4046
4047static const struct file_operations set_tracer_fops = {
4048        .open           = tracing_open_generic,
4049        .read           = tracing_set_trace_read,
4050        .write          = tracing_set_trace_write,
4051        .llseek         = generic_file_llseek,
4052};
4053
4054static const struct file_operations tracing_pipe_fops = {
4055        .open           = tracing_open_pipe,
4056        .poll           = tracing_poll_pipe,
4057        .read           = tracing_read_pipe,
4058        .splice_read    = tracing_splice_read_pipe,
4059        .release        = tracing_release_pipe,
4060        .llseek         = no_llseek,
4061};
4062
4063static const struct file_operations tracing_entries_fops = {
4064        .open           = tracing_entries_open,
4065        .read           = tracing_entries_read,
4066        .write          = tracing_entries_write,
4067        .release        = tracing_entries_release,
4068        .llseek         = generic_file_llseek,
4069};
4070
4071static const struct file_operations tracing_total_entries_fops = {
4072        .open           = tracing_open_generic,
4073        .read           = tracing_total_entries_read,
4074        .llseek         = generic_file_llseek,
4075};
4076
4077static const struct file_operations tracing_free_buffer_fops = {
4078        .write          = tracing_free_buffer_write,
4079        .release        = tracing_free_buffer_release,
4080};
4081
4082static const struct file_operations tracing_mark_fops = {
4083        .open           = tracing_open_generic,
4084        .write          = tracing_mark_write,
4085        .llseek         = generic_file_llseek,
4086};
4087
4088static const struct file_operations trace_clock_fops = {
4089        .open           = tracing_clock_open,
4090        .read           = seq_read,
4091        .llseek         = seq_lseek,
4092        .release        = single_release,
4093        .write          = tracing_clock_write,
4094};
4095
4096struct ftrace_buffer_info {
4097        struct trace_array      *tr;
4098        void                    *spare;
4099        int                     cpu;
4100        unsigned int            read;
4101};
4102
4103static int tracing_buffers_open(struct inode *inode, struct file *filp)
4104{
4105        int cpu = (int)(long)inode->i_private;
4106        struct ftrace_buffer_info *info;
4107
4108        if (tracing_disabled)
4109                return -ENODEV;
4110
4111        info = kzalloc(sizeof(*info), GFP_KERNEL);
4112        if (!info)
4113                return -ENOMEM;
4114
4115        info->tr        = &global_trace;
4116        info->cpu       = cpu;
4117        info->spare     = NULL;
4118        /* Force reading ring buffer for first read */
4119        info->read      = (unsigned int)-1;
4120
4121        filp->private_data = info;
4122
4123        return nonseekable_open(inode, filp);
4124}
4125
4126static ssize_t
4127tracing_buffers_read(struct file *filp, char __user *ubuf,
4128                     size_t count, loff_t *ppos)
4129{
4130        struct ftrace_buffer_info *info = filp->private_data;
4131        ssize_t ret;
4132        size_t size;
4133
4134        if (!count)
4135                return 0;
4136
4137        if (!info->spare)
4138                info->spare = ring_buffer_alloc_read_page(info->tr->buffer, info->cpu);
4139        if (!info->spare)
4140                return -ENOMEM;
4141
4142        /* Do we have previous read data to read? */
4143        if (info->read < PAGE_SIZE)
4144                goto read;
4145
4146        trace_access_lock(info->cpu);
4147        ret = ring_buffer_read_page(info->tr->buffer,
4148                                    &info->spare,
4149                                    count,
4150                                    info->cpu, 0);
4151        trace_access_unlock(info->cpu);
4152        if (ret < 0)
4153                return 0;
4154
4155        info->read = 0;
4156
4157read:
4158        size = PAGE_SIZE - info->read;
4159        if (size > count)
4160                size = count;
4161
4162        ret = copy_to_user(ubuf, info->spare + info->read, size);
4163        if (ret == size)
4164                return -EFAULT;
4165        size -= ret;
4166
4167        *ppos += size;
4168        info->read += size;
4169
4170        return size;
4171}
4172
4173static int tracing_buffers_release(struct inode *inode, struct file *file)
4174{
4175        struct ftrace_buffer_info *info = file->private_data;
4176
4177        if (info->spare)
4178                ring_buffer_free_read_page(info->tr->buffer, info->spare);
4179        kfree(info);
4180
4181        return 0;
4182}
4183
4184struct buffer_ref {
4185        struct ring_buffer      *buffer;
4186        void                    *page;
4187        int                     ref;
4188};
4189
4190static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
4191                                    struct pipe_buffer *buf)
4192{
4193        struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4194
4195        if (--ref->ref)
4196                return;
4197
4198        ring_buffer_free_read_page(ref->buffer, ref->page);
4199        kfree(ref);
4200        buf->private = 0;
4201}
4202
4203static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
4204                                struct pipe_buffer *buf)
4205{
4206        struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4207
4208        ref->ref++;
4209}
4210
4211/* Pipe buffer operations for a buffer. */
4212static const struct pipe_buf_operations buffer_pipe_buf_ops = {
4213        .can_merge              = 0,
4214        .map                    = generic_pipe_buf_map,
4215        .unmap                  = generic_pipe_buf_unmap,
4216        .confirm                = generic_pipe_buf_confirm,
4217        .release                = buffer_pipe_buf_release,
4218        .steal                  = generic_pipe_buf_steal,
4219        .get                    = buffer_pipe_buf_get,
4220};
4221
4222/*
4223 * Callback from splice_to_pipe(), if we need to release some pages
4224 * at the end of the spd in case we error'ed out in filling the pipe.
4225 */
4226static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
4227{
4228        struct buffer_ref *ref =
4229                (struct buffer_ref *)spd->partial[i].private;
4230
4231        if (--ref->ref)
4232                return;
4233
4234        ring_buffer_free_read_page(ref->buffer, ref->page);
4235        kfree(ref);
4236        spd->partial[i].private = 0;
4237}
4238
4239static ssize_t
4240tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4241                            struct pipe_inode_info *pipe, size_t len,
4242                            unsigned int flags)
4243{
4244        struct ftrace_buffer_info *info = file->private_data;
4245        struct partial_page partial_def[PIPE_DEF_BUFFERS];
4246        struct page *pages_def[PIPE_DEF_BUFFERS];
4247        struct splice_pipe_desc spd = {
4248                .pages          = pages_def,
4249                .partial        = partial_def,
4250                .nr_pages_max   = PIPE_DEF_BUFFERS,
4251                .flags          = flags,
4252                .ops            = &buffer_pipe_buf_ops,
4253                .spd_release    = buffer_spd_release,
4254        };
4255        struct buffer_ref *ref;
4256        int entries, size, i;
4257        size_t ret;
4258
4259        if (splice_grow_spd(pipe, &spd))
4260                return -ENOMEM;
4261
4262        if (*ppos & (PAGE_SIZE - 1)) {
4263                WARN_ONCE(1, "Ftrace: previous read must page-align\n");
4264                ret = -EINVAL;
4265                goto out;
4266        }
4267
4268        if (len & (PAGE_SIZE - 1)) {
4269                WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
4270                if (len < PAGE_SIZE) {
4271                        ret = -EINVAL;
4272                        goto out;
4273                }
4274                len &= PAGE_MASK;
4275        }
4276
4277        trace_access_lock(info->cpu);
4278        entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4279
4280        for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
4281                struct page *page;
4282                int r;
4283
4284                ref = kzalloc(sizeof(*ref), GFP_KERNEL);
4285                if (!ref)
4286                        break;
4287
4288                ref->ref = 1;
4289                ref->buffer = info->tr->buffer;
4290                ref->page = ring_buffer_alloc_read_page(ref->buffer, info->cpu);
4291                if (!ref->page) {
4292                        kfree(ref);
4293                        break;
4294                }
4295
4296                r = ring_buffer_read_page(ref->buffer, &ref->page,
4297                                          len, info->cpu, 1);
4298                if (r < 0) {
4299                        ring_buffer_free_read_page(ref->buffer, ref->page);
4300                        kfree(ref);
4301                        break;
4302                }
4303
4304                /*
4305                 * zero out any left over data, this is going to
4306                 * user land.
4307                 */
4308                size = ring_buffer_page_len(ref->page);
4309                if (size < PAGE_SIZE)
4310                        memset(ref->page + size, 0, PAGE_SIZE - size);
4311
4312                page = virt_to_page(ref->page);
4313
4314                spd.pages[i] = page;
4315                spd.partial[i].len = PAGE_SIZE;
4316                spd.partial[i].offset = 0;
4317                spd.partial[i].private = (unsigned long)ref;
4318                spd.nr_pages++;
4319                *ppos += PAGE_SIZE;
4320
4321                entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4322        }
4323
4324        trace_access_unlock(info->cpu);
4325        spd.nr_pages = i;
4326
4327        /* did we read anything? */
4328        if (!spd.nr_pages) {
4329                if (flags & SPLICE_F_NONBLOCK)
4330                        ret = -EAGAIN;
4331                else
4332                        ret = 0;
4333                /* TODO: block */
4334                goto out;
4335        }
4336
4337        ret = splice_to_pipe(pipe, &spd);
4338        splice_shrink_spd(&spd);
4339out:
4340        return ret;
4341}
4342
4343static const struct file_operations tracing_buffers_fops = {
4344        .open           = tracing_buffers_open,
4345        .read           = tracing_buffers_read,
4346        .release        = tracing_buffers_release,
4347        .splice_read    = tracing_buffers_splice_read,
4348        .llseek         = no_llseek,
4349};
4350
4351static ssize_t
4352tracing_stats_read(struct file *filp, char __user *ubuf,
4353                   size_t count, loff_t *ppos)
4354{
4355        unsigned long cpu = (unsigned long)filp->private_data;
4356        struct trace_array *tr = &global_trace;
4357        struct trace_seq *s;
4358        unsigned long cnt;
4359        unsigned long long t;
4360        unsigned long usec_rem;
4361
4362        s = kmalloc(sizeof(*s), GFP_KERNEL);
4363        if (!s)
4364                return -ENOMEM;
4365
4366        trace_seq_init(s);
4367
4368        cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
4369        trace_seq_printf(s, "entries: %ld\n", cnt);
4370
4371        cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
4372        trace_seq_printf(s, "overrun: %ld\n", cnt);
4373
4374        cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
4375        trace_seq_printf(s, "commit overrun: %ld\n", cnt);
4376
4377        cnt = ring_buffer_bytes_cpu(tr->buffer, cpu);
4378        trace_seq_printf(s, "bytes: %ld\n", cnt);
4379
4380        t = ns2usecs(ring_buffer_oldest_event_ts(tr->buffer, cpu));
4381        usec_rem = do_div(t, USEC_PER_SEC);
4382        trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n", t, usec_rem);
4383
4384        t = ns2usecs(ring_buffer_time_stamp(tr->buffer, cpu));
4385        usec_rem = do_div(t, USEC_PER_SEC);
4386        trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
4387
4388        count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
4389
4390        kfree(s);
4391
4392        return count;
4393}
4394
4395static const struct file_operations tracing_stats_fops = {
4396        .open           = tracing_open_generic,
4397        .read           = tracing_stats_read,
4398        .llseek         = generic_file_llseek,
4399};
4400
4401#ifdef CONFIG_DYNAMIC_FTRACE
4402
4403int __weak ftrace_arch_read_dyn_info(char *buf, int size)
4404{
4405        return 0;
4406}
4407
4408static ssize_t
4409tracing_read_dyn_info(struct file *filp, char __user *ubuf,
4410                  size_t cnt, loff_t *ppos)
4411{
4412        static char ftrace_dyn_info_buffer[1024];
4413        static DEFINE_MUTEX(dyn_info_mutex);
4414        unsigned long *p = filp->private_data;
4415        char *buf = ftrace_dyn_info_buffer;
4416        int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
4417        int r;
4418
4419        mutex_lock(&dyn_info_mutex);
4420        r = sprintf(buf, "%ld ", *p);
4421
4422        r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
4423        buf[r++] = '\n';
4424
4425        r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4426
4427        mutex_unlock(&dyn_info_mutex);
4428
4429        return r;
4430}
4431
4432static const struct file_operations tracing_dyn_info_fops = {
4433        .open           = tracing_open_generic,
4434        .read           = tracing_read_dyn_info,
4435        .llseek         = generic_file_llseek,
4436};
4437#endif
4438
4439static struct dentry *d_tracer;
4440
4441struct dentry *tracing_init_dentry(void)
4442{
4443        static int once;
4444
4445        if (d_tracer)
4446                return d_tracer;
4447
4448        if (!debugfs_initialized())
4449                return NULL;
4450
4451        d_tracer = debugfs_create_dir("tracing", NULL);
4452
4453        if (!d_tracer && !once) {
4454                once = 1;
4455                pr_warning("Could not create debugfs directory 'tracing'\n");
4456                return NULL;
4457        }
4458
4459        return d_tracer;
4460}
4461
4462static struct dentry *d_percpu;
4463
4464struct dentry *tracing_dentry_percpu(void)
4465{
4466        static int once;
4467        struct dentry *d_tracer;
4468
4469        if (d_percpu)
4470                return d_percpu;
4471
4472        d_tracer = tracing_init_dentry();
4473
4474        if (!d_tracer)
4475                return NULL;
4476
4477        d_percpu = debugfs_create_dir("per_cpu", d_tracer);
4478
4479        if (!d_percpu && !once) {
4480                once = 1;
4481                pr_warning("Could not create debugfs directory 'per_cpu'\n");
4482                return NULL;
4483        }
4484
4485        return d_percpu;
4486}
4487
4488static void tracing_init_debugfs_percpu(long cpu)
4489{
4490        struct dentry *d_percpu = tracing_dentry_percpu();
4491        struct dentry *d_cpu;
4492        char cpu_dir[30]; /* 30 characters should be more than enough */
4493
4494        if (!d_percpu)
4495                return;
4496
4497        snprintf(cpu_dir, 30, "cpu%ld", cpu);
4498        d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
4499        if (!d_cpu) {
4500                pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
4501                return;
4502        }
4503
4504        /* per cpu trace_pipe */
4505        trace_create_file("trace_pipe", 0444, d_cpu,
4506                        (void *) cpu, &tracing_pipe_fops);
4507
4508        /* per cpu trace */
4509        trace_create_file("trace", 0644, d_cpu,
4510                        (void *) cpu, &tracing_fops);
4511
4512        trace_create_file("trace_pipe_raw", 0444, d_cpu,
4513                        (void *) cpu, &tracing_buffers_fops);
4514
4515        trace_create_file("stats", 0444, d_cpu,
4516                        (void *) cpu, &tracing_stats_fops);
4517
4518        trace_create_file("buffer_size_kb", 0444, d_cpu,
4519                        (void *) cpu, &tracing_entries_fops);
4520}
4521
4522#ifdef CONFIG_FTRACE_SELFTEST
4523/* Let selftest have access to static functions in this file */
4524#include "trace_selftest.c"
4525#endif
4526
4527struct trace_option_dentry {
4528        struct tracer_opt               *opt;
4529        struct tracer_flags             *flags;
4530        struct dentry                   *entry;
4531};
4532
4533static ssize_t
4534trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
4535                        loff_t *ppos)
4536{
4537        struct trace_option_dentry *topt = filp->private_data;
4538        char *buf;
4539
4540        if (topt->flags->val & topt->opt->bit)
4541                buf = "1\n";
4542        else
4543                buf = "0\n";
4544
4545        return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4546}
4547
4548static ssize_t
4549trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
4550                         loff_t *ppos)
4551{
4552        struct trace_option_dentry *topt = filp->private_data;
4553        unsigned long val;
4554        int ret;
4555
4556        ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4557        if (ret)
4558                return ret;
4559
4560        if (val != 0 && val != 1)
4561                return -EINVAL;
4562
4563        if (!!(topt->flags->val & topt->opt->bit) != val) {
4564                mutex_lock(&trace_types_lock);
4565                ret = __set_tracer_option(current_trace, topt->flags,
4566                                          topt->opt, !val);
4567                mutex_unlock(&trace_types_lock);
4568                if (ret)
4569                        return ret;
4570        }
4571
4572        *ppos += cnt;
4573
4574        return cnt;
4575}
4576
4577
4578static const struct file_operations trace_options_fops = {
4579        .open = tracing_open_generic,
4580        .read = trace_options_read,
4581        .write = trace_options_write,
4582        .llseek = generic_file_llseek,
4583};
4584
4585static ssize_t
4586trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
4587                        loff_t *ppos)
4588{
4589        long index = (long)filp->private_data;
4590        char *buf;
4591
4592        if (trace_flags & (1 << index))
4593                buf = "1\n";
4594        else
4595                buf = "0\n";
4596
4597        return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4598}
4599
4600static ssize_t
4601trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
4602                         loff_t *ppos)
4603{
4604        long index = (long)filp->private_data;
4605        unsigned long val;
4606        int ret;
4607
4608        ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4609        if (ret)
4610                return ret;
4611
4612        if (val != 0 && val != 1)
4613                return -EINVAL;
4614        set_tracer_flags(1 << index, val);
4615
4616        *ppos += cnt;
4617
4618        return cnt;
4619}
4620
4621static const struct file_operations trace_options_core_fops = {
4622        .open = tracing_open_generic,
4623        .read = trace_options_core_read,
4624        .write = trace_options_core_write,
4625        .llseek = generic_file_llseek,
4626};
4627
4628struct dentry *trace_create_file(const char *name,
4629                                 umode_t mode,
4630                                 struct dentry *parent,
4631                                 void *data,
4632                                 const struct file_operations *fops)
4633{
4634        struct dentry *ret;
4635
4636        ret = debugfs_create_file(name, mode, parent, data, fops);
4637        if (!ret)
4638                pr_warning("Could not create debugfs '%s' entry\n", name);
4639
4640        return ret;
4641}
4642
4643
4644static struct dentry *trace_options_init_dentry(void)
4645{
4646        struct dentry *d_tracer;
4647        static struct dentry *t_options;
4648
4649        if (t_options)
4650                return t_options;
4651
4652        d_tracer = tracing_init_dentry();
4653        if (!d_tracer)
4654                return NULL;
4655
4656        t_options = debugfs_create_dir("options", d_tracer);
4657        if (!t_options) {
4658                pr_warning("Could not create debugfs directory 'options'\n");
4659                return NULL;
4660        }
4661
4662        return t_options;
4663}
4664
4665static void
4666create_trace_option_file(struct trace_option_dentry *topt,
4667                         struct tracer_flags *flags,
4668                         struct tracer_opt *opt)
4669{
4670        struct dentry *t_options;
4671
4672        t_options = trace_options_init_dentry();
4673        if (!t_options)
4674                return;
4675
4676        topt->flags = flags;
4677        topt->opt = opt;
4678
4679        topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
4680                                    &trace_options_fops);
4681
4682}
4683
4684static struct trace_option_dentry *
4685create_trace_option_files(struct tracer *tracer)
4686{
4687        struct trace_option_dentry *topts;
4688        struct tracer_flags *flags;
4689        struct tracer_opt *opts;
4690        int cnt;
4691
4692        if (!tracer)
4693                return NULL;
4694
4695        flags = tracer->flags;
4696
4697        if (!flags || !flags->opts)
4698                return NULL;
4699
4700        opts = flags->opts;
4701
4702        for (cnt = 0; opts[cnt].name; cnt++)
4703                ;
4704
4705        topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
4706        if (!topts)
4707                return NULL;
4708
4709        for (cnt = 0; opts[cnt].name; cnt++)
4710                create_trace_option_file(&topts[cnt], flags,
4711                                         &opts[cnt]);
4712
4713        return topts;
4714}
4715
4716static void
4717destroy_trace_option_files(struct trace_option_dentry *topts)
4718{
4719        int cnt;
4720
4721        if (!topts)
4722                return;
4723
4724        for (cnt = 0; topts[cnt].opt; cnt++) {
4725                if (topts[cnt].entry)
4726                        debugfs_remove(topts[cnt].entry);
4727        }
4728
4729        kfree(topts);
4730}
4731
4732static struct dentry *
4733create_trace_option_core_file(const char *option, long index)
4734{
4735        struct dentry *t_options;
4736
4737        t_options = trace_options_init_dentry();
4738        if (!t_options)
4739                return NULL;
4740
4741        return trace_create_file(option, 0644, t_options, (void *)index,
4742                                    &trace_options_core_fops);
4743}
4744
4745static __init void create_trace_options_dir(void)
4746{
4747        struct dentry *t_options;
4748        int i;
4749
4750        t_options = trace_options_init_dentry();
4751        if (!t_options)
4752                return;
4753
4754        for (i = 0; trace_options[i]; i++)
4755                create_trace_option_core_file(trace_options[i], i);
4756}
4757
4758static ssize_t
4759rb_simple_read(struct file *filp, char __user *ubuf,
4760               size_t cnt, loff_t *ppos)
4761{
4762        struct trace_array *tr = filp->private_data;
4763        struct ring_buffer *buffer = tr->buffer;
4764        char buf[64];
4765        int r;
4766
4767        if (buffer)
4768                r = ring_buffer_record_is_on(buffer);
4769        else
4770                r = 0;
4771
4772        r = sprintf(buf, "%d\n", r);
4773
4774        return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4775}
4776
4777static ssize_t
4778rb_simple_write(struct file *filp, const char __user *ubuf,
4779                size_t cnt, loff_t *ppos)
4780{
4781        struct trace_array *tr = filp->private_data;
4782        struct ring_buffer *buffer = tr->buffer;
4783        unsigned long val;
4784        int ret;
4785
4786        ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4787        if (ret)
4788                return ret;
4789
4790        if (buffer) {
4791                if (val)
4792                        ring_buffer_record_on(buffer);
4793                else
4794                        ring_buffer_record_off(buffer);
4795        }
4796
4797        (*ppos)++;
4798
4799        return cnt;
4800}
4801
4802static const struct file_operations rb_simple_fops = {
4803        .open           = tracing_open_generic,
4804        .read           = rb_simple_read,
4805        .write          = rb_simple_write,
4806        .llseek         = default_llseek,
4807};
4808
4809static __init int tracer_init_debugfs(void)
4810{
4811        struct dentry *d_tracer;
4812        int cpu;
4813
4814        trace_access_lock_init();
4815
4816        d_tracer = tracing_init_dentry();
4817
4818        trace_create_file("tracing_enabled", 0644, d_tracer,
4819                        &global_trace, &tracing_ctrl_fops);
4820
4821        trace_create_file("trace_options", 0644, d_tracer,
4822                        NULL, &tracing_iter_fops);
4823
4824        trace_create_file("tracing_cpumask", 0644, d_tracer,
4825                        NULL, &tracing_cpumask_fops);
4826
4827        trace_create_file("trace", 0644, d_tracer,
4828                        (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
4829
4830        trace_create_file("available_tracers", 0444, d_tracer,
4831                        &global_trace, &show_traces_fops);
4832
4833        trace_create_file("current_tracer", 0644, d_tracer,
4834                        &global_trace, &set_tracer_fops);
4835
4836#ifdef CONFIG_TRACER_MAX_TRACE
4837        trace_create_file("tracing_max_latency", 0644, d_tracer,
4838                        &tracing_max_latency, &tracing_max_lat_fops);
4839#endif
4840
4841        trace_create_file("tracing_thresh", 0644, d_tracer,
4842                        &tracing_thresh, &tracing_max_lat_fops);
4843
4844        trace_create_file("README", 0444, d_tracer,
4845                        NULL, &tracing_readme_fops);
4846
4847        trace_create_file("trace_pipe", 0444, d_tracer,
4848                        (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
4849
4850        trace_create_file("buffer_size_kb", 0644, d_tracer,
4851                        (void *) RING_BUFFER_ALL_CPUS, &tracing_entries_fops);
4852
4853        trace_create_file("buffer_total_size_kb", 0444, d_tracer,
4854                        &global_trace, &tracing_total_entries_fops);
4855
4856        trace_create_file("free_buffer", 0644, d_tracer,
4857                        &global_trace, &tracing_free_buffer_fops);
4858
4859        trace_create_file("trace_marker", 0220, d_tracer,
4860                        NULL, &tracing_mark_fops);
4861
4862        trace_create_file("saved_cmdlines", 0444, d_tracer,
4863                        NULL, &tracing_saved_cmdlines_fops);
4864
4865        trace_create_file("trace_clock", 0644, d_tracer, NULL,
4866                          &trace_clock_fops);
4867
4868        trace_create_file("tracing_on", 0644, d_tracer,
4869                            &global_trace, &rb_simple_fops);
4870
4871#ifdef CONFIG_DYNAMIC_FTRACE
4872        trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
4873                        &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
4874#endif
4875
4876        create_trace_options_dir();
4877
4878        for_each_tracing_cpu(cpu)
4879                tracing_init_debugfs_percpu(cpu);
4880
4881        return 0;
4882}
4883
4884static int trace_panic_handler(struct notifier_block *this,
4885                               unsigned long event, void *unused)
4886{
4887        if (ftrace_dump_on_oops)
4888                ftrace_dump(ftrace_dump_on_oops);
4889        return NOTIFY_OK;
4890}
4891
4892static struct notifier_block trace_panic_notifier = {
4893        .notifier_call  = trace_panic_handler,
4894        .next           = NULL,
4895        .priority       = 150   /* priority: INT_MAX >= x >= 0 */
4896};
4897
4898static int trace_die_handler(struct notifier_block *self,
4899                             unsigned long val,
4900                             void *data)
4901{
4902        switch (val) {
4903        case DIE_OOPS:
4904                if (ftrace_dump_on_oops)
4905                        ftrace_dump(ftrace_dump_on_oops);
4906                break;
4907        default:
4908                break;
4909        }
4910        return NOTIFY_OK;
4911}
4912
4913static struct notifier_block trace_die_notifier = {
4914        .notifier_call = trace_die_handler,
4915        .priority = 200
4916};
4917
4918/*
4919 * printk is set to max of 1024, we really don't need it that big.
4920 * Nothing should be printing 1000 characters anyway.
4921 */
4922#define TRACE_MAX_PRINT         1000
4923
4924/*
4925 * Define here KERN_TRACE so that we have one place to modify
4926 * it if we decide to change what log level the ftrace dump
4927 * should be at.
4928 */
4929#define KERN_TRACE              KERN_EMERG
4930
4931void
4932trace_printk_seq(struct trace_seq *s)
4933{
4934        /* Probably should print a warning here. */
4935        if (s->len >= 1000)
4936                s->len = 1000;
4937
4938        /* should be zero ended, but we are paranoid. */
4939        s->buffer[s->len] = 0;
4940
4941        printk(KERN_TRACE "%s", s->buffer);
4942
4943        trace_seq_init(s);
4944}
4945
4946void trace_init_global_iter(struct trace_iterator *iter)
4947{
4948        iter->tr = &global_trace;
4949        iter->trace = current_trace;
4950        iter->cpu_file = TRACE_PIPE_ALL_CPU;
4951}
4952
4953static void
4954__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
4955{
4956        static arch_spinlock_t ftrace_dump_lock =
4957                (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
4958        /* use static because iter can be a bit big for the stack */
4959        static struct trace_iterator iter;
4960        unsigned int old_userobj;
4961        static int dump_ran;
4962        unsigned long flags;
4963        int cnt = 0, cpu;
4964
4965        /* only one dump */
4966        local_irq_save(flags);
4967        arch_spin_lock(&ftrace_dump_lock);
4968        if (dump_ran)
4969                goto out;
4970
4971        dump_ran = 1;
4972
4973        tracing_off();
4974
4975        /* Did function tracer already get disabled? */
4976        if (ftrace_is_dead()) {
4977                printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
4978                printk("#          MAY BE MISSING FUNCTION EVENTS\n");
4979        }
4980
4981        if (disable_tracing)
4982                ftrace_kill();
4983
4984        trace_init_global_iter(&iter);
4985
4986        for_each_tracing_cpu(cpu) {
4987                atomic_inc(&iter.tr->data[cpu]->disabled);
4988        }
4989
4990        old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
4991
4992        /* don't look at user memory in panic mode */
4993        trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
4994
4995        /* Simulate the iterator */
4996        iter.tr = &global_trace;
4997        iter.trace = current_trace;
4998
4999        switch (oops_dump_mode) {
5000        case DUMP_ALL:
5001                iter.cpu_file = TRACE_PIPE_ALL_CPU;
5002                break;
5003        case DUMP_ORIG:
5004                iter.cpu_file = raw_smp_processor_id();
5005                break;
5006        case DUMP_NONE:
5007                goto out_enable;
5008        default:
5009                printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
5010                iter.cpu_file = TRACE_PIPE_ALL_CPU;
5011        }
5012
5013        printk(KERN_TRACE "Dumping ftrace buffer:\n");
5014
5015        /*
5016         * We need to stop all tracing on all CPUS to read the
5017         * the next buffer. This is a bit expensive, but is
5018         * not done often. We fill all what we can read,
5019         * and then release the locks again.
5020         */
5021
5022        while (!trace_empty(&iter)) {
5023
5024                if (!cnt)
5025                        printk(KERN_TRACE "---------------------------------\n");
5026
5027                cnt++;
5028
5029                /* reset all but tr, trace, and overruns */
5030                memset(&iter.seq, 0,
5031                       sizeof(struct trace_iterator) -
5032                       offsetof(struct trace_iterator, seq));
5033                iter.iter_flags |= TRACE_FILE_LAT_FMT;
5034                iter.pos = -1;
5035
5036                if (trace_find_next_entry_inc(&iter) != NULL) {
5037                        int ret;
5038
5039                        ret = print_trace_line(&iter);
5040                        if (ret != TRACE_TYPE_NO_CONSUME)
5041                                trace_consume(&iter);
5042                }
5043                touch_nmi_watchdog();
5044
5045                trace_printk_seq(&iter.seq);
5046        }
5047
5048        if (!cnt)
5049                printk(KERN_TRACE "   (ftrace buffer empty)\n");
5050        else
5051                printk(KERN_TRACE "---------------------------------\n");
5052
5053 out_enable:
5054        /* Re-enable tracing if requested */
5055        if (!disable_tracing) {
5056                trace_flags |= old_userobj;
5057
5058                for_each_tracing_cpu(cpu) {
5059                        atomic_dec(&iter.tr->data[cpu]->disabled);
5060                }
5061                tracing_on();
5062        }
5063
5064 out:
5065        arch_spin_unlock(&ftrace_dump_lock);
5066        local_irq_restore(flags);
5067}
5068
5069/* By default: disable tracing after the dump */
5070void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
5071{
5072        __ftrace_dump(true, oops_dump_mode);
5073}
5074EXPORT_SYMBOL_GPL(ftrace_dump);
5075
5076__init static int tracer_alloc_buffers(void)
5077{
5078        int ring_buf_size;
5079        enum ring_buffer_flags rb_flags;
5080        int i;
5081        int ret = -ENOMEM;
5082
5083
5084        if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
5085                goto out;
5086
5087        if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
5088                goto out_free_buffer_mask;
5089
5090        /* Only allocate trace_printk buffers if a trace_printk exists */
5091        if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
5092                trace_printk_init_buffers();
5093
5094        /* To save memory, keep the ring buffer size to its minimum */
5095        if (ring_buffer_expanded)
5096                ring_buf_size = trace_buf_size;
5097        else
5098                ring_buf_size = 1;
5099
5100        rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5101
5102        cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
5103        cpumask_copy(tracing_cpumask, cpu_all_mask);
5104
5105        /* TODO: make the number of buffers hot pluggable with CPUS */
5106        global_trace.buffer = ring_buffer_alloc(ring_buf_size, rb_flags);
5107        if (!global_trace.buffer) {
5108                printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
5109                WARN_ON(1);
5110                goto out_free_cpumask;
5111        }
5112        if (global_trace.buffer_disabled)
5113                tracing_off();
5114
5115
5116#ifdef CONFIG_TRACER_MAX_TRACE
5117        max_tr.buffer = ring_buffer_alloc(1, rb_flags);
5118        if (!max_tr.buffer) {
5119                printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
5120                WARN_ON(1);
5121                ring_buffer_free(global_trace.buffer);
5122                goto out_free_cpumask;
5123        }
5124#endif
5125
5126        /* Allocate the first page for all buffers */
5127        for_each_tracing_cpu(i) {
5128                global_trace.data[i] = &per_cpu(global_trace_cpu, i);
5129                max_tr.data[i] = &per_cpu(max_tr_data, i);
5130        }
5131
5132        set_buffer_entries(&global_trace,
5133                           ring_buffer_size(global_trace.buffer, 0));
5134#ifdef CONFIG_TRACER_MAX_TRACE
5135        set_buffer_entries(&max_tr, 1);
5136#endif
5137
5138        trace_init_cmdlines();
5139
5140        register_tracer(&nop_trace);
5141        current_trace = &nop_trace;
5142        /* All seems OK, enable tracing */
5143        tracing_disabled = 0;
5144
5145        atomic_notifier_chain_register(&panic_notifier_list,
5146                                       &trace_panic_notifier);
5147
5148        register_die_notifier(&trace_die_notifier);
5149
5150        return 0;
5151
5152out_free_cpumask:
5153        free_cpumask_var(tracing_cpumask);
5154out_free_buffer_mask:
5155        free_cpumask_var(tracing_buffer_mask);
5156out:
5157        return ret;
5158}
5159
5160__init static int clear_boot_tracer(void)
5161{
5162        /*
5163         * The default tracer at boot buffer is an init section.
5164         * This function is called in lateinit. If we did not
5165         * find the boot tracer, then clear it out, to prevent
5166         * later registration from accessing the buffer that is
5167         * about to be freed.
5168         */
5169        if (!default_bootup_tracer)
5170                return 0;
5171
5172        printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
5173               default_bootup_tracer);
5174        default_bootup_tracer = NULL;
5175
5176        return 0;
5177}
5178
5179early_initcall(tracer_alloc_buffers);
5180fs_initcall(tracer_init_debugfs);
5181late_initcall(clear_boot_tracer);
5182
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.