linux/kernel/trace/trace.c
<<
>>
Prefs
   1/*
   2 * ring buffer based function tracer
   3 *
   4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
   6 *
   7 * Originally taken from the RT patch by:
   8 *    Arnaldo Carvalho de Melo <acme@redhat.com>
   9 *
  10 * Based on code from the latency_tracer, that is:
  11 *  Copyright (C) 2004-2006 Ingo Molnar
  12 *  Copyright (C) 2004 William Lee Irwin III
  13 */
  14#include <linux/ring_buffer.h>
  15#include <generated/utsrelease.h>
  16#include <linux/stacktrace.h>
  17#include <linux/writeback.h>
  18#include <linux/kallsyms.h>
  19#include <linux/seq_file.h>
  20#include <linux/notifier.h>
  21#include <linux/irqflags.h>
  22#include <linux/debugfs.h>
  23#include <linux/pagemap.h>
  24#include <linux/hardirq.h>
  25#include <linux/linkage.h>
  26#include <linux/uaccess.h>
  27#include <linux/kprobes.h>
  28#include <linux/ftrace.h>
  29#include <linux/module.h>
  30#include <linux/percpu.h>
  31#include <linux/splice.h>
  32#include <linux/kdebug.h>
  33#include <linux/string.h>
  34#include <linux/rwsem.h>
  35#include <linux/slab.h>
  36#include <linux/ctype.h>
  37#include <linux/init.h>
  38#include <linux/poll.h>
  39#include <linux/nmi.h>
  40#include <linux/fs.h>
  41
  42#include "trace.h"
  43#include "trace_output.h"
  44
  45/*
  46 * On boot up, the ring buffer is set to the minimum size, so that
  47 * we do not waste memory on systems that are not using tracing.
  48 */
  49int ring_buffer_expanded;
  50
  51/*
  52 * We need to change this state when a selftest is running.
  53 * A selftest will lurk into the ring-buffer to count the
  54 * entries inserted during the selftest although some concurrent
  55 * insertions into the ring-buffer such as trace_printk could occurred
  56 * at the same time, giving false positive or negative results.
  57 */
  58static bool __read_mostly tracing_selftest_running;
  59
  60/*
  61 * If a tracer is running, we do not want to run SELFTEST.
  62 */
  63bool __read_mostly tracing_selftest_disabled;
  64
  65/* For tracers that don't implement custom flags */
  66static struct tracer_opt dummy_tracer_opt[] = {
  67        { }
  68};
  69
  70static struct tracer_flags dummy_tracer_flags = {
  71        .val = 0,
  72        .opts = dummy_tracer_opt
  73};
  74
  75static int dummy_set_flag(u32 old_flags, u32 bit, int set)
  76{
  77        return 0;
  78}
  79
  80/*
  81 * Kill all tracing for good (never come back).
  82 * It is initialized to 1 but will turn to zero if the initialization
  83 * of the tracer is successful. But that is the only place that sets
  84 * this back to zero.
  85 */
  86static int tracing_disabled = 1;
  87
  88DEFINE_PER_CPU(int, ftrace_cpu_disabled);
  89
  90cpumask_var_t __read_mostly     tracing_buffer_mask;
  91
  92/*
  93 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
  94 *
  95 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
  96 * is set, then ftrace_dump is called. This will output the contents
  97 * of the ftrace buffers to the console.  This is very useful for
  98 * capturing traces that lead to crashes and outputing it to a
  99 * serial console.
 100 *
 101 * It is default off, but you can enable it with either specifying
 102 * "ftrace_dump_on_oops" in the kernel command line, or setting
 103 * /proc/sys/kernel/ftrace_dump_on_oops
 104 * Set 1 if you want to dump buffers of all CPUs
 105 * Set 2 if you want to dump the buffer of the CPU that triggered oops
 106 */
 107
 108enum ftrace_dump_mode ftrace_dump_on_oops;
 109
 110static int tracing_set_tracer(const char *buf);
 111
 112#define MAX_TRACER_SIZE         100
 113static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
 114static char *default_bootup_tracer;
 115
 116static int __init set_cmdline_ftrace(char *str)
 117{
 118        strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
 119        default_bootup_tracer = bootup_tracer_buf;
 120        /* We are using ftrace early, expand it */
 121        ring_buffer_expanded = 1;
 122        return 1;
 123}
 124__setup("ftrace=", set_cmdline_ftrace);
 125
 126static int __init set_ftrace_dump_on_oops(char *str)
 127{
 128        if (*str++ != '=' || !*str) {
 129                ftrace_dump_on_oops = DUMP_ALL;
 130                return 1;
 131        }
 132
 133        if (!strcmp("orig_cpu", str)) {
 134                ftrace_dump_on_oops = DUMP_ORIG;
 135                return 1;
 136        }
 137
 138        return 0;
 139}
 140__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
 141
 142unsigned long long ns2usecs(cycle_t nsec)
 143{
 144        nsec += 500;
 145        do_div(nsec, 1000);
 146        return nsec;
 147}
 148
 149/*
 150 * The global_trace is the descriptor that holds the tracing
 151 * buffers for the live tracing. For each CPU, it contains
 152 * a link list of pages that will store trace entries. The
 153 * page descriptor of the pages in the memory is used to hold
 154 * the link list by linking the lru item in the page descriptor
 155 * to each of the pages in the buffer per CPU.
 156 *
 157 * For each active CPU there is a data field that holds the
 158 * pages for the buffer for that CPU. Each CPU has the same number
 159 * of pages allocated for its buffer.
 160 */
 161static struct trace_array       global_trace;
 162
 163static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
 164
 165int filter_current_check_discard(struct ring_buffer *buffer,
 166                                 struct ftrace_event_call *call, void *rec,
 167                                 struct ring_buffer_event *event)
 168{
 169        return filter_check_discard(call, rec, buffer, event);
 170}
 171EXPORT_SYMBOL_GPL(filter_current_check_discard);
 172
 173cycle_t ftrace_now(int cpu)
 174{
 175        u64 ts;
 176
 177        /* Early boot up does not have a buffer yet */
 178        if (!global_trace.buffer)
 179                return trace_clock_local();
 180
 181        ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
 182        ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
 183
 184        return ts;
 185}
 186
 187/*
 188 * The max_tr is used to snapshot the global_trace when a maximum
 189 * latency is reached. Some tracers will use this to store a maximum
 190 * trace while it continues examining live traces.
 191 *
 192 * The buffers for the max_tr are set up the same as the global_trace.
 193 * When a snapshot is taken, the link list of the max_tr is swapped
 194 * with the link list of the global_trace and the buffers are reset for
 195 * the global_trace so the tracing can continue.
 196 */
 197static struct trace_array       max_tr;
 198
 199static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
 200
 201/* tracer_enabled is used to toggle activation of a tracer */
 202static int                      tracer_enabled = 1;
 203
 204/**
 205 * tracing_is_enabled - return tracer_enabled status
 206 *
 207 * This function is used by other tracers to know the status
 208 * of the tracer_enabled flag.  Tracers may use this function
 209 * to know if it should enable their features when starting
 210 * up. See irqsoff tracer for an example (start_irqsoff_tracer).
 211 */
 212int tracing_is_enabled(void)
 213{
 214        return tracer_enabled;
 215}
 216
 217/*
 218 * trace_buf_size is the size in bytes that is allocated
 219 * for a buffer. Note, the number of bytes is always rounded
 220 * to page size.
 221 *
 222 * This number is purposely set to a low number of 16384.
 223 * If the dump on oops happens, it will be much appreciated
 224 * to not have to wait for all that output. Anyway this can be
 225 * boot time and run time configurable.
 226 */
 227#define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
 228
 229static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
 230
 231/* trace_types holds a link list of available tracers. */
 232static struct tracer            *trace_types __read_mostly;
 233
 234/* current_trace points to the tracer that is currently active */
 235static struct tracer            *current_trace __read_mostly;
 236
 237/*
 238 * trace_types_lock is used to protect the trace_types list.
 239 */
 240static DEFINE_MUTEX(trace_types_lock);
 241
 242/*
 243 * serialize the access of the ring buffer
 244 *
 245 * ring buffer serializes readers, but it is low level protection.
 246 * The validity of the events (which returns by ring_buffer_peek() ..etc)
 247 * are not protected by ring buffer.
 248 *
 249 * The content of events may become garbage if we allow other process consumes
 250 * these events concurrently:
 251 *   A) the page of the consumed events may become a normal page
 252 *      (not reader page) in ring buffer, and this page will be rewrited
 253 *      by events producer.
 254 *   B) The page of the consumed events may become a page for splice_read,
 255 *      and this page will be returned to system.
 256 *
 257 * These primitives allow multi process access to different cpu ring buffer
 258 * concurrently.
 259 *
 260 * These primitives don't distinguish read-only and read-consume access.
 261 * Multi read-only access are also serialized.
 262 */
 263
 264#ifdef CONFIG_SMP
 265static DECLARE_RWSEM(all_cpu_access_lock);
 266static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
 267
 268static inline void trace_access_lock(int cpu)
 269{
 270        if (cpu == TRACE_PIPE_ALL_CPU) {
 271                /* gain it for accessing the whole ring buffer. */
 272                down_write(&all_cpu_access_lock);
 273        } else {
 274                /* gain it for accessing a cpu ring buffer. */
 275
 276                /* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
 277                down_read(&all_cpu_access_lock);
 278
 279                /* Secondly block other access to this @cpu ring buffer. */
 280                mutex_lock(&per_cpu(cpu_access_lock, cpu));
 281        }
 282}
 283
 284static inline void trace_access_unlock(int cpu)
 285{
 286        if (cpu == TRACE_PIPE_ALL_CPU) {
 287                up_write(&all_cpu_access_lock);
 288        } else {
 289                mutex_unlock(&per_cpu(cpu_access_lock, cpu));
 290                up_read(&all_cpu_access_lock);
 291        }
 292}
 293
 294static inline void trace_access_lock_init(void)
 295{
 296        int cpu;
 297
 298        for_each_possible_cpu(cpu)
 299                mutex_init(&per_cpu(cpu_access_lock, cpu));
 300}
 301
 302#else
 303
 304static DEFINE_MUTEX(access_lock);
 305
 306static inline void trace_access_lock(int cpu)
 307{
 308        (void)cpu;
 309        mutex_lock(&access_lock);
 310}
 311
 312static inline void trace_access_unlock(int cpu)
 313{
 314        (void)cpu;
 315        mutex_unlock(&access_lock);
 316}
 317
 318static inline void trace_access_lock_init(void)
 319{
 320}
 321
 322#endif
 323
 324/* trace_wait is a waitqueue for tasks blocked on trace_poll */
 325static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
 326
 327/* trace_flags holds trace_options default values */
 328unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
 329        TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
 330        TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
 331        TRACE_ITER_IRQ_INFO;
 332
 333static int trace_stop_count;
 334static DEFINE_RAW_SPINLOCK(tracing_start_lock);
 335
 336static void wakeup_work_handler(struct work_struct *work)
 337{
 338        wake_up(&trace_wait);
 339}
 340
 341static DECLARE_DELAYED_WORK(wakeup_work, wakeup_work_handler);
 342
 343/**
 344 * tracing_on - enable tracing buffers
 345 *
 346 * This function enables tracing buffers that may have been
 347 * disabled with tracing_off.
 348 */
 349void tracing_on(void)
 350{
 351        if (global_trace.buffer)
 352                ring_buffer_record_on(global_trace.buffer);
 353        /*
 354         * This flag is only looked at when buffers haven't been
 355         * allocated yet. We don't really care about the race
 356         * between setting this flag and actually turning
 357         * on the buffer.
 358         */
 359        global_trace.buffer_disabled = 0;
 360}
 361EXPORT_SYMBOL_GPL(tracing_on);
 362
 363/**
 364 * tracing_off - turn off tracing buffers
 365 *
 366 * This function stops the tracing buffers from recording data.
 367 * It does not disable any overhead the tracers themselves may
 368 * be causing. This function simply causes all recording to
 369 * the ring buffers to fail.
 370 */
 371void tracing_off(void)
 372{
 373        if (global_trace.buffer)
 374                ring_buffer_record_off(global_trace.buffer);
 375        /*
 376         * This flag is only looked at when buffers haven't been
 377         * allocated yet. We don't really care about the race
 378         * between setting this flag and actually turning
 379         * on the buffer.
 380         */
 381        global_trace.buffer_disabled = 1;
 382}
 383EXPORT_SYMBOL_GPL(tracing_off);
 384
 385/**
 386 * tracing_is_on - show state of ring buffers enabled
 387 */
 388int tracing_is_on(void)
 389{
 390        if (global_trace.buffer)
 391                return ring_buffer_record_is_on(global_trace.buffer);
 392        return !global_trace.buffer_disabled;
 393}
 394EXPORT_SYMBOL_GPL(tracing_is_on);
 395
 396/**
 397 * trace_wake_up - wake up tasks waiting for trace input
 398 *
 399 * Schedules a delayed work to wake up any task that is blocked on the
 400 * trace_wait queue. These is used with trace_poll for tasks polling the
 401 * trace.
 402 */
 403void trace_wake_up(void)
 404{
 405        const unsigned long delay = msecs_to_jiffies(2);
 406
 407        if (trace_flags & TRACE_ITER_BLOCK)
 408                return;
 409        schedule_delayed_work(&wakeup_work, delay);
 410}
 411
 412static int __init set_buf_size(char *str)
 413{
 414        unsigned long buf_size;
 415
 416        if (!str)
 417                return 0;
 418        buf_size = memparse(str, &str);
 419        /* nr_entries can not be zero */
 420        if (buf_size == 0)
 421                return 0;
 422        trace_buf_size = buf_size;
 423        return 1;
 424}
 425__setup("trace_buf_size=", set_buf_size);
 426
 427static int __init set_tracing_thresh(char *str)
 428{
 429        unsigned long threshhold;
 430        int ret;
 431
 432        if (!str)
 433                return 0;
 434        ret = strict_strtoul(str, 0, &threshhold);
 435        if (ret < 0)
 436                return 0;
 437        tracing_thresh = threshhold * 1000;
 438        return 1;
 439}
 440__setup("tracing_thresh=", set_tracing_thresh);
 441
 442unsigned long nsecs_to_usecs(unsigned long nsecs)
 443{
 444        return nsecs / 1000;
 445}
 446
 447/* These must match the bit postions in trace_iterator_flags */
 448static const char *trace_options[] = {
 449        "print-parent",
 450        "sym-offset",
 451        "sym-addr",
 452        "verbose",
 453        "raw",
 454        "hex",
 455        "bin",
 456        "block",
 457        "stacktrace",
 458        "trace_printk",
 459        "ftrace_preempt",
 460        "branch",
 461        "annotate",
 462        "userstacktrace",
 463        "sym-userobj",
 464        "printk-msg-only",
 465        "context-info",
 466        "latency-format",
 467        "sleep-time",
 468        "graph-time",
 469        "record-cmd",
 470        "overwrite",
 471        "disable_on_free",
 472        "irq-info",
 473        NULL
 474};
 475
 476static struct {
 477        u64 (*func)(void);
 478        const char *name;
 479} trace_clocks[] = {
 480        { trace_clock_local,    "local" },
 481        { trace_clock_global,   "global" },
 482        { trace_clock_counter,  "counter" },
 483};
 484
 485int trace_clock_id;
 486
 487/*
 488 * trace_parser_get_init - gets the buffer for trace parser
 489 */
 490int trace_parser_get_init(struct trace_parser *parser, int size)
 491{
 492        memset(parser, 0, sizeof(*parser));
 493
 494        parser->buffer = kmalloc(size, GFP_KERNEL);
 495        if (!parser->buffer)
 496                return 1;
 497
 498        parser->size = size;
 499        return 0;
 500}
 501
 502/*
 503 * trace_parser_put - frees the buffer for trace parser
 504 */
 505void trace_parser_put(struct trace_parser *parser)
 506{
 507        kfree(parser->buffer);
 508}
 509
 510/*
 511 * trace_get_user - reads the user input string separated by  space
 512 * (matched by isspace(ch))
 513 *
 514 * For each string found the 'struct trace_parser' is updated,
 515 * and the function returns.
 516 *
 517 * Returns number of bytes read.
 518 *
 519 * See kernel/trace/trace.h for 'struct trace_parser' details.
 520 */
 521int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
 522        size_t cnt, loff_t *ppos)
 523{
 524        char ch;
 525        size_t read = 0;
 526        ssize_t ret;
 527
 528        if (!*ppos)
 529                trace_parser_clear(parser);
 530
 531        ret = get_user(ch, ubuf++);
 532        if (ret)
 533                goto out;
 534
 535        read++;
 536        cnt--;
 537
 538        /*
 539         * The parser is not finished with the last write,
 540         * continue reading the user input without skipping spaces.
 541         */
 542        if (!parser->cont) {
 543                /* skip white space */
 544                while (cnt && isspace(ch)) {
 545                        ret = get_user(ch, ubuf++);
 546                        if (ret)
 547                                goto out;
 548                        read++;
 549                        cnt--;
 550                }
 551
 552                /* only spaces were written */
 553                if (isspace(ch)) {
 554                        *ppos += read;
 555                        ret = read;
 556                        goto out;
 557                }
 558
 559                parser->idx = 0;
 560        }
 561
 562        /* read the non-space input */
 563        while (cnt && !isspace(ch)) {
 564                if (parser->idx < parser->size - 1)
 565                        parser->buffer[parser->idx++] = ch;
 566                else {
 567                        ret = -EINVAL;
 568                        goto out;
 569                }
 570                ret = get_user(ch, ubuf++);
 571                if (ret)
 572                        goto out;
 573                read++;
 574                cnt--;
 575        }
 576
 577        /* We either got finished input or we have to wait for another call. */
 578        if (isspace(ch)) {
 579                parser->buffer[parser->idx] = 0;
 580                parser->cont = false;
 581        } else {
 582                parser->cont = true;
 583                parser->buffer[parser->idx++] = ch;
 584        }
 585
 586        *ppos += read;
 587        ret = read;
 588
 589out:
 590        return ret;
 591}
 592
 593ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
 594{
 595        int len;
 596        int ret;
 597
 598        if (!cnt)
 599                return 0;
 600
 601        if (s->len <= s->readpos)
 602                return -EBUSY;
 603
 604        len = s->len - s->readpos;
 605        if (cnt > len)
 606                cnt = len;
 607        ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
 608        if (ret == cnt)
 609                return -EFAULT;
 610
 611        cnt -= ret;
 612
 613        s->readpos += cnt;
 614        return cnt;
 615}
 616
 617static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
 618{
 619        int len;
 620
 621        if (s->len <= s->readpos)
 622                return -EBUSY;
 623
 624        len = s->len - s->readpos;
 625        if (cnt > len)
 626                cnt = len;
 627        memcpy(buf, s->buffer + s->readpos, cnt);
 628
 629        s->readpos += cnt;
 630        return cnt;
 631}
 632
 633/*
 634 * ftrace_max_lock is used to protect the swapping of buffers
 635 * when taking a max snapshot. The buffers themselves are
 636 * protected by per_cpu spinlocks. But the action of the swap
 637 * needs its own lock.
 638 *
 639 * This is defined as a arch_spinlock_t in order to help
 640 * with performance when lockdep debugging is enabled.
 641 *
 642 * It is also used in other places outside the update_max_tr
 643 * so it needs to be defined outside of the
 644 * CONFIG_TRACER_MAX_TRACE.
 645 */
 646static arch_spinlock_t ftrace_max_lock =
 647        (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
 648
 649unsigned long __read_mostly     tracing_thresh;
 650
 651#ifdef CONFIG_TRACER_MAX_TRACE
 652unsigned long __read_mostly     tracing_max_latency;
 653
 654/*
 655 * Copy the new maximum trace into the separate maximum-trace
 656 * structure. (this way the maximum trace is permanently saved,
 657 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
 658 */
 659static void
 660__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
 661{
 662        struct trace_array_cpu *data = tr->data[cpu];
 663        struct trace_array_cpu *max_data;
 664
 665        max_tr.cpu = cpu;
 666        max_tr.time_start = data->preempt_timestamp;
 667
 668        max_data = max_tr.data[cpu];
 669        max_data->saved_latency = tracing_max_latency;
 670        max_data->critical_start = data->critical_start;
 671        max_data->critical_end = data->critical_end;
 672
 673        memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
 674        max_data->pid = tsk->pid;
 675        max_data->uid = task_uid(tsk);
 676        max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
 677        max_data->policy = tsk->policy;
 678        max_data->rt_priority = tsk->rt_priority;
 679
 680        /* record this tasks comm */
 681        tracing_record_cmdline(tsk);
 682}
 683
 684/**
 685 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
 686 * @tr: tracer
 687 * @tsk: the task with the latency
 688 * @cpu: The cpu that initiated the trace.
 689 *
 690 * Flip the buffers between the @tr and the max_tr and record information
 691 * about which task was the cause of this latency.
 692 */
 693void
 694update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
 695{
 696        struct ring_buffer *buf = tr->buffer;
 697
 698        if (trace_stop_count)
 699                return;
 700
 701        WARN_ON_ONCE(!irqs_disabled());
 702        if (!current_trace->use_max_tr) {
 703                WARN_ON_ONCE(1);
 704                return;
 705        }
 706        arch_spin_lock(&ftrace_max_lock);
 707
 708        tr->buffer = max_tr.buffer;
 709        max_tr.buffer = buf;
 710
 711        __update_max_tr(tr, tsk, cpu);
 712        arch_spin_unlock(&ftrace_max_lock);
 713}
 714
 715/**
 716 * update_max_tr_single - only copy one trace over, and reset the rest
 717 * @tr - tracer
 718 * @tsk - task with the latency
 719 * @cpu - the cpu of the buffer to copy.
 720 *
 721 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
 722 */
 723void
 724update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
 725{
 726        int ret;
 727
 728        if (trace_stop_count)
 729                return;
 730
 731        WARN_ON_ONCE(!irqs_disabled());
 732        if (!current_trace->use_max_tr) {
 733                WARN_ON_ONCE(1);
 734                return;
 735        }
 736
 737        arch_spin_lock(&ftrace_max_lock);
 738
 739        ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
 740
 741        if (ret == -EBUSY) {
 742                /*
 743                 * We failed to swap the buffer due to a commit taking
 744                 * place on this CPU. We fail to record, but we reset
 745                 * the max trace buffer (no one writes directly to it)
 746                 * and flag that it failed.
 747                 */
 748                trace_array_printk(&max_tr, _THIS_IP_,
 749                        "Failed to swap buffers due to commit in progress\n");
 750        }
 751
 752        WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
 753
 754        __update_max_tr(tr, tsk, cpu);
 755        arch_spin_unlock(&ftrace_max_lock);
 756}
 757#endif /* CONFIG_TRACER_MAX_TRACE */
 758
 759/**
 760 * register_tracer - register a tracer with the ftrace system.
 761 * @type - the plugin for the tracer
 762 *
 763 * Register a new plugin tracer.
 764 */
 765int register_tracer(struct tracer *type)
 766{
 767        struct tracer *t;
 768        int ret = 0;
 769
 770        if (!type->name) {
 771                pr_info("Tracer must have a name\n");
 772                return -1;
 773        }
 774
 775        if (strlen(type->name) >= MAX_TRACER_SIZE) {
 776                pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
 777                return -1;
 778        }
 779
 780        mutex_lock(&trace_types_lock);
 781
 782        tracing_selftest_running = true;
 783
 784        for (t = trace_types; t; t = t->next) {
 785                if (strcmp(type->name, t->name) == 0) {
 786                        /* already found */
 787                        pr_info("Tracer %s already registered\n",
 788                                type->name);
 789                        ret = -1;
 790                        goto out;
 791                }
 792        }
 793
 794        if (!type->set_flag)
 795                type->set_flag = &dummy_set_flag;
 796        if (!type->flags)
 797                type->flags = &dummy_tracer_flags;
 798        else
 799                if (!type->flags->opts)
 800                        type->flags->opts = dummy_tracer_opt;
 801        if (!type->wait_pipe)
 802                type->wait_pipe = default_wait_pipe;
 803
 804
 805#ifdef CONFIG_FTRACE_STARTUP_TEST
 806        if (type->selftest && !tracing_selftest_disabled) {
 807                struct tracer *saved_tracer = current_trace;
 808                struct trace_array *tr = &global_trace;
 809
 810                /*
 811                 * Run a selftest on this tracer.
 812                 * Here we reset the trace buffer, and set the current
 813                 * tracer to be this tracer. The tracer can then run some
 814                 * internal tracing to verify that everything is in order.
 815                 * If we fail, we do not register this tracer.
 816                 */
 817                tracing_reset_online_cpus(tr);
 818
 819                current_trace = type;
 820
 821                /* If we expanded the buffers, make sure the max is expanded too */
 822                if (ring_buffer_expanded && type->use_max_tr)
 823                        ring_buffer_resize(max_tr.buffer, trace_buf_size,
 824                                                RING_BUFFER_ALL_CPUS);
 825
 826                /* the test is responsible for initializing and enabling */
 827                pr_info("Testing tracer %s: ", type->name);
 828                ret = type->selftest(type, tr);
 829                /* the test is responsible for resetting too */
 830                current_trace = saved_tracer;
 831                if (ret) {
 832                        printk(KERN_CONT "FAILED!\n");
 833                        /* Add the warning after printing 'FAILED' */
 834                        WARN_ON(1);
 835                        goto out;
 836                }
 837                /* Only reset on passing, to avoid touching corrupted buffers */
 838                tracing_reset_online_cpus(tr);
 839
 840                /* Shrink the max buffer again */
 841                if (ring_buffer_expanded && type->use_max_tr)
 842                        ring_buffer_resize(max_tr.buffer, 1,
 843                                                RING_BUFFER_ALL_CPUS);
 844
 845                printk(KERN_CONT "PASSED\n");
 846        }
 847#endif
 848
 849        type->next = trace_types;
 850        trace_types = type;
 851
 852 out:
 853        tracing_selftest_running = false;
 854        mutex_unlock(&trace_types_lock);
 855
 856        if (ret || !default_bootup_tracer)
 857                goto out_unlock;
 858
 859        if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
 860                goto out_unlock;
 861
 862        printk(KERN_INFO "Starting tracer '%s'\n", type->name);
 863        /* Do we want this tracer to start on bootup? */
 864        tracing_set_tracer(type->name);
 865        default_bootup_tracer = NULL;
 866        /* disable other selftests, since this will break it. */
 867        tracing_selftest_disabled = 1;
 868#ifdef CONFIG_FTRACE_STARTUP_TEST
 869        printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
 870               type->name);
 871#endif
 872
 873 out_unlock:
 874        return ret;
 875}
 876
 877void unregister_tracer(struct tracer *type)
 878{
 879        struct tracer **t;
 880
 881        mutex_lock(&trace_types_lock);
 882        for (t = &trace_types; *t; t = &(*t)->next) {
 883                if (*t == type)
 884                        goto found;
 885        }
 886        pr_info("Tracer %s not registered\n", type->name);
 887        goto out;
 888
 889 found:
 890        *t = (*t)->next;
 891
 892        if (type == current_trace && tracer_enabled) {
 893                tracer_enabled = 0;
 894                tracing_stop();
 895                if (current_trace->stop)
 896                        current_trace->stop(&global_trace);
 897                current_trace = &nop_trace;
 898        }
 899out:
 900        mutex_unlock(&trace_types_lock);
 901}
 902
 903void tracing_reset(struct trace_array *tr, int cpu)
 904{
 905        struct ring_buffer *buffer = tr->buffer;
 906
 907        ring_buffer_record_disable(buffer);
 908
 909        /* Make sure all commits have finished */
 910        synchronize_sched();
 911        ring_buffer_reset_cpu(buffer, cpu);
 912
 913        ring_buffer_record_enable(buffer);
 914}
 915
 916void tracing_reset_online_cpus(struct trace_array *tr)
 917{
 918        struct ring_buffer *buffer = tr->buffer;
 919        int cpu;
 920
 921        ring_buffer_record_disable(buffer);
 922
 923        /* Make sure all commits have finished */
 924        synchronize_sched();
 925
 926        tr->time_start = ftrace_now(tr->cpu);
 927
 928        for_each_online_cpu(cpu)
 929                ring_buffer_reset_cpu(buffer, cpu);
 930
 931        ring_buffer_record_enable(buffer);
 932}
 933
 934void tracing_reset_current(int cpu)
 935{
 936        tracing_reset(&global_trace, cpu);
 937}
 938
 939void tracing_reset_current_online_cpus(void)
 940{
 941        tracing_reset_online_cpus(&global_trace);
 942}
 943
 944#define SAVED_CMDLINES 128
 945#define NO_CMDLINE_MAP UINT_MAX
 946static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
 947static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
 948static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
 949static int cmdline_idx;
 950static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
 951
 952/* temporary disable recording */
 953static atomic_t trace_record_cmdline_disabled __read_mostly;
 954
 955static void trace_init_cmdlines(void)
 956{
 957        memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
 958        memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
 959        cmdline_idx = 0;
 960}
 961
 962int is_tracing_stopped(void)
 963{
 964        return trace_stop_count;
 965}
 966
 967/**
 968 * ftrace_off_permanent - disable all ftrace code permanently
 969 *
 970 * This should only be called when a serious anomally has
 971 * been detected.  This will turn off the function tracing,
 972 * ring buffers, and other tracing utilites. It takes no
 973 * locks and can be called from any context.
 974 */
 975void ftrace_off_permanent(void)
 976{
 977        tracing_disabled = 1;
 978        ftrace_stop();
 979        tracing_off_permanent();
 980}
 981
 982/**
 983 * tracing_start - quick start of the tracer
 984 *
 985 * If tracing is enabled but was stopped by tracing_stop,
 986 * this will start the tracer back up.
 987 */
 988void tracing_start(void)
 989{
 990        struct ring_buffer *buffer;
 991        unsigned long flags;
 992
 993        if (tracing_disabled)
 994                return;
 995
 996        raw_spin_lock_irqsave(&tracing_start_lock, flags);
 997        if (--trace_stop_count) {
 998                if (trace_stop_count < 0) {
 999                        /* Someone screwed up their debugging */
1000                        WARN_ON_ONCE(1);
1001                        trace_stop_count = 0;
1002                }
1003                goto out;
1004        }
1005
1006        /* Prevent the buffers from switching */
1007        arch_spin_lock(&ftrace_max_lock);
1008
1009        buffer = global_trace.buffer;
1010        if (buffer)
1011                ring_buffer_record_enable(buffer);
1012
1013        buffer = max_tr.buffer;
1014        if (buffer)
1015                ring_buffer_record_enable(buffer);
1016
1017        arch_spin_unlock(&ftrace_max_lock);
1018
1019        ftrace_start();
1020 out:
1021        raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1022}
1023
1024/**
1025 * tracing_stop - quick stop of the tracer
1026 *
1027 * Light weight way to stop tracing. Use in conjunction with
1028 * tracing_start.
1029 */
1030void tracing_stop(void)
1031{
1032        struct ring_buffer *buffer;
1033        unsigned long flags;
1034
1035        ftrace_stop();
1036        raw_spin_lock_irqsave(&tracing_start_lock, flags);
1037        if (trace_stop_count++)
1038                goto out;
1039
1040        /* Prevent the buffers from switching */
1041        arch_spin_lock(&ftrace_max_lock);
1042
1043        buffer = global_trace.buffer;
1044        if (buffer)
1045                ring_buffer_record_disable(buffer);
1046
1047        buffer = max_tr.buffer;
1048        if (buffer)
1049                ring_buffer_record_disable(buffer);
1050
1051        arch_spin_unlock(&ftrace_max_lock);
1052
1053 out:
1054        raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1055}
1056
1057void trace_stop_cmdline_recording(void);
1058
1059static void trace_save_cmdline(struct task_struct *tsk)
1060{
1061        unsigned pid, idx;
1062
1063        if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1064                return;
1065
1066        /*
1067         * It's not the end of the world if we don't get
1068         * the lock, but we also don't want to spin
1069         * nor do we want to disable interrupts,
1070         * so if we miss here, then better luck next time.
1071         */
1072        if (!arch_spin_trylock(&trace_cmdline_lock))
1073                return;
1074
1075        idx = map_pid_to_cmdline[tsk->pid];
1076        if (idx == NO_CMDLINE_MAP) {
1077                idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1078
1079                /*
1080                 * Check whether the cmdline buffer at idx has a pid
1081                 * mapped. We are going to overwrite that entry so we
1082                 * need to clear the map_pid_to_cmdline. Otherwise we
1083                 * would read the new comm for the old pid.
1084                 */
1085                pid = map_cmdline_to_pid[idx];
1086                if (pid != NO_CMDLINE_MAP)
1087                        map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1088
1089                map_cmdline_to_pid[idx] = tsk->pid;
1090                map_pid_to_cmdline[tsk->pid] = idx;
1091
1092                cmdline_idx = idx;
1093        }
1094
1095        memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1096
1097        arch_spin_unlock(&trace_cmdline_lock);
1098}
1099
1100void trace_find_cmdline(int pid, char comm[])
1101{
1102        unsigned map;
1103
1104        if (!pid) {
1105                strcpy(comm, "<idle>");
1106                return;
1107        }
1108
1109        if (WARN_ON_ONCE(pid < 0)) {
1110                strcpy(comm, "<XXX>");
1111                return;
1112        }
1113
1114        if (pid > PID_MAX_DEFAULT) {
1115                strcpy(comm, "<...>");
1116                return;
1117        }
1118
1119        preempt_disable();
1120        arch_spin_lock(&trace_cmdline_lock);
1121        map = map_pid_to_cmdline[pid];
1122        if (map != NO_CMDLINE_MAP)
1123                strcpy(comm, saved_cmdlines[map]);
1124        else
1125                strcpy(comm, "<...>");
1126
1127        arch_spin_unlock(&trace_cmdline_lock);
1128        preempt_enable();
1129}
1130
1131void tracing_record_cmdline(struct task_struct *tsk)
1132{
1133        if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
1134            !tracing_is_on())
1135                return;
1136
1137        trace_save_cmdline(tsk);
1138}
1139
1140void
1141tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1142                             int pc)
1143{
1144        struct task_struct *tsk = current;
1145
1146        entry->preempt_count            = pc & 0xff;
1147        entry->pid                      = (tsk) ? tsk->pid : 0;
1148        entry->padding                  = 0;
1149        entry->flags =
1150#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1151                (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1152#else
1153                TRACE_FLAG_IRQS_NOSUPPORT |
1154#endif
1155                ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1156                ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1157                (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1158}
1159EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1160
1161struct ring_buffer_event *
1162trace_buffer_lock_reserve(struct ring_buffer *buffer,
1163                          int type,
1164                          unsigned long len,
1165                          unsigned long flags, int pc)
1166{
1167        struct ring_buffer_event *event;
1168
1169        event = ring_buffer_lock_reserve(buffer, len);
1170        if (event != NULL) {
1171                struct trace_entry *ent = ring_buffer_event_data(event);
1172
1173                tracing_generic_entry_update(ent, flags, pc);
1174                ent->type = type;
1175        }
1176
1177        return event;
1178}
1179
1180static inline void
1181__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1182                             struct ring_buffer_event *event,
1183                             unsigned long flags, int pc,
1184                             int wake)
1185{
1186        ring_buffer_unlock_commit(buffer, event);
1187
1188        ftrace_trace_stack(buffer, flags, 6, pc);
1189        ftrace_trace_userstack(buffer, flags, pc);
1190
1191        if (wake)
1192                trace_wake_up();
1193}
1194
1195void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1196                                struct ring_buffer_event *event,
1197                                unsigned long flags, int pc)
1198{
1199        __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1200}
1201
1202struct ring_buffer_event *
1203trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1204                                  int type, unsigned long len,
1205                                  unsigned long flags, int pc)
1206{
1207        *current_rb = global_trace.buffer;
1208        return trace_buffer_lock_reserve(*current_rb,
1209                                         type, len, flags, pc);
1210}
1211EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1212
1213void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1214                                        struct ring_buffer_event *event,
1215                                        unsigned long flags, int pc)
1216{
1217        __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1218}
1219EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1220
1221void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
1222                                       struct ring_buffer_event *event,
1223                                       unsigned long flags, int pc)
1224{
1225        __trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
1226}
1227EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
1228
1229void trace_nowake_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1230                                            struct ring_buffer_event *event,
1231                                            unsigned long flags, int pc,
1232                                            struct pt_regs *regs)
1233{
1234        ring_buffer_unlock_commit(buffer, event);
1235
1236        ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1237        ftrace_trace_userstack(buffer, flags, pc);
1238}
1239EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit_regs);
1240
1241void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1242                                         struct ring_buffer_event *event)
1243{
1244        ring_buffer_discard_commit(buffer, event);
1245}
1246EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1247
1248void
1249trace_function(struct trace_array *tr,
1250               unsigned long ip, unsigned long parent_ip, unsigned long flags,
1251               int pc)
1252{
1253        struct ftrace_event_call *call = &event_function;
1254        struct ring_buffer *buffer = tr->buffer;
1255        struct ring_buffer_event *event;
1256        struct ftrace_entry *entry;
1257
1258        /* If we are reading the ring buffer, don't trace */
1259        if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1260                return;
1261
1262        event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1263                                          flags, pc);
1264        if (!event)
1265                return;
1266        entry   = ring_buffer_event_data(event);
1267        entry->ip                       = ip;
1268        entry->parent_ip                = parent_ip;
1269
1270        if (!filter_check_discard(call, entry, buffer, event))
1271                ring_buffer_unlock_commit(buffer, event);
1272}
1273
1274void
1275ftrace(struct trace_array *tr, struct trace_array_cpu *data,
1276       unsigned long ip, unsigned long parent_ip, unsigned long flags,
1277       int pc)
1278{
1279        if (likely(!atomic_read(&data->disabled)))
1280                trace_function(tr, ip, parent_ip, flags, pc);
1281}
1282
1283#ifdef CONFIG_STACKTRACE
1284
1285#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1286struct ftrace_stack {
1287        unsigned long           calls[FTRACE_STACK_MAX_ENTRIES];
1288};
1289
1290static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1291static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1292
1293static void __ftrace_trace_stack(struct ring_buffer *buffer,
1294                                 unsigned long flags,
1295                                 int skip, int pc, struct pt_regs *regs)
1296{
1297        struct ftrace_event_call *call = &event_kernel_stack;
1298        struct ring_buffer_event *event;
1299        struct stack_entry *entry;
1300        struct stack_trace trace;
1301        int use_stack;
1302        int size = FTRACE_STACK_ENTRIES;
1303
1304        trace.nr_entries        = 0;
1305        trace.skip              = skip;
1306
1307        /*
1308         * Since events can happen in NMIs there's no safe way to
1309         * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1310         * or NMI comes in, it will just have to use the default
1311         * FTRACE_STACK_SIZE.
1312         */
1313        preempt_disable_notrace();
1314
1315        use_stack = ++__get_cpu_var(ftrace_stack_reserve);
1316        /*
1317         * We don't need any atomic variables, just a barrier.
1318         * If an interrupt comes in, we don't care, because it would
1319         * have exited and put the counter back to what we want.
1320         * We just need a barrier to keep gcc from moving things
1321         * around.
1322         */
1323        barrier();
1324        if (use_stack == 1) {
1325                trace.entries           = &__get_cpu_var(ftrace_stack).calls[0];
1326                trace.max_entries       = FTRACE_STACK_MAX_ENTRIES;
1327
1328                if (regs)
1329                        save_stack_trace_regs(regs, &trace);
1330                else
1331                        save_stack_trace(&trace);
1332
1333                if (trace.nr_entries > size)
1334                        size = trace.nr_entries;
1335        } else
1336                /* From now on, use_stack is a boolean */
1337                use_stack = 0;
1338
1339        size *= sizeof(unsigned long);
1340
1341        event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1342                                          sizeof(*entry) + size, flags, pc);
1343        if (!event)
1344                goto out;
1345        entry = ring_buffer_event_data(event);
1346
1347        memset(&entry->caller, 0, size);
1348
1349        if (use_stack)
1350                memcpy(&entry->caller, trace.entries,
1351                       trace.nr_entries * sizeof(unsigned long));
1352        else {
1353                trace.max_entries       = FTRACE_STACK_ENTRIES;
1354                trace.entries           = entry->caller;
1355                if (regs)
1356                        save_stack_trace_regs(regs, &trace);
1357                else
1358                        save_stack_trace(&trace);
1359        }
1360
1361        entry->size = trace.nr_entries;
1362
1363        if (!filter_check_discard(call, entry, buffer, event))
1364                ring_buffer_unlock_commit(buffer, event);
1365
1366 out:
1367        /* Again, don't let gcc optimize things here */
1368        barrier();
1369        __get_cpu_var(ftrace_stack_reserve)--;
1370        preempt_enable_notrace();
1371
1372}
1373
1374void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1375                             int skip, int pc, struct pt_regs *regs)
1376{
1377        if (!(trace_flags & TRACE_ITER_STACKTRACE))
1378                return;
1379
1380        __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1381}
1382
1383void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1384                        int skip, int pc)
1385{
1386        if (!(trace_flags & TRACE_ITER_STACKTRACE))
1387                return;
1388
1389        __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1390}
1391
1392void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1393                   int pc)
1394{
1395        __ftrace_trace_stack(tr->buffer, flags, skip, pc, NULL);
1396}
1397
1398/**
1399 * trace_dump_stack - record a stack back trace in the trace buffer
1400 */
1401void trace_dump_stack(void)
1402{
1403        unsigned long flags;
1404
1405        if (tracing_disabled || tracing_selftest_running)
1406                return;
1407
1408        local_save_flags(flags);
1409
1410        /* skipping 3 traces, seems to get us at the caller of this function */
1411        __ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count(), NULL);
1412}
1413
1414static DEFINE_PER_CPU(int, user_stack_count);
1415
1416void
1417ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1418{
1419        struct ftrace_event_call *call = &event_user_stack;
1420        struct ring_buffer_event *event;
1421        struct userstack_entry *entry;
1422        struct stack_trace trace;
1423
1424        if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1425                return;
1426
1427        /*
1428         * NMIs can not handle page faults, even with fix ups.
1429         * The save user stack can (and often does) fault.
1430         */
1431        if (unlikely(in_nmi()))
1432                return;
1433
1434        /*
1435         * prevent recursion, since the user stack tracing may
1436         * trigger other kernel events.
1437         */
1438        preempt_disable();
1439        if (__this_cpu_read(user_stack_count))
1440                goto out;
1441
1442        __this_cpu_inc(user_stack_count);
1443
1444        event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1445                                          sizeof(*entry), flags, pc);
1446        if (!event)
1447                goto out_drop_count;
1448        entry   = ring_buffer_event_data(event);
1449
1450        entry->tgid             = current->tgid;
1451        memset(&entry->caller, 0, sizeof(entry->caller));
1452
1453        trace.nr_entries        = 0;
1454        trace.max_entries       = FTRACE_STACK_ENTRIES;
1455        trace.skip              = 0;
1456        trace.entries           = entry->caller;
1457
1458        save_stack_trace_user(&trace);
1459        if (!filter_check_discard(call, entry, buffer, event))
1460                ring_buffer_unlock_commit(buffer, event);
1461
1462 out_drop_count:
1463        __this_cpu_dec(user_stack_count);
1464 out:
1465        preempt_enable();
1466}
1467
1468#ifdef UNUSED
1469static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1470{
1471        ftrace_trace_userstack(tr, flags, preempt_count());
1472}
1473#endif /* UNUSED */
1474
1475#endif /* CONFIG_STACKTRACE */
1476
1477/* created for use with alloc_percpu */
1478struct trace_buffer_struct {
1479        char buffer[TRACE_BUF_SIZE];
1480};
1481
1482static struct trace_buffer_struct *trace_percpu_buffer;
1483static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1484static struct trace_buffer_struct *trace_percpu_irq_buffer;
1485static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1486
1487/*
1488 * The buffer used is dependent on the context. There is a per cpu
1489 * buffer for normal context, softirq contex, hard irq context and
1490 * for NMI context. Thise allows for lockless recording.
1491 *
1492 * Note, if the buffers failed to be allocated, then this returns NULL
1493 */
1494static char *get_trace_buf(void)
1495{
1496        struct trace_buffer_struct *percpu_buffer;
1497        struct trace_buffer_struct *buffer;
1498
1499        /*
1500         * If we have allocated per cpu buffers, then we do not
1501         * need to do any locking.
1502         */
1503        if (in_nmi())
1504                percpu_buffer = trace_percpu_nmi_buffer;
1505        else if (in_irq())
1506                percpu_buffer = trace_percpu_irq_buffer;
1507        else if (in_softirq())
1508                percpu_buffer = trace_percpu_sirq_buffer;
1509        else
1510                percpu_buffer = trace_percpu_buffer;
1511
1512        if (!percpu_buffer)
1513                return NULL;
1514
1515        buffer = per_cpu_ptr(percpu_buffer, smp_processor_id());
1516
1517        return buffer->buffer;
1518}
1519
1520static int alloc_percpu_trace_buffer(void)
1521{
1522        struct trace_buffer_struct *buffers;
1523        struct trace_buffer_struct *sirq_buffers;
1524        struct trace_buffer_struct *irq_buffers;
1525        struct trace_buffer_struct *nmi_buffers;
1526
1527        buffers = alloc_percpu(struct trace_buffer_struct);
1528        if (!buffers)
1529                goto err_warn;
1530
1531        sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1532        if (!sirq_buffers)
1533                goto err_sirq;
1534
1535        irq_buffers = alloc_percpu(struct trace_buffer_struct);
1536        if (!irq_buffers)
1537                goto err_irq;
1538
1539        nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1540        if (!nmi_buffers)
1541                goto err_nmi;
1542
1543        trace_percpu_buffer = buffers;
1544        trace_percpu_sirq_buffer = sirq_buffers;
1545        trace_percpu_irq_buffer = irq_buffers;
1546        trace_percpu_nmi_buffer = nmi_buffers;
1547
1548        return 0;
1549
1550 err_nmi:
1551        free_percpu(irq_buffers);
1552 err_irq:
1553        free_percpu(sirq_buffers);
1554 err_sirq:
1555        free_percpu(buffers);
1556 err_warn:
1557        WARN(1, "Could not allocate percpu trace_printk buffer");
1558        return -ENOMEM;
1559}
1560
1561void trace_printk_init_buffers(void)
1562{
1563        static int buffers_allocated;
1564
1565        if (buffers_allocated)
1566                return;
1567
1568        if (alloc_percpu_trace_buffer())
1569                return;
1570
1571        pr_info("ftrace: Allocated trace_printk buffers\n");
1572
1573        buffers_allocated = 1;
1574}
1575
1576/**
1577 * trace_vbprintk - write binary msg to tracing buffer
1578 *
1579 */
1580int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1581{
1582        struct ftrace_event_call *call = &event_bprint;
1583        struct ring_buffer_event *event;
1584        struct ring_buffer *buffer;
1585        struct trace_array *tr = &global_trace;
1586        struct bprint_entry *entry;
1587        unsigned long flags;
1588        char *tbuffer;
1589        int len = 0, size, pc;
1590
1591        if (unlikely(tracing_selftest_running || tracing_disabled))
1592                return 0;
1593
1594        /* Don't pollute graph traces with trace_vprintk internals */
1595        pause_graph_tracing();
1596
1597        pc = preempt_count();
1598        preempt_disable_notrace();
1599
1600        tbuffer = get_trace_buf();
1601        if (!tbuffer) {
1602                len = 0;
1603                goto out;
1604        }
1605
1606        len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1607
1608        if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
1609                goto out;
1610
1611        local_save_flags(flags);
1612        size = sizeof(*entry) + sizeof(u32) * len;
1613        buffer = tr->buffer;
1614        event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1615                                          flags, pc);
1616        if (!event)
1617                goto out;
1618        entry = ring_buffer_event_data(event);
1619        entry->ip                       = ip;
1620        entry->fmt                      = fmt;
1621
1622        memcpy(entry->buf, tbuffer, sizeof(u32) * len);
1623        if (!filter_check_discard(call, entry, buffer, event)) {
1624                ring_buffer_unlock_commit(buffer, event);
1625                ftrace_trace_stack(buffer, flags, 6, pc);
1626        }
1627
1628out:
1629        preempt_enable_notrace();
1630        unpause_graph_tracing();
1631
1632        return len;
1633}
1634EXPORT_SYMBOL_GPL(trace_vbprintk);
1635
1636int trace_array_printk(struct trace_array *tr,
1637                       unsigned long ip, const char *fmt, ...)
1638{
1639        int ret;
1640        va_list ap;
1641
1642        if (!(trace_flags & TRACE_ITER_PRINTK))
1643                return 0;
1644
1645        va_start(ap, fmt);
1646        ret = trace_array_vprintk(tr, ip, fmt, ap);
1647        va_end(ap);
1648        return ret;
1649}
1650
1651int trace_array_vprintk(struct trace_array *tr,
1652                        unsigned long ip, const char *fmt, va_list args)
1653{
1654        struct ftrace_event_call *call = &event_print;
1655        struct ring_buffer_event *event;
1656        struct ring_buffer *buffer;
1657        int len = 0, size, pc;
1658        struct print_entry *entry;
1659        unsigned long flags;
1660        char *tbuffer;
1661
1662        if (tracing_disabled || tracing_selftest_running)
1663                return 0;
1664
1665        /* Don't pollute graph traces with trace_vprintk internals */
1666        pause_graph_tracing();
1667
1668        pc = preempt_count();
1669        preempt_disable_notrace();
1670
1671
1672        tbuffer = get_trace_buf();
1673        if (!tbuffer) {
1674                len = 0;
1675                goto out;
1676        }
1677
1678        len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
1679        if (len > TRACE_BUF_SIZE)
1680                goto out;
1681
1682        local_save_flags(flags);
1683        size = sizeof(*entry) + len + 1;
1684        buffer = tr->buffer;
1685        event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1686                                          flags, pc);
1687        if (!event)
1688                goto out;
1689        entry = ring_buffer_event_data(event);
1690        entry->ip = ip;
1691
1692        memcpy(&entry->buf, tbuffer, len);
1693        entry->buf[len] = '\0';
1694        if (!filter_check_discard(call, entry, buffer, event)) {
1695                ring_buffer_unlock_commit(buffer, event);
1696                ftrace_trace_stack(buffer, flags, 6, pc);
1697        }
1698 out:
1699        preempt_enable_notrace();
1700        unpause_graph_tracing();
1701
1702        return len;
1703}
1704
1705int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1706{
1707        return trace_array_vprintk(&global_trace, ip, fmt, args);
1708}
1709EXPORT_SYMBOL_GPL(trace_vprintk);
1710
1711static void trace_iterator_increment(struct trace_iterator *iter)
1712{
1713        struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
1714
1715        iter->idx++;
1716        if (buf_iter)
1717                ring_buffer_read(buf_iter, NULL);
1718}
1719
1720static struct trace_entry *
1721peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
1722                unsigned long *lost_events)
1723{
1724        struct ring_buffer_event *event;
1725        struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
1726
1727        if (buf_iter)
1728                event = ring_buffer_iter_peek(buf_iter, ts);
1729        else
1730                event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
1731                                         lost_events);
1732
1733        if (event) {
1734                iter->ent_size = ring_buffer_event_length(event);
1735                return ring_buffer_event_data(event);
1736        }
1737        iter->ent_size = 0;
1738        return NULL;
1739}
1740
1741static struct trace_entry *
1742__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
1743                  unsigned long *missing_events, u64 *ent_ts)
1744{
1745        struct ring_buffer *buffer = iter->tr->buffer;
1746        struct trace_entry *ent, *next = NULL;
1747        unsigned long lost_events = 0, next_lost = 0;
1748        int cpu_file = iter->cpu_file;
1749        u64 next_ts = 0, ts;
1750        int next_cpu = -1;
1751        int next_size = 0;
1752        int cpu;
1753
1754        /*
1755         * If we are in a per_cpu trace file, don't bother by iterating over
1756         * all cpu and peek directly.
1757         */
1758        if (cpu_file > TRACE_PIPE_ALL_CPU) {
1759                if (ring_buffer_empty_cpu(buffer, cpu_file))
1760                        return NULL;
1761                ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
1762                if (ent_cpu)
1763                        *ent_cpu = cpu_file;
1764
1765                return ent;
1766        }
1767
1768        for_each_tracing_cpu(cpu) {
1769
1770                if (ring_buffer_empty_cpu(buffer, cpu))
1771                        continue;
1772
1773                ent = peek_next_entry(iter, cpu, &ts, &lost_events);
1774
1775                /*
1776                 * Pick the entry with the smallest timestamp:
1777                 */
1778                if (ent && (!next || ts < next_ts)) {
1779                        next = ent;
1780                        next_cpu = cpu;
1781                        next_ts = ts;
1782                        next_lost = lost_events;
1783                        next_size = iter->ent_size;
1784                }
1785        }
1786
1787        iter->ent_size = next_size;
1788
1789        if (ent_cpu)
1790                *ent_cpu = next_cpu;
1791
1792        if (ent_ts)
1793                *ent_ts = next_ts;
1794
1795        if (missing_events)
1796                *missing_events = next_lost;
1797
1798        return next;
1799}
1800
1801/* Find the next real entry, without updating the iterator itself */
1802struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1803                                          int *ent_cpu, u64 *ent_ts)
1804{
1805        return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
1806}
1807
1808/* Find the next real entry, and increment the iterator to the next entry */
1809void *trace_find_next_entry_inc(struct trace_iterator *iter)
1810{
1811        iter->ent = __find_next_entry(iter, &iter->cpu,
1812                                      &iter->lost_events, &iter->ts);
1813
1814        if (iter->ent)
1815                trace_iterator_increment(iter);
1816
1817        return iter->ent ? iter : NULL;
1818}
1819
1820static void trace_consume(struct trace_iterator *iter)
1821{
1822        ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
1823                            &iter->lost_events);
1824}
1825
1826static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1827{
1828        struct trace_iterator *iter = m->private;
1829        int i = (int)*pos;
1830        void *ent;
1831
1832        WARN_ON_ONCE(iter->leftover);
1833
1834        (*pos)++;
1835
1836        /* can't go backwards */
1837        if (iter->idx > i)
1838                return NULL;
1839
1840        if (iter->idx < 0)
1841                ent = trace_find_next_entry_inc(iter);
1842        else
1843                ent = iter;
1844
1845        while (ent && iter->idx < i)
1846                ent = trace_find_next_entry_inc(iter);
1847
1848        iter->pos = *pos;
1849
1850        return ent;
1851}
1852
1853void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1854{
1855        struct trace_array *tr = iter->tr;
1856        struct ring_buffer_event *event;
1857        struct ring_buffer_iter *buf_iter;
1858        unsigned long entries = 0;
1859        u64 ts;
1860
1861        tr->data[cpu]->skipped_entries = 0;
1862
1863        buf_iter = trace_buffer_iter(iter, cpu);
1864        if (!buf_iter)
1865                return;
1866
1867        ring_buffer_iter_reset(buf_iter);
1868
1869        /*
1870         * We could have the case with the max latency tracers
1871         * that a reset never took place on a cpu. This is evident
1872         * by the timestamp being before the start of the buffer.
1873         */
1874        while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1875                if (ts >= iter->tr->time_start)
1876                        break;
1877                entries++;
1878                ring_buffer_read(buf_iter, NULL);
1879        }
1880
1881        tr->data[cpu]->skipped_entries = entries;
1882}
1883
1884/*
1885 * The current tracer is copied to avoid a global locking
1886 * all around.
1887 */
1888static void *s_start(struct seq_file *m, loff_t *pos)
1889{
1890        struct trace_iterator *iter = m->private;
1891        static struct tracer *old_tracer;
1892        int cpu_file = iter->cpu_file;
1893        void *p = NULL;
1894        loff_t l = 0;
1895        int cpu;
1896
1897        /* copy the tracer to avoid using a global lock all around */
1898        mutex_lock(&trace_types_lock);
1899        if (unlikely(old_tracer != current_trace && current_trace)) {
1900                old_tracer = current_trace;
1901                *iter->trace = *current_trace;
1902        }
1903        mutex_unlock(&trace_types_lock);
1904
1905        atomic_inc(&trace_record_cmdline_disabled);
1906
1907        if (*pos != iter->pos) {
1908                iter->ent = NULL;
1909                iter->cpu = 0;
1910                iter->idx = -1;
1911
1912                if (cpu_file == TRACE_PIPE_ALL_CPU) {
1913                        for_each_tracing_cpu(cpu)
1914                                tracing_iter_reset(iter, cpu);
1915                } else
1916                        tracing_iter_reset(iter, cpu_file);
1917
1918                iter->leftover = 0;
1919                for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1920                        ;
1921
1922        } else {
1923                /*
1924                 * If we overflowed the seq_file before, then we want
1925                 * to just reuse the trace_seq buffer again.
1926                 */
1927                if (iter->leftover)
1928                        p = iter;
1929                else {
1930                        l = *pos - 1;
1931                        p = s_next(m, p, &l);
1932                }
1933        }
1934
1935        trace_event_read_lock();
1936        trace_access_lock(cpu_file);
1937        return p;
1938}
1939
1940static void s_stop(struct seq_file *m, void *p)
1941{
1942        struct trace_iterator *iter = m->private;
1943
1944        atomic_dec(&trace_record_cmdline_disabled);
1945        trace_access_unlock(iter->cpu_file);
1946        trace_event_read_unlock();
1947}
1948
1949static void
1950get_total_entries(struct trace_array *tr, unsigned long *total, unsigned long *entries)
1951{
1952        unsigned long count;
1953        int cpu;
1954
1955        *total = 0;
1956        *entries = 0;
1957
1958        for_each_tracing_cpu(cpu) {
1959                count = ring_buffer_entries_cpu(tr->buffer, cpu);
1960                /*
1961                 * If this buffer has skipped entries, then we hold all
1962                 * entries for the trace and we need to ignore the
1963                 * ones before the time stamp.
1964                 */
1965                if (tr->data[cpu]->skipped_entries) {
1966                        count -= tr->data[cpu]->skipped_entries;
1967                        /* total is the same as the entries */
1968                        *total += count;
1969                } else
1970                        *total += count +
1971                                ring_buffer_overrun_cpu(tr->buffer, cpu);
1972                *entries += count;
1973        }
1974}
1975
1976static void print_lat_help_header(struct seq_file *m)
1977{
1978        seq_puts(m, "#                  _------=> CPU#            \n");
1979        seq_puts(m, "#                 / _-----=> irqs-off        \n");
1980        seq_puts(m, "#                | / _----=> need-resched    \n");
1981        seq_puts(m, "#                || / _---=> hardirq/softirq \n");
1982        seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
1983        seq_puts(m, "#                |||| /     delay             \n");
1984        seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
1985        seq_puts(m, "#     \\   /      |||||  \\    |   /           \n");
1986}
1987
1988static void print_event_info(struct trace_array *tr, struct seq_file *m)
1989{
1990        unsigned long total;
1991        unsigned long entries;
1992
1993        get_total_entries(tr, &total, &entries);
1994        seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu   #P:%d\n",
1995                   entries, total, num_online_cpus());
1996        seq_puts(m, "#\n");
1997}
1998
1999static void print_func_help_header(struct trace_array *tr, struct seq_file *m)
2000{
2001        print_event_info(tr, m);
2002        seq_puts(m, "#           TASK-PID   CPU#      TIMESTAMP  FUNCTION\n");
2003        seq_puts(m, "#              | |       |          |         |\n");
2004}
2005
2006static void print_func_help_header_irq(struct trace_array *tr, struct seq_file *m)
2007{
2008        print_event_info(tr, m);
2009        seq_puts(m, "#                              _-----=> irqs-off\n");
2010        seq_puts(m, "#                             / _----=> need-resched\n");
2011        seq_puts(m, "#                            | / _---=> hardirq/softirq\n");
2012        seq_puts(m, "#                            || / _--=> preempt-depth\n");
2013        seq_puts(m, "#                            ||| /     delay\n");
2014        seq_puts(m, "#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION\n");
2015        seq_puts(m, "#              | |       |   ||||       |         |\n");
2016}
2017
2018void
2019print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2020{
2021        unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2022        struct trace_array *tr = iter->tr;
2023        struct trace_array_cpu *data = tr->data[tr->cpu];
2024        struct tracer *type = current_trace;
2025        unsigned long entries;
2026        unsigned long total;
2027        const char *name = "preemption";
2028
2029        if (type)
2030                name = type->name;
2031
2032        get_total_entries(tr, &total, &entries);
2033
2034        seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2035                   name, UTS_RELEASE);
2036        seq_puts(m, "# -----------------------------------"
2037                 "---------------------------------\n");
2038        seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2039                   " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2040                   nsecs_to_usecs(data->saved_latency),
2041                   entries,
2042                   total,
2043                   tr->cpu,
2044#if defined(CONFIG_PREEMPT_NONE)
2045                   "server",
2046#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2047                   "desktop",
2048#elif defined(CONFIG_PREEMPT)
2049                   "preempt",
2050#else
2051                   "unknown",
2052#endif
2053                   /* These are reserved for later use */
2054                   0, 0, 0, 0);
2055#ifdef CONFIG_SMP
2056        seq_printf(m, " #P:%d)\n", num_online_cpus());
2057#else
2058        seq_puts(m, ")\n");
2059#endif
2060        seq_puts(m, "#    -----------------\n");
2061        seq_printf(m, "#    | task: %.16s-%d "
2062                   "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2063                   data->comm, data->pid, data->uid, data->nice,
2064                   data->policy, data->rt_priority);
2065        seq_puts(m, "#    -----------------\n");
2066
2067        if (data->critical_start) {
2068                seq_puts(m, "#  => started at: ");
2069                seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2070                trace_print_seq(m, &iter->seq);
2071                seq_puts(m, "\n#  => ended at:   ");
2072                seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2073                trace_print_seq(m, &iter->seq);
2074                seq_puts(m, "\n#\n");
2075        }
2076
2077        seq_puts(m, "#\n");
2078}
2079
2080static void test_cpu_buff_start(struct trace_iterator *iter)
2081{
2082        struct trace_seq *s = &iter->seq;
2083
2084        if (!(trace_flags & TRACE_ITER_ANNOTATE))
2085                return;
2086
2087        if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2088                return;
2089
2090        if (cpumask_test_cpu(iter->cpu, iter->started))
2091                return;
2092
2093        if (iter->tr->data[iter->cpu]->skipped_entries)
2094                return;
2095
2096        cpumask_set_cpu(iter->cpu, iter->started);
2097
2098        /* Don't print started cpu buffer for the first entry of the trace */
2099        if (iter->idx > 1)
2100                trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2101                                iter->cpu);
2102}
2103
2104static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2105{
2106        struct trace_seq *s = &iter->seq;
2107        unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2108        struct trace_entry *entry;
2109        struct trace_event *event;
2110
2111        entry = iter->ent;
2112
2113        test_cpu_buff_start(iter);
2114
2115        event = ftrace_find_event(entry->type);
2116
2117        if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2118                if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2119                        if (!trace_print_lat_context(iter))
2120                                goto partial;
2121                } else {
2122                        if (!trace_print_context(iter))
2123                                goto partial;
2124                }
2125        }
2126
2127        if (event)
2128                return event->funcs->trace(iter, sym_flags, event);
2129
2130        if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2131                goto partial;
2132
2133        return TRACE_TYPE_HANDLED;
2134partial:
2135        return TRACE_TYPE_PARTIAL_LINE;
2136}
2137
2138static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2139{
2140        struct trace_seq *s = &iter->seq;
2141        struct trace_entry *entry;
2142        struct trace_event *event;
2143
2144        entry = iter->ent;
2145
2146        if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2147                if (!trace_seq_printf(s, "%d %d %llu ",
2148                                      entry->pid, iter->cpu, iter->ts))
2149                        goto partial;
2150        }
2151
2152        event = ftrace_find_event(entry->type);
2153        if (event)
2154                return event->funcs->raw(iter, 0, event);
2155
2156        if (!trace_seq_printf(s, "%d ?\n", entry->type))
2157                goto partial;
2158
2159        return TRACE_TYPE_HANDLED;
2160partial:
2161        return TRACE_TYPE_PARTIAL_LINE;
2162}
2163
2164static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2165{
2166        struct trace_seq *s = &iter->seq;
2167        unsigned char newline = '\n';
2168        struct trace_entry *entry;
2169        struct trace_event *event;
2170
2171        entry = iter->ent;
2172
2173        if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2174                SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2175                SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2176                SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2177        }
2178
2179        event = ftrace_find_event(entry->type);
2180        if (event) {
2181                enum print_line_t ret = event->funcs->hex(iter, 0, event);
2182                if (ret != TRACE_TYPE_HANDLED)
2183                        return ret;
2184        }
2185
2186        SEQ_PUT_FIELD_RET(s, newline);
2187
2188        return TRACE_TYPE_HANDLED;
2189}
2190
2191static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2192{
2193        struct trace_seq *s = &iter->seq;
2194        struct trace_entry *entry;
2195        struct trace_event *event;
2196
2197        entry = iter->ent;
2198
2199        if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2200                SEQ_PUT_FIELD_RET(s, entry->pid);
2201                SEQ_PUT_FIELD_RET(s, iter->cpu);
2202                SEQ_PUT_FIELD_RET(s, iter->ts);
2203        }
2204
2205        event = ftrace_find_event(entry->type);
2206        return event ? event->funcs->binary(iter, 0, event) :
2207                TRACE_TYPE_HANDLED;
2208}
2209
2210int trace_empty(struct trace_iterator *iter)
2211{
2212        struct ring_buffer_iter *buf_iter;
2213        int cpu;
2214
2215        /* If we are looking at one CPU buffer, only check that one */
2216        if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
2217                cpu = iter->cpu_file;
2218                buf_iter = trace_buffer_iter(iter, cpu);
2219                if (buf_iter) {
2220                        if (!ring_buffer_iter_empty(buf_iter))
2221                                return 0;
2222                } else {
2223                        if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2224                                return 0;
2225                }
2226                return 1;
2227        }
2228
2229        for_each_tracing_cpu(cpu) {
2230                buf_iter = trace_buffer_iter(iter, cpu);
2231                if (buf_iter) {
2232                        if (!ring_buffer_iter_empty(buf_iter))
2233                                return 0;
2234                } else {
2235                        if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2236                                return 0;
2237                }
2238        }
2239
2240        return 1;
2241}
2242
2243/*  Called with trace_event_read_lock() held. */
2244enum print_line_t print_trace_line(struct trace_iterator *iter)
2245{
2246        enum print_line_t ret;
2247
2248        if (iter->lost_events &&
2249            !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2250                                 iter->cpu, iter->lost_events))
2251                return TRACE_TYPE_PARTIAL_LINE;
2252
2253        if (iter->trace && iter->trace->print_line) {
2254                ret = iter->trace->print_line(iter);
2255                if (ret != TRACE_TYPE_UNHANDLED)
2256                        return ret;
2257        }
2258
2259        if (iter->ent->type == TRACE_BPRINT &&
2260                        trace_flags & TRACE_ITER_PRINTK &&
2261                        trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2262                return trace_print_bprintk_msg_only(iter);
2263
2264        if (iter->ent->type == TRACE_PRINT &&
2265                        trace_flags & TRACE_ITER_PRINTK &&
2266                        trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2267                return trace_print_printk_msg_only(iter);
2268
2269        if (trace_flags & TRACE_ITER_BIN)
2270                return print_bin_fmt(iter);
2271
2272        if (trace_flags & TRACE_ITER_HEX)
2273                return print_hex_fmt(iter);
2274
2275        if (trace_flags & TRACE_ITER_RAW)
2276                return print_raw_fmt(iter);
2277
2278        return print_trace_fmt(iter);
2279}
2280
2281void trace_latency_header(struct seq_file *m)
2282{
2283        struct trace_iterator *iter = m->private;
2284
2285        /* print nothing if the buffers are empty */
2286        if (trace_empty(iter))
2287                return;
2288
2289        if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2290                print_trace_header(m, iter);
2291
2292        if (!(trace_flags & TRACE_ITER_VERBOSE))
2293                print_lat_help_header(m);
2294}
2295
2296void trace_default_header(struct seq_file *m)
2297{
2298        struct trace_iterator *iter = m->private;
2299
2300        if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2301                return;
2302
2303        if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2304                /* print nothing if the buffers are empty */
2305                if (trace_empty(iter))
2306                        return;
2307                print_trace_header(m, iter);
2308                if (!(trace_flags & TRACE_ITER_VERBOSE))
2309                        print_lat_help_header(m);
2310        } else {
2311                if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2312                        if (trace_flags & TRACE_ITER_IRQ_INFO)
2313                                print_func_help_header_irq(iter->tr, m);
2314                        else
2315                                print_func_help_header(iter->tr, m);
2316                }
2317        }
2318}
2319
2320static void test_ftrace_alive(struct seq_file *m)
2321{
2322        if (!ftrace_is_dead())
2323                return;
2324        seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2325        seq_printf(m, "#          MAY BE MISSING FUNCTION EVENTS\n");
2326}
2327
2328static int s_show(struct seq_file *m, void *v)
2329{
2330        struct trace_iterator *iter = v;
2331        int ret;
2332
2333        if (iter->ent == NULL) {
2334                if (iter->tr) {
2335                        seq_printf(m, "# tracer: %s\n", iter->trace->name);
2336                        seq_puts(m, "#\n");
2337                        test_ftrace_alive(m);
2338                }
2339                if (iter->trace && iter->trace->print_header)
2340                        iter->trace->print_header(m);
2341                else
2342                        trace_default_header(m);
2343
2344        } else if (iter->leftover) {
2345                /*
2346                 * If we filled the seq_file buffer earlier, we
2347                 * want to just show it now.
2348                 */
2349                ret = trace_print_seq(m, &iter->seq);
2350
2351                /* ret should this time be zero, but you never know */
2352                iter->leftover = ret;
2353
2354        } else {
2355                print_trace_line(iter);
2356                ret = trace_print_seq(m, &iter->seq);
2357                /*
2358                 * If we overflow the seq_file buffer, then it will
2359                 * ask us for this data again at start up.
2360                 * Use that instead.
2361                 *  ret is 0 if seq_file write succeeded.
2362                 *        -1 otherwise.
2363                 */
2364                iter->leftover = ret;
2365        }
2366
2367        return 0;
2368}
2369
2370static const struct seq_operations tracer_seq_ops = {
2371        .start          = s_start,
2372        .next           = s_next,
2373        .stop           = s_stop,
2374        .show           = s_show,
2375};
2376
2377static struct trace_iterator *
2378__tracing_open(struct inode *inode, struct file *file)
2379{
2380        long cpu_file = (long) inode->i_private;
2381        struct trace_iterator *iter;
2382        int cpu;
2383
2384        if (tracing_disabled)
2385                return ERR_PTR(-ENODEV);
2386
2387        iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2388        if (!iter)
2389                return ERR_PTR(-ENOMEM);
2390
2391        iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2392                                    GFP_KERNEL);
2393        if (!iter->buffer_iter)
2394                goto release;
2395
2396        /*
2397         * We make a copy of the current tracer to avoid concurrent
2398         * changes on it while we are reading.
2399         */
2400        mutex_lock(&trace_types_lock);
2401        iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2402        if (!iter->trace)
2403                goto fail;
2404
2405        if (current_trace)
2406                *iter->trace = *current_trace;
2407
2408        if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2409                goto fail;
2410
2411        if (current_trace && current_trace->print_max)
2412                iter->tr = &max_tr;
2413        else
2414                iter->tr = &global_trace;
2415        iter->pos = -1;
2416        mutex_init(&iter->mutex);
2417        iter->cpu_file = cpu_file;
2418
2419        /* Notify the tracer early; before we stop tracing. */
2420        if (iter->trace && iter->trace->open)
2421                iter->trace->open(iter);
2422
2423        /* Annotate start of buffers if we had overruns */
2424        if (ring_buffer_overruns(iter->tr->buffer))
2425                iter->iter_flags |= TRACE_FILE_ANNOTATE;
2426
2427        /* stop the trace while dumping */
2428        tracing_stop();
2429
2430        if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2431                for_each_tracing_cpu(cpu) {
2432                        iter->buffer_iter[cpu] =
2433                                ring_buffer_read_prepare(iter->tr->buffer, cpu);
2434                }
2435                ring_buffer_read_prepare_sync();
2436                for_each_tracing_cpu(cpu) {
2437                        ring_buffer_read_start(iter->buffer_iter[cpu]);
2438                        tracing_iter_reset(iter, cpu);
2439                }
2440        } else {
2441                cpu = iter->cpu_file;
2442                iter->buffer_iter[cpu] =
2443                        ring_buffer_read_prepare(iter->tr->buffer, cpu);
2444                ring_buffer_read_prepare_sync();
2445                ring_buffer_read_start(iter->buffer_iter[cpu]);
2446                tracing_iter_reset(iter, cpu);
2447        }
2448
2449        mutex_unlock(&trace_types_lock);
2450
2451        return iter;
2452
2453 fail:
2454        mutex_unlock(&trace_types_lock);
2455        kfree(iter->trace);
2456        kfree(iter->buffer_iter);
2457release:
2458        seq_release_private(inode, file);
2459        return ERR_PTR(-ENOMEM);
2460}
2461
2462int tracing_open_generic(struct inode *inode, struct file *filp)
2463{
2464        if (tracing_disabled)
2465                return -ENODEV;
2466
2467        filp->private_data = inode->i_private;
2468        return 0;
2469}
2470
2471static int tracing_release(struct inode *inode, struct file *file)
2472{
2473        struct seq_file *m = file->private_data;
2474        struct trace_iterator *iter;
2475        int cpu;
2476
2477        if (!(file->f_mode & FMODE_READ))
2478                return 0;
2479
2480        iter = m->private;
2481
2482        mutex_lock(&trace_types_lock);
2483        for_each_tracing_cpu(cpu) {
2484                if (iter->buffer_iter[cpu])
2485                        ring_buffer_read_finish(iter->buffer_iter[cpu]);
2486        }
2487
2488        if (iter->trace && iter->trace->close)
2489                iter->trace->close(iter);
2490
2491        /* reenable tracing if it was previously enabled */
2492        tracing_start();
2493        mutex_unlock(&trace_types_lock);
2494
2495        mutex_destroy(&iter->mutex);
2496        free_cpumask_var(iter->started);
2497        kfree(iter->trace);
2498        kfree(iter->buffer_iter);
2499        seq_release_private(inode, file);
2500        return 0;
2501}
2502
2503static int tracing_open(struct inode *inode, struct file *file)
2504{
2505        struct trace_iterator *iter;
2506        int ret = 0;
2507
2508        /* If this file was open for write, then erase contents */
2509        if ((file->f_mode & FMODE_WRITE) &&
2510            (file->f_flags & O_TRUNC)) {
2511                long cpu = (long) inode->i_private;
2512
2513                if (cpu == TRACE_PIPE_ALL_CPU)
2514                        tracing_reset_online_cpus(&global_trace);
2515                else
2516                        tracing_reset(&global_trace, cpu);
2517        }
2518
2519        if (file->f_mode & FMODE_READ) {
2520                iter = __tracing_open(inode, file);
2521                if (IS_ERR(iter))
2522                        ret = PTR_ERR(iter);
2523                else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2524                        iter->iter_flags |= TRACE_FILE_LAT_FMT;
2525        }
2526        return ret;
2527}
2528
2529static void *
2530t_next(struct seq_file *m, void *v, loff_t *pos)
2531{
2532        struct tracer *t = v;
2533
2534        (*pos)++;
2535
2536        if (t)
2537                t = t->next;
2538
2539        return t;
2540}
2541
2542static void *t_start(struct seq_file *m, loff_t *pos)
2543{
2544        struct tracer *t;
2545        loff_t l = 0;
2546
2547        mutex_lock(&trace_types_lock);
2548        for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2549                ;
2550
2551        return t;
2552}
2553
2554static void t_stop(struct seq_file *m, void *p)
2555{
2556        mutex_unlock(&trace_types_lock);
2557}
2558
2559static int t_show(struct seq_file *m, void *v)
2560{
2561        struct tracer *t = v;
2562
2563        if (!t)
2564                return 0;
2565
2566        seq_printf(m, "%s", t->name);
2567        if (t->next)
2568                seq_putc(m, ' ');
2569        else
2570                seq_putc(m, '\n');
2571
2572        return 0;
2573}
2574
2575static const struct seq_operations show_traces_seq_ops = {
2576        .start          = t_start,
2577        .next           = t_next,
2578        .stop           = t_stop,
2579        .show           = t_show,
2580};
2581
2582static int show_traces_open(struct inode *inode, struct file *file)
2583{
2584        if (tracing_disabled)
2585                return -ENODEV;
2586
2587        return seq_open(file, &show_traces_seq_ops);
2588}
2589
2590static ssize_t
2591tracing_write_stub(struct file *filp, const char __user *ubuf,
2592                   size_t count, loff_t *ppos)
2593{
2594        return count;
2595}
2596
2597static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
2598{
2599        if (file->f_mode & FMODE_READ)
2600                return seq_lseek(file, offset, origin);
2601        else
2602                return 0;
2603}
2604
2605static const struct file_operations tracing_fops = {
2606        .open           = tracing_open,
2607        .read           = seq_read,
2608        .write          = tracing_write_stub,
2609        .llseek         = tracing_seek,
2610        .release        = tracing_release,
2611};
2612
2613static const struct file_operations show_traces_fops = {
2614        .open           = show_traces_open,
2615        .read           = seq_read,
2616        .release        = seq_release,
2617        .llseek         = seq_lseek,
2618};
2619
2620/*
2621 * Only trace on a CPU if the bitmask is set:
2622 */
2623static cpumask_var_t tracing_cpumask;
2624
2625/*
2626 * The tracer itself will not take this lock, but still we want
2627 * to provide a consistent cpumask to user-space:
2628 */
2629static DEFINE_MUTEX(tracing_cpumask_update_lock);
2630
2631/*
2632 * Temporary storage for the character representation of the
2633 * CPU bitmask (and one more byte for the newline):
2634 */
2635static char mask_str[NR_CPUS + 1];
2636
2637static ssize_t
2638tracing_cpumask_read(struct file *filp, char __user *ubuf,
2639                     size_t count, loff_t *ppos)
2640{
2641        int len;
2642
2643        mutex_lock(&tracing_cpumask_update_lock);
2644
2645        len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2646        if (count - len < 2) {
2647                count = -EINVAL;
2648                goto out_err;
2649        }
2650        len += sprintf(mask_str + len, "\n");
2651        count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2652
2653out_err:
2654        mutex_unlock(&tracing_cpumask_update_lock);
2655
2656        return count;
2657}
2658
2659static ssize_t
2660tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2661                      size_t count, loff_t *ppos)
2662{
2663        int err, cpu;
2664        cpumask_var_t tracing_cpumask_new;
2665
2666        if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2667                return -ENOMEM;
2668
2669        err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2670        if (err)
2671                goto err_unlock;
2672
2673        mutex_lock(&tracing_cpumask_update_lock);
2674
2675        local_irq_disable();
2676        arch_spin_lock(&ftrace_max_lock);
2677        for_each_tracing_cpu(cpu) {
2678                /*
2679                 * Increase/decrease the disabled counter if we are
2680                 * about to flip a bit in the cpumask:
2681                 */
2682                if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2683                                !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2684                        atomic_inc(&global_trace.data[cpu]->disabled);
2685                        ring_buffer_record_disable_cpu(global_trace.buffer, cpu);
2686                }
2687                if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2688                                cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2689                        atomic_dec(&global_trace.data[cpu]->disabled);
2690                        ring_buffer_record_enable_cpu(global_trace.buffer, cpu);
2691                }
2692        }
2693        arch_spin_unlock(&ftrace_max_lock);
2694        local_irq_enable();
2695
2696        cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2697
2698        mutex_unlock(&tracing_cpumask_update_lock);
2699        free_cpumask_var(tracing_cpumask_new);
2700
2701        return count;
2702
2703err_unlock:
2704        free_cpumask_var(tracing_cpumask_new);
2705
2706        return err;
2707}
2708
2709static const struct file_operations tracing_cpumask_fops = {
2710        .open           = tracing_open_generic,
2711        .read           = tracing_cpumask_read,
2712        .write          = tracing_cpumask_write,
2713        .llseek         = generic_file_llseek,
2714};
2715
2716static int tracing_trace_options_show(struct seq_file *m, void *v)
2717{
2718        struct tracer_opt *trace_opts;
2719        u32 tracer_flags;
2720        int i;
2721
2722        mutex_lock(&trace_types_lock);
2723        tracer_flags = current_trace->flags->val;
2724        trace_opts = current_trace->flags->opts;
2725
2726        for (i = 0; trace_options[i]; i++) {
2727                if (trace_flags & (1 << i))
2728                        seq_printf(m, "%s\n", trace_options[i]);
2729                else
2730                        seq_printf(m, "no%s\n", trace_options[i]);
2731        }
2732
2733        for (i = 0; trace_opts[i].name; i++) {
2734                if (tracer_flags & trace_opts[i].bit)
2735                        seq_printf(m, "%s\n", trace_opts[i].name);
2736                else
2737                        seq_printf(m, "no%s\n", trace_opts[i].name);
2738        }
2739        mutex_unlock(&trace_types_lock);
2740
2741        return 0;
2742}
2743
2744static int __set_tracer_option(struct tracer *trace,
2745                               struct tracer_flags *tracer_flags,
2746                               struct tracer_opt *opts, int neg)
2747{
2748        int ret;
2749
2750        ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
2751        if (ret)
2752                return ret;
2753
2754        if (neg)
2755                tracer_flags->val &= ~opts->bit;
2756        else
2757                tracer_flags->val |= opts->bit;
2758        return 0;
2759}
2760
2761/* Try to assign a tracer specific option */
2762static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2763{
2764        struct tracer_flags *tracer_flags = trace->flags;
2765        struct tracer_opt *opts = NULL;
2766        int i;
2767
2768        for (i = 0; tracer_flags->opts[i].name; i++) {
2769                opts = &tracer_flags->opts[i];
2770
2771                if (strcmp(cmp, opts->name) == 0)
2772                        return __set_tracer_option(trace, trace->flags,
2773                                                   opts, neg);
2774        }
2775
2776        return -EINVAL;
2777}
2778
2779static void set_tracer_flags(unsigned int mask, int enabled)
2780{
2781        /* do nothing if flag is already set */
2782        if (!!(trace_flags & mask) == !!enabled)
2783                return;
2784
2785        if (enabled)
2786                trace_flags |= mask;
2787        else
2788                trace_flags &= ~mask;
2789
2790        if (mask == TRACE_ITER_RECORD_CMD)
2791                trace_event_enable_cmd_record(enabled);
2792
2793        if (mask == TRACE_ITER_OVERWRITE)
2794                ring_buffer_change_overwrite(global_trace.buffer, enabled);
2795}
2796
2797static ssize_t
2798tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2799                        size_t cnt, loff_t *ppos)
2800{
2801        char buf[64];
2802        char *cmp;
2803        int neg = 0;
2804        int ret;
2805        int i;
2806
2807        if (cnt >= sizeof(buf))
2808                return -EINVAL;
2809
2810        if (copy_from_user(&buf, ubuf, cnt))
2811                return -EFAULT;
2812
2813        buf[cnt] = 0;
2814        cmp = strstrip(buf);
2815
2816        if (strncmp(cmp, "no", 2) == 0) {
2817                neg = 1;
2818                cmp += 2;
2819        }
2820
2821        for (i = 0; trace_options[i]; i++) {
2822                if (strcmp(cmp, trace_options[i]) == 0) {
2823                        set_tracer_flags(1 << i, !neg);
2824                        break;
2825                }
2826        }
2827
2828        /* If no option could be set, test the specific tracer options */
2829        if (!trace_options[i]) {
2830                mutex_lock(&trace_types_lock);
2831                ret = set_tracer_option(current_trace, cmp, neg);
2832                mutex_unlock(&trace_types_lock);
2833                if (ret)
2834                        return ret;
2835        }
2836
2837        *ppos += cnt;
2838
2839        return cnt;
2840}
2841
2842static int tracing_trace_options_open(struct inode *inode, struct file *file)
2843{
2844        if (tracing_disabled)
2845                return -ENODEV;
2846        return single_open(file, tracing_trace_options_show, NULL);
2847}
2848
2849static const struct file_operations tracing_iter_fops = {
2850        .open           = tracing_trace_options_open,
2851        .read           = seq_read,
2852        .llseek         = seq_lseek,
2853        .release        = single_release,
2854        .write          = tracing_trace_options_write,
2855};
2856
2857static const char readme_msg[] =
2858        "tracing mini-HOWTO:\n\n"
2859        "# mount -t debugfs nodev /sys/kernel/debug\n\n"
2860        "# cat /sys/kernel/debug/tracing/available_tracers\n"
2861        "wakeup wakeup_rt preemptirqsoff preemptoff irqsoff function nop\n\n"
2862        "# cat /sys/kernel/debug/tracing/current_tracer\n"
2863        "nop\n"
2864        "# echo wakeup > /sys/kernel/debug/tracing/current_tracer\n"
2865        "# cat /sys/kernel/debug/tracing/current_tracer\n"
2866        "wakeup\n"
2867        "# cat /sys/kernel/debug/tracing/trace_options\n"
2868        "noprint-parent nosym-offset nosym-addr noverbose\n"
2869        "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
2870        "# echo 1 > /sys/kernel/debug/tracing/tracing_on\n"
2871        "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
2872        "# echo 0 > /sys/kernel/debug/tracing/tracing_on\n"
2873;
2874
2875static ssize_t
2876tracing_readme_read(struct file *filp, char __user *ubuf,
2877                       size_t cnt, loff_t *ppos)
2878{
2879        return simple_read_from_buffer(ubuf, cnt, ppos,
2880                                        readme_msg, strlen(readme_msg));
2881}
2882
2883static const struct file_operations tracing_readme_fops = {
2884        .open           = tracing_open_generic,
2885        .read           = tracing_readme_read,
2886        .llseek         = generic_file_llseek,
2887};
2888
2889static ssize_t
2890tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
2891                                size_t cnt, loff_t *ppos)
2892{
2893        char *buf_comm;
2894        char *file_buf;
2895        char *buf;
2896        int len = 0;
2897        int pid;
2898        int i;
2899
2900        file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2901        if (!file_buf)
2902                return -ENOMEM;
2903
2904        buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2905        if (!buf_comm) {
2906                kfree(file_buf);
2907                return -ENOMEM;
2908        }
2909
2910        buf = file_buf;
2911
2912        for (i = 0; i < SAVED_CMDLINES; i++) {
2913                int r;
2914
2915                pid = map_cmdline_to_pid[i];
2916                if (pid == -1 || pid == NO_CMDLINE_MAP)
2917                        continue;
2918
2919                trace_find_cmdline(pid, buf_comm);
2920                r = sprintf(buf, "%d %s\n", pid, buf_comm);
2921                buf += r;
2922                len += r;
2923        }
2924
2925        len = simple_read_from_buffer(ubuf, cnt, ppos,
2926                                      file_buf, len);
2927
2928        kfree(file_buf);
2929        kfree(buf_comm);
2930
2931        return len;
2932}
2933
2934static const struct file_operations tracing_saved_cmdlines_fops = {
2935    .open       = tracing_open_generic,
2936    .read       = tracing_saved_cmdlines_read,
2937    .llseek     = generic_file_llseek,
2938};
2939
2940static ssize_t
2941tracing_ctrl_read(struct file *filp, char __user *ubuf,
2942                  size_t cnt, loff_t *ppos)
2943{
2944        char buf[64];
2945        int r;
2946
2947        r = sprintf(buf, "%u\n", tracer_enabled);
2948        return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2949}
2950
2951static ssize_t
2952tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2953                   size_t cnt, loff_t *ppos)
2954{
2955        struct trace_array *tr = filp->private_data;
2956        unsigned long val;
2957        int ret;
2958
2959        ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
2960        if (ret)
2961                return ret;
2962
2963        val = !!val;
2964
2965        mutex_lock(&trace_types_lock);
2966        if (tracer_enabled ^ val) {
2967
2968                /* Only need to warn if this is used to change the state */
2969                WARN_ONCE(1, "tracing_enabled is deprecated. Use tracing_on");
2970
2971                if (val) {
2972                        tracer_enabled = 1;
2973                        if (current_trace->start)
2974                                current_trace->start(tr);
2975                        tracing_start();
2976                } else {
2977                        tracer_enabled = 0;
2978                        tracing_stop();
2979                        if (current_trace->stop)
2980                                current_trace->stop(tr);
2981                }
2982        }
2983        mutex_unlock(&trace_types_lock);
2984
2985        *ppos += cnt;
2986
2987        return cnt;
2988}
2989
2990static ssize_t
2991tracing_set_trace_read(struct file *filp, char __user *ubuf,
2992                       size_t cnt, loff_t *ppos)
2993{
2994        char buf[MAX_TRACER_SIZE+2];
2995        int r;
2996
2997        mutex_lock(&trace_types_lock);
2998        if (current_trace)
2999                r = sprintf(buf, "%s\n", current_trace->name);
3000        else
3001                r = sprintf(buf, "\n");
3002        mutex_unlock(&trace_types_lock);
3003
3004        return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3005}
3006
3007int tracer_init(struct tracer *t, struct trace_array *tr)
3008{
3009        tracing_reset_online_cpus(tr);
3010        return t->init(tr);
3011}
3012
3013static void set_buffer_entries(struct trace_array *tr, unsigned long val)
3014{
3015        int cpu;
3016        for_each_tracing_cpu(cpu)
3017                tr->data[cpu]->entries = val;
3018}
3019
3020static int __tracing_resize_ring_buffer(unsigned long size, int cpu)
3021{
3022        int ret;
3023
3024        /*
3025         * If kernel or user changes the size of the ring buffer
3026         * we use the size that was given, and we can forget about
3027         * expanding it later.
3028         */
3029        ring_buffer_expanded = 1;
3030
3031        ret = ring_buffer_resize(global_trace.buffer, size, cpu);
3032        if (ret < 0)
3033                return ret;
3034
3035        if (!current_trace->use_max_tr)
3036                goto out;
3037
3038        ret = ring_buffer_resize(max_tr.buffer, size, cpu);
3039        if (ret < 0) {
3040                int r = 0;
3041
3042                if (cpu == RING_BUFFER_ALL_CPUS) {
3043                        int i;
3044                        for_each_tracing_cpu(i) {
3045                                r = ring_buffer_resize(global_trace.buffer,
3046                                                global_trace.data[i]->entries,
3047                                                i);
3048                                if (r < 0)
3049                                        break;
3050                        }
3051                } else {
3052                        r = ring_buffer_resize(global_trace.buffer,
3053                                                global_trace.data[cpu]->entries,
3054                                                cpu);
3055                }
3056
3057                if (r < 0) {
3058                        /*
3059                         * AARGH! We are left with different
3060                         * size max buffer!!!!
3061                         * The max buffer is our "snapshot" buffer.
3062                         * When a tracer needs a snapshot (one of the
3063                         * latency tracers), it swaps the max buffer
3064                         * with the saved snap shot. We succeeded to
3065                         * update the size of the main buffer, but failed to
3066                         * update the size of the max buffer. But when we tried
3067                         * to reset the main buffer to the original size, we
3068                         * failed there too. This is very unlikely to
3069                         * happen, but if it does, warn and kill all
3070                         * tracing.
3071                         */
3072                        WARN_ON(1);
3073                        tracing_disabled = 1;
3074                }
3075                return ret;
3076        }
3077
3078        if (cpu == RING_BUFFER_ALL_CPUS)
3079                set_buffer_entries(&max_tr, size);
3080        else
3081                max_tr.data[cpu]->entries = size;
3082
3083 out:
3084        if (cpu == RING_BUFFER_ALL_CPUS)
3085                set_buffer_entries(&global_trace, size);
3086        else
3087                global_trace.data[cpu]->entries = size;
3088
3089        return ret;
3090}
3091
3092static ssize_t tracing_resize_ring_buffer(unsigned long size, int cpu_id)
3093{
3094        int ret = size;
3095
3096        mutex_lock(&trace_types_lock);
3097
3098        if (cpu_id != RING_BUFFER_ALL_CPUS) {
3099                /* make sure, this cpu is enabled in the mask */
3100                if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3101                        ret = -EINVAL;
3102                        goto out;
3103                }
3104        }
3105
3106        ret = __tracing_resize_ring_buffer(size, cpu_id);
3107        if (ret < 0)
3108                ret = -ENOMEM;
3109
3110out:
3111        mutex_unlock(&trace_types_lock);
3112
3113        return ret;
3114}
3115
3116
3117/**
3118 * tracing_update_buffers - used by tracing facility to expand ring buffers
3119 *
3120 * To save on memory when the tracing is never used on a system with it
3121 * configured in. The ring buffers are set to a minimum size. But once
3122 * a user starts to use the tracing facility, then they need to grow
3123 * to their default size.
3124 *
3125 * This function is to be called when a tracer is about to be used.
3126 */
3127int tracing_update_buffers(void)
3128{
3129        int ret = 0;
3130
3131        mutex_lock(&trace_types_lock);
3132        if (!ring_buffer_expanded)
3133                ret = __tracing_resize_ring_buffer(trace_buf_size,
3134                                                RING_BUFFER_ALL_CPUS);
3135        mutex_unlock(&trace_types_lock);
3136
3137        return ret;
3138}
3139
3140struct trace_option_dentry;
3141
3142static struct trace_option_dentry *
3143create_trace_option_files(struct tracer *tracer);
3144
3145static void
3146destroy_trace_option_files(struct trace_option_dentry *topts);
3147
3148static int tracing_set_tracer(const char *buf)
3149{
3150        static struct trace_option_dentry *topts;
3151        struct trace_array *tr = &global_trace;
3152        struct tracer *t;
3153        int ret = 0;
3154
3155        mutex_lock(&trace_types_lock);
3156
3157        if (!ring_buffer_expanded) {
3158                ret = __tracing_resize_ring_buffer(trace_buf_size,
3159                                                RING_BUFFER_ALL_CPUS);
3160                if (ret < 0)
3161                        goto out;
3162                ret = 0;
3163        }
3164
3165        for (t = trace_types; t; t = t->next) {
3166                if (strcmp(t->name, buf) == 0)
3167                        break;
3168        }
3169        if (!t) {
3170                ret = -EINVAL;
3171                goto out;
3172        }
3173        if (t == current_trace)
3174                goto out;
3175
3176        trace_branch_disable();
3177        if (current_trace && current_trace->reset)
3178                current_trace->reset(tr);
3179        if (current_trace && current_trace->use_max_tr) {
3180                /*
3181                 * We don't free the ring buffer. instead, resize it because
3182                 * The max_tr ring buffer has some state (e.g. ring->clock) and
3183                 * we want preserve it.
3184                 */
3185                ring_buffer_resize(max_tr.buffer, 1, RING_BUFFER_ALL_CPUS);
3186                set_buffer_entries(&max_tr, 1);
3187        }
3188        destroy_trace_option_files(topts);
3189
3190        current_trace = &nop_trace;
3191
3192        topts = create_trace_option_files(t);
3193        if (t->use_max_tr) {
3194                int cpu;
3195                /* we need to make per cpu buffer sizes equivalent */
3196                for_each_tracing_cpu(cpu) {
3197                        ret = ring_buffer_resize(max_tr.buffer,
3198                                                global_trace.data[cpu]->entries,
3199                                                cpu);
3200                        if (ret < 0)
3201                                goto out;
3202                        max_tr.data[cpu]->entries =
3203                                        global_trace.data[cpu]->entries;
3204                }
3205        }
3206
3207        if (t->init) {
3208                ret = tracer_init(t, tr);
3209                if (ret)
3210                        goto out;
3211        }
3212
3213        current_trace = t;
3214        trace_branch_enable(tr);
3215 out:
3216        mutex_unlock(&trace_types_lock);
3217
3218        return ret;
3219}
3220
3221static ssize_t
3222tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3223                        size_t cnt, loff_t *ppos)
3224{
3225        char buf[MAX_TRACER_SIZE+1];
3226        int i;
3227        size_t ret;
3228        int err;
3229
3230        ret = cnt;
3231
3232        if (cnt > MAX_TRACER_SIZE)
3233                cnt = MAX_TRACER_SIZE;
3234
3235        if (copy_from_user(&buf, ubuf, cnt))
3236                return -EFAULT;
3237
3238        buf[cnt] = 0;
3239
3240        /* strip ending whitespace. */
3241        for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3242                buf[i] = 0;
3243
3244        err = tracing_set_tracer(buf);
3245        if (err)
3246                return err;
3247
3248        *ppos += ret;
3249
3250        return ret;
3251}
3252
3253static ssize_t
3254tracing_max_lat_read(struct file *filp, char __user *ubuf,
3255                     size_t cnt, loff_t *ppos)
3256{
3257        unsigned long *ptr = filp->private_data;
3258        char buf[64];
3259        int r;
3260
3261        r = snprintf(buf, sizeof(buf), "%ld\n",
3262                     *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
3263        if (r > sizeof(buf))
3264                r = sizeof(buf);
3265        return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3266}
3267
3268static ssize_t
3269tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3270                      size_t cnt, loff_t *ppos)
3271{
3272        unsigned long *ptr = filp->private_data;
3273        unsigned long val;
3274        int ret;
3275
3276        ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3277        if (ret)
3278                return ret;
3279
3280        *ptr = val * 1000;
3281
3282        return cnt;
3283}
3284
3285static int tracing_open_pipe(struct inode *inode, struct file *filp)
3286{
3287        long cpu_file = (long) inode->i_private;
3288        struct trace_iterator *iter;
3289        int ret = 0;
3290
3291        if (tracing_disabled)
3292                return -ENODEV;
3293
3294        mutex_lock(&trace_types_lock);
3295
3296        /* create a buffer to store the information to pass to userspace */
3297        iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3298        if (!iter) {
3299                ret = -ENOMEM;
3300                goto out;
3301        }
3302
3303        /*
3304         * We make a copy of the current tracer to avoid concurrent
3305         * changes on it while we are reading.
3306         */
3307        iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3308        if (!iter->trace) {
3309                ret = -ENOMEM;
3310                goto fail;
3311        }
3312        if (current_trace)
3313                *iter->trace = *current_trace;
3314
3315        if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3316                ret = -ENOMEM;
3317                goto fail;
3318        }
3319
3320        /* trace pipe does not show start of buffer */
3321        cpumask_setall(iter->started);
3322
3323        if (trace_flags & TRACE_ITER_LATENCY_FMT)
3324                iter->iter_flags |= TRACE_FILE_LAT_FMT;
3325
3326        iter->cpu_file = cpu_file;
3327        iter->tr = &global_trace;
3328        mutex_init(&iter->mutex);
3329        filp->private_data = iter;
3330
3331        if (iter->trace->pipe_open)
3332                iter->trace->pipe_open(iter);
3333
3334        nonseekable_open(inode, filp);
3335out:
3336        mutex_unlock(&trace_types_lock);
3337        return ret;
3338
3339fail:
3340        kfree(iter->trace);
3341        kfree(iter);
3342        mutex_unlock(&trace_types_lock);
3343        return ret;
3344}
3345
3346static int tracing_release_pipe(struct inode *inode, struct file *file)
3347{
3348        struct trace_iterator *iter = file->private_data;
3349
3350        mutex_lock(&trace_types_lock);
3351
3352        if (iter->trace->pipe_close)
3353                iter->trace->pipe_close(iter);
3354
3355        mutex_unlock(&trace_types_lock);
3356
3357        free_cpumask_var(iter->started);
3358        mutex_destroy(&iter->mutex);
3359        kfree(iter->trace);
3360        kfree(iter);
3361
3362        return 0;
3363}
3364
3365static unsigned int
3366tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3367{
3368        struct trace_iterator *iter = filp->private_data;
3369
3370        if (trace_flags & TRACE_ITER_BLOCK) {
3371                /*
3372                 * Always select as readable when in blocking mode
3373                 */
3374                return POLLIN | POLLRDNORM;
3375        } else {
3376                if (!trace_empty(iter))
3377                        return POLLIN | POLLRDNORM;
3378                poll_wait(filp, &trace_wait, poll_table);
3379                if (!trace_empty(iter))
3380                        return POLLIN | POLLRDNORM;
3381
3382                return 0;
3383        }
3384}
3385
3386
3387void default_wait_pipe(struct trace_iterator *iter)
3388{
3389        DEFINE_WAIT(wait);
3390
3391        prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
3392
3393        if (trace_empty(iter))
3394                schedule();
3395
3396        finish_wait(&trace_wait, &wait);
3397}
3398
3399/*
3400 * This is a make-shift waitqueue.
3401 * A tracer might use this callback on some rare cases:
3402 *
3403 *  1) the current tracer might hold the runqueue lock when it wakes up
3404 *     a reader, hence a deadlock (sched, function, and function graph tracers)
3405 *  2) the function tracers, trace all functions, we don't want
3406 *     the overhead of calling wake_up and friends
3407 *     (and tracing them too)
3408 *
3409 *     Anyway, this is really very primitive wakeup.
3410 */
3411void poll_wait_pipe(struct trace_iterator *iter)
3412{
3413        set_current_state(TASK_INTERRUPTIBLE);
3414        /* sleep for 100 msecs, and try again. */
3415        schedule_timeout(HZ / 10);
3416}
3417
3418/* Must be called with trace_types_lock mutex held. */
3419static int tracing_wait_pipe(struct file *filp)
3420{
3421        struct trace_iterator *iter = filp->private_data;
3422
3423        while (trace_empty(iter)) {
3424
3425                if ((filp->f_flags & O_NONBLOCK)) {
3426                        return -EAGAIN;
3427                }
3428
3429                mutex_unlock(&iter->mutex);
3430
3431                iter->trace->wait_pipe(iter);
3432
3433                mutex_lock(&iter->mutex);
3434
3435                if (signal_pending(current))
3436                        return -EINTR;
3437
3438                /*
3439                 * We block until we read something and tracing is disabled.
3440                 * We still block if tracing is disabled, but we have never
3441                 * read anything. This allows a user to cat this file, and
3442                 * then enable tracing. But after we have read something,
3443                 * we give an EOF when tracing is again disabled.
3444                 *
3445                 * iter->pos will be 0 if we haven't read anything.
3446                 */
3447                if (!tracer_enabled && iter->pos)
3448                        break;
3449        }
3450
3451        return 1;
3452}
3453
3454/*
3455 * Consumer reader.
3456 */
3457static ssize_t
3458tracing_read_pipe(struct file *filp, char __user *ubuf,
3459                  size_t cnt, loff_t *ppos)
3460{
3461        struct trace_iterator *iter = filp->private_data;
3462        static struct tracer *old_tracer;
3463        ssize_t sret;
3464
3465        /* return any leftover data */
3466        sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3467        if (sret != -EBUSY)
3468                return sret;
3469
3470        trace_seq_init(&iter->seq);
3471
3472        /* copy the tracer to avoid using a global lock all around */
3473        mutex_lock(&trace_types_lock);
3474        if (unlikely(old_tracer != current_trace && current_trace)) {
3475                old_tracer = current_trace;
3476                *iter->trace = *current_trace;
3477        }
3478        mutex_unlock(&trace_types_lock);
3479
3480        /*
3481         * Avoid more than one consumer on a single file descriptor
3482         * This is just a matter of traces coherency, the ring buffer itself
3483         * is protected.
3484         */
3485        mutex_lock(&iter->mutex);
3486        if (iter->trace->read) {
3487                sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3488                if (sret)
3489                        goto out;
3490        }
3491
3492waitagain:
3493        sret = tracing_wait_pipe(filp);
3494        if (sret <= 0)
3495                goto out;
3496
3497        /* stop when tracing is finished */
3498        if (trace_empty(iter)) {
3499                sret = 0;
3500                goto out;
3501        }
3502
3503        if (cnt >= PAGE_SIZE)
3504                cnt = PAGE_SIZE - 1;
3505
3506        /* reset all but tr, trace, and overruns */
3507        memset(&iter->seq, 0,
3508               sizeof(struct trace_iterator) -
3509               offsetof(struct trace_iterator, seq));
3510        iter->pos = -1;
3511
3512        trace_event_read_lock();
3513        trace_access_lock(iter->cpu_file);
3514        while (trace_find_next_entry_inc(iter) != NULL) {
3515                enum print_line_t ret;
3516                int len = iter->seq.len;
3517
3518                ret = print_trace_line(iter);
3519                if (ret == TRACE_TYPE_PARTIAL_LINE) {
3520                        /* don't print partial lines */
3521                        iter->seq.len = len;
3522                        break;
3523                }
3524                if (ret != TRACE_TYPE_NO_CONSUME)
3525                        trace_consume(iter);
3526
3527                if (iter->seq.len >= cnt)
3528                        break;
3529
3530                /*
3531                 * Setting the full flag means we reached the trace_seq buffer
3532                 * size and we should leave by partial output condition above.
3533                 * One of the trace_seq_* functions is not used properly.
3534                 */
3535                WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
3536                          iter->ent->type);
3537        }
3538        trace_access_unlock(iter->cpu_file);
3539        trace_event_read_unlock();
3540
3541        /* Now copy what we have to the user */
3542        sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3543        if (iter->seq.readpos >= iter->seq.len)
3544                trace_seq_init(&iter->seq);
3545
3546        /*
3547         * If there was nothing to send to user, in spite of consuming trace
3548         * entries, go back to wait for more entries.
3549         */
3550        if (sret == -EBUSY)
3551                goto waitagain;
3552
3553out:
3554        mutex_unlock(&iter->mutex);
3555
3556        return sret;
3557}
3558
3559static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3560                                     struct pipe_buffer *buf)
3561{
3562        __free_page(buf->page);
3563}
3564
3565static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3566                                     unsigned int idx)
3567{
3568        __free_page(spd->pages[idx]);
3569}
3570
3571static const struct pipe_buf_operations tracing_pipe_buf_ops = {
3572        .can_merge              = 0,
3573        .map                    = generic_pipe_buf_map,
3574        .unmap                  = generic_pipe_buf_unmap,
3575        .confirm                = generic_pipe_buf_confirm,
3576        .release                = tracing_pipe_buf_release,
3577        .steal                  = generic_pipe_buf_steal,
3578        .get                    = generic_pipe_buf_get,
3579};
3580
3581static size_t
3582tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
3583{
3584        size_t count;
3585        int ret;
3586
3587        /* Seq buffer is page-sized, exactly what we need. */
3588        for (;;) {
3589                count = iter->seq.len;
3590                ret = print_trace_line(iter);
3591                count = iter->seq.len - count;
3592                if (rem < count) {
3593                        rem = 0;
3594                        iter->seq.len -= count;
3595                        break;
3596                }
3597                if (ret == TRACE_TYPE_PARTIAL_LINE) {
3598                        iter->seq.len -= count;
3599                        break;
3600                }
3601
3602                if (ret != TRACE_TYPE_NO_CONSUME)
3603                        trace_consume(iter);
3604                rem -= count;
3605                if (!trace_find_next_entry_inc(iter))   {
3606                        rem = 0;
3607                        iter->ent = NULL;
3608                        break;
3609                }
3610        }
3611
3612        return rem;
3613}
3614
3615static ssize_t tracing_splice_read_pipe(struct file *filp,
3616                                        loff_t *ppos,
3617                                        struct pipe_inode_info *pipe,
3618                                        size_t len,
3619                                        unsigned int flags)
3620{
3621        struct page *pages_def[PIPE_DEF_BUFFERS];
3622        struct partial_page partial_def[PIPE_DEF_BUFFERS];
3623        struct trace_iterator *iter = filp->private_data;
3624        struct splice_pipe_desc spd = {
3625                .pages          = pages_def,
3626                .partial        = partial_def,
3627                .nr_pages       = 0, /* This gets updated below. */
3628                .nr_pages_max   = PIPE_DEF_BUFFERS,
3629                .flags          = flags,
3630                .ops            = &tracing_pipe_buf_ops,
3631                .spd_release    = tracing_spd_release_pipe,
3632        };
3633        static struct tracer *old_tracer;
3634        ssize_t ret;
3635        size_t rem;
3636        unsigned int i;
3637
3638        if (splice_grow_spd(pipe, &spd))
3639                return -ENOMEM;
3640
3641        /* copy the tracer to avoid using a global lock all around */
3642        mutex_lock(&trace_types_lock);
3643        if (unlikely(old_tracer != current_trace && current_trace)) {
3644                old_tracer = current_trace;
3645                *iter->trace = *current_trace;
3646        }
3647        mutex_unlock(&trace_types_lock);
3648
3649        mutex_lock(&iter->mutex);
3650
3651        if (iter->trace->splice_read) {
3652                ret = iter->trace->splice_read(iter, filp,
3653                                               ppos, pipe, len, flags);
3654                if (ret)
3655                        goto out_err;
3656        }
3657
3658        ret = tracing_wait_pipe(filp);
3659        if (ret <= 0)
3660                goto out_err;
3661
3662        if (!iter->ent && !trace_find_next_entry_inc(iter)) {
3663                ret = -EFAULT;
3664                goto out_err;
3665        }
3666
3667        trace_event_read_lock();
3668        trace_access_lock(iter->cpu_file);
3669
3670        /* Fill as many pages as possible. */
3671        for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
3672                spd.pages[i] = alloc_page(GFP_KERNEL);
3673                if (!spd.pages[i])
3674                        break;
3675
3676                rem = tracing_fill_pipe_page(rem, iter);
3677
3678                /* Copy the data into the page, so we can start over. */
3679                ret = trace_seq_to_buffer(&iter->seq,
3680                                          page_address(spd.pages[i]),
3681                                          iter->seq.len);
3682                if (ret < 0) {
3683                        __free_page(spd.pages[i]);
3684                        break;
3685                }
3686                spd.partial[i].offset = 0;
3687                spd.partial[i].len = iter->seq.len;
3688
3689                trace_seq_init(&iter->seq);
3690        }
3691
3692        trace_access_unlock(iter->cpu_file);
3693        trace_event_read_unlock();
3694        mutex_unlock(&iter->mutex);
3695
3696        spd.nr_pages = i;
3697
3698        ret = splice_to_pipe(pipe, &spd);
3699out:
3700        splice_shrink_spd(&spd);
3701        return ret;
3702
3703out_err:
3704        mutex_unlock(&iter->mutex);
3705        goto out;
3706}
3707
3708struct ftrace_entries_info {
3709        struct trace_array      *tr;
3710        int                     cpu;
3711};
3712
3713static int tracing_entries_open(struct inode *inode, struct file *filp)
3714{
3715        struct ftrace_entries_info *info;
3716
3717        if (tracing_disabled)
3718                return -ENODEV;
3719
3720        info = kzalloc(sizeof(*info), GFP_KERNEL);
3721        if (!info)
3722                return -ENOMEM;
3723
3724        info->tr = &global_trace;
3725        info->cpu = (unsigned long)inode->i_private;
3726
3727        filp->private_data = info;
3728
3729        return 0;
3730}
3731
3732static ssize_t
3733tracing_entries_read(struct file *filp, char __user *ubuf,
3734                     size_t cnt, loff_t *ppos)
3735{
3736        struct ftrace_entries_info *info = filp->private_data;
3737        struct trace_array *tr = info->tr;
3738        char buf[64];
3739        int r = 0;
3740        ssize_t ret;
3741
3742        mutex_lock(&trace_types_lock);
3743
3744        if (info->cpu == RING_BUFFER_ALL_CPUS) {
3745                int cpu, buf_size_same;
3746                unsigned long size;
3747
3748                size = 0;
3749                buf_size_same = 1;
3750                /* check if all cpu sizes are same */
3751                for_each_tracing_cpu(cpu) {
3752                        /* fill in the size from first enabled cpu */
3753                        if (size == 0)
3754                                size = tr->data[cpu]->entries;
3755                        if (size != tr->data[cpu]->entries) {
3756                                buf_size_same = 0;
3757                                break;
3758                        }
3759                }
3760
3761                if (buf_size_same) {
3762                        if (!ring_buffer_expanded)
3763                                r = sprintf(buf, "%lu (expanded: %lu)\n",
3764                                            size >> 10,
3765                                            trace_buf_size >> 10);
3766                        else
3767                                r = sprintf(buf, "%lu\n", size >> 10);
3768                } else
3769                        r = sprintf(buf, "X\n");
3770        } else
3771                r = sprintf(buf, "%lu\n", tr->data[info->cpu]->entries >> 10);
3772
3773        mutex_unlock(&trace_types_lock);
3774
3775        ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3776        return ret;
3777}
3778
3779static ssize_t
3780tracing_entries_write(struct file *filp, const char __user *ubuf,
3781                      size_t cnt, loff_t *ppos)
3782{
3783        struct ftrace_entries_info *info = filp->private_data;
3784        unsigned long val;
3785        int ret;
3786
3787        ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3788        if (ret)
3789                return ret;
3790
3791        /* must have at least 1 entry */
3792        if (!val)
3793                return -EINVAL;
3794
3795        /* value is in KB */
3796        val <<= 10;
3797
3798        ret = tracing_resize_ring_buffer(val, info->cpu);
3799        if (ret < 0)
3800                return ret;
3801
3802        *ppos += cnt;
3803
3804        return cnt;
3805}
3806
3807static int
3808tracing_entries_release(struct inode *inode, struct file *filp)
3809{
3810        struct ftrace_entries_info *info = filp->private_data;
3811
3812        kfree(info);
3813
3814        return 0;
3815}
3816
3817static ssize_t
3818tracing_total_entries_read(struct file *filp, char __user *ubuf,
3819                                size_t cnt, loff_t *ppos)
3820{
3821        struct trace_array *tr = filp->private_data;
3822        char buf[64];
3823        int r, cpu;
3824        unsigned long size = 0, expanded_size = 0;
3825
3826        mutex_lock(&trace_types_lock);
3827        for_each_tracing_cpu(cpu) {
3828                size += tr->data[cpu]->entries >> 10;
3829                if (!ring_buffer_expanded)
3830                        expanded_size += trace_buf_size >> 10;
3831        }
3832        if (ring_buffer_expanded)
3833                r = sprintf(buf, "%lu\n", size);
3834        else
3835                r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
3836        mutex_unlock(&trace_types_lock);
3837
3838        return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3839}
3840
3841static ssize_t
3842tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
3843                          size_t cnt, loff_t *ppos)
3844{
3845        /*
3846         * There is no need to read what the user has written, this function
3847         * is just to make sure that there is no error when "echo" is used
3848         */
3849
3850        *ppos += cnt;
3851
3852        return cnt;
3853}
3854
3855static int
3856tracing_free_buffer_release(struct inode *inode, struct file *filp)
3857{
3858        /* disable tracing ? */
3859        if (trace_flags & TRACE_ITER_STOP_ON_FREE)
3860                tracing_off();
3861        /* resize the ring buffer to 0 */
3862        tracing_resize_ring_buffer(0, RING_BUFFER_ALL_CPUS);
3863
3864        return 0;
3865}
3866
3867static ssize_t
3868tracing_mark_write(struct file *filp, const char __user *ubuf,
3869                                        size_t cnt, loff_t *fpos)
3870{
3871        unsigned long addr = (unsigned long)ubuf;
3872        struct ring_buffer_event *event;
3873        struct ring_buffer *buffer;
3874        struct print_entry *entry;
3875        unsigned long irq_flags;
3876        struct page *pages[2];
3877        void *map_page[2];
3878        int nr_pages = 1;
3879        ssize_t written;
3880        int offset;
3881        int size;
3882        int len;
3883        int ret;
3884        int i;
3885
3886        if (tracing_disabled)
3887                return -EINVAL;
3888
3889        if (cnt > TRACE_BUF_SIZE)
3890                cnt = TRACE_BUF_SIZE;
3891
3892        /*
3893         * Userspace is injecting traces into the kernel trace buffer.
3894         * We want to be as non intrusive as possible.
3895         * To do so, we do not want to allocate any special buffers
3896         * or take any locks, but instead write the userspace data
3897         * straight into the ring buffer.
3898         *
3899         * First we need to pin the userspace buffer into memory,
3900         * which, most likely it is, because it just referenced it.
3901         * But there's no guarantee that it is. By using get_user_pages_fast()
3902         * and kmap_atomic/kunmap_atomic() we can get access to the
3903         * pages directly. We then write the data directly into the
3904         * ring buffer.
3905         */
3906        BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
3907
3908        /* check if we cross pages */
3909        if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
3910                nr_pages = 2;
3911
3912        offset = addr & (PAGE_SIZE - 1);
3913        addr &= PAGE_MASK;
3914
3915        ret = get_user_pages_fast(addr, nr_pages, 0, pages);
3916        if (ret < nr_pages) {
3917                while (--ret >= 0)
3918                        put_page(pages[ret]);
3919                written = -EFAULT;
3920                goto out;
3921        }
3922
3923        for (i = 0; i < nr_pages; i++)
3924                map_page[i] = kmap_atomic(pages[i]);
3925
3926        local_save_flags(irq_flags);
3927        size = sizeof(*entry) + cnt + 2; /* possible \n added */
3928        buffer = global_trace.buffer;
3929        event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3930                                          irq_flags, preempt_count());
3931        if (!event) {
3932                /* Ring buffer disabled, return as if not open for write */
3933                written = -EBADF;
3934                goto out_unlock;
3935        }
3936
3937        entry = ring_buffer_event_data(event);
3938        entry->ip = _THIS_IP_;
3939
3940        if (nr_pages == 2) {
3941                len = PAGE_SIZE - offset;
3942                memcpy(&entry->buf, map_page[0] + offset, len);
3943                memcpy(&entry->buf[len], map_page[1], cnt - len);
3944        } else
3945                memcpy(&entry->buf, map_page[0] + offset, cnt);
3946
3947        if (entry->buf[cnt - 1] != '\n') {
3948                entry->buf[cnt] = '\n';
3949                entry->buf[cnt + 1] = '\0';
3950        } else
3951                entry->buf[cnt] = '\0';
3952
3953        ring_buffer_unlock_commit(buffer, event);
3954
3955        written = cnt;
3956
3957        *fpos += written;
3958
3959 out_unlock:
3960        for (i = 0; i < nr_pages; i++){
3961                kunmap_atomic(map_page[i]);
3962                put_page(pages[i]);
3963        }
3964 out:
3965        return written;
3966}
3967
3968static int tracing_clock_show(struct seq_file *m, void *v)
3969{
3970        int i;
3971
3972        for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
3973                seq_printf(m,
3974                        "%s%s%s%s", i ? " " : "",
3975                        i == trace_clock_id ? "[" : "", trace_clocks[i].name,
3976                        i == trace_clock_id ? "]" : "");
3977        seq_putc(m, '\n');
3978
3979        return 0;
3980}
3981
3982static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
3983                                   size_t cnt, loff_t *fpos)
3984{
3985        char buf[64];
3986        const char *clockstr;
3987        int i;
3988
3989        if (cnt >= sizeof(buf))
3990                return -EINVAL;
3991
3992        if (copy_from_user(&buf, ubuf, cnt))
3993                return -EFAULT;
3994
3995        buf[cnt] = 0;
3996
3997        clockstr = strstrip(buf);
3998
3999        for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4000                if (strcmp(trace_clocks[i].name, clockstr) == 0)
4001                        break;
4002        }
4003        if (i == ARRAY_SIZE(trace_clocks))
4004                return -EINVAL;
4005
4006        trace_clock_id = i;
4007
4008        mutex_lock(&trace_types_lock);
4009
4010        ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
4011        if (max_tr.buffer)
4012                ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
4013
4014        mutex_unlock(&trace_types_lock);
4015
4016        *fpos += cnt;
4017
4018        return cnt;
4019}
4020
4021static int tracing_clock_open(struct inode *inode, struct file *file)
4022{
4023        if (tracing_disabled)
4024                return -ENODEV;
4025        return single_open(file, tracing_clock_show, NULL);
4026}
4027
4028static const struct file_operations tracing_max_lat_fops = {
4029        .open           = tracing_open_generic,
4030        .read           = tracing_max_lat_read,
4031        .write          = tracing_max_lat_write,
4032        .llseek         = generic_file_llseek,
4033};
4034
4035static const struct file_operations tracing_ctrl_fops = {
4036        .open           = tracing_open_generic,
4037        .read           = tracing_ctrl_read,
4038        .write          = tracing_ctrl_write,
4039        .llseek         = generic_file_llseek,
4040};
4041
4042static const struct file_operations set_tracer_fops = {
4043        .open           = tracing_open_generic,
4044        .read           = tracing_set_trace_read,
4045        .write          = tracing_set_trace_write,
4046        .llseek         = generic_file_llseek,
4047};
4048
4049static const struct file_operations tracing_pipe_fops = {
4050        .open           = tracing_open_pipe,
4051        .poll           = tracing_poll_pipe,
4052        .read           = tracing_read_pipe,
4053        .splice_read    = tracing_splice_read_pipe,
4054        .release        = tracing_release_pipe,
4055        .llseek         = no_llseek,
4056};
4057
4058static const struct file_operations tracing_entries_fops = {
4059        .open           = tracing_entries_open,
4060        .read           = tracing_entries_read,
4061        .write          = tracing_entries_write,
4062        .release        = tracing_entries_release,
4063        .llseek         = generic_file_llseek,
4064};
4065
4066static const struct file_operations tracing_total_entries_fops = {
4067        .open           = tracing_open_generic,
4068        .read           = tracing_total_entries_read,
4069        .llseek         = generic_file_llseek,
4070};
4071
4072static const struct file_operations tracing_free_buffer_fops = {
4073        .write          = tracing_free_buffer_write,
4074        .release        = tracing_free_buffer_release,
4075};
4076
4077static const struct file_operations tracing_mark_fops = {
4078        .open           = tracing_open_generic,
4079        .write          = tracing_mark_write,
4080        .llseek         = generic_file_llseek,
4081};
4082
4083static const struct file_operations trace_clock_fops = {
4084        .open           = tracing_clock_open,
4085        .read           = seq_read,
4086        .llseek         = seq_lseek,
4087        .release        = single_release,
4088        .write          = tracing_clock_write,
4089};
4090
4091struct ftrace_buffer_info {
4092        struct trace_array      *tr;
4093        void                    *spare;
4094        int                     cpu;
4095        unsigned int            read;
4096};
4097
4098static int tracing_buffers_open(struct inode *inode, struct file *filp)
4099{
4100        int cpu = (int)(long)inode->i_private;
4101        struct ftrace_buffer_info *info;
4102
4103        if (tracing_disabled)
4104                return -ENODEV;
4105
4106        info = kzalloc(sizeof(*info), GFP_KERNEL);
4107        if (!info)
4108                return -ENOMEM;
4109
4110        info->tr        = &global_trace;
4111        info->cpu       = cpu;
4112        info->spare     = NULL;
4113        /* Force reading ring buffer for first read */
4114        info->read      = (unsigned int)-1;
4115
4116        filp->private_data = info;
4117
4118        return nonseekable_open(inode, filp);
4119}
4120
4121static ssize_t
4122tracing_buffers_read(struct file *filp, char __user *ubuf,
4123                     size_t count, loff_t *ppos)
4124{
4125        struct ftrace_buffer_info *info = filp->private_data;
4126        ssize_t ret;
4127        size_t size;
4128
4129        if (!count)
4130                return 0;
4131
4132        if (!info->spare)
4133                info->spare = ring_buffer_alloc_read_page(info->tr->buffer, info->cpu);
4134        if (!info->spare)
4135                return -ENOMEM;
4136
4137        /* Do we have previous read data to read? */
4138        if (info->read < PAGE_SIZE)
4139                goto read;
4140
4141        trace_access_lock(info->cpu);
4142        ret = ring_buffer_read_page(info->tr->buffer,
4143                                    &info->spare,
4144                                    count,
4145                                    info->cpu, 0);
4146        trace_access_unlock(info->cpu);
4147        if (ret < 0)
4148                return 0;
4149
4150        info->read = 0;
4151
4152read:
4153        size = PAGE_SIZE - info->read;
4154        if (size > count)
4155                size = count;
4156
4157        ret = copy_to_user(ubuf, info->spare + info->read, size);
4158        if (ret == size)
4159                return -EFAULT;
4160        size -= ret;
4161
4162        *ppos += size;
4163        info->read += size;
4164
4165        return size;
4166}
4167
4168static int tracing_buffers_release(struct inode *inode, struct file *file)
4169{
4170        struct ftrace_buffer_info *info = file->private_data;
4171
4172        if (info->spare)
4173                ring_buffer_free_read_page(info->tr->buffer, info->spare);
4174        kfree(info);
4175
4176        return 0;
4177}
4178
4179struct buffer_ref {
4180        struct ring_buffer      *buffer;
4181        void                    *page;
4182        int                     ref;
4183};
4184
4185static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
4186                                    struct pipe_buffer *buf)
4187{
4188        struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4189
4190        if (--ref->ref)
4191                return;
4192
4193        ring_buffer_free_read_page(ref->buffer, ref->page);
4194        kfree(ref);
4195        buf->private = 0;
4196}
4197
4198static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
4199                                 struct pipe_buffer *buf)
4200{
4201        return 1;
4202}
4203
4204static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
4205                                struct pipe_buffer *buf)
4206{
4207        struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4208
4209        ref->ref++;
4210}
4211
4212/* Pipe buffer operations for a buffer. */
4213static const struct pipe_buf_operations buffer_pipe_buf_ops = {
4214        .can_merge              = 0,
4215        .map                    = generic_pipe_buf_map,
4216        .unmap                  = generic_pipe_buf_unmap,
4217        .confirm                = generic_pipe_buf_confirm,
4218        .release                = buffer_pipe_buf_release,
4219        .steal                  = buffer_pipe_buf_steal,
4220        .get                    = buffer_pipe_buf_get,
4221};
4222
4223/*
4224 * Callback from splice_to_pipe(), if we need to release some pages
4225 * at the end of the spd in case we error'ed out in filling the pipe.
4226 */
4227static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
4228{
4229        struct buffer_ref *ref =
4230                (struct buffer_ref *)spd->partial[i].private;
4231
4232        if (--ref->ref)
4233                return;
4234
4235        ring_buffer_free_read_page(ref->buffer, ref->page);
4236        kfree(ref);
4237        spd->partial[i].private = 0;
4238}
4239
4240static ssize_t
4241tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4242                            struct pipe_inode_info *pipe, size_t len,
4243                            unsigned int flags)
4244{
4245        struct ftrace_buffer_info *info = file->private_data;
4246        struct partial_page partial_def[PIPE_DEF_BUFFERS];
4247        struct page *pages_def[PIPE_DEF_BUFFERS];
4248        struct splice_pipe_desc spd = {
4249                .pages          = pages_def,
4250                .partial        = partial_def,
4251                .nr_pages_max   = PIPE_DEF_BUFFERS,
4252                .flags          = flags,
4253                .ops            = &buffer_pipe_buf_ops,
4254                .spd_release    = buffer_spd_release,
4255        };
4256        struct buffer_ref *ref;
4257        int entries, size, i;
4258        size_t ret;
4259
4260        if (splice_grow_spd(pipe, &spd))
4261                return -ENOMEM;
4262
4263        if (*ppos & (PAGE_SIZE - 1)) {
4264                WARN_ONCE(1, "Ftrace: previous read must page-align\n");
4265                ret = -EINVAL;
4266                goto out;
4267        }
4268
4269        if (len & (PAGE_SIZE - 1)) {
4270                WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
4271                if (len < PAGE_SIZE) {
4272                        ret = -EINVAL;
4273                        goto out;
4274                }
4275                len &= PAGE_MASK;
4276        }
4277
4278        trace_access_lock(info->cpu);
4279        entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4280
4281        for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
4282                struct page *page;
4283                int r;
4284
4285                ref = kzalloc(sizeof(*ref), GFP_KERNEL);
4286                if (!ref)
4287                        break;
4288
4289                ref->ref = 1;
4290                ref->buffer = info->tr->buffer;
4291                ref->page = ring_buffer_alloc_read_page(ref->buffer, info->cpu);
4292                if (!ref->page) {
4293                        kfree(ref);
4294                        break;
4295                }
4296
4297                r = ring_buffer_read_page(ref->buffer, &ref->page,
4298                                          len, info->cpu, 1);
4299                if (r < 0) {
4300                        ring_buffer_free_read_page(ref->buffer, ref->page);
4301                        kfree(ref);
4302                        break;
4303                }
4304
4305                /*
4306                 * zero out any left over data, this is going to
4307                 * user land.
4308                 */
4309                size = ring_buffer_page_len(ref->page);
4310                if (size < PAGE_SIZE)
4311                        memset(ref->page + size, 0, PAGE_SIZE - size);
4312
4313                page = virt_to_page(ref->page);
4314
4315                spd.pages[i] = page;
4316                spd.partial[i].len = PAGE_SIZE;
4317                spd.partial[i].offset = 0;
4318                spd.partial[i].private = (unsigned long)ref;
4319                spd.nr_pages++;
4320                *ppos += PAGE_SIZE;
4321
4322                entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4323        }
4324
4325        trace_access_unlock(info->cpu);
4326        spd.nr_pages = i;
4327
4328        /* did we read anything? */
4329        if (!spd.nr_pages) {
4330                if (flags & SPLICE_F_NONBLOCK)
4331                        ret = -EAGAIN;
4332                else
4333                        ret = 0;
4334                /* TODO: block */
4335                goto out;
4336        }
4337
4338        ret = splice_to_pipe(pipe, &spd);
4339        splice_shrink_spd(&spd);
4340out:
4341        return ret;
4342}
4343
4344static const struct file_operations tracing_buffers_fops = {
4345        .open           = tracing_buffers_open,
4346        .read           = tracing_buffers_read,
4347        .release        = tracing_buffers_release,
4348        .splice_read    = tracing_buffers_splice_read,
4349        .llseek         = no_llseek,
4350};
4351
4352static ssize_t
4353tracing_stats_read(struct file *filp, char __user *ubuf,
4354                   size_t count, loff_t *ppos)
4355{
4356        unsigned long cpu = (unsigned long)filp->private_data;
4357        struct trace_array *tr = &global_trace;
4358        struct trace_seq *s;
4359        unsigned long cnt;
4360        unsigned long long t;
4361        unsigned long usec_rem;
4362
4363        s = kmalloc(sizeof(*s), GFP_KERNEL);
4364        if (!s)
4365                return -ENOMEM;
4366
4367        trace_seq_init(s);
4368
4369        cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
4370        trace_seq_printf(s, "entries: %ld\n", cnt);
4371
4372        cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
4373        trace_seq_printf(s, "overrun: %ld\n", cnt);
4374
4375        cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
4376        trace_seq_printf(s, "commit overrun: %ld\n", cnt);
4377
4378        cnt = ring_buffer_bytes_cpu(tr->buffer, cpu);
4379        trace_seq_printf(s, "bytes: %ld\n", cnt);
4380
4381        t = ns2usecs(ring_buffer_oldest_event_ts(tr->buffer, cpu));
4382        usec_rem = do_div(t, USEC_PER_SEC);
4383        trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n", t, usec_rem);
4384
4385        t = ns2usecs(ring_buffer_time_stamp(tr->buffer, cpu));
4386        usec_rem = do_div(t, USEC_PER_SEC);
4387        trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
4388
4389        count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
4390
4391        kfree(s);
4392
4393        return count;
4394}
4395
4396static const struct file_operations tracing_stats_fops = {
4397        .open           = tracing_open_generic,
4398        .read           = tracing_stats_read,
4399        .llseek         = generic_file_llseek,
4400};
4401
4402#ifdef CONFIG_DYNAMIC_FTRACE
4403
4404int __weak ftrace_arch_read_dyn_info(char *buf, int size)
4405{
4406        return 0;
4407}
4408
4409static ssize_t
4410tracing_read_dyn_info(struct file *filp, char __user *ubuf,
4411                  size_t cnt, loff_t *ppos)
4412{
4413        static char ftrace_dyn_info_buffer[1024];
4414        static DEFINE_MUTEX(dyn_info_mutex);
4415        unsigned long *p = filp->private_data;
4416        char *buf = ftrace_dyn_info_buffer;
4417        int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
4418        int r;
4419
4420        mutex_lock(&dyn_info_mutex);
4421        r = sprintf(buf, "%ld ", *p);
4422
4423        r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
4424        buf[r++] = '\n';
4425
4426        r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4427
4428        mutex_unlock(&dyn_info_mutex);
4429
4430        return r;
4431}
4432
4433static const struct file_operations tracing_dyn_info_fops = {
4434        .open           = tracing_open_generic,
4435        .read           = tracing_read_dyn_info,
4436        .llseek         = generic_file_llseek,
4437};
4438#endif
4439
4440static struct dentry *d_tracer;
4441
4442struct dentry *tracing_init_dentry(void)
4443{
4444        static int once;
4445
4446        if (d_tracer)
4447                return d_tracer;
4448
4449        if (!debugfs_initialized())
4450                return NULL;
4451
4452        d_tracer = debugfs_create_dir("tracing", NULL);
4453
4454        if (!d_tracer && !once) {
4455                once = 1;
4456                pr_warning("Could not create debugfs directory 'tracing'\n");
4457                return NULL;
4458        }
4459
4460        return d_tracer;
4461}
4462
4463static struct dentry *d_percpu;
4464
4465struct dentry *tracing_dentry_percpu(void)
4466{
4467        static int once;
4468        struct dentry *d_tracer;
4469
4470        if (d_percpu)
4471                return d_percpu;
4472
4473        d_tracer = tracing_init_dentry();
4474
4475        if (!d_tracer)
4476                return NULL;
4477
4478        d_percpu = debugfs_create_dir("per_cpu", d_tracer);
4479
4480        if (!d_percpu && !once) {
4481                once = 1;
4482                pr_warning("Could not create debugfs directory 'per_cpu'\n");
4483                return NULL;
4484        }
4485
4486        return d_percpu;
4487}
4488
4489static void tracing_init_debugfs_percpu(long cpu)
4490{
4491        struct dentry *d_percpu = tracing_dentry_percpu();
4492        struct dentry *d_cpu;
4493        char cpu_dir[30]; /* 30 characters should be more than enough */
4494
4495        if (!d_percpu)
4496                return;
4497
4498        snprintf(cpu_dir, 30, "cpu%ld", cpu);
4499        d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
4500        if (!d_cpu) {
4501                pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
4502                return;
4503        }
4504
4505        /* per cpu trace_pipe */
4506        trace_create_file("trace_pipe", 0444, d_cpu,
4507                        (void *) cpu, &tracing_pipe_fops);
4508
4509        /* per cpu trace */
4510        trace_create_file("trace", 0644, d_cpu,
4511                        (void *) cpu, &tracing_fops);
4512
4513        trace_create_file("trace_pipe_raw", 0444, d_cpu,
4514                        (void *) cpu, &tracing_buffers_fops);
4515
4516        trace_create_file("stats", 0444, d_cpu,
4517                        (void *) cpu, &tracing_stats_fops);
4518
4519        trace_create_file("buffer_size_kb", 0444, d_cpu,
4520                        (void *) cpu, &tracing_entries_fops);
4521}
4522
4523#ifdef CONFIG_FTRACE_SELFTEST
4524/* Let selftest have access to static functions in this file */
4525#include "trace_selftest.c"
4526#endif
4527
4528struct trace_option_dentry {
4529        struct tracer_opt               *opt;
4530        struct tracer_flags             *flags;
4531        struct dentry                   *entry;
4532};
4533
4534static ssize_t
4535trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
4536                        loff_t *ppos)
4537{
4538        struct trace_option_dentry *topt = filp->private_data;
4539        char *buf;
4540
4541        if (topt->flags->val & topt->opt->bit)
4542                buf = "1\n";
4543        else
4544                buf = "0\n";
4545
4546        return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4547}
4548
4549static ssize_t
4550trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
4551                         loff_t *ppos)
4552{
4553        struct trace_option_dentry *topt = filp->private_data;
4554        unsigned long val;
4555        int ret;
4556
4557        ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4558        if (ret)
4559                return ret;
4560
4561        if (val != 0 && val != 1)
4562                return -EINVAL;
4563
4564        if (!!(topt->flags->val & topt->opt->bit) != val) {
4565                mutex_lock(&trace_types_lock);
4566                ret = __set_tracer_option(current_trace, topt->flags,
4567                                          topt->opt, !val);
4568                mutex_unlock(&trace_types_lock);
4569                if (ret)
4570                        return ret;
4571        }
4572
4573        *ppos += cnt;
4574
4575        return cnt;
4576}
4577
4578
4579static const struct file_operations trace_options_fops = {
4580        .open = tracing_open_generic,
4581        .read = trace_options_read,
4582        .write = trace_options_write,
4583        .llseek = generic_file_llseek,
4584};
4585
4586static ssize_t
4587trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
4588                        loff_t *ppos)
4589{
4590        long index = (long)filp->private_data;
4591        char *buf;
4592
4593        if (trace_flags & (1 << index))
4594                buf = "1\n";
4595        else
4596                buf = "0\n";
4597
4598        return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4599}
4600
4601static ssize_t
4602trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
4603                         loff_t *ppos)
4604{
4605        long index = (long)filp->private_data;
4606        unsigned long val;
4607        int ret;
4608
4609        ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4610        if (ret)
4611                return ret;
4612
4613        if (val != 0 && val != 1)
4614                return -EINVAL;
4615        set_tracer_flags(1 << index, val);
4616
4617        *ppos += cnt;
4618
4619        return cnt;
4620}
4621
4622static const struct file_operations trace_options_core_fops = {
4623        .open = tracing_open_generic,
4624        .read = trace_options_core_read,
4625        .write = trace_options_core_write,
4626        .llseek = generic_file_llseek,
4627};
4628
4629struct dentry *trace_create_file(const char *name,
4630                                 umode_t mode,
4631                                 struct dentry *parent,
4632                                 void *data,
4633                                 const struct file_operations *fops)
4634{
4635        struct dentry *ret;
4636
4637        ret = debugfs_create_file(name, mode, parent, data, fops);
4638        if (!ret)
4639                pr_warning("Could not create debugfs '%s' entry\n", name);
4640
4641        return ret;
4642}
4643
4644
4645static struct dentry *trace_options_init_dentry(void)
4646{
4647        struct dentry *d_tracer;
4648        static struct dentry *t_options;
4649
4650        if (t_options)
4651                return t_options;
4652
4653        d_tracer = tracing_init_dentry();
4654        if (!d_tracer)
4655                return NULL;
4656
4657        t_options = debugfs_create_dir("options", d_tracer);
4658        if (!t_options) {
4659                pr_warning("Could not create debugfs directory 'options'\n");
4660                return NULL;
4661        }
4662
4663        return t_options;
4664}
4665
4666static void
4667create_trace_option_file(struct trace_option_dentry *topt,
4668                         struct tracer_flags *flags,
4669                         struct tracer_opt *opt)
4670{
4671        struct dentry *t_options;
4672
4673        t_options = trace_options_init_dentry();
4674        if (!t_options)
4675                return;
4676
4677        topt->flags = flags;
4678        topt->opt = opt;
4679
4680        topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
4681                                    &trace_options_fops);
4682
4683}
4684
4685static struct trace_option_dentry *
4686create_trace_option_files(struct tracer *tracer)
4687{
4688        struct trace_option_dentry *topts;
4689        struct tracer_flags *flags;
4690        struct tracer_opt *opts;
4691        int cnt;
4692
4693        if (!tracer)
4694                return NULL;
4695
4696        flags = tracer->flags;
4697
4698        if (!flags || !flags->opts)
4699                return NULL;
4700
4701        opts = flags->opts;
4702
4703        for (cnt = 0; opts[cnt].name; cnt++)
4704                ;
4705
4706        topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
4707        if (!topts)
4708                return NULL;
4709
4710        for (cnt = 0; opts[cnt].name; cnt++)
4711                create_trace_option_file(&topts[cnt], flags,
4712                                         &opts[cnt]);
4713
4714        return topts;
4715}
4716
4717static void
4718destroy_trace_option_files(struct trace_option_dentry *topts)
4719{
4720        int cnt;
4721
4722        if (!topts)
4723                return;
4724
4725        for (cnt = 0; topts[cnt].opt; cnt++) {
4726                if (topts[cnt].entry)
4727                        debugfs_remove(topts[cnt].entry);
4728        }
4729
4730        kfree(topts);
4731}
4732
4733static struct dentry *
4734create_trace_option_core_file(const char *option, long index)
4735{
4736        struct dentry *t_options;
4737
4738        t_options = trace_options_init_dentry();
4739        if (!t_options)
4740                return NULL;
4741
4742        return trace_create_file(option, 0644, t_options, (void *)index,
4743                                    &trace_options_core_fops);
4744}
4745
4746static __init void create_trace_options_dir(void)
4747{
4748        struct dentry *t_options;
4749        int i;
4750
4751        t_options = trace_options_init_dentry();
4752        if (!t_options)
4753                return;
4754
4755        for (i = 0; trace_options[i]; i++)
4756                create_trace_option_core_file(trace_options[i], i);
4757}
4758
4759static ssize_t
4760rb_simple_read(struct file *filp, char __user *ubuf,
4761               size_t cnt, loff_t *ppos)
4762{
4763        struct trace_array *tr = filp->private_data;
4764        struct ring_buffer *buffer = tr->buffer;
4765        char buf[64];
4766        int r;
4767
4768        if (buffer)
4769                r = ring_buffer_record_is_on(buffer);
4770        else
4771                r = 0;
4772
4773        r = sprintf(buf, "%d\n", r);
4774
4775        return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4776}
4777
4778static ssize_t
4779rb_simple_write(struct file *filp, const char __user *ubuf,
4780                size_t cnt, loff_t *ppos)
4781{
4782        struct trace_array *tr = filp->private_data;
4783        struct ring_buffer *buffer = tr->buffer;
4784        unsigned long val;
4785        int ret;
4786
4787        ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4788        if (ret)
4789                return ret;
4790
4791        if (buffer) {
4792                if (val)
4793                        ring_buffer_record_on(buffer);
4794                else
4795                        ring_buffer_record_off(buffer);
4796        }
4797
4798        (*ppos)++;
4799
4800        return cnt;
4801}
4802
4803static const struct file_operations rb_simple_fops = {
4804        .open           = tracing_open_generic,
4805        .read           = rb_simple_read,
4806        .write          = rb_simple_write,
4807        .llseek         = default_llseek,
4808};
4809
4810static __init int tracer_init_debugfs(void)
4811{
4812        struct dentry *d_tracer;
4813        int cpu;
4814
4815        trace_access_lock_init();
4816
4817        d_tracer = tracing_init_dentry();
4818
4819        trace_create_file("tracing_enabled", 0644, d_tracer,
4820                        &global_trace, &tracing_ctrl_fops);
4821
4822        trace_create_file("trace_options", 0644, d_tracer,
4823                        NULL, &tracing_iter_fops);
4824
4825        trace_create_file("tracing_cpumask", 0644, d_tracer,
4826                        NULL, &tracing_cpumask_fops);
4827
4828        trace_create_file("trace", 0644, d_tracer,
4829                        (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
4830
4831        trace_create_file("available_tracers", 0444, d_tracer,
4832                        &global_trace, &show_traces_fops);
4833
4834        trace_create_file("current_tracer", 0644, d_tracer,
4835                        &global_trace, &set_tracer_fops);
4836
4837#ifdef CONFIG_TRACER_MAX_TRACE
4838        trace_create_file("tracing_max_latency", 0644, d_tracer,
4839                        &tracing_max_latency, &tracing_max_lat_fops);
4840#endif
4841
4842        trace_create_file("tracing_thresh", 0644, d_tracer,
4843                        &tracing_thresh, &tracing_max_lat_fops);
4844
4845        trace_create_file("README", 0444, d_tracer,
4846                        NULL, &tracing_readme_fops);
4847
4848        trace_create_file("trace_pipe", 0444, d_tracer,
4849                        (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
4850
4851        trace_create_file("buffer_size_kb", 0644, d_tracer,
4852                        (void *) RING_BUFFER_ALL_CPUS, &tracing_entries_fops);
4853
4854        trace_create_file("buffer_total_size_kb", 0444, d_tracer,
4855                        &global_trace, &tracing_total_entries_fops);
4856
4857        trace_create_file("free_buffer", 0644, d_tracer,
4858                        &global_trace, &tracing_free_buffer_fops);
4859
4860        trace_create_file("trace_marker", 0220, d_tracer,
4861                        NULL, &tracing_mark_fops);
4862
4863        trace_create_file("saved_cmdlines", 0444, d_tracer,
4864                        NULL, &tracing_saved_cmdlines_fops);
4865
4866        trace_create_file("trace_clock", 0644, d_tracer, NULL,
4867                          &trace_clock_fops);
4868
4869        trace_create_file("tracing_on", 0644, d_tracer,
4870                            &global_trace, &rb_simple_fops);
4871
4872#ifdef CONFIG_DYNAMIC_FTRACE
4873        trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
4874                        &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
4875#endif
4876
4877        create_trace_options_dir();
4878
4879        for_each_tracing_cpu(cpu)
4880                tracing_init_debugfs_percpu(cpu);
4881
4882        return 0;
4883}
4884
4885static int trace_panic_handler(struct notifier_block *this,
4886                               unsigned long event, void *unused)
4887{
4888        if (ftrace_dump_on_oops)
4889                ftrace_dump(ftrace_dump_on_oops);
4890        return NOTIFY_OK;
4891}
4892
4893static struct notifier_block trace_panic_notifier = {
4894        .notifier_call  = trace_panic_handler,
4895        .next           = NULL,
4896        .priority       = 150   /* priority: INT_MAX >= x >= 0 */
4897};
4898
4899static int trace_die_handler(struct notifier_block *self,
4900                             unsigned long val,
4901                             void *data)
4902{
4903        switch (val) {
4904        case DIE_OOPS:
4905                if (ftrace_dump_on_oops)
4906                        ftrace_dump(ftrace_dump_on_oops);
4907                break;
4908        default:
4909                break;
4910        }
4911        return NOTIFY_OK;
4912}
4913
4914static struct notifier_block trace_die_notifier = {
4915        .notifier_call = trace_die_handler,
4916        .priority = 200
4917};
4918
4919/*
4920 * printk is set to max of 1024, we really don't need it that big.
4921 * Nothing should be printing 1000 characters anyway.
4922 */
4923#define TRACE_MAX_PRINT         1000
4924
4925/*
4926 * Define here KERN_TRACE so that we have one place to modify
4927 * it if we decide to change what log level the ftrace dump
4928 * should be at.
4929 */
4930#define KERN_TRACE              KERN_EMERG
4931
4932void
4933trace_printk_seq(struct trace_seq *s)
4934{
4935        /* Probably should print a warning here. */
4936        if (s->len >= 1000)
4937                s->len = 1000;
4938
4939        /* should be zero ended, but we are paranoid. */
4940        s->buffer[s->len] = 0;
4941
4942        printk(KERN_TRACE "%s", s->buffer);
4943
4944        trace_seq_init(s);
4945}
4946
4947void trace_init_global_iter(struct trace_iterator *iter)
4948{
4949        iter->tr = &global_trace;
4950        iter->trace = current_trace;
4951        iter->cpu_file = TRACE_PIPE_ALL_CPU;
4952}
4953
4954static void
4955__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
4956{
4957        static arch_spinlock_t ftrace_dump_lock =
4958                (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
4959        /* use static because iter can be a bit big for the stack */
4960        static struct trace_iterator iter;
4961        unsigned int old_userobj;
4962        static int dump_ran;
4963        unsigned long flags;
4964        int cnt = 0, cpu;
4965
4966        /* only one dump */
4967        local_irq_save(flags);
4968        arch_spin_lock(&ftrace_dump_lock);
4969        if (dump_ran)
4970                goto out;
4971
4972        dump_ran = 1;
4973
4974        tracing_off();
4975
4976        /* Did function tracer already get disabled? */
4977        if (ftrace_is_dead()) {
4978                printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
4979                printk("#          MAY BE MISSING FUNCTION EVENTS\n");
4980        }
4981
4982        if (disable_tracing)
4983                ftrace_kill();
4984
4985        trace_init_global_iter(&iter);
4986
4987        for_each_tracing_cpu(cpu) {
4988                atomic_inc(&iter.tr->data[cpu]->disabled);
4989        }
4990
4991        old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
4992
4993        /* don't look at user memory in panic mode */
4994        trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
4995
4996        /* Simulate the iterator */
4997        iter.tr = &global_trace;
4998        iter.trace = current_trace;
4999
5000        switch (oops_dump_mode) {
5001        case DUMP_ALL:
5002                iter.cpu_file = TRACE_PIPE_ALL_CPU;
5003                break;
5004        case DUMP_ORIG:
5005                iter.cpu_file = raw_smp_processor_id();
5006                break;
5007        case DUMP_NONE:
5008                goto out_enable;
5009        default:
5010                printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
5011                iter.cpu_file = TRACE_PIPE_ALL_CPU;
5012        }
5013
5014        printk(KERN_TRACE "Dumping ftrace buffer:\n");
5015
5016        /*
5017         * We need to stop all tracing on all CPUS to read the
5018         * the next buffer. This is a bit expensive, but is
5019         * not done often. We fill all what we can read,
5020         * and then release the locks again.
5021         */
5022
5023        while (!trace_empty(&iter)) {
5024
5025                if (!cnt)
5026                        printk(KERN_TRACE "---------------------------------\n");
5027
5028                cnt++;
5029
5030                /* reset all but tr, trace, and overruns */
5031                memset(&iter.seq, 0,
5032                       sizeof(struct trace_iterator) -
5033                       offsetof(struct trace_iterator, seq));
5034                iter.iter_flags |= TRACE_FILE_LAT_FMT;
5035                iter.pos = -1;
5036
5037                if (trace_find_next_entry_inc(&iter) != NULL) {
5038                        int ret;
5039
5040                        ret = print_trace_line(&iter);
5041                        if (ret != TRACE_TYPE_NO_CONSUME)
5042                                trace_consume(&iter);
5043                }
5044                touch_nmi_watchdog();
5045
5046                trace_printk_seq(&iter.seq);
5047        }
5048
5049        if (!cnt)
5050                printk(KERN_TRACE "   (ftrace buffer empty)\n");
5051        else
5052                printk(KERN_TRACE "---------------------------------\n");
5053
5054 out_enable:
5055        /* Re-enable tracing if requested */
5056        if (!disable_tracing) {
5057                trace_flags |= old_userobj;
5058
5059                for_each_tracing_cpu(cpu) {
5060                        atomic_dec(&iter.tr->data[cpu]->disabled);
5061                }
5062                tracing_on();
5063        }
5064
5065 out:
5066        arch_spin_unlock(&ftrace_dump_lock);
5067        local_irq_restore(flags);
5068}
5069
5070/* By default: disable tracing after the dump */
5071void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
5072{
5073        __ftrace_dump(true, oops_dump_mode);
5074}
5075EXPORT_SYMBOL_GPL(ftrace_dump);
5076
5077__init static int tracer_alloc_buffers(void)
5078{
5079        int ring_buf_size;
5080        enum ring_buffer_flags rb_flags;
5081        int i;
5082        int ret = -ENOMEM;
5083
5084
5085        if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
5086                goto out;
5087
5088        if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
5089                goto out_free_buffer_mask;
5090
5091        /* Only allocate trace_printk buffers if a trace_printk exists */
5092        if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
5093                trace_printk_init_buffers();
5094
5095        /* To save memory, keep the ring buffer size to its minimum */
5096        if (ring_buffer_expanded)
5097                ring_buf_size = trace_buf_size;
5098        else
5099                ring_buf_size = 1;
5100
5101        rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5102
5103        cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
5104        cpumask_copy(tracing_cpumask, cpu_all_mask);
5105
5106        /* TODO: make the number of buffers hot pluggable with CPUS */
5107        global_trace.buffer = ring_buffer_alloc(ring_buf_size, rb_flags);
5108        if (!global_trace.buffer) {
5109                printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
5110                WARN_ON(1);
5111                goto out_free_cpumask;
5112        }
5113        if (global_trace.buffer_disabled)
5114                tracing_off();
5115
5116
5117#ifdef CONFIG_TRACER_MAX_TRACE
5118        max_tr.buffer = ring_buffer_alloc(1, rb_flags);
5119        if (!max_tr.buffer) {
5120                printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
5121                WARN_ON(1);
5122                ring_buffer_free(global_trace.buffer);
5123                goto out_free_cpumask;
5124        }
5125#endif
5126
5127        /* Allocate the first page for all buffers */
5128        for_each_tracing_cpu(i) {
5129                global_trace.data[i] = &per_cpu(global_trace_cpu, i);
5130                max_tr.data[i] = &per_cpu(max_tr_data, i);
5131        }
5132
5133        set_buffer_entries(&global_trace,
5134                           ring_buffer_size(global_trace.buffer, 0));
5135#ifdef CONFIG_TRACER_MAX_TRACE
5136        set_buffer_entries(&max_tr, 1);
5137#endif
5138
5139        trace_init_cmdlines();
5140
5141        register_tracer(&nop_trace);
5142        current_trace = &nop_trace;
5143        /* All seems OK, enable tracing */
5144        tracing_disabled = 0;
5145
5146        atomic_notifier_chain_register(&panic_notifier_list,
5147                                       &trace_panic_notifier);
5148
5149        register_die_notifier(&trace_die_notifier);
5150
5151        return 0;
5152
5153out_free_cpumask:
5154        free_cpumask_var(tracing_cpumask);
5155out_free_buffer_mask:
5156        free_cpumask_var(tracing_buffer_mask);
5157out:
5158        return ret;
5159}
5160
5161__init static int clear_boot_tracer(void)
5162{
5163        /*
5164         * The default tracer at boot buffer is an init section.
5165         * This function is called in lateinit. If we did not
5166         * find the boot tracer, then clear it out, to prevent
5167         * later registration from accessing the buffer that is
5168         * about to be freed.
5169         */
5170        if (!default_bootup_tracer)
5171                return 0;
5172
5173        printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
5174               default_bootup_tracer);
5175        default_bootup_tracer = NULL;
5176
5177        return 0;
5178}
5179
5180early_initcall(tracer_alloc_buffers);
5181fs_initcall(tracer_init_debugfs);
5182late_initcall(clear_boot_tracer);
5183
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.