linux/kernel/trace/trace_irqsoff.c
<<
>>
Prefs
   1/*
   2 * trace irqs off critical timings
   3 *
   4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
   6 *
   7 * From code in the latency_tracer, that is:
   8 *
   9 *  Copyright (C) 2004-2006 Ingo Molnar
  10 *  Copyright (C) 2004 William Lee Irwin III
  11 */
  12#include <linux/kallsyms.h>
  13#include <linux/debugfs.h>
  14#include <linux/uaccess.h>
  15#include <linux/module.h>
  16#include <linux/ftrace.h>
  17#include <linux/fs.h>
  18
  19#include "trace.h"
  20
  21static struct trace_array               *irqsoff_trace __read_mostly;
  22static int                              tracer_enabled __read_mostly;
  23
  24static DEFINE_PER_CPU(int, tracing_cpu);
  25
  26static DEFINE_RAW_SPINLOCK(max_trace_lock);
  27
  28enum {
  29        TRACER_IRQS_OFF         = (1 << 1),
  30        TRACER_PREEMPT_OFF      = (1 << 2),
  31};
  32
  33static int trace_type __read_mostly;
  34
  35static int save_lat_flag;
  36
  37static void stop_irqsoff_tracer(struct trace_array *tr, int graph);
  38static int start_irqsoff_tracer(struct trace_array *tr, int graph);
  39
  40#ifdef CONFIG_PREEMPT_TRACER
  41static inline int
  42preempt_trace(void)
  43{
  44        return ((trace_type & TRACER_PREEMPT_OFF) && preempt_count());
  45}
  46#else
  47# define preempt_trace() (0)
  48#endif
  49
  50#ifdef CONFIG_IRQSOFF_TRACER
  51static inline int
  52irq_trace(void)
  53{
  54        return ((trace_type & TRACER_IRQS_OFF) &&
  55                irqs_disabled());
  56}
  57#else
  58# define irq_trace() (0)
  59#endif
  60
  61#define TRACE_DISPLAY_GRAPH     1
  62
  63static struct tracer_opt trace_opts[] = {
  64#ifdef CONFIG_FUNCTION_GRAPH_TRACER
  65        /* display latency trace as call graph */
  66        { TRACER_OPT(display-graph, TRACE_DISPLAY_GRAPH) },
  67#endif
  68        { } /* Empty entry */
  69};
  70
  71static struct tracer_flags tracer_flags = {
  72        .val  = 0,
  73        .opts = trace_opts,
  74};
  75
  76#define is_graph() (tracer_flags.val & TRACE_DISPLAY_GRAPH)
  77
  78/*
  79 * Sequence count - we record it when starting a measurement and
  80 * skip the latency if the sequence has changed - some other section
  81 * did a maximum and could disturb our measurement with serial console
  82 * printouts, etc. Truly coinciding maximum latencies should be rare
  83 * and what happens together happens separately as well, so this doesn't
  84 * decrease the validity of the maximum found:
  85 */
  86static __cacheline_aligned_in_smp       unsigned long max_sequence;
  87
  88#ifdef CONFIG_FUNCTION_TRACER
  89/*
  90 * Prologue for the preempt and irqs off function tracers.
  91 *
  92 * Returns 1 if it is OK to continue, and data->disabled is
  93 *            incremented.
  94 *         0 if the trace is to be ignored, and data->disabled
  95 *            is kept the same.
  96 *
  97 * Note, this function is also used outside this ifdef but
  98 *  inside the #ifdef of the function graph tracer below.
  99 *  This is OK, since the function graph tracer is
 100 *  dependent on the function tracer.
 101 */
 102static int func_prolog_dec(struct trace_array *tr,
 103                           struct trace_array_cpu **data,
 104                           unsigned long *flags)
 105{
 106        long disabled;
 107        int cpu;
 108
 109        /*
 110         * Does not matter if we preempt. We test the flags
 111         * afterward, to see if irqs are disabled or not.
 112         * If we preempt and get a false positive, the flags
 113         * test will fail.
 114         */
 115        cpu = raw_smp_processor_id();
 116        if (likely(!per_cpu(tracing_cpu, cpu)))
 117                return 0;
 118
 119        local_save_flags(*flags);
 120        /* slight chance to get a false positive on tracing_cpu */
 121        if (!irqs_disabled_flags(*flags))
 122                return 0;
 123
 124        *data = tr->data[cpu];
 125        disabled = atomic_inc_return(&(*data)->disabled);
 126
 127        if (likely(disabled == 1))
 128                return 1;
 129
 130        atomic_dec(&(*data)->disabled);
 131
 132        return 0;
 133}
 134
 135/*
 136 * irqsoff uses its own tracer function to keep the overhead down:
 137 */
 138static void
 139irqsoff_tracer_call(unsigned long ip, unsigned long parent_ip)
 140{
 141        struct trace_array *tr = irqsoff_trace;
 142        struct trace_array_cpu *data;
 143        unsigned long flags;
 144
 145        if (!func_prolog_dec(tr, &data, &flags))
 146                return;
 147
 148        trace_function(tr, ip, parent_ip, flags, preempt_count());
 149
 150        atomic_dec(&data->disabled);
 151}
 152
 153static struct ftrace_ops trace_ops __read_mostly =
 154{
 155        .func = irqsoff_tracer_call,
 156        .flags = FTRACE_OPS_FL_GLOBAL,
 157};
 158#endif /* CONFIG_FUNCTION_TRACER */
 159
 160#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 161static int irqsoff_set_flag(u32 old_flags, u32 bit, int set)
 162{
 163        int cpu;
 164
 165        if (!(bit & TRACE_DISPLAY_GRAPH))
 166                return -EINVAL;
 167
 168        if (!(is_graph() ^ set))
 169                return 0;
 170
 171        stop_irqsoff_tracer(irqsoff_trace, !set);
 172
 173        for_each_possible_cpu(cpu)
 174                per_cpu(tracing_cpu, cpu) = 0;
 175
 176        tracing_max_latency = 0;
 177        tracing_reset_online_cpus(irqsoff_trace);
 178
 179        return start_irqsoff_tracer(irqsoff_trace, set);
 180}
 181
 182static int irqsoff_graph_entry(struct ftrace_graph_ent *trace)
 183{
 184        struct trace_array *tr = irqsoff_trace;
 185        struct trace_array_cpu *data;
 186        unsigned long flags;
 187        int ret;
 188        int pc;
 189
 190        if (!func_prolog_dec(tr, &data, &flags))
 191                return 0;
 192
 193        pc = preempt_count();
 194        ret = __trace_graph_entry(tr, trace, flags, pc);
 195        atomic_dec(&data->disabled);
 196
 197        return ret;
 198}
 199
 200static void irqsoff_graph_return(struct ftrace_graph_ret *trace)
 201{
 202        struct trace_array *tr = irqsoff_trace;
 203        struct trace_array_cpu *data;
 204        unsigned long flags;
 205        int pc;
 206
 207        if (!func_prolog_dec(tr, &data, &flags))
 208                return;
 209
 210        pc = preempt_count();
 211        __trace_graph_return(tr, trace, flags, pc);
 212        atomic_dec(&data->disabled);
 213}
 214
 215static void irqsoff_trace_open(struct trace_iterator *iter)
 216{
 217        if (is_graph())
 218                graph_trace_open(iter);
 219
 220}
 221
 222static void irqsoff_trace_close(struct trace_iterator *iter)
 223{
 224        if (iter->private)
 225                graph_trace_close(iter);
 226}
 227
 228#define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_CPU | \
 229                            TRACE_GRAPH_PRINT_PROC | \
 230                            TRACE_GRAPH_PRINT_ABS_TIME | \
 231                            TRACE_GRAPH_PRINT_DURATION)
 232
 233static enum print_line_t irqsoff_print_line(struct trace_iterator *iter)
 234{
 235        /*
 236         * In graph mode call the graph tracer output function,
 237         * otherwise go with the TRACE_FN event handler
 238         */
 239        if (is_graph())
 240                return print_graph_function_flags(iter, GRAPH_TRACER_FLAGS);
 241
 242        return TRACE_TYPE_UNHANDLED;
 243}
 244
 245static void irqsoff_print_header(struct seq_file *s)
 246{
 247        if (is_graph())
 248                print_graph_headers_flags(s, GRAPH_TRACER_FLAGS);
 249        else
 250                trace_default_header(s);
 251}
 252
 253static void
 254__trace_function(struct trace_array *tr,
 255                 unsigned long ip, unsigned long parent_ip,
 256                 unsigned long flags, int pc)
 257{
 258        if (is_graph())
 259                trace_graph_function(tr, ip, parent_ip, flags, pc);
 260        else
 261                trace_function(tr, ip, parent_ip, flags, pc);
 262}
 263
 264#else
 265#define __trace_function trace_function
 266
 267static int irqsoff_set_flag(u32 old_flags, u32 bit, int set)
 268{
 269        return -EINVAL;
 270}
 271
 272static int irqsoff_graph_entry(struct ftrace_graph_ent *trace)
 273{
 274        return -1;
 275}
 276
 277static enum print_line_t irqsoff_print_line(struct trace_iterator *iter)
 278{
 279        return TRACE_TYPE_UNHANDLED;
 280}
 281
 282static void irqsoff_graph_return(struct ftrace_graph_ret *trace) { }
 283static void irqsoff_trace_open(struct trace_iterator *iter) { }
 284static void irqsoff_trace_close(struct trace_iterator *iter) { }
 285
 286#ifdef CONFIG_FUNCTION_TRACER
 287static void irqsoff_print_header(struct seq_file *s)
 288{
 289        trace_default_header(s);
 290}
 291#else
 292static void irqsoff_print_header(struct seq_file *s)
 293{
 294        trace_latency_header(s);
 295}
 296#endif /* CONFIG_FUNCTION_TRACER */
 297#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 298
 299/*
 300 * Should this new latency be reported/recorded?
 301 */
 302static int report_latency(cycle_t delta)
 303{
 304        if (tracing_thresh) {
 305                if (delta < tracing_thresh)
 306                        return 0;
 307        } else {
 308                if (delta <= tracing_max_latency)
 309                        return 0;
 310        }
 311        return 1;
 312}
 313
 314static void
 315check_critical_timing(struct trace_array *tr,
 316                      struct trace_array_cpu *data,
 317                      unsigned long parent_ip,
 318                      int cpu)
 319{
 320        cycle_t T0, T1, delta;
 321        unsigned long flags;
 322        int pc;
 323
 324        T0 = data->preempt_timestamp;
 325        T1 = ftrace_now(cpu);
 326        delta = T1-T0;
 327
 328        local_save_flags(flags);
 329
 330        pc = preempt_count();
 331
 332        if (!report_latency(delta))
 333                goto out;
 334
 335        raw_spin_lock_irqsave(&max_trace_lock, flags);
 336
 337        /* check if we are still the max latency */
 338        if (!report_latency(delta))
 339                goto out_unlock;
 340
 341        __trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc);
 342        /* Skip 5 functions to get to the irq/preempt enable function */
 343        __trace_stack(tr, flags, 5, pc);
 344
 345        if (data->critical_sequence != max_sequence)
 346                goto out_unlock;
 347
 348        data->critical_end = parent_ip;
 349
 350        if (likely(!is_tracing_stopped())) {
 351                tracing_max_latency = delta;
 352                update_max_tr_single(tr, current, cpu);
 353        }
 354
 355        max_sequence++;
 356
 357out_unlock:
 358        raw_spin_unlock_irqrestore(&max_trace_lock, flags);
 359
 360out:
 361        data->critical_sequence = max_sequence;
 362        data->preempt_timestamp = ftrace_now(cpu);
 363        __trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc);
 364}
 365
 366static inline void
 367start_critical_timing(unsigned long ip, unsigned long parent_ip)
 368{
 369        int cpu;
 370        struct trace_array *tr = irqsoff_trace;
 371        struct trace_array_cpu *data;
 372        unsigned long flags;
 373
 374        if (likely(!tracer_enabled))
 375                return;
 376
 377        cpu = raw_smp_processor_id();
 378
 379        if (per_cpu(tracing_cpu, cpu))
 380                return;
 381
 382        data = tr->data[cpu];
 383
 384        if (unlikely(!data) || atomic_read(&data->disabled))
 385                return;
 386
 387        atomic_inc(&data->disabled);
 388
 389        data->critical_sequence = max_sequence;
 390        data->preempt_timestamp = ftrace_now(cpu);
 391        data->critical_start = parent_ip ? : ip;
 392
 393        local_save_flags(flags);
 394
 395        __trace_function(tr, ip, parent_ip, flags, preempt_count());
 396
 397        per_cpu(tracing_cpu, cpu) = 1;
 398
 399        atomic_dec(&data->disabled);
 400}
 401
 402static inline void
 403stop_critical_timing(unsigned long ip, unsigned long parent_ip)
 404{
 405        int cpu;
 406        struct trace_array *tr = irqsoff_trace;
 407        struct trace_array_cpu *data;
 408        unsigned long flags;
 409
 410        cpu = raw_smp_processor_id();
 411        /* Always clear the tracing cpu on stopping the trace */
 412        if (unlikely(per_cpu(tracing_cpu, cpu)))
 413                per_cpu(tracing_cpu, cpu) = 0;
 414        else
 415                return;
 416
 417        if (!tracer_enabled)
 418                return;
 419
 420        data = tr->data[cpu];
 421
 422        if (unlikely(!data) ||
 423            !data->critical_start || atomic_read(&data->disabled))
 424                return;
 425
 426        atomic_inc(&data->disabled);
 427
 428        local_save_flags(flags);
 429        __trace_function(tr, ip, parent_ip, flags, preempt_count());
 430        check_critical_timing(tr, data, parent_ip ? : ip, cpu);
 431        data->critical_start = 0;
 432        atomic_dec(&data->disabled);
 433}
 434
 435/* start and stop critical timings used to for stoppage (in idle) */
 436void start_critical_timings(void)
 437{
 438        if (preempt_trace() || irq_trace())
 439                start_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
 440}
 441EXPORT_SYMBOL_GPL(start_critical_timings);
 442
 443void stop_critical_timings(void)
 444{
 445        if (preempt_trace() || irq_trace())
 446                stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
 447}
 448EXPORT_SYMBOL_GPL(stop_critical_timings);
 449
 450#ifdef CONFIG_IRQSOFF_TRACER
 451#ifdef CONFIG_PROVE_LOCKING
 452void time_hardirqs_on(unsigned long a0, unsigned long a1)
 453{
 454        if (!preempt_trace() && irq_trace())
 455                stop_critical_timing(a0, a1);
 456}
 457
 458void time_hardirqs_off(unsigned long a0, unsigned long a1)
 459{
 460        if (!preempt_trace() && irq_trace())
 461                start_critical_timing(a0, a1);
 462}
 463
 464#else /* !CONFIG_PROVE_LOCKING */
 465
 466/*
 467 * Stubs:
 468 */
 469
 470void trace_softirqs_on(unsigned long ip)
 471{
 472}
 473
 474void trace_softirqs_off(unsigned long ip)
 475{
 476}
 477
 478inline void print_irqtrace_events(struct task_struct *curr)
 479{
 480}
 481
 482/*
 483 * We are only interested in hardirq on/off events:
 484 */
 485void trace_hardirqs_on(void)
 486{
 487        if (!preempt_trace() && irq_trace())
 488                stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
 489}
 490EXPORT_SYMBOL(trace_hardirqs_on);
 491
 492void trace_hardirqs_off(void)
 493{
 494        if (!preempt_trace() && irq_trace())
 495                start_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
 496}
 497EXPORT_SYMBOL(trace_hardirqs_off);
 498
 499void trace_hardirqs_on_caller(unsigned long caller_addr)
 500{
 501        if (!preempt_trace() && irq_trace())
 502                stop_critical_timing(CALLER_ADDR0, caller_addr);
 503}
 504EXPORT_SYMBOL(trace_hardirqs_on_caller);
 505
 506void trace_hardirqs_off_caller(unsigned long caller_addr)
 507{
 508        if (!preempt_trace() && irq_trace())
 509                start_critical_timing(CALLER_ADDR0, caller_addr);
 510}
 511EXPORT_SYMBOL(trace_hardirqs_off_caller);
 512
 513#endif /* CONFIG_PROVE_LOCKING */
 514#endif /*  CONFIG_IRQSOFF_TRACER */
 515
 516#ifdef CONFIG_PREEMPT_TRACER
 517void trace_preempt_on(unsigned long a0, unsigned long a1)
 518{
 519        if (preempt_trace() && !irq_trace())
 520                stop_critical_timing(a0, a1);
 521}
 522
 523void trace_preempt_off(unsigned long a0, unsigned long a1)
 524{
 525        if (preempt_trace() && !irq_trace())
 526                start_critical_timing(a0, a1);
 527}
 528#endif /* CONFIG_PREEMPT_TRACER */
 529
 530static int start_irqsoff_tracer(struct trace_array *tr, int graph)
 531{
 532        int ret = 0;
 533
 534        if (!graph)
 535                ret = register_ftrace_function(&trace_ops);
 536        else
 537                ret = register_ftrace_graph(&irqsoff_graph_return,
 538                                            &irqsoff_graph_entry);
 539
 540        if (!ret && tracing_is_enabled())
 541                tracer_enabled = 1;
 542        else
 543                tracer_enabled = 0;
 544
 545        return ret;
 546}
 547
 548static void stop_irqsoff_tracer(struct trace_array *tr, int graph)
 549{
 550        tracer_enabled = 0;
 551
 552        if (!graph)
 553                unregister_ftrace_function(&trace_ops);
 554        else
 555                unregister_ftrace_graph();
 556}
 557
 558static void __irqsoff_tracer_init(struct trace_array *tr)
 559{
 560        save_lat_flag = trace_flags & TRACE_ITER_LATENCY_FMT;
 561        trace_flags |= TRACE_ITER_LATENCY_FMT;
 562
 563        tracing_max_latency = 0;
 564        irqsoff_trace = tr;
 565        /* make sure that the tracer is visible */
 566        smp_wmb();
 567        tracing_reset_online_cpus(tr);
 568
 569        if (start_irqsoff_tracer(tr, is_graph()))
 570                printk(KERN_ERR "failed to start irqsoff tracer\n");
 571}
 572
 573static void irqsoff_tracer_reset(struct trace_array *tr)
 574{
 575        stop_irqsoff_tracer(tr, is_graph());
 576
 577        if (!save_lat_flag)
 578                trace_flags &= ~TRACE_ITER_LATENCY_FMT;
 579}
 580
 581static void irqsoff_tracer_start(struct trace_array *tr)
 582{
 583        tracer_enabled = 1;
 584}
 585
 586static void irqsoff_tracer_stop(struct trace_array *tr)
 587{
 588        tracer_enabled = 0;
 589}
 590
 591#ifdef CONFIG_IRQSOFF_TRACER
 592static int irqsoff_tracer_init(struct trace_array *tr)
 593{
 594        trace_type = TRACER_IRQS_OFF;
 595
 596        __irqsoff_tracer_init(tr);
 597        return 0;
 598}
 599static struct tracer irqsoff_tracer __read_mostly =
 600{
 601        .name           = "irqsoff",
 602        .init           = irqsoff_tracer_init,
 603        .reset          = irqsoff_tracer_reset,
 604        .start          = irqsoff_tracer_start,
 605        .stop           = irqsoff_tracer_stop,
 606        .print_max      = 1,
 607        .print_header   = irqsoff_print_header,
 608        .print_line     = irqsoff_print_line,
 609        .flags          = &tracer_flags,
 610        .set_flag       = irqsoff_set_flag,
 611#ifdef CONFIG_FTRACE_SELFTEST
 612        .selftest    = trace_selftest_startup_irqsoff,
 613#endif
 614        .open           = irqsoff_trace_open,
 615        .close          = irqsoff_trace_close,
 616        .use_max_tr     = 1,
 617};
 618# define register_irqsoff(trace) register_tracer(&trace)
 619#else
 620# define register_irqsoff(trace) do { } while (0)
 621#endif
 622
 623#ifdef CONFIG_PREEMPT_TRACER
 624static int preemptoff_tracer_init(struct trace_array *tr)
 625{
 626        trace_type = TRACER_PREEMPT_OFF;
 627
 628        __irqsoff_tracer_init(tr);
 629        return 0;
 630}
 631
 632static struct tracer preemptoff_tracer __read_mostly =
 633{
 634        .name           = "preemptoff",
 635        .init           = preemptoff_tracer_init,
 636        .reset          = irqsoff_tracer_reset,
 637        .start          = irqsoff_tracer_start,
 638        .stop           = irqsoff_tracer_stop,
 639        .print_max      = 1,
 640        .print_header   = irqsoff_print_header,
 641        .print_line     = irqsoff_print_line,
 642        .flags          = &tracer_flags,
 643        .set_flag       = irqsoff_set_flag,
 644#ifdef CONFIG_FTRACE_SELFTEST
 645        .selftest    = trace_selftest_startup_preemptoff,
 646#endif
 647        .open           = irqsoff_trace_open,
 648        .close          = irqsoff_trace_close,
 649        .use_max_tr     = 1,
 650};
 651# define register_preemptoff(trace) register_tracer(&trace)
 652#else
 653# define register_preemptoff(trace) do { } while (0)
 654#endif
 655
 656#if defined(CONFIG_IRQSOFF_TRACER) && \
 657        defined(CONFIG_PREEMPT_TRACER)
 658
 659static int preemptirqsoff_tracer_init(struct trace_array *tr)
 660{
 661        trace_type = TRACER_IRQS_OFF | TRACER_PREEMPT_OFF;
 662
 663        __irqsoff_tracer_init(tr);
 664        return 0;
 665}
 666
 667static struct tracer preemptirqsoff_tracer __read_mostly =
 668{
 669        .name           = "preemptirqsoff",
 670        .init           = preemptirqsoff_tracer_init,
 671        .reset          = irqsoff_tracer_reset,
 672        .start          = irqsoff_tracer_start,
 673        .stop           = irqsoff_tracer_stop,
 674        .print_max      = 1,
 675        .print_header   = irqsoff_print_header,
 676        .print_line     = irqsoff_print_line,
 677        .flags          = &tracer_flags,
 678        .set_flag       = irqsoff_set_flag,
 679#ifdef CONFIG_FTRACE_SELFTEST
 680        .selftest    = trace_selftest_startup_preemptirqsoff,
 681#endif
 682        .open           = irqsoff_trace_open,
 683        .close          = irqsoff_trace_close,
 684        .use_max_tr     = 1,
 685};
 686
 687# define register_preemptirqsoff(trace) register_tracer(&trace)
 688#else
 689# define register_preemptirqsoff(trace) do { } while (0)
 690#endif
 691
 692__init static int init_irqsoff_tracer(void)
 693{
 694        register_irqsoff(irqsoff_tracer);
 695        register_preemptoff(preemptoff_tracer);
 696        register_preemptirqsoff(preemptirqsoff_tracer);
 697
 698        return 0;
 699}
 700device_initcall(init_irqsoff_tracer);
 701
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.