linux/drivers/hwtracing/coresight/coresight-tmc-etf.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright(C) 2016 Linaro Limited. All rights reserved.
   4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
   5 */
   6
   7#include <linux/atomic.h>
   8#include <linux/circ_buf.h>
   9#include <linux/coresight.h>
  10#include <linux/perf_event.h>
  11#include <linux/slab.h>
  12#include "coresight-priv.h"
  13#include "coresight-tmc.h"
  14#include "coresight-etm-perf.h"
  15
  16static int tmc_set_etf_buffer(struct coresight_device *csdev,
  17                              struct perf_output_handle *handle);
  18
  19static void __tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
  20{
  21        CS_UNLOCK(drvdata->base);
  22
  23        /* Wait for TMCSReady bit to be set */
  24        tmc_wait_for_tmcready(drvdata);
  25
  26        writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
  27        writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
  28                       TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
  29                       TMC_FFCR_TRIGON_TRIGIN,
  30                       drvdata->base + TMC_FFCR);
  31
  32        writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
  33        tmc_enable_hw(drvdata);
  34
  35        CS_LOCK(drvdata->base);
  36}
  37
  38static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
  39{
  40        int rc = coresight_claim_device(drvdata->csdev);
  41
  42        if (rc)
  43                return rc;
  44
  45        __tmc_etb_enable_hw(drvdata);
  46        return 0;
  47}
  48
  49static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata)
  50{
  51        char *bufp;
  52        u32 read_data, lost;
  53
  54        /* Check if the buffer wrapped around. */
  55        lost = readl_relaxed(drvdata->base + TMC_STS) & TMC_STS_FULL;
  56        bufp = drvdata->buf;
  57        drvdata->len = 0;
  58        while (1) {
  59                read_data = readl_relaxed(drvdata->base + TMC_RRD);
  60                if (read_data == 0xFFFFFFFF)
  61                        break;
  62                memcpy(bufp, &read_data, 4);
  63                bufp += 4;
  64                drvdata->len += 4;
  65        }
  66
  67        if (lost)
  68                coresight_insert_barrier_packet(drvdata->buf);
  69        return;
  70}
  71
  72static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
  73{
  74        CS_UNLOCK(drvdata->base);
  75
  76        tmc_flush_and_stop(drvdata);
  77        /*
  78         * When operating in sysFS mode the content of the buffer needs to be
  79         * read before the TMC is disabled.
  80         */
  81        if (drvdata->mode == CS_MODE_SYSFS)
  82                tmc_etb_dump_hw(drvdata);
  83        tmc_disable_hw(drvdata);
  84
  85        CS_LOCK(drvdata->base);
  86}
  87
  88static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
  89{
  90        __tmc_etb_disable_hw(drvdata);
  91        coresight_disclaim_device(drvdata->csdev);
  92}
  93
  94static void __tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
  95{
  96        CS_UNLOCK(drvdata->base);
  97
  98        /* Wait for TMCSReady bit to be set */
  99        tmc_wait_for_tmcready(drvdata);
 100
 101        writel_relaxed(TMC_MODE_HARDWARE_FIFO, drvdata->base + TMC_MODE);
 102        writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI,
 103                       drvdata->base + TMC_FFCR);
 104        writel_relaxed(0x0, drvdata->base + TMC_BUFWM);
 105        tmc_enable_hw(drvdata);
 106
 107        CS_LOCK(drvdata->base);
 108}
 109
 110static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
 111{
 112        int rc = coresight_claim_device(drvdata->csdev);
 113
 114        if (rc)
 115                return rc;
 116
 117        __tmc_etf_enable_hw(drvdata);
 118        return 0;
 119}
 120
 121static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata)
 122{
 123        struct coresight_device *csdev = drvdata->csdev;
 124
 125        CS_UNLOCK(drvdata->base);
 126
 127        tmc_flush_and_stop(drvdata);
 128        tmc_disable_hw(drvdata);
 129        coresight_disclaim_device_unlocked(csdev);
 130        CS_LOCK(drvdata->base);
 131}
 132
 133/*
 134 * Return the available trace data in the buffer from @pos, with
 135 * a maximum limit of @len, updating the @bufpp on where to
 136 * find it.
 137 */
 138ssize_t tmc_etb_get_sysfs_trace(struct tmc_drvdata *drvdata,
 139                                loff_t pos, size_t len, char **bufpp)
 140{
 141        ssize_t actual = len;
 142
 143        /* Adjust the len to available size @pos */
 144        if (pos + actual > drvdata->len)
 145                actual = drvdata->len - pos;
 146        if (actual > 0)
 147                *bufpp = drvdata->buf + pos;
 148        return actual;
 149}
 150
 151static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev)
 152{
 153        int ret = 0;
 154        bool used = false;
 155        char *buf = NULL;
 156        unsigned long flags;
 157        struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 158
 159        /*
 160         * If we don't have a buffer release the lock and allocate memory.
 161         * Otherwise keep the lock and move along.
 162         */
 163        spin_lock_irqsave(&drvdata->spinlock, flags);
 164        if (!drvdata->buf) {
 165                spin_unlock_irqrestore(&drvdata->spinlock, flags);
 166
 167                /* Allocating the memory here while outside of the spinlock */
 168                buf = kzalloc(drvdata->size, GFP_KERNEL);
 169                if (!buf)
 170                        return -ENOMEM;
 171
 172                /* Let's try again */
 173                spin_lock_irqsave(&drvdata->spinlock, flags);
 174        }
 175
 176        if (drvdata->reading) {
 177                ret = -EBUSY;
 178                goto out;
 179        }
 180
 181        /*
 182         * In sysFS mode we can have multiple writers per sink.  Since this
 183         * sink is already enabled no memory is needed and the HW need not be
 184         * touched.
 185         */
 186        if (drvdata->mode == CS_MODE_SYSFS) {
 187                atomic_inc(csdev->refcnt);
 188                goto out;
 189        }
 190
 191        /*
 192         * If drvdata::buf isn't NULL, memory was allocated for a previous
 193         * trace run but wasn't read.  If so simply zero-out the memory.
 194         * Otherwise use the memory allocated above.
 195         *
 196         * The memory is freed when users read the buffer using the
 197         * /dev/xyz.{etf|etb} interface.  See tmc_read_unprepare_etf() for
 198         * details.
 199         */
 200        if (drvdata->buf) {
 201                memset(drvdata->buf, 0, drvdata->size);
 202        } else {
 203                used = true;
 204                drvdata->buf = buf;
 205        }
 206
 207        ret = tmc_etb_enable_hw(drvdata);
 208        if (!ret) {
 209                drvdata->mode = CS_MODE_SYSFS;
 210                atomic_inc(csdev->refcnt);
 211        } else {
 212                /* Free up the buffer if we failed to enable */
 213                used = false;
 214        }
 215out:
 216        spin_unlock_irqrestore(&drvdata->spinlock, flags);
 217
 218        /* Free memory outside the spinlock if need be */
 219        if (!used)
 220                kfree(buf);
 221
 222        return ret;
 223}
 224
 225static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, void *data)
 226{
 227        int ret = 0;
 228        pid_t pid;
 229        unsigned long flags;
 230        struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 231        struct perf_output_handle *handle = data;
 232        struct cs_buffers *buf = etm_perf_sink_config(handle);
 233
 234        spin_lock_irqsave(&drvdata->spinlock, flags);
 235        do {
 236                ret = -EINVAL;
 237                if (drvdata->reading)
 238                        break;
 239                /*
 240                 * No need to continue if the ETB/ETF is already operated
 241                 * from sysFS.
 242                 */
 243                if (drvdata->mode == CS_MODE_SYSFS) {
 244                        ret = -EBUSY;
 245                        break;
 246                }
 247
 248                /* Get a handle on the pid of the process to monitor */
 249                pid = buf->pid;
 250
 251                if (drvdata->pid != -1 && drvdata->pid != pid) {
 252                        ret = -EBUSY;
 253                        break;
 254                }
 255
 256                ret = tmc_set_etf_buffer(csdev, handle);
 257                if (ret)
 258                        break;
 259
 260                /*
 261                 * No HW configuration is needed if the sink is already in
 262                 * use for this session.
 263                 */
 264                if (drvdata->pid == pid) {
 265                        atomic_inc(csdev->refcnt);
 266                        break;
 267                }
 268
 269                ret  = tmc_etb_enable_hw(drvdata);
 270                if (!ret) {
 271                        /* Associate with monitored process. */
 272                        drvdata->pid = pid;
 273                        drvdata->mode = CS_MODE_PERF;
 274                        atomic_inc(csdev->refcnt);
 275                }
 276        } while (0);
 277        spin_unlock_irqrestore(&drvdata->spinlock, flags);
 278
 279        return ret;
 280}
 281
 282static int tmc_enable_etf_sink(struct coresight_device *csdev,
 283                               u32 mode, void *data)
 284{
 285        int ret;
 286
 287        switch (mode) {
 288        case CS_MODE_SYSFS:
 289                ret = tmc_enable_etf_sink_sysfs(csdev);
 290                break;
 291        case CS_MODE_PERF:
 292                ret = tmc_enable_etf_sink_perf(csdev, data);
 293                break;
 294        /* We shouldn't be here */
 295        default:
 296                ret = -EINVAL;
 297                break;
 298        }
 299
 300        if (ret)
 301                return ret;
 302
 303        dev_dbg(&csdev->dev, "TMC-ETB/ETF enabled\n");
 304        return 0;
 305}
 306
 307static int tmc_disable_etf_sink(struct coresight_device *csdev)
 308{
 309        unsigned long flags;
 310        struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 311
 312        spin_lock_irqsave(&drvdata->spinlock, flags);
 313
 314        if (drvdata->reading) {
 315                spin_unlock_irqrestore(&drvdata->spinlock, flags);
 316                return -EBUSY;
 317        }
 318
 319        if (atomic_dec_return(csdev->refcnt)) {
 320                spin_unlock_irqrestore(&drvdata->spinlock, flags);
 321                return -EBUSY;
 322        }
 323
 324        /* Complain if we (somehow) got out of sync */
 325        WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED);
 326        tmc_etb_disable_hw(drvdata);
 327        /* Dissociate from monitored process. */
 328        drvdata->pid = -1;
 329        drvdata->mode = CS_MODE_DISABLED;
 330
 331        spin_unlock_irqrestore(&drvdata->spinlock, flags);
 332
 333        dev_dbg(&csdev->dev, "TMC-ETB/ETF disabled\n");
 334        return 0;
 335}
 336
 337static int tmc_enable_etf_link(struct coresight_device *csdev,
 338                               int inport, int outport)
 339{
 340        int ret = 0;
 341        unsigned long flags;
 342        struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 343        bool first_enable = false;
 344
 345        spin_lock_irqsave(&drvdata->spinlock, flags);
 346        if (drvdata->reading) {
 347                spin_unlock_irqrestore(&drvdata->spinlock, flags);
 348                return -EBUSY;
 349        }
 350
 351        if (atomic_read(&csdev->refcnt[0]) == 0) {
 352                ret = tmc_etf_enable_hw(drvdata);
 353                if (!ret) {
 354                        drvdata->mode = CS_MODE_SYSFS;
 355                        first_enable = true;
 356                }
 357        }
 358        if (!ret)
 359                atomic_inc(&csdev->refcnt[0]);
 360        spin_unlock_irqrestore(&drvdata->spinlock, flags);
 361
 362        if (first_enable)
 363                dev_dbg(&csdev->dev, "TMC-ETF enabled\n");
 364        return ret;
 365}
 366
 367static void tmc_disable_etf_link(struct coresight_device *csdev,
 368                                 int inport, int outport)
 369{
 370        unsigned long flags;
 371        struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 372        bool last_disable = false;
 373
 374        spin_lock_irqsave(&drvdata->spinlock, flags);
 375        if (drvdata->reading) {
 376                spin_unlock_irqrestore(&drvdata->spinlock, flags);
 377                return;
 378        }
 379
 380        if (atomic_dec_return(&csdev->refcnt[0]) == 0) {
 381                tmc_etf_disable_hw(drvdata);
 382                drvdata->mode = CS_MODE_DISABLED;
 383                last_disable = true;
 384        }
 385        spin_unlock_irqrestore(&drvdata->spinlock, flags);
 386
 387        if (last_disable)
 388                dev_dbg(&csdev->dev, "TMC-ETF disabled\n");
 389}
 390
 391static void *tmc_alloc_etf_buffer(struct coresight_device *csdev,
 392                                  struct perf_event *event, void **pages,
 393                                  int nr_pages, bool overwrite)
 394{
 395        int node;
 396        struct cs_buffers *buf;
 397
 398        node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
 399
 400        /* Allocate memory structure for interaction with Perf */
 401        buf = kzalloc_node(sizeof(struct cs_buffers), GFP_KERNEL, node);
 402        if (!buf)
 403                return NULL;
 404
 405        buf->pid = task_pid_nr(event->owner);
 406        buf->snapshot = overwrite;
 407        buf->nr_pages = nr_pages;
 408        buf->data_pages = pages;
 409
 410        return buf;
 411}
 412
 413static void tmc_free_etf_buffer(void *config)
 414{
 415        struct cs_buffers *buf = config;
 416
 417        kfree(buf);
 418}
 419
 420static int tmc_set_etf_buffer(struct coresight_device *csdev,
 421                              struct perf_output_handle *handle)
 422{
 423        int ret = 0;
 424        unsigned long head;
 425        struct cs_buffers *buf = etm_perf_sink_config(handle);
 426
 427        if (!buf)
 428                return -EINVAL;
 429
 430        /* wrap head around to the amount of space we have */
 431        head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
 432
 433        /* find the page to write to */
 434        buf->cur = head / PAGE_SIZE;
 435
 436        /* and offset within that page */
 437        buf->offset = head % PAGE_SIZE;
 438
 439        local_set(&buf->data_size, 0);
 440
 441        return ret;
 442}
 443
 444static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev,
 445                                  struct perf_output_handle *handle,
 446                                  void *sink_config)
 447{
 448        bool lost = false;
 449        int i, cur;
 450        const u32 *barrier;
 451        u32 *buf_ptr;
 452        u64 read_ptr, write_ptr;
 453        u32 status;
 454        unsigned long offset, to_read = 0, flags;
 455        struct cs_buffers *buf = sink_config;
 456        struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 457
 458        if (!buf)
 459                return 0;
 460
 461        /* This shouldn't happen */
 462        if (WARN_ON_ONCE(drvdata->mode != CS_MODE_PERF))
 463                return 0;
 464
 465        spin_lock_irqsave(&drvdata->spinlock, flags);
 466
 467        /* Don't do anything if another tracer is using this sink */
 468        if (atomic_read(csdev->refcnt) != 1)
 469                goto out;
 470
 471        CS_UNLOCK(drvdata->base);
 472
 473        tmc_flush_and_stop(drvdata);
 474
 475        read_ptr = tmc_read_rrp(drvdata);
 476        write_ptr = tmc_read_rwp(drvdata);
 477
 478        /*
 479         * Get a hold of the status register and see if a wrap around
 480         * has occurred.  If so adjust things accordingly.
 481         */
 482        status = readl_relaxed(drvdata->base + TMC_STS);
 483        if (status & TMC_STS_FULL) {
 484                lost = true;
 485                to_read = drvdata->size;
 486        } else {
 487                to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size);
 488        }
 489
 490        /*
 491         * The TMC RAM buffer may be bigger than the space available in the
 492         * perf ring buffer (handle->size).  If so advance the RRP so that we
 493         * get the latest trace data.  In snapshot mode none of that matters
 494         * since we are expected to clobber stale data in favour of the latest
 495         * traces.
 496         */
 497        if (!buf->snapshot && to_read > handle->size) {
 498                u32 mask = tmc_get_memwidth_mask(drvdata);
 499
 500                /*
 501                 * Make sure the new size is aligned in accordance with the
 502                 * requirement explained in function tmc_get_memwidth_mask().
 503                 */
 504                to_read = handle->size & mask;
 505                /* Move the RAM read pointer up */
 506                read_ptr = (write_ptr + drvdata->size) - to_read;
 507                /* Make sure we are still within our limits */
 508                if (read_ptr > (drvdata->size - 1))
 509                        read_ptr -= drvdata->size;
 510                /* Tell the HW */
 511                tmc_write_rrp(drvdata, read_ptr);
 512                lost = true;
 513        }
 514
 515        /*
 516         * Don't set the TRUNCATED flag in snapshot mode because 1) the
 517         * captured buffer is expected to be truncated and 2) a full buffer
 518         * prevents the event from being re-enabled by the perf core,
 519         * resulting in stale data being send to user space.
 520         */
 521        if (!buf->snapshot && lost)
 522                perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
 523
 524        cur = buf->cur;
 525        offset = buf->offset;
 526        barrier = coresight_barrier_pkt;
 527
 528        /* for every byte to read */
 529        for (i = 0; i < to_read; i += 4) {
 530                buf_ptr = buf->data_pages[cur] + offset;
 531                *buf_ptr = readl_relaxed(drvdata->base + TMC_RRD);
 532
 533                if (lost && i < CORESIGHT_BARRIER_PKT_SIZE) {
 534                        *buf_ptr = *barrier;
 535                        barrier++;
 536                }
 537
 538                offset += 4;
 539                if (offset >= PAGE_SIZE) {
 540                        offset = 0;
 541                        cur++;
 542                        /* wrap around at the end of the buffer */
 543                        cur &= buf->nr_pages - 1;
 544                }
 545        }
 546
 547        /*
 548         * In snapshot mode we simply increment the head by the number of byte
 549         * that were written.  User space function  cs_etm_find_snapshot() will
 550         * figure out how many bytes to get from the AUX buffer based on the
 551         * position of the head.
 552         */
 553        if (buf->snapshot)
 554                handle->head += to_read;
 555
 556        CS_LOCK(drvdata->base);
 557out:
 558        spin_unlock_irqrestore(&drvdata->spinlock, flags);
 559
 560        return to_read;
 561}
 562
 563static const struct coresight_ops_sink tmc_etf_sink_ops = {
 564        .enable         = tmc_enable_etf_sink,
 565        .disable        = tmc_disable_etf_sink,
 566        .alloc_buffer   = tmc_alloc_etf_buffer,
 567        .free_buffer    = tmc_free_etf_buffer,
 568        .update_buffer  = tmc_update_etf_buffer,
 569};
 570
 571static const struct coresight_ops_link tmc_etf_link_ops = {
 572        .enable         = tmc_enable_etf_link,
 573        .disable        = tmc_disable_etf_link,
 574};
 575
 576const struct coresight_ops tmc_etb_cs_ops = {
 577        .sink_ops       = &tmc_etf_sink_ops,
 578};
 579
 580const struct coresight_ops tmc_etf_cs_ops = {
 581        .sink_ops       = &tmc_etf_sink_ops,
 582        .link_ops       = &tmc_etf_link_ops,
 583};
 584
 585int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
 586{
 587        enum tmc_mode mode;
 588        int ret = 0;
 589        unsigned long flags;
 590
 591        /* config types are set a boot time and never change */
 592        if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
 593                         drvdata->config_type != TMC_CONFIG_TYPE_ETF))
 594                return -EINVAL;
 595
 596        spin_lock_irqsave(&drvdata->spinlock, flags);
 597
 598        if (drvdata->reading) {
 599                ret = -EBUSY;
 600                goto out;
 601        }
 602
 603        /* Don't interfere if operated from Perf */
 604        if (drvdata->mode == CS_MODE_PERF) {
 605                ret = -EINVAL;
 606                goto out;
 607        }
 608
 609        /* If drvdata::buf is NULL the trace data has been read already */
 610        if (drvdata->buf == NULL) {
 611                ret = -EINVAL;
 612                goto out;
 613        }
 614
 615        /* Disable the TMC if need be */
 616        if (drvdata->mode == CS_MODE_SYSFS) {
 617                /* There is no point in reading a TMC in HW FIFO mode */
 618                mode = readl_relaxed(drvdata->base + TMC_MODE);
 619                if (mode != TMC_MODE_CIRCULAR_BUFFER) {
 620                        ret = -EINVAL;
 621                        goto out;
 622                }
 623                __tmc_etb_disable_hw(drvdata);
 624        }
 625
 626        drvdata->reading = true;
 627out:
 628        spin_unlock_irqrestore(&drvdata->spinlock, flags);
 629
 630        return ret;
 631}
 632
 633int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata)
 634{
 635        char *buf = NULL;
 636        enum tmc_mode mode;
 637        unsigned long flags;
 638
 639        /* config types are set a boot time and never change */
 640        if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
 641                         drvdata->config_type != TMC_CONFIG_TYPE_ETF))
 642                return -EINVAL;
 643
 644        spin_lock_irqsave(&drvdata->spinlock, flags);
 645
 646        /* Re-enable the TMC if need be */
 647        if (drvdata->mode == CS_MODE_SYSFS) {
 648                /* There is no point in reading a TMC in HW FIFO mode */
 649                mode = readl_relaxed(drvdata->base + TMC_MODE);
 650                if (mode != TMC_MODE_CIRCULAR_BUFFER) {
 651                        spin_unlock_irqrestore(&drvdata->spinlock, flags);
 652                        return -EINVAL;
 653                }
 654                /*
 655                 * The trace run will continue with the same allocated trace
 656                 * buffer. As such zero-out the buffer so that we don't end
 657                 * up with stale data.
 658                 *
 659                 * Since the tracer is still enabled drvdata::buf
 660                 * can't be NULL.
 661                 */
 662                memset(drvdata->buf, 0, drvdata->size);
 663                __tmc_etb_enable_hw(drvdata);
 664        } else {
 665                /*
 666                 * The ETB/ETF is not tracing and the buffer was just read.
 667                 * As such prepare to free the trace buffer.
 668                 */
 669                buf = drvdata->buf;
 670                drvdata->buf = NULL;
 671        }
 672
 673        drvdata->reading = false;
 674        spin_unlock_irqrestore(&drvdata->spinlock, flags);
 675
 676        /*
 677         * Free allocated memory outside of the spinlock.  There is no need
 678         * to assert the validity of 'buf' since calling kfree(NULL) is safe.
 679         */
 680        kfree(buf);
 681
 682        return 0;
 683}
 684