linux/arch/arm64/kernel/hw_breakpoint.c
<<
>>
Prefs
   1/*
   2 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
   3 * using the CPU's debug registers.
   4 *
   5 * Copyright (C) 2012 ARM Limited
   6 * Author: Will Deacon <will.deacon@arm.com>
   7 *
   8 * This program is free software; you can redistribute it and/or modify
   9 * it under the terms of the GNU General Public License version 2 as
  10 * published by the Free Software Foundation.
  11 *
  12 * This program is distributed in the hope that it will be useful,
  13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 * GNU General Public License for more details.
  16 *
  17 * You should have received a copy of the GNU General Public License
  18 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 */
  20
  21#define pr_fmt(fmt) "hw-breakpoint: " fmt
  22
  23#include <linux/errno.h>
  24#include <linux/hw_breakpoint.h>
  25#include <linux/perf_event.h>
  26#include <linux/ptrace.h>
  27#include <linux/smp.h>
  28
  29#include <asm/compat.h>
  30#include <asm/current.h>
  31#include <asm/debug-monitors.h>
  32#include <asm/hw_breakpoint.h>
  33#include <asm/kdebug.h>
  34#include <asm/traps.h>
  35#include <asm/cputype.h>
  36#include <asm/system_misc.h>
  37
  38/* Breakpoint currently in use for each BRP. */
  39static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]);
  40
  41/* Watchpoint currently in use for each WRP. */
  42static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]);
  43
  44/* Currently stepping a per-CPU kernel breakpoint. */
  45static DEFINE_PER_CPU(int, stepping_kernel_bp);
  46
  47/* Number of BRP/WRP registers on this CPU. */
  48static int core_num_brps;
  49static int core_num_wrps;
  50
  51/* Determine number of BRP registers available. */
  52static int get_num_brps(void)
  53{
  54        return ((read_cpuid(ID_AA64DFR0_EL1) >> 12) & 0xf) + 1;
  55}
  56
  57/* Determine number of WRP registers available. */
  58static int get_num_wrps(void)
  59{
  60        return ((read_cpuid(ID_AA64DFR0_EL1) >> 20) & 0xf) + 1;
  61}
  62
  63int hw_breakpoint_slots(int type)
  64{
  65        /*
  66         * We can be called early, so don't rely on
  67         * our static variables being initialised.
  68         */
  69        switch (type) {
  70        case TYPE_INST:
  71                return get_num_brps();
  72        case TYPE_DATA:
  73                return get_num_wrps();
  74        default:
  75                pr_warning("unknown slot type: %d\n", type);
  76                return 0;
  77        }
  78}
  79
  80#define READ_WB_REG_CASE(OFF, N, REG, VAL)      \
  81        case (OFF + N):                         \
  82                AARCH64_DBG_READ(N, REG, VAL);  \
  83                break
  84
  85#define WRITE_WB_REG_CASE(OFF, N, REG, VAL)     \
  86        case (OFF + N):                         \
  87                AARCH64_DBG_WRITE(N, REG, VAL); \
  88                break
  89
  90#define GEN_READ_WB_REG_CASES(OFF, REG, VAL)    \
  91        READ_WB_REG_CASE(OFF,  0, REG, VAL);    \
  92        READ_WB_REG_CASE(OFF,  1, REG, VAL);    \
  93        READ_WB_REG_CASE(OFF,  2, REG, VAL);    \
  94        READ_WB_REG_CASE(OFF,  3, REG, VAL);    \
  95        READ_WB_REG_CASE(OFF,  4, REG, VAL);    \
  96        READ_WB_REG_CASE(OFF,  5, REG, VAL);    \
  97        READ_WB_REG_CASE(OFF,  6, REG, VAL);    \
  98        READ_WB_REG_CASE(OFF,  7, REG, VAL);    \
  99        READ_WB_REG_CASE(OFF,  8, REG, VAL);    \
 100        READ_WB_REG_CASE(OFF,  9, REG, VAL);    \
 101        READ_WB_REG_CASE(OFF, 10, REG, VAL);    \
 102        READ_WB_REG_CASE(OFF, 11, REG, VAL);    \
 103        READ_WB_REG_CASE(OFF, 12, REG, VAL);    \
 104        READ_WB_REG_CASE(OFF, 13, REG, VAL);    \
 105        READ_WB_REG_CASE(OFF, 14, REG, VAL);    \
 106        READ_WB_REG_CASE(OFF, 15, REG, VAL)
 107
 108#define GEN_WRITE_WB_REG_CASES(OFF, REG, VAL)   \
 109        WRITE_WB_REG_CASE(OFF,  0, REG, VAL);   \
 110        WRITE_WB_REG_CASE(OFF,  1, REG, VAL);   \
 111        WRITE_WB_REG_CASE(OFF,  2, REG, VAL);   \
 112        WRITE_WB_REG_CASE(OFF,  3, REG, VAL);   \
 113        WRITE_WB_REG_CASE(OFF,  4, REG, VAL);   \
 114        WRITE_WB_REG_CASE(OFF,  5, REG, VAL);   \
 115        WRITE_WB_REG_CASE(OFF,  6, REG, VAL);   \
 116        WRITE_WB_REG_CASE(OFF,  7, REG, VAL);   \
 117        WRITE_WB_REG_CASE(OFF,  8, REG, VAL);   \
 118        WRITE_WB_REG_CASE(OFF,  9, REG, VAL);   \
 119        WRITE_WB_REG_CASE(OFF, 10, REG, VAL);   \
 120        WRITE_WB_REG_CASE(OFF, 11, REG, VAL);   \
 121        WRITE_WB_REG_CASE(OFF, 12, REG, VAL);   \
 122        WRITE_WB_REG_CASE(OFF, 13, REG, VAL);   \
 123        WRITE_WB_REG_CASE(OFF, 14, REG, VAL);   \
 124        WRITE_WB_REG_CASE(OFF, 15, REG, VAL)
 125
 126static u64 read_wb_reg(int reg, int n)
 127{
 128        u64 val = 0;
 129
 130        switch (reg + n) {
 131        GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
 132        GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
 133        GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val);
 134        GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val);
 135        default:
 136                pr_warning("attempt to read from unknown breakpoint register %d\n", n);
 137        }
 138
 139        return val;
 140}
 141
 142static void write_wb_reg(int reg, int n, u64 val)
 143{
 144        switch (reg + n) {
 145        GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
 146        GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
 147        GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val);
 148        GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val);
 149        default:
 150                pr_warning("attempt to write to unknown breakpoint register %d\n", n);
 151        }
 152        isb();
 153}
 154
 155/*
 156 * Convert a breakpoint privilege level to the corresponding exception
 157 * level.
 158 */
 159static enum debug_el debug_exception_level(int privilege)
 160{
 161        switch (privilege) {
 162        case AARCH64_BREAKPOINT_EL0:
 163                return DBG_ACTIVE_EL0;
 164        case AARCH64_BREAKPOINT_EL1:
 165                return DBG_ACTIVE_EL1;
 166        default:
 167                pr_warning("invalid breakpoint privilege level %d\n", privilege);
 168                return -EINVAL;
 169        }
 170}
 171
 172/*
 173 * Install a perf counter breakpoint.
 174 */
 175int arch_install_hw_breakpoint(struct perf_event *bp)
 176{
 177        struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 178        struct perf_event **slot, **slots;
 179        struct debug_info *debug_info = &current->thread.debug;
 180        int i, max_slots, ctrl_reg, val_reg, reg_enable;
 181        u32 ctrl;
 182
 183        if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
 184                /* Breakpoint */
 185                ctrl_reg = AARCH64_DBG_REG_BCR;
 186                val_reg = AARCH64_DBG_REG_BVR;
 187                slots = __get_cpu_var(bp_on_reg);
 188                max_slots = core_num_brps;
 189                reg_enable = !debug_info->bps_disabled;
 190        } else {
 191                /* Watchpoint */
 192                ctrl_reg = AARCH64_DBG_REG_WCR;
 193                val_reg = AARCH64_DBG_REG_WVR;
 194                slots = __get_cpu_var(wp_on_reg);
 195                max_slots = core_num_wrps;
 196                reg_enable = !debug_info->wps_disabled;
 197        }
 198
 199        for (i = 0; i < max_slots; ++i) {
 200                slot = &slots[i];
 201
 202                if (!*slot) {
 203                        *slot = bp;
 204                        break;
 205                }
 206        }
 207
 208        if (WARN_ONCE(i == max_slots, "Can't find any breakpoint slot"))
 209                return -ENOSPC;
 210
 211        /* Ensure debug monitors are enabled at the correct exception level.  */
 212        enable_debug_monitors(debug_exception_level(info->ctrl.privilege));
 213
 214        /* Setup the address register. */
 215        write_wb_reg(val_reg, i, info->address);
 216
 217        /* Setup the control register. */
 218        ctrl = encode_ctrl_reg(info->ctrl);
 219        write_wb_reg(ctrl_reg, i, reg_enable ? ctrl | 0x1 : ctrl & ~0x1);
 220
 221        return 0;
 222}
 223
 224void arch_uninstall_hw_breakpoint(struct perf_event *bp)
 225{
 226        struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 227        struct perf_event **slot, **slots;
 228        int i, max_slots, base;
 229
 230        if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
 231                /* Breakpoint */
 232                base = AARCH64_DBG_REG_BCR;
 233                slots = __get_cpu_var(bp_on_reg);
 234                max_slots = core_num_brps;
 235        } else {
 236                /* Watchpoint */
 237                base = AARCH64_DBG_REG_WCR;
 238                slots = __get_cpu_var(wp_on_reg);
 239                max_slots = core_num_wrps;
 240        }
 241
 242        /* Remove the breakpoint. */
 243        for (i = 0; i < max_slots; ++i) {
 244                slot = &slots[i];
 245
 246                if (*slot == bp) {
 247                        *slot = NULL;
 248                        break;
 249                }
 250        }
 251
 252        if (WARN_ONCE(i == max_slots, "Can't find any breakpoint slot"))
 253                return;
 254
 255        /* Reset the control register. */
 256        write_wb_reg(base, i, 0);
 257
 258        /* Release the debug monitors for the correct exception level.  */
 259        disable_debug_monitors(debug_exception_level(info->ctrl.privilege));
 260}
 261
 262static int get_hbp_len(u8 hbp_len)
 263{
 264        unsigned int len_in_bytes = 0;
 265
 266        switch (hbp_len) {
 267        case ARM_BREAKPOINT_LEN_1:
 268                len_in_bytes = 1;
 269                break;
 270        case ARM_BREAKPOINT_LEN_2:
 271                len_in_bytes = 2;
 272                break;
 273        case ARM_BREAKPOINT_LEN_4:
 274                len_in_bytes = 4;
 275                break;
 276        case ARM_BREAKPOINT_LEN_8:
 277                len_in_bytes = 8;
 278                break;
 279        }
 280
 281        return len_in_bytes;
 282}
 283
 284/*
 285 * Check whether bp virtual address is in kernel space.
 286 */
 287int arch_check_bp_in_kernelspace(struct perf_event *bp)
 288{
 289        unsigned int len;
 290        unsigned long va;
 291        struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 292
 293        va = info->address;
 294        len = get_hbp_len(info->ctrl.len);
 295
 296        return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
 297}
 298
 299/*
 300 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl.
 301 * Hopefully this will disappear when ptrace can bypass the conversion
 302 * to generic breakpoint descriptions.
 303 */
 304int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
 305                           int *gen_len, int *gen_type)
 306{
 307        /* Type */
 308        switch (ctrl.type) {
 309        case ARM_BREAKPOINT_EXECUTE:
 310                *gen_type = HW_BREAKPOINT_X;
 311                break;
 312        case ARM_BREAKPOINT_LOAD:
 313                *gen_type = HW_BREAKPOINT_R;
 314                break;
 315        case ARM_BREAKPOINT_STORE:
 316                *gen_type = HW_BREAKPOINT_W;
 317                break;
 318        case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE:
 319                *gen_type = HW_BREAKPOINT_RW;
 320                break;
 321        default:
 322                return -EINVAL;
 323        }
 324
 325        /* Len */
 326        switch (ctrl.len) {
 327        case ARM_BREAKPOINT_LEN_1:
 328                *gen_len = HW_BREAKPOINT_LEN_1;
 329                break;
 330        case ARM_BREAKPOINT_LEN_2:
 331                *gen_len = HW_BREAKPOINT_LEN_2;
 332                break;
 333        case ARM_BREAKPOINT_LEN_4:
 334                *gen_len = HW_BREAKPOINT_LEN_4;
 335                break;
 336        case ARM_BREAKPOINT_LEN_8:
 337                *gen_len = HW_BREAKPOINT_LEN_8;
 338                break;
 339        default:
 340                return -EINVAL;
 341        }
 342
 343        return 0;
 344}
 345
 346/*
 347 * Construct an arch_hw_breakpoint from a perf_event.
 348 */
 349static int arch_build_bp_info(struct perf_event *bp)
 350{
 351        struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 352
 353        /* Type */
 354        switch (bp->attr.bp_type) {
 355        case HW_BREAKPOINT_X:
 356                info->ctrl.type = ARM_BREAKPOINT_EXECUTE;
 357                break;
 358        case HW_BREAKPOINT_R:
 359                info->ctrl.type = ARM_BREAKPOINT_LOAD;
 360                break;
 361        case HW_BREAKPOINT_W:
 362                info->ctrl.type = ARM_BREAKPOINT_STORE;
 363                break;
 364        case HW_BREAKPOINT_RW:
 365                info->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE;
 366                break;
 367        default:
 368                return -EINVAL;
 369        }
 370
 371        /* Len */
 372        switch (bp->attr.bp_len) {
 373        case HW_BREAKPOINT_LEN_1:
 374                info->ctrl.len = ARM_BREAKPOINT_LEN_1;
 375                break;
 376        case HW_BREAKPOINT_LEN_2:
 377                info->ctrl.len = ARM_BREAKPOINT_LEN_2;
 378                break;
 379        case HW_BREAKPOINT_LEN_4:
 380                info->ctrl.len = ARM_BREAKPOINT_LEN_4;
 381                break;
 382        case HW_BREAKPOINT_LEN_8:
 383                info->ctrl.len = ARM_BREAKPOINT_LEN_8;
 384                break;
 385        default:
 386                return -EINVAL;
 387        }
 388
 389        /*
 390         * On AArch64, we only permit breakpoints of length 4, whereas
 391         * AArch32 also requires breakpoints of length 2 for Thumb.
 392         * Watchpoints can be of length 1, 2, 4 or 8 bytes.
 393         */
 394        if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
 395                if (is_compat_task()) {
 396                        if (info->ctrl.len != ARM_BREAKPOINT_LEN_2 &&
 397                            info->ctrl.len != ARM_BREAKPOINT_LEN_4)
 398                                return -EINVAL;
 399                } else if (info->ctrl.len != ARM_BREAKPOINT_LEN_4) {
 400                        /*
 401                         * FIXME: Some tools (I'm looking at you perf) assume
 402                         *        that breakpoints should be sizeof(long). This
 403                         *        is nonsense. For now, we fix up the parameter
 404                         *        but we should probably return -EINVAL instead.
 405                         */
 406                        info->ctrl.len = ARM_BREAKPOINT_LEN_4;
 407                }
 408        }
 409
 410        /* Address */
 411        info->address = bp->attr.bp_addr;
 412
 413        /*
 414         * Privilege
 415         * Note that we disallow combined EL0/EL1 breakpoints because
 416         * that would complicate the stepping code.
 417         */
 418        if (arch_check_bp_in_kernelspace(bp))
 419                info->ctrl.privilege = AARCH64_BREAKPOINT_EL1;
 420        else
 421                info->ctrl.privilege = AARCH64_BREAKPOINT_EL0;
 422
 423        /* Enabled? */
 424        info->ctrl.enabled = !bp->attr.disabled;
 425
 426        return 0;
 427}
 428
 429/*
 430 * Validate the arch-specific HW Breakpoint register settings.
 431 */
 432int arch_validate_hwbkpt_settings(struct perf_event *bp)
 433{
 434        struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 435        int ret;
 436        u64 alignment_mask, offset;
 437
 438        /* Build the arch_hw_breakpoint. */
 439        ret = arch_build_bp_info(bp);
 440        if (ret)
 441                return ret;
 442
 443        /*
 444         * Check address alignment.
 445         * We don't do any clever alignment correction for watchpoints
 446         * because using 64-bit unaligned addresses is deprecated for
 447         * AArch64.
 448         *
 449         * AArch32 tasks expect some simple alignment fixups, so emulate
 450         * that here.
 451         */
 452        if (is_compat_task()) {
 453                if (info->ctrl.len == ARM_BREAKPOINT_LEN_8)
 454                        alignment_mask = 0x7;
 455                else
 456                        alignment_mask = 0x3;
 457                offset = info->address & alignment_mask;
 458                switch (offset) {
 459                case 0:
 460                        /* Aligned */
 461                        break;
 462                case 1:
 463                        /* Allow single byte watchpoint. */
 464                        if (info->ctrl.len == ARM_BREAKPOINT_LEN_1)
 465                                break;
 466                case 2:
 467                        /* Allow halfword watchpoints and breakpoints. */
 468                        if (info->ctrl.len == ARM_BREAKPOINT_LEN_2)
 469                                break;
 470                default:
 471                        return -EINVAL;
 472                }
 473
 474                info->address &= ~alignment_mask;
 475                info->ctrl.len <<= offset;
 476        } else {
 477                if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE)
 478                        alignment_mask = 0x3;
 479                else
 480                        alignment_mask = 0x7;
 481                if (info->address & alignment_mask)
 482                        return -EINVAL;
 483        }
 484
 485        /*
 486         * Disallow per-task kernel breakpoints since these would
 487         * complicate the stepping code.
 488         */
 489        if (info->ctrl.privilege == AARCH64_BREAKPOINT_EL1 && bp->hw.bp_target)
 490                return -EINVAL;
 491
 492        return 0;
 493}
 494
 495/*
 496 * Enable/disable all of the breakpoints active at the specified
 497 * exception level at the register level.
 498 * This is used when single-stepping after a breakpoint exception.
 499 */
 500static void toggle_bp_registers(int reg, enum debug_el el, int enable)
 501{
 502        int i, max_slots, privilege;
 503        u32 ctrl;
 504        struct perf_event **slots;
 505
 506        switch (reg) {
 507        case AARCH64_DBG_REG_BCR:
 508                slots = __get_cpu_var(bp_on_reg);
 509                max_slots = core_num_brps;
 510                break;
 511        case AARCH64_DBG_REG_WCR:
 512                slots = __get_cpu_var(wp_on_reg);
 513                max_slots = core_num_wrps;
 514                break;
 515        default:
 516                return;
 517        }
 518
 519        for (i = 0; i < max_slots; ++i) {
 520                if (!slots[i])
 521                        continue;
 522
 523                privilege = counter_arch_bp(slots[i])->ctrl.privilege;
 524                if (debug_exception_level(privilege) != el)
 525                        continue;
 526
 527                ctrl = read_wb_reg(reg, i);
 528                if (enable)
 529                        ctrl |= 0x1;
 530                else
 531                        ctrl &= ~0x1;
 532                write_wb_reg(reg, i, ctrl);
 533        }
 534}
 535
 536/*
 537 * Debug exception handlers.
 538 */
 539static int breakpoint_handler(unsigned long unused, unsigned int esr,
 540                              struct pt_regs *regs)
 541{
 542        int i, step = 0, *kernel_step;
 543        u32 ctrl_reg;
 544        u64 addr, val;
 545        struct perf_event *bp, **slots;
 546        struct debug_info *debug_info;
 547        struct arch_hw_breakpoint_ctrl ctrl;
 548
 549        slots = (struct perf_event **)__get_cpu_var(bp_on_reg);
 550        addr = instruction_pointer(regs);
 551        debug_info = &current->thread.debug;
 552
 553        for (i = 0; i < core_num_brps; ++i) {
 554                rcu_read_lock();
 555
 556                bp = slots[i];
 557
 558                if (bp == NULL)
 559                        goto unlock;
 560
 561                /* Check if the breakpoint value matches. */
 562                val = read_wb_reg(AARCH64_DBG_REG_BVR, i);
 563                if (val != (addr & ~0x3))
 564                        goto unlock;
 565
 566                /* Possible match, check the byte address select to confirm. */
 567                ctrl_reg = read_wb_reg(AARCH64_DBG_REG_BCR, i);
 568                decode_ctrl_reg(ctrl_reg, &ctrl);
 569                if (!((1 << (addr & 0x3)) & ctrl.len))
 570                        goto unlock;
 571
 572                counter_arch_bp(bp)->trigger = addr;
 573                perf_bp_event(bp, regs);
 574
 575                /* Do we need to handle the stepping? */
 576                if (!bp->overflow_handler)
 577                        step = 1;
 578unlock:
 579                rcu_read_unlock();
 580        }
 581
 582        if (!step)
 583                return 0;
 584
 585        if (user_mode(regs)) {
 586                debug_info->bps_disabled = 1;
 587                toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 0);
 588
 589                /* If we're already stepping a watchpoint, just return. */
 590                if (debug_info->wps_disabled)
 591                        return 0;
 592
 593                if (test_thread_flag(TIF_SINGLESTEP))
 594                        debug_info->suspended_step = 1;
 595                else
 596                        user_enable_single_step(current);
 597        } else {
 598                toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 0);
 599                kernel_step = &__get_cpu_var(stepping_kernel_bp);
 600
 601                if (*kernel_step != ARM_KERNEL_STEP_NONE)
 602                        return 0;
 603
 604                if (kernel_active_single_step()) {
 605                        *kernel_step = ARM_KERNEL_STEP_SUSPEND;
 606                } else {
 607                        *kernel_step = ARM_KERNEL_STEP_ACTIVE;
 608                        kernel_enable_single_step(regs);
 609                }
 610        }
 611
 612        return 0;
 613}
 614
 615static int watchpoint_handler(unsigned long addr, unsigned int esr,
 616                              struct pt_regs *regs)
 617{
 618        int i, step = 0, *kernel_step, access;
 619        u32 ctrl_reg;
 620        u64 val, alignment_mask;
 621        struct perf_event *wp, **slots;
 622        struct debug_info *debug_info;
 623        struct arch_hw_breakpoint *info;
 624        struct arch_hw_breakpoint_ctrl ctrl;
 625
 626        slots = (struct perf_event **)__get_cpu_var(wp_on_reg);
 627        debug_info = &current->thread.debug;
 628
 629        for (i = 0; i < core_num_wrps; ++i) {
 630                rcu_read_lock();
 631
 632                wp = slots[i];
 633
 634                if (wp == NULL)
 635                        goto unlock;
 636
 637                info = counter_arch_bp(wp);
 638                /* AArch32 watchpoints are either 4 or 8 bytes aligned. */
 639                if (is_compat_task()) {
 640                        if (info->ctrl.len == ARM_BREAKPOINT_LEN_8)
 641                                alignment_mask = 0x7;
 642                        else
 643                                alignment_mask = 0x3;
 644                } else {
 645                        alignment_mask = 0x7;
 646                }
 647
 648                /* Check if the watchpoint value matches. */
 649                val = read_wb_reg(AARCH64_DBG_REG_WVR, i);
 650                if (val != (addr & ~alignment_mask))
 651                        goto unlock;
 652
 653                /* Possible match, check the byte address select to confirm. */
 654                ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i);
 655                decode_ctrl_reg(ctrl_reg, &ctrl);
 656                if (!((1 << (addr & alignment_mask)) & ctrl.len))
 657                        goto unlock;
 658
 659                /*
 660                 * Check that the access type matches.
 661                 * 0 => load, otherwise => store
 662                 */
 663                access = (esr & AARCH64_ESR_ACCESS_MASK) ? HW_BREAKPOINT_W :
 664                         HW_BREAKPOINT_R;
 665                if (!(access & hw_breakpoint_type(wp)))
 666                        goto unlock;
 667
 668                info->trigger = addr;
 669                perf_bp_event(wp, regs);
 670
 671                /* Do we need to handle the stepping? */
 672                if (!wp->overflow_handler)
 673                        step = 1;
 674
 675unlock:
 676                rcu_read_unlock();
 677        }
 678
 679        if (!step)
 680                return 0;
 681
 682        /*
 683         * We always disable EL0 watchpoints because the kernel can
 684         * cause these to fire via an unprivileged access.
 685         */
 686        toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 0);
 687
 688        if (user_mode(regs)) {
 689                debug_info->wps_disabled = 1;
 690
 691                /* If we're already stepping a breakpoint, just return. */
 692                if (debug_info->bps_disabled)
 693                        return 0;
 694
 695                if (test_thread_flag(TIF_SINGLESTEP))
 696                        debug_info->suspended_step = 1;
 697                else
 698                        user_enable_single_step(current);
 699        } else {
 700                toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 0);
 701                kernel_step = &__get_cpu_var(stepping_kernel_bp);
 702
 703                if (*kernel_step != ARM_KERNEL_STEP_NONE)
 704                        return 0;
 705
 706                if (kernel_active_single_step()) {
 707                        *kernel_step = ARM_KERNEL_STEP_SUSPEND;
 708                } else {
 709                        *kernel_step = ARM_KERNEL_STEP_ACTIVE;
 710                        kernel_enable_single_step(regs);
 711                }
 712        }
 713
 714        return 0;
 715}
 716
 717/*
 718 * Handle single-step exception.
 719 */
 720int reinstall_suspended_bps(struct pt_regs *regs)
 721{
 722        struct debug_info *debug_info = &current->thread.debug;
 723        int handled_exception = 0, *kernel_step;
 724
 725        kernel_step = &__get_cpu_var(stepping_kernel_bp);
 726
 727        /*
 728         * Called from single-step exception handler.
 729         * Return 0 if execution can resume, 1 if a SIGTRAP should be
 730         * reported.
 731         */
 732        if (user_mode(regs)) {
 733                if (debug_info->bps_disabled) {
 734                        debug_info->bps_disabled = 0;
 735                        toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 1);
 736                        handled_exception = 1;
 737                }
 738
 739                if (debug_info->wps_disabled) {
 740                        debug_info->wps_disabled = 0;
 741                        toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1);
 742                        handled_exception = 1;
 743                }
 744
 745                if (handled_exception) {
 746                        if (debug_info->suspended_step) {
 747                                debug_info->suspended_step = 0;
 748                                /* Allow exception handling to fall-through. */
 749                                handled_exception = 0;
 750                        } else {
 751                                user_disable_single_step(current);
 752                        }
 753                }
 754        } else if (*kernel_step != ARM_KERNEL_STEP_NONE) {
 755                toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 1);
 756                toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 1);
 757
 758                if (!debug_info->wps_disabled)
 759                        toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1);
 760
 761                if (*kernel_step != ARM_KERNEL_STEP_SUSPEND) {
 762                        kernel_disable_single_step();
 763                        handled_exception = 1;
 764                } else {
 765                        handled_exception = 0;
 766                }
 767
 768                *kernel_step = ARM_KERNEL_STEP_NONE;
 769        }
 770
 771        return !handled_exception;
 772}
 773
 774/*
 775 * Context-switcher for restoring suspended breakpoints.
 776 */
 777void hw_breakpoint_thread_switch(struct task_struct *next)
 778{
 779        /*
 780         *           current        next
 781         * disabled: 0              0     => The usual case, NOTIFY_DONE
 782         *           0              1     => Disable the registers
 783         *           1              0     => Enable the registers
 784         *           1              1     => NOTIFY_DONE. per-task bps will
 785         *                                   get taken care of by perf.
 786         */
 787
 788        struct debug_info *current_debug_info, *next_debug_info;
 789
 790        current_debug_info = &current->thread.debug;
 791        next_debug_info = &next->thread.debug;
 792
 793        /* Update breakpoints. */
 794        if (current_debug_info->bps_disabled != next_debug_info->bps_disabled)
 795                toggle_bp_registers(AARCH64_DBG_REG_BCR,
 796                                    DBG_ACTIVE_EL0,
 797                                    !next_debug_info->bps_disabled);
 798
 799        /* Update watchpoints. */
 800        if (current_debug_info->wps_disabled != next_debug_info->wps_disabled)
 801                toggle_bp_registers(AARCH64_DBG_REG_WCR,
 802                                    DBG_ACTIVE_EL0,
 803                                    !next_debug_info->wps_disabled);
 804}
 805
 806/*
 807 * CPU initialisation.
 808 */
 809static void reset_ctrl_regs(void *unused)
 810{
 811        int i;
 812
 813        for (i = 0; i < core_num_brps; ++i) {
 814                write_wb_reg(AARCH64_DBG_REG_BCR, i, 0UL);
 815                write_wb_reg(AARCH64_DBG_REG_BVR, i, 0UL);
 816        }
 817
 818        for (i = 0; i < core_num_wrps; ++i) {
 819                write_wb_reg(AARCH64_DBG_REG_WCR, i, 0UL);
 820                write_wb_reg(AARCH64_DBG_REG_WVR, i, 0UL);
 821        }
 822}
 823
 824static int hw_breakpoint_reset_notify(struct notifier_block *self,
 825                                                unsigned long action,
 826                                                void *hcpu)
 827{
 828        int cpu = (long)hcpu;
 829        if (action == CPU_ONLINE)
 830                smp_call_function_single(cpu, reset_ctrl_regs, NULL, 1);
 831        return NOTIFY_OK;
 832}
 833
 834static struct notifier_block hw_breakpoint_reset_nb = {
 835        .notifier_call = hw_breakpoint_reset_notify,
 836};
 837
 838/*
 839 * One-time initialisation.
 840 */
 841static int __init arch_hw_breakpoint_init(void)
 842{
 843        core_num_brps = get_num_brps();
 844        core_num_wrps = get_num_wrps();
 845
 846        pr_info("found %d breakpoint and %d watchpoint registers.\n",
 847                core_num_brps, core_num_wrps);
 848
 849        /*
 850         * Reset the breakpoint resources. We assume that a halting
 851         * debugger will leave the world in a nice state for us.
 852         */
 853        smp_call_function(reset_ctrl_regs, NULL, 1);
 854        reset_ctrl_regs(NULL);
 855
 856        /* Register debug fault handlers. */
 857        hook_debug_fault_code(DBG_ESR_EVT_HWBP, breakpoint_handler, SIGTRAP,
 858                              TRAP_HWBKPT, "hw-breakpoint handler");
 859        hook_debug_fault_code(DBG_ESR_EVT_HWWP, watchpoint_handler, SIGTRAP,
 860                              TRAP_HWBKPT, "hw-watchpoint handler");
 861
 862        /* Register hotplug notifier. */
 863        register_cpu_notifier(&hw_breakpoint_reset_nb);
 864
 865        return 0;
 866}
 867arch_initcall(arch_hw_breakpoint_init);
 868
 869void hw_breakpoint_pmu_read(struct perf_event *bp)
 870{
 871}
 872
 873/*
 874 * Dummy function to register with die_notifier.
 875 */
 876int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
 877                                    unsigned long val, void *data)
 878{
 879        return NOTIFY_DONE;
 880}
 881