LXR linux/lib/raid6/recov

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2012 Intel Corporation
   4 * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
   5 */
   6
   7#include <linux/raid/pq.h>
   8
   9#ifdef __KERNEL__
  10#include <asm/neon.h>
  11#else
  12#define kernel_neon_begin()
  13#define kernel_neon_end()
  14#define cpu_has_neon()          (1)
  15#endif
  16
  17static int raid6_has_neon(void)
  18{
  19        return cpu_has_neon();
  20}
  21
  22void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
  23                              uint8_t *dq, const uint8_t *pbmul,
  24                              const uint8_t *qmul);
  25
  26void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
  27                              const uint8_t *qmul);
  28
  29static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
  30                int failb, void **ptrs)
  31{
  32        u8 *p, *q, *dp, *dq;
  33        const u8 *pbmul;        /* P multiplier table for B data */
  34        const u8 *qmul;         /* Q multiplier table (for both) */
  35
  36        p = (u8 *)ptrs[disks - 2];
  37        q = (u8 *)ptrs[disks - 1];
  38
  39        /*
  40         * Compute syndrome with zero for the missing data pages
  41         * Use the dead data pages as temporary storage for
  42         * delta p and delta q
  43         */
  44        dp = (u8 *)ptrs[faila];
  45        ptrs[faila] = (void *)raid6_empty_zero_page;
  46        ptrs[disks - 2] = dp;
  47        dq = (u8 *)ptrs[failb];
  48        ptrs[failb] = (void *)raid6_empty_zero_page;
  49        ptrs[disks - 1] = dq;
  50
  51        raid6_call.gen_syndrome(disks, bytes, ptrs);
  52
  53        /* Restore pointer table */
  54        ptrs[faila]     = dp;
  55        ptrs[failb]     = dq;
  56        ptrs[disks - 2] = p;
  57        ptrs[disks - 1] = q;
  58
  59        /* Now, pick the proper data tables */
  60        pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
  61        qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
  62                                         raid6_gfexp[failb]]];
  63
  64        kernel_neon_begin();
  65        __raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul);
  66        kernel_neon_end();
  67}
  68
  69static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
  70                void **ptrs)
  71{
  72        u8 *p, *q, *dq;
  73        const u8 *qmul;         /* Q multiplier table */
  74
  75        p = (u8 *)ptrs[disks - 2];
  76        q = (u8 *)ptrs[disks - 1];
  77
  78        /*
  79         * Compute syndrome with zero for the missing data page
  80         * Use the dead data page as temporary storage for delta q
  81         */
  82        dq = (u8 *)ptrs[faila];
  83        ptrs[faila] = (void *)raid6_empty_zero_page;
  84        ptrs[disks - 1] = dq;
  85
  86        raid6_call.gen_syndrome(disks, bytes, ptrs);
  87
  88        /* Restore pointer table */
  89        ptrs[faila]     = dq;
  90        ptrs[disks - 1] = q;
  91
  92        /* Now, pick the proper data tables */
  93        qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
  94
  95        kernel_neon_begin();
  96        __raid6_datap_recov_neon(bytes, p, q, dq, qmul);
  97        kernel_neon_end();
  98}
  99
 100const struct raid6_recov_calls raid6_recov_neon = {
 101        .data2          = raid6_2data_recov_neon,
 102        .datap          = raid6_datap_recov_neon,
 103        .valid          = raid6_has_neon,
 104        .name           = "neon",
 105        .priority       = 10,
 106};
 107