linux/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
<<
>>
Prefs
   1/*
   2 * Copyright 2015-2017 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 */
  22
  23#include <linux/pci.h>
  24#include <linux/acpi.h>
  25#include "kfd_crat.h"
  26#include "kfd_priv.h"
  27#include "kfd_topology.h"
  28#include "kfd_iommu.h"
  29#include "amdgpu.h"
  30#include "amdgpu_amdkfd.h"
  31
  32/* GPU Processor ID base for dGPUs for which VCRAT needs to be created.
  33 * GPU processor ID are expressed with Bit[31]=1.
  34 * The base is set to 0x8000_0000 + 0x1000 to avoid collision with GPU IDs
  35 * used in the CRAT.
  36 */
  37static uint32_t gpu_processor_id_low = 0x80001000;
  38
  39/* Return the next available gpu_processor_id and increment it for next GPU
  40 *      @total_cu_count - Total CUs present in the GPU including ones
  41 *                        masked off
  42 */
  43static inline unsigned int get_and_inc_gpu_processor_id(
  44                                unsigned int total_cu_count)
  45{
  46        int current_id = gpu_processor_id_low;
  47
  48        gpu_processor_id_low += total_cu_count;
  49        return current_id;
  50}
  51
  52/* Static table to describe GPU Cache information */
  53struct kfd_gpu_cache_info {
  54        uint32_t        cache_size;
  55        uint32_t        cache_level;
  56        uint32_t        flags;
  57        /* Indicates how many Compute Units share this cache
  58         * within a SA. Value = 1 indicates the cache is not shared
  59         */
  60        uint32_t        num_cu_shared;
  61};
  62
  63static struct kfd_gpu_cache_info kaveri_cache_info[] = {
  64        {
  65                /* TCP L1 Cache per CU */
  66                .cache_size = 16,
  67                .cache_level = 1,
  68                .flags = (CRAT_CACHE_FLAGS_ENABLED |
  69                                CRAT_CACHE_FLAGS_DATA_CACHE |
  70                                CRAT_CACHE_FLAGS_SIMD_CACHE),
  71                .num_cu_shared = 1,
  72        },
  73        {
  74                /* Scalar L1 Instruction Cache (in SQC module) per bank */
  75                .cache_size = 16,
  76                .cache_level = 1,
  77                .flags = (CRAT_CACHE_FLAGS_ENABLED |
  78                                CRAT_CACHE_FLAGS_INST_CACHE |
  79                                CRAT_CACHE_FLAGS_SIMD_CACHE),
  80                .num_cu_shared = 2,
  81        },
  82        {
  83                /* Scalar L1 Data Cache (in SQC module) per bank */
  84                .cache_size = 8,
  85                .cache_level = 1,
  86                .flags = (CRAT_CACHE_FLAGS_ENABLED |
  87                                CRAT_CACHE_FLAGS_DATA_CACHE |
  88                                CRAT_CACHE_FLAGS_SIMD_CACHE),
  89                .num_cu_shared = 2,
  90        },
  91
  92        /* TODO: Add L2 Cache information */
  93};
  94
  95
  96static struct kfd_gpu_cache_info carrizo_cache_info[] = {
  97        {
  98                /* TCP L1 Cache per CU */
  99                .cache_size = 16,
 100                .cache_level = 1,
 101                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 102                                CRAT_CACHE_FLAGS_DATA_CACHE |
 103                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 104                .num_cu_shared = 1,
 105        },
 106        {
 107                /* Scalar L1 Instruction Cache (in SQC module) per bank */
 108                .cache_size = 8,
 109                .cache_level = 1,
 110                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 111                                CRAT_CACHE_FLAGS_INST_CACHE |
 112                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 113                .num_cu_shared = 4,
 114        },
 115        {
 116                /* Scalar L1 Data Cache (in SQC module) per bank. */
 117                .cache_size = 4,
 118                .cache_level = 1,
 119                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 120                                CRAT_CACHE_FLAGS_DATA_CACHE |
 121                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 122                .num_cu_shared = 4,
 123        },
 124
 125        /* TODO: Add L2 Cache information */
 126};
 127
 128#define hawaii_cache_info kaveri_cache_info
 129#define tonga_cache_info carrizo_cache_info
 130#define fiji_cache_info  carrizo_cache_info
 131#define polaris10_cache_info carrizo_cache_info
 132#define polaris11_cache_info carrizo_cache_info
 133#define polaris12_cache_info carrizo_cache_info
 134#define vegam_cache_info carrizo_cache_info
 135
 136/* NOTE: L1 cache information has been updated and L2/L3
 137 * cache information has been added for Vega10 and
 138 * newer ASICs. The unit for cache_size is KiB.
 139 * In future,  check & update cache details
 140 * for every new ASIC is required.
 141 */
 142
 143static struct kfd_gpu_cache_info vega10_cache_info[] = {
 144        {
 145                /* TCP L1 Cache per CU */
 146                .cache_size = 16,
 147                .cache_level = 1,
 148                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 149                                CRAT_CACHE_FLAGS_DATA_CACHE |
 150                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 151                .num_cu_shared = 1,
 152        },
 153        {
 154                /* Scalar L1 Instruction Cache per SQC */
 155                .cache_size = 32,
 156                .cache_level = 1,
 157                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 158                                CRAT_CACHE_FLAGS_INST_CACHE |
 159                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 160                .num_cu_shared = 3,
 161        },
 162        {
 163                /* Scalar L1 Data Cache per SQC */
 164                .cache_size = 16,
 165                .cache_level = 1,
 166                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 167                                CRAT_CACHE_FLAGS_DATA_CACHE |
 168                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 169                .num_cu_shared = 3,
 170        },
 171        {
 172                /* L2 Data Cache per GPU (Total Tex Cache) */
 173                .cache_size = 4096,
 174                .cache_level = 2,
 175                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 176                                CRAT_CACHE_FLAGS_DATA_CACHE |
 177                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 178                .num_cu_shared = 16,
 179        },
 180};
 181
 182static struct kfd_gpu_cache_info raven_cache_info[] = {
 183        {
 184                /* TCP L1 Cache per CU */
 185                .cache_size = 16,
 186                .cache_level = 1,
 187                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 188                                CRAT_CACHE_FLAGS_DATA_CACHE |
 189                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 190                .num_cu_shared = 1,
 191        },
 192        {
 193                /* Scalar L1 Instruction Cache per SQC */
 194                .cache_size = 32,
 195                .cache_level = 1,
 196                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 197                                CRAT_CACHE_FLAGS_INST_CACHE |
 198                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 199                .num_cu_shared = 3,
 200        },
 201        {
 202                /* Scalar L1 Data Cache per SQC */
 203                .cache_size = 16,
 204                .cache_level = 1,
 205                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 206                                CRAT_CACHE_FLAGS_DATA_CACHE |
 207                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 208                .num_cu_shared = 3,
 209        },
 210        {
 211                /* L2 Data Cache per GPU (Total Tex Cache) */
 212                .cache_size = 1024,
 213                .cache_level = 2,
 214                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 215                                CRAT_CACHE_FLAGS_DATA_CACHE |
 216                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 217                .num_cu_shared = 11,
 218        },
 219};
 220
 221static struct kfd_gpu_cache_info renoir_cache_info[] = {
 222        {
 223                /* TCP L1 Cache per CU */
 224                .cache_size = 16,
 225                .cache_level = 1,
 226                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 227                                CRAT_CACHE_FLAGS_DATA_CACHE |
 228                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 229                .num_cu_shared = 1,
 230        },
 231        {
 232                /* Scalar L1 Instruction Cache per SQC */
 233                .cache_size = 32,
 234                .cache_level = 1,
 235                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 236                                CRAT_CACHE_FLAGS_INST_CACHE |
 237                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 238                .num_cu_shared = 3,
 239        },
 240        {
 241                /* Scalar L1 Data Cache per SQC */
 242                .cache_size = 16,
 243                .cache_level = 1,
 244                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 245                                CRAT_CACHE_FLAGS_DATA_CACHE |
 246                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 247                .num_cu_shared = 3,
 248        },
 249        {
 250                /* L2 Data Cache per GPU (Total Tex Cache) */
 251                .cache_size = 1024,
 252                .cache_level = 2,
 253                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 254                                CRAT_CACHE_FLAGS_DATA_CACHE |
 255                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 256                .num_cu_shared = 8,
 257        },
 258};
 259
 260static struct kfd_gpu_cache_info vega12_cache_info[] = {
 261        {
 262                /* TCP L1 Cache per CU */
 263                .cache_size = 16,
 264                .cache_level = 1,
 265                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 266                                CRAT_CACHE_FLAGS_DATA_CACHE |
 267                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 268                .num_cu_shared = 1,
 269        },
 270        {
 271                /* Scalar L1 Instruction Cache per SQC */
 272                .cache_size = 32,
 273                .cache_level = 1,
 274                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 275                                CRAT_CACHE_FLAGS_INST_CACHE |
 276                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 277                .num_cu_shared = 3,
 278        },
 279        {
 280                /* Scalar L1 Data Cache per SQC */
 281                .cache_size = 16,
 282                .cache_level = 1,
 283                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 284                                CRAT_CACHE_FLAGS_DATA_CACHE |
 285                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 286                .num_cu_shared = 3,
 287        },
 288        {
 289                /* L2 Data Cache per GPU (Total Tex Cache) */
 290                .cache_size = 2048,
 291                .cache_level = 2,
 292                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 293                                CRAT_CACHE_FLAGS_DATA_CACHE |
 294                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 295                .num_cu_shared = 5,
 296        },
 297};
 298
 299static struct kfd_gpu_cache_info vega20_cache_info[] = {
 300        {
 301                /* TCP L1 Cache per CU */
 302                .cache_size = 16,
 303                .cache_level = 1,
 304                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 305                                CRAT_CACHE_FLAGS_DATA_CACHE |
 306                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 307                .num_cu_shared = 1,
 308        },
 309        {
 310                /* Scalar L1 Instruction Cache per SQC */
 311                .cache_size = 32,
 312                .cache_level = 1,
 313                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 314                                CRAT_CACHE_FLAGS_INST_CACHE |
 315                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 316                .num_cu_shared = 3,
 317        },
 318        {
 319                /* Scalar L1 Data Cache per SQC */
 320                .cache_size = 16,
 321                .cache_level = 1,
 322                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 323                                CRAT_CACHE_FLAGS_DATA_CACHE |
 324                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 325                .num_cu_shared = 3,
 326        },
 327        {
 328                /* L2 Data Cache per GPU (Total Tex Cache) */
 329                .cache_size = 8192,
 330                .cache_level = 2,
 331                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 332                                CRAT_CACHE_FLAGS_DATA_CACHE |
 333                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 334                .num_cu_shared = 16,
 335        },
 336};
 337
 338static struct kfd_gpu_cache_info aldebaran_cache_info[] = {
 339        {
 340                /* TCP L1 Cache per CU */
 341                .cache_size = 16,
 342                .cache_level = 1,
 343                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 344                                CRAT_CACHE_FLAGS_DATA_CACHE |
 345                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 346                .num_cu_shared = 1,
 347        },
 348        {
 349                /* Scalar L1 Instruction Cache per SQC */
 350                .cache_size = 32,
 351                .cache_level = 1,
 352                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 353                                CRAT_CACHE_FLAGS_INST_CACHE |
 354                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 355                .num_cu_shared = 2,
 356        },
 357        {
 358                /* Scalar L1 Data Cache per SQC */
 359                .cache_size = 16,
 360                .cache_level = 1,
 361                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 362                                CRAT_CACHE_FLAGS_DATA_CACHE |
 363                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 364                .num_cu_shared = 2,
 365        },
 366        {
 367                /* L2 Data Cache per GPU (Total Tex Cache) */
 368                .cache_size = 8192,
 369                .cache_level = 2,
 370                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 371                                CRAT_CACHE_FLAGS_DATA_CACHE |
 372                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 373                .num_cu_shared = 14,
 374        },
 375};
 376
 377static struct kfd_gpu_cache_info navi10_cache_info[] = {
 378        {
 379                /* TCP L1 Cache per CU */
 380                .cache_size = 16,
 381                .cache_level = 1,
 382                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 383                                CRAT_CACHE_FLAGS_DATA_CACHE |
 384                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 385                .num_cu_shared = 1,
 386        },
 387        {
 388                /* Scalar L1 Instruction Cache per SQC */
 389                .cache_size = 32,
 390                .cache_level = 1,
 391                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 392                                CRAT_CACHE_FLAGS_INST_CACHE |
 393                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 394                .num_cu_shared = 2,
 395        },
 396        {
 397                /* Scalar L1 Data Cache per SQC */
 398                .cache_size = 16,
 399                .cache_level = 1,
 400                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 401                                CRAT_CACHE_FLAGS_DATA_CACHE |
 402                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 403                .num_cu_shared = 2,
 404        },
 405        {
 406                /* GL1 Data Cache per SA */
 407                .cache_size = 128,
 408                .cache_level = 1,
 409                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 410                                CRAT_CACHE_FLAGS_DATA_CACHE |
 411                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 412                .num_cu_shared = 10,
 413        },
 414        {
 415                /* L2 Data Cache per GPU (Total Tex Cache) */
 416                .cache_size = 4096,
 417                .cache_level = 2,
 418                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 419                                CRAT_CACHE_FLAGS_DATA_CACHE |
 420                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 421                .num_cu_shared = 10,
 422        },
 423};
 424
 425static struct kfd_gpu_cache_info vangogh_cache_info[] = {
 426        {
 427                /* TCP L1 Cache per CU */
 428                .cache_size = 16,
 429                .cache_level = 1,
 430                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 431                                CRAT_CACHE_FLAGS_DATA_CACHE |
 432                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 433                .num_cu_shared = 1,
 434        },
 435        {
 436                /* Scalar L1 Instruction Cache per SQC */
 437                .cache_size = 32,
 438                .cache_level = 1,
 439                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 440                                CRAT_CACHE_FLAGS_INST_CACHE |
 441                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 442                .num_cu_shared = 2,
 443        },
 444        {
 445                /* Scalar L1 Data Cache per SQC */
 446                .cache_size = 16,
 447                .cache_level = 1,
 448                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 449                                CRAT_CACHE_FLAGS_DATA_CACHE |
 450                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 451                .num_cu_shared = 2,
 452        },
 453        {
 454                /* GL1 Data Cache per SA */
 455                .cache_size = 128,
 456                .cache_level = 1,
 457                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 458                                CRAT_CACHE_FLAGS_DATA_CACHE |
 459                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 460                .num_cu_shared = 8,
 461        },
 462        {
 463                /* L2 Data Cache per GPU (Total Tex Cache) */
 464                .cache_size = 1024,
 465                .cache_level = 2,
 466                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 467                                CRAT_CACHE_FLAGS_DATA_CACHE |
 468                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 469                .num_cu_shared = 8,
 470        },
 471};
 472
 473static struct kfd_gpu_cache_info navi14_cache_info[] = {
 474        {
 475                /* TCP L1 Cache per CU */
 476                .cache_size = 16,
 477                .cache_level = 1,
 478                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 479                                CRAT_CACHE_FLAGS_DATA_CACHE |
 480                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 481                .num_cu_shared = 1,
 482        },
 483        {
 484                /* Scalar L1 Instruction Cache per SQC */
 485                .cache_size = 32,
 486                .cache_level = 1,
 487                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 488                                CRAT_CACHE_FLAGS_INST_CACHE |
 489                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 490                .num_cu_shared = 2,
 491        },
 492        {
 493                /* Scalar L1 Data Cache per SQC */
 494                .cache_size = 16,
 495                .cache_level = 1,
 496                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 497                                CRAT_CACHE_FLAGS_DATA_CACHE |
 498                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 499                .num_cu_shared = 2,
 500        },
 501        {
 502                /* GL1 Data Cache per SA */
 503                .cache_size = 128,
 504                .cache_level = 1,
 505                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 506                                CRAT_CACHE_FLAGS_DATA_CACHE |
 507                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 508                .num_cu_shared = 12,
 509        },
 510        {
 511                /* L2 Data Cache per GPU (Total Tex Cache) */
 512                .cache_size = 2048,
 513                .cache_level = 2,
 514                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 515                                CRAT_CACHE_FLAGS_DATA_CACHE |
 516                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 517                .num_cu_shared = 12,
 518        },
 519};
 520
 521static struct kfd_gpu_cache_info sienna_cichlid_cache_info[] = {
 522        {
 523                /* TCP L1 Cache per CU */
 524                .cache_size = 16,
 525                .cache_level = 1,
 526                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 527                                CRAT_CACHE_FLAGS_DATA_CACHE |
 528                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 529                .num_cu_shared = 1,
 530        },
 531        {
 532                /* Scalar L1 Instruction Cache per SQC */
 533                .cache_size = 32,
 534                .cache_level = 1,
 535                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 536                                CRAT_CACHE_FLAGS_INST_CACHE |
 537                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 538                .num_cu_shared = 2,
 539        },
 540        {
 541                /* Scalar L1 Data Cache per SQC */
 542                .cache_size = 16,
 543                .cache_level = 1,
 544                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 545                                CRAT_CACHE_FLAGS_DATA_CACHE |
 546                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 547                .num_cu_shared = 2,
 548        },
 549        {
 550                /* GL1 Data Cache per SA */
 551                .cache_size = 128,
 552                .cache_level = 1,
 553                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 554                                CRAT_CACHE_FLAGS_DATA_CACHE |
 555                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 556                .num_cu_shared = 10,
 557        },
 558        {
 559                /* L2 Data Cache per GPU (Total Tex Cache) */
 560                .cache_size = 4096,
 561                .cache_level = 2,
 562                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 563                                CRAT_CACHE_FLAGS_DATA_CACHE |
 564                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 565                .num_cu_shared = 10,
 566        },
 567        {
 568                /* L3 Data Cache per GPU */
 569                .cache_size = 128*1024,
 570                .cache_level = 3,
 571                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 572                                CRAT_CACHE_FLAGS_DATA_CACHE |
 573                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 574                .num_cu_shared = 10,
 575        },
 576};
 577
 578static struct kfd_gpu_cache_info navy_flounder_cache_info[] = {
 579        {
 580                /* TCP L1 Cache per CU */
 581                .cache_size = 16,
 582                .cache_level = 1,
 583                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 584                                CRAT_CACHE_FLAGS_DATA_CACHE |
 585                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 586                .num_cu_shared = 1,
 587        },
 588        {
 589                /* Scalar L1 Instruction Cache per SQC */
 590                .cache_size = 32,
 591                .cache_level = 1,
 592                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 593                                CRAT_CACHE_FLAGS_INST_CACHE |
 594                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 595                .num_cu_shared = 2,
 596        },
 597        {
 598                /* Scalar L1 Data Cache per SQC */
 599                .cache_size = 16,
 600                .cache_level = 1,
 601                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 602                                CRAT_CACHE_FLAGS_DATA_CACHE |
 603                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 604                .num_cu_shared = 2,
 605        },
 606        {
 607                /* GL1 Data Cache per SA */
 608                .cache_size = 128,
 609                .cache_level = 1,
 610                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 611                                CRAT_CACHE_FLAGS_DATA_CACHE |
 612                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 613                .num_cu_shared = 10,
 614        },
 615        {
 616                /* L2 Data Cache per GPU (Total Tex Cache) */
 617                .cache_size = 3072,
 618                .cache_level = 2,
 619                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 620                                CRAT_CACHE_FLAGS_DATA_CACHE |
 621                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 622                .num_cu_shared = 10,
 623        },
 624        {
 625                /* L3 Data Cache per GPU */
 626                .cache_size = 96*1024,
 627                .cache_level = 3,
 628                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 629                                CRAT_CACHE_FLAGS_DATA_CACHE |
 630                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 631                .num_cu_shared = 10,
 632        },
 633};
 634
 635static struct kfd_gpu_cache_info dimgrey_cavefish_cache_info[] = {
 636        {
 637                /* TCP L1 Cache per CU */
 638                .cache_size = 16,
 639                .cache_level = 1,
 640                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 641                                CRAT_CACHE_FLAGS_DATA_CACHE |
 642                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 643                .num_cu_shared = 1,
 644        },
 645        {
 646                /* Scalar L1 Instruction Cache per SQC */
 647                .cache_size = 32,
 648                .cache_level = 1,
 649                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 650                                CRAT_CACHE_FLAGS_INST_CACHE |
 651                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 652                .num_cu_shared = 2,
 653        },
 654        {
 655                /* Scalar L1 Data Cache per SQC */
 656                .cache_size = 16,
 657                .cache_level = 1,
 658                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 659                                CRAT_CACHE_FLAGS_DATA_CACHE |
 660                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 661                .num_cu_shared = 2,
 662        },
 663        {
 664                /* GL1 Data Cache per SA */
 665                .cache_size = 128,
 666                .cache_level = 1,
 667                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 668                                CRAT_CACHE_FLAGS_DATA_CACHE |
 669                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 670                .num_cu_shared = 8,
 671        },
 672        {
 673                /* L2 Data Cache per GPU (Total Tex Cache) */
 674                .cache_size = 2048,
 675                .cache_level = 2,
 676                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 677                                CRAT_CACHE_FLAGS_DATA_CACHE |
 678                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 679                .num_cu_shared = 8,
 680        },
 681        {
 682                /* L3 Data Cache per GPU */
 683                .cache_size = 32*1024,
 684                .cache_level = 3,
 685                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 686                                CRAT_CACHE_FLAGS_DATA_CACHE |
 687                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 688                .num_cu_shared = 8,
 689        },
 690};
 691
 692static struct kfd_gpu_cache_info beige_goby_cache_info[] = {
 693        {
 694                /* TCP L1 Cache per CU */
 695                .cache_size = 16,
 696                .cache_level = 1,
 697                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 698                                CRAT_CACHE_FLAGS_DATA_CACHE |
 699                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 700                .num_cu_shared = 1,
 701        },
 702        {
 703                /* Scalar L1 Instruction Cache per SQC */
 704                .cache_size = 32,
 705                .cache_level = 1,
 706                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 707                                CRAT_CACHE_FLAGS_INST_CACHE |
 708                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 709                .num_cu_shared = 2,
 710        },
 711        {
 712                /* Scalar L1 Data Cache per SQC */
 713                .cache_size = 16,
 714                .cache_level = 1,
 715                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 716                                CRAT_CACHE_FLAGS_DATA_CACHE |
 717                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 718                .num_cu_shared = 2,
 719        },
 720        {
 721                /* GL1 Data Cache per SA */
 722                .cache_size = 128,
 723                .cache_level = 1,
 724                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 725                                CRAT_CACHE_FLAGS_DATA_CACHE |
 726                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 727                .num_cu_shared = 8,
 728        },
 729        {
 730                /* L2 Data Cache per GPU (Total Tex Cache) */
 731                .cache_size = 1024,
 732                .cache_level = 2,
 733                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 734                                CRAT_CACHE_FLAGS_DATA_CACHE |
 735                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 736                .num_cu_shared = 8,
 737        },
 738        {
 739                /* L3 Data Cache per GPU */
 740                .cache_size = 16*1024,
 741                .cache_level = 3,
 742                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 743                                CRAT_CACHE_FLAGS_DATA_CACHE |
 744                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 745                .num_cu_shared = 8,
 746        },
 747};
 748
 749static struct kfd_gpu_cache_info yellow_carp_cache_info[] = {
 750        {
 751                /* TCP L1 Cache per CU */
 752                .cache_size = 16,
 753                .cache_level = 1,
 754                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 755                                CRAT_CACHE_FLAGS_DATA_CACHE |
 756                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 757                .num_cu_shared = 1,
 758        },
 759        {
 760                /* Scalar L1 Instruction Cache per SQC */
 761                .cache_size = 32,
 762                .cache_level = 1,
 763                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 764                                CRAT_CACHE_FLAGS_INST_CACHE |
 765                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 766                .num_cu_shared = 2,
 767        },
 768        {
 769                /* Scalar L1 Data Cache per SQC */
 770                .cache_size = 16,
 771                .cache_level = 1,
 772                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 773                                CRAT_CACHE_FLAGS_DATA_CACHE |
 774                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 775                .num_cu_shared = 2,
 776        },
 777        {
 778                /* GL1 Data Cache per SA */
 779                .cache_size = 128,
 780                .cache_level = 1,
 781                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 782                                CRAT_CACHE_FLAGS_DATA_CACHE |
 783                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 784                .num_cu_shared = 6,
 785        },
 786        {
 787                /* L2 Data Cache per GPU (Total Tex Cache) */
 788                .cache_size = 2048,
 789                .cache_level = 2,
 790                .flags = (CRAT_CACHE_FLAGS_ENABLED |
 791                                CRAT_CACHE_FLAGS_DATA_CACHE |
 792                                CRAT_CACHE_FLAGS_SIMD_CACHE),
 793                .num_cu_shared = 6,
 794        },
 795};
 796
 797static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
 798                struct crat_subtype_computeunit *cu)
 799{
 800        dev->node_props.cpu_cores_count = cu->num_cpu_cores;
 801        dev->node_props.cpu_core_id_base = cu->processor_id_low;
 802        if (cu->hsa_capability & CRAT_CU_FLAGS_IOMMU_PRESENT)
 803                dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
 804
 805        pr_debug("CU CPU: cores=%d id_base=%d\n", cu->num_cpu_cores,
 806                        cu->processor_id_low);
 807}
 808
 809static void kfd_populated_cu_info_gpu(struct kfd_topology_device *dev,
 810                struct crat_subtype_computeunit *cu)
 811{
 812        dev->node_props.simd_id_base = cu->processor_id_low;
 813        dev->node_props.simd_count = cu->num_simd_cores;
 814        dev->node_props.lds_size_in_kb = cu->lds_size_in_kb;
 815        dev->node_props.max_waves_per_simd = cu->max_waves_simd;
 816        dev->node_props.wave_front_size = cu->wave_front_size;
 817        dev->node_props.array_count = cu->array_count;
 818        dev->node_props.cu_per_simd_array = cu->num_cu_per_array;
 819        dev->node_props.simd_per_cu = cu->num_simd_per_cu;
 820        dev->node_props.max_slots_scratch_cu = cu->max_slots_scatch_cu;
 821        if (cu->hsa_capability & CRAT_CU_FLAGS_HOT_PLUGGABLE)
 822                dev->node_props.capability |= HSA_CAP_HOT_PLUGGABLE;
 823        pr_debug("CU GPU: id_base=%d\n", cu->processor_id_low);
 824}
 825
 826/* kfd_parse_subtype_cu - parse compute unit subtypes and attach it to correct
 827 * topology device present in the device_list
 828 */
 829static int kfd_parse_subtype_cu(struct crat_subtype_computeunit *cu,
 830                                struct list_head *device_list)
 831{
 832        struct kfd_topology_device *dev;
 833
 834        pr_debug("Found CU entry in CRAT table with proximity_domain=%d caps=%x\n",
 835                        cu->proximity_domain, cu->hsa_capability);
 836        list_for_each_entry(dev, device_list, list) {
 837                if (cu->proximity_domain == dev->proximity_domain) {
 838                        if (cu->flags & CRAT_CU_FLAGS_CPU_PRESENT)
 839                                kfd_populated_cu_info_cpu(dev, cu);
 840
 841                        if (cu->flags & CRAT_CU_FLAGS_GPU_PRESENT)
 842                                kfd_populated_cu_info_gpu(dev, cu);
 843                        break;
 844                }
 845        }
 846
 847        return 0;
 848}
 849
 850static struct kfd_mem_properties *
 851find_subtype_mem(uint32_t heap_type, uint32_t flags, uint32_t width,
 852                struct kfd_topology_device *dev)
 853{
 854        struct kfd_mem_properties *props;
 855
 856        list_for_each_entry(props, &dev->mem_props, list) {
 857                if (props->heap_type == heap_type
 858                                && props->flags == flags
 859                                && props->width == width)
 860                        return props;
 861        }
 862
 863        return NULL;
 864}
 865/* kfd_parse_subtype_mem - parse memory subtypes and attach it to correct
 866 * topology device present in the device_list
 867 */
 868static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem,
 869                                struct list_head *device_list)
 870{
 871        struct kfd_mem_properties *props;
 872        struct kfd_topology_device *dev;
 873        uint32_t heap_type;
 874        uint64_t size_in_bytes;
 875        uint32_t flags = 0;
 876        uint32_t width;
 877
 878        pr_debug("Found memory entry in CRAT table with proximity_domain=%d\n",
 879                        mem->proximity_domain);
 880        list_for_each_entry(dev, device_list, list) {
 881                if (mem->proximity_domain == dev->proximity_domain) {
 882                        /* We're on GPU node */
 883                        if (dev->node_props.cpu_cores_count == 0) {
 884                                /* APU */
 885                                if (mem->visibility_type == 0)
 886                                        heap_type =
 887                                                HSA_MEM_HEAP_TYPE_FB_PRIVATE;
 888                                /* dGPU */
 889                                else
 890                                        heap_type = mem->visibility_type;
 891                        } else
 892                                heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
 893
 894                        if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE)
 895                                flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE;
 896                        if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE)
 897                                flags |= HSA_MEM_FLAGS_NON_VOLATILE;
 898
 899                        size_in_bytes =
 900                                ((uint64_t)mem->length_high << 32) +
 901                                                        mem->length_low;
 902                        width = mem->width;
 903
 904                        /* Multiple banks of the same type are aggregated into
 905                         * one. User mode doesn't care about multiple physical
 906                         * memory segments. It's managed as a single virtual
 907                         * heap for user mode.
 908                         */
 909                        props = find_subtype_mem(heap_type, flags, width, dev);
 910                        if (props) {
 911                                props->size_in_bytes += size_in_bytes;
 912                                break;
 913                        }
 914
 915                        props = kfd_alloc_struct(props);
 916                        if (!props)
 917                                return -ENOMEM;
 918
 919                        props->heap_type = heap_type;
 920                        props->flags = flags;
 921                        props->size_in_bytes = size_in_bytes;
 922                        props->width = width;
 923
 924                        dev->node_props.mem_banks_count++;
 925                        list_add_tail(&props->list, &dev->mem_props);
 926
 927                        break;
 928                }
 929        }
 930
 931        return 0;
 932}
 933
 934/* kfd_parse_subtype_cache - parse cache subtypes and attach it to correct
 935 * topology device present in the device_list
 936 */
 937static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache,
 938                        struct list_head *device_list)
 939{
 940        struct kfd_cache_properties *props;
 941        struct kfd_topology_device *dev;
 942        uint32_t id;
 943        uint32_t total_num_of_cu;
 944
 945        id = cache->processor_id_low;
 946
 947        pr_debug("Found cache entry in CRAT table with processor_id=%d\n", id);
 948        list_for_each_entry(dev, device_list, list) {
 949                total_num_of_cu = (dev->node_props.array_count *
 950                                        dev->node_props.cu_per_simd_array);
 951
 952                /* Cache infomration in CRAT doesn't have proximity_domain
 953                 * information as it is associated with a CPU core or GPU
 954                 * Compute Unit. So map the cache using CPU core Id or SIMD
 955                 * (GPU) ID.
 956                 * TODO: This works because currently we can safely assume that
 957                 *  Compute Units are parsed before caches are parsed. In
 958                 *  future, remove this dependency
 959                 */
 960                if ((id >= dev->node_props.cpu_core_id_base &&
 961                        id <= dev->node_props.cpu_core_id_base +
 962                                dev->node_props.cpu_cores_count) ||
 963                        (id >= dev->node_props.simd_id_base &&
 964                        id < dev->node_props.simd_id_base +
 965                                total_num_of_cu)) {
 966                        props = kfd_alloc_struct(props);
 967                        if (!props)
 968                                return -ENOMEM;
 969
 970                        props->processor_id_low = id;
 971                        props->cache_level = cache->cache_level;
 972                        props->cache_size = cache->cache_size;
 973                        props->cacheline_size = cache->cache_line_size;
 974                        props->cachelines_per_tag = cache->lines_per_tag;
 975                        props->cache_assoc = cache->associativity;
 976                        props->cache_latency = cache->cache_latency;
 977                        memcpy(props->sibling_map, cache->sibling_map,
 978                                        sizeof(props->sibling_map));
 979
 980                        if (cache->flags & CRAT_CACHE_FLAGS_DATA_CACHE)
 981                                props->cache_type |= HSA_CACHE_TYPE_DATA;
 982                        if (cache->flags & CRAT_CACHE_FLAGS_INST_CACHE)
 983                                props->cache_type |= HSA_CACHE_TYPE_INSTRUCTION;
 984                        if (cache->flags & CRAT_CACHE_FLAGS_CPU_CACHE)
 985                                props->cache_type |= HSA_CACHE_TYPE_CPU;
 986                        if (cache->flags & CRAT_CACHE_FLAGS_SIMD_CACHE)
 987                                props->cache_type |= HSA_CACHE_TYPE_HSACU;
 988
 989                        dev->cache_count++;
 990                        dev->node_props.caches_count++;
 991                        list_add_tail(&props->list, &dev->cache_props);
 992
 993                        break;
 994                }
 995        }
 996
 997        return 0;
 998}
 999
1000/* kfd_parse_subtype_iolink - parse iolink subtypes and attach it to correct
1001 * topology device present in the device_list
1002 */
1003static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink,
1004                                        struct list_head *device_list)
1005{
1006        struct kfd_iolink_properties *props = NULL, *props2;
1007        struct kfd_topology_device *dev, *to_dev;
1008        uint32_t id_from;
1009        uint32_t id_to;
1010
1011        id_from = iolink->proximity_domain_from;
1012        id_to = iolink->proximity_domain_to;
1013
1014        pr_debug("Found IO link entry in CRAT table with id_from=%d, id_to %d\n",
1015                        id_from, id_to);
1016        list_for_each_entry(dev, device_list, list) {
1017                if (id_from == dev->proximity_domain) {
1018                        props = kfd_alloc_struct(props);
1019                        if (!props)
1020                                return -ENOMEM;
1021
1022                        props->node_from = id_from;
1023                        props->node_to = id_to;
1024                        props->ver_maj = iolink->version_major;
1025                        props->ver_min = iolink->version_minor;
1026                        props->iolink_type = iolink->io_interface_type;
1027
1028                        if (props->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS)
1029                                props->weight = 20;
1030                        else if (props->iolink_type == CRAT_IOLINK_TYPE_XGMI)
1031                                props->weight = 15 * iolink->num_hops_xgmi;
1032                        else
1033                                props->weight = node_distance(id_from, id_to);
1034
1035                        props->min_latency = iolink->minimum_latency;
1036                        props->max_latency = iolink->maximum_latency;
1037                        props->min_bandwidth = iolink->minimum_bandwidth_mbs;
1038                        props->max_bandwidth = iolink->maximum_bandwidth_mbs;
1039                        props->rec_transfer_size =
1040                                        iolink->recommended_transfer_size;
1041
1042                        dev->io_link_count++;
1043                        dev->node_props.io_links_count++;
1044                        list_add_tail(&props->list, &dev->io_link_props);
1045                        break;
1046                }
1047        }
1048
1049        /* CPU topology is created before GPUs are detected, so CPU->GPU
1050         * links are not built at that time. If a PCIe type is discovered, it
1051         * means a GPU is detected and we are adding GPU->CPU to the topology.
1052         * At this time, also add the corresponded CPU->GPU link if GPU
1053         * is large bar.
1054         * For xGMI, we only added the link with one direction in the crat
1055         * table, add corresponded reversed direction link now.
1056         */
1057        if (props && (iolink->flags & CRAT_IOLINK_FLAGS_BI_DIRECTIONAL)) {
1058                to_dev = kfd_topology_device_by_proximity_domain(id_to);
1059                if (!to_dev)
1060                        return -ENODEV;
1061                /* same everything but the other direction */
1062                props2 = kmemdup(props, sizeof(*props2), GFP_KERNEL);
1063                props2->node_from = id_to;
1064                props2->node_to = id_from;
1065                props2->kobj = NULL;
1066                to_dev->io_link_count++;
1067                to_dev->node_props.io_links_count++;
1068                list_add_tail(&props2->list, &to_dev->io_link_props);
1069        }
1070
1071        return 0;
1072}
1073
1074/* kfd_parse_subtype - parse subtypes and attach it to correct topology device
1075 * present in the device_list
1076 *      @sub_type_hdr - subtype section of crat_image
1077 *      @device_list - list of topology devices present in this crat_image
1078 */
1079static int kfd_parse_subtype(struct crat_subtype_generic *sub_type_hdr,
1080                                struct list_head *device_list)
1081{
1082        struct crat_subtype_computeunit *cu;
1083        struct crat_subtype_memory *mem;
1084        struct crat_subtype_cache *cache;
1085        struct crat_subtype_iolink *iolink;
1086        int ret = 0;
1087
1088        switch (sub_type_hdr->type) {
1089        case CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY:
1090                cu = (struct crat_subtype_computeunit *)sub_type_hdr;
1091                ret = kfd_parse_subtype_cu(cu, device_list);
1092                break;
1093        case CRAT_SUBTYPE_MEMORY_AFFINITY:
1094                mem = (struct crat_subtype_memory *)sub_type_hdr;
1095                ret = kfd_parse_subtype_mem(mem, device_list);
1096                break;
1097        case CRAT_SUBTYPE_CACHE_AFFINITY:
1098                cache = (struct crat_subtype_cache *)sub_type_hdr;
1099                ret = kfd_parse_subtype_cache(cache, device_list);
1100                break;
1101        case CRAT_SUBTYPE_TLB_AFFINITY:
1102                /*
1103                 * For now, nothing to do here
1104                 */
1105                pr_debug("Found TLB entry in CRAT table (not processing)\n");
1106                break;
1107        case CRAT_SUBTYPE_CCOMPUTE_AFFINITY:
1108                /*
1109                 * For now, nothing to do here
1110                 */
1111                pr_debug("Found CCOMPUTE entry in CRAT table (not processing)\n");
1112                break;
1113        case CRAT_SUBTYPE_IOLINK_AFFINITY:
1114                iolink = (struct crat_subtype_iolink *)sub_type_hdr;
1115                ret = kfd_parse_subtype_iolink(iolink, device_list);
1116                break;
1117        default:
1118                pr_warn("Unknown subtype %d in CRAT\n",
1119                                sub_type_hdr->type);
1120        }
1121
1122        return ret;
1123}
1124
1125/* kfd_parse_crat_table - parse CRAT table. For each node present in CRAT
1126 * create a kfd_topology_device and add in to device_list. Also parse
1127 * CRAT subtypes and attach it to appropriate kfd_topology_device
1128 *      @crat_image - input image containing CRAT
1129 *      @device_list - [OUT] list of kfd_topology_device generated after
1130 *                     parsing crat_image
1131 *      @proximity_domain - Proximity domain of the first device in the table
1132 *
1133 *      Return - 0 if successful else -ve value
1134 */
1135int kfd_parse_crat_table(void *crat_image, struct list_head *device_list,
1136                         uint32_t proximity_domain)
1137{
1138        struct kfd_topology_device *top_dev = NULL;
1139        struct crat_subtype_generic *sub_type_hdr;
1140        uint16_t node_id;
1141        int ret = 0;
1142        struct crat_header *crat_table = (struct crat_header *)crat_image;
1143        uint16_t num_nodes;
1144        uint32_t image_len;
1145
1146        if (!crat_image)
1147                return -EINVAL;
1148
1149        if (!list_empty(device_list)) {
1150                pr_warn("Error device list should be empty\n");
1151                return -EINVAL;
1152        }
1153
1154        num_nodes = crat_table->num_domains;
1155        image_len = crat_table->length;
1156
1157        pr_debug("Parsing CRAT table with %d nodes\n", num_nodes);
1158
1159        for (node_id = 0; node_id < num_nodes; node_id++) {
1160                top_dev = kfd_create_topology_device(device_list);
1161                if (!top_dev)
1162                        break;
1163                top_dev->proximity_domain = proximity_domain++;
1164        }
1165
1166        if (!top_dev) {
1167                ret = -ENOMEM;
1168                goto err;
1169        }
1170
1171        memcpy(top_dev->oem_id, crat_table->oem_id, CRAT_OEMID_LENGTH);
1172        memcpy(top_dev->oem_table_id, crat_table->oem_table_id,
1173                        CRAT_OEMTABLEID_LENGTH);
1174        top_dev->oem_revision = crat_table->oem_revision;
1175
1176        sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
1177        while ((char *)sub_type_hdr + sizeof(struct crat_subtype_generic) <
1178                        ((char *)crat_image) + image_len) {
1179                if (sub_type_hdr->flags & CRAT_SUBTYPE_FLAGS_ENABLED) {
1180                        ret = kfd_parse_subtype(sub_type_hdr, device_list);
1181                        if (ret)
1182                                break;
1183                }
1184
1185                sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
1186                                sub_type_hdr->length);
1187        }
1188
1189err:
1190        if (ret)
1191                kfd_release_topology_device_list(device_list);
1192
1193        return ret;
1194}
1195
1196/* Helper function. See kfd_fill_gpu_cache_info for parameter description */
1197static int fill_in_l1_pcache(struct crat_subtype_cache *pcache,
1198                                struct kfd_gpu_cache_info *pcache_info,
1199                                struct kfd_cu_info *cu_info,
1200                                int mem_available,
1201                                int cu_bitmask,
1202                                int cache_type, unsigned int cu_processor_id,
1203                                int cu_block)
1204{
1205        unsigned int cu_sibling_map_mask;
1206        int first_active_cu;
1207
1208        /* First check if enough memory is available */
1209        if (sizeof(struct crat_subtype_cache) > mem_available)
1210                return -ENOMEM;
1211
1212        cu_sibling_map_mask = cu_bitmask;
1213        cu_sibling_map_mask >>= cu_block;
1214        cu_sibling_map_mask &=
1215                ((1 << pcache_info[cache_type].num_cu_shared) - 1);
1216        first_active_cu = ffs(cu_sibling_map_mask);
1217
1218        /* CU could be inactive. In case of shared cache find the first active
1219         * CU. and incase of non-shared cache check if the CU is inactive. If
1220         * inactive active skip it
1221         */
1222        if (first_active_cu) {
1223                memset(pcache, 0, sizeof(struct crat_subtype_cache));
1224                pcache->type = CRAT_SUBTYPE_CACHE_AFFINITY;
1225                pcache->length = sizeof(struct crat_subtype_cache);
1226                pcache->flags = pcache_info[cache_type].flags;
1227                pcache->processor_id_low = cu_processor_id
1228                                         + (first_active_cu - 1);
1229                pcache->cache_level = pcache_info[cache_type].cache_level;
1230                pcache->cache_size = pcache_info[cache_type].cache_size;
1231
1232                /* Sibling map is w.r.t processor_id_low, so shift out
1233                 * inactive CU
1234                 */
1235                cu_sibling_map_mask =
1236                        cu_sibling_map_mask >> (first_active_cu - 1);
1237
1238                pcache->sibling_map[0] = (uint8_t)(cu_sibling_map_mask & 0xFF);
1239                pcache->sibling_map[1] =
1240                                (uint8_t)((cu_sibling_map_mask >> 8) & 0xFF);
1241                pcache->sibling_map[2] =
1242                                (uint8_t)((cu_sibling_map_mask >> 16) & 0xFF);
1243                pcache->sibling_map[3] =
1244                                (uint8_t)((cu_sibling_map_mask >> 24) & 0xFF);
1245                return 0;
1246        }
1247        return 1;
1248}
1249
1250/* Helper function. See kfd_fill_gpu_cache_info for parameter description */
1251static int fill_in_l2_l3_pcache(struct crat_subtype_cache *pcache,
1252                                struct kfd_gpu_cache_info *pcache_info,
1253                                struct kfd_cu_info *cu_info,
1254                                int mem_available,
1255                                int cache_type, unsigned int cu_processor_id)
1256{
1257        unsigned int cu_sibling_map_mask;
1258        int first_active_cu;
1259        int i, j, k;
1260
1261        /* First check if enough memory is available */
1262        if (sizeof(struct crat_subtype_cache) > mem_available)
1263                return -ENOMEM;
1264
1265        cu_sibling_map_mask = cu_info->cu_bitmap[0][0];
1266        cu_sibling_map_mask &=
1267                ((1 << pcache_info[cache_type].num_cu_shared) - 1);
1268        first_active_cu = ffs(cu_sibling_map_mask);
1269
1270        /* CU could be inactive. In case of shared cache find the first active
1271         * CU. and incase of non-shared cache check if the CU is inactive. If
1272         * inactive active skip it
1273         */
1274        if (first_active_cu) {
1275                memset(pcache, 0, sizeof(struct crat_subtype_cache));
1276                pcache->type = CRAT_SUBTYPE_CACHE_AFFINITY;
1277                pcache->length = sizeof(struct crat_subtype_cache);
1278                pcache->flags = pcache_info[cache_type].flags;
1279                pcache->processor_id_low = cu_processor_id
1280                                         + (first_active_cu - 1);
1281                pcache->cache_level = pcache_info[cache_type].cache_level;
1282                pcache->cache_size = pcache_info[cache_type].cache_size;
1283
1284                /* Sibling map is w.r.t processor_id_low, so shift out
1285                 * inactive CU
1286                 */
1287                cu_sibling_map_mask =
1288                        cu_sibling_map_mask >> (first_active_cu - 1);
1289                k = 0;
1290                for (i = 0; i < cu_info->num_shader_engines; i++) {
1291                        for (j = 0; j < cu_info->num_shader_arrays_per_engine;
1292                                j++) {
1293                                pcache->sibling_map[k] =
1294                                 (uint8_t)(cu_sibling_map_mask & 0xFF);
1295                                pcache->sibling_map[k+1] =
1296                                 (uint8_t)((cu_sibling_map_mask >> 8) & 0xFF);
1297                                pcache->sibling_map[k+2] =
1298                                 (uint8_t)((cu_sibling_map_mask >> 16) & 0xFF);
1299                                pcache->sibling_map[k+3] =
1300                                 (uint8_t)((cu_sibling_map_mask >> 24) & 0xFF);
1301                                k += 4;
1302                                cu_sibling_map_mask =
1303                                        cu_info->cu_bitmap[i % 4][j + i / 4];
1304                                cu_sibling_map_mask &= (
1305                                 (1 << pcache_info[cache_type].num_cu_shared)
1306                                 - 1);
1307                        }
1308                }
1309                return 0;
1310        }
1311        return 1;
1312}
1313
1314/* kfd_fill_gpu_cache_info - Fill GPU cache info using kfd_gpu_cache_info
1315 * tables
1316 *
1317 *      @kdev - [IN] GPU device
1318 *      @gpu_processor_id - [IN] GPU processor ID to which these caches
1319 *                          associate
1320 *      @available_size - [IN] Amount of memory available in pcache
1321 *      @cu_info - [IN] Compute Unit info obtained from KGD
1322 *      @pcache - [OUT] memory into which cache data is to be filled in.
1323 *      @size_filled - [OUT] amount of data used up in pcache.
1324 *      @num_of_entries - [OUT] number of caches added
1325 */
1326static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev,
1327                        int gpu_processor_id,
1328                        int available_size,
1329                        struct kfd_cu_info *cu_info,
1330                        struct crat_subtype_cache *pcache,
1331                        int *size_filled,
1332                        int *num_of_entries)
1333{
1334        struct kfd_gpu_cache_info *pcache_info;
1335        int num_of_cache_types = 0;
1336        int i, j, k;
1337        int ct = 0;
1338        int mem_available = available_size;
1339        unsigned int cu_processor_id;
1340        int ret;
1341        unsigned int num_cu_shared;
1342
1343        switch (kdev->device_info->asic_family) {
1344        case CHIP_KAVERI:
1345                pcache_info = kaveri_cache_info;
1346                num_of_cache_types = ARRAY_SIZE(kaveri_cache_info);
1347                break;
1348        case CHIP_HAWAII:
1349                pcache_info = hawaii_cache_info;
1350                num_of_cache_types = ARRAY_SIZE(hawaii_cache_info);
1351                break;
1352        case CHIP_CARRIZO:
1353                pcache_info = carrizo_cache_info;
1354                num_of_cache_types = ARRAY_SIZE(carrizo_cache_info);
1355                break;
1356        case CHIP_TONGA:
1357                pcache_info = tonga_cache_info;
1358                num_of_cache_types = ARRAY_SIZE(tonga_cache_info);
1359                break;
1360        case CHIP_FIJI:
1361                pcache_info = fiji_cache_info;
1362                num_of_cache_types = ARRAY_SIZE(fiji_cache_info);
1363                break;
1364        case CHIP_POLARIS10:
1365                pcache_info = polaris10_cache_info;
1366                num_of_cache_types = ARRAY_SIZE(polaris10_cache_info);
1367                break;
1368        case CHIP_POLARIS11:
1369                pcache_info = polaris11_cache_info;
1370                num_of_cache_types = ARRAY_SIZE(polaris11_cache_info);
1371                break;
1372        case CHIP_POLARIS12:
1373                pcache_info = polaris12_cache_info;
1374                num_of_cache_types = ARRAY_SIZE(polaris12_cache_info);
1375                break;
1376        case CHIP_VEGAM:
1377                pcache_info = vegam_cache_info;
1378                num_of_cache_types = ARRAY_SIZE(vegam_cache_info);
1379                break;
1380        case CHIP_VEGA10:
1381                pcache_info = vega10_cache_info;
1382                num_of_cache_types = ARRAY_SIZE(vega10_cache_info);
1383                break;
1384        case CHIP_VEGA12:
1385                pcache_info = vega12_cache_info;
1386                num_of_cache_types = ARRAY_SIZE(vega12_cache_info);
1387                break;
1388        case CHIP_VEGA20:
1389        case CHIP_ARCTURUS:
1390                pcache_info = vega20_cache_info;
1391                num_of_cache_types = ARRAY_SIZE(vega20_cache_info);
1392                break;
1393        case CHIP_ALDEBARAN:
1394                pcache_info = aldebaran_cache_info;
1395                num_of_cache_types = ARRAY_SIZE(aldebaran_cache_info);
1396                break;
1397        case CHIP_RAVEN:
1398                pcache_info = raven_cache_info;
1399                num_of_cache_types = ARRAY_SIZE(raven_cache_info);
1400                break;
1401        case CHIP_RENOIR:
1402                pcache_info = renoir_cache_info;
1403                num_of_cache_types = ARRAY_SIZE(renoir_cache_info);
1404                break;
1405        case CHIP_NAVI10:
1406        case CHIP_NAVI12:
1407                pcache_info = navi10_cache_info;
1408                num_of_cache_types = ARRAY_SIZE(navi10_cache_info);
1409                break;
1410        case CHIP_NAVI14:
1411                pcache_info = navi14_cache_info;
1412                num_of_cache_types = ARRAY_SIZE(navi14_cache_info);
1413                break;
1414        case CHIP_SIENNA_CICHLID:
1415                pcache_info = sienna_cichlid_cache_info;
1416                num_of_cache_types = ARRAY_SIZE(sienna_cichlid_cache_info);
1417                break;
1418        case CHIP_NAVY_FLOUNDER:
1419                pcache_info = navy_flounder_cache_info;
1420                num_of_cache_types = ARRAY_SIZE(navy_flounder_cache_info);
1421                break;
1422        case CHIP_DIMGREY_CAVEFISH:
1423                pcache_info = dimgrey_cavefish_cache_info;
1424                num_of_cache_types = ARRAY_SIZE(dimgrey_cavefish_cache_info);
1425                break;
1426        case CHIP_VANGOGH:
1427                pcache_info = vangogh_cache_info;
1428                num_of_cache_types = ARRAY_SIZE(vangogh_cache_info);
1429                break;
1430        case CHIP_BEIGE_GOBY:
1431                pcache_info = beige_goby_cache_info;
1432                num_of_cache_types = ARRAY_SIZE(beige_goby_cache_info);
1433                break;
1434        case CHIP_YELLOW_CARP:
1435                pcache_info = yellow_carp_cache_info;
1436                num_of_cache_types = ARRAY_SIZE(yellow_carp_cache_info);
1437                break;
1438        default:
1439                return -EINVAL;
1440        }
1441
1442        *size_filled = 0;
1443        *num_of_entries = 0;
1444
1445        /* For each type of cache listed in the kfd_gpu_cache_info table,
1446         * go through all available Compute Units.
1447         * The [i,j,k] loop will
1448         *              if kfd_gpu_cache_info.num_cu_shared = 1
1449         *                      will parse through all available CU
1450         *              If (kfd_gpu_cache_info.num_cu_shared != 1)
1451         *                      then it will consider only one CU from
1452         *                      the shared unit
1453         */
1454
1455        for (ct = 0; ct < num_of_cache_types; ct++) {
1456          cu_processor_id = gpu_processor_id;
1457          if (pcache_info[ct].cache_level == 1) {
1458            for (i = 0; i < cu_info->num_shader_engines; i++) {
1459              for (j = 0; j < cu_info->num_shader_arrays_per_engine; j++) {
1460                for (k = 0; k < cu_info->num_cu_per_sh;
1461                  k += pcache_info[ct].num_cu_shared) {
1462                  ret = fill_in_l1_pcache(pcache,
1463                                        pcache_info,
1464                                        cu_info,
1465                                        mem_available,
1466                                        cu_info->cu_bitmap[i % 4][j + i / 4],
1467                                        ct,
1468                                        cu_processor_id,
1469                                        k);
1470
1471                  if (ret < 0)
1472                        break;
1473
1474                  if (!ret) {
1475                                pcache++;
1476                                (*num_of_entries)++;
1477                                mem_available -= sizeof(*pcache);
1478                                (*size_filled) += sizeof(*pcache);
1479                  }
1480
1481                  /* Move to next CU block */
1482                  num_cu_shared = ((k + pcache_info[ct].num_cu_shared) <=
1483                                        cu_info->num_cu_per_sh) ?
1484                                        pcache_info[ct].num_cu_shared :
1485                                        (cu_info->num_cu_per_sh - k);
1486                  cu_processor_id += num_cu_shared;
1487                }
1488              }
1489            }
1490          } else {
1491                        ret = fill_in_l2_l3_pcache(pcache,
1492                                pcache_info,
1493                                cu_info,
1494                                mem_available,
1495                                ct,
1496                                cu_processor_id);
1497
1498                        if (ret < 0)
1499                                break;
1500
1501                        if (!ret) {
1502                                pcache++;
1503                                (*num_of_entries)++;
1504                                mem_available -= sizeof(*pcache);
1505                                (*size_filled) += sizeof(*pcache);
1506                        }
1507          }
1508        }
1509
1510        pr_debug("Added [%d] GPU cache entries\n", *num_of_entries);
1511
1512        return 0;
1513}
1514
1515static bool kfd_ignore_crat(void)
1516{
1517        bool ret;
1518
1519        if (ignore_crat)
1520                return true;
1521
1522#ifndef KFD_SUPPORT_IOMMU_V2
1523        ret = true;
1524#else
1525        ret = false;
1526#endif
1527
1528        return ret;
1529}
1530
1531/*
1532 * kfd_create_crat_image_acpi - Allocates memory for CRAT image and
1533 * copies CRAT from ACPI (if available).
1534 * NOTE: Call kfd_destroy_crat_image to free CRAT image memory
1535 *
1536 *      @crat_image: CRAT read from ACPI. If no CRAT in ACPI then
1537 *                   crat_image will be NULL
1538 *      @size: [OUT] size of crat_image
1539 *
1540 *      Return 0 if successful else return error code
1541 */
1542int kfd_create_crat_image_acpi(void **crat_image, size_t *size)
1543{
1544        struct acpi_table_header *crat_table;
1545        acpi_status status;
1546        void *pcrat_image;
1547        int rc = 0;
1548
1549        if (!crat_image)
1550                return -EINVAL;
1551
1552        *crat_image = NULL;
1553
1554        if (kfd_ignore_crat()) {
1555                pr_info("CRAT table disabled by module option\n");
1556                return -ENODATA;
1557        }
1558
1559        /* Fetch the CRAT table from ACPI */
1560        status = acpi_get_table(CRAT_SIGNATURE, 0, &crat_table);
1561        if (status == AE_NOT_FOUND) {
1562                pr_warn("CRAT table not found\n");
1563                return -ENODATA;
1564        } else if (ACPI_FAILURE(status)) {
1565                const char *err = acpi_format_exception(status);
1566
1567                pr_err("CRAT table error: %s\n", err);
1568                return -EINVAL;
1569        }
1570
1571        pcrat_image = kvmalloc(crat_table->length, GFP_KERNEL);
1572        if (!pcrat_image) {
1573                rc = -ENOMEM;
1574                goto out;
1575        }
1576
1577        memcpy(pcrat_image, crat_table, crat_table->length);
1578        *crat_image = pcrat_image;
1579        *size = crat_table->length;
1580out:
1581        acpi_put_table(crat_table);
1582        return rc;
1583}
1584
1585/* Memory required to create Virtual CRAT.
1586 * Since there is no easy way to predict the amount of memory required, the
1587 * following amount is allocated for GPU Virtual CRAT. This is
1588 * expected to cover all known conditions. But to be safe additional check
1589 * is put in the code to ensure we don't overwrite.
1590 */
1591#define VCRAT_SIZE_FOR_GPU      (4 * PAGE_SIZE)
1592
1593/* kfd_fill_cu_for_cpu - Fill in Compute info for the given CPU NUMA node
1594 *
1595 *      @numa_node_id: CPU NUMA node id
1596 *      @avail_size: Available size in the memory
1597 *      @sub_type_hdr: Memory into which compute info will be filled in
1598 *
1599 *      Return 0 if successful else return -ve value
1600 */
1601static int kfd_fill_cu_for_cpu(int numa_node_id, int *avail_size,
1602                                int proximity_domain,
1603                                struct crat_subtype_computeunit *sub_type_hdr)
1604{
1605        const struct cpumask *cpumask;
1606
1607        *avail_size -= sizeof(struct crat_subtype_computeunit);
1608        if (*avail_size < 0)
1609                return -ENOMEM;
1610
1611        memset(sub_type_hdr, 0, sizeof(struct crat_subtype_computeunit));
1612
1613        /* Fill in subtype header data */
1614        sub_type_hdr->type = CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY;
1615        sub_type_hdr->length = sizeof(struct crat_subtype_computeunit);
1616        sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
1617
1618        cpumask = cpumask_of_node(numa_node_id);
1619
1620        /* Fill in CU data */
1621        sub_type_hdr->flags |= CRAT_CU_FLAGS_CPU_PRESENT;
1622        sub_type_hdr->proximity_domain = proximity_domain;
1623        sub_type_hdr->processor_id_low = kfd_numa_node_to_apic_id(numa_node_id);
1624        if (sub_type_hdr->processor_id_low == -1)
1625                return -EINVAL;
1626
1627        sub_type_hdr->num_cpu_cores = cpumask_weight(cpumask);
1628
1629        return 0;
1630}
1631
1632/* kfd_fill_mem_info_for_cpu - Fill in Memory info for the given CPU NUMA node
1633 *
1634 *      @numa_node_id: CPU NUMA node id
1635 *      @avail_size: Available size in the memory
1636 *      @sub_type_hdr: Memory into which compute info will be filled in
1637 *
1638 *      Return 0 if successful else return -ve value
1639 */
1640static int kfd_fill_mem_info_for_cpu(int numa_node_id, int *avail_size,
1641                        int proximity_domain,
1642                        struct crat_subtype_memory *sub_type_hdr)
1643{
1644        uint64_t mem_in_bytes = 0;
1645        pg_data_t *pgdat;
1646        int zone_type;
1647
1648        *avail_size -= sizeof(struct crat_subtype_memory);
1649        if (*avail_size < 0)
1650                return -ENOMEM;
1651
1652        memset(sub_type_hdr, 0, sizeof(struct crat_subtype_memory));
1653
1654        /* Fill in subtype header data */
1655        sub_type_hdr->type = CRAT_SUBTYPE_MEMORY_AFFINITY;
1656        sub_type_hdr->length = sizeof(struct crat_subtype_memory);
1657        sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
1658
1659        /* Fill in Memory Subunit data */
1660
1661        /* Unlike si_meminfo, si_meminfo_node is not exported. So
1662         * the following lines are duplicated from si_meminfo_node
1663         * function
1664         */
1665        pgdat = NODE_DATA(numa_node_id);
1666        for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
1667                mem_in_bytes += zone_managed_pages(&pgdat->node_zones[zone_type]);
1668        mem_in_bytes <<= PAGE_SHIFT;
1669
1670        sub_type_hdr->length_low = lower_32_bits(mem_in_bytes);
1671        sub_type_hdr->length_high = upper_32_bits(mem_in_bytes);
1672        sub_type_hdr->proximity_domain = proximity_domain;
1673
1674        return 0;
1675}
1676
1677#ifdef CONFIG_X86_64
1678static int kfd_fill_iolink_info_for_cpu(int numa_node_id, int *avail_size,
1679                                uint32_t *num_entries,
1680                                struct crat_subtype_iolink *sub_type_hdr)
1681{
1682        int nid;
1683        struct cpuinfo_x86 *c = &cpu_data(0);
1684        uint8_t link_type;
1685
1686        if (c->x86_vendor == X86_VENDOR_AMD)
1687                link_type = CRAT_IOLINK_TYPE_HYPERTRANSPORT;
1688        else
1689                link_type = CRAT_IOLINK_TYPE_QPI_1_1;
1690
1691        *num_entries = 0;
1692
1693        /* Create IO links from this node to other CPU nodes */
1694        for_each_online_node(nid) {
1695                if (nid == numa_node_id) /* node itself */
1696                        continue;
1697
1698                *avail_size -= sizeof(struct crat_subtype_iolink);
1699                if (*avail_size < 0)
1700                        return -ENOMEM;
1701
1702                memset(sub_type_hdr, 0, sizeof(struct crat_subtype_iolink));
1703
1704                /* Fill in subtype header data */
1705                sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
1706                sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
1707                sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
1708
1709                /* Fill in IO link data */
1710                sub_type_hdr->proximity_domain_from = numa_node_id;
1711                sub_type_hdr->proximity_domain_to = nid;
1712                sub_type_hdr->io_interface_type = link_type;
1713
1714                (*num_entries)++;
1715                sub_type_hdr++;
1716        }
1717
1718        return 0;
1719}
1720#endif
1721
1722/* kfd_create_vcrat_image_cpu - Create Virtual CRAT for CPU
1723 *
1724 *      @pcrat_image: Fill in VCRAT for CPU
1725 *      @size:  [IN] allocated size of crat_image.
1726 *              [OUT] actual size of data filled in crat_image
1727 */
1728static int kfd_create_vcrat_image_cpu(void *pcrat_image, size_t *size)
1729{
1730        struct crat_header *crat_table = (struct crat_header *)pcrat_image;
1731        struct acpi_table_header *acpi_table;
1732        acpi_status status;
1733        struct crat_subtype_generic *sub_type_hdr;
1734        int avail_size = *size;
1735        int numa_node_id;
1736#ifdef CONFIG_X86_64
1737        uint32_t entries = 0;
1738#endif
1739        int ret = 0;
1740
1741        if (!pcrat_image)
1742                return -EINVAL;
1743
1744        /* Fill in CRAT Header.
1745         * Modify length and total_entries as subunits are added.
1746         */
1747        avail_size -= sizeof(struct crat_header);
1748        if (avail_size < 0)
1749                return -ENOMEM;
1750
1751        memset(crat_table, 0, sizeof(struct crat_header));
1752        memcpy(&crat_table->signature, CRAT_SIGNATURE,
1753                        sizeof(crat_table->signature));
1754        crat_table->length = sizeof(struct crat_header);
1755
1756        status = acpi_get_table("DSDT", 0, &acpi_table);
1757        if (status != AE_OK)
1758                pr_warn("DSDT table not found for OEM information\n");
1759        else {
1760                crat_table->oem_revision = acpi_table->revision;
1761                memcpy(crat_table->oem_id, acpi_table->oem_id,
1762                                CRAT_OEMID_LENGTH);
1763                memcpy(crat_table->oem_table_id, acpi_table->oem_table_id,
1764                                CRAT_OEMTABLEID_LENGTH);
1765                acpi_put_table(acpi_table);
1766        }
1767        crat_table->total_entries = 0;
1768        crat_table->num_domains = 0;
1769
1770        sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
1771
1772        for_each_online_node(numa_node_id) {
1773                if (kfd_numa_node_to_apic_id(numa_node_id) == -1)
1774                        continue;
1775
1776                /* Fill in Subtype: Compute Unit */
1777                ret = kfd_fill_cu_for_cpu(numa_node_id, &avail_size,
1778                        crat_table->num_domains,
1779                        (struct crat_subtype_computeunit *)sub_type_hdr);
1780                if (ret < 0)
1781                        return ret;
1782                crat_table->length += sub_type_hdr->length;
1783                crat_table->total_entries++;
1784
1785                sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
1786                        sub_type_hdr->length);
1787
1788                /* Fill in Subtype: Memory */
1789                ret = kfd_fill_mem_info_for_cpu(numa_node_id, &avail_size,
1790                        crat_table->num_domains,
1791                        (struct crat_subtype_memory *)sub_type_hdr);
1792                if (ret < 0)
1793                        return ret;
1794                crat_table->length += sub_type_hdr->length;
1795                crat_table->total_entries++;
1796
1797                sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
1798                        sub_type_hdr->length);
1799
1800                /* Fill in Subtype: IO Link */
1801#ifdef CONFIG_X86_64
1802                ret = kfd_fill_iolink_info_for_cpu(numa_node_id, &avail_size,
1803                                &entries,
1804                                (struct crat_subtype_iolink *)sub_type_hdr);
1805                if (ret < 0)
1806                        return ret;
1807
1808                if (entries) {
1809                        crat_table->length += (sub_type_hdr->length * entries);
1810                        crat_table->total_entries += entries;
1811
1812                        sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
1813                                        sub_type_hdr->length * entries);
1814                }
1815#else
1816                pr_info("IO link not available for non x86 platforms\n");
1817#endif
1818
1819                crat_table->num_domains++;
1820        }
1821
1822        /* TODO: Add cache Subtype for CPU.
1823         * Currently, CPU cache information is available in function
1824         * detect_cache_attributes(cpu) defined in the file
1825         * ./arch/x86/kernel/cpu/intel_cacheinfo.c. This function is not
1826         * exported and to get the same information the code needs to be
1827         * duplicated.
1828         */
1829
1830        *size = crat_table->length;
1831        pr_info("Virtual CRAT table created for CPU\n");
1832
1833        return 0;
1834}
1835
1836static int kfd_fill_gpu_memory_affinity(int *avail_size,
1837                struct kfd_dev *kdev, uint8_t type, uint64_t size,
1838                struct crat_subtype_memory *sub_type_hdr,
1839                uint32_t proximity_domain,
1840                const struct kfd_local_mem_info *local_mem_info)
1841{
1842        *avail_size -= sizeof(struct crat_subtype_memory);
1843        if (*avail_size < 0)
1844                return -ENOMEM;
1845
1846        memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_memory));
1847        sub_type_hdr->type = CRAT_SUBTYPE_MEMORY_AFFINITY;
1848        sub_type_hdr->length = sizeof(struct crat_subtype_memory);
1849        sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
1850
1851        sub_type_hdr->proximity_domain = proximity_domain;
1852
1853        pr_debug("Fill gpu memory affinity - type 0x%x size 0x%llx\n",
1854                        type, size);
1855
1856        sub_type_hdr->length_low = lower_32_bits(size);
1857        sub_type_hdr->length_high = upper_32_bits(size);
1858
1859        sub_type_hdr->width = local_mem_info->vram_width;
1860        sub_type_hdr->visibility_type = type;
1861
1862        return 0;
1863}
1864
1865#ifdef CONFIG_ACPI_NUMA
1866static void kfd_find_numa_node_in_srat(struct kfd_dev *kdev)
1867{
1868        struct acpi_table_header *table_header = NULL;
1869        struct acpi_subtable_header *sub_header = NULL;
1870        unsigned long table_end, subtable_len;
1871        u32 pci_id = pci_domain_nr(kdev->pdev->bus) << 16 |
1872                        pci_dev_id(kdev->pdev);
1873        u32 bdf;
1874        acpi_status status;
1875        struct acpi_srat_cpu_affinity *cpu;
1876        struct acpi_srat_generic_affinity *gpu;
1877        int pxm = 0, max_pxm = 0;
1878        int numa_node = NUMA_NO_NODE;
1879        bool found = false;
1880
1881        /* Fetch the SRAT table from ACPI */
1882        status = acpi_get_table(ACPI_SIG_SRAT, 0, &table_header);
1883        if (status == AE_NOT_FOUND) {
1884                pr_warn("SRAT table not found\n");
1885                return;
1886        } else if (ACPI_FAILURE(status)) {
1887                const char *err = acpi_format_exception(status);
1888                pr_err("SRAT table error: %s\n", err);
1889                return;
1890        }
1891
1892        table_end = (unsigned long)table_header + table_header->length;
1893
1894        /* Parse all entries looking for a match. */
1895        sub_header = (struct acpi_subtable_header *)
1896                        ((unsigned long)table_header +
1897                        sizeof(struct acpi_table_srat));
1898        subtable_len = sub_header->length;
1899
1900        while (((unsigned long)sub_header) + subtable_len  < table_end) {
1901                /*
1902                 * If length is 0, break from this loop to avoid
1903                 * infinite loop.
1904                 */
1905                if (subtable_len == 0) {
1906                        pr_err("SRAT invalid zero length\n");
1907                        break;
1908                }
1909
1910                switch (sub_header->type) {
1911                case ACPI_SRAT_TYPE_CPU_AFFINITY:
1912                        cpu = (struct acpi_srat_cpu_affinity *)sub_header;
1913                        pxm = *((u32 *)cpu->proximity_domain_hi) << 8 |
1914                                        cpu->proximity_domain_lo;
1915                        if (pxm > max_pxm)
1916                                max_pxm = pxm;
1917                        break;
1918                case ACPI_SRAT_TYPE_GENERIC_AFFINITY:
1919                        gpu = (struct acpi_srat_generic_affinity *)sub_header;
1920                        bdf = *((u16 *)(&gpu->device_handle[0])) << 16 |
1921                                        *((u16 *)(&gpu->device_handle[2]));
1922                        if (bdf == pci_id) {
1923                                found = true;
1924                                numa_node = pxm_to_node(gpu->proximity_domain);
1925                        }
1926                        break;
1927                default:
1928                        break;
1929                }
1930
1931                if (found)
1932                        break;
1933
1934                sub_header = (struct acpi_subtable_header *)
1935                                ((unsigned long)sub_header + subtable_len);
1936                subtable_len = sub_header->length;
1937        }
1938
1939        acpi_put_table(table_header);
1940
1941        /* Workaround bad cpu-gpu binding case */
1942        if (found && (numa_node < 0 ||
1943                        numa_node > pxm_to_node(max_pxm)))
1944                numa_node = 0;
1945
1946        if (numa_node != NUMA_NO_NODE)
1947                set_dev_node(&kdev->pdev->dev, numa_node);
1948}
1949#endif
1950
1951/* kfd_fill_gpu_direct_io_link - Fill in direct io link from GPU
1952 * to its NUMA node
1953 *      @avail_size: Available size in the memory
1954 *      @kdev - [IN] GPU device
1955 *      @sub_type_hdr: Memory into which io link info will be filled in
1956 *      @proximity_domain - proximity domain of the GPU node
1957 *
1958 *      Return 0 if successful else return -ve value
1959 */
1960static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
1961                        struct kfd_dev *kdev,
1962                        struct crat_subtype_iolink *sub_type_hdr,
1963                        uint32_t proximity_domain)
1964{
1965        struct amdgpu_device *adev = (struct amdgpu_device *)kdev->kgd;
1966
1967        *avail_size -= sizeof(struct crat_subtype_iolink);
1968        if (*avail_size < 0)
1969                return -ENOMEM;
1970
1971        memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_iolink));
1972
1973        /* Fill in subtype header data */
1974        sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
1975        sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
1976        sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
1977        if (kfd_dev_is_large_bar(kdev))
1978                sub_type_hdr->flags |= CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
1979
1980        /* Fill in IOLINK subtype.
1981         * TODO: Fill-in other fields of iolink subtype
1982         */
1983        if (adev->gmc.xgmi.connected_to_cpu) {
1984                /*
1985                 * with host gpu xgmi link, host can access gpu memory whether
1986                 * or not pcie bar type is large, so always create bidirectional
1987                 * io link.
1988                 */
1989                sub_type_hdr->flags |= CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
1990                sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_XGMI;
1991                sub_type_hdr->num_hops_xgmi = 1;
1992        } else {
1993                sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
1994        }
1995
1996        sub_type_hdr->proximity_domain_from = proximity_domain;
1997
1998#ifdef CONFIG_ACPI_NUMA
1999        if (kdev->pdev->dev.numa_node == NUMA_NO_NODE)
2000                kfd_find_numa_node_in_srat(kdev);
2001#endif
2002#ifdef CONFIG_NUMA
2003        if (kdev->pdev->dev.numa_node == NUMA_NO_NODE)
2004                sub_type_hdr->proximity_domain_to = 0;
2005        else
2006                sub_type_hdr->proximity_domain_to = kdev->pdev->dev.numa_node;
2007#else
2008        sub_type_hdr->proximity_domain_to = 0;
2009#endif
2010        return 0;
2011}
2012
2013static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size,
2014                        struct kfd_dev *kdev,
2015                        struct kfd_dev *peer_kdev,
2016                        struct crat_subtype_iolink *sub_type_hdr,
2017                        uint32_t proximity_domain_from,
2018                        uint32_t proximity_domain_to)
2019{
2020        *avail_size -= sizeof(struct crat_subtype_iolink);
2021        if (*avail_size < 0)
2022                return -ENOMEM;
2023
2024        memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_iolink));
2025
2026        sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
2027        sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
2028        sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED |
2029                               CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
2030
2031        sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_XGMI;
2032        sub_type_hdr->proximity_domain_from = proximity_domain_from;
2033        sub_type_hdr->proximity_domain_to = proximity_domain_to;
2034        sub_type_hdr->num_hops_xgmi =
2035                amdgpu_amdkfd_get_xgmi_hops_count(kdev->kgd, peer_kdev->kgd);
2036        return 0;
2037}
2038
2039/* kfd_create_vcrat_image_gpu - Create Virtual CRAT for CPU
2040 *
2041 *      @pcrat_image: Fill in VCRAT for GPU
2042 *      @size:  [IN] allocated size of crat_image.
2043 *              [OUT] actual size of data filled in crat_image
2044 */
2045static int kfd_create_vcrat_image_gpu(void *pcrat_image,
2046                                      size_t *size, struct kfd_dev *kdev,
2047                                      uint32_t proximity_domain)
2048{
2049        struct crat_header *crat_table = (struct crat_header *)pcrat_image;
2050        struct crat_subtype_generic *sub_type_hdr;
2051        struct kfd_local_mem_info local_mem_info;
2052        struct kfd_topology_device *peer_dev;
2053        struct crat_subtype_computeunit *cu;
2054        struct kfd_cu_info cu_info;
2055        int avail_size = *size;
2056        uint32_t total_num_of_cu;
2057        int num_of_cache_entries = 0;
2058        int cache_mem_filled = 0;
2059        uint32_t nid = 0;
2060        int ret = 0;
2061
2062        if (!pcrat_image || avail_size < VCRAT_SIZE_FOR_GPU)
2063                return -EINVAL;
2064
2065        /* Fill the CRAT Header.
2066         * Modify length and total_entries as subunits are added.
2067         */
2068        avail_size -= sizeof(struct crat_header);
2069        if (avail_size < 0)
2070                return -ENOMEM;
2071
2072        memset(crat_table, 0, sizeof(struct crat_header));
2073
2074        memcpy(&crat_table->signature, CRAT_SIGNATURE,
2075                        sizeof(crat_table->signature));
2076        /* Change length as we add more subtypes*/
2077        crat_table->length = sizeof(struct crat_header);
2078        crat_table->num_domains = 1;
2079        crat_table->total_entries = 0;
2080
2081        /* Fill in Subtype: Compute Unit
2082         * First fill in the sub type header and then sub type data
2083         */
2084        avail_size -= sizeof(struct crat_subtype_computeunit);
2085        if (avail_size < 0)
2086                return -ENOMEM;
2087
2088        sub_type_hdr = (struct crat_subtype_generic *)(crat_table + 1);
2089        memset(sub_type_hdr, 0, sizeof(struct crat_subtype_computeunit));
2090
2091        sub_type_hdr->type = CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY;
2092        sub_type_hdr->length = sizeof(struct crat_subtype_computeunit);
2093        sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
2094
2095        /* Fill CU subtype data */
2096        cu = (struct crat_subtype_computeunit *)sub_type_hdr;
2097        cu->flags |= CRAT_CU_FLAGS_GPU_PRESENT;
2098        cu->proximity_domain = proximity_domain;
2099
2100        amdgpu_amdkfd_get_cu_info(kdev->kgd, &cu_info);
2101        cu->num_simd_per_cu = cu_info.simd_per_cu;
2102        cu->num_simd_cores = cu_info.simd_per_cu * cu_info.cu_active_number;
2103        cu->max_waves_simd = cu_info.max_waves_per_simd;
2104
2105        cu->wave_front_size = cu_info.wave_front_size;
2106        cu->array_count = cu_info.num_shader_arrays_per_engine *
2107                cu_info.num_shader_engines;
2108        total_num_of_cu = (cu->array_count * cu_info.num_cu_per_sh);
2109        cu->processor_id_low = get_and_inc_gpu_processor_id(total_num_of_cu);
2110        cu->num_cu_per_array = cu_info.num_cu_per_sh;
2111        cu->max_slots_scatch_cu = cu_info.max_scratch_slots_per_cu;
2112        cu->num_banks = cu_info.num_shader_engines;
2113        cu->lds_size_in_kb = cu_info.lds_size;
2114
2115        cu->hsa_capability = 0;
2116
2117        /* Check if this node supports IOMMU. During parsing this flag will
2118         * translate to HSA_CAP_ATS_PRESENT
2119         */
2120        if (!kfd_iommu_check_device(kdev))
2121                cu->hsa_capability |= CRAT_CU_FLAGS_IOMMU_PRESENT;
2122
2123        crat_table->length += sub_type_hdr->length;
2124        crat_table->total_entries++;
2125
2126        /* Fill in Subtype: Memory. Only on systems with large BAR (no
2127         * private FB), report memory as public. On other systems
2128         * report the total FB size (public+private) as a single
2129         * private heap.
2130         */
2131        amdgpu_amdkfd_get_local_mem_info(kdev->kgd, &local_mem_info);
2132        sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
2133                        sub_type_hdr->length);
2134
2135        if (debug_largebar)
2136                local_mem_info.local_mem_size_private = 0;
2137
2138        if (local_mem_info.local_mem_size_private == 0)
2139                ret = kfd_fill_gpu_memory_affinity(&avail_size,
2140                                kdev, HSA_MEM_HEAP_TYPE_FB_PUBLIC,
2141                                local_mem_info.local_mem_size_public,
2142                                (struct crat_subtype_memory *)sub_type_hdr,
2143                                proximity_domain,
2144                                &local_mem_info);
2145        else
2146                ret = kfd_fill_gpu_memory_affinity(&avail_size,
2147                                kdev, HSA_MEM_HEAP_TYPE_FB_PRIVATE,
2148                                local_mem_info.local_mem_size_public +
2149                                local_mem_info.local_mem_size_private,
2150                                (struct crat_subtype_memory *)sub_type_hdr,
2151                                proximity_domain,
2152                                &local_mem_info);
2153        if (ret < 0)
2154                return ret;
2155
2156        crat_table->length += sizeof(struct crat_subtype_memory);
2157        crat_table->total_entries++;
2158
2159        /* TODO: Fill in cache information. This information is NOT readily
2160         * available in KGD
2161         */
2162        sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
2163                sub_type_hdr->length);
2164        ret = kfd_fill_gpu_cache_info(kdev, cu->processor_id_low,
2165                                avail_size,
2166                                &cu_info,
2167                                (struct crat_subtype_cache *)sub_type_hdr,
2168                                &cache_mem_filled,
2169                                &num_of_cache_entries);
2170
2171        if (ret < 0)
2172                return ret;
2173
2174        crat_table->length += cache_mem_filled;
2175        crat_table->total_entries += num_of_cache_entries;
2176        avail_size -= cache_mem_filled;
2177
2178        /* Fill in Subtype: IO_LINKS
2179         *  Only direct links are added here which is Link from GPU to
2180         *  to its NUMA node. Indirect links are added by userspace.
2181         */
2182        sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
2183                cache_mem_filled);
2184        ret = kfd_fill_gpu_direct_io_link_to_cpu(&avail_size, kdev,
2185                (struct crat_subtype_iolink *)sub_type_hdr, proximity_domain);
2186
2187        if (ret < 0)
2188                return ret;
2189
2190        crat_table->length += sub_type_hdr->length;
2191        crat_table->total_entries++;
2192
2193
2194        /* Fill in Subtype: IO_LINKS
2195         * Direct links from GPU to other GPUs through xGMI.
2196         * We will loop GPUs that already be processed (with lower value
2197         * of proximity_domain), add the link for the GPUs with same
2198         * hive id (from this GPU to other GPU) . The reversed iolink
2199         * (from other GPU to this GPU) will be added
2200         * in kfd_parse_subtype_iolink.
2201         */
2202        if (kdev->hive_id) {
2203                for (nid = 0; nid < proximity_domain; ++nid) {
2204                        peer_dev = kfd_topology_device_by_proximity_domain(nid);
2205                        if (!peer_dev->gpu)
2206                                continue;
2207                        if (peer_dev->gpu->hive_id != kdev->hive_id)
2208                                continue;
2209                        sub_type_hdr = (typeof(sub_type_hdr))(
2210                                (char *)sub_type_hdr +
2211                                sizeof(struct crat_subtype_iolink));
2212                        ret = kfd_fill_gpu_xgmi_link_to_gpu(
2213                                &avail_size, kdev, peer_dev->gpu,
2214                                (struct crat_subtype_iolink *)sub_type_hdr,
2215                                proximity_domain, nid);
2216                        if (ret < 0)
2217                                return ret;
2218                        crat_table->length += sub_type_hdr->length;
2219                        crat_table->total_entries++;
2220                }
2221        }
2222        *size = crat_table->length;
2223        pr_info("Virtual CRAT table created for GPU\n");
2224
2225        return ret;
2226}
2227
2228/* kfd_create_crat_image_virtual - Allocates memory for CRAT image and
2229 *              creates a Virtual CRAT (VCRAT) image
2230 *
2231 * NOTE: Call kfd_destroy_crat_image to free CRAT image memory
2232 *
2233 *      @crat_image: VCRAT image created because ACPI does not have a
2234 *                   CRAT for this device
2235 *      @size: [OUT] size of virtual crat_image
2236 *      @flags: COMPUTE_UNIT_CPU - Create VCRAT for CPU device
2237 *              COMPUTE_UNIT_GPU - Create VCRAT for GPU
2238 *              (COMPUTE_UNIT_CPU | COMPUTE_UNIT_GPU) - Create VCRAT for APU
2239 *                      -- this option is not currently implemented.
2240 *                      The assumption is that all AMD APUs will have CRAT
2241 *      @kdev: Valid kfd_device required if flags contain COMPUTE_UNIT_GPU
2242 *
2243 *      Return 0 if successful else return -ve value
2244 */
2245int kfd_create_crat_image_virtual(void **crat_image, size_t *size,
2246                                  int flags, struct kfd_dev *kdev,
2247                                  uint32_t proximity_domain)
2248{
2249        void *pcrat_image = NULL;
2250        int ret = 0, num_nodes;
2251        size_t dyn_size;
2252
2253        if (!crat_image)
2254                return -EINVAL;
2255
2256        *crat_image = NULL;
2257
2258        /* Allocate the CPU Virtual CRAT size based on the number of online
2259         * nodes. Allocate VCRAT_SIZE_FOR_GPU for GPU virtual CRAT image.
2260         * This should cover all the current conditions. A check is put not
2261         * to overwrite beyond allocated size for GPUs
2262         */
2263        switch (flags) {
2264        case COMPUTE_UNIT_CPU:
2265                num_nodes = num_online_nodes();
2266                dyn_size = sizeof(struct crat_header) +
2267                        num_nodes * (sizeof(struct crat_subtype_computeunit) +
2268                        sizeof(struct crat_subtype_memory) +
2269                        (num_nodes - 1) * sizeof(struct crat_subtype_iolink));
2270                pcrat_image = kvmalloc(dyn_size, GFP_KERNEL);
2271                if (!pcrat_image)
2272                        return -ENOMEM;
2273                *size = dyn_size;
2274                pr_debug("CRAT size is %ld", dyn_size);
2275                ret = kfd_create_vcrat_image_cpu(pcrat_image, size);
2276                break;
2277        case COMPUTE_UNIT_GPU:
2278                if (!kdev)
2279                        return -EINVAL;
2280                pcrat_image = kvmalloc(VCRAT_SIZE_FOR_GPU, GFP_KERNEL);
2281                if (!pcrat_image)
2282                        return -ENOMEM;
2283                *size = VCRAT_SIZE_FOR_GPU;
2284                ret = kfd_create_vcrat_image_gpu(pcrat_image, size, kdev,
2285                                                 proximity_domain);
2286                break;
2287        case (COMPUTE_UNIT_CPU | COMPUTE_UNIT_GPU):
2288                /* TODO: */
2289                ret = -EINVAL;
2290                pr_err("VCRAT not implemented for APU\n");
2291                break;
2292        default:
2293                ret = -EINVAL;
2294        }
2295
2296        if (!ret)
2297                *crat_image = pcrat_image;
2298        else
2299                kvfree(pcrat_image);
2300
2301        return ret;
2302}
2303
2304
2305/* kfd_destroy_crat_image
2306 *
2307 *      @crat_image: [IN] - crat_image from kfd_create_crat_image_xxx(..)
2308 *
2309 */
2310void kfd_destroy_crat_image(void *crat_image)
2311{
2312        kvfree(crat_image);
2313}
2314