linux/drivers/iommu/fsl_pamu_domain.c
<<
>>
Prefs
   1/*
   2 * This program is free software; you can redistribute it and/or modify
   3 * it under the terms of the GNU General Public License, version 2, as
   4 * published by the Free Software Foundation.
   5 *
   6 * This program is distributed in the hope that it will be useful,
   7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
   8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   9 * GNU General Public License for more details.
  10 *
  11 * You should have received a copy of the GNU General Public License
  12 * along with this program; if not, write to the Free Software
  13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  14 *
  15 * Copyright (C) 2013 Freescale Semiconductor, Inc.
  16 * Author: Varun Sethi <varun.sethi@freescale.com>
  17 *
  18 */
  19
  20#define pr_fmt(fmt)    "fsl-pamu-domain: %s: " fmt, __func__
  21
  22#include <linux/init.h>
  23#include <linux/iommu.h>
  24#include <linux/notifier.h>
  25#include <linux/slab.h>
  26#include <linux/module.h>
  27#include <linux/types.h>
  28#include <linux/mm.h>
  29#include <linux/interrupt.h>
  30#include <linux/device.h>
  31#include <linux/of_platform.h>
  32#include <linux/bootmem.h>
  33#include <linux/err.h>
  34#include <asm/io.h>
  35#include <asm/bitops.h>
  36
  37#include <asm/pci-bridge.h>
  38#include <sysdev/fsl_pci.h>
  39
  40#include "fsl_pamu_domain.h"
  41#include "pci.h"
  42
  43/*
  44 * Global spinlock that needs to be held while
  45 * configuring PAMU.
  46 */
  47static DEFINE_SPINLOCK(iommu_lock);
  48
  49static struct kmem_cache *fsl_pamu_domain_cache;
  50static struct kmem_cache *iommu_devinfo_cache;
  51static DEFINE_SPINLOCK(device_domain_lock);
  52
  53static int __init iommu_init_mempool(void)
  54{
  55
  56        fsl_pamu_domain_cache = kmem_cache_create("fsl_pamu_domain",
  57                                         sizeof(struct fsl_dma_domain),
  58                                         0,
  59                                         SLAB_HWCACHE_ALIGN,
  60
  61                                         NULL);
  62        if (!fsl_pamu_domain_cache) {
  63                pr_debug("Couldn't create fsl iommu_domain cache\n");
  64                return -ENOMEM;
  65        }
  66
  67        iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
  68                                         sizeof(struct device_domain_info),
  69                                         0,
  70                                         SLAB_HWCACHE_ALIGN,
  71                                         NULL);
  72        if (!iommu_devinfo_cache) {
  73                pr_debug("Couldn't create devinfo cache\n");
  74                kmem_cache_destroy(fsl_pamu_domain_cache);
  75                return -ENOMEM;
  76        }
  77
  78        return 0;
  79}
  80
  81static phys_addr_t get_phys_addr(struct fsl_dma_domain *dma_domain, dma_addr_t iova)
  82{
  83        u32 win_cnt = dma_domain->win_cnt;
  84        struct dma_window *win_ptr =
  85                                &dma_domain->win_arr[0];
  86        struct iommu_domain_geometry *geom;
  87
  88        geom = &dma_domain->iommu_domain->geometry;
  89
  90        if (!win_cnt || !dma_domain->geom_size) {
  91                pr_debug("Number of windows/geometry not configured for the domain\n");
  92                return 0;
  93        }
  94
  95        if (win_cnt > 1) {
  96                u64 subwin_size;
  97                dma_addr_t subwin_iova;
  98                u32 wnd;
  99
 100                subwin_size = dma_domain->geom_size >> ilog2(win_cnt);
 101                subwin_iova = iova & ~(subwin_size - 1);
 102                wnd = (subwin_iova - geom->aperture_start) >> ilog2(subwin_size);
 103                win_ptr = &dma_domain->win_arr[wnd];
 104        }
 105
 106        if (win_ptr->valid)
 107                return (win_ptr->paddr + (iova & (win_ptr->size - 1)));
 108
 109        return 0;
 110}
 111
 112static int map_subwins(int liodn, struct fsl_dma_domain *dma_domain)
 113{
 114        struct dma_window *sub_win_ptr =
 115                                &dma_domain->win_arr[0];
 116        int i, ret;
 117        unsigned long rpn, flags;
 118
 119        for (i = 0; i < dma_domain->win_cnt; i++) {
 120                if (sub_win_ptr[i].valid) {
 121                        rpn = sub_win_ptr[i].paddr >>
 122                                 PAMU_PAGE_SHIFT;
 123                        spin_lock_irqsave(&iommu_lock, flags);
 124                        ret = pamu_config_spaace(liodn, dma_domain->win_cnt, i,
 125                                                 sub_win_ptr[i].size,
 126                                                 ~(u32)0,
 127                                                 rpn,
 128                                                 dma_domain->snoop_id,
 129                                                 dma_domain->stash_id,
 130                                                 (i > 0) ? 1 : 0,
 131                                                 sub_win_ptr[i].prot);
 132                        spin_unlock_irqrestore(&iommu_lock, flags);
 133                        if (ret) {
 134                                pr_debug("PAMU SPAACE configuration failed for liodn %d\n",
 135                                         liodn);
 136                                return ret;
 137                        }
 138                }
 139        }
 140
 141        return ret;
 142}
 143
 144static int map_win(int liodn, struct fsl_dma_domain *dma_domain)
 145{
 146        int ret;
 147        struct dma_window *wnd = &dma_domain->win_arr[0];
 148        phys_addr_t wnd_addr = dma_domain->iommu_domain->geometry.aperture_start;
 149        unsigned long flags;
 150
 151        spin_lock_irqsave(&iommu_lock, flags);
 152        ret = pamu_config_ppaace(liodn, wnd_addr,
 153                                 wnd->size,
 154                                 ~(u32)0,
 155                                 wnd->paddr >> PAMU_PAGE_SHIFT,
 156                                 dma_domain->snoop_id, dma_domain->stash_id,
 157                                 0, wnd->prot);
 158        spin_unlock_irqrestore(&iommu_lock, flags);
 159        if (ret)
 160                pr_debug("PAMU PAACE configuration failed for liodn %d\n",
 161                        liodn);
 162
 163        return ret;
 164}
 165
 166/* Map the DMA window corresponding to the LIODN */
 167static int map_liodn(int liodn, struct fsl_dma_domain *dma_domain)
 168{
 169        if (dma_domain->win_cnt > 1)
 170                return map_subwins(liodn, dma_domain);
 171        else
 172                return map_win(liodn, dma_domain);
 173
 174}
 175
 176/* Update window/subwindow mapping for the LIODN */
 177static int update_liodn(int liodn, struct fsl_dma_domain *dma_domain, u32 wnd_nr)
 178{
 179        int ret;
 180        struct dma_window *wnd = &dma_domain->win_arr[wnd_nr];
 181        unsigned long flags;
 182
 183        spin_lock_irqsave(&iommu_lock, flags);
 184        if (dma_domain->win_cnt > 1) {
 185                ret = pamu_config_spaace(liodn, dma_domain->win_cnt, wnd_nr,
 186                                         wnd->size,
 187                                         ~(u32)0,
 188                                         wnd->paddr >> PAMU_PAGE_SHIFT,
 189                                         dma_domain->snoop_id,
 190                                         dma_domain->stash_id,
 191                                         (wnd_nr > 0) ? 1 : 0,
 192                                         wnd->prot);
 193                if (ret)
 194                        pr_debug("Subwindow reconfiguration failed for liodn %d\n", liodn);
 195        } else {
 196                phys_addr_t wnd_addr;
 197
 198                wnd_addr = dma_domain->iommu_domain->geometry.aperture_start;
 199
 200                ret = pamu_config_ppaace(liodn, wnd_addr,
 201                                         wnd->size,
 202                                         ~(u32)0,
 203                                         wnd->paddr >> PAMU_PAGE_SHIFT,
 204                                        dma_domain->snoop_id, dma_domain->stash_id,
 205                                        0, wnd->prot);
 206                if (ret)
 207                        pr_debug("Window reconfiguration failed for liodn %d\n", liodn);
 208        }
 209
 210        spin_unlock_irqrestore(&iommu_lock, flags);
 211
 212        return ret;
 213}
 214
 215static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain,
 216                                 u32 val)
 217{
 218        int ret = 0, i;
 219        unsigned long flags;
 220
 221        spin_lock_irqsave(&iommu_lock, flags);
 222        if (!dma_domain->win_arr) {
 223                pr_debug("Windows not configured, stash destination update failed for liodn %d\n", liodn);
 224                spin_unlock_irqrestore(&iommu_lock, flags);
 225                return -EINVAL;
 226        }
 227
 228        for (i = 0; i < dma_domain->win_cnt; i++) {
 229                ret = pamu_update_paace_stash(liodn, i, val);
 230                if (ret) {
 231                        pr_debug("Failed to update SPAACE %d field for liodn %d\n ", i, liodn);
 232                        spin_unlock_irqrestore(&iommu_lock, flags);
 233                        return ret;
 234                }
 235        }
 236
 237        spin_unlock_irqrestore(&iommu_lock, flags);
 238
 239        return ret;
 240}
 241
 242/* Set the geometry parameters for a LIODN */
 243static int pamu_set_liodn(int liodn, struct device *dev,
 244                           struct fsl_dma_domain *dma_domain,
 245                           struct iommu_domain_geometry *geom_attr,
 246                           u32 win_cnt)
 247{
 248        phys_addr_t window_addr, window_size;
 249        phys_addr_t subwin_size;
 250        int ret = 0, i;
 251        u32 omi_index = ~(u32)0;
 252        unsigned long flags;
 253
 254        /*
 255         * Configure the omi_index at the geometry setup time.
 256         * This is a static value which depends on the type of
 257         * device and would not change thereafter.
 258         */
 259        get_ome_index(&omi_index, dev);
 260
 261        window_addr = geom_attr->aperture_start;
 262        window_size = dma_domain->geom_size;
 263
 264        spin_lock_irqsave(&iommu_lock, flags);
 265        ret = pamu_disable_liodn(liodn);
 266        if (!ret)
 267                ret = pamu_config_ppaace(liodn, window_addr, window_size, omi_index,
 268                                         0, dma_domain->snoop_id,
 269                                         dma_domain->stash_id, win_cnt, 0);
 270        spin_unlock_irqrestore(&iommu_lock, flags);
 271        if (ret) {
 272                pr_debug("PAMU PAACE configuration failed for liodn %d, win_cnt =%d\n", liodn, win_cnt);
 273                return ret;
 274        }
 275
 276        if (win_cnt > 1) {
 277                subwin_size = window_size >> ilog2(win_cnt);
 278                for (i = 0; i < win_cnt; i++) {
 279                        spin_lock_irqsave(&iommu_lock, flags);
 280                        ret = pamu_disable_spaace(liodn, i);
 281                        if (!ret)
 282                                ret = pamu_config_spaace(liodn, win_cnt, i,
 283                                                         subwin_size, omi_index,
 284                                                         0, dma_domain->snoop_id,
 285                                                         dma_domain->stash_id,
 286                                                         0, 0);
 287                        spin_unlock_irqrestore(&iommu_lock, flags);
 288                        if (ret) {
 289                                pr_debug("PAMU SPAACE configuration failed for liodn %d\n", liodn);
 290                                return ret;
 291                        }
 292                }
 293        }
 294
 295        return ret;
 296}
 297
 298static int check_size(u64 size, dma_addr_t iova)
 299{
 300        /*
 301         * Size must be a power of two and at least be equal
 302         * to PAMU page size.
 303         */
 304        if (!is_power_of_2(size) || size < PAMU_PAGE_SIZE) {
 305                pr_debug("%s: size too small or not a power of two\n", __func__);
 306                return -EINVAL;
 307        }
 308
 309        /* iova must be page size aligned*/
 310        if (iova & (size - 1)) {
 311                pr_debug("%s: address is not aligned with window size\n", __func__);
 312                return -EINVAL;
 313        }
 314
 315        return 0;
 316}
 317
 318static struct fsl_dma_domain *iommu_alloc_dma_domain(void)
 319{
 320        struct fsl_dma_domain *domain;
 321
 322        domain = kmem_cache_zalloc(fsl_pamu_domain_cache, GFP_KERNEL);
 323        if (!domain)
 324                return NULL;
 325
 326        domain->stash_id = ~(u32)0;
 327        domain->snoop_id = ~(u32)0;
 328        domain->win_cnt = pamu_get_max_subwin_cnt();
 329        domain->geom_size = 0;
 330
 331        INIT_LIST_HEAD(&domain->devices);
 332
 333        spin_lock_init(&domain->domain_lock);
 334
 335        return domain;
 336}
 337
 338static inline struct device_domain_info *find_domain(struct device *dev)
 339{
 340        return dev->archdata.iommu_domain;
 341}
 342
 343static void remove_device_ref(struct device_domain_info *info, u32 win_cnt)
 344{
 345        unsigned long flags;
 346
 347        list_del(&info->link);
 348        spin_lock_irqsave(&iommu_lock, flags);
 349        if (win_cnt > 1)
 350                pamu_free_subwins(info->liodn);
 351        pamu_disable_liodn(info->liodn);
 352        spin_unlock_irqrestore(&iommu_lock, flags);
 353        spin_lock_irqsave(&device_domain_lock, flags);
 354        info->dev->archdata.iommu_domain = NULL;
 355        kmem_cache_free(iommu_devinfo_cache, info);
 356        spin_unlock_irqrestore(&device_domain_lock, flags);
 357}
 358
 359static void detach_device(struct device *dev, struct fsl_dma_domain *dma_domain)
 360{
 361        struct device_domain_info *info, *tmp;
 362        unsigned long flags;
 363
 364        spin_lock_irqsave(&dma_domain->domain_lock, flags);
 365        /* Remove the device from the domain device list */
 366        list_for_each_entry_safe(info, tmp, &dma_domain->devices, link) {
 367                if (!dev || (info->dev == dev))
 368                        remove_device_ref(info, dma_domain->win_cnt);
 369        }
 370        spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 371}
 372
 373static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct device *dev)
 374{
 375        struct device_domain_info *info, *old_domain_info;
 376        unsigned long flags;
 377
 378        spin_lock_irqsave(&device_domain_lock, flags);
 379        /*
 380         * Check here if the device is already attached to domain or not.
 381         * If the device is already attached to a domain detach it.
 382         */
 383        old_domain_info = find_domain(dev);
 384        if (old_domain_info && old_domain_info->domain != dma_domain) {
 385                spin_unlock_irqrestore(&device_domain_lock, flags);
 386                detach_device(dev, old_domain_info->domain);
 387                spin_lock_irqsave(&device_domain_lock, flags);
 388        }
 389
 390        info = kmem_cache_zalloc(iommu_devinfo_cache, GFP_ATOMIC);
 391
 392        info->dev = dev;
 393        info->liodn = liodn;
 394        info->domain = dma_domain;
 395
 396        list_add(&info->link, &dma_domain->devices);
 397        /*
 398         * In case of devices with multiple LIODNs just store
 399         * the info for the first LIODN as all
 400         * LIODNs share the same domain
 401         */
 402        if (!old_domain_info)
 403                dev->archdata.iommu_domain = info;
 404        spin_unlock_irqrestore(&device_domain_lock, flags);
 405
 406}
 407
 408static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain,
 409                                            dma_addr_t iova)
 410{
 411        struct fsl_dma_domain *dma_domain = domain->priv;
 412
 413        if ((iova < domain->geometry.aperture_start) ||
 414                iova > (domain->geometry.aperture_end))
 415                return 0;
 416
 417        return get_phys_addr(dma_domain, iova);
 418}
 419
 420static int fsl_pamu_domain_has_cap(struct iommu_domain *domain,
 421                                      unsigned long cap)
 422{
 423        return cap == IOMMU_CAP_CACHE_COHERENCY;
 424}
 425
 426static void fsl_pamu_domain_destroy(struct iommu_domain *domain)
 427{
 428        struct fsl_dma_domain *dma_domain = domain->priv;
 429
 430        domain->priv = NULL;
 431
 432        /* remove all the devices from the device list */
 433        detach_device(NULL, dma_domain);
 434
 435        dma_domain->enabled = 0;
 436        dma_domain->mapped = 0;
 437
 438        kmem_cache_free(fsl_pamu_domain_cache, dma_domain);
 439}
 440
 441static int fsl_pamu_domain_init(struct iommu_domain *domain)
 442{
 443        struct fsl_dma_domain *dma_domain;
 444
 445        dma_domain = iommu_alloc_dma_domain();
 446        if (!dma_domain) {
 447                pr_debug("dma_domain allocation failed\n");
 448                return -ENOMEM;
 449        }
 450        domain->priv = dma_domain;
 451        dma_domain->iommu_domain = domain;
 452        /* defaul geometry 64 GB i.e. maximum system address */
 453        domain->geometry.aperture_start = 0;
 454        domain->geometry.aperture_end = (1ULL << 36) - 1;
 455        domain->geometry.force_aperture = true;
 456
 457        return 0;
 458}
 459
 460/* Configure geometry settings for all LIODNs associated with domain */
 461static int pamu_set_domain_geometry(struct fsl_dma_domain *dma_domain,
 462                                    struct iommu_domain_geometry *geom_attr,
 463                                    u32 win_cnt)
 464{
 465        struct device_domain_info *info;
 466        int ret = 0;
 467
 468        list_for_each_entry(info, &dma_domain->devices, link) {
 469                ret = pamu_set_liodn(info->liodn, info->dev, dma_domain,
 470                                      geom_attr, win_cnt);
 471                if (ret)
 472                        break;
 473        }
 474
 475        return ret;
 476}
 477
 478/* Update stash destination for all LIODNs associated with the domain */
 479static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val)
 480{
 481        struct device_domain_info *info;
 482        int ret = 0;
 483
 484        list_for_each_entry(info, &dma_domain->devices, link) {
 485                ret = update_liodn_stash(info->liodn, dma_domain, val);
 486                if (ret)
 487                        break;
 488        }
 489
 490        return ret;
 491}
 492
 493/* Update domain mappings for all LIODNs associated with the domain */
 494static int update_domain_mapping(struct fsl_dma_domain *dma_domain, u32 wnd_nr)
 495{
 496        struct device_domain_info *info;
 497        int ret = 0;
 498
 499        list_for_each_entry(info, &dma_domain->devices, link) {
 500                ret = update_liodn(info->liodn, dma_domain, wnd_nr);
 501                if (ret)
 502                        break;
 503        }
 504        return ret;
 505}
 506
 507static int disable_domain_win(struct fsl_dma_domain *dma_domain, u32 wnd_nr)
 508{
 509        struct device_domain_info *info;
 510        int ret = 0;
 511
 512        list_for_each_entry(info, &dma_domain->devices, link) {
 513                if (dma_domain->win_cnt == 1 && dma_domain->enabled) {
 514                        ret = pamu_disable_liodn(info->liodn);
 515                        if (!ret)
 516                                dma_domain->enabled = 0;
 517                } else {
 518                        ret = pamu_disable_spaace(info->liodn, wnd_nr);
 519                }
 520        }
 521
 522        return ret;
 523}
 524
 525static void fsl_pamu_window_disable(struct iommu_domain *domain, u32 wnd_nr)
 526{
 527        struct fsl_dma_domain *dma_domain = domain->priv;
 528        unsigned long flags;
 529        int ret;
 530
 531        spin_lock_irqsave(&dma_domain->domain_lock, flags);
 532        if (!dma_domain->win_arr) {
 533                pr_debug("Number of windows not configured\n");
 534                spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 535                return;
 536        }
 537
 538        if (wnd_nr >= dma_domain->win_cnt) {
 539                pr_debug("Invalid window index\n");
 540                spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 541                return;
 542        }
 543
 544        if (dma_domain->win_arr[wnd_nr].valid) {
 545                ret = disable_domain_win(dma_domain, wnd_nr);
 546                if (!ret) {
 547                        dma_domain->win_arr[wnd_nr].valid = 0;
 548                        dma_domain->mapped--;
 549                }
 550        }
 551
 552        spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 553
 554}
 555
 556static int fsl_pamu_window_enable(struct iommu_domain *domain, u32 wnd_nr,
 557                                  phys_addr_t paddr, u64 size, int prot)
 558{
 559        struct fsl_dma_domain *dma_domain = domain->priv;
 560        struct dma_window *wnd;
 561        int pamu_prot = 0;
 562        int ret;
 563        unsigned long flags;
 564        u64 win_size;
 565
 566        if (prot & IOMMU_READ)
 567                pamu_prot |= PAACE_AP_PERMS_QUERY;
 568        if (prot & IOMMU_WRITE)
 569                pamu_prot |= PAACE_AP_PERMS_UPDATE;
 570
 571        spin_lock_irqsave(&dma_domain->domain_lock, flags);
 572        if (!dma_domain->win_arr) {
 573                pr_debug("Number of windows not configured\n");
 574                spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 575                return -ENODEV;
 576        }
 577
 578        if (wnd_nr >= dma_domain->win_cnt) {
 579                pr_debug("Invalid window index\n");
 580                spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 581                return -EINVAL;
 582        }
 583
 584        win_size = dma_domain->geom_size >> ilog2(dma_domain->win_cnt);
 585        if (size > win_size) {
 586                pr_debug("Invalid window size \n");
 587                spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 588                return -EINVAL;
 589        }
 590
 591        if (dma_domain->win_cnt == 1) {
 592                if (dma_domain->enabled) {
 593                        pr_debug("Disable the window before updating the mapping\n");
 594                        spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 595                        return -EBUSY;
 596                }
 597
 598                ret = check_size(size, domain->geometry.aperture_start);
 599                if (ret) {
 600                        pr_debug("Aperture start not aligned to the size\n");
 601                        spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 602                        return -EINVAL;
 603                }
 604        }
 605
 606        wnd = &dma_domain->win_arr[wnd_nr];
 607        if (!wnd->valid) {
 608                wnd->paddr = paddr;
 609                wnd->size = size;
 610                wnd->prot = pamu_prot;
 611
 612                ret = update_domain_mapping(dma_domain, wnd_nr);
 613                if (!ret) {
 614                        wnd->valid = 1;
 615                        dma_domain->mapped++;
 616                }
 617        } else {
 618                pr_debug("Disable the window before updating the mapping\n");
 619                ret = -EBUSY;
 620        }
 621
 622        spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 623
 624        return ret;
 625}
 626
 627/*
 628 * Attach the LIODN to the DMA domain and configure the geometry
 629 * and window mappings.
 630 */
 631static int handle_attach_device(struct fsl_dma_domain *dma_domain,
 632                                 struct device *dev, const u32 *liodn,
 633                                 int num)
 634{
 635        unsigned long flags;
 636        struct iommu_domain *domain = dma_domain->iommu_domain;
 637        int ret = 0;
 638        int i;
 639
 640        spin_lock_irqsave(&dma_domain->domain_lock, flags);
 641        for (i = 0; i < num; i++) {
 642
 643                /* Ensure that LIODN value is valid */
 644                if (liodn[i] >= PAACE_NUMBER_ENTRIES) {
 645                        pr_debug("Invalid liodn %d, attach device failed for %s\n",
 646                                liodn[i], dev->of_node->full_name);
 647                        ret = -EINVAL;
 648                        break;
 649                }
 650
 651                attach_device(dma_domain, liodn[i], dev);
 652                /*
 653                 * Check if geometry has already been configured
 654                 * for the domain. If yes, set the geometry for
 655                 * the LIODN.
 656                 */
 657                if (dma_domain->win_arr) {
 658                        u32 win_cnt = dma_domain->win_cnt > 1 ? dma_domain->win_cnt : 0;
 659                        ret = pamu_set_liodn(liodn[i], dev, dma_domain,
 660                                              &domain->geometry,
 661                                              win_cnt);
 662                        if (ret)
 663                                break;
 664                        if (dma_domain->mapped) {
 665                                /*
 666                                 * Create window/subwindow mapping for
 667                                 * the LIODN.
 668                                 */
 669                                ret = map_liodn(liodn[i], dma_domain);
 670                                if (ret)
 671                                        break;
 672                        }
 673                }
 674        }
 675        spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 676
 677        return ret;
 678}
 679
 680static int fsl_pamu_attach_device(struct iommu_domain *domain,
 681                                  struct device *dev)
 682{
 683        struct fsl_dma_domain *dma_domain = domain->priv;
 684        const u32 *liodn;
 685        u32 liodn_cnt;
 686        int len, ret = 0;
 687        struct pci_dev *pdev = NULL;
 688        struct pci_controller *pci_ctl;
 689
 690        /*
 691         * Use LIODN of the PCI controller while attaching a
 692         * PCI device.
 693         */
 694        if (dev->bus == &pci_bus_type) {
 695                pdev = to_pci_dev(dev);
 696                pci_ctl = pci_bus_to_host(pdev->bus);
 697                /*
 698                 * make dev point to pci controller device
 699                 * so we can get the LIODN programmed by
 700                 * u-boot.
 701                 */
 702                dev = pci_ctl->parent;
 703        }
 704
 705        liodn = of_get_property(dev->of_node, "fsl,liodn", &len);
 706        if (liodn) {
 707                liodn_cnt = len / sizeof(u32);
 708                ret = handle_attach_device(dma_domain, dev,
 709                                         liodn, liodn_cnt);
 710        } else {
 711                pr_debug("missing fsl,liodn property at %s\n",
 712                          dev->of_node->full_name);
 713                        ret = -EINVAL;
 714        }
 715
 716        return ret;
 717}
 718
 719static void fsl_pamu_detach_device(struct iommu_domain *domain,
 720                                      struct device *dev)
 721{
 722        struct fsl_dma_domain *dma_domain = domain->priv;
 723        const u32 *prop;
 724        int len;
 725        struct pci_dev *pdev = NULL;
 726        struct pci_controller *pci_ctl;
 727
 728        /*
 729         * Use LIODN of the PCI controller while detaching a
 730         * PCI device.
 731         */
 732        if (dev->bus == &pci_bus_type) {
 733                pdev = to_pci_dev(dev);
 734                pci_ctl = pci_bus_to_host(pdev->bus);
 735                /*
 736                 * make dev point to pci controller device
 737                 * so we can get the LIODN programmed by
 738                 * u-boot.
 739                 */
 740                dev = pci_ctl->parent;
 741        }
 742
 743        prop = of_get_property(dev->of_node, "fsl,liodn", &len);
 744        if (prop)
 745                detach_device(dev, dma_domain);
 746        else
 747                pr_debug("missing fsl,liodn property at %s\n",
 748                          dev->of_node->full_name);
 749}
 750
 751static  int configure_domain_geometry(struct iommu_domain *domain, void *data)
 752{
 753        struct iommu_domain_geometry *geom_attr = data;
 754        struct fsl_dma_domain *dma_domain = domain->priv;
 755        dma_addr_t geom_size;
 756        unsigned long flags;
 757
 758        geom_size = geom_attr->aperture_end - geom_attr->aperture_start + 1;
 759        /*
 760         * Sanity check the geometry size. Also, we do not support
 761         * DMA outside of the geometry.
 762         */
 763        if (check_size(geom_size, geom_attr->aperture_start) ||
 764                !geom_attr->force_aperture) {
 765                        pr_debug("Invalid PAMU geometry attributes\n");
 766                        return -EINVAL;
 767                }
 768
 769        spin_lock_irqsave(&dma_domain->domain_lock, flags);
 770        if (dma_domain->enabled) {
 771                pr_debug("Can't set geometry attributes as domain is active\n");
 772                spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 773                return  -EBUSY;
 774        }
 775
 776        /* Copy the domain geometry information */
 777        memcpy(&domain->geometry, geom_attr,
 778               sizeof(struct iommu_domain_geometry));
 779        dma_domain->geom_size = geom_size;
 780
 781        spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 782
 783        return 0;
 784}
 785
 786/* Set the domain stash attribute */
 787static int configure_domain_stash(struct fsl_dma_domain *dma_domain, void *data)
 788{
 789        struct pamu_stash_attribute *stash_attr = data;
 790        unsigned long flags;
 791        int ret;
 792
 793        spin_lock_irqsave(&dma_domain->domain_lock, flags);
 794
 795        memcpy(&dma_domain->dma_stash, stash_attr,
 796                 sizeof(struct pamu_stash_attribute));
 797
 798        dma_domain->stash_id = get_stash_id(stash_attr->cache,
 799                                            stash_attr->cpu);
 800        if (dma_domain->stash_id == ~(u32)0) {
 801                pr_debug("Invalid stash attributes\n");
 802                spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 803                return -EINVAL;
 804        }
 805
 806        ret = update_domain_stash(dma_domain, dma_domain->stash_id);
 807
 808        spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 809
 810        return ret;
 811}
 812
 813/* Configure domain dma state i.e. enable/disable DMA*/
 814static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool enable)
 815{
 816        struct device_domain_info *info;
 817        unsigned long flags;
 818        int ret;
 819
 820        spin_lock_irqsave(&dma_domain->domain_lock, flags);
 821
 822        if (enable && !dma_domain->mapped) {
 823                pr_debug("Can't enable DMA domain without valid mapping\n");
 824                spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 825                return -ENODEV;
 826        }
 827
 828        dma_domain->enabled = enable;
 829        list_for_each_entry(info, &dma_domain->devices,
 830                                 link) {
 831                ret = (enable) ? pamu_enable_liodn(info->liodn) :
 832                        pamu_disable_liodn(info->liodn);
 833                if (ret)
 834                        pr_debug("Unable to set dma state for liodn %d",
 835                                 info->liodn);
 836        }
 837        spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 838
 839        return 0;
 840}
 841
 842static int fsl_pamu_set_domain_attr(struct iommu_domain *domain,
 843                                 enum iommu_attr attr_type, void *data)
 844{
 845        struct fsl_dma_domain *dma_domain = domain->priv;
 846        int ret = 0;
 847
 848
 849        switch (attr_type) {
 850        case DOMAIN_ATTR_GEOMETRY:
 851                ret = configure_domain_geometry(domain, data);
 852                break;
 853        case DOMAIN_ATTR_FSL_PAMU_STASH:
 854                ret = configure_domain_stash(dma_domain, data);
 855                break;
 856        case DOMAIN_ATTR_FSL_PAMU_ENABLE:
 857                ret = configure_domain_dma_state(dma_domain, *(int *)data);
 858                break;
 859        default:
 860                pr_debug("Unsupported attribute type\n");
 861                ret = -EINVAL;
 862                break;
 863        };
 864
 865        return ret;
 866}
 867
 868static int fsl_pamu_get_domain_attr(struct iommu_domain *domain,
 869                                 enum iommu_attr attr_type, void *data)
 870{
 871        struct fsl_dma_domain *dma_domain = domain->priv;
 872        int ret = 0;
 873
 874
 875        switch (attr_type) {
 876        case DOMAIN_ATTR_FSL_PAMU_STASH:
 877                memcpy((struct pamu_stash_attribute *) data, &dma_domain->dma_stash,
 878                                 sizeof(struct pamu_stash_attribute));
 879                break;
 880        case DOMAIN_ATTR_FSL_PAMU_ENABLE:
 881                *(int *)data = dma_domain->enabled;
 882                break;
 883        case DOMAIN_ATTR_FSL_PAMUV1:
 884                *(int *)data = DOMAIN_ATTR_FSL_PAMUV1;
 885                break;
 886        default:
 887                pr_debug("Unsupported attribute type\n");
 888                ret = -EINVAL;
 889                break;
 890        };
 891
 892        return ret;
 893}
 894
 895#define REQ_ACS_FLAGS   (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
 896
 897static struct iommu_group *get_device_iommu_group(struct device *dev)
 898{
 899        struct iommu_group *group;
 900
 901        group = iommu_group_get(dev);
 902        if (!group)
 903                group = iommu_group_alloc();
 904
 905        return group;
 906}
 907
 908static  bool check_pci_ctl_endpt_part(struct pci_controller *pci_ctl)
 909{
 910        u32 version;
 911
 912        /* Check the PCI controller version number by readding BRR1 register */
 913        version = in_be32(pci_ctl->cfg_addr + (PCI_FSL_BRR1 >> 2));
 914        version &= PCI_FSL_BRR1_VER;
 915        /* If PCI controller version is >= 0x204 we can partition endpoints*/
 916        if (version >= 0x204)
 917                return 1;
 918
 919        return 0;
 920}
 921
 922/* Get iommu group information from peer devices or devices on the parent bus */
 923static struct iommu_group *get_shared_pci_device_group(struct pci_dev *pdev)
 924{
 925        struct pci_dev *tmp;
 926        struct iommu_group *group;
 927        struct pci_bus *bus = pdev->bus;
 928
 929        /*
 930         * Traverese the pci bus device list to get
 931         * the shared iommu group.
 932         */
 933        while (bus) {
 934                list_for_each_entry(tmp, &bus->devices, bus_list) {
 935                        if (tmp == pdev)
 936                                continue;
 937                        group = iommu_group_get(&tmp->dev);
 938                        if (group)
 939                                return group;
 940                }
 941
 942                bus = bus->parent;
 943        }
 944
 945        return NULL;
 946}
 947
 948static struct iommu_group *get_pci_device_group(struct pci_dev *pdev)
 949{
 950        struct pci_controller *pci_ctl;
 951        bool pci_endpt_partioning;
 952        struct iommu_group *group = NULL;
 953        struct pci_dev *bridge, *dma_pdev = NULL;
 954
 955        pci_ctl = pci_bus_to_host(pdev->bus);
 956        pci_endpt_partioning = check_pci_ctl_endpt_part(pci_ctl);
 957        /* We can partition PCIe devices so assign device group to the device */
 958        if (pci_endpt_partioning) {
 959                bridge = pci_find_upstream_pcie_bridge(pdev);
 960                if (bridge) {
 961                        if (pci_is_pcie(bridge))
 962                                dma_pdev = pci_get_domain_bus_and_slot(
 963                                                pci_domain_nr(pdev->bus),
 964                                                bridge->subordinate->number, 0);
 965                        if (!dma_pdev)
 966                                dma_pdev = pci_dev_get(bridge);
 967                } else
 968                        dma_pdev = pci_dev_get(pdev);
 969
 970                /* Account for quirked devices */
 971                swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));
 972
 973                /*
 974                 * If it's a multifunction device that does not support our
 975                 * required ACS flags, add to the same group as lowest numbered
 976                 * function that also does not suport the required ACS flags.
 977                 */
 978                if (dma_pdev->multifunction &&
 979                    !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) {
 980                        u8 i, slot = PCI_SLOT(dma_pdev->devfn);
 981
 982                        for (i = 0; i < 8; i++) {
 983                                struct pci_dev *tmp;
 984
 985                                tmp = pci_get_slot(dma_pdev->bus, PCI_DEVFN(slot, i));
 986                                if (!tmp)
 987                                        continue;
 988
 989                                if (!pci_acs_enabled(tmp, REQ_ACS_FLAGS)) {
 990                                        swap_pci_ref(&dma_pdev, tmp);
 991                                        break;
 992                                }
 993                                pci_dev_put(tmp);
 994                        }
 995                }
 996
 997                /*
 998                 * Devices on the root bus go through the iommu.  If that's not us,
 999                 * find the next upstream device and test ACS up to the root bus.
1000                 * Finding the next device may require skipping virtual buses.
1001                 */
1002                while (!pci_is_root_bus(dma_pdev->bus)) {
1003                        struct pci_bus *bus = dma_pdev->bus;
1004
1005                        while (!bus->self) {
1006                                if (!pci_is_root_bus(bus))
1007                                        bus = bus->parent;
1008                                else
1009                                        goto root_bus;
1010                        }
1011
1012                        if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
1013                                break;
1014
1015                        swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));
1016                }
1017
1018root_bus:
1019                group = get_device_iommu_group(&dma_pdev->dev);
1020                pci_dev_put(dma_pdev);
1021                /*
1022                 * PCIe controller is not a paritionable entity
1023                 * free the controller device iommu_group.
1024                 */
1025                if (pci_ctl->parent->iommu_group)
1026                        iommu_group_remove_device(pci_ctl->parent);
1027        } else {
1028                /*
1029                 * All devices connected to the controller will share the
1030                 * PCI controllers device group. If this is the first
1031                 * device to be probed for the pci controller, copy the
1032                 * device group information from the PCI controller device
1033                 * node and remove the PCI controller iommu group.
1034                 * For subsequent devices, the iommu group information can
1035                 * be obtained from sibling devices (i.e. from the bus_devices
1036                 * link list).
1037                 */
1038                if (pci_ctl->parent->iommu_group) {
1039                        group = get_device_iommu_group(pci_ctl->parent);
1040                        iommu_group_remove_device(pci_ctl->parent);
1041                } else
1042                        group = get_shared_pci_device_group(pdev);
1043        }
1044
1045        return group;
1046}
1047
1048static int fsl_pamu_add_device(struct device *dev)
1049{
1050        struct iommu_group *group = NULL;
1051        struct pci_dev *pdev;
1052        const u32 *prop;
1053        int ret, len;
1054
1055        /*
1056         * For platform devices we allocate a separate group for
1057         * each of the devices.
1058         */
1059        if (dev->bus == &pci_bus_type) {
1060                pdev = to_pci_dev(dev);
1061                /* Don't create device groups for virtual PCI bridges */
1062                if (pdev->subordinate)
1063                        return 0;
1064
1065                group = get_pci_device_group(pdev);
1066
1067        } else {
1068                prop = of_get_property(dev->of_node, "fsl,liodn", &len);
1069                if (prop)
1070                        group = get_device_iommu_group(dev);
1071        }
1072
1073        if (!group || IS_ERR(group))
1074                return PTR_ERR(group);
1075
1076        ret = iommu_group_add_device(group, dev);
1077
1078        iommu_group_put(group);
1079        return ret;
1080}
1081
1082static void fsl_pamu_remove_device(struct device *dev)
1083{
1084        iommu_group_remove_device(dev);
1085}
1086
1087static int fsl_pamu_set_windows(struct iommu_domain *domain, u32 w_count)
1088{
1089        struct fsl_dma_domain *dma_domain = domain->priv;
1090        unsigned long flags;
1091        int ret;
1092
1093        spin_lock_irqsave(&dma_domain->domain_lock, flags);
1094        /* Ensure domain is inactive i.e. DMA should be disabled for the domain */
1095        if (dma_domain->enabled) {
1096                pr_debug("Can't set geometry attributes as domain is active\n");
1097                spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1098                return  -EBUSY;
1099        }
1100
1101        /* Ensure that the geometry has been set for the domain */
1102        if (!dma_domain->geom_size) {
1103                pr_debug("Please configure geometry before setting the number of windows\n");
1104                spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1105                return -EINVAL;
1106        }
1107
1108        /*
1109         * Ensure we have valid window count i.e. it should be less than
1110         * maximum permissible limit and should be a power of two.
1111         */
1112        if (w_count > pamu_get_max_subwin_cnt() || !is_power_of_2(w_count)) {
1113                pr_debug("Invalid window count\n");
1114                spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1115                return -EINVAL;
1116        }
1117
1118        ret = pamu_set_domain_geometry(dma_domain, &domain->geometry,
1119                                ((w_count > 1) ? w_count : 0));
1120        if (!ret) {
1121                if (dma_domain->win_arr)
1122                        kfree(dma_domain->win_arr);
1123                dma_domain->win_arr = kzalloc(sizeof(struct dma_window) *
1124                                                          w_count, GFP_ATOMIC);
1125                if (!dma_domain->win_arr) {
1126                        spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1127                        return -ENOMEM;
1128                }
1129                dma_domain->win_cnt = w_count;
1130        }
1131        spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1132
1133        return ret;
1134}
1135
1136static u32 fsl_pamu_get_windows(struct iommu_domain *domain)
1137{
1138        struct fsl_dma_domain *dma_domain = domain->priv;
1139
1140        return dma_domain->win_cnt;
1141}
1142
1143static struct iommu_ops fsl_pamu_ops = {
1144        .domain_init    = fsl_pamu_domain_init,
1145        .domain_destroy = fsl_pamu_domain_destroy,
1146        .attach_dev     = fsl_pamu_attach_device,
1147        .detach_dev     = fsl_pamu_detach_device,
1148        .domain_window_enable = fsl_pamu_window_enable,
1149        .domain_window_disable = fsl_pamu_window_disable,
1150        .domain_get_windows = fsl_pamu_get_windows,
1151        .domain_set_windows = fsl_pamu_set_windows,
1152        .iova_to_phys   = fsl_pamu_iova_to_phys,
1153        .domain_has_cap = fsl_pamu_domain_has_cap,
1154        .domain_set_attr = fsl_pamu_set_domain_attr,
1155        .domain_get_attr = fsl_pamu_get_domain_attr,
1156        .add_device     = fsl_pamu_add_device,
1157        .remove_device  = fsl_pamu_remove_device,
1158};
1159
1160int pamu_domain_init()
1161{
1162        int ret = 0;
1163
1164        ret = iommu_init_mempool();
1165        if (ret)
1166                return ret;
1167
1168        bus_set_iommu(&platform_bus_type, &fsl_pamu_ops);
1169        bus_set_iommu(&pci_bus_type, &fsl_pamu_ops);
1170
1171        return ret;
1172}
1173
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.