1
2
3
4
5
6
7
8
9#include <linux/atomic.h>
10#include <linux/bitfield.h>
11#include <linux/clk.h>
12#include <linux/delay.h>
13#include <linux/dma-iommu.h>
14#include <linux/dma-mapping.h>
15#include <linux/err.h>
16#include <linux/interrupt.h>
17#include <linux/io.h>
18#include <linux/io-64-nonatomic-hi-lo.h>
19#include <linux/io-pgtable.h>
20#include <linux/iommu.h>
21#include <linux/iopoll.h>
22#include <linux/kconfig.h>
23#include <linux/init.h>
24#include <linux/mutex.h>
25#include <linux/of.h>
26#include <linux/of_address.h>
27#include <linux/of_device.h>
28#include <linux/platform_device.h>
29#include <linux/pm.h>
30#include <linux/pm_runtime.h>
31#include <linux/qcom_scm.h>
32#include <linux/slab.h>
33#include <linux/spinlock.h>
34
35#include "arm-smmu.h"
36
37#define SMMU_INTR_SEL_NS 0x2000
38
39enum qcom_iommu_clk {
40 CLK_IFACE,
41 CLK_BUS,
42 CLK_TBU,
43 CLK_NUM,
44};
45
46struct qcom_iommu_ctx;
47
48struct qcom_iommu_dev {
49
50 struct iommu_device iommu;
51 struct device *dev;
52 struct clk_bulk_data clks[CLK_NUM];
53 void __iomem *local_base;
54 u32 sec_id;
55 u8 num_ctxs;
56 struct qcom_iommu_ctx *ctxs[];
57};
58
59struct qcom_iommu_ctx {
60 struct device *dev;
61 void __iomem *base;
62 bool secure_init;
63 u8 asid;
64 struct iommu_domain *domain;
65};
66
67struct qcom_iommu_domain {
68 struct io_pgtable_ops *pgtbl_ops;
69 spinlock_t pgtbl_lock;
70 struct mutex init_mutex;
71 struct iommu_domain domain;
72 struct qcom_iommu_dev *iommu;
73 struct iommu_fwspec *fwspec;
74};
75
76static struct qcom_iommu_domain *to_qcom_iommu_domain(struct iommu_domain *dom)
77{
78 return container_of(dom, struct qcom_iommu_domain, domain);
79}
80
81static const struct iommu_ops qcom_iommu_ops;
82
83static struct qcom_iommu_dev * to_iommu(struct device *dev)
84{
85 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
86
87 if (!fwspec || fwspec->ops != &qcom_iommu_ops)
88 return NULL;
89
90 return dev_iommu_priv_get(dev);
91}
92
93static struct qcom_iommu_ctx * to_ctx(struct qcom_iommu_domain *d, unsigned asid)
94{
95 struct qcom_iommu_dev *qcom_iommu = d->iommu;
96 if (!qcom_iommu)
97 return NULL;
98 return qcom_iommu->ctxs[asid - 1];
99}
100
101static inline void
102iommu_writel(struct qcom_iommu_ctx *ctx, unsigned reg, u32 val)
103{
104 writel_relaxed(val, ctx->base + reg);
105}
106
107static inline void
108iommu_writeq(struct qcom_iommu_ctx *ctx, unsigned reg, u64 val)
109{
110 writeq_relaxed(val, ctx->base + reg);
111}
112
113static inline u32
114iommu_readl(struct qcom_iommu_ctx *ctx, unsigned reg)
115{
116 return readl_relaxed(ctx->base + reg);
117}
118
119static inline u64
120iommu_readq(struct qcom_iommu_ctx *ctx, unsigned reg)
121{
122 return readq_relaxed(ctx->base + reg);
123}
124
125static void qcom_iommu_tlb_sync(void *cookie)
126{
127 struct qcom_iommu_domain *qcom_domain = cookie;
128 struct iommu_fwspec *fwspec = qcom_domain->fwspec;
129 unsigned i;
130
131 for (i = 0; i < fwspec->num_ids; i++) {
132 struct qcom_iommu_ctx *ctx = to_ctx(qcom_domain, fwspec->ids[i]);
133 unsigned int val, ret;
134
135 iommu_writel(ctx, ARM_SMMU_CB_TLBSYNC, 0);
136
137 ret = readl_poll_timeout(ctx->base + ARM_SMMU_CB_TLBSTATUS, val,
138 (val & 0x1) == 0, 0, 5000000);
139 if (ret)
140 dev_err(ctx->dev, "timeout waiting for TLB SYNC\n");
141 }
142}
143
144static void qcom_iommu_tlb_inv_context(void *cookie)
145{
146 struct qcom_iommu_domain *qcom_domain = cookie;
147 struct iommu_fwspec *fwspec = qcom_domain->fwspec;
148 unsigned i;
149
150 for (i = 0; i < fwspec->num_ids; i++) {
151 struct qcom_iommu_ctx *ctx = to_ctx(qcom_domain, fwspec->ids[i]);
152 iommu_writel(ctx, ARM_SMMU_CB_S1_TLBIASID, ctx->asid);
153 }
154
155 qcom_iommu_tlb_sync(cookie);
156}
157
158static void qcom_iommu_tlb_inv_range_nosync(unsigned long iova, size_t size,
159 size_t granule, bool leaf, void *cookie)
160{
161 struct qcom_iommu_domain *qcom_domain = cookie;
162 struct iommu_fwspec *fwspec = qcom_domain->fwspec;
163 unsigned i, reg;
164
165 reg = leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
166
167 for (i = 0; i < fwspec->num_ids; i++) {
168 struct qcom_iommu_ctx *ctx = to_ctx(qcom_domain, fwspec->ids[i]);
169 size_t s = size;
170
171 iova = (iova >> 12) << 12;
172 iova |= ctx->asid;
173 do {
174 iommu_writel(ctx, reg, iova);
175 iova += granule;
176 } while (s -= granule);
177 }
178}
179
180static void qcom_iommu_tlb_flush_walk(unsigned long iova, size_t size,
181 size_t granule, void *cookie)
182{
183 qcom_iommu_tlb_inv_range_nosync(iova, size, granule, false, cookie);
184 qcom_iommu_tlb_sync(cookie);
185}
186
187static void qcom_iommu_tlb_add_page(struct iommu_iotlb_gather *gather,
188 unsigned long iova, size_t granule,
189 void *cookie)
190{
191 qcom_iommu_tlb_inv_range_nosync(iova, granule, granule, true, cookie);
192}
193
194static const struct iommu_flush_ops qcom_flush_ops = {
195 .tlb_flush_all = qcom_iommu_tlb_inv_context,
196 .tlb_flush_walk = qcom_iommu_tlb_flush_walk,
197 .tlb_add_page = qcom_iommu_tlb_add_page,
198};
199
200static irqreturn_t qcom_iommu_fault(int irq, void *dev)
201{
202 struct qcom_iommu_ctx *ctx = dev;
203 u32 fsr, fsynr;
204 u64 iova;
205
206 fsr = iommu_readl(ctx, ARM_SMMU_CB_FSR);
207
208 if (!(fsr & ARM_SMMU_FSR_FAULT))
209 return IRQ_NONE;
210
211 fsynr = iommu_readl(ctx, ARM_SMMU_CB_FSYNR0);
212 iova = iommu_readq(ctx, ARM_SMMU_CB_FAR);
213
214 if (!report_iommu_fault(ctx->domain, ctx->dev, iova, 0)) {
215 dev_err_ratelimited(ctx->dev,
216 "Unhandled context fault: fsr=0x%x, "
217 "iova=0x%016llx, fsynr=0x%x, cb=%d\n",
218 fsr, iova, fsynr, ctx->asid);
219 }
220
221 iommu_writel(ctx, ARM_SMMU_CB_FSR, fsr);
222 iommu_writel(ctx, ARM_SMMU_CB_RESUME, ARM_SMMU_RESUME_TERMINATE);
223
224 return IRQ_HANDLED;
225}
226
227static int qcom_iommu_init_domain(struct iommu_domain *domain,
228 struct qcom_iommu_dev *qcom_iommu,
229 struct device *dev)
230{
231 struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
232 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
233 struct io_pgtable_ops *pgtbl_ops;
234 struct io_pgtable_cfg pgtbl_cfg;
235 int i, ret = 0;
236 u32 reg;
237
238 mutex_lock(&qcom_domain->init_mutex);
239 if (qcom_domain->iommu)
240 goto out_unlock;
241
242 pgtbl_cfg = (struct io_pgtable_cfg) {
243 .pgsize_bitmap = qcom_iommu_ops.pgsize_bitmap,
244 .ias = 32,
245 .oas = 40,
246 .tlb = &qcom_flush_ops,
247 .iommu_dev = qcom_iommu->dev,
248 };
249
250 qcom_domain->iommu = qcom_iommu;
251 qcom_domain->fwspec = fwspec;
252
253 pgtbl_ops = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &pgtbl_cfg, qcom_domain);
254 if (!pgtbl_ops) {
255 dev_err(qcom_iommu->dev, "failed to allocate pagetable ops\n");
256 ret = -ENOMEM;
257 goto out_clear_iommu;
258 }
259
260
261 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
262 domain->geometry.aperture_end = (1ULL << pgtbl_cfg.ias) - 1;
263 domain->geometry.force_aperture = true;
264
265 for (i = 0; i < fwspec->num_ids; i++) {
266 struct qcom_iommu_ctx *ctx = to_ctx(qcom_domain, fwspec->ids[i]);
267
268 if (!ctx->secure_init) {
269 ret = qcom_scm_restore_sec_cfg(qcom_iommu->sec_id, ctx->asid);
270 if (ret) {
271 dev_err(qcom_iommu->dev, "secure init failed: %d\n", ret);
272 goto out_clear_iommu;
273 }
274 ctx->secure_init = true;
275 }
276
277
278 iommu_writeq(ctx, ARM_SMMU_CB_TTBR0,
279 pgtbl_cfg.arm_lpae_s1_cfg.ttbr |
280 FIELD_PREP(ARM_SMMU_TTBRn_ASID, ctx->asid));
281 iommu_writeq(ctx, ARM_SMMU_CB_TTBR1, 0);
282
283
284 iommu_writel(ctx, ARM_SMMU_CB_TCR2,
285 arm_smmu_lpae_tcr2(&pgtbl_cfg));
286 iommu_writel(ctx, ARM_SMMU_CB_TCR,
287 arm_smmu_lpae_tcr(&pgtbl_cfg) | ARM_SMMU_TCR_EAE);
288
289
290 iommu_writel(ctx, ARM_SMMU_CB_S1_MAIR0,
291 pgtbl_cfg.arm_lpae_s1_cfg.mair);
292 iommu_writel(ctx, ARM_SMMU_CB_S1_MAIR1,
293 pgtbl_cfg.arm_lpae_s1_cfg.mair >> 32);
294
295
296 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE |
297 ARM_SMMU_SCTLR_AFE | ARM_SMMU_SCTLR_TRE |
298 ARM_SMMU_SCTLR_M | ARM_SMMU_SCTLR_S1_ASIDPNE |
299 ARM_SMMU_SCTLR_CFCFG;
300
301 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
302 reg |= ARM_SMMU_SCTLR_E;
303
304 iommu_writel(ctx, ARM_SMMU_CB_SCTLR, reg);
305
306 ctx->domain = domain;
307 }
308
309 mutex_unlock(&qcom_domain->init_mutex);
310
311
312 qcom_domain->pgtbl_ops = pgtbl_ops;
313
314 return 0;
315
316out_clear_iommu:
317 qcom_domain->iommu = NULL;
318out_unlock:
319 mutex_unlock(&qcom_domain->init_mutex);
320 return ret;
321}
322
323static struct iommu_domain *qcom_iommu_domain_alloc(unsigned type)
324{
325 struct qcom_iommu_domain *qcom_domain;
326
327 if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
328 return NULL;
329
330
331
332
333
334 qcom_domain = kzalloc(sizeof(*qcom_domain), GFP_KERNEL);
335 if (!qcom_domain)
336 return NULL;
337
338 if (type == IOMMU_DOMAIN_DMA &&
339 iommu_get_dma_cookie(&qcom_domain->domain)) {
340 kfree(qcom_domain);
341 return NULL;
342 }
343
344 mutex_init(&qcom_domain->init_mutex);
345 spin_lock_init(&qcom_domain->pgtbl_lock);
346
347 return &qcom_domain->domain;
348}
349
350static void qcom_iommu_domain_free(struct iommu_domain *domain)
351{
352 struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
353
354 iommu_put_dma_cookie(domain);
355
356 if (qcom_domain->iommu) {
357
358
359
360
361
362
363 pm_runtime_get_sync(qcom_domain->iommu->dev);
364 free_io_pgtable_ops(qcom_domain->pgtbl_ops);
365 pm_runtime_put_sync(qcom_domain->iommu->dev);
366 }
367
368 kfree(qcom_domain);
369}
370
371static int qcom_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
372{
373 struct qcom_iommu_dev *qcom_iommu = to_iommu(dev);
374 struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
375 int ret;
376
377 if (!qcom_iommu) {
378 dev_err(dev, "cannot attach to IOMMU, is it on the same bus?\n");
379 return -ENXIO;
380 }
381
382
383 pm_runtime_get_sync(qcom_iommu->dev);
384 ret = qcom_iommu_init_domain(domain, qcom_iommu, dev);
385 pm_runtime_put_sync(qcom_iommu->dev);
386 if (ret < 0)
387 return ret;
388
389
390
391
392
393 if (qcom_domain->iommu != qcom_iommu) {
394 dev_err(dev, "cannot attach to IOMMU %s while already "
395 "attached to domain on IOMMU %s\n",
396 dev_name(qcom_domain->iommu->dev),
397 dev_name(qcom_iommu->dev));
398 return -EINVAL;
399 }
400
401 return 0;
402}
403
404static void qcom_iommu_detach_dev(struct iommu_domain *domain, struct device *dev)
405{
406 struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
407 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
408 struct qcom_iommu_dev *qcom_iommu = to_iommu(dev);
409 unsigned i;
410
411 if (WARN_ON(!qcom_domain->iommu))
412 return;
413
414 pm_runtime_get_sync(qcom_iommu->dev);
415 for (i = 0; i < fwspec->num_ids; i++) {
416 struct qcom_iommu_ctx *ctx = to_ctx(qcom_domain, fwspec->ids[i]);
417
418
419 iommu_writel(ctx, ARM_SMMU_CB_SCTLR, 0);
420
421 ctx->domain = NULL;
422 }
423 pm_runtime_put_sync(qcom_iommu->dev);
424}
425
426static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova,
427 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
428{
429 int ret;
430 unsigned long flags;
431 struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
432 struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
433
434 if (!ops)
435 return -ENODEV;
436
437 spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
438 ret = ops->map(ops, iova, paddr, size, prot, GFP_ATOMIC);
439 spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
440 return ret;
441}
442
443static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
444 size_t size, struct iommu_iotlb_gather *gather)
445{
446 size_t ret;
447 unsigned long flags;
448 struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
449 struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
450
451 if (!ops)
452 return 0;
453
454
455
456
457
458
459 pm_runtime_get_sync(qcom_domain->iommu->dev);
460 spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
461 ret = ops->unmap(ops, iova, size, gather);
462 spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
463 pm_runtime_put_sync(qcom_domain->iommu->dev);
464
465 return ret;
466}
467
468static void qcom_iommu_flush_iotlb_all(struct iommu_domain *domain)
469{
470 struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
471 struct io_pgtable *pgtable = container_of(qcom_domain->pgtbl_ops,
472 struct io_pgtable, ops);
473 if (!qcom_domain->pgtbl_ops)
474 return;
475
476 pm_runtime_get_sync(qcom_domain->iommu->dev);
477 qcom_iommu_tlb_sync(pgtable->cookie);
478 pm_runtime_put_sync(qcom_domain->iommu->dev);
479}
480
481static void qcom_iommu_iotlb_sync(struct iommu_domain *domain,
482 struct iommu_iotlb_gather *gather)
483{
484 qcom_iommu_flush_iotlb_all(domain);
485}
486
487static phys_addr_t qcom_iommu_iova_to_phys(struct iommu_domain *domain,
488 dma_addr_t iova)
489{
490 phys_addr_t ret;
491 unsigned long flags;
492 struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
493 struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
494
495 if (!ops)
496 return 0;
497
498 spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
499 ret = ops->iova_to_phys(ops, iova);
500 spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
501
502 return ret;
503}
504
505static bool qcom_iommu_capable(enum iommu_cap cap)
506{
507 switch (cap) {
508 case IOMMU_CAP_CACHE_COHERENCY:
509
510
511
512
513 return true;
514 case IOMMU_CAP_NOEXEC:
515 return true;
516 default:
517 return false;
518 }
519}
520
521static struct iommu_device *qcom_iommu_probe_device(struct device *dev)
522{
523 struct qcom_iommu_dev *qcom_iommu = to_iommu(dev);
524 struct device_link *link;
525
526 if (!qcom_iommu)
527 return ERR_PTR(-ENODEV);
528
529
530
531
532
533
534 link = device_link_add(dev, qcom_iommu->dev, DL_FLAG_PM_RUNTIME);
535 if (!link) {
536 dev_err(qcom_iommu->dev, "Unable to create device link between %s and %s\n",
537 dev_name(qcom_iommu->dev), dev_name(dev));
538 return ERR_PTR(-ENODEV);
539 }
540
541 return &qcom_iommu->iommu;
542}
543
544static void qcom_iommu_release_device(struct device *dev)
545{
546 struct qcom_iommu_dev *qcom_iommu = to_iommu(dev);
547
548 if (!qcom_iommu)
549 return;
550
551 iommu_fwspec_free(dev);
552}
553
554static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)
555{
556 struct qcom_iommu_dev *qcom_iommu;
557 struct platform_device *iommu_pdev;
558 unsigned asid = args->args[0];
559
560 if (args->args_count != 1) {
561 dev_err(dev, "incorrect number of iommu params found for %s "
562 "(found %d, expected 1)\n",
563 args->np->full_name, args->args_count);
564 return -EINVAL;
565 }
566
567 iommu_pdev = of_find_device_by_node(args->np);
568 if (WARN_ON(!iommu_pdev))
569 return -EINVAL;
570
571 qcom_iommu = platform_get_drvdata(iommu_pdev);
572
573
574
575
576
577 if (WARN_ON(asid < 1) ||
578 WARN_ON(asid > qcom_iommu->num_ctxs)) {
579 put_device(&iommu_pdev->dev);
580 return -EINVAL;
581 }
582
583 if (!dev_iommu_priv_get(dev)) {
584 dev_iommu_priv_set(dev, qcom_iommu);
585 } else {
586
587
588
589
590 if (WARN_ON(qcom_iommu != dev_iommu_priv_get(dev))) {
591 put_device(&iommu_pdev->dev);
592 return -EINVAL;
593 }
594 }
595
596 return iommu_fwspec_add_ids(dev, &asid, 1);
597}
598
599static const struct iommu_ops qcom_iommu_ops = {
600 .capable = qcom_iommu_capable,
601 .domain_alloc = qcom_iommu_domain_alloc,
602 .domain_free = qcom_iommu_domain_free,
603 .attach_dev = qcom_iommu_attach_dev,
604 .detach_dev = qcom_iommu_detach_dev,
605 .map = qcom_iommu_map,
606 .unmap = qcom_iommu_unmap,
607 .flush_iotlb_all = qcom_iommu_flush_iotlb_all,
608 .iotlb_sync = qcom_iommu_iotlb_sync,
609 .iova_to_phys = qcom_iommu_iova_to_phys,
610 .probe_device = qcom_iommu_probe_device,
611 .release_device = qcom_iommu_release_device,
612 .device_group = generic_device_group,
613 .of_xlate = qcom_iommu_of_xlate,
614 .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M,
615};
616
617static int qcom_iommu_sec_ptbl_init(struct device *dev)
618{
619 size_t psize = 0;
620 unsigned int spare = 0;
621 void *cpu_addr;
622 dma_addr_t paddr;
623 unsigned long attrs;
624 static bool allocated = false;
625 int ret;
626
627 if (allocated)
628 return 0;
629
630 ret = qcom_scm_iommu_secure_ptbl_size(spare, &psize);
631 if (ret) {
632 dev_err(dev, "failed to get iommu secure pgtable size (%d)\n",
633 ret);
634 return ret;
635 }
636
637 dev_info(dev, "iommu sec: pgtable size: %zu\n", psize);
638
639 attrs = DMA_ATTR_NO_KERNEL_MAPPING;
640
641 cpu_addr = dma_alloc_attrs(dev, psize, &paddr, GFP_KERNEL, attrs);
642 if (!cpu_addr) {
643 dev_err(dev, "failed to allocate %zu bytes for pgtable\n",
644 psize);
645 return -ENOMEM;
646 }
647
648 ret = qcom_scm_iommu_secure_ptbl_init(paddr, psize, spare);
649 if (ret) {
650 dev_err(dev, "failed to init iommu pgtable (%d)\n", ret);
651 goto free_mem;
652 }
653
654 allocated = true;
655 return 0;
656
657free_mem:
658 dma_free_attrs(dev, psize, cpu_addr, paddr, attrs);
659 return ret;
660}
661
662static int get_asid(const struct device_node *np)
663{
664 u32 reg;
665
666
667
668
669 if (of_property_read_u32_index(np, "reg", 0, ®))
670 return -ENODEV;
671
672 return reg / 0x1000;
673}
674
675static int qcom_iommu_ctx_probe(struct platform_device *pdev)
676{
677 struct qcom_iommu_ctx *ctx;
678 struct device *dev = &pdev->dev;
679 struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(dev->parent);
680 struct resource *res;
681 int ret, irq;
682
683 ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
684 if (!ctx)
685 return -ENOMEM;
686
687 ctx->dev = dev;
688 platform_set_drvdata(pdev, ctx);
689
690 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
691 ctx->base = devm_ioremap_resource(dev, res);
692 if (IS_ERR(ctx->base))
693 return PTR_ERR(ctx->base);
694
695 irq = platform_get_irq(pdev, 0);
696 if (irq < 0)
697 return -ENODEV;
698
699
700
701
702 iommu_writel(ctx, ARM_SMMU_CB_FSR, iommu_readl(ctx, ARM_SMMU_CB_FSR));
703
704 ret = devm_request_irq(dev, irq,
705 qcom_iommu_fault,
706 IRQF_SHARED,
707 "qcom-iommu-fault",
708 ctx);
709 if (ret) {
710 dev_err(dev, "failed to request IRQ %u\n", irq);
711 return ret;
712 }
713
714 ret = get_asid(dev->of_node);
715 if (ret < 0) {
716 dev_err(dev, "missing reg property\n");
717 return ret;
718 }
719
720 ctx->asid = ret;
721
722 dev_dbg(dev, "found asid %u\n", ctx->asid);
723
724 qcom_iommu->ctxs[ctx->asid - 1] = ctx;
725
726 return 0;
727}
728
729static int qcom_iommu_ctx_remove(struct platform_device *pdev)
730{
731 struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(pdev->dev.parent);
732 struct qcom_iommu_ctx *ctx = platform_get_drvdata(pdev);
733
734 platform_set_drvdata(pdev, NULL);
735
736 qcom_iommu->ctxs[ctx->asid - 1] = NULL;
737
738 return 0;
739}
740
741static const struct of_device_id ctx_of_match[] = {
742 { .compatible = "qcom,msm-iommu-v1-ns" },
743 { .compatible = "qcom,msm-iommu-v1-sec" },
744 { }
745};
746
747static struct platform_driver qcom_iommu_ctx_driver = {
748 .driver = {
749 .name = "qcom-iommu-ctx",
750 .of_match_table = ctx_of_match,
751 },
752 .probe = qcom_iommu_ctx_probe,
753 .remove = qcom_iommu_ctx_remove,
754};
755
756static bool qcom_iommu_has_secure_context(struct qcom_iommu_dev *qcom_iommu)
757{
758 struct device_node *child;
759
760 for_each_child_of_node(qcom_iommu->dev->of_node, child)
761 if (of_device_is_compatible(child, "qcom,msm-iommu-v1-sec"))
762 return true;
763
764 return false;
765}
766
767static int qcom_iommu_device_probe(struct platform_device *pdev)
768{
769 struct device_node *child;
770 struct qcom_iommu_dev *qcom_iommu;
771 struct device *dev = &pdev->dev;
772 struct resource *res;
773 struct clk *clk;
774 int ret, max_asid = 0;
775
776
777
778
779 for_each_child_of_node(dev->of_node, child)
780 max_asid = max(max_asid, get_asid(child));
781
782 qcom_iommu = devm_kzalloc(dev, struct_size(qcom_iommu, ctxs, max_asid),
783 GFP_KERNEL);
784 if (!qcom_iommu)
785 return -ENOMEM;
786 qcom_iommu->num_ctxs = max_asid;
787 qcom_iommu->dev = dev;
788
789 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
790 if (res) {
791 qcom_iommu->local_base = devm_ioremap_resource(dev, res);
792 if (IS_ERR(qcom_iommu->local_base))
793 return PTR_ERR(qcom_iommu->local_base);
794 }
795
796 clk = devm_clk_get(dev, "iface");
797 if (IS_ERR(clk)) {
798 dev_err(dev, "failed to get iface clock\n");
799 return PTR_ERR(clk);
800 }
801 qcom_iommu->clks[CLK_IFACE].clk = clk;
802
803 clk = devm_clk_get(dev, "bus");
804 if (IS_ERR(clk)) {
805 dev_err(dev, "failed to get bus clock\n");
806 return PTR_ERR(clk);
807 }
808 qcom_iommu->clks[CLK_BUS].clk = clk;
809
810 clk = devm_clk_get_optional(dev, "tbu");
811 if (IS_ERR(clk)) {
812 dev_err(dev, "failed to get tbu clock\n");
813 return PTR_ERR(clk);
814 }
815 qcom_iommu->clks[CLK_TBU].clk = clk;
816
817 if (of_property_read_u32(dev->of_node, "qcom,iommu-secure-id",
818 &qcom_iommu->sec_id)) {
819 dev_err(dev, "missing qcom,iommu-secure-id property\n");
820 return -ENODEV;
821 }
822
823 if (qcom_iommu_has_secure_context(qcom_iommu)) {
824 ret = qcom_iommu_sec_ptbl_init(dev);
825 if (ret) {
826 dev_err(dev, "cannot init secure pg table(%d)\n", ret);
827 return ret;
828 }
829 }
830
831 platform_set_drvdata(pdev, qcom_iommu);
832
833 pm_runtime_enable(dev);
834
835
836 ret = devm_of_platform_populate(dev);
837 if (ret) {
838 dev_err(dev, "Failed to populate iommu contexts\n");
839 return ret;
840 }
841
842 ret = iommu_device_sysfs_add(&qcom_iommu->iommu, dev, NULL,
843 dev_name(dev));
844 if (ret) {
845 dev_err(dev, "Failed to register iommu in sysfs\n");
846 return ret;
847 }
848
849 ret = iommu_device_register(&qcom_iommu->iommu, &qcom_iommu_ops, dev);
850 if (ret) {
851 dev_err(dev, "Failed to register iommu\n");
852 return ret;
853 }
854
855 bus_set_iommu(&platform_bus_type, &qcom_iommu_ops);
856
857 if (qcom_iommu->local_base) {
858 pm_runtime_get_sync(dev);
859 writel_relaxed(0xffffffff, qcom_iommu->local_base + SMMU_INTR_SEL_NS);
860 pm_runtime_put_sync(dev);
861 }
862
863 return 0;
864}
865
866static int qcom_iommu_device_remove(struct platform_device *pdev)
867{
868 struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
869
870 bus_set_iommu(&platform_bus_type, NULL);
871
872 pm_runtime_force_suspend(&pdev->dev);
873 platform_set_drvdata(pdev, NULL);
874 iommu_device_sysfs_remove(&qcom_iommu->iommu);
875 iommu_device_unregister(&qcom_iommu->iommu);
876
877 return 0;
878}
879
880static int __maybe_unused qcom_iommu_resume(struct device *dev)
881{
882 struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(dev);
883
884 return clk_bulk_prepare_enable(CLK_NUM, qcom_iommu->clks);
885}
886
887static int __maybe_unused qcom_iommu_suspend(struct device *dev)
888{
889 struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(dev);
890
891 clk_bulk_disable_unprepare(CLK_NUM, qcom_iommu->clks);
892
893 return 0;
894}
895
896static const struct dev_pm_ops qcom_iommu_pm_ops = {
897 SET_RUNTIME_PM_OPS(qcom_iommu_suspend, qcom_iommu_resume, NULL)
898 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
899 pm_runtime_force_resume)
900};
901
902static const struct of_device_id qcom_iommu_of_match[] = {
903 { .compatible = "qcom,msm-iommu-v1" },
904 { }
905};
906
907static struct platform_driver qcom_iommu_driver = {
908 .driver = {
909 .name = "qcom-iommu",
910 .of_match_table = qcom_iommu_of_match,
911 .pm = &qcom_iommu_pm_ops,
912 },
913 .probe = qcom_iommu_device_probe,
914 .remove = qcom_iommu_device_remove,
915};
916
917static int __init qcom_iommu_init(void)
918{
919 int ret;
920
921 ret = platform_driver_register(&qcom_iommu_ctx_driver);
922 if (ret)
923 return ret;
924
925 ret = platform_driver_register(&qcom_iommu_driver);
926 if (ret)
927 platform_driver_unregister(&qcom_iommu_ctx_driver);
928
929 return ret;
930}
931device_initcall(qcom_iommu_init);
932