1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Apple DART (Device Address Resolution Table) IOMMU driver
4 *
5 * Copyright (C) 2021 The Asahi Linux Contributors
6 *
7 * Based on arm/arm-smmu/arm-ssmu.c and arm/arm-smmu-v3/arm-smmu-v3.c
8 * Copyright (C) 2013 ARM Limited
9 * Copyright (C) 2015 ARM Limited
10 * and on exynos-iommu.c
11 * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd.
12 */
13
14#include <linux/atomic.h>
15#include <linux/bitfield.h>
16#include <linux/clk.h>
17#include <linux/dev_printk.h>
18#include <linux/dma-mapping.h>
19#include <linux/err.h>
20#include <linux/interrupt.h>
21#include <linux/io-pgtable.h>
22#include <linux/iommu.h>
23#include <linux/iopoll.h>
24#include <linux/module.h>
25#include <linux/of.h>
26#include <linux/of_address.h>
27#include <linux/of_iommu.h>
28#include <linux/of_platform.h>
29#include <linux/pci.h>
30#include <linux/platform_device.h>
31#include <linux/slab.h>
32#include <linux/swab.h>
33#include <linux/types.h>
34
35#include "dma-iommu.h"
36
37#define DART_MAX_STREAMS 256
38#define DART_MAX_TTBR 4
39#define MAX_DARTS_PER_DEVICE 3
40
41/* Common registers */
42
43#define DART_PARAMS1 0x00
44#define DART_PARAMS1_PAGE_SHIFT GENMASK(27, 24)
45
46#define DART_PARAMS2 0x04
47#define DART_PARAMS2_BYPASS_SUPPORT BIT(0)
48
49/* T8020/T6000 registers */
50
51#define DART_T8020_STREAM_COMMAND 0x20
52#define DART_T8020_STREAM_COMMAND_BUSY BIT(2)
53#define DART_T8020_STREAM_COMMAND_INVALIDATE BIT(20)
54
55#define DART_T8020_STREAM_SELECT 0x34
56
57#define DART_T8020_ERROR 0x40
58#define DART_T8020_ERROR_STREAM GENMASK(27, 24)
59#define DART_T8020_ERROR_CODE GENMASK(11, 0)
60#define DART_T8020_ERROR_FLAG BIT(31)
61
62#define DART_T8020_ERROR_READ_FAULT BIT(4)
63#define DART_T8020_ERROR_WRITE_FAULT BIT(3)
64#define DART_T8020_ERROR_NO_PTE BIT(2)
65#define DART_T8020_ERROR_NO_PMD BIT(1)
66#define DART_T8020_ERROR_NO_TTBR BIT(0)
67
68#define DART_T8020_CONFIG 0x60
69#define DART_T8020_CONFIG_LOCK BIT(15)
70
71#define DART_STREAM_COMMAND_BUSY_TIMEOUT 100
72
73#define DART_T8020_ERROR_ADDR_HI 0x54
74#define DART_T8020_ERROR_ADDR_LO 0x50
75
76#define DART_T8020_STREAMS_ENABLE 0xfc
77
78#define DART_T8020_TCR 0x100
79#define DART_T8020_TCR_TRANSLATE_ENABLE BIT(7)
80#define DART_T8020_TCR_BYPASS_DART BIT(8)
81#define DART_T8020_TCR_BYPASS_DAPF BIT(12)
82
83#define DART_T8020_TTBR 0x200
84#define DART_T8020_USB4_TTBR 0x400
85#define DART_T8020_TTBR_VALID BIT(31)
86#define DART_T8020_TTBR_ADDR_FIELD_SHIFT 0
87#define DART_T8020_TTBR_SHIFT 12
88
89/* T8110 registers */
90
91#define DART_T8110_PARAMS3 0x08
92#define DART_T8110_PARAMS3_PA_WIDTH GENMASK(29, 24)
93#define DART_T8110_PARAMS3_VA_WIDTH GENMASK(21, 16)
94#define DART_T8110_PARAMS3_VER_MAJ GENMASK(15, 8)
95#define DART_T8110_PARAMS3_VER_MIN GENMASK(7, 0)
96
97#define DART_T8110_PARAMS4 0x0c
98#define DART_T8110_PARAMS4_NUM_CLIENTS GENMASK(24, 16)
99#define DART_T8110_PARAMS4_NUM_SIDS GENMASK(8, 0)
100
101#define DART_T8110_TLB_CMD 0x80
102#define DART_T8110_TLB_CMD_BUSY BIT(31)
103#define DART_T8110_TLB_CMD_OP GENMASK(10, 8)
104#define DART_T8110_TLB_CMD_OP_FLUSH_ALL 0
105#define DART_T8110_TLB_CMD_OP_FLUSH_SID 1
106#define DART_T8110_TLB_CMD_STREAM GENMASK(7, 0)
107
108#define DART_T8110_ERROR 0x100
109#define DART_T8110_ERROR_STREAM GENMASK(27, 20)
110#define DART_T8110_ERROR_CODE GENMASK(14, 0)
111#define DART_T8110_ERROR_FLAG BIT(31)
112
113#define DART_T8110_ERROR_MASK 0x104
114
115#define DART_T8110_ERROR_READ_FAULT BIT(5)
116#define DART_T8110_ERROR_WRITE_FAULT BIT(4)
117#define DART_T8110_ERROR_NO_PTE BIT(3)
118#define DART_T8110_ERROR_NO_PMD BIT(2)
119#define DART_T8110_ERROR_NO_PGD BIT(1)
120#define DART_T8110_ERROR_NO_TTBR BIT(0)
121
122#define DART_T8110_ERROR_ADDR_LO 0x170
123#define DART_T8110_ERROR_ADDR_HI 0x174
124
125#define DART_T8110_ERROR_STREAMS 0x1c0
126
127#define DART_T8110_PROTECT 0x200
128#define DART_T8110_UNPROTECT 0x204
129#define DART_T8110_PROTECT_LOCK 0x208
130#define DART_T8110_PROTECT_TTBR_TCR BIT(0)
131
132#define DART_T8110_ENABLE_STREAMS 0xc00
133#define DART_T8110_DISABLE_STREAMS 0xc20
134
135#define DART_T8110_TCR 0x1000
136#define DART_T8110_TCR_REMAP GENMASK(11, 8)
137#define DART_T8110_TCR_REMAP_EN BIT(7)
138#define DART_T8110_TCR_FOUR_LEVEL BIT(3)
139#define DART_T8110_TCR_BYPASS_DAPF BIT(2)
140#define DART_T8110_TCR_BYPASS_DART BIT(1)
141#define DART_T8110_TCR_TRANSLATE_ENABLE BIT(0)
142
143#define DART_T8110_TTBR 0x1400
144#define DART_T8110_TTBR_VALID BIT(0)
145#define DART_T8110_TTBR_ADDR_FIELD_SHIFT 2
146#define DART_T8110_TTBR_SHIFT 14
147
148#define DART_TCR(dart, sid) ((dart)->hw->tcr + ((sid) << 2))
149
150#define DART_TTBR(dart, sid, idx) ((dart)->hw->ttbr + \
151 (((dart)->hw->ttbr_count * (sid)) << 2) + \
152 ((idx) << 2))
153
154struct apple_dart_stream_map;
155
156enum dart_type {
157 DART_T8020,
158 DART_T6000,
159 DART_T8110,
160};
161
162struct apple_dart_hw {
163 enum dart_type type;
164 irqreturn_t (*irq_handler)(int irq, void *dev);
165 int (*invalidate_tlb)(struct apple_dart_stream_map *stream_map);
166
167 u32 oas;
168 enum io_pgtable_fmt fmt;
169
170 int max_sid_count;
171
172 u32 lock;
173 u32 lock_bit;
174
175 u32 error;
176
177 u32 enable_streams;
178
179 u32 tcr;
180 u32 tcr_enabled;
181 u32 tcr_disabled;
182 u32 tcr_bypass;
183 u32 tcr_4level;
184
185 u32 ttbr;
186 u32 ttbr_valid;
187 u32 ttbr_addr_field_shift;
188 u32 ttbr_shift;
189 int ttbr_count;
190};
191
192/*
193 * Private structure associated with each DART device.
194 *
195 * @dev: device struct
196 * @hw: SoC-specific hardware data
197 * @regs: mapped MMIO region
198 * @irq: interrupt number, can be shared with other DARTs
199 * @clks: clocks associated with this DART
200 * @num_clks: number of @clks
201 * @lock: lock for hardware operations involving this dart
202 * @pgsize: pagesize supported by this DART
203 * @supports_bypass: indicates if this DART supports bypass mode
204 * @sid2group: maps stream ids to iommu_groups
205 * @iommu: iommu core device
206 */
207struct apple_dart {
208 struct device *dev;
209 const struct apple_dart_hw *hw;
210
211 void __iomem *regs;
212
213 int irq;
214 struct clk_bulk_data *clks;
215 int num_clks;
216
217 spinlock_t lock;
218
219 u32 ias;
220 u32 oas;
221 u32 pgsize;
222 u32 num_streams;
223 u32 supports_bypass : 1;
224 u32 four_level : 1;
225
226 struct iommu_group *sid2group[DART_MAX_STREAMS];
227 struct iommu_device iommu;
228
229 u32 save_tcr[DART_MAX_STREAMS];
230 u32 save_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR];
231};
232
233/*
234 * Convenience struct to identify streams.
235 *
236 * The normal variant is used inside apple_dart_master_cfg which isn't written
237 * to concurrently.
238 * The atomic variant is used inside apple_dart_domain where we have to guard
239 * against races from potential parallel calls to attach/detach_device.
240 * Note that even inside the atomic variant the apple_dart pointer is not
241 * protected: This pointer is initialized once under the domain init mutex
242 * and never changed again afterwards. Devices with different dart pointers
243 * cannot be attached to the same domain.
244 *
245 * @dart dart pointer
246 * @sid stream id bitmap
247 */
248struct apple_dart_stream_map {
249 struct apple_dart *dart;
250 DECLARE_BITMAP(sidmap, DART_MAX_STREAMS);
251};
252struct apple_dart_atomic_stream_map {
253 struct apple_dart *dart;
254 atomic_long_t sidmap[BITS_TO_LONGS(DART_MAX_STREAMS)];
255};
256
257/*
258 * This structure is attached to each iommu domain handled by a DART.
259 *
260 * @pgtbl_ops: pagetable ops allocated by io-pgtable
261 * @finalized: true if the domain has been completely initialized
262 * @init_lock: protects domain initialization
263 * @stream_maps: streams attached to this domain (valid for DMA/UNMANAGED only)
264 * @domain: core iommu domain pointer
265 */
266struct apple_dart_domain {
267 struct io_pgtable_ops *pgtbl_ops;
268
269 bool finalized;
270 struct mutex init_lock;
271 struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
272
273 struct iommu_domain domain;
274};
275
276/*
277 * This structure is attached to devices with dev_iommu_priv_set() on of_xlate
278 * and contains a list of streams bound to this device.
279 * So far the worst case seen is a single device with two streams
280 * from different darts, such that this simple static array is enough.
281 *
282 * @streams: streams for this device
283 */
284struct apple_dart_master_cfg {
285 /* Intersection of DART capabilitles */
286 u32 supports_bypass : 1;
287
288 struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
289};
290
291/*
292 * Helper macro to iterate over apple_dart_master_cfg.stream_maps and
293 * apple_dart_domain.stream_maps
294 *
295 * @i int used as loop variable
296 * @base pointer to base struct (apple_dart_master_cfg or apple_dart_domain)
297 * @stream pointer to the apple_dart_streams struct for each loop iteration
298 */
299#define for_each_stream_map(i, base, stream_map) \
300 for (i = 0, stream_map = &(base)->stream_maps[0]; \
301 i < MAX_DARTS_PER_DEVICE && stream_map->dart; \
302 stream_map = &(base)->stream_maps[++i])
303
304static struct platform_driver apple_dart_driver;
305static const struct iommu_ops apple_dart_iommu_ops;
306
307static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom)
308{
309 return container_of(dom, struct apple_dart_domain, domain);
310}
311
312static void
313apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map, int levels)
314{
315 struct apple_dart *dart = stream_map->dart;
316 u32 tcr = dart->hw->tcr_enabled;
317 int sid;
318
319 if (levels == 4)
320 tcr |= dart->hw->tcr_4level;
321
322 WARN_ON(levels != 3 && levels != 4);
323 WARN_ON(levels == 4 && !dart->four_level);
324 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
325 writel(val: tcr, addr: dart->regs + DART_TCR(dart, sid));
326}
327
328static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map)
329{
330 struct apple_dart *dart = stream_map->dart;
331 int sid;
332
333 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
334 writel(val: dart->hw->tcr_disabled, addr: dart->regs + DART_TCR(dart, sid));
335}
336
337static void
338apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map)
339{
340 struct apple_dart *dart = stream_map->dart;
341 int sid;
342
343 WARN_ON(!stream_map->dart->supports_bypass);
344 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
345 writel(val: dart->hw->tcr_bypass,
346 addr: dart->regs + DART_TCR(dart, sid));
347}
348
349static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map,
350 u8 idx, phys_addr_t paddr)
351{
352 struct apple_dart *dart = stream_map->dart;
353 int sid;
354
355 WARN_ON(paddr & ((1 << dart->hw->ttbr_shift) - 1));
356 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
357 writel(val: dart->hw->ttbr_valid |
358 (paddr >> dart->hw->ttbr_shift) << dart->hw->ttbr_addr_field_shift,
359 addr: dart->regs + DART_TTBR(dart, sid, idx));
360}
361
362static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map,
363 u8 idx)
364{
365 struct apple_dart *dart = stream_map->dart;
366 int sid;
367
368 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
369 writel(val: 0, addr: dart->regs + DART_TTBR(dart, sid, idx));
370}
371
372static void
373apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map)
374{
375 int i;
376
377 for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i)
378 apple_dart_hw_clear_ttbr(stream_map, idx: i);
379}
380
381static int
382apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map,
383 u32 command)
384{
385 unsigned long flags;
386 int ret, i;
387 u32 command_reg;
388
389 spin_lock_irqsave(&stream_map->dart->lock, flags);
390
391 for (i = 0; i < BITS_TO_U32(stream_map->dart->num_streams); i++)
392 writel(val: stream_map->sidmap[i],
393 addr: stream_map->dart->regs + DART_T8020_STREAM_SELECT + 4 * i);
394 writel(val: command, addr: stream_map->dart->regs + DART_T8020_STREAM_COMMAND);
395
396 ret = readl_poll_timeout_atomic(
397 stream_map->dart->regs + DART_T8020_STREAM_COMMAND, command_reg,
398 !(command_reg & DART_T8020_STREAM_COMMAND_BUSY), 1,
399 DART_STREAM_COMMAND_BUSY_TIMEOUT);
400
401 spin_unlock_irqrestore(lock: &stream_map->dart->lock, flags);
402
403 if (ret) {
404 dev_err(stream_map->dart->dev,
405 "busy bit did not clear after command %x for streams %lx\n",
406 command, stream_map->sidmap[0]);
407 return ret;
408 }
409
410 return 0;
411}
412
413static int
414apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map *stream_map,
415 u32 command)
416{
417 struct apple_dart *dart = stream_map->dart;
418 unsigned long flags;
419 int ret = 0;
420 int sid;
421
422 spin_lock_irqsave(&dart->lock, flags);
423
424 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) {
425 u32 val = FIELD_PREP(DART_T8110_TLB_CMD_OP, command) |
426 FIELD_PREP(DART_T8110_TLB_CMD_STREAM, sid);
427 writel(val, addr: dart->regs + DART_T8110_TLB_CMD);
428
429 ret = readl_poll_timeout_atomic(
430 dart->regs + DART_T8110_TLB_CMD, val,
431 !(val & DART_T8110_TLB_CMD_BUSY), 1,
432 DART_STREAM_COMMAND_BUSY_TIMEOUT);
433
434 if (ret)
435 break;
436
437 }
438
439 spin_unlock_irqrestore(lock: &dart->lock, flags);
440
441 if (ret) {
442 dev_err(stream_map->dart->dev,
443 "busy bit did not clear after command %x for stream %d\n",
444 command, sid);
445 return ret;
446 }
447
448 return 0;
449}
450
451static int
452apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map)
453{
454 return apple_dart_t8020_hw_stream_command(
455 stream_map, DART_T8020_STREAM_COMMAND_INVALIDATE);
456}
457
458static int
459apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map)
460{
461 return apple_dart_t8110_hw_tlb_command(
462 stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID);
463}
464
465static int apple_dart_hw_reset(struct apple_dart *dart)
466{
467 u32 config;
468 struct apple_dart_stream_map stream_map;
469 int i;
470
471 config = readl(addr: dart->regs + dart->hw->lock);
472 if (config & dart->hw->lock_bit) {
473 dev_err(dart->dev, "DART is locked down until reboot: %08x\n",
474 config);
475 return -EINVAL;
476 }
477
478 stream_map.dart = dart;
479 bitmap_zero(dst: stream_map.sidmap, DART_MAX_STREAMS);
480 bitmap_set(map: stream_map.sidmap, start: 0, nbits: dart->num_streams);
481 apple_dart_hw_disable_dma(stream_map: &stream_map);
482 apple_dart_hw_clear_all_ttbrs(stream_map: &stream_map);
483
484 /* enable all streams globally since TCR is used to control isolation */
485 for (i = 0; i < BITS_TO_U32(dart->num_streams); i++)
486 writel(U32_MAX, addr: dart->regs + dart->hw->enable_streams + 4 * i);
487
488 /* clear any pending errors before the interrupt is unmasked */
489 writel(readl(addr: dart->regs + dart->hw->error), addr: dart->regs + dart->hw->error);
490
491 if (dart->hw->type == DART_T8110)
492 writel(val: 0, addr: dart->regs + DART_T8110_ERROR_MASK);
493
494 return dart->hw->invalidate_tlb(&stream_map);
495}
496
497static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain)
498{
499 int i, j;
500 struct apple_dart_atomic_stream_map *domain_stream_map;
501 struct apple_dart_stream_map stream_map;
502
503 for_each_stream_map(i, domain, domain_stream_map) {
504 stream_map.dart = domain_stream_map->dart;
505
506 for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++)
507 stream_map.sidmap[j] = atomic_long_read(v: &domain_stream_map->sidmap[j]);
508
509 stream_map.dart->hw->invalidate_tlb(&stream_map);
510 }
511}
512
513static void apple_dart_flush_iotlb_all(struct iommu_domain *domain)
514{
515 apple_dart_domain_flush_tlb(domain: to_dart_domain(dom: domain));
516}
517
518static void apple_dart_iotlb_sync(struct iommu_domain *domain,
519 struct iommu_iotlb_gather *gather)
520{
521 apple_dart_domain_flush_tlb(domain: to_dart_domain(dom: domain));
522}
523
524static int apple_dart_iotlb_sync_map(struct iommu_domain *domain,
525 unsigned long iova, size_t size)
526{
527 apple_dart_domain_flush_tlb(domain: to_dart_domain(dom: domain));
528 return 0;
529}
530
531static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain,
532 dma_addr_t iova)
533{
534 struct apple_dart_domain *dart_domain = to_dart_domain(dom: domain);
535 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
536
537 if (!ops)
538 return 0;
539
540 return ops->iova_to_phys(ops, iova);
541}
542
543static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova,
544 phys_addr_t paddr, size_t pgsize,
545 size_t pgcount, int prot, gfp_t gfp,
546 size_t *mapped)
547{
548 struct apple_dart_domain *dart_domain = to_dart_domain(dom: domain);
549 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
550
551 if (!ops)
552 return -ENODEV;
553
554 return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp,
555 mapped);
556}
557
558static size_t apple_dart_unmap_pages(struct iommu_domain *domain,
559 unsigned long iova, size_t pgsize,
560 size_t pgcount,
561 struct iommu_iotlb_gather *gather)
562{
563 struct apple_dart_domain *dart_domain = to_dart_domain(dom: domain);
564 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
565
566 return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
567}
568
569static void
570apple_dart_setup_translation(struct apple_dart_domain *domain,
571 struct apple_dart_stream_map *stream_map)
572{
573 int i;
574 struct io_pgtable_cfg *pgtbl_cfg =
575 &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg;
576
577 for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i)
578 apple_dart_hw_set_ttbr(stream_map, idx: i,
579 paddr: pgtbl_cfg->apple_dart_cfg.ttbr[i]);
580 for (; i < stream_map->dart->hw->ttbr_count; ++i)
581 apple_dart_hw_clear_ttbr(stream_map, idx: i);
582
583 apple_dart_hw_enable_translation(stream_map,
584 levels: pgtbl_cfg->apple_dart_cfg.n_levels);
585 stream_map->dart->hw->invalidate_tlb(stream_map);
586}
587
588static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain,
589 struct apple_dart_master_cfg *cfg)
590{
591 struct apple_dart *dart = cfg->stream_maps[0].dart;
592 struct io_pgtable_cfg pgtbl_cfg;
593 int ret = 0;
594 int i, j;
595
596 if (dart->pgsize > PAGE_SIZE)
597 return -EINVAL;
598
599 mutex_lock(&dart_domain->init_lock);
600
601 if (dart_domain->finalized)
602 goto done;
603
604 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
605 dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart;
606 for (j = 0; j < BITS_TO_LONGS(dart->num_streams); j++)
607 atomic_long_set(v: &dart_domain->stream_maps[i].sidmap[j],
608 i: cfg->stream_maps[i].sidmap[j]);
609 }
610
611 pgtbl_cfg = (struct io_pgtable_cfg){
612 .pgsize_bitmap = dart->pgsize,
613 .ias = dart->ias,
614 .oas = dart->oas,
615 .coherent_walk = 1,
616 .iommu_dev = dart->dev,
617 };
618
619 dart_domain->pgtbl_ops = alloc_io_pgtable_ops(fmt: dart->hw->fmt, cfg: &pgtbl_cfg,
620 cookie: &dart_domain->domain);
621 if (!dart_domain->pgtbl_ops) {
622 ret = -ENOMEM;
623 goto done;
624 }
625
626 dart_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
627 dart_domain->domain.geometry.aperture_start = 0;
628 dart_domain->domain.geometry.aperture_end =
629 (dma_addr_t)DMA_BIT_MASK(pgtbl_cfg.ias);
630 dart_domain->domain.geometry.force_aperture = true;
631
632 dart_domain->finalized = true;
633
634done:
635 mutex_unlock(lock: &dart_domain->init_lock);
636 return ret;
637}
638
639static int
640apple_dart_mod_streams(struct apple_dart_atomic_stream_map *domain_maps,
641 struct apple_dart_stream_map *master_maps,
642 bool add_streams)
643{
644 int i, j;
645
646 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
647 if (domain_maps[i].dart != master_maps[i].dart)
648 return -EINVAL;
649 }
650
651 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
652 if (!domain_maps[i].dart)
653 break;
654 for (j = 0; j < BITS_TO_LONGS(domain_maps[i].dart->num_streams); j++) {
655 if (add_streams)
656 atomic_long_or(i: master_maps[i].sidmap[j],
657 v: &domain_maps[i].sidmap[j]);
658 else
659 atomic_long_and(i: ~master_maps[i].sidmap[j],
660 v: &domain_maps[i].sidmap[j]);
661 }
662 }
663
664 return 0;
665}
666
667static int apple_dart_domain_add_streams(struct apple_dart_domain *domain,
668 struct apple_dart_master_cfg *cfg)
669{
670 return apple_dart_mod_streams(domain_maps: domain->stream_maps, master_maps: cfg->stream_maps,
671 add_streams: true);
672}
673
674static int apple_dart_attach_dev_paging(struct iommu_domain *domain,
675 struct device *dev,
676 struct iommu_domain *old)
677{
678 int ret, i;
679 struct apple_dart_stream_map *stream_map;
680 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
681 struct apple_dart_domain *dart_domain = to_dart_domain(dom: domain);
682
683 ret = apple_dart_finalize_domain(dart_domain, cfg);
684 if (ret)
685 return ret;
686
687 ret = apple_dart_domain_add_streams(domain: dart_domain, cfg);
688 if (ret)
689 return ret;
690
691 for_each_stream_map(i, cfg, stream_map)
692 apple_dart_setup_translation(domain: dart_domain, stream_map);
693 return 0;
694}
695
696static int apple_dart_attach_dev_identity(struct iommu_domain *domain,
697 struct device *dev,
698 struct iommu_domain *old)
699{
700 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
701 struct apple_dart_stream_map *stream_map;
702 int i;
703
704 if (!cfg->supports_bypass)
705 return -EINVAL;
706
707 for_each_stream_map(i, cfg, stream_map)
708 apple_dart_hw_enable_bypass(stream_map);
709 return 0;
710}
711
712static const struct iommu_domain_ops apple_dart_identity_ops = {
713 .attach_dev = apple_dart_attach_dev_identity,
714};
715
716static struct iommu_domain apple_dart_identity_domain = {
717 .type = IOMMU_DOMAIN_IDENTITY,
718 .ops = &apple_dart_identity_ops,
719};
720
721static int apple_dart_attach_dev_blocked(struct iommu_domain *domain,
722 struct device *dev,
723 struct iommu_domain *old)
724{
725 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
726 struct apple_dart_stream_map *stream_map;
727 int i;
728
729 for_each_stream_map(i, cfg, stream_map)
730 apple_dart_hw_disable_dma(stream_map);
731 return 0;
732}
733
734static const struct iommu_domain_ops apple_dart_blocked_ops = {
735 .attach_dev = apple_dart_attach_dev_blocked,
736};
737
738static struct iommu_domain apple_dart_blocked_domain = {
739 .type = IOMMU_DOMAIN_BLOCKED,
740 .ops = &apple_dart_blocked_ops,
741};
742
743static struct iommu_device *apple_dart_probe_device(struct device *dev)
744{
745 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
746 struct apple_dart_stream_map *stream_map;
747 int i;
748
749 if (!cfg)
750 return ERR_PTR(error: -ENODEV);
751
752 for_each_stream_map(i, cfg, stream_map)
753 device_link_add(
754 consumer: dev, supplier: stream_map->dart->dev,
755 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
756
757 return &cfg->stream_maps[0].dart->iommu;
758}
759
760static void apple_dart_release_device(struct device *dev)
761{
762 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
763
764 kfree(objp: cfg);
765}
766
767static struct iommu_domain *apple_dart_domain_alloc_paging(struct device *dev)
768{
769 struct apple_dart_domain *dart_domain;
770
771 dart_domain = kzalloc(sizeof(*dart_domain), GFP_KERNEL);
772 if (!dart_domain)
773 return NULL;
774
775 mutex_init(&dart_domain->init_lock);
776
777 if (dev) {
778 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
779 int ret;
780
781 ret = apple_dart_finalize_domain(dart_domain, cfg);
782 if (ret) {
783 kfree(objp: dart_domain);
784 return ERR_PTR(error: ret);
785 }
786 }
787 return &dart_domain->domain;
788}
789
790static void apple_dart_domain_free(struct iommu_domain *domain)
791{
792 struct apple_dart_domain *dart_domain = to_dart_domain(dom: domain);
793
794 free_io_pgtable_ops(ops: dart_domain->pgtbl_ops);
795
796 kfree(objp: dart_domain);
797}
798
799static int apple_dart_of_xlate(struct device *dev,
800 const struct of_phandle_args *args)
801{
802 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
803 struct platform_device *iommu_pdev = of_find_device_by_node(np: args->np);
804 struct apple_dart *dart = platform_get_drvdata(pdev: iommu_pdev);
805 struct apple_dart *cfg_dart;
806 int i, sid;
807
808 put_device(dev: &iommu_pdev->dev);
809
810 if (args->args_count != 1)
811 return -EINVAL;
812 sid = args->args[0];
813
814 if (!cfg) {
815 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
816 if (!cfg)
817 return -ENOMEM;
818 /* Will be ANDed with DART capabilities */
819 cfg->supports_bypass = true;
820 }
821 dev_iommu_priv_set(dev, priv: cfg);
822
823 cfg_dart = cfg->stream_maps[0].dart;
824 if (cfg_dart) {
825 if (cfg_dart->pgsize != dart->pgsize)
826 return -EINVAL;
827 if (cfg_dart->ias != dart->ias)
828 return -EINVAL;
829 }
830
831 cfg->supports_bypass &= dart->supports_bypass;
832
833 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
834 if (cfg->stream_maps[i].dart == dart) {
835 set_bit(nr: sid, addr: cfg->stream_maps[i].sidmap);
836 return 0;
837 }
838 }
839 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
840 if (!cfg->stream_maps[i].dart) {
841 cfg->stream_maps[i].dart = dart;
842 set_bit(nr: sid, addr: cfg->stream_maps[i].sidmap);
843 return 0;
844 }
845 }
846
847 return -EINVAL;
848}
849
850static DEFINE_MUTEX(apple_dart_groups_lock);
851
852static void apple_dart_release_group(void *iommu_data)
853{
854 int i, sid;
855 struct apple_dart_stream_map *stream_map;
856 struct apple_dart_master_cfg *group_master_cfg = iommu_data;
857
858 mutex_lock(&apple_dart_groups_lock);
859
860 for_each_stream_map(i, group_master_cfg, stream_map)
861 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams)
862 stream_map->dart->sid2group[sid] = NULL;
863
864 kfree(objp: iommu_data);
865 mutex_unlock(lock: &apple_dart_groups_lock);
866}
867
868static int apple_dart_merge_master_cfg(struct apple_dart_master_cfg *dst,
869 struct apple_dart_master_cfg *src)
870{
871 /*
872 * We know that this function is only called for groups returned from
873 * pci_device_group and that all Apple Silicon platforms never spread
874 * PCIe devices from the same bus across multiple DARTs such that we can
875 * just assume that both src and dst only have the same single DART.
876 */
877 if (src->stream_maps[1].dart)
878 return -EINVAL;
879 if (dst->stream_maps[1].dart)
880 return -EINVAL;
881 if (src->stream_maps[0].dart != dst->stream_maps[0].dart)
882 return -EINVAL;
883
884 bitmap_or(dst: dst->stream_maps[0].sidmap,
885 src1: dst->stream_maps[0].sidmap,
886 src2: src->stream_maps[0].sidmap,
887 nbits: dst->stream_maps[0].dart->num_streams);
888 return 0;
889}
890
891static struct iommu_group *apple_dart_device_group(struct device *dev)
892{
893 int i, sid;
894 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
895 struct apple_dart_stream_map *stream_map;
896 struct apple_dart_master_cfg *group_master_cfg;
897 struct iommu_group *group = NULL;
898 struct iommu_group *res = ERR_PTR(error: -EINVAL);
899
900 mutex_lock(&apple_dart_groups_lock);
901
902 for_each_stream_map(i, cfg, stream_map) {
903 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) {
904 struct iommu_group *stream_group =
905 stream_map->dart->sid2group[sid];
906
907 if (group && group != stream_group) {
908 res = ERR_PTR(error: -EINVAL);
909 goto out;
910 }
911
912 group = stream_group;
913 }
914 }
915
916 if (group) {
917 res = iommu_group_ref_get(group);
918 goto out;
919 }
920
921#ifdef CONFIG_PCI
922 if (dev_is_pci(dev))
923 group = pci_device_group(dev);
924 else
925#endif
926 group = generic_device_group(dev);
927
928 res = ERR_PTR(error: -ENOMEM);
929 if (!group)
930 goto out;
931
932 group_master_cfg = iommu_group_get_iommudata(group);
933 if (group_master_cfg) {
934 int ret;
935
936 ret = apple_dart_merge_master_cfg(dst: group_master_cfg, src: cfg);
937 if (ret) {
938 dev_err(dev, "Failed to merge DART IOMMU groups.\n");
939 iommu_group_put(group);
940 res = ERR_PTR(error: ret);
941 goto out;
942 }
943 } else {
944 group_master_cfg = kmemdup(cfg, sizeof(*group_master_cfg),
945 GFP_KERNEL);
946 if (!group_master_cfg) {
947 iommu_group_put(group);
948 goto out;
949 }
950
951 iommu_group_set_iommudata(group, iommu_data: group_master_cfg,
952 release: apple_dart_release_group);
953 }
954
955 for_each_stream_map(i, cfg, stream_map)
956 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams)
957 stream_map->dart->sid2group[sid] = group;
958
959 res = group;
960
961out:
962 mutex_unlock(lock: &apple_dart_groups_lock);
963 return res;
964}
965
966static int apple_dart_def_domain_type(struct device *dev)
967{
968 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
969
970 if (cfg->stream_maps[0].dart->pgsize > PAGE_SIZE)
971 return IOMMU_DOMAIN_IDENTITY;
972 if (!cfg->supports_bypass)
973 return IOMMU_DOMAIN_DMA;
974
975 return 0;
976}
977
978#ifndef CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR
979/* Keep things compiling when CONFIG_PCI_APPLE isn't selected */
980#define CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR 0
981#endif
982#define DOORBELL_ADDR (CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR & PAGE_MASK)
983
984static void apple_dart_get_resv_regions(struct device *dev,
985 struct list_head *head)
986{
987 if (IS_ENABLED(CONFIG_PCIE_APPLE) && dev_is_pci(dev)) {
988 struct iommu_resv_region *region;
989 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
990
991 region = iommu_alloc_resv_region(DOORBELL_ADDR,
992 PAGE_SIZE, prot,
993 type: IOMMU_RESV_MSI, GFP_KERNEL);
994 if (!region)
995 return;
996
997 list_add_tail(new: &region->list, head);
998 }
999
1000 iommu_dma_get_resv_regions(dev, list: head);
1001}
1002
1003static const struct iommu_ops apple_dart_iommu_ops = {
1004 .identity_domain = &apple_dart_identity_domain,
1005 .blocked_domain = &apple_dart_blocked_domain,
1006 .domain_alloc_paging = apple_dart_domain_alloc_paging,
1007 .probe_device = apple_dart_probe_device,
1008 .release_device = apple_dart_release_device,
1009 .device_group = apple_dart_device_group,
1010 .of_xlate = apple_dart_of_xlate,
1011 .def_domain_type = apple_dart_def_domain_type,
1012 .get_resv_regions = apple_dart_get_resv_regions,
1013 .owner = THIS_MODULE,
1014 .default_domain_ops = &(const struct iommu_domain_ops) {
1015 .attach_dev = apple_dart_attach_dev_paging,
1016 .map_pages = apple_dart_map_pages,
1017 .unmap_pages = apple_dart_unmap_pages,
1018 .flush_iotlb_all = apple_dart_flush_iotlb_all,
1019 .iotlb_sync = apple_dart_iotlb_sync,
1020 .iotlb_sync_map = apple_dart_iotlb_sync_map,
1021 .iova_to_phys = apple_dart_iova_to_phys,
1022 .free = apple_dart_domain_free,
1023 }
1024};
1025
1026static irqreturn_t apple_dart_t8020_irq(int irq, void *dev)
1027{
1028 struct apple_dart *dart = dev;
1029 const char *fault_name = NULL;
1030 u32 error = readl(addr: dart->regs + DART_T8020_ERROR);
1031 u32 error_code = FIELD_GET(DART_T8020_ERROR_CODE, error);
1032 u32 addr_lo = readl(addr: dart->regs + DART_T8020_ERROR_ADDR_LO);
1033 u32 addr_hi = readl(addr: dart->regs + DART_T8020_ERROR_ADDR_HI);
1034 u64 addr = addr_lo | (((u64)addr_hi) << 32);
1035 u8 stream_idx = FIELD_GET(DART_T8020_ERROR_STREAM, error);
1036
1037 if (!(error & DART_T8020_ERROR_FLAG))
1038 return IRQ_NONE;
1039
1040 /* there should only be a single bit set but let's use == to be sure */
1041 if (error_code == DART_T8020_ERROR_READ_FAULT)
1042 fault_name = "READ FAULT";
1043 else if (error_code == DART_T8020_ERROR_WRITE_FAULT)
1044 fault_name = "WRITE FAULT";
1045 else if (error_code == DART_T8020_ERROR_NO_PTE)
1046 fault_name = "NO PTE FOR IOVA";
1047 else if (error_code == DART_T8020_ERROR_NO_PMD)
1048 fault_name = "NO PMD FOR IOVA";
1049 else if (error_code == DART_T8020_ERROR_NO_TTBR)
1050 fault_name = "NO TTBR FOR IOVA";
1051 else
1052 fault_name = "unknown";
1053
1054 dev_err_ratelimited(
1055 dart->dev,
1056 "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx",
1057 error, stream_idx, error_code, fault_name, addr);
1058
1059 writel(val: error, addr: dart->regs + DART_T8020_ERROR);
1060 return IRQ_HANDLED;
1061}
1062
1063static irqreturn_t apple_dart_t8110_irq(int irq, void *dev)
1064{
1065 struct apple_dart *dart = dev;
1066 const char *fault_name = NULL;
1067 u32 error = readl(addr: dart->regs + DART_T8110_ERROR);
1068 u32 error_code = FIELD_GET(DART_T8110_ERROR_CODE, error);
1069 u32 addr_lo = readl(addr: dart->regs + DART_T8110_ERROR_ADDR_LO);
1070 u32 addr_hi = readl(addr: dart->regs + DART_T8110_ERROR_ADDR_HI);
1071 u64 addr = addr_lo | (((u64)addr_hi) << 32);
1072 u8 stream_idx = FIELD_GET(DART_T8110_ERROR_STREAM, error);
1073
1074 if (!(error & DART_T8110_ERROR_FLAG))
1075 return IRQ_NONE;
1076
1077 /* there should only be a single bit set but let's use == to be sure */
1078 if (error_code == DART_T8110_ERROR_READ_FAULT)
1079 fault_name = "READ FAULT";
1080 else if (error_code == DART_T8110_ERROR_WRITE_FAULT)
1081 fault_name = "WRITE FAULT";
1082 else if (error_code == DART_T8110_ERROR_NO_PTE)
1083 fault_name = "NO PTE FOR IOVA";
1084 else if (error_code == DART_T8110_ERROR_NO_PMD)
1085 fault_name = "NO PMD FOR IOVA";
1086 else if (error_code == DART_T8110_ERROR_NO_PGD)
1087 fault_name = "NO PGD FOR IOVA";
1088 else if (error_code == DART_T8110_ERROR_NO_TTBR)
1089 fault_name = "NO TTBR FOR IOVA";
1090 else
1091 fault_name = "unknown";
1092
1093 dev_err_ratelimited(
1094 dart->dev,
1095 "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx",
1096 error, stream_idx, error_code, fault_name, addr);
1097
1098 writel(val: error, addr: dart->regs + DART_T8110_ERROR);
1099 for (int i = 0; i < BITS_TO_U32(dart->num_streams); i++)
1100 writel(U32_MAX, addr: dart->regs + DART_T8110_ERROR_STREAMS + 4 * i);
1101
1102 return IRQ_HANDLED;
1103}
1104
1105static int apple_dart_probe(struct platform_device *pdev)
1106{
1107 int ret;
1108 u32 dart_params[4];
1109 struct resource *res;
1110 struct apple_dart *dart;
1111 struct device *dev = &pdev->dev;
1112
1113 dart = devm_kzalloc(dev, size: sizeof(*dart), GFP_KERNEL);
1114 if (!dart)
1115 return -ENOMEM;
1116
1117 dart->dev = dev;
1118 dart->hw = of_device_get_match_data(dev);
1119 spin_lock_init(&dart->lock);
1120
1121 dart->regs = devm_platform_get_and_ioremap_resource(pdev, index: 0, res: &res);
1122 if (IS_ERR(ptr: dart->regs))
1123 return PTR_ERR(ptr: dart->regs);
1124
1125 if (resource_size(res) < 0x4000) {
1126 dev_err(dev, "MMIO region too small (%pr)\n", res);
1127 return -EINVAL;
1128 }
1129
1130 dart->irq = platform_get_irq(pdev, 0);
1131 if (dart->irq < 0)
1132 return -ENODEV;
1133
1134 ret = devm_clk_bulk_get_all(dev, clks: &dart->clks);
1135 if (ret < 0)
1136 return ret;
1137 dart->num_clks = ret;
1138
1139 ret = clk_bulk_prepare_enable(num_clks: dart->num_clks, clks: dart->clks);
1140 if (ret)
1141 return ret;
1142
1143 dart_params[0] = readl(addr: dart->regs + DART_PARAMS1);
1144 dart_params[1] = readl(addr: dart->regs + DART_PARAMS2);
1145 dart->pgsize = 1 << FIELD_GET(DART_PARAMS1_PAGE_SHIFT, dart_params[0]);
1146 dart->supports_bypass = dart_params[1] & DART_PARAMS2_BYPASS_SUPPORT;
1147
1148 switch (dart->hw->type) {
1149 case DART_T8020:
1150 case DART_T6000:
1151 dart->ias = 32;
1152 dart->oas = dart->hw->oas;
1153 dart->num_streams = dart->hw->max_sid_count;
1154 break;
1155
1156 case DART_T8110:
1157 dart_params[2] = readl(addr: dart->regs + DART_T8110_PARAMS3);
1158 dart_params[3] = readl(addr: dart->regs + DART_T8110_PARAMS4);
1159 dart->ias = FIELD_GET(DART_T8110_PARAMS3_VA_WIDTH, dart_params[2]);
1160 dart->oas = FIELD_GET(DART_T8110_PARAMS3_PA_WIDTH, dart_params[2]);
1161 dart->num_streams = FIELD_GET(DART_T8110_PARAMS4_NUM_SIDS, dart_params[3]);
1162 dart->four_level = dart->ias > 36;
1163 break;
1164 }
1165
1166 if (dart->num_streams > DART_MAX_STREAMS) {
1167 dev_err(&pdev->dev, "Too many streams (%d > %d)\n",
1168 dart->num_streams, DART_MAX_STREAMS);
1169 ret = -EINVAL;
1170 goto err_clk_disable;
1171 }
1172
1173 ret = apple_dart_hw_reset(dart);
1174 if (ret)
1175 goto err_clk_disable;
1176
1177 ret = request_irq(irq: dart->irq, handler: dart->hw->irq_handler, IRQF_SHARED,
1178 name: "apple-dart fault handler", dev: dart);
1179 if (ret)
1180 goto err_clk_disable;
1181
1182 platform_set_drvdata(pdev, data: dart);
1183
1184 ret = iommu_device_sysfs_add(iommu: &dart->iommu, parent: dev, NULL, fmt: "apple-dart.%s",
1185 dev_name(dev: &pdev->dev));
1186 if (ret)
1187 goto err_free_irq;
1188
1189 ret = iommu_device_register(iommu: &dart->iommu, ops: &apple_dart_iommu_ops, hwdev: dev);
1190 if (ret)
1191 goto err_sysfs_remove;
1192
1193 dev_info(
1194 &pdev->dev,
1195 "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d, AS %d -> %d] initialized\n",
1196 dart->pgsize, dart->num_streams, dart->supports_bypass,
1197 dart->pgsize > PAGE_SIZE, dart->ias, dart->oas);
1198 return 0;
1199
1200err_sysfs_remove:
1201 iommu_device_sysfs_remove(iommu: &dart->iommu);
1202err_free_irq:
1203 free_irq(dart->irq, dart);
1204err_clk_disable:
1205 clk_bulk_disable_unprepare(num_clks: dart->num_clks, clks: dart->clks);
1206
1207 return ret;
1208}
1209
1210static void apple_dart_remove(struct platform_device *pdev)
1211{
1212 struct apple_dart *dart = platform_get_drvdata(pdev);
1213
1214 apple_dart_hw_reset(dart);
1215 free_irq(dart->irq, dart);
1216
1217 iommu_device_unregister(iommu: &dart->iommu);
1218 iommu_device_sysfs_remove(iommu: &dart->iommu);
1219
1220 clk_bulk_disable_unprepare(num_clks: dart->num_clks, clks: dart->clks);
1221}
1222
1223static const struct apple_dart_hw apple_dart_hw_t8103 = {
1224 .type = DART_T8020,
1225 .irq_handler = apple_dart_t8020_irq,
1226 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb,
1227 .oas = 36,
1228 .fmt = APPLE_DART,
1229 .max_sid_count = 16,
1230
1231 .enable_streams = DART_T8020_STREAMS_ENABLE,
1232 .lock = DART_T8020_CONFIG,
1233 .lock_bit = DART_T8020_CONFIG_LOCK,
1234
1235 .error = DART_T8020_ERROR,
1236
1237 .tcr = DART_T8020_TCR,
1238 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE,
1239 .tcr_disabled = 0,
1240 .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART,
1241
1242 .ttbr = DART_T8020_TTBR,
1243 .ttbr_valid = DART_T8020_TTBR_VALID,
1244 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT,
1245 .ttbr_shift = DART_T8020_TTBR_SHIFT,
1246 .ttbr_count = 4,
1247};
1248
1249static const struct apple_dart_hw apple_dart_hw_t8103_usb4 = {
1250 .type = DART_T8020,
1251 .irq_handler = apple_dart_t8020_irq,
1252 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb,
1253 .oas = 36,
1254 .fmt = APPLE_DART,
1255 .max_sid_count = 64,
1256
1257 .enable_streams = DART_T8020_STREAMS_ENABLE,
1258 .lock = DART_T8020_CONFIG,
1259 .lock_bit = DART_T8020_CONFIG_LOCK,
1260
1261 .error = DART_T8020_ERROR,
1262
1263 .tcr = DART_T8020_TCR,
1264 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE,
1265 .tcr_disabled = 0,
1266 .tcr_bypass = 0,
1267
1268 .ttbr = DART_T8020_USB4_TTBR,
1269 .ttbr_valid = DART_T8020_TTBR_VALID,
1270 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT,
1271 .ttbr_shift = DART_T8020_TTBR_SHIFT,
1272 .ttbr_count = 4,
1273};
1274
1275static const struct apple_dart_hw apple_dart_hw_t6000 = {
1276 .type = DART_T6000,
1277 .irq_handler = apple_dart_t8020_irq,
1278 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb,
1279 .oas = 42,
1280 .fmt = APPLE_DART2,
1281 .max_sid_count = 16,
1282
1283 .enable_streams = DART_T8020_STREAMS_ENABLE,
1284 .lock = DART_T8020_CONFIG,
1285 .lock_bit = DART_T8020_CONFIG_LOCK,
1286
1287 .error = DART_T8020_ERROR,
1288
1289 .tcr = DART_T8020_TCR,
1290 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE,
1291 .tcr_disabled = 0,
1292 .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART,
1293
1294 .ttbr = DART_T8020_TTBR,
1295 .ttbr_valid = DART_T8020_TTBR_VALID,
1296 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT,
1297 .ttbr_shift = DART_T8020_TTBR_SHIFT,
1298 .ttbr_count = 4,
1299};
1300
1301static const struct apple_dart_hw apple_dart_hw_t8110 = {
1302 .type = DART_T8110,
1303 .irq_handler = apple_dart_t8110_irq,
1304 .invalidate_tlb = apple_dart_t8110_hw_invalidate_tlb,
1305 .fmt = APPLE_DART2,
1306 .max_sid_count = 256,
1307
1308 .enable_streams = DART_T8110_ENABLE_STREAMS,
1309 .lock = DART_T8110_PROTECT,
1310 .lock_bit = DART_T8110_PROTECT_TTBR_TCR,
1311
1312 .error = DART_T8110_ERROR,
1313
1314 .tcr = DART_T8110_TCR,
1315 .tcr_enabled = DART_T8110_TCR_TRANSLATE_ENABLE,
1316 .tcr_disabled = 0,
1317 .tcr_bypass = DART_T8110_TCR_BYPASS_DAPF | DART_T8110_TCR_BYPASS_DART,
1318 .tcr_4level = DART_T8110_TCR_FOUR_LEVEL,
1319
1320 .ttbr = DART_T8110_TTBR,
1321 .ttbr_valid = DART_T8110_TTBR_VALID,
1322 .ttbr_addr_field_shift = DART_T8110_TTBR_ADDR_FIELD_SHIFT,
1323 .ttbr_shift = DART_T8110_TTBR_SHIFT,
1324 .ttbr_count = 1,
1325};
1326
1327static __maybe_unused int apple_dart_suspend(struct device *dev)
1328{
1329 struct apple_dart *dart = dev_get_drvdata(dev);
1330 unsigned int sid, idx;
1331
1332 for (sid = 0; sid < dart->num_streams; sid++) {
1333 dart->save_tcr[sid] = readl(addr: dart->regs + DART_TCR(dart, sid));
1334 for (idx = 0; idx < dart->hw->ttbr_count; idx++)
1335 dart->save_ttbr[sid][idx] =
1336 readl(addr: dart->regs + DART_TTBR(dart, sid, idx));
1337 }
1338
1339 return 0;
1340}
1341
1342static __maybe_unused int apple_dart_resume(struct device *dev)
1343{
1344 struct apple_dart *dart = dev_get_drvdata(dev);
1345 unsigned int sid, idx;
1346 int ret;
1347
1348 ret = apple_dart_hw_reset(dart);
1349 if (ret) {
1350 dev_err(dev, "Failed to reset DART on resume\n");
1351 return ret;
1352 }
1353
1354 for (sid = 0; sid < dart->num_streams; sid++) {
1355 for (idx = 0; idx < dart->hw->ttbr_count; idx++)
1356 writel(val: dart->save_ttbr[sid][idx],
1357 addr: dart->regs + DART_TTBR(dart, sid, idx));
1358 writel(val: dart->save_tcr[sid], addr: dart->regs + DART_TCR(dart, sid));
1359 }
1360
1361 return 0;
1362}
1363
1364static DEFINE_SIMPLE_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume);
1365
1366static const struct of_device_id apple_dart_of_match[] = {
1367 { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 },
1368 { .compatible = "apple,t8103-usb4-dart", .data = &apple_dart_hw_t8103_usb4 },
1369 { .compatible = "apple,t8110-dart", .data = &apple_dart_hw_t8110 },
1370 { .compatible = "apple,t6000-dart", .data = &apple_dart_hw_t6000 },
1371 {},
1372};
1373MODULE_DEVICE_TABLE(of, apple_dart_of_match);
1374
1375static struct platform_driver apple_dart_driver = {
1376 .driver = {
1377 .name = "apple-dart",
1378 .of_match_table = apple_dart_of_match,
1379 .suppress_bind_attrs = true,
1380 .pm = pm_sleep_ptr(&apple_dart_pm_ops),
1381 },
1382 .probe = apple_dart_probe,
1383 .remove = apple_dart_remove,
1384};
1385
1386module_platform_driver(apple_dart_driver);
1387
1388MODULE_DESCRIPTION("IOMMU API for Apple's DART");
1389MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
1390MODULE_LICENSE("GPL v2");
1391

source code of linux/drivers/iommu/apple-dart.c