1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * CPU-agnostic ARM page table allocator.
4 *
5 * Copyright (C) 2014 ARM Limited
6 *
7 * Author: Will Deacon <[email protected]>
8 */
9
10 #define pr_fmt(fmt) "arm-lpae io-pgtable: " fmt
11
12 #include <linux/atomic.h>
13 #include <linux/bitops.h>
14 #include <linux/io-pgtable.h>
15 #include <linux/kernel.h>
16 #include <linux/sizes.h>
17 #include <linux/slab.h>
18 #include <linux/types.h>
19 #include <linux/dma-mapping.h>
20
21 #include <asm/barrier.h>
22
23 #include "io-pgtable-arm.h"
24 #include "iommu-pages.h"
25
26 #define ARM_LPAE_MAX_ADDR_BITS 52
27 #define ARM_LPAE_S2_MAX_CONCAT_PAGES 16
28 #define ARM_LPAE_MAX_LEVELS 4
29
30 /* Struct accessors */
31 #define io_pgtable_to_data(x) \
32 container_of((x), struct arm_lpae_io_pgtable, iop)
33
34 #define io_pgtable_ops_to_data(x) \
35 io_pgtable_to_data(io_pgtable_ops_to_pgtable(x))
36
37 /*
38 * Calculate the right shift amount to get to the portion describing level l
39 * in a virtual address mapped by the pagetable in d.
40 */
41 #define ARM_LPAE_LVL_SHIFT(l,d) \
42 (((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level) + \
43 ilog2(sizeof(arm_lpae_iopte)))
44
45 #define ARM_LPAE_GRANULE(d) \
46 (sizeof(arm_lpae_iopte) << (d)->bits_per_level)
47 #define ARM_LPAE_PGD_SIZE(d) \
48 (sizeof(arm_lpae_iopte) << (d)->pgd_bits)
49
50 #define ARM_LPAE_PTES_PER_TABLE(d) \
51 (ARM_LPAE_GRANULE(d) >> ilog2(sizeof(arm_lpae_iopte)))
52
53 /*
54 * Calculate the index at level l used to map virtual address a using the
55 * pagetable in d.
56 */
57 #define ARM_LPAE_PGD_IDX(l,d) \
58 ((l) == (d)->start_level ? (d)->pgd_bits - (d)->bits_per_level : 0)
59
60 #define ARM_LPAE_LVL_IDX(a,l,d) \
61 (((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) & \
62 ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1))
63
64 /* Calculate the block/page mapping size at level l for pagetable in d. */
65 #define ARM_LPAE_BLOCK_SIZE(l,d) (1ULL << ARM_LPAE_LVL_SHIFT(l,d))
66
67 /* Page table bits */
68 #define ARM_LPAE_PTE_TYPE_SHIFT 0
69 #define ARM_LPAE_PTE_TYPE_MASK 0x3
70
71 #define ARM_LPAE_PTE_TYPE_BLOCK 1
72 #define ARM_LPAE_PTE_TYPE_TABLE 3
73 #define ARM_LPAE_PTE_TYPE_PAGE 3
74
75 #define ARM_LPAE_PTE_ADDR_MASK GENMASK_ULL(47,12)
76
77 #define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63)
78 #define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53)
79 #define ARM_LPAE_PTE_DBM (((arm_lpae_iopte)1) << 51)
80 #define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10)
81 #define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8)
82 #define ARM_LPAE_PTE_SH_OS (((arm_lpae_iopte)2) << 8)
83 #define ARM_LPAE_PTE_SH_IS (((arm_lpae_iopte)3) << 8)
84 #define ARM_LPAE_PTE_NS (((arm_lpae_iopte)1) << 5)
85 #define ARM_LPAE_PTE_VALID (((arm_lpae_iopte)1) << 0)
86
87 #define ARM_LPAE_PTE_ATTR_LO_MASK (((arm_lpae_iopte)0x3ff) << 2)
88 /* Ignore the contiguous bit for block splitting */
89 #define ARM_LPAE_PTE_ATTR_HI_MASK (ARM_LPAE_PTE_XN | ARM_LPAE_PTE_DBM)
90 #define ARM_LPAE_PTE_ATTR_MASK (ARM_LPAE_PTE_ATTR_LO_MASK | \
91 ARM_LPAE_PTE_ATTR_HI_MASK)
92 /* Software bit for solving coherency races */
93 #define ARM_LPAE_PTE_SW_SYNC (((arm_lpae_iopte)1) << 55)
94
95 /* Stage-1 PTE */
96 #define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6)
97 #define ARM_LPAE_PTE_AP_RDONLY_BIT 7
98 #define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)1) << \
99 ARM_LPAE_PTE_AP_RDONLY_BIT)
100 #define ARM_LPAE_PTE_AP_WR_CLEAN_MASK (ARM_LPAE_PTE_AP_RDONLY | \
101 ARM_LPAE_PTE_DBM)
102 #define ARM_LPAE_PTE_ATTRINDX_SHIFT 2
103 #define ARM_LPAE_PTE_nG (((arm_lpae_iopte)1) << 11)
104
105 /* Stage-2 PTE */
106 #define ARM_LPAE_PTE_HAP_FAULT (((arm_lpae_iopte)0) << 6)
107 #define ARM_LPAE_PTE_HAP_READ (((arm_lpae_iopte)1) << 6)
108 #define ARM_LPAE_PTE_HAP_WRITE (((arm_lpae_iopte)2) << 6)
109 /*
110 * For !FWB these code to:
111 * 1111 = Normal outer write back cachable / Inner Write Back Cachable
112 * Permit S1 to override
113 * 0101 = Normal Non-cachable / Inner Non-cachable
114 * 0001 = Device / Device-nGnRE
115 * For S2FWB these code:
116 * 0110 Force Normal Write Back
117 * 0101 Normal* is forced Normal-NC, Device unchanged
118 * 0001 Force Device-nGnRE
119 */
120 #define ARM_LPAE_PTE_MEMATTR_FWB_WB (((arm_lpae_iopte)0x6) << 2)
121 #define ARM_LPAE_PTE_MEMATTR_OIWB (((arm_lpae_iopte)0xf) << 2)
122 #define ARM_LPAE_PTE_MEMATTR_NC (((arm_lpae_iopte)0x5) << 2)
123 #define ARM_LPAE_PTE_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2)
124
125 /* Register bits */
126 #define ARM_LPAE_VTCR_SL0_MASK 0x3
127
128 #define ARM_LPAE_TCR_T0SZ_SHIFT 0
129
130 #define ARM_LPAE_VTCR_PS_SHIFT 16
131 #define ARM_LPAE_VTCR_PS_MASK 0x7
132
133 #define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3)
134 #define ARM_LPAE_MAIR_ATTR_MASK 0xff
135 #define ARM_LPAE_MAIR_ATTR_DEVICE 0x04
136 #define ARM_LPAE_MAIR_ATTR_NC 0x44
137 #define ARM_LPAE_MAIR_ATTR_INC_OWBRWA 0xf4
138 #define ARM_LPAE_MAIR_ATTR_WBRWA 0xff
139 #define ARM_LPAE_MAIR_ATTR_IDX_NC 0
140 #define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1
141 #define ARM_LPAE_MAIR_ATTR_IDX_DEV 2
142 #define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE 3
143
144 #define ARM_MALI_LPAE_TTBR_ADRMODE_TABLE (3u << 0)
145 #define ARM_MALI_LPAE_TTBR_READ_INNER BIT(2)
146 #define ARM_MALI_LPAE_TTBR_SHARE_OUTER BIT(4)
147
148 #define ARM_MALI_LPAE_MEMATTR_IMP_DEF 0x88ULL
149 #define ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 0x8DULL
150
151 /* IOPTE accessors */
152 #define iopte_deref(pte,d) __va(iopte_to_paddr(pte, d))
153
154 #define iopte_type(pte) \
155 (((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK)
156
157 #define iopte_prot(pte) ((pte) & ARM_LPAE_PTE_ATTR_MASK)
158
159 #define iopte_writeable_dirty(pte) \
160 (((pte) & ARM_LPAE_PTE_AP_WR_CLEAN_MASK) == ARM_LPAE_PTE_DBM)
161
162 #define iopte_set_writeable_clean(ptep) \
163 set_bit(ARM_LPAE_PTE_AP_RDONLY_BIT, (unsigned long *)(ptep))
164
165 struct arm_lpae_io_pgtable {
166 struct io_pgtable iop;
167
168 int pgd_bits;
169 int start_level;
170 int bits_per_level;
171
172 void *pgd;
173 };
174
175 typedef u64 arm_lpae_iopte;
176
iopte_leaf(arm_lpae_iopte pte,int lvl,enum io_pgtable_fmt fmt)177 static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl,
178 enum io_pgtable_fmt fmt)
179 {
180 if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE)
181 return iopte_type(pte) == ARM_LPAE_PTE_TYPE_PAGE;
182
183 return iopte_type(pte) == ARM_LPAE_PTE_TYPE_BLOCK;
184 }
185
iopte_table(arm_lpae_iopte pte,int lvl)186 static inline bool iopte_table(arm_lpae_iopte pte, int lvl)
187 {
188 if (lvl == (ARM_LPAE_MAX_LEVELS - 1))
189 return false;
190 return iopte_type(pte) == ARM_LPAE_PTE_TYPE_TABLE;
191 }
192
paddr_to_iopte(phys_addr_t paddr,struct arm_lpae_io_pgtable * data)193 static arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr,
194 struct arm_lpae_io_pgtable *data)
195 {
196 arm_lpae_iopte pte = paddr;
197
198 /* Of the bits which overlap, either 51:48 or 15:12 are always RES0 */
199 return (pte | (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK;
200 }
201
iopte_to_paddr(arm_lpae_iopte pte,struct arm_lpae_io_pgtable * data)202 static phys_addr_t iopte_to_paddr(arm_lpae_iopte pte,
203 struct arm_lpae_io_pgtable *data)
204 {
205 u64 paddr = pte & ARM_LPAE_PTE_ADDR_MASK;
206
207 if (ARM_LPAE_GRANULE(data) < SZ_64K)
208 return paddr;
209
210 /* Rotate the packed high-order bits back to the top */
211 return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4);
212 }
213
214 /*
215 * Convert an index returned by ARM_LPAE_PGD_IDX(), which can point into
216 * a concatenated PGD, into the maximum number of entries that can be
217 * mapped in the same table page.
218 */
arm_lpae_max_entries(int i,struct arm_lpae_io_pgtable * data)219 static inline int arm_lpae_max_entries(int i, struct arm_lpae_io_pgtable *data)
220 {
221 int ptes_per_table = ARM_LPAE_PTES_PER_TABLE(data);
222
223 return ptes_per_table - (i & (ptes_per_table - 1));
224 }
225
226 /*
227 * Check if concatenated PGDs are mandatory according to Arm DDI0487 (K.a)
228 * 1) R_DXBSH: For 16KB, and 48-bit input size, use level 1 instead of 0.
229 * 2) R_SRKBC: After de-ciphering the table for PA size and valid initial lookup
230 * a) 40 bits PA size with 4K: use level 1 instead of level 0 (2 tables for ias = oas)
231 * b) 40 bits PA size with 16K: use level 2 instead of level 1 (16 tables for ias = oas)
232 * c) 42 bits PA size with 4K: use level 1 instead of level 0 (8 tables for ias = oas)
233 * d) 48 bits PA size with 16K: use level 1 instead of level 0 (2 tables for ias = oas)
234 */
arm_lpae_concat_mandatory(struct io_pgtable_cfg * cfg,struct arm_lpae_io_pgtable * data)235 static inline bool arm_lpae_concat_mandatory(struct io_pgtable_cfg *cfg,
236 struct arm_lpae_io_pgtable *data)
237 {
238 unsigned int ias = cfg->ias;
239 unsigned int oas = cfg->oas;
240
241 /* Covers 1 and 2.d */
242 if ((ARM_LPAE_GRANULE(data) == SZ_16K) && (data->start_level == 0))
243 return (oas == 48) || (ias == 48);
244
245 /* Covers 2.a and 2.c */
246 if ((ARM_LPAE_GRANULE(data) == SZ_4K) && (data->start_level == 0))
247 return (oas == 40) || (oas == 42);
248
249 /* Case 2.b */
250 return (ARM_LPAE_GRANULE(data) == SZ_16K) &&
251 (data->start_level == 1) && (oas == 40);
252 }
253
254 static bool selftest_running = false;
255
__arm_lpae_dma_addr(void * pages)256 static dma_addr_t __arm_lpae_dma_addr(void *pages)
257 {
258 return (dma_addr_t)virt_to_phys(pages);
259 }
260
__arm_lpae_alloc_pages(size_t size,gfp_t gfp,struct io_pgtable_cfg * cfg,void * cookie)261 static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
262 struct io_pgtable_cfg *cfg,
263 void *cookie)
264 {
265 struct device *dev = cfg->iommu_dev;
266 int order = get_order(size);
267 dma_addr_t dma;
268 void *pages;
269
270 VM_BUG_ON((gfp & __GFP_HIGHMEM));
271
272 if (cfg->alloc)
273 pages = cfg->alloc(cookie, size, gfp);
274 else
275 pages = iommu_alloc_pages_node(dev_to_node(dev), gfp, order);
276
277 if (!pages)
278 return NULL;
279
280 if (!cfg->coherent_walk) {
281 dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE);
282 if (dma_mapping_error(dev, dma))
283 goto out_free;
284 /*
285 * We depend on the IOMMU being able to work with any physical
286 * address directly, so if the DMA layer suggests otherwise by
287 * translating or truncating them, that bodes very badly...
288 */
289 if (dma != virt_to_phys(pages))
290 goto out_unmap;
291 }
292
293 return pages;
294
295 out_unmap:
296 dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n");
297 dma_unmap_single(dev, dma, size, DMA_TO_DEVICE);
298
299 out_free:
300 if (cfg->free)
301 cfg->free(cookie, pages, size);
302 else
303 iommu_free_pages(pages, order);
304
305 return NULL;
306 }
307
__arm_lpae_free_pages(void * pages,size_t size,struct io_pgtable_cfg * cfg,void * cookie)308 static void __arm_lpae_free_pages(void *pages, size_t size,
309 struct io_pgtable_cfg *cfg,
310 void *cookie)
311 {
312 if (!cfg->coherent_walk)
313 dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages),
314 size, DMA_TO_DEVICE);
315
316 if (cfg->free)
317 cfg->free(cookie, pages, size);
318 else
319 iommu_free_pages(pages, get_order(size));
320 }
321
__arm_lpae_sync_pte(arm_lpae_iopte * ptep,int num_entries,struct io_pgtable_cfg * cfg)322 static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries,
323 struct io_pgtable_cfg *cfg)
324 {
325 dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep),
326 sizeof(*ptep) * num_entries, DMA_TO_DEVICE);
327 }
328
__arm_lpae_clear_pte(arm_lpae_iopte * ptep,struct io_pgtable_cfg * cfg,int num_entries)329 static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg, int num_entries)
330 {
331 for (int i = 0; i < num_entries; i++)
332 ptep[i] = 0;
333
334 if (!cfg->coherent_walk && num_entries)
335 __arm_lpae_sync_pte(ptep, num_entries, cfg);
336 }
337
338 static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
339 struct iommu_iotlb_gather *gather,
340 unsigned long iova, size_t size, size_t pgcount,
341 int lvl, arm_lpae_iopte *ptep);
342
__arm_lpae_init_pte(struct arm_lpae_io_pgtable * data,phys_addr_t paddr,arm_lpae_iopte prot,int lvl,int num_entries,arm_lpae_iopte * ptep)343 static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
344 phys_addr_t paddr, arm_lpae_iopte prot,
345 int lvl, int num_entries, arm_lpae_iopte *ptep)
346 {
347 arm_lpae_iopte pte = prot;
348 struct io_pgtable_cfg *cfg = &data->iop.cfg;
349 size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data);
350 int i;
351
352 if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1)
353 pte |= ARM_LPAE_PTE_TYPE_PAGE;
354 else
355 pte |= ARM_LPAE_PTE_TYPE_BLOCK;
356
357 for (i = 0; i < num_entries; i++)
358 ptep[i] = pte | paddr_to_iopte(paddr + i * sz, data);
359
360 if (!cfg->coherent_walk)
361 __arm_lpae_sync_pte(ptep, num_entries, cfg);
362 }
363
arm_lpae_init_pte(struct arm_lpae_io_pgtable * data,unsigned long iova,phys_addr_t paddr,arm_lpae_iopte prot,int lvl,int num_entries,arm_lpae_iopte * ptep)364 static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
365 unsigned long iova, phys_addr_t paddr,
366 arm_lpae_iopte prot, int lvl, int num_entries,
367 arm_lpae_iopte *ptep)
368 {
369 int i;
370
371 for (i = 0; i < num_entries; i++)
372 if (iopte_leaf(ptep[i], lvl, data->iop.fmt)) {
373 /* We require an unmap first */
374 WARN_ON(!selftest_running);
375 return -EEXIST;
376 } else if (iopte_type(ptep[i]) == ARM_LPAE_PTE_TYPE_TABLE) {
377 /*
378 * We need to unmap and free the old table before
379 * overwriting it with a block entry.
380 */
381 arm_lpae_iopte *tblp;
382 size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data);
383
384 tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data);
385 if (__arm_lpae_unmap(data, NULL, iova + i * sz, sz, 1,
386 lvl, tblp) != sz) {
387 WARN_ON(1);
388 return -EINVAL;
389 }
390 }
391
392 __arm_lpae_init_pte(data, paddr, prot, lvl, num_entries, ptep);
393 return 0;
394 }
395
arm_lpae_install_table(arm_lpae_iopte * table,arm_lpae_iopte * ptep,arm_lpae_iopte curr,struct arm_lpae_io_pgtable * data)396 static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table,
397 arm_lpae_iopte *ptep,
398 arm_lpae_iopte curr,
399 struct arm_lpae_io_pgtable *data)
400 {
401 arm_lpae_iopte old, new;
402 struct io_pgtable_cfg *cfg = &data->iop.cfg;
403
404 new = paddr_to_iopte(__pa(table), data) | ARM_LPAE_PTE_TYPE_TABLE;
405 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)
406 new |= ARM_LPAE_PTE_NSTABLE;
407
408 /*
409 * Ensure the table itself is visible before its PTE can be.
410 * Whilst we could get away with cmpxchg64_release below, this
411 * doesn't have any ordering semantics when !CONFIG_SMP.
412 */
413 dma_wmb();
414
415 old = cmpxchg64_relaxed(ptep, curr, new);
416
417 if (cfg->coherent_walk || (old & ARM_LPAE_PTE_SW_SYNC))
418 return old;
419
420 /* Even if it's not ours, there's no point waiting; just kick it */
421 __arm_lpae_sync_pte(ptep, 1, cfg);
422 if (old == curr)
423 WRITE_ONCE(*ptep, new | ARM_LPAE_PTE_SW_SYNC);
424
425 return old;
426 }
427
__arm_lpae_map(struct arm_lpae_io_pgtable * data,unsigned long iova,phys_addr_t paddr,size_t size,size_t pgcount,arm_lpae_iopte prot,int lvl,arm_lpae_iopte * ptep,gfp_t gfp,size_t * mapped)428 static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
429 phys_addr_t paddr, size_t size, size_t pgcount,
430 arm_lpae_iopte prot, int lvl, arm_lpae_iopte *ptep,
431 gfp_t gfp, size_t *mapped)
432 {
433 arm_lpae_iopte *cptep, pte;
434 size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data);
435 size_t tblsz = ARM_LPAE_GRANULE(data);
436 struct io_pgtable_cfg *cfg = &data->iop.cfg;
437 int ret = 0, num_entries, max_entries, map_idx_start;
438
439 /* Find our entry at the current level */
440 map_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);
441 ptep += map_idx_start;
442
443 /* If we can install a leaf entry at this level, then do so */
444 if (size == block_size) {
445 max_entries = arm_lpae_max_entries(map_idx_start, data);
446 num_entries = min_t(int, pgcount, max_entries);
447 ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep);
448 if (!ret)
449 *mapped += num_entries * size;
450
451 return ret;
452 }
453
454 /* We can't allocate tables at the final level */
455 if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1))
456 return -EINVAL;
457
458 /* Grab a pointer to the next level */
459 pte = READ_ONCE(*ptep);
460 if (!pte) {
461 cptep = __arm_lpae_alloc_pages(tblsz, gfp, cfg, data->iop.cookie);
462 if (!cptep)
463 return -ENOMEM;
464
465 pte = arm_lpae_install_table(cptep, ptep, 0, data);
466 if (pte)
467 __arm_lpae_free_pages(cptep, tblsz, cfg, data->iop.cookie);
468 } else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) {
469 __arm_lpae_sync_pte(ptep, 1, cfg);
470 }
471
472 if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) {
473 cptep = iopte_deref(pte, data);
474 } else if (pte) {
475 /* We require an unmap first */
476 WARN_ON(!selftest_running);
477 return -EEXIST;
478 }
479
480 /* Rinse, repeat */
481 return __arm_lpae_map(data, iova, paddr, size, pgcount, prot, lvl + 1,
482 cptep, gfp, mapped);
483 }
484
arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable * data,int prot)485 static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data,
486 int prot)
487 {
488 arm_lpae_iopte pte;
489
490 if (data->iop.fmt == ARM_64_LPAE_S1 ||
491 data->iop.fmt == ARM_32_LPAE_S1) {
492 pte = ARM_LPAE_PTE_nG;
493 if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ))
494 pte |= ARM_LPAE_PTE_AP_RDONLY;
495 else if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_HD)
496 pte |= ARM_LPAE_PTE_DBM;
497 if (!(prot & IOMMU_PRIV))
498 pte |= ARM_LPAE_PTE_AP_UNPRIV;
499 } else {
500 pte = ARM_LPAE_PTE_HAP_FAULT;
501 if (prot & IOMMU_READ)
502 pte |= ARM_LPAE_PTE_HAP_READ;
503 if (prot & IOMMU_WRITE)
504 pte |= ARM_LPAE_PTE_HAP_WRITE;
505 }
506
507 /*
508 * Note that this logic is structured to accommodate Mali LPAE
509 * having stage-1-like attributes but stage-2-like permissions.
510 */
511 if (data->iop.fmt == ARM_64_LPAE_S2 ||
512 data->iop.fmt == ARM_32_LPAE_S2) {
513 if (prot & IOMMU_MMIO) {
514 pte |= ARM_LPAE_PTE_MEMATTR_DEV;
515 } else if (prot & IOMMU_CACHE) {
516 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_S2FWB)
517 pte |= ARM_LPAE_PTE_MEMATTR_FWB_WB;
518 else
519 pte |= ARM_LPAE_PTE_MEMATTR_OIWB;
520 } else {
521 pte |= ARM_LPAE_PTE_MEMATTR_NC;
522 }
523 } else {
524 if (prot & IOMMU_MMIO)
525 pte |= (ARM_LPAE_MAIR_ATTR_IDX_DEV
526 << ARM_LPAE_PTE_ATTRINDX_SHIFT);
527 else if (prot & IOMMU_CACHE)
528 pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE
529 << ARM_LPAE_PTE_ATTRINDX_SHIFT);
530 }
531
532 /*
533 * Also Mali has its own notions of shareability wherein its Inner
534 * domain covers the cores within the GPU, and its Outer domain is
535 * "outside the GPU" (i.e. either the Inner or System domain in CPU
536 * terms, depending on coherency).
537 */
538 if (prot & IOMMU_CACHE && data->iop.fmt != ARM_MALI_LPAE)
539 pte |= ARM_LPAE_PTE_SH_IS;
540 else
541 pte |= ARM_LPAE_PTE_SH_OS;
542
543 if (prot & IOMMU_NOEXEC)
544 pte |= ARM_LPAE_PTE_XN;
545
546 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)
547 pte |= ARM_LPAE_PTE_NS;
548
549 if (data->iop.fmt != ARM_MALI_LPAE)
550 pte |= ARM_LPAE_PTE_AF;
551
552 return pte;
553 }
554
arm_lpae_map_pages(struct io_pgtable_ops * ops,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int iommu_prot,gfp_t gfp,size_t * mapped)555 static int arm_lpae_map_pages(struct io_pgtable_ops *ops, unsigned long iova,
556 phys_addr_t paddr, size_t pgsize, size_t pgcount,
557 int iommu_prot, gfp_t gfp, size_t *mapped)
558 {
559 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
560 struct io_pgtable_cfg *cfg = &data->iop.cfg;
561 arm_lpae_iopte *ptep = data->pgd;
562 int ret, lvl = data->start_level;
563 arm_lpae_iopte prot;
564 long iaext = (s64)iova >> cfg->ias;
565
566 if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize))
567 return -EINVAL;
568
569 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
570 iaext = ~iaext;
571 if (WARN_ON(iaext || paddr >> cfg->oas))
572 return -ERANGE;
573
574 if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE)))
575 return -EINVAL;
576
577 prot = arm_lpae_prot_to_pte(data, iommu_prot);
578 ret = __arm_lpae_map(data, iova, paddr, pgsize, pgcount, prot, lvl,
579 ptep, gfp, mapped);
580 /*
581 * Synchronise all PTE updates for the new mapping before there's
582 * a chance for anything to kick off a table walk for the new iova.
583 */
584 wmb();
585
586 return ret;
587 }
588
__arm_lpae_free_pgtable(struct arm_lpae_io_pgtable * data,int lvl,arm_lpae_iopte * ptep)589 static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
590 arm_lpae_iopte *ptep)
591 {
592 arm_lpae_iopte *start, *end;
593 unsigned long table_size;
594
595 if (lvl == data->start_level)
596 table_size = ARM_LPAE_PGD_SIZE(data);
597 else
598 table_size = ARM_LPAE_GRANULE(data);
599
600 start = ptep;
601
602 /* Only leaf entries at the last level */
603 if (lvl == ARM_LPAE_MAX_LEVELS - 1)
604 end = ptep;
605 else
606 end = (void *)ptep + table_size;
607
608 while (ptep != end) {
609 arm_lpae_iopte pte = *ptep++;
610
611 if (!pte || iopte_leaf(pte, lvl, data->iop.fmt))
612 continue;
613
614 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
615 }
616
617 __arm_lpae_free_pages(start, table_size, &data->iop.cfg, data->iop.cookie);
618 }
619
arm_lpae_free_pgtable(struct io_pgtable * iop)620 static void arm_lpae_free_pgtable(struct io_pgtable *iop)
621 {
622 struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop);
623
624 __arm_lpae_free_pgtable(data, data->start_level, data->pgd);
625 kfree(data);
626 }
627
__arm_lpae_unmap(struct arm_lpae_io_pgtable * data,struct iommu_iotlb_gather * gather,unsigned long iova,size_t size,size_t pgcount,int lvl,arm_lpae_iopte * ptep)628 static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
629 struct iommu_iotlb_gather *gather,
630 unsigned long iova, size_t size, size_t pgcount,
631 int lvl, arm_lpae_iopte *ptep)
632 {
633 arm_lpae_iopte pte;
634 struct io_pgtable *iop = &data->iop;
635 int i = 0, num_entries, max_entries, unmap_idx_start;
636
637 /* Something went horribly wrong and we ran out of page table */
638 if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS))
639 return 0;
640
641 unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);
642 ptep += unmap_idx_start;
643 pte = READ_ONCE(*ptep);
644 if (WARN_ON(!pte))
645 return 0;
646
647 /* If the size matches this level, we're in the right place */
648 if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) {
649 max_entries = arm_lpae_max_entries(unmap_idx_start, data);
650 num_entries = min_t(int, pgcount, max_entries);
651
652 /* Find and handle non-leaf entries */
653 for (i = 0; i < num_entries; i++) {
654 pte = READ_ONCE(ptep[i]);
655 if (WARN_ON(!pte))
656 break;
657
658 if (!iopte_leaf(pte, lvl, iop->fmt)) {
659 __arm_lpae_clear_pte(&ptep[i], &iop->cfg, 1);
660
661 /* Also flush any partial walks */
662 io_pgtable_tlb_flush_walk(iop, iova + i * size, size,
663 ARM_LPAE_GRANULE(data));
664 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
665 }
666 }
667
668 /* Clear the remaining entries */
669 __arm_lpae_clear_pte(ptep, &iop->cfg, i);
670
671 if (gather && !iommu_iotlb_gather_queued(gather))
672 for (int j = 0; j < i; j++)
673 io_pgtable_tlb_add_page(iop, gather, iova + j * size, size);
674
675 return i * size;
676 } else if (iopte_leaf(pte, lvl, iop->fmt)) {
677 WARN_ONCE(true, "Unmap of a partial large IOPTE is not allowed");
678 return 0;
679 }
680
681 /* Keep on walkin' */
682 ptep = iopte_deref(pte, data);
683 return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl + 1, ptep);
684 }
685
arm_lpae_unmap_pages(struct io_pgtable_ops * ops,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * gather)686 static size_t arm_lpae_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova,
687 size_t pgsize, size_t pgcount,
688 struct iommu_iotlb_gather *gather)
689 {
690 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
691 struct io_pgtable_cfg *cfg = &data->iop.cfg;
692 arm_lpae_iopte *ptep = data->pgd;
693 long iaext = (s64)iova >> cfg->ias;
694
695 if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount))
696 return 0;
697
698 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
699 iaext = ~iaext;
700 if (WARN_ON(iaext))
701 return 0;
702
703 return __arm_lpae_unmap(data, gather, iova, pgsize, pgcount,
704 data->start_level, ptep);
705 }
706
707 struct io_pgtable_walk_data {
708 struct io_pgtable *iop;
709 void *data;
710 int (*visit)(struct io_pgtable_walk_data *walk_data, int lvl,
711 arm_lpae_iopte *ptep, size_t size);
712 unsigned long flags;
713 u64 addr;
714 const u64 end;
715 };
716
717 static int __arm_lpae_iopte_walk(struct arm_lpae_io_pgtable *data,
718 struct io_pgtable_walk_data *walk_data,
719 arm_lpae_iopte *ptep,
720 int lvl);
721
722 struct iova_to_phys_data {
723 arm_lpae_iopte pte;
724 int lvl;
725 };
726
visit_iova_to_phys(struct io_pgtable_walk_data * walk_data,int lvl,arm_lpae_iopte * ptep,size_t size)727 static int visit_iova_to_phys(struct io_pgtable_walk_data *walk_data, int lvl,
728 arm_lpae_iopte *ptep, size_t size)
729 {
730 struct iova_to_phys_data *data = walk_data->data;
731 data->pte = *ptep;
732 data->lvl = lvl;
733 return 0;
734 }
735
arm_lpae_iova_to_phys(struct io_pgtable_ops * ops,unsigned long iova)736 static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops,
737 unsigned long iova)
738 {
739 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
740 struct iova_to_phys_data d;
741 struct io_pgtable_walk_data walk_data = {
742 .data = &d,
743 .visit = visit_iova_to_phys,
744 .addr = iova,
745 .end = iova + 1,
746 };
747 int ret;
748
749 ret = __arm_lpae_iopte_walk(data, &walk_data, data->pgd, data->start_level);
750 if (ret)
751 return 0;
752
753 iova &= (ARM_LPAE_BLOCK_SIZE(d.lvl, data) - 1);
754 return iopte_to_paddr(d.pte, data) | iova;
755 }
756
visit_pgtable_walk(struct io_pgtable_walk_data * walk_data,int lvl,arm_lpae_iopte * ptep,size_t size)757 static int visit_pgtable_walk(struct io_pgtable_walk_data *walk_data, int lvl,
758 arm_lpae_iopte *ptep, size_t size)
759 {
760 struct arm_lpae_io_pgtable_walk_data *data = walk_data->data;
761 data->ptes[lvl] = *ptep;
762 return 0;
763 }
764
arm_lpae_pgtable_walk(struct io_pgtable_ops * ops,unsigned long iova,void * wd)765 static int arm_lpae_pgtable_walk(struct io_pgtable_ops *ops, unsigned long iova,
766 void *wd)
767 {
768 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
769 struct io_pgtable_walk_data walk_data = {
770 .data = wd,
771 .visit = visit_pgtable_walk,
772 .addr = iova,
773 .end = iova + 1,
774 };
775
776 return __arm_lpae_iopte_walk(data, &walk_data, data->pgd, data->start_level);
777 }
778
io_pgtable_visit(struct arm_lpae_io_pgtable * data,struct io_pgtable_walk_data * walk_data,arm_lpae_iopte * ptep,int lvl)779 static int io_pgtable_visit(struct arm_lpae_io_pgtable *data,
780 struct io_pgtable_walk_data *walk_data,
781 arm_lpae_iopte *ptep, int lvl)
782 {
783 struct io_pgtable *iop = &data->iop;
784 arm_lpae_iopte pte = READ_ONCE(*ptep);
785
786 size_t size = ARM_LPAE_BLOCK_SIZE(lvl, data);
787 int ret = walk_data->visit(walk_data, lvl, ptep, size);
788 if (ret)
789 return ret;
790
791 if (iopte_leaf(pte, lvl, iop->fmt)) {
792 walk_data->addr += size;
793 return 0;
794 }
795
796 if (!iopte_table(pte, lvl)) {
797 return -EINVAL;
798 }
799
800 ptep = iopte_deref(pte, data);
801 return __arm_lpae_iopte_walk(data, walk_data, ptep, lvl + 1);
802 }
803
__arm_lpae_iopte_walk(struct arm_lpae_io_pgtable * data,struct io_pgtable_walk_data * walk_data,arm_lpae_iopte * ptep,int lvl)804 static int __arm_lpae_iopte_walk(struct arm_lpae_io_pgtable *data,
805 struct io_pgtable_walk_data *walk_data,
806 arm_lpae_iopte *ptep,
807 int lvl)
808 {
809 u32 idx;
810 int max_entries, ret;
811
812 if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS))
813 return -EINVAL;
814
815 if (lvl == data->start_level)
816 max_entries = ARM_LPAE_PGD_SIZE(data) / sizeof(arm_lpae_iopte);
817 else
818 max_entries = ARM_LPAE_PTES_PER_TABLE(data);
819
820 for (idx = ARM_LPAE_LVL_IDX(walk_data->addr, lvl, data);
821 (idx < max_entries) && (walk_data->addr < walk_data->end); ++idx) {
822 ret = io_pgtable_visit(data, walk_data, ptep + idx, lvl);
823 if (ret)
824 return ret;
825 }
826
827 return 0;
828 }
829
visit_dirty(struct io_pgtable_walk_data * walk_data,int lvl,arm_lpae_iopte * ptep,size_t size)830 static int visit_dirty(struct io_pgtable_walk_data *walk_data, int lvl,
831 arm_lpae_iopte *ptep, size_t size)
832 {
833 struct iommu_dirty_bitmap *dirty = walk_data->data;
834
835 if (!iopte_leaf(*ptep, lvl, walk_data->iop->fmt))
836 return 0;
837
838 if (iopte_writeable_dirty(*ptep)) {
839 iommu_dirty_bitmap_record(dirty, walk_data->addr, size);
840 if (!(walk_data->flags & IOMMU_DIRTY_NO_CLEAR))
841 iopte_set_writeable_clean(ptep);
842 }
843
844 return 0;
845 }
846
arm_lpae_read_and_clear_dirty(struct io_pgtable_ops * ops,unsigned long iova,size_t size,unsigned long flags,struct iommu_dirty_bitmap * dirty)847 static int arm_lpae_read_and_clear_dirty(struct io_pgtable_ops *ops,
848 unsigned long iova, size_t size,
849 unsigned long flags,
850 struct iommu_dirty_bitmap *dirty)
851 {
852 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
853 struct io_pgtable_cfg *cfg = &data->iop.cfg;
854 struct io_pgtable_walk_data walk_data = {
855 .iop = &data->iop,
856 .data = dirty,
857 .visit = visit_dirty,
858 .flags = flags,
859 .addr = iova,
860 .end = iova + size,
861 };
862 arm_lpae_iopte *ptep = data->pgd;
863 int lvl = data->start_level;
864
865 if (WARN_ON(!size))
866 return -EINVAL;
867 if (WARN_ON((iova + size - 1) & ~(BIT(cfg->ias) - 1)))
868 return -EINVAL;
869 if (data->iop.fmt != ARM_64_LPAE_S1)
870 return -EINVAL;
871
872 return __arm_lpae_iopte_walk(data, &walk_data, ptep, lvl);
873 }
874
arm_lpae_restrict_pgsizes(struct io_pgtable_cfg * cfg)875 static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg)
876 {
877 unsigned long granule, page_sizes;
878 unsigned int max_addr_bits = 48;
879
880 /*
881 * We need to restrict the supported page sizes to match the
882 * translation regime for a particular granule. Aim to match
883 * the CPU page size if possible, otherwise prefer smaller sizes.
884 * While we're at it, restrict the block sizes to match the
885 * chosen granule.
886 */
887 if (cfg->pgsize_bitmap & PAGE_SIZE)
888 granule = PAGE_SIZE;
889 else if (cfg->pgsize_bitmap & ~PAGE_MASK)
890 granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK);
891 else if (cfg->pgsize_bitmap & PAGE_MASK)
892 granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK);
893 else
894 granule = 0;
895
896 switch (granule) {
897 case SZ_4K:
898 page_sizes = (SZ_4K | SZ_2M | SZ_1G);
899 break;
900 case SZ_16K:
901 page_sizes = (SZ_16K | SZ_32M);
902 break;
903 case SZ_64K:
904 max_addr_bits = 52;
905 page_sizes = (SZ_64K | SZ_512M);
906 if (cfg->oas > 48)
907 page_sizes |= 1ULL << 42; /* 4TB */
908 break;
909 default:
910 page_sizes = 0;
911 }
912
913 cfg->pgsize_bitmap &= page_sizes;
914 cfg->ias = min(cfg->ias, max_addr_bits);
915 cfg->oas = min(cfg->oas, max_addr_bits);
916 }
917
918 static struct arm_lpae_io_pgtable *
arm_lpae_alloc_pgtable(struct io_pgtable_cfg * cfg)919 arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg)
920 {
921 struct arm_lpae_io_pgtable *data;
922 int levels, va_bits, pg_shift;
923
924 arm_lpae_restrict_pgsizes(cfg);
925
926 if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K)))
927 return NULL;
928
929 if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS)
930 return NULL;
931
932 if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS)
933 return NULL;
934
935 data = kmalloc(sizeof(*data), GFP_KERNEL);
936 if (!data)
937 return NULL;
938
939 pg_shift = __ffs(cfg->pgsize_bitmap);
940 data->bits_per_level = pg_shift - ilog2(sizeof(arm_lpae_iopte));
941
942 va_bits = cfg->ias - pg_shift;
943 levels = DIV_ROUND_UP(va_bits, data->bits_per_level);
944 data->start_level = ARM_LPAE_MAX_LEVELS - levels;
945
946 /* Calculate the actual size of our pgd (without concatenation) */
947 data->pgd_bits = va_bits - (data->bits_per_level * (levels - 1));
948
949 data->iop.ops = (struct io_pgtable_ops) {
950 .map_pages = arm_lpae_map_pages,
951 .unmap_pages = arm_lpae_unmap_pages,
952 .iova_to_phys = arm_lpae_iova_to_phys,
953 .read_and_clear_dirty = arm_lpae_read_and_clear_dirty,
954 .pgtable_walk = arm_lpae_pgtable_walk,
955 };
956
957 return data;
958 }
959
960 static struct io_pgtable *
arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg * cfg,void * cookie)961 arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
962 {
963 u64 reg;
964 struct arm_lpae_io_pgtable *data;
965 typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr;
966 bool tg1;
967
968 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS |
969 IO_PGTABLE_QUIRK_ARM_TTBR1 |
970 IO_PGTABLE_QUIRK_ARM_OUTER_WBWA |
971 IO_PGTABLE_QUIRK_ARM_HD))
972 return NULL;
973
974 data = arm_lpae_alloc_pgtable(cfg);
975 if (!data)
976 return NULL;
977
978 /* TCR */
979 if (cfg->coherent_walk) {
980 tcr->sh = ARM_LPAE_TCR_SH_IS;
981 tcr->irgn = ARM_LPAE_TCR_RGN_WBWA;
982 tcr->orgn = ARM_LPAE_TCR_RGN_WBWA;
983 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA)
984 goto out_free_data;
985 } else {
986 tcr->sh = ARM_LPAE_TCR_SH_OS;
987 tcr->irgn = ARM_LPAE_TCR_RGN_NC;
988 if (!(cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA))
989 tcr->orgn = ARM_LPAE_TCR_RGN_NC;
990 else
991 tcr->orgn = ARM_LPAE_TCR_RGN_WBWA;
992 }
993
994 tg1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1;
995 switch (ARM_LPAE_GRANULE(data)) {
996 case SZ_4K:
997 tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_4K : ARM_LPAE_TCR_TG0_4K;
998 break;
999 case SZ_16K:
1000 tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_16K : ARM_LPAE_TCR_TG0_16K;
1001 break;
1002 case SZ_64K:
1003 tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_64K : ARM_LPAE_TCR_TG0_64K;
1004 break;
1005 }
1006
1007 switch (cfg->oas) {
1008 case 32:
1009 tcr->ips = ARM_LPAE_TCR_PS_32_BIT;
1010 break;
1011 case 36:
1012 tcr->ips = ARM_LPAE_TCR_PS_36_BIT;
1013 break;
1014 case 40:
1015 tcr->ips = ARM_LPAE_TCR_PS_40_BIT;
1016 break;
1017 case 42:
1018 tcr->ips = ARM_LPAE_TCR_PS_42_BIT;
1019 break;
1020 case 44:
1021 tcr->ips = ARM_LPAE_TCR_PS_44_BIT;
1022 break;
1023 case 48:
1024 tcr->ips = ARM_LPAE_TCR_PS_48_BIT;
1025 break;
1026 case 52:
1027 tcr->ips = ARM_LPAE_TCR_PS_52_BIT;
1028 break;
1029 default:
1030 goto out_free_data;
1031 }
1032
1033 tcr->tsz = 64ULL - cfg->ias;
1034
1035 /* MAIRs */
1036 reg = (ARM_LPAE_MAIR_ATTR_NC
1037 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) |
1038 (ARM_LPAE_MAIR_ATTR_WBRWA
1039 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) |
1040 (ARM_LPAE_MAIR_ATTR_DEVICE
1041 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)) |
1042 (ARM_LPAE_MAIR_ATTR_INC_OWBRWA
1043 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE));
1044
1045 cfg->arm_lpae_s1_cfg.mair = reg;
1046
1047 /* Looking good; allocate a pgd */
1048 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data),
1049 GFP_KERNEL, cfg, cookie);
1050 if (!data->pgd)
1051 goto out_free_data;
1052
1053 /* Ensure the empty pgd is visible before any actual TTBR write */
1054 wmb();
1055
1056 /* TTBR */
1057 cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd);
1058 return &data->iop;
1059
1060 out_free_data:
1061 kfree(data);
1062 return NULL;
1063 }
1064
1065 static struct io_pgtable *
arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg * cfg,void * cookie)1066 arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
1067 {
1068 u64 sl;
1069 struct arm_lpae_io_pgtable *data;
1070 typeof(&cfg->arm_lpae_s2_cfg.vtcr) vtcr = &cfg->arm_lpae_s2_cfg.vtcr;
1071
1072 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_S2FWB))
1073 return NULL;
1074
1075 data = arm_lpae_alloc_pgtable(cfg);
1076 if (!data)
1077 return NULL;
1078
1079 if (arm_lpae_concat_mandatory(cfg, data)) {
1080 if (WARN_ON((ARM_LPAE_PGD_SIZE(data) / sizeof(arm_lpae_iopte)) >
1081 ARM_LPAE_S2_MAX_CONCAT_PAGES))
1082 return NULL;
1083 data->pgd_bits += data->bits_per_level;
1084 data->start_level++;
1085 }
1086
1087 /* VTCR */
1088 if (cfg->coherent_walk) {
1089 vtcr->sh = ARM_LPAE_TCR_SH_IS;
1090 vtcr->irgn = ARM_LPAE_TCR_RGN_WBWA;
1091 vtcr->orgn = ARM_LPAE_TCR_RGN_WBWA;
1092 } else {
1093 vtcr->sh = ARM_LPAE_TCR_SH_OS;
1094 vtcr->irgn = ARM_LPAE_TCR_RGN_NC;
1095 vtcr->orgn = ARM_LPAE_TCR_RGN_NC;
1096 }
1097
1098 sl = data->start_level;
1099
1100 switch (ARM_LPAE_GRANULE(data)) {
1101 case SZ_4K:
1102 vtcr->tg = ARM_LPAE_TCR_TG0_4K;
1103 sl++; /* SL0 format is different for 4K granule size */
1104 break;
1105 case SZ_16K:
1106 vtcr->tg = ARM_LPAE_TCR_TG0_16K;
1107 break;
1108 case SZ_64K:
1109 vtcr->tg = ARM_LPAE_TCR_TG0_64K;
1110 break;
1111 }
1112
1113 switch (cfg->oas) {
1114 case 32:
1115 vtcr->ps = ARM_LPAE_TCR_PS_32_BIT;
1116 break;
1117 case 36:
1118 vtcr->ps = ARM_LPAE_TCR_PS_36_BIT;
1119 break;
1120 case 40:
1121 vtcr->ps = ARM_LPAE_TCR_PS_40_BIT;
1122 break;
1123 case 42:
1124 vtcr->ps = ARM_LPAE_TCR_PS_42_BIT;
1125 break;
1126 case 44:
1127 vtcr->ps = ARM_LPAE_TCR_PS_44_BIT;
1128 break;
1129 case 48:
1130 vtcr->ps = ARM_LPAE_TCR_PS_48_BIT;
1131 break;
1132 case 52:
1133 vtcr->ps = ARM_LPAE_TCR_PS_52_BIT;
1134 break;
1135 default:
1136 goto out_free_data;
1137 }
1138
1139 vtcr->tsz = 64ULL - cfg->ias;
1140 vtcr->sl = ~sl & ARM_LPAE_VTCR_SL0_MASK;
1141
1142 /* Allocate pgd pages */
1143 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data),
1144 GFP_KERNEL, cfg, cookie);
1145 if (!data->pgd)
1146 goto out_free_data;
1147
1148 /* Ensure the empty pgd is visible before any actual TTBR write */
1149 wmb();
1150
1151 /* VTTBR */
1152 cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd);
1153 return &data->iop;
1154
1155 out_free_data:
1156 kfree(data);
1157 return NULL;
1158 }
1159
1160 static struct io_pgtable *
arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg * cfg,void * cookie)1161 arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
1162 {
1163 if (cfg->ias > 32 || cfg->oas > 40)
1164 return NULL;
1165
1166 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
1167 return arm_64_lpae_alloc_pgtable_s1(cfg, cookie);
1168 }
1169
1170 static struct io_pgtable *
arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg * cfg,void * cookie)1171 arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
1172 {
1173 if (cfg->ias > 40 || cfg->oas > 40)
1174 return NULL;
1175
1176 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
1177 return arm_64_lpae_alloc_pgtable_s2(cfg, cookie);
1178 }
1179
1180 static struct io_pgtable *
arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg * cfg,void * cookie)1181 arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)
1182 {
1183 struct arm_lpae_io_pgtable *data;
1184
1185 /* No quirks for Mali (hopefully) */
1186 if (cfg->quirks)
1187 return NULL;
1188
1189 if (cfg->ias > 48 || cfg->oas > 40)
1190 return NULL;
1191
1192 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
1193
1194 data = arm_lpae_alloc_pgtable(cfg);
1195 if (!data)
1196 return NULL;
1197
1198 /* Mali seems to need a full 4-level table regardless of IAS */
1199 if (data->start_level > 0) {
1200 data->start_level = 0;
1201 data->pgd_bits = 0;
1202 }
1203 /*
1204 * MEMATTR: Mali has no actual notion of a non-cacheable type, so the
1205 * best we can do is mimic the out-of-tree driver and hope that the
1206 * "implementation-defined caching policy" is good enough. Similarly,
1207 * we'll use it for the sake of a valid attribute for our 'device'
1208 * index, although callers should never request that in practice.
1209 */
1210 cfg->arm_mali_lpae_cfg.memattr =
1211 (ARM_MALI_LPAE_MEMATTR_IMP_DEF
1212 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) |
1213 (ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC
1214 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) |
1215 (ARM_MALI_LPAE_MEMATTR_IMP_DEF
1216 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV));
1217
1218 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), GFP_KERNEL,
1219 cfg, cookie);
1220 if (!data->pgd)
1221 goto out_free_data;
1222
1223 /* Ensure the empty pgd is visible before TRANSTAB can be written */
1224 wmb();
1225
1226 cfg->arm_mali_lpae_cfg.transtab = virt_to_phys(data->pgd) |
1227 ARM_MALI_LPAE_TTBR_READ_INNER |
1228 ARM_MALI_LPAE_TTBR_ADRMODE_TABLE;
1229 if (cfg->coherent_walk)
1230 cfg->arm_mali_lpae_cfg.transtab |= ARM_MALI_LPAE_TTBR_SHARE_OUTER;
1231
1232 return &data->iop;
1233
1234 out_free_data:
1235 kfree(data);
1236 return NULL;
1237 }
1238
1239 struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns = {
1240 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
1241 .alloc = arm_64_lpae_alloc_pgtable_s1,
1242 .free = arm_lpae_free_pgtable,
1243 };
1244
1245 struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns = {
1246 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
1247 .alloc = arm_64_lpae_alloc_pgtable_s2,
1248 .free = arm_lpae_free_pgtable,
1249 };
1250
1251 struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns = {
1252 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
1253 .alloc = arm_32_lpae_alloc_pgtable_s1,
1254 .free = arm_lpae_free_pgtable,
1255 };
1256
1257 struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns = {
1258 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
1259 .alloc = arm_32_lpae_alloc_pgtable_s2,
1260 .free = arm_lpae_free_pgtable,
1261 };
1262
1263 struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns = {
1264 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
1265 .alloc = arm_mali_lpae_alloc_pgtable,
1266 .free = arm_lpae_free_pgtable,
1267 };
1268
1269 #ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE_SELFTEST
1270
1271 static struct io_pgtable_cfg *cfg_cookie __initdata;
1272
dummy_tlb_flush_all(void * cookie)1273 static void __init dummy_tlb_flush_all(void *cookie)
1274 {
1275 WARN_ON(cookie != cfg_cookie);
1276 }
1277
dummy_tlb_flush(unsigned long iova,size_t size,size_t granule,void * cookie)1278 static void __init dummy_tlb_flush(unsigned long iova, size_t size,
1279 size_t granule, void *cookie)
1280 {
1281 WARN_ON(cookie != cfg_cookie);
1282 WARN_ON(!(size & cfg_cookie->pgsize_bitmap));
1283 }
1284
dummy_tlb_add_page(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)1285 static void __init dummy_tlb_add_page(struct iommu_iotlb_gather *gather,
1286 unsigned long iova, size_t granule,
1287 void *cookie)
1288 {
1289 dummy_tlb_flush(iova, granule, granule, cookie);
1290 }
1291
1292 static const struct iommu_flush_ops dummy_tlb_ops __initconst = {
1293 .tlb_flush_all = dummy_tlb_flush_all,
1294 .tlb_flush_walk = dummy_tlb_flush,
1295 .tlb_add_page = dummy_tlb_add_page,
1296 };
1297
arm_lpae_dump_ops(struct io_pgtable_ops * ops)1298 static void __init arm_lpae_dump_ops(struct io_pgtable_ops *ops)
1299 {
1300 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
1301 struct io_pgtable_cfg *cfg = &data->iop.cfg;
1302
1303 pr_err("cfg: pgsize_bitmap 0x%lx, ias %u-bit\n",
1304 cfg->pgsize_bitmap, cfg->ias);
1305 pr_err("data: %d levels, 0x%zx pgd_size, %u pg_shift, %u bits_per_level, pgd @ %p\n",
1306 ARM_LPAE_MAX_LEVELS - data->start_level, ARM_LPAE_PGD_SIZE(data),
1307 ilog2(ARM_LPAE_GRANULE(data)), data->bits_per_level, data->pgd);
1308 }
1309
1310 #define __FAIL(ops, i) ({ \
1311 WARN(1, "selftest: test failed for fmt idx %d\n", (i)); \
1312 arm_lpae_dump_ops(ops); \
1313 selftest_running = false; \
1314 -EFAULT; \
1315 })
1316
arm_lpae_run_tests(struct io_pgtable_cfg * cfg)1317 static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
1318 {
1319 static const enum io_pgtable_fmt fmts[] __initconst = {
1320 ARM_64_LPAE_S1,
1321 ARM_64_LPAE_S2,
1322 };
1323
1324 int i, j;
1325 unsigned long iova;
1326 size_t size, mapped;
1327 struct io_pgtable_ops *ops;
1328
1329 selftest_running = true;
1330
1331 for (i = 0; i < ARRAY_SIZE(fmts); ++i) {
1332 cfg_cookie = cfg;
1333 ops = alloc_io_pgtable_ops(fmts[i], cfg, cfg);
1334 if (!ops) {
1335 pr_err("selftest: failed to allocate io pgtable ops\n");
1336 return -ENOMEM;
1337 }
1338
1339 /*
1340 * Initial sanity checks.
1341 * Empty page tables shouldn't provide any translations.
1342 */
1343 if (ops->iova_to_phys(ops, 42))
1344 return __FAIL(ops, i);
1345
1346 if (ops->iova_to_phys(ops, SZ_1G + 42))
1347 return __FAIL(ops, i);
1348
1349 if (ops->iova_to_phys(ops, SZ_2G + 42))
1350 return __FAIL(ops, i);
1351
1352 /*
1353 * Distinct mappings of different granule sizes.
1354 */
1355 iova = 0;
1356 for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {
1357 size = 1UL << j;
1358
1359 if (ops->map_pages(ops, iova, iova, size, 1,
1360 IOMMU_READ | IOMMU_WRITE |
1361 IOMMU_NOEXEC | IOMMU_CACHE,
1362 GFP_KERNEL, &mapped))
1363 return __FAIL(ops, i);
1364
1365 /* Overlapping mappings */
1366 if (!ops->map_pages(ops, iova, iova + size, size, 1,
1367 IOMMU_READ | IOMMU_NOEXEC,
1368 GFP_KERNEL, &mapped))
1369 return __FAIL(ops, i);
1370
1371 if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))
1372 return __FAIL(ops, i);
1373
1374 iova += SZ_1G;
1375 }
1376
1377 /* Full unmap */
1378 iova = 0;
1379 for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {
1380 size = 1UL << j;
1381
1382 if (ops->unmap_pages(ops, iova, size, 1, NULL) != size)
1383 return __FAIL(ops, i);
1384
1385 if (ops->iova_to_phys(ops, iova + 42))
1386 return __FAIL(ops, i);
1387
1388 /* Remap full block */
1389 if (ops->map_pages(ops, iova, iova, size, 1,
1390 IOMMU_WRITE, GFP_KERNEL, &mapped))
1391 return __FAIL(ops, i);
1392
1393 if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))
1394 return __FAIL(ops, i);
1395
1396 iova += SZ_1G;
1397 }
1398
1399 /*
1400 * Map/unmap the last largest supported page of the IAS, this can
1401 * trigger corner cases in the concatednated page tables.
1402 */
1403 mapped = 0;
1404 size = 1UL << __fls(cfg->pgsize_bitmap);
1405 iova = (1UL << cfg->ias) - size;
1406 if (ops->map_pages(ops, iova, iova, size, 1,
1407 IOMMU_READ | IOMMU_WRITE |
1408 IOMMU_NOEXEC | IOMMU_CACHE,
1409 GFP_KERNEL, &mapped))
1410 return __FAIL(ops, i);
1411 if (mapped != size)
1412 return __FAIL(ops, i);
1413 if (ops->unmap_pages(ops, iova, size, 1, NULL) != size)
1414 return __FAIL(ops, i);
1415
1416 free_io_pgtable_ops(ops);
1417 }
1418
1419 selftest_running = false;
1420 return 0;
1421 }
1422
arm_lpae_do_selftests(void)1423 static int __init arm_lpae_do_selftests(void)
1424 {
1425 static const unsigned long pgsize[] __initconst = {
1426 SZ_4K | SZ_2M | SZ_1G,
1427 SZ_16K | SZ_32M,
1428 SZ_64K | SZ_512M,
1429 };
1430
1431 static const unsigned int address_size[] __initconst = {
1432 32, 36, 40, 42, 44, 48,
1433 };
1434
1435 int i, j, k, pass = 0, fail = 0;
1436 struct device dev;
1437 struct io_pgtable_cfg cfg = {
1438 .tlb = &dummy_tlb_ops,
1439 .coherent_walk = true,
1440 .iommu_dev = &dev,
1441 };
1442
1443 /* __arm_lpae_alloc_pages() merely needs dev_to_node() to work */
1444 set_dev_node(&dev, NUMA_NO_NODE);
1445
1446 for (i = 0; i < ARRAY_SIZE(pgsize); ++i) {
1447 for (j = 0; j < ARRAY_SIZE(address_size); ++j) {
1448 /* Don't use ias > oas as it is not valid for stage-2. */
1449 for (k = 0; k <= j; ++k) {
1450 cfg.pgsize_bitmap = pgsize[i];
1451 cfg.ias = address_size[k];
1452 cfg.oas = address_size[j];
1453 pr_info("selftest: pgsize_bitmap 0x%08lx, IAS %u OAS %u\n",
1454 pgsize[i], cfg.ias, cfg.oas);
1455 if (arm_lpae_run_tests(&cfg))
1456 fail++;
1457 else
1458 pass++;
1459 }
1460 }
1461 }
1462
1463 pr_info("selftest: completed with %d PASS %d FAIL\n", pass, fail);
1464 return fail ? -EFAULT : 0;
1465 }
1466 subsys_initcall(arm_lpae_do_selftests);
1467 #endif
1468