1 /*
2 * Copyright (c) 2022-2024, Arm Limited. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <assert.h>
8 #include <errno.h>
9 #include <inttypes.h>
10 #include <limits.h>
11 #include <stdint.h>
12
13 #include <arch.h>
14 #include <arch_features.h>
15 #include <arch_helpers.h>
16 #include <common/debug.h>
17 #include "gpt_rme_private.h"
18 #include <lib/gpt_rme/gpt_rme.h>
19 #include <lib/smccc.h>
20 #include <lib/spinlock.h>
21 #include <lib/xlat_tables/xlat_tables_v2.h>
22
23 #if !ENABLE_RME
24 #error "ENABLE_RME must be enabled to use the GPT library"
25 #endif
26
27 /*
28 * Lookup T from PPS
29 *
30 * PPS Size T
31 * 0b000 4GB 32
32 * 0b001 64GB 36
33 * 0b010 1TB 40
34 * 0b011 4TB 42
35 * 0b100 16TB 44
36 * 0b101 256TB 48
37 * 0b110 4PB 52
38 *
39 * See section 15.1.27 of the RME specification.
40 */
41 static const gpt_t_val_e gpt_t_lookup[] = {PPS_4GB_T, PPS_64GB_T,
42 PPS_1TB_T, PPS_4TB_T,
43 PPS_16TB_T, PPS_256TB_T,
44 PPS_4PB_T};
45
46 /*
47 * Lookup P from PGS
48 *
49 * PGS Size P
50 * 0b00 4KB 12
51 * 0b10 16KB 14
52 * 0b01 64KB 16
53 *
54 * Note that pgs=0b10 is 16KB and pgs=0b01 is 64KB, this is not a typo.
55 *
56 * See section 15.1.27 of the RME specification.
57 */
58 static const gpt_p_val_e gpt_p_lookup[] = {PGS_4KB_P, PGS_64KB_P, PGS_16KB_P};
59
60 /*
61 * This structure contains GPT configuration data
62 */
63 typedef struct {
64 uintptr_t plat_gpt_l0_base;
65 gpccr_pps_e pps;
66 gpt_t_val_e t;
67 gpccr_pgs_e pgs;
68 gpt_p_val_e p;
69 } gpt_config_t;
70
71 static gpt_config_t gpt_config;
72
73 /* These variables are used during initialization of the L1 tables */
74 static unsigned int gpt_next_l1_tbl_idx;
75 static uintptr_t gpt_l1_tbl;
76
77 /*
78 * This function checks to see if a GPI value is valid.
79 *
80 * These are valid GPI values.
81 * GPT_GPI_NO_ACCESS U(0x0)
82 * GPT_GPI_SECURE U(0x8)
83 * GPT_GPI_NS U(0x9)
84 * GPT_GPI_ROOT U(0xA)
85 * GPT_GPI_REALM U(0xB)
86 * GPT_GPI_ANY U(0xF)
87 *
88 * Parameters
89 * gpi GPI to check for validity.
90 *
91 * Return
92 * true for a valid GPI, false for an invalid one.
93 */
is_gpi_valid(unsigned int gpi)94 static bool is_gpi_valid(unsigned int gpi)
95 {
96 if ((gpi == GPT_GPI_NO_ACCESS) || (gpi == GPT_GPI_ANY) ||
97 ((gpi >= GPT_GPI_SECURE) && (gpi <= GPT_GPI_REALM))) {
98 return true;
99 }
100 return false;
101 }
102
103 /*
104 * This function checks to see if two PAS regions overlap.
105 *
106 * Parameters
107 * base_1: base address of first PAS
108 * size_1: size of first PAS
109 * base_2: base address of second PAS
110 * size_2: size of second PAS
111 *
112 * Return
113 * True if PAS regions overlap, false if they do not.
114 */
check_pas_overlap(uintptr_t base_1,size_t size_1,uintptr_t base_2,size_t size_2)115 static bool check_pas_overlap(uintptr_t base_1, size_t size_1,
116 uintptr_t base_2, size_t size_2)
117 {
118 if (((base_1 + size_1) > base_2) && ((base_2 + size_2) > base_1)) {
119 return true;
120 }
121 return false;
122 }
123
124 /*
125 * This helper function checks to see if a PAS region from index 0 to
126 * (pas_idx - 1) occupies the L0 region at index l0_idx in the L0 table.
127 *
128 * Parameters
129 * l0_idx: Index of the L0 entry to check
130 * pas_regions: PAS region array
131 * pas_idx: Upper bound of the PAS array index.
132 *
133 * Return
134 * True if a PAS region occupies the L0 region in question, false if not.
135 */
does_previous_pas_exist_here(unsigned int l0_idx,pas_region_t * pas_regions,unsigned int pas_idx)136 static bool does_previous_pas_exist_here(unsigned int l0_idx,
137 pas_region_t *pas_regions,
138 unsigned int pas_idx)
139 {
140 /* Iterate over PAS regions up to pas_idx */
141 for (unsigned int i = 0U; i < pas_idx; i++) {
142 if (check_pas_overlap((GPT_L0GPTSZ_ACTUAL_SIZE * l0_idx),
143 GPT_L0GPTSZ_ACTUAL_SIZE,
144 pas_regions[i].base_pa, pas_regions[i].size)) {
145 return true;
146 }
147 }
148 return false;
149 }
150
151 /*
152 * This function iterates over all of the PAS regions and checks them to ensure
153 * proper alignment of base and size, that the GPI is valid, and that no regions
154 * overlap. As a part of the overlap checks, this function checks existing L0
155 * mappings against the new PAS regions in the event that gpt_init_pas_l1_tables
156 * is called multiple times to place L1 tables in different areas of memory. It
157 * also counts the number of L1 tables needed and returns it on success.
158 *
159 * Parameters
160 * *pas_regions Pointer to array of PAS region structures.
161 * pas_region_cnt Total number of PAS regions in the array.
162 *
163 * Return
164 * Negative Linux error code in the event of a failure, number of L1 regions
165 * required when successful.
166 */
validate_pas_mappings(pas_region_t * pas_regions,unsigned int pas_region_cnt)167 static int validate_pas_mappings(pas_region_t *pas_regions,
168 unsigned int pas_region_cnt)
169 {
170 unsigned int idx;
171 unsigned int l1_cnt = 0U;
172 unsigned int pas_l1_cnt;
173 uint64_t *l0_desc = (uint64_t *)gpt_config.plat_gpt_l0_base;
174
175 assert(pas_regions != NULL);
176 assert(pas_region_cnt != 0U);
177
178 for (idx = 0U; idx < pas_region_cnt; idx++) {
179 /* Check for arithmetic overflow in region */
180 if ((ULONG_MAX - pas_regions[idx].base_pa) <
181 pas_regions[idx].size) {
182 ERROR("GPT: Address overflow in PAS[%u]!\n", idx);
183 return -EOVERFLOW;
184 }
185
186 /* Initial checks for PAS validity */
187 if (((pas_regions[idx].base_pa + pas_regions[idx].size) >
188 GPT_PPS_ACTUAL_SIZE(gpt_config.t)) ||
189 !is_gpi_valid(GPT_PAS_ATTR_GPI(pas_regions[idx].attrs))) {
190 ERROR("GPT: PAS[%u] is invalid!\n", idx);
191 return -EFAULT;
192 }
193
194 /*
195 * Make sure this PAS does not overlap with another one. We
196 * start from idx + 1 instead of 0 since prior PAS mappings will
197 * have already checked themselves against this one.
198 */
199 for (unsigned int i = idx + 1U; i < pas_region_cnt; i++) {
200 if (check_pas_overlap(pas_regions[idx].base_pa,
201 pas_regions[idx].size,
202 pas_regions[i].base_pa,
203 pas_regions[i].size)) {
204 ERROR("GPT: PAS[%u] overlaps with PAS[%u]\n",
205 i, idx);
206 return -EFAULT;
207 }
208 }
209
210 /*
211 * Since this function can be called multiple times with
212 * separate L1 tables we need to check the existing L0 mapping
213 * to see if this PAS would fall into one that has already been
214 * initialized.
215 */
216 for (unsigned int i = GPT_L0_IDX(pas_regions[idx].base_pa);
217 i <= GPT_L0_IDX(pas_regions[idx].base_pa +
218 pas_regions[idx].size - 1UL);
219 i++) {
220 if ((GPT_L0_TYPE(l0_desc[i]) == GPT_L0_TYPE_BLK_DESC) &&
221 (GPT_L0_BLKD_GPI(l0_desc[i]) == GPT_GPI_ANY)) {
222 /* This descriptor is unused so continue */
223 continue;
224 }
225
226 /*
227 * This descriptor has been initialized in a previous
228 * call to this function so cannot be initialized again.
229 */
230 ERROR("GPT: PAS[%u] overlaps with previous L0[%d]!\n",
231 idx, i);
232 return -EFAULT;
233 }
234
235 /* Check for block mapping (L0) type */
236 if (GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs) ==
237 GPT_PAS_ATTR_MAP_TYPE_BLOCK) {
238 /* Make sure base and size are block-aligned */
239 if (!GPT_IS_L0_ALIGNED(pas_regions[idx].base_pa) ||
240 !GPT_IS_L0_ALIGNED(pas_regions[idx].size)) {
241 ERROR("GPT: PAS[%u] is not block-aligned!\n",
242 idx);
243 return -EFAULT;
244 }
245
246 continue;
247 }
248
249 /* Check for granule mapping (L1) type */
250 if (GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs) ==
251 GPT_PAS_ATTR_MAP_TYPE_GRANULE) {
252 /* Make sure base and size are granule-aligned */
253 if (!GPT_IS_L1_ALIGNED(gpt_config.p, pas_regions[idx].base_pa) ||
254 !GPT_IS_L1_ALIGNED(gpt_config.p, pas_regions[idx].size)) {
255 ERROR("GPT: PAS[%u] is not granule-aligned!\n",
256 idx);
257 return -EFAULT;
258 }
259
260 /* Find how many L1 tables this PAS occupies */
261 pas_l1_cnt = (GPT_L0_IDX(pas_regions[idx].base_pa +
262 pas_regions[idx].size - 1UL) -
263 GPT_L0_IDX(pas_regions[idx].base_pa) + 1U);
264
265 /*
266 * This creates a situation where, if multiple PAS
267 * regions occupy the same table descriptor, we can get
268 * an artificially high total L1 table count. The way we
269 * handle this is by checking each PAS against those
270 * before it in the array, and if they both occupy the
271 * same PAS we subtract from pas_l1_cnt and only the
272 * first PAS in the array gets to count it.
273 */
274
275 /*
276 * If L1 count is greater than 1 we know the start and
277 * end PAs are in different L0 regions so we must check
278 * both for overlap against other PAS.
279 */
280 if (pas_l1_cnt > 1) {
281 if (does_previous_pas_exist_here(
282 GPT_L0_IDX(pas_regions[idx].base_pa +
283 pas_regions[idx].size - 1UL),
284 pas_regions, idx)) {
285 pas_l1_cnt--;
286 }
287 }
288
289 if (does_previous_pas_exist_here(
290 GPT_L0_IDX(pas_regions[idx].base_pa),
291 pas_regions, idx)) {
292 pas_l1_cnt--;
293 }
294
295 l1_cnt += pas_l1_cnt;
296 continue;
297 }
298
299 /* If execution reaches this point, mapping type is invalid */
300 ERROR("GPT: PAS[%u] has invalid mapping type 0x%x.\n", idx,
301 GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs));
302 return -EINVAL;
303 }
304
305 return l1_cnt;
306 }
307
308 /*
309 * This function validates L0 initialization parameters.
310 *
311 * Parameters
312 * l0_mem_base Base address of memory used for L0 tables.
313 * l1_mem_size Size of memory available for L0 tables.
314 *
315 * Return
316 * Negative Linux error code in the event of a failure, 0 for success.
317 */
validate_l0_params(gpccr_pps_e pps,uintptr_t l0_mem_base,size_t l0_mem_size)318 static int validate_l0_params(gpccr_pps_e pps, uintptr_t l0_mem_base,
319 size_t l0_mem_size)
320 {
321 size_t l0_alignment;
322
323 /*
324 * Make sure PPS is valid and then store it since macros need this value
325 * to work.
326 */
327 if (pps > GPT_PPS_MAX) {
328 ERROR("GPT: Invalid PPS: 0x%x\n", pps);
329 return -EINVAL;
330 }
331 gpt_config.pps = pps;
332 gpt_config.t = gpt_t_lookup[pps];
333
334 /* Alignment must be the greater of 4KB or l0 table size */
335 l0_alignment = PAGE_SIZE_4KB;
336 if (l0_alignment < GPT_L0_TABLE_SIZE(gpt_config.t)) {
337 l0_alignment = GPT_L0_TABLE_SIZE(gpt_config.t);
338 }
339
340 /* Check base address */
341 if ((l0_mem_base == 0UL) ||
342 ((l0_mem_base & (l0_alignment - 1UL)) != 0UL)) {
343 ERROR("GPT: Invalid L0 base address: 0x%lx\n", l0_mem_base);
344 return -EFAULT;
345 }
346
347 /* Check size */
348 if (l0_mem_size < GPT_L0_TABLE_SIZE(gpt_config.t)) {
349 ERROR("%sL0%s\n", "GPT: Inadequate ", " memory\n");
350 ERROR(" Expected 0x%lx bytes, got 0x%lx bytes\n",
351 GPT_L0_TABLE_SIZE(gpt_config.t),
352 l0_mem_size);
353 return -ENOMEM;
354 }
355
356 return 0;
357 }
358
359 /*
360 * In the event that L1 tables are needed, this function validates
361 * the L1 table generation parameters.
362 *
363 * Parameters
364 * l1_mem_base Base address of memory used for L1 table allocation.
365 * l1_mem_size Total size of memory available for L1 tables.
366 * l1_gpt_cnt Number of L1 tables needed.
367 *
368 * Return
369 * Negative Linux error code in the event of a failure, 0 for success.
370 */
validate_l1_params(uintptr_t l1_mem_base,size_t l1_mem_size,unsigned int l1_gpt_cnt)371 static int validate_l1_params(uintptr_t l1_mem_base, size_t l1_mem_size,
372 unsigned int l1_gpt_cnt)
373 {
374 size_t l1_gpt_mem_sz;
375
376 /* Check if the granularity is supported */
377 if (!xlat_arch_is_granule_size_supported(
378 GPT_PGS_ACTUAL_SIZE(gpt_config.p))) {
379 return -EPERM;
380 }
381
382 /* Make sure L1 tables are aligned to their size */
383 if ((l1_mem_base & (GPT_L1_TABLE_SIZE(gpt_config.p) - 1UL)) != 0UL) {
384 ERROR("GPT: Unaligned L1 GPT base address: 0x%"PRIxPTR"\n",
385 l1_mem_base);
386 return -EFAULT;
387 }
388
389 /* Get total memory needed for L1 tables */
390 l1_gpt_mem_sz = l1_gpt_cnt * GPT_L1_TABLE_SIZE(gpt_config.p);
391
392 /* Check for overflow */
393 if ((l1_gpt_mem_sz / GPT_L1_TABLE_SIZE(gpt_config.p)) != l1_gpt_cnt) {
394 ERROR("GPT: Overflow calculating L1 memory size\n");
395 return -ENOMEM;
396 }
397
398 /* Make sure enough space was supplied */
399 if (l1_mem_size < l1_gpt_mem_sz) {
400 ERROR("%sL1 GPTs%s", "GPT: Inadequate ", " memory\n");
401 ERROR(" Expected 0x%lx bytes, got 0x%lx bytes\n",
402 l1_gpt_mem_sz, l1_mem_size);
403 return -ENOMEM;
404 }
405
406 VERBOSE("GPT: Requested 0x%lx bytes for L1 GPTs\n", l1_gpt_mem_sz);
407 return 0;
408 }
409
410 /*
411 * This function initializes L0 block descriptors (regions that cannot be
412 * transitioned at the granule level) according to the provided PAS.
413 *
414 * Parameters
415 * *pas Pointer to the structure defining the PAS region to
416 * initialize.
417 */
generate_l0_blk_desc(pas_region_t * pas)418 static void generate_l0_blk_desc(pas_region_t *pas)
419 {
420 uint64_t gpt_desc;
421 unsigned int end_idx;
422 unsigned int idx;
423 uint64_t *l0_gpt_arr;
424
425 assert(gpt_config.plat_gpt_l0_base != 0U);
426 assert(pas != NULL);
427
428 /*
429 * Checking of PAS parameters has already been done in
430 * validate_pas_mappings so no need to check the same things again.
431 */
432
433 l0_gpt_arr = (uint64_t *)gpt_config.plat_gpt_l0_base;
434
435 /* Create the GPT Block descriptor for this PAS region */
436 gpt_desc = GPT_L0_BLK_DESC(GPT_PAS_ATTR_GPI(pas->attrs));
437
438 /* Start index of this region in L0 GPTs */
439 idx = GPT_L0_IDX(pas->base_pa);
440
441 /*
442 * Determine number of L0 GPT descriptors covered by
443 * this PAS region and use the count to populate these
444 * descriptors.
445 */
446 end_idx = GPT_L0_IDX(pas->base_pa + pas->size);
447
448 /* Generate the needed block descriptors */
449 for (; idx < end_idx; idx++) {
450 l0_gpt_arr[idx] = gpt_desc;
451 VERBOSE("GPT: L0 entry (BLOCK) index %u [%p]: GPI = 0x%"PRIx64" (0x%"PRIx64")\n",
452 idx, &l0_gpt_arr[idx],
453 (gpt_desc >> GPT_L0_BLK_DESC_GPI_SHIFT) &
454 GPT_L0_BLK_DESC_GPI_MASK, l0_gpt_arr[idx]);
455 }
456 }
457
458 /*
459 * Helper function to determine if the end physical address lies in the same L0
460 * region as the current physical address. If true, the end physical address is
461 * returned else, the start address of the next region is returned.
462 *
463 * Parameters
464 * cur_pa Physical address of the current PA in the loop through
465 * the range.
466 * end_pa Physical address of the end PA in a PAS range.
467 *
468 * Return
469 * The PA of the end of the current range.
470 */
get_l1_end_pa(uintptr_t cur_pa,uintptr_t end_pa)471 static uintptr_t get_l1_end_pa(uintptr_t cur_pa, uintptr_t end_pa)
472 {
473 uintptr_t cur_idx;
474 uintptr_t end_idx;
475
476 cur_idx = GPT_L0_IDX(cur_pa);
477 end_idx = GPT_L0_IDX(end_pa);
478
479 assert(cur_idx <= end_idx);
480
481 if (cur_idx == end_idx) {
482 return end_pa;
483 }
484
485 return (cur_idx + 1U) << GPT_L0_IDX_SHIFT;
486 }
487
488 /*
489 * Helper function to fill out GPI entries in a single L1 table. This function
490 * fills out entire L1 descriptors at a time to save memory writes.
491 *
492 * Parameters
493 * gpi GPI to set this range to
494 * l1 Pointer to L1 table to fill out
495 * first Address of first granule in range.
496 * last Address of last granule in range (inclusive).
497 */
fill_l1_tbl(uint64_t gpi,uint64_t * l1,uintptr_t first,uintptr_t last)498 static void fill_l1_tbl(uint64_t gpi, uint64_t *l1, uintptr_t first,
499 uintptr_t last)
500 {
501 uint64_t gpi_field = GPT_BUILD_L1_DESC(gpi);
502 uint64_t gpi_mask = ULONG_MAX;
503
504 assert(first <= last);
505 assert((first & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1)) == 0U);
506 assert((last & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1)) == 0U);
507 assert(GPT_L0_IDX(first) == GPT_L0_IDX(last));
508 assert(l1 != NULL);
509
510 /* Shift the mask if we're starting in the middle of an L1 entry */
511 gpi_mask = gpi_mask << (GPT_L1_GPI_IDX(gpt_config.p, first) << 2);
512
513 /* Fill out each L1 entry for this region */
514 for (unsigned int i = GPT_L1_IDX(gpt_config.p, first);
515 i <= GPT_L1_IDX(gpt_config.p, last); i++) {
516 /* Account for stopping in the middle of an L1 entry */
517 if (i == GPT_L1_IDX(gpt_config.p, last)) {
518 gpi_mask &= (gpi_mask >> ((15U -
519 GPT_L1_GPI_IDX(gpt_config.p, last)) << 2));
520 }
521
522 /* Write GPI values */
523 assert((l1[i] & gpi_mask) ==
524 (GPT_BUILD_L1_DESC(GPT_GPI_ANY) & gpi_mask));
525 l1[i] = (l1[i] & ~gpi_mask) | (gpi_mask & gpi_field);
526
527 /* Reset mask */
528 gpi_mask = ULONG_MAX;
529 }
530 }
531
532 /*
533 * This function finds the next available unused L1 table and initializes all
534 * granules descriptor entries to GPI_ANY. This ensures that there are no chunks
535 * of GPI_NO_ACCESS (0b0000) memory floating around in the system in the
536 * event that a PAS region stops midway through an L1 table, thus guaranteeing
537 * that all memory not explicitly assigned is GPI_ANY. This function does not
538 * check for overflow conditions, that should be done by the caller.
539 *
540 * Return
541 * Pointer to the next available L1 table.
542 */
get_new_l1_tbl(void)543 static uint64_t *get_new_l1_tbl(void)
544 {
545 /* Retrieve the next L1 table */
546 uint64_t *l1 = (uint64_t *)((uint64_t)(gpt_l1_tbl) +
547 (GPT_L1_TABLE_SIZE(gpt_config.p) *
548 gpt_next_l1_tbl_idx));
549
550 /* Increment L1 counter */
551 gpt_next_l1_tbl_idx++;
552
553 /* Initialize all GPIs to GPT_GPI_ANY */
554 for (unsigned int i = 0U; i < GPT_L1_ENTRY_COUNT(gpt_config.p); i++) {
555 l1[i] = GPT_BUILD_L1_DESC(GPT_GPI_ANY);
556 }
557
558 return l1;
559 }
560
561 /*
562 * When L1 tables are needed, this function creates the necessary L0 table
563 * descriptors and fills out the L1 table entries according to the supplied
564 * PAS range.
565 *
566 * Parameters
567 * *pas Pointer to the structure defining the PAS region.
568 */
generate_l0_tbl_desc(pas_region_t * pas)569 static void generate_l0_tbl_desc(pas_region_t *pas)
570 {
571 uintptr_t end_pa;
572 uintptr_t cur_pa;
573 uintptr_t last_gran_pa;
574 uint64_t *l0_gpt_base;
575 uint64_t *l1_gpt_arr;
576 unsigned int l0_idx;
577
578 assert(gpt_config.plat_gpt_l0_base != 0U);
579 assert(pas != NULL);
580
581 /*
582 * Checking of PAS parameters has already been done in
583 * validate_pas_mappings so no need to check the same things again.
584 */
585
586 end_pa = pas->base_pa + pas->size;
587 l0_gpt_base = (uint64_t *)gpt_config.plat_gpt_l0_base;
588
589 /* We start working from the granule at base PA */
590 cur_pa = pas->base_pa;
591
592 /* Iterate over each L0 region in this memory range */
593 for (l0_idx = GPT_L0_IDX(pas->base_pa);
594 l0_idx <= GPT_L0_IDX(end_pa - 1U);
595 l0_idx++) {
596
597 /*
598 * See if the L0 entry is already a table descriptor or if we
599 * need to create one.
600 */
601 if (GPT_L0_TYPE(l0_gpt_base[l0_idx]) == GPT_L0_TYPE_TBL_DESC) {
602 /* Get the L1 array from the L0 entry */
603 l1_gpt_arr = GPT_L0_TBLD_ADDR(l0_gpt_base[l0_idx]);
604 } else {
605 /* Get a new L1 table from the L1 memory space */
606 l1_gpt_arr = get_new_l1_tbl();
607
608 /* Fill out the L0 descriptor and flush it */
609 l0_gpt_base[l0_idx] = GPT_L0_TBL_DESC(l1_gpt_arr);
610 }
611
612 VERBOSE("GPT: L0 entry (TABLE) index %u [%p] ==> L1 Addr %p (0x%"PRIx64")\n",
613 l0_idx, &l0_gpt_base[l0_idx], l1_gpt_arr, l0_gpt_base[l0_idx]);
614
615 /*
616 * Determine the PA of the last granule in this L0 descriptor.
617 */
618 last_gran_pa = get_l1_end_pa(cur_pa, end_pa) -
619 GPT_PGS_ACTUAL_SIZE(gpt_config.p);
620
621 /*
622 * Fill up L1 GPT entries between these two addresses. This
623 * function needs the addresses of the first granule and last
624 * granule in the range.
625 */
626 fill_l1_tbl(GPT_PAS_ATTR_GPI(pas->attrs), l1_gpt_arr,
627 cur_pa, last_gran_pa);
628
629 /* Advance cur_pa to first granule in next L0 region */
630 cur_pa = get_l1_end_pa(cur_pa, end_pa);
631 }
632 }
633
634 /*
635 * This function flushes a range of L0 descriptors used by a given PAS region
636 * array. There is a chance that some unmodified L0 descriptors would be flushed
637 * in the case that there are "holes" in an array of PAS regions but overall
638 * this should be faster than individually flushing each modified L0 descriptor
639 * as they are created.
640 *
641 * Parameters
642 * *pas Pointer to an array of PAS regions.
643 * pas_count Number of entries in the PAS array.
644 */
flush_l0_for_pas_array(pas_region_t * pas,unsigned int pas_count)645 static void flush_l0_for_pas_array(pas_region_t *pas, unsigned int pas_count)
646 {
647 unsigned int idx;
648 unsigned int start_idx;
649 unsigned int end_idx;
650 uint64_t *l0 = (uint64_t *)gpt_config.plat_gpt_l0_base;
651
652 assert(pas != NULL);
653 assert(pas_count != 0U);
654
655 /* Initial start and end values */
656 start_idx = GPT_L0_IDX(pas[0].base_pa);
657 end_idx = GPT_L0_IDX(pas[0].base_pa + pas[0].size - 1UL);
658
659 /* Find lowest and highest L0 indices used in this PAS array */
660 for (idx = 1U; idx < pas_count; idx++) {
661 if (GPT_L0_IDX(pas[idx].base_pa) < start_idx) {
662 start_idx = GPT_L0_IDX(pas[idx].base_pa);
663 }
664 if (GPT_L0_IDX(pas[idx].base_pa + pas[idx].size - 1UL) > end_idx) {
665 end_idx = GPT_L0_IDX(pas[idx].base_pa + pas[idx].size - 1UL);
666 }
667 }
668
669 /*
670 * Flush all covered L0 descriptors, add 1 because we need to include
671 * the end index value.
672 */
673 flush_dcache_range((uintptr_t)&l0[start_idx],
674 ((end_idx + 1U) - start_idx) * sizeof(uint64_t));
675 }
676
677 /*
678 * Public API to enable granule protection checks once the tables have all been
679 * initialized. This function is called at first initialization and then again
680 * later during warm boots of CPU cores.
681 *
682 * Return
683 * Negative Linux error code in the event of a failure, 0 for success.
684 */
gpt_enable(void)685 int gpt_enable(void)
686 {
687 u_register_t gpccr_el3;
688
689 /*
690 * Granule tables must be initialised before enabling
691 * granule protection.
692 */
693 if (gpt_config.plat_gpt_l0_base == 0UL) {
694 ERROR("GPT: Tables have not been initialized!\n");
695 return -EPERM;
696 }
697
698 /* Write the base address of the L0 tables into GPTBR */
699 write_gptbr_el3(((gpt_config.plat_gpt_l0_base >> GPTBR_BADDR_VAL_SHIFT)
700 >> GPTBR_BADDR_SHIFT) & GPTBR_BADDR_MASK);
701
702 /* GPCCR_EL3.PPS */
703 gpccr_el3 = SET_GPCCR_PPS(gpt_config.pps);
704
705 /* GPCCR_EL3.PGS */
706 gpccr_el3 |= SET_GPCCR_PGS(gpt_config.pgs);
707
708 /*
709 * Since EL3 maps the L1 region as Inner shareable, use the same
710 * shareability attribute for GPC as well so that
711 * GPC fetches are visible to PEs
712 */
713 gpccr_el3 |= SET_GPCCR_SH(GPCCR_SH_IS);
714
715 /* Outer and Inner cacheability set to Normal memory, WB, RA, WA */
716 gpccr_el3 |= SET_GPCCR_ORGN(GPCCR_ORGN_WB_RA_WA);
717 gpccr_el3 |= SET_GPCCR_IRGN(GPCCR_IRGN_WB_RA_WA);
718
719 /* Prepopulate GPCCR_EL3 but don't enable GPC yet */
720 write_gpccr_el3(gpccr_el3);
721 isb();
722
723 /* Invalidate any stale TLB entries and any cached register fields */
724 tlbipaallos();
725 dsb();
726 isb();
727
728 /* Enable GPT */
729 gpccr_el3 |= GPCCR_GPC_BIT;
730
731 /* TODO: Configure GPCCR_EL3_GPCP for Fault control */
732 write_gpccr_el3(gpccr_el3);
733 isb();
734 tlbipaallos();
735 dsb();
736 isb();
737
738 return 0;
739 }
740
741 /*
742 * Public API to disable granule protection checks.
743 */
gpt_disable(void)744 void gpt_disable(void)
745 {
746 u_register_t gpccr_el3 = read_gpccr_el3();
747
748 write_gpccr_el3(gpccr_el3 & ~GPCCR_GPC_BIT);
749 dsbsy();
750 isb();
751 }
752
753 /*
754 * Public API that initializes the entire protected space to GPT_GPI_ANY using
755 * the L0 tables (block descriptors). Ideally, this function is invoked prior
756 * to DDR discovery and initialization. The MMU must be initialized before
757 * calling this function.
758 *
759 * Parameters
760 * pps PPS value to use for table generation
761 * l0_mem_base Base address of L0 tables in memory.
762 * l0_mem_size Total size of memory available for L0 tables.
763 *
764 * Return
765 * Negative Linux error code in the event of a failure, 0 for success.
766 */
gpt_init_l0_tables(gpccr_pps_e pps,uintptr_t l0_mem_base,size_t l0_mem_size)767 int gpt_init_l0_tables(gpccr_pps_e pps, uintptr_t l0_mem_base,
768 size_t l0_mem_size)
769 {
770 int ret;
771 uint64_t gpt_desc;
772
773 /* Ensure that MMU and Data caches are enabled */
774 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0U);
775
776 /* Validate other parameters */
777 ret = validate_l0_params(pps, l0_mem_base, l0_mem_size);
778 if (ret != 0) {
779 return ret;
780 }
781
782 /* Create the descriptor to initialize L0 entries with */
783 gpt_desc = GPT_L0_BLK_DESC(GPT_GPI_ANY);
784
785 /* Iterate through all L0 entries */
786 for (unsigned int i = 0U; i < GPT_L0_REGION_COUNT(gpt_config.t); i++) {
787 ((uint64_t *)l0_mem_base)[i] = gpt_desc;
788 }
789
790 /* Flush updated L0 tables to memory */
791 flush_dcache_range((uintptr_t)l0_mem_base,
792 (size_t)GPT_L0_TABLE_SIZE(gpt_config.t));
793
794 /* Stash the L0 base address once initial setup is complete */
795 gpt_config.plat_gpt_l0_base = l0_mem_base;
796
797 return 0;
798 }
799
800 /*
801 * Public API that carves out PAS regions from the L0 tables and builds any L1
802 * tables that are needed. This function ideally is run after DDR discovery and
803 * initialization. The L0 tables must have already been initialized to GPI_ANY
804 * when this function is called.
805 *
806 * This function can be called multiple times with different L1 memory ranges
807 * and PAS regions if it is desirable to place L1 tables in different locations
808 * in memory. (ex: you have multiple DDR banks and want to place the L1 tables
809 * in the DDR bank that they control)
810 *
811 * Parameters
812 * pgs PGS value to use for table generation.
813 * l1_mem_base Base address of memory used for L1 tables.
814 * l1_mem_size Total size of memory available for L1 tables.
815 * *pas_regions Pointer to PAS regions structure array.
816 * pas_count Total number of PAS regions.
817 *
818 * Return
819 * Negative Linux error code in the event of a failure, 0 for success.
820 */
gpt_init_pas_l1_tables(gpccr_pgs_e pgs,uintptr_t l1_mem_base,size_t l1_mem_size,pas_region_t * pas_regions,unsigned int pas_count)821 int gpt_init_pas_l1_tables(gpccr_pgs_e pgs, uintptr_t l1_mem_base,
822 size_t l1_mem_size, pas_region_t *pas_regions,
823 unsigned int pas_count)
824 {
825 int ret;
826 int l1_gpt_cnt;
827
828 /* Ensure that MMU and Data caches are enabled */
829 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0U);
830
831 /* PGS is needed for validate_pas_mappings so check it now */
832 if (pgs > GPT_PGS_MAX) {
833 ERROR("GPT: Invalid PGS: 0x%x\n", pgs);
834 return -EINVAL;
835 }
836 gpt_config.pgs = pgs;
837 gpt_config.p = gpt_p_lookup[pgs];
838
839 /* Make sure L0 tables have been initialized */
840 if (gpt_config.plat_gpt_l0_base == 0U) {
841 ERROR("GPT: L0 tables must be initialized first!\n");
842 return -EPERM;
843 }
844
845 /* Check if L1 GPTs are required and how many */
846 l1_gpt_cnt = validate_pas_mappings(pas_regions, pas_count);
847 if (l1_gpt_cnt < 0) {
848 return l1_gpt_cnt;
849 }
850
851 VERBOSE("GPT: %i L1 GPTs requested\n", l1_gpt_cnt);
852
853 /* If L1 tables are needed then validate the L1 parameters */
854 if (l1_gpt_cnt > 0) {
855 ret = validate_l1_params(l1_mem_base, l1_mem_size,
856 (unsigned int)l1_gpt_cnt);
857 if (ret != 0) {
858 return ret;
859 }
860
861 /* Set up parameters for L1 table generation */
862 gpt_l1_tbl = l1_mem_base;
863 gpt_next_l1_tbl_idx = 0U;
864 }
865
866 INFO("GPT: Boot Configuration\n");
867 INFO(" PPS/T: 0x%x/%u\n", gpt_config.pps, gpt_config.t);
868 INFO(" PGS/P: 0x%x/%u\n", gpt_config.pgs, gpt_config.p);
869 INFO(" L0GPTSZ/S: 0x%x/%u\n", GPT_L0GPTSZ, GPT_S_VAL);
870 INFO(" PAS count: %u\n", pas_count);
871 INFO(" L0 base: 0x%"PRIxPTR"\n", gpt_config.plat_gpt_l0_base);
872
873 /* Generate the tables in memory */
874 for (unsigned int idx = 0U; idx < pas_count; idx++) {
875 VERBOSE("GPT: PAS[%u]: base 0x%"PRIxPTR"\tsize 0x%lx\tGPI 0x%x\ttype 0x%x\n",
876 idx, pas_regions[idx].base_pa, pas_regions[idx].size,
877 GPT_PAS_ATTR_GPI(pas_regions[idx].attrs),
878 GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs));
879
880 /* Check if a block or table descriptor is required */
881 if (GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs) ==
882 GPT_PAS_ATTR_MAP_TYPE_BLOCK) {
883 generate_l0_blk_desc(&pas_regions[idx]);
884
885 } else {
886 generate_l0_tbl_desc(&pas_regions[idx]);
887 }
888 }
889
890 /* Flush modified L0 tables */
891 flush_l0_for_pas_array(pas_regions, pas_count);
892
893 /* Flush L1 tables if needed */
894 if (l1_gpt_cnt > 0) {
895 flush_dcache_range(l1_mem_base,
896 GPT_L1_TABLE_SIZE(gpt_config.p) *
897 l1_gpt_cnt);
898 }
899
900 /* Make sure that all the entries are written to the memory */
901 dsbishst();
902 tlbipaallos();
903 dsb();
904 isb();
905
906 return 0;
907 }
908
909 /*
910 * Public API to initialize the runtime gpt_config structure based on the values
911 * present in the GPTBR_EL3 and GPCCR_EL3 registers. GPT initialization
912 * typically happens in a bootloader stage prior to setting up the EL3 runtime
913 * environment for the granule transition service so this function detects the
914 * initialization from a previous stage. Granule protection checks must be
915 * enabled already or this function will return an error.
916 *
917 * Return
918 * Negative Linux error code in the event of a failure, 0 for success.
919 */
gpt_runtime_init(void)920 int gpt_runtime_init(void)
921 {
922 u_register_t reg;
923
924 /* Ensure that MMU and Data caches are enabled */
925 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0U);
926
927 /* Ensure GPC are already enabled */
928 if ((read_gpccr_el3() & GPCCR_GPC_BIT) == 0U) {
929 ERROR("GPT: Granule protection checks are not enabled!\n");
930 return -EPERM;
931 }
932
933 /*
934 * Read the L0 table address from GPTBR, we don't need the L1 base
935 * address since those are included in the L0 tables as needed.
936 */
937 reg = read_gptbr_el3();
938 gpt_config.plat_gpt_l0_base = ((reg >> GPTBR_BADDR_SHIFT) &
939 GPTBR_BADDR_MASK) <<
940 GPTBR_BADDR_VAL_SHIFT;
941
942 /* Read GPCCR to get PGS and PPS values */
943 reg = read_gpccr_el3();
944 gpt_config.pps = (reg >> GPCCR_PPS_SHIFT) & GPCCR_PPS_MASK;
945 gpt_config.t = gpt_t_lookup[gpt_config.pps];
946 gpt_config.pgs = (reg >> GPCCR_PGS_SHIFT) & GPCCR_PGS_MASK;
947 gpt_config.p = gpt_p_lookup[gpt_config.pgs];
948
949 VERBOSE("GPT: Runtime Configuration\n");
950 VERBOSE(" PPS/T: 0x%x/%u\n", gpt_config.pps, gpt_config.t);
951 VERBOSE(" PGS/P: 0x%x/%u\n", gpt_config.pgs, gpt_config.p);
952 VERBOSE(" L0GPTSZ/S: 0x%x/%u\n", GPT_L0GPTSZ, GPT_S_VAL);
953 VERBOSE(" L0 base: 0x%"PRIxPTR"\n", gpt_config.plat_gpt_l0_base);
954
955 return 0;
956 }
957
958 /*
959 * The L1 descriptors are protected by a spinlock to ensure that multiple
960 * CPUs do not attempt to change the descriptors at once. In the future it
961 * would be better to have separate spinlocks for each L1 descriptor.
962 */
963 static spinlock_t gpt_lock;
964
965 /*
966 * A helper to write the value (target_pas << gpi_shift) to the index of
967 * the gpt_l1_addr.
968 */
write_gpt(uint64_t * gpt_l1_desc,uint64_t * gpt_l1_addr,unsigned int gpi_shift,unsigned int idx,unsigned int target_pas)969 static inline void write_gpt(uint64_t *gpt_l1_desc, uint64_t *gpt_l1_addr,
970 unsigned int gpi_shift, unsigned int idx,
971 unsigned int target_pas)
972 {
973 *gpt_l1_desc &= ~(GPT_L1_GRAN_DESC_GPI_MASK << gpi_shift);
974 *gpt_l1_desc |= ((uint64_t)target_pas << gpi_shift);
975 gpt_l1_addr[idx] = *gpt_l1_desc;
976 }
977
978 /*
979 * Helper to retrieve the gpt_l1_* information from the base address
980 * returned in gpi_info.
981 */
get_gpi_params(uint64_t base,gpi_info_t * gpi_info)982 static int get_gpi_params(uint64_t base, gpi_info_t *gpi_info)
983 {
984 uint64_t gpt_l0_desc, *gpt_l0_base;
985
986 gpt_l0_base = (uint64_t *)gpt_config.plat_gpt_l0_base;
987 gpt_l0_desc = gpt_l0_base[GPT_L0_IDX(base)];
988 if (GPT_L0_TYPE(gpt_l0_desc) != GPT_L0_TYPE_TBL_DESC) {
989 VERBOSE("GPT: Granule is not covered by a table descriptor!\n");
990 VERBOSE(" Base=0x%"PRIx64"\n", base);
991 return -EINVAL;
992 }
993
994 /* Get the table index and GPI shift from PA */
995 gpi_info->gpt_l1_addr = GPT_L0_TBLD_ADDR(gpt_l0_desc);
996 gpi_info->idx = GPT_L1_IDX(gpt_config.p, base);
997 gpi_info->gpi_shift = GPT_L1_GPI_IDX(gpt_config.p, base) << 2;
998
999 gpi_info->gpt_l1_desc = (gpi_info->gpt_l1_addr)[gpi_info->idx];
1000 gpi_info->gpi = (gpi_info->gpt_l1_desc >> gpi_info->gpi_shift) &
1001 GPT_L1_GRAN_DESC_GPI_MASK;
1002 return 0;
1003 }
1004
1005 /*
1006 * This function is the granule transition delegate service. When a granule
1007 * transition request occurs it is routed to this function to have the request,
1008 * if valid, fulfilled following A1.1.1 Delegate of RME supplement
1009 *
1010 * TODO: implement support for transitioning multiple granules at once.
1011 *
1012 * Parameters
1013 * base Base address of the region to transition, must be
1014 * aligned to granule size.
1015 * size Size of region to transition, must be aligned to granule
1016 * size.
1017 * src_sec_state Security state of the caller.
1018 *
1019 * Return
1020 * Negative Linux error code in the event of a failure, 0 for success.
1021 */
gpt_delegate_pas(uint64_t base,size_t size,unsigned int src_sec_state)1022 int gpt_delegate_pas(uint64_t base, size_t size, unsigned int src_sec_state)
1023 {
1024 gpi_info_t gpi_info;
1025 uint64_t nse;
1026 int res;
1027 unsigned int target_pas;
1028
1029 /* Ensure that the tables have been set up before taking requests */
1030 assert(gpt_config.plat_gpt_l0_base != 0UL);
1031
1032 /* Ensure that caches are enabled */
1033 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0UL);
1034
1035 /* Delegate request can only come from REALM or SECURE */
1036 assert(src_sec_state == SMC_FROM_REALM ||
1037 src_sec_state == SMC_FROM_SECURE);
1038
1039 /* See if this is a single or a range of granule transition */
1040 if (size != GPT_PGS_ACTUAL_SIZE(gpt_config.p)) {
1041 return -EINVAL;
1042 }
1043
1044 /* Check that base and size are valid */
1045 if ((ULONG_MAX - base) < size) {
1046 VERBOSE("GPT: Transition request address overflow!\n");
1047 VERBOSE(" Base=0x%"PRIx64"\n", base);
1048 VERBOSE(" Size=0x%lx\n", size);
1049 return -EINVAL;
1050 }
1051
1052 /* Make sure base and size are valid */
1053 if (((base & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) != 0UL) ||
1054 ((size & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) != 0UL) ||
1055 (size == 0UL) ||
1056 ((base + size) >= GPT_PPS_ACTUAL_SIZE(gpt_config.t))) {
1057 VERBOSE("GPT: Invalid granule transition address range!\n");
1058 VERBOSE(" Base=0x%"PRIx64"\n", base);
1059 VERBOSE(" Size=0x%lx\n", size);
1060 return -EINVAL;
1061 }
1062
1063 target_pas = GPT_GPI_REALM;
1064 if (src_sec_state == SMC_FROM_SECURE) {
1065 target_pas = GPT_GPI_SECURE;
1066 }
1067
1068 /*
1069 * Access to L1 tables is controlled by a global lock to ensure
1070 * that no more than one CPU is allowed to make changes at any
1071 * given time.
1072 */
1073 spin_lock(&gpt_lock);
1074 res = get_gpi_params(base, &gpi_info);
1075 if (res != 0) {
1076 spin_unlock(&gpt_lock);
1077 return res;
1078 }
1079
1080 /* Check that the current address is in NS state */
1081 if (gpi_info.gpi != GPT_GPI_NS) {
1082 VERBOSE("GPT: Only Granule in NS state can be delegated.\n");
1083 VERBOSE(" Caller: %u, Current GPI: %u\n", src_sec_state,
1084 gpi_info.gpi);
1085 spin_unlock(&gpt_lock);
1086 return -EPERM;
1087 }
1088
1089 if (src_sec_state == SMC_FROM_SECURE) {
1090 nse = (uint64_t)GPT_NSE_SECURE << GPT_NSE_SHIFT;
1091 } else {
1092 nse = (uint64_t)GPT_NSE_REALM << GPT_NSE_SHIFT;
1093 }
1094
1095 /*
1096 * In order to maintain mutual distrust between Realm and Secure
1097 * states, remove any data speculatively fetched into the target
1098 * physical address space. Issue DC CIPAPA over address range.
1099 */
1100 if (is_feat_mte2_supported()) {
1101 flush_dcache_to_popa_range_mte2(nse | base,
1102 GPT_PGS_ACTUAL_SIZE(gpt_config.p));
1103 } else {
1104 flush_dcache_to_popa_range(nse | base,
1105 GPT_PGS_ACTUAL_SIZE(gpt_config.p));
1106 }
1107
1108 write_gpt(&gpi_info.gpt_l1_desc, gpi_info.gpt_l1_addr,
1109 gpi_info.gpi_shift, gpi_info.idx, target_pas);
1110 dsboshst();
1111
1112 gpt_tlbi_by_pa_ll(base, GPT_PGS_ACTUAL_SIZE(gpt_config.p));
1113 dsbosh();
1114
1115 nse = (uint64_t)GPT_NSE_NS << GPT_NSE_SHIFT;
1116
1117 if (is_feat_mte2_supported()) {
1118 flush_dcache_to_popa_range_mte2(nse | base,
1119 GPT_PGS_ACTUAL_SIZE(gpt_config.p));
1120 } else {
1121 flush_dcache_to_popa_range(nse | base,
1122 GPT_PGS_ACTUAL_SIZE(gpt_config.p));
1123 }
1124
1125 /* Unlock access to the L1 tables */
1126 spin_unlock(&gpt_lock);
1127
1128 /*
1129 * The isb() will be done as part of context
1130 * synchronization when returning to lower EL.
1131 */
1132 VERBOSE("GPT: Granule 0x%"PRIx64" GPI 0x%x->0x%x\n",
1133 base, gpi_info.gpi, target_pas);
1134
1135 return 0;
1136 }
1137
1138 /*
1139 * This function is the granule transition undelegate service. When a granule
1140 * transition request occurs it is routed to this function where the request is
1141 * validated then fulfilled if possible.
1142 *
1143 * TODO: implement support for transitioning multiple granules at once.
1144 *
1145 * Parameters
1146 * base Base address of the region to transition, must be
1147 * aligned to granule size.
1148 * size Size of region to transition, must be aligned to granule
1149 * size.
1150 * src_sec_state Security state of the caller.
1151 *
1152 * Return
1153 * Negative Linux error code in the event of a failure, 0 for success.
1154 */
gpt_undelegate_pas(uint64_t base,size_t size,unsigned int src_sec_state)1155 int gpt_undelegate_pas(uint64_t base, size_t size, unsigned int src_sec_state)
1156 {
1157 gpi_info_t gpi_info;
1158 uint64_t nse;
1159 int res;
1160
1161 /* Ensure that the tables have been set up before taking requests */
1162 assert(gpt_config.plat_gpt_l0_base != 0UL);
1163
1164 /* Ensure that MMU and caches are enabled */
1165 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0UL);
1166
1167 /* Delegate request can only come from REALM or SECURE */
1168 assert(src_sec_state == SMC_FROM_REALM ||
1169 src_sec_state == SMC_FROM_SECURE);
1170
1171 /* See if this is a single or a range of granule transition */
1172 if (size != GPT_PGS_ACTUAL_SIZE(gpt_config.p)) {
1173 return -EINVAL;
1174 }
1175
1176 /* Check that base and size are valid */
1177 if ((ULONG_MAX - base) < size) {
1178 VERBOSE("GPT: Transition request address overflow!\n");
1179 VERBOSE(" Base=0x%"PRIx64"\n", base);
1180 VERBOSE(" Size=0x%lx\n", size);
1181 return -EINVAL;
1182 }
1183
1184 /* Make sure base and size are valid */
1185 if (((base & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) != 0UL) ||
1186 ((size & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) != 0UL) ||
1187 (size == 0UL) ||
1188 ((base + size) >= GPT_PPS_ACTUAL_SIZE(gpt_config.t))) {
1189 VERBOSE("GPT: Invalid granule transition address range!\n");
1190 VERBOSE(" Base=0x%"PRIx64"\n", base);
1191 VERBOSE(" Size=0x%lx\n", size);
1192 return -EINVAL;
1193 }
1194
1195 /*
1196 * Access to L1 tables is controlled by a global lock to ensure
1197 * that no more than one CPU is allowed to make changes at any
1198 * given time.
1199 */
1200 spin_lock(&gpt_lock);
1201
1202 res = get_gpi_params(base, &gpi_info);
1203 if (res != 0) {
1204 spin_unlock(&gpt_lock);
1205 return res;
1206 }
1207
1208 /* Check that the current address is in the delegated state */
1209 if ((src_sec_state == SMC_FROM_REALM &&
1210 gpi_info.gpi != GPT_GPI_REALM) ||
1211 (src_sec_state == SMC_FROM_SECURE &&
1212 gpi_info.gpi != GPT_GPI_SECURE)) {
1213 VERBOSE("GPT: Only Granule in REALM or SECURE state can be undelegated.\n");
1214 VERBOSE(" Caller: %u Current GPI: %u\n", src_sec_state,
1215 gpi_info.gpi);
1216 spin_unlock(&gpt_lock);
1217 return -EPERM;
1218 }
1219
1220
1221 /* In order to maintain mutual distrust between Realm and Secure
1222 * states, remove access now, in order to guarantee that writes
1223 * to the currently-accessible physical address space will not
1224 * later become observable.
1225 */
1226 write_gpt(&gpi_info.gpt_l1_desc, gpi_info.gpt_l1_addr,
1227 gpi_info.gpi_shift, gpi_info.idx, GPT_GPI_NO_ACCESS);
1228 dsboshst();
1229
1230 gpt_tlbi_by_pa_ll(base, GPT_PGS_ACTUAL_SIZE(gpt_config.p));
1231 dsbosh();
1232
1233 if (src_sec_state == SMC_FROM_SECURE) {
1234 nse = (uint64_t)GPT_NSE_SECURE << GPT_NSE_SHIFT;
1235 } else {
1236 nse = (uint64_t)GPT_NSE_REALM << GPT_NSE_SHIFT;
1237 }
1238
1239 /* Ensure that the scrubbed data has made it past the PoPA */
1240 if (is_feat_mte2_supported()) {
1241 flush_dcache_to_popa_range_mte2(nse | base,
1242 GPT_PGS_ACTUAL_SIZE(gpt_config.p));
1243 } else {
1244 flush_dcache_to_popa_range(nse | base,
1245 GPT_PGS_ACTUAL_SIZE(gpt_config.p));
1246 }
1247
1248 /*
1249 * Remove any data loaded speculatively
1250 * in NS space from before the scrubbing
1251 */
1252 nse = (uint64_t)GPT_NSE_NS << GPT_NSE_SHIFT;
1253
1254 if (is_feat_mte2_supported()) {
1255 flush_dcache_to_popa_range_mte2(nse | base,
1256 GPT_PGS_ACTUAL_SIZE(gpt_config.p));
1257 } else {
1258 flush_dcache_to_popa_range(nse | base,
1259 GPT_PGS_ACTUAL_SIZE(gpt_config.p));
1260 }
1261
1262 /* Clear existing GPI encoding and transition granule. */
1263 write_gpt(&gpi_info.gpt_l1_desc, gpi_info.gpt_l1_addr,
1264 gpi_info.gpi_shift, gpi_info.idx, GPT_GPI_NS);
1265 dsboshst();
1266
1267 /* Ensure that all agents observe the new NS configuration */
1268 gpt_tlbi_by_pa_ll(base, GPT_PGS_ACTUAL_SIZE(gpt_config.p));
1269 dsbosh();
1270
1271 /* Unlock access to the L1 tables. */
1272 spin_unlock(&gpt_lock);
1273
1274 /*
1275 * The isb() will be done as part of context
1276 * synchronization when returning to lower EL.
1277 */
1278 VERBOSE("GPT: Granule 0x%"PRIx64" GPI 0x%x->0x%x\n",
1279 base, gpi_info.gpi, GPT_GPI_NS);
1280
1281 return 0;
1282 }
1283