Lines Matching +full:guest +full:- +full:index +full:- +full:bits
1 // SPDX-License-Identifier: GPL-2.0
3 * KVM guest address space mapping code
21 #include <asm/page-states.h>
30 * The lower bits are ignored by all users of the macro, so it can be used
51 * gmap_alloc - allocate and initialize a guest address space
54 * Returns a guest address space structure.
64 limit = _REGION3_SIZE - 1; in gmap_alloc()
68 limit = _REGION2_SIZE - 1; in gmap_alloc()
72 limit = _REGION1_SIZE - 1; in gmap_alloc()
76 limit = -1UL; in gmap_alloc()
83 INIT_LIST_HEAD(&gmap->children); in gmap_alloc()
84 INIT_RADIX_TREE(&gmap->guest_to_host, GFP_KERNEL_ACCOUNT); in gmap_alloc()
85 INIT_RADIX_TREE(&gmap->host_to_guest, GFP_ATOMIC | __GFP_ACCOUNT); in gmap_alloc()
86 INIT_RADIX_TREE(&gmap->host_to_rmap, GFP_ATOMIC | __GFP_ACCOUNT); in gmap_alloc()
87 spin_lock_init(&gmap->guest_table_lock); in gmap_alloc()
88 spin_lock_init(&gmap->shadow_lock); in gmap_alloc()
89 refcount_set(&gmap->ref_count, 1); in gmap_alloc()
95 gmap->table = table; in gmap_alloc()
96 gmap->asce = atype | _ASCE_TABLE_LENGTH | in gmap_alloc()
98 gmap->asce_end = limit; in gmap_alloc()
109 * gmap_create - create a guest address space
113 * Returns a guest address space structure.
123 gmap->mm = mm; in gmap_create()
124 spin_lock(&mm->context.lock); in gmap_create()
125 list_add_rcu(&gmap->list, &mm->context.gmap_list); in gmap_create()
126 if (list_is_singular(&mm->context.gmap_list)) in gmap_create()
127 gmap_asce = gmap->asce; in gmap_create()
129 gmap_asce = -1UL; in gmap_create()
130 WRITE_ONCE(mm->context.gmap_asce, gmap_asce); in gmap_create()
131 spin_unlock(&mm->context.lock); in gmap_create()
139 __tlb_flush_idte(gmap->asce); in gmap_flush_tlb()
148 unsigned long index; in gmap_radix_tree_free() local
153 index = 0; in gmap_radix_tree_free()
156 radix_tree_for_each_slot(slot, root, &iter, index) { in gmap_radix_tree_free()
157 indices[nr] = iter.index; in gmap_radix_tree_free()
162 index = indices[i]; in gmap_radix_tree_free()
163 radix_tree_delete(root, index); in gmap_radix_tree_free()
173 unsigned long index; in gmap_rmap_radix_tree_free() local
178 index = 0; in gmap_rmap_radix_tree_free()
181 radix_tree_for_each_slot(slot, root, &iter, index) { in gmap_rmap_radix_tree_free()
182 indices[nr] = iter.index; in gmap_rmap_radix_tree_free()
187 index = indices[i]; in gmap_rmap_radix_tree_free()
188 head = radix_tree_delete(root, index); in gmap_rmap_radix_tree_free()
217 * gmap_free - free a guest address space
218 * @gmap: pointer to the guest address space structure
225 if (!(gmap_is_shadow(gmap) && gmap->removed)) in gmap_free()
228 gmap_free_crst(gmap->table, gmap_is_shadow(gmap)); in gmap_free()
230 gmap_radix_tree_free(&gmap->guest_to_host); in gmap_free()
231 gmap_radix_tree_free(&gmap->host_to_guest); in gmap_free()
235 gmap_rmap_radix_tree_free(&gmap->host_to_rmap); in gmap_free()
237 gmap_put(gmap->parent); in gmap_free()
245 * gmap_get - increase reference counter for guest address space
246 * @gmap: pointer to the guest address space structure
252 refcount_inc(&gmap->ref_count); in gmap_get()
258 * gmap_put - decrease reference counter for guest address space
259 * @gmap: pointer to the guest address space structure
261 * If the reference counter reaches zero the guest address space is freed.
265 if (refcount_dec_and_test(&gmap->ref_count)) in gmap_put()
271 * gmap_remove - remove a guest address space but do not free it yet
272 * @gmap: pointer to the guest address space structure
280 if (!list_empty(&gmap->children)) { in gmap_remove()
281 spin_lock(&gmap->shadow_lock); in gmap_remove()
282 list_for_each_entry_safe(sg, next, &gmap->children, list) { in gmap_remove()
283 list_del(&sg->list); in gmap_remove()
286 spin_unlock(&gmap->shadow_lock); in gmap_remove()
288 /* Remove gmap from the pre-mm list */ in gmap_remove()
289 spin_lock(&gmap->mm->context.lock); in gmap_remove()
290 list_del_rcu(&gmap->list); in gmap_remove()
291 if (list_empty(&gmap->mm->context.gmap_list)) in gmap_remove()
293 else if (list_is_singular(&gmap->mm->context.gmap_list)) in gmap_remove()
294 gmap_asce = list_first_entry(&gmap->mm->context.gmap_list, in gmap_remove()
295 struct gmap, list)->asce; in gmap_remove()
297 gmap_asce = -1UL; in gmap_remove()
298 WRITE_ONCE(gmap->mm->context.gmap_asce, gmap_asce); in gmap_remove()
299 spin_unlock(&gmap->mm->context.lock); in gmap_remove()
318 return -ENOMEM; in gmap_alloc_table()
321 spin_lock(&gmap->guest_table_lock); in gmap_alloc_table()
327 spin_unlock(&gmap->guest_table_lock); in gmap_alloc_table()
335 return (unsigned long)radix_tree_lookup(&gmap->host_to_guest, vmaddr >> PMD_SHIFT); in host_to_guest_lookup()
340 return (unsigned long)radix_tree_delete(&gmap->host_to_guest, vmaddr >> PMD_SHIFT); in host_to_guest_delete()
353 * __gmap_unlink_by_vmaddr - unlink a single segment via a host address
354 * @gmap: pointer to the guest address space structure
366 spin_lock(&gmap->guest_table_lock); in __gmap_unlink_by_vmaddr()
374 spin_unlock(&gmap->guest_table_lock); in __gmap_unlink_by_vmaddr()
379 * __gmap_unmap_by_gaddr - unmap a single segment via a guest address
380 * @gmap: pointer to the guest address space structure
381 * @gaddr: address in the guest address space
389 vmaddr = (unsigned long) radix_tree_delete(&gmap->guest_to_host, in __gmap_unmap_by_gaddr()
395 * gmap_unmap_segment - unmap segment from the guest address space
396 * @gmap: pointer to the guest address space structure
397 * @to: address in the guest address space
400 * Returns 0 if the unmap succeeded, -EINVAL if not.
408 if ((to | len) & (PMD_SIZE - 1)) in gmap_unmap_segment()
409 return -EINVAL; in gmap_unmap_segment()
411 return -EINVAL; in gmap_unmap_segment()
414 mmap_write_lock(gmap->mm); in gmap_unmap_segment()
417 mmap_write_unlock(gmap->mm); in gmap_unmap_segment()
425 * gmap_map_segment - map a segment to the guest address space
426 * @gmap: pointer to the guest address space structure
428 * @to: target address in the guest address space
431 * Returns 0 if the mmap succeeded, -EINVAL or -ENOMEM if not.
440 if ((from | to | len) & (PMD_SIZE - 1)) in gmap_map_segment()
441 return -EINVAL; in gmap_map_segment()
443 from + len - 1 > TASK_SIZE_MAX || to + len - 1 > gmap->asce_end) in gmap_map_segment()
444 return -EINVAL; in gmap_map_segment()
447 mmap_write_lock(gmap->mm); in gmap_map_segment()
452 if (radix_tree_insert(&gmap->guest_to_host, in gmap_map_segment()
457 mmap_write_unlock(gmap->mm); in gmap_map_segment()
463 return -ENOMEM; in gmap_map_segment()
468 * __gmap_translate - translate a guest address to a user space address
469 * @gmap: pointer to guest mapping meta data structure
470 * @gaddr: guest address
472 * Returns user space address which corresponds to the guest address or
473 * -EFAULT if no such mapping exists.
485 radix_tree_lookup(&gmap->guest_to_host, gaddr >> PMD_SHIFT); in __gmap_translate()
487 return vmaddr ? (vmaddr | (gaddr & ~PMD_MASK)) : -EFAULT; in __gmap_translate()
492 * gmap_unlink - disconnect a page table from the gmap shadow tables
504 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in gmap_unlink()
516 * __gmap_link - set up shadow page tables to connect a host to a guest address
517 * @gmap: pointer to guest mapping meta data structure
518 * @gaddr: guest address
521 * Returns 0 on success, -ENOMEM for out of memory conditions, and -EFAULT
522 * if the vm address is already mapped to a different guest segment.
540 table = gmap->table; in __gmap_link()
541 if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION1) { in __gmap_link()
546 return -ENOMEM; in __gmap_link()
549 if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION2) { in __gmap_link()
554 return -ENOMEM; in __gmap_link()
557 if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION3) { in __gmap_link()
562 return -ENOMEM; in __gmap_link()
567 mm = gmap->mm; in __gmap_link()
576 return -EFAULT; in __gmap_link()
580 if (pmd_leaf(*pmd) && !gmap->mm->context.allow_gmap_hpage_1m) in __gmap_link()
581 return -EFAULT; in __gmap_link()
587 spin_lock(&gmap->guest_table_lock); in __gmap_link()
589 rc = radix_tree_insert(&gmap->host_to_guest, in __gmap_link()
609 spin_unlock(&gmap->guest_table_lock); in __gmap_link()
626 /* Find the vm address for the guest address */ in __gmap_zap()
627 vmaddr = (unsigned long) radix_tree_lookup(&gmap->guest_to_host, in __gmap_zap()
632 vma = vma_lookup(gmap->mm, vmaddr); in __gmap_zap()
637 ptep = get_locked_pte(gmap->mm, vmaddr, &ptl); in __gmap_zap()
639 ptep_zap_unused(gmap->mm, vmaddr, ptep, 0); in __gmap_zap()
651 mmap_read_lock(gmap->mm); in gmap_discard()
654 /* Find the vm address for the guest address */ in gmap_discard()
656 radix_tree_lookup(&gmap->guest_to_host, in gmap_discard()
662 vma = find_vma(gmap->mm, vmaddr); in gmap_discard()
671 size = min(to - gaddr, PMD_SIZE - (gaddr & ~PMD_MASK)); in gmap_discard()
674 mmap_read_unlock(gmap->mm); in gmap_discard()
682 * gmap_register_pte_notifier - register a pte invalidation callback
688 list_add_rcu(&nb->list, &gmap_notifier_list); in gmap_register_pte_notifier()
694 * gmap_unregister_pte_notifier - remove a pte invalidation callback
700 list_del_rcu(&nb->list); in gmap_unregister_pte_notifier()
707 * gmap_call_notifier - call all registered invalidation callbacks
708 * @gmap: pointer to guest mapping meta data structure
709 * @start: start virtual address in the guest address space
710 * @end: end virtual address in the guest address space
718 nb->notifier_call(gmap, start, end); in gmap_call_notifier()
722 * gmap_table_walk - walk the gmap page tables
723 * @gmap: pointer to guest mapping meta data structure
724 * @gaddr: virtual address in the guest address space
727 * Returns a table entry pointer for the given guest address and @level
730 * @level=2 : returns a pointer to a region-3 table entry (or NULL)
731 * @level=3 : returns a pointer to a region-2 table entry (or NULL)
732 * @level=4 : returns a pointer to a region-1 table entry (or NULL)
741 const int asce_type = gmap->asce & _ASCE_TYPE_MASK; in gmap_table_walk()
742 unsigned long *table = gmap->table; in gmap_table_walk()
744 if (gmap_is_shadow(gmap) && gmap->removed) in gmap_table_walk()
751 gaddr & (-1UL << (31 + (asce_type >> 2) * 11))) in gmap_table_walk()
793 * gmap_pte_op_walk - walk the gmap page table, get the page table lock
795 * @gmap: pointer to guest mapping meta data structure
796 * @gaddr: virtual address in the guest address space
799 * Returns a pointer to the locked pte for a guest address, or NULL
811 return pte_alloc_map_lock(gmap->mm, (pmd_t *) table, gaddr, ptl); in gmap_pte_op_walk()
815 * gmap_pte_op_fixup - force a page in and connect the gmap page table
816 * @gmap: pointer to guest mapping meta data structure
817 * @gaddr: virtual address in the guest address space
822 * -ENOMEM if out of memory and -EFAULT if anything goes wrong while fixing
828 struct mm_struct *mm = gmap->mm; in gmap_pte_op_fixup()
835 return -EFAULT; in gmap_pte_op_fixup()
844 * gmap_pte_op_end - release the page table lock
854 * gmap_pmd_op_walk - walk the gmap tables, get the guest table lock
856 * @gmap: pointer to guest mapping meta data structure
857 * @gaddr: virtual address in the guest address space
859 * Returns a pointer to the pmd for a guest address, or NULL
871 if (!gmap->mm->context.allow_gmap_hpage_1m) in gmap_pmd_op_walk()
874 spin_lock(&gmap->guest_table_lock); in gmap_pmd_op_walk()
876 spin_unlock(&gmap->guest_table_lock); in gmap_pmd_op_walk()
882 spin_unlock(&gmap->guest_table_lock); in gmap_pmd_op_walk()
887 * gmap_pmd_op_end - release the guest_table_lock if needed
888 * @gmap: pointer to the guest mapping meta data structure
894 spin_unlock(&gmap->guest_table_lock); in gmap_pmd_op_end()
898 * gmap_protect_pmd - remove access rights to memory and set pmd notification bits
901 * @bits: notification bits to set
905 * -EAGAIN if a fixup is needed
906 * -EINVAL if unsupported notifier bits have been specified
908 * Expected to be called with sg->mm->mmap_lock in read and
912 pmd_t *pmdp, int prot, unsigned long bits) in gmap_protect_pmd() argument
920 return -EAGAIN; in gmap_protect_pmd()
933 if (bits & GMAP_NOTIFY_MPROT) in gmap_protect_pmd()
937 if (bits & GMAP_NOTIFY_SHADOW) in gmap_protect_pmd()
938 return -EINVAL; in gmap_protect_pmd()
944 * gmap_protect_pte - remove access rights to memory and set pgste bits
945 * @gmap: pointer to guest mapping meta data structure
946 * @gaddr: virtual address in the guest address space
949 * @bits: notification bits to set
951 * Returns 0 if successfully protected, -ENOMEM if out of memory and
952 * -EAGAIN if a fixup is needed.
954 * Expected to be called with sg->mm->mmap_lock in read
957 pmd_t *pmdp, int prot, unsigned long bits) in gmap_protect_pte() argument
965 return -EAGAIN; in gmap_protect_pte()
967 ptep = pte_alloc_map_lock(gmap->mm, pmdp, gaddr, &ptl); in gmap_protect_pte()
969 return -ENOMEM; in gmap_protect_pte()
971 pbits |= (bits & GMAP_NOTIFY_MPROT) ? PGSTE_IN_BIT : 0; in gmap_protect_pte()
972 pbits |= (bits & GMAP_NOTIFY_SHADOW) ? PGSTE_VSIE_BIT : 0; in gmap_protect_pte()
974 rc = ptep_force_prot(gmap->mm, gaddr, ptep, prot, pbits); in gmap_protect_pte()
980 * gmap_protect_range - remove access rights to memory and set pgste bits
981 * @gmap: pointer to guest mapping meta data structure
982 * @gaddr: virtual address in the guest address space
985 * @bits: pgste notification bits to set
990 * -ENOMEM if out of memory;
991 * -EFAULT if gaddr is invalid (or mapping for shadows is missing);
992 * -EAGAIN if the guest mapping is missing and should be fixed by the caller.
994 * Context: Called with sg->mm->mmap_lock in read.
996 int gmap_protect_one(struct gmap *gmap, unsigned long gaddr, int prot, unsigned long bits) in gmap_protect_one() argument
1005 return -EAGAIN; in gmap_protect_one()
1008 rc = gmap_protect_pte(gmap, gaddr, pmdp, prot, bits); in gmap_protect_one()
1012 rc = gmap_protect_pmd(gmap, gaddr, pmdp, prot, bits); in gmap_protect_one()
1023 * gmap_read_table - get an unsigned long value from a guest page table using
1025 * @gmap: pointer to guest mapping meta data structure
1026 * @gaddr: virtual address in the guest address space
1029 * Returns 0 if the value was read, -ENOMEM if out of memory and -EFAULT
1030 * if reading using the virtual address failed. -EINVAL if called on a gmap
1033 * Called with gmap->mm->mmap_lock in read.
1043 return -EINVAL; in gmap_read_table()
1046 rc = -EAGAIN; in gmap_read_table()
1076 * gmap_insert_rmap - add a rmap to the host_to_rmap radix tree
1077 * @sg: pointer to the shadow guest address space structure
1081 * Called with the sg->guest_table_lock
1090 slot = radix_tree_lookup_slot(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT); in gmap_insert_rmap()
1092 rmap->next = radix_tree_deref_slot_protected(slot, in gmap_insert_rmap()
1093 &sg->guest_table_lock); in gmap_insert_rmap()
1094 for (temp = rmap->next; temp; temp = temp->next) { in gmap_insert_rmap()
1095 if (temp->raddr == rmap->raddr) { in gmap_insert_rmap()
1100 radix_tree_replace_slot(&sg->host_to_rmap, slot, rmap); in gmap_insert_rmap()
1102 rmap->next = NULL; in gmap_insert_rmap()
1103 radix_tree_insert(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT, in gmap_insert_rmap()
1109 * gmap_protect_rmap - restrict access rights to memory (RO) and create an rmap
1110 * @sg: pointer to the shadow guest address space structure
1112 * @paddr: address in the parent guest address space
1115 * Returns 0 if successfully protected and the rmap was created, -ENOMEM
1116 * if out of memory and -EFAULT if paddr is invalid.
1129 parent = sg->parent; in gmap_protect_rmap()
1136 return -ENOMEM; in gmap_protect_rmap()
1137 rmap->raddr = raddr; in gmap_protect_rmap()
1143 rc = -EAGAIN; in gmap_protect_rmap()
1146 spin_lock(&sg->guest_table_lock); in gmap_protect_rmap()
1147 rc = ptep_force_prot(parent->mm, paddr, ptep, PROT_READ, in gmap_protect_rmap()
1151 spin_unlock(&sg->guest_table_lock); in gmap_protect_rmap()
1163 len -= PAGE_SIZE; in gmap_protect_rmap()
1176 * gmap_idte_one - invalidate a single region or segment table entry
1177 * @asce: region or segment table *origin* + table-type bits
1182 * The table-type of the @asce identifies the portion of the @vaddr
1183 * that is used as the invalidation index.
1193 * gmap_unshadow_page - remove a page from a shadow page table
1194 * @sg: pointer to the shadow guest address space structure
1195 * @raddr: rmap address in the shadow guest address space
1197 * Called with the sg->guest_table_lock
1207 gmap_call_notifier(sg, raddr, raddr + PAGE_SIZE - 1); in gmap_unshadow_page()
1208 ptep_unshadow_pte(sg->mm, raddr, (pte_t *) table); in gmap_unshadow_page()
1212 * __gmap_unshadow_pgt - remove all entries from a shadow page table
1213 * @sg: pointer to the shadow guest address space structure
1214 * @raddr: rmap address in the shadow guest address space
1217 * Called with the sg->guest_table_lock
1230 * gmap_unshadow_pgt - remove a shadow page table from a segment entry
1231 * @sg: pointer to the shadow guest address space structure
1232 * @raddr: address in the shadow guest address space
1234 * Called with the sg->guest_table_lock
1246 gmap_call_notifier(sg, raddr, raddr + _SEGMENT_SIZE - 1); in gmap_unshadow_pgt()
1247 sto = __pa(ste - ((raddr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT)); in gmap_unshadow_pgt()
1258 * __gmap_unshadow_sgt - remove all entries from a shadow segment table
1259 * @sg: pointer to the shadow guest address space structure
1260 * @raddr: rmap address in the shadow guest address space
1263 * Called with the sg->guest_table_lock
1286 * gmap_unshadow_sgt - remove a shadow segment table from a region-3 entry
1287 * @sg: pointer to the shadow guest address space structure
1288 * @raddr: rmap address in the shadow guest address space
1290 * Called with the shadow->guest_table_lock
1299 r3e = gmap_table_walk(sg, raddr, 2); /* get region-3 pointer */ in gmap_unshadow_sgt()
1302 gmap_call_notifier(sg, raddr, raddr + _REGION3_SIZE - 1); in gmap_unshadow_sgt()
1303 r3o = (unsigned long) (r3e - ((raddr & _REGION3_INDEX) >> _REGION3_SHIFT)); in gmap_unshadow_sgt()
1314 * __gmap_unshadow_r3t - remove all entries from a shadow region-3 table
1315 * @sg: pointer to the shadow guest address space structure
1316 * @raddr: address in the shadow guest address space
1317 * @r3t: pointer to the start of a shadow region-3 table
1319 * Called with the sg->guest_table_lock
1342 * gmap_unshadow_r3t - remove a shadow region-3 table from a region-2 entry
1343 * @sg: pointer to the shadow guest address space structure
1344 * @raddr: rmap address in the shadow guest address space
1346 * Called with the sg->guest_table_lock
1355 r2e = gmap_table_walk(sg, raddr, 3); /* get region-2 pointer */ in gmap_unshadow_r3t()
1358 gmap_call_notifier(sg, raddr, raddr + _REGION2_SIZE - 1); in gmap_unshadow_r3t()
1359 r2o = (unsigned long) (r2e - ((raddr & _REGION2_INDEX) >> _REGION2_SHIFT)); in gmap_unshadow_r3t()
1370 * __gmap_unshadow_r2t - remove all entries from a shadow region-2 table
1371 * @sg: pointer to the shadow guest address space structure
1372 * @raddr: rmap address in the shadow guest address space
1373 * @r2t: pointer to the start of a shadow region-2 table
1375 * Called with the sg->guest_table_lock
1398 * gmap_unshadow_r2t - remove a shadow region-2 table from a region-1 entry
1399 * @sg: pointer to the shadow guest address space structure
1400 * @raddr: rmap address in the shadow guest address space
1402 * Called with the sg->guest_table_lock
1411 r1e = gmap_table_walk(sg, raddr, 4); /* get region-1 pointer */ in gmap_unshadow_r2t()
1414 gmap_call_notifier(sg, raddr, raddr + _REGION1_SIZE - 1); in gmap_unshadow_r2t()
1415 r1o = (unsigned long) (r1e - ((raddr & _REGION1_INDEX) >> _REGION1_SHIFT)); in gmap_unshadow_r2t()
1426 * __gmap_unshadow_r1t - remove all entries from a shadow region-1 table
1427 * @sg: pointer to the shadow guest address space structure
1428 * @raddr: rmap address in the shadow guest address space
1429 * @r1t: pointer to the start of a shadow region-1 table
1431 * Called with the shadow->guest_table_lock
1448 /* Clear entry and flush translation r1t -> r2t */ in __gmap_unshadow_r1t()
1458 * gmap_unshadow - remove a shadow page table completely
1459 * @sg: pointer to the shadow guest address space structure
1461 * Called with sg->guest_table_lock
1468 if (sg->removed) in gmap_unshadow()
1470 sg->removed = 1; in gmap_unshadow()
1471 gmap_call_notifier(sg, 0, -1UL); in gmap_unshadow()
1473 table = __va(sg->asce & _ASCE_ORIGIN); in gmap_unshadow()
1474 switch (sg->asce & _ASCE_TYPE_MASK) { in gmap_unshadow()
1492 * gmap_shadow_r2t - create an empty shadow region 2 table
1493 * @sg: pointer to the shadow guest address space structure
1496 * @fake: r2t references contiguous guest memory block, not a r2t
1499 * four pages of the source table are made read-only in the parent gmap
1503 * Returns 0 if successfully shadowed or already shadowed, -EAGAIN if the
1504 * shadow table structure is incomplete, -ENOMEM if out of memory and
1505 * -EFAULT if an address in the parent gmap could not be resolved.
1507 * Called with sg->mm->mmap_lock in read.
1522 return -ENOMEM; in gmap_shadow_r2t()
1525 spin_lock(&sg->guest_table_lock); in gmap_shadow_r2t()
1526 table = gmap_table_walk(sg, saddr, 4); /* get region-1 pointer */ in gmap_shadow_r2t()
1528 rc = -EAGAIN; /* Race with unshadow */ in gmap_shadow_r2t()
1535 rc = -EAGAIN; /* Race with shadow */ in gmap_shadow_r2t()
1542 if (sg->edat_level >= 1) in gmap_shadow_r2t()
1547 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r2t()
1550 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r2t()
1551 /* Make r2t read-only in parent gmap page table */ in gmap_shadow_r2t()
1555 len = ((r2t & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset; in gmap_shadow_r2t()
1557 spin_lock(&sg->guest_table_lock); in gmap_shadow_r2t()
1561 rc = -EAGAIN; /* Race with unshadow */ in gmap_shadow_r2t()
1567 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r2t()
1570 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r2t()
1577 * gmap_shadow_r3t - create a shadow region 3 table
1578 * @sg: pointer to the shadow guest address space structure
1581 * @fake: r3t references contiguous guest memory block, not a r3t
1583 * Returns 0 if successfully shadowed or already shadowed, -EAGAIN if the
1584 * shadow table structure is incomplete, -ENOMEM if out of memory and
1585 * -EFAULT if an address in the parent gmap could not be resolved.
1587 * Called with sg->mm->mmap_lock in read.
1602 return -ENOMEM; in gmap_shadow_r3t()
1605 spin_lock(&sg->guest_table_lock); in gmap_shadow_r3t()
1606 table = gmap_table_walk(sg, saddr, 3); /* get region-2 pointer */ in gmap_shadow_r3t()
1608 rc = -EAGAIN; /* Race with unshadow */ in gmap_shadow_r3t()
1615 rc = -EAGAIN; /* Race with shadow */ in gmap_shadow_r3t()
1622 if (sg->edat_level >= 1) in gmap_shadow_r3t()
1627 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r3t()
1630 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r3t()
1631 /* Make r3t read-only in parent gmap page table */ in gmap_shadow_r3t()
1635 len = ((r3t & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset; in gmap_shadow_r3t()
1637 spin_lock(&sg->guest_table_lock); in gmap_shadow_r3t()
1641 rc = -EAGAIN; /* Race with unshadow */ in gmap_shadow_r3t()
1647 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r3t()
1650 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r3t()
1657 * gmap_shadow_sgt - create a shadow segment table
1658 * @sg: pointer to the shadow guest address space structure
1661 * @fake: sgt references contiguous guest memory block, not a sgt
1663 * Returns: 0 if successfully shadowed or already shadowed, -EAGAIN if the
1664 * shadow table structure is incomplete, -ENOMEM if out of memory and
1665 * -EFAULT if an address in the parent gmap could not be resolved.
1667 * Called with sg->mm->mmap_lock in read.
1682 return -ENOMEM; in gmap_shadow_sgt()
1685 spin_lock(&sg->guest_table_lock); in gmap_shadow_sgt()
1686 table = gmap_table_walk(sg, saddr, 2); /* get region-3 pointer */ in gmap_shadow_sgt()
1688 rc = -EAGAIN; /* Race with unshadow */ in gmap_shadow_sgt()
1695 rc = -EAGAIN; /* Race with shadow */ in gmap_shadow_sgt()
1702 if (sg->edat_level >= 1) in gmap_shadow_sgt()
1707 spin_unlock(&sg->guest_table_lock); in gmap_shadow_sgt()
1710 spin_unlock(&sg->guest_table_lock); in gmap_shadow_sgt()
1711 /* Make sgt read-only in parent gmap page table */ in gmap_shadow_sgt()
1715 len = ((sgt & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset; in gmap_shadow_sgt()
1717 spin_lock(&sg->guest_table_lock); in gmap_shadow_sgt()
1721 rc = -EAGAIN; /* Race with unshadow */ in gmap_shadow_sgt()
1727 spin_unlock(&sg->guest_table_lock); in gmap_shadow_sgt()
1730 spin_unlock(&sg->guest_table_lock); in gmap_shadow_sgt()
1754 * gmap_shadow_pgt - instantiate a shadow page table
1755 * @sg: pointer to the shadow guest address space structure
1758 * @fake: pgt references contiguous guest memory block, not a pgtable
1760 * Returns 0 if successfully shadowed or already shadowed, -EAGAIN if the
1761 * shadow table structure is incomplete, -ENOMEM if out of memory,
1762 * -EFAULT if an address in the parent gmap could not be resolved and
1764 * Called with gmap->mm->mmap_lock in read
1777 ptdesc = page_table_alloc_pgste(sg->mm); in gmap_shadow_pgt()
1779 return -ENOMEM; in gmap_shadow_pgt()
1786 spin_lock(&sg->guest_table_lock); in gmap_shadow_pgt()
1789 rc = -EAGAIN; /* Race with unshadow */ in gmap_shadow_pgt()
1796 rc = -EAGAIN; /* Race with shadow */ in gmap_shadow_pgt()
1805 spin_unlock(&sg->guest_table_lock); in gmap_shadow_pgt()
1808 spin_unlock(&sg->guest_table_lock); in gmap_shadow_pgt()
1809 /* Make pgt read-only in parent gmap page table (not the pgste) */ in gmap_shadow_pgt()
1813 spin_lock(&sg->guest_table_lock); in gmap_shadow_pgt()
1817 rc = -EAGAIN; /* Race with unshadow */ in gmap_shadow_pgt()
1823 spin_unlock(&sg->guest_table_lock); in gmap_shadow_pgt()
1826 spin_unlock(&sg->guest_table_lock); in gmap_shadow_pgt()
1834 * gmap_shadow_page - create a shadow page mapping
1835 * @sg: pointer to the shadow guest address space structure
1839 * Returns 0 if successfully shadowed or already shadowed, -EAGAIN if the
1840 * shadow table structure is incomplete, -ENOMEM if out of memory and
1841 * -EFAULT if an address in the parent gmap could not be resolved.
1843 * Called with sg->mm->mmap_lock in read.
1856 parent = sg->parent; in gmap_shadow_page()
1861 return -ENOMEM; in gmap_shadow_page()
1862 rmap->raddr = (saddr & PAGE_MASK) | _SHADOW_RMAP_PGTABLE; in gmap_shadow_page()
1874 rc = -EAGAIN; in gmap_shadow_page()
1877 spin_lock(&sg->guest_table_lock); in gmap_shadow_page()
1881 spin_unlock(&sg->guest_table_lock); in gmap_shadow_page()
1886 rc = ptep_shadow_pte(sg->mm, saddr, sptep, tptep, pte); in gmap_shadow_page()
1894 spin_unlock(&sg->guest_table_lock); in gmap_shadow_page()
1909 * gmap_shadow_notify - handle notifications for shadow gmap
1911 * Called with sg->parent->shadow_lock.
1917 unsigned long start, end, bits, raddr; in gmap_shadow_notify() local
1921 spin_lock(&sg->guest_table_lock); in gmap_shadow_notify()
1922 if (sg->removed) { in gmap_shadow_notify()
1923 spin_unlock(&sg->guest_table_lock); in gmap_shadow_notify()
1927 start = sg->orig_asce & _ASCE_ORIGIN; in gmap_shadow_notify()
1928 end = start + ((sg->orig_asce & _ASCE_TABLE_LENGTH) + 1) * PAGE_SIZE; in gmap_shadow_notify()
1929 if (!(sg->orig_asce & _ASCE_REAL_SPACE) && gaddr >= start && in gmap_shadow_notify()
1933 spin_unlock(&sg->guest_table_lock); in gmap_shadow_notify()
1934 list_del(&sg->list); in gmap_shadow_notify()
1939 head = radix_tree_delete(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT); in gmap_shadow_notify()
1941 bits = rmap->raddr & _SHADOW_RMAP_MASK; in gmap_shadow_notify()
1942 raddr = rmap->raddr ^ bits; in gmap_shadow_notify()
1943 switch (bits) { in gmap_shadow_notify()
1962 spin_unlock(&sg->guest_table_lock); in gmap_shadow_notify()
1966 * ptep_notify - call all invalidation callbacks for a specific pte.
1970 * @bits: bits from the pgste that caused the notify call
1976 pte_t *pte, unsigned long bits) in ptep_notify() argument
1984 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in ptep_notify()
1985 spin_lock(&gmap->guest_table_lock); in ptep_notify()
1987 spin_unlock(&gmap->guest_table_lock); in ptep_notify()
1991 if (!list_empty(&gmap->children) && (bits & PGSTE_VSIE_BIT)) { in ptep_notify()
1992 spin_lock(&gmap->shadow_lock); in ptep_notify()
1994 &gmap->children, list) in ptep_notify()
1996 spin_unlock(&gmap->shadow_lock); in ptep_notify()
1998 if (bits & PGSTE_IN_BIT) in ptep_notify()
1999 gmap_call_notifier(gmap, gaddr, gaddr + PAGE_SIZE - 1); in ptep_notify()
2009 gmap_call_notifier(gmap, gaddr, gaddr + HPAGE_SIZE - 1); in pmdp_notify_gmap()
2013 * gmap_pmdp_xchg - exchange a gmap pmd with another
2014 * @gmap: pointer to the guest address space structure
2017 * @gaddr: the affected guest address
2029 __pmdp_idte(gaddr, (pmd_t *)pmdp, IDTE_GUEST_ASCE, gmap->asce, in gmap_pmdp_xchg()
2046 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in gmap_pmdp_clear()
2047 spin_lock(&gmap->guest_table_lock); in gmap_pmdp_clear()
2058 spin_unlock(&gmap->guest_table_lock); in gmap_pmdp_clear()
2064 * gmap_pmdp_invalidate - invalidate all affected guest pmd entries without
2076 * gmap_pmdp_csp - csp all affected guest pmd entries
2087 * gmap_pmdp_idte_local - invalidate and clear a guest pmd entry
2098 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in gmap_pmdp_idte_local()
2099 spin_lock(&gmap->guest_table_lock); in gmap_pmdp_idte_local()
2108 gmap->asce, IDTE_LOCAL); in gmap_pmdp_idte_local()
2113 spin_unlock(&gmap->guest_table_lock); in gmap_pmdp_idte_local()
2120 * gmap_pmdp_idte_global - invalidate and clear a guest pmd entry
2131 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in gmap_pmdp_idte_global()
2132 spin_lock(&gmap->guest_table_lock); in gmap_pmdp_idte_global()
2141 gmap->asce, IDTE_GLOBAL); in gmap_pmdp_idte_global()
2148 spin_unlock(&gmap->guest_table_lock); in gmap_pmdp_idte_global()
2155 * gmap_test_and_clear_dirty_pmd - test and reset segment dirty status
2156 * @gmap: pointer to guest address space
2158 * @gaddr: virtual address in the guest address space
2181 * gmap_sync_dirty_log_pmd - set bitmap based on dirty status of segment
2182 * @gmap: pointer to guest address space
2184 * @gaddr: virtual address in the guest address space
2207 ptep = pte_alloc_map_lock(gmap->mm, pmdp, vmaddr, &ptl); in gmap_sync_dirty_log_pmd()
2210 if (ptep_test_and_clear_uc(gmap->mm, vmaddr, ptep)) in gmap_sync_dirty_log_pmd()
2223 struct vm_area_struct *vma = walk->vma; in thp_split_walk_pmd_entry()
2243 mm->def_flags |= VM_NOHUGEPAGE; in thp_split_mm()
2256 struct mm_struct *mm = current->mm; in s390_enable_sie()
2263 return -EINVAL; in s390_enable_sie()
2265 mm->context.has_pgste = 1; in s390_enable_sie()
2276 unsigned long *found_addr = walk->private; in find_zeropage_pte_entry()
2286 if (!is_cow_mapping(walk->vma->vm_flags)) in find_zeropage_pte_entry()
2287 return -EFAULT; in find_zeropage_pte_entry()
2305 * This must be called after mm->context.allow_cow_sharing was
2312 * mm->context.allow_cow_sharing is set to 0.
2328 if ((vma->vm_flags & VM_PFNMAP) || is_vm_hugetlb_page(vma)) in __s390_unshare_zeropages()
2330 addr = vma->vm_start; in __s390_unshare_zeropages()
2333 rc = walk_page_range_vma(vma, addr, vma->vm_end, in __s390_unshare_zeropages()
2345 return -ENOMEM; in __s390_unshare_zeropages()
2367 if (!mm->context.allow_cow_sharing) in __s390_disable_cow_sharing()
2370 mm->context.allow_cow_sharing = 0; in __s390_disable_cow_sharing()
2377 * from re-enabling it. in __s390_disable_cow_sharing()
2382 mm->context.allow_cow_sharing = 1; in __s390_disable_cow_sharing()
2387 * Disable most COW-sharing of memory pages for the whole process:
2391 * Not that we currently don't bother with COW-shared pages that are shared
2398 mmap_write_lock(current->mm); in s390_disable_cow_sharing()
2399 rc = __s390_disable_cow_sharing(current->mm); in s390_disable_cow_sharing()
2400 mmap_write_unlock(current->mm); in s390_disable_cow_sharing()
2413 ptep_zap_key(walk->mm, addr, pte); in __s390_enable_skey_pte()
2440 * between actual guest memory and the process executable or in __s390_enable_skey_hugetlb()
2450 set_bit(PG_arch_1, &folio->flags); in __s390_enable_skey_hugetlb()
2464 struct mm_struct *mm = current->mm; in s390_enable_skey()
2471 mm->context.uses_skeys = 1; in s390_enable_skey()
2474 mm->context.uses_skeys = 0; in s390_enable_skey()
2491 ptep_zap_unused(walk->mm, addr, pte, 1); in __s390_reset_cmma()
2519 struct reset_walk_state *p = walk->private; in s390_gather_pages()
2525 p->pfns[p->count] = phys_to_pfn(pte_val(pte)); in s390_gather_pages()
2526 p->next = next; in s390_gather_pages()
2527 p->count++; in s390_gather_pages()
2529 return p->count >= GATHER_GET_PAGES; in s390_gather_pages()
2558 * __s390_uv_destroy_range - Call the destroy secure page UVC on each page
2569 * Return: 0 on success, -EINTR if the function stopped before completing
2585 return -EINTR; in __s390_uv_destroy_range()
2592 * s390_replace_asce - Try to replace the current ASCE of a gmap with a copy
2595 * If the ASCE is a SEGMENT type then this function will return -EINVAL,
2597 * to the wrong pages, causing use-after-free and memory corruption.
2611 if ((gmap->asce & _ASCE_TYPE_MASK) == _ASCE_TYPE_SEGMENT) in s390_replace_asce()
2612 return -EINVAL; in s390_replace_asce()
2616 return -ENOMEM; in s390_replace_asce()
2618 memcpy(table, gmap->table, 1UL << (CRST_ALLOC_ORDER + PAGE_SHIFT)); in s390_replace_asce()
2620 /* Set new table origin while preserving existing ASCE control bits */ in s390_replace_asce()
2621 asce = (gmap->asce & ~_ASCE_ORIGIN) | __pa(table); in s390_replace_asce()
2622 WRITE_ONCE(gmap->asce, asce); in s390_replace_asce()
2623 WRITE_ONCE(gmap->mm->context.gmap_asce, asce); in s390_replace_asce()
2624 WRITE_ONCE(gmap->table, table); in s390_replace_asce()