Lines Matching full:pte
69 pr_err("%s:%d: bad pte %016llx.\n", __FILE__, __LINE__, pte_val(e))
76 static inline phys_addr_t __pte_to_phys(pte_t pte) in __pte_to_phys() argument
78 pte_val(pte) &= ~PTE_MAYBE_SHARED; in __pte_to_phys()
79 return (pte_val(pte) & PTE_ADDR_LOW) | in __pte_to_phys()
80 ((pte_val(pte) & PTE_ADDR_HIGH) << PTE_ADDR_HIGH_SHIFT); in __pte_to_phys()
87 #define __pte_to_phys(pte) (pte_val(pte) & PTE_ADDR_LOW) argument
91 #define pte_pfn(pte) (__pte_to_phys(pte) >> PAGE_SHIFT) argument
95 #define pte_none(pte) (!pte_val(pte)) argument
98 #define pte_page(pte) (pfn_to_page(pte_pfn(pte))) argument
103 #define pte_present(pte) (pte_valid(pte) || pte_present_invalid(pte)) argument
104 #define pte_young(pte) (!!(pte_val(pte) & PTE_AF)) argument
105 #define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL)) argument
106 #define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE)) argument
107 #define pte_rdonly(pte) (!!(pte_val(pte) & PTE_RDONLY)) argument
108 #define pte_user(pte) (!!(pte_val(pte) & PTE_USER)) argument
109 #define pte_user_exec(pte) (!(pte_val(pte) & PTE_UXN)) argument
110 #define pte_cont(pte) (!!(pte_val(pte) & PTE_CONT)) argument
111 #define pte_devmap(pte) (!!(pte_val(pte) & PTE_DEVMAP)) argument
112 #define pte_tagged(pte) ((pte_val(pte) & PTE_ATTRINDX_MASK) == \ argument
125 #define pte_hw_dirty(pte) (pte_write(pte) && !pte_rdonly(pte)) argument
126 #define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY)) argument
127 #define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte)) argument
129 #define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID)) argument
130 #define pte_present_invalid(pte) \ argument
131 ((pte_val(pte) & (PTE_VALID | PTE_PRESENT_INVALID)) == PTE_PRESENT_INVALID)
136 #define pte_valid_not_user(pte) \ argument
137 ((pte_val(pte) & (PTE_VALID | PTE_USER | PTE_UXN)) == (PTE_VALID | PTE_UXN))
139 * Returns true if the pte is valid and has the contiguous bit set.
141 #define pte_valid_cont(pte) (pte_valid(pte) && pte_cont(pte)) argument
143 * Could the pte be present in the TLB? We must check mm_tlb_flush_pending
150 #define pte_accessible(mm, pte) \ argument
151 (mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid(pte))
178 #define pte_access_permitted_no_overlay(pte, write) \ argument
179 …(((pte_val(pte) & (PTE_VALID | PTE_USER)) == (PTE_VALID | PTE_USER)) && (!(write) || pte_write(pte…
180 #define pte_access_permitted(pte, write) \ argument
181 (pte_access_permitted_no_overlay(pte, write) && \
182 por_el0_allows_pkey(FIELD_GET(PTE_PO_IDX_MASK, pte_val(pte)), write, false))
188 static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot) in clear_pte_bit() argument
190 pte_val(pte) &= ~pgprot_val(prot); in clear_pte_bit()
191 return pte; in clear_pte_bit()
194 static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot) in set_pte_bit() argument
196 pte_val(pte) |= pgprot_val(prot); in set_pte_bit()
197 return pte; in set_pte_bit()
212 static inline pte_t pte_mkwrite_novma(pte_t pte) in pte_mkwrite_novma() argument
214 pte = set_pte_bit(pte, __pgprot(PTE_WRITE)); in pte_mkwrite_novma()
215 pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY)); in pte_mkwrite_novma()
216 return pte; in pte_mkwrite_novma()
219 static inline pte_t pte_mkclean(pte_t pte) in pte_mkclean() argument
221 pte = clear_pte_bit(pte, __pgprot(PTE_DIRTY)); in pte_mkclean()
222 pte = set_pte_bit(pte, __pgprot(PTE_RDONLY)); in pte_mkclean()
224 return pte; in pte_mkclean()
227 static inline pte_t pte_mkdirty(pte_t pte) in pte_mkdirty() argument
229 pte = set_pte_bit(pte, __pgprot(PTE_DIRTY)); in pte_mkdirty()
231 if (pte_write(pte)) in pte_mkdirty()
232 pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY)); in pte_mkdirty()
234 return pte; in pte_mkdirty()
237 static inline pte_t pte_wrprotect(pte_t pte) in pte_wrprotect() argument
243 if (pte_hw_dirty(pte)) in pte_wrprotect()
244 pte = set_pte_bit(pte, __pgprot(PTE_DIRTY)); in pte_wrprotect()
246 pte = clear_pte_bit(pte, __pgprot(PTE_WRITE)); in pte_wrprotect()
247 pte = set_pte_bit(pte, __pgprot(PTE_RDONLY)); in pte_wrprotect()
248 return pte; in pte_wrprotect()
251 static inline pte_t pte_mkold(pte_t pte) in pte_mkold() argument
253 return clear_pte_bit(pte, __pgprot(PTE_AF)); in pte_mkold()
256 static inline pte_t pte_mkyoung(pte_t pte) in pte_mkyoung() argument
258 return set_pte_bit(pte, __pgprot(PTE_AF)); in pte_mkyoung()
261 static inline pte_t pte_mkspecial(pte_t pte) in pte_mkspecial() argument
263 return set_pte_bit(pte, __pgprot(PTE_SPECIAL)); in pte_mkspecial()
266 static inline pte_t pte_mkcont(pte_t pte) in pte_mkcont() argument
268 return set_pte_bit(pte, __pgprot(PTE_CONT)); in pte_mkcont()
271 static inline pte_t pte_mknoncont(pte_t pte) in pte_mknoncont() argument
273 return clear_pte_bit(pte, __pgprot(PTE_CONT)); in pte_mknoncont()
276 static inline pte_t pte_mkvalid(pte_t pte) in pte_mkvalid() argument
278 return set_pte_bit(pte, __pgprot(PTE_VALID)); in pte_mkvalid()
281 static inline pte_t pte_mkinvalid(pte_t pte) in pte_mkinvalid() argument
283 pte = set_pte_bit(pte, __pgprot(PTE_PRESENT_INVALID)); in pte_mkinvalid()
284 pte = clear_pte_bit(pte, __pgprot(PTE_VALID)); in pte_mkinvalid()
285 return pte; in pte_mkinvalid()
293 static inline pte_t pte_mkdevmap(pte_t pte) in pte_mkdevmap() argument
295 return set_pte_bit(pte, __pgprot(PTE_DEVMAP | PTE_SPECIAL)); in pte_mkdevmap()
299 static inline int pte_uffd_wp(pte_t pte) in pte_uffd_wp() argument
301 return !!(pte_val(pte) & PTE_UFFD_WP); in pte_uffd_wp()
304 static inline pte_t pte_mkuffd_wp(pte_t pte) in pte_mkuffd_wp() argument
306 return pte_wrprotect(set_pte_bit(pte, __pgprot(PTE_UFFD_WP))); in pte_mkuffd_wp()
309 static inline pte_t pte_clear_uffd_wp(pte_t pte) in pte_clear_uffd_wp() argument
311 return clear_pte_bit(pte, __pgprot(PTE_UFFD_WP)); in pte_clear_uffd_wp()
315 static inline void __set_pte_nosync(pte_t *ptep, pte_t pte) in __set_pte_nosync() argument
317 WRITE_ONCE(*ptep, pte); in __set_pte_nosync()
320 static inline void __set_pte(pte_t *ptep, pte_t pte) in __set_pte() argument
322 __set_pte_nosync(ptep, pte); in __set_pte()
325 * Only if the new pte is valid and kernel, otherwise TLB maintenance in __set_pte()
328 if (pte_valid_not_user(pte)) { in __set_pte()
343 * PTE bits configuration in the presence of hardware Dirty Bit Management
353 * the page fault mechanism. Checking the dirty status of a pte becomes:
359 pte_t pte) in __check_safe_pte_update() argument
368 if (!pte_valid(old_pte) || !pte_valid(pte)) in __check_safe_pte_update()
374 * Check for potential race with hardware updates of the pte in __check_safe_pte_update()
378 VM_WARN_ONCE(!pte_young(pte), in __check_safe_pte_update()
380 __func__, pte_val(old_pte), pte_val(pte)); in __check_safe_pte_update()
381 VM_WARN_ONCE(pte_write(old_pte) && !pte_dirty(pte), in __check_safe_pte_update()
383 __func__, pte_val(old_pte), pte_val(pte)); in __check_safe_pte_update()
384 VM_WARN_ONCE(!pgattr_change_is_safe(pte_val(old_pte), pte_val(pte)), in __check_safe_pte_update()
386 __func__, pte_val(old_pte), pte_val(pte)); in __check_safe_pte_update()
389 static inline void __sync_cache_and_tags(pte_t pte, unsigned int nr_pages) in __sync_cache_and_tags() argument
391 if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte)) in __sync_cache_and_tags()
392 __sync_icache_dcache(pte); in __sync_cache_and_tags()
395 * If the PTE would provide user space access to the tags associated in __sync_cache_and_tags()
401 if (system_supports_mte() && pte_access_permitted_no_overlay(pte, false) && in __sync_cache_and_tags()
402 !pte_special(pte) && pte_tagged(pte)) in __sync_cache_and_tags()
403 mte_sync_tags(pte, nr_pages); in __sync_cache_and_tags()
410 static inline pgprot_t pte_pgprot(pte_t pte) in pte_pgprot() argument
412 unsigned long pfn = pte_pfn(pte); in pte_pgprot()
414 return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte)); in pte_pgprot()
418 static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) in pte_advance_pfn() argument
420 return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte)); in pte_advance_pfn()
425 pte_t *ptep, pte_t pte, unsigned int nr) in __set_ptes() argument
427 page_table_check_ptes_set(mm, ptep, pte, nr); in __set_ptes()
428 __sync_cache_and_tags(pte, nr); in __set_ptes()
431 __check_safe_pte_update(mm, ptep, pte); in __set_ptes()
432 __set_pte(ptep, pte); in __set_ptes()
436 pte = pte_advance_pfn(pte, 1); in __set_ptes()
464 static inline pud_t pte_pud(pte_t pte) in pte_pud() argument
466 return __pud(pte_val(pte)); in pte_pud()
479 static inline pmd_t pte_pmd(pte_t pte) in pte_pmd() argument
481 return __pmd(pte_val(pte)); in pte_pmd()
494 static inline pte_t pte_swp_mkexclusive(pte_t pte) in pte_swp_mkexclusive() argument
496 return set_pte_bit(pte, __pgprot(PTE_SWP_EXCLUSIVE)); in pte_swp_mkexclusive()
499 static inline int pte_swp_exclusive(pte_t pte) in pte_swp_exclusive() argument
501 return pte_val(pte) & PTE_SWP_EXCLUSIVE; in pte_swp_exclusive()
504 static inline pte_t pte_swp_clear_exclusive(pte_t pte) in pte_swp_clear_exclusive() argument
506 return clear_pte_bit(pte, __pgprot(PTE_SWP_EXCLUSIVE)); in pte_swp_clear_exclusive()
510 static inline pte_t pte_swp_mkuffd_wp(pte_t pte) in pte_swp_mkuffd_wp() argument
512 return set_pte_bit(pte, __pgprot(PTE_SWP_UFFD_WP)); in pte_swp_mkuffd_wp()
515 static inline int pte_swp_uffd_wp(pte_t pte) in pte_swp_uffd_wp() argument
517 return !!(pte_val(pte) & PTE_SWP_UFFD_WP); in pte_swp_uffd_wp()
520 static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) in pte_swp_clear_uffd_wp() argument
522 return clear_pte_bit(pte, __pgprot(PTE_SWP_UFFD_WP)); in pte_swp_clear_uffd_wp()
530 static inline int pte_protnone(pte_t pte) in pte_protnone() argument
533 * pte_present_invalid() tells us that the pte is invalid from HW in pte_protnone()
541 return pte_present_invalid(pte) && !pte_user(pte) && !pte_user_exec(pte); in pte_protnone()
599 #define pmd_special(pte) (!!((pmd_val(pte) & PTE_SPECIAL))) argument
624 #define pud_special(pte) pte_special(pud_pte(pud)) argument
625 #define pud_mkspecial(pte) pte_pud(pte_mkspecial(pud_pte(pud))) argument
646 pte_t *ptep, pte_t pte, unsigned int nr) in __set_pte_at() argument
648 __sync_cache_and_tags(pte, nr); in __set_pte_at()
649 __check_safe_pte_update(mm, ptep, pte); in __set_pte_at()
650 __set_pte(ptep, pte); in __set_pte_at()
725 #define pte_leaf_size(pte) (pte_cont(pte) ? CONT_PTE_SIZE : PAGE_SIZE) argument
1156 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) in pte_modify() argument
1167 if (pte_hw_dirty(pte)) in pte_modify()
1168 pte = set_pte_bit(pte, __pgprot(PTE_DIRTY)); in pte_modify()
1170 pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask); in pte_modify()
1172 * If we end up clearing hw dirtiness for a sw-dirty PTE, set hardware in pte_modify()
1175 if (pte_sw_dirty(pte)) in pte_modify()
1176 pte = pte_mkdirty(pte); in pte_modify()
1177 return pte; in pte_modify()
1211 static inline bool pte_user_accessible_page(pte_t pte) in pte_user_accessible_page() argument
1213 return pte_valid(pte) && (pte_user(pte) || pte_user_exec(pte)); in pte_user_accessible_page()
1228 * Atomic pte/pmd modifications.
1234 pte_t old_pte, pte; in __ptep_test_and_clear_young() local
1236 pte = __ptep_get(ptep); in __ptep_test_and_clear_young()
1238 old_pte = pte; in __ptep_test_and_clear_young()
1239 pte = pte_mkold(pte); in __ptep_test_and_clear_young()
1240 pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep), in __ptep_test_and_clear_young()
1241 pte_val(old_pte), pte_val(pte)); in __ptep_test_and_clear_young()
1242 } while (pte_val(pte) != pte_val(old_pte)); in __ptep_test_and_clear_young()
1244 return pte_young(pte); in __ptep_test_and_clear_young()
1282 pte_t pte = __pte(xchg_relaxed(&pte_val(*ptep), 0)); in __ptep_get_and_clear() local
1284 page_table_check_pte_clear(mm, pte); in __ptep_get_and_clear()
1286 return pte; in __ptep_get_and_clear()
1305 pte_t pte, tmp_pte; in __get_and_clear_full_ptes() local
1307 pte = __ptep_get_and_clear(mm, addr, ptep); in __get_and_clear_full_ptes()
1313 pte = pte_mkdirty(pte); in __get_and_clear_full_ptes()
1315 pte = pte_mkyoung(pte); in __get_and_clear_full_ptes()
1317 return pte; in __get_and_clear_full_ptes()
1335 pte_t pte) in ___ptep_set_wrprotect() argument
1340 old_pte = pte; in ___ptep_set_wrprotect()
1341 pte = pte_wrprotect(pte); in ___ptep_set_wrprotect()
1342 pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep), in ___ptep_set_wrprotect()
1343 pte_val(old_pte), pte_val(pte)); in ___ptep_set_wrprotect()
1344 } while (pte_val(pte) != pte_val(old_pte)); in ___ptep_set_wrprotect()
1368 pte_t pte, cydp_t flags) in __clear_young_dirty_pte() argument
1373 old_pte = pte; in __clear_young_dirty_pte()
1376 pte = pte_mkold(pte); in __clear_young_dirty_pte()
1378 pte = pte_mkclean(pte); in __clear_young_dirty_pte()
1380 pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep), in __clear_young_dirty_pte()
1381 pte_val(old_pte), pte_val(pte)); in __clear_young_dirty_pte()
1382 } while (pte_val(pte) != pte_val(old_pte)); in __clear_young_dirty_pte()
1389 pte_t pte; in __clear_young_dirty_ptes() local
1392 pte = __ptep_get(ptep); in __clear_young_dirty_ptes()
1395 __set_pte(ptep, pte_mkclean(pte_mkold(pte))); in __clear_young_dirty_ptes()
1397 __clear_young_dirty_pte(vma, addr, ptep, pte, flags); in __clear_young_dirty_ptes()
1443 #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) argument
1506 * the pte is old and cannot be marked young. So we always end up with zeroed
1546 pte_t *ptep, pte_t pte);
1548 pte_t *ptep, pte_t pte);
1552 pte_t *ptep, pte_t pte, unsigned int nr);
1572 unsigned long addr, pte_t *ptep, pte_t pte) in contpte_try_fold() argument
1586 bool palign = (pte_pfn(pte) & contmask) == contmask; in contpte_try_fold()
1589 pte_valid(pte) && !pte_cont(pte) && !pte_special(pte))) in contpte_try_fold()
1590 __contpte_try_fold(mm, addr, ptep, pte); in contpte_try_fold()
1595 unsigned long addr, pte_t *ptep, pte_t pte) in contpte_try_unfold() argument
1597 if (unlikely(pte_valid_cont(pte))) in contpte_try_unfold()
1598 __contpte_try_unfold(mm, addr, ptep, pte); in contpte_try_unfold()
1602 static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte) in pte_batch_hint() argument
1604 if (!pte_valid_cont(pte)) in pte_batch_hint()
1612 * core-mm to manipulate PTE entries within their page tables (or at least this
1623 * its possible a pte will be passed to a setter with the contiguous bit set, so
1631 pte_t pte = __ptep_get(ptep); in ptep_get() local
1633 if (likely(!pte_valid_cont(pte))) in ptep_get()
1634 return pte; in ptep_get()
1636 return contpte_ptep_get(ptep, pte); in ptep_get()
1642 pte_t pte = __ptep_get(ptep); in ptep_get_lockless() local
1644 if (likely(!pte_valid_cont(pte))) in ptep_get_lockless()
1645 return pte; in ptep_get_lockless()
1650 static inline void set_pte(pte_t *ptep, pte_t pte) in set_pte() argument
1656 * any call and emit a warning if there is any attempt to set a pte on in set_pte()
1662 __set_pte(ptep, pte_mknoncont(pte)); in set_pte()
1667 pte_t *ptep, pte_t pte, unsigned int nr) in set_ptes() argument
1669 pte = pte_mknoncont(pte); in set_ptes()
1673 __set_ptes(mm, addr, ptep, pte, 1); in set_ptes()
1674 contpte_try_fold(mm, addr, ptep, pte); in set_ptes()
1676 contpte_set_ptes(mm, addr, ptep, pte, nr); in set_ptes()
1704 pte_t pte; in get_and_clear_full_ptes() local
1708 pte = __get_and_clear_full_ptes(mm, addr, ptep, nr, full); in get_and_clear_full_ptes()
1710 pte = contpte_get_and_clear_full_ptes(mm, addr, ptep, nr, full); in get_and_clear_full_ptes()
1713 return pte; in get_and_clear_full_ptes()
1756 * unfold, and we can remove the contig bit from the pte we read in wrprotect_ptes()