1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/hugetlb.h>
3 #include <asm-generic/tlb.h>
4 #include <asm/pgalloc.h>
5
6 #include "internal.h"
7
reclaim_pt_is_enabled(unsigned long start,unsigned long end,struct zap_details * details)8 bool reclaim_pt_is_enabled(unsigned long start, unsigned long end,
9 struct zap_details *details)
10 {
11 return details && details->reclaim_pt && (end - start >= PMD_SIZE);
12 }
13
try_get_and_clear_pmd(struct mm_struct * mm,pmd_t * pmd,pmd_t * pmdval)14 bool try_get_and_clear_pmd(struct mm_struct *mm, pmd_t *pmd, pmd_t *pmdval)
15 {
16 spinlock_t *pml = pmd_lockptr(mm, pmd);
17
18 if (!spin_trylock(pml))
19 return false;
20
21 *pmdval = pmdp_get_lockless(pmd);
22 pmd_clear(pmd);
23 spin_unlock(pml);
24
25 return true;
26 }
27
free_pte(struct mm_struct * mm,unsigned long addr,struct mmu_gather * tlb,pmd_t pmdval)28 void free_pte(struct mm_struct *mm, unsigned long addr, struct mmu_gather *tlb,
29 pmd_t pmdval)
30 {
31 pte_free_tlb(tlb, pmd_pgtable(pmdval), addr);
32 mm_dec_nr_ptes(mm);
33 }
34
try_to_free_pte(struct mm_struct * mm,pmd_t * pmd,unsigned long addr,struct mmu_gather * tlb)35 void try_to_free_pte(struct mm_struct *mm, pmd_t *pmd, unsigned long addr,
36 struct mmu_gather *tlb)
37 {
38 pmd_t pmdval;
39 spinlock_t *pml, *ptl = NULL;
40 pte_t *start_pte, *pte;
41 int i;
42
43 pml = pmd_lock(mm, pmd);
44 start_pte = pte_offset_map_rw_nolock(mm, pmd, addr, &pmdval, &ptl);
45 if (!start_pte)
46 goto out_ptl;
47 if (ptl != pml)
48 spin_lock_nested(ptl, SINGLE_DEPTH_NESTING);
49
50 /* Check if it is empty PTE page */
51 for (i = 0, pte = start_pte; i < PTRS_PER_PTE; i++, pte++) {
52 if (!pte_none(ptep_get(pte)))
53 goto out_ptl;
54 }
55 pte_unmap(start_pte);
56
57 pmd_clear(pmd);
58
59 if (ptl != pml)
60 spin_unlock(ptl);
61 spin_unlock(pml);
62
63 free_pte(mm, addr, tlb, pmdval);
64
65 return;
66 out_ptl:
67 if (start_pte)
68 pte_unmap_unlock(start_pte, ptl);
69 if (ptl != pml)
70 spin_unlock(pml);
71 }
72