1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Based on arch/arm/mm/copypage.c
4 *
5 * Copyright (C) 2002 Deep Blue Solutions Ltd, All Rights Reserved.
6 * Copyright (C) 2012 ARM Ltd.
7 */
8
9 #include <linux/bitops.h>
10 #include <linux/mm.h>
11
12 #include <asm/page.h>
13 #include <asm/cacheflush.h>
14 #include <asm/cpufeature.h>
15 #include <asm/mte.h>
16
copy_highpage(struct page * to,struct page * from)17 void copy_highpage(struct page *to, struct page *from)
18 {
19 void *kto = page_address(to);
20 void *kfrom = page_address(from);
21 struct folio *src = page_folio(from);
22 struct folio *dst = page_folio(to);
23 unsigned int i, nr_pages;
24
25 copy_page(kto, kfrom);
26
27 if (kasan_hw_tags_enabled())
28 page_kasan_tag_reset(to);
29
30 if (!system_supports_mte())
31 return;
32
33 if (folio_test_hugetlb(src)) {
34 if (!folio_test_hugetlb_mte_tagged(src) ||
35 from != folio_page(src, 0))
36 return;
37
38 WARN_ON_ONCE(!folio_try_hugetlb_mte_tagging(dst));
39
40 /*
41 * Populate tags for all subpages.
42 *
43 * Don't assume the first page is head page since
44 * huge page copy may start from any subpage.
45 */
46 nr_pages = folio_nr_pages(src);
47 for (i = 0; i < nr_pages; i++) {
48 kfrom = page_address(folio_page(src, i));
49 kto = page_address(folio_page(dst, i));
50 mte_copy_page_tags(kto, kfrom);
51 }
52 folio_set_hugetlb_mte_tagged(dst);
53 } else if (page_mte_tagged(from)) {
54 /* It's a new page, shouldn't have been tagged yet */
55 WARN_ON_ONCE(!try_page_mte_tagging(to));
56
57 mte_copy_page_tags(kto, kfrom);
58 set_page_mte_tagged(to);
59 }
60 }
61 EXPORT_SYMBOL(copy_highpage);
62
copy_user_highpage(struct page * to,struct page * from,unsigned long vaddr,struct vm_area_struct * vma)63 void copy_user_highpage(struct page *to, struct page *from,
64 unsigned long vaddr, struct vm_area_struct *vma)
65 {
66 copy_highpage(to, from);
67 flush_dcache_page(to);
68 }
69 EXPORT_SYMBOL_GPL(copy_user_highpage);
70