58 		 "Enable inter server to server copy offload. Default: false");
60 static void cleanup_async_copy(struct nfsd4_copy *copy);
411 	 * field isn't set for some reason, throw warning and just copy  in set_change_info()
960 	 * If we do a zero copy read, then a client will see read data  in nfsd4_read()
964 	 * To ensure proper ordering, we therefore turn off zero copy if  in nfsd4_read()
1307  * nfsd4_has_active_async_copies - Check for ongoing copy operations
1310  * NFSD maintains state for async COPY operations after they complete,
1316  *   %true: At least one active async COPY is ongoing
1317  *   %false: No active async COPY operations were found
1321 	struct nfsd4_copy *copy;  in nfsd4_has_active_async_copies()  local
1325 	list_for_each_entry(copy, &clp->async_copies, copies) {  in nfsd4_has_active_async_copies()
1326 		if (!test_bit(NFSD4_COPY_F_COMPLETED, &copy->cp_flags) &&  in nfsd4_has_active_async_copies()
1327 		    !test_bit(NFSD4_COPY_F_STOPPED, &copy->cp_flags)) {  in nfsd4_has_active_async_copies()
1338  * @nn: Network namespace with possible active copy information
1343 	struct nfsd4_copy *copy;  in nfsd4_async_copy_reaper()  local
1352 			copy = list_entry(pos, struct nfsd4_copy, copies);  in nfsd4_async_copy_reaper()
1353 			if (test_bit(NFSD4_COPY_F_OFFLOAD_DONE, &copy->cp_flags)) {  in nfsd4_async_copy_reaper()
1354 				if (--copy->cp_ttl) {  in nfsd4_async_copy_reaper()
1355 					list_del_init(&copy->copies);  in nfsd4_async_copy_reaper()
1356 					list_add(&copy->copies, &reaplist);  in nfsd4_async_copy_reaper()
1365 		copy = list_first_entry(&reaplist, struct nfsd4_copy, copies);  in nfsd4_async_copy_reaper()
1366 		list_del_init(&copy->copies);  in nfsd4_async_copy_reaper()
1367 		cleanup_async_copy(copy);  in nfsd4_async_copy_reaper()
1371 static void nfs4_put_copy(struct nfsd4_copy *copy)  in nfs4_put_copy()  argument
1373 	if (!refcount_dec_and_test(&copy->refcount))  in nfs4_put_copy()
1375 	kfree(copy->cp_src);  in nfs4_put_copy()
1376 	kfree(copy);  in nfs4_put_copy()
1379 static void nfsd4_stop_copy(struct nfsd4_copy *copy)  in nfsd4_stop_copy()  argument
1381 	trace_nfsd_copy_async_cancel(copy);  in nfsd4_stop_copy()
1382 	if (!test_and_set_bit(NFSD4_COPY_F_STOPPED, &copy->cp_flags))  in nfsd4_stop_copy()
1383 		kthread_stop(copy->copy_task);  in nfsd4_stop_copy()
1384 	nfs4_put_copy(copy);  in nfsd4_stop_copy()
1389 	struct nfsd4_copy *copy = NULL;  in nfsd4_unhash_copy()  local
1393 		copy = list_first_entry(&clp->async_copies, struct nfsd4_copy,  in nfsd4_unhash_copy()
1395 		refcount_inc(&copy->refcount);  in nfsd4_unhash_copy()
1396 		copy->cp_clp = NULL;  in nfsd4_unhash_copy()
1397 		if (!list_empty(&copy->copies))  in nfsd4_unhash_copy()
1398 			list_del_init(&copy->copies);  in nfsd4_unhash_copy()
1401 	return copy;  in nfsd4_unhash_copy()
1406 	struct nfsd4_copy *copy;  in nfsd4_shutdown_copy()  local
1408 	while ((copy = nfsd4_unhash_copy(clp)) != NULL)  in nfsd4_shutdown_copy()
1409 		nfsd4_stop_copy(copy);  in nfsd4_shutdown_copy()
1500  * Support one copy source server for now.
1594  * Verify COPY destination stateid.
1598  * Called with COPY cstate:
1605 		      struct nfsd4_copy *copy)  in nfsd4_setup_inter_ssc()  argument
1608 	stateid_t *s_stid = &copy->cp_src_stateid;  in nfsd4_setup_inter_ssc()
1613 					    &copy->cp_dst_stateid,  in nfsd4_setup_inter_ssc()
1614 					    WR_STATE, &copy->nf_dst, NULL);  in nfsd4_setup_inter_ssc()
1618 	status = nfsd4_interssc_connect(copy->cp_src, rqstp, &copy->ss_nsui);  in nfsd4_setup_inter_ssc()
1624 	copy->c_fh.size = s_fh->fh_handle.fh_size;  in nfsd4_setup_inter_ssc()
1625 	memcpy(copy->c_fh.data, &s_fh->fh_handle.fh_raw, copy->c_fh.size);  in nfsd4_setup_inter_ssc()
1626 	copy->stateid.seqid = cpu_to_be32(s_stid->si_generation);  in nfsd4_setup_inter_ssc()
1627 	memcpy(copy->stateid.other, (void *)&s_stid->si_opaque,  in nfsd4_setup_inter_ssc()
1663 		      struct nfsd4_copy *copy)  in nfsd4_setup_inter_ssc()  argument
1685 		      struct nfsd4_copy *copy)  in nfsd4_setup_intra_ssc()  argument
1687 	return nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,  in nfsd4_setup_intra_ssc()
1688 				 &copy->nf_src, &copy->cp_dst_stateid,  in nfsd4_setup_intra_ssc()
1689 				 &copy->nf_dst);  in nfsd4_setup_intra_ssc()
1696 	struct nfsd4_copy *copy =  in nfsd4_cb_offload_release()  local
1699 	set_bit(NFSD4_COPY_F_OFFLOAD_DONE, &copy->cp_flags);  in nfsd4_cb_offload_release()
1725 static void nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)  in nfsd4_init_copy_res()  argument
1727 	copy->cp_res.wr_stable_how =  in nfsd4_init_copy_res()
1728 		test_bit(NFSD4_COPY_F_COMMITTED, &copy->cp_flags) ?  in nfsd4_init_copy_res()
1730 	nfsd4_copy_set_sync(copy, sync);  in nfsd4_init_copy_res()
1733 static ssize_t _nfsd_copy_file_range(struct nfsd4_copy *copy,  in _nfsd_copy_file_range()  argument
1739 	u64 bytes_total = copy->cp_count;  in _nfsd_copy_file_range()
1740 	u64 src_pos = copy->cp_src_pos;  in _nfsd_copy_file_range()
1741 	u64 dst_pos = copy->cp_dst_pos;  in _nfsd_copy_file_range()
1757 		copy->cp_res.wr_bytes_written += bytes_copied;  in _nfsd_copy_file_range()
1760 	} while (bytes_total > 0 && nfsd4_copy_is_async(copy));  in _nfsd_copy_file_range()
1761 	/* for a non-zero asynchronous copy do a commit of data */  in _nfsd_copy_file_range()
1762 	if (nfsd4_copy_is_async(copy) && copy->cp_res.wr_bytes_written > 0) {  in _nfsd_copy_file_range()
1764 		end = copy->cp_dst_pos + copy->cp_res.wr_bytes_written - 1;  in _nfsd_copy_file_range()
1765 		status = vfs_fsync_range(dst, copy->cp_dst_pos, end, 0);  in _nfsd_copy_file_range()
1769 			set_bit(NFSD4_COPY_F_COMMITTED, &copy->cp_flags);  in _nfsd_copy_file_range()
1774 static __be32 nfsd4_do_copy(struct nfsd4_copy *copy,  in nfsd4_do_copy()  argument
1781 	bytes = _nfsd_copy_file_range(copy, dst, src);  in nfsd4_do_copy()
1783 	/* for async copy, we ignore the error, client can always retry  in nfsd4_do_copy()
1786 	if (bytes < 0 && !copy->cp_res.wr_bytes_written)  in nfsd4_do_copy()
1789 		nfsd4_init_copy_res(copy, sync);  in nfsd4_do_copy()
1816 static void release_copy_files(struct nfsd4_copy *copy)  in release_copy_files()  argument
1818 	if (copy->nf_src)  in release_copy_files()
1819 		nfsd_file_put(copy->nf_src);  in release_copy_files()
1820 	if (copy->nf_dst)  in release_copy_files()
1821 		nfsd_file_put(copy->nf_dst);  in release_copy_files()
1824 static void cleanup_async_copy(struct nfsd4_copy *copy)  in cleanup_async_copy()  argument
1826 	nfs4_free_copy_state(copy);  in cleanup_async_copy()
1827 	release_copy_files(copy);  in cleanup_async_copy()
1828 	if (copy->cp_clp) {  in cleanup_async_copy()
1829 		spin_lock(&copy->cp_clp->async_lock);  in cleanup_async_copy()
1830 		if (!list_empty(&copy->copies))  in cleanup_async_copy()
1831 			list_del_init(&copy->copies);  in cleanup_async_copy()
1832 		spin_unlock(&copy->cp_clp->async_lock);  in cleanup_async_copy()
1834 	nfs4_put_copy(copy);  in cleanup_async_copy()
1837 static void nfsd4_send_cb_offload(struct nfsd4_copy *copy)  in nfsd4_send_cb_offload()  argument
1839 	struct nfsd4_cb_offload *cbo = &copy->cp_cb_offload;  in nfsd4_send_cb_offload()
1841 	memcpy(&cbo->co_res, &copy->cp_res, sizeof(copy->cp_res));  in nfsd4_send_cb_offload()
1842 	memcpy(&cbo->co_fh, &copy->fh, sizeof(copy->fh));  in nfsd4_send_cb_offload()
1843 	cbo->co_nfserr = copy->nfserr;  in nfsd4_send_cb_offload()
1846 	nfsd4_init_cb(&cbo->co_cb, copy->cp_clp, &nfsd4_cb_offload_ops,  in nfsd4_send_cb_offload()
1848 	trace_nfsd_cb_offload(copy->cp_clp, &cbo->co_res.cb_stateid,  in nfsd4_send_cb_offload()
1849 			      &cbo->co_fh, copy->cp_count, copy->nfserr);  in nfsd4_send_cb_offload()
1854  * nfsd4_do_async_copy - kthread function for background server-side COPY
1855  * @data: arguments for COPY operation
1858  *   %0: Copy operation is done.
1862 	struct nfsd4_copy *copy = (struct nfsd4_copy *)data;  in nfsd4_do_async_copy()  local
1864 	trace_nfsd_copy_async(copy);  in nfsd4_do_async_copy()
1865 	if (nfsd4_ssc_is_inter(copy)) {  in nfsd4_do_async_copy()
1868 		filp = nfs42_ssc_open(copy->ss_nsui->nsui_vfsmount,  in nfsd4_do_async_copy()
1869 				      &copy->c_fh, &copy->stateid);  in nfsd4_do_async_copy()
1873 				copy->nfserr = nfserr_wrong_type;  in nfsd4_do_async_copy()
1876 				copy->nfserr = nfserr_offload_denied;  in nfsd4_do_async_copy()
1881 		copy->nfserr = nfsd4_do_copy(copy, filp, copy->nf_dst->nf_file,  in nfsd4_do_async_copy()
1883 		nfsd4_cleanup_inter_ssc(copy->ss_nsui, filp, copy->nf_dst);  in nfsd4_do_async_copy()
1885 		copy->nfserr = nfsd4_do_copy(copy, copy->nf_src->nf_file,  in nfsd4_do_async_copy()
1886 					     copy->nf_dst->nf_file, false);  in nfsd4_do_async_copy()
1892 	set_bit(NFSD4_COPY_F_STOPPED, &copy->cp_flags);  in nfsd4_do_async_copy()
1894 	set_bit(NFSD4_COPY_F_COMPLETED, &copy->cp_flags);  in nfsd4_do_async_copy()
1895 	trace_nfsd_copy_async_done(copy);  in nfsd4_do_async_copy()
1896 	nfsd4_send_cb_offload(copy);  in nfsd4_do_async_copy()
1897 	atomic_dec(&copy->cp_nn->pending_async_copies);  in nfsd4_do_async_copy()
1907 	struct nfsd4_copy *copy = &u->copy;  in nfsd4_copy()  local
1912 	 * Currently, async COPY is not reliable. Force all COPY  in nfsd4_copy()
1914 	 * hangs waiting for COPY completion.  in nfsd4_copy()
1916 	nfsd4_copy_set_sync(copy, true);  in nfsd4_copy()
1918 	result = &copy->cp_res;  in nfsd4_copy()
1921 	copy->cp_clp = cstate->clp;  in nfsd4_copy()
1922 	if (nfsd4_ssc_is_inter(copy)) {  in nfsd4_copy()
1923 		trace_nfsd_copy_inter(copy);  in nfsd4_copy()
1924 		if (!inter_copy_offload_enable || nfsd4_copy_is_sync(copy)) {  in nfsd4_copy()
1928 		status = nfsd4_setup_inter_ssc(rqstp, cstate, copy);  in nfsd4_copy()
1930 			trace_nfsd_copy_done(copy, status);  in nfsd4_copy()
1934 		trace_nfsd_copy_intra(copy);  in nfsd4_copy()
1935 		status = nfsd4_setup_intra_ssc(rqstp, cstate, copy);  in nfsd4_copy()
1937 			trace_nfsd_copy_done(copy, status);  in nfsd4_copy()
1942 	memcpy(&copy->fh, &cstate->current_fh.fh_handle,  in nfsd4_copy()
1944 	if (nfsd4_copy_is_async(copy)) {  in nfsd4_copy()
1952 		/* Arbitrary cap on number of pending async copy operations */  in nfsd4_copy()
1959 		if (!nfs4_init_copy_state(nn, copy))  in nfsd4_copy()
1961 		memcpy(&result->cb_stateid, &copy->cp_stateid.cs_stid,  in nfsd4_copy()
1963 		dup_copy_fields(copy, async_copy);  in nfsd4_copy()
1965 				async_copy, "%s", "copy thread");  in nfsd4_copy()
1975 		status = nfsd4_do_copy(copy, copy->nf_src->nf_file,  in nfsd4_copy()
1976 				       copy->nf_dst->nf_file, true);  in nfsd4_copy()
1979 	trace_nfsd_copy_done(copy, status);  in nfsd4_copy()
1980 	release_copy_files(copy);  in nfsd4_copy()
1986 	if (nfsd4_ssc_is_inter(copy)) {  in nfsd4_copy()
1988 		 * Source's vfsmount of inter-copy will be unmounted  in nfsd4_copy()
1989 		 * by the laundromat. Use copy instead of async_copy  in nfsd4_copy()
1992 		refcount_dec(&copy->ss_nsui->nsui_refcnt);  in nfsd4_copy()
2003 	struct nfsd4_copy *copy;  in find_async_copy_locked()  local
2007 	list_for_each_entry(copy, &clp->async_copies, copies) {  in find_async_copy_locked()
2008 		if (memcmp(&copy->cp_stateid.cs_stid, stateid, NFS4_STATEID_SIZE))  in find_async_copy_locked()
2010 		return copy;  in find_async_copy_locked()
2018 	struct nfsd4_copy *copy;  in find_async_copy()  local
2021 	copy = find_async_copy_locked(clp, stateid);  in find_async_copy()
2022 	if (copy)  in find_async_copy()
2023 		refcount_inc(&copy->refcount);  in find_async_copy()
2025 	return copy;  in find_async_copy()
2034 	struct nfsd4_copy *copy;  in nfsd4_offload_cancel()  local
2037 	copy = find_async_copy(clp, &os->stateid);  in nfsd4_offload_cancel()
2038 	if (!copy) {  in nfsd4_offload_cancel()
2043 		nfsd4_stop_copy(copy);  in nfsd4_offload_cancel()
2122 	struct nfsd4_copy *copy;  in nfsd4_offload_status()  local
2127 	copy = find_async_copy_locked(clp, &os->stateid);  in nfsd4_offload_status()
2128 	if (copy) {  in nfsd4_offload_status()
2129 		os->count = copy->cp_res.wr_bytes_written;  in nfsd4_offload_status()
2130 		if (test_bit(NFSD4_COPY_F_COMPLETED, &copy->cp_flags)) {  in nfsd4_offload_status()
2132 			os->status = copy->nfserr;  in nfsd4_offload_status()
2611 	 * Get the entire list, then copy out only the user attributes  in nfsd4_listxattrs()
2744 	struct nfsd4_copy *copy;  in check_if_stalefh_allowed()  local
2748 	/* traverse all operation and if it's a COPY compound, mark the  in check_if_stalefh_allowed()
2760 			copy = (struct nfsd4_copy *)&op->u;  in check_if_stalefh_allowed()
2766 			if (nfsd4_ssc_is_inter(copy))  in check_if_stalefh_allowed()