100 static inline void tag_set(struct radix_tree_node *node, unsigned int tag,  in tag_set()  argument
103 	__set_bit(offset, node->tags[tag]);  in tag_set()
106 static inline void tag_clear(struct radix_tree_node *node, unsigned int tag,  in tag_clear()  argument
109 	__clear_bit(offset, node->tags[tag]);  in tag_clear()
112 static inline int tag_get(const struct radix_tree_node *node, unsigned int tag,  in tag_get()  argument
115 	return test_bit(offset, node->tags[tag]);  in tag_get()
118 static inline void root_tag_set(struct radix_tree_root *root, unsigned tag)  in root_tag_set()  argument
120 	root->xa_flags |= (__force gfp_t)(1 << (tag + ROOT_TAG_SHIFT));  in root_tag_set()
123 static inline void root_tag_clear(struct radix_tree_root *root, unsigned tag)  in root_tag_clear()  argument
125 	root->xa_flags &= (__force gfp_t)~(1 << (tag + ROOT_TAG_SHIFT));  in root_tag_clear()
133 static inline int root_tag_get(const struct radix_tree_root *root, unsigned tag)  in root_tag_get()  argument
135 	return (__force int)root->xa_flags & (1 << (tag + ROOT_TAG_SHIFT));  in root_tag_get()
149  * Returns 1 if any slot in the node has this tag set.
153 							unsigned int tag)  in any_tag_set()  argument
157 		if (node->tags[tag][idx])  in any_tag_set()
163 static inline void all_tag_set(struct radix_tree_node *node, unsigned int tag)  in all_tag_set()  argument
165 	bitmap_fill(node->tags[tag], RADIX_TREE_MAP_SIZE);  in all_tag_set()
172  * @tag: the tag index
180 radix_tree_find_next_bit(struct radix_tree_node *node, unsigned int tag,  in radix_tree_find_next_bit()  argument
183 	const unsigned long *addr = node->tags[tag];  in radix_tree_find_next_bit()
413 	int tag;  in radix_tree_extend()  local
437 			/* Propagate the aggregated tag info to the new child */  in radix_tree_extend()
438 			for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) {  in radix_tree_extend()
439 				if (root_tag_get(root, tag))  in radix_tree_extend()
440 					tag_set(node, tag, 0);  in radix_tree_extend()
527 		 * also results in a stale slot). So tag the slot as indirect  in radix_tree_shrink()
836 				unsigned int tag, unsigned int offset)  in node_tag_get()  argument
839 		return tag_get(node, tag, offset);  in node_tag_get()
840 	return root_tag_get(root, tag);  in node_tag_get()
940 				unsigned int tag, unsigned int offset)  in node_tag_set()  argument
943 		if (tag_get(node, tag, offset))  in node_tag_set()
945 		tag_set(node, tag, offset);  in node_tag_set()
950 	if (!root_tag_get(root, tag))  in node_tag_set()
951 		root_tag_set(root, tag);  in node_tag_set()
955  *	radix_tree_tag_set - set a tag on a radix tree node
958  *	@tag:		tag index
960  *	Set the search tag (which must be < RADIX_TREE_MAX_TAGS)
964  *	Returns the address of the tagged item.  Setting a tag on a not-present
968 			unsigned long index, unsigned int tag)  in radix_tree_tag_set()  argument
983 		if (!tag_get(parent, tag, offset))  in radix_tree_tag_set()
984 			tag_set(parent, tag, offset);  in radix_tree_tag_set()
987 	/* set the root's tag bit */  in radix_tree_tag_set()
988 	if (!root_tag_get(root, tag))  in radix_tree_tag_set()
989 		root_tag_set(root, tag);  in radix_tree_tag_set()
997 				unsigned int tag, unsigned int offset)  in node_tag_clear()  argument
1000 		if (!tag_get(node, tag, offset))  in node_tag_clear()
1002 		tag_clear(node, tag, offset);  in node_tag_clear()
1003 		if (any_tag_set(node, tag))  in node_tag_clear()
1010 	/* clear the root's tag bit */  in node_tag_clear()
1011 	if (root_tag_get(root, tag))  in node_tag_clear()
1012 		root_tag_clear(root, tag);  in node_tag_clear()
1016  *	radix_tree_tag_clear - clear a tag on a radix tree node
1019  *	@tag:		tag index
1021  *	Clear the search tag (which must be < RADIX_TREE_MAX_TAGS)
1023  *	the leaf node to have no tags set then clear the tag in the
1030 			unsigned long index, unsigned int tag)  in radix_tree_tag_clear()  argument
1048 		node_tag_clear(root, parent, tag, offset);  in radix_tree_tag_clear()
1055   * radix_tree_iter_tag_clear - clear a tag on the current iterator entry
1058   * @tag: tag to clear
1061 			const struct radix_tree_iter *iter, unsigned int tag)  in radix_tree_iter_tag_clear()  argument
1063 	node_tag_clear(root, iter->node, tag, iter_offset(iter));  in radix_tree_iter_tag_clear()
1067  * radix_tree_tag_get - get a tag on a radix tree node
1070  * @tag:		tag index (< RADIX_TREE_MAX_TAGS)
1074  *  0: tag not present or not set
1075  *  1: tag set
1078  * the RCU lock is held, unless tag modification and node deletion are excluded
1082 			unsigned long index, unsigned int tag)  in radix_tree_tag_get()  argument
1087 	if (!root_tag_get(root, tag))  in radix_tree_tag_get()
1100 		if (!tag_get(parent, tag, offset))  in radix_tree_tag_get()
1110 /* Construct iter->tags bit-mask from node->tags[tag] array */
1113 				unsigned tag)  in set_iter_tags()  argument
1123 	iter->tags = node->tags[tag][tag_long] >> tag_bit;  in set_iter_tags()
1129 			iter->tags |= node->tags[tag][tag_long + 1] <<  in set_iter_tags()
1151  * @flags:	RADIX_TREE_ITER_* flags and tag index
1157 	unsigned tag = flags & RADIX_TREE_ITER_TAG_MASK;  in radix_tree_next_chunk()  local
1161 	if ((flags & RADIX_TREE_ITER_TAGGED) && !root_tag_get(root, tag))  in radix_tree_next_chunk()
1198 				!tag_get(node, tag, offset) : !child) {  in radix_tree_next_chunk()
1204 				offset = radix_tree_find_next_bit(node, tag,  in radix_tree_next_chunk()
1235 		set_iter_tags(iter, node, offset, tag);  in radix_tree_next_chunk()
1290  *	                             based on a tag
1295  *	@tag:		the tag index (< RADIX_TREE_MAX_TAGS)
1298  *	have the tag indexed by @tag set.  Places the items at *@results and
1304 		unsigned int tag)  in radix_tree_gang_lookup_tag()  argument
1313 	radix_tree_for_each_tagged(slot, root, &iter, first_index, tag) {  in radix_tree_gang_lookup_tag()
1331  *					  radix tree based on a tag
1336  *	@tag:		the tag index (< RADIX_TREE_MAX_TAGS)
1339  *	have the tag indexed by @tag set.  Places the slots at *@results and
1345 		unsigned int max_items, unsigned int tag)  in radix_tree_gang_lookup_tag_slot()  argument
1354 	radix_tree_for_each_tagged(slot, root, &iter, first_index, tag) {  in radix_tree_gang_lookup_tag_slot()
1370 	int tag;  in __radix_tree_delete()  local
1375 		for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)  in __radix_tree_delete()
1376 			node_tag_clear(root, node, tag, offset);  in __radix_tree_delete()
1454  *	@tag:		tag to test
1456 int radix_tree_tagged(const struct radix_tree_root *root, unsigned int tag)  in radix_tree_tagged()  argument
1458 	return root_tag_get(root, tag);  in radix_tree_tagged()