19 	u64 v = *p & (~0ULL >> high_bit_offset);  in bch2_bkey_packed_to_binary_text()
31 			next_key_bits = 0;  in bch2_bkey_packed_to_binary_text()
70 		      format->bits_per_field[0],  in bch2_bkey_pack_verify()
120 		.w	= 0,  in pack_state_init()
160 	u64 v = 0, offset = le64_to_cpu(state->format->field_offset[field]);  in get_inc_field()
171 	/* avoid shift by 64 if bits is 0 - bits is never 64 here: */  in get_inc_field()
187 			/* avoid shift by 64 if bits is 64 - bits is never 0 here: */  in __set_inc_field()
192 			state->w = 0;  in __set_inc_field()
235 	*w = 0;  in bch2_bkey_transform_key()
237 	for (i = 0; i < BKEY_NR_FIELDS; i++)  in bch2_bkey_transform_key()
281 	out.pad[0]	= 0;  in __bch2_bkey_unpack_key()
327 	*w = 0;  in bch2_bkey_pack_key()
393 		v = ~(~0ULL << bits);  in set_inc_field_lossy()
424 		u64 mask = (~0ULL >> (64 - bits)) << offset;  in bkey_packed_successor()
428 			EBUG_ON(bch2_bkey_cmp_packed(b, out, &k) <= 0);  in bkey_packed_successor()
435 		offset = 0;  in bkey_packed_successor()
443 	for (unsigned i = 0; i < f->nr_fields; i++) {  in bkey_format_has_too_big_fields()
445 		u64 unpacked_max = ~((~0ULL << 1) << (unpacked_bits - 1));  in bkey_format_has_too_big_fields()
447 			? ~((~0ULL << 1) << (f->bits_per_field[i] - 1))  in bkey_format_has_too_big_fields()
448 			: 0;  in bkey_format_has_too_big_fields()
486 	for (i = 0; i < f->key_u64s; i++)  in bch2_bkey_pack_pos_lossy()
487 		w[i] = 0;  in bch2_bkey_pack_pos_lossy()
536 		BUG_ON(bkey_cmp_left_packed(b, out, &orig) >= 0);  in bch2_bkey_pack_pos_lossy()
538 		       bkey_cmp_left_packed(b, &successor, &orig) < 0 &&  in bch2_bkey_pack_pos_lossy()
550 	for (i = 0; i < ARRAY_SIZE(s->field_min); i++)  in bch2_bkey_format_init()
553 	for (i = 0; i < ARRAY_SIZE(s->field_max); i++)  in bch2_bkey_format_init()
554 		s->field_max[i] = 0;  in bch2_bkey_format_init()
556 	/* Make sure we can store a size of 0: */  in bch2_bkey_format_init()
557 	s->field_min[BKEY_FIELD_SIZE] = 0;  in bch2_bkey_format_init()
562 	unsigned field = 0;  in bch2_bkey_format_add_pos()
578 	u64 unpacked_max = ~((~0ULL << 1) << (unpacked_bits - 1));  in set_format_field()
582 	offset = bits == unpacked_bits ? 0 : min(offset, unpacked_max - ((1ULL << bits) - 1));  in set_format_field()
595 	for (i = 0; i < ARRAY_SIZE(s->field_min); i++) {  in bch2_bkey_format_done()
618 	for (i = 0; i < ARRAY_SIZE(ret.bits_per_field); i++) {  in bch2_bkey_format_done()
634 		BUG_ON(bch2_bkey_format_invalid(NULL, &ret, 0, &buf));  in bch2_bkey_format_done()
658 	for (unsigned i = 0; i < f->nr_fields; i++) {  in bch2_bkey_format_invalid()
661 			u64 unpacked_max = ~((~0ULL << 1) << (unpacked_bits - 1));  in bch2_bkey_format_invalid()
664 				? ~((~0ULL << 1) << (packed_bits - 1))  in bch2_bkey_format_invalid()
665 				: 0;  in bch2_bkey_format_invalid()
681 	return 0;  in bch2_bkey_format_invalid()
688 	for (unsigned i = 0; i < ARRAY_SIZE(f->bits_per_field); i++) {  in bch2_bkey_format_to_text()
699  * Bits are indexed from 0 - return is [0, nr_key_bits)
715 	l_v = *l & (~0ULL >> high_bit_offset);  in bch2_bkey_greatest_differing_bit()
716 	r_v = *r & (~0ULL >> high_bit_offset);  in bch2_bkey_greatest_differing_bit()
722 			nr_key_bits = 0;  in bch2_bkey_greatest_differing_bit()
738 	return 0;  in bch2_bkey_greatest_differing_bit()
743  * Bits are indexed from 0 - return is [0, nr_key_bits)
750 	unsigned ret = 0, offset;  in bch2_bkey_ffs()
767 		u64 mask = (~0ULL >> (64 - bits)) << offset;  in bch2_bkey_ffs()
775 		offset = 0;  in bch2_bkey_ffs()
778 	return 0;  in bch2_bkey_ffs()
802 			I2(0x31, 0xc0);  in compile_bkey_field()
816 				I3(0xc7, 0x47, dst_offset);  in compile_bkey_field()
820 				I3(0xc7, 0x47, dst_offset + 4);  in compile_bkey_field()
826 				I4(0x48, 0xc7, 0x47, dst_offset);  in compile_bkey_field()
833 			I3(0xc7, 0x47, dst_offset);  in compile_bkey_field()
845 	for (i = 0; i <= field; i++)  in compile_bkey_field()
853 	if (bit_offset == 0 && bits == 8) {  in compile_bkey_field()
855 		I4(0x0f, 0xb6, 0x46, byte);  in compile_bkey_field()
856 	} else if (bit_offset == 0 && bits == 16) {  in compile_bkey_field()
858 		I4(0x0f, 0xb7, 0x46, byte);  in compile_bkey_field()
867 		I3(0x8b, 0x46, byte);  in compile_bkey_field()
871 			I3(0xc1, 0xe8, bit_offset);  in compile_bkey_field()
875 			unsigned mask = ~0U >> (32 - bits);  in compile_bkey_field()
878 			I1(0x25);  in compile_bkey_field()
890 		I4(0x48, 0x8b, 0x46, byte);  in compile_bkey_field()
897 			I4(0x48, 0xc1, 0xe0, shl);  in compile_bkey_field()
902 			I4(0x48, 0xc1, 0xe8, shr);  in compile_bkey_field()
912 		I4(0x48, 0x8b, 0x46, byte);  in compile_bkey_field()
915 		I3(0x8b, 0x56, byte + 8);  in compile_bkey_field()
922 		I4(0x48, 0xc1, 0xe8, shr);  in compile_bkey_field()
925 		I4(0x48, 0xc1, 0xe2, 64 - shr);  in compile_bkey_field()
928 		I3(0x48, 0x09, 0xd0);  in compile_bkey_field()
934 			I4(0x48, 0xc1, 0xe8, shr);  in compile_bkey_field()
941 		I2(0x48, 0xba);  in compile_bkey_field()
945 		I3(0x48, 0x01, 0xd0);  in compile_bkey_field()
946 	} else if (offset + (~0ULL >> (64 - bits)) > U32_MAX) {  in compile_bkey_field()
948 		I2(0x48, 0x05);  in compile_bkey_field()
953 		I1(0x05);  in compile_bkey_field()
961 		I4(0x48, 0x89, 0x47, dst_offset);  in compile_bkey_field()
965 		I3(0x89, 0x47, dst_offset);  in compile_bkey_field()
987 	I2(0x8b, 0x06);  in bch2_compile_bkey_format()
990 	I5(0x05, BKEY_U64s - format->key_u64s, KEY_FORMAT_CURRENT, 0, 0);  in bch2_compile_bkey_format()
993 	I5(0x25, 0xff, 0xff, 0xff, 0);  in bch2_compile_bkey_format()
996 	I2(0x89, 0x07);  in bch2_compile_bkey_format()
1007 	I1(0xc3);  in bch2_compile_bkey_format()
1079 	struct bkey t = KEY(4134ULL, 1250629070527416633ULL, 0);  in bch2_bkey_pack_test()
1097 	for (i = 0; i < out_s.format->nr_fields; i++) {  in bch2_bkey_pack_test()