110 	void (*ds_synth)(const struct arm_spe_record *record,
309 static struct simd_flags arm_spe__synth_simd_flags(const struct arm_spe_record *record)  in arm_spe__synth_simd_flags()  argument
313 	if ((record->op & ARM_SPE_OP_LDST) && (record->op & ARM_SPE_OP_SVE_LDST))  in arm_spe__synth_simd_flags()
316 	if ((record->op & ARM_SPE_OP_OTHER) && (record->op & ARM_SPE_OP_SVE_OTHER))  in arm_spe__synth_simd_flags()
319 	if (record->type & ARM_SPE_SVE_PARTIAL_PRED)  in arm_spe__synth_simd_flags()
322 	if (record->type & ARM_SPE_SVE_EMPTY_PRED)  in arm_spe__synth_simd_flags()
333 	struct arm_spe_record *record = &speq->decoder->record;  in arm_spe_prep_sample()  local
336 		sample->time = tsc_to_perf_time(record->timestamp, &spe->tc);  in arm_spe_prep_sample()
338 	sample->ip = record->from_ip;  in arm_spe_prep_sample()
344 	sample->simd_flags = arm_spe__synth_simd_flags(record);  in arm_spe_prep_sample()
382 	struct arm_spe_record *record = &speq->decoder->record;  in arm_spe__synth_mem_sample()  local
390 	sample.addr = record->virt_addr;  in arm_spe__synth_mem_sample()
391 	sample.phys_addr = record->phys_addr;  in arm_spe__synth_mem_sample()
393 	sample.weight = record->latency;  in arm_spe__synth_mem_sample()
402 	struct arm_spe_record *record = &speq->decoder->record;  in arm_spe__synth_branch_sample()  local
410 	sample.addr = record->to_ip;  in arm_spe__synth_branch_sample()
411 	sample.weight = record->latency;  in arm_spe__synth_branch_sample()
421 	struct arm_spe_record *record = &speq->decoder->record;  in arm_spe__synth_instruction_sample()  local
437 	sample.addr = record->to_ip;  in arm_spe__synth_instruction_sample()
438 	sample.phys_addr = record->phys_addr;  in arm_spe__synth_instruction_sample()
441 	sample.weight = record->latency;  in arm_spe__synth_instruction_sample()
467 	const struct arm_spe_record *record = &speq->decoder->record;  in arm_spe__sample_flags()  local
470 	if (record->op & ARM_SPE_OP_BRANCH_ERET) {  in arm_spe__sample_flags()
473 		if (record->type & ARM_SPE_BRANCH_MISS)  in arm_spe__sample_flags()
478 static void arm_spe__synth_data_source_common(const struct arm_spe_record *record,  in arm_spe__synth_data_source_common()  argument
493 	if (record->op & ARM_SPE_OP_ST) {  in arm_spe__synth_data_source_common()
500 	switch (record->source) {  in arm_spe__synth_data_source_common()
558 static void arm_spe__synth_data_source_ampereone(const struct arm_spe_record *record,  in arm_spe__synth_data_source_ampereone()  argument
563 	switch (record->source) {  in arm_spe__synth_data_source_ampereone()
584 				record->source);  in arm_spe__synth_data_source_ampereone()
588 	common_record.op = record->op;  in arm_spe__synth_data_source_ampereone()
597 static void arm_spe__synth_memory_level(const struct arm_spe_record *record,  in arm_spe__synth_memory_level()  argument
600 	if (record->type & (ARM_SPE_LLC_ACCESS | ARM_SPE_LLC_MISS)) {  in arm_spe__synth_memory_level()
603 		if (record->type & ARM_SPE_LLC_MISS)  in arm_spe__synth_memory_level()
607 	} else if (record->type & (ARM_SPE_L1D_ACCESS | ARM_SPE_L1D_MISS)) {  in arm_spe__synth_memory_level()
610 		if (record->type & ARM_SPE_L1D_MISS)  in arm_spe__synth_memory_level()
616 	if (record->type & ARM_SPE_REMOTE_ACCESS)  in arm_spe__synth_memory_level()
621 			      const struct arm_spe_record *record,  in arm_spe__synth_ds()  argument
633 		pr_warning_once("Old SPE metadata, re-record to improve decode accuracy\n");  in arm_spe__synth_ds()
661 			data_source_handles[i].ds_synth(record, data_src);  in arm_spe__synth_ds()
670 				      const struct arm_spe_record *record)  in arm_spe__synth_data_source()  argument
675 	if (!is_ldst_op(record->op))  in arm_spe__synth_data_source()
678 	if (record->op & ARM_SPE_OP_LD)  in arm_spe__synth_data_source()
680 	else if (record->op & ARM_SPE_OP_ST)  in arm_spe__synth_data_source()
685 	if (!arm_spe__synth_ds(speq, record, &data_src))  in arm_spe__synth_data_source()
686 		arm_spe__synth_memory_level(record, &data_src);  in arm_spe__synth_data_source()
688 	if (record->type & (ARM_SPE_TLB_ACCESS | ARM_SPE_TLB_MISS)) {  in arm_spe__synth_data_source()
691 		if (record->type & ARM_SPE_TLB_MISS)  in arm_spe__synth_data_source()
702 	const struct arm_spe_record *record = &speq->decoder->record;  in arm_spe_sample()  local
708 	data_src = arm_spe__synth_data_source(speq, record);  in arm_spe_sample()
711 		if (record->type & ARM_SPE_L1D_MISS) {  in arm_spe_sample()
718 		if (record->type & ARM_SPE_L1D_ACCESS) {  in arm_spe_sample()
727 		if (record->type & ARM_SPE_LLC_MISS) {  in arm_spe_sample()
734 		if (record->type & ARM_SPE_LLC_ACCESS) {  in arm_spe_sample()
743 		if (record->type & ARM_SPE_TLB_MISS) {  in arm_spe_sample()
750 		if (record->type & ARM_SPE_TLB_ACCESS) {  in arm_spe_sample()
758 	if (spe->sample_branch && (record->op & ARM_SPE_OP_BRANCH_ERET)) {  in arm_spe_sample()
765 	    (record->type & ARM_SPE_REMOTE_ACCESS)) {  in arm_spe_sample()
773 	 * When data_src is zero it means the record is not a memory operation,  in arm_spe_sample()
776 	if (spe->sample_memory && is_ldst_op(record->op)) {  in arm_spe_sample()
794 	struct arm_spe_record *record;  in arm_spe_run_decoder()  local
803 		 * based the record to synthesize sample; but here the flow is  in arm_spe_run_decoder()
809 		 * has decoded trace data and generated a record, but the record  in arm_spe_run_decoder()
811 		 * to synthesize sample for the left record.  in arm_spe_run_decoder()
812 		 * 2. After decoding trace data, it needs to compare the record  in arm_spe_run_decoder()
813 		 * timestamp with the coming perf event, if the record timestamp  in arm_spe_run_decoder()
815 		 * record into auxtrace heap, thus the record can be deferred to  in arm_spe_run_decoder()
824 		record = &speq->decoder->record;  in arm_spe_run_decoder()
825 		if (!spe->timeless_decoding && record->context_id != (u64)-1) {  in arm_spe_run_decoder()
826 			ret = arm_spe_set_tid(speq, record->context_id);  in arm_spe_run_decoder()
850 		record = &speq->decoder->record;  in arm_spe_run_decoder()
852 		/* Update timestamp for the last record */  in arm_spe_run_decoder()
853 		if (record->timestamp > speq->timestamp)  in arm_spe_run_decoder()
854 			speq->timestamp = record->timestamp;  in arm_spe_run_decoder()
875 	struct arm_spe_record *record;  in arm_spe__setup_queue()  local
905 		record = &speq->decoder->record;  in arm_spe__setup_queue()
907 		speq->timestamp = record->timestamp;  in arm_spe__setup_queue()