112 	struct perf_output_handle handle; /* AUX buffer output handle */
805 	/* Use AUX buffer. No need to allocate it by ourself */  in __hw_perf_event_init()
1183 	 * AUX buffer is used when in diagnostic sampling mode.  in hw_perf_event_update()
1259 static inline unsigned long aux_sdb_index(struct aux_buffer *aux,  in aux_sdb_index()  argument
1262 	return i % aux->sfb.num_sdb;  in aux_sdb_index()
1270 static inline unsigned long aux_sdb_num_alert(struct aux_buffer *aux)  in aux_sdb_num_alert()  argument
1272 	return aux_sdb_num(aux->head, aux->alert_mark);  in aux_sdb_num_alert()
1275 static inline unsigned long aux_sdb_num_empty(struct aux_buffer *aux)  in aux_sdb_num_empty()  argument
1277 	return aux_sdb_num(aux->head, aux->empty_mark);  in aux_sdb_num_empty()
1283 static struct hws_trailer_entry *aux_sdb_trailer(struct aux_buffer *aux,  in aux_sdb_trailer()  argument
1288 	index = aux_sdb_index(aux, index);  in aux_sdb_trailer()
1289 	sdb = aux->sdb_index[index];  in aux_sdb_trailer()
1295  * disabled. Collect the full SDBs in AUX buffer which have not reached
1305 	struct aux_buffer *aux;  in aux_output_end()  local
1308 	aux = perf_get_aux(handle);  in aux_output_end()
1309 	if (!aux)  in aux_output_end()
1312 	range_scan = aux_sdb_num_alert(aux);  in aux_output_end()
1313 	for (i = 0, idx = aux->head; i < range_scan; i++, idx++) {  in aux_output_end()
1314 		te = aux_sdb_trailer(aux, idx);  in aux_output_end()
1322 	te = aux_sdb_trailer(aux, aux->alert_mark);  in aux_output_end()
1336 			    struct aux_buffer *aux,  in aux_output_begin()  argument
1345 	aux->head = handle->head >> PAGE_SHIFT;  in aux_output_begin()
1351 	 * SDBs between aux->head and aux->empty_mark are already ready  in aux_output_begin()
1354 	if (range > aux_sdb_num_empty(aux)) {  in aux_output_begin()
1355 		range_scan = range - aux_sdb_num_empty(aux);  in aux_output_begin()
1356 		idx = aux->empty_mark + 1;  in aux_output_begin()
1358 			te = aux_sdb_trailer(aux, idx);  in aux_output_begin()
1364 		aux->empty_mark = aux->head + range - 1;  in aux_output_begin()
1368 	aux->alert_mark = aux->head + range/2 - 1;  in aux_output_begin()
1369 	te = aux_sdb_trailer(aux, aux->alert_mark);  in aux_output_begin()
1373 	head = aux_sdb_index(aux, aux->head);  in aux_output_begin()
1374 	base = aux->sdbt_index[head / CPUM_SF_SDB_PER_TABLE];  in aux_output_begin()
1377 	cpuhw->lsctl.dear = virt_to_phys((void *)aux->sdb_index[head]);  in aux_output_begin()
1388 static bool aux_set_alert(struct aux_buffer *aux, unsigned long alert_index,  in aux_set_alert()  argument
1394 	te = aux_sdb_trailer(aux, alert_index);  in aux_set_alert()
1415  * @aux:	The AUX buffer to set
1416  * @range:	The range of SDBs to scan started from aux->head
1419  * Set alert indicator on the SDB at index of aux->alert_mark. If this SDB is
1424  * Scan the SDBs in AUX buffer from behind aux->empty_mark. They are used
1427  * If aux->alert_mark fall in this area, just set it. Overflow count is
1430  * SDBs between aux->head and aux->empty_mark are already reset at last time.
1435 static bool aux_reset_buffer(struct aux_buffer *aux, unsigned long range,  in aux_reset_buffer()  argument
1443 	if (range <= aux_sdb_num_empty(aux))  in aux_reset_buffer()
1449 		return aux_set_alert(aux, aux->alert_mark, overflow);  in aux_reset_buffer()
1451 	if (aux->alert_mark <= aux->empty_mark)  in aux_reset_buffer()
1456 		if (!aux_set_alert(aux, aux->alert_mark, overflow))  in aux_reset_buffer()
1464 	range_scan = range - aux_sdb_num_empty(aux);  in aux_reset_buffer()
1465 	idx = aux->empty_mark + 1;  in aux_reset_buffer()
1467 		te = aux_sdb_trailer(aux, idx);  in aux_reset_buffer()
1474 			if (idx == aux->alert_mark)  in aux_reset_buffer()
1483 	aux->empty_mark = aux->head + range - 1;  in aux_reset_buffer()
1493 	struct aux_buffer *aux;  in hw_collect_aux()  local
1500 	aux = perf_get_aux(handle);  in hw_collect_aux()
1501 	if (!aux)  in hw_collect_aux()
1505 	size = aux_sdb_num_alert(aux) << PAGE_SHIFT;  in hw_collect_aux()
1510 	num_sdb = aux->sfb.num_sdb;  in hw_collect_aux()
1513 		aux = perf_aux_output_begin(handle, cpuhw->event);  in hw_collect_aux()
1515 			pr_err("The AUX buffer with %lu pages for the "  in hw_collect_aux()
1520 		if (!aux)  in hw_collect_aux()
1524 		aux->head = handle->head >> PAGE_SHIFT;  in hw_collect_aux()
1527 			aux->alert_mark = aux->head;  in hw_collect_aux()
1529 			aux->alert_mark = aux->head + range/2 - 1;  in hw_collect_aux()
1531 		if (aux_reset_buffer(aux, range, &overflow)) {  in hw_collect_aux()
1538 			pr_err("Sample data caused the AUX buffer with %lu "  in hw_collect_aux()
1539 			       "pages to overflow\n", aux->sfb.num_sdb);  in hw_collect_aux()
1541 			size = aux_sdb_num_alert(aux) << PAGE_SHIFT;  in hw_collect_aux()
1548  * Callback when freeing AUX buffers.
1552 	struct aux_buffer *aux = data;  in aux_buffer_free()  local
1555 	if (!aux)  in aux_buffer_free()
1559 	num_sdbt = aux->sfb.num_sdbt;  in aux_buffer_free()
1561 		free_page(aux->sdbt_index[i]);  in aux_buffer_free()
1563 	kfree(aux->sdbt_index);  in aux_buffer_free()
1564 	kfree(aux->sdb_index);  in aux_buffer_free()
1565 	kfree(aux);  in aux_buffer_free()
1580  * aux_buffer_setup() - Setup AUX buffer for diagnostic mode sampling
1586  * This is the callback when setup an event using AUX buffer. Perf tool can
1588  * for basic samples, AUX buffer belongs to the event. It is scheduled with
1591  * Return the private AUX buffer structure if success or NULL if fails.
1597 	struct aux_buffer *aux;  in aux_buffer_setup()  local
1605 		pr_err("AUX buffer size (%i pages) is larger than the "  in aux_buffer_setup()
1610 		pr_err("AUX buffer size (%i pages) is less than the "  in aux_buffer_setup()
1617 	aux = kzalloc(sizeof(struct aux_buffer), GFP_KERNEL);  in aux_buffer_setup()
1618 	if (!aux)  in aux_buffer_setup()
1620 	sfb = &aux->sfb;  in aux_buffer_setup()
1624 	aux->sdbt_index = kmalloc_array(n_sdbt, sizeof(void *), GFP_KERNEL);  in aux_buffer_setup()
1625 	if (!aux->sdbt_index)  in aux_buffer_setup()
1629 	aux->sdb_index = kmalloc_array(nr_pages, sizeof(void *), GFP_KERNEL);  in aux_buffer_setup()
1630 	if (!aux->sdb_index)  in aux_buffer_setup()
1638 	aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)sfb->sdbt;  in aux_buffer_setup()
1642 	 * Link the provided pages of AUX buffer to SDBT.  in aux_buffer_setup()
1650 			aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)new;  in aux_buffer_setup()
1657 		aux->sdb_index[i] = (unsigned long)pages[i];  in aux_buffer_setup()
1671 	aux->empty_mark = sfb->num_sdb - 1;  in aux_buffer_setup()
1673 	return aux;  in aux_buffer_setup()
1676 	/* SDBs (AUX buffer pages) are freed by caller */  in aux_buffer_setup()
1678 		free_page(aux->sdbt_index[i]);  in aux_buffer_setup()
1679 	kfree(aux->sdb_index);  in aux_buffer_setup()
1681 	kfree(aux->sdbt_index);  in aux_buffer_setup()
1683 	kfree(aux);  in aux_buffer_setup()
1770 	struct aux_buffer *aux;  in cpumsf_pmu_add()  local
1804 		aux = perf_aux_output_begin(&cpuhw->handle, event);  in cpumsf_pmu_add()
1805 		if (!aux) {  in cpumsf_pmu_add()
1809 		err = aux_output_begin(&cpuhw->handle, aux, cpuhw);  in cpumsf_pmu_add()