75 static unsigned int fq_pie_hash(const struct fq_pie_sched_data *q,  in fq_pie_hash()  argument
78 	return reciprocal_scale(skb_get_hash(skb), q->flows_cnt);  in fq_pie_hash()
84 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_classify()  local
91 	    TC_H_MIN(skb->priority) <= q->flows_cnt)  in fq_pie_classify()
94 	filter = rcu_dereference_bh(q->filter_list);  in fq_pie_classify()
96 		return fq_pie_hash(q, skb) + 1;  in fq_pie_classify()
112 		if (TC_H_MIN(res.classid) <= q->flows_cnt)  in fq_pie_classify()
134 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_qdisc_enqueue()  local
152 	sel_flow = &q->flows[idx];  in fq_pie_qdisc_enqueue()
155 	memory_limited = q->memory_usage > q->memory_limit + skb->truesize;  in fq_pie_qdisc_enqueue()
159 		q->stats.overlimit++;  in fq_pie_qdisc_enqueue()
162 		q->overmemory++;  in fq_pie_qdisc_enqueue()
167 	if (!pie_drop_early(sch, &q->p_params, &sel_flow->vars,  in fq_pie_qdisc_enqueue()
170 	} else if (q->p_params.ecn &&  in fq_pie_qdisc_enqueue()
171 		   sel_flow->vars.prob <= (MAX_PROB / 100) * q->ecn_prob &&  in fq_pie_qdisc_enqueue()
176 		q->stats.ecn_mark++;  in fq_pie_qdisc_enqueue()
181 		if (!q->p_params.dq_rate_estimator)  in fq_pie_qdisc_enqueue()
185 		q->stats.packets_in++;  in fq_pie_qdisc_enqueue()
186 		q->memory_usage += skb->truesize;  in fq_pie_qdisc_enqueue()
188 		sch->q.qlen++;  in fq_pie_qdisc_enqueue()
191 			list_add_tail(&sel_flow->flowchain, &q->new_flows);  in fq_pie_qdisc_enqueue()
192 			q->new_flow_count++;  in fq_pie_qdisc_enqueue()
193 			sel_flow->deficit = q->quantum;  in fq_pie_qdisc_enqueue()
202 	q->stats.dropped++;  in fq_pie_qdisc_enqueue()
240 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_qdisc_dequeue()  local
247 	head = &q->new_flows;  in fq_pie_qdisc_dequeue()
249 		head = &q->old_flows;  in fq_pie_qdisc_dequeue()
257 		flow->deficit += q->quantum;  in fq_pie_qdisc_dequeue()
258 		list_move_tail(&flow->flowchain, &q->old_flows);  in fq_pie_qdisc_dequeue()
266 		sch->q.qlen--;  in fq_pie_qdisc_dequeue()
272 		if (head == &q->new_flows && !list_empty(&q->old_flows))  in fq_pie_qdisc_dequeue()
273 			list_move_tail(&flow->flowchain, &q->old_flows);  in fq_pie_qdisc_dequeue()
282 	q->memory_usage -= get_pie_cb(skb)->mem_usage;  in fq_pie_qdisc_dequeue()
283 	pie_process_dequeue(skb, &q->p_params, &flow->vars, flow->backlog);  in fq_pie_qdisc_dequeue()
290 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_change()  local
304 		WRITE_ONCE(q->p_params.limit, limit);  in fq_pie_change()
308 		if (q->flows) {  in fq_pie_change()
313 		q->flows_cnt = nla_get_u32(tb[TCA_FQ_PIE_FLOWS]);  in fq_pie_change()
314 		if (!q->flows_cnt || q->flows_cnt > 65536) {  in fq_pie_change()
327 		WRITE_ONCE(q->p_params.target,  in fq_pie_change()
333 		WRITE_ONCE(q->p_params.tupdate,  in fq_pie_change()
337 		WRITE_ONCE(q->p_params.alpha,  in fq_pie_change()
341 		WRITE_ONCE(q->p_params.beta,  in fq_pie_change()
345 		WRITE_ONCE(q->quantum, nla_get_u32(tb[TCA_FQ_PIE_QUANTUM]));  in fq_pie_change()
348 		WRITE_ONCE(q->memory_limit,  in fq_pie_change()
352 		WRITE_ONCE(q->ecn_prob,  in fq_pie_change()
356 		WRITE_ONCE(q->p_params.ecn,  in fq_pie_change()
360 		WRITE_ONCE(q->p_params.bytemode,  in fq_pie_change()
364 		WRITE_ONCE(q->p_params.dq_rate_estimator,  in fq_pie_change()
368 	while (sch->q.qlen > sch->limit) {  in fq_pie_change()
387 	struct fq_pie_sched_data *q = from_timer(q, t, adapt_timer);  in fq_pie_timer()  local
389 	struct Qdisc *sch = q->sch;  in fq_pie_timer()
398 	max_cnt = min_t(int, q->flows_cnt - q->flows_cursor, 2048);  in fq_pie_timer()
400 		pie_calculate_probability(&q->p_params,  in fq_pie_timer()
401 					  &q->flows[q->flows_cursor].vars,  in fq_pie_timer()
402 					  q->flows[q->flows_cursor].backlog);  in fq_pie_timer()
403 		q->flows_cursor++;  in fq_pie_timer()
406 	tupdate = q->p_params.tupdate;  in fq_pie_timer()
408 	if (q->flows_cursor >= q->flows_cnt) {  in fq_pie_timer()
409 		q->flows_cursor = 0;  in fq_pie_timer()
413 		mod_timer(&q->adapt_timer, jiffies + next);  in fq_pie_timer()
421 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_init()  local
425 	pie_params_init(&q->p_params);  in fq_pie_init()
427 	q->p_params.limit = sch->limit;  in fq_pie_init()
428 	q->quantum = psched_mtu(qdisc_dev(sch));  in fq_pie_init()
429 	q->sch = sch;  in fq_pie_init()
430 	q->ecn_prob = 10;  in fq_pie_init()
431 	q->flows_cnt = 1024;  in fq_pie_init()
432 	q->memory_limit = SZ_32M;  in fq_pie_init()
434 	INIT_LIST_HEAD(&q->new_flows);  in fq_pie_init()
435 	INIT_LIST_HEAD(&q->old_flows);  in fq_pie_init()
436 	timer_setup(&q->adapt_timer, fq_pie_timer, 0);  in fq_pie_init()
445 	err = tcf_block_get(&q->block, &q->filter_list, sch, extack);  in fq_pie_init()
449 	q->flows = kvcalloc(q->flows_cnt, sizeof(struct fq_pie_flow),  in fq_pie_init()
451 	if (!q->flows) {  in fq_pie_init()
455 	for (idx = 0; idx < q->flows_cnt; idx++) {  in fq_pie_init()
456 		struct fq_pie_flow *flow = q->flows + idx;  in fq_pie_init()
462 	mod_timer(&q->adapt_timer, jiffies + HZ / 2);  in fq_pie_init()
467 	q->flows_cnt = 0;  in fq_pie_init()
474 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_dump()  local
483 	    nla_put_u32(skb, TCA_FQ_PIE_FLOWS, READ_ONCE(q->flows_cnt)) ||  in fq_pie_dump()
485 			((u32)PSCHED_TICKS2NS(READ_ONCE(q->p_params.target))) /  in fq_pie_dump()
488 			jiffies_to_usecs(READ_ONCE(q->p_params.tupdate))) ||  in fq_pie_dump()
489 	    nla_put_u32(skb, TCA_FQ_PIE_ALPHA, READ_ONCE(q->p_params.alpha)) ||  in fq_pie_dump()
490 	    nla_put_u32(skb, TCA_FQ_PIE_BETA, READ_ONCE(q->p_params.beta)) ||  in fq_pie_dump()
491 	    nla_put_u32(skb, TCA_FQ_PIE_QUANTUM, READ_ONCE(q->quantum)) ||  in fq_pie_dump()
493 			READ_ONCE(q->memory_limit)) ||  in fq_pie_dump()
494 	    nla_put_u32(skb, TCA_FQ_PIE_ECN_PROB, READ_ONCE(q->ecn_prob)) ||  in fq_pie_dump()
495 	    nla_put_u32(skb, TCA_FQ_PIE_ECN, READ_ONCE(q->p_params.ecn)) ||  in fq_pie_dump()
496 	    nla_put_u32(skb, TCA_FQ_PIE_BYTEMODE, READ_ONCE(q->p_params.bytemode)) ||  in fq_pie_dump()
498 			READ_ONCE(q->p_params.dq_rate_estimator)))  in fq_pie_dump()
510 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_dump_stats()  local
512 		.packets_in	= q->stats.packets_in,  in fq_pie_dump_stats()
513 		.overlimit	= q->stats.overlimit,  in fq_pie_dump_stats()
514 		.overmemory	= q->overmemory,  in fq_pie_dump_stats()
515 		.dropped	= q->stats.dropped,  in fq_pie_dump_stats()
516 		.ecn_mark	= q->stats.ecn_mark,  in fq_pie_dump_stats()
517 		.new_flow_count = q->new_flow_count,  in fq_pie_dump_stats()
518 		.memory_usage   = q->memory_usage,  in fq_pie_dump_stats()
523 	list_for_each(pos, &q->new_flows)  in fq_pie_dump_stats()
526 	list_for_each(pos, &q->old_flows)  in fq_pie_dump_stats()
535 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_reset()  local
538 	INIT_LIST_HEAD(&q->new_flows);  in fq_pie_reset()
539 	INIT_LIST_HEAD(&q->old_flows);  in fq_pie_reset()
540 	for (idx = 0; idx < q->flows_cnt; idx++) {  in fq_pie_reset()
541 		struct fq_pie_flow *flow = q->flows + idx;  in fq_pie_reset()
554 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_destroy()  local
556 	tcf_block_put(q->block);  in fq_pie_destroy()
557 	q->p_params.tupdate = 0;  in fq_pie_destroy()
558 	del_timer_sync(&q->adapt_timer);  in fq_pie_destroy()
559 	kvfree(q->flows);  in fq_pie_destroy()