70 static unsigned int fq_codel_hash(const struct fq_codel_sched_data *q,  in fq_codel_hash()  argument
73 	return reciprocal_scale(skb_get_hash(skb), q->flows_cnt);  in fq_codel_hash()
79 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_classify()  local
86 	    TC_H_MIN(skb->priority) <= q->flows_cnt)  in fq_codel_classify()
89 	filter = rcu_dereference_bh(q->filter_list);  in fq_codel_classify()
91 		return fq_codel_hash(q, skb) + 1;  in fq_codel_classify()
107 		if (TC_H_MIN(res.classid) <= q->flows_cnt)  in fq_codel_classify()
140 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_drop()  local
154 	for (i = 0; i < q->flows_cnt; i++) {  in fq_codel_drop()
155 		if (q->backlogs[i] > maxbacklog) {  in fq_codel_drop()
156 			maxbacklog = q->backlogs[i];  in fq_codel_drop()
164 	flow = &q->flows[idx];  in fq_codel_drop()
177 	q->backlogs[idx] -= len;  in fq_codel_drop()
178 	q->memory_usage -= mem;  in fq_codel_drop()
181 	sch->q.qlen -= i;  in fq_codel_drop()
188 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_enqueue()  local
205 	flow = &q->flows[idx];  in fq_codel_enqueue()
207 	q->backlogs[idx] += qdisc_pkt_len(skb);  in fq_codel_enqueue()
211 		list_add_tail(&flow->flowchain, &q->new_flows);  in fq_codel_enqueue()
212 		q->new_flow_count++;  in fq_codel_enqueue()
213 		flow->deficit = q->quantum;  in fq_codel_enqueue()
216 	q->memory_usage += get_codel_cb(skb)->mem_usage;  in fq_codel_enqueue()
217 	memory_limited = q->memory_usage > q->memory_limit;  in fq_codel_enqueue()
218 	if (++sch->q.qlen <= sch->limit && !memory_limited)  in fq_codel_enqueue()
222 	prev_qlen = sch->q.qlen;  in fq_codel_enqueue()
227 	 * in q->backlogs[] to find a fat flow.  in fq_codel_enqueue()
231 	ret = fq_codel_drop(sch, q->drop_batch_size, to_free);  in fq_codel_enqueue()
233 	prev_qlen -= sch->q.qlen;  in fq_codel_enqueue()
235 	q->drop_overlimit += prev_qlen;  in fq_codel_enqueue()
237 		q->drop_overmemory += prev_qlen;  in fq_codel_enqueue()
259 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in dequeue_func()  local
266 		q->backlogs[flow - q->flows] -= qdisc_pkt_len(skb);  in dequeue_func()
267 		q->memory_usage -= get_codel_cb(skb)->mem_usage;  in dequeue_func()
268 		sch->q.qlen--;  in dequeue_func()
284 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_dequeue()  local
290 	head = &q->new_flows;  in fq_codel_dequeue()
292 		head = &q->old_flows;  in fq_codel_dequeue()
299 		flow->deficit += q->quantum;  in fq_codel_dequeue()
300 		list_move_tail(&flow->flowchain, &q->old_flows);  in fq_codel_dequeue()
304 	skb = codel_dequeue(sch, &sch->qstats.backlog, &q->cparams,  in fq_codel_dequeue()
305 			    &flow->cvars, &q->cstats, qdisc_pkt_len,  in fq_codel_dequeue()
310 		if ((head == &q->new_flows) && !list_empty(&q->old_flows))  in fq_codel_dequeue()
311 			list_move_tail(&flow->flowchain, &q->old_flows);  in fq_codel_dequeue()
319 	if (q->cstats.drop_count) {  in fq_codel_dequeue()
320 		qdisc_tree_reduce_backlog(sch, q->cstats.drop_count,  in fq_codel_dequeue()
321 					  q->cstats.drop_len);  in fq_codel_dequeue()
322 		q->cstats.drop_count = 0;  in fq_codel_dequeue()
323 		q->cstats.drop_len = 0;  in fq_codel_dequeue()
336 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_reset()  local
339 	INIT_LIST_HEAD(&q->new_flows);  in fq_codel_reset()
340 	INIT_LIST_HEAD(&q->old_flows);  in fq_codel_reset()
341 	for (i = 0; i < q->flows_cnt; i++) {  in fq_codel_reset()
342 		struct fq_codel_flow *flow = q->flows + i;  in fq_codel_reset()
348 	memset(q->backlogs, 0, q->flows_cnt * sizeof(u32));  in fq_codel_reset()
349 	q->memory_usage = 0;  in fq_codel_reset()
369 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_change()  local
379 		if (q->flows)  in fq_codel_change()
381 		q->flows_cnt = nla_get_u32(tb[TCA_FQ_CODEL_FLOWS]);  in fq_codel_change()
382 		if (!q->flows_cnt ||  in fq_codel_change()
383 		    q->flows_cnt > 65536)  in fq_codel_change()
398 		WRITE_ONCE(q->cparams.target,  in fq_codel_change()
405 		WRITE_ONCE(q->cparams.ce_threshold,  in fq_codel_change()
410 		WRITE_ONCE(q->cparams.ce_threshold_selector,  in fq_codel_change()
413 		WRITE_ONCE(q->cparams.ce_threshold_mask,  in fq_codel_change()
419 		WRITE_ONCE(q->cparams.interval,  in fq_codel_change()
428 		WRITE_ONCE(q->cparams.ecn,  in fq_codel_change()
432 		WRITE_ONCE(q->quantum, quantum);  in fq_codel_change()
435 		WRITE_ONCE(q->drop_batch_size,  in fq_codel_change()
439 		WRITE_ONCE(q->memory_limit,  in fq_codel_change()
442 	while (sch->q.qlen > sch->limit ||  in fq_codel_change()
443 	       q->memory_usage > q->memory_limit) {  in fq_codel_change()
446 		q->cstats.drop_len += qdisc_pkt_len(skb);  in fq_codel_change()
448 		q->cstats.drop_count++;  in fq_codel_change()
450 	qdisc_tree_reduce_backlog(sch, q->cstats.drop_count, q->cstats.drop_len);  in fq_codel_change()
451 	q->cstats.drop_count = 0;  in fq_codel_change()
452 	q->cstats.drop_len = 0;  in fq_codel_change()
460 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_destroy()  local
462 	tcf_block_put(q->block);  in fq_codel_destroy()
463 	kvfree(q->backlogs);  in fq_codel_destroy()
464 	kvfree(q->flows);  in fq_codel_destroy()
470 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_init()  local
475 	q->flows_cnt = 1024;  in fq_codel_init()
476 	q->memory_limit = 32 << 20; /* 32 MBytes */  in fq_codel_init()
477 	q->drop_batch_size = 64;  in fq_codel_init()
478 	q->quantum = psched_mtu(qdisc_dev(sch));  in fq_codel_init()
479 	INIT_LIST_HEAD(&q->new_flows);  in fq_codel_init()
480 	INIT_LIST_HEAD(&q->old_flows);  in fq_codel_init()
481 	codel_params_init(&q->cparams);  in fq_codel_init()
482 	codel_stats_init(&q->cstats);  in fq_codel_init()
483 	q->cparams.ecn = true;  in fq_codel_init()
484 	q->cparams.mtu = psched_mtu(qdisc_dev(sch));  in fq_codel_init()
492 	err = tcf_block_get(&q->block, &q->filter_list, sch, extack);  in fq_codel_init()
496 	if (!q->flows) {  in fq_codel_init()
497 		q->flows = kvcalloc(q->flows_cnt,  in fq_codel_init()
500 		if (!q->flows) {  in fq_codel_init()
504 		q->backlogs = kvcalloc(q->flows_cnt, sizeof(u32), GFP_KERNEL);  in fq_codel_init()
505 		if (!q->backlogs) {  in fq_codel_init()
509 		for (i = 0; i < q->flows_cnt; i++) {  in fq_codel_init()
510 			struct fq_codel_flow *flow = q->flows + i;  in fq_codel_init()
523 	kvfree(q->flows);  in fq_codel_init()
524 	q->flows = NULL;  in fq_codel_init()
526 	q->flows_cnt = 0;  in fq_codel_init()
532 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_dump()  local
541 			codel_time_to_us(READ_ONCE(q->cparams.target))) ||  in fq_codel_dump()
545 			codel_time_to_us(READ_ONCE(q->cparams.interval))) ||  in fq_codel_dump()
547 			READ_ONCE(q->cparams.ecn)) ||  in fq_codel_dump()
549 			READ_ONCE(q->quantum)) ||  in fq_codel_dump()
551 			READ_ONCE(q->drop_batch_size)) ||  in fq_codel_dump()
553 			READ_ONCE(q->memory_limit)) ||  in fq_codel_dump()
555 			READ_ONCE(q->flows_cnt)))  in fq_codel_dump()
558 	ce_threshold = READ_ONCE(q->cparams.ce_threshold);  in fq_codel_dump()
564 			       READ_ONCE(q->cparams.ce_threshold_selector)))  in fq_codel_dump()
567 			       READ_ONCE(q->cparams.ce_threshold_mask)))  in fq_codel_dump()
579 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_dump_stats()  local
585 	st.qdisc_stats.maxpacket = q->cstats.maxpacket;  in fq_codel_dump_stats()
586 	st.qdisc_stats.drop_overlimit = q->drop_overlimit;  in fq_codel_dump_stats()
587 	st.qdisc_stats.ecn_mark = q->cstats.ecn_mark;  in fq_codel_dump_stats()
588 	st.qdisc_stats.new_flow_count = q->new_flow_count;  in fq_codel_dump_stats()
589 	st.qdisc_stats.ce_mark = q->cstats.ce_mark;  in fq_codel_dump_stats()
590 	st.qdisc_stats.memory_usage  = q->memory_usage;  in fq_codel_dump_stats()
591 	st.qdisc_stats.drop_overmemory = q->drop_overmemory;  in fq_codel_dump_stats()
594 	list_for_each(pos, &q->new_flows)  in fq_codel_dump_stats()
597 	list_for_each(pos, &q->old_flows)  in fq_codel_dump_stats()
620 static void fq_codel_unbind(struct Qdisc *q, unsigned long cl)  in fq_codel_unbind()  argument
627 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_tcf_block()  local
631 	return q->block;  in fq_codel_tcf_block()
644 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_dump_class_stats()  local
649 	if (idx < q->flows_cnt) {  in fq_codel_dump_class_stats()
650 		const struct fq_codel_flow *flow = &q->flows[idx];  in fq_codel_dump_class_stats()
678 		qs.backlog = q->backlogs[idx];  in fq_codel_dump_class_stats()
683 	if (idx < q->flows_cnt)  in fq_codel_dump_class_stats()
690 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_walk()  local
696 	for (i = 0; i < q->flows_cnt; i++) {  in fq_codel_walk()
697 		if (list_empty(&q->flows[i].flowchain)) {  in fq_codel_walk()