Prepare mutable list iterators to cache cursor state

block/bfq-iosched.c

@@ -1209,9 +1209,8 @@ static int bfqq_process_refs(struct bfq_queue *bfqq)
 static void bfq_reset_burst_list(struct bfq_data *bfqd, struct bfq_queue *bfqq)
 {
 	struct bfq_queue *item;
-	struct hlist_node *n;
 
-	hlist_for_each_entry_safe(item, n, &bfqd->burst_list, burst_list_node)
+	hlist_for_each_entry_mutable(item, &bfqd->burst_list, burst_list_node)
 		hlist_del_init(&item->burst_list_node);
 
 	/*
@@ -1236,7 +1235,6 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq)
 
 	if (bfqd->burst_size == bfqd->bfq_large_burst_thresh) {
 		struct bfq_queue *pos, *bfqq_item;
-		struct hlist_node *n;
 
 		/*
 		 * Enough queues have been activated shortly after each
@@ -1260,8 +1258,8 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq)
 		 * belonging to a large burst. So the burst list is not
 		 * needed any more. Remove it.
 		 */
-		hlist_for_each_entry_safe(pos, n, &bfqd->burst_list,
-					  burst_list_node)
+		hlist_for_each_entry_mutable(pos, &bfqd->burst_list,
+					     burst_list_node)
 			hlist_del_init(&pos->burst_list_node);
 	} else /*
 		* Burst not yet large: add bfqq to the burst list. Do
@@ -5330,7 +5328,6 @@ static struct request *bfq_dispatch_request(struct blk_mq_hw_ctx *hctx)
 void bfq_put_queue(struct bfq_queue *bfqq)
 {
 	struct bfq_queue *item;
-	struct hlist_node *n;
 	struct bfq_group *bfqg = bfqq_group(bfqq);
 
 	bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", bfqq, bfqq->ref);
@@ -5391,8 +5388,8 @@ void bfq_put_queue(struct bfq_queue *bfqq)
 		hlist_del_init(&bfqq->woken_list_node);
 
 	/* reset waker for all queues in woken list */
-	hlist_for_each_entry_safe(item, n, &bfqq->woken_list,
-				  woken_list_node) {
+	hlist_for_each_entry_mutable(item, &bfqq->woken_list,
+				     woken_list_node) {
 		item->waker_bfqq = NULL;
 		hlist_del_init(&item->woken_list_node);
 	}
@@ -7141,13 +7138,13 @@ static void bfq_depth_updated(struct request_queue *q)
 static void bfq_exit_queue(struct elevator_queue *e)
 {
 	struct bfq_data *bfqd = e->elevator_data;
-	struct bfq_queue *bfqq, *n;
+	struct bfq_queue *bfqq;
 	unsigned int actuator;
 
 	hrtimer_cancel(&bfqd->idle_slice_timer);
 
 	spin_lock_irq(&bfqd->lock);
-	list_for_each_entry_safe(bfqq, n, &bfqd->idle_list, bfqq_list)
+	list_for_each_entry_mutable(bfqq, &bfqd->idle_list, bfqq_list)
 		bfq_deactivate_bfqq(bfqd, bfqq, false, false);
 	spin_unlock_irq(&bfqd->lock);
 

block/blk-cgroup.c

@@ -997,7 +997,7 @@ static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu)
 {
 	struct llist_head *lhead = per_cpu_ptr(blkcg->lhead, cpu);
 	struct llist_node *lnode;
-	struct blkg_iostat_set *bisc, *next_bisc;
+	struct blkg_iostat_set *bisc;
 	unsigned long flags;
 
 	rcu_read_lock();
@@ -1017,17 +1017,17 @@ static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu)
 	/*
 	 * Iterate only the iostat_cpu's queued in the lockless list.
 	 */
-	llist_for_each_entry_safe(bisc, next_bisc, lnode, lnode) {
+	llist_for_each_entry_mutable(bisc, lnode, lnode) {
 		struct blkcg_gq *blkg = bisc->blkg;
 		struct blkcg_gq *parent = blkg->parent;
 		struct blkg_iostat cur;
 		unsigned int seq;
 
 		/*
-		 * Order assignment of `next_bisc` from `bisc->lnode.next` in
-		 * llist_for_each_entry_safe and clearing `bisc->lqueued` for
-		 * avoiding to assign `next_bisc` with new next pointer added
-		 * in blk_cgroup_bio_start() in case of re-ordering.
+		 * Order the iterator's internal `bisc->lnode.next` load before
+		 * clearing `bisc->lqueued`, so the iterator can't pick up a new
+		 * next pointer added in blk_cgroup_bio_start() in case of
+		 * re-ordering.
 		 *
 		 * The pair barrier is implied in llist_add() in blk_cgroup_bio_start().
 		 */

block/blk-flush.c

@@ -204,7 +204,7 @@ static enum rq_end_io_ret flush_end_io(struct request *flush_rq,
 {
 	struct request_queue *q = flush_rq->q;
 	struct list_head *running;
-	struct request *rq, *n;
+	struct request *rq;
 	unsigned long flags = 0;
 	struct blk_flush_queue *fq = blk_get_flush_queue(flush_rq->mq_ctx);
 
@@ -243,7 +243,7 @@ static enum rq_end_io_ret flush_end_io(struct request *flush_rq,
 	fq->flush_running_idx ^= 1;
 
 	/* and push the waiting requests to the next stage */
-	list_for_each_entry_safe(rq, n, running, queuelist) {
+	list_for_each_entry_mutable(rq, running, queuelist) {
 		unsigned int seq = blk_flush_cur_seq(rq);
 
 		BUG_ON(seq != REQ_FSEQ_PREFLUSH && seq != REQ_FSEQ_POSTFLUSH);

block/blk-iocost.c

@@ -1718,7 +1718,7 @@ static void iocg_flush_stat_leaf(struct ioc_gq *iocg, struct ioc_now *now)
 static void iocg_flush_stat(struct list_head *target_iocgs, struct ioc_now *now)
 {
 	LIST_HEAD(inner_walk);
-	struct ioc_gq *iocg, *tiocg;
+	struct ioc_gq *iocg;
 
 	/* flush leaves and build inner node walk list */
 	list_for_each_entry(iocg, target_iocgs, active_list) {
@@ -1727,7 +1727,7 @@ static void iocg_flush_stat(struct list_head *target_iocgs, struct ioc_now *now)
 	}
 
 	/* keep flushing upwards by walking the inner list backwards */
-	list_for_each_entry_safe_reverse(iocg, tiocg, &inner_walk, walk_list) {
+	list_for_each_entry_mutable_reverse(iocg, &inner_walk, walk_list) {
 		iocg_flush_stat_upward(iocg);
 		list_del_init(&iocg->walk_list);
 	}
@@ -1848,7 +1848,7 @@ static void transfer_surpluses(struct list_head *surpluses, struct ioc_now *now)
 {
 	LIST_HEAD(over_hwa);
 	LIST_HEAD(inner_walk);
-	struct ioc_gq *iocg, *tiocg, *root_iocg;
+	struct ioc_gq *iocg, *root_iocg;
 	u32 after_sum, over_sum, over_target, gamma;
 
 	/*
@@ -1884,7 +1884,7 @@ static void transfer_surpluses(struct list_head *surpluses, struct ioc_now *now)
 		over_target = 0;
 	}
 
-	list_for_each_entry_safe(iocg, tiocg, &over_hwa, walk_list) {
+	list_for_each_entry_mutable(iocg, &over_hwa, walk_list) {
 		if (over_target)
 			iocg->hweight_after_donation =
 				div_u64((u64)iocg->hweight_after_donation *
@@ -2055,7 +2055,7 @@ static void transfer_surpluses(struct list_head *surpluses, struct ioc_now *now)
 	}
 
 	/* walk list should be dissolved after use */
-	list_for_each_entry_safe(iocg, tiocg, &inner_walk, walk_list)
+	list_for_each_entry_mutable(iocg, &inner_walk, walk_list)
 		list_del_init(&iocg->walk_list);
 }
 
@@ -2166,9 +2166,9 @@ static void ioc_forgive_debts(struct ioc *ioc, u64 usage_us_sum, int nr_debtors,
 static int ioc_check_iocgs(struct ioc *ioc, struct ioc_now *now)
 {
 	int nr_debtors = 0;
-	struct ioc_gq *iocg, *tiocg;
+	struct ioc_gq *iocg;
 
-	list_for_each_entry_safe(iocg, tiocg, &ioc->active_iocgs, active_list) {
+	list_for_each_entry_mutable(iocg, &ioc->active_iocgs, active_list) {
 		if (!waitqueue_active(&iocg->waitq) && !iocg->abs_vdebt &&
 		    !iocg->delay && !iocg_is_idle(iocg))
 			continue;
@@ -2234,7 +2234,7 @@ static int ioc_check_iocgs(struct ioc *ioc, struct ioc_now *now)
 static void ioc_timer_fn(struct timer_list *timer)
 {
 	struct ioc *ioc = container_of(timer, struct ioc, timer);
-	struct ioc_gq *iocg, *tiocg;
+	struct ioc_gq *iocg;
 	struct ioc_now now;
 	LIST_HEAD(surpluses);
 	int nr_debtors, nr_shortages = 0, nr_lagging = 0;
@@ -2378,7 +2378,7 @@ static void ioc_timer_fn(struct timer_list *timer)
 	commit_weights(ioc);
 
 	/* surplus list should be dissolved after use */
-	list_for_each_entry_safe(iocg, tiocg, &surpluses, surplus_list)
+	list_for_each_entry_mutable(iocg, &surpluses, surplus_list)
 		list_del_init(&iocg->surplus_list);
 
 	/*

block/blk-mq.c

@@ -1216,9 +1216,9 @@ EXPORT_SYMBOL_GPL(blk_mq_end_request_batch);
 static void blk_complete_reqs(struct llist_head *list)
 {
 	struct llist_node *entry = llist_reverse_order(llist_del_all(list));
-	struct request *rq, *next;
+	struct request *rq;
 
-	llist_for_each_entry_safe(rq, next, entry, ipi_list)
+	llist_for_each_entry_mutable(rq, entry, ipi_list)
 		rq->q->mq_ops->complete(rq);
 }
 
@@ -4383,14 +4383,14 @@ static int blk_mq_alloc_ctxs(struct request_queue *q)
  */
 void blk_mq_release(struct request_queue *q)
 {
-	struct blk_mq_hw_ctx *hctx, *next;
+	struct blk_mq_hw_ctx *hctx;
 	unsigned long i;
 
 	queue_for_each_hw_ctx(q, hctx, i)
 		WARN_ON_ONCE(hctx && list_empty(&hctx->hctx_list));
 
 	/* all hctx are in .unused_hctx_list now */
-	list_for_each_entry_safe(hctx, next, &q->unused_hctx_list, hctx_list) {
+	list_for_each_entry_mutable(hctx, &q->unused_hctx_list, hctx_list) {
 		list_del_init(&hctx->hctx_list);
 		kobject_put(&hctx->kobj);
 	}

block/blk-throttle.c

@@ -1686,10 +1686,10 @@ static void tg_cancel_writeback_bios(struct throtl_grp *tg,
 	tg->flags |= THROTL_TG_CANCELING;
 
 	for (rw = READ; rw <= WRITE; rw++) {
-		struct throtl_qnode *qn, *tmp;
+		struct throtl_qnode *qn;
 		unsigned int nr_bios = 0;
 
-		list_for_each_entry_safe(qn, tmp, &sq->queued[rw], node) {
+		list_for_each_entry_mutable(qn, &sq->queued[rw], node) {
 			struct bio *bio;
 
 			while ((bio = bio_list_pop(&qn->bios_iops))) {

block/kyber-iosched.c

@@ -578,9 +578,9 @@ static void kyber_insert_requests(struct blk_mq_hw_ctx *hctx,
 				  blk_insert_t flags)
 {
 	struct kyber_hctx_data *khd = hctx->sched_data;
-	struct request *rq, *next;
+	struct request *rq;
 
-	list_for_each_entry_safe(rq, next, rq_list, queuelist) {
+	list_for_each_entry_mutable(rq, rq_list, queuelist) {
 		unsigned int sched_domain = kyber_sched_domain(rq->cmd_flags);
 		struct kyber_ctx_queue *kcq = &khd->kcqs[rq->mq_ctx->index_hw[hctx->type]];
 		struct list_head *head = &kcq->rq_list[sched_domain];

block/partitions/ldm.c

@@ -1285,11 +1285,11 @@ static bool ldm_frag_add (const u8 *data, int size, struct list_head *frags)
  */
 static void ldm_frag_free (struct list_head *list)
 {
-	struct list_head *item, *tmp;
+	struct list_head *item;
 
 	BUG_ON (!list);
 
-	list_for_each_safe (item, tmp, list)
+	list_for_each_mutable(item, list)
 		kfree (list_entry (item, struct frag, list));
 }
 
@@ -1400,11 +1400,11 @@ static bool ldm_get_vblks(struct parsed_partitions *state, unsigned long base,
  */
 static void ldm_free_vblks (struct list_head *lh)
 {
-	struct list_head *item, *tmp;
+	struct list_head *item;
 
 	BUG_ON (!lh);
 
-	list_for_each_safe (item, tmp, lh)
+	list_for_each_mutable(item, lh)
 		kfree (list_entry (item, struct vblk, list));
 }
 

block/sed-opal.c

@@ -2573,10 +2573,10 @@ static int check_opal_support(struct opal_dev *dev)
 static void clean_opal_dev(struct opal_dev *dev)
 {
 
-	struct opal_suspend_data *suspend, *next;
+	struct opal_suspend_data *suspend;
 
 	mutex_lock(&dev->dev_lock);
-	list_for_each_entry_safe(suspend, next, &dev->unlk_lst, node) {
+	list_for_each_entry_mutable(suspend, &dev->unlk_lst, node) {
 		list_del(&suspend->node);
 		kfree(suspend);
 	}