workqueue: add a per-cpu backend for unbound pwqs

+/* True if @pool is tied to a specific CPU, rather than an unbound pool. */
+static bool is_pool_cpu_specific(struct worker_pool *pool)
+{
+	return pool->cpu >= 0;
+}
+
 ...
-	if (wq->flags & WQ_UNBOUND) {
+	if (!is_pool_cpu_specific(pool)) {
 		mutex_lock(&wq_pool_mutex);
 		put_unbound_pool(pool);
 		mutex_unlock(&wq_pool_mutex);

diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
index a283766a192aa..5bbbed94d2fa6 100644
--- a/include/linux/workqueue.h
+++ b/include/linux/workqueue.h
@@ -410,6 +410,7 @@ enum wq_flags {
 	__WQ_ORDERED		= 1 << 17, /* internal: workqueue is ordered */
 	__WQ_LEGACY		= 1 << 18, /* internal: create*_workqueue() */
 	__WQ_DEPRECATED		= 1 << 19, /* internal: workqueue is deprecated */
+	__WQ_PERCPU_POOLS	= 1 << 20, /* internal: back unbound pwqs with percpu pools */
 
 	/* BH wq only allows the following flags */
 	__WQ_BH_ALLOWS		= WQ_BH | WQ_HIGHPRI | WQ_PERCPU,

+/*
+ * Create a pwq backing @wq on @cpu with the static per-cpu pool instead of a
+ * dedicated unbound pool. Used by the unbound pwq machinery for a workqueue
+ * that requests the per-cpu backend.
+ */
+static struct pool_workqueue *alloc_percpu_pwq(struct workqueue_struct *wq,
+					       int cpu)
+{
+	struct worker_pool *pool = get_percpu_pool(wq, cpu);
+	struct pool_workqueue *pwq;
+
+	lockdep_assert_held(&wq_pool_mutex);
+
+	pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
+	if (!pwq)
+		return NULL;
+
+	init_pwq(pwq, wq, pool);
+	return pwq;
+}
+

@@ -5643,6 +5664,17 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu)
 	if (!(wq->flags & WQ_UNBOUND) || wq->unbound_attrs->ordered)
 		return;
 
+	if (wq->flags & __WQ_PERCPU_POOLS) {
+		/* nothing to do if @cpu is already backed by its per-cpu pool */
+		if (is_pool_cpu_specific(unbound_pwq(wq, cpu)->pool))
+			return;
+
+		pwq = alloc_percpu_pwq(wq, cpu);
+		if (!pwq)
+			goto use_dfl_pwq;
+		goto install;
+	}
+
 	/*
 	 * We don't wanna alloc/free wq_attrs for each wq for each CPU.
 	 * Let's use a preallocated one.  The following buf is protected by
@@ -5666,6 +5698,7 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu)
 		goto use_dfl_pwq;
 	}
 
+install:
 	/* Install the new pwq. */
 	mutex_lock(&wq->mutex);
 	old_pwq = install_unbound_pwq(wq, cpu, pwq);

diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index df4fc9ccb7b22..cd3d0d54dfddc 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -5766,6 +5766,12 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq)
 
 	if (ret)
 		goto enomem;
+
+	if (wq->flags & __WQ_PERCPU_POOLS) {
+		for_each_possible_cpu(cpu)
+			unbound_wq_update_pwq(wq, cpu);
+	}
+
 	return 0;
 
 enomem: