blob: 5ca46a2e2616f2b3419a751f4eb2ca2af582202d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Tejun Heoc54fce62010-09-10 16:51:36 +02002 * kernel/workqueue.c - generic async execution with shared worker pool
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
Tejun Heoc54fce62010-09-10 16:51:36 +02004 * Copyright (C) 2002 Ingo Molnar
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 *
Tejun Heoc54fce62010-09-10 16:51:36 +02006 * Derived from the taskqueue/keventd code by:
7 * David Woodhouse <dwmw2@infradead.org>
8 * Andrew Morton
9 * Kai Petzke <wpp@marie.physik.tu-berlin.de>
10 * Theodore Ts'o <tytso@mit.edu>
Christoph Lameter89ada672005-10-30 15:01:59 -080011 *
Christoph Lametercde53532008-07-04 09:59:22 -070012 * Made to use alloc_percpu by Christoph Lameter.
Tejun Heoc54fce62010-09-10 16:51:36 +020013 *
14 * Copyright (C) 2010 SUSE Linux Products GmbH
15 * Copyright (C) 2010 Tejun Heo <tj@kernel.org>
16 *
17 * This is the generic async execution mechanism. Work items as are
18 * executed in process context. The worker pool is shared and
19 * automatically managed. There is one worker pool for each CPU and
20 * one extra for works which are better served by workers which are
21 * not bound to any specific CPU.
22 *
23 * Please read Documentation/workqueue.txt for details.
Linus Torvalds1da177e2005-04-16 15:20:36 -070024 */
25
Paul Gortmaker9984de12011-05-23 14:51:41 -040026#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027#include <linux/kernel.h>
28#include <linux/sched.h>
29#include <linux/init.h>
30#include <linux/signal.h>
31#include <linux/completion.h>
32#include <linux/workqueue.h>
33#include <linux/slab.h>
34#include <linux/cpu.h>
35#include <linux/notifier.h>
36#include <linux/kthread.h>
James Bottomley1fa44ec2006-02-23 12:43:43 -060037#include <linux/hardirq.h>
Christoph Lameter46934022006-10-11 01:21:26 -070038#include <linux/mempolicy.h>
Rafael J. Wysocki341a5952006-12-06 20:34:49 -080039#include <linux/freezer.h>
Peter Zijlstrad5abe662006-12-06 20:37:26 -080040#include <linux/kallsyms.h>
41#include <linux/debug_locks.h>
Johannes Berg4e6045f2007-10-18 23:39:55 -070042#include <linux/lockdep.h>
Tejun Heoc34056a2010-06-29 10:07:11 +020043#include <linux/idr.h>
Tejun Heo29c91e92013-03-12 11:30:03 -070044#include <linux/jhash.h>
Sasha Levin42f85702012-12-17 10:01:23 -050045#include <linux/hashtable.h>
Tejun Heo76af4d92013-03-12 11:30:00 -070046#include <linux/rculist.h>
Tejun Heobce90382013-04-01 11:23:32 -070047#include <linux/nodemask.h>
Tejun Heoe22bee72010-06-29 10:07:14 +020048
Tejun Heoea138442013-01-18 14:05:55 -080049#include "workqueue_internal.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070050
Tejun Heoc8e55f32010-06-29 10:07:12 +020051enum {
Tejun Heobc2ae0f2012-07-17 12:39:27 -070052 /*
Tejun Heo24647572013-01-24 11:01:33 -080053 * worker_pool flags
Tejun Heobc2ae0f2012-07-17 12:39:27 -070054 *
Tejun Heo24647572013-01-24 11:01:33 -080055 * A bound pool is either associated or disassociated with its CPU.
Tejun Heobc2ae0f2012-07-17 12:39:27 -070056 * While associated (!DISASSOCIATED), all workers are bound to the
57 * CPU and none has %WORKER_UNBOUND set and concurrency management
58 * is in effect.
59 *
60 * While DISASSOCIATED, the cpu may be offline and all workers have
61 * %WORKER_UNBOUND set and concurrency management disabled, and may
Tejun Heo24647572013-01-24 11:01:33 -080062 * be executing on any CPU. The pool behaves as an unbound one.
Tejun Heobc2ae0f2012-07-17 12:39:27 -070063 *
Tejun Heobc3a1af2013-03-13 19:47:39 -070064 * Note that DISASSOCIATED should be flipped only while holding
65 * manager_mutex to avoid changing binding state while
Tejun Heo24647572013-01-24 11:01:33 -080066 * create_worker() is in progress.
Tejun Heobc2ae0f2012-07-17 12:39:27 -070067 */
Tejun Heo11ebea52012-07-12 14:46:37 -070068 POOL_MANAGE_WORKERS = 1 << 0, /* need to manage workers */
Tejun Heo24647572013-01-24 11:01:33 -080069 POOL_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */
Tejun Heo35b6bb62013-01-24 11:01:33 -080070 POOL_FREEZING = 1 << 3, /* freeze in progress */
Tejun Heodb7bccf2010-06-29 10:07:12 +020071
Tejun Heoc8e55f32010-06-29 10:07:12 +020072 /* worker flags */
73 WORKER_STARTED = 1 << 0, /* started */
74 WORKER_DIE = 1 << 1, /* die die die */
75 WORKER_IDLE = 1 << 2, /* is idle */
Tejun Heoe22bee72010-06-29 10:07:14 +020076 WORKER_PREP = 1 << 3, /* preparing to run works */
Tejun Heofb0e7be2010-06-29 10:07:15 +020077 WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */
Tejun Heof3421792010-07-02 10:03:51 +020078 WORKER_UNBOUND = 1 << 7, /* worker is unbound */
Tejun Heoa9ab7752013-03-19 13:45:21 -070079 WORKER_REBOUND = 1 << 8, /* worker was rebound */
Tejun Heoe22bee72010-06-29 10:07:14 +020080
Tejun Heoa9ab7752013-03-19 13:45:21 -070081 WORKER_NOT_RUNNING = WORKER_PREP | WORKER_CPU_INTENSIVE |
82 WORKER_UNBOUND | WORKER_REBOUND,
Tejun Heodb7bccf2010-06-29 10:07:12 +020083
Tejun Heoe34cdddb2013-01-24 11:01:33 -080084 NR_STD_WORKER_POOLS = 2, /* # standard pools per cpu */
Tejun Heo4ce62e92012-07-13 22:16:44 -070085
Tejun Heo29c91e92013-03-12 11:30:03 -070086 UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
Tejun Heoc8e55f32010-06-29 10:07:12 +020087 BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
Tejun Heodb7bccf2010-06-29 10:07:12 +020088
Tejun Heoe22bee72010-06-29 10:07:14 +020089 MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */
90 IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */
91
Tejun Heo3233cdb2011-02-16 18:10:19 +010092 MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2,
93 /* call for help after 10ms
94 (min two ticks) */
Tejun Heoe22bee72010-06-29 10:07:14 +020095 MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */
96 CREATE_COOLDOWN = HZ, /* time to breath after fail */
Tejun Heoe22bee72010-06-29 10:07:14 +020097
98 /*
99 * Rescue workers are used only on emergencies and shared by
100 * all cpus. Give -20.
101 */
102 RESCUER_NICE_LEVEL = -20,
Tejun Heo32704762012-07-13 22:16:45 -0700103 HIGHPRI_NICE_LEVEL = -20,
Tejun Heoc8e55f32010-06-29 10:07:12 +0200104};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105
106/*
Tejun Heo4690c4a2010-06-29 10:07:10 +0200107 * Structure fields follow one of the following exclusion rules.
108 *
Tejun Heoe41e7042010-08-24 14:22:47 +0200109 * I: Modifiable by initialization/destruction paths and read-only for
110 * everyone else.
Tejun Heo4690c4a2010-06-29 10:07:10 +0200111 *
Tejun Heoe22bee72010-06-29 10:07:14 +0200112 * P: Preemption protected. Disabling preemption is enough and should
113 * only be modified and accessed from the local cpu.
114 *
Tejun Heod565ed62013-01-24 11:01:33 -0800115 * L: pool->lock protected. Access with pool->lock held.
Tejun Heo4690c4a2010-06-29 10:07:10 +0200116 *
Tejun Heod565ed62013-01-24 11:01:33 -0800117 * X: During normal operation, modification requires pool->lock and should
118 * be done only from local cpu. Either disabling preemption on local
119 * cpu or grabbing pool->lock is enough for read access. If
120 * POOL_DISASSOCIATED is set, it's identical to L.
Tejun Heoe22bee72010-06-29 10:07:14 +0200121 *
Tejun Heo822d8402013-03-19 13:45:21 -0700122 * MG: pool->manager_mutex and pool->lock protected. Writes require both
123 * locks. Reads can happen under either lock.
124 *
Lai Jiangshan68e13a62013-03-25 16:57:17 -0700125 * PL: wq_pool_mutex protected.
Tejun Heo76af4d92013-03-12 11:30:00 -0700126 *
Lai Jiangshan68e13a62013-03-25 16:57:17 -0700127 * PR: wq_pool_mutex protected for writes. Sched-RCU protected for reads.
Tejun Heo5bcab332013-03-13 19:47:40 -0700128 *
Lai Jiangshan3c25a552013-03-25 16:57:17 -0700129 * WQ: wq->mutex protected.
130 *
Lai Jiangshanb5927602013-03-25 16:57:19 -0700131 * WR: wq->mutex protected for writes. Sched-RCU protected for reads.
Tejun Heo2e109a22013-03-13 19:47:40 -0700132 *
133 * MD: wq_mayday_lock protected.
Tejun Heo4690c4a2010-06-29 10:07:10 +0200134 */
135
Tejun Heo2eaebdb2013-01-18 14:05:55 -0800136/* struct worker is defined in workqueue_internal.h */
Tejun Heoc34056a2010-06-29 10:07:11 +0200137
Tejun Heobd7bdd42012-07-12 14:46:37 -0700138struct worker_pool {
Tejun Heod565ed62013-01-24 11:01:33 -0800139 spinlock_t lock; /* the pool lock */
Tejun Heod84ff052013-03-12 11:29:59 -0700140 int cpu; /* I: the associated cpu */
Tejun Heo9daf9e62013-01-24 11:01:33 -0800141 int id; /* I: pool ID */
Tejun Heo11ebea52012-07-12 14:46:37 -0700142 unsigned int flags; /* X: flags */
Tejun Heobd7bdd42012-07-12 14:46:37 -0700143
144 struct list_head worklist; /* L: list of pending works */
145 int nr_workers; /* L: total number of workers */
Lai Jiangshanea1abd62012-09-18 09:59:22 -0700146
147 /* nr_idle includes the ones off idle_list for rebinding */
Tejun Heobd7bdd42012-07-12 14:46:37 -0700148 int nr_idle; /* L: currently idle ones */
149
150 struct list_head idle_list; /* X: list of idle workers */
151 struct timer_list idle_timer; /* L: worker idle timeout */
152 struct timer_list mayday_timer; /* L: SOS timer for workers */
153
Tejun Heoc5aa87b2013-03-13 16:51:36 -0700154 /* a workers is either on busy_hash or idle_list, or the manager */
Tejun Heoc9e7cf22013-01-24 11:01:33 -0800155 DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
156 /* L: hash of busy workers */
157
Tejun Heobc3a1af2013-03-13 19:47:39 -0700158 /* see manage_workers() for details on the two manager mutexes */
Tejun Heo34a06bd2013-03-12 11:30:00 -0700159 struct mutex manager_arb; /* manager arbitration */
Tejun Heobc3a1af2013-03-13 19:47:39 -0700160 struct mutex manager_mutex; /* manager exclusion */
Tejun Heo822d8402013-03-19 13:45:21 -0700161 struct idr worker_idr; /* MG: worker IDs and iteration */
Tejun Heoe19e3972013-01-24 11:39:44 -0800162
Tejun Heo7a4e3442013-03-12 11:30:00 -0700163 struct workqueue_attrs *attrs; /* I: worker attributes */
Lai Jiangshan68e13a62013-03-25 16:57:17 -0700164 struct hlist_node hash_node; /* PL: unbound_pool_hash node */
165 int refcnt; /* PL: refcnt for unbound pools */
Tejun Heo7a4e3442013-03-12 11:30:00 -0700166
Tejun Heoe19e3972013-01-24 11:39:44 -0800167 /*
168 * The current concurrency level. As it's likely to be accessed
169 * from other CPUs during try_to_wake_up(), put it in a separate
170 * cacheline.
171 */
172 atomic_t nr_running ____cacheline_aligned_in_smp;
Tejun Heo29c91e92013-03-12 11:30:03 -0700173
174 /*
175 * Destruction of pool is sched-RCU protected to allow dereferences
176 * from get_work_pool().
177 */
178 struct rcu_head rcu;
Tejun Heo8b03ae32010-06-29 10:07:12 +0200179} ____cacheline_aligned_in_smp;
180
181/*
Tejun Heo112202d2013-02-13 19:29:12 -0800182 * The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
183 * of work_struct->data are used for flags and the remaining high bits
184 * point to the pwq; thus, pwqs need to be aligned at two's power of the
185 * number of flag bits.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186 */
Tejun Heo112202d2013-02-13 19:29:12 -0800187struct pool_workqueue {
Tejun Heobd7bdd42012-07-12 14:46:37 -0700188 struct worker_pool *pool; /* I: the associated pool */
Tejun Heo4690c4a2010-06-29 10:07:10 +0200189 struct workqueue_struct *wq; /* I: the owning workqueue */
Tejun Heo73f53c42010-06-29 10:07:11 +0200190 int work_color; /* L: current color */
191 int flush_color; /* L: flushing color */
Tejun Heo8864b4e2013-03-12 11:30:04 -0700192 int refcnt; /* L: reference count */
Tejun Heo73f53c42010-06-29 10:07:11 +0200193 int nr_in_flight[WORK_NR_COLORS];
194 /* L: nr of in_flight works */
Tejun Heo1e19ffc2010-06-29 10:07:12 +0200195 int nr_active; /* L: nr of active works */
Tejun Heoa0a1a5f2010-06-29 10:07:12 +0200196 int max_active; /* L: max active works */
Tejun Heo1e19ffc2010-06-29 10:07:12 +0200197 struct list_head delayed_works; /* L: delayed works */
Lai Jiangshan3c25a552013-03-25 16:57:17 -0700198 struct list_head pwqs_node; /* WR: node on wq->pwqs */
Tejun Heo2e109a22013-03-13 19:47:40 -0700199 struct list_head mayday_node; /* MD: node on wq->maydays */
Tejun Heo8864b4e2013-03-12 11:30:04 -0700200
201 /*
202 * Release of unbound pwq is punted to system_wq. See put_pwq()
203 * and pwq_unbound_release_workfn() for details. pool_workqueue
204 * itself is also sched-RCU protected so that the first pwq can be
Lai Jiangshanb09f4fd2013-03-25 16:57:18 -0700205 * determined without grabbing wq->mutex.
Tejun Heo8864b4e2013-03-12 11:30:04 -0700206 */
207 struct work_struct unbound_release_work;
208 struct rcu_head rcu;
Tejun Heoe904e6c2013-03-12 11:29:57 -0700209} __aligned(1 << WORK_STRUCT_FLAG_BITS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211/*
Tejun Heo73f53c42010-06-29 10:07:11 +0200212 * Structure used to wait for workqueue flush.
213 */
214struct wq_flusher {
Lai Jiangshan3c25a552013-03-25 16:57:17 -0700215 struct list_head list; /* WQ: list of flushers */
216 int flush_color; /* WQ: flush color waiting for */
Tejun Heo73f53c42010-06-29 10:07:11 +0200217 struct completion done; /* flush completion */
218};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219
Tejun Heo226223a2013-03-12 11:30:05 -0700220struct wq_device;
221
Tejun Heo73f53c42010-06-29 10:07:11 +0200222/*
Tejun Heoc5aa87b2013-03-13 16:51:36 -0700223 * The externally visible workqueue. It relays the issued work items to
224 * the appropriate worker_pool through its pool_workqueues.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225 */
226struct workqueue_struct {
Lai Jiangshan87fc7412013-03-25 16:57:18 -0700227 unsigned int flags; /* WQ: WQ_* flags */
Tejun Heo420c0dd2013-03-12 11:29:59 -0700228 struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwq's */
Lai Jiangshan3c25a552013-03-25 16:57:17 -0700229 struct list_head pwqs; /* WR: all pwqs of this wq */
Lai Jiangshan68e13a62013-03-25 16:57:17 -0700230 struct list_head list; /* PL: list of all workqueues */
Tejun Heo73f53c42010-06-29 10:07:11 +0200231
Lai Jiangshan3c25a552013-03-25 16:57:17 -0700232 struct mutex mutex; /* protects this wq */
233 int work_color; /* WQ: current work color */
234 int flush_color; /* WQ: current flush color */
Tejun Heo112202d2013-02-13 19:29:12 -0800235 atomic_t nr_pwqs_to_flush; /* flush in progress */
Lai Jiangshan3c25a552013-03-25 16:57:17 -0700236 struct wq_flusher *first_flusher; /* WQ: first flusher */
237 struct list_head flusher_queue; /* WQ: flush waiters */
238 struct list_head flusher_overflow; /* WQ: flush overflow list */
Tejun Heo73f53c42010-06-29 10:07:11 +0200239
Tejun Heo2e109a22013-03-13 19:47:40 -0700240 struct list_head maydays; /* MD: pwqs requesting rescue */
Tejun Heoe22bee72010-06-29 10:07:14 +0200241 struct worker *rescuer; /* I: rescue worker */
242
Lai Jiangshan87fc7412013-03-25 16:57:18 -0700243 int nr_drainers; /* WQ: drain in progress */
Lai Jiangshana357fc02013-03-25 16:57:19 -0700244 int saved_max_active; /* WQ: saved pwq max_active */
Tejun Heo226223a2013-03-12 11:30:05 -0700245
246#ifdef CONFIG_SYSFS
247 struct wq_device *wq_dev; /* I: for sysfs interface */
248#endif
Johannes Berg4e6045f2007-10-18 23:39:55 -0700249#ifdef CONFIG_LOCKDEP
Tejun Heo4690c4a2010-06-29 10:07:10 +0200250 struct lockdep_map lockdep_map;
Johannes Berg4e6045f2007-10-18 23:39:55 -0700251#endif
Tejun Heob196be82012-01-10 15:11:35 -0800252 char name[]; /* I: workqueue name */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253};
254
Tejun Heoe904e6c2013-03-12 11:29:57 -0700255static struct kmem_cache *pwq_cache;
256
Tejun Heobce90382013-04-01 11:23:32 -0700257static int wq_numa_tbl_len; /* highest possible NUMA node id + 1 */
258static cpumask_var_t *wq_numa_possible_cpumask;
259 /* possible CPUs of each node */
260
261static bool wq_numa_enabled; /* unbound NUMA affinity enabled */
262
Lai Jiangshan68e13a62013-03-25 16:57:17 -0700263static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */
Tejun Heo2e109a22013-03-13 19:47:40 -0700264static DEFINE_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
Tejun Heo5bcab332013-03-13 19:47:40 -0700265
Lai Jiangshan68e13a62013-03-25 16:57:17 -0700266static LIST_HEAD(workqueues); /* PL: list of all workqueues */
267static bool workqueue_freezing; /* PL: have wqs started freezing? */
Tejun Heo7d19c5c2013-03-13 19:47:40 -0700268
269/* the per-cpu worker pools */
270static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
271 cpu_worker_pools);
272
Lai Jiangshan68e13a62013-03-25 16:57:17 -0700273static DEFINE_IDR(worker_pool_idr); /* PR: idr of all pools */
Tejun Heo7d19c5c2013-03-13 19:47:40 -0700274
Lai Jiangshan68e13a62013-03-25 16:57:17 -0700275/* PL: hash of all unbound pools keyed by pool->attrs */
Tejun Heo29c91e92013-03-12 11:30:03 -0700276static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
277
Tejun Heoc5aa87b2013-03-13 16:51:36 -0700278/* I: attributes used when instantiating standard unbound pools on demand */
Tejun Heo29c91e92013-03-12 11:30:03 -0700279static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
280
Tejun Heod320c032010-06-29 10:07:14 +0200281struct workqueue_struct *system_wq __read_mostly;
Tejun Heod320c032010-06-29 10:07:14 +0200282EXPORT_SYMBOL_GPL(system_wq);
Valentin Ilie044c7822012-08-19 00:52:42 +0300283struct workqueue_struct *system_highpri_wq __read_mostly;
Joonsoo Kim1aabe902012-08-15 23:25:39 +0900284EXPORT_SYMBOL_GPL(system_highpri_wq);
Valentin Ilie044c7822012-08-19 00:52:42 +0300285struct workqueue_struct *system_long_wq __read_mostly;
Tejun Heod320c032010-06-29 10:07:14 +0200286EXPORT_SYMBOL_GPL(system_long_wq);
Valentin Ilie044c7822012-08-19 00:52:42 +0300287struct workqueue_struct *system_unbound_wq __read_mostly;
Tejun Heof3421792010-07-02 10:03:51 +0200288EXPORT_SYMBOL_GPL(system_unbound_wq);
Valentin Ilie044c7822012-08-19 00:52:42 +0300289struct workqueue_struct *system_freezable_wq __read_mostly;
Tejun Heo24d51ad2011-02-21 09:52:50 +0100290EXPORT_SYMBOL_GPL(system_freezable_wq);
Tejun Heod320c032010-06-29 10:07:14 +0200291
Tejun Heo7d19c5c2013-03-13 19:47:40 -0700292static int worker_thread(void *__worker);
293static void copy_workqueue_attrs(struct workqueue_attrs *to,
294 const struct workqueue_attrs *from);
295
Tejun Heo97bd2342010-10-05 10:41:14 +0200296#define CREATE_TRACE_POINTS
297#include <trace/events/workqueue.h>
298
Lai Jiangshan68e13a62013-03-25 16:57:17 -0700299#define assert_rcu_or_pool_mutex() \
Tejun Heo5bcab332013-03-13 19:47:40 -0700300 rcu_lockdep_assert(rcu_read_lock_sched_held() || \
Lai Jiangshan68e13a62013-03-25 16:57:17 -0700301 lockdep_is_held(&wq_pool_mutex), \
302 "sched RCU or wq_pool_mutex should be held")
Tejun Heo5bcab332013-03-13 19:47:40 -0700303
Lai Jiangshanb09f4fd2013-03-25 16:57:18 -0700304#define assert_rcu_or_wq_mutex(wq) \
Tejun Heo76af4d92013-03-12 11:30:00 -0700305 rcu_lockdep_assert(rcu_read_lock_sched_held() || \
Lai Jiangshanb5927602013-03-25 16:57:19 -0700306 lockdep_is_held(&wq->mutex), \
Lai Jiangshanb09f4fd2013-03-25 16:57:18 -0700307 "sched RCU or wq->mutex should be held")
Tejun Heo76af4d92013-03-12 11:30:00 -0700308
Tejun Heo822d8402013-03-19 13:45:21 -0700309#ifdef CONFIG_LOCKDEP
310#define assert_manager_or_pool_lock(pool) \
Lai Jiangshan519e3c12013-03-20 03:28:21 +0800311 WARN_ONCE(debug_locks && \
312 !lockdep_is_held(&(pool)->manager_mutex) && \
Tejun Heo822d8402013-03-19 13:45:21 -0700313 !lockdep_is_held(&(pool)->lock), \
314 "pool->manager_mutex or ->lock should be held")
315#else
316#define assert_manager_or_pool_lock(pool) do { } while (0)
317#endif
318
Tejun Heof02ae732013-03-12 11:30:03 -0700319#define for_each_cpu_worker_pool(pool, cpu) \
320 for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
321 (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
Tejun Heo7a62c2c2013-03-12 11:30:03 -0700322 (pool)++)
Tejun Heo4ce62e92012-07-13 22:16:44 -0700323
Tejun Heo49e3cf42013-03-12 11:29:58 -0700324/**
Tejun Heo17116962013-03-12 11:29:58 -0700325 * for_each_pool - iterate through all worker_pools in the system
326 * @pool: iteration cursor
Tejun Heo611c92a2013-03-13 16:51:36 -0700327 * @pi: integer used for iteration
Tejun Heofa1b54e2013-03-12 11:30:00 -0700328 *
Lai Jiangshan68e13a62013-03-25 16:57:17 -0700329 * This must be called either with wq_pool_mutex held or sched RCU read
330 * locked. If the pool needs to be used beyond the locking in effect, the
331 * caller is responsible for guaranteeing that the pool stays online.
Tejun Heofa1b54e2013-03-12 11:30:00 -0700332 *
333 * The if/else clause exists only for the lockdep assertion and can be
334 * ignored.
Tejun Heo17116962013-03-12 11:29:58 -0700335 */
Tejun Heo611c92a2013-03-13 16:51:36 -0700336#define for_each_pool(pool, pi) \
337 idr_for_each_entry(&worker_pool_idr, pool, pi) \
Lai Jiangshan68e13a62013-03-25 16:57:17 -0700338 if (({ assert_rcu_or_pool_mutex(); false; })) { } \
Tejun Heofa1b54e2013-03-12 11:30:00 -0700339 else
Tejun Heo17116962013-03-12 11:29:58 -0700340
341/**
Tejun Heo822d8402013-03-19 13:45:21 -0700342 * for_each_pool_worker - iterate through all workers of a worker_pool
343 * @worker: iteration cursor
344 * @wi: integer used for iteration
345 * @pool: worker_pool to iterate workers of
346 *
347 * This must be called with either @pool->manager_mutex or ->lock held.
348 *
349 * The if/else clause exists only for the lockdep assertion and can be
350 * ignored.
351 */
352#define for_each_pool_worker(worker, wi, pool) \
353 idr_for_each_entry(&(pool)->worker_idr, (worker), (wi)) \
354 if (({ assert_manager_or_pool_lock((pool)); false; })) { } \
355 else
356
357/**
Tejun Heo49e3cf42013-03-12 11:29:58 -0700358 * for_each_pwq - iterate through all pool_workqueues of the specified workqueue
359 * @pwq: iteration cursor
360 * @wq: the target workqueue
Tejun Heo76af4d92013-03-12 11:30:00 -0700361 *
Lai Jiangshanb09f4fd2013-03-25 16:57:18 -0700362 * This must be called either with wq->mutex held or sched RCU read locked.
Tejun Heo794b18b2013-03-13 19:47:40 -0700363 * If the pwq needs to be used beyond the locking in effect, the caller is
364 * responsible for guaranteeing that the pwq stays online.
Tejun Heo76af4d92013-03-12 11:30:00 -0700365 *
366 * The if/else clause exists only for the lockdep assertion and can be
367 * ignored.
Tejun Heo49e3cf42013-03-12 11:29:58 -0700368 */
369#define for_each_pwq(pwq, wq) \
Tejun Heo76af4d92013-03-12 11:30:00 -0700370 list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node) \
Lai Jiangshanb09f4fd2013-03-25 16:57:18 -0700371 if (({ assert_rcu_or_wq_mutex(wq); false; })) { } \
Tejun Heo76af4d92013-03-12 11:30:00 -0700372 else
Tejun Heof3421792010-07-02 10:03:51 +0200373
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900374#ifdef CONFIG_DEBUG_OBJECTS_WORK
375
376static struct debug_obj_descr work_debug_descr;
377
Stanislaw Gruszka99777282011-03-07 09:58:33 +0100378static void *work_debug_hint(void *addr)
379{
380 return ((struct work_struct *) addr)->func;
381}
382
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900383/*
384 * fixup_init is called when:
385 * - an active object is initialized
386 */
387static int work_fixup_init(void *addr, enum debug_obj_state state)
388{
389 struct work_struct *work = addr;
390
391 switch (state) {
392 case ODEBUG_STATE_ACTIVE:
393 cancel_work_sync(work);
394 debug_object_init(work, &work_debug_descr);
395 return 1;
396 default:
397 return 0;
398 }
399}
400
401/*
402 * fixup_activate is called when:
403 * - an active object is activated
404 * - an unknown object is activated (might be a statically initialized object)
405 */
406static int work_fixup_activate(void *addr, enum debug_obj_state state)
407{
408 struct work_struct *work = addr;
409
410 switch (state) {
411
412 case ODEBUG_STATE_NOTAVAILABLE:
413 /*
414 * This is not really a fixup. The work struct was
415 * statically initialized. We just make sure that it
416 * is tracked in the object tracker.
417 */
Tejun Heo22df02b2010-06-29 10:07:10 +0200418 if (test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work))) {
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900419 debug_object_init(work, &work_debug_descr);
420 debug_object_activate(work, &work_debug_descr);
421 return 0;
422 }
423 WARN_ON_ONCE(1);
424 return 0;
425
426 case ODEBUG_STATE_ACTIVE:
427 WARN_ON(1);
428
429 default:
430 return 0;
431 }
432}
433
434/*
435 * fixup_free is called when:
436 * - an active object is freed
437 */
438static int work_fixup_free(void *addr, enum debug_obj_state state)
439{
440 struct work_struct *work = addr;
441
442 switch (state) {
443 case ODEBUG_STATE_ACTIVE:
444 cancel_work_sync(work);
445 debug_object_free(work, &work_debug_descr);
446 return 1;
447 default:
448 return 0;
449 }
450}
451
452static struct debug_obj_descr work_debug_descr = {
453 .name = "work_struct",
Stanislaw Gruszka99777282011-03-07 09:58:33 +0100454 .debug_hint = work_debug_hint,
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900455 .fixup_init = work_fixup_init,
456 .fixup_activate = work_fixup_activate,
457 .fixup_free = work_fixup_free,
458};
459
460static inline void debug_work_activate(struct work_struct *work)
461{
462 debug_object_activate(work, &work_debug_descr);
463}
464
465static inline void debug_work_deactivate(struct work_struct *work)
466{
467 debug_object_deactivate(work, &work_debug_descr);
468}
469
470void __init_work(struct work_struct *work, int onstack)
471{
472 if (onstack)
473 debug_object_init_on_stack(work, &work_debug_descr);
474 else
475 debug_object_init(work, &work_debug_descr);
476}
477EXPORT_SYMBOL_GPL(__init_work);
478
479void destroy_work_on_stack(struct work_struct *work)
480{
481 debug_object_free(work, &work_debug_descr);
482}
483EXPORT_SYMBOL_GPL(destroy_work_on_stack);
484
485#else
486static inline void debug_work_activate(struct work_struct *work) { }
487static inline void debug_work_deactivate(struct work_struct *work) { }
488#endif
489
Tejun Heo9daf9e62013-01-24 11:01:33 -0800490/* allocate ID and assign it to @pool */
491static int worker_pool_assign_id(struct worker_pool *pool)
492{
493 int ret;
494
Lai Jiangshan68e13a62013-03-25 16:57:17 -0700495 lockdep_assert_held(&wq_pool_mutex);
Tejun Heo5bcab332013-03-13 19:47:40 -0700496
Tejun Heofa1b54e2013-03-12 11:30:00 -0700497 do {
498 if (!idr_pre_get(&worker_pool_idr, GFP_KERNEL))
499 return -ENOMEM;
Tejun Heofa1b54e2013-03-12 11:30:00 -0700500 ret = idr_get_new(&worker_pool_idr, pool, &pool->id);
Tejun Heofa1b54e2013-03-12 11:30:00 -0700501 } while (ret == -EAGAIN);
Tejun Heo9daf9e62013-01-24 11:01:33 -0800502
503 return ret;
504}
505
Tejun Heo76af4d92013-03-12 11:30:00 -0700506/**
507 * first_pwq - return the first pool_workqueue of the specified workqueue
508 * @wq: the target workqueue
509 *
Lai Jiangshanb09f4fd2013-03-25 16:57:18 -0700510 * This must be called either with wq->mutex held or sched RCU read locked.
Tejun Heo794b18b2013-03-13 19:47:40 -0700511 * If the pwq needs to be used beyond the locking in effect, the caller is
512 * responsible for guaranteeing that the pwq stays online.
Tejun Heo76af4d92013-03-12 11:30:00 -0700513 */
Tejun Heo7fb98ea2013-03-12 11:30:00 -0700514static struct pool_workqueue *first_pwq(struct workqueue_struct *wq)
Oleg Nesterova848e3b2007-05-09 02:34:17 -0700515{
Lai Jiangshanb09f4fd2013-03-25 16:57:18 -0700516 assert_rcu_or_wq_mutex(wq);
Tejun Heo76af4d92013-03-12 11:30:00 -0700517 return list_first_or_null_rcu(&wq->pwqs, struct pool_workqueue,
518 pwqs_node);
Oleg Nesterova848e3b2007-05-09 02:34:17 -0700519}
520
Tejun Heo73f53c42010-06-29 10:07:11 +0200521static unsigned int work_color_to_flags(int color)
522{
523 return color << WORK_STRUCT_COLOR_SHIFT;
524}
525
526static int get_work_color(struct work_struct *work)
527{
528 return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &
529 ((1 << WORK_STRUCT_COLOR_BITS) - 1);
530}
531
532static int work_next_color(int color)
533{
534 return (color + 1) % WORK_NR_COLORS;
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -0700535}
536
David Howells4594bf12006-12-07 11:33:26 +0000537/*
Tejun Heo112202d2013-02-13 19:29:12 -0800538 * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
539 * contain the pointer to the queued pwq. Once execution starts, the flag
Tejun Heo7c3eed52013-01-24 11:01:33 -0800540 * is cleared and the high bits contain OFFQ flags and pool ID.
Tejun Heo7a22ad72010-06-29 10:07:13 +0200541 *
Tejun Heo112202d2013-02-13 19:29:12 -0800542 * set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling()
543 * and clear_work_data() can be used to set the pwq, pool or clear
Tejun Heobbb68df2012-08-03 10:30:46 -0700544 * work->data. These functions should only be called while the work is
545 * owned - ie. while the PENDING bit is set.
Tejun Heo7a22ad72010-06-29 10:07:13 +0200546 *
Tejun Heo112202d2013-02-13 19:29:12 -0800547 * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
Tejun Heo7c3eed52013-01-24 11:01:33 -0800548 * corresponding to a work. Pool is available once the work has been
Tejun Heo112202d2013-02-13 19:29:12 -0800549 * queued anywhere after initialization until it is sync canceled. pwq is
Tejun Heo7c3eed52013-01-24 11:01:33 -0800550 * available only while the work item is queued.
Tejun Heobbb68df2012-08-03 10:30:46 -0700551 *
552 * %WORK_OFFQ_CANCELING is used to mark a work item which is being
553 * canceled. While being canceled, a work item may have its PENDING set
554 * but stay off timer and worklist for arbitrarily long and nobody should
555 * try to steal the PENDING bit.
David Howells4594bf12006-12-07 11:33:26 +0000556 */
Tejun Heo7a22ad72010-06-29 10:07:13 +0200557static inline void set_work_data(struct work_struct *work, unsigned long data,
558 unsigned long flags)
David Howells365970a2006-11-22 14:54:49 +0000559{
Tejun Heo6183c002013-03-12 11:29:57 -0700560 WARN_ON_ONCE(!work_pending(work));
Tejun Heo7a22ad72010-06-29 10:07:13 +0200561 atomic_long_set(&work->data, data | flags | work_static(work));
David Howells365970a2006-11-22 14:54:49 +0000562}
David Howells365970a2006-11-22 14:54:49 +0000563
Tejun Heo112202d2013-02-13 19:29:12 -0800564static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,
Tejun Heo7a22ad72010-06-29 10:07:13 +0200565 unsigned long extra_flags)
Oleg Nesterov4d707b92010-04-23 17:40:40 +0200566{
Tejun Heo112202d2013-02-13 19:29:12 -0800567 set_work_data(work, (unsigned long)pwq,
568 WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | extra_flags);
Oleg Nesterov4d707b92010-04-23 17:40:40 +0200569}
570
Lai Jiangshan4468a002013-02-06 18:04:53 -0800571static void set_work_pool_and_keep_pending(struct work_struct *work,
572 int pool_id)
573{
574 set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,
575 WORK_STRUCT_PENDING);
576}
577
Tejun Heo7c3eed52013-01-24 11:01:33 -0800578static void set_work_pool_and_clear_pending(struct work_struct *work,
579 int pool_id)
David Howells365970a2006-11-22 14:54:49 +0000580{
Tejun Heo23657bb2012-08-13 17:08:19 -0700581 /*
582 * The following wmb is paired with the implied mb in
583 * test_and_set_bit(PENDING) and ensures all updates to @work made
584 * here are visible to and precede any updates by the next PENDING
585 * owner.
586 */
587 smp_wmb();
Tejun Heo7c3eed52013-01-24 11:01:33 -0800588 set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
Tejun Heo7a22ad72010-06-29 10:07:13 +0200589}
590
591static void clear_work_data(struct work_struct *work)
592{
Tejun Heo7c3eed52013-01-24 11:01:33 -0800593 smp_wmb(); /* see set_work_pool_and_clear_pending() */
594 set_work_data(work, WORK_STRUCT_NO_POOL, 0);
Tejun Heo7a22ad72010-06-29 10:07:13 +0200595}
596
Tejun Heo112202d2013-02-13 19:29:12 -0800597static struct pool_workqueue *get_work_pwq(struct work_struct *work)
Tejun Heo7a22ad72010-06-29 10:07:13 +0200598{
Tejun Heoe1201532010-07-22 14:14:25 +0200599 unsigned long data = atomic_long_read(&work->data);
Tejun Heo7a22ad72010-06-29 10:07:13 +0200600
Tejun Heo112202d2013-02-13 19:29:12 -0800601 if (data & WORK_STRUCT_PWQ)
Tejun Heoe1201532010-07-22 14:14:25 +0200602 return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
603 else
604 return NULL;
Tejun Heo7a22ad72010-06-29 10:07:13 +0200605}
606
Tejun Heo7c3eed52013-01-24 11:01:33 -0800607/**
608 * get_work_pool - return the worker_pool a given work was associated with
609 * @work: the work item of interest
610 *
611 * Return the worker_pool @work was last associated with. %NULL if none.
Tejun Heofa1b54e2013-03-12 11:30:00 -0700612 *
Lai Jiangshan68e13a62013-03-25 16:57:17 -0700613 * Pools are created and destroyed under wq_pool_mutex, and allows read
614 * access under sched-RCU read lock. As such, this function should be
615 * called under wq_pool_mutex or with preemption disabled.
Tejun Heofa1b54e2013-03-12 11:30:00 -0700616 *
617 * All fields of the returned pool are accessible as long as the above
618 * mentioned locking is in effect. If the returned pool needs to be used
619 * beyond the critical section, the caller is responsible for ensuring the
620 * returned pool is and stays online.
Tejun Heo7c3eed52013-01-24 11:01:33 -0800621 */
622static struct worker_pool *get_work_pool(struct work_struct *work)
Tejun Heo7a22ad72010-06-29 10:07:13 +0200623{
Tejun Heoe1201532010-07-22 14:14:25 +0200624 unsigned long data = atomic_long_read(&work->data);
Tejun Heo7c3eed52013-01-24 11:01:33 -0800625 int pool_id;
Tejun Heo7a22ad72010-06-29 10:07:13 +0200626
Lai Jiangshan68e13a62013-03-25 16:57:17 -0700627 assert_rcu_or_pool_mutex();
Tejun Heofa1b54e2013-03-12 11:30:00 -0700628
Tejun Heo112202d2013-02-13 19:29:12 -0800629 if (data & WORK_STRUCT_PWQ)
630 return ((struct pool_workqueue *)
Tejun Heo7c3eed52013-01-24 11:01:33 -0800631 (data & WORK_STRUCT_WQ_DATA_MASK))->pool;
Tejun Heo7a22ad72010-06-29 10:07:13 +0200632
Tejun Heo7c3eed52013-01-24 11:01:33 -0800633 pool_id = data >> WORK_OFFQ_POOL_SHIFT;
634 if (pool_id == WORK_OFFQ_POOL_NONE)
Tejun Heo7a22ad72010-06-29 10:07:13 +0200635 return NULL;
636
Tejun Heofa1b54e2013-03-12 11:30:00 -0700637 return idr_find(&worker_pool_idr, pool_id);
Tejun Heo7c3eed52013-01-24 11:01:33 -0800638}
639
640/**
641 * get_work_pool_id - return the worker pool ID a given work is associated with
642 * @work: the work item of interest
643 *
644 * Return the worker_pool ID @work was last associated with.
645 * %WORK_OFFQ_POOL_NONE if none.
646 */
647static int get_work_pool_id(struct work_struct *work)
648{
Lai Jiangshan54d5b7d2013-02-07 13:14:20 -0800649 unsigned long data = atomic_long_read(&work->data);
Tejun Heo7c3eed52013-01-24 11:01:33 -0800650
Tejun Heo112202d2013-02-13 19:29:12 -0800651 if (data & WORK_STRUCT_PWQ)
652 return ((struct pool_workqueue *)
Lai Jiangshan54d5b7d2013-02-07 13:14:20 -0800653 (data & WORK_STRUCT_WQ_DATA_MASK))->pool->id;
654
655 return data >> WORK_OFFQ_POOL_SHIFT;
Tejun Heo7c3eed52013-01-24 11:01:33 -0800656}
657
Tejun Heobbb68df2012-08-03 10:30:46 -0700658static void mark_work_canceling(struct work_struct *work)
659{
Tejun Heo7c3eed52013-01-24 11:01:33 -0800660 unsigned long pool_id = get_work_pool_id(work);
Tejun Heobbb68df2012-08-03 10:30:46 -0700661
Tejun Heo7c3eed52013-01-24 11:01:33 -0800662 pool_id <<= WORK_OFFQ_POOL_SHIFT;
663 set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
Tejun Heobbb68df2012-08-03 10:30:46 -0700664}
665
666static bool work_is_canceling(struct work_struct *work)
667{
668 unsigned long data = atomic_long_read(&work->data);
669
Tejun Heo112202d2013-02-13 19:29:12 -0800670 return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
Tejun Heobbb68df2012-08-03 10:30:46 -0700671}
672
David Howells365970a2006-11-22 14:54:49 +0000673/*
Tejun Heo32704762012-07-13 22:16:45 -0700674 * Policy functions. These define the policies on how the global worker
675 * pools are managed. Unless noted otherwise, these functions assume that
Tejun Heod565ed62013-01-24 11:01:33 -0800676 * they're being called with pool->lock held.
David Howells365970a2006-11-22 14:54:49 +0000677 */
Tejun Heoe22bee72010-06-29 10:07:14 +0200678
Tejun Heo63d95a92012-07-12 14:46:37 -0700679static bool __need_more_worker(struct worker_pool *pool)
David Howells365970a2006-11-22 14:54:49 +0000680{
Tejun Heoe19e3972013-01-24 11:39:44 -0800681 return !atomic_read(&pool->nr_running);
David Howells365970a2006-11-22 14:54:49 +0000682}
683
Tejun Heoe22bee72010-06-29 10:07:14 +0200684/*
685 * Need to wake up a worker? Called from anything but currently
686 * running workers.
Tejun Heo974271c2012-07-12 14:46:37 -0700687 *
688 * Note that, because unbound workers never contribute to nr_running, this
Tejun Heo706026c2013-01-24 11:01:34 -0800689 * function will always return %true for unbound pools as long as the
Tejun Heo974271c2012-07-12 14:46:37 -0700690 * worklist isn't empty.
Tejun Heoe22bee72010-06-29 10:07:14 +0200691 */
Tejun Heo63d95a92012-07-12 14:46:37 -0700692static bool need_more_worker(struct worker_pool *pool)
David Howells365970a2006-11-22 14:54:49 +0000693{
Tejun Heo63d95a92012-07-12 14:46:37 -0700694 return !list_empty(&pool->worklist) && __need_more_worker(pool);
David Howells365970a2006-11-22 14:54:49 +0000695}
696
Tejun Heoe22bee72010-06-29 10:07:14 +0200697/* Can I start working? Called from busy but !running workers. */
Tejun Heo63d95a92012-07-12 14:46:37 -0700698static bool may_start_working(struct worker_pool *pool)
Tejun Heoe22bee72010-06-29 10:07:14 +0200699{
Tejun Heo63d95a92012-07-12 14:46:37 -0700700 return pool->nr_idle;
Tejun Heoe22bee72010-06-29 10:07:14 +0200701}
702
703/* Do I need to keep working? Called from currently running workers. */
Tejun Heo63d95a92012-07-12 14:46:37 -0700704static bool keep_working(struct worker_pool *pool)
Tejun Heoe22bee72010-06-29 10:07:14 +0200705{
Tejun Heoe19e3972013-01-24 11:39:44 -0800706 return !list_empty(&pool->worklist) &&
707 atomic_read(&pool->nr_running) <= 1;
Tejun Heoe22bee72010-06-29 10:07:14 +0200708}
709
710/* Do we need a new worker? Called from manager. */
Tejun Heo63d95a92012-07-12 14:46:37 -0700711static bool need_to_create_worker(struct worker_pool *pool)
Tejun Heoe22bee72010-06-29 10:07:14 +0200712{
Tejun Heo63d95a92012-07-12 14:46:37 -0700713 return need_more_worker(pool) && !may_start_working(pool);
Tejun Heoe22bee72010-06-29 10:07:14 +0200714}
715
716/* Do I need to be the manager? */
Tejun Heo63d95a92012-07-12 14:46:37 -0700717static bool need_to_manage_workers(struct worker_pool *pool)
Tejun Heoe22bee72010-06-29 10:07:14 +0200718{
Tejun Heo63d95a92012-07-12 14:46:37 -0700719 return need_to_create_worker(pool) ||
Tejun Heo11ebea52012-07-12 14:46:37 -0700720 (pool->flags & POOL_MANAGE_WORKERS);
Tejun Heoe22bee72010-06-29 10:07:14 +0200721}
722
723/* Do we have too many workers and should some go away? */
Tejun Heo63d95a92012-07-12 14:46:37 -0700724static bool too_many_workers(struct worker_pool *pool)
Tejun Heoe22bee72010-06-29 10:07:14 +0200725{
Tejun Heo34a06bd2013-03-12 11:30:00 -0700726 bool managing = mutex_is_locked(&pool->manager_arb);
Tejun Heo63d95a92012-07-12 14:46:37 -0700727 int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
728 int nr_busy = pool->nr_workers - nr_idle;
Tejun Heoe22bee72010-06-29 10:07:14 +0200729
Lai Jiangshanea1abd62012-09-18 09:59:22 -0700730 /*
731 * nr_idle and idle_list may disagree if idle rebinding is in
732 * progress. Never return %true if idle_list is empty.
733 */
734 if (list_empty(&pool->idle_list))
735 return false;
736
Tejun Heoe22bee72010-06-29 10:07:14 +0200737 return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
738}
739
740/*
741 * Wake up functions.
742 */
743
Tejun Heo7e116292010-06-29 10:07:13 +0200744/* Return the first worker. Safe with preemption disabled */
Tejun Heo63d95a92012-07-12 14:46:37 -0700745static struct worker *first_worker(struct worker_pool *pool)
Tejun Heo7e116292010-06-29 10:07:13 +0200746{
Tejun Heo63d95a92012-07-12 14:46:37 -0700747 if (unlikely(list_empty(&pool->idle_list)))
Tejun Heo7e116292010-06-29 10:07:13 +0200748 return NULL;
749
Tejun Heo63d95a92012-07-12 14:46:37 -0700750 return list_first_entry(&pool->idle_list, struct worker, entry);
Tejun Heo7e116292010-06-29 10:07:13 +0200751}
752
753/**
754 * wake_up_worker - wake up an idle worker
Tejun Heo63d95a92012-07-12 14:46:37 -0700755 * @pool: worker pool to wake worker from
Tejun Heo7e116292010-06-29 10:07:13 +0200756 *
Tejun Heo63d95a92012-07-12 14:46:37 -0700757 * Wake up the first idle worker of @pool.
Tejun Heo7e116292010-06-29 10:07:13 +0200758 *
759 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -0800760 * spin_lock_irq(pool->lock).
Tejun Heo7e116292010-06-29 10:07:13 +0200761 */
Tejun Heo63d95a92012-07-12 14:46:37 -0700762static void wake_up_worker(struct worker_pool *pool)
Tejun Heo7e116292010-06-29 10:07:13 +0200763{
Tejun Heo63d95a92012-07-12 14:46:37 -0700764 struct worker *worker = first_worker(pool);
Tejun Heo7e116292010-06-29 10:07:13 +0200765
766 if (likely(worker))
767 wake_up_process(worker->task);
768}
769
Tejun Heo4690c4a2010-06-29 10:07:10 +0200770/**
Tejun Heoe22bee72010-06-29 10:07:14 +0200771 * wq_worker_waking_up - a worker is waking up
772 * @task: task waking up
773 * @cpu: CPU @task is waking up to
774 *
775 * This function is called during try_to_wake_up() when a worker is
776 * being awoken.
777 *
778 * CONTEXT:
779 * spin_lock_irq(rq->lock)
780 */
Tejun Heod84ff052013-03-12 11:29:59 -0700781void wq_worker_waking_up(struct task_struct *task, int cpu)
Tejun Heoe22bee72010-06-29 10:07:14 +0200782{
783 struct worker *worker = kthread_data(task);
784
Joonsoo Kim36576002012-10-26 23:03:49 +0900785 if (!(worker->flags & WORKER_NOT_RUNNING)) {
Tejun Heoec22ca52013-01-24 11:01:33 -0800786 WARN_ON_ONCE(worker->pool->cpu != cpu);
Tejun Heoe19e3972013-01-24 11:39:44 -0800787 atomic_inc(&worker->pool->nr_running);
Joonsoo Kim36576002012-10-26 23:03:49 +0900788 }
Tejun Heoe22bee72010-06-29 10:07:14 +0200789}
790
791/**
792 * wq_worker_sleeping - a worker is going to sleep
793 * @task: task going to sleep
794 * @cpu: CPU in question, must be the current CPU number
795 *
796 * This function is called during schedule() when a busy worker is
797 * going to sleep. Worker on the same cpu can be woken up by
798 * returning pointer to its task.
799 *
800 * CONTEXT:
801 * spin_lock_irq(rq->lock)
802 *
803 * RETURNS:
804 * Worker task on @cpu to wake up, %NULL if none.
805 */
Tejun Heod84ff052013-03-12 11:29:59 -0700806struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu)
Tejun Heoe22bee72010-06-29 10:07:14 +0200807{
808 struct worker *worker = kthread_data(task), *to_wakeup = NULL;
Tejun Heo111c2252013-01-17 17:16:24 -0800809 struct worker_pool *pool;
Tejun Heoe22bee72010-06-29 10:07:14 +0200810
Tejun Heo111c2252013-01-17 17:16:24 -0800811 /*
812 * Rescuers, which may not have all the fields set up like normal
813 * workers, also reach here, let's not access anything before
814 * checking NOT_RUNNING.
815 */
Steven Rostedt2d646722010-12-03 23:12:33 -0500816 if (worker->flags & WORKER_NOT_RUNNING)
Tejun Heoe22bee72010-06-29 10:07:14 +0200817 return NULL;
818
Tejun Heo111c2252013-01-17 17:16:24 -0800819 pool = worker->pool;
Tejun Heo111c2252013-01-17 17:16:24 -0800820
Tejun Heoe22bee72010-06-29 10:07:14 +0200821 /* this can only happen on the local cpu */
Tejun Heo6183c002013-03-12 11:29:57 -0700822 if (WARN_ON_ONCE(cpu != raw_smp_processor_id()))
823 return NULL;
Tejun Heoe22bee72010-06-29 10:07:14 +0200824
825 /*
826 * The counterpart of the following dec_and_test, implied mb,
827 * worklist not empty test sequence is in insert_work().
828 * Please read comment there.
829 *
Tejun Heo628c78e2012-07-17 12:39:27 -0700830 * NOT_RUNNING is clear. This means that we're bound to and
831 * running on the local cpu w/ rq lock held and preemption
832 * disabled, which in turn means that none else could be
Tejun Heod565ed62013-01-24 11:01:33 -0800833 * manipulating idle_list, so dereferencing idle_list without pool
Tejun Heo628c78e2012-07-17 12:39:27 -0700834 * lock is safe.
Tejun Heoe22bee72010-06-29 10:07:14 +0200835 */
Tejun Heoe19e3972013-01-24 11:39:44 -0800836 if (atomic_dec_and_test(&pool->nr_running) &&
837 !list_empty(&pool->worklist))
Tejun Heo63d95a92012-07-12 14:46:37 -0700838 to_wakeup = first_worker(pool);
Tejun Heoe22bee72010-06-29 10:07:14 +0200839 return to_wakeup ? to_wakeup->task : NULL;
840}
841
842/**
843 * worker_set_flags - set worker flags and adjust nr_running accordingly
Tejun Heocb444762010-07-02 10:03:50 +0200844 * @worker: self
Tejun Heod302f012010-06-29 10:07:13 +0200845 * @flags: flags to set
846 * @wakeup: wakeup an idle worker if necessary
847 *
Tejun Heoe22bee72010-06-29 10:07:14 +0200848 * Set @flags in @worker->flags and adjust nr_running accordingly. If
849 * nr_running becomes zero and @wakeup is %true, an idle worker is
850 * woken up.
Tejun Heod302f012010-06-29 10:07:13 +0200851 *
Tejun Heocb444762010-07-02 10:03:50 +0200852 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -0800853 * spin_lock_irq(pool->lock)
Tejun Heod302f012010-06-29 10:07:13 +0200854 */
855static inline void worker_set_flags(struct worker *worker, unsigned int flags,
856 bool wakeup)
857{
Tejun Heobd7bdd42012-07-12 14:46:37 -0700858 struct worker_pool *pool = worker->pool;
Tejun Heoe22bee72010-06-29 10:07:14 +0200859
Tejun Heocb444762010-07-02 10:03:50 +0200860 WARN_ON_ONCE(worker->task != current);
861
Tejun Heoe22bee72010-06-29 10:07:14 +0200862 /*
863 * If transitioning into NOT_RUNNING, adjust nr_running and
864 * wake up an idle worker as necessary if requested by
865 * @wakeup.
866 */
867 if ((flags & WORKER_NOT_RUNNING) &&
868 !(worker->flags & WORKER_NOT_RUNNING)) {
Tejun Heoe22bee72010-06-29 10:07:14 +0200869 if (wakeup) {
Tejun Heoe19e3972013-01-24 11:39:44 -0800870 if (atomic_dec_and_test(&pool->nr_running) &&
Tejun Heobd7bdd42012-07-12 14:46:37 -0700871 !list_empty(&pool->worklist))
Tejun Heo63d95a92012-07-12 14:46:37 -0700872 wake_up_worker(pool);
Tejun Heoe22bee72010-06-29 10:07:14 +0200873 } else
Tejun Heoe19e3972013-01-24 11:39:44 -0800874 atomic_dec(&pool->nr_running);
Tejun Heoe22bee72010-06-29 10:07:14 +0200875 }
876
Tejun Heod302f012010-06-29 10:07:13 +0200877 worker->flags |= flags;
878}
879
880/**
Tejun Heoe22bee72010-06-29 10:07:14 +0200881 * worker_clr_flags - clear worker flags and adjust nr_running accordingly
Tejun Heocb444762010-07-02 10:03:50 +0200882 * @worker: self
Tejun Heod302f012010-06-29 10:07:13 +0200883 * @flags: flags to clear
884 *
Tejun Heoe22bee72010-06-29 10:07:14 +0200885 * Clear @flags in @worker->flags and adjust nr_running accordingly.
Tejun Heod302f012010-06-29 10:07:13 +0200886 *
Tejun Heocb444762010-07-02 10:03:50 +0200887 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -0800888 * spin_lock_irq(pool->lock)
Tejun Heod302f012010-06-29 10:07:13 +0200889 */
890static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
891{
Tejun Heo63d95a92012-07-12 14:46:37 -0700892 struct worker_pool *pool = worker->pool;
Tejun Heoe22bee72010-06-29 10:07:14 +0200893 unsigned int oflags = worker->flags;
894
Tejun Heocb444762010-07-02 10:03:50 +0200895 WARN_ON_ONCE(worker->task != current);
896
Tejun Heod302f012010-06-29 10:07:13 +0200897 worker->flags &= ~flags;
Tejun Heoe22bee72010-06-29 10:07:14 +0200898
Tejun Heo42c025f2011-01-11 15:58:49 +0100899 /*
900 * If transitioning out of NOT_RUNNING, increment nr_running. Note
901 * that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
902 * of multiple flags, not a single flag.
903 */
Tejun Heoe22bee72010-06-29 10:07:14 +0200904 if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
905 if (!(worker->flags & WORKER_NOT_RUNNING))
Tejun Heoe19e3972013-01-24 11:39:44 -0800906 atomic_inc(&pool->nr_running);
Tejun Heod302f012010-06-29 10:07:13 +0200907}
908
909/**
Tejun Heo8cca0ee2010-06-29 10:07:13 +0200910 * find_worker_executing_work - find worker which is executing a work
Tejun Heoc9e7cf22013-01-24 11:01:33 -0800911 * @pool: pool of interest
Tejun Heo8cca0ee2010-06-29 10:07:13 +0200912 * @work: work to find worker for
913 *
Tejun Heoc9e7cf22013-01-24 11:01:33 -0800914 * Find a worker which is executing @work on @pool by searching
915 * @pool->busy_hash which is keyed by the address of @work. For a worker
Tejun Heoa2c1c572012-12-18 10:35:02 -0800916 * to match, its current execution should match the address of @work and
917 * its work function. This is to avoid unwanted dependency between
918 * unrelated work executions through a work item being recycled while still
919 * being executed.
920 *
921 * This is a bit tricky. A work item may be freed once its execution
922 * starts and nothing prevents the freed area from being recycled for
923 * another work item. If the same work item address ends up being reused
924 * before the original execution finishes, workqueue will identify the
925 * recycled work item as currently executing and make it wait until the
926 * current execution finishes, introducing an unwanted dependency.
927 *
Tejun Heoc5aa87b2013-03-13 16:51:36 -0700928 * This function checks the work item address and work function to avoid
929 * false positives. Note that this isn't complete as one may construct a
930 * work function which can introduce dependency onto itself through a
931 * recycled work item. Well, if somebody wants to shoot oneself in the
932 * foot that badly, there's only so much we can do, and if such deadlock
933 * actually occurs, it should be easy to locate the culprit work function.
Tejun Heo8cca0ee2010-06-29 10:07:13 +0200934 *
935 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -0800936 * spin_lock_irq(pool->lock).
Tejun Heo8cca0ee2010-06-29 10:07:13 +0200937 *
938 * RETURNS:
939 * Pointer to worker which is executing @work if found, NULL
940 * otherwise.
941 */
Tejun Heoc9e7cf22013-01-24 11:01:33 -0800942static struct worker *find_worker_executing_work(struct worker_pool *pool,
Tejun Heo8cca0ee2010-06-29 10:07:13 +0200943 struct work_struct *work)
944{
Sasha Levin42f85702012-12-17 10:01:23 -0500945 struct worker *worker;
Sasha Levin42f85702012-12-17 10:01:23 -0500946
Sasha Levinb67bfe02013-02-27 17:06:00 -0800947 hash_for_each_possible(pool->busy_hash, worker, hentry,
Tejun Heoa2c1c572012-12-18 10:35:02 -0800948 (unsigned long)work)
949 if (worker->current_work == work &&
950 worker->current_func == work->func)
Sasha Levin42f85702012-12-17 10:01:23 -0500951 return worker;
952
953 return NULL;
Tejun Heo8cca0ee2010-06-29 10:07:13 +0200954}
955
956/**
Tejun Heobf4ede02012-08-03 10:30:46 -0700957 * move_linked_works - move linked works to a list
958 * @work: start of series of works to be scheduled
959 * @head: target list to append @work to
960 * @nextp: out paramter for nested worklist walking
961 *
962 * Schedule linked works starting from @work to @head. Work series to
963 * be scheduled starts at @work and includes any consecutive work with
964 * WORK_STRUCT_LINKED set in its predecessor.
965 *
966 * If @nextp is not NULL, it's updated to point to the next work of
967 * the last scheduled work. This allows move_linked_works() to be
968 * nested inside outer list_for_each_entry_safe().
969 *
970 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -0800971 * spin_lock_irq(pool->lock).
Tejun Heobf4ede02012-08-03 10:30:46 -0700972 */
973static void move_linked_works(struct work_struct *work, struct list_head *head,
974 struct work_struct **nextp)
975{
976 struct work_struct *n;
977
978 /*
979 * Linked worklist will always end before the end of the list,
980 * use NULL for list head.
981 */
982 list_for_each_entry_safe_from(work, n, NULL, entry) {
983 list_move_tail(&work->entry, head);
984 if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
985 break;
986 }
987
988 /*
989 * If we're already inside safe list traversal and have moved
990 * multiple works to the scheduled queue, the next position
991 * needs to be updated.
992 */
993 if (nextp)
994 *nextp = n;
995}
996
Tejun Heo8864b4e2013-03-12 11:30:04 -0700997/**
998 * get_pwq - get an extra reference on the specified pool_workqueue
999 * @pwq: pool_workqueue to get
1000 *
1001 * Obtain an extra reference on @pwq. The caller should guarantee that
1002 * @pwq has positive refcnt and be holding the matching pool->lock.
1003 */
1004static void get_pwq(struct pool_workqueue *pwq)
1005{
1006 lockdep_assert_held(&pwq->pool->lock);
1007 WARN_ON_ONCE(pwq->refcnt <= 0);
1008 pwq->refcnt++;
1009}
1010
1011/**
1012 * put_pwq - put a pool_workqueue reference
1013 * @pwq: pool_workqueue to put
1014 *
1015 * Drop a reference of @pwq. If its refcnt reaches zero, schedule its
1016 * destruction. The caller should be holding the matching pool->lock.
1017 */
1018static void put_pwq(struct pool_workqueue *pwq)
1019{
1020 lockdep_assert_held(&pwq->pool->lock);
1021 if (likely(--pwq->refcnt))
1022 return;
1023 if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
1024 return;
1025 /*
1026 * @pwq can't be released under pool->lock, bounce to
1027 * pwq_unbound_release_workfn(). This never recurses on the same
1028 * pool->lock as this path is taken only for unbound workqueues and
1029 * the release work item is scheduled on a per-cpu workqueue. To
1030 * avoid lockdep warning, unbound pool->locks are given lockdep
1031 * subclass of 1 in get_unbound_pool().
1032 */
1033 schedule_work(&pwq->unbound_release_work);
1034}
1035
Tejun Heo112202d2013-02-13 19:29:12 -08001036static void pwq_activate_delayed_work(struct work_struct *work)
Tejun Heobf4ede02012-08-03 10:30:46 -07001037{
Tejun Heo112202d2013-02-13 19:29:12 -08001038 struct pool_workqueue *pwq = get_work_pwq(work);
Tejun Heobf4ede02012-08-03 10:30:46 -07001039
1040 trace_workqueue_activate_work(work);
Tejun Heo112202d2013-02-13 19:29:12 -08001041 move_linked_works(work, &pwq->pool->worklist, NULL);
Tejun Heobf4ede02012-08-03 10:30:46 -07001042 __clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work));
Tejun Heo112202d2013-02-13 19:29:12 -08001043 pwq->nr_active++;
Tejun Heobf4ede02012-08-03 10:30:46 -07001044}
1045
Tejun Heo112202d2013-02-13 19:29:12 -08001046static void pwq_activate_first_delayed(struct pool_workqueue *pwq)
Lai Jiangshan3aa62492012-09-18 10:40:00 -07001047{
Tejun Heo112202d2013-02-13 19:29:12 -08001048 struct work_struct *work = list_first_entry(&pwq->delayed_works,
Lai Jiangshan3aa62492012-09-18 10:40:00 -07001049 struct work_struct, entry);
1050
Tejun Heo112202d2013-02-13 19:29:12 -08001051 pwq_activate_delayed_work(work);
Lai Jiangshan3aa62492012-09-18 10:40:00 -07001052}
1053
Tejun Heobf4ede02012-08-03 10:30:46 -07001054/**
Tejun Heo112202d2013-02-13 19:29:12 -08001055 * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
1056 * @pwq: pwq of interest
Tejun Heobf4ede02012-08-03 10:30:46 -07001057 * @color: color of work which left the queue
Tejun Heobf4ede02012-08-03 10:30:46 -07001058 *
1059 * A work either has completed or is removed from pending queue,
Tejun Heo112202d2013-02-13 19:29:12 -08001060 * decrement nr_in_flight of its pwq and handle workqueue flushing.
Tejun Heobf4ede02012-08-03 10:30:46 -07001061 *
1062 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08001063 * spin_lock_irq(pool->lock).
Tejun Heobf4ede02012-08-03 10:30:46 -07001064 */
Tejun Heo112202d2013-02-13 19:29:12 -08001065static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
Tejun Heobf4ede02012-08-03 10:30:46 -07001066{
Tejun Heo8864b4e2013-03-12 11:30:04 -07001067 /* uncolored work items don't participate in flushing or nr_active */
Tejun Heobf4ede02012-08-03 10:30:46 -07001068 if (color == WORK_NO_COLOR)
Tejun Heo8864b4e2013-03-12 11:30:04 -07001069 goto out_put;
Tejun Heobf4ede02012-08-03 10:30:46 -07001070
Tejun Heo112202d2013-02-13 19:29:12 -08001071 pwq->nr_in_flight[color]--;
Tejun Heobf4ede02012-08-03 10:30:46 -07001072
Tejun Heo112202d2013-02-13 19:29:12 -08001073 pwq->nr_active--;
1074 if (!list_empty(&pwq->delayed_works)) {
Lai Jiangshanb3f9f402012-09-18 10:40:00 -07001075 /* one down, submit a delayed one */
Tejun Heo112202d2013-02-13 19:29:12 -08001076 if (pwq->nr_active < pwq->max_active)
1077 pwq_activate_first_delayed(pwq);
Tejun Heobf4ede02012-08-03 10:30:46 -07001078 }
1079
1080 /* is flush in progress and are we at the flushing tip? */
Tejun Heo112202d2013-02-13 19:29:12 -08001081 if (likely(pwq->flush_color != color))
Tejun Heo8864b4e2013-03-12 11:30:04 -07001082 goto out_put;
Tejun Heobf4ede02012-08-03 10:30:46 -07001083
1084 /* are there still in-flight works? */
Tejun Heo112202d2013-02-13 19:29:12 -08001085 if (pwq->nr_in_flight[color])
Tejun Heo8864b4e2013-03-12 11:30:04 -07001086 goto out_put;
Tejun Heobf4ede02012-08-03 10:30:46 -07001087
Tejun Heo112202d2013-02-13 19:29:12 -08001088 /* this pwq is done, clear flush_color */
1089 pwq->flush_color = -1;
Tejun Heobf4ede02012-08-03 10:30:46 -07001090
1091 /*
Tejun Heo112202d2013-02-13 19:29:12 -08001092 * If this was the last pwq, wake up the first flusher. It
Tejun Heobf4ede02012-08-03 10:30:46 -07001093 * will handle the rest.
1094 */
Tejun Heo112202d2013-02-13 19:29:12 -08001095 if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
1096 complete(&pwq->wq->first_flusher->done);
Tejun Heo8864b4e2013-03-12 11:30:04 -07001097out_put:
1098 put_pwq(pwq);
Tejun Heobf4ede02012-08-03 10:30:46 -07001099}
1100
Tejun Heo36e227d2012-08-03 10:30:46 -07001101/**
Tejun Heobbb68df2012-08-03 10:30:46 -07001102 * try_to_grab_pending - steal work item from worklist and disable irq
Tejun Heo36e227d2012-08-03 10:30:46 -07001103 * @work: work item to steal
1104 * @is_dwork: @work is a delayed_work
Tejun Heobbb68df2012-08-03 10:30:46 -07001105 * @flags: place to store irq state
Tejun Heo36e227d2012-08-03 10:30:46 -07001106 *
1107 * Try to grab PENDING bit of @work. This function can handle @work in any
1108 * stable state - idle, on timer or on worklist. Return values are
1109 *
1110 * 1 if @work was pending and we successfully stole PENDING
1111 * 0 if @work was idle and we claimed PENDING
1112 * -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
Tejun Heobbb68df2012-08-03 10:30:46 -07001113 * -ENOENT if someone else is canceling @work, this state may persist
1114 * for arbitrarily long
Tejun Heo36e227d2012-08-03 10:30:46 -07001115 *
Tejun Heobbb68df2012-08-03 10:30:46 -07001116 * On >= 0 return, the caller owns @work's PENDING bit. To avoid getting
Tejun Heoe0aecdd2012-08-21 13:18:24 -07001117 * interrupted while holding PENDING and @work off queue, irq must be
1118 * disabled on entry. This, combined with delayed_work->timer being
1119 * irqsafe, ensures that we return -EAGAIN for finite short period of time.
Tejun Heobbb68df2012-08-03 10:30:46 -07001120 *
1121 * On successful return, >= 0, irq is disabled and the caller is
1122 * responsible for releasing it using local_irq_restore(*@flags).
1123 *
Tejun Heoe0aecdd2012-08-21 13:18:24 -07001124 * This function is safe to call from any context including IRQ handler.
Tejun Heobf4ede02012-08-03 10:30:46 -07001125 */
Tejun Heobbb68df2012-08-03 10:30:46 -07001126static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
1127 unsigned long *flags)
Tejun Heobf4ede02012-08-03 10:30:46 -07001128{
Tejun Heod565ed62013-01-24 11:01:33 -08001129 struct worker_pool *pool;
Tejun Heo112202d2013-02-13 19:29:12 -08001130 struct pool_workqueue *pwq;
Tejun Heobf4ede02012-08-03 10:30:46 -07001131
Tejun Heobbb68df2012-08-03 10:30:46 -07001132 local_irq_save(*flags);
1133
Tejun Heo36e227d2012-08-03 10:30:46 -07001134 /* try to steal the timer if it exists */
1135 if (is_dwork) {
1136 struct delayed_work *dwork = to_delayed_work(work);
1137
Tejun Heoe0aecdd2012-08-21 13:18:24 -07001138 /*
1139 * dwork->timer is irqsafe. If del_timer() fails, it's
1140 * guaranteed that the timer is not queued anywhere and not
1141 * running on the local CPU.
1142 */
Tejun Heo36e227d2012-08-03 10:30:46 -07001143 if (likely(del_timer(&dwork->timer)))
1144 return 1;
1145 }
1146
1147 /* try to claim PENDING the normal way */
Tejun Heobf4ede02012-08-03 10:30:46 -07001148 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
1149 return 0;
1150
1151 /*
1152 * The queueing is in progress, or it is already queued. Try to
1153 * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
1154 */
Tejun Heod565ed62013-01-24 11:01:33 -08001155 pool = get_work_pool(work);
1156 if (!pool)
Tejun Heobbb68df2012-08-03 10:30:46 -07001157 goto fail;
Tejun Heobf4ede02012-08-03 10:30:46 -07001158
Tejun Heod565ed62013-01-24 11:01:33 -08001159 spin_lock(&pool->lock);
Lai Jiangshan0b3dae62013-02-06 18:04:53 -08001160 /*
Tejun Heo112202d2013-02-13 19:29:12 -08001161 * work->data is guaranteed to point to pwq only while the work
1162 * item is queued on pwq->wq, and both updating work->data to point
1163 * to pwq on queueing and to pool on dequeueing are done under
1164 * pwq->pool->lock. This in turn guarantees that, if work->data
1165 * points to pwq which is associated with a locked pool, the work
Lai Jiangshan0b3dae62013-02-06 18:04:53 -08001166 * item is currently queued on that pool.
1167 */
Tejun Heo112202d2013-02-13 19:29:12 -08001168 pwq = get_work_pwq(work);
1169 if (pwq && pwq->pool == pool) {
Tejun Heo16062832013-02-06 18:04:53 -08001170 debug_work_deactivate(work);
Lai Jiangshan3aa62492012-09-18 10:40:00 -07001171
Tejun Heo16062832013-02-06 18:04:53 -08001172 /*
1173 * A delayed work item cannot be grabbed directly because
1174 * it might have linked NO_COLOR work items which, if left
Tejun Heo112202d2013-02-13 19:29:12 -08001175 * on the delayed_list, will confuse pwq->nr_active
Tejun Heo16062832013-02-06 18:04:53 -08001176 * management later on and cause stall. Make sure the work
1177 * item is activated before grabbing.
1178 */
1179 if (*work_data_bits(work) & WORK_STRUCT_DELAYED)
Tejun Heo112202d2013-02-13 19:29:12 -08001180 pwq_activate_delayed_work(work);
Lai Jiangshan3aa62492012-09-18 10:40:00 -07001181
Tejun Heo16062832013-02-06 18:04:53 -08001182 list_del_init(&work->entry);
Tejun Heo112202d2013-02-13 19:29:12 -08001183 pwq_dec_nr_in_flight(get_work_pwq(work), get_work_color(work));
Tejun Heo36e227d2012-08-03 10:30:46 -07001184
Tejun Heo112202d2013-02-13 19:29:12 -08001185 /* work->data points to pwq iff queued, point to pool */
Tejun Heo16062832013-02-06 18:04:53 -08001186 set_work_pool_and_keep_pending(work, pool->id);
Lai Jiangshan4468a002013-02-06 18:04:53 -08001187
Tejun Heo16062832013-02-06 18:04:53 -08001188 spin_unlock(&pool->lock);
1189 return 1;
Tejun Heobf4ede02012-08-03 10:30:46 -07001190 }
Tejun Heod565ed62013-01-24 11:01:33 -08001191 spin_unlock(&pool->lock);
Tejun Heobbb68df2012-08-03 10:30:46 -07001192fail:
1193 local_irq_restore(*flags);
1194 if (work_is_canceling(work))
1195 return -ENOENT;
1196 cpu_relax();
Tejun Heo36e227d2012-08-03 10:30:46 -07001197 return -EAGAIN;
Tejun Heobf4ede02012-08-03 10:30:46 -07001198}
1199
1200/**
Tejun Heo706026c2013-01-24 11:01:34 -08001201 * insert_work - insert a work into a pool
Tejun Heo112202d2013-02-13 19:29:12 -08001202 * @pwq: pwq @work belongs to
Tejun Heo4690c4a2010-06-29 10:07:10 +02001203 * @work: work to insert
1204 * @head: insertion point
1205 * @extra_flags: extra WORK_STRUCT_* flags to set
1206 *
Tejun Heo112202d2013-02-13 19:29:12 -08001207 * Insert @work which belongs to @pwq after @head. @extra_flags is or'd to
Tejun Heo706026c2013-01-24 11:01:34 -08001208 * work_struct flags.
Tejun Heo4690c4a2010-06-29 10:07:10 +02001209 *
1210 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08001211 * spin_lock_irq(pool->lock).
Tejun Heo4690c4a2010-06-29 10:07:10 +02001212 */
Tejun Heo112202d2013-02-13 19:29:12 -08001213static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
1214 struct list_head *head, unsigned int extra_flags)
Oleg Nesterovb89deed2007-05-09 02:33:52 -07001215{
Tejun Heo112202d2013-02-13 19:29:12 -08001216 struct worker_pool *pool = pwq->pool;
Frederic Weisbeckere1d8aa92009-01-12 23:15:46 +01001217
Tejun Heo4690c4a2010-06-29 10:07:10 +02001218 /* we own @work, set data and link */
Tejun Heo112202d2013-02-13 19:29:12 -08001219 set_work_pwq(work, pwq, extra_flags);
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -07001220 list_add_tail(&work->entry, head);
Tejun Heo8864b4e2013-03-12 11:30:04 -07001221 get_pwq(pwq);
Tejun Heoe22bee72010-06-29 10:07:14 +02001222
1223 /*
Tejun Heoc5aa87b2013-03-13 16:51:36 -07001224 * Ensure either wq_worker_sleeping() sees the above
1225 * list_add_tail() or we see zero nr_running to avoid workers lying
1226 * around lazily while there are works to be processed.
Tejun Heoe22bee72010-06-29 10:07:14 +02001227 */
1228 smp_mb();
1229
Tejun Heo63d95a92012-07-12 14:46:37 -07001230 if (__need_more_worker(pool))
1231 wake_up_worker(pool);
Oleg Nesterovb89deed2007-05-09 02:33:52 -07001232}
1233
Tejun Heoc8efcc22010-12-20 19:32:04 +01001234/*
1235 * Test whether @work is being queued from another work executing on the
Tejun Heo8d03ecf2013-02-13 19:29:10 -08001236 * same workqueue.
Tejun Heoc8efcc22010-12-20 19:32:04 +01001237 */
1238static bool is_chained_work(struct workqueue_struct *wq)
1239{
Tejun Heo8d03ecf2013-02-13 19:29:10 -08001240 struct worker *worker;
Tejun Heoc8efcc22010-12-20 19:32:04 +01001241
Tejun Heo8d03ecf2013-02-13 19:29:10 -08001242 worker = current_wq_worker();
1243 /*
1244 * Return %true iff I'm a worker execuing a work item on @wq. If
1245 * I'm @worker, it's safe to dereference it without locking.
1246 */
Tejun Heo112202d2013-02-13 19:29:12 -08001247 return worker && worker->current_pwq->wq == wq;
Tejun Heoc8efcc22010-12-20 19:32:04 +01001248}
1249
Tejun Heod84ff052013-03-12 11:29:59 -07001250static void __queue_work(int cpu, struct workqueue_struct *wq,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001251 struct work_struct *work)
1252{
Tejun Heo112202d2013-02-13 19:29:12 -08001253 struct pool_workqueue *pwq;
Tejun Heoc9178082013-03-12 11:30:04 -07001254 struct worker_pool *last_pool;
Tejun Heo1e19ffc2010-06-29 10:07:12 +02001255 struct list_head *worklist;
Tejun Heo8a2e8e5d2010-08-25 10:33:56 +02001256 unsigned int work_flags;
Joonsoo Kimb75cac92012-08-15 23:25:37 +09001257 unsigned int req_cpu = cpu;
Tejun Heo8930cab2012-08-03 10:30:45 -07001258
1259 /*
1260 * While a work item is PENDING && off queue, a task trying to
1261 * steal the PENDING will busy-loop waiting for it to either get
1262 * queued or lose PENDING. Grabbing PENDING and queueing should
1263 * happen with IRQ disabled.
1264 */
1265 WARN_ON_ONCE(!irqs_disabled());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001266
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +09001267 debug_work_activate(work);
Tejun Heo1e19ffc2010-06-29 10:07:12 +02001268
Tejun Heoc8efcc22010-12-20 19:32:04 +01001269 /* if dying, only works from the same workqueue are allowed */
Tejun Heo618b01e2013-03-12 11:30:04 -07001270 if (unlikely(wq->flags & __WQ_DRAINING) &&
Tejun Heoc8efcc22010-12-20 19:32:04 +01001271 WARN_ON_ONCE(!is_chained_work(wq)))
Tejun Heoe41e7042010-08-24 14:22:47 +02001272 return;
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07001273retry:
Tejun Heoc9178082013-03-12 11:30:04 -07001274 /* pwq which will be used unless @work is executing elsewhere */
Tejun Heoc7fc77f2010-07-02 10:03:51 +02001275 if (!(wq->flags & WQ_UNBOUND)) {
Tejun Heo57469822012-08-03 10:30:45 -07001276 if (cpu == WORK_CPU_UNBOUND)
Tejun Heoc7fc77f2010-07-02 10:03:51 +02001277 cpu = raw_smp_processor_id();
Tejun Heo7fb98ea2013-03-12 11:30:00 -07001278 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
Tejun Heoc9178082013-03-12 11:30:04 -07001279 } else {
1280 pwq = first_pwq(wq);
1281 }
Tejun Heodbf25762012-08-20 14:51:23 -07001282
Tejun Heoc9178082013-03-12 11:30:04 -07001283 /*
1284 * If @work was previously on a different pool, it might still be
1285 * running there, in which case the work needs to be queued on that
1286 * pool to guarantee non-reentrancy.
1287 */
1288 last_pool = get_work_pool(work);
1289 if (last_pool && last_pool != pwq->pool) {
1290 struct worker *worker;
Tejun Heo18aa9ef2010-06-29 10:07:13 +02001291
Tejun Heoc9178082013-03-12 11:30:04 -07001292 spin_lock(&last_pool->lock);
Tejun Heo18aa9ef2010-06-29 10:07:13 +02001293
Tejun Heoc9178082013-03-12 11:30:04 -07001294 worker = find_worker_executing_work(last_pool, work);
Tejun Heo18aa9ef2010-06-29 10:07:13 +02001295
Tejun Heoc9178082013-03-12 11:30:04 -07001296 if (worker && worker->current_pwq->wq == wq) {
1297 pwq = worker->current_pwq;
Tejun Heo8930cab2012-08-03 10:30:45 -07001298 } else {
Tejun Heoc9178082013-03-12 11:30:04 -07001299 /* meh... not running there, queue here */
1300 spin_unlock(&last_pool->lock);
Tejun Heo112202d2013-02-13 19:29:12 -08001301 spin_lock(&pwq->pool->lock);
Tejun Heo8930cab2012-08-03 10:30:45 -07001302 }
Tejun Heof3421792010-07-02 10:03:51 +02001303 } else {
Tejun Heo112202d2013-02-13 19:29:12 -08001304 spin_lock(&pwq->pool->lock);
Tejun Heo502ca9d2010-06-29 10:07:13 +02001305 }
1306
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07001307 /*
1308 * pwq is determined and locked. For unbound pools, we could have
1309 * raced with pwq release and it could already be dead. If its
1310 * refcnt is zero, repeat pwq selection. Note that pwqs never die
1311 * without another pwq replacing it as the first pwq or while a
1312 * work item is executing on it, so the retying is guaranteed to
1313 * make forward-progress.
1314 */
1315 if (unlikely(!pwq->refcnt)) {
1316 if (wq->flags & WQ_UNBOUND) {
1317 spin_unlock(&pwq->pool->lock);
1318 cpu_relax();
1319 goto retry;
1320 }
1321 /* oops */
1322 WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
1323 wq->name, cpu);
1324 }
1325
Tejun Heo112202d2013-02-13 19:29:12 -08001326 /* pwq determined, queue */
1327 trace_workqueue_queue_work(req_cpu, pwq, work);
Tejun Heo502ca9d2010-06-29 10:07:13 +02001328
Dan Carpenterf5b25522012-04-13 22:06:58 +03001329 if (WARN_ON(!list_empty(&work->entry))) {
Tejun Heo112202d2013-02-13 19:29:12 -08001330 spin_unlock(&pwq->pool->lock);
Dan Carpenterf5b25522012-04-13 22:06:58 +03001331 return;
1332 }
Tejun Heo1e19ffc2010-06-29 10:07:12 +02001333
Tejun Heo112202d2013-02-13 19:29:12 -08001334 pwq->nr_in_flight[pwq->work_color]++;
1335 work_flags = work_color_to_flags(pwq->work_color);
Tejun Heo1e19ffc2010-06-29 10:07:12 +02001336
Tejun Heo112202d2013-02-13 19:29:12 -08001337 if (likely(pwq->nr_active < pwq->max_active)) {
Tejun Heocdadf002010-10-05 10:49:55 +02001338 trace_workqueue_activate_work(work);
Tejun Heo112202d2013-02-13 19:29:12 -08001339 pwq->nr_active++;
1340 worklist = &pwq->pool->worklist;
Tejun Heo8a2e8e5d2010-08-25 10:33:56 +02001341 } else {
1342 work_flags |= WORK_STRUCT_DELAYED;
Tejun Heo112202d2013-02-13 19:29:12 -08001343 worklist = &pwq->delayed_works;
Tejun Heo8a2e8e5d2010-08-25 10:33:56 +02001344 }
Tejun Heo1e19ffc2010-06-29 10:07:12 +02001345
Tejun Heo112202d2013-02-13 19:29:12 -08001346 insert_work(pwq, work, worklist, work_flags);
Tejun Heo1e19ffc2010-06-29 10:07:12 +02001347
Tejun Heo112202d2013-02-13 19:29:12 -08001348 spin_unlock(&pwq->pool->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001349}
1350
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -07001351/**
Zhang Ruic1a220e2008-07-23 21:28:39 -07001352 * queue_work_on - queue work on specific cpu
1353 * @cpu: CPU number to execute work on
1354 * @wq: workqueue to use
1355 * @work: work to queue
1356 *
Tejun Heod4283e92012-08-03 10:30:44 -07001357 * Returns %false if @work was already on a queue, %true otherwise.
Zhang Ruic1a220e2008-07-23 21:28:39 -07001358 *
1359 * We queue the work to a specific CPU, the caller must ensure it
1360 * can't go away.
1361 */
Tejun Heod4283e92012-08-03 10:30:44 -07001362bool queue_work_on(int cpu, struct workqueue_struct *wq,
1363 struct work_struct *work)
Zhang Ruic1a220e2008-07-23 21:28:39 -07001364{
Tejun Heod4283e92012-08-03 10:30:44 -07001365 bool ret = false;
Tejun Heo8930cab2012-08-03 10:30:45 -07001366 unsigned long flags;
1367
1368 local_irq_save(flags);
Zhang Ruic1a220e2008-07-23 21:28:39 -07001369
Tejun Heo22df02b2010-06-29 10:07:10 +02001370 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
Tejun Heo4690c4a2010-06-29 10:07:10 +02001371 __queue_work(cpu, wq, work);
Tejun Heod4283e92012-08-03 10:30:44 -07001372 ret = true;
Zhang Ruic1a220e2008-07-23 21:28:39 -07001373 }
Tejun Heo8930cab2012-08-03 10:30:45 -07001374
1375 local_irq_restore(flags);
Zhang Ruic1a220e2008-07-23 21:28:39 -07001376 return ret;
1377}
1378EXPORT_SYMBOL_GPL(queue_work_on);
1379
Tejun Heod8e794d2012-08-03 10:30:45 -07001380void delayed_work_timer_fn(unsigned long __data)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001381{
David Howells52bad642006-11-22 14:54:01 +00001382 struct delayed_work *dwork = (struct delayed_work *)__data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001383
Tejun Heoe0aecdd2012-08-21 13:18:24 -07001384 /* should have been called from irqsafe timer with irq already off */
Lai Jiangshan60c057b2013-02-06 18:04:53 -08001385 __queue_work(dwork->cpu, dwork->wq, &dwork->work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001386}
Konstantin Khlebnikov1438ade52013-01-24 16:36:31 +04001387EXPORT_SYMBOL(delayed_work_timer_fn);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001388
Tejun Heo7beb2ed2012-08-03 10:30:46 -07001389static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
1390 struct delayed_work *dwork, unsigned long delay)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001391{
Tejun Heo7beb2ed2012-08-03 10:30:46 -07001392 struct timer_list *timer = &dwork->timer;
1393 struct work_struct *work = &dwork->work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001394
Tejun Heo7beb2ed2012-08-03 10:30:46 -07001395 WARN_ON_ONCE(timer->function != delayed_work_timer_fn ||
1396 timer->data != (unsigned long)dwork);
Tejun Heofc4b5142012-12-04 07:40:39 -08001397 WARN_ON_ONCE(timer_pending(timer));
1398 WARN_ON_ONCE(!list_empty(&work->entry));
Tejun Heo7beb2ed2012-08-03 10:30:46 -07001399
Tejun Heo8852aac2012-12-01 16:23:42 -08001400 /*
1401 * If @delay is 0, queue @dwork->work immediately. This is for
1402 * both optimization and correctness. The earliest @timer can
1403 * expire is on the closest next tick and delayed_work users depend
1404 * on that there's no such delay when @delay is 0.
1405 */
1406 if (!delay) {
1407 __queue_work(cpu, wq, &dwork->work);
1408 return;
1409 }
1410
Tejun Heo7beb2ed2012-08-03 10:30:46 -07001411 timer_stats_timer_set_start_info(&dwork->timer);
1412
Lai Jiangshan60c057b2013-02-06 18:04:53 -08001413 dwork->wq = wq;
Tejun Heo12650572012-08-08 09:38:42 -07001414 dwork->cpu = cpu;
Tejun Heo7beb2ed2012-08-03 10:30:46 -07001415 timer->expires = jiffies + delay;
1416
1417 if (unlikely(cpu != WORK_CPU_UNBOUND))
1418 add_timer_on(timer, cpu);
1419 else
1420 add_timer(timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001421}
1422
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -07001423/**
1424 * queue_delayed_work_on - queue work on specific CPU after delay
1425 * @cpu: CPU number to execute work on
1426 * @wq: workqueue to use
Randy Dunlapaf9997e2006-12-22 01:06:52 -08001427 * @dwork: work to queue
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -07001428 * @delay: number of jiffies to wait before queueing
1429 *
Tejun Heo715f1302012-08-03 10:30:46 -07001430 * Returns %false if @work was already on a queue, %true otherwise. If
1431 * @delay is zero and @dwork is idle, it will be scheduled for immediate
1432 * execution.
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -07001433 */
Tejun Heod4283e92012-08-03 10:30:44 -07001434bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
1435 struct delayed_work *dwork, unsigned long delay)
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -07001436{
David Howells52bad642006-11-22 14:54:01 +00001437 struct work_struct *work = &dwork->work;
Tejun Heod4283e92012-08-03 10:30:44 -07001438 bool ret = false;
Tejun Heo8930cab2012-08-03 10:30:45 -07001439 unsigned long flags;
1440
1441 /* read the comment in __queue_work() */
1442 local_irq_save(flags);
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -07001443
Tejun Heo22df02b2010-06-29 10:07:10 +02001444 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
Tejun Heo7beb2ed2012-08-03 10:30:46 -07001445 __queue_delayed_work(cpu, wq, dwork, delay);
Tejun Heod4283e92012-08-03 10:30:44 -07001446 ret = true;
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -07001447 }
Tejun Heo8930cab2012-08-03 10:30:45 -07001448
1449 local_irq_restore(flags);
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -07001450 return ret;
1451}
Dave Jonesae90dd52006-06-30 01:40:45 -04001452EXPORT_SYMBOL_GPL(queue_delayed_work_on);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001453
Tejun Heoc8e55f32010-06-29 10:07:12 +02001454/**
Tejun Heo8376fe22012-08-03 10:30:47 -07001455 * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
1456 * @cpu: CPU number to execute work on
1457 * @wq: workqueue to use
1458 * @dwork: work to queue
1459 * @delay: number of jiffies to wait before queueing
1460 *
1461 * If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
1462 * modify @dwork's timer so that it expires after @delay. If @delay is
1463 * zero, @work is guaranteed to be scheduled immediately regardless of its
1464 * current state.
1465 *
1466 * Returns %false if @dwork was idle and queued, %true if @dwork was
1467 * pending and its timer was modified.
1468 *
Tejun Heoe0aecdd2012-08-21 13:18:24 -07001469 * This function is safe to call from any context including IRQ handler.
Tejun Heo8376fe22012-08-03 10:30:47 -07001470 * See try_to_grab_pending() for details.
1471 */
1472bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
1473 struct delayed_work *dwork, unsigned long delay)
1474{
1475 unsigned long flags;
1476 int ret;
1477
1478 do {
1479 ret = try_to_grab_pending(&dwork->work, true, &flags);
1480 } while (unlikely(ret == -EAGAIN));
1481
1482 if (likely(ret >= 0)) {
1483 __queue_delayed_work(cpu, wq, dwork, delay);
1484 local_irq_restore(flags);
1485 }
1486
1487 /* -ENOENT from try_to_grab_pending() becomes %true */
1488 return ret;
1489}
1490EXPORT_SYMBOL_GPL(mod_delayed_work_on);
1491
1492/**
Tejun Heoc8e55f32010-06-29 10:07:12 +02001493 * worker_enter_idle - enter idle state
1494 * @worker: worker which is entering idle state
1495 *
1496 * @worker is entering idle state. Update stats and idle timer if
1497 * necessary.
1498 *
1499 * LOCKING:
Tejun Heod565ed62013-01-24 11:01:33 -08001500 * spin_lock_irq(pool->lock).
Tejun Heoc8e55f32010-06-29 10:07:12 +02001501 */
1502static void worker_enter_idle(struct worker *worker)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001503{
Tejun Heobd7bdd42012-07-12 14:46:37 -07001504 struct worker_pool *pool = worker->pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001505
Tejun Heo6183c002013-03-12 11:29:57 -07001506 if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||
1507 WARN_ON_ONCE(!list_empty(&worker->entry) &&
1508 (worker->hentry.next || worker->hentry.pprev)))
1509 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001510
Tejun Heocb444762010-07-02 10:03:50 +02001511 /* can't use worker_set_flags(), also called from start_worker() */
1512 worker->flags |= WORKER_IDLE;
Tejun Heobd7bdd42012-07-12 14:46:37 -07001513 pool->nr_idle++;
Tejun Heoe22bee72010-06-29 10:07:14 +02001514 worker->last_active = jiffies;
Peter Zijlstrad5abe662006-12-06 20:37:26 -08001515
Tejun Heoc8e55f32010-06-29 10:07:12 +02001516 /* idle_list is LIFO */
Tejun Heobd7bdd42012-07-12 14:46:37 -07001517 list_add(&worker->entry, &pool->idle_list);
Tejun Heodb7bccf2010-06-29 10:07:12 +02001518
Tejun Heo628c78e2012-07-17 12:39:27 -07001519 if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
1520 mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
Tejun Heocb444762010-07-02 10:03:50 +02001521
Tejun Heo544ecf32012-05-14 15:04:50 -07001522 /*
Tejun Heo706026c2013-01-24 11:01:34 -08001523 * Sanity check nr_running. Because wq_unbind_fn() releases
Tejun Heod565ed62013-01-24 11:01:33 -08001524 * pool->lock between setting %WORKER_UNBOUND and zapping
Tejun Heo628c78e2012-07-17 12:39:27 -07001525 * nr_running, the warning may trigger spuriously. Check iff
1526 * unbind is not in progress.
Tejun Heo544ecf32012-05-14 15:04:50 -07001527 */
Tejun Heo24647572013-01-24 11:01:33 -08001528 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
Tejun Heobd7bdd42012-07-12 14:46:37 -07001529 pool->nr_workers == pool->nr_idle &&
Tejun Heoe19e3972013-01-24 11:39:44 -08001530 atomic_read(&pool->nr_running));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001531}
1532
Tejun Heoc8e55f32010-06-29 10:07:12 +02001533/**
1534 * worker_leave_idle - leave idle state
1535 * @worker: worker which is leaving idle state
1536 *
1537 * @worker is leaving idle state. Update stats.
1538 *
1539 * LOCKING:
Tejun Heod565ed62013-01-24 11:01:33 -08001540 * spin_lock_irq(pool->lock).
Tejun Heoc8e55f32010-06-29 10:07:12 +02001541 */
1542static void worker_leave_idle(struct worker *worker)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001543{
Tejun Heobd7bdd42012-07-12 14:46:37 -07001544 struct worker_pool *pool = worker->pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001545
Tejun Heo6183c002013-03-12 11:29:57 -07001546 if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
1547 return;
Tejun Heod302f012010-06-29 10:07:13 +02001548 worker_clr_flags(worker, WORKER_IDLE);
Tejun Heobd7bdd42012-07-12 14:46:37 -07001549 pool->nr_idle--;
Tejun Heoc8e55f32010-06-29 10:07:12 +02001550 list_del_init(&worker->entry);
1551}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001552
Tejun Heoe22bee72010-06-29 10:07:14 +02001553/**
Lai Jiangshanf36dc672013-02-19 12:17:02 -08001554 * worker_maybe_bind_and_lock - try to bind %current to worker_pool and lock it
1555 * @pool: target worker_pool
1556 *
1557 * Bind %current to the cpu of @pool if it is associated and lock @pool.
Tejun Heoe22bee72010-06-29 10:07:14 +02001558 *
1559 * Works which are scheduled while the cpu is online must at least be
1560 * scheduled to a worker which is bound to the cpu so that if they are
1561 * flushed from cpu callbacks while cpu is going down, they are
1562 * guaranteed to execute on the cpu.
1563 *
Lai Jiangshanf5faa072013-02-19 12:17:02 -08001564 * This function is to be used by unbound workers and rescuers to bind
Tejun Heoe22bee72010-06-29 10:07:14 +02001565 * themselves to the target cpu and may race with cpu going down or
1566 * coming online. kthread_bind() can't be used because it may put the
1567 * worker to already dead cpu and set_cpus_allowed_ptr() can't be used
Tejun Heo706026c2013-01-24 11:01:34 -08001568 * verbatim as it's best effort and blocking and pool may be
Tejun Heoe22bee72010-06-29 10:07:14 +02001569 * [dis]associated in the meantime.
1570 *
Tejun Heo706026c2013-01-24 11:01:34 -08001571 * This function tries set_cpus_allowed() and locks pool and verifies the
Tejun Heo24647572013-01-24 11:01:33 -08001572 * binding against %POOL_DISASSOCIATED which is set during
Tejun Heof2d5a0e2012-07-17 12:39:26 -07001573 * %CPU_DOWN_PREPARE and cleared during %CPU_ONLINE, so if the worker
1574 * enters idle state or fetches works without dropping lock, it can
1575 * guarantee the scheduling requirement described in the first paragraph.
Tejun Heoe22bee72010-06-29 10:07:14 +02001576 *
1577 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08001578 * Might sleep. Called without any lock but returns with pool->lock
Tejun Heoe22bee72010-06-29 10:07:14 +02001579 * held.
1580 *
1581 * RETURNS:
Tejun Heo706026c2013-01-24 11:01:34 -08001582 * %true if the associated pool is online (@worker is successfully
Tejun Heoe22bee72010-06-29 10:07:14 +02001583 * bound), %false if offline.
1584 */
Lai Jiangshanf36dc672013-02-19 12:17:02 -08001585static bool worker_maybe_bind_and_lock(struct worker_pool *pool)
Tejun Heod565ed62013-01-24 11:01:33 -08001586__acquires(&pool->lock)
Tejun Heoe22bee72010-06-29 10:07:14 +02001587{
Tejun Heoe22bee72010-06-29 10:07:14 +02001588 while (true) {
1589 /*
1590 * The following call may fail, succeed or succeed
1591 * without actually migrating the task to the cpu if
1592 * it races with cpu hotunplug operation. Verify
Tejun Heo24647572013-01-24 11:01:33 -08001593 * against POOL_DISASSOCIATED.
Tejun Heoe22bee72010-06-29 10:07:14 +02001594 */
Tejun Heo24647572013-01-24 11:01:33 -08001595 if (!(pool->flags & POOL_DISASSOCIATED))
Tejun Heo7a4e3442013-03-12 11:30:00 -07001596 set_cpus_allowed_ptr(current, pool->attrs->cpumask);
Oleg Nesterov85f41862007-05-09 02:34:20 -07001597
Tejun Heod565ed62013-01-24 11:01:33 -08001598 spin_lock_irq(&pool->lock);
Tejun Heo24647572013-01-24 11:01:33 -08001599 if (pool->flags & POOL_DISASSOCIATED)
Tejun Heoe22bee72010-06-29 10:07:14 +02001600 return false;
Lai Jiangshanf5faa072013-02-19 12:17:02 -08001601 if (task_cpu(current) == pool->cpu &&
Tejun Heo7a4e3442013-03-12 11:30:00 -07001602 cpumask_equal(&current->cpus_allowed, pool->attrs->cpumask))
Tejun Heoe22bee72010-06-29 10:07:14 +02001603 return true;
Tejun Heod565ed62013-01-24 11:01:33 -08001604 spin_unlock_irq(&pool->lock);
Oleg Nesterov3af244332007-05-09 02:34:09 -07001605
Tejun Heo5035b202011-04-29 18:08:37 +02001606 /*
1607 * We've raced with CPU hot[un]plug. Give it a breather
1608 * and retry migration. cond_resched() is required here;
1609 * otherwise, we might deadlock against cpu_stop trying to
1610 * bring down the CPU on non-preemptive kernel.
1611 */
Tejun Heoe22bee72010-06-29 10:07:14 +02001612 cpu_relax();
Tejun Heo5035b202011-04-29 18:08:37 +02001613 cond_resched();
Tejun Heoe22bee72010-06-29 10:07:14 +02001614 }
1615}
1616
Tejun Heoc34056a2010-06-29 10:07:11 +02001617static struct worker *alloc_worker(void)
1618{
1619 struct worker *worker;
1620
1621 worker = kzalloc(sizeof(*worker), GFP_KERNEL);
Tejun Heoc8e55f32010-06-29 10:07:12 +02001622 if (worker) {
1623 INIT_LIST_HEAD(&worker->entry);
Tejun Heoaffee4b2010-06-29 10:07:12 +02001624 INIT_LIST_HEAD(&worker->scheduled);
Tejun Heoe22bee72010-06-29 10:07:14 +02001625 /* on creation a worker is in !idle && prep state */
1626 worker->flags = WORKER_PREP;
Tejun Heoc8e55f32010-06-29 10:07:12 +02001627 }
Tejun Heoc34056a2010-06-29 10:07:11 +02001628 return worker;
1629}
1630
1631/**
1632 * create_worker - create a new workqueue worker
Tejun Heo63d95a92012-07-12 14:46:37 -07001633 * @pool: pool the new worker will belong to
Tejun Heoc34056a2010-06-29 10:07:11 +02001634 *
Tejun Heo63d95a92012-07-12 14:46:37 -07001635 * Create a new worker which is bound to @pool. The returned worker
Tejun Heoc34056a2010-06-29 10:07:11 +02001636 * can be started by calling start_worker() or destroyed using
1637 * destroy_worker().
1638 *
1639 * CONTEXT:
1640 * Might sleep. Does GFP_KERNEL allocations.
1641 *
1642 * RETURNS:
1643 * Pointer to the newly created worker.
1644 */
Tejun Heobc2ae0f2012-07-17 12:39:27 -07001645static struct worker *create_worker(struct worker_pool *pool)
Tejun Heoc34056a2010-06-29 10:07:11 +02001646{
Tejun Heo7a4e3442013-03-12 11:30:00 -07001647 const char *pri = pool->attrs->nice < 0 ? "H" : "";
Tejun Heoc34056a2010-06-29 10:07:11 +02001648 struct worker *worker = NULL;
Tejun Heof3421792010-07-02 10:03:51 +02001649 int id = -1;
Tejun Heoc34056a2010-06-29 10:07:11 +02001650
Tejun Heocd549682013-03-13 19:47:39 -07001651 lockdep_assert_held(&pool->manager_mutex);
1652
Tejun Heo822d8402013-03-19 13:45:21 -07001653 /*
1654 * ID is needed to determine kthread name. Allocate ID first
1655 * without installing the pointer.
1656 */
1657 idr_preload(GFP_KERNEL);
Tejun Heod565ed62013-01-24 11:01:33 -08001658 spin_lock_irq(&pool->lock);
Tejun Heo822d8402013-03-19 13:45:21 -07001659
1660 id = idr_alloc(&pool->worker_idr, NULL, 0, 0, GFP_NOWAIT);
1661
Tejun Heod565ed62013-01-24 11:01:33 -08001662 spin_unlock_irq(&pool->lock);
Tejun Heo822d8402013-03-19 13:45:21 -07001663 idr_preload_end();
1664 if (id < 0)
1665 goto fail;
Tejun Heoc34056a2010-06-29 10:07:11 +02001666
1667 worker = alloc_worker();
1668 if (!worker)
1669 goto fail;
1670
Tejun Heobd7bdd42012-07-12 14:46:37 -07001671 worker->pool = pool;
Tejun Heoc34056a2010-06-29 10:07:11 +02001672 worker->id = id;
1673
Tejun Heo29c91e92013-03-12 11:30:03 -07001674 if (pool->cpu >= 0)
Eric Dumazet94dcf292011-03-22 16:30:45 -07001675 worker->task = kthread_create_on_node(worker_thread,
Tejun Heoec22ca52013-01-24 11:01:33 -08001676 worker, cpu_to_node(pool->cpu),
Tejun Heod84ff052013-03-12 11:29:59 -07001677 "kworker/%d:%d%s", pool->cpu, id, pri);
Tejun Heof3421792010-07-02 10:03:51 +02001678 else
1679 worker->task = kthread_create(worker_thread, worker,
Tejun Heoac6104c2013-03-12 11:30:03 -07001680 "kworker/u%d:%d%s",
1681 pool->id, id, pri);
Tejun Heoc34056a2010-06-29 10:07:11 +02001682 if (IS_ERR(worker->task))
1683 goto fail;
1684
Tejun Heoc5aa87b2013-03-13 16:51:36 -07001685 /*
1686 * set_cpus_allowed_ptr() will fail if the cpumask doesn't have any
1687 * online CPUs. It'll be re-applied when any of the CPUs come up.
1688 */
Tejun Heo7a4e3442013-03-12 11:30:00 -07001689 set_user_nice(worker->task, pool->attrs->nice);
1690 set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
Tejun Heo32704762012-07-13 22:16:45 -07001691
Tejun Heo14a40ff2013-03-19 13:45:20 -07001692 /* prevent userland from meddling with cpumask of workqueue workers */
1693 worker->task->flags |= PF_NO_SETAFFINITY;
Tejun Heo7a4e3442013-03-12 11:30:00 -07001694
1695 /*
1696 * The caller is responsible for ensuring %POOL_DISASSOCIATED
1697 * remains stable across this function. See the comments above the
1698 * flag definition for details.
1699 */
1700 if (pool->flags & POOL_DISASSOCIATED)
Tejun Heobc2ae0f2012-07-17 12:39:27 -07001701 worker->flags |= WORKER_UNBOUND;
Oleg Nesterov3af244332007-05-09 02:34:09 -07001702
Tejun Heo822d8402013-03-19 13:45:21 -07001703 /* successful, commit the pointer to idr */
1704 spin_lock_irq(&pool->lock);
1705 idr_replace(&pool->worker_idr, worker, worker->id);
1706 spin_unlock_irq(&pool->lock);
1707
Tejun Heoc34056a2010-06-29 10:07:11 +02001708 return worker;
Tejun Heo822d8402013-03-19 13:45:21 -07001709
Tejun Heoc34056a2010-06-29 10:07:11 +02001710fail:
1711 if (id >= 0) {
Tejun Heod565ed62013-01-24 11:01:33 -08001712 spin_lock_irq(&pool->lock);
Tejun Heo822d8402013-03-19 13:45:21 -07001713 idr_remove(&pool->worker_idr, id);
Tejun Heod565ed62013-01-24 11:01:33 -08001714 spin_unlock_irq(&pool->lock);
Tejun Heoc34056a2010-06-29 10:07:11 +02001715 }
1716 kfree(worker);
1717 return NULL;
1718}
1719
1720/**
1721 * start_worker - start a newly created worker
1722 * @worker: worker to start
1723 *
Tejun Heo706026c2013-01-24 11:01:34 -08001724 * Make the pool aware of @worker and start it.
Tejun Heoc34056a2010-06-29 10:07:11 +02001725 *
1726 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08001727 * spin_lock_irq(pool->lock).
Tejun Heoc34056a2010-06-29 10:07:11 +02001728 */
1729static void start_worker(struct worker *worker)
1730{
Tejun Heocb444762010-07-02 10:03:50 +02001731 worker->flags |= WORKER_STARTED;
Tejun Heobd7bdd42012-07-12 14:46:37 -07001732 worker->pool->nr_workers++;
Tejun Heoc8e55f32010-06-29 10:07:12 +02001733 worker_enter_idle(worker);
Tejun Heoc34056a2010-06-29 10:07:11 +02001734 wake_up_process(worker->task);
1735}
1736
1737/**
Tejun Heoebf44d12013-03-13 19:47:39 -07001738 * create_and_start_worker - create and start a worker for a pool
1739 * @pool: the target pool
1740 *
Tejun Heocd549682013-03-13 19:47:39 -07001741 * Grab the managership of @pool and create and start a new worker for it.
Tejun Heoebf44d12013-03-13 19:47:39 -07001742 */
1743static int create_and_start_worker(struct worker_pool *pool)
1744{
1745 struct worker *worker;
1746
Tejun Heocd549682013-03-13 19:47:39 -07001747 mutex_lock(&pool->manager_mutex);
1748
Tejun Heoebf44d12013-03-13 19:47:39 -07001749 worker = create_worker(pool);
1750 if (worker) {
1751 spin_lock_irq(&pool->lock);
1752 start_worker(worker);
1753 spin_unlock_irq(&pool->lock);
1754 }
1755
Tejun Heocd549682013-03-13 19:47:39 -07001756 mutex_unlock(&pool->manager_mutex);
1757
Tejun Heoebf44d12013-03-13 19:47:39 -07001758 return worker ? 0 : -ENOMEM;
1759}
1760
1761/**
Tejun Heoc34056a2010-06-29 10:07:11 +02001762 * destroy_worker - destroy a workqueue worker
1763 * @worker: worker to be destroyed
1764 *
Tejun Heo706026c2013-01-24 11:01:34 -08001765 * Destroy @worker and adjust @pool stats accordingly.
Tejun Heoc8e55f32010-06-29 10:07:12 +02001766 *
1767 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08001768 * spin_lock_irq(pool->lock) which is released and regrabbed.
Tejun Heoc34056a2010-06-29 10:07:11 +02001769 */
1770static void destroy_worker(struct worker *worker)
1771{
Tejun Heobd7bdd42012-07-12 14:46:37 -07001772 struct worker_pool *pool = worker->pool;
Tejun Heoc34056a2010-06-29 10:07:11 +02001773
Tejun Heocd549682013-03-13 19:47:39 -07001774 lockdep_assert_held(&pool->manager_mutex);
1775 lockdep_assert_held(&pool->lock);
1776
Tejun Heoc34056a2010-06-29 10:07:11 +02001777 /* sanity check frenzy */
Tejun Heo6183c002013-03-12 11:29:57 -07001778 if (WARN_ON(worker->current_work) ||
1779 WARN_ON(!list_empty(&worker->scheduled)))
1780 return;
Tejun Heoc34056a2010-06-29 10:07:11 +02001781
Tejun Heoc8e55f32010-06-29 10:07:12 +02001782 if (worker->flags & WORKER_STARTED)
Tejun Heobd7bdd42012-07-12 14:46:37 -07001783 pool->nr_workers--;
Tejun Heoc8e55f32010-06-29 10:07:12 +02001784 if (worker->flags & WORKER_IDLE)
Tejun Heobd7bdd42012-07-12 14:46:37 -07001785 pool->nr_idle--;
Tejun Heoc8e55f32010-06-29 10:07:12 +02001786
1787 list_del_init(&worker->entry);
Tejun Heocb444762010-07-02 10:03:50 +02001788 worker->flags |= WORKER_DIE;
Tejun Heoc8e55f32010-06-29 10:07:12 +02001789
Tejun Heo822d8402013-03-19 13:45:21 -07001790 idr_remove(&pool->worker_idr, worker->id);
1791
Tejun Heod565ed62013-01-24 11:01:33 -08001792 spin_unlock_irq(&pool->lock);
Tejun Heoc8e55f32010-06-29 10:07:12 +02001793
Tejun Heoc34056a2010-06-29 10:07:11 +02001794 kthread_stop(worker->task);
1795 kfree(worker);
1796
Tejun Heod565ed62013-01-24 11:01:33 -08001797 spin_lock_irq(&pool->lock);
Tejun Heoc34056a2010-06-29 10:07:11 +02001798}
1799
Tejun Heo63d95a92012-07-12 14:46:37 -07001800static void idle_worker_timeout(unsigned long __pool)
Tejun Heoe22bee72010-06-29 10:07:14 +02001801{
Tejun Heo63d95a92012-07-12 14:46:37 -07001802 struct worker_pool *pool = (void *)__pool;
Tejun Heoe22bee72010-06-29 10:07:14 +02001803
Tejun Heod565ed62013-01-24 11:01:33 -08001804 spin_lock_irq(&pool->lock);
Tejun Heoe22bee72010-06-29 10:07:14 +02001805
Tejun Heo63d95a92012-07-12 14:46:37 -07001806 if (too_many_workers(pool)) {
Tejun Heoe22bee72010-06-29 10:07:14 +02001807 struct worker *worker;
1808 unsigned long expires;
1809
1810 /* idle_list is kept in LIFO order, check the last one */
Tejun Heo63d95a92012-07-12 14:46:37 -07001811 worker = list_entry(pool->idle_list.prev, struct worker, entry);
Tejun Heoe22bee72010-06-29 10:07:14 +02001812 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
1813
1814 if (time_before(jiffies, expires))
Tejun Heo63d95a92012-07-12 14:46:37 -07001815 mod_timer(&pool->idle_timer, expires);
Tejun Heoe22bee72010-06-29 10:07:14 +02001816 else {
1817 /* it's been idle for too long, wake up manager */
Tejun Heo11ebea52012-07-12 14:46:37 -07001818 pool->flags |= POOL_MANAGE_WORKERS;
Tejun Heo63d95a92012-07-12 14:46:37 -07001819 wake_up_worker(pool);
Tejun Heoe22bee72010-06-29 10:07:14 +02001820 }
1821 }
1822
Tejun Heod565ed62013-01-24 11:01:33 -08001823 spin_unlock_irq(&pool->lock);
Tejun Heoe22bee72010-06-29 10:07:14 +02001824}
1825
Tejun Heo493a1722013-03-12 11:29:59 -07001826static void send_mayday(struct work_struct *work)
Tejun Heoe22bee72010-06-29 10:07:14 +02001827{
Tejun Heo112202d2013-02-13 19:29:12 -08001828 struct pool_workqueue *pwq = get_work_pwq(work);
1829 struct workqueue_struct *wq = pwq->wq;
Tejun Heo493a1722013-03-12 11:29:59 -07001830
Tejun Heo2e109a22013-03-13 19:47:40 -07001831 lockdep_assert_held(&wq_mayday_lock);
Tejun Heoe22bee72010-06-29 10:07:14 +02001832
Tejun Heo493008a2013-03-12 11:30:03 -07001833 if (!wq->rescuer)
Tejun Heo493a1722013-03-12 11:29:59 -07001834 return;
Tejun Heoe22bee72010-06-29 10:07:14 +02001835
1836 /* mayday mayday mayday */
Tejun Heo493a1722013-03-12 11:29:59 -07001837 if (list_empty(&pwq->mayday_node)) {
1838 list_add_tail(&pwq->mayday_node, &wq->maydays);
Tejun Heoe22bee72010-06-29 10:07:14 +02001839 wake_up_process(wq->rescuer->task);
Tejun Heo493a1722013-03-12 11:29:59 -07001840 }
Tejun Heoe22bee72010-06-29 10:07:14 +02001841}
1842
Tejun Heo706026c2013-01-24 11:01:34 -08001843static void pool_mayday_timeout(unsigned long __pool)
Tejun Heoe22bee72010-06-29 10:07:14 +02001844{
Tejun Heo63d95a92012-07-12 14:46:37 -07001845 struct worker_pool *pool = (void *)__pool;
Tejun Heoe22bee72010-06-29 10:07:14 +02001846 struct work_struct *work;
1847
Tejun Heo2e109a22013-03-13 19:47:40 -07001848 spin_lock_irq(&wq_mayday_lock); /* for wq->maydays */
Tejun Heo493a1722013-03-12 11:29:59 -07001849 spin_lock(&pool->lock);
Tejun Heoe22bee72010-06-29 10:07:14 +02001850
Tejun Heo63d95a92012-07-12 14:46:37 -07001851 if (need_to_create_worker(pool)) {
Tejun Heoe22bee72010-06-29 10:07:14 +02001852 /*
1853 * We've been trying to create a new worker but
1854 * haven't been successful. We might be hitting an
1855 * allocation deadlock. Send distress signals to
1856 * rescuers.
1857 */
Tejun Heo63d95a92012-07-12 14:46:37 -07001858 list_for_each_entry(work, &pool->worklist, entry)
Tejun Heoe22bee72010-06-29 10:07:14 +02001859 send_mayday(work);
1860 }
1861
Tejun Heo493a1722013-03-12 11:29:59 -07001862 spin_unlock(&pool->lock);
Tejun Heo2e109a22013-03-13 19:47:40 -07001863 spin_unlock_irq(&wq_mayday_lock);
Tejun Heoe22bee72010-06-29 10:07:14 +02001864
Tejun Heo63d95a92012-07-12 14:46:37 -07001865 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
Tejun Heoe22bee72010-06-29 10:07:14 +02001866}
1867
1868/**
1869 * maybe_create_worker - create a new worker if necessary
Tejun Heo63d95a92012-07-12 14:46:37 -07001870 * @pool: pool to create a new worker for
Tejun Heoe22bee72010-06-29 10:07:14 +02001871 *
Tejun Heo63d95a92012-07-12 14:46:37 -07001872 * Create a new worker for @pool if necessary. @pool is guaranteed to
Tejun Heoe22bee72010-06-29 10:07:14 +02001873 * have at least one idle worker on return from this function. If
1874 * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
Tejun Heo63d95a92012-07-12 14:46:37 -07001875 * sent to all rescuers with works scheduled on @pool to resolve
Tejun Heoe22bee72010-06-29 10:07:14 +02001876 * possible allocation deadlock.
1877 *
Tejun Heoc5aa87b2013-03-13 16:51:36 -07001878 * On return, need_to_create_worker() is guaranteed to be %false and
1879 * may_start_working() %true.
Tejun Heoe22bee72010-06-29 10:07:14 +02001880 *
1881 * LOCKING:
Tejun Heod565ed62013-01-24 11:01:33 -08001882 * spin_lock_irq(pool->lock) which may be released and regrabbed
Tejun Heoe22bee72010-06-29 10:07:14 +02001883 * multiple times. Does GFP_KERNEL allocations. Called only from
1884 * manager.
1885 *
1886 * RETURNS:
Tejun Heoc5aa87b2013-03-13 16:51:36 -07001887 * %false if no action was taken and pool->lock stayed locked, %true
Tejun Heoe22bee72010-06-29 10:07:14 +02001888 * otherwise.
1889 */
Tejun Heo63d95a92012-07-12 14:46:37 -07001890static bool maybe_create_worker(struct worker_pool *pool)
Tejun Heod565ed62013-01-24 11:01:33 -08001891__releases(&pool->lock)
1892__acquires(&pool->lock)
Tejun Heoe22bee72010-06-29 10:07:14 +02001893{
Tejun Heo63d95a92012-07-12 14:46:37 -07001894 if (!need_to_create_worker(pool))
Tejun Heoe22bee72010-06-29 10:07:14 +02001895 return false;
1896restart:
Tejun Heod565ed62013-01-24 11:01:33 -08001897 spin_unlock_irq(&pool->lock);
Tejun Heo9f9c2362010-07-14 11:31:20 +02001898
Tejun Heoe22bee72010-06-29 10:07:14 +02001899 /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
Tejun Heo63d95a92012-07-12 14:46:37 -07001900 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
Tejun Heoe22bee72010-06-29 10:07:14 +02001901
1902 while (true) {
1903 struct worker *worker;
1904
Tejun Heobc2ae0f2012-07-17 12:39:27 -07001905 worker = create_worker(pool);
Tejun Heoe22bee72010-06-29 10:07:14 +02001906 if (worker) {
Tejun Heo63d95a92012-07-12 14:46:37 -07001907 del_timer_sync(&pool->mayday_timer);
Tejun Heod565ed62013-01-24 11:01:33 -08001908 spin_lock_irq(&pool->lock);
Tejun Heoe22bee72010-06-29 10:07:14 +02001909 start_worker(worker);
Tejun Heo6183c002013-03-12 11:29:57 -07001910 if (WARN_ON_ONCE(need_to_create_worker(pool)))
1911 goto restart;
Tejun Heoe22bee72010-06-29 10:07:14 +02001912 return true;
1913 }
1914
Tejun Heo63d95a92012-07-12 14:46:37 -07001915 if (!need_to_create_worker(pool))
Tejun Heoe22bee72010-06-29 10:07:14 +02001916 break;
1917
Tejun Heoe22bee72010-06-29 10:07:14 +02001918 __set_current_state(TASK_INTERRUPTIBLE);
1919 schedule_timeout(CREATE_COOLDOWN);
Tejun Heo9f9c2362010-07-14 11:31:20 +02001920
Tejun Heo63d95a92012-07-12 14:46:37 -07001921 if (!need_to_create_worker(pool))
Tejun Heoe22bee72010-06-29 10:07:14 +02001922 break;
1923 }
1924
Tejun Heo63d95a92012-07-12 14:46:37 -07001925 del_timer_sync(&pool->mayday_timer);
Tejun Heod565ed62013-01-24 11:01:33 -08001926 spin_lock_irq(&pool->lock);
Tejun Heo63d95a92012-07-12 14:46:37 -07001927 if (need_to_create_worker(pool))
Tejun Heoe22bee72010-06-29 10:07:14 +02001928 goto restart;
1929 return true;
1930}
1931
1932/**
1933 * maybe_destroy_worker - destroy workers which have been idle for a while
Tejun Heo63d95a92012-07-12 14:46:37 -07001934 * @pool: pool to destroy workers for
Tejun Heoe22bee72010-06-29 10:07:14 +02001935 *
Tejun Heo63d95a92012-07-12 14:46:37 -07001936 * Destroy @pool workers which have been idle for longer than
Tejun Heoe22bee72010-06-29 10:07:14 +02001937 * IDLE_WORKER_TIMEOUT.
1938 *
1939 * LOCKING:
Tejun Heod565ed62013-01-24 11:01:33 -08001940 * spin_lock_irq(pool->lock) which may be released and regrabbed
Tejun Heoe22bee72010-06-29 10:07:14 +02001941 * multiple times. Called only from manager.
1942 *
1943 * RETURNS:
Tejun Heoc5aa87b2013-03-13 16:51:36 -07001944 * %false if no action was taken and pool->lock stayed locked, %true
Tejun Heoe22bee72010-06-29 10:07:14 +02001945 * otherwise.
1946 */
Tejun Heo63d95a92012-07-12 14:46:37 -07001947static bool maybe_destroy_workers(struct worker_pool *pool)
Tejun Heoe22bee72010-06-29 10:07:14 +02001948{
1949 bool ret = false;
1950
Tejun Heo63d95a92012-07-12 14:46:37 -07001951 while (too_many_workers(pool)) {
Tejun Heoe22bee72010-06-29 10:07:14 +02001952 struct worker *worker;
1953 unsigned long expires;
1954
Tejun Heo63d95a92012-07-12 14:46:37 -07001955 worker = list_entry(pool->idle_list.prev, struct worker, entry);
Tejun Heoe22bee72010-06-29 10:07:14 +02001956 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
1957
1958 if (time_before(jiffies, expires)) {
Tejun Heo63d95a92012-07-12 14:46:37 -07001959 mod_timer(&pool->idle_timer, expires);
Tejun Heoe22bee72010-06-29 10:07:14 +02001960 break;
1961 }
1962
1963 destroy_worker(worker);
1964 ret = true;
1965 }
1966
1967 return ret;
1968}
1969
1970/**
1971 * manage_workers - manage worker pool
1972 * @worker: self
1973 *
Tejun Heo706026c2013-01-24 11:01:34 -08001974 * Assume the manager role and manage the worker pool @worker belongs
Tejun Heoe22bee72010-06-29 10:07:14 +02001975 * to. At any given time, there can be only zero or one manager per
Tejun Heo706026c2013-01-24 11:01:34 -08001976 * pool. The exclusion is handled automatically by this function.
Tejun Heoe22bee72010-06-29 10:07:14 +02001977 *
1978 * The caller can safely start processing works on false return. On
1979 * true return, it's guaranteed that need_to_create_worker() is false
1980 * and may_start_working() is true.
1981 *
1982 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08001983 * spin_lock_irq(pool->lock) which may be released and regrabbed
Tejun Heoe22bee72010-06-29 10:07:14 +02001984 * multiple times. Does GFP_KERNEL allocations.
1985 *
1986 * RETURNS:
Tejun Heod565ed62013-01-24 11:01:33 -08001987 * spin_lock_irq(pool->lock) which may be released and regrabbed
1988 * multiple times. Does GFP_KERNEL allocations.
Tejun Heoe22bee72010-06-29 10:07:14 +02001989 */
1990static bool manage_workers(struct worker *worker)
1991{
Tejun Heo63d95a92012-07-12 14:46:37 -07001992 struct worker_pool *pool = worker->pool;
Tejun Heoe22bee72010-06-29 10:07:14 +02001993 bool ret = false;
1994
Tejun Heobc3a1af2013-03-13 19:47:39 -07001995 /*
1996 * Managership is governed by two mutexes - manager_arb and
1997 * manager_mutex. manager_arb handles arbitration of manager role.
1998 * Anyone who successfully grabs manager_arb wins the arbitration
1999 * and becomes the manager. mutex_trylock() on pool->manager_arb
2000 * failure while holding pool->lock reliably indicates that someone
2001 * else is managing the pool and the worker which failed trylock
2002 * can proceed to executing work items. This means that anyone
2003 * grabbing manager_arb is responsible for actually performing
2004 * manager duties. If manager_arb is grabbed and released without
2005 * actual management, the pool may stall indefinitely.
2006 *
2007 * manager_mutex is used for exclusion of actual management
2008 * operations. The holder of manager_mutex can be sure that none
2009 * of management operations, including creation and destruction of
2010 * workers, won't take place until the mutex is released. Because
2011 * manager_mutex doesn't interfere with manager role arbitration,
2012 * it is guaranteed that the pool's management, while may be
2013 * delayed, won't be disturbed by someone else grabbing
2014 * manager_mutex.
2015 */
Tejun Heo34a06bd2013-03-12 11:30:00 -07002016 if (!mutex_trylock(&pool->manager_arb))
Tejun Heoe22bee72010-06-29 10:07:14 +02002017 return ret;
2018
Lai Jiangshanee378aa2012-09-10 10:03:44 -07002019 /*
Tejun Heobc3a1af2013-03-13 19:47:39 -07002020 * With manager arbitration won, manager_mutex would be free in
2021 * most cases. trylock first without dropping @pool->lock.
Lai Jiangshanee378aa2012-09-10 10:03:44 -07002022 */
Tejun Heobc3a1af2013-03-13 19:47:39 -07002023 if (unlikely(!mutex_trylock(&pool->manager_mutex))) {
Tejun Heod565ed62013-01-24 11:01:33 -08002024 spin_unlock_irq(&pool->lock);
Tejun Heobc3a1af2013-03-13 19:47:39 -07002025 mutex_lock(&pool->manager_mutex);
Lai Jiangshanee378aa2012-09-10 10:03:44 -07002026 ret = true;
2027 }
2028
Tejun Heo11ebea52012-07-12 14:46:37 -07002029 pool->flags &= ~POOL_MANAGE_WORKERS;
Tejun Heoe22bee72010-06-29 10:07:14 +02002030
2031 /*
2032 * Destroy and then create so that may_start_working() is true
2033 * on return.
2034 */
Tejun Heo63d95a92012-07-12 14:46:37 -07002035 ret |= maybe_destroy_workers(pool);
2036 ret |= maybe_create_worker(pool);
Tejun Heoe22bee72010-06-29 10:07:14 +02002037
Tejun Heobc3a1af2013-03-13 19:47:39 -07002038 mutex_unlock(&pool->manager_mutex);
Tejun Heo34a06bd2013-03-12 11:30:00 -07002039 mutex_unlock(&pool->manager_arb);
Tejun Heoe22bee72010-06-29 10:07:14 +02002040 return ret;
2041}
2042
Tejun Heoa62428c2010-06-29 10:07:10 +02002043/**
2044 * process_one_work - process single work
Tejun Heoc34056a2010-06-29 10:07:11 +02002045 * @worker: self
Tejun Heoa62428c2010-06-29 10:07:10 +02002046 * @work: work to process
2047 *
2048 * Process @work. This function contains all the logics necessary to
2049 * process a single work including synchronization against and
2050 * interaction with other workers on the same cpu, queueing and
2051 * flushing. As long as context requirement is met, any worker can
2052 * call this function to process a work.
2053 *
2054 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08002055 * spin_lock_irq(pool->lock) which is released and regrabbed.
Tejun Heoa62428c2010-06-29 10:07:10 +02002056 */
Tejun Heoc34056a2010-06-29 10:07:11 +02002057static void process_one_work(struct worker *worker, struct work_struct *work)
Tejun Heod565ed62013-01-24 11:01:33 -08002058__releases(&pool->lock)
2059__acquires(&pool->lock)
Tejun Heoa62428c2010-06-29 10:07:10 +02002060{
Tejun Heo112202d2013-02-13 19:29:12 -08002061 struct pool_workqueue *pwq = get_work_pwq(work);
Tejun Heobd7bdd42012-07-12 14:46:37 -07002062 struct worker_pool *pool = worker->pool;
Tejun Heo112202d2013-02-13 19:29:12 -08002063 bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
Tejun Heo73f53c42010-06-29 10:07:11 +02002064 int work_color;
Tejun Heo7e116292010-06-29 10:07:13 +02002065 struct worker *collision;
Tejun Heoa62428c2010-06-29 10:07:10 +02002066#ifdef CONFIG_LOCKDEP
2067 /*
2068 * It is permissible to free the struct work_struct from
2069 * inside the function that is called from it, this we need to
2070 * take into account for lockdep too. To avoid bogus "held
2071 * lock freed" warnings as well as problems when looking into
2072 * work->lockdep_map, make a copy and use that here.
2073 */
Peter Zijlstra4d82a1d2012-05-15 08:06:19 -07002074 struct lockdep_map lockdep_map;
2075
2076 lockdep_copy_map(&lockdep_map, &work->lockdep_map);
Tejun Heoa62428c2010-06-29 10:07:10 +02002077#endif
Tejun Heo6fec10a2012-07-22 10:16:34 -07002078 /*
2079 * Ensure we're on the correct CPU. DISASSOCIATED test is
2080 * necessary to avoid spurious warnings from rescuers servicing the
Tejun Heo24647572013-01-24 11:01:33 -08002081 * unbound or a disassociated pool.
Tejun Heo6fec10a2012-07-22 10:16:34 -07002082 */
Lai Jiangshan5f7dabf2012-09-18 09:59:23 -07002083 WARN_ON_ONCE(!(worker->flags & WORKER_UNBOUND) &&
Tejun Heo24647572013-01-24 11:01:33 -08002084 !(pool->flags & POOL_DISASSOCIATED) &&
Tejun Heoec22ca52013-01-24 11:01:33 -08002085 raw_smp_processor_id() != pool->cpu);
Tejun Heo25511a42012-07-17 12:39:27 -07002086
Tejun Heo7e116292010-06-29 10:07:13 +02002087 /*
2088 * A single work shouldn't be executed concurrently by
2089 * multiple workers on a single cpu. Check whether anyone is
2090 * already processing the work. If so, defer the work to the
2091 * currently executing one.
2092 */
Tejun Heoc9e7cf22013-01-24 11:01:33 -08002093 collision = find_worker_executing_work(pool, work);
Tejun Heo7e116292010-06-29 10:07:13 +02002094 if (unlikely(collision)) {
2095 move_linked_works(work, &collision->scheduled, NULL);
2096 return;
2097 }
2098
Tejun Heo8930cab2012-08-03 10:30:45 -07002099 /* claim and dequeue */
Tejun Heoa62428c2010-06-29 10:07:10 +02002100 debug_work_deactivate(work);
Tejun Heoc9e7cf22013-01-24 11:01:33 -08002101 hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
Tejun Heoc34056a2010-06-29 10:07:11 +02002102 worker->current_work = work;
Tejun Heoa2c1c572012-12-18 10:35:02 -08002103 worker->current_func = work->func;
Tejun Heo112202d2013-02-13 19:29:12 -08002104 worker->current_pwq = pwq;
Tejun Heo73f53c42010-06-29 10:07:11 +02002105 work_color = get_work_color(work);
Tejun Heo7a22ad72010-06-29 10:07:13 +02002106
Tejun Heoa62428c2010-06-29 10:07:10 +02002107 list_del_init(&work->entry);
2108
Tejun Heo649027d2010-06-29 10:07:14 +02002109 /*
Tejun Heofb0e7be2010-06-29 10:07:15 +02002110 * CPU intensive works don't participate in concurrency
2111 * management. They're the scheduler's responsibility.
2112 */
2113 if (unlikely(cpu_intensive))
2114 worker_set_flags(worker, WORKER_CPU_INTENSIVE, true);
2115
Tejun Heo974271c2012-07-12 14:46:37 -07002116 /*
Tejun Heod565ed62013-01-24 11:01:33 -08002117 * Unbound pool isn't concurrency managed and work items should be
Tejun Heo974271c2012-07-12 14:46:37 -07002118 * executed ASAP. Wake up another worker if necessary.
2119 */
Tejun Heo63d95a92012-07-12 14:46:37 -07002120 if ((worker->flags & WORKER_UNBOUND) && need_more_worker(pool))
2121 wake_up_worker(pool);
Tejun Heo974271c2012-07-12 14:46:37 -07002122
Tejun Heo8930cab2012-08-03 10:30:45 -07002123 /*
Tejun Heo7c3eed52013-01-24 11:01:33 -08002124 * Record the last pool and clear PENDING which should be the last
Tejun Heod565ed62013-01-24 11:01:33 -08002125 * update to @work. Also, do this inside @pool->lock so that
Tejun Heo23657bb2012-08-13 17:08:19 -07002126 * PENDING and queued state changes happen together while IRQ is
2127 * disabled.
Tejun Heo8930cab2012-08-03 10:30:45 -07002128 */
Tejun Heo7c3eed52013-01-24 11:01:33 -08002129 set_work_pool_and_clear_pending(work, pool->id);
Tejun Heoa62428c2010-06-29 10:07:10 +02002130
Tejun Heod565ed62013-01-24 11:01:33 -08002131 spin_unlock_irq(&pool->lock);
Tejun Heoa62428c2010-06-29 10:07:10 +02002132
Tejun Heo112202d2013-02-13 19:29:12 -08002133 lock_map_acquire_read(&pwq->wq->lockdep_map);
Tejun Heoa62428c2010-06-29 10:07:10 +02002134 lock_map_acquire(&lockdep_map);
Arjan van de Vene36c8862010-08-21 13:07:26 -07002135 trace_workqueue_execute_start(work);
Tejun Heoa2c1c572012-12-18 10:35:02 -08002136 worker->current_func(work);
Arjan van de Vene36c8862010-08-21 13:07:26 -07002137 /*
2138 * While we must be careful to not use "work" after this, the trace
2139 * point will only record its address.
2140 */
2141 trace_workqueue_execute_end(work);
Tejun Heoa62428c2010-06-29 10:07:10 +02002142 lock_map_release(&lockdep_map);
Tejun Heo112202d2013-02-13 19:29:12 -08002143 lock_map_release(&pwq->wq->lockdep_map);
Tejun Heoa62428c2010-06-29 10:07:10 +02002144
2145 if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
Valentin Ilie044c7822012-08-19 00:52:42 +03002146 pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
2147 " last function: %pf\n",
Tejun Heoa2c1c572012-12-18 10:35:02 -08002148 current->comm, preempt_count(), task_pid_nr(current),
2149 worker->current_func);
Tejun Heoa62428c2010-06-29 10:07:10 +02002150 debug_show_held_locks(current);
2151 dump_stack();
2152 }
2153
Tejun Heod565ed62013-01-24 11:01:33 -08002154 spin_lock_irq(&pool->lock);
Tejun Heoa62428c2010-06-29 10:07:10 +02002155
Tejun Heofb0e7be2010-06-29 10:07:15 +02002156 /* clear cpu intensive status */
2157 if (unlikely(cpu_intensive))
2158 worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2159
Tejun Heoa62428c2010-06-29 10:07:10 +02002160 /* we're done with it, release */
Sasha Levin42f85702012-12-17 10:01:23 -05002161 hash_del(&worker->hentry);
Tejun Heoc34056a2010-06-29 10:07:11 +02002162 worker->current_work = NULL;
Tejun Heoa2c1c572012-12-18 10:35:02 -08002163 worker->current_func = NULL;
Tejun Heo112202d2013-02-13 19:29:12 -08002164 worker->current_pwq = NULL;
2165 pwq_dec_nr_in_flight(pwq, work_color);
Tejun Heoa62428c2010-06-29 10:07:10 +02002166}
2167
Tejun Heoaffee4b2010-06-29 10:07:12 +02002168/**
2169 * process_scheduled_works - process scheduled works
2170 * @worker: self
2171 *
2172 * Process all scheduled works. Please note that the scheduled list
2173 * may change while processing a work, so this function repeatedly
2174 * fetches a work from the top and executes it.
2175 *
2176 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08002177 * spin_lock_irq(pool->lock) which may be released and regrabbed
Tejun Heoaffee4b2010-06-29 10:07:12 +02002178 * multiple times.
2179 */
2180static void process_scheduled_works(struct worker *worker)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002181{
Tejun Heoaffee4b2010-06-29 10:07:12 +02002182 while (!list_empty(&worker->scheduled)) {
2183 struct work_struct *work = list_first_entry(&worker->scheduled,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002184 struct work_struct, entry);
Tejun Heoc34056a2010-06-29 10:07:11 +02002185 process_one_work(worker, work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002186 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002187}
2188
Tejun Heo4690c4a2010-06-29 10:07:10 +02002189/**
2190 * worker_thread - the worker thread function
Tejun Heoc34056a2010-06-29 10:07:11 +02002191 * @__worker: self
Tejun Heo4690c4a2010-06-29 10:07:10 +02002192 *
Tejun Heoc5aa87b2013-03-13 16:51:36 -07002193 * The worker thread function. All workers belong to a worker_pool -
2194 * either a per-cpu one or dynamic unbound one. These workers process all
2195 * work items regardless of their specific target workqueue. The only
2196 * exception is work items which belong to workqueues with a rescuer which
2197 * will be explained in rescuer_thread().
Tejun Heo4690c4a2010-06-29 10:07:10 +02002198 */
Tejun Heoc34056a2010-06-29 10:07:11 +02002199static int worker_thread(void *__worker)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002200{
Tejun Heoc34056a2010-06-29 10:07:11 +02002201 struct worker *worker = __worker;
Tejun Heobd7bdd42012-07-12 14:46:37 -07002202 struct worker_pool *pool = worker->pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002203
Tejun Heoe22bee72010-06-29 10:07:14 +02002204 /* tell the scheduler that this is a workqueue worker */
2205 worker->task->flags |= PF_WQ_WORKER;
Tejun Heoc8e55f32010-06-29 10:07:12 +02002206woke_up:
Tejun Heod565ed62013-01-24 11:01:33 -08002207 spin_lock_irq(&pool->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002208
Tejun Heoa9ab7752013-03-19 13:45:21 -07002209 /* am I supposed to die? */
2210 if (unlikely(worker->flags & WORKER_DIE)) {
Tejun Heod565ed62013-01-24 11:01:33 -08002211 spin_unlock_irq(&pool->lock);
Tejun Heoa9ab7752013-03-19 13:45:21 -07002212 WARN_ON_ONCE(!list_empty(&worker->entry));
2213 worker->task->flags &= ~PF_WQ_WORKER;
2214 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002215 }
2216
Tejun Heoc8e55f32010-06-29 10:07:12 +02002217 worker_leave_idle(worker);
Tejun Heodb7bccf2010-06-29 10:07:12 +02002218recheck:
Tejun Heoe22bee72010-06-29 10:07:14 +02002219 /* no more worker necessary? */
Tejun Heo63d95a92012-07-12 14:46:37 -07002220 if (!need_more_worker(pool))
Tejun Heoe22bee72010-06-29 10:07:14 +02002221 goto sleep;
2222
2223 /* do we need to manage? */
Tejun Heo63d95a92012-07-12 14:46:37 -07002224 if (unlikely(!may_start_working(pool)) && manage_workers(worker))
Tejun Heoe22bee72010-06-29 10:07:14 +02002225 goto recheck;
2226
Tejun Heoc8e55f32010-06-29 10:07:12 +02002227 /*
2228 * ->scheduled list can only be filled while a worker is
2229 * preparing to process a work or actually processing it.
2230 * Make sure nobody diddled with it while I was sleeping.
2231 */
Tejun Heo6183c002013-03-12 11:29:57 -07002232 WARN_ON_ONCE(!list_empty(&worker->scheduled));
Tejun Heoc8e55f32010-06-29 10:07:12 +02002233
Tejun Heoe22bee72010-06-29 10:07:14 +02002234 /*
Tejun Heoa9ab7752013-03-19 13:45:21 -07002235 * Finish PREP stage. We're guaranteed to have at least one idle
2236 * worker or that someone else has already assumed the manager
2237 * role. This is where @worker starts participating in concurrency
2238 * management if applicable and concurrency management is restored
2239 * after being rebound. See rebind_workers() for details.
Tejun Heoe22bee72010-06-29 10:07:14 +02002240 */
Tejun Heoa9ab7752013-03-19 13:45:21 -07002241 worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);
Tejun Heoe22bee72010-06-29 10:07:14 +02002242
2243 do {
Tejun Heoc8e55f32010-06-29 10:07:12 +02002244 struct work_struct *work =
Tejun Heobd7bdd42012-07-12 14:46:37 -07002245 list_first_entry(&pool->worklist,
Tejun Heoc8e55f32010-06-29 10:07:12 +02002246 struct work_struct, entry);
2247
2248 if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
2249 /* optimization path, not strictly necessary */
2250 process_one_work(worker, work);
2251 if (unlikely(!list_empty(&worker->scheduled)))
2252 process_scheduled_works(worker);
2253 } else {
2254 move_linked_works(work, &worker->scheduled, NULL);
2255 process_scheduled_works(worker);
2256 }
Tejun Heo63d95a92012-07-12 14:46:37 -07002257 } while (keep_working(pool));
Tejun Heoc8e55f32010-06-29 10:07:12 +02002258
Tejun Heoe22bee72010-06-29 10:07:14 +02002259 worker_set_flags(worker, WORKER_PREP, false);
Tejun Heod313dd82010-07-02 10:03:51 +02002260sleep:
Tejun Heo63d95a92012-07-12 14:46:37 -07002261 if (unlikely(need_to_manage_workers(pool)) && manage_workers(worker))
Tejun Heoe22bee72010-06-29 10:07:14 +02002262 goto recheck;
Tejun Heod313dd82010-07-02 10:03:51 +02002263
Tejun Heoc8e55f32010-06-29 10:07:12 +02002264 /*
Tejun Heod565ed62013-01-24 11:01:33 -08002265 * pool->lock is held and there's no work to process and no need to
2266 * manage, sleep. Workers are woken up only while holding
2267 * pool->lock or from local cpu, so setting the current state
2268 * before releasing pool->lock is enough to prevent losing any
2269 * event.
Tejun Heoc8e55f32010-06-29 10:07:12 +02002270 */
2271 worker_enter_idle(worker);
2272 __set_current_state(TASK_INTERRUPTIBLE);
Tejun Heod565ed62013-01-24 11:01:33 -08002273 spin_unlock_irq(&pool->lock);
Tejun Heoc8e55f32010-06-29 10:07:12 +02002274 schedule();
2275 goto woke_up;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002276}
2277
Tejun Heoe22bee72010-06-29 10:07:14 +02002278/**
2279 * rescuer_thread - the rescuer thread function
Tejun Heo111c2252013-01-17 17:16:24 -08002280 * @__rescuer: self
Tejun Heoe22bee72010-06-29 10:07:14 +02002281 *
2282 * Workqueue rescuer thread function. There's one rescuer for each
Tejun Heo493008a2013-03-12 11:30:03 -07002283 * workqueue which has WQ_MEM_RECLAIM set.
Tejun Heoe22bee72010-06-29 10:07:14 +02002284 *
Tejun Heo706026c2013-01-24 11:01:34 -08002285 * Regular work processing on a pool may block trying to create a new
Tejun Heoe22bee72010-06-29 10:07:14 +02002286 * worker which uses GFP_KERNEL allocation which has slight chance of
2287 * developing into deadlock if some works currently on the same queue
2288 * need to be processed to satisfy the GFP_KERNEL allocation. This is
2289 * the problem rescuer solves.
2290 *
Tejun Heo706026c2013-01-24 11:01:34 -08002291 * When such condition is possible, the pool summons rescuers of all
2292 * workqueues which have works queued on the pool and let them process
Tejun Heoe22bee72010-06-29 10:07:14 +02002293 * those works so that forward progress can be guaranteed.
2294 *
2295 * This should happen rarely.
2296 */
Tejun Heo111c2252013-01-17 17:16:24 -08002297static int rescuer_thread(void *__rescuer)
Tejun Heoe22bee72010-06-29 10:07:14 +02002298{
Tejun Heo111c2252013-01-17 17:16:24 -08002299 struct worker *rescuer = __rescuer;
2300 struct workqueue_struct *wq = rescuer->rescue_wq;
Tejun Heoe22bee72010-06-29 10:07:14 +02002301 struct list_head *scheduled = &rescuer->scheduled;
Tejun Heoe22bee72010-06-29 10:07:14 +02002302
2303 set_user_nice(current, RESCUER_NICE_LEVEL);
Tejun Heo111c2252013-01-17 17:16:24 -08002304
2305 /*
2306 * Mark rescuer as worker too. As WORKER_PREP is never cleared, it
2307 * doesn't participate in concurrency management.
2308 */
2309 rescuer->task->flags |= PF_WQ_WORKER;
Tejun Heoe22bee72010-06-29 10:07:14 +02002310repeat:
2311 set_current_state(TASK_INTERRUPTIBLE);
2312
Mike Galbraith412d32e2012-11-28 07:17:18 +01002313 if (kthread_should_stop()) {
2314 __set_current_state(TASK_RUNNING);
Tejun Heo111c2252013-01-17 17:16:24 -08002315 rescuer->task->flags &= ~PF_WQ_WORKER;
Tejun Heoe22bee72010-06-29 10:07:14 +02002316 return 0;
Mike Galbraith412d32e2012-11-28 07:17:18 +01002317 }
Tejun Heoe22bee72010-06-29 10:07:14 +02002318
Tejun Heo493a1722013-03-12 11:29:59 -07002319 /* see whether any pwq is asking for help */
Tejun Heo2e109a22013-03-13 19:47:40 -07002320 spin_lock_irq(&wq_mayday_lock);
Tejun Heo493a1722013-03-12 11:29:59 -07002321
2322 while (!list_empty(&wq->maydays)) {
2323 struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
2324 struct pool_workqueue, mayday_node);
Tejun Heo112202d2013-02-13 19:29:12 -08002325 struct worker_pool *pool = pwq->pool;
Tejun Heoe22bee72010-06-29 10:07:14 +02002326 struct work_struct *work, *n;
2327
2328 __set_current_state(TASK_RUNNING);
Tejun Heo493a1722013-03-12 11:29:59 -07002329 list_del_init(&pwq->mayday_node);
2330
Tejun Heo2e109a22013-03-13 19:47:40 -07002331 spin_unlock_irq(&wq_mayday_lock);
Tejun Heoe22bee72010-06-29 10:07:14 +02002332
2333 /* migrate to the target cpu if possible */
Lai Jiangshanf36dc672013-02-19 12:17:02 -08002334 worker_maybe_bind_and_lock(pool);
Lai Jiangshanb3104102013-02-19 12:17:02 -08002335 rescuer->pool = pool;
Tejun Heoe22bee72010-06-29 10:07:14 +02002336
2337 /*
2338 * Slurp in all works issued via this workqueue and
2339 * process'em.
2340 */
Tejun Heo6183c002013-03-12 11:29:57 -07002341 WARN_ON_ONCE(!list_empty(&rescuer->scheduled));
Tejun Heobd7bdd42012-07-12 14:46:37 -07002342 list_for_each_entry_safe(work, n, &pool->worklist, entry)
Tejun Heo112202d2013-02-13 19:29:12 -08002343 if (get_work_pwq(work) == pwq)
Tejun Heoe22bee72010-06-29 10:07:14 +02002344 move_linked_works(work, scheduled, &n);
2345
2346 process_scheduled_works(rescuer);
Tejun Heo75769582011-02-14 14:04:46 +01002347
2348 /*
Tejun Heod565ed62013-01-24 11:01:33 -08002349 * Leave this pool. If keep_working() is %true, notify a
Tejun Heo75769582011-02-14 14:04:46 +01002350 * regular worker; otherwise, we end up with 0 concurrency
2351 * and stalling the execution.
2352 */
Tejun Heo63d95a92012-07-12 14:46:37 -07002353 if (keep_working(pool))
2354 wake_up_worker(pool);
Tejun Heo75769582011-02-14 14:04:46 +01002355
Lai Jiangshanb3104102013-02-19 12:17:02 -08002356 rescuer->pool = NULL;
Tejun Heo493a1722013-03-12 11:29:59 -07002357 spin_unlock(&pool->lock);
Tejun Heo2e109a22013-03-13 19:47:40 -07002358 spin_lock(&wq_mayday_lock);
Tejun Heoe22bee72010-06-29 10:07:14 +02002359 }
2360
Tejun Heo2e109a22013-03-13 19:47:40 -07002361 spin_unlock_irq(&wq_mayday_lock);
Tejun Heo493a1722013-03-12 11:29:59 -07002362
Tejun Heo111c2252013-01-17 17:16:24 -08002363 /* rescuers should never participate in concurrency management */
2364 WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
Tejun Heoe22bee72010-06-29 10:07:14 +02002365 schedule();
2366 goto repeat;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002367}
2368
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -07002369struct wq_barrier {
2370 struct work_struct work;
2371 struct completion done;
2372};
2373
2374static void wq_barrier_func(struct work_struct *work)
2375{
2376 struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
2377 complete(&barr->done);
2378}
2379
Tejun Heo4690c4a2010-06-29 10:07:10 +02002380/**
2381 * insert_wq_barrier - insert a barrier work
Tejun Heo112202d2013-02-13 19:29:12 -08002382 * @pwq: pwq to insert barrier into
Tejun Heo4690c4a2010-06-29 10:07:10 +02002383 * @barr: wq_barrier to insert
Tejun Heoaffee4b2010-06-29 10:07:12 +02002384 * @target: target work to attach @barr to
2385 * @worker: worker currently executing @target, NULL if @target is not executing
Tejun Heo4690c4a2010-06-29 10:07:10 +02002386 *
Tejun Heoaffee4b2010-06-29 10:07:12 +02002387 * @barr is linked to @target such that @barr is completed only after
2388 * @target finishes execution. Please note that the ordering
2389 * guarantee is observed only with respect to @target and on the local
2390 * cpu.
2391 *
2392 * Currently, a queued barrier can't be canceled. This is because
2393 * try_to_grab_pending() can't determine whether the work to be
2394 * grabbed is at the head of the queue and thus can't clear LINKED
2395 * flag of the previous work while there must be a valid next work
2396 * after a work with LINKED flag set.
2397 *
2398 * Note that when @worker is non-NULL, @target may be modified
Tejun Heo112202d2013-02-13 19:29:12 -08002399 * underneath us, so we can't reliably determine pwq from @target.
Tejun Heo4690c4a2010-06-29 10:07:10 +02002400 *
2401 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08002402 * spin_lock_irq(pool->lock).
Tejun Heo4690c4a2010-06-29 10:07:10 +02002403 */
Tejun Heo112202d2013-02-13 19:29:12 -08002404static void insert_wq_barrier(struct pool_workqueue *pwq,
Tejun Heoaffee4b2010-06-29 10:07:12 +02002405 struct wq_barrier *barr,
2406 struct work_struct *target, struct worker *worker)
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -07002407{
Tejun Heoaffee4b2010-06-29 10:07:12 +02002408 struct list_head *head;
2409 unsigned int linked = 0;
2410
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +09002411 /*
Tejun Heod565ed62013-01-24 11:01:33 -08002412 * debugobject calls are safe here even with pool->lock locked
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +09002413 * as we know for sure that this will not trigger any of the
2414 * checks and call back into the fixup functions where we
2415 * might deadlock.
2416 */
Andrew Mortonca1cab32010-10-26 14:22:34 -07002417 INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
Tejun Heo22df02b2010-06-29 10:07:10 +02002418 __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -07002419 init_completion(&barr->done);
Oleg Nesterov83c22522007-05-09 02:33:54 -07002420
Tejun Heoaffee4b2010-06-29 10:07:12 +02002421 /*
2422 * If @target is currently being executed, schedule the
2423 * barrier to the worker; otherwise, put it after @target.
2424 */
2425 if (worker)
2426 head = worker->scheduled.next;
2427 else {
2428 unsigned long *bits = work_data_bits(target);
2429
2430 head = target->entry.next;
2431 /* there can already be other linked works, inherit and set */
2432 linked = *bits & WORK_STRUCT_LINKED;
2433 __set_bit(WORK_STRUCT_LINKED_BIT, bits);
2434 }
2435
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +09002436 debug_work_activate(&barr->work);
Tejun Heo112202d2013-02-13 19:29:12 -08002437 insert_work(pwq, &barr->work, head,
Tejun Heoaffee4b2010-06-29 10:07:12 +02002438 work_color_to_flags(WORK_NO_COLOR) | linked);
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -07002439}
2440
Tejun Heo73f53c42010-06-29 10:07:11 +02002441/**
Tejun Heo112202d2013-02-13 19:29:12 -08002442 * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
Tejun Heo73f53c42010-06-29 10:07:11 +02002443 * @wq: workqueue being flushed
2444 * @flush_color: new flush color, < 0 for no-op
2445 * @work_color: new work color, < 0 for no-op
2446 *
Tejun Heo112202d2013-02-13 19:29:12 -08002447 * Prepare pwqs for workqueue flushing.
Tejun Heo73f53c42010-06-29 10:07:11 +02002448 *
Tejun Heo112202d2013-02-13 19:29:12 -08002449 * If @flush_color is non-negative, flush_color on all pwqs should be
2450 * -1. If no pwq has in-flight commands at the specified color, all
2451 * pwq->flush_color's stay at -1 and %false is returned. If any pwq
2452 * has in flight commands, its pwq->flush_color is set to
2453 * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
Tejun Heo73f53c42010-06-29 10:07:11 +02002454 * wakeup logic is armed and %true is returned.
2455 *
2456 * The caller should have initialized @wq->first_flusher prior to
2457 * calling this function with non-negative @flush_color. If
2458 * @flush_color is negative, no flush color update is done and %false
2459 * is returned.
2460 *
Tejun Heo112202d2013-02-13 19:29:12 -08002461 * If @work_color is non-negative, all pwqs should have the same
Tejun Heo73f53c42010-06-29 10:07:11 +02002462 * work_color which is previous to @work_color and all will be
2463 * advanced to @work_color.
2464 *
2465 * CONTEXT:
Lai Jiangshan3c25a552013-03-25 16:57:17 -07002466 * mutex_lock(wq->mutex).
Tejun Heo73f53c42010-06-29 10:07:11 +02002467 *
2468 * RETURNS:
2469 * %true if @flush_color >= 0 and there's something to flush. %false
2470 * otherwise.
2471 */
Tejun Heo112202d2013-02-13 19:29:12 -08002472static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
Tejun Heo73f53c42010-06-29 10:07:11 +02002473 int flush_color, int work_color)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002474{
Tejun Heo73f53c42010-06-29 10:07:11 +02002475 bool wait = false;
Tejun Heo49e3cf42013-03-12 11:29:58 -07002476 struct pool_workqueue *pwq;
Oleg Nesterov14441962007-05-23 13:57:57 -07002477
Tejun Heo73f53c42010-06-29 10:07:11 +02002478 if (flush_color >= 0) {
Tejun Heo6183c002013-03-12 11:29:57 -07002479 WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
Tejun Heo112202d2013-02-13 19:29:12 -08002480 atomic_set(&wq->nr_pwqs_to_flush, 1);
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +09002481 }
Oleg Nesterov14441962007-05-23 13:57:57 -07002482
Tejun Heo49e3cf42013-03-12 11:29:58 -07002483 for_each_pwq(pwq, wq) {
Tejun Heo112202d2013-02-13 19:29:12 -08002484 struct worker_pool *pool = pwq->pool;
Tejun Heo73f53c42010-06-29 10:07:11 +02002485
Lai Jiangshanb09f4fd2013-03-25 16:57:18 -07002486 spin_lock_irq(&pool->lock);
Tejun Heo73f53c42010-06-29 10:07:11 +02002487
2488 if (flush_color >= 0) {
Tejun Heo6183c002013-03-12 11:29:57 -07002489 WARN_ON_ONCE(pwq->flush_color != -1);
Tejun Heo73f53c42010-06-29 10:07:11 +02002490
Tejun Heo112202d2013-02-13 19:29:12 -08002491 if (pwq->nr_in_flight[flush_color]) {
2492 pwq->flush_color = flush_color;
2493 atomic_inc(&wq->nr_pwqs_to_flush);
Tejun Heo73f53c42010-06-29 10:07:11 +02002494 wait = true;
2495 }
2496 }
2497
2498 if (work_color >= 0) {
Tejun Heo6183c002013-03-12 11:29:57 -07002499 WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
Tejun Heo112202d2013-02-13 19:29:12 -08002500 pwq->work_color = work_color;
Tejun Heo73f53c42010-06-29 10:07:11 +02002501 }
2502
Lai Jiangshanb09f4fd2013-03-25 16:57:18 -07002503 spin_unlock_irq(&pool->lock);
Tejun Heo73f53c42010-06-29 10:07:11 +02002504 }
2505
Tejun Heo112202d2013-02-13 19:29:12 -08002506 if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
Tejun Heo73f53c42010-06-29 10:07:11 +02002507 complete(&wq->first_flusher->done);
2508
2509 return wait;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002510}
2511
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -07002512/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07002513 * flush_workqueue - ensure that any scheduled work has run to completion.
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -07002514 * @wq: workqueue to flush
Linus Torvalds1da177e2005-04-16 15:20:36 -07002515 *
Tejun Heoc5aa87b2013-03-13 16:51:36 -07002516 * This function sleeps until all work items which were queued on entry
2517 * have finished execution, but it is not livelocked by new incoming ones.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002518 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -08002519void flush_workqueue(struct workqueue_struct *wq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002520{
Tejun Heo73f53c42010-06-29 10:07:11 +02002521 struct wq_flusher this_flusher = {
2522 .list = LIST_HEAD_INIT(this_flusher.list),
2523 .flush_color = -1,
2524 .done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done),
2525 };
2526 int next_color;
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -07002527
Ingo Molnar3295f0e2008-08-11 10:30:30 +02002528 lock_map_acquire(&wq->lockdep_map);
2529 lock_map_release(&wq->lockdep_map);
Tejun Heo73f53c42010-06-29 10:07:11 +02002530
Lai Jiangshan3c25a552013-03-25 16:57:17 -07002531 mutex_lock(&wq->mutex);
Tejun Heo73f53c42010-06-29 10:07:11 +02002532
2533 /*
2534 * Start-to-wait phase
2535 */
2536 next_color = work_next_color(wq->work_color);
2537
2538 if (next_color != wq->flush_color) {
2539 /*
2540 * Color space is not full. The current work_color
2541 * becomes our flush_color and work_color is advanced
2542 * by one.
2543 */
Tejun Heo6183c002013-03-12 11:29:57 -07002544 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
Tejun Heo73f53c42010-06-29 10:07:11 +02002545 this_flusher.flush_color = wq->work_color;
2546 wq->work_color = next_color;
2547
2548 if (!wq->first_flusher) {
2549 /* no flush in progress, become the first flusher */
Tejun Heo6183c002013-03-12 11:29:57 -07002550 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
Tejun Heo73f53c42010-06-29 10:07:11 +02002551
2552 wq->first_flusher = &this_flusher;
2553
Tejun Heo112202d2013-02-13 19:29:12 -08002554 if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
Tejun Heo73f53c42010-06-29 10:07:11 +02002555 wq->work_color)) {
2556 /* nothing to flush, done */
2557 wq->flush_color = next_color;
2558 wq->first_flusher = NULL;
2559 goto out_unlock;
2560 }
2561 } else {
2562 /* wait in queue */
Tejun Heo6183c002013-03-12 11:29:57 -07002563 WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
Tejun Heo73f53c42010-06-29 10:07:11 +02002564 list_add_tail(&this_flusher.list, &wq->flusher_queue);
Tejun Heo112202d2013-02-13 19:29:12 -08002565 flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
Tejun Heo73f53c42010-06-29 10:07:11 +02002566 }
2567 } else {
2568 /*
2569 * Oops, color space is full, wait on overflow queue.
2570 * The next flush completion will assign us
2571 * flush_color and transfer to flusher_queue.
2572 */
2573 list_add_tail(&this_flusher.list, &wq->flusher_overflow);
2574 }
2575
Lai Jiangshan3c25a552013-03-25 16:57:17 -07002576 mutex_unlock(&wq->mutex);
Tejun Heo73f53c42010-06-29 10:07:11 +02002577
2578 wait_for_completion(&this_flusher.done);
2579
2580 /*
2581 * Wake-up-and-cascade phase
2582 *
2583 * First flushers are responsible for cascading flushes and
2584 * handling overflow. Non-first flushers can simply return.
2585 */
2586 if (wq->first_flusher != &this_flusher)
2587 return;
2588
Lai Jiangshan3c25a552013-03-25 16:57:17 -07002589 mutex_lock(&wq->mutex);
Tejun Heo73f53c42010-06-29 10:07:11 +02002590
Tejun Heo4ce48b32010-07-02 10:03:51 +02002591 /* we might have raced, check again with mutex held */
2592 if (wq->first_flusher != &this_flusher)
2593 goto out_unlock;
2594
Tejun Heo73f53c42010-06-29 10:07:11 +02002595 wq->first_flusher = NULL;
2596
Tejun Heo6183c002013-03-12 11:29:57 -07002597 WARN_ON_ONCE(!list_empty(&this_flusher.list));
2598 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
Tejun Heo73f53c42010-06-29 10:07:11 +02002599
2600 while (true) {
2601 struct wq_flusher *next, *tmp;
2602
2603 /* complete all the flushers sharing the current flush color */
2604 list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
2605 if (next->flush_color != wq->flush_color)
2606 break;
2607 list_del_init(&next->list);
2608 complete(&next->done);
2609 }
2610
Tejun Heo6183c002013-03-12 11:29:57 -07002611 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
2612 wq->flush_color != work_next_color(wq->work_color));
Tejun Heo73f53c42010-06-29 10:07:11 +02002613
2614 /* this flush_color is finished, advance by one */
2615 wq->flush_color = work_next_color(wq->flush_color);
2616
2617 /* one color has been freed, handle overflow queue */
2618 if (!list_empty(&wq->flusher_overflow)) {
2619 /*
2620 * Assign the same color to all overflowed
2621 * flushers, advance work_color and append to
2622 * flusher_queue. This is the start-to-wait
2623 * phase for these overflowed flushers.
2624 */
2625 list_for_each_entry(tmp, &wq->flusher_overflow, list)
2626 tmp->flush_color = wq->work_color;
2627
2628 wq->work_color = work_next_color(wq->work_color);
2629
2630 list_splice_tail_init(&wq->flusher_overflow,
2631 &wq->flusher_queue);
Tejun Heo112202d2013-02-13 19:29:12 -08002632 flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
Tejun Heo73f53c42010-06-29 10:07:11 +02002633 }
2634
2635 if (list_empty(&wq->flusher_queue)) {
Tejun Heo6183c002013-03-12 11:29:57 -07002636 WARN_ON_ONCE(wq->flush_color != wq->work_color);
Tejun Heo73f53c42010-06-29 10:07:11 +02002637 break;
2638 }
2639
2640 /*
2641 * Need to flush more colors. Make the next flusher
Tejun Heo112202d2013-02-13 19:29:12 -08002642 * the new first flusher and arm pwqs.
Tejun Heo73f53c42010-06-29 10:07:11 +02002643 */
Tejun Heo6183c002013-03-12 11:29:57 -07002644 WARN_ON_ONCE(wq->flush_color == wq->work_color);
2645 WARN_ON_ONCE(wq->flush_color != next->flush_color);
Tejun Heo73f53c42010-06-29 10:07:11 +02002646
2647 list_del_init(&next->list);
2648 wq->first_flusher = next;
2649
Tejun Heo112202d2013-02-13 19:29:12 -08002650 if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
Tejun Heo73f53c42010-06-29 10:07:11 +02002651 break;
2652
2653 /*
2654 * Meh... this color is already done, clear first
2655 * flusher and repeat cascading.
2656 */
2657 wq->first_flusher = NULL;
2658 }
2659
2660out_unlock:
Lai Jiangshan3c25a552013-03-25 16:57:17 -07002661 mutex_unlock(&wq->mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002662}
Dave Jonesae90dd52006-06-30 01:40:45 -04002663EXPORT_SYMBOL_GPL(flush_workqueue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002664
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002665/**
2666 * drain_workqueue - drain a workqueue
2667 * @wq: workqueue to drain
2668 *
2669 * Wait until the workqueue becomes empty. While draining is in progress,
2670 * only chain queueing is allowed. IOW, only currently pending or running
2671 * work items on @wq can queue further work items on it. @wq is flushed
2672 * repeatedly until it becomes empty. The number of flushing is detemined
2673 * by the depth of chaining and should be relatively short. Whine if it
2674 * takes too long.
2675 */
2676void drain_workqueue(struct workqueue_struct *wq)
2677{
2678 unsigned int flush_cnt = 0;
Tejun Heo49e3cf42013-03-12 11:29:58 -07002679 struct pool_workqueue *pwq;
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002680
2681 /*
2682 * __queue_work() needs to test whether there are drainers, is much
2683 * hotter than drain_workqueue() and already looks at @wq->flags.
Tejun Heo618b01e2013-03-12 11:30:04 -07002684 * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002685 */
Lai Jiangshan87fc7412013-03-25 16:57:18 -07002686 mutex_lock(&wq->mutex);
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002687 if (!wq->nr_drainers++)
Tejun Heo618b01e2013-03-12 11:30:04 -07002688 wq->flags |= __WQ_DRAINING;
Lai Jiangshan87fc7412013-03-25 16:57:18 -07002689 mutex_unlock(&wq->mutex);
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002690reflush:
2691 flush_workqueue(wq);
2692
Lai Jiangshanb09f4fd2013-03-25 16:57:18 -07002693 mutex_lock(&wq->mutex);
Tejun Heo76af4d92013-03-12 11:30:00 -07002694
Tejun Heo49e3cf42013-03-12 11:29:58 -07002695 for_each_pwq(pwq, wq) {
Thomas Tuttlefa2563e2011-09-14 16:22:28 -07002696 bool drained;
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002697
Lai Jiangshanb09f4fd2013-03-25 16:57:18 -07002698 spin_lock_irq(&pwq->pool->lock);
Tejun Heo112202d2013-02-13 19:29:12 -08002699 drained = !pwq->nr_active && list_empty(&pwq->delayed_works);
Lai Jiangshanb09f4fd2013-03-25 16:57:18 -07002700 spin_unlock_irq(&pwq->pool->lock);
Thomas Tuttlefa2563e2011-09-14 16:22:28 -07002701
2702 if (drained)
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002703 continue;
2704
2705 if (++flush_cnt == 10 ||
2706 (flush_cnt % 100 == 0 && flush_cnt <= 1000))
Tejun Heoc5aa87b2013-03-13 16:51:36 -07002707 pr_warn("workqueue %s: drain_workqueue() isn't complete after %u tries\n",
Valentin Ilie044c7822012-08-19 00:52:42 +03002708 wq->name, flush_cnt);
Tejun Heo76af4d92013-03-12 11:30:00 -07002709
Lai Jiangshanb09f4fd2013-03-25 16:57:18 -07002710 mutex_unlock(&wq->mutex);
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002711 goto reflush;
2712 }
2713
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002714 if (!--wq->nr_drainers)
Tejun Heo618b01e2013-03-12 11:30:04 -07002715 wq->flags &= ~__WQ_DRAINING;
Lai Jiangshan87fc7412013-03-25 16:57:18 -07002716 mutex_unlock(&wq->mutex);
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002717}
2718EXPORT_SYMBOL_GPL(drain_workqueue);
2719
Tejun Heo606a5022012-08-20 14:51:23 -07002720static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr)
Tejun Heobaf59022010-09-16 10:42:16 +02002721{
2722 struct worker *worker = NULL;
Tejun Heoc9e7cf22013-01-24 11:01:33 -08002723 struct worker_pool *pool;
Tejun Heo112202d2013-02-13 19:29:12 -08002724 struct pool_workqueue *pwq;
Tejun Heobaf59022010-09-16 10:42:16 +02002725
2726 might_sleep();
Tejun Heobaf59022010-09-16 10:42:16 +02002727
Tejun Heofa1b54e2013-03-12 11:30:00 -07002728 local_irq_disable();
2729 pool = get_work_pool(work);
2730 if (!pool) {
2731 local_irq_enable();
2732 return false;
2733 }
2734
2735 spin_lock(&pool->lock);
Lai Jiangshan0b3dae62013-02-06 18:04:53 -08002736 /* see the comment in try_to_grab_pending() with the same code */
Tejun Heo112202d2013-02-13 19:29:12 -08002737 pwq = get_work_pwq(work);
2738 if (pwq) {
2739 if (unlikely(pwq->pool != pool))
Tejun Heobaf59022010-09-16 10:42:16 +02002740 goto already_gone;
Tejun Heo606a5022012-08-20 14:51:23 -07002741 } else {
Tejun Heoc9e7cf22013-01-24 11:01:33 -08002742 worker = find_worker_executing_work(pool, work);
Tejun Heobaf59022010-09-16 10:42:16 +02002743 if (!worker)
2744 goto already_gone;
Tejun Heo112202d2013-02-13 19:29:12 -08002745 pwq = worker->current_pwq;
Tejun Heo606a5022012-08-20 14:51:23 -07002746 }
Tejun Heobaf59022010-09-16 10:42:16 +02002747
Tejun Heo112202d2013-02-13 19:29:12 -08002748 insert_wq_barrier(pwq, barr, work, worker);
Tejun Heod565ed62013-01-24 11:01:33 -08002749 spin_unlock_irq(&pool->lock);
Tejun Heobaf59022010-09-16 10:42:16 +02002750
Tejun Heoe1594892011-01-09 23:32:15 +01002751 /*
2752 * If @max_active is 1 or rescuer is in use, flushing another work
2753 * item on the same workqueue may lead to deadlock. Make sure the
2754 * flusher is not running on the same workqueue by verifying write
2755 * access.
2756 */
Tejun Heo493008a2013-03-12 11:30:03 -07002757 if (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)
Tejun Heo112202d2013-02-13 19:29:12 -08002758 lock_map_acquire(&pwq->wq->lockdep_map);
Tejun Heoe1594892011-01-09 23:32:15 +01002759 else
Tejun Heo112202d2013-02-13 19:29:12 -08002760 lock_map_acquire_read(&pwq->wq->lockdep_map);
2761 lock_map_release(&pwq->wq->lockdep_map);
Tejun Heoe1594892011-01-09 23:32:15 +01002762
Tejun Heobaf59022010-09-16 10:42:16 +02002763 return true;
2764already_gone:
Tejun Heod565ed62013-01-24 11:01:33 -08002765 spin_unlock_irq(&pool->lock);
Tejun Heobaf59022010-09-16 10:42:16 +02002766 return false;
2767}
2768
Oleg Nesterovdb700892008-07-25 01:47:49 -07002769/**
Tejun Heo401a8d02010-09-16 10:36:00 +02002770 * flush_work - wait for a work to finish executing the last queueing instance
2771 * @work: the work to flush
Oleg Nesterovdb700892008-07-25 01:47:49 -07002772 *
Tejun Heo606a5022012-08-20 14:51:23 -07002773 * Wait until @work has finished execution. @work is guaranteed to be idle
2774 * on return if it hasn't been requeued since flush started.
Tejun Heo401a8d02010-09-16 10:36:00 +02002775 *
2776 * RETURNS:
2777 * %true if flush_work() waited for the work to finish execution,
2778 * %false if it was already idle.
Oleg Nesterovdb700892008-07-25 01:47:49 -07002779 */
Tejun Heo401a8d02010-09-16 10:36:00 +02002780bool flush_work(struct work_struct *work)
Oleg Nesterovdb700892008-07-25 01:47:49 -07002781{
Oleg Nesterovdb700892008-07-25 01:47:49 -07002782 struct wq_barrier barr;
2783
Stephen Boyd0976dfc2012-04-20 17:28:50 -07002784 lock_map_acquire(&work->lockdep_map);
2785 lock_map_release(&work->lockdep_map);
2786
Tejun Heo606a5022012-08-20 14:51:23 -07002787 if (start_flush_work(work, &barr)) {
Tejun Heobaf59022010-09-16 10:42:16 +02002788 wait_for_completion(&barr.done);
2789 destroy_work_on_stack(&barr.work);
2790 return true;
Tejun Heo606a5022012-08-20 14:51:23 -07002791 } else {
Tejun Heobaf59022010-09-16 10:42:16 +02002792 return false;
Tejun Heo606a5022012-08-20 14:51:23 -07002793 }
Oleg Nesterovdb700892008-07-25 01:47:49 -07002794}
2795EXPORT_SYMBOL_GPL(flush_work);
2796
Tejun Heo36e227d2012-08-03 10:30:46 -07002797static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
Tejun Heo401a8d02010-09-16 10:36:00 +02002798{
Tejun Heobbb68df2012-08-03 10:30:46 -07002799 unsigned long flags;
Oleg Nesterov1f1f6422007-07-15 23:41:44 -07002800 int ret;
2801
2802 do {
Tejun Heobbb68df2012-08-03 10:30:46 -07002803 ret = try_to_grab_pending(work, is_dwork, &flags);
2804 /*
2805 * If someone else is canceling, wait for the same event it
2806 * would be waiting for before retrying.
2807 */
2808 if (unlikely(ret == -ENOENT))
Tejun Heo606a5022012-08-20 14:51:23 -07002809 flush_work(work);
Oleg Nesterov1f1f6422007-07-15 23:41:44 -07002810 } while (unlikely(ret < 0));
2811
Tejun Heobbb68df2012-08-03 10:30:46 -07002812 /* tell other tasks trying to grab @work to back off */
2813 mark_work_canceling(work);
2814 local_irq_restore(flags);
2815
Tejun Heo606a5022012-08-20 14:51:23 -07002816 flush_work(work);
Tejun Heo7a22ad72010-06-29 10:07:13 +02002817 clear_work_data(work);
Oleg Nesterov1f1f6422007-07-15 23:41:44 -07002818 return ret;
2819}
2820
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002821/**
Tejun Heo401a8d02010-09-16 10:36:00 +02002822 * cancel_work_sync - cancel a work and wait for it to finish
2823 * @work: the work to cancel
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002824 *
Tejun Heo401a8d02010-09-16 10:36:00 +02002825 * Cancel @work and wait for its execution to finish. This function
2826 * can be used even if the work re-queues itself or migrates to
2827 * another workqueue. On return from this function, @work is
2828 * guaranteed to be not pending or executing on any CPU.
Oleg Nesterov1f1f6422007-07-15 23:41:44 -07002829 *
Tejun Heo401a8d02010-09-16 10:36:00 +02002830 * cancel_work_sync(&delayed_work->work) must not be used for
2831 * delayed_work's. Use cancel_delayed_work_sync() instead.
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002832 *
Tejun Heo401a8d02010-09-16 10:36:00 +02002833 * The caller must ensure that the workqueue on which @work was last
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002834 * queued can't be destroyed before this function returns.
Tejun Heo401a8d02010-09-16 10:36:00 +02002835 *
2836 * RETURNS:
2837 * %true if @work was pending, %false otherwise.
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002838 */
Tejun Heo401a8d02010-09-16 10:36:00 +02002839bool cancel_work_sync(struct work_struct *work)
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002840{
Tejun Heo36e227d2012-08-03 10:30:46 -07002841 return __cancel_work_timer(work, false);
Oleg Nesterovb89deed2007-05-09 02:33:52 -07002842}
Oleg Nesterov28e53bd2007-05-09 02:34:22 -07002843EXPORT_SYMBOL_GPL(cancel_work_sync);
Oleg Nesterovb89deed2007-05-09 02:33:52 -07002844
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002845/**
Tejun Heo401a8d02010-09-16 10:36:00 +02002846 * flush_delayed_work - wait for a dwork to finish executing the last queueing
2847 * @dwork: the delayed work to flush
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002848 *
Tejun Heo401a8d02010-09-16 10:36:00 +02002849 * Delayed timer is cancelled and the pending work is queued for
2850 * immediate execution. Like flush_work(), this function only
2851 * considers the last queueing instance of @dwork.
Oleg Nesterov1f1f6422007-07-15 23:41:44 -07002852 *
Tejun Heo401a8d02010-09-16 10:36:00 +02002853 * RETURNS:
2854 * %true if flush_work() waited for the work to finish execution,
2855 * %false if it was already idle.
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002856 */
Tejun Heo401a8d02010-09-16 10:36:00 +02002857bool flush_delayed_work(struct delayed_work *dwork)
2858{
Tejun Heo8930cab2012-08-03 10:30:45 -07002859 local_irq_disable();
Tejun Heo401a8d02010-09-16 10:36:00 +02002860 if (del_timer_sync(&dwork->timer))
Lai Jiangshan60c057b2013-02-06 18:04:53 -08002861 __queue_work(dwork->cpu, dwork->wq, &dwork->work);
Tejun Heo8930cab2012-08-03 10:30:45 -07002862 local_irq_enable();
Tejun Heo401a8d02010-09-16 10:36:00 +02002863 return flush_work(&dwork->work);
2864}
2865EXPORT_SYMBOL(flush_delayed_work);
2866
2867/**
Tejun Heo57b30ae2012-08-21 13:18:24 -07002868 * cancel_delayed_work - cancel a delayed work
2869 * @dwork: delayed_work to cancel
Tejun Heo09383492010-09-16 10:48:29 +02002870 *
Tejun Heo57b30ae2012-08-21 13:18:24 -07002871 * Kill off a pending delayed_work. Returns %true if @dwork was pending
2872 * and canceled; %false if wasn't pending. Note that the work callback
2873 * function may still be running on return, unless it returns %true and the
2874 * work doesn't re-arm itself. Explicitly flush or use
2875 * cancel_delayed_work_sync() to wait on it.
Tejun Heo09383492010-09-16 10:48:29 +02002876 *
Tejun Heo57b30ae2012-08-21 13:18:24 -07002877 * This function is safe to call from any context including IRQ handler.
Tejun Heo09383492010-09-16 10:48:29 +02002878 */
Tejun Heo57b30ae2012-08-21 13:18:24 -07002879bool cancel_delayed_work(struct delayed_work *dwork)
Tejun Heo09383492010-09-16 10:48:29 +02002880{
Tejun Heo57b30ae2012-08-21 13:18:24 -07002881 unsigned long flags;
2882 int ret;
2883
2884 do {
2885 ret = try_to_grab_pending(&dwork->work, true, &flags);
2886 } while (unlikely(ret == -EAGAIN));
2887
2888 if (unlikely(ret < 0))
2889 return false;
2890
Tejun Heo7c3eed52013-01-24 11:01:33 -08002891 set_work_pool_and_clear_pending(&dwork->work,
2892 get_work_pool_id(&dwork->work));
Tejun Heo57b30ae2012-08-21 13:18:24 -07002893 local_irq_restore(flags);
Dan Magenheimerc0158ca2012-10-18 16:31:37 -07002894 return ret;
Tejun Heo09383492010-09-16 10:48:29 +02002895}
Tejun Heo57b30ae2012-08-21 13:18:24 -07002896EXPORT_SYMBOL(cancel_delayed_work);
Tejun Heo09383492010-09-16 10:48:29 +02002897
2898/**
Tejun Heo401a8d02010-09-16 10:36:00 +02002899 * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
2900 * @dwork: the delayed work cancel
2901 *
2902 * This is cancel_work_sync() for delayed works.
2903 *
2904 * RETURNS:
2905 * %true if @dwork was pending, %false otherwise.
2906 */
2907bool cancel_delayed_work_sync(struct delayed_work *dwork)
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002908{
Tejun Heo36e227d2012-08-03 10:30:46 -07002909 return __cancel_work_timer(&dwork->work, true);
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002910}
Oleg Nesterovf5a421a2007-07-15 23:41:44 -07002911EXPORT_SYMBOL(cancel_delayed_work_sync);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002912
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -07002913/**
Tejun Heo31ddd872010-10-19 11:14:49 +02002914 * schedule_on_each_cpu - execute a function synchronously on each online CPU
Andrew Mortonb6136772006-06-25 05:47:49 -07002915 * @func: the function to call
Andrew Mortonb6136772006-06-25 05:47:49 -07002916 *
Tejun Heo31ddd872010-10-19 11:14:49 +02002917 * schedule_on_each_cpu() executes @func on each online CPU using the
2918 * system workqueue and blocks until all CPUs have completed.
Andrew Mortonb6136772006-06-25 05:47:49 -07002919 * schedule_on_each_cpu() is very slow.
Tejun Heo31ddd872010-10-19 11:14:49 +02002920 *
2921 * RETURNS:
2922 * 0 on success, -errno on failure.
Andrew Mortonb6136772006-06-25 05:47:49 -07002923 */
David Howells65f27f32006-11-22 14:55:48 +00002924int schedule_on_each_cpu(work_func_t func)
Christoph Lameter15316ba2006-01-08 01:00:43 -08002925{
2926 int cpu;
Namhyung Kim38f51562010-08-08 14:24:09 +02002927 struct work_struct __percpu *works;
Christoph Lameter15316ba2006-01-08 01:00:43 -08002928
Andrew Mortonb6136772006-06-25 05:47:49 -07002929 works = alloc_percpu(struct work_struct);
2930 if (!works)
Christoph Lameter15316ba2006-01-08 01:00:43 -08002931 return -ENOMEM;
Andrew Mortonb6136772006-06-25 05:47:49 -07002932
Gautham R Shenoy95402b32008-01-25 21:08:02 +01002933 get_online_cpus();
Tejun Heo93981802009-11-17 14:06:20 -08002934
Christoph Lameter15316ba2006-01-08 01:00:43 -08002935 for_each_online_cpu(cpu) {
Ingo Molnar9bfb1832006-12-18 20:05:09 +01002936 struct work_struct *work = per_cpu_ptr(works, cpu);
2937
2938 INIT_WORK(work, func);
Tejun Heob71ab8c2010-06-29 10:07:14 +02002939 schedule_work_on(cpu, work);
Andi Kleen65a64462009-10-14 06:22:47 +02002940 }
Tejun Heo93981802009-11-17 14:06:20 -08002941
2942 for_each_online_cpu(cpu)
2943 flush_work(per_cpu_ptr(works, cpu));
2944
Gautham R Shenoy95402b32008-01-25 21:08:02 +01002945 put_online_cpus();
Andrew Mortonb6136772006-06-25 05:47:49 -07002946 free_percpu(works);
Christoph Lameter15316ba2006-01-08 01:00:43 -08002947 return 0;
2948}
2949
Alan Sterneef6a7d2010-02-12 17:39:21 +09002950/**
2951 * flush_scheduled_work - ensure that any scheduled work has run to completion.
2952 *
2953 * Forces execution of the kernel-global workqueue and blocks until its
2954 * completion.
2955 *
2956 * Think twice before calling this function! It's very easy to get into
2957 * trouble if you don't take great care. Either of the following situations
2958 * will lead to deadlock:
2959 *
2960 * One of the work items currently on the workqueue needs to acquire
2961 * a lock held by your code or its caller.
2962 *
2963 * Your code is running in the context of a work routine.
2964 *
2965 * They will be detected by lockdep when they occur, but the first might not
2966 * occur very often. It depends on what work items are on the workqueue and
2967 * what locks they need, which you have no control over.
2968 *
2969 * In most situations flushing the entire workqueue is overkill; you merely
2970 * need to know that a particular work item isn't queued and isn't running.
2971 * In such cases you should use cancel_delayed_work_sync() or
2972 * cancel_work_sync() instead.
2973 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002974void flush_scheduled_work(void)
2975{
Tejun Heod320c032010-06-29 10:07:14 +02002976 flush_workqueue(system_wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002977}
Dave Jonesae90dd52006-06-30 01:40:45 -04002978EXPORT_SYMBOL(flush_scheduled_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002979
2980/**
James Bottomley1fa44ec2006-02-23 12:43:43 -06002981 * execute_in_process_context - reliably execute the routine with user context
2982 * @fn: the function to execute
James Bottomley1fa44ec2006-02-23 12:43:43 -06002983 * @ew: guaranteed storage for the execute work structure (must
2984 * be available when the work executes)
2985 *
2986 * Executes the function immediately if process context is available,
2987 * otherwise schedules the function for delayed execution.
2988 *
2989 * Returns: 0 - function was executed
2990 * 1 - function was scheduled for execution
2991 */
David Howells65f27f32006-11-22 14:55:48 +00002992int execute_in_process_context(work_func_t fn, struct execute_work *ew)
James Bottomley1fa44ec2006-02-23 12:43:43 -06002993{
2994 if (!in_interrupt()) {
David Howells65f27f32006-11-22 14:55:48 +00002995 fn(&ew->work);
James Bottomley1fa44ec2006-02-23 12:43:43 -06002996 return 0;
2997 }
2998
David Howells65f27f32006-11-22 14:55:48 +00002999 INIT_WORK(&ew->work, fn);
James Bottomley1fa44ec2006-02-23 12:43:43 -06003000 schedule_work(&ew->work);
3001
3002 return 1;
3003}
3004EXPORT_SYMBOL_GPL(execute_in_process_context);
3005
Tejun Heo226223a2013-03-12 11:30:05 -07003006#ifdef CONFIG_SYSFS
3007/*
3008 * Workqueues with WQ_SYSFS flag set is visible to userland via
3009 * /sys/bus/workqueue/devices/WQ_NAME. All visible workqueues have the
3010 * following attributes.
3011 *
3012 * per_cpu RO bool : whether the workqueue is per-cpu or unbound
3013 * max_active RW int : maximum number of in-flight work items
3014 *
3015 * Unbound workqueues have the following extra attributes.
3016 *
3017 * id RO int : the associated pool ID
3018 * nice RW int : nice value of the workers
3019 * cpumask RW mask : bitmask of allowed CPUs for the workers
3020 */
3021struct wq_device {
3022 struct workqueue_struct *wq;
3023 struct device dev;
3024};
3025
3026static struct workqueue_struct *dev_to_wq(struct device *dev)
3027{
3028 struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
3029
3030 return wq_dev->wq;
3031}
3032
3033static ssize_t wq_per_cpu_show(struct device *dev,
3034 struct device_attribute *attr, char *buf)
3035{
3036 struct workqueue_struct *wq = dev_to_wq(dev);
3037
3038 return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
3039}
3040
3041static ssize_t wq_max_active_show(struct device *dev,
3042 struct device_attribute *attr, char *buf)
3043{
3044 struct workqueue_struct *wq = dev_to_wq(dev);
3045
3046 return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
3047}
3048
3049static ssize_t wq_max_active_store(struct device *dev,
3050 struct device_attribute *attr,
3051 const char *buf, size_t count)
3052{
3053 struct workqueue_struct *wq = dev_to_wq(dev);
3054 int val;
3055
3056 if (sscanf(buf, "%d", &val) != 1 || val <= 0)
3057 return -EINVAL;
3058
3059 workqueue_set_max_active(wq, val);
3060 return count;
3061}
3062
3063static struct device_attribute wq_sysfs_attrs[] = {
3064 __ATTR(per_cpu, 0444, wq_per_cpu_show, NULL),
3065 __ATTR(max_active, 0644, wq_max_active_show, wq_max_active_store),
3066 __ATTR_NULL,
3067};
3068
3069static ssize_t wq_pool_id_show(struct device *dev,
3070 struct device_attribute *attr, char *buf)
3071{
3072 struct workqueue_struct *wq = dev_to_wq(dev);
3073 struct worker_pool *pool;
3074 int written;
3075
3076 rcu_read_lock_sched();
3077 pool = first_pwq(wq)->pool;
3078 written = scnprintf(buf, PAGE_SIZE, "%d\n", pool->id);
3079 rcu_read_unlock_sched();
3080
3081 return written;
3082}
3083
3084static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
3085 char *buf)
3086{
3087 struct workqueue_struct *wq = dev_to_wq(dev);
3088 int written;
3089
3090 rcu_read_lock_sched();
3091 written = scnprintf(buf, PAGE_SIZE, "%d\n",
3092 first_pwq(wq)->pool->attrs->nice);
3093 rcu_read_unlock_sched();
3094
3095 return written;
3096}
3097
3098/* prepare workqueue_attrs for sysfs store operations */
3099static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
3100{
3101 struct workqueue_attrs *attrs;
3102
3103 attrs = alloc_workqueue_attrs(GFP_KERNEL);
3104 if (!attrs)
3105 return NULL;
3106
3107 rcu_read_lock_sched();
3108 copy_workqueue_attrs(attrs, first_pwq(wq)->pool->attrs);
3109 rcu_read_unlock_sched();
3110 return attrs;
3111}
3112
3113static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
3114 const char *buf, size_t count)
3115{
3116 struct workqueue_struct *wq = dev_to_wq(dev);
3117 struct workqueue_attrs *attrs;
3118 int ret;
3119
3120 attrs = wq_sysfs_prep_attrs(wq);
3121 if (!attrs)
3122 return -ENOMEM;
3123
3124 if (sscanf(buf, "%d", &attrs->nice) == 1 &&
3125 attrs->nice >= -20 && attrs->nice <= 19)
3126 ret = apply_workqueue_attrs(wq, attrs);
3127 else
3128 ret = -EINVAL;
3129
3130 free_workqueue_attrs(attrs);
3131 return ret ?: count;
3132}
3133
3134static ssize_t wq_cpumask_show(struct device *dev,
3135 struct device_attribute *attr, char *buf)
3136{
3137 struct workqueue_struct *wq = dev_to_wq(dev);
3138 int written;
3139
3140 rcu_read_lock_sched();
3141 written = cpumask_scnprintf(buf, PAGE_SIZE,
3142 first_pwq(wq)->pool->attrs->cpumask);
3143 rcu_read_unlock_sched();
3144
3145 written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
3146 return written;
3147}
3148
3149static ssize_t wq_cpumask_store(struct device *dev,
3150 struct device_attribute *attr,
3151 const char *buf, size_t count)
3152{
3153 struct workqueue_struct *wq = dev_to_wq(dev);
3154 struct workqueue_attrs *attrs;
3155 int ret;
3156
3157 attrs = wq_sysfs_prep_attrs(wq);
3158 if (!attrs)
3159 return -ENOMEM;
3160
3161 ret = cpumask_parse(buf, attrs->cpumask);
3162 if (!ret)
3163 ret = apply_workqueue_attrs(wq, attrs);
3164
3165 free_workqueue_attrs(attrs);
3166 return ret ?: count;
3167}
3168
3169static struct device_attribute wq_sysfs_unbound_attrs[] = {
3170 __ATTR(pool_id, 0444, wq_pool_id_show, NULL),
3171 __ATTR(nice, 0644, wq_nice_show, wq_nice_store),
3172 __ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
3173 __ATTR_NULL,
3174};
3175
3176static struct bus_type wq_subsys = {
3177 .name = "workqueue",
3178 .dev_attrs = wq_sysfs_attrs,
3179};
3180
3181static int __init wq_sysfs_init(void)
3182{
3183 return subsys_virtual_register(&wq_subsys, NULL);
3184}
3185core_initcall(wq_sysfs_init);
3186
3187static void wq_device_release(struct device *dev)
3188{
3189 struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
3190
3191 kfree(wq_dev);
3192}
3193
3194/**
3195 * workqueue_sysfs_register - make a workqueue visible in sysfs
3196 * @wq: the workqueue to register
3197 *
3198 * Expose @wq in sysfs under /sys/bus/workqueue/devices.
3199 * alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
3200 * which is the preferred method.
3201 *
3202 * Workqueue user should use this function directly iff it wants to apply
3203 * workqueue_attrs before making the workqueue visible in sysfs; otherwise,
3204 * apply_workqueue_attrs() may race against userland updating the
3205 * attributes.
3206 *
3207 * Returns 0 on success, -errno on failure.
3208 */
3209int workqueue_sysfs_register(struct workqueue_struct *wq)
3210{
3211 struct wq_device *wq_dev;
3212 int ret;
3213
3214 /*
3215 * Adjusting max_active or creating new pwqs by applyting
3216 * attributes breaks ordering guarantee. Disallow exposing ordered
3217 * workqueues.
3218 */
3219 if (WARN_ON(wq->flags & __WQ_ORDERED))
3220 return -EINVAL;
3221
3222 wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
3223 if (!wq_dev)
3224 return -ENOMEM;
3225
3226 wq_dev->wq = wq;
3227 wq_dev->dev.bus = &wq_subsys;
3228 wq_dev->dev.init_name = wq->name;
3229 wq_dev->dev.release = wq_device_release;
3230
3231 /*
3232 * unbound_attrs are created separately. Suppress uevent until
3233 * everything is ready.
3234 */
3235 dev_set_uevent_suppress(&wq_dev->dev, true);
3236
3237 ret = device_register(&wq_dev->dev);
3238 if (ret) {
3239 kfree(wq_dev);
3240 wq->wq_dev = NULL;
3241 return ret;
3242 }
3243
3244 if (wq->flags & WQ_UNBOUND) {
3245 struct device_attribute *attr;
3246
3247 for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
3248 ret = device_create_file(&wq_dev->dev, attr);
3249 if (ret) {
3250 device_unregister(&wq_dev->dev);
3251 wq->wq_dev = NULL;
3252 return ret;
3253 }
3254 }
3255 }
3256
3257 kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
3258 return 0;
3259}
3260
3261/**
3262 * workqueue_sysfs_unregister - undo workqueue_sysfs_register()
3263 * @wq: the workqueue to unregister
3264 *
3265 * If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
3266 */
3267static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
3268{
3269 struct wq_device *wq_dev = wq->wq_dev;
3270
3271 if (!wq->wq_dev)
3272 return;
3273
3274 wq->wq_dev = NULL;
3275 device_unregister(&wq_dev->dev);
3276}
3277#else /* CONFIG_SYSFS */
3278static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { }
3279#endif /* CONFIG_SYSFS */
3280
Tejun Heo7a4e3442013-03-12 11:30:00 -07003281/**
3282 * free_workqueue_attrs - free a workqueue_attrs
3283 * @attrs: workqueue_attrs to free
3284 *
3285 * Undo alloc_workqueue_attrs().
3286 */
3287void free_workqueue_attrs(struct workqueue_attrs *attrs)
3288{
3289 if (attrs) {
3290 free_cpumask_var(attrs->cpumask);
3291 kfree(attrs);
3292 }
3293}
3294
3295/**
3296 * alloc_workqueue_attrs - allocate a workqueue_attrs
3297 * @gfp_mask: allocation mask to use
3298 *
3299 * Allocate a new workqueue_attrs, initialize with default settings and
3300 * return it. Returns NULL on failure.
3301 */
3302struct workqueue_attrs *alloc_workqueue_attrs(gfp_t gfp_mask)
3303{
3304 struct workqueue_attrs *attrs;
3305
3306 attrs = kzalloc(sizeof(*attrs), gfp_mask);
3307 if (!attrs)
3308 goto fail;
3309 if (!alloc_cpumask_var(&attrs->cpumask, gfp_mask))
3310 goto fail;
3311
Tejun Heo13e2e552013-04-01 11:23:31 -07003312 cpumask_copy(attrs->cpumask, cpu_possible_mask);
Tejun Heo7a4e3442013-03-12 11:30:00 -07003313 return attrs;
3314fail:
3315 free_workqueue_attrs(attrs);
3316 return NULL;
3317}
3318
Tejun Heo29c91e92013-03-12 11:30:03 -07003319static void copy_workqueue_attrs(struct workqueue_attrs *to,
3320 const struct workqueue_attrs *from)
3321{
3322 to->nice = from->nice;
3323 cpumask_copy(to->cpumask, from->cpumask);
3324}
3325
Tejun Heo29c91e92013-03-12 11:30:03 -07003326/* hash value of the content of @attr */
3327static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
3328{
3329 u32 hash = 0;
3330
3331 hash = jhash_1word(attrs->nice, hash);
Tejun Heo13e2e552013-04-01 11:23:31 -07003332 hash = jhash(cpumask_bits(attrs->cpumask),
3333 BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
Tejun Heo29c91e92013-03-12 11:30:03 -07003334 return hash;
3335}
3336
3337/* content equality test */
3338static bool wqattrs_equal(const struct workqueue_attrs *a,
3339 const struct workqueue_attrs *b)
3340{
3341 if (a->nice != b->nice)
3342 return false;
3343 if (!cpumask_equal(a->cpumask, b->cpumask))
3344 return false;
3345 return true;
3346}
3347
Tejun Heo7a4e3442013-03-12 11:30:00 -07003348/**
3349 * init_worker_pool - initialize a newly zalloc'd worker_pool
3350 * @pool: worker_pool to initialize
3351 *
3352 * Initiailize a newly zalloc'd @pool. It also allocates @pool->attrs.
Tejun Heo29c91e92013-03-12 11:30:03 -07003353 * Returns 0 on success, -errno on failure. Even on failure, all fields
3354 * inside @pool proper are initialized and put_unbound_pool() can be called
3355 * on @pool safely to release it.
Tejun Heo7a4e3442013-03-12 11:30:00 -07003356 */
3357static int init_worker_pool(struct worker_pool *pool)
Tejun Heo4e1a1f92013-03-12 11:30:00 -07003358{
3359 spin_lock_init(&pool->lock);
Tejun Heo29c91e92013-03-12 11:30:03 -07003360 pool->id = -1;
3361 pool->cpu = -1;
Tejun Heo4e1a1f92013-03-12 11:30:00 -07003362 pool->flags |= POOL_DISASSOCIATED;
3363 INIT_LIST_HEAD(&pool->worklist);
3364 INIT_LIST_HEAD(&pool->idle_list);
3365 hash_init(pool->busy_hash);
3366
3367 init_timer_deferrable(&pool->idle_timer);
3368 pool->idle_timer.function = idle_worker_timeout;
3369 pool->idle_timer.data = (unsigned long)pool;
3370
3371 setup_timer(&pool->mayday_timer, pool_mayday_timeout,
3372 (unsigned long)pool);
3373
3374 mutex_init(&pool->manager_arb);
Tejun Heobc3a1af2013-03-13 19:47:39 -07003375 mutex_init(&pool->manager_mutex);
Tejun Heo822d8402013-03-19 13:45:21 -07003376 idr_init(&pool->worker_idr);
Tejun Heo7a4e3442013-03-12 11:30:00 -07003377
Tejun Heo29c91e92013-03-12 11:30:03 -07003378 INIT_HLIST_NODE(&pool->hash_node);
3379 pool->refcnt = 1;
3380
3381 /* shouldn't fail above this point */
Tejun Heo7a4e3442013-03-12 11:30:00 -07003382 pool->attrs = alloc_workqueue_attrs(GFP_KERNEL);
3383 if (!pool->attrs)
3384 return -ENOMEM;
3385 return 0;
Tejun Heo4e1a1f92013-03-12 11:30:00 -07003386}
3387
Tejun Heo29c91e92013-03-12 11:30:03 -07003388static void rcu_free_pool(struct rcu_head *rcu)
3389{
3390 struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
3391
Tejun Heo822d8402013-03-19 13:45:21 -07003392 idr_destroy(&pool->worker_idr);
Tejun Heo29c91e92013-03-12 11:30:03 -07003393 free_workqueue_attrs(pool->attrs);
3394 kfree(pool);
3395}
3396
3397/**
3398 * put_unbound_pool - put a worker_pool
3399 * @pool: worker_pool to put
3400 *
3401 * Put @pool. If its refcnt reaches zero, it gets destroyed in sched-RCU
Tejun Heoc5aa87b2013-03-13 16:51:36 -07003402 * safe manner. get_unbound_pool() calls this function on its failure path
3403 * and this function should be able to release pools which went through,
3404 * successfully or not, init_worker_pool().
Tejun Heoa892cac2013-04-01 11:23:32 -07003405 *
3406 * Should be called with wq_pool_mutex held.
Tejun Heo29c91e92013-03-12 11:30:03 -07003407 */
3408static void put_unbound_pool(struct worker_pool *pool)
3409{
3410 struct worker *worker;
3411
Tejun Heoa892cac2013-04-01 11:23:32 -07003412 lockdep_assert_held(&wq_pool_mutex);
3413
3414 if (--pool->refcnt)
Tejun Heo29c91e92013-03-12 11:30:03 -07003415 return;
Tejun Heo29c91e92013-03-12 11:30:03 -07003416
3417 /* sanity checks */
3418 if (WARN_ON(!(pool->flags & POOL_DISASSOCIATED)) ||
Tejun Heoa892cac2013-04-01 11:23:32 -07003419 WARN_ON(!list_empty(&pool->worklist)))
Tejun Heo29c91e92013-03-12 11:30:03 -07003420 return;
Tejun Heo29c91e92013-03-12 11:30:03 -07003421
3422 /* release id and unhash */
3423 if (pool->id >= 0)
3424 idr_remove(&worker_pool_idr, pool->id);
3425 hash_del(&pool->hash_node);
3426
Tejun Heoc5aa87b2013-03-13 16:51:36 -07003427 /*
3428 * Become the manager and destroy all workers. Grabbing
3429 * manager_arb prevents @pool's workers from blocking on
3430 * manager_mutex.
3431 */
Tejun Heo29c91e92013-03-12 11:30:03 -07003432 mutex_lock(&pool->manager_arb);
Tejun Heocd549682013-03-13 19:47:39 -07003433 mutex_lock(&pool->manager_mutex);
Tejun Heo29c91e92013-03-12 11:30:03 -07003434 spin_lock_irq(&pool->lock);
3435
3436 while ((worker = first_worker(pool)))
3437 destroy_worker(worker);
3438 WARN_ON(pool->nr_workers || pool->nr_idle);
3439
3440 spin_unlock_irq(&pool->lock);
Tejun Heocd549682013-03-13 19:47:39 -07003441 mutex_unlock(&pool->manager_mutex);
Tejun Heo29c91e92013-03-12 11:30:03 -07003442 mutex_unlock(&pool->manager_arb);
3443
3444 /* shut down the timers */
3445 del_timer_sync(&pool->idle_timer);
3446 del_timer_sync(&pool->mayday_timer);
3447
3448 /* sched-RCU protected to allow dereferences from get_work_pool() */
3449 call_rcu_sched(&pool->rcu, rcu_free_pool);
3450}
3451
3452/**
3453 * get_unbound_pool - get a worker_pool with the specified attributes
3454 * @attrs: the attributes of the worker_pool to get
3455 *
3456 * Obtain a worker_pool which has the same attributes as @attrs, bump the
3457 * reference count and return it. If there already is a matching
3458 * worker_pool, it will be used; otherwise, this function attempts to
3459 * create a new one. On failure, returns NULL.
Tejun Heoa892cac2013-04-01 11:23:32 -07003460 *
3461 * Should be called with wq_pool_mutex held.
Tejun Heo29c91e92013-03-12 11:30:03 -07003462 */
3463static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
3464{
Tejun Heo29c91e92013-03-12 11:30:03 -07003465 u32 hash = wqattrs_hash(attrs);
3466 struct worker_pool *pool;
Tejun Heo29c91e92013-03-12 11:30:03 -07003467
Tejun Heoa892cac2013-04-01 11:23:32 -07003468 lockdep_assert_held(&wq_pool_mutex);
Tejun Heo29c91e92013-03-12 11:30:03 -07003469
3470 /* do we already have a matching pool? */
Tejun Heo29c91e92013-03-12 11:30:03 -07003471 hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
3472 if (wqattrs_equal(pool->attrs, attrs)) {
3473 pool->refcnt++;
3474 goto out_unlock;
3475 }
3476 }
Tejun Heo29c91e92013-03-12 11:30:03 -07003477
3478 /* nope, create a new one */
3479 pool = kzalloc(sizeof(*pool), GFP_KERNEL);
3480 if (!pool || init_worker_pool(pool) < 0)
3481 goto fail;
3482
Lai Jiangshan12ee4fc2013-03-20 03:28:01 +08003483 if (workqueue_freezing)
3484 pool->flags |= POOL_FREEZING;
3485
Tejun Heo8864b4e2013-03-12 11:30:04 -07003486 lockdep_set_subclass(&pool->lock, 1); /* see put_pwq() */
Tejun Heo29c91e92013-03-12 11:30:03 -07003487 copy_workqueue_attrs(pool->attrs, attrs);
3488
3489 if (worker_pool_assign_id(pool) < 0)
3490 goto fail;
3491
3492 /* create and start the initial worker */
Tejun Heoebf44d12013-03-13 19:47:39 -07003493 if (create_and_start_worker(pool) < 0)
Tejun Heo29c91e92013-03-12 11:30:03 -07003494 goto fail;
3495
Tejun Heo29c91e92013-03-12 11:30:03 -07003496 /* install */
Tejun Heo29c91e92013-03-12 11:30:03 -07003497 hash_add(unbound_pool_hash, &pool->hash_node, hash);
3498out_unlock:
Tejun Heo29c91e92013-03-12 11:30:03 -07003499 return pool;
3500fail:
Tejun Heo29c91e92013-03-12 11:30:03 -07003501 if (pool)
3502 put_unbound_pool(pool);
3503 return NULL;
3504}
3505
Tejun Heo8864b4e2013-03-12 11:30:04 -07003506static void rcu_free_pwq(struct rcu_head *rcu)
3507{
3508 kmem_cache_free(pwq_cache,
3509 container_of(rcu, struct pool_workqueue, rcu));
3510}
3511
3512/*
3513 * Scheduled on system_wq by put_pwq() when an unbound pwq hits zero refcnt
3514 * and needs to be destroyed.
3515 */
3516static void pwq_unbound_release_workfn(struct work_struct *work)
3517{
3518 struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
3519 unbound_release_work);
3520 struct workqueue_struct *wq = pwq->wq;
3521 struct worker_pool *pool = pwq->pool;
Tejun Heobc0caf02013-04-01 11:23:31 -07003522 bool is_last;
Tejun Heo8864b4e2013-03-12 11:30:04 -07003523
3524 if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
3525 return;
3526
Tejun Heo75ccf592013-03-12 11:30:04 -07003527 /*
Lai Jiangshan3c25a552013-03-25 16:57:17 -07003528 * Unlink @pwq. Synchronization against wq->mutex isn't strictly
Tejun Heo75ccf592013-03-12 11:30:04 -07003529 * necessary on release but do it anyway. It's easier to verify
3530 * and consistent with the linking path.
3531 */
Lai Jiangshan3c25a552013-03-25 16:57:17 -07003532 mutex_lock(&wq->mutex);
Tejun Heo8864b4e2013-03-12 11:30:04 -07003533 list_del_rcu(&pwq->pwqs_node);
Tejun Heobc0caf02013-04-01 11:23:31 -07003534 is_last = list_empty(&wq->pwqs);
Lai Jiangshan3c25a552013-03-25 16:57:17 -07003535 mutex_unlock(&wq->mutex);
Tejun Heo8864b4e2013-03-12 11:30:04 -07003536
Tejun Heoa892cac2013-04-01 11:23:32 -07003537 mutex_lock(&wq_pool_mutex);
Tejun Heo8864b4e2013-03-12 11:30:04 -07003538 put_unbound_pool(pool);
Tejun Heoa892cac2013-04-01 11:23:32 -07003539 mutex_unlock(&wq_pool_mutex);
3540
Tejun Heo8864b4e2013-03-12 11:30:04 -07003541 call_rcu_sched(&pwq->rcu, rcu_free_pwq);
3542
3543 /*
3544 * If we're the last pwq going away, @wq is already dead and no one
3545 * is gonna access it anymore. Free it.
3546 */
Tejun Heobc0caf02013-04-01 11:23:31 -07003547 if (is_last)
Tejun Heo8864b4e2013-03-12 11:30:04 -07003548 kfree(wq);
3549}
3550
Tejun Heo0fbd95a2013-03-13 16:51:35 -07003551/**
Tejun Heo699ce092013-03-13 16:51:35 -07003552 * pwq_adjust_max_active - update a pwq's max_active to the current setting
Tejun Heo0fbd95a2013-03-13 16:51:35 -07003553 * @pwq: target pool_workqueue
Tejun Heo0fbd95a2013-03-13 16:51:35 -07003554 *
Tejun Heo699ce092013-03-13 16:51:35 -07003555 * If @pwq isn't freezing, set @pwq->max_active to the associated
3556 * workqueue's saved_max_active and activate delayed work items
3557 * accordingly. If @pwq is freezing, clear @pwq->max_active to zero.
Tejun Heo0fbd95a2013-03-13 16:51:35 -07003558 */
Tejun Heo699ce092013-03-13 16:51:35 -07003559static void pwq_adjust_max_active(struct pool_workqueue *pwq)
Tejun Heo0fbd95a2013-03-13 16:51:35 -07003560{
Tejun Heo699ce092013-03-13 16:51:35 -07003561 struct workqueue_struct *wq = pwq->wq;
3562 bool freezable = wq->flags & WQ_FREEZABLE;
Tejun Heo0fbd95a2013-03-13 16:51:35 -07003563
Tejun Heo699ce092013-03-13 16:51:35 -07003564 /* for @wq->saved_max_active */
Lai Jiangshana357fc02013-03-25 16:57:19 -07003565 lockdep_assert_held(&wq->mutex);
Tejun Heo699ce092013-03-13 16:51:35 -07003566
3567 /* fast exit for non-freezable wqs */
3568 if (!freezable && pwq->max_active == wq->saved_max_active)
3569 return;
3570
Lai Jiangshana357fc02013-03-25 16:57:19 -07003571 spin_lock_irq(&pwq->pool->lock);
Tejun Heo699ce092013-03-13 16:51:35 -07003572
3573 if (!freezable || !(pwq->pool->flags & POOL_FREEZING)) {
3574 pwq->max_active = wq->saved_max_active;
3575
3576 while (!list_empty(&pwq->delayed_works) &&
3577 pwq->nr_active < pwq->max_active)
3578 pwq_activate_first_delayed(pwq);
Lai Jiangshan951a0782013-03-20 10:52:30 -07003579
3580 /*
3581 * Need to kick a worker after thawed or an unbound wq's
3582 * max_active is bumped. It's a slow path. Do it always.
3583 */
3584 wake_up_worker(pwq->pool);
Tejun Heo699ce092013-03-13 16:51:35 -07003585 } else {
3586 pwq->max_active = 0;
3587 }
3588
Lai Jiangshana357fc02013-03-25 16:57:19 -07003589 spin_unlock_irq(&pwq->pool->lock);
Tejun Heo0fbd95a2013-03-13 16:51:35 -07003590}
3591
Tejun Heod2c1d402013-03-12 11:30:04 -07003592static void init_and_link_pwq(struct pool_workqueue *pwq,
3593 struct workqueue_struct *wq,
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003594 struct worker_pool *pool,
3595 struct pool_workqueue **p_last_pwq)
Tejun Heod2c1d402013-03-12 11:30:04 -07003596{
3597 BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);
3598
3599 pwq->pool = pool;
3600 pwq->wq = wq;
3601 pwq->flush_color = -1;
Tejun Heo8864b4e2013-03-12 11:30:04 -07003602 pwq->refcnt = 1;
Tejun Heod2c1d402013-03-12 11:30:04 -07003603 INIT_LIST_HEAD(&pwq->delayed_works);
3604 INIT_LIST_HEAD(&pwq->mayday_node);
Tejun Heo8864b4e2013-03-12 11:30:04 -07003605 INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);
Tejun Heod2c1d402013-03-12 11:30:04 -07003606
Lai Jiangshan3c25a552013-03-25 16:57:17 -07003607 mutex_lock(&wq->mutex);
Tejun Heo75ccf592013-03-12 11:30:04 -07003608
Tejun Heo983ca252013-03-13 16:51:35 -07003609 /*
3610 * Set the matching work_color. This is synchronized with
Lai Jiangshan3c25a552013-03-25 16:57:17 -07003611 * wq->mutex to avoid confusing flush_workqueue().
Tejun Heo983ca252013-03-13 16:51:35 -07003612 */
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003613 if (p_last_pwq)
3614 *p_last_pwq = first_pwq(wq);
Tejun Heo75ccf592013-03-12 11:30:04 -07003615 pwq->work_color = wq->work_color;
Tejun Heo983ca252013-03-13 16:51:35 -07003616
3617 /* sync max_active to the current setting */
3618 pwq_adjust_max_active(pwq);
3619
3620 /* link in @pwq */
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003621 list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
Lai Jiangshana357fc02013-03-25 16:57:19 -07003622
Lai Jiangshan3c25a552013-03-25 16:57:17 -07003623 mutex_unlock(&wq->mutex);
Tejun Heod2c1d402013-03-12 11:30:04 -07003624}
3625
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003626/**
3627 * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
3628 * @wq: the target workqueue
3629 * @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
3630 *
3631 * Apply @attrs to an unbound workqueue @wq. If @attrs doesn't match the
3632 * current attributes, a new pwq is created and made the first pwq which
3633 * will serve all new work items. Older pwqs are released as in-flight
3634 * work items finish. Note that a work item which repeatedly requeues
3635 * itself back-to-back will stay on its current pwq.
3636 *
3637 * Performs GFP_KERNEL allocations. Returns 0 on success and -errno on
3638 * failure.
3639 */
3640int apply_workqueue_attrs(struct workqueue_struct *wq,
3641 const struct workqueue_attrs *attrs)
3642{
Tejun Heo13e2e552013-04-01 11:23:31 -07003643 struct workqueue_attrs *new_attrs;
3644 struct pool_workqueue *pwq = NULL, *last_pwq;
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003645 struct worker_pool *pool;
Tejun Heo48621252013-04-01 11:23:31 -07003646 int ret;
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003647
Tejun Heo8719dce2013-03-12 11:30:04 -07003648 /* only unbound workqueues can change attributes */
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003649 if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
3650 return -EINVAL;
3651
Tejun Heo8719dce2013-03-12 11:30:04 -07003652 /* creating multiple pwqs breaks ordering guarantee */
3653 if (WARN_ON((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs)))
3654 return -EINVAL;
3655
Tejun Heo13e2e552013-04-01 11:23:31 -07003656 /* make a copy of @attrs and sanitize it */
3657 new_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3658 if (!new_attrs)
3659 goto enomem;
3660
3661 copy_workqueue_attrs(new_attrs, attrs);
3662 cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
3663
Tejun Heoa892cac2013-04-01 11:23:32 -07003664 mutex_lock(&wq_pool_mutex);
3665
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003666 pwq = kmem_cache_zalloc(pwq_cache, GFP_KERNEL);
Tejun Heoa892cac2013-04-01 11:23:32 -07003667 if (!pwq) {
3668 mutex_unlock(&wq_pool_mutex);
Tejun Heo13e2e552013-04-01 11:23:31 -07003669 goto enomem;
Tejun Heoa892cac2013-04-01 11:23:32 -07003670 }
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003671
Tejun Heo13e2e552013-04-01 11:23:31 -07003672 pool = get_unbound_pool(new_attrs);
Tejun Heoa892cac2013-04-01 11:23:32 -07003673 if (!pool) {
3674 mutex_unlock(&wq_pool_mutex);
Tejun Heo13e2e552013-04-01 11:23:31 -07003675 goto enomem;
Tejun Heoa892cac2013-04-01 11:23:32 -07003676 }
3677
3678 mutex_unlock(&wq_pool_mutex);
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003679
3680 init_and_link_pwq(pwq, wq, pool, &last_pwq);
3681 if (last_pwq) {
3682 spin_lock_irq(&last_pwq->pool->lock);
3683 put_pwq(last_pwq);
3684 spin_unlock_irq(&last_pwq->pool->lock);
3685 }
3686
Tejun Heo48621252013-04-01 11:23:31 -07003687 ret = 0;
3688 /* fall through */
3689out_free:
3690 free_workqueue_attrs(new_attrs);
3691 return ret;
Tejun Heo13e2e552013-04-01 11:23:31 -07003692
3693enomem:
3694 kmem_cache_free(pwq_cache, pwq);
Tejun Heo48621252013-04-01 11:23:31 -07003695 ret = -ENOMEM;
3696 goto out_free;
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003697}
3698
Tejun Heo30cdf242013-03-12 11:29:57 -07003699static int alloc_and_link_pwqs(struct workqueue_struct *wq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003700{
Tejun Heo49e3cf42013-03-12 11:29:58 -07003701 bool highpri = wq->flags & WQ_HIGHPRI;
Tejun Heo30cdf242013-03-12 11:29:57 -07003702 int cpu;
Frederic Weisbeckere1d8aa92009-01-12 23:15:46 +01003703
Tejun Heo30cdf242013-03-12 11:29:57 -07003704 if (!(wq->flags & WQ_UNBOUND)) {
Tejun Heo420c0dd2013-03-12 11:29:59 -07003705 wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
3706 if (!wq->cpu_pwqs)
Tejun Heo30cdf242013-03-12 11:29:57 -07003707 return -ENOMEM;
3708
3709 for_each_possible_cpu(cpu) {
Tejun Heo7fb98ea2013-03-12 11:30:00 -07003710 struct pool_workqueue *pwq =
3711 per_cpu_ptr(wq->cpu_pwqs, cpu);
Tejun Heo7a62c2c2013-03-12 11:30:03 -07003712 struct worker_pool *cpu_pools =
Tejun Heof02ae732013-03-12 11:30:03 -07003713 per_cpu(cpu_worker_pools, cpu);
Tejun Heo30cdf242013-03-12 11:29:57 -07003714
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003715 init_and_link_pwq(pwq, wq, &cpu_pools[highpri], NULL);
Tejun Heo30cdf242013-03-12 11:29:57 -07003716 }
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003717 return 0;
Tejun Heo30cdf242013-03-12 11:29:57 -07003718 } else {
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003719 return apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
Tejun Heo30cdf242013-03-12 11:29:57 -07003720 }
Oleg Nesterov3af244332007-05-09 02:34:09 -07003721}
3722
Tejun Heof3421792010-07-02 10:03:51 +02003723static int wq_clamp_max_active(int max_active, unsigned int flags,
3724 const char *name)
Tejun Heob71ab8c2010-06-29 10:07:14 +02003725{
Tejun Heof3421792010-07-02 10:03:51 +02003726 int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
3727
3728 if (max_active < 1 || max_active > lim)
Valentin Ilie044c7822012-08-19 00:52:42 +03003729 pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
3730 max_active, name, 1, lim);
Tejun Heob71ab8c2010-06-29 10:07:14 +02003731
Tejun Heof3421792010-07-02 10:03:51 +02003732 return clamp_val(max_active, 1, lim);
Tejun Heob71ab8c2010-06-29 10:07:14 +02003733}
3734
Tejun Heob196be82012-01-10 15:11:35 -08003735struct workqueue_struct *__alloc_workqueue_key(const char *fmt,
Tejun Heod320c032010-06-29 10:07:14 +02003736 unsigned int flags,
3737 int max_active,
3738 struct lock_class_key *key,
Tejun Heob196be82012-01-10 15:11:35 -08003739 const char *lock_name, ...)
Oleg Nesterov3af244332007-05-09 02:34:09 -07003740{
Tejun Heob196be82012-01-10 15:11:35 -08003741 va_list args, args1;
Oleg Nesterov3af244332007-05-09 02:34:09 -07003742 struct workqueue_struct *wq;
Tejun Heo49e3cf42013-03-12 11:29:58 -07003743 struct pool_workqueue *pwq;
Tejun Heob196be82012-01-10 15:11:35 -08003744 size_t namelen;
3745
3746 /* determine namelen, allocate wq and format name */
3747 va_start(args, lock_name);
3748 va_copy(args1, args);
3749 namelen = vsnprintf(NULL, 0, fmt, args) + 1;
3750
3751 wq = kzalloc(sizeof(*wq) + namelen, GFP_KERNEL);
3752 if (!wq)
Tejun Heod2c1d402013-03-12 11:30:04 -07003753 return NULL;
Tejun Heob196be82012-01-10 15:11:35 -08003754
3755 vsnprintf(wq->name, namelen, fmt, args1);
3756 va_end(args);
3757 va_end(args1);
Oleg Nesterov3af244332007-05-09 02:34:09 -07003758
Tejun Heod320c032010-06-29 10:07:14 +02003759 max_active = max_active ?: WQ_DFL_ACTIVE;
Tejun Heob196be82012-01-10 15:11:35 -08003760 max_active = wq_clamp_max_active(max_active, flags, wq->name);
Oleg Nesterov3af244332007-05-09 02:34:09 -07003761
Tejun Heob196be82012-01-10 15:11:35 -08003762 /* init wq */
Tejun Heo97e37d72010-06-29 10:07:10 +02003763 wq->flags = flags;
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02003764 wq->saved_max_active = max_active;
Lai Jiangshan3c25a552013-03-25 16:57:17 -07003765 mutex_init(&wq->mutex);
Tejun Heo112202d2013-02-13 19:29:12 -08003766 atomic_set(&wq->nr_pwqs_to_flush, 0);
Tejun Heo30cdf242013-03-12 11:29:57 -07003767 INIT_LIST_HEAD(&wq->pwqs);
Tejun Heo73f53c42010-06-29 10:07:11 +02003768 INIT_LIST_HEAD(&wq->flusher_queue);
3769 INIT_LIST_HEAD(&wq->flusher_overflow);
Tejun Heo493a1722013-03-12 11:29:59 -07003770 INIT_LIST_HEAD(&wq->maydays);
Oleg Nesterov3af244332007-05-09 02:34:09 -07003771
Johannes Bergeb13ba82008-01-16 09:51:58 +01003772 lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
Oleg Nesterovcce1a162007-05-09 02:34:13 -07003773 INIT_LIST_HEAD(&wq->list);
Oleg Nesterov3af244332007-05-09 02:34:09 -07003774
Tejun Heo30cdf242013-03-12 11:29:57 -07003775 if (alloc_and_link_pwqs(wq) < 0)
Tejun Heod2c1d402013-03-12 11:30:04 -07003776 goto err_free_wq;
Oleg Nesterov3af244332007-05-09 02:34:09 -07003777
Tejun Heo493008a2013-03-12 11:30:03 -07003778 /*
3779 * Workqueues which may be used during memory reclaim should
3780 * have a rescuer to guarantee forward progress.
3781 */
3782 if (flags & WQ_MEM_RECLAIM) {
Tejun Heoe22bee72010-06-29 10:07:14 +02003783 struct worker *rescuer;
3784
Tejun Heod2c1d402013-03-12 11:30:04 -07003785 rescuer = alloc_worker();
Tejun Heoe22bee72010-06-29 10:07:14 +02003786 if (!rescuer)
Tejun Heod2c1d402013-03-12 11:30:04 -07003787 goto err_destroy;
Tejun Heoe22bee72010-06-29 10:07:14 +02003788
Tejun Heo111c2252013-01-17 17:16:24 -08003789 rescuer->rescue_wq = wq;
3790 rescuer->task = kthread_create(rescuer_thread, rescuer, "%s",
Tejun Heob196be82012-01-10 15:11:35 -08003791 wq->name);
Tejun Heod2c1d402013-03-12 11:30:04 -07003792 if (IS_ERR(rescuer->task)) {
3793 kfree(rescuer);
3794 goto err_destroy;
3795 }
Tejun Heoe22bee72010-06-29 10:07:14 +02003796
Tejun Heod2c1d402013-03-12 11:30:04 -07003797 wq->rescuer = rescuer;
Tejun Heo14a40ff2013-03-19 13:45:20 -07003798 rescuer->task->flags |= PF_NO_SETAFFINITY;
Tejun Heoe22bee72010-06-29 10:07:14 +02003799 wake_up_process(rescuer->task);
Oleg Nesterov3af244332007-05-09 02:34:09 -07003800 }
Oleg Nesterov3af244332007-05-09 02:34:09 -07003801
Tejun Heo226223a2013-03-12 11:30:05 -07003802 if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
3803 goto err_destroy;
3804
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02003805 /*
Lai Jiangshan68e13a62013-03-25 16:57:17 -07003806 * wq_pool_mutex protects global freeze state and workqueues list.
3807 * Grab it, adjust max_active and add the new @wq to workqueues
3808 * list.
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02003809 */
Lai Jiangshan68e13a62013-03-25 16:57:17 -07003810 mutex_lock(&wq_pool_mutex);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02003811
Lai Jiangshana357fc02013-03-25 16:57:19 -07003812 mutex_lock(&wq->mutex);
Tejun Heo699ce092013-03-13 16:51:35 -07003813 for_each_pwq(pwq, wq)
3814 pwq_adjust_max_active(pwq);
Lai Jiangshana357fc02013-03-25 16:57:19 -07003815 mutex_unlock(&wq->mutex);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02003816
Tejun Heo15376632010-06-29 10:07:11 +02003817 list_add(&wq->list, &workqueues);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02003818
Lai Jiangshan68e13a62013-03-25 16:57:17 -07003819 mutex_unlock(&wq_pool_mutex);
Tejun Heo15376632010-06-29 10:07:11 +02003820
Oleg Nesterov3af244332007-05-09 02:34:09 -07003821 return wq;
Tejun Heod2c1d402013-03-12 11:30:04 -07003822
3823err_free_wq:
3824 kfree(wq);
3825 return NULL;
3826err_destroy:
3827 destroy_workqueue(wq);
Tejun Heo4690c4a2010-06-29 10:07:10 +02003828 return NULL;
Oleg Nesterov3af244332007-05-09 02:34:09 -07003829}
Tejun Heod320c032010-06-29 10:07:14 +02003830EXPORT_SYMBOL_GPL(__alloc_workqueue_key);
Oleg Nesterov3af244332007-05-09 02:34:09 -07003831
3832/**
3833 * destroy_workqueue - safely terminate a workqueue
3834 * @wq: target workqueue
3835 *
3836 * Safely destroy a workqueue. All work currently pending will be done first.
3837 */
3838void destroy_workqueue(struct workqueue_struct *wq)
3839{
Tejun Heo49e3cf42013-03-12 11:29:58 -07003840 struct pool_workqueue *pwq;
Oleg Nesterov3af244332007-05-09 02:34:09 -07003841
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02003842 /* drain it before proceeding with destruction */
3843 drain_workqueue(wq);
Tejun Heoc8efcc22010-12-20 19:32:04 +01003844
Tejun Heo6183c002013-03-12 11:29:57 -07003845 /* sanity checks */
Lai Jiangshanb09f4fd2013-03-25 16:57:18 -07003846 mutex_lock(&wq->mutex);
Tejun Heo49e3cf42013-03-12 11:29:58 -07003847 for_each_pwq(pwq, wq) {
Tejun Heo6183c002013-03-12 11:29:57 -07003848 int i;
3849
Tejun Heo76af4d92013-03-12 11:30:00 -07003850 for (i = 0; i < WORK_NR_COLORS; i++) {
3851 if (WARN_ON(pwq->nr_in_flight[i])) {
Lai Jiangshanb09f4fd2013-03-25 16:57:18 -07003852 mutex_unlock(&wq->mutex);
Tejun Heo6183c002013-03-12 11:29:57 -07003853 return;
Tejun Heo76af4d92013-03-12 11:30:00 -07003854 }
3855 }
3856
Tejun Heo8864b4e2013-03-12 11:30:04 -07003857 if (WARN_ON(pwq->refcnt > 1) ||
3858 WARN_ON(pwq->nr_active) ||
Tejun Heo76af4d92013-03-12 11:30:00 -07003859 WARN_ON(!list_empty(&pwq->delayed_works))) {
Lai Jiangshanb09f4fd2013-03-25 16:57:18 -07003860 mutex_unlock(&wq->mutex);
Tejun Heo6183c002013-03-12 11:29:57 -07003861 return;
Tejun Heo76af4d92013-03-12 11:30:00 -07003862 }
Tejun Heo6183c002013-03-12 11:29:57 -07003863 }
Lai Jiangshanb09f4fd2013-03-25 16:57:18 -07003864 mutex_unlock(&wq->mutex);
Tejun Heo6183c002013-03-12 11:29:57 -07003865
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02003866 /*
3867 * wq list is used to freeze wq, remove from list after
3868 * flushing is complete in case freeze races us.
3869 */
Lai Jiangshan68e13a62013-03-25 16:57:17 -07003870 mutex_lock(&wq_pool_mutex);
Tejun Heod2c1d402013-03-12 11:30:04 -07003871 list_del_init(&wq->list);
Lai Jiangshan68e13a62013-03-25 16:57:17 -07003872 mutex_unlock(&wq_pool_mutex);
Oleg Nesterov3af244332007-05-09 02:34:09 -07003873
Tejun Heo226223a2013-03-12 11:30:05 -07003874 workqueue_sysfs_unregister(wq);
3875
Tejun Heo493008a2013-03-12 11:30:03 -07003876 if (wq->rescuer) {
Tejun Heoe22bee72010-06-29 10:07:14 +02003877 kthread_stop(wq->rescuer->task);
Xiaotian Feng8d9df9f2010-08-16 09:54:28 +02003878 kfree(wq->rescuer);
Tejun Heo493008a2013-03-12 11:30:03 -07003879 wq->rescuer = NULL;
Tejun Heoe22bee72010-06-29 10:07:14 +02003880 }
3881
Tejun Heo8864b4e2013-03-12 11:30:04 -07003882 if (!(wq->flags & WQ_UNBOUND)) {
3883 /*
3884 * The base ref is never dropped on per-cpu pwqs. Directly
3885 * free the pwqs and wq.
3886 */
3887 free_percpu(wq->cpu_pwqs);
3888 kfree(wq);
3889 } else {
3890 /*
3891 * We're the sole accessor of @wq at this point. Directly
3892 * access the first pwq and put the base ref. As both pwqs
3893 * and pools are sched-RCU protected, the lock operations
3894 * are safe. @wq will be freed when the last pwq is
3895 * released.
3896 */
Tejun Heo29c91e92013-03-12 11:30:03 -07003897 pwq = list_first_entry(&wq->pwqs, struct pool_workqueue,
3898 pwqs_node);
Tejun Heo8864b4e2013-03-12 11:30:04 -07003899 spin_lock_irq(&pwq->pool->lock);
3900 put_pwq(pwq);
3901 spin_unlock_irq(&pwq->pool->lock);
Tejun Heo29c91e92013-03-12 11:30:03 -07003902 }
Oleg Nesterov3af244332007-05-09 02:34:09 -07003903}
3904EXPORT_SYMBOL_GPL(destroy_workqueue);
3905
Tejun Heodcd989c2010-06-29 10:07:14 +02003906/**
3907 * workqueue_set_max_active - adjust max_active of a workqueue
3908 * @wq: target workqueue
3909 * @max_active: new max_active value.
3910 *
3911 * Set max_active of @wq to @max_active.
3912 *
3913 * CONTEXT:
3914 * Don't call from IRQ context.
3915 */
3916void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
3917{
Tejun Heo49e3cf42013-03-12 11:29:58 -07003918 struct pool_workqueue *pwq;
Tejun Heodcd989c2010-06-29 10:07:14 +02003919
Tejun Heo8719dce2013-03-12 11:30:04 -07003920 /* disallow meddling with max_active for ordered workqueues */
3921 if (WARN_ON(wq->flags & __WQ_ORDERED))
3922 return;
3923
Tejun Heof3421792010-07-02 10:03:51 +02003924 max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
Tejun Heodcd989c2010-06-29 10:07:14 +02003925
Lai Jiangshana357fc02013-03-25 16:57:19 -07003926 mutex_lock(&wq->mutex);
Tejun Heodcd989c2010-06-29 10:07:14 +02003927
3928 wq->saved_max_active = max_active;
3929
Tejun Heo699ce092013-03-13 16:51:35 -07003930 for_each_pwq(pwq, wq)
3931 pwq_adjust_max_active(pwq);
Tejun Heodcd989c2010-06-29 10:07:14 +02003932
Lai Jiangshana357fc02013-03-25 16:57:19 -07003933 mutex_unlock(&wq->mutex);
Tejun Heodcd989c2010-06-29 10:07:14 +02003934}
3935EXPORT_SYMBOL_GPL(workqueue_set_max_active);
3936
3937/**
Tejun Heoe6267612013-03-12 17:41:37 -07003938 * current_is_workqueue_rescuer - is %current workqueue rescuer?
3939 *
3940 * Determine whether %current is a workqueue rescuer. Can be used from
3941 * work functions to determine whether it's being run off the rescuer task.
3942 */
3943bool current_is_workqueue_rescuer(void)
3944{
3945 struct worker *worker = current_wq_worker();
3946
Lai Jiangshan6a092df2013-03-20 03:28:03 +08003947 return worker && worker->rescue_wq;
Tejun Heoe6267612013-03-12 17:41:37 -07003948}
3949
3950/**
Tejun Heodcd989c2010-06-29 10:07:14 +02003951 * workqueue_congested - test whether a workqueue is congested
3952 * @cpu: CPU in question
3953 * @wq: target workqueue
3954 *
3955 * Test whether @wq's cpu workqueue for @cpu is congested. There is
3956 * no synchronization around this function and the test result is
3957 * unreliable and only useful as advisory hints or for debugging.
3958 *
3959 * RETURNS:
3960 * %true if congested, %false otherwise.
3961 */
Tejun Heod84ff052013-03-12 11:29:59 -07003962bool workqueue_congested(int cpu, struct workqueue_struct *wq)
Tejun Heodcd989c2010-06-29 10:07:14 +02003963{
Tejun Heo7fb98ea2013-03-12 11:30:00 -07003964 struct pool_workqueue *pwq;
Tejun Heo76af4d92013-03-12 11:30:00 -07003965 bool ret;
3966
Lai Jiangshan88109452013-03-20 03:28:10 +08003967 rcu_read_lock_sched();
Tejun Heo7fb98ea2013-03-12 11:30:00 -07003968
3969 if (!(wq->flags & WQ_UNBOUND))
3970 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
3971 else
3972 pwq = first_pwq(wq);
Tejun Heodcd989c2010-06-29 10:07:14 +02003973
Tejun Heo76af4d92013-03-12 11:30:00 -07003974 ret = !list_empty(&pwq->delayed_works);
Lai Jiangshan88109452013-03-20 03:28:10 +08003975 rcu_read_unlock_sched();
Tejun Heo76af4d92013-03-12 11:30:00 -07003976
3977 return ret;
Tejun Heodcd989c2010-06-29 10:07:14 +02003978}
3979EXPORT_SYMBOL_GPL(workqueue_congested);
3980
3981/**
Tejun Heodcd989c2010-06-29 10:07:14 +02003982 * work_busy - test whether a work is currently pending or running
3983 * @work: the work to be tested
3984 *
3985 * Test whether @work is currently pending or running. There is no
3986 * synchronization around this function and the test result is
3987 * unreliable and only useful as advisory hints or for debugging.
Tejun Heodcd989c2010-06-29 10:07:14 +02003988 *
3989 * RETURNS:
3990 * OR'd bitmask of WORK_BUSY_* bits.
3991 */
3992unsigned int work_busy(struct work_struct *work)
3993{
Tejun Heofa1b54e2013-03-12 11:30:00 -07003994 struct worker_pool *pool;
Tejun Heodcd989c2010-06-29 10:07:14 +02003995 unsigned long flags;
3996 unsigned int ret = 0;
3997
Tejun Heodcd989c2010-06-29 10:07:14 +02003998 if (work_pending(work))
3999 ret |= WORK_BUSY_PENDING;
Tejun Heodcd989c2010-06-29 10:07:14 +02004000
Tejun Heofa1b54e2013-03-12 11:30:00 -07004001 local_irq_save(flags);
4002 pool = get_work_pool(work);
Lai Jiangshan038366c2013-02-06 18:04:53 -08004003 if (pool) {
Tejun Heofa1b54e2013-03-12 11:30:00 -07004004 spin_lock(&pool->lock);
Lai Jiangshan038366c2013-02-06 18:04:53 -08004005 if (find_worker_executing_work(pool, work))
4006 ret |= WORK_BUSY_RUNNING;
Tejun Heofa1b54e2013-03-12 11:30:00 -07004007 spin_unlock(&pool->lock);
Lai Jiangshan038366c2013-02-06 18:04:53 -08004008 }
Tejun Heofa1b54e2013-03-12 11:30:00 -07004009 local_irq_restore(flags);
Tejun Heodcd989c2010-06-29 10:07:14 +02004010
4011 return ret;
4012}
4013EXPORT_SYMBOL_GPL(work_busy);
4014
Tejun Heodb7bccf2010-06-29 10:07:12 +02004015/*
4016 * CPU hotplug.
4017 *
Tejun Heoe22bee72010-06-29 10:07:14 +02004018 * There are two challenges in supporting CPU hotplug. Firstly, there
Tejun Heo112202d2013-02-13 19:29:12 -08004019 * are a lot of assumptions on strong associations among work, pwq and
Tejun Heo706026c2013-01-24 11:01:34 -08004020 * pool which make migrating pending and scheduled works very
Tejun Heoe22bee72010-06-29 10:07:14 +02004021 * difficult to implement without impacting hot paths. Secondly,
Tejun Heo94cf58b2013-01-24 11:01:33 -08004022 * worker pools serve mix of short, long and very long running works making
Tejun Heoe22bee72010-06-29 10:07:14 +02004023 * blocked draining impractical.
4024 *
Tejun Heo24647572013-01-24 11:01:33 -08004025 * This is solved by allowing the pools to be disassociated from the CPU
Tejun Heo628c78e2012-07-17 12:39:27 -07004026 * running as an unbound one and allowing it to be reattached later if the
4027 * cpu comes back online.
Tejun Heodb7bccf2010-06-29 10:07:12 +02004028 */
4029
Tejun Heo706026c2013-01-24 11:01:34 -08004030static void wq_unbind_fn(struct work_struct *work)
Tejun Heodb7bccf2010-06-29 10:07:12 +02004031{
Tejun Heo38db41d2013-01-24 11:01:34 -08004032 int cpu = smp_processor_id();
Tejun Heo4ce62e92012-07-13 22:16:44 -07004033 struct worker_pool *pool;
Tejun Heodb7bccf2010-06-29 10:07:12 +02004034 struct worker *worker;
Tejun Heoa9ab7752013-03-19 13:45:21 -07004035 int wi;
Tejun Heodb7bccf2010-06-29 10:07:12 +02004036
Tejun Heof02ae732013-03-12 11:30:03 -07004037 for_each_cpu_worker_pool(pool, cpu) {
Tejun Heo6183c002013-03-12 11:29:57 -07004038 WARN_ON_ONCE(cpu != smp_processor_id());
Tejun Heo94cf58b2013-01-24 11:01:33 -08004039
Tejun Heobc3a1af2013-03-13 19:47:39 -07004040 mutex_lock(&pool->manager_mutex);
Tejun Heo94cf58b2013-01-24 11:01:33 -08004041 spin_lock_irq(&pool->lock);
4042
4043 /*
Tejun Heobc3a1af2013-03-13 19:47:39 -07004044 * We've blocked all manager operations. Make all workers
Tejun Heo94cf58b2013-01-24 11:01:33 -08004045 * unbound and set DISASSOCIATED. Before this, all workers
4046 * except for the ones which are still executing works from
4047 * before the last CPU down must be on the cpu. After
4048 * this, they may become diasporas.
4049 */
Tejun Heoa9ab7752013-03-19 13:45:21 -07004050 for_each_pool_worker(worker, wi, pool)
Tejun Heoc9e7cf22013-01-24 11:01:33 -08004051 worker->flags |= WORKER_UNBOUND;
Tejun Heodb7bccf2010-06-29 10:07:12 +02004052
Tejun Heo24647572013-01-24 11:01:33 -08004053 pool->flags |= POOL_DISASSOCIATED;
Tejun Heof2d5a0e2012-07-17 12:39:26 -07004054
Tejun Heo94cf58b2013-01-24 11:01:33 -08004055 spin_unlock_irq(&pool->lock);
Tejun Heobc3a1af2013-03-13 19:47:39 -07004056 mutex_unlock(&pool->manager_mutex);
Tejun Heo94cf58b2013-01-24 11:01:33 -08004057 }
Tejun Heoe22bee72010-06-29 10:07:14 +02004058
4059 /*
Tejun Heo628c78e2012-07-17 12:39:27 -07004060 * Call schedule() so that we cross rq->lock and thus can guarantee
4061 * sched callbacks see the %WORKER_UNBOUND flag. This is necessary
4062 * as scheduler callbacks may be invoked from other cpus.
4063 */
4064 schedule();
4065
4066 /*
4067 * Sched callbacks are disabled now. Zap nr_running. After this,
4068 * nr_running stays zero and need_more_worker() and keep_working()
Tejun Heo38db41d2013-01-24 11:01:34 -08004069 * are always true as long as the worklist is not empty. Pools on
4070 * @cpu now behave as unbound (in terms of concurrency management)
4071 * pools which are served by workers tied to the CPU.
Tejun Heo628c78e2012-07-17 12:39:27 -07004072 *
4073 * On return from this function, the current worker would trigger
4074 * unbound chain execution of pending work items if other workers
4075 * didn't already.
Tejun Heoe22bee72010-06-29 10:07:14 +02004076 */
Tejun Heof02ae732013-03-12 11:30:03 -07004077 for_each_cpu_worker_pool(pool, cpu)
Tejun Heoe19e3972013-01-24 11:39:44 -08004078 atomic_set(&pool->nr_running, 0);
Tejun Heodb7bccf2010-06-29 10:07:12 +02004079}
4080
Tejun Heobd7c0892013-03-19 13:45:21 -07004081/**
4082 * rebind_workers - rebind all workers of a pool to the associated CPU
4083 * @pool: pool of interest
4084 *
Tejun Heoa9ab7752013-03-19 13:45:21 -07004085 * @pool->cpu is coming online. Rebind all workers to the CPU.
Tejun Heobd7c0892013-03-19 13:45:21 -07004086 */
4087static void rebind_workers(struct worker_pool *pool)
4088{
Tejun Heoa9ab7752013-03-19 13:45:21 -07004089 struct worker *worker;
4090 int wi;
Tejun Heobd7c0892013-03-19 13:45:21 -07004091
4092 lockdep_assert_held(&pool->manager_mutex);
Tejun Heobd7c0892013-03-19 13:45:21 -07004093
Tejun Heoa9ab7752013-03-19 13:45:21 -07004094 /*
4095 * Restore CPU affinity of all workers. As all idle workers should
4096 * be on the run-queue of the associated CPU before any local
4097 * wake-ups for concurrency management happen, restore CPU affinty
4098 * of all workers first and then clear UNBOUND. As we're called
4099 * from CPU_ONLINE, the following shouldn't fail.
4100 */
4101 for_each_pool_worker(worker, wi, pool)
4102 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4103 pool->attrs->cpumask) < 0);
4104
4105 spin_lock_irq(&pool->lock);
4106
4107 for_each_pool_worker(worker, wi, pool) {
4108 unsigned int worker_flags = worker->flags;
Tejun Heobd7c0892013-03-19 13:45:21 -07004109
4110 /*
Tejun Heoa9ab7752013-03-19 13:45:21 -07004111 * A bound idle worker should actually be on the runqueue
4112 * of the associated CPU for local wake-ups targeting it to
4113 * work. Kick all idle workers so that they migrate to the
4114 * associated CPU. Doing this in the same loop as
4115 * replacing UNBOUND with REBOUND is safe as no worker will
4116 * be bound before @pool->lock is released.
Tejun Heobd7c0892013-03-19 13:45:21 -07004117 */
Tejun Heoa9ab7752013-03-19 13:45:21 -07004118 if (worker_flags & WORKER_IDLE)
4119 wake_up_process(worker->task);
4120
4121 /*
4122 * We want to clear UNBOUND but can't directly call
4123 * worker_clr_flags() or adjust nr_running. Atomically
4124 * replace UNBOUND with another NOT_RUNNING flag REBOUND.
4125 * @worker will clear REBOUND using worker_clr_flags() when
4126 * it initiates the next execution cycle thus restoring
4127 * concurrency management. Note that when or whether
4128 * @worker clears REBOUND doesn't affect correctness.
4129 *
4130 * ACCESS_ONCE() is necessary because @worker->flags may be
4131 * tested without holding any lock in
4132 * wq_worker_waking_up(). Without it, NOT_RUNNING test may
4133 * fail incorrectly leading to premature concurrency
4134 * management operations.
4135 */
4136 WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
4137 worker_flags |= WORKER_REBOUND;
4138 worker_flags &= ~WORKER_UNBOUND;
4139 ACCESS_ONCE(worker->flags) = worker_flags;
Tejun Heobd7c0892013-03-19 13:45:21 -07004140 }
4141
Tejun Heoa9ab7752013-03-19 13:45:21 -07004142 spin_unlock_irq(&pool->lock);
Tejun Heobd7c0892013-03-19 13:45:21 -07004143}
4144
Tejun Heo7dbc7252013-03-19 13:45:21 -07004145/**
4146 * restore_unbound_workers_cpumask - restore cpumask of unbound workers
4147 * @pool: unbound pool of interest
4148 * @cpu: the CPU which is coming up
4149 *
4150 * An unbound pool may end up with a cpumask which doesn't have any online
4151 * CPUs. When a worker of such pool get scheduled, the scheduler resets
4152 * its cpus_allowed. If @cpu is in @pool's cpumask which didn't have any
4153 * online CPU before, cpus_allowed of all its workers should be restored.
4154 */
4155static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
4156{
4157 static cpumask_t cpumask;
4158 struct worker *worker;
4159 int wi;
4160
4161 lockdep_assert_held(&pool->manager_mutex);
4162
4163 /* is @cpu allowed for @pool? */
4164 if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
4165 return;
4166
4167 /* is @cpu the only online CPU? */
4168 cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
4169 if (cpumask_weight(&cpumask) != 1)
4170 return;
4171
4172 /* as we're called from CPU_ONLINE, the following shouldn't fail */
4173 for_each_pool_worker(worker, wi, pool)
4174 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4175 pool->attrs->cpumask) < 0);
4176}
4177
Tejun Heo8db25e72012-07-17 12:39:28 -07004178/*
4179 * Workqueues should be brought up before normal priority CPU notifiers.
4180 * This will be registered high priority CPU notifier.
4181 */
Lai Jiangshan9fdf9b72012-09-18 09:59:23 -07004182static int __cpuinit workqueue_cpu_up_callback(struct notifier_block *nfb,
Tejun Heo8db25e72012-07-17 12:39:28 -07004183 unsigned long action,
4184 void *hcpu)
Oleg Nesterov3af244332007-05-09 02:34:09 -07004185{
Tejun Heod84ff052013-03-12 11:29:59 -07004186 int cpu = (unsigned long)hcpu;
Tejun Heo4ce62e92012-07-13 22:16:44 -07004187 struct worker_pool *pool;
Tejun Heo7dbc7252013-03-19 13:45:21 -07004188 int pi;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004189
Tejun Heo8db25e72012-07-17 12:39:28 -07004190 switch (action & ~CPU_TASKS_FROZEN) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004191 case CPU_UP_PREPARE:
Tejun Heof02ae732013-03-12 11:30:03 -07004192 for_each_cpu_worker_pool(pool, cpu) {
Tejun Heo3ce63372012-07-17 12:39:27 -07004193 if (pool->nr_workers)
4194 continue;
Tejun Heoebf44d12013-03-13 19:47:39 -07004195 if (create_and_start_worker(pool) < 0)
Tejun Heo3ce63372012-07-17 12:39:27 -07004196 return NOTIFY_BAD;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004197 }
Tejun Heodb7bccf2010-06-29 10:07:12 +02004198 break;
Oleg Nesterov00dfcaf2008-04-29 01:00:27 -07004199
Tejun Heo65758202012-07-17 12:39:26 -07004200 case CPU_DOWN_FAILED:
4201 case CPU_ONLINE:
Lai Jiangshan68e13a62013-03-25 16:57:17 -07004202 mutex_lock(&wq_pool_mutex);
Tejun Heo7dbc7252013-03-19 13:45:21 -07004203
4204 for_each_pool(pool, pi) {
Tejun Heobc3a1af2013-03-13 19:47:39 -07004205 mutex_lock(&pool->manager_mutex);
Tejun Heo94cf58b2013-01-24 11:01:33 -08004206
Tejun Heo7dbc7252013-03-19 13:45:21 -07004207 if (pool->cpu == cpu) {
4208 spin_lock_irq(&pool->lock);
4209 pool->flags &= ~POOL_DISASSOCIATED;
4210 spin_unlock_irq(&pool->lock);
Tejun Heoa9ab7752013-03-19 13:45:21 -07004211
Tejun Heo7dbc7252013-03-19 13:45:21 -07004212 rebind_workers(pool);
4213 } else if (pool->cpu < 0) {
4214 restore_unbound_workers_cpumask(pool, cpu);
4215 }
Tejun Heo94cf58b2013-01-24 11:01:33 -08004216
Tejun Heobc3a1af2013-03-13 19:47:39 -07004217 mutex_unlock(&pool->manager_mutex);
Tejun Heo94cf58b2013-01-24 11:01:33 -08004218 }
Tejun Heo7dbc7252013-03-19 13:45:21 -07004219
Lai Jiangshan68e13a62013-03-25 16:57:17 -07004220 mutex_unlock(&wq_pool_mutex);
Tejun Heo8db25e72012-07-17 12:39:28 -07004221 break;
Tejun Heo65758202012-07-17 12:39:26 -07004222 }
4223 return NOTIFY_OK;
4224}
4225
4226/*
4227 * Workqueues should be brought down after normal priority CPU notifiers.
4228 * This will be registered as low priority CPU notifier.
4229 */
Lai Jiangshan9fdf9b72012-09-18 09:59:23 -07004230static int __cpuinit workqueue_cpu_down_callback(struct notifier_block *nfb,
Tejun Heo65758202012-07-17 12:39:26 -07004231 unsigned long action,
4232 void *hcpu)
4233{
Tejun Heod84ff052013-03-12 11:29:59 -07004234 int cpu = (unsigned long)hcpu;
Tejun Heo8db25e72012-07-17 12:39:28 -07004235 struct work_struct unbind_work;
4236
Tejun Heo65758202012-07-17 12:39:26 -07004237 switch (action & ~CPU_TASKS_FROZEN) {
4238 case CPU_DOWN_PREPARE:
Tejun Heo8db25e72012-07-17 12:39:28 -07004239 /* unbinding should happen on the local CPU */
Tejun Heo706026c2013-01-24 11:01:34 -08004240 INIT_WORK_ONSTACK(&unbind_work, wq_unbind_fn);
Joonsoo Kim7635d2f2012-08-15 23:25:41 +09004241 queue_work_on(cpu, system_highpri_wq, &unbind_work);
Tejun Heo8db25e72012-07-17 12:39:28 -07004242 flush_work(&unbind_work);
4243 break;
Tejun Heo65758202012-07-17 12:39:26 -07004244 }
4245 return NOTIFY_OK;
4246}
4247
Rusty Russell2d3854a2008-11-05 13:39:10 +11004248#ifdef CONFIG_SMP
Rusty Russell8ccad402009-01-16 15:31:15 -08004249
Rusty Russell2d3854a2008-11-05 13:39:10 +11004250struct work_for_cpu {
Tejun Heoed48ece2012-09-18 12:48:43 -07004251 struct work_struct work;
Rusty Russell2d3854a2008-11-05 13:39:10 +11004252 long (*fn)(void *);
4253 void *arg;
4254 long ret;
4255};
4256
Tejun Heoed48ece2012-09-18 12:48:43 -07004257static void work_for_cpu_fn(struct work_struct *work)
Rusty Russell2d3854a2008-11-05 13:39:10 +11004258{
Tejun Heoed48ece2012-09-18 12:48:43 -07004259 struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
4260
Rusty Russell2d3854a2008-11-05 13:39:10 +11004261 wfc->ret = wfc->fn(wfc->arg);
4262}
4263
4264/**
4265 * work_on_cpu - run a function in user context on a particular cpu
4266 * @cpu: the cpu to run on
4267 * @fn: the function to run
4268 * @arg: the function arg
4269 *
Rusty Russell31ad9082009-01-16 15:31:15 -08004270 * This will return the value @fn returns.
4271 * It is up to the caller to ensure that the cpu doesn't go offline.
Andrew Morton6b440032009-04-09 09:50:37 -06004272 * The caller must not hold any locks which would prevent @fn from completing.
Rusty Russell2d3854a2008-11-05 13:39:10 +11004273 */
Tejun Heod84ff052013-03-12 11:29:59 -07004274long work_on_cpu(int cpu, long (*fn)(void *), void *arg)
Rusty Russell2d3854a2008-11-05 13:39:10 +11004275{
Tejun Heoed48ece2012-09-18 12:48:43 -07004276 struct work_for_cpu wfc = { .fn = fn, .arg = arg };
Rusty Russell2d3854a2008-11-05 13:39:10 +11004277
Tejun Heoed48ece2012-09-18 12:48:43 -07004278 INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
4279 schedule_work_on(cpu, &wfc.work);
4280 flush_work(&wfc.work);
Rusty Russell2d3854a2008-11-05 13:39:10 +11004281 return wfc.ret;
4282}
4283EXPORT_SYMBOL_GPL(work_on_cpu);
4284#endif /* CONFIG_SMP */
4285
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004286#ifdef CONFIG_FREEZER
Rusty Russelle7577c52009-01-01 10:12:25 +10304287
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004288/**
4289 * freeze_workqueues_begin - begin freezing workqueues
4290 *
Tejun Heo58a69cb2011-02-16 09:25:31 +01004291 * Start freezing workqueues. After this function returns, all freezable
Tejun Heoc5aa87b2013-03-13 16:51:36 -07004292 * workqueues will queue new works to their delayed_works list instead of
Tejun Heo706026c2013-01-24 11:01:34 -08004293 * pool->worklist.
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004294 *
4295 * CONTEXT:
Lai Jiangshana357fc02013-03-25 16:57:19 -07004296 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004297 */
4298void freeze_workqueues_begin(void)
4299{
Tejun Heo17116962013-03-12 11:29:58 -07004300 struct worker_pool *pool;
Tejun Heo24b8a842013-03-12 11:29:58 -07004301 struct workqueue_struct *wq;
4302 struct pool_workqueue *pwq;
Tejun Heo611c92a2013-03-13 16:51:36 -07004303 int pi;
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004304
Lai Jiangshan68e13a62013-03-25 16:57:17 -07004305 mutex_lock(&wq_pool_mutex);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004306
Tejun Heo6183c002013-03-12 11:29:57 -07004307 WARN_ON_ONCE(workqueue_freezing);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004308 workqueue_freezing = true;
4309
Tejun Heo24b8a842013-03-12 11:29:58 -07004310 /* set FREEZING */
Tejun Heo611c92a2013-03-13 16:51:36 -07004311 for_each_pool(pool, pi) {
Tejun Heo5bcab332013-03-13 19:47:40 -07004312 spin_lock_irq(&pool->lock);
Tejun Heo17116962013-03-12 11:29:58 -07004313 WARN_ON_ONCE(pool->flags & POOL_FREEZING);
4314 pool->flags |= POOL_FREEZING;
Tejun Heo5bcab332013-03-13 19:47:40 -07004315 spin_unlock_irq(&pool->lock);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004316 }
4317
Tejun Heo24b8a842013-03-12 11:29:58 -07004318 list_for_each_entry(wq, &workqueues, list) {
Lai Jiangshana357fc02013-03-25 16:57:19 -07004319 mutex_lock(&wq->mutex);
Tejun Heo699ce092013-03-13 16:51:35 -07004320 for_each_pwq(pwq, wq)
4321 pwq_adjust_max_active(pwq);
Lai Jiangshana357fc02013-03-25 16:57:19 -07004322 mutex_unlock(&wq->mutex);
Tejun Heo24b8a842013-03-12 11:29:58 -07004323 }
Tejun Heo5bcab332013-03-13 19:47:40 -07004324
Lai Jiangshan68e13a62013-03-25 16:57:17 -07004325 mutex_unlock(&wq_pool_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004326}
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004327
4328/**
Tejun Heo58a69cb2011-02-16 09:25:31 +01004329 * freeze_workqueues_busy - are freezable workqueues still busy?
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004330 *
4331 * Check whether freezing is complete. This function must be called
4332 * between freeze_workqueues_begin() and thaw_workqueues().
4333 *
4334 * CONTEXT:
Lai Jiangshan68e13a62013-03-25 16:57:17 -07004335 * Grabs and releases wq_pool_mutex.
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004336 *
4337 * RETURNS:
Tejun Heo58a69cb2011-02-16 09:25:31 +01004338 * %true if some freezable workqueues are still busy. %false if freezing
4339 * is complete.
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004340 */
4341bool freeze_workqueues_busy(void)
4342{
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004343 bool busy = false;
Tejun Heo24b8a842013-03-12 11:29:58 -07004344 struct workqueue_struct *wq;
4345 struct pool_workqueue *pwq;
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004346
Lai Jiangshan68e13a62013-03-25 16:57:17 -07004347 mutex_lock(&wq_pool_mutex);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004348
Tejun Heo6183c002013-03-12 11:29:57 -07004349 WARN_ON_ONCE(!workqueue_freezing);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004350
Tejun Heo24b8a842013-03-12 11:29:58 -07004351 list_for_each_entry(wq, &workqueues, list) {
4352 if (!(wq->flags & WQ_FREEZABLE))
4353 continue;
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004354 /*
4355 * nr_active is monotonically decreasing. It's safe
4356 * to peek without lock.
4357 */
Lai Jiangshan88109452013-03-20 03:28:10 +08004358 rcu_read_lock_sched();
Tejun Heo24b8a842013-03-12 11:29:58 -07004359 for_each_pwq(pwq, wq) {
Tejun Heo6183c002013-03-12 11:29:57 -07004360 WARN_ON_ONCE(pwq->nr_active < 0);
Tejun Heo112202d2013-02-13 19:29:12 -08004361 if (pwq->nr_active) {
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004362 busy = true;
Lai Jiangshan88109452013-03-20 03:28:10 +08004363 rcu_read_unlock_sched();
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004364 goto out_unlock;
4365 }
4366 }
Lai Jiangshan88109452013-03-20 03:28:10 +08004367 rcu_read_unlock_sched();
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004368 }
4369out_unlock:
Lai Jiangshan68e13a62013-03-25 16:57:17 -07004370 mutex_unlock(&wq_pool_mutex);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004371 return busy;
4372}
4373
4374/**
4375 * thaw_workqueues - thaw workqueues
4376 *
4377 * Thaw workqueues. Normal queueing is restored and all collected
Tejun Heo706026c2013-01-24 11:01:34 -08004378 * frozen works are transferred to their respective pool worklists.
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004379 *
4380 * CONTEXT:
Lai Jiangshana357fc02013-03-25 16:57:19 -07004381 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004382 */
4383void thaw_workqueues(void)
4384{
Tejun Heo24b8a842013-03-12 11:29:58 -07004385 struct workqueue_struct *wq;
4386 struct pool_workqueue *pwq;
4387 struct worker_pool *pool;
Tejun Heo611c92a2013-03-13 16:51:36 -07004388 int pi;
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004389
Lai Jiangshan68e13a62013-03-25 16:57:17 -07004390 mutex_lock(&wq_pool_mutex);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004391
4392 if (!workqueue_freezing)
4393 goto out_unlock;
4394
Tejun Heo24b8a842013-03-12 11:29:58 -07004395 /* clear FREEZING */
Tejun Heo611c92a2013-03-13 16:51:36 -07004396 for_each_pool(pool, pi) {
Tejun Heo5bcab332013-03-13 19:47:40 -07004397 spin_lock_irq(&pool->lock);
Tejun Heo24b8a842013-03-12 11:29:58 -07004398 WARN_ON_ONCE(!(pool->flags & POOL_FREEZING));
4399 pool->flags &= ~POOL_FREEZING;
Tejun Heo5bcab332013-03-13 19:47:40 -07004400 spin_unlock_irq(&pool->lock);
Tejun Heo24b8a842013-03-12 11:29:58 -07004401 }
Tejun Heo8b03ae32010-06-29 10:07:12 +02004402
Tejun Heo24b8a842013-03-12 11:29:58 -07004403 /* restore max_active and repopulate worklist */
4404 list_for_each_entry(wq, &workqueues, list) {
Lai Jiangshana357fc02013-03-25 16:57:19 -07004405 mutex_lock(&wq->mutex);
Tejun Heo699ce092013-03-13 16:51:35 -07004406 for_each_pwq(pwq, wq)
4407 pwq_adjust_max_active(pwq);
Lai Jiangshana357fc02013-03-25 16:57:19 -07004408 mutex_unlock(&wq->mutex);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004409 }
4410
4411 workqueue_freezing = false;
4412out_unlock:
Lai Jiangshan68e13a62013-03-25 16:57:17 -07004413 mutex_unlock(&wq_pool_mutex);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004414}
4415#endif /* CONFIG_FREEZER */
4416
Tejun Heobce90382013-04-01 11:23:32 -07004417static void __init wq_numa_init(void)
4418{
4419 cpumask_var_t *tbl;
4420 int node, cpu;
4421
4422 /* determine NUMA pwq table len - highest node id + 1 */
4423 for_each_node(node)
4424 wq_numa_tbl_len = max(wq_numa_tbl_len, node + 1);
4425
4426 if (num_possible_nodes() <= 1)
4427 return;
4428
4429 /*
4430 * We want masks of possible CPUs of each node which isn't readily
4431 * available. Build one from cpu_to_node() which should have been
4432 * fully initialized by now.
4433 */
4434 tbl = kzalloc(wq_numa_tbl_len * sizeof(tbl[0]), GFP_KERNEL);
4435 BUG_ON(!tbl);
4436
4437 for_each_node(node)
4438 BUG_ON(!alloc_cpumask_var_node(&tbl[node], GFP_KERNEL, node));
4439
4440 for_each_possible_cpu(cpu) {
4441 node = cpu_to_node(cpu);
4442 if (WARN_ON(node == NUMA_NO_NODE)) {
4443 pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
4444 /* happens iff arch is bonkers, let's just proceed */
4445 return;
4446 }
4447 cpumask_set_cpu(cpu, tbl[node]);
4448 }
4449
4450 wq_numa_possible_cpumask = tbl;
4451 wq_numa_enabled = true;
4452}
4453
Suresh Siddha6ee05782010-07-30 14:57:37 -07004454static int __init init_workqueues(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004455{
Tejun Heo7a4e3442013-03-12 11:30:00 -07004456 int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
4457 int i, cpu;
Tejun Heoc34056a2010-06-29 10:07:11 +02004458
Tejun Heo7c3eed52013-01-24 11:01:33 -08004459 /* make sure we have enough bits for OFFQ pool ID */
4460 BUILD_BUG_ON((1LU << (BITS_PER_LONG - WORK_OFFQ_POOL_SHIFT)) <
Lai Jiangshan6be19582013-02-06 18:04:53 -08004461 WORK_CPU_END * NR_STD_WORKER_POOLS);
Tejun Heob5490072012-08-03 10:30:46 -07004462
Tejun Heoe904e6c2013-03-12 11:29:57 -07004463 WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
4464
4465 pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
4466
Tejun Heo65758202012-07-17 12:39:26 -07004467 cpu_notifier(workqueue_cpu_up_callback, CPU_PRI_WORKQUEUE_UP);
Lai Jiangshana5b4e572012-09-18 09:59:23 -07004468 hotcpu_notifier(workqueue_cpu_down_callback, CPU_PRI_WORKQUEUE_DOWN);
Tejun Heo8b03ae32010-06-29 10:07:12 +02004469
Tejun Heobce90382013-04-01 11:23:32 -07004470 wq_numa_init();
4471
Tejun Heo706026c2013-01-24 11:01:34 -08004472 /* initialize CPU pools */
Tejun Heo29c91e92013-03-12 11:30:03 -07004473 for_each_possible_cpu(cpu) {
Tejun Heo4ce62e92012-07-13 22:16:44 -07004474 struct worker_pool *pool;
Tejun Heo8b03ae32010-06-29 10:07:12 +02004475
Tejun Heo7a4e3442013-03-12 11:30:00 -07004476 i = 0;
Tejun Heof02ae732013-03-12 11:30:03 -07004477 for_each_cpu_worker_pool(pool, cpu) {
Tejun Heo7a4e3442013-03-12 11:30:00 -07004478 BUG_ON(init_worker_pool(pool));
Tejun Heoec22ca52013-01-24 11:01:33 -08004479 pool->cpu = cpu;
Tejun Heo29c91e92013-03-12 11:30:03 -07004480 cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
Tejun Heo7a4e3442013-03-12 11:30:00 -07004481 pool->attrs->nice = std_nice[i++];
4482
Tejun Heo9daf9e62013-01-24 11:01:33 -08004483 /* alloc pool ID */
Lai Jiangshan68e13a62013-03-25 16:57:17 -07004484 mutex_lock(&wq_pool_mutex);
Tejun Heo9daf9e62013-01-24 11:01:33 -08004485 BUG_ON(worker_pool_assign_id(pool));
Lai Jiangshan68e13a62013-03-25 16:57:17 -07004486 mutex_unlock(&wq_pool_mutex);
Tejun Heo4ce62e92012-07-13 22:16:44 -07004487 }
Tejun Heo8b03ae32010-06-29 10:07:12 +02004488 }
4489
Tejun Heoe22bee72010-06-29 10:07:14 +02004490 /* create the initial worker */
Tejun Heo29c91e92013-03-12 11:30:03 -07004491 for_each_online_cpu(cpu) {
Tejun Heo4ce62e92012-07-13 22:16:44 -07004492 struct worker_pool *pool;
Tejun Heoe22bee72010-06-29 10:07:14 +02004493
Tejun Heof02ae732013-03-12 11:30:03 -07004494 for_each_cpu_worker_pool(pool, cpu) {
Tejun Heo29c91e92013-03-12 11:30:03 -07004495 pool->flags &= ~POOL_DISASSOCIATED;
Tejun Heoebf44d12013-03-13 19:47:39 -07004496 BUG_ON(create_and_start_worker(pool) < 0);
Tejun Heo4ce62e92012-07-13 22:16:44 -07004497 }
Tejun Heoe22bee72010-06-29 10:07:14 +02004498 }
4499
Tejun Heo29c91e92013-03-12 11:30:03 -07004500 /* create default unbound wq attrs */
4501 for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
4502 struct workqueue_attrs *attrs;
4503
4504 BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
Tejun Heo29c91e92013-03-12 11:30:03 -07004505 attrs->nice = std_nice[i];
Tejun Heo29c91e92013-03-12 11:30:03 -07004506 unbound_std_wq_attrs[i] = attrs;
4507 }
4508
Tejun Heod320c032010-06-29 10:07:14 +02004509 system_wq = alloc_workqueue("events", 0, 0);
Joonsoo Kim1aabe902012-08-15 23:25:39 +09004510 system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
Tejun Heod320c032010-06-29 10:07:14 +02004511 system_long_wq = alloc_workqueue("events_long", 0, 0);
Tejun Heof3421792010-07-02 10:03:51 +02004512 system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
4513 WQ_UNBOUND_MAX_ACTIVE);
Tejun Heo24d51ad2011-02-21 09:52:50 +01004514 system_freezable_wq = alloc_workqueue("events_freezable",
4515 WQ_FREEZABLE, 0);
Joonsoo Kim1aabe902012-08-15 23:25:39 +09004516 BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||
Tejun Heoae930e02012-08-20 14:51:23 -07004517 !system_unbound_wq || !system_freezable_wq);
Suresh Siddha6ee05782010-07-30 14:57:37 -07004518 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004519}
Suresh Siddha6ee05782010-07-30 14:57:37 -07004520early_initcall(init_workqueues);