blob: f37421fb4f35672edbb8ca1298ad77d3c41c547d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Tejun Heoc54fce62010-09-10 16:51:36 +02002 * kernel/workqueue.c - generic async execution with shared worker pool
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
Tejun Heoc54fce62010-09-10 16:51:36 +02004 * Copyright (C) 2002 Ingo Molnar
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 *
Tejun Heoc54fce62010-09-10 16:51:36 +02006 * Derived from the taskqueue/keventd code by:
7 * David Woodhouse <dwmw2@infradead.org>
8 * Andrew Morton
9 * Kai Petzke <wpp@marie.physik.tu-berlin.de>
10 * Theodore Ts'o <tytso@mit.edu>
Christoph Lameter89ada672005-10-30 15:01:59 -080011 *
Christoph Lametercde53532008-07-04 09:59:22 -070012 * Made to use alloc_percpu by Christoph Lameter.
Tejun Heoc54fce62010-09-10 16:51:36 +020013 *
14 * Copyright (C) 2010 SUSE Linux Products GmbH
15 * Copyright (C) 2010 Tejun Heo <tj@kernel.org>
16 *
17 * This is the generic async execution mechanism. Work items as are
18 * executed in process context. The worker pool is shared and
19 * automatically managed. There is one worker pool for each CPU and
20 * one extra for works which are better served by workers which are
21 * not bound to any specific CPU.
22 *
23 * Please read Documentation/workqueue.txt for details.
Linus Torvalds1da177e2005-04-16 15:20:36 -070024 */
25
Paul Gortmaker9984de12011-05-23 14:51:41 -040026#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027#include <linux/kernel.h>
28#include <linux/sched.h>
29#include <linux/init.h>
30#include <linux/signal.h>
31#include <linux/completion.h>
32#include <linux/workqueue.h>
33#include <linux/slab.h>
34#include <linux/cpu.h>
35#include <linux/notifier.h>
36#include <linux/kthread.h>
James Bottomley1fa44ec2006-02-23 12:43:43 -060037#include <linux/hardirq.h>
Christoph Lameter46934022006-10-11 01:21:26 -070038#include <linux/mempolicy.h>
Rafael J. Wysocki341a5952006-12-06 20:34:49 -080039#include <linux/freezer.h>
Peter Zijlstrad5abe662006-12-06 20:37:26 -080040#include <linux/kallsyms.h>
41#include <linux/debug_locks.h>
Johannes Berg4e6045f2007-10-18 23:39:55 -070042#include <linux/lockdep.h>
Tejun Heoc34056a2010-06-29 10:07:11 +020043#include <linux/idr.h>
Tejun Heo29c91e92013-03-12 11:30:03 -070044#include <linux/jhash.h>
Sasha Levin42f85702012-12-17 10:01:23 -050045#include <linux/hashtable.h>
Tejun Heo76af4d92013-03-12 11:30:00 -070046#include <linux/rculist.h>
Tejun Heoe22bee72010-06-29 10:07:14 +020047
Tejun Heoea138442013-01-18 14:05:55 -080048#include "workqueue_internal.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070049
Tejun Heoc8e55f32010-06-29 10:07:12 +020050enum {
Tejun Heobc2ae0f2012-07-17 12:39:27 -070051 /*
Tejun Heo24647572013-01-24 11:01:33 -080052 * worker_pool flags
Tejun Heobc2ae0f2012-07-17 12:39:27 -070053 *
Tejun Heo24647572013-01-24 11:01:33 -080054 * A bound pool is either associated or disassociated with its CPU.
Tejun Heobc2ae0f2012-07-17 12:39:27 -070055 * While associated (!DISASSOCIATED), all workers are bound to the
56 * CPU and none has %WORKER_UNBOUND set and concurrency management
57 * is in effect.
58 *
59 * While DISASSOCIATED, the cpu may be offline and all workers have
60 * %WORKER_UNBOUND set and concurrency management disabled, and may
Tejun Heo24647572013-01-24 11:01:33 -080061 * be executing on any CPU. The pool behaves as an unbound one.
Tejun Heobc2ae0f2012-07-17 12:39:27 -070062 *
63 * Note that DISASSOCIATED can be flipped only while holding
Tejun Heo24647572013-01-24 11:01:33 -080064 * assoc_mutex to avoid changing binding state while
65 * create_worker() is in progress.
Tejun Heobc2ae0f2012-07-17 12:39:27 -070066 */
Tejun Heo11ebea52012-07-12 14:46:37 -070067 POOL_MANAGE_WORKERS = 1 << 0, /* need to manage workers */
Tejun Heo24647572013-01-24 11:01:33 -080068 POOL_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */
Tejun Heo35b6bb62013-01-24 11:01:33 -080069 POOL_FREEZING = 1 << 3, /* freeze in progress */
Tejun Heodb7bccf2010-06-29 10:07:12 +020070
Tejun Heoc8e55f32010-06-29 10:07:12 +020071 /* worker flags */
72 WORKER_STARTED = 1 << 0, /* started */
73 WORKER_DIE = 1 << 1, /* die die die */
74 WORKER_IDLE = 1 << 2, /* is idle */
Tejun Heoe22bee72010-06-29 10:07:14 +020075 WORKER_PREP = 1 << 3, /* preparing to run works */
Tejun Heofb0e7be2010-06-29 10:07:15 +020076 WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */
Tejun Heof3421792010-07-02 10:03:51 +020077 WORKER_UNBOUND = 1 << 7, /* worker is unbound */
Tejun Heoe22bee72010-06-29 10:07:14 +020078
Lai Jiangshan5f7dabf2012-09-18 09:59:23 -070079 WORKER_NOT_RUNNING = WORKER_PREP | WORKER_UNBOUND |
Tejun Heo403c8212012-07-17 12:39:27 -070080 WORKER_CPU_INTENSIVE,
Tejun Heodb7bccf2010-06-29 10:07:12 +020081
Tejun Heoe34cdddb2013-01-24 11:01:33 -080082 NR_STD_WORKER_POOLS = 2, /* # standard pools per cpu */
Tejun Heo4ce62e92012-07-13 22:16:44 -070083
Tejun Heo29c91e92013-03-12 11:30:03 -070084 UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
Tejun Heoc8e55f32010-06-29 10:07:12 +020085 BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
Tejun Heodb7bccf2010-06-29 10:07:12 +020086
Tejun Heoe22bee72010-06-29 10:07:14 +020087 MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */
88 IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */
89
Tejun Heo3233cdb2011-02-16 18:10:19 +010090 MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2,
91 /* call for help after 10ms
92 (min two ticks) */
Tejun Heoe22bee72010-06-29 10:07:14 +020093 MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */
94 CREATE_COOLDOWN = HZ, /* time to breath after fail */
Tejun Heoe22bee72010-06-29 10:07:14 +020095
96 /*
97 * Rescue workers are used only on emergencies and shared by
98 * all cpus. Give -20.
99 */
100 RESCUER_NICE_LEVEL = -20,
Tejun Heo32704762012-07-13 22:16:45 -0700101 HIGHPRI_NICE_LEVEL = -20,
Tejun Heoc8e55f32010-06-29 10:07:12 +0200102};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103
104/*
Tejun Heo4690c4a2010-06-29 10:07:10 +0200105 * Structure fields follow one of the following exclusion rules.
106 *
Tejun Heoe41e7042010-08-24 14:22:47 +0200107 * I: Modifiable by initialization/destruction paths and read-only for
108 * everyone else.
Tejun Heo4690c4a2010-06-29 10:07:10 +0200109 *
Tejun Heoe22bee72010-06-29 10:07:14 +0200110 * P: Preemption protected. Disabling preemption is enough and should
111 * only be modified and accessed from the local cpu.
112 *
Tejun Heod565ed62013-01-24 11:01:33 -0800113 * L: pool->lock protected. Access with pool->lock held.
Tejun Heo4690c4a2010-06-29 10:07:10 +0200114 *
Tejun Heod565ed62013-01-24 11:01:33 -0800115 * X: During normal operation, modification requires pool->lock and should
116 * be done only from local cpu. Either disabling preemption on local
117 * cpu or grabbing pool->lock is enough for read access. If
118 * POOL_DISASSOCIATED is set, it's identical to L.
Tejun Heoe22bee72010-06-29 10:07:14 +0200119 *
Tejun Heo73f53c42010-06-29 10:07:11 +0200120 * F: wq->flush_mutex protected.
121 *
Tejun Heo4690c4a2010-06-29 10:07:10 +0200122 * W: workqueue_lock protected.
Tejun Heo76af4d92013-03-12 11:30:00 -0700123 *
124 * R: workqueue_lock protected for writes. Sched-RCU protected for reads.
Tejun Heo75ccf592013-03-12 11:30:04 -0700125 *
126 * FR: wq->flush_mutex and workqueue_lock protected for writes. Sched-RCU
127 * protected for reads.
Tejun Heo4690c4a2010-06-29 10:07:10 +0200128 */
129
Tejun Heo2eaebdb2013-01-18 14:05:55 -0800130/* struct worker is defined in workqueue_internal.h */
Tejun Heoc34056a2010-06-29 10:07:11 +0200131
Tejun Heobd7bdd42012-07-12 14:46:37 -0700132struct worker_pool {
Tejun Heod565ed62013-01-24 11:01:33 -0800133 spinlock_t lock; /* the pool lock */
Tejun Heod84ff052013-03-12 11:29:59 -0700134 int cpu; /* I: the associated cpu */
Tejun Heo9daf9e62013-01-24 11:01:33 -0800135 int id; /* I: pool ID */
Tejun Heo11ebea52012-07-12 14:46:37 -0700136 unsigned int flags; /* X: flags */
Tejun Heobd7bdd42012-07-12 14:46:37 -0700137
138 struct list_head worklist; /* L: list of pending works */
139 int nr_workers; /* L: total number of workers */
Lai Jiangshanea1abd62012-09-18 09:59:22 -0700140
141 /* nr_idle includes the ones off idle_list for rebinding */
Tejun Heobd7bdd42012-07-12 14:46:37 -0700142 int nr_idle; /* L: currently idle ones */
143
144 struct list_head idle_list; /* X: list of idle workers */
145 struct timer_list idle_timer; /* L: worker idle timeout */
146 struct timer_list mayday_timer; /* L: SOS timer for workers */
147
Tejun Heoc5aa87b2013-03-13 16:51:36 -0700148 /* a workers is either on busy_hash or idle_list, or the manager */
Tejun Heoc9e7cf22013-01-24 11:01:33 -0800149 DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
150 /* L: hash of busy workers */
151
Tejun Heo34a06bd2013-03-12 11:30:00 -0700152 struct mutex manager_arb; /* manager arbitration */
Tejun Heo24647572013-01-24 11:01:33 -0800153 struct mutex assoc_mutex; /* protect POOL_DISASSOCIATED */
Tejun Heobd7bdd42012-07-12 14:46:37 -0700154 struct ida worker_ida; /* L: for worker IDs */
Tejun Heoe19e3972013-01-24 11:39:44 -0800155
Tejun Heo7a4e3442013-03-12 11:30:00 -0700156 struct workqueue_attrs *attrs; /* I: worker attributes */
Tejun Heoc5aa87b2013-03-13 16:51:36 -0700157 struct hlist_node hash_node; /* W: unbound_pool_hash node */
158 int refcnt; /* W: refcnt for unbound pools */
Tejun Heo7a4e3442013-03-12 11:30:00 -0700159
Tejun Heoe19e3972013-01-24 11:39:44 -0800160 /*
161 * The current concurrency level. As it's likely to be accessed
162 * from other CPUs during try_to_wake_up(), put it in a separate
163 * cacheline.
164 */
165 atomic_t nr_running ____cacheline_aligned_in_smp;
Tejun Heo29c91e92013-03-12 11:30:03 -0700166
167 /*
168 * Destruction of pool is sched-RCU protected to allow dereferences
169 * from get_work_pool().
170 */
171 struct rcu_head rcu;
Tejun Heo8b03ae32010-06-29 10:07:12 +0200172} ____cacheline_aligned_in_smp;
173
174/*
Tejun Heo112202d2013-02-13 19:29:12 -0800175 * The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
176 * of work_struct->data are used for flags and the remaining high bits
177 * point to the pwq; thus, pwqs need to be aligned at two's power of the
178 * number of flag bits.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179 */
Tejun Heo112202d2013-02-13 19:29:12 -0800180struct pool_workqueue {
Tejun Heobd7bdd42012-07-12 14:46:37 -0700181 struct worker_pool *pool; /* I: the associated pool */
Tejun Heo4690c4a2010-06-29 10:07:10 +0200182 struct workqueue_struct *wq; /* I: the owning workqueue */
Tejun Heo73f53c42010-06-29 10:07:11 +0200183 int work_color; /* L: current color */
184 int flush_color; /* L: flushing color */
Tejun Heo8864b4e2013-03-12 11:30:04 -0700185 int refcnt; /* L: reference count */
Tejun Heo73f53c42010-06-29 10:07:11 +0200186 int nr_in_flight[WORK_NR_COLORS];
187 /* L: nr of in_flight works */
Tejun Heo1e19ffc2010-06-29 10:07:12 +0200188 int nr_active; /* L: nr of active works */
Tejun Heoa0a1a5f2010-06-29 10:07:12 +0200189 int max_active; /* L: max active works */
Tejun Heo1e19ffc2010-06-29 10:07:12 +0200190 struct list_head delayed_works; /* L: delayed works */
Tejun Heo75ccf592013-03-12 11:30:04 -0700191 struct list_head pwqs_node; /* FR: node on wq->pwqs */
Tejun Heo493a1722013-03-12 11:29:59 -0700192 struct list_head mayday_node; /* W: node on wq->maydays */
Tejun Heo8864b4e2013-03-12 11:30:04 -0700193
194 /*
195 * Release of unbound pwq is punted to system_wq. See put_pwq()
196 * and pwq_unbound_release_workfn() for details. pool_workqueue
197 * itself is also sched-RCU protected so that the first pwq can be
198 * determined without grabbing workqueue_lock.
199 */
200 struct work_struct unbound_release_work;
201 struct rcu_head rcu;
Tejun Heoe904e6c2013-03-12 11:29:57 -0700202} __aligned(1 << WORK_STRUCT_FLAG_BITS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204/*
Tejun Heo73f53c42010-06-29 10:07:11 +0200205 * Structure used to wait for workqueue flush.
206 */
207struct wq_flusher {
208 struct list_head list; /* F: list of flushers */
209 int flush_color; /* F: flush color waiting for */
210 struct completion done; /* flush completion */
211};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212
Tejun Heo226223a2013-03-12 11:30:05 -0700213struct wq_device;
214
Tejun Heo73f53c42010-06-29 10:07:11 +0200215/*
Tejun Heoc5aa87b2013-03-13 16:51:36 -0700216 * The externally visible workqueue. It relays the issued work items to
217 * the appropriate worker_pool through its pool_workqueues.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218 */
219struct workqueue_struct {
Tejun Heo9c5a2ba2011-04-05 18:01:44 +0200220 unsigned int flags; /* W: WQ_* flags */
Tejun Heo420c0dd2013-03-12 11:29:59 -0700221 struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwq's */
Tejun Heo75ccf592013-03-12 11:30:04 -0700222 struct list_head pwqs; /* FR: all pwqs of this wq */
Tejun Heo4690c4a2010-06-29 10:07:10 +0200223 struct list_head list; /* W: list of all workqueues */
Tejun Heo73f53c42010-06-29 10:07:11 +0200224
225 struct mutex flush_mutex; /* protects wq flushing */
226 int work_color; /* F: current work color */
227 int flush_color; /* F: current flush color */
Tejun Heo112202d2013-02-13 19:29:12 -0800228 atomic_t nr_pwqs_to_flush; /* flush in progress */
Tejun Heo73f53c42010-06-29 10:07:11 +0200229 struct wq_flusher *first_flusher; /* F: first flusher */
230 struct list_head flusher_queue; /* F: flush waiters */
231 struct list_head flusher_overflow; /* F: flush overflow list */
232
Tejun Heo493a1722013-03-12 11:29:59 -0700233 struct list_head maydays; /* W: pwqs requesting rescue */
Tejun Heoe22bee72010-06-29 10:07:14 +0200234 struct worker *rescuer; /* I: rescue worker */
235
Tejun Heo9c5a2ba2011-04-05 18:01:44 +0200236 int nr_drainers; /* W: drain in progress */
Tejun Heo112202d2013-02-13 19:29:12 -0800237 int saved_max_active; /* W: saved pwq max_active */
Tejun Heo226223a2013-03-12 11:30:05 -0700238
239#ifdef CONFIG_SYSFS
240 struct wq_device *wq_dev; /* I: for sysfs interface */
241#endif
Johannes Berg4e6045f2007-10-18 23:39:55 -0700242#ifdef CONFIG_LOCKDEP
Tejun Heo4690c4a2010-06-29 10:07:10 +0200243 struct lockdep_map lockdep_map;
Johannes Berg4e6045f2007-10-18 23:39:55 -0700244#endif
Tejun Heob196be82012-01-10 15:11:35 -0800245 char name[]; /* I: workqueue name */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246};
247
Tejun Heoe904e6c2013-03-12 11:29:57 -0700248static struct kmem_cache *pwq_cache;
249
Tejun Heoc5aa87b2013-03-13 16:51:36 -0700250/* W: hash of all unbound pools keyed by pool->attrs */
Tejun Heo29c91e92013-03-12 11:30:03 -0700251static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
252
Tejun Heoc5aa87b2013-03-13 16:51:36 -0700253/* I: attributes used when instantiating standard unbound pools on demand */
Tejun Heo29c91e92013-03-12 11:30:03 -0700254static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
255
Tejun Heod320c032010-06-29 10:07:14 +0200256struct workqueue_struct *system_wq __read_mostly;
Tejun Heod320c032010-06-29 10:07:14 +0200257EXPORT_SYMBOL_GPL(system_wq);
Valentin Ilie044c7822012-08-19 00:52:42 +0300258struct workqueue_struct *system_highpri_wq __read_mostly;
Joonsoo Kim1aabe902012-08-15 23:25:39 +0900259EXPORT_SYMBOL_GPL(system_highpri_wq);
Valentin Ilie044c7822012-08-19 00:52:42 +0300260struct workqueue_struct *system_long_wq __read_mostly;
Tejun Heod320c032010-06-29 10:07:14 +0200261EXPORT_SYMBOL_GPL(system_long_wq);
Valentin Ilie044c7822012-08-19 00:52:42 +0300262struct workqueue_struct *system_unbound_wq __read_mostly;
Tejun Heof3421792010-07-02 10:03:51 +0200263EXPORT_SYMBOL_GPL(system_unbound_wq);
Valentin Ilie044c7822012-08-19 00:52:42 +0300264struct workqueue_struct *system_freezable_wq __read_mostly;
Tejun Heo24d51ad2011-02-21 09:52:50 +0100265EXPORT_SYMBOL_GPL(system_freezable_wq);
Tejun Heod320c032010-06-29 10:07:14 +0200266
Tejun Heo97bd2342010-10-05 10:41:14 +0200267#define CREATE_TRACE_POINTS
268#include <trace/events/workqueue.h>
269
Tejun Heo76af4d92013-03-12 11:30:00 -0700270#define assert_rcu_or_wq_lock() \
271 rcu_lockdep_assert(rcu_read_lock_sched_held() || \
272 lockdep_is_held(&workqueue_lock), \
273 "sched RCU or workqueue lock should be held")
274
Tejun Heof02ae732013-03-12 11:30:03 -0700275#define for_each_cpu_worker_pool(pool, cpu) \
276 for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
277 (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
Tejun Heo7a62c2c2013-03-12 11:30:03 -0700278 (pool)++)
Tejun Heo4ce62e92012-07-13 22:16:44 -0700279
Sasha Levinb67bfe02013-02-27 17:06:00 -0800280#define for_each_busy_worker(worker, i, pool) \
281 hash_for_each(pool->busy_hash, i, worker, hentry)
Tejun Heodb7bccf2010-06-29 10:07:12 +0200282
Tejun Heo49e3cf42013-03-12 11:29:58 -0700283/**
Tejun Heo17116962013-03-12 11:29:58 -0700284 * for_each_pool - iterate through all worker_pools in the system
285 * @pool: iteration cursor
Tejun Heo611c92a2013-03-13 16:51:36 -0700286 * @pi: integer used for iteration
Tejun Heofa1b54e2013-03-12 11:30:00 -0700287 *
288 * This must be called either with workqueue_lock held or sched RCU read
289 * locked. If the pool needs to be used beyond the locking in effect, the
290 * caller is responsible for guaranteeing that the pool stays online.
291 *
292 * The if/else clause exists only for the lockdep assertion and can be
293 * ignored.
Tejun Heo17116962013-03-12 11:29:58 -0700294 */
Tejun Heo611c92a2013-03-13 16:51:36 -0700295#define for_each_pool(pool, pi) \
296 idr_for_each_entry(&worker_pool_idr, pool, pi) \
Tejun Heofa1b54e2013-03-12 11:30:00 -0700297 if (({ assert_rcu_or_wq_lock(); false; })) { } \
298 else
Tejun Heo17116962013-03-12 11:29:58 -0700299
300/**
Tejun Heo49e3cf42013-03-12 11:29:58 -0700301 * for_each_pwq - iterate through all pool_workqueues of the specified workqueue
302 * @pwq: iteration cursor
303 * @wq: the target workqueue
Tejun Heo76af4d92013-03-12 11:30:00 -0700304 *
305 * This must be called either with workqueue_lock held or sched RCU read
306 * locked. If the pwq needs to be used beyond the locking in effect, the
307 * caller is responsible for guaranteeing that the pwq stays online.
308 *
309 * The if/else clause exists only for the lockdep assertion and can be
310 * ignored.
Tejun Heo49e3cf42013-03-12 11:29:58 -0700311 */
312#define for_each_pwq(pwq, wq) \
Tejun Heo76af4d92013-03-12 11:30:00 -0700313 list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node) \
314 if (({ assert_rcu_or_wq_lock(); false; })) { } \
315 else
Tejun Heof3421792010-07-02 10:03:51 +0200316
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900317#ifdef CONFIG_DEBUG_OBJECTS_WORK
318
319static struct debug_obj_descr work_debug_descr;
320
Stanislaw Gruszka99777282011-03-07 09:58:33 +0100321static void *work_debug_hint(void *addr)
322{
323 return ((struct work_struct *) addr)->func;
324}
325
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900326/*
327 * fixup_init is called when:
328 * - an active object is initialized
329 */
330static int work_fixup_init(void *addr, enum debug_obj_state state)
331{
332 struct work_struct *work = addr;
333
334 switch (state) {
335 case ODEBUG_STATE_ACTIVE:
336 cancel_work_sync(work);
337 debug_object_init(work, &work_debug_descr);
338 return 1;
339 default:
340 return 0;
341 }
342}
343
344/*
345 * fixup_activate is called when:
346 * - an active object is activated
347 * - an unknown object is activated (might be a statically initialized object)
348 */
349static int work_fixup_activate(void *addr, enum debug_obj_state state)
350{
351 struct work_struct *work = addr;
352
353 switch (state) {
354
355 case ODEBUG_STATE_NOTAVAILABLE:
356 /*
357 * This is not really a fixup. The work struct was
358 * statically initialized. We just make sure that it
359 * is tracked in the object tracker.
360 */
Tejun Heo22df02b2010-06-29 10:07:10 +0200361 if (test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work))) {
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900362 debug_object_init(work, &work_debug_descr);
363 debug_object_activate(work, &work_debug_descr);
364 return 0;
365 }
366 WARN_ON_ONCE(1);
367 return 0;
368
369 case ODEBUG_STATE_ACTIVE:
370 WARN_ON(1);
371
372 default:
373 return 0;
374 }
375}
376
377/*
378 * fixup_free is called when:
379 * - an active object is freed
380 */
381static int work_fixup_free(void *addr, enum debug_obj_state state)
382{
383 struct work_struct *work = addr;
384
385 switch (state) {
386 case ODEBUG_STATE_ACTIVE:
387 cancel_work_sync(work);
388 debug_object_free(work, &work_debug_descr);
389 return 1;
390 default:
391 return 0;
392 }
393}
394
395static struct debug_obj_descr work_debug_descr = {
396 .name = "work_struct",
Stanislaw Gruszka99777282011-03-07 09:58:33 +0100397 .debug_hint = work_debug_hint,
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900398 .fixup_init = work_fixup_init,
399 .fixup_activate = work_fixup_activate,
400 .fixup_free = work_fixup_free,
401};
402
403static inline void debug_work_activate(struct work_struct *work)
404{
405 debug_object_activate(work, &work_debug_descr);
406}
407
408static inline void debug_work_deactivate(struct work_struct *work)
409{
410 debug_object_deactivate(work, &work_debug_descr);
411}
412
413void __init_work(struct work_struct *work, int onstack)
414{
415 if (onstack)
416 debug_object_init_on_stack(work, &work_debug_descr);
417 else
418 debug_object_init(work, &work_debug_descr);
419}
420EXPORT_SYMBOL_GPL(__init_work);
421
422void destroy_work_on_stack(struct work_struct *work)
423{
424 debug_object_free(work, &work_debug_descr);
425}
426EXPORT_SYMBOL_GPL(destroy_work_on_stack);
427
428#else
429static inline void debug_work_activate(struct work_struct *work) { }
430static inline void debug_work_deactivate(struct work_struct *work) { }
431#endif
432
Gautham R Shenoy95402b32008-01-25 21:08:02 +0100433/* Serializes the accesses to the list of workqueues. */
434static DEFINE_SPINLOCK(workqueue_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435static LIST_HEAD(workqueues);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +0200436static bool workqueue_freezing; /* W: have wqs started freezing? */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437
Tejun Heoc5aa87b2013-03-13 16:51:36 -0700438/* the per-cpu worker pools */
Tejun Heoe19e3972013-01-24 11:39:44 -0800439static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
Tejun Heof02ae732013-03-12 11:30:03 -0700440 cpu_worker_pools);
Tejun Heof3421792010-07-02 10:03:51 +0200441
Tejun Heofa1b54e2013-03-12 11:30:00 -0700442/*
Tejun Heoc5aa87b2013-03-13 16:51:36 -0700443 * R: idr of all pools. Modifications are protected by workqueue_lock.
444 * Read accesses are protected by sched-RCU protected.
Tejun Heofa1b54e2013-03-12 11:30:00 -0700445 */
Tejun Heo9daf9e62013-01-24 11:01:33 -0800446static DEFINE_IDR(worker_pool_idr);
447
Tejun Heoc34056a2010-06-29 10:07:11 +0200448static int worker_thread(void *__worker);
Tejun Heo226223a2013-03-12 11:30:05 -0700449static void copy_workqueue_attrs(struct workqueue_attrs *to,
450 const struct workqueue_attrs *from);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700451
Tejun Heo9daf9e62013-01-24 11:01:33 -0800452/* allocate ID and assign it to @pool */
453static int worker_pool_assign_id(struct worker_pool *pool)
454{
455 int ret;
456
Tejun Heofa1b54e2013-03-12 11:30:00 -0700457 do {
458 if (!idr_pre_get(&worker_pool_idr, GFP_KERNEL))
459 return -ENOMEM;
460
461 spin_lock_irq(&workqueue_lock);
462 ret = idr_get_new(&worker_pool_idr, pool, &pool->id);
463 spin_unlock_irq(&workqueue_lock);
464 } while (ret == -EAGAIN);
Tejun Heo9daf9e62013-01-24 11:01:33 -0800465
466 return ret;
467}
468
Tejun Heo76af4d92013-03-12 11:30:00 -0700469/**
470 * first_pwq - return the first pool_workqueue of the specified workqueue
471 * @wq: the target workqueue
472 *
473 * This must be called either with workqueue_lock held or sched RCU read
474 * locked. If the pwq needs to be used beyond the locking in effect, the
475 * caller is responsible for guaranteeing that the pwq stays online.
476 */
Tejun Heo7fb98ea2013-03-12 11:30:00 -0700477static struct pool_workqueue *first_pwq(struct workqueue_struct *wq)
Oleg Nesterova848e3b2007-05-09 02:34:17 -0700478{
Tejun Heo76af4d92013-03-12 11:30:00 -0700479 assert_rcu_or_wq_lock();
480 return list_first_or_null_rcu(&wq->pwqs, struct pool_workqueue,
481 pwqs_node);
Oleg Nesterova848e3b2007-05-09 02:34:17 -0700482}
483
Tejun Heo73f53c42010-06-29 10:07:11 +0200484static unsigned int work_color_to_flags(int color)
485{
486 return color << WORK_STRUCT_COLOR_SHIFT;
487}
488
489static int get_work_color(struct work_struct *work)
490{
491 return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &
492 ((1 << WORK_STRUCT_COLOR_BITS) - 1);
493}
494
495static int work_next_color(int color)
496{
497 return (color + 1) % WORK_NR_COLORS;
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -0700498}
499
David Howells4594bf12006-12-07 11:33:26 +0000500/*
Tejun Heo112202d2013-02-13 19:29:12 -0800501 * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
502 * contain the pointer to the queued pwq. Once execution starts, the flag
Tejun Heo7c3eed52013-01-24 11:01:33 -0800503 * is cleared and the high bits contain OFFQ flags and pool ID.
Tejun Heo7a22ad72010-06-29 10:07:13 +0200504 *
Tejun Heo112202d2013-02-13 19:29:12 -0800505 * set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling()
506 * and clear_work_data() can be used to set the pwq, pool or clear
Tejun Heobbb68df2012-08-03 10:30:46 -0700507 * work->data. These functions should only be called while the work is
508 * owned - ie. while the PENDING bit is set.
Tejun Heo7a22ad72010-06-29 10:07:13 +0200509 *
Tejun Heo112202d2013-02-13 19:29:12 -0800510 * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
Tejun Heo7c3eed52013-01-24 11:01:33 -0800511 * corresponding to a work. Pool is available once the work has been
Tejun Heo112202d2013-02-13 19:29:12 -0800512 * queued anywhere after initialization until it is sync canceled. pwq is
Tejun Heo7c3eed52013-01-24 11:01:33 -0800513 * available only while the work item is queued.
Tejun Heobbb68df2012-08-03 10:30:46 -0700514 *
515 * %WORK_OFFQ_CANCELING is used to mark a work item which is being
516 * canceled. While being canceled, a work item may have its PENDING set
517 * but stay off timer and worklist for arbitrarily long and nobody should
518 * try to steal the PENDING bit.
David Howells4594bf12006-12-07 11:33:26 +0000519 */
Tejun Heo7a22ad72010-06-29 10:07:13 +0200520static inline void set_work_data(struct work_struct *work, unsigned long data,
521 unsigned long flags)
David Howells365970a2006-11-22 14:54:49 +0000522{
Tejun Heo6183c002013-03-12 11:29:57 -0700523 WARN_ON_ONCE(!work_pending(work));
Tejun Heo7a22ad72010-06-29 10:07:13 +0200524 atomic_long_set(&work->data, data | flags | work_static(work));
David Howells365970a2006-11-22 14:54:49 +0000525}
David Howells365970a2006-11-22 14:54:49 +0000526
Tejun Heo112202d2013-02-13 19:29:12 -0800527static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,
Tejun Heo7a22ad72010-06-29 10:07:13 +0200528 unsigned long extra_flags)
Oleg Nesterov4d707b92010-04-23 17:40:40 +0200529{
Tejun Heo112202d2013-02-13 19:29:12 -0800530 set_work_data(work, (unsigned long)pwq,
531 WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | extra_flags);
Oleg Nesterov4d707b92010-04-23 17:40:40 +0200532}
533
Lai Jiangshan4468a002013-02-06 18:04:53 -0800534static void set_work_pool_and_keep_pending(struct work_struct *work,
535 int pool_id)
536{
537 set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,
538 WORK_STRUCT_PENDING);
539}
540
Tejun Heo7c3eed52013-01-24 11:01:33 -0800541static void set_work_pool_and_clear_pending(struct work_struct *work,
542 int pool_id)
David Howells365970a2006-11-22 14:54:49 +0000543{
Tejun Heo23657bb2012-08-13 17:08:19 -0700544 /*
545 * The following wmb is paired with the implied mb in
546 * test_and_set_bit(PENDING) and ensures all updates to @work made
547 * here are visible to and precede any updates by the next PENDING
548 * owner.
549 */
550 smp_wmb();
Tejun Heo7c3eed52013-01-24 11:01:33 -0800551 set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
Tejun Heo7a22ad72010-06-29 10:07:13 +0200552}
553
554static void clear_work_data(struct work_struct *work)
555{
Tejun Heo7c3eed52013-01-24 11:01:33 -0800556 smp_wmb(); /* see set_work_pool_and_clear_pending() */
557 set_work_data(work, WORK_STRUCT_NO_POOL, 0);
Tejun Heo7a22ad72010-06-29 10:07:13 +0200558}
559
Tejun Heo112202d2013-02-13 19:29:12 -0800560static struct pool_workqueue *get_work_pwq(struct work_struct *work)
Tejun Heo7a22ad72010-06-29 10:07:13 +0200561{
Tejun Heoe1201532010-07-22 14:14:25 +0200562 unsigned long data = atomic_long_read(&work->data);
Tejun Heo7a22ad72010-06-29 10:07:13 +0200563
Tejun Heo112202d2013-02-13 19:29:12 -0800564 if (data & WORK_STRUCT_PWQ)
Tejun Heoe1201532010-07-22 14:14:25 +0200565 return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
566 else
567 return NULL;
Tejun Heo7a22ad72010-06-29 10:07:13 +0200568}
569
Tejun Heo7c3eed52013-01-24 11:01:33 -0800570/**
571 * get_work_pool - return the worker_pool a given work was associated with
572 * @work: the work item of interest
573 *
574 * Return the worker_pool @work was last associated with. %NULL if none.
Tejun Heofa1b54e2013-03-12 11:30:00 -0700575 *
576 * Pools are created and destroyed under workqueue_lock, and allows read
577 * access under sched-RCU read lock. As such, this function should be
578 * called under workqueue_lock or with preemption disabled.
579 *
580 * All fields of the returned pool are accessible as long as the above
581 * mentioned locking is in effect. If the returned pool needs to be used
582 * beyond the critical section, the caller is responsible for ensuring the
583 * returned pool is and stays online.
Tejun Heo7c3eed52013-01-24 11:01:33 -0800584 */
585static struct worker_pool *get_work_pool(struct work_struct *work)
Tejun Heo7a22ad72010-06-29 10:07:13 +0200586{
Tejun Heoe1201532010-07-22 14:14:25 +0200587 unsigned long data = atomic_long_read(&work->data);
Tejun Heo7c3eed52013-01-24 11:01:33 -0800588 int pool_id;
Tejun Heo7a22ad72010-06-29 10:07:13 +0200589
Tejun Heofa1b54e2013-03-12 11:30:00 -0700590 assert_rcu_or_wq_lock();
591
Tejun Heo112202d2013-02-13 19:29:12 -0800592 if (data & WORK_STRUCT_PWQ)
593 return ((struct pool_workqueue *)
Tejun Heo7c3eed52013-01-24 11:01:33 -0800594 (data & WORK_STRUCT_WQ_DATA_MASK))->pool;
Tejun Heo7a22ad72010-06-29 10:07:13 +0200595
Tejun Heo7c3eed52013-01-24 11:01:33 -0800596 pool_id = data >> WORK_OFFQ_POOL_SHIFT;
597 if (pool_id == WORK_OFFQ_POOL_NONE)
Tejun Heo7a22ad72010-06-29 10:07:13 +0200598 return NULL;
599
Tejun Heofa1b54e2013-03-12 11:30:00 -0700600 return idr_find(&worker_pool_idr, pool_id);
Tejun Heo7c3eed52013-01-24 11:01:33 -0800601}
602
603/**
604 * get_work_pool_id - return the worker pool ID a given work is associated with
605 * @work: the work item of interest
606 *
607 * Return the worker_pool ID @work was last associated with.
608 * %WORK_OFFQ_POOL_NONE if none.
609 */
610static int get_work_pool_id(struct work_struct *work)
611{
Lai Jiangshan54d5b7d2013-02-07 13:14:20 -0800612 unsigned long data = atomic_long_read(&work->data);
Tejun Heo7c3eed52013-01-24 11:01:33 -0800613
Tejun Heo112202d2013-02-13 19:29:12 -0800614 if (data & WORK_STRUCT_PWQ)
615 return ((struct pool_workqueue *)
Lai Jiangshan54d5b7d2013-02-07 13:14:20 -0800616 (data & WORK_STRUCT_WQ_DATA_MASK))->pool->id;
617
618 return data >> WORK_OFFQ_POOL_SHIFT;
Tejun Heo7c3eed52013-01-24 11:01:33 -0800619}
620
Tejun Heobbb68df2012-08-03 10:30:46 -0700621static void mark_work_canceling(struct work_struct *work)
622{
Tejun Heo7c3eed52013-01-24 11:01:33 -0800623 unsigned long pool_id = get_work_pool_id(work);
Tejun Heobbb68df2012-08-03 10:30:46 -0700624
Tejun Heo7c3eed52013-01-24 11:01:33 -0800625 pool_id <<= WORK_OFFQ_POOL_SHIFT;
626 set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
Tejun Heobbb68df2012-08-03 10:30:46 -0700627}
628
629static bool work_is_canceling(struct work_struct *work)
630{
631 unsigned long data = atomic_long_read(&work->data);
632
Tejun Heo112202d2013-02-13 19:29:12 -0800633 return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
Tejun Heobbb68df2012-08-03 10:30:46 -0700634}
635
David Howells365970a2006-11-22 14:54:49 +0000636/*
Tejun Heo32704762012-07-13 22:16:45 -0700637 * Policy functions. These define the policies on how the global worker
638 * pools are managed. Unless noted otherwise, these functions assume that
Tejun Heod565ed62013-01-24 11:01:33 -0800639 * they're being called with pool->lock held.
David Howells365970a2006-11-22 14:54:49 +0000640 */
Tejun Heoe22bee72010-06-29 10:07:14 +0200641
Tejun Heo63d95a92012-07-12 14:46:37 -0700642static bool __need_more_worker(struct worker_pool *pool)
David Howells365970a2006-11-22 14:54:49 +0000643{
Tejun Heoe19e3972013-01-24 11:39:44 -0800644 return !atomic_read(&pool->nr_running);
David Howells365970a2006-11-22 14:54:49 +0000645}
646
Tejun Heoe22bee72010-06-29 10:07:14 +0200647/*
648 * Need to wake up a worker? Called from anything but currently
649 * running workers.
Tejun Heo974271c2012-07-12 14:46:37 -0700650 *
651 * Note that, because unbound workers never contribute to nr_running, this
Tejun Heo706026c2013-01-24 11:01:34 -0800652 * function will always return %true for unbound pools as long as the
Tejun Heo974271c2012-07-12 14:46:37 -0700653 * worklist isn't empty.
Tejun Heoe22bee72010-06-29 10:07:14 +0200654 */
Tejun Heo63d95a92012-07-12 14:46:37 -0700655static bool need_more_worker(struct worker_pool *pool)
David Howells365970a2006-11-22 14:54:49 +0000656{
Tejun Heo63d95a92012-07-12 14:46:37 -0700657 return !list_empty(&pool->worklist) && __need_more_worker(pool);
David Howells365970a2006-11-22 14:54:49 +0000658}
659
Tejun Heoe22bee72010-06-29 10:07:14 +0200660/* Can I start working? Called from busy but !running workers. */
Tejun Heo63d95a92012-07-12 14:46:37 -0700661static bool may_start_working(struct worker_pool *pool)
Tejun Heoe22bee72010-06-29 10:07:14 +0200662{
Tejun Heo63d95a92012-07-12 14:46:37 -0700663 return pool->nr_idle;
Tejun Heoe22bee72010-06-29 10:07:14 +0200664}
665
666/* Do I need to keep working? Called from currently running workers. */
Tejun Heo63d95a92012-07-12 14:46:37 -0700667static bool keep_working(struct worker_pool *pool)
Tejun Heoe22bee72010-06-29 10:07:14 +0200668{
Tejun Heoe19e3972013-01-24 11:39:44 -0800669 return !list_empty(&pool->worklist) &&
670 atomic_read(&pool->nr_running) <= 1;
Tejun Heoe22bee72010-06-29 10:07:14 +0200671}
672
673/* Do we need a new worker? Called from manager. */
Tejun Heo63d95a92012-07-12 14:46:37 -0700674static bool need_to_create_worker(struct worker_pool *pool)
Tejun Heoe22bee72010-06-29 10:07:14 +0200675{
Tejun Heo63d95a92012-07-12 14:46:37 -0700676 return need_more_worker(pool) && !may_start_working(pool);
Tejun Heoe22bee72010-06-29 10:07:14 +0200677}
678
679/* Do I need to be the manager? */
Tejun Heo63d95a92012-07-12 14:46:37 -0700680static bool need_to_manage_workers(struct worker_pool *pool)
Tejun Heoe22bee72010-06-29 10:07:14 +0200681{
Tejun Heo63d95a92012-07-12 14:46:37 -0700682 return need_to_create_worker(pool) ||
Tejun Heo11ebea52012-07-12 14:46:37 -0700683 (pool->flags & POOL_MANAGE_WORKERS);
Tejun Heoe22bee72010-06-29 10:07:14 +0200684}
685
686/* Do we have too many workers and should some go away? */
Tejun Heo63d95a92012-07-12 14:46:37 -0700687static bool too_many_workers(struct worker_pool *pool)
Tejun Heoe22bee72010-06-29 10:07:14 +0200688{
Tejun Heo34a06bd2013-03-12 11:30:00 -0700689 bool managing = mutex_is_locked(&pool->manager_arb);
Tejun Heo63d95a92012-07-12 14:46:37 -0700690 int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
691 int nr_busy = pool->nr_workers - nr_idle;
Tejun Heoe22bee72010-06-29 10:07:14 +0200692
Lai Jiangshanea1abd62012-09-18 09:59:22 -0700693 /*
694 * nr_idle and idle_list may disagree if idle rebinding is in
695 * progress. Never return %true if idle_list is empty.
696 */
697 if (list_empty(&pool->idle_list))
698 return false;
699
Tejun Heoe22bee72010-06-29 10:07:14 +0200700 return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
701}
702
703/*
704 * Wake up functions.
705 */
706
Tejun Heo7e116292010-06-29 10:07:13 +0200707/* Return the first worker. Safe with preemption disabled */
Tejun Heo63d95a92012-07-12 14:46:37 -0700708static struct worker *first_worker(struct worker_pool *pool)
Tejun Heo7e116292010-06-29 10:07:13 +0200709{
Tejun Heo63d95a92012-07-12 14:46:37 -0700710 if (unlikely(list_empty(&pool->idle_list)))
Tejun Heo7e116292010-06-29 10:07:13 +0200711 return NULL;
712
Tejun Heo63d95a92012-07-12 14:46:37 -0700713 return list_first_entry(&pool->idle_list, struct worker, entry);
Tejun Heo7e116292010-06-29 10:07:13 +0200714}
715
716/**
717 * wake_up_worker - wake up an idle worker
Tejun Heo63d95a92012-07-12 14:46:37 -0700718 * @pool: worker pool to wake worker from
Tejun Heo7e116292010-06-29 10:07:13 +0200719 *
Tejun Heo63d95a92012-07-12 14:46:37 -0700720 * Wake up the first idle worker of @pool.
Tejun Heo7e116292010-06-29 10:07:13 +0200721 *
722 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -0800723 * spin_lock_irq(pool->lock).
Tejun Heo7e116292010-06-29 10:07:13 +0200724 */
Tejun Heo63d95a92012-07-12 14:46:37 -0700725static void wake_up_worker(struct worker_pool *pool)
Tejun Heo7e116292010-06-29 10:07:13 +0200726{
Tejun Heo63d95a92012-07-12 14:46:37 -0700727 struct worker *worker = first_worker(pool);
Tejun Heo7e116292010-06-29 10:07:13 +0200728
729 if (likely(worker))
730 wake_up_process(worker->task);
731}
732
Tejun Heo4690c4a2010-06-29 10:07:10 +0200733/**
Tejun Heoe22bee72010-06-29 10:07:14 +0200734 * wq_worker_waking_up - a worker is waking up
735 * @task: task waking up
736 * @cpu: CPU @task is waking up to
737 *
738 * This function is called during try_to_wake_up() when a worker is
739 * being awoken.
740 *
741 * CONTEXT:
742 * spin_lock_irq(rq->lock)
743 */
Tejun Heod84ff052013-03-12 11:29:59 -0700744void wq_worker_waking_up(struct task_struct *task, int cpu)
Tejun Heoe22bee72010-06-29 10:07:14 +0200745{
746 struct worker *worker = kthread_data(task);
747
Joonsoo Kim36576002012-10-26 23:03:49 +0900748 if (!(worker->flags & WORKER_NOT_RUNNING)) {
Tejun Heoec22ca52013-01-24 11:01:33 -0800749 WARN_ON_ONCE(worker->pool->cpu != cpu);
Tejun Heoe19e3972013-01-24 11:39:44 -0800750 atomic_inc(&worker->pool->nr_running);
Joonsoo Kim36576002012-10-26 23:03:49 +0900751 }
Tejun Heoe22bee72010-06-29 10:07:14 +0200752}
753
754/**
755 * wq_worker_sleeping - a worker is going to sleep
756 * @task: task going to sleep
757 * @cpu: CPU in question, must be the current CPU number
758 *
759 * This function is called during schedule() when a busy worker is
760 * going to sleep. Worker on the same cpu can be woken up by
761 * returning pointer to its task.
762 *
763 * CONTEXT:
764 * spin_lock_irq(rq->lock)
765 *
766 * RETURNS:
767 * Worker task on @cpu to wake up, %NULL if none.
768 */
Tejun Heod84ff052013-03-12 11:29:59 -0700769struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu)
Tejun Heoe22bee72010-06-29 10:07:14 +0200770{
771 struct worker *worker = kthread_data(task), *to_wakeup = NULL;
Tejun Heo111c2252013-01-17 17:16:24 -0800772 struct worker_pool *pool;
Tejun Heoe22bee72010-06-29 10:07:14 +0200773
Tejun Heo111c2252013-01-17 17:16:24 -0800774 /*
775 * Rescuers, which may not have all the fields set up like normal
776 * workers, also reach here, let's not access anything before
777 * checking NOT_RUNNING.
778 */
Steven Rostedt2d646722010-12-03 23:12:33 -0500779 if (worker->flags & WORKER_NOT_RUNNING)
Tejun Heoe22bee72010-06-29 10:07:14 +0200780 return NULL;
781
Tejun Heo111c2252013-01-17 17:16:24 -0800782 pool = worker->pool;
Tejun Heo111c2252013-01-17 17:16:24 -0800783
Tejun Heoe22bee72010-06-29 10:07:14 +0200784 /* this can only happen on the local cpu */
Tejun Heo6183c002013-03-12 11:29:57 -0700785 if (WARN_ON_ONCE(cpu != raw_smp_processor_id()))
786 return NULL;
Tejun Heoe22bee72010-06-29 10:07:14 +0200787
788 /*
789 * The counterpart of the following dec_and_test, implied mb,
790 * worklist not empty test sequence is in insert_work().
791 * Please read comment there.
792 *
Tejun Heo628c78e2012-07-17 12:39:27 -0700793 * NOT_RUNNING is clear. This means that we're bound to and
794 * running on the local cpu w/ rq lock held and preemption
795 * disabled, which in turn means that none else could be
Tejun Heod565ed62013-01-24 11:01:33 -0800796 * manipulating idle_list, so dereferencing idle_list without pool
Tejun Heo628c78e2012-07-17 12:39:27 -0700797 * lock is safe.
Tejun Heoe22bee72010-06-29 10:07:14 +0200798 */
Tejun Heoe19e3972013-01-24 11:39:44 -0800799 if (atomic_dec_and_test(&pool->nr_running) &&
800 !list_empty(&pool->worklist))
Tejun Heo63d95a92012-07-12 14:46:37 -0700801 to_wakeup = first_worker(pool);
Tejun Heoe22bee72010-06-29 10:07:14 +0200802 return to_wakeup ? to_wakeup->task : NULL;
803}
804
805/**
806 * worker_set_flags - set worker flags and adjust nr_running accordingly
Tejun Heocb444762010-07-02 10:03:50 +0200807 * @worker: self
Tejun Heod302f012010-06-29 10:07:13 +0200808 * @flags: flags to set
809 * @wakeup: wakeup an idle worker if necessary
810 *
Tejun Heoe22bee72010-06-29 10:07:14 +0200811 * Set @flags in @worker->flags and adjust nr_running accordingly. If
812 * nr_running becomes zero and @wakeup is %true, an idle worker is
813 * woken up.
Tejun Heod302f012010-06-29 10:07:13 +0200814 *
Tejun Heocb444762010-07-02 10:03:50 +0200815 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -0800816 * spin_lock_irq(pool->lock)
Tejun Heod302f012010-06-29 10:07:13 +0200817 */
818static inline void worker_set_flags(struct worker *worker, unsigned int flags,
819 bool wakeup)
820{
Tejun Heobd7bdd42012-07-12 14:46:37 -0700821 struct worker_pool *pool = worker->pool;
Tejun Heoe22bee72010-06-29 10:07:14 +0200822
Tejun Heocb444762010-07-02 10:03:50 +0200823 WARN_ON_ONCE(worker->task != current);
824
Tejun Heoe22bee72010-06-29 10:07:14 +0200825 /*
826 * If transitioning into NOT_RUNNING, adjust nr_running and
827 * wake up an idle worker as necessary if requested by
828 * @wakeup.
829 */
830 if ((flags & WORKER_NOT_RUNNING) &&
831 !(worker->flags & WORKER_NOT_RUNNING)) {
Tejun Heoe22bee72010-06-29 10:07:14 +0200832 if (wakeup) {
Tejun Heoe19e3972013-01-24 11:39:44 -0800833 if (atomic_dec_and_test(&pool->nr_running) &&
Tejun Heobd7bdd42012-07-12 14:46:37 -0700834 !list_empty(&pool->worklist))
Tejun Heo63d95a92012-07-12 14:46:37 -0700835 wake_up_worker(pool);
Tejun Heoe22bee72010-06-29 10:07:14 +0200836 } else
Tejun Heoe19e3972013-01-24 11:39:44 -0800837 atomic_dec(&pool->nr_running);
Tejun Heoe22bee72010-06-29 10:07:14 +0200838 }
839
Tejun Heod302f012010-06-29 10:07:13 +0200840 worker->flags |= flags;
841}
842
843/**
Tejun Heoe22bee72010-06-29 10:07:14 +0200844 * worker_clr_flags - clear worker flags and adjust nr_running accordingly
Tejun Heocb444762010-07-02 10:03:50 +0200845 * @worker: self
Tejun Heod302f012010-06-29 10:07:13 +0200846 * @flags: flags to clear
847 *
Tejun Heoe22bee72010-06-29 10:07:14 +0200848 * Clear @flags in @worker->flags and adjust nr_running accordingly.
Tejun Heod302f012010-06-29 10:07:13 +0200849 *
Tejun Heocb444762010-07-02 10:03:50 +0200850 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -0800851 * spin_lock_irq(pool->lock)
Tejun Heod302f012010-06-29 10:07:13 +0200852 */
853static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
854{
Tejun Heo63d95a92012-07-12 14:46:37 -0700855 struct worker_pool *pool = worker->pool;
Tejun Heoe22bee72010-06-29 10:07:14 +0200856 unsigned int oflags = worker->flags;
857
Tejun Heocb444762010-07-02 10:03:50 +0200858 WARN_ON_ONCE(worker->task != current);
859
Tejun Heod302f012010-06-29 10:07:13 +0200860 worker->flags &= ~flags;
Tejun Heoe22bee72010-06-29 10:07:14 +0200861
Tejun Heo42c025f2011-01-11 15:58:49 +0100862 /*
863 * If transitioning out of NOT_RUNNING, increment nr_running. Note
864 * that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
865 * of multiple flags, not a single flag.
866 */
Tejun Heoe22bee72010-06-29 10:07:14 +0200867 if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
868 if (!(worker->flags & WORKER_NOT_RUNNING))
Tejun Heoe19e3972013-01-24 11:39:44 -0800869 atomic_inc(&pool->nr_running);
Tejun Heod302f012010-06-29 10:07:13 +0200870}
871
872/**
Tejun Heo8cca0ee2010-06-29 10:07:13 +0200873 * find_worker_executing_work - find worker which is executing a work
Tejun Heoc9e7cf22013-01-24 11:01:33 -0800874 * @pool: pool of interest
Tejun Heo8cca0ee2010-06-29 10:07:13 +0200875 * @work: work to find worker for
876 *
Tejun Heoc9e7cf22013-01-24 11:01:33 -0800877 * Find a worker which is executing @work on @pool by searching
878 * @pool->busy_hash which is keyed by the address of @work. For a worker
Tejun Heoa2c1c572012-12-18 10:35:02 -0800879 * to match, its current execution should match the address of @work and
880 * its work function. This is to avoid unwanted dependency between
881 * unrelated work executions through a work item being recycled while still
882 * being executed.
883 *
884 * This is a bit tricky. A work item may be freed once its execution
885 * starts and nothing prevents the freed area from being recycled for
886 * another work item. If the same work item address ends up being reused
887 * before the original execution finishes, workqueue will identify the
888 * recycled work item as currently executing and make it wait until the
889 * current execution finishes, introducing an unwanted dependency.
890 *
Tejun Heoc5aa87b2013-03-13 16:51:36 -0700891 * This function checks the work item address and work function to avoid
892 * false positives. Note that this isn't complete as one may construct a
893 * work function which can introduce dependency onto itself through a
894 * recycled work item. Well, if somebody wants to shoot oneself in the
895 * foot that badly, there's only so much we can do, and if such deadlock
896 * actually occurs, it should be easy to locate the culprit work function.
Tejun Heo8cca0ee2010-06-29 10:07:13 +0200897 *
898 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -0800899 * spin_lock_irq(pool->lock).
Tejun Heo8cca0ee2010-06-29 10:07:13 +0200900 *
901 * RETURNS:
902 * Pointer to worker which is executing @work if found, NULL
903 * otherwise.
904 */
Tejun Heoc9e7cf22013-01-24 11:01:33 -0800905static struct worker *find_worker_executing_work(struct worker_pool *pool,
Tejun Heo8cca0ee2010-06-29 10:07:13 +0200906 struct work_struct *work)
907{
Sasha Levin42f85702012-12-17 10:01:23 -0500908 struct worker *worker;
Sasha Levin42f85702012-12-17 10:01:23 -0500909
Sasha Levinb67bfe02013-02-27 17:06:00 -0800910 hash_for_each_possible(pool->busy_hash, worker, hentry,
Tejun Heoa2c1c572012-12-18 10:35:02 -0800911 (unsigned long)work)
912 if (worker->current_work == work &&
913 worker->current_func == work->func)
Sasha Levin42f85702012-12-17 10:01:23 -0500914 return worker;
915
916 return NULL;
Tejun Heo8cca0ee2010-06-29 10:07:13 +0200917}
918
919/**
Tejun Heobf4ede02012-08-03 10:30:46 -0700920 * move_linked_works - move linked works to a list
921 * @work: start of series of works to be scheduled
922 * @head: target list to append @work to
923 * @nextp: out paramter for nested worklist walking
924 *
925 * Schedule linked works starting from @work to @head. Work series to
926 * be scheduled starts at @work and includes any consecutive work with
927 * WORK_STRUCT_LINKED set in its predecessor.
928 *
929 * If @nextp is not NULL, it's updated to point to the next work of
930 * the last scheduled work. This allows move_linked_works() to be
931 * nested inside outer list_for_each_entry_safe().
932 *
933 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -0800934 * spin_lock_irq(pool->lock).
Tejun Heobf4ede02012-08-03 10:30:46 -0700935 */
936static void move_linked_works(struct work_struct *work, struct list_head *head,
937 struct work_struct **nextp)
938{
939 struct work_struct *n;
940
941 /*
942 * Linked worklist will always end before the end of the list,
943 * use NULL for list head.
944 */
945 list_for_each_entry_safe_from(work, n, NULL, entry) {
946 list_move_tail(&work->entry, head);
947 if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
948 break;
949 }
950
951 /*
952 * If we're already inside safe list traversal and have moved
953 * multiple works to the scheduled queue, the next position
954 * needs to be updated.
955 */
956 if (nextp)
957 *nextp = n;
958}
959
Tejun Heo8864b4e2013-03-12 11:30:04 -0700960/**
961 * get_pwq - get an extra reference on the specified pool_workqueue
962 * @pwq: pool_workqueue to get
963 *
964 * Obtain an extra reference on @pwq. The caller should guarantee that
965 * @pwq has positive refcnt and be holding the matching pool->lock.
966 */
967static void get_pwq(struct pool_workqueue *pwq)
968{
969 lockdep_assert_held(&pwq->pool->lock);
970 WARN_ON_ONCE(pwq->refcnt <= 0);
971 pwq->refcnt++;
972}
973
974/**
975 * put_pwq - put a pool_workqueue reference
976 * @pwq: pool_workqueue to put
977 *
978 * Drop a reference of @pwq. If its refcnt reaches zero, schedule its
979 * destruction. The caller should be holding the matching pool->lock.
980 */
981static void put_pwq(struct pool_workqueue *pwq)
982{
983 lockdep_assert_held(&pwq->pool->lock);
984 if (likely(--pwq->refcnt))
985 return;
986 if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
987 return;
988 /*
989 * @pwq can't be released under pool->lock, bounce to
990 * pwq_unbound_release_workfn(). This never recurses on the same
991 * pool->lock as this path is taken only for unbound workqueues and
992 * the release work item is scheduled on a per-cpu workqueue. To
993 * avoid lockdep warning, unbound pool->locks are given lockdep
994 * subclass of 1 in get_unbound_pool().
995 */
996 schedule_work(&pwq->unbound_release_work);
997}
998
Tejun Heo112202d2013-02-13 19:29:12 -0800999static void pwq_activate_delayed_work(struct work_struct *work)
Tejun Heobf4ede02012-08-03 10:30:46 -07001000{
Tejun Heo112202d2013-02-13 19:29:12 -08001001 struct pool_workqueue *pwq = get_work_pwq(work);
Tejun Heobf4ede02012-08-03 10:30:46 -07001002
1003 trace_workqueue_activate_work(work);
Tejun Heo112202d2013-02-13 19:29:12 -08001004 move_linked_works(work, &pwq->pool->worklist, NULL);
Tejun Heobf4ede02012-08-03 10:30:46 -07001005 __clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work));
Tejun Heo112202d2013-02-13 19:29:12 -08001006 pwq->nr_active++;
Tejun Heobf4ede02012-08-03 10:30:46 -07001007}
1008
Tejun Heo112202d2013-02-13 19:29:12 -08001009static void pwq_activate_first_delayed(struct pool_workqueue *pwq)
Lai Jiangshan3aa62492012-09-18 10:40:00 -07001010{
Tejun Heo112202d2013-02-13 19:29:12 -08001011 struct work_struct *work = list_first_entry(&pwq->delayed_works,
Lai Jiangshan3aa62492012-09-18 10:40:00 -07001012 struct work_struct, entry);
1013
Tejun Heo112202d2013-02-13 19:29:12 -08001014 pwq_activate_delayed_work(work);
Lai Jiangshan3aa62492012-09-18 10:40:00 -07001015}
1016
Tejun Heobf4ede02012-08-03 10:30:46 -07001017/**
Tejun Heo112202d2013-02-13 19:29:12 -08001018 * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
1019 * @pwq: pwq of interest
Tejun Heobf4ede02012-08-03 10:30:46 -07001020 * @color: color of work which left the queue
Tejun Heobf4ede02012-08-03 10:30:46 -07001021 *
1022 * A work either has completed or is removed from pending queue,
Tejun Heo112202d2013-02-13 19:29:12 -08001023 * decrement nr_in_flight of its pwq and handle workqueue flushing.
Tejun Heobf4ede02012-08-03 10:30:46 -07001024 *
1025 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08001026 * spin_lock_irq(pool->lock).
Tejun Heobf4ede02012-08-03 10:30:46 -07001027 */
Tejun Heo112202d2013-02-13 19:29:12 -08001028static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
Tejun Heobf4ede02012-08-03 10:30:46 -07001029{
Tejun Heo8864b4e2013-03-12 11:30:04 -07001030 /* uncolored work items don't participate in flushing or nr_active */
Tejun Heobf4ede02012-08-03 10:30:46 -07001031 if (color == WORK_NO_COLOR)
Tejun Heo8864b4e2013-03-12 11:30:04 -07001032 goto out_put;
Tejun Heobf4ede02012-08-03 10:30:46 -07001033
Tejun Heo112202d2013-02-13 19:29:12 -08001034 pwq->nr_in_flight[color]--;
Tejun Heobf4ede02012-08-03 10:30:46 -07001035
Tejun Heo112202d2013-02-13 19:29:12 -08001036 pwq->nr_active--;
1037 if (!list_empty(&pwq->delayed_works)) {
Lai Jiangshanb3f9f402012-09-18 10:40:00 -07001038 /* one down, submit a delayed one */
Tejun Heo112202d2013-02-13 19:29:12 -08001039 if (pwq->nr_active < pwq->max_active)
1040 pwq_activate_first_delayed(pwq);
Tejun Heobf4ede02012-08-03 10:30:46 -07001041 }
1042
1043 /* is flush in progress and are we at the flushing tip? */
Tejun Heo112202d2013-02-13 19:29:12 -08001044 if (likely(pwq->flush_color != color))
Tejun Heo8864b4e2013-03-12 11:30:04 -07001045 goto out_put;
Tejun Heobf4ede02012-08-03 10:30:46 -07001046
1047 /* are there still in-flight works? */
Tejun Heo112202d2013-02-13 19:29:12 -08001048 if (pwq->nr_in_flight[color])
Tejun Heo8864b4e2013-03-12 11:30:04 -07001049 goto out_put;
Tejun Heobf4ede02012-08-03 10:30:46 -07001050
Tejun Heo112202d2013-02-13 19:29:12 -08001051 /* this pwq is done, clear flush_color */
1052 pwq->flush_color = -1;
Tejun Heobf4ede02012-08-03 10:30:46 -07001053
1054 /*
Tejun Heo112202d2013-02-13 19:29:12 -08001055 * If this was the last pwq, wake up the first flusher. It
Tejun Heobf4ede02012-08-03 10:30:46 -07001056 * will handle the rest.
1057 */
Tejun Heo112202d2013-02-13 19:29:12 -08001058 if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
1059 complete(&pwq->wq->first_flusher->done);
Tejun Heo8864b4e2013-03-12 11:30:04 -07001060out_put:
1061 put_pwq(pwq);
Tejun Heobf4ede02012-08-03 10:30:46 -07001062}
1063
Tejun Heo36e227d2012-08-03 10:30:46 -07001064/**
Tejun Heobbb68df2012-08-03 10:30:46 -07001065 * try_to_grab_pending - steal work item from worklist and disable irq
Tejun Heo36e227d2012-08-03 10:30:46 -07001066 * @work: work item to steal
1067 * @is_dwork: @work is a delayed_work
Tejun Heobbb68df2012-08-03 10:30:46 -07001068 * @flags: place to store irq state
Tejun Heo36e227d2012-08-03 10:30:46 -07001069 *
1070 * Try to grab PENDING bit of @work. This function can handle @work in any
1071 * stable state - idle, on timer or on worklist. Return values are
1072 *
1073 * 1 if @work was pending and we successfully stole PENDING
1074 * 0 if @work was idle and we claimed PENDING
1075 * -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
Tejun Heobbb68df2012-08-03 10:30:46 -07001076 * -ENOENT if someone else is canceling @work, this state may persist
1077 * for arbitrarily long
Tejun Heo36e227d2012-08-03 10:30:46 -07001078 *
Tejun Heobbb68df2012-08-03 10:30:46 -07001079 * On >= 0 return, the caller owns @work's PENDING bit. To avoid getting
Tejun Heoe0aecdd2012-08-21 13:18:24 -07001080 * interrupted while holding PENDING and @work off queue, irq must be
1081 * disabled on entry. This, combined with delayed_work->timer being
1082 * irqsafe, ensures that we return -EAGAIN for finite short period of time.
Tejun Heobbb68df2012-08-03 10:30:46 -07001083 *
1084 * On successful return, >= 0, irq is disabled and the caller is
1085 * responsible for releasing it using local_irq_restore(*@flags).
1086 *
Tejun Heoe0aecdd2012-08-21 13:18:24 -07001087 * This function is safe to call from any context including IRQ handler.
Tejun Heobf4ede02012-08-03 10:30:46 -07001088 */
Tejun Heobbb68df2012-08-03 10:30:46 -07001089static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
1090 unsigned long *flags)
Tejun Heobf4ede02012-08-03 10:30:46 -07001091{
Tejun Heod565ed62013-01-24 11:01:33 -08001092 struct worker_pool *pool;
Tejun Heo112202d2013-02-13 19:29:12 -08001093 struct pool_workqueue *pwq;
Tejun Heobf4ede02012-08-03 10:30:46 -07001094
Tejun Heobbb68df2012-08-03 10:30:46 -07001095 local_irq_save(*flags);
1096
Tejun Heo36e227d2012-08-03 10:30:46 -07001097 /* try to steal the timer if it exists */
1098 if (is_dwork) {
1099 struct delayed_work *dwork = to_delayed_work(work);
1100
Tejun Heoe0aecdd2012-08-21 13:18:24 -07001101 /*
1102 * dwork->timer is irqsafe. If del_timer() fails, it's
1103 * guaranteed that the timer is not queued anywhere and not
1104 * running on the local CPU.
1105 */
Tejun Heo36e227d2012-08-03 10:30:46 -07001106 if (likely(del_timer(&dwork->timer)))
1107 return 1;
1108 }
1109
1110 /* try to claim PENDING the normal way */
Tejun Heobf4ede02012-08-03 10:30:46 -07001111 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
1112 return 0;
1113
1114 /*
1115 * The queueing is in progress, or it is already queued. Try to
1116 * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
1117 */
Tejun Heod565ed62013-01-24 11:01:33 -08001118 pool = get_work_pool(work);
1119 if (!pool)
Tejun Heobbb68df2012-08-03 10:30:46 -07001120 goto fail;
Tejun Heobf4ede02012-08-03 10:30:46 -07001121
Tejun Heod565ed62013-01-24 11:01:33 -08001122 spin_lock(&pool->lock);
Lai Jiangshan0b3dae62013-02-06 18:04:53 -08001123 /*
Tejun Heo112202d2013-02-13 19:29:12 -08001124 * work->data is guaranteed to point to pwq only while the work
1125 * item is queued on pwq->wq, and both updating work->data to point
1126 * to pwq on queueing and to pool on dequeueing are done under
1127 * pwq->pool->lock. This in turn guarantees that, if work->data
1128 * points to pwq which is associated with a locked pool, the work
Lai Jiangshan0b3dae62013-02-06 18:04:53 -08001129 * item is currently queued on that pool.
1130 */
Tejun Heo112202d2013-02-13 19:29:12 -08001131 pwq = get_work_pwq(work);
1132 if (pwq && pwq->pool == pool) {
Tejun Heo16062832013-02-06 18:04:53 -08001133 debug_work_deactivate(work);
Lai Jiangshan3aa62492012-09-18 10:40:00 -07001134
Tejun Heo16062832013-02-06 18:04:53 -08001135 /*
1136 * A delayed work item cannot be grabbed directly because
1137 * it might have linked NO_COLOR work items which, if left
Tejun Heo112202d2013-02-13 19:29:12 -08001138 * on the delayed_list, will confuse pwq->nr_active
Tejun Heo16062832013-02-06 18:04:53 -08001139 * management later on and cause stall. Make sure the work
1140 * item is activated before grabbing.
1141 */
1142 if (*work_data_bits(work) & WORK_STRUCT_DELAYED)
Tejun Heo112202d2013-02-13 19:29:12 -08001143 pwq_activate_delayed_work(work);
Lai Jiangshan3aa62492012-09-18 10:40:00 -07001144
Tejun Heo16062832013-02-06 18:04:53 -08001145 list_del_init(&work->entry);
Tejun Heo112202d2013-02-13 19:29:12 -08001146 pwq_dec_nr_in_flight(get_work_pwq(work), get_work_color(work));
Tejun Heo36e227d2012-08-03 10:30:46 -07001147
Tejun Heo112202d2013-02-13 19:29:12 -08001148 /* work->data points to pwq iff queued, point to pool */
Tejun Heo16062832013-02-06 18:04:53 -08001149 set_work_pool_and_keep_pending(work, pool->id);
Lai Jiangshan4468a002013-02-06 18:04:53 -08001150
Tejun Heo16062832013-02-06 18:04:53 -08001151 spin_unlock(&pool->lock);
1152 return 1;
Tejun Heobf4ede02012-08-03 10:30:46 -07001153 }
Tejun Heod565ed62013-01-24 11:01:33 -08001154 spin_unlock(&pool->lock);
Tejun Heobbb68df2012-08-03 10:30:46 -07001155fail:
1156 local_irq_restore(*flags);
1157 if (work_is_canceling(work))
1158 return -ENOENT;
1159 cpu_relax();
Tejun Heo36e227d2012-08-03 10:30:46 -07001160 return -EAGAIN;
Tejun Heobf4ede02012-08-03 10:30:46 -07001161}
1162
1163/**
Tejun Heo706026c2013-01-24 11:01:34 -08001164 * insert_work - insert a work into a pool
Tejun Heo112202d2013-02-13 19:29:12 -08001165 * @pwq: pwq @work belongs to
Tejun Heo4690c4a2010-06-29 10:07:10 +02001166 * @work: work to insert
1167 * @head: insertion point
1168 * @extra_flags: extra WORK_STRUCT_* flags to set
1169 *
Tejun Heo112202d2013-02-13 19:29:12 -08001170 * Insert @work which belongs to @pwq after @head. @extra_flags is or'd to
Tejun Heo706026c2013-01-24 11:01:34 -08001171 * work_struct flags.
Tejun Heo4690c4a2010-06-29 10:07:10 +02001172 *
1173 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08001174 * spin_lock_irq(pool->lock).
Tejun Heo4690c4a2010-06-29 10:07:10 +02001175 */
Tejun Heo112202d2013-02-13 19:29:12 -08001176static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
1177 struct list_head *head, unsigned int extra_flags)
Oleg Nesterovb89deed2007-05-09 02:33:52 -07001178{
Tejun Heo112202d2013-02-13 19:29:12 -08001179 struct worker_pool *pool = pwq->pool;
Frederic Weisbeckere1d8aa92009-01-12 23:15:46 +01001180
Tejun Heo4690c4a2010-06-29 10:07:10 +02001181 /* we own @work, set data and link */
Tejun Heo112202d2013-02-13 19:29:12 -08001182 set_work_pwq(work, pwq, extra_flags);
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -07001183 list_add_tail(&work->entry, head);
Tejun Heo8864b4e2013-03-12 11:30:04 -07001184 get_pwq(pwq);
Tejun Heoe22bee72010-06-29 10:07:14 +02001185
1186 /*
Tejun Heoc5aa87b2013-03-13 16:51:36 -07001187 * Ensure either wq_worker_sleeping() sees the above
1188 * list_add_tail() or we see zero nr_running to avoid workers lying
1189 * around lazily while there are works to be processed.
Tejun Heoe22bee72010-06-29 10:07:14 +02001190 */
1191 smp_mb();
1192
Tejun Heo63d95a92012-07-12 14:46:37 -07001193 if (__need_more_worker(pool))
1194 wake_up_worker(pool);
Oleg Nesterovb89deed2007-05-09 02:33:52 -07001195}
1196
Tejun Heoc8efcc22010-12-20 19:32:04 +01001197/*
1198 * Test whether @work is being queued from another work executing on the
Tejun Heo8d03ecf2013-02-13 19:29:10 -08001199 * same workqueue.
Tejun Heoc8efcc22010-12-20 19:32:04 +01001200 */
1201static bool is_chained_work(struct workqueue_struct *wq)
1202{
Tejun Heo8d03ecf2013-02-13 19:29:10 -08001203 struct worker *worker;
Tejun Heoc8efcc22010-12-20 19:32:04 +01001204
Tejun Heo8d03ecf2013-02-13 19:29:10 -08001205 worker = current_wq_worker();
1206 /*
1207 * Return %true iff I'm a worker execuing a work item on @wq. If
1208 * I'm @worker, it's safe to dereference it without locking.
1209 */
Tejun Heo112202d2013-02-13 19:29:12 -08001210 return worker && worker->current_pwq->wq == wq;
Tejun Heoc8efcc22010-12-20 19:32:04 +01001211}
1212
Tejun Heod84ff052013-03-12 11:29:59 -07001213static void __queue_work(int cpu, struct workqueue_struct *wq,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001214 struct work_struct *work)
1215{
Tejun Heo112202d2013-02-13 19:29:12 -08001216 struct pool_workqueue *pwq;
Tejun Heoc9178082013-03-12 11:30:04 -07001217 struct worker_pool *last_pool;
Tejun Heo1e19ffc2010-06-29 10:07:12 +02001218 struct list_head *worklist;
Tejun Heo8a2e8e5d2010-08-25 10:33:56 +02001219 unsigned int work_flags;
Joonsoo Kimb75cac92012-08-15 23:25:37 +09001220 unsigned int req_cpu = cpu;
Tejun Heo8930cab2012-08-03 10:30:45 -07001221
1222 /*
1223 * While a work item is PENDING && off queue, a task trying to
1224 * steal the PENDING will busy-loop waiting for it to either get
1225 * queued or lose PENDING. Grabbing PENDING and queueing should
1226 * happen with IRQ disabled.
1227 */
1228 WARN_ON_ONCE(!irqs_disabled());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001229
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +09001230 debug_work_activate(work);
Tejun Heo1e19ffc2010-06-29 10:07:12 +02001231
Tejun Heoc8efcc22010-12-20 19:32:04 +01001232 /* if dying, only works from the same workqueue are allowed */
Tejun Heo618b01e2013-03-12 11:30:04 -07001233 if (unlikely(wq->flags & __WQ_DRAINING) &&
Tejun Heoc8efcc22010-12-20 19:32:04 +01001234 WARN_ON_ONCE(!is_chained_work(wq)))
Tejun Heoe41e7042010-08-24 14:22:47 +02001235 return;
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07001236retry:
Tejun Heoc9178082013-03-12 11:30:04 -07001237 /* pwq which will be used unless @work is executing elsewhere */
Tejun Heoc7fc77f2010-07-02 10:03:51 +02001238 if (!(wq->flags & WQ_UNBOUND)) {
Tejun Heo57469822012-08-03 10:30:45 -07001239 if (cpu == WORK_CPU_UNBOUND)
Tejun Heoc7fc77f2010-07-02 10:03:51 +02001240 cpu = raw_smp_processor_id();
Tejun Heo7fb98ea2013-03-12 11:30:00 -07001241 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
Tejun Heoc9178082013-03-12 11:30:04 -07001242 } else {
1243 pwq = first_pwq(wq);
1244 }
Tejun Heodbf25762012-08-20 14:51:23 -07001245
Tejun Heoc9178082013-03-12 11:30:04 -07001246 /*
1247 * If @work was previously on a different pool, it might still be
1248 * running there, in which case the work needs to be queued on that
1249 * pool to guarantee non-reentrancy.
1250 */
1251 last_pool = get_work_pool(work);
1252 if (last_pool && last_pool != pwq->pool) {
1253 struct worker *worker;
Tejun Heo18aa9ef2010-06-29 10:07:13 +02001254
Tejun Heoc9178082013-03-12 11:30:04 -07001255 spin_lock(&last_pool->lock);
Tejun Heo18aa9ef2010-06-29 10:07:13 +02001256
Tejun Heoc9178082013-03-12 11:30:04 -07001257 worker = find_worker_executing_work(last_pool, work);
Tejun Heo18aa9ef2010-06-29 10:07:13 +02001258
Tejun Heoc9178082013-03-12 11:30:04 -07001259 if (worker && worker->current_pwq->wq == wq) {
1260 pwq = worker->current_pwq;
Tejun Heo8930cab2012-08-03 10:30:45 -07001261 } else {
Tejun Heoc9178082013-03-12 11:30:04 -07001262 /* meh... not running there, queue here */
1263 spin_unlock(&last_pool->lock);
Tejun Heo112202d2013-02-13 19:29:12 -08001264 spin_lock(&pwq->pool->lock);
Tejun Heo8930cab2012-08-03 10:30:45 -07001265 }
Tejun Heof3421792010-07-02 10:03:51 +02001266 } else {
Tejun Heo112202d2013-02-13 19:29:12 -08001267 spin_lock(&pwq->pool->lock);
Tejun Heo502ca9d2010-06-29 10:07:13 +02001268 }
1269
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07001270 /*
1271 * pwq is determined and locked. For unbound pools, we could have
1272 * raced with pwq release and it could already be dead. If its
1273 * refcnt is zero, repeat pwq selection. Note that pwqs never die
1274 * without another pwq replacing it as the first pwq or while a
1275 * work item is executing on it, so the retying is guaranteed to
1276 * make forward-progress.
1277 */
1278 if (unlikely(!pwq->refcnt)) {
1279 if (wq->flags & WQ_UNBOUND) {
1280 spin_unlock(&pwq->pool->lock);
1281 cpu_relax();
1282 goto retry;
1283 }
1284 /* oops */
1285 WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
1286 wq->name, cpu);
1287 }
1288
Tejun Heo112202d2013-02-13 19:29:12 -08001289 /* pwq determined, queue */
1290 trace_workqueue_queue_work(req_cpu, pwq, work);
Tejun Heo502ca9d2010-06-29 10:07:13 +02001291
Dan Carpenterf5b25522012-04-13 22:06:58 +03001292 if (WARN_ON(!list_empty(&work->entry))) {
Tejun Heo112202d2013-02-13 19:29:12 -08001293 spin_unlock(&pwq->pool->lock);
Dan Carpenterf5b25522012-04-13 22:06:58 +03001294 return;
1295 }
Tejun Heo1e19ffc2010-06-29 10:07:12 +02001296
Tejun Heo112202d2013-02-13 19:29:12 -08001297 pwq->nr_in_flight[pwq->work_color]++;
1298 work_flags = work_color_to_flags(pwq->work_color);
Tejun Heo1e19ffc2010-06-29 10:07:12 +02001299
Tejun Heo112202d2013-02-13 19:29:12 -08001300 if (likely(pwq->nr_active < pwq->max_active)) {
Tejun Heocdadf002010-10-05 10:49:55 +02001301 trace_workqueue_activate_work(work);
Tejun Heo112202d2013-02-13 19:29:12 -08001302 pwq->nr_active++;
1303 worklist = &pwq->pool->worklist;
Tejun Heo8a2e8e5d2010-08-25 10:33:56 +02001304 } else {
1305 work_flags |= WORK_STRUCT_DELAYED;
Tejun Heo112202d2013-02-13 19:29:12 -08001306 worklist = &pwq->delayed_works;
Tejun Heo8a2e8e5d2010-08-25 10:33:56 +02001307 }
Tejun Heo1e19ffc2010-06-29 10:07:12 +02001308
Tejun Heo112202d2013-02-13 19:29:12 -08001309 insert_work(pwq, work, worklist, work_flags);
Tejun Heo1e19ffc2010-06-29 10:07:12 +02001310
Tejun Heo112202d2013-02-13 19:29:12 -08001311 spin_unlock(&pwq->pool->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312}
1313
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -07001314/**
Zhang Ruic1a220e2008-07-23 21:28:39 -07001315 * queue_work_on - queue work on specific cpu
1316 * @cpu: CPU number to execute work on
1317 * @wq: workqueue to use
1318 * @work: work to queue
1319 *
Tejun Heod4283e92012-08-03 10:30:44 -07001320 * Returns %false if @work was already on a queue, %true otherwise.
Zhang Ruic1a220e2008-07-23 21:28:39 -07001321 *
1322 * We queue the work to a specific CPU, the caller must ensure it
1323 * can't go away.
1324 */
Tejun Heod4283e92012-08-03 10:30:44 -07001325bool queue_work_on(int cpu, struct workqueue_struct *wq,
1326 struct work_struct *work)
Zhang Ruic1a220e2008-07-23 21:28:39 -07001327{
Tejun Heod4283e92012-08-03 10:30:44 -07001328 bool ret = false;
Tejun Heo8930cab2012-08-03 10:30:45 -07001329 unsigned long flags;
1330
1331 local_irq_save(flags);
Zhang Ruic1a220e2008-07-23 21:28:39 -07001332
Tejun Heo22df02b2010-06-29 10:07:10 +02001333 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
Tejun Heo4690c4a2010-06-29 10:07:10 +02001334 __queue_work(cpu, wq, work);
Tejun Heod4283e92012-08-03 10:30:44 -07001335 ret = true;
Zhang Ruic1a220e2008-07-23 21:28:39 -07001336 }
Tejun Heo8930cab2012-08-03 10:30:45 -07001337
1338 local_irq_restore(flags);
Zhang Ruic1a220e2008-07-23 21:28:39 -07001339 return ret;
1340}
1341EXPORT_SYMBOL_GPL(queue_work_on);
1342
Tejun Heod8e794d2012-08-03 10:30:45 -07001343void delayed_work_timer_fn(unsigned long __data)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001344{
David Howells52bad642006-11-22 14:54:01 +00001345 struct delayed_work *dwork = (struct delayed_work *)__data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001346
Tejun Heoe0aecdd2012-08-21 13:18:24 -07001347 /* should have been called from irqsafe timer with irq already off */
Lai Jiangshan60c057b2013-02-06 18:04:53 -08001348 __queue_work(dwork->cpu, dwork->wq, &dwork->work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001349}
Konstantin Khlebnikov1438ade52013-01-24 16:36:31 +04001350EXPORT_SYMBOL(delayed_work_timer_fn);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001351
Tejun Heo7beb2ed2012-08-03 10:30:46 -07001352static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
1353 struct delayed_work *dwork, unsigned long delay)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001354{
Tejun Heo7beb2ed2012-08-03 10:30:46 -07001355 struct timer_list *timer = &dwork->timer;
1356 struct work_struct *work = &dwork->work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001357
Tejun Heo7beb2ed2012-08-03 10:30:46 -07001358 WARN_ON_ONCE(timer->function != delayed_work_timer_fn ||
1359 timer->data != (unsigned long)dwork);
Tejun Heofc4b5142012-12-04 07:40:39 -08001360 WARN_ON_ONCE(timer_pending(timer));
1361 WARN_ON_ONCE(!list_empty(&work->entry));
Tejun Heo7beb2ed2012-08-03 10:30:46 -07001362
Tejun Heo8852aac2012-12-01 16:23:42 -08001363 /*
1364 * If @delay is 0, queue @dwork->work immediately. This is for
1365 * both optimization and correctness. The earliest @timer can
1366 * expire is on the closest next tick and delayed_work users depend
1367 * on that there's no such delay when @delay is 0.
1368 */
1369 if (!delay) {
1370 __queue_work(cpu, wq, &dwork->work);
1371 return;
1372 }
1373
Tejun Heo7beb2ed2012-08-03 10:30:46 -07001374 timer_stats_timer_set_start_info(&dwork->timer);
1375
Lai Jiangshan60c057b2013-02-06 18:04:53 -08001376 dwork->wq = wq;
Tejun Heo12650572012-08-08 09:38:42 -07001377 dwork->cpu = cpu;
Tejun Heo7beb2ed2012-08-03 10:30:46 -07001378 timer->expires = jiffies + delay;
1379
1380 if (unlikely(cpu != WORK_CPU_UNBOUND))
1381 add_timer_on(timer, cpu);
1382 else
1383 add_timer(timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001384}
1385
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -07001386/**
1387 * queue_delayed_work_on - queue work on specific CPU after delay
1388 * @cpu: CPU number to execute work on
1389 * @wq: workqueue to use
Randy Dunlapaf9997e2006-12-22 01:06:52 -08001390 * @dwork: work to queue
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -07001391 * @delay: number of jiffies to wait before queueing
1392 *
Tejun Heo715f1302012-08-03 10:30:46 -07001393 * Returns %false if @work was already on a queue, %true otherwise. If
1394 * @delay is zero and @dwork is idle, it will be scheduled for immediate
1395 * execution.
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -07001396 */
Tejun Heod4283e92012-08-03 10:30:44 -07001397bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
1398 struct delayed_work *dwork, unsigned long delay)
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -07001399{
David Howells52bad642006-11-22 14:54:01 +00001400 struct work_struct *work = &dwork->work;
Tejun Heod4283e92012-08-03 10:30:44 -07001401 bool ret = false;
Tejun Heo8930cab2012-08-03 10:30:45 -07001402 unsigned long flags;
1403
1404 /* read the comment in __queue_work() */
1405 local_irq_save(flags);
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -07001406
Tejun Heo22df02b2010-06-29 10:07:10 +02001407 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
Tejun Heo7beb2ed2012-08-03 10:30:46 -07001408 __queue_delayed_work(cpu, wq, dwork, delay);
Tejun Heod4283e92012-08-03 10:30:44 -07001409 ret = true;
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -07001410 }
Tejun Heo8930cab2012-08-03 10:30:45 -07001411
1412 local_irq_restore(flags);
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -07001413 return ret;
1414}
Dave Jonesae90dd52006-06-30 01:40:45 -04001415EXPORT_SYMBOL_GPL(queue_delayed_work_on);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001416
Tejun Heoc8e55f32010-06-29 10:07:12 +02001417/**
Tejun Heo8376fe22012-08-03 10:30:47 -07001418 * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
1419 * @cpu: CPU number to execute work on
1420 * @wq: workqueue to use
1421 * @dwork: work to queue
1422 * @delay: number of jiffies to wait before queueing
1423 *
1424 * If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
1425 * modify @dwork's timer so that it expires after @delay. If @delay is
1426 * zero, @work is guaranteed to be scheduled immediately regardless of its
1427 * current state.
1428 *
1429 * Returns %false if @dwork was idle and queued, %true if @dwork was
1430 * pending and its timer was modified.
1431 *
Tejun Heoe0aecdd2012-08-21 13:18:24 -07001432 * This function is safe to call from any context including IRQ handler.
Tejun Heo8376fe22012-08-03 10:30:47 -07001433 * See try_to_grab_pending() for details.
1434 */
1435bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
1436 struct delayed_work *dwork, unsigned long delay)
1437{
1438 unsigned long flags;
1439 int ret;
1440
1441 do {
1442 ret = try_to_grab_pending(&dwork->work, true, &flags);
1443 } while (unlikely(ret == -EAGAIN));
1444
1445 if (likely(ret >= 0)) {
1446 __queue_delayed_work(cpu, wq, dwork, delay);
1447 local_irq_restore(flags);
1448 }
1449
1450 /* -ENOENT from try_to_grab_pending() becomes %true */
1451 return ret;
1452}
1453EXPORT_SYMBOL_GPL(mod_delayed_work_on);
1454
1455/**
Tejun Heoc8e55f32010-06-29 10:07:12 +02001456 * worker_enter_idle - enter idle state
1457 * @worker: worker which is entering idle state
1458 *
1459 * @worker is entering idle state. Update stats and idle timer if
1460 * necessary.
1461 *
1462 * LOCKING:
Tejun Heod565ed62013-01-24 11:01:33 -08001463 * spin_lock_irq(pool->lock).
Tejun Heoc8e55f32010-06-29 10:07:12 +02001464 */
1465static void worker_enter_idle(struct worker *worker)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001466{
Tejun Heobd7bdd42012-07-12 14:46:37 -07001467 struct worker_pool *pool = worker->pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001468
Tejun Heo6183c002013-03-12 11:29:57 -07001469 if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||
1470 WARN_ON_ONCE(!list_empty(&worker->entry) &&
1471 (worker->hentry.next || worker->hentry.pprev)))
1472 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001473
Tejun Heocb444762010-07-02 10:03:50 +02001474 /* can't use worker_set_flags(), also called from start_worker() */
1475 worker->flags |= WORKER_IDLE;
Tejun Heobd7bdd42012-07-12 14:46:37 -07001476 pool->nr_idle++;
Tejun Heoe22bee72010-06-29 10:07:14 +02001477 worker->last_active = jiffies;
Peter Zijlstrad5abe662006-12-06 20:37:26 -08001478
Tejun Heoc8e55f32010-06-29 10:07:12 +02001479 /* idle_list is LIFO */
Tejun Heobd7bdd42012-07-12 14:46:37 -07001480 list_add(&worker->entry, &pool->idle_list);
Tejun Heodb7bccf2010-06-29 10:07:12 +02001481
Tejun Heo628c78e2012-07-17 12:39:27 -07001482 if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
1483 mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
Tejun Heocb444762010-07-02 10:03:50 +02001484
Tejun Heo544ecf32012-05-14 15:04:50 -07001485 /*
Tejun Heo706026c2013-01-24 11:01:34 -08001486 * Sanity check nr_running. Because wq_unbind_fn() releases
Tejun Heod565ed62013-01-24 11:01:33 -08001487 * pool->lock between setting %WORKER_UNBOUND and zapping
Tejun Heo628c78e2012-07-17 12:39:27 -07001488 * nr_running, the warning may trigger spuriously. Check iff
1489 * unbind is not in progress.
Tejun Heo544ecf32012-05-14 15:04:50 -07001490 */
Tejun Heo24647572013-01-24 11:01:33 -08001491 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
Tejun Heobd7bdd42012-07-12 14:46:37 -07001492 pool->nr_workers == pool->nr_idle &&
Tejun Heoe19e3972013-01-24 11:39:44 -08001493 atomic_read(&pool->nr_running));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001494}
1495
Tejun Heoc8e55f32010-06-29 10:07:12 +02001496/**
1497 * worker_leave_idle - leave idle state
1498 * @worker: worker which is leaving idle state
1499 *
1500 * @worker is leaving idle state. Update stats.
1501 *
1502 * LOCKING:
Tejun Heod565ed62013-01-24 11:01:33 -08001503 * spin_lock_irq(pool->lock).
Tejun Heoc8e55f32010-06-29 10:07:12 +02001504 */
1505static void worker_leave_idle(struct worker *worker)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001506{
Tejun Heobd7bdd42012-07-12 14:46:37 -07001507 struct worker_pool *pool = worker->pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001508
Tejun Heo6183c002013-03-12 11:29:57 -07001509 if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
1510 return;
Tejun Heod302f012010-06-29 10:07:13 +02001511 worker_clr_flags(worker, WORKER_IDLE);
Tejun Heobd7bdd42012-07-12 14:46:37 -07001512 pool->nr_idle--;
Tejun Heoc8e55f32010-06-29 10:07:12 +02001513 list_del_init(&worker->entry);
1514}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001515
Tejun Heoe22bee72010-06-29 10:07:14 +02001516/**
Lai Jiangshanf36dc672013-02-19 12:17:02 -08001517 * worker_maybe_bind_and_lock - try to bind %current to worker_pool and lock it
1518 * @pool: target worker_pool
1519 *
1520 * Bind %current to the cpu of @pool if it is associated and lock @pool.
Tejun Heoe22bee72010-06-29 10:07:14 +02001521 *
1522 * Works which are scheduled while the cpu is online must at least be
1523 * scheduled to a worker which is bound to the cpu so that if they are
1524 * flushed from cpu callbacks while cpu is going down, they are
1525 * guaranteed to execute on the cpu.
1526 *
Lai Jiangshanf5faa072013-02-19 12:17:02 -08001527 * This function is to be used by unbound workers and rescuers to bind
Tejun Heoe22bee72010-06-29 10:07:14 +02001528 * themselves to the target cpu and may race with cpu going down or
1529 * coming online. kthread_bind() can't be used because it may put the
1530 * worker to already dead cpu and set_cpus_allowed_ptr() can't be used
Tejun Heo706026c2013-01-24 11:01:34 -08001531 * verbatim as it's best effort and blocking and pool may be
Tejun Heoe22bee72010-06-29 10:07:14 +02001532 * [dis]associated in the meantime.
1533 *
Tejun Heo706026c2013-01-24 11:01:34 -08001534 * This function tries set_cpus_allowed() and locks pool and verifies the
Tejun Heo24647572013-01-24 11:01:33 -08001535 * binding against %POOL_DISASSOCIATED which is set during
Tejun Heof2d5a0e2012-07-17 12:39:26 -07001536 * %CPU_DOWN_PREPARE and cleared during %CPU_ONLINE, so if the worker
1537 * enters idle state or fetches works without dropping lock, it can
1538 * guarantee the scheduling requirement described in the first paragraph.
Tejun Heoe22bee72010-06-29 10:07:14 +02001539 *
1540 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08001541 * Might sleep. Called without any lock but returns with pool->lock
Tejun Heoe22bee72010-06-29 10:07:14 +02001542 * held.
1543 *
1544 * RETURNS:
Tejun Heo706026c2013-01-24 11:01:34 -08001545 * %true if the associated pool is online (@worker is successfully
Tejun Heoe22bee72010-06-29 10:07:14 +02001546 * bound), %false if offline.
1547 */
Lai Jiangshanf36dc672013-02-19 12:17:02 -08001548static bool worker_maybe_bind_and_lock(struct worker_pool *pool)
Tejun Heod565ed62013-01-24 11:01:33 -08001549__acquires(&pool->lock)
Tejun Heoe22bee72010-06-29 10:07:14 +02001550{
Tejun Heoe22bee72010-06-29 10:07:14 +02001551 while (true) {
1552 /*
1553 * The following call may fail, succeed or succeed
1554 * without actually migrating the task to the cpu if
1555 * it races with cpu hotunplug operation. Verify
Tejun Heo24647572013-01-24 11:01:33 -08001556 * against POOL_DISASSOCIATED.
Tejun Heoe22bee72010-06-29 10:07:14 +02001557 */
Tejun Heo24647572013-01-24 11:01:33 -08001558 if (!(pool->flags & POOL_DISASSOCIATED))
Tejun Heo7a4e3442013-03-12 11:30:00 -07001559 set_cpus_allowed_ptr(current, pool->attrs->cpumask);
Oleg Nesterov85f41862007-05-09 02:34:20 -07001560
Tejun Heod565ed62013-01-24 11:01:33 -08001561 spin_lock_irq(&pool->lock);
Tejun Heo24647572013-01-24 11:01:33 -08001562 if (pool->flags & POOL_DISASSOCIATED)
Tejun Heoe22bee72010-06-29 10:07:14 +02001563 return false;
Lai Jiangshanf5faa072013-02-19 12:17:02 -08001564 if (task_cpu(current) == pool->cpu &&
Tejun Heo7a4e3442013-03-12 11:30:00 -07001565 cpumask_equal(&current->cpus_allowed, pool->attrs->cpumask))
Tejun Heoe22bee72010-06-29 10:07:14 +02001566 return true;
Tejun Heod565ed62013-01-24 11:01:33 -08001567 spin_unlock_irq(&pool->lock);
Oleg Nesterov3af244332007-05-09 02:34:09 -07001568
Tejun Heo5035b202011-04-29 18:08:37 +02001569 /*
1570 * We've raced with CPU hot[un]plug. Give it a breather
1571 * and retry migration. cond_resched() is required here;
1572 * otherwise, we might deadlock against cpu_stop trying to
1573 * bring down the CPU on non-preemptive kernel.
1574 */
Tejun Heoe22bee72010-06-29 10:07:14 +02001575 cpu_relax();
Tejun Heo5035b202011-04-29 18:08:37 +02001576 cond_resched();
Tejun Heoe22bee72010-06-29 10:07:14 +02001577 }
1578}
1579
1580/*
Lai Jiangshanea1abd62012-09-18 09:59:22 -07001581 * Rebind an idle @worker to its CPU. worker_thread() will test
Lai Jiangshan5f7dabf2012-09-18 09:59:23 -07001582 * list_empty(@worker->entry) before leaving idle and call this function.
Tejun Heo25511a42012-07-17 12:39:27 -07001583 */
1584static void idle_worker_rebind(struct worker *worker)
1585{
Lai Jiangshan5f7dabf2012-09-18 09:59:23 -07001586 /* CPU may go down again inbetween, clear UNBOUND only on success */
Lai Jiangshanf36dc672013-02-19 12:17:02 -08001587 if (worker_maybe_bind_and_lock(worker->pool))
Lai Jiangshan5f7dabf2012-09-18 09:59:23 -07001588 worker_clr_flags(worker, WORKER_UNBOUND);
Tejun Heo25511a42012-07-17 12:39:27 -07001589
Lai Jiangshanea1abd62012-09-18 09:59:22 -07001590 /* rebind complete, become available again */
1591 list_add(&worker->entry, &worker->pool->idle_list);
Tejun Heod565ed62013-01-24 11:01:33 -08001592 spin_unlock_irq(&worker->pool->lock);
Tejun Heo25511a42012-07-17 12:39:27 -07001593}
1594
1595/*
1596 * Function for @worker->rebind.work used to rebind unbound busy workers to
Tejun Heo403c8212012-07-17 12:39:27 -07001597 * the associated cpu which is coming back online. This is scheduled by
1598 * cpu up but can race with other cpu hotplug operations and may be
1599 * executed twice without intervening cpu down.
Tejun Heoe22bee72010-06-29 10:07:14 +02001600 */
Tejun Heo25511a42012-07-17 12:39:27 -07001601static void busy_worker_rebind_fn(struct work_struct *work)
Tejun Heoe22bee72010-06-29 10:07:14 +02001602{
1603 struct worker *worker = container_of(work, struct worker, rebind_work);
Tejun Heoe22bee72010-06-29 10:07:14 +02001604
Lai Jiangshanf36dc672013-02-19 12:17:02 -08001605 if (worker_maybe_bind_and_lock(worker->pool))
Lai Jiangshaneab6d822012-09-18 09:59:22 -07001606 worker_clr_flags(worker, WORKER_UNBOUND);
Tejun Heoe22bee72010-06-29 10:07:14 +02001607
Tejun Heod565ed62013-01-24 11:01:33 -08001608 spin_unlock_irq(&worker->pool->lock);
Tejun Heoe22bee72010-06-29 10:07:14 +02001609}
1610
Tejun Heo25511a42012-07-17 12:39:27 -07001611/**
Tejun Heo94cf58b2013-01-24 11:01:33 -08001612 * rebind_workers - rebind all workers of a pool to the associated CPU
1613 * @pool: pool of interest
Tejun Heo25511a42012-07-17 12:39:27 -07001614 *
Tejun Heo94cf58b2013-01-24 11:01:33 -08001615 * @pool->cpu is coming online. Rebind all workers to the CPU. Rebinding
Tejun Heo25511a42012-07-17 12:39:27 -07001616 * is different for idle and busy ones.
1617 *
Lai Jiangshanea1abd62012-09-18 09:59:22 -07001618 * Idle ones will be removed from the idle_list and woken up. They will
1619 * add themselves back after completing rebind. This ensures that the
1620 * idle_list doesn't contain any unbound workers when re-bound busy workers
1621 * try to perform local wake-ups for concurrency management.
Tejun Heo25511a42012-07-17 12:39:27 -07001622 *
Lai Jiangshanea1abd62012-09-18 09:59:22 -07001623 * Busy workers can rebind after they finish their current work items.
1624 * Queueing the rebind work item at the head of the scheduled list is
1625 * enough. Note that nr_running will be properly bumped as busy workers
1626 * rebind.
Tejun Heo25511a42012-07-17 12:39:27 -07001627 *
Lai Jiangshanea1abd62012-09-18 09:59:22 -07001628 * On return, all non-manager workers are scheduled for rebind - see
1629 * manage_workers() for the manager special case. Any idle worker
1630 * including the manager will not appear on @idle_list until rebind is
1631 * complete, making local wake-ups safe.
Tejun Heo25511a42012-07-17 12:39:27 -07001632 */
Tejun Heo94cf58b2013-01-24 11:01:33 -08001633static void rebind_workers(struct worker_pool *pool)
Tejun Heo25511a42012-07-17 12:39:27 -07001634{
Lai Jiangshanea1abd62012-09-18 09:59:22 -07001635 struct worker *worker, *n;
Tejun Heo25511a42012-07-17 12:39:27 -07001636 int i;
1637
Tejun Heo94cf58b2013-01-24 11:01:33 -08001638 lockdep_assert_held(&pool->assoc_mutex);
1639 lockdep_assert_held(&pool->lock);
Tejun Heo25511a42012-07-17 12:39:27 -07001640
Lai Jiangshan5f7dabf2012-09-18 09:59:23 -07001641 /* dequeue and kick idle ones */
Tejun Heo94cf58b2013-01-24 11:01:33 -08001642 list_for_each_entry_safe(worker, n, &pool->idle_list, entry) {
1643 /*
1644 * idle workers should be off @pool->idle_list until rebind
1645 * is complete to avoid receiving premature local wake-ups.
1646 */
1647 list_del_init(&worker->entry);
Lai Jiangshan96e65302012-09-02 00:28:19 +08001648
Tejun Heo94cf58b2013-01-24 11:01:33 -08001649 /*
1650 * worker_thread() will see the above dequeuing and call
1651 * idle_worker_rebind().
1652 */
1653 wake_up_process(worker->task);
1654 }
Tejun Heo25511a42012-07-17 12:39:27 -07001655
Tejun Heo94cf58b2013-01-24 11:01:33 -08001656 /* rebind busy workers */
Sasha Levinb67bfe02013-02-27 17:06:00 -08001657 for_each_busy_worker(worker, i, pool) {
Tejun Heo94cf58b2013-01-24 11:01:33 -08001658 struct work_struct *rebind_work = &worker->rebind_work;
1659 struct workqueue_struct *wq;
Tejun Heo25511a42012-07-17 12:39:27 -07001660
Tejun Heo94cf58b2013-01-24 11:01:33 -08001661 if (test_and_set_bit(WORK_STRUCT_PENDING_BIT,
1662 work_data_bits(rebind_work)))
1663 continue;
Tejun Heo25511a42012-07-17 12:39:27 -07001664
Tejun Heo94cf58b2013-01-24 11:01:33 -08001665 debug_work_activate(rebind_work);
Tejun Heo90beca52012-09-04 23:12:33 -07001666
Tejun Heo94cf58b2013-01-24 11:01:33 -08001667 /*
1668 * wq doesn't really matter but let's keep @worker->pool
Tejun Heo112202d2013-02-13 19:29:12 -08001669 * and @pwq->pool consistent for sanity.
Tejun Heo94cf58b2013-01-24 11:01:33 -08001670 */
Tejun Heo7a4e3442013-03-12 11:30:00 -07001671 if (worker->pool->attrs->nice < 0)
Tejun Heo94cf58b2013-01-24 11:01:33 -08001672 wq = system_highpri_wq;
1673 else
1674 wq = system_wq;
Tejun Heoec588152012-09-04 23:16:32 -07001675
Tejun Heo7fb98ea2013-03-12 11:30:00 -07001676 insert_work(per_cpu_ptr(wq->cpu_pwqs, pool->cpu), rebind_work,
Tejun Heo94cf58b2013-01-24 11:01:33 -08001677 worker->scheduled.next,
1678 work_color_to_flags(WORK_NO_COLOR));
Tejun Heoec588152012-09-04 23:16:32 -07001679 }
Tejun Heo25511a42012-07-17 12:39:27 -07001680}
1681
Tejun Heoc34056a2010-06-29 10:07:11 +02001682static struct worker *alloc_worker(void)
1683{
1684 struct worker *worker;
1685
1686 worker = kzalloc(sizeof(*worker), GFP_KERNEL);
Tejun Heoc8e55f32010-06-29 10:07:12 +02001687 if (worker) {
1688 INIT_LIST_HEAD(&worker->entry);
Tejun Heoaffee4b2010-06-29 10:07:12 +02001689 INIT_LIST_HEAD(&worker->scheduled);
Tejun Heo25511a42012-07-17 12:39:27 -07001690 INIT_WORK(&worker->rebind_work, busy_worker_rebind_fn);
Tejun Heoe22bee72010-06-29 10:07:14 +02001691 /* on creation a worker is in !idle && prep state */
1692 worker->flags = WORKER_PREP;
Tejun Heoc8e55f32010-06-29 10:07:12 +02001693 }
Tejun Heoc34056a2010-06-29 10:07:11 +02001694 return worker;
1695}
1696
1697/**
1698 * create_worker - create a new workqueue worker
Tejun Heo63d95a92012-07-12 14:46:37 -07001699 * @pool: pool the new worker will belong to
Tejun Heoc34056a2010-06-29 10:07:11 +02001700 *
Tejun Heo63d95a92012-07-12 14:46:37 -07001701 * Create a new worker which is bound to @pool. The returned worker
Tejun Heoc34056a2010-06-29 10:07:11 +02001702 * can be started by calling start_worker() or destroyed using
1703 * destroy_worker().
1704 *
1705 * CONTEXT:
1706 * Might sleep. Does GFP_KERNEL allocations.
1707 *
1708 * RETURNS:
1709 * Pointer to the newly created worker.
1710 */
Tejun Heobc2ae0f2012-07-17 12:39:27 -07001711static struct worker *create_worker(struct worker_pool *pool)
Tejun Heoc34056a2010-06-29 10:07:11 +02001712{
Tejun Heo7a4e3442013-03-12 11:30:00 -07001713 const char *pri = pool->attrs->nice < 0 ? "H" : "";
Tejun Heoc34056a2010-06-29 10:07:11 +02001714 struct worker *worker = NULL;
Tejun Heof3421792010-07-02 10:03:51 +02001715 int id = -1;
Tejun Heoc34056a2010-06-29 10:07:11 +02001716
Tejun Heod565ed62013-01-24 11:01:33 -08001717 spin_lock_irq(&pool->lock);
Tejun Heobd7bdd42012-07-12 14:46:37 -07001718 while (ida_get_new(&pool->worker_ida, &id)) {
Tejun Heod565ed62013-01-24 11:01:33 -08001719 spin_unlock_irq(&pool->lock);
Tejun Heobd7bdd42012-07-12 14:46:37 -07001720 if (!ida_pre_get(&pool->worker_ida, GFP_KERNEL))
Tejun Heoc34056a2010-06-29 10:07:11 +02001721 goto fail;
Tejun Heod565ed62013-01-24 11:01:33 -08001722 spin_lock_irq(&pool->lock);
Tejun Heoc34056a2010-06-29 10:07:11 +02001723 }
Tejun Heod565ed62013-01-24 11:01:33 -08001724 spin_unlock_irq(&pool->lock);
Tejun Heoc34056a2010-06-29 10:07:11 +02001725
1726 worker = alloc_worker();
1727 if (!worker)
1728 goto fail;
1729
Tejun Heobd7bdd42012-07-12 14:46:37 -07001730 worker->pool = pool;
Tejun Heoc34056a2010-06-29 10:07:11 +02001731 worker->id = id;
1732
Tejun Heo29c91e92013-03-12 11:30:03 -07001733 if (pool->cpu >= 0)
Eric Dumazet94dcf292011-03-22 16:30:45 -07001734 worker->task = kthread_create_on_node(worker_thread,
Tejun Heoec22ca52013-01-24 11:01:33 -08001735 worker, cpu_to_node(pool->cpu),
Tejun Heod84ff052013-03-12 11:29:59 -07001736 "kworker/%d:%d%s", pool->cpu, id, pri);
Tejun Heof3421792010-07-02 10:03:51 +02001737 else
1738 worker->task = kthread_create(worker_thread, worker,
Tejun Heoac6104c2013-03-12 11:30:03 -07001739 "kworker/u%d:%d%s",
1740 pool->id, id, pri);
Tejun Heoc34056a2010-06-29 10:07:11 +02001741 if (IS_ERR(worker->task))
1742 goto fail;
1743
Tejun Heoc5aa87b2013-03-13 16:51:36 -07001744 /*
1745 * set_cpus_allowed_ptr() will fail if the cpumask doesn't have any
1746 * online CPUs. It'll be re-applied when any of the CPUs come up.
1747 */
Tejun Heo7a4e3442013-03-12 11:30:00 -07001748 set_user_nice(worker->task, pool->attrs->nice);
1749 set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
Tejun Heo32704762012-07-13 22:16:45 -07001750
Tejun Heodb7bccf2010-06-29 10:07:12 +02001751 /*
Tejun Heo7a4e3442013-03-12 11:30:00 -07001752 * %PF_THREAD_BOUND is used to prevent userland from meddling with
1753 * cpumask of workqueue workers. This is an abuse. We need
1754 * %PF_NO_SETAFFINITY.
Tejun Heodb7bccf2010-06-29 10:07:12 +02001755 */
Tejun Heo7a4e3442013-03-12 11:30:00 -07001756 worker->task->flags |= PF_THREAD_BOUND;
1757
1758 /*
1759 * The caller is responsible for ensuring %POOL_DISASSOCIATED
1760 * remains stable across this function. See the comments above the
1761 * flag definition for details.
1762 */
1763 if (pool->flags & POOL_DISASSOCIATED)
Tejun Heobc2ae0f2012-07-17 12:39:27 -07001764 worker->flags |= WORKER_UNBOUND;
Oleg Nesterov3af244332007-05-09 02:34:09 -07001765
Tejun Heoc34056a2010-06-29 10:07:11 +02001766 return worker;
1767fail:
1768 if (id >= 0) {
Tejun Heod565ed62013-01-24 11:01:33 -08001769 spin_lock_irq(&pool->lock);
Tejun Heobd7bdd42012-07-12 14:46:37 -07001770 ida_remove(&pool->worker_ida, id);
Tejun Heod565ed62013-01-24 11:01:33 -08001771 spin_unlock_irq(&pool->lock);
Tejun Heoc34056a2010-06-29 10:07:11 +02001772 }
1773 kfree(worker);
1774 return NULL;
1775}
1776
1777/**
1778 * start_worker - start a newly created worker
1779 * @worker: worker to start
1780 *
Tejun Heo706026c2013-01-24 11:01:34 -08001781 * Make the pool aware of @worker and start it.
Tejun Heoc34056a2010-06-29 10:07:11 +02001782 *
1783 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08001784 * spin_lock_irq(pool->lock).
Tejun Heoc34056a2010-06-29 10:07:11 +02001785 */
1786static void start_worker(struct worker *worker)
1787{
Tejun Heocb444762010-07-02 10:03:50 +02001788 worker->flags |= WORKER_STARTED;
Tejun Heobd7bdd42012-07-12 14:46:37 -07001789 worker->pool->nr_workers++;
Tejun Heoc8e55f32010-06-29 10:07:12 +02001790 worker_enter_idle(worker);
Tejun Heoc34056a2010-06-29 10:07:11 +02001791 wake_up_process(worker->task);
1792}
1793
1794/**
1795 * destroy_worker - destroy a workqueue worker
1796 * @worker: worker to be destroyed
1797 *
Tejun Heo706026c2013-01-24 11:01:34 -08001798 * Destroy @worker and adjust @pool stats accordingly.
Tejun Heoc8e55f32010-06-29 10:07:12 +02001799 *
1800 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08001801 * spin_lock_irq(pool->lock) which is released and regrabbed.
Tejun Heoc34056a2010-06-29 10:07:11 +02001802 */
1803static void destroy_worker(struct worker *worker)
1804{
Tejun Heobd7bdd42012-07-12 14:46:37 -07001805 struct worker_pool *pool = worker->pool;
Tejun Heoc34056a2010-06-29 10:07:11 +02001806 int id = worker->id;
1807
1808 /* sanity check frenzy */
Tejun Heo6183c002013-03-12 11:29:57 -07001809 if (WARN_ON(worker->current_work) ||
1810 WARN_ON(!list_empty(&worker->scheduled)))
1811 return;
Tejun Heoc34056a2010-06-29 10:07:11 +02001812
Tejun Heoc8e55f32010-06-29 10:07:12 +02001813 if (worker->flags & WORKER_STARTED)
Tejun Heobd7bdd42012-07-12 14:46:37 -07001814 pool->nr_workers--;
Tejun Heoc8e55f32010-06-29 10:07:12 +02001815 if (worker->flags & WORKER_IDLE)
Tejun Heobd7bdd42012-07-12 14:46:37 -07001816 pool->nr_idle--;
Tejun Heoc8e55f32010-06-29 10:07:12 +02001817
1818 list_del_init(&worker->entry);
Tejun Heocb444762010-07-02 10:03:50 +02001819 worker->flags |= WORKER_DIE;
Tejun Heoc8e55f32010-06-29 10:07:12 +02001820
Tejun Heod565ed62013-01-24 11:01:33 -08001821 spin_unlock_irq(&pool->lock);
Tejun Heoc8e55f32010-06-29 10:07:12 +02001822
Tejun Heoc34056a2010-06-29 10:07:11 +02001823 kthread_stop(worker->task);
1824 kfree(worker);
1825
Tejun Heod565ed62013-01-24 11:01:33 -08001826 spin_lock_irq(&pool->lock);
Tejun Heobd7bdd42012-07-12 14:46:37 -07001827 ida_remove(&pool->worker_ida, id);
Tejun Heoc34056a2010-06-29 10:07:11 +02001828}
1829
Tejun Heo63d95a92012-07-12 14:46:37 -07001830static void idle_worker_timeout(unsigned long __pool)
Tejun Heoe22bee72010-06-29 10:07:14 +02001831{
Tejun Heo63d95a92012-07-12 14:46:37 -07001832 struct worker_pool *pool = (void *)__pool;
Tejun Heoe22bee72010-06-29 10:07:14 +02001833
Tejun Heod565ed62013-01-24 11:01:33 -08001834 spin_lock_irq(&pool->lock);
Tejun Heoe22bee72010-06-29 10:07:14 +02001835
Tejun Heo63d95a92012-07-12 14:46:37 -07001836 if (too_many_workers(pool)) {
Tejun Heoe22bee72010-06-29 10:07:14 +02001837 struct worker *worker;
1838 unsigned long expires;
1839
1840 /* idle_list is kept in LIFO order, check the last one */
Tejun Heo63d95a92012-07-12 14:46:37 -07001841 worker = list_entry(pool->idle_list.prev, struct worker, entry);
Tejun Heoe22bee72010-06-29 10:07:14 +02001842 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
1843
1844 if (time_before(jiffies, expires))
Tejun Heo63d95a92012-07-12 14:46:37 -07001845 mod_timer(&pool->idle_timer, expires);
Tejun Heoe22bee72010-06-29 10:07:14 +02001846 else {
1847 /* it's been idle for too long, wake up manager */
Tejun Heo11ebea52012-07-12 14:46:37 -07001848 pool->flags |= POOL_MANAGE_WORKERS;
Tejun Heo63d95a92012-07-12 14:46:37 -07001849 wake_up_worker(pool);
Tejun Heoe22bee72010-06-29 10:07:14 +02001850 }
1851 }
1852
Tejun Heod565ed62013-01-24 11:01:33 -08001853 spin_unlock_irq(&pool->lock);
Tejun Heoe22bee72010-06-29 10:07:14 +02001854}
1855
Tejun Heo493a1722013-03-12 11:29:59 -07001856static void send_mayday(struct work_struct *work)
Tejun Heoe22bee72010-06-29 10:07:14 +02001857{
Tejun Heo112202d2013-02-13 19:29:12 -08001858 struct pool_workqueue *pwq = get_work_pwq(work);
1859 struct workqueue_struct *wq = pwq->wq;
Tejun Heo493a1722013-03-12 11:29:59 -07001860
1861 lockdep_assert_held(&workqueue_lock);
Tejun Heoe22bee72010-06-29 10:07:14 +02001862
Tejun Heo493008a2013-03-12 11:30:03 -07001863 if (!wq->rescuer)
Tejun Heo493a1722013-03-12 11:29:59 -07001864 return;
Tejun Heoe22bee72010-06-29 10:07:14 +02001865
1866 /* mayday mayday mayday */
Tejun Heo493a1722013-03-12 11:29:59 -07001867 if (list_empty(&pwq->mayday_node)) {
1868 list_add_tail(&pwq->mayday_node, &wq->maydays);
Tejun Heoe22bee72010-06-29 10:07:14 +02001869 wake_up_process(wq->rescuer->task);
Tejun Heo493a1722013-03-12 11:29:59 -07001870 }
Tejun Heoe22bee72010-06-29 10:07:14 +02001871}
1872
Tejun Heo706026c2013-01-24 11:01:34 -08001873static void pool_mayday_timeout(unsigned long __pool)
Tejun Heoe22bee72010-06-29 10:07:14 +02001874{
Tejun Heo63d95a92012-07-12 14:46:37 -07001875 struct worker_pool *pool = (void *)__pool;
Tejun Heoe22bee72010-06-29 10:07:14 +02001876 struct work_struct *work;
1877
Tejun Heo493a1722013-03-12 11:29:59 -07001878 spin_lock_irq(&workqueue_lock); /* for wq->maydays */
1879 spin_lock(&pool->lock);
Tejun Heoe22bee72010-06-29 10:07:14 +02001880
Tejun Heo63d95a92012-07-12 14:46:37 -07001881 if (need_to_create_worker(pool)) {
Tejun Heoe22bee72010-06-29 10:07:14 +02001882 /*
1883 * We've been trying to create a new worker but
1884 * haven't been successful. We might be hitting an
1885 * allocation deadlock. Send distress signals to
1886 * rescuers.
1887 */
Tejun Heo63d95a92012-07-12 14:46:37 -07001888 list_for_each_entry(work, &pool->worklist, entry)
Tejun Heoe22bee72010-06-29 10:07:14 +02001889 send_mayday(work);
1890 }
1891
Tejun Heo493a1722013-03-12 11:29:59 -07001892 spin_unlock(&pool->lock);
1893 spin_unlock_irq(&workqueue_lock);
Tejun Heoe22bee72010-06-29 10:07:14 +02001894
Tejun Heo63d95a92012-07-12 14:46:37 -07001895 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
Tejun Heoe22bee72010-06-29 10:07:14 +02001896}
1897
1898/**
1899 * maybe_create_worker - create a new worker if necessary
Tejun Heo63d95a92012-07-12 14:46:37 -07001900 * @pool: pool to create a new worker for
Tejun Heoe22bee72010-06-29 10:07:14 +02001901 *
Tejun Heo63d95a92012-07-12 14:46:37 -07001902 * Create a new worker for @pool if necessary. @pool is guaranteed to
Tejun Heoe22bee72010-06-29 10:07:14 +02001903 * have at least one idle worker on return from this function. If
1904 * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
Tejun Heo63d95a92012-07-12 14:46:37 -07001905 * sent to all rescuers with works scheduled on @pool to resolve
Tejun Heoe22bee72010-06-29 10:07:14 +02001906 * possible allocation deadlock.
1907 *
Tejun Heoc5aa87b2013-03-13 16:51:36 -07001908 * On return, need_to_create_worker() is guaranteed to be %false and
1909 * may_start_working() %true.
Tejun Heoe22bee72010-06-29 10:07:14 +02001910 *
1911 * LOCKING:
Tejun Heod565ed62013-01-24 11:01:33 -08001912 * spin_lock_irq(pool->lock) which may be released and regrabbed
Tejun Heoe22bee72010-06-29 10:07:14 +02001913 * multiple times. Does GFP_KERNEL allocations. Called only from
1914 * manager.
1915 *
1916 * RETURNS:
Tejun Heoc5aa87b2013-03-13 16:51:36 -07001917 * %false if no action was taken and pool->lock stayed locked, %true
Tejun Heoe22bee72010-06-29 10:07:14 +02001918 * otherwise.
1919 */
Tejun Heo63d95a92012-07-12 14:46:37 -07001920static bool maybe_create_worker(struct worker_pool *pool)
Tejun Heod565ed62013-01-24 11:01:33 -08001921__releases(&pool->lock)
1922__acquires(&pool->lock)
Tejun Heoe22bee72010-06-29 10:07:14 +02001923{
Tejun Heo63d95a92012-07-12 14:46:37 -07001924 if (!need_to_create_worker(pool))
Tejun Heoe22bee72010-06-29 10:07:14 +02001925 return false;
1926restart:
Tejun Heod565ed62013-01-24 11:01:33 -08001927 spin_unlock_irq(&pool->lock);
Tejun Heo9f9c2362010-07-14 11:31:20 +02001928
Tejun Heoe22bee72010-06-29 10:07:14 +02001929 /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
Tejun Heo63d95a92012-07-12 14:46:37 -07001930 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
Tejun Heoe22bee72010-06-29 10:07:14 +02001931
1932 while (true) {
1933 struct worker *worker;
1934
Tejun Heobc2ae0f2012-07-17 12:39:27 -07001935 worker = create_worker(pool);
Tejun Heoe22bee72010-06-29 10:07:14 +02001936 if (worker) {
Tejun Heo63d95a92012-07-12 14:46:37 -07001937 del_timer_sync(&pool->mayday_timer);
Tejun Heod565ed62013-01-24 11:01:33 -08001938 spin_lock_irq(&pool->lock);
Tejun Heoe22bee72010-06-29 10:07:14 +02001939 start_worker(worker);
Tejun Heo6183c002013-03-12 11:29:57 -07001940 if (WARN_ON_ONCE(need_to_create_worker(pool)))
1941 goto restart;
Tejun Heoe22bee72010-06-29 10:07:14 +02001942 return true;
1943 }
1944
Tejun Heo63d95a92012-07-12 14:46:37 -07001945 if (!need_to_create_worker(pool))
Tejun Heoe22bee72010-06-29 10:07:14 +02001946 break;
1947
Tejun Heoe22bee72010-06-29 10:07:14 +02001948 __set_current_state(TASK_INTERRUPTIBLE);
1949 schedule_timeout(CREATE_COOLDOWN);
Tejun Heo9f9c2362010-07-14 11:31:20 +02001950
Tejun Heo63d95a92012-07-12 14:46:37 -07001951 if (!need_to_create_worker(pool))
Tejun Heoe22bee72010-06-29 10:07:14 +02001952 break;
1953 }
1954
Tejun Heo63d95a92012-07-12 14:46:37 -07001955 del_timer_sync(&pool->mayday_timer);
Tejun Heod565ed62013-01-24 11:01:33 -08001956 spin_lock_irq(&pool->lock);
Tejun Heo63d95a92012-07-12 14:46:37 -07001957 if (need_to_create_worker(pool))
Tejun Heoe22bee72010-06-29 10:07:14 +02001958 goto restart;
1959 return true;
1960}
1961
1962/**
1963 * maybe_destroy_worker - destroy workers which have been idle for a while
Tejun Heo63d95a92012-07-12 14:46:37 -07001964 * @pool: pool to destroy workers for
Tejun Heoe22bee72010-06-29 10:07:14 +02001965 *
Tejun Heo63d95a92012-07-12 14:46:37 -07001966 * Destroy @pool workers which have been idle for longer than
Tejun Heoe22bee72010-06-29 10:07:14 +02001967 * IDLE_WORKER_TIMEOUT.
1968 *
1969 * LOCKING:
Tejun Heod565ed62013-01-24 11:01:33 -08001970 * spin_lock_irq(pool->lock) which may be released and regrabbed
Tejun Heoe22bee72010-06-29 10:07:14 +02001971 * multiple times. Called only from manager.
1972 *
1973 * RETURNS:
Tejun Heoc5aa87b2013-03-13 16:51:36 -07001974 * %false if no action was taken and pool->lock stayed locked, %true
Tejun Heoe22bee72010-06-29 10:07:14 +02001975 * otherwise.
1976 */
Tejun Heo63d95a92012-07-12 14:46:37 -07001977static bool maybe_destroy_workers(struct worker_pool *pool)
Tejun Heoe22bee72010-06-29 10:07:14 +02001978{
1979 bool ret = false;
1980
Tejun Heo63d95a92012-07-12 14:46:37 -07001981 while (too_many_workers(pool)) {
Tejun Heoe22bee72010-06-29 10:07:14 +02001982 struct worker *worker;
1983 unsigned long expires;
1984
Tejun Heo63d95a92012-07-12 14:46:37 -07001985 worker = list_entry(pool->idle_list.prev, struct worker, entry);
Tejun Heoe22bee72010-06-29 10:07:14 +02001986 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
1987
1988 if (time_before(jiffies, expires)) {
Tejun Heo63d95a92012-07-12 14:46:37 -07001989 mod_timer(&pool->idle_timer, expires);
Tejun Heoe22bee72010-06-29 10:07:14 +02001990 break;
1991 }
1992
1993 destroy_worker(worker);
1994 ret = true;
1995 }
1996
1997 return ret;
1998}
1999
2000/**
2001 * manage_workers - manage worker pool
2002 * @worker: self
2003 *
Tejun Heo706026c2013-01-24 11:01:34 -08002004 * Assume the manager role and manage the worker pool @worker belongs
Tejun Heoe22bee72010-06-29 10:07:14 +02002005 * to. At any given time, there can be only zero or one manager per
Tejun Heo706026c2013-01-24 11:01:34 -08002006 * pool. The exclusion is handled automatically by this function.
Tejun Heoe22bee72010-06-29 10:07:14 +02002007 *
2008 * The caller can safely start processing works on false return. On
2009 * true return, it's guaranteed that need_to_create_worker() is false
2010 * and may_start_working() is true.
2011 *
2012 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08002013 * spin_lock_irq(pool->lock) which may be released and regrabbed
Tejun Heoe22bee72010-06-29 10:07:14 +02002014 * multiple times. Does GFP_KERNEL allocations.
2015 *
2016 * RETURNS:
Tejun Heod565ed62013-01-24 11:01:33 -08002017 * spin_lock_irq(pool->lock) which may be released and regrabbed
2018 * multiple times. Does GFP_KERNEL allocations.
Tejun Heoe22bee72010-06-29 10:07:14 +02002019 */
2020static bool manage_workers(struct worker *worker)
2021{
Tejun Heo63d95a92012-07-12 14:46:37 -07002022 struct worker_pool *pool = worker->pool;
Tejun Heoe22bee72010-06-29 10:07:14 +02002023 bool ret = false;
2024
Tejun Heo34a06bd2013-03-12 11:30:00 -07002025 if (!mutex_trylock(&pool->manager_arb))
Tejun Heoe22bee72010-06-29 10:07:14 +02002026 return ret;
2027
Lai Jiangshanee378aa2012-09-10 10:03:44 -07002028 /*
2029 * To simplify both worker management and CPU hotplug, hold off
2030 * management while hotplug is in progress. CPU hotplug path can't
Tejun Heo34a06bd2013-03-12 11:30:00 -07002031 * grab @pool->manager_arb to achieve this because that can lead to
2032 * idle worker depletion (all become busy thinking someone else is
2033 * managing) which in turn can result in deadlock under extreme
2034 * circumstances. Use @pool->assoc_mutex to synchronize manager
2035 * against CPU hotplug.
Lai Jiangshanee378aa2012-09-10 10:03:44 -07002036 *
Lai Jiangshanb2eb83d2012-09-18 09:59:23 -07002037 * assoc_mutex would always be free unless CPU hotplug is in
Tejun Heod565ed62013-01-24 11:01:33 -08002038 * progress. trylock first without dropping @pool->lock.
Lai Jiangshanee378aa2012-09-10 10:03:44 -07002039 */
Lai Jiangshanb2eb83d2012-09-18 09:59:23 -07002040 if (unlikely(!mutex_trylock(&pool->assoc_mutex))) {
Tejun Heod565ed62013-01-24 11:01:33 -08002041 spin_unlock_irq(&pool->lock);
Lai Jiangshanb2eb83d2012-09-18 09:59:23 -07002042 mutex_lock(&pool->assoc_mutex);
Lai Jiangshanee378aa2012-09-10 10:03:44 -07002043 /*
2044 * CPU hotplug could have happened while we were waiting
Lai Jiangshanb2eb83d2012-09-18 09:59:23 -07002045 * for assoc_mutex. Hotplug itself can't handle us
Lai Jiangshanee378aa2012-09-10 10:03:44 -07002046 * because manager isn't either on idle or busy list, and
Tejun Heo706026c2013-01-24 11:01:34 -08002047 * @pool's state and ours could have deviated.
Lai Jiangshanee378aa2012-09-10 10:03:44 -07002048 *
Lai Jiangshanb2eb83d2012-09-18 09:59:23 -07002049 * As hotplug is now excluded via assoc_mutex, we can
Lai Jiangshanee378aa2012-09-10 10:03:44 -07002050 * simply try to bind. It will succeed or fail depending
Tejun Heo706026c2013-01-24 11:01:34 -08002051 * on @pool's current state. Try it and adjust
Lai Jiangshanee378aa2012-09-10 10:03:44 -07002052 * %WORKER_UNBOUND accordingly.
2053 */
Lai Jiangshanf36dc672013-02-19 12:17:02 -08002054 if (worker_maybe_bind_and_lock(pool))
Lai Jiangshanee378aa2012-09-10 10:03:44 -07002055 worker->flags &= ~WORKER_UNBOUND;
2056 else
2057 worker->flags |= WORKER_UNBOUND;
2058
2059 ret = true;
2060 }
2061
Tejun Heo11ebea52012-07-12 14:46:37 -07002062 pool->flags &= ~POOL_MANAGE_WORKERS;
Tejun Heoe22bee72010-06-29 10:07:14 +02002063
2064 /*
2065 * Destroy and then create so that may_start_working() is true
2066 * on return.
2067 */
Tejun Heo63d95a92012-07-12 14:46:37 -07002068 ret |= maybe_destroy_workers(pool);
2069 ret |= maybe_create_worker(pool);
Tejun Heoe22bee72010-06-29 10:07:14 +02002070
Lai Jiangshanb2eb83d2012-09-18 09:59:23 -07002071 mutex_unlock(&pool->assoc_mutex);
Tejun Heo34a06bd2013-03-12 11:30:00 -07002072 mutex_unlock(&pool->manager_arb);
Tejun Heoe22bee72010-06-29 10:07:14 +02002073 return ret;
2074}
2075
Tejun Heoa62428c2010-06-29 10:07:10 +02002076/**
2077 * process_one_work - process single work
Tejun Heoc34056a2010-06-29 10:07:11 +02002078 * @worker: self
Tejun Heoa62428c2010-06-29 10:07:10 +02002079 * @work: work to process
2080 *
2081 * Process @work. This function contains all the logics necessary to
2082 * process a single work including synchronization against and
2083 * interaction with other workers on the same cpu, queueing and
2084 * flushing. As long as context requirement is met, any worker can
2085 * call this function to process a work.
2086 *
2087 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08002088 * spin_lock_irq(pool->lock) which is released and regrabbed.
Tejun Heoa62428c2010-06-29 10:07:10 +02002089 */
Tejun Heoc34056a2010-06-29 10:07:11 +02002090static void process_one_work(struct worker *worker, struct work_struct *work)
Tejun Heod565ed62013-01-24 11:01:33 -08002091__releases(&pool->lock)
2092__acquires(&pool->lock)
Tejun Heoa62428c2010-06-29 10:07:10 +02002093{
Tejun Heo112202d2013-02-13 19:29:12 -08002094 struct pool_workqueue *pwq = get_work_pwq(work);
Tejun Heobd7bdd42012-07-12 14:46:37 -07002095 struct worker_pool *pool = worker->pool;
Tejun Heo112202d2013-02-13 19:29:12 -08002096 bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
Tejun Heo73f53c42010-06-29 10:07:11 +02002097 int work_color;
Tejun Heo7e116292010-06-29 10:07:13 +02002098 struct worker *collision;
Tejun Heoa62428c2010-06-29 10:07:10 +02002099#ifdef CONFIG_LOCKDEP
2100 /*
2101 * It is permissible to free the struct work_struct from
2102 * inside the function that is called from it, this we need to
2103 * take into account for lockdep too. To avoid bogus "held
2104 * lock freed" warnings as well as problems when looking into
2105 * work->lockdep_map, make a copy and use that here.
2106 */
Peter Zijlstra4d82a1d2012-05-15 08:06:19 -07002107 struct lockdep_map lockdep_map;
2108
2109 lockdep_copy_map(&lockdep_map, &work->lockdep_map);
Tejun Heoa62428c2010-06-29 10:07:10 +02002110#endif
Tejun Heo6fec10a2012-07-22 10:16:34 -07002111 /*
2112 * Ensure we're on the correct CPU. DISASSOCIATED test is
2113 * necessary to avoid spurious warnings from rescuers servicing the
Tejun Heo24647572013-01-24 11:01:33 -08002114 * unbound or a disassociated pool.
Tejun Heo6fec10a2012-07-22 10:16:34 -07002115 */
Lai Jiangshan5f7dabf2012-09-18 09:59:23 -07002116 WARN_ON_ONCE(!(worker->flags & WORKER_UNBOUND) &&
Tejun Heo24647572013-01-24 11:01:33 -08002117 !(pool->flags & POOL_DISASSOCIATED) &&
Tejun Heoec22ca52013-01-24 11:01:33 -08002118 raw_smp_processor_id() != pool->cpu);
Tejun Heo25511a42012-07-17 12:39:27 -07002119
Tejun Heo7e116292010-06-29 10:07:13 +02002120 /*
2121 * A single work shouldn't be executed concurrently by
2122 * multiple workers on a single cpu. Check whether anyone is
2123 * already processing the work. If so, defer the work to the
2124 * currently executing one.
2125 */
Tejun Heoc9e7cf22013-01-24 11:01:33 -08002126 collision = find_worker_executing_work(pool, work);
Tejun Heo7e116292010-06-29 10:07:13 +02002127 if (unlikely(collision)) {
2128 move_linked_works(work, &collision->scheduled, NULL);
2129 return;
2130 }
2131
Tejun Heo8930cab2012-08-03 10:30:45 -07002132 /* claim and dequeue */
Tejun Heoa62428c2010-06-29 10:07:10 +02002133 debug_work_deactivate(work);
Tejun Heoc9e7cf22013-01-24 11:01:33 -08002134 hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
Tejun Heoc34056a2010-06-29 10:07:11 +02002135 worker->current_work = work;
Tejun Heoa2c1c572012-12-18 10:35:02 -08002136 worker->current_func = work->func;
Tejun Heo112202d2013-02-13 19:29:12 -08002137 worker->current_pwq = pwq;
Tejun Heo73f53c42010-06-29 10:07:11 +02002138 work_color = get_work_color(work);
Tejun Heo7a22ad72010-06-29 10:07:13 +02002139
Tejun Heoa62428c2010-06-29 10:07:10 +02002140 list_del_init(&work->entry);
2141
Tejun Heo649027d2010-06-29 10:07:14 +02002142 /*
Tejun Heofb0e7be2010-06-29 10:07:15 +02002143 * CPU intensive works don't participate in concurrency
2144 * management. They're the scheduler's responsibility.
2145 */
2146 if (unlikely(cpu_intensive))
2147 worker_set_flags(worker, WORKER_CPU_INTENSIVE, true);
2148
Tejun Heo974271c2012-07-12 14:46:37 -07002149 /*
Tejun Heod565ed62013-01-24 11:01:33 -08002150 * Unbound pool isn't concurrency managed and work items should be
Tejun Heo974271c2012-07-12 14:46:37 -07002151 * executed ASAP. Wake up another worker if necessary.
2152 */
Tejun Heo63d95a92012-07-12 14:46:37 -07002153 if ((worker->flags & WORKER_UNBOUND) && need_more_worker(pool))
2154 wake_up_worker(pool);
Tejun Heo974271c2012-07-12 14:46:37 -07002155
Tejun Heo8930cab2012-08-03 10:30:45 -07002156 /*
Tejun Heo7c3eed52013-01-24 11:01:33 -08002157 * Record the last pool and clear PENDING which should be the last
Tejun Heod565ed62013-01-24 11:01:33 -08002158 * update to @work. Also, do this inside @pool->lock so that
Tejun Heo23657bb2012-08-13 17:08:19 -07002159 * PENDING and queued state changes happen together while IRQ is
2160 * disabled.
Tejun Heo8930cab2012-08-03 10:30:45 -07002161 */
Tejun Heo7c3eed52013-01-24 11:01:33 -08002162 set_work_pool_and_clear_pending(work, pool->id);
Tejun Heoa62428c2010-06-29 10:07:10 +02002163
Tejun Heod565ed62013-01-24 11:01:33 -08002164 spin_unlock_irq(&pool->lock);
Tejun Heoa62428c2010-06-29 10:07:10 +02002165
Tejun Heo112202d2013-02-13 19:29:12 -08002166 lock_map_acquire_read(&pwq->wq->lockdep_map);
Tejun Heoa62428c2010-06-29 10:07:10 +02002167 lock_map_acquire(&lockdep_map);
Arjan van de Vene36c8862010-08-21 13:07:26 -07002168 trace_workqueue_execute_start(work);
Tejun Heoa2c1c572012-12-18 10:35:02 -08002169 worker->current_func(work);
Arjan van de Vene36c8862010-08-21 13:07:26 -07002170 /*
2171 * While we must be careful to not use "work" after this, the trace
2172 * point will only record its address.
2173 */
2174 trace_workqueue_execute_end(work);
Tejun Heoa62428c2010-06-29 10:07:10 +02002175 lock_map_release(&lockdep_map);
Tejun Heo112202d2013-02-13 19:29:12 -08002176 lock_map_release(&pwq->wq->lockdep_map);
Tejun Heoa62428c2010-06-29 10:07:10 +02002177
2178 if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
Valentin Ilie044c7822012-08-19 00:52:42 +03002179 pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
2180 " last function: %pf\n",
Tejun Heoa2c1c572012-12-18 10:35:02 -08002181 current->comm, preempt_count(), task_pid_nr(current),
2182 worker->current_func);
Tejun Heoa62428c2010-06-29 10:07:10 +02002183 debug_show_held_locks(current);
2184 dump_stack();
2185 }
2186
Tejun Heod565ed62013-01-24 11:01:33 -08002187 spin_lock_irq(&pool->lock);
Tejun Heoa62428c2010-06-29 10:07:10 +02002188
Tejun Heofb0e7be2010-06-29 10:07:15 +02002189 /* clear cpu intensive status */
2190 if (unlikely(cpu_intensive))
2191 worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2192
Tejun Heoa62428c2010-06-29 10:07:10 +02002193 /* we're done with it, release */
Sasha Levin42f85702012-12-17 10:01:23 -05002194 hash_del(&worker->hentry);
Tejun Heoc34056a2010-06-29 10:07:11 +02002195 worker->current_work = NULL;
Tejun Heoa2c1c572012-12-18 10:35:02 -08002196 worker->current_func = NULL;
Tejun Heo112202d2013-02-13 19:29:12 -08002197 worker->current_pwq = NULL;
2198 pwq_dec_nr_in_flight(pwq, work_color);
Tejun Heoa62428c2010-06-29 10:07:10 +02002199}
2200
Tejun Heoaffee4b2010-06-29 10:07:12 +02002201/**
2202 * process_scheduled_works - process scheduled works
2203 * @worker: self
2204 *
2205 * Process all scheduled works. Please note that the scheduled list
2206 * may change while processing a work, so this function repeatedly
2207 * fetches a work from the top and executes it.
2208 *
2209 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08002210 * spin_lock_irq(pool->lock) which may be released and regrabbed
Tejun Heoaffee4b2010-06-29 10:07:12 +02002211 * multiple times.
2212 */
2213static void process_scheduled_works(struct worker *worker)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002214{
Tejun Heoaffee4b2010-06-29 10:07:12 +02002215 while (!list_empty(&worker->scheduled)) {
2216 struct work_struct *work = list_first_entry(&worker->scheduled,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002217 struct work_struct, entry);
Tejun Heoc34056a2010-06-29 10:07:11 +02002218 process_one_work(worker, work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002219 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002220}
2221
Tejun Heo4690c4a2010-06-29 10:07:10 +02002222/**
2223 * worker_thread - the worker thread function
Tejun Heoc34056a2010-06-29 10:07:11 +02002224 * @__worker: self
Tejun Heo4690c4a2010-06-29 10:07:10 +02002225 *
Tejun Heoc5aa87b2013-03-13 16:51:36 -07002226 * The worker thread function. All workers belong to a worker_pool -
2227 * either a per-cpu one or dynamic unbound one. These workers process all
2228 * work items regardless of their specific target workqueue. The only
2229 * exception is work items which belong to workqueues with a rescuer which
2230 * will be explained in rescuer_thread().
Tejun Heo4690c4a2010-06-29 10:07:10 +02002231 */
Tejun Heoc34056a2010-06-29 10:07:11 +02002232static int worker_thread(void *__worker)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002233{
Tejun Heoc34056a2010-06-29 10:07:11 +02002234 struct worker *worker = __worker;
Tejun Heobd7bdd42012-07-12 14:46:37 -07002235 struct worker_pool *pool = worker->pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002236
Tejun Heoe22bee72010-06-29 10:07:14 +02002237 /* tell the scheduler that this is a workqueue worker */
2238 worker->task->flags |= PF_WQ_WORKER;
Tejun Heoc8e55f32010-06-29 10:07:12 +02002239woke_up:
Tejun Heod565ed62013-01-24 11:01:33 -08002240 spin_lock_irq(&pool->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002241
Lai Jiangshan5f7dabf2012-09-18 09:59:23 -07002242 /* we are off idle list if destruction or rebind is requested */
2243 if (unlikely(list_empty(&worker->entry))) {
Tejun Heod565ed62013-01-24 11:01:33 -08002244 spin_unlock_irq(&pool->lock);
Tejun Heo25511a42012-07-17 12:39:27 -07002245
Lai Jiangshan5f7dabf2012-09-18 09:59:23 -07002246 /* if DIE is set, destruction is requested */
Tejun Heo25511a42012-07-17 12:39:27 -07002247 if (worker->flags & WORKER_DIE) {
2248 worker->task->flags &= ~PF_WQ_WORKER;
2249 return 0;
2250 }
2251
Lai Jiangshan5f7dabf2012-09-18 09:59:23 -07002252 /* otherwise, rebind */
Tejun Heo25511a42012-07-17 12:39:27 -07002253 idle_worker_rebind(worker);
2254 goto woke_up;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002255 }
2256
Tejun Heoc8e55f32010-06-29 10:07:12 +02002257 worker_leave_idle(worker);
Tejun Heodb7bccf2010-06-29 10:07:12 +02002258recheck:
Tejun Heoe22bee72010-06-29 10:07:14 +02002259 /* no more worker necessary? */
Tejun Heo63d95a92012-07-12 14:46:37 -07002260 if (!need_more_worker(pool))
Tejun Heoe22bee72010-06-29 10:07:14 +02002261 goto sleep;
2262
2263 /* do we need to manage? */
Tejun Heo63d95a92012-07-12 14:46:37 -07002264 if (unlikely(!may_start_working(pool)) && manage_workers(worker))
Tejun Heoe22bee72010-06-29 10:07:14 +02002265 goto recheck;
2266
Tejun Heoc8e55f32010-06-29 10:07:12 +02002267 /*
2268 * ->scheduled list can only be filled while a worker is
2269 * preparing to process a work or actually processing it.
2270 * Make sure nobody diddled with it while I was sleeping.
2271 */
Tejun Heo6183c002013-03-12 11:29:57 -07002272 WARN_ON_ONCE(!list_empty(&worker->scheduled));
Tejun Heoc8e55f32010-06-29 10:07:12 +02002273
Tejun Heoe22bee72010-06-29 10:07:14 +02002274 /*
2275 * When control reaches this point, we're guaranteed to have
2276 * at least one idle worker or that someone else has already
2277 * assumed the manager role.
2278 */
2279 worker_clr_flags(worker, WORKER_PREP);
2280
2281 do {
Tejun Heoc8e55f32010-06-29 10:07:12 +02002282 struct work_struct *work =
Tejun Heobd7bdd42012-07-12 14:46:37 -07002283 list_first_entry(&pool->worklist,
Tejun Heoc8e55f32010-06-29 10:07:12 +02002284 struct work_struct, entry);
2285
2286 if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
2287 /* optimization path, not strictly necessary */
2288 process_one_work(worker, work);
2289 if (unlikely(!list_empty(&worker->scheduled)))
2290 process_scheduled_works(worker);
2291 } else {
2292 move_linked_works(work, &worker->scheduled, NULL);
2293 process_scheduled_works(worker);
2294 }
Tejun Heo63d95a92012-07-12 14:46:37 -07002295 } while (keep_working(pool));
Tejun Heoc8e55f32010-06-29 10:07:12 +02002296
Tejun Heoe22bee72010-06-29 10:07:14 +02002297 worker_set_flags(worker, WORKER_PREP, false);
Tejun Heod313dd82010-07-02 10:03:51 +02002298sleep:
Tejun Heo63d95a92012-07-12 14:46:37 -07002299 if (unlikely(need_to_manage_workers(pool)) && manage_workers(worker))
Tejun Heoe22bee72010-06-29 10:07:14 +02002300 goto recheck;
Tejun Heod313dd82010-07-02 10:03:51 +02002301
Tejun Heoc8e55f32010-06-29 10:07:12 +02002302 /*
Tejun Heod565ed62013-01-24 11:01:33 -08002303 * pool->lock is held and there's no work to process and no need to
2304 * manage, sleep. Workers are woken up only while holding
2305 * pool->lock or from local cpu, so setting the current state
2306 * before releasing pool->lock is enough to prevent losing any
2307 * event.
Tejun Heoc8e55f32010-06-29 10:07:12 +02002308 */
2309 worker_enter_idle(worker);
2310 __set_current_state(TASK_INTERRUPTIBLE);
Tejun Heod565ed62013-01-24 11:01:33 -08002311 spin_unlock_irq(&pool->lock);
Tejun Heoc8e55f32010-06-29 10:07:12 +02002312 schedule();
2313 goto woke_up;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002314}
2315
Tejun Heoe22bee72010-06-29 10:07:14 +02002316/**
2317 * rescuer_thread - the rescuer thread function
Tejun Heo111c2252013-01-17 17:16:24 -08002318 * @__rescuer: self
Tejun Heoe22bee72010-06-29 10:07:14 +02002319 *
2320 * Workqueue rescuer thread function. There's one rescuer for each
Tejun Heo493008a2013-03-12 11:30:03 -07002321 * workqueue which has WQ_MEM_RECLAIM set.
Tejun Heoe22bee72010-06-29 10:07:14 +02002322 *
Tejun Heo706026c2013-01-24 11:01:34 -08002323 * Regular work processing on a pool may block trying to create a new
Tejun Heoe22bee72010-06-29 10:07:14 +02002324 * worker which uses GFP_KERNEL allocation which has slight chance of
2325 * developing into deadlock if some works currently on the same queue
2326 * need to be processed to satisfy the GFP_KERNEL allocation. This is
2327 * the problem rescuer solves.
2328 *
Tejun Heo706026c2013-01-24 11:01:34 -08002329 * When such condition is possible, the pool summons rescuers of all
2330 * workqueues which have works queued on the pool and let them process
Tejun Heoe22bee72010-06-29 10:07:14 +02002331 * those works so that forward progress can be guaranteed.
2332 *
2333 * This should happen rarely.
2334 */
Tejun Heo111c2252013-01-17 17:16:24 -08002335static int rescuer_thread(void *__rescuer)
Tejun Heoe22bee72010-06-29 10:07:14 +02002336{
Tejun Heo111c2252013-01-17 17:16:24 -08002337 struct worker *rescuer = __rescuer;
2338 struct workqueue_struct *wq = rescuer->rescue_wq;
Tejun Heoe22bee72010-06-29 10:07:14 +02002339 struct list_head *scheduled = &rescuer->scheduled;
Tejun Heoe22bee72010-06-29 10:07:14 +02002340
2341 set_user_nice(current, RESCUER_NICE_LEVEL);
Tejun Heo111c2252013-01-17 17:16:24 -08002342
2343 /*
2344 * Mark rescuer as worker too. As WORKER_PREP is never cleared, it
2345 * doesn't participate in concurrency management.
2346 */
2347 rescuer->task->flags |= PF_WQ_WORKER;
Tejun Heoe22bee72010-06-29 10:07:14 +02002348repeat:
2349 set_current_state(TASK_INTERRUPTIBLE);
2350
Mike Galbraith412d32e2012-11-28 07:17:18 +01002351 if (kthread_should_stop()) {
2352 __set_current_state(TASK_RUNNING);
Tejun Heo111c2252013-01-17 17:16:24 -08002353 rescuer->task->flags &= ~PF_WQ_WORKER;
Tejun Heoe22bee72010-06-29 10:07:14 +02002354 return 0;
Mike Galbraith412d32e2012-11-28 07:17:18 +01002355 }
Tejun Heoe22bee72010-06-29 10:07:14 +02002356
Tejun Heo493a1722013-03-12 11:29:59 -07002357 /* see whether any pwq is asking for help */
2358 spin_lock_irq(&workqueue_lock);
2359
2360 while (!list_empty(&wq->maydays)) {
2361 struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
2362 struct pool_workqueue, mayday_node);
Tejun Heo112202d2013-02-13 19:29:12 -08002363 struct worker_pool *pool = pwq->pool;
Tejun Heoe22bee72010-06-29 10:07:14 +02002364 struct work_struct *work, *n;
2365
2366 __set_current_state(TASK_RUNNING);
Tejun Heo493a1722013-03-12 11:29:59 -07002367 list_del_init(&pwq->mayday_node);
2368
2369 spin_unlock_irq(&workqueue_lock);
Tejun Heoe22bee72010-06-29 10:07:14 +02002370
2371 /* migrate to the target cpu if possible */
Lai Jiangshanf36dc672013-02-19 12:17:02 -08002372 worker_maybe_bind_and_lock(pool);
Lai Jiangshanb3104102013-02-19 12:17:02 -08002373 rescuer->pool = pool;
Tejun Heoe22bee72010-06-29 10:07:14 +02002374
2375 /*
2376 * Slurp in all works issued via this workqueue and
2377 * process'em.
2378 */
Tejun Heo6183c002013-03-12 11:29:57 -07002379 WARN_ON_ONCE(!list_empty(&rescuer->scheduled));
Tejun Heobd7bdd42012-07-12 14:46:37 -07002380 list_for_each_entry_safe(work, n, &pool->worklist, entry)
Tejun Heo112202d2013-02-13 19:29:12 -08002381 if (get_work_pwq(work) == pwq)
Tejun Heoe22bee72010-06-29 10:07:14 +02002382 move_linked_works(work, scheduled, &n);
2383
2384 process_scheduled_works(rescuer);
Tejun Heo75769582011-02-14 14:04:46 +01002385
2386 /*
Tejun Heod565ed62013-01-24 11:01:33 -08002387 * Leave this pool. If keep_working() is %true, notify a
Tejun Heo75769582011-02-14 14:04:46 +01002388 * regular worker; otherwise, we end up with 0 concurrency
2389 * and stalling the execution.
2390 */
Tejun Heo63d95a92012-07-12 14:46:37 -07002391 if (keep_working(pool))
2392 wake_up_worker(pool);
Tejun Heo75769582011-02-14 14:04:46 +01002393
Lai Jiangshanb3104102013-02-19 12:17:02 -08002394 rescuer->pool = NULL;
Tejun Heo493a1722013-03-12 11:29:59 -07002395 spin_unlock(&pool->lock);
2396 spin_lock(&workqueue_lock);
Tejun Heoe22bee72010-06-29 10:07:14 +02002397 }
2398
Tejun Heo493a1722013-03-12 11:29:59 -07002399 spin_unlock_irq(&workqueue_lock);
2400
Tejun Heo111c2252013-01-17 17:16:24 -08002401 /* rescuers should never participate in concurrency management */
2402 WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
Tejun Heoe22bee72010-06-29 10:07:14 +02002403 schedule();
2404 goto repeat;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002405}
2406
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -07002407struct wq_barrier {
2408 struct work_struct work;
2409 struct completion done;
2410};
2411
2412static void wq_barrier_func(struct work_struct *work)
2413{
2414 struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
2415 complete(&barr->done);
2416}
2417
Tejun Heo4690c4a2010-06-29 10:07:10 +02002418/**
2419 * insert_wq_barrier - insert a barrier work
Tejun Heo112202d2013-02-13 19:29:12 -08002420 * @pwq: pwq to insert barrier into
Tejun Heo4690c4a2010-06-29 10:07:10 +02002421 * @barr: wq_barrier to insert
Tejun Heoaffee4b2010-06-29 10:07:12 +02002422 * @target: target work to attach @barr to
2423 * @worker: worker currently executing @target, NULL if @target is not executing
Tejun Heo4690c4a2010-06-29 10:07:10 +02002424 *
Tejun Heoaffee4b2010-06-29 10:07:12 +02002425 * @barr is linked to @target such that @barr is completed only after
2426 * @target finishes execution. Please note that the ordering
2427 * guarantee is observed only with respect to @target and on the local
2428 * cpu.
2429 *
2430 * Currently, a queued barrier can't be canceled. This is because
2431 * try_to_grab_pending() can't determine whether the work to be
2432 * grabbed is at the head of the queue and thus can't clear LINKED
2433 * flag of the previous work while there must be a valid next work
2434 * after a work with LINKED flag set.
2435 *
2436 * Note that when @worker is non-NULL, @target may be modified
Tejun Heo112202d2013-02-13 19:29:12 -08002437 * underneath us, so we can't reliably determine pwq from @target.
Tejun Heo4690c4a2010-06-29 10:07:10 +02002438 *
2439 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08002440 * spin_lock_irq(pool->lock).
Tejun Heo4690c4a2010-06-29 10:07:10 +02002441 */
Tejun Heo112202d2013-02-13 19:29:12 -08002442static void insert_wq_barrier(struct pool_workqueue *pwq,
Tejun Heoaffee4b2010-06-29 10:07:12 +02002443 struct wq_barrier *barr,
2444 struct work_struct *target, struct worker *worker)
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -07002445{
Tejun Heoaffee4b2010-06-29 10:07:12 +02002446 struct list_head *head;
2447 unsigned int linked = 0;
2448
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +09002449 /*
Tejun Heod565ed62013-01-24 11:01:33 -08002450 * debugobject calls are safe here even with pool->lock locked
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +09002451 * as we know for sure that this will not trigger any of the
2452 * checks and call back into the fixup functions where we
2453 * might deadlock.
2454 */
Andrew Mortonca1cab32010-10-26 14:22:34 -07002455 INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
Tejun Heo22df02b2010-06-29 10:07:10 +02002456 __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -07002457 init_completion(&barr->done);
Oleg Nesterov83c22522007-05-09 02:33:54 -07002458
Tejun Heoaffee4b2010-06-29 10:07:12 +02002459 /*
2460 * If @target is currently being executed, schedule the
2461 * barrier to the worker; otherwise, put it after @target.
2462 */
2463 if (worker)
2464 head = worker->scheduled.next;
2465 else {
2466 unsigned long *bits = work_data_bits(target);
2467
2468 head = target->entry.next;
2469 /* there can already be other linked works, inherit and set */
2470 linked = *bits & WORK_STRUCT_LINKED;
2471 __set_bit(WORK_STRUCT_LINKED_BIT, bits);
2472 }
2473
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +09002474 debug_work_activate(&barr->work);
Tejun Heo112202d2013-02-13 19:29:12 -08002475 insert_work(pwq, &barr->work, head,
Tejun Heoaffee4b2010-06-29 10:07:12 +02002476 work_color_to_flags(WORK_NO_COLOR) | linked);
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -07002477}
2478
Tejun Heo73f53c42010-06-29 10:07:11 +02002479/**
Tejun Heo112202d2013-02-13 19:29:12 -08002480 * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
Tejun Heo73f53c42010-06-29 10:07:11 +02002481 * @wq: workqueue being flushed
2482 * @flush_color: new flush color, < 0 for no-op
2483 * @work_color: new work color, < 0 for no-op
2484 *
Tejun Heo112202d2013-02-13 19:29:12 -08002485 * Prepare pwqs for workqueue flushing.
Tejun Heo73f53c42010-06-29 10:07:11 +02002486 *
Tejun Heo112202d2013-02-13 19:29:12 -08002487 * If @flush_color is non-negative, flush_color on all pwqs should be
2488 * -1. If no pwq has in-flight commands at the specified color, all
2489 * pwq->flush_color's stay at -1 and %false is returned. If any pwq
2490 * has in flight commands, its pwq->flush_color is set to
2491 * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
Tejun Heo73f53c42010-06-29 10:07:11 +02002492 * wakeup logic is armed and %true is returned.
2493 *
2494 * The caller should have initialized @wq->first_flusher prior to
2495 * calling this function with non-negative @flush_color. If
2496 * @flush_color is negative, no flush color update is done and %false
2497 * is returned.
2498 *
Tejun Heo112202d2013-02-13 19:29:12 -08002499 * If @work_color is non-negative, all pwqs should have the same
Tejun Heo73f53c42010-06-29 10:07:11 +02002500 * work_color which is previous to @work_color and all will be
2501 * advanced to @work_color.
2502 *
2503 * CONTEXT:
2504 * mutex_lock(wq->flush_mutex).
2505 *
2506 * RETURNS:
2507 * %true if @flush_color >= 0 and there's something to flush. %false
2508 * otherwise.
2509 */
Tejun Heo112202d2013-02-13 19:29:12 -08002510static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
Tejun Heo73f53c42010-06-29 10:07:11 +02002511 int flush_color, int work_color)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002512{
Tejun Heo73f53c42010-06-29 10:07:11 +02002513 bool wait = false;
Tejun Heo49e3cf42013-03-12 11:29:58 -07002514 struct pool_workqueue *pwq;
Oleg Nesterov14441962007-05-23 13:57:57 -07002515
Tejun Heo73f53c42010-06-29 10:07:11 +02002516 if (flush_color >= 0) {
Tejun Heo6183c002013-03-12 11:29:57 -07002517 WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
Tejun Heo112202d2013-02-13 19:29:12 -08002518 atomic_set(&wq->nr_pwqs_to_flush, 1);
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +09002519 }
Oleg Nesterov14441962007-05-23 13:57:57 -07002520
Tejun Heo76af4d92013-03-12 11:30:00 -07002521 local_irq_disable();
2522
Tejun Heo49e3cf42013-03-12 11:29:58 -07002523 for_each_pwq(pwq, wq) {
Tejun Heo112202d2013-02-13 19:29:12 -08002524 struct worker_pool *pool = pwq->pool;
Tejun Heo73f53c42010-06-29 10:07:11 +02002525
Tejun Heo76af4d92013-03-12 11:30:00 -07002526 spin_lock(&pool->lock);
Tejun Heo73f53c42010-06-29 10:07:11 +02002527
2528 if (flush_color >= 0) {
Tejun Heo6183c002013-03-12 11:29:57 -07002529 WARN_ON_ONCE(pwq->flush_color != -1);
Tejun Heo73f53c42010-06-29 10:07:11 +02002530
Tejun Heo112202d2013-02-13 19:29:12 -08002531 if (pwq->nr_in_flight[flush_color]) {
2532 pwq->flush_color = flush_color;
2533 atomic_inc(&wq->nr_pwqs_to_flush);
Tejun Heo73f53c42010-06-29 10:07:11 +02002534 wait = true;
2535 }
2536 }
2537
2538 if (work_color >= 0) {
Tejun Heo6183c002013-03-12 11:29:57 -07002539 WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
Tejun Heo112202d2013-02-13 19:29:12 -08002540 pwq->work_color = work_color;
Tejun Heo73f53c42010-06-29 10:07:11 +02002541 }
2542
Tejun Heo76af4d92013-03-12 11:30:00 -07002543 spin_unlock(&pool->lock);
Tejun Heo73f53c42010-06-29 10:07:11 +02002544 }
2545
Tejun Heo76af4d92013-03-12 11:30:00 -07002546 local_irq_enable();
2547
Tejun Heo112202d2013-02-13 19:29:12 -08002548 if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
Tejun Heo73f53c42010-06-29 10:07:11 +02002549 complete(&wq->first_flusher->done);
2550
2551 return wait;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002552}
2553
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -07002554/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07002555 * flush_workqueue - ensure that any scheduled work has run to completion.
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -07002556 * @wq: workqueue to flush
Linus Torvalds1da177e2005-04-16 15:20:36 -07002557 *
Tejun Heoc5aa87b2013-03-13 16:51:36 -07002558 * This function sleeps until all work items which were queued on entry
2559 * have finished execution, but it is not livelocked by new incoming ones.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002560 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -08002561void flush_workqueue(struct workqueue_struct *wq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002562{
Tejun Heo73f53c42010-06-29 10:07:11 +02002563 struct wq_flusher this_flusher = {
2564 .list = LIST_HEAD_INIT(this_flusher.list),
2565 .flush_color = -1,
2566 .done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done),
2567 };
2568 int next_color;
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -07002569
Ingo Molnar3295f0e2008-08-11 10:30:30 +02002570 lock_map_acquire(&wq->lockdep_map);
2571 lock_map_release(&wq->lockdep_map);
Tejun Heo73f53c42010-06-29 10:07:11 +02002572
2573 mutex_lock(&wq->flush_mutex);
2574
2575 /*
2576 * Start-to-wait phase
2577 */
2578 next_color = work_next_color(wq->work_color);
2579
2580 if (next_color != wq->flush_color) {
2581 /*
2582 * Color space is not full. The current work_color
2583 * becomes our flush_color and work_color is advanced
2584 * by one.
2585 */
Tejun Heo6183c002013-03-12 11:29:57 -07002586 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
Tejun Heo73f53c42010-06-29 10:07:11 +02002587 this_flusher.flush_color = wq->work_color;
2588 wq->work_color = next_color;
2589
2590 if (!wq->first_flusher) {
2591 /* no flush in progress, become the first flusher */
Tejun Heo6183c002013-03-12 11:29:57 -07002592 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
Tejun Heo73f53c42010-06-29 10:07:11 +02002593
2594 wq->first_flusher = &this_flusher;
2595
Tejun Heo112202d2013-02-13 19:29:12 -08002596 if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
Tejun Heo73f53c42010-06-29 10:07:11 +02002597 wq->work_color)) {
2598 /* nothing to flush, done */
2599 wq->flush_color = next_color;
2600 wq->first_flusher = NULL;
2601 goto out_unlock;
2602 }
2603 } else {
2604 /* wait in queue */
Tejun Heo6183c002013-03-12 11:29:57 -07002605 WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
Tejun Heo73f53c42010-06-29 10:07:11 +02002606 list_add_tail(&this_flusher.list, &wq->flusher_queue);
Tejun Heo112202d2013-02-13 19:29:12 -08002607 flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
Tejun Heo73f53c42010-06-29 10:07:11 +02002608 }
2609 } else {
2610 /*
2611 * Oops, color space is full, wait on overflow queue.
2612 * The next flush completion will assign us
2613 * flush_color and transfer to flusher_queue.
2614 */
2615 list_add_tail(&this_flusher.list, &wq->flusher_overflow);
2616 }
2617
2618 mutex_unlock(&wq->flush_mutex);
2619
2620 wait_for_completion(&this_flusher.done);
2621
2622 /*
2623 * Wake-up-and-cascade phase
2624 *
2625 * First flushers are responsible for cascading flushes and
2626 * handling overflow. Non-first flushers can simply return.
2627 */
2628 if (wq->first_flusher != &this_flusher)
2629 return;
2630
2631 mutex_lock(&wq->flush_mutex);
2632
Tejun Heo4ce48b32010-07-02 10:03:51 +02002633 /* we might have raced, check again with mutex held */
2634 if (wq->first_flusher != &this_flusher)
2635 goto out_unlock;
2636
Tejun Heo73f53c42010-06-29 10:07:11 +02002637 wq->first_flusher = NULL;
2638
Tejun Heo6183c002013-03-12 11:29:57 -07002639 WARN_ON_ONCE(!list_empty(&this_flusher.list));
2640 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
Tejun Heo73f53c42010-06-29 10:07:11 +02002641
2642 while (true) {
2643 struct wq_flusher *next, *tmp;
2644
2645 /* complete all the flushers sharing the current flush color */
2646 list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
2647 if (next->flush_color != wq->flush_color)
2648 break;
2649 list_del_init(&next->list);
2650 complete(&next->done);
2651 }
2652
Tejun Heo6183c002013-03-12 11:29:57 -07002653 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
2654 wq->flush_color != work_next_color(wq->work_color));
Tejun Heo73f53c42010-06-29 10:07:11 +02002655
2656 /* this flush_color is finished, advance by one */
2657 wq->flush_color = work_next_color(wq->flush_color);
2658
2659 /* one color has been freed, handle overflow queue */
2660 if (!list_empty(&wq->flusher_overflow)) {
2661 /*
2662 * Assign the same color to all overflowed
2663 * flushers, advance work_color and append to
2664 * flusher_queue. This is the start-to-wait
2665 * phase for these overflowed flushers.
2666 */
2667 list_for_each_entry(tmp, &wq->flusher_overflow, list)
2668 tmp->flush_color = wq->work_color;
2669
2670 wq->work_color = work_next_color(wq->work_color);
2671
2672 list_splice_tail_init(&wq->flusher_overflow,
2673 &wq->flusher_queue);
Tejun Heo112202d2013-02-13 19:29:12 -08002674 flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
Tejun Heo73f53c42010-06-29 10:07:11 +02002675 }
2676
2677 if (list_empty(&wq->flusher_queue)) {
Tejun Heo6183c002013-03-12 11:29:57 -07002678 WARN_ON_ONCE(wq->flush_color != wq->work_color);
Tejun Heo73f53c42010-06-29 10:07:11 +02002679 break;
2680 }
2681
2682 /*
2683 * Need to flush more colors. Make the next flusher
Tejun Heo112202d2013-02-13 19:29:12 -08002684 * the new first flusher and arm pwqs.
Tejun Heo73f53c42010-06-29 10:07:11 +02002685 */
Tejun Heo6183c002013-03-12 11:29:57 -07002686 WARN_ON_ONCE(wq->flush_color == wq->work_color);
2687 WARN_ON_ONCE(wq->flush_color != next->flush_color);
Tejun Heo73f53c42010-06-29 10:07:11 +02002688
2689 list_del_init(&next->list);
2690 wq->first_flusher = next;
2691
Tejun Heo112202d2013-02-13 19:29:12 -08002692 if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
Tejun Heo73f53c42010-06-29 10:07:11 +02002693 break;
2694
2695 /*
2696 * Meh... this color is already done, clear first
2697 * flusher and repeat cascading.
2698 */
2699 wq->first_flusher = NULL;
2700 }
2701
2702out_unlock:
2703 mutex_unlock(&wq->flush_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002704}
Dave Jonesae90dd52006-06-30 01:40:45 -04002705EXPORT_SYMBOL_GPL(flush_workqueue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002706
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002707/**
2708 * drain_workqueue - drain a workqueue
2709 * @wq: workqueue to drain
2710 *
2711 * Wait until the workqueue becomes empty. While draining is in progress,
2712 * only chain queueing is allowed. IOW, only currently pending or running
2713 * work items on @wq can queue further work items on it. @wq is flushed
2714 * repeatedly until it becomes empty. The number of flushing is detemined
2715 * by the depth of chaining and should be relatively short. Whine if it
2716 * takes too long.
2717 */
2718void drain_workqueue(struct workqueue_struct *wq)
2719{
2720 unsigned int flush_cnt = 0;
Tejun Heo49e3cf42013-03-12 11:29:58 -07002721 struct pool_workqueue *pwq;
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002722
2723 /*
2724 * __queue_work() needs to test whether there are drainers, is much
2725 * hotter than drain_workqueue() and already looks at @wq->flags.
Tejun Heo618b01e2013-03-12 11:30:04 -07002726 * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002727 */
Tejun Heoe98d5b12013-03-12 11:29:57 -07002728 spin_lock_irq(&workqueue_lock);
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002729 if (!wq->nr_drainers++)
Tejun Heo618b01e2013-03-12 11:30:04 -07002730 wq->flags |= __WQ_DRAINING;
Tejun Heoe98d5b12013-03-12 11:29:57 -07002731 spin_unlock_irq(&workqueue_lock);
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002732reflush:
2733 flush_workqueue(wq);
2734
Tejun Heo76af4d92013-03-12 11:30:00 -07002735 local_irq_disable();
2736
Tejun Heo49e3cf42013-03-12 11:29:58 -07002737 for_each_pwq(pwq, wq) {
Thomas Tuttlefa2563e2011-09-14 16:22:28 -07002738 bool drained;
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002739
Tejun Heo76af4d92013-03-12 11:30:00 -07002740 spin_lock(&pwq->pool->lock);
Tejun Heo112202d2013-02-13 19:29:12 -08002741 drained = !pwq->nr_active && list_empty(&pwq->delayed_works);
Tejun Heo76af4d92013-03-12 11:30:00 -07002742 spin_unlock(&pwq->pool->lock);
Thomas Tuttlefa2563e2011-09-14 16:22:28 -07002743
2744 if (drained)
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002745 continue;
2746
2747 if (++flush_cnt == 10 ||
2748 (flush_cnt % 100 == 0 && flush_cnt <= 1000))
Tejun Heoc5aa87b2013-03-13 16:51:36 -07002749 pr_warn("workqueue %s: drain_workqueue() isn't complete after %u tries\n",
Valentin Ilie044c7822012-08-19 00:52:42 +03002750 wq->name, flush_cnt);
Tejun Heo76af4d92013-03-12 11:30:00 -07002751
2752 local_irq_enable();
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002753 goto reflush;
2754 }
2755
Tejun Heo76af4d92013-03-12 11:30:00 -07002756 spin_lock(&workqueue_lock);
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002757 if (!--wq->nr_drainers)
Tejun Heo618b01e2013-03-12 11:30:04 -07002758 wq->flags &= ~__WQ_DRAINING;
Tejun Heo76af4d92013-03-12 11:30:00 -07002759 spin_unlock(&workqueue_lock);
2760
2761 local_irq_enable();
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02002762}
2763EXPORT_SYMBOL_GPL(drain_workqueue);
2764
Tejun Heo606a5022012-08-20 14:51:23 -07002765static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr)
Tejun Heobaf59022010-09-16 10:42:16 +02002766{
2767 struct worker *worker = NULL;
Tejun Heoc9e7cf22013-01-24 11:01:33 -08002768 struct worker_pool *pool;
Tejun Heo112202d2013-02-13 19:29:12 -08002769 struct pool_workqueue *pwq;
Tejun Heobaf59022010-09-16 10:42:16 +02002770
2771 might_sleep();
Tejun Heobaf59022010-09-16 10:42:16 +02002772
Tejun Heofa1b54e2013-03-12 11:30:00 -07002773 local_irq_disable();
2774 pool = get_work_pool(work);
2775 if (!pool) {
2776 local_irq_enable();
2777 return false;
2778 }
2779
2780 spin_lock(&pool->lock);
Lai Jiangshan0b3dae62013-02-06 18:04:53 -08002781 /* see the comment in try_to_grab_pending() with the same code */
Tejun Heo112202d2013-02-13 19:29:12 -08002782 pwq = get_work_pwq(work);
2783 if (pwq) {
2784 if (unlikely(pwq->pool != pool))
Tejun Heobaf59022010-09-16 10:42:16 +02002785 goto already_gone;
Tejun Heo606a5022012-08-20 14:51:23 -07002786 } else {
Tejun Heoc9e7cf22013-01-24 11:01:33 -08002787 worker = find_worker_executing_work(pool, work);
Tejun Heobaf59022010-09-16 10:42:16 +02002788 if (!worker)
2789 goto already_gone;
Tejun Heo112202d2013-02-13 19:29:12 -08002790 pwq = worker->current_pwq;
Tejun Heo606a5022012-08-20 14:51:23 -07002791 }
Tejun Heobaf59022010-09-16 10:42:16 +02002792
Tejun Heo112202d2013-02-13 19:29:12 -08002793 insert_wq_barrier(pwq, barr, work, worker);
Tejun Heod565ed62013-01-24 11:01:33 -08002794 spin_unlock_irq(&pool->lock);
Tejun Heobaf59022010-09-16 10:42:16 +02002795
Tejun Heoe1594892011-01-09 23:32:15 +01002796 /*
2797 * If @max_active is 1 or rescuer is in use, flushing another work
2798 * item on the same workqueue may lead to deadlock. Make sure the
2799 * flusher is not running on the same workqueue by verifying write
2800 * access.
2801 */
Tejun Heo493008a2013-03-12 11:30:03 -07002802 if (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)
Tejun Heo112202d2013-02-13 19:29:12 -08002803 lock_map_acquire(&pwq->wq->lockdep_map);
Tejun Heoe1594892011-01-09 23:32:15 +01002804 else
Tejun Heo112202d2013-02-13 19:29:12 -08002805 lock_map_acquire_read(&pwq->wq->lockdep_map);
2806 lock_map_release(&pwq->wq->lockdep_map);
Tejun Heoe1594892011-01-09 23:32:15 +01002807
Tejun Heobaf59022010-09-16 10:42:16 +02002808 return true;
2809already_gone:
Tejun Heod565ed62013-01-24 11:01:33 -08002810 spin_unlock_irq(&pool->lock);
Tejun Heobaf59022010-09-16 10:42:16 +02002811 return false;
2812}
2813
Oleg Nesterovdb700892008-07-25 01:47:49 -07002814/**
Tejun Heo401a8d02010-09-16 10:36:00 +02002815 * flush_work - wait for a work to finish executing the last queueing instance
2816 * @work: the work to flush
Oleg Nesterovdb700892008-07-25 01:47:49 -07002817 *
Tejun Heo606a5022012-08-20 14:51:23 -07002818 * Wait until @work has finished execution. @work is guaranteed to be idle
2819 * on return if it hasn't been requeued since flush started.
Tejun Heo401a8d02010-09-16 10:36:00 +02002820 *
2821 * RETURNS:
2822 * %true if flush_work() waited for the work to finish execution,
2823 * %false if it was already idle.
Oleg Nesterovdb700892008-07-25 01:47:49 -07002824 */
Tejun Heo401a8d02010-09-16 10:36:00 +02002825bool flush_work(struct work_struct *work)
Oleg Nesterovdb700892008-07-25 01:47:49 -07002826{
Oleg Nesterovdb700892008-07-25 01:47:49 -07002827 struct wq_barrier barr;
2828
Stephen Boyd0976dfc2012-04-20 17:28:50 -07002829 lock_map_acquire(&work->lockdep_map);
2830 lock_map_release(&work->lockdep_map);
2831
Tejun Heo606a5022012-08-20 14:51:23 -07002832 if (start_flush_work(work, &barr)) {
Tejun Heobaf59022010-09-16 10:42:16 +02002833 wait_for_completion(&barr.done);
2834 destroy_work_on_stack(&barr.work);
2835 return true;
Tejun Heo606a5022012-08-20 14:51:23 -07002836 } else {
Tejun Heobaf59022010-09-16 10:42:16 +02002837 return false;
Tejun Heo606a5022012-08-20 14:51:23 -07002838 }
Oleg Nesterovdb700892008-07-25 01:47:49 -07002839}
2840EXPORT_SYMBOL_GPL(flush_work);
2841
Tejun Heo36e227d2012-08-03 10:30:46 -07002842static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
Tejun Heo401a8d02010-09-16 10:36:00 +02002843{
Tejun Heobbb68df2012-08-03 10:30:46 -07002844 unsigned long flags;
Oleg Nesterov1f1f6422007-07-15 23:41:44 -07002845 int ret;
2846
2847 do {
Tejun Heobbb68df2012-08-03 10:30:46 -07002848 ret = try_to_grab_pending(work, is_dwork, &flags);
2849 /*
2850 * If someone else is canceling, wait for the same event it
2851 * would be waiting for before retrying.
2852 */
2853 if (unlikely(ret == -ENOENT))
Tejun Heo606a5022012-08-20 14:51:23 -07002854 flush_work(work);
Oleg Nesterov1f1f6422007-07-15 23:41:44 -07002855 } while (unlikely(ret < 0));
2856
Tejun Heobbb68df2012-08-03 10:30:46 -07002857 /* tell other tasks trying to grab @work to back off */
2858 mark_work_canceling(work);
2859 local_irq_restore(flags);
2860
Tejun Heo606a5022012-08-20 14:51:23 -07002861 flush_work(work);
Tejun Heo7a22ad72010-06-29 10:07:13 +02002862 clear_work_data(work);
Oleg Nesterov1f1f6422007-07-15 23:41:44 -07002863 return ret;
2864}
2865
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002866/**
Tejun Heo401a8d02010-09-16 10:36:00 +02002867 * cancel_work_sync - cancel a work and wait for it to finish
2868 * @work: the work to cancel
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002869 *
Tejun Heo401a8d02010-09-16 10:36:00 +02002870 * Cancel @work and wait for its execution to finish. This function
2871 * can be used even if the work re-queues itself or migrates to
2872 * another workqueue. On return from this function, @work is
2873 * guaranteed to be not pending or executing on any CPU.
Oleg Nesterov1f1f6422007-07-15 23:41:44 -07002874 *
Tejun Heo401a8d02010-09-16 10:36:00 +02002875 * cancel_work_sync(&delayed_work->work) must not be used for
2876 * delayed_work's. Use cancel_delayed_work_sync() instead.
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002877 *
Tejun Heo401a8d02010-09-16 10:36:00 +02002878 * The caller must ensure that the workqueue on which @work was last
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002879 * queued can't be destroyed before this function returns.
Tejun Heo401a8d02010-09-16 10:36:00 +02002880 *
2881 * RETURNS:
2882 * %true if @work was pending, %false otherwise.
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002883 */
Tejun Heo401a8d02010-09-16 10:36:00 +02002884bool cancel_work_sync(struct work_struct *work)
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002885{
Tejun Heo36e227d2012-08-03 10:30:46 -07002886 return __cancel_work_timer(work, false);
Oleg Nesterovb89deed2007-05-09 02:33:52 -07002887}
Oleg Nesterov28e53bd2007-05-09 02:34:22 -07002888EXPORT_SYMBOL_GPL(cancel_work_sync);
Oleg Nesterovb89deed2007-05-09 02:33:52 -07002889
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002890/**
Tejun Heo401a8d02010-09-16 10:36:00 +02002891 * flush_delayed_work - wait for a dwork to finish executing the last queueing
2892 * @dwork: the delayed work to flush
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002893 *
Tejun Heo401a8d02010-09-16 10:36:00 +02002894 * Delayed timer is cancelled and the pending work is queued for
2895 * immediate execution. Like flush_work(), this function only
2896 * considers the last queueing instance of @dwork.
Oleg Nesterov1f1f6422007-07-15 23:41:44 -07002897 *
Tejun Heo401a8d02010-09-16 10:36:00 +02002898 * RETURNS:
2899 * %true if flush_work() waited for the work to finish execution,
2900 * %false if it was already idle.
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002901 */
Tejun Heo401a8d02010-09-16 10:36:00 +02002902bool flush_delayed_work(struct delayed_work *dwork)
2903{
Tejun Heo8930cab2012-08-03 10:30:45 -07002904 local_irq_disable();
Tejun Heo401a8d02010-09-16 10:36:00 +02002905 if (del_timer_sync(&dwork->timer))
Lai Jiangshan60c057b2013-02-06 18:04:53 -08002906 __queue_work(dwork->cpu, dwork->wq, &dwork->work);
Tejun Heo8930cab2012-08-03 10:30:45 -07002907 local_irq_enable();
Tejun Heo401a8d02010-09-16 10:36:00 +02002908 return flush_work(&dwork->work);
2909}
2910EXPORT_SYMBOL(flush_delayed_work);
2911
2912/**
Tejun Heo57b30ae2012-08-21 13:18:24 -07002913 * cancel_delayed_work - cancel a delayed work
2914 * @dwork: delayed_work to cancel
Tejun Heo09383492010-09-16 10:48:29 +02002915 *
Tejun Heo57b30ae2012-08-21 13:18:24 -07002916 * Kill off a pending delayed_work. Returns %true if @dwork was pending
2917 * and canceled; %false if wasn't pending. Note that the work callback
2918 * function may still be running on return, unless it returns %true and the
2919 * work doesn't re-arm itself. Explicitly flush or use
2920 * cancel_delayed_work_sync() to wait on it.
Tejun Heo09383492010-09-16 10:48:29 +02002921 *
Tejun Heo57b30ae2012-08-21 13:18:24 -07002922 * This function is safe to call from any context including IRQ handler.
Tejun Heo09383492010-09-16 10:48:29 +02002923 */
Tejun Heo57b30ae2012-08-21 13:18:24 -07002924bool cancel_delayed_work(struct delayed_work *dwork)
Tejun Heo09383492010-09-16 10:48:29 +02002925{
Tejun Heo57b30ae2012-08-21 13:18:24 -07002926 unsigned long flags;
2927 int ret;
2928
2929 do {
2930 ret = try_to_grab_pending(&dwork->work, true, &flags);
2931 } while (unlikely(ret == -EAGAIN));
2932
2933 if (unlikely(ret < 0))
2934 return false;
2935
Tejun Heo7c3eed52013-01-24 11:01:33 -08002936 set_work_pool_and_clear_pending(&dwork->work,
2937 get_work_pool_id(&dwork->work));
Tejun Heo57b30ae2012-08-21 13:18:24 -07002938 local_irq_restore(flags);
Dan Magenheimerc0158ca2012-10-18 16:31:37 -07002939 return ret;
Tejun Heo09383492010-09-16 10:48:29 +02002940}
Tejun Heo57b30ae2012-08-21 13:18:24 -07002941EXPORT_SYMBOL(cancel_delayed_work);
Tejun Heo09383492010-09-16 10:48:29 +02002942
2943/**
Tejun Heo401a8d02010-09-16 10:36:00 +02002944 * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
2945 * @dwork: the delayed work cancel
2946 *
2947 * This is cancel_work_sync() for delayed works.
2948 *
2949 * RETURNS:
2950 * %true if @dwork was pending, %false otherwise.
2951 */
2952bool cancel_delayed_work_sync(struct delayed_work *dwork)
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002953{
Tejun Heo36e227d2012-08-03 10:30:46 -07002954 return __cancel_work_timer(&dwork->work, true);
Oleg Nesterov6e84d642007-05-09 02:34:46 -07002955}
Oleg Nesterovf5a421a2007-07-15 23:41:44 -07002956EXPORT_SYMBOL(cancel_delayed_work_sync);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002957
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -07002958/**
Tejun Heo31ddd872010-10-19 11:14:49 +02002959 * schedule_on_each_cpu - execute a function synchronously on each online CPU
Andrew Mortonb6136772006-06-25 05:47:49 -07002960 * @func: the function to call
Andrew Mortonb6136772006-06-25 05:47:49 -07002961 *
Tejun Heo31ddd872010-10-19 11:14:49 +02002962 * schedule_on_each_cpu() executes @func on each online CPU using the
2963 * system workqueue and blocks until all CPUs have completed.
Andrew Mortonb6136772006-06-25 05:47:49 -07002964 * schedule_on_each_cpu() is very slow.
Tejun Heo31ddd872010-10-19 11:14:49 +02002965 *
2966 * RETURNS:
2967 * 0 on success, -errno on failure.
Andrew Mortonb6136772006-06-25 05:47:49 -07002968 */
David Howells65f27f32006-11-22 14:55:48 +00002969int schedule_on_each_cpu(work_func_t func)
Christoph Lameter15316ba2006-01-08 01:00:43 -08002970{
2971 int cpu;
Namhyung Kim38f51562010-08-08 14:24:09 +02002972 struct work_struct __percpu *works;
Christoph Lameter15316ba2006-01-08 01:00:43 -08002973
Andrew Mortonb6136772006-06-25 05:47:49 -07002974 works = alloc_percpu(struct work_struct);
2975 if (!works)
Christoph Lameter15316ba2006-01-08 01:00:43 -08002976 return -ENOMEM;
Andrew Mortonb6136772006-06-25 05:47:49 -07002977
Gautham R Shenoy95402b32008-01-25 21:08:02 +01002978 get_online_cpus();
Tejun Heo93981802009-11-17 14:06:20 -08002979
Christoph Lameter15316ba2006-01-08 01:00:43 -08002980 for_each_online_cpu(cpu) {
Ingo Molnar9bfb1832006-12-18 20:05:09 +01002981 struct work_struct *work = per_cpu_ptr(works, cpu);
2982
2983 INIT_WORK(work, func);
Tejun Heob71ab8c2010-06-29 10:07:14 +02002984 schedule_work_on(cpu, work);
Andi Kleen65a64462009-10-14 06:22:47 +02002985 }
Tejun Heo93981802009-11-17 14:06:20 -08002986
2987 for_each_online_cpu(cpu)
2988 flush_work(per_cpu_ptr(works, cpu));
2989
Gautham R Shenoy95402b32008-01-25 21:08:02 +01002990 put_online_cpus();
Andrew Mortonb6136772006-06-25 05:47:49 -07002991 free_percpu(works);
Christoph Lameter15316ba2006-01-08 01:00:43 -08002992 return 0;
2993}
2994
Alan Sterneef6a7d2010-02-12 17:39:21 +09002995/**
2996 * flush_scheduled_work - ensure that any scheduled work has run to completion.
2997 *
2998 * Forces execution of the kernel-global workqueue and blocks until its
2999 * completion.
3000 *
3001 * Think twice before calling this function! It's very easy to get into
3002 * trouble if you don't take great care. Either of the following situations
3003 * will lead to deadlock:
3004 *
3005 * One of the work items currently on the workqueue needs to acquire
3006 * a lock held by your code or its caller.
3007 *
3008 * Your code is running in the context of a work routine.
3009 *
3010 * They will be detected by lockdep when they occur, but the first might not
3011 * occur very often. It depends on what work items are on the workqueue and
3012 * what locks they need, which you have no control over.
3013 *
3014 * In most situations flushing the entire workqueue is overkill; you merely
3015 * need to know that a particular work item isn't queued and isn't running.
3016 * In such cases you should use cancel_delayed_work_sync() or
3017 * cancel_work_sync() instead.
3018 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003019void flush_scheduled_work(void)
3020{
Tejun Heod320c032010-06-29 10:07:14 +02003021 flush_workqueue(system_wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003022}
Dave Jonesae90dd52006-06-30 01:40:45 -04003023EXPORT_SYMBOL(flush_scheduled_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003024
3025/**
James Bottomley1fa44ec2006-02-23 12:43:43 -06003026 * execute_in_process_context - reliably execute the routine with user context
3027 * @fn: the function to execute
James Bottomley1fa44ec2006-02-23 12:43:43 -06003028 * @ew: guaranteed storage for the execute work structure (must
3029 * be available when the work executes)
3030 *
3031 * Executes the function immediately if process context is available,
3032 * otherwise schedules the function for delayed execution.
3033 *
3034 * Returns: 0 - function was executed
3035 * 1 - function was scheduled for execution
3036 */
David Howells65f27f32006-11-22 14:55:48 +00003037int execute_in_process_context(work_func_t fn, struct execute_work *ew)
James Bottomley1fa44ec2006-02-23 12:43:43 -06003038{
3039 if (!in_interrupt()) {
David Howells65f27f32006-11-22 14:55:48 +00003040 fn(&ew->work);
James Bottomley1fa44ec2006-02-23 12:43:43 -06003041 return 0;
3042 }
3043
David Howells65f27f32006-11-22 14:55:48 +00003044 INIT_WORK(&ew->work, fn);
James Bottomley1fa44ec2006-02-23 12:43:43 -06003045 schedule_work(&ew->work);
3046
3047 return 1;
3048}
3049EXPORT_SYMBOL_GPL(execute_in_process_context);
3050
Tejun Heo226223a2013-03-12 11:30:05 -07003051#ifdef CONFIG_SYSFS
3052/*
3053 * Workqueues with WQ_SYSFS flag set is visible to userland via
3054 * /sys/bus/workqueue/devices/WQ_NAME. All visible workqueues have the
3055 * following attributes.
3056 *
3057 * per_cpu RO bool : whether the workqueue is per-cpu or unbound
3058 * max_active RW int : maximum number of in-flight work items
3059 *
3060 * Unbound workqueues have the following extra attributes.
3061 *
3062 * id RO int : the associated pool ID
3063 * nice RW int : nice value of the workers
3064 * cpumask RW mask : bitmask of allowed CPUs for the workers
3065 */
3066struct wq_device {
3067 struct workqueue_struct *wq;
3068 struct device dev;
3069};
3070
3071static struct workqueue_struct *dev_to_wq(struct device *dev)
3072{
3073 struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
3074
3075 return wq_dev->wq;
3076}
3077
3078static ssize_t wq_per_cpu_show(struct device *dev,
3079 struct device_attribute *attr, char *buf)
3080{
3081 struct workqueue_struct *wq = dev_to_wq(dev);
3082
3083 return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
3084}
3085
3086static ssize_t wq_max_active_show(struct device *dev,
3087 struct device_attribute *attr, char *buf)
3088{
3089 struct workqueue_struct *wq = dev_to_wq(dev);
3090
3091 return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
3092}
3093
3094static ssize_t wq_max_active_store(struct device *dev,
3095 struct device_attribute *attr,
3096 const char *buf, size_t count)
3097{
3098 struct workqueue_struct *wq = dev_to_wq(dev);
3099 int val;
3100
3101 if (sscanf(buf, "%d", &val) != 1 || val <= 0)
3102 return -EINVAL;
3103
3104 workqueue_set_max_active(wq, val);
3105 return count;
3106}
3107
3108static struct device_attribute wq_sysfs_attrs[] = {
3109 __ATTR(per_cpu, 0444, wq_per_cpu_show, NULL),
3110 __ATTR(max_active, 0644, wq_max_active_show, wq_max_active_store),
3111 __ATTR_NULL,
3112};
3113
3114static ssize_t wq_pool_id_show(struct device *dev,
3115 struct device_attribute *attr, char *buf)
3116{
3117 struct workqueue_struct *wq = dev_to_wq(dev);
3118 struct worker_pool *pool;
3119 int written;
3120
3121 rcu_read_lock_sched();
3122 pool = first_pwq(wq)->pool;
3123 written = scnprintf(buf, PAGE_SIZE, "%d\n", pool->id);
3124 rcu_read_unlock_sched();
3125
3126 return written;
3127}
3128
3129static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
3130 char *buf)
3131{
3132 struct workqueue_struct *wq = dev_to_wq(dev);
3133 int written;
3134
3135 rcu_read_lock_sched();
3136 written = scnprintf(buf, PAGE_SIZE, "%d\n",
3137 first_pwq(wq)->pool->attrs->nice);
3138 rcu_read_unlock_sched();
3139
3140 return written;
3141}
3142
3143/* prepare workqueue_attrs for sysfs store operations */
3144static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
3145{
3146 struct workqueue_attrs *attrs;
3147
3148 attrs = alloc_workqueue_attrs(GFP_KERNEL);
3149 if (!attrs)
3150 return NULL;
3151
3152 rcu_read_lock_sched();
3153 copy_workqueue_attrs(attrs, first_pwq(wq)->pool->attrs);
3154 rcu_read_unlock_sched();
3155 return attrs;
3156}
3157
3158static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
3159 const char *buf, size_t count)
3160{
3161 struct workqueue_struct *wq = dev_to_wq(dev);
3162 struct workqueue_attrs *attrs;
3163 int ret;
3164
3165 attrs = wq_sysfs_prep_attrs(wq);
3166 if (!attrs)
3167 return -ENOMEM;
3168
3169 if (sscanf(buf, "%d", &attrs->nice) == 1 &&
3170 attrs->nice >= -20 && attrs->nice <= 19)
3171 ret = apply_workqueue_attrs(wq, attrs);
3172 else
3173 ret = -EINVAL;
3174
3175 free_workqueue_attrs(attrs);
3176 return ret ?: count;
3177}
3178
3179static ssize_t wq_cpumask_show(struct device *dev,
3180 struct device_attribute *attr, char *buf)
3181{
3182 struct workqueue_struct *wq = dev_to_wq(dev);
3183 int written;
3184
3185 rcu_read_lock_sched();
3186 written = cpumask_scnprintf(buf, PAGE_SIZE,
3187 first_pwq(wq)->pool->attrs->cpumask);
3188 rcu_read_unlock_sched();
3189
3190 written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
3191 return written;
3192}
3193
3194static ssize_t wq_cpumask_store(struct device *dev,
3195 struct device_attribute *attr,
3196 const char *buf, size_t count)
3197{
3198 struct workqueue_struct *wq = dev_to_wq(dev);
3199 struct workqueue_attrs *attrs;
3200 int ret;
3201
3202 attrs = wq_sysfs_prep_attrs(wq);
3203 if (!attrs)
3204 return -ENOMEM;
3205
3206 ret = cpumask_parse(buf, attrs->cpumask);
3207 if (!ret)
3208 ret = apply_workqueue_attrs(wq, attrs);
3209
3210 free_workqueue_attrs(attrs);
3211 return ret ?: count;
3212}
3213
3214static struct device_attribute wq_sysfs_unbound_attrs[] = {
3215 __ATTR(pool_id, 0444, wq_pool_id_show, NULL),
3216 __ATTR(nice, 0644, wq_nice_show, wq_nice_store),
3217 __ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
3218 __ATTR_NULL,
3219};
3220
3221static struct bus_type wq_subsys = {
3222 .name = "workqueue",
3223 .dev_attrs = wq_sysfs_attrs,
3224};
3225
3226static int __init wq_sysfs_init(void)
3227{
3228 return subsys_virtual_register(&wq_subsys, NULL);
3229}
3230core_initcall(wq_sysfs_init);
3231
3232static void wq_device_release(struct device *dev)
3233{
3234 struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
3235
3236 kfree(wq_dev);
3237}
3238
3239/**
3240 * workqueue_sysfs_register - make a workqueue visible in sysfs
3241 * @wq: the workqueue to register
3242 *
3243 * Expose @wq in sysfs under /sys/bus/workqueue/devices.
3244 * alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
3245 * which is the preferred method.
3246 *
3247 * Workqueue user should use this function directly iff it wants to apply
3248 * workqueue_attrs before making the workqueue visible in sysfs; otherwise,
3249 * apply_workqueue_attrs() may race against userland updating the
3250 * attributes.
3251 *
3252 * Returns 0 on success, -errno on failure.
3253 */
3254int workqueue_sysfs_register(struct workqueue_struct *wq)
3255{
3256 struct wq_device *wq_dev;
3257 int ret;
3258
3259 /*
3260 * Adjusting max_active or creating new pwqs by applyting
3261 * attributes breaks ordering guarantee. Disallow exposing ordered
3262 * workqueues.
3263 */
3264 if (WARN_ON(wq->flags & __WQ_ORDERED))
3265 return -EINVAL;
3266
3267 wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
3268 if (!wq_dev)
3269 return -ENOMEM;
3270
3271 wq_dev->wq = wq;
3272 wq_dev->dev.bus = &wq_subsys;
3273 wq_dev->dev.init_name = wq->name;
3274 wq_dev->dev.release = wq_device_release;
3275
3276 /*
3277 * unbound_attrs are created separately. Suppress uevent until
3278 * everything is ready.
3279 */
3280 dev_set_uevent_suppress(&wq_dev->dev, true);
3281
3282 ret = device_register(&wq_dev->dev);
3283 if (ret) {
3284 kfree(wq_dev);
3285 wq->wq_dev = NULL;
3286 return ret;
3287 }
3288
3289 if (wq->flags & WQ_UNBOUND) {
3290 struct device_attribute *attr;
3291
3292 for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
3293 ret = device_create_file(&wq_dev->dev, attr);
3294 if (ret) {
3295 device_unregister(&wq_dev->dev);
3296 wq->wq_dev = NULL;
3297 return ret;
3298 }
3299 }
3300 }
3301
3302 kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
3303 return 0;
3304}
3305
3306/**
3307 * workqueue_sysfs_unregister - undo workqueue_sysfs_register()
3308 * @wq: the workqueue to unregister
3309 *
3310 * If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
3311 */
3312static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
3313{
3314 struct wq_device *wq_dev = wq->wq_dev;
3315
3316 if (!wq->wq_dev)
3317 return;
3318
3319 wq->wq_dev = NULL;
3320 device_unregister(&wq_dev->dev);
3321}
3322#else /* CONFIG_SYSFS */
3323static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { }
3324#endif /* CONFIG_SYSFS */
3325
Tejun Heo7a4e3442013-03-12 11:30:00 -07003326/**
3327 * free_workqueue_attrs - free a workqueue_attrs
3328 * @attrs: workqueue_attrs to free
3329 *
3330 * Undo alloc_workqueue_attrs().
3331 */
3332void free_workqueue_attrs(struct workqueue_attrs *attrs)
3333{
3334 if (attrs) {
3335 free_cpumask_var(attrs->cpumask);
3336 kfree(attrs);
3337 }
3338}
3339
3340/**
3341 * alloc_workqueue_attrs - allocate a workqueue_attrs
3342 * @gfp_mask: allocation mask to use
3343 *
3344 * Allocate a new workqueue_attrs, initialize with default settings and
3345 * return it. Returns NULL on failure.
3346 */
3347struct workqueue_attrs *alloc_workqueue_attrs(gfp_t gfp_mask)
3348{
3349 struct workqueue_attrs *attrs;
3350
3351 attrs = kzalloc(sizeof(*attrs), gfp_mask);
3352 if (!attrs)
3353 goto fail;
3354 if (!alloc_cpumask_var(&attrs->cpumask, gfp_mask))
3355 goto fail;
3356
3357 cpumask_setall(attrs->cpumask);
3358 return attrs;
3359fail:
3360 free_workqueue_attrs(attrs);
3361 return NULL;
3362}
3363
Tejun Heo29c91e92013-03-12 11:30:03 -07003364static void copy_workqueue_attrs(struct workqueue_attrs *to,
3365 const struct workqueue_attrs *from)
3366{
3367 to->nice = from->nice;
3368 cpumask_copy(to->cpumask, from->cpumask);
3369}
3370
3371/*
3372 * Hacky implementation of jhash of bitmaps which only considers the
3373 * specified number of bits. We probably want a proper implementation in
3374 * include/linux/jhash.h.
3375 */
3376static u32 jhash_bitmap(const unsigned long *bitmap, int bits, u32 hash)
3377{
3378 int nr_longs = bits / BITS_PER_LONG;
3379 int nr_leftover = bits % BITS_PER_LONG;
3380 unsigned long leftover = 0;
3381
3382 if (nr_longs)
3383 hash = jhash(bitmap, nr_longs * sizeof(long), hash);
3384 if (nr_leftover) {
3385 bitmap_copy(&leftover, bitmap + nr_longs, nr_leftover);
3386 hash = jhash(&leftover, sizeof(long), hash);
3387 }
3388 return hash;
3389}
3390
3391/* hash value of the content of @attr */
3392static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
3393{
3394 u32 hash = 0;
3395
3396 hash = jhash_1word(attrs->nice, hash);
3397 hash = jhash_bitmap(cpumask_bits(attrs->cpumask), nr_cpu_ids, hash);
3398 return hash;
3399}
3400
3401/* content equality test */
3402static bool wqattrs_equal(const struct workqueue_attrs *a,
3403 const struct workqueue_attrs *b)
3404{
3405 if (a->nice != b->nice)
3406 return false;
3407 if (!cpumask_equal(a->cpumask, b->cpumask))
3408 return false;
3409 return true;
3410}
3411
Tejun Heo7a4e3442013-03-12 11:30:00 -07003412/**
3413 * init_worker_pool - initialize a newly zalloc'd worker_pool
3414 * @pool: worker_pool to initialize
3415 *
3416 * Initiailize a newly zalloc'd @pool. It also allocates @pool->attrs.
Tejun Heo29c91e92013-03-12 11:30:03 -07003417 * Returns 0 on success, -errno on failure. Even on failure, all fields
3418 * inside @pool proper are initialized and put_unbound_pool() can be called
3419 * on @pool safely to release it.
Tejun Heo7a4e3442013-03-12 11:30:00 -07003420 */
3421static int init_worker_pool(struct worker_pool *pool)
Tejun Heo4e1a1f92013-03-12 11:30:00 -07003422{
3423 spin_lock_init(&pool->lock);
Tejun Heo29c91e92013-03-12 11:30:03 -07003424 pool->id = -1;
3425 pool->cpu = -1;
Tejun Heo4e1a1f92013-03-12 11:30:00 -07003426 pool->flags |= POOL_DISASSOCIATED;
3427 INIT_LIST_HEAD(&pool->worklist);
3428 INIT_LIST_HEAD(&pool->idle_list);
3429 hash_init(pool->busy_hash);
3430
3431 init_timer_deferrable(&pool->idle_timer);
3432 pool->idle_timer.function = idle_worker_timeout;
3433 pool->idle_timer.data = (unsigned long)pool;
3434
3435 setup_timer(&pool->mayday_timer, pool_mayday_timeout,
3436 (unsigned long)pool);
3437
3438 mutex_init(&pool->manager_arb);
3439 mutex_init(&pool->assoc_mutex);
3440 ida_init(&pool->worker_ida);
Tejun Heo7a4e3442013-03-12 11:30:00 -07003441
Tejun Heo29c91e92013-03-12 11:30:03 -07003442 INIT_HLIST_NODE(&pool->hash_node);
3443 pool->refcnt = 1;
3444
3445 /* shouldn't fail above this point */
Tejun Heo7a4e3442013-03-12 11:30:00 -07003446 pool->attrs = alloc_workqueue_attrs(GFP_KERNEL);
3447 if (!pool->attrs)
3448 return -ENOMEM;
3449 return 0;
Tejun Heo4e1a1f92013-03-12 11:30:00 -07003450}
3451
Tejun Heo29c91e92013-03-12 11:30:03 -07003452static void rcu_free_pool(struct rcu_head *rcu)
3453{
3454 struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
3455
3456 ida_destroy(&pool->worker_ida);
3457 free_workqueue_attrs(pool->attrs);
3458 kfree(pool);
3459}
3460
3461/**
3462 * put_unbound_pool - put a worker_pool
3463 * @pool: worker_pool to put
3464 *
3465 * Put @pool. If its refcnt reaches zero, it gets destroyed in sched-RCU
Tejun Heoc5aa87b2013-03-13 16:51:36 -07003466 * safe manner. get_unbound_pool() calls this function on its failure path
3467 * and this function should be able to release pools which went through,
3468 * successfully or not, init_worker_pool().
Tejun Heo29c91e92013-03-12 11:30:03 -07003469 */
3470static void put_unbound_pool(struct worker_pool *pool)
3471{
3472 struct worker *worker;
3473
3474 spin_lock_irq(&workqueue_lock);
3475 if (--pool->refcnt) {
3476 spin_unlock_irq(&workqueue_lock);
3477 return;
3478 }
3479
3480 /* sanity checks */
3481 if (WARN_ON(!(pool->flags & POOL_DISASSOCIATED)) ||
3482 WARN_ON(!list_empty(&pool->worklist))) {
3483 spin_unlock_irq(&workqueue_lock);
3484 return;
3485 }
3486
3487 /* release id and unhash */
3488 if (pool->id >= 0)
3489 idr_remove(&worker_pool_idr, pool->id);
3490 hash_del(&pool->hash_node);
3491
3492 spin_unlock_irq(&workqueue_lock);
3493
Tejun Heoc5aa87b2013-03-13 16:51:36 -07003494 /*
3495 * Become the manager and destroy all workers. Grabbing
3496 * manager_arb prevents @pool's workers from blocking on
3497 * manager_mutex.
3498 */
Tejun Heo29c91e92013-03-12 11:30:03 -07003499 mutex_lock(&pool->manager_arb);
3500 spin_lock_irq(&pool->lock);
3501
3502 while ((worker = first_worker(pool)))
3503 destroy_worker(worker);
3504 WARN_ON(pool->nr_workers || pool->nr_idle);
3505
3506 spin_unlock_irq(&pool->lock);
3507 mutex_unlock(&pool->manager_arb);
3508
3509 /* shut down the timers */
3510 del_timer_sync(&pool->idle_timer);
3511 del_timer_sync(&pool->mayday_timer);
3512
3513 /* sched-RCU protected to allow dereferences from get_work_pool() */
3514 call_rcu_sched(&pool->rcu, rcu_free_pool);
3515}
3516
3517/**
3518 * get_unbound_pool - get a worker_pool with the specified attributes
3519 * @attrs: the attributes of the worker_pool to get
3520 *
3521 * Obtain a worker_pool which has the same attributes as @attrs, bump the
3522 * reference count and return it. If there already is a matching
3523 * worker_pool, it will be used; otherwise, this function attempts to
3524 * create a new one. On failure, returns NULL.
3525 */
3526static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
3527{
3528 static DEFINE_MUTEX(create_mutex);
3529 u32 hash = wqattrs_hash(attrs);
3530 struct worker_pool *pool;
3531 struct worker *worker;
3532
3533 mutex_lock(&create_mutex);
3534
3535 /* do we already have a matching pool? */
3536 spin_lock_irq(&workqueue_lock);
3537 hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
3538 if (wqattrs_equal(pool->attrs, attrs)) {
3539 pool->refcnt++;
3540 goto out_unlock;
3541 }
3542 }
3543 spin_unlock_irq(&workqueue_lock);
3544
3545 /* nope, create a new one */
3546 pool = kzalloc(sizeof(*pool), GFP_KERNEL);
3547 if (!pool || init_worker_pool(pool) < 0)
3548 goto fail;
3549
Tejun Heo8864b4e2013-03-12 11:30:04 -07003550 lockdep_set_subclass(&pool->lock, 1); /* see put_pwq() */
Tejun Heo29c91e92013-03-12 11:30:03 -07003551 copy_workqueue_attrs(pool->attrs, attrs);
3552
3553 if (worker_pool_assign_id(pool) < 0)
3554 goto fail;
3555
3556 /* create and start the initial worker */
3557 worker = create_worker(pool);
3558 if (!worker)
3559 goto fail;
3560
3561 spin_lock_irq(&pool->lock);
3562 start_worker(worker);
3563 spin_unlock_irq(&pool->lock);
3564
3565 /* install */
3566 spin_lock_irq(&workqueue_lock);
3567 hash_add(unbound_pool_hash, &pool->hash_node, hash);
3568out_unlock:
3569 spin_unlock_irq(&workqueue_lock);
3570 mutex_unlock(&create_mutex);
3571 return pool;
3572fail:
3573 mutex_unlock(&create_mutex);
3574 if (pool)
3575 put_unbound_pool(pool);
3576 return NULL;
3577}
3578
Tejun Heo8864b4e2013-03-12 11:30:04 -07003579static void rcu_free_pwq(struct rcu_head *rcu)
3580{
3581 kmem_cache_free(pwq_cache,
3582 container_of(rcu, struct pool_workqueue, rcu));
3583}
3584
3585/*
3586 * Scheduled on system_wq by put_pwq() when an unbound pwq hits zero refcnt
3587 * and needs to be destroyed.
3588 */
3589static void pwq_unbound_release_workfn(struct work_struct *work)
3590{
3591 struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
3592 unbound_release_work);
3593 struct workqueue_struct *wq = pwq->wq;
3594 struct worker_pool *pool = pwq->pool;
3595
3596 if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
3597 return;
3598
Tejun Heo75ccf592013-03-12 11:30:04 -07003599 /*
3600 * Unlink @pwq. Synchronization against flush_mutex isn't strictly
3601 * necessary on release but do it anyway. It's easier to verify
3602 * and consistent with the linking path.
3603 */
3604 mutex_lock(&wq->flush_mutex);
Tejun Heo8864b4e2013-03-12 11:30:04 -07003605 spin_lock_irq(&workqueue_lock);
3606 list_del_rcu(&pwq->pwqs_node);
3607 spin_unlock_irq(&workqueue_lock);
Tejun Heo75ccf592013-03-12 11:30:04 -07003608 mutex_unlock(&wq->flush_mutex);
Tejun Heo8864b4e2013-03-12 11:30:04 -07003609
3610 put_unbound_pool(pool);
3611 call_rcu_sched(&pwq->rcu, rcu_free_pwq);
3612
3613 /*
3614 * If we're the last pwq going away, @wq is already dead and no one
3615 * is gonna access it anymore. Free it.
3616 */
3617 if (list_empty(&wq->pwqs))
3618 kfree(wq);
3619}
3620
Tejun Heo0fbd95a2013-03-13 16:51:35 -07003621/**
Tejun Heo699ce092013-03-13 16:51:35 -07003622 * pwq_adjust_max_active - update a pwq's max_active to the current setting
Tejun Heo0fbd95a2013-03-13 16:51:35 -07003623 * @pwq: target pool_workqueue
Tejun Heo0fbd95a2013-03-13 16:51:35 -07003624 *
Tejun Heo699ce092013-03-13 16:51:35 -07003625 * If @pwq isn't freezing, set @pwq->max_active to the associated
3626 * workqueue's saved_max_active and activate delayed work items
3627 * accordingly. If @pwq is freezing, clear @pwq->max_active to zero.
Tejun Heo0fbd95a2013-03-13 16:51:35 -07003628 */
Tejun Heo699ce092013-03-13 16:51:35 -07003629static void pwq_adjust_max_active(struct pool_workqueue *pwq)
Tejun Heo0fbd95a2013-03-13 16:51:35 -07003630{
Tejun Heo699ce092013-03-13 16:51:35 -07003631 struct workqueue_struct *wq = pwq->wq;
3632 bool freezable = wq->flags & WQ_FREEZABLE;
Tejun Heo0fbd95a2013-03-13 16:51:35 -07003633
Tejun Heo699ce092013-03-13 16:51:35 -07003634 /* for @wq->saved_max_active */
3635 lockdep_assert_held(&workqueue_lock);
3636
3637 /* fast exit for non-freezable wqs */
3638 if (!freezable && pwq->max_active == wq->saved_max_active)
3639 return;
3640
3641 spin_lock(&pwq->pool->lock);
3642
3643 if (!freezable || !(pwq->pool->flags & POOL_FREEZING)) {
3644 pwq->max_active = wq->saved_max_active;
3645
3646 while (!list_empty(&pwq->delayed_works) &&
3647 pwq->nr_active < pwq->max_active)
3648 pwq_activate_first_delayed(pwq);
3649 } else {
3650 pwq->max_active = 0;
3651 }
3652
3653 spin_unlock(&pwq->pool->lock);
Tejun Heo0fbd95a2013-03-13 16:51:35 -07003654}
3655
Tejun Heod2c1d402013-03-12 11:30:04 -07003656static void init_and_link_pwq(struct pool_workqueue *pwq,
3657 struct workqueue_struct *wq,
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003658 struct worker_pool *pool,
3659 struct pool_workqueue **p_last_pwq)
Tejun Heod2c1d402013-03-12 11:30:04 -07003660{
3661 BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);
3662
3663 pwq->pool = pool;
3664 pwq->wq = wq;
3665 pwq->flush_color = -1;
Tejun Heo8864b4e2013-03-12 11:30:04 -07003666 pwq->refcnt = 1;
Tejun Heod2c1d402013-03-12 11:30:04 -07003667 INIT_LIST_HEAD(&pwq->delayed_works);
3668 INIT_LIST_HEAD(&pwq->mayday_node);
Tejun Heo8864b4e2013-03-12 11:30:04 -07003669 INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);
Tejun Heod2c1d402013-03-12 11:30:04 -07003670
Tejun Heo75ccf592013-03-12 11:30:04 -07003671 mutex_lock(&wq->flush_mutex);
3672 spin_lock_irq(&workqueue_lock);
3673
Tejun Heo983ca252013-03-13 16:51:35 -07003674 /*
3675 * Set the matching work_color. This is synchronized with
3676 * flush_mutex to avoid confusing flush_workqueue().
3677 */
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003678 if (p_last_pwq)
3679 *p_last_pwq = first_pwq(wq);
Tejun Heo75ccf592013-03-12 11:30:04 -07003680 pwq->work_color = wq->work_color;
Tejun Heo983ca252013-03-13 16:51:35 -07003681
3682 /* sync max_active to the current setting */
3683 pwq_adjust_max_active(pwq);
3684
3685 /* link in @pwq */
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003686 list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
Tejun Heo75ccf592013-03-12 11:30:04 -07003687
3688 spin_unlock_irq(&workqueue_lock);
3689 mutex_unlock(&wq->flush_mutex);
Tejun Heod2c1d402013-03-12 11:30:04 -07003690}
3691
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003692/**
3693 * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
3694 * @wq: the target workqueue
3695 * @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
3696 *
3697 * Apply @attrs to an unbound workqueue @wq. If @attrs doesn't match the
3698 * current attributes, a new pwq is created and made the first pwq which
3699 * will serve all new work items. Older pwqs are released as in-flight
3700 * work items finish. Note that a work item which repeatedly requeues
3701 * itself back-to-back will stay on its current pwq.
3702 *
3703 * Performs GFP_KERNEL allocations. Returns 0 on success and -errno on
3704 * failure.
3705 */
3706int apply_workqueue_attrs(struct workqueue_struct *wq,
3707 const struct workqueue_attrs *attrs)
3708{
3709 struct pool_workqueue *pwq, *last_pwq;
3710 struct worker_pool *pool;
3711
Tejun Heo8719dce2013-03-12 11:30:04 -07003712 /* only unbound workqueues can change attributes */
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003713 if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
3714 return -EINVAL;
3715
Tejun Heo8719dce2013-03-12 11:30:04 -07003716 /* creating multiple pwqs breaks ordering guarantee */
3717 if (WARN_ON((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs)))
3718 return -EINVAL;
3719
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003720 pwq = kmem_cache_zalloc(pwq_cache, GFP_KERNEL);
3721 if (!pwq)
3722 return -ENOMEM;
3723
3724 pool = get_unbound_pool(attrs);
3725 if (!pool) {
3726 kmem_cache_free(pwq_cache, pwq);
3727 return -ENOMEM;
3728 }
3729
3730 init_and_link_pwq(pwq, wq, pool, &last_pwq);
3731 if (last_pwq) {
3732 spin_lock_irq(&last_pwq->pool->lock);
3733 put_pwq(last_pwq);
3734 spin_unlock_irq(&last_pwq->pool->lock);
3735 }
3736
3737 return 0;
3738}
3739
Tejun Heo30cdf242013-03-12 11:29:57 -07003740static int alloc_and_link_pwqs(struct workqueue_struct *wq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003741{
Tejun Heo49e3cf42013-03-12 11:29:58 -07003742 bool highpri = wq->flags & WQ_HIGHPRI;
Tejun Heo30cdf242013-03-12 11:29:57 -07003743 int cpu;
Frederic Weisbeckere1d8aa92009-01-12 23:15:46 +01003744
Tejun Heo30cdf242013-03-12 11:29:57 -07003745 if (!(wq->flags & WQ_UNBOUND)) {
Tejun Heo420c0dd2013-03-12 11:29:59 -07003746 wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
3747 if (!wq->cpu_pwqs)
Tejun Heo30cdf242013-03-12 11:29:57 -07003748 return -ENOMEM;
3749
3750 for_each_possible_cpu(cpu) {
Tejun Heo7fb98ea2013-03-12 11:30:00 -07003751 struct pool_workqueue *pwq =
3752 per_cpu_ptr(wq->cpu_pwqs, cpu);
Tejun Heo7a62c2c2013-03-12 11:30:03 -07003753 struct worker_pool *cpu_pools =
Tejun Heof02ae732013-03-12 11:30:03 -07003754 per_cpu(cpu_worker_pools, cpu);
Tejun Heo30cdf242013-03-12 11:29:57 -07003755
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003756 init_and_link_pwq(pwq, wq, &cpu_pools[highpri], NULL);
Tejun Heo30cdf242013-03-12 11:29:57 -07003757 }
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003758 return 0;
Tejun Heo30cdf242013-03-12 11:29:57 -07003759 } else {
Tejun Heo9e8cd2f2013-03-12 11:30:04 -07003760 return apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
Tejun Heo30cdf242013-03-12 11:29:57 -07003761 }
Oleg Nesterov3af244332007-05-09 02:34:09 -07003762}
3763
Tejun Heof3421792010-07-02 10:03:51 +02003764static int wq_clamp_max_active(int max_active, unsigned int flags,
3765 const char *name)
Tejun Heob71ab8c2010-06-29 10:07:14 +02003766{
Tejun Heof3421792010-07-02 10:03:51 +02003767 int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
3768
3769 if (max_active < 1 || max_active > lim)
Valentin Ilie044c7822012-08-19 00:52:42 +03003770 pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
3771 max_active, name, 1, lim);
Tejun Heob71ab8c2010-06-29 10:07:14 +02003772
Tejun Heof3421792010-07-02 10:03:51 +02003773 return clamp_val(max_active, 1, lim);
Tejun Heob71ab8c2010-06-29 10:07:14 +02003774}
3775
Tejun Heob196be82012-01-10 15:11:35 -08003776struct workqueue_struct *__alloc_workqueue_key(const char *fmt,
Tejun Heod320c032010-06-29 10:07:14 +02003777 unsigned int flags,
3778 int max_active,
3779 struct lock_class_key *key,
Tejun Heob196be82012-01-10 15:11:35 -08003780 const char *lock_name, ...)
Oleg Nesterov3af244332007-05-09 02:34:09 -07003781{
Tejun Heob196be82012-01-10 15:11:35 -08003782 va_list args, args1;
Oleg Nesterov3af244332007-05-09 02:34:09 -07003783 struct workqueue_struct *wq;
Tejun Heo49e3cf42013-03-12 11:29:58 -07003784 struct pool_workqueue *pwq;
Tejun Heob196be82012-01-10 15:11:35 -08003785 size_t namelen;
3786
3787 /* determine namelen, allocate wq and format name */
3788 va_start(args, lock_name);
3789 va_copy(args1, args);
3790 namelen = vsnprintf(NULL, 0, fmt, args) + 1;
3791
3792 wq = kzalloc(sizeof(*wq) + namelen, GFP_KERNEL);
3793 if (!wq)
Tejun Heod2c1d402013-03-12 11:30:04 -07003794 return NULL;
Tejun Heob196be82012-01-10 15:11:35 -08003795
3796 vsnprintf(wq->name, namelen, fmt, args1);
3797 va_end(args);
3798 va_end(args1);
Oleg Nesterov3af244332007-05-09 02:34:09 -07003799
Tejun Heod320c032010-06-29 10:07:14 +02003800 max_active = max_active ?: WQ_DFL_ACTIVE;
Tejun Heob196be82012-01-10 15:11:35 -08003801 max_active = wq_clamp_max_active(max_active, flags, wq->name);
Oleg Nesterov3af244332007-05-09 02:34:09 -07003802
Tejun Heob196be82012-01-10 15:11:35 -08003803 /* init wq */
Tejun Heo97e37d72010-06-29 10:07:10 +02003804 wq->flags = flags;
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02003805 wq->saved_max_active = max_active;
Tejun Heo73f53c42010-06-29 10:07:11 +02003806 mutex_init(&wq->flush_mutex);
Tejun Heo112202d2013-02-13 19:29:12 -08003807 atomic_set(&wq->nr_pwqs_to_flush, 0);
Tejun Heo30cdf242013-03-12 11:29:57 -07003808 INIT_LIST_HEAD(&wq->pwqs);
Tejun Heo73f53c42010-06-29 10:07:11 +02003809 INIT_LIST_HEAD(&wq->flusher_queue);
3810 INIT_LIST_HEAD(&wq->flusher_overflow);
Tejun Heo493a1722013-03-12 11:29:59 -07003811 INIT_LIST_HEAD(&wq->maydays);
Oleg Nesterov3af244332007-05-09 02:34:09 -07003812
Johannes Bergeb13ba82008-01-16 09:51:58 +01003813 lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
Oleg Nesterovcce1a162007-05-09 02:34:13 -07003814 INIT_LIST_HEAD(&wq->list);
Oleg Nesterov3af244332007-05-09 02:34:09 -07003815
Tejun Heo30cdf242013-03-12 11:29:57 -07003816 if (alloc_and_link_pwqs(wq) < 0)
Tejun Heod2c1d402013-03-12 11:30:04 -07003817 goto err_free_wq;
Oleg Nesterov3af244332007-05-09 02:34:09 -07003818
Tejun Heo493008a2013-03-12 11:30:03 -07003819 /*
3820 * Workqueues which may be used during memory reclaim should
3821 * have a rescuer to guarantee forward progress.
3822 */
3823 if (flags & WQ_MEM_RECLAIM) {
Tejun Heoe22bee72010-06-29 10:07:14 +02003824 struct worker *rescuer;
3825
Tejun Heod2c1d402013-03-12 11:30:04 -07003826 rescuer = alloc_worker();
Tejun Heoe22bee72010-06-29 10:07:14 +02003827 if (!rescuer)
Tejun Heod2c1d402013-03-12 11:30:04 -07003828 goto err_destroy;
Tejun Heoe22bee72010-06-29 10:07:14 +02003829
Tejun Heo111c2252013-01-17 17:16:24 -08003830 rescuer->rescue_wq = wq;
3831 rescuer->task = kthread_create(rescuer_thread, rescuer, "%s",
Tejun Heob196be82012-01-10 15:11:35 -08003832 wq->name);
Tejun Heod2c1d402013-03-12 11:30:04 -07003833 if (IS_ERR(rescuer->task)) {
3834 kfree(rescuer);
3835 goto err_destroy;
3836 }
Tejun Heoe22bee72010-06-29 10:07:14 +02003837
Tejun Heod2c1d402013-03-12 11:30:04 -07003838 wq->rescuer = rescuer;
Tejun Heoe22bee72010-06-29 10:07:14 +02003839 rescuer->task->flags |= PF_THREAD_BOUND;
3840 wake_up_process(rescuer->task);
Oleg Nesterov3af244332007-05-09 02:34:09 -07003841 }
Oleg Nesterov3af244332007-05-09 02:34:09 -07003842
Tejun Heo226223a2013-03-12 11:30:05 -07003843 if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
3844 goto err_destroy;
3845
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02003846 /*
Tejun Heo699ce092013-03-13 16:51:35 -07003847 * workqueue_lock protects global freeze state and workqueues list.
3848 * Grab it, adjust max_active and add the new workqueue to
3849 * workqueues list.
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02003850 */
Tejun Heoe98d5b12013-03-12 11:29:57 -07003851 spin_lock_irq(&workqueue_lock);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02003852
Tejun Heo699ce092013-03-13 16:51:35 -07003853 for_each_pwq(pwq, wq)
3854 pwq_adjust_max_active(pwq);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02003855
Tejun Heo15376632010-06-29 10:07:11 +02003856 list_add(&wq->list, &workqueues);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02003857
Tejun Heoe98d5b12013-03-12 11:29:57 -07003858 spin_unlock_irq(&workqueue_lock);
Tejun Heo15376632010-06-29 10:07:11 +02003859
Oleg Nesterov3af244332007-05-09 02:34:09 -07003860 return wq;
Tejun Heod2c1d402013-03-12 11:30:04 -07003861
3862err_free_wq:
3863 kfree(wq);
3864 return NULL;
3865err_destroy:
3866 destroy_workqueue(wq);
Tejun Heo4690c4a2010-06-29 10:07:10 +02003867 return NULL;
Oleg Nesterov3af244332007-05-09 02:34:09 -07003868}
Tejun Heod320c032010-06-29 10:07:14 +02003869EXPORT_SYMBOL_GPL(__alloc_workqueue_key);
Oleg Nesterov3af244332007-05-09 02:34:09 -07003870
3871/**
3872 * destroy_workqueue - safely terminate a workqueue
3873 * @wq: target workqueue
3874 *
3875 * Safely destroy a workqueue. All work currently pending will be done first.
3876 */
3877void destroy_workqueue(struct workqueue_struct *wq)
3878{
Tejun Heo49e3cf42013-03-12 11:29:58 -07003879 struct pool_workqueue *pwq;
Oleg Nesterov3af244332007-05-09 02:34:09 -07003880
Tejun Heo9c5a2ba2011-04-05 18:01:44 +02003881 /* drain it before proceeding with destruction */
3882 drain_workqueue(wq);
Tejun Heoc8efcc22010-12-20 19:32:04 +01003883
Tejun Heo76af4d92013-03-12 11:30:00 -07003884 spin_lock_irq(&workqueue_lock);
3885
Tejun Heo6183c002013-03-12 11:29:57 -07003886 /* sanity checks */
Tejun Heo49e3cf42013-03-12 11:29:58 -07003887 for_each_pwq(pwq, wq) {
Tejun Heo6183c002013-03-12 11:29:57 -07003888 int i;
3889
Tejun Heo76af4d92013-03-12 11:30:00 -07003890 for (i = 0; i < WORK_NR_COLORS; i++) {
3891 if (WARN_ON(pwq->nr_in_flight[i])) {
3892 spin_unlock_irq(&workqueue_lock);
Tejun Heo6183c002013-03-12 11:29:57 -07003893 return;
Tejun Heo76af4d92013-03-12 11:30:00 -07003894 }
3895 }
3896
Tejun Heo8864b4e2013-03-12 11:30:04 -07003897 if (WARN_ON(pwq->refcnt > 1) ||
3898 WARN_ON(pwq->nr_active) ||
Tejun Heo76af4d92013-03-12 11:30:00 -07003899 WARN_ON(!list_empty(&pwq->delayed_works))) {
3900 spin_unlock_irq(&workqueue_lock);
Tejun Heo6183c002013-03-12 11:29:57 -07003901 return;
Tejun Heo76af4d92013-03-12 11:30:00 -07003902 }
Tejun Heo6183c002013-03-12 11:29:57 -07003903 }
3904
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02003905 /*
3906 * wq list is used to freeze wq, remove from list after
3907 * flushing is complete in case freeze races us.
3908 */
Tejun Heod2c1d402013-03-12 11:30:04 -07003909 list_del_init(&wq->list);
Tejun Heo76af4d92013-03-12 11:30:00 -07003910
Tejun Heoe98d5b12013-03-12 11:29:57 -07003911 spin_unlock_irq(&workqueue_lock);
Oleg Nesterov3af244332007-05-09 02:34:09 -07003912
Tejun Heo226223a2013-03-12 11:30:05 -07003913 workqueue_sysfs_unregister(wq);
3914
Tejun Heo493008a2013-03-12 11:30:03 -07003915 if (wq->rescuer) {
Tejun Heoe22bee72010-06-29 10:07:14 +02003916 kthread_stop(wq->rescuer->task);
Xiaotian Feng8d9df9f2010-08-16 09:54:28 +02003917 kfree(wq->rescuer);
Tejun Heo493008a2013-03-12 11:30:03 -07003918 wq->rescuer = NULL;
Tejun Heoe22bee72010-06-29 10:07:14 +02003919 }
3920
Tejun Heo8864b4e2013-03-12 11:30:04 -07003921 if (!(wq->flags & WQ_UNBOUND)) {
3922 /*
3923 * The base ref is never dropped on per-cpu pwqs. Directly
3924 * free the pwqs and wq.
3925 */
3926 free_percpu(wq->cpu_pwqs);
3927 kfree(wq);
3928 } else {
3929 /*
3930 * We're the sole accessor of @wq at this point. Directly
3931 * access the first pwq and put the base ref. As both pwqs
3932 * and pools are sched-RCU protected, the lock operations
3933 * are safe. @wq will be freed when the last pwq is
3934 * released.
3935 */
Tejun Heo29c91e92013-03-12 11:30:03 -07003936 pwq = list_first_entry(&wq->pwqs, struct pool_workqueue,
3937 pwqs_node);
Tejun Heo8864b4e2013-03-12 11:30:04 -07003938 spin_lock_irq(&pwq->pool->lock);
3939 put_pwq(pwq);
3940 spin_unlock_irq(&pwq->pool->lock);
Tejun Heo29c91e92013-03-12 11:30:03 -07003941 }
Oleg Nesterov3af244332007-05-09 02:34:09 -07003942}
3943EXPORT_SYMBOL_GPL(destroy_workqueue);
3944
Tejun Heodcd989c2010-06-29 10:07:14 +02003945/**
3946 * workqueue_set_max_active - adjust max_active of a workqueue
3947 * @wq: target workqueue
3948 * @max_active: new max_active value.
3949 *
3950 * Set max_active of @wq to @max_active.
3951 *
3952 * CONTEXT:
3953 * Don't call from IRQ context.
3954 */
3955void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
3956{
Tejun Heo49e3cf42013-03-12 11:29:58 -07003957 struct pool_workqueue *pwq;
Tejun Heodcd989c2010-06-29 10:07:14 +02003958
Tejun Heo8719dce2013-03-12 11:30:04 -07003959 /* disallow meddling with max_active for ordered workqueues */
3960 if (WARN_ON(wq->flags & __WQ_ORDERED))
3961 return;
3962
Tejun Heof3421792010-07-02 10:03:51 +02003963 max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
Tejun Heodcd989c2010-06-29 10:07:14 +02003964
Tejun Heoe98d5b12013-03-12 11:29:57 -07003965 spin_lock_irq(&workqueue_lock);
Tejun Heodcd989c2010-06-29 10:07:14 +02003966
3967 wq->saved_max_active = max_active;
3968
Tejun Heo699ce092013-03-13 16:51:35 -07003969 for_each_pwq(pwq, wq)
3970 pwq_adjust_max_active(pwq);
Tejun Heodcd989c2010-06-29 10:07:14 +02003971
Tejun Heoe98d5b12013-03-12 11:29:57 -07003972 spin_unlock_irq(&workqueue_lock);
Tejun Heodcd989c2010-06-29 10:07:14 +02003973}
3974EXPORT_SYMBOL_GPL(workqueue_set_max_active);
3975
3976/**
Tejun Heoe6267612013-03-12 17:41:37 -07003977 * current_is_workqueue_rescuer - is %current workqueue rescuer?
3978 *
3979 * Determine whether %current is a workqueue rescuer. Can be used from
3980 * work functions to determine whether it's being run off the rescuer task.
3981 */
3982bool current_is_workqueue_rescuer(void)
3983{
3984 struct worker *worker = current_wq_worker();
3985
3986 return worker && worker == worker->current_pwq->wq->rescuer;
3987}
3988
3989/**
Tejun Heodcd989c2010-06-29 10:07:14 +02003990 * workqueue_congested - test whether a workqueue is congested
3991 * @cpu: CPU in question
3992 * @wq: target workqueue
3993 *
3994 * Test whether @wq's cpu workqueue for @cpu is congested. There is
3995 * no synchronization around this function and the test result is
3996 * unreliable and only useful as advisory hints or for debugging.
3997 *
3998 * RETURNS:
3999 * %true if congested, %false otherwise.
4000 */
Tejun Heod84ff052013-03-12 11:29:59 -07004001bool workqueue_congested(int cpu, struct workqueue_struct *wq)
Tejun Heodcd989c2010-06-29 10:07:14 +02004002{
Tejun Heo7fb98ea2013-03-12 11:30:00 -07004003 struct pool_workqueue *pwq;
Tejun Heo76af4d92013-03-12 11:30:00 -07004004 bool ret;
4005
4006 preempt_disable();
Tejun Heo7fb98ea2013-03-12 11:30:00 -07004007
4008 if (!(wq->flags & WQ_UNBOUND))
4009 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
4010 else
4011 pwq = first_pwq(wq);
Tejun Heodcd989c2010-06-29 10:07:14 +02004012
Tejun Heo76af4d92013-03-12 11:30:00 -07004013 ret = !list_empty(&pwq->delayed_works);
4014 preempt_enable();
4015
4016 return ret;
Tejun Heodcd989c2010-06-29 10:07:14 +02004017}
4018EXPORT_SYMBOL_GPL(workqueue_congested);
4019
4020/**
Tejun Heodcd989c2010-06-29 10:07:14 +02004021 * work_busy - test whether a work is currently pending or running
4022 * @work: the work to be tested
4023 *
4024 * Test whether @work is currently pending or running. There is no
4025 * synchronization around this function and the test result is
4026 * unreliable and only useful as advisory hints or for debugging.
Tejun Heodcd989c2010-06-29 10:07:14 +02004027 *
4028 * RETURNS:
4029 * OR'd bitmask of WORK_BUSY_* bits.
4030 */
4031unsigned int work_busy(struct work_struct *work)
4032{
Tejun Heofa1b54e2013-03-12 11:30:00 -07004033 struct worker_pool *pool;
Tejun Heodcd989c2010-06-29 10:07:14 +02004034 unsigned long flags;
4035 unsigned int ret = 0;
4036
Tejun Heodcd989c2010-06-29 10:07:14 +02004037 if (work_pending(work))
4038 ret |= WORK_BUSY_PENDING;
Tejun Heodcd989c2010-06-29 10:07:14 +02004039
Tejun Heofa1b54e2013-03-12 11:30:00 -07004040 local_irq_save(flags);
4041 pool = get_work_pool(work);
Lai Jiangshan038366c2013-02-06 18:04:53 -08004042 if (pool) {
Tejun Heofa1b54e2013-03-12 11:30:00 -07004043 spin_lock(&pool->lock);
Lai Jiangshan038366c2013-02-06 18:04:53 -08004044 if (find_worker_executing_work(pool, work))
4045 ret |= WORK_BUSY_RUNNING;
Tejun Heofa1b54e2013-03-12 11:30:00 -07004046 spin_unlock(&pool->lock);
Lai Jiangshan038366c2013-02-06 18:04:53 -08004047 }
Tejun Heofa1b54e2013-03-12 11:30:00 -07004048 local_irq_restore(flags);
Tejun Heodcd989c2010-06-29 10:07:14 +02004049
4050 return ret;
4051}
4052EXPORT_SYMBOL_GPL(work_busy);
4053
Tejun Heodb7bccf2010-06-29 10:07:12 +02004054/*
4055 * CPU hotplug.
4056 *
Tejun Heoe22bee72010-06-29 10:07:14 +02004057 * There are two challenges in supporting CPU hotplug. Firstly, there
Tejun Heo112202d2013-02-13 19:29:12 -08004058 * are a lot of assumptions on strong associations among work, pwq and
Tejun Heo706026c2013-01-24 11:01:34 -08004059 * pool which make migrating pending and scheduled works very
Tejun Heoe22bee72010-06-29 10:07:14 +02004060 * difficult to implement without impacting hot paths. Secondly,
Tejun Heo94cf58b2013-01-24 11:01:33 -08004061 * worker pools serve mix of short, long and very long running works making
Tejun Heoe22bee72010-06-29 10:07:14 +02004062 * blocked draining impractical.
4063 *
Tejun Heo24647572013-01-24 11:01:33 -08004064 * This is solved by allowing the pools to be disassociated from the CPU
Tejun Heo628c78e2012-07-17 12:39:27 -07004065 * running as an unbound one and allowing it to be reattached later if the
4066 * cpu comes back online.
Tejun Heodb7bccf2010-06-29 10:07:12 +02004067 */
4068
Tejun Heo706026c2013-01-24 11:01:34 -08004069static void wq_unbind_fn(struct work_struct *work)
Tejun Heodb7bccf2010-06-29 10:07:12 +02004070{
Tejun Heo38db41d2013-01-24 11:01:34 -08004071 int cpu = smp_processor_id();
Tejun Heo4ce62e92012-07-13 22:16:44 -07004072 struct worker_pool *pool;
Tejun Heodb7bccf2010-06-29 10:07:12 +02004073 struct worker *worker;
Tejun Heodb7bccf2010-06-29 10:07:12 +02004074 int i;
4075
Tejun Heof02ae732013-03-12 11:30:03 -07004076 for_each_cpu_worker_pool(pool, cpu) {
Tejun Heo6183c002013-03-12 11:29:57 -07004077 WARN_ON_ONCE(cpu != smp_processor_id());
Tejun Heo94cf58b2013-01-24 11:01:33 -08004078
4079 mutex_lock(&pool->assoc_mutex);
4080 spin_lock_irq(&pool->lock);
4081
4082 /*
4083 * We've claimed all manager positions. Make all workers
4084 * unbound and set DISASSOCIATED. Before this, all workers
4085 * except for the ones which are still executing works from
4086 * before the last CPU down must be on the cpu. After
4087 * this, they may become diasporas.
4088 */
Tejun Heo4ce62e92012-07-13 22:16:44 -07004089 list_for_each_entry(worker, &pool->idle_list, entry)
Tejun Heo403c8212012-07-17 12:39:27 -07004090 worker->flags |= WORKER_UNBOUND;
Tejun Heodb7bccf2010-06-29 10:07:12 +02004091
Sasha Levinb67bfe02013-02-27 17:06:00 -08004092 for_each_busy_worker(worker, i, pool)
Tejun Heoc9e7cf22013-01-24 11:01:33 -08004093 worker->flags |= WORKER_UNBOUND;
Tejun Heodb7bccf2010-06-29 10:07:12 +02004094
Tejun Heo24647572013-01-24 11:01:33 -08004095 pool->flags |= POOL_DISASSOCIATED;
Tejun Heof2d5a0e2012-07-17 12:39:26 -07004096
Tejun Heo94cf58b2013-01-24 11:01:33 -08004097 spin_unlock_irq(&pool->lock);
4098 mutex_unlock(&pool->assoc_mutex);
4099 }
Tejun Heoe22bee72010-06-29 10:07:14 +02004100
4101 /*
Tejun Heo628c78e2012-07-17 12:39:27 -07004102 * Call schedule() so that we cross rq->lock and thus can guarantee
4103 * sched callbacks see the %WORKER_UNBOUND flag. This is necessary
4104 * as scheduler callbacks may be invoked from other cpus.
4105 */
4106 schedule();
4107
4108 /*
4109 * Sched callbacks are disabled now. Zap nr_running. After this,
4110 * nr_running stays zero and need_more_worker() and keep_working()
Tejun Heo38db41d2013-01-24 11:01:34 -08004111 * are always true as long as the worklist is not empty. Pools on
4112 * @cpu now behave as unbound (in terms of concurrency management)
4113 * pools which are served by workers tied to the CPU.
Tejun Heo628c78e2012-07-17 12:39:27 -07004114 *
4115 * On return from this function, the current worker would trigger
4116 * unbound chain execution of pending work items if other workers
4117 * didn't already.
Tejun Heoe22bee72010-06-29 10:07:14 +02004118 */
Tejun Heof02ae732013-03-12 11:30:03 -07004119 for_each_cpu_worker_pool(pool, cpu)
Tejun Heoe19e3972013-01-24 11:39:44 -08004120 atomic_set(&pool->nr_running, 0);
Tejun Heodb7bccf2010-06-29 10:07:12 +02004121}
4122
Tejun Heo8db25e72012-07-17 12:39:28 -07004123/*
4124 * Workqueues should be brought up before normal priority CPU notifiers.
4125 * This will be registered high priority CPU notifier.
4126 */
Lai Jiangshan9fdf9b72012-09-18 09:59:23 -07004127static int __cpuinit workqueue_cpu_up_callback(struct notifier_block *nfb,
Tejun Heo8db25e72012-07-17 12:39:28 -07004128 unsigned long action,
4129 void *hcpu)
Oleg Nesterov3af244332007-05-09 02:34:09 -07004130{
Tejun Heod84ff052013-03-12 11:29:59 -07004131 int cpu = (unsigned long)hcpu;
Tejun Heo4ce62e92012-07-13 22:16:44 -07004132 struct worker_pool *pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004133
Tejun Heo8db25e72012-07-17 12:39:28 -07004134 switch (action & ~CPU_TASKS_FROZEN) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004135 case CPU_UP_PREPARE:
Tejun Heof02ae732013-03-12 11:30:03 -07004136 for_each_cpu_worker_pool(pool, cpu) {
Tejun Heo3ce63372012-07-17 12:39:27 -07004137 struct worker *worker;
4138
4139 if (pool->nr_workers)
4140 continue;
4141
4142 worker = create_worker(pool);
4143 if (!worker)
4144 return NOTIFY_BAD;
4145
Tejun Heod565ed62013-01-24 11:01:33 -08004146 spin_lock_irq(&pool->lock);
Tejun Heo3ce63372012-07-17 12:39:27 -07004147 start_worker(worker);
Tejun Heod565ed62013-01-24 11:01:33 -08004148 spin_unlock_irq(&pool->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004149 }
Tejun Heodb7bccf2010-06-29 10:07:12 +02004150 break;
Oleg Nesterov00dfcaf2008-04-29 01:00:27 -07004151
Tejun Heo65758202012-07-17 12:39:26 -07004152 case CPU_DOWN_FAILED:
4153 case CPU_ONLINE:
Tejun Heof02ae732013-03-12 11:30:03 -07004154 for_each_cpu_worker_pool(pool, cpu) {
Tejun Heo94cf58b2013-01-24 11:01:33 -08004155 mutex_lock(&pool->assoc_mutex);
4156 spin_lock_irq(&pool->lock);
4157
Tejun Heo24647572013-01-24 11:01:33 -08004158 pool->flags &= ~POOL_DISASSOCIATED;
Tejun Heo94cf58b2013-01-24 11:01:33 -08004159 rebind_workers(pool);
4160
4161 spin_unlock_irq(&pool->lock);
4162 mutex_unlock(&pool->assoc_mutex);
4163 }
Tejun Heo8db25e72012-07-17 12:39:28 -07004164 break;
Tejun Heo65758202012-07-17 12:39:26 -07004165 }
4166 return NOTIFY_OK;
4167}
4168
4169/*
4170 * Workqueues should be brought down after normal priority CPU notifiers.
4171 * This will be registered as low priority CPU notifier.
4172 */
Lai Jiangshan9fdf9b72012-09-18 09:59:23 -07004173static int __cpuinit workqueue_cpu_down_callback(struct notifier_block *nfb,
Tejun Heo65758202012-07-17 12:39:26 -07004174 unsigned long action,
4175 void *hcpu)
4176{
Tejun Heod84ff052013-03-12 11:29:59 -07004177 int cpu = (unsigned long)hcpu;
Tejun Heo8db25e72012-07-17 12:39:28 -07004178 struct work_struct unbind_work;
4179
Tejun Heo65758202012-07-17 12:39:26 -07004180 switch (action & ~CPU_TASKS_FROZEN) {
4181 case CPU_DOWN_PREPARE:
Tejun Heo8db25e72012-07-17 12:39:28 -07004182 /* unbinding should happen on the local CPU */
Tejun Heo706026c2013-01-24 11:01:34 -08004183 INIT_WORK_ONSTACK(&unbind_work, wq_unbind_fn);
Joonsoo Kim7635d2f2012-08-15 23:25:41 +09004184 queue_work_on(cpu, system_highpri_wq, &unbind_work);
Tejun Heo8db25e72012-07-17 12:39:28 -07004185 flush_work(&unbind_work);
4186 break;
Tejun Heo65758202012-07-17 12:39:26 -07004187 }
4188 return NOTIFY_OK;
4189}
4190
Rusty Russell2d3854a2008-11-05 13:39:10 +11004191#ifdef CONFIG_SMP
Rusty Russell8ccad402009-01-16 15:31:15 -08004192
Rusty Russell2d3854a2008-11-05 13:39:10 +11004193struct work_for_cpu {
Tejun Heoed48ece2012-09-18 12:48:43 -07004194 struct work_struct work;
Rusty Russell2d3854a2008-11-05 13:39:10 +11004195 long (*fn)(void *);
4196 void *arg;
4197 long ret;
4198};
4199
Tejun Heoed48ece2012-09-18 12:48:43 -07004200static void work_for_cpu_fn(struct work_struct *work)
Rusty Russell2d3854a2008-11-05 13:39:10 +11004201{
Tejun Heoed48ece2012-09-18 12:48:43 -07004202 struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
4203
Rusty Russell2d3854a2008-11-05 13:39:10 +11004204 wfc->ret = wfc->fn(wfc->arg);
4205}
4206
4207/**
4208 * work_on_cpu - run a function in user context on a particular cpu
4209 * @cpu: the cpu to run on
4210 * @fn: the function to run
4211 * @arg: the function arg
4212 *
Rusty Russell31ad9082009-01-16 15:31:15 -08004213 * This will return the value @fn returns.
4214 * It is up to the caller to ensure that the cpu doesn't go offline.
Andrew Morton6b440032009-04-09 09:50:37 -06004215 * The caller must not hold any locks which would prevent @fn from completing.
Rusty Russell2d3854a2008-11-05 13:39:10 +11004216 */
Tejun Heod84ff052013-03-12 11:29:59 -07004217long work_on_cpu(int cpu, long (*fn)(void *), void *arg)
Rusty Russell2d3854a2008-11-05 13:39:10 +11004218{
Tejun Heoed48ece2012-09-18 12:48:43 -07004219 struct work_for_cpu wfc = { .fn = fn, .arg = arg };
Rusty Russell2d3854a2008-11-05 13:39:10 +11004220
Tejun Heoed48ece2012-09-18 12:48:43 -07004221 INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
4222 schedule_work_on(cpu, &wfc.work);
4223 flush_work(&wfc.work);
Rusty Russell2d3854a2008-11-05 13:39:10 +11004224 return wfc.ret;
4225}
4226EXPORT_SYMBOL_GPL(work_on_cpu);
4227#endif /* CONFIG_SMP */
4228
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004229#ifdef CONFIG_FREEZER
Rusty Russelle7577c52009-01-01 10:12:25 +10304230
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004231/**
4232 * freeze_workqueues_begin - begin freezing workqueues
4233 *
Tejun Heo58a69cb2011-02-16 09:25:31 +01004234 * Start freezing workqueues. After this function returns, all freezable
Tejun Heoc5aa87b2013-03-13 16:51:36 -07004235 * workqueues will queue new works to their delayed_works list instead of
Tejun Heo706026c2013-01-24 11:01:34 -08004236 * pool->worklist.
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004237 *
4238 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08004239 * Grabs and releases workqueue_lock and pool->lock's.
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004240 */
4241void freeze_workqueues_begin(void)
4242{
Tejun Heo17116962013-03-12 11:29:58 -07004243 struct worker_pool *pool;
Tejun Heo24b8a842013-03-12 11:29:58 -07004244 struct workqueue_struct *wq;
4245 struct pool_workqueue *pwq;
Tejun Heo611c92a2013-03-13 16:51:36 -07004246 int pi;
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004247
Tejun Heoe98d5b12013-03-12 11:29:57 -07004248 spin_lock_irq(&workqueue_lock);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004249
Tejun Heo6183c002013-03-12 11:29:57 -07004250 WARN_ON_ONCE(workqueue_freezing);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004251 workqueue_freezing = true;
4252
Tejun Heo24b8a842013-03-12 11:29:58 -07004253 /* set FREEZING */
Tejun Heo611c92a2013-03-13 16:51:36 -07004254 for_each_pool(pool, pi) {
Tejun Heo17116962013-03-12 11:29:58 -07004255 spin_lock(&pool->lock);
Tejun Heo17116962013-03-12 11:29:58 -07004256 WARN_ON_ONCE(pool->flags & POOL_FREEZING);
4257 pool->flags |= POOL_FREEZING;
Tejun Heo17116962013-03-12 11:29:58 -07004258 spin_unlock(&pool->lock);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004259 }
4260
Tejun Heo24b8a842013-03-12 11:29:58 -07004261 /* suppress further executions by setting max_active to zero */
4262 list_for_each_entry(wq, &workqueues, list) {
Tejun Heo699ce092013-03-13 16:51:35 -07004263 for_each_pwq(pwq, wq)
4264 pwq_adjust_max_active(pwq);
Tejun Heo24b8a842013-03-12 11:29:58 -07004265 }
4266
Tejun Heoe98d5b12013-03-12 11:29:57 -07004267 spin_unlock_irq(&workqueue_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004268}
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004269
4270/**
Tejun Heo58a69cb2011-02-16 09:25:31 +01004271 * freeze_workqueues_busy - are freezable workqueues still busy?
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004272 *
4273 * Check whether freezing is complete. This function must be called
4274 * between freeze_workqueues_begin() and thaw_workqueues().
4275 *
4276 * CONTEXT:
4277 * Grabs and releases workqueue_lock.
4278 *
4279 * RETURNS:
Tejun Heo58a69cb2011-02-16 09:25:31 +01004280 * %true if some freezable workqueues are still busy. %false if freezing
4281 * is complete.
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004282 */
4283bool freeze_workqueues_busy(void)
4284{
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004285 bool busy = false;
Tejun Heo24b8a842013-03-12 11:29:58 -07004286 struct workqueue_struct *wq;
4287 struct pool_workqueue *pwq;
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004288
Tejun Heoe98d5b12013-03-12 11:29:57 -07004289 spin_lock_irq(&workqueue_lock);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004290
Tejun Heo6183c002013-03-12 11:29:57 -07004291 WARN_ON_ONCE(!workqueue_freezing);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004292
Tejun Heo24b8a842013-03-12 11:29:58 -07004293 list_for_each_entry(wq, &workqueues, list) {
4294 if (!(wq->flags & WQ_FREEZABLE))
4295 continue;
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004296 /*
4297 * nr_active is monotonically decreasing. It's safe
4298 * to peek without lock.
4299 */
Tejun Heo24b8a842013-03-12 11:29:58 -07004300 for_each_pwq(pwq, wq) {
Tejun Heo6183c002013-03-12 11:29:57 -07004301 WARN_ON_ONCE(pwq->nr_active < 0);
Tejun Heo112202d2013-02-13 19:29:12 -08004302 if (pwq->nr_active) {
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004303 busy = true;
4304 goto out_unlock;
4305 }
4306 }
4307 }
4308out_unlock:
Tejun Heoe98d5b12013-03-12 11:29:57 -07004309 spin_unlock_irq(&workqueue_lock);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004310 return busy;
4311}
4312
4313/**
4314 * thaw_workqueues - thaw workqueues
4315 *
4316 * Thaw workqueues. Normal queueing is restored and all collected
Tejun Heo706026c2013-01-24 11:01:34 -08004317 * frozen works are transferred to their respective pool worklists.
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004318 *
4319 * CONTEXT:
Tejun Heod565ed62013-01-24 11:01:33 -08004320 * Grabs and releases workqueue_lock and pool->lock's.
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004321 */
4322void thaw_workqueues(void)
4323{
Tejun Heo24b8a842013-03-12 11:29:58 -07004324 struct workqueue_struct *wq;
4325 struct pool_workqueue *pwq;
4326 struct worker_pool *pool;
Tejun Heo611c92a2013-03-13 16:51:36 -07004327 int pi;
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004328
Tejun Heoe98d5b12013-03-12 11:29:57 -07004329 spin_lock_irq(&workqueue_lock);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004330
4331 if (!workqueue_freezing)
4332 goto out_unlock;
4333
Tejun Heo24b8a842013-03-12 11:29:58 -07004334 /* clear FREEZING */
Tejun Heo611c92a2013-03-13 16:51:36 -07004335 for_each_pool(pool, pi) {
Tejun Heo24b8a842013-03-12 11:29:58 -07004336 spin_lock(&pool->lock);
4337 WARN_ON_ONCE(!(pool->flags & POOL_FREEZING));
4338 pool->flags &= ~POOL_FREEZING;
4339 spin_unlock(&pool->lock);
4340 }
Tejun Heo8b03ae32010-06-29 10:07:12 +02004341
Tejun Heo24b8a842013-03-12 11:29:58 -07004342 /* restore max_active and repopulate worklist */
4343 list_for_each_entry(wq, &workqueues, list) {
Tejun Heo699ce092013-03-13 16:51:35 -07004344 for_each_pwq(pwq, wq)
4345 pwq_adjust_max_active(pwq);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004346 }
4347
Tejun Heo24b8a842013-03-12 11:29:58 -07004348 /* kick workers */
Tejun Heo611c92a2013-03-13 16:51:36 -07004349 for_each_pool(pool, pi) {
Tejun Heo24b8a842013-03-12 11:29:58 -07004350 spin_lock(&pool->lock);
4351 wake_up_worker(pool);
4352 spin_unlock(&pool->lock);
4353 }
4354
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004355 workqueue_freezing = false;
4356out_unlock:
Tejun Heoe98d5b12013-03-12 11:29:57 -07004357 spin_unlock_irq(&workqueue_lock);
Tejun Heoa0a1a5f2010-06-29 10:07:12 +02004358}
4359#endif /* CONFIG_FREEZER */
4360
Suresh Siddha6ee05782010-07-30 14:57:37 -07004361static int __init init_workqueues(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004362{
Tejun Heo7a4e3442013-03-12 11:30:00 -07004363 int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
4364 int i, cpu;
Tejun Heoc34056a2010-06-29 10:07:11 +02004365
Tejun Heo7c3eed52013-01-24 11:01:33 -08004366 /* make sure we have enough bits for OFFQ pool ID */
4367 BUILD_BUG_ON((1LU << (BITS_PER_LONG - WORK_OFFQ_POOL_SHIFT)) <
Lai Jiangshan6be19582013-02-06 18:04:53 -08004368 WORK_CPU_END * NR_STD_WORKER_POOLS);
Tejun Heob5490072012-08-03 10:30:46 -07004369
Tejun Heoe904e6c2013-03-12 11:29:57 -07004370 WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
4371
4372 pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
4373
Tejun Heo65758202012-07-17 12:39:26 -07004374 cpu_notifier(workqueue_cpu_up_callback, CPU_PRI_WORKQUEUE_UP);
Lai Jiangshana5b4e572012-09-18 09:59:23 -07004375 hotcpu_notifier(workqueue_cpu_down_callback, CPU_PRI_WORKQUEUE_DOWN);
Tejun Heo8b03ae32010-06-29 10:07:12 +02004376
Tejun Heo706026c2013-01-24 11:01:34 -08004377 /* initialize CPU pools */
Tejun Heo29c91e92013-03-12 11:30:03 -07004378 for_each_possible_cpu(cpu) {
Tejun Heo4ce62e92012-07-13 22:16:44 -07004379 struct worker_pool *pool;
Tejun Heo8b03ae32010-06-29 10:07:12 +02004380
Tejun Heo7a4e3442013-03-12 11:30:00 -07004381 i = 0;
Tejun Heof02ae732013-03-12 11:30:03 -07004382 for_each_cpu_worker_pool(pool, cpu) {
Tejun Heo7a4e3442013-03-12 11:30:00 -07004383 BUG_ON(init_worker_pool(pool));
Tejun Heoec22ca52013-01-24 11:01:33 -08004384 pool->cpu = cpu;
Tejun Heo29c91e92013-03-12 11:30:03 -07004385 cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
Tejun Heo7a4e3442013-03-12 11:30:00 -07004386 pool->attrs->nice = std_nice[i++];
4387
Tejun Heo9daf9e62013-01-24 11:01:33 -08004388 /* alloc pool ID */
4389 BUG_ON(worker_pool_assign_id(pool));
Tejun Heo4ce62e92012-07-13 22:16:44 -07004390 }
Tejun Heo8b03ae32010-06-29 10:07:12 +02004391 }
4392
Tejun Heoe22bee72010-06-29 10:07:14 +02004393 /* create the initial worker */
Tejun Heo29c91e92013-03-12 11:30:03 -07004394 for_each_online_cpu(cpu) {
Tejun Heo4ce62e92012-07-13 22:16:44 -07004395 struct worker_pool *pool;
Tejun Heoe22bee72010-06-29 10:07:14 +02004396
Tejun Heof02ae732013-03-12 11:30:03 -07004397 for_each_cpu_worker_pool(pool, cpu) {
Tejun Heo4ce62e92012-07-13 22:16:44 -07004398 struct worker *worker;
4399
Tejun Heo29c91e92013-03-12 11:30:03 -07004400 pool->flags &= ~POOL_DISASSOCIATED;
Tejun Heo24647572013-01-24 11:01:33 -08004401
Tejun Heobc2ae0f2012-07-17 12:39:27 -07004402 worker = create_worker(pool);
Tejun Heo4ce62e92012-07-13 22:16:44 -07004403 BUG_ON(!worker);
Tejun Heod565ed62013-01-24 11:01:33 -08004404 spin_lock_irq(&pool->lock);
Tejun Heo4ce62e92012-07-13 22:16:44 -07004405 start_worker(worker);
Tejun Heod565ed62013-01-24 11:01:33 -08004406 spin_unlock_irq(&pool->lock);
Tejun Heo4ce62e92012-07-13 22:16:44 -07004407 }
Tejun Heoe22bee72010-06-29 10:07:14 +02004408 }
4409
Tejun Heo29c91e92013-03-12 11:30:03 -07004410 /* create default unbound wq attrs */
4411 for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
4412 struct workqueue_attrs *attrs;
4413
4414 BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
4415
4416 attrs->nice = std_nice[i];
4417 cpumask_setall(attrs->cpumask);
4418
4419 unbound_std_wq_attrs[i] = attrs;
4420 }
4421
Tejun Heod320c032010-06-29 10:07:14 +02004422 system_wq = alloc_workqueue("events", 0, 0);
Joonsoo Kim1aabe902012-08-15 23:25:39 +09004423 system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
Tejun Heod320c032010-06-29 10:07:14 +02004424 system_long_wq = alloc_workqueue("events_long", 0, 0);
Tejun Heof3421792010-07-02 10:03:51 +02004425 system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
4426 WQ_UNBOUND_MAX_ACTIVE);
Tejun Heo24d51ad2011-02-21 09:52:50 +01004427 system_freezable_wq = alloc_workqueue("events_freezable",
4428 WQ_FREEZABLE, 0);
Joonsoo Kim1aabe902012-08-15 23:25:39 +09004429 BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||
Tejun Heoae930e02012-08-20 14:51:23 -07004430 !system_unbound_wq || !system_freezable_wq);
Suresh Siddha6ee05782010-07-30 14:57:37 -07004431 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004432}
Suresh Siddha6ee05782010-07-30 14:57:37 -07004433early_initcall(init_workqueues);