rcu: Permit call_rcu() from CPU_DYING notifiers
[linux-2.6.git] / kernel / padata.c
index b452599..89fe3d1 100644 (file)
@@ -1,6 +1,8 @@
 /*
  * padata.c - generic interface to process data streams in parallel
  *
+ * See Documentation/padata.txt for an api documentation.
+ *
  * Copyright (C) 2008, 2009 secunet Security Networks AG
  * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com>
  *
@@ -29,7 +31,6 @@
 #include <linux/sysfs.h>
 #include <linux/rcupdate.h>
 
-#define MAX_SEQ_NR (INT_MAX - NR_CPUS)
 #define MAX_OBJ_NUM 1000
 
 static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index)
@@ -43,18 +44,19 @@ static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index)
        return target_cpu;
 }
 
-static int padata_cpu_hash(struct padata_priv *padata)
+static int padata_cpu_hash(struct parallel_data *pd)
 {
        int cpu_index;
-       struct parallel_data *pd;
-
-       pd =  padata->pd;
 
        /*
         * Hash the sequence numbers to the cpus by taking
         * seq_nr mod. number of cpus in use.
         */
-       cpu_index =  padata->seq_nr % cpumask_weight(pd->cpumask.pcpu);
+
+       spin_lock(&pd->seq_lock);
+       cpu_index =  pd->seq_nr % cpumask_weight(pd->cpumask.pcpu);
+       pd->seq_nr++;
+       spin_unlock(&pd->seq_lock);
 
        return padata_index_to_cpu(pd, cpu_index);
 }
@@ -132,12 +134,7 @@ int padata_do_parallel(struct padata_instance *pinst,
        padata->pd = pd;
        padata->cb_cpu = cb_cpu;
 
-       if (unlikely(atomic_read(&pd->seq_nr) == pd->max_seq_nr))
-               atomic_set(&pd->seq_nr, -1);
-
-       padata->seq_nr = atomic_inc_return(&pd->seq_nr);
-
-       target_cpu = padata_cpu_hash(padata);
+       target_cpu = padata_cpu_hash(pd);
        queue = per_cpu_ptr(pd->pqueue, target_cpu);
 
        spin_lock(&queue->parallel.lock);
@@ -173,7 +170,7 @@ EXPORT_SYMBOL(padata_do_parallel);
 static struct padata_priv *padata_get_next(struct parallel_data *pd)
 {
        int cpu, num_cpus;
-       int next_nr, next_index;
+       unsigned int next_nr, next_index;
        struct padata_parallel_queue *queue, *next_queue;
        struct padata_priv *padata;
        struct padata_list *reorder;
@@ -189,14 +186,6 @@ static struct padata_priv *padata_get_next(struct parallel_data *pd)
        cpu = padata_index_to_cpu(pd, next_index);
        next_queue = per_cpu_ptr(pd->pqueue, cpu);
 
-       if (unlikely(next_nr > pd->max_seq_nr)) {
-               next_nr = next_nr - pd->max_seq_nr - 1;
-               next_index = next_nr % num_cpus;
-               cpu = padata_index_to_cpu(pd, next_index);
-               next_queue = per_cpu_ptr(pd->pqueue, cpu);
-               pd->processed = 0;
-       }
-
        padata = NULL;
 
        reorder = &next_queue->reorder;
@@ -205,8 +194,6 @@ static struct padata_priv *padata_get_next(struct parallel_data *pd)
                padata = list_entry(reorder->list.next,
                                    struct padata_priv, list);
 
-               BUG_ON(next_nr != padata->seq_nr);
-
                spin_lock(&reorder->lock);
                list_del_init(&padata->list);
                atomic_dec(&pd->reorder_objects);
@@ -230,6 +217,7 @@ out:
 
 static void padata_reorder(struct parallel_data *pd)
 {
+       int cb_cpu;
        struct padata_priv *padata;
        struct padata_serial_queue *squeue;
        struct padata_instance *pinst = pd->pinst;
@@ -270,13 +258,14 @@ static void padata_reorder(struct parallel_data *pd)
                        return;
                }
 
-               squeue = per_cpu_ptr(pd->squeue, padata->cb_cpu);
+               cb_cpu = padata->cb_cpu;
+               squeue = per_cpu_ptr(pd->squeue, cb_cpu);
 
                spin_lock(&squeue->serial.lock);
                list_add_tail(&padata->list, &squeue->serial.list);
                spin_unlock(&squeue->serial.lock);
 
-               queue_work_on(padata->cb_cpu, pinst->wq, &squeue->work);
+               queue_work_on(cb_cpu, pinst->wq, &squeue->work);
        }
 
        spin_unlock_bh(&pd->lock);
@@ -367,13 +356,13 @@ static int padata_setup_cpumasks(struct parallel_data *pd,
        if (!alloc_cpumask_var(&pd->cpumask.pcpu, GFP_KERNEL))
                return -ENOMEM;
 
-       cpumask_and(pd->cpumask.pcpu, pcpumask, cpu_active_mask);
+       cpumask_and(pd->cpumask.pcpu, pcpumask, cpu_online_mask);
        if (!alloc_cpumask_var(&pd->cpumask.cbcpu, GFP_KERNEL)) {
                free_cpumask_var(pd->cpumask.cbcpu);
                return -ENOMEM;
        }
 
-       cpumask_and(pd->cpumask.cbcpu, cbcpumask, cpu_active_mask);
+       cpumask_and(pd->cpumask.cbcpu, cbcpumask, cpu_online_mask);
        return 0;
 }
 
@@ -400,7 +389,7 @@ static void padata_init_squeues(struct parallel_data *pd)
 /* Initialize all percpu queues used by parallel workers */
 static void padata_init_pqueues(struct parallel_data *pd)
 {
-       int cpu_index, num_cpus, cpu;
+       int cpu_index, cpu;
        struct padata_parallel_queue *pqueue;
 
        cpu_index = 0;
@@ -415,9 +404,6 @@ static void padata_init_pqueues(struct parallel_data *pd)
                INIT_WORK(&pqueue->work, padata_parallel_worker);
                atomic_set(&pqueue->num_obj, 0);
        }
-
-       num_cpus = cpumask_weight(pd->cpumask.pcpu);
-       pd->max_seq_nr = num_cpus ? (MAX_SEQ_NR / num_cpus) * num_cpus - 1 : 0;
 }
 
 /* Allocate and initialize the internal cpumask dependend resources. */
@@ -444,7 +430,7 @@ static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst,
        padata_init_pqueues(pd);
        padata_init_squeues(pd);
        setup_timer(&pd->timer, padata_reorder_timer, (unsigned long)pd);
-       atomic_set(&pd->seq_nr, -1);
+       pd->seq_nr = 0;
        atomic_set(&pd->reorder_objects, 0);
        atomic_set(&pd->refcnt, 0);
        pd->pinst = pinst;
@@ -580,7 +566,7 @@ EXPORT_SYMBOL(padata_unregister_cpumask_notifier);
 static bool padata_validate_cpumask(struct padata_instance *pinst,
                                    const struct cpumask *cpumask)
 {
-       if (!cpumask_intersects(cpumask, cpu_active_mask)) {
+       if (!cpumask_intersects(cpumask, cpu_online_mask)) {
                pinst->flags |= PADATA_INVALID;
                return false;
        }
@@ -694,7 +680,7 @@ static int __padata_add_cpu(struct padata_instance *pinst, int cpu)
 {
        struct parallel_data *pd;
 
-       if (cpumask_test_cpu(cpu, cpu_active_mask)) {
+       if (cpumask_test_cpu(cpu, cpu_online_mask)) {
                pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu,
                                     pinst->cpumask.cbcpu);
                if (!pd)
@@ -762,6 +748,9 @@ static int __padata_remove_cpu(struct padata_instance *pinst, int cpu)
                        return -ENOMEM;
 
                padata_replace(pinst, pd);
+
+               cpumask_clear_cpu(cpu, pd->cpumask.cbcpu);
+               cpumask_clear_cpu(cpu, pd->cpumask.pcpu);
        }
 
        return 0;