index e6c2517..82095bf 100644 (file)
@@ -47,9 +47,7 @@ static int convert_prio(int prio)
}

#define for_each_cpupri_active(array, idx)                    \
-  for (idx = find_first_bit(array, CPUPRI_NR_PRIORITIES);     \
-       idx < CPUPRI_NR_PRIORITIES;                            \
-       idx = find_next_bit(array, CPUPRI_NR_PRIORITIES, idx+1))
+       for_each_set_bit(idx, array, CPUPRI_NR_PRIORITIES)

/**
* cpupri_find - find the best (lowest-pri) CPU in the system
@@ -81,8 +79,21 @@ int cpupri_find(struct cpupri *cp, struct task_struct *p,
continue;

+
+                       /*
+                        * We have to ensure that we have at least one bit
+                        * still set in the array, since the map could have
+                        * been concurrently emptied between the first and
+                        * condition, simply act as though we never hit this
+                        * priority level and continue on.
+                        */
+                               continue;
+               }
+
return 1;
}

@@ -114,32 +125,34 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)

/*
* If the cpu was currently mapped to a different value, we
-        * first need to unmap the old value
+        * need to map it to the new value then remove the old value.
+        * Note, we must add the new value first, otherwise we risk the
+        * cpu being cleared from pri_active, and this cpu could be
+        * missed for a push or pull.
*/
-       if (likely(oldpri != CPUPRI_INVALID)) {
-               struct cpupri_vec *vec  = &cp->pri_to_cpu[oldpri];
-
-               spin_lock_irqsave(&vec->lock, flags);
-
-               vec->count--;
-               if (!vec->count)
-                       clear_bit(oldpri, cp->pri_active);
-
-               spin_unlock_irqrestore(&vec->lock, flags);
-       }
-
if (likely(newpri != CPUPRI_INVALID)) {
struct cpupri_vec *vec = &cp->pri_to_cpu[newpri];

-               spin_lock_irqsave(&vec->lock, flags);
+               raw_spin_lock_irqsave(&vec->lock, flags);

vec->count++;
if (vec->count == 1)
set_bit(newpri, cp->pri_active);

-               spin_unlock_irqrestore(&vec->lock, flags);
+               raw_spin_unlock_irqrestore(&vec->lock, flags);
+       }
+       if (likely(oldpri != CPUPRI_INVALID)) {
+               struct cpupri_vec *vec  = &cp->pri_to_cpu[oldpri];
+
+               raw_spin_lock_irqsave(&vec->lock, flags);
+
+               vec->count--;
+               if (!vec->count)
+                       clear_bit(oldpri, cp->pri_active);
+
+               raw_spin_unlock_irqrestore(&vec->lock, flags);
}

*currpri = newpri;
@@ -165,7 +178,7 @@ int cpupri_init(struct cpupri *cp, bool bootmem)
for (i = 0; i < CPUPRI_NR_PRIORITIES; i++) {
struct cpupri_vec *vec = &cp->pri_to_cpu[i];

-               spin_lock_init(&vec->lock);
+               raw_spin_lock_init(&vec->lock);
vec->count = 0;